@reconcrap/people-network-memory 0.1.0

package/src/people_network_memory/fixtures/extractor_eval.py

@@ -0,0 +1,364 @@
+"""Evaluation fixtures for the LLM ingestion extractor."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import Protocol
+
+from people_network_memory.domain.models import SocialInteraction
+
+
+class ExtractorLike(Protocol):
+    def extract(self, interaction: SocialInteraction) -> SocialInteraction:
+        ...
+
+
+@dataclass(frozen=True)
+class ExtractorExpected:
+    participants: list[str] = field(default_factory=list)
+    mentioned_people: list[str] = field(default_factory=list)
+    aliases: dict[str, list[str]] = field(default_factory=dict)
+    follow_up_terms: list[list[str]] = field(default_factory=list)
+    forbidden_follow_up_terms: list[list[str]] = field(default_factory=list)
+    follow_up_due_dates: list[str] = field(default_factory=list)
+    attributed_claim_terms: list[list[str]] = field(default_factory=list)
+    direct_fact_terms: list[list[str]] = field(default_factory=list)
+
+
+@dataclass(frozen=True)
+class ExtractorEvalCase:
+    case_id: str
+    source_text: str
+    expected: ExtractorExpected
+    occurred_at: datetime | None = None
+
+
+EXTRACTOR_EVAL_CASES: list[ExtractorEvalCase] = [
+    ExtractorEvalCase(
+        case_id="zh_promise_explicit",
+        source_text=(
+            "今天在Manner Coffee见了测试赵小北，她在北辰智能负责人形机器人"
+            "感知算法招聘，喜欢微信联系。我答应下周三上午10点给她发三位"
+            "感知算法候选人。"
+        ),
+        occurred_at=datetime(2026, 5, 15, tzinfo=timezone.utc),
+        expected=ExtractorExpected(
+            participants=["测试赵小北"],
+            follow_up_terms=[["测试赵小北", "三位", "感知算法候选人"]],
+            follow_up_due_dates=["2026-05-20"],
+            direct_fact_terms=[["测试赵小北", "北辰智能"], ["测试赵小北", "微信"]],
+        ),
+    ),
+    ExtractorEvalCase(
+        case_id="zh_commitment_soft",
+        source_text=(
+            "在三顿半碰到测试林小雨，她在星河机器人做人形机器人算法招聘。"
+            "我回头把两个感知算法候选人发给她，最好下周前。"
+        ),
+        occurred_at=datetime(2026, 5, 15, tzinfo=timezone.utc),
+        expected=ExtractorExpected(
+            participants=["测试林小雨"],
+            follow_up_terms=[["两个", "感知算法候选人"]],
+            direct_fact_terms=[["测试林小雨", "星河机器人"]],
+        ),
+    ),
+    ExtractorEvalCase(
+        case_id="zh_asked_me",
+        source_text=(
+            "今天和测试陈一诺聊了云栖机器人的控制算法招聘，"
+            "她让我明天下午3点前发两个控制算法候选人。"
+        ),
+        occurred_at=datetime(2026, 5, 15, tzinfo=timezone.utc),
+        expected=ExtractorExpected(
+            participants=["测试陈一诺"],
+            follow_up_terms=[["两个", "控制算法候选人"]],
+            follow_up_due_dates=["2026-05-16"],
+        ),
+    ),
+    ExtractorEvalCase(
+        case_id="zh_alias",
+        source_text=(
+            "今天在潘家园见了测试王小明（阿明），聊了北京机器人项目。"
+            "他在摸金科技做AI硬件。"
+        ),
+        expected=ExtractorExpected(
+            participants=["测试王小明"],
+            aliases={"测试王小明": ["阿明"]},
+            direct_fact_terms=[["测试王小明", "摸金科技"], ["测试王小明", "AI硬件"]],
+        ),
+    ),
+    ExtractorEvalCase(
+        case_id="zh_contact_preference_not_follow_up",
+        source_text=(
+            "今天又见了测试王小明（阿明），他现在也在关注具身智能芯片供应链，"
+            "之后最好用邮件联系。"
+        ),
+        expected=ExtractorExpected(
+            participants=["测试王小明"],
+            aliases={"测试王小明": ["阿明"]},
+            forbidden_follow_up_terms=[["邮件联系"]],
+        ),
+    ),
+    ExtractorEvalCase(
+        case_id="zh_secondhand_claim",
+        source_text=(
+            "今天在% Arabica见了测试陈一诺，她说测试王凯可能要离开云栖机器人。"
+            "陈一诺自己在云栖机器人做控制算法招聘。"
+        ),
+        expected=ExtractorExpected(
+            participants=["测试陈一诺"],
+            mentioned_people=["测试王凯"],
+            attributed_claim_terms=[["测试陈一诺", "测试王凯", "可能要离开"]],
+            direct_fact_terms=[["测试陈一诺", "云栖机器人"]],
+        ),
+    ),
+    ExtractorEvalCase(
+        case_id="en_need_to_follow_up",
+        source_text=(
+            "Met Alice Zhang at Blue Bottle. We discussed robotics hiring. "
+            "Need to send her two founder intros next week."
+        ),
+        expected=ExtractorExpected(
+            participants=["Alice Zhang"],
+            follow_up_terms=[["Alice Zhang", "two founder intros"]],
+        ),
+    ),
+    ExtractorEvalCase(
+        case_id="en_asked_me",
+        source_text=(
+            "Had coffee with Clara Wu at % Arabica. Clara asked me to WhatsApp "
+            "her the AI hardware shortlist tomorrow."
+        ),
+        occurred_at=datetime(2026, 5, 15, tzinfo=timezone.utc),
+        expected=ExtractorExpected(
+            participants=["Clara Wu"],
+            follow_up_terms=[["WhatsApp", "AI hardware shortlist"]],
+            follow_up_due_dates=["2026-05-16"],
+        ),
+    ),
+    ExtractorEvalCase(
+        case_id="en_mentioned_not_present",
+        source_text=(
+            "Met Alice Zhang at Seesaw. Alice mentioned Bob Li for founder intros, "
+            "but Bob was not there."
+        ),
+        expected=ExtractorExpected(
+            participants=["Alice Zhang"],
+            mentioned_people=["Bob Li"],
+            attributed_claim_terms=[["Alice", "Bob Li", "founder intros"]],
+        ),
+    ),
+]
+
+
+def evaluate_extractor(
+    extractor: ExtractorLike,
+    *,
+    cases: list[ExtractorEvalCase] | None = None,
+    max_cases: int | None = None,
+) -> dict[str, object]:
+    selected = list(cases or EXTRACTOR_EVAL_CASES)
+    if max_cases is not None:
+        selected = selected[:max_cases]
+    case_payloads: list[dict[str, object]] = []
+    passed = 0
+    for case in selected:
+        interaction = SocialInteraction(
+            source_text=case.source_text,
+            occurred_at=case.occurred_at,
+        )
+        actual = extractor.extract(interaction)
+        checks = _case_checks(case.expected, actual)
+        ok = all(item["ok"] for item in checks)
+        passed += int(ok)
+        case_payloads.append(
+            {
+                "case_id": case.case_id,
+                "ok": ok,
+                "source_text": case.source_text,
+                "expected": _expected_payload(case.expected),
+                "actual": _actual_payload(actual),
+                "checks": checks,
+            }
+        )
+    checked = len(selected)
+    return {
+        "ok": passed == checked,
+        "checked": checked,
+        "passed": passed,
+        "failed": checked - passed,
+        "pass_rate": (passed / checked) if checked else 0.0,
+        "cases": case_payloads,
+    }
+
+
+def _case_checks(
+    expected: ExtractorExpected, actual: SocialInteraction
+) -> list[dict[str, object]]:
+    checks: list[dict[str, object]] = []
+    participant_labels = [item.person.label for item in actual.participants]
+    mentioned_labels = [item.person.label for item in actual.mentioned_people]
+    follow_ups = [
+        " ".join(
+            [
+                item.description,
+                *[person.label for person in item.related_people],
+            ]
+        )
+        for item in actual.follow_ups
+    ]
+    due_dates = [item.due_at.isoformat() for item in actual.follow_ups if item.due_at]
+    claims = [
+        " ".join(
+            part
+            for part in [
+                item.speaker.label if item.speaker else "",
+                item.subject.label if item.subject else "",
+                item.claim_text,
+            ]
+            if part
+        )
+        for item in actual.attributed_claims
+    ]
+    mention_claim_equivalents = [
+        " ".join(
+            part
+            for part in [
+                item.mentioned_by.label if item.mentioned_by else "",
+                item.person.label,
+                item.context or "",
+            ]
+            if part
+        )
+        for item in actual.mentioned_people
+    ]
+    facts = [
+        " ".join([item.subject.label, item.predicate, item.value, *map(str, item.metadata.values())])
+        for item in actual.direct_facts
+    ]
+    for label in expected.participants:
+        checks.append(
+            _check(
+                "participant",
+                label,
+                _contains_label(participant_labels, label),
+                participant_labels,
+            )
+        )
+    for label in expected.mentioned_people:
+        checks.append(
+            _check(
+                "mentioned_person",
+                label,
+                _contains_label(mentioned_labels, label),
+                mentioned_labels,
+            )
+        )
+    for label, aliases in expected.aliases.items():
+        actual_aliases = {
+            item.person.label: item.person.aliases for item in actual.participants
+        }
+        for alias in aliases:
+            checks.append(
+                _check(
+                    "alias",
+                    f"{label} -> {alias}",
+                    alias in actual_aliases.get(label, []),
+                    actual_aliases,
+                )
+            )
+    for terms in expected.follow_up_terms:
+        checks.append(
+            _check(
+                "follow_up_terms",
+                terms,
+                _any_text_has_terms(follow_ups, terms),
+                follow_ups,
+            )
+        )
+    for terms in expected.forbidden_follow_up_terms:
+        checks.append(
+            _check(
+                "forbidden_follow_up_terms",
+                terms,
+                not _any_text_has_terms(follow_ups, terms),
+                follow_ups,
+            )
+        )
+    for due_date in expected.follow_up_due_dates:
+        checks.append(
+            _check("follow_up_due_at", due_date, due_date in due_dates, due_dates)
+        )
+    for terms in expected.attributed_claim_terms:
+        checks.append(
+            _check(
+                "attributed_claim_terms",
+                terms,
+                _any_text_has_terms(claims + mention_claim_equivalents, terms),
+                claims + mention_claim_equivalents,
+            )
+        )
+    for terms in expected.direct_fact_terms:
+        checks.append(
+            _check(
+                "direct_fact_terms",
+                terms,
+                _any_text_has_terms(facts, terms),
+                facts,
+            )
+        )
+    return checks
+
+
+def _contains_label(labels: list[str], expected: str) -> bool:
+    expected_folded = expected.casefold()
+    return any(
+        expected_folded == label.casefold() or expected_folded in label.casefold()
+        for label in labels
+    )
+
+
+def _any_text_has_terms(texts: list[str], terms: list[str]) -> bool:
+    return any(
+        all(term.casefold() in text.casefold() for term in terms)
+        for text in texts
+    )
+
+
+def _check(kind: str, expected: object, ok: bool, actual: object) -> dict[str, object]:
+    return {"kind": kind, "expected": expected, "ok": ok, "actual": actual}
+
+
+def _expected_payload(expected: ExtractorExpected) -> dict[str, object]:
+    return {
+        "participants": expected.participants,
+        "mentioned_people": expected.mentioned_people,
+        "aliases": expected.aliases,
+        "follow_up_terms": expected.follow_up_terms,
+        "forbidden_follow_up_terms": expected.forbidden_follow_up_terms,
+        "follow_up_due_dates": expected.follow_up_due_dates,
+        "attributed_claim_terms": expected.attributed_claim_terms,
+        "direct_fact_terms": expected.direct_fact_terms,
+    }
+
+
+def _actual_payload(actual: SocialInteraction) -> dict[str, object]:
+    return {
+        "place": actual.place,
+        "participants": [
+            item.person.model_dump(mode="json") for item in actual.participants
+        ],
+        "mentioned_people": [
+            item.model_dump(mode="json") for item in actual.mentioned_people
+        ],
+        "topics": actual.topics,
+        "direct_facts": [item.model_dump(mode="json") for item in actual.direct_facts],
+        "attributed_claims": [
+            item.model_dump(mode="json") for item in actual.attributed_claims
+        ],
+        "follow_ups": [item.model_dump(mode="json") for item in actual.follow_ups],
+        "relationships": [
+            item.model_dump(mode="json") for item in actual.relationships
+        ],
+    }

package/src/people_network_memory/fixtures/generator.py

@@ -0,0 +1,290 @@
+"""Deterministic social-network fixture generator."""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from random import Random
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from people_network_memory.domain.models import (
+    AttributedClaim,
+    DirectFact,
+    FollowUpTask,
+    MentionedPerson,
+    Participant,
+    PersonRef,
+    RelationshipAssertion,
+    SensitivityLabel,
+    SocialInteraction,
+)
+
+
+class FixturePerson(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    person_id: str
+    name: str
+    company: str
+    previous_companies: list[str]
+    schools: list[str]
+    interests: list[str]
+
+
+class EvalQuery(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    query: str
+    expected_people: list[str] = Field(default_factory=list)
+    expected_terms: list[str] = Field(default_factory=list)
+    category: Literal["mentioned", "vague", "bilingual", "follow_up", "profile"]
+    source_interaction_index: int | None = None
+
+
+class MockDataset(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    people: list[FixturePerson]
+    organizations: list[str]
+    schools: list[str]
+    places: list[str]
+    interactions: list[SocialInteraction]
+    eval_queries: list[EvalQuery]
+
+    def summary(self) -> dict[str, int]:
+        return {
+            "people": len(self.people),
+            "organizations": len(self.organizations),
+            "schools": len(self.schools),
+            "places": len(self.places),
+            "interactions": len(self.interactions),
+            "eval_queries": len(self.eval_queries),
+        }
+
+
+FIRST_NAMES = [
+    "Alice",
+    "Bob",
+    "Chen",
+    "Diana",
+    "Evan",
+    "Fiona",
+    "Grace",
+    "Henry",
+    "Iris",
+    "Jason",
+    "Kelly",
+    "Leo",
+    "Mina",
+    "Nora",
+    "Oscar",
+    "Priya",
+    "Qian",
+    "Rita",
+    "Sam",
+    "Tina",
+]
+LAST_NAMES = ["Zhang", "Li", "Wang", "Chen", "Liu", "Guo", "Tan", "Kim"]
+ORGS = [
+    "Tencent Robotics",
+    "ByteDance AI Lab",
+    "Alibaba Cloud",
+    "Meituan",
+    "SenseTime",
+    "DJI",
+    "Ant Group",
+    "Xpeng Robotics",
+    "MiniMax",
+    "Moonshot AI",
+]
+SCHOOLS = ["Tsinghua", "Peking University", "Zhejiang University", "SJTU", "Fudan", "CMU"]
+PLACES = [
+    "Blue Bottle Coffee",
+    "Xintiandi",
+    "Z Park",
+    "People Square",
+    "Sanlitun",
+    "West Lake",
+    "张江咖啡馆",
+    "深圳湾创业广场",
+    "The Bund",
+    "Nanshan Library",
+]
+INTERESTS = [
+    "robotics hiring",
+    "embodied AI",
+    "fundraising",
+    "jazz",
+    "trail running",
+    "Japanese food",
+    "robot learning",
+    "founder intros",
+    "product strategy",
+    "coffee",
+]
+
+
+def generate_mock_dataset(seed: int = 42) -> MockDataset:
+    rng = Random(seed)
+    people: list[FixturePerson] = []
+    for index in range(42):
+        first = FIRST_NAMES[index % len(FIRST_NAMES)]
+        last = LAST_NAMES[(index * 3) % len(LAST_NAMES)]
+        if index in {8, 19, 31}:
+            first, last = "Bob", "Li"
+        company = ORGS[index % len(ORGS)]
+        previous = [ORGS[(index + 2) % len(ORGS)], ORGS[(index + 5) % len(ORGS)]]
+        schools = [SCHOOLS[index % len(SCHOOLS)], SCHOOLS[(index + 2) % len(SCHOOLS)]]
+        interests = [INTERESTS[index % len(INTERESTS)], INTERESTS[(index + 3) % len(INTERESTS)]]
+        people.append(
+            FixturePerson(
+                person_id=f"fixture_person_{index:03d}",
+                name=f"{first} {last}",
+                company=company,
+                previous_companies=previous,
+                schools=schools,
+                interests=interests,
+            )
+        )
+
+    interactions: list[SocialInteraction] = []
+    for index in range(90):
+        person = people[index % len(people)]
+        other = people[(index + 7) % len(people)]
+        place = PLACES[index % len(PLACES)]
+        topic = INTERESTS[index % len(INTERESTS)]
+        source_text = (
+            f"Met {person.name} at {place}. We discussed {topic}. "
+            f"{person.name} mentioned {other.name} may be useful for {INTERESTS[(index + 2) % len(INTERESTS)]}."
+        )
+        if index % 11 == 0:
+            source_text = (
+                f"在{place}见了{person.name}，聊了{topic}，"
+                f"{person.name}提到{other.name}正在关注{INTERESTS[(index + 2) % len(INTERESTS)]}。"
+            )
+        interaction = SocialInteraction(
+            source_text=source_text,
+            occurred_at=datetime(2026, 5, (index % 25) + 1, tzinfo=timezone.utc),
+            interaction_type="coffee" if "Coffee" in place or "咖啡" in place else "meeting",
+            place=place,
+            participants=[Participant(person=_ref(person))],
+            mentioned_people=[
+                MentionedPerson(
+                    person=_ref(other),
+                    mentioned_by=_ref(person),
+                    context=INTERESTS[(index + 2) % len(INTERESTS)],
+                )
+            ],
+            topics=[topic],
+            direct_facts=[
+                DirectFact(subject=_ref(person), predicate="interest", value=topic),
+                DirectFact(subject=_ref(person), predicate="works_at", value=person.company),
+            ],
+            attributed_claims=[
+                AttributedClaim(
+                    speaker=_ref(person),
+                    subject=_ref(other),
+                    claim_text=(
+                        f"{person.name} said {other.name} may be useful for "
+                        f"{INTERESTS[(index + 2) % len(INTERESTS)]}."
+                    ),
+                )
+            ],
+            follow_ups=[
+                FollowUpTask(
+                    description=f"Follow up with {person.name} about {topic}",
+                    related_people=[_ref(person)],
+                )
+            ]
+            if index % 5 == 0
+            else [],
+            relationships=[
+                RelationshipAssertion(
+                    source=_ref(person),
+                    target=_ref(other),
+                    relationship_type="knows_person",
+                    directed=False,
+                )
+            ],
+            sensitivity=[SensitivityLabel.DO_NOT_SURFACE_UNPROMPTED]
+            if index % 17 == 0
+            else [],
+            metadata={"fixture_index": index, "random_check": rng.randint(1, 1000)},
+        )
+        interactions.append(interaction)
+
+    eval_queries: list[EvalQuery] = []
+    for index in range(10):
+        person = people[index]
+        other = people[(index + 7) % len(people)]
+        eval_queries.append(
+            EvalQuery(
+                query=f"who mentioned {other.name}",
+                expected_people=[person.name],
+                expected_terms=[other.name],
+                category="mentioned",
+                source_interaction_index=index,
+            )
+        )
+    for index in range(10, 20):
+        person = people[index]
+        eval_queries.append(
+            EvalQuery(
+                query=f"person from {PLACES[index % len(PLACES)]} talking about {INTERESTS[index % len(INTERESTS)]}",
+                expected_people=[person.name],
+                expected_terms=[PLACES[index % len(PLACES)], INTERESTS[index % len(INTERESTS)]],
+                category="vague",
+                source_interaction_index=index,
+            )
+        )
+    for index in range(20, 25):
+        person = people[index]
+        eval_queries.append(
+            EvalQuery(
+                query=f"在{PLACES[index % len(PLACES)]}聊{INTERESTS[index % len(INTERESTS)]}的人",
+                expected_people=[person.name],
+                expected_terms=[INTERESTS[index % len(INTERESTS)]],
+                category="bilingual",
+                source_interaction_index=index,
+            )
+        )
+    for index in range(25, 75, 5):
+        person = people[index % len(people)]
+        topic = INTERESTS[index % len(INTERESTS)]
+        eval_queries.append(
+            EvalQuery(
+                query=f"what did I promise {person.name}",
+                expected_people=[person.name],
+                expected_terms=[f"Follow up with {person.name} about {topic}"],
+                category="follow_up",
+                source_interaction_index=index,
+            )
+        )
+    for index in range(35, 42):
+        person = people[index % len(people)]
+        eval_queries.append(
+            EvalQuery(
+                query=f"{person.company} {person.interests[0]}",
+                expected_people=[person.name],
+                expected_terms=[person.company, person.interests[0]],
+                category="profile",
+                source_interaction_index=index,
+            )
+        )
+    return MockDataset(
+        people=people,
+        organizations=ORGS,
+        schools=SCHOOLS,
+        places=PLACES,
+        interactions=interactions,
+        eval_queries=eval_queries,
+    )
+
+
+def _ref(person: FixturePerson) -> PersonRef:
+    return PersonRef(
+        label=person.name,
+        person_id=person.person_id,
+        company_hint=person.company,
+    )