@reconcrap/people-network-memory 0.1.0

package/src/people_network_memory/fixtures/eval.py

@@ -0,0 +1,398 @@
+"""Fixture retrieval evaluation helpers."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from people_network_memory.application.services import RecordInteractionService, RetrieveContextService
+from people_network_memory.domain.models import RetrievalItem, SensitivityLabel
+from people_network_memory.fixtures.generator import MockDataset
+from people_network_memory.infrastructure.id_generator import SequentialIdGenerator
+from people_network_memory.infrastructure.in_memory_store import InMemoryPeopleStore
+from people_network_memory.infrastructure.retrieval_intent import text_answers_mentioned_query
+
+
+@dataclass(frozen=True)
+class EvalResult:
+    checked: int
+    top3_hits: int
+    top5_hits: int
+    evidence_complete: int
+    ingested_interactions: int = 0
+    returned_results: int = 0
+    evidence_results: int = 0
+    sensitive_leaks: int = 0
+    no_result_count: int = 0
+    candidate_queries: int = 0
+    skipped_unanswerable_queries: int = 0
+    cases: list[dict[str, object]] = field(default_factory=list)
+
+    @property
+    def recall_at_3(self) -> float:
+        return self.top3_hits / self.checked if self.checked else 0.0
+
+    @property
+    def recall_at_5(self) -> float:
+        return self.top5_hits / self.checked if self.checked else 0.0
+
+    @property
+    def evidence_rate(self) -> float:
+        return self.evidence_complete / self.checked if self.checked else 0.0
+
+    @property
+    def returned_result_evidence_rate(self) -> float:
+        if not self.returned_results:
+            return 0.0
+        return self.evidence_results / self.returned_results
+
+    def to_json(
+        self,
+        *,
+        include_cases: bool = True,
+        failures_only: bool = False,
+    ) -> dict[str, object]:
+        payload: dict[str, object] = {
+            "checked": self.checked,
+            "candidate_queries": self.candidate_queries,
+            "skipped_unanswerable_queries": self.skipped_unanswerable_queries,
+            "ingested_interactions": self.ingested_interactions,
+            "top3_hits": self.top3_hits,
+            "top5_hits": self.top5_hits,
+            "recall_at_3": round(self.recall_at_3, 4),
+            "recall_at_5": round(self.recall_at_5, 4),
+            "evidence_rate": round(self.evidence_rate, 4),
+            "returned_results": self.returned_results,
+            "evidence_results": self.evidence_results,
+            "returned_result_evidence_rate": round(self.returned_result_evidence_rate, 4),
+            "semantic_result_count": self._semantic_result_count(),
+            "sensitive_leaks": self.sensitive_leaks,
+            "no_result_count": self.no_result_count,
+            "category_breakdown": self._category_breakdown(),
+            "passes_v1_thresholds": self.recall_at_3 >= 0.70
+            and self.recall_at_5 >= 0.85
+            and self.returned_result_evidence_rate == 1.0
+            and self.sensitive_leaks == 0,
+        }
+        if include_cases:
+            payload["cases"] = [
+                case for case in self.cases if not failures_only or _case_failed(case)
+            ]
+            payload["case_filter"] = "failures_only" if failures_only else "all"
+        return payload
+
+    def _category_breakdown(self) -> dict[str, dict[str, object]]:
+        buckets: dict[str, dict[str, int]] = {}
+        for case in self.cases:
+            category = str(case.get("category", "unknown"))
+            bucket = buckets.setdefault(
+                category,
+                {
+                    "checked": 0,
+                    "top3_hits": 0,
+                    "top5_hits": 0,
+                    "evidence_complete": 0,
+                    "sensitive_leaks": 0,
+                    "no_result_count": 0,
+                },
+            )
+            bucket["checked"] += 1
+            bucket["top3_hits"] += 1 if case.get("hit_top3") else 0
+            bucket["top5_hits"] += 1 if case.get("hit_top5") else 0
+            bucket["evidence_complete"] += 1 if case.get("evidence_complete") else 0
+            bucket["sensitive_leaks"] += int(case.get("sensitive_leak_count", 0))
+            bucket["no_result_count"] += 1 if int(case.get("result_count", 0)) == 0 else 0
+        return {
+            category: {
+                **bucket,
+                "recall_at_3": round(bucket["top3_hits"] / bucket["checked"], 4),
+                "recall_at_5": round(bucket["top5_hits"] / bucket["checked"], 4),
+                "evidence_rate": round(bucket["evidence_complete"] / bucket["checked"], 4),
+            }
+            for category, bucket in sorted(buckets.items())
+        }
+
+    def _semantic_result_count(self) -> int:
+        count = 0
+        for case in self.cases:
+            for result in case.get("actual_results", []):
+                if isinstance(result, dict) and result.get("title") == "Semantic interaction match":
+                    count += 1
+        return count
+
+
+def evaluate_dataset(
+    dataset: MockDataset,
+    *,
+    max_interactions: int | None = None,
+    max_queries: int | None = None,
+    only_answerable: bool = True,
+) -> EvalResult:
+    store = InMemoryPeopleStore()
+    record = RecordInteractionService(
+        memory_store=store,
+        identity_index=store,
+        review_queue=store,
+        id_generator=SequentialIdGenerator(),
+    )
+    retrieve = RetrieveContextService(graph_search=store, review_queue=store)
+    return evaluate_services(
+        dataset,
+        record_service=record,
+        retrieve_service=retrieve,
+        max_interactions=max_interactions,
+        max_queries=max_queries,
+        only_answerable=only_answerable,
+    )
+
+
+def evaluate_services(
+    dataset: MockDataset,
+    *,
+    record_service: RecordInteractionService,
+    retrieve_service: RetrieveContextService,
+    max_interactions: int | None = None,
+    max_queries: int | None = None,
+    only_answerable: bool = True,
+) -> EvalResult:
+    interactions = dataset.interactions[:max_interactions] if max_interactions else dataset.interactions
+    for interaction in interactions:
+        record_service.record(interaction)
+    return evaluate_retrieval_service(
+        dataset,
+        retrieve_service=retrieve_service,
+        indexed_interactions=len(interactions),
+        max_queries=max_queries,
+        only_answerable=only_answerable,
+    )
+
+
+def evaluate_retrieval_service(
+    dataset: MockDataset,
+    *,
+    retrieve_service: RetrieveContextService,
+    indexed_interactions: int | None = None,
+    max_queries: int | None = None,
+    only_answerable: bool = True,
+) -> EvalResult:
+    indexed_count = indexed_interactions if indexed_interactions is not None else len(dataset.interactions)
+    candidate_queries = dataset.eval_queries[:max_queries] if max_queries else dataset.eval_queries
+    skipped_unanswerable_queries = 0
+    if indexed_interactions is not None and only_answerable:
+        queries = [
+            query
+            for query in candidate_queries
+            if query.source_interaction_index is None
+            or query.source_interaction_index < indexed_count
+        ]
+        skipped_unanswerable_queries = len(candidate_queries) - len(queries)
+    else:
+        queries = candidate_queries
+    top3_hits = 0
+    top5_hits = 0
+    evidence_complete = 0
+    returned_results = 0
+    evidence_results = 0
+    sensitive_leaks = 0
+    no_result_count = 0
+    cases: list[dict[str, object]] = []
+    for query_index, query in enumerate(queries, start=1):
+        response = retrieve_service.retrieve(
+            query.query,
+            limit=5,
+            sensitivity_policy="strict",
+            output_context="private",
+        )
+        returned_results += len(response.results)
+        evidence_results += sum(1 for item in response.results if item.evidence)
+        leak_count = sum(1 for item in response.results if _has_sensitive_leak(item.sensitivity))
+        sensitive_leaks += leak_count
+        if not response.results:
+            no_result_count += 1
+        evidence_ok = bool(response.results) and all(item.evidence for item in response.results)
+        if evidence_ok:
+            evidence_complete += 1
+        expected = query.expected_people + query.expected_terms
+        strict_top5 = _first_strict_match_rank(
+            category=query.category,
+            results=response.results,
+            expected_people=query.expected_people,
+            expected_terms=query.expected_terms,
+            max_rank=5,
+        )
+        strict_top3 = _first_strict_match_rank(
+            category=query.category,
+            results=response.results,
+            expected_people=query.expected_people,
+            expected_terms=query.expected_terms,
+            max_rank=3,
+        )
+        joined_top5 = " ".join(item.matched_text + " " + item.title for item in response.results)
+        matched_top5 = [item for item in expected if _contains(joined_top5, item)]
+        hit_top5 = strict_top5 is not None
+        hit_top3 = strict_top3 is not None
+        if hit_top5:
+            top5_hits += 1
+        if hit_top3:
+            top3_hits += 1
+        cases.append(
+            {
+                "query": query.query,
+                "query_index": query_index,
+                "category": query.category,
+                "source_interaction_index": query.source_interaction_index,
+                "expected": expected,
+                "expected_people": query.expected_people,
+                "expected_terms": query.expected_terms,
+                "matched_expected": matched_top5,
+                "missed_expected": [item for item in expected if item not in matched_top5],
+                "hit_top3": hit_top3,
+                "hit_top5": hit_top5,
+                "strict_match_rank": strict_top5,
+                "expected_person_rank": _first_expected_person_rank(
+                    response.results,
+                    query.expected_people,
+                    max_rank=5,
+                ),
+                "result_count": len(response.results),
+                "evidence_complete": evidence_ok,
+                "sensitive_leak_count": leak_count,
+                "top_matches": [
+                    {
+                        "title": item.title,
+                        "matched_text": item.matched_text,
+                        "why_matched": item.why_matched,
+                        "score": round(item.score, 4),
+                        "person_ids": item.person_ids,
+                        "sensitivity": [label.value for label in item.sensitivity],
+                        "evidence_ids": [evidence.evidence_id for evidence in item.evidence],
+                    }
+                    for item in response.results[:3]
+                ],
+                "actual_results": [
+                    _serialize_actual_result(rank, item)
+                    for rank, item in enumerate(response.results, start=1)
+                ],
+            }
+        )
+    return EvalResult(
+        checked=len(queries),
+        ingested_interactions=indexed_count,
+        candidate_queries=len(candidate_queries),
+        skipped_unanswerable_queries=skipped_unanswerable_queries,
+        top3_hits=top3_hits,
+        top5_hits=top5_hits,
+        evidence_complete=evidence_complete,
+        returned_results=returned_results,
+        evidence_results=evidence_results,
+        sensitive_leaks=sensitive_leaks,
+        no_result_count=no_result_count,
+        cases=cases,
+    )
+
+
+def _has_sensitive_leak(labels: list[SensitivityLabel]) -> bool:
+    blocked = {
+        SensitivityLabel.SENSITIVE,
+        SensitivityLabel.DO_NOT_SURFACE_UNPROMPTED,
+    }
+    return any(label in blocked for label in labels)
+
+
+def _serialize_actual_result(rank: int, item: RetrievalItem) -> dict[str, object]:
+    return {
+        "rank": rank,
+        "item_id": item.item_id,
+        "kind": item.kind,
+        "title": item.title,
+        "matched_text": item.matched_text,
+        "why_matched": item.why_matched,
+        "score": round(item.score, 4),
+        "person_ids": item.person_ids,
+        "sensitivity": [label.value for label in item.sensitivity],
+        "is_secondhand": item.is_secondhand,
+        "evidence": [
+            {
+                "evidence_id": evidence.evidence_id,
+                "source_text": evidence.source_text,
+                "recorded_at": evidence.recorded_at.isoformat(),
+                "speaker_person_id": evidence.speaker_person_id,
+                "speaker_label": evidence.speaker_label,
+                "confidence": round(evidence.confidence, 4),
+            }
+            for evidence in item.evidence
+        ],
+    }
+
+
+def _first_strict_match_rank(
+    *,
+    category: str,
+    results: list[RetrievalItem],
+    expected_people: list[str],
+    expected_terms: list[str],
+    max_rank: int,
+) -> int | None:
+    for rank, item in enumerate(results[:max_rank], start=1):
+        if _result_satisfies_query(
+            category=category,
+            item=item,
+            expected_people=expected_people,
+            expected_terms=expected_terms,
+        ):
+            return rank
+    return None
+
+
+def _result_satisfies_query(
+    *,
+    category: str,
+    item: RetrievalItem,
+    expected_people: list[str],
+    expected_terms: list[str],
+) -> bool:
+    text = f"{item.title} {item.matched_text}"
+    if category == "mentioned":
+        target = expected_terms[0] if expected_terms else ""
+        return (
+            bool(expected_people)
+            and any(_contains(text, person) for person in expected_people)
+            and text_answers_mentioned_query(text, target)
+        )
+    if category == "follow_up":
+        return (
+            item.kind == "follow_up"
+            and all(_contains(text, person) for person in expected_people)
+            and all(_contains(text, term) for term in expected_terms)
+        )
+    if category == "profile":
+        return all(_contains(text, term) for term in expected_terms)
+    if category in {"vague", "bilingual"}:
+        return all(_contains(text, term) for term in expected_terms)
+    return all(_contains(text, item) for item in [*expected_people, *expected_terms])
+
+
+def _first_expected_person_rank(
+    results: list[RetrievalItem],
+    expected_people: list[str],
+    *,
+    max_rank: int,
+) -> int | None:
+    if not expected_people:
+        return None
+    for rank, item in enumerate(results[:max_rank], start=1):
+        text = f"{item.title} {item.matched_text}"
+        if any(_contains(text, person) for person in expected_people):
+            return rank
+    return None
+
+
+def _contains(text: str, expected: str) -> bool:
+    return expected.lower() in text.lower()
+
+
+def _case_failed(case: dict[str, object]) -> bool:
+    return (
+        not bool(case.get("hit_top5"))
+        or not bool(case.get("evidence_complete"))
+        or int(case.get("sensitive_leak_count", 0)) > 0
+    )