@reconcrap/people-network-memory 0.1.0

package/src/people_network_memory/fixtures/report.py

@@ -0,0 +1,252 @@
+"""Human-readable eval report rendering."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+
+def render_eval_markdown_report(
+    payload: dict[str, Any],
+    *,
+    title: str = "Personal Network Memory Retrieval Test Report",
+    compare_payload: dict[str, Any] | None = None,
+) -> str:
+    comparison = _cases_by_index(compare_payload) if compare_payload else {}
+    lines: list[str] = [
+        f"# {title}",
+        "",
+        "## Summary",
+        "",
+        f"- Checked queries: {payload.get('checked')}",
+        f"- Recall@3: {payload.get('recall_at_3')}",
+        f"- Recall@5: {payload.get('recall_at_5')}",
+        f"- Evidence rate: {payload.get('evidence_rate')}",
+        f"- Returned-result evidence rate: {payload.get('returned_result_evidence_rate')}",
+        f"- Sensitive leaks: {payload.get('sensitive_leaks')}",
+        f"- Semantic result count: {payload.get('semantic_result_count')}",
+        f"- Passes V1 thresholds: {payload.get('passes_v1_thresholds')}",
+        *_rank_summary_lines(payload),
+        "",
+    ]
+    if compare_payload:
+        lines.extend(
+            [
+                "## Comparison Baseline",
+                "",
+                f"- Baseline checked queries: {compare_payload.get('checked')}",
+                f"- Baseline Recall@3: {compare_payload.get('recall_at_3')}",
+                f"- Baseline Recall@5: {compare_payload.get('recall_at_5')}",
+                f"- Baseline passes V1 thresholds: {compare_payload.get('passes_v1_thresholds')}",
+                *_rank_summary_lines(compare_payload, prefix="Baseline "),
+                f"- Top result changed by judged retrieval: {_changed_top_count(payload, compare_payload)}",
+                "",
+            ]
+        )
+
+    for case in payload.get("cases", []):
+        if not isinstance(case, dict):
+            continue
+        index = case.get("query_index")
+        baseline = comparison.get(index)
+        status = "PASS" if case.get("hit_top5") else "FAIL"
+        lines.extend(
+            [
+                f"## Case {index}: {case.get('category')} - {status}",
+                "",
+                f"Query: `{case.get('query')}`",
+                "",
+                "Expected Results:",
+                _expected_line(case),
+                "",
+            ]
+        )
+        if baseline:
+            lines.extend(
+                [
+                    "Deterministic Actual Results:",
+                    *_actual_lines(baseline.get("actual_results", [])),
+                    "",
+                    "LLM-Judged Actual Results:",
+                    *_actual_lines(case.get("actual_results", [])),
+                    "",
+                ]
+            )
+        else:
+            lines.extend(
+                [
+                    "Actual Results:",
+                    *_actual_lines(case.get("actual_results", [])),
+                    "",
+                ]
+            )
+        lines.extend(
+            [
+                "Evaluation Reasoning:",
+                _reasoning_line(case),
+                "",
+            ]
+        )
+        if baseline:
+            lines.extend(
+                [
+                    "Comparison Note:",
+                    _comparison_line(case, baseline),
+                    "",
+                ]
+            )
+    return "\n".join(lines).rstrip() + "\n"
+
+
+def write_eval_markdown_report(
+    payload: dict[str, Any],
+    output: str | Path,
+    *,
+    title: str = "Personal Network Memory Retrieval Test Report",
+    compare_payload: dict[str, Any] | None = None,
+) -> None:
+    path = Path(output)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(
+        render_eval_markdown_report(payload, title=title, compare_payload=compare_payload),
+        encoding="utf-8",
+    )
+
+
+def _cases_by_index(payload: dict[str, Any] | None) -> dict[object, dict[str, Any]]:
+    if not payload:
+        return {}
+    cases: dict[object, dict[str, Any]] = {}
+    for case in payload.get("cases", []):
+        if isinstance(case, dict):
+            cases[case.get("query_index")] = case
+    return cases
+
+
+def _rank_summary_lines(payload: dict[str, Any], *, prefix: str = "") -> list[str]:
+    cases = [case for case in payload.get("cases", []) if isinstance(case, dict)]
+    if not cases:
+        return []
+    strict_top1 = sum(1 for case in cases if case.get("strict_match_rank") == 1)
+    expected_top1 = sum(1 for case in cases if case.get("expected_person_rank") == 1)
+    expected_top3 = sum(
+        1
+        for case in cases
+        if isinstance(case.get("expected_person_rank"), int)
+        and case["expected_person_rank"] <= 3
+    )
+    expected_top5 = sum(
+        1
+        for case in cases
+        if isinstance(case.get("expected_person_rank"), int)
+        and case["expected_person_rank"] <= 5
+    )
+    total = len(cases)
+    return [
+        f"- {prefix}Strict match at rank 1: {strict_top1}/{total}",
+        f"- {prefix}Expected person at rank 1: {expected_top1}/{total}",
+        f"- {prefix}Expected person within top 3: {expected_top3}/{total}",
+        f"- {prefix}Expected person within top 5: {expected_top5}/{total}",
+    ]
+
+
+def _changed_top_count(payload: dict[str, Any], baseline: dict[str, Any]) -> str:
+    baseline_cases = _cases_by_index(baseline)
+    changed = 0
+    checked = 0
+    for case in payload.get("cases", []):
+        if not isinstance(case, dict):
+            continue
+        base_case = baseline_cases.get(case.get("query_index"))
+        if not base_case:
+            continue
+        checked += 1
+        if _top_item_id(case) != _top_item_id(base_case):
+            changed += 1
+    return f"{changed}/{checked}" if checked else "0/0"
+
+
+def _expected_line(case: dict[str, Any]) -> str:
+    people = ", ".join(str(item) for item in case.get("expected_people", [])) or "None"
+    terms = ", ".join(str(item) for item in case.get("expected_terms", [])) or "None"
+    return f"- Expected people: {people}; expected terms/facts: {terms}"
+
+
+def _actual_lines(results: object) -> list[str]:
+    if not isinstance(results, list) or not results:
+        return ["- No results returned."]
+    lines: list[str] = []
+    for result in results[:5]:
+        if not isinstance(result, dict):
+            continue
+        evidence = result.get("evidence", [])
+        evidence_text = ""
+        if isinstance(evidence, list) and evidence:
+            first = evidence[0]
+            if isinstance(first, dict):
+                evidence_text = str(first.get("source_text", ""))
+        lines.append(
+            "- Rank {rank}: {title} ({kind}) - {matched_text}".format(
+                rank=result.get("rank"),
+                title=result.get("title"),
+                kind=result.get("kind"),
+                matched_text=_compact(str(result.get("matched_text", ""))),
+            )
+        )
+        lines.append(f"  - Why returned: {_compact(str(result.get('why_matched', '')))}")
+        if evidence_text:
+            lines.append(f"  - Evidence: {_compact(evidence_text)}")
+    return lines
+
+
+def _reasoning_line(case: dict[str, Any]) -> str:
+    category = str(case.get("category"))
+    strict_rank = case.get("strict_match_rank")
+    expected_rank = case.get("expected_person_rank")
+    if strict_rank is None:
+        return (
+            "- FAIL: no top-5 result satisfied the intent-aware rule for this category. "
+            f"Matched expected strings: {case.get('matched_expected', [])}; "
+            f"missed: {case.get('missed_expected', [])}."
+        )
+    if category == "mentioned":
+        rule = "a result must identify the speaker and state that they mentioned the target"
+    elif category == "follow_up":
+        rule = "a result must be a follow-up item for the requested person"
+    elif category == "profile":
+        rule = "one result must combine the requested profile terms for a person"
+    elif category in {"vague", "bilingual"}:
+        rule = "one result must include the requested vague place/topic clues"
+    else:
+        rule = "one result must satisfy all expected terms"
+    suffix = (
+        f" Expected fixture person first appears at rank {expected_rank}."
+        if expected_rank is not None
+        else ""
+    )
+    return f"- PASS: strict match at rank {strict_rank}; {rule}.{suffix}"
+
+
+def _comparison_line(case: dict[str, Any], baseline: dict[str, Any]) -> str:
+    judged_top = _top_item_id(case)
+    baseline_top = _top_item_id(baseline)
+    if judged_top == baseline_top:
+        return "- The LLM-judged top result is the same as the deterministic top result."
+    return (
+        "- The LLM judge changed the top result from "
+        f"`{baseline_top}` to `{judged_top}`."
+    )
+
+
+def _top_item_id(case: dict[str, Any]) -> str:
+    results = case.get("actual_results", [])
+    if isinstance(results, list) and results and isinstance(results[0], dict):
+        return str(results[0].get("item_id") or results[0].get("title"))
+    return "none"
+
+
+def _compact(text: str, *, limit: int = 420) -> str:
+    compacted = " ".join(text.split())
+    if len(compacted) <= limit:
+        return compacted
+    return compacted[: limit - 3].rstrip() + "..."

package/src/people_network_memory/graphiti_adapter/__init__.py

@@ -0,0 +1,9 @@
+"""Graphiti adapter boundary.
+
+This package is the only place allowed to import Graphiti.
+"""
+
+from people_network_memory.graphiti_adapter.graphiti_store import GraphitiGraphStore
+
+__all__ = ["GraphitiGraphStore"]
+

package/src/people_network_memory/graphiti_adapter/episode_formatter.py

@@ -0,0 +1,70 @@
+"""Compact Graphiti episode formatting.
+
+Graphiti's extraction step is LLM-backed, so large structured JSON episodes can
+be slow and fragile. This formatter keeps the social-memory semantics explicit
+while sending short text snippets to Graphiti.
+"""
+
+from __future__ import annotations
+
+from people_network_memory.domain.models import SocialInteraction
+
+
+def format_graphiti_episode(interaction: SocialInteraction) -> str:
+    lines: list[str] = []
+    if interaction.occurred_at:
+        lines.append(f"Date: {interaction.occurred_at.date().isoformat()}.")
+    if interaction.interaction_type:
+        lines.append(f"Interaction type: {interaction.interaction_type}.")
+    if interaction.place:
+        lines.append(f"Place: {interaction.place}.")
+    participants = [participant.person.label for participant in interaction.participants]
+    if participants:
+        lines.append("Participants: " + ", ".join(participants) + ".")
+    mentioned = [
+        _mentioned_sentence(index, mention)
+        for index, mention in enumerate(interaction.mentioned_people, start=1)
+    ]
+    lines.extend(mentioned)
+    for topic in interaction.topics:
+        lines.append(f"Discussed topic: {topic}.")
+    for fact in interaction.direct_facts:
+        lines.append(
+            f"Direct fact: {fact.subject.label} {fact.predicate.replace('_', ' ')} {fact.value}."
+        )
+    for claim in interaction.attributed_claims:
+        lines.append(
+            "Attributed claim: "
+            f"{claim.speaker.label} said about {claim.subject.label}: {claim.claim_text}"
+        )
+    for relationship in interaction.relationships:
+        lines.append(
+            "Relationship: "
+            f"{relationship.source.label} {relationship.relationship_type.replace('_', ' ')} "
+            f"{relationship.target.label}."
+        )
+    for follow_up in interaction.follow_ups:
+        related = ", ".join(person.label for person in follow_up.related_people)
+        if related:
+            lines.append(f"Follow-up: {follow_up.description} Related people: {related}.")
+        else:
+            lines.append(f"Follow-up: {follow_up.description}.")
+    if interaction.sensitivity:
+        labels = ", ".join(label.value for label in interaction.sensitivity)
+        lines.append(f"Sensitivity labels: {labels}.")
+    lines.append(f"Original note: {interaction.source_text}")
+    return "\n".join(lines)
+
+
+def _mentioned_sentence(index: int, mention: object) -> str:
+    person = getattr(mention, "person")
+    mentioned_by = getattr(mention, "mentioned_by", None)
+    context = getattr(mention, "context", None)
+    prefix = f"Mentioned person {index}: "
+    if mentioned_by:
+        sentence = f"{mentioned_by.label} mentioned {person.label}"
+    else:
+        sentence = f"{person.label} was mentioned"
+    if context:
+        sentence += f" in context: {context}"
+    return prefix + sentence + "."