@reconcrap/people-network-memory 0.1.0

package/src/people_network_memory/harness_adapters/openclaw/integration_eval.py

@@ -0,0 +1,508 @@
+"""Deterministic harness-memory integration evals.
+
+These checks model the contract that a harness should follow after reading the
+OpenClaw skill: use this MCP for people/network evidence, then apply the
+harness's own user memory for style, goals, and task constraints.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Literal
+
+from people_network_memory.config import PeopleMemoryConfig
+from people_network_memory.harness_adapters.openclaw.smoke import (
+    route_prompt_for_openclaw_skill,
+)
+from people_network_memory.mcp_server.runtime import build_runtime
+
+
+@dataclass(frozen=True)
+class HarnessIntegrationCase:
+    case_id: str
+    prompt: str
+    harness_memory: list[str]
+    expected_mode: Literal["recall", "brief"] = "recall"
+    sensitivity_policy: Literal["personal", "strict", "task_aware"] = "personal"
+    output_context: Literal["private", "shareable"] = "private"
+    expected_social_terms: list[str] = field(default_factory=list)
+    expected_user_memory_terms: list[str] = field(default_factory=list)
+    forbidden_social_fact_terms: list[str] = field(default_factory=list)
+    require_get_person: bool = False
+    require_follow_up: bool = False
+    require_no_named_person_evidence: str | None = None
+    expect_conflict_resolution: bool = False
+
+    def to_json(self) -> dict[str, Any]:
+        return {
+            "case_id": self.case_id,
+            "prompt": self.prompt,
+            "harness_memory": self.harness_memory,
+            "expected_mode": self.expected_mode,
+            "sensitivity_policy": self.sensitivity_policy,
+            "output_context": self.output_context,
+            "expected_social_terms": self.expected_social_terms,
+            "expected_user_memory_terms": self.expected_user_memory_terms,
+            "forbidden_social_fact_terms": self.forbidden_social_fact_terms,
+            "require_get_person": self.require_get_person,
+            "require_follow_up": self.require_follow_up,
+            "require_no_named_person_evidence": self.require_no_named_person_evidence,
+            "expect_conflict_resolution": self.expect_conflict_resolution,
+        }
+
+
+HARNESS_INTEGRATION_CASES = [
+    HarnessIntegrationCase(
+        case_id="dinner_group_curates_social_graph_with_user_preferences",
+        prompt=(
+            "Who should I invite to dinner in Shanghai for robotics and founder intros? "
+            "Use what you know about my preferences."
+        ),
+        harness_memory=[
+            "The user prefers small dinners, practical robotics conversations, and warm founder introductions."
+        ],
+        expected_social_terms=["Shanghai", "robotics", "founder intros"],
+        expected_user_memory_terms=["small dinners", "practical robotics conversations"],
+        require_get_person=True,
+    ),
+    HarnessIntegrationCase(
+        case_id="shareable_intro_uses_style_memory_without_sensitive_claims",
+        prompt="Draft a warm intro message to Alice Zhang about a robotics founder dinner.",
+        harness_memory=[
+            "The user prefers concise warm messages and does not want sensitive claims in shareable drafts."
+        ],
+        sensitivity_policy="task_aware",
+        output_context="shareable",
+        expected_social_terms=["Alice Zhang", "Tencent Robotics", "robotics"],
+        expected_user_memory_terms=["concise warm messages"],
+        forbidden_social_fact_terms=["may leave Ant Group"],
+        require_get_person=True,
+    ),
+    HarnessIntegrationCase(
+        case_id="conflict_prefers_mcp_for_person_fact",
+        prompt="Brief me on Alice Zhang's current company before I meet her.",
+        harness_memory=["Older harness memory says Alice Zhang works at ByteDance AI Lab."],
+        expected_mode="brief",
+        expected_social_terms=["Alice Zhang", "Tencent Robotics"],
+        expected_user_memory_terms=["Older harness memory"],
+        forbidden_social_fact_terms=["ByteDance AI Lab"],
+        require_get_person=True,
+        expect_conflict_resolution=True,
+    ),
+    HarnessIntegrationCase(
+        case_id="missing_social_graph_evidence_is_separated_from_user_memory",
+        prompt="Should I invite Kai Wu to the robotics dinner?",
+        harness_memory=["The user likes robotics founders and prefers warm intros."],
+        expected_social_terms=[],
+        expected_user_memory_terms=["robotics founders", "warm intros"],
+        require_no_named_person_evidence="Kai Wu",
+    ),
+    HarnessIntegrationCase(
+        case_id="promise_query_uses_follow_up_plus_user_style",
+        prompt="What did I promise Alice Zhang and how should I phrase the reminder in my usual style?",
+        harness_memory=["The user prefers concise action-item reminders."],
+        expected_social_terms=["founder contacts"],
+        expected_user_memory_terms=["concise action-item reminders"],
+        require_follow_up=True,
+        require_get_person=True,
+    ),
+]
+
+
+def run_harness_memory_integration_eval() -> dict[str, Any]:
+    runtime = build_runtime(PeopleMemoryConfig(test_mode=True))
+    try:
+        _seed_social_memory(runtime.tools)
+        cases = [_run_case(runtime.tools, case) for case in HARNESS_INTEGRATION_CASES]
+        return {
+            "ok": all(case["ok"] for case in cases),
+            "checked": len(cases),
+            "passed": sum(1 for case in cases if case["ok"]),
+            "failed": sum(1 for case in cases if not case["ok"]),
+            "cases": cases,
+        }
+    finally:
+        runtime.close()
+
+
+def _seed_social_memory(tools: Any) -> None:
+    alice_record = tools.record_interaction(
+        {
+            "source_text": (
+                "Met Alice Zhang at Blue Bottle Coffee in Shanghai. We discussed robotics "
+                "hiring and founder intros. Alice Zhang works at Tencent Robotics as product lead."
+            ),
+            "place": "Blue Bottle Coffee Shanghai",
+            "participants": [{"person": {"label": "Alice Zhang"}}],
+            "topics": ["robotics hiring", "founder intros"],
+            "direct_facts": [
+                {
+                    "subject": {"label": "Alice Zhang"},
+                    "predicate": "works_at",
+                    "value": "Tencent Robotics",
+                    "metadata": {"role": "product lead"},
+                },
+                {
+                    "subject": {"label": "Alice Zhang"},
+                    "predicate": "interest",
+                    "value": "robotics hiring",
+                },
+                {
+                    "subject": {"label": "Alice Zhang"},
+                    "predicate": "interest",
+                    "value": "founder intros",
+                },
+            ],
+        }
+    )
+    alice_id = alice_record["person_ref_map"]["Alice Zhang"]
+    bob_record = tools.record_interaction(
+        {
+            "source_text": (
+                "Met Bob Li at Zhangjiang Cafe in Shanghai. We discussed robot learning "
+                "and founder intros. Bob Li works at Ant Group."
+            ),
+            "place": "Zhangjiang Cafe Shanghai",
+            "participants": [{"person": {"label": "Bob Li"}}],
+            "topics": ["robot learning", "founder intros"],
+            "direct_facts": [
+                {
+                    "subject": {"label": "Bob Li"},
+                    "predicate": "works_at",
+                    "value": "Ant Group",
+                },
+                {
+                    "subject": {"label": "Bob Li"},
+                    "predicate": "interest",
+                    "value": "robot learning",
+                },
+                {
+                    "subject": {"label": "Bob Li"},
+                    "predicate": "interest",
+                    "value": "founder intros",
+                },
+            ],
+        }
+    )
+    bob_id = bob_record["person_ref_map"]["Bob Li"]
+    tools.record_interaction(
+        {
+            "source_text": (
+                "Had dinner with Fiona Chen in Shanghai. We discussed Japanese food and "
+                "founder intros. Fiona Chen knows Alice Zhang."
+            ),
+            "place": "Shanghai",
+            "interaction_type": "dinner",
+            "participants": [{"person": {"label": "Fiona Chen"}}],
+            "mentioned_people": [
+                {
+                    "person": {"label": "Alice Zhang", "person_id": alice_id},
+                    "mentioned_by": {"label": "Fiona Chen"},
+                    "context": "Fiona Chen knows Alice Zhang.",
+                }
+            ],
+            "topics": ["Japanese food", "founder intros"],
+            "relationships": [
+                {
+                    "source": {"label": "Fiona Chen"},
+                    "target": {"label": "Alice Zhang", "person_id": alice_id},
+                    "relationship_type": "knows",
+                }
+            ],
+        }
+    )
+    tools.record_interaction(
+        {
+            "source_text": (
+                "Alice Zhang said Bob Li may leave Ant Group. Treat this as sensitive "
+                "secondhand context, not as a shareable fact."
+            ),
+            "participants": [{"person": {"label": "Alice Zhang", "person_id": alice_id}}],
+            "mentioned_people": [
+                {
+                    "person": {"label": "Bob Li", "person_id": bob_id},
+                    "mentioned_by": {"label": "Alice Zhang", "person_id": alice_id},
+                    "context": "Alice Zhang said Bob Li may leave Ant Group.",
+                }
+            ],
+            "attributed_claims": [
+                {
+                    "speaker": {"label": "Alice Zhang", "person_id": alice_id},
+                    "subject": {"label": "Bob Li", "person_id": bob_id},
+                    "claim_text": "Alice Zhang said Bob Li may leave Ant Group.",
+                    "sensitivity": ["sensitive", "secondhand", "unverified"],
+                }
+            ],
+        }
+    )
+    tools.record_interaction(
+        {
+            "source_text": (
+                "Promised Alice Zhang I would send two founder contacts next week."
+            ),
+            "participants": [{"person": {"label": "Alice Zhang", "person_id": alice_id}}],
+            "follow_ups": [
+                {
+                    "description": "Follow up with Alice Zhang by sending two founder contacts.",
+                    "related_people": [{"label": "Alice Zhang", "person_id": alice_id}],
+                }
+            ],
+        }
+    )
+
+
+def _run_case(tools: Any, case: HarnessIntegrationCase) -> dict[str, Any]:
+    route = route_prompt_for_openclaw_skill(case.prompt)
+    retrieve_payload = {
+        "query": case.prompt,
+        "mode": case.expected_mode,
+        "limit": 8,
+        "sensitivity_policy": case.sensitivity_policy,
+        "output_context": case.output_context,
+    }
+    retrieval = tools.retrieve_network_context(retrieve_payload)
+    person_cards = _get_shortlisted_cards(tools, retrieval, enabled=case.require_get_person)
+    simulated = _simulate_harness_answer(case, retrieval, person_cards)
+    checks = _checks(case, route, retrieval, person_cards, simulated)
+    return {
+        "ok": all(check["ok"] for check in checks),
+        "case": case.to_json(),
+        "route": route,
+        "tool_calls": _tool_calls(retrieve_payload, person_cards),
+        "retrieval_result_count": len(retrieval["results"]),
+        "top_results": [_compact_result(item) for item in retrieval["results"][:5]],
+        "simulated_harness_answer": simulated,
+        "checks": checks,
+    }
+
+
+def _get_shortlisted_cards(tools: Any, retrieval: dict[str, Any], *, enabled: bool) -> list[dict[str, Any]]:
+    if not enabled:
+        return []
+    person_ids: list[str] = []
+    for result in retrieval["results"]:
+        for person_id in result.get("person_ids", []):
+            if person_id not in person_ids:
+                person_ids.append(person_id)
+    cards: list[dict[str, Any]] = []
+    for person_id in person_ids[:5]:
+        card = tools.get_person({"person_id": person_id})
+        if card.get("found"):
+            cards.append(card)
+    return cards
+
+
+def _simulate_harness_answer(
+    case: HarnessIntegrationCase,
+    retrieval: dict[str, Any],
+    person_cards: list[dict[str, Any]],
+) -> dict[str, Any]:
+    evidence_text = _joined_retrieval_text(retrieval, person_cards)
+    social_terms = [term for term in case.expected_social_terms if _contains(evidence_text, term)]
+    user_memory_text = " ".join(case.harness_memory)
+    user_memory_terms = [
+        term for term in case.expected_user_memory_terms if _contains(user_memory_text, term)
+    ]
+    forbidden_as_social_fact = [
+        term for term in case.forbidden_social_fact_terms if _contains(evidence_text, term)
+    ]
+    shortlisted_people = [
+        {
+            "person_id": card["person_id"],
+            "display_name": card["display_name"],
+            "headline": card.get("headline"),
+            "why": _person_reason(card),
+        }
+        for card in person_cards
+    ]
+    named_missing = case.require_no_named_person_evidence
+    no_named_person_evidence = (
+        bool(named_missing)
+        and not any(_contains(_result_text(item), named_missing or "") for item in retrieval["results"])
+    )
+    conflict_resolution = None
+    if case.expect_conflict_resolution:
+        conflict_resolution = {
+            "person_fact_source": "mcp_evidence",
+            "preferred_fact": "Alice Zhang works at Tencent Robotics.",
+            "conflicting_harness_memory": "Older harness memory says Alice Zhang works at ByteDance AI Lab.",
+            "rule": "prefer MCP evidence for social-memory facts",
+        }
+    return {
+        "answer_strategy": (
+            "Use MCP evidence for people/network facts, then apply harness user memory "
+            "for tone, preferences, current goals, and task constraints."
+        ),
+        "social_evidence_terms_used": social_terms,
+        "user_memory_terms_used": user_memory_terms,
+        "shortlisted_people": shortlisted_people,
+        "forbidden_terms_seen_only_as_retrieved_context": forbidden_as_social_fact,
+        "no_named_person_evidence": no_named_person_evidence,
+        "conflict_resolution": conflict_resolution,
+        "source_boundary": {
+            "people_network_memory": "MCP retrieval and person cards",
+            "user_memory": "harness memory supplied with the case",
+        },
+    }
+
+
+def _checks(
+    case: HarnessIntegrationCase,
+    route: dict[str, Any],
+    retrieval: dict[str, Any],
+    person_cards: list[dict[str, Any]],
+    simulated: dict[str, Any],
+) -> list[dict[str, Any]]:
+    checks = [
+        _check(
+            "routes_to_retrieval",
+            route.get("tool") == "retrieve_network_context"
+            and route.get("mode", "recall") == case.expected_mode,
+            {"route": route, "expected_mode": case.expected_mode},
+        ),
+        _check(
+            "uses_harness_user_memory_terms",
+            set(case.expected_user_memory_terms).issubset(simulated["user_memory_terms_used"]),
+            {
+                "expected": case.expected_user_memory_terms,
+                "used": simulated["user_memory_terms_used"],
+            },
+        ),
+        _check(
+            "uses_mcp_social_evidence_terms",
+            set(case.expected_social_terms).issubset(simulated["social_evidence_terms_used"]),
+            {
+                "expected": case.expected_social_terms,
+                "used": simulated["social_evidence_terms_used"],
+            },
+        ),
+    ]
+    if case.require_get_person:
+        checks.append(
+            _check(
+                "hydrates_shortlisted_people",
+                bool(person_cards),
+                {"card_count": len(person_cards)},
+            )
+        )
+    if case.require_follow_up:
+        checks.append(
+            _check(
+                "prioritizes_follow_up_results",
+                any(item.get("kind") == "follow_up" for item in retrieval["results"][:3]),
+                {"top_kinds": [item.get("kind") for item in retrieval["results"][:3]]},
+            )
+        )
+    if case.forbidden_social_fact_terms:
+        checks.append(
+            _check(
+                "does_not_promote_forbidden_harness_or_sensitive_terms_as_social_fact",
+                not simulated["forbidden_terms_seen_only_as_retrieved_context"],
+                {
+                    "forbidden": case.forbidden_social_fact_terms,
+                    "seen": simulated["forbidden_terms_seen_only_as_retrieved_context"],
+                },
+            )
+        )
+    if case.require_no_named_person_evidence:
+        checks.append(
+            _check(
+                "separates_missing_social_graph_evidence_from_user_memory",
+                simulated["no_named_person_evidence"],
+                {"missing_name": case.require_no_named_person_evidence},
+            )
+        )
+    if case.expect_conflict_resolution:
+        checks.append(
+            _check(
+                "conflict_prefers_mcp_for_social_fact",
+                bool(simulated["conflict_resolution"])
+                and _contains(_joined_retrieval_text(retrieval, person_cards), "Tencent Robotics"),
+                {"conflict_resolution": simulated["conflict_resolution"]},
+            )
+        )
+    return checks
+
+
+def _tool_calls(retrieve_payload: dict[str, Any], person_cards: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    calls = [{"tool": "retrieve_network_context", "payload": retrieve_payload}]
+    calls.extend(
+        {"tool": "get_person", "payload": {"person_id": card["person_id"]}}
+        for card in person_cards
+    )
+    return calls
+
+
+def _compact_result(item: dict[str, Any]) -> dict[str, Any]:
+    return {
+        "kind": item.get("kind"),
+        "title": item.get("title"),
+        "matched_text": item.get("matched_text"),
+        "why_matched": item.get("why_matched"),
+        "person_ids": item.get("person_ids", []),
+        "evidence": [
+            {
+                "source_text": evidence.get("source_text"),
+                "recorded_at": evidence.get("recorded_at"),
+            }
+            for evidence in item.get("evidence", [])[:2]
+        ],
+    }
+
+
+def _check(name: str, ok: bool, detail: dict[str, Any]) -> dict[str, Any]:
+    return {"name": name, "ok": bool(ok), "detail": detail}
+
+
+def _joined_retrieval_text(retrieval: dict[str, Any], person_cards: list[dict[str, Any]]) -> str:
+    result_text = " ".join(_result_text(item) for item in retrieval["results"])
+    card_text = " ".join(_card_text(card) for card in person_cards)
+    return f"{result_text} {card_text}"
+
+
+def _result_text(item: dict[str, Any]) -> str:
+    evidence_text = " ".join(
+        str(evidence.get("source_text", "")) for evidence in item.get("evidence", [])
+    )
+    return " ".join(
+        [
+            str(item.get("title", "")),
+            str(item.get("matched_text", "")),
+            str(item.get("why_matched", "")),
+            evidence_text,
+        ]
+    )
+
+
+def _card_text(card: dict[str, Any]) -> str:
+    work = " ".join(str(item.get("organization", "")) for item in card.get("work_history", []))
+    interests = " ".join(str(item) for item in card.get("interests", []))
+    follow_ups = " ".join(str(item.get("description", "")) for item in card.get("open_follow_ups", []))
+    return " ".join(
+        [
+            str(card.get("display_name", "")),
+            str(card.get("headline", "")),
+            str(card.get("summary", "")),
+            work,
+            interests,
+            follow_ups,
+        ]
+    )
+
+
+def _person_reason(card: dict[str, Any]) -> str:
+    parts = []
+    if card.get("work_history"):
+        parts.append(
+            "works at "
+            + ", ".join(str(item.get("organization", "")) for item in card["work_history"][:2])
+        )
+    if card.get("interests"):
+        parts.append("interests: " + ", ".join(str(item) for item in card["interests"][:3]))
+    if card.get("open_follow_ups"):
+        parts.append("has open follow-up")
+    return "; ".join(parts) or "matched retrieved social-memory evidence"
+
+
+def _contains(text: str, term: str) -> bool:
+    return term.lower() in text.lower()