@reconcrap/people-network-memory 0.1.0

package/src/people_network_memory/infrastructure/llm_extractor.py

@@ -0,0 +1,476 @@
+"""Optional OpenAI-compatible ingestion extractor.
+
+The extractor is intentionally an infrastructure adapter. Application services
+only depend on the InteractionExtractor port and keep deterministic validation,
+identity policy, and evidence handling after this step.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass
+from typing import Any
+
+import httpx
+from pydantic import ValidationError
+
+from people_network_memory.config import PeopleMemoryConfig
+from people_network_memory.domain.models import SocialInteraction
+from people_network_memory.ports.errors import ConfigError
+
+
+INTERACTION_PATCH_FIELDS = {
+    "occurred_at",
+    "interaction_type",
+    "place",
+    "participants",
+    "mentioned_people",
+    "topics",
+    "direct_facts",
+    "attributed_claims",
+    "follow_ups",
+    "relationships",
+    "sensitivity",
+    "metadata",
+}
+
+CONTACT_PREFERENCE_ONLY_RE = re.compile(
+    r"(?:之后|以后|后续|下次|以后有事)?\s*"
+    r"(?:最好|偏好|喜欢|倾向于)?\s*(?:用|通过)?\s*"
+    r"(?:微信|短信|电话|邮件|邮箱|whatsapp|飞书|钉钉)\s*"
+    r"(?:联系|沟通|跟进)",
+    flags=re.IGNORECASE,
+)
+CONCRETE_FOLLOW_UP_CUE_RE = re.compile(
+    r"(?:"
+    r"明天|后天|下周|周[一二三四五六日天]|星期[一二三四五六日天]|礼拜[一二三四五六日天]|"
+    r"\d{1,2}[月/.-]\d{1,2}|20\d{2}[-/]\d{1,2}[-/]\d{1,2}|"
+    r"答应|承诺|说好|约好|提醒我|需要我|让我|"
+    r"发|发送|推荐|介绍|引荐|安排|确认|准备|"
+    r"tomorrow|next\s+week|next\s+\w+day|by\s+\w+day|"
+    r"promised|asked\s+me\s+to|need\s+to|send|introduce|share|confirm|schedule"
+    r")",
+    flags=re.IGNORECASE,
+)
+
+
+@dataclass(frozen=True)
+class LlmExtractorSettings:
+    base_url: str
+    model: str
+    api_key: str
+    timeout_seconds: float = 30.0
+    response_format: str = "none"
+
+    @classmethod
+    def from_config(cls, config: PeopleMemoryConfig) -> "LlmExtractorSettings":
+        missing: list[str] = []
+        if not config.llm_base_url:
+            missing.append("PEOPLE_MEMORY_LLM_BASE_URL")
+        if not config.llm_model:
+            missing.append("PEOPLE_MEMORY_LLM_MODEL")
+        if not config.llm_api_key:
+            missing.append("PEOPLE_MEMORY_LLM_API_KEY")
+        if missing:
+            raise ConfigError("LLM ingestion extractor requires: " + ", ".join(missing))
+        return cls(
+            base_url=config.llm_base_url or "",
+            model=config.llm_model or "",
+            api_key=config.llm_api_key or "",
+            timeout_seconds=config.ingestion_extractor_timeout_seconds,
+            response_format=config.llm_response_format,
+        )
+
+
+class OpenAICompatibleInteractionExtractor:
+    """Extract structure from messy social notes before deterministic normalization."""
+
+    def __init__(self, settings: LlmExtractorSettings) -> None:
+        self._settings = settings
+
+    @classmethod
+    def from_config(
+        cls, config: PeopleMemoryConfig
+    ) -> "OpenAICompatibleInteractionExtractor":
+        return cls(LlmExtractorSettings.from_config(config))
+
+    def extract(self, interaction: SocialInteraction) -> SocialInteraction:
+        try:
+            payload = self._request_interaction_patch(interaction)
+            extracted = _interaction_from_patch(interaction, payload)
+        except (
+            httpx.HTTPError,
+            KeyError,
+            TypeError,
+            ValueError,
+            json.JSONDecodeError,
+            ValidationError,
+        ):
+            return interaction
+        return _merge_interactions(interaction, extracted)
+
+    def _request_interaction_patch(self, interaction: SocialInteraction) -> dict[str, Any]:
+        url = self._settings.base_url.rstrip("/") + "/chat/completions"
+        payload: dict[str, Any] = {
+            "model": self._settings.model,
+            "messages": [
+                {
+                    "role": "system",
+                    "content": (
+                        "You extract structured social-memory data from messy user notes. "
+                        "Return only JSON. Preserve the user's original source text. "
+                        "Use only facts visible in the note. Do not invent person IDs, "
+                        "dates, companies, or relationships. Separate people present from "
+                        "people merely mentioned. Store 'A said B...' as an attributed "
+                        "claim, not a direct fact about B. For Chinese aliases like "
+                        "胡八一（胖子）, 胡八一又叫胖子, or 胡八一绰号胖子, use 胡八一 as "
+                        "the person label and put 胖子 in that PersonRef's aliases. "
+                        "Preserve visible Chinese names exactly; do not remove prefixes "
+                        "such as 测试 or infer a different legal name. Extract follow-ups "
+                        "and commitments broadly, not only literal promise wording: examples "
+                        "include 我答应, 我回头, 记得, 提醒我, 她让我, 需要我, 下次帮她, "
+                        "need to, should, promised, asked me to, follow up, send, introduce. "
+                        "A follow-up is a user action or reminder after the interaction; do "
+                        "not store it as an attributed claim. Contact preferences such as "
+                        "之后最好用邮件联系, prefers WeChat, or likes email are preferences, "
+                        "not follow-ups, unless the user gave a concrete task or date. "
+                        "If a relative due date is clear "
+                        "and existing_structured_fields has occurred_at, normalize due_at as "
+                        "YYYY-MM-DD; otherwise omit due_at or set it to null. Keep follow-up "
+                        "descriptions specific enough to act on, including person, action, "
+                        "object, and visible time phrase when present."
+                    ),
+                },
+                {
+                    "role": "user",
+                    "content": json.dumps(
+                        {
+                            "source_text": interaction.source_text,
+                            "existing_structured_fields": _existing_structured_payload(
+                                interaction
+                            ),
+                            "return_schema": {
+                                "interaction_type": (
+                                    "meeting|coffee|dinner|call|message|intro|event|other"
+                                ),
+                                "place": "string or null",
+                                "participants": [
+                                    {"person": {"label": "string", "aliases": ["string"]}}
+                                ],
+                                "mentioned_people": [
+                                    {
+                                        "person": {"label": "string"},
+                                        "mentioned_by": {"label": "string"},
+                                        "context": "string or null",
+                                    }
+                                ],
+                                "topics": ["string"],
+                                "direct_facts": [
+                                    {
+                                        "subject": {"label": "string"},
+                                        "predicate": "works_at|studied_at|interest|...",
+                                        "value": "string",
+                                        "metadata": {
+                                            "role": "string when a role/title is visible"
+                                        },
+                                    }
+                                ],
+                                "attributed_claims": [
+                                    {
+                                        "speaker": {"label": "string"},
+                                        "subject": {"label": "string"},
+                                        "claim_text": "verbatim claim sentence",
+                                        "claim_type": "optional short category",
+                                    }
+                                ],
+                                "follow_ups": [
+                                    {
+                                        "description": (
+                                            "specific action item, preserving user language"
+                                        ),
+                                        "due_at": "YYYY-MM-DD or null",
+                                        "related_people": [{"label": "string"}],
+                                    }
+                                ],
+                                "relationships": [
+                                    {
+                                        "source": {"label": "string"},
+                                        "target": {"label": "string"},
+                                        "relationship_type": "knows|works_with|...",
+                                    }
+                                ],
+                            },
+                        },
+                        ensure_ascii=False,
+                    ),
+                },
+            ],
+            "temperature": 0,
+        }
+        if self._settings.response_format == "json_object":
+            payload["response_format"] = {"type": "json_object"}
+        response = httpx.post(
+            url,
+            headers={
+                "Authorization": f"Bearer {self._settings.api_key}",
+                "Content-Type": "application/json",
+            },
+            json=payload,
+            timeout=self._settings.timeout_seconds,
+        )
+        response.raise_for_status()
+        content = response.json()["choices"][0]["message"]["content"]
+        return _parse_json_object(content)
+
+
+def _existing_structured_payload(interaction: SocialInteraction) -> dict[str, object]:
+    payload = interaction.model_dump(mode="json", exclude={"source_text"})
+    return {key: value for key, value in payload.items() if value not in (None, [], {})}
+
+
+def _interaction_from_patch(
+    original: SocialInteraction, patch: dict[str, Any]
+) -> SocialInteraction:
+    if "interaction" in patch and isinstance(patch["interaction"], dict):
+        patch = patch["interaction"]
+    if "record_interaction" in patch and isinstance(patch["record_interaction"], dict):
+        patch = patch["record_interaction"]
+    clean_patch = {
+        key: _coerce_extractor_field(key, _without_person_ids(value))
+        for key, value in patch.items()
+        if key in INTERACTION_PATCH_FIELDS
+    }
+    clean_patch["source_text"] = original.source_text
+    return SocialInteraction.model_validate(clean_patch)
+
+
+def _without_person_ids(value: Any) -> Any:
+    if isinstance(value, dict):
+        return {
+            key: _without_person_ids(item)
+            for key, item in value.items()
+            if key != "person_id"
+        }
+    if isinstance(value, list):
+        return [_without_person_ids(item) for item in value]
+    return value
+
+
+def _coerce_extractor_field(field_name: str, value: Any) -> Any:
+    if field_name == "participants" and isinstance(value, list):
+        return [_coerce_participant(item) for item in value]
+    if field_name == "mentioned_people" and isinstance(value, list):
+        return [_coerce_mentioned_person(item) for item in value]
+    if field_name == "direct_facts" and isinstance(value, list):
+        return [_coerce_direct_fact(item) for item in value]
+    if field_name == "attributed_claims" and isinstance(value, list):
+        return [_coerce_attributed_claim(item) for item in value]
+    if field_name == "follow_ups" and isinstance(value, list):
+        return [
+            coerced
+            for coerced in (_coerce_follow_up(item) for item in value)
+            if not _looks_like_contact_preference_only_follow_up(coerced)
+        ]
+    if field_name == "relationships" and isinstance(value, list):
+        return [_coerce_relationship(item) for item in value]
+    return value
+
+
+def _coerce_participant(item: Any) -> Any:
+    if isinstance(item, str):
+        return {"person": {"label": item}}
+    if not isinstance(item, dict):
+        return item
+    if "person" in item:
+        return {**item, "person": _coerce_person_ref(item["person"])}
+    if "name" in item:
+        role = item.get("role", "participant")
+        return {"person": {"label": item["name"]}, "role": role}
+    return item
+
+
+def _coerce_mentioned_person(item: Any) -> Any:
+    if isinstance(item, str):
+        return {"person": {"label": item}}
+    if not isinstance(item, dict):
+        return item
+    coerced = dict(item)
+    if "person" in coerced:
+        coerced["person"] = _coerce_person_ref(coerced["person"])
+    elif "name" in coerced:
+        coerced["person"] = {"label": coerced.pop("name")}
+    if "mentioned_by" in coerced:
+        coerced["mentioned_by"] = _coerce_person_ref(coerced["mentioned_by"])
+    return coerced
+
+
+def _coerce_direct_fact(item: Any) -> Any:
+    if not isinstance(item, dict):
+        return item
+    coerced = dict(item)
+    if "subject" in coerced:
+        coerced["subject"] = _coerce_person_ref(coerced["subject"])
+    return coerced
+
+
+def _coerce_attributed_claim(item: Any) -> Any:
+    if not isinstance(item, dict):
+        return item
+    coerced = dict(item)
+    if "speaker" in coerced:
+        coerced["speaker"] = _coerce_person_ref(coerced["speaker"])
+    if "subject" in coerced:
+        coerced["subject"] = _coerce_person_ref(coerced["subject"])
+    return coerced
+
+
+def _coerce_follow_up(item: Any) -> Any:
+    if isinstance(item, str):
+        return {"description": item}
+    if not isinstance(item, dict):
+        return item
+    coerced = dict(item)
+    if "what" in coerced and "description" not in coerced:
+        coerced["description"] = coerced.pop("what")
+    if "due" in coerced and "due_at" not in coerced:
+        coerced["due_at"] = coerced.pop("due")
+    if isinstance(coerced.get("related_people"), list):
+        coerced["related_people"] = [
+            _coerce_person_ref(ref) for ref in coerced["related_people"]
+        ]
+    return coerced
+
+
+def _looks_like_contact_preference_only_follow_up(item: Any) -> bool:
+    if isinstance(item, str):
+        description = item
+    elif isinstance(item, dict):
+        description = str(item.get("description") or item.get("what") or "")
+    else:
+        return False
+    normalized = " ".join(description.casefold().split())
+    if not CONTACT_PREFERENCE_ONLY_RE.search(normalized):
+        return False
+    without_contact_phrase = CONTACT_PREFERENCE_ONLY_RE.sub("", normalized).strip()
+    without_contact_phrase = re.sub(
+        r"^[,，。；;\s]+|[,，。；;\s]+$", "", without_contact_phrase
+    )
+    if CONCRETE_FOLLOW_UP_CUE_RE.search(without_contact_phrase):
+        return False
+    return True
+
+
+def _coerce_relationship(item: Any) -> Any:
+    if not isinstance(item, dict):
+        return item
+    coerced = dict(item)
+    if "source" in coerced:
+        coerced["source"] = _coerce_person_ref(coerced["source"])
+    if "target" in coerced:
+        coerced["target"] = _coerce_person_ref(coerced["target"])
+    return coerced
+
+
+def _coerce_person_ref(item: Any) -> Any:
+    if isinstance(item, str):
+        return {"label": item}
+    if isinstance(item, dict) and "name" in item and "label" not in item:
+        coerced = dict(item)
+        coerced["label"] = coerced.pop("name")
+        return coerced
+    return item
+
+
+def _merge_interactions(
+    base: SocialInteraction, extracted: SocialInteraction
+) -> SocialInteraction:
+    updates: dict[str, object] = {}
+    if not base.occurred_at and extracted.occurred_at:
+        updates["occurred_at"] = extracted.occurred_at
+    if base.interaction_type == "meeting" and extracted.interaction_type != "meeting":
+        updates["interaction_type"] = extracted.interaction_type
+    if not base.place and extracted.place:
+        updates["place"] = extracted.place
+    for field_name in [
+        "participants",
+        "mentioned_people",
+        "topics",
+        "direct_facts",
+        "attributed_claims",
+        "follow_ups",
+        "relationships",
+        "sensitivity",
+    ]:
+        base_items = list(getattr(base, field_name))
+        extracted_items = list(getattr(extracted, field_name))
+        updates[field_name] = _merge_list_field(field_name, base_items, extracted_items)
+    if extracted.metadata:
+        updates["metadata"] = {**extracted.metadata, **base.metadata}
+    return base.model_copy(update=updates)
+
+
+def _merge_list_field(
+    field_name: str, base_items: list[Any], extracted_items: list[Any]
+) -> list[Any]:
+    result = list(base_items)
+    seen = {_dedupe_key(field_name, item) for item in result}
+    for item in extracted_items:
+        key = _dedupe_key(field_name, item)
+        if key in seen:
+            continue
+        seen.add(key)
+        result.append(item)
+    return result
+
+
+def _dedupe_key(field_name: str, item: Any) -> tuple[object, ...]:
+    if field_name == "topics" or field_name == "sensitivity":
+        return (str(item).casefold(),)
+    if field_name == "participants":
+        return (_person_key(item.person),)
+    if field_name == "mentioned_people":
+        mentioned_by = item.mentioned_by.label if item.mentioned_by else ""
+        return (_person_key(item.person), mentioned_by.casefold())
+    if field_name == "direct_facts":
+        return (
+            _person_key(item.subject),
+            item.predicate.casefold(),
+            item.value.casefold(),
+        )
+    if field_name == "attributed_claims":
+        speaker = item.speaker.label if item.speaker else ""
+        subject = item.subject.label if item.subject else ""
+        return (speaker.casefold(), subject.casefold(), item.claim_text.casefold())
+    if field_name == "follow_ups":
+        return (item.description.casefold(),)
+    if field_name == "relationships":
+        return (
+            _person_key(item.source),
+            _person_key(item.target),
+            item.relationship_type.casefold(),
+        )
+    return (repr(item),)
+
+
+def _person_key(ref: Any) -> str:
+    return str(getattr(ref, "label", "")).casefold().strip()
+
+
+def _parse_json_object(text: str) -> dict[str, Any]:
+    stripped = text.strip()
+    if stripped.startswith("```"):
+        lines = stripped.splitlines()
+        if len(lines) >= 3 and lines[-1].strip() == "```":
+            stripped = "\n".join(lines[1:-1]).strip()
+    try:
+        parsed = json.loads(stripped)
+    except json.JSONDecodeError:
+        start = stripped.find("{")
+        end = stripped.rfind("}")
+        if start < 0 or end <= start:
+            raise
+        parsed = json.loads(stripped[start : end + 1])
+    if not isinstance(parsed, dict):
+        raise ValueError("LLM extractor response must be a JSON object")
+    return parsed

package/src/people_network_memory/infrastructure/llm_identity_advisor.py

@@ -0,0 +1,200 @@
+"""Optional OpenAI-compatible identity disambiguation advisor."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from typing import Any
+
+import httpx
+from pydantic import ValidationError
+
+from people_network_memory.config import PeopleMemoryConfig
+from people_network_memory.domain.models import (
+    IdentityAdvice,
+    IdentityCandidate,
+    PersonRef,
+    SocialInteraction,
+)
+from people_network_memory.ports.errors import ConfigError
+
+
+@dataclass(frozen=True)
+class LlmIdentityAdvisorSettings:
+    base_url: str
+    model: str
+    api_key: str
+    timeout_seconds: float = 30.0
+    response_format: str = "none"
+
+    @classmethod
+    def from_config(cls, config: PeopleMemoryConfig) -> "LlmIdentityAdvisorSettings":
+        missing: list[str] = []
+        if not config.llm_base_url:
+            missing.append("PEOPLE_MEMORY_LLM_BASE_URL")
+        if not config.llm_model:
+            missing.append("PEOPLE_MEMORY_LLM_MODEL")
+        if not config.llm_api_key:
+            missing.append("PEOPLE_MEMORY_LLM_API_KEY")
+        if missing:
+            raise ConfigError("LLM identity advisor requires: " + ", ".join(missing))
+        return cls(
+            base_url=config.llm_base_url or "",
+            model=config.llm_model or "",
+            api_key=config.llm_api_key or "",
+            timeout_seconds=config.identity_advisor_timeout_seconds,
+            response_format=config.llm_response_format,
+        )
+
+
+class OpenAICompatibleIdentityAdvisor:
+    """Advises on identity ambiguity without owning merge/link decisions."""
+
+    def __init__(self, settings: LlmIdentityAdvisorSettings) -> None:
+        self._settings = settings
+
+    @classmethod
+    def from_config(cls, config: PeopleMemoryConfig) -> "OpenAICompatibleIdentityAdvisor":
+        return cls(LlmIdentityAdvisorSettings.from_config(config))
+
+    def advise(
+        self,
+        *,
+        interaction: SocialInteraction,
+        ref: PersonRef,
+        candidates: list[IdentityCandidate],
+    ) -> IdentityAdvice:
+        if not candidates:
+            return IdentityAdvice(recommendation="unknown")
+        try:
+            return self._request_advice(interaction=interaction, ref=ref, candidates=candidates)
+        except (
+            httpx.HTTPError,
+            KeyError,
+            TypeError,
+            ValueError,
+            json.JSONDecodeError,
+            ValidationError,
+        ):
+            return IdentityAdvice(recommendation="unknown")
+
+    def _request_advice(
+        self,
+        *,
+        interaction: SocialInteraction,
+        ref: PersonRef,
+        candidates: list[IdentityCandidate],
+    ) -> IdentityAdvice:
+        url = self._settings.base_url.rstrip("/") + "/chat/completions"
+        payload: dict[str, Any] = {
+            "model": self._settings.model,
+            "messages": [
+                {
+                    "role": "system",
+                    "content": (
+                        "You are an identity disambiguation advisor for a private "
+                        "personal-network memory tool. Decide whether the current "
+                        "reference appears to mean the same person as an existing "
+                        "candidate, a different person, or ambiguous. Use only the "
+                        "provided source note and candidate evidence. Return JSON "
+                        "only. This is advisory: never recommend merging unless the "
+                        "source clearly supports the same person."
+                    ),
+                },
+                {
+                    "role": "user",
+                    "content": json.dumps(
+                        {
+                            "source_text": interaction.source_text,
+                            "current_reference": {
+                                "label": ref.label,
+                                "aliases": ref.aliases,
+                                "company_hint": ref.company_hint,
+                            },
+                            "candidates": [_candidate_payload(item) for item in candidates],
+                            "return_schema": {
+                                "recommendation": (
+                                    "same_person|different_person|"
+                                    "ambiguous_needs_review|unknown"
+                                ),
+                                "confidence": "number 0..1",
+                                "candidate_person_id": "string or null",
+                                "reasons": ["short reason strings"],
+                                "evidence": ["short source/candidate evidence strings"],
+                            },
+                        },
+                        ensure_ascii=False,
+                    ),
+                },
+            ],
+            "temperature": 0,
+        }
+        if self._settings.response_format == "json_object":
+            payload["response_format"] = {"type": "json_object"}
+        response = httpx.post(
+            url,
+            headers={
+                "Authorization": f"Bearer {self._settings.api_key}",
+                "Content-Type": "application/json",
+            },
+            json=payload,
+            timeout=self._settings.timeout_seconds,
+        )
+        response.raise_for_status()
+        content = response.json()["choices"][0]["message"]["content"]
+        parsed = _parse_json_object(content)
+        recommendation = str(parsed.get("recommendation", "unknown"))
+        if recommendation not in {
+            "same_person",
+            "different_person",
+            "ambiguous_needs_review",
+            "unknown",
+        }:
+            recommendation = "unknown"
+        return IdentityAdvice(
+            recommendation=recommendation,  # type: ignore[arg-type]
+            confidence=float(parsed.get("confidence") or 0.0),
+            candidate_person_id=(
+                str(parsed["candidate_person_id"])
+                if parsed.get("candidate_person_id") is not None
+                else None
+            ),
+            reasons=_string_list(parsed.get("reasons")),
+            evidence=_string_list(parsed.get("evidence")),
+        )
+
+
+def _candidate_payload(candidate: IdentityCandidate) -> dict[str, object]:
+    return {
+        "person_id": candidate.person_id,
+        "display_name": candidate.display_name,
+        "score": candidate.score,
+        "evidence": candidate.evidence[:5],
+        "exact_identifier_match": candidate.exact_identifier_match,
+        "exact_name_match": candidate.exact_name_match,
+    }
+
+
+def _string_list(value: object) -> list[str]:
+    if not isinstance(value, list):
+        return []
+    return [str(item) for item in value if str(item).strip()]
+
+
+def _parse_json_object(text: str) -> dict[str, Any]:
+    stripped = text.strip()
+    if stripped.startswith("```"):
+        lines = stripped.splitlines()
+        if len(lines) >= 3 and lines[-1].strip() == "```":
+            stripped = "\n".join(lines[1:-1]).strip()
+    try:
+        parsed = json.loads(stripped)
+    except json.JSONDecodeError:
+        start = stripped.find("{")
+        end = stripped.rfind("}")
+        if start < 0 or end <= start:
+            raise
+        parsed = json.loads(stripped[start : end + 1])
+    if not isinstance(parsed, dict):
+        raise ValueError("LLM identity advisor response must be a JSON object")
+    return parsed