@geravant/sinain 1.11.0 → 1.13.0

package/sinain-memory/eval/benchmarks/ingest.py

@@ -0,0 +1,152 @@
+"""Ingestion pipeline — benchmark conversations → sinain triplestore.
+
+Runs session_distiller.py + knowledge_integrator.py via subprocess (exact production path).
+Caches results aggressively to avoid repeated LLM calls.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import shutil
+import tempfile
+from pathlib import Path
+from subprocess import run, PIPE, TimeoutExpired
+
+from .base_adapter import BenchmarkInstance
+from .config import DISTILLER_TIMEOUT_S, INTEGRATOR_TIMEOUT_S
+
+
+def _scripts_dir() -> Path:
+    """Locate sinain-memory scripts directory."""
+    return Path(__file__).resolve().parent.parent.parent
+
+
+def _content_hash(sessions: list[list[dict]]) -> str:
+    """Hash session content for caching."""
+    raw = json.dumps(sessions, sort_keys=True, ensure_ascii=False)
+    return hashlib.sha256(raw.encode()).hexdigest()[:16]
+
+
+def _run_script(script_name: str, args: list[str], timeout: int) -> str | None:
+    """Run a Python script from sinain-memory, return stdout or None on failure."""
+    script_path = _scripts_dir() / script_name
+    if not script_path.exists():
+        print(f"[ingest] {script_name} not found at {script_path}")
+        return None
+
+    env = {**os.environ, "PYTHONPATH": str(_scripts_dir())}
+    # Ensure a working model is available (common.py defaults may reference unreleased models)
+    if "SINAIN_BENCH_MODEL" in os.environ:
+        env["SINAIN_FAST_MODEL"] = os.environ["SINAIN_BENCH_MODEL"]
+    try:
+        result = run(
+            ["python3", str(script_path)] + args,
+            capture_output=True, text=True, timeout=timeout, env=env,
+        )
+        if result.returncode != 0:
+            print(f"[ingest] {script_name} failed: {result.stderr[:200]}")
+            return None
+        return result.stdout.strip()
+    except TimeoutExpired:
+        print(f"[ingest] {script_name} timed out ({timeout}s)")
+        return None
+
+
+def ingest_instance(
+    instance: BenchmarkInstance,
+    cache_dir: Path,
+) -> Path | None:
+    """Ingest a benchmark instance into a triplestore. Returns db_path or None.
+
+    Uses caching: if the same haystack was already ingested, returns the cached DB.
+    """
+    ch = _content_hash(instance.sessions)
+    cache_path = cache_dir / "stores" / f"{ch}.db"
+
+    if cache_path.exists():
+        return cache_path
+
+    cache_path.parent.mkdir(parents=True, exist_ok=True)
+
+    # Create temp memory directory
+    tmp = tempfile.mkdtemp(prefix="sinain-bench-")
+    mem_dir = Path(tmp) / "memory"
+    for subdir in ["", "playbook-logs", "playbook-archive"]:
+        (mem_dir / subdir).mkdir(parents=True, exist_ok=True)
+
+    # Write a minimal playbook so integrator doesn't fail
+    (mem_dir / "sinain-playbook.md").write_text("# Sinain Playbook\n\n(benchmark run)\n")
+
+    success = False
+    try:
+        # Batch sessions into chunks of ~10 for fewer LLM calls.
+        # Each chunk becomes one distiller call with a combined transcript.
+        BATCH_SIZE = 10
+        num_sessions = len(instance.sessions)
+        batch_idx = 0
+
+        for start in range(0, num_sessions, BATCH_SIZE):
+            batch = instance.sessions[start:start + BATCH_SIZE]
+            # Flatten batch into one transcript
+            combined: list[dict] = []
+            for session in batch:
+                combined.extend(session)
+            if len(combined) < 3:
+                continue
+
+            first_ts = combined[0].get("ts", "2025-01-01T10:00:00Z")
+            meta = json.dumps({
+                "ts": first_ts,
+                "sessionKey": f"benchmark-batch-{batch_idx}",
+                "durationMs": len(combined) * 30000,
+            })
+            batch_idx += 1
+
+            # Step 1: Distill the batch
+            digest_json = _run_script("session_distiller.py", [
+                "--memory-dir", str(mem_dir),
+                "--transcript", json.dumps(combined),
+                "--session-meta", meta,
+            ], DISTILLER_TIMEOUT_S)
+
+            if not digest_json:
+                continue
+
+            try:
+                digest = json.loads(digest_json)
+            except json.JSONDecodeError:
+                continue
+
+            if digest.get("isEmpty") or digest.get("error"):
+                continue
+
+            # Step 2: Integrate into knowledge graph
+            _run_script("knowledge_integrator.py", [
+                "--memory-dir", str(mem_dir),
+                "--digest", json.dumps(digest),
+            ], INTEGRATOR_TIMEOUT_S)
+
+        # Copy the resulting DB to cache
+        db_path = mem_dir / "knowledge-graph.db"
+        if db_path.exists() and db_path.stat().st_size > 0:
+            shutil.copy2(db_path, cache_path)
+            success = True
+
+    finally:
+        shutil.rmtree(tmp, ignore_errors=True)
+
+    return cache_path if success else None
+
+
+def get_knowledge_doc(db_path: Path) -> str:
+    """Render a sinain-knowledge.md style document from a triplestore."""
+    import sys
+    sys.path.insert(0, str(_scripts_dir()))
+    from graph_query import query_top_facts, format_facts_text
+
+    facts = query_top_facts(str(db_path), limit=30)
+    if not facts:
+        return "(no knowledge available)"
+    return format_facts_text(facts, max_chars=6000)

package/sinain-memory/eval/benchmarks/judges/qa_judge.py

@@ -0,0 +1,81 @@
+"""LLM-as-Judge: QA answer quality evaluator (LongMemEval-compatible, 1-5 scale).
+
+Uses GPT-4o via OpenRouter for comparability with published results.
+"""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+# Add sinain-memory to path for common imports
+_koog_dir = str(Path(__file__).resolve().parent.parent.parent.parent)
+if _koog_dir not in sys.path:
+    sys.path.insert(0, _koog_dir)
+
+from common import LLMError, call_llm, extract_json  # noqa: E402
+
+SYSTEM_PROMPT = """\
+You are evaluating whether a predicted answer correctly answers a question.
+The gold (reference) answer is provided.
+
+Score on a scale of 1-5:
+  5: Perfect — captures all key information from the gold answer, no errors
+  4: Mostly correct — minor omissions or imprecision, main point is right
+  3: Partially correct — captures some key points but misses important details
+  2: Related but mostly wrong — touches the topic but answer is largely incorrect
+  1: Completely wrong, contradicts the gold answer, or says "I don't know" when the answer exists
+
+Special cases:
+- If the gold answer indicates abstention is correct (e.g. "I don't know" or "not mentioned"),
+  then a predicted "I don't know" scores 5.
+- Numeric answers within 10% of gold = full credit.
+- Getting the gist right but missing specifics = 3-4 depending on importance.
+
+Respond with ONLY a JSON object: {"score": <1-5>, "reasoning": "brief explanation"}"""
+
+
+def judge_qa(
+    question: str,
+    gold_answer: str,
+    predicted_answer: str,
+    *,
+    condition: str = "",
+    model: str | None = None,
+) -> dict | None:
+    """Score a QA answer. Returns {"score": 1-5, "reasoning": str} or None on failure."""
+    user_parts = [
+        f"## Question\n{question}",
+        f"\n## Gold Answer\n{gold_answer}",
+        f"\n## Predicted Answer\n{predicted_answer}",
+    ]
+    if condition:
+        user_parts.append(f"\n## Context Condition: {condition}")
+
+    try:
+        kwargs: dict = {
+            "system_prompt": SYSTEM_PROMPT,
+            "user_prompt": "\n".join(user_parts),
+            "max_tokens": 200,
+            "json_mode": True,
+        }
+        if model:
+            kwargs["model"] = model
+        else:
+            kwargs["script"] = "meeting_benchmark"
+
+        raw = call_llm(**kwargs)
+        result = extract_json(raw)
+
+        score = result.get("score")
+        reasoning = result.get("reasoning", "")
+
+        if not isinstance(score, (int, float)) or not (1 <= score <= 5):
+            print(f"[warn] qa_judge returned invalid score: {score}", file=sys.stderr)
+            return None
+
+        return {"score": int(score), "reasoning": str(reasoning)[:300]}
+
+    except (ValueError, LLMError, KeyError) as e:
+        print(f"[warn] qa_judge call failed: {e}", file=sys.stderr)
+        return None

package/sinain-memory/eval/benchmarks/longmemeval_adapter.py

@@ -0,0 +1,177 @@
+"""LongMemEval (ICLR 2025) adapter — download + parse into sinain format.
+
+Dataset: https://huggingface.co/datasets/xiaowu0162/longmemeval-cleaned
+Paper: https://arxiv.org/abs/2410.10813
+
+Fields per item:
+  question_id, question_type, question, answer, question_date,
+  haystack_session_ids, haystack_dates, haystack_sessions, answer_session_ids
+
+haystack_sessions entries: {"role": "user"/"assistant", "content": "...", "has_answer": bool}
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+
+from .base_adapter import BenchmarkAdapter, BenchmarkInstance, BenchmarkQuestion
+
+
+def _download_dataset(data_dir: Path) -> Path:
+    """Download LongMemEval from HuggingFace if not cached."""
+    cache_path = data_dir / "longmemeval" / "longmemeval_s_cleaned.json"
+    if cache_path.exists():
+        return cache_path
+
+    cache_path.parent.mkdir(parents=True, exist_ok=True)
+
+    url = "https://huggingface.co/datasets/xiaowu0162/longmemeval-cleaned/resolve/main/longmemeval_s_cleaned.json"
+    print(f"[longmemeval] downloading from {url} ...")
+
+    # Use curl to avoid macOS Python SSL cert issues
+    import subprocess
+    result = subprocess.run(
+        ["curl", "-fSL", "-o", str(cache_path), url],
+        capture_output=True, text=True, timeout=120,
+    )
+    if result.returncode != 0:
+        raise RuntimeError(f"Download failed: {result.stderr[:200]}")
+    print(f"[longmemeval] saved to {cache_path} ({cache_path.stat().st_size} bytes)")
+    return cache_path
+
+
+def _session_hash(sessions: list[dict]) -> str:
+    """Content hash for a haystack (for grouping questions with shared context)."""
+    raw = json.dumps(sessions, sort_keys=True, ensure_ascii=False)
+    return hashlib.sha256(raw.encode()).hexdigest()[:16]
+
+
+def _sessions_to_feed_items(
+    haystack_sessions: list[list[dict]],
+    haystack_session_ids: list[str],
+    haystack_dates: list[str],
+) -> list[list[dict]]:
+    """Convert LongMemEval haystack into sinain feed item sessions.
+
+    haystack_sessions is a list of sessions, each a list of turn dicts:
+    sessions[i][j] = {"role": "user"/"assistant", "content": "..."}
+
+    Each session becomes a list of feed items with synthesized timestamps.
+    User turns → source: "audio", assistant turns → source: "agent".
+    """
+    result: list[list[dict]] = []
+
+    for i, session_turns in enumerate(haystack_sessions):
+        if not session_turns:
+            continue
+
+        base_ts = haystack_dates[i] if i < len(haystack_dates) else "2025-01-01T10:00:00Z"
+        base_dt = _parse_date(base_ts)
+
+        items = []
+        for j, turn in enumerate(session_turns):
+            ts = (base_dt + timedelta(seconds=30 * j)).isoformat()
+            source = "audio" if turn.get("role") == "user" else "agent"
+            items.append({
+                "source": source,
+                "text": turn.get("content", ""),
+                "ts": ts,
+                "channel": "benchmark",
+            })
+        if items:
+            result.append(items)
+
+    return result
+
+
+def _parse_date(s: str) -> datetime:
+    """Best-effort date parsing."""
+    for fmt in ("%Y-%m-%dT%H:%M:%SZ", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%d", "%m/%d/%Y"):
+        try:
+            return datetime.strptime(s, fmt).replace(tzinfo=timezone.utc)
+        except (ValueError, TypeError):
+            continue
+    return datetime(2025, 1, 1, 10, 0, tzinfo=timezone.utc)
+
+
+class LongMemEvalAdapter(BenchmarkAdapter):
+    """Adapter for LongMemEval (ICLR 2025) benchmark."""
+
+    @property
+    def name(self) -> str:
+        return "longmemeval"
+
+    def load_dataset(self, data_dir: str) -> list[BenchmarkInstance]:
+        """Download and parse LongMemEval, grouping questions by shared haystack."""
+        path = _download_dataset(Path(data_dir))
+        with open(path) as f:
+            raw_items = json.load(f)
+
+        # Group questions by haystack content hash
+        groups: dict[str, dict] = {}
+        for item in raw_items:
+            h = _session_hash(item.get("haystack_sessions", []))
+            if h not in groups:
+                groups[h] = {
+                    "haystack_sessions": item["haystack_sessions"],
+                    "haystack_session_ids": item.get("haystack_session_ids", []),
+                    "haystack_dates": item.get("haystack_dates", []),
+                    "questions": [],
+                }
+            groups[h]["questions"].append(item)
+
+        instances = []
+        for h, group in groups.items():
+            feed_sessions = _sessions_to_feed_items(
+                group["haystack_sessions"],
+                group["haystack_session_ids"],
+                group["haystack_dates"],
+            )
+
+            questions = []
+            for item in group["questions"]:
+                questions.append(BenchmarkQuestion(
+                    id=item["question_id"],
+                    text=item["question"],
+                    gold_answer=str(item["answer"]),
+                    category=item.get("question_type", "unknown"),
+                    evidence_session_ids=item.get("answer_session_ids", []),
+                    metadata={
+                        "question_date": item.get("question_date", ""),
+                    },
+                ))
+
+            instances.append(BenchmarkInstance(
+                id=f"lme-{h}",
+                sessions=feed_sessions,
+                questions=questions,
+                raw_sessions=group["haystack_sessions"],
+                metadata={
+                    "haystack_hash": h,
+                    "num_sessions": len(feed_sessions),
+                    "num_turns": len(group["haystack_sessions"]),
+                },
+            ))
+
+        print(f"[longmemeval] loaded {sum(len(i.questions) for i in instances)} questions "
+              f"across {len(instances)} unique haystacks")
+        return instances
+
+    def format_full_context(self, instance: BenchmarkInstance) -> str:
+        """Render the full conversation history for the baseline condition."""
+        lines = []
+        for session in instance.raw_sessions:
+            if isinstance(session, list):
+                for turn in session:
+                    role = turn.get("role", "unknown").capitalize()
+                    content = turn.get("content", "")
+                    lines.append(f"{role}: {content}")
+                lines.append("---")  # session separator
+            elif isinstance(session, dict):
+                role = session.get("role", "unknown").capitalize()
+                content = session.get("content", "")
+                lines.append(f"{role}: {content}")
+        return "\n\n".join(lines)

package/sinain-memory/eval/benchmarks/meeting_adapter.py

@@ -0,0 +1,81 @@
+"""Meeting Memory adapter — loads QA pairs + ground-truth transcript.
+
+Unlike LongMemEval, this adapter does NOT ingest. The real sinain pipeline
+(start-local.sh) produces the knowledge-graph.db during a live capture run.
+This adapter only loads QA pairs and provides the full-context baseline.
+
+Data layout:
+    eval/benchmarks/data/meeting/
+        <name>.txt          — ground-truth transcript
+        <name>_qa.json      — gold QA pairs
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from .base_adapter import BenchmarkAdapter, BenchmarkInstance, BenchmarkQuestion
+
+
+class MeetingMemoryAdapter(BenchmarkAdapter):
+    """Adapter for meeting memory benchmarks (real pipeline capture)."""
+
+    @property
+    def name(self) -> str:
+        return "meeting"
+
+    def load_dataset(self, data_dir: str) -> list[BenchmarkInstance]:
+        """Load meeting QA pairs and transcripts from data/meeting/."""
+        meeting_dir = Path(data_dir) / "meeting"
+        if not meeting_dir.exists():
+            raise FileNotFoundError(f"Meeting data not found: {meeting_dir}")
+
+        instances = []
+        for qa_path in sorted(meeting_dir.glob("*_qa.json")):
+            # Derive transcript path: foo_qa.json → foo.txt
+            stem = qa_path.stem.replace("_qa", "")
+            transcript_path = qa_path.parent / f"{stem}.txt"
+            if not transcript_path.exists():
+                print(f"[meeting] warning: no transcript for {qa_path.name}, skipping")
+                continue
+
+            # Load QA pairs
+            with open(qa_path) as f:
+                raw_questions = json.load(f)
+
+            questions = []
+            for item in raw_questions:
+                questions.append(BenchmarkQuestion(
+                    id=item["id"],
+                    text=item["question"],
+                    gold_answer=item["gold_answer"],
+                    category=item.get("category", "unknown"),
+                    evidence_session_ids=item.get("evidence_timestamps", []),
+                    metadata={
+                        "evidence_timestamps": item.get("evidence_timestamps", []),
+                    },
+                ))
+
+            # Load transcript
+            transcript_text = transcript_path.read_text(encoding="utf-8")
+
+            instances.append(BenchmarkInstance(
+                id=f"meeting-{stem}",
+                sessions=[],  # Not used — real pipeline does ingestion
+                questions=questions,
+                raw_sessions=[],
+                metadata={
+                    "transcript_path": str(transcript_path),
+                    "raw_transcript": transcript_text,
+                    "stem": stem,
+                },
+            ))
+
+        total_q = sum(len(i.questions) for i in instances)
+        print(f"[meeting] loaded {total_q} questions across {len(instances)} meetings")
+        return instances
+
+    def format_full_context(self, instance: BenchmarkInstance) -> str:
+        """Return the raw transcript verbatim for the full-context baseline."""
+        return instance.metadata.get("raw_transcript", "")