@geravant/sinain 1.13.0 → 1.15.0

package/sinain-memory/eval/benchmarks/evaluate.py

@@ -1,146 +0,0 @@
-"""Evaluation pipeline — score answers and compute aggregate metrics.
-
-Combines:
-- LLM-as-Judge (QA scoring, 1-5 scale)
-- Retrieval metrics (Recall@k, NDCG@k)
-- Token F1 overlap (mechanical, free)
-"""
-
-from __future__ import annotations
-
-import math
-import re
-from collections import defaultdict
-
-from .base_adapter import BenchmarkQuestion
-from .config import K_VALUES
-
-
-# ── Token F1 (mechanical, no LLM needed) ─────────────────────────────────────
-
-def _tokenize(text: str) -> list[str]:
-    """Simple whitespace + punctuation tokenizer."""
-    return re.findall(r"\w+", text.lower())
-
-
-def token_f1(predicted: str, gold: str | int) -> float:
-    """Compute token-level F1 between predicted and gold answers."""
-    pred_tokens = set(_tokenize(str(predicted)))
-    gold_tokens = set(_tokenize(str(gold)))
-    if not gold_tokens or not pred_tokens:
-        return 0.0
-    overlap = pred_tokens & gold_tokens
-    if not overlap:
-        return 0.0
-    precision = len(overlap) / len(pred_tokens)
-    recall = len(overlap) / len(gold_tokens)
-    return 2 * precision * recall / (precision + recall)
-
-
-# ── Retrieval metrics (reuse logic from retrieval_evaluator.py) ───────────────
-
-def dcg_at_k(relevant_positions: list[int], k: int) -> float:
-    """Discounted Cumulative Gain at k."""
-    score = 0.0
-    for pos in relevant_positions:
-        if pos < k:
-            score += 1.0 / math.log2(pos + 2)
-    return score
-
-
-def ndcg_at_k(relevant_positions: list[int], num_relevant: int, k: int) -> float:
-    """Normalized DCG at k."""
-    dcg = dcg_at_k(relevant_positions, k)
-    ideal_positions = list(range(min(num_relevant, k)))
-    idcg = dcg_at_k(ideal_positions, k)
-    return dcg / idcg if idcg > 0 else 0.0
-
-
-def compute_retrieval_metrics(
-    retrieved_ids: list[str],
-    expected_ids: list[str],
-    k_values: list[int] | None = None,
-) -> dict:
-    """Compute Recall@k and NDCG@k for a single question."""
-    ks = k_values or K_VALUES
-    expected_set = set(expected_ids)
-    relevant_positions = [i for i, rid in enumerate(retrieved_ids) if rid in expected_set]
-
-    result = {}
-    for k in ks:
-        hit = any(pos < k for pos in relevant_positions)
-        result[f"recall@{k}"] = 1.0 if hit else 0.0
-        result[f"ndcg@{k}"] = ndcg_at_k(relevant_positions, len(expected_set), k)
-    return result
-
-
-# ── Aggregate metrics ─────────────────────────────────────────────────────────
-
-def aggregate_results(per_question: list[dict]) -> dict:
-    """Compute aggregate metrics from per-question results.
-
-    Each per_question entry has:
-      {id, category, retrieval: {recall@k, ndcg@k}, answers: {condition: {score, f1}}}
-    """
-    if not per_question:
-        return {"error": "no results"}
-
-    # Per-condition scores
-    condition_scores: dict[str, list[float]] = defaultdict(list)
-    condition_f1s: dict[str, list[float]] = defaultdict(list)
-    # Per-category per-condition
-    cat_scores: dict[str, dict[str, list[float]]] = defaultdict(lambda: defaultdict(list))
-    # Retrieval
-    retrieval_metrics: dict[str, list[float]] = defaultdict(list)
-
-    for q in per_question:
-        cat = q.get("category", "unknown")
-
-        for cond, data in q.get("answers", {}).items():
-            if data.get("score") is not None:
-                condition_scores[cond].append(data["score"])
-                cat_scores[cat][cond].append(data["score"])
-            if data.get("f1") is not None:
-                condition_f1s[cond].append(data["f1"])
-
-        for metric, val in q.get("retrieval", {}).items():
-            if isinstance(val, (int, float)):
-                retrieval_metrics[metric].append(val)
-
-    def _mean(lst: list[float]) -> float:
-        return round(sum(lst) / len(lst), 4) if lst else 0.0
-
-    # Build summary
-    conditions = {}
-    for cond in sorted(condition_scores):
-        conditions[cond] = {
-            "mean_score": _mean(condition_scores[cond]),
-            "mean_f1": _mean(condition_f1s.get(cond, [])),
-            "n": len(condition_scores[cond]),
-        }
-
-    # IPR: sinain-memory vs full-context
-    sm_scores = condition_scores.get("sinain-memory", [])
-    fc_scores = condition_scores.get("full-context", [])
-    ipr = _mean(sm_scores) / _mean(fc_scores) if fc_scores and _mean(fc_scores) > 0 else None
-
-    # Category breakdown
-    categories = {}
-    for cat in sorted(cat_scores):
-        categories[cat] = {}
-        for cond in sorted(cat_scores[cat]):
-            categories[cat][cond] = {
-                "mean_score": _mean(cat_scores[cat][cond]),
-                "n": len(cat_scores[cat][cond]),
-            }
-
-    # Retrieval summary
-    retrieval = {k: _mean(v) for k, v in sorted(retrieval_metrics.items())}
-
-    return {
-        "total_questions": len(per_question),
-        "conditions": conditions,
-        "ipr": round(ipr, 4) if ipr else None,
-        "categories": categories,
-        "retrieval": retrieval,
-    }

package/sinain-memory/eval/benchmarks/ingest.py

@@ -1,152 +0,0 @@
-"""Ingestion pipeline — benchmark conversations → sinain triplestore.
-
-Runs session_distiller.py + knowledge_integrator.py via subprocess (exact production path).
-Caches results aggressively to avoid repeated LLM calls.
-"""
-
-from __future__ import annotations
-
-import hashlib
-import json
-import os
-import shutil
-import tempfile
-from pathlib import Path
-from subprocess import run, PIPE, TimeoutExpired
-
-from .base_adapter import BenchmarkInstance
-from .config import DISTILLER_TIMEOUT_S, INTEGRATOR_TIMEOUT_S
-
-
-def _scripts_dir() -> Path:
-    """Locate sinain-memory scripts directory."""
-    return Path(__file__).resolve().parent.parent.parent
-
-
-def _content_hash(sessions: list[list[dict]]) -> str:
-    """Hash session content for caching."""
-    raw = json.dumps(sessions, sort_keys=True, ensure_ascii=False)
-    return hashlib.sha256(raw.encode()).hexdigest()[:16]
-
-
-def _run_script(script_name: str, args: list[str], timeout: int) -> str | None:
-    """Run a Python script from sinain-memory, return stdout or None on failure."""
-    script_path = _scripts_dir() / script_name
-    if not script_path.exists():
-        print(f"[ingest] {script_name} not found at {script_path}")
-        return None
-
-    env = {**os.environ, "PYTHONPATH": str(_scripts_dir())}
-    # Ensure a working model is available (common.py defaults may reference unreleased models)
-    if "SINAIN_BENCH_MODEL" in os.environ:
-        env["SINAIN_FAST_MODEL"] = os.environ["SINAIN_BENCH_MODEL"]
-    try:
-        result = run(
-            ["python3", str(script_path)] + args,
-            capture_output=True, text=True, timeout=timeout, env=env,
-        )
-        if result.returncode != 0:
-            print(f"[ingest] {script_name} failed: {result.stderr[:200]}")
-            return None
-        return result.stdout.strip()
-    except TimeoutExpired:
-        print(f"[ingest] {script_name} timed out ({timeout}s)")
-        return None
-
-
-def ingest_instance(
-    instance: BenchmarkInstance,
-    cache_dir: Path,
-) -> Path | None:
-    """Ingest a benchmark instance into a triplestore. Returns db_path or None.
-
-    Uses caching: if the same haystack was already ingested, returns the cached DB.
-    """
-    ch = _content_hash(instance.sessions)
-    cache_path = cache_dir / "stores" / f"{ch}.db"
-
-    if cache_path.exists():
-        return cache_path
-
-    cache_path.parent.mkdir(parents=True, exist_ok=True)
-
-    # Create temp memory directory
-    tmp = tempfile.mkdtemp(prefix="sinain-bench-")
-    mem_dir = Path(tmp) / "memory"
-    for subdir in ["", "playbook-logs", "playbook-archive"]:
-        (mem_dir / subdir).mkdir(parents=True, exist_ok=True)
-
-    # Write a minimal playbook so integrator doesn't fail
-    (mem_dir / "sinain-playbook.md").write_text("# Sinain Playbook\n\n(benchmark run)\n")
-
-    success = False
-    try:
-        # Batch sessions into chunks of ~10 for fewer LLM calls.
-        # Each chunk becomes one distiller call with a combined transcript.
-        BATCH_SIZE = 10
-        num_sessions = len(instance.sessions)
-        batch_idx = 0
-
-        for start in range(0, num_sessions, BATCH_SIZE):
-            batch = instance.sessions[start:start + BATCH_SIZE]
-            # Flatten batch into one transcript
-            combined: list[dict] = []
-            for session in batch:
-                combined.extend(session)
-            if len(combined) < 3:
-                continue
-
-            first_ts = combined[0].get("ts", "2025-01-01T10:00:00Z")
-            meta = json.dumps({
-                "ts": first_ts,
-                "sessionKey": f"benchmark-batch-{batch_idx}",
-                "durationMs": len(combined) * 30000,
-            })
-            batch_idx += 1
-
-            # Step 1: Distill the batch
-            digest_json = _run_script("session_distiller.py", [
-                "--memory-dir", str(mem_dir),
-                "--transcript", json.dumps(combined),
-                "--session-meta", meta,
-            ], DISTILLER_TIMEOUT_S)
-
-            if not digest_json:
-                continue
-
-            try:
-                digest = json.loads(digest_json)
-            except json.JSONDecodeError:
-                continue
-
-            if digest.get("isEmpty") or digest.get("error"):
-                continue
-
-            # Step 2: Integrate into knowledge graph
-            _run_script("knowledge_integrator.py", [
-                "--memory-dir", str(mem_dir),
-                "--digest", json.dumps(digest),
-            ], INTEGRATOR_TIMEOUT_S)
-
-        # Copy the resulting DB to cache
-        db_path = mem_dir / "knowledge-graph.db"
-        if db_path.exists() and db_path.stat().st_size > 0:
-            shutil.copy2(db_path, cache_path)
-            success = True
-
-    finally:
-        shutil.rmtree(tmp, ignore_errors=True)
-
-    return cache_path if success else None
-
-
-def get_knowledge_doc(db_path: Path) -> str:
-    """Render a sinain-knowledge.md style document from a triplestore."""
-    import sys
-    sys.path.insert(0, str(_scripts_dir()))
-    from graph_query import query_top_facts, format_facts_text
-
-    facts = query_top_facts(str(db_path), limit=30)
-    if not facts:
-        return "(no knowledge available)"
-    return format_facts_text(facts, max_chars=6000)

package/sinain-memory/eval/benchmarks/judges/qa_judge.py

@@ -1,81 +0,0 @@
-"""LLM-as-Judge: QA answer quality evaluator (LongMemEval-compatible, 1-5 scale).
-
-Uses GPT-4o via OpenRouter for comparability with published results.
-"""
-
-from __future__ import annotations
-
-import sys
-from pathlib import Path
-
-# Add sinain-memory to path for common imports
-_koog_dir = str(Path(__file__).resolve().parent.parent.parent.parent)
-if _koog_dir not in sys.path:
-    sys.path.insert(0, _koog_dir)
-
-from common import LLMError, call_llm, extract_json  # noqa: E402
-
-SYSTEM_PROMPT = """\
-You are evaluating whether a predicted answer correctly answers a question.
-The gold (reference) answer is provided.
-
-Score on a scale of 1-5:
-  5: Perfect — captures all key information from the gold answer, no errors
-  4: Mostly correct — minor omissions or imprecision, main point is right
-  3: Partially correct — captures some key points but misses important details
-  2: Related but mostly wrong — touches the topic but answer is largely incorrect
-  1: Completely wrong, contradicts the gold answer, or says "I don't know" when the answer exists
-
-Special cases:
-- If the gold answer indicates abstention is correct (e.g. "I don't know" or "not mentioned"),
-  then a predicted "I don't know" scores 5.
-- Numeric answers within 10% of gold = full credit.
-- Getting the gist right but missing specifics = 3-4 depending on importance.
-
-Respond with ONLY a JSON object: {"score": <1-5>, "reasoning": "brief explanation"}"""
-
-
-def judge_qa(
-    question: str,
-    gold_answer: str,
-    predicted_answer: str,
-    *,
-    condition: str = "",
-    model: str | None = None,
-) -> dict | None:
-    """Score a QA answer. Returns {"score": 1-5, "reasoning": str} or None on failure."""
-    user_parts = [
-        f"## Question\n{question}",
-        f"\n## Gold Answer\n{gold_answer}",
-        f"\n## Predicted Answer\n{predicted_answer}",
-    ]
-    if condition:
-        user_parts.append(f"\n## Context Condition: {condition}")
-
-    try:
-        kwargs: dict = {
-            "system_prompt": SYSTEM_PROMPT,
-            "user_prompt": "\n".join(user_parts),
-            "max_tokens": 200,
-            "json_mode": True,
-        }
-        if model:
-            kwargs["model"] = model
-        else:
-            kwargs["script"] = "meeting_benchmark"
-
-        raw = call_llm(**kwargs)
-        result = extract_json(raw)
-
-        score = result.get("score")
-        reasoning = result.get("reasoning", "")
-
-        if not isinstance(score, (int, float)) or not (1 <= score <= 5):
-            print(f"[warn] qa_judge returned invalid score: {score}", file=sys.stderr)
-            return None
-
-        return {"score": int(score), "reasoning": str(reasoning)[:300]}
-
-    except (ValueError, LLMError, KeyError) as e:
-        print(f"[warn] qa_judge call failed: {e}", file=sys.stderr)
-        return None

package/sinain-memory/eval/benchmarks/longmemeval_adapter.py

@@ -1,177 +0,0 @@
-"""LongMemEval (ICLR 2025) adapter — download + parse into sinain format.
-
-Dataset: https://huggingface.co/datasets/xiaowu0162/longmemeval-cleaned
-Paper: https://arxiv.org/abs/2410.10813
-
-Fields per item:
-  question_id, question_type, question, answer, question_date,
-  haystack_session_ids, haystack_dates, haystack_sessions, answer_session_ids
-
-haystack_sessions entries: {"role": "user"/"assistant", "content": "...", "has_answer": bool}
-"""
-
-from __future__ import annotations
-
-import hashlib
-import json
-from datetime import datetime, timedelta, timezone
-from pathlib import Path
-
-from .base_adapter import BenchmarkAdapter, BenchmarkInstance, BenchmarkQuestion
-
-
-def _download_dataset(data_dir: Path) -> Path:
-    """Download LongMemEval from HuggingFace if not cached."""
-    cache_path = data_dir / "longmemeval" / "longmemeval_s_cleaned.json"
-    if cache_path.exists():
-        return cache_path
-
-    cache_path.parent.mkdir(parents=True, exist_ok=True)
-
-    url = "https://huggingface.co/datasets/xiaowu0162/longmemeval-cleaned/resolve/main/longmemeval_s_cleaned.json"
-    print(f"[longmemeval] downloading from {url} ...")
-
-    # Use curl to avoid macOS Python SSL cert issues
-    import subprocess
-    result = subprocess.run(
-        ["curl", "-fSL", "-o", str(cache_path), url],
-        capture_output=True, text=True, timeout=120,
-    )
-    if result.returncode != 0:
-        raise RuntimeError(f"Download failed: {result.stderr[:200]}")
-    print(f"[longmemeval] saved to {cache_path} ({cache_path.stat().st_size} bytes)")
-    return cache_path
-
-
-def _session_hash(sessions: list[dict]) -> str:
-    """Content hash for a haystack (for grouping questions with shared context)."""
-    raw = json.dumps(sessions, sort_keys=True, ensure_ascii=False)
-    return hashlib.sha256(raw.encode()).hexdigest()[:16]
-
-
-def _sessions_to_feed_items(
-    haystack_sessions: list[list[dict]],
-    haystack_session_ids: list[str],
-    haystack_dates: list[str],
-) -> list[list[dict]]:
-    """Convert LongMemEval haystack into sinain feed item sessions.
-
-    haystack_sessions is a list of sessions, each a list of turn dicts:
-    sessions[i][j] = {"role": "user"/"assistant", "content": "..."}
-
-    Each session becomes a list of feed items with synthesized timestamps.
-    User turns → source: "audio", assistant turns → source: "agent".
-    """
-    result: list[list[dict]] = []
-
-    for i, session_turns in enumerate(haystack_sessions):
-        if not session_turns:
-            continue
-
-        base_ts = haystack_dates[i] if i < len(haystack_dates) else "2025-01-01T10:00:00Z"
-        base_dt = _parse_date(base_ts)
-
-        items = []
-        for j, turn in enumerate(session_turns):
-            ts = (base_dt + timedelta(seconds=30 * j)).isoformat()
-            source = "audio" if turn.get("role") == "user" else "agent"
-            items.append({
-                "source": source,
-                "text": turn.get("content", ""),
-                "ts": ts,
-                "channel": "benchmark",
-            })
-        if items:
-            result.append(items)
-
-    return result
-
-
-def _parse_date(s: str) -> datetime:
-    """Best-effort date parsing."""
-    for fmt in ("%Y-%m-%dT%H:%M:%SZ", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%d", "%m/%d/%Y"):
-        try:
-            return datetime.strptime(s, fmt).replace(tzinfo=timezone.utc)
-        except (ValueError, TypeError):
-            continue
-    return datetime(2025, 1, 1, 10, 0, tzinfo=timezone.utc)
-
-
-class LongMemEvalAdapter(BenchmarkAdapter):
-    """Adapter for LongMemEval (ICLR 2025) benchmark."""
-
-    @property
-    def name(self) -> str:
-        return "longmemeval"
-
-    def load_dataset(self, data_dir: str) -> list[BenchmarkInstance]:
-        """Download and parse LongMemEval, grouping questions by shared haystack."""
-        path = _download_dataset(Path(data_dir))
-        with open(path) as f:
-            raw_items = json.load(f)
-
-        # Group questions by haystack content hash
-        groups: dict[str, dict] = {}
-        for item in raw_items:
-            h = _session_hash(item.get("haystack_sessions", []))
-            if h not in groups:
-                groups[h] = {
-                    "haystack_sessions": item["haystack_sessions"],
-                    "haystack_session_ids": item.get("haystack_session_ids", []),
-                    "haystack_dates": item.get("haystack_dates", []),
-                    "questions": [],
-                }
-            groups[h]["questions"].append(item)
-
-        instances = []
-        for h, group in groups.items():
-            feed_sessions = _sessions_to_feed_items(
-                group["haystack_sessions"],
-                group["haystack_session_ids"],
-                group["haystack_dates"],
-            )
-
-            questions = []
-            for item in group["questions"]:
-                questions.append(BenchmarkQuestion(
-                    id=item["question_id"],
-                    text=item["question"],
-                    gold_answer=str(item["answer"]),
-                    category=item.get("question_type", "unknown"),
-                    evidence_session_ids=item.get("answer_session_ids", []),
-                    metadata={
-                        "question_date": item.get("question_date", ""),
-                    },
-                ))
-
-            instances.append(BenchmarkInstance(
-                id=f"lme-{h}",
-                sessions=feed_sessions,
-                questions=questions,
-                raw_sessions=group["haystack_sessions"],
-                metadata={
-                    "haystack_hash": h,
-                    "num_sessions": len(feed_sessions),
-                    "num_turns": len(group["haystack_sessions"]),
-                },
-            ))
-
-        print(f"[longmemeval] loaded {sum(len(i.questions) for i in instances)} questions "
-              f"across {len(instances)} unique haystacks")
-        return instances
-
-    def format_full_context(self, instance: BenchmarkInstance) -> str:
-        """Render the full conversation history for the baseline condition."""
-        lines = []
-        for session in instance.raw_sessions:
-            if isinstance(session, list):
-                for turn in session:
-                    role = turn.get("role", "unknown").capitalize()
-                    content = turn.get("content", "")
-                    lines.append(f"{role}: {content}")
-                lines.append("---")  # session separator
-            elif isinstance(session, dict):
-                role = session.get("role", "unknown").capitalize()
-                content = session.get("content", "")
-                lines.append(f"{role}: {content}")
-        return "\n\n".join(lines)

package/sinain-memory/eval/benchmarks/meeting_adapter.py

@@ -1,81 +0,0 @@
-"""Meeting Memory adapter — loads QA pairs + ground-truth transcript.
-
-Unlike LongMemEval, this adapter does NOT ingest. The real sinain pipeline
-(start-local.sh) produces the knowledge-graph.db during a live capture run.
-This adapter only loads QA pairs and provides the full-context baseline.
-
-Data layout:
-    eval/benchmarks/data/meeting/
-        <name>.txt          — ground-truth transcript
-        <name>_qa.json      — gold QA pairs
-"""
-
-from __future__ import annotations
-
-import json
-from pathlib import Path
-
-from .base_adapter import BenchmarkAdapter, BenchmarkInstance, BenchmarkQuestion
-
-
-class MeetingMemoryAdapter(BenchmarkAdapter):
-    """Adapter for meeting memory benchmarks (real pipeline capture)."""
-
-    @property
-    def name(self) -> str:
-        return "meeting"
-
-    def load_dataset(self, data_dir: str) -> list[BenchmarkInstance]:
-        """Load meeting QA pairs and transcripts from data/meeting/."""
-        meeting_dir = Path(data_dir) / "meeting"
-        if not meeting_dir.exists():
-            raise FileNotFoundError(f"Meeting data not found: {meeting_dir}")
-
-        instances = []
-        for qa_path in sorted(meeting_dir.glob("*_qa.json")):
-            # Derive transcript path: foo_qa.json → foo.txt
-            stem = qa_path.stem.replace("_qa", "")
-            transcript_path = qa_path.parent / f"{stem}.txt"
-            if not transcript_path.exists():
-                print(f"[meeting] warning: no transcript for {qa_path.name}, skipping")
-                continue
-
-            # Load QA pairs
-            with open(qa_path) as f:
-                raw_questions = json.load(f)
-
-            questions = []
-            for item in raw_questions:
-                questions.append(BenchmarkQuestion(
-                    id=item["id"],
-                    text=item["question"],
-                    gold_answer=item["gold_answer"],
-                    category=item.get("category", "unknown"),
-                    evidence_session_ids=item.get("evidence_timestamps", []),
-                    metadata={
-                        "evidence_timestamps": item.get("evidence_timestamps", []),
-                    },
-                ))
-
-            # Load transcript
-            transcript_text = transcript_path.read_text(encoding="utf-8")
-
-            instances.append(BenchmarkInstance(
-                id=f"meeting-{stem}",
-                sessions=[],  # Not used — real pipeline does ingestion
-                questions=questions,
-                raw_sessions=[],
-                metadata={
-                    "transcript_path": str(transcript_path),
-                    "raw_transcript": transcript_text,
-                    "stem": stem,
-                },
-            ))
-
-        total_q = sum(len(i.questions) for i in instances)
-        print(f"[meeting] loaded {total_q} questions across {len(instances)} meetings")
-        return instances
-
-    def format_full_context(self, instance: BenchmarkInstance) -> str:
-        """Return the raw transcript verbatim for the full-context baseline."""
-        return instance.metadata.get("raw_transcript", "")