@geravant/sinain 1.10.1 → 1.12.0

package/sinain-memory/eval/benchmarks/longmemeval_adapter.py

@@ -0,0 +1,177 @@
+"""LongMemEval (ICLR 2025) adapter — download + parse into sinain format.
+
+Dataset: https://huggingface.co/datasets/xiaowu0162/longmemeval-cleaned
+Paper: https://arxiv.org/abs/2410.10813
+
+Fields per item:
+  question_id, question_type, question, answer, question_date,
+  haystack_session_ids, haystack_dates, haystack_sessions, answer_session_ids
+
+haystack_sessions entries: {"role": "user"/"assistant", "content": "...", "has_answer": bool}
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+
+from .base_adapter import BenchmarkAdapter, BenchmarkInstance, BenchmarkQuestion
+
+
+def _download_dataset(data_dir: Path) -> Path:
+    """Download LongMemEval from HuggingFace if not cached."""
+    cache_path = data_dir / "longmemeval" / "longmemeval_s_cleaned.json"
+    if cache_path.exists():
+        return cache_path
+
+    cache_path.parent.mkdir(parents=True, exist_ok=True)
+
+    url = "https://huggingface.co/datasets/xiaowu0162/longmemeval-cleaned/resolve/main/longmemeval_s_cleaned.json"
+    print(f"[longmemeval] downloading from {url} ...")
+
+    # Use curl to avoid macOS Python SSL cert issues
+    import subprocess
+    result = subprocess.run(
+        ["curl", "-fSL", "-o", str(cache_path), url],
+        capture_output=True, text=True, timeout=120,
+    )
+    if result.returncode != 0:
+        raise RuntimeError(f"Download failed: {result.stderr[:200]}")
+    print(f"[longmemeval] saved to {cache_path} ({cache_path.stat().st_size} bytes)")
+    return cache_path
+
+
+def _session_hash(sessions: list[dict]) -> str:
+    """Content hash for a haystack (for grouping questions with shared context)."""
+    raw = json.dumps(sessions, sort_keys=True, ensure_ascii=False)
+    return hashlib.sha256(raw.encode()).hexdigest()[:16]
+
+
+def _sessions_to_feed_items(
+    haystack_sessions: list[list[dict]],
+    haystack_session_ids: list[str],
+    haystack_dates: list[str],
+) -> list[list[dict]]:
+    """Convert LongMemEval haystack into sinain feed item sessions.
+
+    haystack_sessions is a list of sessions, each a list of turn dicts:
+    sessions[i][j] = {"role": "user"/"assistant", "content": "..."}
+
+    Each session becomes a list of feed items with synthesized timestamps.
+    User turns → source: "audio", assistant turns → source: "agent".
+    """
+    result: list[list[dict]] = []
+
+    for i, session_turns in enumerate(haystack_sessions):
+        if not session_turns:
+            continue
+
+        base_ts = haystack_dates[i] if i < len(haystack_dates) else "2025-01-01T10:00:00Z"
+        base_dt = _parse_date(base_ts)
+
+        items = []
+        for j, turn in enumerate(session_turns):
+            ts = (base_dt + timedelta(seconds=30 * j)).isoformat()
+            source = "audio" if turn.get("role") == "user" else "agent"
+            items.append({
+                "source": source,
+                "text": turn.get("content", ""),
+                "ts": ts,
+                "channel": "benchmark",
+            })
+        if items:
+            result.append(items)
+
+    return result
+
+
+def _parse_date(s: str) -> datetime:
+    """Best-effort date parsing."""
+    for fmt in ("%Y-%m-%dT%H:%M:%SZ", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%d", "%m/%d/%Y"):
+        try:
+            return datetime.strptime(s, fmt).replace(tzinfo=timezone.utc)
+        except (ValueError, TypeError):
+            continue
+    return datetime(2025, 1, 1, 10, 0, tzinfo=timezone.utc)
+
+
+class LongMemEvalAdapter(BenchmarkAdapter):
+    """Adapter for LongMemEval (ICLR 2025) benchmark."""
+
+    @property
+    def name(self) -> str:
+        return "longmemeval"
+
+    def load_dataset(self, data_dir: str) -> list[BenchmarkInstance]:
+        """Download and parse LongMemEval, grouping questions by shared haystack."""
+        path = _download_dataset(Path(data_dir))
+        with open(path) as f:
+            raw_items = json.load(f)
+
+        # Group questions by haystack content hash
+        groups: dict[str, dict] = {}
+        for item in raw_items:
+            h = _session_hash(item.get("haystack_sessions", []))
+            if h not in groups:
+                groups[h] = {
+                    "haystack_sessions": item["haystack_sessions"],
+                    "haystack_session_ids": item.get("haystack_session_ids", []),
+                    "haystack_dates": item.get("haystack_dates", []),
+                    "questions": [],
+                }
+            groups[h]["questions"].append(item)
+
+        instances = []
+        for h, group in groups.items():
+            feed_sessions = _sessions_to_feed_items(
+                group["haystack_sessions"],
+                group["haystack_session_ids"],
+                group["haystack_dates"],
+            )
+
+            questions = []
+            for item in group["questions"]:
+                questions.append(BenchmarkQuestion(
+                    id=item["question_id"],
+                    text=item["question"],
+                    gold_answer=str(item["answer"]),
+                    category=item.get("question_type", "unknown"),
+                    evidence_session_ids=item.get("answer_session_ids", []),
+                    metadata={
+                        "question_date": item.get("question_date", ""),
+                    },
+                ))
+
+            instances.append(BenchmarkInstance(
+                id=f"lme-{h}",
+                sessions=feed_sessions,
+                questions=questions,
+                raw_sessions=group["haystack_sessions"],
+                metadata={
+                    "haystack_hash": h,
+                    "num_sessions": len(feed_sessions),
+                    "num_turns": len(group["haystack_sessions"]),
+                },
+            ))
+
+        print(f"[longmemeval] loaded {sum(len(i.questions) for i in instances)} questions "
+              f"across {len(instances)} unique haystacks")
+        return instances
+
+    def format_full_context(self, instance: BenchmarkInstance) -> str:
+        """Render the full conversation history for the baseline condition."""
+        lines = []
+        for session in instance.raw_sessions:
+            if isinstance(session, list):
+                for turn in session:
+                    role = turn.get("role", "unknown").capitalize()
+                    content = turn.get("content", "")
+                    lines.append(f"{role}: {content}")
+                lines.append("---")  # session separator
+            elif isinstance(session, dict):
+                role = session.get("role", "unknown").capitalize()
+                content = session.get("content", "")
+                lines.append(f"{role}: {content}")
+        return "\n\n".join(lines)

package/sinain-memory/eval/benchmarks/query.py

@@ -0,0 +1,172 @@
+"""Query pipeline — benchmark questions → LLM answers under 3 conditions.
+
+Condition A (sinain-memory): answer from knowledge graph facts
+Condition B (full-context):  answer from full conversation history
+Condition C (knowledge-doc): answer from portable knowledge document
+"""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+# Add sinain-memory to path
+_koog_dir = str(Path(__file__).resolve().parent.parent.parent)
+if _koog_dir not in sys.path:
+    sys.path.insert(0, _koog_dir)
+
+from common import call_llm  # noqa: E402
+
+from .base_adapter import BenchmarkQuestion
+from .config import QA_MODEL, MAX_FACTS_PER_QUERY
+
+
+def _extract_keywords(query: str) -> list[str]:
+    """Extract search keywords (reuses logic from retrieval_evaluator)."""
+    import re
+    words = re.findall(r"[a-zA-Z][a-zA-Z0-9-]+", query.lower())
+    stopwords = {
+        "the", "is", "in", "on", "for", "and", "or", "of", "to", "a", "an",
+        "it", "was", "not", "how", "what", "when", "does", "did", "do", "my",
+        "your", "their", "have", "has", "had", "are", "were", "been", "being",
+        "about", "from", "with", "that", "this", "which", "who", "whom",
+        "where", "why", "can", "could", "would", "should",
+    }
+    return [w for w in words if len(w) > 2 and w not in stopwords]
+
+
+def _get_all_facts_text(db_path: str) -> str:
+    """Dump ALL facts from the knowledge graph as formatted text.
+
+    Sinain triplestores are small (10-50 facts per session), so including
+    everything is feasible and avoids tag-matching failures.
+    """
+    from graph_query import query_top_facts, format_facts_text
+
+    facts = query_top_facts(db_path, limit=50)
+    if not facts:
+        return "(no knowledge available)"
+    return format_facts_text(facts, max_chars=6000)
+
+
+def _query_knowledge(db_path: str, question: str) -> str:
+    """Query sinain knowledge graph for facts relevant to a question.
+
+    Strategy: first try tag-based retrieval (targeted). If nothing found,
+    fall back to full DB dump (sinain stores are small enough).
+    """
+    from graph_query import query_facts_by_entities, format_facts_text
+
+    keywords = _extract_keywords(question)
+    if keywords:
+        facts = query_facts_by_entities(db_path, keywords, max_facts=MAX_FACTS_PER_QUERY)
+        if facts:
+            return format_facts_text(facts, max_chars=3000)
+
+    # Fallback: include all facts (DB is small, typically 10-30 facts)
+    return _get_all_facts_text(db_path)
+
+
+def _get_retrieved_facts(db_path: str, question: str, k: int = 10) -> list[dict]:
+    """Get facts retrieved for a question (for retrieval evaluation)."""
+    from graph_query import query_facts_by_entities, query_top_facts
+
+    keywords = _extract_keywords(question)
+    if keywords:
+        facts = query_facts_by_entities(db_path, keywords, max_facts=k)
+        if facts:
+            return facts
+
+    # Fallback: top facts by confidence
+    return query_top_facts(db_path, limit=k)
+
+
+def compute_content_recall(
+    retrieved_facts: list[dict],
+    gold_answer: str,
+    k_values: list[int] | None = None,
+) -> dict:
+    """Content-based retrieval metric: do retrieved facts contain the answer?
+
+    Instead of matching entity IDs (which don't align between LongMemEval
+    session IDs and sinain entity IDs), we check whether the gold answer's
+    key terms appear in any retrieved fact's content.
+    """
+    from .config import K_VALUES
+    ks = k_values or K_VALUES
+
+    gold_terms = set(_extract_keywords(str(gold_answer)))
+    if not gold_terms:
+        return {f"content_recall@{k}": 0.0 for k in ks}
+
+    result = {}
+    for k in ks:
+        top_k = retrieved_facts[:k]
+        # Check if any fact in top-k contains gold answer terms
+        for fact in top_k:
+            fact_text = f"{fact.get('entity', '')} {fact.get('value', '')}".lower()
+            fact_terms = set(_extract_keywords(fact_text))
+            overlap = gold_terms & fact_terms
+            if len(overlap) >= max(1, len(gold_terms) // 2):  # ≥50% of gold terms
+                result[f"content_recall@{k}"] = 1.0
+                break
+        else:
+            result[f"content_recall@{k}"] = 0.0
+
+    return result
+
+
+def answer_question(
+    question: BenchmarkQuestion,
+    condition: str,
+    *,
+    db_path: str | None = None,
+    full_context: str | None = None,
+    knowledge_doc: str | None = None,
+    model: str | None = None,
+) -> str:
+    """Generate an answer for a benchmark question under a specific condition.
+
+    Returns the LLM's answer text.
+    """
+    qa_model = model or QA_MODEL
+
+    if condition == "sinain-memory":
+        assert db_path, "db_path required for sinain-memory condition"
+        facts = _query_knowledge(db_path, question.text)
+        system = (
+            "Answer the question using ONLY the provided knowledge facts. "
+            "If the facts don't contain enough information to answer, say \"I don't know.\""
+        )
+        user = f"## Knowledge Facts\n{facts}\n\n## Question\n{question.text}"
+
+    elif condition == "full-context":
+        assert full_context, "full_context required for full-context condition"
+        system = (
+            "Answer the question based on the conversation history below. "
+            "Be concise and specific."
+        )
+        # Truncate context if too large (some models have limits)
+        ctx = full_context[:100_000] if len(full_context) > 100_000 else full_context
+        user = f"## Conversation History\n{ctx}\n\n## Question\n{question.text}"
+
+    elif condition == "knowledge-doc":
+        assert knowledge_doc, "knowledge_doc required for knowledge-doc condition"
+        system = (
+            "Answer the question using ONLY the knowledge document provided. "
+            "If the document doesn't contain enough information, say \"I don't know.\""
+        )
+        user = f"## Knowledge Document\n{knowledge_doc}\n\n## Question\n{question.text}"
+
+    else:
+        raise ValueError(f"Unknown condition: {condition}")
+
+    try:
+        return call_llm(
+            system_prompt=system,
+            user_prompt=user,
+            model=qa_model,
+            max_tokens=300,
+        ).strip()
+    except Exception as e:
+        return f"(error: {e})"

package/sinain-memory/eval/benchmarks/report.py

@@ -0,0 +1,87 @@
+"""Report generation — markdown, JSON, and LaTeX output."""
+
+from __future__ import annotations
+
+import json
+from datetime import datetime, timezone
+
+
+def generate_markdown(benchmark_name: str, summary: dict, details: list[dict]) -> str:
+    """Generate a publishable markdown report."""
+    lines = [
+        f"# Sinain Knowledge Graph — {benchmark_name} Results",
+        f"\nGenerated: {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}",
+        "",
+    ]
+
+    # Headline IPR
+    ipr = summary.get("ipr")
+    if ipr:
+        lines.append(f"**Information Preservation Rate (IPR)**: {ipr:.1%}")
+        lines.append("")
+
+    # Condition scores table
+    conditions = summary.get("conditions", {})
+    if conditions:
+        cond_names = sorted(conditions.keys())
+        header = "| Condition | Mean Score (1-5) | Mean F1 | N |"
+        sep = "|-----------|------------------|---------|---|"
+        lines.extend([header, sep])
+        for cond in cond_names:
+            c = conditions[cond]
+            lines.append(f"| {cond} | {c['mean_score']:.2f} | {c.get('mean_f1', 0):.2f} | {c['n']} |")
+        lines.append("")
+
+    # Retrieval metrics
+    retrieval = summary.get("retrieval", {})
+    if retrieval:
+        lines.append("## Retrieval Quality")
+        lines.append("| Metric | Score |")
+        lines.append("|--------|-------|")
+        for k, v in sorted(retrieval.items()):
+            lines.append(f"| {k} | {v:.1%} |")
+        lines.append("")
+
+    # Category breakdown
+    categories = summary.get("categories", {})
+    if categories:
+        lines.append("## By Category")
+        cond_names = sorted(set(c for cat in categories.values() for c in cat))
+        header = "| Category | " + " | ".join(cond_names) + " |"
+        sep = "|----------|" + "|".join(["------"] * len(cond_names)) + "|"
+        lines.extend([header, sep])
+        for cat in sorted(categories):
+            cells = []
+            for cond in cond_names:
+                if cond in categories[cat]:
+                    cells.append(f"{categories[cat][cond]['mean_score']:.2f} (n={categories[cat][cond]['n']})")
+                else:
+                    cells.append("-")
+            lines.append(f"| {cat} | " + " | ".join(cells) + " |")
+        lines.append("")
+
+    # Failures (worst questions)
+    if details:
+        sm_details = [d for d in details if d.get("answers", {}).get("sinain-memory", {}).get("score") is not None]
+        sm_details.sort(key=lambda d: d["answers"]["sinain-memory"]["score"])
+        if sm_details:
+            lines.append("## Hardest Questions for sinain-memory (bottom 5)")
+            for d in sm_details[:5]:
+                sm = d["answers"]["sinain-memory"]
+                fc = d["answers"].get("full-context", {})
+                lines.append(f"- **{d['id']}** [{d['category']}]: score={sm['score']}/5 "
+                             f"(full-context: {fc.get('score', '?')}/5)")
+                lines.append(f"  Q: {d['question'][:100]}...")
+            lines.append("")
+
+    return "\n".join(lines)
+
+
+def generate_json(benchmark_name: str, summary: dict, details: list[dict]) -> str:
+    """Generate JSON report."""
+    return json.dumps({
+        "benchmark": benchmark_name,
+        "timestamp": datetime.now(timezone.utc).isoformat(),
+        "summary": summary,
+        "details": details,
+    }, indent=2, ensure_ascii=False)