@geravant/sinain 1.12.0 → 1.13.0

package/sinain-memory/eval/benchmarks/meeting_adapter.py

@@ -0,0 +1,81 @@
+"""Meeting Memory adapter — loads QA pairs + ground-truth transcript.
+
+Unlike LongMemEval, this adapter does NOT ingest. The real sinain pipeline
+(start-local.sh) produces the knowledge-graph.db during a live capture run.
+This adapter only loads QA pairs and provides the full-context baseline.
+
+Data layout:
+    eval/benchmarks/data/meeting/
+        <name>.txt          — ground-truth transcript
+        <name>_qa.json      — gold QA pairs
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from .base_adapter import BenchmarkAdapter, BenchmarkInstance, BenchmarkQuestion
+
+
+class MeetingMemoryAdapter(BenchmarkAdapter):
+    """Adapter for meeting memory benchmarks (real pipeline capture)."""
+
+    @property
+    def name(self) -> str:
+        return "meeting"
+
+    def load_dataset(self, data_dir: str) -> list[BenchmarkInstance]:
+        """Load meeting QA pairs and transcripts from data/meeting/."""
+        meeting_dir = Path(data_dir) / "meeting"
+        if not meeting_dir.exists():
+            raise FileNotFoundError(f"Meeting data not found: {meeting_dir}")
+
+        instances = []
+        for qa_path in sorted(meeting_dir.glob("*_qa.json")):
+            # Derive transcript path: foo_qa.json → foo.txt
+            stem = qa_path.stem.replace("_qa", "")
+            transcript_path = qa_path.parent / f"{stem}.txt"
+            if not transcript_path.exists():
+                print(f"[meeting] warning: no transcript for {qa_path.name}, skipping")
+                continue
+
+            # Load QA pairs
+            with open(qa_path) as f:
+                raw_questions = json.load(f)
+
+            questions = []
+            for item in raw_questions:
+                questions.append(BenchmarkQuestion(
+                    id=item["id"],
+                    text=item["question"],
+                    gold_answer=item["gold_answer"],
+                    category=item.get("category", "unknown"),
+                    evidence_session_ids=item.get("evidence_timestamps", []),
+                    metadata={
+                        "evidence_timestamps": item.get("evidence_timestamps", []),
+                    },
+                ))
+
+            # Load transcript
+            transcript_text = transcript_path.read_text(encoding="utf-8")
+
+            instances.append(BenchmarkInstance(
+                id=f"meeting-{stem}",
+                sessions=[],  # Not used — real pipeline does ingestion
+                questions=questions,
+                raw_sessions=[],
+                metadata={
+                    "transcript_path": str(transcript_path),
+                    "raw_transcript": transcript_text,
+                    "stem": stem,
+                },
+            ))
+
+        total_q = sum(len(i.questions) for i in instances)
+        print(f"[meeting] loaded {total_q} questions across {len(instances)} meetings")
+        return instances
+
+    def format_full_context(self, instance: BenchmarkInstance) -> str:
+        """Return the raw transcript verbatim for the full-context baseline."""
+        return instance.metadata.get("raw_transcript", "")

package/sinain-memory/eval/benchmarks/meeting_runner.py

@@ -0,0 +1,230 @@
+#!/usr/bin/env python3
+"""Meeting Memory Benchmark runner — evaluates a captured knowledge-graph.db.
+
+Standalone script: does NOT modify or import runner.py.
+Reuses shared infrastructure: query, judge, evaluate, report.
+
+Usage:
+    python3 eval/benchmarks/meeting_runner.py \
+        --db /tmp/sinain-bench-XXXX/knowledge-graph.db \
+        --conditions sinain-memory,full-context
+
+    # Quick test (3 questions):
+    python3 eval/benchmarks/meeting_runner.py \
+        --db /tmp/sinain-bench-XXXX/knowledge-graph.db \
+        --subset 3
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+# Add sinain-memory to path
+_koog_dir = str(Path(__file__).resolve().parent.parent.parent)
+if _koog_dir not in sys.path:
+    sys.path.insert(0, _koog_dir)
+
+from eval.benchmarks.config import DATA_DIR, RESULTS_DIR, QA_MODEL, JUDGE_MODEL
+from eval.benchmarks.meeting_adapter import MeetingMemoryAdapter
+from eval.benchmarks.query import answer_question, _get_retrieved_facts, compute_content_recall
+from eval.benchmarks.evaluate import token_f1, aggregate_results
+from eval.benchmarks.judges.qa_judge import judge_qa
+from eval.benchmarks.report import generate_markdown, generate_json
+
+
+def run_meeting_benchmark(
+    db_path: str,
+    conditions: list[str],
+    *,
+    subset: int | None = None,
+    meeting_filter: str | None = None,
+    qa_model: str = QA_MODEL,
+    judge_model: str = JUDGE_MODEL,
+    output_dir: Path = RESULTS_DIR,
+    data_dir: Path = DATA_DIR,
+    resume: bool = False,
+) -> tuple[dict, list[dict]]:
+    """Run meeting benchmark against a captured knowledge-graph.db."""
+
+    adapter = MeetingMemoryAdapter()
+    instances = adapter.load_dataset(str(data_dir))
+
+    if meeting_filter:
+        instances = [i for i in instances if meeting_filter in i.id]
+        print(f"[meeting] filtered to {len(instances)} meeting(s) matching '{meeting_filter}'")
+
+    if not instances:
+        print("[meeting] no meeting data found — check data/meeting/ directory")
+        return {"error": "no data"}, []
+
+    # Validate DB exists for sinain-memory condition
+    if "sinain-memory" in conditions:
+        if not db_path or not Path(db_path).exists():
+            print(f"[meeting] ERROR: --db path does not exist: {db_path}")
+            print("  Run the capture pipeline first (see plan Part 1)")
+            sys.exit(1)
+
+    print(f"\n{'='*60}")
+    print(f"  Meeting Memory Benchmark")
+    print(f"  Conditions: {', '.join(conditions)}")
+    print(f"  DB: {db_path or '(none)'}")
+    print(f"  QA model: {qa_model}")
+    print(f"  Judge model: {judge_model}")
+    print(f"{'='*60}\n")
+
+    # Flatten questions
+    all_questions = []
+    for inst in instances:
+        for q in inst.questions:
+            all_questions.append((inst, q))
+
+    if subset:
+        all_questions = all_questions[:subset]
+
+    total = len(all_questions)
+    print(f"[meeting] evaluating {total} questions\n")
+
+    # Resume support
+    resume_path = output_dir / "meeting_progress.jsonl"
+    completed: dict[str, dict] = {}
+    if resume and resume_path.exists():
+        for line in resume_path.read_text().strip().split("\n"):
+            if line:
+                entry = json.loads(line)
+                completed[entry["id"]] = entry
+
+    output_dir.mkdir(parents=True, exist_ok=True)
+    details: list[dict] = []
+
+    for idx, (inst, question) in enumerate(all_questions):
+        qid = question.id
+
+        if qid in completed:
+            details.append(completed[qid])
+            continue
+
+        print(f"[{idx+1}/{total}] {qid} [{question.category}]")
+        print(f"  Q: {question.text[:80]}...")
+
+        full_context = adapter.format_full_context(inst)
+
+        # Retrieval metrics
+        retrieval = {}
+        if db_path and "sinain-memory" in conditions:
+            retrieved_facts = _get_retrieved_facts(db_path, question.text)
+            retrieval = compute_content_recall(retrieved_facts, question.gold_answer)
+
+        # Generate answers per condition
+        answers = {}
+        for cond in conditions:
+            if cond == "sinain-memory" and not db_path:
+                answers[cond] = {"text": "(no DB)", "score": 1, "f1": 0.0}
+                continue
+
+            print(f"  [{cond}] generating answer...")
+            answer_text = answer_question(
+                question, cond,
+                db_path=db_path,
+                full_context=full_context,
+                model=qa_model,
+            )
+
+            f1 = token_f1(answer_text, question.gold_answer)
+
+            judge_result = judge_qa(
+                question.text, question.gold_answer, answer_text,
+                condition=cond, model=judge_model,
+            )
+            score = judge_result["score"] if judge_result else None
+            reasoning = judge_result["reasoning"] if judge_result else None
+
+            answers[cond] = {
+                "text": answer_text[:500],
+                "score": score,
+                "f1": round(f1, 4),
+                "reasoning": reasoning,
+            }
+            print(f"    score={score}/5 f1={f1:.2f}")
+
+        entry = {
+            "id": qid,
+            "question": question.text,
+            "gold_answer": question.gold_answer,
+            "category": question.category,
+            "retrieval": retrieval,
+            "answers": answers,
+        }
+        details.append(entry)
+
+        # Save progress
+        with open(resume_path, "a") as f:
+            f.write(json.dumps(entry, ensure_ascii=False) + "\n")
+
+    summary = aggregate_results(details)
+    return summary, details
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Meeting Memory Benchmark")
+    parser.add_argument("--db", required=False, default=None,
+                        help="Path to knowledge-graph.db from capture run")
+    parser.add_argument("--conditions", default="sinain-memory,full-context",
+                        help="Comma-separated conditions (sinain-memory, full-context)")
+    parser.add_argument("--subset", type=int, default=None,
+                        help="Run only first N questions")
+    parser.add_argument("--meeting", default=None,
+                        help="Filter to specific meeting by stem (e.g. al-futaim-prep-5min)")
+    parser.add_argument("--qa-model", default=QA_MODEL)
+    parser.add_argument("--judge-model", default=JUDGE_MODEL)
+    parser.add_argument("--output-dir", type=Path, default=RESULTS_DIR)
+    parser.add_argument("--data-dir", type=Path, default=DATA_DIR)
+    parser.add_argument("--format", default="json,markdown")
+    parser.add_argument("--resume", action="store_true")
+    args = parser.parse_args()
+
+    conditions = [c.strip() for c in args.conditions.split(",")]
+    formats = [f.strip() for f in args.format.split(",")]
+
+    summary, details = run_meeting_benchmark(
+        args.db, conditions,
+        subset=args.subset,
+        meeting_filter=args.meeting,
+        qa_model=args.qa_model,
+        judge_model=args.judge_model,
+        output_dir=args.output_dir,
+        data_dir=args.data_dir,
+        resume=args.resume,
+    )
+
+    # Write outputs
+    args.output_dir.mkdir(parents=True, exist_ok=True)
+
+    if "json" in formats:
+        json_path = args.output_dir / "meeting_results.json"
+        json_path.write_text(generate_json("meeting", summary, details))
+        print(f"\n[output] JSON: {json_path}")
+
+    if "markdown" in formats:
+        md_path = args.output_dir / "meeting_results.md"
+        md_path.write_text(generate_markdown("meeting", summary, details))
+        print(f"[output] Markdown: {md_path}")
+
+    # Print summary
+    print(f"\n{'='*60}")
+    print(f"  Meeting Memory Benchmark — Summary")
+    print(f"{'='*60}")
+    ipr = summary.get("ipr")
+    if ipr:
+        print(f"  IPR: {ipr:.1%}")
+    for cond, data in summary.get("conditions", {}).items():
+        print(f"  {cond}: {data['mean_score']:.2f}/5 (n={data['n']})")
+    for k, v in summary.get("retrieval", {}).items():
+        print(f"  {k}: {v:.1%}")
+    print()
+
+
+if __name__ == "__main__":
+    main()

package/sinain-memory/eval/benchmarks/query.py

@@ -52,30 +52,51 @@ def _get_all_facts_text(db_path: str) -> str:
 def _query_knowledge(db_path: str, question: str) -> str:
     """Query sinain knowledge graph for facts relevant to a question.
 
-    Strategy: first try tag-based retrieval (targeted). If nothing found,
-    fall back to full DB dump (sinain stores are small enough).
+    Strategy: retrieve broadly, then re-rank by keyword overlap with the question.
+    This ensures specific facts (CTO background) beat generic ones (meeting schedule)
+    when the question asks about the CTO.
     """
-    from graph_query import query_facts_by_entities, format_facts_text
+    from graph_query import query_facts_hybrid, query_top_facts, format_facts_text
 
-    keywords = _extract_keywords(question)
-    if keywords:
-        facts = query_facts_by_entities(db_path, keywords, max_facts=MAX_FACTS_PER_QUERY)
-        if facts:
-            return format_facts_text(facts, max_chars=3000)
+    # Retrieve a broad candidate set
+    candidates = query_facts_hybrid(db_path, question, max_facts=30)
+    if not candidates:
+        candidates = query_top_facts(db_path, limit=30)
+    if not candidates:
+        return "(no knowledge available)"
 
-    # Fallback: include all facts (DB is small, typically 10-30 facts)
-    return _get_all_facts_text(db_path)
+    # Re-rank by embedding similarity if available, fall back to keyword overlap
+    try:
+        from embed_client import rank_by_similarity
+        fact_texts = [str(f.get("value", "")) for f in candidates]
+        ranked_indices = rank_by_similarity(question, fact_texts)
+        if ranked_indices is not None:
+            ranked = [candidates[i] for i, _ in ranked_indices[:MAX_FACTS_PER_QUERY]]
+            return format_facts_text(ranked, max_chars=3000)
+    except Exception:
+        pass
+
+    # Fallback: keyword overlap ranking
+    q_keywords = set(_extract_keywords(question))
+    def _relevance(fact: dict) -> float:
+        value = str(fact.get("value", "")).lower()
+        entity = str(fact.get("entity", "")).lower()
+        fact_words = set(_extract_keywords(value + " " + entity))
+        if not q_keywords:
+            return 0.0
+        return len(q_keywords & fact_words) / len(q_keywords)
+
+    ranked = sorted(candidates, key=_relevance, reverse=True)
+    return format_facts_text(ranked[:MAX_FACTS_PER_QUERY], max_chars=3000)
 
 
 def _get_retrieved_facts(db_path: str, question: str, k: int = 10) -> list[dict]:
     """Get facts retrieved for a question (for retrieval evaluation)."""
-    from graph_query import query_facts_by_entities, query_top_facts
+    from graph_query import query_facts_hybrid, query_top_facts
 
-    keywords = _extract_keywords(question)
-    if keywords:
-        facts = query_facts_by_entities(db_path, keywords, max_facts=k)
-        if facts:
-            return facts
+    facts = query_facts_hybrid(db_path, question, max_facts=k)
+    if facts:
+        return facts
 
     # Fallback: top facts by confidence
     return query_top_facts(db_path, limit=k)