@geravant/sinain 1.12.0 → 1.14.0

package/sinain-memory/eval/benchmarks/query.py

@@ -1,172 +0,0 @@
-"""Query pipeline — benchmark questions → LLM answers under 3 conditions.
-
-Condition A (sinain-memory): answer from knowledge graph facts
-Condition B (full-context):  answer from full conversation history
-Condition C (knowledge-doc): answer from portable knowledge document
-"""
-
-from __future__ import annotations
-
-import sys
-from pathlib import Path
-
-# Add sinain-memory to path
-_koog_dir = str(Path(__file__).resolve().parent.parent.parent)
-if _koog_dir not in sys.path:
-    sys.path.insert(0, _koog_dir)
-
-from common import call_llm  # noqa: E402
-
-from .base_adapter import BenchmarkQuestion
-from .config import QA_MODEL, MAX_FACTS_PER_QUERY
-
-
-def _extract_keywords(query: str) -> list[str]:
-    """Extract search keywords (reuses logic from retrieval_evaluator)."""
-    import re
-    words = re.findall(r"[a-zA-Z][a-zA-Z0-9-]+", query.lower())
-    stopwords = {
-        "the", "is", "in", "on", "for", "and", "or", "of", "to", "a", "an",
-        "it", "was", "not", "how", "what", "when", "does", "did", "do", "my",
-        "your", "their", "have", "has", "had", "are", "were", "been", "being",
-        "about", "from", "with", "that", "this", "which", "who", "whom",
-        "where", "why", "can", "could", "would", "should",
-    }
-    return [w for w in words if len(w) > 2 and w not in stopwords]
-
-
-def _get_all_facts_text(db_path: str) -> str:
-    """Dump ALL facts from the knowledge graph as formatted text.
-
-    Sinain triplestores are small (10-50 facts per session), so including
-    everything is feasible and avoids tag-matching failures.
-    """
-    from graph_query import query_top_facts, format_facts_text
-
-    facts = query_top_facts(db_path, limit=50)
-    if not facts:
-        return "(no knowledge available)"
-    return format_facts_text(facts, max_chars=6000)
-
-
-def _query_knowledge(db_path: str, question: str) -> str:
-    """Query sinain knowledge graph for facts relevant to a question.
-
-    Strategy: first try tag-based retrieval (targeted). If nothing found,
-    fall back to full DB dump (sinain stores are small enough).
-    """
-    from graph_query import query_facts_by_entities, format_facts_text
-
-    keywords = _extract_keywords(question)
-    if keywords:
-        facts = query_facts_by_entities(db_path, keywords, max_facts=MAX_FACTS_PER_QUERY)
-        if facts:
-            return format_facts_text(facts, max_chars=3000)
-
-    # Fallback: include all facts (DB is small, typically 10-30 facts)
-    return _get_all_facts_text(db_path)
-
-
-def _get_retrieved_facts(db_path: str, question: str, k: int = 10) -> list[dict]:
-    """Get facts retrieved for a question (for retrieval evaluation)."""
-    from graph_query import query_facts_by_entities, query_top_facts
-
-    keywords = _extract_keywords(question)
-    if keywords:
-        facts = query_facts_by_entities(db_path, keywords, max_facts=k)
-        if facts:
-            return facts
-
-    # Fallback: top facts by confidence
-    return query_top_facts(db_path, limit=k)
-
-
-def compute_content_recall(
-    retrieved_facts: list[dict],
-    gold_answer: str,
-    k_values: list[int] | None = None,
-) -> dict:
-    """Content-based retrieval metric: do retrieved facts contain the answer?
-
-    Instead of matching entity IDs (which don't align between LongMemEval
-    session IDs and sinain entity IDs), we check whether the gold answer's
-    key terms appear in any retrieved fact's content.
-    """
-    from .config import K_VALUES
-    ks = k_values or K_VALUES
-
-    gold_terms = set(_extract_keywords(str(gold_answer)))
-    if not gold_terms:
-        return {f"content_recall@{k}": 0.0 for k in ks}
-
-    result = {}
-    for k in ks:
-        top_k = retrieved_facts[:k]
-        # Check if any fact in top-k contains gold answer terms
-        for fact in top_k:
-            fact_text = f"{fact.get('entity', '')} {fact.get('value', '')}".lower()
-            fact_terms = set(_extract_keywords(fact_text))
-            overlap = gold_terms & fact_terms
-            if len(overlap) >= max(1, len(gold_terms) // 2):  # ≥50% of gold terms
-                result[f"content_recall@{k}"] = 1.0
-                break
-        else:
-            result[f"content_recall@{k}"] = 0.0
-
-    return result
-
-
-def answer_question(
-    question: BenchmarkQuestion,
-    condition: str,
-    *,
-    db_path: str | None = None,
-    full_context: str | None = None,
-    knowledge_doc: str | None = None,
-    model: str | None = None,
-) -> str:
-    """Generate an answer for a benchmark question under a specific condition.
-
-    Returns the LLM's answer text.
-    """
-    qa_model = model or QA_MODEL
-
-    if condition == "sinain-memory":
-        assert db_path, "db_path required for sinain-memory condition"
-        facts = _query_knowledge(db_path, question.text)
-        system = (
-            "Answer the question using ONLY the provided knowledge facts. "
-            "If the facts don't contain enough information to answer, say \"I don't know.\""
-        )
-        user = f"## Knowledge Facts\n{facts}\n\n## Question\n{question.text}"
-
-    elif condition == "full-context":
-        assert full_context, "full_context required for full-context condition"
-        system = (
-            "Answer the question based on the conversation history below. "
-            "Be concise and specific."
-        )
-        # Truncate context if too large (some models have limits)
-        ctx = full_context[:100_000] if len(full_context) > 100_000 else full_context
-        user = f"## Conversation History\n{ctx}\n\n## Question\n{question.text}"
-
-    elif condition == "knowledge-doc":
-        assert knowledge_doc, "knowledge_doc required for knowledge-doc condition"
-        system = (
-            "Answer the question using ONLY the knowledge document provided. "
-            "If the document doesn't contain enough information, say \"I don't know.\""
-        )
-        user = f"## Knowledge Document\n{knowledge_doc}\n\n## Question\n{question.text}"
-
-    else:
-        raise ValueError(f"Unknown condition: {condition}")
-
-    try:
-        return call_llm(
-            system_prompt=system,
-            user_prompt=user,
-            model=qa_model,
-            max_tokens=300,
-        ).strip()
-    except Exception as e:
-        return f"(error: {e})"

package/sinain-memory/eval/benchmarks/report.py

@@ -1,87 +0,0 @@
-"""Report generation — markdown, JSON, and LaTeX output."""
-
-from __future__ import annotations
-
-import json
-from datetime import datetime, timezone
-
-
-def generate_markdown(benchmark_name: str, summary: dict, details: list[dict]) -> str:
-    """Generate a publishable markdown report."""
-    lines = [
-        f"# Sinain Knowledge Graph — {benchmark_name} Results",
-        f"\nGenerated: {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}",
-        "",
-    ]
-
-    # Headline IPR
-    ipr = summary.get("ipr")
-    if ipr:
-        lines.append(f"**Information Preservation Rate (IPR)**: {ipr:.1%}")
-        lines.append("")
-
-    # Condition scores table
-    conditions = summary.get("conditions", {})
-    if conditions:
-        cond_names = sorted(conditions.keys())
-        header = "| Condition | Mean Score (1-5) | Mean F1 | N |"
-        sep = "|-----------|------------------|---------|---|"
-        lines.extend([header, sep])
-        for cond in cond_names:
-            c = conditions[cond]
-            lines.append(f"| {cond} | {c['mean_score']:.2f} | {c.get('mean_f1', 0):.2f} | {c['n']} |")
-        lines.append("")
-
-    # Retrieval metrics
-    retrieval = summary.get("retrieval", {})
-    if retrieval:
-        lines.append("## Retrieval Quality")
-        lines.append("| Metric | Score |")
-        lines.append("|--------|-------|")
-        for k, v in sorted(retrieval.items()):
-            lines.append(f"| {k} | {v:.1%} |")
-        lines.append("")
-
-    # Category breakdown
-    categories = summary.get("categories", {})
-    if categories:
-        lines.append("## By Category")
-        cond_names = sorted(set(c for cat in categories.values() for c in cat))
-        header = "| Category | " + " | ".join(cond_names) + " |"
-        sep = "|----------|" + "|".join(["------"] * len(cond_names)) + "|"
-        lines.extend([header, sep])
-        for cat in sorted(categories):
-            cells = []
-            for cond in cond_names:
-                if cond in categories[cat]:
-                    cells.append(f"{categories[cat][cond]['mean_score']:.2f} (n={categories[cat][cond]['n']})")
-                else:
-                    cells.append("-")
-            lines.append(f"| {cat} | " + " | ".join(cells) + " |")
-        lines.append("")
-
-    # Failures (worst questions)
-    if details:
-        sm_details = [d for d in details if d.get("answers", {}).get("sinain-memory", {}).get("score") is not None]
-        sm_details.sort(key=lambda d: d["answers"]["sinain-memory"]["score"])
-        if sm_details:
-            lines.append("## Hardest Questions for sinain-memory (bottom 5)")
-            for d in sm_details[:5]:
-                sm = d["answers"]["sinain-memory"]
-                fc = d["answers"].get("full-context", {})
-                lines.append(f"- **{d['id']}** [{d['category']}]: score={sm['score']}/5 "
-                             f"(full-context: {fc.get('score', '?')}/5)")
-                lines.append(f"  Q: {d['question'][:100]}...")
-            lines.append("")
-
-    return "\n".join(lines)
-
-
-def generate_json(benchmark_name: str, summary: dict, details: list[dict]) -> str:
-    """Generate JSON report."""
-    return json.dumps({
-        "benchmark": benchmark_name,
-        "timestamp": datetime.now(timezone.utc).isoformat(),
-        "summary": summary,
-        "details": details,
-    }, indent=2, ensure_ascii=False)

package/sinain-memory/eval/benchmarks/runner.py

@@ -1,276 +0,0 @@
-#!/usr/bin/env python3
-"""Benchmark runner — evaluates sinain's knowledge graph against published benchmarks.
-
-Usage:
-    python3 eval/benchmarks/runner.py --benchmarks longmemeval --subset 5
-    python3 eval/benchmarks/runner.py --benchmarks longmemeval --conditions sinain-memory,full-context
-    python3 eval/benchmarks/runner.py --benchmarks longmemeval --format markdown --resume
-"""
-
-from __future__ import annotations
-
-import argparse
-import json
-import sys
-from pathlib import Path
-
-# Add sinain-memory to path
-_koog_dir = str(Path(__file__).resolve().parent.parent.parent)
-if _koog_dir not in sys.path:
-    sys.path.insert(0, _koog_dir)
-
-from eval.benchmarks.config import DATA_DIR, RESULTS_DIR, QA_MODEL, JUDGE_MODEL
-from eval.benchmarks.base_adapter import BenchmarkAdapter, BenchmarkInstance
-from eval.benchmarks.longmemeval_adapter import LongMemEvalAdapter
-from eval.benchmarks.ingest import ingest_instance, get_knowledge_doc
-from eval.benchmarks.query import answer_question, _get_retrieved_facts, compute_content_recall
-from eval.benchmarks.evaluate import (
-    token_f1, aggregate_results,
-)
-from eval.benchmarks.judges.qa_judge import judge_qa
-from eval.benchmarks.report import generate_markdown, generate_json
-
-
-def _get_adapter(name: str) -> BenchmarkAdapter:
-    if name == "longmemeval":
-        return LongMemEvalAdapter()
-    raise ValueError(f"Unknown benchmark: {name}. Available: longmemeval")
-
-
-def _load_resume(resume_path: Path) -> dict[str, dict]:
-    """Load previously computed results for resume support."""
-    results = {}
-    if resume_path.exists():
-        for line in resume_path.read_text().strip().split("\n"):
-            if line:
-                entry = json.loads(line)
-                results[entry["id"]] = entry
-    return results
-
-
-def run_benchmark(
-    benchmark_name: str,
-    conditions: list[str],
-    *,
-    subset: int | None = None,
-    qa_model: str = QA_MODEL,
-    judge_model: str = JUDGE_MODEL,
-    output_dir: Path = RESULTS_DIR,
-    cache_dir: Path = DATA_DIR,
-    resume: bool = False,
-    skip_llm: bool = False,
-    stratified: bool = False,
-) -> tuple[dict, list[dict]]:
-    """Run a benchmark end-to-end. Returns (summary, details)."""
-
-    adapter = _get_adapter(benchmark_name)
-
-    # Load dataset
-    print(f"\n{'='*60}")
-    print(f"  Benchmark: {benchmark_name}")
-    print(f"  Conditions: {', '.join(conditions)}")
-    print(f"  QA model: {qa_model}")
-    print(f"  Judge model: {judge_model}")
-    print(f"{'='*60}\n")
-
-    instances = adapter.load_dataset(str(cache_dir))
-
-    # Flatten questions
-    all_questions = []
-    for inst in instances:
-        for q in inst.questions:
-            all_questions.append((inst, q))
-
-    if subset:
-        if stratified:
-            # Take equal samples from each question category
-            from collections import defaultdict
-            by_cat: dict[str, list] = defaultdict(list)
-            for pair in all_questions:
-                by_cat[pair[1].category].append(pair)
-            per_cat = max(1, subset // len(by_cat))
-            sampled = []
-            for cat in sorted(by_cat):
-                sampled.extend(by_cat[cat][:per_cat])
-            all_questions = sampled[:subset]
-        else:
-            all_questions = all_questions[:subset]
-
-    total = len(all_questions)
-    print(f"[runner] evaluating {total} questions\n")
-
-    # Resume support
-    resume_path = output_dir / f"{benchmark_name}_progress.jsonl"
-    completed = _load_resume(resume_path) if resume else {}
-    output_dir.mkdir(parents=True, exist_ok=True)
-
-    # Track ingested instances
-    instance_dbs: dict[str, Path | None] = {}
-    instance_docs: dict[str, str] = {}
-
-    details: list[dict] = []
-
-    for idx, (inst, question) in enumerate(all_questions):
-        qid = question.id
-
-        # Skip if already done
-        if qid in completed:
-            details.append(completed[qid])
-            continue
-
-        print(f"[{idx+1}/{total}] {qid} [{question.category}]")
-
-        # Ingest instance if not done yet
-        if inst.id not in instance_dbs:
-            if "sinain-memory" in conditions or "knowledge-doc" in conditions:
-                print(f"  ingesting {inst.id} ({len(inst.sessions)} sessions)...")
-                instance_dbs[inst.id] = ingest_instance(inst, cache_dir / benchmark_name)
-                db = instance_dbs[inst.id]
-                if db:
-                    instance_docs[inst.id] = get_knowledge_doc(db)
-                    print(f"  -> ingested ({db.stat().st_size} bytes)")
-                else:
-                    instance_docs[inst.id] = "(ingestion failed)"
-                    print(f"  -> ingestion failed")
-            else:
-                instance_dbs[inst.id] = None
-                instance_docs[inst.id] = ""
-
-        db_path = instance_dbs.get(inst.id)
-        knowledge_doc = instance_docs.get(inst.id, "")
-        full_context = adapter.format_full_context(inst)
-
-        # Retrieval metrics (content-based: do retrieved facts contain the answer?)
-        retrieval = {}
-        if db_path and "sinain-memory" in conditions:
-            retrieved_facts = _get_retrieved_facts(str(db_path), question.text)
-            retrieval = compute_content_recall(
-                retrieved_facts, question.gold_answer,
-            )
-
-        # Generate answers per condition
-        answers = {}
-        for cond in conditions:
-            if skip_llm:
-                answers[cond] = {"text": "(skipped)", "score": None, "f1": None}
-                continue
-
-            # Skip sinain-memory/knowledge-doc if ingestion failed
-            if cond in ("sinain-memory", "knowledge-doc") and not db_path:
-                answers[cond] = {"text": "(ingestion failed)", "score": 1, "f1": 0.0, "reasoning": "ingestion failed"}
-                print(f"  [{cond}] skipped (ingestion failed)")
-                continue
-
-            print(f"  [{cond}] generating answer...")
-            answer_text = answer_question(
-                question, cond,
-                db_path=str(db_path) if db_path else None,
-                full_context=full_context,
-                knowledge_doc=knowledge_doc,
-                model=qa_model,
-            )
-
-            # Score
-            f1 = token_f1(answer_text, question.gold_answer)
-
-            judge_result = judge_qa(
-                question.text, question.gold_answer, answer_text,
-                condition=cond, model=judge_model,
-            )
-            score = judge_result["score"] if judge_result else None
-            reasoning = judge_result["reasoning"] if judge_result else None
-
-            answers[cond] = {
-                "text": answer_text[:500],
-                "score": score,
-                "f1": round(f1, 4),
-                "reasoning": reasoning,
-            }
-            print(f"    score={score}/5 f1={f1:.2f}")
-
-        entry = {
-            "id": qid,
-            "question": question.text,
-            "gold_answer": question.gold_answer,
-            "category": question.category,
-            "retrieval": retrieval,
-            "answers": answers,
-        }
-        details.append(entry)
-
-        # Save progress incrementally
-        with open(resume_path, "a") as f:
-            f.write(json.dumps(entry, ensure_ascii=False) + "\n")
-
-    # Aggregate
-    summary = aggregate_results(details)
-    return summary, details
-
-
-def main() -> None:
-    parser = argparse.ArgumentParser(description="Sinain Knowledge Graph Benchmark Runner")
-    parser.add_argument("--benchmarks", default="longmemeval",
-                        help="Comma-separated benchmark names (longmemeval, locomo)")
-    parser.add_argument("--conditions", default="sinain-memory,full-context,knowledge-doc",
-                        help="Comma-separated conditions to evaluate")
-    parser.add_argument("--subset", type=int, default=None,
-                        help="Run only first N questions (for dev iteration)")
-    parser.add_argument("--qa-model", default=QA_MODEL, help="Model for QA generation")
-    parser.add_argument("--judge-model", default=JUDGE_MODEL, help="Model for QA judging")
-    parser.add_argument("--output-dir", type=Path, default=RESULTS_DIR)
-    parser.add_argument("--cache-dir", type=Path, default=DATA_DIR)
-    parser.add_argument("--format", default="json,markdown",
-                        help="Output formats (json, markdown)")
-    parser.add_argument("--resume", action="store_true", help="Resume from partial results")
-    parser.add_argument("--skip-llm", action="store_true",
-                        help="Skip LLM calls (retrieval + mechanical metrics only)")
-    parser.add_argument("--stratified", action="store_true",
-                        help="Sample equally from each question category (with --subset)")
-    args = parser.parse_args()
-
-    conditions = [c.strip() for c in args.conditions.split(",")]
-    formats = [f.strip() for f in args.format.split(",")]
-
-    for bench_name in args.benchmarks.split(","):
-        bench_name = bench_name.strip()
-        summary, details = run_benchmark(
-            bench_name, conditions,
-            subset=args.subset,
-            qa_model=args.qa_model,
-            judge_model=args.judge_model,
-            output_dir=args.output_dir,
-            cache_dir=args.cache_dir,
-            resume=args.resume,
-            skip_llm=args.skip_llm,
-            stratified=args.stratified,
-        )
-
-        # Write outputs
-        args.output_dir.mkdir(parents=True, exist_ok=True)
-
-        if "json" in formats:
-            json_path = args.output_dir / f"{bench_name}_results.json"
-            json_path.write_text(generate_json(bench_name, summary, details))
-            print(f"\n[output] JSON: {json_path}")
-
-        if "markdown" in formats:
-            md_path = args.output_dir / f"{bench_name}_results.md"
-            md_path.write_text(generate_markdown(bench_name, summary, details))
-            print(f"[output] Markdown: {md_path}")
-
-        # Print summary
-        print(f"\n{'='*60}")
-        print(f"  {bench_name} — Summary")
-        print(f"{'='*60}")
-        ipr = summary.get("ipr")
-        if ipr:
-            print(f"  IPR: {ipr:.1%}")
-        for cond, data in summary.get("conditions", {}).items():
-            print(f"  {cond}: {data['mean_score']:.2f}/5 (n={data['n']})")
-        for k, v in summary.get("retrieval", {}).items():
-            print(f"  {k}: {v:.1%}")
-        print()
-
-
-if __name__ == "__main__":
-    main()

package/sinain-memory/eval/judges/base_judge.py

@@ -1,61 +0,0 @@
-"""Shared infrastructure for LLM-as-Judge evaluators.
-
-Provides ``run_judge()`` which calls the LLM with a rubric prompt and
-extracts a ``{"score": 1-4, "reasoning": "..."}`` response.
-"""
-
-from __future__ import annotations
-
-import json
-import sys
-from pathlib import Path
-
-# Add parent dirs so ``common`` is importable when running from anywhere.
-_koog_dir = str(Path(__file__).resolve().parent.parent.parent)
-if _koog_dir not in sys.path:
-    sys.path.insert(0, _koog_dir)
-
-from common import LLMError, call_llm, extract_json  # noqa: E402
-
-
-def run_judge(
-    system_prompt: str,
-    user_prompt: str,
-    *,
-    model: str | None = None,
-    max_tokens: int = 200,
-    timeout: int = 30,
-) -> dict | None:
-    """Call LLM with a judge prompt and return ``{"score": int, "reasoning": str}`` or None.
-
-    *model* defaults to the ``eval.judges.model`` setting resolved externally.
-    When None, falls back to ``common.call_llm`` defaults (which reads koog-config).
-    """
-    try:
-        kwargs: dict = {
-            "system_prompt": system_prompt,
-            "user_prompt": user_prompt,
-            "max_tokens": max_tokens,
-            "json_mode": True,
-        }
-        # Use script-based config resolution if no explicit model
-        if model:
-            kwargs["model"] = model
-        else:
-            kwargs["script"] = "tick_evaluator"
-
-        raw = call_llm(**kwargs)
-        result = extract_json(raw)
-
-        score = result.get("score")
-        reasoning = result.get("reasoning", "")
-
-        if not isinstance(score, (int, float)) or not (1 <= score <= 4):
-            print(f"[warn] judge returned invalid score: {score}", file=sys.stderr)
-            return None
-
-        return {"score": int(score), "reasoning": str(reasoning)[:300]}
-
-    except (ValueError, LLMError, KeyError) as e:
-        print(f"[warn] judge call failed: {e}", file=sys.stderr)
-        return None

package/sinain-memory/eval/judges/curation_judge.py

@@ -1,46 +0,0 @@
-"""LLM-as-Judge: Playbook curation quality evaluator."""
-
-from __future__ import annotations
-
-from .base_judge import run_judge
-
-SYSTEM_PROMPT = """\
-You are an evaluator scoring the quality of playbook curation changes.
-
-The curator follows a directive and three laws:
-  Law 1: Don't remove error-prevention patterns
-  Law 2: Preserve high-scoring approaches
-  Law 3: Then evolve
-
-Rate the curation on a 1-4 scale:
-  4: Changes perfectly match directive + evidence, three laws respected
-  3: Good changes, minor alignment issues with directive
-  2: Changes misaligned with directive or weak evidence
-  1: Destructive changes, violated three laws, or ignored directive entirely
-
-Respond with ONLY a JSON object: {"score": <1-4>, "reasoning": "brief explanation"}"""
-
-
-def judge_curation(
-    curator_result: dict,
-    directive: str,
-    playbook_before: str = "",
-    **kwargs,
-) -> dict | None:
-    """Evaluate playbook curation quality. Returns {"score": 1-4, "reasoning": str} or None."""
-    changes = curator_result.get("changes", {})
-    stale_actions = curator_result.get("staleItemActions", [])
-    lines = curator_result.get("playbookLines", "?")
-
-    parts = [
-        f"## Curate Directive\n{directive}",
-        f"\n## Changes Made\nAdded: {changes.get('added', [])}\nPruned: {changes.get('pruned', [])}\nPromoted: {changes.get('promoted', [])}",
-        f"\n## Stale Item Actions\n{stale_actions}",
-        f"\n## Playbook Lines After: {lines}",
-    ]
-
-    if playbook_before:
-        # Truncate to keep prompt manageable
-        parts.append(f"\n## Playbook Before (excerpt)\n{playbook_before[:1500]}")
-
-    return run_judge(SYSTEM_PROMPT, "\n".join(parts), **kwargs)