@geravant/sinain 1.13.0 → 1.14.0

package/sinain-memory/eval/assertions.py

@@ -1,267 +0,0 @@
-"""Behavioral assertion library for sinain-koog tick evaluation.
-
-Each assertion function validates a runtime invariant of the pipeline.
-Returns ``{"name": str, "passed": bool, "detail": str}``.
-"""
-
-from __future__ import annotations
-
-
-def _result(name: str, passed: bool, detail: str) -> dict:
-    return {"name": name, "passed": passed, "detail": detail}
-
-
-# ---------------------------------------------------------------------------
-# Playbook curator assertions
-# ---------------------------------------------------------------------------
-
-def assert_playbook_under_limit(curator_result: dict, limit: int = 50) -> dict:
-    """Verify playbook body stays under the line limit."""
-    lines = curator_result.get("playbookLines", 0)
-    if lines <= limit:
-        return _result("playbook_under_limit", True, f"body has {lines} lines (limit {limit})")
-    return _result("playbook_under_limit", False, f"body has {lines} lines, exceeds limit of {limit}")
-
-
-def assert_curator_respected_directive(curator_result: dict, directive: str) -> dict:
-    """Check that curator changes align with the curate directive."""
-    changes = curator_result.get("changes", {})
-    added = len(changes.get("added", []))
-    pruned = len(changes.get("pruned", []))
-
-    if directive == "aggressive_prune":
-        # Should have pruned items
-        if pruned > 0:
-            return _result("curator_respected_directive", True,
-                           f"aggressive_prune: pruned {pruned} items")
-        if added == 0 and pruned == 0:
-            return _result("curator_respected_directive", True,
-                           "aggressive_prune: no changes (acceptable if playbook already lean)")
-        return _result("curator_respected_directive", False,
-                       f"aggressive_prune: added {added} but pruned {pruned} — expected pruning")
-
-    if directive == "stability":
-        # Should not aggressively prune established patterns
-        if pruned > added + 2:
-            return _result("curator_respected_directive", False,
-                           f"stability: pruned {pruned} items (only added {added}) — too aggressive for stability mode")
-        return _result("curator_respected_directive", True,
-                       f"stability: added {added}, pruned {pruned} — conservative")
-
-    # normal / insufficient_data — any reasonable mix is fine
-    return _result("curator_respected_directive", True,
-                   f"{directive}: added {added}, pruned {pruned}")
-
-
-# ---------------------------------------------------------------------------
-# Signal analyzer assertions
-# ---------------------------------------------------------------------------
-
-def assert_no_repeat_action(signal_result: dict, recent_logs: list[dict], window: int = 3) -> dict:
-    """Verify recommendedAction doesn't repeat the last N ticks' actions."""
-    action = signal_result.get("recommendedAction")
-    if action is None or action.get("action") == "skip":
-        return _result("no_repeat_action", True, "no action recommended (skip/null)")
-
-    task = (action.get("task") or "").lower().strip()
-    if not task:
-        return _result("no_repeat_action", True, "no task description to compare")
-
-    # Collect recent action tasks
-    recent_tasks: list[str] = []
-    for log in recent_logs[:window]:
-        log_actions = log.get("actionsConsidered", [])
-        for a in log_actions:
-            if a.get("chosen"):
-                recent_tasks.append((a.get("reason") or a.get("task") or "").lower().strip())
-
-    # Check for near-duplicate (substring match to catch rephrasing)
-    for prev_task in recent_tasks:
-        if not prev_task:
-            continue
-        # If >60% of words overlap, consider it a repeat
-        task_words = set(task.split())
-        prev_words = set(prev_task.split())
-        if not task_words or not prev_words:
-            continue
-        overlap = len(task_words & prev_words) / max(len(task_words), len(prev_words))
-        if overlap > 0.6:
-            return _result("no_repeat_action", False,
-                           f"action task '{task[:60]}' overlaps with recent '{prev_task[:60]}' ({overlap:.0%} word overlap)")
-
-    return _result("no_repeat_action", True,
-                   f"action task is distinct from last {window} ticks")
-
-
-def assert_signal_confidence_threshold(signal_result: dict, threshold: float = 0.5) -> dict:
-    """Verify actions are only recommended above the confidence threshold."""
-    action = signal_result.get("recommendedAction")
-    if action is None or action.get("action") == "skip":
-        return _result("signal_confidence_threshold", True, "no action recommended")
-
-    confidence = action.get("confidence")
-    if confidence is None:
-        return _result("signal_confidence_threshold", False,
-                       "action recommended but no confidence value provided")
-
-    if confidence >= threshold:
-        return _result("signal_confidence_threshold", True,
-                       f"confidence {confidence:.2f} >= threshold {threshold}")
-    return _result("signal_confidence_threshold", False,
-                   f"confidence {confidence:.2f} < threshold {threshold}")
-
-
-# ---------------------------------------------------------------------------
-# Insight synthesizer assertions
-# ---------------------------------------------------------------------------
-
-def assert_insight_char_limit(synth_result: dict, limit: int = 500) -> dict:
-    """Verify suggestion+insight stays under the character limit."""
-    if synth_result.get("skip", False):
-        return _result("insight_char_limit", True, "output skipped")
-
-    suggestion = synth_result.get("suggestion", "")
-    insight = synth_result.get("insight", "")
-    total = len(suggestion) + len(insight)
-
-    if total <= limit:
-        return _result("insight_char_limit", True, f"total {total} chars (limit {limit})")
-    return _result("insight_char_limit", False, f"total {total} chars exceeds limit of {limit}")
-
-
-def assert_skip_reason_specific(synth_result: dict) -> dict:
-    """If skip=true, verify the reason is specific (not generic boilerplate)."""
-    if not synth_result.get("skip", False):
-        return _result("skip_reason_specific", True, "output not skipped")
-
-    reason = (synth_result.get("skipReason") or "").strip()
-    if not reason:
-        return _result("skip_reason_specific", False, "skip=true but no skipReason provided")
-
-    # Check against known-generic patterns
-    generic_phrases = [
-        "no new data",
-        "nothing new",
-        "no updates",
-        "insufficient data",
-        "not enough information",
-        "no changes",
-    ]
-    reason_lower = reason.lower()
-    for phrase in generic_phrases:
-        if reason_lower == phrase or (len(reason_lower) < 30 and phrase in reason_lower):
-            return _result("skip_reason_specific", False,
-                           f"skipReason is too generic: '{reason}'")
-
-    return _result("skip_reason_specific", True, f"skipReason is specific ({len(reason)} chars)")
-
-
-# ---------------------------------------------------------------------------
-# Memory miner assertions
-# ---------------------------------------------------------------------------
-
-def assert_miner_references_sources(miner_result: dict, daily_files: list[str]) -> dict:
-    """Verify mining findings reference actual source files that were provided."""
-    mined = miner_result.get("minedSources", [])
-    if not mined:
-        return _result("miner_references_sources", True, "no sources mined (early return)")
-
-    # daily_files contains basenames like "2026-02-21.md"
-    known_basenames = set(daily_files)
-    unknown = [s for s in mined if s not in known_basenames]
-
-    if unknown:
-        return _result("miner_references_sources", False,
-                       f"minedSources references unknown files: {unknown}")
-    return _result("miner_references_sources", True,
-                   f"all {len(mined)} mined sources are valid")
-
-
-# ---------------------------------------------------------------------------
-# Cross-script / structural assertions
-# ---------------------------------------------------------------------------
-
-def assert_schema_valid(script_name: str, output: dict, schema_errors: list[str]) -> dict:
-    """Wrap schema validation result as an assertion."""
-    if not schema_errors:
-        return _result(f"schema_valid_{script_name}", True, "output matches schema")
-    return _result(f"schema_valid_{script_name}", False,
-                   f"{len(schema_errors)} schema errors: {'; '.join(schema_errors[:3])}")
-
-
-def assert_playbook_header_footer_intact(playbook_text: str) -> dict:
-    """Verify the playbook still has its mining-index header and effectiveness footer."""
-    has_header = "<!-- mining-index:" in playbook_text
-    has_footer = "<!-- effectiveness:" in playbook_text
-
-    if has_header and has_footer:
-        return _result("playbook_header_footer_intact", True,
-                       "both mining-index and effectiveness comments present")
-    missing = []
-    if not has_header:
-        missing.append("mining-index")
-    if not has_footer:
-        missing.append("effectiveness")
-    return _result("playbook_header_footer_intact", False,
-                   f"missing playbook comments: {', '.join(missing)}")
-
-
-# ---------------------------------------------------------------------------
-# Runner: execute all applicable assertions for a tick
-# ---------------------------------------------------------------------------
-
-def run_tick_assertions(
-    log_entry: dict,
-    recent_logs: list[dict],
-    playbook_text: str,
-    daily_files: list[str],
-) -> list[dict]:
-    """Run all applicable assertions against a single tick's log entry.
-
-    Returns a list of assertion result dicts.
-    """
-    results: list[dict] = []
-
-    # Signal analyzer assertions
-    signals = log_entry.get("signals")
-    if signals is not None:
-        results.append(assert_signal_confidence_threshold(
-            {"signals": signals, "recommendedAction": log_entry.get("recommendedAction")},
-        ))
-        results.append(assert_no_repeat_action(
-            {"signals": signals, "recommendedAction": log_entry.get("recommendedAction")},
-            recent_logs,
-        ))
-
-    # Curator assertions — playbookChanges can be {"note": "skipped"} or full output
-    curator = log_entry.get("playbookChanges")
-    if isinstance(curator, dict) and "changes" in curator:
-        curator_with_lines = {**curator}
-        if "playbookLines" not in curator_with_lines:
-            curator_with_lines["playbookLines"] = curator.get("playbookLines", 0)
-        results.append(assert_playbook_under_limit(curator_with_lines))
-
-        directive = log_entry.get("curateDirective", "normal")
-        results.append(assert_curator_respected_directive(curator_with_lines, directive))
-
-    # Insight synthesizer assertions — output can be null (pipeline-level skip)
-    output = log_entry.get("output")
-    if isinstance(output, dict):
-        results.append(assert_insight_char_limit(output))
-        results.append(assert_skip_reason_specific(output))
-
-    # Mining assertions — log uses miningFindings (str) and minedSources (list)
-    mining = log_entry.get("miningResult")
-    if mining is not None:
-        results.append(assert_miner_references_sources(mining, daily_files))
-    elif log_entry.get("minedSources"):
-        # Reconstruct mining result from flat log fields
-        results.append(assert_miner_references_sources(
-            {"minedSources": log_entry.get("minedSources", [])}, daily_files
-        ))
-
-    # Playbook health (if we have playbook text)
-    if playbook_text:
-        results.append(assert_playbook_header_footer_intact(playbook_text))
-
-    return results

package/sinain-memory/eval/benchmarks/base_adapter.py

@@ -1,43 +0,0 @@
-"""Base adapter and data classes for benchmark evaluation."""
-
-from __future__ import annotations
-
-from abc import ABC, abstractmethod
-from dataclasses import dataclass, field
-
-
-@dataclass
-class BenchmarkQuestion:
-    id: str
-    text: str
-    gold_answer: str
-    category: str  # single-session, multi-session, temporal, etc.
-    evidence_session_ids: list[str] = field(default_factory=list)
-    metadata: dict = field(default_factory=dict)
-
-
-@dataclass
-class BenchmarkInstance:
-    """A set of conversations + questions that share the same context."""
-    id: str
-    sessions: list[list[dict]]  # list of sessions, each a list of feed items {source, text, ts}
-    questions: list[BenchmarkQuestion] = field(default_factory=list)
-    raw_sessions: list[dict] = field(default_factory=list)  # original benchmark format (for full-context condition)
-    metadata: dict = field(default_factory=dict)
-
-
-class BenchmarkAdapter(ABC):
-    """Abstract adapter: converts a published benchmark into sinain's format."""
-
-    @property
-    @abstractmethod
-    def name(self) -> str:
-        """Benchmark name (e.g. 'longmemeval', 'locomo')."""
-
-    @abstractmethod
-    def load_dataset(self, data_dir: str) -> list[BenchmarkInstance]:
-        """Download (if needed) and parse the benchmark dataset."""
-
-    @abstractmethod
-    def format_full_context(self, instance: BenchmarkInstance) -> str:
-        """Render the full conversation history as a text string for the baseline condition."""

package/sinain-memory/eval/benchmarks/config.py

@@ -1,23 +0,0 @@
-"""Benchmark configuration — models, paths, thresholds."""
-
-from pathlib import Path
-
-BENCHMARKS_DIR = Path(__file__).resolve().parent
-DATA_DIR = BENCHMARKS_DIR / "data"
-RESULTS_DIR = BENCHMARKS_DIR / "results"
-
-# LLM models (via OpenRouter)
-QA_MODEL = "google/gemini-2.5-flash"
-JUDGE_MODEL = "openai/gpt-4o"
-
-# Retrieval
-K_VALUES = [1, 3, 5, 10]
-MAX_FACTS_PER_QUERY = 10
-
-# Ingestion
-DISTILLER_TIMEOUT_S = 30
-INTEGRATOR_TIMEOUT_S = 60
-
-# Dataset URLs
-LONGMEMEVAL_HF = "xiaowu0162/longmemeval-cleaned"
-LOCOMO_GITHUB = "https://raw.githubusercontent.com/snap-research/locomo/main/data/locomo10.json"

package/sinain-memory/eval/benchmarks/evaluate.py

@@ -1,146 +0,0 @@
-"""Evaluation pipeline — score answers and compute aggregate metrics.
-
-Combines:
-- LLM-as-Judge (QA scoring, 1-5 scale)
-- Retrieval metrics (Recall@k, NDCG@k)
-- Token F1 overlap (mechanical, free)
-"""
-
-from __future__ import annotations
-
-import math
-import re
-from collections import defaultdict
-
-from .base_adapter import BenchmarkQuestion
-from .config import K_VALUES
-
-
-# ── Token F1 (mechanical, no LLM needed) ─────────────────────────────────────
-
-def _tokenize(text: str) -> list[str]:
-    """Simple whitespace + punctuation tokenizer."""
-    return re.findall(r"\w+", text.lower())
-
-
-def token_f1(predicted: str, gold: str | int) -> float:
-    """Compute token-level F1 between predicted and gold answers."""
-    pred_tokens = set(_tokenize(str(predicted)))
-    gold_tokens = set(_tokenize(str(gold)))
-    if not gold_tokens or not pred_tokens:
-        return 0.0
-    overlap = pred_tokens & gold_tokens
-    if not overlap:
-        return 0.0
-    precision = len(overlap) / len(pred_tokens)
-    recall = len(overlap) / len(gold_tokens)
-    return 2 * precision * recall / (precision + recall)
-
-
-# ── Retrieval metrics (reuse logic from retrieval_evaluator.py) ───────────────
-
-def dcg_at_k(relevant_positions: list[int], k: int) -> float:
-    """Discounted Cumulative Gain at k."""
-    score = 0.0
-    for pos in relevant_positions:
-        if pos < k:
-            score += 1.0 / math.log2(pos + 2)
-    return score
-
-
-def ndcg_at_k(relevant_positions: list[int], num_relevant: int, k: int) -> float:
-    """Normalized DCG at k."""
-    dcg = dcg_at_k(relevant_positions, k)
-    ideal_positions = list(range(min(num_relevant, k)))
-    idcg = dcg_at_k(ideal_positions, k)
-    return dcg / idcg if idcg > 0 else 0.0
-
-
-def compute_retrieval_metrics(
-    retrieved_ids: list[str],
-    expected_ids: list[str],
-    k_values: list[int] | None = None,
-) -> dict:
-    """Compute Recall@k and NDCG@k for a single question."""
-    ks = k_values or K_VALUES
-    expected_set = set(expected_ids)
-    relevant_positions = [i for i, rid in enumerate(retrieved_ids) if rid in expected_set]
-
-    result = {}
-    for k in ks:
-        hit = any(pos < k for pos in relevant_positions)
-        result[f"recall@{k}"] = 1.0 if hit else 0.0
-        result[f"ndcg@{k}"] = ndcg_at_k(relevant_positions, len(expected_set), k)
-    return result
-
-
-# ── Aggregate metrics ─────────────────────────────────────────────────────────
-
-def aggregate_results(per_question: list[dict]) -> dict:
-    """Compute aggregate metrics from per-question results.
-
-    Each per_question entry has:
-      {id, category, retrieval: {recall@k, ndcg@k}, answers: {condition: {score, f1}}}
-    """
-    if not per_question:
-        return {"error": "no results"}
-
-    # Per-condition scores
-    condition_scores: dict[str, list[float]] = defaultdict(list)
-    condition_f1s: dict[str, list[float]] = defaultdict(list)
-    # Per-category per-condition
-    cat_scores: dict[str, dict[str, list[float]]] = defaultdict(lambda: defaultdict(list))
-    # Retrieval
-    retrieval_metrics: dict[str, list[float]] = defaultdict(list)
-
-    for q in per_question:
-        cat = q.get("category", "unknown")
-
-        for cond, data in q.get("answers", {}).items():
-            if data.get("score") is not None:
-                condition_scores[cond].append(data["score"])
-                cat_scores[cat][cond].append(data["score"])
-            if data.get("f1") is not None:
-                condition_f1s[cond].append(data["f1"])
-
-        for metric, val in q.get("retrieval", {}).items():
-            if isinstance(val, (int, float)):
-                retrieval_metrics[metric].append(val)
-
-    def _mean(lst: list[float]) -> float:
-        return round(sum(lst) / len(lst), 4) if lst else 0.0
-
-    # Build summary
-    conditions = {}
-    for cond in sorted(condition_scores):
-        conditions[cond] = {
-            "mean_score": _mean(condition_scores[cond]),
-            "mean_f1": _mean(condition_f1s.get(cond, [])),
-            "n": len(condition_scores[cond]),
-        }
-
-    # IPR: sinain-memory vs full-context
-    sm_scores = condition_scores.get("sinain-memory", [])
-    fc_scores = condition_scores.get("full-context", [])
-    ipr = _mean(sm_scores) / _mean(fc_scores) if fc_scores and _mean(fc_scores) > 0 else None
-
-    # Category breakdown
-    categories = {}
-    for cat in sorted(cat_scores):
-        categories[cat] = {}
-        for cond in sorted(cat_scores[cat]):
-            categories[cat][cond] = {
-                "mean_score": _mean(cat_scores[cat][cond]),
-                "n": len(cat_scores[cat][cond]),
-            }
-
-    # Retrieval summary
-    retrieval = {k: _mean(v) for k, v in sorted(retrieval_metrics.items())}
-
-    return {
-        "total_questions": len(per_question),
-        "conditions": conditions,
-        "ipr": round(ipr, 4) if ipr else None,
-        "categories": categories,
-        "retrieval": retrieval,
-    }

package/sinain-memory/eval/benchmarks/ingest.py

@@ -1,152 +0,0 @@
-"""Ingestion pipeline — benchmark conversations → sinain triplestore.
-
-Runs session_distiller.py + knowledge_integrator.py via subprocess (exact production path).
-Caches results aggressively to avoid repeated LLM calls.
-"""
-
-from __future__ import annotations
-
-import hashlib
-import json
-import os
-import shutil
-import tempfile
-from pathlib import Path
-from subprocess import run, PIPE, TimeoutExpired
-
-from .base_adapter import BenchmarkInstance
-from .config import DISTILLER_TIMEOUT_S, INTEGRATOR_TIMEOUT_S
-
-
-def _scripts_dir() -> Path:
-    """Locate sinain-memory scripts directory."""
-    return Path(__file__).resolve().parent.parent.parent
-
-
-def _content_hash(sessions: list[list[dict]]) -> str:
-    """Hash session content for caching."""
-    raw = json.dumps(sessions, sort_keys=True, ensure_ascii=False)
-    return hashlib.sha256(raw.encode()).hexdigest()[:16]
-
-
-def _run_script(script_name: str, args: list[str], timeout: int) -> str | None:
-    """Run a Python script from sinain-memory, return stdout or None on failure."""
-    script_path = _scripts_dir() / script_name
-    if not script_path.exists():
-        print(f"[ingest] {script_name} not found at {script_path}")
-        return None
-
-    env = {**os.environ, "PYTHONPATH": str(_scripts_dir())}
-    # Ensure a working model is available (common.py defaults may reference unreleased models)
-    if "SINAIN_BENCH_MODEL" in os.environ:
-        env["SINAIN_FAST_MODEL"] = os.environ["SINAIN_BENCH_MODEL"]
-    try:
-        result = run(
-            ["python3", str(script_path)] + args,
-            capture_output=True, text=True, timeout=timeout, env=env,
-        )
-        if result.returncode != 0:
-            print(f"[ingest] {script_name} failed: {result.stderr[:200]}")
-            return None
-        return result.stdout.strip()
-    except TimeoutExpired:
-        print(f"[ingest] {script_name} timed out ({timeout}s)")
-        return None
-
-
-def ingest_instance(
-    instance: BenchmarkInstance,
-    cache_dir: Path,
-) -> Path | None:
-    """Ingest a benchmark instance into a triplestore. Returns db_path or None.
-
-    Uses caching: if the same haystack was already ingested, returns the cached DB.
-    """
-    ch = _content_hash(instance.sessions)
-    cache_path = cache_dir / "stores" / f"{ch}.db"
-
-    if cache_path.exists():
-        return cache_path
-
-    cache_path.parent.mkdir(parents=True, exist_ok=True)
-
-    # Create temp memory directory
-    tmp = tempfile.mkdtemp(prefix="sinain-bench-")
-    mem_dir = Path(tmp) / "memory"
-    for subdir in ["", "playbook-logs", "playbook-archive"]:
-        (mem_dir / subdir).mkdir(parents=True, exist_ok=True)
-
-    # Write a minimal playbook so integrator doesn't fail
-    (mem_dir / "sinain-playbook.md").write_text("# Sinain Playbook\n\n(benchmark run)\n")
-
-    success = False
-    try:
-        # Batch sessions into chunks of ~10 for fewer LLM calls.
-        # Each chunk becomes one distiller call with a combined transcript.
-        BATCH_SIZE = 10
-        num_sessions = len(instance.sessions)
-        batch_idx = 0
-
-        for start in range(0, num_sessions, BATCH_SIZE):
-            batch = instance.sessions[start:start + BATCH_SIZE]
-            # Flatten batch into one transcript
-            combined: list[dict] = []
-            for session in batch:
-                combined.extend(session)
-            if len(combined) < 3:
-                continue
-
-            first_ts = combined[0].get("ts", "2025-01-01T10:00:00Z")
-            meta = json.dumps({
-                "ts": first_ts,
-                "sessionKey": f"benchmark-batch-{batch_idx}",
-                "durationMs": len(combined) * 30000,
-            })
-            batch_idx += 1
-
-            # Step 1: Distill the batch
-            digest_json = _run_script("session_distiller.py", [
-                "--memory-dir", str(mem_dir),
-                "--transcript", json.dumps(combined),
-                "--session-meta", meta,
-            ], DISTILLER_TIMEOUT_S)
-
-            if not digest_json:
-                continue
-
-            try:
-                digest = json.loads(digest_json)
-            except json.JSONDecodeError:
-                continue
-
-            if digest.get("isEmpty") or digest.get("error"):
-                continue
-
-            # Step 2: Integrate into knowledge graph
-            _run_script("knowledge_integrator.py", [
-                "--memory-dir", str(mem_dir),
-                "--digest", json.dumps(digest),
-            ], INTEGRATOR_TIMEOUT_S)
-
-        # Copy the resulting DB to cache
-        db_path = mem_dir / "knowledge-graph.db"
-        if db_path.exists() and db_path.stat().st_size > 0:
-            shutil.copy2(db_path, cache_path)
-            success = True
-
-    finally:
-        shutil.rmtree(tmp, ignore_errors=True)
-
-    return cache_path if success else None
-
-
-def get_knowledge_doc(db_path: Path) -> str:
-    """Render a sinain-knowledge.md style document from a triplestore."""
-    import sys
-    sys.path.insert(0, str(_scripts_dir()))
-    from graph_query import query_top_facts, format_facts_text
-
-    facts = query_top_facts(str(db_path), limit=30)
-    if not facts:
-        return "(no knowledge available)"
-    return format_facts_text(facts, max_chars=6000)