@geravant/sinain 1.12.0 → 1.14.0

package/sinain-memory/eval/judges/insight_judge.py

@@ -1,48 +0,0 @@
-"""LLM-as-Judge: Insight synthesis quality evaluator."""
-
-from __future__ import annotations
-
-from .base_judge import run_judge
-
-SYSTEM_PROMPT = """\
-You are an evaluator scoring the quality of an insight synthesizer's output.
-
-The synthesizer produces two parts:
-  - Suggestion: actionable recommendation grounded in playbook/data
-  - Insight: surprising cross-domain connection from accumulated observations
-
-Rate the output on a 1-4 scale:
-  4: Suggestion is actionable with specific reference, insight connects 2+ distinct observations
-  3: One component is excellent, the other adequate
-  2: Generic suggestion or obvious insight
-  1: Hallucinated content, not grounded in playbook/logs
-
-If the output was skipped, rate the skip decision:
-  4: Skip is well-justified with specific references to what was checked
-  3: Skip is reasonable
-  2: Should not have skipped — there was material to work with
-  1: Skip reason is generic/lazy
-
-Respond with ONLY a JSON object: {"score": <1-4>, "reasoning": "brief explanation"}"""
-
-
-def judge_insight(
-    synth_result: dict,
-    playbook_excerpt: str = "",
-    **kwargs,
-) -> dict | None:
-    """Evaluate insight synthesis quality. Returns {"score": 1-4, "reasoning": str} or None."""
-    skipped = synth_result.get("skip", False)
-
-    parts = []
-    if skipped:
-        parts.append(f"## Status: SKIPPED\nReason: {synth_result.get('skipReason', 'none given')}")
-    else:
-        parts.append(f"## Suggestion\n{synth_result.get('suggestion', '')}")
-        parts.append(f"\n## Insight\n{synth_result.get('insight', '')}")
-        parts.append(f"\n## Total Chars: {synth_result.get('totalChars', '?')}")
-
-    if playbook_excerpt:
-        parts.append(f"\n## Playbook Context (excerpt)\n{playbook_excerpt[:1000]}")
-
-    return run_judge(SYSTEM_PROMPT, "\n".join(parts), **kwargs)

package/sinain-memory/eval/judges/mining_judge.py

@@ -1,42 +0,0 @@
-"""LLM-as-Judge: Memory mining quality evaluator."""
-
-from __future__ import annotations
-
-from .base_judge import run_judge
-
-SYSTEM_PROMPT = """\
-You are an evaluator scoring the quality of a memory mining agent's findings.
-
-The miner reads daily memory files and extracts patterns, preferences, and insights
-that should be added to the evolving playbook.
-
-Rate the mining output on a 1-4 scale:
-  4: Found non-obvious cross-day patterns, all grounded in source files
-  3: Valid patterns found, properly grounded in provided daily files
-  2: Only surface-level observations from source files
-  1: Hallucinated patterns not present in provided daily files
-
-Respond with ONLY a JSON object: {"score": <1-4>, "reasoning": "brief explanation"}"""
-
-
-def judge_mining(
-    miner_result: dict,
-    mined_file_excerpts: dict[str, str] | None = None,
-    **kwargs,
-) -> dict | None:
-    """Evaluate memory mining quality. Returns {"score": 1-4, "reasoning": str} or None."""
-    parts = [
-        f"## Findings\n{miner_result.get('findings', '')}",
-        f"\n## New Patterns\n{miner_result.get('newPatterns', [])}",
-        f"\n## Contradictions\n{miner_result.get('contradictions', [])}",
-        f"\n## Preferences\n{miner_result.get('preferences', [])}",
-        f"\n## Mined Sources\n{miner_result.get('minedSources', [])}",
-    ]
-
-    if mined_file_excerpts:
-        for name, content in mined_file_excerpts.items():
-            # Truncate large files
-            excerpt = content[:1500] if len(content) > 1500 else content
-            parts.append(f"\n## Source File: {name}\n{excerpt}")
-
-    return run_judge(SYSTEM_PROMPT, "\n".join(parts), **kwargs)

package/sinain-memory/eval/judges/signal_judge.py

@@ -1,45 +0,0 @@
-"""LLM-as-Judge: Signal detection quality evaluator."""
-
-from __future__ import annotations
-
-from .base_judge import run_judge
-
-SYSTEM_PROMPT = """\
-You are an evaluator scoring the quality of a signal detection system.
-
-Rate the signal analysis on a 1-4 scale:
-  4: All real signals detected, action is highly relevant and specific
-  3: Key signals detected, action is reasonable
-  2: Missed important signals or action is vague
-  1: Hallucinated signals or inappropriate action
-
-Respond with ONLY a JSON object: {"score": <1-4>, "reasoning": "brief explanation"}"""
-
-
-def judge_signal(
-    signal_result: dict,
-    session_summary: str,
-    recent_logs: list[dict] | None = None,
-    **kwargs,
-) -> dict | None:
-    """Evaluate signal detection quality. Returns {"score": 1-4, "reasoning": str} or None."""
-    parts = [f"## Session Summary\n{session_summary}"]
-
-    signals = signal_result.get("signals", [])
-    action = signal_result.get("recommendedAction")
-    idle = signal_result.get("idle", False)
-
-    parts.append(f"\n## Detected Signals\n{signals}")
-    parts.append(f"\n## Recommended Action\n{action}")
-    parts.append(f"\n## Idle: {idle}")
-
-    if recent_logs:
-        recent_actions = []
-        for log in recent_logs[:3]:
-            for a in log.get("actionsConsidered", []):
-                if a.get("chosen"):
-                    recent_actions.append(a)
-        if recent_actions:
-            parts.append(f"\n## Recent Actions (should not repeat)\n{recent_actions}")
-
-    return run_judge(SYSTEM_PROMPT, "\n".join(parts), **kwargs)

package/sinain-memory/eval/retrieval_benchmark.jsonl

@@ -1,12 +0,0 @@
-{"query": "OCR pipeline stalls on macOS 14", "expected_entities": ["fact:ocr-backpressure", "fact:sck-capture"], "category": "error-resolution"}
-{"query": "camera conflicts with screen capture", "expected_entities": ["fact:camera-conflict", "fact:coremediaio"], "category": "error-resolution"}
-{"query": "audio gain not applied in pipeline", "expected_entities": ["fact:audio-gain"], "category": "bug-fix"}
-{"query": "Flutter ProviderNotFoundException in secondary window", "expected_entities": ["fact:flutter-provider", "fact:multi-window"], "category": "error-resolution"}
-{"query": "user prefers concise Telegram messages", "expected_entities": ["fact:telegram-preference"], "category": "user-preference"}
-{"query": "PyObjC performRequests_error_ returns bool not tuple", "expected_entities": ["fact:pyobjc-api"], "category": "bug-fix"}
-{"query": "ScreenCaptureKit zero-copy IOSurface", "expected_entities": ["fact:sck-capture", "fact:iosurface"], "category": "tool-knowledge"}
-{"query": "OpenClaw gateway workspace not initialized", "expected_entities": ["fact:workspace-init"], "category": "error-resolution"}
-{"query": "react-native metro bundler cache invalidation", "expected_entities": ["fact:react-native-metro"], "category": "tool-knowledge"}
-{"query": "sinain agent session key format", "expected_entities": ["fact:session-key"], "category": "tool-knowledge"}
-{"query": "what was the OCR backend last month", "expected_entities": ["fact:ocr-backend"], "category": "temporal"}
-{"query": "when did we switch from CGDisplayCreateImage to ScreenCaptureKit", "expected_entities": ["fact:sck-capture", "fact:cgdisplay-deprecation"], "category": "temporal"}

package/sinain-memory/eval/retrieval_evaluator.py

@@ -1,186 +0,0 @@
-#!/usr/bin/env python3
-"""Retrieval Quality Evaluator — Recall@k and NDCG@k for knowledge graph queries.
-
-Inspired by mempalace's LongMemEval benchmark infrastructure. Measures whether the
-right knowledge surfaces when the agent needs it, complementing sinain's existing
-output quality evaluation (schemas + assertions + LLM judges).
-
-Usage:
-    python3 eval/retrieval_evaluator.py \
-        --db memory/knowledge-graph.db \
-        --benchmark eval/retrieval_benchmark.jsonl \
-        [--k 1,3,5] [--format json|text]
-
-Benchmark dataset format (JSONL):
-    {"query": "OCR pipeline stalls on macOS 14", "expected_entities": ["fact:sck-capture-fix"], "category": "error-resolution"}
-"""
-
-import argparse
-import json
-import math
-import sys
-from collections import defaultdict
-from pathlib import Path
-
-
-def load_benchmark(path: str) -> list[dict]:
-    """Load benchmark QA pairs from JSONL."""
-    items = []
-    with open(path) as f:
-        for line in f:
-            line = line.strip()
-            if line:
-                items.append(json.loads(line))
-    return items
-
-
-def extract_keywords(query: str) -> list[str]:
-    """Extract search keywords from a natural language query."""
-    import re
-    words = re.findall(r"[a-zA-Z][a-zA-Z0-9-]+", query.lower())
-    stopwords = {"the", "is", "in", "on", "for", "and", "or", "of", "to", "a", "an", "it", "was", "not", "how", "what", "when", "does"}
-    return [w for w in words if len(w) > 2 and w not in stopwords]
-
-
-def dcg_at_k(relevant_positions: list[int], k: int) -> float:
-    """Compute Discounted Cumulative Gain at k."""
-    score = 0.0
-    for pos in relevant_positions:
-        if pos < k:
-            score += 1.0 / math.log2(pos + 2)  # +2 because position is 0-indexed
-    return score
-
-
-def ndcg_at_k(relevant_positions: list[int], num_relevant: int, k: int) -> float:
-    """Compute Normalized DCG at k."""
-    dcg = dcg_at_k(relevant_positions, k)
-    # Ideal DCG: all relevant items at top positions
-    ideal_positions = list(range(min(num_relevant, k)))
-    idcg = dcg_at_k(ideal_positions, k)
-    return dcg / idcg if idcg > 0 else 0.0
-
-
-def evaluate_retrieval(
-    benchmark_path: str,
-    db_path: str,
-    k_values: list[int] = [1, 3, 5],
-) -> dict:
-    """Run benchmark queries against graph_query.py, compute Recall@k and NDCG@k."""
-    # Import graph_query from parent dir
-    sys.path.insert(0, str(Path(__file__).parent.parent))
-    from graph_query import query_facts_by_entities
-
-    items = load_benchmark(benchmark_path)
-    if not items:
-        return {"error": "Empty benchmark dataset"}
-
-    max_k = max(k_values)
-    metrics: dict[str, list[float]] = defaultdict(list)
-    category_metrics: dict[str, dict[str, list[float]]] = defaultdict(lambda: defaultdict(list))
-    details: list[dict] = []
-
-    for item in items:
-        query = item["query"]
-        expected = set(item.get("expected_entities", []))
-        category = item.get("category", "general")
-        keywords = extract_keywords(query)
-
-        if not keywords or not expected:
-            continue
-
-        results = query_facts_by_entities(db_path, keywords, max_facts=max_k)
-        result_ids = [r["entityId"] for r in results]
-
-        # Find positions of relevant results
-        relevant_positions = []
-        for i, rid in enumerate(result_ids):
-            if rid in expected:
-                relevant_positions.append(i)
-
-        for k in k_values:
-            hit = any(pos < k for pos in relevant_positions)
-            recall = 1.0 if hit else 0.0
-            ndcg = ndcg_at_k(relevant_positions, len(expected), k)
-
-            metrics[f"recall@{k}"].append(recall)
-            metrics[f"ndcg@{k}"].append(ndcg)
-            category_metrics[category][f"recall@{k}"].append(recall)
-            category_metrics[category][f"ndcg@{k}"].append(ndcg)
-
-        details.append({
-            "query": query,
-            "category": category,
-            "expected": list(expected),
-            "retrieved": result_ids[:max_k],
-            "hit@1": any(pos < 1 for pos in relevant_positions),
-            "hit@5": any(pos < 5 for pos in relevant_positions),
-        })
-
-    # Aggregate
-    summary = {
-        "total_queries": len(items),
-        "evaluated": len(details),
-    }
-    for metric_name, values in sorted(metrics.items()):
-        summary[metric_name] = round(sum(values) / len(values), 4) if values else 0.0
-
-    # Per-category breakdown
-    categories = {}
-    for cat, cat_metrics in sorted(category_metrics.items()):
-        categories[cat] = {
-            "count": len(next(iter(cat_metrics.values()))),
-        }
-        for metric_name, values in sorted(cat_metrics.items()):
-            categories[cat][metric_name] = round(sum(values) / len(values), 4) if values else 0.0
-
-    return {
-        "summary": summary,
-        "categories": categories,
-        "details": details,
-    }
-
-
-def format_report_text(result: dict) -> str:
-    """Format evaluation result as human-readable text for daily report injection."""
-    lines = ["## Retrieval Quality"]
-    s = result["summary"]
-    for key in sorted(s):
-        if key.startswith("recall@") or key.startswith("ndcg@"):
-            lines.append(f"- {key}: {s[key]:.2%}")
-
-    if result.get("categories"):
-        lines.append("")
-        lines.append("**By category:**")
-        for cat, cm in sorted(result["categories"].items()):
-            r5 = cm.get("recall@5", 0)
-            lines.append(f"- {cat} (n={cm['count']}): recall@5={r5:.0%}")
-
-    # Weakest category
-    cats = result.get("categories", {})
-    if cats:
-        weakest = min(cats.items(), key=lambda x: x[1].get("recall@5", 1.0))
-        if weakest[1].get("recall@5", 1.0) < 0.8:
-            lines.append(f"\n**Weakest**: {weakest[0]} ({weakest[1].get('recall@5', 0):.0%})")
-
-    return "\n".join(lines)
-
-
-def main() -> None:
-    parser = argparse.ArgumentParser(description="Retrieval Quality Evaluator")
-    parser.add_argument("--db", required=True, help="Path to knowledge-graph.db")
-    parser.add_argument("--benchmark", required=True, help="Path to retrieval_benchmark.jsonl")
-    parser.add_argument("--k", default="1,3,5", help="Comma-separated k values for Recall@k")
-    parser.add_argument("--format", choices=["json", "text"], default="json", help="Output format")
-    args = parser.parse_args()
-
-    k_values = [int(k) for k in args.k.split(",")]
-    result = evaluate_retrieval(args.benchmark, args.db, k_values)
-
-    if args.format == "text":
-        print(format_report_text(result))
-    else:
-        print(json.dumps(result, indent=2, ensure_ascii=False))
-
-
-if __name__ == "__main__":
-    main()

package/sinain-memory/eval/schemas.py

@@ -1,247 +0,0 @@
-"""JSON Schema definitions for all sinain-koog script outputs.
-
-Each schema corresponds to the JSON printed by output_json() in its respective
-script.  Used by tick_evaluator.py for mechanical validation (Tier 1 eval).
-"""
-
-import json
-from typing import Any
-
-
-# ---------------------------------------------------------------------------
-# signal_analyzer.py output
-# ---------------------------------------------------------------------------
-
-SIGNAL_ANALYZER_SCHEMA: dict = {
-    "type": "object",
-    "required": ["signals", "recommendedAction", "idle"],
-    "properties": {
-        "signals": {
-            "type": "array",
-            "items": {"type": "string"},
-        },
-        "recommendedAction": {
-            "oneOf": [
-                {"type": "null"},
-                {
-                    "type": "object",
-                    "required": ["action"],
-                    "properties": {
-                        "action": {"enum": ["sessions_spawn", "telegram_tip", "skip"]},
-                        "task": {"type": "string"},
-                        "confidence": {"type": "number", "minimum": 0, "maximum": 1},
-                    },
-                },
-            ],
-        },
-        "idle": {"type": "boolean"},
-    },
-}
-
-# ---------------------------------------------------------------------------
-# feedback_analyzer.py output
-# ---------------------------------------------------------------------------
-
-FEEDBACK_ANALYZER_SCHEMA: dict = {
-    "type": "object",
-    "required": ["feedbackScores", "effectiveness", "curateDirective"],
-    "properties": {
-        "feedbackScores": {
-            "type": "object",
-            "required": ["avg"],
-            "properties": {
-                "avg": {"type": "number"},
-                "high": {"type": "array", "items": {"type": "string"}},
-                "low": {"type": "array", "items": {"type": "string"}},
-            },
-        },
-        "effectiveness": {
-            "type": "object",
-            "required": ["outputs", "positive", "negative", "neutral", "rate"],
-            "properties": {
-                "outputs": {"type": "integer", "minimum": 0},
-                "positive": {"type": "integer", "minimum": 0},
-                "negative": {"type": "integer", "minimum": 0},
-                "neutral": {"type": "integer", "minimum": 0},
-                "rate": {"type": "number", "minimum": 0, "maximum": 1},
-            },
-        },
-        "curateDirective": {
-            "enum": ["aggressive_prune", "normal", "stability", "insufficient_data"],
-        },
-        "interpretation": {"type": "string"},
-    },
-}
-
-# ---------------------------------------------------------------------------
-# memory_miner.py output
-# ---------------------------------------------------------------------------
-
-MEMORY_MINER_SCHEMA: dict = {
-    "type": "object",
-    "required": ["findings", "newPatterns"],
-    "properties": {
-        "findings": {"type": "string"},
-        "newPatterns": {"type": "array", "items": {"type": "string"}},
-        "contradictions": {"type": "array", "items": {"type": "string"}},
-        "preferences": {"type": "array", "items": {"type": "string"}},
-        "minedSources": {"type": "array", "items": {"type": "string"}},
-    },
-}
-
-# ---------------------------------------------------------------------------
-# playbook_curator.py output
-# ---------------------------------------------------------------------------
-
-PLAYBOOK_CURATOR_SCHEMA: dict = {
-    "type": "object",
-    "required": ["changes", "playbookLines"],
-    "properties": {
-        "changes": {
-            "type": "object",
-            "required": ["added", "pruned", "promoted"],
-            "properties": {
-                "added": {"type": "array", "items": {"type": "string"}},
-                "pruned": {"type": "array", "items": {"type": "string"}},
-                "promoted": {"type": "array", "items": {"type": "string"}},
-            },
-        },
-        "staleItemActions": {"type": "array", "items": {"type": "string"}},
-        "playbookLines": {"type": "integer", "minimum": 0},
-        "error": {"type": "string"},
-    },
-}
-
-# ---------------------------------------------------------------------------
-# insight_synthesizer.py output (non-skip case)
-# ---------------------------------------------------------------------------
-
-INSIGHT_SYNTHESIZER_SCHEMA: dict = {
-    "type": "object",
-    "required": ["skip"],
-    "properties": {
-        "skip": {"type": "boolean"},
-        "suggestion": {"type": "string"},
-        "insight": {"type": "string"},
-        "totalChars": {"type": "integer", "minimum": 0},
-        "skipReason": {"type": "string"},
-    },
-}
-
-# ---------------------------------------------------------------------------
-# module_manager.py extract output
-# ---------------------------------------------------------------------------
-
-MODULE_EXTRACT_SCHEMA: dict = {
-    "type": "object",
-    "required": ["extracted", "domain", "status"],
-    "properties": {
-        "extracted": {"type": "string"},
-        "domain": {"type": "string"},
-        "patternsEstablished": {"type": "integer", "minimum": 0},
-        "patternsEmerging": {"type": "integer", "minimum": 0},
-        "vocabularyTerms": {"type": "integer", "minimum": 0},
-        "modulePath": {"type": "string"},
-        "status": {"enum": ["suspended", "active"]},
-        "activateWith": {"type": "string"},
-    },
-}
-
-
-# ---------------------------------------------------------------------------
-# Registry: script name → schema
-# ---------------------------------------------------------------------------
-
-SCHEMA_REGISTRY: dict[str, dict] = {
-    "signal_analyzer": SIGNAL_ANALYZER_SCHEMA,
-    "feedback_analyzer": FEEDBACK_ANALYZER_SCHEMA,
-    "memory_miner": MEMORY_MINER_SCHEMA,
-    "playbook_curator": PLAYBOOK_CURATOR_SCHEMA,
-    "insight_synthesizer": INSIGHT_SYNTHESIZER_SCHEMA,
-    "module_extract": MODULE_EXTRACT_SCHEMA,
-}
-
-
-# ---------------------------------------------------------------------------
-# Lightweight JSON Schema validator (no external dependency)
-# ---------------------------------------------------------------------------
-
-def validate(instance: Any, schema: dict) -> list[str]:
-    """Validate *instance* against a JSON Schema subset.
-
-    Returns a list of error strings (empty = valid).  Supports:
-    type, required, properties, items, enum, oneOf, minimum, maximum.
-    """
-    errors: list[str] = []
-    _validate(instance, schema, "", errors)
-    return errors
-
-
-def _validate(instance: Any, schema: dict, path: str, errors: list[str]) -> None:
-    # --- oneOf ---
-    if "oneOf" in schema:
-        matches = 0
-        for sub in schema["oneOf"]:
-            sub_errors: list[str] = []
-            _validate(instance, sub, path, sub_errors)
-            if not sub_errors:
-                matches += 1
-        if matches == 0:
-            errors.append(f"{path or '.'}: does not match any oneOf variant")
-        return
-
-    # --- enum ---
-    if "enum" in schema:
-        if instance not in schema["enum"]:
-            errors.append(f"{path or '.'}: {instance!r} not in {schema['enum']}")
-        return
-
-    # --- type ---
-    expected_type = schema.get("type")
-    if expected_type:
-        ok = _type_check(instance, expected_type)
-        if not ok:
-            errors.append(f"{path or '.'}: expected {expected_type}, got {type(instance).__name__}")
-            return
-
-    # --- required ---
-    if "required" in schema and isinstance(instance, dict):
-        for key in schema["required"]:
-            if key not in instance:
-                errors.append(f"{path}.{key}: required field missing")
-
-    # --- properties ---
-    if "properties" in schema and isinstance(instance, dict):
-        for key, sub_schema in schema["properties"].items():
-            if key in instance:
-                _validate(instance[key], sub_schema, f"{path}.{key}", errors)
-
-    # --- items ---
-    if "items" in schema and isinstance(instance, list):
-        for i, item in enumerate(instance):
-            _validate(item, schema["items"], f"{path}[{i}]", errors)
-
-    # --- minimum / maximum ---
-    if isinstance(instance, (int, float)):
-        if "minimum" in schema and instance < schema["minimum"]:
-            errors.append(f"{path or '.'}: {instance} < minimum {schema['minimum']}")
-        if "maximum" in schema and instance > schema["maximum"]:
-            errors.append(f"{path or '.'}: {instance} > maximum {schema['maximum']}")
-
-
-def _type_check(instance: Any, expected: str) -> bool:
-    if expected == "object":
-        return isinstance(instance, dict)
-    if expected == "array":
-        return isinstance(instance, list)
-    if expected == "string":
-        return isinstance(instance, str)
-    if expected == "number":
-        return isinstance(instance, (int, float))
-    if expected == "integer":
-        return isinstance(instance, int) and not isinstance(instance, bool)
-    if expected == "boolean":
-        return isinstance(instance, bool)
-    if expected == "null":
-        return instance is None
-    return True