@geravant/sinain 1.0.1

package/sinain-memory/eval/judges/base_judge.py

@@ -0,0 +1,61 @@
+"""Shared infrastructure for LLM-as-Judge evaluators.
+
+Provides ``run_judge()`` which calls the LLM with a rubric prompt and
+extracts a ``{"score": 1-4, "reasoning": "..."}`` response.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+# Add parent dirs so ``common`` is importable when running from anywhere.
+_koog_dir = str(Path(__file__).resolve().parent.parent.parent)
+if _koog_dir not in sys.path:
+    sys.path.insert(0, _koog_dir)
+
+from common import LLMError, call_llm, extract_json  # noqa: E402
+
+
+def run_judge(
+    system_prompt: str,
+    user_prompt: str,
+    *,
+    model: str | None = None,
+    max_tokens: int = 200,
+    timeout: int = 30,
+) -> dict | None:
+    """Call LLM with a judge prompt and return ``{"score": int, "reasoning": str}`` or None.
+
+    *model* defaults to the ``eval.judges.model`` setting resolved externally.
+    When None, falls back to ``common.call_llm`` defaults (which reads koog-config).
+    """
+    try:
+        kwargs: dict = {
+            "system_prompt": system_prompt,
+            "user_prompt": user_prompt,
+            "max_tokens": max_tokens,
+            "json_mode": True,
+        }
+        # Use script-based config resolution if no explicit model
+        if model:
+            kwargs["model"] = model
+        else:
+            kwargs["script"] = "tick_evaluator"
+
+        raw = call_llm(**kwargs)
+        result = extract_json(raw)
+
+        score = result.get("score")
+        reasoning = result.get("reasoning", "")
+
+        if not isinstance(score, (int, float)) or not (1 <= score <= 4):
+            print(f"[warn] judge returned invalid score: {score}", file=sys.stderr)
+            return None
+
+        return {"score": int(score), "reasoning": str(reasoning)[:300]}
+
+    except (ValueError, LLMError, KeyError) as e:
+        print(f"[warn] judge call failed: {e}", file=sys.stderr)
+        return None

package/sinain-memory/eval/judges/curation_judge.py

@@ -0,0 +1,46 @@
+"""LLM-as-Judge: Playbook curation quality evaluator."""
+
+from __future__ import annotations
+
+from .base_judge import run_judge
+
+SYSTEM_PROMPT = """\
+You are an evaluator scoring the quality of playbook curation changes.
+
+The curator follows a directive and three laws:
+  Law 1: Don't remove error-prevention patterns
+  Law 2: Preserve high-scoring approaches
+  Law 3: Then evolve
+
+Rate the curation on a 1-4 scale:
+  4: Changes perfectly match directive + evidence, three laws respected
+  3: Good changes, minor alignment issues with directive
+  2: Changes misaligned with directive or weak evidence
+  1: Destructive changes, violated three laws, or ignored directive entirely
+
+Respond with ONLY a JSON object: {"score": <1-4>, "reasoning": "brief explanation"}"""
+
+
+def judge_curation(
+    curator_result: dict,
+    directive: str,
+    playbook_before: str = "",
+    **kwargs,
+) -> dict | None:
+    """Evaluate playbook curation quality. Returns {"score": 1-4, "reasoning": str} or None."""
+    changes = curator_result.get("changes", {})
+    stale_actions = curator_result.get("staleItemActions", [])
+    lines = curator_result.get("playbookLines", "?")
+
+    parts = [
+        f"## Curate Directive\n{directive}",
+        f"\n## Changes Made\nAdded: {changes.get('added', [])}\nPruned: {changes.get('pruned', [])}\nPromoted: {changes.get('promoted', [])}",
+        f"\n## Stale Item Actions\n{stale_actions}",
+        f"\n## Playbook Lines After: {lines}",
+    ]
+
+    if playbook_before:
+        # Truncate to keep prompt manageable
+        parts.append(f"\n## Playbook Before (excerpt)\n{playbook_before[:1500]}")
+
+    return run_judge(SYSTEM_PROMPT, "\n".join(parts), **kwargs)

package/sinain-memory/eval/judges/insight_judge.py

@@ -0,0 +1,48 @@
+"""LLM-as-Judge: Insight synthesis quality evaluator."""
+
+from __future__ import annotations
+
+from .base_judge import run_judge
+
+SYSTEM_PROMPT = """\
+You are an evaluator scoring the quality of an insight synthesizer's output.
+
+The synthesizer produces two parts:
+  - Suggestion: actionable recommendation grounded in playbook/data
+  - Insight: surprising cross-domain connection from accumulated observations
+
+Rate the output on a 1-4 scale:
+  4: Suggestion is actionable with specific reference, insight connects 2+ distinct observations
+  3: One component is excellent, the other adequate
+  2: Generic suggestion or obvious insight
+  1: Hallucinated content, not grounded in playbook/logs
+
+If the output was skipped, rate the skip decision:
+  4: Skip is well-justified with specific references to what was checked
+  3: Skip is reasonable
+  2: Should not have skipped — there was material to work with
+  1: Skip reason is generic/lazy
+
+Respond with ONLY a JSON object: {"score": <1-4>, "reasoning": "brief explanation"}"""
+
+
+def judge_insight(
+    synth_result: dict,
+    playbook_excerpt: str = "",
+    **kwargs,
+) -> dict | None:
+    """Evaluate insight synthesis quality. Returns {"score": 1-4, "reasoning": str} or None."""
+    skipped = synth_result.get("skip", False)
+
+    parts = []
+    if skipped:
+        parts.append(f"## Status: SKIPPED\nReason: {synth_result.get('skipReason', 'none given')}")
+    else:
+        parts.append(f"## Suggestion\n{synth_result.get('suggestion', '')}")
+        parts.append(f"\n## Insight\n{synth_result.get('insight', '')}")
+        parts.append(f"\n## Total Chars: {synth_result.get('totalChars', '?')}")
+
+    if playbook_excerpt:
+        parts.append(f"\n## Playbook Context (excerpt)\n{playbook_excerpt[:1000]}")
+
+    return run_judge(SYSTEM_PROMPT, "\n".join(parts), **kwargs)

package/sinain-memory/eval/judges/mining_judge.py

@@ -0,0 +1,42 @@
+"""LLM-as-Judge: Memory mining quality evaluator."""
+
+from __future__ import annotations
+
+from .base_judge import run_judge
+
+SYSTEM_PROMPT = """\
+You are an evaluator scoring the quality of a memory mining agent's findings.
+
+The miner reads daily memory files and extracts patterns, preferences, and insights
+that should be added to the evolving playbook.
+
+Rate the mining output on a 1-4 scale:
+  4: Found non-obvious cross-day patterns, all grounded in source files
+  3: Valid patterns found, properly grounded in provided daily files
+  2: Only surface-level observations from source files
+  1: Hallucinated patterns not present in provided daily files
+
+Respond with ONLY a JSON object: {"score": <1-4>, "reasoning": "brief explanation"}"""
+
+
+def judge_mining(
+    miner_result: dict,
+    mined_file_excerpts: dict[str, str] | None = None,
+    **kwargs,
+) -> dict | None:
+    """Evaluate memory mining quality. Returns {"score": 1-4, "reasoning": str} or None."""
+    parts = [
+        f"## Findings\n{miner_result.get('findings', '')}",
+        f"\n## New Patterns\n{miner_result.get('newPatterns', [])}",
+        f"\n## Contradictions\n{miner_result.get('contradictions', [])}",
+        f"\n## Preferences\n{miner_result.get('preferences', [])}",
+        f"\n## Mined Sources\n{miner_result.get('minedSources', [])}",
+    ]
+
+    if mined_file_excerpts:
+        for name, content in mined_file_excerpts.items():
+            # Truncate large files
+            excerpt = content[:1500] if len(content) > 1500 else content
+            parts.append(f"\n## Source File: {name}\n{excerpt}")
+
+    return run_judge(SYSTEM_PROMPT, "\n".join(parts), **kwargs)

package/sinain-memory/eval/judges/signal_judge.py

@@ -0,0 +1,45 @@
+"""LLM-as-Judge: Signal detection quality evaluator."""
+
+from __future__ import annotations
+
+from .base_judge import run_judge
+
+SYSTEM_PROMPT = """\
+You are an evaluator scoring the quality of a signal detection system.
+
+Rate the signal analysis on a 1-4 scale:
+  4: All real signals detected, action is highly relevant and specific
+  3: Key signals detected, action is reasonable
+  2: Missed important signals or action is vague
+  1: Hallucinated signals or inappropriate action
+
+Respond with ONLY a JSON object: {"score": <1-4>, "reasoning": "brief explanation"}"""
+
+
+def judge_signal(
+    signal_result: dict,
+    session_summary: str,
+    recent_logs: list[dict] | None = None,
+    **kwargs,
+) -> dict | None:
+    """Evaluate signal detection quality. Returns {"score": 1-4, "reasoning": str} or None."""
+    parts = [f"## Session Summary\n{session_summary}"]
+
+    signals = signal_result.get("signals", [])
+    action = signal_result.get("recommendedAction")
+    idle = signal_result.get("idle", False)
+
+    parts.append(f"\n## Detected Signals\n{signals}")
+    parts.append(f"\n## Recommended Action\n{action}")
+    parts.append(f"\n## Idle: {idle}")
+
+    if recent_logs:
+        recent_actions = []
+        for log in recent_logs[:3]:
+            for a in log.get("actionsConsidered", []):
+                if a.get("chosen"):
+                    recent_actions.append(a)
+        if recent_actions:
+            parts.append(f"\n## Recent Actions (should not repeat)\n{recent_actions}")
+
+    return run_judge(SYSTEM_PROMPT, "\n".join(parts), **kwargs)

package/sinain-memory/eval/schemas.py

@@ -0,0 +1,247 @@
+"""JSON Schema definitions for all sinain-koog script outputs.
+
+Each schema corresponds to the JSON printed by output_json() in its respective
+script.  Used by tick_evaluator.py for mechanical validation (Tier 1 eval).
+"""
+
+import json
+from typing import Any
+
+
+# ---------------------------------------------------------------------------
+# signal_analyzer.py output
+# ---------------------------------------------------------------------------
+
+SIGNAL_ANALYZER_SCHEMA: dict = {
+    "type": "object",
+    "required": ["signals", "recommendedAction", "idle"],
+    "properties": {
+        "signals": {
+            "type": "array",
+            "items": {"type": "string"},
+        },
+        "recommendedAction": {
+            "oneOf": [
+                {"type": "null"},
+                {
+                    "type": "object",
+                    "required": ["action"],
+                    "properties": {
+                        "action": {"enum": ["sessions_spawn", "telegram_tip", "skip"]},
+                        "task": {"type": "string"},
+                        "confidence": {"type": "number", "minimum": 0, "maximum": 1},
+                    },
+                },
+            ],
+        },
+        "idle": {"type": "boolean"},
+    },
+}
+
+# ---------------------------------------------------------------------------
+# feedback_analyzer.py output
+# ---------------------------------------------------------------------------
+
+FEEDBACK_ANALYZER_SCHEMA: dict = {
+    "type": "object",
+    "required": ["feedbackScores", "effectiveness", "curateDirective"],
+    "properties": {
+        "feedbackScores": {
+            "type": "object",
+            "required": ["avg"],
+            "properties": {
+                "avg": {"type": "number"},
+                "high": {"type": "array", "items": {"type": "string"}},
+                "low": {"type": "array", "items": {"type": "string"}},
+            },
+        },
+        "effectiveness": {
+            "type": "object",
+            "required": ["outputs", "positive", "negative", "neutral", "rate"],
+            "properties": {
+                "outputs": {"type": "integer", "minimum": 0},
+                "positive": {"type": "integer", "minimum": 0},
+                "negative": {"type": "integer", "minimum": 0},
+                "neutral": {"type": "integer", "minimum": 0},
+                "rate": {"type": "number", "minimum": 0, "maximum": 1},
+            },
+        },
+        "curateDirective": {
+            "enum": ["aggressive_prune", "normal", "stability", "insufficient_data"],
+        },
+        "interpretation": {"type": "string"},
+    },
+}
+
+# ---------------------------------------------------------------------------
+# memory_miner.py output
+# ---------------------------------------------------------------------------
+
+MEMORY_MINER_SCHEMA: dict = {
+    "type": "object",
+    "required": ["findings", "newPatterns"],
+    "properties": {
+        "findings": {"type": "string"},
+        "newPatterns": {"type": "array", "items": {"type": "string"}},
+        "contradictions": {"type": "array", "items": {"type": "string"}},
+        "preferences": {"type": "array", "items": {"type": "string"}},
+        "minedSources": {"type": "array", "items": {"type": "string"}},
+    },
+}
+
+# ---------------------------------------------------------------------------
+# playbook_curator.py output
+# ---------------------------------------------------------------------------
+
+PLAYBOOK_CURATOR_SCHEMA: dict = {
+    "type": "object",
+    "required": ["changes", "playbookLines"],
+    "properties": {
+        "changes": {
+            "type": "object",
+            "required": ["added", "pruned", "promoted"],
+            "properties": {
+                "added": {"type": "array", "items": {"type": "string"}},
+                "pruned": {"type": "array", "items": {"type": "string"}},
+                "promoted": {"type": "array", "items": {"type": "string"}},
+            },
+        },
+        "staleItemActions": {"type": "array", "items": {"type": "string"}},
+        "playbookLines": {"type": "integer", "minimum": 0},
+        "error": {"type": "string"},
+    },
+}
+
+# ---------------------------------------------------------------------------
+# insight_synthesizer.py output (non-skip case)
+# ---------------------------------------------------------------------------
+
+INSIGHT_SYNTHESIZER_SCHEMA: dict = {
+    "type": "object",
+    "required": ["skip"],
+    "properties": {
+        "skip": {"type": "boolean"},
+        "suggestion": {"type": "string"},
+        "insight": {"type": "string"},
+        "totalChars": {"type": "integer", "minimum": 0},
+        "skipReason": {"type": "string"},
+    },
+}
+
+# ---------------------------------------------------------------------------
+# module_manager.py extract output
+# ---------------------------------------------------------------------------
+
+MODULE_EXTRACT_SCHEMA: dict = {
+    "type": "object",
+    "required": ["extracted", "domain", "status"],
+    "properties": {
+        "extracted": {"type": "string"},
+        "domain": {"type": "string"},
+        "patternsEstablished": {"type": "integer", "minimum": 0},
+        "patternsEmerging": {"type": "integer", "minimum": 0},
+        "vocabularyTerms": {"type": "integer", "minimum": 0},
+        "modulePath": {"type": "string"},
+        "status": {"enum": ["suspended", "active"]},
+        "activateWith": {"type": "string"},
+    },
+}
+
+
+# ---------------------------------------------------------------------------
+# Registry: script name → schema
+# ---------------------------------------------------------------------------
+
+SCHEMA_REGISTRY: dict[str, dict] = {
+    "signal_analyzer": SIGNAL_ANALYZER_SCHEMA,
+    "feedback_analyzer": FEEDBACK_ANALYZER_SCHEMA,
+    "memory_miner": MEMORY_MINER_SCHEMA,
+    "playbook_curator": PLAYBOOK_CURATOR_SCHEMA,
+    "insight_synthesizer": INSIGHT_SYNTHESIZER_SCHEMA,
+    "module_extract": MODULE_EXTRACT_SCHEMA,
+}
+
+
+# ---------------------------------------------------------------------------
+# Lightweight JSON Schema validator (no external dependency)
+# ---------------------------------------------------------------------------
+
+def validate(instance: Any, schema: dict) -> list[str]:
+    """Validate *instance* against a JSON Schema subset.
+
+    Returns a list of error strings (empty = valid).  Supports:
+    type, required, properties, items, enum, oneOf, minimum, maximum.
+    """
+    errors: list[str] = []
+    _validate(instance, schema, "", errors)
+    return errors
+
+
+def _validate(instance: Any, schema: dict, path: str, errors: list[str]) -> None:
+    # --- oneOf ---
+    if "oneOf" in schema:
+        matches = 0
+        for sub in schema["oneOf"]:
+            sub_errors: list[str] = []
+            _validate(instance, sub, path, sub_errors)
+            if not sub_errors:
+                matches += 1
+        if matches == 0:
+            errors.append(f"{path or '.'}: does not match any oneOf variant")
+        return
+
+    # --- enum ---
+    if "enum" in schema:
+        if instance not in schema["enum"]:
+            errors.append(f"{path or '.'}: {instance!r} not in {schema['enum']}")
+        return
+
+    # --- type ---
+    expected_type = schema.get("type")
+    if expected_type:
+        ok = _type_check(instance, expected_type)
+        if not ok:
+            errors.append(f"{path or '.'}: expected {expected_type}, got {type(instance).__name__}")
+            return
+
+    # --- required ---
+    if "required" in schema and isinstance(instance, dict):
+        for key in schema["required"]:
+            if key not in instance:
+                errors.append(f"{path}.{key}: required field missing")
+
+    # --- properties ---
+    if "properties" in schema and isinstance(instance, dict):
+        for key, sub_schema in schema["properties"].items():
+            if key in instance:
+                _validate(instance[key], sub_schema, f"{path}.{key}", errors)
+
+    # --- items ---
+    if "items" in schema and isinstance(instance, list):
+        for i, item in enumerate(instance):
+            _validate(item, schema["items"], f"{path}[{i}]", errors)
+
+    # --- minimum / maximum ---
+    if isinstance(instance, (int, float)):
+        if "minimum" in schema and instance < schema["minimum"]:
+            errors.append(f"{path or '.'}: {instance} < minimum {schema['minimum']}")
+        if "maximum" in schema and instance > schema["maximum"]:
+            errors.append(f"{path or '.'}: {instance} > maximum {schema['maximum']}")
+
+
+def _type_check(instance: Any, expected: str) -> bool:
+    if expected == "object":
+        return isinstance(instance, dict)
+    if expected == "array":
+        return isinstance(instance, list)
+    if expected == "string":
+        return isinstance(instance, str)
+    if expected == "number":
+        return isinstance(instance, (int, float))
+    if expected == "integer":
+        return isinstance(instance, int) and not isinstance(instance, bool)
+    if expected == "boolean":
+        return isinstance(instance, bool)
+    if expected == "null":
+        return instance is None
+    return True

package/sinain-memory/eval_delta.py

@@ -0,0 +1,109 @@
+#!/usr/bin/env python3
+"""Before/after benchmark delta tool.
+
+Loads two daily snapshots and computes per-metric deltas to determine
+whether a change (e.g. prompt tuning) helped or hurt.
+
+Usage:
+    python3 sinain-koog/eval_delta.py --memory-dir memory/ --after 2026-03-08
+    python3 sinain-koog/eval_delta.py --memory-dir memory/ --before 2026-03-06 --after 2026-03-08 --label "tuned insight prompt"
+"""
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+_koog_dir = str(Path(__file__).resolve().parent)
+if _koog_dir not in sys.path:
+    sys.path.insert(0, _koog_dir)
+
+from eval_reporter import compute_delta, load_previous_snapshot
+
+
+def load_snapshot(report_dir: Path, date_str: str) -> dict | None:
+    """Load a snapshot for the given date."""
+    snap_path = report_dir / f"{date_str}.snapshot.json"
+    if not snap_path.exists():
+        return None
+    try:
+        return json.loads(snap_path.read_text(encoding="utf-8"))
+    except (json.JSONDecodeError, OSError):
+        return None
+
+
+def format_human(delta: dict[str, dict], label: str = "") -> str:
+    """Format delta as a human-readable summary."""
+    lines = []
+    if label:
+        lines.append(f"Label: {label}")
+    lines.append("")
+
+    improved = [k for k, v in delta.items() if v["status"] == "IMPROVED"]
+    regressed = [k for k, v in delta.items() if v["status"] == "REGRESSED"]
+    same = [k for k, v in delta.items() if v["status"] == "SAME"]
+
+    for metric, info in sorted(delta.items()):
+        marker = {"IMPROVED": "↑", "REGRESSED": "↓", "SAME": "→"}[info["status"]]
+        sign = "+" if info["delta"] > 0 else ""
+        lines.append(f"  {marker} {metric}: {info['before']} → {info['after']} ({sign}{info['delta']})")
+
+    lines.append("")
+    lines.append(f"Summary: {len(improved)} improved, {len(regressed)} regressed, {len(same)} same")
+
+    if regressed:
+        lines.append(f"Regressions: {', '.join(regressed)}")
+
+    return "\n".join(lines)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Before/after benchmark delta comparison")
+    parser.add_argument("--memory-dir", required=True, help="Path to memory/ directory")
+    parser.add_argument("--before", default=None, help="Date of 'before' snapshot (YYYY-MM-DD). Auto-detected if omitted")
+    parser.add_argument("--after", required=True, help="Date of 'after' snapshot (YYYY-MM-DD)")
+    parser.add_argument("--label", default="", help="Optional label for the change being measured")
+    parser.add_argument("--json", action="store_true", help="Output raw JSON instead of human-readable text")
+    args = parser.parse_args()
+
+    report_dir = Path(args.memory_dir) / "eval-reports"
+    if not report_dir.is_dir():
+        print(f"[eval-delta] report directory not found: {report_dir}", file=sys.stderr)
+        sys.exit(1)
+
+    # Load "after" snapshot
+    after_snap = load_snapshot(report_dir, args.after)
+    if not after_snap:
+        print(f"[eval-delta] no snapshot found for --after {args.after}", file=sys.stderr)
+        sys.exit(1)
+
+    # Load "before" snapshot
+    if args.before:
+        before_date = args.before
+        before_snap = load_snapshot(report_dir, before_date)
+        if not before_snap:
+            print(f"[eval-delta] no snapshot found for --before {before_date}", file=sys.stderr)
+            sys.exit(1)
+    else:
+        before_date, before_snap = load_previous_snapshot(report_dir, args.after)
+        if not before_snap:
+            print(f"[eval-delta] no previous snapshot found before {args.after}", file=sys.stderr)
+            sys.exit(1)
+
+    delta = compute_delta(before_snap, after_snap)
+
+    if args.json:
+        output = {
+            "before": before_date,
+            "after": args.after,
+            "label": args.label,
+            "delta": delta,
+        }
+        print(json.dumps(output, indent=2))
+    else:
+        print(f"Delta: {before_date} → {args.after}")
+        print(format_human(delta, args.label))
+
+
+if __name__ == "__main__":
+    main()