@geravant/sinain 1.0.1

package/sinain-memory/eval_reporter.py

@@ -0,0 +1,642 @@
+#!/usr/bin/env python3
+"""Tier 2 Evaluation: Daily report generator — runs as server cron job (daily 03:00).
+
+Aggregates 24h of eval-logs, computes quality metrics, detects regressions,
+uses LLM to interpret trends and write a daily report to memory/eval-reports/.
+
+Invocation (cron):
+    uv run --with requests python3 sinain-koog/eval_reporter.py \
+        --memory-dir memory/ [--days 1]
+"""
+
+import argparse
+import json
+import sys
+from collections import Counter
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+
+_koog_dir = str(Path(__file__).resolve().parent)
+if _koog_dir not in sys.path:
+    sys.path.insert(0, _koog_dir)
+
+from common import LLMError, _load_config, _read_jsonl, call_llm, extract_json, read_recent_logs
+
+
+# ---------------------------------------------------------------------------
+# Config (duplicated from tick_evaluator to avoid circular import)
+# ---------------------------------------------------------------------------
+
+_EVAL_DEFAULTS = {
+    "level": "mechanical",
+    "sampleRate": 0.2,
+    "judges": {"model": "smart", "maxTokens": 200, "timeout": 30},
+    "dailyReport": True,
+    "regressionThresholds": {
+        "assertionPassRate": 0.85,
+        "effectivenessRate": 0.4,
+        "skipRate": 0.8,
+    },
+}
+
+
+def load_eval_config(memory_dir: str) -> dict:
+    """Load eval config with runtime overrides from memory/eval-config.json."""
+    base = _load_config().get("eval", {})
+    cfg = {**_EVAL_DEFAULTS, **base}
+
+    override_path = Path(memory_dir) / "eval-config.json"
+    if override_path.exists():
+        try:
+            override = json.loads(override_path.read_text(encoding="utf-8"))
+            cfg.update(override)
+        except (json.JSONDecodeError, OSError) as e:
+            print(f"[warn] eval-config.json override failed: {e}", file=sys.stderr)
+
+    return cfg
+
+
+# ---------------------------------------------------------------------------
+# Aggregation
+# ---------------------------------------------------------------------------
+
+def load_eval_logs(memory_dir: str, days: int = 1) -> list[dict]:
+    """Load eval-log entries from the last N days."""
+    log_dir = Path(memory_dir) / "eval-logs"
+    if not log_dir.is_dir():
+        return []
+
+    cutoff = datetime.now(timezone.utc) - timedelta(days=days)
+    entries: list[dict] = []
+
+    for jsonl_file in sorted(log_dir.glob("*.jsonl"), reverse=True):
+        try:
+            file_date = datetime.strptime(jsonl_file.stem, "%Y-%m-%d").replace(tzinfo=timezone.utc)
+        except ValueError:
+            continue
+        if file_date < cutoff - timedelta(days=1):
+            break
+        entries.extend(_read_jsonl(jsonl_file))
+
+    return entries
+
+
+def extract_run_summaries(eval_entries: list[dict]) -> tuple[list[dict], list[dict]]:
+    """Separate run_summary metadata entries from tick eval results.
+
+    Returns (tick_entries, run_summaries).
+    """
+    ticks = []
+    summaries = []
+    for e in eval_entries:
+        if e.get("_type") == "run_summary":
+            summaries.append(e)
+        else:
+            ticks.append(e)
+    return ticks, summaries
+
+
+def compute_aggregates(eval_entries: list[dict]) -> dict:
+    """Compute daily aggregate metrics from eval-log entries."""
+    if not eval_entries:
+        return {"tickCount": 0}
+
+    # Schema validity
+    schema_total = sum(e.get("schema", {}).get("total", 0) for e in eval_entries)
+    schema_valid = sum(e.get("schema", {}).get("valid", 0) for e in eval_entries)
+    schema_rate = round(schema_valid / schema_total, 3) if schema_total > 0 else 1.0
+
+    # Assertion pass rate
+    assert_total = sum(e.get("assertions", {}).get("total", 0) for e in eval_entries)
+    assert_passed = sum(e.get("assertions", {}).get("passed", 0) for e in eval_entries)
+    assert_rate = round(assert_passed / assert_total, 3) if assert_total > 0 else 1.0
+
+    # Assertion failure histogram
+    failure_counter: Counter = Counter()
+    for e in eval_entries:
+        for f in e.get("assertions", {}).get("failures", []):
+            failure_counter[f.get("name", "unknown")] += 1
+
+    # Judge score distribution + sub-score aggregation
+    judge_scores: dict[str, list[int]] = {}
+    sub_scores: dict[str, dict[str, list[int]]] = {}  # {judge: {dim: [scores]}}
+    for e in eval_entries:
+        judges = e.get("judges")
+        if not judges:
+            continue
+        for judge_name, result in judges.items():
+            if isinstance(result, dict) and "score" in result:
+                judge_scores.setdefault(judge_name, []).append(result["score"])
+                # Collect multi-dimensional sub-scores if present
+                scores_dict = result.get("scores")
+                if isinstance(scores_dict, dict):
+                    judge_subs = sub_scores.setdefault(judge_name, {})
+                    for dim, val in scores_dict.items():
+                        if isinstance(val, (int, float)):
+                            judge_subs.setdefault(dim, []).append(int(val))
+
+    judge_avg = None
+    if judge_scores:
+        all_scores = [s for scores in judge_scores.values() for s in scores]
+        judge_avg = round(sum(all_scores) / len(all_scores), 2) if all_scores else None
+
+    # Pass rate trend
+    pass_rates = [e.get("passRate", 1.0) for e in eval_entries]
+    avg_pass_rate = round(sum(pass_rates) / len(pass_rates), 3)
+
+    # Build sub-score summary: {judge: {dim: {count, avg}}}
+    sub_score_summary: dict[str, dict[str, dict]] = {}
+    for judge_name, dims in sub_scores.items():
+        sub_score_summary[judge_name] = {
+            dim: {"count": len(vals), "avg": round(sum(vals) / len(vals), 2)}
+            for dim, vals in dims.items()
+        }
+
+    return {
+        "tickCount": len(eval_entries),
+        "schemaValidity": {"total": schema_total, "valid": schema_valid, "rate": schema_rate},
+        "assertionPassRate": {"total": assert_total, "passed": assert_passed, "rate": assert_rate},
+        "failureHistogram": dict(failure_counter.most_common(10)),
+        "judgeScores": {k: {"count": len(v), "avg": round(sum(v) / len(v), 2), "dist": dict(Counter(v))}
+                        for k, v in judge_scores.items()},
+        "judgeAvg": judge_avg,
+        "subScores": sub_score_summary,
+        "avgPassRate": avg_pass_rate,
+    }
+
+
+def compute_playbook_health(playbook_logs: list[dict]) -> dict:
+    """Compute playbook health metrics from heartbeat logs."""
+    line_counts: list[int] = []
+    total_added = 0
+    total_pruned = 0
+
+    for entry in playbook_logs:
+        changes = entry.get("playbookChanges", {})
+        if isinstance(changes, dict):
+            pl = changes.get("playbookLines")
+            if isinstance(pl, int):
+                line_counts.append(pl)
+            total_added += len(changes.get("added", []))
+            total_pruned += len(changes.get("pruned", []))
+
+    tick_count = len(playbook_logs) or 1
+    return {
+        "lineCountTrend": line_counts[-5:] if line_counts else [],
+        "avgChurnPerTick": round((total_added + total_pruned) / tick_count, 1),
+        "totalAdded": total_added,
+        "totalPruned": total_pruned,
+    }
+
+
+def _percentile(sorted_vals: list[float], p: float) -> float:
+    """Compute the p-th percentile (0-100) from a pre-sorted list."""
+    if not sorted_vals:
+        return 0.0
+    k = (len(sorted_vals) - 1) * p / 100.0
+    f = int(k)
+    c = f + 1 if f + 1 < len(sorted_vals) else f
+    return round(sorted_vals[f] + (k - f) * (sorted_vals[c] - sorted_vals[f]), 1)
+
+
+def compute_latency_stats(playbook_logs: list[dict]) -> dict[str, dict]:
+    """Aggregate per-script latency statistics from playbook-log entries.
+
+    Returns {scriptName: {count, avg, p50, p95}} for each script key found
+    in the latencyMs field, plus a "total" entry for totalLatencyMs.
+    """
+    buckets: dict[str, list[float]] = {}
+    for entry in playbook_logs:
+        lat = entry.get("latencyMs")
+        if isinstance(lat, dict):
+            for script, ms in lat.items():
+                if isinstance(ms, (int, float)):
+                    buckets.setdefault(script, []).append(float(ms))
+        total = entry.get("totalLatencyMs")
+        if isinstance(total, (int, float)):
+            buckets.setdefault("total", []).append(float(total))
+
+    stats: dict[str, dict] = {}
+    for name, vals in buckets.items():
+        vals.sort()
+        stats[name] = {
+            "count": len(vals),
+            "avg": round(sum(vals) / len(vals), 1),
+            "p50": _percentile(vals, 50),
+            "p95": _percentile(vals, 95),
+        }
+    return stats
+
+
+def compute_skip_rate(playbook_logs: list[dict]) -> float:
+    """Compute the insight synthesizer skip rate."""
+    total = 0
+    skipped = 0
+    for entry in playbook_logs:
+        output = entry.get("output")
+        if output is not None:
+            total += 1
+            if output.get("skip", False):
+                skipped += 1
+    return round(skipped / total, 2) if total > 0 else 0.0
+
+
+# ---------------------------------------------------------------------------
+# Regression detection
+# ---------------------------------------------------------------------------
+
+def detect_regressions(aggregates: dict, thresholds: dict, skip_rate: float) -> list[str]:
+    """Detect regressions based on thresholds."""
+    regressions: list[str] = []
+
+    assert_rate = aggregates.get("assertionPassRate", {}).get("rate", 1.0)
+    if assert_rate < thresholds.get("assertionPassRate", 0.85):
+        regressions.append(
+            f"Assertion pass rate {assert_rate:.1%} below threshold {thresholds['assertionPassRate']:.0%}"
+        )
+
+    if skip_rate > thresholds.get("skipRate", 0.8):
+        regressions.append(
+            f"Skip rate {skip_rate:.0%} above threshold {thresholds['skipRate']:.0%} — synthesizer rarely producing output"
+        )
+
+    # Repeated failures
+    histogram = aggregates.get("failureHistogram", {})
+    for name, count in histogram.items():
+        if count >= 3:
+            regressions.append(f"Assertion '{name}' failed {count} times (systemic issue)")
+
+    return regressions
+
+
+# ---------------------------------------------------------------------------
+# Report generation
+# ---------------------------------------------------------------------------
+
+def generate_report_markdown(
+    date_str: str,
+    aggregates: dict,
+    playbook_health: dict,
+    skip_rate: float,
+    regressions: list[str],
+    llm_interpretation: str = "",
+    run_summaries: list[dict] | None = None,
+    latency_stats: dict[str, dict] | None = None,
+) -> str:
+    """Generate the daily eval report as markdown."""
+    lines: list[str] = []
+    lines.append(f"# Eval Report — {date_str}\n")
+
+    # Quality Gates
+    lines.append("## Quality Gates")
+    schema = aggregates.get("schemaValidity", {})
+    s_rate = schema.get("rate", 1.0)
+    s_icon = "✓" if s_rate >= 0.95 else "⚠"
+    lines.append(f"- {s_icon} Schema validity: {s_rate:.0%} ({schema.get('valid', 0)}/{schema.get('total', 0)} checks)")
+
+    a = aggregates.get("assertionPassRate", {})
+    a_rate = a.get("rate", 1.0)
+    a_icon = "✓" if a_rate >= 0.85 else "⚠"
+    lines.append(f"- {a_icon} Assertion pass rate: {a_rate:.0%} ({a.get('passed', 0)}/{a.get('total', 0)} checks)")
+
+    j_avg = aggregates.get("judgeAvg")
+    if j_avg is not None:
+        j_icon = "✓" if j_avg >= 3.0 else "⚠"
+        judge_count = sum(v.get("count", 0) for v in aggregates.get("judgeScores", {}).values())
+        lines.append(f"- {j_icon} Mean judge score: {j_avg}/4.0 ({judge_count} evaluations)")
+
+    skip_icon = "✓" if skip_rate < 0.8 else "⚠"
+    lines.append(f"- {skip_icon} Skip rate: {skip_rate:.0%}")
+    lines.append(f"- Ticks evaluated: {aggregates.get('tickCount', 0)}")
+
+    # Partial run warning
+    if run_summaries:
+        partial_runs = [s for s in run_summaries if s.get("isPartial")]
+        if partial_runs:
+            total_failed = sum(s.get("failed", 0) for s in partial_runs)
+            total_attempted = sum(s.get("attempted", 0) for s in partial_runs)
+            lines.append(f"- ⚠ PARTIAL: {total_failed}/{total_attempted} tick evaluations "
+                         f"failed across {len(partial_runs)} run(s)")
+    lines.append("")
+
+    # Assertion Failures
+    histogram = aggregates.get("failureHistogram", {})
+    if histogram:
+        lines.append("## Assertion Failures (top failures)")
+        for i, (name, count) in enumerate(sorted(histogram.items(), key=lambda x: -x[1])[:5], 1):
+            lines.append(f"{i}. {name} — {count} failures")
+        lines.append("")
+
+    # Judge Score Breakdown
+    judge_scores = aggregates.get("judgeScores", {})
+    if judge_scores:
+        lines.append("## Judge Scores")
+        for judge_name, info in judge_scores.items():
+            dist = info.get("dist", {})
+            dist_str = ", ".join(f"{k}★={v}" for k, v in sorted(dist.items()))
+            lines.append(f"- {judge_name}: avg {info.get('avg', '?')}/4.0 ({dist_str})")
+        lines.append("")
+
+    # Sub-Score Breakdown (multi-dimensional rubrics)
+    sub_scores = aggregates.get("subScores", {})
+    if sub_scores:
+        lines.append("## Sub-Scores (per dimension)")
+        for judge_name, dims in sorted(sub_scores.items()):
+            dim_parts = []
+            for dim, info in sorted(dims.items()):
+                dim_parts.append(f"{dim}={info['avg']}/4.0")
+            lines.append(f"- {judge_name}: {', '.join(dim_parts)}")
+        lines.append("")
+
+    # Playbook Health
+    lines.append("## Playbook Health")
+    lines.append(f"- Line count trend: {playbook_health.get('lineCountTrend', [])}")
+    lines.append(f"- Avg churn/tick: {playbook_health.get('avgChurnPerTick', 0)} changes")
+    lines.append(f"- Total added: {playbook_health.get('totalAdded', 0)}, pruned: {playbook_health.get('totalPruned', 0)}")
+    lines.append("")
+
+    # Latency
+    if latency_stats:
+        lines.append("## Latency")
+        for script, info in sorted(latency_stats.items()):
+            lines.append(f"- {script}: avg {info['avg']}ms, p50 {info['p50']}ms, "
+                         f"p95 {info['p95']}ms ({info['count']} samples)")
+        lines.append("")
+
+    # Regressions
+    if regressions:
+        lines.append("## ⚠ Regressions Detected")
+        for r in regressions:
+            lines.append(f"- {r}")
+        lines.append("")
+
+    # LLM Interpretation
+    if llm_interpretation:
+        lines.append("## Analysis & Recommendations")
+        lines.append(llm_interpretation)
+        lines.append("")
+
+    return "\n".join(lines) + "\n"
+
+
+def build_snapshot(aggregates: dict, skip_rate: float, regressions: list[str]) -> dict:
+    """Build a compact snapshot of key metrics for delta comparison."""
+    judge_scores = aggregates.get("judgeScores", {})
+    per_judge = {name: info.get("avg") for name, info in judge_scores.items() if info.get("avg") is not None}
+
+    # Top 3 assertion failures
+    histogram = aggregates.get("failureHistogram", {})
+    top_failures = [name for name, _ in sorted(histogram.items(), key=lambda x: -x[1])[:3]]
+
+    return {
+        "assertionPassRate": aggregates.get("assertionPassRate", {}).get("rate"),
+        "schemaRate": aggregates.get("schemaValidity", {}).get("rate"),
+        "judgeAvg": aggregates.get("judgeAvg"),
+        "skipRate": skip_rate,
+        "perJudgeAvg": per_judge,
+        "topFailures": top_failures,
+        "regressionCount": len(regressions),
+        "tickCount": aggregates.get("tickCount", 0),
+    }
+
+
+def load_previous_snapshot(report_dir: Path, current_date: str) -> tuple[str | None, dict | None]:
+    """Find the most recent snapshot before current_date.
+
+    Returns (date_str, snapshot_dict) or (None, None).
+    """
+    snapshots = sorted(report_dir.glob("*.snapshot.json"), reverse=True)
+    for snap_path in snapshots:
+        date_str = snap_path.stem.replace(".snapshot", "")
+        if date_str < current_date:
+            try:
+                return date_str, json.loads(snap_path.read_text(encoding="utf-8"))
+            except (json.JSONDecodeError, OSError):
+                continue
+    return None, None
+
+
+def compute_delta(before: dict, after: dict) -> dict[str, dict]:
+    """Compute per-metric deltas between two snapshots.
+
+    Returns {metric: {before, after, delta, status}} where status is
+    IMPROVED, REGRESSED, or SAME.
+    """
+    # Metrics where higher is better
+    higher_better = {"assertionPassRate", "schemaRate", "judgeAvg"}
+    # Metrics where lower is better
+    lower_better = {"skipRate", "regressionCount"}
+
+    result: dict[str, dict] = {}
+    for key in higher_better | lower_better:
+        b = before.get(key)
+        a = after.get(key)
+        if b is None or a is None:
+            continue
+        delta = round(a - b, 4) if isinstance(a, float) else a - b
+        if key in higher_better:
+            status = "IMPROVED" if delta > 0.001 else ("REGRESSED" if delta < -0.001 else "SAME")
+        else:
+            status = "IMPROVED" if delta < -0.001 else ("REGRESSED" if delta > 0.001 else "SAME")
+        result[key] = {"before": b, "after": a, "delta": delta, "status": status}
+
+    # Per-judge deltas
+    before_judges = before.get("perJudgeAvg", {})
+    after_judges = after.get("perJudgeAvg", {})
+    for judge in set(before_judges) | set(after_judges):
+        b = before_judges.get(judge)
+        a = after_judges.get(judge)
+        if b is None or a is None:
+            continue
+        delta = round(a - b, 2)
+        status = "IMPROVED" if delta > 0.05 else ("REGRESSED" if delta < -0.05 else "SAME")
+        result[f"judge:{judge}"] = {"before": b, "after": a, "delta": delta, "status": status}
+
+    return result
+
+
+def format_delta_section(prev_date: str, delta: dict[str, dict]) -> str:
+    """Format delta comparison as a markdown section."""
+    lines = [f"## Delta vs Previous ({prev_date})"]
+    for metric, info in sorted(delta.items()):
+        marker = {"IMPROVED": "↑", "REGRESSED": "↓", "SAME": "→"}.get(info["status"], "?")
+        sign = "+" if info["delta"] > 0 else ""
+        lines.append(f"- {marker} {metric}: {info['before']} → {info['after']} ({sign}{info['delta']}) [{info['status']}]")
+    lines.append("")
+    return "\n".join(lines)
+
+
+def _sample_judge_details(eval_entries: list[dict], max_entries: int = 8) -> str:
+    """Extract sampled judge reasonings + assertion failures for cross-tick synthesis."""
+    # Sample evenly across the day
+    step = max(1, len(eval_entries) // max_entries)
+    sampled = eval_entries[::step][:max_entries]
+
+    parts: list[str] = []
+    for i, entry in enumerate(sampled):
+        tick_ts = entry.get("tickTs", "?")
+        section = [f"### Tick {i+1} ({tick_ts})"]
+
+        # Judge reasonings (truncated)
+        judges = entry.get("judges")
+        if judges:
+            for judge_name, result in judges.items():
+                if isinstance(result, dict) and "reasoning" in result:
+                    reasoning = str(result["reasoning"])[:150]
+                    score = result.get("score", "?")
+                    section.append(f"  {judge_name} ({score}/4): {reasoning}")
+
+        # Assertion failures
+        failures = entry.get("assertions", {}).get("failures", [])
+        if failures:
+            for f in failures[:3]:
+                section.append(f"  FAIL: {f.get('name', '?')} — {str(f.get('detail', ''))[:100]}")
+
+        parts.append("\n".join(section))
+
+    return "\n\n".join(parts)
+
+
+def get_llm_interpretation(
+    aggregates: dict,
+    regressions: list[str],
+    playbook_health: dict,
+    eval_entries: list[dict] | None = None,
+) -> str:
+    """Use LLM to interpret trends and write recommendations.
+
+    When eval_entries is provided, includes sampled judge reasonings for
+    cross-tick pattern synthesis.
+    """
+    system_prompt = (
+        "You are an evaluation analyst for a personal AI assistant pipeline. "
+        "Analyze the metrics AND individual tick evaluations to identify cross-cutting patterns. "
+        "Respond with ONLY a JSON object:\n"
+        '{"patterns": ["pattern 1", ...], '
+        '"bottleneck": "detection|generation|both|none", '
+        '"recommendations": ["rec 1", ...]}\n\n'
+        "- patterns: 2-4 recurring themes across individual ticks (reference specific judges/assertions)\n"
+        "- bottleneck: whether issues stem from signal detection (input), insight generation (output), both, or none\n"
+        "- recommendations: 3-5 actionable next steps"
+    )
+
+    user_prompt = (
+        f"## Aggregates\n{json.dumps(aggregates, indent=2)}\n\n"
+        f"## Regressions\n{regressions}\n\n"
+        f"## Playbook Health\n{json.dumps(playbook_health, indent=2)}"
+    )
+
+    if eval_entries:
+        details = _sample_judge_details(eval_entries)
+        user_prompt += f"\n\n## Individual Tick Evaluations (sampled)\n{details}"
+
+    try:
+        raw = call_llm(system_prompt, user_prompt, script="eval_reporter", json_mode=True)
+        result = extract_json(raw)
+
+        sections: list[str] = []
+
+        # Bottleneck
+        bottleneck = result.get("bottleneck", "none")
+        if bottleneck != "none":
+            sections.append(f"**Bottleneck**: {bottleneck}")
+
+        # Patterns
+        patterns = result.get("patterns", [])
+        if patterns:
+            sections.append("**Patterns**:")
+            sections.extend(f"- {p}" for p in patterns)
+
+        # Recommendations
+        recs = result.get("recommendations", [])
+        if recs:
+            sections.append("\n**Recommendations**:")
+            sections.extend(f"- {r}" for r in recs)
+
+        return "\n".join(sections) if sections else ""
+    except (ValueError, LLMError) as e:
+        print(f"[eval-reporter] LLM interpretation failed: {e}", file=sys.stderr)
+
+    return ""
+
+
+# ---------------------------------------------------------------------------
+# Main
+# ---------------------------------------------------------------------------
+
+def main():
+    parser = argparse.ArgumentParser(description="Sinain Koog Daily Eval Reporter (Tier 2)")
+    parser.add_argument("--memory-dir", required=True, help="Path to memory/ directory")
+    parser.add_argument("--days", type=int, default=1, help="Number of days to aggregate (default: 1)")
+    args = parser.parse_args()
+
+    memory_dir = args.memory_dir
+    eval_config = load_eval_config(memory_dir)
+    thresholds = eval_config.get("regressionThresholds", _EVAL_DEFAULTS["regressionThresholds"])
+
+    # Load eval logs
+    raw_eval_entries = load_eval_logs(memory_dir, days=args.days)
+    if not raw_eval_entries:
+        print("[eval-reporter] no eval-log entries found", file=sys.stderr)
+        return
+
+    # Separate tick results from run summary metadata
+    eval_entries, run_summaries = extract_run_summaries(raw_eval_entries)
+    if not eval_entries:
+        print("[eval-reporter] no tick eval entries (only run summaries)", file=sys.stderr)
+        return
+
+    # Load playbook logs for health metrics
+    playbook_logs = read_recent_logs(memory_dir, days=args.days)
+
+    # Compute metrics
+    aggregates = compute_aggregates(eval_entries)
+    playbook_health = compute_playbook_health(playbook_logs)
+    skip_rate = compute_skip_rate(playbook_logs)
+    latency_stats = compute_latency_stats(playbook_logs)
+    regressions = detect_regressions(aggregates, thresholds, skip_rate)
+
+    print(f"[eval-reporter] {aggregates['tickCount']} ticks, "
+          f"schema={aggregates.get('schemaValidity', {}).get('rate', '?')}, "
+          f"assertions={aggregates.get('assertionPassRate', {}).get('rate', '?')}, "
+          f"regressions={len(regressions)}", file=sys.stderr)
+
+    # LLM interpretation (if report feature is on and we have enough data)
+    llm_interpretation = ""
+    if eval_config.get("dailyReport", True) and aggregates["tickCount"] >= 2:
+        llm_interpretation = get_llm_interpretation(
+            aggregates, regressions, playbook_health, eval_entries=eval_entries,
+        )
+
+    # Generate report
+    today = datetime.now(timezone.utc).strftime("%Y-%m-%d")
+    report = generate_report_markdown(
+        today, aggregates, playbook_health, skip_rate, regressions, llm_interpretation,
+        run_summaries=run_summaries,
+        latency_stats=latency_stats,
+    )
+
+    # Write report
+    report_dir = Path(memory_dir) / "eval-reports"
+    report_dir.mkdir(parents=True, exist_ok=True)
+
+    # Write snapshot for delta comparison
+    snapshot = build_snapshot(aggregates, skip_rate, regressions)
+    snapshot_file = report_dir / f"{today}.snapshot.json"
+    snapshot_file.write_text(json.dumps(snapshot, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
+
+    # Compute delta vs previous snapshot and append to report
+    prev_date, prev_snapshot = load_previous_snapshot(report_dir, today)
+    if prev_snapshot:
+        delta = compute_delta(prev_snapshot, snapshot)
+        if delta:
+            report += "\n" + format_delta_section(prev_date, delta) + "\n"
+
+    report_file = report_dir / f"{today}.md"
+    report_file.write_text(report, encoding="utf-8")
+
+    print(f"[eval-reporter] report + snapshot written to {report_dir}", file=sys.stderr)
+
+
+if __name__ == "__main__":
+    main()