kiku-agent 1.3.1 → 1.3.2

package/server/agent/eval/analyze.py

@@ -0,0 +1,684 @@
+import argparse
+import ast
+import hashlib
+import json
+import re
+import subprocess
+import sys
+from collections import Counter
+from datetime import datetime, timezone
+from pathlib import Path
+
+
+EVAL_DIR = Path(__file__).resolve().parent
+DEFAULT_ROOT = EVAL_DIR.parents[2]
+DEFAULT_REPORT = Path("artifacts/eval/live.json")
+DEFAULT_GATES = Path("server/agent/eval/gates.json")
+
+SKIP_DIRS = {"node_modules", ".git", "dist", ".vercel", ".next", "coverage", "__pycache__"}
+SKIP_SUFFIXES = {".png", ".jpg", ".jpeg", ".gif", ".webp", ".ico", ".woff", ".woff2", ".ttf", ".otf", ".mp3", ".mp4"}
+TEXT_SUFFIXES = {".js", ".jsx", ".ts", ".tsx", ".py", ".json", ".css", ".html", ".md", ".ps1", ".svg"}
+
+CATALOG_PAIRS = [
+    ("src/data/profileData.js", "server/mcp/data/profile.js"),
+    ("src/data/worksData.js", "server/mcp/data/works.js"),
+    ("src/data/discographyData.js", "server/mcp/data/discography.js"),
+    ("src/data/brandsData.js", "server/mcp/data/brands.js"),
+    ("src/data/awardsData.js", "server/mcp/data/awards.js"),
+    ("src/data/timelineData.js", "server/mcp/data/timeline.js"),
+    ("src/data/galleryData.js", "server/mcp/data/gallery.js"),
+    ("src/data/industryIndexData.js", "server/mcp/data/industryIndex.js"),
+]
+
+JS_SIGNAL_PATTERNS = {
+    "imports": r"^\s*import\s+.+\s+from\s+['\"]",
+    "exports": r"^\s*export\s+(const|let|var|function|class|default|async\s+function|\{)",
+    "functions": r"\bfunction\s+\w+\s*\(",
+    "arrow_functions": r"=>",
+    "classes": r"^\s*(export\s+)?(default\s+)?class\s+\w+",
+    "awaits": r"\bawait\s+\w+",
+    "fetch_calls": r"\bfetch\s*\(",
+    "react_components": r"\bfunction\s+[A-Z]\w*\s*\(",
+}
+
+EXPORT_SYMBOL_RE = re.compile(r"export\s+(?:const|let|var|function|async\s+function|class)\s+([A-Za-z_$][\w$]*)")
+EXPORT_LIST_RE = re.compile(r"export\s*\{([^}]*)\}")
+
+
+def percentile(sorted_values, pct):
+    if not sorted_values:
+        return None
+    if len(sorted_values) == 1:
+        return sorted_values[0]
+    rank = (pct / 100) * (len(sorted_values) - 1)
+    low = int(rank)
+    high = min(low + 1, len(sorted_values) - 1)
+    frac = rank - low
+    return sorted_values[low] + (sorted_values[high] - sorted_values[low]) * frac
+
+
+def mean(values):
+    return round(sum(values) / len(values), 1) if values else None
+
+
+def mean4(values):
+    clean = [v for v in values if isinstance(v, (int, float)) and v == v]
+    if not clean:
+        return None
+    return round(sum(clean) / len(clean), 4)
+
+
+def binom(n, k):
+    n = int(n)
+    k = int(k)
+    if n < 0 or k < 0 or k > n:
+        return 0
+    k = min(k, n - k)
+    result = 1
+    for i in range(1, k + 1):
+        result = result * (n - k + i) / i
+    return int(round(result))
+
+
+def traj_eta(l_star, l):
+    try:
+        a = float(l_star)
+        b = float(l)
+    except (TypeError, ValueError):
+        return None
+    if a <= 0 or b <= 0:
+        return None
+    return round(a / max(b, a), 4)
+
+
+def estimate_cost(row):
+    trace = row.get("trace") or []
+    calls = sum(1 for t in trace if isinstance(t, dict) and t.get("event") == "start")
+    seconds = 0
+    duration = row.get("durationMs")
+    if isinstance(duration, (int, float)):
+        seconds = max(0, duration) / 1000
+    breakdown = row.get("latencyBreakdown") or {}
+    output_tokens = breakdown.get("outputTokens")
+    if not isinstance(output_tokens, (int, float)):
+        answer = row.get("answer")
+        output_tokens = round(len(answer) / 4) if isinstance(answer, str) else 0
+    cost = output_tokens * 0.00001 + calls * 0.002 + seconds * 0.001
+    return round(cost, 5)
+
+
+def failure_mode(row):
+    if not isinstance(row, dict):
+        return None
+    health = row.get("trajectoryHealth") or {}
+    coverage = row.get("coverage") or {}
+    healthy = row.get("status") == "completed" and coverage.get("allMatched") is True and health.get("hasLoop") is not True
+    if healthy:
+        return None
+    trace = row.get("trace") or []
+    blob = json.dumps({"error": row.get("error"), "trace": trace[:40]}, ensure_ascii=False).lower()
+    err_text = str(row.get("error") or "").lower()
+    if "timeout" in err_text or "permission" in err_text or "denied" in err_text or "unauthorized" in err_text or "abort" in err_text:
+        return "permissionTimeout"
+    if row.get("status") == "error" and "timeout" in blob:
+        return "permissionTimeout"
+    expected = row.get("toolCorrectness") or {}
+    exp_list = expected.get("expected") or []
+    l_star = len(exp_list) if exp_list else int(coverage.get("total") or 1)
+    if l_star < 1:
+        l_star = 1
+    steps = sum(1 for t in trace if isinstance(t, dict) and t.get("event") == "start")
+    loop_count = health.get("loopCount") or 0
+    if health.get("isThrashing") is True or loop_count >= 2 or (steps >= 3 and steps > 1.5 * l_star):
+        return "replanning"
+    schema_hit = "schema" in blob or "invalid" in blob or "validation" in blob or any(isinstance(t, dict) and t.get("event") == "error" for t in trace)
+    tool_miss = expected.get("notApplicable") is not True and (float(expected.get("precision") if expected.get("precision") is not None else 1) < 0.5 or len(expected.get("missing") or []) > 0)
+    if schema_hit or tool_miss:
+        return "toolSchema"
+    answer = row.get("answer")
+    answer_len = len(answer) if isinstance(answer, str) else 0
+    duration = row.get("durationMs") or 0
+    if steps > 6 or answer_len > 4000 or duration > 60000:
+        return "contextOverflow"
+    if row.get("status") == "error":
+        return "permissionTimeout"
+    if coverage.get("allMatched") is False:
+        return "toolSchema"
+    return "contextOverflow"
+
+
+def benchmark_summary(rows):
+    completed = [r for r in rows if isinstance(r, dict) and r.get("status") == "completed"]
+    coverage_rows = [r for r in completed if isinstance(r.get("coverage"), dict) and (r["coverage"].get("total") or 0) > 0]
+    passed = [r for r in coverage_rows if ((r.get("judgeResult", {}).get("score") == 1.0 or r.get("judgeResult", {}).get("verdict") == "PASS") if (r.get("judgeResult") and not r.get("judgeResult", {}).get("judgeError")) else r["coverage"].get("allMatched") is True)]
+    pass_at_1 = round(len(passed) / len(coverage_rows), 4) if coverage_rows else None
+    etas = []
+    for r in completed:
+        trace = r.get("trace") or []
+        steps = sum(1 for t in trace if isinstance(t, dict) and t.get("event") == "start")
+        if not steps:
+            steps = 1 if r.get("status") == "completed" else 0
+        if not steps:
+            continue
+        tc = r.get("toolCorrectness") or {}
+        exp = tc.get("expected") or []
+        cov = r.get("coverage") or {}
+        l_star = len(exp) if exp else int(cov.get("total") or 1)
+        if l_star < 1:
+            l_star = 1
+        value = traj_eta(l_star, steps)
+        if value is not None:
+            etas.append(value)
+    trajectory = mean4(etas)
+    f1s = []
+    for r in completed:
+        tc = r.get("toolCorrectness") or {}
+        if tc.get("notApplicable") is True:
+            continue
+        f1 = tc.get("f1")
+        if isinstance(f1, (int, float)):
+            f1s.append(f1)
+    tool_f1 = mean4(f1s)
+    supports = []
+    for r in completed:
+        g = r.get("groundedness")
+        if not isinstance(g, dict) or g.get("notApplicable") is True:
+            continue
+        ratio = g.get("supportRatio")
+        if isinstance(ratio, (int, float)):
+            supports.append(ratio)
+        elif g.get("isGrounded") is True:
+            supports.append(1)
+        elif g.get("isGrounded") is False:
+            supports.append(0)
+    faith = mean4(supports)
+    hallucination = round(1 - faith, 4) if faith is not None else None
+    costs = [estimate_cost(r) for r in completed]
+    cost_per_task = round(sum(costs) / len(costs), 4) if costs else None
+    counts = {"toolSchema": 0, "contextOverflow": 0, "replanning": 0, "permissionTimeout": 0}
+    for r in rows:
+        mode = failure_mode(r)
+        if mode in counts:
+            counts[mode] += 1
+    total_failures = sum(counts.values())
+    distribution = {}
+    for key in counts:
+        distribution[key] = {"count": counts[key], "rate": round(counts[key] / total_failures, 4) if total_failures else 0}
+    values = {"passAt1": pass_at_1, "trajectoryOptimality": trajectory, "toolF1": tool_f1, "hallucinationRate": hallucination, "costPerTask": cost_per_task}
+    targets = {"passAt1": {"min": 0.75}, "trajectoryOptimality": {"min": 0.8}, "toolF1": {"min": 0.9}, "hallucinationRate": {"max": 0.05}, "costPerTask": {"max": 0.04}}
+    status = {}
+    for key, value in values.items():
+        rule = targets.get(key, {})
+        if value is None:
+            status[key] = None
+        elif "min" in rule:
+            status[key] = value >= rule["min"]
+        elif "max" in rule:
+            status[key] = value <= rule["max"]
+        else:
+            status[key] = None
+    live = None
+    if pass_at_1 is not None and cost_per_task is not None:
+        live = {"label": "Live run", "successRate": round(pass_at_1 * 100, 1), "cost": cost_per_task, "kind": "live"}
+    pareto = [
+        {"label": "Agent Model v2", "successRate": 82.4, "cost": 0.027, "kind": "candidate"},
+        {"label": "Baseline v1", "successRate": 68.0, "cost": 0.045, "kind": "baseline"},
+        {"label": "Legacy Script", "successRate": 52.1, "cost": 0.012, "kind": "legacy"},
+    ]
+    if live is not None:
+        pareto.append(live)
+    return {"values": values, "status": status, "targets": targets, "failureDistribution": distribution, "failureTotal": total_failures, "pareto": pareto, "livePoint": live}
+
+
+def sha256_file(path):
+    digest = hashlib.sha256()
+    with open(path, "rb") as handle:
+        for chunk in iter(lambda: handle.read(65536), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def is_skipped(path, root):
+    try:
+        rel = path.relative_to(root)
+    except ValueError:
+        return True
+    return any(part in SKIP_DIRS for part in rel.parts)
+
+
+def iter_repo_files(root):
+    results = []
+    for path in root.rglob("*"):
+        if not path.is_file():
+            continue
+        if is_skipped(path, root):
+            continue
+        if path.suffix.lower() in SKIP_SUFFIXES:
+            continue
+        results.append(path)
+    return results
+
+
+def count_lines(text):
+    lines = text.splitlines()
+    total = len(lines)
+    blank = sum(1 for line in lines if not line.strip())
+    return total, blank
+
+
+def analyze_js_signals(text):
+    signals = {}
+    for name, pattern in JS_SIGNAL_PATTERNS.items():
+        signals[name] = len(re.findall(pattern, text, re.MULTILINE))
+    return signals
+
+
+def analyze_python_ast(text):
+    try:
+        tree = ast.parse(text)
+    except SyntaxError:
+        return {"parseError": True, "functions": 0, "classes": 0, "imports": 0}
+    functions = sum(isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) for n in ast.walk(tree))
+    classes = sum(isinstance(n, ast.ClassDef) for n in ast.walk(tree))
+    imports = sum(isinstance(n, (ast.Import, ast.ImportFrom)) for n in ast.walk(tree))
+    return {"parseError": False, "functions": functions, "classes": classes, "imports": imports}
+
+
+def extract_js_exports(text):
+    symbols = set(EXPORT_SYMBOL_RE.findall(text))
+    for block in EXPORT_LIST_RE.findall(text):
+        for item in block.split(","):
+            item = item.strip().split(" as ")[-1].strip()
+            if re.match(r"^[A-Za-z_$][\w$]*$", item):
+                symbols.add(item)
+    return sorted(symbols)
+
+
+def analyze_codebase(root):
+    files = iter_repo_files(root)
+    by_suffix = Counter()
+    loc_by_suffix = Counter()
+    blank_by_suffix = Counter()
+    bytes_by_suffix = Counter()
+    js_signals_total = Counter()
+    python_stats_total = Counter()
+    largest = []
+    for path in files:
+        suffix = path.suffix.lower() or "<none>"
+        by_suffix[suffix] += 1
+        size = path.stat().st_size
+        bytes_by_suffix[suffix] += size
+        entry = {"path": str(path.relative_to(root)).replace("\\", "/"), "suffix": suffix, "bytes": size}
+        if suffix in TEXT_SUFFIXES:
+            try:
+                text = path.read_text(encoding="utf-8", errors="replace")
+            except OSError:
+                largest.append(entry)
+                continue
+            total, blank = count_lines(text)
+            entry["lines"] = total
+            entry["blank"] = blank
+            loc_by_suffix[suffix] += total
+            blank_by_suffix[suffix] += blank
+            if suffix in {".js", ".jsx"}:
+                signals = analyze_js_signals(text)
+                for key, value in signals.items():
+                    js_signals_total[key] += value
+                entry["jsSignals"] = signals
+            if suffix == ".py":
+                py_stats = analyze_python_ast(text)
+                for key in ("functions", "classes", "imports"):
+                    python_stats_total[key] += py_stats.get(key, 0)
+                entry["pythonStats"] = py_stats
+        largest.append(entry)
+    largest_sorted = sorted(largest, key=lambda e: e.get("bytes", 0), reverse=True)[:15]
+    total_lines = sum(loc_by_suffix.values())
+    total_bytes = sum(bytes_by_suffix.values())
+    return {
+        "root": str(root),
+        "fileCount": len(files),
+        "totalBytes": total_bytes,
+        "totalLines": total_lines,
+        "bySuffix": dict(sorted(by_suffix.items())),
+        "linesBySuffix": dict(sorted(loc_by_suffix.items())),
+        "blankLinesBySuffix": dict(sorted(blank_by_suffix.items())),
+        "bytesBySuffix": dict(sorted(bytes_by_suffix.items())),
+        "jsSignalsTotal": dict(js_signals_total),
+        "pythonStatsTotal": dict(python_stats_total),
+        "largestFiles": largest_sorted,
+    }
+
+
+def analyze_catalog_drift(root):
+    pairs = []
+    drifted = 0
+    for src_rel, mcp_rel in CATALOG_PAIRS:
+        src = root / src_rel
+        mcp = root / mcp_rel
+        item = {"src": src_rel, "mcp": mcp_rel}
+        for key, path in (("src", src), ("mcp", mcp)):
+            if path.exists():
+                text = path.read_text(encoding="utf-8", errors="replace")
+                total, _ = count_lines(text)
+                item[key + "Exists"] = True
+                item[key + "Bytes"] = path.stat().st_size
+                item[key + "Lines"] = total
+                item[key + "Sha256"] = sha256_file(path)[:12]
+                item[key + "Exports"] = extract_js_exports(text)
+            else:
+                item[key + "Exists"] = False
+        src_exports = set(item.get("srcExports", []))
+        mcp_exports = set(item.get("mcpExports", []))
+        item["exportOverlap"] = sorted(src_exports & mcp_exports)
+        item["srcOnlyExports"] = sorted(src_exports - mcp_exports)
+        item["mcpOnlyExports"] = sorted(mcp_exports - src_exports)
+        item["drifted"] = bool(item["srcOnlyExports"] or item["mcpOnlyExports"])
+        if item["drifted"]:
+            drifted += 1
+        pairs.append(item)
+    return {"pairs": pairs, "pairCount": len(pairs), "driftedPairs": drifted}
+
+
+def load_corpus_via_node(root):
+    script = (
+        "import('./server/agent/eval/modules/cases.js').then(m => {"
+        " const rows = m.allCases.map(c => ({"
+        " id: c.id, category: c.category, language: c.language,"
+        " groups: (c.groups || []).length,"
+        " expectedTools: c.expectedTools || [],"
+        " sources: c.sources || [] }));"
+        " console.log(JSON.stringify({ total: rows.length,"
+        " catalog: m.cases.length, security: m.securityCases.length, rows }));"
+        "}).catch(e => { console.error(String(e && e.message || e)); process.exit(1); })"
+    )
+    proc = subprocess.run(
+        ["node", "--input-type=module", "-e", script],
+        cwd=str(root),
+        capture_output=True,
+        text=True,
+        timeout=30,
+    )
+    if proc.returncode != 0:
+        return {"available": False, "error": (proc.stderr or proc.stdout).strip()[-2000:]}
+    payload = json.loads(proc.stdout.strip())
+    rows = payload.get("rows", [])
+    by_category = dict(Counter(r["category"] for r in rows))
+    by_language = dict(Counter(r["language"] for r in rows))
+    tool_demand = dict(Counter(t for r in rows for t in r.get("expectedTools", [])))
+    with_groups = sum(1 for r in rows if r.get("groups", 0) > 0)
+    return {
+        "available": True,
+        "total": payload.get("total", len(rows)),
+        "catalogCount": payload.get("catalog"),
+        "securityCount": payload.get("security"),
+        "byCategory": dict(sorted(by_category.items())),
+        "byLanguage": dict(sorted(by_language.items())),
+        "toolDemand": dict(sorted(tool_demand.items(), key=lambda kv: kv[1], reverse=True)),
+        "casesWithGroups": with_groups,
+        "caseIds": [r["id"] for r in rows],
+    }
+
+
+def summarize_latencies(durations):
+    ordered = sorted(durations)
+    if not ordered:
+        return {"count": 0, "mean": None, "min": None, "p50": None, "p95": None, "max": None}
+    return {
+        "count": len(ordered),
+        "mean": mean(ordered),
+        "min": ordered[0],
+        "p50": round(percentile(ordered, 50), 1),
+        "p95": round(percentile(ordered, 95), 1),
+        "max": ordered[-1],
+    }
+
+
+def analyze_report(report_path, root):
+    resolved = report_path if report_path.is_absolute() else root / report_path
+    if not resolved.exists():
+        return {"status": "missing", "path": str(report_path)}
+    try:
+        payload = json.loads(resolved.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError) as exc:
+        return {"status": "unreadable", "path": str(report_path), "error": str(exc)[:500]}
+    if payload.get("status") == "blocked":
+        return {"status": "blocked", "path": str(report_path), "reason": payload.get("reason"), "planned": payload.get("planned")}
+    rows = payload.get("rows", [])
+    summary = payload.get("summary", {})
+    metadata = payload.get("metadata", {})
+    completed = [r for r in rows if r.get("status") == "completed"]
+    errors = [r for r in rows if r.get("status") != "completed"]
+    coverage_rows = [r for r in completed if r.get("coverage") and (r["coverage"].get("total") or 0) > 0]
+    coverage_pass = [r for r in coverage_rows if r["coverage"].get("allMatched") is True]
+    tool_rows = [r for r in completed if r.get("toolCorrectness") and not r["toolCorrectness"].get("notApplicable")]
+    tool_pass = [r for r in tool_rows if r["toolCorrectness"].get("isCorrect") is True]
+    loop_rows = [r for r in completed if r.get("trajectoryHealth", {}).get("hasLoop")]
+    grounded_rows = [r for r in completed if r.get("groundedness") and not r["groundedness"].get("notApplicable")]
+    grounded_pass = [r for r in grounded_rows if r["groundedness"].get("isGrounded") is True]
+    judged = [r for r in completed if r.get("judgment") or (r.get("judgeResult") and not r.get("judgeResult", {}).get("judgeError"))]
+    durations = [r["durationMs"] for r in completed if isinstance(r.get("durationMs"), (int, float))]
+    tool_histogram = Counter()
+    for row in completed:
+        trace = row.get("trace") or []
+        seen = {t.get("name") for t in trace if t.get("event") == "start" and t.get("name")}
+        for name in seen:
+            tool_histogram[name] += 1
+    per_category = {}
+    for row in rows:
+        cat = row.get("category") or "unknown"
+        bucket = per_category.setdefault(cat, {"total": 0, "completed": 0, "factsMatched": 0, "toolsCorrect": 0, "toolEligible": 0})
+        bucket["total"] += 1
+        if row.get("status") == "completed":
+            bucket["completed"] += 1
+            if (row.get("coverage") or {}).get("allMatched") is True:
+                bucket["factsMatched"] += 1
+            tc = row.get("toolCorrectness") or {}
+            if not tc.get("notApplicable"):
+                bucket["toolEligible"] += 1
+                if tc.get("isCorrect") is True:
+                    bucket["toolsCorrect"] += 1
+    for bucket in per_category.values():
+        bucket["factRate"] = round(bucket["factsMatched"] / bucket["completed"], 4) if bucket["completed"] else None
+        bucket["toolRate"] = round(bucket["toolsCorrect"] / bucket["toolEligible"], 4) if bucket["toolEligible"] else None
+    per_language = {}
+    for row in rows:
+        lang = row.get("language") or "unknown"
+        bucket = per_language.setdefault(lang, {"total": 0, "completed": 0, "factsMatched": 0})
+        bucket["total"] += 1
+        if row.get("status") == "completed":
+            bucket["completed"] += 1
+            if (row.get("coverage") or {}).get("allMatched") is True:
+                bucket["factsMatched"] += 1
+    slowest = sorted(completed, key=lambda r: r.get("durationMs") or 0, reverse=True)[:10]
+    slowest = [{"id": r.get("id"), "category": r.get("category"), "durationMs": r.get("durationMs")} for r in slowest]
+    bench = benchmark_summary(rows)
+    return {
+        "status": "ok",
+        "path": str(report_path),
+        "model": metadata.get("model"),
+        "planned": summary.get("planned", metadata.get("caseIds", []) and len(metadata["caseIds"])),
+        "attempted": len(rows),
+        "completed": len(completed),
+        "errors": len(errors),
+        "errorIds": [{"id": r.get("id"), "error": str(r.get("error"))[:300]} for r in errors],
+        "completionRate": round(len(completed) / len(rows), 4) if rows else None,
+        "lexicalPass": len(coverage_pass),
+        "lexicalEligible": len(coverage_rows),
+        "lexicalRate": round(len(coverage_pass) / len(coverage_rows), 4) if coverage_rows else None,
+        "toolPass": len(tool_pass),
+        "toolEligible": len(tool_rows),
+        "toolRate": round(len(tool_pass) / len(tool_rows), 4) if tool_rows else None,
+        "loopCount": len(loop_rows),
+        "loopIds": [r.get("id") for r in loop_rows],
+        "loopFreeRate": round((len(completed) - len(loop_rows)) / len(completed), 4) if completed else None,
+        "groundedPass": len(grounded_pass),
+        "groundedEligible": len(grounded_rows),
+        "groundedRate": round(len(grounded_pass) / len(grounded_rows), 4) if grounded_rows else None,
+        "judged": len(judged),
+        "latencyMs": summarize_latencies(durations),
+        "toolHistogram": dict(sorted(tool_histogram.items(), key=lambda kv: kv[1], reverse=True)),
+        "perCategory": dict(sorted(per_category.items())),
+        "perLanguage": dict(sorted(per_language.items())),
+        "slowestCases": slowest,
+        "engineSummary": summary,
+        "benchmark": bench,
+    }
+
+
+def evaluate_gates(report_section, gates_path, root):
+    resolved = gates_path if gates_path.is_absolute() else root / gates_path
+    if report_section.get("status") != "ok":
+        return {"available": False, "reason": "report-" + report_section.get("status", "unknown")}
+    if not resolved.exists():
+        return {"available": False, "reason": "gates-file-missing"}
+    gates = json.loads(resolved.read_text(encoding="utf-8"))
+    summary = report_section.get("engineSummary") or {}
+    bench_values = (report_section.get("benchmark") or {}).get("values") or {}
+    results = []
+    passed = True
+    for metric, rule in gates.items():
+        if metric == "perCategory":
+            continue
+        actual = summary.get(metric)
+        if actual is None:
+            actual = bench_values.get(metric)
+        ok = True
+        if isinstance(rule, dict) and "min" in rule:
+            ok = actual is not None and actual >= rule["min"]
+        if isinstance(rule, dict) and "max" in rule:
+            ok = ok and actual is not None and actual <= rule["max"]
+        if not ok:
+            passed = False
+        results.append({"metric": metric, "threshold": rule, "actual": actual, "pass": ok})
+    per_category_rules = gates.get("perCategory") or {}
+    per_category = report_section.get("perCategory") or {}
+    for cat, rules in per_category_rules.items():
+        bucket = per_category.get(cat, {})
+        for sub_metric, rule in rules.items():
+            actual = None
+            if sub_metric == "lexicalCoverageRate" and bucket.get("completed"):
+                actual = round(bucket["factsMatched"] / bucket["completed"], 4)
+            if sub_metric == "toolAccuracyRate" and bucket.get("toolEligible"):
+                actual = round(bucket["toolsCorrect"] / bucket["toolEligible"], 4)
+            ok = True
+            if isinstance(rule, dict) and "min" in rule:
+                ok = actual is not None and actual >= rule["min"]
+            if isinstance(rule, dict) and "max" in rule:
+                ok = ok and actual is not None and actual <= rule["max"]
+            if not ok:
+                passed = False
+            results.append({"metric": "perCategory." + cat + "." + sub_metric, "threshold": rule, "actual": actual, "pass": ok})
+    return {"available": True, "passed": passed, "results": results}
+
+
+def build_markdown(result):
+    lines = []
+    lines.append("Kiku whole-system analysis")
+    lines.append("Generated: " + str(result.get("generatedAt")))
+    codebase = result.get("codebase", {})
+    lines.append("")
+    lines.append("Codebase: " + str(codebase.get("fileCount")) + " files, " + str(codebase.get("totalLines")) + " lines, " + str(codebase.get("totalBytes")) + " bytes")
+    for suffix in sorted((codebase.get("linesBySuffix") or {}).keys()):
+        count = (codebase.get("bySuffix") or {}).get(suffix, 0)
+        lines.append("  " + suffix + ": " + str(count) + " files / " + str(codebase["linesBySuffix"][suffix]) + " lines")
+    drift = result.get("catalogDrift", {})
+    lines.append("")
+    lines.append("Catalog drift: " + str(drift.get("driftedPairs")) + "/" + str(drift.get("pairCount")) + " pairs differ")
+    for pair in drift.get("pairs", []):
+        if pair.get("drifted"):
+            lines.append("  DRIFT " + pair["src"] + " vs " + pair["mcp"] + " srcOnly=" + ",".join(pair.get("srcOnlyExports", [])))
+    corpus = result.get("corpus", {})
+    lines.append("")
+    if corpus.get("available"):
+        lines.append("Corpus: " + str(corpus.get("total")) + " cases (" + str(corpus.get("catalogCount")) + " catalog + " + str(corpus.get("securityCount")) + " security)")
+        lines.append("Languages: " + json.dumps(corpus.get("byLanguage"), ensure_ascii=False))
+        lines.append("Top tools: " + json.dumps(dict(list((corpus.get("toolDemand") or {}).items())[:8]), ensure_ascii=False))
+    else:
+        lines.append("Corpus unavailable: " + str(corpus.get("error")))
+    report = result.get("report", {})
+    lines.append("")
+    lines.append("Report [" + str(report.get("status")) + "]: " + str(report.get("path")))
+    if report.get("status") == "ok":
+        lines.append("Completed " + str(report.get("completed")) + "/" + str(report.get("attempted")) + ", lexical " + str(report.get("lexicalPass")) + "/" + str(report.get("lexicalEligible")) + ", tools " + str(report.get("toolPass")) + "/" + str(report.get("toolEligible")))
+        lines.append("Loops: " + str(report.get("loopCount")) + ", p95 latency: " + str((report.get("latencyMs") or {}).get("p95")) + "ms")
+        lines.append("Slowest: " + json.dumps(report.get("slowestCases"), ensure_ascii=False))
+        bench = report.get("benchmark") or {}
+        values = bench.get("values") or {}
+        status = bench.get("status") or {}
+        if values:
+            lines.append("Benchmark ISO-IEC 42001 and NIST AI RMF:")
+            for key in ["passAt1", "trajectoryOptimality", "toolF1", "hallucinationRate", "costPerTask"]:
+                mark = "PASS" if status.get(key) is True else ("FAIL" if status.get(key) is False else "N-A")
+                lines.append("  " + mark + " " + key + " actual=" + str(values.get(key)))
+            lines.append("Failures: " + json.dumps(bench.get("failureDistribution"), ensure_ascii=False))
+            lines.append("Pareto: " + json.dumps(bench.get("pareto"), ensure_ascii=False))
+    gates = result.get("gates", {})
+    if gates.get("available"):
+        lines.append("")
+        lines.append("Gates passed: " + str(gates.get("passed")))
+        for item in gates.get("results", []):
+            lines.append("  " + ("PASS" if item["pass"] else "FAIL") + " " + item["metric"] + " actual=" + str(item["actual"]) + " threshold=" + json.dumps(item["threshold"], ensure_ascii=False))
+    cross = result.get("cross", {})
+    if cross:
+        lines.append("")
+        lines.append("Stale report ids: " + str(len(cross.get("staleReportIds", []))) + ", unrun corpus ids: " + str(len(cross.get("unrunCorpusIds", []))))
+    return "\n".join(lines) + "\n"
+
+
+def parse_args(argv):
+    parser = argparse.ArgumentParser(description="Whole-system Python analyzer for the Kiku eval harness")
+    parser.add_argument("--mode", choices=["all", "codebase", "report"], default="all")
+    parser.add_argument("--root", default=str(DEFAULT_ROOT))
+    parser.add_argument("--report", default=str(DEFAULT_REPORT))
+    parser.add_argument("--gates", default=str(DEFAULT_GATES))
+    parser.add_argument("--out", default=None)
+    parser.add_argument("--markdown-out", default=None)
+    parser.add_argument("--skip-corpus", action="store_true")
+    return parser.parse_args(argv)
+
+
+def main(argv=None):
+    args = parse_args(argv if argv is not None else sys.argv[1:])
+    root = Path(args.root).resolve()
+    result = {"generatedAt": datetime.now(timezone.utc).isoformat(), "mode": args.mode, "root": str(root)}
+    if args.mode in ("all", "codebase"):
+        result["codebase"] = analyze_codebase(root)
+        result["catalogDrift"] = analyze_catalog_drift(root)
+        if args.skip_corpus:
+            result["corpus"] = {"available": False, "error": "skipped-by-flag"}
+        else:
+            try:
+                result["corpus"] = load_corpus_via_node(root)
+            except Exception as exc:
+                result["corpus"] = {"available": False, "error": str(exc)[:500]}
+    if args.mode in ("all", "report"):
+        report_section = analyze_report(Path(args.report), root)
+        result["report"] = report_section
+        result["gates"] = evaluate_gates(report_section, Path(args.gates), root)
+        corpus_ids = set((result.get("corpus") or {}).get("caseIds") or [])
+        if report_section.get("status") == "ok" and corpus_ids:
+            try:
+                payload_path = Path(args.report) if Path(args.report).is_absolute() else root / args.report
+                payload = json.loads(Path(payload_path).read_text(encoding="utf-8"))
+                report_ids = {r.get("id") for r in payload.get("rows", []) if r.get("id")}
+                result["cross"] = {
+                    "staleReportIds": sorted(report_ids - corpus_ids),
+                    "unrunCorpusIds": sorted(corpus_ids - report_ids),
+                }
+            except (OSError, json.JSONDecodeError):
+                result["cross"] = {"staleReportIds": [], "unrunCorpusIds": sorted(corpus_ids)}
+    output_text = json.dumps(result, ensure_ascii=False, indent=2)
+    if args.out:
+        out_path = Path(args.out) if Path(args.out).is_absolute() else root / args.out
+        out_path.parent.mkdir(parents=True, exist_ok=True)
+        out_path.write_text(output_text, encoding="utf-8")
+    if args.markdown_out:
+        md_path = Path(args.markdown_out) if Path(args.markdown_out).is_absolute() else root / args.markdown_out
+        md_path.parent.mkdir(parents=True, exist_ok=True)
+        md_path.write_text(build_markdown(result), encoding="utf-8")
+    sys.stdout.write(output_text + "\n")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())