ltcai 9.7.0 → 9.9.0

package/latticeai/services/change_proposals.py

@@ -21,9 +21,13 @@ change audit log. The classification itself lives in
 from __future__ import annotations
 
 import difflib
+import hashlib
 import logging
+import os
+import tempfile
+import threading
 from pathlib import Path
-from typing import Any, Callable, Dict, List, Optional
+from typing import Any, Callable, Dict, List, Optional, Tuple
 
 from latticeai.core.tool_governor import classify_tool_call
 
@@ -34,6 +38,55 @@ _MAX_DIFF_LINES = 400
 _SMALL_TIER_DIFF_LINES = 40
 
 
+class ProposalConflictError(ValueError):
+    """The target file changed (or the proposal was already resolved) between
+    staging and approval — applying now would silently destroy user edits.
+
+    Subclasses :class:`ValueError` deliberately: API surfaces that only know
+    ``except ValueError`` degrade to a 4xx instead of applying or crashing,
+    while conflict-aware surfaces catch this class first and answer **409**
+    with a rebase hint.
+    """
+
+    def __init__(
+        self,
+        *,
+        reason: str,
+        path: str,
+        kind: str,
+        base_sha256: str = "",
+        current_sha256: str = "",
+        rebase_hint: str = "",
+    ) -> None:
+        self.reason = reason
+        self.path = path
+        self.kind = kind
+        self.base_sha256 = base_sha256
+        self.current_sha256 = current_sha256
+        self.rebase_hint = rebase_hint or (
+            "제안 생성 이후 파일 상태가 바뀌었습니다. 이 제안을 거부하고 "
+            "현재 파일 내용을 기준으로 제안을 다시 생성하세요."
+        )
+        super().__init__(f"change proposal conflict ({reason}): {path}")
+
+    def to_detail(self) -> Dict[str, Any]:
+        """409 response body — no staged content, just what the UI needs."""
+        return {
+            "error": "change_proposal_conflict",
+            "conflict": True,
+            "reason": self.reason,
+            "path": self.path,
+            "kind": self.kind,
+            "base_sha256": self.base_sha256,
+            "current_sha256": self.current_sha256,
+            "rebase_hint": self.rebase_hint,
+        }
+
+
+def _sha256_text(content: str) -> str:
+    return hashlib.sha256(content.encode("utf-8")).hexdigest()
+
+
 def _unified_diff(before: str, after: str, path: str) -> List[str]:
     lines = list(
         difflib.unified_diff(
@@ -60,6 +113,11 @@ class ChangeProposalService:
         self._review_queue = review_queue
         self._resolve_path = resolve_path
         self._audit = audit or (lambda *a, **kw: None)
+        # One protected section for status-check → conflict-check → apply →
+        # transition. In-process serialization is the right weight here: the
+        # review queue lives in this process and the write itself is atomic
+        # (temp file + os.replace), so a cross-process file lock adds nothing.
+        self._apply_lock = threading.Lock()
 
     # ── classification / staging (agent-loop governor port) ─────────────
 
@@ -125,15 +183,24 @@ class ChangeProposalService:
             return False
 
     def _read_before(self, path: str) -> str:
+        return self._snapshot(path)[1]
+
+    def _snapshot(self, path: str) -> Tuple[bool, str]:
+        """(exists, content) of the target *as the proposal pipeline sees it*.
+
+        Both staging and the approve-time conflict check go through this one
+        normalization (truncate → utf-8 with replacement), so an unchanged
+        file always hashes identically at both points in time.
+        """
         try:
             target = self._resolve_path(path)
             if not target.is_file():
-                return ""
+                return False, ""
             raw = target.read_bytes()[:_MAX_STAGED_BYTES]
-            return raw.decode("utf-8", errors="replace")
+            return True, raw.decode("utf-8", errors="replace")
         except Exception:
             LOGGER.exception("change proposal read failed")
-            return ""
+            return False, ""
 
     def _staged_content(self, name: str, args: Dict[str, Any]) -> Optional[str]:
         if name == "write_file":
@@ -164,7 +231,7 @@ class ChangeProposalService:
         workspace_id: Optional[str] = None,
         context: Optional[Dict[str, Any]] = None,
     ) -> Dict[str, Any]:
-        before = self._read_before(path)
+        base_exists, before = self._snapshot(path)
         diff = _unified_diff(before, new_content, path)
         tier = "small" if len(diff) <= _SMALL_TIER_DIFF_LINES else "large"
         provenance: Dict[str, Any] = {"proposed_by": proposed_by, "reason": reason}
@@ -183,6 +250,11 @@ class ChangeProposalService:
                 "tier": tier,
                 "before_bytes": len(before.encode("utf-8")),
                 "after_bytes": len(new_content.encode("utf-8")),
+                # Base snapshot for the approve-time conflict check: an empty
+                # base_sha256 with base_exists=False means "proposed against a
+                # missing file", never "hash of the empty string".
+                "base_exists": base_exists,
+                "base_sha256": _sha256_text(before) if base_exists else "",
             },
             provenance=provenance,
             user_email=user_email,
@@ -203,7 +275,7 @@ class ChangeProposalService:
         user_email: Optional[str] = None,
         workspace_id: Optional[str] = None,
     ) -> Dict[str, Any]:
-        before = self._read_before(path)
+        base_exists, before = self._snapshot(path)
         item = self._review_queue.create(
             title=f"파일 삭제 제안: {path}",
             summary=reason or "기존 파일을 삭제하는 작업이라 검토 후 적용됩니다.",
@@ -215,6 +287,8 @@ class ChangeProposalService:
                 "tier": "large",
                 "before_bytes": len(before.encode("utf-8")),
                 "after_bytes": 0,
+                "base_exists": base_exists,
+                "base_sha256": _sha256_text(before) if base_exists else "",
             },
             provenance={"proposed_by": proposed_by, "reason": reason},
             user_email=user_email,
@@ -274,28 +348,96 @@ class ChangeProposalService:
         self, item_id: str, *, user_email: Optional[str] = None,
         workspace_id: Optional[str] = None,
     ) -> Dict[str, Any]:
-        item = self._review_queue.get(item_id, workspace_id=workspace_id)
-        if item.get("source") != "change_proposal":
-            raise KeyError(f"not a change proposal: {item_id}")
-        payload = item.get("payload") or {}
-        kind = item.get("kind")
-        path = str(payload.get("path") or "")
-        target = self._resolve_path(path)
-        if kind == "file_update":
-            target.parent.mkdir(parents=True, exist_ok=True)
-            target.write_text(str(payload.get("new_content") or ""), encoding="utf-8")
-        elif kind == "file_delete":
-            if target.is_file():
-                target.unlink()
-        else:
-            raise ValueError(f"unknown change proposal kind: {kind}")
-        approved = self._review_queue.approve(item_id, workspace_id=workspace_id)
+        """Apply the staged change exactly as reviewed — but only if the file
+        on disk still matches the base snapshot the reviewer looked at.
+
+        Raises :class:`ProposalConflictError` (→ HTTP 409) when the target
+        drifted since staging or the proposal was already resolved; nothing
+        touches disk in that case, so out-of-band user edits are preserved.
+        """
+        with self._apply_lock:
+            item = self._review_queue.get(item_id, workspace_id=workspace_id)
+            if item.get("source") != "change_proposal":
+                raise KeyError(f"not a change proposal: {item_id}")
+            payload = item.get("payload") or {}
+            kind = str(item.get("kind") or "")
+            path = str(payload.get("path") or "")
+            if kind not in ("file_update", "file_delete"):
+                raise ValueError(f"unknown change proposal kind: {kind}")
+
+            # Duplicate/concurrent approval guard: a resolved proposal must
+            # never re-apply (the disk may have moved on since it was applied).
+            status = str(item.get("status") or "pending")
+            if status not in ("pending", "snoozed"):
+                raise ProposalConflictError(
+                    reason=f"already_{status}", path=path, kind=kind,
+                    rebase_hint="이미 처리된 제안입니다. 다시 적용할 수 없습니다.",
+                )
+
+            self._check_base_unchanged(payload, path=path, kind=kind)
+
+            target = self._resolve_path(path)
+            if kind == "file_update":
+                self._atomic_write(target, str(payload.get("new_content") or ""))
+            else:  # file_delete — existence re-verified inside the lock
+                if target.is_file():
+                    target.unlink()
+            approved = self._review_queue.approve(item_id, workspace_id=workspace_id)
         self._audit(
             "change_proposal_applied", user_email=user_email,
             proposal_id=item_id, path=path, kind=kind,
         )
         return {"item": approved, "applied": True, "path": path, "kind": kind}
 
+    def _check_base_unchanged(
+        self, payload: Dict[str, Any], *, path: str, kind: str
+    ) -> None:
+        """Compare the disk state *now* against the staged base snapshot."""
+        if "base_sha256" not in payload or "base_exists" not in payload:
+            # Legacy proposal staged before base snapshots existed — keep the
+            # historical apply-as-reviewed behavior rather than rejecting it.
+            return
+        base_exists = bool(payload.get("base_exists"))
+        base_sha256 = str(payload.get("base_sha256") or "")
+        current_exists, current_content = self._snapshot(path)
+        current_sha256 = _sha256_text(current_content) if current_exists else ""
+        if not base_exists:
+            if current_exists:
+                raise ProposalConflictError(
+                    reason="file_created_since_proposal", path=path, kind=kind,
+                    current_sha256=current_sha256,
+                )
+            return
+        if not current_exists:
+            raise ProposalConflictError(
+                reason="file_deleted_since_proposal", path=path, kind=kind,
+                base_sha256=base_sha256,
+            )
+        if current_sha256 != base_sha256:
+            raise ProposalConflictError(
+                reason="file_modified_since_proposal", path=path, kind=kind,
+                base_sha256=base_sha256, current_sha256=current_sha256,
+            )
+
+    @staticmethod
+    def _atomic_write(target: Path, content: str) -> None:
+        """Write via a same-directory temp file + ``os.replace`` so readers
+        never observe a partially written proposal."""
+        target.parent.mkdir(parents=True, exist_ok=True)
+        fd, tmp_name = tempfile.mkstemp(
+            dir=str(target.parent), prefix=f".{target.name}.", suffix=".staged"
+        )
+        try:
+            with os.fdopen(fd, "w", encoding="utf-8") as handle:
+                handle.write(content)
+            os.replace(tmp_name, target)
+        except BaseException:
+            try:
+                os.unlink(tmp_name)
+            except OSError:
+                pass
+            raise
+
     def reject(
         self, item_id: str, *, user_email: Optional[str] = None,
         workspace_id: Optional[str] = None, reason: str = "",
@@ -319,4 +461,4 @@ class ChangeProposalService:
         return {"item": dismissed, "applied": False, "reason": reason}
 
 
-__all__ = ["ChangeProposalService"]
+__all__ = ["ChangeProposalService", "ProposalConflictError"]

package/latticeai/services/product_readiness.py

@@ -18,7 +18,7 @@ from typing import Any, Dict, List
 
 from latticeai.services.architecture_readiness import architecture_readiness
 
-PRODUCT_VERSION_TARGET = "9.7.0"
+PRODUCT_VERSION_TARGET = "9.9.0"
 
 
 @dataclass(frozen=True)

package/latticeai/services/tool_dispatch.py

@@ -21,6 +21,7 @@ from latticeai.core.agent_prompts import (
     PLANNER_PROMPT,
 )
 from latticeai.core.policy import role_has_capability
+from latticeai.core.tool_governor import classify_tool_call
 from latticeai.core.tool_registry import ToolPermission, ToolPolicy
 from latticeai.tools import AGENT_ROOT, DEFAULT_TOOL_REGISTRY, ToolError, ensure_agent_root
 
@@ -122,6 +123,22 @@ class ToolDispatchService:
                 detail=f"'{tool_name}' 툴에는 '{capability}' capability가 필요합니다.",
             )
 
+    def _governed_path_exists(self, path: str) -> bool:
+        """Best-effort existence check for governance classification.
+
+        Resolves workspace-relative paths under ``AGENT_ROOT`` and honors
+        absolute paths (home-sandbox writes). Never raises — an unresolvable
+        path is treated as non-existent (additive), which the fail-closed
+        guard only escalates when the target actually exists.
+        """
+        try:
+            candidate = Path(path)
+            if not candidate.is_absolute():
+                candidate = Path(AGENT_ROOT) / candidate
+            return candidate.exists()
+        except Exception:
+            return False
+
     def user_role(self, current_user: str) -> str:
         users = self.load_users()
         try:
@@ -162,6 +179,22 @@ class ToolDispatchService:
                 status_code=403,
                 detail=f"'{tool_name}' 툴은 파괴적 작업으로 차단되었습니다.",
             )
+        # Fail-closed governance: a call that would rewrite existing content but
+        # cannot be staged as a reviewable proposal is blocked, never applied
+        # silently. New-file (additive) calls are unaffected.
+        verdict = classify_tool_call(
+            tool_name, args or {},
+            policy=dict(policy), path_exists=self._governed_path_exists,
+        )
+        if verdict.get("fail_closed"):
+            raise HTTPException(
+                status_code=409,
+                detail=(
+                    f"'{tool_name}' 툴은 기존 콘텐츠를 변경하지만 검토 가능한 제안으로 "
+                    "적용할 수 없어 차단되었습니다. 새 파일 이름으로 생성하거나 "
+                    "지원되는 편집 도구(write_file/edit_file)를 사용하세요."
+                ),
+            )
         if (
             require_auto_approval
             and not trusted_admin

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ltcai",
-  "version": "9.7.0",
+  "version": "9.9.0",
   "description": "Lattice AI — local-first Digital Brain that keeps your knowledge durable across any AI model.",
   "homepage": "https://github.com/TaeSooPark-PTS/LatticeAI#readme",
   "repository": {
@@ -24,10 +24,12 @@
     "build:assets": "vite build && node scripts/build_frontend_assets.mjs",
     "build:python": "node scripts/run_python.mjs -m build",
     "check:python": "node scripts/run_python.mjs scripts/check_python.py",
-    "lint": "npm run lint:python && node --check tests/visual/mock_server.cjs && node --check tests/visual/v3.spec.js && npm run lint:frontend && npm run frontend:openapi:check && node scripts/check_i18n_literals.mjs && npm run test:browser-extension",
+    "lint": "npm run lint:python && node --check tests/visual/mock_server.cjs && node --check tests/visual/v3.spec.js && npm run lint:frontend && npm run frontend:openapi:check && node scripts/check_i18n_literals.mjs && npm run check:bundle && npm run test:browser-extension",
     "lint:python": "node scripts/run_python.mjs -m ruff check .",
     "lint:frontend": "node scripts/lint_frontend.mjs",
-    "docs:check-links": "node scripts/check_markdown_links.mjs",
+    "check:bundle": "node scripts/check_bundle_budget.mjs",
+    "docs:check-links": "node scripts/check_markdown_links.mjs && node scripts/check_doc_status.mjs",
+    "docs:check-status": "node scripts/check_doc_status.mjs",
     "docs:check-current": "node scripts/check_current_release_docs.mjs",
     "typecheck": "npm run typecheck:frontend && cd vscode-extension && npm run build",
     "typecheck:frontend": "npx tsc -p tsconfig.json --noEmit",

package/scripts/bench_models.py

@@ -0,0 +1,312 @@
+#!/usr/bin/env python3
+"""Model robustness benchmark harness for the Lattice agent loop.
+
+What it measures
+================
+The product claim is that the Brain stays durable "across any AI model" — a
+weaker model may emit sloppier output, but the agent loop repairs it and still
+completes the task. This harness turns that claim into a **matrix** of three
+numbers per model tier:
+
+* ``success_rate`` — fraction of model outputs the loop could turn into a valid
+  action (i.e. the task can proceed).
+* ``repair_rate``  — of the successful parses, the fraction that ONLY succeeded
+  because the loop's tolerant parser had to repair the output (fences, prose,
+  trailing commas, ``<think>`` blocks, Python-dict literals). High repair_rate
+  = "this tier leans hard on the loop's robustness".
+* ``latency_ms``   — see the honesty note below.
+
+It measures **real code**: every output is fed through
+``latticeai.core.agent.extract_action_details`` — the exact parser/repair
+function the production loop uses (``agent.py`` calls it in plan/execute/verify).
+An ``agent-loop`` reference row additionally runs the real
+``latticeai.core.agent_eval.run_agent_eval`` state machine over its 20
+scripted scenarios.
+
+Two modes
+=========
+1. **scripted** (default, always runnable, no model/network): a curated corpus
+   of model-realistic outputs per quality tier (frontier / mid-local /
+   weak-local). Proves the harness works and exposes the loop's repair boundary
+   deterministically.
+2. **live** (opt-in, ``--live-endpoint``): sends a fixed set of agent prompts to
+   an OpenAI-compatible local endpoint (e.g. LM Studio / llama.cpp / vLLM) and
+   runs the *real* completions through the same parser, measuring true
+   end-to-end generation latency. Falls back to scripted with an honest message
+   if the endpoint is unreachable.
+
+Honesty note on latency
+========================
+In **scripted** mode ``latency_ms`` is the *parse+repair* cost only
+(microseconds); it is NOT model inference time and must not be read as such.
+Real generation latency is only meaningful in **live** mode.
+
+Usage
+=====
+    .venv/bin/python scripts/bench_models.py                    # scripted matrix
+    .venv/bin/python scripts/bench_models.py --json out.json    # + machine output
+    .venv/bin/python scripts/bench_models.py \
+        --live-endpoint http://127.0.0.1:1234/v1 --model my-local-model
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import statistics
+import sys
+import time
+import urllib.error
+import urllib.request
+from pathlib import Path
+from typing import Any, Dict, List, Tuple
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from latticeai.core.agent import extract_action_details  # noqa: E402
+from latticeai.core.agent_eval import run_agent_eval  # noqa: E402
+
+
+# ── Scripted corpora ─────────────────────────────────────────────────────
+# Each entry is one model output for a canonical agent turn. The tiers encode
+# how the SAME intent degrades as model quality drops. All "should_parse=True"
+# entries are recoverable by the real loop; "should_parse=False" entries are
+# genuinely broken (past the repair boundary) and SHOULD fail — that is what
+# makes success_rate < 1.0 meaningful rather than a rigged demo.
+
+_CLEAN = [
+    ('{"action": "plan", "goal": "ingest", "steps": [{"action": "write_file"}]}', True),
+    ('{"action": "write_file", "args": {"path": "note.txt", "content": "hi"}}', True),
+    ('{"action": "final", "message": "done"}', True),
+    ('{"action": "verdict", "verdict": "PASS", "next_state": "DONE", "reason": "ok"}', True),
+    ('{"action": "knowledge_graph_search", "args": {"query": "roadmap"}}', True),
+    ('{"action": "read_file", "args": {"path": "README.md"}}', True),
+]
+
+_MID = [
+    # markdown-fenced JSON (very common)
+    ('```json\n{"action": "plan", "goal": "ingest", "steps": []}\n```', True),
+    ('```\n{"action": "write_file", "args": {"path": "a.txt", "content": "x"}}\n```', True),
+    # prose preamble then the object
+    ('Sure! Here is the next step:\n{"action": "final", "message": "done"}', True),
+    # trailing comma before closing brace
+    ('{"action": "read_file", "args": {"path": "README.md",}}', True),
+    # trailing comma in array
+    ('{"action": "plan", "goal": "g", "steps": [{"action": "read_file"},]}', True),
+    ('```json\n{"action": "verdict", "verdict": "PASS", "next_state": "DONE", "reason": "ok"}\n```', True),
+]
+
+_WEAK = [
+    # <think> reasoning block that itself contains braces, then the action
+    ('<think>I should write the file {maybe}</think>\n{"action": "write_file", "args": {"path": "n.txt", "content": "c"}}', True),
+    ('<reasoning>ok</reasoning> {"action": "final", "message": "done"}', True),
+    # Python dict literal (single quotes, True) — ast.literal_eval path
+    ("{'action': 'read_file', 'args': {'path': 'x.txt'}}", True),
+    ("{'action': 'verdict', 'verdict': 'PASS', 'next_state': 'DONE', 'reason': 'ok'}", True),
+    # fenced + trailing comma + prose all at once
+    ('Here you go:\n```json\n{"action": "plan", "goal": "g", "steps": [],}\n```', True),
+    # genuinely broken: pure prose, no JSON object at all -> must fail
+    ("I think we are done here, nothing else to do.", False),
+    # genuinely broken: object but missing the required "action" field
+    ('{"message": "done", "status": "ok"}', False),
+    # genuinely broken: truncated / unbalanced braces past repair
+    ('{"action": "write_file", "args": {"path": "n.txt"', False),
+]
+
+_PROFILES: Dict[str, List[Tuple[str, bool]]] = {
+    "frontier (clean JSON)": _CLEAN,
+    "mid-local (fenced/prose/commas)": _MID,
+    "weak-local (think/py-literal/broken)": _WEAK,
+}
+
+
+def _bench_corpus(corpus: List[Tuple[str, bool]]) -> Dict[str, Any]:
+    total = len(corpus)
+    parsed = 0
+    repaired = 0
+    latencies: List[float] = []
+    mismatches: List[str] = []
+    repair_kinds: Dict[str, int] = {}
+    for raw, should_parse in corpus:
+        t0 = time.perf_counter()
+        try:
+            _action, repairs = extract_action_details(raw)
+            ok = True
+        except (ValueError, Exception):  # noqa: BLE001 - parser raises ValueError
+            ok = False
+            repairs = []
+        latencies.append((time.perf_counter() - t0) * 1000.0)
+        if ok:
+            parsed += 1
+            if repairs:
+                repaired += 1
+                for kind in repairs:
+                    repair_kinds[kind] = repair_kinds.get(kind, 0) + 1
+        if ok != should_parse:
+            mismatches.append(raw[:60])
+    return {
+        "total": total,
+        "parsed": parsed,
+        "repaired": repaired,
+        "success_rate": round(parsed / total, 4) if total else 0.0,
+        "repair_rate": round(repaired / parsed, 4) if parsed else 0.0,
+        "latency_ms_mean": round(statistics.fmean(latencies), 4) if latencies else 0.0,
+        "repair_kinds": repair_kinds,
+        # A healthy harness has zero mismatches: every "should_parse" call
+        # matched reality. Non-empty means the corpus/parser disagree.
+        "harness_mismatches": mismatches,
+    }
+
+
+# ── Live (OpenAI-compatible) path ────────────────────────────────────────
+_LIVE_PROMPTS = [
+    "Return ONLY one JSON object for the next agent step: read the file README.md.",
+    "Return ONLY one JSON object: plan to ingest a document, then finish.",
+    "Return ONLY one JSON object: write 'hello' to notes.txt.",
+    "Return ONLY one JSON object: a PASS verdict moving the loop to DONE.",
+    "Return ONLY one JSON object: search the knowledge graph for 'roadmap'.",
+]
+
+_SYSTEM = (
+    "You are the executor of an agent loop. Every reply MUST be exactly one "
+    'JSON object with an "action" field and nothing else.'
+)
+
+
+def _openai_chat(base_url: str, model: str, prompt: str, *, timeout: float) -> Tuple[str, float]:
+    url = base_url.rstrip("/") + "/chat/completions"
+    body = json.dumps(
+        {
+            "model": model,
+            "messages": [
+                {"role": "system", "content": _SYSTEM},
+                {"role": "user", "content": prompt},
+            ],
+            "temperature": 0.2,
+            "max_tokens": 256,
+        }
+    ).encode("utf-8")
+    req = urllib.request.Request(  # noqa: S310 - operator-supplied local endpoint
+        url, data=body, headers={"Content-Type": "application/json"}, method="POST"
+    )
+    t0 = time.perf_counter()
+    with urllib.request.urlopen(req, timeout=timeout) as res:  # noqa: S310
+        payload = json.loads(res.read().decode("utf-8", errors="replace"))
+    latency = (time.perf_counter() - t0) * 1000.0
+    text = payload["choices"][0]["message"]["content"]
+    return text, latency
+
+
+def _bench_live(base_url: str, model: str, *, timeout: float) -> Dict[str, Any]:
+    total = 0
+    parsed = 0
+    repaired = 0
+    latencies: List[float] = []
+    for prompt in _LIVE_PROMPTS:
+        total += 1
+        try:
+            text, latency = _openai_chat(base_url, model, prompt, timeout=timeout)
+        except (urllib.error.URLError, OSError, KeyError, ValueError) as exc:
+            return {"error": f"live endpoint unreachable/invalid: {exc}"}
+        latencies.append(latency)
+        try:
+            _action, repairs = extract_action_details(text)
+            parsed += 1
+            if repairs:
+                repaired += 1
+        except Exception:  # noqa: BLE001
+            pass
+    return {
+        "total": total,
+        "parsed": parsed,
+        "repaired": repaired,
+        "success_rate": round(parsed / total, 4) if total else 0.0,
+        "repair_rate": round(repaired / parsed, 4) if parsed else 0.0,
+        "latency_ms_mean": round(statistics.fmean(latencies), 2) if latencies else 0.0,
+    }
+
+
+# ── Reporting ────────────────────────────────────────────────────────────
+def _fmt_row(name: str, r: Dict[str, Any]) -> str:
+    return (
+        f"  {name:<40} "
+        f"success={r['success_rate']:<7} "
+        f"repair={r['repair_rate']:<7} "
+        f"n={r.get('total', '-'):<4} "
+        f"latency_ms={r['latency_ms_mean']}"
+    )
+
+
+def main(argv: List[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description="Model robustness benchmark harness")
+    parser.add_argument("--json", dest="json_out", help="write full report JSON to this path")
+    parser.add_argument("--live-endpoint", help="OpenAI-compatible base URL, e.g. http://127.0.0.1:1234/v1")
+    parser.add_argument("--model", help="model id for the live endpoint")
+    parser.add_argument("--timeout", type=float, default=30.0, help="live request timeout seconds")
+    args = parser.parse_args(argv)
+
+    report: Dict[str, Any] = {"mode": "scripted", "profiles": {}}
+
+    # Scripted matrix (always).
+    for name, corpus in _PROFILES.items():
+        report["profiles"][name] = _bench_corpus(corpus)
+
+    # Real agent-loop reference row.
+    loop = run_agent_eval()
+    report["agent_loop_reference"] = {
+        "scenarios": loop["scenarios"],
+        "passed": loop["passed"],
+        "success_rate": loop["success_rate"],
+        "recovery_rate": loop["recovery_rate"],
+        "parse_errors": loop["parse_errors"],
+        "parse_recovered": loop["parse_recovered"],
+    }
+
+    # Optional live row.
+    if args.live_endpoint:
+        if not args.model:
+            print("--live-endpoint requires --model", file=sys.stderr)
+            return 2
+        live = _bench_live(args.live_endpoint, args.model, timeout=args.timeout)
+        report["mode"] = "scripted+live"
+        report["live"] = {"endpoint": args.live_endpoint, "model": args.model, **live}
+
+    # Human-readable matrix.
+    print("Model robustness benchmark (real parser: extract_action_details)")
+    print("=" * 72)
+    print("Scripted tiers (latency_ms = parse+repair only, NOT model inference):")
+    mismatch_total = 0
+    for name, r in report["profiles"].items():
+        print(_fmt_row(name, r))
+        if r["repair_kinds"]:
+            print(f"      repairs used: {r['repair_kinds']}")
+        mismatch_total += len(r["harness_mismatches"])
+    ref = report["agent_loop_reference"]
+    print("-" * 72)
+    print(
+        f"  agent-loop (real state machine)          "
+        f"success={ref['success_rate']:<7} "
+        f"recovery={ref['recovery_rate']:<7} "
+        f"scenarios={ref['scenarios']}"
+    )
+    if "live" in report:
+        lv = report["live"]
+        print("-" * 72)
+        if "error" in lv:
+            print(f"  live [{lv['model']}] SKIPPED: {lv['error']}")
+        else:
+            print(_fmt_row(f"live [{lv['model']}] (real inference)", lv))
+    print("=" * 72)
+    print(f"harness self-check: {mismatch_total} corpus/parser mismatches (0 = healthy)")
+
+    if args.json_out:
+        Path(args.json_out).write_text(json.dumps(report, indent=2), encoding="utf-8")
+        print(f"wrote {args.json_out}")
+
+    # Non-zero exit only if the harness itself is inconsistent (a real bug),
+    # never because a weak tier scored low — low scores are the finding.
+    return 1 if mismatch_total else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())