ltcai 9.9.3 → 9.9.5

package/scripts/bench_agent_smoke.py

@@ -0,0 +1,410 @@
+#!/usr/bin/env python3
+"""Weekly real-model agent-loop smoke (review Wave 3.4) — FAIL-OPEN.
+
+What it measures
+================
+``scripts/agent_eval.py`` proves the loop's state machine against *scripted*
+model replies. This harness closes the remaining gap: it drives a small set of
+canonical agent tasks through the REAL :class:`latticeai.core.agent.
+SingleAgentRuntime` (plan → approve → execute → verify), with the generation
+port wired to each *locally installed* MLX model — real inference, no HTTP
+server. Per model and task it reports:
+
+* ``final_state``   — the loop's honest terminal state (DONE / NEEDS_REVIEW /
+  FAILED), plus the ``agent_eval`` result-class bucket;
+* ``steps``         — LLM calls the run needed;
+* parse repairs     — ``parse_errors`` / ``parse_recovered`` and the LoopTrace
+  ``repairs`` histogram (how hard the loop worked to keep this model on rails);
+* ``duration_s``    — true end-to-end wall time including inference.
+
+Model access pattern
+====================
+Identical to ``scripts/bench_models.py --filegen``: installed gemma/qwen/llama
+MLX models are discovered through the product's own model catalog + on-disk HF
+checks, loaded with the real ``LLMRouter``, and unloaded afterwards. The tool
+port is in-memory (canned results, nothing touches disk or network) — this is
+a smoke of the LOOP over real model output, not of the tool implementations.
+
+FAIL-OPEN by design
+===================
+This is a scheduled/weekly report, never a CI gate:
+
+* no models installed (or catalog import fails, e.g. on a CI runner) →
+  an honest "no models available — skipped (fail-open)" report, exit 0;
+* a model that fails to load → a per-model skip entry, exit 0;
+* any run-level crash → a skip report naming the error, exit 0;
+* weak results are the finding, never a failure.
+
+Non-zero exits happen only for real script errors (bad flags → argparse's 2).
+
+Usage
+=====
+    .venv/bin/python scripts/bench_agent_smoke.py               # human table
+    .venv/bin/python scripts/bench_agent_smoke.py --json        # JSON to stdout
+    .venv/bin/python scripts/bench_agent_smoke.py --tasks 1 --max-steps 6
+    .venv/bin/python scripts/bench_agent_smoke.py --model mlx-community/...
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import sys
+import tempfile
+import time
+from pathlib import Path
+from typing import Any, Awaitable, Callable, Dict, List, Optional, Tuple
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from latticeai.core.agent import (  # noqa: E402
+    AgentDeps,
+    AgentRunContext,
+    AgentState,
+    SingleAgentRuntime,
+)
+from latticeai.core.agent_eval import classify_result  # noqa: E402
+from latticeai.core.agent_prompts import (  # noqa: E402
+    CRITIC_PROMPT,
+    EXECUTOR_PROMPT,
+    MEMORY_UPDATER_PROMPT,
+    PLANNER_PROMPT,
+)
+from latticeai.tools import ToolError  # noqa: E402
+
+
+# ── canonical smoke tasks ────────────────────────────────────────────────
+# Small, file-flavored tasks users actually ask for. The tool port is canned,
+# so the measurement is "can this model steer the real loop to completion",
+# not "is the artifact beautiful".
+SMOKE_TASKS: List[Tuple[str, str]] = [
+    ("haiku-file", "Write a short haiku about autumn into haiku.txt."),
+    ("hello-script", "Create a Python file hello.py that prints hello."),
+    (
+        "list-then-summarize",
+        "List the files in the workspace, then summarize what you see in one sentence.",
+    ),
+]
+
+_SMOKE_MODEL_FAMILIES = ("gemma", "qwen", "llama")
+
+_AUTO_POLICY = {
+    "auto_approve": True, "risk": "low", "shell": False, "network": False,
+    "destructive": False, "sandbox": False, "rollback": "none",
+}
+
+
+def discover_agent_models() -> List[Dict[str, str]]:
+    """Installed local MLX models — the exact discovery pattern of
+    ``scripts/bench_models.py --filegen`` (product catalog + on-disk HF
+    checks). Any import/probe failure is fail-open: an empty list, never an
+    exception. Messages go to stderr so ``--json`` stdout stays parseable."""
+    try:
+        from latticeai.models.router import (
+            _looks_like_hf_model_dir,
+            hf_cache_model_dir,
+            hf_model_dir,
+        )
+        from latticeai.services.model_catalog import ENGINE_MODEL_CATALOG
+    except Exception as exc:  # noqa: BLE001 - discovery must never crash the report
+        print(
+            f"agent-smoke: model catalog unavailable ({exc}); no models discovered",
+            file=sys.stderr,
+        )
+        return []
+    found: List[Dict[str, str]] = []
+    for entry in ENGINE_MODEL_CATALOG.get("local_mlx", []):
+        model_id = str(entry.get("id") or "")
+        lowered = model_id.lower()
+        family = next((f for f in _SMOKE_MODEL_FAMILIES if f in lowered), None)
+        if not family:
+            continue
+        try:
+            downloaded = (
+                _looks_like_hf_model_dir(hf_model_dir(model_id))
+                or hf_cache_model_dir(model_id) is not None
+            )
+        except Exception:  # noqa: BLE001
+            downloaded = False
+        if downloaded:
+            found.append({"id": model_id, "family": family})
+    return found
+
+
+# ── HTTP-less single-agent runtime over an in-memory tool port ───────────
+
+class _SmokeReq:
+    conversation_id = None
+    temperature = 0.2
+    workspace_id = None
+    source = "agent_smoke"
+
+    def __init__(self, message: str) -> None:
+        self.message = message
+
+
+def _build_smoke_deps(
+    generate_as: Callable[..., Awaitable[Any]],
+    tool_log: List[Dict[str, Any]],
+) -> AgentDeps:
+    """Real production prompts + a canned in-memory tool port.
+
+    The port mirrors the fake in ``latticeai.core.agent_eval``: it records
+    calls, fails a pathless write like the real dispatcher, and never touches
+    disk or network. All tools are auto-approve — governance behaviour is
+    agent_eval's job; this smoke measures loop mechanics over real inference.
+    """
+
+    async def generate(**kwargs):
+        return '{"action": "noop"}'
+
+    def execute_tool(name: str, args: dict) -> dict:
+        if name in ("write_file", "generate_file") and not args.get("path"):
+            raise ToolError(f"{name} requires args.path")
+        tool_log.append({"name": name, "args": args})
+        if name == "list_dir":
+            return {"ok": True, "entries": ["README.md", "haiku.txt", "hello.py"]}
+        if name == "read_file":
+            return {"ok": True, "path": args.get("path", ""), "content": "smoke fixture file"}
+        return {"ok": True, "path": args.get("path", "")}
+
+    return AgentDeps(
+        generate_as=generate_as,
+        generate=generate,
+        execute_tool=execute_tool,
+        policy_for=lambda name, args: dict(_AUTO_POLICY),
+        risk_level=lambda p: p["risk"],
+        check_role=lambda name, user: None,
+        tool_governance={
+            "write_file": dict(_AUTO_POLICY),
+            "read_file": dict(_AUTO_POLICY),
+            "list_dir": dict(_AUTO_POLICY),
+            "generate_file": dict(_AUTO_POLICY),
+            "knowledge_graph_ingest": dict(_AUTO_POLICY),
+            "knowledge_graph_search": dict(_AUTO_POLICY),
+        },
+        file_create_actions=frozenset({"write_file", "generate_file"}),
+        recent_chat_context=lambda **kw: "",
+        clear_history=lambda keep: {"ok": True},
+        knowledge_save=lambda *a, **kw: None,
+        audit=lambda *a, **kw: None,
+        planner_prompt=PLANNER_PROMPT,
+        executor_prompt=EXECUTOR_PROMPT,
+        critic_prompt=CRITIC_PROMPT,
+        memory_updater_prompt=MEMORY_UPDATER_PROMPT,
+        agent_root=Path(tempfile.gettempdir()) / "agent-smoke",
+    )
+
+
+async def _run_one_task(
+    generate_as: Callable[..., Awaitable[Any]],
+    task_id: str,
+    prompt: str,
+    *,
+    max_steps: int = 8,
+    model_id: Optional[str] = None,
+) -> Dict[str, Any]:
+    """Drive one task through the real state machine; reduce to a report row."""
+    tool_log: List[Dict[str, Any]] = []
+    runtime = SingleAgentRuntime(_build_smoke_deps(generate_as, tool_log))
+    ctx = AgentRunContext()
+    ctx.state = AgentState.PLANNING
+    ctx.executing_model = model_id
+    ctx.reviewing_model = model_id
+    req = _SmokeReq(prompt)
+
+    t0 = time.perf_counter()
+    await runtime.plan(ctx, req, "en", "smoke@local", model_id=model_id)
+    # The operator launched this run deliberately — the human approval the
+    # gate asks for. The gate itself (and denial paths) are covered by
+    # agent_eval; a real model's free-form plan must not dead-end the smoke.
+    runtime.approve(ctx, "smoke@local", approved_by_human=True)
+    if ctx.state == AgentState.EXECUTING:
+        await runtime.run_to_completion(
+            ctx, req, "en", "smoke@local", max_steps=max_steps, max_retry=2
+        )
+    duration = round(time.perf_counter() - t0, 2)
+
+    summary = ctx.trace.summary()
+    executed = [call["name"] for call in tool_log]
+    return {
+        "task": task_id,
+        "final_state": ctx.state.value,
+        "result_class": classify_result(ctx.state.value, ctx.trace.events, summary, executed),
+        "steps": summary["llm_calls"],
+        "parse_errors": summary["parse_errors"],
+        "parse_recovered": summary["parse_recovered"],
+        "repairs_total": sum((summary.get("repairs") or {}).values()),
+        "repairs": summary.get("repairs") or {},
+        "tool_calls": len(tool_log),
+        "duration_s": duration,
+    }
+
+
+def _make_router_generate_as(router: Any, model_id: str) -> Callable[..., Awaitable[str]]:
+    """Bridge the runtime's generate_as port to the real LLMRouter (the same
+    ``generate_as`` call shape bench_models' filegen mode uses)."""
+
+    async def generate_as(_model_id, message, context, max_tokens, temperature):
+        return str(
+            await router.generate_as(
+                _model_id or model_id,
+                message=message,
+                context=context,
+                max_tokens=max_tokens,
+                temperature=temperature,
+            )
+        )
+
+    return generate_as
+
+
+async def _smoke_run_models(
+    models: List[Dict[str, str]],
+    tasks: List[Tuple[str, str]],
+    max_steps: int,
+) -> List[Dict[str, Any]]:
+    from latticeai.models.router import LLMRouter
+
+    router = LLMRouter()
+    results: List[Dict[str, Any]] = []
+    for model in models:
+        model_id = model["id"]
+        try:
+            await router.load_model(model_id)
+        except Exception as exc:  # noqa: BLE001 - fail-open per model
+            results.append({
+                "model": model_id, "family": model["family"],
+                "skipped": True, "reason": f"load failed: {exc}",
+            })
+            continue
+        generate_as = _make_router_generate_as(router, model_id)
+        rows: List[Dict[str, Any]] = []
+        for task_id, prompt in tasks:
+            try:
+                rows.append(await _run_one_task(
+                    generate_as, task_id, prompt,
+                    max_steps=max_steps, model_id=model_id,
+                ))
+            except Exception as exc:  # noqa: BLE001 - one broken task never sinks the report
+                rows.append({
+                    "task": task_id, "final_state": "HARNESS_ERROR",
+                    "result_class": "failed", "error": str(exc),
+                })
+        results.append({
+            "model": model_id,
+            "family": model["family"],
+            "tasks": rows,
+            "tasks_total": len(rows),
+            "completed": sum(1 for r in rows if r.get("final_state") == "DONE"),
+            "duration_s": round(sum(float(r.get("duration_s") or 0.0) for r in rows), 2),
+        })
+        try:
+            router.unload_model(model_id)
+        except Exception:  # noqa: BLE001
+            pass
+    return results
+
+
+def run_agent_smoke(
+    models: Optional[List[Dict[str, str]]] = None,
+    tasks: Optional[List[Tuple[str, str]]] = None,
+    max_steps: int = 8,
+) -> Dict[str, Any]:
+    """Weekly per-model agent-loop report. Fail-open: never raises, never gates."""
+    if models is None:
+        models = discover_agent_models()
+    selected_tasks = list(tasks or SMOKE_TASKS)
+    if not models:
+        return {
+            "mode": "agent-smoke",
+            "status": "skipped",
+            "reason": (
+                "no models available — install a local gemma/qwen/llama model "
+                "via the app's model picker, then re-run"
+            ),
+            "models": [],
+        }
+    try:
+        results = asyncio.run(_smoke_run_models(models, selected_tasks, max_steps))
+    except Exception as exc:  # noqa: BLE001 - fail-open at the run level too
+        return {
+            "mode": "agent-smoke", "status": "skipped",
+            "reason": f"smoke run failed: {exc}", "models": [],
+        }
+    return {"mode": "agent-smoke", "status": "ok", "models": results}
+
+
+def format_smoke_report(report: Dict[str, Any]) -> str:
+    lines = [
+        "Weekly agent-loop smoke (real models through SingleAgentRuntime)",
+        "=" * 72,
+    ]
+    if report.get("status") == "skipped":
+        lines.append(f"no models available — skipped (fail-open): {report.get('reason')}")
+        lines.append("=" * 72)
+        return "\n".join(lines)
+    for model in report.get("models", []):
+        if model.get("skipped"):
+            lines.append(f"  {model['model']}: SKIPPED — {model.get('reason')}")
+            continue
+        lines.append(
+            f"  {model['model']} "
+            f"(completed {model['completed']}/{model['tasks_total']}, "
+            f"{model['duration_s']}s)"
+        )
+        for row in model.get("tasks", []):
+            if row.get("final_state") == "HARNESS_ERROR":
+                lines.append(f"    {row['task']:<22} HARNESS_ERROR  {row.get('error')}")
+                continue
+            lines.append(
+                f"    {row['task']:<22} {row['final_state']:<13} "
+                f"steps={row['steps']} parse_errors={row['parse_errors']} "
+                f"repairs={row['repairs_total']} tools={row['tool_calls']} "
+                f"{row['duration_s']}s"
+            )
+    lines.append("=" * 72)
+    lines.append(
+        "note: fail-open weekly report — weak results are the finding, never a failure"
+    )
+    return "\n".join(lines)
+
+
+def main(argv: List[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        description="Weekly real-model agent-loop smoke (fail-open, never a gate)"
+    )
+    parser.add_argument(
+        "--json", dest="json_out", action="store_true",
+        help="print the machine-readable report JSON to stdout",
+    )
+    parser.add_argument(
+        "--tasks", type=int, default=len(SMOKE_TASKS),
+        help=f"how many smoke tasks to run per model (1-{len(SMOKE_TASKS)}, default all)",
+    )
+    parser.add_argument(
+        "--max-steps", type=int, default=8, help="agent-loop step budget per task",
+    )
+    parser.add_argument(
+        "--model", help="restrict the run to one installed model id",
+    )
+    args = parser.parse_args(argv)
+
+    count = max(1, min(int(args.tasks), len(SMOKE_TASKS)))
+    models = discover_agent_models()
+    if args.model:
+        models = [m for m in models if m["id"] == args.model]
+    report = run_agent_smoke(
+        models=models, tasks=SMOKE_TASKS[:count], max_steps=max(2, int(args.max_steps)),
+    )
+    if args.json_out:
+        print(json.dumps(report, ensure_ascii=False, indent=2))
+    else:
+        print(format_smoke_report(report))
+    # FAIL-OPEN: missing or weak models never produce a non-zero exit.
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/scripts/check_current_release_docs.mjs

@@ -6,7 +6,7 @@ const root = process.cwd();
 const pkg = JSON.parse(readFileSync(path.join(root, "package.json"), "utf8"));
 const version = pkg.version;
 const releaseDir = `output/release/v${version}`;
-const releaseTheme = "Closed Loops";
+const releaseTheme = "Closed Gaps";
 const title = `${version} — ${releaseTheme}`;
 const escapedVersion = version.replaceAll(".", "\\.");
 

package/scripts/funnel_soft_gate.py

@@ -0,0 +1,192 @@
+#!/usr/bin/env python3
+"""Funnel soft gate (review Wave 3.3) — funnel metrics as an advisory verdict.
+
+Reads the same ``funnel_metrics.json`` production writes and grades it through
+``FunnelMetricsService.snapshot()`` — no rate logic is duplicated here. Two
+funnel-health rates carry WARN thresholds:
+
+* ``code_only_rate``    — WARN above 0.05 (goal: >95% of recognized file
+  requests deliver a real artifact, not a code/prose-only answer);
+* ``needs_review_rate`` — WARN above 0.30 (verifier fail-closed is healthy,
+  but a third of runs ending NEEDS_REVIEW means verification is starving).
+
+Rates whose denominator is still zero come back ``None`` from the snapshot;
+they are reported as "no data" and never warn — a fresh install stays green.
+``real_file_rate``, ``approval_resume_rate`` and TTFV are shown as
+informational context only.
+
+Exit codes
+==========
+* default         — advisory: ALWAYS ``0``; warnings are printed, not enforced;
+* ``--strict``    — CI opt-in: ``1`` when any WARN fired ("no data" never
+  fails, even under --strict);
+* bad flags       — argparse's usual ``2`` (the only real script error).
+
+Usage
+=====
+    .venv/bin/python scripts/funnel_soft_gate.py                 # advisory report
+    .venv/bin/python scripts/funnel_soft_gate.py --strict        # CI gate opt-in
+    .venv/bin/python scripts/funnel_soft_gate.py --json          # machine output
+    .venv/bin/python scripts/funnel_soft_gate.py --path /tmp/m.json
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from latticeai.services.funnel_metrics import FunnelMetricsService  # noqa: E402
+
+# Advisory thresholds (overridable per run via flags). WARN fires strictly
+# ABOVE the threshold, so a rate sitting exactly on the goal boundary passes.
+CODE_ONLY_WARN_THRESHOLD = 0.05
+NEEDS_REVIEW_WARN_THRESHOLD = 0.30
+
+
+def default_metrics_path() -> Path:
+    """The exact file production writes: ``Config.from_env`` data-dir pattern
+    (``LATTICEAI_DATA_DIR`` env or ``~/.ltcai``) + the ``funnel_metrics.json``
+    filename from ``latticeai/runtime/persistence_runtime.py``."""
+    data_dir = Path(os.environ.get("LATTICEAI_DATA_DIR") or (Path.home() / ".ltcai"))
+    return data_dir / "funnel_metrics.json"
+
+
+def evaluate_snapshot(
+    snapshot: Dict[str, Any],
+    *,
+    code_only_warn: float = CODE_ONLY_WARN_THRESHOLD,
+    needs_review_warn: float = NEEDS_REVIEW_WARN_THRESHOLD,
+) -> Dict[str, Any]:
+    """Grade a ``FunnelMetricsService.snapshot()`` payload into verdicts."""
+    rates = snapshot.get("rates") or {}
+    checks: List[Dict[str, Any]] = []
+
+    def check(name: str, threshold: float, goal: str) -> None:
+        rate: Optional[float] = rates.get(name)
+        if rate is None:
+            verdict, detail = "no_data", "no denominator recorded yet"
+        elif rate > threshold:
+            verdict, detail = "warn", f"{rate} > {threshold} — {goal}"
+        else:
+            verdict, detail = "ok", f"{rate} <= {threshold}"
+        checks.append({
+            "name": name, "rate": rate, "threshold": threshold,
+            "verdict": verdict, "detail": detail,
+        })
+
+    check("code_only_rate", code_only_warn,
+          "goal: >95% of file requests deliver real files")
+    check("needs_review_rate", needs_review_warn,
+          "goal: <30% of agent runs end NEEDS_REVIEW")
+
+    warnings = [c["name"] for c in checks if c["verdict"] == "warn"]
+    if warnings:
+        status = "warn"
+    elif all(c["verdict"] == "no_data" for c in checks):
+        status = "no_data"
+    else:
+        status = "ok"
+    return {
+        "status": status,
+        "warnings": warnings,
+        "checks": checks,
+        # Informational context — never gated.
+        "info": {
+            "real_file_rate": rates.get("real_file_rate"),
+            "approval_resume_rate": rates.get("approval_resume_rate"),
+            "ttfv_seconds": snapshot.get("ttfv_seconds"),
+            "counters": snapshot.get("counters") or {},
+        },
+    }
+
+
+def format_report(payload: Dict[str, Any]) -> str:
+    lines = [
+        f"Funnel soft gate (advisory) — {payload['path']}",
+        "=" * 72,
+    ]
+    for check in payload["checks"]:
+        rate = "no data" if check["rate"] is None else str(check["rate"])
+        verdict = {"warn": "WARN", "ok": "ok", "no_data": "-"}[check["verdict"]]
+        lines.append(
+            f"  {check['name']:<20} {rate:<8} {verdict:<5} {check['detail']}"
+        )
+    info = payload["info"]
+    lines.append("-" * 72)
+    lines.append(
+        f"  info: real_file_rate={info['real_file_rate']} "
+        f"approval_resume_rate={info['approval_resume_rate']} "
+        f"ttfv_seconds={info['ttfv_seconds']}"
+    )
+    lines.append("=" * 72)
+    if payload["status"] == "warn":
+        mode = (
+            "STRICT — exiting 1" if payload["strict"]
+            else "advisory — exit 0; opt into enforcement with --strict"
+        )
+        lines.append(f"verdict: WARN ({', '.join(payload['warnings'])}) [{mode}]")
+    elif payload["status"] == "no_data":
+        lines.append("verdict: no data yet — nothing to grade (exit 0)")
+    else:
+        lines.append("verdict: OK — funnel rates within goals")
+    return "\n".join(lines)
+
+
+def main(argv: List[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        description="Advisory funnel-metrics gate (warn on funnel-health regressions)"
+    )
+    parser.add_argument(
+        "--path",
+        help="metrics JSON path (default: the production data-dir funnel_metrics.json)",
+    )
+    parser.add_argument(
+        "--strict", action="store_true",
+        help="exit 1 when any WARN fired (CI opt-in; 'no data' still exits 0)",
+    )
+    parser.add_argument(
+        "--json", dest="json_out", action="store_true",
+        help="print the machine-readable verdict JSON to stdout",
+    )
+    parser.add_argument(
+        "--code-only-warn", type=float, default=CODE_ONLY_WARN_THRESHOLD,
+        help=f"code_only_rate WARN threshold (default {CODE_ONLY_WARN_THRESHOLD})",
+    )
+    parser.add_argument(
+        "--needs-review-warn", type=float, default=NEEDS_REVIEW_WARN_THRESHOLD,
+        help=f"needs_review_rate WARN threshold (default {NEEDS_REVIEW_WARN_THRESHOLD})",
+    )
+    args = parser.parse_args(argv)
+
+    path = Path(args.path) if args.path else default_metrics_path()
+    # A missing or corrupt file yields the service's zeroed state → all rates
+    # None → "no data" (never a crash, never a failure).
+    snapshot = FunnelMetricsService(path).snapshot()
+    result = evaluate_snapshot(
+        snapshot,
+        code_only_warn=args.code_only_warn,
+        needs_review_warn=args.needs_review_warn,
+    )
+    exit_code = 1 if (args.strict and result["status"] == "warn") else 0
+    payload = {
+        "mode": "funnel-soft-gate",
+        "path": str(path),
+        "strict": bool(args.strict),
+        "exit_code": exit_code,
+        **result,
+    }
+    if args.json_out:
+        print(json.dumps(payload, ensure_ascii=False, indent=2))
+    else:
+        print(format_report(payload))
+    return exit_code
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())