omnilane 0.42.5 → 0.42.7

package/scripts/lib/probe.py

@@ -3,11 +3,14 @@
 
 Runs one CLI invocation, captures raw stdout/stderr to files, and writes a
 descriptor with the original command/stream fields plus a vendor-specific
-verdict, its reason, and the observed model when the response proves it.
+verdict, its reason, the observed model, and the tier of evidence that model
+rests on.
 """
 import argparse
+import glob
 import json
 import os
+import re
 import subprocess
 import time
 from datetime import datetime, timezone
@@ -16,6 +19,88 @@ from pathlib import Path
 SWEEP_ID = os.environ.get("OMNILANE_TRANSPORT_SWEEP_ID", "overlay-reprobe-20260909")
 DEFAULT_ROOT = Path.home() / ".omnilane" / "transport-evidence" / SWEEP_ID
 
+# Ordered strongest first. The tier is read off the evidence a run produced, not
+# off the vendor: a CLI that starts reporting a billed model earns the higher
+# tier with no change here.
+TIER_BILLED = "billed-model"       # the provider named the model it charged for
+TIER_ECHO = "client-echo"          # the CLI recorded the model it asked for
+TIER_SELECTOR = "selector-only"    # the CLI accepted the selector and said no more
+EVIDENCE_TIERS = (TIER_BILLED, TIER_ECHO, TIER_SELECTOR)
+
+
+def codex_failure(stdout_text: str) -> str:
+    """The last thing codex's event stream said went wrong, if anything.
+
+    Later events supersede earlier ones: a retry notice is progress, the message
+    on `turn.failed` is the outcome.
+    """
+    latest = ""
+    for line in stdout_text.splitlines():
+        line = line.strip()
+        if not line.startswith("{"):
+            continue
+        try:
+            event = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        if not isinstance(event, dict):
+            continue
+        if event.get("type") == "turn.failed":
+            message = (event.get("error") or {}).get("message")
+        elif event.get("type") == "error":
+            message = event.get("message")
+        else:
+            continue
+        if isinstance(message, str) and message:
+            latest = message
+    return f"; codex-event: {latest[:200]}" if latest else ""
+
+
+def _requested_model(command: list) -> str | None:
+    requested = None
+    for index, argument in enumerate(command):
+        if not isinstance(argument, str):
+            continue
+        if argument in ("--model", "-m") and index + 1 < len(command):
+            requested = command[index + 1]
+        elif argument.startswith("--model="):
+            requested = argument.split("=", 1)[1]
+    return requested
+
+
+def record_values(record_text: str) -> list[str]:
+    """Comparable values from a client record digest: `key<TAB>value` lines.
+
+    `#` lines carry provenance and display labels that must never be compared —
+    agy's backend label is "Gemini 3.8 Flash (Low)", not a model identifier.
+    """
+    values = []
+    for line in (record_text or "").splitlines():
+        if line.startswith("#") or "\t" not in line:
+            continue
+        value = line.split("\t", 1)[1].strip()
+        if value:
+            values.append(value)
+    return values
+
+
+def _client_record(
+    evidence_json: dict,
+    record_text: str,
+    passed_reason: str,
+) -> tuple[str, str, str | None, str]:
+    """Judge a CLI's own on-disk record of the request it sent."""
+    values = record_values(record_text)
+    if not values:
+        return "pass", f"{passed_reason}; no-client-record", None, TIER_SELECTOR
+    observed_model = ", ".join(sorted(set(values)))
+    requested_model = _requested_model(evidence_json.get("command", []))
+    if not requested_model:
+        return "fail", "missing-requested-model", observed_model, TIER_SELECTOR
+    if requested_model not in values:
+        return "fail", "client-record-mismatch", observed_model, TIER_SELECTOR
+    return "pass", f"{passed_reason}-and-client-record-matched", observed_model, TIER_ECHO
+
 
 def verdict(
     evidence_json: dict,
@@ -23,25 +108,27 @@ def verdict(
     stderr_text: str,
     vendor: str,
     expected_token: str | None,
-) -> tuple[str, str, str | None]:
+    extra: dict | None = None,
+) -> tuple[str, str, str | None, str]:
     """Judge raw evidence without reading files, running commands or mutating it.
 
-    Only Claude's billed modelUsage keys currently prove the responding model.
-    A requested selector (including the Codex banner) is not observed identity.
+    `extra` carries text the caller already gathered from disk, so this stays a
+    pure function that can re-judge an old sweep offline.
     """
+    extra = extra or {}
     if evidence_json.get("timed_out"):
-        return "fail", "timeout", None
+        return "fail", "timeout", None, TIER_SELECTOR
     if not expected_token:
-        return "fail", "missing-expected-token", None
+        return "fail", "missing-expected-token", None, TIER_SELECTOR
 
     exit_code = evidence_json.get("exit_code")
     if vendor == "claude":
         try:
             response = json.loads(stdout_text)
         except (json.JSONDecodeError, TypeError):
-            return "fail", "invalid-json", None
+            return "fail", "invalid-json", None, TIER_SELECTOR
         if not isinstance(response, dict):
-            return "fail", "invalid-json-result", None
+            return "fail", "invalid-json-result", None, TIER_SELECTOR
         usage = response.get("modelUsage")
         models = sorted(usage) if isinstance(usage, dict) else []
         observed_model = ", ".join(models) or None
@@ -56,48 +143,168 @@ def verdict(
                 reason = "api-error"
             else:
                 reason = "result-error"
-            return "fail", f"{reason}: {result[:120]}", observed_model
+            return "fail", f"{reason}: {result[:120]}", observed_model, TIER_SELECTOR
         if not models:
-            return "fail", "missing-model-usage", None
-        command = evidence_json.get("command", [])
-        requested_model = None
-        for index, argument in enumerate(command):
-            if argument == "--model" and index + 1 < len(command):
-                requested_model = command[index + 1]
-            elif isinstance(argument, str) and argument.startswith("--model="):
-                requested_model = argument.split("=", 1)[1]
+            return "fail", "missing-model-usage", None, TIER_SELECTOR
+        requested_model = _requested_model(evidence_json.get("command", []))
         if not requested_model:
-            return "fail", "missing-requested-model", observed_model
+            return "fail", "missing-requested-model", observed_model, TIER_SELECTOR
         if models != [requested_model]:
-            return "fail", "model-mismatch", observed_model
+            return "fail", "model-mismatch", observed_model, TIER_SELECTOR
         if "unknown --effort" in stderr_text.lower():
-            return "fail", "effort-silently-defaulted", observed_model
+            return "fail", "effort-silently-defaulted", observed_model, TIER_SELECTOR
         if exit_code != 0:
-            return "fail", f"exit-code: {exit_code}", observed_model
+            return "fail", f"exit-code: {exit_code}", observed_model, TIER_SELECTOR
         if expected_token not in result:
-            return "fail", "missing-expected-token", observed_model
-        return "pass", "expected-token-and-model-matched", observed_model
+            return "fail", "missing-expected-token", observed_model, TIER_SELECTOR
+        return "pass", "expected-token-and-model-matched", observed_model, TIER_BILLED
+
+    if vendor == "grok":
+        if exit_code != 0:
+            return "fail", f"exit-code: {exit_code}: {stderr_text[:120]}", None, TIER_SELECTOR
+        if stderr_text:
+            return "fail", f"unexpected-stderr: {stderr_text[:120]}", None, TIER_SELECTOR
+        if expected_token not in stdout_text:
+            return "fail", "missing-expected-token", None, TIER_SELECTOR
+        try:
+            response = json.loads(stdout_text)
+        except (json.JSONDecodeError, TypeError):
+            response = None
+        usage = response.get("modelUsage") if isinstance(response, dict) else None
+        models = sorted(usage) if isinstance(usage, dict) else []
+        if not models:
+            return "pass", "expected-token-and-clean-stderr; no-billed-model", None, TIER_SELECTOR
+        observed_model = ", ".join(models)
+        requested_model = _requested_model(evidence_json.get("command", []))
+        if not requested_model:
+            return "fail", "missing-requested-model", observed_model, TIER_SELECTOR
+        # Grok bills `grok-4.6` as `grok-4.6-build`. Accept that one suffix and
+        # nothing else: a prefix test would let `grok-4.6-anything` pass.
+        if models not in ([requested_model], [f"{requested_model}-build"]):
+            return "fail", "model-mismatch", observed_model, TIER_SELECTOR
+        return "pass", "expected-token-and-billed-model-matched", observed_model, TIER_BILLED
 
-    if vendor in ("grok", "agy"):
+    if vendor == "agy":
         if exit_code != 0:
-            return "fail", f"exit-code: {exit_code}: {stderr_text[:120]}", None
+            return "fail", f"exit-code: {exit_code}: {stderr_text[:120]}", None, TIER_SELECTOR
         if stderr_text:
-            return "fail", f"unexpected-stderr: {stderr_text[:120]}", None
+            return "fail", f"unexpected-stderr: {stderr_text[:120]}", None, TIER_SELECTOR
         if expected_token not in stdout_text:
-            return "fail", "missing-expected-token", None
-        return "pass", "expected-token-and-clean-stderr", None
+            return "fail", "missing-expected-token", None, TIER_SELECTOR
+        return _client_record(evidence_json, extra.get("cli_log", ""),
+                              "expected-token-and-clean-stderr")
 
     if vendor == "codex":
         diagnostics = [line[:120] for line in stderr_text.splitlines()
                        if "error" in line.lower() or "warning" in line.lower()]
         review = "; stderr-review: " + " | ".join(diagnostics) if diagnostics else ""
+        # Under `--json` the refusal that ended the run is an stdout event, not
+        # a stderr line, so a failure would otherwise be recorded as a bare exit
+        # code and leave unproven[] saying nothing a reader can act on.
+        why = codex_failure(stdout_text)
         if exit_code != 0:
-            return "fail", f"exit-code: {exit_code}{review}", None
+            return "fail", f"exit-code: {exit_code}{why}{review}", None, TIER_SELECTOR
         if expected_token not in stdout_text:
-            return "fail", f"missing-expected-token{review}", None
-        return "pass", f"expected-token-matched{review}", None
+            return "fail", f"missing-expected-token{why}{review}", None, TIER_SELECTOR
+        result, reason, observed, tier = _client_record(
+            evidence_json, extra.get("rollout", ""), "expected-token-matched")
+        return result, f"{reason}{review}", observed, tier
+
+    return "fail", f"unsupported-vendor: {vendor}", None, TIER_SELECTOR
+
+
+def _sha256(path: Path) -> str:
+    import hashlib
+
+    digest = hashlib.sha256()
+    with open(path, "rb") as stream:
+        for block in iter(lambda: stream.read(1 << 20), b""):
+            digest.update(block)
+    return digest.hexdigest()
+
+
+def _walk_models(node, prefix: str, found: dict) -> None:
+    if isinstance(node, dict):
+        for key, value in node.items():
+            here = f"{prefix}.{key}" if prefix else key
+            if key == "model" and isinstance(value, str):
+                found.setdefault(here, value)
+            _walk_models(value, here, found)
+    elif isinstance(node, list):
+        for item in node:
+            _walk_models(item, f"{prefix}[]", found)
+
+
+def codex_record(stdout_text: str, sessions_dir: Path | None = None) -> str:
+    """Digest the rollout codex persisted for the thread this run started.
+
+    Needs `codex exec --json` for the thread id and no `--ephemeral`, which
+    would suppress the rollout this reads.
+    """
+    thread_id = None
+    for line in stdout_text.splitlines():
+        line = line.strip()
+        if not line.startswith("{"):
+            continue
+        try:
+            event = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        if isinstance(event, dict) and isinstance(event.get("thread_id"), str):
+            thread_id = event["thread_id"]
+            break
+    if not thread_id:
+        return ""
+    root = sessions_dir or Path.home() / ".codex" / "sessions"
+    matches = sorted(glob.glob(str(root / "**" / f"rollout-*{thread_id}.jsonl"), recursive=True))
+    if not matches:
+        return ""
+    path = Path(matches[0])
+    found: dict[str, str] = {}
+    for line in path.read_text(errors="replace").splitlines():
+        line = line.strip()
+        if not line.startswith("{"):
+            continue
+        try:
+            event = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        _walk_models(event, "", found)
+    lines = [f"# thread_id\t{thread_id}", f"# source\t{path}", f"# sha256\t{_sha256(path)}"]
+    lines += [f"{key}\t{value}" for key, value in sorted(found.items())]
+    return "\n".join(lines) + "\n"
+
+
+AGY_MODEL = re.compile(r"Resolving model (\S+)")
+AGY_LABEL = re.compile(r'selected model override to backend: label="([^"]+)"')
+
 
-    return "fail", f"unsupported-vendor: {vendor}", None
+def agy_record(app_root: Path, since: float) -> str:
+    """Digest the model agy resolved in the logs this run wrote under app_root.
+
+    `since` rejects logs from an earlier run, so a reused app root cannot lend
+    its model string to a later probe.
+    """
+    app_root = Path(app_root).expanduser()
+    if not app_root.is_dir():
+        return ""
+    lines: list[str] = []
+    values: dict[str, str] = {}
+    for path in sorted(app_root.rglob("cli*.log")):
+        if not path.is_file() or path.stat().st_mtime < since - 2:
+            continue
+        text = path.read_text(errors="replace")
+        models = AGY_MODEL.findall(text)
+        if not models:
+            continue
+        lines += [f"# source\t{path}", f"# sha256\t{_sha256(path)}"]
+        lines += [f"# label\t{label}" for label in dict.fromkeys(AGY_LABEL.findall(text))]
+        for model in models:
+            values.setdefault(f"{path.name}:resolved-model:{model}", model)
+    if not values:
+        return ""
+    lines += [f"{key}\t{value}" for key, value in sorted(values.items())]
+    return "\n".join(lines) + "\n"
 
 
 def probe(
@@ -109,6 +316,7 @@ def probe(
     *,
     vendor: str | None = None,
     expected_token: str | None = None,
+    app_root: Path | None = None,
 ) -> dict:
     if not expected_token:
         raise ValueError("expected_token is required before running a probe")
@@ -148,10 +356,24 @@ def probe(
         "stdout": str(out_path),
         "stderr": str(err_path),
     }
-    record["verdict"], record["verdict_reason"], record["observed_model"] = verdict(
-        record, out_path.read_text(errors="replace"), err_path.read_text(errors="replace"),
-        vendor, expected_token,
-    )
+    stdout_text = out_path.read_text(errors="replace")
+    extra = {}
+    if vendor == "codex":
+        extra["rollout"] = codex_record(stdout_text)
+    elif vendor == "agy" and app_root is not None:
+        extra["cli_log"] = agy_record(app_root, started)
+    for key, text in extra.items():
+        if not text:
+            continue
+        # Stored so the sweep hashes it and verdict() can re-judge it offline.
+        record_path = evidence / f"{name}.{key}"
+        record_path.write_text(text)
+        record[key] = str(record_path)
+    record["verdict"], record["verdict_reason"], record["observed_model"], \
+        record["evidence_tier"] = verdict(
+            record, stdout_text, err_path.read_text(errors="replace"),
+            vendor, expected_token, extra,
+        )
     (evidence / f"{name}.json").write_text(json.dumps(record, indent=2) + "\n")
     return record
 
@@ -174,11 +396,13 @@ if __name__ == "__main__":
     parser.add_argument("--expect", required=True, help="expected response token")
     parser.add_argument("--vendor", choices=("claude", "grok", "agy", "codex"),
                         help="defaults to the command executable's basename")
+    parser.add_argument("--app-root", type=Path,
+                        help="agy app data root for this run, to read back its cli log")
     parser.add_argument("name")
     parser.add_argument("command", nargs=argparse.REMAINDER)
     args = parser.parse_args()
     if not args.command:
         parser.error("command is required")
-    rec = probe(args.name, args.command, root=args.root,
+    rec = probe(args.name, args.command, root=args.root, app_root=args.app_root,
                 vendor=args.vendor, expected_token=args.expect)
     print(json.dumps(rec, ensure_ascii=False))

package/skills/omnilane/SKILL.md

@@ -19,10 +19,13 @@ You (the main loop) may be Claude, GPT, Grok, or Gemini. The procedure is identi
    Read-only work uses advise; edits require `--mode work --workdir <repo>`.
    `<repo>/scripts/dispatch.sh --caller-context FILE [--executor auto|native|cli] [--native-context FILE] [--vendor V] [--mode work] [--workdir DIR] <lane> "<task>"`
 
-   A model caller MUST pass `--caller-context`; without it every dispatch is
-   refused with `missing-caller-context` before a job exists. Build that file
-   before the first dispatch — see **Frozen exact-AA downward gate** for the
-   schema, a worked example, and what to do when your own effort is unverifiable.
+   A model caller needs a caller identity. Dispatch reads it from the CLI that
+   launched you when that CLI names its model and effort, so an ordinary session
+   passes nothing. When it cannot, dispatch is refused with
+   `missing-caller-context` before a job exists: run `omnilane whoami` (or
+   `<repo>/bin/omnilane whoami`) and pass the file it prints as
+   `--caller-context`. See **Frozen exact-AA downward gate** for the schema and
+   what to do when your effort is genuinely unverifiable.
 
    Add `--background` for long tasks; poll with `scripts/jobs.sh status|result <id>`.
    Use `--thread NAME` when later claude, codex, grok or gemini dispatches
@@ -355,15 +358,23 @@ and `snapshot_id` must equal the registry's own `snapshot.id`.
 Set `inherited_ceiling` to your own row's score when you are the root caller, or
 to the ceiling you were handed when you are a child.
 
-**When your harness names a model but no effort, look before you guess.** The
-launching process usually carries the exact flags. Walk your own ancestor chain
-(`ps -o ppid=,comm= -p <pid>` upward, then `ps -o args= -p <ancestor>`) and read
-its `--model` / `--effort`. Match the ancestor chain rather than the first
-matching process on the host — a second session of the same CLI is common and
-its flags are not yours. This is request-selector evidence, the same class the
-transport overlay carries, and it does not certify upstream identity.
-
-Only when that genuinely yields nothing: ask the operator, or declare the
+**Your identity is read from the CLI that launched you.** When no
+`--caller-context` is given, dispatch walks up the process tree to the nearest
+vendor CLI and reads the selector it was started with: `--model` / `--effort`
+for claude, `-m` and `-c model_reasoning_effort` for codex, `--reasoning-effort`
+for grok, the effort-encoded model id for agy. `omnilane whoami` runs the same
+walk and prints the resulting caller-context file, for a retry or to see what
+you will be held to. Nearest wins, so a codex worker started by a Claude session
+is a codex caller, and another session of the same CLI elsewhere on the host is
+never consulted. It does not guess: a missing flag, a model alias, or a Claude
+effort whose only scored row is non-reasoning is refused with the reason. This
+is request-selector evidence, the same class the transport overlay carries; it
+does not certify upstream identity, but unlike a hand-written file the model
+cannot edit it. An explicit `--caller-context` or `--operator-asserted-human`
+still wins, and `OMNILANE_AA_CALLER_FROM_PROCESS=0` restores the file-only
+contract.
+
+Only when `omnilane whoami` refuses: ask the operator, or declare the
 lowest-scoring row of your model and say so in your report. Understating only
 narrows what you may dispatch to, so it fails in the safe direction — but it is
 the fallback, not the first move, and an unnecessarily low ceiling silently
@@ -372,7 +383,9 @@ declared effort to unblock a refused target, and never assert
 `--operator-asserted-human` on your own behalf.
 
 Three refusal codes mean different things and need different fixes.
-`missing-caller-context` means you passed no file — write one.
+`missing-caller-context` means no identity reached the gate: you passed no file
+and dispatch could not read one from your launching CLI. Run `omnilane whoami`;
+its refusal says exactly why, and that reason is what to fix or report.
 `runtime-mapping-unverified` means the file is fine but the *target* has no proven
 host-local request selector; that is fixed by a `--transport-overlay` entry backed
 by real evidence, never by editing the frozen registry (its sha256 is pinned in
@@ -388,9 +401,24 @@ selector evidence. Evidence entries carry a `vendor` tag: a tagged entry that
 drifts marks only its own vendor stale, and the other three keep dispatching.
 Untagged evidence — `probe-manifest.json`, and any overlay built before the tags
 existed — still fails the whole gate closed, which is what an unpatched host
-looks like. Codex and Claude evidence paths embed version directories
-(`releases/0.153.4-…`, `versions/2.1.263`), so their upgrades remove the file
-rather than change its digest; both are treated as staleness, not corruption.
+looks like. Codex and Claude resolve through version directories
+(`releases/0.153.4-…`, `versions/2.1.266`), so their upgrades remove the anchored
+file rather than change its digest; both are treated as staleness, not corruption.
+
+Do not expect these upgrades to be operator actions. agy and grok update
+themselves in the background when invoked — agy's own `cli.log` records
+`auto_updater.go: Spawned background update process`, and both binaries changed
+under a probing session on 2026-09-10, minutes after their first call. Overlay
+drift is therefore a routine consequence of using a vendor, not an occasional
+maintenance event, which is why per-vendor degradation matters more than it
+looks. It also means any test asserting a fixed number of verified live
+mappings will go red on its own schedule.
+
+Because of that, `build_overlay.py` anchors the executable `shutil.which` resolves
+rather than a version written into the script. A pinned path drifts out of use
+silently: before 0.42.6 the overlay hashed claude `2.1.263` while every dispatch
+ran `2.1.266`, so eleven mappings were "verified" against a binary that had not
+run for a day.
 
 Re-signing is a probe, a rebuild, and an install, in that order. Back up
 `~/.omnilane/transport-contracts.local.json` first; restoring it is the rollback.
@@ -408,6 +436,30 @@ Keep the sweep where its default `--root` puts it,
 inside a repository is one `git clean -fdx` away from taking every vendor down
 at once — the same global refusal a re-signing session is usually trying to end.
 
+Every mapping carries an `evidence_tier` saying how strongly its probe pinned the
+responder. `billed-model` means the provider named the model it charged for —
+Claude's `modelUsage`, grok's under `--output-format json`. `client-echo` means
+the CLI wrote down the model it asked for — codex's session rollout, agy's
+`cli.log` resolver line. `selector-only` means the CLI accepted the selector and
+said nothing more. Put plainly: `client-echo` is the CLI's copy of your order,
+`billed-model` is the provider's receipt. Neither certifies upstream identity,
+but only one of them was written by the party that answered.
+
+The tier is reported, never enforced. Dispatch still turns on `runtime_verified`
+alone, so a mapping that drops to `selector-only` keeps working and simply shows
+up in doctor as worth re-probing. Do not add a tier check to the gate: that would
+rebuild the failure 0.42.5 removed, where evidence quality could refuse a lane
+that runs. The tier is read off the evidence a run produced rather than assigned
+per vendor, so a sweep predating 0.42.6 re-judges as `selector-only` and a CLI
+that starts reporting a billed model is promoted with no code change.
+
+Two probe details follow from this. Codex needs `exec --json` (the thread id that
+locates the rollout) and must *not* use `--ephemeral`, which suppresses the very
+rollout the tier reads. agy needs its own app data directory, prepared exactly
+the way `run-gemini.sh` does it — `prepare-agy-mode.py --mode advise` returns a
+path relative to `~/.gemini` that is passed as `--app_data_dir=`; the environment
+variables that look like they would do this are ignored.
+
 Never sign a probe you did not read. `probe.py` records a `verdict` because exit
 status alone is not evidence: the Claude CLI answers a quota refusal with a JSON
 body carrying `is_error`, and it accepts an unknown `--effort` by silently using