omnilane 0.12.0 → 0.14.0

package/completions/_omnilane

@@ -53,7 +53,7 @@ _omnilane() {
   local command="${words[2]:-}" sub sub_index=3
   local -a lanes ids
   if (( CURRENT == 2 )); then
-    _values 'command' version list route dispatch jobs doctor release-audit ui configure completion help
+    _values 'command' version list route dispatch jobs doctor benchmark release-audit ui configure completion help
     return
   fi
   case "$command" in
@@ -78,7 +78,7 @@ _omnilane() {
         sub="${words[4]:-}"
       fi
       if (( CURRENT == sub_index )); then
-        _values 'job command' list status result tail retry stats wait audit prune help
+        _values 'job command' list status result tail retry stats recommend wait audit prune help
       elif (( CURRENT == sub_index + 1 )) &&
            [[ "$sub" == status || "$sub" == result || "$sub" == wait ||
               "$sub" == tail || "$sub" == retry ]]; then
@@ -92,7 +92,13 @@ _omnilane() {
         _arguments '--background[re-dispatch in the background]'
       elif [[ "$sub" == wait ]]; then
         _arguments '--timeout[maximum wait seconds]:seconds:'
-      elif [[ "$sub" == stats || "$sub" == audit ]]; then
+      elif [[ "$sub" == stats ]]; then
+        _arguments '--last[maximum jobs]:count:' '--lane[filter by lane]:lane:' \
+          '--vendor[filter by vendor]:vendor:' '--json[versioned metadata output]'
+      elif [[ "$sub" == recommend ]]; then
+        _arguments '--last[maximum jobs]:count:' '--lane[filter by lane]:lane:' \
+          '--min-samples[minimum completed samples]:count:' '--json[versioned metadata output]'
+      elif [[ "$sub" == audit ]]; then
         _arguments '--last[maximum jobs]:count:' '--json[versioned metadata output]'
       elif [[ "$sub" == prune ]]; then
         _arguments '--keep[completed jobs to retain]:count:' \
@@ -100,7 +106,18 @@ _omnilane() {
           '--apply[delete eligible jobs]'
       fi
       ;;
-    doctor) _values 'option' --json ;;
+    doctor)
+      _arguments '--json[versioned JSON output]' '--strict[fail on warnings]' \
+        '--probe[run one opt-in live provider probe]:vendor:' \
+        '--probe-timeout[probe timeout in seconds]:seconds:'
+      ;;
+    benchmark)
+      _arguments '--json[versioned JSON output]' '--run[invoke providers in advise mode]' \
+        '*--vendor[vendor to compare]:vendor:(codex claude grok gemini kimi qwen opencode openrouter deepseek zai mistral groq cerebras)' \
+        '--timeout[per-workload timeout]:seconds:' \
+        '--workloads[TSV workload file]:file:_files' \
+        '*--cost-per-call[user-supplied vendor USD estimate]:vendor=USD:'
+      ;;
     release-audit)
       _arguments '--target[release version]:version:' '--allow-dirty[permit a dirty worktree]' \
         '--require-tag[require an annotated tag at HEAD]' '--manifest[include tracked inventory]' \

package/completions/omnilane.bash

@@ -58,7 +58,7 @@ _omnilane() {
   prev="${COMP_WORDS[COMP_CWORD-1]:-}"
   command="${COMP_WORDS[1]:-}"
   if [[ "$COMP_CWORD" -eq 1 ]]; then
-    words="version list route dispatch jobs doctor release-audit ui configure completion help"
+      words="version list route dispatch jobs doctor benchmark release-audit ui configure completion help"
   else
     case "$command" in
       route|dispatch)
@@ -84,7 +84,7 @@ _omnilane() {
           sub="${COMP_WORDS[3]:-}"
         fi
         if [[ "$COMP_CWORD" -eq "$sub_index" ]]; then
-          words="list status result tail retry stats wait audit prune help"
+          words="list status result tail retry stats recommend wait audit prune help"
         elif [[ "$COMP_CWORD" -eq $((sub_index + 1)) &&
                 ( "$sub" == status || "$sub" == result || "$sub" == wait ||
                   "$sub" == tail || "$sub" == retry ) ]]; then
@@ -99,7 +99,11 @@ _omnilane() {
           words="--background"
         elif [[ "$sub" == wait ]]; then
           words="--timeout"
-        elif [[ "$sub" == stats || "$sub" == audit ]]; then
+        elif [[ "$sub" == stats ]]; then
+          words="--last --lane --vendor --json"
+        elif [[ "$sub" == recommend ]]; then
+          words="--last --lane --min-samples --json"
+        elif [[ "$sub" == audit ]]; then
           words="--last --json"
         elif [[ "$sub" == prune ]]; then
           words="--keep --older-than --apply"
@@ -107,7 +111,18 @@ _omnilane() {
           return
         fi
         ;;
-      doctor) words="--json" ;;
+      doctor) words="--json --strict --probe --probe-timeout" ;;
+      benchmark)
+        case "$prev" in
+          --vendor) words="codex claude grok gemini kimi qwen opencode openrouter deepseek zai mistral groq cerebras" ;;
+          --workloads)
+            COMPREPLY=( $(compgen -f -- "$cur") )
+            return
+            ;;
+          --timeout|--cost-per-call) return ;;
+          *) words="--json --run --vendor --timeout --workloads --cost-per-call" ;;
+        esac
+        ;;
       release-audit) words="--target --allow-dirty --require-tag --manifest --json" ;;
       ui) words="start status url stop" ;;
       completion) words="bash zsh" ;;

package/completions/omnilane.fish

@@ -10,6 +10,7 @@ complete -c omnilane -f -n __fish_use_subcommand -a dispatch      -d 'dispatch o
 complete -c omnilane -f -n __fish_use_subcommand -a jobs          -d 'inspect background jobs'
 complete -c omnilane -f -n __fish_use_subcommand -a mcp           -d 'MCP stdio server'
 complete -c omnilane -f -n __fish_use_subcommand -a doctor        -d 'read-only health report'
+complete -c omnilane -f -n __fish_use_subcommand -a benchmark     -d 'fixed quality/cost comparison'
 complete -c omnilane -f -n __fish_use_subcommand -a release-audit -d 'offline release gate'
 complete -c omnilane -f -n __fish_use_subcommand -a ui            -d 'Live Board server'
 complete -c omnilane -f -n __fish_use_subcommand -a configure     -d 'lane routing (menu or set/get/unset/list/diff)'
@@ -29,19 +30,30 @@ complete -c omnilane -n '__fish_seen_subcommand_from route dispatch' -l dry-run
 
 # jobs subcommands (only before a jobs subcommand is chosen).
 complete -c omnilane -f \
-  -n '__fish_seen_subcommand_from jobs; and not __fish_seen_subcommand_from list status result tail retry stats wait audit prune cancel rm help' \
-  -a 'list status result tail retry stats wait audit prune cancel rm help' -d 'jobs subcommand'
+  -n '__fish_seen_subcommand_from jobs; and not __fish_seen_subcommand_from list status result tail retry stats recommend wait audit prune cancel rm help' \
+  -a 'list status result tail retry stats recommend wait audit prune cancel rm help' -d 'jobs subcommand'
 complete -c omnilane -n '__fish_seen_subcommand_from jobs' -l json -d 'JSON output'
 # jobs list / stats filters.
-complete -c omnilane -n '__fish_seen_subcommand_from jobs; and __fish_seen_subcommand_from list stats' -l lane   -x                                                                                    -d 'filter by lane'
+complete -c omnilane -n '__fish_seen_subcommand_from jobs; and __fish_seen_subcommand_from list stats recommend' -l lane -x -d 'filter by lane'
 complete -c omnilane -n '__fish_seen_subcommand_from jobs; and __fish_seen_subcommand_from list stats' -l vendor -x -a 'codex claude grok gemini kimi qwen opencode openrouter deepseek zai mistral groq cerebras exec' -d 'filter by vendor'
 complete -c omnilane -n '__fish_seen_subcommand_from jobs; and __fish_seen_subcommand_from list'       -l status -x -a 'running done'                                                                    -d 'filter by status'
+complete -c omnilane -n '__fish_seen_subcommand_from jobs; and __fish_seen_subcommand_from stats recommend audit' -l last -x -d 'maximum recent jobs'
+complete -c omnilane -n '__fish_seen_subcommand_from jobs; and __fish_seen_subcommand_from recommend' -l min-samples -x -d 'minimum completed samples'
 
 # configure subcommands (non-interactive).
 complete -c omnilane -f -n '__fish_seen_subcommand_from configure; and not __fish_seen_subcommand_from set get unset list diff' -a 'set get unset list diff' -d 'configure action'
 
 # doctor / release-audit / ui / completion.
 complete -c omnilane -n '__fish_seen_subcommand_from doctor' -l json -d 'JSON output'
+complete -c omnilane -n '__fish_seen_subcommand_from doctor' -l strict -d 'fail when any warning is reported'
+complete -c omnilane -n '__fish_seen_subcommand_from doctor' -l probe -x -a 'codex claude grok gemini kimi qwen opencode openrouter deepseek zai mistral groq cerebras' -d 'run one opt-in live probe'
+complete -c omnilane -n '__fish_seen_subcommand_from doctor' -l probe-timeout -x -d 'probe timeout in seconds'
+complete -c omnilane -n '__fish_seen_subcommand_from benchmark' -l json -d 'JSON output'
+complete -c omnilane -n '__fish_seen_subcommand_from benchmark' -l run -d 'invoke providers in advise mode'
+complete -c omnilane -n '__fish_seen_subcommand_from benchmark' -l vendor -x -a 'codex claude grok gemini kimi qwen opencode openrouter deepseek zai mistral groq cerebras' -d 'vendor to compare; repeatable'
+complete -c omnilane -n '__fish_seen_subcommand_from benchmark' -l timeout -x -d 'per-workload timeout'
+complete -c omnilane -n '__fish_seen_subcommand_from benchmark' -l workloads -r -d 'TSV workload file'
+complete -c omnilane -n '__fish_seen_subcommand_from benchmark' -l cost-per-call -x -d 'user-supplied VENDOR=USD estimate'
 complete -c omnilane -n '__fish_seen_subcommand_from release-audit' -l target -x       -d 'target version'
 complete -c omnilane -n '__fish_seen_subcommand_from release-audit' -l allow-dirty     -d 'permit a dirty tree'
 complete -c omnilane -n '__fish_seen_subcommand_from release-audit' -l require-tag     -d 'require an annotated tag'

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "omnilane",
-  "version": "0.12.0",
+  "version": "0.14.0",
   "description": "One routing table, every harness — classify subtasks into lanes and dispatch each lane to the best vendor's agentic CLI (Codex, Claude, Gemini, Grok) using your existing subscription logins.",
   "bin": {
     "omnilane": "bin/omnilane"
@@ -11,6 +11,7 @@
     "!scripts/__pycache__",
     "!**/*.pyc",
     "completions/",
+    "benchmarks/",
     "ui/",
     "routing.yaml",
     "routing.local.yaml.example",

package/routing.yaml

@@ -27,7 +27,7 @@ hard-judgment:   claude claude-opus-5 xhigh | codex gpt-5.6-sol max   # ordered
 taste-final:     claude claude-opus-5 high | codex gpt-5.6-sol max    # user-facing prose, prompt/doc polish, Chinese phrasing, style arbitration
 consult:         codex gpt-5.6-sol max | claude claude-opus-5 high | grok grok-4.5 - | gemini "Gemini 3.1 Pro (High)" -  # direct named-model consultation; use --vendor to prevent fallback
 ui-draft:        codex gpt-5.6-sol xhigh | claude claude-opus-5 high  # only with a design system / reference images; open-ended visual taste -> taste-final
-long-context:    gemini "Gemini 3.1 Pro (High)" - | claude claude-opus-5 high | codex gpt-5.6-sol high  # all have 1M context; Gemini leads single-needle retrieval at full length and is the cheapest way to sweep volume, Flash for fast loops. For multi-hop synthesis across a large corpus prefer the claude candidate: published multi-needle scores at 1M favour Claude by a wide margin (see docs/model-capabilities-2026-07.md)
+long-context:    gemini "Gemini 3.1 Pro (High)" - | codex gpt-5.6-sol high | claude claude-opus-5 high  # all have 1M context; ordered on AA-LCR, which scores exactly this lane's work — extracting and synthesising across long documents — and where Gemini leads both fallbacks. Corrected 2026-08-03: this comment used to send multi-hop synthesis to the Claude candidate on second-hand prior-generation figures, and current first-party per-effort data reverses that, so the two fallbacks swapped. Caveat in docs: AA-LCR runs at 10k-100k tokens, so nothing here settles behaviour at a full 1M
 fast-agentic:    codex gpt-5.6-luna max | gemini "Gemini 3.6 Flash (High)" -  # fast multi-step tool loops. Reordered 2026-08-03: Luna leads Flash on agentic benchmarks AND costs a fraction as much per task, so Flash's remaining edge is raw throughput alone. Keep Flash first only if your loops are latency-bound. Both take image input, so the lane's multimodal checks are unaffected
 live-search:     grok grok-4.5 - | off          # native X/web search lane; no real substitute
 coding-overflow: grok grok-4.5 - | kimi kimi-k3 - | qwen qwen3-coder-plus - | opencode - - | off  # codex-quota relief valve: mid-tier coding; Grok 4.5 is a capable mid-tier coder but AA measures a high hallucination rate — verify every factual claim it ships. qwen3-coder-plus = 2025-09-23 snapshot alias (Qwen 3.6 Plus exists; re-evaluate before swapping). kimi/qwen model fields are CLI aliases — adjust to your login. opencode "-" model = its own configured default.

package/scripts/benchmark.py

@@ -0,0 +1,446 @@
+#!/usr/bin/env python3
+"""Fixed, body-free routing quality/cost benchmark."""
+
+import argparse
+from decimal import Decimal, InvalidOperation
+import json
+import os
+from pathlib import Path
+import re
+import shlex
+import subprocess
+import sys
+import tempfile
+import time
+
+
+ROOT = Path(__file__).resolve().parents[1]
+DEFAULT_WORKLOADS = ROOT / "benchmarks" / "workloads.tsv"
+NAME_RE = re.compile(r"^[a-z][a-z0-9-]*$")
+ID_RE = re.compile(r"^[a-z][a-z0-9_-]*$")
+
+
+class BenchmarkError(Exception):
+    pass
+
+
+def parse_args(argv=None):
+    parser = argparse.ArgumentParser(
+        prog="omnilane benchmark",
+        description=(
+            "Compare fixed-workload routing quality and user-supplied per-call "
+            "costs. Provider calls require --run."
+        ),
+    )
+    parser.add_argument("--json", action="store_true", help="emit JSON")
+    parser.add_argument(
+        "--run",
+        action="store_true",
+        help="invoke providers in advise mode; default is routing-only dry-run",
+    )
+    parser.add_argument(
+        "--vendor",
+        action="append",
+        default=[],
+        metavar="VENDOR",
+        help="vendor to compare; repeatable (default: configured vendors)",
+    )
+    parser.add_argument(
+        "--timeout",
+        type=int,
+        default=60,
+        metavar="SECONDS",
+        help="per-workload timeout, 1..600 (default: 60)",
+    )
+    parser.add_argument(
+        "--workloads",
+        type=Path,
+        default=DEFAULT_WORKLOADS,
+        metavar="FILE",
+        help="fixed TSV workload file",
+    )
+    parser.add_argument(
+        "--cost-per-call",
+        action="append",
+        default=[],
+        metavar="VENDOR=USD",
+        help="optional user-supplied estimated USD per call; repeatable",
+    )
+    args = parser.parse_args(argv)
+    if not 1 <= args.timeout <= 600:
+        parser.error("--timeout must be between 1 and 600 seconds")
+    return args
+
+
+def unique(values):
+    return list(dict.fromkeys(values))
+
+
+def decode_plan_value(value):
+    try:
+        parsed = shlex.split(value, posix=True)
+    except ValueError:
+        return value
+    return parsed[0] if len(parsed) == 1 else value
+
+
+def parse_plan(text):
+    plan = {}
+    for line in text.splitlines():
+        if "=" not in line:
+            continue
+        key, value = line.split("=", 1)
+        if re.fullmatch(r"[a-z_]+", key):
+            plan[key] = decode_plan_value(value)
+    return plan
+
+
+def load_workloads(path):
+    try:
+        lines = path.read_text(encoding="utf-8").splitlines()
+    except OSError as exc:
+        raise BenchmarkError(f"cannot read workloads: {path}: {exc}") from exc
+
+    workloads = []
+    seen = set()
+    for line_number, line in enumerate(lines, 1):
+        if not line.strip() or line.lstrip().startswith("#"):
+            continue
+        fields = line.split("\t", 4)
+        if len(fields) != 5:
+            raise BenchmarkError(
+                f"{path}:{line_number}: expected 5 tab-separated fields"
+            )
+        workload_id, lane, weight_text, pattern, prompt = fields
+        if not ID_RE.fullmatch(workload_id) or workload_id in seen:
+            raise BenchmarkError(f"{path}:{line_number}: invalid or duplicate id")
+        if not NAME_RE.fullmatch(lane):
+            raise BenchmarkError(f"{path}:{line_number}: invalid lane")
+        try:
+            weight = int(weight_text)
+        except ValueError as exc:
+            raise BenchmarkError(f"{path}:{line_number}: weight must be an integer") from exc
+        if not 1 <= weight <= 100:
+            raise BenchmarkError(f"{path}:{line_number}: weight must be 1..100")
+        try:
+            re.compile(pattern)
+        except re.error as exc:
+            raise BenchmarkError(f"{path}:{line_number}: invalid regex: {exc}") from exc
+        if not prompt.strip():
+            raise BenchmarkError(f"{path}:{line_number}: prompt must not be empty")
+        seen.add(workload_id)
+        workloads.append(
+            {
+                "id": workload_id,
+                "lane": lane,
+                "weight": weight,
+                "pattern": pattern,
+                "prompt": prompt,
+            }
+        )
+    if not workloads:
+        raise BenchmarkError(f"no workloads found: {path}")
+    return workloads
+
+
+def parse_costs(items):
+    costs = {}
+    for item in items:
+        if "=" not in item:
+            raise BenchmarkError("--cost-per-call must use VENDOR=USD")
+        vendor, raw = item.split("=", 1)
+        if not NAME_RE.fullmatch(vendor):
+            raise BenchmarkError(f"invalid cost vendor: {vendor}")
+        try:
+            value = Decimal(raw)
+        except InvalidOperation as exc:
+            raise BenchmarkError(f"invalid cost for {vendor}: {raw}") from exc
+        if not value.is_finite() or value < 0:
+            raise BenchmarkError(f"cost for {vendor} must be non-negative")
+        costs[vendor] = value
+    return costs
+
+
+def list_routes(dispatch, env):
+    result = subprocess.run(
+        [str(dispatch), "--list"],
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        env=env,
+        timeout=15,
+        check=False,
+    )
+    if result.returncode != 0:
+        detail = result.stderr.strip().splitlines()[-1:] or ["unknown error"]
+        raise BenchmarkError(f"cannot list routing: {detail[0]}")
+    lanes = []
+    vendors = []
+    for line in result.stdout.splitlines():
+        if ":" not in line:
+            continue
+        lane, raw = line.split(":", 1)
+        lane = lane.strip()
+        if not NAME_RE.fullmatch(lane):
+            continue
+        lanes.append(lane)
+        try:
+            fields = shlex.split(raw.strip())
+        except ValueError:
+            fields = raw.split()
+        if fields and NAME_RE.fullmatch(fields[0]) and fields[0] not in {
+            "off",
+            "exec",
+            "vote",
+        }:
+            vendors.append(fields[0])
+    return unique(lanes), unique(vendors)
+
+
+def resolve_route(dispatch, vendor, preferred_lane, lanes, prompt, env):
+    last_error = "vendor is not configured in an available lane"
+    for lane in unique([preferred_lane] + list(lanes)):
+        result = subprocess.run(
+            [
+                str(dispatch),
+                "--dry-run",
+                "--mode",
+                "advise",
+                "--vendor",
+                vendor,
+                lane,
+                prompt,
+            ],
+            text=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            env=env,
+            timeout=15,
+            check=False,
+        )
+        if result.returncode == 0:
+            plan = parse_plan(result.stdout)
+            if plan.get("provider_invoked") != "no":
+                return None, "dry-run contract did not confirm provider_invoked=no"
+            if plan.get("vendor") != vendor:
+                return None, "dry-run resolved a different vendor"
+            plan.setdefault("lane", lane)
+            return plan, ""
+        detail = result.stderr.strip().splitlines()
+        if detail:
+            last_error = detail[-1]
+    return None, last_error
+
+
+def money(value):
+    return format(value.quantize(Decimal("0.01")), "f")
+
+
+def cost_report(vendor, calls, costs):
+    if vendor not in costs:
+        return None
+    per_call = costs[vendor]
+    return {
+        "basis": "user-supplied-per-call",
+        "currency": "USD",
+        "per_call_usd": money(per_call),
+        "estimated_total_usd": money(per_call * calls),
+    }
+
+
+def run_one(repo, vendor, plan, workload, timeout, env):
+    runner = repo / "scripts" / "runners" / f"run-{vendor}.sh"
+    base = {
+        "id": workload["id"],
+        "requested_lane": workload["lane"],
+        "resolved_lane": plan.get("lane", workload["lane"]),
+        "model": plan.get("model", "-"),
+        "effort": plan.get("effort", "-"),
+        "weight": workload["weight"],
+    }
+    if not runner.is_file() or not os.access(runner, os.X_OK):
+        return dict(base, status="runner_error", duration_seconds=0.0, response_bytes=0), True
+
+    started = time.monotonic()
+    with tempfile.TemporaryDirectory(prefix="omnilane-benchmark-") as temporary:
+        temporary_path = Path(temporary)
+        prompt_file = temporary_path / "prompt.txt"
+        output_file = temporary_path / "response.txt"
+        prompt_file.write_text(str(workload["prompt"]) + "\n", encoding="utf-8")
+        run_env = env.copy()
+        run_env["OMNILANE_TIMEOUT"] = str(timeout)
+        try:
+            result = subprocess.run(
+                [
+                    str(runner),
+                    "advise",
+                    str(repo),
+                    plan.get("model", "-"),
+                    plan.get("effort", "-"),
+                    str(prompt_file),
+                    str(output_file),
+                ],
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+                env=run_env,
+                timeout=timeout + 5,
+                check=False,
+            )
+        except subprocess.TimeoutExpired:
+            elapsed = round(time.monotonic() - started, 3)
+            return dict(base, status="runner_error", duration_seconds=elapsed, response_bytes=0), True
+        elapsed = round(time.monotonic() - started, 3)
+        try:
+            body = output_file.read_text(encoding="utf-8")
+            response_bytes = output_file.stat().st_size
+        except OSError:
+            body = ""
+            response_bytes = 0
+        if result.returncode != 0:
+            status = "runner_error"
+            runner_error = True
+        else:
+            status = "passed" if re.search(str(workload["pattern"]), body) else "failed"
+            runner_error = False
+        return dict(
+            base,
+            status=status,
+            duration_seconds=elapsed,
+            response_bytes=response_bytes,
+        ), runner_error
+
+
+def execute(args):
+    repo = Path(os.environ.get("OMNILANE_BENCHMARK_REPO", str(ROOT))).resolve()
+    dispatch = repo / "scripts" / "dispatch.sh"
+    if not dispatch.is_file() or not os.access(dispatch, os.X_OK):
+        raise BenchmarkError(f"dispatch is not executable: {dispatch}")
+    env = os.environ.copy()
+    workloads = load_workloads(args.workloads)
+    costs = parse_costs(args.cost_per_call)
+    lanes, configured_vendors = list_routes(dispatch, env)
+    vendors = unique(args.vendor or configured_vendors)
+    if not vendors:
+        raise BenchmarkError("no configured benchmark vendors found")
+    for vendor in vendors:
+        if not NAME_RE.fullmatch(vendor) or vendor in {"off", "exec", "vote"}:
+            raise BenchmarkError(f"unsupported benchmark vendor: {vendor}")
+
+    report = {
+        "schema_version": 1,
+        "command": "benchmark",
+        "mode": "run" if args.run else "dry-run",
+        "provider_invoked": False,
+        "workload_count": len(workloads),
+        "workloads_file": str(args.workloads.resolve()),
+        "vendors": [],
+    }
+    had_error = False
+    provider_invoked = False
+    for vendor in vendors:
+        vendor_started = time.monotonic()
+        items = []
+        passed = 0
+        score_possible = 0
+        score_earned = 0
+        for workload in workloads:
+            score_possible += int(workload["weight"])
+            plan, error = resolve_route(
+                dispatch,
+                vendor,
+                workload["lane"],
+                lanes,
+                workload["prompt"],
+                env,
+            )
+            if plan is None:
+                items.append(
+                    {
+                        "id": workload["id"],
+                        "requested_lane": workload["lane"],
+                        "weight": workload["weight"],
+                        "status": "unavailable",
+                        "duration_seconds": 0.0,
+                        "response_bytes": 0,
+                        "error": error,
+                    }
+                )
+                had_error = True
+                continue
+            if not args.run:
+                items.append(
+                    {
+                        "id": workload["id"],
+                        "requested_lane": workload["lane"],
+                        "resolved_lane": plan.get("lane", workload["lane"]),
+                        "model": plan.get("model", "-"),
+                        "effort": plan.get("effort", "-"),
+                        "weight": workload["weight"],
+                        "status": "planned",
+                        "duration_seconds": 0.0,
+                        "response_bytes": 0,
+                    }
+                )
+                continue
+            provider_invoked = True
+            item, runner_error = run_one(
+                repo, vendor, plan, workload, args.timeout, env
+            )
+            items.append(item)
+            had_error = had_error or runner_error
+            if item["status"] == "passed":
+                passed += 1
+                score_earned += int(workload["weight"])
+
+        quality = None
+        if args.run and score_possible:
+            quality = round((score_earned / score_possible) * 100, 2)
+            if quality.is_integer():
+                quality = int(quality)
+        report["vendors"].append(
+            {
+                "vendor": vendor,
+                "planned_calls": len(workloads),
+                "passed": passed,
+                "score_earned": score_earned,
+                "score_possible": score_possible,
+                "quality_percent": quality,
+                "duration_seconds": round(time.monotonic() - vendor_started, 3),
+                "cost": cost_report(vendor, len(workloads), costs),
+                "workloads": items,
+            }
+        )
+    report["provider_invoked"] = provider_invoked
+    return report, 1 if had_error else 0
+
+
+def print_human(report):
+    invoked = "yes" if report["provider_invoked"] else "no"
+    print(f"mode={report['mode']} provider_invoked={invoked}")
+    for vendor in report["vendors"]:
+        quality = vendor["quality_percent"]
+        quality_text = "planned" if quality is None else f"{quality}%"
+        cost = vendor["cost"]
+        cost_text = "not supplied" if cost is None else f"USD {cost['estimated_total_usd']}"
+        print(
+            f"{vendor['vendor']}: calls={vendor['planned_calls']} "
+            f"quality={quality_text} estimated_cost={cost_text}"
+        )
+
+
+def main(argv=None):
+    args = parse_args(argv)
+    try:
+        report, return_code = execute(args)
+    except BenchmarkError as exc:
+        print(f"omnilane benchmark: {exc}", file=sys.stderr)
+        return 2
+    if args.json:
+        print(json.dumps(report, ensure_ascii=False, separators=(",", ":")))
+    else:
+        print_human(report)
+    return return_code
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/scripts/check.sh

@@ -75,7 +75,7 @@ fi
 
 # 4) python compile of the UI and test modules
 py_files=()
-for rel in scripts/ui.py tests/test_ui.py tests/test_ci_policy.py tests/ui_browser_harness.py; do
+for rel in scripts/ui.py scripts/benchmark.py tests/test_ui.py tests/test_benchmark.py tests/test_ci_policy.py tests/ui_browser_harness.py; do
   [[ -f "$REPO/$rel" ]] && py_files+=("$REPO/$rel")
 done
 if ! command -v python3 >/dev/null 2>&1; then