@rulemetric/cli 0.13.0 → 0.14.1

package/dist/harbor/run.sh

@@ -0,0 +1,268 @@
+#!/usr/bin/env bash
+# RuleMetric Harbor runner.
+#
+# Default = OUR OWN HARNESS: the host agent (rm_host_claude.py → the user's own
+# `claude` CLI, subscription auth) against local Docker. Two opt-in ways to
+# move cost off this machine, both keyed from ~/.config/rulemetric/harbor.env
+# (names in harbor.env.example; the file is never in the repo):
+#
+#   --cloud <provider>   run the TASK CONTAINERS on a Harbor cloud environment
+#                        (daytona|modal|e2b|runloop|…) instead of local Docker.
+#                        The agent still runs here; compute is offloaded.
+#   --byok               bill the LLM to ANTHROPIC_API_KEY from harbor.env
+#                        instead of the subscription login. Works with the
+#                        host agent (key inherited by the child `claude`) and
+#                        is REQUIRED for --remote-agent.
+#   --remote-agent       use Harbor's built-in `claude-code` agent INSIDE the
+#                        sandbox (nothing runs on this host per trial). Needs
+#                        --byok. Delivery is limited to --prompt-file and
+#                        --skill; --claude-md needs the host agent.
+#   --dry-run            print the harbor command(s) with secrets redacted and
+#                        exit without running anything.
+#
+# Usage:
+#   ./run.sh run <task-dir> [options] [harbor flags...]            one arm
+#   ./run.sh ab  <task-dir> (--instructions <file> | --prompt-file <file>) [options] [harbor flags...]
+#                                                                  baseline + with-arm, prints the delta
+#
+#   --engine claude|codex  which HARNESS the arms run on (default claude). The
+#                          instruction file follows the engine: CLAUDE.md for
+#                          claude, AGENTS.md for codex. --claude-md is kept as an
+#                          alias of --instructions.
+#
+# Common harbor flags (passed through): -m <model>, --n-attempts N, -n <concurrency>,
+# --ak container_shell=true (local Docker only), --ak skills_dirs=..., --skill <dir>.
+set -euo pipefail
+
+HERE="$(cd "$(dirname "$0")" && pwd)"
+JOBS_DIR="${HARBOR_JOBS_DIR:-/tmp/harbor-jobs}"   # NEVER inside the repo: config.json records agent env
+ENV_FILE="${HARBOR_ENV_FILE:-$HOME/.config/rulemetric/harbor.env}"
+CA="$HOME/.mitmproxy/mitmproxy-ca-cert.pem"
+[ -f "$CA" ] && export SSL_CERT_FILE="$CA"
+export PYTHONPATH="$HERE${PYTHONPATH:+:$PYTHONPATH}"
+
+MODE="${1:?usage: run.sh run|ab <task-dir> [options] [flags]}"; shift
+TASK="${1:?task dir required}"; shift
+[ -d "$TASK" ] || TASK="$HERE/$TASK"
+
+# ── option parsing (ours) — everything else passes through to harbor ─────────
+CLOUD=""; BYOK=0; REMOTE=0; DRY=0; CLAUDE_MD=""; PROMPT_FILE=""; ENGINE="claude"
+ARGS=()
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --cloud)        CLOUD="${2:?--cloud needs a provider}"; shift 2 ;;
+    --byok)         BYOK=1; shift ;;
+    --remote-agent) REMOTE=1; shift ;;
+    --dry-run)      DRY=1; shift ;;
+    --engine)       ENGINE="${2:?--engine needs claude|codex}"; shift 2 ;;
+    # --instructions is the engine-neutral spelling; --claude-md is kept so
+    # every existing invocation and doc keeps working unchanged.
+    --claude-md|--instructions) CLAUDE_MD="${2:?needs a file}"; shift 2 ;;
+    --prompt-file)  PROMPT_FILE="${2:?--prompt-file needs a file}"; shift 2 ;;
+    *)              ARGS+=("$1"); shift ;;
+  esac
+done
+
+die() { echo "run.sh: $*" >&2; exit 1; }
+
+# ── credentials: only ever from the env file (or an already-exported var) ────
+load_env_file() {
+  [ -f "$ENV_FILE" ] || return 0
+  local perms
+  # GNU `stat -c` FIRST, BSD `stat -f` second. The reverse order was silently
+  # broken on Linux: there `-f` is a VALID flag meaning "filesystem status", so
+  # it exits 0 and the `||` fallback never runs — $perms captured a whole
+  # filesystem dump ("File: … Type: ext2/ext3 … Blocks: …"), never equalled
+  # "600", and the warning printed that dump instead of a mode. A permissions
+  # check on a secrets file that cannot fire is worse than none, because the
+  # noise reads like it ran. macOS has no `-c`, so it falls through to `-f`.
+  perms=$(stat -c '%a' "$ENV_FILE" 2>/dev/null || stat -f '%Lp' "$ENV_FILE" 2>/dev/null || echo '')
+  if [ -z "$perms" ]; then
+    echo "run.sh: warning: could not read the mode of $ENV_FILE; ensure it is chmod 600" >&2
+  elif [ "$perms" != "600" ]; then
+    echo "run.sh: warning: $ENV_FILE is mode $perms; chmod 600 it" >&2
+  fi
+  set -a; # shellcheck disable=SC1090
+  . "$ENV_FILE"; set +a
+}
+require_var() { [ -n "${!1:-}" ] || die "$2 needs $1 — set it in $ENV_FILE (see harbor.env.example)"; }
+
+if [ -n "$CLOUD" ] || [ "$BYOK" = 1 ]; then load_env_file; fi
+
+# ── cloud preflight: fail here, by name, not three minutes into a sandbox build
+if [ -n "$CLOUD" ]; then
+  case "$CLOUD" in
+    docker) CLOUD="" ;;
+    daytona)
+      if [ -z "${DAYTONA_API_KEY:-}" ] && { [ -z "${DAYTONA_JWT_TOKEN:-}" ] || [ -z "${DAYTONA_ORGANIZATION_ID:-}" ]; }; then
+        die "--cloud daytona needs DAYTONA_API_KEY (or DAYTONA_JWT_TOKEN + DAYTONA_ORGANIZATION_ID) in $ENV_FILE"
+      fi ;;
+    modal)   require_var MODAL_TOKEN_ID "--cloud modal"; require_var MODAL_TOKEN_SECRET "--cloud modal" ;;
+    e2b)     require_var E2B_API_KEY "--cloud e2b" ;;
+    runloop) require_var RUNLOOP_API_KEY "--cloud runloop" ;;
+    *)       echo "run.sh: warning: no credential preflight for provider '$CLOUD'; harbor will validate" >&2 ;;
+  esac
+  for a in ${ARGS[@]+"${ARGS[@]}"}; do
+    case "$a" in container_shell=*|--ak=container_shell*) die "--ak container_shell needs local Docker; drop it or drop --cloud" ;; esac
+  done
+fi
+
+# ── engine selection ─────────────────────────────────────────────────────────
+# The instruction file is NOT the same file across harnesses: Claude Code reads
+# CLAUDE.md, Codex reads AGENTS.md. Delivering the wrong one produces a with-arm
+# that ran without its instruction and still reports a clean number — the exact
+# defect that made `run --claude-md X` an undelivered baseline until 2026-08-31.
+case "$ENGINE" in
+  claude) AGENT_SPEC="rm_host_claude:HostClaudeCode"; MD_KEY="claude_md"; ENGINE_BIN="claude" ;;
+  codex)  AGENT_SPEC="rm_host_codex:HostCodex";       MD_KEY="agents_md"; ENGINE_BIN="codex"  ;;
+  *)      echo "run.sh: --engine must be claude or codex (got '$ENGINE')" >&2; exit 1 ;;
+esac
+
+# ── BYOK / remote agent ──────────────────────────────────────────────────────
+AGENT_FLAGS=(--agent "$AGENT_SPEC")
+AE=()
+if [ "$REMOTE" = 1 ]; then
+  [ "$ENGINE" = claude ] || die "--remote-agent is Harbor's built-in claude-code agent; it has no codex equivalent — drop --engine codex or drop --remote-agent"
+  [ "$BYOK" = 1 ] || die "--remote-agent runs Harbor's built-in claude-code inside the sandbox; it needs --byok (no keychain there)"
+  [ -z "$CLAUDE_MD" ] || die "--claude-md is host-agent delivery; with --remote-agent use --prompt-file or --skill"
+  AGENT_FLAGS=(--agent claude-code)
+fi
+
+# The host engine CLI is required only when we are actually going to SPAWN it.
+# This check deliberately sits AFTER argument validation and skips two cases:
+#
+#  --dry-run      prints the argv and never invokes harbor or the engine. The
+#                 contract suite runs the whole option matrix this way with no
+#                 harbor install (see harbor-run-contract.test.ts); requiring a
+#                 real `claude`/`codex` here made 18 of those cases fail on CI
+#                 while passing on a laptop that happens to have the CLI.
+#  --remote-agent runs Harbor's built-in claude-code INSIDE the sandbox, so the
+#                 host CLI is not in the loop at all.
+#
+# Order matters as much as the gate: when this ran before the refusals above,
+# `--engine codex --remote-agent` reported "CLI not found" instead of the real
+# reason, so the message named the wrong problem.
+if [ "$DRY" != 1 ] && [ "$REMOTE" != 1 ]; then
+  command -v "$ENGINE_BIN" >/dev/null 2>&1 \
+    || die "host \`$ENGINE_BIN\` CLI not found on PATH — --engine $ENGINE needs it"
+fi
+
+# Tooling preflight for UNATTENDED runs (2026-09-02). Until this existed the
+# only checked dependency was the engine CLI, and a run missing `harbor` died at
+# line 181 with bash's own `harbor: command not found` — AFTER announcing arm
+# 1/2, naming neither the tool nor the reason. That is survivable when a human
+# is watching and useless at 3am.
+#
+# `harbor` installs to ~/.local/bin, which is NOT on the launchd worker's PATH
+# (launchd omits Homebrew and friends). Docker Desktop installed but ASLEEP is
+# the ordinary state of a laptop overnight, and presence on PATH cannot see it —
+# only `docker info` can. Each refusal names its own tool: a scheduled job that
+# misreports why it failed costs more than one that simply fails.
+if [ "$DRY" != 1 ]; then
+  command -v harbor >/dev/null 2>&1 \
+    || die "\`harbor\` not found on PATH — install it, or add its dir (usually ~/.local/bin) to PATH. A launchd/cron PATH does not include it by default."
+  # --cloud moves the task containers off this machine; only the local docker
+  # environment needs a live daemon here.
+  if [ -z "$CLOUD" ] || [ "$CLOUD" = docker ]; then
+    docker info >/dev/null 2>&1 \
+      || die "the Docker daemon is not responding — \`docker info\` failed. Start Docker Desktop (an unattended run cannot); or use --cloud <provider> to run the containers off this machine."
+  fi
+fi
+if [ "$BYOK" = 1 ]; then
+  if [ -n "${ANTHROPIC_API_KEY:-}" ]; then
+    export ANTHROPIC_API_KEY                       # host agent: the child `claude` inherits it and bills the key
+    [ "$REMOTE" = 1 ] && AE+=(--ae "ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY")
+  elif [ -n "${CLAUDE_CODE_OAUTH_TOKEN:-}" ]; then
+    # The host agent strips CLAUDE_CODE_* from the child env on purpose (nested-
+    # session hygiene), so an OAuth token can only reach the REMOTE agent.
+    [ "$REMOTE" = 1 ] || die "--byok with CLAUDE_CODE_OAUTH_TOKEN only works with --remote-agent; set ANTHROPIC_API_KEY for the host agent"
+    AE+=(--ae "CLAUDE_CODE_OAUTH_TOKEN=$CLAUDE_CODE_OAUTH_TOKEN" --ae "CLAUDE_FORCE_OAUTH=1")
+  else
+    die "--byok needs ANTHROPIC_API_KEY (or CLAUDE_CODE_OAUTH_TOKEN for --remote-agent) in $ENV_FILE"
+  fi
+fi
+
+ENV_FLAGS=(); [ -n "$CLOUD" ] && ENV_FLAGS=(--env "$CLOUD")
+
+redact() { # print argv with any KEY=VALUE secret values masked
+  local out=() a
+  for a in "$@"; do
+    case "$a" in
+      *_API_KEY=*|*_TOKEN=*|*_SECRET=*|*_JWT_TOKEN=*) out+=("${a%%=*}=***") ;;
+      *) out+=("$(printf '%q' "$a")") ;;
+    esac
+  done
+  printf '%s\n' "${out[*]}"
+}
+
+hrun() { # job-name, then extra flags
+  local name="$1"; shift
+  local cmd=(harbor run -p "$TASK" "${AGENT_FLAGS[@]}" ${ENV_FLAGS[@]+"${ENV_FLAGS[@]}"} ${AE[@]+"${AE[@]}"} \
+    -o "$JOBS_DIR" --job-name "$name" -q -y "$@")
+  if [ "$DRY" = 1 ]; then echo "+ $(redact "${cmd[@]}")"; return 0; fi
+  "${cmd[@]}"
+}
+
+# Per-arm reward + cost telemetry (plan §5.3) from summarize.py: reward from
+# harbor's result.json, cost/tokens/turns summed over every claude-output.json
+# the host agent wrote. Remote-agent jobs report cost as n/a, never 0.
+summary()  { python3 "$HERE/summarize.py" "$JOBS_DIR/$1" --pretty; }
+field()    { python3 "$HERE/summarize.py" "$JOBS_DIR/$1" | python3 -c "import json,sys; v=json.load(sys.stdin)['$2']; print('' if v is None else v)"; }
+
+describe() {
+  echo "── harness: $([ "$REMOTE" = 1 ] && echo 'built-in claude-code (remote agent)' || echo "host agent (own $ENGINE_BIN CLI)")" \
+       "· compute: ${CLOUD:-local docker} · billing: $([ "$BYOK" = 1 ] && echo BYOK || echo subscription)"
+}
+
+case "$MODE" in
+  run)
+    # run mode delivers --claude-md / --prompt-file exactly like ab's with-arm.
+    # (Until 2026-08-31 it silently IGNORED them — every `run --claude-md X`
+    # trial was an undelivered baseline; the removal arms measured nothing.)
+    WITH=()
+    [ -z "$CLAUDE_MD" ] || { [ -f "$CLAUDE_MD" ] || CLAUDE_MD="$HERE/$CLAUDE_MD"; WITH+=(--ak "$MD_KEY=$CLAUDE_MD"); }
+    [ -z "$PROMPT_FILE" ] || { [ -f "$PROMPT_FILE" ] || PROMPT_FILE="$HERE/$PROMPT_FILE"; WITH+=(--ak "append_system_prompt=$(cat "$PROMPT_FILE")"); }
+    describe
+    NAME="run-$(date +%H%M%S)"
+    hrun "$NAME" ${WITH[@]+"${WITH[@]}"} ${ARGS[@]+"${ARGS[@]}"}
+    [ "$DRY" = 1 ] && exit 0
+    echo ""
+    echo "══ result ($(basename "$TASK")): $(summary "$NAME")"
+    echo "   job      : $JOBS_DIR/$NAME"
+    ;;
+  ab)
+    [ -n "$CLAUDE_MD" ] || [ -n "$PROMPT_FILE" ] || die "ab mode needs --claude-md <file> or --prompt-file <file>"
+    [ -z "$CLAUDE_MD" ] || [ -f "$CLAUDE_MD" ] || CLAUDE_MD="$HERE/$CLAUDE_MD"
+    [ -z "$PROMPT_FILE" ] || [ -f "$PROMPT_FILE" ] || PROMPT_FILE="$HERE/$PROMPT_FILE"
+    WITH=()
+    [ -z "$CLAUDE_MD" ] || WITH+=(--ak "$MD_KEY=$CLAUDE_MD")
+    [ -z "$PROMPT_FILE" ] || WITH+=(--ak "append_system_prompt=$(cat "$PROMPT_FILE")")
+    STAMP=$(date +%H%M%S)
+    describe
+    echo "── arm 1/2: baseline"
+    hrun "ab-$STAMP-baseline" ${ARGS[@]+"${ARGS[@]}"}
+    echo "── arm 2/2: with $(basename "${CLAUDE_MD:-$PROMPT_FILE}")"
+    hrun "ab-$STAMP-with" "${WITH[@]}" ${ARGS[@]+"${ARGS[@]}"}
+    [ "$DRY" = 1 ] && exit 0
+    B=$(field "ab-$STAMP-baseline" reward_mean); W=$(field "ab-$STAMP-with" reward_mean)
+    echo ""
+    echo "══ A/B result ($(basename "$TASK"))"
+    echo "   baseline : $(summary "ab-$STAMP-baseline")"
+    echo "   with-arm : $(summary "ab-$STAMP-with")"
+    python3 - "$B" "$W" <<'PY'
+import sys
+b, w = sys.argv[1:3]
+print(f"   delta    : {float(w)-float(b):+.3f} reward" if b and w else "   delta    : n/a (an arm has no reward)")
+PY
+    # Cost: a PER-TRIAL delta with a bootstrap CI and a decision, not the
+    # difference of two sums the display used to print. A sum has no variance and
+    # is confounded by trial count, so it could never tell a saving from noise.
+    # Exit 3 = an arm measured no cost at all; that is UNKNOWN, and printing
+    # nothing is the honest outcome (cost_delta.py has already said why on stderr).
+    python3 "$HERE/cost_delta.py" "$JOBS_DIR/ab-$STAMP-baseline" "$JOBS_DIR/ab-$STAMP-with" --brief \
+      | sed 's/^/   /' || true
+    echo "   jobs     : $JOBS_DIR/ab-$STAMP-{baseline,with}"
+    ;;
+  *)
+    die "unknown mode: $MODE (use run|ab)" ;;
+esac

package/dist/harbor/summarize.py

@@ -0,0 +1,248 @@
+"""Per-arm summary of a Harbor job: reward plus cost telemetry (plan §5.3).
+
+    python3 summarize.py <job-dir>            -> one JSON object on stdout
+    python3 summarize.py <job-dir> --pretty   -> aligned text for the terminal
+
+Reward comes from the job-level result.json (harbor's own aggregate). Cost
+comes from every `claude-output.json` the host agent wrote under the job dir
+(one per trial, `claude -p --output-format json`): total_cost_usd, duration,
+turns and token counts are SUMMED across trials. Cost is reported as null, never
+zero, when no such file exists — "we did not measure" must not read as "it was
+free".
+
+The codex host agent writes `codex-output.txt` instead. Since 2026-09-02 that
+file is JSONL (`codex exec --json`), whose final `turn.completed` event carries
+a real usage envelope, so codex arms now report TOKENS. They still report no
+`cost_usd`: codex states token counts, not a price, and inventing one from a
+local price table would be a number nobody measured.
+
+That leaves exactly one arm with nothing to read — the remote agent, which
+bills the key directly. The n/a copy therefore still states the FACT rather
+than asserting a cause it cannot verify: a control that lies about WHY it has
+no number is worse than one that says it has none.
+
+Kept dependency-free and importable so the contract test can exercise it on a
+synthetic job dir without harbor installed.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+from typing import Any
+
+TOKEN_FIELDS = (
+    "input_tokens",
+    "output_tokens",
+    "cache_read_input_tokens",
+    "cache_creation_input_tokens",
+)
+
+
+# codex's usage keys -> the claude-shaped fields this module aggregates, so a
+# mixed job sums into one comparable set. codex has no cache-CREATION counter.
+CODEX_TOKEN_MAP = {
+    "input_tokens": "input_tokens",
+    "output_tokens": "output_tokens",
+    "cached_input_tokens": "cache_read_input_tokens",
+}
+
+
+def parse_codex_usage(text: str) -> dict[str, int] | None:
+    """Token usage from `codex exec --json` JSONL, or None when absent.
+
+    Verified against codex-cli 0.144.0, which emits a final:
+      {"type":"turn.completed","usage":{"input_tokens":17970,
+       "cached_input_tokens":9984,"output_tokens":5,"reasoning_output_tokens":0}}
+
+    None means UNKNOWN, never zero. Later events win — a multi-turn run reports
+    cumulative usage on its final turn. Non-JSON lines are skipped: the stream
+    is progress output and a future codex may add shapes we do not know.
+    """
+    found: dict[str, int] | None = None
+    for line in text.splitlines():
+        line = line.strip()
+        if not line.startswith("{"):
+            continue
+        try:
+            event = json.loads(line)
+        except ValueError:
+            continue
+        if not isinstance(event, dict):
+            continue
+        usage = event.get("usage")
+        if not isinstance(usage, dict):
+            continue
+        mapped = {k: 0 for k in TOKEN_FIELDS}
+        for src, dst in CODEX_TOKEN_MAP.items():
+            try:
+                mapped[dst] = int(usage.get(src) or 0)
+            except (TypeError, ValueError):
+                mapped[dst] = 0
+        found = mapped
+    return found
+
+
+def _mean_reward(result: dict[str, Any]) -> float | None:
+    try:
+        evals = result["stats"]["evals"]
+        first = next(iter(evals.values()))
+        return float(first["metrics"][0]["mean"])
+    except (KeyError, StopIteration, IndexError, TypeError, ValueError):
+        return None
+
+
+def trial_reward(agent_output: Path) -> float | None:
+    """This trial's own reward from harbor's per-trial result.json, or None.
+
+    Layout: <job>/<trial>/agent/claude-output.json alongside
+            <job>/<trial>/result.json -> verifier_result.rewards.reward
+
+    None means UNKNOWN (no verdict written -- a crashed or interrupted trial),
+    never 0.0. Scoring an unknown as a failure would make a crashed run look like
+    an instruction that did not work.
+    """
+    path = agent_output.parent.parent / "result.json"
+    try:
+        data = json.loads(path.read_text())
+    except (OSError, ValueError):
+        return None
+    if not isinstance(data, dict):
+        return None
+    rewards = ((data.get("verifier_result") or {}).get("rewards")) or {}
+    value = rewards.get("reward")
+    try:
+        return float(value)
+    except (TypeError, ValueError):
+        return None
+
+
+def summarize(job_dir: Path) -> dict[str, Any]:
+    job_dir = Path(job_dir)
+    result_path = job_dir / "result.json"
+    result = json.loads(result_path.read_text()) if result_path.exists() else {}
+
+    outputs = sorted(job_dir.rglob("claude-output.json"))
+    cost = 0.0
+    duration_ms = 0
+    turns = 0
+    tokens = {k: 0 for k in TOKEN_FIELDS}
+    errors = 0
+    parsed = 0
+    # Per-trial records, kept alongside the sums. The sums answer "what did this
+    # arm cost"; only the per-trial series can answer "is the difference real",
+    # because a sum has no variance and is confounded by trial count. cost_delta.py
+    # reads this; the sums stay for the existing run.sh display contract.
+    trials: list[dict[str, Any]] = []
+    for path in outputs:
+        try:
+            out = json.loads(path.read_text())
+        except (OSError, ValueError):
+            continue
+        if not isinstance(out, dict):
+            continue
+        parsed += 1
+        trial_cost = float(out.get("total_cost_usd") or 0.0)
+        trial_duration = int(out.get("duration_ms") or 0)
+        trial_turns = int(out.get("num_turns") or 0)
+        cost += trial_cost
+        duration_ms += trial_duration
+        turns += trial_turns
+        if out.get("is_error"):
+            errors += 1
+        usage = out.get("usage") or {}
+        for k in TOKEN_FIELDS:
+            tokens[k] += int(usage.get(k) or 0)
+        trials.append(
+            {
+                "trial": str(path.parent.relative_to(job_dir)),
+                "cost_usd": trial_cost,
+                "duration_ms": trial_duration,
+                "turns": trial_turns,
+                "is_error": bool(out.get("is_error")),
+                # Paired with the cost so cost-per-success can be resampled: the
+                # trial that cost this much is the trial that did or did not pass.
+                # None when harbor wrote no verdict for the trial -- unknown, and
+                # cost_delta.py drops it rather than counting it as a failure.
+                "reward": trial_reward(path),
+            }
+        )
+
+    # Codex arms: tokens only. cost_usd stays None because codex reports no price.
+    codex_parsed = 0
+    for path in sorted(job_dir.rglob("codex-output.txt")):
+        try:
+            usage = parse_codex_usage(path.read_text(errors="replace"))
+        except OSError:
+            continue
+        if usage is None:
+            continue
+        codex_parsed += 1
+        for k in TOKEN_FIELDS:
+            tokens[k] += usage[k]
+
+    measured_tokens = parsed > 0 or codex_parsed > 0
+
+    stats = result.get("stats") or {}
+    n_trials = stats.get("n_trials")
+    if n_trials is None:
+        n_trials = len([p for p in job_dir.iterdir() if (p / "result.json").exists()]) if job_dir.exists() else 0
+
+    return {
+        "job": job_dir.name,
+        "reward_mean": _mean_reward(result),
+        "n_trials": n_trials,
+        # null, not 0, when nothing was measured (remote agent, or no trials ran).
+        # cost_usd is claude-only: codex reports tokens, never a price.
+        "cost_usd": round(cost, 6) if parsed else None,
+        "duration_ms": duration_ms if parsed else None,
+        "turns": turns if parsed else None,
+        "tokens": tokens if measured_tokens else None,
+        "agent_errors": errors if parsed else None,
+        # null, not [], when nothing was measured — same reason as cost_usd: an
+        # empty series and an unmeasured arm must not read alike downstream.
+        "trials": trials if parsed else None,
+        "host_agent_outputs": parsed,
+        "codex_agent_outputs": codex_parsed,
+    }
+
+
+def pretty(s: dict[str, Any]) -> str:
+    reward = "n/a" if s["reward_mean"] is None else f"{s['reward_mean']:.3f}"
+    if s["cost_usd"] is None and s["tokens"] is not None:
+        # A codex arm: real token counts, but codex states no price, so a dollar
+        # figure here would be invented rather than measured.
+        t = s["tokens"]
+        cost = (
+            f"cost n/a (codex reports tokens, not price) · "
+            f"in {t['input_tokens']} / out {t['output_tokens']} / "
+            f"cache-read {t['cache_read_input_tokens']} tokens"
+        )
+    elif s["cost_usd"] is None:
+        cost = (
+            "n/a (no per-trial cost or token output — the remote agent bills "
+            "the key directly; a host claude or codex arm would have written one)"
+        )
+    else:
+        t = s["tokens"]
+        cost = (
+            f"${s['cost_usd']:.4f} · {s['duration_ms'] / 1000:.0f}s · {s['turns']} turns · "
+            f"in {t['input_tokens']} / out {t['output_tokens']} / "
+            f"cache-read {t['cache_read_input_tokens']} tokens"
+            + (f" · {s['agent_errors']} agent error(s)" if s["agent_errors"] else "")
+        )
+    return f"reward {reward} over {s['n_trials']} trial(s) · {cost}"
+
+
+def main(argv: list[str]) -> int:
+    if len(argv) < 2:
+        print("usage: summarize.py <job-dir> [--pretty]", file=sys.stderr)
+        return 2
+    s = summarize(Path(argv[1]))
+    print(pretty(s) if "--pretty" in argv[2:] else json.dumps(s))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))

package/dist/lib/active-org-refresh.js

@@ -1 +1 @@
-import{a,b,c}from"../chunk-25RDT32M.js";import"../chunk-3AV3GTMP.js";import"../chunk-APDRR6FS.js";import"../chunk-J5HW45PZ.js";import"../chunk-PZNWH4VQ.js";import"../chunk-ZLLER3HP.js";export{c as _resetRefreshAttempt,b as bootstrapActiveOrg,a as maybeRefreshActiveOrg};
+import{a,b,c}from"../chunk-4FJYVYCU.js";import"../chunk-3AV3GTMP.js";import"../chunk-76UKAFNM.js";import"../chunk-42A4PUW7.js";import"../chunk-XQYAVSUQ.js";import"../chunk-PZNWH4VQ.js";import"../chunk-ZLLER3HP.js";export{c as _resetRefreshAttempt,b as bootstrapActiveOrg,a as maybeRefreshActiveOrg};

package/dist/lib/agent-loop.js

@@ -1 +1 @@
-import{a,b,c,d,e}from"../chunk-MLV2RHRE.js";import"../chunk-QFKUSNUB.js";import"../chunk-4N43IBXH.js";import"../chunk-IEQDLURA.js";import"../chunk-KINIFUZQ.js";import"../chunk-VIQ25ORK.js";import"../chunk-6ZPSVZNH.js";import"../chunk-RGUCR7T6.js";import"../chunk-WY3VVX3R.js";import"../chunk-KV5A77XW.js";import"../chunk-ZYGBABMZ.js";import"../chunk-U7TBEC3I.js";import"../chunk-Y7QIOPN5.js";import"../chunk-HOQPVOMF.js";import"../chunk-DUBBL57L.js";import"../chunk-WHQ4BDSM.js";import"../chunk-PPXOC7IV.js";import"../chunk-3MDEGOUG.js";import"../chunk-JNFB22FD.js";import"../chunk-HLQZR4XJ.js";import"../chunk-5RBLIVBF.js";import"../chunk-SN3CHV66.js";import"../chunk-6W2GSV4O.js";import"../chunk-JWFRI32Y.js";import"../chunk-25Q3Y7TC.js";import"../chunk-MNQPCNGQ.js";import"../chunk-PZS55PW2.js";import"../chunk-WQX4RRBQ.js";import"../chunk-3ZHVP3KE.js";import"../chunk-WOJ733IB.js";import"../chunk-RJX75Y7J.js";import"../chunk-SJKS3PYX.js";import"../chunk-2EX27VJ3.js";import"../chunk-UM4CBCXI.js";import"../chunk-3UKZJAFS.js";import"../chunk-4COLIGD7.js";import"../chunk-VXUKXGRX.js";import"../chunk-PVGUDJD7.js";import"../chunk-GIRPEMTW.js";import"../chunk-SIGZEQ3K.js";import"../chunk-4FSQSETR.js";import"../chunk-VH2EAP4J.js";import"../chunk-J6QMBMC2.js";import"../chunk-UYPBIXF2.js";import"../chunk-OZNKMJ5B.js";import"../chunk-F6DAHCJC.js";import"../chunk-GBWQENAH.js";import"../chunk-LJRE3LZ2.js";import"../chunk-YI2OWO6R.js";import"../chunk-V2USQH27.js";import"../chunk-RK663O4M.js";import"../chunk-6GAIYJ6K.js";import"../chunk-6UEFCMR2.js";import"../chunk-MODXEGCU.js";import"../chunk-RBSEE2Z6.js";import"../chunk-BG55WDDX.js";import"../chunk-H3BCWSYO.js";import"../chunk-M7QSC64D.js";import"../chunk-LTQ5BJYL.js";import"../chunk-G7V4ZZZ3.js";import"../chunk-6WQG6C7F.js";import"../chunk-LB24UKHZ.js";import"../chunk-NRYLYMDM.js";import"../chunk-U3XKXJ3A.js";import"../chunk-5MXZJWQS.js";import"../chunk-N7ODX6PO.js";import"../chunk-OZQNSELV.js";import"../chunk-HFGO5762.js";import"../chunk-VS5ROGDP.js";import"../chunk-KCOJLTVX.js";import"../chunk-GN3ZHQ46.js";import"../chunk-U2NT5OSQ.js";import"../chunk-KGURGIZ2.js";import"../chunk-UPFH72PF.js";import"../chunk-WH3IPXXQ.js";import"../chunk-CDY7TQFZ.js";import"../chunk-VBXL7XO7.js";import"../chunk-HLNHBZLI.js";import"../chunk-BZS5QRWB.js";import"../chunk-SBGXIPRN.js";import"../chunk-6XTLP2G7.js";import"../chunk-PJ3WHQ67.js";import"../chunk-APDRR6FS.js";import"../chunk-J5HW45PZ.js";import"../chunk-PZNWH4VQ.js";import"../chunk-ZLLER3HP.js";export{b as CAPABILITIES,a as HANDLERS,c as availableCapabilities,d as pollForWork,e as runAgentLoop};
+import{a,b,c,d,e}from"../chunk-BAITMD76.js";import"../chunk-W23OYQXI.js";import"../chunk-EMLLGASZ.js";import"../chunk-IEQDLURA.js";import"../chunk-KINIFUZQ.js";import"../chunk-V55ROUNP.js";import"../chunk-XMA6SWOT.js";import"../chunk-RAQLVO2X.js";import"../chunk-DHX4U6EC.js";import"../chunk-NYNHX7OX.js";import"../chunk-VXL5H3VZ.js";import"../chunk-QAMHZY2P.js";import"../chunk-RMRVUI7B.js";import"../chunk-ISFPPDDB.js";import"../chunk-ZYGBABMZ.js";import"../chunk-5RBLIVBF.js";import"../chunk-AIYFT4P6.js";import"../chunk-PZS55PW2.js";import"../chunk-3IMRLHJJ.js";import"../chunk-3ZHVP3KE.js";import"../chunk-EUHRTEVJ.js";import"../chunk-HOQPVOMF.js";import"../chunk-GKVHFXUQ.js";import"../chunk-LJRJ2RAC.js";import"../chunk-E7GQJML3.js";import"../chunk-QRIJ3AHP.js";import"../chunk-CT6GJQIR.js";import"../chunk-OVS7ZDJ4.js";import"../chunk-AGEZAKCC.js";import"../chunk-I6ZQOMZS.js";import"../chunk-6ANODRDT.js";import"../chunk-3UKZJAFS.js";import"../chunk-XJ25HFIT.js";import"../chunk-SJKS3PYX.js";import"../chunk-YK2IEHLB.js";import"../chunk-PIVEJTNF.js";import"../chunk-7MTYUTUN.js";import"../chunk-MMLRWCBW.js";import"../chunk-GIRPEMTW.js";import"../chunk-SIGZEQ3K.js";import"../chunk-DQYYHX6K.js";import"../chunk-2UK4DZFW.js";import"../chunk-3FNTV2TW.js";import"../chunk-JTYVBR5H.js";import"../chunk-B5X6HCBK.js";import"../chunk-T75KAEJO.js";import"../chunk-RXZLBSDO.js";import"../chunk-LJRE3LZ2.js";import"../chunk-MXHAXY66.js";import"../chunk-FYZ6HHCV.js";import"../chunk-OP4U3XBZ.js";import"../chunk-HNYHZO2J.js";import"../chunk-6UEFCMR2.js";import"../chunk-MODXEGCU.js";import"../chunk-RBSEE2Z6.js";import"../chunk-KOYHJFB3.js";import"../chunk-POEMCITL.js";import"../chunk-M7QSC64D.js";import"../chunk-XGMOVQ5G.js";import"../chunk-LB24UKHZ.js";import"../chunk-G7V4ZZZ3.js";import"../chunk-43VKTVM3.js";import"../chunk-AHLVY52I.js";import"../chunk-SCQK52SL.js";import"../chunk-ZK3ACE4T.js";import"../chunk-N4YPDZJF.js";import"../chunk-PPAHIVG5.js";import"../chunk-5MXZJWQS.js";import"../chunk-CT7FWSIU.js";import"../chunk-N7ODX6PO.js";import"../chunk-OZQNSELV.js";import"../chunk-HFGO5762.js";import"../chunk-VS5ROGDP.js";import"../chunk-H6FVXVGG.js";import"../chunk-SWSOEYC7.js";import"../chunk-U2NT5OSQ.js";import"../chunk-KGURGIZ2.js";import"../chunk-GYNZ3OZF.js";import"../chunk-UPFH72PF.js";import"../chunk-DVU5PHPM.js";import"../chunk-CDY7TQFZ.js";import"../chunk-VBXL7XO7.js";import"../chunk-FBSZ6KCM.js";import"../chunk-JP6ULOEX.js";import"../chunk-SG4UDDHB.js";import"../chunk-GPBN4ROE.js";import"../chunk-6XTLP2G7.js";import"../chunk-PJ3WHQ67.js";import"../chunk-76UKAFNM.js";import"../chunk-42A4PUW7.js";import"../chunk-XQYAVSUQ.js";import"../chunk-PZNWH4VQ.js";import"../chunk-ZLLER3HP.js";export{b as CAPABILITIES,a as HANDLERS,c as availableCapabilities,d as pollForWork,e as runAgentLoop};

package/dist/lib/antigravity-transcript.js

@@ -1 +1 @@
-import{a,b,c,d,e,f,g}from"../chunk-QTCWOIT4.js";import"../chunk-APDRR6FS.js";import"../chunk-J5HW45PZ.js";import"../chunk-PZNWH4VQ.js";import"../chunk-ZLLER3HP.js";export{d as classifyRecord,e as emitRecordsAsEvents,c as extractUserPrompt,b as findMaxStepIndex,a as readNewRecords,g as stateApi,f as tailAntigravityTranscript};
+import{a,b,c,d,e,f,g}from"../chunk-WWQWHC4K.js";import"../chunk-76UKAFNM.js";import"../chunk-42A4PUW7.js";import"../chunk-XQYAVSUQ.js";import"../chunk-PZNWH4VQ.js";import"../chunk-ZLLER3HP.js";export{d as classifyRecord,e as emitRecordsAsEvents,c as extractUserPrompt,b as findMaxStepIndex,a as readNewRecords,g as stateApi,f as tailAntigravityTranscript};

package/dist/lib/api-client.js

@@ -1 +1 @@
-import{a,b,c,d,e,f,g,h,i}from"../chunk-APDRR6FS.js";import"../chunk-J5HW45PZ.js";import"../chunk-PZNWH4VQ.js";import"../chunk-ZLLER3HP.js";export{b as ApiError,a as DEFAULT_API_URL,d as api,i as apiDelete,e as apiGet,h as apiPatch,f as apiPost,g as apiPut,c as getBaseUrl};
+import{a,b,c,d,e,f,g,h,i}from"../chunk-76UKAFNM.js";import"../chunk-42A4PUW7.js";import"../chunk-XQYAVSUQ.js";import"../chunk-PZNWH4VQ.js";import"../chunk-ZLLER3HP.js";export{b as ApiError,a as DEFAULT_API_URL,d as api,i as apiDelete,e as apiGet,h as apiPatch,f as apiPost,g as apiPut,c as getBaseUrl};

package/dist/lib/autorun-batch.js

@@ -0,0 +1 @@
+import{a}from"../chunk-DQYYHX6K.js";import"../chunk-ZLLER3HP.js";export{a as selectNightlyCases};

package/dist/lib/capture-scope-config.js

@@ -1 +1 @@
-import{a,b,c,d,e,f,g}from"../chunk-SHYP27B7.js";import"../chunk-QXPYGN7I.js";import"../chunk-ZLLER3HP.js";export{f as applyScopeEdit,g as backupUnparseableConfig,a as captureConfigPath,e as isUsableRoot,d as normaliseRoot,b as readCaptureConfig,c as writeCaptureConfig};
+import{a,b,c,d,e,f,g}from"../chunk-VALRPZXH.js";import"../chunk-QI3FJKGK.js";import"../chunk-ZLLER3HP.js";export{f as applyScopeEdit,g as backupUnparseableConfig,a as captureConfigPath,e as isUsableRoot,d as normaliseRoot,b as readCaptureConfig,c as writeCaptureConfig};

package/dist/lib/codex-capture-scope.js

@@ -1 +1 @@
-import{a,b,c,d,e,f,g,h}from"../chunk-QXPYGN7I.js";import"../chunk-ZLLER3HP.js";export{a as CAPTURE_CONFIG_FILENAME,h as createCodexScopeWatcher,e as describeCaptureScope,f as describeCodexCaptureScope,g as describeHooksCaptureScope,d as isPathInScope,b as loadCodexCaptureScope,c as loadHooksCaptureScope};
+import{a,b,c,d,e,f,g,h,i,j,k}from"../chunk-QI3FJKGK.js";import"../chunk-ZLLER3HP.js";export{a as CAPTURE_CONFIG_FILENAME,j as createCodexScopeWatcher,k as createPiScopeWatcher,f as describeCaptureScope,g as describeCodexCaptureScope,h as describeHooksCaptureScope,i as describePiCaptureScope,e as isPathInScope,b as loadCodexCaptureScope,c as loadHooksCaptureScope,d as loadPiCaptureScope};

package/dist/lib/codex-exec.js

@@ -1 +1 @@
-import{a,b,c,d}from"../chunk-BZS5QRWB.js";import"../chunk-6XTLP2G7.js";import"../chunk-PJ3WHQ67.js";import"../chunk-ZLLER3HP.js";export{b as buildCodexArgs,a as codexModelId,d as executeCodexEval,c as parseCodexUsage};
+import{a,b,c,d}from"../chunk-SG4UDDHB.js";import"../chunk-6XTLP2G7.js";import"../chunk-PJ3WHQ67.js";import"../chunk-ZLLER3HP.js";export{b as buildCodexArgs,a as codexModelId,d as executeCodexEval,c as parseCodexUsage};

package/dist/lib/ensure-api-key.js

@@ -1 +1 @@
-import{a}from"../chunk-M2VVRQKA.js";import"../chunk-5XK6LB2T.js";import"../chunk-APDRR6FS.js";import"../chunk-J5HW45PZ.js";import"../chunk-PZNWH4VQ.js";import"../chunk-ZLLER3HP.js";export{a as ensureDeviceApiKey};
+import{a}from"../chunk-RG3R4EAB.js";import"../chunk-FKTT7A4H.js";import"../chunk-76UKAFNM.js";import"../chunk-42A4PUW7.js";import"../chunk-XQYAVSUQ.js";import"../chunk-PZNWH4VQ.js";import"../chunk-ZLLER3HP.js";export{a as ensureDeviceApiKey};

package/dist/lib/ensure-fresh-token.js

@@ -1 +1 @@
-import{a}from"../chunk-P2WSDHIA.js";import"../chunk-APDRR6FS.js";import"../chunk-J5HW45PZ.js";import"../chunk-PZNWH4VQ.js";import"../chunk-ZLLER3HP.js";export{a as ensureFreshToken};
+import{a}from"../chunk-VIFOLTZT.js";import"../chunk-76UKAFNM.js";import"../chunk-42A4PUW7.js";import"../chunk-XQYAVSUQ.js";import"../chunk-PZNWH4VQ.js";import"../chunk-ZLLER3HP.js";export{a as ensureFreshToken};

package/dist/lib/eval-accumulation.js

@@ -0,0 +1 @@
+import{a}from"../chunk-AHLVY52I.js";import"../chunk-ZLLER3HP.js";export{a as findOpenAccumulation};

package/dist/lib/eval-analyzer.js

@@ -1 +1 @@
-import{a}from"../chunk-BG55WDDX.js";import"../chunk-HLNHBZLI.js";import"../chunk-BZS5QRWB.js";import"../chunk-SBGXIPRN.js";import"../chunk-6XTLP2G7.js";import"../chunk-PJ3WHQ67.js";import"../chunk-ZLLER3HP.js";export{a as analyzeEvalResults};
+import{a}from"../chunk-KOYHJFB3.js";import"../chunk-FBSZ6KCM.js";import"../chunk-JP6ULOEX.js";import"../chunk-SG4UDDHB.js";import"../chunk-GPBN4ROE.js";import"../chunk-6XTLP2G7.js";import"../chunk-PJ3WHQ67.js";import"../chunk-ZLLER3HP.js";export{a as analyzeEvalResults};

package/dist/lib/eval-benchmark.js

@@ -1 +1 @@
-import{a,b,c}from"../chunk-O2HCD6XL.js";import"../chunk-VBXL7XO7.js";import"../chunk-APDRR6FS.js";import"../chunk-J5HW45PZ.js";import"../chunk-PZNWH4VQ.js";import"../chunk-ZLLER3HP.js";export{b as aggregateBenchmark,c as postBenchmark,a as summarizeRuns};
+import{a,b,c}from"../chunk-IO46ICZY.js";import"../chunk-VBXL7XO7.js";import"../chunk-76UKAFNM.js";import"../chunk-42A4PUW7.js";import"../chunk-XQYAVSUQ.js";import"../chunk-PZNWH4VQ.js";import"../chunk-ZLLER3HP.js";export{b as aggregateBenchmark,c as postBenchmark,a as summarizeRuns};

package/dist/lib/eval-candidate-run.js

@@ -1 +1 @@
-import{a,b,c,d,e,f,g}from"../chunk-NRYLYMDM.js";import"../chunk-KCOJLTVX.js";import"../chunk-GN3ZHQ46.js";import"../chunk-U2NT5OSQ.js";import"../chunk-KGURGIZ2.js";import"../chunk-UPFH72PF.js";import"../chunk-WH3IPXXQ.js";import"../chunk-CDY7TQFZ.js";import"../chunk-HLNHBZLI.js";import"../chunk-BZS5QRWB.js";import"../chunk-SBGXIPRN.js";import"../chunk-6XTLP2G7.js";import"../chunk-PJ3WHQ67.js";import"../chunk-APDRR6FS.js";import"../chunk-J5HW45PZ.js";import"../chunk-PZNWH4VQ.js";import"../chunk-ZLLER3HP.js";export{a as CANDIDATE_ARM,b as PRIOR_VERSION_ARM,d as accountForFailures,e as attributeShortfall,f as buildCandidateQueue,c as pairCandidateRuns,g as runCandidateBatchAndPost};
+import{a,b,c,d,e,f,g,h,i}from"../chunk-SCQK52SL.js";import"../chunk-H6FVXVGG.js";import"../chunk-SWSOEYC7.js";import"../chunk-U2NT5OSQ.js";import"../chunk-KGURGIZ2.js";import"../chunk-GYNZ3OZF.js";import"../chunk-UPFH72PF.js";import"../chunk-DVU5PHPM.js";import"../chunk-CDY7TQFZ.js";import"../chunk-FBSZ6KCM.js";import"../chunk-JP6ULOEX.js";import"../chunk-SG4UDDHB.js";import"../chunk-GPBN4ROE.js";import"../chunk-6XTLP2G7.js";import"../chunk-PJ3WHQ67.js";import"../chunk-76UKAFNM.js";import"../chunk-42A4PUW7.js";import"../chunk-XQYAVSUQ.js";import"../chunk-PZNWH4VQ.js";import"../chunk-ZLLER3HP.js";export{a as CANDIDATE_ARM,b as PRIOR_VERSION_ARM,f as accountForFailures,g as attributeShortfall,h as buildCandidateQueue,d as measurementBatchKey,c as observedExecutionKey,e as pairCandidateRuns,i as runCandidateBatchAndPost};

package/dist/lib/eval-case-generator.js

@@ -1 +1 @@
-import{a,b}from"../chunk-RK663O4M.js";import"../chunk-U3XKXJ3A.js";import"../chunk-5MXZJWQS.js";import"../chunk-N7ODX6PO.js";import"../chunk-BZS5QRWB.js";import"../chunk-SBGXIPRN.js";import"../chunk-6XTLP2G7.js";import"../chunk-PJ3WHQ67.js";import"../chunk-ZLLER3HP.js";export{a as buildFeedbackBlock,b as generateEvalCases};
+import{a,b}from"../chunk-OP4U3XBZ.js";import"../chunk-ZK3ACE4T.js";import"../chunk-N4YPDZJF.js";import"../chunk-PPAHIVG5.js";import"../chunk-5MXZJWQS.js";import"../chunk-CT7FWSIU.js";import"../chunk-N7ODX6PO.js";import"../chunk-FBSZ6KCM.js";import"../chunk-JP6ULOEX.js";import"../chunk-SG4UDDHB.js";import"../chunk-GPBN4ROE.js";import"../chunk-6XTLP2G7.js";import"../chunk-PJ3WHQ67.js";import"../chunk-XQYAVSUQ.js";import"../chunk-ZLLER3HP.js";export{a as buildFeedbackBlock,b as generateEvalCases};

package/dist/lib/eval-conversation-handler.js

@@ -1 +1 @@
-import{a}from"../chunk-PVGUDJD7.js";import"../chunk-RK663O4M.js";import"../chunk-BG55WDDX.js";import"../chunk-U3XKXJ3A.js";import"../chunk-5MXZJWQS.js";import"../chunk-N7ODX6PO.js";import"../chunk-WH3IPXXQ.js";import"../chunk-CDY7TQFZ.js";import"../chunk-HLNHBZLI.js";import"../chunk-BZS5QRWB.js";import"../chunk-SBGXIPRN.js";import"../chunk-6XTLP2G7.js";import"../chunk-PJ3WHQ67.js";import"../chunk-APDRR6FS.js";import"../chunk-J5HW45PZ.js";import"../chunk-PZNWH4VQ.js";import"../chunk-ZLLER3HP.js";export{a as processConversationJob};
+import{a}from"../chunk-MMLRWCBW.js";import"../chunk-OP4U3XBZ.js";import"../chunk-KOYHJFB3.js";import"../chunk-ZK3ACE4T.js";import"../chunk-N4YPDZJF.js";import"../chunk-PPAHIVG5.js";import"../chunk-5MXZJWQS.js";import"../chunk-CT7FWSIU.js";import"../chunk-N7ODX6PO.js";import"../chunk-DVU5PHPM.js";import"../chunk-CDY7TQFZ.js";import"../chunk-FBSZ6KCM.js";import"../chunk-JP6ULOEX.js";import"../chunk-SG4UDDHB.js";import"../chunk-GPBN4ROE.js";import"../chunk-6XTLP2G7.js";import"../chunk-PJ3WHQ67.js";import"../chunk-76UKAFNM.js";import"../chunk-42A4PUW7.js";import"../chunk-XQYAVSUQ.js";import"../chunk-PZNWH4VQ.js";import"../chunk-ZLLER3HP.js";export{a as processConversationJob};

package/dist/lib/eval-executor.js

@@ -1 +1 @@
-import{a,b,c,d,e,f}from"../chunk-HLNHBZLI.js";import"../chunk-BZS5QRWB.js";import"../chunk-SBGXIPRN.js";import"../chunk-6XTLP2G7.js";import"../chunk-PJ3WHQ67.js";import"../chunk-ZLLER3HP.js";export{d as buildClaudeArgs,e as describeExitFailure,f as executeEval,c as parseClaudeJsonOutput,b as resolveCodexEvalModel,a as resolveEvalEngine};
+import{a,b,c,d,e,f}from"../chunk-FBSZ6KCM.js";import"../chunk-JP6ULOEX.js";import"../chunk-SG4UDDHB.js";import"../chunk-GPBN4ROE.js";import"../chunk-6XTLP2G7.js";import"../chunk-PJ3WHQ67.js";import"../chunk-ZLLER3HP.js";export{d as buildClaudeArgs,e as describeExitFailure,f as executeEval,c as parseClaudeJsonOutput,b as resolveCodexEvalModel,a as resolveEvalEngine};

package/dist/lib/eval-grader.js

@@ -1 +1 @@
-import{a}from"../chunk-WH3IPXXQ.js";import"../chunk-CDY7TQFZ.js";import"../chunk-HLNHBZLI.js";import"../chunk-BZS5QRWB.js";import"../chunk-SBGXIPRN.js";import"../chunk-6XTLP2G7.js";import"../chunk-PJ3WHQ67.js";import"../chunk-ZLLER3HP.js";export{a as gradeEvalOutput};
+import{a}from"../chunk-DVU5PHPM.js";import"../chunk-CDY7TQFZ.js";import"../chunk-FBSZ6KCM.js";import"../chunk-JP6ULOEX.js";import"../chunk-SG4UDDHB.js";import"../chunk-GPBN4ROE.js";import"../chunk-6XTLP2G7.js";import"../chunk-PJ3WHQ67.js";import"../chunk-ZLLER3HP.js";export{a as gradeEvalOutput};

package/dist/lib/eval-meta-judge.js

@@ -1 +1 @@
-import{a}from"../chunk-VXUKXGRX.js";import"../chunk-HLNHBZLI.js";import"../chunk-BZS5QRWB.js";import"../chunk-SBGXIPRN.js";import"../chunk-6XTLP2G7.js";import"../chunk-PJ3WHQ67.js";import"../chunk-ZLLER3HP.js";export{a as judgeEvalQuality};
+import{a}from"../chunk-7MTYUTUN.js";import"../chunk-FBSZ6KCM.js";import"../chunk-JP6ULOEX.js";import"../chunk-SG4UDDHB.js";import"../chunk-GPBN4ROE.js";import"../chunk-6XTLP2G7.js";import"../chunk-PJ3WHQ67.js";import"../chunk-ZLLER3HP.js";export{a as judgeEvalQuality};