@aigne/afs-llm-bench 1.12.0-beta.5

package/dist/runner.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.mjs","names":["promptIds","fs"],"sources":["../src/runner.ts"],"sourcesContent":["/**\n * Phase 1 — runOne + execRun matrix + results writer.\n *\n * Dispatches a single (model, prompt) sample through `/dev/agent/.actions/run`,\n * scores the trace via `scoring.ts`, and writes per-model aggregates to a JSON\n * file. Credentials are NOT held by bench — the caller forwards a model id and\n * ai-device routing injects vault secrets at the LLM call site.\n *\n * See plan.md L294-378 (impl) + L745-752 (test spec).\n */\n\nimport { randomBytes } from \"node:crypto\";\nimport { promises as fs } from \"node:fs\";\nimport { join } from \"node:path\";\nimport type { AgentRunResult } from \"@aigne/afs-agent\";\nimport { ValidationError } from \"./errors.js\";\nimport { aggregateJudgeScores, DEFAULT_JUDGE_MODEL, judgeInstructionFollowing } from \"./judge.js\";\nimport { DEFAULT_OUTLIER_THRESHOLD, findOutliers, type OutlierEntry } from \"./outlier.js\";\nimport type { Prompt } from \"./prompts.js\";\nimport {\n  type BenchPrompt,\n  type DimensionScores,\n  OBJECTIVE_DIMENSIONS,\n  scoreFromTrace,\n} from \"./scoring.js\";\n\nexport { ValidationError };\n\n// ─── Constants ────────────────────────────────────────────────────────────\n\nconst DEFAULT_SAMPLES = 3;\nconst MAX_MODELS = 5;\nconst MAX_PROMPTS_PER_RUN = 5;\nconst MAX_MODELS_FULL = 200;\nconst MAX_PROMPTS_FULL = 200;\nconst MAX_SAMPLES = 100;\nconst MAX_PARALLEL = 4;\n// Per-sample token budget. agent-run counts cumulative tokens across all\n// rounds (each round re-sends prior tool results in context), so the cap\n// scales roughly with round count. Hard cap is 2× prompt.max_rounds (see\n// runOne) — for the longest prompt (12 rounds → 24-round cap) reasoning\n// models can hit 200K+ before stopping. 240K covers that without giving\n// runaway loops unbounded room. 2026-04-28: opus-4-7 × 02-ui-control hit\n// 140K at round 12/16 with the 120K cap.\nconst DEFAULT_TOTAL_TOKENS = 240_000;\nconst DEFAULT_WALL_TIME_MS = 5 * 60_000;\n\n/**\n * Hub used to expand short model ids (e.g. `\"claude-haiku-4-5\"`) into a full\n * AFS path before dispatching to `/dev/agent/.actions/run`. agent-run's own\n * `resolveAgentModelPath` defaults to AigneHub, which requires credits — we\n * default to Vertex so OAuth-backed local installs work without billing.\n *\n * Override via `runFull` arg `hub` (or by passing the full\n * `/dev/ai/hubs/<hub>/models/<id>` path as the model id).\n */\nconst DEFAULT_HUB = \"vertex\";\n\n/** Expand a short model id to its full hub path. Full paths pass through. */\nfunction resolveModelPath(model: string, hub: string = DEFAULT_HUB): string {\n  if (model.startsWith(\"/\")) return model;\n  return `/dev/ai/hubs/${hub}/models/${model}`;\n}\nconst RATE_LIMIT_MAX_RETRIES = 3;\nconst RATE_LIMIT_BASE_BACKOFF_MS = 2000;\n// Hard cap on serialised main-results payload. Includes per_prompt + stddev but\n// NOT raw traces (which go to the opt-in `<stamp>-raw.json` sidecar). 10 MB is\n// generous enough for 200 model × 200 prompt × 5-dim scores yet small enough\n// that an accidental include of trace[] surfaces as a hard fail rather than\n// quietly bloating the results dir over many runs.\nconst MAX_RESULTS_SIZE_BYTES = 10 * 1024 * 1024;\n\n// Phase 1 of Run 3 will fully implement; this phase only validates the arg.\nconst ALLOWED_JUDGE_DIMENSIONS = [\"instruction_following\"] as const;\n\n/**\n * Hard cap on multi-judge fanout (Phase 0 of Run 4). Each judge is one extra\n * `/dev/ai/.actions/chat` call per successful sample, so a 200×200 matrix\n * with 100 samples explodes quickly. 10 is generous enough for ensemble\n * experiments and small enough that runaway configs surface as validation\n * errors rather than silently burning cost.\n */\nconst MAX_JUDGES = 10;\n// Same vocabulary as `judgeModel` inside judge.ts — judge ids must be safe\n// hub path segments (no `/`, no `..`). Re-validated at the judge layer too,\n// but failing fast keeps a hostile config from racing N parallel calls.\nconst JUDGE_ID_PATTERN = /^[A-Za-z0-9._-]+$/;\n\n// Secret VALUE patterns — redact the actual credential material, not just the\n// label. These run BEFORE the label patterns so composite shapes like\n// `Authorization: Bearer <token>` have the *token* redacted (a bare `\\bbearer\\b`\n// label rule would blank only the word \"Bearer\" and leave the token on disk —\n// issue #830). Anchored to real credential shapes (word boundary + length, and\n// for `sk-` a required digit) so ordinary prose such as \"risk-assessment\" or\n// \"task-oriented\" is left untouched. Serialised JSON escapes newlines as `\\n`,\n// so the PEM pattern matches across the escaped block. Best-effort: this is a\n// safety net, not a guarantee — the design keeps credentials out of traces.\nconst SECRET_VALUE_PATTERNS: RegExp[] = [\n  // PEM private-key blocks (RSA / EC / OPENSSH / plain PRIVATE KEY).\n  /-----BEGIN [A-Z0-9 ]*?PRIVATE KEY-----[\\s\\S]*?-----END [A-Z0-9 ]*?PRIVATE KEY-----/g,\n  // `Bearer <token>` — redact the token, not just the \"Bearer\" label.\n  /\\bBearer\\s+[A-Za-z0-9._~+/=-]+/gi,\n  // OpenAI-style keys (`sk-`, `sk-proj-`, …). Require a digit + length so\n  // benign hyphenated words (\"risk-assessment\") never match.\n  /\\bsk-[A-Za-z0-9_-]*[0-9][A-Za-z0-9_-]{6,}/g,\n  // Google OAuth access tokens.\n  /\\bya29\\.[A-Za-z0-9._-]+/g,\n  // Slack tokens (xoxb-/xoxp-/xoxa-/xoxr-/xoxs-).\n  /\\bxox[baprs]-[A-Za-z0-9-]+/gi,\n  // AWS access key ids.\n  /\\bAKIA[0-9A-Z]{16}\\b/g,\n];\n\n// Sensitive LABEL substrings that must never appear in a results file.\n// Sanitisation runs over the JSON-serialised payload before it lands on disk;\n// matches are redacted in place. The list is conservative (false positives are\n// cheap). Label rules complement the value rules above (e.g. redacting the\n// literal key name `api_key` / `Authorization`), not replace them.\nconst SENSITIVE_PATTERNS: RegExp[] = [\n  /\\bapi[_-]?key\\b/gi,\n  /\\bauthorization\\b/gi,\n  /\\bbearer\\b/gi,\n  /\\bservice[_-]?account\\b/gi,\n  /\\bprivate[_-]?key\\b/gi,\n  /\\bAKIA[0-9A-Z]{0,16}\\b/g,\n];\n\n// ─── Tools / system prompt for benchmark agent-run ────────────────────────\n\nexport interface ToolEntry {\n  path: string;\n  ops: string[];\n  maxDepth?: number;\n}\n\n/** Permissive read-only AFS sandbox so benchmark prompts can explore mounts. */\nexport function defaultBenchTools(): ToolEntry[] {\n  return [{ path: \"/**\", ops: [\"list\", \"read\", \"explain\", \"stat\", \"exec\"], maxDepth: 10 }];\n}\n\nexport function defaultBenchSystem(): string {\n  return [\n    \"You are an AFS power user evaluating mounts and actions.\",\n    \"Tools: afs_list, afs_read, afs_explain, afs_stat, afs_exec.\",\n    \"Prefer parallel tool calls when listing or explaining sibling paths.\",\n    \"Stop and summarize as soon as you have enough context to answer.\",\n  ].join(\"\\n\");\n}\n\n// ─── runOne dispatch ─────────────────────────────────────────────────────\n\nexport interface ExecLike {\n  exec(\n    path: string,\n    args: Record<string, unknown>,\n  ): Promise<{\n    success: boolean;\n    data?: Record<string, unknown> | unknown;\n    error?: { code?: string; message?: string };\n  }>;\n}\n\nexport interface RunOneInput {\n  deps: ExecLike;\n  model: string;\n  /** Hub used to expand short `model` ids; ignored when `model` is a full\n   *  AFS path. Defaults to `\"vertex\"` (see DEFAULT_HUB). */\n  hub?: string;\n  prompt: Prompt;\n  benchPrompt: BenchPrompt;\n}\n\nexport interface RunOneResult {\n  model: string;\n  prompt: string;\n  scores: DimensionScores;\n  raw: AgentRunResult;\n  /**\n   * Per-judge raw scores from a multi-judge dispatch (Phase 0 of Run 4).\n   * Keyed by judge model id; value is `null` when that judge failed for this\n   * sample. Absent when no judging was requested or only single-judge mode\n   * was used (existing single-judge behaviour stays in `scores.instruction_following`).\n   */\n  judges_raw?: Record<string, number | null>;\n}\n\nexport async function runOne(input: RunOneInput): Promise<RunOneResult> {\n  const taskBody = input.prompt.sections.Task ?? input.prompt.body;\n  const systemBody = input.prompt.sections[\"System prompt\"] ?? defaultBenchSystem();\n\n  const exec = await input.deps.exec(\"/dev/agent/.actions/run\", {\n    task: taskBody,\n    model: resolveModelPath(input.model, input.hub),\n    tools: defaultBenchTools(),\n    budget: {\n      // Hard cap is 2× the prompt's `max_rounds` so reasoning models that\n      // legitimately need a few extra rounds aren't truncated mid-thought.\n      // Scoring still uses `prompt.max_rounds` (unchanged) for self_stop /\n      // efficiency — a model that finishes after the threshold gets 0 on\n      // those dims, but the sample is captured instead of failing as ERR.\n      // 2026-04-28: Phase 1.7 saw claude-opus-4-7 × 02-ui-control fail all\n      // 3/3 samples at round 8/8; with 2× cap (16) the model finishes and\n      // self_stop = 0 reflects that it didn't stop in time.\n      max_rounds: input.prompt.frontmatter.max_rounds * 2,\n      total_tokens: DEFAULT_TOTAL_TOKENS,\n      wall_time_ms: DEFAULT_WALL_TIME_MS,\n    },\n    system: systemBody,\n  });\n\n  // Budget exhaustion is a graceful \"model couldn't finish in time\" outcome,\n  // not a runner failure: agent-run returns success=false + error.code=\n  // \"BUDGET_EXHAUSTED\" but `data` carries a fully-formed AgentRunResult with\n  // status=\"budget_exhausted\" and the partial trace. Treat it as a 0-score\n  // sample so it counts toward n_samples and pulls the model's average down\n  // (the entire point of the self_stop dimension). Otherwise reasoning\n  // models that legitimately can't stop in time silently disappear from the\n  // affected prompt's per_prompt average and rank artificially high.\n  //\n  // True runner failures (network, dispatch, agent-run crash) keep the\n  // throw-and-record-as-error path so the maintainer can triage them.\n  const isBudgetExhausted =\n    !exec.success &&\n    exec.error?.code === \"BUDGET_EXHAUSTED\" &&\n    exec.data &&\n    typeof exec.data === \"object\" &&\n    (exec.data as Record<string, unknown>).status === \"budget_exhausted\";\n\n  if (!exec.success && !isBudgetExhausted) {\n    // Real agent-run errors live in exec.data.error (status=\"error\" branch);\n    // exec.error.message is for AFS-level dispatch failures. Try data first\n    // so deep failures (e.g. \"agent-run crashed: ...\") aren't masked by the\n    // generic fallback.\n    const dataErr =\n      typeof exec.data === \"object\" && exec.data !== null\n        ? ((exec.data as Record<string, unknown>).error as string | undefined)\n        : undefined;\n    const err = dataErr ?? exec.error?.message ?? \"agent-run dispatch failed\";\n    throw new Error(`[runOne] agent-run failed for model=${input.model}: ${err}`);\n  }\n\n  const result = exec.data as AgentRunResult;\n  return {\n    model: input.model,\n    prompt: input.prompt.id,\n    scores: scoreFromTrace(input.benchPrompt, result),\n    raw: result,\n  };\n}\n\n// ─── prompt → BenchPrompt ─────────────────────────────────────────────────\n\nexport function promptToBenchPrompt(prompt: Prompt): BenchPrompt {\n  return {\n    max_rounds: prompt.frontmatter.max_rounds,\n    expected_paths: extractExpectedPaths(prompt),\n    expected_outcomes: extractExpectedOutcomes(prompt),\n  };\n}\n\n/**\n * Extract the raw `# Expected outcomes` section text. Returns the entire\n * section body (including any markdown table) so the LLM judge has the full\n * rubric. Empty string when the section is missing — the judge still runs\n * but has no rubric to score against; in practice this means a low score.\n */\nexport function extractExpectedOutcomes(prompt: Prompt): string {\n  const key = Object.keys(prompt.sections).find((k) => k.startsWith(\"Expected outcomes\"));\n  if (!key) return \"\";\n  return prompt.sections[key]!.trim();\n}\n\n/**\n * Extract expected_paths from the `# Expected paths` body section's first\n * fenced code block. Strips `// …` line comments. Lines that don't start with\n * `/` are ignored (so prose inside the block doesn't pollute the path set).\n */\nexport function extractExpectedPaths(prompt: Prompt): string[] {\n  const key = Object.keys(prompt.sections).find((k) => k.startsWith(\"Expected paths\"));\n  if (!key) return [];\n  const section = prompt.sections[key]!;\n  const fenced = section.match(/```[^\\n]*\\n([\\s\\S]*?)```/);\n  const block = fenced ? fenced[1]! : section;\n  const out: string[] = [];\n  for (const raw of block.split(/\\r?\\n/)) {\n    const noComment = raw.split(\"//\")[0]!.trim();\n    if (noComment.startsWith(\"/\")) out.push(noComment);\n  }\n  return out;\n}\n\n// ─── Validation ───────────────────────────────────────────────────────────\n\nexport interface ValidatedRunArgs {\n  models: string[];\n  prompts: \"all\" | string[];\n  samples: number;\n  parallel: number;\n  /** Hub for short-model-id expansion. See `RunOneInput.hub`. */\n  hub?: string;\n}\n\nexport function validateRunArgs(args: Record<string, unknown>): ValidatedRunArgs {\n  const models = args.models;\n  if (!Array.isArray(models)) {\n    throw new ValidationError(\"Phase 1: 'models' must be an explicit array (no 'all' shortcut)\");\n  }\n  if (models.length < 1 || models.length > MAX_MODELS) {\n    throw new ValidationError(`Phase 1: 'models' length must be in [1, ${MAX_MODELS}]`);\n  }\n  for (const m of models) {\n    if (typeof m !== \"string\" || m.length === 0) {\n      throw new ValidationError(\"Phase 1: 'models' entries must be non-empty strings\");\n    }\n  }\n\n  const promptsArg = args.prompts;\n  let prompts: \"all\" | string[];\n  if (promptsArg === \"all\") {\n    prompts = \"all\";\n  } else if (Array.isArray(promptsArg)) {\n    if (promptsArg.length < 1 || promptsArg.length > MAX_PROMPTS_PER_RUN) {\n      throw new ValidationError(`Phase 1: 'prompts' length must be in [1, ${MAX_PROMPTS_PER_RUN}]`);\n    }\n    for (const p of promptsArg) {\n      if (typeof p !== \"string\" || p.length === 0) {\n        throw new ValidationError(\"Phase 1: 'prompts' entries must be non-empty strings\");\n      }\n    }\n    prompts = promptsArg as string[];\n  } else {\n    throw new ValidationError(\"Phase 1: 'prompts' must be 'all' or an array of prompt ids\");\n  }\n\n  const samples = clampInt(args.samples, DEFAULT_SAMPLES, 1, MAX_SAMPLES);\n  const parallel = clampInt(args.parallel, 1, 1, MAX_PARALLEL);\n\n  let hub: string | undefined;\n  if (args.hub !== undefined) {\n    if (typeof args.hub !== \"string\" || args.hub.length === 0) {\n      throw new ValidationError(\"'hub' must be a non-empty string\");\n    }\n    if (!/^[A-Za-z0-9._-]+$/.test(args.hub)) {\n      throw new ValidationError(`'hub' has invalid characters: ${args.hub}`);\n    }\n    hub = args.hub;\n  }\n\n  return {\n    models: models as string[],\n    prompts,\n    samples,\n    parallel,\n    ...(hub !== undefined ? { hub } : {}),\n  };\n}\n\nfunction clampInt(raw: unknown, fallback: number, lo: number, hi: number): number {\n  const v = typeof raw === \"number\" && Number.isFinite(raw) ? Math.floor(raw) : fallback;\n  return Math.max(lo, Math.min(hi, v));\n}\n\n// ─── runFull validation (relaxed for full-matrix headless runs) ──────────\n\nexport interface ValidatedRunFullArgs {\n  models: string[];\n  prompts: \"all\" | string[];\n  samples: number;\n  parallel: number;\n  /**\n   * Hub used to expand short model ids — passed through to runOne's\n   * `resolveModelPath`. Falls back to `DEFAULT_HUB` when omitted. Models\n   * given as a full `/dev/ai/hubs/<hub>/models/<id>` path bypass this.\n   */\n  hub?: string;\n  judgeFor?: ReadonlyArray<(typeof ALLOWED_JUDGE_DIMENSIONS)[number]>;\n  /**\n   * Multi-judge ensemble (Phase 0 of Run 4). When present, takes precedence\n   * over `judgeModel` — each judge dispatches its own chat call per sample\n   * and the per-sample scores are aggregated by `aggregateJudgeScores`.\n   * Shares the same `judgeHub` for every judge (default `\"vertex\"`).\n   * Length 1 is allowed and reduces to single-judge passthrough.\n   */\n  judges?: string[];\n  /**\n   * When true, write a sidecar `<stamp>-raw.json` next to the main results\n   * file containing `trace[]` for every successful sample. Default false to\n   * keep the main results file small and avoid leaking trace details into\n   * the dir that maintainers commit.\n   */\n  includeRaw: boolean;\n  /**\n   * Per-dimension stddev cap for outlier flagging (Phase 3 of Run 4 /\n   * plan.md L1070-1072). When `undefined`, the runner uses\n   * `DEFAULT_OUTLIER_THRESHOLD` (0.15). Clamped to `[0.01, 1.0]` so a\n   * misconfig can't disable outlier detection (`0`) or flag every dim (`>1`).\n   */\n  outlierThreshold?: number;\n  /**\n   * When true, after the main matrix completes the runner schedules one\n   * extra batch of `min(N, 3)` samples for each (model, prompt) flagged by\n   * `findOutliers` and recomputes the aggregate + outliers from the merged\n   * samples. Bounded so total samples per (model, prompt) ≤ `2 × N` (no\n   * runaway budget).\n   */\n  outlierRerun: boolean;\n}\n\nexport function validateRunFullArgs(args: Record<string, unknown>): ValidatedRunFullArgs {\n  const models = args.models;\n  if (!Array.isArray(models)) {\n    throw new ValidationError(\"'models' must be an explicit array (no 'all' shortcut)\");\n  }\n  if (models.length < 1 || models.length > MAX_MODELS_FULL) {\n    throw new ValidationError(`'models' length must be in [1, ${MAX_MODELS_FULL}]`);\n  }\n  for (const m of models) {\n    if (typeof m !== \"string\" || m.length === 0) {\n      throw new ValidationError(\"'models' entries must be non-empty strings\");\n    }\n  }\n\n  const promptsArg = args.prompts;\n  let prompts: \"all\" | string[];\n  if (promptsArg === \"all\") {\n    prompts = \"all\";\n  } else if (Array.isArray(promptsArg)) {\n    if (promptsArg.length < 1 || promptsArg.length > MAX_PROMPTS_FULL) {\n      throw new ValidationError(`'prompts' length must be in [1, ${MAX_PROMPTS_FULL}]`);\n    }\n    for (const p of promptsArg) {\n      if (typeof p !== \"string\" || p.length === 0) {\n        throw new ValidationError(\"'prompts' entries must be non-empty strings\");\n      }\n    }\n    prompts = promptsArg as string[];\n  } else {\n    throw new ValidationError(\"'prompts' must be 'all' or an array of prompt ids\");\n  }\n\n  const samples = clampInt(args.samples, DEFAULT_SAMPLES, 1, MAX_SAMPLES);\n  const parallel = clampInt(args.parallel, 1, 1, MAX_PARALLEL);\n\n  let judgeFor: ValidatedRunFullArgs[\"judgeFor\"];\n  if (args.judgeFor !== undefined) {\n    if (!Array.isArray(args.judgeFor)) {\n      throw new ValidationError(\"'judgeFor' must be an array of dimension names\");\n    }\n    const allowed = new Set(ALLOWED_JUDGE_DIMENSIONS as readonly string[]);\n    for (const d of args.judgeFor) {\n      if (typeof d !== \"string\" || !allowed.has(d)) {\n        throw new ValidationError(\n          `'judgeFor' entries must be one of: ${ALLOWED_JUDGE_DIMENSIONS.join(\", \")}`,\n        );\n      }\n    }\n    judgeFor = args.judgeFor as ValidatedRunFullArgs[\"judgeFor\"];\n  }\n\n  let includeRaw = false;\n  if (args.includeRaw !== undefined) {\n    if (typeof args.includeRaw !== \"boolean\") {\n      throw new ValidationError(\"'includeRaw' must be a boolean\");\n    }\n    includeRaw = args.includeRaw;\n  }\n\n  let outlierThreshold: number | undefined;\n  if (args.outlierThreshold !== undefined) {\n    if (typeof args.outlierThreshold !== \"number\" || !Number.isFinite(args.outlierThreshold)) {\n      throw new ValidationError(\"'outlierThreshold' must be a finite number\");\n    }\n    outlierThreshold = Math.max(0.01, Math.min(1.0, args.outlierThreshold));\n  }\n\n  let outlierRerun = false;\n  if (args.outlierRerun !== undefined) {\n    if (typeof args.outlierRerun !== \"boolean\") {\n      throw new ValidationError(\"'outlierRerun' must be a boolean\");\n    }\n    outlierRerun = args.outlierRerun;\n  }\n\n  let judges: string[] | undefined;\n  if (args.judges !== undefined) {\n    if (!Array.isArray(args.judges)) {\n      throw new ValidationError(\"'judges' must be an array of judge model ids\");\n    }\n    if (args.judges.length < 1 || args.judges.length > MAX_JUDGES) {\n      throw new ValidationError(`'judges' length must be in [1, ${MAX_JUDGES}]`);\n    }\n    for (const j of args.judges) {\n      if (typeof j !== \"string\" || j.length === 0) {\n        throw new ValidationError(\"'judges' entries must be non-empty strings\");\n      }\n      if (!JUDGE_ID_PATTERN.test(j)) {\n        throw new ValidationError(`'judges' entry \"${j}\" has invalid characters`);\n      }\n    }\n    judges = args.judges as string[];\n  }\n\n  let hub: string | undefined;\n  if (args.hub !== undefined) {\n    if (typeof args.hub !== \"string\" || args.hub.length === 0) {\n      throw new ValidationError(\"'hub' must be a non-empty string\");\n    }\n    if (!/^[A-Za-z0-9._-]+$/.test(args.hub)) {\n      throw new ValidationError(`'hub' has invalid characters: ${args.hub}`);\n    }\n    hub = args.hub;\n  }\n\n  return {\n    models: models as string[],\n    prompts,\n    samples,\n    parallel,\n    ...(hub !== undefined ? { hub } : {}),\n    judgeFor,\n    judges,\n    includeRaw,\n    outlierThreshold,\n    outlierRerun,\n  };\n}\n\n// ─── Aggregation ──────────────────────────────────────────────────────────\n\nconst DIMENSIONS = OBJECTIVE_DIMENSIONS;\n\nexport interface SampleError {\n  /** Coarse-grained error class for triage. */\n  kind: \"rate_limit\" | \"auth\" | \"other\";\n  /** Path that failed (almost always /dev/agent/.actions/run). */\n  path?: string;\n  message: string;\n  /** Number of attempts made before giving up (1 for non-retryable errors). */\n  attempts: number;\n}\n\nexport interface ModelAggregate {\n  n_samples: number;\n  scores: DimensionScores;\n  stddev: Record<string, number>;\n  /**\n   * Per-prompt mean scores: `{ <prompt_id>: { <dim>: mean across that prompt's\n   * samples } }`. Lets a maintainer audit which prompts dragged the model\n   * average down without re-running the matrix. Keys are sorted alphabetically\n   * for stable diffing across runs. Empty object when there are no samples.\n   */\n  per_prompt: Record<string, DimensionScores>;\n  /**\n   * Per-domain aggregate score (Phase 1 of Run 4 / plan.md L1054-1061).\n   * Single number per domain — the mean of every sample's per-dimension\n   * average for prompts that belong to that domain. Domain is read from the\n   * prompt's `frontmatter.domain` (default `\"english\"` per INTENT). Absent\n   * when the caller doesn't supply the prompt list, so unit tests that only\n   * pass `samples` keep their existing shape.\n   */\n  domains?: Record<string, number>;\n  /**\n   * Multi-judge audit trail (Phase 0 of Run 4). Per-judge mean across all the\n   * model's samples that received that judge's verdict; `null` when no sample\n   * succeeded for that judge. Absent when no multi-judge dispatch happened.\n   * Lets a maintainer detect a single rogue judge dragging the aggregate.\n   */\n  judges_raw?: Record<string, number | null>;\n  /** Per-sample failures; absent when all samples succeed. */\n  errors?: SampleError[];\n}\n\n/**\n * Allowed values for `prompt.frontmatter.domain` (plan.md L1086-1089). Kept\n * here so the runner-side aggregator and the patcher's catalog whitelist stay\n * in sync — adding a new domain requires updating both. Default is `english`\n * for prompts that omit the field.\n */\nconst ALLOWED_DOMAINS = [\"chinese\", \"english\", \"coding\"] as const;\ntype Domain = (typeof ALLOWED_DOMAINS)[number];\n\nfunction promptDomain(prompt: Prompt): Domain {\n  return (prompt.frontmatter.domain ?? \"english\") as Domain;\n}\n\n/** Mean of the populated dimension scores for a single sample. */\nfunction sampleDimensionMean(s: DimensionScores): number {\n  const vals: number[] = DIMENSIONS.map((d) => s[d]);\n  if (typeof s.instruction_following === \"number\" && Number.isFinite(s.instruction_following)) {\n    vals.push(s.instruction_following);\n  }\n  return vals.reduce((a, b) => a + b, 0) / vals.length;\n}\n\n/**\n * Population stddev (divides by N, not N-1) — matches the convention the\n * benchmark spec uses. For N=1 this is 0 by definition; the existing path\n * just returns 0 without dividing. N=0 short-circuits to 0 above the loop.\n */\nexport function meanAndStddev(values: number[]): { mean: number; stddev: number } {\n  if (values.length === 0) return { mean: 0, stddev: 0 };\n  const mean = values.reduce((a, b) => a + b, 0) / values.length;\n  const stddev = Math.sqrt(values.reduce((a, b) => a + (b - mean) ** 2, 0) / values.length);\n  return { mean, stddev };\n}\n\nfunction emptyDimensionScores(): DimensionScores {\n  return {\n    exploration: 0,\n    tool_reliability: 0,\n    self_stop: 0,\n    parallelism: 0,\n    efficiency: 0,\n  };\n}\n\nexport function aggregateAcrossPrompts(\n  samples: RunOneResult[],\n  prompts?: Prompt[],\n): ModelAggregate {\n  const scores = emptyDimensionScores();\n  const stddev: Record<string, number> = { ...scores };\n  if (samples.length === 0) return { n_samples: 0, scores, stddev, per_prompt: {} };\n\n  for (const dim of DIMENSIONS) {\n    const values = samples.map((s) => s.scores[dim]);\n    const { mean, stddev: sd } = meanAndStddev(values);\n    scores[dim] = mean;\n    stddev[dim] = sd;\n  }\n\n  // instruction_following is judged externally and may be missing on a per-\n  // sample basis (judge failure / not requested). Aggregate over the present\n  // values only; if all are missing the dimension stays absent from `scores`.\n  const ifValues: number[] = [];\n  for (const s of samples) {\n    const v = s.scores.instruction_following;\n    if (typeof v === \"number\" && Number.isFinite(v)) ifValues.push(v);\n  }\n  if (ifValues.length > 0) {\n    const { mean, stddev: sd } = meanAndStddev(ifValues);\n    scores.instruction_following = mean;\n    stddev.instruction_following = sd;\n  }\n\n  // Group samples by prompt id, then per-prompt mean per dimension. Sorted\n  // keys keep the on-disk JSON byte-stable across runs that reach the same\n  // (model, prompt) set.\n  const byPrompt = new Map<string, RunOneResult[]>();\n  for (const s of samples) {\n    const arr = byPrompt.get(s.prompt) ?? [];\n    arr.push(s);\n    byPrompt.set(s.prompt, arr);\n  }\n  const per_prompt: Record<string, DimensionScores> = {};\n  for (const promptId of [...byPrompt.keys()].sort()) {\n    const promptSamples = byPrompt.get(promptId)!;\n    const dims = emptyDimensionScores();\n    for (const dim of DIMENSIONS) {\n      dims[dim] = meanAndStddev(promptSamples.map((s) => s.scores[dim])).mean;\n    }\n    const ifPerPrompt: number[] = [];\n    for (const s of promptSamples) {\n      const v = s.scores.instruction_following;\n      if (typeof v === \"number\" && Number.isFinite(v)) ifPerPrompt.push(v);\n    }\n    if (ifPerPrompt.length > 0) {\n      dims.instruction_following = meanAndStddev(ifPerPrompt).mean;\n    }\n    per_prompt[promptId] = dims;\n  }\n\n  // Multi-judge audit trail: per-judge mean across all samples. Map entry\n  // present (even with empty array) means \"judge attempted at least once,\n  // even if every result was null\" — so the catalog still surfaces the\n  // judge id with a null mean rather than silently dropping it.\n  const judgeScores = new Map<string, number[]>();\n  for (const s of samples) {\n    if (!s.judges_raw) continue;\n    for (const [id, score] of Object.entries(s.judges_raw)) {\n      const arr = judgeScores.get(id) ?? [];\n      if (typeof score === \"number\" && Number.isFinite(score)) arr.push(score);\n      judgeScores.set(id, arr);\n    }\n  }\n  const out: ModelAggregate = { n_samples: samples.length, scores, stddev, per_prompt };\n\n  // Domain aggregation (Phase 1 of Run 4): group samples by their prompt's\n  // `frontmatter.domain` and average each sample's per-dimension mean inside\n  // the group. Single number per domain — matches the catalog's\n  // `entry.afs.domains` schema (number per key, not nested per-dim).\n  if (prompts && prompts.length > 0) {\n    const domainOf = new Map<string, Domain>();\n    for (const p of prompts) domainOf.set(p.id, promptDomain(p));\n    const samplesByDomain = new Map<Domain, RunOneResult[]>();\n    for (const s of samples) {\n      const d = domainOf.get(s.prompt) ?? \"english\";\n      const arr = samplesByDomain.get(d) ?? [];\n      arr.push(s);\n      samplesByDomain.set(d, arr);\n    }\n    if (samplesByDomain.size > 0) {\n      const domains: Record<string, number> = {};\n      for (const d of [...samplesByDomain.keys()].sort()) {\n        const ds = samplesByDomain.get(d)!;\n        const means = ds.map((s) => sampleDimensionMean(s.scores));\n        domains[d] = means.reduce((a, b) => a + b, 0) / means.length;\n      }\n      out.domains = domains;\n    }\n  }\n\n  if (judgeScores.size > 0) {\n    const judges_raw: Record<string, number | null> = {};\n    for (const id of [...judgeScores.keys()].sort()) {\n      const vs = judgeScores.get(id)!;\n      judges_raw[id] = vs.length === 0 ? null : meanAndStddev(vs).mean;\n    }\n    out.judges_raw = judges_raw;\n  }\n  return out;\n}\n\n// ─── Matrix runner ────────────────────────────────────────────────────────\n\nexport interface RunMatrixInput {\n  deps: ExecLike;\n  prompts: Prompt[];\n  models: string[];\n  samples: number;\n  parallel: number;\n  resultsDir: string;\n  /** Hub for short-model-id expansion. See `RunOneInput.hub`. */\n  hub?: string;\n}\n\nexport interface ResultsFile {\n  version: string;\n  /**\n   * Judge model id(s) used to score `instruction_following`. Single-judge\n   * runs serialise as a string (back-compat with patcher.ts); multi-judge\n   * (Phase 0 of Run 4) serialises as an array. `null` when no judging happened.\n   */\n  judge_model?: string | string[] | null;\n  config: {\n    samples_per_pair: number;\n    models_tested: string[];\n    prompts_used: string[];\n  };\n  results: Record<string, ModelAggregate>;\n  /**\n   * Per-(model, prompt, dim) tuples whose stddev exceeded the configured\n   * threshold (Phase 3 of Run 4). Always present — empty array when nothing\n   * was flagged. Audit-only field; **not** propagated to the catalog by\n   * `patcher.ts`. See `outlier.ts` for the algorithm.\n   */\n  outliers?: OutlierEntry[];\n}\n\nexport interface RunMatrixResult {\n  results: Record<string, ModelAggregate>;\n  file: string;\n  ranBatch: { models: string[]; prompts: string[]; samples: number };\n}\n\nexport async function runMatrix(input: RunMatrixInput): Promise<RunMatrixResult> {\n  const benchPromptCache = new Map<string, BenchPrompt>();\n  for (const p of input.prompts) benchPromptCache.set(p.id, promptToBenchPrompt(p));\n\n  const results: Record<string, ModelAggregate> = {};\n  for (const model of input.models) {\n    const samples: RunOneResult[] = [];\n    for (const prompt of input.prompts) {\n      // Same-prompt N samples scheduled with the parallel cap. Different\n      // prompts run sequentially per-model (Phase 1 simplicity).\n      const batches = chunk(\n        Array.from({ length: input.samples }, () => prompt),\n        input.parallel,\n      );\n      for (const batch of batches) {\n        const out = await Promise.all(\n          batch.map((p) =>\n            runOne({\n              deps: input.deps,\n              model,\n              hub: input.hub,\n              prompt: p,\n              benchPrompt: benchPromptCache.get(p.id)!,\n            }),\n          ),\n        );\n        samples.push(...out);\n      }\n    }\n    results[model] = aggregateAcrossPrompts(samples, input.prompts);\n  }\n\n  const promptIds = input.prompts.map((p) => p.id);\n  const payload: ResultsFile = {\n    version: new Date().toISOString(),\n    config: {\n      samples_per_pair: input.samples,\n      models_tested: input.models,\n      prompts_used: promptIds,\n    },\n    results,\n  };\n\n  const filename = await writeResultsAtomic(payload, input.resultsDir);\n  return {\n    results,\n    file: filename,\n    ranBatch: { models: input.models, prompts: promptIds, samples: input.samples },\n  };\n}\n\nfunction chunk<T>(arr: T[], size: number): T[][] {\n  const out: T[][] = [];\n  for (let i = 0; i < arr.length; i += size) out.push(arr.slice(i, i + size));\n  return out;\n}\n\n// ─── Full headless matrix runner with incremental flush + retry ──────────\n\nexport interface ProgressEvent {\n  model: string;\n  prompt: string;\n  sampleIndex: number;\n  totalSamples: number;\n  /** \"ok\" if scoring succeeded, \"error\" if all retries exhausted. */\n  status: \"ok\" | \"error\";\n  /** Five-dim summary on success; only `error` on failure. */\n  scores?: DimensionScores;\n  error?: SampleError;\n}\n\nexport interface RunFullInput {\n  deps: ExecLike;\n  prompts: Prompt[];\n  models: string[];\n  samples: number;\n  parallel: number;\n  resultsDir: string;\n  /** Hub for short-model-id expansion. See `RunOneInput.hub`. */\n  hub?: string;\n  /**\n   * Sleep hook for backoff; defaults to setTimeout. Tests inject a no-op so\n   * they don't actually wait 6+ seconds for backoff windows.\n   */\n  sleep?: (ms: number) => Promise<void>;\n  /** Per-sample progress callback (fires after each retry-resolved sample). */\n  onProgress?: (event: ProgressEvent) => void;\n  /** Fires after each (model, prompt) pair flushes to disk. */\n  onPairComplete?: (model: string, promptId: string, partialFile: string) => void | Promise<void>;\n  /** Fires after each model completes (after final pair flush). */\n  onModelComplete?: (model: string, partialFile: string) => void | Promise<void>;\n  /**\n   * Optional dimensions to evaluate via LLM-as-judge. When the array\n   * includes `\"instruction_following\"`, each successful sample is sent to\n   * `/dev/ai/.actions/chat` for scoring before progress is emitted.\n   */\n  judgeFor?: ReadonlyArray<(typeof ALLOWED_JUDGE_DIMENSIONS)[number]>;\n  /**\n   * Optional override for the judge model id. Validated against\n   * `[A-Za-z0-9._-]+` inside `judgeInstructionFollowing`. Defaults to\n   * `claude-opus-4-7` (routed through ai-device). Ignored when `judges`\n   * is set (multi-judge takes precedence; tasks.md L23).\n   */\n  judgeModel?: string;\n  /**\n   * Optional override for the hub the judge model lives on. Defaults to\n   * `aignehub`. Used together with `judgeModel` to drive ai-device's\n   * bypass-mode dispatch.\n   */\n  judgeHub?: string;\n  /**\n   * Multi-judge ensemble (Phase 0 of Run 4). When set with at least one\n   * judge id, this **overrides** `judgeModel` — every successful sample\n   * dispatches one chat call per judge id and `aggregateJudgeScores`\n   * collapses the N scores into a single `instruction_following`. All\n   * judges share the same `judgeHub`.\n   */\n  judges?: string[];\n  /**\n   * When true, write a sidecar `<stamp>-raw.json` next to the main results\n   * file containing the AgentRunResult (trace[] et al) for every successful\n   * sample. Off by default — raw traces can be large and may contain prompt\n   * fragments the maintainer doesn't want to commit.\n   */\n  includeRaw?: boolean;\n  /**\n   * Per-dimension stddev cap for outlier flagging (Phase 3 of Run 4).\n   * Defaults to `DEFAULT_OUTLIER_THRESHOLD` (0.15). Validated/clamped at\n   * the action boundary (`validateRunFullArgs`).\n   */\n  outlierThreshold?: number;\n  /**\n   * When true, after the main matrix the runner schedules one extra batch\n   * of `min(N, 3)` samples per (model, prompt) flagged by `findOutliers`,\n   * appends them to the existing samples, and recomputes the aggregate +\n   * outliers. Bounded so total samples per (model, prompt) ≤ `2 × N`.\n   */\n  outlierRerun?: boolean;\n}\n\n/**\n * Per-sample raw payload kept around when `includeRaw: true`. Indexed\n * `model → prompt → [{ sampleIndex, raw }]` so a maintainer can audit a\n * specific cell of the matrix without scanning the whole sidecar.\n */\nexport interface RawResultsFile {\n  version: string;\n  config: {\n    samples_per_pair: number;\n    models_tested: string[];\n    prompts_used: string[];\n  };\n  raw: Record<string, Record<string, Array<{ sampleIndex: number; raw: AgentRunResult }>>>;\n}\n\nexport interface RunFullResult {\n  results: Record<string, ModelAggregate>;\n  file: string;\n  /** Path to the sidecar raw-trace JSON when `includeRaw: true`; otherwise null. */\n  rawFile: string | null;\n  ranBatch: { models: string[]; prompts: string[]; samples: number };\n}\n\n/**\n * `runMatrixFull` is the headless full-matrix runner used by `/bench/.actions/runFull`.\n *\n * Differences vs `runMatrix`:\n *  - Allows up to 200 models / 200 prompts (vs 5/5 for the fast `run` action).\n *  - Flushes the results JSON to disk after every (model, prompt) pair so a\n *    SIGKILL mid-run preserves the entries already completed.\n *  - Retries rate-limit errors with exponential backoff (2s/4s/8s, max 3\n *    retries). Auth errors (`INVALID_API_KEY`, `UNAUTHENTICATED`, etc.) fail\n *    immediately without retry.\n *  - Records per-sample failures on the model aggregate's `errors[]` instead\n *    of throwing — one bad sample doesn't block the rest of the matrix.\n *  - Emits per-sample progress via `onProgress` so callers can stream live\n *    output (`▸ <model> × <prompt> sample N/M: ...`).\n */\nexport async function runMatrixFull(input: RunFullInput): Promise<RunFullResult> {\n  const benchPromptCache = new Map<string, BenchPrompt>();\n  for (const p of input.prompts) benchPromptCache.set(p.id, promptToBenchPrompt(p));\n\n  const sleep = input.sleep ?? defaultSleep;\n  const promptIds = input.prompts.map((p) => p.id);\n  // Single ISO timestamp for the whole run. The on-disk filename derives from\n  // it (`:` → `-`, `.` → `-`) so each pair flush rewrites the same file.\n  const isoVersion = new Date().toISOString();\n\n  const includeRaw = input.includeRaw ?? false;\n\n  // Resolve effective judge ids. When `judges` is non-empty it wins; otherwise\n  // fall back to single-judge (the existing `judgeModel` / DEFAULT_JUDGE_MODEL\n  // path inside `judgeInstructionFollowing`). Empty effectiveJudges means\n  // single-judge dispatch (existing behaviour).\n  const effectiveJudges: string[] =\n    input.judges && input.judges.length > 0 ? [...input.judges] : [];\n  // What gets written into ResultsFile.judge_model — string when single-judge\n  // active, array when multi, null when no judging requested.\n  const judgeForRequested = input.judgeFor?.includes(\"instruction_following\") ?? false;\n  let resultsFileJudgeModel: string | string[] | null = null;\n  if (judgeForRequested) {\n    if (effectiveJudges.length > 1) resultsFileJudgeModel = [...effectiveJudges];\n    else if (effectiveJudges.length === 1) resultsFileJudgeModel = effectiveJudges[0]!;\n    else resultsFileJudgeModel = input.judgeModel ?? DEFAULT_JUDGE_MODEL;\n  }\n\n  const results: Record<string, ModelAggregate> = {};\n  // Per-model sample buffers retained for the post-loop outlier scan + rerun.\n  // The intra-loop `samples` and `errors` aliases below point at these so the\n  // existing pair-flush paths keep working unchanged.\n  const samplesByModel: Record<string, RunOneResult[]> = {};\n  const errorsByModel: Record<string, SampleError[]> = {};\n  // Held in memory only when includeRaw is true. Indexed\n  // model → prompt → [{ sampleIndex, raw }]. Flushed alongside the main\n  // results file at every (model, prompt) boundary so a SIGKILL leaves a\n  // consistent sidecar pairing.\n  const rawByModel: Record<\n    string,\n    Record<string, Array<{ sampleIndex: number; raw: AgentRunResult }>>\n  > = {};\n  let lastFile = \"\";\n  let lastRawFile: string | null = null;\n\n  const outlierThreshold = input.outlierThreshold ?? DEFAULT_OUTLIER_THRESHOLD;\n  const outlierRerun = input.outlierRerun ?? false;\n\n  // Encapsulate per-batch dispatch (run + retry + judge + record) so the\n  // outlier-rerun pass can reuse the same execution path without copy/pasting\n  // judge + bookkeeping logic.\n  const runSampleBatch = async (\n    model: string,\n    prompt: Prompt,\n    sampleIndices: number[],\n    totalSamples: number,\n  ): Promise<void> => {\n    for (const batch of chunk(sampleIndices, input.parallel)) {\n      const settled = await Promise.all(\n        batch.map(async (sampleIndex) => ({\n          sampleIndex,\n          outcome: await runOneWithRetry({\n            deps: input.deps,\n            model,\n            hub: input.hub,\n            prompt,\n            benchPrompt: benchPromptCache.get(prompt.id)!,\n            sleep,\n          }),\n        })),\n      );\n\n      for (const { sampleIndex, outcome } of settled) {\n        const event: ProgressEvent = {\n          model,\n          prompt: prompt.id,\n          sampleIndex,\n          totalSamples,\n          status: outcome.kind === \"ok\" ? \"ok\" : \"error\",\n        };\n        if (outcome.kind === \"ok\") {\n          if (judgeForRequested) {\n            if (effectiveJudges.length > 0) {\n              const perJudge = await Promise.all(\n                effectiveJudges.map((j) =>\n                  judgeInstructionFollowing(\n                    input.deps,\n                    benchPromptCache.get(prompt.id)!,\n                    outcome.value.raw,\n                    j,\n                    input.judgeHub,\n                  ),\n                ),\n              );\n              const judgesRaw: Record<string, number | null> = {};\n              effectiveJudges.forEach((j, i) => {\n                judgesRaw[j] = perJudge[i]!;\n              });\n              outcome.value.judges_raw = judgesRaw;\n              const aggregate = aggregateJudgeScores(perJudge);\n              if (aggregate !== null) {\n                outcome.value.scores.instruction_following = aggregate;\n              }\n            } else {\n              const score = await judgeInstructionFollowing(\n                input.deps,\n                benchPromptCache.get(prompt.id)!,\n                outcome.value.raw,\n                input.judgeModel,\n                input.judgeHub,\n              );\n              if (score !== null) {\n                outcome.value.scores.instruction_following = score;\n              }\n            }\n          }\n          samplesByModel[model]!.push(outcome.value);\n          event.scores = outcome.value.scores;\n          if (includeRaw) {\n            rawByModel[model]![prompt.id]!.push({ sampleIndex, raw: outcome.value.raw });\n          }\n        } else {\n          errorsByModel[model]!.push(outcome.error);\n          event.error = outcome.error;\n        }\n        input.onProgress?.(event);\n      }\n    }\n  };\n\n  // Build a snapshot ResultsFile from the current per-model state. Used for\n  // every flush so the schema (incl. `outliers`) stays consistent across\n  // intermediate (`outliers: []`) and final (post-detection) writes.\n  const buildResultsFile = (outliers: OutlierEntry[]): ResultsFile => ({\n    version: isoVersion,\n    judge_model: resultsFileJudgeModel,\n    config: {\n      samples_per_pair: input.samples,\n      models_tested: input.models,\n      prompts_used: promptIds,\n    },\n    results,\n    outliers,\n  });\n\n  for (const model of input.models) {\n    samplesByModel[model] = [];\n    errorsByModel[model] = [];\n    if (includeRaw) rawByModel[model] = {};\n\n    for (const prompt of input.prompts) {\n      if (includeRaw) rawByModel[model]![prompt.id] = [];\n      // Same-prompt N samples scheduled with the parallel cap. Different\n      // prompts run sequentially per-model (matches runMatrix semantics).\n      const sampleIndices = Array.from({ length: input.samples }, (_, i) => i + 1);\n      await runSampleBatch(model, prompt, sampleIndices, input.samples);\n\n      // Flush partial results after every (model, prompt) pair so a kill\n      // mid-run preserves what's been completed. Outliers are written as\n      // `[]` here because the scan only runs once at end-of-matrix; an\n      // intermediate flush is correct-but-stale on that field.\n      const agg = aggregateAcrossPrompts(samplesByModel[model]!, input.prompts);\n      const errs = errorsByModel[model]!;\n      results[model] = errs.length > 0 ? { ...agg, errors: errs } : agg;\n      lastFile = await writeResultsAtomic(buildResultsFile([]), input.resultsDir);\n      if (includeRaw) {\n        lastRawFile = await writeRawResultsAtomic(\n          {\n            version: isoVersion,\n            config: {\n              samples_per_pair: input.samples,\n              models_tested: input.models,\n              prompts_used: promptIds,\n            },\n            raw: rawByModel,\n          },\n          input.resultsDir,\n        );\n      }\n      await input.onPairComplete?.(model, prompt.id, lastFile);\n    }\n\n    await input.onModelComplete?.(model, lastFile);\n  }\n\n  // ─── Outlier scan + optional rerun ──────────────────────────────────────\n  let outliers = findOutliers(samplesByModel, outlierThreshold);\n\n  if (outlierRerun && outliers.length > 0) {\n    const promptById = new Map(input.prompts.map((p) => [p.id, p]));\n    // model → set of prompt ids needing a rerun. Nested sets dedupe naturally\n    // when multiple flagged dims point at the same (model, prompt).\n    const rerunByModel = new Map<string, Set<string>>();\n    for (const o of outliers) {\n      if (!promptById.has(o.prompt)) continue;\n      const set = rerunByModel.get(o.model) ?? new Set<string>();\n      set.add(o.prompt);\n      rerunByModel.set(o.model, set);\n    }\n\n    for (const [model, promptIds] of rerunByModel) {\n      // One filter pass per model populates per-prompt counts so we don't\n      // rescan `samplesByModel[model]` once per outlier pair.\n      const countByPrompt = new Map<string, number>();\n      for (const s of samplesByModel[model]!) {\n        countByPrompt.set(s.prompt, (countByPrompt.get(s.prompt) ?? 0) + 1);\n      }\n      for (const promptId of promptIds) {\n        const prompt = promptById.get(promptId)!;\n        const existingForPair = countByPrompt.get(promptId) ?? 0;\n        // Cap so the post-rerun count never exceeds `2 × input.samples`.\n        const extra = Math.min(3, 2 * input.samples - existingForPair);\n        if (extra <= 0) continue;\n        const sampleIndices = Array.from({ length: extra }, (_, i) => existingForPair + i + 1);\n        await runSampleBatch(model, prompt, sampleIndices, existingForPair + extra);\n      }\n    }\n\n    // Recompute affected models' aggregates so on-disk results reflect the\n    // appended samples. Models without any rerun-affected pair stay as-is.\n    for (const model of rerunByModel.keys()) {\n      const agg = aggregateAcrossPrompts(samplesByModel[model]!, input.prompts);\n      const errs = errorsByModel[model]!;\n      results[model] = errs.length > 0 ? { ...agg, errors: errs } : agg;\n    }\n\n    outliers = findOutliers(samplesByModel, outlierThreshold);\n  }\n\n  // Final flush carries the outliers field — earlier pair flushes wrote `[]`.\n  lastFile = await writeResultsAtomic(buildResultsFile(outliers), input.resultsDir);\n  if (includeRaw) {\n    lastRawFile = await writeRawResultsAtomic(\n      {\n        version: isoVersion,\n        config: {\n          samples_per_pair: input.samples,\n          models_tested: input.models,\n          prompts_used: promptIds,\n        },\n        raw: rawByModel,\n      },\n      input.resultsDir,\n    );\n  }\n\n  return {\n    results,\n    file: lastFile,\n    rawFile: lastRawFile,\n    ranBatch: { models: input.models, prompts: promptIds, samples: input.samples },\n  };\n}\n\ninterface RunOneRetryInput {\n  deps: ExecLike;\n  model: string;\n  hub?: string;\n  prompt: Prompt;\n  benchPrompt: BenchPrompt;\n  sleep: (ms: number) => Promise<void>;\n}\n\ntype RetryOutcome = { kind: \"ok\"; value: RunOneResult } | { kind: \"error\"; error: SampleError };\n\n/**\n * Wraps `runOne` with bounded exponential backoff for rate-limit errors and\n * fast-fail for auth errors. Other errors are non-retryable but classified as\n * \"other\" so they still surface in `errors[]`.\n */\nasync function runOneWithRetry(input: RunOneRetryInput): Promise<RetryOutcome> {\n  for (let attempts = 1; ; attempts++) {\n    try {\n      const value = await runOne({\n        deps: input.deps,\n        model: input.model,\n        hub: input.hub,\n        prompt: input.prompt,\n        benchPrompt: input.benchPrompt,\n      });\n      return { kind: \"ok\", value };\n    } catch (err) {\n      const message = err instanceof Error ? err.message : String(err);\n      const cls = classifyError(message);\n      if (cls === \"rate_limit\" && attempts <= RATE_LIMIT_MAX_RETRIES) {\n        // 2s, 4s, 8s — bounded so a stuck model doesn't burn the whole batch.\n        await input.sleep(RATE_LIMIT_BASE_BACKOFF_MS * 2 ** (attempts - 1));\n        continue;\n      }\n      return {\n        kind: \"error\",\n        error: { kind: cls, path: \"/dev/agent/.actions/run\", message, attempts },\n      };\n    }\n  }\n}\n\nconst RATE_LIMIT_PATTERNS = /(rate.*limit|429|too many requests)/i;\nconst AUTH_ERROR_PATTERNS = /(invalid[ _]api[ _]key|unauthenticated|unauthorized)/i;\n\nfunction classifyError(message: string): SampleError[\"kind\"] {\n  if (RATE_LIMIT_PATTERNS.test(message)) return \"rate_limit\";\n  if (AUTH_ERROR_PATTERNS.test(message)) return \"auth\";\n  return \"other\";\n}\n\nfunction defaultSleep(ms: number): Promise<void> {\n  return new Promise((resolve) => setTimeout(resolve, ms));\n}\n\n// ─── Atomic results writer ────────────────────────────────────────────────\n\n/**\n * Write `payload` to `<dir>/<ISO-date>.json` atomically (write to a randomised\n * `.tmp-<id>.json` then rename). Best-effort redaction runs before\n * serialisation: both sensitive labels (api_key / Authorization / …) and\n * common credential VALUE shapes (Bearer tokens, `sk-`/`ya29.` keys, PEM\n * blocks, AWS/Slack ids) are blanked so credentials that leak into agent-run\n * traces are scrubbed from the on-disk record. This is a safety net, not a\n * guarantee — the design keeps credentials out of traces. Aborts (without\n * writing) when\n * the serialised payload would exceed `MAX_RESULTS_SIZE_BYTES` — the failure\n * mode is intentionally noisy so a bug that bloats the results dir never lands\n * on disk silently.\n */\nexport async function writeResultsAtomic(payload: ResultsFile, dir: string): Promise<string> {\n  const safe = sanitiseForResults(payload);\n  const json = `${JSON.stringify(safe, null, 2)}\\n`;\n  const bytes = Buffer.byteLength(json, \"utf8\");\n  if (bytes > MAX_RESULTS_SIZE_BYTES) {\n    throw new ValidationError(\n      `results JSON is ${bytes} bytes — exceeds ${MAX_RESULTS_SIZE_BYTES} byte cap. ` +\n        \"If raw traces are needed pass `includeRaw: true` (writes a separate `<stamp>-raw.json`).\",\n    );\n  }\n  const stamp = payload.version.replace(/[:.]/g, \"-\");\n  return writeAtomic(dir, `${stamp}.json`, json);\n}\n\n/**\n * Write the raw-trace sidecar to `<dir>/<ISO-date>-raw.json` atomically. Uses\n * the same sensitive-string redaction as the main results file. Has no size\n * cap — the maintainer opted in via `includeRaw: true` and a hard fail\n * mid-run would discard the agent traces they explicitly asked to keep.\n */\nexport async function writeRawResultsAtomic(payload: RawResultsFile, dir: string): Promise<string> {\n  const safe = sanitiseForResults(payload);\n  const json = `${JSON.stringify(safe, null, 2)}\\n`;\n  const stamp = payload.version.replace(/[:.]/g, \"-\");\n  return writeAtomic(dir, `${stamp}-raw.json`, json);\n}\n\nasync function writeAtomic(dir: string, filename: string, json: string): Promise<string> {\n  await fs.mkdir(dir, { recursive: true });\n  const finalPath = join(dir, filename);\n  const tmpPath = join(dir, `.tmp-${randomBytes(6).toString(\"hex\")}.json`);\n  await fs.writeFile(tmpPath, json, \"utf8\");\n  try {\n    await fs.rename(tmpPath, finalPath);\n  } catch (err) {\n    // Best-effort cleanup so a failed rename doesn't litter the dir.\n    await fs.rm(tmpPath, { force: true });\n    throw err;\n  }\n  return finalPath;\n}\n\nfunction sanitiseForResults<T>(value: T): T {\n  const json = JSON.stringify(value);\n  let cleaned = json;\n  // Value patterns first so composite `Bearer <token>` shapes redact the token\n  // before the bare `\\bbearer\\b` label rule blanks only the word (issue #830).\n  for (const re of SECRET_VALUE_PATTERNS) cleaned = cleaned.replace(re, \"[REDACTED]\");\n  for (const re of SENSITIVE_PATTERNS) cleaned = cleaned.replace(re, \"[REDACTED]\");\n  return JSON.parse(cleaned) as T;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;AA8BA,MAAM,kBAAkB;AACxB,MAAM,aAAa;AACnB,MAAM,sBAAsB;AAC5B,MAAM,kBAAkB;AACxB,MAAM,mBAAmB;AACzB,MAAM,cAAc;AACpB,MAAM,eAAe;AAQrB,MAAM,uBAAuB;AAC7B,MAAM,uBAAuB,IAAI;;;;;;;;;;AAWjC,MAAM,cAAc;;AAGpB,SAAS,iBAAiB,OAAe,MAAc,aAAqB;AAC1E,KAAI,MAAM,WAAW,IAAI,CAAE,QAAO;AAClC,QAAO,gBAAgB,IAAI,UAAU;;AAEvC,MAAM,yBAAyB;AAC/B,MAAM,6BAA6B;AAMnC,MAAM,yBAAyB,KAAK,OAAO;AAG3C,MAAM,2BAA2B,CAAC,wBAAwB;;;;;;;;AAS1D,MAAM,aAAa;AAInB,MAAM,mBAAmB;AAWzB,MAAM,wBAAkC;CAEtC;CAEA;CAGA;CAEA;CAEA;CAEA;CACD;AAOD,MAAM,qBAA+B;CACnC;CACA;CACA;CACA;CACA;CACA;CACD;;AAWD,SAAgB,oBAAiC;AAC/C,QAAO,CAAC;EAAE,MAAM;EAAO,KAAK;GAAC;GAAQ;GAAQ;GAAW;GAAQ;GAAO;EAAE,UAAU;EAAI,CAAC;;AAG1F,SAAgB,qBAA6B;AAC3C,QAAO;EACL;EACA;EACA;EACA;EACD,CAAC,KAAK,KAAK;;AAwCd,eAAsB,OAAO,OAA2C;CACtE,MAAM,WAAW,MAAM,OAAO,SAAS,QAAQ,MAAM,OAAO;CAC5D,MAAM,aAAa,MAAM,OAAO,SAAS,oBAAoB,oBAAoB;CAEjF,MAAM,OAAO,MAAM,MAAM,KAAK,KAAK,2BAA2B;EAC5D,MAAM;EACN,OAAO,iBAAiB,MAAM,OAAO,MAAM,IAAI;EAC/C,OAAO,mBAAmB;EAC1B,QAAQ;GASN,YAAY,MAAM,OAAO,YAAY,aAAa;GAClD,cAAc;GACd,cAAc;GACf;EACD,QAAQ;EACT,CAAC;CAaF,MAAM,oBACJ,CAAC,KAAK,WACN,KAAK,OAAO,SAAS,sBACrB,KAAK,QACL,OAAO,KAAK,SAAS,YACpB,KAAK,KAAiC,WAAW;AAEpD,KAAI,CAAC,KAAK,WAAW,CAAC,mBAAmB;EASvC,MAAM,OAHJ,OAAO,KAAK,SAAS,YAAY,KAAK,SAAS,OACzC,KAAK,KAAiC,QACxC,WACiB,KAAK,OAAO,WAAW;AAC9C,QAAM,IAAI,MAAM,uCAAuC,MAAM,MAAM,IAAI,MAAM;;CAG/E,MAAM,SAAS,KAAK;AACpB,QAAO;EACL,OAAO,MAAM;EACb,QAAQ,MAAM,OAAO;EACrB,QAAQ,eAAe,MAAM,aAAa,OAAO;EACjD,KAAK;EACN;;AAKH,SAAgB,oBAAoB,QAA6B;AAC/D,QAAO;EACL,YAAY,OAAO,YAAY;EAC/B,gBAAgB,qBAAqB,OAAO;EAC5C,mBAAmB,wBAAwB,OAAO;EACnD;;;;;;;;AASH,SAAgB,wBAAwB,QAAwB;CAC9D,MAAM,MAAM,OAAO,KAAK,OAAO,SAAS,CAAC,MAAM,MAAM,EAAE,WAAW,oBAAoB,CAAC;AACvF,KAAI,CAAC,IAAK,QAAO;AACjB,QAAO,OAAO,SAAS,KAAM,MAAM;;;;;;;AAQrC,SAAgB,qBAAqB,QAA0B;CAC7D,MAAM,MAAM,OAAO,KAAK,OAAO,SAAS,CAAC,MAAM,MAAM,EAAE,WAAW,iBAAiB,CAAC;AACpF,KAAI,CAAC,IAAK,QAAO,EAAE;CACnB,MAAM,UAAU,OAAO,SAAS;CAChC,MAAM,SAAS,QAAQ,MAAM,2BAA2B;CACxD,MAAM,QAAQ,SAAS,OAAO,KAAM;CACpC,MAAM,MAAgB,EAAE;AACxB,MAAK,MAAM,OAAO,MAAM,MAAM,QAAQ,EAAE;EACtC,MAAM,YAAY,IAAI,MAAM,KAAK,CAAC,GAAI,MAAM;AAC5C,MAAI,UAAU,WAAW,IAAI,CAAE,KAAI,KAAK,UAAU;;AAEpD,QAAO;;AAcT,SAAgB,gBAAgB,MAAiD;CAC/E,MAAM,SAAS,KAAK;AACpB,KAAI,CAAC,MAAM,QAAQ,OAAO,CACxB,OAAM,IAAI,gBAAgB,kEAAkE;AAE9F,KAAI,OAAO,SAAS,KAAK,OAAO,SAAS,WACvC,OAAM,IAAI,gBAAgB,2CAA2C,WAAW,GAAG;AAErF,MAAK,MAAM,KAAK,OACd,KAAI,OAAO,MAAM,YAAY,EAAE,WAAW,EACxC,OAAM,IAAI,gBAAgB,sDAAsD;CAIpF,MAAM,aAAa,KAAK;CACxB,IAAI;AACJ,KAAI,eAAe,MACjB,WAAU;UACD,MAAM,QAAQ,WAAW,EAAE;AACpC,MAAI,WAAW,SAAS,KAAK,WAAW,SAAS,oBAC/C,OAAM,IAAI,gBAAgB,4CAA4C,oBAAoB,GAAG;AAE/F,OAAK,MAAM,KAAK,WACd,KAAI,OAAO,MAAM,YAAY,EAAE,WAAW,EACxC,OAAM,IAAI,gBAAgB,uDAAuD;AAGrF,YAAU;OAEV,OAAM,IAAI,gBAAgB,6DAA6D;CAGzF,MAAM,UAAU,SAAS,KAAK,SAAS,iBAAiB,GAAG,YAAY;CACvE,MAAM,WAAW,SAAS,KAAK,UAAU,GAAG,GAAG,aAAa;CAE5D,IAAI;AACJ,KAAI,KAAK,QAAQ,QAAW;AAC1B,MAAI,OAAO,KAAK,QAAQ,YAAY,KAAK,IAAI,WAAW,EACtD,OAAM,IAAI,gBAAgB,mCAAmC;AAE/D,MAAI,CAAC,oBAAoB,KAAK,KAAK,IAAI,CACrC,OAAM,IAAI,gBAAgB,iCAAiC,KAAK,MAAM;AAExE,QAAM,KAAK;;AAGb,QAAO;EACG;EACR;EACA;EACA;EACA,GAAI,QAAQ,SAAY,EAAE,KAAK,GAAG,EAAE;EACrC;;AAGH,SAAS,SAAS,KAAc,UAAkB,IAAY,IAAoB;CAChF,MAAM,IAAI,OAAO,QAAQ,YAAY,OAAO,SAAS,IAAI,GAAG,KAAK,MAAM,IAAI,GAAG;AAC9E,QAAO,KAAK,IAAI,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;;AAiDtC,SAAgB,oBAAoB,MAAqD;CACvF,MAAM,SAAS,KAAK;AACpB,KAAI,CAAC,MAAM,QAAQ,OAAO,CACxB,OAAM,IAAI,gBAAgB,yDAAyD;AAErF,KAAI,OAAO,SAAS,KAAK,OAAO,SAAS,gBACvC,OAAM,IAAI,gBAAgB,kCAAkC,gBAAgB,GAAG;AAEjF,MAAK,MAAM,KAAK,OACd,KAAI,OAAO,MAAM,YAAY,EAAE,WAAW,EACxC,OAAM,IAAI,gBAAgB,6CAA6C;CAI3E,MAAM,aAAa,KAAK;CACxB,IAAI;AACJ,KAAI,eAAe,MACjB,WAAU;UACD,MAAM,QAAQ,WAAW,EAAE;AACpC,MAAI,WAAW,SAAS,KAAK,WAAW,SAAS,iBAC/C,OAAM,IAAI,gBAAgB,mCAAmC,iBAAiB,GAAG;AAEnF,OAAK,MAAM,KAAK,WACd,KAAI,OAAO,MAAM,YAAY,EAAE,WAAW,EACxC,OAAM,IAAI,gBAAgB,8CAA8C;AAG5E,YAAU;OAEV,OAAM,IAAI,gBAAgB,oDAAoD;CAGhF,MAAM,UAAU,SAAS,KAAK,SAAS,iBAAiB,GAAG,YAAY;CACvE,MAAM,WAAW,SAAS,KAAK,UAAU,GAAG,GAAG,aAAa;CAE5D,IAAI;AACJ,KAAI,KAAK,aAAa,QAAW;AAC/B,MAAI,CAAC,MAAM,QAAQ,KAAK,SAAS,CAC/B,OAAM,IAAI,gBAAgB,iDAAiD;EAE7E,MAAM,UAAU,IAAI,IAAI,yBAA8C;AACtE,OAAK,MAAM,KAAK,KAAK,SACnB,KAAI,OAAO,MAAM,YAAY,CAAC,QAAQ,IAAI,EAAE,CAC1C,OAAM,IAAI,gBACR,sCAAsC,yBAAyB,KAAK,KAAK,GAC1E;AAGL,aAAW,KAAK;;CAGlB,IAAI,aAAa;AACjB,KAAI,KAAK,eAAe,QAAW;AACjC,MAAI,OAAO,KAAK,eAAe,UAC7B,OAAM,IAAI,gBAAgB,iCAAiC;AAE7D,eAAa,KAAK;;CAGpB,IAAI;AACJ,KAAI,KAAK,qBAAqB,QAAW;AACvC,MAAI,OAAO,KAAK,qBAAqB,YAAY,CAAC,OAAO,SAAS,KAAK,iBAAiB,CACtF,OAAM,IAAI,gBAAgB,6CAA6C;AAEzE,qBAAmB,KAAK,IAAI,KAAM,KAAK,IAAI,GAAK,KAAK,iBAAiB,CAAC;;CAGzE,IAAI,eAAe;AACnB,KAAI,KAAK,iBAAiB,QAAW;AACnC,MAAI,OAAO,KAAK,iBAAiB,UAC/B,OAAM,IAAI,gBAAgB,mCAAmC;AAE/D,iBAAe,KAAK;;CAGtB,IAAI;AACJ,KAAI,KAAK,WAAW,QAAW;AAC7B,MAAI,CAAC,MAAM,QAAQ,KAAK,OAAO,CAC7B,OAAM,IAAI,gBAAgB,+CAA+C;AAE3E,MAAI,KAAK,OAAO,SAAS,KAAK,KAAK,OAAO,SAAS,WACjD,OAAM,IAAI,gBAAgB,kCAAkC,WAAW,GAAG;AAE5E,OAAK,MAAM,KAAK,KAAK,QAAQ;AAC3B,OAAI,OAAO,MAAM,YAAY,EAAE,WAAW,EACxC,OAAM,IAAI,gBAAgB,6CAA6C;AAEzE,OAAI,CAAC,iBAAiB,KAAK,EAAE,CAC3B,OAAM,IAAI,gBAAgB,mBAAmB,EAAE,0BAA0B;;AAG7E,WAAS,KAAK;;CAGhB,IAAI;AACJ,KAAI,KAAK,QAAQ,QAAW;AAC1B,MAAI,OAAO,KAAK,QAAQ,YAAY,KAAK,IAAI,WAAW,EACtD,OAAM,IAAI,gBAAgB,mCAAmC;AAE/D,MAAI,CAAC,oBAAoB,KAAK,KAAK,IAAI,CACrC,OAAM,IAAI,gBAAgB,iCAAiC,KAAK,MAAM;AAExE,QAAM,KAAK;;AAGb,QAAO;EACG;EACR;EACA;EACA;EACA,GAAI,QAAQ,SAAY,EAAE,KAAK,GAAG,EAAE;EACpC;EACA;EACA;EACA;EACA;EACD;;AAKH,MAAM,aAAa;AAoDnB,SAAS,aAAa,QAAwB;AAC5C,QAAQ,OAAO,YAAY,UAAU;;;AAIvC,SAAS,oBAAoB,GAA4B;CACvD,MAAM,OAAiB,WAAW,KAAK,MAAM,EAAE,GAAG;AAClD,KAAI,OAAO,EAAE,0BAA0B,YAAY,OAAO,SAAS,EAAE,sBAAsB,CACzF,MAAK,KAAK,EAAE,sBAAsB;AAEpC,QAAO,KAAK,QAAQ,GAAG,MAAM,IAAI,GAAG,EAAE,GAAG,KAAK;;;;;;;AAQhD,SAAgB,cAAc,QAAoD;AAChF,KAAI,OAAO,WAAW,EAAG,QAAO;EAAE,MAAM;EAAG,QAAQ;EAAG;CACtD,MAAM,OAAO,OAAO,QAAQ,GAAG,MAAM,IAAI,GAAG,EAAE,GAAG,OAAO;AAExD,QAAO;EAAE;EAAM,QADA,KAAK,KAAK,OAAO,QAAQ,GAAG,MAAM,KAAK,IAAI,SAAS,GAAG,EAAE,GAAG,OAAO,OAAO;EAClE;;AAGzB,SAAS,uBAAwC;AAC/C,QAAO;EACL,aAAa;EACb,kBAAkB;EAClB,WAAW;EACX,aAAa;EACb,YAAY;EACb;;AAGH,SAAgB,uBACd,SACA,SACgB;CAChB,MAAM,SAAS,sBAAsB;CACrC,MAAM,SAAiC,EAAE,GAAG,QAAQ;AACpD,KAAI,QAAQ,WAAW,EAAG,QAAO;EAAE,WAAW;EAAG;EAAQ;EAAQ,YAAY,EAAE;EAAE;AAEjF,MAAK,MAAM,OAAO,YAAY;EAE5B,MAAM,EAAE,MAAM,QAAQ,OAAO,cADd,QAAQ,KAAK,MAAM,EAAE,OAAO,KAAK,CACE;AAClD,SAAO,OAAO;AACd,SAAO,OAAO;;CAMhB,MAAM,WAAqB,EAAE;AAC7B,MAAK,MAAM,KAAK,SAAS;EACvB,MAAM,IAAI,EAAE,OAAO;AACnB,MAAI,OAAO,MAAM,YAAY,OAAO,SAAS,EAAE,CAAE,UAAS,KAAK,EAAE;;AAEnE,KAAI,SAAS,SAAS,GAAG;EACvB,MAAM,EAAE,MAAM,QAAQ,OAAO,cAAc,SAAS;AACpD,SAAO,wBAAwB;AAC/B,SAAO,wBAAwB;;CAMjC,MAAM,2BAAW,IAAI,KAA6B;AAClD,MAAK,MAAM,KAAK,SAAS;EACvB,MAAM,MAAM,SAAS,IAAI,EAAE,OAAO,IAAI,EAAE;AACxC,MAAI,KAAK,EAAE;AACX,WAAS,IAAI,EAAE,QAAQ,IAAI;;CAE7B,MAAM,aAA8C,EAAE;AACtD,MAAK,MAAM,YAAY,CAAC,GAAG,SAAS,MAAM,CAAC,CAAC,MAAM,EAAE;EAClD,MAAM,gBAAgB,SAAS,IAAI,SAAS;EAC5C,MAAM,OAAO,sBAAsB;AACnC,OAAK,MAAM,OAAO,WAChB,MAAK,OAAO,cAAc,cAAc,KAAK,MAAM,EAAE,OAAO,KAAK,CAAC,CAAC;EAErE,MAAM,cAAwB,EAAE;AAChC,OAAK,MAAM,KAAK,eAAe;GAC7B,MAAM,IAAI,EAAE,OAAO;AACnB,OAAI,OAAO,MAAM,YAAY,OAAO,SAAS,EAAE,CAAE,aAAY,KAAK,EAAE;;AAEtE,MAAI,YAAY,SAAS,EACvB,MAAK,wBAAwB,cAAc,YAAY,CAAC;AAE1D,aAAW,YAAY;;CAOzB,MAAM,8BAAc,IAAI,KAAuB;AAC/C,MAAK,MAAM,KAAK,SAAS;AACvB,MAAI,CAAC,EAAE,WAAY;AACnB,OAAK,MAAM,CAAC,IAAI,UAAU,OAAO,QAAQ,EAAE,WAAW,EAAE;GACtD,MAAM,MAAM,YAAY,IAAI,GAAG,IAAI,EAAE;AACrC,OAAI,OAAO,UAAU,YAAY,OAAO,SAAS,MAAM,CAAE,KAAI,KAAK,MAAM;AACxE,eAAY,IAAI,IAAI,IAAI;;;CAG5B,MAAM,MAAsB;EAAE,WAAW,QAAQ;EAAQ;EAAQ;EAAQ;EAAY;AAMrF,KAAI,WAAW,QAAQ,SAAS,GAAG;EACjC,MAAM,2BAAW,IAAI,KAAqB;AAC1C,OAAK,MAAM,KAAK,QAAS,UAAS,IAAI,EAAE,IAAI,aAAa,EAAE,CAAC;EAC5D,MAAM,kCAAkB,IAAI,KAA6B;AACzD,OAAK,MAAM,KAAK,SAAS;GACvB,MAAM,IAAI,SAAS,IAAI,EAAE,OAAO,IAAI;GACpC,MAAM,MAAM,gBAAgB,IAAI,EAAE,IAAI,EAAE;AACxC,OAAI,KAAK,EAAE;AACX,mBAAgB,IAAI,GAAG,IAAI;;AAE7B,MAAI,gBAAgB,OAAO,GAAG;GAC5B,MAAM,UAAkC,EAAE;AAC1C,QAAK,MAAM,KAAK,CAAC,GAAG,gBAAgB,MAAM,CAAC,CAAC,MAAM,EAAE;IAElD,MAAM,QADK,gBAAgB,IAAI,EAAE,CAChB,KAAK,MAAM,oBAAoB,EAAE,OAAO,CAAC;AAC1D,YAAQ,KAAK,MAAM,QAAQ,GAAG,MAAM,IAAI,GAAG,EAAE,GAAG,MAAM;;AAExD,OAAI,UAAU;;;AAIlB,KAAI,YAAY,OAAO,GAAG;EACxB,MAAM,aAA4C,EAAE;AACpD,OAAK,MAAM,MAAM,CAAC,GAAG,YAAY,MAAM,CAAC,CAAC,MAAM,EAAE;GAC/C,MAAM,KAAK,YAAY,IAAI,GAAG;AAC9B,cAAW,MAAM,GAAG,WAAW,IAAI,OAAO,cAAc,GAAG,CAAC;;AAE9D,MAAI,aAAa;;AAEnB,QAAO;;AA6CT,eAAsB,UAAU,OAAiD;CAC/E,MAAM,mCAAmB,IAAI,KAA0B;AACvD,MAAK,MAAM,KAAK,MAAM,QAAS,kBAAiB,IAAI,EAAE,IAAI,oBAAoB,EAAE,CAAC;CAEjF,MAAM,UAA0C,EAAE;AAClD,MAAK,MAAM,SAAS,MAAM,QAAQ;EAChC,MAAM,UAA0B,EAAE;AAClC,OAAK,MAAM,UAAU,MAAM,SAAS;GAGlC,MAAM,UAAU,MACd,MAAM,KAAK,EAAE,QAAQ,MAAM,SAAS,QAAQ,OAAO,EACnD,MAAM,SACP;AACD,QAAK,MAAM,SAAS,SAAS;IAC3B,MAAM,MAAM,MAAM,QAAQ,IACxB,MAAM,KAAK,MACT,OAAO;KACL,MAAM,MAAM;KACZ;KACA,KAAK,MAAM;KACX,QAAQ;KACR,aAAa,iBAAiB,IAAI,EAAE,GAAG;KACxC,CAAC,CACH,CACF;AACD,YAAQ,KAAK,GAAG,IAAI;;;AAGxB,UAAQ,SAAS,uBAAuB,SAAS,MAAM,QAAQ;;CAGjE,MAAM,YAAY,MAAM,QAAQ,KAAK,MAAM,EAAE,GAAG;AAYhD,QAAO;EACL;EACA,MAHe,MAAM,mBAVM;GAC3B,0BAAS,IAAI,MAAM,EAAC,aAAa;GACjC,QAAQ;IACN,kBAAkB,MAAM;IACxB,eAAe,MAAM;IACrB,cAAc;IACf;GACD;GACD,EAEkD,MAAM,WAAW;EAIlE,UAAU;GAAE,QAAQ,MAAM;GAAQ,SAAS;GAAW,SAAS,MAAM;GAAS;EAC/E;;AAGH,SAAS,MAAS,KAAU,MAAqB;CAC/C,MAAM,MAAa,EAAE;AACrB,MAAK,IAAI,IAAI,GAAG,IAAI,IAAI,QAAQ,KAAK,KAAM,KAAI,KAAK,IAAI,MAAM,GAAG,IAAI,KAAK,CAAC;AAC3E,QAAO;;;;;;;;;;;;;;;;;AA4HT,eAAsB,cAAc,OAA6C;CAC/E,MAAM,mCAAmB,IAAI,KAA0B;AACvD,MAAK,MAAM,KAAK,MAAM,QAAS,kBAAiB,IAAI,EAAE,IAAI,oBAAoB,EAAE,CAAC;CAEjF,MAAM,QAAQ,MAAM,SAAS;CAC7B,MAAM,YAAY,MAAM,QAAQ,KAAK,MAAM,EAAE,GAAG;CAGhD,MAAM,8BAAa,IAAI,MAAM,EAAC,aAAa;CAE3C,MAAM,aAAa,MAAM,cAAc;CAMvC,MAAM,kBACJ,MAAM,UAAU,MAAM,OAAO,SAAS,IAAI,CAAC,GAAG,MAAM,OAAO,GAAG,EAAE;CAGlE,MAAM,oBAAoB,MAAM,UAAU,SAAS,wBAAwB,IAAI;CAC/E,IAAI,wBAAkD;AACtD,KAAI,kBACF,KAAI,gBAAgB,SAAS,EAAG,yBAAwB,CAAC,GAAG,gBAAgB;UACnE,gBAAgB,WAAW,EAAG,yBAAwB,gBAAgB;KAC1E,yBAAwB,MAAM,cAAc;CAGnD,MAAM,UAA0C,EAAE;CAIlD,MAAM,iBAAiD,EAAE;CACzD,MAAM,gBAA+C,EAAE;CAKvD,MAAM,aAGF,EAAE;CACN,IAAI,WAAW;CACf,IAAI,cAA6B;CAEjC,MAAM,mBAAmB,MAAM,oBAAoB;CACnD,MAAM,eAAe,MAAM,gBAAgB;CAK3C,MAAM,iBAAiB,OACrB,OACA,QACA,eACA,iBACkB;AAClB,OAAK,MAAM,SAAS,MAAM,eAAe,MAAM,SAAS,EAAE;GACxD,MAAM,UAAU,MAAM,QAAQ,IAC5B,MAAM,IAAI,OAAO,iBAAiB;IAChC;IACA,SAAS,MAAM,gBAAgB;KAC7B,MAAM,MAAM;KACZ;KACA,KAAK,MAAM;KACX;KACA,aAAa,iBAAiB,IAAI,OAAO,GAAG;KAC5C;KACD,CAAC;IACH,EAAE,CACJ;AAED,QAAK,MAAM,EAAE,aAAa,aAAa,SAAS;IAC9C,MAAM,QAAuB;KAC3B;KACA,QAAQ,OAAO;KACf;KACA;KACA,QAAQ,QAAQ,SAAS,OAAO,OAAO;KACxC;AACD,QAAI,QAAQ,SAAS,MAAM;AACzB,SAAI,kBACF,KAAI,gBAAgB,SAAS,GAAG;MAC9B,MAAM,WAAW,MAAM,QAAQ,IAC7B,gBAAgB,KAAK,MACnB,0BACE,MAAM,MACN,iBAAiB,IAAI,OAAO,GAAG,EAC/B,QAAQ,MAAM,KACd,GACA,MAAM,SACP,CACF,CACF;MACD,MAAM,YAA2C,EAAE;AACnD,sBAAgB,SAAS,GAAG,MAAM;AAChC,iBAAU,KAAK,SAAS;QACxB;AACF,cAAQ,MAAM,aAAa;MAC3B,MAAM,YAAY,qBAAqB,SAAS;AAChD,UAAI,cAAc,KAChB,SAAQ,MAAM,OAAO,wBAAwB;YAE1C;MACL,MAAM,QAAQ,MAAM,0BAClB,MAAM,MACN,iBAAiB,IAAI,OAAO,GAAG,EAC/B,QAAQ,MAAM,KACd,MAAM,YACN,MAAM,SACP;AACD,UAAI,UAAU,KACZ,SAAQ,MAAM,OAAO,wBAAwB;;AAInD,oBAAe,OAAQ,KAAK,QAAQ,MAAM;AAC1C,WAAM,SAAS,QAAQ,MAAM;AAC7B,SAAI,WACF,YAAW,OAAQ,OAAO,IAAK,KAAK;MAAE;MAAa,KAAK,QAAQ,MAAM;MAAK,CAAC;WAEzE;AACL,mBAAc,OAAQ,KAAK,QAAQ,MAAM;AACzC,WAAM,QAAQ,QAAQ;;AAExB,UAAM,aAAa,MAAM;;;;CAQ/B,MAAM,oBAAoB,gBAA2C;EACnE,SAAS;EACT,aAAa;EACb,QAAQ;GACN,kBAAkB,MAAM;GACxB,eAAe,MAAM;GACrB,cAAc;GACf;EACD;EACA;EACD;AAED,MAAK,MAAM,SAAS,MAAM,QAAQ;AAChC,iBAAe,SAAS,EAAE;AAC1B,gBAAc,SAAS,EAAE;AACzB,MAAI,WAAY,YAAW,SAAS,EAAE;AAEtC,OAAK,MAAM,UAAU,MAAM,SAAS;AAClC,OAAI,WAAY,YAAW,OAAQ,OAAO,MAAM,EAAE;AAIlD,SAAM,eAAe,OAAO,QADN,MAAM,KAAK,EAAE,QAAQ,MAAM,SAAS,GAAG,GAAG,MAAM,IAAI,EAAE,EACzB,MAAM,QAAQ;GAMjE,MAAM,MAAM,uBAAuB,eAAe,QAAS,MAAM,QAAQ;GACzE,MAAM,OAAO,cAAc;AAC3B,WAAQ,SAAS,KAAK,SAAS,IAAI;IAAE,GAAG;IAAK,QAAQ;IAAM,GAAG;AAC9D,cAAW,MAAM,mBAAmB,iBAAiB,EAAE,CAAC,EAAE,MAAM,WAAW;AAC3E,OAAI,WACF,eAAc,MAAM,sBAClB;IACE,SAAS;IACT,QAAQ;KACN,kBAAkB,MAAM;KACxB,eAAe,MAAM;KACrB,cAAc;KACf;IACD,KAAK;IACN,EACD,MAAM,WACP;AAEH,SAAM,MAAM,iBAAiB,OAAO,OAAO,IAAI,SAAS;;AAG1D,QAAM,MAAM,kBAAkB,OAAO,SAAS;;CAIhD,IAAI,WAAW,aAAa,gBAAgB,iBAAiB;AAE7D,KAAI,gBAAgB,SAAS,SAAS,GAAG;EACvC,MAAM,aAAa,IAAI,IAAI,MAAM,QAAQ,KAAK,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;EAG/D,MAAM,+BAAe,IAAI,KAA0B;AACnD,OAAK,MAAM,KAAK,UAAU;AACxB,OAAI,CAAC,WAAW,IAAI,EAAE,OAAO,CAAE;GAC/B,MAAM,MAAM,aAAa,IAAI,EAAE,MAAM,oBAAI,IAAI,KAAa;AAC1D,OAAI,IAAI,EAAE,OAAO;AACjB,gBAAa,IAAI,EAAE,OAAO,IAAI;;AAGhC,OAAK,MAAM,CAAC,OAAOA,gBAAc,cAAc;GAG7C,MAAM,gCAAgB,IAAI,KAAqB;AAC/C,QAAK,MAAM,KAAK,eAAe,OAC7B,eAAc,IAAI,EAAE,SAAS,cAAc,IAAI,EAAE,OAAO,IAAI,KAAK,EAAE;AAErE,QAAK,MAAM,YAAYA,aAAW;IAChC,MAAM,SAAS,WAAW,IAAI,SAAS;IACvC,MAAM,kBAAkB,cAAc,IAAI,SAAS,IAAI;IAEvD,MAAM,QAAQ,KAAK,IAAI,GAAG,IAAI,MAAM,UAAU,gBAAgB;AAC9D,QAAI,SAAS,EAAG;AAEhB,UAAM,eAAe,OAAO,QADN,MAAM,KAAK,EAAE,QAAQ,OAAO,GAAG,GAAG,MAAM,kBAAkB,IAAI,EAAE,EACnC,kBAAkB,MAAM;;;AAM/E,OAAK,MAAM,SAAS,aAAa,MAAM,EAAE;GACvC,MAAM,MAAM,uBAAuB,eAAe,QAAS,MAAM,QAAQ;GACzE,MAAM,OAAO,cAAc;AAC3B,WAAQ,SAAS,KAAK,SAAS,IAAI;IAAE,GAAG;IAAK,QAAQ;IAAM,GAAG;;AAGhE,aAAW,aAAa,gBAAgB,iBAAiB;;AAI3D,YAAW,MAAM,mBAAmB,iBAAiB,SAAS,EAAE,MAAM,WAAW;AACjF,KAAI,WACF,eAAc,MAAM,sBAClB;EACE,SAAS;EACT,QAAQ;GACN,kBAAkB,MAAM;GACxB,eAAe,MAAM;GACrB,cAAc;GACf;EACD,KAAK;EACN,EACD,MAAM,WACP;AAGH,QAAO;EACL;EACA,MAAM;EACN,SAAS;EACT,UAAU;GAAE,QAAQ,MAAM;GAAQ,SAAS;GAAW,SAAS,MAAM;GAAS;EAC/E;;;;;;;AAmBH,eAAe,gBAAgB,OAAgD;AAC7E,MAAK,IAAI,WAAW,IAAK,WACvB,KAAI;AAQF,SAAO;GAAE,MAAM;GAAM,OAPP,MAAM,OAAO;IACzB,MAAM,MAAM;IACZ,OAAO,MAAM;IACb,KAAK,MAAM;IACX,QAAQ,MAAM;IACd,aAAa,MAAM;IACpB,CAAC;GAC0B;UACrB,KAAK;EACZ,MAAM,UAAU,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI;EAChE,MAAM,MAAM,cAAc,QAAQ;AAClC,MAAI,QAAQ,gBAAgB,YAAY,wBAAwB;AAE9D,SAAM,MAAM,MAAM,6BAA6B,MAAM,WAAW,GAAG;AACnE;;AAEF,SAAO;GACL,MAAM;GACN,OAAO;IAAE,MAAM;IAAK,MAAM;IAA2B;IAAS;IAAU;GACzE;;;AAKP,MAAM,sBAAsB;AAC5B,MAAM,sBAAsB;AAE5B,SAAS,cAAc,SAAsC;AAC3D,KAAI,oBAAoB,KAAK,QAAQ,CAAE,QAAO;AAC9C,KAAI,oBAAoB,KAAK,QAAQ,CAAE,QAAO;AAC9C,QAAO;;AAGT,SAAS,aAAa,IAA2B;AAC/C,QAAO,IAAI,SAAS,YAAY,WAAW,SAAS,GAAG,CAAC;;;;;;;;;;;;;;;AAkB1D,eAAsB,mBAAmB,SAAsB,KAA8B;CAC3F,MAAM,OAAO,mBAAmB,QAAQ;CACxC,MAAM,OAAO,GAAG,KAAK,UAAU,MAAM,MAAM,EAAE,CAAC;CAC9C,MAAM,QAAQ,OAAO,WAAW,MAAM,OAAO;AAC7C,KAAI,QAAQ,uBACV,OAAM,IAAI,gBACR,mBAAmB,MAAM,mBAAmB,uBAAuB,yGAEpE;AAGH,QAAO,YAAY,KAAK,GADV,QAAQ,QAAQ,QAAQ,SAAS,IAAI,CAClB,QAAQ,KAAK;;;;;;;;AAShD,eAAsB,sBAAsB,SAAyB,KAA8B;CACjG,MAAM,OAAO,mBAAmB,QAAQ;CACxC,MAAM,OAAO,GAAG,KAAK,UAAU,MAAM,MAAM,EAAE,CAAC;AAE9C,QAAO,YAAY,KAAK,GADV,QAAQ,QAAQ,QAAQ,SAAS,IAAI,CAClB,YAAY,KAAK;;AAGpD,eAAe,YAAY,KAAa,UAAkB,MAA+B;AACvF,OAAMC,SAAG,MAAM,KAAK,EAAE,WAAW,MAAM,CAAC;CACxC,MAAM,YAAY,KAAK,KAAK,SAAS;CACrC,MAAM,UAAU,KAAK,KAAK,QAAQ,YAAY,EAAE,CAAC,SAAS,MAAM,CAAC,OAAO;AACxE,OAAMA,SAAG,UAAU,SAAS,MAAM,OAAO;AACzC,KAAI;AACF,QAAMA,SAAG,OAAO,SAAS,UAAU;UAC5B,KAAK;AAEZ,QAAMA,SAAG,GAAG,SAAS,EAAE,OAAO,MAAM,CAAC;AACrC,QAAM;;AAER,QAAO;;AAGT,SAAS,mBAAsB,OAAa;CAE1C,IAAI,UADS,KAAK,UAAU,MAAM;AAIlC,MAAK,MAAM,MAAM,sBAAuB,WAAU,QAAQ,QAAQ,IAAI,aAAa;AACnF,MAAK,MAAM,MAAM,mBAAoB,WAAU,QAAQ,QAAQ,IAAI,aAAa;AAChF,QAAO,KAAK,MAAM,QAAQ"}

package/dist/scoring.cjs

@@ -0,0 +1,47 @@
+
+//#region src/scoring.ts
+/**
+* Objective dimensions scored from `AgentRunResult` alone — does NOT include
+* `instruction_following`, which is judge-fed and may be absent on a per-sample
+* basis. Aggregators (`runner.aggregateAcrossPrompts`, `outlier.findOutliers`)
+* iterate this list and handle IF as a separate, optional case.
+*/
+const OBJECTIVE_DIMENSIONS = [
+	"exploration",
+	"tool_reliability",
+	"self_stop",
+	"parallelism",
+	"efficiency"
+];
+function clamp(value, min, max) {
+	if (value < min) return min;
+	if (value > max) return max;
+	return value;
+}
+function completedCurve(rounds, maxRounds) {
+	if (maxRounds <= 0) return rounds === 0 ? 1 : 0;
+	return clamp(1 - rounds / maxRounds, 0, 1);
+}
+function scoreFromTrace(prompt, result) {
+	const allCalls = (result.trace ?? []).flatMap((r) => r.tool_calls);
+	const completed = result.status === "completed";
+	const expected = new Set(prompt.expected_paths);
+	let matched = 0;
+	const visited = /* @__PURE__ */ new Set();
+	for (const c of allCalls) if (c.path) visited.add(c.path);
+	for (const p of expected) if (visited.has(p)) matched++;
+	const exploration = expected.size === 0 ? 0 : matched / expected.size;
+	const total = allCalls.length;
+	const failed = allCalls.filter((c) => c.status !== "ok").length;
+	return {
+		exploration,
+		tool_reliability: total === 0 ? 0 : 1 - failed / total,
+		self_stop: completed ? completedCurve(result.rounds, prompt.max_rounds) : 0,
+		parallelism: result.rounds === 0 ? 0 : clamp((total - result.rounds) / (result.rounds * 3), 0, 1),
+		efficiency: completed ? completedCurve(result.rounds, prompt.max_rounds) : 0
+	};
+}
+
+//#endregion
+exports.OBJECTIVE_DIMENSIONS = OBJECTIVE_DIMENSIONS;
+exports.scoreFromTrace = scoreFromTrace;

package/dist/scoring.d.cts

@@ -0,0 +1,28 @@
+import { AgentRunResult } from "@aigne/afs-agent";
+
+//#region src/scoring.d.ts
+interface BenchPrompt {
+  /** Round budget the runner gave to agent-run (drives self_stop & efficiency). */
+  max_rounds: number;
+  /** Canonical AFS paths the prompt expects the agent to visit at least once. */
+  expected_paths: string[];
+  /**
+   * Raw text of the prompt's `# Expected outcomes` section, passed verbatim
+   * to the LLM judge for `instruction_following`. Empty string when the
+   * prompt has no rubric section. Set by `runner.promptToBenchPrompt`.
+   */
+  expected_outcomes?: string;
+}
+interface DimensionScores {
+  exploration: number;
+  tool_reliability: number;
+  self_stop: number;
+  parallelism: number;
+  efficiency: number;
+  /** Phase 1 always omits this; Phase 2 fills it via LLM judge. */
+  instruction_following?: number;
+}
+declare function scoreFromTrace(prompt: BenchPrompt, result: AgentRunResult): DimensionScores;
+//#endregion
+export { BenchPrompt, DimensionScores, scoreFromTrace };
+//# sourceMappingURL=scoring.d.cts.map

package/dist/scoring.d.cts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scoring.d.cts","names":[],"sources":["../src/scoring.ts"],"mappings":";;;UAQiB,WAAA;EAEf;EAAA,UAAA;EAQA;EANA,cAAA;EAMiB;AAGnB;;;;EAHE,iBAAA;AAAA;AAAA,UAGe,eAAA;EACf,WAAA;EACA,gBAAA;EACA,SAAA;EACA,WAAA;EACA,UAAA;EAgCc;EA9Bd,qBAAA;AAAA;AAAA,iBA8Bc,cAAA,CAAe,MAAA,EAAQ,WAAA,EAAa,MAAA,EAAQ,cAAA,GAAiB,eAAA"}

package/dist/scoring.d.mts

@@ -0,0 +1,28 @@
+import { AgentRunResult } from "@aigne/afs-agent";
+
+//#region src/scoring.d.ts
+interface BenchPrompt {
+  /** Round budget the runner gave to agent-run (drives self_stop & efficiency). */
+  max_rounds: number;
+  /** Canonical AFS paths the prompt expects the agent to visit at least once. */
+  expected_paths: string[];
+  /**
+   * Raw text of the prompt's `# Expected outcomes` section, passed verbatim
+   * to the LLM judge for `instruction_following`. Empty string when the
+   * prompt has no rubric section. Set by `runner.promptToBenchPrompt`.
+   */
+  expected_outcomes?: string;
+}
+interface DimensionScores {
+  exploration: number;
+  tool_reliability: number;
+  self_stop: number;
+  parallelism: number;
+  efficiency: number;
+  /** Phase 1 always omits this; Phase 2 fills it via LLM judge. */
+  instruction_following?: number;
+}
+declare function scoreFromTrace(prompt: BenchPrompt, result: AgentRunResult): DimensionScores;
+//#endregion
+export { BenchPrompt, DimensionScores, scoreFromTrace };
+//# sourceMappingURL=scoring.d.mts.map

package/dist/scoring.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scoring.d.mts","names":[],"sources":["../src/scoring.ts"],"mappings":";;;UAQiB,WAAA;EAEf;EAAA,UAAA;EAQA;EANA,cAAA;EAMiB;AAGnB;;;;EAHE,iBAAA;AAAA;AAAA,UAGe,eAAA;EACf,WAAA;EACA,gBAAA;EACA,SAAA;EACA,WAAA;EACA,UAAA;EAgCc;EA9Bd,qBAAA;AAAA;AAAA,iBA8Bc,cAAA,CAAe,MAAA,EAAQ,WAAA,EAAa,MAAA,EAAQ,cAAA,GAAiB,eAAA"}

package/dist/scoring.mjs

@@ -0,0 +1,46 @@
+//#region src/scoring.ts
+/**
+* Objective dimensions scored from `AgentRunResult` alone — does NOT include
+* `instruction_following`, which is judge-fed and may be absent on a per-sample
+* basis. Aggregators (`runner.aggregateAcrossPrompts`, `outlier.findOutliers`)
+* iterate this list and handle IF as a separate, optional case.
+*/
+const OBJECTIVE_DIMENSIONS = [
+	"exploration",
+	"tool_reliability",
+	"self_stop",
+	"parallelism",
+	"efficiency"
+];
+function clamp(value, min, max) {
+	if (value < min) return min;
+	if (value > max) return max;
+	return value;
+}
+function completedCurve(rounds, maxRounds) {
+	if (maxRounds <= 0) return rounds === 0 ? 1 : 0;
+	return clamp(1 - rounds / maxRounds, 0, 1);
+}
+function scoreFromTrace(prompt, result) {
+	const allCalls = (result.trace ?? []).flatMap((r) => r.tool_calls);
+	const completed = result.status === "completed";
+	const expected = new Set(prompt.expected_paths);
+	let matched = 0;
+	const visited = /* @__PURE__ */ new Set();
+	for (const c of allCalls) if (c.path) visited.add(c.path);
+	for (const p of expected) if (visited.has(p)) matched++;
+	const exploration = expected.size === 0 ? 0 : matched / expected.size;
+	const total = allCalls.length;
+	const failed = allCalls.filter((c) => c.status !== "ok").length;
+	return {
+		exploration,
+		tool_reliability: total === 0 ? 0 : 1 - failed / total,
+		self_stop: completed ? completedCurve(result.rounds, prompt.max_rounds) : 0,
+		parallelism: result.rounds === 0 ? 0 : clamp((total - result.rounds) / (result.rounds * 3), 0, 1),
+		efficiency: completed ? completedCurve(result.rounds, prompt.max_rounds) : 0
+	};
+}
+
+//#endregion
+export { OBJECTIVE_DIMENSIONS, scoreFromTrace };
+//# sourceMappingURL=scoring.mjs.map

package/dist/scoring.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"scoring.mjs","names":[],"sources":["../src/scoring.ts"],"sourcesContent":["/**\n * Phase 1 — pure scoring of the 5 objective AFS dimensions from an\n * `AgentRunResult`. `instruction_following` is left undefined here; Phase 2\n * fills it via an LLM judge against `expected_outcomes`. See plan.md L212-293.\n */\n\nimport type { AgentRunResult } from \"@aigne/afs-agent\";\n\nexport interface BenchPrompt {\n  /** Round budget the runner gave to agent-run (drives self_stop & efficiency). */\n  max_rounds: number;\n  /** Canonical AFS paths the prompt expects the agent to visit at least once. */\n  expected_paths: string[];\n  /**\n   * Raw text of the prompt's `# Expected outcomes` section, passed verbatim\n   * to the LLM judge for `instruction_following`. Empty string when the\n   * prompt has no rubric section. Set by `runner.promptToBenchPrompt`.\n   */\n  expected_outcomes?: string;\n}\n\nexport interface DimensionScores {\n  exploration: number;\n  tool_reliability: number;\n  self_stop: number;\n  parallelism: number;\n  efficiency: number;\n  /** Phase 1 always omits this; Phase 2 fills it via LLM judge. */\n  instruction_following?: number;\n}\n\n/**\n * Objective dimensions scored from `AgentRunResult` alone — does NOT include\n * `instruction_following`, which is judge-fed and may be absent on a per-sample\n * basis. Aggregators (`runner.aggregateAcrossPrompts`, `outlier.findOutliers`)\n * iterate this list and handle IF as a separate, optional case.\n */\nexport const OBJECTIVE_DIMENSIONS = [\n  \"exploration\",\n  \"tool_reliability\",\n  \"self_stop\",\n  \"parallelism\",\n  \"efficiency\",\n] as const satisfies ReadonlyArray<keyof DimensionScores>;\n\nfunction clamp(value: number, min: number, max: number): number {\n  if (value < min) return min;\n  if (value > max) return max;\n  return value;\n}\n\n// `1 - rounds/max_rounds` clamped, with a maxRounds<=0 short-circuit so the\n// curve still has a defined limit (rounds=0 → 1, rounds>0 → 0) without dividing.\nfunction completedCurve(rounds: number, maxRounds: number): number {\n  if (maxRounds <= 0) return rounds === 0 ? 1 : 0;\n  return clamp(1 - rounds / maxRounds, 0, 1);\n}\n\nexport function scoreFromTrace(prompt: BenchPrompt, result: AgentRunResult): DimensionScores {\n  const allCalls = (result.trace ?? []).flatMap((r) => r.tool_calls);\n  const completed = result.status === \"completed\";\n\n  // Empty `path` strings are dropped — they signal a tool call that doesn't\n  // address an AFS path (e.g. .actions/x without a target).\n  const expected = new Set(prompt.expected_paths);\n  let matched = 0;\n  const visited = new Set<string>();\n  for (const c of allCalls) if (c.path) visited.add(c.path);\n  for (const p of expected) if (visited.has(p)) matched++;\n  const exploration = expected.size === 0 ? 0 : matched / expected.size;\n\n  // `circuit_breaker` counts as failed (the breaker tripped before the tool\n  // ran). 0 calls → 0: no signal means we can't claim \"reliable\".\n  const total = allCalls.length;\n  const failed = allCalls.filter((c) => c.status !== \"ok\").length;\n  const tool_reliability = total === 0 ? 0 : 1 - failed / total;\n\n  const self_stop = completed ? completedCurve(result.rounds, prompt.max_rounds) : 0;\n\n  // 1 call/round (sequential) → 0; ≥4 calls/round on average → 1.\n  const parallelism =\n    result.rounds === 0 ? 0 : clamp((total - result.rounds) / (result.rounds * 3), 0, 1);\n\n  // Rounds-normalized so models with larger context windows (bigger system\n  // prompt / tool schema) aren't penalized for token volume.\n  const efficiency = completed ? completedCurve(result.rounds, prompt.max_rounds) : 0;\n\n  return { exploration, tool_reliability, self_stop, parallelism, efficiency };\n}\n"],"mappings":";;;;;;;AAqCA,MAAa,uBAAuB;CAClC;CACA;CACA;CACA;CACA;CACD;AAED,SAAS,MAAM,OAAe,KAAa,KAAqB;AAC9D,KAAI,QAAQ,IAAK,QAAO;AACxB,KAAI,QAAQ,IAAK,QAAO;AACxB,QAAO;;AAKT,SAAS,eAAe,QAAgB,WAA2B;AACjE,KAAI,aAAa,EAAG,QAAO,WAAW,IAAI,IAAI;AAC9C,QAAO,MAAM,IAAI,SAAS,WAAW,GAAG,EAAE;;AAG5C,SAAgB,eAAe,QAAqB,QAAyC;CAC3F,MAAM,YAAY,OAAO,SAAS,EAAE,EAAE,SAAS,MAAM,EAAE,WAAW;CAClE,MAAM,YAAY,OAAO,WAAW;CAIpC,MAAM,WAAW,IAAI,IAAI,OAAO,eAAe;CAC/C,IAAI,UAAU;CACd,MAAM,0BAAU,IAAI,KAAa;AACjC,MAAK,MAAM,KAAK,SAAU,KAAI,EAAE,KAAM,SAAQ,IAAI,EAAE,KAAK;AACzD,MAAK,MAAM,KAAK,SAAU,KAAI,QAAQ,IAAI,EAAE,CAAE;CAC9C,MAAM,cAAc,SAAS,SAAS,IAAI,IAAI,UAAU,SAAS;CAIjE,MAAM,QAAQ,SAAS;CACvB,MAAM,SAAS,SAAS,QAAQ,MAAM,EAAE,WAAW,KAAK,CAAC;AAazD,QAAO;EAAE;EAAa,kBAZG,UAAU,IAAI,IAAI,IAAI,SAAS;EAYhB,WAVtB,YAAY,eAAe,OAAO,QAAQ,OAAO,WAAW,GAAG;EAU9B,aANjD,OAAO,WAAW,IAAI,IAAI,OAAO,QAAQ,OAAO,WAAW,OAAO,SAAS,IAAI,GAAG,EAAE;EAMtB,YAF7C,YAAY,eAAe,OAAO,QAAQ,OAAO,WAAW,GAAG;EAEN"}

package/manifest.json

@@ -0,0 +1,39 @@
+{
+  "name": "llm-bench",
+  "description": "LLM AFS benchmark suite — score model AFS-friendliness, patch ai-device catalog.\n- list /prompts to see available prompts\n- list /results to see prior runs\n- exec /.actions/run to score a (models × prompts) matrix (Phase 1)\n- exec /.actions/patchCatalog to merge a result file into ai-device catalog (Phase 1)",
+  "uriTemplate": "llm-bench://",
+  "category": "ai",
+  "schema": {
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "type": "object",
+    "properties": {
+      "name": {
+        "type": "string",
+        "description": "Mount slug — last segment of the mount path (defaults to \"llm-bench\")."
+      },
+      "description": {
+        "type": "string",
+        "description": "Optional human-readable description shown in registry listings."
+      },
+      "promptsDir": {
+        "type": "string",
+        "description": "Absolute path to the prompt set on disk. Defaults to repo's `benchmarks/llm-afs/prompts`."
+      },
+      "resultsDir": {
+        "type": "string",
+        "description": "Absolute path to the results directory. Defaults to repo's `benchmarks/llm-afs/results`."
+      }
+    },
+    "additionalProperties": false
+  },
+  "tags": ["ai", "benchmark", "llm", "evaluation", "catalog"],
+  "capabilityTags": ["read-only", "auth:none", "local"],
+  "security": {
+    "riskLevel": "local",
+    "resourceAccess": ["local-filesystem"],
+    "dataSensitivity": [],
+    "notes": [
+      "Phase 0 is scaffold-only: list/read of prompts + results, NOT_IMPLEMENTED on actions. Phase 1 will run benchmarks and patch the ai-device catalog (maintainer tool boundary)."
+    ]
+  }
+}

package/package.json

@@ -0,0 +1,62 @@
+{
+  "name": "@aigne/afs-llm-bench",
+  "version": "1.12.0-beta.5",
+  "description": "AFS provider for the LLM-AFS benchmark suite — runs AFS-friendliness evals and patches scores into the ai-device catalog.",
+  "license": "UNLICENSED",
+  "publishConfig": {
+    "access": "public"
+  },
+  "author": "Arcblock <blocklet@arcblock.io> https://github.com/arcblock",
+  "homepage": "https://github.com/arcblock/arc",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/arcblock/arc"
+  },
+  "bugs": {
+    "url": "https://github.com/arcblock/arc/issues"
+  },
+  "type": "module",
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.mjs",
+  "types": "./dist/index.d.cts",
+  "exports": {
+    ".": {
+      "require": "./dist/index.cjs",
+      "import": "./dist/index.mjs"
+    },
+    "./*": "./*"
+  },
+  "files": [
+    "dist",
+    "LICENSE",
+    "README.md",
+    "CHANGELOG.md",
+    "manifest.json"
+  ],
+  "dependencies": {
+    "gray-matter": "^4.0.3",
+    "js-yaml": "^4.1.1",
+    "ufo": "^1.6.3",
+    "zod": "^4.0.0",
+    "@aigne/afs": "^1.12.0-beta.5",
+    "@aigne/afs-agent": "^1.11.0-beta.13"
+  },
+  "devDependencies": {
+    "@types/bun": "^1.3.6",
+    "@types/js-yaml": "^4.0.9",
+    "npm-run-all": "^4.1.5",
+    "rimraf": "^6.1.2",
+    "tsdown": "0.20.0-beta.3",
+    "typescript": "5.9.2",
+    "@aigne/afs-ai-device": "1.12.0-beta.5",
+    "@aigne/afs-testing": "1.12.0-beta.5",
+    "@aigne/typescript-config": "0.0.0"
+  },
+  "scripts": {
+    "build": "tsdown",
+    "check-types": "tsc --noEmit",
+    "clean": "rimraf dist coverage",
+    "test": "bun test",
+    "test:coverage": "bun test --coverage --coverage-reporter=lcov --coverage-reporter=text"
+  }
+}