@aigne/afs-llm-bench 1.12.0-beta.5

package/dist/runner.mjs

@@ -0,0 +1,697 @@
+import { ValidationError } from "./errors.mjs";
+import { DEFAULT_JUDGE_MODEL, aggregateJudgeScores, judgeInstructionFollowing } from "./judge.mjs";
+import { OBJECTIVE_DIMENSIONS, scoreFromTrace } from "./scoring.mjs";
+import { DEFAULT_OUTLIER_THRESHOLD, findOutliers } from "./outlier.mjs";
+import { promises } from "node:fs";
+import { randomBytes } from "node:crypto";
+import { join } from "node:path";
+
+//#region src/runner.ts
+/**
+* Phase 1 — runOne + execRun matrix + results writer.
+*
+* Dispatches a single (model, prompt) sample through `/dev/agent/.actions/run`,
+* scores the trace via `scoring.ts`, and writes per-model aggregates to a JSON
+* file. Credentials are NOT held by bench — the caller forwards a model id and
+* ai-device routing injects vault secrets at the LLM call site.
+*
+* See plan.md L294-378 (impl) + L745-752 (test spec).
+*/
+const DEFAULT_SAMPLES = 3;
+const MAX_MODELS = 5;
+const MAX_PROMPTS_PER_RUN = 5;
+const MAX_MODELS_FULL = 200;
+const MAX_PROMPTS_FULL = 200;
+const MAX_SAMPLES = 100;
+const MAX_PARALLEL = 4;
+const DEFAULT_TOTAL_TOKENS = 24e4;
+const DEFAULT_WALL_TIME_MS = 5 * 6e4;
+/**
+* Hub used to expand short model ids (e.g. `"claude-haiku-4-5"`) into a full
+* AFS path before dispatching to `/dev/agent/.actions/run`. agent-run's own
+* `resolveAgentModelPath` defaults to AigneHub, which requires credits — we
+* default to Vertex so OAuth-backed local installs work without billing.
+*
+* Override via `runFull` arg `hub` (or by passing the full
+* `/dev/ai/hubs/<hub>/models/<id>` path as the model id).
+*/
+const DEFAULT_HUB = "vertex";
+/** Expand a short model id to its full hub path. Full paths pass through. */
+function resolveModelPath(model, hub = DEFAULT_HUB) {
+	if (model.startsWith("/")) return model;
+	return `/dev/ai/hubs/${hub}/models/${model}`;
+}
+const RATE_LIMIT_MAX_RETRIES = 3;
+const RATE_LIMIT_BASE_BACKOFF_MS = 2e3;
+const MAX_RESULTS_SIZE_BYTES = 10 * 1024 * 1024;
+const ALLOWED_JUDGE_DIMENSIONS = ["instruction_following"];
+/**
+* Hard cap on multi-judge fanout (Phase 0 of Run 4). Each judge is one extra
+* `/dev/ai/.actions/chat` call per successful sample, so a 200×200 matrix
+* with 100 samples explodes quickly. 10 is generous enough for ensemble
+* experiments and small enough that runaway configs surface as validation
+* errors rather than silently burning cost.
+*/
+const MAX_JUDGES = 10;
+const JUDGE_ID_PATTERN = /^[A-Za-z0-9._-]+$/;
+const SECRET_VALUE_PATTERNS = [
+	/-----BEGIN [A-Z0-9 ]*?PRIVATE KEY-----[\s\S]*?-----END [A-Z0-9 ]*?PRIVATE KEY-----/g,
+	/\bBearer\s+[A-Za-z0-9._~+/=-]+/gi,
+	/\bsk-[A-Za-z0-9_-]*[0-9][A-Za-z0-9_-]{6,}/g,
+	/\bya29\.[A-Za-z0-9._-]+/g,
+	/\bxox[baprs]-[A-Za-z0-9-]+/gi,
+	/\bAKIA[0-9A-Z]{16}\b/g
+];
+const SENSITIVE_PATTERNS = [
+	/\bapi[_-]?key\b/gi,
+	/\bauthorization\b/gi,
+	/\bbearer\b/gi,
+	/\bservice[_-]?account\b/gi,
+	/\bprivate[_-]?key\b/gi,
+	/\bAKIA[0-9A-Z]{0,16}\b/g
+];
+/** Permissive read-only AFS sandbox so benchmark prompts can explore mounts. */
+function defaultBenchTools() {
+	return [{
+		path: "/**",
+		ops: [
+			"list",
+			"read",
+			"explain",
+			"stat",
+			"exec"
+		],
+		maxDepth: 10
+	}];
+}
+function defaultBenchSystem() {
+	return [
+		"You are an AFS power user evaluating mounts and actions.",
+		"Tools: afs_list, afs_read, afs_explain, afs_stat, afs_exec.",
+		"Prefer parallel tool calls when listing or explaining sibling paths.",
+		"Stop and summarize as soon as you have enough context to answer."
+	].join("\n");
+}
+async function runOne(input) {
+	const taskBody = input.prompt.sections.Task ?? input.prompt.body;
+	const systemBody = input.prompt.sections["System prompt"] ?? defaultBenchSystem();
+	const exec = await input.deps.exec("/dev/agent/.actions/run", {
+		task: taskBody,
+		model: resolveModelPath(input.model, input.hub),
+		tools: defaultBenchTools(),
+		budget: {
+			max_rounds: input.prompt.frontmatter.max_rounds * 2,
+			total_tokens: DEFAULT_TOTAL_TOKENS,
+			wall_time_ms: DEFAULT_WALL_TIME_MS
+		},
+		system: systemBody
+	});
+	const isBudgetExhausted = !exec.success && exec.error?.code === "BUDGET_EXHAUSTED" && exec.data && typeof exec.data === "object" && exec.data.status === "budget_exhausted";
+	if (!exec.success && !isBudgetExhausted) {
+		const err = (typeof exec.data === "object" && exec.data !== null ? exec.data.error : void 0) ?? exec.error?.message ?? "agent-run dispatch failed";
+		throw new Error(`[runOne] agent-run failed for model=${input.model}: ${err}`);
+	}
+	const result = exec.data;
+	return {
+		model: input.model,
+		prompt: input.prompt.id,
+		scores: scoreFromTrace(input.benchPrompt, result),
+		raw: result
+	};
+}
+function promptToBenchPrompt(prompt) {
+	return {
+		max_rounds: prompt.frontmatter.max_rounds,
+		expected_paths: extractExpectedPaths(prompt),
+		expected_outcomes: extractExpectedOutcomes(prompt)
+	};
+}
+/**
+* Extract the raw `# Expected outcomes` section text. Returns the entire
+* section body (including any markdown table) so the LLM judge has the full
+* rubric. Empty string when the section is missing — the judge still runs
+* but has no rubric to score against; in practice this means a low score.
+*/
+function extractExpectedOutcomes(prompt) {
+	const key = Object.keys(prompt.sections).find((k) => k.startsWith("Expected outcomes"));
+	if (!key) return "";
+	return prompt.sections[key].trim();
+}
+/**
+* Extract expected_paths from the `# Expected paths` body section's first
+* fenced code block. Strips `// …` line comments. Lines that don't start with
+* `/` are ignored (so prose inside the block doesn't pollute the path set).
+*/
+function extractExpectedPaths(prompt) {
+	const key = Object.keys(prompt.sections).find((k) => k.startsWith("Expected paths"));
+	if (!key) return [];
+	const section = prompt.sections[key];
+	const fenced = section.match(/```[^\n]*\n([\s\S]*?)```/);
+	const block = fenced ? fenced[1] : section;
+	const out = [];
+	for (const raw of block.split(/\r?\n/)) {
+		const noComment = raw.split("//")[0].trim();
+		if (noComment.startsWith("/")) out.push(noComment);
+	}
+	return out;
+}
+function validateRunArgs(args) {
+	const models = args.models;
+	if (!Array.isArray(models)) throw new ValidationError("Phase 1: 'models' must be an explicit array (no 'all' shortcut)");
+	if (models.length < 1 || models.length > MAX_MODELS) throw new ValidationError(`Phase 1: 'models' length must be in [1, ${MAX_MODELS}]`);
+	for (const m of models) if (typeof m !== "string" || m.length === 0) throw new ValidationError("Phase 1: 'models' entries must be non-empty strings");
+	const promptsArg = args.prompts;
+	let prompts;
+	if (promptsArg === "all") prompts = "all";
+	else if (Array.isArray(promptsArg)) {
+		if (promptsArg.length < 1 || promptsArg.length > MAX_PROMPTS_PER_RUN) throw new ValidationError(`Phase 1: 'prompts' length must be in [1, ${MAX_PROMPTS_PER_RUN}]`);
+		for (const p of promptsArg) if (typeof p !== "string" || p.length === 0) throw new ValidationError("Phase 1: 'prompts' entries must be non-empty strings");
+		prompts = promptsArg;
+	} else throw new ValidationError("Phase 1: 'prompts' must be 'all' or an array of prompt ids");
+	const samples = clampInt(args.samples, DEFAULT_SAMPLES, 1, MAX_SAMPLES);
+	const parallel = clampInt(args.parallel, 1, 1, MAX_PARALLEL);
+	let hub;
+	if (args.hub !== void 0) {
+		if (typeof args.hub !== "string" || args.hub.length === 0) throw new ValidationError("'hub' must be a non-empty string");
+		if (!/^[A-Za-z0-9._-]+$/.test(args.hub)) throw new ValidationError(`'hub' has invalid characters: ${args.hub}`);
+		hub = args.hub;
+	}
+	return {
+		models,
+		prompts,
+		samples,
+		parallel,
+		...hub !== void 0 ? { hub } : {}
+	};
+}
+function clampInt(raw, fallback, lo, hi) {
+	const v = typeof raw === "number" && Number.isFinite(raw) ? Math.floor(raw) : fallback;
+	return Math.max(lo, Math.min(hi, v));
+}
+function validateRunFullArgs(args) {
+	const models = args.models;
+	if (!Array.isArray(models)) throw new ValidationError("'models' must be an explicit array (no 'all' shortcut)");
+	if (models.length < 1 || models.length > MAX_MODELS_FULL) throw new ValidationError(`'models' length must be in [1, ${MAX_MODELS_FULL}]`);
+	for (const m of models) if (typeof m !== "string" || m.length === 0) throw new ValidationError("'models' entries must be non-empty strings");
+	const promptsArg = args.prompts;
+	let prompts;
+	if (promptsArg === "all") prompts = "all";
+	else if (Array.isArray(promptsArg)) {
+		if (promptsArg.length < 1 || promptsArg.length > MAX_PROMPTS_FULL) throw new ValidationError(`'prompts' length must be in [1, ${MAX_PROMPTS_FULL}]`);
+		for (const p of promptsArg) if (typeof p !== "string" || p.length === 0) throw new ValidationError("'prompts' entries must be non-empty strings");
+		prompts = promptsArg;
+	} else throw new ValidationError("'prompts' must be 'all' or an array of prompt ids");
+	const samples = clampInt(args.samples, DEFAULT_SAMPLES, 1, MAX_SAMPLES);
+	const parallel = clampInt(args.parallel, 1, 1, MAX_PARALLEL);
+	let judgeFor;
+	if (args.judgeFor !== void 0) {
+		if (!Array.isArray(args.judgeFor)) throw new ValidationError("'judgeFor' must be an array of dimension names");
+		const allowed = new Set(ALLOWED_JUDGE_DIMENSIONS);
+		for (const d of args.judgeFor) if (typeof d !== "string" || !allowed.has(d)) throw new ValidationError(`'judgeFor' entries must be one of: ${ALLOWED_JUDGE_DIMENSIONS.join(", ")}`);
+		judgeFor = args.judgeFor;
+	}
+	let includeRaw = false;
+	if (args.includeRaw !== void 0) {
+		if (typeof args.includeRaw !== "boolean") throw new ValidationError("'includeRaw' must be a boolean");
+		includeRaw = args.includeRaw;
+	}
+	let outlierThreshold;
+	if (args.outlierThreshold !== void 0) {
+		if (typeof args.outlierThreshold !== "number" || !Number.isFinite(args.outlierThreshold)) throw new ValidationError("'outlierThreshold' must be a finite number");
+		outlierThreshold = Math.max(.01, Math.min(1, args.outlierThreshold));
+	}
+	let outlierRerun = false;
+	if (args.outlierRerun !== void 0) {
+		if (typeof args.outlierRerun !== "boolean") throw new ValidationError("'outlierRerun' must be a boolean");
+		outlierRerun = args.outlierRerun;
+	}
+	let judges;
+	if (args.judges !== void 0) {
+		if (!Array.isArray(args.judges)) throw new ValidationError("'judges' must be an array of judge model ids");
+		if (args.judges.length < 1 || args.judges.length > MAX_JUDGES) throw new ValidationError(`'judges' length must be in [1, ${MAX_JUDGES}]`);
+		for (const j of args.judges) {
+			if (typeof j !== "string" || j.length === 0) throw new ValidationError("'judges' entries must be non-empty strings");
+			if (!JUDGE_ID_PATTERN.test(j)) throw new ValidationError(`'judges' entry "${j}" has invalid characters`);
+		}
+		judges = args.judges;
+	}
+	let hub;
+	if (args.hub !== void 0) {
+		if (typeof args.hub !== "string" || args.hub.length === 0) throw new ValidationError("'hub' must be a non-empty string");
+		if (!/^[A-Za-z0-9._-]+$/.test(args.hub)) throw new ValidationError(`'hub' has invalid characters: ${args.hub}`);
+		hub = args.hub;
+	}
+	return {
+		models,
+		prompts,
+		samples,
+		parallel,
+		...hub !== void 0 ? { hub } : {},
+		judgeFor,
+		judges,
+		includeRaw,
+		outlierThreshold,
+		outlierRerun
+	};
+}
+const DIMENSIONS = OBJECTIVE_DIMENSIONS;
+function promptDomain(prompt) {
+	return prompt.frontmatter.domain ?? "english";
+}
+/** Mean of the populated dimension scores for a single sample. */
+function sampleDimensionMean(s) {
+	const vals = DIMENSIONS.map((d) => s[d]);
+	if (typeof s.instruction_following === "number" && Number.isFinite(s.instruction_following)) vals.push(s.instruction_following);
+	return vals.reduce((a, b) => a + b, 0) / vals.length;
+}
+/**
+* Population stddev (divides by N, not N-1) — matches the convention the
+* benchmark spec uses. For N=1 this is 0 by definition; the existing path
+* just returns 0 without dividing. N=0 short-circuits to 0 above the loop.
+*/
+function meanAndStddev(values) {
+	if (values.length === 0) return {
+		mean: 0,
+		stddev: 0
+	};
+	const mean = values.reduce((a, b) => a + b, 0) / values.length;
+	return {
+		mean,
+		stddev: Math.sqrt(values.reduce((a, b) => a + (b - mean) ** 2, 0) / values.length)
+	};
+}
+function emptyDimensionScores() {
+	return {
+		exploration: 0,
+		tool_reliability: 0,
+		self_stop: 0,
+		parallelism: 0,
+		efficiency: 0
+	};
+}
+function aggregateAcrossPrompts(samples, prompts) {
+	const scores = emptyDimensionScores();
+	const stddev = { ...scores };
+	if (samples.length === 0) return {
+		n_samples: 0,
+		scores,
+		stddev,
+		per_prompt: {}
+	};
+	for (const dim of DIMENSIONS) {
+		const { mean, stddev: sd } = meanAndStddev(samples.map((s) => s.scores[dim]));
+		scores[dim] = mean;
+		stddev[dim] = sd;
+	}
+	const ifValues = [];
+	for (const s of samples) {
+		const v = s.scores.instruction_following;
+		if (typeof v === "number" && Number.isFinite(v)) ifValues.push(v);
+	}
+	if (ifValues.length > 0) {
+		const { mean, stddev: sd } = meanAndStddev(ifValues);
+		scores.instruction_following = mean;
+		stddev.instruction_following = sd;
+	}
+	const byPrompt = /* @__PURE__ */ new Map();
+	for (const s of samples) {
+		const arr = byPrompt.get(s.prompt) ?? [];
+		arr.push(s);
+		byPrompt.set(s.prompt, arr);
+	}
+	const per_prompt = {};
+	for (const promptId of [...byPrompt.keys()].sort()) {
+		const promptSamples = byPrompt.get(promptId);
+		const dims = emptyDimensionScores();
+		for (const dim of DIMENSIONS) dims[dim] = meanAndStddev(promptSamples.map((s) => s.scores[dim])).mean;
+		const ifPerPrompt = [];
+		for (const s of promptSamples) {
+			const v = s.scores.instruction_following;
+			if (typeof v === "number" && Number.isFinite(v)) ifPerPrompt.push(v);
+		}
+		if (ifPerPrompt.length > 0) dims.instruction_following = meanAndStddev(ifPerPrompt).mean;
+		per_prompt[promptId] = dims;
+	}
+	const judgeScores = /* @__PURE__ */ new Map();
+	for (const s of samples) {
+		if (!s.judges_raw) continue;
+		for (const [id, score] of Object.entries(s.judges_raw)) {
+			const arr = judgeScores.get(id) ?? [];
+			if (typeof score === "number" && Number.isFinite(score)) arr.push(score);
+			judgeScores.set(id, arr);
+		}
+	}
+	const out = {
+		n_samples: samples.length,
+		scores,
+		stddev,
+		per_prompt
+	};
+	if (prompts && prompts.length > 0) {
+		const domainOf = /* @__PURE__ */ new Map();
+		for (const p of prompts) domainOf.set(p.id, promptDomain(p));
+		const samplesByDomain = /* @__PURE__ */ new Map();
+		for (const s of samples) {
+			const d = domainOf.get(s.prompt) ?? "english";
+			const arr = samplesByDomain.get(d) ?? [];
+			arr.push(s);
+			samplesByDomain.set(d, arr);
+		}
+		if (samplesByDomain.size > 0) {
+			const domains = {};
+			for (const d of [...samplesByDomain.keys()].sort()) {
+				const means = samplesByDomain.get(d).map((s) => sampleDimensionMean(s.scores));
+				domains[d] = means.reduce((a, b) => a + b, 0) / means.length;
+			}
+			out.domains = domains;
+		}
+	}
+	if (judgeScores.size > 0) {
+		const judges_raw = {};
+		for (const id of [...judgeScores.keys()].sort()) {
+			const vs = judgeScores.get(id);
+			judges_raw[id] = vs.length === 0 ? null : meanAndStddev(vs).mean;
+		}
+		out.judges_raw = judges_raw;
+	}
+	return out;
+}
+async function runMatrix(input) {
+	const benchPromptCache = /* @__PURE__ */ new Map();
+	for (const p of input.prompts) benchPromptCache.set(p.id, promptToBenchPrompt(p));
+	const results = {};
+	for (const model of input.models) {
+		const samples = [];
+		for (const prompt of input.prompts) {
+			const batches = chunk(Array.from({ length: input.samples }, () => prompt), input.parallel);
+			for (const batch of batches) {
+				const out = await Promise.all(batch.map((p) => runOne({
+					deps: input.deps,
+					model,
+					hub: input.hub,
+					prompt: p,
+					benchPrompt: benchPromptCache.get(p.id)
+				})));
+				samples.push(...out);
+			}
+		}
+		results[model] = aggregateAcrossPrompts(samples, input.prompts);
+	}
+	const promptIds = input.prompts.map((p) => p.id);
+	return {
+		results,
+		file: await writeResultsAtomic({
+			version: (/* @__PURE__ */ new Date()).toISOString(),
+			config: {
+				samples_per_pair: input.samples,
+				models_tested: input.models,
+				prompts_used: promptIds
+			},
+			results
+		}, input.resultsDir),
+		ranBatch: {
+			models: input.models,
+			prompts: promptIds,
+			samples: input.samples
+		}
+	};
+}
+function chunk(arr, size) {
+	const out = [];
+	for (let i = 0; i < arr.length; i += size) out.push(arr.slice(i, i + size));
+	return out;
+}
+/**
+* `runMatrixFull` is the headless full-matrix runner used by `/bench/.actions/runFull`.
+*
+* Differences vs `runMatrix`:
+*  - Allows up to 200 models / 200 prompts (vs 5/5 for the fast `run` action).
+*  - Flushes the results JSON to disk after every (model, prompt) pair so a
+*    SIGKILL mid-run preserves the entries already completed.
+*  - Retries rate-limit errors with exponential backoff (2s/4s/8s, max 3
+*    retries). Auth errors (`INVALID_API_KEY`, `UNAUTHENTICATED`, etc.) fail
+*    immediately without retry.
+*  - Records per-sample failures on the model aggregate's `errors[]` instead
+*    of throwing — one bad sample doesn't block the rest of the matrix.
+*  - Emits per-sample progress via `onProgress` so callers can stream live
+*    output (`▸ <model> × <prompt> sample N/M: ...`).
+*/
+async function runMatrixFull(input) {
+	const benchPromptCache = /* @__PURE__ */ new Map();
+	for (const p of input.prompts) benchPromptCache.set(p.id, promptToBenchPrompt(p));
+	const sleep = input.sleep ?? defaultSleep;
+	const promptIds = input.prompts.map((p) => p.id);
+	const isoVersion = (/* @__PURE__ */ new Date()).toISOString();
+	const includeRaw = input.includeRaw ?? false;
+	const effectiveJudges = input.judges && input.judges.length > 0 ? [...input.judges] : [];
+	const judgeForRequested = input.judgeFor?.includes("instruction_following") ?? false;
+	let resultsFileJudgeModel = null;
+	if (judgeForRequested) if (effectiveJudges.length > 1) resultsFileJudgeModel = [...effectiveJudges];
+	else if (effectiveJudges.length === 1) resultsFileJudgeModel = effectiveJudges[0];
+	else resultsFileJudgeModel = input.judgeModel ?? DEFAULT_JUDGE_MODEL;
+	const results = {};
+	const samplesByModel = {};
+	const errorsByModel = {};
+	const rawByModel = {};
+	let lastFile = "";
+	let lastRawFile = null;
+	const outlierThreshold = input.outlierThreshold ?? DEFAULT_OUTLIER_THRESHOLD;
+	const outlierRerun = input.outlierRerun ?? false;
+	const runSampleBatch = async (model, prompt, sampleIndices, totalSamples) => {
+		for (const batch of chunk(sampleIndices, input.parallel)) {
+			const settled = await Promise.all(batch.map(async (sampleIndex) => ({
+				sampleIndex,
+				outcome: await runOneWithRetry({
+					deps: input.deps,
+					model,
+					hub: input.hub,
+					prompt,
+					benchPrompt: benchPromptCache.get(prompt.id),
+					sleep
+				})
+			})));
+			for (const { sampleIndex, outcome } of settled) {
+				const event = {
+					model,
+					prompt: prompt.id,
+					sampleIndex,
+					totalSamples,
+					status: outcome.kind === "ok" ? "ok" : "error"
+				};
+				if (outcome.kind === "ok") {
+					if (judgeForRequested) if (effectiveJudges.length > 0) {
+						const perJudge = await Promise.all(effectiveJudges.map((j) => judgeInstructionFollowing(input.deps, benchPromptCache.get(prompt.id), outcome.value.raw, j, input.judgeHub)));
+						const judgesRaw = {};
+						effectiveJudges.forEach((j, i) => {
+							judgesRaw[j] = perJudge[i];
+						});
+						outcome.value.judges_raw = judgesRaw;
+						const aggregate = aggregateJudgeScores(perJudge);
+						if (aggregate !== null) outcome.value.scores.instruction_following = aggregate;
+					} else {
+						const score = await judgeInstructionFollowing(input.deps, benchPromptCache.get(prompt.id), outcome.value.raw, input.judgeModel, input.judgeHub);
+						if (score !== null) outcome.value.scores.instruction_following = score;
+					}
+					samplesByModel[model].push(outcome.value);
+					event.scores = outcome.value.scores;
+					if (includeRaw) rawByModel[model][prompt.id].push({
+						sampleIndex,
+						raw: outcome.value.raw
+					});
+				} else {
+					errorsByModel[model].push(outcome.error);
+					event.error = outcome.error;
+				}
+				input.onProgress?.(event);
+			}
+		}
+	};
+	const buildResultsFile = (outliers$1) => ({
+		version: isoVersion,
+		judge_model: resultsFileJudgeModel,
+		config: {
+			samples_per_pair: input.samples,
+			models_tested: input.models,
+			prompts_used: promptIds
+		},
+		results,
+		outliers: outliers$1
+	});
+	for (const model of input.models) {
+		samplesByModel[model] = [];
+		errorsByModel[model] = [];
+		if (includeRaw) rawByModel[model] = {};
+		for (const prompt of input.prompts) {
+			if (includeRaw) rawByModel[model][prompt.id] = [];
+			await runSampleBatch(model, prompt, Array.from({ length: input.samples }, (_, i) => i + 1), input.samples);
+			const agg = aggregateAcrossPrompts(samplesByModel[model], input.prompts);
+			const errs = errorsByModel[model];
+			results[model] = errs.length > 0 ? {
+				...agg,
+				errors: errs
+			} : agg;
+			lastFile = await writeResultsAtomic(buildResultsFile([]), input.resultsDir);
+			if (includeRaw) lastRawFile = await writeRawResultsAtomic({
+				version: isoVersion,
+				config: {
+					samples_per_pair: input.samples,
+					models_tested: input.models,
+					prompts_used: promptIds
+				},
+				raw: rawByModel
+			}, input.resultsDir);
+			await input.onPairComplete?.(model, prompt.id, lastFile);
+		}
+		await input.onModelComplete?.(model, lastFile);
+	}
+	let outliers = findOutliers(samplesByModel, outlierThreshold);
+	if (outlierRerun && outliers.length > 0) {
+		const promptById = new Map(input.prompts.map((p) => [p.id, p]));
+		const rerunByModel = /* @__PURE__ */ new Map();
+		for (const o of outliers) {
+			if (!promptById.has(o.prompt)) continue;
+			const set = rerunByModel.get(o.model) ?? /* @__PURE__ */ new Set();
+			set.add(o.prompt);
+			rerunByModel.set(o.model, set);
+		}
+		for (const [model, promptIds$1] of rerunByModel) {
+			const countByPrompt = /* @__PURE__ */ new Map();
+			for (const s of samplesByModel[model]) countByPrompt.set(s.prompt, (countByPrompt.get(s.prompt) ?? 0) + 1);
+			for (const promptId of promptIds$1) {
+				const prompt = promptById.get(promptId);
+				const existingForPair = countByPrompt.get(promptId) ?? 0;
+				const extra = Math.min(3, 2 * input.samples - existingForPair);
+				if (extra <= 0) continue;
+				await runSampleBatch(model, prompt, Array.from({ length: extra }, (_, i) => existingForPair + i + 1), existingForPair + extra);
+			}
+		}
+		for (const model of rerunByModel.keys()) {
+			const agg = aggregateAcrossPrompts(samplesByModel[model], input.prompts);
+			const errs = errorsByModel[model];
+			results[model] = errs.length > 0 ? {
+				...agg,
+				errors: errs
+			} : agg;
+		}
+		outliers = findOutliers(samplesByModel, outlierThreshold);
+	}
+	lastFile = await writeResultsAtomic(buildResultsFile(outliers), input.resultsDir);
+	if (includeRaw) lastRawFile = await writeRawResultsAtomic({
+		version: isoVersion,
+		config: {
+			samples_per_pair: input.samples,
+			models_tested: input.models,
+			prompts_used: promptIds
+		},
+		raw: rawByModel
+	}, input.resultsDir);
+	return {
+		results,
+		file: lastFile,
+		rawFile: lastRawFile,
+		ranBatch: {
+			models: input.models,
+			prompts: promptIds,
+			samples: input.samples
+		}
+	};
+}
+/**
+* Wraps `runOne` with bounded exponential backoff for rate-limit errors and
+* fast-fail for auth errors. Other errors are non-retryable but classified as
+* "other" so they still surface in `errors[]`.
+*/
+async function runOneWithRetry(input) {
+	for (let attempts = 1;; attempts++) try {
+		return {
+			kind: "ok",
+			value: await runOne({
+				deps: input.deps,
+				model: input.model,
+				hub: input.hub,
+				prompt: input.prompt,
+				benchPrompt: input.benchPrompt
+			})
+		};
+	} catch (err) {
+		const message = err instanceof Error ? err.message : String(err);
+		const cls = classifyError(message);
+		if (cls === "rate_limit" && attempts <= RATE_LIMIT_MAX_RETRIES) {
+			await input.sleep(RATE_LIMIT_BASE_BACKOFF_MS * 2 ** (attempts - 1));
+			continue;
+		}
+		return {
+			kind: "error",
+			error: {
+				kind: cls,
+				path: "/dev/agent/.actions/run",
+				message,
+				attempts
+			}
+		};
+	}
+}
+const RATE_LIMIT_PATTERNS = /(rate.*limit|429|too many requests)/i;
+const AUTH_ERROR_PATTERNS = /(invalid[ _]api[ _]key|unauthenticated|unauthorized)/i;
+function classifyError(message) {
+	if (RATE_LIMIT_PATTERNS.test(message)) return "rate_limit";
+	if (AUTH_ERROR_PATTERNS.test(message)) return "auth";
+	return "other";
+}
+function defaultSleep(ms) {
+	return new Promise((resolve) => setTimeout(resolve, ms));
+}
+/**
+* Write `payload` to `<dir>/<ISO-date>.json` atomically (write to a randomised
+* `.tmp-<id>.json` then rename). Best-effort redaction runs before
+* serialisation: both sensitive labels (api_key / Authorization / …) and
+* common credential VALUE shapes (Bearer tokens, `sk-`/`ya29.` keys, PEM
+* blocks, AWS/Slack ids) are blanked so credentials that leak into agent-run
+* traces are scrubbed from the on-disk record. This is a safety net, not a
+* guarantee — the design keeps credentials out of traces. Aborts (without
+* writing) when
+* the serialised payload would exceed `MAX_RESULTS_SIZE_BYTES` — the failure
+* mode is intentionally noisy so a bug that bloats the results dir never lands
+* on disk silently.
+*/
+async function writeResultsAtomic(payload, dir) {
+	const safe = sanitiseForResults(payload);
+	const json = `${JSON.stringify(safe, null, 2)}\n`;
+	const bytes = Buffer.byteLength(json, "utf8");
+	if (bytes > MAX_RESULTS_SIZE_BYTES) throw new ValidationError(`results JSON is ${bytes} bytes — exceeds ${MAX_RESULTS_SIZE_BYTES} byte cap. If raw traces are needed pass \`includeRaw: true\` (writes a separate \`<stamp>-raw.json\`).`);
+	return writeAtomic(dir, `${payload.version.replace(/[:.]/g, "-")}.json`, json);
+}
+/**
+* Write the raw-trace sidecar to `<dir>/<ISO-date>-raw.json` atomically. Uses
+* the same sensitive-string redaction as the main results file. Has no size
+* cap — the maintainer opted in via `includeRaw: true` and a hard fail
+* mid-run would discard the agent traces they explicitly asked to keep.
+*/
+async function writeRawResultsAtomic(payload, dir) {
+	const safe = sanitiseForResults(payload);
+	const json = `${JSON.stringify(safe, null, 2)}\n`;
+	return writeAtomic(dir, `${payload.version.replace(/[:.]/g, "-")}-raw.json`, json);
+}
+async function writeAtomic(dir, filename, json) {
+	await promises.mkdir(dir, { recursive: true });
+	const finalPath = join(dir, filename);
+	const tmpPath = join(dir, `.tmp-${randomBytes(6).toString("hex")}.json`);
+	await promises.writeFile(tmpPath, json, "utf8");
+	try {
+		await promises.rename(tmpPath, finalPath);
+	} catch (err) {
+		await promises.rm(tmpPath, { force: true });
+		throw err;
+	}
+	return finalPath;
+}
+function sanitiseForResults(value) {
+	let cleaned = JSON.stringify(value);
+	for (const re of SECRET_VALUE_PATTERNS) cleaned = cleaned.replace(re, "[REDACTED]");
+	for (const re of SENSITIVE_PATTERNS) cleaned = cleaned.replace(re, "[REDACTED]");
+	return JSON.parse(cleaned);
+}
+
+//#endregion
+export { aggregateAcrossPrompts, defaultBenchSystem, defaultBenchTools, extractExpectedOutcomes, extractExpectedPaths, meanAndStddev, promptToBenchPrompt, runMatrix, runMatrixFull, runOne, validateRunArgs, validateRunFullArgs, writeRawResultsAtomic, writeResultsAtomic };
+//# sourceMappingURL=runner.mjs.map