@aigne/afs-llm-bench 1.12.0-beta.5

package/dist/bench.mjs

@@ -0,0 +1,944 @@
+import { ValidationError } from "./errors.mjs";
+import { runMatrix, runMatrixFull, validateRunArgs, validateRunFullArgs } from "./runner.mjs";
+import { patchCatalog } from "./patcher.mjs";
+import { BadPromptError, loadPromptsFromDir } from "./prompts.mjs";
+import { generateReport } from "./reporter.mjs";
+import { __decorate } from "./_virtual/_@oxc-project_runtime@0.108.0/helpers/decorate.mjs";
+import { existsSync } from "node:fs";
+import { AFSBaseProvider, Actions, Explain, List, Meta, Read, Stat, getPlatform, makeNsLog } from "@aigne/afs";
+import { joinURL } from "ufo";
+import { z } from "zod";
+
+//#region src/bench.ts
+/**
+* AFSBench — LLM AFS benchmark provider (Phase 0 scaffold).
+*
+* Routes:
+*   /                                — directory (list → prompts, results, .actions)
+*   /prompts                          — directory (list → prompt entries)
+*   /prompts/:id                      — file (parsed prompt content)
+*   /results                          — directory (list → result JSON files)
+*   /results/:date                    — file (raw result JSON)
+*   /.actions/run                     — Phase 1: matrix runner. Phase 0: NOT_IMPLEMENTED.
+*   /.actions/patchCatalog            — Phase 1: catalog patcher. Phase 0: NOT_IMPLEMENTED.
+*
+* Phase 1 will plug `runner.ts` and `patcher.ts` into the action handlers; for
+* now they always return `{ success: false, error: { code: "NOT_IMPLEMENTED" } }`.
+*/
+const log = makeNsLog("provider:llm-bench");
+const DEFAULT_PROMPTS_DIR = "benchmarks/llm-afs/prompts";
+const DEFAULT_RESULTS_DIR = "benchmarks/llm-afs/results";
+const DEFAULT_CATALOG_PATH = "providers/ai/device/catalog.json";
+const optionsSchema = z.object({
+	name: z.string().optional(),
+	description: z.string().optional(),
+	promptsDir: z.string().optional().describe("Absolute path to the prompt set on disk (defaults to benchmarks/llm-afs/prompts)."),
+	resultsDir: z.string().optional().describe("Absolute path to the results dir (defaults to benchmarks/llm-afs/results)."),
+	catalogPath: z.string().optional().describe("Absolute path to providers/ai/device/catalog.json (maintainer dev artifact). Default: workspace-root-relative.")
+});
+const ROOT_ACTIONS = [
+	{
+		name: "run",
+		description: "Run a (models × prompts × samples) benchmark matrix (Phase 1).",
+		schema: {
+			type: "object",
+			description: "Run (models × prompts × samples) matrix and write results JSON to disk. Phase 1.",
+			properties: {
+				models: {
+					description: "Canonical model ids to test. Phase 1: required explicit array, length 1-5; no 'all' shortcut.",
+					type: "array",
+					items: { type: "string" }
+				},
+				prompts: {
+					description: "Prompt ids to run, or 'all' (Phase 1: array length 1-5).",
+					oneOf: [{
+						type: "string",
+						enum: ["all"]
+					}, {
+						type: "array",
+						items: { type: "string" }
+					}]
+				},
+				samples: {
+					type: "number",
+					description: "Samples per (model, prompt) pair, clamped to [1, 100]. Default 3."
+				},
+				parallel: {
+					type: "number",
+					description: "Concurrent samples within one (model, prompt). Clamped to [1, 4]. Default 1."
+				}
+			},
+			required: ["models"]
+		}
+	},
+	{
+		name: "runFull",
+		description: "Headless full-matrix runner with incremental flush + rate-limit backoff (Phase 2.1).",
+		schema: {
+			type: "object",
+			description: "Headless full-matrix benchmark runner. Larger limits than `run` (≤200 models / ≤200 prompts), incremental flush after every (model, prompt), exponential backoff for rate-limit errors, fast-fail for invalid api-key.",
+			properties: {
+				models: {
+					description: "Canonical model ids to test. Required array, length 1-200.",
+					type: "array",
+					items: { type: "string" }
+				},
+				prompts: {
+					description: "Prompt ids to run, or 'all' (array length 1-200).",
+					oneOf: [{
+						type: "string",
+						enum: ["all"]
+					}, {
+						type: "array",
+						items: { type: "string" }
+					}]
+				},
+				samples: {
+					type: "number",
+					description: "Samples per (model, prompt) pair, clamped to [1, 100]. Default 3."
+				},
+				parallel: {
+					type: "number",
+					description: "Concurrent samples within one (model, prompt). Clamped to [1, 4]. Default 1."
+				},
+				judgeFor: {
+					type: "array",
+					description: "Dimensions to evaluate via LLM-as-judge. Phase 0 of Run 3 only validates this arg; Phase 1 wires the judge.",
+					items: {
+						type: "string",
+						enum: ["instruction_following"]
+					}
+				},
+				judges: {
+					type: "array",
+					description: "Multi-judge ensemble (Phase 0 of Run 4). N judge model ids; takes precedence over `judgeModel`. N=1 is single-judge passthrough; N≥2 aggregates per spec (succ=2 mean, succ=3 median, succ≥4 trimmed mean). Length 1-10. Shares the same judgeHub.",
+					items: { type: "string" },
+					minItems: 1,
+					maxItems: 10
+				},
+				includeRaw: {
+					type: "boolean",
+					description: "When true, write a sidecar `<stamp>-raw.json` next to the main results file containing AgentRunResult.trace[] for every successful sample. Default false to keep main results small and avoid leaking trace details."
+				},
+				outlierThreshold: {
+					type: "number",
+					description: "Per-dimension stddev cap for outlier flagging (Phase 3 of Run 4). Defaults to 0.15. Clamped to [0.01, 1.0]."
+				},
+				outlierRerun: {
+					type: "boolean",
+					description: "When true, after the main matrix the runner schedules one extra batch (≤ 3 samples) for each (model, prompt) flagged as an outlier and recomputes the aggregate + outliers. Total samples per (model, prompt) bounded by 2 × N. Default false."
+				}
+			},
+			required: ["models"]
+		}
+	},
+	{
+		name: "patchCatalog",
+		description: "Merge a results JSON into the ai-device catalog `afs` subtree (Phase 1).",
+		schema: {
+			type: "object",
+			description: "Merge a results JSON into providers/ai/device/catalog.json under each model's `afs` subtree. Phase 1.",
+			properties: { results: {
+				type: "string",
+				description: "Path to the results JSON to merge (default: latest under resultsDir)."
+			} }
+		}
+	},
+	{
+		name: "report",
+		description: "Render a results JSON as a markdown or HTML benchmark report (Phase 2 of Run 4).",
+		schema: {
+			type: "object",
+			description: "Render a results JSON as a human-readable benchmark report (markdown or HTML). Phase 2 of Run 4.",
+			properties: {
+				results: {
+					type: "string",
+					description: "Path to the results JSON to render (must resolve inside the results dir)."
+				},
+				format: {
+					type: "string",
+					enum: ["markdown", "html"],
+					description: "Output format. Defaults to markdown."
+				},
+				previous: {
+					type: "string",
+					description: "Optional path to a prior results JSON; when supplied the report adds a delta column."
+				},
+				includeRaw: {
+					type: "boolean",
+					description: "When true, embed the raw results JSON inside the report. Default false to avoid leaking trace data into shared reports."
+				}
+			},
+			required: ["results"]
+		}
+	}
+];
+function findWorkspaceRoot(start) {
+	const path = getPlatform().path;
+	let dir = start;
+	for (let i = 0; i < 12; i++) {
+		if (existsSync(path.join(dir, "pnpm-workspace.yaml"))) return dir;
+		const parent = path.dirname(dir);
+		if (parent === dir) return void 0;
+		dir = parent;
+	}
+}
+function resolveAbsolute(p, fallback) {
+	const path = getPlatform().path;
+	const value = p ?? fallback;
+	if (path.isAbsolute(value)) return value;
+	const cwd = getPlatform().process?.cwd?.() ?? process.cwd();
+	const base = p === void 0 ? findWorkspaceRoot(cwd) ?? cwd : cwd;
+	return path.resolve(base, value);
+}
+async function safeReaddir(dir) {
+	const fs = getPlatform().fs;
+	if (!fs || !await fs.exists(dir)) return [];
+	return fs.readdir(dir);
+}
+async function readResultFile(resultsDir, date) {
+	const fs = getPlatform().fs;
+	const path = getPlatform().path;
+	if (!fs) return void 0;
+	const filename = date.endsWith(".json") ? date : `${date}.json`;
+	const target = path.join(resultsDir, filename);
+	if (!await fs.exists(target)) return void 0;
+	const raw = await fs.readTextFile(target);
+	let parsed;
+	try {
+		parsed = JSON.parse(raw);
+	} catch {
+		parsed = raw;
+	}
+	return {
+		raw,
+		parsed
+	};
+}
+var AFSBench = class extends AFSBaseProvider {
+	name;
+	description;
+	accessMode = "readwrite";
+	promptsDir;
+	resultsDir;
+	catalogPath;
+	static schema() {
+		return optionsSchema;
+	}
+	static manifest() {
+		return {
+			name: "llm-bench",
+			description: "LLM AFS benchmark suite — score model AFS-friendliness, patch ai-device catalog.\n- list /prompts to see available benchmark prompts\n- list /results to see prior runs\n- exec /.actions/run to score a (models × prompts × samples) matrix (Phase 1)\n- exec /.actions/patchCatalog to merge a result file into the ai-device catalog (Phase 1)",
+			uriTemplate: "llm-bench://",
+			category: "ai",
+			schema: optionsSchema,
+			tags: [
+				"ai",
+				"benchmark",
+				"llm",
+				"evaluation",
+				"catalog"
+			],
+			capabilityTags: [
+				"read-only",
+				"auth:none",
+				"local"
+			],
+			security: {
+				riskLevel: "local",
+				resourceAccess: ["local-filesystem"],
+				dataSensitivity: [],
+				notes: ["Phase 0 is scaffold-only (NOT_IMPLEMENTED actions). Phase 1 will run benchmarks via agent-run and write the ai-device catalog (maintainer tool boundary, see decisions.md #24)."]
+			}
+		};
+	}
+	static treeSchema() {
+		return {
+			operations: [
+				"list",
+				"read",
+				"exec",
+				"stat",
+				"explain"
+			],
+			tree: {
+				"/": {
+					kind: "llm-bench:root",
+					operations: [
+						"list",
+						"read",
+						"exec"
+					],
+					actions: [
+						"run",
+						"runFull",
+						"patchCatalog",
+						"report"
+					]
+				},
+				"/prompts": {
+					kind: "llm-bench:directory",
+					operations: ["list", "read"]
+				},
+				"/prompts/{id}": {
+					kind: "llm-bench:prompt",
+					operations: ["read"]
+				},
+				"/results": {
+					kind: "llm-bench:directory",
+					operations: ["list", "read"]
+				},
+				"/results/{date}": {
+					kind: "llm-bench:result",
+					operations: ["read"]
+				}
+			},
+			bestFor: ["LLM AFS-friendliness benchmark", "catalog scoring"],
+			notFor: ["runtime telemetry — see ai-device BenchmarkStore for that"]
+		};
+	}
+	constructor(options = {}) {
+		super();
+		const parsed = optionsSchema.parse(options);
+		this.name = parsed.name ?? "llm-bench";
+		this.description = parsed.description ?? "LLM AFS benchmark suite";
+		this.promptsDir = resolveAbsolute(parsed.promptsDir, DEFAULT_PROMPTS_DIR);
+		this.resultsDir = resolveAbsolute(parsed.resultsDir, DEFAULT_RESULTS_DIR);
+		this.catalogPath = resolveAbsolute(parsed.catalogPath, DEFAULT_CATALOG_PATH);
+	}
+	/** Test/debug helpers. */
+	getPromptsDir() {
+		return this.promptsDir;
+	}
+	getResultsDir() {
+		return this.resultsDir;
+	}
+	getCatalogPath() {
+		return this.catalogPath;
+	}
+	async loadPrompts() {
+		const { prompts } = await loadPromptsFromDir(this.promptsDir);
+		return prompts;
+	}
+	async listResultFiles() {
+		const path = getPlatform().path;
+		return (await safeReaddir(this.resultsDir)).filter((f) => path.extname(f).toLowerCase() === ".json").sort();
+	}
+	async listRoot(_ctx) {
+		const promptCount = (await this.loadPrompts()).length;
+		const resultCount = (await this.listResultFiles()).length;
+		return { data: [this.buildEntry("/prompts", { meta: {
+			kind: "llm-bench:directory",
+			childrenCount: promptCount
+		} }), this.buildEntry("/results", { meta: {
+			kind: "llm-bench:directory",
+			childrenCount: resultCount
+		} })] };
+	}
+	async readRoot(_ctx) {
+		return this.buildEntry("/", {
+			content: {
+				provider: "llm-bench",
+				promptsDir: this.promptsDir,
+				resultsDir: this.resultsDir
+			},
+			meta: {
+				kind: "llm-bench:root",
+				childrenCount: 2
+			}
+		});
+	}
+	async statRoot(_ctx) {
+		return { data: this.buildEntry("/", { meta: {
+			kind: "llm-bench:root",
+			childrenCount: 2
+		} }) };
+	}
+	async metaRoot(_ctx) {
+		const prompts = await this.loadPrompts();
+		const results = await this.listResultFiles();
+		return this.buildEntry("/.meta", {
+			content: {
+				provider: "llm-bench",
+				phase: "0-scaffold",
+				promptsDir: this.promptsDir,
+				resultsDir: this.resultsDir,
+				promptCount: prompts.length,
+				resultCount: results.length
+			},
+			meta: {
+				kind: "llm-bench:root",
+				childrenCount: 2
+			}
+		});
+	}
+	async explainRoot(_ctx) {
+		return {
+			content: [
+				"# LLM AFS Benchmark",
+				"",
+				"Score how well an LLM uses AFS tools (exploration, tool reliability,",
+				"self-stop, parallelism, efficiency, instruction-following). Scores get",
+				"patched into the ai-device catalog so policies can rank by real performance.",
+				"",
+				"## Phase 0 (scaffold)",
+				"",
+				"- `list /prompts` — benchmark prompt set (parsed from markdown)",
+				"- `list /results` — historical run JSON files",
+				"- `exec /.actions/run` / `/.actions/patchCatalog` — placeholders, return NOT_IMPLEMENTED until Phase 1 lands",
+				"",
+				"See `intent/llm-afs-benchmark/INTENT.md` for the full design."
+			].join("\n"),
+			format: "markdown"
+		};
+	}
+	async readCapabilities(_ctx) {
+		const operations = this.getOperationsDeclaration();
+		const manifest = {
+			schemaVersion: 1,
+			provider: this.name,
+			description: this.description,
+			tools: [],
+			operations,
+			actions: [{
+				description: "Root actions",
+				catalog: ROOT_ACTIONS.map((a) => ({
+					name: a.name,
+					description: a.description,
+					inputSchema: a.schema
+				})),
+				discovery: {
+					pathTemplate: "/.actions",
+					note: "Phase 0 returns NOT_IMPLEMENTED for both actions; full behaviour lands in Phase 1."
+				}
+			}]
+		};
+		return this.buildEntry("/.meta/.capabilities", {
+			content: manifest,
+			meta: {
+				kind: "afs:capabilities",
+				...operations
+			}
+		});
+	}
+	async listPrompts(_ctx) {
+		return { data: (await this.loadPrompts()).map((p) => this.buildEntry(joinURL("/prompts", p.id), {
+			id: p.id,
+			meta: {
+				kind: "llm-bench:prompt",
+				childrenCount: 0,
+				title: p.frontmatter.title,
+				domain: p.frontmatter.domain ?? "english",
+				maxRounds: p.frontmatter.max_rounds
+			}
+		})) };
+	}
+	async readPrompts(_ctx) {
+		const prompts = await this.loadPrompts();
+		return this.buildEntry("/prompts", {
+			content: { promptIds: prompts.map((p) => p.id) },
+			meta: {
+				kind: "llm-bench:directory",
+				childrenCount: prompts.length
+			}
+		});
+	}
+	async statPrompts(_ctx) {
+		const prompts = await this.loadPrompts();
+		return { data: this.buildEntry("/prompts", { meta: {
+			kind: "llm-bench:directory",
+			childrenCount: prompts.length
+		} }) };
+	}
+	async metaPrompts(_ctx) {
+		const prompts = await this.loadPrompts();
+		return this.buildEntry("/prompts/.meta", {
+			content: {
+				promptIds: prompts.map((p) => p.id),
+				promptsDir: this.promptsDir
+			},
+			meta: {
+				kind: "llm-bench:directory",
+				childrenCount: prompts.length
+			}
+		});
+	}
+	async explainPrompts(_ctx) {
+		const prompts = await this.loadPrompts();
+		return {
+			content: [
+				"# Prompts",
+				"",
+				`Loaded from ${this.promptsDir}. ${prompts.length} prompt(s) parsed.`,
+				"",
+				...prompts.map((p) => `- \`${p.id}\` — ${p.frontmatter.title}`)
+			].join("\n"),
+			format: "markdown"
+		};
+	}
+	async findPromptOrUndefined(id) {
+		return (await this.loadPrompts()).find((p) => p.id === id);
+	}
+	async listPrompt(_ctx) {
+		return { data: [] };
+	}
+	async readPrompt(ctx) {
+		const prompt = await this.findPromptOrUndefined(ctx.params.id);
+		if (!prompt) return void 0;
+		return this.buildEntry(joinURL("/prompts", prompt.id), {
+			id: prompt.id,
+			content: prompt.body,
+			meta: {
+				kind: "llm-bench:prompt",
+				childrenCount: 0,
+				title: prompt.frontmatter.title,
+				domain: prompt.frontmatter.domain ?? "english",
+				maxRounds: prompt.frontmatter.max_rounds,
+				sections: Object.keys(prompt.sections),
+				filename: prompt.filename
+			}
+		});
+	}
+	async statPrompt(ctx) {
+		const prompt = await this.findPromptOrUndefined(ctx.params.id);
+		if (!prompt) return { data: void 0 };
+		return { data: this.buildEntry(joinURL("/prompts", prompt.id), {
+			id: prompt.id,
+			meta: {
+				kind: "llm-bench:prompt",
+				childrenCount: 0
+			}
+		}) };
+	}
+	async metaPrompt(ctx) {
+		const prompt = await this.findPromptOrUndefined(ctx.params.id);
+		if (!prompt) return void 0;
+		return this.buildEntry(joinURL("/prompts", prompt.id, ".meta"), {
+			content: {
+				id: prompt.id,
+				title: prompt.frontmatter.title,
+				domain: prompt.frontmatter.domain ?? "english",
+				maxRounds: prompt.frontmatter.max_rounds,
+				weightDim: prompt.frontmatter.weight_dim,
+				sections: Object.keys(prompt.sections)
+			},
+			meta: {
+				kind: "llm-bench:prompt",
+				childrenCount: 0,
+				title: prompt.frontmatter.title,
+				domain: prompt.frontmatter.domain ?? "english",
+				maxRounds: prompt.frontmatter.max_rounds
+			}
+		});
+	}
+	async explainPrompt(ctx) {
+		const prompt = await this.findPromptOrUndefined(ctx.params.id);
+		if (!prompt) return {
+			content: `# Prompt not found\n\nNo prompt registered with id \`${ctx.params.id}\`.`,
+			format: "markdown"
+		};
+		return {
+			content: [
+				`# ${prompt.frontmatter.title}`,
+				"",
+				`Id: \`${prompt.id}\` · Domain: ${prompt.frontmatter.domain ?? "english"} · Max rounds: ${prompt.frontmatter.max_rounds}`,
+				"",
+				"## Sections",
+				...Object.keys(prompt.sections).map((s) => `- ${s}`)
+			].join("\n"),
+			format: "markdown"
+		};
+	}
+	async listResults(_ctx) {
+		return { data: (await this.listResultFiles()).map((filename) => {
+			const id = filename.replace(/\.json$/i, "");
+			return this.buildEntry(joinURL("/results", id), {
+				id,
+				meta: {
+					kind: "llm-bench:result",
+					childrenCount: 0,
+					filename
+				}
+			});
+		}) };
+	}
+	async readResults(_ctx) {
+		const files = await this.listResultFiles();
+		return this.buildEntry("/results", {
+			content: { runs: files },
+			meta: {
+				kind: "llm-bench:directory",
+				childrenCount: files.length
+			}
+		});
+	}
+	async statResults(_ctx) {
+		const files = await this.listResultFiles();
+		return { data: this.buildEntry("/results", { meta: {
+			kind: "llm-bench:directory",
+			childrenCount: files.length
+		} }) };
+	}
+	async metaResults(_ctx) {
+		const files = await this.listResultFiles();
+		return this.buildEntry("/results/.meta", {
+			content: {
+				runs: files,
+				resultsDir: this.resultsDir
+			},
+			meta: {
+				kind: "llm-bench:directory",
+				childrenCount: files.length
+			}
+		});
+	}
+	async explainResults(_ctx) {
+		const files = await this.listResultFiles();
+		return {
+			content: [
+				"# Results",
+				"",
+				`Historical runs under ${this.resultsDir}. ${files.length} file(s).`,
+				"",
+				"Each `.json` file is a snapshot from `arc afs exec /bench/.actions/run` (Phase 1)."
+			].join("\n"),
+			format: "markdown"
+		};
+	}
+	async listResult(_ctx) {
+		return { data: [] };
+	}
+	async readResult(ctx) {
+		const file = await readResultFile(this.resultsDir, ctx.params.date);
+		if (!file) return void 0;
+		const id = ctx.params.date.replace(/\.json$/i, "");
+		return this.buildEntry(joinURL("/results", id), {
+			id,
+			content: file.parsed,
+			meta: {
+				kind: "llm-bench:result",
+				childrenCount: 0
+			}
+		});
+	}
+	async statResult(ctx) {
+		if (!await readResultFile(this.resultsDir, ctx.params.date)) return { data: void 0 };
+		const id = ctx.params.date.replace(/\.json$/i, "");
+		return { data: this.buildEntry(joinURL("/results", id), {
+			id,
+			meta: {
+				kind: "llm-bench:result",
+				childrenCount: 0
+			}
+		}) };
+	}
+	async metaResult(ctx) {
+		const file = await readResultFile(this.resultsDir, ctx.params.date);
+		if (!file) return void 0;
+		const id = ctx.params.date.replace(/\.json$/i, "");
+		return this.buildEntry(joinURL("/results", id, ".meta"), {
+			content: {
+				id,
+				sizeBytes: file.raw.length
+			},
+			meta: {
+				kind: "llm-bench:result",
+				childrenCount: 0
+			}
+		});
+	}
+	async listRootActions(_ctx) {
+		return { data: ROOT_ACTIONS.map((a) => this.buildEntry(joinURL("/.actions", a.name), {
+			id: a.name,
+			content: {
+				name: a.name,
+				description: a.description,
+				inputSchema: a.schema
+			},
+			meta: { kind: "action" }
+		})) };
+	}
+	async execRun(ctx, args) {
+		try {
+			const validated = validateRunArgs(args);
+			const allPrompts = await this.loadPrompts();
+			const requestedIds = validated.prompts === "all" ? allPrompts.map((p) => p.id) : validated.prompts;
+			const selected = [];
+			for (const id of requestedIds) {
+				const found = allPrompts.find((p) => p.id === id);
+				if (!found) throw new BadPromptError(id, `unknown prompt id: ${id}`);
+				selected.push(found);
+			}
+			const afs = ctx.context?.afs;
+			const afsExec = afs?.exec?.bind(afs);
+			if (!afsExec) return {
+				success: false,
+				error: {
+					code: "NO_AFS_CONTEXT",
+					message: "AFS root unavailable in route context — cannot dispatch /dev/agent/.actions/run"
+				}
+			};
+			const out = await runMatrix({
+				deps: { exec: (path, dispatchArgs) => afsExec(path, dispatchArgs, {}) },
+				prompts: selected,
+				models: validated.models,
+				samples: validated.samples,
+				parallel: validated.parallel,
+				resultsDir: this.resultsDir,
+				...validated.hub !== void 0 ? { hub: validated.hub } : {}
+			});
+			return {
+				success: true,
+				data: {
+					results: out.results,
+					file: out.file,
+					ranBatch: out.ranBatch
+				}
+			};
+		} catch (err) {
+			if (err instanceof ValidationError) return {
+				success: false,
+				error: {
+					code: "VALIDATION",
+					message: err.message
+				}
+			};
+			if (err instanceof BadPromptError) return {
+				success: false,
+				error: {
+					code: "BAD_PROMPT",
+					message: err.message
+				}
+			};
+			return {
+				success: false,
+				error: {
+					code: "RUNNER_ERROR",
+					message: err instanceof Error ? err.message : String(err)
+				}
+			};
+		}
+	}
+	async execRunFull(ctx, args) {
+		try {
+			const validated = validateRunFullArgs(args);
+			const allPrompts = await this.loadPrompts();
+			const requestedIds = validated.prompts === "all" ? allPrompts.map((p) => p.id) : validated.prompts;
+			const selected = [];
+			for (const id of requestedIds) {
+				const found = allPrompts.find((p) => p.id === id);
+				if (!found) throw new BadPromptError(id, `unknown prompt id: ${id}`);
+				selected.push(found);
+			}
+			const afs = ctx.context?.afs;
+			const afsExec = afs?.exec?.bind(afs);
+			if (!afsExec) return {
+				success: false,
+				error: {
+					code: "NO_AFS_CONTEXT",
+					message: "AFS root unavailable in route context — cannot dispatch /dev/agent/.actions/run"
+				}
+			};
+			const out = await runMatrixFull({
+				deps: { exec: (path, dispatchArgs) => afsExec(path, dispatchArgs, {}) },
+				prompts: selected,
+				models: validated.models,
+				samples: validated.samples,
+				parallel: validated.parallel,
+				resultsDir: this.resultsDir,
+				...validated.hub !== void 0 ? { hub: validated.hub } : {},
+				judgeFor: validated.judgeFor,
+				judges: validated.judges,
+				includeRaw: validated.includeRaw,
+				outlierThreshold: validated.outlierThreshold,
+				outlierRerun: validated.outlierRerun,
+				onProgress: progressLogger
+			});
+			return {
+				success: true,
+				data: {
+					results: out.results,
+					file: out.file,
+					rawFile: out.rawFile,
+					ranBatch: out.ranBatch,
+					judgeFor: validated.judgeFor ?? null
+				}
+			};
+		} catch (err) {
+			if (err instanceof ValidationError) return {
+				success: false,
+				error: {
+					code: "VALIDATION",
+					message: err.message
+				}
+			};
+			if (err instanceof BadPromptError) return {
+				success: false,
+				error: {
+					code: "BAD_PROMPT",
+					message: err.message
+				}
+			};
+			return {
+				success: false,
+				error: {
+					code: "RUNNER_ERROR",
+					message: err instanceof Error ? err.message : String(err)
+				}
+			};
+		}
+	}
+	async execPatchCatalog(ctx, args) {
+		try {
+			const resultsArg = args.results;
+			if (typeof resultsArg !== "string" || resultsArg.length === 0) throw new ValidationError("'results' must be a non-empty string path");
+			const afs = ctx.context?.afs;
+			const afsExec = afs?.exec?.bind(afs);
+			const afsRead = afs?.read?.bind(afs);
+			const afsList = afs?.list?.bind(afs);
+			if (!afsRead || !afsList) return {
+				success: false,
+				error: {
+					code: "NO_AFS_CONTEXT",
+					message: "AFS unavailable in route context — cannot fetch /dev/ai/score-weights. Ensure llm-bench is mounted alongside ai-device."
+				}
+			};
+			return {
+				success: true,
+				data: await patchCatalog({
+					resultsPath: resultsArg,
+					catalogPath: this.catalogPath,
+					resultsRoot: this.resultsDir,
+					loadPolicyWeights: () => loadWeightsViaAFS(afsList, afsRead),
+					refreshRegistry: afsExec ? async () => {
+						const result = await afsExec("/dev/ai/.actions/refreshRegistry", {}, {});
+						if (!result.success) throw new Error(`refreshRegistry failed: ${result.error?.message ?? "(no message)"}`);
+					} : void 0
+				})
+			};
+		} catch (err) {
+			if (err instanceof ValidationError) return {
+				success: false,
+				error: {
+					code: "VALIDATION",
+					message: err.message
+				}
+			};
+			return {
+				success: false,
+				error: {
+					code: "PATCH_ERROR",
+					message: err instanceof Error ? err.message : String(err)
+				}
+			};
+		}
+	}
+	async execReport(ctx, args) {
+		try {
+			const resultsArg = args.results;
+			if (typeof resultsArg !== "string" || resultsArg.length === 0) throw new ValidationError("'results' must be a non-empty string path");
+			const previousArg = args.previous;
+			if (previousArg !== void 0 && (typeof previousArg !== "string" || previousArg.length === 0)) throw new ValidationError("'previous' must be a non-empty string path when provided");
+			const formatArg = args.format;
+			if (formatArg !== void 0 && formatArg !== "markdown" && formatArg !== "html") throw new ValidationError("'format' must be 'markdown' or 'html'");
+			const includeRawArg = args.includeRaw;
+			if (includeRawArg !== void 0 && typeof includeRawArg !== "boolean") throw new ValidationError("'includeRaw' must be a boolean when provided");
+			const afs = ctx.context?.afs;
+			const afsRead = afs?.read?.bind(afs);
+			const afsList = afs?.list?.bind(afs);
+			const report = await generateReport({
+				resultsPath: resultsArg,
+				resultsRoot: this.resultsDir,
+				previousPath: previousArg,
+				format: formatArg,
+				includeRaw: includeRawArg === true,
+				loadPolicyWeights: afsRead && afsList ? () => loadWeightsViaAFS(afsList, afsRead) : () => Promise.resolve({})
+			});
+			return {
+				success: true,
+				data: {
+					content: report.content,
+					format: report.format,
+					generatedAt: report.generatedAt
+				}
+			};
+		} catch (err) {
+			if (err instanceof ValidationError) return {
+				success: false,
+				error: {
+					code: "VALIDATION",
+					message: err.message
+				}
+			};
+			return {
+				success: false,
+				error: {
+					code: "REPORT_ERROR",
+					message: err instanceof Error ? err.message : String(err)
+				}
+			};
+		}
+	}
+};
+__decorate([List("/")], AFSBench.prototype, "listRoot", null);
+__decorate([Read("/")], AFSBench.prototype, "readRoot", null);
+__decorate([Stat("/")], AFSBench.prototype, "statRoot", null);
+__decorate([Meta("/")], AFSBench.prototype, "metaRoot", null);
+__decorate([Explain("/")], AFSBench.prototype, "explainRoot", null);
+__decorate([Read("/.meta/.capabilities")], AFSBench.prototype, "readCapabilities", null);
+__decorate([List("/prompts")], AFSBench.prototype, "listPrompts", null);
+__decorate([Read("/prompts")], AFSBench.prototype, "readPrompts", null);
+__decorate([Stat("/prompts")], AFSBench.prototype, "statPrompts", null);
+__decorate([Meta("/prompts")], AFSBench.prototype, "metaPrompts", null);
+__decorate([Explain("/prompts")], AFSBench.prototype, "explainPrompts", null);
+__decorate([List("/prompts/:id")], AFSBench.prototype, "listPrompt", null);
+__decorate([Read("/prompts/:id")], AFSBench.prototype, "readPrompt", null);
+__decorate([Stat("/prompts/:id")], AFSBench.prototype, "statPrompt", null);
+__decorate([Meta("/prompts/:id")], AFSBench.prototype, "metaPrompt", null);
+__decorate([Explain("/prompts/:id")], AFSBench.prototype, "explainPrompt", null);
+__decorate([List("/results")], AFSBench.prototype, "listResults", null);
+__decorate([Read("/results")], AFSBench.prototype, "readResults", null);
+__decorate([Stat("/results")], AFSBench.prototype, "statResults", null);
+__decorate([Meta("/results")], AFSBench.prototype, "metaResults", null);
+__decorate([Explain("/results")], AFSBench.prototype, "explainResults", null);
+__decorate([List("/results/:date")], AFSBench.prototype, "listResult", null);
+__decorate([Read("/results/:date")], AFSBench.prototype, "readResult", null);
+__decorate([Stat("/results/:date")], AFSBench.prototype, "statResult", null);
+__decorate([Meta("/results/:date")], AFSBench.prototype, "metaResult", null);
+__decorate([Actions("/")], AFSBench.prototype, "listRootActions", null);
+__decorate([Actions.Exec("/", "run", void 0, { effect: "write" })], AFSBench.prototype, "execRun", null);
+__decorate([Actions.Exec("/", "runFull", void 0, { effect: "write" })], AFSBench.prototype, "execRunFull", null);
+__decorate([Actions.Exec("/", "patchCatalog", void 0, { effect: "write" })], AFSBench.prototype, "execPatchCatalog", null);
+__decorate([Actions.Exec("/", "report", void 0, { effect: "read" })], AFSBench.prototype, "execReport", null);
+function progressLogger(event) {
+	const tag = `${event.model} × ${event.prompt} sample ${event.sampleIndex}/${event.totalSamples}`;
+	if (event.status === "ok" && event.scores) {
+		const s = event.scores;
+		const parts = [
+			`expl=${s.exploration.toFixed(2)}`,
+			`tool=${s.tool_reliability.toFixed(2)}`,
+			`stop=${s.self_stop.toFixed(2)}`,
+			`par=${s.parallelism.toFixed(2)}`,
+			`eff=${s.efficiency.toFixed(2)}`
+		];
+		if (typeof s.instruction_following === "number") parts.push(`inst=${s.instruction_following.toFixed(2)}`);
+		log.info(`▸ ${tag}: ✓ scores={${parts.join(" ")}}`);
+	} else if (event.status === "error" && event.error) log.info(`▸ ${tag}: ✗ ${event.error.kind} after ${event.error.attempts} attempt(s) — ${event.error.message}`);
+}
+async function loadWeightsViaAFS(list, read) {
+	const entries = (await list("/dev/ai/score-weights")).data ?? [];
+	const pairs = await Promise.all(entries.map(async (entry) => {
+		const policy = entry.id ?? entry.path.split("/").pop();
+		if (!policy) return null;
+		const content = (await read(`/dev/ai/score-weights/${policy}`)).data?.content;
+		if (!content?.weights || typeof content.weights !== "object") return null;
+		return [policy, content.weights];
+	}));
+	const out = {};
+	for (const pair of pairs) if (pair) out[pair[0]] = pair[1];
+	return out;
+}
+
+//#endregion
+export { AFSBench };
+//# sourceMappingURL=bench.mjs.map