@aigne/afs-llm-bench 1.12.0-beta.5

package/dist/patcher.mjs

@@ -0,0 +1,291 @@
+import { ValidationError } from "./errors.mjs";
+import "./runner.mjs";
+import { promises } from "node:fs";
+import { makeNsLog } from "@aigne/afs";
+import { z } from "zod";
+import { randomBytes } from "node:crypto";
+import * as path from "node:path";
+
+//#region src/patcher.ts
+/**
+* Phase 3 — patchCatalog. Merges a results JSON into the ai-device catalog's
+* `entry.afs` subtree. Maintainer tool: directly reads/writes catalog.json
+* (a git-tracked dev artifact, not runtime data) — see decisions.md #24.
+*
+* Invariants (plan.md L459-468):
+*   1. Models missing from results keep their existing `entry.afs` (deep-equal).
+*   2. Only `r.scores` is copied (no other result fields leak into catalog).
+*   3. The written `entry.afs` block has alphabetically sorted keys (clean diff).
+*   4. Atomic write: tmp → rename, no half-written catalog on failure.
+*   5. Prototype-pollution: __proto__/constructor/prototype rejected anywhere.
+*   6. Path traversal: resultsPath must resolve inside resultsRoot.
+*   7. Cache reload via `refreshRegistry` — failures propagate so the caller
+*      knows the catalog landed but routing still sees the previous snapshot.
+*   8. Vault credentials sanitisation: api_key / Authorization / Bearer / etc.
+*      are rejected as field names anywhere in `results.<model>` payloads.
+*
+* Other catalog entries are written unchanged (key order preserved) so the
+* git diff is limited to the `afs` block of patched models.
+*
+* Phase 5: weight tables are no longer inlined — the bench provider passes
+* `loadPolicyWeights` (which the wiring in `bench.ts` resolves through
+* `/dev/ai/score-weights/<name>`), so this module remains independent of
+* ai-device internals.
+*/
+const log = makeNsLog("provider:llm-bench");
+const RESERVED_KEYS = new Set([
+	"__proto__",
+	"constructor",
+	"prototype",
+	"__defineGetter__",
+	"__defineSetter__",
+	"__lookupGetter__",
+	"__lookupSetter__"
+]);
+const SENSITIVE_KEY_PATTERNS = [
+	/^api[_-]?key$/i,
+	/^authorization$/i,
+	/^bearer$/i,
+	/^service[_-]?account$/i,
+	/^private[_-]?key$/i,
+	/^client[_-]?secret$/i,
+	/^secret$/i,
+	/^credentials?$/i
+];
+const ALLOWED_RESULT_ENTRY_FIELDS = new Set([
+	"n_samples",
+	"scores",
+	"domains",
+	"stddev",
+	"per_prompt",
+	"errors"
+]);
+const MAX_RESULT_ENTRIES = 500;
+const MAX_N_SAMPLES = 1e3;
+const ScoresSchema = z.object({
+	exploration: z.number().min(0).max(1).optional(),
+	tool_reliability: z.number().min(0).max(1).optional(),
+	self_stop: z.number().min(0).max(1).optional(),
+	parallelism: z.number().min(0).max(1).optional(),
+	efficiency: z.number().min(0).max(1).optional(),
+	instruction_following: z.number().min(0).max(1).optional()
+}).strict();
+const DomainsSchema = z.object({
+	chinese: z.number().min(0).max(1).optional(),
+	english: z.number().min(0).max(1).optional(),
+	coding: z.number().min(0).max(1).optional()
+}).strict();
+const ResultEntrySchema = z.object({
+	n_samples: z.number().int().min(1).max(MAX_N_SAMPLES),
+	scores: ScoresSchema,
+	domains: DomainsSchema.optional(),
+	stddev: z.record(z.string(), z.number()).optional(),
+	per_prompt: z.record(z.string().regex(/^[A-Za-z0-9._-]+$/), ScoresSchema).optional(),
+	errors: z.array(z.unknown()).optional()
+}).strict();
+const JudgeIdSchema = z.string().regex(/^[A-Za-z0-9._-]+$/);
+const JudgeModelSchema = z.union([JudgeIdSchema, z.array(JudgeIdSchema).min(1)]).nullable().optional();
+const ResultsRootSchema = z.object({
+	version: z.string().min(1),
+	judge_model: JudgeModelSchema,
+	config: z.unknown().optional(),
+	results: z.record(z.string().regex(/^[A-Za-z0-9._-]+$/), ResultEntrySchema),
+	outliers: z.unknown().optional()
+}).strict();
+/**
+* Validate raw JSON shape + reject reserved/sensitive keys anywhere.
+* Throws `ValidationError` on any rejection.
+*
+* Layers (in order):
+*   1. Deep-walk: reject `__proto__` / `constructor` / `prototype` and any
+*      key matching a credential pattern (`api_key`, `Authorization`, …).
+*   2. Resource-exhaustion guard: results entries ≤ MAX_RESULT_ENTRIES.
+*   3. Per-entry whitelist: only `n_samples`/`scores`/`domains`/`stddev`.
+*   4. Zod schema parse (z.strict() also catches unknown scores dimensions).
+*/
+function validateResultsShape(raw) {
+	rejectDangerousKeysDeep(raw);
+	if (raw && typeof raw === "object" && !Array.isArray(raw)) {
+		const r = raw.results;
+		if (r && typeof r === "object" && !Array.isArray(r)) {
+			const keys = Object.keys(r);
+			if (keys.length > MAX_RESULT_ENTRIES) throw new ValidationError(`results contains ${keys.length} entries; max ${MAX_RESULT_ENTRIES}`);
+			for (const k of keys) {
+				const entry = r[k];
+				if (entry && typeof entry === "object" && !Array.isArray(entry)) {
+					for (const f of Object.keys(entry)) if (!ALLOWED_RESULT_ENTRY_FIELDS.has(f)) throw new ValidationError(`unknown field "${f}" in results.${k}`);
+				}
+			}
+		}
+	}
+	const parsed = ResultsRootSchema.safeParse(raw);
+	if (!parsed.success) throw new ValidationError(`results JSON is invalid: ${parsed.error.message}`);
+	return {
+		version: parsed.data.version,
+		judge_model: parsed.data.judge_model ?? null,
+		results: parsed.data.results
+	};
+}
+function rejectDangerousKeysDeep(value, p = "$") {
+	if (value === null || typeof value !== "object") return;
+	if (Array.isArray(value)) {
+		for (let i = 0; i < value.length; i++) rejectDangerousKeysDeep(value[i], `${p}[${i}]`);
+		return;
+	}
+	for (const k of Object.keys(value)) {
+		if (RESERVED_KEYS.has(k)) throw new ValidationError(`reserved key "${k}" not allowed at ${p}`);
+		for (const re of SENSITIVE_KEY_PATTERNS) if (re.test(k)) throw new ValidationError(`sensitive field "${k}" not allowed at ${p}`);
+		rejectDangerousKeysDeep(value[k], `${p}.${k}`);
+	}
+}
+/**
+* Resolve `p` against cwd if relative, then ensure it lives inside `resultsRoot`.
+* Defends against `..` segments and absolute paths that escape the root.
+* Returns the absolute path on success; throws `ValidationError` otherwise.
+*
+* Note: this is purely lexical (no `fs.realpath`) — a maintainer-created
+* symlink inside `resultsRoot` pointing outside is not caught here. The threat
+* model is a maintainer tool, so the lexical check is intentionally cheap; if
+* a symlink-based vector materialises in production, swap to `fs.realpath`.
+*/
+function assertWithinResultsDir(p, resultsRoot) {
+	const absRoot = path.resolve(resultsRoot);
+	const abs = path.resolve(p);
+	const rel = path.relative(absRoot, abs);
+	if (rel === "" || rel.startsWith("..") || path.isAbsolute(rel)) throw new ValidationError(`results path outside ${resultsRoot}: ${p}`);
+	return abs;
+}
+/**
+* Return a deeply-key-sorted clone of an `afs` block. Plain objects get keys
+* in alphabetical order; arrays preserve order; primitives pass through.
+* Does not mutate input.
+*/
+function sortAfsBlock(value) {
+	if (value === null || typeof value !== "object") return value;
+	if (Array.isArray(value)) return value.map((v) => sortAfsBlock(v));
+	const sorted = {};
+	for (const k of Object.keys(value).sort()) sorted[k] = sortAfsBlock(value[k]);
+	return sorted;
+}
+const BIOME_LINE_WIDTH = 100;
+const BIOME_INDENT = "  ";
+function stringifyBiomeJSON(value, currentIndent = "", startColumn = 0) {
+	if (value === null) return "null";
+	if (typeof value === "boolean" || typeof value === "number") return String(value);
+	if (typeof value === "string") return JSON.stringify(value);
+	if (Array.isArray(value)) {
+		if (value.length === 0) return "[]";
+		if (value.every((v) => v === null || typeof v !== "object")) {
+			const inline = `[${value.map((v) => stringifyBiomeJSON(v)).join(", ")}]`;
+			if (startColumn + inline.length <= BIOME_LINE_WIDTH) return inline;
+		}
+		const inner$1 = currentIndent + BIOME_INDENT;
+		return `[\n${value.map((v) => `${inner$1}${stringifyBiomeJSON(v, inner$1, inner$1.length)}`).join(",\n")}\n${currentIndent}]`;
+	}
+	if (typeof value !== "object") return JSON.stringify(value);
+	const entries = Object.entries(value);
+	if (entries.length === 0) return "{}";
+	const inner = currentIndent + BIOME_INDENT;
+	return `{\n${entries.map(([k, v]) => {
+		const keyPart = `${inner}${JSON.stringify(k)}: `;
+		return `${keyPart}${stringifyBiomeJSON(v, inner, keyPart.length)}`;
+	}).join(",\n")}\n${currentIndent}}`;
+}
+/** Write `obj` as biome-compatible pretty-printed JSON to `target` via tmp+rename. */
+async function atomicWriteJSON(target, obj) {
+	const dir = path.dirname(target);
+	await promises.mkdir(dir, { recursive: true });
+	const tmp = path.join(dir, `.tmp-${randomBytes(6).toString("hex")}.json`);
+	const payload = `${stringifyBiomeJSON(obj)}\n`;
+	await promises.writeFile(tmp, payload, "utf8");
+	try {
+		await promises.rename(tmp, target);
+	} catch (err) {
+		await promises.rm(tmp, { force: true });
+		throw err;
+	}
+}
+function meanOf(xs) {
+	if (xs.length === 0) return 0;
+	let s = 0;
+	for (const x of xs) s += x;
+	return s / xs.length;
+}
+function getNested(obj, p) {
+	const parts = p.split(".");
+	let cur = obj;
+	for (const k of parts) {
+		if (cur === null || typeof cur !== "object") return void 0;
+		cur = cur[k];
+	}
+	return cur;
+}
+function computeWeightedScore(entry, weights) {
+	let total = 0;
+	for (const [pathKey, w] of Object.entries(weights)) {
+		const v = getNested(entry, pathKey);
+		if (typeof v === "number" && Number.isFinite(v)) total += v * w;
+	}
+	return total;
+}
+async function patchCatalog(input) {
+	const safeResultsPath = assertWithinResultsDir(input.resultsPath, input.resultsRoot);
+	let raw;
+	try {
+		raw = await promises.readFile(safeResultsPath, "utf8");
+	} catch (err) {
+		throw new ValidationError(`failed to read results: ${err instanceof Error ? err.message : String(err)}`);
+	}
+	let parsedRaw;
+	try {
+		parsedRaw = JSON.parse(raw);
+	} catch (err) {
+		throw new ValidationError(`results is not valid JSON: ${err instanceof Error ? err.message : String(err)}`);
+	}
+	const validated = validateResultsShape(parsedRaw);
+	const catalogText = await promises.readFile(input.catalogPath, "utf8");
+	const catalog = JSON.parse(catalogText);
+	if (!Array.isArray(catalog)) throw new ValidationError("catalog.json must be an array of entries");
+	const summary = {
+		changes: [],
+		skipped: []
+	};
+	if (catalog.length === 0) return summary;
+	const weightsTable = await input.loadPolicyWeights();
+	if (!weightsTable || typeof weightsTable !== "object" || Array.isArray(weightsTable)) throw new ValidationError("loadPolicyWeights must return a plain object of { policy: weights }");
+	for (const [policy, weights] of Object.entries(weightsTable)) if (!weights || typeof weights !== "object" || Array.isArray(weights)) throw new ValidationError(`weights for policy "${policy}" must be a plain object`);
+	for (const entry of catalog) {
+		const r = validated.results[entry.model];
+		if (!r) {
+			summary.skipped.push(entry.model);
+			continue;
+		}
+		const before = entry.afs;
+		const policyScores = {};
+		for (const [name, w] of Object.entries(weightsTable)) policyScores[name] = computeWeightedScore(r, w);
+		entry.afs = sortAfsBlock({
+			status: "benchmarked",
+			version: validated.version,
+			last_seen_in_run: validated.version,
+			judge: validated.judge_model,
+			samples: r.n_samples,
+			scores: { ...r.scores },
+			overall: meanOf(Object.values(r.scores)),
+			policyScores,
+			...r.domains ? { domains: { ...r.domains } } : {}
+		});
+		summary.changes.push({
+			model: entry.model,
+			before,
+			after: entry.afs
+		});
+	}
+	await atomicWriteJSON(input.catalogPath, catalog);
+	if (input.refreshRegistry) await input.refreshRegistry();
+	if (summary.changes.length > 0) log.warn(`[patchCatalog] ${summary.changes.length} catalog entr${summary.changes.length === 1 ? "y" : "ies"} updated. Run \`pnpm --filter @aigne/afs-ai-device --filter @aigne/arc-cli build && arc service restart\` to make the daemon's policy router pick them up — the catalog is bundled into ai-device's dist and refreshRegistry only rebuilds the hub registry, not the catalog.`);
+	return summary;
+}
+
+//#endregion
+export { assertWithinResultsDir, atomicWriteJSON, patchCatalog, sortAfsBlock, validateResultsShape };
+//# sourceMappingURL=patcher.mjs.map

package/dist/patcher.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"patcher.mjs","names":["inner","fs"],"sources":["../src/patcher.ts"],"sourcesContent":["/**\n * Phase 3 — patchCatalog. Merges a results JSON into the ai-device catalog's\n * `entry.afs` subtree. Maintainer tool: directly reads/writes catalog.json\n * (a git-tracked dev artifact, not runtime data) — see decisions.md #24.\n *\n * Invariants (plan.md L459-468):\n *   1. Models missing from results keep their existing `entry.afs` (deep-equal).\n *   2. Only `r.scores` is copied (no other result fields leak into catalog).\n *   3. The written `entry.afs` block has alphabetically sorted keys (clean diff).\n *   4. Atomic write: tmp → rename, no half-written catalog on failure.\n *   5. Prototype-pollution: __proto__/constructor/prototype rejected anywhere.\n *   6. Path traversal: resultsPath must resolve inside resultsRoot.\n *   7. Cache reload via `refreshRegistry` — failures propagate so the caller\n *      knows the catalog landed but routing still sees the previous snapshot.\n *   8. Vault credentials sanitisation: api_key / Authorization / Bearer / etc.\n *      are rejected as field names anywhere in `results.<model>` payloads.\n *\n * Other catalog entries are written unchanged (key order preserved) so the\n * git diff is limited to the `afs` block of patched models.\n *\n * Phase 5: weight tables are no longer inlined — the bench provider passes\n * `loadPolicyWeights` (which the wiring in `bench.ts` resolves through\n * `/dev/ai/score-weights/<name>`), so this module remains independent of\n * ai-device internals.\n */\n\nimport { randomBytes } from \"node:crypto\";\nimport { promises as fs } from \"node:fs\";\nimport * as path from \"node:path\";\nimport { makeNsLog } from \"@aigne/afs\";\nimport { z } from \"zod\";\nimport { ValidationError } from \"./runner.js\";\n\nconst log = makeNsLog(\"provider:llm-bench\");\n\n// ─── Types ───────────────────────────────────────────────────────────────\n\nexport interface AFSScores {\n  status: \"benchmarked\" | \"untested\" | \"deprecated_score\";\n  version: string;\n  last_seen_in_run: string;\n  /**\n   * Judge model id(s) used to score `instruction_following`. String for\n   * single-judge runs (back-compat), array for multi-judge ensemble runs\n   * (Phase 0 of Run 4 — plan.md L1052), `null` when no judging happened.\n   */\n  judge: string | string[] | null;\n  samples: number;\n  scores: Record<string, number>;\n  overall: number;\n  policyScores: Record<string, number>;\n  domains?: Record<string, number>;\n}\n\nexport interface CatalogEntry {\n  model: string;\n  afs?: AFSScores;\n  [k: string]: unknown;\n}\n\nexport interface PatchSummary {\n  changes: Array<{ model: string; before: AFSScores | undefined; after: AFSScores }>;\n  skipped: string[];\n}\n\nexport type PolicyWeightsTable = Record<string, Record<string, number>>;\n\nexport interface PatchCatalogInput {\n  resultsPath: string;\n  catalogPath: string;\n  resultsRoot: string;\n  /**\n   * Resolve every policy → its weight map. Bench wires this through AFS\n   * (`/dev/ai/score-weights/<name>`), keeping this module free of ai-device\n   * imports. Returning an empty object means no `policyScores` are produced.\n   */\n  loadPolicyWeights: () => Promise<PolicyWeightsTable>;\n  /**\n   * Triggered after the catalog has been atomically written so the device's\n   * registry cache picks the new scores up immediately. Failures propagate —\n   * the catalog already landed, but the caller should know routing still\n   * sees the previous snapshot until the next refresh tick.\n   */\n  refreshRegistry?: () => Promise<void>;\n}\n\n// ─── Validation ───────────────────────────────────────────────────────────\n\nconst RESERVED_KEYS = new Set([\n  \"__proto__\",\n  \"constructor\",\n  \"prototype\",\n  \"__defineGetter__\",\n  \"__defineSetter__\",\n  \"__lookupGetter__\",\n  \"__lookupSetter__\",\n]);\n\nconst SENSITIVE_KEY_PATTERNS: RegExp[] = [\n  /^api[_-]?key$/i,\n  /^authorization$/i,\n  /^bearer$/i,\n  /^service[_-]?account$/i,\n  /^private[_-]?key$/i,\n  /^client[_-]?secret$/i,\n  /^secret$/i,\n  /^credentials?$/i,\n];\n\n// Whitelist of fields permitted on each results.<model> entry. Anything else\n// is rejected — keeps unexpected fields (rogue, sensitive, prototype-poll) out\n// of the catalog. `stddev` and `per_prompt` are kept because the runner emits\n// them (Phase 1 / Phase 2 of Run 3 respectively); they intentionally don't\n// carry across into entry.afs — the catalog stores aggregated scores only.\nconst ALLOWED_RESULT_ENTRY_FIELDS = new Set([\n  \"n_samples\",\n  \"scores\",\n  \"domains\",\n  \"stddev\",\n  \"per_prompt\",\n  \"errors\",\n]);\n\nconst MAX_RESULT_ENTRIES = 500;\nconst MAX_N_SAMPLES = 1000;\n\nconst ScoresSchema = z\n  .object({\n    exploration: z.number().min(0).max(1).optional(),\n    tool_reliability: z.number().min(0).max(1).optional(),\n    self_stop: z.number().min(0).max(1).optional(),\n    parallelism: z.number().min(0).max(1).optional(),\n    efficiency: z.number().min(0).max(1).optional(),\n    instruction_following: z.number().min(0).max(1).optional(),\n  })\n  .strict();\n\n// Domain enum mirrors `prompts.ts` — both layers reject anything outside the\n// whitelist so a hand-crafted results JSON can't push arbitrary keys into\n// `entry.afs.domains` (plan.md L1089 — Step 3.2 Security). `.strict()`\n// rejects unknown keys; each enum value is `.optional()` because a partial\n// matrix (eg. only `english` prompts ran) shouldn't fail validation.\nconst DomainsSchema = z\n  .object({\n    chinese: z.number().min(0).max(1).optional(),\n    english: z.number().min(0).max(1).optional(),\n    coding: z.number().min(0).max(1).optional(),\n  })\n  .strict();\n\nconst ResultEntrySchema = z\n  .object({\n    n_samples: z.number().int().min(1).max(MAX_N_SAMPLES),\n    scores: ScoresSchema,\n    domains: DomainsSchema.optional(),\n    stddev: z.record(z.string(), z.number()).optional(),\n    // Per-prompt mean scores; structurally `{ <prompt_id>: <ScoresSchema> }`.\n    // Validated but not propagated into entry.afs — the catalog only stores\n    // the aggregated `scores`.\n    per_prompt: z.record(z.string().regex(/^[A-Za-z0-9._-]+$/), ScoresSchema).optional(),\n    // Per-sample failures recorded by runMatrixFull; passthrough only.\n    errors: z.array(z.unknown()).optional(),\n  })\n  .strict();\n\n// Judge id pattern matches `JUDGE_MODEL_PATTERN` in judge.ts — both layers\n// reject path-traversal tokens so the catalog never persists a hostile id.\nconst JudgeIdSchema = z.string().regex(/^[A-Za-z0-9._-]+$/);\nconst JudgeModelSchema = z\n  .union([JudgeIdSchema, z.array(JudgeIdSchema).min(1)])\n  .nullable()\n  .optional();\n\nconst ResultsRootSchema = z\n  .object({\n    version: z.string().min(1),\n    judge_model: JudgeModelSchema,\n    config: z.unknown().optional(),\n    results: z.record(z.string().regex(/^[A-Za-z0-9._-]+$/), ResultEntrySchema),\n    // Run 4 phase 3 (outlier filter) emits `outliers: Array<{...}>` on the\n    // results root. patcher.ts treats it as audit-only metadata — accepted\n    // but not propagated into `entry.afs`.\n    outliers: z.unknown().optional(),\n  })\n  .strict();\n\nexport interface ValidatedResults {\n  version: string;\n  judge_model: string | string[] | null;\n  results: Record<string, ResultEntry>;\n}\n\ninterface ResultEntry {\n  n_samples: number;\n  scores: Record<string, number>;\n  domains?: Record<string, number>;\n}\n\n/**\n * Validate raw JSON shape + reject reserved/sensitive keys anywhere.\n * Throws `ValidationError` on any rejection.\n *\n * Layers (in order):\n *   1. Deep-walk: reject `__proto__` / `constructor` / `prototype` and any\n *      key matching a credential pattern (`api_key`, `Authorization`, …).\n *   2. Resource-exhaustion guard: results entries ≤ MAX_RESULT_ENTRIES.\n *   3. Per-entry whitelist: only `n_samples`/`scores`/`domains`/`stddev`.\n *   4. Zod schema parse (z.strict() also catches unknown scores dimensions).\n */\nexport function validateResultsShape(raw: unknown): ValidatedResults {\n  rejectDangerousKeysDeep(raw);\n\n  if (raw && typeof raw === \"object\" && !Array.isArray(raw)) {\n    const r = (raw as Record<string, unknown>).results;\n    if (r && typeof r === \"object\" && !Array.isArray(r)) {\n      const keys = Object.keys(r);\n      if (keys.length > MAX_RESULT_ENTRIES) {\n        throw new ValidationError(\n          `results contains ${keys.length} entries; max ${MAX_RESULT_ENTRIES}`,\n        );\n      }\n      for (const k of keys) {\n        const entry = (r as Record<string, unknown>)[k];\n        if (entry && typeof entry === \"object\" && !Array.isArray(entry)) {\n          for (const f of Object.keys(entry as Record<string, unknown>)) {\n            if (!ALLOWED_RESULT_ENTRY_FIELDS.has(f)) {\n              throw new ValidationError(`unknown field \"${f}\" in results.${k}`);\n            }\n          }\n        }\n      }\n    }\n  }\n\n  const parsed = ResultsRootSchema.safeParse(raw);\n  if (!parsed.success) {\n    throw new ValidationError(`results JSON is invalid: ${parsed.error.message}`);\n  }\n  return {\n    version: parsed.data.version,\n    judge_model: parsed.data.judge_model ?? null,\n    // Single-judge writes a string; multi-judge writes an array — in the\n    // catalog `entry.afs.judge` mirrors that shape (plan.md L1052).\n    results: parsed.data.results as Record<string, ResultEntry>,\n  };\n}\n\nfunction rejectDangerousKeysDeep(value: unknown, p = \"$\"): void {\n  if (value === null || typeof value !== \"object\") return;\n  if (Array.isArray(value)) {\n    for (let i = 0; i < value.length; i++) rejectDangerousKeysDeep(value[i], `${p}[${i}]`);\n    return;\n  }\n  for (const k of Object.keys(value as Record<string, unknown>)) {\n    if (RESERVED_KEYS.has(k)) {\n      throw new ValidationError(`reserved key \"${k}\" not allowed at ${p}`);\n    }\n    for (const re of SENSITIVE_KEY_PATTERNS) {\n      if (re.test(k)) {\n        throw new ValidationError(`sensitive field \"${k}\" not allowed at ${p}`);\n      }\n    }\n    rejectDangerousKeysDeep((value as Record<string, unknown>)[k], `${p}.${k}`);\n  }\n}\n\n// ─── Path safety ──────────────────────────────────────────────────────────\n\n/**\n * Resolve `p` against cwd if relative, then ensure it lives inside `resultsRoot`.\n * Defends against `..` segments and absolute paths that escape the root.\n * Returns the absolute path on success; throws `ValidationError` otherwise.\n *\n * Note: this is purely lexical (no `fs.realpath`) — a maintainer-created\n * symlink inside `resultsRoot` pointing outside is not caught here. The threat\n * model is a maintainer tool, so the lexical check is intentionally cheap; if\n * a symlink-based vector materialises in production, swap to `fs.realpath`.\n */\nexport function assertWithinResultsDir(p: string, resultsRoot: string): string {\n  const absRoot = path.resolve(resultsRoot);\n  const abs = path.resolve(p);\n  const rel = path.relative(absRoot, abs);\n  if (rel === \"\" || rel.startsWith(\"..\") || path.isAbsolute(rel)) {\n    throw new ValidationError(`results path outside ${resultsRoot}: ${p}`);\n  }\n  return abs;\n}\n\n// ─── Stable-key serialisation ────────────────────────────────────────────\n\n/**\n * Return a deeply-key-sorted clone of an `afs` block. Plain objects get keys\n * in alphabetical order; arrays preserve order; primitives pass through.\n * Does not mutate input.\n */\nexport function sortAfsBlock<T>(value: T): T {\n  if (value === null || typeof value !== \"object\") return value;\n  if (Array.isArray(value)) return value.map((v) => sortAfsBlock(v)) as unknown as T;\n  const sorted: Record<string, unknown> = {};\n  for (const k of Object.keys(value as Record<string, unknown>).sort()) {\n    sorted[k] = sortAfsBlock((value as Record<string, unknown>)[k]);\n  }\n  return sorted as T;\n}\n\n// ─── Atomic JSON write ───────────────────────────────────────────────────\n\n// Match biome's pretty-print for JSON: indent 2 spaces, inline arrays of\n// primitives when the line fits within lineWidth=100. Without this, every\n// `JSON.stringify(catalog, null, 2)` re-expands `aliases` / `recommendedFor`\n// / `modality.input` etc. into multi-line form, and the next `pnpm format`\n// run in CI immediately churns 1000+ lines of diff back. The maintainer\n// patcher writes the catalog once per benchmark — it must not fight the\n// formatter on every cycle.\nconst BIOME_LINE_WIDTH = 100;\nconst BIOME_INDENT = \"  \";\n\nfunction stringifyBiomeJSON(value: unknown, currentIndent = \"\", startColumn = 0): string {\n  if (value === null) return \"null\";\n  if (typeof value === \"boolean\" || typeof value === \"number\") return String(value);\n  if (typeof value === \"string\") return JSON.stringify(value);\n\n  if (Array.isArray(value)) {\n    if (value.length === 0) return \"[]\";\n    const allPrimitive = value.every((v) => v === null || typeof v !== \"object\");\n    if (allPrimitive) {\n      const inline = `[${value.map((v) => stringifyBiomeJSON(v)).join(\", \")}]`;\n      if (startColumn + inline.length <= BIOME_LINE_WIDTH) return inline;\n    }\n    const inner = currentIndent + BIOME_INDENT;\n    const items = value.map((v) => `${inner}${stringifyBiomeJSON(v, inner, inner.length)}`);\n    return `[\\n${items.join(\",\\n\")}\\n${currentIndent}]`;\n  }\n\n  if (typeof value !== \"object\") return JSON.stringify(value);\n\n  const entries = Object.entries(value as Record<string, unknown>);\n  if (entries.length === 0) return \"{}\";\n  const inner = currentIndent + BIOME_INDENT;\n  const lines = entries.map(([k, v]) => {\n    const keyPart = `${inner}${JSON.stringify(k)}: `;\n    return `${keyPart}${stringifyBiomeJSON(v, inner, keyPart.length)}`;\n  });\n  return `{\\n${lines.join(\",\\n\")}\\n${currentIndent}}`;\n}\n\n/** Write `obj` as biome-compatible pretty-printed JSON to `target` via tmp+rename. */\nexport async function atomicWriteJSON(target: string, obj: unknown): Promise<void> {\n  const dir = path.dirname(target);\n  await fs.mkdir(dir, { recursive: true });\n  const tmp = path.join(dir, `.tmp-${randomBytes(6).toString(\"hex\")}.json`);\n  const payload = `${stringifyBiomeJSON(obj)}\\n`;\n  await fs.writeFile(tmp, payload, \"utf8\");\n  try {\n    await fs.rename(tmp, target);\n  } catch (err) {\n    await fs.rm(tmp, { force: true });\n    throw err;\n  }\n}\n\n// ─── Scoring helpers ─────────────────────────────────────────────────────\n\nfunction meanOf(xs: number[]): number {\n  if (xs.length === 0) return 0;\n  let s = 0;\n  for (const x of xs) s += x;\n  return s / xs.length;\n}\n\nfunction getNested(obj: unknown, p: string): unknown {\n  const parts = p.split(\".\");\n  let cur: unknown = obj;\n  for (const k of parts) {\n    if (cur === null || typeof cur !== \"object\") return undefined;\n    cur = (cur as Record<string, unknown>)[k];\n  }\n  return cur;\n}\n\nfunction computeWeightedScore(entry: ResultEntry, weights: Record<string, number>): number {\n  let total = 0;\n  for (const [pathKey, w] of Object.entries(weights)) {\n    const v = getNested(entry, pathKey);\n    if (typeof v === \"number\" && Number.isFinite(v)) total += v * w;\n  }\n  return total;\n}\n\n// ─── patchCatalog ────────────────────────────────────────────────────────\n\nexport async function patchCatalog(input: PatchCatalogInput): Promise<PatchSummary> {\n  const safeResultsPath = assertWithinResultsDir(input.resultsPath, input.resultsRoot);\n\n  let raw: string;\n  try {\n    raw = await fs.readFile(safeResultsPath, \"utf8\");\n  } catch (err) {\n    throw new ValidationError(\n      `failed to read results: ${err instanceof Error ? err.message : String(err)}`,\n    );\n  }\n\n  let parsedRaw: unknown;\n  try {\n    parsedRaw = JSON.parse(raw);\n  } catch (err) {\n    throw new ValidationError(\n      `results is not valid JSON: ${err instanceof Error ? err.message : String(err)}`,\n    );\n  }\n\n  const validated = validateResultsShape(parsedRaw);\n\n  const catalogText = await fs.readFile(input.catalogPath, \"utf8\");\n  const catalog = JSON.parse(catalogText) as CatalogEntry[];\n  if (!Array.isArray(catalog)) {\n    throw new ValidationError(\"catalog.json must be an array of entries\");\n  }\n\n  const summary: PatchSummary = { changes: [], skipped: [] };\n\n  // Skip the AFS round-trip for empty catalogs — nothing to score.\n  if (catalog.length === 0) return summary;\n\n  // Pull weights up front (one round-trip, then a parallel fanout in the\n  // bench wiring) so the per-entry loop is sync. Validate shape at the\n  // choke point — refuse to write rather than persist garbage.\n  const weightsTable = await input.loadPolicyWeights();\n  if (!weightsTable || typeof weightsTable !== \"object\" || Array.isArray(weightsTable)) {\n    throw new ValidationError(\n      \"loadPolicyWeights must return a plain object of { policy: weights }\",\n    );\n  }\n  for (const [policy, weights] of Object.entries(weightsTable)) {\n    if (!weights || typeof weights !== \"object\" || Array.isArray(weights)) {\n      throw new ValidationError(`weights for policy \"${policy}\" must be a plain object`);\n    }\n  }\n\n  for (const entry of catalog) {\n    const r = validated.results[entry.model];\n    if (!r) {\n      summary.skipped.push(entry.model);\n      continue;\n    }\n    const before = entry.afs;\n\n    const policyScores: Record<string, number> = {};\n    for (const [name, w] of Object.entries(weightsTable)) {\n      policyScores[name] = computeWeightedScore(r, w);\n    }\n\n    const afsBlock: AFSScores = {\n      status: \"benchmarked\",\n      version: validated.version,\n      last_seen_in_run: validated.version,\n      judge: validated.judge_model,\n      samples: r.n_samples,\n      scores: { ...r.scores },\n      overall: meanOf(Object.values(r.scores)),\n      policyScores,\n      ...(r.domains ? { domains: { ...r.domains } } : {}),\n    };\n\n    entry.afs = sortAfsBlock(afsBlock);\n    summary.changes.push({ model: entry.model, before, after: entry.afs });\n  }\n\n  await atomicWriteJSON(input.catalogPath, catalog);\n\n  // Phase 5: surface refresh failures — the catalog write already succeeded,\n  // but the caller needs to know the device snapshot is stale until the\n  // periodic refresh runs.\n  if (input.refreshRegistry) await input.refreshRegistry();\n\n  // The ai-device package imports `catalog.json` at build time\n  // (`providers/ai/device/src/index.ts:42`), so a running daemon caches the\n  // pre-patch catalog snapshot. `refreshRegistry` rebuilds the hub registry\n  // from the cached catalog; it does NOT pick up the new file. Until that\n  // import becomes a runtime read, the maintainer flow is:\n  //\n  //   1. patchCatalog (this function) — file written ✓, registry refreshed ✓\n  //   2. pnpm --filter @aigne/afs-ai-device --filter @aigne/arc-cli build\n  //   3. arc service restart\n  //\n  // Without 2+3 the new scores are visible via `arc afs read /catalog/<id>`\n  // but `/dev/ai/.actions/chat` policy ranking still uses old scores.\n  if (summary.changes.length > 0) {\n    log.warn(\n      `[patchCatalog] ${summary.changes.length} catalog entr${summary.changes.length === 1 ? \"y\" : \"ies\"} updated. ` +\n        \"Run `pnpm --filter @aigne/afs-ai-device --filter @aigne/arc-cli build && arc service restart` \" +\n        \"to make the daemon's policy router pick them up — the catalog is bundled into ai-device's dist \" +\n        \"and refreshRegistry only rebuilds the hub registry, not the catalog.\",\n    );\n  }\n\n  return summary;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAiCA,MAAM,MAAM,UAAU,qBAAqB;AAuD3C,MAAM,gBAAgB,IAAI,IAAI;CAC5B;CACA;CACA;CACA;CACA;CACA;CACA;CACD,CAAC;AAEF,MAAM,yBAAmC;CACvC;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACD;AAOD,MAAM,8BAA8B,IAAI,IAAI;CAC1C;CACA;CACA;CACA;CACA;CACA;CACD,CAAC;AAEF,MAAM,qBAAqB;AAC3B,MAAM,gBAAgB;AAEtB,MAAM,eAAe,EAClB,OAAO;CACN,aAAa,EAAE,QAAQ,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,UAAU;CAChD,kBAAkB,EAAE,QAAQ,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,UAAU;CACrD,WAAW,EAAE,QAAQ,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,UAAU;CAC9C,aAAa,EAAE,QAAQ,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,UAAU;CAChD,YAAY,EAAE,QAAQ,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,UAAU;CAC/C,uBAAuB,EAAE,QAAQ,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,UAAU;CAC3D,CAAC,CACD,QAAQ;AAOX,MAAM,gBAAgB,EACnB,OAAO;CACN,SAAS,EAAE,QAAQ,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,UAAU;CAC5C,SAAS,EAAE,QAAQ,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,UAAU;CAC5C,QAAQ,EAAE,QAAQ,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,UAAU;CAC5C,CAAC,CACD,QAAQ;AAEX,MAAM,oBAAoB,EACvB,OAAO;CACN,WAAW,EAAE,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,IAAI,cAAc;CACrD,QAAQ;CACR,SAAS,cAAc,UAAU;CACjC,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,QAAQ,CAAC,CAAC,UAAU;CAInD,YAAY,EAAE,OAAO,EAAE,QAAQ,CAAC,MAAM,oBAAoB,EAAE,aAAa,CAAC,UAAU;CAEpF,QAAQ,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC,UAAU;CACxC,CAAC,CACD,QAAQ;AAIX,MAAM,gBAAgB,EAAE,QAAQ,CAAC,MAAM,oBAAoB;AAC3D,MAAM,mBAAmB,EACtB,MAAM,CAAC,eAAe,EAAE,MAAM,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC,CACrD,UAAU,CACV,UAAU;AAEb,MAAM,oBAAoB,EACvB,OAAO;CACN,SAAS,EAAE,QAAQ,CAAC,IAAI,EAAE;CAC1B,aAAa;CACb,QAAQ,EAAE,SAAS,CAAC,UAAU;CAC9B,SAAS,EAAE,OAAO,EAAE,QAAQ,CAAC,MAAM,oBAAoB,EAAE,kBAAkB;CAI3E,UAAU,EAAE,SAAS,CAAC,UAAU;CACjC,CAAC,CACD,QAAQ;;;;;;;;;;;;AAyBX,SAAgB,qBAAqB,KAAgC;AACnE,yBAAwB,IAAI;AAE5B,KAAI,OAAO,OAAO,QAAQ,YAAY,CAAC,MAAM,QAAQ,IAAI,EAAE;EACzD,MAAM,IAAK,IAAgC;AAC3C,MAAI,KAAK,OAAO,MAAM,YAAY,CAAC,MAAM,QAAQ,EAAE,EAAE;GACnD,MAAM,OAAO,OAAO,KAAK,EAAE;AAC3B,OAAI,KAAK,SAAS,mBAChB,OAAM,IAAI,gBACR,oBAAoB,KAAK,OAAO,gBAAgB,qBACjD;AAEH,QAAK,MAAM,KAAK,MAAM;IACpB,MAAM,QAAS,EAA8B;AAC7C,QAAI,SAAS,OAAO,UAAU,YAAY,CAAC,MAAM,QAAQ,MAAM,EAC7D;UAAK,MAAM,KAAK,OAAO,KAAK,MAAiC,CAC3D,KAAI,CAAC,4BAA4B,IAAI,EAAE,CACrC,OAAM,IAAI,gBAAgB,kBAAkB,EAAE,eAAe,IAAI;;;;;CAQ7E,MAAM,SAAS,kBAAkB,UAAU,IAAI;AAC/C,KAAI,CAAC,OAAO,QACV,OAAM,IAAI,gBAAgB,4BAA4B,OAAO,MAAM,UAAU;AAE/E,QAAO;EACL,SAAS,OAAO,KAAK;EACrB,aAAa,OAAO,KAAK,eAAe;EAGxC,SAAS,OAAO,KAAK;EACtB;;AAGH,SAAS,wBAAwB,OAAgB,IAAI,KAAW;AAC9D,KAAI,UAAU,QAAQ,OAAO,UAAU,SAAU;AACjD,KAAI,MAAM,QAAQ,MAAM,EAAE;AACxB,OAAK,IAAI,IAAI,GAAG,IAAI,MAAM,QAAQ,IAAK,yBAAwB,MAAM,IAAI,GAAG,EAAE,GAAG,EAAE,GAAG;AACtF;;AAEF,MAAK,MAAM,KAAK,OAAO,KAAK,MAAiC,EAAE;AAC7D,MAAI,cAAc,IAAI,EAAE,CACtB,OAAM,IAAI,gBAAgB,iBAAiB,EAAE,mBAAmB,IAAI;AAEtE,OAAK,MAAM,MAAM,uBACf,KAAI,GAAG,KAAK,EAAE,CACZ,OAAM,IAAI,gBAAgB,oBAAoB,EAAE,mBAAmB,IAAI;AAG3E,0BAAyB,MAAkC,IAAI,GAAG,EAAE,GAAG,IAAI;;;;;;;;;;;;;AAgB/E,SAAgB,uBAAuB,GAAW,aAA6B;CAC7E,MAAM,UAAU,KAAK,QAAQ,YAAY;CACzC,MAAM,MAAM,KAAK,QAAQ,EAAE;CAC3B,MAAM,MAAM,KAAK,SAAS,SAAS,IAAI;AACvC,KAAI,QAAQ,MAAM,IAAI,WAAW,KAAK,IAAI,KAAK,WAAW,IAAI,CAC5D,OAAM,IAAI,gBAAgB,wBAAwB,YAAY,IAAI,IAAI;AAExE,QAAO;;;;;;;AAUT,SAAgB,aAAgB,OAAa;AAC3C,KAAI,UAAU,QAAQ,OAAO,UAAU,SAAU,QAAO;AACxD,KAAI,MAAM,QAAQ,MAAM,CAAE,QAAO,MAAM,KAAK,MAAM,aAAa,EAAE,CAAC;CAClE,MAAM,SAAkC,EAAE;AAC1C,MAAK,MAAM,KAAK,OAAO,KAAK,MAAiC,CAAC,MAAM,CAClE,QAAO,KAAK,aAAc,MAAkC,GAAG;AAEjE,QAAO;;AAYT,MAAM,mBAAmB;AACzB,MAAM,eAAe;AAErB,SAAS,mBAAmB,OAAgB,gBAAgB,IAAI,cAAc,GAAW;AACvF,KAAI,UAAU,KAAM,QAAO;AAC3B,KAAI,OAAO,UAAU,aAAa,OAAO,UAAU,SAAU,QAAO,OAAO,MAAM;AACjF,KAAI,OAAO,UAAU,SAAU,QAAO,KAAK,UAAU,MAAM;AAE3D,KAAI,MAAM,QAAQ,MAAM,EAAE;AACxB,MAAI,MAAM,WAAW,EAAG,QAAO;AAE/B,MADqB,MAAM,OAAO,MAAM,MAAM,QAAQ,OAAO,MAAM,SAAS,EAC1D;GAChB,MAAM,SAAS,IAAI,MAAM,KAAK,MAAM,mBAAmB,EAAE,CAAC,CAAC,KAAK,KAAK,CAAC;AACtE,OAAI,cAAc,OAAO,UAAU,iBAAkB,QAAO;;EAE9D,MAAMA,UAAQ,gBAAgB;AAE9B,SAAO,MADO,MAAM,KAAK,MAAM,GAAGA,UAAQ,mBAAmB,GAAGA,SAAOA,QAAM,OAAO,GAAG,CACpE,KAAK,MAAM,CAAC,IAAI,cAAc;;AAGnD,KAAI,OAAO,UAAU,SAAU,QAAO,KAAK,UAAU,MAAM;CAE3D,MAAM,UAAU,OAAO,QAAQ,MAAiC;AAChE,KAAI,QAAQ,WAAW,EAAG,QAAO;CACjC,MAAM,QAAQ,gBAAgB;AAK9B,QAAO,MAJO,QAAQ,KAAK,CAAC,GAAG,OAAO;EACpC,MAAM,UAAU,GAAG,QAAQ,KAAK,UAAU,EAAE,CAAC;AAC7C,SAAO,GAAG,UAAU,mBAAmB,GAAG,OAAO,QAAQ,OAAO;GAChE,CACiB,KAAK,MAAM,CAAC,IAAI,cAAc;;;AAInD,eAAsB,gBAAgB,QAAgB,KAA6B;CACjF,MAAM,MAAM,KAAK,QAAQ,OAAO;AAChC,OAAMC,SAAG,MAAM,KAAK,EAAE,WAAW,MAAM,CAAC;CACxC,MAAM,MAAM,KAAK,KAAK,KAAK,QAAQ,YAAY,EAAE,CAAC,SAAS,MAAM,CAAC,OAAO;CACzE,MAAM,UAAU,GAAG,mBAAmB,IAAI,CAAC;AAC3C,OAAMA,SAAG,UAAU,KAAK,SAAS,OAAO;AACxC,KAAI;AACF,QAAMA,SAAG,OAAO,KAAK,OAAO;UACrB,KAAK;AACZ,QAAMA,SAAG,GAAG,KAAK,EAAE,OAAO,MAAM,CAAC;AACjC,QAAM;;;AAMV,SAAS,OAAO,IAAsB;AACpC,KAAI,GAAG,WAAW,EAAG,QAAO;CAC5B,IAAI,IAAI;AACR,MAAK,MAAM,KAAK,GAAI,MAAK;AACzB,QAAO,IAAI,GAAG;;AAGhB,SAAS,UAAU,KAAc,GAAoB;CACnD,MAAM,QAAQ,EAAE,MAAM,IAAI;CAC1B,IAAI,MAAe;AACnB,MAAK,MAAM,KAAK,OAAO;AACrB,MAAI,QAAQ,QAAQ,OAAO,QAAQ,SAAU,QAAO;AACpD,QAAO,IAAgC;;AAEzC,QAAO;;AAGT,SAAS,qBAAqB,OAAoB,SAAyC;CACzF,IAAI,QAAQ;AACZ,MAAK,MAAM,CAAC,SAAS,MAAM,OAAO,QAAQ,QAAQ,EAAE;EAClD,MAAM,IAAI,UAAU,OAAO,QAAQ;AACnC,MAAI,OAAO,MAAM,YAAY,OAAO,SAAS,EAAE,CAAE,UAAS,IAAI;;AAEhE,QAAO;;AAKT,eAAsB,aAAa,OAAiD;CAClF,MAAM,kBAAkB,uBAAuB,MAAM,aAAa,MAAM,YAAY;CAEpF,IAAI;AACJ,KAAI;AACF,QAAM,MAAMA,SAAG,SAAS,iBAAiB,OAAO;UACzC,KAAK;AACZ,QAAM,IAAI,gBACR,2BAA2B,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI,GAC5E;;CAGH,IAAI;AACJ,KAAI;AACF,cAAY,KAAK,MAAM,IAAI;UACpB,KAAK;AACZ,QAAM,IAAI,gBACR,8BAA8B,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI,GAC/E;;CAGH,MAAM,YAAY,qBAAqB,UAAU;CAEjD,MAAM,cAAc,MAAMA,SAAG,SAAS,MAAM,aAAa,OAAO;CAChE,MAAM,UAAU,KAAK,MAAM,YAAY;AACvC,KAAI,CAAC,MAAM,QAAQ,QAAQ,CACzB,OAAM,IAAI,gBAAgB,2CAA2C;CAGvE,MAAM,UAAwB;EAAE,SAAS,EAAE;EAAE,SAAS,EAAE;EAAE;AAG1D,KAAI,QAAQ,WAAW,EAAG,QAAO;CAKjC,MAAM,eAAe,MAAM,MAAM,mBAAmB;AACpD,KAAI,CAAC,gBAAgB,OAAO,iBAAiB,YAAY,MAAM,QAAQ,aAAa,CAClF,OAAM,IAAI,gBACR,sEACD;AAEH,MAAK,MAAM,CAAC,QAAQ,YAAY,OAAO,QAAQ,aAAa,CAC1D,KAAI,CAAC,WAAW,OAAO,YAAY,YAAY,MAAM,QAAQ,QAAQ,CACnE,OAAM,IAAI,gBAAgB,uBAAuB,OAAO,0BAA0B;AAItF,MAAK,MAAM,SAAS,SAAS;EAC3B,MAAM,IAAI,UAAU,QAAQ,MAAM;AAClC,MAAI,CAAC,GAAG;AACN,WAAQ,QAAQ,KAAK,MAAM,MAAM;AACjC;;EAEF,MAAM,SAAS,MAAM;EAErB,MAAM,eAAuC,EAAE;AAC/C,OAAK,MAAM,CAAC,MAAM,MAAM,OAAO,QAAQ,aAAa,CAClD,cAAa,QAAQ,qBAAqB,GAAG,EAAE;AAejD,QAAM,MAAM,aAZgB;GAC1B,QAAQ;GACR,SAAS,UAAU;GACnB,kBAAkB,UAAU;GAC5B,OAAO,UAAU;GACjB,SAAS,EAAE;GACX,QAAQ,EAAE,GAAG,EAAE,QAAQ;GACvB,SAAS,OAAO,OAAO,OAAO,EAAE,OAAO,CAAC;GACxC;GACA,GAAI,EAAE,UAAU,EAAE,SAAS,EAAE,GAAG,EAAE,SAAS,EAAE,GAAG,EAAE;GACnD,CAEiC;AAClC,UAAQ,QAAQ,KAAK;GAAE,OAAO,MAAM;GAAO;GAAQ,OAAO,MAAM;GAAK,CAAC;;AAGxE,OAAM,gBAAgB,MAAM,aAAa,QAAQ;AAKjD,KAAI,MAAM,gBAAiB,OAAM,MAAM,iBAAiB;AAcxD,KAAI,QAAQ,QAAQ,SAAS,EAC3B,KAAI,KACF,kBAAkB,QAAQ,QAAQ,OAAO,eAAe,QAAQ,QAAQ,WAAW,IAAI,MAAM,MAAM,+QAIpG;AAGH,QAAO"}

package/dist/prompts.cjs

@@ -0,0 +1,123 @@
+const require_rolldown_runtime = require('./_virtual/rolldown_runtime.cjs');
+let _aigne_afs = require("@aigne/afs");
+let zod = require("zod");
+let gray_matter = require("gray-matter");
+gray_matter = require_rolldown_runtime.__toESM(gray_matter);
+let js_yaml = require("js-yaml");
+
+//#region src/prompts.ts
+/**
+* Prompt loader for the LLM-AFS benchmark suite.
+*
+* Each prompt is a Markdown file with YAML frontmatter (`gray-matter`-parsed)
+* and a body sliced by H1 sections (`# Task`, `# System prompt`, etc.).
+*
+* Phase 0 only needs the load + parse path so that `list /bench/prompts` and
+* `read /bench/prompts/<id>` return real entries. Runner / patcher consume the
+* parsed shape in Phase 1.
+*/
+const GRAY_MATTER_OPTIONS = { engines: { yaml: {
+	parse: (str) => (0, js_yaml.load)(str),
+	stringify: (data) => (0, js_yaml.dump)(data)
+} } };
+const promptFrontmatterSchema = zod.z.object({
+	id: zod.z.string().min(1, "frontmatter.id is required"),
+	title: zod.z.string().min(1, "frontmatter.title is required"),
+	domain: zod.z.enum([
+		"chinese",
+		"english",
+		"coding"
+	]).optional(),
+	max_rounds: zod.z.number().int().positive("max_rounds must be a positive integer"),
+	weight_dim: zod.z.string().optional()
+});
+/**
+* Slice markdown body into sections keyed by H1 heading text.
+*
+* Section headings are case-sensitive and trimmed. Content before the first H1
+* is dropped — by convention every benchmark prompt starts with `# Task`.
+*/
+function sliceBodyByH1(body) {
+	const out = {};
+	const lines = body.split(/\r?\n/);
+	let currentName;
+	let currentBuf = [];
+	const flush = () => {
+		if (currentName !== void 0) out[currentName] = currentBuf.join("\n").trim();
+	};
+	for (const line of lines) {
+		const match = /^#\s+(.+?)\s*$/.exec(line);
+		if (match && !line.startsWith("##")) {
+			flush();
+			currentName = match[1].trim();
+			currentBuf = [];
+		} else if (currentName !== void 0) currentBuf.push(line);
+	}
+	flush();
+	return out;
+}
+/**
+* Parse a single prompt file. Throws `BadPromptError` (regular Error subclass)
+* if frontmatter is missing required fields.
+*/
+var BadPromptError = class extends Error {
+	name = "BadPromptError";
+	constructor(filename, message) {
+		super(`[${filename}] ${message}`);
+		this.filename = filename;
+	}
+};
+function parsePrompt(filename, raw) {
+	const parsed = (0, gray_matter.default)(raw, GRAY_MATTER_OPTIONS);
+	const fm = promptFrontmatterSchema.safeParse(parsed.data);
+	if (!fm.success) throw new BadPromptError(filename, `invalid frontmatter — ${fm.error.issues.map((i) => `${i.path.join(".")}: ${i.message}`).join("; ")}`);
+	const body = parsed.content.replace(/^\s+/, "");
+	return {
+		id: fm.data.id,
+		filename,
+		frontmatter: fm.data,
+		body,
+		sections: sliceBodyByH1(body)
+	};
+}
+/**
+* Load every `*.md` file under `dir` and parse it into a Prompt. Missing
+* directory → empty array (Phase 0 must not crash on unconfigured promptsDir).
+*
+* Files that fail frontmatter validation are skipped with their error returned
+* in `errors` so callers can surface them (Phase 1 runner, conformance probe).
+*/
+async function loadPromptsFromDir(dir) {
+	const platform = (0, _aigne_afs.getPlatform)();
+	const fs = platform.fs;
+	const path = platform.path;
+	if (!fs) return {
+		prompts: [],
+		errors: []
+	};
+	if (!await fs.exists(dir)) return {
+		prompts: [],
+		errors: []
+	};
+	const entries = await fs.readdir(dir);
+	const prompts = [];
+	const errors = [];
+	for (const filename of entries.slice().sort()) {
+		if (path.extname(filename).toLowerCase() !== ".md") continue;
+		const raw = await fs.readTextFile(path.join(dir, filename));
+		try {
+			prompts.push(parsePrompt(filename, raw));
+		} catch (err) {
+			if (err instanceof BadPromptError) errors.push(err);
+			else throw err;
+		}
+	}
+	return {
+		prompts,
+		errors
+	};
+}
+
+//#endregion
+exports.BadPromptError = BadPromptError;
+exports.loadPromptsFromDir = loadPromptsFromDir;

package/dist/prompts.d.cts

@@ -0,0 +1,30 @@
+import { z } from "zod";
+
+//#region src/prompts.d.ts
+declare const promptFrontmatterSchema: z.ZodObject<{
+  id: z.ZodString;
+  title: z.ZodString;
+  domain: z.ZodOptional<z.ZodEnum<{
+    chinese: "chinese";
+    english: "english";
+    coding: "coding";
+  }>>;
+  max_rounds: z.ZodNumber;
+  weight_dim: z.ZodOptional<z.ZodString>;
+}, z.core.$strip>;
+type PromptFrontmatter = z.infer<typeof promptFrontmatterSchema>;
+interface Prompt {
+  /** Stable id from frontmatter, used as the path segment under `/prompts/`. */
+  id: string;
+  /** Source filename relative to `promptsDir`. */
+  filename: string;
+  /** Validated frontmatter. */
+  frontmatter: PromptFrontmatter;
+  /** Raw markdown body (excludes frontmatter). */
+  body: string;
+  /** Body sliced by `# H1` headings — section name → section body. */
+  sections: Record<string, string>;
+}
+//#endregion
+export { Prompt, PromptFrontmatter };
+//# sourceMappingURL=prompts.d.cts.map

package/dist/prompts.d.cts.map

@@ -0,0 +1 @@
+{"version":3,"file":"prompts.d.cts","names":[],"sources":["../src/prompts.ts"],"mappings":";;;cA4BM,uBAAA,EAAuB,CAAA,CAAA,SAAA;;;;;;;;;;;KAQjB,iBAAA,GAAoB,CAAA,CAAE,KAAA,QAAa,uBAAA;AAAA,UAE9B,MAAA;;EAEf,EAAA;;EAEA,QAAA;;EAEA,WAAA,EAAa,iBAAA;;EAEb,IAAA;;EAEA,QAAA,EAAU,MAAA;AAAA"}

package/dist/prompts.d.mts

@@ -0,0 +1,30 @@
+import { z } from "zod";
+
+//#region src/prompts.d.ts
+declare const promptFrontmatterSchema: z.ZodObject<{
+  id: z.ZodString;
+  title: z.ZodString;
+  domain: z.ZodOptional<z.ZodEnum<{
+    chinese: "chinese";
+    english: "english";
+    coding: "coding";
+  }>>;
+  max_rounds: z.ZodNumber;
+  weight_dim: z.ZodOptional<z.ZodString>;
+}, z.core.$strip>;
+type PromptFrontmatter = z.infer<typeof promptFrontmatterSchema>;
+interface Prompt {
+  /** Stable id from frontmatter, used as the path segment under `/prompts/`. */
+  id: string;
+  /** Source filename relative to `promptsDir`. */
+  filename: string;
+  /** Validated frontmatter. */
+  frontmatter: PromptFrontmatter;
+  /** Raw markdown body (excludes frontmatter). */
+  body: string;
+  /** Body sliced by `# H1` headings — section name → section body. */
+  sections: Record<string, string>;
+}
+//#endregion
+export { Prompt, PromptFrontmatter };
+//# sourceMappingURL=prompts.d.mts.map

package/dist/prompts.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"prompts.d.mts","names":[],"sources":["../src/prompts.ts"],"mappings":";;;cA4BM,uBAAA,EAAuB,CAAA,CAAA,SAAA;;;;;;;;;;;KAQjB,iBAAA,GAAoB,CAAA,CAAE,KAAA,QAAa,uBAAA;AAAA,UAE9B,MAAA;;EAEf,EAAA;;EAEA,QAAA;;EAEA,WAAA,EAAa,iBAAA;;EAEb,IAAA;;EAEA,QAAA,EAAU,MAAA;AAAA"}

package/dist/prompts.mjs

@@ -0,0 +1,121 @@
+import { getPlatform } from "@aigne/afs";
+import { z } from "zod";
+import matter from "gray-matter";
+import { dump, load } from "js-yaml";
+
+//#region src/prompts.ts
+/**
+* Prompt loader for the LLM-AFS benchmark suite.
+*
+* Each prompt is a Markdown file with YAML frontmatter (`gray-matter`-parsed)
+* and a body sliced by H1 sections (`# Task`, `# System prompt`, etc.).
+*
+* Phase 0 only needs the load + parse path so that `list /bench/prompts` and
+* `read /bench/prompts/<id>` return real entries. Runner / patcher consume the
+* parsed shape in Phase 1.
+*/
+const GRAY_MATTER_OPTIONS = { engines: { yaml: {
+	parse: (str) => load(str),
+	stringify: (data) => dump(data)
+} } };
+const promptFrontmatterSchema = z.object({
+	id: z.string().min(1, "frontmatter.id is required"),
+	title: z.string().min(1, "frontmatter.title is required"),
+	domain: z.enum([
+		"chinese",
+		"english",
+		"coding"
+	]).optional(),
+	max_rounds: z.number().int().positive("max_rounds must be a positive integer"),
+	weight_dim: z.string().optional()
+});
+/**
+* Slice markdown body into sections keyed by H1 heading text.
+*
+* Section headings are case-sensitive and trimmed. Content before the first H1
+* is dropped — by convention every benchmark prompt starts with `# Task`.
+*/
+function sliceBodyByH1(body) {
+	const out = {};
+	const lines = body.split(/\r?\n/);
+	let currentName;
+	let currentBuf = [];
+	const flush = () => {
+		if (currentName !== void 0) out[currentName] = currentBuf.join("\n").trim();
+	};
+	for (const line of lines) {
+		const match = /^#\s+(.+?)\s*$/.exec(line);
+		if (match && !line.startsWith("##")) {
+			flush();
+			currentName = match[1].trim();
+			currentBuf = [];
+		} else if (currentName !== void 0) currentBuf.push(line);
+	}
+	flush();
+	return out;
+}
+/**
+* Parse a single prompt file. Throws `BadPromptError` (regular Error subclass)
+* if frontmatter is missing required fields.
+*/
+var BadPromptError = class extends Error {
+	name = "BadPromptError";
+	constructor(filename, message) {
+		super(`[${filename}] ${message}`);
+		this.filename = filename;
+	}
+};
+function parsePrompt(filename, raw) {
+	const parsed = matter(raw, GRAY_MATTER_OPTIONS);
+	const fm = promptFrontmatterSchema.safeParse(parsed.data);
+	if (!fm.success) throw new BadPromptError(filename, `invalid frontmatter — ${fm.error.issues.map((i) => `${i.path.join(".")}: ${i.message}`).join("; ")}`);
+	const body = parsed.content.replace(/^\s+/, "");
+	return {
+		id: fm.data.id,
+		filename,
+		frontmatter: fm.data,
+		body,
+		sections: sliceBodyByH1(body)
+	};
+}
+/**
+* Load every `*.md` file under `dir` and parse it into a Prompt. Missing
+* directory → empty array (Phase 0 must not crash on unconfigured promptsDir).
+*
+* Files that fail frontmatter validation are skipped with their error returned
+* in `errors` so callers can surface them (Phase 1 runner, conformance probe).
+*/
+async function loadPromptsFromDir(dir) {
+	const platform = getPlatform();
+	const fs = platform.fs;
+	const path = platform.path;
+	if (!fs) return {
+		prompts: [],
+		errors: []
+	};
+	if (!await fs.exists(dir)) return {
+		prompts: [],
+		errors: []
+	};
+	const entries = await fs.readdir(dir);
+	const prompts = [];
+	const errors = [];
+	for (const filename of entries.slice().sort()) {
+		if (path.extname(filename).toLowerCase() !== ".md") continue;
+		const raw = await fs.readTextFile(path.join(dir, filename));
+		try {
+			prompts.push(parsePrompt(filename, raw));
+		} catch (err) {
+			if (err instanceof BadPromptError) errors.push(err);
+			else throw err;
+		}
+	}
+	return {
+		prompts,
+		errors
+	};
+}
+
+//#endregion
+export { BadPromptError, loadPromptsFromDir };
+//# sourceMappingURL=prompts.mjs.map