@archsight/aios 1.2.0 → 1.3.1

package/scripts/analyze-prompt-run-results.mjs

@@ -0,0 +1,187 @@
+#!/usr/bin/env node
+
+import fs from "node:fs";
+import path from "node:path";
+
+const root = fs.realpathSync(process.cwd());
+const errors = [];
+
+const fixturePath = repoPath("prompts/evaluations/engineering-business-basic-fixtures.json");
+const scorecardPath = repoPath("prompts/evaluations/engineering-business-basic-scorecard.json");
+const fixture = readJson(fixturePath);
+const scorecard = readJson(scorecardPath);
+const args = parseArgs(process.argv.slice(2));
+
+function repoPath(...parts) {
+  const target = path.join(root, ...parts);
+  const relative = path.relative(root, target);
+  if (relative.startsWith("..") || path.isAbsolute(relative)) {
+    throw new Error(`Path traversal detected: ${target}`);
+  }
+  return target;
+}
+
+function parseArgs(argv) {
+  const parsed = {
+    file: undefined,
+    out: undefined
+  };
+
+  for (let index = 0; index < argv.length; index += 1) {
+    const arg = argv[index];
+    if (arg === "--file") {
+      const value = argv[index + 1];
+      if (!value) {
+        errors.push("--file requires a path");
+      } else {
+        parsed.file = repoPath(value);
+        index += 1;
+      }
+    } else if (arg === "--out") {
+      const value = argv[index + 1];
+      if (!value) {
+        errors.push("--out requires a path");
+      } else {
+        parsed.out = repoPath(value);
+        index += 1;
+      }
+    } else {
+      errors.push(`Unknown argument: ${arg}`);
+    }
+  }
+
+  if (!parsed.file) {
+    errors.push("--file is required");
+  }
+
+  return parsed;
+}
+
+function readJson(filePath) {
+  try {
+    return JSON.parse(fs.readFileSync(filePath, "utf8"));
+  } catch (error) {
+    errors.push(`${path.relative(root, filePath)}: invalid JSON (${error.message})`);
+    return undefined;
+  }
+}
+
+function outputText(value) {
+  if (Array.isArray(value)) return value.join("\n");
+  if (typeof value === "string") return value;
+  return "";
+}
+
+function weightedScore(scores, criteria) {
+  const totalWeight = criteria.reduce((sum, item) => sum + item.weight, 0);
+  return (
+    criteria.reduce((sum, item) => {
+      return sum + scores[item.id] * item.weight;
+    }, 0) / totalWeight
+  );
+}
+
+function analyzeResults(results) {
+  const outputsByRunId = new Map((results.outputs ?? []).map((item) => [item.runId, item]));
+  const scorecardByCase = new Map((scorecard.cases ?? []).map((item) => [item.caseId, item]));
+  const criteria = scorecard.criteria ?? [];
+
+  const caseResults = (fixture.cases ?? []).map((item) => {
+    const weak = outputsByRunId.get(`${item.id}::weak`);
+    const basic = outputsByRunId.get(`${item.id}::basic`);
+    const weakText = outputText(weak?.output);
+    const basicText = outputText(basic?.output);
+    const score = scorecardByCase.get(item.id);
+
+    const weakMissingSections = item.expectedStrongSections.filter((section) => !weakText.includes(section));
+    const basicMissingSections = item.expectedStrongSections.filter((section) => !basicText.includes(section));
+    const weakProhibitedClaims = item.bannedClaims.filter((claim) => weakText.includes(claim));
+    const basicProhibitedClaims = item.bannedClaims.filter((claim) => basicText.includes(claim));
+    const basicPass = basicMissingSections.length === 0 && basicProhibitedClaims.length === 0;
+    const weakDiagnostics = weakMissingSections.length + weakProhibitedClaims.length;
+
+    const weakScore = score ? weightedScore(score.weakScores, criteria) : 0;
+    const basicScore = score ? weightedScore(score.basicScores, criteria) : 0;
+
+    return {
+      caseId: item.id,
+      scenario: item.scenario,
+      basicPass,
+      weakDiagnostics,
+      weakMissingSections,
+      basicMissingSections,
+      weakProhibitedClaims,
+      basicProhibitedClaims,
+      scorecardWinner: score?.winner ?? "unknown",
+      scoreDelta: Number((basicScore - weakScore).toFixed(2)),
+      decisionBasis: score?.decisionBasis ?? ""
+    };
+  });
+
+  return {
+    schema: 1,
+    name: "engineering-business-basic-run-results-analysis",
+    version: fixture.version,
+    sourceFile: path.relative(root, args.file),
+    totalCases: caseResults.length,
+    basicPassCount: caseResults.filter((item) => item.basicPass).length,
+    weakDiagnosticCaseCount: caseResults.filter((item) => item.weakDiagnostics > 0).length,
+    scorecardBasicWinnerCount: caseResults.filter((item) => item.scorecardWinner === "basic").length,
+    caseResults
+  };
+}
+
+function renderMarkdown(analysis) {
+  const lines = [
+    "# 工程业务基础提示词运行结果分析",
+    "",
+    `- 来源文件：\`${analysis.sourceFile}\``,
+    `- 覆盖 case：${analysis.totalCases}`,
+    `- 基础提示词通过门禁：${analysis.basicPassCount}/${analysis.totalCases}`,
+    `- 普通提示词存在结构或边界诊断：${analysis.weakDiagnosticCaseCount}/${analysis.totalCases}`,
+    `- scorecard 判定基础提示词更优：${analysis.scorecardBasicWinnerCount}/${analysis.totalCases}`,
+    "",
+    "> 本报告只分析已归档的脱敏 run results，不代表未运行模型时的真实效果。",
+    "",
+    "| Case | Basic Gate | Weak Diagnostics | Score Delta | Decision |",
+    "|---|---:|---:|---:|---|"
+  ];
+
+  for (const item of analysis.caseResults) {
+    lines.push(
+      `| ${item.caseId} | ${item.basicPass ? "pass" : "fail"} | ${item.weakDiagnostics} | ${item.scoreDelta} | ${item.scorecardWinner} |`
+    );
+  }
+
+  lines.push("", "## 分场景说明", "");
+  for (const item of analysis.caseResults) {
+    lines.push(`### ${item.caseId}`, "");
+    lines.push(`- 场景：${item.scenario}`);
+    lines.push(`- 评分卡依据：${item.decisionBasis}`);
+    lines.push(`- Basic 缺失章节：${item.basicMissingSections.length === 0 ? "无" : item.basicMissingSections.join("；")}`);
+    lines.push(`- Basic 禁止结论：${item.basicProhibitedClaims.length === 0 ? "无" : item.basicProhibitedClaims.join("；")}`);
+    lines.push(`- Weak 缺失章节：${item.weakMissingSections.length === 0 ? "无" : item.weakMissingSections.join("；")}`);
+    lines.push(`- Weak 禁止结论：${item.weakProhibitedClaims.length === 0 ? "无" : item.weakProhibitedClaims.join("；")}`);
+    lines.push("");
+  }
+
+  return `${lines.join("\n")}\n`;
+}
+
+const results = args.file ? readJson(args.file) : undefined;
+const analysis = results && fixture && scorecard ? analyzeResults(results) : undefined;
+const report = analysis ? renderMarkdown(analysis) : "";
+
+if (errors.length > 0) {
+  console.error(`Prompt run results analysis failed with ${errors.length} error(s):`);
+  for (const error of errors) console.error(`- ${error}`);
+  process.exit(1);
+}
+
+if (args.out) {
+  fs.mkdirSync(path.dirname(args.out), { recursive: true });
+  fs.writeFileSync(args.out, report, "utf8");
+  console.log(`Prompt run results analysis written: ${path.relative(root, args.out)}`);
+} else {
+  process.stdout.write(report);
+}

package/scripts/build-prompt-run-pack.mjs

@@ -0,0 +1,248 @@
+#!/usr/bin/env node
+
+import fs from "node:fs";
+import path from "node:path";
+
+const root = fs.realpathSync(process.cwd());
+const errors = [];
+
+const args = parseArgs(process.argv.slice(2));
+const fixturePath = repoPath(args.fixture);
+const fixture = readJson(fixturePath);
+
+const sensitiveTerms = [
+  "立信",
+  "费敏",
+  "闻总",
+  "谭总",
+  "茅盾中学",
+  "鸿益",
+  "太鑫",
+  "飞双",
+  "魔毯",
+  "客户内部",
+  "培训演示",
+  "基础内测",
+  "内测模式",
+  "内测包",
+  "嘉兴",
+  "绍兴",
+  "杭州",
+  "20,000",
+  "20000",
+  "28,000",
+  "28000",
+  "17,800",
+  "17800"
+];
+
+function repoPath(...parts) {
+  const target = path.join(root, ...parts);
+  const relative = path.relative(root, target);
+  if (relative.startsWith("..") || path.isAbsolute(relative)) {
+    throw new Error(`Path traversal detected: ${target}`);
+  }
+  return target;
+}
+
+function parseArgs(argv) {
+  const parsed = {
+    out: undefined,
+    fixture: "prompts/evaluations/engineering-business-basic-fixtures.json"
+  };
+
+  for (let index = 0; index < argv.length; index += 1) {
+    const arg = argv[index];
+    if (arg === "--out") {
+      const value = argv[index + 1];
+      if (!value) {
+        errors.push("--out requires a path");
+      } else {
+        parsed.out = repoPath(value);
+        index += 1;
+      }
+    } else if (arg === "--fixture") {
+      const value = argv[index + 1];
+      if (!value) {
+        errors.push("--fixture requires a path");
+      } else {
+        repoPath(value);
+        parsed.fixture = value.replace(/\\/g, "/");
+        index += 1;
+      }
+    } else if (arg === "--check") {
+      continue;
+    } else {
+      errors.push(`Unknown argument: ${arg}`);
+    }
+  }
+
+  return parsed;
+}
+
+function readJson(filePath) {
+  try {
+    return JSON.parse(fs.readFileSync(filePath, "utf8"));
+  } catch (error) {
+    errors.push(`${path.relative(root, filePath)}: invalid JSON (${error.message})`);
+    return undefined;
+  }
+}
+
+function readText(relativePath) {
+  try {
+    return fs.readFileSync(repoPath(relativePath), "utf8");
+  } catch (error) {
+    errors.push(`${relativePath}: ${error.message}`);
+    return "";
+  }
+}
+
+function check(condition, message) {
+  if (!condition) errors.push(message);
+}
+
+function includesSensitiveTerm(value) {
+  const raw = typeof value === "string" ? value : JSON.stringify(value);
+  return sensitiveTerms.filter((term) => raw.includes(term));
+}
+
+function caseInput(item) {
+  if (item.markdownFixturePath) {
+    return [readText(item.markdownFixturePath)];
+  }
+  return item.sampleInput;
+}
+
+function runPackName() {
+  if (fixture?.name === "engineering-business-basic-fixtures") {
+    return "engineering-business-basic-run-pack";
+  }
+  return `${fixture?.name ?? "prompt-fixture"}-run-pack`;
+}
+
+function buildRunPack() {
+  if (!fixture) return undefined;
+
+  const runs = [];
+  for (const item of fixture.cases ?? []) {
+    const basicPrompt = readText(item.promptPath);
+    const sampleInput = caseInput(item);
+    const common = {
+      caseId: item.id,
+      skillId: item.skillId,
+      scenario: item.scenario,
+      promptVersion: fixture.version,
+      inputSummary: item.inputSummary,
+      inputFormat: item.markdownFixturePath ? "markdown" : "inline-list",
+      sampleInput,
+      expectedStrongSections: item.expectedStrongSections,
+      bannedClaims: item.bannedClaims,
+      weakFailureModes: item.weakFailureModes,
+      ...(item.markdownFixturePath ? { markdownFixturePath: item.markdownFixturePath } : {})
+    };
+
+    runs.push({
+      ...common,
+      runId: `${item.id}::weak`,
+      variant: "weak",
+      promptSource: "fixture.weakPrompt",
+      prompt: item.weakPrompt
+    });
+    runs.push({
+      ...common,
+      runId: `${item.id}::basic`,
+      variant: "basic",
+      promptSource: item.promptPath,
+      prompt: basicPrompt
+    });
+  }
+
+  return {
+    schema: 1,
+    name: runPackName(),
+    version: fixture.version,
+    fixture: args.fixture,
+    dataBoundary:
+      "De-identified weak/basic prompt run pack. Public advisory fixtures use Markdown-normalized synthetic inputs; do not add real customer names, contacts, project names, amounts, dates, locations, or raw source documents.",
+    runInstructions: [
+      "For each item, use prompt as the instruction and sampleInput as the user-provided material.",
+      "When inputFormat is markdown, pass the Markdown text as the material under review.",
+      "Run weak and basic variants separately for the same caseId.",
+      "Save model outputs into the model-output JSON schema and validate with validate-prompt-model-outputs.mjs.",
+      "Compare weak and basic outputs using engineering-business-basic-scorecard.json."
+    ],
+    runs
+  };
+}
+
+function validateRunPack(runPack) {
+  if (!fixture || !runPack) return;
+
+  check(runPack.schema === 1, "run pack: schema must be 1");
+  check(runPack.version === fixture.version, `run pack: version must match fixture version ${fixture.version}`);
+  check(runPack.fixture === args.fixture, "run pack: fixture path mismatch");
+  check(Array.isArray(runPack.runs), "run pack: runs must be an array");
+  check(runPack.runs?.length === (fixture.cases?.length ?? 0) * 2, "run pack: must include weak and basic runs for every case");
+
+  const sensitiveHits = includesSensitiveTerm(runPack);
+  check(sensitiveHits.length === 0, `run pack: sensitive terms leaked (${sensitiveHits.join(", ")})`);
+
+  const fixtureById = new Map((fixture.cases ?? []).map((item) => [item.id, item]));
+  const seenRunIds = new Set();
+  const variantsByCase = new Map();
+
+  for (const item of runPack.runs ?? []) {
+    check(typeof item.runId === "string" && item.runId.length > 0, "run pack item: missing runId");
+    check(!seenRunIds.has(item.runId), `${item.runId}: duplicate runId`);
+    seenRunIds.add(item.runId);
+
+    const sourceCase = fixtureById.get(item.caseId);
+    check(Boolean(sourceCase), `${item.runId}: caseId not found in fixture`);
+    check(item.variant === "weak" || item.variant === "basic", `${item.runId}: variant must be weak or basic`);
+    check(item.skillId === sourceCase?.skillId, `${item.runId}: skillId mismatch`);
+    check(item.promptVersion === fixture.version, `${item.runId}: promptVersion mismatch`);
+    check(Array.isArray(item.sampleInput) && item.sampleInput.length > 0, `${item.runId}: sampleInput must be non-empty`);
+    check(typeof item.prompt === "string" && item.prompt.length > 0, `${item.runId}: prompt must be non-empty`);
+    if (sourceCase?.markdownFixturePath) {
+      check(item.inputFormat === "markdown", `${item.runId}: markdown fixture must set inputFormat=markdown`);
+      check(item.markdownFixturePath === sourceCase.markdownFixturePath, `${item.runId}: markdownFixturePath mismatch`);
+      check(item.sampleInput.length === 1, `${item.runId}: markdown fixture sampleInput must contain one markdown string`);
+      check(item.sampleInput[0] === readText(sourceCase.markdownFixturePath), `${item.runId}: sampleInput must match markdown fixture content`);
+    }
+
+    const variants = variantsByCase.get(item.caseId) ?? new Set();
+    variants.add(item.variant);
+    variantsByCase.set(item.caseId, variants);
+
+    if (item.variant === "weak") {
+      check(item.prompt === sourceCase?.weakPrompt, `${item.runId}: weak prompt must match fixture weakPrompt`);
+      check(item.promptSource === "fixture.weakPrompt", `${item.runId}: weak promptSource mismatch`);
+    } else {
+      check(item.prompt === readText(sourceCase?.promptPath), `${item.runId}: basic prompt must match promptPath content`);
+      check(item.promptSource === sourceCase?.promptPath, `${item.runId}: basic promptSource mismatch`);
+    }
+  }
+
+  for (const caseId of fixtureById.keys()) {
+    const variants = variantsByCase.get(caseId);
+    check(variants?.has("weak") && variants?.has("basic"), `${caseId}: missing weak/basic pair`);
+  }
+}
+
+const runPack = buildRunPack();
+validateRunPack(runPack);
+
+if (errors.length > 0) {
+  console.error(`Prompt run pack build failed with ${errors.length} error(s):`);
+  for (const error of errors) console.error(`- ${error}`);
+  process.exit(1);
+}
+
+if (args.out) {
+  fs.mkdirSync(path.dirname(args.out), { recursive: true });
+  fs.writeFileSync(args.out, `${JSON.stringify(runPack, null, 2)}\n`, "utf8");
+  console.log(`Prompt run pack written: ${path.relative(root, args.out)}`);
+} else {
+  console.log("Prompt run pack validation passed.");
+}

package/scripts/validate-prompt-fixtures.mjs

@@ -0,0 +1,225 @@
+#!/usr/bin/env node
+
+import fs from "node:fs";
+import path from "node:path";
+
+const root = fs.realpathSync(process.cwd());
+const errors = [];
+
+const fixturePaths = [
+  "prompts/evaluations/engineering-business-basic-fixtures.json",
+  "prompts/evaluations/engineering-business-public-advisory-fixtures.json"
+];
+const fixtures = fixturePaths.map((fixturePath) => ({
+  path: fixturePath,
+  data: readJson(repoPath(fixturePath))
+}));
+
+const sensitiveTerms = [
+  "立信",
+  "费敏",
+  "闻总",
+  "谭总",
+  "茅盾中学",
+  "鸿益",
+  "太鑫",
+  "飞双",
+  "魔毯",
+  "客户内部",
+  "培训演示",
+  "基础内测",
+  "内测模式",
+  "内测包",
+  "嘉兴",
+  "绍兴",
+  "杭州",
+  "20,000",
+  "20000",
+  "28,000",
+  "28000",
+  "17,800",
+  "17800"
+];
+
+const requiredCaseFields = [
+  "id",
+  "skillId",
+  "promptPath",
+  "scenario",
+  "sourceSignals",
+  "advisoryComparison",
+  "weakPrompt",
+  "inputSummary",
+  "sampleInput",
+  "requiredPromptTerms",
+  "expectedOutputShape",
+  "expectedStrongSections",
+  "weakFailureModes",
+  "bannedClaims"
+];
+
+function repoPath(...parts) {
+  const target = path.join(root, ...parts);
+  const relative = path.relative(root, target);
+  if (relative.startsWith("..") || path.isAbsolute(relative)) {
+    throw new Error(`Path traversal detected: ${target}`);
+  }
+  return target;
+}
+
+function readJson(filePath) {
+  try {
+    return JSON.parse(fs.readFileSync(filePath, "utf8"));
+  } catch (error) {
+    errors.push(`${path.relative(root, filePath)}: invalid JSON (${error.message})`);
+    return undefined;
+  }
+}
+
+function readText(relativePath) {
+  try {
+    return fs.readFileSync(repoPath(relativePath), "utf8");
+  } catch (error) {
+    errors.push(`${relativePath}: ${error.message}`);
+    return "";
+  }
+}
+
+function check(condition, message) {
+  if (!condition) errors.push(message);
+}
+
+function includesSensitiveTerm(value) {
+  const raw = typeof value === "string" ? value : JSON.stringify(value);
+  return sensitiveTerms.filter((term) => raw.includes(term));
+}
+
+function validateMarkdownFixture(fixturePath, item) {
+  const markdownPath = item.markdownFixturePath;
+  check(typeof markdownPath === "string" && markdownPath.endsWith(".md"), `${fixturePath}/${item.id}: markdownFixturePath must point to a markdown file`);
+  if (typeof markdownPath !== "string") {
+    return;
+  }
+
+  const markdown = readText(markdownPath);
+  check(markdown.length > 0, `${fixturePath}/${item.id}: markdown fixture is empty or missing`);
+  check(markdown.includes(`caseId: ${item.id}`), `${fixturePath}/${item.id}: markdown fixture must include matching caseId`);
+  check(markdown.includes(`skillId: ${item.skillId}`), `${fixturePath}/${item.id}: markdown fixture must include matching skillId`);
+  check(markdown.includes("> 数据说明：以下客户、项目、人员、地点、日期、金额、编号均为虚构。"), `${fixturePath}/${item.id}: markdown fixture must state synthetic boundary`);
+
+  const sensitiveHits = includesSensitiveTerm(markdown);
+  check(sensitiveHits.length === 0, `${fixturePath}/${item.id}: markdown fixture leaked sensitive terms (${sensitiveHits.join(", ")})`);
+}
+
+function validateSyntheticFields(fixturePath, fixture, item) {
+  if (!fixture.name?.includes("public-advisory")) {
+    return;
+  }
+
+  check(fixture.syntheticDataPolicy?.entitiesAreFictional === true, `${fixturePath}: syntheticDataPolicy must mark entities as fictional`);
+  check(item.syntheticFields && typeof item.syntheticFields === "object", `${item.id}: missing syntheticFields`);
+
+  const requiredSyntheticFields = [
+    "customer",
+    "project",
+    "location",
+    "dates",
+    "people",
+    "numbers",
+    "documentRefs"
+  ];
+
+  for (const field of requiredSyntheticFields) {
+    check(Array.isArray(item.syntheticFields?.[field]) && item.syntheticFields[field].length > 0, `${item.id}: syntheticFields.${field} must be a non-empty array`);
+  }
+
+  const raw = JSON.stringify(item.syntheticFields ?? {});
+  check(raw.includes("虚构") || raw.includes("示例"), `${item.id}: syntheticFields must clearly mark fictional data`);
+  validateMarkdownFixture(fixturePath, item);
+}
+
+function validateFixture(fixturePath, fixture) {
+  if (!fixture) {
+    return;
+  }
+
+  check(fixture.schema === 1, `${fixturePath}: schema must be 1`);
+  check(Array.isArray(fixture.cases), `${fixturePath}: cases must be an array`);
+  check(fixture.cases?.length === 6, `${fixturePath}: must cover 6 engineering-business cases`);
+
+  const seenIds = new Set();
+  const seenSkills = new Set();
+
+  for (const item of fixture.cases ?? []) {
+    for (const field of requiredCaseFields) {
+      check(Object.prototype.hasOwnProperty.call(item, field), `${fixturePath}/${item.id ?? "unknown"}: missing ${field}`);
+    }
+
+    check(!seenIds.has(item.id), `${fixturePath}/${item.id}: duplicate case id`);
+    seenIds.add(item.id);
+    check(!seenSkills.has(item.skillId), `${fixturePath}/${item.skillId}: duplicate skill fixture`);
+    seenSkills.add(item.skillId);
+
+    check(/^aios-[a-z0-9-]+$/.test(item.skillId), `${fixturePath}/${item.id}: invalid skillId`);
+    check(item.promptPath === `skills/${item.skillId}/prompts/basic-prompt.md`, `${fixturePath}/${item.id}: promptPath must match skillId`);
+
+    for (const field of [
+      "inputSummary",
+      "sourceSignals",
+      "advisoryComparison",
+      "sampleInput",
+      "requiredPromptTerms",
+      "expectedOutputShape",
+      "expectedStrongSections",
+      "weakFailureModes",
+      "bannedClaims"
+    ]) {
+      check(Array.isArray(item[field]) && item[field].length > 0, `${fixturePath}/${item.id}: ${field} must be a non-empty array`);
+    }
+
+    for (const signal of item.sourceSignals ?? []) {
+      check(
+        /^(advisory|source-shape|boundary-shape):/.test(signal),
+        `${fixturePath}/${item.id}: source signal must use an abstract prefix (${signal})`
+      );
+    }
+
+    const sensitiveHits = includesSensitiveTerm(item);
+    check(sensitiveHits.length === 0, `${fixturePath}/${item.id}: sensitive terms leaked (${sensitiveHits.join(", ")})`);
+    validateSyntheticFields(fixturePath, fixture, item);
+
+    const prompt = readText(item.promptPath);
+    check(prompt.length > 0, `${fixturePath}/${item.id}: prompt file is empty or missing`);
+    for (const term of item.requiredPromptTerms ?? []) {
+      check(prompt.includes(term), `${fixturePath}/${item.id}: prompt missing required term "${term}"`);
+    }
+    for (const section of item.expectedStrongSections ?? []) {
+      check(prompt.includes(section), `${fixturePath}/${item.id}: prompt missing expected strong section "${section}"`);
+    }
+    for (const bannedClaim of item.bannedClaims ?? []) {
+      check(prompt.includes(bannedClaim), `${fixturePath}/${item.id}: prompt should explicitly list prohibited claim "${bannedClaim}"`);
+    }
+  }
+
+  const expectedSkills = [
+    "aios-commercial-tender",
+    "aios-commercial-contract",
+    "aios-construction-daily",
+    "aios-construction-meeting",
+    "aios-commercial-variation",
+    "aios-construction-scheme"
+  ].sort();
+  check(JSON.stringify([...seenSkills].sort()) === JSON.stringify(expectedSkills), `${fixturePath}: skill coverage mismatch`);
+}
+
+for (const fixture of fixtures) {
+  validateFixture(fixture.path, fixture.data);
+}
+
+if (errors.length > 0) {
+  console.error(`Prompt fixture validation failed with ${errors.length} error(s):`);
+  for (const error of errors) console.error(`- ${error}`);
+  process.exit(1);
+}
+
+console.log("Prompt fixture validation passed.");