@archsight/aios 1.1.0 → 1.3.0

package/scripts/validate-prompt-model-outputs.mjs

@@ -0,0 +1,201 @@
+#!/usr/bin/env node
+
+import fs from "node:fs";
+import path from "node:path";
+
+const root = fs.realpathSync(process.cwd());
+const errors = [];
+
+const defaultOutputPath = "prompts/evaluations/engineering-business-basic-model-output.example.json";
+const args = parseArgs(process.argv.slice(2));
+const outputPath = args.file ?? repoPath(defaultOutputPath);
+const fixturePath = repoPath("prompts/evaluations/engineering-business-basic-fixtures.json");
+const fixture = readJson(fixturePath);
+const outputFile = args.init ? undefined : readJson(outputPath);
+
+const sensitiveTerms = [
+  "立信",
+  "费敏",
+  "闻总",
+  "谭总",
+  "茅盾中学",
+  "鸿益",
+  "太鑫",
+  "飞双",
+  "魔毯",
+  "客户内部",
+  "培训演示",
+  "基础内测",
+  "内测模式",
+  "内测包"
+];
+
+function repoPath(...parts) {
+  const target = path.join(root, ...parts);
+  const relative = path.relative(root, target);
+  if (relative.startsWith("..") || path.isAbsolute(relative)) {
+    throw new Error(`Path traversal detected: ${target}`);
+  }
+  return target;
+}
+
+function parseArgs(argv) {
+  const parsed = {
+    file: undefined,
+    init: undefined,
+    force: false
+  };
+
+  for (let index = 0; index < argv.length; index += 1) {
+    const arg = argv[index];
+    if (arg === "--file") {
+      const value = argv[index + 1];
+      if (!value) {
+        errors.push("--file requires a path");
+      } else {
+        parsed.file = repoPath(value);
+        index += 1;
+      }
+    } else if (arg === "--init") {
+      const value = argv[index + 1];
+      if (!value) {
+        errors.push("--init requires a path");
+      } else {
+        parsed.init = repoPath(value);
+        index += 1;
+      }
+    } else if (arg === "--force") {
+      parsed.force = true;
+    } else {
+      errors.push(`Unknown argument: ${arg}`);
+    }
+  }
+
+  return parsed;
+}
+
+function readJson(filePath) {
+  try {
+    return JSON.parse(fs.readFileSync(filePath, "utf8"));
+  } catch (error) {
+    errors.push(`${path.relative(root, filePath)}: invalid JSON (${error.message})`);
+    return undefined;
+  }
+}
+
+function check(condition, message) {
+  if (!condition) errors.push(message);
+}
+
+function includesSensitiveTerm(value) {
+  const raw = typeof value === "string" ? value : JSON.stringify(value);
+  return sensitiveTerms.filter((term) => raw.includes(term));
+}
+
+function outputText(value) {
+  if (Array.isArray(value)) return value.join("\n");
+  if (typeof value === "string") return value;
+  return "";
+}
+
+function createOutputTemplate() {
+  if (!fixture) return;
+
+  if (fs.existsSync(args.init) && !args.force) {
+    errors.push(`${path.relative(root, args.init)} already exists; pass --force to overwrite`);
+    return;
+  }
+
+  const template = {
+    schema: 1,
+    name: "engineering-business-basic-model-output-run",
+    version: fixture.version ?? "0.1",
+    fixture: "prompts/evaluations/engineering-business-basic-fixtures.json",
+    isExample: false,
+    dataBoundary:
+      "Fill this file with de-identified model outputs only. Do not include customer names, contacts, project names, exact amounts, or raw source documents.",
+    outputs: (fixture.cases ?? []).map((item) => ({
+      caseId: item.id,
+      promptVersion: fixture.version ?? "0.1",
+      model: "",
+      ranAt: "",
+      notes: "",
+      promptPath: item.promptPath,
+      scenario: item.scenario,
+      expectedSections: item.expectedStrongSections,
+      bannedClaims: item.bannedClaims,
+      output: []
+    }))
+  };
+
+  fs.mkdirSync(path.dirname(args.init), { recursive: true });
+  fs.writeFileSync(args.init, `${JSON.stringify(template, null, 2)}\n`, "utf8");
+  console.log(`Prompt model output template written: ${path.relative(root, args.init)}`);
+}
+
+if (args.init) {
+  if (errors.length === 0) createOutputTemplate();
+  if (errors.length > 0) {
+    console.error(`Prompt model output validation failed with ${errors.length} error(s):`);
+    for (const error of errors) console.error(`- ${error}`);
+    process.exit(1);
+  }
+  process.exit(0);
+}
+
+if (fixture && outputFile) {
+  check(outputFile.schema === 1, "model output file: schema must be 1");
+  check(typeof outputFile.version === "string" && outputFile.version.length > 0, "model output file: version must be a string");
+  check(Array.isArray(outputFile.outputs), "model output file: outputs must be an array");
+  check(
+    outputFile.fixture === "prompts/evaluations/engineering-business-basic-fixtures.json",
+    "model output file: fixture path mismatch"
+  );
+
+  const casesById = new Map((fixture.cases ?? []).map((item) => [item.id, item]));
+  const expectedIds = [...casesById.keys()].sort();
+  const actualIds = (outputFile.outputs ?? []).map((item) => item.caseId).sort();
+
+  check(JSON.stringify(actualIds) === JSON.stringify(expectedIds), "model output file: case coverage mismatch");
+
+  const seenIds = new Set();
+  for (const item of outputFile.outputs ?? []) {
+    check(typeof item.caseId === "string" && item.caseId.length > 0, "model output item: missing caseId");
+    check(!seenIds.has(item.caseId), `${item.caseId}: duplicate model output`);
+    seenIds.add(item.caseId);
+
+    const sourceCase = casesById.get(item.caseId);
+    check(Boolean(sourceCase), `${item.caseId}: caseId not found in fixtures`);
+
+    check(typeof item.promptVersion === "string" && item.promptVersion.length > 0, `${item.caseId}: missing promptVersion`);
+    check(item.promptVersion === fixture.version, `${item.caseId}: promptVersion must match fixture version ${fixture.version}`);
+    check(typeof item.model === "string" && item.model.length > 0, `${item.caseId}: missing model`);
+    check(typeof item.ranAt === "string" && item.ranAt.length > 0, `${item.caseId}: missing ranAt`);
+    check(!Number.isNaN(Date.parse(item.ranAt)), `${item.caseId}: ranAt must be a parseable timestamp`);
+    check(typeof item.notes === "string", `${item.caseId}: notes must be a string`);
+    if (outputFile.isExample !== true) {
+      check(item.model !== "example-skeleton", `${item.caseId}: non-example output must use a real model identifier`);
+    }
+
+    const text = outputText(item.output);
+    check(text.length > 0, `${item.caseId}: output must be a non-empty string or string array`);
+
+    const sensitiveHits = includesSensitiveTerm(item);
+    check(sensitiveHits.length === 0, `${item.caseId}: sensitive terms leaked (${sensitiveHits.join(", ")})`);
+
+    for (const section of sourceCase?.expectedStrongSections ?? []) {
+      check(text.includes(section), `${item.caseId}: output missing expected section "${section}"`);
+    }
+    for (const bannedClaim of sourceCase?.bannedClaims ?? []) {
+      check(!text.includes(bannedClaim), `${item.caseId}: output contains prohibited claim "${bannedClaim}"`);
+    }
+  }
+}
+
+if (errors.length > 0) {
+  console.error(`Prompt model output validation failed with ${errors.length} error(s):`);
+  for (const error of errors) console.error(`- ${error}`);
+  process.exit(1);
+}
+
+console.log("Prompt model output validation passed.");

package/scripts/validate-prompt-run-results.mjs

@@ -0,0 +1,259 @@
+#!/usr/bin/env node
+
+import fs from "node:fs";
+import path from "node:path";
+
+const root = fs.realpathSync(process.cwd());
+const errors = [];
+const diagnostics = [];
+
+const fixturePath = repoPath("prompts/evaluations/engineering-business-basic-fixtures.json");
+const fixture = readJson(fixturePath);
+const args = parseArgs(process.argv.slice(2));
+
+const sensitiveTerms = [
+  "立信",
+  "费敏",
+  "闻总",
+  "谭总",
+  "茅盾中学",
+  "鸿益",
+  "太鑫",
+  "飞双",
+  "魔毯",
+  "客户内部",
+  "培训演示",
+  "基础内测",
+  "内测模式",
+  "内测包"
+];
+
+function repoPath(...parts) {
+  const target = path.join(root, ...parts);
+  const relative = path.relative(root, target);
+  if (relative.startsWith("..") || path.isAbsolute(relative)) {
+    throw new Error(`Path traversal detected: ${target}`);
+  }
+  return target;
+}
+
+function parseArgs(argv) {
+  const parsed = {
+    file: undefined,
+    init: undefined,
+    force: false,
+    checkTemplate: argv.length === 0
+  };
+
+  for (let index = 0; index < argv.length; index += 1) {
+    const arg = argv[index];
+    if (arg === "--file") {
+      const value = argv[index + 1];
+      if (!value) {
+        errors.push("--file requires a path");
+      } else {
+        parsed.file = repoPath(value);
+        index += 1;
+      }
+    } else if (arg === "--init") {
+      const value = argv[index + 1];
+      if (!value) {
+        errors.push("--init requires a path");
+      } else {
+        parsed.init = repoPath(value);
+        index += 1;
+      }
+    } else if (arg === "--force") {
+      parsed.force = true;
+    } else if (arg === "--check-template") {
+      parsed.checkTemplate = true;
+    } else {
+      errors.push(`Unknown argument: ${arg}`);
+    }
+  }
+
+  if ([parsed.file, parsed.init, parsed.checkTemplate].filter(Boolean).length > 1) {
+    errors.push("Use only one mode: --file, --init, or --check-template");
+  }
+
+  return parsed;
+}
+
+function readJson(filePath) {
+  try {
+    return JSON.parse(fs.readFileSync(filePath, "utf8"));
+  } catch (error) {
+    errors.push(`${path.relative(root, filePath)}: invalid JSON (${error.message})`);
+    return undefined;
+  }
+}
+
+function check(condition, message) {
+  if (!condition) errors.push(message);
+}
+
+function includesSensitiveTerm(value) {
+  const raw = typeof value === "string" ? value : JSON.stringify(value);
+  return sensitiveTerms.filter((term) => raw.includes(term));
+}
+
+function outputText(value) {
+  if (Array.isArray(value)) return value.join("\n");
+  if (typeof value === "string") return value;
+  return "";
+}
+
+function expectedRuns() {
+  if (!fixture) return [];
+
+  return (fixture.cases ?? []).flatMap((item) => [
+    {
+      runId: `${item.id}::weak`,
+      caseId: item.id,
+      variant: "weak",
+      promptSource: "fixture.weakPrompt",
+      promptVersion: fixture.version,
+      expectedStrongSections: item.expectedStrongSections,
+      bannedClaims: item.bannedClaims,
+      weakFailureModes: item.weakFailureModes
+    },
+    {
+      runId: `${item.id}::basic`,
+      caseId: item.id,
+      variant: "basic",
+      promptSource: item.promptPath,
+      promptVersion: fixture.version,
+      expectedStrongSections: item.expectedStrongSections,
+      bannedClaims: item.bannedClaims,
+      weakFailureModes: item.weakFailureModes
+    }
+  ]);
+}
+
+function createTemplate() {
+  return {
+    schema: 1,
+    name: "engineering-business-basic-run-results",
+    version: fixture?.version ?? "0.1",
+    fixture: "prompts/evaluations/engineering-business-basic-fixtures.json",
+    runPack: "prompts/evaluations/engineering-business-basic-run-pack.generated.json",
+    isExample: false,
+    dataBoundary:
+      "Fill this file with de-identified weak/basic model outputs only. Do not include customer names, contacts, project names, exact amounts, or raw source documents.",
+    outputs: expectedRuns().map((item) => ({
+      runId: item.runId,
+      caseId: item.caseId,
+      variant: item.variant,
+      promptVersion: item.promptVersion,
+      model: "",
+      ranAt: "",
+      notes: "",
+      promptSource: item.promptSource,
+      expectedStrongSections: item.expectedStrongSections,
+      bannedClaims: item.bannedClaims,
+      weakFailureModes: item.weakFailureModes,
+      output: []
+    }))
+  };
+}
+
+function validateTemplateShape(template) {
+  check(template.schema === 1, "run results: schema must be 1");
+  check(template.version === fixture?.version, `run results: version must match fixture version ${fixture?.version}`);
+  check(template.fixture === "prompts/evaluations/engineering-business-basic-fixtures.json", "run results: fixture path mismatch");
+  check(Array.isArray(template.outputs), "run results: outputs must be an array");
+  check(template.outputs?.length === expectedRuns().length, "run results: output count must match weak/basic run count");
+
+  const sensitiveHits = includesSensitiveTerm(template);
+  check(sensitiveHits.length === 0, `run results: sensitive terms leaked (${sensitiveHits.join(", ")})`);
+
+  const expectedByRunId = new Map(expectedRuns().map((item) => [item.runId, item]));
+  const actualRunIds = (template.outputs ?? []).map((item) => item.runId).sort();
+  const expectedRunIds = [...expectedByRunId.keys()].sort();
+  check(JSON.stringify(actualRunIds) === JSON.stringify(expectedRunIds), "run results: runId coverage mismatch");
+
+  const seen = new Set();
+  for (const item of template.outputs ?? []) {
+    const expected = expectedByRunId.get(item.runId);
+    check(!seen.has(item.runId), `${item.runId}: duplicate output`);
+    seen.add(item.runId);
+    check(Boolean(expected), `${item.runId}: runId not found in expected run pack`);
+    check(item.caseId === expected?.caseId, `${item.runId}: caseId mismatch`);
+    check(item.variant === expected?.variant, `${item.runId}: variant mismatch`);
+    check(item.promptVersion === expected?.promptVersion, `${item.runId}: promptVersion mismatch`);
+    check(item.promptSource === expected?.promptSource, `${item.runId}: promptSource mismatch`);
+    check(Array.isArray(item.expectedStrongSections), `${item.runId}: expectedStrongSections must be an array`);
+    check(Array.isArray(item.bannedClaims), `${item.runId}: bannedClaims must be an array`);
+    check(Array.isArray(item.weakFailureModes), `${item.runId}: weakFailureModes must be an array`);
+  }
+}
+
+function validateRunResults(results) {
+  validateTemplateShape(results);
+
+  for (const item of results.outputs ?? []) {
+    check(typeof item.model === "string" && item.model.length > 0, `${item.runId}: missing model`);
+    check(item.model !== "example-skeleton", `${item.runId}: model must be a real model identifier`);
+    check(typeof item.ranAt === "string" && item.ranAt.length > 0, `${item.runId}: missing ranAt`);
+    check(!Number.isNaN(Date.parse(item.ranAt)), `${item.runId}: ranAt must be a parseable timestamp`);
+    check(typeof item.notes === "string", `${item.runId}: notes must be a string`);
+
+    const text = outputText(item.output);
+    check(text.length > 0, `${item.runId}: output must be a non-empty string or string array`);
+
+    const sensitiveHits = includesSensitiveTerm(item);
+    check(sensitiveHits.length === 0, `${item.runId}: sensitive terms leaked (${sensitiveHits.join(", ")})`);
+
+    const missingSections = (item.expectedStrongSections ?? []).filter((section) => !text.includes(section));
+    const prohibitedClaims = (item.bannedClaims ?? []).filter((claim) => text.includes(claim));
+
+    if (item.variant === "basic") {
+      for (const section of missingSections) {
+        errors.push(`${item.runId}: basic output missing expected section "${section}"`);
+      }
+      for (const claim of prohibitedClaims) {
+        errors.push(`${item.runId}: basic output contains prohibited claim "${claim}"`);
+      }
+    } else if (missingSections.length > 0 || prohibitedClaims.length > 0) {
+      diagnostics.push({
+        runId: item.runId,
+        missingSections,
+        prohibitedClaims
+      });
+    }
+  }
+}
+
+if (args.init) {
+  const template = createTemplate();
+  validateTemplateShape(template);
+  if (fs.existsSync(args.init) && !args.force) {
+    errors.push(`${path.relative(root, args.init)} already exists; pass --force to overwrite`);
+  }
+  if (errors.length === 0) {
+    fs.mkdirSync(path.dirname(args.init), { recursive: true });
+    fs.writeFileSync(args.init, `${JSON.stringify(template, null, 2)}\n`, "utf8");
+    console.log(`Prompt run results template written: ${path.relative(root, args.init)}`);
+    process.exit(0);
+  }
+} else if (args.file) {
+  const results = readJson(args.file);
+  if (results) validateRunResults(results);
+} else {
+  validateTemplateShape(createTemplate());
+  if (errors.length === 0) {
+    console.log("Prompt run results template validation passed.");
+    process.exit(0);
+  }
+}
+
+if (errors.length > 0) {
+  console.error(`Prompt run results validation failed with ${errors.length} error(s):`);
+  for (const error of errors) console.error(`- ${error}`);
+  process.exit(1);
+}
+
+console.log("Prompt run results validation passed.");
+if (diagnostics.length > 0) {
+  console.log(`Weak output diagnostics: ${diagnostics.length} run(s) need comparison review.`);
+}

package/scripts/validate-prompt-scorecard.mjs

@@ -0,0 +1,133 @@
+#!/usr/bin/env node
+
+import fs from "node:fs";
+import path from "node:path";
+
+const root = fs.realpathSync(process.cwd());
+const errors = [];
+
+const fixturePath = repoPath("prompts/evaluations/engineering-business-basic-fixtures.json");
+const scorecardPath = repoPath("prompts/evaluations/engineering-business-basic-scorecard.json");
+const fixture = readJson(fixturePath);
+const scorecard = readJson(scorecardPath);
+
+const sensitiveTerms = [
+  "立信",
+  "费敏",
+  "闻总",
+  "谭总",
+  "茅盾中学",
+  "鸿益",
+  "太鑫",
+  "飞双",
+  "魔毯",
+  "客户内部",
+  "培训演示",
+  "基础内测",
+  "内测模式",
+  "内测包"
+];
+
+function repoPath(...parts) {
+  const target = path.join(root, ...parts);
+  const relative = path.relative(root, target);
+  if (relative.startsWith("..") || path.isAbsolute(relative)) {
+    throw new Error(`Path traversal detected: ${target}`);
+  }
+  return target;
+}
+
+function readJson(filePath) {
+  try {
+    return JSON.parse(fs.readFileSync(filePath, "utf8"));
+  } catch (error) {
+    errors.push(`${path.relative(root, filePath)}: invalid JSON (${error.message})`);
+    return undefined;
+  }
+}
+
+function check(condition, message) {
+  if (!condition) errors.push(message);
+}
+
+function includesSensitiveTerm(value) {
+  const raw = typeof value === "string" ? value : JSON.stringify(value);
+  return sensitiveTerms.filter((term) => raw.includes(term));
+}
+
+function weightedScore(scores, criteria) {
+  const totalWeight = criteria.reduce((sum, item) => sum + item.weight, 0);
+  return (
+    criteria.reduce((sum, item) => {
+      return sum + scores[item.id] * item.weight;
+    }, 0) / totalWeight
+  );
+}
+
+if (fixture && scorecard) {
+  check(scorecard.schema === 1, "scorecard: schema must be 1");
+  check(scorecard.version === fixture.version, `scorecard: version must match fixture version ${fixture.version}`);
+  check(scorecard.fixture === "prompts/evaluations/engineering-business-basic-fixtures.json", "scorecard: fixture path mismatch");
+  check(Array.isArray(scorecard.criteria) && scorecard.criteria.length > 0, "scorecard: criteria must be a non-empty array");
+  check(Array.isArray(scorecard.cases) && scorecard.cases.length === fixture.cases.length, "scorecard: case count mismatch");
+
+  const sensitiveHits = includesSensitiveTerm(scorecard);
+  check(sensitiveHits.length === 0, `scorecard: sensitive terms leaked (${sensitiveHits.join(", ")})`);
+
+  const criteriaIds = new Set();
+  let totalWeight = 0;
+  for (const criterion of scorecard.criteria ?? []) {
+    check(/^[a-z0-9_]+$/.test(criterion.id), `${criterion.id ?? "unknown"}: invalid criterion id`);
+    check(!criteriaIds.has(criterion.id), `${criterion.id}: duplicate criterion`);
+    criteriaIds.add(criterion.id);
+    check(Number.isInteger(criterion.weight) && criterion.weight > 0, `${criterion.id}: weight must be a positive integer`);
+    totalWeight += criterion.weight ?? 0;
+    check(typeof criterion.description === "string" && criterion.description.length > 0, `${criterion.id}: missing description`);
+  }
+  check(totalWeight === 100, `scorecard: criteria weights must total 100, got ${totalWeight}`);
+
+  const fixtureById = new Map((fixture.cases ?? []).map((item) => [item.id, item]));
+  const expectedIds = [...fixtureById.keys()].sort();
+  const actualIds = (scorecard.cases ?? []).map((item) => item.caseId).sort();
+  check(JSON.stringify(actualIds) === JSON.stringify(expectedIds), "scorecard: case coverage mismatch");
+
+  for (const item of scorecard.cases ?? []) {
+    const sourceCase = fixtureById.get(item.caseId);
+    check(Boolean(sourceCase), `${item.caseId}: caseId not found in fixtures`);
+    check(item.winner === "basic", `${item.caseId}: winner must be basic`);
+    check(typeof item.decisionBasis === "string" && item.decisionBasis.length > 0, `${item.caseId}: missing decisionBasis`);
+    check(Array.isArray(item.basicPromptGains) && item.basicPromptGains.length > 0, `${item.caseId}: basicPromptGains must be non-empty`);
+    check(
+      JSON.stringify(item.observedWeakFailures ?? []) === JSON.stringify(sourceCase?.weakFailureModes ?? []),
+      `${item.caseId}: observedWeakFailures must match fixture weakFailureModes`
+    );
+
+    for (const scoreSetName of ["weakScores", "basicScores"]) {
+      const scoreSet = item[scoreSetName] ?? {};
+      const scoreIds = Object.keys(scoreSet).sort();
+      check(JSON.stringify(scoreIds) === JSON.stringify([...criteriaIds].sort()), `${item.caseId}: ${scoreSetName} coverage mismatch`);
+      for (const criterionId of criteriaIds) {
+        const value = scoreSet[criterionId];
+        check(Number.isInteger(value) && value >= 1 && value <= 5, `${item.caseId}: ${scoreSetName}.${criterionId} must be 1-5`);
+      }
+    }
+
+    const weakTotal = weightedScore(item.weakScores, scorecard.criteria);
+    const basicTotal = weightedScore(item.basicScores, scorecard.criteria);
+    check(
+      basicTotal - weakTotal >= scorecard.minimumWeightedDelta,
+      `${item.caseId}: weighted improvement ${Number(basicTotal - weakTotal).toFixed(2)} is below minimum ${scorecard.minimumWeightedDelta}`
+    );
+  }
+
+  check(scorecard.overallDecision?.winner === "basic", "scorecard: overall winner must be basic");
+  check(typeof scorecard.overallDecision?.notAClaim === "string", "scorecard: overallDecision.notAClaim is required");
+}
+
+if (errors.length > 0) {
+  console.error(`Prompt scorecard validation failed with ${errors.length} error(s):`);
+  for (const error of errors) console.error(`- ${error}`);
+  process.exit(1);
+}
+
+console.log("Prompt scorecard validation passed.");