@soleur/opencode 0.0.0-stage → 3.354.2

package/skills/eval-harness/scripts/eval-gate.cjs

@@ -0,0 +1,318 @@
+#!/usr/bin/env node
+// eval-gate.cjs — thin orchestrator for the validation-gated classifier-skill-edit loop.
+//
+// The verdict MATH lives in verdict.cjs (pure); this file only does I/O: resolve the
+// gated-skills registry, extract current+candidate blocks, project them into skill-arm
+// prompts, shell promptfoo SKILL-ARM ONLY (current vs candidate — the baseline control
+// arm is irrelevant to a regression gate), normalize promptfoo JSON into the shape
+// verdict.cjs expects, and print the verdict. FAIL-CLOSED: any error => non-zero exit +
+// stderr and the verdict defaults to NOT accept.
+//
+// Flags:
+//   --check <file>           Lookup-only: print {gated, target, block_id} for whether
+//                            <file> is a gated source. Exit 0. NO API.
+//   --target <id>            Gated target id (go-routing | ticket-triage).
+//   --candidate-file <path>  The edited source file holding the candidate block.
+//   --target-task <json|path> The synthesized row whose candidate samples must pass.
+//   --repeat N               Samples per cell (default 5).
+//   --append-on-accept       On accept, append the (synthesized-only) target task to
+//                            tasks/<target>.jsonl.
+//   --dry-run                Print the skill-arm-only API-call estimate; exit. NO API.
+"use strict";
+
+const fs = require("node:fs");
+const os = require("node:os");
+const path = require("node:path");
+const { execFileSync } = require("node:child_process");
+const { extractBlock } = require("./extract-block.cjs");
+const { renderSkillPrompt, tokensFor } = require("./gen-skill-prompt.cjs");
+const { computeVerdict } = require("./verdict.cjs");
+
+const SKILL_DIR = path.resolve(__dirname, "..");
+const REPO_ROOT = path.resolve(SKILL_DIR, "..", "..", "..", "..");
+const REGISTRY = path.join(SKILL_DIR, "gated-skills.json");
+const MODELS_FILE = path.join(SKILL_DIR, "models.generated.json");
+
+// Per-target promptfoo resources (enum SSOT does NOT follow the target id, so it is
+// listed explicitly). Corpus tasks file follows tasks/<target>.jsonl by convention.
+const TARGET_RESOURCES = {
+  "go-routing": { tasks: "tasks/go-routing.jsonl", enumPath: "enums/go-routes.json" },
+  "ticket-triage": { tasks: "tasks/ticket-triage.jsonl", enumPath: "enums/triage-levels.json" },
+  "lane-inference": { tasks: "tasks/lane-inference.jsonl", enumPath: "enums/lane.json" },
+  "incident-threshold": { tasks: "tasks/incident-threshold.jsonl", enumPath: "enums/incident-threshold.json" },
+};
+
+function die(message, payload) {
+  process.stderr.write(`eval-gate: ${message}\n`);
+  // Fail-closed: surface a NOT-accept verdict to any JSON consumer.
+  process.stdout.write(JSON.stringify(Object.assign({ accept: false, error: message }, payload || {})) + "\n");
+  process.exit(1);
+}
+
+function loadRegistry() {
+  return JSON.parse(fs.readFileSync(REGISTRY, "utf8"));
+}
+
+function parseArgs(argv) {
+  const args = { repeat: 5, appendOnAccept: false, dryRun: false };
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i];
+    switch (a) {
+      case "--check": args.check = argv[++i]; break;
+      case "--target": args.target = argv[++i]; break;
+      case "--candidate-file": args.candidateFile = argv[++i]; break;
+      case "--target-task": args.targetTask = argv[++i]; break;
+      case "--repeat": args.repeat = parseInt(argv[++i], 10); break;
+      case "--append-on-accept": args.appendOnAccept = true; break;
+      case "--dry-run": args.dryRun = true; break;
+      default:
+        process.stderr.write(`eval-gate: unknown flag ${a}\n`);
+        process.exit(2);
+    }
+  }
+  // Fix 5: a non-positive / non-integer --repeat would make the dry-run estimate
+  // dishonest and send String(NaN) to promptfoo. Fail-closed.
+  if (!(Number.isInteger(args.repeat) && args.repeat >= 1)) {
+    die("--repeat must be a positive integer");
+  }
+  return args;
+}
+
+// findBySourceFile(file): the registry entry whose source_file resolves to the same
+// absolute path as `file` (resolved against CWD), or null.
+function findBySourceFile(file) {
+  const wanted = path.resolve(process.cwd(), file);
+  const wantedRepoRel = path.resolve(REPO_ROOT, file);
+  return (
+    loadRegistry().find((e) => {
+      const abs = path.resolve(REPO_ROOT, e.source_file);
+      return abs === wanted || abs === wantedRepoRel;
+    }) || null
+  );
+}
+
+function entryForTarget(target) {
+  const entry = loadRegistry().find((e) => e.target === target);
+  if (!entry) die(`unknown target ${JSON.stringify(target)}`);
+  return entry;
+}
+
+function countTasks(target) {
+  const res = TARGET_RESOURCES[target];
+  if (!res) die(`no promptfoo resources for target ${JSON.stringify(target)}`);
+  const text = fs.readFileSync(path.join(SKILL_DIR, res.tasks), "utf8");
+  return text.split(/\r?\n/).filter((l) => l.trim() !== "").length;
+}
+
+// --- synthesized-only guard (TR2 / cq-test-fixtures-synthesized-only) ----------
+// Reject inputs that look like real user data / PII. Heuristic, fail-closed.
+function synthesizedCheck(input) {
+  const s = String(input);
+  const realEmail = /\b[\w.+-]+@(?!example\.|test\.|acme\.)[\w-]+\.[\w.-]+\b/i;
+  const cardOrPhone = /\b\d[\d -]{10,}\d\b/;
+  const ssn = /\b\d{3}-\d{2}-\d{4}\b/;
+  if (realEmail.test(s)) return { ok: false, reason: "input contains a real-looking email address" };
+  if (ssn.test(s)) return { ok: false, reason: "input contains an SSN-shaped sequence" };
+  if (cardOrPhone.test(s)) return { ok: false, reason: "input contains a card/phone-shaped digit sequence" };
+  return { ok: true };
+}
+
+// parsePromptfooOutput(json): normalize promptfoo --output JSON into the verdict shape.
+// task_id = vars.input (the unique per-task key); correct is read from the MEASUREMENT
+// assert (measure-classification.cjs), identified by its "classification-correct" /
+// "classification-incorrect" reason prefix — robust to assert ordering. Fail-closed: a
+// row with no measurement component throws.
+function parsePromptfooOutput(json) {
+  const rows = (json && json.results && Array.isArray(json.results.results))
+    ? json.results.results
+    : (Array.isArray(json && json.results) ? json.results : null);
+  if (!rows) throw new Error("promptfoo output has no results array");
+  return rows.map((r) => {
+    const vars = r.vars || (r.testCase && r.testCase.vars) || {};
+    const task_id = vars.input;
+    if (typeof task_id !== "string") throw new Error("promptfoo result missing vars.input");
+    const comps = (r.gradingResult && Array.isArray(r.gradingResult.componentResults))
+      ? r.gradingResult.componentResults
+      : [];
+    const measure = comps.find((c) => typeof c.reason === "string" && /^classification-(in)?correct/.test(c.reason));
+    if (!measure) throw new Error(`no measurement component for task ${JSON.stringify(task_id)}`);
+    return { task_id, correct: /^classification-correct/.test(measure.reason) };
+  });
+}
+
+// runPromptfoo(promptText, target, combinedTasksFile, repeat): SKILL-ARM ONLY run of a
+// single prompt; returns the normalized results array. Thin shell-out.
+function runPromptfoo(promptText, target, combinedTasksFile, repeat) {
+  const res = TARGET_RESOURCES[target];
+  const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "eval-gate-"));
+  const promptFile = path.join(tmpDir, "skill.txt");
+  const configFile = path.join(tmpDir, "config.json");
+  const outFile = path.join(tmpDir, "results.json");
+  fs.writeFileSync(promptFile, promptText);
+  const config = {
+    description: `eval-gate skill-arm-only (${target})`,
+    providers: `file://${MODELS_FILE}`,
+    prompts: [`file://${promptFile}`],
+    defaultTest: {
+      vars: { enum: `file://${path.join(SKILL_DIR, res.enumPath)}` },
+      assert: [
+        { type: "javascript", value: `file://${path.join(SKILL_DIR, "scripts", "measure-classification.cjs")}` },
+        { type: "javascript", value: `file://${path.join(SKILL_DIR, "scripts", "gate-classification.cjs")}` },
+      ],
+    },
+    tests: `file://${combinedTasksFile}`,
+  };
+  fs.writeFileSync(configFile, JSON.stringify(config));
+  execFileSync("npx", ["promptfoo", "eval", "-c", configFile, "--repeat", String(repeat), "--output", outFile], {
+    cwd: SKILL_DIR,
+    stdio: ["ignore", "inherit", "inherit"],
+  });
+  return parsePromptfooOutput(JSON.parse(fs.readFileSync(outFile, "utf8")));
+}
+
+function main() {
+  const args = parseArgs(process.argv.slice(2));
+
+  // --check: lookup-only, NO API.
+  if (args.check) {
+    const entry = findBySourceFile(args.check);
+    process.stdout.write(JSON.stringify({
+      gated: Boolean(entry),
+      target: entry ? entry.target : null,
+      block_id: entry ? entry.block_id : null,
+    }) + "\n");
+    return;
+  }
+
+  if (!args.target) die("--target is required (or use --check)");
+
+  // --dry-run: API-cost disclosure (skill-arm-only), NO API.
+  if (args.dryRun) {
+    const models = JSON.parse(fs.readFileSync(MODELS_FILE, "utf8")).length;
+    const corpus = countTasks(args.target);
+    const tasks = corpus + 1; // corpus + the target task
+    // Two skill-arm runs (current vs candidate); the baseline control arm is skipped.
+    const estimate = 2 * models * tasks * args.repeat;
+    process.stdout.write(JSON.stringify({
+      dry_run: true,
+      target: args.target,
+      arms: "skill-only (current vs candidate; baseline control arm skipped)",
+      models,
+      corpus_tasks: corpus,
+      target_tasks: 1,
+      repeat: args.repeat,
+      estimated_api_calls: estimate,
+      formula: "2 (current+candidate) x models x (corpus_tasks + 1) x repeat",
+      note: "estimate only — no API call made",
+    }) + "\n");
+    return;
+  }
+
+  // Real run.
+  if (!args.candidateFile) die("--candidate-file is required for a real run");
+
+  const entry = entryForTarget(args.target);
+  let currentBlock, candidateBlock;
+  try {
+    const currentSource = fs.readFileSync(path.join(REPO_ROOT, entry.source_file), "utf8");
+    const candidateSource = fs.readFileSync(args.candidateFile, "utf8");
+    currentBlock = extractBlock(currentSource, entry.block_start_marker, entry.block_end_marker);
+    candidateBlock = extractBlock(candidateSource, entry.block_start_marker, entry.block_end_marker);
+  } catch (err) {
+    die(err.message);
+  }
+
+  // Fix 4: an in-place-edited source would make currentBlock === candidateBlock
+  // (the candidate IS the live file), masquerading as "no gated-block change". Refuse
+  // before the short-circuit — the proposed edit must live at a distinct temp path.
+  if (path.resolve(args.candidateFile) === path.resolve(REPO_ROOT, entry.source_file)) {
+    die("candidate-file must differ from the live source file (write the proposed edit to a temp path)");
+  }
+
+  // Ungateable no-op: the edit did not change the gated block. Short-circuit BEFORE
+  // requiring a target task or spending any API — there is nothing to gate.
+  if (currentBlock === candidateBlock) {
+    process.stdout.write(JSON.stringify({ accept: true, reason: "no gated-block change" }) + "\n");
+    return;
+  }
+
+  if (!args.targetTask) die("--target-task is required for a real run with a changed block");
+
+  // Parse the target task (inline JSON or a path).
+  let targetTaskObj;
+  try {
+    const raw = fs.existsSync(args.targetTask)
+      ? fs.readFileSync(args.targetTask, "utf8")
+      : args.targetTask;
+    targetTaskObj = JSON.parse(raw);
+  } catch (err) {
+    die(`cannot parse --target-task: ${err.message}`);
+  }
+  const targetInput = targetTaskObj && targetTaskObj.vars && targetTaskObj.vars.input;
+  const targetGolden = targetTaskObj && targetTaskObj.vars && targetTaskObj.vars.golden_label;
+  if (typeof targetInput !== "string" || typeof targetGolden !== "string") {
+    die("--target-task must be {\"vars\":{\"input\":\"...\",\"golden_label\":\"...\"}}");
+  }
+
+  // Fix 2: synthesized/PII guard runs on the target-task input IMMEDIATELY after parse,
+  // BEFORE the combined tasks file is built or any API call is made — a real-data-shaped
+  // target task must never reach the LLM providers (fail-closed, before any API spend).
+  const synth = synthesizedCheck(targetInput);
+  if (!synth.ok) die(`target task is not synthesized: ${synth.reason}`);
+
+  // Build the combined tasks file: corpus + the target task row.
+  const res = TARGET_RESOURCES[args.target];
+  const corpusText = fs.readFileSync(path.join(SKILL_DIR, res.tasks), "utf8").replace(/\s*$/, "");
+  const combinedDir = fs.mkdtempSync(path.join(os.tmpdir(), "eval-gate-tasks-"));
+  const combinedFile = path.join(combinedDir, "combined.jsonl");
+  fs.writeFileSync(combinedFile, corpusText + "\n" + JSON.stringify(targetTaskObj) + "\n");
+
+  const tokens = tokensFor(args.target);
+  const currentPrompt = renderSkillPrompt(args.target, currentBlock, tokens);
+  const candidatePrompt = renderSkillPrompt(args.target, candidateBlock, tokens);
+
+  let verdict;
+  try {
+    const currentResults = runPromptfoo(currentPrompt, args.target, combinedFile, args.repeat);
+    const candidateResults = runPromptfoo(candidatePrompt, args.target, combinedFile, args.repeat);
+    verdict = computeVerdict(currentResults, candidateResults, { task_id: targetInput }, {});
+  } catch (err) {
+    die(`gate run failed: ${err.message}`);
+  }
+
+  // --append-on-accept: on accept, append the synthesized target task to the corpus.
+  // Fix 3: the input was already gated as synthesized before the run (Fix 2), so this
+  // defensive re-check should never fail. If it ever did, a skipped corpus-write is NOT
+  // a gate failure — keep the real verdict (accept reflects the GATE decision) and record
+  // append_skipped rather than flipping top-level accept to false.
+  if (args.appendOnAccept && verdict.accept) {
+    const guard = synthesizedCheck(targetInput);
+    if (!guard.ok) {
+      verdict.append_skipped = true;
+      verdict.append_skipped_reason = guard.reason;
+    } else {
+      const corpusPath = path.join(SKILL_DIR, res.tasks);
+      const existing = fs.readFileSync(corpusPath, "utf8").replace(/\s*$/, "");
+      fs.writeFileSync(corpusPath, existing + "\n" + JSON.stringify(targetTaskObj) + "\n");
+      verdict.appended_to = res.tasks;
+    }
+  }
+
+  process.stdout.write(JSON.stringify(verdict) + "\n");
+}
+
+// Export the per-target resource map so the registry-coverage consistency test can
+// assert every gated target is wired here (a missing entry ships the gate DORMANT —
+// --check reports gated:true but --target/--dry-run die fail-closed). Requiring this
+// module must NOT run the CLI, hence the require.main guard below.
+module.exports = { TARGET_RESOURCES };
+
+// Fix 1: any uncaught throw (registry parse, missing corpus file, mkdtemp, render)
+// must still emit the {accept:false} stdout contract via die(), not just exit non-zero.
+if (require.main === module) {
+  try {
+    main();
+  } catch (e) {
+    die(e && e.message ? e.message : String(e));
+  }
+}

package/skills/eval-harness/scripts/extract-block.cjs

@@ -0,0 +1,87 @@
+#!/usr/bin/env node
+// extract-block.cjs — deterministic projection of a gated classifier block.
+//
+// A "gated block" is the span of a source file (e.g. the /go routing table, the
+// ticket-triage P-level rubric) wrapped in HTML-comment sentinels:
+//
+//   <!-- eval-gate:block:<block-id>:start -->
+//   ...the classifier rules (the SSOT)...
+//   <!-- eval-gate:block:<block-id>:end -->
+//
+// This is the mechanical link that makes the eval-harness skill-arm prompt a
+// projection of the production source (NOT a hand-copied paraphrase). It is used
+// (a) to build the skill-arm prompt and (b) to detect whether an edit changed the
+// block at all.
+//
+// Pure / deterministic: extractBlock() takes text in, returns text out, no I/O.
+//
+// CLI: node extract-block.cjs <source-file> <block-id>
+//   Prints the text strictly between the markers (exclusive of the marker lines),
+//   trimmed. Exits non-zero with a clear stderr message if either marker is missing.
+"use strict";
+
+const fs = require("node:fs");
+
+// markersFor(blockId): the canonical start/end sentinels for a block id.
+function markersFor(blockId) {
+  return {
+    start: `<!-- eval-gate:block:${blockId}:start -->`,
+    end: `<!-- eval-gate:block:${blockId}:end -->`,
+  };
+}
+
+// extractBlock(sourceText, startMarker, endMarker): the text strictly between the
+// two markers, trimmed. Throws (fail-closed) if either marker is absent or the end
+// precedes the start.
+function extractBlock(sourceText, startMarker, endMarker) {
+  if (typeof sourceText !== "string") {
+    throw new TypeError("extractBlock: sourceText must be a string");
+  }
+  const startIdx = sourceText.indexOf(startMarker);
+  if (startIdx === -1) {
+    throw new Error(`extract-block: start marker not found: ${startMarker}`);
+  }
+  const endIdx = sourceText.indexOf(endMarker);
+  if (endIdx === -1) {
+    throw new Error(`extract-block: end marker not found: ${endMarker}`);
+  }
+  if (endIdx < startIdx) {
+    throw new Error(
+      `extract-block: end marker ${endMarker} precedes start marker ${startMarker}`,
+    );
+  }
+  const between = sourceText.slice(startIdx + startMarker.length, endIdx);
+  return between.trim();
+}
+
+// extractBlockById(sourceText, blockId): convenience wrapper using the canonical
+// marker convention for a block id.
+function extractBlockById(sourceText, blockId) {
+  const { start, end } = markersFor(blockId);
+  return extractBlock(sourceText, start, end);
+}
+
+module.exports = { extractBlock, extractBlockById, markersFor };
+
+if (require.main === module) {
+  const [sourceFile, blockId] = process.argv.slice(2);
+  if (!sourceFile || !blockId) {
+    process.stderr.write(
+      "usage: node extract-block.cjs <source-file> <block-id>\n",
+    );
+    process.exit(2);
+  }
+  let text;
+  try {
+    text = fs.readFileSync(sourceFile, "utf8");
+  } catch (err) {
+    process.stderr.write(`extract-block: cannot read ${sourceFile}: ${err.message}\n`);
+    process.exit(1);
+  }
+  try {
+    process.stdout.write(extractBlockById(text, blockId) + "\n");
+  } catch (err) {
+    process.stderr.write(`${err.message}\n`);
+    process.exit(1);
+  }
+}

package/skills/eval-harness/scripts/gate-classification.cjs

@@ -0,0 +1,28 @@
+// GATE assert (the ponytail `correctness.js` analog).
+//
+// promptfoo custom JavaScript assert. FAILS when the model's emitted label is NOT
+// a member of the target's closed enum — a malformed or hallucinated label breaks
+// the skill's contract regardless of whether it would have been the "right" answer.
+// This is the deterministic contract check: the output must be a well-formed label.
+//
+// Signature: module.exports = (output, context) => GradingResult
+//   context.vars.enum : the closed set of allowed labels — a direct array or a
+//   `file://`/relative path to the enum SSOT (promptfoo passes the `file://` ref
+//   verbatim, so loadEnum() reads the file itself).
+"use strict";
+
+const { extractLabel, loadEnum } = require("./parse-label.cjs");
+
+module.exports = (output, context) => {
+  const vars = (context && context.vars) || {};
+  const allowed = loadEnum(vars);
+  const got = extractLabel(output, allowed);
+  const inEnum = allowed.includes(got);
+  return {
+    pass: inEnum,
+    score: inEnum ? 1.0 : 0.0,
+    reason: inEnum
+      ? `in-enum label: ${got}`
+      : `out-of-enum label ${JSON.stringify(got)} — not one of [${allowed.join(", ")}]`,
+  };
+};

package/skills/eval-harness/scripts/gen-models.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+# Single-source the three current Claude model IDs into models.generated.json.
+#
+# The model IDs live in ONE place — the TypeScript registry under
+# apps/web-platform/server/inngest/. promptfoo YAML cannot import TS, so this
+# generator reads the IDs VERBATIM from the registry and emits a provider list
+# the promptfooconfig-*.yaml files reference via `providers: file://models.generated.json`.
+# No model literal is ever hardcoded in a config-class file, which also keeps the
+# model-launch-review auto-fixer from rewriting a stale literal here.
+set -euo pipefail
+
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ROOT="$(git -C "$HERE" rev-parse --show-toplevel)"
+CONST="$ROOT/apps/web-platform/server/inngest/leader-prompts/constants.ts"
+TIERS="$ROOT/apps/web-platform/server/inngest/model-tiers.ts"
+OUT="$HERE/../models.generated.json"
+
+extract() { # file, exported-const-name -> the claude-* literal
+  grep -E "^export const $2" "$1" | grep -oE '"claude-[^"]+"' | tr -d '"' | head -1
+}
+
+SONNET="$(extract "$CONST" "SONNET_MODEL")"
+HAIKU="$(extract "$CONST" "HAIKU_MODEL")"
+OPUS="$(extract "$TIERS" "AUDIT_MODEL")"
+
+for pair in "SONNET_MODEL:$SONNET" "HAIKU_MODEL:$HAIKU" "AUDIT_MODEL:$OPUS"; do
+  if [[ -z "${pair#*:}" ]]; then
+    echo "gen-models: failed to read ${pair%%:*} from the registry" >&2
+    exit 1
+  fi
+done
+
+cat > "$OUT" <<EOF
+[
+  "anthropic:messages:$OPUS",
+  "anthropic:messages:$SONNET",
+  "anthropic:messages:$HAIKU"
+]
+EOF
+
+echo "gen-models: wrote $OUT"

package/skills/eval-harness/scripts/gen-skill-prompt.cjs

@@ -0,0 +1,177 @@
+#!/usr/bin/env node
+// gen-skill-prompt.cjs — project a gated classifier block into a promptfoo
+// skill-arm prompt.
+//
+// The skill-arm prompt is a MECHANICAL PROJECTION of the source block (extracted
+// by extract-block.cjs), not a hand-copied paraphrase. This file owns the fixed
+// wrapper (instruction header + the block + the {{input}} trailer) per target.
+// Regenerate the committed prompts/<target>-skill.txt whenever the source block
+// changes; the AC4 round-trip test asserts generated == committed byte-for-byte.
+//
+// renderSkillPrompt(target, block, tokens) is PURE (text in, text out). The CLI
+// reads the source file on disk via gated-skills.json, extracts the block, reads
+// the target's enum for the token list, renders, and writes the projection.
+//
+// CLI:
+//   node gen-skill-prompt.cjs <target>        # write prompts/<target>-skill.txt
+//   node gen-skill-prompt.cjs --all           # regenerate every gated target
+//   node gen-skill-prompt.cjs <target> --stdout   # print, do not write
+"use strict";
+
+const fs = require("node:fs");
+const path = require("node:path");
+const { extractBlock } = require("./extract-block.cjs");
+
+const SKILL_DIR = path.resolve(__dirname, "..");
+const REPO_ROOT = path.resolve(SKILL_DIR, "..", "..", "..", "..");
+const REGISTRY = path.join(SKILL_DIR, "gated-skills.json");
+
+// Per-target wrapper. `enumPath` is relative to the skill dir; `render` is pure.
+const TARGET_CONFIG = {
+  "go-routing": {
+    enumPath: "enums/go-routes.json",
+    render(block, tokens) {
+      return [
+        "You are the Soleur /go router. Classify the user's request into EXACTLY ONE route token using this routing table.",
+        "",
+        "Routing table:",
+        block,
+        "",
+        `Respond with ONLY the single route token (one of: ${tokens.join(", ")}). No explanation, no punctuation, no extra words.`,
+        "",
+        "User request: {{input}}",
+        "Route:",
+        "",
+      ].join("\n");
+    },
+  },
+  "ticket-triage": {
+    enumPath: "enums/triage-levels.json",
+    render(block, tokens) {
+      return [
+        "You are the Soleur ticket-triage classifier. Assign a priority level to the GitHub issue using this rubric.",
+        "",
+        "Priority rubric:",
+        block,
+        "",
+        `Respond with ONLY the single priority token (one of: ${tokens.join(", ")}). No explanation, no extra words.`,
+        "",
+        "Issue:",
+        "{{input}}",
+        "Priority:",
+        "",
+      ].join("\n");
+    },
+  },
+  "lane-inference": {
+    enumPath: "enums/lane.json",
+    render(block, tokens) {
+      return [
+        "You are the Soleur brainstorm lane classifier. Classify the feature description into EXACTLY ONE lane using this rule.",
+        "",
+        "Lane inference rule:",
+        block,
+        "",
+        `Respond with ONLY the single lane token (one of: ${tokens.join(", ")}). No explanation, no punctuation, no extra words.`,
+        "",
+        "Feature description: {{input}}",
+        "Lane:",
+        "",
+      ].join("\n");
+    },
+  },
+  "incident-threshold": {
+    enumPath: "enums/incident-threshold.json",
+    render(block, tokens) {
+      return [
+        "You are the Soleur incident brand-survival classifier. Assign a threshold to the incident description using this rubric.",
+        "",
+        "Classification rubric:",
+        block,
+        "",
+        `Respond with ONLY the single threshold value (exactly one of: ${tokens.join(", ")}). No explanation, no extra words.`,
+        "",
+        "Incident:",
+        "{{input}}",
+        "Threshold:",
+        "",
+      ].join("\n");
+    },
+  },
+};
+
+// renderSkillPrompt(target, block, tokens): PURE projection of a block into the
+// target's skill-arm prompt. Throws on an unknown target (fail-closed).
+function renderSkillPrompt(target, block, tokens) {
+  const cfg = TARGET_CONFIG[target];
+  if (!cfg) {
+    throw new Error(`gen-skill-prompt: unknown target ${JSON.stringify(target)}`);
+  }
+  return cfg.render(block, tokens);
+}
+
+function loadRegistry() {
+  return JSON.parse(fs.readFileSync(REGISTRY, "utf8"));
+}
+
+function entryFor(target) {
+  const entry = loadRegistry().find((e) => e.target === target);
+  if (!entry) {
+    throw new Error(`gen-skill-prompt: target ${JSON.stringify(target)} not in gated-skills.json`);
+  }
+  return entry;
+}
+
+function tokensFor(target) {
+  const cfg = TARGET_CONFIG[target];
+  const enumAbs = path.join(SKILL_DIR, cfg.enumPath);
+  const arr = JSON.parse(fs.readFileSync(enumAbs, "utf8"));
+  if (!Array.isArray(arr)) {
+    throw new Error(`gen-skill-prompt: enum ${cfg.enumPath} is not an array`);
+  }
+  return arr;
+}
+
+// generateFromDisk(target): read source_file, extract the block, render. Returns
+// { prompt, projectedPromptPath }.
+function generateFromDisk(target) {
+  const entry = entryFor(target);
+  const sourceAbs = path.join(REPO_ROOT, entry.source_file);
+  const sourceText = fs.readFileSync(sourceAbs, "utf8");
+  const block = extractBlock(sourceText, entry.block_start_marker, entry.block_end_marker);
+  const prompt = renderSkillPrompt(target, block, tokensFor(target));
+  return { prompt, projectedPromptPath: path.join(REPO_ROOT, entry.projected_prompt_path) };
+}
+
+module.exports = { renderSkillPrompt, generateFromDisk, TARGET_CONFIG, tokensFor };
+
+if (require.main === module) {
+  const args = process.argv.slice(2);
+  const toStdout = args.includes("--stdout");
+  const all = args.includes("--all");
+  const targets = all
+    ? loadRegistry().map((e) => e.target)
+    : args.filter((a) => !a.startsWith("--"));
+
+  if (targets.length === 0) {
+    process.stderr.write(
+      "usage: node gen-skill-prompt.cjs <target> [--stdout] | --all\n",
+    );
+    process.exit(2);
+  }
+
+  try {
+    for (const target of targets) {
+      const { prompt, projectedPromptPath } = generateFromDisk(target);
+      if (toStdout) {
+        process.stdout.write(prompt);
+      } else {
+        fs.writeFileSync(projectedPromptPath, prompt);
+        process.stderr.write(`gen-skill-prompt: wrote ${projectedPromptPath}\n`);
+      }
+    }
+  } catch (err) {
+    process.stderr.write(`${err.message}\n`);
+    process.exit(1);
+  }
+}

package/skills/eval-harness/scripts/measure-classification.cjs

@@ -0,0 +1,30 @@
+// MEASUREMENT assert (the ponytail `loc.js` analog).
+//
+// promptfoo custom JavaScript assert. Records the classification-correct rate as
+// a score and ALWAYS passes — it measures, it never gates. The recorded number is
+// the metric under test: 1.0 when the model's emitted label matches the golden
+// label, 0.0 otherwise. Aggregated across runs (repeat: N) and across the skill
+// vs baseline arms, the mean of this score IS the baseline-vs-skill delta.
+//
+// Signature: module.exports = (output, context) => GradingResult
+//   context.vars.golden_label : the expected label (route or P-level)
+//   context.vars.enum         : the closed set of allowed labels — array or a
+//   `file://`/relative path to the SSOT (resolved by loadEnum()).
+"use strict";
+
+const { extractLabel, loadEnum } = require("./parse-label.cjs");
+
+module.exports = (output, context) => {
+  const vars = (context && context.vars) || {};
+  const golden = vars.golden_label;
+  const allowed = loadEnum(vars);
+  const got = extractLabel(output, allowed);
+  const correct = got === golden;
+  return {
+    pass: true, // measurement only — never gates
+    score: correct ? 1.0 : 0.0,
+    reason: correct
+      ? `classification-correct: ${got}`
+      : `classification-incorrect: got ${JSON.stringify(got)}, expected ${JSON.stringify(golden)}`,
+  };
+};