@soleur/opencode 0.0.0-stage → 3.354.2

package/skills/eval-harness/scripts/parse-label.cjs

@@ -0,0 +1,93 @@
+// Shared label parser for the eval-harness asserts.
+//
+// extractLabel(output, allowed): given a raw model output string and the closed
+// set of allowed labels (the target's enum), return the canonical enum member the
+// output denotes, or — when no enum member is present — the raw first non-empty
+// line (so the gate assert can reject a malformed / hallucinated label).
+//
+// Classifier-agnostic: works for /go routes (lowercase, hyphenated) and
+// ticket-triage P-levels (P1/P2/P3) alike. Matching is case-insensitive; a
+// hyphen counts as part of a label token so "clo-attestation" is not split.
+"use strict";
+
+const fs = require("node:fs");
+const path = require("node:path");
+
+// loadEnum(vars): resolve the target's closed label set to a real array.
+//
+// promptfoo's handling of a `defaultTest.vars` `file://*.json` value passed to a
+// custom JS assert is version-dependent — confirmed empirically that it may send
+// EITHER the RESOLVED FILE CONTENTS (the JSON array text as a string) OR the literal
+// unresolved ref `"file://enums/...json"`. The assert must handle both, plus a direct
+// array (unit tests). Returns [] on any failure so the gate fails closed (out-of-enum)
+// rather than throwing. (An earlier change dropped the JSON-array-text branch as
+// "speculative"; that silently broke the gate — promptfoo's resolved-contents shape IS
+// that branch's input. The regression guard lives in test/parse-label.test.sh.)
+function loadEnum(vars) {
+  const raw = vars && vars.enum;
+  if (Array.isArray(raw)) return raw;
+  if (typeof raw !== "string" || raw.trim() === "") return [];
+  const s = raw.trim();
+  // (a) JSON array literal / resolved file contents — starts with `[`, JSON.parse.
+  // (b) a `file://`/relative/absolute path — strip `file://`, read the file.
+  if (s.startsWith("[")) {
+    try {
+      const a = JSON.parse(s);
+      if (Array.isArray(a)) return a;
+    } catch {
+      /* not valid JSON — fall through to file resolution */
+    }
+  }
+  const ref = s.replace(/^file:\/\//, "");
+  const p = path.isAbsolute(ref) ? ref : path.resolve(__dirname, "..", ref);
+  try {
+    const a = JSON.parse(fs.readFileSync(p, "utf8"));
+    if (Array.isArray(a)) return a;
+  } catch {
+    /* unreadable / not an array — fall through */
+  }
+  return [];
+}
+
+function extractLabel(output, allowed) {
+  const text = String(output == null ? "" : output).trim();
+  const set = Array.isArray(allowed) ? allowed : [];
+
+  // 1. Exact (case-insensitive) match against the whole trimmed output.
+  for (const a of set) {
+    if (text.toLowerCase() === String(a).toLowerCase()) return a;
+  }
+
+  // 2. Word-boundary search: return the allowed label that appears EARLIEST in
+  //    the text (lowest character position), NOT the first member in enum order.
+  //    Enum-order would bias the result toward whichever label is declared first
+  //    in the type, independent of what the model actually said. Tie-break =
+  //    earliest mention. Hedged / negated outputs ("not P1, use P2") are
+  //    inherently ambiguous and resolve to the first-mentioned label — the
+  //    harness's validity rests on the prompt's "respond with ONLY the token"
+  //    instruction (handled by the exact-match tier above); this tier is a
+  //    lenient fallback for disobedient prose. The hyphen is treated as an
+  //    in-token character so "clo-attestation" is not split.
+  let best = null;
+  let bestIdx = Infinity;
+  for (const a of set) {
+    const esc = String(a).replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+    const re = new RegExp(`(^|[^A-Za-z0-9-])(${esc})([^A-Za-z0-9-]|$)`, "i");
+    const m = re.exec(text);
+    if (m) {
+      const idx = m.index + m[1].length; // position of the label token itself
+      if (idx < bestIdx) {
+        bestIdx = idx;
+        best = a;
+      }
+    }
+  }
+  if (best !== null) return best;
+
+  // 3. No enum member found — return the raw first non-empty line so a gate
+  //    assert can reject it as out-of-enum.
+  const firstLine = text.split(/\r?\n/).map((s) => s.trim()).filter(Boolean)[0];
+  return firstLine == null ? text : firstLine;
+}
+
+module.exports = { extractLabel, loadEnum };

package/skills/eval-harness/scripts/verdict.cjs

@@ -0,0 +1,132 @@
+// verdict.cjs — PURE accept/reject computation for a gated classifier-skill edit.
+//
+// NO I/O, NO shell-out, NO LLM. The only deterministic seam in the gate: it consumes
+// already-scored promptfoo results and decides whether a candidate block edit is safe
+// to apply. The orchestrator (eval-gate.cjs) runs promptfoo and normalizes its JSON
+// into the shape below; the verdict math lives here so it is unit-testable with zero
+// API spend (AC2).
+//
+// Normalized results shape
+// ------------------------
+//   currentResults, candidateResults : Array<{ task_id: string, correct: boolean }>
+//     One entry per model×repeat sample per corpus task. "current" = the block on disk
+//     today; "candidate" = the edited block. The target task's samples are included in
+//     these arrays (tagged with targetTask.task_id) and are SEPARATED OUT internally:
+//       - corpus rate excludes the target task_id (avoids double-counting the row whose
+//         pass we test independently);
+//       - target_task_passes uses ONLY the candidate samples for targetTask.task_id.
+//   targetTask : { task_id }  — the row whose candidate samples must pass.
+//   opts : { epsilon?, targetThreshold? }
+//     epsilon default = 1 / (number of distinct corpus task_ids)  [one-task-equivalent].
+//     targetThreshold default = 0.5.
+//
+// Decision
+// --------
+//   current_rate       = pooled mean correctness over current corpus samples (ex-target)
+//   candidate_rate     = pooled mean correctness over candidate corpus samples (ex-target)
+//   corpus_regressed   = candidate_rate < current_rate - epsilon   (strict < ; boundary
+//                        equality is NOT a regression — inclusive)
+//   target_task_passes = pooled mean correctness of candidate target samples >= targetThreshold
+//   corpus_untrustworthy = current_rate < MIN_TRUSTED_CORPUS_RATE  (absolute floor: below
+//                        this the baseline eval is too degraded to trust a RELATIVE
+//                        comparison — a "no regression" verdict off a broken baseline is
+//                        meaningless)
+//   accept             = !corpus_regressed && target_task_passes && !corpus_untrustworthy
+//
+// Fail-closed: malformed input throws (the orchestrator treats a throw as NOT accept).
+"use strict";
+
+// Absolute corpus trust floor: below this current_rate the baseline eval is too degraded
+// to trust a relative current-vs-candidate comparison, so accept is forced false.
+const MIN_TRUSTED_CORPUS_RATE = 0.5;
+
+function assertResults(results, name) {
+  if (!Array.isArray(results)) {
+    throw new TypeError(`computeVerdict: ${name} must be an array`);
+  }
+  for (const r of results) {
+    if (!r || typeof r !== "object") {
+      throw new TypeError(`computeVerdict: ${name} entry must be an object`);
+    }
+    if (typeof r.task_id !== "string" || r.task_id === "") {
+      throw new TypeError(`computeVerdict: ${name} entry missing string task_id`);
+    }
+    if (typeof r.correct !== "boolean") {
+      throw new TypeError(`computeVerdict: ${name} entry .correct must be a boolean`);
+    }
+  }
+}
+
+// poolRate(samples): mean correctness (1.0/0.0 per sample). null when empty.
+function poolRate(samples) {
+  if (samples.length === 0) return null;
+  const correct = samples.reduce((acc, s) => acc + (s.correct ? 1 : 0), 0);
+  return correct / samples.length;
+}
+
+function distinctTaskIds(results) {
+  return [...new Set(results.map((r) => r.task_id))];
+}
+
+function computeVerdict(currentResults, candidateResults, targetTask, opts) {
+  assertResults(currentResults, "currentResults");
+  assertResults(candidateResults, "candidateResults");
+  if (!targetTask || typeof targetTask.task_id !== "string" || targetTask.task_id === "") {
+    throw new TypeError("computeVerdict: targetTask must have a string task_id");
+  }
+  const targetId = targetTask.task_id;
+  const targetThreshold =
+    opts && typeof opts.targetThreshold === "number" ? opts.targetThreshold : 0.5;
+
+  const curCorpus = currentResults.filter((r) => r.task_id !== targetId);
+  const candCorpus = candidateResults.filter((r) => r.task_id !== targetId);
+  const candTarget = candidateResults.filter((r) => r.task_id === targetId);
+
+  if (curCorpus.length === 0 || candCorpus.length === 0) {
+    throw new Error("computeVerdict: no corpus samples (excluding the target task) to compare");
+  }
+  if (candTarget.length === 0) {
+    throw new Error(
+      `computeVerdict: no candidate samples for target task ${JSON.stringify(targetId)}`,
+    );
+  }
+
+  // epsilon default = one-task-equivalent over the distinct corpus tasks.
+  const corpusTaskCount = distinctTaskIds(curCorpus).length;
+  const epsilon =
+    opts && typeof opts.epsilon === "number" ? opts.epsilon : 1 / corpusTaskCount;
+
+  const current_rate = poolRate(curCorpus);
+  const candidate_rate = poolRate(candCorpus);
+  const target_rate = poolRate(candTarget);
+
+  const corpus_regressed = candidate_rate < current_rate - epsilon;
+  const target_task_passes = target_rate >= targetThreshold;
+  const corpus_untrustworthy = current_rate < MIN_TRUSTED_CORPUS_RATE;
+  const accept = !corpus_regressed && target_task_passes && !corpus_untrustworthy;
+
+  // per_task breakdown over the union of corpus + target task ids.
+  const allIds = [...new Set([...distinctTaskIds(currentResults), ...distinctTaskIds(candidateResults)])];
+  const per_task = allIds.map((task_id) => ({
+    task_id,
+    is_target: task_id === targetId,
+    current_rate: poolRate(currentResults.filter((r) => r.task_id === task_id)),
+    candidate_rate: poolRate(candidateResults.filter((r) => r.task_id === task_id)),
+  }));
+
+  return {
+    accept,
+    corpus_regressed,
+    target_task_passes,
+    corpus_untrustworthy,
+    current_rate,
+    candidate_rate,
+    target_rate,
+    epsilon,
+    target_threshold: targetThreshold,
+    min_trusted_corpus_rate: MIN_TRUSTED_CORPUS_RATE,
+    per_task,
+  };
+}
+
+module.exports = { computeVerdict };

package/skills/eval-harness/tasks/go-routing.jsonl

@@ -0,0 +1,8 @@
+{"vars": {"input": "The dashboard throws a 500 when I click Export. It worked yesterday — can you fix it?", "golden_label": "fix"}}
+{"vars": {"input": "The checkout API has been returning 500s to every user for the last 20 minutes. Production is down right now and customers are stuck.", "golden_label": "incident"}}
+{"vars": {"input": "Go through every issue labeled stale-flaky and close them all out — clean up that whole backlog.", "golden_label": "drain"}}
+{"vars": {"input": "A customer emailed asking us to delete all of their personal data and send them a copy of everything we hold on them.", "golden_label": "legal-threshold"}}
+{"vars": {"input": "Before we vendor this dependency — is its AGPL license going to be a problem for our closed-source product?", "golden_label": "legal-threshold"}}
+{"vars": {"input": "Can you review PR #4502 and check whether the auth changes are sound?", "golden_label": "review"}}
+{"vars": {"input": "I'd like to add a dark-mode toggle to the settings page — where should we start?", "golden_label": "default"}}
+{"vars": {"input": "#6325 implement Phase F", "golden_label": "implement"}}

package/skills/eval-harness/tasks/incident-threshold.jsonl

@@ -0,0 +1,7 @@
+{"vars": {"input": "A typo in an internal CI log message; no user-facing surface and no data touched.", "golden_label": "none"}}
+{"vars": {"input": "Updated a developer-only runbook page; no credential, billing, or user surface affected.", "golden_label": "none"}}
+{"vars": {"input": "One customer's API key was accidentally written to an application log for several hours.", "golden_label": "single-user incident"}}
+{"vars": {"input": "A single user's stored payment card failed to save, blocking their subscription for an hour.", "golden_label": "single-user incident"}}
+{"vars": {"input": "A misconfigured query exposed many users' email addresses across multiple tenants for a day.", "golden_label": "aggregate pattern"}}
+{"vars": {"input": "A webhook retry bug double-charged every active subscriber in one billing run.", "golden_label": "aggregate pattern"}}
+{"vars": {"input": "A bad deploy logged out all users across three enterprise tenants and lost their unsaved work.", "golden_label": "aggregate pattern"}}

package/skills/eval-harness/tasks/lane-inference.jsonl

@@ -0,0 +1,7 @@
+{"vars": {"input": "Bump the lockfile and run a format-only pass across the repo; no logic changes.", "golden_label": "procedural"}}
+{"vars": {"input": "Scaffold a new empty docs page directory and do a rename-only cleanup of one file.", "golden_label": "procedural"}}
+{"vars": {"input": "Adjust the pricing tier labels shown on the public marketing site.", "golden_label": "single-domain"}}
+{"vars": {"input": "Refine the onboarding welcome email copy for clarity.", "golden_label": "single-domain"}}
+{"vars": {"input": "Add an audit log capturing every auth event that touches regulated user data.", "golden_label": "cross-domain"}}
+{"vars": {"input": "Migrate billing records to a new payment processor via a schema migration.", "golden_label": "cross-domain"}}
+{"vars": {"input": "Run a security audit, then a lint-fix formatting pass across the infra configs.", "golden_label": "cross-domain"}}

package/skills/eval-harness/tasks/ticket-triage.jsonl

@@ -0,0 +1,6 @@
+{"vars": {"input": "Title: Checkout completely broken in production\n\nEvery payment attempt returns 'card declined', even with known-good test cards. No customer can complete a purchase right now. Started ~15 minutes ago.", "golden_label": "P1"}}
+{"vars": {"input": "Title: New signups cannot activate\n\nLogin works for existing users, but every new signup's email-verification link returns a 404. No new user has been able to activate their account since this morning. No error is thrown — it just dead-ends.", "golden_label": "P1"}}
+{"vars": {"input": "Title: CSV export drops the last row on large workspaces\n\nWorkspaces with more than 100 items lose the final row in the exported CSV. Affects many users. Workaround: export twice and diff. Not urgent but should be fixed soon.", "golden_label": "P2"}}
+{"vars": {"input": "Title: Search is slow on large histories\n\nSearch results take 8-10 seconds to load for workspaces with large conversation histories. Still usable, just sluggish; no data is wrong.", "golden_label": "P2"}}
+{"vars": {"input": "Title: URGENT!! Help tooltip typo\n\nThe footer 'Help' link tooltip reads 'Helpp'. Purely cosmetic — flagging as urgent because it's on every page.", "golden_label": "P3"}}
+{"vars": {"input": "Title: Footer copyright year is stale\n\nThe site footer still shows '2024'. Purely cosmetic, no functional impact.", "golden_label": "P3"}}

package/skills/eval-harness/tasks/tool-selection.jsonl

@@ -0,0 +1,15 @@
+{"vars": {"phase": "brainstorm", "input": "Approaches explored and key decisions captured in the brainstorm doc; the worktree and spec.md are written. The design is settled and we're ready to produce the file-by-file implementation plan with an ordered task list.", "golden_label": "plan"}}
+{"vars": {"phase": "brainstorm", "input": "We chose approach B and documented the rationale and non-goals. The exploration is done; the next move is to convert these settled decisions into an ordered implementation plan with tasks.", "golden_label": "plan"}}
+{"vars": {"phase": "plan", "input": "plan.md and tasks.md are committed. Before any code is written we want to spawn parallel research agents that enrich each plan section with codebase patterns, prior art, and learnings.", "golden_label": "deepen-plan"}}
+{"vars": {"phase": "plan", "input": "The implementation plan is drafted and committed. Before writing code we want independent reviewers (DHH, Kieran, simplicity) to critique the plan's approach and flag over-engineering.", "golden_label": "plan-review"}}
+{"vars": {"phase": "plan", "input": "plan.md and tasks.md are reviewed and approved. There is nothing left to research or critique; it is time to start executing the first task with incremental commits.", "golden_label": "work"}}
+{"vars": {"phase": "work", "input": "Implementation is complete, every task in tasks.md is checked off, and the unit tests pass green. Before opening a PR we want a multi-agent pass to catch bugs and convention issues.", "golden_label": "review"}}
+{"vars": {"phase": "work", "input": "We just finished the GREEN implementation for a UI change under app/(dashboard). jsdom can't catch CSS-layout or nav-state regressions, so we need the auth-seeded headless browser gate before review.", "golden_label": "qa"}}
+{"vars": {"phase": "work", "input": "The implementation is written but the test suite has six failing specs. We want to autonomously iterate diagnosis-and-fix with checkpoint commits until the whole suite is green.", "golden_label": "test-fix-loop"}}
+{"vars": {"phase": "review", "input": "All review findings are resolved and the full suite is green. The feature is verified and ready to commit, push, open the PR, and merge.", "golden_label": "ship"}}
+{"vars": {"phase": "review", "input": "The draft PR has fourteen unresolved reviewer comments spread across several files. We want to address every thread in parallel and mark them resolved.", "golden_label": "resolve-pr-parallel"}}
+{"vars": {"phase": "review", "input": "The review pass is otherwise clean, but it flagged that a dashboard layout change was never visually verified. Before shipping we need the auth-seeded headless browser gate on the affected route.", "golden_label": "qa"}}
+{"vars": {"phase": "ship", "input": "We're about to ship. First we need the pre-flight checks on database migrations, security headers, and lockfile integrity before merging.", "golden_label": "preflight"}}
+{"vars": {"phase": "ship", "input": "The PR is approved, CI is green, and preflight passed. We're ready to merge the feature branch into main with automatic conflict resolution and worktree cleanup.", "golden_label": "merge-pr"}}
+{"vars": {"phase": "ship", "input": "The PR just merged to main. We need to verify the deploy succeeded and that production is healthy before moving on.", "golden_label": "postmerge"}}
+{"vars": {"phase": "ship", "input": "Several PRs merged this week. We want to generate a formatted release summary from their labels, linked issues, and descriptions.", "golden_label": "changelog"}}

package/skills/eval-harness/test/eval-gate.test.sh

@@ -0,0 +1,132 @@
+#!/usr/bin/env bash
+# Deterministic unit test for the eval-gate.cjs orchestrator — NO API.
+# Covers the three code paths that never call promptfoo:
+#   --check     gated-source lookup (true/false)
+#   --dry-run   skill-arm-only API-cost disclosure (no API call)
+#   no-op       candidate block == current block => {accept:true, "no gated-block change"}
+set -euo pipefail
+
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SKILL_DIR="$(cd "$HERE/.." && pwd)"
+REPO_ROOT="$(cd "$SKILL_DIR/../../../.." && pwd)"
+GATE="$SKILL_DIR/scripts/eval-gate.cjs"
+fails=0
+
+pass() { echo "ok   [$1]"; }
+fail() { echo "FAIL [$1]: $2"; fails=$((fails + 1)); }
+
+jqget() { node -e 'const o=JSON.parse(require("fs").readFileSync(0,"utf8"));process.stdout.write(String(o[process.argv[1]]))' "$1"; }
+
+cd "$REPO_ROOT"
+
+# --- --check: a gated source returns gated:true + its target/block_id ---
+out=$(node "$GATE" --check plugins/soleur/commands/go.md)
+if [[ "$(echo "$out" | jqget gated)" == "true" && "$(echo "$out" | jqget target)" == "go-routing" && "$(echo "$out" | jqget block_id)" == "go-routing" ]]; then
+  pass "--check gated source (go.md)"
+else
+  fail "--check gated source (go.md)" "got: $out"
+fi
+
+out=$(node "$GATE" --check plugins/soleur/agents/support/ticket-triage.md)
+if [[ "$(echo "$out" | jqget gated)" == "true" && "$(echo "$out" | jqget target)" == "ticket-triage" ]]; then
+  pass "--check gated source (ticket-triage.md)"
+else
+  fail "--check gated source (ticket-triage.md)" "got: $out"
+fi
+
+# --- --check: a non-gated file returns gated:false ---
+out=$(node "$GATE" --check plugins/soleur/skills/eval-harness/README.md)
+if [[ "$(echo "$out" | jqget gated)" == "false" ]]; then
+  pass "--check non-gated file"
+else
+  fail "--check non-gated file" "got: $out"
+fi
+
+# --- --dry-run: prints estimate, dry_run:true, no API ---
+out=$(node "$GATE" --dry-run --target go-routing --repeat 5)
+if [[ "$(echo "$out" | jqget dry_run)" == "true" ]]; then
+  calls=$(echo "$out" | jqget estimated_api_calls)
+  # 2 (current+candidate) x 3 models x (8 corpus + 1 target) x 5 repeat = 270
+  if [[ "$calls" == "270" ]]; then
+    pass "--dry-run estimate (go-routing, repeat 5) = $calls"
+  else
+    fail "--dry-run estimate" "estimated_api_calls=$calls (want 270)"
+  fi
+else
+  fail "--dry-run" "got: $out"
+fi
+
+# --- no-op: candidate-file is a DISTINCT copy with an unchanged block => accept, no API ---
+# (must be a temp copy, not the live source — Fix 4 refuses an in-place candidate-file).
+tmpcopy="$(mktemp --suffix=.md)"
+trap 'rm -f "$tmpcopy"' EXIT
+cp "$REPO_ROOT/plugins/soleur/commands/go.md" "$tmpcopy"
+out=$(node "$GATE" --target go-routing --candidate-file "$tmpcopy")
+if [[ "$(echo "$out" | jqget accept)" == "true" && "$(echo "$out" | jqget reason)" == "no gated-block change" ]]; then
+  pass "no-op (unchanged block) accepts without API"
+else
+  fail "no-op (unchanged block)" "got: $out"
+fi
+
+# --- Fix 4: candidate-file == the live source file is refused (fail-closed) ---
+out=$(node "$GATE" --target go-routing --candidate-file plugins/soleur/commands/go.md || true)
+if [[ "$(echo "$out" | jqget accept)" == "false" && "$(echo "$out" | jqget error)" == *"must differ from the live source file"* ]]; then
+  pass "candidate-file == live source is refused"
+else
+  fail "candidate-file == live source is refused" "got: $out"
+fi
+
+# --- --check: the two added gated sources resolve to their targets ---
+out=$(node "$GATE" --check plugins/soleur/skills/brainstorm/references/brainstorm-domain-config.md)
+if [[ "$(echo "$out" | jqget gated)" == "true" && "$(echo "$out" | jqget target)" == "lane-inference" && "$(echo "$out" | jqget block_id)" == "lane-inference" ]]; then
+  pass "--check gated source (brainstorm-domain-config.md -> lane-inference)"
+else
+  fail "--check gated source (lane-inference)" "got: $out"
+fi
+
+out=$(node "$GATE" --check plugins/soleur/skills/incident/SKILL.md)
+if [[ "$(echo "$out" | jqget gated)" == "true" && "$(echo "$out" | jqget target)" == "incident-threshold" ]]; then
+  pass "--check gated source (incident/SKILL.md -> incident-threshold)"
+else
+  fail "--check gated source (incident-threshold)" "got: $out"
+fi
+
+# --- gate path: --dry-run resolves TARGET_RESOURCES for the new targets (no API). ---
+# A missing TARGET_RESOURCES entry would die() here — so this is the load-bearing proof
+# that the GATE (not just the measurement config) is actually wired for the new surfaces.
+for t in lane-inference incident-threshold; do
+  out=$(node "$GATE" --dry-run --target "$t" --repeat 5)
+  calls=$(echo "$out" | jqget estimated_api_calls)
+  # 2 (current+candidate) x 3 models x (7 corpus + 1 target) x 5 repeat = 240
+  if [[ "$(echo "$out" | jqget dry_run)" == "true" && "$calls" == "240" ]]; then
+    pass "--dry-run estimate ($t, repeat 5) = $calls"
+  else
+    fail "--dry-run ($t)" "got: $out"
+  fi
+done
+
+# --- registry-coverage: every gated target is wired in BOTH per-target maps ---
+# A target present in gated-skills.json but missing from TARGET_RESOURCES ships the gate
+# DORMANT (--check says gated:true while --target/--dry-run die fail-closed). This guard
+# makes that drift fail CI for any future target, not just these two.
+if node -e '
+  const reg = require(process.argv[1]);
+  const { TARGET_CONFIG } = require(process.argv[2]);
+  const { TARGET_RESOURCES } = require(process.argv[3]);
+  const missing = [];
+  for (const e of reg) {
+    if (!TARGET_CONFIG[e.target]) missing.push(e.target + " (gen-skill-prompt TARGET_CONFIG)");
+    if (!TARGET_RESOURCES[e.target]) missing.push(e.target + " (eval-gate TARGET_RESOURCES)");
+  }
+  if (missing.length) { process.stderr.write("uncovered targets: " + missing.join(", ") + "\n"); process.exit(1); }
+' "$SKILL_DIR/gated-skills.json" "$SKILL_DIR/scripts/gen-skill-prompt.cjs" "$SKILL_DIR/scripts/eval-gate.cjs" 2>/tmp/eval-gate-coverage.err; then
+  pass "registry-coverage: every target in TARGET_CONFIG and TARGET_RESOURCES"
+else
+  fail "registry-coverage" "$(cat /tmp/eval-gate-coverage.err)"
+fi
+
+if [[ "$fails" -gt 0 ]]; then
+  echo "eval-gate: $fails assertion(s) failed"
+  exit 1
+fi
+echo "eval-gate: all assertions passed"

package/skills/eval-harness/test/extract-block.test.sh

@@ -0,0 +1,87 @@
+#!/usr/bin/env bash
+# Deterministic unit test for extract-block.cjs + gen-skill-prompt.cjs (no live LLM).
+# Asserts:
+#   AC4 round-trip — regenerating each target's projected prompt from the source
+#                    block equals the committed projection byte-for-byte.
+#   extractBlock throws loudly when a marker is missing (fail-closed).
+#   extractBlock returns the trimmed text strictly between the markers.
+set -euo pipefail
+SUITE_TMP=$(mktemp -d)
+[[ "$SUITE_TMP" == /* ]] || { echo "FATAL: SUITE_TMP is not absolute: $SUITE_TMP" >&2; exit 2; }
+trap 'rm -rf "$SUITE_TMP"' EXIT
+
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SKILL_DIR="$(cd "$HERE/.." && pwd)"
+REPO_ROOT="$(cd "$SKILL_DIR/../../../.." && pwd)"
+EXTRACT="$SKILL_DIR/scripts/extract-block.cjs"
+GEN="$SKILL_DIR/scripts/gen-skill-prompt.cjs"
+REGISTRY="$SKILL_DIR/gated-skills.json"
+fails=0
+
+pass() { echo "ok   [$1]"; }
+fail() { echo "FAIL [$1]: $2"; fails=$((fails + 1)); }
+
+# --- AC4 round-trip: generated projection == committed projection (per registry target) ---
+# Target loop is derived from gated-skills.json (target + projected_prompt_path) rather than
+# a hardcoded list, so a newly-registered gated classifier is round-trip-covered automatically.
+# projected_prompt_path is repo-root-relative (e.g. plugins/soleur/skills/eval-harness/prompts/go-skill.txt).
+roundtrips=0
+while IFS=$'\t' read -r target projected; do
+  [[ -z "$target" ]] && continue
+  committed="$REPO_ROOT/$projected"
+  if diff -u <(node "$GEN" "$target" --stdout) "$committed" > "$SUITE_TMP/eval-gate-roundtrip.diff" 2>&1; then
+    pass "round-trip $target == committed projection"
+  else
+    fail "round-trip $target" "generated projection differs from committed $committed (run: node scripts/gen-skill-prompt.cjs $target)"
+    cat "$SUITE_TMP/eval-gate-roundtrip.diff"
+  fi
+  roundtrips=$((roundtrips + 1))
+done < <(node -e 'JSON.parse(require("fs").readFileSync(process.argv[1],"utf8")).forEach(e=>process.stdout.write(e.target+"\t"+e.projected_prompt_path+"\n"))' "$REGISTRY")
+
+# Minimum-cardinality guard: a registry-derived loop silently exits 0 with ZERO coverage if the
+# registry is empty / unreadable / the node one-liner emits nothing. Fail loud instead.
+if [[ "$roundtrips" -lt 1 ]]; then
+  fail "registry round-trip coverage" "0 round-trip targets executed — empty/unreadable $REGISTRY would otherwise pass with no coverage"
+fi
+
+# --- extractBlock returns trimmed text between markers ---
+got=$(node -e '
+  const { extractBlock } = require(process.argv[1]);
+  const src = "pre\n<!-- eval-gate:block:x:start -->\n  HELLO  \n<!-- eval-gate:block:x:end -->\npost";
+  process.stdout.write(extractBlock(src, "<!-- eval-gate:block:x:start -->", "<!-- eval-gate:block:x:end -->"));
+' "$EXTRACT")
+if [[ "$got" == "HELLO" ]]; then
+  pass "extractBlock trims block text"
+else
+  fail "extractBlock trims block text" "got '$got' (want 'HELLO')"
+fi
+
+# --- extractBlock throws loudly on a missing marker (fail-closed) ---
+if node -e '
+  const { extractBlock } = require(process.argv[1]);
+  extractBlock("no markers here", "<!-- eval-gate:block:x:start -->", "<!-- eval-gate:block:x:end -->");
+' "$EXTRACT" 2>/dev/null; then
+  fail "extractBlock missing-marker throws" "did not throw on a missing start marker"
+else
+  pass "extractBlock missing-marker throws (fail-closed)"
+fi
+
+# --- CLI exits non-zero with stderr when a marker is missing ---
+tmp="$(mktemp -p "$SUITE_TMP")"
+printf 'this file has no sentinels\n' > "$tmp"
+if node "$EXTRACT" "$tmp" go-routing 2>"$SUITE_TMP/eval-gate-cli.err"; then
+  fail "extract-block CLI missing-marker exit" "CLI exited 0 on a file without markers"
+else
+  if grep -q "marker not found" "$SUITE_TMP/eval-gate-cli.err"; then
+    pass "extract-block CLI missing-marker exits non-zero with clear stderr"
+  else
+    fail "extract-block CLI missing-marker stderr" "stderr lacked a clear message: $(cat "$SUITE_TMP/eval-gate-cli.err")"
+  fi
+fi
+rm -f "$tmp"
+
+if [[ "$fails" -gt 0 ]]; then
+  echo "extract-block: $fails assertion(s) failed"
+  exit 1
+fi
+echo "extract-block: all assertions passed"

package/skills/eval-harness/test/gate-classification.test.sh

@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+# Deterministic unit test for the GATE assert (scripts/gate-classification.cjs).
+# No live LLM: every "model output" is a stubbed string. Asserts the contract:
+#   - pass: true  when the parsed label IS a member of the target's enum
+#   - pass: false when the parsed label is NOT in the enum (malformed/hallucinated)
+#   - exercised for BOTH the 7-route enum AND the P1/P2/P3 enum
+set -euo pipefail
+
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SKILL_DIR="$(cd "$HERE/.." && pwd)"
+ASSERT="$SKILL_DIR/scripts/gate-classification.cjs"
+fails=0
+
+# run <output> <enum-json> <expect-pass> <label>
+run() {
+  local output="$1" enumjson="$2" want_pass="$3" label="$4"
+  local got_pass
+  got_pass=$(node -e '
+    const fn = require(process.argv[1]);
+    const r = fn(process.argv[2], { vars: { enum: JSON.parse(process.argv[3]) } });
+    process.stdout.write(String(r.pass));
+  ' "$ASSERT" "$output" "$enumjson" )
+  if [[ "$got_pass" != "$want_pass" ]]; then
+    echo "FAIL [$label]: output='$output' -> pass=$got_pass (want $want_pass)"
+    fails=$((fails + 1))
+  else
+    echo "ok   [$label]: pass=$got_pass"
+  fi
+}
+
+GO='["fix","drain","clo-attestation","review","legal-threshold","incident","default"]'
+TR='["P1","P2","P3"]'
+
+# --- /go route enum ---
+run "fix"                    "$GO" "true"  "in-enum route passes"
+run "clo-attestation"        "$GO" "true"  "in-enum hyphenated route passes"
+run "The route is review."   "$GO" "true"  "in-enum route in prose passes"
+run "banana"                 "$GO" "false" "out-of-enum route fails"
+run "P1"                     "$GO" "false" "wrong-namespace label fails"
+
+# --- ticket-triage P-level enum ---
+run "P1"                     "$TR" "true"  "in-enum P-level passes"
+run "p2"                     "$TR" "true"  "in-enum P-level case-insensitive passes"
+run "P9"                     "$TR" "false" "out-of-enum P-level fails"
+run "fix"                    "$TR" "false" "wrong-namespace label fails"
+
+# --- file:// enum shape (promptfoo passes defaultTest.vars file:// refs verbatim;
+#     the #5358 live-run bug). enum arrives as a RAW STRING, not a parsed array. ---
+fileref() {
+  local output="$1" enumref="$2" want_pass="$3" label="$4"
+  local got
+  got=$(node -e '
+    const fn = require(process.argv[1]);
+    const r = fn(process.argv[2], { vars: { enum: process.argv[3] } });
+    process.stdout.write(String(r.pass));
+  ' "$ASSERT" "$output" "$enumref")
+  if [[ "$got" != "$want_pass" ]]; then
+    echo "FAIL [$label]: output='$output' enum='$enumref' -> pass=$got (want $want_pass)"
+    fails=$((fails + 1))
+  else
+    echo "ok   [$label]: pass=$got"
+  fi
+}
+fileref "fix"    "file://enums/go-routes.json"     "true"  "file:// enum + in-enum route passes"
+fileref "banana" "file://enums/go-routes.json"     "false" "file:// enum + out-of-enum route fails"
+fileref "P1"     "file://enums/triage-levels.json" "true"  "file:// enum + in-enum P-level passes"
+# Resolved-contents shape (JSON array text) — what promptfoo actually passes when it
+# resolves a defaultTest.vars file:// ref to file CONTENTS. Regression guard for the
+# gate-vacuous bug (the enum arrived as JSON-array-text, not a path/literal-ref).
+fileref "fix"    '["fix","drain","clo-attestation","review","legal-threshold","incident","default"]' "true"  "contents-string enum + in-enum passes"
+fileref "banana" '["fix","drain","clo-attestation","review","legal-threshold","incident","default"]' "false" "contents-string enum + out-of-enum fails"
+
+if [[ "$fails" -gt 0 ]]; then
+  echo "gate-classification: $fails assertion(s) failed"
+  exit 1
+fi
+echo "gate-classification: all assertions passed"