@soleur/opencode 0.0.0-stage → 3.354.2

package/skills/eval-harness/test/gen-models.test.sh

@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+# Deterministic test for scripts/gen-models.sh: the generated models.generated.json
+# must contain exactly the three current model IDs read VERBATIM from the TS registry
+# (single source of truth), each wrapped in the `anthropic:messages:` provider prefix.
+# No hardcoded model literals here — every expected value is re-derived from the registry.
+set -euo pipefail
+
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SKILL_DIR="$(cd "$HERE/.." && pwd)"
+ROOT="$(git -C "$SKILL_DIR" rev-parse --show-toplevel)"
+CONST="$ROOT/apps/web-platform/server/inngest/leader-prompts/constants.ts"
+TIERS="$ROOT/apps/web-platform/server/inngest/model-tiers.ts"
+GEN="$SKILL_DIR/scripts/gen-models.sh"
+OUT="$SKILL_DIR/models.generated.json"
+fails=0
+
+# Re-derive the expected IDs from the registry (mirrors gen-models.sh extraction).
+extract() { grep -E "^export const $2" "$1" | grep -oE '"claude-[^"]+"' | tr -d '"' | head -1; }
+SONNET=$(extract "$CONST" "SONNET_MODEL")
+HAIKU=$(extract "$CONST" "HAIKU_MODEL")
+OPUS=$(extract "$TIERS" "AUDIT_MODEL")
+
+for v in "$SONNET" "$HAIKU" "$OPUS"; do
+  if [[ -z "$v" ]]; then echo "FAIL: could not read a model ID from the registry"; exit 1; fi
+done
+
+# COMMITTED-ARTIFACT PARITY. Running the generator in place and then asserting
+# against what it just wrote can only prove the generator is self-consistent —
+# it silently CLOBBERS a hand-edit instead of reporting it, so the checked-in
+# file could drift from the registry indefinitely with this test green. Snapshot
+# first, regenerate, and diff: the committed file must already be correct.
+if [[ ! -f "$OUT" ]]; then
+  echo "FAIL: $OUT is missing from the repo"; exit 1
+fi
+_committed="$(mktemp)"
+# Single owning trap (ADR-129) — covers every exit path below, including the
+# STALE failure and any unexpected die between allocation and the diff.
+trap 'rm -f "$_committed"' EXIT
+cp "$OUT" "$_committed"
+
+bash "$GEN"
+
+if ! diff -q "$_committed" "$OUT" >/dev/null 2>&1; then
+  echo "FAIL: $OUT is STALE — it differs from what gen-models.sh produces."
+  echo "      Run: bash $GEN   (and commit the result)"
+  diff -u "$_committed" "$OUT" || true
+  exit 1
+fi
+echo "ok   committed artifact matches the generator (no drift)"
+
+check() {
+  local needle="anthropic:messages:$1"
+  if node -e 'const a=require(process.argv[1]); process.exit(a.includes(process.argv[2])?0:1)' "$OUT" "$needle"; then
+    echo "ok   present: $needle"
+  else
+    echo "FAIL missing: $needle"
+    fails=$((fails + 1))
+  fi
+}
+check "$OPUS"
+check "$SONNET"
+check "$HAIKU"
+
+# Exactly three providers, no extras, no hardcoded literals leaked.
+COUNT=$(node -e 'const a=require(process.argv[1]); process.stdout.write(String(a.length))' "$OUT")
+if [[ "$COUNT" != "3" ]]; then
+  echo "FAIL: expected 3 providers, got $COUNT"
+  fails=$((fails + 1))
+fi
+
+if [[ "$fails" -gt 0 ]]; then
+  echo "gen-models: $fails assertion(s) failed"
+  exit 1
+fi
+echo "gen-models: all assertions passed"

package/skills/eval-harness/test/measure-classification.test.sh

@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+# Deterministic unit test for the MEASUREMENT assert (scripts/measure-classification.cjs).
+# No live LLM: every "model output" is a stubbed string. Asserts the contract:
+#   - ALWAYS pass: true (the measurement records, never gates)
+#   - score 1.0 when the parsed label matches golden_label, 0.0 otherwise
+#   - works for BOTH a /go route AND a ticket-triage P-level
+set -euo pipefail
+
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SKILL_DIR="$(cd "$HERE/.." && pwd)"
+ASSERT="$SKILL_DIR/scripts/measure-classification.cjs"
+fails=0
+
+# run <output> <golden_label> <enum-json> <expect-pass> <expect-score> <label>
+run() {
+  local output="$1" golden="$2" enumjson="$3" want_pass="$4" want_score="$5" label="$6"
+  local result
+  result=$(node -e '
+    const fn = require(process.argv[1]);
+    const r = fn(process.argv[2], { vars: { golden_label: process.argv[3], enum: JSON.parse(process.argv[4]) } });
+    process.stdout.write(JSON.stringify({ pass: r.pass, score: r.score }));
+  ' "$ASSERT" "$output" "$golden" "$enumjson" )
+  local got_pass got_score
+  got_pass=$(node -e 'process.stdout.write(String(JSON.parse(process.argv[1]).pass))' "$result")
+  got_score=$(node -e 'process.stdout.write(String(JSON.parse(process.argv[1]).score))' "$result")
+  if [[ "$got_pass" != "$want_pass" || "$got_score" != "$want_score" ]]; then
+    echo "FAIL [$label]: output='$output' golden='$golden' -> pass=$got_pass score=$got_score (want pass=$want_pass score=$want_score)"
+    fails=$((fails + 1))
+  else
+    echo "ok   [$label]: pass=$got_pass score=$got_score"
+  fi
+}
+
+GO='["fix","drain","clo-attestation","review","legal-threshold","incident","default"]'
+TR='["P1","P2","P3"]'
+
+# --- /go route cases ---
+run "fix"                     "fix"     "$GO" "true" "1" "route exact match"
+run "The route is \`fix\`."   "fix"     "$GO" "true" "1" "route embedded in prose"
+run "default"                 "fix"     "$GO" "true" "0" "route mismatch scores 0 but still passes"
+run "incident"                "incident" "$GO" "true" "1" "non-default route match"
+run "banana"                  "fix"     "$GO" "true" "0" "hallucinated route scores 0 but still passes"
+
+# --- ticket-triage P-level cases ---
+run "P1"                      "P1"      "$TR" "true" "1" "P-level exact match"
+run "Priority: P2"            "P2"      "$TR" "true" "1" "P-level embedded in prose"
+run "p3"                      "P3"      "$TR" "true" "1" "P-level case-insensitive match"
+run "P1"                      "P3"      "$TR" "true" "0" "P-level mismatch scores 0 but still passes"
+
+# --- file:// enum shape (the promptfoo defaultTest.vars shape — #5358 regression).
+#     enum arrives as a RAW STRING; loadEnum must read the SSOT file to score. ---
+fileref() {
+  local output="$1" golden="$2" enumref="$3" want_score="$4" label="$5"
+  local got
+  got=$(node -e '
+    const fn = require(process.argv[1]);
+    const r = fn(process.argv[2], { vars: { golden_label: process.argv[3], enum: process.argv[4] } });
+    process.stdout.write(String(r.score));
+  ' "$ASSERT" "$output" "$golden" "$enumref")
+  if [[ "$got" != "$want_score" ]]; then
+    echo "FAIL [$label]: output='$output' enum='$enumref' -> score=$got (want $want_score)"
+    fails=$((fails + 1))
+  else
+    echo "ok   [$label]: score=$got"
+  fi
+}
+fileref "fix" "fix" "file://enums/go-routes.json"     "1" "file:// enum + correct route scores 1"
+fileref "P2"  "P2"  "file://enums/triage-levels.json" "1" "file:// enum + correct P-level scores 1"
+
+if [[ "$fails" -gt 0 ]]; then
+  echo "measure-classification: $fails assertion(s) failed"
+  exit 1
+fi
+echo "measure-classification: all assertions passed"

package/skills/eval-harness/test/parse-label.test.sh

@@ -0,0 +1,129 @@
+#!/usr/bin/env bash
+# Deterministic unit test for the shared label parser (scripts/parse-label.cjs).
+# Pins the documented three-tier contract:
+#   1. exact (case-insensitive) match against the whole trimmed output
+#   2. word-boundary search → the EARLIEST-occurring allowed label (not enum order)
+#   3. fallback → the raw first non-empty line (so the gate can reject it)
+# Also pins the documented limitations: hyphenated labels are not split, a label
+# that is a strict substring of a longer token does not match, and hedged/negated
+# prose resolves to the first-mentioned label.
+set -euo pipefail
+
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SKILL_DIR="$(cd "$HERE/.." && pwd)"
+PARSER="$SKILL_DIR/scripts/parse-label.cjs"
+fails=0
+
+# run <output> <enum-json> <expected-label> <case-label>
+run() {
+  local output="$1" enumjson="$2" want="$3" label="$4"
+  local got
+  got=$(node -e '
+    const { extractLabel } = require(process.argv[1]);
+    process.stdout.write(String(extractLabel(process.argv[2], JSON.parse(process.argv[3]))));
+  ' "$PARSER" "$output" "$enumjson")
+  if [[ "$got" != "$want" ]]; then
+    echo "FAIL [$label]: output='$output' -> '$got' (want '$want')"
+    fails=$((fails + 1))
+  else
+    echo "ok   [$label]: '$got'"
+  fi
+}
+
+# run_empty <enum-json> <expected> <case-label>  (output is the empty/undefined case)
+run_empty() {
+  local enumjson="$1" want="$2" label="$3"
+  local got
+  got=$(node -e '
+    const { extractLabel } = require(process.argv[1]);
+    const out = process.argv[3] === "__NULL__" ? null : process.argv[3];
+    process.stdout.write(String(extractLabel(out, JSON.parse(process.argv[2]))));
+  ' "$PARSER" "$enumjson" "$4")
+  if [[ "$got" != "$want" ]]; then
+    echo "FAIL [$label]: -> '$got' (want '$want')"
+    fails=$((fails + 1))
+  else
+    echo "ok   [$label]: '$got'"
+  fi
+}
+
+GO='["fix","drain","clo-attestation","review","legal-threshold","incident","default"]'
+TR='["P1","P2","P3"]'
+
+# --- Tier 1: exact match ---
+run "fix"                     "$GO" "fix"             "tier1 exact route"
+run "P1"                      "$TR" "P1"              "tier1 exact P-level"
+run "p2"                      "$TR" "P2"              "tier1 exact case-insensitive"
+run "  review  "              "$GO" "review"          "tier1 exact after trim"
+
+# --- Tier 2: word-boundary, EARLIEST-occurrence wins (not enum order) ---
+run "The route is review."    "$GO" "review"          "tier2 single label in prose"
+run "review, not fix"         "$GO" "review"          "tier2 earliest wins over enum order (fix precedes review in enum)"
+run "default, or maybe fix"   "$GO" "default"         "tier2 earliest wins (default precedes fix in text)"
+run "Route: incident"         "$GO" "incident"        "tier2 label after prompt label"
+run "clo-attestation"         "$GO" "clo-attestation" "tier2 hyphenated label not split"
+run "use legal-threshold here" "$GO" "legal-threshold" "tier2 hyphenated label in prose"
+# Documented limitation: negation resolves to the first-mentioned label.
+run "not P1, use P2"          "$TR" "P1"              "tier2 documented negation limitation (first-mentioned)"
+
+# --- Tier 2 substring safety: a strict substring of a longer token must NOT match ---
+run "P10"                     "$TR" "P10"             "tier2 P1 not matched inside P10 (falls to raw)"
+
+# --- Tier 3: fallback to raw first non-empty line (gate will reject) ---
+run "banana"                  "$GO" "banana"          "tier3 out-of-enum raw token"
+run $'   \nfirst junk line\nsecond' "$GO" "first junk line" "tier3 first non-empty line"
+
+# --- Null / empty handling ---
+run_empty "$GO" "" "empty string output" ""
+run_empty "$GO" "" "null output" "__NULL__"
+
+# --- loadEnum: the var shapes promptfoo actually passes ---
+# promptfoo's handling of a `defaultTest.vars` `file://*.json` value is
+# version-dependent: it may pass the RESOLVED FILE CONTENTS (JSON array text as a
+# string) OR the literal unresolved `file://...` ref. loadEnum must handle BOTH
+# (plus a direct array), returning [] (gate fails closed) on anything else. The
+# resolved-contents case is the one a prior simplification dropped, vacuuming the
+# gate — these cases lock both shapes so it cannot recur.
+loadenum() {
+  local enumval="$1" want_len="$2" want_first="$3" label="$4"
+  local got
+  got=$(node -e '
+    const { loadEnum } = require(process.argv[1]);
+    const v = process.argv[2] === "__ARRAY__"
+      ? ["a","b","c"]
+      : process.argv[2];
+    const e = loadEnum({ enum: v });
+    process.stdout.write(JSON.stringify({ len: e.length, first: (e[0] ?? null) }));
+  ' "$SKILL_DIR/scripts/parse-label.cjs" "$enumval")
+  local got_len got_first
+  got_len=$(node -e 'process.stdout.write(String(JSON.parse(process.argv[1]).len))' "$got")
+  got_first=$(node -e 'process.stdout.write(String(JSON.parse(process.argv[1]).first))' "$got")
+  if [[ "$got_len" != "$want_len" || "$got_first" != "$want_first" ]]; then
+    echo "FAIL [$label]: enum='$enumval' -> len=$got_len first=$got_first (want len=$want_len first=$want_first)"
+    fails=$((fails + 1))
+  else
+    echo "ok   [$label]: len=$got_len first=$got_first"
+  fi
+}
+
+# promptfoo's file:// var handling is version-dependent: it may pass the literal
+# `file://...` ref OR the RESOLVED FILE CONTENTS (JSON array text). Both must work.
+loadenum "file://enums/go-routes.json"     11 "fix" "loadEnum file:// literal ref (one promptfoo shape)"
+loadenum "file://enums/triage-levels.json" 3 "P1"  "loadEnum file:// triage-levels"
+loadenum "enums/go-routes.json"            11 "fix" "loadEnum bare relative path"
+# The resolved-contents shape — what promptfoo actually passes when it resolves a
+# defaultTest.vars file:// ref to the file's CONTENTS. Regression guard for the
+# gate-vacuous bug (loadEnum must JSON.parse a JSON-array-text string, not treat it
+# as a path). JSON array text as a string.
+loadenum '["fix","drain","drain-prs","clo-attestation","review","legal-threshold","incident","implement","questionnaire","glossary","default"]' 11 "fix" "loadEnum JSON-array-text contents (promptfoo file:// resolved-contents shape)"
+loadenum '[ "a", "b", "c" ]'               3 "a"   "loadEnum whitespaced JSON-array-text contents"
+loadenum "__ARRAY__"                       3 "a"   "loadEnum direct array passthrough"
+loadenum "file://enums/does-not-exist.json" 0 "null" "loadEnum unreadable file -> [] (gate fails closed)"
+loadenum "not json and not a path"         0 "null" "loadEnum garbage -> [] (gate fails closed)"
+loadenum ""                                0 "null" "loadEnum empty string -> []"
+
+if [[ "$fails" -gt 0 ]]; then
+  echo "parse-label: $fails assertion(s) failed"
+  exit 1
+fi
+echo "parse-label: all assertions passed"

package/skills/eval-harness/test/registry-completeness.test.sh

@@ -0,0 +1,109 @@
+#!/usr/bin/env bash
+# Deterministic completeness test for the gated-skills registry — NO API.
+# Asserts bidirectional parity between `eval-gate:block:<id>:start` markers in
+# source files and `block_id` entries in gated-skills.json, so a new/renamed
+# gated classifier cannot be added without wiring its projection check.
+#   DEDUP    — each scanned id appears exactly once in source (sort -u can't hide a dup).
+#   PARITY   — source-scanned id set == registry block_id set (block-id level).
+#   CHARSET  — every registry block_id matches ^[a-z][a-z0-9-]*$ (scanner-recognizable).
+#   NEGATIVE — in-memory injected id/dup is flagged (verify-the-verifier).
+# Mirrors test/eval-gate.test.sh: pure bash + node one-liners (no jq), accumulate-then-exit.
+set -euo pipefail
+
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SKILL_DIR="$(cd "$HERE/.." && pwd)"
+REPO_ROOT="$(cd "$SKILL_DIR/../../../.." && pwd)"
+REGISTRY="$SKILL_DIR/gated-skills.json"
+fails=0
+
+pass() { echo "ok   [$1]"; }
+fail() { echo "FAIL [$1]: $2"; fails=$((fails + 1)); }
+
+cd "$REPO_ROOT"
+
+# Source-marker scan: ids from `eval-gate:block:<id>:start` across plugins/soleur/, minus the
+# eval-harness skill dir itself (its SKILL.md / registry / tests quote the literal markers in
+# prose, so including them would false-positive the parity check). `|| true` keeps a
+# legitimately-empty scan from aborting under errexit — PARITY then reports it cleanly.
+# Scope assumption (fail-open corner, unreachable today): every gated source_file lives under
+# plugins/soleur/ and outside eval-harness/. If the registry ever gains a source_file outside
+# this scan root (or inside eval-harness/), widen/adjust the pathspec or its marker is invisible.
+scan_ids() {
+  git grep -hoE 'eval-gate:block:[a-z][a-z0-9-]*:start' \
+    -- 'plugins/soleur/' ':(exclude)plugins/soleur/skills/eval-harness/' 2>/dev/null \
+    | sed -E 's/eval-gate:block:(.*):start/\1/' || true
+}
+
+# Registry block_ids (no jq dependency — node one-liner, mirrors eval-gate.test.sh).
+registry_ids() {
+  node -e 'JSON.parse(require("fs").readFileSync(process.argv[1],"utf8")).forEach(e=>console.log(e.block_id))' "$REGISTRY"
+}
+
+# --- DEDUP guard: each source id appears exactly once (BEFORE sort -u would hide a dup) ---
+dups="$(scan_ids | sort | uniq -d)"
+if [[ -z "$dups" ]]; then
+  pass "DEDUP — each gated block_id appears exactly once in source"
+else
+  fail "DEDUP — duplicate source marker(s)" "gated block '$(echo "$dups" | tr '\n' ' ')' appears more than once in source — each block_id maps to one registry source_file"
+fi
+
+# --- PARITY (the feature): source-scanned id set == registry id set, block-id level ---
+src_set="$(scan_ids | sort -u)"
+reg_set="$(registry_ids | sort -u)"
+
+# Characterize the live scan (Kieran: don't just exit-0; pin the production command's output).
+# Sorted set; update when a gated classifier surface is added/removed (PARITY below is the
+# robust invariant — this literal is the belt-and-suspenders scanner-output pin).
+expected_set=$'go-routing\nincident-threshold\nlane-inference\nticket-triage'
+if [[ "$src_set" == "$expected_set" ]]; then
+  pass "live scan characterized as exactly {go-routing, incident-threshold, lane-inference, ticket-triage}"
+else
+  fail "live scan characterization" "scanned ids: [$(echo "$src_set" | tr '\n' ' ')] (want: go-routing incident-threshold lane-inference ticket-triage)"
+fi
+
+# `|| true` inside the substitution: diff exits 1 on differences, which would otherwise
+# abort the script (set -e + pipefail) BEFORE fail() prints the clear drift message.
+parity_diff="$(diff <(echo "$src_set") <(echo "$reg_set") || true)"
+if [[ -z "$parity_diff" ]]; then
+  pass "PARITY — source markers and registry block_ids match"
+else
+  fail "PARITY — registry/source drift" "$(echo "$parity_diff" | tr '\n' ' ') ('<' source-only = unregistered marker: add a registry entry + enums/tasks/promptfooconfig + node scripts/gen-skill-prompt.cjs <id>; '>' registry-only = orphan/renamed entry)"
+fi
+
+# --- CHARSET guard: every registry block_id is scanner-recognizable ---
+bad_ids=""
+while IFS= read -r id; do
+  [[ "$id" =~ ^[a-z][a-z0-9-]*$ ]] || bad_ids+="$id "
+done < <(registry_ids)
+if [[ -z "$bad_ids" ]]; then
+  pass "CHARSET — every registry block_id matches ^[a-z][a-z0-9-]*\$"
+else
+  fail "CHARSET — non-conforming block_id" "registry block_id(s) '$bad_ids' must match ^[a-z][a-z0-9-]*\$ — the scanner regex only recognizes lowercase-hyphen ids"
+fi
+
+# --- NEGATIVE sanity (in-memory; git grep is tracked-only and cannot see a tmp fixture, so a
+#     file-based negative would test a different backend than production and false-pass) ---
+# (a) an injected unregistered id must be flagged by the PARITY comparison — exercised through the
+#     SAME captured-string idiom the live check uses, and asserting the injected id lands on the
+#     source-only ('<') side, so this verifies the verifier rather than just that diff(A+x,A)≠∅.
+inj_src="$(printf '%s\nzz-injected-unregistered' "$src_set" | sort -u)"
+inj_diff="$(diff <(echo "$inj_src") <(echo "$reg_set") || true)"
+if [[ -n "$inj_diff" && "$inj_diff" == *"< zz-injected-unregistered"* ]]; then
+  pass "NEGATIVE — injected unregistered id is flagged by PARITY (source-only side)"
+else
+  fail "NEGATIVE parity sanity" "injected unregistered id was NOT flagged on the source-only side (got: $(echo "$inj_diff" | tr '\n' ' '))"
+fi
+
+# (b) an injected duplicate id must be flagged by the DEDUP guard.
+inj_dups="$(printf '%s\ngo-routing' "$(scan_ids)" | sort | uniq -d)"
+if [[ -n "$inj_dups" ]]; then
+  pass "NEGATIVE — injected duplicate id is flagged by DEDUP"
+else
+  fail "NEGATIVE dedup sanity" "an injected duplicate id was NOT flagged by the dedup check"
+fi
+
+if [[ "$fails" -gt 0 ]]; then
+  echo "registry-completeness: $fails assertion(s) failed"
+  exit 1
+fi
+echo "registry-completeness: all assertions passed"

package/skills/eval-harness/test/verdict.test.sh

@@ -0,0 +1,123 @@
+#!/usr/bin/env bash
+# Deterministic unit test for the PURE verdict engine (scripts/verdict.cjs).
+# ZERO live API — every input is a recorded normalized-results fixture.
+#
+# Normalized results shape (the seam the orchestrator wires compound/heal-skill to):
+#   currentResults, candidateResults : Array<{ task_id: string, correct: boolean }>
+#     — one entry per model×repeat sample per corpus task.
+#   targetTask : { task_id }  — the row whose candidate samples must pass (>= 0.5).
+#   opts : { epsilon?, targetThreshold=0.5 }  — epsilon default = 1/(#distinct corpus task_ids).
+#
+# Cases (AC2): (a) accept, (b) reject-corpus-regress, (c) reject-target-fail,
+#              (d) epsilon-boundary (inclusive == not-regressed).
+set -euo pipefail
+
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SKILL_DIR="$(cd "$HERE/.." && pwd)"
+VERDICT="$SKILL_DIR/scripts/verdict.cjs"
+export SKILL_DIR
+
+node -e "$(cat <<'EOF'
+const assert = require("node:assert");
+const path = require("node:path");
+const { computeVerdict } = require(path.join(process.env.SKILL_DIR, "scripts", "verdict.cjs"));
+
+let fails = 0;
+function check(label, fn) {
+  try { fn(); console.log("ok   [" + label + "]"); }
+  catch (e) { console.log("FAIL [" + label + "]: " + e.message); fails++; }
+}
+
+// Helper: N samples of a task with `c` correct, rest incorrect.
+function samples(task_id, total, correct) {
+  const out = [];
+  for (let i = 0; i < total; i++) out.push({ task_id, correct: i < correct });
+  return out;
+}
+
+const TARGET = { task_id: "T" };
+
+// --- (a) accept: corpus flat (0.75 -> 0.75), target passes (1.0). epsilon = 1/4 = 0.25
+check("accept: improve target, corpus flat", () => {
+  const corpusCur = [...samples("t1",1,1), ...samples("t2",1,1), ...samples("t3",1,1), ...samples("t4",1,0)];
+  const corpusCand = [...samples("t1",1,1), ...samples("t2",1,1), ...samples("t3",1,1), ...samples("t4",1,0)];
+  const cur = [...corpusCur, ...samples("T",3,1)];
+  const cand = [...corpusCand, ...samples("T",3,3)];
+  const v = computeVerdict(cur, cand, TARGET, {});
+  assert.strictEqual(v.epsilon, 0.25, "epsilon default = 1/4");
+  assert.strictEqual(v.corpus_regressed, false);
+  assert.strictEqual(v.target_task_passes, true);
+  assert.strictEqual(v.accept, true);
+  assert.ok(Array.isArray(v.per_task));
+});
+
+// --- (b) reject: corpus regresses beyond epsilon (1.0 -> 0.25), target passes
+check("reject: corpus regresses beyond epsilon", () => {
+  const cur = [...samples("t1",1,1), ...samples("t2",1,1), ...samples("t3",1,1), ...samples("t4",1,1), ...samples("T",1,1)];
+  const cand = [...samples("t1",1,1), ...samples("t2",1,0), ...samples("t3",1,0), ...samples("t4",1,0), ...samples("T",1,1)];
+  const v = computeVerdict(cur, cand, TARGET, {});
+  assert.strictEqual(v.corpus_regressed, true);
+  assert.strictEqual(v.accept, false);
+});
+
+// --- (c) reject: target task fails (<0.5), corpus flat
+check("reject: target task fails (<0.5)", () => {
+  const corpus = [...samples("t1",1,1), ...samples("t2",1,1), ...samples("t3",1,1), ...samples("t4",1,0)];
+  const cur = [...corpus, ...samples("T",3,2)];
+  const cand = [...corpus, ...samples("T",3,1)]; // 1/3 = 0.333 < 0.5
+  const v = computeVerdict(cur, cand, TARGET, {});
+  assert.strictEqual(v.corpus_regressed, false);
+  assert.strictEqual(v.target_task_passes, false);
+  assert.strictEqual(v.accept, false);
+});
+
+// --- (d) epsilon-boundary: candidate exactly at current_rate - epsilon => NOT regressed
+check("epsilon-boundary inclusive == not regressed", () => {
+  // current corpus rate = 1.0, epsilon = 0.25, candidate corpus = 0.75 exactly.
+  const cur = [...samples("t1",1,1), ...samples("t2",1,1), ...samples("t3",1,1), ...samples("t4",1,1), ...samples("T",1,1)];
+  const cand = [...samples("t1",1,1), ...samples("t2",1,1), ...samples("t3",1,1), ...samples("t4",1,0), ...samples("T",1,1)];
+  const v = computeVerdict(cur, cand, TARGET, {});
+  assert.strictEqual(v.current_rate, 1.0);
+  assert.strictEqual(v.candidate_rate, 0.75);
+  assert.strictEqual(v.corpus_regressed, false, "boundary is inclusive: == is not a regression");
+  assert.strictEqual(v.accept, true);
+});
+
+// --- (e) reject: corpus below the absolute trust floor (both arms degraded) ---
+// current corpus rate = 0.25 (< MIN_TRUSTED_CORPUS_RATE 0.5); candidate flat at 0.25 so
+// there is no *relative* regression, and the target passes — but the baseline is too
+// degraded to trust, so accept must be forced false.
+check("reject: corpus below trust floor (both arms degraded)", () => {
+  const corpus = [...samples("t1",1,1), ...samples("t2",1,0), ...samples("t3",1,0), ...samples("t4",1,0)]; // 0.25
+  const cur = [...corpus, ...samples("T",3,3)];
+  const cand = [...corpus, ...samples("T",3,3)];
+  const v = computeVerdict(cur, cand, TARGET, {});
+  assert.strictEqual(v.current_rate, 0.25);
+  assert.strictEqual(v.corpus_regressed, false, "flat corpus is not a relative regression");
+  assert.strictEqual(v.target_task_passes, true);
+  assert.strictEqual(v.corpus_untrustworthy, true);
+  assert.strictEqual(v.min_trusted_corpus_rate, 0.5);
+  assert.strictEqual(v.accept, false, "untrustworthy baseline forces accept=false");
+});
+
+// --- malformed input throws (fail-closed) ---
+check("throws on non-array currentResults", () => {
+  assert.throws(() => computeVerdict(null, [], TARGET, {}));
+});
+check("throws on entry missing correct boolean", () => {
+  assert.throws(() => computeVerdict([{ task_id: "t1" }], [{ task_id: "t1", correct: true }], TARGET, {}));
+});
+check("throws on missing targetTask.task_id", () => {
+  const ok = [...samples("t1",1,1), ...samples("T",1,1)];
+  assert.throws(() => computeVerdict(ok, ok, {}, {}));
+});
+check("throws when target has no candidate samples", () => {
+  const cur = [...samples("t1",1,1), ...samples("T",1,1)];
+  const cand = [...samples("t1",1,1)]; // no T
+  assert.throws(() => computeVerdict(cur, cand, TARGET, {}));
+});
+
+if (fails > 0) { console.log("verdict: " + fails + " assertion(s) failed"); process.exit(1); }
+console.log("verdict: all assertions passed");
+EOF
+)"

package/skills/every-style-editor/SKILL.md

@@ -0,0 +1,142 @@
+---
+name: every-style-editor
+description: "This skill should be used when reviewing or editing copy for adherence to Every's style guide. It provides systematic line-by-line review for grammar, punctuation, mechanics, and style compliance."
+---
+
+<!-- grok-harness-invoke:start -->
+**Grok Build (`plugins/soleur/lib/harness.ts` `invokeSkill()`):** Read this SKILL.md in this process and run it to completion. A one-segment `soleur:<name>` in this document names a SKILL — on Grok Build, Read `plugins/soleur/skills/<name>/SKILL.md` in this process; it is not a nested tool_use. A multi-segment id such as `soleur:<domain>:<name>` names an AGENT: spawn it, never Read it, and on Grok Build spawn_subagent takes the id with its colons replaced by hyphens (`agentIdToGrokSubagentType`). **Claude Code:** Skill tool for a skill (`soleur:<name>`), Task tool with `subagent_type` for an agent. Forbidden is executing a subset, not the Read.
+<!-- grok-harness-invoke:end -->
+
+# Every Style Editor
+
+This skill provides a systematic approach to reviewing copy against Every's comprehensive style guide. It transforms Claude into a meticulous line editor and proofreader specializing in grammar, mechanics, and style guide compliance.
+
+## When to Use This Skill
+
+Use this skill when:
+
+- Reviewing articles, blog posts, newsletters, or any written content
+- Ensuring copy follows Every's specific style conventions
+- Providing feedback on grammar, punctuation, and mechanics
+- Flagging deviations from the Every style guide
+- Preparing clean copy for human editorial review
+
+## Skill Overview
+
+This skill enables performing a comprehensive review of written content in four phases:
+
+1. **Initial Assessment** - Understanding context and document type
+2. **Detailed Line Edit** - Checking every sentence for compliance
+3. **Mechanical Review** - Verifying formatting and consistency
+4. **Recommendations** - Providing actionable improvement suggestions
+
+## How to Use This Skill
+
+### Step 1: Initial Assessment
+
+Begin by reading the entire piece to understand:
+
+- Document type (article, knowledge base entry, social post, etc.)
+- Target audience
+- Overall tone and voice
+- Content context
+
+### Step 2: Detailed Line Edit
+
+Review each paragraph systematically, checking for:
+
+- Sentence structure and grammar correctness
+- Punctuation usage (commas, semicolons, em dashes, etc.)
+- Capitalization rules (especially job titles, headlines)
+- Word choice and usage (overused words, passive voice)
+- Adherence to Every style guide rules
+
+Reference the complete [EVERY_WRITE_STYLE.md](./references/EVERY_WRITE_STYLE.md) for specific rules when in doubt.
+
+### Step 3: Mechanical Review
+
+Verify:
+
+- Spacing and formatting consistency
+- Style choices applied uniformly throughout
+- Special elements (lists, quotes, citations)
+- Proper use of italics and formatting
+- Number formatting (numerals vs. spelled out)
+- Link formatting and descriptions
+
+### Step 4: Output Results
+
+Present findings using this structure:
+
+```
+DOCUMENT REVIEW SUMMARY
+=====================
+Document Type: [type]
+Word Count: [approximate]
+Overall Assessment: [brief overview]
+
+ERRORS FOUND: [total number]
+
+DETAILED CORRECTIONS
+===================
+
+[For each error found:]
+
+**Location**: [Paragraph #, Sentence #]
+**Issue Type**: [Grammar/Punctuation/Mechanics/Style Guide]
+**Original**: "[exact text with error]"
+**Correction**: "[corrected text]"
+**Rule Reference**: [Specific style guide rule violated]
+**Explanation**: [Brief explanation of why this is an error]
+
+---
+
+RECURRING ISSUES
+===============
+[List patterns of errors that appear multiple times]
+
+STYLE GUIDE COMPLIANCE CHECKLIST
+==============================
+✓ [Rule followed correctly]
+✗ [Rule violated - with count of violations]
+
+FINAL RECOMMENDATIONS
+===================
+[2-3 actionable suggestions for improving the draft]
+```
+
+## Style Guide Reference
+
+The complete Every style guide is included in [EVERY_WRITE_STYLE.md](./references/EVERY_WRITE_STYLE.md). Key areas to focus on:
+
+- **Quick Rules**: Title case for headlines, sentence case elsewhere
+- **Tone**: Active voice, avoid overused words (actually, very, just), be specific
+- **Numbers**: Spell out one through nine; use numerals for 10+
+- **Punctuation**: Oxford commas, em dashes without spaces, proper quotation mark usage
+- **Capitalization**: Lowercase job titles, company as singular (it), teams as plural (they)
+- **Emphasis**: Italics only (no bold for emphasis)
+- **Links**: 2-4 words, don't say "click here"
+
+## Key Principles
+
+- **Be specific**: Always quote the exact text with the error
+- **Reference rules**: Cite the specific style guide rule for each correction
+- **Maintain voice**: Preserve the author's voice while correcting errors
+- **Prioritize clarity**: Focus on changes that improve readability
+- **Be constructive**: Frame feedback to help writers improve
+- **Flag ambiguous cases**: When style guide doesn't address an issue, explain options and recommend the clearest choice
+
+## Common Areas to Focus On
+
+Based on Every's style guide, pay special attention to:
+
+- Punctuation (comma usage, semicolons, apostrophes, quotation marks)
+- Capitalization (proper nouns, titles, sentence starts)
+- Numbers (when to spell out vs. use numerals)
+- Passive voice (replace with active whenever possible)
+- Overused words (actually, very, just)
+- Lists (parallel structure, punctuation, capitalization)
+- Hyphenation (compound adjectives, except adverbs)
+- Word usage (fewer vs. less, they vs. them)
+- Company references (singular "it", teams as plural "they")
+- Job title capitalization