@ccoalm/ccl-skills 0.8.0 → 0.10.0

package/dist/assets/marketplace/plugins/ccl-skills/skills/skill-extraction-workflow/scripts/test_eval_routing_bank_grader_diagnostics.sh

@@ -35,6 +35,16 @@ description: Use when choosing test layers and regression evidence.
 # Testing Strategy
 EOF
 
+# A second skill keeps the outcome space larger than {expected, none}, so
+# acceptable[] fixtures below stay non-vacuous under the anti-gaming rule.
+mkdir -p "$REPO/skills/tighten-doc"
+cat > "$REPO/skills/tighten-doc/SKILL.md" <<'EOF'
+---
+description: Use when polishing document wording.
+---
+# Tighten Doc
+EOF
+
 cat > "$REPO/eval/routing-tasks.jsonl" <<'EOF'
 {"id":"fake-claude-auth","utterance":"补一个回归测试","expected_skill":"testing-strategy","frozen_at_sha":""}
 EOF
@@ -187,4 +197,216 @@ set -e
 [ "$twice_rc" = 3 ] || fail "a grader that hangs on both attempts must stay unmeasured; rc=$twice_rc"
 assert_contains "grader_timeout_1s" "$twice_out" "a doubly-timed-out task should still report the timeout"
 
+# --- negative-control sentinel + failure-mode labels -------------------------
+# expected_skill "none" is an outcome, not a catalog entry: a grader that picks a
+# real skill for such a row must FAIL the task and label it "absorbed", and its
+# clarify flag must be counted as a first-class metric.
+none_bank="$TMP/bank-none.jsonl"
+cat > "$none_bank" <<'EOF'
+{"id":"neg-probe","utterance":"帮我写封邮件","expected_skill":"none","frozen_at_sha":""}
+EOF
+none_json="$TMP/report-none.json"
+cat > "$FAKE_BIN/claude" <<'EOF'
+#!/usr/bin/env bash
+printf '%s\n' '{"selected_skill": "testing-strategy", "clarify": true, "confidence": 0.4, "rationale_short": "nearest neighbor"}'
+EOF
+chmod +x "$FAKE_BIN/claude"
+set +e
+none_out="$(PATH="$FAKE_BIN:$PATH" ruby "$EVAL_SCRIPT" "$REPO" --bank "$none_bank" --json "$none_json" --timeout 60 2>&1)"
+none_rc=$?
+set -e
+[ "$none_rc" = 0 ] || fail "sentinel run should complete advisory rc=0, got rc=$none_rc; out=$none_out"
+none_status="$(ruby -rjson -e 'print JSON.parse(File.read(ARGV[0])).dig("results", 0, "status")' "$none_json")"
+[ "$none_status" = "FAIL" ] || fail "a skill claiming an expected-none row must FAIL, got: $none_status"
+none_labels="$(ruby -rjson -e 'print JSON.parse(File.read(ARGV[0])).dig("results", 0, "labels").join(",")' "$none_json")"
+assert_contains "absorbed" "$none_labels" "expected-none row claimed by a skill must carry the absorbed label"
+none_clarify="$(ruby -rjson -e 'print JSON.parse(File.read(ARGV[0]))["clarify_count"]' "$none_json")"
+[ "$none_clarify" = 1 ] || fail "clarify flag must be counted (expected clarify_count=1, got $none_clarify)"
+none_lowconf="$(ruby -rjson -e 'print JSON.parse(File.read(ARGV[0]))["low_confidence_count"]' "$none_json")"
+[ "$none_lowconf" = 1 ] || fail "confidence 0.4 must count as low-confidence (<0.5), got $none_lowconf"
+# And the honest direction: a grader answering "none" on the same row must PASS.
+cat > "$FAKE_BIN/claude" <<'EOF'
+#!/usr/bin/env bash
+printf '%s\n' '{"selected_skill": "none", "clarify": false, "confidence": 0.9, "rationale_short": "out of catalog"}'
+EOF
+chmod +x "$FAKE_BIN/claude"
+set +e
+none2_out="$(PATH="$FAKE_BIN:$PATH" ruby "$EVAL_SCRIPT" "$REPO" --bank "$none_bank" --json "$none_json" --timeout 60 2>&1)"
+set -e
+none2_status="$(ruby -rjson -e 'print JSON.parse(File.read(ARGV[0])).dig("results", 0, "status")' "$none_json")"
+[ "$none2_status" = "PASS" ] || fail "a none verdict on an expected-none row must PASS, got: $none2_status; out=$none2_out"
+
+# --- replicas: conservative consensus, ownership_split, agreement metric ------
+split_bank="$TMP/bank-split.jsonl"
+cat > "$split_bank" <<'EOF'
+{"id":"split-probe","utterance":"补一个回归测试","expected_skill":"testing-strategy","frozen_at_sha":""}
+EOF
+split_json="$TMP/report-split.json"
+rm -f "$GRADER_PID_FILE.calls"
+cat > "$FAKE_BIN/claude" <<'EOF'
+#!/usr/bin/env bash
+if [ -f "$GRADER_PID_FILE.calls" ]; then
+  printf '%s\n' '{"selected_skill": "none", "clarify": false, "confidence": 0.6, "rationale_short": "second replica disagrees"}'
+else
+  : > "$GRADER_PID_FILE.calls"
+  printf '%s\n' '{"selected_skill": "testing-strategy", "clarify": false, "confidence": 0.9, "rationale_short": "first replica"}'
+fi
+EOF
+chmod +x "$FAKE_BIN/claude"
+set +e
+split_out="$(PATH="$FAKE_BIN:$PATH" ruby "$EVAL_SCRIPT" "$REPO" --bank "$split_bank" --replicas 2 --json "$split_json" --timeout 60 2>&1)"
+split_rc=$?
+set -e
+[ "$split_rc" = 0 ] || fail "replica run should complete advisory rc=0, got rc=$split_rc; out=$split_out"
+split_status="$(ruby -rjson -e 'print JSON.parse(File.read(ARGV[0])).dig("results", 0, "status")' "$split_json")"
+[ "$split_status" = "FAIL" ] || fail "one failing replica must fail the task (conservative consensus), got: $split_status"
+split_labels="$(ruby -rjson -e 'print JSON.parse(File.read(ARGV[0])).dig("results", 0, "labels").join(",")' "$split_json")"
+assert_contains "ownership_split" "$split_labels" "disagreeing replicas must carry the ownership_split label"
+split_agree="$(ruby -rjson -e 'r=JSON.parse(File.read(ARGV[0]))["replica_agreement"]; print "#{r["agree"]}/#{r["measured"]}"' "$split_json")"
+[ "$split_agree" = "0/1" ] || fail "expected replica_agreement 0/1, got: $split_agree"
+split_verdicts="$(ruby -rjson -e 'print JSON.parse(File.read(ARGV[0])).dig("results", 0, "verdicts").length' "$split_json")"
+[ "$split_verdicts" = 2 ] || fail "expected 2 recorded verdicts, got: $split_verdicts"
+
+# --- replicas: a PASS+ERROR pair is PARTIALLY measured, never a clean PASS ----
+# Task-level consensus sees only observed verdicts, so without replica-level
+# error accounting a failed replica would vanish (zero grader-errors reported).
+partial_bank="$TMP/bank-partial.jsonl"
+cat > "$partial_bank" <<'EOF'
+{"id":"partial-probe","utterance":"补一个回归测试","expected_skill":"testing-strategy","frozen_at_sha":""}
+EOF
+partial_json="$TMP/report-partial.json"
+rm -f "$GRADER_PID_FILE.partial"
+cat > "$FAKE_BIN/claude" <<'EOF'
+#!/usr/bin/env bash
+if [ -f "$GRADER_PID_FILE.partial" ]; then
+  printf 'grader exploded\n' >&2
+  exit 1
+fi
+: > "$GRADER_PID_FILE.partial"
+printf '%s\n' '{"selected_skill": "testing-strategy", "clarify": false, "confidence": 0.9, "rationale_short": "first replica ok"}'
+EOF
+chmod +x "$FAKE_BIN/claude"
+set +e
+partial_out="$(PATH="$FAKE_BIN:$PATH" ruby "$EVAL_SCRIPT" "$REPO" --bank "$partial_bank" --replicas 2 --json "$partial_json" --timeout 60 2>&1)"
+partial_rc=$?
+set -e
+[ "$partial_rc" = 0 ] || fail "partial-error run should complete advisory rc=0, got rc=$partial_rc; out=$partial_out"
+partial_status="$(ruby -rjson -e 'print JSON.parse(File.read(ARGV[0])).dig("results", 0, "status")' "$partial_json")"
+[ "$partial_status" = "PASS" ] || fail "consensus over observed verdicts should stay PASS, got: $partial_status"
+partial_labels="$(ruby -rjson -e 'print JSON.parse(File.read(ARGV[0])).dig("results", 0, "labels").join(",")' "$partial_json")"
+assert_contains "partial_error" "$partial_labels" "a PASS+ERROR pair must carry the partial_error label"
+partial_ev="$(ruby -rjson -e 'print JSON.parse(File.read(ARGV[0]))["error_verdicts"]' "$partial_json")"
+[ "$partial_ev" = 1 ] || fail "replica-level errors must be counted (expected error_verdicts=1, got $partial_ev)"
+assert_contains "grader-error verdicts: 1" "$partial_out" "partial errors must be surfaced in human output"
+
+# --- acceptable[]: a defensible alternate outcome passes, and is marked -------
+acc_bank="$TMP/bank-acc.jsonl"
+cat > "$acc_bank" <<'EOF'
+{"id":"acc-probe","utterance":"用 COBOL 写个批处理","expected_skill":"testing-strategy","acceptable":["none"],"frozen_at_sha":""}
+EOF
+acc_json="$TMP/report-acc.json"
+cat > "$FAKE_BIN/claude" <<'EOF'
+#!/usr/bin/env bash
+printf '%s\n' '{"selected_skill": "none", "clarify": false, "confidence": 0.8, "rationale_short": "alternate outcome"}'
+EOF
+chmod +x "$FAKE_BIN/claude"
+set +e
+acc_out="$(PATH="$FAKE_BIN:$PATH" ruby "$EVAL_SCRIPT" "$REPO" --bank "$acc_bank" --json "$acc_json" --timeout 60 2>&1)"
+acc_rc=$?
+set -e
+[ "$acc_rc" = 0 ] || fail "acceptable run should complete advisory rc=0, got rc=$acc_rc; out=$acc_out"
+acc_status="$(ruby -rjson -e 'print JSON.parse(File.read(ARGV[0])).dig("results", 0, "status")' "$acc_json")"
+[ "$acc_status" = "PASS" ] || fail "a selection inside acceptable[] must PASS, got: $acc_status"
+acc_hit="$(ruby -rjson -e 'print JSON.parse(File.read(ARGV[0])).dig("results", 0, "acceptable_hit")' "$acc_json")"
+[ "$acc_hit" = "true" ] || fail "acceptable_hit must be recorded, got: $acc_hit"
+
+# --- baseline ruler guard: mismatched bank or replicas suppresses the diff ----
+# The docs declare reports from different (bank, replicas) configurations
+# non-comparable; the runner must enforce that, or a stale baseline emits false
+# newly_failed/newly_passed regressions.
+ruler_bank="$TMP/bank-ruler.jsonl"
+cat > "$ruler_bank" <<'EOF'
+{"id":"ruler-probe","utterance":"补一个回归测试","expected_skill":"testing-strategy","frozen_at_sha":""}
+EOF
+cat > "$FAKE_BIN/claude" <<'EOF'
+#!/usr/bin/env bash
+printf '%s\n' '{"selected_skill": "none", "clarify": false, "confidence": 0.9, "rationale_short": "fails vs expected"}'
+EOF
+chmod +x "$FAKE_BIN/claude"
+ruler_now="$TMP/report-ruler-now.json"
+set +e
+PATH="$FAKE_BIN:$PATH" ruby "$EVAL_SCRIPT" "$REPO" --bank "$ruler_bank" --json "$ruler_now" --timeout 60 >/dev/null 2>&1
+set -e
+[ -f "$ruler_now" ] || fail "expected current-ruler report to be written"
+# Baseline A: same shape but a different bank fingerprint, task previously PASS.
+ruler_base_bank="$TMP/report-ruler-base-bank.json"
+ruby -rjson -e '
+r = JSON.parse(File.read(ARGV[0]))
+r["routing_surface"]["bank_sha256"] = "0" * 64
+r["results"][0]["status"] = "PASS"
+File.write(ARGV[1], JSON.generate(r))
+' "$ruler_now" "$ruler_base_bank"
+mismatch_json="$TMP/report-ruler-mismatch.json"
+set +e
+mismatch_out="$(PATH="$FAKE_BIN:$PATH" ruby "$EVAL_SCRIPT" "$REPO" --bank "$ruler_bank" --baseline "$ruler_base_bank" --json "$mismatch_json" --timeout 60 2>&1)"
+set -e
+assert_contains "baseline not compared" "$mismatch_out" "a bank-fingerprint mismatch must suppress the baseline diff"
+mm_nf="$(ruby -rjson -e 'print JSON.parse(File.read(ARGV[0]))["newly_failed"].length' "$mismatch_json")"
+[ "$mm_nf" = 0 ] || fail "newly_failed must be empty under a mismatched-bank baseline, got $mm_nf entries"
+mm_cmp="$(ruby -rjson -e 'print JSON.parse(File.read(ARGV[0]))["baseline_comparable"].inspect' "$mismatch_json")"
+[ "$mm_cmp" = "false" ] || fail "baseline_comparable must be false for a bank mismatch, got: $mm_cmp"
+# Baseline B: same bank fingerprint but a different replica count.
+ruler_base_rep="$TMP/report-ruler-base-rep.json"
+ruby -rjson -e '
+r = JSON.parse(File.read(ARGV[0]))
+r["replicas"] = 2
+r["results"][0]["status"] = "PASS"
+File.write(ARGV[1], JSON.generate(r))
+' "$ruler_now" "$ruler_base_rep"
+set +e
+repmm_out="$(PATH="$FAKE_BIN:$PATH" ruby "$EVAL_SCRIPT" "$REPO" --bank "$ruler_bank" --baseline "$ruler_base_rep" --json "$mismatch_json" --timeout 60 2>&1)"
+set -e
+assert_contains "replica count differs" "$repmm_out" "a replica-count mismatch must suppress the baseline diff"
+# Control: an identical-ruler baseline still produces the diff.
+ruler_base_ok="$TMP/report-ruler-base-ok.json"
+ruby -rjson -e '
+r = JSON.parse(File.read(ARGV[0]))
+r["results"][0]["status"] = "PASS"
+File.write(ARGV[1], JSON.generate(r))
+' "$ruler_now" "$ruler_base_ok"
+set +e
+okdiff_out="$(PATH="$FAKE_BIN:$PATH" ruby "$EVAL_SCRIPT" "$REPO" --bank "$ruler_bank" --baseline "$ruler_base_ok" --json "$mismatch_json" --timeout 60 2>&1)"
+set -e
+assert_contains "newly_failed=ruler-probe" "$okdiff_out" "an identical-ruler baseline must still yield the regression diff"
+
+# --- schema strictness: vacuous rows and non-list fields are usage errors ----
+vac_bank="$TMP/bank-vacuous.jsonl"
+cat > "$vac_bank" <<'EOF'
+{"id":"vac-probe","utterance":"x","expected_skill":"testing-strategy","acceptable":["tighten-doc","none"],"frozen_at_sha":""}
+EOF
+set +e
+vac_out="$(ruby "$EVAL_SCRIPT" "$REPO" --bank "$vac_bank" --dry-run 2>&1)"
+vac_rc=$?
+set -e
+[ "$vac_rc" = 2 ] || fail "a row covering the whole outcome space must be a usage error (rc=2), got rc=$vac_rc; out=$vac_out"
+assert_contains "vacuous row" "$vac_out" "vacuous-row rejection should name the failure"
+
+badtype_bank="$TMP/bank-badtype.jsonl"
+cat > "$badtype_bank" <<'EOF'
+{"id":"badtype-probe","utterance":"x","expected_skill":"testing-strategy","must_not_route_to":1,"frozen_at_sha":""}
+EOF
+set +e
+badtype_out="$(ruby "$EVAL_SCRIPT" "$REPO" --bank "$badtype_bank" --dry-run 2>&1)"
+badtype_rc=$?
+set -e
+[ "$badtype_rc" = 2 ] || fail "a non-list must_not_route_to must be a usage error (rc=2), got rc=$badtype_rc; out=$badtype_out"
+assert_contains "must_not_route_to must be a list" "$badtype_out" "non-list field must produce the documented diagnostic, not a stack trace"
+
+# --- --replicas argument strictness ------------------------------------------
+set +e
+rep_out="$(ruby "$EVAL_SCRIPT" "$REPO" --replicas 0 2>&1)"
+rep_rc=$?
+set -e
+[ "$rep_rc" = 2 ] || fail "--replicas 0 must be a usage error (rc=2), got rc=$rep_rc; out=$rep_out"
+
 echo "test_eval_routing_bank_grader_diagnostics: ok"

package/dist/assets/marketplace/plugins/ccl-skills/skills/skill-extraction-workflow/scripts/test_extraction_review_gate.sh

@@ -33,7 +33,7 @@ import sys
 from pathlib import Path
 
 args = [item.decode() for item in Path(sys.argv[1]).read_bytes().split(b"\0") if item]
-assert args[:2] == ["--challenge-budget", "2"], args
+assert args[:2] == ["--challenge-budget", "1"], args
 assert args.count("--challenge-budget") == 1, args
 PY
 
@@ -47,14 +47,14 @@ import sys
 from pathlib import Path
 
 args = [item.decode() for item in Path(sys.argv[1]).read_bytes().split(b"\0") if item]
-assert args[:2] == ["--challenge-budget", "2"], args
+assert args[:2] == ["--challenge-budget", "1"], args
 assert args[-1] == "--base", args
 assert args.count("--challenge-budget") == 1, args
 PY
 
 # Exercise the installed wrapper/controller pair without invoking a model. The
 # same real controller defaults to budget 0 when called directly, while the
-# extraction wrapper must make the emitted receipt report budget 2.
+# extraction wrapper must make the emitted receipt report budget 1.
 python3 - "$TMP/real-controller.diff" "$TMP/real-controller-plan.json" <<'PY'
 import json
 import sys
@@ -74,7 +74,7 @@ conclusions = {
     "safety": "The probe selects only the implementer family and invokes no external reviewer.",
     "failure_paths": "The no-independent-reviewer boundary remains structured and fail closed.",
     "tests_evidence": "Direct and wrapped calls provide a differential budget assertion.",
-    "compatibility": "The generic controller default remains zero while extraction fixes two.",
+    "compatibility": "The generic controller default remains zero while extraction fixes one.",
 }
 skills = {
     "correctness": "skill-extraction-workflow",
@@ -84,8 +84,8 @@ skills = {
     "compatibility": "terminal-cli-dev",
 }
 plan = {
-    "intent": "Prove the extraction wrapper and real review controller agree on budget two.",
-    "acceptance": ["The wrapped real-controller receipt reports challenge_budget two."],
+    "intent": "Prove the extraction wrapper and real review controller agree on budget one.",
+    "acceptance": ["The wrapped real-controller receipt reports challenge_budget one."],
     "self_review": [
         {
             "concern": concern,
@@ -134,7 +134,7 @@ assert_rc "$wrapped_rc" 2 "wrapped real controller must stop before model infere
 assert_contains '"reason_code":"no_independent_reviewer_available"' "$direct_out" "direct real-controller boundary"
 assert_contains '"reason_code":"no_independent_reviewer_available"' "$wrapped_out" "wrapped real-controller boundary"
 assert_contains '"challenge_budget":0' "$direct_out" "generic controller default budget"
-assert_contains '"challenge_budget":2' "$wrapped_out" "wrapper-enforced real-controller budget"
+assert_contains '"challenge_budget":1' "$wrapped_out" "wrapper-enforced real-controller budget"
 [ ! -e "$TMP/codex-invoked" ] || fail "same-family Codex executable was invoked"
 
 # Join the producer and consumer contracts. First feed the exact real receipt
@@ -231,7 +231,7 @@ for spelling in --challenge-budget --challenge-budget=4 --challenge-b=4; do
   rc=$?
   set -e
   assert_rc "$rc" 2 "caller budget override must be rejected"
-  assert_contains "challenge budget is fixed at 2" "$out" "override reason"
+  assert_contains "challenge budget is fixed at 1" "$out" "override reason"
   [ ! -s "$TMP/args" ] || fail "controller ran after budget override"
 done
 
@@ -321,16 +321,22 @@ assert "--challenge-budget" not in quickstart, (
     "quickstart must not let callers override the extraction review budget"
 )
 assert re.search(
-    r"[Rr]ound 2.{0,500}ready_for_human_decision",
+    r"[Rr]ound 2 challenge.{0,500}ready_for_human_decision",
     quickstart,
     re.DOTALL,
 ), "quickstart does not validate an early-clean round-2 terminal checkpoint"
 assert re.search(
-    r"[Rr]ound 3.{0,500}continuation_authorization_required",
+    r"[Rr]ound 2 findings.{0,200}continuation_authorization_required",
     quickstart,
     re.DOTALL,
 ), "quickstart does not validate the exhausted-budget terminal checkpoint"
 assert "baseline_race" in quickstart
+assert "at most two challenges" not in quickstart, (
+    "quickstart still advertises the retired two-challenge budget"
+)
+assert "At the third Agent-autonomous round" not in dual, (
+    "dual-track still gates the lane at the retired third round"
+)
 PY
 
 echo "test_extraction_review_gate: ok"

package/dist/assets/marketplace/plugins/ccl-skills/skills/skill-extraction-workflow/scripts/test_frozen_case_sanctity.sh

@@ -0,0 +1,178 @@
+#!/usr/bin/env bash
+# Frozen-case sanctity gate (regressions-are-sacred, deterministic half).
+#
+# SCOPE — read this before citing the lane as evidence. A frozen eval case
+# (routing task-bank row, golden trace) is a pinned judgment: deleting it or
+# rewriting its judgment fields makes a red disappear without anyone ruling on
+# it. This lane makes that trade VISIBLE: any base-relative deletion or
+# judgment-field change of a frozen case must be named by an ADDED
+# source-register line carrying `case-retired: <id>` or `case-rescoped: <id>`.
+# It does NOT judge whether the retirement/rescope is justified — that ruling
+# belongs to the round's independent review (the register row is what puts it
+# in front of the reviewer). Like the impact-chain gate, it trusts the author
+# to write the declaration; it defends against SILENT trades, not forged ones.
+# Without CCL_SKILL_BASE_REF (or an unresolvable base) it degrades to an
+# explicit skip token — a skipped run is not a passed run.
+set -u
+
+ROOT="${SANCTITY_ROOT:-$(cd "$(dirname "$0")/../../.." && pwd)}"
+[ -n "${SANCTITY_ROOT:-}" ] && echo "NOTICE: SANCTITY_ROOT set — testing tree: $ROOT" >&2
+REGISTER="skills/skill-extraction-workflow/references/source-register.md"
+BANK="eval/routing-tasks.jsonl"
+TRACES_DIR="eval/golden-traces"
+
+BASE="${CCL_SKILL_BASE_REF:-}"
+if [ -z "$BASE" ]; then
+  echo "frozen_case_sanctity_skipped no-base-ref (set CCL_SKILL_BASE_REF to enable; skipped is not passed)"
+  exit 0
+fi
+if ! git -C "$ROOT" rev-parse --verify --quiet "${BASE}^{commit}" >/dev/null; then
+  echo "frozen_case_sanctity_skipped base-unresolvable ($BASE; skipped is not passed)"
+  exit 0
+fi
+
+TMPDIR_SANCTITY="$(mktemp -d)"
+trap 'rm -rf "$TMPDIR_SANCTITY"' EXIT
+
+# Base snapshots; a path absent at base means every current case is new (no
+# violation possible on that surface).
+git -C "$ROOT" show "$BASE:$BANK" > "$TMPDIR_SANCTITY/base-bank.jsonl" 2>/dev/null || : > "$TMPDIR_SANCTITY/base-bank.jsonl"
+git -C "$ROOT" show "$BASE:$REGISTER" > "$TMPDIR_SANCTITY/base-register.md" 2>/dev/null || : > "$TMPDIR_SANCTITY/base-register.md"
+mkdir -p "$TMPDIR_SANCTITY/base-traces"
+# -r: enumerate blobs recursively so a trace in a subdirectory is still guarded
+# (the head-side walk below recurses symmetrically).
+n=0
+git -C "$ROOT" ls-tree -r --name-only "$BASE" -- "$TRACES_DIR/" 2>/dev/null | while IFS= read -r tp; do
+  case "$tp" in
+    *.json)
+      n=$((n+1))
+      git -C "$ROOT" show "$BASE:$tp" > "$TMPDIR_SANCTITY/base-traces/trace-$n-$(basename "$tp")" 2>/dev/null || true ;;
+  esac
+done
+
+python3 - "$ROOT" "$TMPDIR_SANCTITY" "$REGISTER" "$BANK" "$TRACES_DIR" <<'PY'
+import json, os, sys
+
+root, tmp, register_rel, bank_rel, traces_rel = sys.argv[1:6]
+fail = 0
+
+def bad(msg):
+    global fail
+    print(f"FAIL: {msg}", file=sys.stderr)
+    fail = 1
+
+def load_bank(path):
+    rows = {}
+    if not os.path.exists(path):
+        return rows
+    with open(path, encoding="utf-8") as fh:
+        for raw in fh:
+            raw = raw.strip()
+            if not raw:
+                continue
+            try:
+                row = json.loads(raw)
+            except json.JSONDecodeError:
+                continue  # malformed rows are the integrity lane's finding, not ours
+            rid = row.get("id")
+            if rid:
+                rows[rid] = row
+    return rows
+
+def judgment(row):
+    # The judgment surface of a bank case: what outcome passes or fails it.
+    # Provenance/commentary fields (source, why_expected, frozen_at_sha) may
+    # change without re-scoping the case.
+    return (
+        row.get("expected_skill"),
+        sorted(row.get("acceptable") or []) if isinstance(row.get("acceptable"), list) else row.get("acceptable"),
+        sorted(row.get("must_not_route_to") or []) if isinstance(row.get("must_not_route_to"), list) else row.get("must_not_route_to"),
+    )
+
+base_bank = load_bank(os.path.join(tmp, "base-bank.jsonl"))
+head_bank = load_bank(os.path.join(root, bank_rel))
+
+violations = []  # (kind, id)
+for rid, row in base_bank.items():
+    if rid not in head_bank:
+        violations.append(("deleted bank case", rid))
+    elif judgment(row) != judgment(head_bank[rid]):
+        violations.append(("re-scoped bank case (judgment fields changed)", rid))
+
+def load_trace(path):
+    try:
+        with open(path, encoding="utf-8") as fh:
+            return json.load(fh)
+    except (OSError, json.JSONDecodeError):
+        return None
+
+base_traces = {}
+base_dir = os.path.join(tmp, "base-traces")
+for name in sorted(os.listdir(base_dir)):
+    data = load_trace(os.path.join(base_dir, name))
+    if isinstance(data, dict) and data.get("id"):
+        base_traces[data["id"]] = data
+
+head_traces = {}
+head_dir = os.path.join(root, traces_rel)
+if os.path.isdir(head_dir):
+    for dirpath, _dirnames, filenames in os.walk(head_dir):
+        for name in sorted(filenames):
+            if not name.endswith(".json"):
+                continue
+            data = load_trace(os.path.join(dirpath, name))
+            if isinstance(data, dict) and data.get("id"):
+                head_traces[data["id"]] = data
+
+for tid, data in base_traces.items():
+    if tid not in head_traces:
+        violations.append(("deleted golden trace", tid))
+    elif data.get("assert") != head_traces[tid].get("assert"):
+        violations.append(("re-scoped golden trace (assert block changed)", tid))
+
+# Added register lines = head lines not present at base (the register is
+# append-only, so a set diff is the added-row surface). Only added Markdown
+# TABLE ROWS may carry adjudication credit — an adjudication is a register row
+# a reviewer rules on, so an HTML comment or loose prose that merely mentions
+# the token mints nothing.
+with open(os.path.join(tmp, "base-register.md"), encoding="utf-8") as fh:
+    base_lines = set(fh.read().splitlines())
+register_path = os.path.join(root, register_rel)
+added_rows = []
+if os.path.exists(register_path):
+    with open(register_path, encoding="utf-8") as fh:
+        added_rows = [
+            ln for ln in fh.read().splitlines()
+            if ln not in base_lines and ln.lstrip().startswith("|")
+        ]
+added_blob = "\n".join(added_rows)
+
+import re
+def adjudicated(case_id):
+    # Token-bounded: `case-retired: foo` must not be credited by
+    # `case-retired: foobar` (ids use [A-Za-z0-9_-]).
+    pattern = re.compile(
+        r"case-(?:retired|rescoped):\s*" + re.escape(case_id) + r"(?![A-Za-z0-9_-])"
+    )
+    return bool(pattern.search(added_blob))
+
+for kind, cid in violations:
+    if adjudicated(cid):
+        continue
+    bad(
+        f"{kind} '{cid}' with no adjudication row — a frozen case may not be "
+        f"silently traded away; add a source-register row this round containing "
+        f"`case-retired: {cid}` or `case-rescoped: {cid}` (with the why), so the "
+        f"independent review rules on it"
+    )
+
+if fail:
+    sys.exit(1)
+print(f"frozen_case_sanctity_ok bank_base={len(base_bank)} traces_base={len(base_traces)} violations=0")
+PY
+status=$?
+if [ $status -ne 0 ]; then
+  echo "frozen_case_sanctity_failed" >&2
+  exit 1
+fi
+exit 0

package/dist/assets/marketplace/plugins/ccl-skills/skills/skill-extraction-workflow/scripts/test_frozen_case_sanctity_selfproof.sh

@@ -0,0 +1,108 @@
+#!/usr/bin/env bash
+# Self-proof for test_frozen_case_sanctity.sh: builds a synthetic repo and
+# proves the oracle can fail for the right reason on every guarded surface —
+# deletion, judgment re-scope, subdirectory golden-trace deletion — and that
+# adjudication credit cannot be minted by a prefix-colliding id or an HTML
+# comment (only an added register TABLE ROW with a token-bounded id counts).
+# The unmutated control and the properly-adjudicated leg must stay green, and
+# the no-base leg must print the explicit skip token. Runs in mktemp only.
+set -u
+
+SCRIPTS_DIR="$(cd "$(dirname "$0")" && pwd)"
+GATE="$SCRIPTS_DIR/test_frozen_case_sanctity.sh"
+[ -f "$GATE" ] || { echo "FAIL: gate script missing ($GATE)" >&2; exit 1; }
+
+WORK="$(mktemp -d)"
+trap 'rm -rf "$WORK"' EXIT
+REPO="$WORK/repo"
+mkdir -p "$REPO/eval/golden-traces/nested" "$REPO/skills/skill-extraction-workflow/references"
+
+BANK="$REPO/eval/routing-tasks.jsonl"
+REG="$REPO/skills/skill-extraction-workflow/references/source-register.md"
+cat > "$BANK" <<'EOF'
+{"id": "a1", "utterance": "u1", "expected_skill": "s-one", "why_expected": "w", "frozen_at_sha": "root"}
+{"id": "a1x", "utterance": "u2", "expected_skill": "s-two", "why_expected": "w", "frozen_at_sha": "root"}
+EOF
+cat > "$REPO/eval/golden-traces/nested/t1.json" <<'EOF'
+{"id": "trace-nested-one", "assert": {"must_invoke_skill": "s-one"}}
+EOF
+printf '| base row |\n' > "$REG"
+
+git -C "$REPO" init -q
+git -C "$REPO" -c user.email=t@t -c user.name=t add -A
+git -C "$REPO" -c user.email=t@t -c user.name=t commit -qm base
+BASE_SHA="$(git -C "$REPO" rev-parse HEAD)"
+
+pass=0
+fail=0
+ok()  { pass=$((pass+1)); }
+bad() { fail=$((fail+1)); echo "FAIL: $1" >&2; }
+
+run_gate() {
+  SANCTITY_ROOT="$REPO" CCL_SKILL_BASE_REF="$BASE_SHA" bash "$GATE" > "$WORK/out.log" 2> "$WORK/err.log"
+}
+
+reset_tree() { git -C "$REPO" checkout -q -- .; }
+
+# Leg 1: unmutated control is green with the success token.
+reset_tree
+if run_gate && grep -q "frozen_case_sanctity_ok" "$WORK/out.log"; then ok; else bad "control leg not green"; fi
+
+# Leg 2: deleting bank case a1 reds, naming a1.
+reset_tree
+grep -v '"id": "a1",' "$BANK" > "$BANK.t" && mv "$BANK.t" "$BANK"
+if run_gate; then bad "bank deletion passed"; else
+  grep -q "deleted bank case 'a1'" "$WORK/err.log" && ok || bad "bank deletion red but not attributed to a1"
+fi
+
+# Leg 3: re-scoping a1's expected_skill reds.
+reset_tree
+python3 - "$BANK" <<'PY'
+import json, sys
+path = sys.argv[1]
+rows = [json.loads(l) for l in open(path) if l.strip()]
+rows[0]["expected_skill"] = "s-two"
+open(path, "w").write("\n".join(json.dumps(r) for r in rows) + "\n")
+PY
+if run_gate; then bad "re-scope passed"; else
+  grep -q "re-scoped bank case (judgment fields changed) 'a1'" "$WORK/err.log" && ok || bad "re-scope red but not attributed"
+fi
+
+# Leg 4: deleting the SUBDIRECTORY golden trace reds (recursive base walk).
+reset_tree
+rm "$REPO/eval/golden-traces/nested/t1.json"
+if run_gate; then bad "nested trace deletion passed"; else
+  grep -q "deleted golden trace 'trace-nested-one'" "$WORK/err.log" && ok || bad "trace deletion red but not attributed"
+fi
+
+# Leg 5: a prefix-colliding id mints no credit (case-retired: a1x != a1).
+reset_tree
+grep -v '"id": "a1",' "$BANK" > "$BANK.t" && mv "$BANK.t" "$BANK"
+printf '| retired | case-retired: a1x | reason |\n' >> "$REG"
+if run_gate; then bad "prefix-colliding adjudication credited"; else
+  grep -q "deleted bank case 'a1'" "$WORK/err.log" && ok || bad "prefix leg red but not attributed"
+fi
+
+# Leg 6: an HTML comment mints no credit (row-only surface).
+reset_tree
+grep -v '"id": "a1",' "$BANK" > "$BANK.t" && mv "$BANK.t" "$BANK"
+printf '<!-- case-retired: a1 -->\n' >> "$REG"
+if run_gate; then bad "HTML-comment adjudication credited"; else
+  grep -q "deleted bank case 'a1'" "$WORK/err.log" && ok || bad "comment leg red but not attributed"
+fi
+
+# Leg 7: a real added table row with the exact id is credited — green.
+reset_tree
+grep -v '"id": "a1",' "$BANK" > "$BANK.t" && mv "$BANK.t" "$BANK"
+printf '| retired | case-retired: a1 | superseded by a2 probes |\n' >> "$REG"
+if run_gate && grep -q "frozen_case_sanctity_ok" "$WORK/out.log"; then ok; else bad "adjudicated deletion not green"; fi
+
+# Leg 8: no base ref prints the explicit skip token and exits 0. env -u so an
+# outer CI-supplied CCL_SKILL_BASE_REF cannot leak into this leg (the known
+# nested-suite leak class).
+reset_tree
+if env -u CCL_SKILL_BASE_REF SANCTITY_ROOT="$REPO" bash "$GATE" > "$WORK/out.log" 2>&1 && grep -q "frozen_case_sanctity_skipped no-base-ref" "$WORK/out.log"; then ok; else bad "no-base leg missing skip token"; fi
+
+echo "frozen_case_sanctity_selfproof: pass=$pass fail=$fail"
+[ "$fail" -eq 0 ] && [ "$pass" -eq 8 ] || exit 1
+echo "frozen_case_sanctity_selfproof_ok"