@ccoalm/ccl-skills 0.9.0 → 0.10.0

package/dist/assets/marketplace/plugins/ccl-skills/skills/skill-extraction-workflow/scripts/test_routing_bank_integrity.sh

@@ -71,20 +71,52 @@ with open(bank, encoding="utf-8") as fh:
         elif rid:
             seen[rid] = lineno
 
+        # "none" is the runner's negative-control / coverage-gap sentinel
+        # (eval-routing-bank.rb): the expected outcome is that no catalog skill
+        # claims the utterance. It is an outcome, never a routable target.
         expected = row.get("expected_skill")
-        if expected and expected not in installed:
+        if expected and expected != "none" and expected not in installed:
             bad(f"line {lineno} ({rid}): expected_skill '{expected}' is not a skill in skills/")
 
-        must_not = row.get("must_not_route_to") or []
-        if not isinstance(must_not, list):
-            bad(f"line {lineno} ({rid}): must_not_route_to must be a list")
+        # Type-check the ORIGINAL value: `x or []` would first convert an
+        # explicitly invalid "" (or 0/false) into a clean empty list, so the
+        # deterministic lane would pass a row the Ruby runner rejects.
+        must_not = row.get("must_not_route_to")
+        if must_not is None:
+            must_not = []
+        elif not isinstance(must_not, list):
+            bad(f"line {lineno} ({rid}): must_not_route_to must be a list, got {must_not!r}")
             must_not = []
         for target in must_not:
-            if target not in installed:
+            if target == "none":
+                bad(f"line {lineno} ({rid}): must_not_route_to names the sentinel 'none' (an outcome, not a routable target)")
+            elif target not in installed:
                 bad(f"line {lineno} ({rid}): must_not_route_to names '{target}', not a skill in skills/")
             if target == expected:
                 bad(f"line {lineno} ({rid}): '{target}' is both expected_skill and must_not_route_to")
 
+        acceptable = row.get("acceptable")
+        if acceptable is None:
+            acceptable = []
+        elif not isinstance(acceptable, list):
+            bad(f"line {lineno} ({rid}): acceptable must be a list, got {acceptable!r}")
+            acceptable = []
+        for target in acceptable:
+            if target != "none" and target not in installed:
+                bad(f"line {lineno} ({rid}): acceptable names '{target}', not a skill in skills/ (nor the 'none' sentinel)")
+            if target == expected:
+                bad(f"line {lineno} ({rid}): '{target}' is both expected_skill and acceptable (redundant)")
+            if target in must_not:
+                bad(f"line {lineno} ({rid}): '{target}' is both acceptable and must_not_route_to (contradictory)")
+
+        # Anti-gaming mirror of the runner's vacuous-row rule: expected +
+        # acceptable must leave at least one outcome that would fail the row,
+        # or every valid grader selection passes and the fixture fakes green.
+        if isinstance(acceptable, list) and expected:
+            outcome_space = set(installed) | {"none"}
+            if not outcome_space - ({expected} | set(acceptable)):
+                bad(f"line {lineno} ({rid}): expected_skill plus acceptable cover every possible outcome (vacuous row)")
+
 # A truncated or emptied bank must fail rather than vacuously pass.
 MIN_ROWS = 100
 if rows < MIN_ROWS:
@@ -203,3 +235,52 @@ print(f"routing_bank_provenance_info: source present on {with_source}/{rows} row
       f"(docs/f4-skill-effectiveness-harness.md lists it as required; not enforced here)")
 print(f"routing_bank_integrity_ok rows={rows} (structure only — routing NOT evaluated)")
 PY
+main_rc=$?
+[ "$main_rc" -eq 0 ] || exit "$main_rc"
+
+# --- validator self-proof (oracle-can-fail) ----------------------------------
+# A green main run proves nothing about the new field predicates unless the
+# validator is also seen to RED on applied mutants. Run this same script once
+# against a synthetic root carrying one mutant per predicate, and require both
+# the non-zero exit and each predicate's own FAIL message. Guarded against
+# recursion: the inner invocation skips this section.
+if [ -z "${BANK_INTEGRITY_SELFPROOF:-}" ]; then
+  SP_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/bank-integrity-selfproof.XXXXXX")"
+  trap 'rm -rf "$SP_ROOT"' EXIT
+  mkdir -p "$SP_ROOT/skills/testing-strategy" "$SP_ROOT/skills/tighten-doc" "$SP_ROOT/eval"
+  printf -- '---\ndescription: stub\n---\n' > "$SP_ROOT/skills/testing-strategy/SKILL.md"
+  printf -- '---\ndescription: stub\n---\n' > "$SP_ROOT/skills/tighten-doc/SKILL.md"
+  cat > "$SP_ROOT/eval/routing-tasks.jsonl" <<'MUTANTS'
+{"id": "bad-mustnot-none", "utterance": "x", "expected_skill": "testing-strategy", "why_expected": "y", "frozen_at_sha": "root", "must_not_route_to": ["none"]}
+{"id": "bad-acc-restate", "utterance": "x", "expected_skill": "testing-strategy", "why_expected": "y", "frozen_at_sha": "root", "acceptable": ["testing-strategy"]}
+{"id": "bad-acc-unknown", "utterance": "x", "expected_skill": "testing-strategy", "why_expected": "y", "frozen_at_sha": "root", "acceptable": ["not-a-skill"]}
+{"id": "bad-acc-contradict", "utterance": "x", "expected_skill": "testing-strategy", "why_expected": "y", "frozen_at_sha": "root", "acceptable": ["tighten-doc"], "must_not_route_to": ["tighten-doc"]}
+{"id": "bad-acc-empty-string", "utterance": "x", "expected_skill": "testing-strategy", "why_expected": "y", "frozen_at_sha": "root", "acceptable": ""}
+{"id": "bad-mustnot-empty-string", "utterance": "x", "expected_skill": "testing-strategy", "why_expected": "y", "frozen_at_sha": "root", "must_not_route_to": ""}
+{"id": "bad-none-typo", "utterance": "x", "expected_skill": "not-a-skill", "why_expected": "y", "frozen_at_sha": "root"}
+{"id": "bad-universal-pass", "utterance": "x", "expected_skill": "testing-strategy", "why_expected": "y", "frozen_at_sha": "root", "acceptable": ["tighten-doc", "none"]}
+MUTANTS
+  sp_out="$(BANK_INTEGRITY_SELFPROOF=1 BANK_ROOT="$SP_ROOT" bash "$0" 2>&1)"
+  sp_rc=$?
+  if [ "$sp_rc" -eq 0 ]; then
+    echo "FAIL: validator self-proof — the mutant bank exited 0 (the oracle cannot fail)" >&2
+    exit 1
+  fi
+  for expect in \
+    "bad-mustnot-none): must_not_route_to names the sentinel 'none'" \
+    "bad-acc-restate): 'testing-strategy' is both expected_skill and acceptable" \
+    "bad-acc-unknown): acceptable names 'not-a-skill'" \
+    "bad-acc-contradict): 'tighten-doc' is both acceptable and must_not_route_to" \
+    "bad-acc-empty-string): acceptable must be a list, got ''" \
+    "bad-mustnot-empty-string): must_not_route_to must be a list, got ''" \
+    "bad-none-typo): expected_skill 'not-a-skill' is not a skill in skills/" \
+    "bad-universal-pass): expected_skill plus acceptable cover every possible outcome (vacuous row)"; do
+    case "$sp_out" in
+      *"$expect"*) : ;;
+      *) echo "FAIL: validator self-proof — expected mutant message not found: $expect" >&2
+         echo "$sp_out" >&2
+         exit 1 ;;
+    esac
+  done
+  echo "routing_bank_validator_selfproof_ok: 8 applied mutants red on their own assertions"
+fi

package/dist/assets/marketplace/plugins/ccl-skills/skills/skill-extraction-workflow/scripts/test_validate_extraction_review_state.sh

@@ -140,7 +140,7 @@ def make_fixture(
     receipt_mutators=None,
     completion=True,
     completion_mutator=None,
-    budget=2,
+    budget=1,
     base_shas=None,
     closeout=None,
     unmatched=0,
@@ -168,9 +168,9 @@ def make_fixture(
     for index, findings in enumerate(receipt_findings, start=1):
         mode = "review" if index == 1 else "challenge"
         status = "findings" if findings else "passed"
-        remaining = 3 - index
+        remaining = 2 - index
         if status == "findings":
-            state = "post_review_budget" if index == 3 else "findings_pending"
+            state = "post_review_budget" if index == 2 else "findings_pending"
         else:
             state = "reviewed"
         receipt = {
@@ -181,7 +181,7 @@ def make_fixture(
             "review_chain_tracked": True,
             "review_chain_id": "extraction-chain",
             "autonomous_review_index": index,
-            "autonomous_review_budget": 3,
+            "autonomous_review_budget": 2,
             "autonomous_reviews_remaining": remaining,
             "autonomous_review_allowed": remaining > 0,
             "challenge_index": 0 if index == 1 else index - 1,
@@ -221,7 +221,7 @@ def make_fixture(
             "review_chain_tracked": True,
             "review_chain_id": final["review_chain_id"],
             "autonomous_review_index": final["autonomous_review_index"],
-            "autonomous_review_budget": 3,
+            "autonomous_review_budget": 2,
             "autonomous_reviews_remaining": final["autonomous_reviews_remaining"],
             "autonomous_review_allowed": False,
             "challenge_budget": budget,
@@ -411,9 +411,9 @@ run("float-schema", float_schema["ledger"], 1, "schema_version must be 3")
 
 float_budget = make_fixture(
     "float-budget",
-    receipt_mutators={2: lambda row: row.update(challenge_budget=2.0)},
+    receipt_mutators={2: lambda row: row.update(challenge_budget=1.0)},
 )
-run("float-budget", float_budget["ledger"], 1, "challenge_budget must be 2")
+run("float-budget", float_budget["ledger"], 1, "challenge_budget must be 1")
 
 nan_schema = make_fixture("nan-schema")
 nan_schema["ledger"]["schema_version"] = float("nan")
@@ -493,11 +493,17 @@ scope_mismatch = make_fixture("scope-mismatch", receipt_mutators={2: drift_scope
 run("scope-mismatch", scope_mismatch["ledger"], 1, "review scope changed")
 
 budget_four = make_fixture("budget-four", budget=4)
-run("budget-four", budget_four["ledger"], 1, "challenge_budget must be 2")
+run("budget-four", budget_four["ledger"], 1, "challenge_budget must be 1")
 
 review_only = make_fixture("review-only", receipt_findings=[[]])
 run("review-only", review_only["ledger"], 1, "ready requires at least one tracked challenge")
 
+# A caller must not out-run the wrapper budget by supplying extra receipts: a
+# third receipt under budget 1 claims a negative remaining count and would
+# otherwise reach completion validation as a ready-state budget bypass.
+over_budget = make_fixture("over-budget", receipt_findings=[[], [], []])
+run("over-budget", over_budget["ledger"], 1, "exceeds the wrapper budget")
+
 missing_complete = make_fixture("missing-complete")
 missing_complete["ledger"]["completion_receipt"] = None
 run("missing-complete", missing_complete["ledger"], 1, "requires a completion receipt")
@@ -537,7 +543,7 @@ run(
 
 continuation = make_fixture(
     "continuation",
-    receipt_findings=[[finding(1)], [], [finding(2)]],
+    receipt_findings=[[finding(1)], [finding(2)]],
     completion=False,
     open_last=True,
 )
@@ -550,10 +556,10 @@ run(
 
 continuation_unconsumed_first_drift = make_fixture(
     "continuation-unconsumed-first-drift",
-    receipt_findings=[[finding(1)], [], [finding(2)]],
+    receipt_findings=[[finding(1)], [finding(2)]],
     completion=False,
     open_last=True,
-    base_shas=[A, A, A, B],
+    base_shas=[A, A, B],
 )
 run(
     "continuation-unconsumed-first-drift",
@@ -564,7 +570,7 @@ run(
 
 continuation_candidate = make_fixture(
     "continuation-candidate",
-    receipt_findings=[[finding(1)], [], [finding(2)]],
+    receipt_findings=[[finding(1)], [finding(2)]],
     completion=False,
     open_last=True,
 )
@@ -595,8 +601,8 @@ run(
 
 unknown_state = make_fixture(
     "unknown-state",
-    receipt_findings=[[finding(1)], [], [finding(2)]],
-    receipt_mutators={3: lambda row: row.update(review_state="potato")},
+    receipt_findings=[[finding(1)], [finding(2)]],
+    receipt_mutators={2: lambda row: row.update(review_state="potato")},
     completion=False,
     open_last=True,
 )
@@ -605,10 +611,10 @@ run("unknown-state", unknown_state["ledger"], 1, "unknown controller review_stat
 
 passed_continuation = make_fixture(
     "passed-continuation",
-    receipt_findings=[[finding(1)], [], []],
+    receipt_findings=[[finding(1)], []],
     completion=False,
 )
-run("passed-continuation", passed_continuation["ledger"], 1, "round 3 findings in post_review_budget")
+run("passed-continuation", passed_continuation["ledger"], 1, "findings in post_review_budget")
 
 bad_finding_hash = make_fixture("bad-finding-hash")
 bad_finding_hash["ledger"]["finding_classes"][0]["occurrences"][0][
@@ -744,7 +750,7 @@ run(
 
 historical_open = make_fixture(
     "historical-open",
-    receipt_findings=[[finding(1)], [finding(2)], []],
+    receipt_findings=[[finding(1), finding(2)], []],
 )
 historical_open["ledger"]["finding_classes"][0]["occurrences"][0][
     "disposition"
@@ -756,7 +762,7 @@ run("historical-open", historical_open["ledger"], 1, "unresolved finding occurre
 
 historical_open_linked = make_fixture(
     "historical-open-linked",
-    receipt_findings=[[finding(1)], [finding(2)], []],
+    receipt_findings=[[finding(1), finding(2)], []],
 )
 historical_open_linked_occurrences = historical_open_linked["ledger"][
     "finding_classes"
@@ -825,7 +831,7 @@ run("needs-human", needs_human["ledger"], 1, "unresolved finding occurrence")
 
 third_occurrence = make_fixture(
     "third-occurrence",
-    receipt_findings=[[finding(1)], [finding(2)], [finding(3)]],
+    receipt_findings=[[finding(1), finding(2)], [finding(3)]],
     completion=False,
     unmatched=1,
     open_last=True,
@@ -1045,7 +1051,7 @@ for closing_disposition in sorted(CLOSED_DISPOSITIONS):
     case_name = f"historical-needs-human-{closing_disposition.replace('_', '-')}"
     historical_needs_human_linked = make_fixture(
         case_name,
-        receipt_findings=[[finding(1)], [finding(2)], []],
+        receipt_findings=[[finding(1), finding(2)], []],
     )
     historical_needs_human_occurrences = historical_needs_human_linked["ledger"][
         "finding_classes"
@@ -1128,7 +1134,7 @@ duplicate_completion_result = invoke(
 
 duplicate_sweep = make_fixture(
     "duplicate-sweep",
-    receipt_findings=[[finding(1)], [finding(2)], [finding(3)]],
+    receipt_findings=[[finding(1), finding(2)], [finding(3)]],
     completion=False,
     open_last=True,
 )

package/dist/assets/marketplace/plugins/ccl-skills/skills/skill-extraction-workflow/scripts/validate_extraction_review_state.py

@@ -30,6 +30,11 @@ TERMINAL_STATES = {
 }
 EXTERNAL_REVIEW_STATES = {"reviewed", "findings_pending", "post_review_budget"}
 KNOWN_REVIEW_STATES = EXTERNAL_REVIEW_STATES | {"self_reviewed"}
+# The extraction wrapper fixes the autonomous lane at one review plus one
+# challenge; every numeric bound below derives from these two constants so a
+# future budget change lands in exactly one place.
+WRAPPER_CHALLENGE_BUDGET = 1
+WRAPPER_AUTONOMOUS_ROUNDS = WRAPPER_CHALLENGE_BUDGET + 1
 DISPOSITIONS = {
     "fixed",
     "source_refuted",
@@ -272,8 +277,8 @@ def validate_scope(receipt: dict[str, Any], label: str) -> str:
     ]
     if len(normalized_risks) != len(set(normalized_risks)):
         fail(f"{label}.review_scope.risk_tags contains duplicates")
-    if scope["challenge_budget"] != 2 or type(scope["challenge_budget"]) is not int:
-        fail(f"{label}.review_scope.challenge_budget must be 2")
+    if scope["challenge_budget"] != WRAPPER_CHALLENGE_BUDGET or type(scope["challenge_budget"]) is not int:
+        fail(f"{label}.review_scope.challenge_budget must be {WRAPPER_CHALLENGE_BUDGET}")
     if (
         scope["wording_only_proof_sha256"] is not None
         or scope["wording_only_scope_sha256"] is not None
@@ -313,6 +318,11 @@ def validate_controller_receipts(
         payload["candidate_sha256"], "ledger.candidate_sha256"
     )
     for expected_index, value in enumerate(refs, start=1):
+        # A caller supplying more receipts than the wrapper can mint would
+        # otherwise claim a negative remaining count and reach completion
+        # validation as a ready-state budget bypass.
+        if expected_index > WRAPPER_AUTONOMOUS_ROUNDS:
+            fail("controller chain exceeds the wrapper budget")
         ref = exact_object(
             value, {"sequence", "file", "sha256"}, f"controller_receipts[{expected_index - 1}]"
         )
@@ -352,13 +362,13 @@ def validate_controller_receipts(
             receipt.get("challenge_index")
         ) is not int:
             fail(f"controller receipt {expected_index} challenge_index is invalid")
-        if receipt.get("challenge_budget") != 2 or type(receipt.get("challenge_budget")) is not int:
-            fail(f"controller receipt {expected_index} challenge_budget must be 2")
-        if receipt.get("autonomous_review_budget") != 3 or type(
+        if receipt.get("challenge_budget") != WRAPPER_CHALLENGE_BUDGET or type(receipt.get("challenge_budget")) is not int:
+            fail(f"controller receipt {expected_index} challenge_budget must be {WRAPPER_CHALLENGE_BUDGET}")
+        if receipt.get("autonomous_review_budget") != WRAPPER_AUTONOMOUS_ROUNDS or type(
             receipt.get("autonomous_review_budget")
         ) is not int:
-            fail(f"controller receipt {expected_index} autonomous_review_budget must be 3")
-        expected_remaining = 3 - expected_index
+            fail(f"controller receipt {expected_index} autonomous_review_budget must be {WRAPPER_AUTONOMOUS_ROUNDS}")
+        expected_remaining = WRAPPER_AUTONOMOUS_ROUNDS - expected_index
         if receipt.get("autonomous_reviews_remaining") != expected_remaining or type(
             receipt.get("autonomous_reviews_remaining")
         ) is not int:
@@ -415,7 +425,7 @@ def validate_controller_receipts(
             fail(f"controller receipt {expected_index} has unknown controller review_state {state}")
         expected_state = (
             "post_review_budget"
-            if receipt["status"] == "findings" and expected_index == 3
+            if receipt["status"] == "findings" and expected_index == WRAPPER_AUTONOMOUS_ROUNDS
             else "findings_pending"
             if receipt["status"] == "findings"
             else "reviewed"
@@ -469,17 +479,17 @@ def validate_completion_receipt(
         fail("completion receipt cannot close a final external receipt with findings")
     if receipt.get("review_chain_tracked") is not True or receipt.get("review_chain_id") != chain_id:
         fail("completion receipt review_chain_id does not match the controller chain")
-    if receipt.get("challenge_budget") != 2 or type(receipt.get("challenge_budget")) is not int:
-        fail("completion receipt challenge_budget must be 2")
-    if receipt.get("autonomous_review_budget") != 3 or type(
+    if receipt.get("challenge_budget") != WRAPPER_CHALLENGE_BUDGET or type(receipt.get("challenge_budget")) is not int:
+        fail(f"completion receipt challenge_budget must be {WRAPPER_CHALLENGE_BUDGET}")
+    if receipt.get("autonomous_review_budget") != WRAPPER_AUTONOMOUS_ROUNDS or type(
         receipt.get("autonomous_review_budget")
     ) is not int:
-        fail("completion receipt autonomous_review_budget must be 3")
+        fail(f"completion receipt autonomous_review_budget must be {WRAPPER_AUTONOMOUS_ROUNDS}")
     if receipt.get("autonomous_review_index") != len(receipts) or type(
         receipt.get("autonomous_review_index")
     ) is not int:
         fail("completion receipt autonomous_review_index does not match the final round")
-    expected_remaining = 3 - len(receipts)
+    expected_remaining = WRAPPER_AUTONOMOUS_ROUNDS - len(receipts)
     if receipt.get("autonomous_reviews_remaining") != expected_remaining or type(
         receipt.get("autonomous_reviews_remaining")
     ) is not int:
@@ -941,12 +951,12 @@ def validate(payload: dict[str, Any], ledger_dir: Path) -> tuple[str, int, int]:
     elif closeout == "continuation_authorization_required":
         final = receipts[-1]
         if (
-            len(receipts) != 3
+            len(receipts) != WRAPPER_AUTONOMOUS_ROUNDS
             or final.get("status") != "findings"
             or final.get("review_state") != "post_review_budget"
             or final.get("human_decision_required") is not True
         ):
-            fail("continuation_authorization_required requires round 3 findings in post_review_budget")
+            fail(f"continuation_authorization_required requires final-round (round {WRAPPER_AUTONOMOUS_ROUNDS}) findings in post_review_budget")
     elif closeout == "baseline_race" and not delta:
         fail("baseline_race requires a non-empty unreviewed_delta")