@ccoalm/ccl-skills 0.9.0 → 0.10.0

package/dist/assets/marketplace/plugins/ccl-skills/skills/skill-extraction-workflow/scripts/check-size-budget.sh

@@ -21,6 +21,18 @@
 #     advisory = no signal). A agent-context/session-start.md absent from base is treated as a NEW
 #     every-session injection and blocks like a new severe entrypoint.
 #
+# Blocking (delta-scoped, skills/*/references/**/*.md — the reference-file line
+#   ratchet; design invariants in references/attention-budget-ratchet.md):
+#   - a changed reference that is NEW or CROSSES over 500 physical lines (base
+#     was not over)               => block ("new reference over line budget")
+#   - a changed reference already over 500 lines in base that GREW
+#     (head_lines > base_lines)   => block ("over-limit reference grew")
+#   references/source-register.md is structurally excluded (append-only ledger;
+#   its growth is governed by the append-only contract, not a line cap) — the
+#   exclusion lives in this gate, never in the candidate. A NEW reference over
+#   100 lines with no "##" section headings draws an advisory token (never
+#   blocks). Rename credit is path-paired and non-growing, same as entrypoints.
+#
 # Advisory (never blocks on its own): the recommended-band body-char debt
 # counters, severe-debt counters, per-file deltas, and the agent-context/session-start.md 13000B
 # byte tripwire BAND remain VISIBILITY / DEBT-MANAGEMENT only (the band flags the
@@ -113,6 +125,11 @@ end
 WORD_BUDGET_MAX = 5000
 WORD_TOKEN_PATTERN = /\p{Han}|\uFFFD|(?:(?!\p{Han})[\p{L}\p{N}])+(?:[\u0027\u2019_-](?:(?!\p{Han})[\p{L}\p{N}])+)*/u
 
+REF_LINE_MAX = 500
+REF_NAV_MIN_LINES = 100
+REF_PATH_RE = %r{\Askills/[^/]+/references/.+\.md\z}
+REF_LEDGER_RE = %r{\Askills/[^/]+/references/source-register\.md\z}
+
 Metric = Struct.new(:body_chars, :body_words, :bytes, keyword_init: true)
 
 def metric_for_text(text)
@@ -195,6 +212,42 @@ def head_metric_for_path(root, rel)
   metric_for_file(full)
 end
 
+def ref_info_for_text(raw)
+  text = raw.b.dup.force_encoding(Encoding::UTF_8).scrub { |invalid| "�" * invalid.bytesize }
+  # Line-ending normalization comes first: a CR-only or CRLF file must measure the
+  # same as the LF file a reader sees, otherwise a 501-line CR-delimited file reads
+  # as one line and earns a clean verdict. Headings count EXACT H2 only, because an
+  # H3-only file gives a chunked read no section structure to navigate by.
+  text = text.gsub(/\r\n?/, "\n")
+  lines = text.lines
+  { lines: lines.length, headings: lines.count { |l| l.match?(/\A##[ \t]/) } }
+end
+
+def base_ref_info_for_path(root, base, rel)
+  return :unknown if base.nil?
+  return :missing unless git_success?("cat-file", "-e", "#{base}:#{rel}", root: root)
+  ref_info_for_text(git_text("show", "#{base}:#{rel}", root: root))
+end
+
+def head_ref_info_for_path(root, rel)
+  full = File.join(root, rel)
+  return :missing unless File.file?(full)
+  ref_info_for_text(File.binread(full))
+end
+
+def fmt_ref_value(info)
+  case info
+  when :unknown then "unknown"
+  when :missing then "missing"
+  else info[:lines].to_s
+  end
+end
+
+def fmt_ref_delta(base_info, head_info)
+  return "unknown" unless base_info.is_a?(Hash) && head_info.is_a?(Hash)
+  format("%+d", head_info[:lines] - base_info[:lines])
+end
+
 def fmt_metric_value(metric, field)
   case metric
   when :unknown then "unknown"
@@ -437,6 +490,144 @@ if bootstrap_changed
   # :missing (deleted) => nothing left to size-gate, same as a deleted entrypoint
 end
 
+# --- reference-file line-budget ratchet (delta-scoped) ------------------------
+# Design invariants: references/attention-budget-ratchet.md. Physical lines are
+# the stable proxy metric; the append-only ledger references/source-register.md
+# is excluded by gate design (its growth is governed by the append-only
+# contract) and the exclusion lives HERE so a candidate cannot nominate its own
+# exemption.
+ref_blocks = []
+ref_partials = []
+ref_changed = []
+ref_scan_error = nil
+begin
+  if system("git", "-C", root, "rev-parse", "--is-inside-work-tree", out: File::NULL, err: File::NULL)
+    rpaths = []
+    rpaths.concat(git_lines("diff", "--name-only", base, "HEAD", "--", "skills", root: root)) if base
+    rpaths.concat(git_lines("diff", "--name-only", "--", "skills", root: root))
+    rpaths.concat(git_lines("diff", "--cached", "--name-only", "--", "skills", root: root))
+    rpaths.concat(git_lines("ls-files", "--others", "--exclude-standard", "--", "skills", root: root))
+    ref_changed = rpaths.select { |p| p.match?(REF_PATH_RE) }.reject { |p| p.match?(REF_LEDGER_RE) }.uniq
+  end
+rescue StandardError => e
+  ref_scan_error = "#{e.class.name} #{e.message.lines.first.to_s.strip}"
+  warn "changed_reference_line_scan_skipped: #{ref_scan_error}"
+end
+# A scan failure must not silently read as "no references changed" — that would
+# false-green a real growth, so it fails closed as a partial.
+ref_partials << "skills/*/references: changed-detection failed (#{ref_scan_error}) — line-budget check unavailable (fail-closed)" if ref_scan_error
+
+ref_move_map = {}
+begin
+  if system("git", "-C", root, "rev-parse", "--is-inside-work-tree", out: File::NULL, err: File::NULL)
+    status_args = []
+    status_args << ["diff", "--name-status", base, "HEAD"] if base
+    status_args << ["diff", "--name-status", "--cached"]
+    status_args << ["diff", "--name-status"]
+    status_args.each do |cmd|
+      git_lines(*cmd, "--", "skills", root: root).each do |line|
+        status, *paths = line.split("\t")
+        next unless status.start_with?("R") && paths.length >= 2
+        next unless status[1..].to_i >= 90
+        old_rel = paths[0].match?(REF_PATH_RE) ? paths[0] : nil
+        new_rel = paths[1].match?(REF_PATH_RE) ? paths[1] : nil
+        ref_move_map[new_rel] = old_rel if old_rel && new_rel
+      end
+    end
+  end
+rescue StandardError => e
+  warn "reference_move_pair_scan_skipped: #{e.class.name}"
+end
+
+# The census is visibility only, so an unreadable or vanished file must degrade the
+# COUNTER to unknown rather than abort the program: aborting here would skip the
+# per-file fail-closed partials below and emit no verdict token at all.
+head_ref_over_count = 0
+head_ref_census_error = nil
+begin
+  Dir[File.join(root, "skills", "*", "references", "**", "*.md")].each do |path|
+    rel = path.start_with?(root + "/") ? path[(root.length + 1)..] : path.sub(%r{\A\./}, "")
+    next unless rel.match?(REF_PATH_RE)
+    next if rel.match?(REF_LEDGER_RE)
+    info = begin
+      head_ref_info_for_path(root, rel)
+    rescue StandardError => e
+      head_ref_census_error ||= "#{rel}: #{e.class.name}"
+      next
+    end
+    head_ref_over_count += 1 if info.is_a?(Hash) && info[:lines] > REF_LINE_MAX
+  end
+rescue StandardError => e
+  head_ref_census_error ||= e.class.name
+end
+if head_ref_census_error
+  warn "reference_line_census_partial: head over-limit census incomplete (#{head_ref_census_error}) — counter reported unknown; per-file verdicts below are unaffected"
+  head_ref_over_count = nil
+end
+base_ref_over_count = nil
+if base
+  begin
+    base_ref_over_count = 0
+    git_lines("grep", "-c", "-e", "", base, "--", "skills", root: root).each do |line|
+      rest, _, cnt = line.rpartition(":")
+      _, _, rel = rest.partition(":")
+      next unless rel.match?(REF_PATH_RE)
+      next if rel.match?(REF_LEDGER_RE)
+      base_ref_over_count += 1 if cnt.to_i > REF_LINE_MAX
+    end
+  rescue StandardError => e
+    base_ref_over_count = nil
+    warn "reference_line_trend_partial: base=unknown reason=#{e.class.name}"
+  end
+end
+puts "reference_line_over_limit_count_base=#{base_ref_over_count.nil? ? "unknown" : base_ref_over_count}"
+puts "reference_line_over_limit_count_head=#{head_ref_over_count.nil? ? "unknown" : head_ref_over_count}"
+puts "reference_line_over_limit_count_delta=#{(base_ref_over_count.nil? || head_ref_over_count.nil?) ? "unknown" : format("%+d", head_ref_over_count - base_ref_over_count)}"
+
+# Ledger visibility: excluded from the ratchet, still reported when over.
+Dir[File.join(root, "skills", "*", "references", "source-register.md")].sort.each do |path|
+  rel = path.start_with?(root + "/") ? path[(root.length + 1)..] : path.sub(%r{\A\./}, "")
+  info = begin
+    head_ref_info_for_path(root, rel)
+  rescue StandardError
+    :missing
+  end
+  if info.is_a?(Hash) && info[:lines] > REF_LINE_MAX
+    puts "reference_line_budget_ledger_excluded: #{rel} lines=#{info[:lines]} — append-only ledger, excluded by gate design (growth governed by the append-only contract)"
+  end
+end
+
+ref_changed.sort.each do |rel|
+  begin
+    head_info = head_ref_info_for_path(root, rel)
+  rescue StandardError => e
+    ref_partials << "#{rel}: head lines unreadable (#{e.class.name}) — line-budget check unavailable (fail-closed)"
+    next
+  end
+  next if head_info == :missing # deleted reference: nothing left to gate
+  base_info = base_ref_info_for_path(root, base, rel)
+  puts "changed_reference_line_delta: #{rel} base_lines=#{fmt_ref_value(base_info)} head_lines=#{head_info[:lines]} delta_lines=#{fmt_ref_delta(base_info, head_info)}"
+  if base_info == :missing && head_info[:lines] > REF_NAV_MIN_LINES && head_info[:headings].zero?
+    warn "reference_nav_advisory: #{rel} lines=#{head_info[:lines]} headings=0 — a new reference over #{REF_NAV_MIN_LINES} lines needs ## section structure so chunked reads and greps can navigate (advisory, never blocks)"
+  end
+  next unless head_info[:lines] > REF_LINE_MAX
+  if base_info == :unknown
+    ref_partials << "#{rel}: base unknown — reference line-budget check unavailable (fail-closed)"
+  elsif base_info == :missing || base_info[:lines] <= REF_LINE_MAX
+    source_rel = ref_move_map[rel]
+    source_info = source_rel ? base_ref_info_for_path(root, base, source_rel) : nil
+    if source_info.is_a?(Hash) && source_info[:lines] > REF_LINE_MAX && head_info[:lines] <= source_info[:lines]
+      puts "reference_line_budget_move_ok: #{rel} head_lines=#{head_info[:lines]} allowed_lines=#{source_info[:lines]} moved_from=#{source_rel}"
+    else
+      ref_blocks << "#{rel}: new reference over line budget head_lines=#{head_info[:lines]} (> #{REF_LINE_MAX}) — split it by subtopic before landing (see references/attention-budget-ratchet.md; there is no exempt marker or waiver flag)"
+    end
+  elsif head_info[:lines] > base_info[:lines]
+    ref_blocks << "#{rel}: over-limit reference grew base_lines=#{base_info[:lines]} head_lines=#{head_info[:lines]} — shrink or stay level; fund additions by consolidating text in the same file"
+  else
+    puts "reference_line_budget_legacy_ok: #{rel} base_lines=#{base_info[:lines]} head_lines=#{head_info[:lines]} allowed_lines=#{base_info[:lines]}"
+  end
+end
+
 # Advisory markers only when the blocking verdict is clean — a consumer reading
 # the preserved legacy token must never see ok next to a block.
 #
@@ -451,18 +642,20 @@ end
 # — but the token says un-evaluated, not ok.
 # NOTE: this comment lives inside the single-quoted `ruby -e` program; an
 # apostrophe here terminates the shell quote and breaks the script.
-if blocks.empty? && partials.empty? && word_blocks.empty? && word_partials.empty?
+if blocks.empty? && partials.empty? && word_blocks.empty? && word_partials.empty? && ref_blocks.empty? && ref_partials.empty?
   puts "size_budget_advisory_#{size_state}"
   puts "entrypoint_size_budget_advisory_ok"
   if base.nil?
     puts "entrypoint_size_blocking_unevaluated: base=unknown — committed changes were NOT delta-checked; this is not a pass"
     puts "entrypoint_word_budget_blocking_unevaluated: base=unknown — committed changes were NOT delta-checked; this is not a pass"
+    puts "reference_line_budget_blocking_unevaluated: base=unknown — committed changes were NOT delta-checked; this is not a pass"
     if File.file?(File.join(root, "agent-context/session-start.md"))
       puts "bootstrap_size_delta_unevaluated: base=unknown — agent-context/session-start.md committed changes were NOT delta-checked; this is not a pass"
     end
   else
     puts "entrypoint_size_blocking_ok"
     puts "entrypoint_word_budget_blocking_ok"
+    puts "reference_line_budget_blocking_ok"
   end
   exit 0
 end
@@ -470,7 +663,10 @@ blocks.each { |b| warn "entrypoint_size_block: #{b}" }
 partials.each { |b| warn "entrypoint_size_block_partial: #{b}" }
 word_blocks.each { |b| warn "entrypoint_word_budget_block: #{b}" }
 word_partials.each { |b| warn "entrypoint_word_budget_block_partial: #{b}" }
+ref_blocks.each { |b| warn "reference_line_block: #{b}" }
+ref_partials.each { |b| warn "reference_line_block_partial: #{b}" }
 puts "entrypoint_word_budget_blocking_failed" unless word_blocks.empty? && word_partials.empty?
+puts "reference_line_budget_blocking_failed" unless ref_blocks.empty? && ref_partials.empty?
 # Legacy aggregate token for every blocking verdict owned by this size-budget
 # script, including the body-word rule. Keep it unconditional so existing
 # consumers cannot miss a new word-only failure.

package/dist/assets/marketplace/plugins/ccl-skills/skills/skill-extraction-workflow/scripts/contract-anchors.tsv

@@ -0,0 +1,15 @@
+# Contract-anchor table for check-contract-anchors.sh (same directory).
+# One row per pinned load-bearing contract literal:
+#   id <TAB> repo-relative path <TAB> pinned literal (exactly-once, >=16 chars) <TAB> note
+# The note names the sanitized provenance label and why the wording is load-bearing.
+# To change pinned wording intentionally: edit the contract sentence AND this row
+# in the same MR. Keep this set small — load-bearing contracts only, never style.
+verdict-taxonomy-discriminator	skills/testing-strategy/references/ci-fixtures-and-flake-control.md	One discriminating predicate decides the verdict	071-chainA-r1f1: fault-origin discriminator sentence; deleting it reverts verdicts to case-by-case judgment
+verdict-taxonomy-false-green	skills/testing-strategy/references/ci-fixtures-and-flake-control.md	excludes `infra-error` cases to present a clean total is a false-green report	071-chainA-r1f1: report-completeness clause; deleting it legalizes infra-error-excluding clean totals
+stop-predicate-materially-differing	skills/product-rd-workflow/SKILL.md	materially differing viable approaches (none dominant-and-reversible)	071-chainB-r1f1: stop-condition predicate wording; inverting it flips blocked/continue classification
+stop-predicate-evidenced-cause	skills/product-rd-workflow/SKILL.md	a fix lacking evidenced cause	071-chainB-r1f1: stop-condition predicate wording; weakening it admits plausible-but-unevidenced fixes
+burn-rate-tiers-entry	skills/platform-observability/SKILL.md	page 14.4× 1h/5m, page 6× 6h/30m, ticket 1× 3d/6h	071-chainC-r1f4: SRE Workbook multiwindow tiers, externally verified value (specs/071 source-verification); silent renumbering must not pass
+burn-rate-page-row-reference	skills/platform-observability/references/sli-slo-design.md	| Page (P0) | 1h | 5m | 14.4 |	071-chainC-r1f4: reference-side copy of the page tier; pinning both sides makes entry/reference drift visible
+coverage-tier-provenance	skills/testing-strategy/references/test-code-authoring-patterns.md	60% acceptable / 75% commendable / 90% exemplary 分档	071-chainC-r1f4: externally verified coverage tiers (specs/071 source-verification)
+pairwise-trigger-range	skills/test-artifact-management/references/classical-test-design-techniques.md	2-way 累计触发 53–97%	071-chainC-r1f4: NIST SP 800-142 empirical range, externally verified (specs/071 source-verification)
+bva-two-vs-three-value	skills/test-artifact-management/references/classical-test-design-techniques.md	2-value（边界 + 下一格）和 3-value（边界 + 两侧）	071-chainC-r1f4: ISTQB v4 BVA variant definitions, externally verified (specs/071 source-verification)

package/dist/assets/marketplace/plugins/ccl-skills/skills/skill-extraction-workflow/scripts/eval-routing-bank.rb

@@ -16,13 +16,23 @@
 # Usage:
 #   eval-routing-bank.rb <repo-root> [--bank <path>] [--model <name>] [--limit N]
 #                        [--dry-run] [--json <path>] [--baseline <path>] [--timeout S]
-#                        [--desc-budget-chars N] [--with-bootstrap]
+#                        [--desc-budget-chars N] [--with-bootstrap] [--replicas N]
 #
 # --desc-budget-chars N simulates a consumer that truncates each skill description
 # to its first N characters before routing (e.g. Codex compresses the skill listing
 # under a ~2%-of-context budget, 8000 chars when the window is unknown, shortening
 # descriptions first). Run the bank once plain and once with a budget to see which
 # routes only survive on the description tail — those triggers need front-loading.
+#
+# expected_skill "none" marks negative controls (out-of-library utterances) and
+# coverage-gap probes (in-domain utterances no skill owns): the correct outcome
+# is rejection, and a catalog skill claiming them is labeled "absorbed".
+# --replicas N grades each task N times: task status is the conservative
+# consensus (every observed replica must PASS), replica top1 agreement is
+# reported, and disagreeing replicas label the task "ownership_split". clarify
+# and low-confidence counts are first-class report fields either way.
+# Reports from different (bank, replicas) configurations are different rulers —
+# do not diff them as a regression signal.
 # Exit: 0 = ran (advisory); 2 = usage error; 3 = grader entirely unavailable.
 
 require "yaml"
@@ -39,7 +49,7 @@ end
 
 root = ARGV[0]
 if root.nil? || root.start_with?("-")
-  warn "usage: eval-routing-bank.rb <repo-root> [--bank p] [--model m] [--limit N] [--dry-run] [--json p] [--baseline p] [--timeout S] [--desc-budget-chars N]"
+  warn "usage: eval-routing-bank.rb <repo-root> [--bank p] [--model m] [--limit N] [--dry-run] [--json p] [--baseline p] [--timeout S] [--desc-budget-chars N] [--replicas N]"
   exit 2
 end
 bank_path = arg("--bank", File.join(root, "eval", "routing-tasks.jsonl"))
@@ -65,6 +75,17 @@ if ARGV.include?("--desc-budget-chars")
   desc_budget = b.to_i
 end
 timeout_s = (t = arg("--timeout")) ? t.to_i : 60
+replicas = 1
+if ARGV.include?("--replicas")
+  r = arg("--replicas")
+  # Same strictness as --desc-budget-chars: a silent to_i coercion would turn a
+  # typo into "1 replica" and the agreement metric would quietly measure nothing.
+  unless r && r =~ /\A[1-9]\d*\z/
+    warn "--replicas requires a positive integer value, got #{r.inspect}"
+    exit 2
+  end
+  replicas = r.to_i
+end
 
 unless File.file?(bank_path)
   warn "eval_bank_missing: #{bank_path}"
@@ -89,16 +110,57 @@ end
 tasks = tasks.first(limit) if limit
 
 # Schema validation: a task must be answerable, and must not be self-contradictory.
+# expected_skill "none" is the negative-control / coverage-gap sentinel: the
+# correct routing outcome is that NO catalog skill claims the utterance.
+all_outcomes = Dir[File.join(root, "skills", "*", "SKILL.md")]
+               .map { |p| File.basename(File.dirname(p)) } + ["none"]
 tasks.each do |t|
   id = t["id"] || "(no id)"
   if t["utterance"].to_s.strip.empty? || t["expected_skill"].to_s.strip.empty?
     warn "eval_bank_invalid_task: #{id}: utterance and expected_skill are required"
     exit 2
   end
+  # Type-check the ORIGINAL values FIRST: a present-but-non-list field (e.g. ""
+  # or a bare string or a number) must fail as a usage error before any
+  # membership check touches it — Ruby strings are truthy and respond to
+  # include? (substring semantics), and non-strings would raise a bare
+  # NoMethodError instead of the documented invalid-bank diagnostic.
+  %w[must_not_route_to acceptable].each do |f|
+    next unless t.key?(f)
+    unless t[f].is_a?(Array)
+      warn "eval_bank_invalid_task: #{id}: #{f} must be a list, got #{t[f].inspect}"
+      exit 2
+    end
+  end
   if (t["must_not_route_to"] || []).include?(t["expected_skill"])
     warn "eval_bank_invalid_task: #{id}: expected_skill is also in must_not_route_to (impossible)"
     exit 2
   end
+  if (t["must_not_route_to"] || []).include?("none")
+    warn "eval_bank_invalid_task: #{id}: \"none\" is a sentinel outcome, not a routable target for must_not_route_to"
+    exit 2
+  end
+  # Optional acceptable[] names defensible alternate outcomes (a skill name or
+  # "none") for utterances with more than one correct route — e.g. a coverage-gap
+  # ask where both coordinator intake and rejection are right. It must not
+  # restate expected_skill or contradict must_not_route_to.
+  acc = t["acceptable"] || []
+  if acc.include?(t["expected_skill"])
+    warn "eval_bank_invalid_task: #{id}: acceptable restates expected_skill"
+    exit 2
+  end
+  unless (acc & (t["must_not_route_to"] || [])).empty?
+    warn "eval_bank_invalid_task: #{id}: acceptable and must_not_route_to overlap (contradictory)"
+    exit 2
+  end
+  # Anti-gaming: expected + acceptable must leave at least one outcome that
+  # would FAIL the row. A row covering the complete catalog-and-none outcome
+  # space passes on every valid grader selection — a vacuous fixture that fakes
+  # green regardless of description behavior.
+  if (all_outcomes - ([t["expected_skill"]] + acc)).empty?
+    warn "eval_bank_invalid_task: #{id}: expected_skill plus acceptable cover every possible outcome (vacuous row)"
+    exit 2
+  end
 end
 
 # --- anti-game ---------------------------------------------------------------
@@ -188,9 +250,12 @@ def build_prompt(catalog, utterance, bootstrap_layer = nil)
     user utterance and the catalog of skills (name + description), pick the SINGLE
     best skill that should handle it. Use only the catalog; route by the
     descriptions' Use-when / Proactively / Skip rules.
+    If NO skill's description covers the utterance, answer "none" — do not
+    force-fit the nearest neighbor. Set "clarify" to true only when you would
+    need to ask the user a clarifying question before committing to a route.
 
     Output ONLY a JSON object on one line, no other text:
-    {"selected_skill": "<exact skill name from the catalog>", "confidence": <0.0-1.0>, "rationale_short": "<one short clause>"}
+    {"selected_skill": "<exact skill name from the catalog, or none>", "clarify": <true|false>, "confidence": <0.0-1.0>, "rationale_short": "<one short clause>"}
 
     #{bootstrap_layer ? "== ALWAYS-ON ENTRY ROUTING LAYER (injected into every session; takes precedence for entry routing) ==\n#{bootstrap_layer}\n" : ""}
     == SKILL CATALOG ==
@@ -309,39 +374,77 @@ grader_available = true
 tasks.each do |t|
   exp = t["expected_skill"]
   must_not = t["must_not_route_to"] || []
+  acceptable = t["acceptable"] || []
   frozen_ref = t["frozen_at_sha"] == "root" ? `git -C #{Shellwords.escape(root)} rev-list --max-parents=0 HEAD`.lines.first.to_s.strip : t["frozen_at_sha"]
   frozen_ok = ancestor?(root, frozen_ref)
   prompt = build_prompt(catalog, t["utterance"], bootstrap_layer)
-  parsed, error = grade(model, timeout_s, prompt)
-  # A task that produced no observation contributes nothing to the totals, and an
-  # unmeasured task is indistinguishable from a routing failure in the numbers
-  # this bank reports. Both recoverable causes are sampling accidents rather than
-  # verdicts — a hard timeout is usually machine load, and unparseable output is
-  # usually one stray quote in a generated rationale — so each gets one retry.
-  # Repairing malformed output instead of re-asking for it was tried and removed:
-  # interpreting text that is by definition malformed has no natural boundary,
-  # and three review rounds each found a different shape that a repair would read
-  # as a verdict. Re-asking needs no such interpretation. Nothing else is retried:
-  # an auth failure and a missing CLI do reproduce on a second call.
-  if error&.start_with?("grader_timeout_") || error&.start_with?("no_json_in_output")
-    parsed, retry_error = grade(model, timeout_s, prompt)
-    error = parsed ? nil : retry_error
-  end
-  if error == "claude_not_found"
-    grader_available = false
-    break
+  verdicts = []
+  replicas.times do |ri|
+    parsed, error = grade(model, timeout_s, prompt)
+    # A verdict that produced no observation contributes nothing to the totals,
+    # and an unmeasured task is indistinguishable from a routing failure in the
+    # numbers this bank reports. Both recoverable causes are sampling accidents
+    # rather than verdicts — a hard timeout is usually machine load, and
+    # unparseable output is usually one stray quote in a generated rationale —
+    # so each gets one retry.
+    # Repairing malformed output instead of re-asking for it was tried and removed:
+    # interpreting text that is by definition malformed has no natural boundary,
+    # and three review rounds each found a different shape that a repair would read
+    # as a verdict. Re-asking needs no such interpretation. Nothing else is retried:
+    # an auth failure and a missing CLI do reproduce on a second call.
+    if error&.start_with?("grader_timeout_") || error&.start_with?("no_json_in_output")
+      parsed, retry_error = grade(model, timeout_s, prompt)
+      error = parsed ? nil : retry_error
+    end
+    if error == "claude_not_found"
+      grader_available = false
+      break
+    end
+    selected = parsed && parsed["selected_skill"]
+    clarify = parsed && parsed["clarify"] == true
+    confidence = parsed && parsed["confidence"]
+    v_status =
+      if error then "ERROR"
+      elsif (selected == exp || acceptable.include?(selected)) && !must_not.include?(selected) then "PASS"
+      else "FAIL"
+      end
+    verdicts << { replica: ri + 1, selected: selected, clarify: clarify,
+                  confidence: confidence, status: v_status, error: error }
   end
-  selected = parsed && parsed["selected_skill"]
-  confidence = parsed && parsed["confidence"]
+  break unless grader_available
+  # Task-level consensus over replicas (replicas=1 reproduces the old per-task
+  # semantics exactly). Conservative: one failing replica fails the task —
+  # a route that only sometimes lands is not a stable route.
+  observed = verdicts.reject { |v| v[:status] == "ERROR" }
   status =
-    if error then "ERROR"
-    elsif selected == exp && !must_not.include?(selected) then "PASS"
+    if observed.empty? then "ERROR"
+    elsif observed.all? { |v| v[:status] == "PASS" } then "PASS"
     else "FAIL"
     end
+  selections = observed.map { |v| v[:selected] }.uniq
+  # Failure-mode labels (vocabulary in references/eval-routing.md):
+  #   absorbed        — an utterance that should be rejected (expected "none")
+  #                     or kept away from named bait neighbors (must_not_route_to)
+  #                     was claimed by such a skill anyway
+  #   ownership_split — replicas disagreed on the top pick (unstable ownership)
+  labels = []
+  absorbed = (exp == "none" && observed.any? { |v| v[:selected] && v[:selected] != "none" }) ||
+             observed.any? { |v| must_not.include?(v[:selected]) }
+  labels << "absorbed" if absorbed
+  labels << "ownership_split" if selections.size > 1
+  # A task where some replicas erred but others graded is PARTIALLY measured:
+  # the consensus above sees only the observed verdicts, so without this label
+  # a PASS+ERROR pair would read as a clean PASS and the run as fully sampled.
+  labels << "partial_error" if verdicts.any? { |v| v[:status] == "ERROR" } && !observed.empty?
   results << {
-    id: t["id"], utterance: t["utterance"], expected: exp, selected: selected,
-    confidence: confidence, must_not_route_to: must_not, status: status,
-    error: error, frozen_at_sha_is_ancestor: frozen_ok
+    id: t["id"], utterance: t["utterance"], expected: exp, acceptable: acceptable,
+    selected: observed.first && observed.first[:selected],
+    confidence: observed.first && observed.first[:confidence],
+    clarify: observed.any? { |v| v[:clarify] },
+    acceptable_hit: observed.any? { |v| acceptable.include?(v[:selected]) },
+    must_not_route_to: must_not, status: status, labels: labels,
+    verdicts: verdicts, error: (verdicts.find { |v| v[:error] } || {})[:error],
+    frozen_at_sha_is_ancestor: frozen_ok
   }
 end
 
@@ -355,43 +458,114 @@ fails = results.select { |r| r[:status] == "FAIL" }
 errors = results.select { |r| r[:status] == "ERROR" }
 drift = results.reject { |r| r[:frozen_at_sha_is_ancestor] }
 
+# First-class routing-quality metrics beyond pass/fail (counted over observed,
+# non-ERROR verdicts): clarify rate, low-confidence rate, and — when replicas
+# >= 2 — how often all replicas of one task picked the same top skill.
+all_observed = results.flat_map { |r| r[:verdicts].reject { |v| v[:status] == "ERROR" } }
+clarify_count = all_observed.count { |v| v[:clarify] }
+low_conf_count = all_observed.count { |v| v[:confidence].is_a?(Numeric) && v[:confidence] < 0.5 }
+# Replica-level error accounting: task-level `error` counts only fully
+# unmeasured tasks, so a PASS+ERROR pair would otherwise report zero
+# grader-errors while a replica silently went missing.
+error_verdicts = results.sum { |r| r[:verdicts].count { |v| v[:status] == "ERROR" } }
+partial_error_ids = results.select { |r| r[:labels].include?("partial_error") }.map { |r| r[:id] }
+agreement_measured = 0
+agreement_agree = 0
+if replicas >= 2
+  results.each do |r|
+    obs = r[:verdicts].reject { |v| v[:status] == "ERROR" }
+    next if obs.size < 2
+    agreement_measured += 1
+    agreement_agree += 1 if obs.map { |v| v[:selected] }.uniq.size == 1
+  end
+end
+
 # baseline diff (newly failed / newly passed). The baseline is a prior report
-# (a hash with a "results" array) or a bare results array.
+# (a hash with a "results" array) or a bare results array. Reports from a
+# different bank content or replica count are DIFFERENT RULERS (documented
+# above): comparing them emits false regressions/improvements, so a
+# demonstrated mismatch suppresses the diff instead of computing it. A bare
+# results array carries no fingerprint — its comparability is unverifiable and
+# is flagged as such rather than silently trusted.
 newly_failed = []
 newly_passed = []
+baseline_comparable = nil
+baseline_incomparable_reason = nil
 if baseline_path && File.file?(baseline_path)
   base_json = JSON.parse(File.read(baseline_path))
-  base_results = base_json.is_a?(Hash) ? (base_json["results"] || []) : base_json
-  base_status = base_results.to_h { |r| [r["id"], r["status"]] }
-  results.each do |r|
-    was = base_status[r[:id]]
-    newly_failed << r[:id] if was == "PASS" && r[:status] == "FAIL"
-    newly_passed << r[:id] if was == "FAIL" && r[:status] == "PASS"
+  if base_json.is_a?(Hash)
+    base_bank_sha = base_json.dig("routing_surface", "bank_sha256")
+    base_replicas = base_json["replicas"] || 1 # legacy reports predate --replicas
+    if base_bank_sha && base_bank_sha != routing_surface[:bank_sha256]
+      baseline_comparable = false
+      baseline_incomparable_reason = "bank content differs (baseline #{base_bank_sha[0, 12]}… vs current #{routing_surface[:bank_sha256][0, 12]}…)"
+    elsif base_replicas != replicas
+      baseline_comparable = false
+      baseline_incomparable_reason = "replica count differs (baseline #{base_replicas} vs current #{replicas})"
+    else
+      baseline_comparable = base_bank_sha ? true : "unverified (baseline carries no bank fingerprint)"
+    end
+  else
+    baseline_comparable = "unverified (bare results array carries no fingerprint)"
+  end
+  unless baseline_comparable == false
+    base_results = base_json.is_a?(Hash) ? (base_json["results"] || []) : base_json
+    base_status = base_results.to_h { |r| [r["id"], r["status"]] }
+    results.each do |r|
+      was = base_status[r[:id]]
+      newly_failed << r[:id] if was == "PASS" && r[:status] == "FAIL"
+      newly_passed << r[:id] if was == "FAIL" && r[:status] == "PASS"
+    end
   end
 end
 
 report = {
   model: model, tasks: results.size, pass: passes, fail: fails.size, error: errors.size,
+  replicas: replicas, verdicts: all_observed.size,
+  error_verdicts: error_verdicts, partial_error_tasks: partial_error_ids,
+  clarify_count: clarify_count, low_confidence_count: low_conf_count,
+  replica_agreement: (replicas >= 2 ? { agree: agreement_agree, measured: agreement_measured } : nil),
   desc_budget_chars: desc_budget, routing_surface: routing_surface,
   co_change_bank_and_descriptions: co_change, co_change_check_available: co_change_check_ok,
   frozen_drift: drift.map { |r| r[:id] },
+  baseline_comparable: baseline_comparable,
+  baseline_incomparable_reason: baseline_incomparable_reason,
   newly_failed: newly_failed, newly_passed: newly_passed, results: results
 }
 File.write(json_path, JSON.pretty_generate(report)) if json_path
 
 puts "eval-routing-bank (#{model}): #{passes}/#{results.size} pass, #{fails.size} fail, #{errors.size} grader-error"
 puts "  arm: desc-budget-chars=#{desc_budget}" if desc_budget
+unless all_observed.empty?
+  line = "  clarify: #{clarify_count}/#{all_observed.size} verdicts, low-confidence(<0.5): #{low_conf_count}/#{all_observed.size}"
+  line += ", replica top1 agreement: #{agreement_agree}/#{agreement_measured}" if replicas >= 2
+  puts line
+end
+if error_verdicts.positive?
+  puts "  ⚠ grader-error verdicts: #{error_verdicts}#{partial_error_ids.empty? ? '' : " (partially measured tasks: #{partial_error_ids.join(', ')})"}"
+end
 puts "  ⚠ co-change check unavailable: base ref #{base.inspect} not resolvable — could not verify bank/description co-change" unless co_change_check_ok
 puts "  ⚠ co-change: this change touches BOTH the bank and a SKILL.md description (verify the bank was not edited to pass)" if co_change
 puts "  ⚠ frozen-drift (not ancestor of HEAD, excluded from regression judgment): #{drift.map { |r| r[:id] }.join(', ')}" unless drift.empty?
 unless fails.empty?
   puts "  FAIL:"
-  fails.each { |r| puts "    - #{r[:id]}: expected #{r[:expected]} got #{r[:selected].inspect} (conf #{r[:confidence]})" }
+  fails.each do |r|
+    tag = r[:labels].empty? ? "" : " [#{r[:labels].join(',')}]"
+    obs = r[:verdicts].reject { |v| v[:status] == "ERROR" }
+    got = obs.map { |v| v[:selected].inspect }.uniq.join(" | ")
+    conf = obs.map { |v| v[:confidence] }.join(",")
+    puts "    - #{r[:id]}: expected #{r[:expected]} got #{got} (conf #{conf})#{tag}"
+  end
 end
 unless errors.empty?
   puts "  GRADER-ERROR:"
   errors.each { |r| puts "    - #{r[:id]}: #{r[:error]}" }
 end
+if baseline_comparable == false
+  puts "  ⚠ baseline not compared — different ruler: #{baseline_incomparable_reason}"
+elsif baseline_comparable.is_a?(String)
+  puts "  ⚠ baseline comparability #{baseline_comparable}"
+end
 unless newly_failed.empty? && newly_passed.empty?
   puts "  vs baseline: newly_failed=#{newly_failed.join(',')} newly_passed=#{newly_passed.join(',')}"
 end

package/dist/assets/marketplace/plugins/ccl-skills/skills/skill-extraction-workflow/scripts/extraction_review_gate.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Extraction-owned autonomous review wrapper: one review plus two challenges.
+# Extraction-owned autonomous review wrapper: one review plus one challenge.
 set -euo pipefail
 
 SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
@@ -8,7 +8,7 @@ CONTROLLER="$SCRIPT_DIR/../../code-review/scripts/review_gate.sh"
 for arg in "$@"; do
   case "$arg" in
     --challenge-b*)
-      echo "extraction_review_gate_error: challenge budget is fixed at 2" >&2
+      echo "extraction_review_gate_error: challenge budget is fixed at 1" >&2
       exit 2
       ;;
   esac
@@ -19,4 +19,4 @@ if [[ ! -x "$CONTROLLER" ]]; then
   exit 2
 fi
 
-exec bash "$CONTROLLER" --challenge-budget 2 "$@"
+exec bash "$CONTROLLER" --challenge-budget 1 "$@"