@ccoalm/ccl-skills 0.13.0 → 0.15.0

package/dist/assets/marketplace/plugins/ccl-skills/skills/skill-extraction-workflow/scripts/eval-routing-bank.rb

@@ -42,6 +42,14 @@ require "shellwords"
 require "timeout"
 require "digest"
 
+# Resolution floor for ACTING on a measured case. A report taken below it
+# LOCATES candidates; it does not license a description edit, because at three
+# replicas a case that routes correctly 80-90% of the time reads as failing and
+# a one-off deviation reads as a finding. `references/eval-routing.md` owns the
+# rule and states the same number; test_eval_routing_bank_resolution.sh pins the
+# two sides together so they cannot drift apart.
+ACTION_RESOLUTION_MIN_REPLICAS = 10
+
 def arg(flag, default = nil)
   i = ARGV.index(flag)
   i ? ARGV[i + 1] : default
@@ -183,15 +191,25 @@ desc_changed = changed_files.any? { |f| f.end_with?("/SKILL.md") }
 co_change = bank_changed && desc_changed
 
 # --- build skill routing surface (the same descriptions the agent routes on) -
-catalog = Dir[File.join(root, "skills", "*", "SKILL.md")].sort.map do |path|
+# One filtered list feeds BOTH the prompt and the set of selectable names, so a
+# skill the prompt never offered (no frontmatter, empty description) cannot be a
+# valid selection: review noted the two were built by separate transformations
+# whose filtering could drift apart, and a name allowed but never shown is exactly
+# the shape that drift would let a verdict claim.
+# map + compact rather than filter_map: the runner has to work on the oldest
+# Ruby a host ships (macOS system Ruby is 2.6), and filter_map is 2.7+.
+catalog_entries = Dir[File.join(root, "skills", "*", "SKILL.md")].sort.map do |path|
   name = File.basename(File.dirname(path))
   m = File.read(path).match(/\A---\s*\n(.*?)\n---\s*\n/m)
   next unless m
   desc = (YAML.safe_load(m[1]) rescue {})["description"].to_s.strip
   next if desc.empty?
   desc = desc[0, desc_budget] if desc_budget && desc.length > desc_budget
-  "### #{name}\n#{desc}"
-end.compact.join("\n\n")
+  [name, "### #{name}\n#{desc}"]
+end.compact
+catalog = catalog_entries.map(&:last).join("\n\n")
+# The set of names a verdict may legitimately select; "none" is accepted separately.
+catalog_names = catalog_entries.map(&:first)
 
 # --- optional always-on entry-routing layer -----------------------------------
 # The catalog above is the description-only surface. Hosts ALSO inject
@@ -401,6 +419,16 @@ tasks.each do |t|
       break
     end
     selected = parsed && parsed["selected_skill"]
+    # A parseable answer is not yet a usable observation. The prompt's contract is
+    # an exact catalog name or "none"; anything else -- a missing key, a name the
+    # catalog does not carry, a non-string -- is grader output that says nothing
+    # about routing, and counting it as a verdict would let it feed the per-case
+    # resolution floor exactly as a well-formed one does. It is an ERROR, not a
+    # FAIL: a FAIL is evidence against the route, this is absence of evidence.
+    if parsed && error.nil? && !(selected == "none" || catalog_names.include?(selected))
+      error = "invalid_selection: #{selected.inspect[0, 80]}"
+      parsed = nil
+    end
     clarify = parsed && parsed["clarify"] == true
     confidence = parsed && parsed["confidence"]
     v_status =
@@ -519,6 +547,30 @@ if baseline_path && File.file?(baseline_path)
   end
 end
 
+# The floor is on VALID observations, not on the requested replica count: a run
+# asked for ten replicas can come back with seven usable verdicts once a grader
+# times out or returns unparsable output, and a report that called itself
+# actionable on the request alone would license an edit the evidence cannot
+# support. Observed in this repository: a fourteen-task run at --replicas 10
+# returned six grader errors and left two tasks at seven valid observations.
+#
+# The verdict is PER CASE, because an edit is licensed per case. A report-wide
+# flag alone is unsound in the other direction: a subset run over case A can be
+# actionable while saying nothing about case B, and a consumer reading only the
+# top-level boolean would take it as licence for an edit to B. Each result
+# therefore carries its own `actionable`, and the report-level field is the
+# conjunction over the cases the run actually measured -- true only when every
+# measured case clears the floor, and never a statement about a case absent from
+# `results`.
+results.each do |r|
+  valid = r[:verdicts].count { |v| v[:status] != "ERROR" }
+  r[:valid_observations] = valid
+  r[:actionable] = replicas >= ACTION_RESOLUTION_MIN_REPLICAS &&
+    valid >= ACTION_RESOLUTION_MIN_REPLICAS
+end
+min_valid_observations = results.map { |r| r[:valid_observations] }.min.to_i
+action_resolution = !results.empty? && results.all? { |r| r[:actionable] }
+
 report = {
   model: model, tasks: results.size, pass: passes, fail: fails.size, error: errors.size,
   replicas: replicas, verdicts: all_observed.size,
@@ -526,6 +578,10 @@ report = {
   clarify_count: clarify_count, low_confidence_count: low_conf_count,
   replica_agreement: (replicas >= 2 ? { agree: agreement_agree, measured: agreement_measured } : nil),
   desc_budget_chars: desc_budget, routing_surface: routing_surface,
+  action_resolution: action_resolution,
+  action_resolution_scope: "cases measured by this run only; see each result's actionable field",
+  action_resolution_min_replicas: ACTION_RESOLUTION_MIN_REPLICAS,
+  min_valid_observations: min_valid_observations,
   co_change_bank_and_descriptions: co_change, co_change_check_available: co_change_check_ok,
   frozen_drift: drift.map { |r| r[:id] },
   baseline_comparable: baseline_comparable,
@@ -535,6 +591,9 @@ report = {
 File.write(json_path, JSON.pretty_generate(report)) if json_path
 
 puts "eval-routing-bank (#{model}): #{passes}/#{results.size} pass, #{fails.size} fail, #{errors.size} grader-error"
+unless action_resolution
+  puts "  \u26a0 screening_resolution_only: replicas=#{replicas}, weakest task has #{min_valid_observations} valid observations, floor #{ACTION_RESOLUTION_MIN_REPLICAS} — this report locates candidates, it does not license a description edit; a per-case edit needs #{ACTION_RESOLUTION_MIN_REPLICAS} valid observations of that case (references/eval-routing.md)"
+end
 puts "  arm: desc-budget-chars=#{desc_budget}" if desc_budget
 unless all_observed.empty?
   line = "  clarify: #{clarify_count}/#{all_observed.size} verdicts, low-confidence(<0.5): #{low_conf_count}/#{all_observed.size}"

package/dist/assets/marketplace/plugins/ccl-skills/skills/skill-extraction-workflow/scripts/impact-chain-gate.rb

@@ -52,19 +52,123 @@ LEDGER_PATH = "skills/skill-extraction-workflow/references/source-register.md"
 # the routing-surface class below was one such patch) only re-instantiates it on
 # the next input. The predicate now reads the round.
 #
-# Rounds are cut at the commits that touch the ledger itself, walked first-parent
-# so one merged worktree round collapses to one boundary. The partition comes from
-# git alone: an author cannot widen, move, or nominate their own scope.
-round_heads = git_read.call("rev-list", "--first-parent", "--reverse", "#{base_ref}..HEAD", "--", LEDGER_PATH)
-                      .split("\n").map(&:strip).reject(&:empty?)
-# Each round spans (previous ledger boundary, this one] so the work commits that
+# Rounds are cut at the commits that touch the ledger itself, walked along a
+# first-parent line. The partition comes from git alone: an author cannot widen,
+# move, or nominate their own scope.
+#
+# THE SAME PARTITION BEFORE AND AFTER THE MERGE. A branch is judged on its own
+# first-parent line while it is a pull request (CI checks out the branch head),
+# and once merged that whole line sits behind ONE first-parent step of the
+# integration branch. Reading that step as one boundary gave the same history a
+# different partition after it landed — every round on the branch collapsed into
+# one, and a row whose validity depends on its round being narrow (a routing-
+# surface `#description` anchor in a commit that changed nothing else) turned red
+# without a byte of it changing. That is the "verdict moved after it landed"
+# defect in a new coat, and it surfaced on every post-merge evaluation: the push
+# build of the integration branch and the promotion pull request. So a merge that
+# git itself reproduces from its two parents is EXPANDED in place: its second
+# parent's line, from the fork point to the merged head, is walked with the same
+# rule, recursively, and contributes exactly the rounds it had as a branch.
+#
+# A merge is expanded only when git can rebuild it — two parents, a tree equal to
+# `git merge-tree --write-tree` of those parents, and a second parent that is
+# neither already on the base nor already on the line being walked (a sync merge
+# brings nothing that needs a round). Anything else — a hand-resolved merge, a
+# conflicted one, an octopus — keeps today's single boundary at the merge, so
+# content git did not derive from the parents is never left in no round.
+ROUND_WALK_MAX_DEPTH = 8
+ancestor_of = lambda do |commit, tip|
+  IO.popen(["git", "-C", root, "merge-base", "--is-ancestor", commit, tip], err: File::NULL, &:read)
+  status = $?.exitstatus
+  next true if status == 0
+  next false if status == 1
+  warn "impact_chain_git_failed: git merge-base --is-ancestor #{commit} #{tip} exited #{status}"
+  exit 1
+end
+# The tree git produces merging `second` into `first`; nil when that merge
+# conflicts (whoever resolved it was not git). Needs git 2.38+, the same floor
+# the review-ledger binder already requires for the identical invariant.
+automatic_merge_tree = lambda do |first, second|
+  out = IO.popen(["git", "-C", root, "merge-tree", "--write-tree", first, second], err: File::NULL, &:read)
+  status = $?.exitstatus
+  next out.to_s.lines.first.to_s.strip if status == 0
+  next nil if status == 1
+  warn "impact_chain_git_failed: git merge-tree --write-tree #{first} #{second} exited #{status} (git 2.38 or newer is required)"
+  exit 1
+end
+# [first parent, second parent, fork point] when `commit` is a merge git can
+# rebuild from its parents and whose second parent carries a line of its own;
+# nil when the merge keeps today's single-boundary treatment. `line_base` is the
+# base of the line being walked: a merge whose second parent is already below
+# that base is a sync of what the line was cut from (the target advancing under
+# a branch), and is a sync on the branch's own line exactly as it is on the
+# integration line — judging it against the outer base alone would expand it
+# during promotion and strand the branch's earlier work in a rowless span.
+expandable_merge = lambda do |commit, line_base|
+  parents = git_read.call("rev-list", "--parents", "-n", "1", commit).split[1..] || []
+  next nil unless parents.length == 2
+  first, second = parents
+  next nil if ancestor_of.call(second, base_ref) || ancestor_of.call(second, line_base) || ancestor_of.call(second, first)
+  own_tree = git_read.call("rev-parse", "#{commit}^{tree}").strip
+  next nil unless automatic_merge_tree.call(first, second) == own_tree
+  # The fork point read FAILS CLOSED like every other git read here: a lookup
+  # that errored would otherwise read as "no fork point", skip the expansion,
+  # and hand the merge the collapsed span — the lenient verdict — on a git
+  # failure nobody sees. Exit 1 is git's own "no common ancestor" and means
+  # there is genuinely no line to expand from.
+  fork_out = IO.popen(["git", "-C", root, "merge-base", first, second], err: File::NULL, &:read)
+  fork_status = $?.exitstatus
+  next nil if fork_status == 1
+  unless fork_status == 0
+    warn "impact_chain_git_failed: git merge-base #{first} #{second} exited #{fork_status}"
+    exit 1
+  end
+  fork = fork_out.to_s.split("\n").first.to_s.strip
+  next nil if fork.empty?
+  [first, second, fork]
+end
+# Each round spans (previous boundary, this one] so the work commits that
 # precede a ledger append are inside the round they belong to — landing the change
 # and appending the row in separate commits is the normal shape, not an evasion.
-round_bounds = ([base_ref] + round_heads).each_cons(2).to_a
-# The trailing span — owner changes committed after the last ledger append — is a
+# The trailing span — owner changes committed after the last boundary — is a
 # round too. It holds no rows, so its owners fall through to the presence check
-# and the gate still fails closed on undeclared work.
-round_bounds << [round_heads.last || base_ref, "HEAD"]
+# and the gate still fails closed on undeclared work. Both hold on every line the
+# walk visits, the integration branch and each expanded merge alike.
+round_bounds_for = lambda do |from, to, depth|
+  if depth > ROUND_WALK_MAX_DEPTH
+    warn "impact_chain_round_walk_too_deep: merges nested more than #{ROUND_WALK_MAX_DEPTH} levels between #{from} and #{to}"
+    exit 1
+  end
+  list = lambda do |*options, pathspec|
+    git_read.call("rev-list", "--first-parent", "--reverse", *options, "#{from}..#{to}", *pathspec)
+            .split("\n").map(&:strip).reject(&:empty?)
+  end
+  line = list.call([])
+  ledger_heads = list.call(["--", LEDGER_PATH])
+  merges = list.call("--merges", [])
+  spans = []
+  prev = from
+  line.each do |commit|
+    expansion = merges.include?(commit) ? expandable_merge.call(commit, from) : nil
+    if expansion
+      first, second, fork = expansion
+      spans << [prev, first] unless prev == first
+      spans.concat(round_bounds_for.call(fork, second, depth + 1))
+      prev = commit
+    elsif ledger_heads.include?(commit)
+      spans << [prev, commit]
+      prev = commit
+    end
+  end
+  spans << [prev, to]
+  spans
+end
+round_bounds = round_bounds_for.call(base_ref, "HEAD", 0)
+# Diagnostic only: print the partition so a verdict can be read against the
+# rounds it was judged in. Off by default so no suite's output assertions move.
+if ENV["CCL_IMPACT_CHAIN_TRACE_ROUNDS"] == "1"
+  round_bounds.each { |span_base, span_head| warn "impact_chain_round: #{span_base[0, 12]}..#{span_head[0, 12]}" }
+end
 # Everything a predicate needs to judge one span. Built lazily per span and
 # memoized: a round whose rows are all RED-baseline never pays for the rename
 # derivation.

package/dist/assets/marketplace/plugins/ccl-skills/skills/skill-extraction-workflow/scripts/reference-access-census.sh

@@ -0,0 +1,157 @@
+#!/usr/bin/env bash
+# Reference-access census: which of a skill package's files were actually
+# touched by real agent sessions on THIS host, over a lookback window.
+#
+# Why this exists: a rule set must not grow monotonically, but "retire what is
+# not pulling its weight" needs a signal that is not the author's opinion. The
+# closest observable proxy on a local host is the per-file access record in the
+# agent's own session transcripts (Claude Code `~/.claude/projects/**/*.jsonl`,
+# Codex `~/.codex/sessions/**/*.jsonl`): a reference that no session mentioned
+# in N days is a relocation/retirement candidate; one mentioned in most sessions
+# that loaded the skill is a candidate for promotion into the entrypoint. This
+# is the local instantiation of the usage counters that context-evolution
+# methods keep per bullet (helpful/harmful counts) and of the "ignored content
+# is unnecessary or poorly signaled" observation in the official skill-authoring
+# guidance — advisory only, never a gate (Goodhart: a count that becomes a
+# target gets gamed by mentioning files).
+#
+# Privacy contract: the transcripts are private per-host data. This script
+# prints ONLY repo-relative skill file paths, per-file session counts, and
+# last-touched dates. It never prints transcript text, prompts, absolute paths
+# outside the skill tree, or session ids. Its output is safe to paste into a
+# private charter; it is still not shared-tree content by itself.
+#
+# Counting unit: a SESSION (one transcript file) counts once per skill file it
+# mentions, whatever the tool (Read, sed, grep, Skill load). "Mentioned" is a
+# superset of "read to depth" — treat a count as an upper bound on real use.
+#
+# Usage:
+#   reference-access-census.sh [--skill <name>] [--days <n>] [--repo-root <dir>]
+#                              [--logs <dir>[,<dir>...]]
+# Defaults: skill=skill-extraction-workflow, days=60, repo-root=cwd-derived,
+#           logs=$HOME/.claude/projects,$HOME/.codex/sessions
+# Exit 0 on a completed census or an honest unevaluated result (no transcripts);
+# exit 2 on usage errors and on input errors (unreadable/vanished/unexecutable
+# inputs) — counts are withheld rather than printed as zeros.
+set -euo pipefail
+
+skill="skill-extraction-workflow"
+days=60
+repo_root=""
+logs="${HOME:+${HOME}/.claude/projects,${HOME}/.codex/sessions}"
+logs_explicit=0
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --skill|--days|--repo-root|--logs)
+      if [ $# -lt 2 ]; then echo "reference_access_census_usage_error: $1 needs a value" >&2; exit 2; fi ;;
+  esac
+  case "$1" in
+    --skill) skill="$2"; shift 2 ;;
+    --days) days="$2"; shift 2 ;;
+    --repo-root) repo_root="$2"; shift 2 ;;
+    --logs) logs="$2"; logs_explicit=1; shift 2 ;;
+    -h|--help) sed -n '2,32p' "$0"; exit 0 ;;
+    *) echo "reference_access_census_usage_error: unknown argument (see --help)" >&2; exit 2 ;;
+  esac
+done
+if [ -z "$repo_root" ]; then
+  repo_root="$(git rev-parse --show-toplevel 2>/dev/null || pwd)"
+fi
+skill_dir="$repo_root/skills/$skill"
+if [ ! -d "$skill_dir" ]; then
+  echo "reference_access_census_usage_error: --skill names no tracked skill package under the repo root" >&2; exit 2
+fi
+case "$days" in ''|*[!0-9]*) echo "reference_access_census_usage_error: --days must be an integer" >&2; exit 2 ;; esac
+
+# Inventory: every tracked markdown file in the package (entrypoint + references).
+files=()
+while IFS= read -r f; do files+=("$f"); done < <(cd "$repo_root" && git ls-files "skills/$skill/SKILL.md" "skills/$skill/references/*.md" 2>/dev/null | sort)
+if [ "${#files[@]}" -eq 0 ]; then
+  echo "reference_access_census_usage_error: --skill names a package with no tracked SKILL.md or references" >&2; exit 2
+fi
+
+# Candidate transcripts: any .jsonl under the log roots modified within the window.
+tmp="$(mktemp -d)"; trap 'rm -rf "$tmp"' EXIT
+roots=(); if [ -n "$logs" ]; then IFS=',' read -r -a roots <<<"$logs"; fi
+: > "$tmp/candidates0"; : > "$tmp/errors"
+# Input errors are never zeros: every scan phase collects stderr, and a non-empty
+# error log withholds the table (exit 2) instead of printing counts that would
+# read as evidence. grep's no-match status writes nothing to stderr, so it never
+# trips this; unreadable, vanished, or unexecutable inputs do. The error log is
+# summarized as a count only — file names would be private paths.
+withhold_if_errors() {
+  if [ -s "$tmp/errors" ]; then
+    n="$(wc -l < "$tmp/errors" | tr -d ' ')"
+    echo "reference_access_census_unevaluated: $n input error(s) while scanning transcripts (unreadable, vanished, or unexecutable inputs); counts withheld"
+    exit 2
+  fi
+}
+# A default root that is absent is normal (the host may run only one agent); an
+# explicitly supplied root that is absent is an input error — the caller asked
+# for a scan that cannot happen, and skipping it would read as a real count.
+for r in "${roots[@]}"; do
+  [ -n "$r" ] || continue
+  if [ ! -d "$r" ]; then
+    if [ "$logs_explicit" -eq 1 ]; then echo "missing explicit log root" >> "$tmp/errors"; fi
+    continue
+  fi
+  find "$r" -type f -name '*.jsonl' -mtime "-$days" -print0 >> "$tmp/candidates0" 2>> "$tmp/errors" || echo "transcript listing failed under one log root" >> "$tmp/errors"
+done
+withhold_if_errors
+# The inventory is NUL-delimited (a pathname may contain a newline) and
+# deduplicated by pathname: overlapping roots (a root and its own subtree, or
+# the same root twice) list a transcript more than once, and a session must
+# count once. Two names for one file (symlinked roots) are not canonicalized.
+sort -zu "$tmp/candidates0" -o "$tmp/candidates0"
+total_sessions="$(tr -cd '\0' < "$tmp/candidates0" | wc -c | tr -d ' ')"
+if [ "$total_sessions" -eq 0 ]; then
+  echo "reference_access_census_unevaluated: no transcripts within ${days}d under ${#roots[@]} log root(s); counts withheld"
+  exit 0
+fi
+
+# Sessions that mention the package at all (fixed-string prefilter keeps this fast).
+# xargs batches keep this under ARG_MAX; grep exit 1 (no match in a batch) is not an error.
+# grep exit 1 (no match in a batch) is success; any other non-zero status —
+# with or without a stderr line — is an input error, so each batch runs under a
+# small wrapper that normalizes 1 to 0 and lets xargs report anything else.
+# `|| rc=$?` keeps the batch alive when the caller exported SHELLOPTS=errexit:
+# a bare `grep; rc=$?` would exit the child at status 1 before normalizing it.
+grep_batch() { local rc=0; grep -lF --null "$@" || rc=$?; [ "$rc" -le 1 ] && return 0; return "$rc"; }
+export -f grep_batch 2>/dev/null || true
+if ! xargs -0 -n 200 bash -c 'grep_batch "$@"' _ "skills/$skill/" < "$tmp/candidates0" > "$tmp/touching0" 2>> "$tmp/errors"; then
+  echo "transcript scan failed in at least one batch" >> "$tmp/errors"
+fi
+withhold_if_errors
+touching="$(tr -cd '\0' < "$tmp/touching0" | wc -c | tr -d ' ')"
+
+# Per-file session count + last-touched date (mtime of the newest touching transcript).
+: > "$tmp/rows"
+for f in "${files[@]}"; do
+  rel="${f#skills/$skill/}"
+  if [ "$touching" -eq 0 ]; then n=0; last="-"; else
+    if ! xargs -0 -n 200 bash -c 'grep_batch "$@"' _ "$f" < "$tmp/touching0" > "$tmp/hits0" 2>> "$tmp/errors"; then
+      echo "transcript scan failed in at least one batch" >> "$tmp/errors"
+    fi
+    withhold_if_errors
+    n="$(tr -cd '\0' < "$tmp/hits0" | wc -c | tr -d ' ')"
+    if [ "$n" -gt 0 ]; then
+      # BSD stat first, GNU stat as the fallback; only the fallback's failure is an
+      # error. Every substitution is guarded so a failing pipeline cannot abort the
+      # script under set -e before withhold_if_errors runs.
+      last="$(xargs -0 stat -f '%Sm' -t '%Y-%m-%d' < "$tmp/hits0" 2>/dev/null | sort | tail -1)" || last=""
+      if [ -z "$last" ]; then
+        last="$(xargs -0 stat -c '%y' < "$tmp/hits0" 2>> "$tmp/errors" | cut -c1-10 | sort | tail -1)" || last=""
+      fi
+      if [ -z "$last" ]; then
+        [ -s "$tmp/errors" ] || echo "stat produced no timestamp for a touching transcript" >> "$tmp/errors"
+        withhold_if_errors
+      fi
+    else last="-"; fi
+  fi
+  if [ "$touching" -gt 0 ]; then share="$(( n * 100 / touching ))%"; else share="-"; fi
+  printf '%s | %s | %s | %s\n' "$rel" "$n" "$last" "$share" >> "$tmp/rows"
+done
+printf '%s\n' "reference_access_census: skill=$skill window=${days}d transcripts=$total_sessions sessions_touching_package=$touching"
+printf '%s\n' "file | sessions | last_touched | share_of_touching"
+sort -t'|' -k2,2nr "$tmp/rows"
+echo "reference_access_census_ok"