@ccoalm/ccl-skills 0.12.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/marketplace/plugins/ccl-skills/skills/defect-diagnosis/SKILL.md +21 -12
- package/dist/assets/marketplace/plugins/ccl-skills/skills/defect-diagnosis/references/diagnosis-playbook.md +42 -1
- package/dist/assets/marketplace/plugins/ccl-skills/skills/llm-inference-integration/SKILL.md +7 -4
- package/dist/assets/marketplace/plugins/ccl-skills/skills/llm-inference-integration/references/agent-context-freshness.md +4 -0
- package/dist/assets/marketplace/plugins/ccl-skills/skills/llm-inference-integration/references/agent-tool-dispatch.md +2 -0
- package/dist/assets/marketplace/plugins/ccl-skills/skills/llm-inference-integration/references/inference-capacity-operations.md +20 -1
- package/dist/assets/marketplace/plugins/ccl-skills/skills/llm-inference-integration/references/llm-client-gateway.md +11 -0
- package/dist/assets/marketplace/plugins/ccl-skills/skills/llm-inference-integration/references/model-prompt-evaluation.md +3 -2
- package/dist/assets/marketplace/plugins/ccl-skills/skills/llm-inference-integration/references/retrieval-agent-safety.md +5 -0
- package/dist/assets/marketplace/plugins/ccl-skills/skills/multi-agent-delegation/SKILL.md +13 -9
- package/dist/assets/marketplace/plugins/ccl-skills/skills/multi-agent-delegation/references/multi-agent-delegation-playbook.md +12 -1
- package/dist/assets/marketplace/plugins/ccl-skills/skills/skill-extraction-workflow/SKILL.md +14 -41
- package/dist/assets/marketplace/plugins/ccl-skills/skills/skill-extraction-workflow/references/attention-budget-ratchet.md +11 -0
- package/dist/assets/marketplace/plugins/ccl-skills/skills/skill-extraction-workflow/references/correction-routing-map.md +22 -0
- package/dist/assets/marketplace/plugins/ccl-skills/skills/skill-extraction-workflow/references/coverage-exhaustion-traps.md +7 -0
- package/dist/assets/marketplace/plugins/ccl-skills/skills/skill-extraction-workflow/references/description-authoring.md +26 -0
- package/dist/assets/marketplace/plugins/ccl-skills/skills/skill-extraction-workflow/references/dual-track-review-gate.md +2 -2
- package/dist/assets/marketplace/plugins/ccl-skills/skills/skill-extraction-workflow/references/eval-routing.md +2 -0
- package/dist/assets/marketplace/plugins/ccl-skills/skills/skill-extraction-workflow/references/external-practice-controls.md +6 -0
- package/dist/assets/marketplace/plugins/ccl-skills/skills/skill-extraction-workflow/references/extraction-quickstart.md +4 -2
- package/dist/assets/marketplace/plugins/ccl-skills/skills/skill-extraction-workflow/references/harness-patterns-and-eval.md +22 -0
- package/dist/assets/marketplace/plugins/ccl-skills/skills/skill-extraction-workflow/references/incident-postmortem-extraction.md +8 -0
- package/dist/assets/marketplace/plugins/ccl-skills/skills/skill-extraction-workflow/references/rule-consolidation.md +1 -0
- package/dist/assets/marketplace/plugins/ccl-skills/skills/skill-extraction-workflow/references/source-register.md +70 -0
- package/dist/assets/marketplace/plugins/ccl-skills/skills/skill-extraction-workflow/references/uiux-judgment-extraction.md +11 -0
- package/dist/assets/marketplace/plugins/ccl-skills/skills/skill-extraction-workflow/references/validation-and-landing.md +11 -0
- package/dist/assets/marketplace/plugins/ccl-skills/skills/skill-extraction-workflow/scripts/entrypoint_form_census.py +169 -0
- package/dist/assets/marketplace/plugins/ccl-skills/skills/skill-extraction-workflow/scripts/reference-access-census.sh +157 -0
- package/dist/assets/marketplace/plugins/ccl-skills/skills/skill-extraction-workflow/scripts/review_ledger_binding.py +454 -103
- package/dist/assets/marketplace/plugins/ccl-skills/skills/skill-extraction-workflow/scripts/test_check_ccl_regressions.sh +8 -0
- package/dist/assets/marketplace/plugins/ccl-skills/skills/skill-extraction-workflow/scripts/test_entrypoint_form_census.sh +174 -0
- package/dist/assets/marketplace/plugins/ccl-skills/skills/skill-extraction-workflow/scripts/test_reference_access_census.sh +209 -0
- package/dist/assets/marketplace/plugins/ccl-skills/skills/skill-extraction-workflow/scripts/test_review_ledger_binding.sh +331 -0
- package/dist/assets/release.json +56 -31
- package/package.json +1 -1
|
@@ -31,6 +31,7 @@
|
|
|
31
31
|
# - test_uiux_delivery_contract.sh
|
|
32
32
|
# - test_uiux_loading_budget.sh
|
|
33
33
|
# - test_obligation_ledger.sh
|
|
34
|
+
# - test_reference_access_census.sh
|
|
34
35
|
# --full runs --fast plus the heavy full-checker regressions:
|
|
35
36
|
# - test_check_ccl_r0_status.sh
|
|
36
37
|
# - test_check_ccl_source_register_lifecycle.sh
|
|
@@ -107,6 +108,13 @@ fast_tests=(
|
|
|
107
108
|
# applied deletion mutation in a throwaway copy (spec 031 review disposition).
|
|
108
109
|
test_controlled_escalation_pins.sh
|
|
109
110
|
test_check_ccl_size_budget.sh
|
|
111
|
+
# Entrypoint guidance-form census (what counts as one rule, a prohibition, a
|
|
112
|
+
# named baseline failure): synthetic fixtures plus the shipped entrypoint,
|
|
113
|
+
# seconds. Pins the definitions, never any round's figures.
|
|
114
|
+
test_entrypoint_form_census.sh
|
|
115
|
+
# Usage census over synthetic transcripts (counts, privacy contract,
|
|
116
|
+
# unevaluated sentinel, ARG_MAX batching): no host logs, seconds.
|
|
117
|
+
test_reference_access_census.sh
|
|
110
118
|
test_check_ccl_skill_catalog.sh
|
|
111
119
|
test_check_ccl_parallel_stack_parity.sh
|
|
112
120
|
test_check_mr_target_freshness.sh
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# Regression for the entrypoint form census. The census is an instrument whose
|
|
3
|
+
# only value is that a later round recomputes the same quantity, so the cases
|
|
4
|
+
# below pin the definitions the script documents -- what counts as one rule,
|
|
5
|
+
# what counts as a prohibition, and what counts as a named baseline failure --
|
|
6
|
+
# rather than the figures of any particular entrypoint, which move every round.
|
|
7
|
+
set -uo pipefail
|
|
8
|
+
|
|
9
|
+
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
|
|
10
|
+
CENSUS="$DIR/entrypoint_form_census.py"
|
|
11
|
+
WORK="$(mktemp -d "${TMPDIR:-/tmp}/entrypoint-form-census.XXXXXX")"
|
|
12
|
+
trap 'rm -rf "$WORK"' EXIT
|
|
13
|
+
|
|
14
|
+
fails=0
|
|
15
|
+
check() { if eval "$2"; then echo "ok - $1"; else echo "FAIL - $1"; fails=$((fails+1)); fi; }
|
|
16
|
+
|
|
17
|
+
[ -f "$CENSUS" ] || { echo "FAIL: census missing: $CENSUS" >&2; exit 1; }
|
|
18
|
+
|
|
19
|
+
field() { printf '%s\n' "$1" | sed -n "s/^$2=//p"; }
|
|
20
|
+
|
|
21
|
+
# One rule per top-level bullet: the sub-bullet and the continuation line belong
|
|
22
|
+
# to the rule above them, and the blank line between them does not close it.
|
|
23
|
+
# Getting this wrong inflates every per-rule figure the census reports.
|
|
24
|
+
# The prohibitions are placed in the continuation and the sub-bullet, never in
|
|
25
|
+
# the bullet's own first line: a rule COUNT cannot detect a parser that drops
|
|
26
|
+
# continuations, because a dropped continuation is still not a new top-level
|
|
27
|
+
# bullet. The token total is the quantity that moves when the defect is present,
|
|
28
|
+
# so that is what the case asserts.
|
|
29
|
+
cat >"$WORK/one-rule.md" <<'MD'
|
|
30
|
+
# Entry
|
|
31
|
+
|
|
32
|
+
## Core Rules
|
|
33
|
+
|
|
34
|
+
### A group
|
|
35
|
+
|
|
36
|
+
- A rule head with no imperative of its own.
|
|
37
|
+
|
|
38
|
+
A continuation line that must hold.
|
|
39
|
+
|
|
40
|
+
- A sub-bullet that must also hold.
|
|
41
|
+
|
|
42
|
+
## After
|
|
43
|
+
|
|
44
|
+
- A bullet outside Core Rules that must not be counted.
|
|
45
|
+
MD
|
|
46
|
+
out="$(python3 "$CENSUS" "$WORK/one-rule.md")"; rc=$?
|
|
47
|
+
check "sub-bullets and continuations separated by blank lines stay inside one rule" \
|
|
48
|
+
'[ "$rc" = 0 ] && [ "$(field "$out" rules)" = 1 ] && [ "$(field "$out" prohibitive_tokens)" = 2 ]'
|
|
49
|
+
# This case owns the section boundary alone: a parser that ran past `## After`
|
|
50
|
+
# would report two rules, so the rule count is the quantity that moves here.
|
|
51
|
+
check "the census stops at the next same-level heading" \
|
|
52
|
+
'[ "$(field "$out" rules)" = 1 ]'
|
|
53
|
+
|
|
54
|
+
# A prohibition with no named baseline failure is the diagnostic set; the same
|
|
55
|
+
# rule with its failure named must leave that set. The pair is the control: if
|
|
56
|
+
# the failure-shape vocabulary stopped matching, the first case would still pass
|
|
57
|
+
# and only the second would fail, which is what makes the pair informative.
|
|
58
|
+
cat >"$WORK/unanchored.md" <<'MD'
|
|
59
|
+
# Entry
|
|
60
|
+
|
|
61
|
+
## Core Rules
|
|
62
|
+
|
|
63
|
+
### A group
|
|
64
|
+
|
|
65
|
+
- A rule that must never be skipped.
|
|
66
|
+
MD
|
|
67
|
+
out="$(python3 "$CENSUS" "$WORK/unanchored.md")"
|
|
68
|
+
check "a prohibition with no named baseline failure is reported unanchored" \
|
|
69
|
+
'[ "$(field "$out" unanchored_prohibition_rules)" = 1 ] && [ "$(field "$out" rules_naming_baseline_failure)" = 0 ]'
|
|
70
|
+
|
|
71
|
+
cat >"$WORK/anchored.md" <<'MD'
|
|
72
|
+
# Entry
|
|
73
|
+
|
|
74
|
+
## Core Rules
|
|
75
|
+
|
|
76
|
+
### A group
|
|
77
|
+
|
|
78
|
+
- A rule that must never be skipped. Failure shape: the step is skipped under time pressure.
|
|
79
|
+
MD
|
|
80
|
+
out="$(python3 "$CENSUS" "$WORK/anchored.md")"
|
|
81
|
+
check "the same prohibition with its baseline failure named leaves the unanchored set" \
|
|
82
|
+
'[ "$(field "$out" unanchored_prohibition_rules)" = 0 ] && [ "$(field "$out" rules_naming_baseline_failure)" = 1 ]'
|
|
83
|
+
|
|
84
|
+
# A rule with no prohibitive token is not in the diagnostic set whether or not it
|
|
85
|
+
# names a failure: the set is prohibitions the form table has something to say
|
|
86
|
+
# about, not every rule that omits a rationale.
|
|
87
|
+
cat >"$WORK/no-prohibition.md" <<'MD'
|
|
88
|
+
# Entry
|
|
89
|
+
|
|
90
|
+
## Core Rules
|
|
91
|
+
|
|
92
|
+
### A group
|
|
93
|
+
|
|
94
|
+
- State the artifact's parts, in order.
|
|
95
|
+
MD
|
|
96
|
+
out="$(python3 "$CENSUS" "$WORK/no-prohibition.md")"
|
|
97
|
+
check "a rule carrying no prohibitive token is outside the diagnostic set" \
|
|
98
|
+
'[ "$(field "$out" prohibitive_tokens)" = 0 ] && [ "$(field "$out" unanchored_prohibition_rules)" = 0 ]'
|
|
99
|
+
|
|
100
|
+
# The capitalised spellings are emphasis in this package and are counted; a word
|
|
101
|
+
# that merely contains one of them is not a token.
|
|
102
|
+
cat >"$WORK/tokens.md" <<'MD'
|
|
103
|
+
# Entry
|
|
104
|
+
|
|
105
|
+
## Core Rules
|
|
106
|
+
|
|
107
|
+
### A group
|
|
108
|
+
|
|
109
|
+
- This MUST hold and NEVER lapses, and the mustard and nevertheless here are not tokens.
|
|
110
|
+
MD
|
|
111
|
+
out="$(python3 "$CENSUS" "$WORK/tokens.md")"
|
|
112
|
+
check "capitalised emphasis counts as prohibitive and a word merely containing one does not" \
|
|
113
|
+
'[ "$(field "$out" prohibitive_tokens)" = 2 ]'
|
|
114
|
+
|
|
115
|
+
# The alternation is leftmost and non-overlapping, so a phrase absorbs the words
|
|
116
|
+
# inside it. Two independent review lenses read the previous comment as promising
|
|
117
|
+
# the opposite, which is exactly the failure this case now pins: the documented
|
|
118
|
+
# counting rule and the reported figure have to be the same ruler.
|
|
119
|
+
cat >"$WORK/overlap.md" <<'MD'
|
|
120
|
+
# Entry
|
|
121
|
+
|
|
122
|
+
## Core Rules
|
|
123
|
+
|
|
124
|
+
### A group
|
|
125
|
+
|
|
126
|
+
- This must not be counted twice, and this cannot be either.
|
|
127
|
+
MD
|
|
128
|
+
out="$(python3 "$CENSUS" "$WORK/overlap.md")"
|
|
129
|
+
check "a phrase absorbs the word inside it: 'must not' is one token, not two" \
|
|
130
|
+
'[ "$(field "$out" prohibitive_tokens)" = 2 ]'
|
|
131
|
+
|
|
132
|
+
# An existing but empty Core Rules section is an error, not a zero-rule census:
|
|
133
|
+
# a section that lost its rules must not report as a clean sheet either.
|
|
134
|
+
cat >"$WORK/empty-core.md" <<'MD'
|
|
135
|
+
# Entry
|
|
136
|
+
|
|
137
|
+
## Core Rules
|
|
138
|
+
|
|
139
|
+
### A group
|
|
140
|
+
|
|
141
|
+
Prose with no top-level bullet.
|
|
142
|
+
|
|
143
|
+
## After
|
|
144
|
+
|
|
145
|
+
- A bullet outside Core Rules.
|
|
146
|
+
MD
|
|
147
|
+
out="$(python3 "$CENSUS" "$WORK/empty-core.md" 2>&1)"; rc=$?
|
|
148
|
+
check "an existing but empty Core Rules section is an error, never a zero-rule census" \
|
|
149
|
+
'[ "$rc" != 0 ] && case "$out" in *entrypoint_form_census_error*) true;; *) false;; esac'
|
|
150
|
+
|
|
151
|
+
# An entrypoint with no Core Rules section is an error, not a zero: reporting
|
|
152
|
+
# zero rules would let a renamed or moved section read as a clean sheet.
|
|
153
|
+
cat >"$WORK/no-core.md" <<'MD'
|
|
154
|
+
# Entry
|
|
155
|
+
|
|
156
|
+
## Something Else
|
|
157
|
+
|
|
158
|
+
- A bullet.
|
|
159
|
+
MD
|
|
160
|
+
out="$(python3 "$CENSUS" "$WORK/no-core.md" 2>&1)"; rc=$?
|
|
161
|
+
check "a file with no Core Rules heading is an error, never an empty census" \
|
|
162
|
+
'[ "$rc" != 0 ] && case "$out" in *entrypoint_form_census_error*) true;; *) false;; esac'
|
|
163
|
+
|
|
164
|
+
# The shipped entrypoint must parse, or the instrument silently stops measuring
|
|
165
|
+
# the only file it exists for.
|
|
166
|
+
out="$(python3 "$CENSUS" "$DIR/../SKILL.md")"; rc=$?
|
|
167
|
+
check "the shipped entrypoint parses and reports a non-zero rule count" \
|
|
168
|
+
'[ "$rc" = 0 ] && [ "$(field "$out" rules)" -gt 0 ] && case "$out" in *entrypoint_form_census_ok*) true;; *) false;; esac'
|
|
169
|
+
|
|
170
|
+
if [ "$fails" -ne 0 ]; then
|
|
171
|
+
echo "test_entrypoint_form_census: $fails failing case(s)"
|
|
172
|
+
exit 1
|
|
173
|
+
fi
|
|
174
|
+
echo "test_entrypoint_form_census: ok"
|
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# Regression for reference-access-census.sh: synthetic transcripts only, no
|
|
3
|
+
# host logs are read. Proves (1) per-file session counts and the touching
|
|
4
|
+
# denominator, (2) the unevaluated sentinel when no transcript exists (never a
|
|
5
|
+
# zero table), (3) the privacy contract — output carries no transcript text,
|
|
6
|
+
# no absolute log path, no session id, (2b) input errors (an unreadable
|
|
7
|
+
# transcript) withhold the table with exit 2 instead of printing zeros or the ok
|
|
8
|
+
# token, (4) usage errors exit 2, and (5) the
|
|
9
|
+
# ARG_MAX regression: a candidate set larger than one xargs batch still counts
|
|
10
|
+
# (the first version silently reported 0 on a large window).
|
|
11
|
+
set -euo pipefail
|
|
12
|
+
|
|
13
|
+
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
14
|
+
CENSUS="$HERE/reference-access-census.sh"
|
|
15
|
+
TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT
|
|
16
|
+
fail() { echo "FAIL: $*" >&2; exit 1; }
|
|
17
|
+
|
|
18
|
+
# --- fixture repo: one skill package with an entrypoint and three references ---
|
|
19
|
+
REPO="$TMP/repo"; mkdir -p "$REPO/skills/demo-skill/references"
|
|
20
|
+
git -C "$TMP" init -q repo 2>/dev/null || git init -q "$REPO"
|
|
21
|
+
printf '# demo\n' > "$REPO/skills/demo-skill/SKILL.md"
|
|
22
|
+
for r in alpha beta gamma; do printf '# %s\n' "$r" > "$REPO/skills/demo-skill/references/$r.md"; done
|
|
23
|
+
git -C "$REPO" add -A && git -C "$REPO" -c user.email=t@t -c user.name=t commit -qm fixture
|
|
24
|
+
|
|
25
|
+
# --- fixture transcripts: two log roots, distinct mention shapes ---
|
|
26
|
+
LOGS="$TMP/logs-claude"; mkdir -p "$LOGS/proj-a" "$TMP/logs-codex/2026/09"
|
|
27
|
+
SECRET_PROMPT="user-private-prompt-text-7f3a"
|
|
28
|
+
SESSION_ID="session-9c1e2d3b-secret"
|
|
29
|
+
# session 1 (claude shape): reads alpha and beta via a Read tool call
|
|
30
|
+
printf '{"type":"tool_use","name":"Read","input":{"file_path":"/abs/checkout/skills/demo-skill/references/alpha.md"},"prompt":"%s","session":"%s"}\n' "$SECRET_PROMPT" "$SESSION_ID" > "$LOGS/proj-a/one.jsonl"
|
|
31
|
+
printf '{"type":"tool_use","name":"Read","input":{"file_path":"/abs/checkout/skills/demo-skill/references/beta.md"}}\n' >> "$LOGS/proj-a/one.jsonl"
|
|
32
|
+
# session 2 (codex shape): shell command mentioning alpha twice (must count once)
|
|
33
|
+
printf '{"cmd":"sed -n 1,40p skills/demo-skill/references/alpha.md && cat skills/demo-skill/references/alpha.md"}\n' > "$TMP/logs-codex/2026/09/rollout-two.jsonl"
|
|
34
|
+
# session 3: touches the package (SKILL.md) but no reference
|
|
35
|
+
printf '{"cmd":"cat skills/demo-skill/SKILL.md"}\n' > "$TMP/logs-codex/2026/09/rollout-three.jsonl"
|
|
36
|
+
# session 4: unrelated transcript (must not count as touching)
|
|
37
|
+
printf '{"cmd":"ls skills/other-skill/"}\n' > "$LOGS/proj-a/four.jsonl"
|
|
38
|
+
|
|
39
|
+
# deterministic, differing mtimes so the last_touched column can be asserted exactly (newest wins)
|
|
40
|
+
touch -t "$(date -v-3d +%Y%m%d%H%M 2>/dev/null || date -d '3 days ago' +%Y%m%d%H%M)" "$LOGS/proj-a/one.jsonl"
|
|
41
|
+
touch -t "$(date -v-1d +%Y%m%d%H%M 2>/dev/null || date -d '1 day ago' +%Y%m%d%H%M)" "$TMP/logs-codex/2026/09/rollout-two.jsonl"
|
|
42
|
+
NEWEST="$(date -v-1d +%Y-%m-%d 2>/dev/null || date -d '1 day ago' +%Y-%m-%d)"
|
|
43
|
+
OLDER="$(date -v-3d +%Y-%m-%d 2>/dev/null || date -d '3 days ago' +%Y-%m-%d)"
|
|
44
|
+
# a stale transcript (older than the window) that mentions tracked files must not move any count, share, or date
|
|
45
|
+
printf '{"cmd":"cat skills/demo-skill/references/alpha.md skills/demo-skill/references/gamma.md"}\n' > "$LOGS/proj-a/stale.jsonl"
|
|
46
|
+
touch -t "$(date -v-45d +%Y%m%d%H%M 2>/dev/null || date -d '45 days ago' +%Y%m%d%H%M)" "$LOGS/proj-a/stale.jsonl"
|
|
47
|
+
run() { bash "$CENSUS" --repo-root "$REPO" --skill demo-skill --days 30 --logs "$LOGS,$TMP/logs-codex"; }
|
|
48
|
+
out="$(run)" || fail "census exited non-zero on a valid fixture"
|
|
49
|
+
|
|
50
|
+
# (1) counts and denominator
|
|
51
|
+
printf '%s\n' "$out" | grep -qF 'transcripts=4 sessions_touching_package=3' || fail "denominator wrong:\n$out"
|
|
52
|
+
printf '%s\n' "$out" | grep -qF "references/alpha.md | 2 | $NEWEST | 66%" || fail "alpha should count 2 sessions with the newest touching date $NEWEST:\n$out"
|
|
53
|
+
printf '%s\n' "$out" | grep -qF "references/beta.md | 1 | $OLDER | 33%" || fail "beta should count 1 session dated $OLDER:\n$out"
|
|
54
|
+
printf '%s\n' "$out" | grep -qE '^references/gamma\.md \| 0 \| - \| 0%' || fail "gamma should count 0 with no date:\n$out"
|
|
55
|
+
printf '%s\n' "$out" | grep -qE '^SKILL\.md \| 1 \|' || fail "SKILL.md should count 1 session:\n$out"
|
|
56
|
+
[ "$(printf '%s\n' "$out" | tail -1)" = "reference_access_census_ok" ] || fail "last token must be the ok marker"
|
|
57
|
+
|
|
58
|
+
# (3) privacy contract
|
|
59
|
+
printf '%s\n' "$out" | grep -qF "$SECRET_PROMPT" && fail "transcript text leaked into census output"
|
|
60
|
+
printf '%s\n' "$out" | grep -qF "$SESSION_ID" && fail "session id leaked into census output"
|
|
61
|
+
printf '%s\n' "$out" | grep -qF "$LOGS" && fail "absolute log path leaked into census output"
|
|
62
|
+
printf '%s\n' "$out" | grep -qF "/abs/checkout" && fail "absolute checkout path leaked into census output"
|
|
63
|
+
|
|
64
|
+
# (2) unevaluated sentinel: no transcripts at all
|
|
65
|
+
EMPTY="$TMP/empty-logs"; mkdir -p "$EMPTY"
|
|
66
|
+
eout="$(bash "$CENSUS" --repo-root "$REPO" --skill demo-skill --days 30 --logs "$EMPTY")" || fail "empty-log run must exit 0"
|
|
67
|
+
[ "$(printf '%s\n' "$eout" | tail -1)" = "reference_access_census_unevaluated: no transcripts within 30d under 1 log root(s); counts withheld" ] || fail "missing unevaluated sentinel:\n$eout"
|
|
68
|
+
printf '%s\n' "$eout" | grep -qE '\| 0 \|' && fail "an unevaluated run must not print a zero table"
|
|
69
|
+
printf '%s\n' "$eout" | grep -qF "$EMPTY" && fail "unevaluated sentinel leaked the log root path"
|
|
70
|
+
printf '%s\n' "$eout" | grep -qF "$TMP" && fail "unevaluated sentinel leaked an absolute path"
|
|
71
|
+
|
|
72
|
+
# (2b) input errors withhold the table: an unreadable transcript is exit 2 + unevaluated, never zeros or ok
|
|
73
|
+
UNREAD="$TMP/logs-unreadable"; mkdir -p "$UNREAD"
|
|
74
|
+
printf '{"cmd":"cat skills/demo-skill/references/alpha.md"}\n' > "$UNREAD/readable.jsonl"
|
|
75
|
+
printf '{"cmd":"cat skills/demo-skill/references/beta.md"}\n' > "$UNREAD/locked.jsonl"
|
|
76
|
+
chmod 000 "$UNREAD/locked.jsonl"
|
|
77
|
+
if [ -r "$UNREAD/locked.jsonl" ]; then
|
|
78
|
+
echo "note: chmod 000 is readable here (root); skipping the unreadable-input leg"
|
|
79
|
+
else
|
|
80
|
+
uout="$(bash "$CENSUS" --repo-root "$REPO" --skill demo-skill --days 30 --logs "$UNREAD" 2>"$TMP/unread.stderr")" && fail "unreadable transcript must exit non-zero"
|
|
81
|
+
urc=$?
|
|
82
|
+
uout="$(bash "$CENSUS" --repo-root "$REPO" --skill demo-skill --days 30 --logs "$UNREAD" 2>/dev/null || true)"
|
|
83
|
+
bash "$CENSUS" --repo-root "$REPO" --skill demo-skill --days 30 --logs "$UNREAD" >/dev/null 2>&1 || [ $? -eq 2 ] || fail "unreadable transcript must exit 2"
|
|
84
|
+
printf '%s\n' "$uout" | grep -qE 'reference_access_census_unevaluated: [0-9]+ input error\(s\)' || fail "unreadable transcript must withhold counts as unevaluated:\n$uout"
|
|
85
|
+
printf '%s\n' "$uout" | grep -qF 'reference_access_census_ok' && fail "ok token printed after an input error"
|
|
86
|
+
printf '%s\n' "$uout" | grep -qE '\| [0-9]+ \|' && fail "a table was printed after an input error"
|
|
87
|
+
printf '%s\n' "$uout" | grep -qF "$UNREAD" && fail "input-error sentinel leaked the log path"
|
|
88
|
+
fi
|
|
89
|
+
chmod 644 "$UNREAD/locked.jsonl"
|
|
90
|
+
|
|
91
|
+
# (4) usage errors exit 2
|
|
92
|
+
if bash "$CENSUS" --repo-root "$REPO" --skill no-such-skill --logs "$LOGS" >/dev/null 2>&1; then fail "unknown skill must exit 2"; fi
|
|
93
|
+
bash "$CENSUS" --repo-root "$REPO" --skill no-such-skill --logs "$LOGS" >/dev/null 2>&1 || [ $? -eq 2 ] || fail "unknown skill must exit 2 exactly"
|
|
94
|
+
bash "$CENSUS" --repo-root "$REPO" --skill demo-skill --days abc --logs "$LOGS" >/dev/null 2>&1 || [ $? -eq 2 ] || fail "non-integer --days must exit 2"
|
|
95
|
+
|
|
96
|
+
# (5) ARG_MAX regression: the candidate path list must exceed THIS host's exec limit
|
|
97
|
+
# (getconf ARG_MAX) so that a `$(cat candidates)` expansion — the first version's shape,
|
|
98
|
+
# which silently reported 0 on a real 60-day window — cannot fit in one exec. The fixture
|
|
99
|
+
# size is derived from the limit, never a fixed byte figure; a host whose limit exceeds the
|
|
100
|
+
# fixture bound runs only the counting leg and says so. Verified RED by applying that exact
|
|
101
|
+
# mutation to a disposable copy of the script.
|
|
102
|
+
DPAD=""; i=0; while [ $i -lt 150 ]; do DPAD="${DPAD}d"; i=$((i+1)); done
|
|
103
|
+
BIG="$TMP/logs-big/$DPAD"; mkdir -p "$BIG"
|
|
104
|
+
PAD=""; i=0; while [ $i -lt 40 ]; do PAD="${PAD}x"; i=$((i+1)); done
|
|
105
|
+
arg_max="$(getconf ARG_MAX 2>/dev/null || echo 2097152)"
|
|
106
|
+
per_path=$(( ${#BIG} + 60 ))
|
|
107
|
+
need=$(( (arg_max + 262144) / per_path + 1 ))
|
|
108
|
+
[ "$need" -lt 11000 ] && need=11000
|
|
109
|
+
probe=1
|
|
110
|
+
if [ "$need" -gt 60000 ]; then
|
|
111
|
+
echo "note: ARG_MAX=$arg_max would need $need fixture files; the E2BIG probe is skipped on this host and only the counting leg runs"
|
|
112
|
+
need=11000; probe=0
|
|
113
|
+
fi
|
|
114
|
+
( cd "$BIG" && i=1; while [ $i -le $need ]; do : > "s$i-$PAD.jsonl"; i=$((i+1)); done )
|
|
115
|
+
find "$BIG" -type f -name '*.jsonl' > "$TMP/big-list"
|
|
116
|
+
inv_bytes=$(wc -c < "$TMP/big-list" | tr -d ' ')
|
|
117
|
+
if [ "$probe" -eq 1 ]; then
|
|
118
|
+
erc=0; bash -c 'grep -lF "skills/demo-skill/" $(cat "$1") >/dev/null 2>&1' _ "$TMP/big-list" 2>"$TMP/e2big.err" || erc=$?
|
|
119
|
+
[ "$erc" -eq 126 ] || grep -qi "argument list too long" "$TMP/e2big.err" || fail "ARG_MAX fixture ($inv_bytes bytes, ARG_MAX=$arg_max) does not exceed this host's exec limit (rc=$erc), so the leg cannot kill the single-exec mutation"
|
|
120
|
+
fi
|
|
121
|
+
for i in 7 5208 10999; do printf '{"cmd":"cat skills/demo-skill/references/gamma.md"}\n' > "$BIG/s$i-$PAD.jsonl"; done
|
|
122
|
+
bout="$(bash "$CENSUS" --repo-root "$REPO" --skill demo-skill --days 30 --logs "$BIG")" || fail "big run exited non-zero"
|
|
123
|
+
printf '%s\n' "$bout" | grep -qF "transcripts=$need sessions_touching_package=3" || fail "large candidate set miscounted (ARG_MAX regression):\n$bout"
|
|
124
|
+
printf '%s\n' "$bout" | grep -qE '^references/gamma\.md \| 3 \|' || fail "gamma should count 3 across xargs batches:\n$bout"
|
|
125
|
+
|
|
126
|
+
# (6) a flag without its value is a usage error (exit 2), not an unbound-variable abort
|
|
127
|
+
bash "$CENSUS" --repo-root "$REPO" --skill >/dev/null 2>&1 || [ $? -eq 2 ] || fail "flag without value must exit 2"
|
|
128
|
+
|
|
129
|
+
# (7) an unknown argument never echoes its value (an absolute path passed by mistake stays private)
|
|
130
|
+
uerr="$(bash "$CENSUS" --repo-root "$REPO" --skill demo-skill --logs "$LOGS" "/private/mistyped/path" 2>&1 >/dev/null || true)"
|
|
131
|
+
bash "$CENSUS" --repo-root "$REPO" --skill demo-skill --logs "$LOGS" "/private/mistyped/path" >/dev/null 2>&1 || [ $? -eq 2 ] || fail "unknown argument must exit 2"
|
|
132
|
+
printf '%s\n' "$uerr" | grep -qF "/private/mistyped/path" && fail "unknown-argument error echoed the caller's value"
|
|
133
|
+
printf '%s\n' "$uerr" | grep -qF "unknown argument" || fail "unknown-argument error must still say what went wrong"
|
|
134
|
+
|
|
135
|
+
# (8) an explicitly supplied log root that does not exist is an input error — alone or mixed with a valid root
|
|
136
|
+
for roots in "$TMP/no-such-root" "$TMP/no-such-root,$LOGS"; do
|
|
137
|
+
mout="$(bash "$CENSUS" --repo-root "$REPO" --skill demo-skill --days 30 --logs "$roots" 2>/dev/null || true)"
|
|
138
|
+
bash "$CENSUS" --repo-root "$REPO" --skill demo-skill --days 30 --logs "$roots" >/dev/null 2>&1 || [ $? -eq 2 ] || fail "missing explicit log root must exit 2 ($roots)"
|
|
139
|
+
printf '%s\n' "$mout" | grep -qF 'reference_access_census_unevaluated: 1 input error(s)' || fail "missing explicit log root must withhold counts:\n$mout"
|
|
140
|
+
printf '%s\n' "$mout" | grep -qF 'reference_access_census_ok' && fail "ok token printed with a missing explicit log root"
|
|
141
|
+
printf '%s\n' "$mout" | grep -qE '\| [0-9]+ \|' && fail "a table was printed with a missing explicit log root"
|
|
142
|
+
printf '%s\n' "$mout" | grep -qF "$TMP" && fail "missing-root sentinel leaked a path"
|
|
143
|
+
done
|
|
144
|
+
# a missing DEFAULT root stays normal: with no --logs and an empty HOME the result is the no-transcript sentinel, exit 0
|
|
145
|
+
dout="$(HOME="$TMP/empty-home" bash "$CENSUS" --repo-root "$REPO" --skill demo-skill --days 30)" || fail "absent default roots must not be an input error"
|
|
146
|
+
[ "$(printf '%s\n' "$dout" | tail -1)" = "reference_access_census_unevaluated: no transcripts within 30d under 2 log root(s); counts withheld" ] || fail "absent default roots must yield the no-transcript sentinel:\n$dout"
|
|
147
|
+
|
|
148
|
+
# (10) overlapping log roots (a root and its own subtree, or a root listed twice) count each transcript once
|
|
149
|
+
oout="$(bash "$CENSUS" --repo-root "$REPO" --skill demo-skill --days 30 --logs "$TMP/logs-codex,$TMP/logs-codex/2026,$TMP/logs-codex")" || fail "overlapping roots run exited non-zero"
|
|
150
|
+
printf '%s\n' "$oout" | grep -qF 'transcripts=2 sessions_touching_package=2' || fail "overlapping roots must not double-count transcripts:\n$oout"
|
|
151
|
+
printf '%s\n' "$oout" | grep -qE '^references/alpha\.md \| 1 \|' || fail "alpha must count once under overlapping roots:\n$oout"
|
|
152
|
+
|
|
153
|
+
# (11) an invalid --skill value is never echoed back (a mistyped private identifier stays private)
|
|
154
|
+
kerr="$(bash "$CENSUS" --repo-root "$REPO" --skill "acme-internal-secret-project" --logs "$LOGS" 2>&1 >/dev/null || true)"
|
|
155
|
+
printf '%s\n' "$kerr" | grep -qF "acme-internal-secret-project" && fail "unknown --skill error echoed the caller's value"
|
|
156
|
+
printf '%s\n' "$kerr" | grep -qF "usage_error" || fail "unknown --skill must still be reported as a usage error"
|
|
157
|
+
|
|
158
|
+
# (12) a transcript whose name contains a newline is one candidate, not two broken ones
|
|
159
|
+
NL="$TMP/logs-newline"; mkdir -p "$NL"; printf '{"cmd":"cat skills/demo-skill/references/beta.md"}\n' > "$NL/odd
|
|
160
|
+
name.jsonl"
|
|
161
|
+
nout="$(bash "$CENSUS" --repo-root "$REPO" --skill demo-skill --days 30 --logs "$NL")" || fail "newline-named transcript run exited non-zero"
|
|
162
|
+
printf '%s\n' "$nout" | grep -qF 'transcripts=1 sessions_touching_package=1' || fail "newline in a transcript name must not split the candidate:\n$nout"
|
|
163
|
+
|
|
164
|
+
# (13) HOME unset and no --logs: no default roots, the no-transcript sentinel, exit 0 (never an unbound-variable abort)
|
|
165
|
+
hout="$(env -u HOME bash "$CENSUS" --repo-root "$REPO" --skill demo-skill --days 30)" || fail "unset HOME must not abort the census"
|
|
166
|
+
[ "$(printf '%s\n' "$hout" | tail -1)" = "reference_access_census_unevaluated: no transcripts within 30d under 0 log root(s); counts withheld" ] || fail "unset HOME must yield the no-transcript sentinel:\n$hout"
|
|
167
|
+
|
|
168
|
+
# (14) a transcript-read error forced by a command shim (root-independent twin of the chmod leg)
|
|
169
|
+
GSHIM="$TMP/gshim"; mkdir -p "$GSHIM"
|
|
170
|
+
printf '#!/usr/bin/env bash\necho "grep: transcript: Input/output error" >&2\nexit 2\n' > "$GSHIM/grep"; chmod +x "$GSHIM/grep"
|
|
171
|
+
gout="$(PATH="$GSHIM:$PATH" bash "$CENSUS" --repo-root "$REPO" --skill demo-skill --days 30 --logs "$LOGS" 2>/dev/null || true)"
|
|
172
|
+
PATH="$GSHIM:$PATH" bash "$CENSUS" --repo-root "$REPO" --skill demo-skill --days 30 --logs "$LOGS" >/dev/null 2>&1 || [ $? -eq 2 ] || fail "a grep read error must exit 2"
|
|
173
|
+
printf '%s\n' "$gout" | grep -qF 'reference_access_census_unevaluated:' || fail "a grep read error must withhold counts:\n$gout"
|
|
174
|
+
printf '%s\n' "$gout" | grep -qF 'reference_access_census_ok' && fail "ok token printed after a grep read error"
|
|
175
|
+
|
|
176
|
+
# (15) a grep that fails silently (exit 2, no stderr) is still an input error: withhold, exit 2, no ok token
|
|
177
|
+
QSHIM="$TMP/qshim"; mkdir -p "$QSHIM"
|
|
178
|
+
printf '#!/usr/bin/env bash\nexit 2\n' > "$QSHIM/grep"; chmod +x "$QSHIM/grep"
|
|
179
|
+
qout="$(PATH="$QSHIM:$PATH" bash "$CENSUS" --repo-root "$REPO" --skill demo-skill --days 30 --logs "$LOGS" 2>/dev/null || true)"
|
|
180
|
+
PATH="$QSHIM:$PATH" bash "$CENSUS" --repo-root "$REPO" --skill demo-skill --days 30 --logs "$LOGS" >/dev/null 2>&1 || [ $? -eq 2 ] || fail "a silent grep failure must exit 2"
|
|
181
|
+
printf '%s\n' "$qout" | grep -qF 'reference_access_census_unevaluated:' || fail "a silent grep failure must withhold counts:\n$qout"
|
|
182
|
+
printf '%s\n' "$qout" | grep -qF 'reference_access_census_ok' && fail "ok token printed after a silent grep failure"
|
|
183
|
+
|
|
184
|
+
# (16) an inherited errexit (caller exported SHELLOPTS=errexit) must not turn an all-no-match batch into an input error
|
|
185
|
+
NM="$TMP/logs-nomatch"; mkdir -p "$NM"; printf '{"cmd":"ls skills/other-skill/"}\n' > "$NM/quiet.jsonl"
|
|
186
|
+
mout2="$(env SHELLOPTS=errexit bash "$CENSUS" --repo-root "$REPO" --skill demo-skill --days 30 --logs "$NM")" || fail "inherited errexit must not abort a valid census"
|
|
187
|
+
printf '%s\n' "$mout2" | grep -qF 'transcripts=1 sessions_touching_package=0' || fail "all-no-match batch under inherited errexit must be a valid zero census:\n$mout2"
|
|
188
|
+
[ "$(printf '%s\n' "$mout2" | tail -1)" = "reference_access_census_ok" ] || fail "inherited errexit must not withhold a valid census"
|
|
189
|
+
|
|
190
|
+
# (17) a find that fails silently (exit 2, no stderr) is an input error: withhold, exit 2, no ok token
|
|
191
|
+
FSHIM="$TMP/fshim"; mkdir -p "$FSHIM"
|
|
192
|
+
printf '#!/usr/bin/env bash\nexit 2\n' > "$FSHIM/find"; chmod +x "$FSHIM/find"
|
|
193
|
+
fout="$(PATH="$FSHIM:$PATH" bash "$CENSUS" --repo-root "$REPO" --skill demo-skill --days 30 --logs "$LOGS" 2>/dev/null || true)"
|
|
194
|
+
frc=0; PATH="$FSHIM:$PATH" bash "$CENSUS" --repo-root "$REPO" --skill demo-skill --days 30 --logs "$LOGS" >/dev/null 2>&1 || frc=$?
|
|
195
|
+
[ "$frc" -eq 2 ] || fail "a silent find failure must exit 2 (got $frc)"
|
|
196
|
+
printf '%s\n' "$fout" | grep -qE 'reference_access_census_unevaluated: [0-9]+ input error' || fail "a silent find failure must withhold counts as an input error, not the no-transcript sentinel:\n$fout"
|
|
197
|
+
printf '%s\n' "$fout" | grep -qF 'reference_access_census_ok' && fail "ok token printed after a silent find failure"
|
|
198
|
+
|
|
199
|
+
# (9) a failing stat (a transcript vanishing before the timestamp read) withholds the table with exit 2 —
|
|
200
|
+
# deterministic via a PATH shim, independent of filesystem permissions or root
|
|
201
|
+
SHIM="$TMP/shim"; mkdir -p "$SHIM"
|
|
202
|
+
printf '#!/usr/bin/env bash\necho "stat: cannot stat: No such file or directory" >&2\nexit 1\n' > "$SHIM/stat"; chmod +x "$SHIM/stat"
|
|
203
|
+
sout="$(PATH="$SHIM:$PATH" bash "$CENSUS" --repo-root "$REPO" --skill demo-skill --days 30 --logs "$LOGS,$TMP/logs-codex" 2>/dev/null || true)"
|
|
204
|
+
PATH="$SHIM:$PATH" bash "$CENSUS" --repo-root "$REPO" --skill demo-skill --days 30 --logs "$LOGS,$TMP/logs-codex" >/dev/null 2>&1 || [ $? -eq 2 ] || fail "failing stat must exit 2, not the pipeline status"
|
|
205
|
+
printf '%s\n' "$sout" | grep -qF 'reference_access_census_unevaluated:' || fail "failing stat must print the unevaluated sentinel:\n$sout"
|
|
206
|
+
printf '%s\n' "$sout" | grep -qF 'reference_access_census_ok' && fail "ok token printed after a failing stat"
|
|
207
|
+
printf '%s\n' "$sout" | grep -qE '\| [0-9]+ \|' && fail "a table was printed after a failing stat"
|
|
208
|
+
|
|
209
|
+
echo "test_reference_access_census_ok"
|