bigpowers 2.63.0 → 2.64.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.pi/package.json +1 -1
- package/.pi/prompts/build-epic.md +2 -1
- package/.pi/prompts/develop-tdd.md +4 -0
- package/.pi/prompts/investigate-bug.md +5 -0
- package/.pi/prompts/plan-refactor.md +5 -0
- package/.pi/prompts/run-benchmark.md +67 -48
- package/.pi/prompts/security-review.md +4 -0
- package/.pi/prompts/setup-environment.md +15 -0
- package/.pi/prompts/wire-observability.md +4 -0
- package/.pi/skills/build-epic/SKILL.md +2 -1
- package/.pi/skills/develop-tdd/SKILL.md +4 -0
- package/.pi/skills/investigate-bug/SKILL.md +5 -0
- package/.pi/skills/plan-refactor/SKILL.md +5 -0
- package/.pi/skills/run-benchmark/SKILL.md +67 -48
- package/.pi/skills/security-review/SKILL.md +4 -0
- package/.pi/skills/setup-environment/SKILL.md +15 -0
- package/.pi/skills/wire-observability/SKILL.md +4 -0
- package/CHANGELOG.md +25 -0
- package/CLAUDE.md +4 -0
- package/CONVENTIONS.md +10 -0
- package/README.md +6 -0
- package/SKILL-INDEX.md +1 -1
- package/package.json +5 -2
- package/scripts/build-receipts.sh +107 -0
- package/scripts/build-skill-graph.sh +226 -0
- package/scripts/build-skill-index.sh +10 -20
- package/scripts/generate-skill-index.sh +2 -4
- package/scripts/record-cycle-time.sh +6 -4
- package/scripts/regenerate-lockfile.sh +10 -38
- package/scripts/sync-skills.sh +106 -79
- package/scripts/validate-okf.sh +28 -0
- package/skills/build-epic/SKILL.md +2 -1
- package/skills/develop-tdd/SKILL.md +4 -0
- package/skills/investigate-bug/SKILL.md +5 -0
- package/skills/plan-refactor/SKILL.md +5 -0
- package/skills/run-benchmark/SKILL.md +67 -49
- package/skills/security-review/SKILL.md +4 -0
- package/skills/setup-environment/SKILL.md +15 -0
- package/skills/wire-observability/SKILL.md +4 -0
- package/skills-lock.json +83 -83
package/scripts/sync-skills.sh
CHANGED
|
@@ -1,14 +1,12 @@
|
|
|
1
1
|
#!/usr/bin/env bash
|
|
2
|
-
# story: e13s03
|
|
2
|
+
# story: e13s03 e28s03
|
|
3
3
|
# sync-skills.sh — generate Cursor, Gemini CLI, and pi artifacts from SKILL.md source files
|
|
4
|
+
# Architecture: Parse→IR→Render. Each target is its own render_<target>() function.
|
|
4
5
|
# Run this after adding or updating any skill. Symlinks carry changes through automatically.
|
|
5
6
|
set -euo pipefail
|
|
6
7
|
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
# This allows e29s02 to git-mv skill dirs into skills/ without changing this script.
|
|
10
|
-
SKILLS_ROOT="$REPO_ROOT"
|
|
11
|
-
[[ -d "$REPO_ROOT/skills" ]] && SKILLS_ROOT="$REPO_ROOT/skills"
|
|
8
|
+
source "$(dirname "${BASH_SOURCE[0]}")/lib/skill-common.sh"
|
|
9
|
+
|
|
12
10
|
CURSOR_RULES="$REPO_ROOT/.cursor/rules"
|
|
13
11
|
GEMINI_EXT_DIR="$REPO_ROOT/.gemini/extensions/bigpowers"
|
|
14
12
|
GEMINI_SKILLS="$GEMINI_EXT_DIR/skills"
|
|
@@ -26,100 +24,136 @@ rm -rf "${GEMINI_COMMANDS:?}"/*
|
|
|
26
24
|
rm -rf "${PI_SKILLS:?}"/*
|
|
27
25
|
rm -rf "${PI_PROMPTS:?}"/*
|
|
28
26
|
|
|
29
|
-
#
|
|
30
|
-
#
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
# Escape double quotes and backslashes for safe double-quoted YAML output
|
|
48
|
-
description_escaped=$(echo "$description" | sed 's/\\/\\\\/g; s/"/\\"/g')
|
|
49
|
-
|
|
50
|
-
[[ -z "$name" ]] && continue
|
|
51
|
-
|
|
52
|
-
# Build concatenated content: SKILL.md body + all other *.md files alphabetically
|
|
53
|
-
# Strip frontmatter from SKILL.md (content between second --- and EOF)
|
|
27
|
+
# ── IR: shared intermediate representation ──────────────────────────
|
|
28
|
+
# These globals are set by the main loop per skill and consumed by render functions.
|
|
29
|
+
IR_NAME=""
|
|
30
|
+
IR_MODEL=""
|
|
31
|
+
IR_DESCRIPTION=""
|
|
32
|
+
IR_DESC_ESCAPED=""
|
|
33
|
+
IR_BODY=""
|
|
34
|
+
|
|
35
|
+
# ── build_body ──────────────────────────────────────────────────────
|
|
36
|
+
# Builds the concatenated content: SKILL.md body + all other *.md files.
|
|
37
|
+
# Args: $1 = path to SKILL.md, $2 = skill directory
|
|
38
|
+
build_body() {
|
|
39
|
+
local skill_md="$1"
|
|
40
|
+
local skill_dir="$2"
|
|
41
|
+
local body
|
|
42
|
+
|
|
43
|
+
# Strip frontmatter from SKILL.md (content after second ---)
|
|
54
44
|
body=$(awk '/^---/{f++; next} f>=2{print}' "$skill_md")
|
|
55
45
|
|
|
46
|
+
# Append extra *.md files alphabetically
|
|
47
|
+
local extra_md
|
|
56
48
|
for extra_md in $(find "$skill_dir" -maxdepth 1 -name "*.md" ! -name "SKILL.md" | LC_ALL=C sort); do
|
|
57
49
|
body="$body"$'\n\n'"---"$'\n\n'"$(cat "$extra_md")"
|
|
58
50
|
done
|
|
59
51
|
|
|
60
52
|
# Strip disable-model-invocation lines
|
|
61
|
-
|
|
53
|
+
echo "$body" | grep -v 'disable-model-invocation'
|
|
54
|
+
}
|
|
62
55
|
|
|
63
|
-
|
|
64
|
-
|
|
56
|
+
# ── Render target: Cursor ───────────────────────────────────────────
|
|
57
|
+
# Writes .cursor/rules/<name>.mdc with YAML frontmatter + body.
|
|
58
|
+
render_cursor() {
|
|
59
|
+
local cursor_file="$CURSOR_RULES/$IR_NAME.mdc"
|
|
65
60
|
{
|
|
66
61
|
echo "---"
|
|
67
|
-
echo "description: \"$
|
|
62
|
+
echo "description: \"$IR_DESC_ESCAPED\""
|
|
68
63
|
echo "alwaysApply: false"
|
|
69
64
|
echo "---"
|
|
70
65
|
echo ""
|
|
71
|
-
echo "$
|
|
66
|
+
echo "$IR_BODY"
|
|
72
67
|
} > "$cursor_file"
|
|
68
|
+
}
|
|
73
69
|
|
|
74
|
-
|
|
75
|
-
|
|
70
|
+
# ── Render target: Gemini Agent Skill ───────────────────────────────
|
|
71
|
+
# Writes .gemini/extensions/bigpowers/skills/<name>/SKILL.md
|
|
72
|
+
render_gemini_skill() {
|
|
73
|
+
mkdir -p "$GEMINI_SKILLS/$IR_NAME"
|
|
76
74
|
{
|
|
77
75
|
echo "---"
|
|
78
|
-
echo "name: $
|
|
79
|
-
echo "description: \"$
|
|
76
|
+
echo "name: $IR_NAME"
|
|
77
|
+
echo "description: \"$IR_DESC_ESCAPED\""
|
|
80
78
|
echo "---"
|
|
81
79
|
echo ""
|
|
82
|
-
echo "$
|
|
83
|
-
} > "$GEMINI_SKILLS/$
|
|
80
|
+
echo "$IR_BODY"
|
|
81
|
+
} > "$GEMINI_SKILLS/$IR_NAME/SKILL.md"
|
|
82
|
+
}
|
|
84
83
|
|
|
85
|
-
|
|
86
|
-
|
|
84
|
+
# ── Render target: Gemini Slash Command ─────────────────────────────
|
|
85
|
+
# Writes .gemini/extensions/bigpowers/commands/<name>.toml + prompt file
|
|
86
|
+
render_gemini_command() {
|
|
87
87
|
mkdir -p "$GEMINI_COMMANDS/prompts"
|
|
88
|
-
prompt_file="commands/prompts/$
|
|
89
|
-
echo "$
|
|
90
|
-
|
|
88
|
+
local prompt_file="commands/prompts/$IR_NAME.md"
|
|
89
|
+
echo "$IR_BODY" > "$GEMINI_EXT_DIR/$prompt_file"
|
|
91
90
|
{
|
|
92
|
-
echo "description = \"$
|
|
91
|
+
echo "description = \"$IR_DESC_ESCAPED\""
|
|
93
92
|
echo "prompt = \"@{$prompt_file}\""
|
|
94
|
-
} > "$GEMINI_COMMANDS/$
|
|
93
|
+
} > "$GEMINI_COMMANDS/$IR_NAME.toml"
|
|
94
|
+
}
|
|
95
95
|
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
96
|
+
# ── Render target: pi Agent Skill ───────────────────────────────────
|
|
97
|
+
# Writes .pi/skills/<name>/SKILL.md
|
|
98
|
+
render_pi_skill() {
|
|
99
|
+
mkdir -p "$PI_SKILLS/$IR_NAME"
|
|
99
100
|
{
|
|
100
101
|
echo "---"
|
|
101
|
-
echo "name: $
|
|
102
|
-
echo "description: \"$
|
|
103
|
-
[[ -n "$
|
|
102
|
+
echo "name: $IR_NAME"
|
|
103
|
+
echo "description: \"$IR_DESC_ESCAPED\""
|
|
104
|
+
[[ -n "$IR_MODEL" ]] && echo "model: $IR_MODEL"
|
|
104
105
|
echo "---"
|
|
105
106
|
echo ""
|
|
106
|
-
echo "$
|
|
107
|
-
} > "$PI_SKILLS/$
|
|
107
|
+
echo "$IR_BODY"
|
|
108
|
+
} > "$PI_SKILLS/$IR_NAME/SKILL.md"
|
|
109
|
+
}
|
|
108
110
|
|
|
109
|
-
|
|
110
|
-
|
|
111
|
+
# ── Render target: pi Prompt Template ───────────────────────────────
|
|
112
|
+
# Writes .pi/prompts/<name>.md
|
|
113
|
+
render_pi_prompt() {
|
|
111
114
|
{
|
|
112
115
|
echo "---"
|
|
113
|
-
echo "description: $
|
|
116
|
+
echo "description: $IR_DESCRIPTION"
|
|
114
117
|
echo "---"
|
|
115
118
|
echo ""
|
|
116
|
-
echo "$
|
|
117
|
-
} > "$PI_PROMPTS/$
|
|
119
|
+
echo "$IR_BODY"
|
|
120
|
+
} > "$PI_PROMPTS/$IR_NAME.md"
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
# ── Target registry ─────────────────────────────────────────────────
|
|
124
|
+
# Order matters: render functions are called in this order per skill.
|
|
125
|
+
TARGETS=(render_cursor render_gemini_skill render_gemini_command render_pi_skill render_pi_prompt)
|
|
126
|
+
|
|
127
|
+
skill_count=0
|
|
128
|
+
|
|
129
|
+
# ── Main loop: Parse → IR → Render per skill ────────────────────────
|
|
130
|
+
while IFS= read -r skill_dir; do
|
|
131
|
+
skill_md="$skill_dir/SKILL.md"
|
|
132
|
+
|
|
133
|
+
parse_frontmatter "$skill_md" || continue
|
|
134
|
+
|
|
135
|
+
IR_NAME="$_PF_NAME"
|
|
136
|
+
IR_MODEL="$_PF_MODEL"
|
|
137
|
+
IR_DESCRIPTION="$_PF_DESCRIPTION"
|
|
138
|
+
|
|
139
|
+
# Escape double quotes and backslashes for safe double-quoted YAML output
|
|
140
|
+
IR_DESC_ESCAPED=$(echo "$IR_DESCRIPTION" | sed 's/\\/\\\\/g; s/"/\\"/g')
|
|
141
|
+
|
|
142
|
+
[[ -z "$IR_NAME" ]] && continue
|
|
143
|
+
|
|
144
|
+
IR_BODY=$(build_body "$skill_md" "$skill_dir")
|
|
145
|
+
|
|
146
|
+
# Dispatch to each render target
|
|
147
|
+
for target_fn in "${TARGETS[@]}"; do
|
|
148
|
+
"$target_fn"
|
|
149
|
+
done
|
|
118
150
|
|
|
119
151
|
skill_count=$((skill_count + 1))
|
|
120
|
-
done
|
|
152
|
+
done < <(iterate_skills)
|
|
153
|
+
|
|
154
|
+
# ── Post-loop: manifest, opencode, regeneration, guards, README badge ──
|
|
121
155
|
|
|
122
|
-
# Assemble final gemini-extension.json
|
|
156
|
+
# Assemble final gemini-extension.json
|
|
123
157
|
pkg_version=$(jq -r '.version' "$REPO_ROOT/package.json")
|
|
124
158
|
pkg_desc=$(jq -r '.description' "$REPO_ROOT/package.json")
|
|
125
159
|
|
|
@@ -128,8 +162,7 @@ jq -n --arg name "bigpowers" \
|
|
|
128
162
|
--arg desc "${skill_count} skills — ${pkg_desc}" \
|
|
129
163
|
'{name: $name, version: $version, description: $desc}' > "$GEMINI_MANIFEST"
|
|
130
164
|
|
|
131
|
-
#
|
|
132
|
-
# Enables pi install (local path, npm, or git) with auto-discovered skills and prompts
|
|
165
|
+
# Write pi package config: .pi/package.json
|
|
133
166
|
jq -n --arg version "$pkg_version" \
|
|
134
167
|
--arg desc "${skill_count} skills — ${pkg_desc}" \
|
|
135
168
|
'{
|
|
@@ -143,9 +176,7 @@ jq -n --arg version "$pkg_version" \
|
|
|
143
176
|
}
|
|
144
177
|
}' > "$PI_PACKAGE_JSON"
|
|
145
178
|
|
|
146
|
-
#
|
|
147
|
-
# Skills are loaded on-demand via opencode's native skill tool, not instructions.
|
|
148
|
-
# Full opencode integration lives in the bigpowers-opencode repo.
|
|
179
|
+
# Write OpenCode configuration: opencode.json
|
|
149
180
|
{
|
|
150
181
|
echo "{"
|
|
151
182
|
echo " \"\$schema\": \"https://opencode.ai/config.json\","
|
|
@@ -153,7 +184,7 @@ jq -n --arg version "$pkg_version" \
|
|
|
153
184
|
echo "}"
|
|
154
185
|
} > "$REPO_ROOT/opencode.json"
|
|
155
186
|
|
|
156
|
-
#
|
|
187
|
+
# Sync to bigpowers-opencode repo (if --opencode path is provided)
|
|
157
188
|
OPN_TARGET=""
|
|
158
189
|
for arg in "$@"; do
|
|
159
190
|
case "$arg" in
|
|
@@ -194,8 +225,7 @@ if [[ -x "$REPO_ROOT/scripts/build-skill-index.sh" ]]; then
|
|
|
194
225
|
bash "$REPO_ROOT/scripts/build-skill-index.sh" || true
|
|
195
226
|
fi
|
|
196
227
|
|
|
197
|
-
# Prune orphan cursor rules
|
|
198
|
-
# Allowlist: hand-maintained rules that intentionally have no SKILL.md source.
|
|
228
|
+
# Prune orphan cursor rules
|
|
199
229
|
CURSOR_KEEP="context7.mdc"
|
|
200
230
|
for mdc in "$CURSOR_RULES"/*.mdc; do
|
|
201
231
|
[[ -e "$mdc" ]] || continue
|
|
@@ -207,8 +237,7 @@ for mdc in "$CURSOR_RULES"/*.mdc; do
|
|
|
207
237
|
fi
|
|
208
238
|
done
|
|
209
239
|
|
|
210
|
-
# Stamp the README skill-count badge
|
|
211
|
-
# Portable (BSD/GNU): write to temp file instead of sed -i.
|
|
240
|
+
# Stamp the README skill-count badge
|
|
212
241
|
readme="$REPO_ROOT/README.md"
|
|
213
242
|
if [[ -f "$readme" ]] && grep -q 'badge/skills-' "$readme"; then
|
|
214
243
|
readme_tmp=$(mktemp)
|
|
@@ -229,7 +258,7 @@ echo " → SKILL-INDEX.md (auto-generated skill reference)"
|
|
|
229
258
|
echo " → opencode.json (CLAUDE.md + CONVENTIONS.md instructions)"
|
|
230
259
|
[[ -n "$OPN_TARGET" ]] && echo " → bigpowers-opencode: $opencode_count skills"
|
|
231
260
|
|
|
232
|
-
# Regression guard (BUG-2026-06-02T164500)
|
|
261
|
+
# Regression guard (BUG-2026-06-02T164500)
|
|
233
262
|
trace_mdc="$REPO_ROOT/.cursor/rules/trace-requirement.mdc"
|
|
234
263
|
if [[ -f "$trace_mdc" ]] && ! grep -q 'release-plan.yaml + epic' "$trace_mdc"; then
|
|
235
264
|
echo "sync-skills: FAIL — '+' missing from trace-requirement; use sed -E for whitespace collapse" >&2
|
|
@@ -257,15 +286,13 @@ if [[ -f "$validate_script" ]] && command -v python3 &>/dev/null; then
|
|
|
257
286
|
fi
|
|
258
287
|
fi
|
|
259
288
|
|
|
260
|
-
# Regression guard (BUG-2026-07-02T103911): no bash 4+ features
|
|
261
|
-
# macOS ships bash 3.2 which lacks declare -A (associative arrays)
|
|
289
|
+
# Regression guard (BUG-2026-07-02T103911): no bash 4+ features
|
|
262
290
|
if grep -rn '^declare -A\|^[[:space:]]*declare -A' scripts/sync-skills.sh scripts/generate-skill-index.sh scripts/regenerate-lockfile.sh scripts/build-skill-index.sh 2>/dev/null; then
|
|
263
291
|
echo "sync-skills: FAIL — bash 4+ features detected in install-chain scripts (not macOS-compatible)" >&2
|
|
264
292
|
exit 1
|
|
265
293
|
fi
|
|
266
294
|
|
|
267
|
-
# Regenerate derived reference tables
|
|
268
|
-
# Only in dev context (with .git) — not during consumer npm install where docs/ is excluded
|
|
295
|
+
# Regenerate derived reference tables
|
|
269
296
|
if [[ -d "$REPO_ROOT/.git" ]]; then
|
|
270
297
|
bash "$REPO_ROOT/scripts/generate-reference-tables.sh"
|
|
271
298
|
fi
|
package/scripts/validate-okf.sh
CHANGED
|
@@ -194,6 +194,31 @@ for m in d.get('migrations',[]):
|
|
|
194
194
|
printf "${GREEN}PASS${NC} %s (%s migrations indexed)\n" "$name" "$count"
|
|
195
195
|
}
|
|
196
196
|
|
|
197
|
+
# ---- Receipts validation (e41s04) ----------------------------------------
|
|
198
|
+
validate_receipts() {
|
|
199
|
+
local f="$ROOT/specs/receipts.json"
|
|
200
|
+
if [ ! -f "$f" ]; then printf "${YELLOW}SKIP${NC} receipts.json not found\n"; return; fi
|
|
201
|
+
python3 -c "
|
|
202
|
+
import json, sys
|
|
203
|
+
d=json.load(open('$f'))
|
|
204
|
+
vs={'measured','estimated','backfilled','absent'}
|
|
205
|
+
ss=('compliance','golden_suite','metrics','traceability')
|
|
206
|
+
errs=0
|
|
207
|
+
for sec in ss:
|
|
208
|
+
data=d.get(sec)
|
|
209
|
+
if data is None: continue
|
|
210
|
+
src=data.get('source','')
|
|
211
|
+
if not src: print(f'FAIL: receipts.json — {sec}: missing source tag'); errs+=1; continue
|
|
212
|
+
if src not in vs: print(f'FAIL: receipts.json — {sec}: unknown source \"{src}\"'); errs+=1; continue
|
|
213
|
+
hv='value' in data
|
|
214
|
+
if src=='absent' and hv: print(f'FAIL: receipts.json — {sec}: tagged absent but carries value'); errs+=1
|
|
215
|
+
elif src!='absent' and not hv: print(f'FAIL: receipts.json — {sec}: tagged {src} but missing value'); errs+=1
|
|
216
|
+
if errs==0: print('PASS receipts.json')
|
|
217
|
+
else: print(f'{errs} violation(s) in receipts.json')
|
|
218
|
+
sys.exit(errs)
|
|
219
|
+
" || { EXIT_CODE=1; return; }
|
|
220
|
+
}
|
|
221
|
+
|
|
197
222
|
# ---- Main validation dispatcher ----------------------------------------
|
|
198
223
|
validate_bundle() {
|
|
199
224
|
local file="$1"
|
|
@@ -280,4 +305,7 @@ else
|
|
|
280
305
|
exit 1
|
|
281
306
|
fi
|
|
282
307
|
|
|
308
|
+
# e41s04: always validate receipts.json
|
|
309
|
+
validate_receipts
|
|
310
|
+
|
|
283
311
|
exit "$EXIT_CODE"
|
|
@@ -36,7 +36,8 @@ Orchestrates the **build** flow for a single epic: survey → plan tasks → kic
|
|
|
36
36
|
2. **Step 0 — Threat Model:** Run `security-review` against the epic's scope (read from the epic capsule). Output `specs/security/epics/<epic-id>/THREAT_MODEL.md` with surface area, vulnerability categories, risk level, and mitigation guidance.
|
|
37
37
|
3. **Assess Impact (Step 2):** Before writing tasks, run `assess-impact --lightweight` on the proposed change. If the risk score exceeds 7, gate — require a `grill-me` session. Write the impact report to `specs/IMPACT-<epic>-<story>.md`. For net-new code with no existing dependents, skip.
|
|
38
38
|
4. **BCP Tracking (Step 2):** After `plan-work` completes, read the `bcps:` count (Business Complexity Points story size) from the epic capsule and carry it into `state.yaml` as `epic_cycle.story_bcps = N`.
|
|
39
|
-
5.
|
|
39
|
+
5. **BCP Plus (Step 2, optional):** When the epic capsule carries a `bcp_plus_breakdown` (13-dimension sizing), also carry it into `state.yaml` as `epic_cycle.bcp_plus`. The breakdown maps each of the 13 dimensions to an integer count. NFR dimensions (11–13) that are N/A per the NFR Gate may be set to 0.
|
|
40
|
+
6. If `epic_cycle.step` missing, set to `1`.
|
|
40
41
|
6. Run **only the current step** (resume mode) unless user asked for full auto-run.
|
|
41
42
|
7. After step verify passes, increment `epic_cycle.step` in `state.yaml` (or `bash scripts/bp-yaml-set.sh` if available).
|
|
42
43
|
8. On story complete, set `execution-status.yaml` story key to `done`; run `bash scripts/sync-status-from-epics.sh`.
|
|
@@ -102,3 +102,7 @@ If this cycle modified files in `.github/workflows/`, run the CI dry-run procedu
|
|
|
102
102
|
|
|
103
103
|
Gate: READY -> next: verify-work
|
|
104
104
|
Writes: state.yaml handoff.next_skill = verify-work
|
|
105
|
+
|
|
106
|
+
## BCP Plus Integration
|
|
107
|
+
|
|
108
|
+
At story completion, if the story was sized with BCP Plus (13-dimension breakdown), log the `bcp_plus.total` alongside the standard `bcps:` count in the story's tasks.yaml. The breakdown is available from the epic capsule's `bcp_plus_breakdown` field. See `docs/references/bcp-plus.md` for the full methodology and NFR Gate pattern.
|
|
@@ -120,3 +120,8 @@ A numbered list of RED-GREEN cycles:
|
|
|
120
120
|
|
|
121
121
|
After writing the bug file, print a one-line summary of the root cause and suggest running `kickoff-branch` next to create a fix branch.
|
|
122
122
|
|
|
123
|
+
## References
|
|
124
|
+
|
|
125
|
+
- [Feathers' Seams & Characterization Tests](../../docs/references/feathers.md) — seam types, characterization tests, and the legacy code change algorithm
|
|
126
|
+
- [Fowler's Code Smells](../../docs/references/fowler.md) — identifying structural problems by smell before investigating
|
|
127
|
+
|
|
@@ -78,5 +78,10 @@ Any further notes about the refactor.
|
|
|
78
78
|
|
|
79
79
|
After writing `specs/REFACTOR_LATEST.md`, suggest running `kickoff-branch` next to create a refactor branch.
|
|
80
80
|
|
|
81
|
+
## References
|
|
82
|
+
|
|
83
|
+
- [Fowler's Refactoring Catalog](../../docs/references/fowler.md) — canonical refactoring vocabulary and code-smell taxonomy
|
|
84
|
+
- [Beck's Tidy First?](../../docs/references/kent-beck.md) — structural change before behavioral change
|
|
85
|
+
|
|
81
86
|
## Verify
|
|
82
87
|
|
|
@@ -1,69 +1,87 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: run-benchmark
|
|
3
|
+
# story: e48s01
|
|
3
4
|
model: haiku
|
|
4
5
|
effort: standard
|
|
5
|
-
description: Run skill quality benchmarks from specs/benchmarks/ definitions
|
|
6
|
+
description: Run skill quality benchmarks from specs/benchmarks/ definitions — N-run with/without-skill delta grading, train/validation split, pass@k + benchmark.json reports. Use before and after evolve-skill to prove quality changes are improvements, not regressions.
|
|
6
7
|
---
|
|
7
8
|
|
|
8
9
|
# Run Benchmark
|
|
9
10
|
|
|
10
11
|
> **HARD GATE** — Do NOT use benchmark scores to declare a skill "good" or "bad" in isolation. Benchmarks measure relative quality vs. a baseline — they catch regressions, they do not certify correctness.
|
|
11
12
|
|
|
12
|
-
Reads benchmark definitions from `specs/benchmarks/`, executes each scenario's grader, and writes a structured `pass@k` report that `evolve-skill` consumes.
|
|
13
|
+
Reads benchmark definitions from `specs/benchmarks/`, executes each scenario's grader with and without the skill loaded, and writes a structured `pass@k` report with delta grading that `evolve-skill` consumes.
|
|
13
14
|
|
|
14
|
-
##
|
|
15
|
+
## With/Without-Skill Delta Grading
|
|
15
16
|
|
|
16
|
-
|
|
17
|
-
# Benchmark a single skill
|
|
18
|
-
run-benchmark <skill-name>
|
|
17
|
+
Every scenario runs N times (default 3) in two modes: **with** the skill loaded and **without** (bare agent with only CLAUDE.md). The delta `Δ = pass@k_with − pass@k_without` isolates the skill's causal contribution. A negative delta is a regression flag.
|
|
19
18
|
|
|
20
|
-
|
|
21
|
-
run-benchmark --all
|
|
19
|
+
## Train/Validation Split
|
|
22
20
|
|
|
23
|
-
|
|
24
|
-
run-benchmark <skill-name> --baseline
|
|
25
|
-
```
|
|
21
|
+
Benchmark definitions partition scenarios into two sets:
|
|
26
22
|
|
|
27
|
-
|
|
23
|
+
| Set | Tag | Purpose |
|
|
24
|
+
|-----|-----|---------|
|
|
25
|
+
| **Train** | `split: train` | Development scenarios — used while iterating. Hitting 100% on train is expected. |
|
|
26
|
+
| **Validation** | `split: validation` | Held-out scenarios — the real quality signal. Overfitting train while validation stagnates is a design smell. |
|
|
28
27
|
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
2. **Run each scenario** — For each scenario in `scenarios[]`:
|
|
32
|
-
- **Code grader:** Run `grader.command` in repo root via `bash -c`. Exit 0 → PASS. Non-zero → FAIL. Timeout: 15 seconds.
|
|
33
|
-
- **Rubric grader:** Present each criterion to the agent as a yes/no question about the scenario output. ≥ 80% yes → PASS, else FAIL.
|
|
34
|
-
|
|
35
|
-
3. **Calculate pass@k** — `pass@k = sum(weight of PASS scenarios) / sum(all weights)`. Round to 2 decimal places.
|
|
36
|
-
|
|
37
|
-
4. **Write report** to `specs/benchmarks/reports/BENCHMARK-<skill>-<YYYY-MM-DD>.yaml`:
|
|
38
|
-
|
|
39
|
-
```yaml
|
|
40
|
-
skill: survey-context
|
|
41
|
-
run_date: "2026-06-22"
|
|
42
|
-
pass_at_k: 0.83
|
|
43
|
-
total_scenarios: 3
|
|
44
|
-
passed: 2
|
|
45
|
-
failed: 1
|
|
46
|
-
scenarios:
|
|
47
|
-
- id: s01
|
|
48
|
-
name: "detects active epic from state.yaml"
|
|
49
|
-
result: PASS
|
|
50
|
-
weight: 1.0
|
|
51
|
-
- id: s02
|
|
52
|
-
name: "reads release-plan.yaml and reports next epic"
|
|
53
|
-
result: PASS
|
|
54
|
-
weight: 1.0
|
|
55
|
-
- id: s03
|
|
56
|
-
name: "handles missing state.yaml gracefully"
|
|
57
|
-
result: FAIL
|
|
58
|
-
weight: 0.5
|
|
59
|
-
failure_note: "crashed instead of suggesting state.yaml creation"
|
|
60
|
-
```
|
|
28
|
+
`pass@k` is reported separately for train and validation. Validation score is authoritative; train score is iteration guidance only.
|
|
61
29
|
|
|
62
|
-
|
|
30
|
+
## Usage
|
|
63
31
|
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
32
|
+
```bash
|
|
33
|
+
run-benchmark <skill-name> # benchmark single skill
|
|
34
|
+
run-benchmark --all # benchmark all with definitions
|
|
35
|
+
run-benchmark <skill-name> --baseline # pin results as baseline
|
|
36
|
+
```
|
|
68
37
|
|
|
38
|
+
## Process
|
|
69
39
|
|
|
40
|
+
1. **Locate definition** — Read `specs/benchmarks/<skill>.yaml`. If absent, stop with message.
|
|
41
|
+
|
|
42
|
+
2. **Partition scenarios** — Split by `split` field (`train` → iteration, `validation` → authoritative, default: `validation`).
|
|
43
|
+
|
|
44
|
+
3. **Run each scenario (N-run delta)** — For each scenario, run grader N times (default 3, configurable via `runs:`):
|
|
45
|
+
- **Without skill:** Agent with only CLAUDE.md/CONVENTIONS.md
|
|
46
|
+
- **With skill:** Agent with the skill under test active
|
|
47
|
+
- Code grader: `bash -c <command>`, exit 0 → PASS. Timeout: 15s.
|
|
48
|
+
- Rubric grader: yes/no per criterion, ≥ 80% yes → PASS.
|
|
49
|
+
- Record: `{scenario_id: {with: [P/F,...], without: [P/F,...]}}`
|
|
50
|
+
|
|
51
|
+
4. **Calculate scores** — Per split (train, validation) and mode (with, without):
|
|
52
|
+
- `pass@k = sum(weight × pass_rate) / sum(weights)` where `pass_rate = passes/runs`
|
|
53
|
+
- `Δ = pass@k_with − pass@k_without` — causal contribution
|
|
54
|
+
- Round to 2 decimal places
|
|
55
|
+
|
|
56
|
+
5. **Write benchmark.json** to `specs/benchmarks/reports/benchmark-<skill>.json`:
|
|
57
|
+
```json
|
|
58
|
+
{"skill":"survey-context","run_date":"2026-06-22","runs_per_scenario":3,"train":{"with_skill":0.92,"without_skill":0.67,"delta":0.25,"scenarios":["s01","s02"]},"validation":{"with_skill":0.83,"without_skill":0.60,"delta":0.23,"scenarios":["s03","s04","s05"]}}
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
6. **Write YAML report** to `specs/benchmarks/reports/BENCHMARK-<skill>-<YYYY-MM-DD>.yaml`:
|
|
62
|
+
```yaml
|
|
63
|
+
skill: survey-context
|
|
64
|
+
run_date: "2026-06-22"
|
|
65
|
+
runs_per_scenario: 3
|
|
66
|
+
train:
|
|
67
|
+
pass_at_k_with: 0.92
|
|
68
|
+
pass_at_k_without: 0.67
|
|
69
|
+
delta: 0.25
|
|
70
|
+
validation:
|
|
71
|
+
pass_at_k_with: 0.83
|
|
72
|
+
pass_at_k_without: 0.60
|
|
73
|
+
delta: 0.23
|
|
74
|
+
scenarios:
|
|
75
|
+
- id: s01
|
|
76
|
+
split: train
|
|
77
|
+
with_pass_rate: 1.0
|
|
78
|
+
without_pass_rate: 0.67
|
|
79
|
+
delta: 0.33
|
|
80
|
+
weight: 1.0
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
7. **Baseline** (`--baseline`) — Copy to `BASELINE-<skill>.yaml` + `baseline-<skill>.json`.
|
|
84
|
+
|
|
85
|
+
8. **Compare to baseline** — `IMPROVED: Δ 0.17 → 0.25` / `REGRESSION: Δ 0.25 → 0.17 — do NOT ship` / `STABLE`.
|
|
86
|
+
|
|
87
|
+
9. **Delta threshold gate** — Validation Δ < 0.0 blocks release. Δ < 0.05 warns (marginal). Min meaningful threshold: 0.05.
|
|
@@ -30,6 +30,10 @@ description: >
|
|
|
30
30
|
|
|
31
31
|
Covered: SQLi, XSS, SSRF, command injection, auth bypass, unsafe deserialization, path traversal, IDOR, crypto flaws, secrets exposure, template injection, NoSQLi
|
|
32
32
|
|
|
33
|
+
## BCP Plus Integration
|
|
34
|
+
|
|
35
|
+
This skill maps to **BCP Plus dimension 12 (Security & Compliance)**. When BCP Plus sizing is active, the threat model categories above correspond to sub-elements within dimension 12. The NFR Gate rule applies: standard-expectation items (e.g., "use HTTPS", "hash passwords") score 0 with a one-line rationale; only above-standard security requirements contribute to the dimension 12 count. See `docs/references/bcp-plus.md` for the full 13-dimension framework and NFR Gate pattern.
|
|
36
|
+
|
|
33
37
|
## Integration points
|
|
34
38
|
|
|
35
39
|
| Skill | Touchpoint |
|
|
@@ -20,6 +20,21 @@ Idempotent prep so BUILD phase commands succeed on first run.
|
|
|
20
20
|
5. Run smoke: lint + one fast test or `--version` on key tools.
|
|
21
21
|
6. Record versions in `specs/state.yaml` under Environment.
|
|
22
22
|
|
|
23
|
+
## BCP Plus Counter (optional)
|
|
24
|
+
|
|
25
|
+
The `big-counter` tool is an optional dependency for BCP Plus 13-dimension story sizing:
|
|
26
|
+
|
|
27
|
+
```bash
|
|
28
|
+
# Install from PyPI (recommended)
|
|
29
|
+
pip install big-counter
|
|
30
|
+
|
|
31
|
+
# Or from npm
|
|
32
|
+
npm install -g big-counter
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
Verify the install: `big-counter --version`
|
|
36
|
+
Skip if BCP Plus sizing is not needed for this project.
|
|
37
|
+
|
|
23
38
|
## Verify
|
|
24
39
|
|
|
25
40
|
→ verify: `test -f CLAUDE.md && grep -q 'Test' CLAUDE.md && echo "OK: CLAUDE.md has test table" || echo "MISSING: test commands"`
|
|
@@ -93,3 +93,7 @@ Place setup scripts in `scripts/setup.sh` (or language-appropriate equivalent).
|
|
|
93
93
|
- [ ] Health check command returns success
|
|
94
94
|
- [ ] No sensitive data in log output
|
|
95
95
|
|
|
96
|
+
## BCP Plus Integration
|
|
97
|
+
|
|
98
|
+
This skill maps to **BCP Plus dimension 11 (Quality Attributes)** — specifically performance, reliability, and availability instrumentation. Structured logging, health checks, and idempotent setup scripts are observable manifestations of quality-attribute requirements. The NFR Gate rule applies: standard-expectation items (e.g., "the app should log errors") score 0 with a one-line rationale; only above-standard quality requirements (e.g., p99 latency tracking, SLO-based alerting) contribute to the dimension 11 count. See `docs/references/bcp-plus.md` for the full 13-dimension framework and NFR Gate pattern.
|
|
99
|
+
|