bigpowers 2.73.3 → 2.73.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.pi/package.json +1 -1
- package/.pi/prompts/gate-trace.md +1 -1
- package/.pi/prompts/maintain-wiki.md +1 -1
- package/.pi/prompts/security-review.md +1 -1
- package/.pi/skills/gate-trace/SKILL.md +1 -1
- package/.pi/skills/maintain-wiki/SKILL.md +1 -1
- package/.pi/skills/security-review/SKILL.md +1 -1
- package/CHANGELOG.md +14 -0
- package/package.json +1 -1
- package/scripts/lib/golden-suite-agent.sh +80 -0
- package/scripts/lib/golden-suite-gates.sh +105 -0
- package/scripts/lib/golden-suite-report.sh +157 -0
- package/scripts/lib/golden-suite-run.sh +79 -0
- package/scripts/lib/skill-common.sh +21 -0
- package/scripts/lib/sync-post.sh +162 -0
- package/scripts/lib/sync-render.sh +126 -0
- package/scripts/lib/validate-okf-kinds.sh +272 -0
- package/scripts/okf-add-references.py +79 -0
- package/scripts/run-golden-suite.sh +4 -488
- package/scripts/run-verification-gates.sh +4 -488
- package/scripts/sync-skills.sh +12 -612
- package/scripts/validate-okf.sh +2 -318
package/.pi/package.json
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
---
|
|
2
|
-
description: "Deterministic traceability quality gate — reads coverage matrix + blind-spot data, applies decision rules with oracle confidence downgrade, emits PASS/CONCERNS/FAIL/WAIVED verdict. Use before release-branch to gate merges on traceability."
|
|
2
|
+
description: "Deterministic traceability quality gate — reads coverage matrix + blind-spot data, applies decision rules with oracle confidence downgrade, emits PASS/CONCERNS/FAIL/WAIVED verdict. Use before release-branch to gate merges on traceability."
|
|
3
3
|
---
|
|
4
4
|
|
|
5
5
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
---
|
|
2
|
-
description: "Agent-maintained OKF wiki — INGEST source docs, LINT for issues, QUERY across concept pages. Run as part of build-epic Step 8 and verify-work Phase 3."
|
|
2
|
+
description: "Agent-maintained OKF wiki — INGEST source docs, LINT for issues, QUERY across concept pages. Run as part of build-epic Step 8 and verify-work Phase 3."
|
|
3
3
|
---
|
|
4
4
|
|
|
5
5
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
---
|
|
2
|
-
description: >
|
|
2
|
+
description: >
|
|
3
3
|
---
|
|
4
4
|
|
|
5
5
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: gate-trace
|
|
3
|
-
description: "\"Deterministic traceability quality gate — reads coverage matrix + blind-spot data, applies decision rules with oracle confidence downgrade, emits PASS/CONCERNS/FAIL/WAIVED verdict. Use before release-branch to gate merges on traceability.\"
|
|
3
|
+
description: "\"Deterministic traceability quality gate — reads coverage matrix + blind-spot data, applies decision rules with oracle confidence downgrade, emits PASS/CONCERNS/FAIL/WAIVED verdict. Use before release-branch to gate merges on traceability.\""
|
|
4
4
|
model: haiku
|
|
5
5
|
---
|
|
6
6
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: maintain-wiki
|
|
3
|
-
description: "\"Agent-maintained OKF wiki — INGEST source docs, LINT for issues, QUERY across concept pages. Run as part of build-epic Step 8 and verify-work Phase 3.\"
|
|
3
|
+
description: "\"Agent-maintained OKF wiki — INGEST source docs, LINT for issues, QUERY across concept pages. Run as part of build-epic Step 8 and verify-work Phase 3.\""
|
|
4
4
|
model: haiku
|
|
5
5
|
---
|
|
6
6
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: security-review
|
|
3
|
-
description: ">
|
|
3
|
+
description: ">"
|
|
4
4
|
model: sonnet
|
|
5
5
|
---
|
|
6
6
|
|
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,17 @@
|
|
|
1
|
+
## [2.73.5](https://github.com/danielvm-git/bigpowers/compare/v2.73.4...v2.73.5) (2026-07-06)
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
### Bug Fixes
|
|
5
|
+
|
|
6
|
+
* **verify:** split golden suite and validate-okf into lib modules ([#60](https://github.com/danielvm-git/bigpowers/issues/60)) ([d0c1a7a](https://github.com/danielvm-git/bigpowers/commit/d0c1a7ae8811b37d532471a9eb758936964344f4))
|
|
7
|
+
|
|
8
|
+
## [2.73.4](https://github.com/danielvm-git/bigpowers/compare/v2.73.3...v2.73.4) (2026-07-06)
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
### Bug Fixes
|
|
12
|
+
|
|
13
|
+
* **sync:** split sync-skills.sh into lib modules under 300-line cap ([#59](https://github.com/danielvm-git/bigpowers/issues/59)) ([eed704f](https://github.com/danielvm-git/bigpowers/commit/eed704fe87a9f89dec211bbc6c6695afee884e52))
|
|
14
|
+
|
|
1
15
|
## [2.73.3](https://github.com/danielvm-git/bigpowers/compare/v2.73.2...v2.73.3) (2026-07-06)
|
|
2
16
|
|
|
3
17
|
|
package/package.json
CHANGED
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# story: e42s04
|
|
3
|
+
# Agent-driven golden story runner — sourced by golden-suite-run.sh
|
|
4
|
+
|
|
5
|
+
if [ -n "${GOLDEN_AGENT_LOADED:-}" ]; then return 0; fi
|
|
6
|
+
GOLDEN_AGENT_LOADED=1
|
|
7
|
+
|
|
8
|
+
golden_run_agent_stories() {
|
|
9
|
+
local yaml_file story_id PASS_K PASS_THRESHOLD story_pass story_fail flake_note LOCK_FILE
|
|
10
|
+
|
|
11
|
+
AGENT_PASS=0 AGENT_FAIL=0 AGENT_SKIP=0 AGENT_FLAKE=0 AGENT_TOTAL=0
|
|
12
|
+
AGENT_RESULTS=()
|
|
13
|
+
DETERMINISTIC_PASSED=false
|
|
14
|
+
[[ "$FAIL_COUNT" -eq 0 ]] && DETERMINISTIC_PASSED=true
|
|
15
|
+
|
|
16
|
+
if ! $AGENT_MODE; then return 0; fi
|
|
17
|
+
if ! $DETERMINISTIC_PASSED; then
|
|
18
|
+
echo ""
|
|
19
|
+
echo -e "${GOLDEN_YELLOW}── Agent stories skipped: deterministic gates failed ──${GOLDEN_NC}"
|
|
20
|
+
AGENT_RESULTS+=(" status: skipped" " reason: deterministic gates failed")
|
|
21
|
+
return 0
|
|
22
|
+
fi
|
|
23
|
+
|
|
24
|
+
echo ""
|
|
25
|
+
echo "── Agent-Driven Golden Stories ──"
|
|
26
|
+
|
|
27
|
+
if [[ ! -d "$GOLDEN_DIR" ]]; then
|
|
28
|
+
echo -e "${GOLDEN_YELLOW}WARN: $GOLDEN_DIR not found — agent stories skipped${GOLDEN_NC}"
|
|
29
|
+
AGENT_RESULTS+=(" status: skipped" " reason: golden directory not found")
|
|
30
|
+
(( AGENT_SKIP += 1 ))
|
|
31
|
+
return 0
|
|
32
|
+
fi
|
|
33
|
+
|
|
34
|
+
for yaml_file in "$GOLDEN_DIR"/g-*.yaml; do
|
|
35
|
+
[[ -f "$yaml_file" ]] || continue
|
|
36
|
+
story_id=$(basename "$yaml_file" .yaml)
|
|
37
|
+
echo -n "[$story_id] "
|
|
38
|
+
|
|
39
|
+
PASS_K=$($PYTHON -c "import yaml; d=yaml.safe_load(open('$yaml_file')); print(d.get('pass_at_k',{}).get('k',3))" 2>/dev/null || echo 3)
|
|
40
|
+
PASS_THRESHOLD=$($PYTHON -c "import yaml; d=yaml.safe_load(open('$yaml_file')); print(d.get('pass_at_k',{}).get('threshold',2))" 2>/dev/null || echo 2)
|
|
41
|
+
|
|
42
|
+
story_pass=0 story_fail=0
|
|
43
|
+
LOCK_FILE=".github/workflows/e42-golden-deepseek.lock.yml"
|
|
44
|
+
|
|
45
|
+
for ((attempt=1; attempt<=PASS_K; attempt++)); do
|
|
46
|
+
if [[ ! -f "$LOCK_FILE" ]]; then
|
|
47
|
+
echo -e "${GOLDEN_YELLOW}SKIP${GOLDEN_NC} (no compiled workflow for $story_id)"
|
|
48
|
+
break
|
|
49
|
+
fi
|
|
50
|
+
if $PYTHON -c "
|
|
51
|
+
import yaml
|
|
52
|
+
d = yaml.safe_load(open('$yaml_file'))
|
|
53
|
+
assert d.get('grader',{}).get('type') == 'code', 'grader must be code'
|
|
54
|
+
assert d['grader'].get('command'), 'grader command required'
|
|
55
|
+
assert d.get('pass_at_k',{}).get('k') == $PASS_K
|
|
56
|
+
assert d['pass_at_k'].get('threshold') == $PASS_THRESHOLD
|
|
57
|
+
" 2>/dev/null; then
|
|
58
|
+
(( story_pass += 1 ))
|
|
59
|
+
else
|
|
60
|
+
(( story_fail += 1 ))
|
|
61
|
+
fi
|
|
62
|
+
done
|
|
63
|
+
|
|
64
|
+
if [[ "$story_pass" -eq 0 && "$story_fail" -eq 0 ]]; then
|
|
65
|
+
echo -e "${GOLDEN_YELLOW}SKIP${GOLDEN_NC}"
|
|
66
|
+
AGENT_RESULTS+=(" - id: $story_id" " status: skipped" " pass_at_k: \"0/$PASS_K\"")
|
|
67
|
+
(( AGENT_SKIP += 1 ))
|
|
68
|
+
elif [[ "$story_pass" -ge "$PASS_THRESHOLD" ]]; then
|
|
69
|
+
flake_note=""
|
|
70
|
+
[[ "$story_fail" -gt 0 ]] && flake_note=" (flake: $story_fail)" && (( AGENT_FLAKE += story_fail ))
|
|
71
|
+
echo -e "${GOLDEN_GREEN}PASS${GOLDEN_NC}${flake_note} (pass@k: $story_pass/$PASS_K)"
|
|
72
|
+
AGENT_RESULTS+=(" - id: $story_id" " status: pass" " pass_at_k: \"$story_pass/$PASS_K\"" " flake_count: $story_fail")
|
|
73
|
+
(( AGENT_PASS += 1 ))
|
|
74
|
+
else
|
|
75
|
+
echo -e "${GOLDEN_RED}FAIL${GOLDEN_NC} (pass@k: $story_pass/$PASS_K, threshold $PASS_THRESHOLD)"
|
|
76
|
+
AGENT_RESULTS+=(" - id: $story_id" " status: fail" " pass_at_k: \"$story_pass/$PASS_K\"" " flake_count: $story_fail")
|
|
77
|
+
(( AGENT_FAIL += 1 ))
|
|
78
|
+
fi
|
|
79
|
+
done
|
|
80
|
+
}
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# story: e42s04 e45s02
|
|
3
|
+
# Deterministic gate runner for golden suite — sourced by golden-suite-run.sh
|
|
4
|
+
|
|
5
|
+
if [ -n "${GOLDEN_GATES_LOADED:-}" ]; then return 0; fi
|
|
6
|
+
GOLDEN_GATES_LOADED=1
|
|
7
|
+
|
|
8
|
+
GOLDEN_GATES=(
|
|
9
|
+
"compliance:npm run compliance:false"
|
|
10
|
+
"g04-selftest:bash scripts/golden-g04-selftest.sh:true"
|
|
11
|
+
"g06-migrate-version:bash tests/run-golden-migrate-version.sh:true"
|
|
12
|
+
"g07-negative-path:bash scripts/golden-g07-negative-path.sh:false"
|
|
13
|
+
"g08-anti-vacuity:bash scripts/golden-g08-anti-vacuity.sh:false"
|
|
14
|
+
"g09-yaml-roundtrip:bash scripts/golden-g09-yaml-roundtrip.sh:false"
|
|
15
|
+
"g10-trace-anti-vacuity:bash scripts/golden-g10-trace-anti-vacuity.sh:false"
|
|
16
|
+
"g11-gitignore-venv:bash scripts/golden-g11-gitignore-venv.sh:false"
|
|
17
|
+
"specs-parse:bash scripts/validate-specs-yaml.sh:false"
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
GOLDEN_GREEN='\033[0;32m'
|
|
21
|
+
GOLDEN_RED='\033[0;31m'
|
|
22
|
+
GOLDEN_YELLOW='\033[0;33m'
|
|
23
|
+
GOLDEN_NC='\033[0m'
|
|
24
|
+
|
|
25
|
+
golden_agent_dry_run() {
|
|
26
|
+
local yaml_count=0 yaml_ok=0 yaml_fail=0 yaml_file story_id
|
|
27
|
+
|
|
28
|
+
echo "Agent Golden Stories — DRY RUN"
|
|
29
|
+
echo "Golden YAML directory: $GOLDEN_DIR"
|
|
30
|
+
|
|
31
|
+
if [[ ! -d "$GOLDEN_DIR" ]]; then
|
|
32
|
+
echo -e "${GOLDEN_YELLOW}WARN: $GOLDEN_DIR not found${GOLDEN_NC}"
|
|
33
|
+
return 0
|
|
34
|
+
fi
|
|
35
|
+
|
|
36
|
+
for yaml_file in "$GOLDEN_DIR"/g-*.yaml; do
|
|
37
|
+
[[ -f "$yaml_file" ]] || continue
|
|
38
|
+
(( yaml_count += 1 ))
|
|
39
|
+
story_id=$(basename "$yaml_file" .yaml)
|
|
40
|
+
|
|
41
|
+
if $PYTHON -c "import yaml; yaml.safe_load(open('$yaml_file'))" 2>/dev/null; then
|
|
42
|
+
local grader_type pass_k
|
|
43
|
+
grader_type=$($PYTHON -c "import yaml; d=yaml.safe_load(open('$yaml_file')); print(d.get('grader',{}).get('type',''))" 2>/dev/null)
|
|
44
|
+
pass_k=$($PYTHON -c "import yaml; d=yaml.safe_load(open('$yaml_file')); pk=d.get('pass_at_k',{}); print(f\"{pk.get('k',0)}-of-{pk.get('threshold',0)}\")" 2>/dev/null)
|
|
45
|
+
echo -e " ${GOLDEN_GREEN}OK${GOLDEN_NC} $story_id — grader=$grader_type, pass@k=$pass_k"
|
|
46
|
+
(( yaml_ok += 1 ))
|
|
47
|
+
else
|
|
48
|
+
echo -e " ${GOLDEN_RED}FAIL${GOLDEN_NC} $story_id — YAML parse error"
|
|
49
|
+
(( yaml_fail += 1 ))
|
|
50
|
+
fi
|
|
51
|
+
done
|
|
52
|
+
|
|
53
|
+
echo ""
|
|
54
|
+
echo "Agent YAML validation: $yaml_ok/$yaml_count parseable"
|
|
55
|
+
if [[ "$yaml_fail" -gt 0 ]]; then
|
|
56
|
+
echo -e "${GOLDEN_RED}$yaml_fail YAML(s) failed to parse${GOLDEN_NC}"
|
|
57
|
+
return 1
|
|
58
|
+
fi
|
|
59
|
+
return 0
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
golden_run_deterministic_gates() {
|
|
63
|
+
local gate name cmd is_optional main_cmd script_path
|
|
64
|
+
PASS_COUNT=0 FAIL_COUNT=0 SKIP_COUNT=0
|
|
65
|
+
GATE_RESULTS=()
|
|
66
|
+
|
|
67
|
+
for gate in "${GOLDEN_GATES[@]}"; do
|
|
68
|
+
IFS=':' read -r name cmd is_optional <<< "$gate"
|
|
69
|
+
echo -n "[$name] "
|
|
70
|
+
|
|
71
|
+
main_cmd=$(echo "$cmd" | awk '{print $1}')
|
|
72
|
+
if [[ "$main_cmd" == "bash" ]]; then
|
|
73
|
+
script_path=$(echo "$cmd" | awk '{print $2}')
|
|
74
|
+
if [[ ! -f "$script_path" ]]; then
|
|
75
|
+
if [[ "$is_optional" == "true" ]]; then
|
|
76
|
+
echo -e "${GOLDEN_YELLOW}SKIP${GOLDEN_NC} (script not yet implemented)"
|
|
77
|
+
GATE_RESULTS+=(" - name: $name" " exit_code: skipped" " duration_seconds: 0" " status: skipped" " note: script not found: $script_path")
|
|
78
|
+
(( SKIP_COUNT += 1 ))
|
|
79
|
+
continue
|
|
80
|
+
fi
|
|
81
|
+
echo -e "${GOLDEN_RED}FAIL${GOLDEN_NC} (script not found: $script_path)"
|
|
82
|
+
GATE_RESULTS+=(" - name: $name" " exit_code: 127" " duration_seconds: 0" " status: fail" " error: script not found: $script_path")
|
|
83
|
+
(( FAIL_COUNT += 1 ))
|
|
84
|
+
continue
|
|
85
|
+
fi
|
|
86
|
+
fi
|
|
87
|
+
|
|
88
|
+
local START_TIME END_TIME DURATION_MS DURATION_SEC GATE_EXIT
|
|
89
|
+
START_TIME=$(date +%s%N)
|
|
90
|
+
eval "$cmd" >/dev/null 2>&1 && GATE_EXIT=$? || GATE_EXIT=$?
|
|
91
|
+
END_TIME=$(date +%s%N)
|
|
92
|
+
DURATION_MS=$(( (END_TIME - START_TIME) / 1000000 ))
|
|
93
|
+
DURATION_SEC=$(echo "scale=2; $DURATION_MS / 1000" | bc 2>/dev/null || echo "0")
|
|
94
|
+
|
|
95
|
+
if [[ "$GATE_EXIT" -eq 0 ]]; then
|
|
96
|
+
echo -e "${GOLDEN_GREEN}PASS${GOLDEN_NC} (${DURATION_SEC}s)"
|
|
97
|
+
GATE_RESULTS+=(" - name: $name" " exit_code: 0" " duration_seconds: $DURATION_SEC" " status: pass")
|
|
98
|
+
(( PASS_COUNT += 1 ))
|
|
99
|
+
else
|
|
100
|
+
echo -e "${GOLDEN_RED}FAIL${GOLDEN_NC} (${DURATION_SEC}s, exit $GATE_EXIT)"
|
|
101
|
+
GATE_RESULTS+=(" - name: $name" " exit_code: $GATE_EXIT" " duration_seconds: $DURATION_SEC" " status: fail")
|
|
102
|
+
(( FAIL_COUNT += 1 ))
|
|
103
|
+
fi
|
|
104
|
+
done
|
|
105
|
+
}
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# story: e31s04 e45s02
|
|
3
|
+
# Golden suite reporting — verdict, YAML/OKF output, size budget
|
|
4
|
+
|
|
5
|
+
if [ -n "${GOLDEN_REPORT_LOADED:-}" ]; then return 0; fi
|
|
6
|
+
GOLDEN_REPORT_LOADED=1
|
|
7
|
+
|
|
8
|
+
golden_compute_verdict() {
|
|
9
|
+
TOTAL=$((PASS_COUNT + FAIL_COUNT + SKIP_COUNT))
|
|
10
|
+
AGENT_TOTAL=$((AGENT_PASS + AGENT_FAIL + AGENT_SKIP))
|
|
11
|
+
if [[ "$FAIL_COUNT" -gt 0 || "$AGENT_FAIL" -gt 0 ]]; then OVERALL="fail"; else OVERALL="pass"; fi
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
golden_print_verdict() {
|
|
15
|
+
golden_compute_verdict
|
|
16
|
+
echo ""
|
|
17
|
+
echo "══════════════════════════════════════════"
|
|
18
|
+
echo " GOLDEN SUITE VERDICT"
|
|
19
|
+
echo "══════════════════════════════════════════"
|
|
20
|
+
echo ""
|
|
21
|
+
echo "Deterministic gates:"
|
|
22
|
+
echo -e " ${PASS_COUNT}/${TOTAL} passed"
|
|
23
|
+
[[ "$SKIP_COUNT" -gt 0 ]] && echo -e " ${SKIP_COUNT} skipped"
|
|
24
|
+
[[ "$FAIL_COUNT" -gt 0 ]] && echo -e " ${GOLDEN_RED}${FAIL_COUNT} failed${GOLDEN_NC}"
|
|
25
|
+
|
|
26
|
+
if $AGENT_MODE; then
|
|
27
|
+
echo ""
|
|
28
|
+
echo "Agent-driven stories:"
|
|
29
|
+
echo -e " ${AGENT_PASS}/${AGENT_TOTAL} passed"
|
|
30
|
+
[[ "$AGENT_FAIL" -gt 0 ]] && echo -e " ${GOLDEN_RED}${AGENT_FAIL} failed${GOLDEN_NC}"
|
|
31
|
+
[[ "$AGENT_SKIP" -gt 0 ]] && echo -e " ${AGENT_SKIP} skipped"
|
|
32
|
+
[[ "$AGENT_FLAKE" -gt 0 ]] && echo -e " ${GOLDEN_YELLOW}${AGENT_FLAKE} flake(s) detected${GOLDEN_NC} (absorbed by pass@k 2-of-3)"
|
|
33
|
+
fi
|
|
34
|
+
|
|
35
|
+
echo ""
|
|
36
|
+
echo -e "Combined verdict: $([[ "$OVERALL" == "pass" ]] && echo -e "${GOLDEN_GREEN}PASS${GOLDEN_NC}" || echo -e "${GOLDEN_RED}FAIL${GOLDEN_NC}")"
|
|
37
|
+
echo "Report: $REPORT_FILE"
|
|
38
|
+
echo "══════════════════════════════════════════"
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
golden_write_reports() {
|
|
42
|
+
local PASS_RATE AGENT_PASS_RATE OKF_DIR OKF_FILE THRESHOLD GATE_STATUS mode
|
|
43
|
+
|
|
44
|
+
PASS_RATE=$(echo "scale=2; $PASS_COUNT / $TOTAL" | bc 2>/dev/null || echo "0")
|
|
45
|
+
AGENT_PASS_RATE="n/a"
|
|
46
|
+
[[ "$AGENT_TOTAL" -gt 0 ]] && AGENT_PASS_RATE=$(echo "scale=2; $AGENT_PASS / $AGENT_TOTAL" | bc 2>/dev/null || echo "0")
|
|
47
|
+
mode=$([ "$BASELINE_MODE" = true ] && echo 'baseline' || echo 'daily')
|
|
48
|
+
|
|
49
|
+
cat > "$REPORT_FILE" << YAML_EOF
|
|
50
|
+
timestamp: "$TIMESTAMP"
|
|
51
|
+
git_commit: "$GIT_COMMIT"
|
|
52
|
+
suite: "golden-combined"
|
|
53
|
+
mode: "$mode"
|
|
54
|
+
deterministic:
|
|
55
|
+
total: $TOTAL
|
|
56
|
+
passed: $PASS_COUNT
|
|
57
|
+
failed: $FAIL_COUNT
|
|
58
|
+
skipped: $SKIP_COUNT
|
|
59
|
+
pass_rate: $PASS_RATE
|
|
60
|
+
gates:
|
|
61
|
+
$(printf '%s\n' "${GATE_RESULTS[@]}")
|
|
62
|
+
agent_stories:
|
|
63
|
+
total: $AGENT_TOTAL
|
|
64
|
+
passed: $AGENT_PASS
|
|
65
|
+
failed: $AGENT_FAIL
|
|
66
|
+
skipped: $AGENT_SKIP
|
|
67
|
+
flake_count: $AGENT_FLAKE
|
|
68
|
+
pass_rate: $AGENT_PASS_RATE
|
|
69
|
+
summary:
|
|
70
|
+
combined_verdict: "$OVERALL"
|
|
71
|
+
YAML_EOF
|
|
72
|
+
|
|
73
|
+
OKF_DIR="specs/verifications/reports"
|
|
74
|
+
mkdir -p "$OKF_DIR"
|
|
75
|
+
OKF_FILE="$OKF_DIR/GOLDEN-${TODAY}.okf.md"
|
|
76
|
+
THRESHOLD=100
|
|
77
|
+
GATE_STATUS="$OVERALL"
|
|
78
|
+
|
|
79
|
+
cat > "$OKF_FILE" << OKF_EOF
|
|
80
|
+
---
|
|
81
|
+
okf_kind: verification-report
|
|
82
|
+
okf_version: "0.1"
|
|
83
|
+
score: ${PASS_RATE}
|
|
84
|
+
gate_status: "${GATE_STATUS}"
|
|
85
|
+
threshold: ${THRESHOLD}
|
|
86
|
+
total_pass: ${PASS_COUNT}
|
|
87
|
+
total_fail: ${FAIL_COUNT}
|
|
88
|
+
total_skip: ${SKIP_COUNT}
|
|
89
|
+
generated_by: ${GOLDEN_GENERATED_BY}
|
|
90
|
+
generated_at: "${TIMESTAMP}"
|
|
91
|
+
git_commit: "${GIT_COMMIT}"
|
|
92
|
+
---
|
|
93
|
+
|
|
94
|
+
# Golden Suite — ${TODAY}
|
|
95
|
+
|
|
96
|
+
**Score:** ${PASS_RATE} (${PASS_COUNT}/${TOTAL} passed)
|
|
97
|
+
**Gate:** ${GATE_STATUS} | **Threshold:** ${THRESHOLD}%
|
|
98
|
+
|
|
99
|
+
See \`${REPORT_FILE}\` for the full YAML report.
|
|
100
|
+
OKF_EOF
|
|
101
|
+
|
|
102
|
+
echo "OKF report: $OKF_FILE"
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
golden_write_baseline_sizes() {
|
|
106
|
+
local skill_md skill_name skill_bytes
|
|
107
|
+
echo "" >> "$REPORT_FILE"
|
|
108
|
+
echo "# Skill byte counts (baseline for e31s04 size budget)" >> "$REPORT_FILE"
|
|
109
|
+
echo "skill_sizes:" >> "$REPORT_FILE"
|
|
110
|
+
for skill_md in skills/*/SKILL.md; do
|
|
111
|
+
skill_name=$(basename "$(dirname "$skill_md")")
|
|
112
|
+
skill_bytes=$(wc -c < "$skill_md" | tr -d ' ')
|
|
113
|
+
echo " $skill_name: $skill_bytes" >> "$REPORT_FILE"
|
|
114
|
+
done
|
|
115
|
+
echo "Baseline pinned: $REPORT_FILE"
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
golden_check_size_budget() {
|
|
119
|
+
local WARN_THRESHOLD=20 FAIL_THRESHOLD=50 skill_md skill_name current_bytes baseline_bytes pct
|
|
120
|
+
local SIZE_WARNINGS=0 SIZE_FAILURES=0 NEW_SKILLS=0
|
|
121
|
+
|
|
122
|
+
if [[ ! -f "$BASELINE_FILE" ]]; then
|
|
123
|
+
echo "No baseline found. Run --baseline first."
|
|
124
|
+
return 1
|
|
125
|
+
fi
|
|
126
|
+
|
|
127
|
+
echo ""
|
|
128
|
+
echo "Size Budget Check (+${WARN_THRESHOLD}% warn, +${FAIL_THRESHOLD}% fail):"
|
|
129
|
+
for skill_md in skills/*/SKILL.md; do
|
|
130
|
+
skill_name=$(basename "$(dirname "$skill_md")")
|
|
131
|
+
current_bytes=$(wc -c < "$skill_md" | tr -d ' ')
|
|
132
|
+
baseline_bytes=$(grep "^ $skill_name:" "$BASELINE_FILE" 2>/dev/null | awk '{print $2}')
|
|
133
|
+
|
|
134
|
+
if [[ -z "$baseline_bytes" ]]; then
|
|
135
|
+
echo -e " ${GOLDEN_YELLOW}NEW${GOLDEN_NC} $skill_name: ${current_bytes}b (not in baseline)"
|
|
136
|
+
(( NEW_SKILLS += 1 )) || true
|
|
137
|
+
continue
|
|
138
|
+
fi
|
|
139
|
+
[[ "$baseline_bytes" -eq 0 ]] && baseline_bytes=1
|
|
140
|
+
pct=$(( (current_bytes - baseline_bytes) * 100 / baseline_bytes ))
|
|
141
|
+
|
|
142
|
+
if [[ "$pct" -ge "$FAIL_THRESHOLD" ]]; then
|
|
143
|
+
echo -e " ${GOLDEN_RED}FAIL${GOLDEN_NC} $skill_name: ${current_bytes}b (+${pct}%)"
|
|
144
|
+
(( SIZE_FAILURES += 1 )) || true
|
|
145
|
+
elif [[ "$pct" -ge "$WARN_THRESHOLD" ]]; then
|
|
146
|
+
echo -e " ${GOLDEN_YELLOW}WARN${GOLDEN_NC} $skill_name: ${current_bytes}b (+${pct}%)"
|
|
147
|
+
(( SIZE_WARNINGS += 1 )) || true
|
|
148
|
+
else
|
|
149
|
+
echo -e " ${GOLDEN_GREEN}OK${GOLDEN_NC} $skill_name: ${current_bytes}b"
|
|
150
|
+
fi
|
|
151
|
+
done
|
|
152
|
+
|
|
153
|
+
echo ""
|
|
154
|
+
echo "Size summary: $SIZE_WARNINGS warnings, $SIZE_FAILURES failures, $NEW_SKILLS new"
|
|
155
|
+
[[ "$SIZE_FAILURES" -gt 0 ]] && return 1
|
|
156
|
+
return 0
|
|
157
|
+
}
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# story: e42s04 e45s02
|
|
3
|
+
# Golden suite orchestrator — sourced by run-verification-gates.sh / run-golden-suite.sh
|
|
4
|
+
|
|
5
|
+
if [ -n "${GOLDEN_SUITE_RUN_LOADED:-}" ]; then return 0; fi
|
|
6
|
+
GOLDEN_SUITE_RUN_LOADED=1
|
|
7
|
+
|
|
8
|
+
_LIB="$(dirname "${BASH_SOURCE[0]}")"
|
|
9
|
+
source "$_LIB/golden-suite-gates.sh"
|
|
10
|
+
source "$_LIB/golden-suite-agent.sh"
|
|
11
|
+
source "$_LIB/golden-suite-report.sh"
|
|
12
|
+
|
|
13
|
+
golden_suite_main() {
|
|
14
|
+
local arg gate name cmd optional
|
|
15
|
+
|
|
16
|
+
REPORT_DIR="specs/benchmarks/reports"
|
|
17
|
+
BASELINE_FILE="$REPORT_DIR/BASELINE-GOLDEN.yaml"
|
|
18
|
+
GOLDEN_DIR="specs/benchmarks/golden"
|
|
19
|
+
DRY_RUN=false BASELINE_MODE=false CHECK_SIZE=false AGENT_MODE=false
|
|
20
|
+
|
|
21
|
+
for arg in "$@"; do
|
|
22
|
+
case "$arg" in
|
|
23
|
+
--dry-run) DRY_RUN=true ;;
|
|
24
|
+
--baseline) BASELINE_MODE=true ;;
|
|
25
|
+
--check-size) CHECK_SIZE=true ;;
|
|
26
|
+
--agent) AGENT_MODE=true ;;
|
|
27
|
+
*)
|
|
28
|
+
echo "Unknown flag: $arg"
|
|
29
|
+
echo "Usage: ${GOLDEN_USAGE_NAME} [--agent] [--dry-run] [--baseline] [--check-size]"
|
|
30
|
+
exit 2
|
|
31
|
+
;;
|
|
32
|
+
esac
|
|
33
|
+
done
|
|
34
|
+
|
|
35
|
+
if $AGENT_MODE && ! command -v gh-aw &>/dev/null && ! command -v gh &>/dev/null; then
|
|
36
|
+
echo -e "${GOLDEN_YELLOW}WARNING: gh-aw not installed — agent gate skipped${GOLDEN_NC}" >&2
|
|
37
|
+
AGENT_MODE=false
|
|
38
|
+
fi
|
|
39
|
+
|
|
40
|
+
if $DRY_RUN; then
|
|
41
|
+
if $AGENT_MODE; then
|
|
42
|
+
echo "Golden Suite — DRY RUN (deterministic + agent)"
|
|
43
|
+
echo ""
|
|
44
|
+
echo "Deterministic gates (in order):"
|
|
45
|
+
for gate in "${GOLDEN_GATES[@]}"; do
|
|
46
|
+
IFS=':' read -r name cmd optional <<< "$gate"
|
|
47
|
+
echo " - $name: $cmd"
|
|
48
|
+
done
|
|
49
|
+
echo ""
|
|
50
|
+
golden_agent_dry_run
|
|
51
|
+
exit $?
|
|
52
|
+
fi
|
|
53
|
+
echo "Golden Suite — DRY RUN"
|
|
54
|
+
echo "Gates (in order):"
|
|
55
|
+
for gate in "${GOLDEN_GATES[@]}"; do
|
|
56
|
+
IFS=':' read -r name cmd optional <<< "$gate"
|
|
57
|
+
echo " - $name: $cmd"
|
|
58
|
+
done
|
|
59
|
+
echo "Report: $REPORT_DIR/GOLDEN-YYYY-MM-DD.yaml"
|
|
60
|
+
echo "OK (dry run)"
|
|
61
|
+
exit 0
|
|
62
|
+
fi
|
|
63
|
+
|
|
64
|
+
mkdir -p "$REPORT_DIR"
|
|
65
|
+
TODAY=$(date +%Y-%m-%d)
|
|
66
|
+
TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
|
67
|
+
GIT_COMMIT=$(git rev-parse HEAD 2>/dev/null || echo "unknown")
|
|
68
|
+
if $BASELINE_MODE; then REPORT_FILE="$BASELINE_FILE"; else REPORT_FILE="$REPORT_DIR/GOLDEN-${TODAY}.yaml"; fi
|
|
69
|
+
|
|
70
|
+
golden_run_deterministic_gates
|
|
71
|
+
golden_run_agent_stories
|
|
72
|
+
golden_print_verdict
|
|
73
|
+
golden_write_reports
|
|
74
|
+
|
|
75
|
+
if $BASELINE_MODE; then golden_write_baseline_sizes; fi
|
|
76
|
+
if $CHECK_SIZE; then golden_check_size_budget; exit $?; fi
|
|
77
|
+
[[ "$OVERALL" == "fail" ]] && exit 1
|
|
78
|
+
exit 0
|
|
79
|
+
}
|
|
@@ -91,6 +91,27 @@ parse_frontmatter() {
|
|
|
91
91
|
export SKILL_NAME SKILL_MODEL SKILL_DESC _PF_NAME _PF_MODEL _PF_DESCRIPTION
|
|
92
92
|
}
|
|
93
93
|
|
|
94
|
+
# parse_frontmatter_okf — like parse_frontmatter but also extracts effort field
|
|
95
|
+
parse_frontmatter_okf() {
|
|
96
|
+
local file="$1"
|
|
97
|
+
_PF_EFFORT=""
|
|
98
|
+
|
|
99
|
+
if [[ ! -f "$file" ]] || ! grep -q '^---$' "$file"; then
|
|
100
|
+
return 1
|
|
101
|
+
fi
|
|
102
|
+
|
|
103
|
+
_PF_NAME=$(awk '/^---/{f++} f==1 && /^name:/{print; exit}' "$file" | sed 's/^name:[[:space:]]*//')
|
|
104
|
+
_PF_MODEL=$(awk '/^---/{f++} f==1 && /^model:/{print; exit}' "$file" | sed 's/^model:[[:space:]]*//')
|
|
105
|
+
_PF_EFFORT=$(awk '/^---/{f++} f==1 && /^effort:/{print; exit}' "$file" | sed 's/^effort:[[:space:]]*//')
|
|
106
|
+
_PF_DESCRIPTION=$(awk '/^---/{f++; next} f==1 && /^description:/{p=1; sub(/^description:[[:space:]]*/,""); print; next} f==1 && p && /^[a-z]+:/{exit} f==1 && p{print}' "$file" \
|
|
107
|
+
| tr -d '\n' \
|
|
108
|
+
| sed -E 's/[[:space:]]+/ /g')
|
|
109
|
+
|
|
110
|
+
[[ -z "$_PF_NAME" ]] && return 1
|
|
111
|
+
export _PF_NAME _PF_MODEL _PF_EFFORT _PF_DESCRIPTION
|
|
112
|
+
return 0
|
|
113
|
+
}
|
|
114
|
+
|
|
94
115
|
# iterate_skills — list skill directories containing SKILL.md, stable sorted order
|
|
95
116
|
iterate_skills() {
|
|
96
117
|
resolve_repo_root 2>/dev/null || true
|