@ericrisco/rsc 1.0.6 → 1.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/manifest.json +1 -1
- package/package.json +1 -1
- package/scripts/lib/behavior-score.js +58 -1
- package/scripts/skill-behavior-eval.js +15 -5
- package/scripts/skill-harden-rubric.md +69 -7
- package/scripts/skill-harden.workflow.js +116 -12
- package/skills/analyze/SKILL.md +22 -0
- package/skills/clarify/SKILL.md +22 -0
- package/skills/constitution/SKILL.md +22 -0
- package/skills/plan/SKILL.md +22 -0
- package/skills/review/SKILL.md +22 -0
package/manifest.json
CHANGED
package/package.json
CHANGED
|
@@ -63,6 +63,60 @@ export function behavioralGate(agg) {
|
|
|
63
63
|
return { pass, reasons, mustFix };
|
|
64
64
|
}
|
|
65
65
|
|
|
66
|
+
// The hold-out gate — does a fix TRANSFER, or was it memorized?
|
|
67
|
+
//
|
|
68
|
+
// The fix loop scores the edited skill against a FRESH scenario the fixer never saw. Because the
|
|
69
|
+
// eval engine always runs that scenario twice (with and without the skill), the hold-out already
|
|
70
|
+
// carries its own lift: no pre-fix baseline is needed.
|
|
71
|
+
//
|
|
72
|
+
// The bar is deliberately NOT the main gate's LIFT_MIN. One fresh scenario graded once is noisy, so
|
|
73
|
+
// only the unambiguous signal acts: the edited skill failing to beat the bare agent on unseen work.
|
|
74
|
+
// Anything positive transfers. Raising this to LIFT_MIN would reject good fixes on variance alone
|
|
75
|
+
// (see 02-DOCS/wiki/sdd/specs/generalization-gate.md).
|
|
76
|
+
export function holdoutGate(scored) {
|
|
77
|
+
const agg = scored && scored.aggregate;
|
|
78
|
+
if (!agg || agg.n === 0 || typeof agg.lift !== 'number') {
|
|
79
|
+
return {
|
|
80
|
+
verdict: 'block',
|
|
81
|
+
kind: 'indeterminate',
|
|
82
|
+
lift: null,
|
|
83
|
+
reason: 'Hold-out could not be scored (no gradeable fresh scenario). Failing closed: an unverified edit is not a verified one.',
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
if (agg.lift <= 0) {
|
|
87
|
+
return {
|
|
88
|
+
verdict: 'block',
|
|
89
|
+
kind: 'regression',
|
|
90
|
+
lift: agg.lift,
|
|
91
|
+
reason: `Hold-out lift ${agg.lift} <= 0: on a scenario the fixer never saw, the edited skill does not beat no-skill. The edit did not transfer.`,
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
return {
|
|
95
|
+
verdict: 'pass',
|
|
96
|
+
kind: 'transfer',
|
|
97
|
+
lift: agg.lift,
|
|
98
|
+
reason: `Hold-out lift ${agg.lift} > 0: the edit still helps on a scenario the fixer never saw.`,
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
export function formatHoldoutVerdict(scored, gate) {
|
|
103
|
+
const agg = (scored && scored.aggregate) || {};
|
|
104
|
+
const lines = [];
|
|
105
|
+
lines.push(`# Hold-out verdict — ${(scored && scored.skillId) || '(unknown skill)'}`);
|
|
106
|
+
lines.push('');
|
|
107
|
+
lines.push(`**${gate.verdict === 'pass' ? 'PASS ✅' : 'BLOCK ❌'}** · ${gate.kind} · lift ${gate.lift == null ? 'n/a' : gate.lift} · absolute ${agg.absoluteScore == null ? 'n/a' : agg.absoluteScore} · n=${agg.n || 0}${agg.dropped ? ` (${agg.dropped} dropped)` : ''}`);
|
|
108
|
+
lines.push('');
|
|
109
|
+
lines.push(gate.reason);
|
|
110
|
+
// The fresh scenario is generated by the same model that fixes. Echoing it here is the only way to
|
|
111
|
+
// audit by hand whether the generator went easy on itself — a risk the spec accepts, not solves.
|
|
112
|
+
if (scored && scored.holdoutScenario) {
|
|
113
|
+
lines.push('');
|
|
114
|
+
lines.push('**Fresh scenario used:**');
|
|
115
|
+
lines.push(`> ${String(scored.holdoutScenario).replace(/\n/g, '\n> ')}`);
|
|
116
|
+
}
|
|
117
|
+
return lines.join('\n');
|
|
118
|
+
}
|
|
119
|
+
|
|
66
120
|
// raw: { skillId, scenarios: [{index, xIsTreatment, gradeX, gradeY, error?}] }
|
|
67
121
|
export function scoreFromRaw(raw) {
|
|
68
122
|
const scenarios = (raw && Array.isArray(raw.scenarios) ? raw.scenarios : []).map((s) => {
|
|
@@ -78,7 +132,10 @@ export function scoreFromRaw(raw) {
|
|
|
78
132
|
});
|
|
79
133
|
const agg = aggregate(scenarios);
|
|
80
134
|
const gate = behavioralGate(agg);
|
|
81
|
-
|
|
135
|
+
const out = { skillId: (raw && raw.skillId) || null, scenarios, aggregate: agg, gate };
|
|
136
|
+
// Carried through untouched so the hold-out verdict can echo the fresh scenario for hand-audit.
|
|
137
|
+
if (raw && raw.holdoutScenario) out.holdoutScenario = raw.holdoutScenario;
|
|
138
|
+
return out;
|
|
82
139
|
}
|
|
83
140
|
|
|
84
141
|
export function formatScorecard(scored) {
|
|
@@ -1,11 +1,15 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
// skill-behavior-eval.js — turn the behavior workflow's raw JSON into a scorecard.
|
|
3
3
|
// Usage:
|
|
4
|
-
// node scripts/skill-behavior-eval.js --score
|
|
5
|
-
//
|
|
4
|
+
// node scripts/skill-behavior-eval.js --score <raw.json> (or pipe raw JSON on stdin with -)
|
|
5
|
+
// node scripts/skill-behavior-eval.js --holdout <raw.json> (fresh-scenario transfer gate)
|
|
6
|
+
// Exits 0 if the gate passes, 1 if it fails/blocks, 2 on usage/parse error.
|
|
7
|
+
//
|
|
8
|
+
// --holdout exists so the fix loop's blocking decision is a comparison of numbers an agent reports,
|
|
9
|
+
// never a judgement an agent makes (constitution P1).
|
|
6
10
|
|
|
7
11
|
import { readFileSync } from 'node:fs';
|
|
8
|
-
import { scoreFromRaw, formatScorecard } from './lib/behavior-score.js';
|
|
12
|
+
import { scoreFromRaw, formatScorecard, holdoutGate, formatHoldoutVerdict } from './lib/behavior-score.js';
|
|
9
13
|
|
|
10
14
|
function readInput(argPath) {
|
|
11
15
|
if (argPath && argPath !== '-') return readFileSync(argPath, 'utf8');
|
|
@@ -14,8 +18,9 @@ function readInput(argPath) {
|
|
|
14
18
|
|
|
15
19
|
function main() {
|
|
16
20
|
const args = process.argv.slice(2);
|
|
17
|
-
|
|
18
|
-
|
|
21
|
+
const mode = args[0];
|
|
22
|
+
if (mode !== '--score' && mode !== '--holdout') {
|
|
23
|
+
process.stderr.write('usage: skill-behavior-eval.js --score|--holdout <raw.json|->\n');
|
|
19
24
|
process.exit(2);
|
|
20
25
|
}
|
|
21
26
|
let raw;
|
|
@@ -26,6 +31,11 @@ function main() {
|
|
|
26
31
|
process.exit(2);
|
|
27
32
|
}
|
|
28
33
|
const scored = scoreFromRaw(raw);
|
|
34
|
+
if (mode === '--holdout') {
|
|
35
|
+
const gate = holdoutGate(scored);
|
|
36
|
+
process.stdout.write(formatHoldoutVerdict(scored, gate) + '\n');
|
|
37
|
+
process.exit(gate.verdict === 'pass' ? 0 : 1);
|
|
38
|
+
}
|
|
29
39
|
process.stdout.write(formatScorecard(scored) + '\n');
|
|
30
40
|
process.exit(scored.gate.pass ? 0 : 1);
|
|
31
41
|
}
|
|
@@ -3,7 +3,13 @@
|
|
|
3
3
|
The `skill-harden` workflow turns a FAILING behavioral score into edits, without gaming the eval.
|
|
4
4
|
These are the rules its agents follow.
|
|
5
5
|
|
|
6
|
-
|
|
6
|
+
The three guards below — **diff judge**, **hold-out**, **Generalization gate** — are mandatory *and*
|
|
7
|
+
mechanised: `tests/harden-generalization-gate.test.js` asserts, for each of those three, that the
|
|
8
|
+
loop implements what this file promises. **Adding a fourth guard here means adding its pair to that
|
|
9
|
+
test**; a guard documented but unchecked gives the safety without the protection, which is the
|
|
10
|
+
failure mode this repo has now hit nine times (constitution P2).
|
|
11
|
+
|
|
12
|
+
## Diagnosis (skill-fault vs eval-fault vs capability)
|
|
7
13
|
|
|
8
14
|
Given the failing `mustFix`, both A/B outputs, and the grader's per-item evidence, decide the FAULT:
|
|
9
15
|
|
|
@@ -15,8 +21,44 @@ Given the failing `mustFix`, both A/B outputs, and the grader's per-item evidenc
|
|
|
15
21
|
2. **Phantom-context must_include** — an item demands workspace artifacts the isolated eval
|
|
16
22
|
agent cannot have (e.g. `user-profile.md`, a real sibling skill to delegate to), capping the
|
|
17
23
|
absolute unfairly.
|
|
24
|
+
- **capability** — the gap cannot be closed by words in a skill body. It needs an **executable
|
|
25
|
+
capability**: something that fires on a real execution event and returns to the agent information
|
|
26
|
+
it cannot obtain on its own — a hook, a script, a tool, a sub-agent. Advice that merely tells the
|
|
27
|
+
agent to remember something is **not** a capability: it is not anchored to any observable event, so
|
|
28
|
+
it can fire when it should not and stay silent when it should not.
|
|
29
|
+
|
|
30
|
+
This loop can only ever write **guidance** (SKILL.md, references/). So `capability` is not a fix
|
|
31
|
+
route, it is the honest exit: the loop edits nothing, names the missing capability and the surface it
|
|
32
|
+
would live on, and stops. Writing a paragraph that pretends to cover a missing capability raises the
|
|
33
|
+
score without the capability existing — the worst outcome available.
|
|
34
|
+
|
|
35
|
+
A `capability` verdict **must name** the capability and its surface. One that cannot is an excuse,
|
|
36
|
+
not a diagnosis → choose `skill`.
|
|
37
|
+
|
|
38
|
+
Default to **skill** when unsure: blaming the eval or the surface area is the easier, less honest path.
|
|
39
|
+
|
|
40
|
+
## The Generalization gate (ex-ante — it binds before the edit is written)
|
|
41
|
+
|
|
42
|
+
This gate governs the **content** of every line the fixer writes, and it travels **in the fixer's own
|
|
43
|
+
instruction**, not in a review afterwards. Writing freely and reverting later burns the round, and the
|
|
44
|
+
loop only has two.
|
|
45
|
+
|
|
46
|
+
Everything added to a skill is global context that will apply to cases the skill has never seen. So:
|
|
47
|
+
|
|
48
|
+
- **Allowed:** a reusable criterion that carries its own **applicability condition** — something the
|
|
49
|
+
agent can evaluate on a case it has never encountered.
|
|
50
|
+
- **Banned outright:** identifiers or titles from the eval scenario; names of its files, symbols or
|
|
51
|
+
fixtures; rules that branch on its specific data; and reciting the finding as if it were a rule
|
|
52
|
+
("the last round showed X needs Y" — that is an answer, not a criterion).
|
|
53
|
+
|
|
54
|
+
**Litmus test, before every line:** *would this still help on a case in this domain I have never
|
|
55
|
+
seen?* If no → rewrite it as a criterion, or drop it. Not knowledge → not in the skill.
|
|
18
56
|
|
|
19
|
-
|
|
57
|
+
Why ex-ante and not only as review: moving this constraint from after-the-edit to
|
|
58
|
+
before-the-edit is what buys generalization cheaply. The measured version of this result — plus the
|
|
59
|
+
warning that richer feedback **without** this gate makes held-out performance *worse* — is in
|
|
60
|
+
[HarnessCompass](https://arxiv.org/abs/2608.01918); the reasoning it drove is recorded in
|
|
61
|
+
`02-DOCS/wiki/sdd/specs/generalization-gate.md`.
|
|
20
62
|
|
|
21
63
|
## Eval-fix guard (independent judge)
|
|
22
64
|
|
|
@@ -27,14 +69,34 @@ skill-fault this round.
|
|
|
27
69
|
|
|
28
70
|
## Skill-fix guards (both required)
|
|
29
71
|
|
|
30
|
-
1. **Diff judge.** Read the SKILL.md/references diff
|
|
31
|
-
decision rules, concrete guidance)
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
72
|
+
1. **Diff judge.** Read the SKILL.md/references diff and check BOTH: does it add genuine capability
|
|
73
|
+
(method, decision rules, concrete guidance) rather than echoing the `must_include` wording into
|
|
74
|
+
the body? *and* does every added line survive the Generalization gate above? Fail either → revert
|
|
75
|
+
the edit; it is not a fix.
|
|
76
|
+
2. **Hold-out.** Re-score the EDITED skill on a FRESH scenario from its domain that the fixer never
|
|
77
|
+
saw. A real improvement generalizes; a memorized one does not.
|
|
78
|
+
- The eval engine runs every scenario with **and** without the skill, so the fresh scenario
|
|
79
|
+
already carries its own lift. No pre-fix baseline is needed.
|
|
80
|
+
- The verdict is computed by `node scripts/skill-behavior-eval.js --holdout <raw.json>` — a
|
|
81
|
+
comparison of numbers, deterministic and reproducible, never an agent's opinion (P1). Exit 0
|
|
82
|
+
passes, exit 1 blocks.
|
|
83
|
+
- **Blocks on `lift <= 0`** (`regression`): on unseen work the edited skill does not beat the bare
|
|
84
|
+
agent, so the edit did not transfer. Also blocks when the fresh scenario could not be run or
|
|
85
|
+
graded (`indeterminate`) — failing closed, because an unverified edit is not a verified one.
|
|
86
|
+
- The bar is deliberately **not** the main gate's `lift >= 1.0`: one fresh scenario graded once is
|
|
87
|
+
noisy, so only the unambiguous signal acts. Upgrade path if it proves too noisy in practice:
|
|
88
|
+
N=3 fresh scenarios, majority.
|
|
89
|
+
- **A block reverts the edit.** The round produced nothing and is recorded with its reason.
|
|
35
90
|
|
|
36
91
|
## Stopping & honesty
|
|
37
92
|
|
|
38
93
|
- Max 2 rounds. On give-up, report the honest final score and a recommendation — never a faked pass.
|
|
94
|
+
- **A main-gate pass whose last surviving edit failed the hold-out does not commit.** It is a fix
|
|
95
|
+
tuned to the cases it was measured against; the loop reverts it and reports why. Conservative on
|
|
96
|
+
purpose: a lost commit costs one re-run, a certified overfit costs the catalog.
|
|
97
|
+
- A `capability` verdict never commits either: it returns the missing capability and its surface, and
|
|
98
|
+
routes the work to `specify`.
|
|
99
|
+
- Every non-commit says which condition failed, in the returned result. A denial without its exit is
|
|
100
|
+
abandonment (P6).
|
|
39
101
|
- Persistent lift-fail at a high absolute → recommend deprecate/merge: the skill does not justify
|
|
40
102
|
its own existence.
|
|
@@ -39,12 +39,28 @@ const SCORE_SCHEMA = {
|
|
|
39
39
|
const FAULT_SCHEMA = {
|
|
40
40
|
type: 'object',
|
|
41
41
|
properties: {
|
|
42
|
-
|
|
42
|
+
// 'capability' is the honest third exit: this loop can only ever write guidance (SKILL.md /
|
|
43
|
+
// references), so a failure that needs an executable capability is out of its reach. Saying so
|
|
44
|
+
// beats writing a paragraph that pretends to cover it.
|
|
45
|
+
fault: { type: 'string', enum: ['skill', 'eval', 'capability'] },
|
|
43
46
|
rationale: { type: 'string' },
|
|
47
|
+
missingCapability: { type: 'string' },
|
|
48
|
+
surface: { type: 'string' },
|
|
44
49
|
},
|
|
45
50
|
required: ['fault', 'rationale'],
|
|
46
51
|
}
|
|
47
52
|
|
|
53
|
+
const HOLDOUT_VERDICT_SCHEMA = {
|
|
54
|
+
type: 'object',
|
|
55
|
+
properties: {
|
|
56
|
+
verdict: { type: 'string', enum: ['pass', 'block'] },
|
|
57
|
+
kind: { type: 'string', enum: ['transfer', 'regression', 'indeterminate'] },
|
|
58
|
+
lift: { type: 'number' },
|
|
59
|
+
reason: { type: 'string' },
|
|
60
|
+
},
|
|
61
|
+
required: ['verdict', 'kind', 'reason'],
|
|
62
|
+
}
|
|
63
|
+
|
|
48
64
|
const GUARD_SCHEMA = {
|
|
49
65
|
type: 'object',
|
|
50
66
|
properties: {
|
|
@@ -75,7 +91,26 @@ async function scoreRaw(raw, round) {
|
|
|
75
91
|
)
|
|
76
92
|
}
|
|
77
93
|
|
|
94
|
+
// The hold-out gate: score the EDITED skill against a fresh scenario the fixer never saw, and let a
|
|
95
|
+
// deterministic CLI decide. The verdict is a comparison of numbers the agent reports, never a
|
|
96
|
+
// judgement the agent makes (constitution P1).
|
|
97
|
+
async function holdoutVerdict(raw, scenario, round) {
|
|
98
|
+
const payload = JSON.stringify({ ...raw, holdoutScenario: scenario })
|
|
99
|
+
return agent(
|
|
100
|
+
`Write this JSON to /tmp/harden-${skillId}-holdout-r${round}.json exactly, then run ` +
|
|
101
|
+
`\`node scripts/skill-behavior-eval.js --holdout /tmp/harden-${skillId}-holdout-r${round}.json\`. ` +
|
|
102
|
+
`Read its markdown output and return {verdict, kind, lift, reason} verbatim from that output — ` +
|
|
103
|
+
`verdict "pass" if the CLI exit code was 0, "block" if it was 1. Do NOT form your own opinion ` +
|
|
104
|
+
`about whether the fix was good: report what the CLI decided.\n\nJSON:\n${payload}`,
|
|
105
|
+
{ label: `holdout:r${round}`, phase: 'Diagnose & Fix', schema: HOLDOUT_VERDICT_SCHEMA },
|
|
106
|
+
)
|
|
107
|
+
}
|
|
108
|
+
|
|
78
109
|
const history = []
|
|
110
|
+
const holdout = [] // one record per fix round: did the hold-out run, and what did it say
|
|
111
|
+
let lastFixHoldout = null // verdict of the last round that left an edit on disk
|
|
112
|
+
let lastFault = null // last diagnosis, so the report can name a capability we cannot write
|
|
113
|
+
let outcome = null
|
|
79
114
|
let round = 0
|
|
80
115
|
let committed = null
|
|
81
116
|
|
|
@@ -96,9 +131,16 @@ while (true) {
|
|
|
96
131
|
const fault = await agent(
|
|
97
132
|
`Follow scripts/skill-harden-rubric.md (Diagnosis). The skill "${skillId}" FAILED its behavioral gate. ` +
|
|
98
133
|
`mustFix:\n- ${score.mustFix.join('\n- ')}\n\nGrader signals (both A/B outputs, per-item evidence):\n${evidence}\n\n` +
|
|
99
|
-
`Decide fault = 'skill' or '
|
|
134
|
+
`Decide fault = 'skill', 'eval' or 'capability' and give a rationale. Default to 'skill' when unsure.\n\n` +
|
|
135
|
+
`Choose 'capability' ONLY when the gap cannot be closed by words in a skill body: it needs an ` +
|
|
136
|
+
`executable capability — something that fires on a real execution event and returns to the agent ` +
|
|
137
|
+
`information it cannot obtain on its own (a hook, a script, a tool, a sub-agent). Advice that only ` +
|
|
138
|
+
`tells the agent to remember something is NOT a capability. If you choose 'capability' you MUST name ` +
|
|
139
|
+
`it in missingCapability and name where it would live in surface; a verdict that cannot name the ` +
|
|
140
|
+
`capability is an excuse, not a diagnosis — pick 'skill' instead.`,
|
|
100
141
|
{ label: `diagnose:r${round}`, phase: 'Diagnose & Fix', schema: FAULT_SCHEMA },
|
|
101
142
|
)
|
|
143
|
+
lastFault = fault
|
|
102
144
|
|
|
103
145
|
if (fault.fault === 'eval') {
|
|
104
146
|
const judged = await agent(
|
|
@@ -115,31 +157,69 @@ while (true) {
|
|
|
115
157
|
}
|
|
116
158
|
}
|
|
117
159
|
|
|
160
|
+
if (fault.fault === 'capability') {
|
|
161
|
+
// Out of this loop's reach: it can only write words. Stop honestly instead of producing guidance
|
|
162
|
+
// that raises the score without the capability existing. Nothing is edited here on purpose.
|
|
163
|
+
outcome = 'capability-out-of-reach'
|
|
164
|
+
log(`${skillId}: needs a capability, not guidance — ${fault.missingCapability || '(unnamed)'} in ${fault.surface || '(unnamed surface)'}`)
|
|
165
|
+
break
|
|
166
|
+
}
|
|
167
|
+
|
|
118
168
|
if (fault.fault === 'skill') {
|
|
119
169
|
await agent(
|
|
120
170
|
`Follow the author-skill discipline. The skill "${skillId}" must genuinely cover this mustFix without ` +
|
|
121
171
|
`keyword-stuffing:\n- ${score.mustFix.join('\n- ')}\n\nEdit skills/${skillId}/SKILL.md (body) and/or files under ` +
|
|
122
172
|
`skills/${skillId}/references/ to add the REAL missing capability (method, decision rules, concrete guidance). ` +
|
|
123
|
-
`Do not touch evals/. Apply the edits now
|
|
173
|
+
`Do not touch evals/. Apply the edits now.\n\n` +
|
|
174
|
+
`HARD CONSTRAINT — the Generalization gate (scripts/skill-harden-rubric.md defines it; it binds every ` +
|
|
175
|
+
`line you write, and it binds BEFORE you write, not after):\n` +
|
|
176
|
+
`Everything you add is global context that will apply to cases this skill has never seen. Write a ` +
|
|
177
|
+
`reusable criterion that carries its own applicability condition — never the answer to the one ` +
|
|
178
|
+
`scenario that failed. Banned outright: identifiers or titles from the eval scenario, names of its ` +
|
|
179
|
+
`files/symbols/fixtures, rules that branch on its specific data, and reciting the finding as if it ` +
|
|
180
|
+
`were a rule ("the last round showed X needs Y").\n` +
|
|
181
|
+
`Litmus test before every line: "would this still help on a case in this domain I have never seen?" ` +
|
|
182
|
+
`If no, rewrite it as a criterion or drop it. A fix that only lifts this scenario's score will be ` +
|
|
183
|
+
`caught by the hold-out and reverted, and the round will be wasted.`,
|
|
124
184
|
{ label: `fix:r${round}`, phase: 'Diagnose & Fix' },
|
|
125
185
|
)
|
|
126
186
|
const diffJudge = await agent(
|
|
127
187
|
`Follow scripts/skill-harden-rubric.md (Skill-fix guard 1). Run \`git diff -- skills/${skillId}/SKILL.md skills/${skillId}/references\` ` +
|
|
128
|
-
`and judge: does the diff add genuine capability, or just echo the mustFix wording to satisfy the
|
|
129
|
-
`
|
|
188
|
+
`and judge BOTH: (a) does the diff add genuine capability, or just echo the mustFix wording to satisfy the ` +
|
|
189
|
+
`grader? and (b) does every added line survive the Generalization gate — a reusable criterion with its ` +
|
|
190
|
+
`applicability condition, not the answer to the observed scenario (no eval-specific names, no branching on ` +
|
|
191
|
+
`its data, no recited findings)? Fail either check and it is not a fix. ` +
|
|
192
|
+
`If it fails, run \`git checkout -- skills/${skillId}/SKILL.md skills/${skillId}/references\` to revert it. ` +
|
|
130
193
|
`Return {genuine, rationale}.`,
|
|
131
194
|
{ label: `diff-judge:r${round}`, phase: 'Diagnose & Fix', schema: GUARD_SCHEMA },
|
|
132
195
|
)
|
|
133
196
|
|
|
134
|
-
if (diffJudge.genuine) {
|
|
135
|
-
|
|
136
|
-
|
|
197
|
+
if (!diffJudge.genuine) {
|
|
198
|
+
holdout.push({ round, ran: false, why: 'edit reverted by the diff judge — nothing to validate' })
|
|
199
|
+
} else {
|
|
200
|
+
// Guard 2: hold-out. Score the EDITED skill on a fresh scenario the fixer never saw. The eval
|
|
201
|
+
// engine runs every scenario with AND without the skill, so this already carries its own lift.
|
|
202
|
+
const fresh = await agent(
|
|
137
203
|
`Invent ONE fresh capability scenario for the "${skillId}" skill's domain that is NOT in its cases.yaml and ` +
|
|
138
204
|
`does NOT enumerate its own requirements. Return {scenario, mustInclude:[3-6 outcome-level checks]}.`,
|
|
139
205
|
{ label: `holdout-gen:r${round}`, phase: 'Diagnose & Fix', schema: HOLDOUT_SCHEMA },
|
|
140
206
|
)
|
|
141
|
-
const holdoutRaw = await workflow(EVAL, { skillId, scenarios: [
|
|
142
|
-
await
|
|
207
|
+
const holdoutRaw = await workflow(EVAL, { skillId, scenarios: [fresh] })
|
|
208
|
+
const verdict = await holdoutVerdict(holdoutRaw, fresh.scenario, round)
|
|
209
|
+
lastFixHoldout = verdict
|
|
210
|
+
holdout.push({ round, ran: true, scenario: fresh.scenario, ...verdict })
|
|
211
|
+
|
|
212
|
+
if (verdict.verdict === 'block') {
|
|
213
|
+
// The edit lifted this skill's own cases.yaml but does not transfer to work the fixer never
|
|
214
|
+
// saw. A memorized fix is not a fix: revert it. This is the guard the rubric always promised
|
|
215
|
+
// and the code never enforced — see 02-DOCS/wiki/sdd/specs/generalization-gate.md.
|
|
216
|
+
log(`${skillId}: hold-out ${verdict.kind} (lift ${verdict.lift}) — reverting round ${round}'s edit`)
|
|
217
|
+
await agent(
|
|
218
|
+
`Run exactly \`git checkout -- skills/${skillId}/SKILL.md skills/${skillId}/references\` and nothing else. ` +
|
|
219
|
+
`The edit failed the hold-out gate (${verdict.kind}) so it must not survive. Return the command's output.`,
|
|
220
|
+
{ label: `holdout-revert:r${round}`, phase: 'Diagnose & Fix', effort: 'low' },
|
|
221
|
+
)
|
|
222
|
+
}
|
|
143
223
|
}
|
|
144
224
|
}
|
|
145
225
|
|
|
@@ -147,7 +227,13 @@ while (true) {
|
|
|
147
227
|
}
|
|
148
228
|
|
|
149
229
|
const passed = history.length > 0 && history[history.length - 1].pass === true
|
|
150
|
-
|
|
230
|
+
// A main-gate pass whose last surviving edit failed the hold-out is NOT a pass: it is a fix that
|
|
231
|
+
// only works on the cases it was tuned against. Conservative on purpose — a lost commit costs one
|
|
232
|
+
// re-run, a certified overfit costs the catalog.
|
|
233
|
+
const holdoutClean = !lastFixHoldout || lastFixHoldout.verdict === 'pass'
|
|
234
|
+
let notCommittedBecause = null
|
|
235
|
+
|
|
236
|
+
if (passed && holdoutClean && !noCommit) {
|
|
151
237
|
phase('Commit')
|
|
152
238
|
const commit = await agent(
|
|
153
239
|
`The skill "${skillId}" now passes its behavioral gate. Commit ONLY its files: ` +
|
|
@@ -156,6 +242,24 @@ if (passed && !noCommit) {
|
|
|
156
242
|
{ label: `commit:${skillId}`, phase: 'Commit' },
|
|
157
243
|
)
|
|
158
244
|
committed = (commit || '').trim()
|
|
245
|
+
} else if (outcome === 'capability-out-of-reach') {
|
|
246
|
+
notCommittedBecause = `needs a capability this loop cannot write: ${lastFault.missingCapability || '(unnamed)'} in ${lastFault.surface || '(unnamed surface)'}. Route it through specify — guidance cannot close this gap.`
|
|
247
|
+
} else if (!passed) {
|
|
248
|
+
notCommittedBecause = 'the behavioral gate still fails. Read the last scorecard and fix by hand, or deprecate the skill.'
|
|
249
|
+
} else if (!holdoutClean) {
|
|
250
|
+
notCommittedBecause = `the last edit passed cases.yaml but failed the hold-out (${lastFixHoldout.kind}, lift ${lastFixHoldout.lift}) and was reverted. Re-run to try a different fix, or write the criterion by hand.`
|
|
251
|
+
} else if (noCommit) {
|
|
252
|
+
notCommittedBecause = 'noCommit was requested.'
|
|
159
253
|
}
|
|
160
254
|
|
|
161
|
-
return {
|
|
255
|
+
return {
|
|
256
|
+
skillId,
|
|
257
|
+
rounds: round + 1,
|
|
258
|
+
history,
|
|
259
|
+
holdout,
|
|
260
|
+
outcome: outcome || (passed && holdoutClean ? 'passed' : 'gave-up'),
|
|
261
|
+
missingCapability: lastFault && lastFault.fault === 'capability' ? { capability: lastFault.missingCapability, surface: lastFault.surface, rationale: lastFault.rationale } : null,
|
|
262
|
+
committed,
|
|
263
|
+
passed: passed && holdoutClean,
|
|
264
|
+
notCommittedBecause,
|
|
265
|
+
}
|
package/skills/analyze/SKILL.md
CHANGED
|
@@ -101,6 +101,28 @@ Render it at the dial's verbosity. Do not log a decision to `decisions.md` — a
|
|
|
101
101
|
| Siding with the plan when it contradicts the constitution | The constitution wins by definition — flag CRITICAL. If the principle itself is wrong, that is a `constitution` change the user makes, not a quiet override here. |
|
|
102
102
|
| Guessing what a vague requirement meant | Guessing defeats the gate. Mark AMBIGUOUS and route to `clarify`; do not encode your guess. |
|
|
103
103
|
|
|
104
|
+
## Result envelope
|
|
105
|
+
|
|
106
|
+
End with the parseable block every SDD phase shares, so the dispatcher can chain without
|
|
107
|
+
interpreting prose (contract: `../sdd/SKILL.md`):
|
|
108
|
+
|
|
109
|
+
```json result-envelope
|
|
110
|
+
{
|
|
111
|
+
"status": "complete|blocked|failed",
|
|
112
|
+
"executive_summary": "Cross-read of spec/plan/tasks against the constitution; findings ranked.",
|
|
113
|
+
"artifact": "02-DOCS/wiki/sdd/analysis/<slug>.md",
|
|
114
|
+
"next_recommended": "implement",
|
|
115
|
+
"risk": "low|medium|high",
|
|
116
|
+
"skill_resolution": {
|
|
117
|
+
"used": ["analyze"],
|
|
118
|
+
"missing": [],
|
|
119
|
+
"fallback": [],
|
|
120
|
+
"compact_rules": ["Read adversarially across artifacts, not inside one.", "A finding without a location is an opinion."]
|
|
121
|
+
},
|
|
122
|
+
"evidence": ["report path exists", "blockers listed with artifact + location", "constitution conflicts named"]
|
|
123
|
+
}
|
|
124
|
+
```
|
|
125
|
+
|
|
104
126
|
## Next in the chain
|
|
105
127
|
|
|
106
128
|
On `GATE: PASS` (or once the user consciously accepts the remaining MEDIUM/LOW findings), proceed to **`implement`** — execute the tasks with TDD discipline, delegating concrete test tooling to the relevant stack skill (`fastapi`, `nextjs`, `go`, `flutter`, `postgresdb`). On `GATE: BLOCKED`, route each CRITICAL/HIGH finding to its owning phase (`clarify`, `plan`, `tasks`, or `constitution`), let the user resolve, then re-run `analyze`. The gate only opens once.
|
package/skills/clarify/SKILL.md
CHANGED
|
@@ -113,6 +113,28 @@ After baking back, the spec line becomes a bounded, testable behavior with accep
|
|
|
113
113
|
|
|
114
114
|
The gate is passed when the spec, constitution and profile were all read (settled questions cited, not re-asked); all ten taxonomy categories were considered; only the build-changing gaps were put to the user, as dial-sized decisions with recommendations; every answer is baked into the spec body with observable acceptance criteria, logged under `## Clarifications` and bounded under `## Out of scope`; and the final re-read opened no new gap.
|
|
115
115
|
|
|
116
|
+
## Result envelope
|
|
117
|
+
|
|
118
|
+
End with the parseable block every SDD phase shares, so the dispatcher can chain without
|
|
119
|
+
interpreting prose (contract: `../sdd/SKILL.md`):
|
|
120
|
+
|
|
121
|
+
```json result-envelope
|
|
122
|
+
{
|
|
123
|
+
"status": "complete|blocked|failed",
|
|
124
|
+
"executive_summary": "Open points resolved; the spec is de-risked and ready to plan against.",
|
|
125
|
+
"artifact": "02-DOCS/wiki/sdd/specs/<slug>.md",
|
|
126
|
+
"next_recommended": "plan",
|
|
127
|
+
"risk": "low|medium|high",
|
|
128
|
+
"skill_resolution": {
|
|
129
|
+
"used": ["clarify"],
|
|
130
|
+
"missing": [],
|
|
131
|
+
"fallback": [],
|
|
132
|
+
"compact_rules": ["Ask only what changes the spec.", "An unanswered question is recorded, never invented."]
|
|
133
|
+
},
|
|
134
|
+
"evidence": ["answers folded back into the spec", "remaining open points listed with their owner"]
|
|
135
|
+
}
|
|
136
|
+
```
|
|
137
|
+
|
|
116
138
|
## Next in the chain
|
|
117
139
|
|
|
118
140
|
Hand off to **`plan`** — turn the now-sharp spec into a technical implementation plan (architecture, interfaces, data flow, testing strategy, risks), deferring stack specifics to the relevant stack skill. The chain continues: clarify → **plan** → tasks → analyze → implement → verify → review → ship. `debug` is callable any time if what you are "clarifying" turns out to be a runtime fault, not a spec gap.
|
|
@@ -132,6 +132,28 @@ This skill's `02-DOCS` record is the constitution at `02-DOCS/wiki/sdd/constitut
|
|
|
132
132
|
|
|
133
133
|
Every later rsc-sdd phase reads this file before it works. The harness maintains and improves the article over the life of the project; this skill is the place that ratifies and amends it.
|
|
134
134
|
|
|
135
|
+
## Result envelope
|
|
136
|
+
|
|
137
|
+
End with the parseable block every SDD phase shares, so the dispatcher can chain without
|
|
138
|
+
interpreting prose (contract: `../sdd/SKILL.md`):
|
|
139
|
+
|
|
140
|
+
```json result-envelope
|
|
141
|
+
{
|
|
142
|
+
"status": "complete|blocked|failed",
|
|
143
|
+
"executive_summary": "Constitution written with N numbered, testable rules the later phases inherit.",
|
|
144
|
+
"artifact": "02-DOCS/wiki/sdd/constitution.md",
|
|
145
|
+
"next_recommended": "specify",
|
|
146
|
+
"risk": "low|medium|high",
|
|
147
|
+
"skill_resolution": {
|
|
148
|
+
"used": ["constitution"],
|
|
149
|
+
"missing": [],
|
|
150
|
+
"fallback": [],
|
|
151
|
+
"compact_rules": ["Principles are inherited constraints, not choices to re-make.", "Every rule is testable or it is a preference."]
|
|
152
|
+
},
|
|
153
|
+
"evidence": ["constitution path exists", "rules numbered and testable", "decisions log appended"]
|
|
154
|
+
}
|
|
155
|
+
```
|
|
156
|
+
|
|
135
157
|
## Next in the chain
|
|
136
158
|
|
|
137
159
|
The constitution is the guardrail; now describe what to build. Hand off to **`../specify/SKILL.md`** — turn a fuzzy intent into a spec (what & why, no implementation), grounded in these principles. The full chain: **constitution → specify → clarify → plan → tasks → analyze → implement → verify → review → ship** (with `debug`, `worktrees`, `parallel` callable on demand). The dispatcher is `../sdd/SKILL.md`.
|
package/skills/plan/SKILL.md
CHANGED
|
@@ -141,6 +141,28 @@ regardless. (`implement` re-checks this as a hard gate before its first commit.)
|
|
|
141
141
|
- Concrete framework/ORM/test-runner mechanics → the stack skill (`../fastapi/SKILL.md`,
|
|
142
142
|
`../nextjs/SKILL.md`, `../go/SKILL.md`, `../flutter/SKILL.md`, `../postgresdb/SKILL.md`).
|
|
143
143
|
|
|
144
|
+
## Result envelope
|
|
145
|
+
|
|
146
|
+
End with the parseable block every SDD phase shares, so the dispatcher can chain without
|
|
147
|
+
interpreting prose (contract: `../sdd/SKILL.md`):
|
|
148
|
+
|
|
149
|
+
```json result-envelope
|
|
150
|
+
{
|
|
151
|
+
"status": "complete|blocked|failed",
|
|
152
|
+
"executive_summary": "Technical plan derived from the clarified spec, with the isolation decision made.",
|
|
153
|
+
"artifact": "02-DOCS/wiki/sdd/plans/<slug>.md",
|
|
154
|
+
"next_recommended": "tasks",
|
|
155
|
+
"risk": "low|medium|high",
|
|
156
|
+
"skill_resolution": {
|
|
157
|
+
"used": ["plan"],
|
|
158
|
+
"missing": [],
|
|
159
|
+
"fallback": [],
|
|
160
|
+
"compact_rules": ["The plan answers HOW; the spec owns WHAT.", "Name the isolation choice before the build starts."]
|
|
161
|
+
},
|
|
162
|
+
"evidence": ["plan path exists", "each spec acceptance criterion has an approach", "risks and rollback stated"]
|
|
163
|
+
}
|
|
164
|
+
```
|
|
165
|
+
|
|
144
166
|
## Next in the chain
|
|
145
167
|
|
|
146
168
|
Plan written, indexed, decisions logged → propose isolation (above), then hand off to
|
package/skills/review/SKILL.md
CHANGED
|
@@ -237,6 +237,28 @@ Review is mostly a conversation, but two artifacts persist into the harness wiki
|
|
|
237
237
|
|
|
238
238
|
Index both in `02-DOCS/wiki/index.md` (the Knowledge map; root `CLAUDE.md` keeps only a short pointer) under the `sdd/` topic — the harness owns that map; this skill just keeps its rows honest.
|
|
239
239
|
|
|
240
|
+
## Result envelope
|
|
241
|
+
|
|
242
|
+
End with the parseable block every SDD phase shares, so the dispatcher can chain without
|
|
243
|
+
interpreting prose (contract: `../sdd/SKILL.md`):
|
|
244
|
+
|
|
245
|
+
```json result-envelope
|
|
246
|
+
{
|
|
247
|
+
"status": "complete|blocked|failed",
|
|
248
|
+
"executive_summary": "Adversarial review against spec/plan/constitution; verdict and blocking findings.",
|
|
249
|
+
"artifact": "02-DOCS/wiki/sdd/reviews/<slug>.md",
|
|
250
|
+
"next_recommended": "ship",
|
|
251
|
+
"risk": "low|medium|high",
|
|
252
|
+
"skill_resolution": {
|
|
253
|
+
"used": ["review"],
|
|
254
|
+
"missing": [],
|
|
255
|
+
"fallback": [],
|
|
256
|
+
"compact_rules": ["Rank or it is noise: blocker / should-fix / nit.", "A finding without a repro is a question, not a defect."]
|
|
257
|
+
},
|
|
258
|
+
"evidence": ["verdict stated (APPROVE | APPROVE WITH NITS | CHANGES REQUESTED)", "each blocker carries location + repro + fix"]
|
|
259
|
+
}
|
|
260
|
+
```
|
|
261
|
+
|
|
240
262
|
## Next in the chain
|
|
241
263
|
|
|
242
264
|
When the diff carries an **APPROVE** / **APPROVE WITH NITS** verdict and every blocker is resolved, hand off to **ship** — close the branch via PR / merge / cleanup, with **git authorship as Eric, never Claude**. If the review came back **CHANGES REQUESTED**, the loop goes back to **implement** (fix), then **verify** (re-prove green), then back here for re-review. Don't ship a diff that hasn't earned its verdict.
|