@olegkoval/agent-skills 1.49.2 → 1.50.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/adapters/claude/olko-github-pr/skills/lekker-review/SKILL.md +26 -1
- package/adapters/claude/olko-github-pr/skills/lekker-review/references/agents/completeness-critic.md +12 -0
- package/adapters/claude/olko-github-pr/skills/lekker-review/references/agents/implementation.md +26 -1
- package/adapters/claude/olko-github-pr/skills/lekker-review/references/agents/prover.md +8 -1
- package/adapters/claude/olko-github-pr/skills/lekker-review/references/agents/test-quality.md +20 -2
- package/adapters/claude/olko-github-pr/skills/lekker-review/references/agents/verifier.md +25 -0
- package/adapters/claude/olko-github-pr/skills/lekker-review/references/fix-mode.md +23 -0
- package/package.json +1 -1
- package/plugins/olko-apple-kit/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-creative/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-garmin-kit/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-git-tools/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-github-pr/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-github-pr/skills/lekker-review/SKILL.md +26 -1
- package/plugins/olko-github-pr/skills/lekker-review/references/agents/completeness-critic.md +12 -0
- package/plugins/olko-github-pr/skills/lekker-review/references/agents/implementation.md +26 -1
- package/plugins/olko-github-pr/skills/lekker-review/references/agents/prover.md +8 -1
- package/plugins/olko-github-pr/skills/lekker-review/references/agents/test-quality.md +20 -2
- package/plugins/olko-github-pr/skills/lekker-review/references/agents/verifier.md +25 -0
- package/plugins/olko-github-pr/skills/lekker-review/references/fix-mode.md +23 -0
- package/plugins/olko-github-pr/skills/lekker-review/workflow.js +87 -5
- package/plugins/olko-obsidian/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-product/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-reflection/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-release/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-skill-meta/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-web-ops/.claude-plugin/plugin.json +1 -1
|
@@ -87,6 +87,31 @@ No nitpicking. Critical and Important findings are reserved for things that
|
|
|
87
87
|
could cause bugs, outages, data loss, security incidents, or real performance
|
|
88
88
|
problems at scale.
|
|
89
89
|
|
|
90
|
+
**Boundary-first review rule:** for stateful or integration-heavy changes,
|
|
91
|
+
review the complete behavioral chain rather than the changed helper alone:
|
|
92
|
+
|
|
93
|
+
```text
|
|
94
|
+
production entry point
|
|
95
|
+
→ decision point
|
|
96
|
+
→ runtime realization
|
|
97
|
+
→ downstream consumer
|
|
98
|
+
→ persistence/cache/cleanup owner
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
Require the review to distinguish logical identity from physical/runtime
|
|
102
|
+
identity, and to test the relevant transition matrix: success, `null`, `undefined`,
|
|
103
|
+
exception, retry, fallback, rotation, cache reuse/eviction, proxy parity, and
|
|
104
|
+
quoted/history input where applicable. A copied helper test is isolated
|
|
105
|
+
evidence, not proof that the production boundary works.
|
|
106
|
+
|
|
107
|
+
Every verdict-affecting finding should carry a boundary map and an evidence
|
|
108
|
+
class. The report must distinguish source inspection, isolated tests,
|
|
109
|
+
repository tests, composed production-entry probes, production probes, and
|
|
110
|
+
hosted CI. Never upgrade one evidence class into another in the wording.
|
|
111
|
+
|
|
112
|
+
After a fix, re-run the original proof *and* audit the newly changed lifecycle
|
|
113
|
+
boundary for regressions. A green reproduction is necessary, not sufficient.
|
|
114
|
+
|
|
90
115
|
**HARD RULE: the `## 💰 Review Cost` block is mandatory.** Every review MUST
|
|
91
116
|
end with an evidence-backed cost block and no `<N>` placeholders. Use real
|
|
92
117
|
numbers for measurements the host exposes; follow `references/output-format.md`
|
|
@@ -290,7 +315,7 @@ page for this PR.
|
|
|
290
315
|
| Review agents | 2 triage (haiku) | 5 specialists (sonnet) | 5 specialists (sonnet) |
|
|
291
316
|
| Per-finding verification | Criticals + Importants | Criticals + Importants | Criticals + Importants |
|
|
292
317
|
| Completeness critic | skip | skip | included |
|
|
293
|
-
| Proof-of-bug (failing test per Critical) | skip | included (max 5) | included (max 5) |
|
|
318
|
+
| Proof-of-bug (failing test per Critical or composed-boundary Important) | skip | included (max 5) | included (max 5) |
|
|
294
319
|
| Living review artifact | included | included | included |
|
|
295
320
|
| Housekeeping (optional memory/notes writeback) | skip | included | included |
|
|
296
321
|
| `--post` | supported | supported | supported |
|
package/adapters/claude/olko-github-pr/skills/lekker-review/references/agents/completeness-critic.md
CHANGED
|
@@ -14,6 +14,18 @@ were declared "no findings" too quickly. For each angle:
|
|
|
14
14
|
Be concrete: cite diff lines, not vibes. If you genuinely cannot find a missed
|
|
15
15
|
angle, return "No gaps found."
|
|
16
16
|
|
|
17
|
+
Always consider these maintainer-grade gaps before returning no gaps:
|
|
18
|
+
- the real production entry point and downstream consumer were not exercised;
|
|
19
|
+
- the selected value was not traced through runtime realization, persistence,
|
|
20
|
+
cleanup, cache reuse, or fallback;
|
|
21
|
+
- logical identity was conflated with physical/runtime identity;
|
|
22
|
+
- a state-transition matrix is missing for `None`, exceptions, retries,
|
|
23
|
+
rotation, proxying, or alternate transport;
|
|
24
|
+
- the fix was checked only against the original symptom, not for a new defect
|
|
25
|
+
introduced at the changed ordering or lifecycle boundary;
|
|
26
|
+
- the evidence is an isolated harness or local test but the review claims
|
|
27
|
+
composed behavior or hosted-CI acceptance.
|
|
28
|
+
|
|
17
29
|
Agent findings:
|
|
18
30
|
<AGENT_FINDINGS_SUMMARY>
|
|
19
31
|
|
package/adapters/claude/olko-github-pr/skills/lekker-review/references/agents/implementation.md
CHANGED
|
@@ -37,7 +37,32 @@ Axes to cover:
|
|
|
37
37
|
sort/dedup on large arrays that could be done at DB level.
|
|
38
38
|
- Integration Contracts: Shopify API misuse, BC API assumptions, webhook
|
|
39
39
|
idempotency, external API pagination not handled.
|
|
40
|
-
|
|
40
|
+
|
|
41
|
+
Boundary contract and lifecycle map (required for every Critical/Important
|
|
42
|
+
finding involving routing, persistence, caching, fallback, proxying, or an
|
|
43
|
+
external integration):
|
|
44
|
+
- `entryPoint`: identify the real production trigger that reaches the changed
|
|
45
|
+
code (HTTP route, webhook, CLI command, queue consumer, gateway ingress, etc.).
|
|
46
|
+
- `decisionPoint`: identify where the behavior is selected or transformed.
|
|
47
|
+
- `realizationPoint`: identify where the selected behavior becomes runtime state.
|
|
48
|
+
- `consumer`: identify the downstream consumer that actually relies on the
|
|
49
|
+
contract, not merely the helper that constructs it.
|
|
50
|
+
- `stateOwner`: identify the durable/session/cache owner when state crosses a
|
|
51
|
+
turn, request, retry, or process boundary.
|
|
52
|
+
- `identityDimensions`: list every identity that must remain distinct, such as
|
|
53
|
+
logical session key, physical session id, provider alias, resolved provider,
|
|
54
|
+
model, and API mode.
|
|
55
|
+
- `transitions`: list the relevant success, `null`, `undefined`, exception, retry, fallback,
|
|
56
|
+
session-rotation, cache-reuse, and proxy/alternate-transport transitions.
|
|
57
|
+
- `evidenceClass`: label the strongest evidence actually checked: `source`,
|
|
58
|
+
`isolated-test`, `repository-test`, `composed-test`, `production-probe`, or
|
|
59
|
+
`hosted-ci`. Never call an isolated harness a production-path proof.
|
|
60
|
+
|
|
61
|
+
Do not report a stateful/integration finding without this boundary object. If
|
|
62
|
+
the production entry point or downstream consumer cannot be identified, mark
|
|
63
|
+
the claim as unverified rather than inferring behavior from a helper alone.
|
|
64
|
+
|
|
65
|
+
GraphQL pagination (GQL-1): for every GraphQL query in the diff that uses a
|
|
41
66
|
nodes connection (`nodes { ... }`):
|
|
42
67
|
(a) Check that `pageInfo { hasNextPage endCursor }` is present alongside nodes: if missing, Critical.
|
|
43
68
|
(b) Check that all pages are fetched (a loop or recursion using endCursor): a single-page fetch is a bug, Critical.
|
|
@@ -4,11 +4,18 @@
|
|
|
4
4
|
|
|
5
5
|
## Mission
|
|
6
6
|
|
|
7
|
-
A Critical finding
|
|
7
|
+
A Critical finding, or an Important finding with a composed boundary map, is an
|
|
8
|
+
argument until someone runs it. Your job is to turn it
|
|
8
9
|
into a demonstration: write ONE test that asserts the CORRECT behavior of the
|
|
9
10
|
code the finding targets, run it in the real worktree, and capture it failing
|
|
10
11
|
for the exact reason the finding claims.
|
|
11
12
|
|
|
13
|
+
For a finding with `boundary`, prefer entering through the recorded production
|
|
14
|
+
entry point and reaching the recorded downstream consumer. Do not replace a
|
|
15
|
+
composed proof with a copied helper test just because the helper is easier to
|
|
16
|
+
import. If full composition is impossible, report the strongest lower evidence
|
|
17
|
+
class honestly.
|
|
18
|
+
|
|
12
19
|
Write the test as if the bug were already fixed. It must fail today precisely
|
|
13
20
|
*because* it isn't fixed. **Never write a test that asserts the buggy behavior
|
|
14
21
|
just to have something red** -- a proof that asserts wrongness is worthless and
|
package/adapters/claude/olko-github-pr/skills/lekker-review/references/agents/test-quality.md
CHANGED
|
@@ -128,8 +128,26 @@ g) Mock smell: test the behavior, not the way it's built
|
|
|
128
128
|
(time, randomness, injected network failure), or a dependency that genuinely
|
|
129
129
|
cannot run in the test environment. The tell for a good mock: the behavior
|
|
130
130
|
under test only exists because the boundary did something (e.g. a retry
|
|
131
|
-
|
|
132
|
-
|
|
131
|
+
banner that appears only when the fetch rejects). Even then, prefer an
|
|
132
|
+
injected simple fake over a module-level spy, and assert what the user sees.
|
|
133
|
+
|
|
134
|
+
h) Composed boundary and transition coverage
|
|
135
|
+
For every stateful or integration-heavy change, do not stop at testing the
|
|
136
|
+
helper. Trace the real entry point through the decision point, runtime
|
|
137
|
+
realization, downstream consumer, and persistence/cleanup owner. Require a
|
|
138
|
+
test or explicit evidence for the relevant transition matrix:
|
|
139
|
+
- success, `None`, exception, and disabled-feature paths;
|
|
140
|
+
- retry and fallback behavior;
|
|
141
|
+
- session/request rotation and cache reuse or eviction;
|
|
142
|
+
- proxy or alternate transport parity;
|
|
143
|
+
- quoted/history input versus newly authored input;
|
|
144
|
+
- provider/model/API-mode identity when routing is involved.
|
|
145
|
+
|
|
146
|
+
A test that only asserts a helper return value is `isolated-test` evidence,
|
|
147
|
+
not `composed-test` evidence. Flag a Critical/Important finding when the
|
|
148
|
+
changed contract is consumed through a different path than the test covers,
|
|
149
|
+
or when the test can pass while the stateful consumer still receives the
|
|
150
|
+
wrong identity or ordering.
|
|
133
151
|
|
|
134
152
|
Report each per-line issue as:
|
|
135
153
|
`test-file:line, <concise description of the gap or weakness>`
|
|
@@ -123,6 +123,31 @@ convention finding with no verifiable precedent does not ship.
|
|
|
123
123
|
|
|
124
124
|
---
|
|
125
125
|
|
|
126
|
+
## Step 4b -- Boundary contract and evidence check
|
|
127
|
+
|
|
128
|
+
For any Critical/Important finding involving routing, persistence, caching,
|
|
129
|
+
fallback, proxying, quoted input, or an external integration, read the
|
|
130
|
+
finding's `boundary` object. It should identify the real production entry
|
|
131
|
+
point, decision point, runtime realization point, downstream consumer, state
|
|
132
|
+
owner, and relevant identity dimensions/transitions.
|
|
133
|
+
|
|
134
|
+
Reconstruct the path in `WORKTREE_PATH` from entry point to consumer. A copied
|
|
135
|
+
helper test or a mock that never reaches the consumer is only isolated
|
|
136
|
+
evidence. Do not confirm a verdict-affecting finding as production-path proof
|
|
137
|
+
unless the claimed failure reaches the stateful consumer or the review clearly
|
|
138
|
+
labels the evidence as isolated.
|
|
139
|
+
|
|
140
|
+
If the finding has no boundary object or no usable production entry point or
|
|
141
|
+
consumer, keep it unverified and non-blocking; drop it when the missing boundary
|
|
142
|
+
is the crux of the claim. The workflow enforces this as an unverified observation
|
|
143
|
+
even if a verifier confirms it. Record the strongest evidence class actually checked
|
|
144
|
+
(`source`, `isolated-test`, `repository-test`, `composed-test`,
|
|
145
|
+
`production-probe`, or `hosted-ci`) in the reasoning.
|
|
146
|
+
|
|
147
|
+
Build a small transition matrix for the affected contract. At minimum check
|
|
148
|
+
success, `null`, `undefined`, exception, retry/fallback, state rotation, cache reuse or
|
|
149
|
+
eviction, and proxy/alternate transport whenever those transitions exist.
|
|
150
|
+
|
|
126
151
|
## Step 5 -- Fix validation
|
|
127
152
|
|
|
128
153
|
Before confirming the finding, identify the callers and consumers of the changed
|
|
@@ -210,6 +210,29 @@ check after any revert triggered here before moving on.
|
|
|
210
210
|
|
|
211
211
|
---
|
|
212
212
|
|
|
213
|
+
## Step 5c -- Fix delta audit
|
|
214
|
+
|
|
215
|
+
After the original proof flips red → green, audit the *new* boundary created by
|
|
216
|
+
the fix. Do not assume that making the original assertion pass makes the
|
|
217
|
+
lifecycle safe.
|
|
218
|
+
|
|
219
|
+
For each fixed Critical/Important finding involving state, routing, ordering,
|
|
220
|
+
proxying, persistence, caching, fallback, or quoted input:
|
|
221
|
+
|
|
222
|
+
1. Re-read the entire changed path from the production entry point to the
|
|
223
|
+
downstream consumer.
|
|
224
|
+
2. Re-run the relevant transition matrix: success, `None`, exception, retry,
|
|
225
|
+
fallback, rotation, cache reuse/eviction, and alternate transport where
|
|
226
|
+
applicable.
|
|
227
|
+
3. Check that logical identity, runtime identity, provider/API identity, and
|
|
228
|
+
durable identity are still distinct where the contract requires it.
|
|
229
|
+
4. Add a focused regression probe when the fix moved work across an ordering or
|
|
230
|
+
lifecycle boundary. The probe must enter through the real path when feasible.
|
|
231
|
+
|
|
232
|
+
If the audit finds a new failure, revert the whole group and mark it
|
|
233
|
+
`failed: fix-delta-regression`. A green original proof is not sufficient to
|
|
234
|
+
commit a fix that breaks a neighboring transition.
|
|
235
|
+
|
|
213
236
|
## Step 6 -- Commit (local only)
|
|
214
237
|
|
|
215
238
|
One commit per file group, in group order. Stage explicitly -- never `git add -A`,
|
package/package.json
CHANGED
|
@@ -86,6 +86,31 @@ No nitpicking. Critical and Important findings are reserved for things that
|
|
|
86
86
|
could cause bugs, outages, data loss, security incidents, or real performance
|
|
87
87
|
problems at scale.
|
|
88
88
|
|
|
89
|
+
**Boundary-first review rule:** for stateful or integration-heavy changes,
|
|
90
|
+
review the complete behavioral chain rather than the changed helper alone:
|
|
91
|
+
|
|
92
|
+
```text
|
|
93
|
+
production entry point
|
|
94
|
+
→ decision point
|
|
95
|
+
→ runtime realization
|
|
96
|
+
→ downstream consumer
|
|
97
|
+
→ persistence/cache/cleanup owner
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
Require the review to distinguish logical identity from physical/runtime
|
|
101
|
+
identity, and to test the relevant transition matrix: success, `null`, `undefined`,
|
|
102
|
+
exception, retry, fallback, rotation, cache reuse/eviction, proxy parity, and
|
|
103
|
+
quoted/history input where applicable. A copied helper test is isolated
|
|
104
|
+
evidence, not proof that the production boundary works.
|
|
105
|
+
|
|
106
|
+
Every verdict-affecting finding should carry a boundary map and an evidence
|
|
107
|
+
class. The report must distinguish source inspection, isolated tests,
|
|
108
|
+
repository tests, composed production-entry probes, production probes, and
|
|
109
|
+
hosted CI. Never upgrade one evidence class into another in the wording.
|
|
110
|
+
|
|
111
|
+
After a fix, re-run the original proof *and* audit the newly changed lifecycle
|
|
112
|
+
boundary for regressions. A green reproduction is necessary, not sufficient.
|
|
113
|
+
|
|
89
114
|
**HARD RULE: the `## 💰 Review Cost` block is mandatory.** Every review MUST
|
|
90
115
|
end with an evidence-backed cost block and no `<N>` placeholders. Use real
|
|
91
116
|
numbers for measurements the host exposes; follow `references/output-format.md`
|
|
@@ -289,7 +314,7 @@ page for this PR.
|
|
|
289
314
|
| Review agents | 2 triage (haiku) | 5 specialists (sonnet) | 5 specialists (sonnet) |
|
|
290
315
|
| Per-finding verification | Criticals + Importants | Criticals + Importants | Criticals + Importants |
|
|
291
316
|
| Completeness critic | skip | skip | included |
|
|
292
|
-
| Proof-of-bug (failing test per Critical) | skip | included (max 5) | included (max 5) |
|
|
317
|
+
| Proof-of-bug (failing test per Critical or composed-boundary Important) | skip | included (max 5) | included (max 5) |
|
|
293
318
|
| Living review artifact | included | included | included |
|
|
294
319
|
| Housekeeping (optional memory/notes writeback) | skip | included | included |
|
|
295
320
|
| `--post` | supported | supported | supported |
|
package/plugins/olko-github-pr/skills/lekker-review/references/agents/completeness-critic.md
CHANGED
|
@@ -14,6 +14,18 @@ were declared "no findings" too quickly. For each angle:
|
|
|
14
14
|
Be concrete: cite diff lines, not vibes. If you genuinely cannot find a missed
|
|
15
15
|
angle, return "No gaps found."
|
|
16
16
|
|
|
17
|
+
Always consider these maintainer-grade gaps before returning no gaps:
|
|
18
|
+
- the real production entry point and downstream consumer were not exercised;
|
|
19
|
+
- the selected value was not traced through runtime realization, persistence,
|
|
20
|
+
cleanup, cache reuse, or fallback;
|
|
21
|
+
- logical identity was conflated with physical/runtime identity;
|
|
22
|
+
- a state-transition matrix is missing for `None`, exceptions, retries,
|
|
23
|
+
rotation, proxying, or alternate transport;
|
|
24
|
+
- the fix was checked only against the original symptom, not for a new defect
|
|
25
|
+
introduced at the changed ordering or lifecycle boundary;
|
|
26
|
+
- the evidence is an isolated harness or local test but the review claims
|
|
27
|
+
composed behavior or hosted-CI acceptance.
|
|
28
|
+
|
|
17
29
|
Agent findings:
|
|
18
30
|
<AGENT_FINDINGS_SUMMARY>
|
|
19
31
|
|
|
@@ -37,7 +37,32 @@ Axes to cover:
|
|
|
37
37
|
sort/dedup on large arrays that could be done at DB level.
|
|
38
38
|
- Integration Contracts: Shopify API misuse, BC API assumptions, webhook
|
|
39
39
|
idempotency, external API pagination not handled.
|
|
40
|
-
|
|
40
|
+
|
|
41
|
+
Boundary contract and lifecycle map (required for every Critical/Important
|
|
42
|
+
finding involving routing, persistence, caching, fallback, proxying, or an
|
|
43
|
+
external integration):
|
|
44
|
+
- `entryPoint`: identify the real production trigger that reaches the changed
|
|
45
|
+
code (HTTP route, webhook, CLI command, queue consumer, gateway ingress, etc.).
|
|
46
|
+
- `decisionPoint`: identify where the behavior is selected or transformed.
|
|
47
|
+
- `realizationPoint`: identify where the selected behavior becomes runtime state.
|
|
48
|
+
- `consumer`: identify the downstream consumer that actually relies on the
|
|
49
|
+
contract, not merely the helper that constructs it.
|
|
50
|
+
- `stateOwner`: identify the durable/session/cache owner when state crosses a
|
|
51
|
+
turn, request, retry, or process boundary.
|
|
52
|
+
- `identityDimensions`: list every identity that must remain distinct, such as
|
|
53
|
+
logical session key, physical session id, provider alias, resolved provider,
|
|
54
|
+
model, and API mode.
|
|
55
|
+
- `transitions`: list the relevant success, `null`, `undefined`, exception, retry, fallback,
|
|
56
|
+
session-rotation, cache-reuse, and proxy/alternate-transport transitions.
|
|
57
|
+
- `evidenceClass`: label the strongest evidence actually checked: `source`,
|
|
58
|
+
`isolated-test`, `repository-test`, `composed-test`, `production-probe`, or
|
|
59
|
+
`hosted-ci`. Never call an isolated harness a production-path proof.
|
|
60
|
+
|
|
61
|
+
Do not report a stateful/integration finding without this boundary object. If
|
|
62
|
+
the production entry point or downstream consumer cannot be identified, mark
|
|
63
|
+
the claim as unverified rather than inferring behavior from a helper alone.
|
|
64
|
+
|
|
65
|
+
GraphQL pagination (GQL-1): for every GraphQL query in the diff that uses a
|
|
41
66
|
nodes connection (`nodes { ... }`):
|
|
42
67
|
(a) Check that `pageInfo { hasNextPage endCursor }` is present alongside nodes: if missing, Critical.
|
|
43
68
|
(b) Check that all pages are fetched (a loop or recursion using endCursor): a single-page fetch is a bug, Critical.
|
|
@@ -4,11 +4,18 @@
|
|
|
4
4
|
|
|
5
5
|
## Mission
|
|
6
6
|
|
|
7
|
-
A Critical finding
|
|
7
|
+
A Critical finding, or an Important finding with a composed boundary map, is an
|
|
8
|
+
argument until someone runs it. Your job is to turn it
|
|
8
9
|
into a demonstration: write ONE test that asserts the CORRECT behavior of the
|
|
9
10
|
code the finding targets, run it in the real worktree, and capture it failing
|
|
10
11
|
for the exact reason the finding claims.
|
|
11
12
|
|
|
13
|
+
For a finding with `boundary`, prefer entering through the recorded production
|
|
14
|
+
entry point and reaching the recorded downstream consumer. Do not replace a
|
|
15
|
+
composed proof with a copied helper test just because the helper is easier to
|
|
16
|
+
import. If full composition is impossible, report the strongest lower evidence
|
|
17
|
+
class honestly.
|
|
18
|
+
|
|
12
19
|
Write the test as if the bug were already fixed. It must fail today precisely
|
|
13
20
|
*because* it isn't fixed. **Never write a test that asserts the buggy behavior
|
|
14
21
|
just to have something red** -- a proof that asserts wrongness is worthless and
|
|
@@ -128,8 +128,26 @@ g) Mock smell: test the behavior, not the way it's built
|
|
|
128
128
|
(time, randomness, injected network failure), or a dependency that genuinely
|
|
129
129
|
cannot run in the test environment. The tell for a good mock: the behavior
|
|
130
130
|
under test only exists because the boundary did something (e.g. a retry
|
|
131
|
-
|
|
132
|
-
|
|
131
|
+
banner that appears only when the fetch rejects). Even then, prefer an
|
|
132
|
+
injected simple fake over a module-level spy, and assert what the user sees.
|
|
133
|
+
|
|
134
|
+
h) Composed boundary and transition coverage
|
|
135
|
+
For every stateful or integration-heavy change, do not stop at testing the
|
|
136
|
+
helper. Trace the real entry point through the decision point, runtime
|
|
137
|
+
realization, downstream consumer, and persistence/cleanup owner. Require a
|
|
138
|
+
test or explicit evidence for the relevant transition matrix:
|
|
139
|
+
- success, `None`, exception, and disabled-feature paths;
|
|
140
|
+
- retry and fallback behavior;
|
|
141
|
+
- session/request rotation and cache reuse or eviction;
|
|
142
|
+
- proxy or alternate transport parity;
|
|
143
|
+
- quoted/history input versus newly authored input;
|
|
144
|
+
- provider/model/API-mode identity when routing is involved.
|
|
145
|
+
|
|
146
|
+
A test that only asserts a helper return value is `isolated-test` evidence,
|
|
147
|
+
not `composed-test` evidence. Flag a Critical/Important finding when the
|
|
148
|
+
changed contract is consumed through a different path than the test covers,
|
|
149
|
+
or when the test can pass while the stateful consumer still receives the
|
|
150
|
+
wrong identity or ordering.
|
|
133
151
|
|
|
134
152
|
Report each per-line issue as:
|
|
135
153
|
`test-file:line, <concise description of the gap or weakness>`
|
|
@@ -123,6 +123,31 @@ convention finding with no verifiable precedent does not ship.
|
|
|
123
123
|
|
|
124
124
|
---
|
|
125
125
|
|
|
126
|
+
## Step 4b -- Boundary contract and evidence check
|
|
127
|
+
|
|
128
|
+
For any Critical/Important finding involving routing, persistence, caching,
|
|
129
|
+
fallback, proxying, quoted input, or an external integration, read the
|
|
130
|
+
finding's `boundary` object. It should identify the real production entry
|
|
131
|
+
point, decision point, runtime realization point, downstream consumer, state
|
|
132
|
+
owner, and relevant identity dimensions/transitions.
|
|
133
|
+
|
|
134
|
+
Reconstruct the path in `WORKTREE_PATH` from entry point to consumer. A copied
|
|
135
|
+
helper test or a mock that never reaches the consumer is only isolated
|
|
136
|
+
evidence. Do not confirm a verdict-affecting finding as production-path proof
|
|
137
|
+
unless the claimed failure reaches the stateful consumer or the review clearly
|
|
138
|
+
labels the evidence as isolated.
|
|
139
|
+
|
|
140
|
+
If the finding has no boundary object or no usable production entry point or
|
|
141
|
+
consumer, keep it unverified and non-blocking; drop it when the missing boundary
|
|
142
|
+
is the crux of the claim. The workflow enforces this as an unverified observation
|
|
143
|
+
even if a verifier confirms it. Record the strongest evidence class actually checked
|
|
144
|
+
(`source`, `isolated-test`, `repository-test`, `composed-test`,
|
|
145
|
+
`production-probe`, or `hosted-ci`) in the reasoning.
|
|
146
|
+
|
|
147
|
+
Build a small transition matrix for the affected contract. At minimum check
|
|
148
|
+
success, `null`, `undefined`, exception, retry/fallback, state rotation, cache reuse or
|
|
149
|
+
eviction, and proxy/alternate transport whenever those transitions exist.
|
|
150
|
+
|
|
126
151
|
## Step 5 -- Fix validation
|
|
127
152
|
|
|
128
153
|
Before confirming the finding, identify the callers and consumers of the changed
|
|
@@ -210,6 +210,29 @@ check after any revert triggered here before moving on.
|
|
|
210
210
|
|
|
211
211
|
---
|
|
212
212
|
|
|
213
|
+
## Step 5c -- Fix delta audit
|
|
214
|
+
|
|
215
|
+
After the original proof flips red → green, audit the *new* boundary created by
|
|
216
|
+
the fix. Do not assume that making the original assertion pass makes the
|
|
217
|
+
lifecycle safe.
|
|
218
|
+
|
|
219
|
+
For each fixed Critical/Important finding involving state, routing, ordering,
|
|
220
|
+
proxying, persistence, caching, fallback, or quoted input:
|
|
221
|
+
|
|
222
|
+
1. Re-read the entire changed path from the production entry point to the
|
|
223
|
+
downstream consumer.
|
|
224
|
+
2. Re-run the relevant transition matrix: success, `None`, exception, retry,
|
|
225
|
+
fallback, rotation, cache reuse/eviction, and alternate transport where
|
|
226
|
+
applicable.
|
|
227
|
+
3. Check that logical identity, runtime identity, provider/API identity, and
|
|
228
|
+
durable identity are still distinct where the contract requires it.
|
|
229
|
+
4. Add a focused regression probe when the fix moved work across an ordering or
|
|
230
|
+
lifecycle boundary. The probe must enter through the real path when feasible.
|
|
231
|
+
|
|
232
|
+
If the audit finds a new failure, revert the whole group and mark it
|
|
233
|
+
`failed: fix-delta-regression`. A green original proof is not sufficient to
|
|
234
|
+
commit a fix that breaks a neighboring transition.
|
|
235
|
+
|
|
213
236
|
## Step 6 -- Commit (local only)
|
|
214
237
|
|
|
215
238
|
One commit per file group, in group order. Stage explicitly -- never `git add -A`,
|
|
@@ -8,11 +8,30 @@ export const meta = {
|
|
|
8
8
|
// Schemas
|
|
9
9
|
// ---------------------------------------------------------------------------
|
|
10
10
|
|
|
11
|
+
const BOUNDARY_SCHEMA = {
|
|
12
|
+
type: 'object',
|
|
13
|
+
required: ['entryPoint', 'consumer'],
|
|
14
|
+
properties: {
|
|
15
|
+
entryPoint: { type: 'string', minLength: 1, pattern: '\\S' },
|
|
16
|
+
decisionPoint: { type: 'string' },
|
|
17
|
+
realizationPoint: { type: 'string' },
|
|
18
|
+
consumer: { type: 'string', minLength: 1, pattern: '\\S' },
|
|
19
|
+
stateOwner: { type: 'string' },
|
|
20
|
+
evidenceClass: { type: 'string' },
|
|
21
|
+
identityDimensions: { type: 'array', items: { type: 'string' } },
|
|
22
|
+
transitions: { type: 'array', items: { type: 'string' } },
|
|
23
|
+
},
|
|
24
|
+
}
|
|
25
|
+
|
|
11
26
|
const FINDINGS_ARRAY_SCHEMA = {
|
|
12
27
|
type: 'array',
|
|
13
28
|
items: {
|
|
14
29
|
type: 'object',
|
|
15
30
|
required: ['file', 'line', 'severity', 'title', 'description', 'badCode', 'fix'],
|
|
31
|
+
anyOf: [
|
|
32
|
+
{ properties: { severity: { enum: ['observation', 'idiomatic'] } } },
|
|
33
|
+
{ required: ['boundary'] },
|
|
34
|
+
],
|
|
16
35
|
properties: {
|
|
17
36
|
file: { type: 'string' },
|
|
18
37
|
line: { type: 'integer' },
|
|
@@ -23,6 +42,7 @@ const FINDINGS_ARRAY_SCHEMA = {
|
|
|
23
42
|
fix: { type: 'string' },
|
|
24
43
|
precedent: { type: 'string' },
|
|
25
44
|
rule: { type: 'string', minLength: 1, pattern: '\\S' },
|
|
45
|
+
boundary: BOUNDARY_SCHEMA,
|
|
26
46
|
},
|
|
27
47
|
},
|
|
28
48
|
}
|
|
@@ -267,6 +287,41 @@ function shouldVerify(finding) {
|
|
|
267
287
|
return finding.severity === 'critical' || finding.severity === 'important'
|
|
268
288
|
}
|
|
269
289
|
|
|
290
|
+
/**
|
|
291
|
+
* Does the finding carry enough of a composed boundary map to attempt a
|
|
292
|
+
* proof? Requires an entry point and a consumer, plus at least one of a
|
|
293
|
+
* realization point, state owner, or recorded transition.
|
|
294
|
+
*/
|
|
295
|
+
function hasBoundaryProofSurface(finding) {
|
|
296
|
+
const boundary = finding && finding.boundary
|
|
297
|
+
if (!boundary || typeof boundary !== 'object') {
|
|
298
|
+
return false
|
|
299
|
+
}
|
|
300
|
+
const hasText = function(value) {
|
|
301
|
+
return typeof value === 'string' && value.trim().length > 0
|
|
302
|
+
}
|
|
303
|
+
const hasList = function(value) {
|
|
304
|
+
return Array.isArray(value) && value.length > 0
|
|
305
|
+
}
|
|
306
|
+
return hasText(boundary.entryPoint)
|
|
307
|
+
&& hasText(boundary.consumer)
|
|
308
|
+
&& (
|
|
309
|
+
hasText(boundary.realizationPoint)
|
|
310
|
+
|| hasText(boundary.stateOwner)
|
|
311
|
+
|| hasList(boundary.transitions)
|
|
312
|
+
)
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
/**
|
|
316
|
+
* Does this finding qualify for the Prove stage? Every Critical finding
|
|
317
|
+
* qualifies; an Important finding only qualifies when it has a composed
|
|
318
|
+
* boundary map to prove against.
|
|
319
|
+
*/
|
|
320
|
+
function shouldProve(finding) {
|
|
321
|
+
return finding.severity === 'critical'
|
|
322
|
+
|| (finding.severity === 'important' && hasBoundaryProofSurface(finding))
|
|
323
|
+
}
|
|
324
|
+
|
|
270
325
|
function normalizeProof(result) {
|
|
271
326
|
const hasText = function(value) {
|
|
272
327
|
return typeof value === 'string' && value.trim().length > 0
|
|
@@ -520,7 +575,29 @@ const budgetAtStart = budget.spent()
|
|
|
520
575
|
// rule-specific checks and skip only its runtime-oriented challenges.
|
|
521
576
|
// -------------------------------------------------------------------------
|
|
522
577
|
|
|
578
|
+
// A model verdict cannot replace the required production boundary map.
|
|
579
|
+
function enforceBoundary(finding) {
|
|
580
|
+
if (!shouldVerify(finding)) return finding
|
|
581
|
+
const boundary = finding.boundary
|
|
582
|
+
const hasText = function(value) {
|
|
583
|
+
return typeof value === 'string' && value.trim().length > 0
|
|
584
|
+
}
|
|
585
|
+
if (boundary && typeof boundary === 'object' && !Array.isArray(boundary)
|
|
586
|
+
&& hasText(boundary.entryPoint) && hasText(boundary.consumer)) {
|
|
587
|
+
return finding
|
|
588
|
+
}
|
|
589
|
+
downgradedCount++
|
|
590
|
+
const unverified = Object.assign({}, finding, {
|
|
591
|
+
severity: 'observation',
|
|
592
|
+
verificationStatus: 'unverified',
|
|
593
|
+
verifierReasoning: 'Missing boundary object or identified production entry point/downstream consumer; downgraded to non-blocking',
|
|
594
|
+
})
|
|
595
|
+
delete unverified.proof
|
|
596
|
+
return unverified
|
|
597
|
+
}
|
|
598
|
+
|
|
523
599
|
async function verifyAll(findings) {
|
|
600
|
+
findings = findings.map(enforceBoundary)
|
|
524
601
|
const inScope = findings.filter(function(f) {
|
|
525
602
|
return shouldVerify(f)
|
|
526
603
|
})
|
|
@@ -704,7 +781,8 @@ const budgetAtStart = budget.spent()
|
|
|
704
781
|
}
|
|
705
782
|
|
|
706
783
|
// Verify the new finding through the same verifier path
|
|
707
|
-
const newFinding = Object.assign({}, reResult.findings[0])
|
|
784
|
+
const newFinding = enforceBoundary(Object.assign({}, reResult.findings[0]))
|
|
785
|
+
if (newFinding.verificationStatus === 'unverified') return [newFinding]
|
|
708
786
|
newFinding._dimension = `critic:${angle.axis}`
|
|
709
787
|
|
|
710
788
|
agentCount++
|
|
@@ -749,9 +827,13 @@ const budgetAtStart = budget.spent()
|
|
|
749
827
|
|
|
750
828
|
} // end ENGINE === 'workflow' branch
|
|
751
829
|
|
|
830
|
+
// Recheck after critic dedup and imported results, before proof promotion.
|
|
831
|
+
finalFindings = finalFindings.map(enforceBoundary)
|
|
832
|
+
|
|
752
833
|
// -------------------------------------------------------------------------
|
|
753
|
-
// Prove: for each Critical finding
|
|
754
|
-
//
|
|
834
|
+
// Prove: for each Critical finding and each Important finding with a
|
|
835
|
+
// composed boundary map (excluding hard rules, which are policy violations
|
|
836
|
+
// with no runtime failure to demonstrate), attempt to produce an
|
|
755
837
|
// executable failing test in the worktree. Only runs with a worktree and
|
|
756
838
|
// outside scan depth, and is capped so a pathological finding count can't
|
|
757
839
|
// blow the budget.
|
|
@@ -770,7 +852,7 @@ const budgetAtStart = budget.spent()
|
|
|
770
852
|
|
|
771
853
|
if (worktreePath && depth !== 'scan') {
|
|
772
854
|
const proveCandidates = finalFindings.filter(function(f) {
|
|
773
|
-
return f
|
|
855
|
+
return shouldProve(f) && !isHardRule(f)
|
|
774
856
|
})
|
|
775
857
|
|
|
776
858
|
if (proveCandidates.length > 0) {
|
|
@@ -779,7 +861,7 @@ const budgetAtStart = budget.spent()
|
|
|
779
861
|
const capped = proveCandidates.slice(0, 5)
|
|
780
862
|
const skipped = proveCandidates.length - capped.length
|
|
781
863
|
if (skipped > 0) {
|
|
782
|
-
log(`Prove: capping at 5 provers, skipping ${skipped} additional
|
|
864
|
+
log(`Prove: capping at 5 provers, skipping ${skipped} additional finding(s)`)
|
|
783
865
|
}
|
|
784
866
|
|
|
785
867
|
await batched(capped.map(function(finding) {
|