@olegkoval/agent-skills 1.49.1 → 1.50.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/prompts/codexloop.prompt.md +15 -2
- package/.kiro/steering/codexloop.md +14 -1
- package/.windsurf/rules/codexloop.md +14 -1
- package/README.md +2 -1
- package/adapters/claude/olko-github-pr/skills/codexloop/SKILL.md +15 -2
- package/adapters/claude/olko-github-pr/skills/lekker-review/SKILL.md +26 -1
- package/adapters/claude/olko-github-pr/skills/lekker-review/references/agents/completeness-critic.md +12 -0
- package/adapters/claude/olko-github-pr/skills/lekker-review/references/agents/implementation.md +26 -1
- package/adapters/claude/olko-github-pr/skills/lekker-review/references/agents/prover.md +8 -1
- package/adapters/claude/olko-github-pr/skills/lekker-review/references/agents/test-quality.md +20 -2
- package/adapters/claude/olko-github-pr/skills/lekker-review/references/agents/verifier.md +25 -0
- package/adapters/claude/olko-github-pr/skills/lekker-review/references/fix-mode.md +23 -0
- package/adapters/claude/olko-reflection/skills/agent-ops-retro/scripts/mine-transcripts.mjs +10 -2
- package/adapters/cursor/olko-github-pr/skills/codexloop/SKILL.md +15 -2
- package/adapters/cursor/olko-reflection/skills/agent-ops-retro/scripts/mine-transcripts.mjs +10 -2
- package/adapters/grok/olko-github-pr/skills/codexloop/SKILL.md +15 -2
- package/adapters/grok/olko-reflection/skills/agent-ops-retro/scripts/mine-transcripts.mjs +10 -2
- package/catalog/skills.json +1 -1
- package/package.json +1 -1
- package/plugins/olko-apple-kit/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-creative/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-garmin-kit/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-git-tools/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-github-pr/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-github-pr/skills/codexloop/SKILL.md +15 -2
- package/plugins/olko-github-pr/skills/lekker-review/SKILL.md +26 -1
- package/plugins/olko-github-pr/skills/lekker-review/references/agents/completeness-critic.md +12 -0
- package/plugins/olko-github-pr/skills/lekker-review/references/agents/implementation.md +26 -1
- package/plugins/olko-github-pr/skills/lekker-review/references/agents/prover.md +8 -1
- package/plugins/olko-github-pr/skills/lekker-review/references/agents/test-quality.md +20 -2
- package/plugins/olko-github-pr/skills/lekker-review/references/agents/verifier.md +25 -0
- package/plugins/olko-github-pr/skills/lekker-review/references/fix-mode.md +23 -0
- package/plugins/olko-github-pr/skills/lekker-review/workflow.js +87 -5
- package/plugins/olko-obsidian/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-product/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-reflection/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-reflection/skills/agent-ops-retro/scripts/mine-transcripts.mjs +10 -2
- package/plugins/olko-release/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-skill-meta/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-web-ops/.claude-plugin/plugin.json +1 -1
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
description: "Iteratively drives a GitHub PR to zero unresolved OpenAI Codex review comments, verifying every finding against the real code first, fixes only the correct ones, and rebuts false positives without changing correct code."
|
|
5
5
|
---
|
|
6
6
|
|
|
7
|
-
Use the
|
|
7
|
+
Use the codexloop skill.
|
|
8
8
|
|
|
9
9
|
# Codexloop
|
|
10
10
|
|
|
@@ -63,6 +63,12 @@ Switch to the PR branch if not already on it. Capture `OWNER`/`REPO` (`gh repo v
|
|
|
63
63
|
|
|
64
64
|
## 2. The loop (max 5 iterations)
|
|
65
65
|
|
|
66
|
+
At the end of every invocation, post exactly one top-level PR conversation comment summarizing
|
|
67
|
+
the result, including iterations, comments fixed, comments rebutted, and anything remaining. This
|
|
68
|
+
is required even when the loop times out or reaches its iteration cap; report that state honestly.
|
|
69
|
+
End the comment with: **Useful? React with 👍 / 👎.** Do not post a separate summary comment for
|
|
70
|
+
each iteration.
|
|
71
|
+
|
|
66
72
|
Keep an explicit iteration counter and stop at 5: the cap is a real bound to enforce, not a
|
|
67
73
|
figure of speech. Each pass through A–G is one iteration; on hitting the cap, go straight to the
|
|
68
74
|
report and list what is still unresolved rather than starting a sixth.
|
|
@@ -184,6 +190,10 @@ gh pr view <PR> --json headRefOid -q .headRefOid # must match
|
|
|
184
190
|
|
|
185
191
|
If they differ, stop: the fix is not on the PR, so nothing may be resolved yet.
|
|
186
192
|
|
|
193
|
+
Every fixed Codex thread must be replied to and resolved after this push is verified. A fix is not
|
|
194
|
+
complete while its thread remains unresolved. If no code changed, rebutted threads can be replied
|
|
195
|
+
to and resolved without a push.
|
|
196
|
+
|
|
187
197
|
### F. Reply to and resolve every addressed thread
|
|
188
198
|
|
|
189
199
|
Only now, with the fixes pushed, reply and resolve. Fetch unresolved threads, **following
|
|
@@ -220,7 +230,10 @@ gh api graphql -f query='mutation { resolveReviewThread(input: {threadId: "THREA
|
|
|
220
230
|
```
|
|
221
231
|
|
|
222
232
|
Resolve a thread only for comments authored by `$BOT` that you have fixed or rebutted: never
|
|
223
|
-
blanket-resolve, and never resolve a human reviewer's thread.
|
|
233
|
+
blanket-resolve, and never resolve a human reviewer's thread. For each fixed comment, verify the
|
|
234
|
+
push first, reply with the change made, then resolve the thread and verify `isResolved: true`.
|
|
235
|
+
For each rebutted comment, reply with the evidence and verify resolution. Do not report a comment
|
|
236
|
+
as fixed or rebutted until its thread is confirmed resolved.
|
|
224
237
|
|
|
225
238
|
Threads you are **rebutting** need no push, so they may be replied to and resolved regardless of
|
|
226
239
|
whether step D changed code.
|
|
@@ -62,6 +62,12 @@ Switch to the PR branch if not already on it. Capture `OWNER`/`REPO` (`gh repo v
|
|
|
62
62
|
|
|
63
63
|
## 2. The loop (max 5 iterations)
|
|
64
64
|
|
|
65
|
+
At the end of every invocation, post exactly one top-level PR conversation comment summarizing
|
|
66
|
+
the result, including iterations, comments fixed, comments rebutted, and anything remaining. This
|
|
67
|
+
is required even when the loop times out or reaches its iteration cap; report that state honestly.
|
|
68
|
+
End the comment with: **Useful? React with 👍 / 👎.** Do not post a separate summary comment for
|
|
69
|
+
each iteration.
|
|
70
|
+
|
|
65
71
|
Keep an explicit iteration counter and stop at 5: the cap is a real bound to enforce, not a
|
|
66
72
|
figure of speech. Each pass through A–G is one iteration; on hitting the cap, go straight to the
|
|
67
73
|
report and list what is still unresolved rather than starting a sixth.
|
|
@@ -183,6 +189,10 @@ gh pr view <PR> --json headRefOid -q .headRefOid # must match
|
|
|
183
189
|
|
|
184
190
|
If they differ, stop: the fix is not on the PR, so nothing may be resolved yet.
|
|
185
191
|
|
|
192
|
+
Every fixed Codex thread must be replied to and resolved after this push is verified. A fix is not
|
|
193
|
+
complete while its thread remains unresolved. If no code changed, rebutted threads can be replied
|
|
194
|
+
to and resolved without a push.
|
|
195
|
+
|
|
186
196
|
### F. Reply to and resolve every addressed thread
|
|
187
197
|
|
|
188
198
|
Only now, with the fixes pushed, reply and resolve. Fetch unresolved threads, **following
|
|
@@ -219,7 +229,10 @@ gh api graphql -f query='mutation { resolveReviewThread(input: {threadId: "THREA
|
|
|
219
229
|
```
|
|
220
230
|
|
|
221
231
|
Resolve a thread only for comments authored by `$BOT` that you have fixed or rebutted: never
|
|
222
|
-
blanket-resolve, and never resolve a human reviewer's thread.
|
|
232
|
+
blanket-resolve, and never resolve a human reviewer's thread. For each fixed comment, verify the
|
|
233
|
+
push first, reply with the change made, then resolve the thread and verify `isResolved: true`.
|
|
234
|
+
For each rebutted comment, reply with the evidence and verify resolution. Do not report a comment
|
|
235
|
+
as fixed or rebutted until its thread is confirmed resolved.
|
|
223
236
|
|
|
224
237
|
Threads you are **rebutting** need no push, so they may be replied to and resolved regardless of
|
|
225
238
|
whether step D changed code.
|
|
@@ -61,6 +61,12 @@ Switch to the PR branch if not already on it. Capture `OWNER`/`REPO` (`gh repo v
|
|
|
61
61
|
|
|
62
62
|
## 2. The loop (max 5 iterations)
|
|
63
63
|
|
|
64
|
+
At the end of every invocation, post exactly one top-level PR conversation comment summarizing
|
|
65
|
+
the result, including iterations, comments fixed, comments rebutted, and anything remaining. This
|
|
66
|
+
is required even when the loop times out or reaches its iteration cap; report that state honestly.
|
|
67
|
+
End the comment with: **Useful? React with 👍 / 👎.** Do not post a separate summary comment for
|
|
68
|
+
each iteration.
|
|
69
|
+
|
|
64
70
|
Keep an explicit iteration counter and stop at 5: the cap is a real bound to enforce, not a
|
|
65
71
|
figure of speech. Each pass through A–G is one iteration; on hitting the cap, go straight to the
|
|
66
72
|
report and list what is still unresolved rather than starting a sixth.
|
|
@@ -182,6 +188,10 @@ gh pr view <PR> --json headRefOid -q .headRefOid # must match
|
|
|
182
188
|
|
|
183
189
|
If they differ, stop: the fix is not on the PR, so nothing may be resolved yet.
|
|
184
190
|
|
|
191
|
+
Every fixed Codex thread must be replied to and resolved after this push is verified. A fix is not
|
|
192
|
+
complete while its thread remains unresolved. If no code changed, rebutted threads can be replied
|
|
193
|
+
to and resolved without a push.
|
|
194
|
+
|
|
185
195
|
### F. Reply to and resolve every addressed thread
|
|
186
196
|
|
|
187
197
|
Only now, with the fixes pushed, reply and resolve. Fetch unresolved threads, **following
|
|
@@ -218,7 +228,10 @@ gh api graphql -f query='mutation { resolveReviewThread(input: {threadId: "THREA
|
|
|
218
228
|
```
|
|
219
229
|
|
|
220
230
|
Resolve a thread only for comments authored by `$BOT` that you have fixed or rebutted: never
|
|
221
|
-
blanket-resolve, and never resolve a human reviewer's thread.
|
|
231
|
+
blanket-resolve, and never resolve a human reviewer's thread. For each fixed comment, verify the
|
|
232
|
+
push first, reply with the change made, then resolve the thread and verify `isResolved: true`.
|
|
233
|
+
For each rebutted comment, reply with the evidence and verify resolution. Do not report a comment
|
|
234
|
+
as fixed or rebutted until its thread is confirmed resolved.
|
|
222
235
|
|
|
223
236
|
Threads you are **rebutting** need no push, so they may be replied to and resolved regardless of
|
|
224
237
|
whether step D changed code.
|
package/README.md
CHANGED
|
@@ -115,7 +115,8 @@ cd agent-skills
|
|
|
115
115
|
./scripts/install-codex-symlinks.sh
|
|
116
116
|
```
|
|
117
117
|
|
|
118
|
-
Then mention a lookup name in a new Codex session
|
|
118
|
+
Then mention a lookup name in a new Codex session. The Codexloop skill is available directly as
|
|
119
|
+
`$codexloop` (or by name in a prompt):
|
|
119
120
|
|
|
120
121
|
```text
|
|
121
122
|
Use the olko:semantic-release-beta skill to add prereleases on a beta branch.
|
|
@@ -8,7 +8,7 @@ description: >
|
|
|
8
8
|
review", "fix codex comments", "codex loop", or wants to iterate on Codex PR review feedback.
|
|
9
9
|
compatibility: GitHub only (Codex code review is a GitHub app). Requires git + gh (GitHub CLI) authenticated, and the Codex / ChatGPT connector app installed on the repo with code review enabled.
|
|
10
10
|
metadata:
|
|
11
|
-
version: "1.
|
|
11
|
+
version: "1.1"
|
|
12
12
|
allowed-tools: Bash(gh:*) Bash(git:*)
|
|
13
13
|
---
|
|
14
14
|
<!-- Generated by scripts/build-adapters.sh. Do not edit directly. -->
|
|
@@ -70,6 +70,12 @@ Switch to the PR branch if not already on it. Capture `OWNER`/`REPO` (`gh repo v
|
|
|
70
70
|
|
|
71
71
|
## 2. The loop (max 5 iterations)
|
|
72
72
|
|
|
73
|
+
At the end of every invocation, post exactly one top-level PR conversation comment summarizing
|
|
74
|
+
the result, including iterations, comments fixed, comments rebutted, and anything remaining. This
|
|
75
|
+
is required even when the loop times out or reaches its iteration cap; report that state honestly.
|
|
76
|
+
End the comment with: **Useful? React with 👍 / 👎.** Do not post a separate summary comment for
|
|
77
|
+
each iteration.
|
|
78
|
+
|
|
73
79
|
Keep an explicit iteration counter and stop at 5: the cap is a real bound to enforce, not a
|
|
74
80
|
figure of speech. Each pass through A–G is one iteration; on hitting the cap, go straight to the
|
|
75
81
|
report and list what is still unresolved rather than starting a sixth.
|
|
@@ -191,6 +197,10 @@ gh pr view <PR> --json headRefOid -q .headRefOid # must match
|
|
|
191
197
|
|
|
192
198
|
If they differ, stop: the fix is not on the PR, so nothing may be resolved yet.
|
|
193
199
|
|
|
200
|
+
Every fixed Codex thread must be replied to and resolved after this push is verified. A fix is not
|
|
201
|
+
complete while its thread remains unresolved. If no code changed, rebutted threads can be replied
|
|
202
|
+
to and resolved without a push.
|
|
203
|
+
|
|
194
204
|
### F. Reply to and resolve every addressed thread
|
|
195
205
|
|
|
196
206
|
Only now, with the fixes pushed, reply and resolve. Fetch unresolved threads, **following
|
|
@@ -227,7 +237,10 @@ gh api graphql -f query='mutation { resolveReviewThread(input: {threadId: "THREA
|
|
|
227
237
|
```
|
|
228
238
|
|
|
229
239
|
Resolve a thread only for comments authored by `$BOT` that you have fixed or rebutted: never
|
|
230
|
-
blanket-resolve, and never resolve a human reviewer's thread.
|
|
240
|
+
blanket-resolve, and never resolve a human reviewer's thread. For each fixed comment, verify the
|
|
241
|
+
push first, reply with the change made, then resolve the thread and verify `isResolved: true`.
|
|
242
|
+
For each rebutted comment, reply with the evidence and verify resolution. Do not report a comment
|
|
243
|
+
as fixed or rebutted until its thread is confirmed resolved.
|
|
231
244
|
|
|
232
245
|
Threads you are **rebutting** need no push, so they may be replied to and resolved regardless of
|
|
233
246
|
whether step D changed code.
|
|
@@ -87,6 +87,31 @@ No nitpicking. Critical and Important findings are reserved for things that
|
|
|
87
87
|
could cause bugs, outages, data loss, security incidents, or real performance
|
|
88
88
|
problems at scale.
|
|
89
89
|
|
|
90
|
+
**Boundary-first review rule:** for stateful or integration-heavy changes,
|
|
91
|
+
review the complete behavioral chain rather than the changed helper alone:
|
|
92
|
+
|
|
93
|
+
```text
|
|
94
|
+
production entry point
|
|
95
|
+
→ decision point
|
|
96
|
+
→ runtime realization
|
|
97
|
+
→ downstream consumer
|
|
98
|
+
→ persistence/cache/cleanup owner
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
Require the review to distinguish logical identity from physical/runtime
|
|
102
|
+
identity, and to test the relevant transition matrix: success, `null`, `undefined`,
|
|
103
|
+
exception, retry, fallback, rotation, cache reuse/eviction, proxy parity, and
|
|
104
|
+
quoted/history input where applicable. A copied helper test is isolated
|
|
105
|
+
evidence, not proof that the production boundary works.
|
|
106
|
+
|
|
107
|
+
Every verdict-affecting finding should carry a boundary map and an evidence
|
|
108
|
+
class. The report must distinguish source inspection, isolated tests,
|
|
109
|
+
repository tests, composed production-entry probes, production probes, and
|
|
110
|
+
hosted CI. Never upgrade one evidence class into another in the wording.
|
|
111
|
+
|
|
112
|
+
After a fix, re-run the original proof *and* audit the newly changed lifecycle
|
|
113
|
+
boundary for regressions. A green reproduction is necessary, not sufficient.
|
|
114
|
+
|
|
90
115
|
**HARD RULE: the `## 💰 Review Cost` block is mandatory.** Every review MUST
|
|
91
116
|
end with an evidence-backed cost block and no `<N>` placeholders. Use real
|
|
92
117
|
numbers for measurements the host exposes; follow `references/output-format.md`
|
|
@@ -290,7 +315,7 @@ page for this PR.
|
|
|
290
315
|
| Review agents | 2 triage (haiku) | 5 specialists (sonnet) | 5 specialists (sonnet) |
|
|
291
316
|
| Per-finding verification | Criticals + Importants | Criticals + Importants | Criticals + Importants |
|
|
292
317
|
| Completeness critic | skip | skip | included |
|
|
293
|
-
| Proof-of-bug (failing test per Critical) | skip | included (max 5) | included (max 5) |
|
|
318
|
+
| Proof-of-bug (failing test per Critical or composed-boundary Important) | skip | included (max 5) | included (max 5) |
|
|
294
319
|
| Living review artifact | included | included | included |
|
|
295
320
|
| Housekeeping (optional memory/notes writeback) | skip | included | included |
|
|
296
321
|
| `--post` | supported | supported | supported |
|
package/adapters/claude/olko-github-pr/skills/lekker-review/references/agents/completeness-critic.md
CHANGED
|
@@ -14,6 +14,18 @@ were declared "no findings" too quickly. For each angle:
|
|
|
14
14
|
Be concrete: cite diff lines, not vibes. If you genuinely cannot find a missed
|
|
15
15
|
angle, return "No gaps found."
|
|
16
16
|
|
|
17
|
+
Always consider these maintainer-grade gaps before returning no gaps:
|
|
18
|
+
- the real production entry point and downstream consumer were not exercised;
|
|
19
|
+
- the selected value was not traced through runtime realization, persistence,
|
|
20
|
+
cleanup, cache reuse, or fallback;
|
|
21
|
+
- logical identity was conflated with physical/runtime identity;
|
|
22
|
+
- a state-transition matrix is missing for `None`, exceptions, retries,
|
|
23
|
+
rotation, proxying, or alternate transport;
|
|
24
|
+
- the fix was checked only against the original symptom, not for a new defect
|
|
25
|
+
introduced at the changed ordering or lifecycle boundary;
|
|
26
|
+
- the evidence is an isolated harness or local test but the review claims
|
|
27
|
+
composed behavior or hosted-CI acceptance.
|
|
28
|
+
|
|
17
29
|
Agent findings:
|
|
18
30
|
<AGENT_FINDINGS_SUMMARY>
|
|
19
31
|
|
package/adapters/claude/olko-github-pr/skills/lekker-review/references/agents/implementation.md
CHANGED
|
@@ -37,7 +37,32 @@ Axes to cover:
|
|
|
37
37
|
sort/dedup on large arrays that could be done at DB level.
|
|
38
38
|
- Integration Contracts: Shopify API misuse, BC API assumptions, webhook
|
|
39
39
|
idempotency, external API pagination not handled.
|
|
40
|
-
|
|
40
|
+
|
|
41
|
+
Boundary contract and lifecycle map (required for every Critical/Important
|
|
42
|
+
finding involving routing, persistence, caching, fallback, proxying, or an
|
|
43
|
+
external integration):
|
|
44
|
+
- `entryPoint`: identify the real production trigger that reaches the changed
|
|
45
|
+
code (HTTP route, webhook, CLI command, queue consumer, gateway ingress, etc.).
|
|
46
|
+
- `decisionPoint`: identify where the behavior is selected or transformed.
|
|
47
|
+
- `realizationPoint`: identify where the selected behavior becomes runtime state.
|
|
48
|
+
- `consumer`: identify the downstream consumer that actually relies on the
|
|
49
|
+
contract, not merely the helper that constructs it.
|
|
50
|
+
- `stateOwner`: identify the durable/session/cache owner when state crosses a
|
|
51
|
+
turn, request, retry, or process boundary.
|
|
52
|
+
- `identityDimensions`: list every identity that must remain distinct, such as
|
|
53
|
+
logical session key, physical session id, provider alias, resolved provider,
|
|
54
|
+
model, and API mode.
|
|
55
|
+
- `transitions`: list the relevant success, `null`, `undefined`, exception, retry, fallback,
|
|
56
|
+
session-rotation, cache-reuse, and proxy/alternate-transport transitions.
|
|
57
|
+
- `evidenceClass`: label the strongest evidence actually checked: `source`,
|
|
58
|
+
`isolated-test`, `repository-test`, `composed-test`, `production-probe`, or
|
|
59
|
+
`hosted-ci`. Never call an isolated harness a production-path proof.
|
|
60
|
+
|
|
61
|
+
Do not report a stateful/integration finding without this boundary object. If
|
|
62
|
+
the production entry point or downstream consumer cannot be identified, mark
|
|
63
|
+
the claim as unverified rather than inferring behavior from a helper alone.
|
|
64
|
+
|
|
65
|
+
GraphQL pagination (GQL-1): for every GraphQL query in the diff that uses a
|
|
41
66
|
nodes connection (`nodes { ... }`):
|
|
42
67
|
(a) Check that `pageInfo { hasNextPage endCursor }` is present alongside nodes: if missing, Critical.
|
|
43
68
|
(b) Check that all pages are fetched (a loop or recursion using endCursor): a single-page fetch is a bug, Critical.
|
|
@@ -4,11 +4,18 @@
|
|
|
4
4
|
|
|
5
5
|
## Mission
|
|
6
6
|
|
|
7
|
-
A Critical finding
|
|
7
|
+
A Critical finding, or an Important finding with a composed boundary map, is an
|
|
8
|
+
argument until someone runs it. Your job is to turn it
|
|
8
9
|
into a demonstration: write ONE test that asserts the CORRECT behavior of the
|
|
9
10
|
code the finding targets, run it in the real worktree, and capture it failing
|
|
10
11
|
for the exact reason the finding claims.
|
|
11
12
|
|
|
13
|
+
For a finding with `boundary`, prefer entering through the recorded production
|
|
14
|
+
entry point and reaching the recorded downstream consumer. Do not replace a
|
|
15
|
+
composed proof with a copied helper test just because the helper is easier to
|
|
16
|
+
import. If full composition is impossible, report the strongest lower evidence
|
|
17
|
+
class honestly.
|
|
18
|
+
|
|
12
19
|
Write the test as if the bug were already fixed. It must fail today precisely
|
|
13
20
|
*because* it isn't fixed. **Never write a test that asserts the buggy behavior
|
|
14
21
|
just to have something red** -- a proof that asserts wrongness is worthless and
|
package/adapters/claude/olko-github-pr/skills/lekker-review/references/agents/test-quality.md
CHANGED
|
@@ -128,8 +128,26 @@ g) Mock smell: test the behavior, not the way it's built
|
|
|
128
128
|
(time, randomness, injected network failure), or a dependency that genuinely
|
|
129
129
|
cannot run in the test environment. The tell for a good mock: the behavior
|
|
130
130
|
under test only exists because the boundary did something (e.g. a retry
|
|
131
|
-
|
|
132
|
-
|
|
131
|
+
banner that appears only when the fetch rejects). Even then, prefer an
|
|
132
|
+
injected simple fake over a module-level spy, and assert what the user sees.
|
|
133
|
+
|
|
134
|
+
h) Composed boundary and transition coverage
|
|
135
|
+
For every stateful or integration-heavy change, do not stop at testing the
|
|
136
|
+
helper. Trace the real entry point through the decision point, runtime
|
|
137
|
+
realization, downstream consumer, and persistence/cleanup owner. Require a
|
|
138
|
+
test or explicit evidence for the relevant transition matrix:
|
|
139
|
+
- success, `None`, exception, and disabled-feature paths;
|
|
140
|
+
- retry and fallback behavior;
|
|
141
|
+
- session/request rotation and cache reuse or eviction;
|
|
142
|
+
- proxy or alternate transport parity;
|
|
143
|
+
- quoted/history input versus newly authored input;
|
|
144
|
+
- provider/model/API-mode identity when routing is involved.
|
|
145
|
+
|
|
146
|
+
A test that only asserts a helper return value is `isolated-test` evidence,
|
|
147
|
+
not `composed-test` evidence. Flag a Critical/Important finding when the
|
|
148
|
+
changed contract is consumed through a different path than the test covers,
|
|
149
|
+
or when the test can pass while the stateful consumer still receives the
|
|
150
|
+
wrong identity or ordering.
|
|
133
151
|
|
|
134
152
|
Report each per-line issue as:
|
|
135
153
|
`test-file:line, <concise description of the gap or weakness>`
|
|
@@ -123,6 +123,31 @@ convention finding with no verifiable precedent does not ship.
|
|
|
123
123
|
|
|
124
124
|
---
|
|
125
125
|
|
|
126
|
+
## Step 4b -- Boundary contract and evidence check
|
|
127
|
+
|
|
128
|
+
For any Critical/Important finding involving routing, persistence, caching,
|
|
129
|
+
fallback, proxying, quoted input, or an external integration, read the
|
|
130
|
+
finding's `boundary` object. It should identify the real production entry
|
|
131
|
+
point, decision point, runtime realization point, downstream consumer, state
|
|
132
|
+
owner, and relevant identity dimensions/transitions.
|
|
133
|
+
|
|
134
|
+
Reconstruct the path in `WORKTREE_PATH` from entry point to consumer. A copied
|
|
135
|
+
helper test or a mock that never reaches the consumer is only isolated
|
|
136
|
+
evidence. Do not confirm a verdict-affecting finding as production-path proof
|
|
137
|
+
unless the claimed failure reaches the stateful consumer or the review clearly
|
|
138
|
+
labels the evidence as isolated.
|
|
139
|
+
|
|
140
|
+
If the finding has no boundary object or no usable production entry point or
|
|
141
|
+
consumer, keep it unverified and non-blocking; drop it when the missing boundary
|
|
142
|
+
is the crux of the claim. The workflow enforces this as an unverified observation
|
|
143
|
+
even if a verifier confirms it. Record the strongest evidence class actually checked
|
|
144
|
+
(`source`, `isolated-test`, `repository-test`, `composed-test`,
|
|
145
|
+
`production-probe`, or `hosted-ci`) in the reasoning.
|
|
146
|
+
|
|
147
|
+
Build a small transition matrix for the affected contract. At minimum check
|
|
148
|
+
success, `null`, `undefined`, exception, retry/fallback, state rotation, cache reuse or
|
|
149
|
+
eviction, and proxy/alternate transport whenever those transitions exist.
|
|
150
|
+
|
|
126
151
|
## Step 5 -- Fix validation
|
|
127
152
|
|
|
128
153
|
Before confirming the finding, identify the callers and consumers of the changed
|
|
@@ -210,6 +210,29 @@ check after any revert triggered here before moving on.
|
|
|
210
210
|
|
|
211
211
|
---
|
|
212
212
|
|
|
213
|
+
## Step 5c -- Fix delta audit
|
|
214
|
+
|
|
215
|
+
After the original proof flips red → green, audit the *new* boundary created by
|
|
216
|
+
the fix. Do not assume that making the original assertion pass makes the
|
|
217
|
+
lifecycle safe.
|
|
218
|
+
|
|
219
|
+
For each fixed Critical/Important finding involving state, routing, ordering,
|
|
220
|
+
proxying, persistence, caching, fallback, or quoted input:
|
|
221
|
+
|
|
222
|
+
1. Re-read the entire changed path from the production entry point to the
|
|
223
|
+
downstream consumer.
|
|
224
|
+
2. Re-run the relevant transition matrix: success, `None`, exception, retry,
|
|
225
|
+
fallback, rotation, cache reuse/eviction, and alternate transport where
|
|
226
|
+
applicable.
|
|
227
|
+
3. Check that logical identity, runtime identity, provider/API identity, and
|
|
228
|
+
durable identity are still distinct where the contract requires it.
|
|
229
|
+
4. Add a focused regression probe when the fix moved work across an ordering or
|
|
230
|
+
lifecycle boundary. The probe must enter through the real path when feasible.
|
|
231
|
+
|
|
232
|
+
If the audit finds a new failure, revert the whole group and mark it
|
|
233
|
+
`failed: fix-delta-regression`. A green original proof is not sufficient to
|
|
234
|
+
commit a fix that breaks a neighboring transition.
|
|
235
|
+
|
|
213
236
|
## Step 6 -- Commit (local only)
|
|
214
237
|
|
|
215
238
|
One commit per file group, in group order. Stage explicitly -- never `git add -A`,
|
|
@@ -45,7 +45,6 @@ function walk(dir, out = []) {
|
|
|
45
45
|
return out;
|
|
46
46
|
}
|
|
47
47
|
|
|
48
|
-
// A user-role text record that is really harness output, not something a person typed.
|
|
49
48
|
// A user-role text record that is really harness output, not something a person typed.
|
|
50
49
|
// Each alternative below was found by a run that mis-counted. The first pass caught the
|
|
51
50
|
// obvious wrappers and still let 26% through: skill body dumps, image attachments,
|
|
@@ -62,6 +61,15 @@ const HARNESS = new RegExp('^(' + [
|
|
|
62
61
|
'\\[Image #\\d+\\]', // bare attachment
|
|
63
62
|
'\\[Image: source:',
|
|
64
63
|
'\\[Pasted text',
|
|
64
|
+
// v3: the 2026-09-29 run found v2 still let 66% through, almost all of it these five.
|
|
65
|
+
'Another Claude session sent a message', // subagent / cross-session hand-back
|
|
66
|
+
'<agent-message',
|
|
67
|
+
'This session is being continued from a previous conversation', // compaction resume
|
|
68
|
+
'Stop hook feedback:',
|
|
69
|
+
'\\[SYSTEM NOTIFICATION',
|
|
70
|
+
'Your response above was cut off',
|
|
71
|
+
'\\(Re-invocation of /',
|
|
72
|
+
'\\[\\d+ prior /loop wakeup',
|
|
65
73
|
].join('|') + ')');
|
|
66
74
|
|
|
67
75
|
const textOf = (msg) => {
|
|
@@ -261,7 +269,7 @@ if (problems.length) {
|
|
|
261
269
|
console.error('INCONSISTENT REPORT, refusing to write:\n ' + problems.join('\n '));
|
|
262
270
|
process.exit(3);
|
|
263
271
|
}
|
|
264
|
-
report.echoFilterVersion =
|
|
272
|
+
report.echoFilterVersion = 3;
|
|
265
273
|
|
|
266
274
|
fs.writeFileSync(OUT, JSON.stringify(report, null, 1));
|
|
267
275
|
fs.writeFileSync(OUT.replace(/\.json$/, '') + '-turns.json', JSON.stringify(S.humanTurns, null, 1));
|
|
@@ -8,7 +8,7 @@ description: >
|
|
|
8
8
|
review", "fix codex comments", "codex loop", or wants to iterate on Codex PR review feedback.
|
|
9
9
|
compatibility: GitHub only (Codex code review is a GitHub app). Requires git + gh (GitHub CLI) authenticated, and the Codex / ChatGPT connector app installed on the repo with code review enabled.
|
|
10
10
|
metadata:
|
|
11
|
-
version: "1.
|
|
11
|
+
version: "1.1"
|
|
12
12
|
allowed-tools: Bash(gh:*) Bash(git:*)
|
|
13
13
|
---
|
|
14
14
|
<!-- Generated by scripts/build-adapters.sh. Do not edit directly. -->
|
|
@@ -70,6 +70,12 @@ Switch to the PR branch if not already on it. Capture `OWNER`/`REPO` (`gh repo v
|
|
|
70
70
|
|
|
71
71
|
## 2. The loop (max 5 iterations)
|
|
72
72
|
|
|
73
|
+
At the end of every invocation, post exactly one top-level PR conversation comment summarizing
|
|
74
|
+
the result, including iterations, comments fixed, comments rebutted, and anything remaining. This
|
|
75
|
+
is required even when the loop times out or reaches its iteration cap; report that state honestly.
|
|
76
|
+
End the comment with: **Useful? React with 👍 / 👎.** Do not post a separate summary comment for
|
|
77
|
+
each iteration.
|
|
78
|
+
|
|
73
79
|
Keep an explicit iteration counter and stop at 5: the cap is a real bound to enforce, not a
|
|
74
80
|
figure of speech. Each pass through A–G is one iteration; on hitting the cap, go straight to the
|
|
75
81
|
report and list what is still unresolved rather than starting a sixth.
|
|
@@ -191,6 +197,10 @@ gh pr view <PR> --json headRefOid -q .headRefOid # must match
|
|
|
191
197
|
|
|
192
198
|
If they differ, stop: the fix is not on the PR, so nothing may be resolved yet.
|
|
193
199
|
|
|
200
|
+
Every fixed Codex thread must be replied to and resolved after this push is verified. A fix is not
|
|
201
|
+
complete while its thread remains unresolved. If no code changed, rebutted threads can be replied
|
|
202
|
+
to and resolved without a push.
|
|
203
|
+
|
|
194
204
|
### F. Reply to and resolve every addressed thread
|
|
195
205
|
|
|
196
206
|
Only now, with the fixes pushed, reply and resolve. Fetch unresolved threads, **following
|
|
@@ -227,7 +237,10 @@ gh api graphql -f query='mutation { resolveReviewThread(input: {threadId: "THREA
|
|
|
227
237
|
```
|
|
228
238
|
|
|
229
239
|
Resolve a thread only for comments authored by `$BOT` that you have fixed or rebutted: never
|
|
230
|
-
blanket-resolve, and never resolve a human reviewer's thread.
|
|
240
|
+
blanket-resolve, and never resolve a human reviewer's thread. For each fixed comment, verify the
|
|
241
|
+
push first, reply with the change made, then resolve the thread and verify `isResolved: true`.
|
|
242
|
+
For each rebutted comment, reply with the evidence and verify resolution. Do not report a comment
|
|
243
|
+
as fixed or rebutted until its thread is confirmed resolved.
|
|
231
244
|
|
|
232
245
|
Threads you are **rebutting** need no push, so they may be replied to and resolved regardless of
|
|
233
246
|
whether step D changed code.
|
|
@@ -45,7 +45,6 @@ function walk(dir, out = []) {
|
|
|
45
45
|
return out;
|
|
46
46
|
}
|
|
47
47
|
|
|
48
|
-
// A user-role text record that is really harness output, not something a person typed.
|
|
49
48
|
// A user-role text record that is really harness output, not something a person typed.
|
|
50
49
|
// Each alternative below was found by a run that mis-counted. The first pass caught the
|
|
51
50
|
// obvious wrappers and still let 26% through: skill body dumps, image attachments,
|
|
@@ -62,6 +61,15 @@ const HARNESS = new RegExp('^(' + [
|
|
|
62
61
|
'\\[Image #\\d+\\]', // bare attachment
|
|
63
62
|
'\\[Image: source:',
|
|
64
63
|
'\\[Pasted text',
|
|
64
|
+
// v3: the 2026-09-29 run found v2 still let 66% through, almost all of it these five.
|
|
65
|
+
'Another Claude session sent a message', // subagent / cross-session hand-back
|
|
66
|
+
'<agent-message',
|
|
67
|
+
'This session is being continued from a previous conversation', // compaction resume
|
|
68
|
+
'Stop hook feedback:',
|
|
69
|
+
'\\[SYSTEM NOTIFICATION',
|
|
70
|
+
'Your response above was cut off',
|
|
71
|
+
'\\(Re-invocation of /',
|
|
72
|
+
'\\[\\d+ prior /loop wakeup',
|
|
65
73
|
].join('|') + ')');
|
|
66
74
|
|
|
67
75
|
const textOf = (msg) => {
|
|
@@ -261,7 +269,7 @@ if (problems.length) {
|
|
|
261
269
|
console.error('INCONSISTENT REPORT, refusing to write:\n ' + problems.join('\n '));
|
|
262
270
|
process.exit(3);
|
|
263
271
|
}
|
|
264
|
-
report.echoFilterVersion =
|
|
272
|
+
report.echoFilterVersion = 3;
|
|
265
273
|
|
|
266
274
|
fs.writeFileSync(OUT, JSON.stringify(report, null, 1));
|
|
267
275
|
fs.writeFileSync(OUT.replace(/\.json$/, '') + '-turns.json', JSON.stringify(S.humanTurns, null, 1));
|
|
@@ -8,7 +8,7 @@ description: >
|
|
|
8
8
|
review", "fix codex comments", "codex loop", or wants to iterate on Codex PR review feedback.
|
|
9
9
|
compatibility: GitHub only (Codex code review is a GitHub app). Requires git + gh (GitHub CLI) authenticated, and the Codex / ChatGPT connector app installed on the repo with code review enabled.
|
|
10
10
|
metadata:
|
|
11
|
-
version: "1.
|
|
11
|
+
version: "1.1"
|
|
12
12
|
allowed-tools: Bash(gh:*) Bash(git:*)
|
|
13
13
|
---
|
|
14
14
|
<!-- Generated by scripts/build-adapters.sh. Do not edit directly. -->
|
|
@@ -70,6 +70,12 @@ Switch to the PR branch if not already on it. Capture `OWNER`/`REPO` (`gh repo v
|
|
|
70
70
|
|
|
71
71
|
## 2. The loop (max 5 iterations)
|
|
72
72
|
|
|
73
|
+
At the end of every invocation, post exactly one top-level PR conversation comment summarizing
|
|
74
|
+
the result, including iterations, comments fixed, comments rebutted, and anything remaining. This
|
|
75
|
+
is required even when the loop times out or reaches its iteration cap; report that state honestly.
|
|
76
|
+
End the comment with: **Useful? React with 👍 / 👎.** Do not post a separate summary comment for
|
|
77
|
+
each iteration.
|
|
78
|
+
|
|
73
79
|
Keep an explicit iteration counter and stop at 5: the cap is a real bound to enforce, not a
|
|
74
80
|
figure of speech. Each pass through A–G is one iteration; on hitting the cap, go straight to the
|
|
75
81
|
report and list what is still unresolved rather than starting a sixth.
|
|
@@ -191,6 +197,10 @@ gh pr view <PR> --json headRefOid -q .headRefOid # must match
|
|
|
191
197
|
|
|
192
198
|
If they differ, stop: the fix is not on the PR, so nothing may be resolved yet.
|
|
193
199
|
|
|
200
|
+
Every fixed Codex thread must be replied to and resolved after this push is verified. A fix is not
|
|
201
|
+
complete while its thread remains unresolved. If no code changed, rebutted threads can be replied
|
|
202
|
+
to and resolved without a push.
|
|
203
|
+
|
|
194
204
|
### F. Reply to and resolve every addressed thread
|
|
195
205
|
|
|
196
206
|
Only now, with the fixes pushed, reply and resolve. Fetch unresolved threads, **following
|
|
@@ -227,7 +237,10 @@ gh api graphql -f query='mutation { resolveReviewThread(input: {threadId: "THREA
|
|
|
227
237
|
```
|
|
228
238
|
|
|
229
239
|
Resolve a thread only for comments authored by `$BOT` that you have fixed or rebutted: never
|
|
230
|
-
blanket-resolve, and never resolve a human reviewer's thread.
|
|
240
|
+
blanket-resolve, and never resolve a human reviewer's thread. For each fixed comment, verify the
|
|
241
|
+
push first, reply with the change made, then resolve the thread and verify `isResolved: true`.
|
|
242
|
+
For each rebutted comment, reply with the evidence and verify resolution. Do not report a comment
|
|
243
|
+
as fixed or rebutted until its thread is confirmed resolved.
|
|
231
244
|
|
|
232
245
|
Threads you are **rebutting** need no push, so they may be replied to and resolved regardless of
|
|
233
246
|
whether step D changed code.
|
|
@@ -45,7 +45,6 @@ function walk(dir, out = []) {
|
|
|
45
45
|
return out;
|
|
46
46
|
}
|
|
47
47
|
|
|
48
|
-
// A user-role text record that is really harness output, not something a person typed.
|
|
49
48
|
// A user-role text record that is really harness output, not something a person typed.
|
|
50
49
|
// Each alternative below was found by a run that mis-counted. The first pass caught the
|
|
51
50
|
// obvious wrappers and still let 26% through: skill body dumps, image attachments,
|
|
@@ -62,6 +61,15 @@ const HARNESS = new RegExp('^(' + [
|
|
|
62
61
|
'\\[Image #\\d+\\]', // bare attachment
|
|
63
62
|
'\\[Image: source:',
|
|
64
63
|
'\\[Pasted text',
|
|
64
|
+
// v3: the 2026-09-29 run found v2 still let 66% through, almost all of it these five.
|
|
65
|
+
'Another Claude session sent a message', // subagent / cross-session hand-back
|
|
66
|
+
'<agent-message',
|
|
67
|
+
'This session is being continued from a previous conversation', // compaction resume
|
|
68
|
+
'Stop hook feedback:',
|
|
69
|
+
'\\[SYSTEM NOTIFICATION',
|
|
70
|
+
'Your response above was cut off',
|
|
71
|
+
'\\(Re-invocation of /',
|
|
72
|
+
'\\[\\d+ prior /loop wakeup',
|
|
65
73
|
].join('|') + ')');
|
|
66
74
|
|
|
67
75
|
const textOf = (msg) => {
|
|
@@ -261,7 +269,7 @@ if (problems.length) {
|
|
|
261
269
|
console.error('INCONSISTENT REPORT, refusing to write:\n ' + problems.join('\n '));
|
|
262
270
|
process.exit(3);
|
|
263
271
|
}
|
|
264
|
-
report.echoFilterVersion =
|
|
272
|
+
report.echoFilterVersion = 3;
|
|
265
273
|
|
|
266
274
|
fs.writeFileSync(OUT, JSON.stringify(report, null, 1));
|
|
267
275
|
fs.writeFileSync(OUT.replace(/\.json$/, '') + '-turns.json', JSON.stringify(S.humanTurns, null, 1));
|
package/catalog/skills.json
CHANGED
|
@@ -144,7 +144,7 @@
|
|
|
144
144
|
},
|
|
145
145
|
{
|
|
146
146
|
"name": "codexloop",
|
|
147
|
-
"lookupName": "
|
|
147
|
+
"lookupName": "codexloop",
|
|
148
148
|
"path": "plugins/olko-github-pr/skills/codexloop",
|
|
149
149
|
"description": "Iteratively drives a GitHub PR to zero unresolved OpenAI Codex review comments, verifying every finding against the real code first, fixes only the correct ones, and rebuts false positives without changing correct code.",
|
|
150
150
|
"tags": [
|
package/package.json
CHANGED
|
@@ -8,7 +8,7 @@ description: >
|
|
|
8
8
|
review", "fix codex comments", "codex loop", or wants to iterate on Codex PR review feedback.
|
|
9
9
|
compatibility: GitHub only (Codex code review is a GitHub app). Requires git + gh (GitHub CLI) authenticated, and the Codex / ChatGPT connector app installed on the repo with code review enabled.
|
|
10
10
|
metadata:
|
|
11
|
-
version: "1.
|
|
11
|
+
version: "1.1"
|
|
12
12
|
allowed-tools: Bash(gh:*) Bash(git:*)
|
|
13
13
|
---
|
|
14
14
|
|
|
@@ -69,6 +69,12 @@ Switch to the PR branch if not already on it. Capture `OWNER`/`REPO` (`gh repo v
|
|
|
69
69
|
|
|
70
70
|
## 2. The loop (max 5 iterations)
|
|
71
71
|
|
|
72
|
+
At the end of every invocation, post exactly one top-level PR conversation comment summarizing
|
|
73
|
+
the result, including iterations, comments fixed, comments rebutted, and anything remaining. This
|
|
74
|
+
is required even when the loop times out or reaches its iteration cap; report that state honestly.
|
|
75
|
+
End the comment with: **Useful? React with 👍 / 👎.** Do not post a separate summary comment for
|
|
76
|
+
each iteration.
|
|
77
|
+
|
|
72
78
|
Keep an explicit iteration counter and stop at 5: the cap is a real bound to enforce, not a
|
|
73
79
|
figure of speech. Each pass through A–G is one iteration; on hitting the cap, go straight to the
|
|
74
80
|
report and list what is still unresolved rather than starting a sixth.
|
|
@@ -190,6 +196,10 @@ gh pr view <PR> --json headRefOid -q .headRefOid # must match
|
|
|
190
196
|
|
|
191
197
|
If they differ, stop: the fix is not on the PR, so nothing may be resolved yet.
|
|
192
198
|
|
|
199
|
+
Every fixed Codex thread must be replied to and resolved after this push is verified. A fix is not
|
|
200
|
+
complete while its thread remains unresolved. If no code changed, rebutted threads can be replied
|
|
201
|
+
to and resolved without a push.
|
|
202
|
+
|
|
193
203
|
### F. Reply to and resolve every addressed thread
|
|
194
204
|
|
|
195
205
|
Only now, with the fixes pushed, reply and resolve. Fetch unresolved threads, **following
|
|
@@ -226,7 +236,10 @@ gh api graphql -f query='mutation { resolveReviewThread(input: {threadId: "THREA
|
|
|
226
236
|
```
|
|
227
237
|
|
|
228
238
|
Resolve a thread only for comments authored by `$BOT` that you have fixed or rebutted: never
|
|
229
|
-
blanket-resolve, and never resolve a human reviewer's thread.
|
|
239
|
+
blanket-resolve, and never resolve a human reviewer's thread. For each fixed comment, verify the
|
|
240
|
+
push first, reply with the change made, then resolve the thread and verify `isResolved: true`.
|
|
241
|
+
For each rebutted comment, reply with the evidence and verify resolution. Do not report a comment
|
|
242
|
+
as fixed or rebutted until its thread is confirmed resolved.
|
|
230
243
|
|
|
231
244
|
Threads you are **rebutting** need no push, so they may be replied to and resolved regardless of
|
|
232
245
|
whether step D changed code.
|
|
@@ -86,6 +86,31 @@ No nitpicking. Critical and Important findings are reserved for things that
|
|
|
86
86
|
could cause bugs, outages, data loss, security incidents, or real performance
|
|
87
87
|
problems at scale.
|
|
88
88
|
|
|
89
|
+
**Boundary-first review rule:** for stateful or integration-heavy changes,
|
|
90
|
+
review the complete behavioral chain rather than the changed helper alone:
|
|
91
|
+
|
|
92
|
+
```text
|
|
93
|
+
production entry point
|
|
94
|
+
→ decision point
|
|
95
|
+
→ runtime realization
|
|
96
|
+
→ downstream consumer
|
|
97
|
+
→ persistence/cache/cleanup owner
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
Require the review to distinguish logical identity from physical/runtime
|
|
101
|
+
identity, and to test the relevant transition matrix: success, `null`, `undefined`,
|
|
102
|
+
exception, retry, fallback, rotation, cache reuse/eviction, proxy parity, and
|
|
103
|
+
quoted/history input where applicable. A copied helper test is isolated
|
|
104
|
+
evidence, not proof that the production boundary works.
|
|
105
|
+
|
|
106
|
+
Every verdict-affecting finding should carry a boundary map and an evidence
|
|
107
|
+
class. The report must distinguish source inspection, isolated tests,
|
|
108
|
+
repository tests, composed production-entry probes, production probes, and
|
|
109
|
+
hosted CI. Never upgrade one evidence class into another in the wording.
|
|
110
|
+
|
|
111
|
+
After a fix, re-run the original proof *and* audit the newly changed lifecycle
|
|
112
|
+
boundary for regressions. A green reproduction is necessary, not sufficient.
|
|
113
|
+
|
|
89
114
|
**HARD RULE: the `## 💰 Review Cost` block is mandatory.** Every review MUST
|
|
90
115
|
end with an evidence-backed cost block and no `<N>` placeholders. Use real
|
|
91
116
|
numbers for measurements the host exposes; follow `references/output-format.md`
|
|
@@ -289,7 +314,7 @@ page for this PR.
|
|
|
289
314
|
| Review agents | 2 triage (haiku) | 5 specialists (sonnet) | 5 specialists (sonnet) |
|
|
290
315
|
| Per-finding verification | Criticals + Importants | Criticals + Importants | Criticals + Importants |
|
|
291
316
|
| Completeness critic | skip | skip | included |
|
|
292
|
-
| Proof-of-bug (failing test per Critical) | skip | included (max 5) | included (max 5) |
|
|
317
|
+
| Proof-of-bug (failing test per Critical or composed-boundary Important) | skip | included (max 5) | included (max 5) |
|
|
293
318
|
| Living review artifact | included | included | included |
|
|
294
319
|
| Housekeeping (optional memory/notes writeback) | skip | included | included |
|
|
295
320
|
| `--post` | supported | supported | supported |
|
package/plugins/olko-github-pr/skills/lekker-review/references/agents/completeness-critic.md
CHANGED
|
@@ -14,6 +14,18 @@ were declared "no findings" too quickly. For each angle:
|
|
|
14
14
|
Be concrete: cite diff lines, not vibes. If you genuinely cannot find a missed
|
|
15
15
|
angle, return "No gaps found."
|
|
16
16
|
|
|
17
|
+
Always consider these maintainer-grade gaps before returning no gaps:
|
|
18
|
+
- the real production entry point and downstream consumer were not exercised;
|
|
19
|
+
- the selected value was not traced through runtime realization, persistence,
|
|
20
|
+
cleanup, cache reuse, or fallback;
|
|
21
|
+
- logical identity was conflated with physical/runtime identity;
|
|
22
|
+
- a state-transition matrix is missing for `None`, exceptions, retries,
|
|
23
|
+
rotation, proxying, or alternate transport;
|
|
24
|
+
- the fix was checked only against the original symptom, not for a new defect
|
|
25
|
+
introduced at the changed ordering or lifecycle boundary;
|
|
26
|
+
- the evidence is an isolated harness or local test but the review claims
|
|
27
|
+
composed behavior or hosted-CI acceptance.
|
|
28
|
+
|
|
17
29
|
Agent findings:
|
|
18
30
|
<AGENT_FINDINGS_SUMMARY>
|
|
19
31
|
|
|
@@ -37,7 +37,32 @@ Axes to cover:
|
|
|
37
37
|
sort/dedup on large arrays that could be done at DB level.
|
|
38
38
|
- Integration Contracts: Shopify API misuse, BC API assumptions, webhook
|
|
39
39
|
idempotency, external API pagination not handled.
|
|
40
|
-
|
|
40
|
+
|
|
41
|
+
Boundary contract and lifecycle map (required for every Critical/Important
|
|
42
|
+
finding involving routing, persistence, caching, fallback, proxying, or an
|
|
43
|
+
external integration):
|
|
44
|
+
- `entryPoint`: identify the real production trigger that reaches the changed
|
|
45
|
+
code (HTTP route, webhook, CLI command, queue consumer, gateway ingress, etc.).
|
|
46
|
+
- `decisionPoint`: identify where the behavior is selected or transformed.
|
|
47
|
+
- `realizationPoint`: identify where the selected behavior becomes runtime state.
|
|
48
|
+
- `consumer`: identify the downstream consumer that actually relies on the
|
|
49
|
+
contract, not merely the helper that constructs it.
|
|
50
|
+
- `stateOwner`: identify the durable/session/cache owner when state crosses a
|
|
51
|
+
turn, request, retry, or process boundary.
|
|
52
|
+
- `identityDimensions`: list every identity that must remain distinct, such as
|
|
53
|
+
logical session key, physical session id, provider alias, resolved provider,
|
|
54
|
+
model, and API mode.
|
|
55
|
+
- `transitions`: list the relevant success, `null`, `undefined`, exception, retry, fallback,
|
|
56
|
+
session-rotation, cache-reuse, and proxy/alternate-transport transitions.
|
|
57
|
+
- `evidenceClass`: label the strongest evidence actually checked: `source`,
|
|
58
|
+
`isolated-test`, `repository-test`, `composed-test`, `production-probe`, or
|
|
59
|
+
`hosted-ci`. Never call an isolated harness a production-path proof.
|
|
60
|
+
|
|
61
|
+
Do not report a stateful/integration finding without this boundary object. If
|
|
62
|
+
the production entry point or downstream consumer cannot be identified, mark
|
|
63
|
+
the claim as unverified rather than inferring behavior from a helper alone.
|
|
64
|
+
|
|
65
|
+
GraphQL pagination (GQL-1): for every GraphQL query in the diff that uses a
|
|
41
66
|
nodes connection (`nodes { ... }`):
|
|
42
67
|
(a) Check that `pageInfo { hasNextPage endCursor }` is present alongside nodes: if missing, Critical.
|
|
43
68
|
(b) Check that all pages are fetched (a loop or recursion using endCursor): a single-page fetch is a bug, Critical.
|
|
@@ -4,11 +4,18 @@
|
|
|
4
4
|
|
|
5
5
|
## Mission
|
|
6
6
|
|
|
7
|
-
A Critical finding
|
|
7
|
+
A Critical finding, or an Important finding with a composed boundary map, is an
|
|
8
|
+
argument until someone runs it. Your job is to turn it
|
|
8
9
|
into a demonstration: write ONE test that asserts the CORRECT behavior of the
|
|
9
10
|
code the finding targets, run it in the real worktree, and capture it failing
|
|
10
11
|
for the exact reason the finding claims.
|
|
11
12
|
|
|
13
|
+
For a finding with `boundary`, prefer entering through the recorded production
|
|
14
|
+
entry point and reaching the recorded downstream consumer. Do not replace a
|
|
15
|
+
composed proof with a copied helper test just because the helper is easier to
|
|
16
|
+
import. If full composition is impossible, report the strongest lower evidence
|
|
17
|
+
class honestly.
|
|
18
|
+
|
|
12
19
|
Write the test as if the bug were already fixed. It must fail today precisely
|
|
13
20
|
*because* it isn't fixed. **Never write a test that asserts the buggy behavior
|
|
14
21
|
just to have something red** -- a proof that asserts wrongness is worthless and
|
|
@@ -128,8 +128,26 @@ g) Mock smell: test the behavior, not the way it's built
|
|
|
128
128
|
(time, randomness, injected network failure), or a dependency that genuinely
|
|
129
129
|
cannot run in the test environment. The tell for a good mock: the behavior
|
|
130
130
|
under test only exists because the boundary did something (e.g. a retry
|
|
131
|
-
|
|
132
|
-
|
|
131
|
+
banner that appears only when the fetch rejects). Even then, prefer an
|
|
132
|
+
injected simple fake over a module-level spy, and assert what the user sees.
|
|
133
|
+
|
|
134
|
+
h) Composed boundary and transition coverage
|
|
135
|
+
For every stateful or integration-heavy change, do not stop at testing the
|
|
136
|
+
helper. Trace the real entry point through the decision point, runtime
|
|
137
|
+
realization, downstream consumer, and persistence/cleanup owner. Require a
|
|
138
|
+
test or explicit evidence for the relevant transition matrix:
|
|
139
|
+
- success, `None`, exception, and disabled-feature paths;
|
|
140
|
+
- retry and fallback behavior;
|
|
141
|
+
- session/request rotation and cache reuse or eviction;
|
|
142
|
+
- proxy or alternate transport parity;
|
|
143
|
+
- quoted/history input versus newly authored input;
|
|
144
|
+
- provider/model/API-mode identity when routing is involved.
|
|
145
|
+
|
|
146
|
+
A test that only asserts a helper return value is `isolated-test` evidence,
|
|
147
|
+
not `composed-test` evidence. Flag a Critical/Important finding when the
|
|
148
|
+
changed contract is consumed through a different path than the test covers,
|
|
149
|
+
or when the test can pass while the stateful consumer still receives the
|
|
150
|
+
wrong identity or ordering.
|
|
133
151
|
|
|
134
152
|
Report each per-line issue as:
|
|
135
153
|
`test-file:line, <concise description of the gap or weakness>`
|
|
@@ -123,6 +123,31 @@ convention finding with no verifiable precedent does not ship.
|
|
|
123
123
|
|
|
124
124
|
---
|
|
125
125
|
|
|
126
|
+
## Step 4b -- Boundary contract and evidence check
|
|
127
|
+
|
|
128
|
+
For any Critical/Important finding involving routing, persistence, caching,
|
|
129
|
+
fallback, proxying, quoted input, or an external integration, read the
|
|
130
|
+
finding's `boundary` object. It should identify the real production entry
|
|
131
|
+
point, decision point, runtime realization point, downstream consumer, state
|
|
132
|
+
owner, and relevant identity dimensions/transitions.
|
|
133
|
+
|
|
134
|
+
Reconstruct the path in `WORKTREE_PATH` from entry point to consumer. A copied
|
|
135
|
+
helper test or a mock that never reaches the consumer is only isolated
|
|
136
|
+
evidence. Do not confirm a verdict-affecting finding as production-path proof
|
|
137
|
+
unless the claimed failure reaches the stateful consumer or the review clearly
|
|
138
|
+
labels the evidence as isolated.
|
|
139
|
+
|
|
140
|
+
If the finding has no boundary object or no usable production entry point or
|
|
141
|
+
consumer, keep it unverified and non-blocking; drop it when the missing boundary
|
|
142
|
+
is the crux of the claim. The workflow enforces this as an unverified observation
|
|
143
|
+
even if a verifier confirms it. Record the strongest evidence class actually checked
|
|
144
|
+
(`source`, `isolated-test`, `repository-test`, `composed-test`,
|
|
145
|
+
`production-probe`, or `hosted-ci`) in the reasoning.
|
|
146
|
+
|
|
147
|
+
Build a small transition matrix for the affected contract. At minimum check
|
|
148
|
+
success, `null`, `undefined`, exception, retry/fallback, state rotation, cache reuse or
|
|
149
|
+
eviction, and proxy/alternate transport whenever those transitions exist.
|
|
150
|
+
|
|
126
151
|
## Step 5 -- Fix validation
|
|
127
152
|
|
|
128
153
|
Before confirming the finding, identify the callers and consumers of the changed
|
|
@@ -210,6 +210,29 @@ check after any revert triggered here before moving on.
|
|
|
210
210
|
|
|
211
211
|
---
|
|
212
212
|
|
|
213
|
+
## Step 5c -- Fix delta audit
|
|
214
|
+
|
|
215
|
+
After the original proof flips red → green, audit the *new* boundary created by
|
|
216
|
+
the fix. Do not assume that making the original assertion pass makes the
|
|
217
|
+
lifecycle safe.
|
|
218
|
+
|
|
219
|
+
For each fixed Critical/Important finding involving state, routing, ordering,
|
|
220
|
+
proxying, persistence, caching, fallback, or quoted input:
|
|
221
|
+
|
|
222
|
+
1. Re-read the entire changed path from the production entry point to the
|
|
223
|
+
downstream consumer.
|
|
224
|
+
2. Re-run the relevant transition matrix: success, `None`, exception, retry,
|
|
225
|
+
fallback, rotation, cache reuse/eviction, and alternate transport where
|
|
226
|
+
applicable.
|
|
227
|
+
3. Check that logical identity, runtime identity, provider/API identity, and
|
|
228
|
+
durable identity are still distinct where the contract requires it.
|
|
229
|
+
4. Add a focused regression probe when the fix moved work across an ordering or
|
|
230
|
+
lifecycle boundary. The probe must enter through the real path when feasible.
|
|
231
|
+
|
|
232
|
+
If the audit finds a new failure, revert the whole group and mark it
|
|
233
|
+
`failed: fix-delta-regression`. A green original proof is not sufficient to
|
|
234
|
+
commit a fix that breaks a neighboring transition.
|
|
235
|
+
|
|
213
236
|
## Step 6 -- Commit (local only)
|
|
214
237
|
|
|
215
238
|
One commit per file group, in group order. Stage explicitly -- never `git add -A`,
|
|
@@ -8,11 +8,30 @@ export const meta = {
|
|
|
8
8
|
// Schemas
|
|
9
9
|
// ---------------------------------------------------------------------------
|
|
10
10
|
|
|
11
|
+
const BOUNDARY_SCHEMA = {
|
|
12
|
+
type: 'object',
|
|
13
|
+
required: ['entryPoint', 'consumer'],
|
|
14
|
+
properties: {
|
|
15
|
+
entryPoint: { type: 'string', minLength: 1, pattern: '\\S' },
|
|
16
|
+
decisionPoint: { type: 'string' },
|
|
17
|
+
realizationPoint: { type: 'string' },
|
|
18
|
+
consumer: { type: 'string', minLength: 1, pattern: '\\S' },
|
|
19
|
+
stateOwner: { type: 'string' },
|
|
20
|
+
evidenceClass: { type: 'string' },
|
|
21
|
+
identityDimensions: { type: 'array', items: { type: 'string' } },
|
|
22
|
+
transitions: { type: 'array', items: { type: 'string' } },
|
|
23
|
+
},
|
|
24
|
+
}
|
|
25
|
+
|
|
11
26
|
const FINDINGS_ARRAY_SCHEMA = {
|
|
12
27
|
type: 'array',
|
|
13
28
|
items: {
|
|
14
29
|
type: 'object',
|
|
15
30
|
required: ['file', 'line', 'severity', 'title', 'description', 'badCode', 'fix'],
|
|
31
|
+
anyOf: [
|
|
32
|
+
{ properties: { severity: { enum: ['observation', 'idiomatic'] } } },
|
|
33
|
+
{ required: ['boundary'] },
|
|
34
|
+
],
|
|
16
35
|
properties: {
|
|
17
36
|
file: { type: 'string' },
|
|
18
37
|
line: { type: 'integer' },
|
|
@@ -23,6 +42,7 @@ const FINDINGS_ARRAY_SCHEMA = {
|
|
|
23
42
|
fix: { type: 'string' },
|
|
24
43
|
precedent: { type: 'string' },
|
|
25
44
|
rule: { type: 'string', minLength: 1, pattern: '\\S' },
|
|
45
|
+
boundary: BOUNDARY_SCHEMA,
|
|
26
46
|
},
|
|
27
47
|
},
|
|
28
48
|
}
|
|
@@ -267,6 +287,41 @@ function shouldVerify(finding) {
|
|
|
267
287
|
return finding.severity === 'critical' || finding.severity === 'important'
|
|
268
288
|
}
|
|
269
289
|
|
|
290
|
+
/**
|
|
291
|
+
* Does the finding carry enough of a composed boundary map to attempt a
|
|
292
|
+
* proof? Requires an entry point and a consumer, plus at least one of a
|
|
293
|
+
* realization point, state owner, or recorded transition.
|
|
294
|
+
*/
|
|
295
|
+
function hasBoundaryProofSurface(finding) {
|
|
296
|
+
const boundary = finding && finding.boundary
|
|
297
|
+
if (!boundary || typeof boundary !== 'object') {
|
|
298
|
+
return false
|
|
299
|
+
}
|
|
300
|
+
const hasText = function(value) {
|
|
301
|
+
return typeof value === 'string' && value.trim().length > 0
|
|
302
|
+
}
|
|
303
|
+
const hasList = function(value) {
|
|
304
|
+
return Array.isArray(value) && value.length > 0
|
|
305
|
+
}
|
|
306
|
+
return hasText(boundary.entryPoint)
|
|
307
|
+
&& hasText(boundary.consumer)
|
|
308
|
+
&& (
|
|
309
|
+
hasText(boundary.realizationPoint)
|
|
310
|
+
|| hasText(boundary.stateOwner)
|
|
311
|
+
|| hasList(boundary.transitions)
|
|
312
|
+
)
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
/**
|
|
316
|
+
* Does this finding qualify for the Prove stage? Every Critical finding
|
|
317
|
+
* qualifies; an Important finding only qualifies when it has a composed
|
|
318
|
+
* boundary map to prove against.
|
|
319
|
+
*/
|
|
320
|
+
function shouldProve(finding) {
|
|
321
|
+
return finding.severity === 'critical'
|
|
322
|
+
|| (finding.severity === 'important' && hasBoundaryProofSurface(finding))
|
|
323
|
+
}
|
|
324
|
+
|
|
270
325
|
function normalizeProof(result) {
|
|
271
326
|
const hasText = function(value) {
|
|
272
327
|
return typeof value === 'string' && value.trim().length > 0
|
|
@@ -520,7 +575,29 @@ const budgetAtStart = budget.spent()
|
|
|
520
575
|
// rule-specific checks and skip only its runtime-oriented challenges.
|
|
521
576
|
// -------------------------------------------------------------------------
|
|
522
577
|
|
|
578
|
+
// A model verdict cannot replace the required production boundary map.
|
|
579
|
+
function enforceBoundary(finding) {
|
|
580
|
+
if (!shouldVerify(finding)) return finding
|
|
581
|
+
const boundary = finding.boundary
|
|
582
|
+
const hasText = function(value) {
|
|
583
|
+
return typeof value === 'string' && value.trim().length > 0
|
|
584
|
+
}
|
|
585
|
+
if (boundary && typeof boundary === 'object' && !Array.isArray(boundary)
|
|
586
|
+
&& hasText(boundary.entryPoint) && hasText(boundary.consumer)) {
|
|
587
|
+
return finding
|
|
588
|
+
}
|
|
589
|
+
downgradedCount++
|
|
590
|
+
const unverified = Object.assign({}, finding, {
|
|
591
|
+
severity: 'observation',
|
|
592
|
+
verificationStatus: 'unverified',
|
|
593
|
+
verifierReasoning: 'Missing boundary object or identified production entry point/downstream consumer; downgraded to non-blocking',
|
|
594
|
+
})
|
|
595
|
+
delete unverified.proof
|
|
596
|
+
return unverified
|
|
597
|
+
}
|
|
598
|
+
|
|
523
599
|
async function verifyAll(findings) {
|
|
600
|
+
findings = findings.map(enforceBoundary)
|
|
524
601
|
const inScope = findings.filter(function(f) {
|
|
525
602
|
return shouldVerify(f)
|
|
526
603
|
})
|
|
@@ -704,7 +781,8 @@ const budgetAtStart = budget.spent()
|
|
|
704
781
|
}
|
|
705
782
|
|
|
706
783
|
// Verify the new finding through the same verifier path
|
|
707
|
-
const newFinding = Object.assign({}, reResult.findings[0])
|
|
784
|
+
const newFinding = enforceBoundary(Object.assign({}, reResult.findings[0]))
|
|
785
|
+
if (newFinding.verificationStatus === 'unverified') return [newFinding]
|
|
708
786
|
newFinding._dimension = `critic:${angle.axis}`
|
|
709
787
|
|
|
710
788
|
agentCount++
|
|
@@ -749,9 +827,13 @@ const budgetAtStart = budget.spent()
|
|
|
749
827
|
|
|
750
828
|
} // end ENGINE === 'workflow' branch
|
|
751
829
|
|
|
830
|
+
// Recheck after critic dedup and imported results, before proof promotion.
|
|
831
|
+
finalFindings = finalFindings.map(enforceBoundary)
|
|
832
|
+
|
|
752
833
|
// -------------------------------------------------------------------------
|
|
753
|
-
// Prove: for each Critical finding
|
|
754
|
-
//
|
|
834
|
+
// Prove: for each Critical finding and each Important finding with a
|
|
835
|
+
// composed boundary map (excluding hard rules, which are policy violations
|
|
836
|
+
// with no runtime failure to demonstrate), attempt to produce an
|
|
755
837
|
// executable failing test in the worktree. Only runs with a worktree and
|
|
756
838
|
// outside scan depth, and is capped so a pathological finding count can't
|
|
757
839
|
// blow the budget.
|
|
@@ -770,7 +852,7 @@ const budgetAtStart = budget.spent()
|
|
|
770
852
|
|
|
771
853
|
if (worktreePath && depth !== 'scan') {
|
|
772
854
|
const proveCandidates = finalFindings.filter(function(f) {
|
|
773
|
-
return f
|
|
855
|
+
return shouldProve(f) && !isHardRule(f)
|
|
774
856
|
})
|
|
775
857
|
|
|
776
858
|
if (proveCandidates.length > 0) {
|
|
@@ -779,7 +861,7 @@ const budgetAtStart = budget.spent()
|
|
|
779
861
|
const capped = proveCandidates.slice(0, 5)
|
|
780
862
|
const skipped = proveCandidates.length - capped.length
|
|
781
863
|
if (skipped > 0) {
|
|
782
|
-
log(`Prove: capping at 5 provers, skipping ${skipped} additional
|
|
864
|
+
log(`Prove: capping at 5 provers, skipping ${skipped} additional finding(s)`)
|
|
783
865
|
}
|
|
784
866
|
|
|
785
867
|
await batched(capped.map(function(finding) {
|
|
@@ -45,7 +45,6 @@ function walk(dir, out = []) {
|
|
|
45
45
|
return out;
|
|
46
46
|
}
|
|
47
47
|
|
|
48
|
-
// A user-role text record that is really harness output, not something a person typed.
|
|
49
48
|
// A user-role text record that is really harness output, not something a person typed.
|
|
50
49
|
// Each alternative below was found by a run that mis-counted. The first pass caught the
|
|
51
50
|
// obvious wrappers and still let 26% through: skill body dumps, image attachments,
|
|
@@ -62,6 +61,15 @@ const HARNESS = new RegExp('^(' + [
|
|
|
62
61
|
'\\[Image #\\d+\\]', // bare attachment
|
|
63
62
|
'\\[Image: source:',
|
|
64
63
|
'\\[Pasted text',
|
|
64
|
+
// v3: the 2026-09-29 run found v2 still let 66% through, almost all of it these five.
|
|
65
|
+
'Another Claude session sent a message', // subagent / cross-session hand-back
|
|
66
|
+
'<agent-message',
|
|
67
|
+
'This session is being continued from a previous conversation', // compaction resume
|
|
68
|
+
'Stop hook feedback:',
|
|
69
|
+
'\\[SYSTEM NOTIFICATION',
|
|
70
|
+
'Your response above was cut off',
|
|
71
|
+
'\\(Re-invocation of /',
|
|
72
|
+
'\\[\\d+ prior /loop wakeup',
|
|
65
73
|
].join('|') + ')');
|
|
66
74
|
|
|
67
75
|
const textOf = (msg) => {
|
|
@@ -261,7 +269,7 @@ if (problems.length) {
|
|
|
261
269
|
console.error('INCONSISTENT REPORT, refusing to write:\n ' + problems.join('\n '));
|
|
262
270
|
process.exit(3);
|
|
263
271
|
}
|
|
264
|
-
report.echoFilterVersion =
|
|
272
|
+
report.echoFilterVersion = 3;
|
|
265
273
|
|
|
266
274
|
fs.writeFileSync(OUT, JSON.stringify(report, null, 1));
|
|
267
275
|
fs.writeFileSync(OUT.replace(/\.json$/, '') + '-turns.json', JSON.stringify(S.humanTurns, null, 1));
|