@olegkoval/agent-skills 1.14.1 → 1.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +2 -1
- package/.cursor-plugin/index.json +5 -0
- package/catalog/skills.json +21 -0
- package/collections/software-development.json +1 -0
- package/package.json +1 -1
- package/packages/software-development/self-critique/SKILL.md +148 -0
- package/packages/software-development/self-critique/adapters/claude/plugin.json +5 -0
- package/packages/software-development/self-critique/adapters/claude/skills/self-critique/SKILL.md +149 -0
- package/packages/software-development/self-critique/adapters/claude/skills/self-critique/scripts/critique-nudge.mjs +57 -0
- package/packages/software-development/self-critique/adapters/codex/README.md +21 -0
- package/packages/software-development/self-critique/adapters/cursor/plugin.json +6 -0
- package/packages/software-development/self-critique/adapters/cursor/skills/self-critique/SKILL.md +150 -0
- package/packages/software-development/self-critique/adapters/cursor/skills/self-critique/scripts/critique-nudge.mjs +57 -0
- package/packages/software-development/self-critique/scripts/critique-nudge.mjs +57 -0
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "olko-agent-skills",
|
|
3
3
|
"description": "Agent-agnostic skill catalog for Codex, Claude, Cursor, and other skill-aware tools.",
|
|
4
|
-
"version": "1.14.
|
|
4
|
+
"version": "1.14.1",
|
|
5
5
|
"author": {
|
|
6
6
|
"name": "Oleg Koval"
|
|
7
7
|
},
|
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
"repository": "https://github.com/oleg-koval/agent-skills",
|
|
10
10
|
"license": "MIT",
|
|
11
11
|
"skills": [
|
|
12
|
+
"./packages/software-development/self-critique",
|
|
12
13
|
"./packages/software-development/docs-index-keeper",
|
|
13
14
|
"./packages/software-development/semantic-release-beta",
|
|
14
15
|
"./packages/software-development/gh-cli",
|
|
@@ -1,6 +1,11 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "agent-skills",
|
|
3
3
|
"plugins": [
|
|
4
|
+
{
|
|
5
|
+
"name": "olko:self-critique",
|
|
6
|
+
"source": "./packages/software-development/self-critique/adapters/cursor",
|
|
7
|
+
"description": "Adversarially critique your own last answer: spawn a critic agent that verifies claims against live sources, finds the related issue or pattern you missed, then loop until satisfied and report where you were wrong, iterations-to-satisfy, the numeric improvement, and a final score."
|
|
8
|
+
},
|
|
4
9
|
{
|
|
5
10
|
"name": "olko:docs-index-keeper",
|
|
6
11
|
"source": "./packages/software-development/docs-index-keeper/adapters/cursor",
|
package/catalog/skills.json
CHANGED
|
@@ -1,6 +1,27 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "agent-skills",
|
|
3
3
|
"packages": [
|
|
4
|
+
{
|
|
5
|
+
"name": "self-critique",
|
|
6
|
+
"lookupName": "olko:self-critique",
|
|
7
|
+
"category": "software-development",
|
|
8
|
+
"path": "packages/software-development/self-critique",
|
|
9
|
+
"description": "Adversarially critique your own last answer: spawn a critic agent that verifies claims against live sources, finds the related issue or pattern you missed, then loop until satisfied and report where you were wrong, iterations-to-satisfy, the numeric improvement, and a final score.",
|
|
10
|
+
"tags": [
|
|
11
|
+
"critical-thinking",
|
|
12
|
+
"self-review",
|
|
13
|
+
"verification",
|
|
14
|
+
"adversarial",
|
|
15
|
+
"agents",
|
|
16
|
+
"quality"
|
|
17
|
+
],
|
|
18
|
+
"adapters": [
|
|
19
|
+
"codex",
|
|
20
|
+
"claude",
|
|
21
|
+
"cursor",
|
|
22
|
+
"copilot"
|
|
23
|
+
]
|
|
24
|
+
},
|
|
4
25
|
{
|
|
5
26
|
"name": "docs-index-keeper",
|
|
6
27
|
"lookupName": "olko:docs-index-keeper",
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
"name": "software-development",
|
|
3
3
|
"description": "Skills for software development, release automation, git workflows, documentation, prompt tooling, and product building.",
|
|
4
4
|
"packages": [
|
|
5
|
+
"self-critique",
|
|
5
6
|
"docs-index-keeper",
|
|
6
7
|
"semantic-release-beta",
|
|
7
8
|
"gh-cli",
|
package/package.json
CHANGED
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: self-critique
|
|
3
|
+
description: Adversarially critique your own last substantial answer before the user has to. Spawn a critic agent that verifies every claim against live sources, checks similar and related issues, finds patterns, and addresses comments directly to you. Loop (revise, re-score) until satisfied, then report where you were wrong, iterations-to-satisfy, the numeric improvement, and a final score. Use when the user asks you to criticize, challenge, stress-test, or red-team your own answer, or accepts a "critical-thinking review" offer.
|
|
4
|
+
license: MIT
|
|
5
|
+
compatibility: Codex, Claude Code, Cursor, and other Agent Skills compatible tools.
|
|
6
|
+
metadata:
|
|
7
|
+
author: Oleg Koval
|
|
8
|
+
tags:
|
|
9
|
+
- critical-thinking
|
|
10
|
+
- self-review
|
|
11
|
+
- verification
|
|
12
|
+
- adversarial
|
|
13
|
+
- agents
|
|
14
|
+
- quality
|
|
15
|
+
---
|
|
16
|
+
|
|
17
|
+
# Self-critique
|
|
18
|
+
|
|
19
|
+
Run an adversarial review of **your own previous answer** and report the result in numbers.
|
|
20
|
+
|
|
21
|
+
## Overview
|
|
22
|
+
|
|
23
|
+
A confident answer is not a verified one. This skill turns your last substantial reply into the
|
|
24
|
+
subject of a hostile review: a separate critic agent tries to refute it against live sources, hunts
|
|
25
|
+
for the related issue or pattern you missed, and scores it. You then revise and re-score until the
|
|
26
|
+
score stops moving, and report the outcome honestly, including where you were wrong.
|
|
27
|
+
|
|
28
|
+
The point is to surface error before the user does, and to make "is this actually right?" a measured
|
|
29
|
+
quantity instead of a vibe.
|
|
30
|
+
|
|
31
|
+
## When to Use
|
|
32
|
+
|
|
33
|
+
- The user asks you to criticize, challenge, stress-test, red-team, or "poke holes in" your own answer.
|
|
34
|
+
- The user accepts a "critical-thinking review" offer.
|
|
35
|
+
- You have just produced a substantial analysis, recommendation, or investigation and want a
|
|
36
|
+
confidence number before the user acts on it.
|
|
37
|
+
|
|
38
|
+
Do not use for trivial one-shot replies or simple chat. The overhead only pays off on load-bearing
|
|
39
|
+
answers.
|
|
40
|
+
|
|
41
|
+
## Procedure
|
|
42
|
+
|
|
43
|
+
1. **Snapshot the target.** Capture your last substantial answer verbatim, plus the concrete claims
|
|
44
|
+
it makes and the source each claim rests on. This is `v1`.
|
|
45
|
+
|
|
46
|
+
2. **Spawn the critic.** Launch one independent agent with live-source access (issue tracker, code
|
|
47
|
+
host, error monitor, the repo). Use the prompt template below. The critic must return: `score`
|
|
48
|
+
(0-100), `verdict` (AGREE / AGREE-WITH-CAVEATS / DISAGREE), `where_wrong` (list), `missed`
|
|
49
|
+
(related issues or patterns you failed to surface), `unverifiable` (claims it could not confirm),
|
|
50
|
+
and `fixes` (ranked, actionable).
|
|
51
|
+
|
|
52
|
+
3. **Decide and loop.** Stop if `score >= 90` OR no actionable `fixes` remain. Otherwise revise the
|
|
53
|
+
answer using the fixes to produce `v2`, then re-spawn the critic to score `v2`. Repeat.
|
|
54
|
+
**Hard cap: 3 critic rounds.** (A known ceiling: raise it only if a round is still finding real,
|
|
55
|
+
fixable defects.)
|
|
56
|
+
|
|
57
|
+
4. **Report, always, even after a single round:**
|
|
58
|
+
- **Where you were wrong** - consolidated, each item with the receipt the critic cited.
|
|
59
|
+
- **Iterations to satisfy** - how many critic rounds it took to reach the stop condition (1 means
|
|
60
|
+
it was solid on the first pass).
|
|
61
|
+
- **Improvement in numbers** - the score trajectory, for example `72 -> 88 -> 94 (+22 pts, +31%)`.
|
|
62
|
+
If only one round ran, state `no revision needed`.
|
|
63
|
+
- **Final score** - the critic's last score and verdict.
|
|
64
|
+
- **Unverifiable** - anything you should not present as fact.
|
|
65
|
+
|
|
66
|
+
## Critic prompt template
|
|
67
|
+
|
|
68
|
+
Pass this to the critic agent, filling in the two placeholders. It is deliberately adversarial:
|
|
69
|
+
the default stance is that the answer is wrong until a live source proves otherwise.
|
|
70
|
+
|
|
71
|
+
> You are an adversarial reviewer. Your job is to verify and, where possible, **refute** the answer
|
|
72
|
+
> below. Default to skepticism: assume a claim is wrong until a live source proves it. Address every
|
|
73
|
+
> comment **directly to the author** ("You claimed X; the source says Y").
|
|
74
|
+
>
|
|
75
|
+
> **The answer under review (verbatim):**
|
|
76
|
+
> {{ANSWER}}
|
|
77
|
+
>
|
|
78
|
+
> **Its claims and the source each rests on:**
|
|
79
|
+
> {{CLAIMS_AND_SOURCES}}
|
|
80
|
+
>
|
|
81
|
+
> **Do this:**
|
|
82
|
+
> 1. Re-verify every claim against the **live** source (issue tracker / code host / error monitor /
|
|
83
|
+
> the repo) - never trust the answer's own summary. Quote the field, line, PR body, or issue you
|
|
84
|
+
> checked.
|
|
85
|
+
> 2. **Check similar and related issues, and find patterns.** Is this part of a cluster? Is there a
|
|
86
|
+
> closer sibling, parent epic, or duplicate the answer should have surfaced but did not? Name it
|
|
87
|
+
> with its ID and the shared signal (same error, same code path, same root cause).
|
|
88
|
+
> 3. Stress-test the **strongest** load-bearing claim and the answer's biggest blind spot
|
|
89
|
+
> specifically. Where is it over-confident? Where is symptom-level reasoning passing itself off as
|
|
90
|
+
> a proven root cause?
|
|
91
|
+
> 4. List exactly **where the author was wrong or imprecise**, and what they **missed**.
|
|
92
|
+
> 5. Flag any claim you **could not verify** so it is not presented as fact.
|
|
93
|
+
>
|
|
94
|
+
> **Output, terse, with a receipt for every line:**
|
|
95
|
+
> - `verdict`: AGREE | AGREE-WITH-CAVEATS | DISAGREE
|
|
96
|
+
> - `score`: 0-100 (rubric: 100 = every claim verified, nothing missed, confidence correctly
|
|
97
|
+
> calibrated; subtract for each unverified claim, each missed sibling or pattern, and each
|
|
98
|
+
> over- or under-statement)
|
|
99
|
+
> - `where_wrong`: bullet list, each with its source receipt
|
|
100
|
+
> - `missed`: related issues or patterns the author failed to surface, with IDs
|
|
101
|
+
> - `unverifiable`: claims that need a source you do not have
|
|
102
|
+
> - `fixes`: ranked, actionable - what to change to raise the score
|
|
103
|
+
|
|
104
|
+
## Optional: auto-offer via a Claude Code Stop hook
|
|
105
|
+
|
|
106
|
+
A hook is a shell command, so it cannot run this loop itself. What it can do is **nudge**: after a
|
|
107
|
+
substantial answer, it offers the review and you run the skill only if the user says yes.
|
|
108
|
+
|
|
109
|
+
The hook ships with this skill at `scripts/critique-nudge.mjs`. Register it as a `Stop` hook in your
|
|
110
|
+
Claude Code settings (`.claude/settings.json` for a team, `.claude/settings.local.json` for
|
|
111
|
+
yourself):
|
|
112
|
+
|
|
113
|
+
```json
|
|
114
|
+
{
|
|
115
|
+
"hooks": {
|
|
116
|
+
"Stop": [
|
|
117
|
+
{
|
|
118
|
+
"hooks": [
|
|
119
|
+
{
|
|
120
|
+
"type": "command",
|
|
121
|
+
"command": "node \"$CLAUDE_PROJECT_DIR/.claude/hooks/critique-nudge.mjs\"",
|
|
122
|
+
"timeout": 15
|
|
123
|
+
}
|
|
124
|
+
]
|
|
125
|
+
}
|
|
126
|
+
]
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
Copy `scripts/critique-nudge.mjs` to `.claude/hooks/critique-nudge.mjs` (or point the command at
|
|
132
|
+
wherever you keep it). After an answer whose final prose is at least 1500 characters, the hook blocks
|
|
133
|
+
the turn once and instructs the agent to append a one-line "critical-thinking review" offer, then
|
|
134
|
+
stop. Two loop guards keep it from firing forever: it does nothing when `stop_hook_active` is set, and
|
|
135
|
+
nothing when the answer already contains the offer phrase. On any malformed input it exits silently
|
|
136
|
+
and never blocks the user.
|
|
137
|
+
|
|
138
|
+
The 1500-character threshold is a heuristic with a known ceiling: it misses short but dense
|
|
139
|
+
tool-heavy turns. If that matters, extend the script to also count tool calls since the last user
|
|
140
|
+
message.
|
|
141
|
+
|
|
142
|
+
## Notes
|
|
143
|
+
|
|
144
|
+
- One critic round is fine and common. If the answer was already solid, report
|
|
145
|
+
`iterations: 1, no revision needed` and the single score rather than forcing a revision.
|
|
146
|
+
- Diversity beats redundancy: if a claim can fail in more than one way, give the critic distinct
|
|
147
|
+
lenses (correctness, completeness, does-it-reproduce) instead of repeating the same check.
|
|
148
|
+
- The critic verifies against live sources; it does not rewrite the answer. You revise; it re-scores.
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "olko-self-critique",
|
|
3
|
+
"description": "Adversarially critique your own last answer: spawn a critic agent that verifies claims against live sources, finds the related issue or pattern you missed, then loop until satisfied and report where you were wrong, iterations-to-satisfy, the numeric improvement, and a final score.",
|
|
4
|
+
"skills": "./skills"
|
|
5
|
+
}
|
package/packages/software-development/self-critique/adapters/claude/skills/self-critique/SKILL.md
ADDED
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: self-critique
|
|
3
|
+
description: Adversarially critique your own last substantial answer before the user has to. Spawn a critic agent that verifies every claim against live sources, checks similar and related issues, finds patterns, and addresses comments directly to you. Loop (revise, re-score) until satisfied, then report where you were wrong, iterations-to-satisfy, the numeric improvement, and a final score. Use when the user asks you to criticize, challenge, stress-test, or red-team your own answer, or accepts a "critical-thinking review" offer.
|
|
4
|
+
license: MIT
|
|
5
|
+
compatibility: Codex, Claude Code, Cursor, and other Agent Skills compatible tools.
|
|
6
|
+
metadata:
|
|
7
|
+
author: Oleg Koval
|
|
8
|
+
tags:
|
|
9
|
+
- critical-thinking
|
|
10
|
+
- self-review
|
|
11
|
+
- verification
|
|
12
|
+
- adversarial
|
|
13
|
+
- agents
|
|
14
|
+
- quality
|
|
15
|
+
---
|
|
16
|
+
<!-- Generated by scripts/build-adapters.sh. Do not edit directly. -->
|
|
17
|
+
|
|
18
|
+
# Self-critique
|
|
19
|
+
|
|
20
|
+
Run an adversarial review of **your own previous answer** and report the result in numbers.
|
|
21
|
+
|
|
22
|
+
## Overview
|
|
23
|
+
|
|
24
|
+
A confident answer is not a verified one. This skill turns your last substantial reply into the
|
|
25
|
+
subject of a hostile review: a separate critic agent tries to refute it against live sources, hunts
|
|
26
|
+
for the related issue or pattern you missed, and scores it. You then revise and re-score until the
|
|
27
|
+
score stops moving, and report the outcome honestly, including where you were wrong.
|
|
28
|
+
|
|
29
|
+
The point is to surface error before the user does, and to make "is this actually right?" a measured
|
|
30
|
+
quantity instead of a vibe.
|
|
31
|
+
|
|
32
|
+
## When to Use
|
|
33
|
+
|
|
34
|
+
- The user asks you to criticize, challenge, stress-test, red-team, or "poke holes in" your own answer.
|
|
35
|
+
- The user accepts a "critical-thinking review" offer.
|
|
36
|
+
- You have just produced a substantial analysis, recommendation, or investigation and want a
|
|
37
|
+
confidence number before the user acts on it.
|
|
38
|
+
|
|
39
|
+
Do not use for trivial one-shot replies or simple chat. The overhead only pays off on load-bearing
|
|
40
|
+
answers.
|
|
41
|
+
|
|
42
|
+
## Procedure
|
|
43
|
+
|
|
44
|
+
1. **Snapshot the target.** Capture your last substantial answer verbatim, plus the concrete claims
|
|
45
|
+
it makes and the source each claim rests on. This is `v1`.
|
|
46
|
+
|
|
47
|
+
2. **Spawn the critic.** Launch one independent agent with live-source access (issue tracker, code
|
|
48
|
+
host, error monitor, the repo). Use the prompt template below. The critic must return: `score`
|
|
49
|
+
(0-100), `verdict` (AGREE / AGREE-WITH-CAVEATS / DISAGREE), `where_wrong` (list), `missed`
|
|
50
|
+
(related issues or patterns you failed to surface), `unverifiable` (claims it could not confirm),
|
|
51
|
+
and `fixes` (ranked, actionable).
|
|
52
|
+
|
|
53
|
+
3. **Decide and loop.** Stop if `score >= 90` OR no actionable `fixes` remain. Otherwise revise the
|
|
54
|
+
answer using the fixes to produce `v2`, then re-spawn the critic to score `v2`. Repeat.
|
|
55
|
+
**Hard cap: 3 critic rounds.** (A known ceiling: raise it only if a round is still finding real,
|
|
56
|
+
fixable defects.)
|
|
57
|
+
|
|
58
|
+
4. **Report, always, even after a single round:**
|
|
59
|
+
- **Where you were wrong** - consolidated, each item with the receipt the critic cited.
|
|
60
|
+
- **Iterations to satisfy** - how many critic rounds it took to reach the stop condition (1 means
|
|
61
|
+
it was solid on the first pass).
|
|
62
|
+
- **Improvement in numbers** - the score trajectory, for example `72 -> 88 -> 94 (+22 pts, +31%)`.
|
|
63
|
+
If only one round ran, state `no revision needed`.
|
|
64
|
+
- **Final score** - the critic's last score and verdict.
|
|
65
|
+
- **Unverifiable** - anything you should not present as fact.
|
|
66
|
+
|
|
67
|
+
## Critic prompt template
|
|
68
|
+
|
|
69
|
+
Pass this to the critic agent, filling in the two placeholders. It is deliberately adversarial:
|
|
70
|
+
the default stance is that the answer is wrong until a live source proves otherwise.
|
|
71
|
+
|
|
72
|
+
> You are an adversarial reviewer. Your job is to verify and, where possible, **refute** the answer
|
|
73
|
+
> below. Default to skepticism: assume a claim is wrong until a live source proves it. Address every
|
|
74
|
+
> comment **directly to the author** ("You claimed X; the source says Y").
|
|
75
|
+
>
|
|
76
|
+
> **The answer under review (verbatim):**
|
|
77
|
+
> {{ANSWER}}
|
|
78
|
+
>
|
|
79
|
+
> **Its claims and the source each rests on:**
|
|
80
|
+
> {{CLAIMS_AND_SOURCES}}
|
|
81
|
+
>
|
|
82
|
+
> **Do this:**
|
|
83
|
+
> 1. Re-verify every claim against the **live** source (issue tracker / code host / error monitor /
|
|
84
|
+
> the repo) - never trust the answer's own summary. Quote the field, line, PR body, or issue you
|
|
85
|
+
> checked.
|
|
86
|
+
> 2. **Check similar and related issues, and find patterns.** Is this part of a cluster? Is there a
|
|
87
|
+
> closer sibling, parent epic, or duplicate the answer should have surfaced but did not? Name it
|
|
88
|
+
> with its ID and the shared signal (same error, same code path, same root cause).
|
|
89
|
+
> 3. Stress-test the **strongest** load-bearing claim and the answer's biggest blind spot
|
|
90
|
+
> specifically. Where is it over-confident? Where is symptom-level reasoning passing itself off as
|
|
91
|
+
> a proven root cause?
|
|
92
|
+
> 4. List exactly **where the author was wrong or imprecise**, and what they **missed**.
|
|
93
|
+
> 5. Flag any claim you **could not verify** so it is not presented as fact.
|
|
94
|
+
>
|
|
95
|
+
> **Output, terse, with a receipt for every line:**
|
|
96
|
+
> - `verdict`: AGREE | AGREE-WITH-CAVEATS | DISAGREE
|
|
97
|
+
> - `score`: 0-100 (rubric: 100 = every claim verified, nothing missed, confidence correctly
|
|
98
|
+
> calibrated; subtract for each unverified claim, each missed sibling or pattern, and each
|
|
99
|
+
> over- or under-statement)
|
|
100
|
+
> - `where_wrong`: bullet list, each with its source receipt
|
|
101
|
+
> - `missed`: related issues or patterns the author failed to surface, with IDs
|
|
102
|
+
> - `unverifiable`: claims that need a source you do not have
|
|
103
|
+
> - `fixes`: ranked, actionable - what to change to raise the score
|
|
104
|
+
|
|
105
|
+
## Optional: auto-offer via a Claude Code Stop hook
|
|
106
|
+
|
|
107
|
+
A hook is a shell command, so it cannot run this loop itself. What it can do is **nudge**: after a
|
|
108
|
+
substantial answer, it offers the review and you run the skill only if the user says yes.
|
|
109
|
+
|
|
110
|
+
The hook ships with this skill at `scripts/critique-nudge.mjs`. Register it as a `Stop` hook in your
|
|
111
|
+
Claude Code settings (`.claude/settings.json` for a team, `.claude/settings.local.json` for
|
|
112
|
+
yourself):
|
|
113
|
+
|
|
114
|
+
```json
|
|
115
|
+
{
|
|
116
|
+
"hooks": {
|
|
117
|
+
"Stop": [
|
|
118
|
+
{
|
|
119
|
+
"hooks": [
|
|
120
|
+
{
|
|
121
|
+
"type": "command",
|
|
122
|
+
"command": "node \"$CLAUDE_PROJECT_DIR/.claude/hooks/critique-nudge.mjs\"",
|
|
123
|
+
"timeout": 15
|
|
124
|
+
}
|
|
125
|
+
]
|
|
126
|
+
}
|
|
127
|
+
]
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
Copy `scripts/critique-nudge.mjs` to `.claude/hooks/critique-nudge.mjs` (or point the command at
|
|
133
|
+
wherever you keep it). After an answer whose final prose is at least 1500 characters, the hook blocks
|
|
134
|
+
the turn once and instructs the agent to append a one-line "critical-thinking review" offer, then
|
|
135
|
+
stop. Two loop guards keep it from firing forever: it does nothing when `stop_hook_active` is set, and
|
|
136
|
+
nothing when the answer already contains the offer phrase. On any malformed input it exits silently
|
|
137
|
+
and never blocks the user.
|
|
138
|
+
|
|
139
|
+
The 1500-character threshold is a heuristic with a known ceiling: it misses short but dense
|
|
140
|
+
tool-heavy turns. If that matters, extend the script to also count tool calls since the last user
|
|
141
|
+
message.
|
|
142
|
+
|
|
143
|
+
## Notes
|
|
144
|
+
|
|
145
|
+
- One critic round is fine and common. If the answer was already solid, report
|
|
146
|
+
`iterations: 1, no revision needed` and the single score rather than forcing a revision.
|
|
147
|
+
- Diversity beats redundancy: if a claim can fail in more than one way, give the critic distinct
|
|
148
|
+
lenses (correctness, completeness, does-it-reproduce) instead of repeating the same check.
|
|
149
|
+
- The critic verifies against live sources; it does not rewrite the answer. You revise; it re-scores.
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Stop hook: after a substantial answer, make the model offer the user a
|
|
3
|
+
// critical-thinking review (adversarial critic agent, per the /self-critique skill).
|
|
4
|
+
// The hook only NUDGES. It cannot spawn agents or run the loop itself.
|
|
5
|
+
//
|
|
6
|
+
// ponytail: "substantial" = final assistant prose >= THRESHOLD chars. Known ceiling:
|
|
7
|
+
// misses short-but-dense tool-heavy turns. Upgrade path: also count tool_use blocks
|
|
8
|
+
// since the last user message if length alone misfires.
|
|
9
|
+
import { readFileSync } from 'node:fs';
|
|
10
|
+
|
|
11
|
+
const THRESHOLD = 1500;
|
|
12
|
+
const OFFER_MARKER = /critical-thinking review/i; // the offer line must contain this; doubles as a loop guard
|
|
13
|
+
|
|
14
|
+
function bail() { process.exit(0); } // any uncertainty => do nothing, never block the user
|
|
15
|
+
|
|
16
|
+
let raw = '';
|
|
17
|
+
try { raw = readFileSync(0, 'utf8'); } catch { bail(); }
|
|
18
|
+
|
|
19
|
+
let input;
|
|
20
|
+
try { input = JSON.parse(raw); } catch { bail(); }
|
|
21
|
+
|
|
22
|
+
// Loop guard #1: if this hook already forced a continuation, let the turn end.
|
|
23
|
+
if (input.stop_hook_active) bail();
|
|
24
|
+
|
|
25
|
+
const path = input.transcript_path;
|
|
26
|
+
if (!path) bail();
|
|
27
|
+
|
|
28
|
+
let lines;
|
|
29
|
+
try { lines = readFileSync(path, 'utf8').trim().split('\n'); } catch { bail(); }
|
|
30
|
+
|
|
31
|
+
// Walk from the end; grab the final assistant message that has text content.
|
|
32
|
+
let text = '';
|
|
33
|
+
for (let i = lines.length - 1; i >= 0; i--) {
|
|
34
|
+
let obj;
|
|
35
|
+
try { obj = JSON.parse(lines[i]); } catch { continue; }
|
|
36
|
+
const msg = obj.message;
|
|
37
|
+
if (obj.type === 'assistant' && msg && Array.isArray(msg.content)) {
|
|
38
|
+
text = msg.content.filter((c) => c.type === 'text').map((c) => c.text).join('\n');
|
|
39
|
+
if (text.trim()) break;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
if (text.length < THRESHOLD) bail();
|
|
44
|
+
|
|
45
|
+
// Loop guard #2: the answer already carries the offer (e.g. I added it last turn).
|
|
46
|
+
if (OFFER_MARKER.test(text)) bail();
|
|
47
|
+
|
|
48
|
+
const reason =
|
|
49
|
+
'That was a substantial answer. Before ending the turn, append exactly ONE short line ' +
|
|
50
|
+
'offering the user a "critical-thinking review": ask whether they want you to run /self-critique ' +
|
|
51
|
+
'(spawn an adversarial critic agent that challenges this answer, checks similar issues, finds ' +
|
|
52
|
+
'patterns, and reports where you were wrong, iterations-to-satisfy, the numeric improvement, and a ' +
|
|
53
|
+
'final score). The offer line MUST contain the phrase "critical-thinking review". Do NOT run the ' +
|
|
54
|
+
'critique now and do not re-explain the answer. Just offer it, then stop.';
|
|
55
|
+
|
|
56
|
+
process.stdout.write(JSON.stringify({ decision: 'block', reason }));
|
|
57
|
+
bail();
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# Codex Adapter for self-critique
|
|
2
|
+
|
|
3
|
+
This is a Codex-specific adapter for the `olko:self-critique` skill.
|
|
4
|
+
The canonical skill definition is in `../../../SKILL.md`.
|
|
5
|
+
|
|
6
|
+
## Usage
|
|
7
|
+
|
|
8
|
+
Invoke in a Codex session:
|
|
9
|
+
|
|
10
|
+
```
|
|
11
|
+
Use the olko:self-critique skill to adversarially review your last answer and report a confidence score.
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
## Workflow
|
|
15
|
+
|
|
16
|
+
See `../../../SKILL.md` for the full workflow: spawn a critic agent, verify every claim against live
|
|
17
|
+
sources, find the related issue or pattern you missed, then loop (revise, re-score) until satisfied
|
|
18
|
+
and report where you were wrong, iterations-to-satisfy, the numeric improvement, and a final score.
|
|
19
|
+
|
|
20
|
+
The optional Claude Code `Stop` hook (`scripts/critique-nudge.mjs`) is Claude-specific and does not
|
|
21
|
+
apply to Codex; the skill procedure itself is agent-agnostic.
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "olko:self-critique",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "Adversarially critique your own last answer: spawn a critic agent that verifies claims against live sources, finds the related issue or pattern you missed, then loop until satisfied and report where you were wrong, iterations-to-satisfy, the numeric improvement, and a final score.",
|
|
5
|
+
"skills": "skills/"
|
|
6
|
+
}
|
package/packages/software-development/self-critique/adapters/cursor/skills/self-critique/SKILL.md
ADDED
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: self-critique
|
|
3
|
+
description: Adversarially critique your own last substantial answer before the user has to. Spawn a critic agent that verifies every claim against live sources, checks similar and related issues, finds patterns, and addresses comments directly to you. Loop (revise, re-score) until satisfied, then report where you were wrong, iterations-to-satisfy, the numeric improvement, and a final score. Use when the user asks you to criticize, challenge, stress-test, or red-team your own answer, or accepts a "critical-thinking review" offer.
|
|
4
|
+
license: MIT
|
|
5
|
+
compatibility: Codex, Claude Code, Cursor, and other Agent Skills compatible tools.
|
|
6
|
+
metadata:
|
|
7
|
+
targets: ["cursor"]
|
|
8
|
+
author: Oleg Koval
|
|
9
|
+
tags:
|
|
10
|
+
- critical-thinking
|
|
11
|
+
- self-review
|
|
12
|
+
- verification
|
|
13
|
+
- adversarial
|
|
14
|
+
- agents
|
|
15
|
+
- quality
|
|
16
|
+
---
|
|
17
|
+
<!-- Generated by scripts/build-adapters.sh. Do not edit directly. -->
|
|
18
|
+
|
|
19
|
+
# Self-critique
|
|
20
|
+
|
|
21
|
+
Run an adversarial review of **your own previous answer** and report the result in numbers.
|
|
22
|
+
|
|
23
|
+
## Overview
|
|
24
|
+
|
|
25
|
+
A confident answer is not a verified one. This skill turns your last substantial reply into the
|
|
26
|
+
subject of a hostile review: a separate critic agent tries to refute it against live sources, hunts
|
|
27
|
+
for the related issue or pattern you missed, and scores it. You then revise and re-score until the
|
|
28
|
+
score stops moving, and report the outcome honestly, including where you were wrong.
|
|
29
|
+
|
|
30
|
+
The point is to surface error before the user does, and to make "is this actually right?" a measured
|
|
31
|
+
quantity instead of a vibe.
|
|
32
|
+
|
|
33
|
+
## When to Use
|
|
34
|
+
|
|
35
|
+
- The user asks you to criticize, challenge, stress-test, red-team, or "poke holes in" your own answer.
|
|
36
|
+
- The user accepts a "critical-thinking review" offer.
|
|
37
|
+
- You have just produced a substantial analysis, recommendation, or investigation and want a
|
|
38
|
+
confidence number before the user acts on it.
|
|
39
|
+
|
|
40
|
+
Do not use for trivial one-shot replies or simple chat. The overhead only pays off on load-bearing
|
|
41
|
+
answers.
|
|
42
|
+
|
|
43
|
+
## Procedure
|
|
44
|
+
|
|
45
|
+
1. **Snapshot the target.** Capture your last substantial answer verbatim, plus the concrete claims
|
|
46
|
+
it makes and the source each claim rests on. This is `v1`.
|
|
47
|
+
|
|
48
|
+
2. **Spawn the critic.** Launch one independent agent with live-source access (issue tracker, code
|
|
49
|
+
host, error monitor, the repo). Use the prompt template below. The critic must return: `score`
|
|
50
|
+
(0-100), `verdict` (AGREE / AGREE-WITH-CAVEATS / DISAGREE), `where_wrong` (list), `missed`
|
|
51
|
+
(related issues or patterns you failed to surface), `unverifiable` (claims it could not confirm),
|
|
52
|
+
and `fixes` (ranked, actionable).
|
|
53
|
+
|
|
54
|
+
3. **Decide and loop.** Stop if `score >= 90` OR no actionable `fixes` remain. Otherwise revise the
|
|
55
|
+
answer using the fixes to produce `v2`, then re-spawn the critic to score `v2`. Repeat.
|
|
56
|
+
**Hard cap: 3 critic rounds.** (A known ceiling: raise it only if a round is still finding real,
|
|
57
|
+
fixable defects.)
|
|
58
|
+
|
|
59
|
+
4. **Report, always, even after a single round:**
|
|
60
|
+
- **Where you were wrong** - consolidated, each item with the receipt the critic cited.
|
|
61
|
+
- **Iterations to satisfy** - how many critic rounds it took to reach the stop condition (1 means
|
|
62
|
+
it was solid on the first pass).
|
|
63
|
+
- **Improvement in numbers** - the score trajectory, for example `72 -> 88 -> 94 (+22 pts, +31%)`.
|
|
64
|
+
If only one round ran, state `no revision needed`.
|
|
65
|
+
- **Final score** - the critic's last score and verdict.
|
|
66
|
+
- **Unverifiable** - anything you should not present as fact.
|
|
67
|
+
|
|
68
|
+
## Critic prompt template
|
|
69
|
+
|
|
70
|
+
Pass this to the critic agent, filling in the two placeholders. It is deliberately adversarial:
|
|
71
|
+
the default stance is that the answer is wrong until a live source proves otherwise.
|
|
72
|
+
|
|
73
|
+
> You are an adversarial reviewer. Your job is to verify and, where possible, **refute** the answer
|
|
74
|
+
> below. Default to skepticism: assume a claim is wrong until a live source proves it. Address every
|
|
75
|
+
> comment **directly to the author** ("You claimed X; the source says Y").
|
|
76
|
+
>
|
|
77
|
+
> **The answer under review (verbatim):**
|
|
78
|
+
> {{ANSWER}}
|
|
79
|
+
>
|
|
80
|
+
> **Its claims and the source each rests on:**
|
|
81
|
+
> {{CLAIMS_AND_SOURCES}}
|
|
82
|
+
>
|
|
83
|
+
> **Do this:**
|
|
84
|
+
> 1. Re-verify every claim against the **live** source (issue tracker / code host / error monitor /
|
|
85
|
+
> the repo) - never trust the answer's own summary. Quote the field, line, PR body, or issue you
|
|
86
|
+
> checked.
|
|
87
|
+
> 2. **Check similar and related issues, and find patterns.** Is this part of a cluster? Is there a
|
|
88
|
+
> closer sibling, parent epic, or duplicate the answer should have surfaced but did not? Name it
|
|
89
|
+
> with its ID and the shared signal (same error, same code path, same root cause).
|
|
90
|
+
> 3. Stress-test the **strongest** load-bearing claim and the answer's biggest blind spot
|
|
91
|
+
> specifically. Where is it over-confident? Where is symptom-level reasoning passing itself off as
|
|
92
|
+
> a proven root cause?
|
|
93
|
+
> 4. List exactly **where the author was wrong or imprecise**, and what they **missed**.
|
|
94
|
+
> 5. Flag any claim you **could not verify** so it is not presented as fact.
|
|
95
|
+
>
|
|
96
|
+
> **Output, terse, with a receipt for every line:**
|
|
97
|
+
> - `verdict`: AGREE | AGREE-WITH-CAVEATS | DISAGREE
|
|
98
|
+
> - `score`: 0-100 (rubric: 100 = every claim verified, nothing missed, confidence correctly
|
|
99
|
+
> calibrated; subtract for each unverified claim, each missed sibling or pattern, and each
|
|
100
|
+
> over- or under-statement)
|
|
101
|
+
> - `where_wrong`: bullet list, each with its source receipt
|
|
102
|
+
> - `missed`: related issues or patterns the author failed to surface, with IDs
|
|
103
|
+
> - `unverifiable`: claims that need a source you do not have
|
|
104
|
+
> - `fixes`: ranked, actionable - what to change to raise the score
|
|
105
|
+
|
|
106
|
+
## Optional: auto-offer via a Claude Code Stop hook
|
|
107
|
+
|
|
108
|
+
A hook is a shell command, so it cannot run this loop itself. What it can do is **nudge**: after a
|
|
109
|
+
substantial answer, it offers the review and you run the skill only if the user says yes.
|
|
110
|
+
|
|
111
|
+
The hook ships with this skill at `scripts/critique-nudge.mjs`. Register it as a `Stop` hook in your
|
|
112
|
+
Claude Code settings (`.claude/settings.json` for a team, `.claude/settings.local.json` for
|
|
113
|
+
yourself):
|
|
114
|
+
|
|
115
|
+
```json
|
|
116
|
+
{
|
|
117
|
+
"hooks": {
|
|
118
|
+
"Stop": [
|
|
119
|
+
{
|
|
120
|
+
"hooks": [
|
|
121
|
+
{
|
|
122
|
+
"type": "command",
|
|
123
|
+
"command": "node \"$CLAUDE_PROJECT_DIR/.claude/hooks/critique-nudge.mjs\"",
|
|
124
|
+
"timeout": 15
|
|
125
|
+
}
|
|
126
|
+
]
|
|
127
|
+
}
|
|
128
|
+
]
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
Copy `scripts/critique-nudge.mjs` to `.claude/hooks/critique-nudge.mjs` (or point the command at
|
|
134
|
+
wherever you keep it). After an answer whose final prose is at least 1500 characters, the hook blocks
|
|
135
|
+
the turn once and instructs the agent to append a one-line "critical-thinking review" offer, then
|
|
136
|
+
stop. Two loop guards keep it from firing forever: it does nothing when `stop_hook_active` is set, and
|
|
137
|
+
nothing when the answer already contains the offer phrase. On any malformed input it exits silently
|
|
138
|
+
and never blocks the user.
|
|
139
|
+
|
|
140
|
+
The 1500-character threshold is a heuristic with a known ceiling: it misses short but dense
|
|
141
|
+
tool-heavy turns. If that matters, extend the script to also count tool calls since the last user
|
|
142
|
+
message.
|
|
143
|
+
|
|
144
|
+
## Notes
|
|
145
|
+
|
|
146
|
+
- One critic round is fine and common. If the answer was already solid, report
|
|
147
|
+
`iterations: 1, no revision needed` and the single score rather than forcing a revision.
|
|
148
|
+
- Diversity beats redundancy: if a claim can fail in more than one way, give the critic distinct
|
|
149
|
+
lenses (correctness, completeness, does-it-reproduce) instead of repeating the same check.
|
|
150
|
+
- The critic verifies against live sources; it does not rewrite the answer. You revise; it re-scores.
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Stop hook: after a substantial answer, make the model offer the user a
|
|
3
|
+
// critical-thinking review (adversarial critic agent, per the /self-critique skill).
|
|
4
|
+
// The hook only NUDGES. It cannot spawn agents or run the loop itself.
|
|
5
|
+
//
|
|
6
|
+
// ponytail: "substantial" = final assistant prose >= THRESHOLD chars. Known ceiling:
|
|
7
|
+
// misses short-but-dense tool-heavy turns. Upgrade path: also count tool_use blocks
|
|
8
|
+
// since the last user message if length alone misfires.
|
|
9
|
+
import { readFileSync } from 'node:fs';
|
|
10
|
+
|
|
11
|
+
const THRESHOLD = 1500;
|
|
12
|
+
const OFFER_MARKER = /critical-thinking review/i; // the offer line must contain this; doubles as a loop guard
|
|
13
|
+
|
|
14
|
+
function bail() { process.exit(0); } // any uncertainty => do nothing, never block the user
|
|
15
|
+
|
|
16
|
+
let raw = '';
|
|
17
|
+
try { raw = readFileSync(0, 'utf8'); } catch { bail(); }
|
|
18
|
+
|
|
19
|
+
let input;
|
|
20
|
+
try { input = JSON.parse(raw); } catch { bail(); }
|
|
21
|
+
|
|
22
|
+
// Loop guard #1: if this hook already forced a continuation, let the turn end.
|
|
23
|
+
if (input.stop_hook_active) bail();
|
|
24
|
+
|
|
25
|
+
const path = input.transcript_path;
|
|
26
|
+
if (!path) bail();
|
|
27
|
+
|
|
28
|
+
let lines;
|
|
29
|
+
try { lines = readFileSync(path, 'utf8').trim().split('\n'); } catch { bail(); }
|
|
30
|
+
|
|
31
|
+
// Walk from the end; grab the final assistant message that has text content.
|
|
32
|
+
let text = '';
|
|
33
|
+
for (let i = lines.length - 1; i >= 0; i--) {
|
|
34
|
+
let obj;
|
|
35
|
+
try { obj = JSON.parse(lines[i]); } catch { continue; }
|
|
36
|
+
const msg = obj.message;
|
|
37
|
+
if (obj.type === 'assistant' && msg && Array.isArray(msg.content)) {
|
|
38
|
+
text = msg.content.filter((c) => c.type === 'text').map((c) => c.text).join('\n');
|
|
39
|
+
if (text.trim()) break;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
if (text.length < THRESHOLD) bail();
|
|
44
|
+
|
|
45
|
+
// Loop guard #2: the answer already carries the offer (e.g. I added it last turn).
|
|
46
|
+
if (OFFER_MARKER.test(text)) bail();
|
|
47
|
+
|
|
48
|
+
const reason =
|
|
49
|
+
'That was a substantial answer. Before ending the turn, append exactly ONE short line ' +
|
|
50
|
+
'offering the user a "critical-thinking review": ask whether they want you to run /self-critique ' +
|
|
51
|
+
'(spawn an adversarial critic agent that challenges this answer, checks similar issues, finds ' +
|
|
52
|
+
'patterns, and reports where you were wrong, iterations-to-satisfy, the numeric improvement, and a ' +
|
|
53
|
+
'final score). The offer line MUST contain the phrase "critical-thinking review". Do NOT run the ' +
|
|
54
|
+
'critique now and do not re-explain the answer. Just offer it, then stop.';
|
|
55
|
+
|
|
56
|
+
process.stdout.write(JSON.stringify({ decision: 'block', reason }));
|
|
57
|
+
bail();
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Stop hook: after a substantial answer, make the model offer the user a
|
|
3
|
+
// critical-thinking review (adversarial critic agent, per the /self-critique skill).
|
|
4
|
+
// The hook only NUDGES. It cannot spawn agents or run the loop itself.
|
|
5
|
+
//
|
|
6
|
+
// ponytail: "substantial" = final assistant prose >= THRESHOLD chars. Known ceiling:
|
|
7
|
+
// misses short-but-dense tool-heavy turns. Upgrade path: also count tool_use blocks
|
|
8
|
+
// since the last user message if length alone misfires.
|
|
9
|
+
import { readFileSync } from 'node:fs';
|
|
10
|
+
|
|
11
|
+
const THRESHOLD = 1500;
|
|
12
|
+
const OFFER_MARKER = /critical-thinking review/i; // the offer line must contain this; doubles as a loop guard
|
|
13
|
+
|
|
14
|
+
function bail() { process.exit(0); } // any uncertainty => do nothing, never block the user
|
|
15
|
+
|
|
16
|
+
let raw = '';
|
|
17
|
+
try { raw = readFileSync(0, 'utf8'); } catch { bail(); }
|
|
18
|
+
|
|
19
|
+
let input;
|
|
20
|
+
try { input = JSON.parse(raw); } catch { bail(); }
|
|
21
|
+
|
|
22
|
+
// Loop guard #1: if this hook already forced a continuation, let the turn end.
|
|
23
|
+
if (input.stop_hook_active) bail();
|
|
24
|
+
|
|
25
|
+
const path = input.transcript_path;
|
|
26
|
+
if (!path) bail();
|
|
27
|
+
|
|
28
|
+
let lines;
|
|
29
|
+
try { lines = readFileSync(path, 'utf8').trim().split('\n'); } catch { bail(); }
|
|
30
|
+
|
|
31
|
+
// Walk from the end; grab the final assistant message that has text content.
|
|
32
|
+
let text = '';
|
|
33
|
+
for (let i = lines.length - 1; i >= 0; i--) {
|
|
34
|
+
let obj;
|
|
35
|
+
try { obj = JSON.parse(lines[i]); } catch { continue; }
|
|
36
|
+
const msg = obj.message;
|
|
37
|
+
if (obj.type === 'assistant' && msg && Array.isArray(msg.content)) {
|
|
38
|
+
text = msg.content.filter((c) => c.type === 'text').map((c) => c.text).join('\n');
|
|
39
|
+
if (text.trim()) break;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
if (text.length < THRESHOLD) bail();
|
|
44
|
+
|
|
45
|
+
// Loop guard #2: the answer already carries the offer (e.g. I added it last turn).
|
|
46
|
+
if (OFFER_MARKER.test(text)) bail();
|
|
47
|
+
|
|
48
|
+
const reason =
|
|
49
|
+
'That was a substantial answer. Before ending the turn, append exactly ONE short line ' +
|
|
50
|
+
'offering the user a "critical-thinking review": ask whether they want you to run /self-critique ' +
|
|
51
|
+
'(spawn an adversarial critic agent that challenges this answer, checks similar issues, finds ' +
|
|
52
|
+
'patterns, and reports where you were wrong, iterations-to-satisfy, the numeric improvement, and a ' +
|
|
53
|
+
'final score). The offer line MUST contain the phrase "critical-thinking review". Do NOT run the ' +
|
|
54
|
+
'critique now and do not re-explain the answer. Just offer it, then stop.';
|
|
55
|
+
|
|
56
|
+
process.stdout.write(JSON.stringify({ decision: 'block', reason }));
|
|
57
|
+
bail();
|