hotmilk 0.1.13 → 0.1.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +12 -10
- package/README.md +237 -53
- package/agents/README.md +3 -3
- package/agents/assistant.md +1 -1
- package/agents/coach.md +26 -4
- package/agents/designer.md +1 -1
- package/agents/planner.md +2 -2
- package/hotmilk.json +11 -0
- package/package.json +43 -24
- package/prompts/prompt-eval.md +241 -0
- package/prompts/tidy.md +3 -1
- package/prompts/translate.md +3 -1
- package/skills/make-docs/SKILL.md +43 -0
- package/skills/pioneer/SKILL.md +135 -131
- package/skills/pioneer/references/autoresearch-routing.md +114 -0
- package/skills/pioneer/references/chat-plan.md +50 -0
- package/skills/pioneer/references/goal-gate.md +54 -0
- package/skills/pioneer/references/graph-recon-gate.md +50 -0
- package/skills/pioneer/references/observational-memory-routing.md +90 -0
- package/skills/pioneer/references/openspec-routing.md +50 -0
- package/skills/pioneer/references/plannotator-routing.md +83 -0
- package/skills/pioneer/references/prompt-eval-gate.md +38 -0
- package/skills/pioneer/references/shazam-routing.md +58 -0
- package/skills/recommend-research/SKILL.md +94 -6
- package/skills/recommend-research/references/example-narrow.md +53 -0
- package/skills/recommend-research/references/scope-gate.md +46 -0
- package/skills/recommend-research/references/shortlist-gate.md +69 -0
- package/skills/recommend-research/references/verify-gate.md +73 -0
- package/skills/update-docs/SKILL.md +86 -0
- package/skills/update-docs/references/doc-inventory.md +69 -0
- package/skills/update-docs/references/drift-verification.md +59 -0
- package/src/bootstrap/btw.ts +49 -0
- package/src/bootstrap/context-stack.ts +71 -14
- package/src/bootstrap/dashboard-settings.ts +16 -2
- package/src/bootstrap/dashboard.ts +85 -10
- package/src/bootstrap/defaults.ts +26 -0
- package/src/bootstrap/extensions.ts +10 -1
- package/src/bootstrap/global-extension-sources.ts +43 -4
- package/src/bootstrap/graph.ts +12 -0
- package/src/bootstrap/project-trust.ts +55 -0
- package/src/bootstrap/resolve-bundled.ts +28 -4
- package/src/bootstrap/session.ts +61 -1
- package/src/bootstrap/subagents-doctor.ts +12 -0
- package/src/config/bundled-extensions.ts +42 -0
- package/src/config/bundled-package-registry.ts +3 -0
- package/src/config/hotmilk.ts +76 -1
- package/src/config/mcp.ts +45 -12
- package/src/config/resolve.ts +21 -0
- package/src/config/runtime.ts +6 -0
- package/src/controller/input.ts +15 -0
- package/src/controller/mode.ts +32 -0
- package/src/index.ts +5 -0
- package/src/ui/footer.ts +63 -1
- package/src/ui/github-user.ts +52 -4
- package/src/ui/session-logo.ts +31 -3
- package/themes/monokai.json +1 -1
- package/skills/empirical-prompt-tuning/SKILL.md +0 -232
package/src/ui/footer.ts
CHANGED
|
@@ -13,7 +13,12 @@ import {
|
|
|
13
13
|
resolveGithubFooterContextAsync,
|
|
14
14
|
} from "./github-user.ts";
|
|
15
15
|
|
|
16
|
-
/**
|
|
16
|
+
/**
|
|
17
|
+
* Pure footer clock formatting (no pi-coding-agent imports — safe for unit tests).
|
|
18
|
+
*
|
|
19
|
+
* @param date - Date to format
|
|
20
|
+
* @returns formatted time string in HH:mm:ss format
|
|
21
|
+
*/
|
|
17
22
|
export function formatFooterTime(date: Date): string {
|
|
18
23
|
return new Intl.DateTimeFormat(undefined, {
|
|
19
24
|
hour: "2-digit",
|
|
@@ -23,6 +28,12 @@ export function formatFooterTime(date: Date): string {
|
|
|
23
28
|
}).format(date);
|
|
24
29
|
}
|
|
25
30
|
|
|
31
|
+
/**
|
|
32
|
+
* Sanitize footer status text by removing newlines, tabs, and extra spaces.
|
|
33
|
+
*
|
|
34
|
+
* @param text - raw status text
|
|
35
|
+
* @returns sanitized text
|
|
36
|
+
*/
|
|
26
37
|
function sanitizeStatusText(text: string): string {
|
|
27
38
|
return text
|
|
28
39
|
.replace(/[\r\n\t]/g, " ")
|
|
@@ -30,6 +41,15 @@ function sanitizeStatusText(text: string): string {
|
|
|
30
41
|
.trim();
|
|
31
42
|
}
|
|
32
43
|
|
|
44
|
+
/**
|
|
45
|
+
* Format extension status lines for the footer.
|
|
46
|
+
*
|
|
47
|
+
* @param footerData - footer data provider
|
|
48
|
+
* @param width - terminal width
|
|
49
|
+
* @param dim - dimming function
|
|
50
|
+
* @param ellipsis - ellipsis string
|
|
51
|
+
* @returns formatted status lines
|
|
52
|
+
*/
|
|
33
53
|
function extensionStatusLines(
|
|
34
54
|
footerData: ReadonlyFooterDataProvider,
|
|
35
55
|
width: number,
|
|
@@ -48,10 +68,24 @@ function extensionStatusLines(
|
|
|
48
68
|
.map((text) => truncateToWidth(dim(text), width, ellipsis));
|
|
49
69
|
}
|
|
50
70
|
|
|
71
|
+
/**
|
|
72
|
+
* Check if a footer line has visible content.
|
|
73
|
+
*
|
|
74
|
+
* @param line - line to check
|
|
75
|
+
* @returns true if line has visible content
|
|
76
|
+
*/
|
|
51
77
|
function isVisibleFooterLine(line: string): boolean {
|
|
52
78
|
return visibleWidth(line) > 0;
|
|
53
79
|
}
|
|
54
80
|
|
|
81
|
+
/**
|
|
82
|
+
* Append metadata (time, term program) to the last footer line if it fits.
|
|
83
|
+
*
|
|
84
|
+
* @param lines - existing footer lines
|
|
85
|
+
* @param meta - metadata to append
|
|
86
|
+
* @param width - terminal width
|
|
87
|
+
* @returns updated lines
|
|
88
|
+
*/
|
|
55
89
|
function appendMetaToLastLine(lines: string[], meta: string, width: number): string[] {
|
|
56
90
|
const visibleLines = lines.filter(isVisibleFooterLine);
|
|
57
91
|
if (visibleLines.length === 0) {
|
|
@@ -75,6 +109,12 @@ function appendMetaToLastLine(lines: string[], meta: string, width: number): str
|
|
|
75
109
|
|
|
76
110
|
const FOOTER_TIME_REFRESH_MS = 30_000;
|
|
77
111
|
|
|
112
|
+
/**
|
|
113
|
+
* Get the latest thinking level from session manager entries.
|
|
114
|
+
*
|
|
115
|
+
* @param sessionManager - Pi session manager
|
|
116
|
+
* @returns latest thinking level
|
|
117
|
+
*/
|
|
78
118
|
function latestThinkingLevel(sessionManager: ExtensionContext["sessionManager"]): ThinkingLevel {
|
|
79
119
|
const entries = sessionManager.getEntries();
|
|
80
120
|
for (let i = entries.length - 1; i >= 0; i--) {
|
|
@@ -86,6 +126,12 @@ function latestThinkingLevel(sessionManager: ExtensionContext["sessionManager"])
|
|
|
86
126
|
return "off";
|
|
87
127
|
}
|
|
88
128
|
|
|
129
|
+
/**
|
|
130
|
+
* Create a footer-compatible session object from extension context.
|
|
131
|
+
*
|
|
132
|
+
* @param ctx - extension context
|
|
133
|
+
* @returns agent session for footer
|
|
134
|
+
*/
|
|
89
135
|
function footerSessionFromContext(ctx: ExtensionContext): AgentSession {
|
|
90
136
|
return {
|
|
91
137
|
get state() {
|
|
@@ -100,6 +146,16 @@ function footerSessionFromContext(ctx: ExtensionContext): AgentSession {
|
|
|
100
146
|
} as AgentSession;
|
|
101
147
|
}
|
|
102
148
|
|
|
149
|
+
/**
|
|
150
|
+
* Decorate the pwd line with GitHub user handle if available.
|
|
151
|
+
*
|
|
152
|
+
* @param pwdLine - current working directory line
|
|
153
|
+
* @param githubUser - resolved GitHub user
|
|
154
|
+
* @param repoOwner - repo owner
|
|
155
|
+
* @param width - terminal width
|
|
156
|
+
* @param theme - Pi theme
|
|
157
|
+
* @returns decorated pwd line
|
|
158
|
+
*/
|
|
103
159
|
function decoratePwdLine(
|
|
104
160
|
pwdLine: string,
|
|
105
161
|
githubUser: string | undefined,
|
|
@@ -118,6 +174,12 @@ function decoratePwdLine(
|
|
|
118
174
|
return truncateToWidth(decorated, width, theme.fg("dim", "..."));
|
|
119
175
|
}
|
|
120
176
|
|
|
177
|
+
/**
|
|
178
|
+
* Install the hotmilk footer with GitHub user, clock, and extension status lines.
|
|
179
|
+
*
|
|
180
|
+
* @param ctx - extension context
|
|
181
|
+
* @param termProgram - terminal program identifier
|
|
182
|
+
*/
|
|
121
183
|
export function setupHotmilkFooter(ctx: ExtensionContext, termProgram: string): void {
|
|
122
184
|
if (!ctx.hasUI) {
|
|
123
185
|
return;
|
package/src/ui/github-user.ts
CHANGED
|
@@ -1,3 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* GitHub user / repo-owner resolution for the footer.
|
|
3
|
+
*
|
|
4
|
+
* Reads `$GITHUB_USER`, `gh api user`, git config, and the `origin` remote
|
|
5
|
+
* to show `@handle` and highlight repo ownership in the status bar.
|
|
6
|
+
*/
|
|
7
|
+
|
|
1
8
|
import { execFile, execFileSync } from "node:child_process";
|
|
2
9
|
import { promisify } from "node:util";
|
|
3
10
|
|
|
@@ -57,11 +64,23 @@ async function runCommandAsync(
|
|
|
57
64
|
const GITHUB_REMOTE_OWNER =
|
|
58
65
|
/^(?:git@github\.com:|https?:\/\/github\.com\/|ssh:\/\/git@github\.com\/)([^/]+)\//;
|
|
59
66
|
|
|
60
|
-
/**
|
|
67
|
+
/**
|
|
68
|
+
* Parse the owner/user from common github.com remote URL shapes.
|
|
69
|
+
*
|
|
70
|
+
* @param url - remote URL like `https://github.com/owner/repo.git`
|
|
71
|
+
* @returns the owner string, or undefined if not a GitHub remote
|
|
72
|
+
*/
|
|
61
73
|
export function parseGithubUserFromRemote(url: string): string | undefined {
|
|
62
74
|
return GITHUB_REMOTE_OWNER.exec(url.trim())?.[1];
|
|
63
75
|
}
|
|
64
76
|
|
|
77
|
+
/**
|
|
78
|
+
* Append the GitHub handle to the prompt path segment.
|
|
79
|
+
*
|
|
80
|
+
* @param pwd - current working directory string
|
|
81
|
+
* @param githubUser - resolved GitHub handle
|
|
82
|
+
* @returns footer path text
|
|
83
|
+
*/
|
|
65
84
|
export function formatFooterPwdWithGithubUser(pwd: string, githubUser: string | undefined): string {
|
|
66
85
|
if (!githubUser) {
|
|
67
86
|
return pwd;
|
|
@@ -69,16 +88,28 @@ export function formatFooterPwdWithGithubUser(pwd: string, githubUser: string |
|
|
|
69
88
|
return `${pwd} · @${githubUser}`;
|
|
70
89
|
}
|
|
71
90
|
|
|
91
|
+
/**
|
|
92
|
+
* Check whether the GitHub user owns the current repo.
|
|
93
|
+
*
|
|
94
|
+
* @param githubUser - resolved GitHub handle
|
|
95
|
+
* @param repoOwner - owner parsed from `origin`
|
|
96
|
+
*/
|
|
72
97
|
export function isGithubRepoOwner(githubUser: string, repoOwner: string | undefined): boolean {
|
|
73
98
|
return repoOwner !== undefined && githubUser.toLowerCase() === repoOwner.toLowerCase();
|
|
74
99
|
}
|
|
75
100
|
|
|
101
|
+
/** Options for {@link resolveRepoOwner}. */
|
|
76
102
|
export type ResolveRepoOwnerOptions = {
|
|
77
103
|
cwd?: string;
|
|
104
|
+
/** Override command executor for tests. */
|
|
78
105
|
runCommand?: (file: string, args: string[], cwd?: string) => string | undefined;
|
|
79
106
|
};
|
|
80
107
|
|
|
81
|
-
/**
|
|
108
|
+
/**
|
|
109
|
+
* Resolve the GitHub repo owner from the `origin` remote.
|
|
110
|
+
*
|
|
111
|
+
* @param opts - options, including optional command executor for tests
|
|
112
|
+
*/
|
|
82
113
|
export function resolveRepoOwner(opts: ResolveRepoOwnerOptions = {}): string | undefined {
|
|
83
114
|
const cwd = opts.cwd ?? process.cwd();
|
|
84
115
|
const exec = opts.runCommand ?? runCommand;
|
|
@@ -89,13 +120,21 @@ export function resolveRepoOwner(opts: ResolveRepoOwnerOptions = {}): string | u
|
|
|
89
120
|
return parseGithubUserFromRemote(remote);
|
|
90
121
|
}
|
|
91
122
|
|
|
123
|
+
/** Options for {@link resolveGithubUsername}. */
|
|
92
124
|
export type ResolveGithubUsernameOptions = {
|
|
93
125
|
cwd?: string;
|
|
94
126
|
env?: NodeJS.ProcessEnv;
|
|
127
|
+
/** Override command executor for tests. */
|
|
95
128
|
runCommand?: (file: string, args: string[], cwd?: string) => string | undefined;
|
|
96
129
|
};
|
|
97
130
|
|
|
98
|
-
/**
|
|
131
|
+
/**
|
|
132
|
+
* Best-effort GitHub login for footer display.
|
|
133
|
+
*
|
|
134
|
+
* Resolution order: `$GITHUB_USER` → `gh api user` → git config → origin remote.
|
|
135
|
+
*
|
|
136
|
+
* @param opts - options, including optional command executor for tests
|
|
137
|
+
*/
|
|
99
138
|
export function resolveGithubUsername(opts: ResolveGithubUsernameOptions = {}): string | undefined {
|
|
100
139
|
const cwd = opts.cwd ?? process.cwd();
|
|
101
140
|
const env = opts.env ?? process.env;
|
|
@@ -117,18 +156,27 @@ export function resolveGithubUsername(opts: ResolveGithubUsernameOptions = {}):
|
|
|
117
156
|
return remote ? parseGithubUserFromRemote(remote) : undefined;
|
|
118
157
|
}
|
|
119
158
|
|
|
159
|
+
/** Footer payload containing the GitHub handle and repo owner. */
|
|
120
160
|
export type GithubFooterContext = {
|
|
121
161
|
githubUser?: string;
|
|
122
162
|
repoOwner?: string;
|
|
123
163
|
};
|
|
124
164
|
|
|
165
|
+
/** Options for {@link resolveGithubFooterContextAsync}. */
|
|
125
166
|
export type ResolveGithubFooterContextOptions = {
|
|
126
167
|
cwd?: string;
|
|
127
168
|
env?: NodeJS.ProcessEnv;
|
|
169
|
+
/** Override async command executor for tests. */
|
|
128
170
|
runCommandAsync?: (file: string, args: string[], cwd?: string) => Promise<string | undefined>;
|
|
129
171
|
};
|
|
130
172
|
|
|
131
|
-
/**
|
|
173
|
+
/**
|
|
174
|
+
* Resolve footer GitHub handle and repo owner asynchronously.
|
|
175
|
+
*
|
|
176
|
+
* Non-blocking so the footer can refresh after UI startup.
|
|
177
|
+
*
|
|
178
|
+
* @param opts - options, including optional async command executor for tests
|
|
179
|
+
*/
|
|
132
180
|
export async function resolveGithubFooterContextAsync(
|
|
133
181
|
opts: ResolveGithubFooterContextOptions = {},
|
|
134
182
|
): Promise<GithubFooterContext> {
|
package/src/ui/session-logo.ts
CHANGED
|
@@ -13,10 +13,18 @@ const LOGO_LINES = HOTMILK_LOGO.split("\n");
|
|
|
13
13
|
/** Same lead time as gentle-pi `startup-banner.ts`. */
|
|
14
14
|
const LOGO_SHOW_DELAY_MS = 50;
|
|
15
15
|
|
|
16
|
+
/** Timeout handle for delayed logo display. */
|
|
16
17
|
let showTimeout: ReturnType<typeof setTimeout> | undefined;
|
|
18
|
+
/** Active TUI instance for logo rendering. */
|
|
17
19
|
let activeTui: TUI | undefined;
|
|
18
20
|
|
|
19
|
-
/**
|
|
21
|
+
/**
|
|
22
|
+
* Center fixed-width ASCII lines within the terminal width.
|
|
23
|
+
*
|
|
24
|
+
* @param lines - ASCII lines to center
|
|
25
|
+
* @param width - terminal width
|
|
26
|
+
* @returns centered lines
|
|
27
|
+
*/
|
|
20
28
|
export function centerAsciiLines(lines: readonly string[], width: number): string[] {
|
|
21
29
|
const contentWidth = Math.max(...lines.map((line) => line.length), 1);
|
|
22
30
|
const pad = Math.max(0, Math.floor((width - contentWidth) / 2));
|
|
@@ -24,11 +32,17 @@ export function centerAsciiLines(lines: readonly string[], width: number): strin
|
|
|
24
32
|
return lines.map((line) => prefix + line.padEnd(contentWidth));
|
|
25
33
|
}
|
|
26
34
|
|
|
27
|
-
/**
|
|
35
|
+
/**
|
|
36
|
+
* Skip only `/reload`; show on fresh launch and session resume (gentle-pi shows every session_start).
|
|
37
|
+
*
|
|
38
|
+
* @param reason - session start reason
|
|
39
|
+
* @returns true if logo should be shown
|
|
40
|
+
*/
|
|
28
41
|
export function shouldShowSessionLogo(reason: string): boolean {
|
|
29
42
|
return reason !== "reload";
|
|
30
43
|
}
|
|
31
44
|
|
|
45
|
+
/** Clear any pending logo display timers. */
|
|
32
46
|
function clearLogoTimers(): void {
|
|
33
47
|
if (showTimeout) {
|
|
34
48
|
clearTimeout(showTimeout);
|
|
@@ -36,6 +50,11 @@ function clearLogoTimers(): void {
|
|
|
36
50
|
}
|
|
37
51
|
}
|
|
38
52
|
|
|
53
|
+
/**
|
|
54
|
+
* Dismiss the hotmilk logo from the header.
|
|
55
|
+
*
|
|
56
|
+
* @param ctx - extension context
|
|
57
|
+
*/
|
|
39
58
|
function dismissHotmilkLogo(ctx: ExtensionContext): void {
|
|
40
59
|
if (!ctx.hasUI) return;
|
|
41
60
|
ctx.ui.setHeader(undefined);
|
|
@@ -43,6 +62,11 @@ function dismissHotmilkLogo(ctx: ExtensionContext): void {
|
|
|
43
62
|
activeTui = undefined;
|
|
44
63
|
}
|
|
45
64
|
|
|
65
|
+
/**
|
|
66
|
+
* Show the hotmilk session logo with a delay.
|
|
67
|
+
*
|
|
68
|
+
* @param ctx - extension context
|
|
69
|
+
*/
|
|
46
70
|
function showHotmilkSessionLogo(ctx: ExtensionContext): void {
|
|
47
71
|
if (!ctx.hasUI) return;
|
|
48
72
|
|
|
@@ -65,7 +89,11 @@ function showHotmilkSessionLogo(ctx: ExtensionContext): void {
|
|
|
65
89
|
}, LOGO_SHOW_DELAY_MS);
|
|
66
90
|
}
|
|
67
91
|
|
|
68
|
-
/**
|
|
92
|
+
/**
|
|
93
|
+
* Register persistent header logo on session start; clear on shutdown only.
|
|
94
|
+
*
|
|
95
|
+
* @param pi - Pi extension API
|
|
96
|
+
*/
|
|
69
97
|
export function registerHotmilkSessionLogo(pi: ExtensionAPI): void {
|
|
70
98
|
pi.on("session_start", (event, ctx) => {
|
|
71
99
|
if (!shouldShowSessionLogo(event.reason)) return;
|
package/themes/monokai.json
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
{
|
|
2
|
-
"$schema": "https://raw.githubusercontent.com/
|
|
2
|
+
"$schema": "https://raw.githubusercontent.com/earendil-works/pi/main/packages/coding-agent/src/modes/interactive/theme/theme-schema.json",
|
|
3
3
|
"name": "monokai",
|
|
4
4
|
"vars": {
|
|
5
5
|
"bg": "#272822",
|
|
@@ -1,232 +0,0 @@
|
|
|
1
|
-
---
|
|
2
|
-
name: empirical-prompt-tuning
|
|
3
|
-
description: "Harden prompts: fresh executor each iteration, checklist scoring, one themed delta per loop until convergence. After editing skills, AGENTS.md, CLAUDE.md, or task prompts. [Triggers: empirical prompt tuning, prompt evaluation, tune prompt, fresh agent evaluation]"
|
|
4
|
-
---
|
|
5
|
-
|
|
6
|
-
# Empirical Prompt Tuning
|
|
7
|
-
|
|
8
|
-
A prompt's quality is opaque to its author. **Dispatch a fresh agent, run the prompt, evaluate on two axes, iterate until plateau.** Do not stop before the plateau.
|
|
9
|
-
|
|
10
|
-
## Core Concept: Fresh Agent
|
|
11
|
-
|
|
12
|
-
Any executor with **no authoring context** and **no memory of prior iterations**. The implementation does not matter; the absence of authoring bias does. Self re-reading never qualifies.
|
|
13
|
-
|
|
14
|
-
### Dispatch Methods (pick one)
|
|
15
|
-
|
|
16
|
-
| Method | How | Notes |
|
|
17
|
-
| ------------------------------ | -------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------- |
|
|
18
|
-
| **pi-subagents** (recommended) | `subagent({ agent: "assistant", task: "...", context: "fresh", model: "<model-id>", skill: false, clarify: false })` | Always set `context: "fresh"`, explicit `model`, `skill: false`, `clarify: false` |
|
|
19
|
-
| **Claude Code Task** | `{ agent: "worker", task: "...", context: "fresh" }` | Alternative when PI unavailable |
|
|
20
|
-
| **Separate session** | New terminal: `claude` or `pi` | Manual but reliable |
|
|
21
|
-
| **Direct API** | POST to `/v1/messages` with empty history | For programmatic testing |
|
|
22
|
-
|
|
23
|
-
**NOT fresh**: `context: "fork"` (inherits history), same session, self re-reading.
|
|
24
|
-
|
|
25
|
-
For parallel scenarios, use `subagent({ tasks: [...] })` with models from the **same provider** to avoid cross-provider timeouts.
|
|
26
|
-
|
|
27
|
-
## Workflow
|
|
28
|
-
|
|
29
|
-
```
|
|
30
|
-
0. Static coherence check (no dispatch)
|
|
31
|
-
1. Baseline: stakes tier + scenarios + requirement checklists
|
|
32
|
-
2. Dispatch fresh agent → execute → self-report
|
|
33
|
-
3. Two-sided evaluation (self-report + instructor metrics)
|
|
34
|
-
4. Apply ONE themed delta
|
|
35
|
-
5. New fresh agent → re-evaluate
|
|
36
|
-
6. Stop at 2 consecutive clears (or 3 for high-stakes)
|
|
37
|
-
```
|
|
38
|
-
|
|
39
|
-
### Step 0 — Static Coherence Check
|
|
40
|
-
|
|
41
|
-
Compare what the prompt **advertises** vs what the body **delivers**. Fix divergences before any dispatch.
|
|
42
|
-
|
|
43
|
-
| Target type | "Advertises" source | Check |
|
|
44
|
-
| --------------------------- | ---------------------------- | ------------------------------------------- |
|
|
45
|
-
| Skill file | Frontmatter `description` | Triggers/use-cases match body coverage? |
|
|
46
|
-
| CLAUDE.md section | Section heading | Heading promise matches actionable content? |
|
|
47
|
-
| Task prompt / system prompt | Opening instruction or title | Stated goal matches the actual rules? |
|
|
48
|
-
|
|
49
|
-
Skipping this causes false positives — the fresh agent silently re-interprets the body.
|
|
50
|
-
|
|
51
|
-
### Step 1 — Baseline Preparation
|
|
52
|
-
|
|
53
|
-
**Choose and freeze a stakes tier** before anything else:
|
|
54
|
-
|
|
55
|
-
| Stakes | Scenarios | `[critical]` tags | Stop condition | Iteration cap |
|
|
56
|
-
| --------------------------------- | --------------------- | ----------------- | -------------------- | ------------- |
|
|
57
|
-
| **High** (core skill, automation) | 3 (1 median + 2 edge) | ≥ 2 | 3 consecutive clears | none |
|
|
58
|
-
| **Standard** (default) | 2 (1 median + 1 edge) | ≥ 1 | 2 consecutive clears | 5 |
|
|
59
|
-
| **Low** (small, low-blast) | 2 (1 median + 1 edge) | ≥ 1 | ship at 80% | 3 |
|
|
60
|
-
|
|
61
|
-
For each scenario, write a **requirement checklist** (3–7 items). Rules:
|
|
62
|
-
|
|
63
|
-
- At least one item tagged `[critical]` — without it, success judgment is vacuous.
|
|
64
|
-
- Fix the checklist before running. Do not adjust after seeing results.
|
|
65
|
-
- **Translating user complaints into checklist items**: If the user says "it produces shallow output", decompose "shallow" into observable, testable criteria (e.g., "identifies at least one concrete bug", "includes line references", "provides actionable suggestions"). Each criterion becomes a checklist item. The user's complaint itself is not a valid checklist item.
|
|
66
|
-
- `[critical]` means: if this item fails, the whole scenario fails (binary ×), regardless of other items.
|
|
67
|
-
- Non-critical items score: ○ = 1, partial = 0.5, × = 0. Accuracy = sum / total.
|
|
68
|
-
|
|
69
|
-
### Step 2–3 — Dispatch and Execute
|
|
70
|
-
|
|
71
|
-
Hand the fresh agent this contract:
|
|
72
|
-
|
|
73
|
-
```
|
|
74
|
-
You are an executor reading <target prompt name> as a blank slate.
|
|
75
|
-
You have no prior exposure to this prompt; treat it as newly encountered.
|
|
76
|
-
|
|
77
|
-
## Target Prompt
|
|
78
|
-
<paste the full target prompt here>
|
|
79
|
-
|
|
80
|
-
## Scenario
|
|
81
|
-
<one paragraph describing the situation>
|
|
82
|
-
|
|
83
|
-
## Requirement Checklist
|
|
84
|
-
1. [critical] <item>
|
|
85
|
-
2. <item>
|
|
86
|
-
...
|
|
87
|
-
|
|
88
|
-
## Task
|
|
89
|
-
1. Execute the scenario following the target prompt and produce the deliverable.
|
|
90
|
-
2. On completion, respond with the report below.
|
|
91
|
-
|
|
92
|
-
## Report Structure
|
|
93
|
-
- Deliverable: <the artifact or run summary>
|
|
94
|
-
- Requirement status: for each item, ○ / × / partial (with reason)
|
|
95
|
-
- Ambiguities: places where wording was open to interpretation (bullets)
|
|
96
|
-
- Discretionary fills: decisions the instructions did not specify (bullets)
|
|
97
|
-
- Retries: how many times you redid a decision and why
|
|
98
|
-
```
|
|
99
|
-
|
|
100
|
-
**Always paste the full prompt** in the contract (not a file path) — ensures portability across all harness types.
|
|
101
|
-
|
|
102
|
-
### Step 4 — Two-Sided Evaluation
|
|
103
|
-
|
|
104
|
-
| Axis | Source | How to measure |
|
|
105
|
-
| ----------------------- | -------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
106
|
-
| **Success/Failure** | Instructor | ○ only if every `[critical]` met. Binary. |
|
|
107
|
-
| **Accuracy** | Instructor | ○/partial/× per item → percentage |
|
|
108
|
-
| **Step count** | Environment metadata | `tool_uses` (Claude Code), tool-use blocks (API). For **text-generation targets** (no tool use): substitute with a concrete output-unit count — e.g., number of review comments, checklist items produced, or code blocks emitted. State the chosen unit once at the top of the iteration log. |
|
|
109
|
-
| **Duration** | Environment metadata | `duration_ms` or wall-clock |
|
|
110
|
-
| **Retry count** | Self-report | Times the agent redid a decision |
|
|
111
|
-
| **Ambiguities** | Self-report | Qualitative — primary signal for improvement |
|
|
112
|
-
| **Discretionary fills** | Self-report | Surfaces implicit requirements |
|
|
113
|
-
|
|
114
|
-
**Weighting**: Qualitative signals (ambiguities, fills) are primary. Quantitative metrics (time, steps) are supporting. Chasing time alone makes prompts brittle.
|
|
115
|
-
|
|
116
|
-
**On failure**: append one line to Ambiguities naming which `[critical]` item dropped.
|
|
117
|
-
|
|
118
|
-
#### Step Count as Structure Signal
|
|
119
|
-
|
|
120
|
-
Even at 100% accuracy, step-count skew across scenarios exposes structural flaws:
|
|
121
|
-
|
|
122
|
-
- One scenario at 3–5× the others → the prompt lacks self-containment for that case (executor wandered).
|
|
123
|
-
- Fix: add inline guidance or a minimal complete example.
|
|
124
|
-
|
|
125
|
-
### Step 5 — Apply ONE Delta
|
|
126
|
-
|
|
127
|
-
1. Identify the most impactful ambiguity.
|
|
128
|
-
2. **State which checklist item the edit satisfies** before editing — edits inferred from axis names alone rarely land.
|
|
129
|
-
3. Apply the minimum edit. One semantic theme per iteration. A cluster of 2–3 related micro-edits is one theme; unrelated edits go to the next iteration.
|
|
130
|
-
|
|
131
|
-
#### Correction Propagation Patterns
|
|
132
|
-
|
|
133
|
-
| Pattern | What happens | Lesson |
|
|
134
|
-
| ------------------ | ------------------------------------------- | ----------------------------------------------------------- |
|
|
135
|
-
| Conservative drift | Edit aimed at multiple axes, only one moved | Multi-axis shots usually miss |
|
|
136
|
-
| Upside drift | One structural edit satisfied multiple axes | Information combos are naturally multi-axis |
|
|
137
|
-
| Null drift | Edit didn't move any axis | Axis names ≠ judgment text. Tie edits to threshold wording. |
|
|
138
|
-
|
|
139
|
-
### Step 6 — Re-evaluate
|
|
140
|
-
|
|
141
|
-
Dispatch a **new** fresh agent. Never reuse — it learned from the prior run. Repeat from Step 2.
|
|
142
|
-
|
|
143
|
-
### Step 7 — Stopping Criteria
|
|
144
|
-
|
|
145
|
-
**Converged** when for N consecutive iterations (N = 2 standard, 3 high-stakes):
|
|
146
|
-
|
|
147
|
-
- New ambiguities: 0
|
|
148
|
-
- Accuracy gain: ≤ +3 points
|
|
149
|
-
- Step count change: within ±10%
|
|
150
|
-
- Duration change: within ±15%
|
|
151
|
-
|
|
152
|
-
**At convergence** (all tiers): add one hold-out scenario not used in any prior iteration. If accuracy drops ≥ 15 points → overfitting. Return to scenario design. This is mandatory, not optional.
|
|
153
|
-
|
|
154
|
-
**Diverged**: after 3+ iterations ambiguities aren't decreasing → the prompt's design is wrong. Stop patching, rewrite.
|
|
155
|
-
|
|
156
|
-
**Resource cutoff**: ship at 80% when improvement cost exceeds prompt importance.
|
|
157
|
-
|
|
158
|
-
### Presentation Format
|
|
159
|
-
|
|
160
|
-
Record each iteration as:
|
|
161
|
-
|
|
162
|
-
```
|
|
163
|
-
## Iteration N
|
|
164
|
-
|
|
165
|
-
### Changes (delta from previous)
|
|
166
|
-
- <one-line edit description>
|
|
167
|
-
|
|
168
|
-
### Results
|
|
169
|
-
| Scenario | Success | Accuracy | Steps | Duration | Retries |
|
|
170
|
-
|---|---|---|---|---|---|
|
|
171
|
-
| A | ○ | 90% | 4 | 20s | 0 |
|
|
172
|
-
| B | × | 60% | 9 | 41s | 2 |
|
|
173
|
-
|
|
174
|
-
### Ambiguities (new this iteration)
|
|
175
|
-
- <Scenario B>: [critical] item N was × — <reason>
|
|
176
|
-
- <Scenario A>: (none new)
|
|
177
|
-
|
|
178
|
-
### Discretionary fills (new this iteration)
|
|
179
|
-
- <Scenario B>: <what was filled>
|
|
180
|
-
|
|
181
|
-
### Next edit
|
|
182
|
-
- <one-line minimal edit>
|
|
183
|
-
|
|
184
|
-
(Convergence: X consecutive clears / Y iterations until stop)
|
|
185
|
-
```
|
|
186
|
-
|
|
187
|
-
- Never leave an axis blank (write "0", not "—").
|
|
188
|
-
- "None new" is explicit signal.
|
|
189
|
-
- Steps column may be replaced with turn count in plain-chat environments — note the substitution once.
|
|
190
|
-
|
|
191
|
-
## Red Flags
|
|
192
|
-
|
|
193
|
-
| Rationalization | Reality |
|
|
194
|
-
| ------------------------------------------------ | -------------------------------------------------------------------- |
|
|
195
|
-
| "Re-reading it myself is equivalent." | Structurally impossible to objectively view text you just wrote. |
|
|
196
|
-
| "One scenario is enough." | Overfits. Minimum 2. |
|
|
197
|
-
| "Zero ambiguities once, so done." | May be coincidence. Require consecutive clears. |
|
|
198
|
-
| "Fix multiple things in one pass." | Lose attribution. One theme per iteration. |
|
|
199
|
-
| "Metrics look good, ignore qualitative." | Faster ≠ better. Qualitative is primary. |
|
|
200
|
-
| "Reuse the same fresh agent." | It learned. Dispatch anew every time. |
|
|
201
|
-
| "Split every micro-edit into its own iteration." | Over-splitting. One semantic unit = one iteration. |
|
|
202
|
-
| "Rewrite from scratch." | Valid only after 3+ stalled iterations. Before that, it's an escape. |
|
|
203
|
-
|
|
204
|
-
## Environment Constraints
|
|
205
|
-
|
|
206
|
-
### When Fresh Agent Is Unavailable
|
|
207
|
-
|
|
208
|
-
This skill **requires** a fresh agent. Do not apply it when:
|
|
209
|
-
|
|
210
|
-
- Inside a subagent with no dispatch privileges
|
|
211
|
-
- `subagent` / Task tool is disabled
|
|
212
|
-
- No API key or rate-limited
|
|
213
|
-
- Cannot obtain fresh context through any method
|
|
214
|
-
|
|
215
|
-
### Fallback Order
|
|
216
|
-
|
|
217
|
-
1. Ask the user to open a separate session
|
|
218
|
-
2. Report: "empirical evaluation skipped: no fresh agent available"
|
|
219
|
-
3. **Never substitute self re-reading**
|
|
220
|
-
|
|
221
|
-
### Structural-Audit Mode
|
|
222
|
-
|
|
223
|
-
For coherence/clarity checks only (not empirical behavior): mark the request with _"structural-audit mode: check textual coherence only, do not execute."_ This is a supplement — it cannot count toward consecutive-clear convergence.
|
|
224
|
-
|
|
225
|
-
### pi-subagents Troubleshooting
|
|
226
|
-
|
|
227
|
-
| Symptom | Cause | Fix |
|
|
228
|
-
| --------------------------- | ------------------------ | -------------------------- |
|
|
229
|
-
| "No API key found" | Default model resolution | Specify `model` explicitly |
|
|
230
|
-
| Agent reads wrong files | Skill/cwd pollution | Add `skill: false` |
|
|
231
|
-
| Parallel timeout (exit 143) | Cross-provider mixing | Same-provider models only |
|
|
232
|
-
| Subagent hangs | Provider instability | Try different model id |
|