@computesdk/pi-loops 0.0.0-stage → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +124 -2
- package/agents/bold.md +10 -0
- package/agents/council.md +6 -0
- package/agents/judge.md +6 -0
- package/agents/safe.md +10 -0
- package/agents/wild.md +10 -0
- package/extensions/loop.ts +1019 -0
- package/lib/judge.ts +193 -0
- package/lib/measure.ts +38 -0
- package/lib/rubric.ts +159 -0
- package/lib/sheet.ts +33 -0
- package/lib/shot.ts +90 -0
- package/package.json +64 -4
- package/prompts/init.md +6 -0
- package/prompts/loop.md +17 -0
|
@@ -0,0 +1,1019 @@
|
|
|
1
|
+
// Loops as Pi units: one `loop` tool. Your session is the parent; each attempt is a forked Pi session in its own
|
|
2
|
+
// git worktree, driven by a style agent (agents/*.md); the attempt sets its project up and measures itself. The judge is an
|
|
3
|
+
// agent with read/bash that re-measures every candidate where it lives and reads the diffs; the council is a one-shot session.
|
|
4
|
+
// State: .pi/loops/<run>.json (+ the worktrees under .pi/loops/<run>/). Nothing else is written.
|
|
5
|
+
import fs from "node:fs";
|
|
6
|
+
import path from "node:path";
|
|
7
|
+
import { fileURLToPath } from "node:url";
|
|
8
|
+
import { randomBytes } from "node:crypto";
|
|
9
|
+
import { createAgentSession, DefaultResourceLoader, SessionManager, getAgentDir, type AgentSession, type ModelRegistry } from "@earendil-works/pi-coding-agent";
|
|
10
|
+
import { Text } from "@earendil-works/pi-tui";
|
|
11
|
+
import { Type } from "typebox";
|
|
12
|
+
import { pickWinner, panelJudge } from "../lib/judge.ts";
|
|
13
|
+
import { better as betterOf, git, restoreProtected, sh, diffText, errText, type GitRunner } from "../lib/measure.ts";
|
|
14
|
+
import { addRule, fromRules, newRubric, parseJson, rubricText, writeRubric } from "../lib/rubric.ts";
|
|
15
|
+
import { sheet } from "../lib/sheet.ts";
|
|
16
|
+
|
|
17
|
+
const HERE = path.dirname(fileURLToPath(import.meta.url));
|
|
18
|
+
/** Package root: `agents/`, `prompts/` and `lib/` sit one level up from `extensions/`. */
|
|
19
|
+
const ROOT = path.resolve(HERE, "..");
|
|
20
|
+
export const STYLES = ["safe", "bold", "wild"];
|
|
21
|
+
export const DEFAULTS = { attempts: 3, rounds: 1, timeoutMin: 15, tokenBudget: null, repeat: 2, minGain: 0.05, judgeSamples: 3, judgePairs: 2, judgePanel: true, council: true, models: {} };
|
|
22
|
+
const STATE_DIR = path.join(".pi", "loops");
|
|
23
|
+
|
|
24
|
+
/** One agent markdown file: frontmatter (name, model, tools, …) plus body. */
|
|
25
|
+
export type Agent = { role?: string; file?: string; name?: string; model?: string; tools?: string[]; body: string; [key: string]: any };
|
|
26
|
+
/** The model each role runs on: `loops.models` where set, else the attempts and council spread over the pool. */
|
|
27
|
+
export type ModelPlan = { spread: boolean; pool: string[]; safe?: string | null; bold?: string | null; wild?: string | null; judge?: string | null; council?: (string | null)[]; [key: string]: any };
|
|
28
|
+
/** Merged settings: DEFAULTS, then `loops` in settings.json, then ~/.pi/agent/loops.json, then <repo>/.pi/loops.json. */
|
|
29
|
+
export type LoopSettings = { attempts: number; rounds: number; timeoutMin: number; tokenBudget: number | null; repeat: number; minGain: number; judgeSamples: number; judgePairs: number; judgePanel: boolean; council: boolean; models: Record<string, any>; check?: string; bench?: string; metric?: any; protect?: string[]; serve?: string; routes?: string[]; [key: string]: any };
|
|
30
|
+
/** One finished attempt: its worktree, session, cost, and how it ended. */
|
|
31
|
+
export type Attempt = { style: string; label: string; dir?: string; model?: string | null; session?: string | null; tokens?: number; cost?: number; summary?: string; status: string; error?: string; commit?: string; branch?: string; cut?: string; restored?: boolean; bench?: any; review?: any; [key: string]: any };
|
|
32
|
+
/** An attempt that finished and committed: the only kind the judge sees. */
|
|
33
|
+
export type Finished = Attempt & { dir: string; commit: string };
|
|
34
|
+
const finished = (a: Attempt): a is Finished => a.status === "done" && !!a.commit && !!a.dir;
|
|
35
|
+
|
|
36
|
+
/** A model as Pi names it (`provider` + `id`). */
|
|
37
|
+
export type ModelRef = { provider: string; id: string };
|
|
38
|
+
/** A model as Pi hands it to extensions: what `createAgentSession` takes. */
|
|
39
|
+
export type Model = ReturnType<ModelRegistry["getAvailable"]>[number];
|
|
40
|
+
/** The model-registry methods Loops calls. */
|
|
41
|
+
export type ModelRegistryLike = { getAvailable?: () => ModelRef[]; find?: (provider: string, id: string) => ModelRef | undefined };
|
|
42
|
+
/** The session-manager methods Loops calls. */
|
|
43
|
+
export type SessionManagerLike = { getBranch?: () => any[]; getSessionDir?: () => string; getSessionFile?: () => string | undefined };
|
|
44
|
+
/** The subset of Pi's UI the extension calls. */
|
|
45
|
+
export type Ui = {
|
|
46
|
+
notify: (message: string, type?: string) => void;
|
|
47
|
+
select: (title: string, options: string[]) => Promise<string | null>;
|
|
48
|
+
input?: (title: string, placeholder?: string) => Promise<string | null>;
|
|
49
|
+
editor?: (title: string, text: string) => Promise<void>;
|
|
50
|
+
setStatus?: (key: string, text?: string) => void;
|
|
51
|
+
};
|
|
52
|
+
/** The context a model picker needs: no session manager, no UI beyond notify/select/input. */
|
|
53
|
+
export type PickCtx = { cwd: string; model?: ModelRef | null; modelRegistry?: ModelRegistryLike; ui: Ui };
|
|
54
|
+
/** The Pi command/tool context Loops uses (only these fields are touched). */
|
|
55
|
+
export type Ctx = PickCtx & {
|
|
56
|
+
hasUI?: boolean;
|
|
57
|
+
sessionManager?: SessionManagerLike;
|
|
58
|
+
switchSession?: (file: string) => Promise<void>;
|
|
59
|
+
};
|
|
60
|
+
/** The subset of Pi's extension API the extension calls. */
|
|
61
|
+
export type Pi = {
|
|
62
|
+
registerTool?: (tool: ToolDef) => void;
|
|
63
|
+
registerCommand?: (name: string, command: { description?: string; handler: (args: string, ctx: Ctx) => Promise<void> | void }) => void;
|
|
64
|
+
on?: (event: string, handler: (...args: any[]) => any) => void;
|
|
65
|
+
appendEntry?: (type: string, data?: any) => void;
|
|
66
|
+
sendUserMessage?: (message: any, options?: any) => void | Promise<void>;
|
|
67
|
+
getSettings?: () => any;
|
|
68
|
+
[key: string]: any;
|
|
69
|
+
};
|
|
70
|
+
/** A Pi tool registration (only the parts Loops uses). */
|
|
71
|
+
export type ToolDef<P = any> = {
|
|
72
|
+
name: string;
|
|
73
|
+
label: string;
|
|
74
|
+
description: string;
|
|
75
|
+
parameters: any;
|
|
76
|
+
outputSchema?: any;
|
|
77
|
+
execute: (id: string, params: P, signal: AbortSignal, onUpdate: (update: any) => void, ctx: Ctx) => Promise<any>;
|
|
78
|
+
renderCall?: (args: any, theme: any) => any;
|
|
79
|
+
renderResult?: (result: any, options: { expanded: boolean; isPartial: boolean }, theme: any) => any;
|
|
80
|
+
};
|
|
81
|
+
/** A candidate for the judge: a checked-out directory at a commit. */
|
|
82
|
+
export type Candidate = { label: string; dir: string; commit: string; fresh?: boolean };
|
|
83
|
+
/** One round of a run. */
|
|
84
|
+
export type Round = { n: number; from?: string; status: string; attempts: Attempt[]; judge?: any; judgeSession?: string | null; judgeTokens?: number; judgeCost?: number; startedAt?: number; endedAt?: number; sheet?: string | null };
|
|
85
|
+
/** A run's persisted state (`.pi/loops/<run>.json`). */
|
|
86
|
+
export type State = {
|
|
87
|
+
v?: number;
|
|
88
|
+
id: string;
|
|
89
|
+
goal: string;
|
|
90
|
+
kind: string;
|
|
91
|
+
status: string;
|
|
92
|
+
baseCommit: string;
|
|
93
|
+
metric?: any;
|
|
94
|
+
rules: string[];
|
|
95
|
+
protect?: string[];
|
|
96
|
+
build?: string;
|
|
97
|
+
shots?: string[];
|
|
98
|
+
directions?: Record<string, string>;
|
|
99
|
+
best: { label?: string | null; commit: string; session?: string | null; branch?: string | null; dir?: string | null; score?: number | null; reason?: string | null; round?: number };
|
|
100
|
+
winner?: any;
|
|
101
|
+
rounds: Round[];
|
|
102
|
+
rubric?: any;
|
|
103
|
+
models?: any;
|
|
104
|
+
editedBaseline?: boolean;
|
|
105
|
+
openNext?: string | null;
|
|
106
|
+
parentSession?: string | null;
|
|
107
|
+
check?: string;
|
|
108
|
+
bench?: string;
|
|
109
|
+
[key: string]: any;
|
|
110
|
+
};
|
|
111
|
+
/** The parameters of the `loop` tool, and of a brief. */
|
|
112
|
+
export type LoopParamsIn = {
|
|
113
|
+
goal?: string;
|
|
114
|
+
check?: string;
|
|
115
|
+
bench?: string;
|
|
116
|
+
metric?: any;
|
|
117
|
+
protect?: string[];
|
|
118
|
+
build?: string;
|
|
119
|
+
shots?: string[];
|
|
120
|
+
rounds?: number;
|
|
121
|
+
from?: string;
|
|
122
|
+
rules?: string[];
|
|
123
|
+
routes?: string[];
|
|
124
|
+
directions?: Record<string, string>;
|
|
125
|
+
confirm?: boolean;
|
|
126
|
+
wait?: boolean;
|
|
127
|
+
};
|
|
128
|
+
/** A brief: the tool params plus whatever the project's .pi/loops.json contributes. */
|
|
129
|
+
export type SpecView = LoopParamsIn & { serve?: string | null };
|
|
130
|
+
/** What the build wrote, or why it could not. */
|
|
131
|
+
export type ShotFiles = { error?: string; files?: string[]; have: string[] };
|
|
132
|
+
/** One attempt's live progress, as the tool's details carry it. */
|
|
133
|
+
export type AttemptDetail = { style: string; status: string; tool: string | null; tools: number; tokens: number; text: string };
|
|
134
|
+
/** The tool's structured details: what the renderer draws while a round runs. */
|
|
135
|
+
export type Details = { run: string | null; kind: string; round: number | null; judge?: any; attempts: AttemptDetail[]; phase: string; spec?: any; [key: string]: any };
|
|
136
|
+
|
|
137
|
+
/* ---------- agents (Pi agent markdown) ---------- */
|
|
138
|
+
export function parseAgent(md: string): Agent {
|
|
139
|
+
const m = /^---\n([\s\S]*?)\n---\n?([\s\S]*)$/.exec(md);
|
|
140
|
+
const fm: Record<string, string> = {};
|
|
141
|
+
if (m) for (const line of m[1].split("\n")) { const i = line.indexOf(":"); if (i > 0) fm[line.slice(0, i).trim()] = line.slice(i + 1).trim(); }
|
|
142
|
+
const tools = fm.tools === undefined ? undefined : fm.tools.split(",").map((s) => s.trim()).filter(Boolean);
|
|
143
|
+
return { ...fm, tools, body: (m ? m[2] : md).trim() };
|
|
144
|
+
}
|
|
145
|
+
/** agents/<role>.md from the project (.pi/agents), the user (~/.pi/agent/agents), then this package. */
|
|
146
|
+
export function loadAgent(role: string, { cwd, agentDir = getAgentDir() }: { cwd?: string; agentDir?: string } = {}): Agent {
|
|
147
|
+
for (const dir of [cwd && path.join(cwd, ".pi", "agents"), path.join(agentDir, "agents"), path.join(ROOT, "agents")]) {
|
|
148
|
+
const f = dir && path.join(dir, `${role}.md`);
|
|
149
|
+
if (f && fs.existsSync(f)) return { role, file: f, ...parseAgent(fs.readFileSync(f, "utf8")) };
|
|
150
|
+
}
|
|
151
|
+
throw new Error(`no agent markdown for ${role}`);
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/* ---------- config: ~/.pi/agent/loops.json (your defaults) + <repo>/.pi/loops.json (this repo; wins per key) ---------- */
|
|
155
|
+
export const CONFIG_FILE = "loops.json";
|
|
156
|
+
export const configFiles = (cwd: string, agentDir: string = getAgentDir()) => ({ user: path.join(agentDir, CONFIG_FILE), project: path.join(cwd, ".pi", CONFIG_FILE) });
|
|
157
|
+
function readJson(f: string): Record<string, any> { if (!fs.existsSync(f)) return {}; const j = JSON.parse(fs.readFileSync(f, "utf8")); if (!j || typeof j !== "object" || Array.isArray(j)) throw new Error(`${f} is not a JSON object`); return j; }
|
|
158
|
+
export function loopSettings(pi: Pi | null, cwd: string, agentDir: string = getAgentDir()): LoopSettings {
|
|
159
|
+
let s: Record<string, any> = {};
|
|
160
|
+
const merge = (j: Record<string, any>) => { s = { ...s, ...j, models: { ...(s.models ?? {}), ...(j.models ?? {}) } }; };
|
|
161
|
+
// legacy, lowest precedence: `loops` in Pi's settings.json (user, then project), or whatever Pi hands us
|
|
162
|
+
for (const f of [path.join(agentDir, "settings.json"), path.join(cwd, ".pi", "settings.json")]) { try { merge(readJson(f).loops ?? {}); } catch {} }
|
|
163
|
+
try { merge(pi?.getSettings?.()?.loops ?? {}); } catch {}
|
|
164
|
+
const { user, project } = configFiles(cwd, agentDir);
|
|
165
|
+
for (const f of [user, project]) { let j = {}; try { j = readJson(f); } catch {} merge(j); }
|
|
166
|
+
return { ...DEFAULTS, ...s, models: { ...DEFAULTS.models, ...(s.models ?? {}) } };
|
|
167
|
+
}
|
|
168
|
+
/** Merge `patch` into <repo>/.pi/loops.json (models merged one level deeper). Throws when the file exists but isn't a JSON object. */
|
|
169
|
+
export function saveConfig(cwd: string, patch: Record<string, any>): Record<string, any> {
|
|
170
|
+
const f = configFiles(cwd).project;
|
|
171
|
+
const cur = readJson(f);
|
|
172
|
+
const next: Record<string, any> = { ...cur, ...patch, ...(patch.models ? { models: { ...(cur.models ?? {}), ...patch.models } } : {}) };
|
|
173
|
+
for (const k of Object.keys(next)) if (next[k] === null) delete next[k];
|
|
174
|
+
fs.mkdirSync(path.dirname(f), { recursive: true });
|
|
175
|
+
fs.writeFileSync(f, JSON.stringify(next, null, 2) + "\n");
|
|
176
|
+
return next;
|
|
177
|
+
}
|
|
178
|
+
export function parseBudget(v: unknown): number | null {
|
|
179
|
+
const m = String(v ?? "").trim().toLowerCase().match(/^(\d+(?:\.\d+)?)\s*([km])?$/);
|
|
180
|
+
if (!m) return null;
|
|
181
|
+
return Math.round(Number(m[1]) * (m[2] === "m" ? 1_000_000 : m[2] === "k" ? 1_000 : 1));
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
const modelSpec = (m: ModelRef) => `${m.provider}/${m.id}`;
|
|
185
|
+
const globRe = (pat: string) => new RegExp("^" + pat.split("*").map((x) => x.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join(".*") + "$");
|
|
186
|
+
/** Which model each role runs on. Explicit `loops.models` wins; otherwise the attempts and council members are spread over the pool
|
|
187
|
+
* (`loops.models.pool`, else Pi's `enabledModels`) so a round compares models too; with no pool everything runs on the current model. */
|
|
188
|
+
export function resolveModels({ settings, available = [], current = null, enabledModels = [] }: { settings: { models?: Record<string, any> }; available?: ModelRef[]; current?: string | null; enabledModels?: string[] }): ModelPlan {
|
|
189
|
+
const m = settings.models ?? {};
|
|
190
|
+
const specs = available.map(modelSpec);
|
|
191
|
+
const patterns: string[] = Array.isArray(m.pool) && m.pool.length ? m.pool : enabledModels;
|
|
192
|
+
let pool = patterns.flatMap((p) => specs.filter((sp) => sp === p || sp.endsWith("/" + p) || (p.includes("*") && globRe(p).test(sp))));
|
|
193
|
+
pool = [...new Set(pool)];
|
|
194
|
+
if (current && pool.includes(current)) pool = [current, ...pool.filter((x) => x !== current)];
|
|
195
|
+
const spread = pool.length >= 2;
|
|
196
|
+
const out: ModelPlan = { spread, pool };
|
|
197
|
+
STYLES.forEach((style, i) => { out[style] = m[style] ?? (pool.length ? pool[i % pool.length] : null); });
|
|
198
|
+
out.judge = m.judge ?? null;
|
|
199
|
+
const council = Array.isArray(m.council) ? m.council : m.council ? [m.council, m.council] : pool.length ? pool.slice(0, 3) : [null, null];
|
|
200
|
+
out.council = council.length >= 2 ? council : [...council, ...council].slice(0, 2);
|
|
201
|
+
return out;
|
|
202
|
+
}
|
|
203
|
+
export function modelsOf(ctx: PickCtx, settings: LoopSettings, pi: Pi | null): ModelPlan {
|
|
204
|
+
let enabled: string[] = [];
|
|
205
|
+
try { enabled = pi?.getSettings?.()?.enabledModels ?? []; } catch {}
|
|
206
|
+
if (!enabled.length) { try { enabled = JSON.parse(fs.readFileSync(path.join(getAgentDir(), "settings.json"), "utf8")).enabledModels ?? []; } catch {} }
|
|
207
|
+
const available = ctx.modelRegistry?.getAvailable?.() ?? [];
|
|
208
|
+
return resolveModels({ settings, available, current: ctx.model ? modelSpec(ctx.model) : null, enabledModels: enabled });
|
|
209
|
+
}
|
|
210
|
+
export const shortModel = (spec?: string | null) => (spec ? spec.split("/").pop() ?? spec : "current");
|
|
211
|
+
function pickModel(ctx: Ctx, spec?: string | null): Model {
|
|
212
|
+
const reg = ctx.modelRegistry;
|
|
213
|
+
// At runtime these are full Pi models; the registry is typed loosely so tests can pass { provider, id } stubs.
|
|
214
|
+
if (spec) {
|
|
215
|
+
const [provider, ...rest] = spec.split("/");
|
|
216
|
+
const m = (rest.length ? reg?.find?.(provider, rest.join("/")) : reg?.getAvailable?.().find((x) => x.id === spec)) as Model | undefined;
|
|
217
|
+
if (m) return m;
|
|
218
|
+
throw new Error(`model "${spec}" is not available (check /login, models.json or the loops.models setting)`);
|
|
219
|
+
}
|
|
220
|
+
const m = (ctx.model ?? reg?.getAvailable?.()[0]) as Model | undefined;
|
|
221
|
+
if (!m) throw new Error("no model available: log in with /login first");
|
|
222
|
+
return m;
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
/* ---------- state: .pi/loops/<run>.json ---------- */
|
|
226
|
+
const stateFile = (cwd: string, id: string) => path.join(cwd, STATE_DIR, `${id}.json`);
|
|
227
|
+
export function saveState(cwd: string, state: State) {
|
|
228
|
+
const f = stateFile(cwd, state.id);
|
|
229
|
+
fs.mkdirSync(path.dirname(f), { recursive: true });
|
|
230
|
+
const gi = path.join(cwd, ".gitignore");
|
|
231
|
+
const cur = fs.existsSync(gi) ? fs.readFileSync(gi, "utf8") : "";
|
|
232
|
+
if (!cur.split("\n").some((l) => /^\/?\.pi\/loops\/?$/.test(l.trim()))) fs.writeFileSync(gi, (cur && !cur.endsWith("\n") ? cur + "\n" : cur) + ".pi/loops\n");
|
|
233
|
+
fs.writeFileSync(f + ".tmp", JSON.stringify(state, null, 2));
|
|
234
|
+
fs.renameSync(f + ".tmp", f);
|
|
235
|
+
return f;
|
|
236
|
+
}
|
|
237
|
+
/** Screenshots and the shots dir stay out of every worktree's commits: the repo's info/exclude applies to all of them. */
|
|
238
|
+
export function excludeShots(cwd: string, shots: string[] | null | undefined, g: GitRunner = git) {
|
|
239
|
+
const want = [".loops-shots/", ...(shots ?? [])].map((f) => f.replace(/^\.\//, ""));
|
|
240
|
+
const common = path.resolve(cwd, g(cwd, "rev-parse", "--git-common-dir"));
|
|
241
|
+
const f = path.join(common, "info", "exclude");
|
|
242
|
+
const cur = fs.existsSync(f) ? fs.readFileSync(f, "utf8") : "";
|
|
243
|
+
const missing = want.filter((w) => !cur.split("\n").some((l) => l.trim() === w));
|
|
244
|
+
if (!missing.length) return;
|
|
245
|
+
fs.mkdirSync(path.dirname(f), { recursive: true });
|
|
246
|
+
fs.writeFileSync(f, (cur && !cur.endsWith("\n") ? cur + "\n" : cur) + missing.join("\n") + "\n");
|
|
247
|
+
}
|
|
248
|
+
/** The runs this session started (newest first), from its own transcript entries — not the repo's whole .pi/loops/. */
|
|
249
|
+
export function sessionRuns(ctx: { sessionManager?: SessionManagerLike }): string[] {
|
|
250
|
+
try { return (ctx.sessionManager?.getBranch?.() ?? []).filter((e) => e.type === "custom" && e.customType === "loops-run" && e.data?.id).map((e) => e.data.id).reverse().filter((id, i, a) => a.indexOf(id) === i); } catch { return []; }
|
|
251
|
+
}
|
|
252
|
+
export function loadState(cwd: string, from?: string | null): State {
|
|
253
|
+
const dir = path.join(cwd, STATE_DIR);
|
|
254
|
+
let id = from;
|
|
255
|
+
if (!id || id === "last") {
|
|
256
|
+
const runs = fs.existsSync(dir) ? fs.readdirSync(dir).filter((f) => f.endsWith(".json")).sort((a, b) => fs.statSync(path.join(dir, a)).mtimeMs - fs.statSync(path.join(dir, b)).mtimeMs || a.localeCompare(b)) : [];
|
|
257
|
+
if (!runs.length) throw new Error(`no runs in ${STATE_DIR}/ to resume`);
|
|
258
|
+
id = runs[runs.length - 1].replace(/\.json$/, "");
|
|
259
|
+
}
|
|
260
|
+
const f = stateFile(cwd, id.replace(/\.json$/, ""));
|
|
261
|
+
if (!fs.existsSync(f)) throw new Error(`no run ${id} in ${STATE_DIR}/`);
|
|
262
|
+
return JSON.parse(fs.readFileSync(f, "utf8"));
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
/* ---------- prompts ---------- */
|
|
266
|
+
export function attemptPrompt(state: State, style: string, n: number, settings: Partial<LoopSettings> = DEFAULTS) {
|
|
267
|
+
const rules = state.rules.length ? `Rules:\n${state.rules.map((r) => "- " + r).join("\n")}\n` : "";
|
|
268
|
+
const how = [state.check && `Check (must pass): ${state.check}`, state.bench && `Benchmark: ${state.bench} (prints ${state.metric?.unit ?? "ms"}; ${state.metric?.better ?? "lower"} is better)`,
|
|
269
|
+
state.protect?.length && `Protected files, do not edit: ${state.protect.join(", ")}`, state.build && `Build: ${state.build}${state.shots?.length ? ` (must write ${state.shots.join(", ")})` : ""}`].filter(Boolean).join("\n");
|
|
270
|
+
const best = state.best;
|
|
271
|
+
const lead = n === 1 ? `Round 1 of a hill-climbing loop. Goal: ${state.goal}`
|
|
272
|
+
: `Round ${n}. The version you are looking at was judged the best so far${best.score != null ? ` (${best.score} ${state.metric?.unit ?? ""})` : ""}${best.reason ? `. Feedback: "${best.reason}"` : ""}. Improve on it further.\nGoal: ${state.goal}`;
|
|
273
|
+
return `${lead}\n${rules}${how}\nBudget: about ${settings.tokenBudget ? `${Math.round(settings.tokenBudget / 1000)}k tokens` : "unlimited tokens"} and ${settings.timeoutMin ?? "unlimited"} minutes; whatever is committed when it runs out is judged, so land a working change early and refine after. Your direction: ${style.toUpperCase()}${state.directions?.[style] ? ` — ${state.directions[style]}` : ""}. Work in the current directory only. It is a fresh checkout: if the project needs setting up (dependencies installed, the way the repo's docs say), do that first. Before you finish, run the check${state.bench ? " and the benchmark" : state.build ? " and the build" : ""} yourself${state.bench ? " and put the benchmark number in your summary" : ""}. Leave no scratch or helper files behind unless the goal needs them (delete temporary checks before you finish). When finished, reply with exactly ONE line summarising what you changed.`;
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
/** session.prompt takes text + images separately: number each image in place so the text still says which candidate it belongs to. */
|
|
277
|
+
export function flattenParts(content: string | any[]): { text: string; images: any[] } {
|
|
278
|
+
const parts = typeof content === "string" ? [{ type: "text", text: content }] : content;
|
|
279
|
+
const images: any[] = [];
|
|
280
|
+
const text = parts.map((p) => (p.type === "image" ? (images.push(p), `[image ${images.length}]`) : p.text)).join("\n");
|
|
281
|
+
return { text, images };
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
/* ---------- rounds running in the background, so the parent can steer them while they run ---------- */
|
|
285
|
+
export const RUNS = new Map<string, any>();
|
|
286
|
+
export function liveRun(id?: string | null) {
|
|
287
|
+
if (id && id !== "last") { const r = RUNS.get(id); if (!r) throw new Error(`${id} is not running${RUNS.size ? ` (running: ${[...RUNS.keys()].join(", ")})` : ""}`); return r; }
|
|
288
|
+
const r = [...RUNS.values()].pop();
|
|
289
|
+
if (!r) throw new Error("no loop is running");
|
|
290
|
+
return r;
|
|
291
|
+
}
|
|
292
|
+
export function findAttempt(live: { attempts: Map<string, any> }, name: unknown): string | null {
|
|
293
|
+
const want = String(name).toLowerCase();
|
|
294
|
+
return [...live.attempts.keys()].find((k) => k.toLowerCase() === want || k.toLowerCase().endsWith("-" + want)) ?? null;
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
/* ---------- live child sessions, so a session_shutdown mid-round can stop them ---------- */
|
|
298
|
+
const active = new Set<any>();
|
|
299
|
+
let shuttingDown = false;
|
|
300
|
+
const track = (session: AgentSession) => { if (shuttingDown) { try { session.dispose(); } catch {} throw new Error("Pi is shutting down"); } active.add(session); return session; };
|
|
301
|
+
const release = (session: AgentSession) => { active.delete(session); try { session.dispose(); } catch {} };
|
|
302
|
+
export async function stopAll() { shuttingDown = true; const all = [...active]; active.clear(); await Promise.allSettled(all.map((s) => s.abort?.().catch(() => {}))); for (const s of all) { try { s.dispose(); } catch {} } return all.length; }
|
|
303
|
+
|
|
304
|
+
/* ---------- one-shot agent sessions (visual judge panel / council): no tools, Pi auth + session header ---------- */
|
|
305
|
+
async function oneShot({ role, model, content, cwd, ctx, settings, signal }: { role: string; model?: string | null; content: any; cwd: string; ctx: Ctx; settings: LoopSettings; signal?: AbortSignal }) {
|
|
306
|
+
const agentDir = getAgentDir();
|
|
307
|
+
const agent = loadAgent(role, { cwd, agentDir });
|
|
308
|
+
const loader = new DefaultResourceLoader({ cwd, agentDir, noExtensions: true, noSkills: true, noPromptTemplates: true, noThemes: true, noContextFiles: true, systemPrompt: agent.body });
|
|
309
|
+
await loader.reload();
|
|
310
|
+
const { session: created } = await createAgentSession({ cwd, agentDir, model: pickModel(ctx, model ?? settings.models[role] ?? agent.model), noTools: "all", resourceLoader: loader, sessionManager: SessionManager.inMemory(cwd) });
|
|
311
|
+
const session = track(created);
|
|
312
|
+
const { text, images } = flattenParts(content);
|
|
313
|
+
const onAbort = () => session.abort().catch(() => {});
|
|
314
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
315
|
+
try {
|
|
316
|
+
await session.prompt(text, { images, expandPromptTemplates: false });
|
|
317
|
+
let last = session.messages.filter((m) => m.role === "assistant").pop();
|
|
318
|
+
if (last?.stopReason === "error") throw new Error(last.errorMessage || `${role} errored`);
|
|
319
|
+
if (!(session.getLastAssistantText() ?? "").trim()) {
|
|
320
|
+
await session.prompt("You answered with nothing. Answer now with the JSON object asked for and nothing else.", { expandPromptTemplates: false });
|
|
321
|
+
last = session.messages.filter((m) => m.role === "assistant").pop();
|
|
322
|
+
if (last?.stopReason === "error") throw new Error(last.errorMessage || `${role} errored`);
|
|
323
|
+
}
|
|
324
|
+
return parseJson(session.getLastAssistantText());
|
|
325
|
+
} finally { signal?.removeEventListener("abort", onAbort); release(session); }
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
/* ---------- an attempt: forked session + worktree + style agent ---------- */
|
|
329
|
+
async function runAttempt({ state, style, n, dir, ctx, settings, model, signal, onProgress, live }: { state: State; style: string; n: number; dir: string; ctx: Ctx; settings: LoopSettings; model?: string | null; signal?: AbortSignal; onProgress: (progress: any) => void; live?: any }): Promise<Attempt> {
|
|
330
|
+
const agentDir = getAgentDir();
|
|
331
|
+
const agent = loadAgent(style, { cwd: ctx.cwd, agentDir });
|
|
332
|
+
const sessionDir = ctx.sessionManager?.getSessionDir?.();
|
|
333
|
+
const src = state.best.session ?? ctx.sessionManager?.getSessionFile?.();
|
|
334
|
+
const sessionManager = src && fs.existsSync(src) ? SessionManager.forkFrom(src, dir, sessionDir) : SessionManager.create(dir, sessionDir);
|
|
335
|
+
const loader = new DefaultResourceLoader({ cwd: dir, agentDir, noExtensions: true, noPromptTemplates: true, noThemes: true, appendSystemPrompt: [agent.body] });
|
|
336
|
+
await loader.reload();
|
|
337
|
+
const picked = pickModel(ctx, model ?? settings.models[style] ?? agent.model);
|
|
338
|
+
const { session: created } = await createAgentSession({ cwd: dir, agentDir, model: picked, tools: agent.tools?.length ? agent.tools : undefined, resourceLoader: loader, sessionManager });
|
|
339
|
+
const session = track(created);
|
|
340
|
+
live?.attempts.set(`R${n}-${style}`, session);
|
|
341
|
+
const p: { style: string; status: string; tool: string | null; tools: number; tokens: number; cost: number; text: string } = { style, status: "running", tool: null, tools: 0, tokens: 0, cost: 0, text: "" };
|
|
342
|
+
let ended: string | null = null;
|
|
343
|
+
const unsub = session.subscribe((e) => {
|
|
344
|
+
if (e.type === "tool_execution_start") { p.tool = e.toolName; p.tools++; onProgress(p); }
|
|
345
|
+
else if (e.type === "tool_execution_end") { p.tool = null; onProgress(p); }
|
|
346
|
+
else if (e.type === "message_end" && e.message?.role === "assistant") {
|
|
347
|
+
p.tokens += e.message.usage?.totalTokens || 0;
|
|
348
|
+
p.cost += typeof e.message.usage?.cost === "number" ? e.message.usage.cost : e.message.usage?.cost?.total || 0;
|
|
349
|
+
const t = (e.message.content || []).filter((c) => c.type === "text").map((c) => c.text).join("").trim();
|
|
350
|
+
if (t) p.text = t.slice(-200);
|
|
351
|
+
onProgress(p);
|
|
352
|
+
if (settings.tokenBudget && p.tokens > settings.tokenBudget && !ended) { ended = "budget"; session.abort().catch(() => {}); }
|
|
353
|
+
}
|
|
354
|
+
});
|
|
355
|
+
const timer = settings.timeoutMin ? setTimeout(() => { ended ??= "timeout"; session.abort().catch(() => {}); }, settings.timeoutMin * 60_000) : null;
|
|
356
|
+
const onAbort = () => { ended ??= "aborted"; session.abort().catch(() => {}); };
|
|
357
|
+
if (signal?.aborted) onAbort(); else signal?.addEventListener("abort", onAbort, { once: true });
|
|
358
|
+
let err = "";
|
|
359
|
+
try { await session.prompt(attemptPrompt(state, style, n, settings), { expandPromptTemplates: false }); } catch (e) { err = errText(e); }
|
|
360
|
+
finally { if (timer) clearTimeout(timer); signal?.removeEventListener("abort", onAbort); unsub(); }
|
|
361
|
+
const last = session.messages.filter((m) => m.role === "assistant").pop();
|
|
362
|
+
if (!ended && last?.stopReason === "error") err ||= last.errorMessage || "agent error";
|
|
363
|
+
const out = { style, label: `R${n}-${style}`, dir, model: modelSpec(picked), session: session.sessionFile ?? null, tokens: p.tokens, cost: +p.cost.toFixed(4), summary: (session.getLastAssistantText() ?? "").trim().split("\n").pop()?.slice(0, 200) ?? "" };
|
|
364
|
+
release(session);
|
|
365
|
+
live?.attempts.delete(`R${n}-${style}`);
|
|
366
|
+
if (ended === "aborted") return { ...out, status: ended, error: err };
|
|
367
|
+
if (!ended && err) return { ...out, status: "error", error: err.slice(0, 300) };
|
|
368
|
+
try { git(dir, "add", "-A"); if (state.shots?.length) { try { git(dir, "reset", "-q", "--", ...state.shots); } catch {} } git(dir, "commit", "-q", "-m", `${out.label}: ${out.summary || "attempt"}`); } catch {}
|
|
369
|
+
const commit = git(dir, "rev-parse", "HEAD");
|
|
370
|
+
if (commit === state.best.commit) return { ...out, status: ended ?? "unchanged", commit, error: err };
|
|
371
|
+
return { ...out, status: "done", commit, ...(ended ? { cut: ended } : {}) };
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
/* ---------- the judge: an agent with read/bash re-measures every candidate where it lives and reads the diffs ---------- */
|
|
375
|
+
export function verifyPrompt(state: State, n: number, cands: Candidate[], { repeat = 2 }: { repeat?: number } = {}) {
|
|
376
|
+
const unit = state.metric?.unit || "ms", better = state.metric?.better ?? "lower";
|
|
377
|
+
const rules = state.rules.length ? `Rules every attempt must follow:\n${state.rules.map((r) => "- " + r).join("\n")}\n` : "";
|
|
378
|
+
const rubric = state.rubric ? `Rubric (the council wrote it; a change that fails a criterion is not ok):\n${rubricText(state.rubric)}\n` : "";
|
|
379
|
+
const list = cands.map((c) => `- ${c.label}: ${c.dir} at commit ${c.commit.slice(0, 7)}${c.label === "CURRENT" ? (c.fresh ? " — the best so far, a fresh checkout: set it up the way the attempts did before measuring" : n === 1 ? " — the best so far; the user's own checkout, already set up. Do not modify it." : " — the best so far (the previous winner's worktree, already set up)") : " — set up by the attempt itself; if the check or benchmark cannot run only because of missing setup, set it up (install dependencies); never edit source files"}`).join("\n");
|
|
380
|
+
const protect = state.protect?.length ? ` (${state.protect.join(", ")} were restored to the base version before judging; ignore them)` : "";
|
|
381
|
+
return `You judge round ${n} of a hill-climbing loop. Goal: ${state.goal}
|
|
382
|
+
${rules}${rubric}Check (must exit 0): ${state.check ?? "none"}
|
|
383
|
+
Benchmark: ${state.bench} — prints ${unit}; ${better} is better. Run it ${repeat} times per candidate and keep the best sample.
|
|
384
|
+
|
|
385
|
+
Candidates. Each is a checked-out directory; measure every one yourself, in that directory, on this machine, under the same conditions:
|
|
386
|
+
${list}
|
|
387
|
+
|
|
388
|
+
For each attempt (not CURRENT) read its change too: \`git diff ${cands[0].commit.slice(0, 7)} HEAD\` in its directory${protect}. Veto rule breaks and cheating: special-casing benchmark inputs, caching results across calls, skipping work the goal requires, weakening the check or the benchmark.
|
|
389
|
+
|
|
390
|
+
Do not edit any file anywhere. When done, reply with exactly one JSON object and nothing else:
|
|
391
|
+
{"CURRENT":{"value":<best sample, number or null>,"note":"<=15 words"},"<attempt label>":{"check":true|false,"value":<best sample, number or null>,"ok":true|false,"note":"<=20 words: what the change does, or why it failed or is vetoed"}}`;
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
async function judgeRound({ state, n, cands, cwd, ctx, settings, model, signal, onProgress }: { state: State; n: number; cands: Candidate[]; cwd: string; ctx: Ctx; settings: LoopSettings; model?: string | null; signal?: AbortSignal; onProgress: (progress: any) => void }) {
|
|
395
|
+
return agentJSON({ role: "judge", model, prompt: verifyPrompt(state, n, cands, settings), ok: (out: any) => "CURRENT" in out, cwd, ctx, settings, signal, onProgress });
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
async function agentJSON({ role, model, prompt, ok, cwd, ctx, settings, signal, onProgress }: { role: string; model?: string | null; prompt: string; ok: (out: any) => boolean; cwd: string; ctx: Ctx; settings: LoopSettings; signal?: AbortSignal; onProgress: (progress: any) => void }) {
|
|
399
|
+
const agentDir = getAgentDir();
|
|
400
|
+
const agent = loadAgent(role, { cwd, agentDir });
|
|
401
|
+
const loader = new DefaultResourceLoader({ cwd, agentDir, noExtensions: true, noPromptTemplates: true, noThemes: true, appendSystemPrompt: [agent.body] });
|
|
402
|
+
await loader.reload();
|
|
403
|
+
const { session: created } = await createAgentSession({ cwd, agentDir, model: pickModel(ctx, model ?? settings.models[role] ?? agent.model), tools: agent.tools?.length ? agent.tools : undefined, resourceLoader: loader, sessionManager: SessionManager.create(cwd, ctx.sessionManager?.getSessionDir?.()) });
|
|
404
|
+
const session = track(created);
|
|
405
|
+
const p: { tool: string | null; tools: number; tokens: number; cost: number } = { tool: null, tools: 0, tokens: 0, cost: 0 };
|
|
406
|
+
const unsub = session.subscribe((e) => {
|
|
407
|
+
if (e.type === "tool_execution_start") { p.tool = e.toolName; p.tools++; onProgress(p); }
|
|
408
|
+
else if (e.type === "tool_execution_end") { p.tool = null; onProgress(p); }
|
|
409
|
+
else if (e.type === "message_end" && e.message?.role === "assistant") { p.tokens += e.message.usage?.totalTokens || 0; p.cost += typeof e.message.usage?.cost === "number" ? e.message.usage.cost : e.message.usage?.cost?.total || 0; onProgress(p); }
|
|
410
|
+
});
|
|
411
|
+
const onAbort = () => session.abort().catch(() => {});
|
|
412
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
413
|
+
try {
|
|
414
|
+
for (let i = 0; i < 2; i++) {
|
|
415
|
+
await session.prompt(prompt, { expandPromptTemplates: false });
|
|
416
|
+
const last = session.messages.filter((m) => m.role === "assistant").pop();
|
|
417
|
+
if (last?.stopReason === "error") throw new Error(last.errorMessage || `${role} errored`);
|
|
418
|
+
try { const out = parseJson(session.getLastAssistantText()); if (out && typeof out === "object" && ok(out)) return { out, session: session.sessionFile ?? null, tokens: p.tokens, cost: +p.cost.toFixed(4) }; } catch {}
|
|
419
|
+
prompt = "Reply with only the JSON object described, nothing else.";
|
|
420
|
+
}
|
|
421
|
+
throw new Error(`${role} never answered in shape`);
|
|
422
|
+
} finally { signal?.removeEventListener("abort", onAbort); unsub(); release(session); }
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
/* ---------- the tool ---------- */
|
|
426
|
+
const Metric = Type.Object({ key: Type.Optional(Type.String({ description: "JSON key in the bench output; default ms" })), unit: Type.Optional(Type.String()), better: Type.Optional(Type.Union([Type.Literal("lower"), Type.Literal("higher")])) });
|
|
427
|
+
const RoundOut = Type.Object({ n: Type.Number(), pick: Type.Union([Type.String(), Type.Null()]), why: Type.Union([Type.String(), Type.Null()]), scores: Type.Record(Type.String(), Type.Union([Type.Number(), Type.Null()])), reasons: Type.Record(Type.String(), Type.String()), sheet: Type.Union([Type.String(), Type.Null()]),
|
|
428
|
+
attempts: Type.Array(Type.Object({ label: Type.String(), style: Type.String(), status: Type.String(), model: Type.Union([Type.String(), Type.Null()]), commit: Type.Union([Type.String(), Type.Null()]), session: Type.Union([Type.String(), Type.Null()]), summary: Type.String(), cost: Type.Number() })) });
|
|
429
|
+
export const LoopResult = Type.Object({
|
|
430
|
+
run: Type.String(), status: Type.String(), kind: Type.String(), goal: Type.String(), baseCommit: Type.String(),
|
|
431
|
+
metric: Type.Object({ key: Type.Union([Type.String(), Type.Null()]), unit: Type.Union([Type.String(), Type.Null()]), better: Type.Union([Type.String(), Type.Null()]) }),
|
|
432
|
+
best: Type.Object({ label: Type.Union([Type.String(), Type.Null()]), commit: Type.Union([Type.String(), Type.Null()]), score: Type.Union([Type.Number(), Type.Null()]), branch: Type.Union([Type.String(), Type.Null()]) }),
|
|
433
|
+
rounds: Type.Array(RoundOut),
|
|
434
|
+
});
|
|
435
|
+
export function structuredResult(state: Pick<State, "id" | "status" | "kind" | "goal" | "baseCommit" | "metric" | "best">, rounds: Round[]) {
|
|
436
|
+
const n = (v: unknown): number | null => (typeof v === "number" && Number.isFinite(v) ? v : null);
|
|
437
|
+
return {
|
|
438
|
+
run: state.id, status: state.status, kind: state.kind, goal: state.goal, baseCommit: state.baseCommit,
|
|
439
|
+
metric: { key: state.metric?.key ?? null, unit: state.metric?.unit ?? null, better: state.metric?.better ?? null },
|
|
440
|
+
best: { label: state.best?.label ?? null, commit: state.best?.commit ?? null, score: n(state.best?.score), branch: state.best?.branch ?? null },
|
|
441
|
+
rounds: rounds.map((r) => ({ n: r.n, pick: r.judge?.pick ?? null, why: r.judge?.why ?? null, scores: Object.fromEntries(Object.entries(r.judge?.scores ?? {}).map(([k, v]) => [k, n(v)])), reasons: Object.fromEntries(Object.entries(r.judge?.reasons ?? {}).map(([k, v]) => [k, String(v)])), sheet: r.sheet ?? null,
|
|
442
|
+
attempts: r.attempts.map((a) => ({ label: a.label, style: a.style, status: a.status, model: a.model ?? null, commit: a.commit ?? null, session: a.session ?? null, summary: a.summary ?? "", cost: n(a.cost) ?? 0 })) })),
|
|
443
|
+
};
|
|
444
|
+
}
|
|
445
|
+
export const LoopParams = Type.Object({
|
|
446
|
+
confirm: Type.Optional(Type.Boolean({ description: "Show the spec and ask approve / change / discuss before running (default true in the TUI; never without a UI)" })),
|
|
447
|
+
wait: Type.Optional(Type.Boolean({ description: "Block until the round is done (always the case without a UI, e.g. pi -p); default in the TUI: return once the round has started and deliver the card as a message" })),
|
|
448
|
+
goal: Type.Optional(Type.String({ description: "What to improve. Required unless `from` is set." })),
|
|
449
|
+
check: Type.Optional(Type.String({ description: "Command that must pass (tests)" })),
|
|
450
|
+
bench: Type.Optional(Type.String({ description: "Command that prints the metric (a JSON line or a number); sets bench mode" })),
|
|
451
|
+
metric: Type.Optional(Metric),
|
|
452
|
+
protect: Type.Optional(Type.Array(Type.String(), { description: "Files attempts may not change (restored before judging)" })),
|
|
453
|
+
build: Type.Optional(Type.String({ description: "Visual mode: command that builds the page and writes the screenshots" })),
|
|
454
|
+
shots: Type.Optional(Type.Array(Type.String(), { description: "Visual mode: screenshot files the build writes, judged as images" })),
|
|
455
|
+
rounds: Type.Optional(Type.Number({ description: "Rounds to run now (default 1)" })),
|
|
456
|
+
from: Type.Optional(Type.String({ description: "Run id (or \"last\") to continue from its last judged round" })),
|
|
457
|
+
rules: Type.Optional(Type.Array(Type.String(), { description: "Rules every attempt must follow; the judge vetoes breaches" })),
|
|
458
|
+
routes: Type.Optional(Type.Array(Type.String(), { description: "Visual goals: routes to screenshot (e.g. /pricing) with the project's `serve` from .pi/loops.json; builds `build` + `shots` for you" })),
|
|
459
|
+
directions: Type.Optional(Type.Object({ safe: Type.Optional(Type.String()), bold: Type.Optional(Type.String()), wild: Type.Optional(Type.String()) }, { description: "Goal-specific angle for each attempt style" })),
|
|
460
|
+
});
|
|
461
|
+
export const InitParams = Type.Object({
|
|
462
|
+
serve: Type.Optional(Type.String({ description: "Starts the app on $PORT, e.g. `npm run dev -- --port $PORT`; omit when there is no web app" })),
|
|
463
|
+
check: Type.Optional(Type.String({ description: "Tests / typecheck command that must pass" })),
|
|
464
|
+
bench: Type.Optional(Type.String({ description: "Script that prints the number to climb" })),
|
|
465
|
+
metric: Type.Optional(Metric),
|
|
466
|
+
protect: Type.Optional(Type.Array(Type.String(), { description: "Files attempts may not edit" })),
|
|
467
|
+
});
|
|
468
|
+
|
|
469
|
+
const fmt = (x: unknown, unit?: string | null) => (x == null ? "—" : `${x} ${unit ?? ""}`.trim());
|
|
470
|
+
export function roundTable(state: State, round: Round): string {
|
|
471
|
+
const unit = state.metric?.unit ?? (state.kind === "bench" ? "ms" : "");
|
|
472
|
+
const j = round.judge ?? {};
|
|
473
|
+
const kept = (label: string) => (j.pick === label ? "✓" : "");
|
|
474
|
+
const rows = round.attempts.map((a) => [kept(a.label), a.label, a.style, a.model ? shortModel(a.model) : "", a.status !== "done" ? a.status : a.vetoed || (a.review && a.review.ok === false) ? "vetoed" : fmt(j.scores?.[a.label] ?? a.bench?.value, unit), (j.reasons?.[a.label] ?? a.error ?? a.summary ?? "").slice(0, 90)]);
|
|
475
|
+
if (j.scores?.CURRENT != null) rows.unshift([kept("CURRENT"), "CURRENT", "best so far", "", fmt(j.scores.CURRENT, unit), j.reasons?.CURRENT ?? ""]);
|
|
476
|
+
const head = ["", "attempt", "style", "model", state.kind === "bench" ? "result" : "score", "judge"];
|
|
477
|
+
const w = head.map((h, i) => Math.max(h.length, ...rows.map((r) => String(r[i]).length)));
|
|
478
|
+
const line = (r: string[]) => r.map((c, i) => String(c).padEnd(w[i])).join(" ").trimEnd();
|
|
479
|
+
return [line(head), line(w.map((n, i) => (head[i] ? "-" : " ").repeat(n))), ...rows.map(line)].join("\n");
|
|
480
|
+
}
|
|
481
|
+
/** What the winner changed vs the baseline: files tracked in the baseline (safe to apply) vs files the attempt added or removed. */
|
|
482
|
+
export function winnerFiles(cwd: string, base: string, commit: string, g: GitRunner = git) {
|
|
483
|
+
const changed: string[] = [], added: string[] = [], removed: string[] = [];
|
|
484
|
+
for (const line of g(cwd, "diff", "--name-status", `${base}..${commit}`).split("\n").filter(Boolean)) {
|
|
485
|
+
const [st, ...rest] = line.split("\t");
|
|
486
|
+
const f = rest[rest.length - 1];
|
|
487
|
+
if (st.startsWith("A")) added.push(f); else if (st.startsWith("D")) removed.push(f); else changed.push(f);
|
|
488
|
+
}
|
|
489
|
+
return { changed, added, removed };
|
|
490
|
+
}
|
|
491
|
+
export function applyHint(best: { commit: string; label?: string | null; score?: number | null }, files: { changed: string[]; added: string[]; removed: string[] }): string[] {
|
|
492
|
+
const q = (xs: string[]) => xs.map((f) => (/[^\w./-]/.test(f) ? JSON.stringify(f) : f)).join(" ");
|
|
493
|
+
const lines: string[] = [];
|
|
494
|
+
if (files.changed.length) lines.push(`Apply (tracked files only): git checkout ${best.commit.slice(0, 7)} -- ${q(files.changed)}`);
|
|
495
|
+
else lines.push("Apply: nothing — the winner changed no tracked file.");
|
|
496
|
+
if (files.added.length) lines.push(`Not applied — new files the attempt left in its worktree: ${q(files.added)} (take one explicitly with git checkout ${best.commit.slice(0, 7)} -- <file>)`);
|
|
497
|
+
if (files.removed.length) lines.push(`Not applied — files the attempt deleted: ${q(files.removed)} (git rm them yourself if intended)`);
|
|
498
|
+
return lines;
|
|
499
|
+
}
|
|
500
|
+
|
|
501
|
+
/** Take a run's best into the checkout: the winner's changed tracked files from its commit, committed. New/deleted files are reported, not applied. */
|
|
502
|
+
export function applyBest(cwd: string, state: State, g: GitRunner = git) {
|
|
503
|
+
const best: Partial<State["best"]> = state.best ?? {};
|
|
504
|
+
if (!best.commit || best.commit === state.baseCommit) throw new Error(`${state.id}: nothing to apply — no attempt beat the baseline`);
|
|
505
|
+
const files = winnerFiles(cwd, state.baseCommit, best.commit, g);
|
|
506
|
+
if (!files.changed.length) return { applied: [], files, commit: null };
|
|
507
|
+
const dirty = g(cwd, "status", "--porcelain", "--untracked-files=no").split("\n").filter((l) => l.trim()).map((l) => l.trim().split(/\s+/).pop());
|
|
508
|
+
const clash = files.changed.filter((f) => dirty.includes(f));
|
|
509
|
+
if (clash.length) throw new Error(`uncommitted changes in ${clash.join(", ")} — commit or stash them first`);
|
|
510
|
+
g(cwd, "checkout", best.commit, "--", ...files.changed);
|
|
511
|
+
g(cwd, "add", "--", ...files.changed);
|
|
512
|
+
if (!g(cwd, "status", "--porcelain", "--", ...files.changed).trim()) return { applied: [], files, commit: null, already: true };
|
|
513
|
+
const score = best.score != null ? ` (${best.score}${state.metric?.unit ? " " + state.metric.unit : ""})` : "";
|
|
514
|
+
g(cwd, "commit", "-q", "-m", `${(state.goal ?? "loop").slice(0, 60)} — ${best.label} from ${state.id}${score}`);
|
|
515
|
+
return { applied: files.changed, files, commit: g(cwd, "rev-parse", "HEAD") };
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
/** The climb so far, one line per round: CURRENT, each attempt with its delta, what was kept. */
|
|
519
|
+
export function climbText(state: State): string {
|
|
520
|
+
const unit = state.metric?.unit ?? "";
|
|
521
|
+
const lines: string[] = [];
|
|
522
|
+
for (const r of state.rounds ?? []) {
|
|
523
|
+
const j = r.judge; if (!j) { lines.push(`r${r.n} (not judged)`); continue; }
|
|
524
|
+
const cur = j.scores?.CURRENT;
|
|
525
|
+
const parts = r.attempts.map((a) => {
|
|
526
|
+
const v = j.scores?.[a.label];
|
|
527
|
+
const who = a.model ? `${a.style}/${shortModel(a.model)}` : a.style;
|
|
528
|
+
if (a.status !== "done") return `${who} ${a.status}`;
|
|
529
|
+
if (v == null) return `${who} ${(j.reasons?.[a.label] ?? "no number").slice(0, 24)}`;
|
|
530
|
+
const d = cur != null && cur !== 0 ? ` (${v < cur ? "" : "+"}${(((v - cur) / cur) * 100).toFixed(1)}%)` : "";
|
|
531
|
+
return `${who} ${fmt(v, unit)}${d}${j.pick === a.label ? " ✓" : ""}`;
|
|
532
|
+
});
|
|
533
|
+
lines.push(`r${r.n} CURRENT ${cur != null ? fmt(cur, unit) : "?"} │ ${parts.join(" · ")} → kept ${j.pick}${j.human ? " (yours)" : ""}${r.sheet ? ` [${r.sheet}]` : ""}`);
|
|
534
|
+
}
|
|
535
|
+
return lines.join("\n");
|
|
536
|
+
}
|
|
537
|
+
|
|
538
|
+
export function resultText(state: State, rounds: Round[], cwd: string): string {
|
|
539
|
+
const out: string[] = [];
|
|
540
|
+
const unit = state.metric?.unit ?? (state.kind === "bench" ? "ms" : "");
|
|
541
|
+
// Scan order: the verdict (what was kept, why) and the next action first, the attempt table second, supporting detail last.
|
|
542
|
+
const last = rounds[rounds.length - 1];
|
|
543
|
+
for (const r of rounds) {
|
|
544
|
+
const j = r.judge;
|
|
545
|
+
const score = j && j.pick !== "CURRENT" ? j.scores?.[j.pick] : null;
|
|
546
|
+
out.push(`Round ${r.n}${j ? ` — kept ${j.pick}${score != null ? ` at ${fmt(score, unit)}` : ""}` : " — not judged"}`);
|
|
547
|
+
if (j?.why) out.push(`Why: ${j.why}`);
|
|
548
|
+
for (const l of Object.keys(j?.byModel ?? {})) { const xs = Object.values(j.byModel[l]) as number[]; if (xs.length > 1 && Math.max(...xs) - Math.min(...xs) >= 3) out.push(`Panel split on ${l}: ${Object.entries(j.byModel[l]).map(([m, x]) => `${m} ${x}`).join(" · ")}`); }
|
|
549
|
+
if (r === last) out.push(`Next round: loop({ from: "${state.id}" })`);
|
|
550
|
+
out.push("", roundTable(state, r), "");
|
|
551
|
+
}
|
|
552
|
+
const best = state.best;
|
|
553
|
+
if (best.commit && best.commit !== state.baseCommit) {
|
|
554
|
+
let stat = ""; let files: { changed: string[]; added: string[]; removed: string[] } = { changed: [], added: [], removed: [] };
|
|
555
|
+
try { stat = git(cwd, "diff", "--stat", `${state.baseCommit}..${best.commit}`); files = winnerFiles(cwd, state.baseCommit, best.commit); } catch {}
|
|
556
|
+
out.push(`Best so far: ${best.label} (${best.commit.slice(0, 7)}, branch ${best.branch ?? "?"})${best.score != null ? ` at ${fmt(best.score, state.metric?.unit)}` : ""}`);
|
|
557
|
+
if (stat) out.push(stat);
|
|
558
|
+
out.push(...applyHint(best, files), "");
|
|
559
|
+
} else out.push("No attempt beat the baseline yet.", "");
|
|
560
|
+
if ((state.rounds?.length ?? 0) > 1 || state.shots?.length) out.push("Climb:", climbText(state), "");
|
|
561
|
+
if (state.models && !state.models.spread && !STYLES.some((st) => state.models[st])) out.push("All roles ran on your current model — /loop models gives each attempt its own (or set enabledModels in Pi's settings to spread them).", "");
|
|
562
|
+
if (state.openNext) out.push(`/loop open ${state.openNext} — resume that attempt's session and ask it what it did.`, "");
|
|
563
|
+
return out.join("\n").trim();
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
export function specText(p: SpecView): string {
|
|
567
|
+
const dirs = p.directions ?? {};
|
|
568
|
+
const anyDirs = Object.values(dirs).some(Boolean);
|
|
569
|
+
const lines = [
|
|
570
|
+
`goal ${p.goal ?? ""}`,
|
|
571
|
+
`rules ${p.rules?.length ? "\n" + p.rules.map((r) => ` - ${r}`).join("\n") : "none"}`,
|
|
572
|
+
`angles ${anyDirs ? "\n" + STYLES.filter((st) => dirs[st]).map((st) => ` ${st.padEnd(5)} ${dirs[st]}`).join("\n") : "the standard safe / bold / wild"}`,
|
|
573
|
+
`shots ${p.routes?.length ? p.routes.join(", ") : p.shots?.length ? `${p.shots.join(", ")} (build: ${p.build ?? "none"})` : "none — judged on the diffs"}`,
|
|
574
|
+
`project check ${p.check ?? "none"} · bench ${p.bench ? `${p.bench} (${p.metric?.unit ?? p.metric?.key ?? "number"}, ${p.metric?.better ?? "lower"} is better)` : "none"} · protect ${p.protect?.length ? p.protect.join(", ") : "none"}${p.routes?.length ? ` · serve ${p.serve ?? "MISSING — run /loop init"}` : ""}`,
|
|
575
|
+
];
|
|
576
|
+
return lines.join("\n");
|
|
577
|
+
}
|
|
578
|
+
|
|
579
|
+
export const SETUP_KEYS = ["serve", "check", "bench", "metric", "protect"];
|
|
580
|
+
export function projectSetup(cwd: string): Record<string, any> { const c = loopSettings(null, cwd); return Object.fromEntries(SETUP_KEYS.filter((k) => c[k] != null && (!Array.isArray(c[k]) || c[k].length)).map((k) => [k, c[k]])); }
|
|
581
|
+
export function setupText(p: Record<string, any>): string {
|
|
582
|
+
return [`serve ${p.serve ?? "none (no web app to screenshot)"}`, `check ${p.check ?? "none"}`, `bench ${p.bench ? `${p.bench} (${p.metric?.unit ?? p.metric?.key ?? "number"}, ${p.metric?.better ?? "lower"} is better)` : "none"}`, `protect ${p.protect?.length ? p.protect.join(", ") : "none"}`].join("\n");
|
|
583
|
+
}
|
|
584
|
+
const INIT_STEP = `Set the project up for loops first (one time): from package.json and the config files actually present — read, don't run, no bash — work out serve (the command that starts the app on $PORT, e.g. \`npm run dev -- --port $PORT\`; none if this isn't a web app), check (tests / typecheck as this repo runs them; none if none), bench (a script that prints the number to climb, with its unit and whether lower or higher is better; usually none) and protect (the check / bench / config files attempts could game), then call \`loop_init\` with them — it shows them to me and saves .pi/loops.json.`;
|
|
585
|
+
export function projectText(cwd: string): string {
|
|
586
|
+
const p = projectSetup(cwd);
|
|
587
|
+
if (!p.serve && !p.check) return `This repo is not set up for loops yet. ${INIT_STEP} Then the brief below.`;
|
|
588
|
+
return `Project setup (.pi/loops.json, written once by /loop init; say if it is wrong):\n${setupText(p)}`;
|
|
589
|
+
}
|
|
590
|
+
/** The dialog version: a select's title is not scrollable, so keep it to a few short lines; the full brief goes to the tool's progress output. */
|
|
591
|
+
export function specBrief(p: SpecView, width: number = 96): string {
|
|
592
|
+
const cut = (t: unknown) => { const s = String(t ?? "").replace(/\s+/g, " ").trim(); return s.length > width ? s.slice(0, width - 1) + "…" : s; };
|
|
593
|
+
const dirs = p.directions ?? {};
|
|
594
|
+
const anyDirs = Object.values(dirs).some(Boolean);
|
|
595
|
+
return [
|
|
596
|
+
`goal ${cut(p.goal)}`,
|
|
597
|
+
`rules ${p.rules?.length ? `${p.rules.length}: ${cut(p.rules.join(" · "))}` : "none"}`,
|
|
598
|
+
`angles ${anyDirs ? cut(STYLES.filter((st) => dirs[st]).map((st) => `${st}: ${dirs[st]}`).join(" · ")) : "standard safe / bold / wild"}`,
|
|
599
|
+
`shots ${p.routes?.length ? cut(p.routes.join(", ")) : p.shots?.length ? cut(p.shots.join(", ")) : "none — judged on the diffs"}`,
|
|
600
|
+
`project ${cut(`check ${p.check ?? "none"} · bench ${p.bench ?? "none"} · protect ${p.protect?.length ? p.protect.join(", ") : "none"}${p.routes?.length ? ` · serve ${p.serve ?? "MISSING — run /loop init"}` : ""}`)}`,
|
|
601
|
+
].join("\n");
|
|
602
|
+
}
|
|
603
|
+
|
|
604
|
+
export function loopPrompt(ask: string, cwd: string = process.cwd()): string {
|
|
605
|
+
const md = fs.readFileSync(path.join(ROOT, "prompts", "loop.md"), "utf8");
|
|
606
|
+
const body = md.replace(/^---[\s\S]*?---\n/, "");
|
|
607
|
+
return body.replace("$PROJECT", () => projectText(cwd)).replace("$@", () => ask.trim());
|
|
608
|
+
}
|
|
609
|
+
export function initPrompt() { return fs.readFileSync(path.join(ROOT, "prompts", "init.md"), "utf8").replace(/^---[\s\S]*?---\n/, "").replace("$INIT", () => INIT_STEP); }
|
|
610
|
+
|
|
611
|
+
export const modelsText = (models: Record<string, any>) => Object.entries(models).filter(([k]) => k !== "pool").map(([k, v]) => `${k} ${Array.isArray(v) ? v.map(shortModel).join("+") : shortModel(v)}`).join(" · ") || "everything on the current model";
|
|
612
|
+
|
|
613
|
+
/** What `/loop models` reports: the models it saved and where, or why it could not save them. */
|
|
614
|
+
export type PickResult = { models: Record<string, any>; file: string } | { error: string };
|
|
615
|
+
|
|
616
|
+
/** Walk through one model per role and save them to <repo>/.pi/loops.json; null when cancelled or there is nothing to pick from, { error } when the file can't be written. */
|
|
617
|
+
export async function pickModels(ctx: PickCtx, { reason }: { reason?: string } = {}): Promise<PickResult | null> {
|
|
618
|
+
const available = (ctx.modelRegistry?.getAvailable?.() ?? []).map(modelSpec);
|
|
619
|
+
if (!available.length) { ctx.ui.notify("no models available — /login first", "error"); return null; }
|
|
620
|
+
const current = ctx.model ? modelSpec(ctx.model) : null;
|
|
621
|
+
const TYPE = "type a model…";
|
|
622
|
+
const options = [...(current ? [`${current} (current)`] : []), TYPE, ...available.filter((m) => m !== current)];
|
|
623
|
+
const pick = async (title: string) => {
|
|
624
|
+
for (;;) {
|
|
625
|
+
const r = await ctx.ui.select(title, options);
|
|
626
|
+
if (r == null) return null;
|
|
627
|
+
if (r !== TYPE) return r.replace(/ \(current\)$/, "");
|
|
628
|
+
const typed = (await ctx.ui.input?.(title, "part of a model id, e.g. haiku or gpt-5.6"))?.trim().toLowerCase();
|
|
629
|
+
if (!typed) continue;
|
|
630
|
+
const hits = available.filter((m) => m.toLowerCase().includes(typed));
|
|
631
|
+
if (hits.length === 1) return hits[0];
|
|
632
|
+
ctx.ui.notify(hits.length ? `${hits.length} models match "${typed}": ${hits.map(shortModel).slice(0, 8).join(", ")}${hits.length > 8 ? "…" : ""} — be more specific` : `no model matches "${typed}"`, "warning");
|
|
633
|
+
}
|
|
634
|
+
};
|
|
635
|
+
const models: Record<string, any> = {};
|
|
636
|
+
const hint: Record<string, string> = { safe: "smallest safe change", bold: "structural redesign", wild: "unconventional idea" };
|
|
637
|
+
for (const role of STYLES) { const r = await pick(`${reason ? reason + ": " : ""}model for the ${role} attempt (${hint[role]}) — each attempt its own`); if (r == null) return null; models[role] = r; }
|
|
638
|
+
const judge = await pick("Model for the judge (measures, vetoes — your strongest)"); if (judge == null) return null;
|
|
639
|
+
models.judge = judge;
|
|
640
|
+
models.council = [...new Set(STYLES.map((r) => models[r]))];
|
|
641
|
+
if (models.council.length < 2) models.council.push(judge);
|
|
642
|
+
const file = configFiles(ctx.cwd).project;
|
|
643
|
+
let saved;
|
|
644
|
+
try { saved = saveConfig(ctx.cwd, { models }); } catch (e) { const error = `${path.relative(ctx.cwd, file)}: ${errText(e).slice(0, 80)}; fix it and run /loop models again`; ctx.ui.notify(error, "error"); return { error }; }
|
|
645
|
+
return { models: saved.models, file: path.relative(ctx.cwd, file) };
|
|
646
|
+
}
|
|
647
|
+
|
|
648
|
+
/** The /loops card: what this repo's loops run with, and the last run. */
|
|
649
|
+
export function configCard(settings: LoopSettings, cwd: string, lastState?: State | null, effective: ModelPlan = resolveModels({ settings })): string {
|
|
650
|
+
const m = settings.models ?? {};
|
|
651
|
+
const roles = STYLES.map((r) => `${r} ${shortModel(effective[r])}`).join(" · ");
|
|
652
|
+
const council = (effective.council ?? []).map(shortModel).join("+");
|
|
653
|
+
const lines = [
|
|
654
|
+
`models ${roles} · judge ${shortModel(effective.judge)} · council ${council}${m.pool?.length ? ` · pool ${m.pool.join(", ")}` : ""}`,
|
|
655
|
+
`rounds ${settings.rounds ?? 1} per /loop`,
|
|
656
|
+
`budget ${settings.tokenBudget ? `${Math.round(settings.tokenBudget / 1000)}k tokens per attempt` : "no token cap (the timeout limits attempts)"}`,
|
|
657
|
+
`timeout ${settings.timeoutMin} min per attempt`,
|
|
658
|
+
`minGain ${settings.minGain} (bench mode)`,
|
|
659
|
+
];
|
|
660
|
+
if (lastState) {
|
|
661
|
+
const r = lastState.rounds?.at(-1);
|
|
662
|
+
lines.push(`last run ${lastState.id} — ${lastState.status}${r?.judge ? `, r${r.n} kept ${r.judge.pick}` : ""}${lastState.best?.commit && lastState.best.commit !== lastState.baseCommit ? `, best ${lastState.best.label} (${lastState.best.commit.slice(0, 7)})` : ""}`);
|
|
663
|
+
}
|
|
664
|
+
lines.push(`config ${path.relative(cwd, configFiles(cwd).project)} (this repo) · ${configFiles(cwd).user} (your defaults)`);
|
|
665
|
+
return lines.join("\n");
|
|
666
|
+
}
|
|
667
|
+
|
|
668
|
+
export async function openSession(ctx: Ctx, label: string, runId?: string | null): Promise<boolean> {
|
|
669
|
+
let state: State; try { state = loadState(ctx.cwd, runId || "last"); } catch (e) { ctx.ui.notify(errText(e), "error"); return false; }
|
|
670
|
+
const rounds = state.rounds ?? [];
|
|
671
|
+
const file = label === "judge" ? [...rounds].reverse().find((r) => r.judgeSession)?.judgeSession : rounds.flatMap((r) => r.attempts).find((a) => a.label.toLowerCase() === label.toLowerCase())?.session;
|
|
672
|
+
if (!file || !fs.existsSync(file)) { ctx.ui.notify(`no session file for ${label} in ${state.id}`, "error"); return false; }
|
|
673
|
+
await ctx.switchSession?.(file);
|
|
674
|
+
return true;
|
|
675
|
+
}
|
|
676
|
+
|
|
677
|
+
export function runsText(cwd: string): string {
|
|
678
|
+
const dir = path.join(cwd, STATE_DIR);
|
|
679
|
+
const files = fs.existsSync(dir) ? fs.readdirSync(dir).filter((f) => f.endsWith(".json")).sort() : [];
|
|
680
|
+
if (!files.length) return `no runs in ${STATE_DIR}/`;
|
|
681
|
+
return files.map((f) => { try { const st = JSON.parse(fs.readFileSync(path.join(dir, f), "utf8")); return `${st.id} — ${st.status} — ${st.goal.slice(0, 60)}\n${climbText(st).split("\n").map((l) => " " + l).join("\n")}`; } catch { return f; } }).join("\n");
|
|
682
|
+
}
|
|
683
|
+
|
|
684
|
+
export const SUBCOMMANDS = ["init", "models", "rounds", "budget", "timeout", "open", "runs"];
|
|
685
|
+
/** Bare, or a subcommand word with at most its argument(s) — anything longer is a goal ("rounds of golf should load faster"); bad values error in loopsCommand instead of starting a round. */
|
|
686
|
+
export function isSubcommand(args?: string | null): boolean {
|
|
687
|
+
const w = (args ?? "").trim().split(/\s+/).filter(Boolean);
|
|
688
|
+
if (!w.length) return true;
|
|
689
|
+
const sub = w[0].toLowerCase();
|
|
690
|
+
if (!SUBCOMMANDS.includes(sub)) return false;
|
|
691
|
+
if (sub === "models" || sub === "runs" || sub === "init") return w.length === 1;
|
|
692
|
+
if (sub === "open") return w.length <= 3;
|
|
693
|
+
return w.length <= 2;
|
|
694
|
+
}
|
|
695
|
+
const LOOPS_USAGE = "/loop — card and picker · /loop <goal> — run a round · /loop init — project setup (serve, check, bench, protect) · /loop models · /loop rounds <n> · /loop budget <tokens, e.g. 800k|1.5m> · /loop timeout <min> · /loop open <R1-safe|judge> [run] · /loop runs";
|
|
696
|
+
export async function loopsCommand(args: string, ctx: Ctx, pi: Pi): Promise<void> {
|
|
697
|
+
const [first, ...rest] = (args ?? "").trim().split(/\s+/).filter(Boolean);
|
|
698
|
+
const sub = first?.toLowerCase();
|
|
699
|
+
const settings = loopSettings(pi, ctx.cwd);
|
|
700
|
+
const set = (patch: Record<string, any>, what: string) => { try { saveConfig(ctx.cwd, patch); ctx.ui.notify(`${path.relative(ctx.cwd, configFiles(ctx.cwd).project)}: ${what}`, "info"); } catch (e) { ctx.ui.notify(errText(e), "error"); } };
|
|
701
|
+
switch (sub) {
|
|
702
|
+
case undefined: {
|
|
703
|
+
let last: State | null = null; try { last = loadState(ctx.cwd, sessionRuns(ctx).find((id) => fs.existsSync(stateFile(ctx.cwd, id))) ?? "last"); } catch {}
|
|
704
|
+
ctx.ui.notify(configCard(settings, ctx.cwd, last, modelsOf(ctx, settings, pi)), "info");
|
|
705
|
+
const choice = await ctx.ui.select("Change (esc to leave)", ["models", "rounds", "budget", "timeout"]);
|
|
706
|
+
if (!choice) return;
|
|
707
|
+
if (choice === "models") return loopsCommand("models", ctx, pi);
|
|
708
|
+
const v = await ctx.ui.input?.(`${choice} — currently ${choice === "rounds" ? settings.rounds ?? 1 : choice === "budget" ? (settings.tokenBudget ? `${Math.round(settings.tokenBudget / 1000)}k` : "none") : settings.timeoutMin}`, choice === "budget" ? "e.g. 800k or 1.5m" : "a number");
|
|
709
|
+
if (v == null || !v.trim()) return;
|
|
710
|
+
return loopsCommand(`${choice} ${v.trim()}`, ctx, pi);
|
|
711
|
+
}
|
|
712
|
+
case "models": { const r = await pickModels(ctx); if (r && "models" in r) ctx.ui.notify(`${r.file}: ${modelsText(r.models)}`, "info"); return; }
|
|
713
|
+
case "rounds": { const n = Number(rest[0]); if (!Number.isInteger(n) || n < 1) { ctx.ui.notify("/loop rounds <n ≥ 1>", "error"); return; } return set({ rounds: n }, `rounds ${n} per /loop`); }
|
|
714
|
+
case "budget": { const n = parseBudget(rest[0]); if (!n) { ctx.ui.notify("/loop budget <tokens: 800k, 1.5m, 400000>", "error"); return; } return set({ tokenBudget: n }, `budget ${Math.round(n / 1000)}k tokens per attempt`); }
|
|
715
|
+
case "timeout": { const n = Number(rest[0]); if (!(Number.isFinite(n) && n > 0)) { ctx.ui.notify("/loop timeout <minutes>", "error"); return; } return set({ timeoutMin: n }, `timeout ${n} min per attempt`); }
|
|
716
|
+
case "open": { if (!rest[0]) { ctx.ui.notify("/loop open <R1-safe|judge> [run id]", "info"); return; } await openSession(ctx, rest[0], rest[1]); return; }
|
|
717
|
+
case "init": await pi.sendUserMessage?.(initPrompt()); return;
|
|
718
|
+
case "runs": ctx.ui.notify(runsText(ctx.cwd), "info"); return;
|
|
719
|
+
default: ctx.ui.notify(LOOPS_USAGE, "info");
|
|
720
|
+
}
|
|
721
|
+
}
|
|
722
|
+
|
|
723
|
+
export default function loopsExtension(pi: Pi): void {
|
|
724
|
+
pi.registerCommand?.("loop", {
|
|
725
|
+
description: "Run a hill-climbing round: /loop <goal>[; check is <cmd>; bench is <cmd> and prints <unit>]. Bare /loop shows this repo's loops card; /loop models|rounds <n>|budget <tokens>|timeout <min>|open <attempt>|runs configure and inspect",
|
|
726
|
+
handler: async (args, ctx) => {
|
|
727
|
+
if (isSubcommand(args)) return loopsCommand(args, ctx, pi);
|
|
728
|
+
await pi.sendUserMessage?.(loopPrompt(args, ctx.cwd));
|
|
729
|
+
},
|
|
730
|
+
});
|
|
731
|
+
pi.on?.("session_shutdown", async () => { for (const r of RUNS.values()) r.ctrl.abort(); await stopAll(); });
|
|
732
|
+
async function run(params: LoopParamsIn, signal: AbortSignal | undefined, onUpdate: ((update: any) => void) | undefined, ctx: Ctx, live: any) {
|
|
733
|
+
const cwd = ctx.cwd;
|
|
734
|
+
let settings = loopSettings(pi, cwd);
|
|
735
|
+
const chosen = Object.keys(settings.models).some((k) => k !== "pool");
|
|
736
|
+
if (!params.from && !chosen && ctx.hasUI && (ctx.modelRegistry?.getAvailable?.() ?? []).length > 1) {
|
|
737
|
+
const r = await pickModels(ctx, { reason: "First loop here" });
|
|
738
|
+
if (r && "error" in r) throw new Error(r.error);
|
|
739
|
+
if (r && "models" in r) settings = { ...settings, models: { ...settings.models, ...r.models } };
|
|
740
|
+
}
|
|
741
|
+
const models = modelsOf(ctx, settings, pi);
|
|
742
|
+
let state: State;
|
|
743
|
+
if (params.from) {
|
|
744
|
+
const mine = params.from === "last" ? sessionRuns(ctx).find((id) => fs.existsSync(stateFile(cwd, id))) : null;
|
|
745
|
+
state = loadState(cwd, mine ?? params.from);
|
|
746
|
+
for (const k of ["check", "bench", "metric", "protect", "build", "shots"] as const) if (params[k] != null) state[k] = params[k];
|
|
747
|
+
for (const r of params.rules ?? []) if (!state.rules.includes(r)) { state.rules.push(r); if (state.rubric) addRule(state.rubric, r); }
|
|
748
|
+
} else {
|
|
749
|
+
if (!params.goal) throw new Error("loop needs a goal (or `from` to continue a run)");
|
|
750
|
+
let baseCommit: string; try { baseCommit = git(cwd, "rev-parse", "HEAD"); } catch { throw new Error("loop needs a git repo with at least one commit"); }
|
|
751
|
+
const edited = git(cwd, "status", "--porcelain", "--untracked-files=no").split("\n").some((l) => l.trim() && !/\s\.gitignore$/.test(l));
|
|
752
|
+
const stashed = edited ? git(cwd, "stash", "create", "loops baseline") : "";
|
|
753
|
+
if (stashed) baseCommit = stashed;
|
|
754
|
+
const id = `loop-${new Date().toISOString().replace(/[-:]/g, "").slice(0, 15)}-${randomBytes(2).toString("hex")}`;
|
|
755
|
+
const kind = params.bench ? "bench" : "visual";
|
|
756
|
+
let build = params.build, shots: string[] = params.shots ?? [];
|
|
757
|
+
if (!build && params.routes?.length) {
|
|
758
|
+
if (!settings.serve) throw new Error("routes need `serve` in .pi/loops.json — run /loop init first");
|
|
759
|
+
const tool = path.join(ROOT, "lib", "shot.ts"), q = (v: unknown) => `'${String(v).replace(/'/g, `'\\''`)}'`;
|
|
760
|
+
const slug = (r: string) => r.replace(/^\/+|\/+$/g, "").replace(/[^a-z0-9]+/gi, "-").toLowerCase() || "index";
|
|
761
|
+
shots = params.routes.map((r) => `.loops-shots/${slug(r)}.png`);
|
|
762
|
+
build = params.routes.map((r, i) => `node ${q(tool)} --serve ${q(settings.serve)} --url ${q(r)} --out ${q(shots[i])}`).join(" && ");
|
|
763
|
+
}
|
|
764
|
+
state = { v: 2, id, goal: params.goal, kind, editedBaseline: !!stashed, check: params.check ?? settings.check, bench: params.bench, metric: { key: "ms", unit: kind === "bench" ? "ms" : "", better: "lower", ...(params.bench && params.bench === settings.bench ? settings.metric ?? {} : {}), ...(params.metric ?? {}) }, protect: params.protect ?? settings.protect ?? [], build, shots, rules: params.rules ?? [], directions: params.directions ?? {}, baseCommit, best: { label: "base", commit: baseCommit, session: null, score: null, reason: null }, winner: null, status: "running", rounds: [], rubric: null, parentSession: ctx.sessionManager?.getSessionFile?.() ?? null };
|
|
765
|
+
}
|
|
766
|
+
state.status = "running";
|
|
767
|
+
saveState(cwd, state);
|
|
768
|
+
const link = () => { try { pi.appendEntry?.("loops-run", { id: state.id, status: state.status, goal: state.goal, kind: state.kind, best: state.best?.label ?? null, rounds: state.rounds?.length ?? 0 }); } catch {} };
|
|
769
|
+
link();
|
|
770
|
+
if (state.shots?.length) { try { excludeShots(cwd, state.shots); } catch {} }
|
|
771
|
+
const runDir = path.join(cwd, STATE_DIR, state.id);
|
|
772
|
+
const details: Details = { run: state.id, kind: state.kind, round: null, judge: null, attempts: [], phase: "starting" };
|
|
773
|
+
if (live) { live.id = state.id; live.state = state; live.details = details; live.cwd = cwd; live.started?.(); }
|
|
774
|
+
const push = (phase?: string) => { details.phase = phase ?? details.phase; if (live) { try { ctx.ui.setStatus?.("loops", `${state.id} r${details.round ?? "?"} · ${details.phase}`); } catch {} } onUpdate?.({ content: [{ type: "text", text: `${state.id} r${details.round ?? "?"} · ${details.phase}\n${details.attempts.map((a) => ` ${a.style.padEnd(5)} ${a.status.padEnd(8)} ${a.tool ? `[${a.tool}] ` : ""}${a.tools} tools, ${a.tokens} tok${a.text ? ` — ${a.text.split("\n").pop()?.slice(0, 80) ?? ""}` : ""}`).join("\n")}` }], details: structuredClone(details) }); };
|
|
775
|
+
const askModel = (role: string, i: number = 0) => (content: any) => oneShot({ role, model: role === "council" ? (models.council ?? [])[i % (models.council?.length || 1)] : models[role], content, cwd, ctx, settings, signal });
|
|
776
|
+
const askAs = (role: string, model?: string | null) => (content: any) => oneShot({ role, model, content, cwd, ctx, settings, signal });
|
|
777
|
+
// the judge panel: every council model (the attempts' models) scores every candidate, labels blind; the judge model keeps vetoes and the head-to-head
|
|
778
|
+
const panel = settings.council !== false && settings.judgePanel !== false && (models.council?.length ?? 0) > 1 ? (models.council ?? []).map((m) => ({ name: shortModel(m), ask: askAs("judge", m) })) : undefined;
|
|
779
|
+
state.models = { ...Object.fromEntries(STYLES.map((st) => [st, models[st]])), judge: models.judge, council: models.council, spread: models.spread };
|
|
780
|
+
const needsYou = async (title: string, options: string[], fallback: string): Promise<string> => {
|
|
781
|
+
if (ctx.hasUI && !live) { const r = await ctx.ui.select(title, options); return r ?? fallback; }
|
|
782
|
+
return fallback;
|
|
783
|
+
};
|
|
784
|
+
|
|
785
|
+
if (!state.rubric && settings.council !== false && state.kind === "bench") {
|
|
786
|
+
push("council writes the rubric");
|
|
787
|
+
try { state.rubric = newRubric(await writeRubric({ goal: state.goal, kind: state.kind, rules: state.rules, subject: "the code", members: models.council?.length ?? 0, ask: (role, prompt, i) => askModel(role, i)(prompt) }), "council"); }
|
|
788
|
+
catch (e) { state.rubric = fromRules(state.rules, errText(e).slice(0, 200)); }
|
|
789
|
+
saveState(cwd, state);
|
|
790
|
+
}
|
|
791
|
+
|
|
792
|
+
const done: Round[] = [];
|
|
793
|
+
const want = Math.max(1, params.rounds ?? settings.rounds ?? 1);
|
|
794
|
+
for (let k = 0; k < want; k++) {
|
|
795
|
+
if (signal?.aborted) break;
|
|
796
|
+
let round = state.rounds[state.rounds.length - 1];
|
|
797
|
+
if (!round || round.judge) { round = { n: state.rounds.length + 1, from: state.best.commit, status: "running", attempts: [], judge: null, startedAt: Date.now() }; state.rounds.push(round); saveState(cwd, state); }
|
|
798
|
+
const n = round.n;
|
|
799
|
+
details.round = n;
|
|
800
|
+
const styles = STYLES.slice(0, Math.min(STYLES.length, settings.attempts));
|
|
801
|
+
const keep = round.attempts.filter(finished);
|
|
802
|
+
details.attempts = styles.map((s) => keep.find((a) => a.style === s) ? { style: s, status: "done", tool: null, tools: 0, tokens: 0, text: "kept from before" } : { style: s, status: "queued", tool: null, tools: 0, tokens: 0, text: "" });
|
|
803
|
+
push("attempts");
|
|
804
|
+
const results = await Promise.all(styles.map(async (style) => {
|
|
805
|
+
const kept = keep.find((a) => a.style === style);
|
|
806
|
+
if (kept) return kept;
|
|
807
|
+
const dir = path.join(runDir, `r${n}-${style}`);
|
|
808
|
+
const branch = `loops/${state.id}/r${n}-${style}`;
|
|
809
|
+
fs.mkdirSync(runDir, { recursive: true });
|
|
810
|
+
try { git(cwd, "worktree", "remove", "-f", dir); } catch {}
|
|
811
|
+
git(cwd, "worktree", "add", "-q", "-f", "-B", branch, dir, round.from ?? state.baseCommit);
|
|
812
|
+
const slot = details.attempts.find((a) => a.style === style)!;
|
|
813
|
+
const r = await runAttempt({ state, style, n, dir, ctx, settings, model: models[style], signal, live, onProgress: (p) => { Object.assign(slot, p); push(); } });
|
|
814
|
+
Object.assign(slot, { status: r.status, tool: null, text: r.summary || r.error || "" }); push();
|
|
815
|
+
return { ...r, branch, dir: path.relative(cwd, dir) };
|
|
816
|
+
}));
|
|
817
|
+
round.attempts = results;
|
|
818
|
+
if (signal?.aborted) { round.status = "aborted"; saveState(cwd, state); done.push(round); break; }
|
|
819
|
+
saveState(cwd, state);
|
|
820
|
+
|
|
821
|
+
push("judging");
|
|
822
|
+
const valid = results.filter(finished);
|
|
823
|
+
let judge: any;
|
|
824
|
+
const curRel = n > 1 && state.best.dir && fs.existsSync(path.join(cwd, state.best.dir)) ? state.best.dir : null;
|
|
825
|
+
let curDir = curRel ? path.join(cwd, curRel) : cwd, curFresh = false;
|
|
826
|
+
if (n > 1 && !curRel) { curDir = path.join(runDir, "best"); curFresh = true; try { git(cwd, "worktree", "remove", "-f", curDir); } catch {} git(cwd, "worktree", "add", "-q", "-f", "--detach", curDir, state.best.commit); }
|
|
827
|
+
for (const a of valid) { const adir = path.join(cwd, a.dir); if (restoreProtected({ cwd: adir, protect: state.protect, baseCommit: state.baseCommit })) { a.commit = git(adir, "rev-parse", "HEAD"); a.restored = true; } }
|
|
828
|
+
const shotFiles = (dir: string): ShotFiles => {
|
|
829
|
+
const files = (state.shots ?? []).map((f) => path.join(dir, f));
|
|
830
|
+
const missing = files.filter((f) => !fs.existsSync(f));
|
|
831
|
+
return missing.length ? { error: `no screenshot: ${missing.map((f) => path.basename(f)).join(", ")}`, have: files.filter((f) => fs.existsSync(f)) } : { files, have: files };
|
|
832
|
+
};
|
|
833
|
+
const ensureShots = async (dir: string, force: boolean = false): Promise<ShotFiles> => { if (state.shots?.length && state.build && (force || shotFiles(dir).error)) await sh(dir, state.build); return shotFiles(dir); };
|
|
834
|
+
if (state.kind === "bench") {
|
|
835
|
+
const cands: Candidate[] = [{ label: "CURRENT", dir: curDir, commit: state.best.commit, fresh: curFresh }, ...valid.map((a) => ({ label: a.label, dir: path.join(cwd, a.dir), commit: a.commit }))];
|
|
836
|
+
const num = (v: unknown): number | null => (typeof v === "number" && Number.isFinite(v) ? v : null);
|
|
837
|
+
let cur: { value: number | null; note: any } = { value: null, note: null }, failure: string | null = null;
|
|
838
|
+
try {
|
|
839
|
+
const j = await judgeRound({ state, n, cands, cwd, ctx, settings, model: models.judge, signal, onProgress: (p) => { details.judge = { ...p }; push(`judging${p.tool ? ` [${p.tool}]` : ""} · ${p.tools} tools, ${p.tokens} tok`); } });
|
|
840
|
+
round.judgeSession = j.session; round.judgeTokens = j.tokens; round.judgeCost = j.cost;
|
|
841
|
+
cur = { value: num(j.out.CURRENT?.value), note: j.out.CURRENT?.note };
|
|
842
|
+
for (const a of valid) {
|
|
843
|
+
const r = j.out[a.label] ?? {};
|
|
844
|
+
a.bench = r.check === false ? { passed: false, tests: String(r.note ?? "check failed") } : { passed: true, value: num(r.value), ...(num(r.value) == null ? { error: String(r.note || "the judge got no benchmark number") } : {}) };
|
|
845
|
+
a.review = { ok: r.ok !== false, note: String(r.note ?? "") };
|
|
846
|
+
const slot = details.attempts.find((x) => x.style === a.style); if (slot) { slot.text = a.bench.error ?? (a.bench.passed ? `${fmt(a.bench.value, state.metric.unit)}${a.review.ok ? "" : " vetoed"}` : "check fails"); push(); }
|
|
847
|
+
}
|
|
848
|
+
} catch (e) { failure = errText(e).slice(0, 200); for (const a of valid) a.bench = { passed: true, error: `not judged: ${failure}` }; }
|
|
849
|
+
judge = pickWinner({ cur: { value: cur.value }, attempts: valid, metric: state.metric, minGain: settings.minGain });
|
|
850
|
+
if (failure) judge.why = `The judge failed (${failure}); kept the current best.`;
|
|
851
|
+
if (cur.value == null) judge.reasons.CURRENT = failure ?? cur.note ?? "baseline could not be measured";
|
|
852
|
+
} else {
|
|
853
|
+
const shots = shotFiles;
|
|
854
|
+
const reasons: Record<string, string> = {};
|
|
855
|
+
const curShots = await ensureShots(curDir, curFresh);
|
|
856
|
+
const codeOnly = !state.shots?.length;
|
|
857
|
+
const diffOf = (dir: string, commit: string) => diffText(dir, state.best.commit, commit);
|
|
858
|
+
const candidates: { label: string; shots?: string[]; text?: string }[] = curShots.files ? [{ label: "CURRENT", shots: curShots.files, ...(codeOnly ? { text: "CURRENT is the code as it stands; every other candidate below is a diff against it. Judge whether each diff makes the code better for the goal (correctness, clarity, idiom, no needless churn or deleted safety)." } : {}) }] : [];
|
|
859
|
+
if (!curShots.files) reasons.CURRENT = `${curShots.error} in ${path.relative(cwd, curDir) || "."}; attempts were scored on the goal alone`;
|
|
860
|
+
else if (n === 1 && state.editedBaseline) reasons.CURRENT = "your checkout, uncommitted edits included; attempts started from the same state";
|
|
861
|
+
for (const a of valid) {
|
|
862
|
+
const adir = path.join(cwd, a.dir);
|
|
863
|
+
if (a.restored && state.build) { const b = await sh(adir, state.build); if (b.code !== 0) { a.bench = { passed: false, tests: "rebuild after restoring protected files failed: " + b.out.slice(-200) }; reasons[a.label] = a.bench.tests.slice(0, 120); continue; } }
|
|
864
|
+
if (state.check) { const c = await sh(adir, state.check); if (c.code !== 0) { a.bench = { passed: false, tests: c.out.slice(-200) }; reasons[a.label] = "check fails"; continue; } }
|
|
865
|
+
a.bench = { passed: true };
|
|
866
|
+
let s = shots(adir);
|
|
867
|
+
if (s.error && state.build) { const b = await sh(adir, state.build); if (b.code !== 0) { a.bench.error = `build failed: ${b.out.trim().split("\n").pop()?.slice(0, 160) ?? ""}`; reasons[a.label] = a.bench.error.slice(0, 120); continue; } s = shots(adir); }
|
|
868
|
+
if (s.error) { a.bench.error = s.error; reasons[a.label] = s.error.slice(0, 120); continue; }
|
|
869
|
+
candidates.push({ label: a.label, shots: s.files, ...(codeOnly ? { text: diffOf(adir, a.commit) } : {}) });
|
|
870
|
+
}
|
|
871
|
+
const img = (f: string) => ({ type: "image", data: fs.readFileSync(f).toString("base64"), mimeType: f.endsWith(".jpg") || f.endsWith(".jpeg") ? "image/jpeg" : "image/png" });
|
|
872
|
+
judge = candidates.some((c) => c.label !== "CURRENT") ? await panelJudge({ ask: askModel("judge"), panel, img, subject: codeOnly ? "the code" : "the page", goal: state.goal, rules: state.rules, rubric: state.rubric ?? undefined, candidates, samples: settings.judgeSamples, pairs: settings.judgePairs }) : { scores: {}, reasons: {}, pick: "CURRENT", why: valid.length ? `No attempt could be judged: ${valid.map((a) => `${a.label} — ${reasons[a.label] ?? a.bench?.error ?? "?"}`).join("; ") || "none finished"}.` : "No attempt finished." };
|
|
873
|
+
Object.assign(judge.reasons, reasons);
|
|
874
|
+
}
|
|
875
|
+
for (const a of results) if (a.status !== "done") judge.reasons[a.label] ??= a.error ? `${a.status}: ${a.error}` : a.status;
|
|
876
|
+
for (const a of results) if (a.cut) judge.reasons[a.label] = `${judge.reasons[a.label] ? judge.reasons[a.label] + " — " : ""}cut off by ${a.cut === "budget" ? "the token budget" : a.cut === "timeout" ? "the time limit" : a.cut}`;
|
|
877
|
+
|
|
878
|
+
if (state.shots?.length) {
|
|
879
|
+
push("sheet");
|
|
880
|
+
const unit = state.metric?.unit ?? "";
|
|
881
|
+
const tag = (label: string, a?: Attempt) => judge.reasons?.[label] && (a?.bench?.passed === false || a?.bench?.error) ? `${label} — ${String(judge.reasons[label]).slice(0, 28)}` : `${label}${judge.scores?.[label] != null ? ` ${fmt(judge.scores[label], unit)}` : ""}${judge.pick === label ? " ✓" : ""}`;
|
|
882
|
+
const tiles = [{ label: tag("CURRENT"), files: (await ensureShots(curDir, curFresh)).have }];
|
|
883
|
+
for (const a of valid) tiles.push({ label: tag(a.label, a), files: (await ensureShots(path.join(cwd, a.dir))).have });
|
|
884
|
+
try { const f = sheet({ tiles, out: path.join(runDir, `r${n}.png`) }); if (f) round.sheet = path.relative(cwd, f); } catch {}
|
|
885
|
+
}
|
|
886
|
+
|
|
887
|
+
const takeable = valid.filter((a) => a.bench?.passed && !a.bench.error);
|
|
888
|
+
if (live?.pick) {
|
|
889
|
+
const want = String(live.pick).toLowerCase();
|
|
890
|
+
const a = takeable.find((x) => x.label.toLowerCase() === want || x.style === want);
|
|
891
|
+
if (a) { judge.pick = a.label; judge.why = `You picked ${a.label}. ${judge.why}`; judge.human = true; }
|
|
892
|
+
else if (want === "current") { judge.pick = "CURRENT"; judge.why = `You kept CURRENT. ${judge.why}`; judge.human = true; }
|
|
893
|
+
else judge.why = `${judge.why} (your pick ${live.pick} was not a passing attempt)`;
|
|
894
|
+
live.pick = null;
|
|
895
|
+
}
|
|
896
|
+
if (ctx.hasUI && !live && takeable.length) {
|
|
897
|
+
for (;;) {
|
|
898
|
+
const others = takeable.filter((a) => a.label !== judge.pick);
|
|
899
|
+
const opts = [`Keep ${judge.pick}${judge.pick === "CURRENT" ? " (nothing beat it)" : " (judge's pick)"}`, ...others.map((a) => `Take ${a.label} instead`), ...valid.map((a) => `Show ${a.label}'s diff`), ...valid.filter((a) => a.session).map((a) => `Open ${a.label}'s session after the run`)];
|
|
900
|
+
const choice = await ctx.ui.select(`Round ${n}: ${judge.why}${round.sheet ? ` Sheet: ${round.sheet}` : ""}`, opts);
|
|
901
|
+
if (choice == null || choice.startsWith("Keep")) break;
|
|
902
|
+
const a = valid.find((x) => choice.includes(x.label));
|
|
903
|
+
if (!a) break;
|
|
904
|
+
if (choice.startsWith("Take")) { judge.pick = a.label; judge.why = `You picked ${a.label}. ${judge.why}`; judge.human = true; break; }
|
|
905
|
+
if (choice.startsWith("Show")) { let d: string; try { d = `${git(cwd, "diff", "--stat", `${round.from}..${a.commit}`)}\n\n${git(cwd, "diff", `${round.from}..${a.commit}`)}`.slice(0, 60000); } catch (e) { d = errText(e); } await ctx.ui.editor?.(`${a.label} vs CURRENT (read-only; close to return)`, d); }
|
|
906
|
+
if (choice.startsWith("Open")) { state.openNext = a.label; ctx.ui.notify(`After the run: /loop open ${a.label} resumes ${a.label}'s session (/loop open judge for the judge).`, "info"); }
|
|
907
|
+
}
|
|
908
|
+
} else if (judge.pick === "CURRENT" && takeable.length) {
|
|
909
|
+
const choice = await needsYou(`Round ${n}: ${judge.why} Keep the current best?`, ["Keep the current best", ...takeable.map((a) => `Take ${a.label} anyway`)], "Keep the current best");
|
|
910
|
+
const forced = takeable.find((a) => choice === `Take ${a.label} anyway`);
|
|
911
|
+
if (forced) { judge.pick = forced.label; judge.why = `You picked ${forced.label}. ${judge.why}`; judge.human = true; }
|
|
912
|
+
}
|
|
913
|
+
round.judge = judge;
|
|
914
|
+
round.status = "done";
|
|
915
|
+
round.endedAt = Date.now();
|
|
916
|
+
if (judge.pick !== "CURRENT") {
|
|
917
|
+
const w = results.filter(finished).find((a) => a.label === judge.pick);
|
|
918
|
+
if (w) {
|
|
919
|
+
state.best = { label: w.label, commit: w.commit, session: w.session, branch: w.branch, dir: w.dir, score: judge.scores[w.label] ?? null, reason: judge.reasons[w.label] ?? null, round: n };
|
|
920
|
+
let files: any = null; try { files = winnerFiles(cwd, state.baseCommit, w.commit); } catch {}
|
|
921
|
+
state.winner = { style: w.style, label: w.label, round: n, commit: w.commit, branch: w.branch, session: w.session, bench: state.kind === "bench" ? { value: judge.scores[w.label] ?? null, unit: state.metric?.unit ?? "" } : null, score: judge.scores[w.label] ?? null, reason: judge.reasons[w.label] ?? null, files };
|
|
922
|
+
}
|
|
923
|
+
}
|
|
924
|
+
saveState(cwd, state);
|
|
925
|
+
done.push(round);
|
|
926
|
+
push(`round ${n}: kept ${judge.pick}`);
|
|
927
|
+
}
|
|
928
|
+
const aborted = !!signal?.aborted;
|
|
929
|
+
state.status = aborted ? "aborted" : "done";
|
|
930
|
+
saveState(cwd, state);
|
|
931
|
+
link();
|
|
932
|
+
const text = (aborted ? `Aborted during round ${details.round}; attempts' worktrees are kept under ${path.relative(cwd, runDir)}/. Resume with loop({ from: "${state.id}" }).\n\n` : "") + resultText(state, done, cwd);
|
|
933
|
+
const lastSheet = [...done].reverse().find((r) => r.sheet)?.sheet;
|
|
934
|
+
const content: any[] = [{ type: "text", text }];
|
|
935
|
+
if (lastSheet) { try { content.push({ type: "image", data: fs.readFileSync(path.join(cwd, lastSheet)).toString("base64"), mimeType: "image/png" }); } catch {} }
|
|
936
|
+
return { content, structuredContent: structuredResult(state, state.rounds), details: { ...structuredClone(details), phase: aborted ? "aborted" : "done", status: state.status, best: state.best, winner: state.winner, rounds: done.map((r) => ({ n: r.n, pick: r.judge?.pick ?? null, why: r.judge?.why ?? null })) } };
|
|
937
|
+
}
|
|
938
|
+
|
|
939
|
+
pi.registerTool?.({
|
|
940
|
+
name: "loop",
|
|
941
|
+
label: "loop",
|
|
942
|
+
outputSchema: LoopResult,
|
|
943
|
+
description: "Run hill-climbing round(s) on the current git repo: three forked attempt sessions (safe/bold/wild agents) in their own worktrees, a judge keeps the best. Bench mode when `bench` is set (each attempt sets up and measures itself; a judge agent re-measures every candidate, reads the diffs and vetoes rule breaks); visual mode otherwise (`build` writes `shots`, a judge panel scores them). `from` continues a run from its last judged round.",
|
|
944
|
+
parameters: LoopParams,
|
|
945
|
+
async execute(_id, params, signal, onUpdate, ctx) {
|
|
946
|
+
if (ctx.hasUI && !params.from && params.confirm !== false) {
|
|
947
|
+
const st = loopSettings(pi, ctx.cwd);
|
|
948
|
+
const view = { ...params, check: params.check ?? st.check, protect: params.protect ?? st.protect, serve: st.serve, metric: params.metric ?? (params.bench && params.bench === st.bench ? st.metric : undefined) };
|
|
949
|
+
const spec = specText(view);
|
|
950
|
+
onUpdate?.({ content: [{ type: "text", text: `Brief\n${spec}` }], details: { run: null, kind: params.bench ? "bench" : "visual", round: null, attempts: [], phase: "brief" } });
|
|
951
|
+
const notRun = (phase: string, text: string) => ({ content: [{ type: "text", text }], details: { run: null, kind: params.bench ? "bench" : "visual", round: null, attempts: [], phase, spec: { ...params } }, structuredContent: structuredResult({ id: "", status: phase, kind: params.bench ? "bench" : "visual", goal: params.goal ?? "", baseCommit: "", metric: params.metric ?? {}, best: { commit: "" } }, []) });
|
|
952
|
+
const choice = await ctx.ui.select(`Brief for this loop (full text above)\n\n${specBrief(view)}\n`, ["Approve — run the round", "Change…", "Discuss"]);
|
|
953
|
+
if (choice?.startsWith("Change")) {
|
|
954
|
+
const note = (await ctx.ui.input?.("What should change?", "e.g. also protect src/lib/data.ts; check is npm test"))?.trim();
|
|
955
|
+
return notRun("change", note ? `Not run. Change requested: ${note}\nRevise the spec accordingly and call loop again.` : "Not run. Revise the spec with me in chat, then call loop again.");
|
|
956
|
+
}
|
|
957
|
+
if (!choice?.startsWith("Approve")) return notRun("discuss", "Not run. Discuss the spec with me in chat (what is unclear, what should be different), then call loop again when we agree.");
|
|
958
|
+
}
|
|
959
|
+
if (!ctx.hasUI || params.wait) return run(params, signal, onUpdate, ctx, null);
|
|
960
|
+
const live: any = { id: null, state: null, details: null, cwd: ctx.cwd, attempts: new Map(), pick: null, last: null, error: null, ctrl: new AbortController(), startedAt: Date.now() };
|
|
961
|
+
const started = new Promise((resolve) => { live.started = resolve; });
|
|
962
|
+
live.promise = run(params, live.ctrl.signal, (u) => { live.last = u; }, ctx, live).then(
|
|
963
|
+
(r) => { try { pi.sendUserMessage?.(r.content, { deliverAs: "followUp" }); } catch {} },
|
|
964
|
+
(e) => { live.error = e; if (live.id) { try { pi.sendUserMessage?.(`Loop ${live.id} failed: ${e.message}`, { deliverAs: "followUp" }); } catch {} } }
|
|
965
|
+
).finally(() => { if (live.id) RUNS.delete(live.id); live.started?.(); try { ctx.ui.setStatus?.("loops", undefined); } catch {} });
|
|
966
|
+
await started;
|
|
967
|
+
if (!live.id) throw live.error ?? new Error("the loop did not start");
|
|
968
|
+
RUNS.set(live.id, live);
|
|
969
|
+
const text = `${live.id} started (${live.state.kind}; ${STYLES.slice(0, Math.min(STYLES.length, loopSettings(pi, ctx.cwd).attempts)).join(" / ")} attempts). The round card arrives here as a message when the judge is done. Until then: loop_status for progress, loop_talk to tell an attempt something, loop_rule to add a rule, loop_pick to decide the winner yourself.`;
|
|
970
|
+
return { content: [{ type: "text", text }], details: { run: live.id, kind: live.state.kind, round: null, attempts: [], phase: "started" }, structuredContent: structuredResult(live.state, []) };
|
|
971
|
+
},
|
|
972
|
+
renderCall(args, theme) {
|
|
973
|
+
const what = args.from ? `continue ${args.from}` : (args.goal ?? "").slice(0, 70);
|
|
974
|
+
return new Text(theme.fg("toolTitle", theme.bold("loop ")) + theme.fg("accent", what) + theme.fg("muted", ` ×${args.rounds ?? 1}${args.bench ? " bench" : args.goal ? " visual" : ""}`), 0, 0);
|
|
975
|
+
},
|
|
976
|
+
renderResult(result, { expanded, isPartial }, theme) {
|
|
977
|
+
const d = result.details;
|
|
978
|
+
if (!d) return new Text(result.content?.[0]?.text ?? "", 0, 0);
|
|
979
|
+
const head = theme.fg(isPartial ? "warning" : d.phase === "aborted" ? "error" : "success", `${d.run} r${d.round ?? "?"} · ${d.phase}`);
|
|
980
|
+
const lines = ((d.attempts ?? []) as AttemptDetail[]).map((a) => ` ${theme.fg("accent", a.style.padEnd(5))} ${theme.fg(a.status === "done" ? "success" : a.status === "running" ? "warning" : "muted", a.status.padEnd(9))}${a.tool ? theme.fg("dim", `[${a.tool}] `) : ""}${theme.fg("muted", `${a.tools} tools · ${a.tokens} tok`)}${a.text ? theme.fg("dim", ` — ${a.text.split("\n").pop()?.slice(0, 70) ?? ""}`) : ""}`);
|
|
981
|
+
const body = !isPartial || d.phase === "brief" ? "\n" + (result.content?.[0]?.text ?? "") : "";
|
|
982
|
+
return new Text([head, ...lines].join("\n") + body, 0, 0);
|
|
983
|
+
},
|
|
984
|
+
});
|
|
985
|
+
const said = (t: string) => ({ content: [{ type: "text", text: t }] });
|
|
986
|
+
const idParam = Type.Optional(Type.String({ description: "Run id; default: the round started most recently" }));
|
|
987
|
+
pi.registerTool?.({ name: "loop_init", label: "loop_init", description: "Project setup for loops, written once to .pi/loops.json: serve (start the app on $PORT), check, bench + metric, protect. In the TUI it shows them and asks Save / Change / Discuss.", parameters: InitParams,
|
|
988
|
+
async execute(_id, p, _signal, _onUpdate, ctx) {
|
|
989
|
+
const fields = Object.fromEntries(SETUP_KEYS.filter((k) => p[k] != null && (!Array.isArray(p[k]) || p[k].length)).map((k) => [k, p[k]]));
|
|
990
|
+
if (ctx.hasUI) {
|
|
991
|
+
const choice = await ctx.ui.select(`Project setup for loops (.pi/loops.json)\n\n${setupText(fields)}\n`, ["Save", "Change…", "Discuss"]);
|
|
992
|
+
if (choice?.startsWith("Change")) { const note = (await ctx.ui.input?.("What should change?", "e.g. serve is npm start -- -p $PORT; no check"))?.trim(); return { content: [{ type: "text", text: note ? `Not saved. Change requested: ${note}\nRevise and call loop_init again.` : "Not saved. Revise with me in chat, then call loop_init again." }], details: { phase: "change", note } }; }
|
|
993
|
+
if (choice !== "Save") return { content: [{ type: "text", text: "Not saved. Discuss the setup with me in chat, then call loop_init again." }], details: { phase: "discuss" } };
|
|
994
|
+
}
|
|
995
|
+
saveConfig(ctx.cwd, { ...Object.fromEntries(SETUP_KEYS.map((k) => [k, null])), ...fields });
|
|
996
|
+
return { content: [{ type: "text", text: `Saved ${path.relative(ctx.cwd, configFiles(ctx.cwd).project)}:\n${setupText(fields)}` }], details: { phase: "saved", setup: fields } };
|
|
997
|
+
} });
|
|
998
|
+
pi.registerTool?.({ name: "loop_status", label: "loop_status", description: "Progress of a loop round running in the background: phase, each attempt's status, tools and tokens.", parameters: Type.Object({ id: idParam }),
|
|
999
|
+
async execute(_id, p) { const r = liveRun(p.id); return { content: r.last?.content ?? [{ type: "text", text: `${r.id} starting` }], details: r.details }; } });
|
|
1000
|
+
pi.registerTool?.({ name: "loop_talk", label: "loop_talk", description: "Tell a running attempt something (R1-bold, or just bold): delivered into its session as a steering message it acts on next.", parameters: Type.Object({ attempt: Type.String(), message: Type.String(), id: idParam }),
|
|
1001
|
+
async execute(_id, p) { const r = liveRun(p.id); const key = findAttempt(r, p.attempt); if (!key) throw new Error(`${p.attempt} is not running (running: ${[...r.attempts.keys()].join(", ") || "none"})`); await r.attempts.get(key).steer(`Message from the person running this loop: ${p.message}`); return said(`told ${key}: ${p.message}`); } });
|
|
1002
|
+
pi.registerTool?.({ name: "loop_rule", label: "loop_rule", description: "Add a rule to a running loop: every running attempt is told now, the judge enforces it from this round on, and it stays for later rounds.", parameters: Type.Object({ rule: Type.String(), id: idParam }),
|
|
1003
|
+
async execute(_id, p) { const r = liveRun(p.id); if (!r.state.rules.includes(p.rule)) { r.state.rules.push(p.rule); if (r.state.rubric) addRule(r.state.rubric, p.rule); saveState(r.cwd, r.state); } await Promise.allSettled([...r.attempts.values()].map((s) => s.steer(`New rule for this loop, from the person running it: ${p.rule}`))); return said(`rule added: ${p.rule} (told ${r.attempts.size} running attempt${r.attempts.size === 1 ? "" : "s"})`); } });
|
|
1004
|
+
pi.registerTool?.({ name: "loop_pick", label: "loop_pick", description: "Decide the running round yourself: when the judge is done, keep this attempt (R1-wild, or just wild; or CURRENT) as the best if its check passes, whatever the judge scored.", parameters: Type.Object({ attempt: Type.String(), id: idParam }),
|
|
1005
|
+
async execute(_id, p) { const r = liveRun(p.id); r.pick = p.attempt; return said(`when round ${r.details?.round ?? "?"} of ${r.id} is judged, ${p.attempt} is kept`); } });
|
|
1006
|
+
pi.registerTool?.({ name: "loop_apply", label: "loop_apply", description: "Take a run's best into the checkout: checks out the winner's changed tracked files from its commit and commits them (files it added or deleted are listed, not applied). Default: this session's last run.", parameters: Type.Object({ id: idParam }),
|
|
1007
|
+
async execute(_id, p, _signal, _onUpdate, ctx) {
|
|
1008
|
+
const mine = !p.id || p.id === "last" ? sessionRuns(ctx).find((id) => fs.existsSync(stateFile(ctx.cwd, id))) : null;
|
|
1009
|
+
const state = loadState(ctx.cwd, mine ?? p.id);
|
|
1010
|
+
if (RUNS.has(state.id)) throw new Error(`${state.id} is still running — loop_stop it or wait for the card`);
|
|
1011
|
+
const r = applyBest(ctx.cwd, state);
|
|
1012
|
+
const lines = [r.commit ? `Applied ${state.best.label} (${state.best.commit.slice(0, 7)}) as ${r.commit.slice(0, 7)}: ${r.applied.join(", ")}` : r.already ? `${state.best.label} is already in the checkout.` : "Nothing applied — the winner changed no tracked file."];
|
|
1013
|
+
if (r.files.added.length) lines.push(`Not applied — files it added: ${r.files.added.join(", ")} (git checkout ${state.best.commit.slice(0, 7)} -- <file> to take one)`);
|
|
1014
|
+
if (r.files.removed.length) lines.push(`Not applied — files it deleted: ${r.files.removed.join(", ")}`);
|
|
1015
|
+
return { ...said(lines.join("\n")), details: { run: state.id, applied: r.applied, commit: r.commit, files: r.files } };
|
|
1016
|
+
} });
|
|
1017
|
+
pi.registerTool?.({ name: "loop_stop", label: "loop_stop", description: "Stop a round running in the background: aborts its attempts and judge. What they committed stays in the worktrees; loop({ from }) resumes the run.", parameters: Type.Object({ id: idParam }),
|
|
1018
|
+
async execute(_id, p) { const r = liveRun(p.id); r.ctrl.abort(); await r.promise.catch(() => {}); return said(`${r.id} stopped. Resume with loop({ from: "${r.id}" }).`); } });
|
|
1019
|
+
}
|