@computesdk/pi-loops 1.0.1 → 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -4
- package/extensions/loop.ts +152 -35
- package/lib/judge.ts +1 -1
- package/lib/shot.ts +171 -23
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -42,9 +42,9 @@ Morph and Particle ship as their own packages, so you can install either one wit
|
|
|
42
42
|
| Provider | Package | Key | Models |
|
|
43
43
|
| --- | --- | --- | --- |
|
|
44
44
|
| Particle AI | `pi-particle` | `PARTICLE_API_KEY` (`pai-…`) | `particle-1`, `deepseek-ai/deepseek-v41-flash-1017`, `glm-5.3-flash` |
|
|
45
|
-
| Morph |
|
|
45
|
+
| Morph | `@computesdk/pi-morph` | `MORPH_API_KEY` | `morph-kimik3`, `morph-kimik3-fast`, `morph-glm53-744b`, `morph-glm53flash`, `morph-dsv41flash`, `morph-dsv41flash-1017`, `morph-dsv4flash` |
|
|
46
46
|
|
|
47
|
-
They live in this repo under `packages/pi-morph` and `packages/pi-particle`; install them with `pi install npm
|
|
47
|
+
They live in this repo under `packages/pi-morph` and `packages/pi-particle`; install them with `pi install npm:@computesdk/pi-morph` (or `pi install npm:pi-particle`). Loops itself is model-agnostic — it runs on whatever your TUI is on.
|
|
48
48
|
|
|
49
49
|
## The one tool
|
|
50
50
|
|
|
@@ -70,6 +70,8 @@ loop({
|
|
|
70
70
|
|
|
71
71
|
**Visual mode** (no `bench`): each attempt runs `build` itself and must leave the `shots` files (PNG/JPEG) in its worktree; a **judge** panel scores them (median of 3) and the top two go head-to-head against CURRENT, both orders; a candidate wins with a clear majority and a score not below CURRENT's. CURRENT's shots are taken with the same `build` in your checkout (round 1) or the previous winner's worktree. Uncommitted edits in your checkout are part of the baseline in both modes: round 1 forks from a commit of your working tree (`git stash create`, your checkout is untouched), so attempts and CURRENT start from the same state.
|
|
72
72
|
|
|
73
|
+
Screenshots come from `lib/shot.ts`: headless Chrome driven over the Chrome DevTools Protocol, no dependencies (CDP over Node's built-in `WebSocket`). The default is the **whole page**, not the 900px fold — a judge scoring a screenshot should see all of it — with `--viewport` for the fold and `--selector '#card'` for one element. Any Chrome/Chromium works, including `chrome-headless-shell`, the headless-only binary that has no `--screenshot` flag at all and is what `npx playwright install chromium-headless-shell` puts in `~/.cache/ms-playwright`; `LOOPS_CHROME` overrides discovery. Loops looks for one **before** the round starts and, when there is none, fails once with the install hint instead of three `build failed` lines. A route that 404s or refuses the connection is an error, never a picture of the browser's error page. The sheet is `montage` (ImageMagick) when installed.
|
|
74
|
+
|
|
73
75
|
**The pick is yours.** After every judged round you get the round card: when the run has `shots` (either mode) a side-by-side **sheet** of CURRENT and the attempts with their numbers (`.pi/loops/<run>/r<n>.png`, also shown inline in terminals with image support), and a picker — keep the judge's pick, take another attempt instead, show an attempt's diff, or mark an attempt's session to open afterwards (`/loop-open R1-bold`, `/loop-open judge`). The result ends with the **climb**: one line per round, CURRENT, each attempt with its delta, what was kept.
|
|
74
76
|
|
|
75
77
|
**Steering while it runs.** In the TUI the tool returns as soon as the round has started and the card arrives as a message when the judge is done, so the conversation stays open: `loop_status` (progress), `loop_talk("R1-bold", "…")` (a steering message into that attempt's session), `loop_rule("…")` (told to every running attempt, enforced by the judge from this round on, kept for later rounds), `loop_pick("wild")` (keep that attempt, or `CURRENT`, whatever the judge scores), `loop_stop()` (abort; worktrees stay, `loop({ from })` resumes). After the card, `loop_apply()` takes the run's best into the checkout — the winner's changed tracked files from its commit, as one commit; files it added or deleted are listed, not applied. Headless (`pi -p`) or with `wait: true` the tool blocks until the round is done, as before.
|
|
@@ -108,7 +110,7 @@ Config is Loops' own file, the way Pi extensions keep theirs: `<repo>/.pi/loops.
|
|
|
108
110
|
}
|
|
109
111
|
```
|
|
110
112
|
|
|
111
|
-
**Judging.** Visual and code-only rounds are scored by a panel: every council model (by default the attempts' models) scores every candidate 0–10 against the goal and rules, labels blind, median across models and samples; the `judge` model keeps the vetoes and the head-to-head against CURRENT that decides the round. The card says when the panel split (`Panel split on R1-wild: kimi-k3 3 · qwen3.8-flash 8 · glm-5.3 8`). `"judgePanel": false` in `.pi/loops.json` goes back to the judge alone; bench rounds are measured, not scored.
|
|
113
|
+
**Judging.** A visual round is judged on screenshots, so the judge — and every panel member, when the panel is on — must be a model that can see images; one that cannot is refused before the round starts (it would score every candidate the same, which reads as a tie), with the fix in the message. The one-shot sessions the judge and council run in load no extensions, so Loops copies the host's provider registrations into them: providers added by an extension (rather than built into Pi) can judge just as the attempts can, and nothing else about that session changes — no tools, no extension commands. Visual and code-only rounds are scored by a panel: every council model (by default the attempts' models) scores every candidate 0–10 against the goal and rules, labels blind, median across models and samples; the `judge` model keeps the vetoes and the head-to-head against CURRENT that decides the round. The card says when the panel split (`Panel split on R1-wild: kimi-k3 3 · qwen3.8-flash 8 · glm-5.3 8`). `"judgePanel": false` in `.pi/loops.json` goes back to the judge alone; bench rounds are measured, not scored.
|
|
112
114
|
|
|
113
115
|
**Models.** A round is also a comparison of models: the three attempts and the council members should each run on a different one. The first `/loop` in a repo (interactive, nothing set yet, more than one model available) asks four times — a model for each attempt, one for the judge (the council is the attempts' models) — and writes them to `.pi/loops.json`; `/loop models` re-opens that picker. `models` keys are the roles — `safe`, `bold`, `wild`, `judge`, `scout`, and `council` (a string or a list, one member per model). Roles left unset are spread over the **pool**: `models.pool` (a list of `provider/id`, `id` or `provider/*` patterns), else Pi's own `enabledModels`, starting from your current model; with no pool everything runs on your current model and the result says so. The round table and climb show each attempt's model. `"scout": false` turns the scout off (a bare goal then runs in visual mode on the goal alone).
|
|
114
116
|
|
|
@@ -120,6 +122,6 @@ Ctrl+C aborts the attempts and leaves their worktrees and sessions in place. `lo
|
|
|
120
122
|
|
|
121
123
|
## Library
|
|
122
124
|
|
|
123
|
-
`lib/measure.ts` (shell runner, protect restore, diff), `lib/judge.ts` (`pickWinner` over the judge's numbers, the visual `panelJudge`), `lib/rubric.ts` (council synthesis, human rules), `lib/shot.ts` (serve + headless-Chrome screenshot CLI), `lib/sheet.ts` (the labelled sheet, via ImageMagick's `montage` when installed). All model calls are injected, so `pnpm test` runs them against stubs — no model needed. The whole package is TypeScript: Pi loads the `.ts` files directly (no build step), and `pnpm check` (or `pnpm -r check` from the repo root) checks them. It ships no `main`/`exports` on purpose — Node refuses to strip types inside `node_modules`, so a programmatic `import "@computesdk/pi-loops"` would need a build; Pi loads the files through jiti and does not.
|
|
125
|
+
`lib/measure.ts` (shell runner, protect restore, diff), `lib/judge.ts` (`pickWinner` over the judge's numbers, the visual `panelJudge`), `lib/rubric.ts` (council synthesis, human rules), `lib/shot.ts` (serve + headless-Chrome screenshot CLI over CDP: full page, viewport or one selector), `lib/sheet.ts` (the labelled sheet, via ImageMagick's `montage` when installed). All model calls are injected, so `pnpm test` runs them against stubs — no model needed. The whole package is TypeScript: Pi loads the `.ts` files directly (no build step), and `pnpm check` (or `pnpm -r check` from the repo root) checks them. It ships no `main`/`exports` on purpose — Node refuses to strip types inside `node_modules`, so a programmatic `import "@computesdk/pi-loops"` would need a build; Pi loads the files through jiti and does not.
|
|
124
126
|
|
|
125
127
|
`test/fixtures/fuzzy/bench` is the test repo: `cp -r test/fixtures/fuzzy/bench /tmp/fuzzy && cd /tmp/fuzzy && git init && git add -A && git commit -m base`, then run the `/loop` above.
|
package/extensions/loop.ts
CHANGED
|
@@ -6,13 +6,14 @@ import fs from "node:fs";
|
|
|
6
6
|
import path from "node:path";
|
|
7
7
|
import { fileURLToPath } from "node:url";
|
|
8
8
|
import { randomBytes } from "node:crypto";
|
|
9
|
-
import { createAgentSession, DefaultResourceLoader, SessionManager, getAgentDir, type AgentSession, type ModelRegistry } from "@earendil-works/pi-coding-agent";
|
|
9
|
+
import { createAgentSession, createAgentSessionFromServices, createAgentSessionServices, DefaultResourceLoader, SessionManager, getAgentDir, withFileMutationQueue, type AgentSession, type ModelRegistry } from "@earendil-works/pi-coding-agent";
|
|
10
10
|
import { Text } from "@earendil-works/pi-tui";
|
|
11
11
|
import { Type } from "typebox";
|
|
12
12
|
import { pickWinner, panelJudge } from "../lib/judge.ts";
|
|
13
13
|
import { better as betterOf, git, restoreProtected, sh, diffText, errText, type GitRunner } from "../lib/measure.ts";
|
|
14
14
|
import { addRule, fromRules, newRubric, parseJson, rubricText, writeRubric } from "../lib/rubric.ts";
|
|
15
15
|
import { sheet } from "../lib/sheet.ts";
|
|
16
|
+
import { findChrome } from "../lib/shot.ts";
|
|
16
17
|
|
|
17
18
|
const HERE = path.dirname(fileURLToPath(import.meta.url));
|
|
18
19
|
/** Package root: `agents/`, `prompts/` and `lib/` sit one level up from `extensions/`. */
|
|
@@ -20,6 +21,46 @@ const ROOT = path.resolve(HERE, "..");
|
|
|
20
21
|
export const STYLES = ["safe", "bold", "wild"];
|
|
21
22
|
export const DEFAULTS = { attempts: 3, rounds: 1, timeoutMin: 15, tokenBudget: null, repeat: 2, minGain: 0.05, judgeSamples: 3, judgePairs: 2, judgePanel: true, council: true, models: {} };
|
|
22
23
|
const STATE_DIR = path.join(".pi", "loops");
|
|
24
|
+
/** Usage of a whole round: every attempt session, judge sample and council member, so the tool can report it. */
|
|
25
|
+
export type Meter = { input: number; output: number; cacheRead: number; cacheWrite: number; totalTokens: number; cost: { input: number; output: number; cacheRead: number; cacheWrite: number; total: number } };
|
|
26
|
+
export const newMeter = (): Meter => ({ input: 0, output: 0, cacheRead: 0, cacheWrite: 0, totalTokens: 0, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } });
|
|
27
|
+
/** Add one message's `usage`. Providers report what they report: anything missing counts as zero, never as NaN. */
|
|
28
|
+
export function addUsage(m: Meter | undefined, u: any) {
|
|
29
|
+
if (!m || !u) return;
|
|
30
|
+
m.input += u.input || 0; m.output += u.output || 0; m.cacheRead += u.cacheRead || 0; m.cacheWrite += u.cacheWrite || 0;
|
|
31
|
+
m.totalTokens += u.totalTokens || 0;
|
|
32
|
+
m.cost.input += u.cost?.input || 0; m.cost.output += u.cost?.output || 0;
|
|
33
|
+
m.cost.cacheRead += u.cost?.cacheRead || 0; m.cost.cacheWrite += u.cost?.cacheWrite || 0;
|
|
34
|
+
m.cost.total += typeof u.cost === "number" ? u.cost : u.cost?.total || 0;
|
|
35
|
+
}
|
|
36
|
+
/** `3.2k tok · $0.0410`, for the card and the result. */
|
|
37
|
+
export const meterText = (m: Meter | undefined) => m && m.totalTokens ? `${Math.round(m.totalTokens / 100) / 10}k tok · $${m.cost.total.toFixed(4)}` : "";
|
|
38
|
+
|
|
39
|
+
/** A `build` that is our own screenshot CLI (written by `routes`): it needs a browser, so check for one up front. */
|
|
40
|
+
const SELF_SHOT = /[\\/]shot\.ts/;
|
|
41
|
+
|
|
42
|
+
/** Can this model see images? `null` when the model object does not say (a stub, or a catalog entry without `input`). */
|
|
43
|
+
export const seesImages = (m: unknown): boolean | null => {
|
|
44
|
+
const input = (m as { input?: unknown } | null | undefined)?.input;
|
|
45
|
+
return Array.isArray(input) ? input.includes("image") : null;
|
|
46
|
+
};
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* A visual round is judged on screenshots, so a judge that cannot see them scores every candidate the same and the round
|
|
50
|
+
* reads as a tie: worse than not running, because it looks like it worked. Refuse up front instead. Bare-goal rounds are
|
|
51
|
+
* `kind: "visual"` too but are judged on diffs, so the gate is about screenshots, not the kind.
|
|
52
|
+
* Returns the problem, or null when every role that looks at pictures can.
|
|
53
|
+
*/
|
|
54
|
+
export const judgedOnScreenshots = (state: { kind?: string; shots?: string[] | null }) => state.kind === "visual" && !!state.shots?.length;
|
|
55
|
+
|
|
56
|
+
export function visionProblem(roles: { role: string; key: string; spec: string | null | undefined; model: unknown }[]): string | null {
|
|
57
|
+
for (const r of roles) {
|
|
58
|
+
if (r.model == null || seesImages(r.model) !== false) continue;
|
|
59
|
+
return `visual mode judges screenshots, and the ${r.role}${r.spec ? ` (${r.spec})` : ""} cannot see images — it would score every candidate the same. ` +
|
|
60
|
+
`Pick a vision model for the ${r.role} (/loop models, or "models": { "${r.key}": "…" } in .pi/loops.json), or pass \`bench\` so the round is measured instead of looked at.`;
|
|
61
|
+
}
|
|
62
|
+
return null;
|
|
63
|
+
}
|
|
23
64
|
|
|
24
65
|
/** One agent markdown file: frontmatter (name, model, tools, …) plus body. */
|
|
25
66
|
export type Agent = { role?: string; file?: string; name?: string; model?: string; tools?: string[]; body: string; [key: string]: any };
|
|
@@ -38,7 +79,13 @@ export type ModelRef = { provider: string; id: string };
|
|
|
38
79
|
/** A model as Pi hands it to extensions: what `createAgentSession` takes. */
|
|
39
80
|
export type Model = ReturnType<ModelRegistry["getAvailable"]>[number];
|
|
40
81
|
/** The model-registry methods Loops calls. */
|
|
41
|
-
export type ModelRegistryLike = {
|
|
82
|
+
export type ModelRegistryLike = {
|
|
83
|
+
getAvailable?: () => ModelRef[]; find?: (provider: string, id: string) => ModelRef | undefined;
|
|
84
|
+
/** Providers an *extension* registered (pi-particle, pi-morph, anything not built into Pi), with their config. */
|
|
85
|
+
getRegisteredProviderIds?: () => readonly string[];
|
|
86
|
+
getRegisteredProviderConfig?: (provider: string) => unknown;
|
|
87
|
+
getRegisteredNativeProvider?: (provider: string) => unknown;
|
|
88
|
+
};
|
|
42
89
|
/** The session-manager methods Loops calls. */
|
|
43
90
|
export type SessionManagerLike = { getBranch?: () => any[]; getSessionDir?: () => string; getSessionFile?: () => string | undefined };
|
|
44
91
|
/** The subset of Pi's UI the extension calls. */
|
|
@@ -74,6 +121,12 @@ export type ToolDef<P = any> = {
|
|
|
74
121
|
description: string;
|
|
75
122
|
parameters: any;
|
|
76
123
|
outputSchema?: any;
|
|
124
|
+
/** `model-only` for the tools that ask the user or orchestrate: never callable from scripts. */
|
|
125
|
+
exposure?: "direct" | "model-only" | "codemode" | "deferred" | "hidden";
|
|
126
|
+
/** "sequential" for tools that share in-memory state, so one message's calls cannot interleave. */
|
|
127
|
+
executionMode?: "sequential" | "parallel";
|
|
128
|
+
/** MCP tool hints: what a permission extension needs to decide whether to ask. */
|
|
129
|
+
annotations?: { readOnlyHint?: boolean; destructiveHint?: boolean; idempotentHint?: boolean; openWorldHint?: boolean };
|
|
77
130
|
execute: (id: string, params: P, signal: AbortSignal, onUpdate: (update: any) => void, ctx: Ctx) => Promise<any>;
|
|
78
131
|
renderCall?: (args: any, theme: any) => any;
|
|
79
132
|
renderResult?: (result: any, options: { expanded: boolean; isPartial: boolean }, theme: any) => any;
|
|
@@ -166,6 +219,10 @@ export function loopSettings(pi: Pi | null, cwd: string, agentDir: string = getA
|
|
|
166
219
|
return { ...DEFAULTS, ...s, models: { ...DEFAULTS.models, ...(s.models ?? {}) } };
|
|
167
220
|
}
|
|
168
221
|
/** Merge `patch` into <repo>/.pi/loops.json (models merged one level deeper). Throws when the file exists but isn't a JSON object. */
|
|
222
|
+
/** `saveConfig` is a read-merge-write, so the whole of it takes the file queue; two tools must not interleave. */
|
|
223
|
+
export const saveConfigQueued = (cwd: string, patch: Record<string, any>) =>
|
|
224
|
+
withFileMutationQueue(configFiles(cwd).project, async () => saveConfig(cwd, patch));
|
|
225
|
+
|
|
169
226
|
export function saveConfig(cwd: string, patch: Record<string, any>): Record<string, any> {
|
|
170
227
|
const f = configFiles(cwd).project;
|
|
171
228
|
const cur = readJson(f);
|
|
@@ -302,17 +359,45 @@ const release = (session: AgentSession) => { active.delete(session); try { sessi
|
|
|
302
359
|
export async function stopAll() { shuttingDown = true; const all = [...active]; active.clear(); await Promise.allSettled(all.map((s) => s.abort?.().catch(() => {}))); for (const s of all) { try { s.dispose(); } catch {} } return all.length; }
|
|
303
360
|
|
|
304
361
|
/* ---------- one-shot agent sessions (visual judge panel / council): no tools, Pi auth + session header ---------- */
|
|
305
|
-
|
|
362
|
+
/**
|
|
363
|
+
* A one-shot session (judge, council) loads no extensions — that keeps it lean and tool-free — so a provider that an
|
|
364
|
+
* *extension* adds (rather than one built into Pi) is unknown inside it and its credential cannot be resolved. Say that
|
|
365
|
+
* instead of pi's "No API key found", which points at /login and sends people the wrong way.
|
|
366
|
+
*/
|
|
367
|
+
function explainAuth(e: unknown, role: string, spec: string | null, ctx: Ctx): unknown {
|
|
368
|
+
const provider = errText(e).match(/No API key found for ([\w.-]+)/)?.[1]?.replace(/\.+$/, ""); // pi ends the sentence with a period
|
|
369
|
+
if (!provider) return e;
|
|
370
|
+
// the host can see a model on that provider (so this session is missing something the host has), not a bare missing login
|
|
371
|
+
const hostKnows = (ctx.modelRegistry?.getAvailable?.() ?? []).some((m) => (m as { provider?: string }).provider === provider);
|
|
372
|
+
if (!hostKnows) return e;
|
|
373
|
+
return new Error(`the ${role} cannot run on ${spec ?? provider}: "${provider}" is a Pi extension's provider, and a one-shot ${role} session loads no extensions, so it has no credentials there — give the ${role} a provider built into Pi (attempts can still use ${provider}).`, { cause: e });
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
async function oneShot({ role, model, content, cwd, ctx, settings, meter, signal }: { role: string; model?: string | null; content: any; cwd: string; ctx: Ctx; settings: LoopSettings; meter?: Meter; signal?: AbortSignal }) {
|
|
306
377
|
const agentDir = getAgentDir();
|
|
378
|
+
const spec = model ?? settings.models[role] ?? null;
|
|
307
379
|
const agent = loadAgent(role, { cwd, agentDir });
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
const { session: created } = await createAgentSession({ cwd, agentDir, model: pickModel(ctx, model ?? settings.models[role] ?? agent.model), noTools: "all", resourceLoader: loader, sessionManager: SessionManager.inMemory(cwd) });
|
|
311
|
-
const session = track(created);
|
|
312
|
-
const { text, images } = flattenParts(content);
|
|
313
|
-
const onAbort = () => session.abort().catch(() => {});
|
|
314
|
-
signal?.addEventListener("abort", onAbort, { once: true });
|
|
380
|
+
let session: AgentSession | null = null;
|
|
381
|
+
const onAbort = () => session?.abort().catch(() => {});
|
|
315
382
|
try {
|
|
383
|
+
// what a one-shot wants: no extensions, no skills, no tools — just the role's system prompt and one answer
|
|
384
|
+
const services = await createAgentSessionServices({ cwd, agentDir,
|
|
385
|
+
resourceLoaderOptions: { noExtensions: true, noSkills: true, noPromptTemplates: true, noThemes: true, noContextFiles: true, systemPrompt: agent.body } });
|
|
386
|
+
// "no extensions" would also drop the *providers* extensions add (anything not built into Pi), and an unregistered
|
|
387
|
+
// provider has no credentials. Copy the host's registrations over: config only, no extension code runs, still no tools.
|
|
388
|
+
const known = new Set(services.modelRuntime.getRegisteredProviderIds());
|
|
389
|
+
for (const id of ctx.modelRegistry?.getRegisteredProviderIds?.() ?? []) {
|
|
390
|
+
if (known.has(id)) continue;
|
|
391
|
+
const config = ctx.modelRegistry?.getRegisteredProviderConfig?.(id);
|
|
392
|
+
if (config) { services.modelRuntime.registerProvider(id, config as any); continue; }
|
|
393
|
+
const native = ctx.modelRegistry?.getRegisteredNativeProvider?.(id); // a provider built with createProvider, not config
|
|
394
|
+
if (native) services.modelRuntime.registerNativeProvider(native as any);
|
|
395
|
+
}
|
|
396
|
+
// creating the session resolves the credential too, so it belongs inside this try: that is where an unusable provider shows up
|
|
397
|
+
const { session: created } = await createAgentSessionFromServices({ services, sessionManager: SessionManager.inMemory(cwd), model: pickModel(ctx, spec ?? agent.model), noTools: "all" });
|
|
398
|
+
session = track(created);
|
|
399
|
+
const { text, images } = flattenParts(content);
|
|
400
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
316
401
|
await session.prompt(text, { images, expandPromptTemplates: false });
|
|
317
402
|
let last = session.messages.filter((m) => m.role === "assistant").pop();
|
|
318
403
|
if (last?.stopReason === "error") throw new Error(last.errorMessage || `${role} errored`);
|
|
@@ -322,16 +407,23 @@ async function oneShot({ role, model, content, cwd, ctx, settings, signal }: { r
|
|
|
322
407
|
if (last?.stopReason === "error") throw new Error(last.errorMessage || `${role} errored`);
|
|
323
408
|
}
|
|
324
409
|
return parseJson(session.getLastAssistantText());
|
|
325
|
-
}
|
|
410
|
+
} catch (e) { throw explainAuth(e, role, spec, ctx); }
|
|
411
|
+
finally {
|
|
412
|
+
signal?.removeEventListener("abort", onAbort);
|
|
413
|
+
if (session) {
|
|
414
|
+
// a one-shot is a whole session with real spend: hand it to the round's meter before the session goes away
|
|
415
|
+
for (const m of session.messages) if (m.role === "assistant") addUsage(meter, (m as { usage?: unknown }).usage);
|
|
416
|
+
release(session);
|
|
417
|
+
}
|
|
418
|
+
}
|
|
326
419
|
}
|
|
327
420
|
|
|
328
421
|
/* ---------- an attempt: forked session + worktree + style agent ---------- */
|
|
329
|
-
async function runAttempt({ state, style, n, dir, ctx, settings, model, signal, onProgress, live }: { state: State; style: string; n: number; dir: string; ctx:
|
|
422
|
+
async function runAttempt({ state, style, n, dir, cwd, ctx, settings, model, meter, sessionDir, src, signal, onProgress, live }: { state: State; style: string; n: number; dir: string; cwd: string; ctx: PickCtx; settings: LoopSettings; model?: string | null; meter?: Meter; sessionDir?: string; src?: string | null; signal?: AbortSignal; onProgress: (progress: any) => void; live?: any }): Promise<Attempt> {
|
|
330
423
|
const agentDir = getAgentDir();
|
|
331
|
-
const agent = loadAgent(style, { cwd
|
|
332
|
-
const
|
|
333
|
-
const
|
|
334
|
-
const sessionManager = src && fs.existsSync(src) ? SessionManager.forkFrom(src, dir, sessionDir) : SessionManager.create(dir, sessionDir);
|
|
424
|
+
const agent = loadAgent(style, { cwd, agentDir });
|
|
425
|
+
const fork = state.best.session ?? src;
|
|
426
|
+
const sessionManager = fork && fs.existsSync(fork) ? SessionManager.forkFrom(fork, dir, sessionDir) : SessionManager.create(dir, sessionDir);
|
|
335
427
|
const loader = new DefaultResourceLoader({ cwd: dir, agentDir, noExtensions: true, noPromptTemplates: true, noThemes: true, appendSystemPrompt: [agent.body] });
|
|
336
428
|
await loader.reload();
|
|
337
429
|
const picked = pickModel(ctx, model ?? settings.models[style] ?? agent.model);
|
|
@@ -346,6 +438,7 @@ async function runAttempt({ state, style, n, dir, ctx, settings, model, signal,
|
|
|
346
438
|
else if (e.type === "message_end" && e.message?.role === "assistant") {
|
|
347
439
|
p.tokens += e.message.usage?.totalTokens || 0;
|
|
348
440
|
p.cost += typeof e.message.usage?.cost === "number" ? e.message.usage.cost : e.message.usage?.cost?.total || 0;
|
|
441
|
+
addUsage(meter, e.message.usage);
|
|
349
442
|
const t = (e.message.content || []).filter((c) => c.type === "text").map((c) => c.text).join("").trim();
|
|
350
443
|
if (t) p.text = t.slice(-200);
|
|
351
444
|
onProgress(p);
|
|
@@ -641,7 +734,7 @@ export async function pickModels(ctx: PickCtx, { reason }: { reason?: string } =
|
|
|
641
734
|
if (models.council.length < 2) models.council.push(judge);
|
|
642
735
|
const file = configFiles(ctx.cwd).project;
|
|
643
736
|
let saved;
|
|
644
|
-
try { saved =
|
|
737
|
+
try { saved = await saveConfigQueued(ctx.cwd, { models }); } catch (e) { const error = `${path.relative(ctx.cwd, file)}: ${errText(e).slice(0, 80)}; fix it and run /loop models again`; ctx.ui.notify(error, "error"); return { error }; }
|
|
645
738
|
return { models: saved.models, file: path.relative(ctx.cwd, file) };
|
|
646
739
|
}
|
|
647
740
|
|
|
@@ -697,7 +790,7 @@ export async function loopsCommand(args: string, ctx: Ctx, pi: Pi): Promise<void
|
|
|
697
790
|
const [first, ...rest] = (args ?? "").trim().split(/\s+/).filter(Boolean);
|
|
698
791
|
const sub = first?.toLowerCase();
|
|
699
792
|
const settings = loopSettings(pi, ctx.cwd);
|
|
700
|
-
const set = (patch: Record<string, any>, what: string) => { try {
|
|
793
|
+
const set = async (patch: Record<string, any>, what: string) => { try { await saveConfigQueued(ctx.cwd, patch); ctx.ui.notify(`${path.relative(ctx.cwd, configFiles(ctx.cwd).project)}: ${what}`, "info"); } catch (e) { ctx.ui.notify(errText(e), "error"); } };
|
|
701
794
|
switch (sub) {
|
|
702
795
|
case undefined: {
|
|
703
796
|
let last: State | null = null; try { last = loadState(ctx.cwd, sessionRuns(ctx).find((id) => fs.existsSync(stateFile(ctx.cwd, id))) ?? "last"); } catch {}
|
|
@@ -728,7 +821,9 @@ export default function loopsExtension(pi: Pi): void {
|
|
|
728
821
|
await pi.sendUserMessage?.(loopPrompt(args, ctx.cwd));
|
|
729
822
|
},
|
|
730
823
|
});
|
|
731
|
-
|
|
824
|
+
// state is module-scoped, so a session that starts after one has shut down must not inherit its latch
|
|
825
|
+
pi.on?.("session_start", async () => { shuttingDown = false; RUNS.clear(); });
|
|
826
|
+
pi.on?.("session_shutdown", async () => { shuttingDown = true; for (const r of RUNS.values()) r.ctrl.abort(); await stopAll(); });
|
|
732
827
|
async function run(params: LoopParamsIn, signal: AbortSignal | undefined, onUpdate: ((update: any) => void) | undefined, ctx: Ctx, live: any) {
|
|
733
828
|
const cwd = ctx.cwd;
|
|
734
829
|
let settings = loopSettings(pi, cwd);
|
|
@@ -763,17 +858,37 @@ export default function loopsExtension(pi: Pi): void {
|
|
|
763
858
|
}
|
|
764
859
|
state = { v: 2, id, goal: params.goal, kind, editedBaseline: !!stashed, check: params.check ?? settings.check, bench: params.bench, metric: { key: "ms", unit: kind === "bench" ? "ms" : "", better: "lower", ...(params.bench && params.bench === settings.bench ? settings.metric ?? {} : {}), ...(params.metric ?? {}) }, protect: params.protect ?? settings.protect ?? [], build, shots, rules: params.rules ?? [], directions: params.directions ?? {}, baseCommit, best: { label: "base", commit: baseCommit, session: null, score: null, reason: null }, winner: null, status: "running", rounds: [], rubric: null, parentSession: ctx.sessionManager?.getSessionFile?.() ?? null };
|
|
765
860
|
}
|
|
861
|
+
// a run that rebuilds its shots with our CLI needs a browser: fail once here, before three attempt sessions and a
|
|
862
|
+
// judge panel start, whatever wrote the `build` (routes, resume, or by hand). The message names the fix.
|
|
863
|
+
if (SELF_SHOT.test(state.build ?? "")) findChrome();
|
|
864
|
+
// a visual round is only as good as the eyes on it: refuse a judge (or panel member) that cannot see the screenshots
|
|
865
|
+
if (judgedOnScreenshots(state)) {
|
|
866
|
+
const asModel = (spec: string | null | undefined) => { try { return pickModel(ctx, spec); } catch { return null; } }; // an unavailable model reports itself later
|
|
867
|
+
const panelOn = settings.council !== false && settings.judgePanel !== false && (models.council?.length ?? 0) > 1;
|
|
868
|
+
const problem = visionProblem([
|
|
869
|
+
{ role: "judge", key: "judge", spec: models.judge, model: asModel(models.judge) },
|
|
870
|
+
...(panelOn ? (models.council ?? []).map((spec, i) => ({ role: `panel member ${i + 1}`, key: "council", spec, model: asModel(spec) })) : []),
|
|
871
|
+
]);
|
|
872
|
+
if (problem) throw new Error(problem);
|
|
873
|
+
}
|
|
766
874
|
state.status = "running";
|
|
767
875
|
saveState(cwd, state);
|
|
768
876
|
const link = () => { try { pi.appendEntry?.("loops-run", { id: state.id, status: state.status, goal: state.goal, kind: state.kind, best: state.best?.label ?? null, rounds: state.rounds?.length ?? 0 }); } catch {} };
|
|
769
877
|
link();
|
|
770
878
|
if (state.shots?.length) { try { excludeShots(cwd, state.shots); } catch {} }
|
|
771
879
|
const runDir = path.join(cwd, STATE_DIR, state.id);
|
|
880
|
+
// every nested model call in this round — attempts, judge samples, council — adds to one meter, reported as the result's `usage`
|
|
881
|
+
const meter = newMeter();
|
|
882
|
+
// a TUI call returns as soon as the round starts, and session replacement invalidates the old ctx: capture what the
|
|
883
|
+
// background run needs as plain data now (the model registry is the one shared object that stays valid)
|
|
884
|
+
const sessionDir = ctx.sessionManager?.getSessionDir?.();
|
|
885
|
+
const sessionFile = ctx.sessionManager?.getSessionFile?.() ?? null;
|
|
886
|
+
const setStatus = (text?: string) => { try { ctx.ui.setStatus?.("loops", text); } catch {} };
|
|
772
887
|
const details: Details = { run: state.id, kind: state.kind, round: null, judge: null, attempts: [], phase: "starting" };
|
|
773
888
|
if (live) { live.id = state.id; live.state = state; live.details = details; live.cwd = cwd; live.started?.(); }
|
|
774
|
-
const push = (phase?: string) => { details.phase = phase ?? details.phase; if (live)
|
|
775
|
-
const askModel = (role: string, i: number = 0) => (content: any) => oneShot({ role, model: role === "council" ? (models.council ?? [])[i % (models.council?.length || 1)] : models[role], content, cwd, ctx, settings, signal });
|
|
776
|
-
const askAs = (role: string, model?: string | null) => (content: any) => oneShot({ role, model, content, cwd, ctx, settings, signal });
|
|
889
|
+
const push = (phase?: string) => { details.phase = phase ?? details.phase; if (live) setStatus(`${state.id} r${details.round ?? "?"} · ${details.phase}`); onUpdate?.({ content: [{ type: "text", text: `${state.id} r${details.round ?? "?"} · ${details.phase}\n${details.attempts.map((a) => ` ${a.style.padEnd(5)} ${a.status.padEnd(8)} ${a.tool ? `[${a.tool}] ` : ""}${a.tools} tools, ${a.tokens} tok${a.text ? ` — ${a.text.split("\n").pop()?.slice(0, 80) ?? ""}` : ""}`).join("\n")}` }], details: structuredClone(details) }); };
|
|
890
|
+
const askModel = (role: string, i: number = 0) => (content: any) => oneShot({ role, model: role === "council" ? (models.council ?? [])[i % (models.council?.length || 1)] : models[role], content, cwd, ctx, settings, meter, signal });
|
|
891
|
+
const askAs = (role: string, model?: string | null) => (content: any) => oneShot({ role, model, content, cwd, ctx, settings, meter, signal });
|
|
777
892
|
// the judge panel: every council model (the attempts' models) scores every candidate, labels blind; the judge model keeps vetoes and the head-to-head
|
|
778
893
|
const panel = settings.council !== false && settings.judgePanel !== false && (models.council?.length ?? 0) > 1 ? (models.council ?? []).map((m) => ({ name: shortModel(m), ask: askAs("judge", m) })) : undefined;
|
|
779
894
|
state.models = { ...Object.fromEntries(STYLES.map((st) => [st, models[st]])), judge: models.judge, council: models.council, spread: models.spread };
|
|
@@ -810,7 +925,7 @@ export default function loopsExtension(pi: Pi): void {
|
|
|
810
925
|
try { git(cwd, "worktree", "remove", "-f", dir); } catch {}
|
|
811
926
|
git(cwd, "worktree", "add", "-q", "-f", "-B", branch, dir, round.from ?? state.baseCommit);
|
|
812
927
|
const slot = details.attempts.find((a) => a.style === style)!;
|
|
813
|
-
const r = await runAttempt({ state, style, n, dir, ctx, settings, model: models[style], signal, live, onProgress: (p) => { Object.assign(slot, p); push(); } });
|
|
928
|
+
const r = await runAttempt({ state, style, n, dir, cwd, ctx, settings, model: models[style], meter, sessionDir, src: sessionFile, signal, live, onProgress: (p) => { Object.assign(slot, p); push(); } });
|
|
814
929
|
Object.assign(slot, { status: r.status, tool: null, text: r.summary || r.error || "" }); push();
|
|
815
930
|
return { ...r, branch, dir: path.relative(cwd, dir) };
|
|
816
931
|
}));
|
|
@@ -931,13 +1046,14 @@ export default function loopsExtension(pi: Pi): void {
|
|
|
931
1046
|
link();
|
|
932
1047
|
const text = (aborted ? `Aborted during round ${details.round}; attempts' worktrees are kept under ${path.relative(cwd, runDir)}/. Resume with loop({ from: "${state.id}" }).\n\n` : "") + resultText(state, done, cwd);
|
|
933
1048
|
const lastSheet = [...done].reverse().find((r) => r.sheet)?.sheet;
|
|
934
|
-
const
|
|
1049
|
+
const spend = meterText(meter);
|
|
1050
|
+
const content: any[] = [{ type: "text", text: spend ? `${text}\nSpent ${spend} on this run's models.` : text }];
|
|
935
1051
|
if (lastSheet) { try { content.push({ type: "image", data: fs.readFileSync(path.join(cwd, lastSheet)).toString("base64"), mimeType: "image/png" }); } catch {} }
|
|
936
|
-
return { content, structuredContent: structuredResult(state, state.rounds), details: { ...structuredClone(details), phase: aborted ? "aborted" : "done", status: state.status, best: state.best, winner: state.winner, rounds: done.map((r) => ({ n: r.n, pick: r.judge?.pick ?? null, why: r.judge?.why ?? null })) } };
|
|
1052
|
+
return { content, usage: meter.totalTokens ? meter : undefined, structuredContent: structuredResult(state, state.rounds), details: { ...structuredClone(details), phase: aborted ? "aborted" : "done", status: state.status, best: state.best, winner: state.winner, spent: spend || null, rounds: done.map((r) => ({ n: r.n, pick: r.judge?.pick ?? null, why: r.judge?.why ?? null })) } };
|
|
937
1053
|
}
|
|
938
1054
|
|
|
939
1055
|
pi.registerTool?.({
|
|
940
|
-
name: "loop",
|
|
1056
|
+
name: "loop", exposure: "model-only", executionMode: "sequential", annotations: { readOnlyHint: false, destructiveHint: true, idempotentHint: false, openWorldHint: true },
|
|
941
1057
|
label: "loop",
|
|
942
1058
|
outputSchema: LoopResult,
|
|
943
1059
|
description: "Run hill-climbing round(s) on the current git repo: three forked attempt sessions (safe/bold/wild agents) in their own worktrees, a judge keeps the best. Bench mode when `bench` is set (each attempt sets up and measures itself; a judge agent re-measures every candidate, reads the diffs and vetoes rule breaks); visual mode otherwise (`build` writes `shots`, a judge panel scores them). `from` continues a run from its last judged round.",
|
|
@@ -984,7 +1100,7 @@ export default function loopsExtension(pi: Pi): void {
|
|
|
984
1100
|
});
|
|
985
1101
|
const said = (t: string) => ({ content: [{ type: "text", text: t }] });
|
|
986
1102
|
const idParam = Type.Optional(Type.String({ description: "Run id; default: the round started most recently" }));
|
|
987
|
-
pi.registerTool?.({ name: "loop_init", label: "loop_init", description: "Project setup for loops, written once to .pi/loops.json: serve (start the app on $PORT), check, bench + metric, protect. In the TUI it shows them and asks Save / Change / Discuss.", parameters: InitParams,
|
|
1103
|
+
pi.registerTool?.({ name: "loop_init", exposure: "model-only", executionMode: "sequential", annotations: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: false }, label: "loop_init", description: "Project setup for loops, written once to .pi/loops.json: serve (start the app on $PORT), check, bench + metric, protect. In the TUI it shows them and asks Save / Change / Discuss.", parameters: InitParams,
|
|
988
1104
|
async execute(_id, p, _signal, _onUpdate, ctx) {
|
|
989
1105
|
const fields = Object.fromEntries(SETUP_KEYS.filter((k) => p[k] != null && (!Array.isArray(p[k]) || p[k].length)).map((k) => [k, p[k]]));
|
|
990
1106
|
if (ctx.hasUI) {
|
|
@@ -992,28 +1108,29 @@ export default function loopsExtension(pi: Pi): void {
|
|
|
992
1108
|
if (choice?.startsWith("Change")) { const note = (await ctx.ui.input?.("What should change?", "e.g. serve is npm start -- -p $PORT; no check"))?.trim(); return { content: [{ type: "text", text: note ? `Not saved. Change requested: ${note}\nRevise and call loop_init again.` : "Not saved. Revise with me in chat, then call loop_init again." }], details: { phase: "change", note } }; }
|
|
993
1109
|
if (choice !== "Save") return { content: [{ type: "text", text: "Not saved. Discuss the setup with me in chat, then call loop_init again." }], details: { phase: "discuss" } };
|
|
994
1110
|
}
|
|
995
|
-
|
|
1111
|
+
await saveConfigQueued(ctx.cwd, { ...Object.fromEntries(SETUP_KEYS.map((k) => [k, null])), ...fields });
|
|
996
1112
|
return { content: [{ type: "text", text: `Saved ${path.relative(ctx.cwd, configFiles(ctx.cwd).project)}:\n${setupText(fields)}` }], details: { phase: "saved", setup: fields } };
|
|
997
1113
|
} });
|
|
998
|
-
pi.registerTool?.({ name: "loop_status", label: "loop_status", description: "Progress of a loop round running in the background: phase, each attempt's status, tools and tokens.", parameters: Type.Object({ id: idParam }),
|
|
1114
|
+
pi.registerTool?.({ name: "loop_status", annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }, label: "loop_status", description: "Progress of a loop round running in the background: phase, each attempt's status, tools and tokens.", parameters: Type.Object({ id: idParam }),
|
|
999
1115
|
async execute(_id, p) { const r = liveRun(p.id); return { content: r.last?.content ?? [{ type: "text", text: `${r.id} starting` }], details: r.details }; } });
|
|
1000
|
-
pi.registerTool?.({ name: "loop_talk", label: "loop_talk", description: "Tell a running attempt something (R1-bold, or just bold): delivered into its session as a steering message it acts on next.", parameters: Type.Object({ attempt: Type.String(), message: Type.String(), id: idParam }),
|
|
1116
|
+
pi.registerTool?.({ name: "loop_talk", executionMode: "sequential", annotations: { readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: true }, label: "loop_talk", description: "Tell a running attempt something (R1-bold, or just bold): delivered into its session as a steering message it acts on next.", parameters: Type.Object({ attempt: Type.String(), message: Type.String(), id: idParam }),
|
|
1001
1117
|
async execute(_id, p) { const r = liveRun(p.id); const key = findAttempt(r, p.attempt); if (!key) throw new Error(`${p.attempt} is not running (running: ${[...r.attempts.keys()].join(", ") || "none"})`); await r.attempts.get(key).steer(`Message from the person running this loop: ${p.message}`); return said(`told ${key}: ${p.message}`); } });
|
|
1002
|
-
pi.registerTool?.({ name: "loop_rule", label: "loop_rule", description: "Add a rule to a running loop: every running attempt is told now, the judge enforces it from this round on, and it stays for later rounds.", parameters: Type.Object({ rule: Type.String(), id: idParam }),
|
|
1118
|
+
pi.registerTool?.({ name: "loop_rule", executionMode: "sequential", annotations: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: false }, label: "loop_rule", description: "Add a rule to a running loop: every running attempt is told now, the judge enforces it from this round on, and it stays for later rounds.", parameters: Type.Object({ rule: Type.String(), id: idParam }),
|
|
1003
1119
|
async execute(_id, p) { const r = liveRun(p.id); if (!r.state.rules.includes(p.rule)) { r.state.rules.push(p.rule); if (r.state.rubric) addRule(r.state.rubric, p.rule); saveState(r.cwd, r.state); } await Promise.allSettled([...r.attempts.values()].map((s) => s.steer(`New rule for this loop, from the person running it: ${p.rule}`))); return said(`rule added: ${p.rule} (told ${r.attempts.size} running attempt${r.attempts.size === 1 ? "" : "s"})`); } });
|
|
1004
|
-
pi.registerTool?.({ name: "loop_pick", label: "loop_pick", description: "Decide the running round yourself: when the judge is done, keep this attempt (R1-wild, or just wild; or CURRENT) as the best if its check passes, whatever the judge scored.", parameters: Type.Object({ attempt: Type.String(), id: idParam }),
|
|
1120
|
+
pi.registerTool?.({ name: "loop_pick", executionMode: "sequential", annotations: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: false }, label: "loop_pick", description: "Decide the running round yourself: when the judge is done, keep this attempt (R1-wild, or just wild; or CURRENT) as the best if its check passes, whatever the judge scored.", parameters: Type.Object({ attempt: Type.String(), id: idParam }),
|
|
1005
1121
|
async execute(_id, p) { const r = liveRun(p.id); r.pick = p.attempt; return said(`when round ${r.details?.round ?? "?"} of ${r.id} is judged, ${p.attempt} is kept`); } });
|
|
1006
|
-
pi.registerTool?.({ name: "loop_apply", label: "loop_apply", description: "Take a run's best into the checkout: checks out the winner's changed tracked files from its commit and commits them (files it added or deleted are listed, not applied). Default: this session's last run.", parameters: Type.Object({ id: idParam }),
|
|
1122
|
+
pi.registerTool?.({ name: "loop_apply", executionMode: "sequential", annotations: { readOnlyHint: false, destructiveHint: true, idempotentHint: true, openWorldHint: false }, label: "loop_apply", description: "Take a run's best into the checkout: checks out the winner's changed tracked files from its commit and commits them (files it added or deleted are listed, not applied). Default: this session's last run.", parameters: Type.Object({ id: idParam }),
|
|
1007
1123
|
async execute(_id, p, _signal, _onUpdate, ctx) {
|
|
1008
1124
|
const mine = !p.id || p.id === "last" ? sessionRuns(ctx).find((id) => fs.existsSync(stateFile(ctx.cwd, id))) : null;
|
|
1009
1125
|
const state = loadState(ctx.cwd, mine ?? p.id);
|
|
1010
1126
|
if (RUNS.has(state.id)) throw new Error(`${state.id} is still running — loop_stop it or wait for the card`);
|
|
1011
|
-
|
|
1127
|
+
// a git checkout + commit in the user's checkout: serialise it against other mutations of that repo
|
|
1128
|
+
const r = await withFileMutationQueue(path.join(ctx.cwd, ".git"), async () => applyBest(ctx.cwd, state));
|
|
1012
1129
|
const lines = [r.commit ? `Applied ${state.best.label} (${state.best.commit.slice(0, 7)}) as ${r.commit.slice(0, 7)}: ${r.applied.join(", ")}` : r.already ? `${state.best.label} is already in the checkout.` : "Nothing applied — the winner changed no tracked file."];
|
|
1013
1130
|
if (r.files.added.length) lines.push(`Not applied — files it added: ${r.files.added.join(", ")} (git checkout ${state.best.commit.slice(0, 7)} -- <file> to take one)`);
|
|
1014
1131
|
if (r.files.removed.length) lines.push(`Not applied — files it deleted: ${r.files.removed.join(", ")}`);
|
|
1015
1132
|
return { ...said(lines.join("\n")), details: { run: state.id, applied: r.applied, commit: r.commit, files: r.files } };
|
|
1016
1133
|
} });
|
|
1017
|
-
pi.registerTool?.({ name: "loop_stop", label: "loop_stop", description: "Stop a round running in the background: aborts its attempts and judge. What they committed stays in the worktrees; loop({ from }) resumes the run.", parameters: Type.Object({ id: idParam }),
|
|
1134
|
+
pi.registerTool?.({ name: "loop_stop", executionMode: "sequential", annotations: { readOnlyHint: false, destructiveHint: true, idempotentHint: true, openWorldHint: false }, label: "loop_stop", description: "Stop a round running in the background: aborts its attempts and judge. What they committed stays in the worktrees; loop({ from }) resumes the run.", parameters: Type.Object({ id: idParam }),
|
|
1018
1135
|
async execute(_id, p) { const r = liveRun(p.id); r.ctrl.abort(); await r.promise.catch(() => {}); return said(`${r.id} stopped. Resume with loop({ from: "${r.id}" }).`); } });
|
|
1019
1136
|
}
|
package/lib/judge.ts
CHANGED
|
@@ -139,7 +139,7 @@ export async function panelJudge(o: JudgeOptions): Promise<PanelResult> {
|
|
|
139
139
|
const per = Math.max(1, Math.ceil(n / panel.length));
|
|
140
140
|
const settled = await Promise.allSettled(panel.flatMap((m) => Array.from({ length: per }, () => scoreOnce({ ...o, ask: m.ask }).then((r) => ({ ...r, by: m.name })))));
|
|
141
141
|
const runs = settled.filter((r): r is PromiseFulfilledResult<any> => r.status === "fulfilled").map((r) => r.value);
|
|
142
|
-
if (!runs.length) throw new Error(`judge failed (all samples): ${[...new Set(settled.map((s) => String(s.status === "rejected" ? s.reason?.message ?? s.reason : "unknown").slice(0,
|
|
142
|
+
if (!runs.length) throw new Error(`judge failed (all samples): ${[...new Set(settled.map((s) => String(s.status === "rejected" ? s.reason?.message ?? s.reason : "unknown").slice(0, 240)))].join(" | ")}`);
|
|
143
143
|
const labels = o.candidates.map((c) => c.label);
|
|
144
144
|
const scores: Record<string, number> = {}, spread: Record<string, number> = {}, reasons: Record<string, string> = {};
|
|
145
145
|
for (const l of labels) {
|
package/lib/shot.ts
CHANGED
|
@@ -1,26 +1,63 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
// Screenshot a route of a web app with headless Chrome,
|
|
2
|
+
// Screenshot a route of a web app with headless Chrome, driving it over the Chrome DevTools Protocol,
|
|
3
|
+
// starting the app first when asked. No dependencies: CDP over Node's built-in WebSocket.
|
|
3
4
|
// node shot.ts --url /pricing --out shots/pricing.png --serve "npm start" (app is given $PORT)
|
|
4
5
|
// node shot.ts --url http://localhost:3000/ --out shots/home.png (already running)
|
|
5
6
|
// Options: --port N (default: a free one), --width 1440 --height 900, --wait ms (settle time after load, default 1500),
|
|
6
|
-
// --timeout s (how long to wait for the server, default 180), --
|
|
7
|
-
|
|
7
|
+
// --timeout s (how long to wait for the server, default 180), --load-timeout s (the page, default 30),
|
|
8
|
+
// --chrome <path> (or $LOOPS_CHROME / $CHROME),
|
|
9
|
+
// --viewport (capture the viewport only; the default is the whole page), --selector <css> (capture one element).
|
|
10
|
+
// The default is the whole page, not the fold: a judge scoring a screenshot should see all of it.
|
|
11
|
+
// Any Chrome/Chromium works, including chrome-headless-shell — the headless binary Playwright installs, which has
|
|
12
|
+
// no `--screenshot` flag at all and can only be driven over CDP. That is why this file speaks CDP.
|
|
13
|
+
import { spawn, type ChildProcess } from "node:child_process";
|
|
8
14
|
import fs from "node:fs";
|
|
9
15
|
import net from "node:net";
|
|
16
|
+
import os from "node:os";
|
|
10
17
|
import path from "node:path";
|
|
11
18
|
import { fileURLToPath } from "node:url";
|
|
12
19
|
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
"
|
|
20
|
+
/** Commands looked up on $PATH, in this order. The last three are the headless-only binaries. */
|
|
21
|
+
const NAMES = ["google-chrome", "google-chrome-stable", "chromium", "chromium-browser", "chrome", "brave-browser", "microsoft-edge",
|
|
22
|
+
"chrome-headless-shell", "headless-shell", "headless_shell"];
|
|
23
|
+
/** Absolute paths, for installs that are not on $PATH. */
|
|
24
|
+
const PATHS = ["/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", "/Applications/Chromium.app/Contents/MacOS/Chromium",
|
|
25
|
+
"/Applications/Brave Browser.app/Contents/MacOS/Brave Browser", "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge",
|
|
26
|
+
"/headless-shell/headless-shell", // exe.dev VMs ship the headless shell Shelley drives here
|
|
27
|
+
"/usr/lib/chromium/chromium", "/snap/bin/chromium", "/opt/google/chrome/chrome"];
|
|
16
28
|
|
|
29
|
+
const which = (cmd: string) => { for (const dir of (process.env.PATH ?? "").split(path.delimiter)) { const p = path.join(dir, cmd); if (fs.existsSync(p)) return p; } return null; };
|
|
30
|
+
const isPath = (c: string) => c.includes("/");
|
|
31
|
+
|
|
32
|
+
/** Newest first: Playwright's cache is versioned, and the last install is the one to use. */
|
|
33
|
+
function playwrightShells(): string[] {
|
|
34
|
+
const root = process.env.PLAYWRIGHT_BROWSERS_PATH || path.join(os.homedir(), ".cache", "ms-playwright");
|
|
35
|
+
let entries: string[];
|
|
36
|
+
try { entries = fs.readdirSync(root); } catch { return []; }
|
|
37
|
+
const found: string[] = [];
|
|
38
|
+
for (const e of entries.sort().reverse()) {
|
|
39
|
+
if (e.startsWith("chromium_headless_shell-")) found.push(path.join(root, e, "chrome-headless-shell-linux64", "chrome-headless-shell"), path.join(root, e, "chrome-headless-shell-mac-arm64", "chrome-headless-shell"), path.join(root, e, "chrome-headless-shell-mac-x64", "chrome-headless-shell"));
|
|
40
|
+
else if (e.startsWith("chromium-")) found.push(path.join(root, e, "chrome-linux", "chrome"), path.join(root, e, "chrome-mac", "Chromium.app", "Contents", "MacOS", "Chromium"));
|
|
41
|
+
}
|
|
42
|
+
return found;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** Where a browser could be, in the order we trust it: what you asked for, $PATH, known install dirs. */
|
|
46
|
+
export function chromeCandidates(explicit?: string): string[] {
|
|
47
|
+
const named = [explicit, process.env.LOOPS_CHROME, process.env.CHROME, process.env.PUPPETEER_EXECUTABLE_PATH].filter((c): c is string => !!c);
|
|
48
|
+
const fromNames = NAMES.map(which).filter((p): p is string => !!p);
|
|
49
|
+
return [...named, ...fromNames, ...PATHS, ...playwrightShells()];
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** The first usable browser binary; throws a message that says what to do about it when there is none. */
|
|
17
53
|
export function findChrome(explicit?: string): string {
|
|
18
|
-
const
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
for (const dir of (process.env.PATH ?? "").split(path.delimiter)) { const p = path.join(dir, c); if (fs.existsSync(p)) return p; }
|
|
54
|
+
for (const c of chromeCandidates(explicit)) {
|
|
55
|
+
if (isPath(c)) { if (fs.existsSync(c)) return c; continue; }
|
|
56
|
+
const p = which(c); if (p) return p;
|
|
22
57
|
}
|
|
23
|
-
throw new Error("no Chrome/Chromium found
|
|
58
|
+
throw new Error("no Chrome/Chromium found — visual mode screenshots routes with headless Chrome. Any Chromium works: install one, " +
|
|
59
|
+
"`npx playwright install chromium-headless-shell` puts one where loops looks (~/.cache/ms-playwright), or " +
|
|
60
|
+
"point LOOPS_CHROME at a binary, e.g. LOOPS_CHROME=/usr/bin/chromium");
|
|
24
61
|
}
|
|
25
62
|
|
|
26
63
|
export const freePort = () => new Promise<number>((res, rej) => { const s = net.createServer(); s.unref(); s.on("error", rej); s.listen(0, "127.0.0.1", () => { const addr = s.address(); const port = typeof addr === "object" && addr ? addr.port : 0; s.close(() => res(port)); }); });
|
|
@@ -35,12 +72,124 @@ async function waitFor(url: string, timeoutMs: number, child: ChildProcess | nul
|
|
|
35
72
|
throw new Error(`server did not answer at ${url} within ${timeoutMs / 1000}s`);
|
|
36
73
|
}
|
|
37
74
|
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
75
|
+
/* ---------- CDP: launch, connect, capture ---------- */
|
|
76
|
+
|
|
77
|
+
type Send = (method: string, params?: Record<string, unknown>, sessionId?: string) => Promise<any>;
|
|
78
|
+
type Cdp = { send: Send; close: () => void; on: (method: string, fn: (params: any) => void) => () => void };
|
|
79
|
+
|
|
80
|
+
async function connect(wsUrl: string): Promise<Cdp> {
|
|
81
|
+
const sock = new WebSocket(wsUrl);
|
|
82
|
+
const pending = new Map<number, { ok: (v: any) => void; fail: (e: Error) => void }>();
|
|
83
|
+
const listeners = new Map<string, Set<(params: any) => void>>();
|
|
84
|
+
let id = 0;
|
|
85
|
+
sock.addEventListener("message", (e) => {
|
|
86
|
+
const m = JSON.parse(String(e.data));
|
|
87
|
+
if (m.id) { const p = pending.get(m.id); if (!p) return; pending.delete(m.id); if (m.error) p.fail(new Error(m.error.message)); else p.ok(m.result); return; }
|
|
88
|
+
for (const fn of listeners.get(m.method) ?? []) { try { fn(m.params); } catch {} }
|
|
89
|
+
});
|
|
90
|
+
await new Promise<void>((res, rej) => {
|
|
91
|
+
sock.addEventListener("open", () => res());
|
|
92
|
+
sock.addEventListener("error", () => rej(new Error("could not connect to the browser's DevTools endpoint")));
|
|
93
|
+
});
|
|
94
|
+
const send: Send = (method, params = {}, sessionId) => new Promise((ok, fail) => {
|
|
95
|
+
const n = ++id; pending.set(n, { ok, fail });
|
|
96
|
+
sock.send(JSON.stringify({ id: n, method, params, ...(sessionId ? { sessionId } : {}) }));
|
|
97
|
+
});
|
|
98
|
+
return {
|
|
99
|
+
send, close: () => { try { sock.close(); } catch {} },
|
|
100
|
+
on: (method, fn) => { const s = listeners.get(method) ?? new Set(); s.add(fn); listeners.set(method, s); return () => { s.delete(fn); }; },
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/** Start the browser on a port Chrome picks itself, and read the endpoint it prints. */
|
|
105
|
+
function launch(bin: string): { child: ChildProcess; endpoint: Promise<string> } {
|
|
106
|
+
const child = spawn(bin, ["--headless=new", "--no-sandbox", "--disable-gpu", "--hide-scrollbars", "--disable-dev-shm-usage",
|
|
107
|
+
"--remote-debugging-port=0", "about:blank"], { stdio: ["ignore", "ignore", "pipe"], detached: true });
|
|
108
|
+
const endpoint = new Promise<string>((res, rej) => {
|
|
109
|
+
let buf = "";
|
|
110
|
+
const t = setTimeout(() => rej(new Error(`the browser did not open a DevTools endpoint: ${buf.split("\n").filter((l) => !/dbus|gpu|vulkan|fontconfig/i.test(l)).slice(-2).join(" ").slice(0, 240)}`)), 30000);
|
|
111
|
+
child.stderr?.on("data", (d) => { buf += String(d); const m = buf.match(/DevTools listening on (ws:\/\/\S+)/); if (m) { clearTimeout(t); res(m[1]); } });
|
|
112
|
+
child.on("exit", (code) => { clearTimeout(t); rej(new Error(`the browser exited (${code}) before it was ready`)); });
|
|
113
|
+
});
|
|
114
|
+
return { child, endpoint };
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
const kill = (child: ChildProcess) => {
|
|
118
|
+
const pid = child.pid; if (!pid) return;
|
|
119
|
+
try { process.kill(-pid, "SIGTERM"); } catch { try { child.kill("SIGTERM"); } catch {} }
|
|
120
|
+
setTimeout(() => { try { process.kill(-pid, "SIGKILL"); } catch {} }, 1500).unref();
|
|
121
|
+
};
|
|
122
|
+
|
|
123
|
+
/** An element's box in page coordinates, for `--selector`. */
|
|
124
|
+
async function boxOf(to: Send, selector: string) {
|
|
125
|
+
await to("DOM.enable");
|
|
126
|
+
const { root } = await to("DOM.getDocument", { depth: -1 });
|
|
127
|
+
const { nodeId } = await to("DOM.querySelector", { nodeId: root.nodeId, selector });
|
|
128
|
+
if (!nodeId) throw new Error(`no element matches ${selector}`);
|
|
129
|
+
const { model } = await to("DOM.getBoxModel", { nodeId });
|
|
130
|
+
const [x1, y1, , , x2, , , y2] = model.border as number[]; // TL, TR, BR, BL → width/height from the opposite corners
|
|
131
|
+
return { x: x1, y: y1, width: Math.max(x2 - x1, 1), height: Math.max(y2 - y1, 1) };
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/** Chrome refuses absurd surfaces; a 12000px page still shows everything a judge needs. */
|
|
135
|
+
const MAX_HEIGHT = 12000;
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* Screenshot `url` into `out`: the whole page unless `viewport`, or one element when `selector` is given.
|
|
139
|
+
* Throws when no browser is installed (findChrome's message says what to do), when the page does not load
|
|
140
|
+
* (a route that 404s or refuses the connection is an error, never a screenshot of the browser's error page),
|
|
141
|
+
* or on a bad selector.
|
|
142
|
+
*/
|
|
143
|
+
export async function capture({ url, out, width = 1440, height = 900, wait = 1500, loadTimeout = 30, viewport = false, selector, chrome, cwd = process.cwd() }: {
|
|
144
|
+
url: string; out: string; width?: number; height?: number; wait?: number; /** seconds to wait for the page to load */ loadTimeout?: number;
|
|
145
|
+
viewport?: boolean; selector?: string; chrome?: string; cwd?: string;
|
|
146
|
+
}): Promise<string> {
|
|
147
|
+
const bin = findChrome(chrome);
|
|
148
|
+
const { child, endpoint } = launch(bin);
|
|
149
|
+
let cdp: Cdp | null = null;
|
|
150
|
+
try {
|
|
151
|
+
cdp = await connect(await endpoint);
|
|
152
|
+
// about:blank first, then navigate: the page has to be watched from before its first request
|
|
153
|
+
const { targetId } = await cdp.send("Target.createTarget", { url: "about:blank" });
|
|
154
|
+
const { sessionId } = await cdp.send("Target.attachToTarget", { targetId, flatten: true });
|
|
155
|
+
const to: Send = (method, params = {}) => cdp!.send(method, params, sessionId);
|
|
156
|
+
await to("Page.enable");
|
|
157
|
+
await to("Network.enable");
|
|
158
|
+
await to("Emulation.setDeviceMetricsOverride", { width, height, deviceScaleFactor: 1, mobile: false });
|
|
159
|
+
const failures: string[] = [];
|
|
160
|
+
let loaded = false;
|
|
161
|
+
cdp.on("Page.loadEventFired", () => { loaded = true; });
|
|
162
|
+
cdp.on("Network.loadingFailed", (p) => { if (p?.type === "Document") failures.push(p.errorText ?? "the document failed to load"); });
|
|
163
|
+
cdp.on("Network.responseReceived", (p) => { if (p?.type === "Document" && p.response?.status >= 400) failures.push(`HTTP ${p.response.status}`); });
|
|
164
|
+
const nav = await to("Page.navigate", { url });
|
|
165
|
+
if (nav?.errorText) failures.push(nav.errorText);
|
|
166
|
+
for (const t0 = Date.now(); !loaded && !failures.length && Date.now() - t0 < loadTimeout * 1000;) await new Promise((r) => setTimeout(r, 100));
|
|
167
|
+
if (wait) await new Promise((r) => setTimeout(r, wait));
|
|
168
|
+
if (failures.length) throw new Error(`${url}: ${failures[0]}`);
|
|
169
|
+
const layout = await to("Page.getLayoutMetrics");
|
|
170
|
+
const pageSize = layout.cssContentSize ?? layout.contentSize;
|
|
171
|
+
const clip = selector ? await boxOf(to, selector)
|
|
172
|
+
: viewport ? { x: 0, y: 0, width, height }
|
|
173
|
+
: { x: 0, y: 0, width: Math.max(Math.ceil(pageSize.width), width), height: Math.min(Math.max(Math.ceil(pageSize.height), height), MAX_HEIGHT) };
|
|
174
|
+
const { data } = await to("Page.captureScreenshot", { format: "png", captureBeyondViewport: true, fromSurface: true, clip: { ...clip, scale: 1 } });
|
|
175
|
+
const file = path.resolve(cwd, out);
|
|
176
|
+
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
177
|
+
fs.writeFileSync(file, Buffer.from(data, "base64"));
|
|
178
|
+
return file;
|
|
179
|
+
} finally {
|
|
180
|
+
cdp?.close();
|
|
181
|
+
kill(child);
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/* ---------- CLI: serve the app, then capture ---------- */
|
|
186
|
+
|
|
187
|
+
export async function shot({ url, out, serve, port, width = 1440, height = 900, wait = 1500, timeout = 180, chrome, cwd = process.cwd(), viewport = false, selector, loadTimeout = 30, log = () => {} }: {
|
|
188
|
+
url?: string; out?: string; serve?: string; port?: number; width?: number; height?: number; wait?: number; timeout?: number;
|
|
189
|
+
chrome?: string; cwd?: string; viewport?: boolean; selector?: string; loadTimeout?: number; log?: (msg: string) => void;
|
|
41
190
|
}) {
|
|
42
191
|
if (!url || !out) throw new Error("shot needs --url and --out");
|
|
43
|
-
|
|
192
|
+
findChrome(chrome); // fail before starting anything, with the message that says what to install
|
|
44
193
|
let child: ChildProcess | null = null, logFile: string | null = null;
|
|
45
194
|
if (serve) {
|
|
46
195
|
port ??= await freePort();
|
|
@@ -53,12 +202,9 @@ export async function shot({ url, out, serve, port, width = 1440, height = 900,
|
|
|
53
202
|
const target = /^https?:\/\//.test(url) ? url : `http://localhost:${port ?? (() => { throw new Error("a relative --url needs --serve or --port"); })()}${url.startsWith("/") ? "" : "/"}${url}`;
|
|
54
203
|
try {
|
|
55
204
|
if (child || port) await waitFor(target, timeout * 1000, child);
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
const r = spawnSync(bin, args, { cwd, encoding: "utf8", timeout: 120000 });
|
|
60
|
-
if (r.status !== 0 || !fs.existsSync(path.resolve(cwd, out))) throw new Error(`chrome failed (${r.status}): ${(r.stderr || "").split("\n").filter((l) => !/dbus|gpu|vulkan|fontconfig/i.test(l)).slice(-3).join(" ")}`);
|
|
61
|
-
return path.resolve(cwd, out);
|
|
205
|
+
const file = await capture({ url: target, out, width, height, wait, loadTimeout, viewport, selector, chrome, cwd });
|
|
206
|
+
log(`${selector ?? (viewport ? "viewport" : "full page")} → ${out}`);
|
|
207
|
+
return file;
|
|
62
208
|
} finally {
|
|
63
209
|
if (child) {
|
|
64
210
|
if (child.pid) { try { process.kill(-child.pid, "SIGTERM"); } catch {} }
|
|
@@ -69,7 +215,7 @@ export async function shot({ url, out, serve, port, width = 1440, height = 900,
|
|
|
69
215
|
}
|
|
70
216
|
}
|
|
71
217
|
|
|
72
|
-
type ShotArgs = Partial<{ url: string; out: string; serve: string; port: number; width: number; height: number; wait: number; timeout: number; chrome: string }>;
|
|
218
|
+
type ShotArgs = Partial<{ url: string; out: string; serve: string; port: number; width: number; height: number; wait: number; timeout: number; loadTimeout: number; chrome: string; selector: string; viewport: boolean }>;
|
|
73
219
|
|
|
74
220
|
function parseArgs(argv: string[]): ShotArgs {
|
|
75
221
|
const o: Record<string, unknown> = {};
|
|
@@ -77,7 +223,9 @@ function parseArgs(argv: string[]): ShotArgs {
|
|
|
77
223
|
const a = argv[i];
|
|
78
224
|
if (!a.startsWith("--")) continue;
|
|
79
225
|
const k = a.slice(2); const v = argv[i + 1];
|
|
80
|
-
if (
|
|
226
|
+
if (k === "load-timeout") { (o as any).loadTimeout = Number(v); i++; continue; }
|
|
227
|
+
if (["width", "height", "wait", "timeout", "load-timeout", "port"].includes(k)) { o[k] = Number(v); i++; }
|
|
228
|
+
else if (k === "viewport" || k === "full") { o.viewport = k === "viewport"; }
|
|
81
229
|
else { o[k] = v; i++; }
|
|
82
230
|
}
|
|
83
231
|
return o as ShotArgs;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@computesdk/pi-loops",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.3",
|
|
4
4
|
"description": "Hill-climbing loops as Pi units: one `loop` tool, style agents, forked attempt sessions in git worktrees, a judge keeps the best.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"pi-package",
|
|
@@ -49,7 +49,7 @@
|
|
|
49
49
|
],
|
|
50
50
|
"scripts": {
|
|
51
51
|
"check": "tsc --noEmit",
|
|
52
|
-
"test": "tsc --noEmit && node test/rubric.ts && node test/judge.ts && node test/measure.ts && node test/loop.ts"
|
|
52
|
+
"test": "tsc --noEmit && node test/rubric.ts && node test/judge.ts && node test/measure.ts && node test/shot.ts && node test/loop.ts"
|
|
53
53
|
},
|
|
54
54
|
"peerDependencies": {
|
|
55
55
|
"@earendil-works/pi-coding-agent": "*",
|