@computesdk/pi-loops 0.0.0-stage → 0.1.0

package/lib/judge.ts

@@ -0,0 +1,193 @@
+// Judging math from the POC. Bench numbers and review verdicts come from the judge agent (extensions/loop.ts); the visual panel's
+// model calls go through the injected `ask(content) -> parsed JSON` (a one-shot agent session, a stub in tests). Pure otherwise.
+
+import type { Metric } from "./measure.ts";
+
+/** One attempt as the judge agent measured it. */
+export type Attempt = {
+  label: string;
+  bench?: { passed?: boolean; tests?: string; value?: number; error?: string };
+  review?: { ok?: boolean; note?: string };
+  vetoed?: boolean;
+};
+
+/** A bench score; `null`/`undefined` when the bench produced none. */
+export type Score = number | null | undefined;
+/** One candidate as the visual judge sees it: screenshots, and/or its code change as a diff. */
+export type Candidate = { label: string; shots?: string[]; text?: string };
+/** A part of a judge prompt: text, or whatever `img` makes of an image file. */
+export type Part = { type: string; text?: string; [key: string]: unknown };
+/** One judge model call: prompt parts in, parsed JSON answer out. */
+export type Ask = (content: Part[]) => Promise<any>;
+/** The rubric as the judge uses it: each criterion's name, its share of 100, and the text. */
+export type JudgeCriterion = { name: string; pct: number; description?: string; check?: string };
+export type JudgeRubric = { criteria: JudgeCriterion[] };
+/** What every judge prompt is told: the subject, the goal, the rules, the rubric in force. */
+export type JudgeBrief = { subject: string; goal: string; rules: string[]; rubric?: JudgeRubric };
+export interface JudgeOptions extends JudgeBrief {
+  ask: Ask;
+  /** One `{ name, ask }` per council model; without it the judge scores alone. */
+  panel?: { name: string; ask: Ask }[];
+  img: (file: string) => Part;
+  candidates: Candidate[];
+  samples?: number;
+  pairs?: number;
+  scoreSlack?: number;
+}
+/** One candidate's head-to-head record against CURRENT. */
+export type Vote = { wins: number; losses: number; ties: number; votes: number; why?: any; decidedBy?: string[] };
+/** One model's answer to a compare call. */
+export type Comparison = { winner: string | null; why: any; criterion: any };
+/** The bench verdict: a score and a reason per label, the winner, and one line of why. */
+export type BenchVerdict = { scores: Record<string, Score>; reasons: Record<string, string>; pick: string; why: string };
+/** The panel verdict. */
+export type PanelResult = {
+  scores: Record<string, number>;
+  reasons: Record<string, string>;
+  spread: Record<string, number>;
+  pick: string;
+  why: string;
+  h2h: Record<string, Vote>;
+  samples: number;
+  rubric?: Record<string, { scores: Record<string, number>; total: number }>;
+  byModel?: Record<string, Record<string, number>>;
+};
+
+const median = (xs: number[]) => { const s = [...xs].sort((a, b) => a - b); const m = s.length >> 1; return s.length % 2 ? s[m] : (s[m - 1] + s[m]) / 2; };
+const round1 = (x: number) => Math.round(x * 10) / 10;
+/** A missing score as a number, so it can be compared and sorted (NaN loses every comparison). */
+const num = (v: Score) => v ?? NaN;
+
+/* ---------- bench pick: check gate, benchmark scores, judge veto, minGain ---------- */
+/**
+ * attempts: [{ label, bench: { passed, tests, value?, error? }, review?: { ok, note } }] as the judge agent measured them; cur: { value } (CURRENT re-measured by the same judge).
+ * Returns { scores, reasons, pick, why }.
+ */
+export function pickWinner({ cur, attempts, metric = {}, minGain = 0.05 }: { cur: { value?: number | null }; attempts: Attempt[]; metric?: Metric; minGain?: number }): BenchVerdict {
+  const better = metric.better === "higher" ? (a: number, b: number) => a > b : (a: number, b: number) => a < b;
+  const gain = metric.better === "higher" ? (cur: number, v: Score) => Number(v) / cur : (cur: number, v: Score) => cur / Number(v);
+  const unit = metric.unit ?? "";
+  const scores: Record<string, Score> = {}, reasons: Record<string, string> = {};
+  scores.CURRENT = cur.value; reasons.CURRENT = `${cur.value} ${unit} re-measured`;
+  for (const a of attempts) {
+    const m = a.bench ?? {};
+    if (!m.passed) { reasons[a.label] = "tests fail: " + ((m.tests ?? "").split("\n").find((l) => /FAIL|failed|Error/.test(l)) || m.tests || "").slice(0, 120); continue; }
+    if (m.error) { reasons[a.label] = m.error.slice(0, 120); continue; }
+    if (a.review && !a.review.ok) { reasons[a.label] = "vetoed: " + String(a.review.note); a.vetoed = true; continue; }
+    scores[a.label] = m.value;
+    const x = cur.value == null ? NaN : gain(cur.value, m.value);
+    reasons[a.label] = `${m.value} ${unit}${Number.isFinite(x) ? ` (${x >= 1 ? x.toFixed(2) + "× better" : (1 / x).toFixed(2) + "× worse"})` : ""}${a.review?.note ? ` · ${a.review.note}` : ""}`;
+  }
+  const ok = attempts.filter((a) => scores[a.label] != null).sort((x, y) => (better(num(scores[x.label]), num(scores[y.label])) ? -1 : 1));
+  const top = ok[0];
+  const wins = !!top && (cur.value == null || better(num(scores[top.label]), cur.value * (metric.better === "higher" ? 1 + minGain : 1 - minGain)));
+  const pick = wins && top ? top.label : "CURRENT";
+  const failed = attempts.length - ok.length;
+  const why = wins && top
+    ? `${top.label} is best at ${scores[top.label]} ${unit} vs ${cur.value} ${unit} now (${gain(Number(cur.value), scores[top.label]).toFixed(2)}×); tests pass${top.review?.note ? `; review: ${top.review.note}` : ""}.${failed ? ` ${failed} attempt(s) failed tests or were vetoed.` : ""}`
+    : top ? `Best attempt (${top.label}, ${scores[top.label]} ${unit}) is not >${minGain * 100}% better than ${cur.value} ${unit}.` : `No attempt passed tests and review.`;
+  return { scores, reasons, pick, why };
+}
+
+/* ---------- visual panel: scores (median of samples) + head-to-head votes against CURRENT ---------- */
+const header = (o: JudgeBrief) => `${o.subject}.\nGoal: ${o.goal}\nRules (weigh these heavily):\n${o.rules.map((r) => "- " + r).join("\n")}\n` +
+  (o.rubric ? `\nRubric (the council wrote it; judge against these criteria and nothing else, each weighted by its share of 100):\n${o.rubric.criteria.map((c) => `- ${c.name} (${c.pct}%): ${c.description}${c.check ? ` Check: ${c.check}` : ""}`).join("\n")}\n` : "");
+const scoreShape = (o: JudgeBrief) => o.rubric
+  ? `Return JSON: {"scores":{"<label>":number},"criteria":{"<label>":{"<criterion name>":0-10}},"reasons":{"<label>":"<=15 words: the criterion that most drove the score, and why"}}`
+  : `Return JSON: {"scores":{"<label>":number},"reasons":{"<label>":"<=15 words, main strength or flaw"}}`;
+const compareShape = (o: JudgeBrief) => o.rubric
+  ? `Return JSON: {"better":"1"|"2"|"equal","criterion":"<the rubric criterion that decided it>","why":"<=20 words, citing that criterion"}`
+  : `Return JSON: {"better":"1"|"2"|"equal","why":"<=20 words"}`;
+const cites = (why: any, criterion: any) => (criterion && why && !why.toLowerCase().includes(String(criterion).toLowerCase().slice(0, 12)) ? `${criterion}: ${why}` : why);
+const shotsOf = (o: { img: (file: string) => Part }, c: Candidate): Part[] => [...(c.shots ?? []).map((f) => o.img(f)), ...(c.text ? [{ type: "text", text: c.text }] : [])];
+
+async function scoreOnce(o: JudgeOptions) {
+  const content: Part[] = [{ type: "text", text: `You are the judge in a hill-climbing loop improving ${header(o)}\nCandidates follow, each with screenshot(s) and/or its code change as a diff against CURRENT. CURRENT is the best version so far. Score every candidate 0-10 against the goal and rules; be strict and consistent.\n${scoreShape(o)}` }];
+  for (const c of o.candidates) { content.push({ type: "text", text: `--- ${c.label}` }, ...shotsOf(o, c)); }
+  return o.ask(content);
+}
+async function compareOnce(o: JudgeOptions, first: Candidate, second: Candidate): Promise<Comparison> {
+  const content: Part[] = [{ type: "text", text: `You compare two versions of ${header(o)}\nTwo versions follow, "1" and "2" (the order is random). Which one better achieves the goal under the rules? If they're genuinely equal, say so.\n${compareShape(o)}` }];
+  for (const [i, c] of [first, second].entries()) content.push({ type: "text", text: `--- Version ${i + 1}` }, ...shotsOf(o, c));
+  const r = await o.ask(content);
+  return { winner: r.better === "1" ? first.label : r.better === "2" ? second.label : null, why: cites(r.why, o.rubric && r.criterion), criterion: o.rubric ? r.criterion : undefined };
+}
+/** votes for `cand` vs `cur`: both orders, `pairs` times each; a close first batch gets a second. */
+export async function headToHead(o: JudgeOptions, cand: Candidate, cur: Candidate): Promise<Vote> {
+  const batch = async (): Promise<Comparison[]> => {
+    const jobs: Promise<Comparison>[] = [];
+    for (let i = 0; i < (o.pairs ?? 2); i++) jobs.push(compareOnce(o, cand, cur), compareOnce(o, cur, cand));
+    return (await Promise.allSettled(jobs)).filter((v): v is PromiseFulfilledResult<Comparison> => v.status === "fulfilled").map((v) => v.value);
+  };
+  let votes = await batch();
+  const w0 = votes.filter((v) => v.winner === cand.label).length, l0 = votes.filter((v) => v.winner === cur.label).length;
+  if (w0 > 0 && l0 > 0) votes = votes.concat(await batch());
+  const wins = votes.filter((v) => v.winner === cand.label).length, losses = votes.filter((v) => v.winner === cur.label).length;
+  const lead = votes.find((v) => v.winner === cand.label) ?? votes[0];
+  const decidedBy = o.rubric ? [...votes.reduce((m, v) => (v.criterion ? m.set(v.criterion, (m.get(v.criterion) ?? 0) + 1) : m), new Map<string, number>())].sort((a, b) => b[1] - a[1]).map(([c]) => c) : undefined;
+  return { wins, losses, ties: votes.length - wins - losses, votes: votes.length, why: lead?.why, ...(decidedBy ? { decidedBy } : {}) };
+}
+/**
+ * o: { ask, panel?: [{ name, ask }], img, subject, goal, rules, rubric?, candidates: [{label, shots?: [file], text?: diff}] incl. "CURRENT", samples?=3, pairs?=2, scoreSlack?=0.3 }
+ * Returns { scores, reasons, spread, pick, why, h2h, samples, rubric? }. A candidate wins only with a head-to-head majority
+ * (wins > 2×losses and ≥ half the votes) and a score not below CURRENT's (or a sweep within `scoreSlack`).
+ */
+export async function panelJudge(o: JudgeOptions): Promise<PanelResult> {
+  const n = o.samples ?? 3;
+  // scoring: the panel (one `{ name, ask }` per council model, labels blind) or the judge alone; `samples` calls spread over the panel
+  const panel = o.panel?.length ? o.panel : [{ name: "judge", ask: o.ask }];
+  const per = Math.max(1, Math.ceil(n / panel.length));
+  const settled = await Promise.allSettled(panel.flatMap((m) => Array.from({ length: per }, () => scoreOnce({ ...o, ask: m.ask }).then((r) => ({ ...r, by: m.name })))));
+  const runs = settled.filter((r): r is PromiseFulfilledResult<any> => r.status === "fulfilled").map((r) => r.value);
+  if (!runs.length) throw new Error(`judge failed (all samples): ${[...new Set(settled.map((s) => String(s.status === "rejected" ? s.reason?.message ?? s.reason : "unknown").slice(0, 120)))].join(" | ")}`);
+  const labels = o.candidates.map((c) => c.label);
+  const scores: Record<string, number> = {}, spread: Record<string, number> = {}, reasons: Record<string, string> = {};
+  for (const l of labels) {
+    const xs = runs.map((r) => Number(r.scores?.[l])).filter(Number.isFinite);
+    if (!xs.length) continue;
+    scores[l] = round1(median(xs));
+    spread[l] = round1(Math.max(...xs) - Math.min(...xs));
+    const near = runs.reduce((a, r) => (Math.abs(r.scores?.[l] - scores[l]) < Math.abs(a.scores?.[l] - scores[l]) ? r : a));
+    reasons[l] = near.reasons?.[l];
+  }
+  let byCriterion: PanelResult["rubric"];
+  if (o.rubric) {
+    byCriterion = {};
+    for (const l of labels) {
+      const per: Record<string, number> = {};
+      for (const c of o.rubric.criteria) {
+        const xs = runs.map((r) => Number(r.criteria?.[l]?.[c.name])).filter(Number.isFinite);
+        if (xs.length) per[c.name] = round1(median(xs));
+      }
+      if (!Object.keys(per).length) continue;
+      const covered = o.rubric.criteria.filter((c) => per[c.name] != null);
+      byCriterion[l] = { scores: per, total: round1(covered.reduce((s, c) => s + per[c.name] * c.pct, 0) / Math.max(1, covered.reduce((s, c) => s + c.pct, 0))) };
+    }
+  }
+  let byModel: PanelResult["byModel"];
+  if (o.panel?.length) {
+    byModel = {};
+    for (const l of labels) {
+      const m: Record<string, number[]> = {};
+      for (const r of runs) { const x = Number(r.scores?.[l]); if (Number.isFinite(x)) (m[r.by] ??= []).push(x); }
+      if (Object.keys(m).length) byModel[l] = Object.fromEntries(Object.entries(m).map(([k, xs]) => [k, round1(median(xs))]));
+    }
+  }
+  const byLabel = new Map(o.candidates.map((c) => [c.label, c]));
+  const cur = o.candidates.find((c) => c.label === "CURRENT");
+  const ranked = labels.filter((l) => l !== "CURRENT" && scores[l] != null).sort((a, b) => scores[b] - scores[a]).slice(0, 2);
+  const h2h: Record<string, Vote> = {};
+  await Promise.all(ranked.map(async (l) => { const c = byLabel.get(l); h2h[l] = cur && c ? await headToHead(o, c, cur) : { wins: 1, losses: 0, ties: 0, votes: 1 }; }));
+  const rate = (l: string) => h2h[l].wins / Math.max(1, h2h[l].votes);
+  const qualifies = (h: Vote) => h.wins > 2 * h.losses && h.wins * 2 >= h.votes;
+  const sweep = (h: Vote) => h.losses === 0 && h.wins >= 4;
+  const scoreOk = (l: string) => scores.CURRENT == null || scores[l] >= scores.CURRENT || (sweep(h2h[l]) && scores[l] >= scores.CURRENT - (o.scoreSlack ?? 0.3));
+  const winners = ranked.filter((l) => qualifies(h2h[l]) && scoreOk(l)).sort((a, b) => rate(b) - rate(a) || scores[b] - scores[a]);
+  const pick = winners[0] ?? "CURRENT";
+  const v = (l: string) => `${h2h[l].wins}–${h2h[l].losses}${h2h[l].ties ? `, ${h2h[l].ties} even` : ""}`;
+  const why = pick !== "CURRENT"
+    ? `${pick} beat the current best head-to-head (${v(pick)} of ${h2h[pick].votes} votes${scores.CURRENT != null && scores[pick] < scores.CURRENT ? `; its score ${scores[pick]} is within noise of ${scores.CURRENT}` : ""}): ${h2h[pick].why ?? reasons[pick]}`
+    : ranked.some((l) => qualifies(h2h[l])) ? `Head-to-head favoured ${ranked.filter((l) => qualifies(h2h[l])).join(", ")}, but its score was clearly below the current best's (${scores.CURRENT}), so the signals disagree; kept the current best.`
+    : ranked.length ? `Nothing clearly beat the current best head-to-head (${ranked.map((l) => `${l} ${v(l)}`).join("; ")}).` : "No candidates.";
+  return { scores, reasons, spread, pick, why, h2h, samples: runs.length, ...(byCriterion ? { rubric: byCriterion } : {}), ...(byModel ? { byModel } : {}) };
+}

package/lib/measure.ts

@@ -0,0 +1,38 @@
+// Git helpers around attempts: shell runner, protected-file restore, diff. `git` is injectable for tests.
+import { spawn, execFileSync } from "node:child_process";
+
+export const sh = (cwd: string, cmd: string, ms = 180_000) => new Promise<{ code: number | null; out: string }>((resolve) => {
+  const p = spawn("bash", ["-lc", cmd], { cwd, stdio: ["ignore", "pipe", "pipe"] });
+  let out = "";
+  p.stdout.on("data", (d) => (out += d)); p.stderr.on("data", (d) => (out += d));
+  const t = setTimeout(() => p.kill("SIGKILL"), ms);
+  p.on("close", (code) => { clearTimeout(t); resolve({ code, out: out.trim() }); });
+});
+/** `git` is injectable so tests can stub it out. */
+export type GitRunner = (cwd: string, ...args: string[]) => string;
+export const git: GitRunner = (cwd, ...a) => execFileSync("git", a, { cwd, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }).trim();
+/** A thrown value's message: `catch (e)` hands you `unknown`, and anything at all can be thrown. */
+export const errText = (e: unknown) => (e instanceof Error ? e.message : String(e));
+/** The patch between two commits for a judge: stat + body, cut at `max` chars; a patch too big even for the buffer falls back to the stat alone (never "unavailable"). */
+export function diffText(cwd: string, base: string, commit: string, max = 60_000) {
+  const run = (...a: string[]) => execFileSync("git", ["diff", ...a, `${base}..${commit}`], { cwd, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], maxBuffer: 256 * 1024 * 1024 });
+  let d;
+  try { d = run("--stat", "-p"); } catch { try { d = run("--stat") + "\n… (patch too large to show; stat only)"; } catch (e) { return `(diff unavailable: ${errText(e).slice(0, 80)})`; } }
+  d = d.trim();
+  return d.length > max ? d.slice(0, max) + "\n… (diff truncated)" : d || "(no change)";
+}
+
+export function restoreProtected({ cwd, protect = [], baseCommit, git: g = git }: { cwd: string; protect?: string[]; baseCommit: string; git?: GitRunner }) {
+  if (!protect.length) return false;
+  g(cwd, "checkout", baseCommit, "--", ...protect);
+  try { g(cwd, "commit", "-q", "-m", "restore protected files", "--", ...protect); return true; } catch { return false; }
+}
+
+/** The diff the judge sees: attempt vs the version it started from, protected files excluded. */
+export const diffFor = ({ cwd, from, to = "HEAD", protect = [], git: g = git, max = 30000 }: { cwd: string; from: string; to?: string; protect?: string[]; git?: GitRunner; max?: number }) =>
+  g(cwd, "diff", from, to, "--", ".", ...protect.map((f) => `:!${f}`)).slice(0, max);
+
+/** Which direction counts as better for a metric. */
+export type Metric = { key?: string; unit?: string; better?: "higher" | "lower" };
+
+export const better = (metric: Metric = {}) => (metric.better === "higher" ? (a: number, b: number) => a > b : (a: number, b: number) => a < b);

package/lib/rubric.ts

@@ -0,0 +1,159 @@
+// The judging rubric: written by the council before round 1, extended by the human's `rules`.
+// Pure data + prompts; `ask(role, prompt)` is injected (one-shot agent session in the extension, a stub in tests).
+
+/** A criterion as proposed by a council member, a rule, or a human; `normalise` fills in the rest. */
+export type CriterionInput = { name?: unknown; weight?: unknown; description?: unknown; check?: unknown; source?: string; was?: Criterion };
+/** A criterion after `normalise`: text cleaned, weight clamped, `pct` set so the column sums to 100. */
+export type Criterion = { name: string; weight: number; description: string; check: string; source: string; pct: number; was?: Criterion };
+export type RubricEntry = { t: number; kind: string } & Record<string, unknown>;
+export type Rubric = { v: number; criteria: Criterion[]; summary: string; source: string; members?: number; history: RubricEntry[] };
+/** What the council is told: the goal, the rules to cover, and the rubric in force. */
+export type RubricContext = { goal: string; kind?: string; rules?: string[]; subject?: string; history?: string; current?: Criterion[] };
+
+const WEIGHT_RANGE: [number, number] = [1, 5];
+const clamp = (x: unknown, [lo, hi]: [number, number]) => (Number.isFinite(Number(x)) ? Math.min(hi, Math.max(lo, Math.round(Number(x)))) : hi);
+const trim = (s: unknown, n: number) => String(s ?? "").replace(/\s+/g, " ").trim().slice(0, n);
+const key = (s: unknown) => trim(s, 80).toLowerCase().replace(/[^a-z0-9]+/g, " ").trim();
+const entry = (kind: string, extra: Record<string, unknown>): RubricEntry => ({ t: Date.now(), kind, ...extra });
+
+/** JSON from a model answer: the whole text, else the fenced block, else the first balanced {…} / […]. */
+export function parseJson(text: unknown): any {
+  let s = String(text ?? "").trim();
+  const fence = /```(?:json)?\s*([\s\S]*?)```/i.exec(s);
+  if (fence) s = fence[1].trim();
+  try { return JSON.parse(s); } catch {}
+  const start = s.search(/[{[]/);
+  if (start < 0) throw new Error(`no JSON in model answer: ${s.slice(0, 120)}`);
+  const open = s[start], close = open === "{" ? "}" : "]";
+  let depth = 0, inStr = false, esc = false;
+  for (let i = start; i < s.length; i++) {
+    const c = s[i];
+    if (inStr) { if (esc) esc = false; else if (c === "\\") esc = true; else if (c === '"') inStr = false; continue; }
+    if (c === '"') inStr = true;
+    else if (c === "{" || c === "[") depth++;
+    else if (c === "}" || c === "]") { depth--; if (depth === 0) { if (c !== close) break; return JSON.parse(s.slice(start, i + 1)); } }
+  }
+  throw new Error(`unbalanced JSON in model answer: ${s.slice(start, start + 120)}`);
+}
+
+/** Clean a criteria list: drop empties, merge duplicate names (first wins, weight = max), clamp weights, set `pct` to sum 100. */
+export function normalise(criteria: CriterionInput[] | undefined): Criterion[] {
+  const out: Criterion[] = [];
+  for (const c of criteria ?? []) {
+    const name = trim(c?.name, 60);
+    if (!name) continue;
+    const prev = out.find((x) => key(x.name) === key(name));
+    const w = clamp(c.weight, WEIGHT_RANGE);
+    if (prev) { prev.weight = Math.max(prev.weight, w); prev.description ||= trim(c.description, 300); prev.check ||= trim(c.check, 300); continue; }
+    out.push({ name, weight: w, description: trim(c.description, 300), check: trim(c.check, 300), source: c.source || "council", pct: 0, ...(c.was ? { was: c.was } : {}) });
+  }
+  const total = out.reduce((s, c) => s + c.weight, 0) || 1;
+  let given = 0;
+  for (const c of out) { c.pct = Math.floor((c.weight / total) * 100); given += c.pct; }
+  for (let i = 0; given < 100 && out.length; i = (i + 1) % out.length) { out[i].pct++; given++; }
+  out.sort((a, b) => b.weight - a.weight);
+  return out;
+}
+
+const CRITERION_SHAPE = `{"name":"<=6 words","weight":1-5,"description":"what good looks like, one sentence","check":"how a judge verifies it from what it is shown, one sentence"}`;
+export function proposePrompt(o: RubricContext) {
+  const bench = o.kind === "bench";
+  return `Several coding agents will each produce a version of ${o.subject ?? "the work"}; a judge then ${bench ? "checks each version's diff against the rules (the benchmark itself measures speed, so the rubric is about correctness, rule-following and code quality, not about being fast)" : "scores screenshots of each version against the goal and rules"}. The judge can only use this rubric, so criteria must be observable${bench ? " in a diff and test output" : " in a screenshot"} and distinct from each other.
+Goal: ${o.goal}
+Rules already stated (every rule must be covered by some criterion):
+${(o.rules ?? []).map((r) => "- " + r).join("\n") || "- (none)"}
+${o.history ? `\nWhat has happened in the loop so far (use it to sharpen or add criteria the current rubric missed):\n${o.history}\n${o.current ? `Current rubric:\n${rubricText({ criteria: o.current })}\n` : ""}` : ""}
+Propose 4 to 7 criteria. Weight 5 = decisive, 1 = nice to have.
+Return JSON: {"criteria":[${CRITERION_SHAPE}]}`;
+}
+export function synthPrompt(o: RubricContext, proposals: any[]) {
+  return `You chair the council that writes the judging rubric for a hill-climbing loop improving ${o.subject ?? "the work"}.
+Goal: ${o.goal}
+Rules stated (each must be covered):
+${(o.rules ?? []).map((r) => "- " + r).join("\n") || "- (none)"}
+
+${proposals.length} council member(s) proposed criteria:
+${proposals.map((p, i) => `--- Member ${i + 1}\n${JSON.stringify(p.criteria)}`).join("\n")}
+${o.history ? `\nLoop history so far:\n${o.history}\n${o.current ? `The rubric in force (prefer small edits to it over a rewrite; keep criteria marked source "human"):\n${JSON.stringify(o.current)}\n` : ""}` : ""}
+Merge them into one rubric of 4 to 7 criteria: combine duplicates, keep the sharpest wording of "description" and "check", drop anything the judge cannot observe, keep weights on the 1–5 scale (5 = decisive). Then write a one-sentence summary of how the judge will decide.
+Return JSON: {"criteria":[${CRITERION_SHAPE}],"summary":"<=30 words"}`;
+}
+
+/**
+ * Convene the council: `members` proposals (role "council"), one synthesis (role "judge").
+ * `ask(role, prompt)` returns the parsed JSON answer. `history`/`current` make it a re-convene.
+ * Returns { criteria, summary, source: "council", members }.
+ */
+export async function writeRubric({ goal, kind, rules, subject, history, current, members = 2, ask }: {
+  goal: string; kind?: string; rules?: string[]; subject?: string; history?: string;
+  current?: Criterion[]; members?: number; ask: (role: string, prompt: string, i?: number) => Promise<any>;
+}) {
+  const o: RubricContext = { goal, kind, rules, subject, history, current };
+  const settled = await Promise.allSettled(Array.from({ length: members }, (_, i) => ask("council", proposePrompt(o), i)));
+  const proposals = settled
+    .filter((r): r is PromiseFulfilledResult<any> => r.status === "fulfilled" && Array.isArray(r.value?.criteria) && r.value.criteria.length)
+    .map((r) => r.value);
+  if (!proposals.length) throw new Error(`council failed: ${settled.map((r) => (r.status === "rejected" ? r.reason?.message ?? String(r.reason) : "no criteria")).join("; ").slice(0, 200)}`);
+  const s = await ask("judge", synthPrompt(o, proposals));
+  let criteria = normalise(Array.isArray(s.criteria) && s.criteria.length ? s.criteria : proposals.flatMap((p) => p.criteria));
+  if (criteria.length < 2) criteria = normalise(proposals.flatMap((p) => p.criteria));
+  for (const h of (current ?? []).filter((c) => c.source === "human")) {
+    const same = criteria.find((c) => key(c.name) === key(h.name));
+    if (same) same.source = "human"; else criteria = normalise([...criteria, h]);
+  }
+  return { criteria, summary: trim(s.summary, 240) || `${criteria.length} criteria, weighted by the council`, source: "council", members: proposals.length };
+}
+
+export function newRubric(r: { criteria: CriterionInput[]; summary?: string; members?: number }, source: string): Rubric {
+  const criteria = normalise(r.criteria.map((c) => ({ ...c, source: c.source || source })));
+  return { v: 1, criteria, summary: r.summary || `${criteria.length} criteria from the ${source}`, source, ...(r.members ? { members: r.members } : {}), history: [entry(source, { criteria })] };
+}
+/** The fallback when no council member answers: every rule is a decisive criterion. */
+export const fromRules = (rules: string[], why: string): Rubric => ({ ...newRubric({ criteria: rules.map((text) => ruleCriterion(text, "rule")), summary: "Each rule weighs equally (the council was unavailable)." }, "rule"), history: [entry("rule", { why })] });
+
+const shortName = (text: unknown) => { const t = trim(text, 300).replace(/[.;:,]+$/, ""); if (t.length <= 80) return t; const cut = t.slice(0, 80); return `${cut.slice(0, Math.max(cut.lastIndexOf(" "), 40))}…`; };
+const ruleCriterion = (text: unknown, source: string): CriterionInput => ({ name: shortName(text), weight: 5, description: trim(text, 300), check: `Does the version follow this rule: ${trim(text, 200)}?`, source });
+/** A human rule becomes a decisive criterion (replacing a council criterion of the same name, remembered in `was`). */
+export function addRule(rubric: Rubric, text: string, by = "you") {
+  if (rubric.criteria.some((c) => c.source === "human" && c.description === trim(text, 300))) return rubric;
+  const rule = ruleCriterion(text, "human");
+  const same = rubric.criteria.find((c) => key(c.name) === key(rule.name));
+  rubric.criteria = normalise(same ? rubric.criteria.map((c) => (c === same ? { ...rule, name: same.name, was: { ...same } } : c)) : [...rubric.criteria, rule]);
+  rubric.history.push(entry("rule", { text, by }));
+  return rubric;
+}
+/** Drop the rule with that text wherever it sits (restoring what it took over); falls back to the criterion name. */
+export function removeRule(rubric: Rubric, text: string) {
+  const t = trim(text, 300);
+  const byText = (c: Criterion | undefined) => c?.description === t;
+  const byName = (c: Criterion | undefined) => !!c && key(c.name) === key(t);
+  const chain = (c: Criterion | undefined): Criterion[] => (c ? [c, ...chain(c.was)] : []);
+  const drop = (c: Criterion): Criterion => {
+    const w = c.was;
+    if (w && byText(w)) return { ...c, was: w.was };
+    return w ? { ...c, was: drop(w) } : { ...c };
+  };
+  let hit = false;
+  const take = (match: (c: Criterion | undefined) => boolean) => (c: Criterion): Criterion[] => {
+    if (match(c)) { hit = true; return c.was ? [c.was] : []; }
+    if (match === byText && chain(c.was).some(byText)) { hit = true; return [drop(c)]; }
+    return [c];
+  };
+  let next = rubric.criteria.flatMap(take(byText));
+  if (!hit) next = rubric.criteria.flatMap(take(byName));
+  if (!hit) return rubric;
+  rubric.criteria = normalise(next);
+  rubric.history.push(entry("unrule", { text }));
+  return rubric;
+}
+/** A re-convened council's proposal replaces the criteria (history keeps the old ones). */
+export function applyRevision(rubric: Rubric, r: { criteria: CriterionInput[]; summary?: string; members?: number }) {
+  const criteria = normalise(r.criteria);
+  const changed = JSON.stringify(criteria.map(({ name, weight }) => [name, weight])) !== JSON.stringify(rubric.criteria.map(({ name, weight }) => [name, weight]));
+  rubric.criteria = criteria;
+  if (r.summary) rubric.summary = r.summary;
+  rubric.history.push(entry("revisit", { criteria, members: r.members, changed }));
+  return { rubric, changed };
+}
+/** One line per criterion, the way both the judge prompt and the needs-you card show it. */
+export const rubricText = (rubric: { criteria: Criterion[] }) => rubric.criteria.map((c) => `- ${c.name} (${c.pct}%${c.source === "human" ? ", yours" : ""})${c.description && c.description !== c.name ? `: ${c.description}` : ""}${c.check ? ` Check: ${c.check}` : ""}`).join("\n");

package/lib/sheet.ts

@@ -0,0 +1,33 @@
+// A side-by-side sheet of candidate screenshots with labels, via ImageMagick's montage (returns null when it isn't installed).
+import { spawnSync } from "node:child_process";
+import fs from "node:fs";
+import path from "node:path";
+
+export function haveMontage() { return spawnSync("montage", ["-version"], { encoding: "utf8" }).status === 0; }
+
+/** ImageMagick's default label font (helvetica) is often missing; pick one it actually has. */
+function pickFont() {
+  const list = spawnSync("convert", ["-list", "font"], { encoding: "utf8" }).stdout ?? "";
+  const fonts = [...list.matchAll(/^\s*Font:\s*(\S+)/gm)].map((m) => m[1]);
+  return fonts.find((f) => /^(DejaVu-Sans|Liberation-Sans|Helvetica|Arial|Noto-Sans)$/i.test(f)) ?? fonts.find((f) => /sans/i.test(f)) ?? fonts[0];
+}
+
+type Tile = { label: string; files?: string[] };
+
+/** tiles: [{ label, files: [png...] }] → one PNG at `out`; one row per screenshot index, one column per candidate. */
+export function sheet({ tiles, out, tileWidth = 480 }: { tiles: Tile[]; out: string; tileWidth?: number }) {
+  const cols = tiles.filter((t): t is Tile & { files: string[] } => !!t.files?.length);
+  if (!cols.length || !haveMontage()) return null;
+  const rows = Math.max(...cols.map((t) => t.files.length));
+  const args = [];
+  for (let r = 0; r < rows; r++) for (const t of cols) {
+    const f = t.files[r];
+    if (f && fs.existsSync(f)) args.push("-label", r === 0 ? t.label : path.basename(f), f);
+    else args.push("-label", r === 0 ? t.label : "", "xc:#222222");
+  }
+  const font = pickFont();
+  args.push("-tile", `${cols.length}x${rows}`, "-geometry", `${tileWidth}x+8+8`, "-background", "#111111", "-fill", "#eeeeee", "-pointsize", "22", ...(font ? ["-font", font] : []), out);
+  fs.mkdirSync(path.dirname(out), { recursive: true });
+  const r = spawnSync("montage", args, { encoding: "utf8", timeout: 120000 });
+  return r.status === 0 && fs.existsSync(out) ? out : null;
+}

package/lib/shot.ts

@@ -0,0 +1,90 @@
+#!/usr/bin/env node
+// Screenshot a route of a web app with headless Chrome, starting the app first when asked. No dependencies.
+//   node shot.ts --url /pricing --out shots/pricing.png --serve "npm start"        (app is given $PORT)
+//   node shot.ts --url http://localhost:3000/ --out shots/home.png                 (already running)
+// Options: --port N (default: a free one), --width 1440 --height 900, --wait ms (settle time after load, default 1500),
+//          --timeout s (how long to wait for the server, default 180), --chrome <path> (or $LOOPS_CHROME / $CHROME).
+import { spawn, spawnSync, type ChildProcess } from "node:child_process";
+import fs from "node:fs";
+import net from "node:net";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+const CHROMES = ["google-chrome", "google-chrome-stable", "chromium", "chromium-browser", "chrome", "brave-browser", "microsoft-edge",
+  "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", "/Applications/Chromium.app/Contents/MacOS/Chromium",
+  "/Applications/Brave Browser.app/Contents/MacOS/Brave Browser", "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge"];
+
+export function findChrome(explicit?: string): string {
+  const cands = [explicit, process.env.LOOPS_CHROME, process.env.CHROME, process.env.PUPPETEER_EXECUTABLE_PATH, ...CHROMES].filter((c): c is string => !!c);
+  for (const c of cands) {
+    if (c.includes("/")) { if (fs.existsSync(c)) return c; continue; }
+    for (const dir of (process.env.PATH ?? "").split(path.delimiter)) { const p = path.join(dir, c); if (fs.existsSync(p)) return p; }
+  }
+  throw new Error("no Chrome/Chromium found; install one or set LOOPS_CHROME=/path/to/chrome");
+}
+
+export const freePort = () => new Promise<number>((res, rej) => { const s = net.createServer(); s.unref(); s.on("error", rej); s.listen(0, "127.0.0.1", () => { const addr = s.address(); const port = typeof addr === "object" && addr ? addr.port : 0; s.close(() => res(port)); }); });
+
+async function waitFor(url: string, timeoutMs: number, child: ChildProcess | null) {
+  const t0 = Date.now();
+  while (Date.now() - t0 < timeoutMs) {
+    if (child && child.exitCode != null) throw new Error(`server exited with code ${child.exitCode} before answering`);
+    try { const r = await fetch(url, { redirect: "manual", signal: AbortSignal.timeout(5000) }); if (r.status < 500) return; } catch {}
+    await new Promise((r) => setTimeout(r, 500));
+  }
+  throw new Error(`server did not answer at ${url} within ${timeoutMs / 1000}s`);
+}
+
+export async function shot({ url, out, serve, port, width = 1440, height = 900, wait = 1500, timeout = 180, chrome, cwd = process.cwd(), log = () => {} }: {
+  url?: string; out?: string; serve?: string; port?: number; width?: number; height?: number;
+  wait?: number; timeout?: number; chrome?: string; cwd?: string; log?: (msg: string) => void;
+}) {
+  if (!url || !out) throw new Error("shot needs --url and --out");
+  const bin = findChrome(chrome);
+  let child: ChildProcess | null = null, logFile: string | null = null;
+  if (serve) {
+    port ??= await freePort();
+    logFile = path.join(cwd, `.loops-serve-${port}.log`);
+    const fd = fs.openSync(logFile, "w");
+    child = spawn("sh", ["-c", serve], { cwd, env: { ...process.env, PORT: String(port) }, stdio: ["ignore", fd, fd], detached: true });
+    fs.closeSync(fd);
+    log(`serving on :${port} (${serve})`);
+  }
+  const target = /^https?:\/\//.test(url) ? url : `http://localhost:${port ?? (() => { throw new Error("a relative --url needs --serve or --port"); })()}${url.startsWith("/") ? "" : "/"}${url}`;
+  try {
+    if (child || port) await waitFor(target, timeout * 1000, child);
+    fs.mkdirSync(path.dirname(path.resolve(cwd, out)), { recursive: true });
+    const args = ["--headless=new", "--no-sandbox", "--disable-gpu", "--hide-scrollbars", "--disable-dev-shm-usage", `--window-size=${width},${height}`,
+      `--virtual-time-budget=${wait}`, `--screenshot=${path.resolve(cwd, out)}`, target];
+    const r = spawnSync(bin, args, { cwd, encoding: "utf8", timeout: 120000 });
+    if (r.status !== 0 || !fs.existsSync(path.resolve(cwd, out))) throw new Error(`chrome failed (${r.status}): ${(r.stderr || "").split("\n").filter((l) => !/dbus|gpu|vulkan|fontconfig/i.test(l)).slice(-3).join(" ")}`);
+    return path.resolve(cwd, out);
+  } finally {
+    if (child) {
+      if (child.pid) { try { process.kill(-child.pid, "SIGTERM"); } catch {} }
+      await new Promise((r) => setTimeout(r, 1500));
+      if (child.pid) { try { process.kill(-child.pid, "SIGKILL"); } catch {} }
+      try { if (logFile) fs.rmSync(logFile, { force: true }); } catch {}
+    }
+  }
+}
+
+type ShotArgs = Partial<{ url: string; out: string; serve: string; port: number; width: number; height: number; wait: number; timeout: number; chrome: string }>;
+
+function parseArgs(argv: string[]): ShotArgs {
+  const o: Record<string, unknown> = {};
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i];
+    if (!a.startsWith("--")) continue;
+    const k = a.slice(2); const v = argv[i + 1];
+    if (["width", "height", "wait", "timeout", "port"].includes(k)) { o[k] = Number(v); i++; }
+    else { o[k] = v; i++; }
+  }
+  return o as ShotArgs;
+}
+
+if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
+  shot({ ...parseArgs(process.argv.slice(2)), log: (m: string) => console.error(m) })
+    .then((f) => { console.log(f); })
+    .catch((e) => { console.error(`shot: ${e.message}`); process.exit(1); });
+}

package/package.json

@@ -1,6 +1,66 @@
 {
   "name": "@computesdk/pi-loops",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.1.0",
+  "description": "Hill-climbing loops as Pi units: one `loop` tool, style agents, forked attempt sessions in git worktrees, a judge keeps the best.",
+  "keywords": [
+    "pi-package",
+    "pi",
+    "pi-extension",
+    "loops",
+    "hill-climbing",
+    "agents",
+    "git-worktree",
+    "judge",
+    "benchmark",
+    "optimisation"
+  ],
+  "license": "MIT",
+  "type": "module",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/computesdk/pi-extensions.git",
+    "directory": "packages/pi-loops"
+  },
+  "homepage": "https://github.com/computesdk/pi-extensions/tree/main/packages/pi-loops#readme",
+  "bugs": {
+    "url": "https://github.com/computesdk/pi-extensions/issues"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "engines": {
+    "node": ">=22.18.0"
+  },
+  "pi": {
+    "extensions": [
+      "./extensions/loop.ts"
+    ],
+    "prompts": [
+      "./prompts/*.md"
+    ]
+  },
+  "files": [
+    "extensions",
+    "lib",
+    "agents",
+    "prompts",
+    "README.md",
+    "LICENSE"
+  ],
+  "scripts": {
+    "check": "tsc --noEmit",
+    "test": "tsc --noEmit && node test/rubric.ts && node test/judge.ts && node test/measure.ts && node test/loop.ts"
+  },
+  "peerDependencies": {
+    "@earendil-works/pi-coding-agent": "*",
+    "@earendil-works/pi-tui": "*",
+    "typebox": "*"
+  },
+  "devDependencies": {
+    "@earendil-works/pi-coding-agent": "catalog:",
+    "@earendil-works/pi-tui": "catalog:",
+    "@types/node": "catalog:",
+    "typebox": "catalog:",
+    "typescript": "catalog:"
+  }
+}

package/prompts/init.md

@@ -0,0 +1,6 @@
+---
+description: Set this project up for loops (serve, check, bench, protect) — written once to .pi/loops.json
+---
+$INIT
+
+If it comes back "change: …", revise as asked and call loop_init again; if "discuss", talk it through with me here first. When it is saved, say so in one line.

package/prompts/loop.md

@@ -0,0 +1,17 @@
+---
+description: Run a hill-climbing round — three forked attempts, a judge keeps the best
+argument-hint: <goal>
+---
+Goal for a hill-climbing loop: $@
+
+$PROJECT
+
+Write the brief the attempts get. Read, don't run — read tools only, no bash, no edits; the attempts and the judge run things. The brief:
+
+goal: what good looks like for this ask, one or two sentences an attempt can aim at
+rules: what the judge must enforce, as short testable statements (truthfulness, scope, style, what must not change), or none
+directions: a goal-specific angle for each of safe / bold / wild, or none for the standard ones
+routes: when the ask is about how something looks — the route(s) to screenshot (needs the project's serve); else none
+bench: the project's bench command, only when the ask is about its number; else none
+
+Then call the `loop` tool once with exactly those fields (`goal`, `rules`, `directions`, `routes`, `bench` only where the brief has them; check / protect come from the project setup, pass them only to override) — don't paste the brief in chat first; the tool shows it to me and asks approve / change / discuss. If it comes back "change: …", revise and call it again; if "discuss", talk it through with me here and call it again when we agree. Once approved it returns as soon as the round has started; the round card (table, climb, winner's diff stat) arrives here as a message when the judge is done — show it as-is and tell me how to run another round (`loop({ from })`). While the round runs, act on what I say with the steering tools: `loop_status` when I ask how it is going, `loop_talk` to tell one attempt something, `loop_rule` to add a rule for every attempt and the judge, `loop_pick` when I decide the winner myself, `loop_stop` to abort it; after the card, `loop_apply` when I say to take the winner.