@magland/gentl 0.0.0-stage → 0.1.0

package/dist/run.js

@@ -0,0 +1,423 @@
+// The agent loop: one model call per turn, one action per turn, everything recorded in ./runs/<id>/.
+import { randomBytes } from "node:crypto";
+import * as fs from "node:fs";
+import * as path from "node:path";
+import * as readline from "node:readline";
+import { createModelClient, Interrupted } from "./model.js";
+import { evaluateRun } from "./evaluate.js";
+import { describeFinalState } from "./finalstate.js";
+import { uploadRun } from "./hub.js";
+import { configureLocalCopy, describeLocalCopy, localCopyInfo, prefixCollisions } from "./localcopy.js";
+import { applyNotes, ASK_RESULT, describeAction, hasNotesDecision, NOTES_REMINDER, parseResponse, RESPONSE_SCHEMA, systemPrompt, validateAction } from "./protocol.js";
+import { fence } from "./format.js";
+import { ntok } from "./tokens.js";
+import { DockerSandbox } from "./docker.js";
+import { HostSandbox } from "./host.js";
+export const FORMAT_VERSION = 1;
+function formatAction(a) {
+    switch (a.type) {
+        case "run":
+            return `### Action: run${a.max_chars !== undefined ? ` (max_chars ${a.max_chars})` : ""}\n${fence(String(a.cmd), "bash")}`;
+        case "write_file":
+            return `### Action: write_file \`${a.path}\`\n${fence(String(a.content))}`;
+        case "edit_file":
+            return `### Action: edit_file \`${a.path}\`\nold:\n${fence(String(a.old))}new:\n${fence(String(a.new))}`;
+        case "ask":
+            return `### Action: ask\n\n${a.question}\n`;
+        case "finish":
+            return `### Action: finish (success: ${a.success})\n\n${a.summary}\n${a.friction ? `\nFriction:\n\n${a.friction}\n` : ""}`;
+        default:
+            return `### Action (unrecognized)\n${fence(JSON.stringify(a, null, 2), "json")}`;
+    }
+}
+function formatSummary(s) {
+    const outcome = s.finished ? `finished, self-reported ${s.self_reported_success ? "success" : "failure"}`
+        : `did not finish (${s.stop_reason}${s.error ? `: ${s.error}` : ""})`;
+    const rows = [
+        ["Outcome", outcome],
+        ["Turns", s.turns],
+        ["Read tokens total", s.read_tokens_total],
+        ["Notes tokens (total / max)", `${s.notes_tokens_total} / ${s.notes_tokens_max}`],
+        ["Result tokens (total / max)", `${s.result_tokens_total} / ${s.result_tokens_max}`],
+        ["Overhead tokens (system prompt + task, once)", s.overhead_tokens],
+        ["Model output tokens", s.model_output_tokens],
+        ["Parse errors / invalid / repeated / declined actions",
+            `${s.parse_errors} / ${s.invalid_actions} / ${s.repeated_actions} / ${s.declined_actions}`],
+        ["Questions asked", s.asks.length],
+        ["Notes reminders (responses without a notes decision)", s.notes_reminders],
+        ["Cost", `$${s.cost_usd.toFixed(4)}${s.backend === "claude-code" ? " (notional; Claude Code subscription)" : ""}`],
+        ["Duration", `${Math.round(s.duration_s)} s`],
+    ];
+    return rows.map(([k, v]) => `- ${k}: ${v}`).join("\n") + "\n";
+}
+function firstLine(s, max) {
+    const line = s.split("\n")[0];
+    const cut = line.length > max ? line.slice(0, max) : line;
+    return cut === s ? cut : `${cut} …`;
+}
+function indent(text, maxLines) {
+    const lines = text.replace(/\n$/, "").split("\n");
+    const shown = lines.slice(0, maxLines).map((l) => `       ${l}`);
+    if (lines.length > maxLines)
+        shown.push(`       (${lines.length - maxLines} more lines)`);
+    return shown.join("\n");
+}
+let activePrompt = null;
+/** Ask a question on the terminal. Ctrl-C or Ctrl-D ends the run. */
+function ask(question) {
+    const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
+    activePrompt = rl;
+    return new Promise((resolve, reject) => {
+        let answered = false;
+        rl.on("SIGINT", () => {
+            process.stdout.write("\n");
+            rl.close();
+        });
+        rl.on("close", () => {
+            activePrompt = null;
+            if (!answered)
+                reject(new Interrupted());
+        });
+        rl.question(question, (answer) => {
+            answered = true;
+            rl.close();
+            resolve(answer.trim());
+        });
+    });
+}
+/** Show the action and ask for approval. Returns null if approved, else the reason given (possibly empty). */
+async function confirm(a, lineShown) {
+    if (a.type === "run" && lineShown !== a.cmd)
+        console.log(indent(String(a.cmd), 40));
+    if (a.type === "write_file")
+        console.log(indent(String(a.content), 30));
+    if (a.type === "edit_file")
+        console.log(`       old:\n${indent(String(a.old), 15)}\n       new:\n${indent(String(a.new), 15)}`);
+    for (;;) {
+        const ans = (await ask("     ok? [y/n] ")).toLowerCase();
+        if (ans === "y" || ans === "yes")
+            return null;
+        if (ans === "n" || ans === "no")
+            return await ask("     reason (optional): ");
+    }
+}
+function actionLine(a) {
+    switch (a.type) {
+        case "run":
+            return `run: ${firstLine(String(a.cmd), 90)}`;
+        case "write_file":
+        case "edit_file":
+            return `${a.type}: ${a.path}`;
+        case "ask":
+            return `ask: ${firstLine(String(a.question), 90)}`;
+        case "finish":
+            return `finish: ${a.success ? "success" : "failure"}`;
+        default:
+            return firstLine(JSON.stringify(a), 90);
+    }
+}
+export function timestamp(d) {
+    const p = (n) => String(n).padStart(2, "0");
+    return `${d.getFullYear()}${p(d.getMonth() + 1)}${p(d.getDate())}-${p(d.getHours())}${p(d.getMinutes())}${p(d.getSeconds())}`;
+}
+export async function runTask(opts) {
+    const started = new Date();
+    const runId = `${timestamp(started)}-${opts.taskName}`;
+    const lc = opts.localCopy;
+    const MIRROR_MOUNT = "/srv/git";
+    const sb = opts.sandbox.kind === "docker"
+        // The random suffix keeps names unique when two gentl processes start the same task in the same second.
+        ? new DockerSandbox(`gentl-${runId.toLowerCase().replace(/[^a-z0-9_.-]/g, "-")}-${randomBytes(3).toString("hex")}`, opts.sandbox.image, lc ? [{ host: lc.mirrorDir, container: MIRROR_MOUNT }] : [])
+        : new HostSandbox();
+    // Signal handlers go in as soon as the sandbox exists, so no signal can leave it behind.
+    // Ctrl-C stops the running command, or cancels the model call and ends the run.
+    // SIGTERM and SIGHUP end the run wherever it is; the run directory is still completed.
+    // During cleanup further signals are ignored, so a second Ctrl-C cannot leave the sandbox behind.
+    let abortModel = null;
+    let stopRequested = false;
+    let cleaningUp = false;
+    const onSigint = () => {
+        if (cleaningUp)
+            return console.log("     still cleaning up (removing the sandbox); please wait");
+        if (sb.interrupt())
+            return;
+        stopRequested = true;
+        abortModel?.abort();
+    };
+    const onTerm = () => {
+        if (cleaningUp)
+            return;
+        stopRequested = true;
+        sb.interrupt();
+        abortModel?.abort();
+        activePrompt?.close();
+    };
+    process.on("SIGINT", onSigint);
+    process.on("SIGTERM", onTerm);
+    process.on("SIGHUP", onTerm);
+    const localCopyError = lc
+        ? configureLocalCopy(sb, lc, sb.kind === "docker" ? `${MIRROR_MOUNT}/${lc.mirrorName}` : path.join(lc.mirrorDir, lc.mirrorName))
+        : null;
+    const runDir = path.resolve(opts.runsDir, runId);
+    fs.mkdirSync(runDir, { recursive: true });
+    const events = fs.openSync(path.join(runDir, "events.jsonl"), "w");
+    const transcript = fs.openSync(path.join(runDir, "transcript.md"), "w");
+    const emit = (ev) => fs.writeSync(events, JSON.stringify(ev) + "\n");
+    const tr = (text) => fs.writeSync(transcript, text + "\n");
+    const log = (msg) => {
+        console.log(`     ${msg}`);
+        tr(`_${msg}_\n`);
+    };
+    const system = systemPrompt(sb.workdir, sb.kind === "docker" ? "linux" : process.platform);
+    const client = createModelClient(opts.model, { name: "turn", schema: RESPONSE_SCHEMA }, log, sb.kind === "host" ? sb.workdir : undefined);
+    const s = {
+        format: FORMAT_VERSION, gentl_version: opts.version, run_id: runId, task_name: opts.taskName,
+        task_source: opts.taskSource, task: opts.task, model: opts.model.model, backend: opts.model.backend,
+        base_url: opts.model.backend === "openrouter" ? opts.model.baseUrl : null,
+        sandbox: sb.kind, image: sb instanceof DockerSandbox ? sb.image : null,
+        image_id: sb instanceof DockerSandbox ? sb.imageId : null, confirm: opts.confirm,
+        local_copy: lc && !localCopyError ? localCopyInfo(lc) : null, structured_output: true, started: started.toISOString(), ended: "", duration_s: 0,
+        system_prompt: system, overhead_tokens: ntok(system) + ntok(opts.task), turns: 0,
+        read_tokens_total: 0, notes_tokens_total: 0, notes_tokens_max: 0, result_tokens_total: 0, result_tokens_max: 0,
+        model_output_tokens: 0, provider_prompt_tokens: 0, provider_completion_tokens: 0, cost_usd: 0,
+        parse_errors: 0, invalid_actions: 0, repeated_actions: 0, declined_actions: 0, notes_reminders: 0, asks: [],
+        stop_reason: null, error: null, finished: false,
+        self_reported_success: null, self_report_summary: null, self_report_friction: null, final_repos: null,
+    };
+    const ts = opts.taskSource;
+    const source = ts.kind === "file" ? ts.path : ts.kind === "hub" ? `${ts.title} (${ts.url})` : "given on the command line";
+    console.log(`gentl ${opts.version}  run ${runId}`);
+    console.log(`model ${opts.model.model} (${opts.model.backend})  ${sb.describe()}${opts.confirm ? "  (confirm each action)" : ""}`);
+    if (lc && !localCopyError) {
+        console.log(`local copy: ${describeLocalCopy(lc)}`);
+        for (const u of prefixCollisions(opts.task, lc.urls)) {
+            console.log(`warning: the task mentions ${u}, which the local copy would also redirect (git insteadOf matches by prefix); use --remote if that is wrong`);
+        }
+    }
+    if (localCopyError)
+        console.log(`warning: local copy disabled: ${localCopyError}`);
+    console.log(`\nturn  +read (cumulative)  notes  action`);
+    tr(`# gentl run ${runId}\n\n- Model: ${opts.model.model} (${opts.model.backend})\n- Task: ${source}\n- Sandbox: ${sb.describe()}\n- Local copy: ${lc && !localCopyError ? describeLocalCopy(lc) : "none"}\n` +
+        `- Started: ${started.toISOString()}\n\n## Task\n\n${opts.task}\n\n## System prompt\n\n${fence(system)}`);
+    emit({ type: "start", run_id: runId, task: opts.task, task_source: opts.taskSource, model: opts.model.model,
+        backend: opts.model.backend, base_url: s.base_url, sandbox: sb.kind, sandbox_description: sb.describe(), workdir: sb.workdir, confirm: opts.confirm, system_prompt: system });
+    let notes = "";
+    let last = null;
+    let lastActionJson = null;
+    try {
+        for (let turn = 1; turn <= opts.maxTurns; turn++) {
+            if (stopRequested)
+                throw new Interrupted();
+            let user = `# Task\n${opts.task}\n\n# Your notes\n${notes || "(empty)"}\n`;
+            if (last)
+                user += `\n# Last action\n${last.action}\n\n# Result\n${last.result}\n`;
+            const notesTok = ntok(notes);
+            let resultTok = last ? ntok(last.action) + ntok(last.result) : 0;
+            s.turns = turn;
+            s.notes_tokens_total += notesTok;
+            s.notes_tokens_max = Math.max(s.notes_tokens_max, notesTok);
+            s.result_tokens_total += resultTok;
+            s.result_tokens_max = Math.max(s.result_tokens_max, resultTok);
+            s.read_tokens_total = s.notes_tokens_total + s.result_tokens_total;
+            tr(`---\n\n## Turn ${turn}\n\nRead this turn: notes ${notesTok} + result ${resultTok} tokens (cumulative ${s.read_tokens_total})\n`);
+            const callModel = async (input) => {
+                const t0 = Date.now();
+                abortModel = new AbortController();
+                const reply = await client.call(system, input, abortModel.signal);
+                abortModel = null;
+                s.structured_output = client.structured;
+                const outTok = ntok(reply.text);
+                s.model_output_tokens += outTok;
+                s.provider_prompt_tokens += Number(reply.usage.prompt_tokens ?? 0);
+                s.provider_completion_tokens += Number(reply.usage.completion_tokens ?? 0);
+                s.cost_usd += Number(reply.usage.cost ?? 0);
+                const seconds = (Date.now() - t0) / 1000;
+                tr(`_model: ${seconds.toFixed(1)} s, ${outTok} output tokens_\n`);
+                return { reply, seconds };
+            };
+            const tryParse = (text) => {
+                try {
+                    return { parsed: parseResponse(text), error: null };
+                }
+                catch (e) {
+                    return { parsed: null, error: e.message };
+                }
+            };
+            let { reply, seconds: modelSeconds } = await callModel(user);
+            let { parsed, error: parseError } = tryParse(reply.text);
+            const ev = { type: "turn", turn, input: user, notes_tokens: notesTok, usage: reply.usage };
+            // No notes decision: ask again within the turn, while the result is still in view. The reminder counts as read.
+            if (parsed && !hasNotesDecision(parsed)) {
+                s.notes_reminders++;
+                const reminderTok = ntok(NOTES_REMINDER);
+                resultTok += reminderTok;
+                s.result_tokens_total += reminderTok;
+                s.result_tokens_max = Math.max(s.result_tokens_max, resultTok);
+                s.read_tokens_total = s.notes_tokens_total + s.result_tokens_total;
+                tr(`### No notes decision; reminder sent (${reminderTok} tokens)\n\nFirst response:\n${fence(reply.text)}`);
+                Object.assign(ev, { response_without_notes: reply.text, usage_without_notes: reply.usage });
+                const retryInput = `${user}\n# Reminder\n${NOTES_REMINDER}\n`;
+                ({ reply, seconds: modelSeconds } = await callModel(retryInput));
+                ({ parsed, error: parseError } = tryParse(reply.text));
+                Object.assign(ev, { input: retryInput, usage: reply.usage, notes_reminder: true });
+            }
+            Object.assign(ev, { response: reply.text, result_tokens: resultTok, model_seconds: modelSeconds });
+            const prefix = `t${String(turn).padStart(2, "0")}  +${resultTok + notesTok} (${s.read_tokens_total.toLocaleString("en-US")})`.padEnd(26) +
+                String(notesTok).padEnd(7) + (ev.notes_reminder ? "[reminded] " : "");
+            if (!parsed) {
+                s.parse_errors++;
+                const msg = parseError;
+                console.log(`${prefix}parse error: ${msg}`);
+                tr(`### Parse error: ${msg}\n\nRaw response:\n${fence(reply.text)}`);
+                last = { action: "(none: your response could not be parsed)",
+                    result: `error: ${msg}. Respond with one JSON object as described.` };
+                emit({ ...ev, parse_error: msg });
+                continue;
+            }
+            const n = applyNotes(notes, parsed);
+            notes = n.notes;
+            const a = parsed.action;
+            const line = actionLine(a);
+            console.log(prefix + line);
+            tr(`### Notes (${n.ops.length ? n.ops.join(", ") : parsed.notes_keep === true ? "kept" : "unchanged"}; ${ntok(notes)} tokens)\n\n${fence(notes || "(empty)")}`);
+            for (const f of n.feedback)
+                tr(`_${f}_\n`);
+            tr(formatAction(a));
+            Object.assign(ev, { parsed, notes, notes_ops: n.ops, notes_feedback: n.feedback });
+            const invalid = validateAction(a);
+            if (!invalid && a.type === "finish") {
+                Object.assign(s, { finished: true, stop_reason: "finish", self_reported_success: a.success,
+                    self_report_summary: a.summary, self_report_friction: a.friction ?? null });
+                emit(ev);
+                break;
+            }
+            const notices = [];
+            const actionJson = JSON.stringify(a);
+            if (actionJson === lastActionJson) {
+                s.repeated_actions++;
+                notices.push("note: this action is identical to your previous one.");
+            }
+            lastActionJson = actionJson;
+            let full;
+            let shown;
+            if (invalid) {
+                s.invalid_actions++;
+                full = shown = `error: malformed action: ${invalid}`;
+            }
+            else if (a.type === "ask") {
+                s.asks.push({ turn, question: String(a.question) });
+                full = shown = ASK_RESULT;
+            }
+            else {
+                const reason = opts.confirm ? await confirm(a, line.replace(/^run: /, "")) : null;
+                if (reason !== null) {
+                    s.declined_actions++;
+                    full = shown = `The person running gentl declined this action.${reason ? ` Their reason: ${reason}` : ""}`;
+                    ev.declined = reason;
+                }
+                else if (a.type === "run") {
+                    const r = await sb.run(String(a.cmd), a.max_chars);
+                    if (r.sandboxError)
+                        throw new Error(`the sandbox failed: ${r.sandboxError.trim()}`);
+                    ({ full, shown } = r);
+                    Object.assign(ev, { truncation: r.truncation ?? undefined, exit_code: r.exitCode, signal: r.signal, cmd_seconds: r.seconds, timed_out: r.timedOut,
+                        interrupted: r.interrupted });
+                }
+                else if (a.type === "write_file") {
+                    full = shown = sb.writeFile(String(a.path), String(a.content));
+                }
+                else {
+                    full = shown = sb.editFile(String(a.path), String(a.old), String(a.new));
+                }
+            }
+            shown = [...notices, ...n.feedback, shown].join("\n");
+            last = { action: describeAction(a), result: shown };
+            emit({ ...ev, notices, result_full: full, result_shown: shown });
+            const truncNote = shown.length < full.length ? `; truncated, full result ${full.length} chars in events.jsonl` : "";
+            tr(`### Result (${ntok(shown)} tokens${truncNote})\n${fence(shown)}`);
+        }
+        if (!s.stop_reason)
+            s.stop_reason = "max_turns";
+    }
+    catch (e) {
+        if (e instanceof Interrupted)
+            s.stop_reason = "interrupted";
+        else {
+            s.stop_reason = "error";
+            s.error = e.message;
+        }
+    }
+    finally {
+        cleaningUp = true;
+        const ended = new Date();
+        s.ended = ended.toISOString();
+        s.duration_s = (ended.getTime() - started.getTime()) / 1000;
+        try {
+            const fin = describeFinalState(sb);
+            fs.writeFileSync(path.join(runDir, "final-state.md"), fin.text);
+            s.final_repos = fin.repos;
+        }
+        catch (e) {
+            log(`could not record the final state: ${e.message}`);
+        }
+        fs.writeFileSync(path.join(runDir, "summary.json"), JSON.stringify(s, null, 2) + "\n");
+        emit({ type: "end", stop_reason: s.stop_reason, error: s.error });
+        tr(`---\n\n## Summary\n\n${formatSummary(s)}`);
+        fs.closeSync(events);
+        fs.closeSync(transcript);
+        if (!opts.keep)
+            sb.remove();
+        else if (sb instanceof DockerSandbox)
+            console.log(`\ncontainer kept: ${sb.name} (remove with: docker rm -f ${sb.name})`);
+        else if (sb instanceof HostSandbox)
+            console.log(`\nworkspace kept: ${sb.root}`);
+        process.off("SIGINT", onSigint);
+        process.off("SIGTERM", onTerm);
+        process.off("SIGHUP", onTerm);
+    }
+    console.log(`\n${formatSummary(s)}`);
+    if (s.self_report_summary)
+        console.log(`Agent's summary: ${s.self_report_summary}\n`);
+    if (s.self_report_friction)
+        console.log(`Agent's friction report: ${s.self_report_friction}\n`);
+    if (s.stop_reason === "finish" || s.stop_reason === "max_turns") {
+        const ev = await evaluate(runDir, opts.evaluatorModel);
+        if (ev)
+            s.evaluation = { model: ev.model, success: ev.success, reasons: ev.reasons, cost_usd: ev.cost_usd };
+    }
+    console.log(`Run directory: ${runDir}`);
+    if (opts.upload) {
+        if (s.stop_reason === "finish" || s.stop_reason === "max_turns") {
+            try {
+                console.log(`Uploaded: ${await uploadRun(runDir, opts.upload)}`);
+            }
+            catch (e) {
+                console.log(`Upload failed (${e.message}). Retry with: gentl upload ${path.relative(process.cwd(), runDir)}`);
+            }
+        }
+        else
+            console.log(`Not uploaded: the run did not finish (${s.stop_reason}).`);
+    }
+    return { runDir, summary: s };
+}
+/** Run the evaluator on a run directory and print its verdict and friction report. Ctrl-C skips it. */
+export async function evaluate(runDir, cfg) {
+    console.log(`Evaluating with ${cfg.model} ...`);
+    const ac = new AbortController();
+    const onSigint = () => ac.abort();
+    process.on("SIGINT", onSigint);
+    try {
+        const ev = await evaluateRun(runDir, cfg, ac.signal, (msg) => console.log(`     ${msg}`));
+        console.log(`\nVerdict: ${ev.success ? "success" : "failure"}\n\n${ev.reasons}\n\nFriction report:\n\n${ev.friction_report}\n`);
+        console.log(`Evaluator cost: $${ev.cost_usd.toFixed(4)}; written to friction.md, evaluation.json and summary.json`);
+        return ev;
+    }
+    catch (e) {
+        const msg = e instanceof Interrupted ? "interrupted" : e.message;
+        console.log(`Evaluation skipped (${msg}). Run it later with: gentl evaluate ${path.relative(process.cwd(), runDir)}`);
+        return null;
+    }
+    finally {
+        process.off("SIGINT", onSigint);
+    }
+}

package/dist/sandbox.js

@@ -0,0 +1,129 @@
+// Where the agent's actions run. A sandbox runs commands with a timeout in their own process group, writes and
+// edits files, and runs short commands for gentl itself (e.g. to record the final state).
+import { spawnSync } from "node:child_process";
+import { CMD_TIMEOUT_S, countOccurrences, DEFAULT_MAX_CHARS, MAX_MAX_CHARS } from "./protocol.js";
+const CAPTURE_HEAD = 1 << 20; // bytes of output kept from the start of a command
+const CAPTURE_TAIL = 1 << 20; // and from the end
+/** Keep the head and tail, with a note on what was cut and how to see more. */
+export function truncate(out, maxChars) {
+    if (out.length <= maxChars)
+        return out;
+    const head = out.slice(0, Math.floor((maxChars * 3) / 5));
+    const tail = out.slice(out.length - Math.floor((maxChars * 2) / 5));
+    const omitted = out.slice(head.length, out.length - tail.length);
+    const lines = countOccurrences(omitted, "\n");
+    return `${head}\n[... ${omitted.length} chars, ${lines} lines omitted; narrow the command (grep, sed -n, head) or raise max_chars (at most ${MAX_MAX_CHARS}) ...]\n${tail}`;
+}
+/** Collects output, keeping only the first and last megabyte of very long output. */
+class Capture {
+    head = Buffer.alloc(0);
+    tail = [];
+    tailLen = 0;
+    dropped = 0;
+    add(b) {
+        if (this.head.length < CAPTURE_HEAD) {
+            const n = Math.min(CAPTURE_HEAD - this.head.length, b.length);
+            this.head = Buffer.concat([this.head, b.subarray(0, n)]);
+            b = b.subarray(n);
+        }
+        if (!b.length)
+            return;
+        this.tail.push(b);
+        this.tailLen += b.length;
+        while (this.tailLen - this.tail[0].length >= CAPTURE_TAIL) {
+            const first = this.tail.shift();
+            this.tailLen -= first.length;
+            this.dropped += first.length;
+        }
+    }
+    text() {
+        const gap = this.dropped ? `\n[... ${this.dropped} bytes not captured ...]\n` : "";
+        return this.head.toString("utf8") + gap + Buffer.concat(this.tail).toString("utf8");
+    }
+}
+export class Sandbox {
+    current = null;
+    run(cmd, maxChars = DEFAULT_MAX_CHARS) {
+        maxChars = Math.max(200, Math.min(MAX_MAX_CHARS, Math.floor(maxChars)));
+        const t0 = Date.now();
+        const started = this.start(cmd);
+        const { child } = started;
+        const cur = { started, interrupted: false };
+        this.current = cur;
+        const cap = new Capture();
+        child.stdout.on("data", (b) => cap.add(b));
+        let sandboxStderr = "";
+        child.stderr.on("data", (b) => {
+            if (started.sandboxStderr)
+                sandboxStderr += b.toString("utf8");
+            else
+                cap.add(b);
+        });
+        let timedOut = false;
+        const timer = setTimeout(() => {
+            timedOut = true;
+            started.kill();
+        }, CMD_TIMEOUT_S * 1000);
+        return new Promise((resolve) => {
+            let done = false;
+            const finish = (code, signal) => {
+                if (done)
+                    return;
+                done = true;
+                clearTimeout(timer);
+                this.current = null;
+                const classified = started.sandboxStderr?.(sandboxStderr) ?? { keep: "", fatal: null };
+                if (classified.keep)
+                    cap.add(Buffer.from(classified.keep));
+                const out = cap.text();
+                const status = cur.interrupted ? "stopped by the person running gentl (Ctrl-C)"
+                    : timedOut ? `timed out after ${CMD_TIMEOUT_S} s` : code !== null ? `exit code ${code}` : `killed by ${signal}`;
+                const truncation = out.length > maxChars
+                    ? { head: status.length + 1 + Math.floor((maxChars * 3) / 5), tail: Math.floor((maxChars * 2) / 5) } : null;
+                resolve({ exitCode: code, signal, full: `${status}\n${out}`, shown: `${status}\n${truncate(out, maxChars)}`,
+                    truncation, seconds: (Date.now() - t0) / 1000, timedOut, interrupted: cur.interrupted,
+                    sandboxError: classified.fatal });
+            };
+            child.on("error", (e) => {
+                cap.add(Buffer.from(`error: ${e.message}\n`));
+                finish(null, "spawn error");
+            });
+            child.on("close", (code, signal) => finish(code, signal));
+            // A background process (`server &`) can keep the pipes open after the command exits; don't wait for it.
+            child.on("exit", (code, signal) => setTimeout(() => {
+                child.stdout.destroy();
+                child.stderr.destroy();
+                finish(code, signal);
+            }, 1000));
+        });
+    }
+    /** Stop the running command (Ctrl-C). Returns false if no command is running. */
+    interrupt() {
+        if (!this.current)
+            return false;
+        this.current.interrupted = true;
+        this.current.started.kill();
+        return true;
+    }
+    writeFile(p, content) {
+        const r = this.exec(["sh", "-c", 'mkdir -p "$(dirname "$1")" && cat > "$1"', "sh", p], undefined, content);
+        return r.ok ? `wrote ${p}` : `error: ${r.out.trim()}`;
+    }
+    editFile(p, old, replacement) {
+        const r = this.exec(["cat", "--", p]);
+        if (!r.ok)
+            return `error: ${r.out.trim()}`;
+        const n = countOccurrences(r.out, old);
+        if (n !== 1)
+            return `error: the old text occurs ${n} times in ${p}; it must occur exactly once`;
+        const w = this.exec(["sh", "-c", 'cat > "$1"', "sh", p], undefined, r.out.replace(old, () => replacement));
+        return w.ok ? `edited ${p}` : `error: ${w.out.trim()}`;
+    }
+}
+/** spawnSync wrapper returning stdout (plus stderr on failure). */
+export function execLocal(cmd, args, opts) {
+    const p = spawnSync(cmd, args, { ...opts, encoding: "utf8", maxBuffer: 64 << 20 });
+    if (p.error)
+        return { ok: false, out: p.error.message };
+    return { ok: p.status === 0, out: (p.stdout ?? "") + (p.status === 0 ? "" : p.stderr ?? "") };
+}

package/dist/suite.js

@@ -0,0 +1,114 @@
+// Batches: a suite (a directory of task files) and/or repeated runs (--repeat N). Runs go one after another into
+// runs/<timestamp>-<name>/, and a results table is printed and written there.
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { runTask, timestamp } from "./run.js";
+/** Task files in a suite directory: *.md, except README.md and files starting with "_". */
+export function suiteTasks(dir) {
+    return fs.readdirSync(dir)
+        .filter((f) => f.endsWith(".md") && f.toLowerCase() !== "readme.md" && !f.startsWith("_"))
+        .sort()
+        .map((f) => {
+        const file = path.join(dir, f);
+        return { taskName: path.basename(f, ".md"), task: fs.readFileSync(file, "utf8").trim(), taskSource: { kind: "file", path: file } };
+    });
+}
+function quantile(sorted, q) {
+    const pos = (sorted.length - 1) * q;
+    const lo = Math.floor(pos);
+    const hi = Math.ceil(pos);
+    return sorted[lo] + (sorted[hi] - sorted[lo]) * (pos - lo);
+}
+function stats(values) {
+    if (!values.length)
+        return null;
+    const s = [...values].sort((a, b) => a - b);
+    return { median: quantile(s, 0.5), q1: quantile(s, 0.25), q3: quantile(s, 0.75), min: s[0], max: s[s.length - 1] };
+}
+function aggregate(task, rows) {
+    const ended = rows.filter((r) => r.run_dir !== null);
+    const ok = ended.filter((r) => r.verdict === true);
+    return {
+        task, runs: ended.length, successes: ok.length,
+        read_tokens: stats(ok.map((r) => r.read_tokens_total)), turns: stats(ok.map((r) => r.turns)),
+        notes_tokens_max: stats(ok.map((r) => r.notes_tokens_max)),
+        cost_usd: ended.reduce((a, r) => a + (r.cost_usd ?? 0), 0),
+    };
+}
+function rowFor(task, rep, batchDir, runDir, s) {
+    return {
+        task, rep, run_dir: path.relative(batchDir, runDir), stop_reason: s.stop_reason ?? "unknown",
+        verdict: s.evaluation?.success ?? null, self_reported_success: s.self_reported_success, turns: s.turns,
+        read_tokens_total: s.read_tokens_total, notes_tokens_max: s.notes_tokens_max, asks: s.asks.length,
+        cost_usd: s.cost_usd + (s.evaluation?.cost_usd ?? 0),
+    };
+}
+function emptyRow(task, rep, stop) {
+    return { task, rep, run_dir: null, stop_reason: stop, verdict: null, self_reported_success: null, turns: null,
+        read_tokens_total: null, notes_tokens_max: null, asks: null, cost_usd: null };
+}
+function table(header, body) {
+    const widths = header.map((h, i) => Math.max(h.length, ...body.map((b) => b[i].length)));
+    const line = (cells) => `| ${cells.map((c, i) => c.padEnd(widths[i])).join(" | ")} |`;
+    return [line(header), line(widths.map((w) => "-".repeat(w))), ...body.map(line)].join("\n") + "\n";
+}
+const num = (v) => (v === null ? "-" : Math.round(v).toLocaleString("en-US"));
+const cost = (v) => (v === null ? "-" : `$${v.toFixed(4)}`);
+function runsTable(rows, repeat) {
+    const yn = (v) => (v === null ? "-" : v ? "yes" : "no");
+    return table(["task", ...(repeat > 1 ? ["rep"] : []), "verdict", "self-report", "stop", "turns", "read tokens", "notes max", "asks", "cost"], rows.map((r) => [r.task, ...(repeat > 1 ? [String(r.rep)] : []), r.verdict === null ? "-" : r.verdict ? "success" : "failure",
+        yn(r.self_reported_success), r.stop_reason, num(r.turns), num(r.read_tokens_total), num(r.notes_tokens_max),
+        num(r.asks), cost(r.cost_usd)]));
+}
+function aggregateTable(aggs) {
+    const range = (s) => (s ? `${num(s.min)}-${num(s.max)}` : "-");
+    const iqr = (s) => (s ? `${num(s.q1)}-${num(s.q3)}` : "-");
+    return table(["task", "success", "read median", "read IQR", "read range", "turns median", "notes max median", "cost"], aggs.map((a) => [a.task, `${a.successes}/${a.runs}`, num(a.read_tokens?.median ?? null), iqr(a.read_tokens),
+        range(a.read_tokens), num(a.turns?.median ?? null), num(a.notes_tokens_max?.median ?? null), cost(a.cost_usd)]));
+}
+/** Run each task `repeat` times. Returns false if any run failed with an error. */
+export async function runBatch(name, tasks, repeat, base) {
+    if (!tasks.length)
+        throw new Error("no tasks to run");
+    const started = new Date();
+    const batchDir = path.resolve("runs", `${timestamp(started)}-${name}`);
+    fs.mkdirSync(batchDir, { recursive: true });
+    const total = tasks.length * repeat;
+    console.log(`${name}: ${tasks.length} task(s) x ${repeat}, runs in ${batchDir}`);
+    const rows = [];
+    let stopped = false;
+    let i = 0;
+    for (const t of tasks) {
+        for (let rep = 1; rep <= repeat; rep++) {
+            i++;
+            if (stopped) {
+                rows.push(emptyRow(t.taskName, rep, "skipped"));
+                continue;
+            }
+            const label = repeat > 1 ? `${t.taskName} (run ${rep}/${repeat})` : t.taskName;
+            console.log(`\n=== ${i}/${total}: ${label} ===\n`);
+            try {
+                const out = await runTask({ ...base, task: t.task, taskName: repeat > 1 ? `${t.taskName}-r${rep}` : t.taskName,
+                    taskSource: t.taskSource, runsDir: batchDir });
+                rows.push(rowFor(t.taskName, rep, batchDir, out.runDir, out.summary));
+                if (out.summary.stop_reason === "interrupted") {
+                    console.log("\nRun interrupted; skipping the remaining runs.");
+                    stopped = true;
+                }
+            }
+            catch (e) {
+                console.log(`error: ${e.message}`);
+                rows.push(emptyRow(t.taskName, rep, "error"));
+            }
+        }
+    }
+    const aggs = tasks.map((t) => aggregate(t.taskName, rows.filter((r) => r.task === t.taskName)));
+    let report = runsTable(rows, repeat);
+    if (repeat > 1)
+        report = `Per task (read tokens, turns and notes over runs the evaluator judged successful):\n\n${aggregateTable(aggs)}\nAll runs:\n\n${report}`;
+    fs.writeFileSync(path.join(batchDir, "results.md"), `# ${name}\n\nStarted ${started.toISOString()}; ${tasks.length} task(s), ${repeat} run(s) each.\n\n${report}`);
+    fs.writeFileSync(path.join(batchDir, "results.json"), JSON.stringify({ format: 1, name, repeat, started: started.toISOString(),
+        ended: new Date().toISOString(), tasks: aggs, runs: rows }, null, 2) + "\n");
+    console.log(`\n=== ${name} ===\n\n${report}\nBatch directory: ${batchDir}`);
+    return rows.every((r) => r.stop_reason !== "error");
+}

package/dist/tokens.js

@@ -0,0 +1,7 @@
+import { Tiktoken } from "js-tiktoken/lite";
+import o200k_base from "js-tiktoken/ranks/o200k_base";
+const enc = new Tiktoken(o200k_base);
+/** Tokens in text (o200k_base). Special-token strings are counted as plain text. */
+export function ntok(text) {
+    return enc.encode(text, [], []).length;
+}