@magland/gentl 0.0.0-stage → 0.1.0

package/dist/cli.js

@@ -0,0 +1,316 @@
+#!/usr/bin/env node
+import * as fs from "node:fs";
+import * as path from "node:path";
+import * as readline from "node:readline";
+import { parseArgs } from "node:util";
+import { checkDocker, ensureDefaultImage } from "./docker.js";
+import { fetchHubTask, hubCredentials, parseHubTaskUrl, runDirsIn, runHub, uploadRun } from "./hub.js";
+import { findRepo, prepareLocalCopy, removeLocalCopy } from "./localcopy.js";
+import { evaluate, runTask } from "./run.js";
+import { runBatch, suiteTasks } from "./suite.js";
+const VERSION = JSON.parse(fs.readFileSync(new URL("../package.json", import.meta.url), "utf8")).version;
+const DEFAULT_MODEL = "anthropic/claude-haiku-5.5";
+const DEFAULT_BASE_URL = "https://openrouter.ai/api/v1";
+const USAGE = `gentl ${VERSION}: a readability test for codebases
+
+usage:
+  gentl run (TASK.md | DIR | URL | -t TEXT) [options]
+                                  run the agent on a task, then evaluate the run; DIR runs each
+                                  *.md in it as a suite; URL is a task on a hub (https://<hub>/tasks/<id>)
+  gentl evaluate RUN_DIR [options]  (re-)evaluate a finished run
+  gentl upload RUN_DIR...           upload runs (or batch directories of runs) to their task's hub
+
+options:
+  -t, --task TEXT     task text given directly
+  --image IMG         Docker image for the sandbox (default: built from gentl's sandbox/Dockerfile)
+  --host              run in a fresh temp directory on this machine instead of Docker
+  --confirm           ask before each action (the default with --host)
+  --yes               don't ask: run a hub task without confirming it, and with --host, run actions
+                      without asking
+  --upload            upload each finished run to the hub its task came from
+  --model M           model for the agent (default ${DEFAULT_MODEL})
+  --evaluator-model M model for the evaluator (default ${DEFAULT_MODEL})
+  --backend B         openrouter (any OpenAI-compatible endpoint; key in OPENROUTER_API_KEY) or
+                      claude-code (your Claude Code login, via claude -p); default from
+                      GENTL_BACKEND, else openrouter
+  --base-url URL      OpenAI-compatible endpoint (default ${DEFAULT_BASE_URL})
+  --max-turns N       stop after N turns (default 50)
+  --repeat N          run each task N times and report the success rate and median read tokens
+  --local             serve clones of the task's repo from your checkout, uncommitted changes included
+                      (the default when the task file, suite or current directory is in a git repo)
+  --remote            clone the published repo instead
+  --keep              keep the container or temp workspace instead of deleting it
+
+The API key is read from OPENROUTER_API_KEY, in the environment or in ./.env.
+Runs are written to ./runs/<id>/.`;
+/** A variable from the environment, or from ./.env. */
+function envValue(name) {
+    if (process.env[name])
+        return process.env[name];
+    if (!fs.existsSync(".env"))
+        return undefined;
+    for (const line of fs.readFileSync(".env", "utf8").split("\n")) {
+        const m = line.match(/^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*?)\s*$/);
+        if (m && m[1] === name)
+            return m[2].replace(/^(['"])(.*)\1$/, "$2");
+    }
+    return undefined;
+}
+function fail(msg) {
+    console.error(`gentl: ${msg}`);
+    process.exit(2);
+}
+async function run(argv) {
+    const { values, positionals } = parseArgs({
+        args: argv,
+        allowPositionals: true,
+        options: {
+            task: { type: "string", short: "t" },
+            host: { type: "boolean" },
+            image: { type: "string" },
+            confirm: { type: "boolean" },
+            yes: { type: "boolean" },
+            local: { type: "boolean" },
+            upload: { type: "boolean" },
+            remote: { type: "boolean" },
+            model: { type: "string", default: DEFAULT_MODEL },
+            "evaluator-model": { type: "string", default: DEFAULT_MODEL },
+            backend: { type: "string" },
+            "base-url": { type: "string", default: DEFAULT_BASE_URL },
+            "max-turns": { type: "string", default: "50" },
+            repeat: { type: "string", default: "1" },
+            keep: { type: "boolean" },
+            help: { type: "boolean", short: "h" },
+        },
+    });
+    if (values.help) {
+        console.log(USAGE);
+        return;
+    }
+    if ((values.task === undefined) === (positionals.length !== 1))
+        fail("give one task: a task file or -t TEXT");
+    if (values.host && values.image)
+        fail("--image applies to the Docker sandbox, not --host");
+    if (values.confirm && values.yes)
+        fail("--confirm and --yes contradict each other");
+    if (values.local && values.remote)
+        fail("--local and --remote contradict each other");
+    const confirm = values.host ? !values.yes : !!values.confirm;
+    if (confirm && !process.stdin.isTTY)
+        fail("confirming each action needs a terminal (use --yes with --host to skip it)");
+    const maxTurns = Number(values["max-turns"]);
+    if (!Number.isInteger(maxTurns) || maxTurns < 1)
+        fail("--max-turns must be a positive integer");
+    const repeat = Number(values.repeat);
+    if (!Number.isInteger(repeat) || repeat < 1)
+        fail("--repeat must be a positive integer");
+    const modelFor = modelConfig(values.backend, values["base-url"]);
+    let sandbox;
+    if (values.host) {
+        sandbox = { kind: "host" };
+        if (!confirm)
+            console.log("warning: host mode without confirmation; the agent's commands run on this machine unchecked");
+    }
+    else {
+        try {
+            checkDocker();
+            sandbox = { kind: "docker", image: values.image ?? ensureDefaultImage() };
+        }
+        catch (e) {
+            fail(e.message);
+        }
+    }
+    // What to run: a suite directory, a task file, or task text.
+    let target;
+    if (values.task !== undefined) {
+        const task = values.task.trim();
+        target = { kind: "task", task, taskName: (task.toLowerCase().match(/[a-z0-9]+/g) ?? ["task"]).slice(0, 5).join("-"),
+            taskSource: { kind: "text" } };
+    }
+    else {
+        const p = positionals[0];
+        if (/^https?:\/\//.test(p)) {
+            if (!parseHubTaskUrl(p))
+                fail(`not a hub task URL: ${p} (expected https://<hub>/tasks/<id>)`);
+            if (values.local)
+                fail("--local does not apply to hub tasks; they always clone the published repo");
+            let t;
+            try {
+                t = await fetchHubTask(p);
+            }
+            catch (e) {
+                fail(e.message);
+            }
+            console.log(`Task ${t.id} from ${t.hub}: ${t.title}\n\n${t.text.replace(/^/gm, "    ")}\n`);
+            if (!values.yes) {
+                if (!process.stdin.isTTY)
+                    fail("a hub task needs confirming in a terminal (or pass --yes)");
+                if (!(await yesNo("Run this task? [y/n] "))) {
+                    console.log("Not running it.");
+                    process.exit(0);
+                }
+            }
+            const slug = (t.title.toLowerCase().match(/[a-z0-9]+/g) ?? []).slice(0, 5).join("-");
+            target = { kind: "task", task: t.text, taskName: `hub${t.id}${slug ? `-${slug}` : ""}`,
+                taskSource: { kind: "hub", url: t.url, hub: t.hub, id: t.id, title: t.title } };
+        }
+        else if (!fs.existsSync(p))
+            fail(`no such file: ${p}`);
+        else
+            target = fs.statSync(p).isDirectory() ? { kind: "suite", dir: p }
+                : { kind: "task", task: fs.readFileSync(p, "utf8").trim(), taskName: path.basename(p).replace(/\.[^.]*$/, ""),
+                    taskSource: { kind: "file", path: p } };
+    }
+    if (target.kind === "task" && !target.task)
+        fail("the task is empty");
+    // Uploads go to the hub the task came from; credentials are settled before any run starts.
+    let upload = null;
+    if (values.upload) {
+        if (target.kind !== "task" || target.taskSource.kind !== "hub")
+            fail("--upload works only for tasks from a hub");
+        try {
+            upload = await hubCredentials(target.taskSource.hub);
+        }
+        catch (e) {
+            fail(e.message);
+        }
+    }
+    // The local copy: the repo containing the task file or suite (or the current directory for -t).
+    // Hub tasks always clone the published repo, so their results match public code.
+    const isHub = target.kind === "task" && target.taskSource.kind === "hub";
+    if (isHub && values.local)
+        fail("--local does not apply to hub tasks; they always clone the published repo");
+    let localCopy = null;
+    if (!values.remote && !isHub) {
+        const dir = target.kind === "suite" ? target.dir : target.taskSource.kind === "file" ? path.dirname(target.taskSource.path) : ".";
+        const found = findRepo(path.resolve(dir));
+        if ("reason" in found) {
+            if (values.local)
+                fail(`--local: ${found.reason}`);
+        }
+        else {
+            try {
+                localCopy = prepareLocalCopy(found.repo, found.origin);
+            }
+            catch (e) {
+                fail(e.message);
+            }
+        }
+    }
+    const common = { sandbox, confirm, localCopy, upload, maxTurns, keep: !!values.keep, version: VERSION,
+        model: modelFor(values.model), evaluatorModel: modelFor(values["evaluator-model"]) };
+    let code = 0;
+    try {
+        if (target.kind === "suite") {
+            code = (await runBatch(path.basename(path.resolve(target.dir)), suiteTasks(target.dir), repeat, common)) ? 0 : 1;
+        }
+        else if (repeat > 1) {
+            code = (await runBatch(`${target.taskName}-x${repeat}`, [target], repeat, common)) ? 0 : 1;
+        }
+        else {
+            const out = await runTask({ ...common, task: target.task, taskName: target.taskName, taskSource: target.taskSource, runsDir: "runs" });
+            code = out.summary.stop_reason === "error" ? 1 : 0;
+        }
+    }
+    catch (e) {
+        console.error(`gentl: ${e.message}`);
+        code = 2;
+    }
+    finally {
+        if (localCopy)
+            removeLocalCopy(localCopy);
+    }
+    process.exit(code);
+}
+/** Ask a yes/no question; Ctrl-C or Ctrl-D counts as no. */
+function yesNo(question) {
+    const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
+    return new Promise((resolve) => {
+        // Resolve before closing: close() fires the "close" handler synchronously.
+        rl.question(question, (a) => {
+            resolve(/^y(es)?$/i.test(a.trim()));
+            rl.close();
+        });
+        rl.on("close", () => resolve(false));
+    });
+}
+async function uploadCommand(argv) {
+    const { values, positionals } = parseArgs({ args: argv, allowPositionals: true, options: { help: { type: "boolean", short: "h" } } });
+    if (values.help || !positionals.length) {
+        console.log(USAGE);
+        process.exit(values.help ? 0 : 2);
+    }
+    const dirs = positionals.flatMap((p) => (fs.existsSync(p) ? runDirsIn(p) : fail(`no such directory: ${p}`)));
+    if (!dirs.length)
+        fail("no run directories found");
+    const creds = new Map();
+    let failed = 0;
+    for (const dir of dirs) {
+        try {
+            const hub = runHub(dir);
+            if (!hub)
+                throw new Error("its task did not come from a hub, so there is nowhere to upload it");
+            if (!creds.has(hub))
+                creds.set(hub, await hubCredentials(hub));
+            console.log(`${dir}: uploaded ${await uploadRun(dir, creds.get(hub))}`);
+        }
+        catch (e) {
+            failed++;
+            console.log(`${dir}: ${e.message}`);
+        }
+    }
+    process.exit(failed ? 1 : 0);
+}
+/** Checks the backend settings once and returns a function giving the config for a model. */
+function modelConfig(backendOpt, baseUrl) {
+    const backend = backendOpt ?? envValue("GENTL_BACKEND") ?? "openrouter";
+    if (backend === "claude-code")
+        return (model) => ({ backend, model, baseUrl: "", apiKey: "" });
+    if (backend !== "openrouter")
+        fail(`unknown backend ${JSON.stringify(backend)}; use openrouter or claude-code`);
+    const apiKey = envValue("OPENROUTER_API_KEY");
+    if (!apiKey)
+        fail("OPENROUTER_API_KEY is not set (environment or ./.env); or use --backend claude-code");
+    return (model) => ({ backend: "openrouter", model, baseUrl, apiKey });
+}
+async function evaluateCommand(argv) {
+    const { values, positionals } = parseArgs({
+        args: argv,
+        allowPositionals: true,
+        options: {
+            "evaluator-model": { type: "string", default: DEFAULT_MODEL },
+            backend: { type: "string" },
+            "base-url": { type: "string", default: DEFAULT_BASE_URL },
+            help: { type: "boolean", short: "h" },
+        },
+    });
+    if (values.help) {
+        console.log(USAGE);
+        return;
+    }
+    if (positionals.length !== 1)
+        fail("give one run directory");
+    const runDir = path.resolve(positionals[0]);
+    for (const f of ["summary.json", "events.jsonl"]) {
+        if (!fs.existsSync(path.join(runDir, f)))
+            fail(`${f} not found in ${runDir}`);
+    }
+    const ok = await evaluate(runDir, modelConfig(values.backend, values["base-url"])(values["evaluator-model"]));
+    process.exit(ok ? 0 : 1);
+}
+async function main() {
+    const [cmd, ...rest] = process.argv.slice(2);
+    if (cmd === "run")
+        await run(rest);
+    else if (cmd === "evaluate")
+        await evaluateCommand(rest);
+    else if (cmd === "upload")
+        await uploadCommand(rest);
+    else if (cmd === "--version" || cmd === "-V")
+        console.log(VERSION);
+    else {
+        console.log(USAGE);
+        process.exit(cmd === "-h" || cmd === "--help" ? 0 : 2);
+    }
+}
+main().catch((e) => fail(e instanceof Error ? e.message : String(e)));

package/dist/docker.js

@@ -0,0 +1,76 @@
+// Docker mode: a fresh container per run, from the bundled default image or one given with --image.
+// The image needs bash and setsid (util-linux); git is needed by most tasks and by the final-state record.
+import { spawn, spawnSync } from "node:child_process";
+import { createHash } from "node:crypto";
+import * as fs from "node:fs";
+import { execLocal, Sandbox } from "./sandbox.js";
+const WORKDIR = "/work";
+function docker(args, input) {
+    return execLocal("docker", args, { input });
+}
+/** The default image, built from sandbox/Dockerfile on first use; its tag follows the Dockerfile's content. */
+export function ensureDefaultImage() {
+    const dockerfile = fs.readFileSync(new URL("../sandbox/Dockerfile", import.meta.url));
+    const tag = "gentl-sandbox:" + createHash("sha256").update(dockerfile).digest("hex").slice(0, 12);
+    if (docker(["image", "inspect", tag]).ok)
+        return tag;
+    console.log(`Building the sandbox image ${tag} (first run only) ...`);
+    const p = spawnSync("docker", ["build", "-t", tag, "-"], { input: dockerfile, stdio: ["pipe", "inherit", "inherit"] });
+    if (p.status !== 0)
+        throw new Error(`docker build failed for ${tag}`);
+    return tag;
+}
+export function checkDocker() {
+    const r = docker(["version", "--format", "{{.Server.Version}}"]);
+    if (!r.ok)
+        throw new Error(`Docker is not available (${r.out.trim().split("\n")[0]}). Install or start Docker, or use --host.`);
+}
+/** Messages from the docker client and setsid (the command's own stderr is merged into stdout in the container).
+ * setsid's note about a child killed by a signal is dropped; a missing or stopped container is fatal. */
+function classifyDockerStderr(text) {
+    const lines = text.split("\n").filter((l) => l.trim() && !/^setsid: child \d+ did not exit normally/.test(l));
+    const fatal = lines.find((l) => /No such container|is not running|Cannot connect to the Docker daemon/i.test(l)) ?? null;
+    return { keep: lines.length ? lines.join("\n") + "\n" : "", fatal };
+}
+export class DockerSandbox extends Sandbox {
+    name;
+    image;
+    kind = "docker";
+    workdir = WORKDIR;
+    imageId;
+    count = 0;
+    /** `mounts` are read-only bind mounts (host directory to container directory). */
+    constructor(name, image, mounts = []) {
+        super();
+        this.name = name;
+        this.image = image;
+        const volumes = mounts.flatMap((m) => ["-v", `${m.host}:${m.container}:ro`]);
+        const r = docker(["run", "-d", "--init", "--name", name, "-w", WORKDIR, ...volumes, "--entrypoint", "sleep", image, "infinity"]);
+        if (!r.ok)
+            throw new Error(`could not start container from ${image}: ${r.out.trim()}`);
+        this.imageId = docker(["image", "inspect", "-f", "{{.Id}}", image]).out.trim();
+    }
+    describe() {
+        return `docker, image ${this.image}, container ${this.name}`;
+    }
+    start(cmd) {
+        // setsid gives the command its own process group inside the container; its id goes to a pid file so the group
+        // can be killed (killing the local docker client would leave the command running). The command's stderr is
+        // merged into stdout inside the container, so the client's stderr holds only messages from setsid and docker.
+        const pidFile = `/tmp/.gentl-cmd-${++this.count}.pid`;
+        const child = spawn("docker", ["exec", "-w", WORKDIR, this.name, "setsid", "-w", "bash", "-c",
+            `echo $$ > ${pidFile}; exec bash -c "$1" 2>&1`, "gentl", cmd], { detached: true, stdio: ["ignore", "pipe", "pipe"] });
+        const kill = () => {
+            docker(["exec", this.name, "bash", "-c", `kill -9 -- -$(cat ${pidFile}) 2>/dev/null`]);
+            setTimeout(() => child.kill("SIGKILL"), 2000).unref();
+        };
+        return { child, kill, sandboxStderr: classifyDockerStderr };
+    }
+    exec(args, cwd, input) {
+        const w = cwd ? (cwd.startsWith("/") ? cwd : `${WORKDIR}/${cwd}`) : WORKDIR;
+        return docker(["exec", ...(input !== undefined ? ["-i"] : []), "-w", w, this.name, ...args], input);
+    }
+    remove() {
+        docker(["rm", "-f", this.name]);
+    }
+}

package/dist/evaluate.js

@@ -0,0 +1,126 @@
+// The evaluator: one model call that reads a run directory and writes a verdict and a friction report.
+// It depends only on the run directory, so a run can be re-evaluated later (`gentl evaluate`).
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { clip, fence } from "./format.js";
+import { createModelClient } from "./model.js";
+import { parseFirstJson } from "./protocol.js";
+const RESULT_HIDDEN_CAP = 6000; // chars of the part of a result the agent did not see
+const RESULT_CAP = 12000; // chars of any single result
+const LOG_CAP = 500000; // chars of the whole log; beyond this, results are clipped harder
+export const EVALUATOR_PROMPT = `You evaluate one run of gentl, a readability test for codebases. An agent was given a task as a newcomer to some software (typically: clone a repo, set it up from its docs, make a change, verify it). The agent had no memory between turns except short notes it kept, and it saw only its last action's result each turn, often truncated. Its success is self-reported, so you judge it.
+
+You receive the task, the log of every turn (notes, action, result; parts of results the agent did not see are marked), the questions the agent asked (no one answered them), the agent's finish report, and the final state of the workspace (git status and diff of each repo, and other files).
+
+Write two things.
+
+1. A verdict: did the agent actually accomplish the task? Judge from the evidence in the log and the final state, not from the agent's claims. Every part of the task must be done, and verified where the task implies it (for example, tests actually ran and passed). Give short reasons that cite turn numbers.
+
+2. A friction report for the maintainers of the software, in markdown, with these sections:
+- Docs used: which docs, files or pages the agent relied on, and for what (cite turns).
+- Missing or wrong: what the agent needed but could not find in the docs, or found wrong or outdated. Be specific: which file or section, and what it should say.
+- Guesses: where the agent went ahead without support from the docs (inferred from config, CI files or code, or trial and error).
+- Prior knowledge: commands, APIs, flags or facts the agent used that never appeared in any result it saw before that point, so they came from its training and the docs were not tested there.
+- Questions: each question the agent asked, and whether the docs answer it.
+Keep it short and concrete, as bullets; write "none" for an empty section. Separate problems in the software's docs from problems caused by the agent's own mistakes or by the test environment (for example a missing system package), and say which is which.
+
+Respond with one JSON object: {"success": true|false, "reasons": "...", "friction_report": "..."}`;
+const EVAL_SCHEMA = {
+    name: "evaluation",
+    schema: {
+        type: "object",
+        properties: { success: { type: "boolean" }, reasons: { type: "string" }, friction_report: { type: "string" } },
+        required: ["success", "reasons", "friction_report"],
+        additionalProperties: false,
+    },
+};
+/** A result as the evaluator sees it: the full result, with the part the agent did not see marked. */
+function resultForEvaluator(ev, cap) {
+    const full = ev.result_full ?? "";
+    const notices = (ev.notices ?? []).map((n) => `${n}\n`).join("");
+    const t = ev.truncation;
+    if (!t)
+        return notices + clip(full, cap);
+    const hidden = full.slice(t.head, full.length - t.tail);
+    return notices + full.slice(0, t.head) +
+        `\n[--- the agent did not see the next ${hidden.length} chars ---]\n${clip(hidden, Math.min(RESULT_HIDDEN_CAP, cap))}\n[--- end of the part the agent did not see ---]\n` +
+        full.slice(full.length - t.tail);
+}
+function formatActionForEvaluator(a) {
+    if (!a || typeof a !== "object")
+        return "(none)";
+    switch (a.type) {
+        case "run":
+            return `run:\n${fence(String(a.cmd), "bash")}`;
+        case "write_file":
+            return `write_file ${a.path}:\n${fence(clip(String(a.content), RESULT_CAP))}`;
+        case "edit_file":
+            return `edit_file ${a.path}\nold:\n${fence(String(a.old))}new:\n${fence(String(a.new))}`;
+        case "ask":
+            return `ask: ${a.question}`;
+        default:
+            return fence(JSON.stringify(a, null, 2), "json");
+    }
+}
+export function buildEvaluatorInput(runDir) {
+    const summary = JSON.parse(fs.readFileSync(path.join(runDir, "summary.json"), "utf8"));
+    const events = fs.readFileSync(path.join(runDir, "events.jsonl"), "utf8").split("\n").filter(Boolean).map((l) => JSON.parse(l));
+    const finalPath = path.join(runDir, "final-state.md");
+    const finalState = fs.existsSync(finalPath) ? fs.readFileSync(finalPath, "utf8") : "(not recorded)";
+    const turns = events.filter((e) => e.type === "turn");
+    const build = (cap) => {
+        let log = "";
+        for (const ev of turns) {
+            log += `## Turn ${ev.turn}\n\n`;
+            if (ev.parse_error) {
+                log += `The response could not be parsed (${ev.parse_error}).\n\n`;
+                continue;
+            }
+            log += `Notes after this turn:\n${fence(ev.notes || "(empty)")}\nAction: ${formatActionForEvaluator(ev.parsed?.action)}\n`;
+            if (ev.declined !== undefined)
+                log += `\nThe person running gentl declined this action${ev.declined ? `: ${ev.declined}` : ""}.\n`;
+            if (ev.result_full !== undefined)
+                log += `\nResult:\n${fence(resultForEvaluator(ev, cap))}`;
+            log += "\n";
+        }
+        return log;
+    };
+    let log = build(RESULT_CAP);
+    if (log.length > LOG_CAP)
+        log = build(2000);
+    const asks = (summary.asks ?? []);
+    const finish = summary.finished
+        ? `success: ${summary.self_reported_success}\nsummary: ${summary.self_report_summary}\nfriction: ${summary.self_report_friction ?? "(none given)"}`
+        : `The agent did not finish (stop reason: ${summary.stop_reason}).`;
+    const lc = summary.local_copy
+        ? ` Clones of ${summary.local_copy.origin} were served from the maintainer's local working copy (with any uncommitted changes), so the agent's clone may differ from the published repo and its origin URL is a local file:// path; this is intended, not a problem with the run.`
+        : "";
+    return `# Task\n\n${summary.task}\n\n# Run\n\nModel ${summary.model}, sandbox ${summary.sandbox}, ${summary.turns} turns, stop reason ${summary.stop_reason}.${lc}\n\n` +
+        `# Log\n\n${log}\n# Questions the agent asked\n\n${asks.length ? asks.map((a) => `- turn ${a.turn}: ${a.question}`).join("\n") : "none"}\n\n` +
+        `# The agent's finish report\n\n${finish}\n\n# Final state of the workspace\n\n${finalState}\n`;
+}
+/** Evaluate a run directory and write evaluation.json and friction.md; also records the verdict in summary.json. */
+export async function evaluateRun(runDir, cfg, signal, log) {
+    const input = buildEvaluatorInput(runDir);
+    const client = createModelClient(cfg, EVAL_SCHEMA, log);
+    const reply = await client.call(EVALUATOR_PROMPT, input, signal);
+    let parsed;
+    try {
+        parsed = parseFirstJson(reply.text, (v) => typeof v.success === "boolean");
+    }
+    catch {
+        throw new Error(`could not parse the evaluator's response: ${reply.text.slice(0, 300)}`);
+    }
+    const ev = {
+        model: cfg.model, success: parsed.success, reasons: String(parsed.reasons ?? ""),
+        friction_report: String(parsed.friction_report ?? ""), structured_output: client.structured,
+        usage: reply.usage, cost_usd: Number(reply.usage.cost ?? 0), input_chars: input.length, created: new Date().toISOString(),
+    };
+    fs.writeFileSync(path.join(runDir, "evaluation.json"), JSON.stringify(ev, null, 2) + "\n");
+    fs.writeFileSync(path.join(runDir, "friction.md"), `# Evaluation\n\nVerdict: ${ev.success ? "success" : "failure"} (evaluator ${ev.model})\n\n${ev.reasons}\n\n# Friction report\n\n${ev.friction_report}\n`);
+    const summaryPath = path.join(runDir, "summary.json");
+    const summary = JSON.parse(fs.readFileSync(summaryPath, "utf8"));
+    summary.evaluation = { model: ev.model, success: ev.success, reasons: ev.reasons, cost_usd: ev.cost_usd };
+    fs.writeFileSync(summaryPath, JSON.stringify(summary, null, 2) + "\n");
+    return ev;
+}

package/dist/finalstate.js

@@ -0,0 +1,64 @@
+// The final state of the workspace, for the evaluator: git status and diff of each repo, plus loose files.
+// Everything runs inside the sandbox through `exec`, so it works the same for host and Docker.
+import { clip, fence } from "./format.js";
+const SKIP_DIRS = ["node_modules", ".venv", "venv", "env", "site-packages", "__pycache__", "build", "dist", "target"];
+const MAX_DEPTH = 4; // how deep to look for .git below the working directory
+const DIFF_CAP = 40000; // chars per repo diff
+const FILE_CAP = 10000; // bytes per untracked or loose file
+const FILES_CAP = 40000; // chars of file contents per section
+const MAX_FILES = 40; // files listed per section
+const PRUNE = SKIP_DIRS.map((d) => `-name ${d}`).join(" -o ");
+function lines(s) {
+    return s.split("\n").map((l) => l.trim()).filter(Boolean);
+}
+function filesSection(sb, cwd, rels) {
+    let out = "";
+    let budget = FILES_CAP;
+    for (const rel of rels.slice(0, MAX_FILES)) {
+        const r = sb.exec(["bash", "-c", `[ -f "$1" ] && [ "$(wc -c < "$1")" -le ${FILE_CAP} ] && cat "$1"`, "_", rel], cwd);
+        if (!r.ok || r.out.includes("\0") || r.out.length > budget) {
+            out += `\n${rel}: (not shown: binary, large, or over the size budget)\n`;
+            continue;
+        }
+        budget -= r.out.length;
+        out += `\n${rel}:\n${fence(r.out)}`;
+    }
+    if (rels.length > MAX_FILES)
+        out += `\n(${rels.length - MAX_FILES} more files not listed)\n`;
+    return out;
+}
+export function describeFinalState(sb) {
+    const sh = (script, cwd) => sb.exec(["bash", "-c", script], cwd);
+    const git = (cwd, ...args) => sb.exec(["git", "-c", "core.quotepath=off", ...args], cwd);
+    const top = sh("ls -1Ap");
+    const repoDirs = lines(sh(`find . -maxdepth ${MAX_DEPTH} \\( ${PRUNE} \\) -prune -o -name .git -prune -print`).out)
+        .map((p) => p.replace(/\/\.git$/, "").replace(/^\.\//, "") || ".")
+        .sort();
+    const repos = [];
+    let text = `Top level of the working directory: ${lines(top.out).join(", ") || "(empty)"}\n\n`;
+    for (const rel of repoDirs) {
+        const upstream = git(rel, "rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{u}");
+        const base = upstream.ok ? upstream.out.trim() : "HEAD";
+        const status = git(rel, "status", "--short", "--branch").out;
+        const commits = base !== "HEAD" ? git(rel, "log", "--oneline", `${base}..HEAD`).out : "";
+        const diff = git(rel, "diff", base).out;
+        const untracked = lines(git(rel, "ls-files", "--others", "--exclude-standard").out);
+        repos.push({ path: rel, changed: !!(commits.trim() || diff.trim() || untracked.length) });
+        text += `## Repo ${rel}\n\ngit status:\n${fence(status || "(clean)")}`;
+        if (commits.trim())
+            text += `\nCommits made since ${base}:\n${fence(commits)}`;
+        text += `\ngit diff ${base} (committed and uncommitted changes to tracked files):\n${fence(clip(diff, DIFF_CAP) || "(none)", "diff")}`;
+        if (untracked.length)
+            text += `\nUntracked files:\n${filesSection(sb, rel, untracked)}`;
+        text += "\n";
+    }
+    // Files in the working directory outside any repo (e.g. an example script the agent wrote), two levels deep.
+    const loose = lines(sh(`find . -maxdepth 2 \\( -name '.?*' -o ${PRUNE} \\) -prune -o -type f -print`).out)
+        .map((p) => p.replace(/^\.\//, ""))
+        .filter((p) => !repoDirs.some((r) => r === "." || p.startsWith(r + "/")));
+    if (loose.length)
+        text += `## Files outside any repo\n${filesSection(sb, ".", loose)}`;
+    if (!repoDirs.length)
+        text += "No git repos found in the working directory.\n";
+    return { text, repos };
+}

package/dist/format.js

@@ -0,0 +1,15 @@
+/** Code block whose fence is longer than any backtick run in the text. */
+export function fence(text, lang = "") {
+    const runs = text.match(/`{3,}/g) ?? [];
+    const n = Math.max(3, ...runs.map((r) => r.length + 1));
+    const f = "`".repeat(n);
+    return `${f}${lang}\n${text.replace(/\n+$/, "")}\n${f}\n`;
+}
+/** Keep the start and end of a long text, noting how much was cut. */
+export function clip(text, maxChars) {
+    if (text.length <= maxChars)
+        return text;
+    const head = text.slice(0, Math.floor(maxChars / 2));
+    const tail = text.slice(text.length - Math.floor(maxChars / 2));
+    return `${head}\n[... ${text.length - head.length - tail.length} chars cut ...]\n${tail}`;
+}

package/dist/host.js

@@ -0,0 +1,64 @@
+// Host mode: a fresh temp directory on this machine, with work/ (where the agent's commands run) and home/ (HOME).
+import { spawn, spawnSync } from "node:child_process";
+import * as fs from "node:fs";
+import * as os from "node:os";
+import * as path from "node:path";
+import { execLocal, Sandbox } from "./sandbox.js";
+// Environment variables passed through from the user's environment; everything else (API keys, tokens, XDG dirs) is dropped.
+const PASSTHROUGH_ENV = ["PATH", "LANG", "LANGUAGE", "LC_ALL", "LC_CTYPE", "TZ", "USER", "LOGNAME", "SHELL",
+    "http_proxy", "https_proxy", "no_proxy", "HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY"];
+function killGroup(pid) {
+    if (!pid)
+        return;
+    try {
+        process.kill(-pid, "SIGKILL");
+    }
+    catch {
+        // already gone
+    }
+}
+export class HostSandbox extends Sandbox {
+    kind = "host";
+    root;
+    workdir;
+    env = {};
+    groups = new Set(); // process groups of all commands, killed on remove()
+    constructor() {
+        super();
+        this.root = fs.mkdtempSync(path.join(os.tmpdir(), "gentl-"));
+        this.workdir = path.join(this.root, "work");
+        const home = path.join(this.root, "home");
+        fs.mkdirSync(this.workdir);
+        fs.mkdirSync(home);
+        for (const k of PASSTHROUGH_ENV)
+            if (process.env[k] !== undefined)
+                this.env[k] = process.env[k];
+        Object.assign(this.env, { HOME: home, TERM: "dumb", NO_COLOR: "1", PAGER: "cat", GIT_PAGER: "cat",
+            GIT_TERMINAL_PROMPT: "0" });
+    }
+    describe() {
+        return `host, workspace ${this.root}`;
+    }
+    start(cmd) {
+        const child = spawn("bash", ["-c", cmd], { cwd: this.workdir, env: this.env, detached: true,
+            stdio: ["ignore", "pipe", "pipe"] });
+        if (child.pid)
+            this.groups.add(child.pid);
+        return { child, kill: () => killGroup(child.pid) };
+    }
+    exec(args, cwd, input) {
+        return execLocal(args[0], args.slice(1), { cwd: path.resolve(this.workdir, cwd ?? "."), env: this.env, input });
+    }
+    remove() {
+        for (const pid of this.groups)
+            killGroup(pid);
+        try {
+            fs.rmSync(this.root, { recursive: true, force: true });
+        }
+        catch {
+            // Some caches (e.g. Go modules) are read-only.
+            spawnSync("chmod", ["-R", "u+w", this.root]);
+            fs.rmSync(this.root, { recursive: true, force: true });
+        }
+    }
+}