@turri-dev/cli 0.0.0-stage → 0.1.1

package/dist/regenerate/jobs.js

@@ -0,0 +1,427 @@
+import { randomUUID } from "node:crypto";
+import { availableParallelism } from "node:os";
+import { withPulse } from "./shell.js";
+import { describeEnvironmentError, EnvironmentError, missingCredentials } from "./environment-error.js";
+import { addAgentReport, describeAgents, JobStoppedError, withActivity } from "./activity.js";
+import { startJobLog } from "./job-log.js";
+import { chosenStack, withStack } from "../config.js";
+const DEFAULT_TAIL_CHARS = 4000;
+const ACTIVITY_LINES = 10;
+/** A long render's full transcript is kept, up to about this much text. */
+const MAX_TRANSCRIPT_CHARS = 600_000;
+/** What a stopped job says when the person stopped it. */
+export const STOPPED_ON_REQUEST = "Stopped on request.";
+export function createJobRegistry(options) {
+    const jobs = new Map();
+    // The slots: how many jobs run, and the ones whose lanes are free waiting for one, in the order they got there.
+    let limit = options?.maxRunning !== undefined && options.maxRunning >= 1 ? Math.floor(options.maxRunning) : undefined;
+    let active = 0;
+    const waiters = [];
+    const takeSlot = () => new Promise((resolve) => {
+        if (limit === undefined || active < limit) {
+            active += 1;
+            resolve();
+        }
+        else
+            waiters.push(() => ((active += 1), resolve()));
+    });
+    const giveSlot = () => {
+        active -= 1;
+        waiters.shift()?.();
+    };
+    const settled = new Map();
+    const tailChars = options?.tailChars ?? DEFAULT_TAIL_CHARS;
+    /** The last job to claim each lane. A new job waits for it and becomes the new tail. */
+    const lanes = new Map();
+    const controllers = new Map();
+    /** Why turri stopped a job, when it did: a job stopped without a reason was stopped by the person. */
+    const stopReasons = new Map();
+    /**
+     * Jobs whose work is over. Not the same as a finished status: a queued job that was
+     * stopped says "stopped" at once, but still sits in its lanes until the job ahead of it
+     * settles - and whatever queues behind it has to wait that long too.
+     */
+    const done = new Set();
+    let sequence = 0;
+    return {
+        setMaxRunning(count) {
+            limit = count !== undefined && count >= 1 ? Math.floor(count) : undefined;
+            // Raised: the jobs waiting for a slot that now has room start.
+            while (waiters.length > 0 && (limit === undefined || active < limit))
+                waiters.shift()();
+        },
+        start(input) {
+            const id = randomUUID().slice(0, 8);
+            const claimed = [...new Set(input.lanes ?? [])];
+            const blockers = [
+                ...new Set(claimed
+                    .map((lane) => lanes.get(lane))
+                    .filter((other) => other !== undefined && jobs.has(other) && !done.has(other))),
+            ];
+            for (const lane of claimed)
+                lanes.set(lane, id);
+            const now = Date.now();
+            const job = {
+                id,
+                kind: input.kind,
+                summary: input.summary,
+                ...(input.module !== undefined ? { module: input.module } : {}),
+                ...(input.pulse !== undefined ? { pulse: input.pulse } : {}),
+                ...(input.title !== undefined ? { title: input.title } : {}),
+                diff: input.diff,
+                model: input.model,
+                ...(input.follows !== undefined ? { follows: input.follows } : {}),
+                status: blockers.length > 0 || (limit !== undefined && active >= limit) ? "queued" : "running",
+                sequence: (sequence += 1),
+                queuedAt: now,
+                startedAt: now,
+                waitingOn: blockers,
+                tail: "",
+                activity: [],
+                usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, cost: 0 },
+                agents: [],
+                transcript: [],
+            };
+            jobs.set(job.id, job);
+            let transcriptChars = 0;
+            const changed = () => {
+                // Like onFinish: a display that throws must not take the job down.
+                try {
+                    options?.onChange?.(job);
+                }
+                catch {
+                    /* ignore */
+                }
+            };
+            changed();
+            const controller = new AbortController();
+            controllers.set(id, controller);
+            /** The job's transcript on disk, from when it starts running - see job-log.ts. */
+            let log;
+            const sink = {
+                signal: controller.signal,
+                job: { id, kind: input.kind, summary: input.summary },
+                specConflict(conflict) {
+                    job.specConflict = conflict;
+                    this.transcript({ kind: "stage", text: `the spec contradicts itself: functionalities ${conflict.specs.map((spec) => `${spec.module} ${spec.frid}`).join(" and ")} cannot both hold` });
+                    changed();
+                },
+                stage(label) {
+                    job.stage = label;
+                    this.transcript({ kind: "stage", text: label });
+                    changed();
+                },
+                spec(spec) {
+                    // The same functionality going on to its next stage keeps its clock; another starts a new one.
+                    const since = spec !== undefined && job.building?.label === spec.label ? job.building.since : Date.now();
+                    job.building = spec === undefined ? undefined : { ...spec, since };
+                    if (spec !== undefined)
+                        this.transcript({ kind: "spec", text: `${spec.label}\n${spec.text}` });
+                    changed();
+                },
+                transcript(entry) {
+                    log?.entry(entry);
+                    // The screen keeps the clipped text; the log has the whole.
+                    const { full: _full, ...shown } = entry;
+                    entry = shown;
+                    const last = job.transcript.at(-1);
+                    // Deltas of a running reply join the entry they continue - the same agent's.
+                    if ((entry.kind === "text" || entry.kind === "thinking") && last?.kind === entry.kind && last.agent?.label === entry.agent?.label) {
+                        last.text += entry.text;
+                    }
+                    else {
+                        job.transcript.push({ ...entry });
+                    }
+                    transcriptChars += entry.text.length;
+                    while (transcriptChars > MAX_TRANSCRIPT_CHARS && job.transcript.length > 1) {
+                        transcriptChars -= job.transcript.shift().text.length;
+                    }
+                },
+                activity(line) {
+                    job.activity = [...job.activity, line].slice(-ACTIVITY_LINES);
+                    changed();
+                },
+                note(line) {
+                    log?.note(line);
+                },
+                usage(spent) {
+                    job.usage = {
+                        input: job.usage.input + spent.input,
+                        output: job.usage.output + spent.output,
+                        cacheRead: job.usage.cacheRead + spent.cacheRead,
+                        cacheWrite: job.usage.cacheWrite + spent.cacheWrite,
+                        cost: job.usage.cost + spent.cost,
+                    };
+                    changed();
+                },
+                agent(report) {
+                    addAgentReport(job.agents, report);
+                },
+                now(label, opts) {
+                    job.now =
+                        label === undefined
+                            ? undefined
+                            : { label, since: opts?.keepSince && job.now ? job.now.since : Date.now() };
+                    changed();
+                },
+            };
+            // Whether the job ahead succeeded or failed, this one gets its turn: each reports on its own -
+            // unless the registry retries, and one ahead of it failed for good: then nothing behind it runs.
+            const turn = Promise.allSettled(blockers.map((other) => settled.get(other)));
+            const guarded = options?.retries !== undefined;
+            const retries = options?.retries ?? 0;
+            /** A job's work ran, and did not do what it was asked: it threw, or said it did not pass. */
+            const didNotPass = (outcome) => {
+                const details = outcome.details;
+                return details?.passed === false && details.skipped !== true;
+            };
+            // A pulse: what its agents leave running is killed when it ends, and the modules it works in are its own.
+            const runOnce = () => {
+                // A credential agent-env names that is not there stops the pulse before any agent starts.
+                const own = input.lanes?.[0];
+                const missing = own !== undefined ? missingCredentials(own) : undefined;
+                if (missing !== undefined)
+                    return Promise.reject(missing);
+                return withPulse(() => withActivity(sink, () => input.run((delta) => {
+                    job.tail = (job.tail + delta).slice(-tailChars);
+                })), { modules: input.lanes ?? [] });
+            };
+            const runGuarded = async () => {
+                const failures = [];
+                for (let attempt = 0;; attempt += 1) {
+                    let outcome;
+                    let error;
+                    try {
+                        outcome = await runOnce();
+                    }
+                    catch (thrown) {
+                        error = thrown;
+                    }
+                    // Asked to stop is not a failure: no retry, and nothing behind it fails for it.
+                    if (controller.signal.aborted) {
+                        if (error !== undefined)
+                            throw error;
+                        return outcome;
+                    }
+                    if (error === undefined && !didNotPass(outcome))
+                        return outcome;
+                    // The environment lacks something: no retry would find it - the pulse stops, the module's later pulses wait,
+                    // and the person is told. What the harness found itself is worked out at once; what an agent said goes as it is.
+                    if (error instanceof EnvironmentError) {
+                        job.failedForGood = true;
+                        if (error.source === "harness" && options?.diagnose !== undefined) {
+                            sink.stage("the environment lacks something - working out what to do about it");
+                            job.diagnosis = await withActivity(sink, () => options.diagnose(job, [error.message, describeEnvironmentError(error, input.rerun)], log?.file)).catch(() => undefined);
+                        }
+                        job.environment = describeEnvironmentError(error, input.rerun);
+                        throw new Error([job.environment, ...(job.diagnosis !== undefined ? ["", job.diagnosis] : [])].join("\n"));
+                    }
+                    failures.push(error !== undefined ? (error instanceof Error ? error.message : String(error)) : outcome.report);
+                    if (attempt < retries) {
+                        sink.stage(`the pulse failed - trying it once more (${attempt + 1} of ${retries})`);
+                        continue;
+                    }
+                    job.failedForGood = true;
+                    if (options?.diagnose !== undefined) {
+                        sink.stage("the pulse failed again - working out why, and what has to be done");
+                        job.diagnosis = await withActivity(sink, () => options.diagnose(job, failures, log?.file)).catch(() => undefined);
+                    }
+                    throw new Error([
+                        `Failed ${failures.length} time${failures.length === 1 ? "" : "s"}, so it and every pulse queued behind it have stopped.`,
+                        ...(job.diagnosis !== undefined ? ["", job.diagnosis] : []),
+                        "",
+                        "The last attempt ended with:",
+                        failures.at(-1).trim().slice(-4000),
+                    ].join("\n"));
+                }
+            };
+            /** It holds one of the run slots, to give back when it ends. */
+            let holding = false;
+            const promise = turn
+                .then(async () => {
+                // Stopped while it waited: it never starts.
+                if (controller.signal.aborted)
+                    throw new JobStoppedError();
+                // A job ahead of it failed for good: it does not start, and fails for good too - so does what waits on it.
+                const failedAhead = guarded ? blockers.map((other) => jobs.get(other)).find((other) => other?.failedForGood === true) : undefined;
+                if (failedAhead !== undefined) {
+                    job.failedForGood = true;
+                    if (failedAhead.environment !== undefined) {
+                        throw new Error(`Not run: pulse ${failedAhead.id} ahead of it in the queue (${pulseName(failedAhead)}) stopped, and this waits for it.\n\n${failedAhead.environment}`);
+                    }
+                    throw new Error(`Not run: pulse ${failedAhead.id} ahead of it in the queue (${pulseName(failedAhead)}) failed twice, so nothing queued behind it runs.` +
+                        (failedAhead.diagnosis !== undefined ? `\n\n${failedAhead.diagnosis}` : ""));
+                }
+                // Its lanes are free; it still needs one of the run slots.
+                if (limit !== undefined && active >= limit) {
+                    job.status = "queued";
+                    job.waitingOn = [];
+                    job.stage = `waiting for a free slot - ${active} of ${limit} pulses running at once (parallel-jobs)`;
+                    changed();
+                }
+                await takeSlot();
+                holding = true;
+                if (controller.signal.aborted)
+                    throw new JobStoppedError();
+                if (job.status === "queued") {
+                    job.status = "running";
+                    job.startedAt = Date.now();
+                    job.waitingOn = [];
+                    job.stage = undefined;
+                    changed();
+                }
+                log = startJobLog(job, claimed);
+                if (log !== undefined)
+                    job.logFile = log.file;
+                // What the job's agents start runs until it ends, and no longer (withPulse) - with the stack chosen as it
+                // starts, whatever is chosen while it runs.
+                return withStack(chosenStack(), () => (guarded ? runGuarded() : runOnce()));
+            })
+                .then((outcome) => {
+                // Asked to stop, it is stopped - whether its work threw as it unwound or returned what it had.
+                job.status = controller.signal.aborted ? "stopped" : "succeeded";
+                if (controller.signal.aborted)
+                    job.error = stopReasons.get(id) ?? STOPPED_ON_REQUEST;
+                job.report = outcome.report;
+                job.announce = outcome.announce;
+                job.details = outcome.details;
+                return job;
+            }, (error) => {
+                // Asked to stop is not a failure, whatever the work threw as it unwound.
+                job.status = controller.signal.aborted ? "stopped" : "failed";
+                job.error = controller.signal.aborted ? (stopReasons.get(id) ?? STOPPED_ON_REQUEST) : error instanceof Error ? error.message : String(error);
+                return job;
+            })
+                .then((finished) => {
+                if (holding) {
+                    holding = false;
+                    giveSlot();
+                }
+                finished.finishedAt = Date.now();
+                log?.end(finished.status === "succeeded" ? "finished" : finished.status === "stopped" ? "stopped" : "finished - failed", finished.announce ?? finished.report ?? finished.error, describeAgents(finished.agents));
+                // A failing listener must not take the job down with it; the result is
+                // already recorded and can still be collected by asking.
+                try {
+                    options?.onFinish?.(finished);
+                }
+                catch {
+                    /* ignore */
+                }
+                return finished;
+            });
+            settled.set(job.id, promise);
+            promise.then(() => {
+                controllers.delete(id);
+                done.add(id);
+                // Release the lanes this job still holds, so a later job does not wait on a ghost.
+                for (const lane of claimed)
+                    if (lanes.get(lane) === id)
+                        lanes.delete(lane);
+            });
+            return job;
+        },
+        get(id) {
+            return jobs.get(id);
+        },
+        stop(id, reason) {
+            const job = jobs.get(id);
+            const controller = controllers.get(id);
+            if (job === undefined || controller === undefined || done.has(id) || job.status === "stopped")
+                return undefined;
+            // Who stopped it, and why: turri, saying so - or, without a reason, the person.
+            if (reason !== undefined)
+                stopReasons.set(id, reason);
+            controller.abort();
+            // A queued job is done with at once, so the queue behind it shows it gone; a running
+            // one stays running until its agent has actually stopped.
+            if (job.status === "queued") {
+                job.status = "stopped";
+                job.waitingOn = [];
+                job.error = stopReasons.get(id) ?? STOPPED_ON_REQUEST;
+                options?.onChange?.(job);
+            }
+            return job;
+        },
+        list() {
+            return [...jobs.values()].sort((a, b) => b.sequence - a.sequence);
+        },
+        notify() {
+            try {
+                options?.onNotify?.();
+            }
+            catch {
+                /* a display that throws must not take the work down */
+            }
+        },
+        async wait(id) {
+            const promise = settled.get(id);
+            if (promise === undefined)
+                throw new Error(`No such pulse: ${id}`);
+            return promise;
+        },
+    };
+}
+export function formatDuration(job) {
+    return formatElapsed((job.finishedAt ?? Date.now()) - job.startedAt);
+}
+/** "42s", "3m 12s", "1h 5m": how long something has run. */
+export function formatElapsed(ms) {
+    const seconds = Math.max(0, Math.round(ms / 1000));
+    if (seconds < 60)
+        return `${seconds}s`;
+    if (seconds < 3600)
+        return `${Math.floor(seconds / 60)}m ${seconds % 60}s`;
+    return `${Math.floor(seconds / 3600)}h ${Math.floor((seconds % 3600) / 60)}m`;
+}
+/**
+ * What a pulse is called wherever it is shown in a line: its module, kind and number - `shorty · regenerate pulse 3`,
+ * `shorty · conformance pulse 6` - never what it regenerates, which is the agents' words and long. A job of no module,
+ * what was asked for.
+ */
+export function pulseName(job) {
+    if (job.module === undefined)
+        return job.summary;
+    return job.pulse !== undefined ? `${job.module} · ${job.kind} pulse ${job.pulse}` : `${job.module} ${job.kind}`;
+}
+/** What the agent is told when it checks on a job. */
+export function describeJob(job) {
+    if (job.status === "queued") {
+        return `${pulseName(job)}: queued behind ${job.waitingOn.map((id) => `pulse ${id}`).join(" and ")}. It starts when that finishes.`;
+    }
+    const header = `${pulseName(job)}: ${job.status}, ${formatDuration(job)}`;
+    if (job.status === "running") {
+        return [
+            header,
+            ...(job.building ? ["", `building ${job.building.label}:`, job.building.text] : []),
+            ...(job.stage ? ["", `stage: ${job.stage}`] : []),
+            ...(job.activity.length > 0 ? ["", "<recent activity>", ...job.activity, "</recent activity>"] : []),
+            "",
+            "<latest output>",
+            job.tail.trimEnd() || "(nothing yet)",
+            "</latest output>",
+        ].join("\n");
+    }
+    if (job.status === "failed")
+        return `${header}\n\n${job.error ?? "no error recorded"}`;
+    if (job.status === "stopped")
+        return `${header}\n\n${job.error ?? STOPPED_ON_REQUEST}`;
+    const agents = describeAgents(job.agents ?? []);
+    return `${header}\n\n${job.report ?? "(no report)"}${agents.length > 0 ? `\n\n<its agents>\n${agents.join("\n")}\n</its agents>` : ""}`;
+}
+/**
+ * Whether a tool should block rather than hand back a job id.
+ *
+ * Background only makes sense where the session outlives the turn. In print, json and rpc
+ * modes the process ends as soon as the answer is written, taking any detached work with
+ * it, so those wait unless told otherwise.
+ */
+export function shouldWait(mode, wait) {
+    return wait ?? mode !== "tui";
+}
+/**
+ * How many jobs run at once when nothing says: half this machine's cores, at least one and at most four - a pulse's
+ * builds and tests run here, beside the person's own work.
+ */
+export function defaultParallelJobs(cores = availableParallelism()) {
+    return Math.min(4, Math.max(1, Math.floor(cores / 2)));
+}

package/dist/regenerate/lanes.js

@@ -0,0 +1,32 @@
+import { basename, dirname, resolve } from "node:path";
+import { buildDirUnder, modulesRoot, projectDirOf } from "./paths.js";
+import { resolveRequiresChain } from "./requires.js";
+/**
+ * What a job on this spec may write, as lanes for the job registry.
+ *
+ * A module's directory holds its code and its conformance tests, and building a module can
+ * build everything it requires first - so the lanes are every module in the requires chain.
+ * That is more than a single functionality touches, and deliberately so: two jobs that
+ * might write the same directory wait for each other, and a wait costs minutes where a
+ * collision costs the build.
+ *
+ * The module's own folder is a lane whatever the spec says at the moment it is read. A spec with
+ * no functional specs yet - the scaffold a new project starts with - or one that does not parse
+ * names no chain, and a job queued then must still wait for, and be waited for by, every other
+ * job on that module: two that claimed different lanes for the same folder ran side by side and
+ * wrote over each other.
+ */
+export function moduleLanes(plainFile, buildRoot) {
+    const extra = buildRoot !== undefined ? [resolve(buildRoot)] : [];
+    // The folder the module builds in, by its name - the file's - as every chain names it too.
+    const own = dirname(buildDirUnder(resolve(buildRoot ?? modulesRoot(projectDirOf(plainFile))), basename(plainFile, ".plain")));
+    try {
+        const chain = resolveRequiresChain(plainFile, [], buildRoot, { allowNoFunctionalities: true });
+        return [...new Set([own, ...chain.map((entry) => dirname(entry.buildDir)), ...extra])];
+    }
+    catch {
+        // A spec that does not parse fails inside the job with a proper error. Until then it
+        // still must not run alongside another job on the same module, or on the same file.
+        return [...new Set([own, resolve(plainFile), ...extra])];
+    }
+}