@asrunjs/asrun 0.8.0 → 0.10.0

package/dist/scripts/markers-confirm.js

@@ -0,0 +1,139 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { CASES_DIR, REPORTS_DIR, ROOT, TESTS_DIR } from '../lib/paths.js';
+import { markerSentence, parseCaseDocument, removeMarkers } from '../lib/case-doc.js';
+/*
+ * `pnpm markers:confirm --case <id> --env <env> [--report <run-id|file>] [--dry-run]`
+ *
+ * Settles the `🔍 to be confirmed` lines of one case from the last run instead of from a browser: a marker goes when
+ * every test of the case passed in the newest report AND the spec carries the line as
+ * `// unconfirmed: <the sentence>`. It answers "does the page still say this?", never "is this the right
+ * expectation": that stays a person's decision, which the case panel shows as "why it needs you".
+ *
+ * A report is evidence for the wording it asserted, not for a line no assertion reads, so a marker without an
+ * `// unconfirmed:` line in the spec is left alone and named in the output.
+ */
+const RUN_ID = /^\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}(-[a-z0-9]+(-[a-z0-9]+)*)?$/;
+const UNCONFIRMED = /\/\/[ \t]*unconfirmed:[ \t]*(.+)$/gm;
+/** Shorter matches are noise: a two-word sentence would match half a spec. */
+const MIN_MATCH = 16;
+const args = process.argv.slice(2);
+const option = (name) => {
+    const index = args.indexOf(name);
+    return index >= 0 ? args[index + 1] : undefined;
+};
+const caseId = option('--case');
+const env = option('--env');
+const reportArg = option('--report');
+const dryRun = args.includes('--dry-run');
+if (args.includes('--help') || !caseId || !env) {
+    console.log('Usage: pnpm markers:confirm --case <case-id> --env <env> [--report <run-id|file>] [--dry-run]');
+    console.log('Removes the "to be confirmed" markers of one case when that run passed and the spec checks them.');
+    process.exit(args.includes('--help') ? 0 : 1);
+}
+/** The report to trust: the one named by `--report` (a run id or a summary file), else the newest run. */
+function chosenSummary() {
+    if (reportArg === undefined)
+        return newestSummary();
+    const file = reportArg.endsWith('.json')
+        ? path.resolve(ROOT, reportArg)
+        : path.join(REPORTS_DIR, reportArg, 'summary.json');
+    if (!fs.existsSync(file))
+        return null;
+    return { where: path.relative(ROOT, file).replace(/\\/g, '/'), summary: JSON.parse(fs.readFileSync(file, 'utf8')) };
+}
+/** The newest report: `latest/` when it exists, otherwise the newest run folder that has a summary. */
+function newestSummary() {
+    const latest = path.join(REPORTS_DIR, 'latest', 'summary.json');
+    if (fs.existsSync(latest))
+        return { where: 'reports/latest/summary.json', summary: JSON.parse(fs.readFileSync(latest, 'utf8')) };
+    if (!fs.existsSync(REPORTS_DIR))
+        return null;
+    const runs = fs
+        .readdirSync(REPORTS_DIR)
+        .filter((name) => RUN_ID.test(name) && fs.existsSync(path.join(REPORTS_DIR, name, 'summary.json')))
+        .sort();
+    const last = runs.at(-1);
+    if (!last)
+        return null;
+    return {
+        where: `reports/${last}/summary.json`,
+        summary: JSON.parse(fs.readFileSync(path.join(REPORTS_DIR, last, 'summary.json'), 'utf8')),
+    };
+}
+const normalize = (value) => value
+    .replace(/`/g, '')
+    .replace(/[ \t]{2,}/g, ' ')
+    .replace(/[.;:,]+$/, '')
+    .trim()
+    .toLowerCase();
+const casePath = path.join(CASES_DIR, `${caseId}.md`);
+const specPath = path.join(TESTS_DIR, `${caseId}.spec.ts`);
+if (!fs.existsSync(casePath)) {
+    console.error(`FAIL ${caseId}: no case document at cases/${caseId}.md`);
+    process.exit(1);
+}
+const report = chosenSummary();
+if (report === null) {
+    console.error('FAIL: no run report found. Run the case first: pnpm asrun --env ' + env + ' --grep ' + caseId);
+    process.exit(1);
+}
+const results = report.summary.results.filter((result) => result.caseId === caseId);
+if (results.length === 0) {
+    console.error(`FAIL ${caseId}: ${report.where} holds no result for this case. Run it first.`);
+    process.exit(1);
+}
+const runEnv = report.summary.env ?? results.find((result) => result.env !== undefined)?.env;
+if (runEnv !== undefined && runEnv !== env) {
+    console.error(`FAIL ${caseId}: the newest report is a ${runEnv} run, not ${env}. Run the case in ${env} first.`);
+    process.exit(1);
+}
+const unsettled = results.filter((result) => result.status !== 'passed');
+if (unsettled.length > 0) {
+    console.error(`FAIL ${caseId}: ${unsettled.length} of ${results.length} test(s) did not pass in ${report.where} (${unsettled
+        .map((result) => `${result.profile}: ${result.status}`)
+        .join(', ')}); nothing is confirmed from a run that did not pass.`);
+    process.exit(1);
+}
+const text = fs.readFileSync(casePath, 'utf8');
+const doc = parseCaseDocument(text);
+if (doc.markers.length === 0) {
+    console.log(`${caseId}: no marker to settle.`);
+    process.exit(0);
+}
+if (!fs.existsSync(specPath)) {
+    console.log(`${caseId}: no spec yet (tests/${caseId}.spec.ts), so no run can confirm a marker.`);
+    process.exit(0);
+}
+const spec = fs.readFileSync(specPath, 'utf8');
+const quoted = [...spec.matchAll(UNCONFIRMED)].map((match) => normalize(match[1]));
+const confirmed = [];
+const left = [];
+for (const marker of doc.markers) {
+    const sentence = normalize(markerSentence(text, marker));
+    const match = sentence.length >= MIN_MATCH
+        ? quoted.find((quote) => quote.includes(sentence) || sentence.includes(quote))
+        : undefined;
+    if (match === undefined) {
+        left.push({ marker, reason: 'the spec has no "// unconfirmed:" line for it, so no assertion reads it' });
+        continue;
+    }
+    confirmed.push(marker);
+}
+for (const { marker, reason } of left) {
+    console.log(`keep ${caseId}:${marker.line}  ${markerSentence(text, marker).slice(0, 100)}`);
+    console.log(`     ${reason}`);
+}
+if (confirmed.length === 0) {
+    console.log(`${caseId}: nothing to settle from ${report.where}.`);
+    process.exit(0);
+}
+for (const marker of confirmed) {
+    console.log(`pass ${caseId}:${marker.line}  ${markerSentence(text, marker).slice(0, 100)}`);
+}
+if (dryRun) {
+    console.log(`dry run: ${confirmed.length} marker(s) would go, ${left.length} would stay.`);
+    process.exit(0);
+}
+fs.writeFileSync(casePath, removeMarkers(text, confirmed));
+console.log(`wrote cases/${caseId}.md: ${confirmed.length} marker(s) settled by ${report.where}, ${left.length} left.`);

package/dist/ui/server/actions.js

@@ -16,6 +16,21 @@ import { AI_COMMANDS, AI_COMMAND_WRITES, AI_TOOLS, continueCommandLine, continue
 const REPORTS_DIR = path.join(ROOT, 'reports');
 const REPORT_ID = /^\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}(-[a-z0-9]+(-[a-z0-9]+)*)?$/;
 const newSessionId = (tool) => tool === 'claude' ? crypto.randomUUID() : `ses_${crypto.randomBytes(12).toString('hex')}`;
+/**
+ * The request as the person made it, for "Run again". A confirmation they gave is left out on purpose: write cases and
+ * an overwritten spec are asked about again on the next run.
+ */
+function replayRequest(request) {
+    if (request.action === 'run') {
+        const { confirmWritesIn: _confirmed, ...rest } = request;
+        return rest;
+    }
+    if (request.action === 'ai') {
+        const { force: _force, ...rest } = request;
+        return rest;
+    }
+    return request;
+}
 export const startJobSchema = z.discriminatedUnion('action', [
     z.object({ action: z.literal('signin'), profile: z.string().min(1), env: z.string().min(1) }),
     z.object({ action: z.literal('visit'), profile: z.string().min(1), env: z.string().min(1) }),
@@ -157,6 +172,7 @@ export function buildJobSpec(request) {
             args: [...scriptArgs('login.ts'), request.profile],
             locks: [authLock(request.profile)],
             acceptsInput: true,
+            rerun: replayRequest(request),
             onFinish: () => void verifyProfiles({ ids: [request.profile] }),
         };
     }
@@ -175,16 +191,20 @@ export function buildJobSpec(request) {
             args: [...scriptArgs('explore.ts'), request.profile, ...(startPath ? [startPath] : [])],
             locks: [`visit:${request.profile}`],
             acceptsInput: false,
+            rerun: replayRequest(request),
         };
     }
     // An AI run on a case, a first turn or a continuation: the same permissions and locks either way.
     // Generating and healing write the spec and the case values and run the tests, so they must not overlap with a run.
-    // Reviewing only edits the case file and looks at pages, so it takes just the case lock.
+    // Reviewing only edits the case file, but every AI command drives the browser through the workspace's one
+    // `playwright-test` MCP server: two runs at once interleave on it and leave it wedged for every later run (one
+    // stops the other's paused test, and its call queue never drains). So the `ai` lock is workspace-wide, and only
+    // the case lock narrows it further.
     const aiSpec = (tool, command, caseId, env, sessionId, run, model, thinking) => ({
         kind: 'ai',
         env,
         ...run,
-        locks: [`ai:${caseId}`, ...(AI_COMMAND_WRITES[command] === 'files' ? ['run', ...allLocks()] : [])],
+        locks: ['ai', `ai:${caseId}`, ...(AI_COMMAND_WRITES[command] === 'files' ? ['run', ...allLocks()] : [])],
         acceptsInput: false,
         stream: tool,
         ai: { tool, command, caseId, env, sessionId, ...(model && { model }), ...(thinking && { thinking }) },
@@ -209,15 +229,20 @@ export function buildJobSpec(request) {
         if (request.attended && request.command !== 'explore-case') {
             throw new ActionError('Only explore-case can be attended.');
         }
-        if ((request.profiles || request.hints !== undefined || request.report) && request.command !== 'heal-spec') {
-            throw new ActionError('Only heal-spec takes profiles, hints and a report.');
+        if (request.report && request.command !== 'heal-spec') {
+            throw new ActionError('Only heal-spec reads a report.');
+        }
+        // The spec is generated for every profile the case names (and runs for every profile), so a selection could only
+        // narrow it wrongly.
+        if (request.profiles && request.command === 'gen-spec') {
+            throw new ActionError('gen-spec takes no profiles: it uses every profile the case holds information for.');
         }
         const profilesInEnv = listProfiles(request.env, knownProfiles).map((profile) => profile.id);
         if (request.profiles?.some((id) => !profilesInEnv.includes(id))) {
             throw new ActionError(`Unknown profile for "${request.env}".`);
         }
         if (request.profiles?.length === 0)
-            throw new ActionError('Choose at least one profile to heal.');
+            throw new ActionError('Choose at least one profile.');
         if (request.report !== undefined && !(REPORT_ID.test(request.report) && fs.existsSync(path.join(REPORTS_DIR, request.report, 'summary.json')))) {
             throw new ActionError(`Unknown report "${request.report}".`);
         }
@@ -234,25 +259,33 @@ export function buildJobSpec(request) {
             thinking: request.thinking,
             force: request.force,
             attended: request.attended,
-            // Naming every profile is the same as leaving it out: the agent then takes all that are logged in.
+            // Naming every profile of the environment is the same as leaving it out: the agent then takes all that are logged in.
             profiles: request.profiles && request.profiles.length < profilesInEnv.length ? request.profiles : undefined,
             hints: hintsPath,
             report: request.report,
         };
         const session = { id: newSessionId(request.tool), resume: false };
         const { file, args } = headlessInvocation(request.tool, request.command, request.caseId, options, session);
-        return aiSpec(request.tool, request.command, request.caseId, request.env, session.id, {
-            command: headlessCommandLine(request.tool, request.command, request.caseId, options, session),
-            file,
-            args,
-        }, request.model, request.thinking);
+        return {
+            ...aiSpec(request.tool, request.command, request.caseId, request.env, session.id, {
+                command: headlessCommandLine(request.tool, request.command, request.caseId, options, session),
+                file,
+                args,
+            }, request.model, request.thinking),
+            rerun: replayRequest(request),
+        };
     }
     if (request.action === 'continue') {
         const earlier = jobs.get(request.jobId);
         const ai = earlier?.spec.ai;
         if (!earlier || !ai)
             throw new ActionError('Unknown job, or it was not an AI run on a case.');
-        if (earlier.info.status !== 'succeeded' || earlier.info.resultState !== 'needs-input') {
+        const finishedWithQuestions = earlier.info.status === 'succeeded' && earlier.info.resultState === 'needs-input';
+        // A server restart can leave a run that had already written its questions: answering them is still the next step.
+        const recoveredWithQuestions = earlier.info.status === 'interrupted' &&
+            earlier.info.resultState === 'needs-input' &&
+            earlier.info.leftover?.state !== 'alive';
+        if (!finishedWithQuestions && !recoveredWithQuestions) {
             throw new ActionError('Only a run that finished with RESULT: needs-input can be continued.');
         }
         if (findContinuation(jobs.list(), earlier.info.id)) {
@@ -267,12 +300,16 @@ export function buildJobSpec(request) {
         fs.writeFileSync(path.join(ROOT, replyPath), request.reply.trim() + '\n');
         // The next turn keeps how the first one was told to run (model and reasoning), so the log stays the same.
         const { file, args } = continueInvocation(ai.tool, ai.command, replyPath, ai.sessionId, ai.model, ai.thinking);
-        return aiSpec(ai.tool, ai.command, ai.caseId, ai.env, ai.sessionId, {
-            command: continueCommandLine(ai.tool, ai.command, replyPath, ai.sessionId, ai.model, ai.thinking),
-            file,
-            args,
-            continuesJobId: earlier.info.id,
-        }, ai.model, ai.thinking);
+        return {
+            ...aiSpec(ai.tool, ai.command, ai.caseId, ai.env, ai.sessionId, {
+                command: continueCommandLine(ai.tool, ai.command, replyPath, ai.sessionId, ai.model, ai.thinking),
+                file,
+                args,
+                continuesJobId: earlier.info.id,
+            }, ai.model, ai.thinking),
+            // "Run again" on a continued run repeats the run it continued, not the answers that were sent to it.
+            ...(earlier.spec.rerun && { rerun: earlier.spec.rerun }),
+        };
     }
     if (request.action === 'make-cases') {
         if (request.memo.trim() === '')
@@ -290,6 +327,7 @@ export function buildJobSpec(request) {
             locks: ['ai:make-cases'],
             acceptsInput: false,
             stream: request.tool,
+            rerun: replayRequest(request),
         };
     }
     if (!knownEnvs.includes(request.env))
@@ -359,6 +397,7 @@ export function buildJobSpec(request) {
         reportId,
         locks: ['run', ...allLocks()],
         acceptsInput: false,
+        rerun: replayRequest(request),
         ...(updateBaselines.length > 0 && { baselineCases: updateBaselines }),
         onFinish: () => void verifyProfiles({ env: request.env }),
     };

package/dist/ui/server/app.js

@@ -111,7 +111,7 @@ export function createApp({ port }) {
     const assertProfileIdle = (id) => {
         const busy = jobs
             .list()
-            .find((job) => job.status === 'running' &&
+            .find((job) => jobs.holdsLock(job) &&
             (job.kind === 'run' ||
                 job.kind === 'ai' ||
                 (job.profile === id && (job.kind === 'signin' || job.kind === 'visit'))));
@@ -157,7 +157,7 @@ export function createApp({ port }) {
     api.get('/case-impact/:id{.+}', (c) => c.json(caseImpact(c.req.param('id'))));
     // Moving or deleting files under a running agent or test run would break it.
     const assertNoRunningWork = () => {
-        const busy = jobs.list().find((job) => job.status === 'running' && (job.kind === 'ai' || job.kind === 'run'));
+        const busy = jobs.list().find((job) => jobs.holdsLock(job) && (job.kind === 'ai' || job.kind === 'run'));
         if (busy)
             throw new CaseError(`Wait for "${busy.command}" to finish first.`, 409);
     };
@@ -280,6 +280,11 @@ export function createApp({ port }) {
         return c.json(await listAiModels(tool, c.req.query('refresh') === '1'));
     });
     api.get('/jobs', (c) => c.json(jobs.list()));
+    // What "Run again" starts. The stored request has no write-case or overwrite confirmation: the page asks again.
+    api.get('/jobs/:id/rerun', (c) => {
+        const request = jobs.rerunRequest(c.req.param('id'));
+        return request ? c.json({ request }) : c.json({ error: 'This job cannot be started again.' }, 404);
+    });
     api.post('/jobs', async (c) => {
         const request = startJobSchema.parse(await c.req.json());
         return c.json(jobs.start(buildJobSpec(request)), 201);

package/dist/ui/server/job-store.d.ts

@@ -0,0 +1,60 @@
+import type { AiCommand, AiTool } from '../shared/ai-commands.js';
+import type { JobChunk, JobInfo, StartJobRequest } from '../shared/types.js';
+/** What a job was started with, as far as it has to survive a restart. */
+export interface PersistedJobSpec {
+    kind: JobInfo['kind'];
+    profile?: string;
+    env?: string;
+    command: string;
+    file: string;
+    args: string[];
+    locks: string[];
+    acceptsInput: boolean;
+    stream?: AiTool;
+    baselineCases?: string[];
+    reportId?: string;
+    ai?: {
+        tool: AiTool;
+        command: AiCommand;
+        caseId: string;
+        env: string;
+        sessionId: string;
+        model?: string;
+        thinking?: boolean;
+    };
+    continuesJobId?: string;
+    /** What "Run again" starts: the person's request, without the confirmations they gave. */
+    rerun?: StartJobRequest;
+    /** The file the command resolved to on PATH, kept so a process that outlived the server can be recognized. */
+    spawnedPath?: string;
+}
+/** One job on disk: what the UI shows, how to continue it, and which process to stop after a restart. */
+export interface PersistedJob {
+    version: 1;
+    info: JobInfo;
+    spec: PersistedJobSpec;
+    /** The root process that was spawned; absent when it could not be started. */
+    pid?: number;
+}
+export declare const jobDir: (id: string) => string;
+export declare const chunksFile: (id: string) => string;
+export declare const logFile: (id: string) => string;
+export declare function ensureJobDir(id: string): void;
+/** Writes the record through a temporary file, so a crash cannot leave half a JSON document behind. */
+export declare function writeRecord(record: PersistedJob): void;
+/** Every record that can still be read, oldest first. A broken one is skipped: its log stays where it is. */
+export declare function readJobs(): PersistedJob[];
+/** The last chunks of a job, within the same memory budget the live job uses. */
+export declare function readChunks(id: string, maxChars: number): {
+    chunks: JobChunk[];
+    nextSeq: number;
+    bufferedChars: number;
+};
+/** Adds a chunk to a job that is not running any more (a note about the restart). */
+export declare function appendChunkNow(id: string, chunk: JobChunk, logText: string): void;
+/**
+ * Deletes what is no longer worth keeping: older than the retention window, or beyond the newest `keep` finished jobs.
+ * Jobs still running in this server (`live`) and a leftover process that may still be alive are never deleted: that
+ * record is the only way to manage them. Flat `.log` files from before records existed are pruned by age.
+ */
+export declare function pruneJobs(keep: number, maxAgeMs: number, live?: ReadonlySet<string>): void;

package/dist/ui/server/job-store.js

@@ -0,0 +1,198 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { ROOT } from '../../lib/paths.js';
+const JOBS_DIR = path.join(ROOT, '.tmp', 'ui-jobs');
+const RECORD_FILE = 'record.json';
+const CHUNKS_FILE = 'chunks.jsonl';
+const LOG_FILE = 'output.log';
+/** A temporary record file left behind by a crash is only useful while its writer may still be running. */
+const STALE_TEMP_MS = 60 * 60 * 1000;
+// Job ids are `crypto.randomBytes(6).toString('hex')`; the pattern also keeps a stray folder out of the paths below.
+const JOB_ID = /^[a-f0-9]{8,32}$/;
+/** How much of a chunks file is read back into memory; the full text stays in output.log. */
+const MAX_TAIL_BYTES = 8 * 1024 * 1024;
+export const jobDir = (id) => path.join(JOBS_DIR, id);
+export const chunksFile = (id) => path.join(jobDir(id), CHUNKS_FILE);
+export const logFile = (id) => path.join(jobDir(id), LOG_FILE);
+export function ensureJobDir(id) {
+    fs.mkdirSync(jobDir(id), { recursive: true });
+}
+function readRecord(id) {
+    try {
+        const parsed = JSON.parse(fs.readFileSync(path.join(jobDir(id), RECORD_FILE), 'utf8'));
+        if (parsed?.version !== 1 || parsed.info?.id !== id)
+            return null;
+        return parsed;
+    }
+    catch {
+        return null;
+    }
+}
+/** Writes the record through a temporary file, so a crash cannot leave half a JSON document behind. */
+export function writeRecord(record) {
+    const target = path.join(jobDir(record.info.id), RECORD_FILE);
+    ensureJobDir(record.info.id);
+    const temporary = `${target}.${process.pid}.tmp`;
+    try {
+        fs.writeFileSync(temporary, JSON.stringify(record));
+        fs.renameSync(temporary, target);
+    }
+    catch (error) {
+        fs.rmSync(temporary, { force: true });
+        throw error;
+    }
+}
+/** Every record that can still be read, oldest first. A broken one is skipped: its log stays where it is. */
+export function readJobs() {
+    if (!fs.existsSync(JOBS_DIR))
+        return [];
+    const jobs = [];
+    for (const entry of fs.readdirSync(JOBS_DIR, { withFileTypes: true })) {
+        if (!entry.isDirectory() || !JOB_ID.test(entry.name))
+            continue;
+        const record = readRecord(entry.name);
+        if (record)
+            jobs.push(record);
+        else
+            console.warn(`Ignoring the job record in ${path.join(JOBS_DIR, entry.name)}: it could not be read.`);
+    }
+    return jobs.sort((left, right) => left.info.startedAt.localeCompare(right.info.startedAt));
+}
+/** The last chunks of a job, within the same memory budget the live job uses. */
+export function readChunks(id, maxChars) {
+    let text = '';
+    try {
+        const file = chunksFile(id);
+        const size = fs.statSync(file).size;
+        const length = Math.min(size, Math.max(maxChars * 2, MAX_TAIL_BYTES));
+        const start = size - length;
+        if (length > 0) {
+            const buffer = Buffer.alloc(length);
+            const descriptor = fs.openSync(file, 'r');
+            try {
+                fs.readSync(descriptor, buffer, 0, length, start);
+            }
+            finally {
+                fs.closeSync(descriptor);
+            }
+            text = buffer.toString('utf8');
+            // The tail starts in the middle of a line: that chunk is not valid and is dropped below.
+            if (start > 0)
+                text = text.slice(text.indexOf('\n') + 1);
+        }
+    }
+    catch {
+        return { chunks: [], nextSeq: 0, bufferedChars: 0 };
+    }
+    const chunks = [];
+    let bufferedChars = 0;
+    for (const line of text.split('\n')) {
+        if (line.trim() === '')
+            continue;
+        try {
+            const chunk = JSON.parse(line);
+            if (typeof chunk?.seq !== 'number' || typeof chunk.data !== 'string')
+                continue;
+            chunks.push(chunk);
+            bufferedChars += chunk.data.length;
+        }
+        catch {
+            // A crash can leave the last line half-written; it is not a chunk.
+        }
+    }
+    while (bufferedChars > maxChars && chunks.length > 1) {
+        bufferedChars -= chunks.shift().data.length;
+    }
+    const nextSeq = chunks.reduce((next, chunk) => Math.max(next, chunk.seq + 1), 0);
+    return { chunks, nextSeq, bufferedChars };
+}
+/** Adds a chunk to a job that is not running any more (a note about the restart). */
+export function appendChunkNow(id, chunk, logText) {
+    ensureJobDir(id);
+    fs.appendFileSync(chunksFile(id), `${JSON.stringify(chunk)}\n`);
+    fs.appendFileSync(logFile(id), logText);
+}
+/**
+ * Deletes what is no longer worth keeping: older than the retention window, or beyond the newest `keep` finished jobs.
+ * Jobs still running in this server (`live`) and a leftover process that may still be alive are never deleted: that
+ * record is the only way to manage them. Flat `.log` files from before records existed are pruned by age.
+ */
+export function pruneJobs(keep, maxAgeMs, live = new Set()) {
+    if (!fs.existsSync(JOBS_DIR))
+        return;
+    const now = Date.now();
+    const records = [];
+    for (const entry of fs.readdirSync(JOBS_DIR, { withFileTypes: true })) {
+        const full = path.join(JOBS_DIR, entry.name);
+        if (!entry.isDirectory()) {
+            // The log of a job from before this format; no record, so only its age can decide.
+            if (/\.log$/i.test(entry.name)) {
+                try {
+                    if (now - fs.statSync(full).mtimeMs > maxAgeMs)
+                        fs.rmSync(full, { force: true });
+                }
+                catch {
+                    // It disappeared while it was being looked at.
+                }
+            }
+            else if (/^record\.json\..+\.tmp$/i.test(entry.name)) {
+                try {
+                    if (now - fs.statSync(full).mtimeMs > STALE_TEMP_MS)
+                        fs.rmSync(full, { force: true });
+                }
+                catch {
+                    // It disappeared while it was being looked at.
+                }
+            }
+            continue;
+        }
+        if (!JOB_ID.test(entry.name) || live.has(entry.name))
+            continue;
+        const record = readRecord(entry.name);
+        // A directory that never got its record (a crash between creating the folder and writing it) is dropped once it is
+        // old enough that no writer can still be filling it.
+        if (!record) {
+            try {
+                if (now - fs.statSync(full).mtimeMs > STALE_TEMP_MS)
+                    fs.rmSync(full, { recursive: true, force: true });
+            }
+            catch {
+                // It disappeared while it was being looked at.
+            }
+            continue;
+        }
+        const leftover = record.info.leftover?.state;
+        const endedAt = record.info.endedAt ?? record.info.interruptedAt ?? record.info.startedAt;
+        const parsed = endedAt ? Date.parse(endedAt) : Number.NaN;
+        let time;
+        try {
+            time = Number.isFinite(parsed) ? parsed : fs.statSync(full).mtimeMs;
+        }
+        catch {
+            continue;
+        }
+        // A record from a crashed server has to survive until the next start can mark it; an orphan is pinned for ever,
+        // because it is the only way to stop its process.
+        const pinned = record.info.status === 'running' ||
+            (record.info.status === 'interrupted' && (leftover === 'alive' || leftover === 'unknown'));
+        records.push({ id: entry.name, time, pinned });
+    }
+    records.sort((left, right) => right.time - left.time);
+    let kept = 0;
+    for (const record of records) {
+        if (record.pinned)
+            continue;
+        const tooOld = now - record.time > maxAgeMs;
+        // `kept` counts the newest finished jobs, so the list is cleared from the back.
+        const tooMany = kept >= keep;
+        kept += 1;
+        if (!tooOld && !tooMany)
+            continue;
+        try {
+            fs.rmSync(jobDir(record.id), { recursive: true, force: true });
+        }
+        catch {
+            // A file still open in another process: leave it for the next cleanup.
+        }
+    }
+}