@asrunjs/asrun 0.9.0 → 0.11.0

package/dist/ui/server/app.js

@@ -111,7 +111,7 @@ export function createApp({ port }) {
     const assertProfileIdle = (id) => {
         const busy = jobs
             .list()
-            .find((job) => job.status === 'running' &&
+            .find((job) => jobs.holdsLock(job) &&
             (job.kind === 'run' ||
                 job.kind === 'ai' ||
                 (job.profile === id && (job.kind === 'signin' || job.kind === 'visit'))));
@@ -157,7 +157,7 @@ export function createApp({ port }) {
     api.get('/case-impact/:id{.+}', (c) => c.json(caseImpact(c.req.param('id'))));
     // Moving or deleting files under a running agent or test run would break it.
     const assertNoRunningWork = () => {
-        const busy = jobs.list().find((job) => job.status === 'running' && (job.kind === 'ai' || job.kind === 'run'));
+        const busy = jobs.list().find((job) => jobs.holdsLock(job) && (job.kind === 'ai' || job.kind === 'run'));
         if (busy)
             throw new CaseError(`Wait for "${busy.command}" to finish first.`, 409);
     };
@@ -280,6 +280,11 @@ export function createApp({ port }) {
         return c.json(await listAiModels(tool, c.req.query('refresh') === '1'));
     });
     api.get('/jobs', (c) => c.json(jobs.list()));
+    // What "Run again" starts. The stored request has no write-case or overwrite confirmation: the page asks again.
+    api.get('/jobs/:id/rerun', (c) => {
+        const request = jobs.rerunRequest(c.req.param('id'));
+        return request ? c.json({ request }) : c.json({ error: 'This job cannot be started again.' }, 404);
+    });
     api.post('/jobs', async (c) => {
         const request = startJobSchema.parse(await c.req.json());
         return c.json(jobs.start(buildJobSpec(request)), 201);

package/dist/ui/server/job-store.d.ts

@@ -0,0 +1,60 @@
+import type { AiCommand, AiTool } from '../shared/ai-commands.js';
+import type { JobChunk, JobInfo, StartJobRequest } from '../shared/types.js';
+/** What a job was started with, as far as it has to survive a restart. */
+export interface PersistedJobSpec {
+    kind: JobInfo['kind'];
+    profile?: string;
+    env?: string;
+    command: string;
+    file: string;
+    args: string[];
+    locks: string[];
+    acceptsInput: boolean;
+    stream?: AiTool;
+    baselineCases?: string[];
+    reportId?: string;
+    ai?: {
+        tool: AiTool;
+        command: AiCommand;
+        caseId: string;
+        env: string;
+        sessionId: string;
+        model?: string;
+        thinking?: boolean;
+    };
+    continuesJobId?: string;
+    /** What "Run again" starts: the person's request, without the confirmations they gave. */
+    rerun?: StartJobRequest;
+    /** The file the command resolved to on PATH, kept so a process that outlived the server can be recognized. */
+    spawnedPath?: string;
+}
+/** One job on disk: what the UI shows, how to continue it, and which process to stop after a restart. */
+export interface PersistedJob {
+    version: 1;
+    info: JobInfo;
+    spec: PersistedJobSpec;
+    /** The root process that was spawned; absent when it could not be started. */
+    pid?: number;
+}
+export declare const jobDir: (id: string) => string;
+export declare const chunksFile: (id: string) => string;
+export declare const logFile: (id: string) => string;
+export declare function ensureJobDir(id: string): void;
+/** Writes the record through a temporary file, so a crash cannot leave half a JSON document behind. */
+export declare function writeRecord(record: PersistedJob): void;
+/** Every record that can still be read, oldest first. A broken one is skipped: its log stays where it is. */
+export declare function readJobs(): PersistedJob[];
+/** The last chunks of a job, within the same memory budget the live job uses. */
+export declare function readChunks(id: string, maxChars: number): {
+    chunks: JobChunk[];
+    nextSeq: number;
+    bufferedChars: number;
+};
+/** Adds a chunk to a job that is not running any more (a note about the restart). */
+export declare function appendChunkNow(id: string, chunk: JobChunk, logText: string): void;
+/**
+ * Deletes what is no longer worth keeping: older than the retention window, or beyond the newest `keep` finished jobs.
+ * Jobs still running in this server (`live`) and a leftover process that may still be alive are never deleted: that
+ * record is the only way to manage them. Flat `.log` files from before records existed are pruned by age.
+ */
+export declare function pruneJobs(keep: number, maxAgeMs: number, live?: ReadonlySet<string>): void;

package/dist/ui/server/job-store.js

@@ -0,0 +1,198 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { ROOT } from '../../lib/paths.js';
+const JOBS_DIR = path.join(ROOT, '.tmp', 'ui-jobs');
+const RECORD_FILE = 'record.json';
+const CHUNKS_FILE = 'chunks.jsonl';
+const LOG_FILE = 'output.log';
+/** A temporary record file left behind by a crash is only useful while its writer may still be running. */
+const STALE_TEMP_MS = 60 * 60 * 1000;
+// Job ids are `crypto.randomBytes(6).toString('hex')`; the pattern also keeps a stray folder out of the paths below.
+const JOB_ID = /^[a-f0-9]{8,32}$/;
+/** How much of a chunks file is read back into memory; the full text stays in output.log. */
+const MAX_TAIL_BYTES = 8 * 1024 * 1024;
+export const jobDir = (id) => path.join(JOBS_DIR, id);
+export const chunksFile = (id) => path.join(jobDir(id), CHUNKS_FILE);
+export const logFile = (id) => path.join(jobDir(id), LOG_FILE);
+export function ensureJobDir(id) {
+    fs.mkdirSync(jobDir(id), { recursive: true });
+}
+function readRecord(id) {
+    try {
+        const parsed = JSON.parse(fs.readFileSync(path.join(jobDir(id), RECORD_FILE), 'utf8'));
+        if (parsed?.version !== 1 || parsed.info?.id !== id)
+            return null;
+        return parsed;
+    }
+    catch {
+        return null;
+    }
+}
+/** Writes the record through a temporary file, so a crash cannot leave half a JSON document behind. */
+export function writeRecord(record) {
+    const target = path.join(jobDir(record.info.id), RECORD_FILE);
+    ensureJobDir(record.info.id);
+    const temporary = `${target}.${process.pid}.tmp`;
+    try {
+        fs.writeFileSync(temporary, JSON.stringify(record));
+        fs.renameSync(temporary, target);
+    }
+    catch (error) {
+        fs.rmSync(temporary, { force: true });
+        throw error;
+    }
+}
+/** Every record that can still be read, oldest first. A broken one is skipped: its log stays where it is. */
+export function readJobs() {
+    if (!fs.existsSync(JOBS_DIR))
+        return [];
+    const jobs = [];
+    for (const entry of fs.readdirSync(JOBS_DIR, { withFileTypes: true })) {
+        if (!entry.isDirectory() || !JOB_ID.test(entry.name))
+            continue;
+        const record = readRecord(entry.name);
+        if (record)
+            jobs.push(record);
+        else
+            console.warn(`Ignoring the job record in ${path.join(JOBS_DIR, entry.name)}: it could not be read.`);
+    }
+    return jobs.sort((left, right) => left.info.startedAt.localeCompare(right.info.startedAt));
+}
+/** The last chunks of a job, within the same memory budget the live job uses. */
+export function readChunks(id, maxChars) {
+    let text = '';
+    try {
+        const file = chunksFile(id);
+        const size = fs.statSync(file).size;
+        const length = Math.min(size, Math.max(maxChars * 2, MAX_TAIL_BYTES));
+        const start = size - length;
+        if (length > 0) {
+            const buffer = Buffer.alloc(length);
+            const descriptor = fs.openSync(file, 'r');
+            try {
+                fs.readSync(descriptor, buffer, 0, length, start);
+            }
+            finally {
+                fs.closeSync(descriptor);
+            }
+            text = buffer.toString('utf8');
+            // The tail starts in the middle of a line: that chunk is not valid and is dropped below.
+            if (start > 0)
+                text = text.slice(text.indexOf('\n') + 1);
+        }
+    }
+    catch {
+        return { chunks: [], nextSeq: 0, bufferedChars: 0 };
+    }
+    const chunks = [];
+    let bufferedChars = 0;
+    for (const line of text.split('\n')) {
+        if (line.trim() === '')
+            continue;
+        try {
+            const chunk = JSON.parse(line);
+            if (typeof chunk?.seq !== 'number' || typeof chunk.data !== 'string')
+                continue;
+            chunks.push(chunk);
+            bufferedChars += chunk.data.length;
+        }
+        catch {
+            // A crash can leave the last line half-written; it is not a chunk.
+        }
+    }
+    while (bufferedChars > maxChars && chunks.length > 1) {
+        bufferedChars -= chunks.shift().data.length;
+    }
+    const nextSeq = chunks.reduce((next, chunk) => Math.max(next, chunk.seq + 1), 0);
+    return { chunks, nextSeq, bufferedChars };
+}
+/** Adds a chunk to a job that is not running any more (a note about the restart). */
+export function appendChunkNow(id, chunk, logText) {
+    ensureJobDir(id);
+    fs.appendFileSync(chunksFile(id), `${JSON.stringify(chunk)}\n`);
+    fs.appendFileSync(logFile(id), logText);
+}
+/**
+ * Deletes what is no longer worth keeping: older than the retention window, or beyond the newest `keep` finished jobs.
+ * Jobs still running in this server (`live`) and a leftover process that may still be alive are never deleted: that
+ * record is the only way to manage them. Flat `.log` files from before records existed are pruned by age.
+ */
+export function pruneJobs(keep, maxAgeMs, live = new Set()) {
+    if (!fs.existsSync(JOBS_DIR))
+        return;
+    const now = Date.now();
+    const records = [];
+    for (const entry of fs.readdirSync(JOBS_DIR, { withFileTypes: true })) {
+        const full = path.join(JOBS_DIR, entry.name);
+        if (!entry.isDirectory()) {
+            // The log of a job from before this format; no record, so only its age can decide.
+            if (/\.log$/i.test(entry.name)) {
+                try {
+                    if (now - fs.statSync(full).mtimeMs > maxAgeMs)
+                        fs.rmSync(full, { force: true });
+                }
+                catch {
+                    // It disappeared while it was being looked at.
+                }
+            }
+            else if (/^record\.json\..+\.tmp$/i.test(entry.name)) {
+                try {
+                    if (now - fs.statSync(full).mtimeMs > STALE_TEMP_MS)
+                        fs.rmSync(full, { force: true });
+                }
+                catch {
+                    // It disappeared while it was being looked at.
+                }
+            }
+            continue;
+        }
+        if (!JOB_ID.test(entry.name) || live.has(entry.name))
+            continue;
+        const record = readRecord(entry.name);
+        // A directory that never got its record (a crash between creating the folder and writing it) is dropped once it is
+        // old enough that no writer can still be filling it.
+        if (!record) {
+            try {
+                if (now - fs.statSync(full).mtimeMs > STALE_TEMP_MS)
+                    fs.rmSync(full, { recursive: true, force: true });
+            }
+            catch {
+                // It disappeared while it was being looked at.
+            }
+            continue;
+        }
+        const leftover = record.info.leftover?.state;
+        const endedAt = record.info.endedAt ?? record.info.interruptedAt ?? record.info.startedAt;
+        const parsed = endedAt ? Date.parse(endedAt) : Number.NaN;
+        let time;
+        try {
+            time = Number.isFinite(parsed) ? parsed : fs.statSync(full).mtimeMs;
+        }
+        catch {
+            continue;
+        }
+        // A record from a crashed server has to survive until the next start can mark it; an orphan is pinned for ever,
+        // because it is the only way to stop its process.
+        const pinned = record.info.status === 'running' ||
+            (record.info.status === 'interrupted' && (leftover === 'alive' || leftover === 'unknown'));
+        records.push({ id: entry.name, time, pinned });
+    }
+    records.sort((left, right) => right.time - left.time);
+    let kept = 0;
+    for (const record of records) {
+        if (record.pinned)
+            continue;
+        const tooOld = now - record.time > maxAgeMs;
+        // `kept` counts the newest finished jobs, so the list is cleared from the back.
+        const tooMany = kept >= keep;
+        kept += 1;
+        if (!tooOld && !tooMany)
+            continue;
+        try {
+            fs.rmSync(jobDir(record.id), { recursive: true, force: true });
+        }
+        catch {
+            // A file still open in another process: leave it for the next cleanup.
+        }
+    }
+}

package/dist/ui/server/jobs.d.ts

@@ -1,7 +1,7 @@
 import { type ChildProcess } from 'node:child_process';
 import { EventEmitter } from 'node:events';
 import fs from 'node:fs';
-import type { JobChunk, JobInfo, JobKind } from '../shared/types.js';
+import type { JobChunk, JobInfo, JobKind, StartJobRequest } from '../shared/types.js';
 import type { AiCommand, AiTool } from '../shared/ai-commands.js';
 import { AiStreamFormatter } from './ai-stream.js';
 export declare class JobConflictError extends Error {
@@ -36,6 +36,10 @@ export interface JobSpec {
     };
     /** The job whose conversation this one continues. */
     continuesJobId?: string;
+    /** What "Run again" starts: the request as the person made it, without the confirmations they gave. */
+    rerun?: StartJobRequest;
+    /** The file the command resolved to on PATH, used to recognize the process after a restart. */
+    spawnedPath?: string;
     /** Runs synchronously right before the final status is published. */
     onFinish?: (job: JobInfo) => void;
 }
@@ -47,34 +51,72 @@ interface JobRecord {
     nextSeq: number;
     emitter: EventEmitter;
     child: ChildProcess | null;
+    /** The root process, also kept on disk so a job that outlived the server can still be stopped. */
+    pid: number | null;
     cancelRequested: boolean;
-    logStream: fs.WriteStream;
+    /** Set when the stall watchdog stopped the run: it ends as failed, not as a cancel the person asked for. */
+    stalled: boolean;
+    /** When the last block was added (a heartbeat line does not count, or it would keep the run alive). */
+    lastActivityAt: number;
+    /** Null for a job loaded from disk: it does not write any more. */
+    logStream: fs.WriteStream | null;
+    chunkStream: fs.WriteStream | null;
+    /** False for a finished job loaded from disk: its chunks are read only when someone opens it. */
+    chunksLoaded: boolean;
     formatter: AiStreamFormatter | null;
     heartbeat: NodeJS.Timeout | null;
 }
 declare class JobManager {
     private jobs;
+    private leftoverTimer;
     /** Emits `job` with a JobInfo on every status change of any job. */
     readonly events: EventEmitter<[never]>;
     constructor();
     list(): JobInfo[];
     get(id: string): JobRecord | undefined;
+    /** The request "Run again" starts, or undefined when this job was not started from one. */
+    rerunRequest(id: string): StartJobRequest | undefined;
+    /**
+     * True while this job still holds its locks. A job a restart left behind holds them only while its process is
+     * confirmed to be alive: starting another agent next to it would wedge the one MCP server for both.
+     */
+    holdsLock(info: JobInfo): boolean;
     start(spec: JobSpec): JobInfo;
     write(id: string, text: string): boolean;
     cancel(id: string): boolean;
     /** Stops everything; called when the server shuts down. */
     killAll(): void;
+    /**
+     * Reads the jobs of earlier server runs. One that was still running is marked `interrupted`: its output stops where
+     * the server died, and its process is either gone or a leftover the person can stop. An AI run that had already
+     * written its `RESULT:` line is recovered as succeeded, so its answers can still be answered.
+     */
+    private recover;
+    /** Reads a finished job's chunks the first time someone opens it; a job this server runs already has them. */
+    private ensureChunks;
+    private recoverInterrupted;
+    /** Keeps the leftover state current, so a process that exits by itself stops blocking new jobs. */
+    private startLeftoverProbe;
+    private checkLeftovers;
+    /** After a stop was requested, make sure the process is really gone; a signal that was ignored is not a stop. */
+    private verifyLeftoverStopped;
     /** Publishes a change of "waiting for the person" (a handoff card) like any other status change. */
     private syncAwaiting;
-    private signalGroup;
+    /**
+     * Stops a run whose agent has said nothing for `STALL_MS`: a tool call that never returns leaves the CLI waiting for
+     * ever, and the log would only keep repeating "still waiting for ...". The block above the cancel names what to do.
+     */
+    private watchStall;
     /** A block of an AI run's log, with the time it was produced and how the UI should present it. */
     private pushPart;
     private push;
     /** Keeps the chunk for the browser and writes it to the job's log file, which is what is left after the run. */
     private store;
+    /** A chunk for a job that no longer runs: appended to both files, never to a stream that is still open. */
+    private pushRecovered;
+    private persist;
     private finish;
     private pruneFinished;
-    private pruneOldLogs;
 }
 export declare const jobs: JobManager;
 export {};