@wjarka/cezarion 0.14.9-dev.915 → 0.14.9-dev.936

package/dist/workflows/run.d.ts

@@ -219,6 +219,16 @@ export declare function pastedAttachmentsNote(attachments: PersistedAttachment[]
 export declare const VARIANT_LETTERS: readonly ['A', 'B', 'C'];
 /** Only an original workflow that has never executed may return to its queue. */
 export declare function isUntouchedCancelledRun(run: RunRecord): boolean;
+/** #469: why destroy could not prove a crashed generation's termination. */
+export type WorkerTerminationBlocker = {
+    kind: 'unreadable';
+} | {
+    kind: 'controller';
+    pid: number;
+} | {
+    kind: 'processes';
+    pids: number[];
+};
 /**
  * The mini workflow engine: executes a `WorkflowDef` against a repo, one step
  * at a time, persisting every event to the RunStore (which the SSE endpoints
@@ -248,9 +258,53 @@ export declare class RunManager {
      * exception or an interrupt request cannot stand in for session termination. */
     private trackWorkerSessionResult;
     private persistWorkerCompletion;
+    /** #469: the generation a dead controller left `starting`, if this manager may judge it. A
+     * generation this process controls is never judged here: the in-memory execution map owns it,
+     * and a disposed manager's still-running sessions look like orphans. `busy` is transient (this
+     * manager holds the run); `unknown` is a present record that proves nothing. */
+    private orphanState;
+    private orphanedWorkerGeneration;
+    /** Completes a crashed generation's proof once every one of its processes is proven gone (#469).
+     * Sync and signal-free; recovery, resume, delivery, collect and destroy then take their normal paths.
+     * A non-gone probe is cached briefly: on darwin the scan is a synchronous `lsof` on the event loop. */
+    settleOrphanedWorkerExecution(runId: string, opts?: {
+        admitting?: boolean;
+        fresh?: boolean;
+    }): boolean;
+    private readonly orphanProbes;
+    private readonly orphanReprobes;
+    private readonly orphanBlockers;
+    private readonly reportedOrphanBlockers;
+    private orphanReprobeMs;
+    private orphanReprobeLimitMs;
+    private orphanReprobeSlowMs;
+    private orphanTermGraceMs;
+    /** #469: a survivor that dies after recovery has no other wake source (no exit callback for a
+     * process another cezar spawned). Unref'd; finalization's `run` event then lets worker waits and
+     * outcomes observe it. Fast for the first window, then slow but uncapped, so a long-lived
+     * survivor is still noticed when it exits. A tick while this manager holds the run is skipped;
+     * only terminal reasons (proof settled, generation changed, run gone, unknown record, dispose)
+     * stop it. */
+    private armOrphanReprobe;
+    private clearOrphanReprobe;
+    /** Why the last destroy could not prove termination (#469), and whether that differs from the
+     * reason last reported, so a retry loop appends one lifecycle event per distinct blocker set. */
+    takeWorkerTerminationBlocker(runId: string): {
+        blocker: WorkerTerminationBlocker;
+        changed: boolean;
+    } | undefined;
+    /** Continue over a live orphan names what still runs; other refusals keep the generic text. */
+    private orphanAdmissionRefusal;
+    /** Destroy only (#469): signal the recorded, token-verified survivors of a generation whose
+     * controller is proven dead, then finalize on `gone`. Scan-only processes are never signalled. */
+    private reapOrphanedWorker;
+    /** Best effort: a failed write leaves the working-directory scan as this process's evidence. */
+    private recordWorkerProcess;
     private trackTurnEnd;
     requestWorkerStop(runId: string): WorkerStopResult;
-    awaitRunTermination(runId: string, timeoutMs: number): Promise<boolean>;
+    awaitRunTermination(runId: string, timeoutMs: number, opts?: {
+        reapOrphans?: boolean;
+    }): Promise<boolean>;
     /** Private cleanup capability, never serialized. Obtain after stop/termination
      * under the service's destroy lock; each invocation rechecks the same generation. */
     getWorkerNoMaterializationProof(runId: string): WorkerNoMaterializationProof | undefined;

package/dist/workflows/run.js

@@ -9,6 +9,7 @@ import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync
 import { dirname, join } from 'node:path';
 import { parseAskMarkerResult, stripAskMarker, } from '../core/ask.js';
 import { hasRegisteredRunProcess, onUsage, registerRunProcess, unregisterRunProcess } from '../core/process-usage.js';
+import { inspectGeneration, isCurrentProcess, processStartToken, recordedProcessLive } from '../delegation/process-liveness.js';
 import { parseUsageLimit } from '../core/usage-limit.js';
 import { createRunner } from '../core/runner-factory.js';
 import { modelConflictsWithRunner } from '../core/model-presets.js';
@@ -41,7 +42,7 @@ import { answersQuestion, openQuestions, questionMessage } from '../delegation/q
 import { workerOutcome } from '../runs/delegation-state.js';
 import { loadWorkflows } from './load.js';
 import { reclaimWorktrees, rematerializeReclaimedWorktree } from '../runs/retention.js';
-import { AgentTempDirError, agentTmpEnv, removeAgentTmpDir, sweepAgentTmpDirs, } from '../runs/agent-tmpdir.js';
+import { AgentTempDirError, agentTmpEnv, agentTmpDirLocations, removeAgentTmpDir, sweepAgentTmpDirs, } from '../runs/agent-tmpdir.js';
 import { extractTaskRefs, refineTaskRefs, titleRefNumber } from '../runs/task-refs.js';
 import { parseTaskMarkers, stripTaskMarkers } from '../runs/task-markers.js';
 import { autoNamingActive, generateRunName, liveTitleUpdatesEnabled, postValidateTitle } from '../runs/auto-name.js';
@@ -426,6 +427,12 @@ export function isUntouchedCancelledRun(run) {
     return run.status === 'cancelled' && !run.startedAt && !!run.workflowDef &&
         run.steps.every(step => step.status === 'pending' && !step.startedAt && !step.sessionId);
 }
+/** #469: how long a non-gone orphan probe stands before the next scan. */
+const ORPHAN_PROBE_CACHE_MS = 2_000;
+/** #469: admission refused because a process of the previous generation still runs. */
+class WorkerOrphanAliveError extends Error {
+}
+const admissionError = (error, fallback) => error instanceof WorkerOrphanAliveError ? error.message : fallback;
 /**
  * The mini workflow engine: executes a `WorkflowDef` against a repo, one step
  * at a time, persisting every event to the RunStore (which the SSE endpoints
@@ -455,6 +462,13 @@ export class RunManager {
     beginWorkerExecution(runId, admitted = true) {
         if (this.store.getRun(runId)?.delegation?.role !== 'worker')
             return;
+        // #469: one site for Continue, --resume, parent replies and queued revival after a crash.
+        // Fresh: admission is one-shot, so a cached "alive" must not refuse an orphan that has since died.
+        if (!this.executions.has(runId))
+            this.settleOrphanedWorkerExecution(runId, { admitting: true, fresh: true });
+        const refusal = this.executions.has(runId) ? undefined : this.orphanAdmissionRefusal(runId);
+        if (refusal)
+            throw new WorkerOrphanAliveError(refusal);
         const existing = this.executions.get(runId);
         if (existing) {
             const proof = this.store.readWorkerExecution(runId);
@@ -469,6 +483,9 @@ export class RunManager {
         const promise = new Promise(done => { resolve = done; });
         this.executions.set(runId, { generation, admitted, promise, resolve, sessions: new Set(), turns: new Set(), deliveries: new Set() });
         this.finalizedWorkers.delete(runId);
+        // #469: an earlier generation's destroy blocker never describes this one.
+        this.orphanBlockers.delete(runId);
+        this.reportedOrphanBlockers.delete(runId);
         this.stoppedWorkers.delete(runId);
     }
     async finishWorkerExecution(runId) {
@@ -518,6 +535,196 @@ export class RunManager {
         }
         catch { /* Finalized in this process; preserve exact generation for an explicit retry. */ }
     }
+    /** #469: the generation a dead controller left `starting`, if this manager may judge it. A
+     * generation this process controls is never judged here: the in-memory execution map owns it,
+     * and a disposed manager's still-running sessions look like orphans. `busy` is transient (this
+     * manager holds the run); `unknown` is a present record that proves nothing. */
+    orphanState(runId, admitting = false) {
+        const run = this.store.getRun(runId);
+        if (run?.delegation?.role !== 'worker' || this.disposed)
+            return { state: 'none' };
+        if (this.executions.has(runId) || this.active.has(runId) || this.starting.has(runId) || (!admitting && this.queue.includes(runId)))
+            return { state: 'busy' };
+        const proof = this.store.readWorkerExecution(runId);
+        if (proof?.phase !== 'starting')
+            return { state: 'none' };
+        const record = this.store.readWorkerProcesses(runId, proof.generation);
+        // A present record that proves nothing (unreadable, malformed, another generation) blocks both
+        // finalization and reaping; only an absent one is legacy, scan-only evidence.
+        if (record === 'unknown')
+            return { state: 'unknown' };
+        if (record !== 'absent' && isCurrentProcess(record.controller))
+            return { state: 'none' };
+        // Finalization deletes the scratch too, so a process working there keeps the generation alive.
+        return { state: 'orphan', generation: proof.generation, ...(record === 'absent' ? {} : { record }),
+            paths: [run.delegation.workspace.path, ...agentTmpDirLocations(this.dataDir, runId)],
+            // No process of this worker can predate its record (1 s slack for tick rounding).
+            ...(Number.isFinite(Date.parse(run.createdAt)) ? { since: Date.parse(run.createdAt) - 1_000 } : {}) };
+    }
+    orphanedWorkerGeneration(runId, admitting = false) {
+        const orphan = this.orphanState(runId, admitting);
+        return orphan.state === 'orphan' ? orphan : undefined;
+    }
+    /** Completes a crashed generation's proof once every one of its processes is proven gone (#469).
+     * Sync and signal-free; recovery, resume, delivery, collect and destroy then take their normal paths.
+     * A non-gone probe is cached briefly: on darwin the scan is a synchronous `lsof` on the event loop. */
+    settleOrphanedWorkerExecution(runId, opts = {}) {
+        const orphan = this.orphanedWorkerGeneration(runId, opts.admitting);
+        if (!orphan)
+            return false;
+        const cached = this.orphanProbes.get(runId);
+        if (!opts.fresh && cached?.generation === orphan.generation && Date.now() - cached.at < ORPHAN_PROBE_CACHE_MS)
+            return false;
+        const probe = inspectGeneration(orphan);
+        if (probe.liveness !== 'gone') {
+            this.orphanProbes.set(runId, { generation: orphan.generation, at: Date.now(), probe });
+            return false;
+        }
+        // A stale `alive` must not outlive a `gone` probe, even when the commit below fails.
+        this.orphanProbes.delete(runId);
+        try {
+            if (!this.store.commitWorkerExecutionComplete(runId, orphan.generation))
+                return false;
+        }
+        catch {
+            return false;
+        }
+        this.orphanProbes.delete(runId);
+        this.orphanBlockers.delete(runId);
+        this.reportedOrphanBlockers.delete(runId);
+        this.clearOrphanReprobe(runId);
+        this.store.appendEvent(runId, { type: 'lifecycle', message: "the interrupted worker's processes are gone; its execution was finalized" });
+        removeAgentTmpDir(this.dataDir, runId);
+        return true;
+    }
+    orphanProbes = new Map();
+    orphanReprobes = new Map();
+    orphanBlockers = new Map();
+    reportedOrphanBlockers = new Map();
+    // Private and overridable so tests need not wait out production cadence.
+    orphanReprobeMs = 15_000;
+    orphanReprobeLimitMs = 15 * 60_000;
+    orphanReprobeSlowMs = 60_000;
+    orphanTermGraceMs = 10_000;
+    /** #469: a survivor that dies after recovery has no other wake source (no exit callback for a
+     * process another cezar spawned). Unref'd; finalization's `run` event then lets worker waits and
+     * outcomes observe it. Fast for the first window, then slow but uncapped, so a long-lived
+     * survivor is still noticed when it exits. A tick while this manager holds the run is skipped;
+     * only terminal reasons (proof settled, generation changed, run gone, unknown record, dispose)
+     * stop it. */
+    armOrphanReprobe(runId) {
+        const armed = this.orphanedWorkerGeneration(runId);
+        if (this.disposed || this.orphanReprobes.has(runId) || !armed)
+            return;
+        const slowAfter = Date.now() + this.orphanReprobeLimitMs;
+        const schedule = () => {
+            const timer = setTimeout(() => {
+                const orphan = this.orphanState(runId);
+                if (orphan.state === 'busy')
+                    return schedule();
+                if (orphan.state !== 'orphan' || orphan.generation !== armed.generation || this.settleOrphanedWorkerExecution(runId))
+                    this.clearOrphanReprobe(runId);
+                else
+                    schedule();
+            }, Date.now() < slowAfter ? this.orphanReprobeMs : this.orphanReprobeSlowMs);
+            timer.unref?.();
+            this.orphanReprobes.set(runId, timer);
+        };
+        schedule();
+    }
+    clearOrphanReprobe(runId) {
+        const timer = this.orphanReprobes.get(runId);
+        if (timer)
+            clearTimeout(timer);
+        this.orphanReprobes.delete(runId);
+    }
+    /** Why the last destroy could not prove termination (#469), and whether that differs from the
+     * reason last reported, so a retry loop appends one lifecycle event per distinct blocker set. */
+    takeWorkerTerminationBlocker(runId) {
+        const blocker = this.orphanBlockers.get(runId);
+        if (!blocker)
+            return undefined;
+        const key = JSON.stringify(blocker);
+        const changed = this.reportedOrphanBlockers.get(runId) !== key;
+        this.reportedOrphanBlockers.set(runId, key);
+        return { blocker, changed };
+    }
+    /** Continue over a live orphan names what still runs; other refusals keep the generic text. */
+    orphanAdmissionRefusal(runId) {
+        const orphan = this.orphanedWorkerGeneration(runId, true);
+        const probe = orphan && this.orphanProbes.get(runId);
+        if (!probe || probe.generation !== orphan.generation)
+            return undefined;
+        if (probe.probe.controller !== undefined)
+            return `the worker is still controlled by a live cezar (pid ${probe.probe.controller})`;
+        const pid = probe.probe.pids[0];
+        return pid === undefined ? undefined : `a process of the previous execution is still running (pid ${pid})`;
+    }
+    /** Destroy only (#469): signal the recorded, token-verified survivors of a generation whose
+     * controller is proven dead, then finalize on `gone`. Scan-only processes are never signalled. */
+    async reapOrphanedWorker(runId, timeoutMs) {
+        const deadline = Date.now() + Math.min(30_000, Math.max(0, Number.isFinite(timeoutMs) ? timeoutMs : 0));
+        this.orphanBlockers.delete(runId);
+        const initial = this.orphanState(runId);
+        if (initial.state === 'unknown') {
+            this.orphanBlockers.set(runId, { kind: 'unreadable' });
+            return;
+        }
+        const orphan = this.orphanedWorkerGeneration(runId);
+        if (!orphan || this.settleOrphanedWorkerExecution(runId, { fresh: true }))
+            return;
+        const same = () => this.orphanedWorkerGeneration(runId)?.generation === orphan.generation;
+        // >= 500 ms: every probe may be a synchronous darwin `lsof`.
+        const pause = () => new Promise(resolve => setTimeout(resolve, Math.max(0, Math.min(500, deadline - Date.now()))));
+        // A live controller is another cezar's: nothing to signal or wait for. Otherwise signal only
+        // recorded, token-verified survivors (none for a legacy record-less generation), then wait
+        // out every other holder until the deadline; the scan's processes are never signalled.
+        if (!orphan.record || !recordedProcessLive(orphan.record.controller)) {
+            if (orphan.record) {
+                const targets = orphan.record.processes.filter(entry => entry.startToken !== undefined && recordedProcessLive(entry));
+                const signal = (name) => {
+                    for (const entry of targets) {
+                        if (!same())
+                            return;
+                        // Re-verified immediately before every signal, exactly: a reused PID is never touched.
+                        if (processStartToken(entry.pid) === entry.startToken)
+                            try {
+                                process.kill(entry.pid, name);
+                            }
+                            catch { /* already gone */ }
+                    }
+                };
+                signal('SIGTERM');
+                const killAt = Math.min(deadline, Date.now() + this.orphanTermGraceMs);
+                while (Date.now() < killAt && targets.some(recordedProcessLive))
+                    await pause();
+                if (targets.some(recordedProcessLive))
+                    signal('SIGKILL');
+            }
+            while (!this.settleOrphanedWorkerExecution(runId, { fresh: true }) && Date.now() < deadline && same())
+                await pause();
+        }
+        if (!same())
+            return;
+        const probe = this.orphanProbes.get(runId)?.probe;
+        if (probe?.controller !== undefined)
+            this.orphanBlockers.set(runId, { kind: 'controller', pid: probe.controller });
+        else if (probe?.pids.length)
+            this.orphanBlockers.set(runId, { kind: 'processes', pids: probe.pids });
+    }
+    /** Best effort: a failed write leaves the working-directory scan as this process's evidence. */
+    recordWorkerProcess(runId, pid) {
+        const generation = this.executions.get(runId)?.generation;
+        if (!generation)
+            return;
+        try {
+            if (!this.store.appendWorkerProcess(runId, generation, pid))
+                console.warn(`[cez] worker ${runId} process record unavailable; relying on the working-directory scan`);
+        }
+        catch {
+            console.warn(`[cez] worker ${runId} process record write failed; relying on the working-directory scan`);
+        }
+    }
     trackTurnEnd(runId, text) {
         const task = this.recordTurnEnd(runId, text);
         const turns = this.executions.get(runId)?.turns;
@@ -532,12 +739,16 @@ export class RunManager {
         this.cancel(runId);
         return { workerId: runId, state: !this.executions.has(runId) && this.store.readWorkerExecution(runId)?.phase === 'complete' ? 'terminated' : 'stopping' };
     }
-    async awaitRunTermination(runId, timeoutMs) {
+    async awaitRunTermination(runId, timeoutMs, opts = {}) {
         const execution = this.executions.get(runId);
         if (!execution) {
             const generation = this.finalizedWorkers.get(runId);
             if (!this.disposed && generation)
                 this.persistWorkerCompletion(runId, generation);
+            else if (!generation && opts.reapOrphans)
+                await this.reapOrphanedWorker(runId, timeoutMs);
+            else if (!generation)
+                this.settleOrphanedWorkerExecution(runId);
             return this.store.readWorkerExecution(runId)?.phase === 'complete';
         }
         if (this.disposed)
@@ -717,6 +928,11 @@ export class RunManager {
         this.ciResources.release();
         this.delegationProvisioner = undefined;
         this.finalizedWorkers.clear();
+        for (const runId of [...this.orphanReprobes.keys()])
+            this.clearOrphanReprobe(runId);
+        this.orphanProbes.clear();
+        this.orphanBlockers.clear();
+        this.reportedOrphanBlockers.clear();
         for (const settle of this.terminationWaiters)
             settle();
         this.store.off('run', this.onDelegationRun);
@@ -1480,11 +1696,11 @@ export class RunManager {
                         try {
                             this.beginWorkerExecution(candidate);
                         }
-                        catch {
+                        catch (error) {
                             this.queue.splice(next, 1);
                             this.pendingJobs.delete(candidate);
                             this.pendingContinuations.delete(candidate);
-                            this.store.updateRun(candidate, { status: 'failed', error: 'worker execution checkpoint unavailable', finishedAt: new Date().toISOString() });
+                            this.store.updateRun(candidate, { status: 'failed', error: admissionError(error, 'worker execution checkpoint unavailable'), finishedAt: new Date().toISOString() });
                             continue;
                         }
                     }
@@ -1722,6 +1938,11 @@ export class RunManager {
                         this.store.appendEvent(run.id, { type: 'note', message: `replaying ${replayed.length} message${replayed.length === 1 ? '' : 's'} accepted before the restart but not read` });
                 }
             }
+            // #469: a generation whose processes died with the old controller is finalized first, so it
+            // re-launches through the ordinary Continue path and its scratch is not retained.
+            for (const run of this.store.listRuns())
+                if (run.delegation?.role === 'worker' && !this.settleOrphanedWorkerExecution(run.id))
+                    this.armOrphanReprobe(run.id);
             this.reconcileWorkerWaits();
             const live = this.store
                 .listRuns()
@@ -4503,8 +4724,8 @@ export class RunManager {
                         } };
                 this.beginWorkerExecution(runId, false);
             }
-            catch {
-                return { ok: false, error: 'original execution checkpoint unavailable' };
+            catch (error) {
+                return { ok: false, error: admissionError(error, 'original execution checkpoint unavailable') };
             }
             const content = [
                 ...(opts.text?.trim() ? [{ type: 'text', text: opts.text.trim() }] : []), ...(opts.images ?? []),
@@ -4525,8 +4746,8 @@ export class RunManager {
         try {
             this.beginWorkerExecution(runId, !deferForCapacity);
         }
-        catch {
-            return { ok: false, error: 'worker execution checkpoint unavailable' };
+        catch (error) {
+            return { ok: false, error: admissionError(error, 'worker execution checkpoint unavailable') };
         }
         // Everything that could refuse this continuation has now passed, so a pending usage-limit
         // resume is superseded either way: this IS that resume (it re-stamps its own counter), or a
@@ -5168,8 +5389,10 @@ export class RunManager {
         try {
             this.flushDeferred(runId);
             this.flushAgentInputs(runId);
-            if (session.pid !== undefined)
+            if (session.pid !== undefined) {
                 registerRunProcess(runId, session.pid);
+                this.recordWorkerProcess(runId, session.pid);
+            }
             setupComplete = true;
             const result = await session.result.finally(() => state.agentInputFlight?.settled).finally(() => this.requeueUnreadAtClose(runId, state, session));
             if (this.preserveRunAfterDisposal(runId, state))
@@ -6033,8 +6256,10 @@ export class RunManager {
         try {
             this.flushDeferred(runId);
             this.flushAgentInputs(runId);
-            if (session.pid !== undefined)
+            if (session.pid !== undefined) {
                 registerRunProcess(runId, session.pid);
+                this.recordWorkerProcess(runId, session.pid);
+            }
             setupComplete = true;
             const result = await session.result.finally(() => state.agentInputFlight?.settled).finally(() => this.requeueUnreadAtClose(runId, state, session));
             if (this.isDisposedDelegatedRun(runId))