@detergent-software/conductor 3.5.0-dev.9 → 3.5.1-dev.1

package/build/manager.js

@@ -117,8 +117,11 @@ const TRANSCRIPT_SCAN_MS = 60_000;
 // The process scan for `claude attach` clients runs at most this often, and only while a worker is
 // attached: on Windows it takes about a second (ATK-520).
 const CLIENT_SCAN_MS = 30_000;
-// This many scans in a row without the client, once one had found it, mean the tab closed.
+// This many scans in a row without the client, once one had found it, mean the tab closed. So do
+// as many with no client ever found for a tab the daemon opened, counted from CLIENT_START_MS after
+// the attach, so its client has had time to start (ATK-878).
 const CLIENT_MISSES = 2;
+const CLIENT_START_MS = 15_000;
 // A compaction with no boundary in the transcript this long after it started, its turn over, failed.
 const COMPACT_MINUTES = 10;
 // A resumed worker with no reply in its transcript this long after the resume, its turn over, parks
@@ -133,6 +136,8 @@ const SWEPT = ['running', 'parked', 'stopped-unexpectedly'];
 // listing only shows whether someone woke a parked or stopped session outside the daemon, and gates
 // a parked worker's compaction, both of which can wait this long.
 const IDLE_LISTING_MS = 60_000;
+// The events of a parked worker's session running again, as the inbox reads them.
+const RUNNING_AGAIN = new Set(['worker.resumed', 'worker.revived', 'worker.running-again']);
 // The resume text: facts only, as a user turn (protocol 2's wake text), such as
 // `[conductor] event: time 2026-10-10T22:05:00.000Z reached 22:05`. The worker re-reads the source.
 // Each fact once, at its first wait's time: a merge resolves a PR's merged, checks and comment
@@ -155,6 +160,9 @@ export function sessionManager(deps) {
     const busy = new Set();
     const tasks = new Set();
     const stalls = new Map();
+    // By worker id, apart from stalls, which a compaction's re-park starts afresh (in memory: a restart
+    // checks each long park again). Dropped when the worker runs again or retires, as its item is.
+    const parkHeld = new Map();
     const transcripts = new Map();
     const transcriptScans = new Map();
     // worker id → a session id the listing shows for it that it hasn't followed yet, since when
@@ -174,7 +182,8 @@ export function sessionManager(deps) {
     const held = new Set();
     // Workers whose compaction failed: not tried again while the daemon runs (their inbox item says so).
     const uncompactable = new Set();
-    // worker id → the attach (its attachedAt) a retire was last refused for, so each is logged once.
+    // worker id → the attach (its attachedAt) a retire was last refused for, and the reasons logged
+    // for it, so each is logged once per attach.
     const attachRefusals = new Map();
     // Every session the conductor starts loads its plugin, and a resume whose saved --plugin-dir went
     // missing runs on without it, silently (ATK-522). The mod reports each session start (pluginSeen),
@@ -208,6 +217,10 @@ export function sessionManager(deps) {
             subject: worker.item,
         };
         store.appendEvent(entry);
+        // What resolves a long park's held-process item (inbox.ts) ends its check too. A compaction's
+        // resume doesn't: the item stays open through it.
+        if (RUNNING_AGAIN.has(kind) || (kind === 'worker.attached' && data.resumed === true))
+            parkHeld.delete(worker.id);
     };
     // Its session was (re)started at `since`, before the claude call: its report must come after.
     // Only a worker launched with the plugin keeps it in its saved options: one launched before it (by
@@ -357,6 +370,91 @@ export function sessionManager(deps) {
             ? { error: `couldn't stop ${processesText(held)}`, held, ok: false, step: name, ...done }
             : { ok: true, step: name, ...done };
     };
+    // A worker parked stopProcessesAfterParkMinutes or longer has its worktree's processes stopped, as a
+    // retire stops them, once per park (ATK-876): a dev server it left up holds its memory through the
+    // wait. Its ports stay assigned, so a server its work skill starts again serves on them; the resume
+    // starts nothing. The clock is the park, which a compaction's re-park starts again; after a daemon
+    // restart, with no park seen, it is the last stop. A listing that fails is tried again
+    // PARK_RETRY_MS later. Processes that wouldn't stop are listed again until they have exited
+    // (recheckHeld), through the re-parks of a compaction too.
+    const stopLongParked = async (worker) => {
+        const after = deps.stopProcessesAfterParkMinutes;
+        if (after <= 0 || worker.worktree === null || worker.stoppedAt === null)
+            return;
+        const state = stalls.get(worker.id) ?? {};
+        const minutes = (now().getTime() - (state.parkedAt ?? Date.parse(worker.stoppedAt))) / 60_000;
+        const check = state.parkProcesses;
+        if (minutes < after || check === 'checked' || (check !== undefined && now().getTime() < check.retryAt)) {
+            const held = parkHeld.get(worker.id);
+            if (held !== undefined && now().getTime() >= held.retryAt)
+                await recheckHeld(worker, worker.worktree, held, minutes);
+            return;
+        }
+        state.parkProcesses = 'checked';
+        stalls.set(worker.id, state);
+        const done = await stopProcesses(worker.worktree, worktreePorts(store, worker.worktree));
+        if (done.note?.includes('not checked'))
+            log.log('warn', 'parked worker processes not all checked', { note: done.note, worker: worker.id });
+        if (done.stopped === undefined && done.held === undefined) {
+            // A listing that failed outright (no `none`) is tried again; none found waits for the retire.
+            if (done.note?.startsWith('not checked'))
+                state.parkProcesses = { retryAt: now().getTime() + PARK_RETRY_MS };
+            return;
+        }
+        if (done.held !== undefined) {
+            // An earlier park's held ones that this pass stopped or no longer lists stay until a check sees
+            // them gone, since their item still names them: a compaction's re-park checks again.
+            const held = done.held;
+            const earlier = (parkHeld.get(worker.id)?.held ?? []).filter((p) => !held.some((h) => h.pid === p.pid));
+            parkHeld.set(worker.id, { held: [...held, ...earlier], retryAt: now().getTime() + PARK_RETRY_MS });
+        }
+        event(worker, 'worker.processes-stopped', {
+            minutes: Math.round(minutes),
+            ...(done.note === undefined ? {} : { note: done.note }),
+            stopped: done.stopped ?? [],
+            ...(done.held === undefined ? {} : { error: done.error, held: done.held }),
+        });
+    };
+    // A long park's processes that wouldn't stop, listed again every PARK_RETRY_MS (ATK-880): they can
+    // exit on their own, or the operator can end them, while the worker stays parked, and their item's
+    // kill commands would then name PIDs the OS may give to an unrelated process. This stops nothing.
+    // One has exited when its PID is no longer alive, or when a complete listing of the worktree no
+    // longer shows one that runs from it: the listing is the worktree's, so a reused PID isn't in it.
+    // One found by its port or its parent can still run unlisted (the port probe failed, the parent
+    // exited), so it needs its PID gone. Those that exited are logged as worker.processes-exited, with
+    // those still held, and the inbox raises the item again for the rest. A listing that fails changes
+    // nothing, nor does one the worker's resume or retire overtook.
+    const recheckHeld = async (worker, worktree, check, minutes) => {
+        check.retryAt = now().getTime() + PARK_RETRY_MS;
+        let found;
+        try {
+            found = await deps.processes.find(worktree, worktreePorts(store, worktree));
+        }
+        catch (error) {
+            log.log('warn', "parked worker's held processes not checked again", {
+                error: messageOf(error),
+                worker: worker.id,
+            });
+            return;
+        }
+        if (parkHeld.get(worker.id) !== check)
+            return;
+        const listed = new Set(found.processes.map((p) => p.pid));
+        const complete = found.unseen.length === 0;
+        const exited = [];
+        const still = [];
+        for (const p of check.held) {
+            const gone = !alive(p.pid) || (!listed.has(p.pid) && complete && p.why === 'path');
+            (gone ? exited : still).push(p);
+        }
+        if (exited.length === 0)
+            return;
+        if (still.length === 0)
+            parkHeld.delete(worker.id);
+        else
+            check.held = still;
+        event(worker, 'worker.processes-exited', { exited, held: still, minutes: Math.round(minutes) });
+    };
     // What still holds the worktree after a removal failed: its processes as listed again, or none
     // when the OS doesn't say (on Windows, a process whose working folder alone is in it).
     const holders = async (worktree, ports) => deps.processes
@@ -535,7 +633,7 @@ export function sessionManager(deps) {
             return;
         }
         const parked = update(worker, { status: 'parked', stoppedAt: now().toISOString() });
-        stalls.delete(worker.id);
+        stalls.set(worker.id, { parkedAt: now().getTime() });
         starting.delete(worker.id);
         replies.delete(worker.id);
         event(parked, 'worker.parked', { waits: waits.map((w) => w.condition) }, decision.rule);
@@ -696,8 +794,10 @@ export function sessionManager(deps) {
     };
     // Once the process scan has seen the worker's `claude attach` client and then missed it
     // CLIENT_MISSES times in a row, and its turn is over, the tab closed: the worker is the daemon's
-    // again (ATK-520). A client seen still doesn't hold it past the releases below: a tab left open
-    // with nobody in it would hold its waits as before ATK-516, and a park closes the tab.
+    // again (ATK-520). So it is when a tab the daemon opened is closed before any scan found its
+    // client (ATK-878; `watchClient` says when). A client seen still doesn't hold it past the
+    // releases below: a tab left open with nobody in it would hold its waits as before ATK-516, and
+    // a park closes the tab.
     // Attached means "the operator may be in it" only while the session is
     // in use. Once its turn has been over `idleMinutes`, the worker is the daemon's again, and its
     // waits park and resume it as usual. A resolved wait takes it back at once when the transcript
@@ -735,23 +835,44 @@ export function sessionManager(deps) {
     };
     // What a client of the worker's session attaches by: its short id, session id or name.
     const clientNames = (worker) => [worker.handle, worker.sessionId, worker.name].filter((name) => name !== null);
+    const clientWatch = (opened) => ({
+        misses: 0,
+        names: [],
+        opened,
+        scan: 0,
+        seen: false,
+    });
     // Each scan since the attach counts once. One that finds the client marks it seen and starts the
-    // misses over; one that doesn't, after it was seen, is a miss. So a tab that never opened (its
-    // terminal failed, and the operator runs `claude attach` by hand) or hasn't started its client yet
-    // is no closed tab. The names include every one the worker had since the attach: a client keeps
-    // the id it attached by when the worker follows a new one. A scan that couldn't read the
-    // processes tells nothing.
-    const watchClient = (worker, state) => {
-        if (worker.attachedAt === null)
-            return;
-        const watch = (state.attachedClient ??= { misses: 0, names: [], scan: 0, seen: false });
+    // misses over; one that doesn't, after it was seen, is a miss. A tab the daemon opened can close
+    // before any scan finds its client (ATK-878), so for one the scans never saw, a scan from
+    // CLIENT_START_MS after the attach without it is a miss too: the grace lets its client start. A
+    // tab that didn't open (its terminal failed, and the operator may run `claude attach` by hand), a
+    // session woken outside the daemon, or an attach from before a daemon restart is no closed tab
+    // until a scan has seen its client. The names include every one the worker had since the attach:
+    // a client keeps the id it attached by when the worker follows a new one. A scan that couldn't
+    // read the processes tells nothing.
+    const stallOf = (workerId) => {
+        const state = stalls.get(workerId) ?? {};
+        stalls.set(workerId, state);
+        return state;
+    };
+    // The attached worker's client watch, with the names it has now.
+    const watchOf = (worker, state) => {
+        const watch = (state.attachedClient ??= clientWatch(false));
         for (const name of clientNames(worker))
             if (!watch.names.includes(name))
                 watch.names.push(name);
+        return watch;
+    };
+    const watchClient = (worker, state) => {
+        if (worker.attachedAt === null)
+            return;
+        const watch = watchOf(worker, state);
         const scan = clientScan;
         if (scan === null || scan.targets === null || scan.scan === watch.scan)
             return;
-        if (scan.at < Date.parse(worker.attachedAt))
+        const attachedAt = Date.parse(worker.attachedAt);
+        if (scan.at < attachedAt)
             return;
         const { targets } = scan;
         watch.scan = scan.scan;
@@ -759,9 +880,15 @@ export function sessionManager(deps) {
             watch.seen = true;
             watch.misses = 0;
         }
-        else if (watch.seen)
+        else if (watch.seen || (watch.opened && scan.at >= attachedAt + CLIENT_START_MS))
             watch.misses += 1;
     };
+    const detach = (worker, data) => {
+        forgetAttach(worker.id);
+        const released = update(worker, { attachedAt: null });
+        event(released, 'worker.detached', data);
+        return released;
+    };
     const releaseAttached = (worker, session, resolved) => {
         const state = stalls.get(worker.id) ?? {};
         stalls.set(worker.id, state);
@@ -770,23 +897,27 @@ export function sessionManager(deps) {
             state.attachedIdleSince = undefined;
             return worker;
         }
-        const detach = (data) => {
-            forgetAttach(worker.id);
-            const released = update(worker, { attachedAt: null });
-            event(released, 'worker.detached', data);
-            return released;
-        };
         // Nobody has it open: its waits park and resume it as usual, with no idle spell first.
-        if ((state.attachedClient?.misses ?? 0) >= CLIENT_MISSES)
-            return detach({ reason: 'tab closed' });
         const attachedAt = worker.attachedAt;
+        const client = state.attachedClient;
+        if (client && client.misses >= CLIENT_MISSES) {
+            // A turn since the attach with no client ever found: the tab may be in use where the scan can't
+            // see its client (a `ps` that shows only a process title), or the attach came mid-turn. Either
+            // way the worker is held as before ATK-878, until the idle release.
+            if (!client.seen && attachedAt !== null && turnSince(worker, Date.parse(attachedAt))) {
+                client.opened = false;
+                client.misses = 0;
+            }
+            else
+                return detach(worker, { reason: 'tab closed', seen: client.seen });
+        }
         if (resolved.length > 0 && attachedAt !== null) {
             const since = Date.parse(attachedAt);
             // Once seen, a turn since the attach stays seen: the transcript isn't read again.
             state.attachedUsed ||= usedSince(worker, since);
             const fresh = resolved.filter((w) => w.resolvedAt !== null && Date.parse(w.resolvedAt) > since);
             if (fresh.length > 0 && !state.attachedUsed) {
-                return detach({ reason: 'wait resolved', waits: fresh.map((w) => w.condition) });
+                return detach(worker, { reason: 'wait resolved', waits: fresh.map((w) => w.condition) });
             }
             if (!state.attachedHeld) {
                 state.attachedHeld = true;
@@ -797,7 +928,36 @@ export function sessionManager(deps) {
         const minutes = minutesSince(state.attachedIdleSince);
         if (minutes < config.idleMinutes)
             return worker;
-        return detach({ idleMinutes: Math.floor(minutes) });
+        return detach(worker, { idleMinutes: Math.floor(minutes) });
+    };
+    // An attached worker's tab, for a retire (ATK-890): `closed` when a scan no longer finds a client
+    // that one found before, or, for a tab the daemon opened that no scan found, by the sweep's
+    // CLIENT_MISSES (its client may start slowly); `open` when the client is found; `unseen` when no
+    // scan ever found it and the session took a turn since the attach (held as the sweep holds it,
+    // ATK-878) or the daemon didn't open the tab; `unknown` when the processes couldn't be read, or
+    // the tab may still be starting its client. The operator's retire scans afresh; the scheduler's
+    // retries reuse a readable scan from after the attach under CLIENT_SCAN_MS old, as the sweep does.
+    const attachedTab = async (worker, attachedAt, reuse) => {
+        const since = Date.parse(attachedAt);
+        const recent = clientScan !== null &&
+            clientScan.targets !== null &&
+            clientScan.at >= since &&
+            now().getTime() - clientScan.at < CLIENT_SCAN_MS;
+        if (!reuse || !recent)
+            await scanClients();
+        const state = stallOf(worker.id);
+        watchClient(worker, state);
+        const watch = watchOf(worker, state);
+        const targets = clientScan?.targets ?? null;
+        if (targets === null)
+            return 'unknown';
+        if (watch.names.some((name) => targets.has(name)))
+            return 'open';
+        if (watch.seen)
+            return 'closed';
+        if (!watch.opened || turnSince(worker, since))
+            return 'unseen';
+        return watch.misses >= CLIENT_MISSES ? 'closed' : 'unknown';
     };
     const sweepWorker = async (worker) => {
         let current = worker;
@@ -893,10 +1053,13 @@ export function sessionManager(deps) {
         // (ATK-519): the retire, tried again, takes it on.
         if (waits.length === 0)
             return;
-        if (resolved.length > 0)
+        if (resolved.length > 0) {
             await resume(current, resolved, session);
-        else
-            await compact(current, session);
+            return;
+        }
+        // Not with a resolved wait: the resume about to happen may want its server.
+        await stopLongParked(current);
+        await compact(current, session);
     };
     // The transcript is under the slug of the session's cwd, which is usually its worktree. Failing
     // that, every project directory is scanned, at most once a minute per worker.
@@ -939,13 +1102,12 @@ export function sessionManager(deps) {
         const last = lastTurnOf(worker);
         return last === undefined || (last !== null && last.getTime() > since);
     };
-    // No transcript, no evidence of a turn.
-    const tookTurnSinceStop = (worker) => {
-        if (worker.stoppedAt === null)
-            return true;
+    // A turn since `since` (ms). No transcript, no evidence of a turn.
+    const turnSince = (worker, since) => {
         const last = lastTurnOf(worker);
-        return last !== null && last !== undefined && last.getTime() > Date.parse(worker.stoppedAt);
+        return last !== null && last !== undefined && last.getTime() > since;
     };
+    const tookTurnSinceStop = (worker) => worker.stoppedAt === null || turnSince(worker, Date.parse(worker.stoppedAt));
     // Whether a resumed worker still waits for the model to answer its event. A reply after the resume
     // delivers the event's waits; so does no transcript to tell, as before ATK-863.
     // An attached one is the operator's, who has the event in front of them: delivered too, so the
@@ -1187,6 +1349,11 @@ export function sessionManager(deps) {
                         current = update(current, { attachedAt: now().toISOString() });
                         forgetAttach(current.id);
                     }
+                    // Seen: once it's gone, the tab closed, also for a retire before the next sweep's scan
+                    // (ATK-890). The watch skips this scan, which can start before the attach.
+                    const watch = watchOf(current, stallOf(current.id));
+                    watch.seen = true;
+                    watch.misses = 0;
                     event(current, 'worker.attached', { resumed, tabOpen: true });
                     return { already: true, hint: tabs.hint(window), resumed, tab: null, worker: current };
                 }
@@ -1214,6 +1381,12 @@ export function sessionManager(deps) {
                 current = resumed ? nowRunning(current, patch) : update(current, { ...patch, status: 'running' });
                 forgetAttach(current.id);
                 const tab = await tabs.open({ cwd: current.worktree ?? cloneOf(current), handle, title: current.name, window });
+                if (tab === null) {
+                    // Its client is due: a scan that never finds it means the tab closed (ATK-878).
+                    const state = stalls.get(current.id) ?? {};
+                    stalls.set(current.id, state);
+                    state.attachedClient = clientWatch(true);
+                }
                 event(current, 'worker.attached', { resumed, ...(tab ? { tabError: tab } : {}) });
                 if (started !== null)
                     expectPlugin(current, started);
@@ -1262,7 +1435,7 @@ export function sessionManager(deps) {
                 .find((w) => w.sessionId === sessionId || newSessionIds.get(w.id)?.sessionId === sessionId) ?? null);
         },
         retire: async (workerId, { cause, force = false, keep = false } = {}) => {
-            const worker = activeWorker(workerId);
+            let worker = activeWorker(workerId);
             if (busy.has(worker.id) || worker.status === 'launching') {
                 return {
                     outcome: 'refused',
@@ -1282,22 +1455,37 @@ export function sessionManager(deps) {
                     });
                     return { outcome: 'refused', reason, transient, ...(work ? { work } : {}), worker };
                 };
-                // The operator has its session open in a tab (ATK-519). The scheduler tries again every tick,
-                // so it is logged once per attach. Only a running worker is attached: one whose session
-                // stopped unexpectedly keeps the attach it had.
+                // The operator has its session open in a tab (ATK-519). A tab a scan finds closed is released
+                // here and the retire goes on (ATK-890). The scheduler tries again every tick, so each refusal
+                // is logged once per attach. Only a running worker is attached: one whose session stopped
+                // unexpectedly keeps the attach it had.
                 const attachedAt = worker.status === 'running' ? worker.attachedAt : null;
                 if (attachedAt !== null && !force) {
-                    const reason = 'attached: retire once its tab is released, or retire --force';
-                    if (attachRefusals.get(worker.id) !== attachedAt) {
-                        attachRefusals.set(worker.id, attachedAt);
-                        event(worker, 'worker.retire-refused', {
-                            attached: true,
-                            reason,
-                            transient: true,
-                            ...(cause === undefined ? {} : { cause }),
-                        });
+                    const tab = await attachedTab(worker, attachedAt, cause !== undefined);
+                    if (tab === 'closed') {
+                        const seen = stalls.get(worker.id)?.attachedClient?.seen ?? false;
+                        worker = detach(worker, { reason: 'tab closed', retire: true, seen });
+                    }
+                    else {
+                        const reason = tab === 'unseen'
+                            ? 'attached, but no scan has found its tab: retire --force retires it'
+                            : 'attached: retire once its tab is released, or retire --force';
+                        const logged = attachRefusals.get(worker.id);
+                        if (logged?.attachedAt !== attachedAt)
+                            attachRefusals.set(worker.id, { attachedAt, reasons: new Set() });
+                        const reasons = attachRefusals.get(worker.id)?.reasons;
+                        if (reasons && !reasons.has(reason)) {
+                            reasons.add(reason);
+                            event(worker, 'worker.retire-refused', {
+                                attached: true,
+                                reason,
+                                tab,
+                                transient: true,
+                                ...(cause === undefined ? {} : { cause }),
+                            });
+                        }
+                        return { attached: true, outcome: 'refused', reason, transient: true, worker };
                     }
-                    return { attached: true, outcome: 'refused', reason, transient: true, worker };
                 }
                 attachRefusals.delete(worker.id);
                 // Fresh: the last sweep's listing can be a minute old. Null when it couldn't be read.
@@ -1408,6 +1596,7 @@ export function sessionManager(deps) {
                 if (!worker.probe)
                     releaseLocks(store, worker.item, 'worker retired');
                 stalls.delete(worker.id);
+                parkHeld.delete(worker.id);
                 starting.delete(worker.id);
                 replies.delete(worker.id);
                 resends.delete(worker.id);