@platless/axom-cli 0.1.0 → 0.5.9

package/dist/workstation/keepers/policy.js

@@ -0,0 +1,576 @@
+// Daemon-side: what the Workstations page sees of the keepers (heartbeat `keepers[]`, spec §8.1) and the keeper REPLACEMENT POLICY
+// (spec §8.3, Task 26). Not a keeper-side module: keeper processes never import it. Daemon business modules (acp/, agentSessions,
+// update/) are reached only through the injected sides below.
+//
+// replacement of a generation (computed here, locally, after the CLI that brought the new build was installed — the server only
+// PREDICTS it from the release manifest before an update, Task 27):
+//   none       it is the current build's generation
+//   available  another build, its protocol inside the current build's `daemonProtocol` window: NOTHING happens by itself (an older
+//              keeper-terminal even keeps serving new work while the current build is not running, keepers/terminal/generations.ts)
+//   required   its protocol is outside that window
+//   mandatory  the current build's manifest says `mandatory`
+//
+// Policy, acted on at most once per (kind, generation) — a decision is persisted (`<configDir>/keepers/policy.json`), so neither a
+// daemon restart nor a user's cancel brings an automatic one back:
+//   terminal   required/mandatory: queued `when-idle` — new work goes to the current build; the older generation is stopped once it
+//              has no open PTY and no running exec (`ptyOpen == 0 && execRunning == 0`). The user may queue it for an `available`
+//              one (replace-when-idle), stop it now (replace-now: its PTYs and commands end, listed in `affected`) or cancel.
+//   acp        required/mandatory: rolling — new sessions already go to the current generation (Task 21); every session held on an
+//              older one gets a persistent intent to move there when idle (`moveOffOlderKeeperGenerations`, Task 24); an older
+//              generation that hosts no adapter any more is stopped (`retireIdleGenerations`). Never for `available` unless asked.
+//              replace-now stops the older generations at once (their in-flight turns end; the sessions reopen through session/load).
+//   legacy     the legacy tools keeper (spec §9) retires by itself (keepers/legacy.ts); it is reported `required`/`when-idle`, and
+//              replace-now stops it at once.
+import { promises as fs } from 'node:fs';
+import path from 'node:path';
+import { keeperOneShot, readKeeperToken } from './kit/client.js';
+import { inProtocolWindow, isProtocolWindow, protocolWindow } from './kit/protocol.js';
+import { TERMINAL_PROTOCOL_VERSION } from './terminal/protocol.js';
+import { ACP_PROTOCOL_VERSION } from './acp/protocol.js';
+import { KEEPER_AFFECTED_MAX_ITEMS, KEEPER_LABEL_MAX_CHARS, KEEPER_STATUS_MAX_ENTRIES, KEEPER_STATUS_MAX_ITEMS, } from '../protocol/management.js';
+/** How often the policy looks at the keepers (and acts on queued replacements). */
+export const KEEPER_POLICY_TICK_MS = 30_000;
+/** One keeper's sample (info + activity on fresh connections) may take this long; past it the keeper is `unavailable`. */
+export const KEEPER_SAMPLE_TIMEOUT_MS = 1000;
+/** A sample this young is reused (the heartbeat and the policy tick share it). */
+export const KEEPER_SAMPLE_REUSE_MS = 2000;
+/** Decisions kept in the policy file (oldest dropped first). */
+export const KEEPER_DECISIONS_LIMIT = 64;
+/** `keeper_control` results kept for a repeated commandId. */
+export const KEEPER_CONTROL_RESULTS_LIMIT = 64;
+/** The policy file, under `<configDir>/keepers/`. */
+export const KEEPER_POLICY_FILE = 'policy.json';
+/** What replacing generation `gen` takes (see the file comment). `protocolVersion` null = unknown (not reachable): `available`. */
+export function replacementOf(input) {
+    if (input.gen === input.current)
+        return 'none';
+    if (input.protocolVersion !== null && !inProtocolWindow(input.window, input.protocolVersion))
+        return 'required';
+    return input.mandatory ? 'mandatory' : 'available';
+}
+/** Whether a replacement of this kind happens without the user asking (spec §8.3): never for `available`. */
+export const replacesByItself = (replacement) => replacement === 'required' || replacement === 'mandatory';
+/** A keeper-terminal generation may be stopped by a queued replacement: no open PTY and no running exec. */
+export function terminalIdle(metrics) {
+    return (metrics.ptyOpen ?? 0) === 0 && (metrics.execRunning ?? 0) === 0;
+}
+const clip = (text, max = KEEPER_LABEL_MAX_CHARS) => (text.length > max ? text.slice(0, max) : text);
+/** A label is a name, never a command line: the first word, without a path. */
+export function nameLabel(text, fallback) {
+    const first = (text ?? '').trim().split(/\s+/)[0] ?? '';
+    const base = first.includes('/') ? path.basename(first) : first;
+    return clip(base || fallback);
+}
+/** A shared keeper collection that does not finish within this is abandoned (its callers fail; the next call starts anew). */
+export const KEEPER_COLLECT_TIMEOUT_MS = 5000;
+export class KeeperSampleTimeoutError extends Error {
+    constructor(timeoutMs) { super(`Keepers could not be sampled within ${timeoutMs} ms`); this.name = 'KeeperSampleTimeoutError'; }
+}
+/**
+ * The default probe: `keeper.info` and `keeper.activity`, each on its own fresh connection (window-exempt methods; the hello
+ * accepts any protocol, so a keeper outside the window is still described). Starts nothing.
+ */
+export function oneShotProbe(pathsFor) {
+    return async (kind, gen, timeoutMs) => {
+        const paths = pathsFor(kind, gen);
+        const token = await readKeeperToken(paths.tokenFile);
+        const [info, activity] = await Promise.all([
+            keeperOneShot(paths.socket, 'keeper.info', {}, timeoutMs, { kind, token }),
+            keeperOneShot(paths.socket, 'keeper.activity', {}, timeoutMs, { kind, token }),
+        ]);
+        if (info.gen !== gen || info.kind !== kind)
+            throw new Error('keeper identity mismatch');
+        return { hello: info, draining: info.draining === true, activity: activity };
+    };
+}
+/** `keeper.drain` on a fresh connection (window-exempt). Starts nothing. */
+export function oneShotDrain(pathsFor, timeoutMs = 2000) {
+    return async (kind, gen) => {
+        const paths = pathsFor(kind, gen);
+        await keeperOneShot(paths.socket, 'keeper.drain', {}, timeoutMs, { kind, token: await readKeeperToken(paths.tokenFile) });
+    };
+}
+const decisionKey = (kind, gen) => `${kind}:${gen}`;
+export class KeeperPolicy {
+    options;
+    decisions = new Map();
+    loaded;
+    sample;
+    collecting;
+    results = new Map();
+    /** Policy actions run one at a time (decisions, stops). */
+    acting = Promise.resolve();
+    now;
+    log;
+    constructor(options) {
+        this.options = options;
+        this.now = options.now ?? Date.now;
+        this.log = options.log ?? (() => { });
+    }
+    // ---- decisions (persisted) --------------------------------------------------------------------------------------------------
+    get file() { return path.join(this.options.dir, KEEPER_POLICY_FILE); }
+    load() {
+        this.loaded ??= (async () => {
+            try {
+                const value = JSON.parse(await fs.readFile(this.file, 'utf8'));
+                if (value.v === 1 && value.decisions && typeof value.decisions === 'object') {
+                    for (const [key, decision] of Object.entries(value.decisions)) {
+                        if (decision && ['when-idle', 'cancelled', 'done'].includes(decision.mode) && ['auto', 'user'].includes(decision.by))
+                            this.decisions.set(key, decision);
+                    }
+                }
+            }
+            catch { /* none yet, or unreadable: start empty (an automatic decision may then be taken again — harmless) */ }
+        })();
+        return this.loaded;
+    }
+    async save() {
+        while (this.decisions.size > KEEPER_DECISIONS_LIMIT)
+            this.decisions.delete(this.decisions.keys().next().value);
+        const body = JSON.stringify({ v: 1, decisions: Object.fromEntries(this.decisions) });
+        try {
+            await fs.mkdir(this.options.dir, { recursive: true, mode: 0o700 });
+            const temp = `${this.file}.${process.pid}.${this.now()}.tmp`;
+            await fs.writeFile(temp, body, { mode: 0o600 });
+            await fs.rename(temp, this.file);
+        }
+        catch (error) {
+            this.log(`[keepers] policy file not written: ${error instanceof Error ? error.message : String(error)}`);
+        }
+    }
+    async decide(kind, gen, decision) {
+        const key = decisionKey(kind, gen);
+        this.decisions.delete(key);
+        this.decisions.set(key, decision);
+        this.sample = undefined; // a status sampled before carries the old `pending`
+        await this.save();
+    }
+    /** The decision on (kind, gen), if any (tests and diagnostics). */
+    async decisionOf(kind, gen) {
+        await this.load();
+        return this.decisions.get(decisionKey(kind, gen));
+    }
+    pendingOf(kind, gen) {
+        return this.decisions.get(decisionKey(kind, gen))?.mode === 'when-idle' ? 'when-idle' : null;
+    }
+    serialized(work) {
+        const run = this.acting.catch(() => undefined).then(work);
+        this.acting = run.catch(() => undefined);
+        return run;
+    }
+    // ---- serving generation (keeper-terminal) -----------------------------------------------------------------------------------
+    /**
+     * `TerminalGenerations.picker`: an older keeper-terminal generation keeps serving new work only while the current build does not
+     * run, its replacement is merely `available` and nobody queued it. Otherwise (or when unsure) the current build serves.
+     */
+    async terminalServing(current, older) {
+        const terminal = this.options.terminal;
+        if (!terminal)
+            return undefined;
+        await this.load();
+        if ((await terminal.live()).includes(current))
+            return undefined;
+        const manifest = await terminal.manifest();
+        const window = windowOf(manifest, TERMINAL_PROTOCOL_VERSION);
+        let best;
+        const eligible = (gen) => {
+            const decision = this.decisions.get(decisionKey('terminal', gen));
+            return !decision?.drained && decision?.mode !== 'when-idle' && decision?.mode !== 'done';
+        };
+        for (const gen of older) {
+            if (!eligible(gen))
+                continue;
+            const hello = await terminal.reachable(gen).catch(() => null);
+            // Asked again after the await: a replacement queued meanwhile wins (its stop drains and re-probes before stopping, too).
+            if (!hello || !eligible(gen))
+                continue;
+            if (replacementOf({ gen, current, protocolVersion: hello.protocolVersion, window, mandatory: manifest.mandatory === true }) !== 'available')
+                continue;
+            if (!best || hello.startedAt > best.startedAt)
+                best = { gen, startedAt: hello.startedAt };
+        }
+        return best?.gen;
+    }
+    // ---- sampling -----------------------------------------------------------------------------------------------------------
+    /**
+     * The heartbeat's `keepers` (spec §8.1). A keeper that cannot be sampled is `unavailable`; rejects only with
+     * {@link KeeperSampleTimeoutError} when the whole collection hung (the heartbeat then sends `keepers:null`).
+     */
+    async statuses() {
+        return (await this.sampled()).map((entry) => entry.status).slice(0, KEEPER_STATUS_MAX_ENTRIES);
+    }
+    sampled(fresh = false) {
+        // One collection at a time: one in flight is joined, never piled up behind every heartbeat. It has its own deadline: a step with
+        // no bound of its own (a wedged file or /proc read) fails it — every joined caller gets KeeperSampleTimeoutError, and the next call
+        // starts a new collection — so no caller (heartbeat, tick, keeper_control) ever waits on it longer than the deadline.
+        if (this.collecting)
+            return this.collecting;
+        const cached = this.sample;
+        if (!fresh && cached && this.now() - cached.at < KEEPER_SAMPLE_REUSE_MS)
+            return cached.result;
+        const timeoutMs = this.options.collectTimeoutMs ?? KEEPER_COLLECT_TIMEOUT_MS;
+        let timer;
+        const collecting = this.collect();
+        collecting.catch(() => undefined); // settling after the deadline is nobody's business
+        const result = Promise.race([
+            collecting,
+            new Promise((_, reject) => { timer = setTimeout(() => reject(new KeeperSampleTimeoutError(timeoutMs)), timeoutMs); timer.unref?.(); }),
+        ]).finally(() => clearTimeout(timer));
+        this.collecting = result;
+        const done = () => { if (this.collecting === result)
+            this.collecting = undefined; };
+        const failed = () => { done(); if (this.sample?.result === result)
+            this.sample = undefined; };
+        result.then(done, failed);
+        this.sample = { at: this.now(), result };
+        return result;
+    }
+    async collect() {
+        await this.load();
+        const [terminal, acp] = await Promise.all([this.collectTerminal(), this.collectAcp()]);
+        return [...terminal, ...acp];
+    }
+    async probe(kind, gen) {
+        try {
+            return await this.options.probe(kind, gen, this.options.sampleTimeoutMs ?? KEEPER_SAMPLE_TIMEOUT_MS);
+        }
+        catch {
+            return null;
+        }
+    }
+    async collectTerminal() {
+        const out = [];
+        const terminal = this.options.terminal;
+        if (terminal) {
+            try {
+                const [current, live, manifest] = await Promise.all([terminal.currentGen(), terminal.live(), terminal.manifest()]);
+                const window = windowOf(manifest, TERMINAL_PROTOCOL_VERSION);
+                const gens = live.includes(current) ? live : [current, ...live];
+                out.push(...await Promise.all(gens.map(async (gen) => {
+                    const isLive = live.includes(gen);
+                    const sample = isLive ? await this.probe('terminal', gen) : null;
+                    const protocolVersion = sample?.hello.protocolVersion ?? (gen === current ? manifest.protocolVersion : null);
+                    return {
+                        live: isLive, current: gen === current,
+                        status: {
+                            kind: 'terminal', gen, state: !isLive ? 'stopped' : sample ? 'running' : 'unavailable',
+                            protocolVersion: protocolVersion ?? 0,
+                            buildVersion: sample?.hello.buildVersion ?? (gen === current ? manifest.buildVersion : ''),
+                            replacement: replacementOf({ gen, current, protocolVersion, window, mandatory: manifest.mandatory === true }),
+                            pending: gen === current ? null : this.pendingOf('terminal', gen),
+                            metrics: terminalMetrics(sample?.activity),
+                            items: terminalItems(sample?.activity),
+                        },
+                    };
+                })));
+            }
+            catch (error) {
+                this.log(`[keepers] keeper-terminal could not be described: ${error instanceof Error ? error.message : String(error)}`);
+            }
+        }
+        const legacy = this.options.legacy;
+        if (legacy?.live)
+            out.push({ live: true, current: false, status: this.legacyStatus(legacy) });
+        return out;
+    }
+    legacyStatus(legacy) {
+        const { state, open } = legacy.status();
+        const shell = nameLabel(this.options.legacyShell?.(), 'shell');
+        const sessions = legacy.lastSessions();
+        return {
+            kind: 'terminal', gen: 'legacy', legacy: true, state: state === 'present' || state === 'stopping' ? 'running' : 'unavailable', // busy = not answering
+            protocolVersion: 0, buildVersion: 'legacy', replacement: 'required', pending: 'when-idle',
+            metrics: { ptyOpen: open ?? sessions.filter((session) => !session.exited).length, ptyActive: 0, execRunning: 0 },
+            items: sessions.slice(0, KEEPER_STATUS_MAX_ITEMS).map((session) => ({
+                type: 'pty', id: clip(session.id, 256), label: shell, state: session.exited ? 'exited' : 'idle', sinceEpochMs: 0,
+            })),
+        };
+    }
+    async collectAcp() {
+        const acp = this.options.acp;
+        if (!acp)
+            return [];
+        try {
+            const [current, live, manifest] = await Promise.all([acp.currentGen(), acp.live(), acp.manifest()]);
+            const window = windowOf(manifest, ACP_PROTOCOL_VERSION);
+            const gens = live.includes(current) ? live : [current, ...live];
+            return await Promise.all(gens.map(async (gen) => {
+                const isLive = live.includes(gen);
+                const sample = isLive ? await this.probe('acp', gen) : null;
+                const protocolVersion = sample?.hello.protocolVersion ?? (gen === current ? manifest.protocolVersion : null);
+                const items = this.acpItems(sample?.activity);
+                return {
+                    live: isLive, current: gen === current,
+                    status: {
+                        kind: 'acp', gen, state: !isLive ? 'stopped' : sample ? 'running' : 'unavailable',
+                        protocolVersion: protocolVersion ?? 0,
+                        buildVersion: sample?.hello.buildVersion ?? (gen === current ? manifest.buildVersion : ''),
+                        replacement: replacementOf({ gen, current, protocolVersion, window, mandatory: manifest.mandatory === true }),
+                        pending: gen === current ? null : this.pendingOf('acp', gen),
+                        metrics: {
+                            sessions: finite(sample?.activity.metrics.sessionsRunning),
+                            busy: items.filter((item) => item.state === 'busy').length,
+                        },
+                        items,
+                    },
+                };
+            }));
+        }
+        catch (error) {
+            this.log(`[keepers] keeper-acp could not be described: ${error instanceof Error ? error.message : String(error)}`);
+            return [];
+        }
+    }
+    /** keeper-acp's mechanical items joined with this daemon's view: busy/idle is the daemon's judgement (spec §3.5). */
+    acpItems(activity) {
+        const items = [];
+        for (const item of activity?.items ?? []) {
+            if (item.type !== 'acp' || items.length >= KEEPER_STATUS_MAX_ITEMS)
+                continue;
+            const ref = refOfSessionKey(item.id);
+            const view = this.options.acp?.session(ref);
+            const agentId = view?.agentId ?? item.label;
+            const running = item.state === 'running';
+            items.push({
+                type: 'session', id: clip(ref, 256), label: clip((this.options.agentName?.(agentId) ?? agentId) || 'Agent'),
+                state: running ? (view?.busy ? 'busy' : 'idle') : item.state === 'starting' ? 'starting' : 'exited',
+                sinceEpochMs: item.sinceEpochMs,
+            });
+        }
+        return items;
+    }
+    // ---- the policy ---------------------------------------------------------------------------------------------------------
+    /** One policy pass (periodic): automatic decisions, then queued replacements that can run now. Never rejects. */
+    tick() {
+        return this.serialized(async () => {
+            const sampled = await this.sampled(true);
+            await this.automatic(sampled);
+            await this.runQueued(sampled);
+        }).catch((error) => { this.log(`[keepers] policy pass failed: ${error instanceof Error ? error.message : String(error)}`); });
+    }
+    /** required/mandatory, never decided before (a cancel or a restart never brings an automatic decision back): queue it. */
+    async automatic(sampled) {
+        for (const { status, live, current } of sampled) {
+            if (status.legacy || current || !live || !replacesByItself(status.replacement))
+                continue;
+            if (this.decisions.has(decisionKey(status.kind, status.gen)))
+                continue;
+            const refs = status.kind === 'acp' ? await this.moveOff([status.gen]) : undefined;
+            await this.decide(status.kind, status.gen, { mode: 'when-idle', by: 'auto', at: this.now(), ...(refs ? { refs } : {}) });
+            this.log(`[keepers] ${status.kind} generation ${status.gen} is ${status.replacement}: replaced once idle`);
+        }
+    }
+    async moveOff(gens) {
+        if (gens.length === 0)
+            return [];
+        try {
+            return await this.options.acp.moveOff(gens);
+        }
+        catch (error) {
+            this.log(`[keepers] moving sessions off older keeper-acp generations failed: ${error instanceof Error ? error.message : String(error)}`);
+            return [];
+        }
+    }
+    /** Queued (`when-idle`) replacements: an idle keeper-terminal generation is stopped; keeper-acp keeps migrating and retires. */
+    async runQueued(sampled) {
+        const acpPending = [];
+        for (const { status, live, current } of sampled) {
+            if (status.legacy || current || this.pendingOf(status.kind, status.gen) !== 'when-idle')
+                continue;
+            if (status.kind === 'acp') {
+                if (live)
+                    acpPending.push(status.gen);
+                continue;
+            }
+            if (!live || status.state !== 'running' || !terminalIdle(status.metrics))
+                continue;
+            // Asked again right before stopping: a PTY or an exec that started since the sample keeps it.
+            const again = await this.probe('terminal', status.gen);
+            if (!again || !terminalIdle(terminalMetrics(again.activity)))
+                continue;
+            // A create the picker routed there before the decision may still be on its way: drain first (keeper-terminal refuses new work
+            // from now on), then stop only if a probe AFTER the drain still finds it idle.
+            try {
+                await this.options.terminal.drain(status.gen);
+            }
+            catch {
+                continue;
+            }
+            const decision = this.decisions.get(decisionKey('terminal', status.gen));
+            if (decision && !decision.drained)
+                await this.decide('terminal', status.gen, { ...decision, drained: true });
+            const drained = await this.probe('terminal', status.gen);
+            if (!drained || !terminalIdle(terminalMetrics(drained.activity)))
+                continue;
+            await this.stopTerminal(status.gen, 'graceful', 'idle');
+        }
+        // Queued decisions of generations that are gone are done — only for a kind whose generations could be listed at all (its current
+        // generation is always in a successful listing), so a failed listing never ends a decision.
+        const listedKinds = new Set(sampled.filter((entry) => entry.current).map((entry) => entry.status.kind));
+        for (const [key, decision] of [...this.decisions]) {
+            if (decision.mode !== 'when-idle')
+                continue;
+            const [kind, gen] = key.split(':');
+            if (!listedKinds.has(kind))
+                continue;
+            if (!sampled.some((entry) => entry.status.kind === kind && entry.status.gen === gen && entry.live)) {
+                await this.decide(kind, gen, { ...decision, mode: 'done', at: this.now() });
+            }
+        }
+        if (acpPending.length > 0 && this.options.acp) {
+            // Sessions adopted on such a generation since the decision get their intent too (idempotent); their refs join the decision, so a
+            // cancel withdraws every intent it caused.
+            const refs = await this.moveOff(acpPending);
+            for (const gen of acpPending) {
+                const decision = this.decisions.get(decisionKey('acp', gen));
+                if (!decision || decision.mode !== 'when-idle')
+                    continue;
+                const merged = [...new Set([...(decision.refs ?? []), ...refs])];
+                if (merged.length !== (decision.refs ?? []).length)
+                    await this.decide('acp', gen, { ...decision, refs: merged });
+            }
+            await this.options.acp.retireIdle().catch(() => undefined);
+        }
+    }
+    async stopTerminal(gen, mode, why) {
+        const terminal = this.options.terminal;
+        const result = await terminal.stop(gen, mode).catch((error) => ({ stopped: false, pid: null, reason: error instanceof Error ? error.message : String(error) }));
+        this.log(`[keepers] keeper-terminal generation ${gen} (${why}): ${result.stopped ? 'stopped' : `not stopped${result.reason ? ` (${result.reason})` : ''}`}`);
+        const previous = this.decisions.get(decisionKey('terminal', gen));
+        if (result.stopped)
+            await this.decide('terminal', gen, { mode: 'done', by: previous?.by ?? 'user', at: this.now(), ...(previous?.refs ? { refs: previous.refs } : {}) });
+        this.sample = undefined;
+        return result.stopped;
+    }
+    // ---- keeper_control -------------------------------------------------------------------------------------------------------
+    /** Answers `keeper_control`; a repeated commandId gets the same answer (spec §8.1). Never rejects. */
+    control(frame) {
+        const known = this.results.get(frame.commandId);
+        if (known)
+            return known;
+        const result = this.serialized(() => this.apply(frame)).catch((error) => {
+            // Nothing was done when the keepers could not even be sampled: the same commandId may be tried again.
+            if (error instanceof KeeperSampleTimeoutError)
+                queueMicrotask(() => { if (this.results.get(frame.commandId) === result)
+                    this.results.delete(frame.commandId); });
+            return {
+                t: 'keeper_control_result', commandId: frame.commandId, kind: frame.kind, action: frame.action, ok: false, affected: [],
+                error: error instanceof Error ? error.message : String(error),
+            };
+        });
+        this.results.set(frame.commandId, result);
+        while (this.results.size > KEEPER_CONTROL_RESULTS_LIMIT)
+            this.results.delete(this.results.keys().next().value);
+        return result;
+    }
+    async apply(frame) {
+        const sampled = await this.sampled(true);
+        const older = sampled.filter((entry) => entry.status.kind === frame.kind && !entry.current && entry.live && !entry.status.legacy);
+        const legacy = frame.kind === 'terminal' ? sampled.find((entry) => entry.status.legacy) : undefined;
+        const affected = [];
+        let ok = true;
+        let incomplete = false;
+        let error;
+        if (frame.kind === 'acp' && !this.options.acp)
+            throw new Error('keeper-acp is not managed by this daemon');
+        if (frame.kind === 'terminal' && !this.options.terminal)
+            throw new Error('keeper-terminal is not managed by this daemon');
+        switch (frame.action) {
+            case 'replace-when-idle': {
+                const refs = frame.kind === 'acp' ? await this.moveOff(older.map((entry) => entry.status.gen)) : undefined;
+                for (const { status } of older)
+                    await this.decide(frame.kind, status.gen, { mode: 'when-idle', by: 'user', at: this.now(), ...(refs ? { refs } : {}) });
+                if (older.length > 0)
+                    await this.runQueued(await this.sampled(true));
+                break;
+            }
+            case 'replace-now': {
+                for (const { status } of older) {
+                    const interrupted = status.items.filter((item) => item.state !== 'exited');
+                    const stopped = frame.kind === 'terminal'
+                        ? await this.stopTerminal(status.gen, 'force', 'replaced now')
+                        : await this.stopAcp(status.gen);
+                    if (stopped) {
+                        affected.push(...interrupted.map((item) => ({ type: item.type, id: item.id, label: item.label, gen: status.gen })));
+                        // Not every interrupted thing is named: the keeper could not be asked, or its items were cut at the bound.
+                        const running = frame.kind === 'terminal' ? (status.metrics.ptyOpen ?? 0) + (status.metrics.execRunning ?? 0) : (status.metrics.sessions ?? 0);
+                        if (status.state !== 'running' || interrupted.length < running)
+                            incomplete = true;
+                    }
+                    else {
+                        ok = false;
+                        error = `generation ${status.gen} could not be stopped`;
+                    }
+                }
+                if (legacy && this.options.legacy) {
+                    const shell = nameLabel(this.options.legacyShell?.(), 'shell');
+                    for (const session of await this.options.legacy.replaceNow())
+                        affected.push({ type: 'pty', id: session.id, label: shell, gen: 'legacy' });
+                    if (this.options.legacy.live) {
+                        ok = false;
+                        error = 'the legacy tools keeper could not be stopped';
+                    }
+                }
+                break;
+            }
+            case 'cancel': {
+                for (const [key, decision] of [...this.decisions]) {
+                    if (!key.startsWith(`${frame.kind}:`) || decision.mode !== 'when-idle')
+                        continue;
+                    if (frame.kind === 'acp' && decision.refs?.length)
+                        await this.options.acp.cancelMoves(decision.refs).catch(() => undefined);
+                    await this.decide(frame.kind, key.slice(frame.kind.length + 1), { ...decision, mode: 'cancelled', by: 'user', at: this.now() });
+                }
+                break;
+            }
+        }
+        this.sample = undefined;
+        // From the decisions themselves (a sample in flight may predate them); the legacy keeper is always queued while it lives.
+        const queued = [...this.decisions].some(([key, decision]) => key.startsWith(`${frame.kind}:`) && decision.mode === 'when-idle')
+            || (frame.kind === 'terminal' && this.options.legacy?.live === true);
+        if (affected.length > KEEPER_AFFECTED_MAX_ITEMS)
+            incomplete = true;
+        return {
+            t: 'keeper_control_result', commandId: frame.commandId, kind: frame.kind, action: frame.action, ok,
+            affected: affected.slice(0, KEEPER_AFFECTED_MAX_ITEMS), pending: queued ? 'when-idle' : null,
+            ...(incomplete ? { affectedIncomplete: true } : {}), ...(error ? { error } : {}),
+        };
+    }
+    async stopAcp(gen) {
+        const acp = this.options.acp;
+        const result = await acp.stop(gen, 'force').catch((error) => ({ stopped: false, pid: null, reason: error instanceof Error ? error.message : String(error) }));
+        this.log(`[keepers] keeper-acp generation ${gen} (replaced now): ${result.stopped ? 'stopped' : `not stopped${result.reason ? ` (${result.reason})` : ''}`}`);
+        const previous = this.decisions.get(decisionKey('acp', gen));
+        if (result.stopped)
+            await this.decide('acp', gen, { mode: 'done', by: 'user', at: this.now(), ...(previous?.refs ? { refs: previous.refs } : {}) });
+        return result.stopped;
+    }
+}
+/** The window the current build's daemon drives (its manifest's `daemonProtocol`, which the release manifest carries too). */
+function windowOf(manifest, protocol) {
+    return isProtocolWindow(manifest.daemonProtocol) ? manifest.daemonProtocol : protocolWindow(protocol);
+}
+const finite = (value) => (typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : 0);
+function terminalMetrics(activity) {
+    const metrics = activity?.metrics ?? {};
+    return { ptyOpen: finite(metrics.ptyOpen), ptyActive: finite(metrics.ptyActive), execRunning: finite(metrics.execRunning) };
+}
+/** keeper-terminal's items: PTYs (foreground process name) and execs (the server's label); bounded, labels cut. */
+function terminalItems(activity) {
+    const items = [];
+    for (const item of activity?.items ?? []) {
+        if ((item.type !== 'pty' && item.type !== 'exec') || items.length >= KEEPER_STATUS_MAX_ITEMS)
+            continue;
+        items.push({
+            type: item.type, id: clip(item.id, 256),
+            // A PTY label is already a process name; an exec label is the server's description — neither is a command line.
+            label: clip(item.label || (item.type === 'pty' ? 'shell' : 'Command')),
+            state: item.state, sinceEpochMs: item.sinceEpochMs, ...(item.jobId ? { jobId: item.jobId } : {}),
+        });
+    }
+    return items;
+}
+/** The session ref a keeper-acp session key belongs to (`<ref>` or a switch successor `<ref>#s<n>`, Task 24). */
+export function refOfSessionKey(key) {
+    return key.replace(/#s\d+$/, '');
+}

package/dist/workstation/keepers/stop.js

@@ -0,0 +1,34 @@
+/** The kit keepers a stopped workstation stops, every running generation of each. */
+export const STOPPED_KEEPER_KINDS = ['acp', 'terminal'];
+/** How long a graceful stop may take before it is forced (keeper-acp gives its adapters 1.5 s, the kit's grace is 3 s). */
+export const KEEPER_STOP_WAIT_MS = 10_000;
+/**
+ * Stops every running generation of keeper-acp and keeper-terminal: gracefully (`keeper.stop`: adapters, PTYs and execs are terminated
+ * by the keeper itself), then forced when it did not stop in time. Never throws; reports what happened per generation.
+ */
+export async function stopKeepers(launcher, options = {}) {
+    const log = options.log ?? (() => { });
+    const reports = [];
+    for (const kind of STOPPED_KEEPER_KINDS) {
+        let gens;
+        try {
+            gens = await launcher.liveGenerations(kind);
+        }
+        catch (error) {
+            log(`keeper-${kind}: could not be listed: ${messageOf(error)}`);
+            continue;
+        }
+        for (const gen of gens) {
+            let forced = false;
+            let result = await launcher.stop(kind, gen, 'graceful', options.waitMs ?? KEEPER_STOP_WAIT_MS).catch((error) => ({ stopped: false, pid: null, reason: messageOf(error) }));
+            if (!result.stopped) {
+                forced = true;
+                result = await launcher.stop(kind, gen, 'force').catch((error) => ({ stopped: false, pid: null, reason: messageOf(error) }));
+            }
+            log(`keeper-${kind} ${gen}: ${result.stopped ? (forced ? 'stopped (forced)' : 'stopped') : `could not be stopped${result.reason ? ` (${result.reason})` : ''}`}`);
+            reports.push({ kind, gen, result, forced });
+        }
+    }
+    return reports;
+}
+const messageOf = (error) => (error instanceof Error ? error.message : String(error));