@ours.network/fleet 1.1.0 → 1.1.2

package/dist/session/acp.js

@@ -1,5 +1,5 @@
 import { spawn } from 'node:child_process';
-import { randomUUID } from 'node:crypto';
+import { createHash, randomUUID } from 'node:crypto';
 import { existsSync, lstatSync, readFileSync, readlinkSync, realpathSync, writeFileSync, } from 'node:fs';
 import { basename, dirname, isAbsolute, join, relative, resolve } from 'node:path';
 import { Readable, Writable } from 'node:stream';
@@ -7,6 +7,7 @@ import * as acp from '@agentclientprotocol/sdk';
 import { normalizeSessionUpdate } from './conversation-normalizer.js';
 import { ConversationEventStore } from './conversation-store.js';
 import { SessionEvents } from './events.js';
+import { DEFAULT_STALL_TIMEOUT_MS, STALL_RECOVERY_PROMPT, StallWatchdog, StallToolHistory, hasStallRecoveryClaim } from './stall-watchdog.js';
 import { ACP_CANCEL_DEADLINE_EXCEEDED, SessionControlError, classifyChildExit, sessionBackendCapabilities, turnResult, } from './types.js';
 const CANCEL_SETTLE_GRACE_MS = 15_000;
 const CANCEL_TERMINATE_GRACE_MS = 5_000;
@@ -158,6 +159,7 @@ function canonicallyWithin(root, candidates) {
 function conversationSource(origin) {
     switch (origin?.kind) {
         case 'owner-admin-console': return { source: 'owner_admin_console', persistBody: true };
+        case 'stall-watchdog': return { source: 'fleet_monitor', persistBody: false };
         case 'startup': return { source: 'startup', persistBody: true };
         case 'owner': return { source: 'owner_channel', persistBody: false };
         case 'fleet-monitor': return { source: 'fleet_monitor', persistBody: false };
@@ -265,6 +267,15 @@ export class AcpSession {
     terminate;
     /** ACP-authenticated in-flight calls, including independently reserved permissions. */
     activeToolCalls = new Map();
+    stallWatchdog;
+    stallToolHistory;
+    stallRecoveryClaimed = false;
+    managedTurnCount = 0;
+    steeringWasUsed = false;
+    steeringRequests = 0;
+    retryNativeTurnId;
+    stallTimer;
+    stallAttempt;
     toolBoundaryWaiters = new Set();
     activeTurn;
     constructor(options, child, connection) {
@@ -285,6 +296,9 @@ export class AcpSession {
         this.terminated.catch(() => undefined);
         child.stderr.on('data', chunk => options.log(`[${options.name}] acp: ${String(chunk).trimEnd()}`));
         child.once('exit', (code, signal) => {
+            if (this.stallTimer)
+                clearInterval(this.stallTimer);
+            this.stallTimer = undefined;
             if (this.cancelForceKill)
                 clearTimeout(this.cancelForceKill);
             this.cancelForceKill = undefined;
@@ -341,7 +355,8 @@ export class AcpSession {
         let instance;
         const app = acp.client({ name: 'ours-fleet' })
             .onNotification(acp.methods.client.session.update, ({ params }) => {
-            instance?.recordUpdate(params.update);
+            if (instance && (!instance.sessionId || params.sessionId === instance.sessionId))
+                instance.recordUpdate(params.update);
         })
             .onRequest(acp.methods.client.session.requestPermission, ({ params }) => {
             if (!instance)
@@ -353,6 +368,7 @@ export class AcpSession {
         instance = new AcpSession(options, child, connection);
         try {
             await instance.initialize();
+            instance.startStallWatchdog();
             instance.recoverOpenPrompts();
             return instance;
         }
@@ -471,10 +487,14 @@ export class AcpSession {
     reserveTool(toolCallId) {
         if (toolCallId)
             this.toolCall(toolCallId).lifecycle = true;
+        else if (this.activeTurn)
+            this.activeTurn.boundaryUnknown = true;
     }
     reservePermission(toolCallId, permissionId) {
         if (toolCallId)
             this.toolCall(toolCallId).permissions.set(permissionId, 'pending');
+        else if (this.activeTurn)
+            this.activeTurn.boundaryUnknown = true;
     }
     allowPermission(toolCallId, permissionId) {
         if (!toolCallId)
@@ -559,6 +579,8 @@ export class AcpSession {
      * steering response from becoming a tight replay loop.
      */
     async steerOrQueueWake(text, options) {
+        if (this.stallRecoveryClaimed)
+            return this.submitPrompt(text, { ...options, interrupt: false, steer: false });
         const steered = await this.steerPrompt(text);
         if (steered.accepted || steered.detail !== 'ACP steering failed'
             || this.closing || !this.isAlive())
@@ -622,6 +644,8 @@ export class AcpSession {
      * for it is what turned a busy agent into a timeout and then into "dead".
      */
     async queuePrompt(text, options = {}) {
+        if (this.stallRecoveryClaimed && options.origin?.kind === 'fleet-monitor')
+            options = { ...options, interrupt: false, steer: false };
         if (this.cancelRecoveryReason)
             throw new SessionControlError('control-unavailable', 'ACP adapter restart is in progress after the cancellation deadline', ACP_CANCEL_DEADLINE_EXCEEDED);
         if (this.closing || !this.sessionId || !this.isAlive())
@@ -781,6 +805,8 @@ export class AcpSession {
         }
     }
     async cancelActive(source) {
+        if (this.stallAttempt && source !== 'stall-watchdog' && source !== 'fleet-monitor')
+            this.stallAttempt.superseded = true;
         if (!this.sessionId)
             return;
         const active = this.activeTurn;
@@ -989,6 +1015,11 @@ export class AcpSession {
     }
     async close() {
         this.closing = true;
+        if (this.stallTimer)
+            clearInterval(this.stallTimer);
+        this.stallTimer = undefined;
+        if (this.stallAttempt)
+            this.stallAttempt.superseded = true;
         if (this.cancelEscalation)
             clearTimeout(this.cancelEscalation);
         this.cancelEscalation = undefined;
@@ -1130,13 +1161,158 @@ export class AcpSession {
         this.runtimeModel = runtimeSelector(options, 'model');
         this.reasoningEffort = runtimeSelector(options, 'thought_level') ?? reasoningFromModelId(modelId);
     }
+    startStallWatchdog() {
+        if (!this.options.stallRecovery)
+            return;
+        const timeoutMs = this.options.stallRecovery.timeoutMs ?? DEFAULT_STALL_TIMEOUT_MS;
+        this.stallRecoveryClaimed = hasStallRecoveryClaim(this.options.stateDir, this.sessionId);
+        this.stallToolHistory = new StallToolHistory(this.options.stateDir, this.sessionId, this.options.mode === 'resume');
+        this.stallWatchdog = new StallWatchdog({
+            stateDir: this.options.stateDir, timeoutMs, previouslyClaimed: this.stallRecoveryClaimed, now: () => Date.now(),
+            observe: () => this.stallObservation(),
+            recover: (observed, report) => this.recoverStall(observed, report),
+            diagnostic: diagnostic => {
+                if (['interrupt_requested', 'blocked_previous_attempt', 'blocked_persistence'].includes(diagnostic.status))
+                    this.stallRecoveryClaimed = true;
+                this.events.emit('stall_recovery', { status: diagnostic.status, stallDiagnostic: diagnostic });
+                this.options.log(`[${this.options.name}] ACP stall recovery: ${diagnostic.status}; `
+                    + 'inspect structured session events before continuing; never replay uncertain side effects');
+            },
+        });
+        this.stallTimer = setInterval(() => this.checkStallWatchdog(), this.options.stallRecovery.tickMs ?? Math.min(10_000, timeoutMs));
+        this.stallTimer.unref?.();
+    }
+    checkStallWatchdog() {
+        void this.stallWatchdog?.tick();
+        const attempt = this.stallAttempt;
+        if (attempt?.recoveryStartedAt === undefined || attempt.blocked || attempt.superseded)
+            return;
+        const observed = this.stallObservation();
+        if (!observed?.safe || observed.turnId !== attempt.recoveryId)
+            return;
+        const last = observed.lastProgressAt || attempt.recoveryStartedAt;
+        const timeoutMs = this.options.stallRecovery?.timeoutMs ?? DEFAULT_STALL_TIMEOUT_MS;
+        if (Date.now() - last >= timeoutMs * 2) {
+            attempt.blocked = true;
+            try {
+                attempt.report('blocked_restall');
+            }
+            catch { /* durable claim already prevents retry */ }
+        }
+    }
+    stallObservation() {
+        const active = this.activeTurn;
+        if (!active || !this.sessionId)
+            return undefined;
+        return {
+            sessionId: this.sessionId, generation: this.sessionGeneration, turnId: active.id,
+            startedAt: active.startedAt, lastProgressAt: active.lastProgressAt, progressCount: active.progressCount,
+            transportFailures: active.transportFailures,
+            boundaryEvidenceAvailable: this.stallToolHistory?.available() !== false,
+            safe: this.isAlive() && !this.closing && this.readiness === 'running'
+                && !active.cancellationSource && !active.boundaryUnknown && !this.steeringOccupied
+                && this.steeringRequests === 0 && this.stallToolHistory?.available() !== false
+                && this.activeToolCalls.size === 0 && this.pendingPermissions.size === 0,
+        };
+    }
+    async recoverStall(observed, report) {
+        const current = this.stallObservation();
+        if (!current?.safe || current.turnId !== observed.turnId
+            || current.lastProgressAt !== observed.lastProgressAt || current.progressCount !== observed.progressCount) {
+            report('superseded');
+            return;
+        }
+        const active = this.activeTurn;
+        let resolveReady;
+        const ready = new Promise(resolve => { resolveReady = resolve; });
+        const attempt = this.stallAttempt = {
+            turnId: active.id, recoveryId: randomUUID(), ready, superseded: false,
+            report, resumed: false, blocked: false,
+        };
+        active.cancellationSource = 'stall-watchdog';
+        // This intentionally does not use cancelActive: automatic recovery may
+        // never enter its SIGTERM/SIGKILL escalation or settle a permission.
+        let timer;
+        try {
+            const completed = await Promise.race([
+                (async () => {
+                    await this.connection.agent.notify(acp.methods.agent.session.cancel, { sessionId: this.sessionId });
+                    await active.settled;
+                    return true;
+                })(),
+                new Promise(resolve => {
+                    timer = setTimeout(() => resolve(false), this.options.stallRecovery?.cancelWaitMs ?? CANCEL_SETTLE_GRACE_MS);
+                    timer.unref?.();
+                }),
+            ]);
+            if (!completed || attempt.superseded || this.closing || !this.isAlive()) {
+                attempt.blocked = true;
+                report(attempt.superseded || this.closing ? 'superseded' : 'blocked_cancel');
+                resolveReady(false);
+            }
+            else
+                resolveReady(true);
+        }
+        catch {
+            attempt.blocked = true;
+            try {
+                report('blocked_cancel');
+            }
+            finally {
+                resolveReady(false);
+            }
+        }
+        finally {
+            if (timer)
+                clearTimeout(timer);
+        }
+    }
+    /** Keep the original queue slot (including startup) until recovery finishes. */
     async runPrompt(text, turnId = randomUUID(), origin) {
+        const result = await this.runSinglePrompt(text, turnId, origin);
+        const attempt = this.stallAttempt;
+        if (!attempt || attempt.turnId !== turnId)
+            return result;
+        const ready = await attempt.ready;
+        if (attempt.superseded || this.closing)
+            return result;
+        if (!ready || result.outcome !== 'cancelled' || result.cancellationSource !== 'stall-watchdog') {
+            // An RPC error/refusal/ambiguous terminal answer to cancellation is not
+            // permission to replay work or to fail startup and restart the process.
+            if (!attempt.blocked) {
+                attempt.blocked = true;
+                try {
+                    attempt.report('blocked_cancel');
+                }
+                catch { /* claim remains durable */ }
+            }
+            return turnResult(true, 'cancelled', 'diagnostic cancellation requires operator attention', undefined, 'stall-watchdog');
+        }
+        try {
+            attempt.report('recovery_started');
+            attempt.recoveryStartedAt = Date.now();
+            const recoveryOrigin = origin?.kind === 'scheduled-loop'
+                ? origin : { kind: 'stall-watchdog' };
+            this.admitToLedger(attempt.recoveryId, STALL_RECOVERY_PROMPT, 0, { origin: recoveryOrigin });
+            const recovered = await this.runSinglePrompt(STALL_RECOVERY_PROMPT, attempt.recoveryId, recoveryOrigin);
+            attempt.report(attempt.superseded ? 'superseded'
+                : recovered.succeeded && attempt.resumed ? 'recovery_completed' : 'blocked_recovery');
+            // A failed diagnostic turn must not make startup tear down the session.
+            return recovered.succeeded && attempt.resumed ? recovered : turnResult(true, 'cancelled', 'diagnostic recovery requires operator attention', undefined, 'stall-watchdog');
+        }
+        catch {
+            return turnResult(true, 'cancelled', 'diagnostic recovery requires operator attention', undefined, 'stall-watchdog');
+        }
+    }
+    async runSinglePrompt(text, turnId = randomUUID(), origin) {
         if (!this.sessionId || !this.isAlive())
             return turnResult(false, 'failed', this.lastError ?? 'ACP session is offline');
         this.readiness = 'running';
+        this.managedTurnCount++;
+        this.retryNativeTurnId = undefined;
         let settle;
         const settled = new Promise(resolve => { settle = resolve; });
-        this.activeTurn = { id: turnId, output: '', origin, settled, settle };
+        this.activeTurn = { toolEvidence: new Map(), startedAt: Date.now(), toolIds: new Set(), lastProgressAt: 0, progressCount: 0, transportFailures: 0, boundaryUnknown: false, id: turnId, output: '', origin, settled, settle };
         this.events.emit('state', { turnId, status: 'running', origin });
         this.conversation.appendSafe({
             kind: 'prompt.started', sessionGeneration: this.sessionGeneration,
@@ -1180,7 +1356,8 @@ export class AcpSession {
             this.lastError = origin?.kind === 'scheduled-loop' ? 'scheduled-loop turn failed' : detail;
             this.readiness = this.isAlive() ? 'idle' : 'failed';
             this.events.emit('error', {
-                turnId, origin,
+                turnId, origin: this.activeTurn?.cancellationSource === 'stall-watchdog'
+                    ? { kind: 'stall-watchdog' } : origin,
                 text: origin?.kind === 'scheduled-loop' ? 'scheduled-loop turn failed' : this.lastError,
             });
             if (this.isAlive())
@@ -1208,8 +1385,10 @@ export class AcpSession {
         }
     }
     async steerPrompt(text) {
+        this.steeringWasUsed = true;
         if (!this.sessionId || !this.isAlive())
             return turnResult(false, 'failed', this.lastError ?? 'ACP session is offline');
+        this.steeringRequests++;
         try {
             const response = await Promise.race([
                 this.connection.agent.request('_session/steering', {
@@ -1234,6 +1413,9 @@ export class AcpSession {
             this.events.emit('error', { text: detail });
             return turnResult(false, 'failed', detail);
         }
+        finally {
+            this.steeringRequests--;
+        }
     }
     requestPermission(params) {
         // `kinds` is a PRIORITY order. Scanning the agent's option array instead
@@ -1409,6 +1591,48 @@ export class AcpSession {
             && params.options.some(option => option.optionId === 'allow_once' && option.kind === 'allow_once')
             && params.options.some(option => option.optionId === 'decline' && option.kind === 'reject_once');
     }
+    /** Pinned codex-acp 1.1.7 structured metadata, never stderr or assistant text. */
+    recordStallMetadata(update) {
+        if (this.options.permissionMetadataSource !== 'codex-acp'
+            || update.sessionUpdate !== 'session_info_update' || !this.activeTurn)
+            return;
+        const meta = update._meta?.codex;
+        if (!meta || typeof meta !== 'object' || Array.isArray(meta))
+            return;
+        const codex = meta;
+        const status = codex.threadStatus;
+        if (Object.prototype.hasOwnProperty.call(codex, 'threadStatus')) {
+            const value = status && typeof status === 'object' && !Array.isArray(status)
+                ? status : {};
+            // Unknown or modal thread status is a permanent conservative fence for
+            // this turn. A later delayed idle/active status must not clear it.
+            if (value.type !== 'active' || !Array.isArray(value.activeFlags)
+                || value.activeFlags.length > 0)
+                this.activeTurn.boundaryUnknown = true;
+        }
+        const error = codex.error;
+        if (!error || typeof error !== 'object' || Array.isArray(error))
+            return;
+        const value = error;
+        const info = value.codexErrorInfo;
+        if (value.willRetry !== true || typeof value.turnId !== 'string' || !value.turnId
+            || !info || typeof info !== 'object' || Array.isArray(info))
+            return;
+        if (!['responseStreamConnectionFailed', 'responseStreamDisconnected']
+            .some(key => Object.prototype.hasOwnProperty.call(info, key)))
+            return;
+        // ACP 1.1.7 does not expose a native-turn-to-prompt mapping. Only the first
+        // fresh managed turn with no steering can be correlated without guessing;
+        // later/resumed turns retain the conservative generic no-progress path.
+        if (this.options.mode !== 'fresh' || this.managedTurnCount !== 1 || this.steeringWasUsed)
+            return;
+        if (this.retryNativeTurnId && this.retryNativeTurnId !== value.turnId) {
+            this.activeTurn.boundaryUnknown = true;
+            return;
+        }
+        this.retryNativeTurnId = value.turnId;
+        this.activeTurn.transportFailures++;
+    }
     recordUpdate(update) {
         // Replayed history is not current activity: `session/load` would otherwise
         // make a cold session look like it had just been working. The same reason
@@ -1418,6 +1642,53 @@ export class AcpSession {
             this.lastUpdateAt = new Date().toISOString();
             this.refreshSteeringOccupancy();
         }
+        if (!this.replaying && this.activeTurn) {
+            this.recordStallMetadata(update);
+            const active = this.activeTurn;
+            const kind = update.sessionUpdate;
+            if (this.options.stallRecovery && (kind === 'tool_call' || kind === 'tool_call_update')) {
+                if (!update.toolCallId || active.toolIds.size >= 4096)
+                    active.boundaryUnknown = true;
+                else
+                    active.toolIds.add(update.toolCallId);
+                if (this.stallToolHistory?.observe(update.toolCallId, active.id) === false)
+                    active.boundaryUnknown = true;
+            }
+            let meaningful = ((kind === 'agent_message_chunk' || kind === 'agent_thought_chunk')
+                && (update.content.type !== 'text' || update.content.text.length > 0))
+                || kind === 'tool_call' || kind === 'tool_call_update' || kind === 'plan';
+            if (this.options.stallRecovery && (kind === 'tool_call' || kind === 'tool_call_update' || kind === 'plan')) {
+                const fingerprint = createHash('sha256').update(JSON.stringify(update)).digest('hex');
+                if (kind === 'plan') {
+                    meaningful = fingerprint !== active.planEvidence;
+                    active.planEvidence = fingerprint;
+                }
+                else if (active.toolEvidence.size < 4096 || active.toolEvidence.has(update.toolCallId)) {
+                    meaningful = fingerprint !== active.toolEvidence.get(update.toolCallId);
+                    active.toolEvidence.set(update.toolCallId, fingerprint);
+                }
+            }
+            if (this.options.stallRecovery && ![
+                'agent_message_chunk', 'agent_thought_chunk', 'tool_call', 'tool_call_update', 'plan',
+                'available_commands_update', 'current_mode_update', 'config_option_update', 'session_info_update', 'usage_update',
+            ].includes(kind))
+                active.boundaryUnknown = true;
+            if (meaningful) {
+                active.lastProgressAt = Date.now();
+                active.progressCount++;
+                active.transportFailures = 0;
+                const attempt = this.stallAttempt;
+                if (attempt?.recoveryId === active.id && !attempt.resumed) {
+                    attempt.resumed = true;
+                    try {
+                        attempt.report('progress_resumed');
+                    }
+                    catch {
+                        attempt.blocked = true;
+                    }
+                }
+            }
+        }
         const scheduled = this.activeTurn?.origin?.kind === 'scheduled-loop';
         const messagePhase = update.sessionUpdate === 'agent_message_chunk'
             ? this.codexMessagePhase(update) : undefined;
@@ -1475,6 +1746,9 @@ export class AcpSession {
                     this.releaseTool(update.toolCallId);
                 else if (update.status !== undefined)
                     this.reserveTool(update.toolCallId);
+                else if (this.options.stallRecovery && this.activeTurn
+                    && !this.activeToolCalls.has(update.toolCallId))
+                    this.activeTurn.boundaryUnknown = true;
                 break;
             default:
                 break;

package/dist/session/stall-watchdog.d.ts

@@ -0,0 +1,60 @@
+export declare const DEFAULT_STALL_TIMEOUT_MS: number;
+export declare const STALL_RECOVERY_PROMPT: string;
+export type StallStatus = 'interrupt_requested' | 'recovery_started' | 'progress_resumed' | 'recovery_completed' | 'blocked_cancel' | 'blocked_recovery' | 'blocked_restall' | 'blocked_persistence' | 'blocked_previous_attempt' | 'blocked_evidence' | 'superseded';
+export interface StallDiagnostic {
+    version: 1;
+    kind: 'stall_recovery';
+    eventId: string;
+    session: string;
+    turn: string;
+    status: StallStatus;
+    evidence: 'adapter_transport' | 'no_progress';
+    idleMs: number;
+}
+export interface StallObservation {
+    sessionId: string;
+    generation: string;
+    turnId: string;
+    startedAt: number;
+    lastProgressAt: number;
+    progressCount: number;
+    transportFailures: number;
+    safe: boolean;
+    boundaryEvidenceAvailable?: boolean;
+}
+/** Presence, including an incomplete claim, restores conservative mail policy. */
+export declare function hasStallRecoveryClaim(stateDir: string, sessionId: string): boolean;
+/** ACP has no turn IDs on tool updates. Reuse across turns is ambiguous. */
+export declare class StallToolHistory {
+    private readonly turns;
+    private healthy;
+    private readonly directory;
+    private readonly path;
+    private readonly session;
+    constructor(stateDir: string, sessionId: string, resume: boolean);
+    available(): boolean;
+    /** Record before relying on a tool event. False means cancellation is unsafe. */
+    observe(toolId: string, turnId: string): boolean;
+    private persist;
+}
+/**
+ * One durable attempt per ACP session, deliberately stricter than one per turn.
+ * A restarted supervisor never guesses whether cancellation or recovery ran.
+ * Claim files are never reclaimed automatically, including malformed/empty ones.
+ */
+export declare class StallWatchdog {
+    private readonly options;
+    private checking;
+    private disabled;
+    private unavailableTurn?;
+    constructor(options: {
+        stateDir: string;
+        timeoutMs: number;
+        previouslyClaimed?: boolean;
+        now(): number;
+        observe(): StallObservation | undefined;
+        recover(observed: StallObservation, report: (status: StallStatus) => void): Promise<void>;
+        diagnostic(event: StallDiagnostic): void;
+    });
+    tick(): Promise<void>;
+}