@qodo/sdk 0.12.0 → 0.13.1

package/dist/api/websocket.js

@@ -28,6 +28,12 @@ export var ConnectionState;
 export var ReadyState;
 (function (ReadyState) {
     ReadyState["IDLE"] = "IDLE";
+    /**
+     * Post-connect, waiting for ResumeAck from the server before sending any
+     * turn. Replaces the old WAITING_INITIAL_READY — the server's authoritative
+     * view is now the precondition for proceeding, not a bare "Ready" signal.
+     */
+    ReadyState["AWAITING_RESUME_ACK"] = "AWAITING_RESUME_ACK";
     ReadyState["WAITING_INITIAL_READY"] = "WAITING_INITIAL_READY";
     ReadyState["READY"] = "READY";
     ReadyState["MESSAGE_SENT"] = "MESSAGE_SENT";
@@ -73,8 +79,60 @@ export class WebSocketClient extends EventEmitter {
     latestCheckpointId;
     lastSentMessage;
     readyTimer;
-    pendingOutbox = [];
     receivedResponsesSinceMessage = false; // Track if any responses received after sending message
+    /**
+     * Protocol-v2 counters. Public read-only via `getMetrics()` — wired into
+     * SDK telemetry so observability dashboards can alert on e.g. rising
+     * `reconcile_failures_total` without digging through debug logs.
+     */
+    metrics = {
+        reconnects_total: 0,
+        turns_sent_total: 0,
+        turns_acked_total: 0,
+        turns_replayed_total: 0,
+        resume_acks_total: 0,
+        reconcile_failures_total: 0,
+        stalls_detected_total: 0,
+        idempotent_replays_total: 0,
+    };
+    /** Last time the socket transitioned from CONNECTED to not-CONNECTED; used for downtime metrics. */
+    lastDisconnectAt;
+    getMetrics() {
+        return { ...this.metrics };
+    }
+    /**
+     * Outbox of turns the client has sent (or is about to send) but has not
+     * received a TurnAck for. Keyed by turn_id so idempotent replay across
+     * reconnect is an O(1) lookup. Only pruned on TurnAck or ResumeAck.
+     */
+    outbox = new Map();
+    /**
+     * FIFO order of turn_ids in the outbox. We send one turn at a time — the
+     * Ready/MESSAGE_SENT ping-pong enforces this — but the order matters for
+     * replay after reconcile, so we track it explicitly rather than relying on
+     * Map iteration order across Node versions.
+     */
+    outboxOrder = [];
+    /** SLA timer: Resume must be ack'd within this window or the socket is dead. */
+    resumeAckTimer;
+    /** SLA timer: post-ResumeAck, the server must produce forward progress or we surface a stall. */
+    postResumeProgressTimer;
+    /**
+     * Short grace period after a bare Ready arrives while we're awaiting a
+     * ResumeAck. If no ResumeAck shows up in that window, we assume the
+     * backend is a pre-v2 deployment that only emits Ready and fall back —
+     * instead of hanging for 15s until the ResumeAck SLA closes the socket.
+     */
+    legacyFallbackTimer;
+    /**
+     * Flips true on the first transition to READY and stays true for the life
+     * of the instance. Represents "initial handshake has completed at least
+     * once." Used by ``handleClose`` to distinguish startup failures (fail
+     * fast with a real error) from mid-session drops (enter reconnect loop).
+     * Decoupling this from specific ReadyState values means new handshake
+     * states can be added without silently changing close behavior.
+     */
+    hasCompletedHandshake = false;
     /** Expose current ready state so callers can detect post-await state changes. */
     getReadyState() {
         return this.readyState;
@@ -101,6 +159,18 @@ export class WebSocketClient extends EventEmitter {
         if (this.readyState !== newState) {
             const oldState = this.readyState;
             this.readyState = newState;
+            // First entry into READY marks the handshake as complete. Any close
+            // before this point is a startup failure; any close after is a
+            // mid-session drop that should trigger the reconnect loop.
+            if (newState === ReadyState.READY && !this.hasCompletedHandshake) {
+                this.hasCompletedHandshake = true;
+            }
+            // Leaving AWAITING_RESUME_ACK — either we got what we wanted
+            // (ResumeAck) or we gave up. Either way the legacy-fallback grace
+            // timer is no longer relevant.
+            if (oldState === ReadyState.AWAITING_RESUME_ACK) {
+                this.clearLegacyFallbackTimer();
+            }
             this.debug(`[WebSocketClient] Ready state transition: ${oldState} → ${newState}` +
                 (context ? ` | Context: ${context}` : '') +
                 ` | Session: ${this.sessionId?.substring(0, 8)}...` +
@@ -108,6 +178,308 @@ export class WebSocketClient extends EventEmitter {
             this.emit('readyStateChanged', { oldState, newState, context });
         }
     }
+    /**
+     * SLA for the Resume→ResumeAck round trip. Tuned to cover a cold graph load
+     * from postgres with some headroom, but tight enough that a silently dead
+     * socket reveals itself before the outer timeout layer even notices.
+     */
+    static RESUME_ACK_TIMEOUT_MS = 15000;
+    /**
+     * SLA for "something must happen after ResumeAck." If the server says it's
+     * awaiting nothing and we also have nothing to send, we stay in READY (not
+     * a stall). Otherwise a quiet socket for this long is a stall.
+     */
+    static POST_RESUME_PROGRESS_TIMEOUT_MS = 30000;
+    /**
+     * Grace period after a bare Ready arrives in AWAITING_RESUME_ACK. Long
+     * enough to cover the backend's ResumeAck round-trip latency, short
+     * enough that a true legacy server doesn't hang the SDK startup.
+     */
+    static LEGACY_READY_FALLBACK_MS = 2000;
+    addToOutbox(turn) {
+        if (this.outbox.has(turn.turn_id))
+            return;
+        this.outbox.set(turn.turn_id, turn);
+        this.outboxOrder.push(turn.turn_id);
+        this.metrics.turns_sent_total++;
+    }
+    removeFromOutbox(turnId) {
+        if (!this.outbox.delete(turnId))
+            return false;
+        const idx = this.outboxOrder.indexOf(turnId);
+        if (idx >= 0)
+            this.outboxOrder.splice(idx, 1);
+        return true;
+    }
+    outboxSnapshot() {
+        return this.outboxOrder
+            .map(id => this.outbox.get(id))
+            .filter((t) => t !== undefined);
+    }
+    /** Build the client half of a PendingTurn for the Resume envelope. */
+    pendingTurnDescriptor(turn) {
+        if (turn.type === 'UserQuery') {
+            return { turn_id: turn.turn_id, type: 'prompt' };
+        }
+        return {
+            turn_id: turn.turn_id,
+            type: 'tool_result',
+            ...(turn.tool_call_id ? { tool_call_id: turn.tool_call_id } : {}),
+        };
+    }
+    clearResumeAckTimer() {
+        if (this.resumeAckTimer) {
+            clearTimeout(this.resumeAckTimer);
+            this.resumeAckTimer = undefined;
+        }
+    }
+    clearPostResumeProgressTimer() {
+        if (this.postResumeProgressTimer) {
+            clearTimeout(this.postResumeProgressTimer);
+            this.postResumeProgressTimer = undefined;
+        }
+    }
+    startResumeAckTimer() {
+        this.clearResumeAckTimer();
+        this.resumeAckTimer = setTimeout(() => {
+            this.debug(`[WebSocketClient] ResumeAck timeout expired | ` +
+                `State: ${this.readyState} | ` +
+                `Session: ${this.sessionId?.substring(0, 8)}...`);
+            // The server didn't reply to Resume — treat the socket as dead and cycle it.
+            try {
+                if (this.ws && this.ws.readyState === WebSocket.OPEN) {
+                    this.ws.close(4001, 'ResumeAck timeout');
+                }
+            }
+            catch { /* cleanup: forcing close on an already-closing socket is expected to no-op */ }
+            this.emit('resumeAckTimeout');
+        }, WebSocketClient.RESUME_ACK_TIMEOUT_MS);
+        this.resumeAckTimer.unref?.();
+    }
+    clearLegacyFallbackTimer() {
+        if (this.legacyFallbackTimer) {
+            clearTimeout(this.legacyFallbackTimer);
+            this.legacyFallbackTimer = undefined;
+        }
+    }
+    /**
+     * Start the legacy-server fallback grace period.
+     *
+     * Called when we receive a bare ``Ready`` while still in
+     * ``AWAITING_RESUME_ACK``. In the normal v2 flow the server sends Ready
+     * first and ResumeAck right after — we want to wait for ResumeAck. But if
+     * the server is a pre-v2 deployment that never emits ResumeAck, waiting
+     * for the 15s SLA would stall every startup. Instead, we wait a short
+     * grace period; if no ResumeAck arrives, we treat the Ready we already
+     * saw as a legacy-protocol signal and transition to READY.
+     */
+    startLegacyFallbackTimer() {
+        // Idempotent — multiple Ready messages shouldn't pile up timers.
+        if (this.legacyFallbackTimer)
+            return;
+        this.legacyFallbackTimer = setTimeout(() => {
+            this.legacyFallbackTimer = undefined;
+            if (this.readyState !== ReadyState.AWAITING_RESUME_ACK)
+                return;
+            this.debug(`[WebSocketClient] Legacy fallback: Ready seen without ResumeAck, ` +
+                `assuming pre-v2 backend | outbox: ${this.outbox.size}`);
+            // Cancel the longer ResumeAck SLA — we've decided the server doesn't speak it.
+            this.clearResumeAckTimer();
+            this.setReadyState(ReadyState.READY, 'Legacy fallback after Ready without ResumeAck');
+            // Drain anything queued. Legacy flow has no TurnAck, so outbox
+            // entries will linger; a later reconnect (against a v2 backend)
+            // would reconcile them, but that's out of scope for this path.
+            void this.processOutbox();
+        }, WebSocketClient.LEGACY_READY_FALLBACK_MS);
+        this.legacyFallbackTimer.unref?.();
+    }
+    startPostResumeProgressTimer() {
+        this.clearPostResumeProgressTimer();
+        this.postResumeProgressTimer = setTimeout(() => {
+            this.debug(`[WebSocketClient] Post-ResumeAck stall | ` +
+                `State: ${this.readyState} | ` +
+                `Outbox size: ${this.outbox.size} | ` +
+                `Session: ${this.sessionId?.substring(0, 8)}...`);
+            this.metrics.stalls_detected_total++;
+            this.emit('stallDetected', {
+                stage: 'post_resume',
+                outboxSize: this.outbox.size,
+            });
+        }, WebSocketClient.POST_RESUME_PROGRESS_TIMEOUT_MS);
+        this.postResumeProgressTimer.unref?.();
+    }
+    /**
+     * Send the protocol-v2 Resume envelope. Called on every (re)connect before
+     * any turn can flow. Gives the backend enough state to reply with an
+     * authoritative ResumeAck.
+     */
+    sendResume() {
+        if (!this.ws || this.state !== ConnectionState.CONNECTED) {
+            this.debug('[WebSocketClient] sendResume: socket not connected, skipping');
+            return;
+        }
+        const payload = {
+            last_seen_checkpoint: this.latestCheckpointId ?? null,
+            pending_turns: this.outboxSnapshot().map(t => this.pendingTurnDescriptor(t)),
+        };
+        const formatted = `Resume ${JSON.stringify(payload)}\n`;
+        try {
+            this.ws.send(formatted);
+            this.debug(`[WebSocketClient] Sent Resume | ` +
+                `pending_turns: ${payload.pending_turns.length} | ` +
+                `last_seen_checkpoint: ${(this.latestCheckpointId ?? 'none').toString().substring(0, 8)}`);
+            this.setReadyState(ReadyState.AWAITING_RESUME_ACK, 'Resume sent');
+            this.startResumeAckTimer();
+        }
+        catch (err) {
+            this.debug('[WebSocketClient] Error sending Resume:', err);
+        }
+    }
+    /**
+     * Apply the server's authoritative view. This is the anti-deadlock primitive:
+     * every reconnect path runs through here and produces an explicit decision
+     * instead of both sides waiting on each other.
+     *
+     * Reconcile only runs on the FIRST ResumeAck of a (re)connect — the one we
+     * were explicitly waiting for. Any subsequent ResumeAcks (e.g. the server
+     * re-emits one as a loop-top heartbeat, or replies to our Resume probe) are
+     * informational; acting on them would double-send turns we've already
+     * replayed.
+     */
+    async handleResumeAck(message) {
+        const args = message?.data?.tool_args ?? {};
+        const currentCheckpoint = args.current_checkpoint;
+        const lastConsumedTurnId = args.last_consumed_turn_id;
+        const awaiting = args.awaiting ?? { type: 'none' };
+        const awaitingType = awaiting.type ?? 'none';
+        const awaitingToolCallId = awaiting.tool_call_id;
+        this.clearResumeAckTimer();
+        if (currentCheckpoint && currentCheckpoint !== this.latestCheckpointId) {
+            this.latestCheckpointId = currentCheckpoint;
+        }
+        const isReconcilePhase = this.readyState === ReadyState.AWAITING_RESUME_ACK;
+        this.metrics.resume_acks_total++;
+        this.emit('resumeAck', {
+            current_checkpoint: currentCheckpoint,
+            last_consumed_turn_id: lastConsumedTurnId,
+            awaiting: { type: awaitingType, tool_call_id: awaitingToolCallId },
+            outbox_size: this.outbox.size,
+        });
+        if (!isReconcilePhase) {
+            this.debug(`[WebSocketClient] Informational ResumeAck (already reconciled) | ` +
+                `State: ${this.readyState}`);
+            return;
+        }
+        // 1) Drop turns the server has already consumed (implicit ack via Resume).
+        if (lastConsumedTurnId && this.outbox.has(lastConsumedTurnId)) {
+            this.removeFromOutbox(lastConsumedTurnId);
+            this.debug(`[WebSocketClient] ResumeAck dropped acked turn from outbox | ` +
+                `turn_id: ${lastConsumedTurnId}`);
+        }
+        // 2) Reconcile what the server is waiting for against what we hold.
+        let reconcileFailedReason;
+        let resumeToSend;
+        if (awaitingType === 'tool_result') {
+            // Find a pending tool_result for the exact tool_call_id the server wants.
+            const candidate = this.outboxSnapshot().find(t => t.type === 'IDERetrievalAnswer' && t.tool_call_id === awaitingToolCallId);
+            if (candidate) {
+                resumeToSend = candidate;
+            }
+            else {
+                // Look for a MISMATCHING tool_result in the outbox — that's a true
+                // conflict (server awaits Y, client holds X). Emit reconcileFailed so
+                // the consumer can decide whether to abort or recompute.
+                const mismatchingToolResult = this.outboxSnapshot().find(t => t.type === 'IDERetrievalAnswer' && t.tool_call_id !== awaitingToolCallId);
+                if (mismatchingToolResult) {
+                    reconcileFailedReason = awaitingToolCallId
+                        ? `Server awaits tool_result for tool_call_id=${awaitingToolCallId}, but outbox holds tool_result for ${mismatchingToolResult.tool_call_id}`
+                        : 'Server awaits tool_result but outbox holds tool_result for a different tool_call_id';
+                }
+                // Otherwise: outbox is empty (or holds only UserQuery). This is a
+                // legitimate in-flight state — the tool is executing in the MCP layer
+                // and will send the tool_result when done. No failure here.
+            }
+        }
+        else if (awaitingType === 'prompt' || awaitingType === 'none') {
+            // Server is idle — any leftover outbox turns are stale consumed or never-delivered
+            // probes. We keep prompt-style outbox entries (they will flow via normal send),
+            // but drop any stale tool_results since the graph is no longer at that step.
+            for (const turn of this.outboxSnapshot()) {
+                if (turn.type === 'IDERetrievalAnswer') {
+                    this.removeFromOutbox(turn.turn_id);
+                    this.debug(`[WebSocketClient] ResumeAck dropped stale tool_result from outbox | ` +
+                        `turn_id: ${turn.turn_id}`);
+                }
+            }
+            // If we still have a UserQuery queued (client sent a prompt but never saw TurnAck)
+            // we should replay it.
+            const promptCandidate = this.outboxSnapshot().find(t => t.type === 'UserQuery');
+            if (promptCandidate)
+                resumeToSend = promptCandidate;
+        }
+        if (reconcileFailedReason) {
+            this.debug(`[WebSocketClient] Reconcile failed | ${reconcileFailedReason} | ` +
+                `outbox: ${this.outbox.size}`);
+            this.metrics.reconcile_failures_total++;
+            this.emit('reconcileFailed', { reason: reconcileFailedReason, awaiting });
+            // Don't lock up — transition to READY so the consumer's own error handling
+            // can decide what to do. The consumer can still send a new prompt.
+            this.setReadyState(ReadyState.READY, 'Reconcile failed');
+            return;
+        }
+        // 3) Happy path: transition to READY and replay anything still pending.
+        this.setReadyState(ReadyState.READY, 'ResumeAck reconciled');
+        if (resumeToSend) {
+            this.startPostResumeProgressTimer();
+            this.metrics.turns_replayed_total++;
+            await this.sendImmediately({
+                type: resumeToSend.type,
+                data: resumeToSend.data,
+                timestamp: resumeToSend.timestamp,
+            });
+            this.receivedResponsesSinceMessage = false;
+            this.setReadyState(ReadyState.MESSAGE_SENT, `Replayed ${resumeToSend.type} after reconcile`);
+            this.startReadyTimer();
+        }
+        else if (this.outbox.size > 0) {
+            // No specific replay target, but we have queued work — drain it.
+            await this.processOutbox();
+        }
+    }
+    handleTurnAck(message) {
+        const args = message?.data?.tool_args ?? {};
+        const turnId = args.turn_id;
+        const checkpointId = args.checkpoint_id;
+        if (!turnId) {
+            this.debug('[WebSocketClient] TurnAck received without turn_id, ignoring');
+            return;
+        }
+        const removed = this.removeFromOutbox(turnId);
+        if (checkpointId)
+            this.latestCheckpointId = checkpointId;
+        this.clearPostResumeProgressTimer();
+        // TurnAck is the v2 completion signal — the in-flight turn is consumed,
+        // so the TurnAck/Ready SLA started on send is no longer needed.
+        this.clearReadyTimer();
+        // Transition out of MESSAGE_SENT so the next queued turn can drain and
+        // a subsequent legacy-Ready (informational) no longer looks mid-send.
+        if (this.readyState === ReadyState.MESSAGE_SENT) {
+            this.setReadyState(ReadyState.READY, 'TurnAck received — turn durably consumed');
+            // If more turns piled up while we were waiting, drain one now.
+            void this.processOutbox();
+        }
+        this.metrics.turns_acked_total++;
+        // A TurnAck for a turn we no longer have in the outbox means the server
+        // re-ACK'd an already-dropped turn (idempotent replay confirmation).
+        if (!removed)
+            this.metrics.idempotent_replays_total++;
+        this.debug(`[WebSocketClient] TurnAck received | ` +
+            `turn_id: ${turnId} | ` +
+            `checkpoint: ${checkpointId?.substring(0, 8) || 'none'} | ` +
+            `outbox_removed: ${removed} | ` +
+            `outbox_remaining: ${this.outbox.size}`);
+        this.emit('turnAck', { turn_id: turnId, checkpoint_id: checkpointId });
+    }
     async handleReadyMessage(message) {
         const checkpointId = message.data?.tool_args?.checkpoint_id;
         const previousCheckpoint = this.latestCheckpointId;
@@ -124,6 +496,15 @@ export class WebSocketClient extends EventEmitter {
         }
         // Handle based on current state
         switch (this.readyState) {
+            case ReadyState.AWAITING_RESUME_ACK:
+                // In the normal v2 flow the server sends Ready then ResumeAck;
+                // Ready here is informational and we stay put until ResumeAck.
+                // Start a short legacy-fallback grace period so that a pre-v2
+                // backend (Ready only, no ResumeAck) doesn't stall the whole
+                // 15s ResumeAck SLA before we make progress.
+                this.debug('[WebSocketClient] Ready received while awaiting ResumeAck — starting legacy fallback grace');
+                this.startLegacyFallbackTimer();
+                break;
             case ReadyState.WAITING_INITIAL_READY:
                 this.clearReadyTimer();
                 this.setReadyState(ReadyState.READY, 'Initial Ready received after connection');
@@ -135,21 +516,20 @@ export class WebSocketClient extends EventEmitter {
                 await this.processOutbox();
                 break;
             case ReadyState.MESSAGE_SENT:
-                // Ready in MESSAGE_SENT should only be accepted if responses were received
-                if (!this.receivedResponsesSinceMessage) {
-                    this.debug(`[WebSocketClient] Unexpected Ready without responses | ` +
-                        `Checkpoint: ${checkpointId?.substring(0, 8) || 'none'} | ` +
-                        `This indicates server failed to process the message`);
-                    // Trigger checkpoint recovery - server acknowledged but didn't process
-                    this.clearReadyTimer();
-                    await this.initiateCheckpointRecovery('Ready received without any responses (server processing failure)');
-                    return;
-                }
-                // Normal case: Ready after responses signals server finished processing
-                this.debug('[WebSocketClient] Ready received, responses complete');
-                this.clearReadyTimer();
-                this.setReadyState(ReadyState.READY, 'Ready received, server finished processing');
-                await this.processOutbox();
+                // Under protocol-v2, ``TurnAck`` is the authoritative signal that our
+                // message was consumed — not ``Ready``. The backend sends Ready at
+                // the top of every loop iteration (including between turns and after
+                // handling a Resume probe), so Ready can legitimately arrive while
+                // we're still waiting on TurnAck — sometimes even before the graph's
+                // own output lands, purely due to network ordering. Acting on it
+                // (the old "server failed to process" path that force-reconnected)
+                // created a tight infinite loop where every UserQuery triggered a
+                // spurious reconnect.
+                //
+                // Treat Ready here as informational. The timer started in
+                // ``sendMessage`` / ``processOutbox`` still bounds how long we wait
+                // for TurnAck, and ``handleTurnAck`` is the real state transition.
+                this.debug('[WebSocketClient] Ready received in MESSAGE_SENT — ignoring (v2 relies on TurnAck)');
                 break;
             case ReadyState.READY:
                 this.debug('[WebSocketClient] Ready received while already in READY state (duplicate)');
@@ -189,32 +569,48 @@ export class WebSocketClient extends EventEmitter {
     async sendMessage(type, data) {
         // Reset idle timer on any outgoing message
         this.resetIdleTimer();
-        const messageToSend = { type, data, timestamp: Date.now() };
+        // Ensure every turn carries a turn_id — the backend idempotency key.
+        // Accepts a caller-supplied id (rare; only tests need that) and otherwise
+        // mints one here so existing call sites don't need updates.
+        const turn_id = (data && typeof data === 'object' && typeof data.turn_id === 'string' && data.turn_id)
+            ? data.turn_id
+            : uuid();
+        const dataWithTurnId = (data && typeof data === 'object')
+            ? { ...data, turn_id }
+            : { turn_id, payload: data };
+        const tool_call_id = type === 'IDERetrievalAnswer'
+            ? (dataWithTurnId.tool_id ?? dataWithTurnId.tool_call_id)
+            : undefined;
+        const turn = {
+            turn_id,
+            type,
+            data: dataWithTurnId,
+            tool_call_id,
+            timestamp: Date.now(),
+        };
+        this.addToOutbox(turn);
         this.debug(`[WebSocketClient] sendMessage called | ` +
             `Type: ${type} | ` +
+            `turn_id: ${turn_id} | ` +
             `ReadyState: ${this.readyState} | ` +
-            `ConnectionState: ${this.state}`);
-        // Save as last sent message for potential recovery
-        this.lastSentMessage = messageToSend;
-        this.debug(`[WebSocketClient] Last sent message cached | ` +
-            `Type: ${type} | ` +
-            `Timestamp: ${messageToSend.timestamp}`);
+            `ConnectionState: ${this.state} | ` +
+            `outbox_size: ${this.outbox.size}`);
+        // Save as last sent message for potential recovery (legacy path)
+        this.lastSentMessage = { type, data: dataWithTurnId, timestamp: turn.timestamp };
         if (this.readyState === ReadyState.READY && this.ws && this.state === ConnectionState.CONNECTED) {
-            // Send immediately
-            this.receivedResponsesSinceMessage = false; // Reset flag when sending new message
-            await this.sendImmediately(messageToSend);
+            this.receivedResponsesSinceMessage = false;
+            await this.sendImmediately({ type, data: dataWithTurnId, timestamp: turn.timestamp });
             this.setReadyState(ReadyState.MESSAGE_SENT, `Sent ${type} message`);
-            this.startReadyTimer(); // Expect Ready after server processes message
+            this.startReadyTimer();
         }
         else {
-            // Queue until Ready
-            this.pendingOutbox.push(messageToSend);
-            this.debug(`[WebSocketClient] Message queued | ` +
+            this.debug(`[WebSocketClient] Turn parked in outbox | ` +
                 `Type: ${type} | ` +
-                `Queue size: ${this.pendingOutbox.length} | ` +
+                `turn_id: ${turn_id} | ` +
+                `outbox_size: ${this.outbox.size} | ` +
                 `Reason: ReadyState=${this.readyState}, Connected=${this.state === ConnectionState.CONNECTED}`);
             if (this.state === ConnectionState.DISCONNECTED) {
-                this.debug('[WebSocketClient] Initiating connection due to queued message');
+                this.debug('[WebSocketClient] Initiating connection due to queued turn');
                 await this.connect();
             }
         }
@@ -226,29 +622,38 @@ export class WebSocketClient extends EventEmitter {
             `Type: ${msg.type} | ` +
             `Size: ${formatted.length} bytes`);
     }
+    /**
+     * Drain the outbox into the socket. Only one turn is in-flight at a time
+     * (MESSAGE_SENT gate enforces this). Called on ResumeAck (after reconcile)
+     * and on transitions back to READY.
+     */
     async processOutbox() {
         this.debug(`[WebSocketClient] Processing outbox | ` +
-            `Queue size: ${this.pendingOutbox.length} | ` +
+            `outbox_size: ${this.outbox.size} | ` +
             `ReadyState: ${this.readyState}`);
-        if (this.pendingOutbox.length > 0 && this.readyState === ReadyState.READY) {
-            const msg = this.pendingOutbox.shift();
-            this.debug(`[WebSocketClient] Sending queued message | ` +
-                `Type: ${msg.type} | ` +
-                `Queued at: ${msg.timestamp} | ` +
-                `Wait time: ${Date.now() - msg.timestamp}ms | ` +
-                `Remaining in queue: ${this.pendingOutbox.length}`);
-            this.receivedResponsesSinceMessage = false; // Reset flag when sending queued message
-            await this.sendImmediately(msg);
-            this.setReadyState(ReadyState.MESSAGE_SENT, `Sent queued ${msg.type} message`);
-            this.startReadyTimer(); // Expect Ready after server processes message
-        }
-        else if (this.pendingOutbox.length === 0) {
+        if (this.outbox.size === 0) {
             this.debug('[WebSocketClient] Outbox is empty, no messages to process');
+            return;
         }
-        else {
+        if (this.readyState !== ReadyState.READY) {
             this.debug(`[WebSocketClient] Cannot process outbox | ` +
                 `ReadyState: ${this.readyState} (expected READY)`);
+            return;
         }
+        const nextId = this.outboxOrder[0];
+        const turn = nextId ? this.outbox.get(nextId) : undefined;
+        if (!turn)
+            return;
+        this.debug(`[WebSocketClient] Sending queued turn | ` +
+            `Type: ${turn.type} | ` +
+            `turn_id: ${turn.turn_id} | ` +
+            `Queued at: ${turn.timestamp} | ` +
+            `Wait: ${Date.now() - turn.timestamp}ms | ` +
+            `outbox_remaining: ${this.outbox.size}`);
+        this.receivedResponsesSinceMessage = false;
+        await this.sendImmediately({ type: turn.type, data: turn.data, timestamp: turn.timestamp });
+        this.setReadyState(ReadyState.MESSAGE_SENT, `Sent queued ${turn.type} message`);
+        this.startReadyTimer();
     }
     startReadyTimer() {
         this.clearReadyTimer();
@@ -268,44 +673,32 @@ export class WebSocketClient extends EventEmitter {
             this.readyTimer = undefined;
         }
     }
+    /**
+     * Legacy recovery entry point.
+     *
+     * Under protocol v2 the outbox holds every un-ACK'd turn by turn_id and the
+     * Resume/ResumeAck handshake replays it authoritatively. All we need to do
+     * here is cycle the socket — the open handler will send Resume, the server
+     * will reply with ResumeAck, and reconciliation will happen automatically.
+     */
     async initiateCheckpointRecovery(reason) {
-        this.debug(`[WebSocketClient] ===== CHECKPOINT RECOVERY INITIATED ===== | ` +
+        this.debug(`[WebSocketClient] Forcing reconnect for recovery | ` +
             `Reason: ${reason} | ` +
             `Checkpoint: ${this.latestCheckpointId?.substring(0, 8) || 'none'} | ` +
-            `Last message type: ${this.lastSentMessage?.type} | ` +
-            `Current state: ${this.readyState}`);
-        this.setReadyState(ReadyState.CHECKPOINT_RECOVERY, reason);
-        // Disconnect current connection
-        this.debug('[WebSocketClient] Disconnecting for checkpoint recovery');
-        this.disconnect();
-        // Reconnect with checkpoint (URL will include checkpoint_id)
-        this.debug(`[WebSocketClient] Reconnecting with checkpoint | ` +
-            `Checkpoint ID: ${this.latestCheckpointId?.substring(0, 8) || 'none'} | ` +
-            `Session ID: ${this.sessionId?.substring(0, 8)}...`);
+            `outbox_size: ${this.outbox.size} | ` +
+            `State: ${this.readyState}`);
+        this.emit('checkpointRecovery', { reason, checkpoint: this.latestCheckpointId });
         try {
-            await this.establishConnection();
-            this.debug('[WebSocketClient] Reconnection successful during recovery');
-            // Wait for Ready state is set in handleReadyMessage when initial Ready is received
-            // Resend last message once Ready received
-            if (this.lastSentMessage) {
-                this.pendingOutbox.unshift(this.lastSentMessage);
-                this.debug(`[WebSocketClient] Last message re-queued for recovery | ` +
-                    `Type: ${this.lastSentMessage.type} | ` +
-                    `Original timestamp: ${this.lastSentMessage.timestamp}`);
-            }
-            else {
-                this.debug('[WebSocketClient] No last message to resend during recovery');
+            // Close the socket hard. handleClose will schedule reconnection via
+            // attemptReconnection, and the open handler will send a fresh Resume.
+            if (this.ws && this.ws.readyState === WebSocket.OPEN) {
+                this.ws.close(4002, `Recovery: ${reason}`);
             }
-            // Emit event for monitoring
-            this.emit('checkpointRecovery', { reason, checkpoint: this.latestCheckpointId });
         }
         catch (error) {
-            this.debug(`[WebSocketClient] Checkpoint recovery failed | ` +
+            this.debug(`[WebSocketClient] Recovery reconnect failed | ` +
                 `Error: ${error instanceof Error ? error.message : error}`);
-            this.setReadyState(ReadyState.IDLE, 'Recovery failed');
-            throw error;
         }
-        this.debug('[WebSocketClient] ===== CHECKPOINT RECOVERY COMPLETED =====');
     }
     disconnect() {
         // Hold a reference to the current socket to close it safely
@@ -388,17 +781,18 @@ export class WebSocketClient extends EventEmitter {
                     this.emit('connected');
                     this.debug(`[WebSocketClient] WebSocket connected | ` +
                         `Session: ${this.sessionId?.substring(0, 8)}... | ` +
-                        `Checkpoint: ${this.latestCheckpointId?.substring(0, 8) || 'none'}`);
-                    // Transition to waiting for initial Ready
-                    this.setReadyState(ReadyState.WAITING_INITIAL_READY, 'Connection opened, awaiting initial Ready');
-                    this.startReadyTimer(); // Wait for initial Ready
-                    // Start idle timer immediately upon connection
+                        `Checkpoint: ${this.latestCheckpointId?.substring(0, 8) || 'none'} | ` +
+                        `outbox: ${this.outbox.size}`);
+                    // Start idle timer and heartbeat immediately so dead sockets surface.
                     this.startIdleTimer();
-                    // Start heartbeat
                     this.startHeartbeat();
-                    // Process any queued messages (old mechanism, will be replaced by pendingOutbox)
-                    this.processQueuedMessages();
-                    this.debug('[WebSocketClient] Connection established, waiting for initial Ready signal');
+                    // Protocol v2: every (re)connect opens with a Resume envelope. The
+                    // server's ResumeAck is the authoritative signal — the old "wait for
+                    // Ready" handshake is left as a fallback for servers that don't yet
+                    // understand Resume (they emit Ready first, which we treat as info).
+                    this.setReadyState(ReadyState.AWAITING_RESUME_ACK, 'Connection opened, sending Resume');
+                    this.sendResume();
+                    this.debug('[WebSocketClient] Connection established, waiting for ResumeAck');
                     resolve();
                 });
                 ws.on('message', (data) => {
@@ -407,22 +801,33 @@ export class WebSocketClient extends EventEmitter {
                     this.resetIdleTimer();
                     this.lastPongReceived = Date.now(); // any message is also a ping
                     const message = data.toString();
-                    // Try to detect Ready messages
+                    // Intercept protocol-v2 control messages (ResumeAck, TurnAck, Ready)
+                    // before forwarding to the consumer. These are transport concerns.
                     try {
                         const parsed = JSON.parse(message);
-                        if (parsed && parsed.data && parsed.data.tool === 'Ready') {
-                            // Handle Ready message separately
+                        const tool = parsed?.data?.tool;
+                        if (tool === 'Ready') {
                             this.handleReadyMessage(parsed);
                             return;
                         }
+                        if (tool === 'ResumeAck') {
+                            this.handleResumeAck(parsed);
+                            return;
+                        }
+                        if (tool === 'TurnAck') {
+                            this.handleTurnAck(parsed);
+                            return;
+                        }
                     }
                     catch (e) {
-                        // Not JSON or not a Ready message, continue with normal flow
+                        // Not JSON or not a control message — fall through to consumer.
                     }
                     // Track that we received a non-Ready message (response)
                     if (this.readyState === ReadyState.MESSAGE_SENT) {
                         this.receivedResponsesSinceMessage = true;
                     }
+                    // A domain event after ResumeAck counts as forward progress — clear the stall timer.
+                    this.clearPostResumeProgressTimer();
                     // Emit non-Ready messages to AgentAPI
                     this.emit('message', message);
                 });
@@ -533,8 +938,11 @@ export class WebSocketClient extends EventEmitter {
             this.setState(ConnectionState.FAILED);
             this.emit('error', new Error('Authentication failed'));
         }
-        else if (this.readyState === ReadyState.WAITING_INITIAL_READY) {
-            // Connection closed during initial handshake - fail fast with clear error
+        else if (!this.hasCompletedHandshake) {
+            // No successful handshake has completed on this instance. Treat any
+            // close here as a startup failure — auth issues, wrong URL, server
+            // rejecting the protocol, etc. should surface as a real error rather
+            // than enter an indefinite reconnect loop that hides the root cause.
             this.setState(ConnectionState.FAILED);
             const errorMessage = this.buildConnectionErrorMessage(code, reason);
             this.emit('error', new Error(errorMessage));
@@ -573,13 +981,22 @@ export class WebSocketClient extends EventEmitter {
         this.isReconnecting = true;
         this.setState(ConnectionState.RECONNECTING);
         this.reconnectAttempts++;
+        this.metrics.reconnects_total++;
+        // Stamp the disconnect time on the first reconnect attempt of a cycle so
+        // the eventual `reconnected` event can carry an accurate downtime_ms.
+        if (this.reconnectAttempts === 1 && !this.lastDisconnectAt) {
+            this.lastDisconnectAt = Date.now();
+        }
+        this.emit('reconnecting', { attempt: this.reconnectAttempts });
         const delay = this.calculateBackoffDelay(this.reconnectAttempts);
         this.debug(`Attempting reconnection ${this.reconnectAttempts} in ${delay}ms`);
         this.reconnectTimer = setTimeout(async () => {
             try {
                 await this.establishConnection();
                 this.isReconnecting = false;
-                this.emit('reconnected');
+                const downtime_ms = this.lastDisconnectAt ? Date.now() - this.lastDisconnectAt : undefined;
+                this.lastDisconnectAt = undefined;
+                this.emit('reconnected', { downtime_ms });
             }
             catch (error) {
                 this.debug('Reconnection failed:', error instanceof Error ? error.message : error);
@@ -690,6 +1107,9 @@ export class WebSocketClient extends EventEmitter {
         this.clearIdleTimer();
         this.stopHeartbeat();
         this.clearReadyTimer();
+        this.clearResumeAckTimer();
+        this.clearPostResumeProgressTimer();
+        this.clearLegacyFallbackTimer();
         if (this.reconnectTimer) {
             clearTimeout(this.reconnectTimer);
             this.reconnectTimer = undefined;
@@ -704,17 +1124,35 @@ export class WebSocketClient extends EventEmitter {
             `State: ${this.state} | ` +
             `ReadyState: ${this.readyState} | ` +
             `Queued messages: ${this.messageQueue.length} | ` +
-            `Pending outbox: ${this.pendingOutbox.length}`);
+            `Outbox: ${this.outbox.size}`);
         this.clearTimers();
         this.disconnect();
         this.messageQueue = [];
-        this.pendingOutbox = [];
+        this.outbox.clear();
+        this.outboxOrder.length = 0;
         this.isReconnecting = false;
         this.latestCheckpointId = undefined;
         this.lastSentMessage = undefined;
+        this.hasCompletedHandshake = false;
         this.setReadyState(ReadyState.IDLE, 'Cleanup performed');
         this.removeAllListeners();
         this.debug('[WebSocketClient] Cleanup completed');
     }
+    // --- Test hooks -----------------------------------------------------------
+    // These are intentionally tiny, read-only accessors so the chaos harness
+    // can assert on internal state without resorting to `any` casts. They are
+    // not part of the public SDK API.
+    /** @internal */
+    __getOutboxIds() {
+        return [...this.outboxOrder];
+    }
+    /** @internal */
+    __getOutboxSize() {
+        return this.outbox.size;
+    }
+    /** @internal */
+    __getLatestCheckpointId() {
+        return this.latestCheckpointId;
+    }
 }
 //# sourceMappingURL=websocket.js.map