mixdog 0.9.87 → 0.9.89

package/src/runtime/agent/orchestrator/providers/openai-ws-stream.mjs

@@ -19,7 +19,8 @@ import {
     extractCachedTokens,
     appendAgentTrace,
 } from '../agent-trace.mjs';
-import { populateHttpStatusFromMessage } from './retry-classifier.mjs';
+import { typedStatusFrom } from './retry-classifier.mjs';
+import { stampStreamOutcome, STREAM_TRANSPORTS } from './lib/stream-outcome.mjs';
 import { makeInvalidToolArgsMarker } from './openai-compat-stream.mjs';
 import { createLeakGuard, createToolCallDedupe, dedupeToolCallList } from './anthropic-leaked-toolcall.mjs';
 import {
@@ -51,7 +52,6 @@ import {
 } from './openai-ws-events.mjs';
 import {
     createActiveToolItemTracker,
-    hasCompleteToolCall as sharedHasCompleteToolCall,
 } from './tool-stream-state.mjs';
 import {
     enableSessionTransportTracking,
@@ -125,6 +125,21 @@ function _writeWsLifecycleTrace(lifecycle) {
     process.stderr.write(`[ws-trace] t=${new Date().toISOString()} lifecycle=${lifecycle}\n`);
 }
 
+// Wire-level `end_turn` on a terminal Responses frame (codex-rs
+// codex-api/src/sse/responses.rs ResponseCompleted.end_turn: Option<bool>,
+// consumed in core/src/session/turn.rs:2299 as "false ⇒ needs follow-up").
+// The field is optional: only a real boolean is normalized; anything else —
+// including a missing field — stays undefined so absence is preserved and no
+// caller can mistake "server said nothing" for "server said true/false".
+export function _endTurnFromEvent(event) {
+    if (!event || typeof event !== 'object') return undefined;
+    const fromResponse = event.response?.end_turn;
+    if (typeof fromResponse === 'boolean') return fromResponse;
+    const topLevel = event.end_turn;
+    if (typeof topLevel === 'boolean') return topLevel;
+    return undefined;
+}
+
 function _responseItemKey(item, fallbackIndex = 0) {
     if (!item || typeof item !== 'object') return `primitive:${fallbackIndex}`;
     if (item.id) return `${item.type || 'item'}:id:${item.id}`;
@@ -309,12 +324,6 @@ export async function _streamResponse({
     // input starts streaming (before it lands in pendingCalls/toolCalls).
     // Gates partial-final SUCCESS so a stall mid tool-input never looks text-only.
     let _toolInFlight = false;
-    // Early tool-call settle latch. Set when a fully-formed tool call is
-    // available but the server never emits response.completed/response.done —
-    // we resolve successfully anyway and surface `closeSocket` so the pool
-    // discards (never reuses) the socket, which may still carry the missing
-    // terminal frames as orphans.
-    let _earlySettled = false;
     // Fix 2: cross-path name+args dedupe. A text-leaked synthetic and an
     // identical native function_call must fire onToolCall exactly once. Every
     // dispatch site routes through emitToolCallDedupe.
@@ -494,6 +503,9 @@ export async function _streamResponse({
     let usage;
     let stopReason = null;
     let incompleteReason = null;
+    // Normalized wire `end_turn` from the terminal frame; stays undefined
+    // unless the server actually supplied a boolean.
+    let endTurn;
     let done = false;
     let terminalError = null;
     // Mid-stream retry classifier needs to distinguish "stream died before we
@@ -640,19 +652,41 @@ export async function _streamResponse({
                 firstMeaningfulTimer = null;
             }
         };
+        // Streamed partial state for a stream that ended WITHOUT the terminal
+        // response.completed/done frame. The transport never synthesizes a
+        // completion: it reports the continuation plus what it had, and the
+        // loop decides (partial text failure / explicit tool-call turn).
+        // `pendingToolUse` means "a tool input never finished streaming" — a
+        // fully assembled call is not pending.
+        const _attachWsPartialState = (err) => {
+            if (!err) return err;
+            // Fold the held leak-guard tail into `content` FIRST so the partial
+            // snapshot keeps legitimate trailing text; finish() then skips
+            // flushLeak (terminalError set) without losing it.
+            try { flushLeak(); } catch { /* best-effort */ }
+            try {
+                err.partialContent = content;
+                err.partialToolCalls = toolCalls.length ? toolCalls.slice() : undefined;
+                err.pendingToolUse = pendingCalls.size > 0
+                    || activeToolItems.size > 0
+                    || _toolInFlight === true;
+                err.partialModel = model || undefined;
+            } catch { /* best-effort enrichment */ }
+            return err;
+        };
         const resetInterChunk = () => {
             if (interChunkTimer) clearTimeout(interChunkTimer);
             interChunkTimer = setTimeout(() => {
-                // Early settle: a complete tool call is already captured with no
-                // deferred salvage pending, so silence here is the missing
-                // response.completed/done — resolve with the tool call instead
-                // of a stall error the loop would retry.
-                if (settleEarlyOnToolCall('inter_chunk')) return;
                 if (process.env.MIXDOG_DEBUG_AGENT) {
                     process.stderr.write(`[agent-trace] ws-timeout kind=inter-chunk afterMs=${interChunkMs}\n`);
                 }
                 traceWsTimeout('inter_chunk_timeout', interChunkMs);
                 terminalError = new Error(`WS stream: inter-chunk inactivity for ${interChunkMs}ms`);
+                // No terminal frame arrived, so this is a CONTINUATION, never a
+                // synthesized completion. Attach the streamed partial state so
+                // the loop's partial/tool recovery can finalize it explicitly.
+                terminalError.streamStalled = true;
+                _attachWsPartialState(terminalError);
                 try { socket.close(4000, 'inter_chunk_timeout'); } catch {}
                 // Same half-open guard as the pre-stream watchdog: reject directly
                 // so a stuck socket.close() cannot leave the Promise pending.
@@ -666,29 +700,12 @@ export async function _streamResponse({
             if (!semanticIdleEnabled) return;
             if (semanticIdleTimer) clearTimeout(semanticIdleTimer);
             semanticIdleTimer = setTimeout(() => {
-                // Early settle before treating tool-work silence as a stall: if
-                // a complete tool call is available and no deferred salvage
-                // remains, the turn result is already final even without
-                // response.completed/done.
-                if (settleEarlyOnToolCall('semantic_idle')) return;
                 traceWsTimeout('semantic_idle_timeout', semanticIdleMs);
                 terminalError = streamStalledError('Responses WS', semanticIdleMs, { emittedToolCall: !!midState?.emittedToolCall });
                 // Partial-final recovery: attach streamed partial state so
                 // a wedged FINAL no-tool summary can be accepted as partial-final
                 // success by the loop. pendingToolUse gates out mid-flight tools.
-                // Fold the held leak-guard tail into `content` FIRST so the
-                // partial snapshot below keeps legitimate trailing text; finish()
-                // then skips flushLeak (terminalError set) without losing it.
-                flushLeak();
-                try {
-                    terminalError.partialContent = content;
-                    terminalError.partialToolCalls = toolCalls.length ? toolCalls.slice() : undefined;
-                    terminalError.pendingToolUse = pendingCalls.size > 0
-                        || !!midState?.emittedToolCall
-                        || toolCalls.length > 0
-                        || _toolInFlight === true;
-                    terminalError.partialModel = model || undefined;
-                } catch { /* best-effort enrichment */ }
+                _attachWsPartialState(terminalError);
                 try { terminalError.wsCloseCode = 4000; } catch {}
                 try { socket.close(4000, 'semantic_idle_timeout'); } catch {}
                 finish();
@@ -737,7 +754,33 @@ export async function _streamResponse({
             // Flush any partial-sentinel tail held back mid-stream so
             // legitimate trailing text is never lost (streamed-text path).
             cleanup();
-            if (terminalError) { reject(terminalError); return; }
+            if (terminalError) {
+                // Canonical stream-outcome contract: ONE authoritative record
+                // of what this WS turn produced, stamped on every reject path
+                // (watchdog stall, abnormal close, server error, abort). Legacy
+                // aliases are preserved by stampStreamOutcome().
+                try {
+                    stampStreamOutcome(terminalError, {
+                        transport: STREAM_TRANSPORTS.WS,
+                        provider: 'openai-responses',
+                        terminalObserved: midState.sawCompleted === true,
+                        continuation: midState.sawCompleted !== true,
+                        textEmitted: midState.emittedText === true,
+                        textObservedChars: content.length,
+                        reasoningEmitted: midState.emittedReasoning === true,
+                        toolCallsStarted: midState.startedToolCall === true || pendingCalls.size > 0,
+                        toolCallsComplete: toolCalls.length,
+                        toolCallsDispatched: midState.emittedToolCall === true ? Math.max(1, toolCalls.length) : 0,
+                        pendingToolInput: pendingCalls.size > 0 || _toolInFlight === true,
+                        userAbort: midState.userAbort === true,
+                        // A terminal frame with end_turn=false keeps the same
+                        // user turn open: terminal observed, still continuation.
+                        ...(endTurn === false ? { continuationDeclared: true } : {}),
+                    });
+                } catch { /* stamping is best-effort; aliases already set */ }
+                reject(terminalError);
+                return;
+            }
             resolve({
                 content,
                 model,
@@ -754,44 +797,13 @@ export async function _streamResponse({
                 // for the WS path (sendViaWebSocket spreads this result
                 // through to the provider caller unchanged).
                 ...(stopReason === 'length' && content.length > 0 ? { truncated: true } : {}),
-                // Early tool-call settle: tell the caller to drop (not pool)
-                // this socket. The server may still emit the skipped
-                // response.completed/done as orphan frames; reusing the socket
-                // would read them as the next request's stream and wedge it.
-                ...(_earlySettled ? { closeSocket: true } : {}),
                 incompleteReason: incompleteReason || undefined,
+                // Only present when the terminal frame carried the wire field.
+                ...(typeof endTurn === 'boolean' ? { endTurn } : {}),
                 responseId: responseId || undefined,
                 serviceTier: responseServiceTier || undefined,
             });
         };
-        // Early tool-call settle guard. A fully-formed tool call (real call_id +
-        // name, not a deferred salvage placeholder) means the turn's actionable
-        // result is already complete. When the server then goes silent without
-        // response.completed/response.done, resolve successfully with that tool
-        // call rather than surfacing a stall error the loop gates out via
-        // pendingToolUse and retries. Returns true if it settled the stream.
-        const hasCompleteToolCall = () => sharedHasCompleteToolCall({
-            toolCalls,
-            pendingSize: pendingCalls.size,
-            activeSize: activeToolItems.size,
-            toolInFlight: _toolInFlight,
-        });
-        const settleEarlyOnToolCall = (reason) => {
-            if (done || terminalError) return false;
-            if (!hasCompleteToolCall()) return false;
-            _earlySettled = true;
-            // Treat as a normal completion for downstream state: we have the
-            // canonical tool call, only the terminal lifecycle frame is missing.
-            midState.sawCompleted = true;
-            midState.wsEarlySettle = reason;
-            done = true;
-            // Close and mark keep=false (via closeSocket in the resolve). The
-            // late/never response.completed cannot then leak onto a pooled reuse.
-            try { socket.close(4000, `early_settle_${reason}`); } catch {}
-            finish();
-            return true;
-        };
-
         messageHandler = (data) => {
             const frameBytes = _incomingFrameByteLength(data);
             if (maxIncomingFrameBytes > 0 && frameBytes != null && frameBytes > maxIncomingFrameBytes) {
@@ -837,7 +849,10 @@ export async function _streamResponse({
                 const err = new Error(event.error.message || 'Responses WS error');
                 try {
                     err.payload = event.error;
-                    populateHttpStatusFromMessage(err);
+                    // Typed evidence only: a server error event without a
+                    // numeric status stays unclassified (surfaced, not retried).
+                    const typed = typedStatusFrom(event.error, event);
+                    if (typed) err.httpStatus = typed;
                 } catch {}
                 terminalError = err;
                 finish();
@@ -1187,6 +1202,13 @@ export async function _streamResponse({
                         finish();
                         break;
                     }
+                    // Wire-level completion signal (optional). Captured after
+                    // salvage validation so a failed stream never reports a
+                    // turn-completion hint; absence stays undefined.
+                    {
+                        const wireEndTurn = _endTurnFromEvent(event);
+                        if (typeof wireEndTurn === 'boolean') endTurn = wireEndTurn;
+                    }
                     midState.sawCompleted = true;
                     done = true;
                     finish();
@@ -1212,7 +1234,10 @@ export async function _streamResponse({
                             new Error(`${errLabel} response.done failed: ${msg}`),
                             { responseFailed: event },
                         );
-                        populateHttpStatusFromMessage(terminalError, msg);
+                        {
+                            const typed = typedStatusFrom(event.response?.error, event.error, event);
+                            if (typed) terminalError.httpStatus = typed;
+                        }
                         done = true;
                         finish();
                         break;
@@ -1244,6 +1269,13 @@ export async function _streamResponse({
                         finish();
                         break;
                     }
+                    // Success-shaped response.done (status '' or 'completed')
+                    // is the terminal frame for streams that never emit
+                    // response.completed — carry the same optional end_turn.
+                    {
+                        const wireEndTurn = _endTurnFromEvent(event);
+                        if (typeof wireEndTurn === 'boolean') endTurn = wireEndTurn;
+                    }
                     midState.sawCompleted = true;
                     done = true;
                     finish();
@@ -1280,20 +1312,23 @@ export async function _streamResponse({
                     terminalError = Object.assign(new Error(`${errLabel} response.failed: ${msg}`), {
                         responseFailed: event,
                     });
-                    // Sniff the server message for transient/auth/permanent
-                    // hints so the handshake / mid-stream retry classifiers
-                    // can route by httpStatus. Without this, server-side
-                    // events like "Our servers are currently overloaded"
-                    // surfaced as unclassified errors and skipped the
-                    // 5xx retry bucket entirely.
-                    populateHttpStatusFromMessage(terminalError, msg);
+                    // TYPED status only (event/error payload). A failure whose
+                    // only evidence is message text is 'unknown' to the retry
+                    // classifiers and is surfaced instead of replayed.
+                    {
+                        const typed = typedStatusFrom(event.response?.error, event.error, event);
+                        if (typed) terminalError.httpStatus = typed;
+                    }
                     finish();
                     break;
                 }
                 case 'error': {
                     const errMsg = String(event.message || event.error?.message || 'unknown');
                     terminalError = new Error(`${errLabel} error: ${errMsg}`);
-                    populateHttpStatusFromMessage(terminalError, errMsg);
+                    {
+                        const typed = typedStatusFrom(event.error, event);
+                        if (typed) terminalError.httpStatus = typed;
+                    }
                     finish();
                     break;
                 }