mixdog 0.9.152 → 0.9.154

package/src/runtime/agent/orchestrator/providers/openai-oauth-http-sse.mjs

@@ -41,6 +41,10 @@ import { createActiveToolItemTracker } from './tool-stream-state.mjs';
 import { createProviderReplay } from './lib/provider-replay.mjs';
 import { parseProviderJsonBatch } from './stream-json-pool.mjs';
 import { envFlag as _envFlag } from './lib/env-utils.mjs';
+import {
+    activateCodexTurnState,
+    captureCodexTurnState,
+} from './openai-turn-state.mjs';
 export { envPositiveInt as _envPositiveInt } from './lib/env-utils.mjs';
 export { _envFlag };
 
@@ -164,7 +168,13 @@ function _pushOutputTextAnnotations(part, citations, citationKeys) {
     }
 }
 
-function _buildOpenAIHttpFallbackHeaders({ auth, cacheKey, statelessConversation = false }) {
+export function _buildOpenAIHttpFallbackHeaders({
+    auth,
+    cacheKey,
+    statelessConversation = false,
+    poolKey = null,
+    turnId = null,
+}) {
     if (auth?.type === 'openai-direct') {
         // Public API-key auth: Bearer <OPENAI_API_KEY>, no chatgpt-account-id /
         // originator (mirrors openai-ws-pool _buildHandshakeHeaders' direct
@@ -195,6 +205,8 @@ function _buildOpenAIHttpFallbackHeaders({ auth, cacheKey, statelessConversation
         headers['session-id'] = sid;
         headers['thread-id'] = sid;
     }
+    const turnState = activateCodexTurnState(poolKey, turnId);
+    if (turnState) headers['x-codex-turn-state'] = turnState;
     return headers;
 }
 
@@ -251,7 +263,14 @@ export async function sendViaHttpSse({
     const totalTimeout = createPassthroughSignal(externalSignal);
     const statelessConversation = opts?.statelessConversation === true
         || _envFlag('MIXDOG_OAI_STATELESS_HTTP', false);
-    const headers = _buildOpenAIHttpFallbackHeaders({ auth, cacheKey, statelessConversation });
+    const turnId = opts?.turnId || opts?.codexTurnId || opts?.session?.turnId || null;
+    const headers = _buildOpenAIHttpFallbackHeaders({
+        auth,
+        cacheKey,
+        statelessConversation,
+        poolKey,
+        turnId,
+    });
     const fetchStartedAt = Date.now();
     const responsesUrl = auth?.type === 'openai-direct'
         ? OPENAI_DIRECT_RESPONSES_URL
@@ -380,6 +399,10 @@ export async function sendViaHttpSse({
         totalTimeout.cleanup();
         throw err;
     }
+    if (auth?.type !== 'openai-direct') {
+        const responseTurnState = response.headers?.get?.('x-codex-turn-state');
+        if (responseTurnState) captureCodexTurnState(poolKey, turnId, responseTurnState);
+    }
     if (!response.body) {
         totalTimeout.cleanup();
         throw Object.assign(new Error('OpenAI OAuth HTTP fallback returned no response body'), { initialResponseError: true });

package/src/runtime/agent/orchestrator/providers/openai-oauth-ws.mjs

@@ -32,6 +32,7 @@ import {
     grokCacheChainTraceFields,
     appendAgentTrace,
 } from '../agent-trace.mjs';
+import { traceCacheBreak } from '../cache-break-trace.mjs';
 import {
     classifyHandshakeError,
     classifyMidstreamError,
@@ -65,7 +66,10 @@ import {
     parseToolSearchArgs,
     _streamResponse,
 } from './openai-ws-stream.mjs';
-import { _buildResponseCreateFrame } from './openai-ws-delta.mjs';
+import {
+    _buildResponseCreateFrame,
+    _requestInputMismatchDiagnostics,
+} from './openai-ws-delta.mjs';
 import { resolveOpenAiTransportPolicy } from './openai-transport-policy.mjs';
 import { envPositiveInt } from './lib/env-utils.mjs';
 
@@ -589,6 +593,10 @@ export async function sendViaWebSocket({
     // One compact row per logical iteration. Values aggregate all handshake
     // and mid-stream attempts, including warmup, without retaining request data.
     const sendSpan = {
+        admissionQueueWaitMs: Math.max(0, Number(sendOpts?._providerAdmission?.queueWaitMs) || 0),
+        providerConcurrentRequests: Math.max(1, Number(sendOpts?._providerAdmission?.active) || 1),
+        providerQueuedRequests: Math.max(0, Number(sendOpts?._providerAdmission?.queued) || 0),
+        poolOwnerWaitMs: 0,
         poolAcquireMs: 0,
         requestBuildSerializationMs: 0,
         preResponseCreatedMs: 0,
@@ -598,14 +606,44 @@ export async function sendViaWebSocket({
         acquireAttempts: 0,
         acquireMode: null,
         emitted: false,
+        timing: null,
     };
-    const emitSendSpan = (outcome) => {
-        if (sendSpan.emitted) return;
+    const emitSendSpan = (outcome, target = null) => {
+        if (sendSpan.emitted) {
+            if (target && sendSpan.timing) {
+                try { target.transportTiming = sendSpan.timing; } catch {}
+            }
+            return sendSpan.timing;
+        }
         sendSpan.emitted = true;
+        const socketAcquireMs = Math.max(0, sendSpan.poolAcquireMs - sendSpan.poolOwnerWaitMs);
+        const timing = {
+            transport: 'websocket',
+            outcome,
+            admissionQueueWaitMs: sendSpan.admissionQueueWaitMs,
+            providerConcurrentRequests: sendSpan.providerConcurrentRequests,
+            providerQueuedRequests: sendSpan.providerQueuedRequests,
+            poolOwnerWaitMs: sendSpan.poolOwnerWaitMs,
+            socketAcquireMs,
+            poolAcquireMs: sendSpan.poolAcquireMs,
+            requestBuildSerializationMs: sendSpan.requestBuildSerializationMs,
+            preResponseCreatedMs: sendSpan.preResponseCreatedMs,
+            providerFirstEventMs: sendSpan.firstEventMs,
+            retryBackoffMs: sendSpan.retryBackoffMs,
+            handshakeRetries: sendSpan.handshakeRetries,
+            acquireAttempts: sendSpan.acquireAttempts,
+            acquireMode: sendSpan.acquireMode || 'failed',
+        };
+        sendSpan.timing = timing;
         const payload = {
             provider: traceProvider,
             model: useModel,
             transport: 'websocket',
+            admission_queue_wait_ms: timing.admissionQueueWaitMs,
+            provider_concurrent_requests: timing.providerConcurrentRequests,
+            provider_queued_requests: timing.providerQueuedRequests,
+            pool_owner_wait_ms: timing.poolOwnerWaitMs,
+            socket_acquire_ms: timing.socketAcquireMs,
             acquire_mode: sendSpan.acquireMode || 'failed',
             acquire_attempts: sendSpan.acquireAttempts,
             handshake_retries: sendSpan.handshakeRetries,
@@ -625,6 +663,10 @@ export async function sendViaWebSocket({
                 payload,
             });
         } catch {}
+        if (target) {
+            try { target.transportTiming = timing; } catch {}
+        }
+        return timing;
     };
     // Single caller-visible recovery path for both handshake/acquire retries
     // and retryable stream failures. The session/TUI stage bridge renders this
@@ -699,6 +741,7 @@ export async function sendViaWebSocket({
         } catch (err) {
             _stampWarmup(err);
             sendSpan.poolAcquireMs += performance.now() - handshakeStart;
+            sendSpan.poolOwnerWaitMs += Math.max(0, Number(err?.ownerWaitMs) || 0);
             // Provenance only; policy remains provider-owned below. This lets
             // the direct wrapper distinguish an upgrade rejection from an
             // application error carrying the same HTTP status.
@@ -744,6 +787,7 @@ export async function sendViaWebSocket({
             const retryable = classifier
                 && Number(err?.httpStatus || 0) !== 401
                 && Number(err?.httpStatus || 0) !== 426
+                && err?.unsafeToRetry !== true
                 && !externalSignal?.aborted;
             if (retryable && attemptIndex < MAX_MIDSTREAM_RETRIES) {
                 if (!firstAttemptError) {
@@ -777,7 +821,7 @@ export async function sendViaWebSocket({
                     try { err.midstreamClassifier = classifier; } catch {}
                     try { err.wsRetriesExhausted = true; } catch {}
                 }
-                emitSendSpan('error');
+                emitSendSpan('error', err);
                 throw _stampTool(_stampLiveText(err));
             }
             if (attemptIndex > 0 && firstAttemptError) {
@@ -794,6 +838,7 @@ export async function sendViaWebSocket({
         }
         const { entry, reused } = acquired;
         sendSpan.poolAcquireMs += performance.now() - handshakeStart;
+        sendSpan.poolOwnerWaitMs += Math.max(0, Number(acquired?.ownerWaitMs) || 0);
         sendSpan.acquireMode = acquired.prewarmed
             ? 'prewarmed'
             : entry?.ephemeral ? 'ephemeral' : (reused ? 'reused' : 'new');
@@ -863,6 +908,7 @@ export async function sendViaWebSocket({
         let strippedResponseItems = 0;
         let skippedResponseItems = 0;
         let responseOutputMismatch = null;
+        let requestInputMismatch = null;
         let wireFrameHadTurnState = false;
         let wireFrameMetadataTrace = _metadataTrace(null);
         let framePrefixHash = null;
@@ -950,7 +996,6 @@ export async function sendViaWebSocket({
                 entry.lastInputPrefixHash = createHash('sha256')
                     .update(JSON.stringify(warmupInputArr))
                     .digest('hex');
-                entry.turnStateFromPrewarm = true;
                 try {
                     const warmupPayload = {
                         provider: traceProvider,
@@ -976,7 +1021,7 @@ export async function sendViaWebSocket({
             // Codex performs generate:false during session startup, then hands
             // this live client session to the first real turn. Startup callers
             // stop here; the pooled entry retains its response id, request
-            // snapshot, socket, and turn-state for the later real send.
+            // snapshot and socket for the later real send.
             if (sendOpts?._startupPrewarmOnly === true) {
                 const responseId = warmupResult?.responseId
                     || startupWarmupResponseId
@@ -984,8 +1029,7 @@ export async function sendViaWebSocket({
                     || null;
                 if (responseId) entry.startupWarmupResponseId = responseId;
                 else releaseWebSocket({ entry, poolKey, keep: false });
-                emitSendSpan('ok');
-                return {
+                const out = {
                     content: '',
                     model: warmupResult?.model || useModel,
                     toolCalls: [],
@@ -1001,6 +1045,8 @@ export async function sendViaWebSocket({
                         ? { entry, poolKey, cacheKey }
                         : null,
                 };
+                out.transportTiming = emitSendSpan('ok');
+                return out;
             }
 
             // A completed generate:false prewarm is a valid continuation
@@ -1015,6 +1061,7 @@ export async function sendViaWebSocket({
             strippedResponseItems = delta.strippedResponseItems || 0;
             skippedResponseItems = delta.skippedResponseItems || 0;
             responseOutputMismatch = delta.responseOutputMismatch || null;
+            requestInputMismatch = delta.requestInputMismatch || null;
             const wireFrame = _withCodexWsClientMetadata(frame, entry, useCodexWsClientMetadata, codexMetadataContext);
             wireFrameHadTurnState = !!wireFrame?.client_metadata?.['x-codex-turn-state'];
             wireFrameMetadataTrace = _metadataTrace(wireFrame?.client_metadata);
@@ -1312,6 +1359,12 @@ export async function sendViaWebSocket({
             body: requestBody,
             traceProvider,
         });
+        if (cacheContinuityResetReason === 'input_prefix_mismatch' && !requestInputMismatch) {
+            requestInputMismatch = _requestInputMismatchDiagnostics(
+                requestBody?.input,
+                entry?.lastRequestInput,
+            );
+        }
         if (result.responseId && keepResponseChain) {
             entry.lastResponseId = result.responseId;
             entry.lastRequestSansInput = _stableStringify(_sansInput(requestBody, {
@@ -1431,6 +1484,23 @@ export async function sendViaWebSocket({
                 });
             } catch {}
         }
+        if (cacheObservation.actualMiss) {
+            traceCacheBreak({
+                sessionId: poolKey,
+                iteration,
+                classification: 'provider_miss',
+                reason: cacheObservation.missReason || 'warm_session_cache_miss',
+                source: 'provider_usage',
+                provider: traceProvider,
+                model: liveModel,
+                transport: 'websocket',
+                cachedTokens: cacheObservation.cachedTokens,
+                promptTokens: cacheObservation.promptTokens,
+                uncachedTokens: cacheObservation.uncachedTokens,
+                cacheRatio: cacheObservation.cacheRatio,
+                actualCacheMiss: true,
+            }, { traceFn: _agentTraceFn });
+        }
         // Rebase after a genuine provider retreat so one eviction produces one
         // diagnostic instead of a long run of duplicate "dropped" rows. The
         // request that exposed the retreat has already rebuilt the prefix; its
@@ -1494,7 +1564,7 @@ export async function sendViaWebSocket({
                 ws_client_metadata_has_turn_state: wireFrameHadTurnState,
                 ws_entry_turn_state_available: useCodexWsClientMetadata && !!entry.turnState,
                 // Fingerprint only. The token is a routing credential, so it is
-                // hashed like every other handshake secret; the length still
+                // hashed rather than logged; the length still
                 // distinguishes a real server token from a stub value, and the
                 // hash shows whether one session keeps a single pin or is
                 // re-issued (reconnect / turn rollover).
@@ -1505,6 +1575,7 @@ export async function sendViaWebSocket({
                 chain_stripped_response_items: strippedResponseItems,
                 chain_skipped_response_items: skippedResponseItems,
                 ...(responseOutputMismatch || {}),
+                ...(requestInputMismatch || {}),
                 chain_response_items: Array.isArray(result.responseItems) ? result.responseItems.length : 0,
                 body_input_items: Array.isArray(requestBody.input) ? requestBody.input.length : null,
                 frame_input_items: Array.isArray(frame.input) ? frame.input.length : null,
@@ -1532,10 +1603,29 @@ export async function sendViaWebSocket({
                 const intentionalTransition = typeof sendOpts?.cacheBreakIntent === 'string'
                     ? sendOpts.cacheBreakIntent
                     : null;
+                traceCacheBreak({
+                    sessionId: poolKey,
+                    iteration,
+                    classification: intentionalTransition ? 'intentional' : 'provider_transition',
+                    reason: mode === 'delta' ? deltaReason : (deltaReason || 'full_frame'),
+                    source: 'openai_ws_delta',
+                    provider: traceProvider,
+                    model: liveModel,
+                    transport: 'websocket',
+                    intentionalTransition,
+                    cachedTokens: cacheObservation.cachedTokens,
+                    promptTokens: cacheObservation.promptTokens,
+                    uncachedTokens: cacheObservation.uncachedTokens,
+                    cacheRatio: cacheObservation.cacheRatio,
+                    actualCacheMiss: cacheObservation.actualMiss,
+                    ...(requestInputMismatch || {}),
+                }, { traceFn: null });
                 _agentTraceFn({
                     sessionId: poolKey,
                     iteration,
                     kind: 'cache_break',
+                    classification: intentionalTransition ? 'intentional' : 'provider_transition',
+                    source: 'openai_ws_delta',
                     provider: traceProvider,
                     model: liveModel,
                     payload: {
@@ -1544,6 +1634,8 @@ export async function sendViaWebSocket({
                         transport: 'websocket',
                         ws_mode: mode,
                         reason: mode === 'delta' ? deltaReason : (deltaReason || 'full_frame'),
+                        classification: intentionalTransition ? 'intentional' : 'provider_transition',
+                        source: 'openai_ws_delta',
                         intentional_transition: intentionalTransition,
                         request_tool_choice: requestBody.tool_choice ?? null,
                         cache_key_hash: transportCacheKeyHash,
@@ -1556,6 +1648,7 @@ export async function sendViaWebSocket({
                         chain_stripped_response_items: strippedResponseItems,
                         chain_skipped_response_items: skippedResponseItems,
                         ...(responseOutputMismatch || {}),
+                        ...(requestInputMismatch || {}),
                         chain_response_items: Array.isArray(result.responseItems) ? result.responseItems.length : 0,
                         body_input_items: Array.isArray(requestBody.input) ? requestBody.input.length : null,
                         frame_input_items: Array.isArray(frame.input) ? frame.input.length : null,
@@ -1584,7 +1677,7 @@ export async function sendViaWebSocket({
         // Leave a breadcrumb on the result so downstream callers can observe
         // that a retry was used (0 = first-try success, up to 2 for ws_1006/1011).
         try { Object.defineProperty(out, '__midstreamRetries', { value: attemptIndex, enumerable: false }); } catch {}
-        emitSendSpan('ok');
+        out.transportTiming = emitSendSpan('ok');
         return out;
     }
     // Unreachable — the loop either returns or throws above.

package/src/runtime/agent/orchestrator/providers/openai-turn-state.mjs

@@ -0,0 +1,68 @@
+const MAX_TURN_STATE_SCOPES = 4096;
+const _turnStateByScope = new Map();
+
+function _clean(value) {
+    return typeof value === 'string' ? value.trim() : '';
+}
+
+function _ensureCapacity(scopeKey) {
+    if (_turnStateByScope.has(scopeKey) || _turnStateByScope.size < MAX_TURN_STATE_SCOPES) return;
+    const oldest = _turnStateByScope.keys().next().value;
+    if (oldest !== undefined) _turnStateByScope.delete(oldest);
+}
+
+// A turn-state token belongs to one logical turn, not to the physical
+// connection that happened to carry it. Activating a new turn clears the prior
+// value even when the caller reuses the same WebSocket or HTTP session.
+export function activateCodexTurnState(scopeKey, turnId, owner = null) {
+    const scope = _clean(scopeKey);
+    const turn = _clean(turnId);
+    if (!scope || !turn) return null;
+    const current = _turnStateByScope.get(scope);
+    if (current?.turnId === turn) {
+        // One physical WS connection owns the token while it is live. HTTP
+        // fallback has no WS owner and may read the same turn-scoped value.
+        if (owner && current.owner && current.owner !== owner) return null;
+        if (owner && !current.owner) current.owner = owner;
+        return current.value || null;
+    }
+    _ensureCapacity(scope);
+    _turnStateByScope.set(scope, { turnId: turn, value: null, owner: owner || null });
+    return null;
+}
+
+// The first server value wins for the whole logical turn.
+export function captureCodexTurnState(scopeKey, turnId, value, owner = null) {
+    const scope = _clean(scopeKey);
+    const turn = _clean(turnId);
+    const token = _clean(value);
+    if (!scope || !turn || !token) return null;
+    activateCodexTurnState(scope, turn, owner);
+    const current = _turnStateByScope.get(scope);
+    if (!current || current.turnId !== turn) return null;
+    if (owner && current.owner && current.owner !== owner) return null;
+    if (owner && !current.owner) current.owner = owner;
+    if (current.value) return current.value;
+    current.value = token;
+    return token;
+}
+
+// A replacement connection may adopt the turn token only after the physical
+// connection that owned it has actually closed.
+export function retireCodexTurnStateOwner(scopeKey, owner) {
+    const scope = _clean(scopeKey);
+    if (!scope || !owner) return;
+    const current = _turnStateByScope.get(scope);
+    if (current?.owner === owner) current.owner = null;
+}
+
+export function clearCodexTurnStateScope(scopeKey) {
+    const scope = _clean(scopeKey);
+    if (scope) _turnStateByScope.delete(scope);
+}
+
+export function clearAllCodexTurnStates() {
+    _turnStateByScope.clear();
+}
+
+export const _clearCodexTurnStatesForTest = clearAllCodexTurnStates;

package/src/runtime/agent/orchestrator/providers/openai-ws-delta.mjs

@@ -196,6 +196,31 @@ function _normalizedResponseItemDiagnosticHash(item) {
         .digest('hex');
 }
 
+export function _requestInputMismatchDiagnostics(currentInput, previousInput) {
+    const current = Array.isArray(currentInput) ? currentInput : [];
+    const previous = Array.isArray(previousInput) ? previousInput : [];
+    let index = Math.min(current.length, previous.length);
+    const limit = Math.min(current.length, previous.length);
+    for (let cursor = 0; cursor < limit; cursor += 1) {
+        if (_requestInputItemsMatch(current[cursor], previous[cursor])) continue;
+        index = cursor;
+        break;
+    }
+    const expected = previous[index];
+    const actual = current[index];
+    return {
+        inputPrefixMismatchIndex: index,
+        inputPrefixMismatchPreviousCount: previous.length,
+        inputPrefixMismatchCurrentCount: current.length,
+        inputPrefixMismatchExpectedType: expected?.type
+            || (expected?.role === 'assistant' ? 'message' : (expected ? 'unknown' : 'missing')),
+        inputPrefixMismatchExpectedHash: _normalizedResponseItemDiagnosticHash(expected),
+        inputPrefixMismatchActualType: actual?.type
+            || (actual?.role === 'assistant' ? 'message' : (actual ? 'unknown' : 'missing')),
+        inputPrefixMismatchActualHash: _normalizedResponseItemDiagnosticHash(actual),
+    };
+}
+
 function _responseOutputMismatchDiagnostics(inputItem, responseItem, replayItemCount, responseItemCount) {
     return {
         response_output_mismatch_expected_type: responseItem?.type || 'unknown',
@@ -312,16 +337,11 @@ export function _buildResponseCreateFrame(body, { previousResponseId = null, inp
 
 export function _computeDelta({ entry, body, traceProvider }) {
     // DEFAULT: full-frame sends. codex's delta path is only cache-safe with
-    // the x-codex-turn-state sticky-routing token. R11-R14 (2026-07-03)
-    // measured zero turn-state EVENTS across 200+ calls despite
-    // UA/version/beta-features/client_metadata parity, and delta at 18-28% warm
-    // miss vs full-frame 0.0%. The event channel is still silent, but the
-    // conclusion "the backend never issues it to us" was wrong: the token
-    // arrives on the handshake 101 response instead (2026-08-21 bench: 81 of 88
-    // calls carried it from iteration 1, before any response event existed), so
-    // the pool captures it at upgrade and the frame metadata replays it.
-    // Without the sticky token, previous_response_id requests hop cache nodes
-    // and only the first prefix blocks hit.
+    // the x-codex-turn-state sticky-routing token. The token is request-scoped:
+    // response metadata establishes it, later response.create frames replay it
+    // for the same logical turn, and a new turn starts empty even when it reuses
+    // the physical connection. Handshake state is deliberately ignored because
+    // a connection can outlive the turn that created it.
     // Delta gating flows through the single transport-policy switch
     // (MIXDOG_OAI_TRANSPORT: ws-full | ws-delta | http-sse). Default ws-delta
     // selects the refs-compatible safe delta; 'ws-full'/'http-sse' force full
@@ -379,7 +399,12 @@ export function _computeDelta({ entry, body, traceProvider }) {
     const curInput = Array.isArray(body.input) ? body.input : [];
     const afterPreviousInput = _stripRequestPrefix(curInput, entry.lastRequestInput);
     if (!afterPreviousInput) {
-        return { mode: 'full', reason: 'input_prefix_mismatch', frame: buildFrame(body) };
+        return {
+            mode: 'full',
+            reason: 'input_prefix_mismatch',
+            requestInputMismatch: _requestInputMismatchDiagnostics(curInput, entry.lastRequestInput),
+            frame: buildFrame(body),
+        };
     }
     const stripped = _stripResponseItemsFromHead(afterPreviousInput, entry.lastResponseItems);
     if (!stripped.ok) {