mixdog 0.9.44 → 0.9.46

package/src/runtime/agent/orchestrator/providers/openai-ws-pool.mjs

@@ -9,6 +9,7 @@
 import WebSocket from 'ws';
 import { errText } from '../../../shared/err-text.mjs';
 import { createHash, randomBytes } from 'crypto';
+import { performance } from 'node:perf_hooks';
 import { appendFileSync, mkdirSync, writeFileSync } from 'node:fs';
 import { join } from 'node:path';
 import { codexOriginator, codexUserAgent, codexVersionHeader } from './codex-client-meta.mjs';
@@ -305,6 +306,14 @@ function _isOpen(entry) {
     return entry?.socket?.readyState === WebSocket.OPEN;
 }
 
+function _setTransportReferenced(entry, referenced) {
+    const transport = entry?.socket?._socket;
+    try {
+        if (referenced) transport?.ref?.();
+        else transport?.unref?.();
+    } catch {}
+}
+
 function _clearLiveness(entry) {
     if (entry?.pingTimer) {
         clearInterval(entry.pingTimer);
@@ -377,7 +386,7 @@ function _armLiveness(poolKey, entry) {
 // for a server event that will never arrive. Tag any failure with
 // `wsSendFailed=true` so _classifyMidstreamError routes the next attempt
 // through a fresh socket.
-export function _sendFrame(entry, frame) {
+export function _sendFrame(entry, frame, sendSpan = null) {
     return new Promise((resolve, reject) => {
         const socket = entry?.socket;
         if (!socket || socket.readyState !== WebSocket.OPEN) {
@@ -387,6 +396,7 @@ export function _sendFrame(entry, frame) {
             return;
         }
         let payload;
+        const serializeStart = performance.now();
         try { payload = JSON.stringify(frame); }
         catch (e) {
             const err = e instanceof Error ? e : new Error(String(e));
@@ -394,6 +404,10 @@ export function _sendFrame(entry, frame) {
             reject(err);
             return;
         }
+        if (sendSpan && typeof sendSpan === 'object') {
+            sendSpan.requestBuildSerializationMs = (sendSpan.requestBuildSerializationMs || 0)
+                + (performance.now() - serializeStart);
+        }
         _dumpFrame(payload);
         try {
             // Do NOT await the send callback: on a wedged-but-OPEN socket the
@@ -691,6 +705,7 @@ export async function acquireWebSocket({ auth, poolKey, cacheKey, codexHeaders,
             // front makes the find() above skip it; on probe failure it is
             // evicted (removed from arr) so the loop continues cleanly.
             idle.busy = true;
+            _setTransportReferenced(idle, true);
             if (WS_PING_ENABLED && Date.now() - (idle.lastAliveAt || 0) >= WS_LIVENESS_STALE_MS) {
                 const alive = await _pingProbe(idle, WS_PONG_TIMEOUT_MS);
                 if (!alive) {
@@ -803,6 +818,9 @@ export function releaseWebSocket({ entry, poolKey, keep }) {
     entry.lastAliveAt = Date.now();
     _scheduleIdleClose(poolKey, entry);
     if (WS_PING_ENABLED) _armLiveness(poolKey, entry);
+    // Idle pooled sockets retain reuse state without retaining the process.
+    // acquireWebSocket re-refs the transport before probing or handing it out.
+    _setTransportReferenced(entry, false);
 }
 
 // Drain-complete fence — set true once _closeAllPooledSockets runs so any

package/src/runtime/agent/orchestrator/providers/openai-ws-stream.mjs

@@ -14,6 +14,7 @@
  * openai-oauth-ws.mjs re-exports.
  */
 import { randomBytes } from 'crypto';
+import { performance } from 'node:perf_hooks';
 import {
     extractCachedTokens,
     appendAgentTrace,
@@ -250,6 +251,12 @@ export async function _streamResponse({
     const errLabel = _wsErrLabel(traceProvider);
     const socket = entry.socket;
     enableSessionTransportTracking(state?.sessionId);
+    // An already-open (possibly pooled) socket proves transport availability;
+    // response.created/reasoning/text/tool events independently satisfy the
+    // semantic deadline.
+    if (socket?.readyState === WebSocket.OPEN) {
+        markSessionTransportActivity(state?.sessionId);
+    }
     const preResponseCreatedMs = _positiveInt(_timeouts?.preResponseCreatedMs, WS_PRE_RESPONSE_CREATED_MS);
     const interChunkMs = _positiveInt(_timeouts?.interChunkMs, WS_INTER_CHUNK_MS);
     // First-MEANINGFUL-frame deadline. Distinct from preResponseCreatedMs (a
@@ -422,6 +429,7 @@ export async function _streamResponse({
         toolCalls.push(call);
         midState.emittedToolCall = true;
         try { onToolCall?.(call); } catch {}
+        try { onStreamDelta?.('tool'); } catch {}
     };
     const relayLeakText = (delta) => {
         if (!leakGuard.enabled) {
@@ -430,17 +438,22 @@ export async function _streamResponse({
                 if (state) state.emittedText = true;
                 try { onTextDelta(delta); } catch {}
             }
-            return;
+            if (delta) {
+                try { onStreamDelta?.('text'); } catch {}
+            }
+            return { text: !!delta, tool: false };
         }
         const { text, calls } = leakGuard.push(delta);
         if (text) {
             content += text;
+            try { onStreamDelta?.('text'); } catch {}
             if (onTextDelta) {
                 if (state) state.emittedText = true;
                 try { onTextDelta(text); } catch {}
             }
         }
         for (const c of calls) dispatchLeakedCall(c);
+        return { text: !!text, tool: calls.length > 0 };
     };
     const flushLeak = () => {
         if (!leakGuard.enabled) return;
@@ -763,6 +776,11 @@ export async function _streamResponse({
         };
 
         messageHandler = (data) => {
+            if (midState.sendSpan && midState.sendStartedAt != null
+                && midState.sendSpanAttemptFirstEvent !== true) {
+                midState.sendSpanAttemptFirstEvent = true;
+                midState.sendSpan.firstEventMs += performance.now() - midState.sendStartedAt;
+            }
             resetIdle();
             // resetIdle() above resets the SINGLE inter-chunk idle timer on
             // EVERY received frame — response.created, metadata,
@@ -798,6 +816,11 @@ export async function _streamResponse({
             switch (event.type) {
                 case 'response.created':
                     midState.sawResponseCreated = true;
+                    if (midState.sendSpan && midState.sendStartedAt != null
+                        && midState.sendSpanAttemptResponseCreated !== true) {
+                        midState.sendSpanAttemptResponseCreated = true;
+                        midState.sendSpan.preResponseCreatedMs += performance.now() - midState.sendStartedAt;
+                    }
                     if (event.response?.model) model = event.response.model;
                     if (event.response?.id) responseId = event.response.id;
                     // Server ack (first event). resetIdle() at the top of
@@ -807,21 +830,20 @@ export async function _streamResponse({
                     // first-meaningful watchdog (keepalive/metadata frames do
                     // NOT reach this case, so they never clear it).
                     clearFirstMeaningfulWatchdog();
+                    try { onStreamDelta?.('semantic'); } catch {}
                     // Do not arm semantic idle until actual model progress.
                     // Transport-only reasoning heartbeats are bounded by the
                     // outer first-visible ceiling instead.
                     break;
                 case 'response.output_text.delta':
-                    try {
+                    if (event.delta) try {
                         if (!_firstDeltaEmitted) {
                             _firstDeltaEmitted = true;
                             if (process.env.MIXDOG_DEBUG_AGENT) {
                                 process.stderr.write(`[agent-trace] ws-first-delta sinceStreaming=${Date.now() - _streamingStart}ms\n`);
                             }
                         }
-                        onStreamDelta?.();
                     } catch {}
-                    bumpSemanticIdle();
                     // Live text relay (gateway): forward the raw text chunk so
                     // the client renders first tokens before the final replay.
                     // Tool-call/argument deltas intentionally stay off this path.
@@ -832,7 +854,10 @@ export async function _streamResponse({
                     // Routed through the leaked-tool-call guard: appends to
                     // `content`, forwards visible text via onTextDelta, and
                     // recovers/dispatches any leaked known-tool call.
-                    relayLeakText(event.delta || '');
+                    {
+                        const progress = relayLeakText(event.delta || '');
+                        if (progress?.text || progress?.tool) bumpSemanticIdle();
+                    }
                     break;
                 case 'response.reasoning_text.delta':
                 case 'response.reasoning_summary_text.delta':
@@ -841,7 +866,7 @@ export async function _streamResponse({
                     // Only non-empty reasoning text is model progress. Empty
                     // deltas remain transport activity via resetIdle() above.
                     if (event.delta) {
-                        try { onStreamDelta?.(); } catch {}
+                        try { onStreamDelta?.('reasoning'); } catch {}
                         bumpSemanticIdle();
                     }
                     break;
@@ -869,17 +894,18 @@ export async function _streamResponse({
                     // timer so long server-side tool latency after item-added
                     // (before any arg delta) is not mistaken for a silent stall.
                     resetSemanticIdle();
+                    try { onStreamDelta?.(_toolInFlight ? 'tool' : 'semantic'); } catch {}
                     break;
                 case 'response.function_call_arguments.delta':
                     markActiveToolItem(null, event.item_id);
                     _toolInFlight = true;
-                    try { onStreamDelta?.(); } catch {}
+                    try { onStreamDelta?.('tool'); } catch {}
                     bumpSemanticIdle();
                     break;
                 case 'response.custom_tool_call_input.delta':
                     markActiveToolItem(null, event.item_id);
                     _toolInFlight = true;
-                    try { onStreamDelta?.(); } catch {}
+                    try { onStreamDelta?.('tool'); } catch {}
                     bumpSemanticIdle();
                     break;
                 case 'response.function_call_arguments.done': {
@@ -935,7 +961,7 @@ export async function _streamResponse({
                             _deferred: true,
                         });
                     }
-                    try { onStreamDelta?.(); } catch {}
+                    try { onStreamDelta?.('tool'); } catch {}
                     bumpSemanticIdle();
                     break;
                 }
@@ -992,6 +1018,13 @@ export async function _streamResponse({
                     // Item-done is genuine lifecycle progress — reset semantic
                     // idle so latency before the next item/args does not stall.
                     resetSemanticIdle();
+                    try {
+                        onStreamDelta?.(
+                            event.item?.type === 'reasoning'
+                                ? 'reasoning'
+                                : (_toolInFlight || /tool|function_call/.test(event.item?.type || '') ? 'tool' : 'semantic'),
+                        );
+                    } catch {}
                     break;
                 case 'response.completed': {
                     const completedServiceTier = event.response?.service_tier || event.response?.serviceTier || '';
@@ -1016,6 +1049,7 @@ export async function _streamResponse({
                     }
                     if (!model && event.response?.model) model = event.response.model;
                     if (!responseId && event.response?.id) responseId = event.response.id;
+                    let reportedBundleProgress = false;
                     if (event.response?.output) {
                         for (const item of event.response.output) {
                             pushResponseItem(item);
@@ -1030,9 +1064,18 @@ export async function _streamResponse({
                                         if (leakGuard.enabled) {
                                             const { text, calls } = leakGuard.push(c.text || '', true);
                                             content += text;
+                                            if (text) {
+                                                try { onStreamDelta?.('text'); } catch {}
+                                                reportedBundleProgress = true;
+                                            }
                                             for (const lc of calls) dispatchLeakedCall(lc);
+                                            if (calls.length) reportedBundleProgress = true;
                                         } else {
                                             content += c.text || '';
+                                            if (c.text) {
+                                                try { onStreamDelta?.('text'); } catch {}
+                                                reportedBundleProgress = true;
+                                            }
                                         }
                                         pushOutputTextAnnotations(c);
                                     }
@@ -1043,15 +1086,29 @@ export async function _streamResponse({
                                     if (c.type === 'output_text') pushOutputTextAnnotations(c);
                                 }
                             }
-                            if (item.type === 'web_search_call') pushWebSearchCall(item);
-                            if (item.type === 'tool_search_call') pushToolSearchCall(item);
-                            if (item.type === 'custom_tool_call') pushCustomToolCall(item);
+                            if (item.type === 'web_search_call') {
+                                pushWebSearchCall(item);
+                                try { onStreamDelta?.('tool'); } catch {}
+                                reportedBundleProgress = true;
+                            }
+                            if (item.type === 'tool_search_call') {
+                                pushToolSearchCall(item);
+                                try { onStreamDelta?.('tool'); } catch {}
+                                reportedBundleProgress = true;
+                            }
+                            if (item.type === 'custom_tool_call') {
+                                pushCustomToolCall(item);
+                                try { onStreamDelta?.('tool'); } catch {}
+                                reportedBundleProgress = true;
+                            }
                             // Salvage path: some streams emit reasoning only
                             // inside the final response.completed.output
                             // bundle (no per-item .done event). Dedup by id.
                             if (item.type === 'reasoning'
                                 && !reasoningItems.some(r => r.id === item.id)) {
                                 pushReasoningItem(item);
+                                try { onStreamDelta?.('reasoning'); } catch {}
+                                reportedBundleProgress = true;
                             }
                             // Salvage path for function_call: when
                             // arguments.done fired before (or without) a
@@ -1072,9 +1129,14 @@ export async function _streamResponse({
                                         emitToolCallDedupe(tc);
                                     }
                                 }
+                                try { onStreamDelta?.('tool'); } catch {}
+                                reportedBundleProgress = true;
                             }
                         }
                     }
+                    if (!reportedBundleProgress) {
+                        try { onStreamDelta?.('semantic'); } catch {}
+                    }
                     // Salvage validation. Any deferred call still missing
                     // id/name would propagate to the next turn as a
                     // function_call_output the server can't anchor. Fail the
@@ -1209,7 +1271,7 @@ export async function _streamResponse({
                         // Only non-empty reasoning deltas are model progress;
                         // empty variants remain transport activity only.
                         if (event.delta) {
-                            try { onStreamDelta?.(); } catch {}
+                            try { onStreamDelta?.('reasoning'); } catch {}
                             bumpSemanticIdle();
                         }
                     }
@@ -1290,13 +1352,19 @@ export async function _streamResponse({
         socket.on('message', messageHandler);
         socket.on('close', closeHandler);
         socket.on('error', errorHandler);
+        tracePingHandler = () => {
+            markSessionTransportActivity(midState.sessionId);
+            if (WS_TRACE_ENABLED) _writeWsLifecycleTrace('ping');
+        };
+        tracePongHandler = () => {
+            markSessionTransportActivity(midState.sessionId);
+            if (WS_TRACE_ENABLED) _writeWsLifecycleTrace('pong');
+        };
+        socket.on('ping', tracePingHandler);
+        socket.on('pong', tracePongHandler);
         if (WS_TRACE_ENABLED) {
             traceOpenHandler = () => _writeWsLifecycleTrace('open');
-            tracePingHandler = () => _writeWsLifecycleTrace('ping');
-            tracePongHandler = () => _writeWsLifecycleTrace('pong');
             socket.on('open', traceOpenHandler);
-            socket.on('ping', tracePingHandler);
-            socket.on('pong', tracePongHandler);
             if (socket.readyState === WebSocket.OPEN) _writeWsLifecycleTrace('open');
         }
         armPreStreamWatchdog();

package/src/runtime/agent/orchestrator/providers/openai-ws.mjs

@@ -19,7 +19,7 @@ import { sanitizeModelList } from './model-list-sanitize.mjs';
 import { sendViaHttpSse, _envFlag } from './openai-oauth-http-sse.mjs';
 import { shouldFallbackTransport } from './retry-classifier.mjs';
 import { resolveOpenAiTransportPolicy } from './openai-transport-policy.mjs';
-import { loadConfig } from '../config.mjs';
+import { getAgentApiKey } from '../../../shared/provider-api-key.mjs';
 import {
     resolveProviderCacheKey,
     resolveProviderPromptCacheLane,
@@ -57,17 +57,15 @@ export class OpenAIDirectProvider {
         return k;
     }
     // Auth-recovery mirror of openai-compat.reloadApiKey: on a 401/403 the key
-    // was likely rotated in config after this provider instance was built, so
-    // re-read providers.openai.apiKey from disk before the single retry.
+    // was likely rotated after this provider instance was built, so re-read
+    // only OpenAI's environment/keychain sources before the single retry.
     // Returns the fresh key (or null if none) — no client to rebuild here since
     // the WS/HTTP transports take the key per-call via the `auth` object.
     reloadApiKey() {
         try {
-            const freshConfig = loadConfig();
-            const cfg = freshConfig.providers?.openai;
-            const newKey = cfg?.apiKey || this.config.apiKey;
+            const newKey = getAgentApiKey('openai') || this.config.apiKey;
             if (newKey) {
-                this.config = { ...(this.config || {}), ...(cfg || {}), apiKey: newKey };
+                this.config = { ...(this.config || {}), apiKey: newKey };
                 return newKey;
             }
         } catch { /* best effort */ }
@@ -185,7 +183,7 @@ export class OpenAIDirectProvider {
             const unsafeToRetry = err?.liveTextEmitted === true
                 || err?.emittedToolCall === true
                 || err?.unsafeToRetry === true;
-            // (1) 401/403 → reload apiKey from config and retry once over WS.
+            // (1) 401/403 → reload only OpenAI auth and retry once over WS.
             //     shouldFallbackTransport denies 401/403, so this branch owns
             //     its own guard.
             if ((status === 401 || status === 403) && !unsafeToRetry) {

package/src/runtime/agent/orchestrator/providers/registry.mjs

@@ -56,27 +56,52 @@ function configSignature(cfg) {
     }
 }
 
-async function loadProviderExport(cacheKey, spec, exportName) {
+function abortError(signal) {
+    return signal?.reason instanceof Error ? signal.reason : new Error(String(signal?.reason || 'provider initialization aborted'));
+}
+
+function throwIfAborted(signal) {
+    if (signal?.aborted) throw abortError(signal);
+}
+
+function awaitWithAbort(promise, signal) {
+    if (!(signal instanceof AbortSignal)) return promise;
+    throwIfAborted(signal);
+    let listener = null;
+    const aborted = new Promise((_, reject) => {
+        listener = () => reject(abortError(signal));
+        signal.addEventListener('abort', listener, { once: true });
+    });
+    return Promise.race([promise, aborted]).finally(() => {
+        if (listener) {
+            try { signal.removeEventListener('abort', listener); } catch { /* ignore */ }
+        }
+    });
+}
+
+async function loadProviderExport(cacheKey, spec, exportName, signal = null) {
     if (!providerModulePromises.has(cacheKey)) {
         providerModulePromises.set(cacheKey, import(spec));
     }
-    const mod = await providerModulePromises.get(cacheKey);
+    const mod = await awaitWithAbort(providerModulePromises.get(cacheKey), signal);
+    throwIfAborted(signal);
     const value = mod?.[exportName];
     if (typeof value !== 'function') throw new Error(`provider export missing: ${exportName}`);
+    throwIfAborted(signal);
     providerCtors.set(cacheKey, value);
     return value;
 }
 
-async function loadProviderCtor(name) {
-    if (name === 'anthropic') return loadProviderExport('anthropic', './anthropic.mjs', 'AnthropicProvider');
-    if (name === 'gemini') return loadProviderExport('gemini', './gemini.mjs', 'GeminiProvider');
-    if (name === 'openai-oauth') return loadProviderExport('openai-oauth', './openai-oauth.mjs', 'OpenAIOAuthProvider');
-    if (name === 'anthropic-oauth') return loadProviderExport('anthropic-oauth', './anthropic-oauth.mjs', 'AnthropicOAuthProvider');
-    if (name === 'grok-oauth') return loadProviderExport('grok-oauth', './grok-oauth.mjs', 'GrokOAuthProvider');
-    if (name === 'openai') return loadProviderExport('openai', './openai-ws.mjs', 'OpenAIDirectProvider');
-    if (name === 'opencode-go') return loadProviderExport('opencode-go', './opencode-go.mjs', 'OpenCodeGoProvider');
+async function loadProviderCtor(name, signal = null) {
+    if (name === 'anthropic') return loadProviderExport('anthropic', './anthropic.mjs', 'AnthropicProvider', signal);
+    if (name === 'gemini') return loadProviderExport('gemini', './gemini.mjs', 'GeminiProvider', signal);
+    if (name === 'openai-oauth') return loadProviderExport('openai-oauth', './openai-oauth.mjs', 'OpenAIOAuthProvider', signal);
+    if (name === 'anthropic-oauth') return loadProviderExport('anthropic-oauth', './anthropic-oauth.mjs', 'AnthropicOAuthProvider', signal);
+    if (name === 'grok-oauth') return loadProviderExport('grok-oauth', './grok-oauth.mjs', 'GrokOAuthProvider', signal);
+    if (name === 'openai') return loadProviderExport('openai', './openai-ws.mjs', 'OpenAIDirectProvider', signal);
+    if (name === 'opencode-go') return loadProviderExport('opencode-go', './opencode-go.mjs', 'OpenCodeGoProvider', signal);
     if (Object.prototype.hasOwnProperty.call(OPENAI_COMPAT_PRESETS, name)) {
-        return loadProviderExport('openai-compat', './openai-compat.mjs', 'OpenAICompatProvider');
+        return loadProviderExport('openai-compat', './openai-compat.mjs', 'OpenAICompatProvider', signal);
     }
     throw new Error(`unknown enabled provider: ${name}`);
 }
@@ -88,11 +113,12 @@ function instantiateProvider(name, Ctor, cfg) {
     return new Ctor(cfg);
 }
 
-export async function initProviders(config) {
+export async function initProviders(config, { signal = null } = {}) {
+    throwIfAborted(signal);
     const sig = configSignature(config);
     // Coalesce: an identical config is already mid-init — attach to it.
     if (sig !== null && _inFlightPromise && _inFlightSig === sig) {
-        return _inFlightPromise;
+        return awaitWithAbort(_inFlightPromise, signal);
     }
     // Fast path: chain idle and the live registry already reflects this exact
     // config — nothing to tear down or rebuild.
@@ -103,7 +129,7 @@ export async function initProviders(config) {
     // signatures can never run their clear()+rebuild concurrently, regardless
     // of caller-side gating (agent-tool gateOnPrior may release a queued init
     // before the prior one settled). Errors do not poison the chain.
-    const run = () => _initProvidersUnsynchronized(config);
+    const run = () => _initProvidersUnsynchronized(config, signal);
     const next = _initChain.then(run, run);
     _initChain = next.then(() => {}, () => {});
     const settle = () => {
@@ -118,10 +144,11 @@ export async function initProviders(config) {
     );
     _inFlightSig = sig;
     _inFlightPromise = tracked;
-    return tracked;
+    return awaitWithAbort(tracked, signal);
 }
 
-async function _initProvidersUnsynchronized(config) {
+async function _initProvidersUnsynchronized(config, signal = null) {
+    throwIfAborted(signal);
     // Invariant: never wipe the live registry based on an empty / all-disabled
     // config. Without this guard, a stale `loadAgentConfig()` (e.g. mid-reload
     // or a transient FS hiccup) would land here as `{}` or `{...,enabled:false}`,
@@ -145,7 +172,8 @@ async function _initProvidersUnsynchronized(config) {
             return { name, inst: providers.get(name), sig };
         }
         try {
-            const Ctor = await loadProviderCtor(name);
+            const Ctor = await loadProviderCtor(name, signal);
+            throwIfAborted(signal);
             const inst = instantiateProvider(name, Ctor, cfg);
             return { name, inst, sig };
         }
@@ -155,6 +183,7 @@ async function _initProvidersUnsynchronized(config) {
         }
     }));
     for (const result of enabledResults) {
+        throwIfAborted(signal);
         if (!result) continue;
         if (result.error) throw result.error;
         next.set(result.name, result.inst);
@@ -182,6 +211,7 @@ async function _initProvidersUnsynchronized(config) {
             if (signatures.has(name)) nextSignatures.set(name, signatures.get(name));
         }
     }
+    throwIfAborted(signal);
     providers.clear();
     for (const [k, v] of next) providers.set(k, v);
     signatures.clear();