mixdog 0.9.53 → 0.9.55

package/src/runtime/agent/orchestrator/providers/gemini-stream.mjs

@@ -44,6 +44,41 @@ export function geminiTruncatedStreamError(message) {
     );
 }
 
+export function geminiStreamCorruptionError(message, cause = null) {
+    return Object.assign(
+        new Error(message, cause ? { cause } : undefined),
+        {
+            name: 'GeminiStreamCorruptionError',
+            code: 'TRUNCATED_STREAM',
+            truncatedStream: true,
+            streamCorruption: true,
+        },
+    );
+}
+
+function isGeminiSdkStreamParseError(err) {
+    let cursor = err;
+    const seen = new Set();
+    for (let depth = 0; cursor && depth < 5 && !seen.has(cursor); depth++) {
+        seen.add(cursor);
+        if (cursor instanceof SyntaxError) return true;
+        const name = String(cursor?.name || '');
+        const message = String(cursor?.message || '');
+        if (name === 'GoogleGenerativeAIError'
+            && /(?:parse|parsing|json|unexpected token|unexpected end|unterminated)/i.test(message)) {
+            return true;
+        }
+        cursor = cursor?.cause;
+    }
+    return false;
+}
+
+function normalizeGeminiSdkStreamError(err, label) {
+    return isGeminiSdkStreamParseError(err)
+        ? geminiStreamCorruptionError(`${label} corrupt SDK SSE JSON`, err)
+        : err;
+}
+
 // CC-rule safety stamp for Gemini stream failures (provider-stall audit):
 // once text has been relayed to the live gateway or a leaked tool call was
 // dispatched, replaying the request would double-render/double-execute — the
@@ -63,10 +98,10 @@ export function stampGeminiStreamFailure(err, {
     const finalizedText = textLeakGuard?.getVisibleText?.();
     const visibleText = typeof finalizedText === 'string' ? finalizedText : relayedText;
     const visible = visibleText.length > 0;
-    // A native functionCall chunk (not a text-leaked one) is also an in-flight
-    // tool use — replaying would double-dispatch, and partial-final must NOT
-    // treat it as a clean no-tool summary.
-    const pendingTool = leaked || sawFunctionCall === true;
+    // Native functionCall chunks are only dispatched after successful stream
+    // completion in gemini.mjs. Until then a reconnect is safe. Text-leaked
+    // calls are dispatched while parsing and remain a hard replay boundary.
+    const pendingTool = leaked;
     try {
         if (visible) { err.liveTextEmitted = true; err.unsafeToRetry = true; }
         if (pendingTool) { err.emittedToolCall = true; err.unsafeToRetry = true; }
@@ -277,6 +312,10 @@ export function createGeminiTextLeakGuard({ knownToolNames, onTextDelta, onToolC
 
 export async function consumeGeminiRestStreamResponse(response, { signal, onStreamDelta, onTextDelta, textLeakGuard, label }) {
     if (!response?.body) throw new Error(`${label}: missing response body`);
+    if (signal?.aborted) {
+        const reason = signal.reason;
+        throw reason instanceof Error ? reason : new Error(`${label} aborted`);
+    }
     const reader = response.body.getReader();
     const decoder = new TextDecoder();
     let buffer = '';
@@ -332,10 +371,6 @@ export async function consumeGeminiRestStreamResponse(response, { signal, onStre
     };
 
     if (signal) {
-        if (signal.aborted) {
-            const reason = signal.reason;
-            throw reason instanceof Error ? reason : new Error(`${label} aborted`);
-        }
         signal.addEventListener('abort', onAbort, { once: true });
     }
 
@@ -373,7 +408,11 @@ export async function consumeGeminiRestStreamResponse(response, { signal, onStre
                 const data = line.slice(6).trim();
                 if (!data || data === '[DONE]') continue;
                 let parsed;
-                try { parsed = JSON.parse(data); } catch { continue; }
+                try {
+                    parsed = JSON.parse(data);
+                } catch (cause) {
+                    throw geminiStreamCorruptionError(`${label} corrupt SSE JSON`, cause);
+                }
                 if (!sawStreamChunk) {
                     sawStreamChunk = true;
                     clearFirstByteTimer();
@@ -407,7 +446,9 @@ export async function consumeGeminiRestStreamResponse(response, { signal, onStre
                             if (t) relayedText += t;
                             relayGeminiStreamText(t, { onTextDelta, textLeakGuard });
                         }
-                    } catch { /* skip malformed tail */ }
+                    } catch (cause) {
+                        throw geminiStreamCorruptionError(`${label} corrupt SSE tail JSON`, cause);
+                    }
                 }
             }
         }
@@ -437,7 +478,16 @@ export async function consumeGeminiRestStreamResponse(response, { signal, onStre
     return aggregated;
 }
 
-export async function consumeGeminiSdkStream(streamResult, { signal, onStreamDelta, onTextDelta, textLeakGuard, label }) {
+export async function consumeGeminiSdkStream(streamResult, {
+    signal,
+    onStreamDelta,
+    onTextDelta,
+    textLeakGuard,
+    label,
+    cancelGeneration,
+    firstByteTimeoutMs = GEMINI_FIRST_BYTE_TIMEOUT_MS,
+    cancellationGraceMs = 250,
+}) {
     let sawStreamChunk = false;
     let idleTimedOut = false;
     let idleTimer = null;
@@ -453,15 +503,53 @@ export async function consumeGeminiSdkStream(streamResult, { signal, onStreamDel
         try { textLeakGuard?.finalize(); } catch {}
     };
 
+    let iterator = null;
+    let cancellation = null;
+    let forcedFailure = null;
+    const cancelInFlight = (err) => {
+        if (cancellation) return cancellation;
+        forcedFailure = err;
+        cancellation = (async () => {
+            try { cancelGeneration?.(err); } catch {}
+            let returnPromise;
+            try {
+                returnPromise = Promise.resolve(iterator?.return?.());
+            } catch {
+                return;
+            }
+            // iterator.return() is best-effort cleanup. A broken SDK iterator
+            // must not deadlock the timeout path and prevent withRetry from
+            // beginning the next attempt after the generation was aborted.
+            let graceTimer = null;
+            try {
+                await Promise.race([
+                    returnPromise.catch(() => {}),
+                    new Promise((resolve) => {
+                        graceTimer = setTimeout(resolve, Math.max(0, cancellationGraceMs));
+                    }),
+                ]);
+            } finally {
+                if (graceTimer) clearTimeout(graceTimer);
+                // Keep observing a late rejection after the grace race expires.
+                returnPromise.catch(() => {});
+            }
+        })();
+        return cancellation;
+    };
+
+    const rejectAfterCancellation = (reject, err) => {
+        cancelInFlight(err).then(() => reject(err), () => reject(err));
+    };
+
     const armFirstByteTimer = () => {
         if (firstByteTimer) clearTimeout(firstByteTimer);
         firstByteTimer = setTimeout(() => {
             if (firstByteReject) {
-                const e = geminiTimeoutError(`${label} first byte`, GEMINI_FIRST_BYTE_TIMEOUT_MS);
-                const r = firstByteReject; firstByteReject = null; r(e);
+                const e = geminiTimeoutError(`${label} first byte`, firstByteTimeoutMs);
+                const r = firstByteReject; firstByteReject = null;
+                rejectAfterCancellation(r, e);
             }
-        }, GEMINI_FIRST_BYTE_TIMEOUT_MS);
-        if (firstByteTimer.unref) firstByteTimer.unref();
+        }, firstByteTimeoutMs);
     };
 
     const clearFirstByteTimer = () => {
@@ -479,10 +567,10 @@ export async function consumeGeminiSdkStream(streamResult, { signal, onStreamDel
             idleTimedOut = true;
             if (inFlightReject) {
                 const e = geminiTimeoutError(`${label} SSE idle`, PROVIDER_SSE_IDLE_TIMEOUT_MS);
-                const r = inFlightReject; inFlightReject = null; r(e);
+                const r = inFlightReject; inFlightReject = null;
+                rejectAfterCancellation(r, e);
             }
         }, PROVIDER_SSE_IDLE_TIMEOUT_MS);
-        if (idleTimer.unref) idleTimer.unref();
     };
 
     if (signal?.aborted) {
@@ -490,7 +578,17 @@ export async function consumeGeminiSdkStream(streamResult, { signal, onStreamDel
         throw reason instanceof Error ? reason : new Error(`${label} aborted`);
     }
 
-    const iterator = streamResult.stream[Symbol.asyncIterator]();
+    iterator = streamResult.stream[Symbol.asyncIterator]();
+    // The SDK tees the parsed stream into `response`. Even though local
+    // aggregation normally avoids awaiting it, its rejection must remain
+    // observed across parser/abort retries.
+    let responsePromise;
+    try {
+        responsePromise = Promise.resolve(streamResult.response);
+    } catch (err) {
+        responsePromise = Promise.reject(err);
+    }
+    responsePromise.catch(() => {});
 
     // Wire the abort signal to actually CANCEL iteration (mirror of the REST
     // consumer's onAbort -> reader.cancel). Without this, a parent / client /
@@ -511,10 +609,7 @@ export async function consumeGeminiSdkStream(streamResult, { signal, onStreamDel
                 firstByteReject = null;
                 r(err);
             }
-            try {
-                const ret = iterator.return?.();
-                if (ret && typeof ret.catch === 'function') ret.catch(() => {});
-            } catch {}
+            cancelInFlight(err).catch(() => {});
         };
         signal.addEventListener('abort', onSignalAbort, { once: true });
     }
@@ -536,12 +631,26 @@ export async function consumeGeminiSdkStream(streamResult, { signal, onStreamDel
                         (value) => {
                             inFlightReject = null;
                             firstByteReject = null;
-                            resolve(value);
+                            if (forcedFailure) {
+                                cancellation.then(
+                                    () => reject(forcedFailure),
+                                    () => reject(forcedFailure),
+                                );
+                            } else {
+                                resolve(value);
+                            }
                         },
                         (err) => {
                             inFlightReject = null;
                             firstByteReject = null;
-                            reject(err);
+                            if (forcedFailure) {
+                                cancellation.then(
+                                    () => reject(forcedFailure),
+                                    () => reject(forcedFailure),
+                                );
+                            } else {
+                                reject(err);
+                            }
                         },
                     );
                 });
@@ -553,7 +662,7 @@ export async function consumeGeminiSdkStream(streamResult, { signal, onStreamDel
                     const reason = signal.reason;
                     throw reason instanceof Error ? reason : new Error(`${label} aborted`);
                 }
-                throw err;
+                throw normalizeGeminiSdkStreamError(err, label);
             }
             if (step.done) break;
             if (!sawStreamChunk) {
@@ -605,9 +714,9 @@ export async function consumeGeminiSdkStream(streamResult, { signal, onStreamDel
     } else {
         let response;
         try {
-            response = await streamResult.response;
+            response = await responsePromise;
         } catch (err) {
-            let failure = err;
+            let failure = normalizeGeminiSdkStreamError(err, label);
             if (signal?.aborted) {
                 const reason = signal.reason;
                 failure = reason instanceof Error ? reason : new Error(`${label} aborted`);

package/src/runtime/agent/orchestrator/providers/gemini.mjs

@@ -7,6 +7,8 @@ import {
     PROVIDER_MAX_BEFORE_WARN_MS,
     PROVIDER_SSE_IDLE_TIMEOUT_MS,
     PROVIDER_SSE_IDLE_WATCHDOG_ENABLED,
+    PROVIDER_CACHE_CREATE_TIMEOUT_MS,
+    PROVIDER_CACHE_CREATE_TOTAL_TIMEOUT_MS,
     providerTimeoutError,
     resolveTimeoutMs,
     createTimeoutSignal,
@@ -48,6 +50,7 @@ import {
     GEMINI_GLOBAL_CACHE_DELETE_GRACE_MS,
     _geminiCredentialFingerprint,
     _invalidateGeminiCachesForCredentialFingerprint,
+    _invalidateGeminiCacheName,
 } from './gemini-cache.mjs';
 import {
     GEMINI_MODELS as MODELS,
@@ -68,6 +71,41 @@ export { fetchGeminiModelPages, resolveLatestGeminiModel, ensureLatestGeminiMode
 // De-dupes concurrent force-refreshes so they share one HTTP round-trip,
 // mirroring anthropic-oauth's _modelRefreshInFlight.
 let _modelRefreshInFlight = null;
+const GEMINI_AVAILABILITY_TIMEOUT_MS = 1_000;
+
+function geminiRestError(res, text, label) {
+    let payload = null;
+    try { payload = text ? JSON.parse(text) : null; } catch {}
+    const detail = payload?.error || payload || null;
+    const err = new Error(`${label} ${res.status}: ${text.slice(0, 300)}`);
+    err.status = res.status;
+    err.httpStatus = res.status;
+    err.headers = res.headers;
+    if (detail) {
+        err.error = detail;
+        err.data = payload;
+        if (detail.status) err.geminiStatus = detail.status;
+        if (Array.isArray(detail.details)) err.details = detail.details;
+        const retryAfter = res.headers?.get?.('retry-after')
+            ?? res.headers?.get?.('retry-after-ms');
+        // RESOURCE_EXHAUSTED without a server retry window is deterministic
+        // quota exhaustion. With Retry-After present, leave it request-local
+        // so withRetry can honor the mandated delay.
+        if (detail.status && (retryAfter == null || retryAfter === '')) {
+            err.code = detail.status;
+        }
+    }
+    return err;
+}
+
+function isGeminiCachedContentError(err, cacheName) {
+    const status = Number(err?.status || err?.httpStatus || 0);
+    if (status !== 400 && status !== 404) return false;
+    const text = `${err?.message || ''} ${JSON.stringify(err?.data || '')}`.toLowerCase();
+    return text.includes('cachedcontent')
+        || text.includes('cached content')
+        || (cacheName && text.includes(String(cacheName).toLowerCase()));
+}
 
 // --- Cache accounting/trace: extracted to gemini-cache.mjs ---
 // --- Stream consumption/guards: extracted to gemini-stream.mjs ---
@@ -129,7 +167,8 @@ export class GeminiProvider {
     // one cache slot warm without re-creation overhead; storage cost (~$0.5/M
     // tokens/hour) is dwarfed by the 75% input-price discount on hits beyond
     // a few iterations.
-    async _ensureGeminiCache({ apiKey, model, systemInstruction, geminiTools, toolConfig, contents, opts }) {
+    async _ensureGeminiCache({ apiKey, model, systemInstruction, geminiTools, toolConfig, contents, opts, skipExplicitCache = false }) {
+        if (skipExplicitCache) return null;
         if (Array.isArray(opts?.nativeTools) && opts.nativeTools.length) return null;
         // Kill-switch: MIXDOG_GEMINI_EXPLICIT_CACHE=0 skips cachedContents
         // entirely and relies on Gemini's implicit prefix caching (2.5+/3.x
@@ -314,17 +353,23 @@ export class GeminiProvider {
             // only the 20s ceiling. Without merging opts.signal a session that is
             // aborted (stall-watchdog / closeSession) mid-cache-create leaves this
             // preflight request running until its own timeout fires.
-            const res = await this._fetch(url, {
-                method: 'POST',
-                headers: { 'Content-Type': 'application/json' },
-                body: JSON.stringify(body),
-                signal: opts.signal
-                    ? AbortSignal.any([opts.signal, AbortSignal.timeout(20_000)])
-                    : AbortSignal.timeout(20_000),
-                dispatcher: getLlmDispatcher(),
-            });
-            if (!res.ok) {
-                const text = await res.text().catch(() => '');
+            const createTotal = createTimeoutSignal(
+                opts.signal,
+                PROVIDER_CACHE_CREATE_TOTAL_TIMEOUT_MS,
+                'Gemini cachedContents.create total',
+            );
+            let data;
+            try {
+                data = await withRetry(async ({ signal: attemptSignal }) => {
+                    const res = await this._fetch(url, {
+                        method: 'POST',
+                        headers: { 'Content-Type': 'application/json' },
+                        body: JSON.stringify(body),
+                        signal: attemptSignal,
+                        dispatcher: getLlmDispatcher(),
+                    });
+                    if (res.ok) return await res.json();
+                    const text = await res.text().catch(() => '');
                 try {
                     appendAgentTrace({
                         sessionId: opts.sessionId || opts.session?.id || null,
@@ -339,9 +384,15 @@ export class GeminiProvider {
                         },
                     });
                 } catch {}
-                return null;
+                    throw geminiRestError(res, text, 'Gemini cachedContents.create');
+                }, {
+                    signal: createTotal.signal,
+                    perAttemptTimeoutMs: PROVIDER_CACHE_CREATE_TIMEOUT_MS,
+                    perAttemptLabel: 'Gemini cachedContents.create',
+                });
+            } finally {
+                createTotal.cleanup();
             }
-            const data = await res.json();
             const cacheName = data?.name || null;
             if (!cacheName) return null;
             const cacheTokenSize = Number(data?.usageMetadata?.totalTokenCount || 0) || 0;
@@ -422,7 +473,9 @@ export class GeminiProvider {
         geminiGlobalCacheCreates.set(globalCacheKey, createTask);
         try {
             const created = await createTask;
-            if (!created?.cacheName) return canAttachState ? state.cacheName : null;
+            // A failed refresh must not silently retain a cache that may have
+            // expired or been evicted server-side. The caller proceeds uncached.
+            if (!created?.cacheName) return null;
             _attachGeminiCacheState(opts, created, currentIter);
             return created.cacheName;
         } finally {
@@ -460,7 +513,7 @@ export class GeminiProvider {
         }
     }
 
-    async _doSend(messages, model, tools, sendOpts) {
+    async _doSend(messages, model, tools, sendOpts, internal = {}) {
         const opts = sendOpts || {};
         const signal = opts.signal || null;
         const onStreamDelta = typeof opts.onStreamDelta === 'function' ? opts.onStreamDelta : null;
@@ -510,6 +563,7 @@ export class GeminiProvider {
             toolConfig,
             contents,
             opts,
+            skipExplicitCache: internal.skipExplicitCache === true,
         });
         try { opts.onStageChange?.('requesting'); } catch {}
 
@@ -569,9 +623,7 @@ export class GeminiProvider {
                         }
                         if (!res.ok) {
                             const text = await res.text().catch(() => '');
-                            const err = new Error(`Gemini REST streamGenerateContent ${res.status}: ${text.slice(0, 300)}`);
-                            err.status = res.status;
-                            throw err;
+                            throw geminiRestError(res, text, 'Gemini REST streamGenerateContent');
                         }
                         textLeakGuard = buildTextLeakGuard();
                         return await consumeGeminiRestStreamResponse(res, {
@@ -590,6 +642,18 @@ export class GeminiProvider {
                         },
                     },
                 );
+            } catch (err) {
+                if (!internal.skipExplicitCache
+                    && err?.unsafeToRetry !== true
+                    && isGeminiCachedContentError(err, cachedContent)) {
+                    _invalidateGeminiCacheName(cachedContent);
+                    if (opts.providerState?.gemini?.cacheName === cachedContent) {
+                        const { gemini: _dropGemini, ...rest } = opts.providerState;
+                        opts.providerState = rest;
+                    }
+                    return await this._doSend(messages, model, tools, opts, { skipExplicitCache: true });
+                }
+                throw err;
             } finally {
                 restPassthrough.cleanup();
             }
@@ -666,6 +730,9 @@ export class GeminiProvider {
                                 onTextDelta,
                                 textLeakGuard,
                                 label: 'Gemini SDK streamGenerateContent',
+                                cancelGeneration: (reason) => {
+                                    if (!reqController.signal.aborted) reqController.abort(reason);
+                                },
                             });
                         } finally {
                             clearConnectTimer();
@@ -867,13 +934,30 @@ export class GeminiProvider {
     }
 
     async isAvailable() {
+        const controller = new AbortController();
+        const timeout = setTimeout(() => {
+            controller.abort(providerTimeoutError('Gemini availability probe', GEMINI_AVAILABILITY_TIMEOUT_MS));
+        }, GEMINI_AVAILABILITY_TIMEOUT_MS);
         try {
             const model = this.genAI.getGenerativeModel({ model: DEFAULT_MODEL });
-            await model.generateContent('hi');
+            const generation = Promise.resolve(model.generateContent('hi', { signal: controller.signal }));
+            generation.catch(() => {});
+            await Promise.race([
+                generation,
+                new Promise((_, reject) => {
+                    if (controller.signal.aborted) {
+                        reject(controller.signal.reason);
+                        return;
+                    }
+                    controller.signal.addEventListener('abort', () => reject(controller.signal.reason), { once: true });
+                }),
+            ]);
             return true;
         }
         catch {
             return false;
+        } finally {
+            clearTimeout(timeout);
         }
     }
 }