@juspay/neurolink 12.20.0 → 12.20.2

package/dist/proxy/otelLogSink.d.ts

@@ -52,12 +52,22 @@ export declare function getProxyOtelLogSnapshot(): {
         exportUnconfirmed: number;
         dropped: number;
         outstanding: number;
+        outstandingBytes: number;
+        highWaterBytes: number;
+        byteLimitDrops: number;
+        retryingRecords: number;
+        retriedBatches: number;
+        retentionExpired: number;
+        lastTransientFailureAt: string | undefined;
         lastAcknowledgedAt: string | undefined;
         lastFailureAt: string | undefined;
         highWaterOutstanding: number;
         failureHistoryEvicted: number;
-        kind: "metadata" | "bodies";
+        kind: "metadata" | "bodies" | "indexes" | "diagnostics";
         capacity: number;
+        byteCapacity: number;
+        retentionMs: number;
+        oldestOutstandingAgeMs: number;
     }[];
 };
 /** Bounded provider flush belongs after final request and lifecycle publication. */

package/dist/proxy/otelLogSink.js

@@ -5,7 +5,7 @@ import { getProxyRequestTraceContext, proxyLogContext, } from "./proxyTraceConte
 import { setImmediate as yieldToRequests } from "node:timers/promises";
 import { SeverityNumber } from "@opentelemetry/api-logs";
 import { ExportResultCode } from "@opentelemetry/core";
-import { createProxyOtlpLogTransport } from "./otlpLogTransport.js";
+import { createProxyOtlpLogTransport, getProxyOtlpRetryAfter, } from "./otlpLogTransport.js";
 import { resourceFromAttributes } from "@opentelemetry/resources";
 import { BatchLogRecordProcessor, LoggerProvider, } from "@opentelemetry/sdk-logs";
 import { sanitizeForLog } from "../utils/logSanitize.js";
@@ -21,6 +21,21 @@ const BODY_OTLP_CHUNK_SIZE = 128 * 1024;
 const OTLP_EXPORT_TIMEOUT_MS = 30_000;
 const OTLP_EXPORT_CALLBACK_DEADLINE_MS = OTLP_EXPORT_TIMEOUT_MS + 1_000;
 const BODY_PUBLICATION_ACTIVE_DEADLINE_MS = 20_000;
+const METADATA_RETENTION_MS = 120_000;
+const DIAGNOSTIC_KINDS = new Set(["console", "runtime", "telemetry_delivery"]);
+/** Payload accounting includes UTF-8 attributes and fixed per-record overhead. */
+function retainedRecordBytes(record) {
+    try {
+        return (512 +
+            Buffer.byteLength(typeof record.body === "string"
+                ? record.body
+                : (JSON.stringify(record.body) ?? ""), "utf8") +
+            Buffer.byteLength(JSON.stringify(record.attributes), "utf8"));
+    }
+    catch {
+        return Infinity;
+    }
+}
 const bodyDelivery = {
     attempted: 0,
     transportAcknowledged: 0,
@@ -39,9 +54,12 @@ export function isProxyOtelOnly() {
     return process.env.NEUROLINK_PROXY_LOG_SINK === "otel";
 }
 /** Reserve capacity including exports in flight, independently for metadata and bodies. */
-function createTrackedProcessor(url, capacity, kind) {
+function createTrackedProcessor(url, capacity, kind, byteCapacity) {
     const unsettled = new Set();
     const enqueuedAt = new WeakMap();
+    const retainedBytes = new WeakMap();
+    const stopRetentionWaiters = new Set();
+    const retentionMs = kind === "metadata" || kind === "indexes" ? METADATA_RETENTION_MS : 0;
     const capacityWaiters = new Set();
     const flushWaiters = new Set();
     const state = {
@@ -51,6 +69,13 @@ function createTrackedProcessor(url, capacity, kind) {
         exportUnconfirmed: 0,
         dropped: 0,
         outstanding: 0,
+        outstandingBytes: 0,
+        highWaterBytes: 0,
+        byteLimitDrops: 0,
+        retryingRecords: 0,
+        retriedBatches: 0,
+        retentionExpired: 0,
+        lastTransientFailureAt: undefined,
         lastAcknowledgedAt: undefined,
         lastFailureAt: undefined,
         highWaterOutstanding: 0,
@@ -113,17 +138,32 @@ function createTrackedProcessor(url, capacity, kind) {
                 }
             }
             let settled = false;
-            const deadline = setTimeout(() => settle({
-                code: ExportResultCode.FAILED,
-                error: new Error("OTLP export callback deadline exceeded"),
-            }), OTLP_EXPORT_CALLBACK_DEADLINE_MS);
-            deadline.unref();
+            let retrying = false;
+            let retryTimer;
+            let deadline;
+            let lastFailure;
+            let attempt = 0;
+            const oldestEnqueuedAt = Math.min(...records.map((record) => enqueuedAt.get(record) ?? exportStartedAt));
+            const retainUntil = oldestEnqueuedAt + retentionMs;
+            const stopRetention = () => {
+                // Only a waiting retry is settled early. An active HTTP export keeps
+                // ownership until its callback/deadline so capacity stays truthful.
+                if (retryTimer && lastFailure) {
+                    settle(lastFailure);
+                }
+            };
+            stopRetentionWaiters.add(stopRetention);
             const settle = (result) => {
                 if (settled) {
                     return;
                 }
                 settled = true;
                 clearTimeout(deadline);
+                clearTimeout(retryTimer);
+                stopRetentionWaiters.delete(stopRetention);
+                if (retrying) {
+                    state.retryingRecords -= records.length;
+                }
                 state.outstanding -= records.length;
                 if (result.code === ExportResultCode.SUCCESS) {
                     state.transportAcknowledged += records.length;
@@ -137,6 +177,7 @@ function createTrackedProcessor(url, capacity, kind) {
                 }
                 for (const record of records) {
                     unsettled.delete(record);
+                    state.outstandingBytes -= retainedBytes.get(record) ?? 0;
                     const id = record.attributes?.["body.capture_id"];
                     const publication = typeof id === "string" ? bodyPublications.get(id) : undefined;
                     if (publication) {
@@ -160,15 +201,64 @@ function createTrackedProcessor(url, capacity, kind) {
                 }
                 callback(result);
             };
-            try {
-                transport.export(records, settle);
-            }
-            catch (error) {
-                settle({
+            const send = () => {
+                retryTimer = undefined;
+                if (retentionMs && performance.now() >= retainUntil) {
+                    state.retentionExpired += records.length;
+                    settle({
+                        code: ExportResultCode.FAILED,
+                        error: new Error("OTLP metadata retention budget exceeded"),
+                    });
+                    return;
+                }
+                attempt++;
+                if (attempt > 1) {
+                    state.retriedBatches++;
+                }
+                let attemptSettled = false;
+                const completed = (result) => {
+                    if (attemptSettled || settled) {
+                        return;
+                    }
+                    attemptSettled = true;
+                    clearTimeout(deadline);
+                    const retryAfter = result.code === ExportResultCode.FAILED
+                        ? getProxyOtlpRetryAfter(result.error)
+                        : undefined;
+                    if (retentionMs && retryAfter !== undefined && !shuttingDown) {
+                        state.lastTransientFailureAt = new Date().toISOString();
+                        const wait = Math.max(retryAfter, Math.min(10_000, 1000 * 2 ** Math.min(attempt - 1, 4)) *
+                            (0.8 + Math.random() * 0.4));
+                        if (performance.now() + wait < retainUntil) {
+                            if (!retrying) {
+                                retrying = true;
+                                state.retryingRecords += records.length;
+                            }
+                            lastFailure = result;
+                            retryTimer = setTimeout(send, wait);
+                            retryTimer.unref();
+                            return;
+                        }
+                        state.retentionExpired += records.length;
+                    }
+                    settle(result);
+                };
+                deadline = setTimeout(() => completed({
                     code: ExportResultCode.FAILED,
-                    error: error instanceof Error ? error : new Error(String(error)),
-                });
-            }
+                    error: new Error("OTLP export callback deadline exceeded"),
+                }), OTLP_EXPORT_CALLBACK_DEADLINE_MS);
+                deadline.unref();
+                try {
+                    transport.export(records, completed);
+                }
+                catch (error) {
+                    completed({
+                        code: ExportResultCode.FAILED,
+                        error: error instanceof Error ? error : new Error(String(error)),
+                    });
+                }
+            };
+            send();
         },
         shutdown: () => transport.shutdown(),
     };
@@ -176,7 +266,7 @@ function createTrackedProcessor(url, capacity, kind) {
         maxQueueSize: capacity,
         maxExportBatchSize: 64,
         scheduledDelayMillis: kind === "bodies" ? 25 : 1000,
-        exportTimeoutMillis: OTLP_EXPORT_CALLBACK_DEADLINE_MS,
+        exportTimeoutMillis: retentionMs + OTLP_EXPORT_CALLBACK_DEADLINE_MS + 1000,
     });
     const processor = {
         onEmit(record) {
@@ -187,8 +277,13 @@ function createTrackedProcessor(url, capacity, kind) {
             if (publication) {
                 publication.emitted++;
             }
-            if (state.outstanding >= capacity) {
+            const bytes = retainedRecordBytes(record);
+            if (state.outstanding >= capacity ||
+                state.outstandingBytes + bytes > byteCapacity) {
                 state.dropped++;
+                if (state.outstandingBytes + bytes > byteCapacity) {
+                    state.byteLimitDrops++;
+                }
                 rememberFailure([record], "queue_full");
                 if (publication) {
                     publication.dropped++;
@@ -198,6 +293,9 @@ function createTrackedProcessor(url, capacity, kind) {
             }
             state.submitted++;
             state.outstanding++;
+            state.outstandingBytes += bytes;
+            state.highWaterBytes = Math.max(state.highWaterBytes, state.outstandingBytes);
+            retainedBytes.set(record, bytes);
             unsettled.add(record);
             enqueuedAt.set(record, performance.now());
             state.highWaterOutstanding = Math.max(state.highWaterOutstanding, state.outstanding);
@@ -250,7 +348,28 @@ function createTrackedProcessor(url, capacity, kind) {
         capacityWaiters.add(check);
         check();
     });
-    return { state, processor, capacity, kind, waitForCapacity };
+    return {
+        state,
+        processor,
+        capacity,
+        byteCapacity,
+        retentionMs,
+        kind,
+        waitForCapacity,
+        abortPending: () => transport.abortPending(),
+        stopRetention: () => {
+            for (const stop of stopRetentionWaiters) {
+                stop();
+            }
+        },
+        oldestOutstandingAgeMs: () => {
+            let oldest = performance.now();
+            for (const record of unsettled) {
+                oldest = Math.min(oldest, enqueuedAt.get(record) ?? oldest);
+            }
+            return unsettled.size ? Math.max(0, performance.now() - oldest) : 0;
+        },
+    };
 }
 /**
  * Publish bounded captures concurrently into a shared batch queue. Transport
@@ -450,10 +569,13 @@ export function initializeProxyOtelLogs(role = "worker") {
             throw new Error("Proxy OTLP logs require HTTPS for non-loopback collectors");
         }
     }
-    const metadata = createTrackedProcessor(endpoint, 2048, "metadata");
-    const bodies = createTrackedProcessor(bodyEndpoint, 256, "bodies");
-    queues.push(metadata, bodies);
+    const metadata = createTrackedProcessor(endpoint, 2048, "metadata", 8 * 1024 * 1024);
+    const bodies = createTrackedProcessor(bodyEndpoint, 256, "bodies", 40 * 1024 * 1024);
+    const diagnostics = createTrackedProcessor(endpoint, 512, "diagnostics", 2 * 1024 * 1024);
+    const indexes = createTrackedProcessor(endpoint, 1024, "indexes", 8 * 1024 * 1024);
+    queues.push(metadata, bodies, diagnostics, indexes);
     provider = new LoggerProvider({
+        forceFlushTimeoutMillis: METADATA_RETENTION_MS + OTLP_EXPORT_CALLBACK_DEADLINE_MS + 2000,
         resource: resourceFromAttributes({
             "service.name": process.env.OTEL_SERVICE_NAME ?? "neurolink-proxy",
             "service.instance.id": `${role}-${process.pid}`,
@@ -465,7 +587,13 @@ export function initializeProxyOtelLogs(role = "worker") {
                 onEmit(record, context) {
                     (record.attributes?.["proxy.record_kind"] === "body"
                         ? bodies
-                        : metadata).processor.onEmit(record, context);
+                        : record.attributes?.["proxy.record_kind"] === "body_capture_index"
+                            ? indexes
+                            : record.attributes?.["proxy.lifecycle.event"] ===
+                                "runtime_sample" ||
+                                DIAGNOSTIC_KINDS.has(String(record.attributes?.["proxy.record_kind"]))
+                                ? diagnostics
+                                : metadata).processor.onEmit(record, context);
                 },
                 forceFlush: async () => {
                     await Promise.all(queues.map((q) => q.processor.forceFlush()));
@@ -577,6 +705,9 @@ export function getProxyOtelLogSnapshot() {
         queues: queues.map((q) => ({
             kind: q.kind,
             capacity: q.capacity,
+            byteCapacity: q.byteCapacity,
+            retentionMs: q.retentionMs,
+            oldestOutstandingAgeMs: q.oldestOutstandingAgeMs(),
             ...q.state,
             recentFailures: q.state.recentFailures.map((failure) => ({
                 ...failure,
@@ -593,10 +724,27 @@ export async function flushProxyOtelLogs() {
 /** Release this process's exporter and restore console ownership. */
 export async function shutdownProxyOtelLogs() {
     shuttingDown = true;
-    await Promise.allSettled([...bodyPublicationOperations]);
-    restoreConsole?.();
-    restoreConsole = undefined;
-    await provider?.shutdown();
+    for (const queue of queues) {
+        queue.stopRetention();
+    }
+    // Service cleanup gives telemetry five seconds. End waiting retries now,
+    // then let healthy exports drain; cancel unfinished HTTP work before that
+    // outer deadline. Cancellation remains unconfirmed, never acknowledged.
+    const deadline = setTimeout(() => {
+        for (const queue of queues) {
+            queue.abortPending();
+        }
+    }, 4000);
+    deadline.unref();
+    try {
+        await Promise.allSettled([...bodyPublicationOperations]);
+        restoreConsole?.();
+        restoreConsole = undefined;
+        await provider?.shutdown();
+    }
+    finally {
+        clearTimeout(deadline);
+    }
     provider = undefined;
     queues.length = 0;
     invalidRecords = 0;
@@ -624,7 +772,6 @@ export function withProxyOtelLogShutdown(handler) {
             await handler(arg);
         }
         finally {
-            await flushProxyOtelLogs().catch(() => undefined);
             await shutdownProxyOtelLogs().catch(() => undefined);
         }
     };

package/dist/proxy/otlpLogTransport.d.ts

@@ -1,3 +1,5 @@
-import type { LogRecordExporter } from "@opentelemetry/sdk-logs";
+import type { ProxyOtlpLogTransport } from "../types/index.js";
+/** Only explicit transient transport failures may be retained and retried. */
+export declare function getProxyOtlpRetryAfter(error: Error | undefined): number | undefined;
 /** Both proxy queues use this transport; success is not backend persistence. */
-export declare function createProxyOtlpLogTransport(endpoint: string, timeoutMillis: number): LogRecordExporter;
+export declare function createProxyOtlpLogTransport(endpoint: string, timeoutMillis: number): ProxyOtlpLogTransport;

package/dist/proxy/otlpLogTransport.js

@@ -36,6 +36,12 @@ class ProxyOtlpResponseError extends Error {
         this.name = "ProxyOtlpResponseError";
     }
 }
+/** Only explicit transient transport failures may be retained and retried. */
+export function getProxyOtlpRetryAfter(error) {
+    return error instanceof ProxyOtlpResponseError && error.retryable
+        ? (error.retryAfterMs ?? 0)
+        : undefined;
+}
 function signalSetting(suffix) {
     return (process.env[`OTEL_EXPORTER_OTLP_LOGS_${suffix}`]?.trim() ||
         process.env[`OTEL_EXPORTER_OTLP_${suffix}`]?.trim() ||
@@ -200,6 +206,7 @@ export function createProxyOtlpLogTransport(endpoint, timeoutMillis) {
     const request = url.protocol === "https:" ? httpsRequest : httpRequest;
     let closed = false;
     const active = new Set();
+    const controllers = new Set();
     const sendOnce = (data, signal) => new Promise((resolve, reject) => {
         let receivedResponse = false;
         const req = request(url, {
@@ -256,13 +263,13 @@ export function createProxyOtlpLogTransport(endpoint, timeoutMillis) {
         });
         req.on("error", (error) => reject(new ProxyOtlpResponseError(signal.aborted
             ? "OTLP export deadline exceeded"
-            : "OTLP transport failed", !signal.aborted &&
-            !receivedResponse &&
-            RETRYABLE_NETWORK_CODES.has(error.code ?? ""))));
+            : "OTLP transport failed", !receivedResponse &&
+            (signal.aborted || RETRYABLE_NETWORK_CODES.has(error.code ?? "")))));
         req.end(data);
     });
     const send = async (records) => {
         const controller = new AbortController();
+        controllers.add(controller);
         const deadlineAt = Date.now() + timeoutMillis;
         const deadline = setTimeout(() => controller.abort(), timeoutMillis);
         deadline.unref();
@@ -286,19 +293,19 @@ export function createProxyOtlpLogTransport(endpoint, timeoutMillis) {
                         throw error;
                     }
                     if (attempt >= MAX_EXPORT_ATTEMPTS) {
-                        throw new ProxyOtlpResponseError("OTLP retry attempt limit reached");
+                        throw new ProxyOtlpResponseError("OTLP retry attempt limit reached", true, error.retryAfterMs);
                     }
                     const exponential = Math.min(5000, 1000 * 2 ** Math.min(attempt - 1, 3));
                     // Retry-After is a lower bound, never permission to hammer a busy
                     // collector. Even Retry-After: 0 retains exponential backoff+jitter.
                     const wait = Math.max(error.retryAfterMs ?? 0, exponential * (0.8 + Math.random() * 0.4));
                     if (wait >= deadlineAt - Date.now()) {
-                        throw new ProxyOtlpResponseError("OTLP retry exceeds export deadline");
+                        throw new ProxyOtlpResponseError("OTLP retry exceeds export deadline", true, wait);
                     }
                     await delay(wait, undefined, { signal: controller.signal });
                 }
             }
-            throw new ProxyOtlpResponseError("OTLP export deadline exceeded");
+            throw new ProxyOtlpResponseError("OTLP export deadline exceeded", true);
         }
         catch (error) {
             if (error instanceof ProxyOtlpResponseError) {
@@ -306,13 +313,21 @@ export function createProxyOtlpLogTransport(endpoint, timeoutMillis) {
             }
             throw new ProxyOtlpResponseError(controller.signal.aborted
                 ? "OTLP export deadline exceeded"
-                : "OTLP export failed");
+                : "OTLP export failed", controller.signal.aborted);
         }
         finally {
             clearTimeout(deadline);
+            controllers.delete(controller);
         }
     };
     return {
+        abortPending() {
+            closed = true;
+            for (const controller of controllers) {
+                controller.abort();
+            }
+            agent.destroy();
+        },
         export(records, callback) {
             if (closed) {
                 callback({

package/dist/proxy/requestLogger.js

@@ -7,6 +7,7 @@
  */
 import { emitProxyOtelEvent, getProxyOtelLogSnapshot, initializeProxyOtelLogs, isProxyOtelOnly, publishProxyOtelBody, } from "./otelLogSink.js";
 import { randomUUID } from "node:crypto";
+import { createProxyBodyCapturePolicy } from "./bodyCapturePolicy.js";
 import { join } from "path";
 import { homedir } from "os";
 import { logger } from "../utils/logger.js";
@@ -26,6 +27,7 @@ import { withTimeout } from "../utils/async/withTimeout.js";
 import { resolveProxyLogTraceContext, proxyLogContext, } from "./proxyTraceContext.js";
 let logDir = null;
 let logEnabled = false;
+let bodyCapturePolicy = createProxyBodyCapturePolicy();
 const pendingLogOperations = new Set();
 const REQUEST_LOG_IO_TIMEOUT_MS = 5_000;
 // launchd gives the service 45 seconds after SIGTERM. Leave fifteen seconds
@@ -61,6 +63,7 @@ export function getRequestLoggerSnapshot() {
         attempts: { ...metadataSinks.attempts },
         debug: { ...metadataSinks.debug },
         bodyCapture: getBodyCaptureWorkerSnapshot(),
+        bodyCapturePolicy: bodyCapturePolicy.snapshot(),
     };
 }
 /**
@@ -165,6 +168,10 @@ export function initRequestLogger(enabled = true, customLogsDir) {
     // Lifecycle metadata deliberately shares the request logger's enablement,
     // directory permissions, retention boundary, and operator privacy control.
     logEnabled = enabled;
+    bodyCapturePolicy = createProxyBodyCapturePolicy({
+        bytesPerMinute: process.env.NEUROLINK_PROXY_BODY_BYTES_PER_MINUTE,
+        deduplicate: process.env.NEUROLINK_PROXY_BODY_DEDUPLICATION !== "false",
+    });
     if (!enabled) {
         configureProxyLifecycleLogger({ enabled: false });
         return;
@@ -699,13 +706,14 @@ export async function logBodyCapture(entry) {
             indexEntry.traceFlags = traceCtx.traceFlags;
         }
         if (isProxyOtelOnly()) {
-            const delivery = await emitOtlpBodyLogRecord({
+            const publication = await bodyCapturePolicy.publish(metadata, stored, () => emitOtlpBodyLogRecord({
                 ...metadata,
                 traceId: traceCtx?.traceId ?? metadata.traceId,
                 spanId: traceCtx?.spanId ?? metadata.spanId,
                 traceFlags: traceCtx?.traceFlags ?? metadata.traceFlags,
-            }, stored);
-            indexEntry.bodyDelivery = delivery ?? {
+            }, stored));
+            indexEntry.bodyReference = publication.reference;
+            indexEntry.bodyDelivery = publication.delivery ?? {
                 status: processed.error
                     ? "capture_rejected"
                     : stored.redactedBody === undefined

package/dist/proxy/restartControl.js

@@ -104,6 +104,21 @@ export async function startProxyRestartControl(options) {
         // its existing streams. Full quiescence is only required before replacing
         // the supervisor itself (updateCoordinator.isProxyRuntimeSettled).
     };
+    const readServingStatus = async () => {
+        let lastError;
+        for (let attempt = 1; attempt <= 3; attempt++) {
+            try {
+                return statusSchema.parse(await options.getStatus());
+            }
+            catch (error) {
+                lastError = error;
+                if (attempt < 3) {
+                    await new Promise((resolve) => setTimeout(resolve, attempt * 100));
+                }
+            }
+        }
+        throw lastError;
+    };
     const operate = async (restart) => {
         if (busy || closing) {
             return result("refused", "A restart check or activation is already in progress.");
@@ -130,7 +145,7 @@ export async function startProxyRestartControl(options) {
                 throw new Error("The serving worker exited during preflight.");
             }
             previousWorkerPid = active.pid;
-            const status = statusSchema.parse(await options.getStatus());
+            const status = await readServingStatus();
             assertSupervisorLogging(status, options.getTelemetry?.());
             if (status.pid !== previousWorkerPid ||
                 status.version !== active.version) {
@@ -156,7 +171,7 @@ export async function startProxyRestartControl(options) {
             handoffBaseline = options.server.snapshot();
             const after = await options.server.replace(version);
             activated = true;
-            const fresh = statusSchema.parse(await options.getStatus());
+            const fresh = await readServingStatus();
             if (fresh.pid !== after.active?.pid ||
                 fresh.pid === previousWorkerPid ||
                 fresh.version !== version) {
@@ -372,7 +387,7 @@ export async function requestProxyRestart(identity, check) {
                 connection: "close",
             },
         });
-        const timeout = setTimeout(() => req.destroy(new Error("Restart result timed out; outcome is unknown. Inspect proxy status before retrying.")), check ? 15_000 : 150_000);
+        const timeout = setTimeout(() => req.destroy(new Error("Restart result timed out; outcome is unknown. Inspect proxy status before retrying.")), check ? 30_000 : 150_000);
         req.once("error", (error) => {
             clearTimeout(timeout);
             reject(error);

package/dist/proxy/rollingProxyServer.js

@@ -14,9 +14,14 @@ export async function startRollingProxyServer(options) {
     let requestedReplacementSchedule = 0;
     let requestedReplacementPending = false;
     let requestedReplacementReason = "environment";
+    let stalledGeneration;
+    let stallReplacementFailures = 0;
+    let nextStallReplacementAt = 0;
     let replacementQueueTail = null;
     const recoveryDelayMs = Math.max(1, options.recoveryDelayMs ?? DEFAULT_RECOVERY_DELAY_MS);
     const maxRecoveryDelayMs = Math.max(recoveryDelayMs, options.maxRecoveryDelayMs ?? DEFAULT_MAX_RECOVERY_DELAY_MS);
+    const stallReplacementDelayMs = Math.max(1, options.stallReplacementDelayMs ?? 60_000);
+    const maxStallReplacementDelayMs = Math.max(stallReplacementDelayMs, options.maxStallReplacementDelayMs ?? 15 * 60_000);
     const queueReplacement = (operation) => {
         const predecessor = replacementQueueTail;
         const result = (predecessor ?? Promise.resolve()).then(operation);
@@ -61,6 +66,21 @@ export async function startRollingProxyServer(options) {
         recoveryTimer.unref?.();
     };
     const stateChanged = (snapshot) => {
+        if (requestedReplacementReason !== "environment" &&
+            stalledGeneration !== undefined &&
+            snapshot.active?.generation !== stalledGeneration) {
+            // A pressure retry belongs only to the generation that stalled. Once it
+            // exits or is replaced, ordinary recovery owns the next generation.
+            if (requestedReplacementTimer) {
+                clearTimeout(requestedReplacementTimer);
+                requestedReplacementTimer = undefined;
+            }
+            requestedReplacementSchedule += 1;
+            requestedReplacementPending = false;
+            stalledGeneration = undefined;
+            stallReplacementFailures = 0;
+            nextStallReplacementAt = 0;
+        }
         try {
             options.onStateChange?.(snapshot);
         }
@@ -78,6 +98,12 @@ export async function startRollingProxyServer(options) {
         requestedReplacementPending = true;
         if (request) {
             requestedReplacementReason = request.reason;
+            if (request.reason !== "environment" &&
+                stalledGeneration !== request.generation) {
+                stalledGeneration = request.generation;
+                stallReplacementFailures = 0;
+                nextStallReplacementAt = 0;
+            }
         }
         if (requestedReplacementTimer || replacementQueueTail) {
             return;
@@ -89,22 +115,41 @@ export async function startRollingProxyServer(options) {
                 return;
             }
             if (closing || !supervisor.snapshot().active) {
+                requestedReplacementPending = false;
                 return;
             }
             requestedReplacementPending = false;
             const replacementVersion = desiredVersion;
             const replacementReason = requestedReplacementReason;
+            const replacementGeneration = supervisor.snapshot().active?.generation;
             void queueReplacement(async () => {
                 if (closing || !supervisor.snapshot().active) {
                     return;
                 }
                 options.log?.(`[proxy-supervisor] preparing same-version worker replacement version=${replacementVersion} reason=${replacementReason}`);
                 await supervisor.replace(replacementVersion);
+                stallReplacementFailures = 0;
+                nextStallReplacementAt = 0;
                 options.log?.(`[proxy-supervisor] same-version worker replacement complete version=${replacementVersion} reason=${replacementReason}`);
             }).catch((error) => {
                 options.log?.(`[proxy-supervisor] same-version worker replacement failed version=${replacementVersion} reason=${replacementReason}: ${error instanceof Error ? error.message : String(error)}`);
+                // A failed candidate does not invalidate the old generation's streams.
+                // Retry pressure recovery with a growing, capped delay. New requests
+                // cannot reset that delay, and an unrelated successful rotation ends it.
+                if (replacementReason !== "environment" &&
+                    !closing &&
+                    supervisor.snapshot().active?.generation ===
+                        replacementGeneration &&
+                    schedule === requestedReplacementSchedule) {
+                    stallReplacementFailures += 1;
+                    const delay = Math.min(maxStallReplacementDelayMs, stallReplacementDelayMs *
+                        2 ** Math.min(stallReplacementFailures - 1, 16));
+                    nextStallReplacementAt = Date.now() + delay;
+                    options.log?.(`[proxy-supervisor] stall replacement retry deferred failures=${stallReplacementFailures} delayMs=${delay}`);
+                    scheduleRequestedReplacement();
+                }
             });
-        }, 50);
+        }, Math.max(50, nextStallReplacementAt - Date.now()));
         requestedReplacementTimer.unref?.();
     }
     const supervisor = new RollingWorkerSupervisor({
@@ -189,6 +234,8 @@ export async function startRollingProxyServer(options) {
             try {
                 const snapshot = await queueReplacement(() => supervisor.replace(expectedVersion));
                 recoveryFailures = 0;
+                stallReplacementFailures = 0;
+                nextStallReplacementAt = 0;
                 return snapshot;
             }
             catch (error) {