@juspay/neurolink 10.12.5 → 10.12.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -646,6 +646,7 @@ export type AnthropicSuccessResult = {
646
646
  failure?: {
647
647
  message: string;
648
648
  rateLimit: boolean;
649
+ retryDelayMs?: number;
649
650
  };
650
651
  } | {
651
652
  response: Response | unknown;
@@ -702,6 +703,8 @@ export type AnthropicAuthRetryResult = {
702
703
  response?: Response | unknown;
703
704
  holdsAccountAdmission?: boolean;
704
705
  continueLoop: boolean;
706
+ /** Failure-path pacing before rotating after provider-wide overload. */
707
+ retryDelayMs?: number;
705
708
  lastError: unknown;
706
709
  authFailureMessage: string | null;
707
710
  sawRateLimit: boolean;
@@ -713,6 +716,8 @@ export type AnthropicNonOkResult = {
713
716
  response?: Response | unknown;
714
717
  continueLoop: boolean;
715
718
  retrySameAccount?: boolean;
719
+ /** Failure-path pacing before rotating after provider-wide overload. */
720
+ retryDelayMs?: number;
716
721
  lastError: unknown;
717
722
  authFailureMessage: string | null;
718
723
  sawTransientFailure: boolean;
@@ -1457,6 +1462,9 @@ export type ProxyAnalysisReport = {
1457
1462
  attemptLatency: boolean;
1458
1463
  cacheUsage: boolean;
1459
1464
  routingDecisions: boolean;
1465
+ /** True only when every stream needed for cross-stream request/attempt
1466
+ * reconciliation begins at or before the requested analysis window. */
1467
+ comparableRequestAttempts: boolean;
1460
1468
  };
1461
1469
  dataQuality: {
1462
1470
  linesRead: number;
@@ -1468,6 +1476,8 @@ export type ProxyAnalysisReport = {
1468
1476
  observedFrom: string | null;
1469
1477
  observedTo: string | null;
1470
1478
  startsAtOrBeforeRequestedWindow: boolean;
1479
+ /** Whether this stream can support claims covering the full window. */
1480
+ completeWindow: boolean;
1471
1481
  }>;
1472
1482
  bodyArtifacts: {
1473
1483
  capturesIndexed: number;
@@ -2073,6 +2083,14 @@ export type RollingWorkerFailureDetails = {
2073
2083
  workerExitSignal?: string | null;
2074
2084
  supervisorAction?: "none" | "sigkill_after_transfer_failure";
2075
2085
  };
2086
+ export type RollingWorkerSupervisorEvent = {
2087
+ at: string;
2088
+ type: "activated" | "failure" | "failed_transfer" | "rejected_socket";
2089
+ generation: number | null;
2090
+ version: string | null;
2091
+ phase?: "startup" | "activation" | "runtime" | "transfer";
2092
+ reason?: string;
2093
+ };
2076
2094
  export type RollingWorkerSupervisorSnapshot = {
2077
2095
  generation: number;
2078
2096
  active: {
@@ -2093,6 +2111,8 @@ export type RollingWorkerSupervisorSnapshot = {
2093
2111
  queuedSockets: number;
2094
2112
  rejectedSockets: number;
2095
2113
  failedTransfers: number;
2114
+ /** Bounded generation-scoped evidence for attributing lifetime counters. */
2115
+ recentEvents: RollingWorkerSupervisorEvent[];
2096
2116
  lastFailure: ({
2097
2117
  at: string;
2098
2118
  generation: number;
@@ -2305,7 +2325,18 @@ export type ProxyNeurolinkRuntime = {
2305
2325
  neurolink: {
2306
2326
  getToolRegistry(): MCPToolRegistry;
2307
2327
  };
2308
- cleanupLogs: (daysToKeep?: number, maxFiles?: number) => void;
2328
+ logsDir: string;
2329
+ };
2330
+ /** Data passed to the isolated proxy log-retention worker. */
2331
+ export type ProxyLogCleanupWorkerData = {
2332
+ logsDir: string;
2333
+ maxAgeDays: number;
2334
+ maxSizeMb: number;
2335
+ };
2336
+ /** Lifecycle handle for non-blocking proxy log retention. */
2337
+ export type ProxyLogCleanupScheduler = {
2338
+ trigger: () => boolean;
2339
+ stop: () => Promise<void>;
2309
2340
  };
2310
2341
  /** Hono app + readiness state created by the proxy start command. */
2311
2342
  export type ProxyStartApp = {
@@ -0,0 +1,12 @@
1
+ import type { ProxyLogCleanupScheduler } from "../types/index.js";
2
+ /**
3
+ * Runs retention in a worker so large log trees cannot delay readiness or
4
+ * block active streams. Concurrent runs are coalesced into the active scan.
5
+ */
6
+ export declare function startProxyLogCleanupScheduler(params: {
7
+ logsDir: string;
8
+ maxAgeDays?: number;
9
+ maxSizeMb?: number;
10
+ initialDelayMs?: number;
11
+ intervalMs?: number;
12
+ }): ProxyLogCleanupScheduler;
@@ -0,0 +1,73 @@
1
+ import { Worker } from "node:worker_threads";
2
+ import { withTimeout } from "../utils/async/withTimeout.js";
3
+ import { logger } from "../utils/logger.js";
4
+ const DEFAULT_INITIAL_DELAY_MS = 30_000;
5
+ const DEFAULT_INTERVAL_MS = 60 * 60 * 1000;
6
+ const WORKER_TERMINATION_TIMEOUT_MS = 5_000;
7
+ /**
8
+ * Runs retention in a worker so large log trees cannot delay readiness or
9
+ * block active streams. Concurrent runs are coalesced into the active scan.
10
+ */
11
+ export function startProxyLogCleanupScheduler(params) {
12
+ const maxAgeDays = params.maxAgeDays ?? 7;
13
+ const maxSizeMb = params.maxSizeMb ?? 500;
14
+ let activeWorker;
15
+ let stopped = false;
16
+ const trigger = () => {
17
+ if (stopped || activeWorker) {
18
+ return false;
19
+ }
20
+ let worker;
21
+ try {
22
+ worker = new Worker(params.workerUrl ??
23
+ new URL("./logCleanupWorkerEntry.js", import.meta.url), {
24
+ execArgv: process.execArgv.filter((argument) => !argument.startsWith("--input-type")),
25
+ workerData: {
26
+ logsDir: params.logsDir,
27
+ maxAgeDays,
28
+ maxSizeMb,
29
+ },
30
+ });
31
+ }
32
+ catch (error) {
33
+ logger.debug(`[proxy] could not start background log cleanup: ${error instanceof Error ? error.message : String(error)}`);
34
+ return false;
35
+ }
36
+ activeWorker = worker;
37
+ worker.unref();
38
+ worker.once("error", (error) => {
39
+ logger.debug(`[proxy] background log cleanup failed: ${error instanceof Error ? error.message : String(error)}`);
40
+ });
41
+ worker.once("exit", (code) => {
42
+ if (activeWorker === worker) {
43
+ activeWorker = undefined;
44
+ }
45
+ if (code !== 0 && !stopped) {
46
+ logger.debug(`[proxy] background log cleanup exited with code ${code}`);
47
+ }
48
+ });
49
+ return true;
50
+ };
51
+ const initialTimer = setTimeout(trigger, params.initialDelayMs ?? DEFAULT_INITIAL_DELAY_MS);
52
+ initialTimer.unref();
53
+ const interval = setInterval(trigger, params.intervalMs ?? DEFAULT_INTERVAL_MS);
54
+ interval.unref();
55
+ return {
56
+ trigger,
57
+ stop: async () => {
58
+ stopped = true;
59
+ clearTimeout(initialTimer);
60
+ clearInterval(interval);
61
+ const worker = activeWorker;
62
+ activeWorker = undefined;
63
+ if (worker) {
64
+ try {
65
+ await withTimeout(worker.terminate(), WORKER_TERMINATION_TIMEOUT_MS, "Timed out terminating the proxy log cleanup worker");
66
+ }
67
+ catch (error) {
68
+ logger.debug(`[proxy] background log cleanup termination failed: ${error instanceof Error ? error.message : String(error)}`);
69
+ }
70
+ }
71
+ },
72
+ };
73
+ }
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,14 @@
1
+ import { parentPort, workerData } from "node:worker_threads";
2
+ import { cleanupLogsAt } from "./requestLogger.js";
3
+ const data = workerData;
4
+ try {
5
+ cleanupLogsAt(data.logsDir, data.maxAgeDays, data.maxSizeMb);
6
+ parentPort?.postMessage({ ok: true });
7
+ }
8
+ catch (error) {
9
+ parentPort?.postMessage({
10
+ ok: false,
11
+ error: error instanceof Error ? error.message : String(error),
12
+ });
13
+ process.exitCode = 1;
14
+ }
@@ -737,6 +737,10 @@ export async function analyzeProxyLogs(options) {
737
737
  const artifactsReferenced = artifactsPresent + artifactsMissing;
738
738
  const finalSummary = summarizeFinalRequests(finalRequests, terminalStreamErrors, attemptsByRequest, accounts);
739
739
  const routingSummary = summarizeRouting(finalRequests);
740
+ const streamComplete = (stream) => {
741
+ const range = observedRanges[stream];
742
+ return range.from !== null && range.from <= sinceMs;
743
+ };
740
744
  return {
741
745
  generatedAt: new Date(nowMs).toISOString(),
742
746
  since: new Date(sinceMs).toISOString(),
@@ -755,6 +759,7 @@ export async function analyzeProxyLogs(options) {
755
759
  attemptLatency: attemptLatency.length > 0,
756
760
  cacheUsage: finalSummary.cache.requestsWithUsage > 0,
757
761
  routingDecisions: routingSummary.totalRecords > 0,
762
+ comparableRequestAttempts: streamComplete("requests") && streamComplete("attempts"),
758
763
  },
759
764
  dataQuality: {
760
765
  linesRead,
@@ -768,6 +773,7 @@ export async function analyzeProxyLogs(options) {
768
773
  observedFrom: range.from === null ? null : new Date(range.from).toISOString(),
769
774
  observedTo: range.to === null ? null : new Date(range.to).toISOString(),
770
775
  startsAtOrBeforeRequestedWindow: range.from !== null && range.from <= sinceMs,
776
+ completeWindow: range.from !== null && range.from <= sinceMs,
771
777
  },
772
778
  ])),
773
779
  bodyArtifacts: {
@@ -70,3 +70,9 @@ export declare function logStreamError(entry: {
70
70
  * Non-fatal — proxy keeps working even if cleanup fails.
71
71
  */
72
72
  export declare function cleanupLogs(maxAgeDays?: number, maxSizeMb?: number): void;
73
+ /**
74
+ * Path-scoped retention implementation used by the proxy cleanup worker.
75
+ * This function is intentionally synchronous: callers must run it outside the
76
+ * request-serving process when the directory can contain many artifacts.
77
+ */
78
+ export declare function cleanupLogsAt(activeLogDir: string, maxAgeDays?: number, maxSizeMb?: number): void;
@@ -700,60 +700,70 @@ export async function logStreamError(entry) {
700
700
  * Non-fatal — proxy keeps working even if cleanup fails.
701
701
  */
702
702
  export function cleanupLogs(maxAgeDays = 7, maxSizeMb = 500) {
703
- if (!logDir || !existsSync(logDir)) {
703
+ if (!logDir) {
704
704
  return;
705
705
  }
706
706
  try {
707
- const activeLogDir = logDir;
708
- const files = collectManagedLogFiles(activeLogDir).sort((a, b) => a.mtime - b.mtime); // oldest first
709
- const currentDate = new Date().toISOString().split("T")[0];
710
- const currentMetadataLogs = new Set(["proxy", "proxy-attempts", "proxy-debug", "proxy-lifecycle"].map((prefix) => join(activeLogDir, `${prefix}-${currentDate}.jsonl`)));
711
- const canDelete = (file) => !currentMetadataLogs.has(file.path);
712
- const cutoff = Date.now() - maxAgeDays * 24 * 60 * 60 * 1000;
713
- let deletedCount = 0;
714
- let freedBytes = 0;
715
- // Pass 1: delete files older than maxAgeDays
716
- const remaining = [];
717
- for (const file of files) {
718
- if (file.mtime < cutoff && canDelete(file)) {
719
- unlinkSync(file.path);
720
- deletedCount++;
721
- freedBytes += file.size;
722
- }
723
- else {
724
- remaining.push(file);
725
- }
726
- }
727
- const bodiesDir = join(logDir, "bodies");
728
- if (existsSync(bodiesDir)) {
729
- pruneEmptyDirectories(bodiesDir, bodiesDir);
730
- }
731
- // Pass 2: if total size exceeds maxSizeMb, delete oldest until under limit
732
- const maxBytes = maxSizeMb * 1024 * 1024;
733
- let totalSize = remaining.reduce((sum, f) => sum + f.size, 0);
734
- const deletionCandidates = remaining.filter(canDelete);
735
- // Current-day metadata is the only reliable source for final-request,
736
- // attempt, lifecycle, and body-index reconciliation. Keep those indexes
737
- // intact during size cleanup; body artifacts and older indexes remain
738
- // eligible for eviction.
739
- while (totalSize > maxBytes && deletionCandidates.length > 0) {
740
- const oldest = deletionCandidates.shift();
741
- if (!oldest) {
742
- break;
743
- }
744
- unlinkSync(oldest.path);
745
- totalSize -= oldest.size;
707
+ cleanupLogsAt(logDir, maxAgeDays, maxSizeMb);
708
+ }
709
+ catch {
710
+ // Non-fatal for legacy in-process callers.
711
+ }
712
+ }
713
+ /**
714
+ * Path-scoped retention implementation used by the proxy cleanup worker.
715
+ * This function is intentionally synchronous: callers must run it outside the
716
+ * request-serving process when the directory can contain many artifacts.
717
+ */
718
+ export function cleanupLogsAt(activeLogDir, maxAgeDays = 7, maxSizeMb = 500) {
719
+ if (!existsSync(activeLogDir)) {
720
+ return;
721
+ }
722
+ const files = collectManagedLogFiles(activeLogDir).sort((a, b) => a.mtime - b.mtime); // oldest first
723
+ const currentDate = new Date().toISOString().split("T")[0];
724
+ const currentMetadataLogs = new Set(["proxy", "proxy-attempts", "proxy-debug", "proxy-lifecycle"].map((prefix) => join(activeLogDir, `${prefix}-${currentDate}.jsonl`)));
725
+ const canDelete = (file) => !currentMetadataLogs.has(file.path);
726
+ const cutoff = Date.now() - maxAgeDays * 24 * 60 * 60 * 1000;
727
+ let deletedCount = 0;
728
+ let freedBytes = 0;
729
+ // Pass 1: delete files older than maxAgeDays
730
+ const remaining = [];
731
+ for (const file of files) {
732
+ if (file.mtime < cutoff && canDelete(file)) {
733
+ unlinkSync(file.path);
746
734
  deletedCount++;
747
- freedBytes += oldest.size;
735
+ freedBytes += file.size;
748
736
  }
749
- if (existsSync(bodiesDir)) {
750
- pruneEmptyDirectories(bodiesDir, bodiesDir);
737
+ else {
738
+ remaining.push(file);
751
739
  }
752
- if (deletedCount > 0) {
753
- logger.info(`[proxy] log cleanup: deleted ${deletedCount} file(s), freed ${(freedBytes / 1024 / 1024).toFixed(1)} MB`);
740
+ }
741
+ const bodiesDir = join(activeLogDir, "bodies");
742
+ if (existsSync(bodiesDir)) {
743
+ pruneEmptyDirectories(bodiesDir, bodiesDir);
744
+ }
745
+ // Pass 2: if total size exceeds maxSizeMb, delete oldest until under limit
746
+ const maxBytes = maxSizeMb * 1024 * 1024;
747
+ let totalSize = remaining.reduce((sum, f) => sum + f.size, 0);
748
+ const deletionCandidates = remaining.filter(canDelete);
749
+ // Current-day metadata is the only reliable source for final-request,
750
+ // attempt, lifecycle, and body-index reconciliation. Keep those indexes
751
+ // intact during size cleanup; body artifacts and older indexes remain
752
+ // eligible for eviction.
753
+ while (totalSize > maxBytes && deletionCandidates.length > 0) {
754
+ const oldest = deletionCandidates.shift();
755
+ if (!oldest) {
756
+ break;
754
757
  }
758
+ unlinkSync(oldest.path);
759
+ totalSize -= oldest.size;
760
+ deletedCount++;
761
+ freedBytes += oldest.size;
755
762
  }
756
- catch {
757
- // Non-fatal
763
+ if (existsSync(bodiesDir)) {
764
+ pruneEmptyDirectories(bodiesDir, bodiesDir);
765
+ }
766
+ if (deletedCount > 0) {
767
+ logger.info(`[proxy] log cleanup: deleted ${deletedCount} file(s), freed ${(freedBytes / 1024 / 1024).toFixed(1)} MB`);
758
768
  }
759
769
  }
@@ -14,6 +14,7 @@ export declare class RollingWorkerSupervisor {
14
14
  private replacement;
15
15
  private rejectedSockets;
16
16
  private failedTransfers;
17
+ private readonly recentEvents;
17
18
  private lastFailure;
18
19
  private closed;
19
20
  private shutdownPromise;
@@ -38,5 +39,6 @@ export declare class RollingWorkerSupervisor {
38
39
  private describeTransferError;
39
40
  private extractLifecycleFailureDetails;
40
41
  private recordFailure;
42
+ private recordEvent;
41
43
  private publishState;
42
44
  }
@@ -3,6 +3,7 @@ const DEFAULT_READY_TIMEOUT_MS = 30_000;
3
3
  const DEFAULT_SOCKET_QUEUE_LIMIT = 1_024;
4
4
  const DEFAULT_SOCKET_QUEUE_TIMEOUT_MS = 30_000;
5
5
  const DEFAULT_SHUTDOWN_TIMEOUT_MS = 30_000;
6
+ const MAX_RECENT_SUPERVISOR_EVENTS = 100;
6
7
  /**
7
8
  * Owns worker generations while the caller owns the public listening socket.
8
9
  * Sockets are transferred once to the active worker, so response bytes never
@@ -18,6 +19,7 @@ export class RollingWorkerSupervisor {
18
19
  replacement = null;
19
20
  rejectedSockets = 0;
20
21
  failedTransfers = 0;
22
+ recentEvents = [];
21
23
  lastFailure = null;
22
24
  closed = false;
23
25
  shutdownPromise = null;
@@ -56,6 +58,7 @@ export class RollingWorkerSupervisor {
56
58
  queuedSockets: this.queuedSockets.length,
57
59
  rejectedSockets: this.rejectedSockets,
58
60
  failedTransfers: this.failedTransfers,
61
+ recentEvents: [...this.recentEvents],
59
62
  lastFailure: this.lastFailure,
60
63
  };
61
64
  }
@@ -302,6 +305,11 @@ export class RollingWorkerSupervisor {
302
305
  this.maybeDrainWorker(previous);
303
306
  }
304
307
  this.options.log?.(`[proxy-supervisor] activated generation=${generation} pid=${handle.pid} version=${expectedVersion}`);
308
+ this.recordEvent({
309
+ type: "activated",
310
+ generation,
311
+ version: expectedVersion,
312
+ });
305
313
  this.publishState();
306
314
  finish();
307
315
  });
@@ -410,6 +418,13 @@ export class RollingWorkerSupervisor {
410
418
  handleTransferFailure(worker, socket, error) {
411
419
  this.failedTransfers += 1;
412
420
  const detail = this.describeTransferError(error);
421
+ this.recordEvent({
422
+ type: "failed_transfer",
423
+ generation: worker.generation,
424
+ version: worker.version,
425
+ phase: "transfer",
426
+ reason: detail,
427
+ });
413
428
  const lifecycle = this.extractLifecycleFailureDetails(error, worker.handle.pid);
414
429
  this.recordFailure(worker.generation, worker.version, "transfer", `worker ${worker.handle.pid} failed to accept a transferred socket: ${detail}`, {
415
430
  ...lifecycle.details,
@@ -432,10 +447,16 @@ export class RollingWorkerSupervisor {
432
447
  }
433
448
  this.publishState();
434
449
  }
435
- this.rejectSocket(socket);
450
+ this.rejectSocket(socket, worker.generation, worker.version, "transfer_failure");
436
451
  }
437
- rejectSocket(socket) {
452
+ rejectSocket(socket, generation = this.active?.generation ?? null, version = this.active?.version ?? null, reason = "unavailable") {
438
453
  this.rejectedSockets += 1;
454
+ this.recordEvent({
455
+ type: "rejected_socket",
456
+ generation,
457
+ version,
458
+ reason,
459
+ });
439
460
  socket.destroy();
440
461
  this.publishState();
441
462
  }
@@ -479,6 +500,23 @@ export class RollingWorkerSupervisor {
479
500
  message: message.slice(0, 1_000),
480
501
  ...details,
481
502
  };
503
+ this.recordEvent({
504
+ type: "failure",
505
+ generation,
506
+ version,
507
+ phase,
508
+ reason: message,
509
+ });
510
+ }
511
+ recordEvent(event) {
512
+ this.recentEvents.push({
513
+ at: new Date().toISOString(),
514
+ ...event,
515
+ ...(event.reason ? { reason: event.reason.slice(0, 1_000) } : {}),
516
+ });
517
+ if (this.recentEvents.length > MAX_RECENT_SUPERVISOR_EVENTS) {
518
+ this.recentEvents.splice(0, this.recentEvents.length - MAX_RECENT_SUPERVISOR_EVENTS);
519
+ }
482
520
  }
483
521
  publishState() {
484
522
  try {
@@ -310,6 +310,7 @@ declare function shouldAttemptClaudeFallback(loopState: AnthropicLoopState): boo
310
310
  export declare function createClaudeProxyRoutes(modelRouter?: ModelRouterInterface, basePath?: string, accountStrategy?: "round-robin" | "fill-first", passthroughMode?: boolean, primaryAccountKey?: string, accountAllowlistOrRuntimeOptions?: AccountAllowlist | ClaudeProxyRouteRuntimeOptions): RouteGroup;
311
311
  declare function reconcileEligibleAccountRuntimeState(account: ProxyPassthroughAccount): void;
312
312
  export declare function getTransientSameAccountRetryDelayMs(retryNumber: number): number;
313
+ export declare function getOverloadRotationDelayMs(attemptNumber: number): number;
313
314
  declare function describeTransportError(error: unknown): string;
314
315
  /**
315
316
  * Determine whether a POST can be retried without risking duplicate provider
@@ -400,6 +401,7 @@ export declare const __testHooks: {
400
401
  describeTransportError: typeof describeTransportError;
401
402
  redactProviderErrorMessage: typeof redactProviderErrorMessage;
402
403
  isUpstreamOverload: typeof isUpstreamOverload;
404
+ getOverloadRotationDelayMs: typeof getOverloadRotationDelayMs;
403
405
  shouldAttemptClaudeFallback: typeof shouldAttemptClaudeFallback;
404
406
  executeClaudeFallbackWithRetry: typeof executeClaudeFallbackWithRetry;
405
407
  buildClaudeAnthropicFailureResponse: typeof buildClaudeAnthropicFailureResponse;
@@ -63,6 +63,7 @@ let lastKnownAccountCount = 0;
63
63
  const MAX_AUTH_RETRIES = 5;
64
64
  const MAX_TRANSIENT_SAME_ACCOUNT_RETRIES = 2;
65
65
  const TRANSIENT_SAME_ACCOUNT_RETRY_DELAYS_MS = [250, 1_000];
66
+ const OVERLOAD_ACCOUNT_ROTATION_DELAYS_MS = [250, 500, 1_000, 2_000];
66
67
  const MAX_FALLBACK_NETWORK_RETRIES = 1;
67
68
  const FALLBACK_STREAM_IDLE_TIMEOUT_MS = 2 * 60 * 1000;
68
69
  /** Maximum upstream 429 attempts per account before rotating — for a TRANSIENT
@@ -2883,7 +2884,13 @@ async function handleAnthropicStreamingSuccessResponse(args) {
2883
2884
  });
2884
2885
  return {
2885
2886
  retryNextAccount: true,
2886
- failure: { message: preflight.message, rateLimit: isRateLimit },
2887
+ failure: {
2888
+ message: preflight.message,
2889
+ rateLimit: isRateLimit,
2890
+ ...(preflight.errorType === "overloaded_error"
2891
+ ? { retryDelayMs: getOverloadRotationDelayMs(attemptNumber) }
2892
+ : {}),
2893
+ },
2887
2894
  };
2888
2895
  }
2889
2896
  logAttempt(response.status, undefined, undefined, {
@@ -3484,6 +3491,9 @@ async function handleAnthropicAuthRetry(args) {
3484
3491
  const failure = successResult.failure;
3485
3492
  return {
3486
3493
  continueLoop: true,
3494
+ ...(failure?.retryDelayMs
3495
+ ? { retryDelayMs: failure.retryDelayMs }
3496
+ : {}),
3487
3497
  lastError: failure?.message ?? currentLastError,
3488
3498
  authFailureMessage: currentAuthFailureMessage,
3489
3499
  sawRateLimit: currentSawRateLimit || Boolean(failure?.rateLimit),
@@ -3941,6 +3951,9 @@ async function handleAnthropicNonOkResponse(args) {
3941
3951
  return {
3942
3952
  continueLoop: true,
3943
3953
  retrySameAccount: !upstreamOverload,
3954
+ ...(upstreamOverload
3955
+ ? { retryDelayMs: getOverloadRotationDelayMs(attemptNumber) }
3956
+ : {}),
3944
3957
  lastError: currentLastError,
3945
3958
  authFailureMessage: currentAuthFailureMessage,
3946
3959
  sawTransientFailure: currentSawTransientFailure,
@@ -4553,7 +4566,8 @@ async function handleAnthropicRoutedClaudeRequest(args) {
4553
4566
  effectiveAccounts.every((account) => !isAccountAdmissionAvailable(account.key, accountAdmissionCapacity))) {
4554
4567
  queuedAccountAdmission = await acquireFirstAvailableAccountAdmission(effectiveAccounts.map((account) => account.key), accountAdmissionCapacity, ctx.abortSignal);
4555
4568
  }
4556
- accountLoop: for (const account of effectiveAccounts) {
4569
+ accountLoop: for (const [accountIndex, account,] of effectiveAccounts.entries()) {
4570
+ const hasNextAccount = accountIndex < effectiveAccounts.length - 1;
4557
4571
  const accountState = getOrCreateRuntimeState(account.key);
4558
4572
  let transientSameAccountRetries = 0;
4559
4573
  let rateLimitSameAccountRetries = 0;
@@ -4763,6 +4777,10 @@ async function handleAnthropicRoutedClaudeRequest(args) {
4763
4777
  return authRetryResult.response;
4764
4778
  }
4765
4779
  if (authRetryResult.continueLoop) {
4780
+ if (hasNextAccount && authRetryResult.retryDelayMs) {
4781
+ logger.always(`[proxy] pacing cross-account SSE overload rotation for ${authRetryResult.retryDelayMs}ms after auth refresh`);
4782
+ await sleep(authRetryResult.retryDelayMs);
4783
+ }
4766
4784
  continue accountLoop;
4767
4785
  }
4768
4786
  }
@@ -4804,6 +4822,10 @@ async function handleAnthropicRoutedClaudeRequest(args) {
4804
4822
  if (nonOkResult.retrySameAccount) {
4805
4823
  logger.always(`[proxy] exhausted transient same-account retries for account=${account.label}; rotating`);
4806
4824
  }
4825
+ if (hasNextAccount && nonOkResult.retryDelayMs) {
4826
+ logger.always(`[proxy] pacing cross-account overload rotation for ${nonOkResult.retryDelayMs}ms`);
4827
+ await sleep(nonOkResult.retryDelayMs);
4828
+ }
4807
4829
  continue accountLoop;
4808
4830
  }
4809
4831
  break accountLoop;
@@ -4845,6 +4867,10 @@ async function handleAnthropicRoutedClaudeRequest(args) {
4845
4867
  loopState.lastError = successResult.failure.message;
4846
4868
  loopState.sawRateLimit ||= successResult.failure.rateLimit;
4847
4869
  loopState.sawTransientFailure ||= !successResult.failure.rateLimit;
4870
+ if (hasNextAccount && successResult.failure.retryDelayMs) {
4871
+ logger.always(`[proxy] pacing cross-account SSE overload rotation for ${successResult.failure.retryDelayMs}ms`);
4872
+ await sleep(successResult.failure.retryDelayMs);
4873
+ }
4848
4874
  }
4849
4875
  continue accountLoop;
4850
4876
  }
@@ -5315,6 +5341,10 @@ export function getTransientSameAccountRetryDelayMs(retryNumber) {
5315
5341
  const index = Math.min(Math.max(retryNumber - 1, 0), TRANSIENT_SAME_ACCOUNT_RETRY_DELAYS_MS.length - 1);
5316
5342
  return TRANSIENT_SAME_ACCOUNT_RETRY_DELAYS_MS[index] ?? 0;
5317
5343
  }
5344
+ export function getOverloadRotationDelayMs(attemptNumber) {
5345
+ const index = Math.min(Math.max(attemptNumber - 1, 0), OVERLOAD_ACCOUNT_ROTATION_DELAYS_MS.length - 1);
5346
+ return jitteredDelay(OVERLOAD_ACCOUNT_ROTATION_DELAYS_MS[index] ?? 250);
5347
+ }
5318
5348
  async function sleep(ms) {
5319
5349
  await new Promise((resolve) => setTimeout(resolve, ms));
5320
5350
  }
@@ -5586,6 +5616,7 @@ export const __testHooks = {
5586
5616
  describeTransportError,
5587
5617
  redactProviderErrorMessage,
5588
5618
  isUpstreamOverload,
5619
+ getOverloadRotationDelayMs,
5589
5620
  shouldAttemptClaudeFallback,
5590
5621
  executeClaudeFallbackWithRetry,
5591
5622
  buildClaudeAnthropicFailureResponse,
@@ -646,6 +646,7 @@ export type AnthropicSuccessResult = {
646
646
  failure?: {
647
647
  message: string;
648
648
  rateLimit: boolean;
649
+ retryDelayMs?: number;
649
650
  };
650
651
  } | {
651
652
  response: Response | unknown;
@@ -702,6 +703,8 @@ export type AnthropicAuthRetryResult = {
702
703
  response?: Response | unknown;
703
704
  holdsAccountAdmission?: boolean;
704
705
  continueLoop: boolean;
706
+ /** Failure-path pacing before rotating after provider-wide overload. */
707
+ retryDelayMs?: number;
705
708
  lastError: unknown;
706
709
  authFailureMessage: string | null;
707
710
  sawRateLimit: boolean;
@@ -713,6 +716,8 @@ export type AnthropicNonOkResult = {
713
716
  response?: Response | unknown;
714
717
  continueLoop: boolean;
715
718
  retrySameAccount?: boolean;
719
+ /** Failure-path pacing before rotating after provider-wide overload. */
720
+ retryDelayMs?: number;
716
721
  lastError: unknown;
717
722
  authFailureMessage: string | null;
718
723
  sawTransientFailure: boolean;
@@ -1457,6 +1462,9 @@ export type ProxyAnalysisReport = {
1457
1462
  attemptLatency: boolean;
1458
1463
  cacheUsage: boolean;
1459
1464
  routingDecisions: boolean;
1465
+ /** True only when every stream needed for cross-stream request/attempt
1466
+ * reconciliation begins at or before the requested analysis window. */
1467
+ comparableRequestAttempts: boolean;
1460
1468
  };
1461
1469
  dataQuality: {
1462
1470
  linesRead: number;
@@ -1468,6 +1476,8 @@ export type ProxyAnalysisReport = {
1468
1476
  observedFrom: string | null;
1469
1477
  observedTo: string | null;
1470
1478
  startsAtOrBeforeRequestedWindow: boolean;
1479
+ /** Whether this stream can support claims covering the full window. */
1480
+ completeWindow: boolean;
1471
1481
  }>;
1472
1482
  bodyArtifacts: {
1473
1483
  capturesIndexed: number;
@@ -2073,6 +2083,14 @@ export type RollingWorkerFailureDetails = {
2073
2083
  workerExitSignal?: string | null;
2074
2084
  supervisorAction?: "none" | "sigkill_after_transfer_failure";
2075
2085
  };
2086
+ export type RollingWorkerSupervisorEvent = {
2087
+ at: string;
2088
+ type: "activated" | "failure" | "failed_transfer" | "rejected_socket";
2089
+ generation: number | null;
2090
+ version: string | null;
2091
+ phase?: "startup" | "activation" | "runtime" | "transfer";
2092
+ reason?: string;
2093
+ };
2076
2094
  export type RollingWorkerSupervisorSnapshot = {
2077
2095
  generation: number;
2078
2096
  active: {
@@ -2093,6 +2111,8 @@ export type RollingWorkerSupervisorSnapshot = {
2093
2111
  queuedSockets: number;
2094
2112
  rejectedSockets: number;
2095
2113
  failedTransfers: number;
2114
+ /** Bounded generation-scoped evidence for attributing lifetime counters. */
2115
+ recentEvents: RollingWorkerSupervisorEvent[];
2096
2116
  lastFailure: ({
2097
2117
  at: string;
2098
2118
  generation: number;
@@ -2305,7 +2325,18 @@ export type ProxyNeurolinkRuntime = {
2305
2325
  neurolink: {
2306
2326
  getToolRegistry(): MCPToolRegistry;
2307
2327
  };
2308
- cleanupLogs: (daysToKeep?: number, maxFiles?: number) => void;
2328
+ logsDir: string;
2329
+ };
2330
+ /** Data passed to the isolated proxy log-retention worker. */
2331
+ export type ProxyLogCleanupWorkerData = {
2332
+ logsDir: string;
2333
+ maxAgeDays: number;
2334
+ maxSizeMb: number;
2335
+ };
2336
+ /** Lifecycle handle for non-blocking proxy log retention. */
2337
+ export type ProxyLogCleanupScheduler = {
2338
+ trigger: () => boolean;
2339
+ stop: () => Promise<void>;
2309
2340
  };
2310
2341
  /** Hono app + readiness state created by the proxy start command. */
2311
2342
  export type ProxyStartApp = {