@juspay/neurolink 10.8.22 → 10.9.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -45,6 +45,8 @@ const PROXY_TELEMETRY_SCRIPT_PATH = fileURLToPath(new URL("../../../scripts/obse
45
45
  const PROXY_LIFECYCLE_SHUTDOWN_TIMEOUT_MS = 5_000;
46
46
  const LEGACY_STATUS_ACCOUNT_CACHE_TTL_MS = 5_000;
47
47
  const PROXY_STATUS_TOKEN_READ_TIMEOUT_MS = 2_000;
48
+ const PROXY_STATUS_RECONCILE_TIMEOUT_MS = 750;
49
+ const PROXY_STATUS_ACCOUNT_INVENTORY_TIMEOUT_MS = 750;
48
50
  let legacyStatusAccountCache;
49
51
  // Allowed drift between a pid's OS-reported start time and the persisted
50
52
  // ProxySupervisorState.startTime before processLooksLikeProxySupervisor
@@ -1592,9 +1594,20 @@ export async function createProxyStartApp(params) {
1592
1594
  const activeAccountAllowlist = runtimeConfig
1593
1595
  ? runtimeConfig.accountAllowlist
1594
1596
  : params.accountAllowlist;
1595
- const { getReconciledUsageSnapshot, getUsageStatsPersistenceStatus } = await import("../../lib/proxy/usageStats.js");
1597
+ const { getReconciledUsageSnapshot, getUsageSnapshot, getUsageStatsPersistenceStatus, } = await import("../../lib/proxy/usageStats.js");
1596
1598
  const { loadAccountCooldowns } = await import("../../lib/proxy/accountCooldown.js");
1597
- const usageSnapshot = await getReconciledUsageSnapshot();
1599
+ let usageSnapshot = getUsageSnapshot();
1600
+ let snapshotSource = "memory";
1601
+ try {
1602
+ usageSnapshot = await withTimeout(getReconciledUsageSnapshot(), PROXY_STATUS_RECONCILE_TIMEOUT_MS, "[proxy] /status usage reconciliation timed out");
1603
+ snapshotSource = "reconciled";
1604
+ }
1605
+ catch (error) {
1606
+ // Status must not become an outage amplifier when a cross-process lock or
1607
+ // a slow filesystem stalls reconciliation. The process-local snapshot is
1608
+ // coherent and its source is explicit to callers.
1609
+ logger.debug(`[proxy] /status using memory stats snapshot: ${error instanceof Error ? error.message : String(error)}`);
1610
+ }
1598
1611
  const { stats, terminalErrors } = usageSnapshot;
1599
1612
  const terminalErrorDetailsComparable = usageSnapshot.statsVersion === usageSnapshot.terminalErrorsVersion;
1600
1613
  const lastTerminalError = terminalErrors.recent.at(-1) ?? null;
@@ -1608,40 +1621,52 @@ export async function createProxyStartApp(params) {
1608
1621
  const supervisorState = loadProxySupervisorState();
1609
1622
  const rollingSupervisorRunning = isRollingHandoffCapable(supervisorState);
1610
1623
  const updateState = loadUpdateState();
1611
- const cooldowns = await loadAccountCooldowns();
1624
+ const cooldowns = await withTimeout(loadAccountCooldowns(), PROXY_STATUS_ACCOUNT_INVENTORY_TIMEOUT_MS, "[proxy] /status cooldown inspection timed out").catch((error) => {
1625
+ logger.debug(`[proxy] /status using empty cooldown snapshot: ${error instanceof Error ? error.message : String(error)}`);
1626
+ return {};
1627
+ });
1612
1628
  const storedAccountKeys = new Set();
1613
1629
  const storedAccountExpirations = new Map();
1614
1630
  const disabledAccountKeys = new Set();
1615
1631
  let accountInventoryLoaded = false;
1616
1632
  try {
1617
1633
  const { tokenStore } = await import("../../lib/auth/tokenStore.js");
1618
- const storedKeys = await tokenStore.listByPrefix("anthropic:");
1634
+ const storedKeys = await withTimeout(tokenStore.listByPrefix("anthropic:"), PROXY_STATUS_ACCOUNT_INVENTORY_TIMEOUT_MS, "[proxy] /status account enumeration timed out");
1619
1635
  for (const key of storedKeys) {
1620
- const normalizedKey = normalizeAnthropicAccountKey(key);
1621
- storedAccountKeys.add(normalizedKey);
1636
+ storedAccountKeys.add(normalizeAnthropicAccountKey(key));
1622
1637
  }
1623
- await Promise.all(storedKeys.map(async (key) => {
1624
- const normalizedKey = normalizeAnthropicAccountKey(key);
1625
- try {
1626
- const tokens = await withTimeout(tokenStore.peekTokens(key), PROXY_STATUS_TOKEN_READ_TIMEOUT_MS, "[proxy] /status token inspection timed out");
1627
- if (tokens) {
1628
- storedAccountExpirations.set(normalizedKey, tokens.expiresAt);
1638
+ // Once account names are known, preserve them even when optional token
1639
+ // metadata is slow. That keeps the status table useful and avoids
1640
+ // incorrectly presenting known accounts as removed.
1641
+ accountInventoryLoaded = true;
1642
+ const inventory = await withTimeout((async () => {
1643
+ const tokenExpirations = await Promise.all(storedKeys.map(async (key) => {
1644
+ try {
1645
+ const tokens = await withTimeout(tokenStore.peekTokens(key), PROXY_STATUS_TOKEN_READ_TIMEOUT_MS, "[proxy] /status token inspection timed out");
1646
+ return tokens ? [key, tokens.expiresAt] : undefined;
1629
1647
  }
1648
+ catch (error) {
1649
+ logger.debug(`[proxy] /status: failed to inspect token metadata for ${normalizeAnthropicAccountKey(key)}: ${error instanceof Error ? error.message : String(error)}`);
1650
+ return undefined;
1651
+ }
1652
+ }));
1653
+ const disabledKeys = await tokenStore.listDisabled();
1654
+ return { tokenExpirations, disabledKeys };
1655
+ })(), PROXY_STATUS_ACCOUNT_INVENTORY_TIMEOUT_MS, "[proxy] /status account metadata timed out");
1656
+ for (const expiration of inventory.tokenExpirations) {
1657
+ if (expiration) {
1658
+ storedAccountExpirations.set(normalizeAnthropicAccountKey(expiration[0]), expiration[1]);
1630
1659
  }
1631
- catch (err) {
1632
- logger.debug(`[proxy] /status: failed to inspect token metadata for ${normalizedKey}: ${err instanceof Error ? err.message : String(err)}`);
1633
- }
1634
- }));
1635
- for (const key of await tokenStore.listDisabled()) {
1660
+ }
1661
+ for (const key of inventory.disabledKeys) {
1636
1662
  disabledAccountKeys.add(normalizeAnthropicAccountKey(key));
1637
1663
  }
1638
- accountInventoryLoaded = true;
1639
1664
  }
1640
1665
  catch (err) {
1641
1666
  logger.debug(`[proxy] /status: failed to resolve account cooldown labels: ${err instanceof Error ? err.message : String(err)}`);
1642
1667
  }
1643
1668
  const legacyAccountLabel = accountInventoryLoaded
1644
- ? await resolveLegacyStatusAccountLabel(storedAccountKeys.size)
1669
+ ? await withTimeout(resolveLegacyStatusAccountLabel(storedAccountKeys.size), PROXY_STATUS_ACCOUNT_INVENTORY_TIMEOUT_MS, "[proxy] /status legacy account inspection timed out").catch(() => null)
1645
1670
  : null;
1646
1671
  const now = Date.now();
1647
1672
  const health = buildProxyHealthResponse(readiness, {
@@ -1649,7 +1674,12 @@ export async function createProxyStartApp(params) {
1649
1674
  passthrough: activePassthrough,
1650
1675
  version: PROXY_VERSION,
1651
1676
  });
1652
- const primaryAccount = await resolveStatusPrimaryAccount(activeProxyConfig);
1677
+ const primaryAccount = await withTimeout(resolveStatusPrimaryAccount(activeProxyConfig), PROXY_STATUS_ACCOUNT_INVENTORY_TIMEOUT_MS, "[proxy] /status primary account inspection timed out").catch(() => ({
1678
+ configured: activeProxyConfig?.routing?.primaryAccount?.trim() || null,
1679
+ key: null,
1680
+ label: null,
1681
+ source: "fallback",
1682
+ }));
1653
1683
  const activeUpdaterPid = supervisorState?.updaterPid ?? runtimeState?.updaterPid;
1654
1684
  const accountRows = Object.values(stats.accounts).map((account) => {
1655
1685
  const normalizedKey = normalizeAnthropicAccountKey(account.label);
@@ -1813,6 +1843,7 @@ export async function createProxyStartApp(params) {
1813
1843
  terminalErrorDetailsComparable,
1814
1844
  terminalErrorDetailsMissing,
1815
1845
  terminalErrorDetailsExcess,
1846
+ snapshotSource,
1816
1847
  accounts: accountRows,
1817
1848
  primaryAccount,
1818
1849
  persistence: getUsageStatsPersistenceStatus(),
@@ -1,4 +1,69 @@
1
1
  import type { AllToolsMap, BasicToolsMap, FilesystemToolsMap, Tool, UtilityToolsMap } from "../types/index.js";
2
+ /**
3
+ * Vertex location for the websearchGrounding tool:
4
+ * NEUROLINK_WEBSEARCH_LOCATION, else `global`.
5
+ *
6
+ * Deliberately does NOT inherit GOOGLE_VERTEX_LOCATION. The search models are
7
+ * served only from the `global`, `us` and `eu` endpoints — never from a single
8
+ * region — so inheriting a regional value produces a 404 for the model rather
9
+ * than a working search. That is not hypothetical: a deployment pinned to
10
+ * us-east5 for Claude-on-Vertex sent web search to us-east5 too and every
11
+ * query failed with NOT_FOUND.
12
+ *
13
+ * Set NEUROLINK_WEBSEARCH_LOCATION to `us` or `eu` where the multi-region
14
+ * endpoints are required, e.g. for data residency.
15
+ */
16
+ export declare function resolveWebsearchLocation(): string;
17
+ /** Model for websearchGrounding — NEUROLINK_WEBSEARCH_MODEL, else the default. */
18
+ export declare function resolveWebsearchModel(): string;
19
+ /**
20
+ * The generateContent request for a web search. Google Search grounding is
21
+ * requested natively through `config.tools`.
22
+ */
23
+ export declare function buildWebsearchRequest(model: string, query: string, maxWords: number): {
24
+ model: string;
25
+ contents: {
26
+ role: string;
27
+ parts: {
28
+ text: string;
29
+ }[];
30
+ }[];
31
+ config: {
32
+ tools: {
33
+ googleSearch: {};
34
+ }[];
35
+ };
36
+ };
37
+ /**
38
+ * Display domain for one grounding chunk.
39
+ *
40
+ * Blank `domain` values count as absent, and the uri fallback is guarded by
41
+ * `URL.canParse` — an unparseable provider uri would otherwise throw and take
42
+ * the entire search down with it.
43
+ */
44
+ export declare function resolveGroundingDomain(web: {
45
+ uri?: string;
46
+ domain?: string;
47
+ }): string;
48
+ /**
49
+ * Map grounding chunks onto search results, capped at `limitedResults`.
50
+ * Returns an empty array when there is no usable grounding metadata, which the
51
+ * caller replaces with a single synthesised result.
52
+ */
53
+ export declare function buildWebsearchResults(groundingMetadata: {
54
+ groundingChunks?: {
55
+ web?: {
56
+ uri?: string;
57
+ title?: string;
58
+ domain?: string;
59
+ };
60
+ }[];
61
+ } | undefined, searchContent: string, limitedResults: number): {
62
+ title: string;
63
+ url: string;
64
+ snippet: string;
65
+ domain: string;
66
+ }[];
2
67
  /**
3
68
  * Direct tool definitions that work immediately with Gemini/AI SDK
4
69
  * These bypass MCP complexity and provide reliable agent functionality
@@ -6,6 +6,8 @@ import { logger } from "../utils/logger.js";
6
6
  import { CSVProcessor } from "../utils/csvProcessor.js";
7
7
  import { shouldEnableBashTool } from "../utils/toolUtils.js";
8
8
  import { tool } from "../utils/tool.js";
9
+ import { withTimeout } from "../utils/async/withTimeout.js";
10
+ import { TIMEOUTS } from "../constants/timeouts.js";
9
11
  const MAX_OUTPUT_BYTES = 102400; // 100KB
10
12
  function truncateOutput(output) {
11
13
  if (output.length > MAX_OUTPUT_BYTES) {
@@ -35,16 +37,77 @@ function resolveWithinCwd(filePath) {
35
37
  }
36
38
  return { path: resolvedPath };
37
39
  }
38
- // Runtime Google Search tool creation - bypasses TypeScript strict typing
39
- function createGoogleSearchTools() {
40
- const searchTool = {};
41
- // Dynamically assign google_search property at runtime
42
- Object.defineProperty(searchTool, "google_search", {
43
- value: {},
44
- enumerable: true,
45
- configurable: true,
46
- });
47
- return [searchTool];
40
+ /**
41
+ * Vertex location for the websearchGrounding tool:
42
+ * NEUROLINK_WEBSEARCH_LOCATION, else `global`.
43
+ *
44
+ * Deliberately does NOT inherit GOOGLE_VERTEX_LOCATION. The search models are
45
+ * served only from the `global`, `us` and `eu` endpoints — never from a single
46
+ * region — so inheriting a regional value produces a 404 for the model rather
47
+ * than a working search. That is not hypothetical: a deployment pinned to
48
+ * us-east5 for Claude-on-Vertex sent web search to us-east5 too and every
49
+ * query failed with NOT_FOUND.
50
+ *
51
+ * Set NEUROLINK_WEBSEARCH_LOCATION to `us` or `eu` where the multi-region
52
+ * endpoints are required, e.g. for data residency.
53
+ */
54
+ export function resolveWebsearchLocation() {
55
+ return process.env.NEUROLINK_WEBSEARCH_LOCATION?.trim() || "global";
56
+ }
57
+ /** Model for websearchGrounding — NEUROLINK_WEBSEARCH_MODEL, else the default. */
58
+ export function resolveWebsearchModel() {
59
+ return (process.env.NEUROLINK_WEBSEARCH_MODEL?.trim() || "gemini-3.1-flash-lite");
60
+ }
61
+ /**
62
+ * The generateContent request for a web search. Google Search grounding is
63
+ * requested natively through `config.tools`.
64
+ */
65
+ export function buildWebsearchRequest(model, query, maxWords) {
66
+ return {
67
+ model,
68
+ contents: [
69
+ {
70
+ role: "user",
71
+ parts: [
72
+ {
73
+ text: `Search for: "${query}". Provide a concise summary in no more than ${maxWords} words.`,
74
+ },
75
+ ],
76
+ },
77
+ ],
78
+ config: { tools: [{ googleSearch: {} }] },
79
+ };
80
+ }
81
+ /**
82
+ * Display domain for one grounding chunk.
83
+ *
84
+ * Blank `domain` values count as absent, and the uri fallback is guarded by
85
+ * `URL.canParse` — an unparseable provider uri would otherwise throw and take
86
+ * the entire search down with it.
87
+ */
88
+ export function resolveGroundingDomain(web) {
89
+ return (web.domain?.trim() ||
90
+ (web.uri && URL.canParse(web.uri) ? new URL(web.uri).hostname : "unknown"));
91
+ }
92
+ /**
93
+ * Map grounding chunks onto search results, capped at `limitedResults`.
94
+ * Returns an empty array when there is no usable grounding metadata, which the
95
+ * caller replaces with a single synthesised result.
96
+ */
97
+ export function buildWebsearchResults(groundingMetadata, searchContent, limitedResults) {
98
+ const searchResults = [];
99
+ for (const chunk of groundingMetadata?.groundingChunks?.slice(0, limitedResults) ?? []) {
100
+ if (chunk.web) {
101
+ searchResults.push({
102
+ title: chunk.web.title || "No title",
103
+ url: chunk.web.uri || "",
104
+ // Full content — maxWords already bounds the length.
105
+ snippet: searchContent,
106
+ domain: resolveGroundingDomain(chunk.web),
107
+ });
108
+ }
109
+ }
110
+ return searchResults;
48
111
  }
49
112
  /**
50
113
  * Direct tool definitions that work immediately with Gemini/AI SDK
@@ -593,7 +656,7 @@ export const directAgentTools = {
593
656
  try {
594
657
  const hasCredentials = process.env.GOOGLE_APPLICATION_CREDENTIALS;
595
658
  const hasProjectId = process.env.GOOGLE_VERTEX_PROJECT;
596
- const projectLocation = process.env.GOOGLE_VERTEX_LOCATION || "us-central1";
659
+ const projectLocation = resolveWebsearchLocation();
597
660
  if (!hasCredentials || !hasProjectId) {
598
661
  return {
599
662
  success: false,
@@ -605,31 +668,19 @@ export const directAgentTools = {
605
668
  };
606
669
  }
607
670
  const limitedResults = Math.min(Math.max(maxResults, 1), 5);
608
- const { VertexAI } = await import("@google-cloud/vertexai");
609
- const vertex_ai = new VertexAI({
671
+ const { GoogleGenAI } = await import("@google/genai");
672
+ const vertex_ai = new GoogleGenAI({
673
+ vertexai: true,
610
674
  project: hasProjectId,
611
675
  location: projectLocation,
612
676
  });
613
- const websearchModel = process.env.NEUROLINK_WEBSEARCH_MODEL?.trim() ||
614
- "gemini-2.5-flash-lite";
615
- const model = vertex_ai.getGenerativeModel({
616
- model: websearchModel,
617
- tools: createGoogleSearchTools(),
618
- });
619
- // Search query with word limit constraint
620
- const searchPrompt = `Search for: "${query}". Provide a concise summary in no more than ${maxWords} words.`;
677
+ const websearchModel = resolveWebsearchModel();
621
678
  const startTime = Date.now();
622
- const response = await model.generateContent({
623
- contents: [
624
- {
625
- role: "user",
626
- parts: [{ text: searchPrompt }],
627
- },
628
- ],
629
- });
679
+ // A stalled Vertex call would otherwise leave the tool pending
680
+ // indefinitely; the outer catch turns a timeout into the failure shape.
681
+ const result = await withTimeout(vertex_ai.models.generateContent(buildWebsearchRequest(websearchModel, query, maxWords)), TIMEOUTS.TOOL.EXECUTION_DEFAULT_MS, "[websearchGrounding] Vertex AI web search request timed out");
630
682
  const responseTime = Date.now() - startTime;
631
683
  // Extract grounding metadata and search results
632
- const result = response.response;
633
684
  const candidates = result.candidates;
634
685
  if (!candidates || candidates.length === 0) {
635
686
  return {
@@ -638,33 +689,18 @@ export const directAgentTools = {
638
689
  query,
639
690
  };
640
691
  }
641
- const content = candidates[0].content;
642
- if (!content || !content.parts || content.parts.length === 0) {
692
+ // Extract raw search content. The aggregated `text` getter joins every
693
+ // text part, so it survives responses that lead with a non-text part.
694
+ const searchContent = result.text?.trim() || "";
695
+ if (!searchContent) {
643
696
  return {
644
697
  success: false,
645
698
  error: "No search content found",
646
699
  query,
647
700
  };
648
701
  }
649
- // Extract raw search content
650
- const searchContent = content.parts[0].text || "";
651
702
  // Extract grounding sources if available
652
- const groundingMetadata = candidates[0]?.groundingMetadata;
653
- const searchResults = [];
654
- if (groundingMetadata?.groundingChunks) {
655
- for (const chunk of groundingMetadata.groundingChunks.slice(0, limitedResults)) {
656
- if (chunk.web) {
657
- searchResults.push({
658
- title: chunk.web.title || "No title",
659
- url: chunk.web.uri || "",
660
- snippet: searchContent, // Use full content since maxWords already limits length
661
- domain: chunk.web.uri
662
- ? new URL(chunk.web.uri).hostname
663
- : "unknown",
664
- });
665
- }
666
- }
667
- }
703
+ const searchResults = buildWebsearchResults(candidates[0]?.groundingMetadata, searchContent, limitedResults);
668
704
  // If no grounding metadata, create basic result structure
669
705
  if (searchResults.length === 0) {
670
706
  searchResults.push({
@@ -1,3 +1,4 @@
1
+ import { appendFile } from "node:fs/promises";
1
2
  import type { ProxyLifecycleEventInput, ProxyLifecycleLoggerOptions, ProxyLifecycleLoggerSnapshot } from "../types/index.js";
2
3
  export declare function hashProxyLifecycleSessionId(sessionId: string | undefined): string | undefined;
3
4
  export declare function configureProxyLifecycleLogger(options: ProxyLifecycleLoggerOptions): void;
@@ -6,3 +7,7 @@ export declare function logProxyLifecycleEvent(input: ProxyLifecycleEventInput):
6
7
  export declare function flushProxyLifecycleEvents(): Promise<void>;
7
8
  export declare function getProxyLifecycleLoggerSnapshot(): ProxyLifecycleLoggerSnapshot;
8
9
  export declare function resetProxyLifecycleLoggerForTests(): void;
10
+ /** Isolated failure injection for lifecycle durability tests. */
11
+ export declare const __proxyLifecycleTestHooks: {
12
+ setAppendFileForTests(append: typeof appendFile): void;
13
+ };
@@ -9,6 +9,8 @@ const SCHEMA_VERSION = 1;
9
9
  const DEFAULT_QUEUE_CAPACITY = 10_000;
10
10
  const DEFAULT_BATCH_SIZE = 256;
11
11
  const DEFAULT_FLUSH_INTERVAL_MS = 25;
12
+ const DEFAULT_MAX_WRITE_RETRIES = 3;
13
+ const MAX_WRITE_RETRY_DELAY_MS = 1_000;
12
14
  const LIFECYCLE_APPEND_TIMEOUT_MS = 2_000;
13
15
  const MAX_SHORT_FIELD_LENGTH = 256;
14
16
  const SESSION_KEY_FILE = ".proxy-lifecycle-session-key";
@@ -17,6 +19,7 @@ let lifecycleLogDir;
17
19
  let queueCapacity = DEFAULT_QUEUE_CAPACITY;
18
20
  let batchSize = DEFAULT_BATCH_SIZE;
19
21
  let flushIntervalMs = DEFAULT_FLUSH_INTERVAL_MS;
22
+ let maxWriteRetries = DEFAULT_MAX_WRITE_RETRIES;
20
23
  let processInstanceId = randomUUID();
21
24
  let sessionHashKey = randomBytes(32);
22
25
  let nextSequence = 1;
@@ -28,10 +31,13 @@ let queueDrops = 0;
28
31
  let invalidDrops = 0;
29
32
  let writeDrops = 0;
30
33
  let writeFailures = 0;
34
+ let writeRetries = 0;
31
35
  let inFlight = 0;
32
36
  let queue = [];
33
37
  let flushTimer;
34
38
  let flushInFlight;
39
+ let nextFlushDelayMs;
40
+ let appendLifecycleFile = appendFile;
35
41
  function positiveInteger(value, fallback) {
36
42
  return Number.isInteger(value) && (value ?? 0) > 0
37
43
  ? value
@@ -123,14 +129,14 @@ function clearScheduledFlush() {
123
129
  flushTimer = undefined;
124
130
  }
125
131
  }
126
- function scheduleFlush() {
132
+ function scheduleFlush(delayMs = flushIntervalMs) {
127
133
  if (flushTimer || flushInFlight || queue.length === 0) {
128
134
  return;
129
135
  }
130
136
  flushTimer = setTimeout(() => {
131
137
  flushTimer = undefined;
132
138
  void startFlush();
133
- }, flushIntervalMs);
139
+ }, delayMs);
134
140
  flushTimer.unref?.();
135
141
  }
136
142
  async function flushBatch() {
@@ -143,27 +149,53 @@ async function flushBatch() {
143
149
  const byPath = new Map();
144
150
  for (const item of batch) {
145
151
  const path = join(item.logDir, `proxy-lifecycle-${item.date}.jsonl`);
146
- const lines = byPath.get(path) ?? [];
147
- lines.push(`${JSON.stringify(item.record)}\n`);
148
- byPath.set(path, lines);
152
+ const items = byPath.get(path) ?? [];
153
+ items.push(item);
154
+ byPath.set(path, items);
149
155
  }
150
- for (const [path, lines] of byPath) {
156
+ const retries = [];
157
+ let retryDelayMs = 0;
158
+ for (const [path, items] of byPath) {
159
+ const lines = items.map((item) => `${JSON.stringify(item.record)}\n`);
151
160
  try {
152
161
  // This best-effort telemetry sink intentionally avoids fsync so request
153
162
  // throughput is not coupled to storage latency. Loss is surfaced by
154
163
  // writeDrops/writeFailures rather than delaying proxy responses.
155
- await withTimeout(appendFile(path, lines.join(""), { mode: 0o600 }), LIFECYCLE_APPEND_TIMEOUT_MS, "Timed out writing proxy lifecycle metadata");
164
+ await withTimeout(appendLifecycleFile(path, lines.join(""), { mode: 0o600 }), LIFECYCLE_APPEND_TIMEOUT_MS, "Timed out writing proxy lifecycle metadata");
156
165
  written += lines.length;
157
166
  }
158
167
  catch (error) {
159
- dropped += lines.length;
160
- writeDrops += lines.length;
161
168
  writeFailures += 1;
169
+ const retryable = items.filter((item) => item.writeRetries < maxWriteRetries);
170
+ const exhausted = items.length - retryable.length;
171
+ if (retryable.length > 0) {
172
+ const nextRetries = retryable.map((item) => ({
173
+ ...item,
174
+ writeRetries: item.writeRetries + 1,
175
+ }));
176
+ retries.push(...nextRetries);
177
+ writeRetries += nextRetries.length;
178
+ retryDelayMs = Math.max(retryDelayMs, Math.min(MAX_WRITE_RETRY_DELAY_MS, flushIntervalMs *
179
+ 2 ** Math.max(...nextRetries.map((item) => item.writeRetries))));
180
+ }
181
+ if (exhausted > 0) {
182
+ dropped += exhausted;
183
+ writeDrops += exhausted;
184
+ }
162
185
  logger.warn("[proxy] lifecycle metadata write failed", {
186
+ path,
187
+ retrying: retryable.length,
188
+ dropped: exhausted,
163
189
  error: error instanceof Error ? error.message : String(error),
164
190
  });
165
191
  }
166
192
  }
193
+ if (retries.length > 0) {
194
+ // Keep retried records ahead of newly admitted records. This preserves
195
+ // per-file sequence order while continuing to keep request paths async.
196
+ queue.unshift(...retries);
197
+ nextFlushDelayMs = Math.max(nextFlushDelayMs ?? 0, retryDelayMs || flushIntervalMs);
198
+ }
167
199
  }
168
200
  finally {
169
201
  inFlight = Math.max(0, inFlight - batch.length);
@@ -179,21 +211,27 @@ function startFlush() {
179
211
  if (flushInFlight === currentFlush) {
180
212
  flushInFlight = undefined;
181
213
  }
182
- scheduleFlush();
214
+ const delayMs = nextFlushDelayMs;
215
+ nextFlushDelayMs = undefined;
216
+ scheduleFlush(delayMs);
183
217
  }, () => {
184
218
  if (flushInFlight === currentFlush) {
185
219
  flushInFlight = undefined;
186
220
  }
187
- scheduleFlush();
221
+ const delayMs = nextFlushDelayMs;
222
+ nextFlushDelayMs = undefined;
223
+ scheduleFlush(delayMs);
188
224
  });
189
225
  return currentFlush;
190
226
  }
191
227
  export function configureProxyLifecycleLogger(options) {
192
228
  clearScheduledFlush();
229
+ nextFlushDelayMs = undefined;
193
230
  loggerEnabled = false;
194
231
  lifecycleLogDir = undefined;
195
232
  queueCapacity = positiveInteger(options.queueCapacity, DEFAULT_QUEUE_CAPACITY);
196
233
  batchSize = positiveInteger(options.batchSize, DEFAULT_BATCH_SIZE);
234
+ maxWriteRetries = positiveInteger(options.maxWriteRetries, DEFAULT_MAX_WRITE_RETRIES);
197
235
  flushIntervalMs = positiveInteger(options.flushIntervalMs, DEFAULT_FLUSH_INTERVAL_MS);
198
236
  if (options.enabled && options.logDir) {
199
237
  try {
@@ -268,6 +306,7 @@ export function logProxyLifecycleEvent(input) {
268
306
  logDir: lifecycleLogDir,
269
307
  date: String(record.timestamp).slice(0, 10),
270
308
  record,
309
+ writeRetries: 0,
271
310
  });
272
311
  enqueued += 1;
273
312
  scheduleFlush();
@@ -303,6 +342,7 @@ export function getProxyLifecycleLoggerSnapshot() {
303
342
  invalidDrops,
304
343
  writeDrops,
305
344
  writeFailures,
345
+ writeRetries,
306
346
  pending: queue.length,
307
347
  inFlight,
308
348
  flushing: flushInFlight !== undefined,
@@ -315,6 +355,7 @@ export function resetProxyLifecycleLoggerForTests() {
315
355
  queueCapacity = DEFAULT_QUEUE_CAPACITY;
316
356
  batchSize = DEFAULT_BATCH_SIZE;
317
357
  flushIntervalMs = DEFAULT_FLUSH_INTERVAL_MS;
358
+ maxWriteRetries = DEFAULT_MAX_WRITE_RETRIES;
318
359
  processInstanceId = randomUUID();
319
360
  sessionHashKey = randomBytes(32);
320
361
  nextSequence = 1;
@@ -326,8 +367,17 @@ export function resetProxyLifecycleLoggerForTests() {
326
367
  invalidDrops = 0;
327
368
  writeDrops = 0;
328
369
  writeFailures = 0;
370
+ writeRetries = 0;
329
371
  inFlight = 0;
330
372
  queue = [];
331
373
  flushInFlight = undefined;
374
+ nextFlushDelayMs = undefined;
375
+ appendLifecycleFile = appendFile;
332
376
  }
377
+ /** Isolated failure injection for lifecycle durability tests. */
378
+ export const __proxyLifecycleTestHooks = {
379
+ setAppendFileForTests(append) {
380
+ appendLifecycleFile = append;
381
+ },
382
+ };
333
383
  //# sourceMappingURL=proxyLifecycle.js.map
@@ -77,6 +77,8 @@ export declare function recordFinalSuccess(accountLabel?: string, accountType?:
77
77
  export declare function recordAttemptError(accountLabel: string, accountType: string, status: number, rateLimitKind?: "transient" | "quota"): void;
78
78
  export declare function recordFinalError(status: number, accountLabel?: string, accountType?: string, details?: ProxyTerminalErrorDetails): void;
79
79
  export declare function getStats(): ProxyStats;
80
+ /** Return the process-local coherent snapshot without filesystem reconciliation. */
81
+ export declare function getUsageSnapshot(): ProxyUsageStatsSnapshot;
80
82
  export declare function getReconciledStats(): Promise<ProxyStats>;
81
83
  export declare function getReconciledUsageSnapshot(): Promise<ProxyUsageStatsSnapshot>;
82
84
  export declare function getAccountStats(label: string): AccountStats | undefined;
@@ -1121,6 +1121,10 @@ export function recordFinalError(status, accountLabel, accountType, details) {
1121
1121
  export function getStats() {
1122
1122
  return defaultStore.getStats();
1123
1123
  }
1124
+ /** Return the process-local coherent snapshot without filesystem reconciliation. */
1125
+ export function getUsageSnapshot() {
1126
+ return defaultStore.getUsageSnapshot();
1127
+ }
1124
1128
  export async function getReconciledStats() {
1125
1129
  return defaultStore.reconcile();
1126
1130
  }
@@ -1246,6 +1246,8 @@ export type ProxyLifecycleLoggerSnapshot = {
1246
1246
  invalidDrops: number;
1247
1247
  writeDrops: number;
1248
1248
  writeFailures: number;
1249
+ /** Events requeued after a transient lifecycle metadata write failure. */
1250
+ writeRetries: number;
1249
1251
  pending: number;
1250
1252
  inFlight: number;
1251
1253
  flushing: boolean;
@@ -1257,12 +1259,15 @@ export type ProxyLifecycleLoggerOptions = {
1257
1259
  queueCapacity?: number;
1258
1260
  batchSize?: number;
1259
1261
  flushIntervalMs?: number;
1262
+ /** Bounded retries for a metadata batch that cannot be appended immediately. */
1263
+ maxWriteRetries?: number;
1260
1264
  };
1261
1265
  /** Serialized lifecycle line awaiting a bounded batch write. */
1262
1266
  export type QueuedProxyLifecycleEvent = {
1263
1267
  logDir: string;
1264
1268
  date: string;
1265
1269
  record: Record<string, unknown>;
1270
+ writeRetries: number;
1266
1271
  };
1267
1272
  /** Percentile summary used by offline proxy log analysis. */
1268
1273
  export type ProxyLatencySummary = {
@@ -2167,6 +2172,8 @@ export type StatusStats = {
2167
2172
  terminalErrorDetailsComparable?: boolean;
2168
2173
  terminalErrorDetailsMissing?: number;
2169
2174
  terminalErrorDetailsExcess?: number;
2175
+ /** Whether this status response reconciled shared state or used local memory. */
2176
+ snapshotSource?: "reconciled" | "memory";
2170
2177
  accounts?: {
2171
2178
  label: string;
2172
2179
  type: string;
@@ -1,3 +1,4 @@
1
+ import { appendFile } from "node:fs/promises";
1
2
  import type { ProxyLifecycleEventInput, ProxyLifecycleLoggerOptions, ProxyLifecycleLoggerSnapshot } from "../types/index.js";
2
3
  export declare function hashProxyLifecycleSessionId(sessionId: string | undefined): string | undefined;
3
4
  export declare function configureProxyLifecycleLogger(options: ProxyLifecycleLoggerOptions): void;
@@ -6,3 +7,7 @@ export declare function logProxyLifecycleEvent(input: ProxyLifecycleEventInput):
6
7
  export declare function flushProxyLifecycleEvents(): Promise<void>;
7
8
  export declare function getProxyLifecycleLoggerSnapshot(): ProxyLifecycleLoggerSnapshot;
8
9
  export declare function resetProxyLifecycleLoggerForTests(): void;
10
+ /** Isolated failure injection for lifecycle durability tests. */
11
+ export declare const __proxyLifecycleTestHooks: {
12
+ setAppendFileForTests(append: typeof appendFile): void;
13
+ };