@juspay/neurolink 10.9.0 → 10.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/dist/cli/commands/proxy.js +51 -20
- package/dist/lib/proxy/proxyLifecycle.d.ts +5 -0
- package/dist/lib/proxy/proxyLifecycle.js +61 -11
- package/dist/lib/proxy/usageStats.d.ts +2 -0
- package/dist/lib/proxy/usageStats.js +4 -0
- package/dist/lib/types/proxy.d.ts +7 -0
- package/dist/proxy/proxyLifecycle.d.ts +5 -0
- package/dist/proxy/proxyLifecycle.js +61 -11
- package/dist/proxy/usageStats.d.ts +2 -0
- package/dist/proxy/usageStats.js +4 -0
- package/dist/types/proxy.d.ts +7 -0
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,9 @@
|
|
|
1
|
+
## [10.9.1](https://github.com/juspay/neurolink/compare/v10.9.0...v10.9.1) (2026-08-05)
|
|
2
|
+
|
|
3
|
+
### Bug Fixes
|
|
4
|
+
|
|
5
|
+
- **(proxy):** harden status and lifecycle telemetry ([03399a8](https://github.com/juspay/neurolink/commit/03399a8ac36861c96774fee8ae2465c2eed39026))
|
|
6
|
+
|
|
1
7
|
## [10.9.0](https://github.com/juspay/neurolink/compare/v10.8.22...v10.9.0) (2026-08-05)
|
|
2
8
|
|
|
3
9
|
### Features
|
|
@@ -45,6 +45,8 @@ const PROXY_TELEMETRY_SCRIPT_PATH = fileURLToPath(new URL("../../../scripts/obse
|
|
|
45
45
|
const PROXY_LIFECYCLE_SHUTDOWN_TIMEOUT_MS = 5_000;
|
|
46
46
|
const LEGACY_STATUS_ACCOUNT_CACHE_TTL_MS = 5_000;
|
|
47
47
|
const PROXY_STATUS_TOKEN_READ_TIMEOUT_MS = 2_000;
|
|
48
|
+
const PROXY_STATUS_RECONCILE_TIMEOUT_MS = 750;
|
|
49
|
+
const PROXY_STATUS_ACCOUNT_INVENTORY_TIMEOUT_MS = 750;
|
|
48
50
|
let legacyStatusAccountCache;
|
|
49
51
|
// Allowed drift between a pid's OS-reported start time and the persisted
|
|
50
52
|
// ProxySupervisorState.startTime before processLooksLikeProxySupervisor
|
|
@@ -1592,9 +1594,20 @@ export async function createProxyStartApp(params) {
|
|
|
1592
1594
|
const activeAccountAllowlist = runtimeConfig
|
|
1593
1595
|
? runtimeConfig.accountAllowlist
|
|
1594
1596
|
: params.accountAllowlist;
|
|
1595
|
-
const { getReconciledUsageSnapshot, getUsageStatsPersistenceStatus } = await import("../../lib/proxy/usageStats.js");
|
|
1597
|
+
const { getReconciledUsageSnapshot, getUsageSnapshot, getUsageStatsPersistenceStatus, } = await import("../../lib/proxy/usageStats.js");
|
|
1596
1598
|
const { loadAccountCooldowns } = await import("../../lib/proxy/accountCooldown.js");
|
|
1597
|
-
|
|
1599
|
+
let usageSnapshot = getUsageSnapshot();
|
|
1600
|
+
let snapshotSource = "memory";
|
|
1601
|
+
try {
|
|
1602
|
+
usageSnapshot = await withTimeout(getReconciledUsageSnapshot(), PROXY_STATUS_RECONCILE_TIMEOUT_MS, "[proxy] /status usage reconciliation timed out");
|
|
1603
|
+
snapshotSource = "reconciled";
|
|
1604
|
+
}
|
|
1605
|
+
catch (error) {
|
|
1606
|
+
// Status must not become an outage amplifier when a cross-process lock or
|
|
1607
|
+
// a slow filesystem stalls reconciliation. The process-local snapshot is
|
|
1608
|
+
// coherent and its source is explicit to callers.
|
|
1609
|
+
logger.debug(`[proxy] /status using memory stats snapshot: ${error instanceof Error ? error.message : String(error)}`);
|
|
1610
|
+
}
|
|
1598
1611
|
const { stats, terminalErrors } = usageSnapshot;
|
|
1599
1612
|
const terminalErrorDetailsComparable = usageSnapshot.statsVersion === usageSnapshot.terminalErrorsVersion;
|
|
1600
1613
|
const lastTerminalError = terminalErrors.recent.at(-1) ?? null;
|
|
@@ -1608,40 +1621,52 @@ export async function createProxyStartApp(params) {
|
|
|
1608
1621
|
const supervisorState = loadProxySupervisorState();
|
|
1609
1622
|
const rollingSupervisorRunning = isRollingHandoffCapable(supervisorState);
|
|
1610
1623
|
const updateState = loadUpdateState();
|
|
1611
|
-
const cooldowns = await loadAccountCooldowns()
|
|
1624
|
+
const cooldowns = await withTimeout(loadAccountCooldowns(), PROXY_STATUS_ACCOUNT_INVENTORY_TIMEOUT_MS, "[proxy] /status cooldown inspection timed out").catch((error) => {
|
|
1625
|
+
logger.debug(`[proxy] /status using empty cooldown snapshot: ${error instanceof Error ? error.message : String(error)}`);
|
|
1626
|
+
return {};
|
|
1627
|
+
});
|
|
1612
1628
|
const storedAccountKeys = new Set();
|
|
1613
1629
|
const storedAccountExpirations = new Map();
|
|
1614
1630
|
const disabledAccountKeys = new Set();
|
|
1615
1631
|
let accountInventoryLoaded = false;
|
|
1616
1632
|
try {
|
|
1617
1633
|
const { tokenStore } = await import("../../lib/auth/tokenStore.js");
|
|
1618
|
-
const storedKeys = await tokenStore.listByPrefix("anthropic:");
|
|
1634
|
+
const storedKeys = await withTimeout(tokenStore.listByPrefix("anthropic:"), PROXY_STATUS_ACCOUNT_INVENTORY_TIMEOUT_MS, "[proxy] /status account enumeration timed out");
|
|
1619
1635
|
for (const key of storedKeys) {
|
|
1620
|
-
|
|
1621
|
-
storedAccountKeys.add(normalizedKey);
|
|
1636
|
+
storedAccountKeys.add(normalizeAnthropicAccountKey(key));
|
|
1622
1637
|
}
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1638
|
+
// Once account names are known, preserve them even when optional token
|
|
1639
|
+
// metadata is slow. That keeps the status table useful and avoids
|
|
1640
|
+
// incorrectly presenting known accounts as removed.
|
|
1641
|
+
accountInventoryLoaded = true;
|
|
1642
|
+
const inventory = await withTimeout((async () => {
|
|
1643
|
+
const tokenExpirations = await Promise.all(storedKeys.map(async (key) => {
|
|
1644
|
+
try {
|
|
1645
|
+
const tokens = await withTimeout(tokenStore.peekTokens(key), PROXY_STATUS_TOKEN_READ_TIMEOUT_MS, "[proxy] /status token inspection timed out");
|
|
1646
|
+
return tokens ? [key, tokens.expiresAt] : undefined;
|
|
1629
1647
|
}
|
|
1648
|
+
catch (error) {
|
|
1649
|
+
logger.debug(`[proxy] /status: failed to inspect token metadata for ${normalizeAnthropicAccountKey(key)}: ${error instanceof Error ? error.message : String(error)}`);
|
|
1650
|
+
return undefined;
|
|
1651
|
+
}
|
|
1652
|
+
}));
|
|
1653
|
+
const disabledKeys = await tokenStore.listDisabled();
|
|
1654
|
+
return { tokenExpirations, disabledKeys };
|
|
1655
|
+
})(), PROXY_STATUS_ACCOUNT_INVENTORY_TIMEOUT_MS, "[proxy] /status account metadata timed out");
|
|
1656
|
+
for (const expiration of inventory.tokenExpirations) {
|
|
1657
|
+
if (expiration) {
|
|
1658
|
+
storedAccountExpirations.set(normalizeAnthropicAccountKey(expiration[0]), expiration[1]);
|
|
1630
1659
|
}
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
}
|
|
1634
|
-
}));
|
|
1635
|
-
for (const key of await tokenStore.listDisabled()) {
|
|
1660
|
+
}
|
|
1661
|
+
for (const key of inventory.disabledKeys) {
|
|
1636
1662
|
disabledAccountKeys.add(normalizeAnthropicAccountKey(key));
|
|
1637
1663
|
}
|
|
1638
|
-
accountInventoryLoaded = true;
|
|
1639
1664
|
}
|
|
1640
1665
|
catch (err) {
|
|
1641
1666
|
logger.debug(`[proxy] /status: failed to resolve account cooldown labels: ${err instanceof Error ? err.message : String(err)}`);
|
|
1642
1667
|
}
|
|
1643
1668
|
const legacyAccountLabel = accountInventoryLoaded
|
|
1644
|
-
? await resolveLegacyStatusAccountLabel(storedAccountKeys.size)
|
|
1669
|
+
? await withTimeout(resolveLegacyStatusAccountLabel(storedAccountKeys.size), PROXY_STATUS_ACCOUNT_INVENTORY_TIMEOUT_MS, "[proxy] /status legacy account inspection timed out").catch(() => null)
|
|
1645
1670
|
: null;
|
|
1646
1671
|
const now = Date.now();
|
|
1647
1672
|
const health = buildProxyHealthResponse(readiness, {
|
|
@@ -1649,7 +1674,12 @@ export async function createProxyStartApp(params) {
|
|
|
1649
1674
|
passthrough: activePassthrough,
|
|
1650
1675
|
version: PROXY_VERSION,
|
|
1651
1676
|
});
|
|
1652
|
-
const primaryAccount = await resolveStatusPrimaryAccount(activeProxyConfig)
|
|
1677
|
+
const primaryAccount = await withTimeout(resolveStatusPrimaryAccount(activeProxyConfig), PROXY_STATUS_ACCOUNT_INVENTORY_TIMEOUT_MS, "[proxy] /status primary account inspection timed out").catch(() => ({
|
|
1678
|
+
configured: activeProxyConfig?.routing?.primaryAccount?.trim() || null,
|
|
1679
|
+
key: null,
|
|
1680
|
+
label: null,
|
|
1681
|
+
source: "fallback",
|
|
1682
|
+
}));
|
|
1653
1683
|
const activeUpdaterPid = supervisorState?.updaterPid ?? runtimeState?.updaterPid;
|
|
1654
1684
|
const accountRows = Object.values(stats.accounts).map((account) => {
|
|
1655
1685
|
const normalizedKey = normalizeAnthropicAccountKey(account.label);
|
|
@@ -1813,6 +1843,7 @@ export async function createProxyStartApp(params) {
|
|
|
1813
1843
|
terminalErrorDetailsComparable,
|
|
1814
1844
|
terminalErrorDetailsMissing,
|
|
1815
1845
|
terminalErrorDetailsExcess,
|
|
1846
|
+
snapshotSource,
|
|
1816
1847
|
accounts: accountRows,
|
|
1817
1848
|
primaryAccount,
|
|
1818
1849
|
persistence: getUsageStatsPersistenceStatus(),
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { appendFile } from "node:fs/promises";
|
|
1
2
|
import type { ProxyLifecycleEventInput, ProxyLifecycleLoggerOptions, ProxyLifecycleLoggerSnapshot } from "../types/index.js";
|
|
2
3
|
export declare function hashProxyLifecycleSessionId(sessionId: string | undefined): string | undefined;
|
|
3
4
|
export declare function configureProxyLifecycleLogger(options: ProxyLifecycleLoggerOptions): void;
|
|
@@ -6,3 +7,7 @@ export declare function logProxyLifecycleEvent(input: ProxyLifecycleEventInput):
|
|
|
6
7
|
export declare function flushProxyLifecycleEvents(): Promise<void>;
|
|
7
8
|
export declare function getProxyLifecycleLoggerSnapshot(): ProxyLifecycleLoggerSnapshot;
|
|
8
9
|
export declare function resetProxyLifecycleLoggerForTests(): void;
|
|
10
|
+
/** Isolated failure injection for lifecycle durability tests. */
|
|
11
|
+
export declare const __proxyLifecycleTestHooks: {
|
|
12
|
+
setAppendFileForTests(append: typeof appendFile): void;
|
|
13
|
+
};
|
|
@@ -9,6 +9,8 @@ const SCHEMA_VERSION = 1;
|
|
|
9
9
|
const DEFAULT_QUEUE_CAPACITY = 10_000;
|
|
10
10
|
const DEFAULT_BATCH_SIZE = 256;
|
|
11
11
|
const DEFAULT_FLUSH_INTERVAL_MS = 25;
|
|
12
|
+
const DEFAULT_MAX_WRITE_RETRIES = 3;
|
|
13
|
+
const MAX_WRITE_RETRY_DELAY_MS = 1_000;
|
|
12
14
|
const LIFECYCLE_APPEND_TIMEOUT_MS = 2_000;
|
|
13
15
|
const MAX_SHORT_FIELD_LENGTH = 256;
|
|
14
16
|
const SESSION_KEY_FILE = ".proxy-lifecycle-session-key";
|
|
@@ -17,6 +19,7 @@ let lifecycleLogDir;
|
|
|
17
19
|
let queueCapacity = DEFAULT_QUEUE_CAPACITY;
|
|
18
20
|
let batchSize = DEFAULT_BATCH_SIZE;
|
|
19
21
|
let flushIntervalMs = DEFAULT_FLUSH_INTERVAL_MS;
|
|
22
|
+
let maxWriteRetries = DEFAULT_MAX_WRITE_RETRIES;
|
|
20
23
|
let processInstanceId = randomUUID();
|
|
21
24
|
let sessionHashKey = randomBytes(32);
|
|
22
25
|
let nextSequence = 1;
|
|
@@ -28,10 +31,13 @@ let queueDrops = 0;
|
|
|
28
31
|
let invalidDrops = 0;
|
|
29
32
|
let writeDrops = 0;
|
|
30
33
|
let writeFailures = 0;
|
|
34
|
+
let writeRetries = 0;
|
|
31
35
|
let inFlight = 0;
|
|
32
36
|
let queue = [];
|
|
33
37
|
let flushTimer;
|
|
34
38
|
let flushInFlight;
|
|
39
|
+
let nextFlushDelayMs;
|
|
40
|
+
let appendLifecycleFile = appendFile;
|
|
35
41
|
function positiveInteger(value, fallback) {
|
|
36
42
|
return Number.isInteger(value) && (value ?? 0) > 0
|
|
37
43
|
? value
|
|
@@ -123,14 +129,14 @@ function clearScheduledFlush() {
|
|
|
123
129
|
flushTimer = undefined;
|
|
124
130
|
}
|
|
125
131
|
}
|
|
126
|
-
function scheduleFlush() {
|
|
132
|
+
function scheduleFlush(delayMs = flushIntervalMs) {
|
|
127
133
|
if (flushTimer || flushInFlight || queue.length === 0) {
|
|
128
134
|
return;
|
|
129
135
|
}
|
|
130
136
|
flushTimer = setTimeout(() => {
|
|
131
137
|
flushTimer = undefined;
|
|
132
138
|
void startFlush();
|
|
133
|
-
},
|
|
139
|
+
}, delayMs);
|
|
134
140
|
flushTimer.unref?.();
|
|
135
141
|
}
|
|
136
142
|
async function flushBatch() {
|
|
@@ -143,27 +149,53 @@ async function flushBatch() {
|
|
|
143
149
|
const byPath = new Map();
|
|
144
150
|
for (const item of batch) {
|
|
145
151
|
const path = join(item.logDir, `proxy-lifecycle-${item.date}.jsonl`);
|
|
146
|
-
const
|
|
147
|
-
|
|
148
|
-
byPath.set(path,
|
|
152
|
+
const items = byPath.get(path) ?? [];
|
|
153
|
+
items.push(item);
|
|
154
|
+
byPath.set(path, items);
|
|
149
155
|
}
|
|
150
|
-
|
|
156
|
+
const retries = [];
|
|
157
|
+
let retryDelayMs = 0;
|
|
158
|
+
for (const [path, items] of byPath) {
|
|
159
|
+
const lines = items.map((item) => `${JSON.stringify(item.record)}\n`);
|
|
151
160
|
try {
|
|
152
161
|
// This best-effort telemetry sink intentionally avoids fsync so request
|
|
153
162
|
// throughput is not coupled to storage latency. Loss is surfaced by
|
|
154
163
|
// writeDrops/writeFailures rather than delaying proxy responses.
|
|
155
|
-
await withTimeout(
|
|
164
|
+
await withTimeout(appendLifecycleFile(path, lines.join(""), { mode: 0o600 }), LIFECYCLE_APPEND_TIMEOUT_MS, "Timed out writing proxy lifecycle metadata");
|
|
156
165
|
written += lines.length;
|
|
157
166
|
}
|
|
158
167
|
catch (error) {
|
|
159
|
-
dropped += lines.length;
|
|
160
|
-
writeDrops += lines.length;
|
|
161
168
|
writeFailures += 1;
|
|
169
|
+
const retryable = items.filter((item) => item.writeRetries < maxWriteRetries);
|
|
170
|
+
const exhausted = items.length - retryable.length;
|
|
171
|
+
if (retryable.length > 0) {
|
|
172
|
+
const nextRetries = retryable.map((item) => ({
|
|
173
|
+
...item,
|
|
174
|
+
writeRetries: item.writeRetries + 1,
|
|
175
|
+
}));
|
|
176
|
+
retries.push(...nextRetries);
|
|
177
|
+
writeRetries += nextRetries.length;
|
|
178
|
+
retryDelayMs = Math.max(retryDelayMs, Math.min(MAX_WRITE_RETRY_DELAY_MS, flushIntervalMs *
|
|
179
|
+
2 ** Math.max(...nextRetries.map((item) => item.writeRetries))));
|
|
180
|
+
}
|
|
181
|
+
if (exhausted > 0) {
|
|
182
|
+
dropped += exhausted;
|
|
183
|
+
writeDrops += exhausted;
|
|
184
|
+
}
|
|
162
185
|
logger.warn("[proxy] lifecycle metadata write failed", {
|
|
186
|
+
path,
|
|
187
|
+
retrying: retryable.length,
|
|
188
|
+
dropped: exhausted,
|
|
163
189
|
error: error instanceof Error ? error.message : String(error),
|
|
164
190
|
});
|
|
165
191
|
}
|
|
166
192
|
}
|
|
193
|
+
if (retries.length > 0) {
|
|
194
|
+
// Keep retried records ahead of newly admitted records. This preserves
|
|
195
|
+
// per-file sequence order while continuing to keep request paths async.
|
|
196
|
+
queue.unshift(...retries);
|
|
197
|
+
nextFlushDelayMs = Math.max(nextFlushDelayMs ?? 0, retryDelayMs || flushIntervalMs);
|
|
198
|
+
}
|
|
167
199
|
}
|
|
168
200
|
finally {
|
|
169
201
|
inFlight = Math.max(0, inFlight - batch.length);
|
|
@@ -179,21 +211,27 @@ function startFlush() {
|
|
|
179
211
|
if (flushInFlight === currentFlush) {
|
|
180
212
|
flushInFlight = undefined;
|
|
181
213
|
}
|
|
182
|
-
|
|
214
|
+
const delayMs = nextFlushDelayMs;
|
|
215
|
+
nextFlushDelayMs = undefined;
|
|
216
|
+
scheduleFlush(delayMs);
|
|
183
217
|
}, () => {
|
|
184
218
|
if (flushInFlight === currentFlush) {
|
|
185
219
|
flushInFlight = undefined;
|
|
186
220
|
}
|
|
187
|
-
|
|
221
|
+
const delayMs = nextFlushDelayMs;
|
|
222
|
+
nextFlushDelayMs = undefined;
|
|
223
|
+
scheduleFlush(delayMs);
|
|
188
224
|
});
|
|
189
225
|
return currentFlush;
|
|
190
226
|
}
|
|
191
227
|
export function configureProxyLifecycleLogger(options) {
|
|
192
228
|
clearScheduledFlush();
|
|
229
|
+
nextFlushDelayMs = undefined;
|
|
193
230
|
loggerEnabled = false;
|
|
194
231
|
lifecycleLogDir = undefined;
|
|
195
232
|
queueCapacity = positiveInteger(options.queueCapacity, DEFAULT_QUEUE_CAPACITY);
|
|
196
233
|
batchSize = positiveInteger(options.batchSize, DEFAULT_BATCH_SIZE);
|
|
234
|
+
maxWriteRetries = positiveInteger(options.maxWriteRetries, DEFAULT_MAX_WRITE_RETRIES);
|
|
197
235
|
flushIntervalMs = positiveInteger(options.flushIntervalMs, DEFAULT_FLUSH_INTERVAL_MS);
|
|
198
236
|
if (options.enabled && options.logDir) {
|
|
199
237
|
try {
|
|
@@ -268,6 +306,7 @@ export function logProxyLifecycleEvent(input) {
|
|
|
268
306
|
logDir: lifecycleLogDir,
|
|
269
307
|
date: String(record.timestamp).slice(0, 10),
|
|
270
308
|
record,
|
|
309
|
+
writeRetries: 0,
|
|
271
310
|
});
|
|
272
311
|
enqueued += 1;
|
|
273
312
|
scheduleFlush();
|
|
@@ -303,6 +342,7 @@ export function getProxyLifecycleLoggerSnapshot() {
|
|
|
303
342
|
invalidDrops,
|
|
304
343
|
writeDrops,
|
|
305
344
|
writeFailures,
|
|
345
|
+
writeRetries,
|
|
306
346
|
pending: queue.length,
|
|
307
347
|
inFlight,
|
|
308
348
|
flushing: flushInFlight !== undefined,
|
|
@@ -315,6 +355,7 @@ export function resetProxyLifecycleLoggerForTests() {
|
|
|
315
355
|
queueCapacity = DEFAULT_QUEUE_CAPACITY;
|
|
316
356
|
batchSize = DEFAULT_BATCH_SIZE;
|
|
317
357
|
flushIntervalMs = DEFAULT_FLUSH_INTERVAL_MS;
|
|
358
|
+
maxWriteRetries = DEFAULT_MAX_WRITE_RETRIES;
|
|
318
359
|
processInstanceId = randomUUID();
|
|
319
360
|
sessionHashKey = randomBytes(32);
|
|
320
361
|
nextSequence = 1;
|
|
@@ -326,8 +367,17 @@ export function resetProxyLifecycleLoggerForTests() {
|
|
|
326
367
|
invalidDrops = 0;
|
|
327
368
|
writeDrops = 0;
|
|
328
369
|
writeFailures = 0;
|
|
370
|
+
writeRetries = 0;
|
|
329
371
|
inFlight = 0;
|
|
330
372
|
queue = [];
|
|
331
373
|
flushInFlight = undefined;
|
|
374
|
+
nextFlushDelayMs = undefined;
|
|
375
|
+
appendLifecycleFile = appendFile;
|
|
332
376
|
}
|
|
377
|
+
/** Isolated failure injection for lifecycle durability tests. */
|
|
378
|
+
export const __proxyLifecycleTestHooks = {
|
|
379
|
+
setAppendFileForTests(append) {
|
|
380
|
+
appendLifecycleFile = append;
|
|
381
|
+
},
|
|
382
|
+
};
|
|
333
383
|
//# sourceMappingURL=proxyLifecycle.js.map
|
|
@@ -77,6 +77,8 @@ export declare function recordFinalSuccess(accountLabel?: string, accountType?:
|
|
|
77
77
|
export declare function recordAttemptError(accountLabel: string, accountType: string, status: number, rateLimitKind?: "transient" | "quota"): void;
|
|
78
78
|
export declare function recordFinalError(status: number, accountLabel?: string, accountType?: string, details?: ProxyTerminalErrorDetails): void;
|
|
79
79
|
export declare function getStats(): ProxyStats;
|
|
80
|
+
/** Return the process-local coherent snapshot without filesystem reconciliation. */
|
|
81
|
+
export declare function getUsageSnapshot(): ProxyUsageStatsSnapshot;
|
|
80
82
|
export declare function getReconciledStats(): Promise<ProxyStats>;
|
|
81
83
|
export declare function getReconciledUsageSnapshot(): Promise<ProxyUsageStatsSnapshot>;
|
|
82
84
|
export declare function getAccountStats(label: string): AccountStats | undefined;
|
|
@@ -1121,6 +1121,10 @@ export function recordFinalError(status, accountLabel, accountType, details) {
|
|
|
1121
1121
|
export function getStats() {
|
|
1122
1122
|
return defaultStore.getStats();
|
|
1123
1123
|
}
|
|
1124
|
+
/** Return the process-local coherent snapshot without filesystem reconciliation. */
|
|
1125
|
+
export function getUsageSnapshot() {
|
|
1126
|
+
return defaultStore.getUsageSnapshot();
|
|
1127
|
+
}
|
|
1124
1128
|
export async function getReconciledStats() {
|
|
1125
1129
|
return defaultStore.reconcile();
|
|
1126
1130
|
}
|
|
@@ -1246,6 +1246,8 @@ export type ProxyLifecycleLoggerSnapshot = {
|
|
|
1246
1246
|
invalidDrops: number;
|
|
1247
1247
|
writeDrops: number;
|
|
1248
1248
|
writeFailures: number;
|
|
1249
|
+
/** Events requeued after a transient lifecycle metadata write failure. */
|
|
1250
|
+
writeRetries: number;
|
|
1249
1251
|
pending: number;
|
|
1250
1252
|
inFlight: number;
|
|
1251
1253
|
flushing: boolean;
|
|
@@ -1257,12 +1259,15 @@ export type ProxyLifecycleLoggerOptions = {
|
|
|
1257
1259
|
queueCapacity?: number;
|
|
1258
1260
|
batchSize?: number;
|
|
1259
1261
|
flushIntervalMs?: number;
|
|
1262
|
+
/** Bounded retries for a metadata batch that cannot be appended immediately. */
|
|
1263
|
+
maxWriteRetries?: number;
|
|
1260
1264
|
};
|
|
1261
1265
|
/** Serialized lifecycle line awaiting a bounded batch write. */
|
|
1262
1266
|
export type QueuedProxyLifecycleEvent = {
|
|
1263
1267
|
logDir: string;
|
|
1264
1268
|
date: string;
|
|
1265
1269
|
record: Record<string, unknown>;
|
|
1270
|
+
writeRetries: number;
|
|
1266
1271
|
};
|
|
1267
1272
|
/** Percentile summary used by offline proxy log analysis. */
|
|
1268
1273
|
export type ProxyLatencySummary = {
|
|
@@ -2167,6 +2172,8 @@ export type StatusStats = {
|
|
|
2167
2172
|
terminalErrorDetailsComparable?: boolean;
|
|
2168
2173
|
terminalErrorDetailsMissing?: number;
|
|
2169
2174
|
terminalErrorDetailsExcess?: number;
|
|
2175
|
+
/** Whether this status response reconciled shared state or used local memory. */
|
|
2176
|
+
snapshotSource?: "reconciled" | "memory";
|
|
2170
2177
|
accounts?: {
|
|
2171
2178
|
label: string;
|
|
2172
2179
|
type: string;
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { appendFile } from "node:fs/promises";
|
|
1
2
|
import type { ProxyLifecycleEventInput, ProxyLifecycleLoggerOptions, ProxyLifecycleLoggerSnapshot } from "../types/index.js";
|
|
2
3
|
export declare function hashProxyLifecycleSessionId(sessionId: string | undefined): string | undefined;
|
|
3
4
|
export declare function configureProxyLifecycleLogger(options: ProxyLifecycleLoggerOptions): void;
|
|
@@ -6,3 +7,7 @@ export declare function logProxyLifecycleEvent(input: ProxyLifecycleEventInput):
|
|
|
6
7
|
export declare function flushProxyLifecycleEvents(): Promise<void>;
|
|
7
8
|
export declare function getProxyLifecycleLoggerSnapshot(): ProxyLifecycleLoggerSnapshot;
|
|
8
9
|
export declare function resetProxyLifecycleLoggerForTests(): void;
|
|
10
|
+
/** Isolated failure injection for lifecycle durability tests. */
|
|
11
|
+
export declare const __proxyLifecycleTestHooks: {
|
|
12
|
+
setAppendFileForTests(append: typeof appendFile): void;
|
|
13
|
+
};
|
|
@@ -9,6 +9,8 @@ const SCHEMA_VERSION = 1;
|
|
|
9
9
|
const DEFAULT_QUEUE_CAPACITY = 10_000;
|
|
10
10
|
const DEFAULT_BATCH_SIZE = 256;
|
|
11
11
|
const DEFAULT_FLUSH_INTERVAL_MS = 25;
|
|
12
|
+
const DEFAULT_MAX_WRITE_RETRIES = 3;
|
|
13
|
+
const MAX_WRITE_RETRY_DELAY_MS = 1_000;
|
|
12
14
|
const LIFECYCLE_APPEND_TIMEOUT_MS = 2_000;
|
|
13
15
|
const MAX_SHORT_FIELD_LENGTH = 256;
|
|
14
16
|
const SESSION_KEY_FILE = ".proxy-lifecycle-session-key";
|
|
@@ -17,6 +19,7 @@ let lifecycleLogDir;
|
|
|
17
19
|
let queueCapacity = DEFAULT_QUEUE_CAPACITY;
|
|
18
20
|
let batchSize = DEFAULT_BATCH_SIZE;
|
|
19
21
|
let flushIntervalMs = DEFAULT_FLUSH_INTERVAL_MS;
|
|
22
|
+
let maxWriteRetries = DEFAULT_MAX_WRITE_RETRIES;
|
|
20
23
|
let processInstanceId = randomUUID();
|
|
21
24
|
let sessionHashKey = randomBytes(32);
|
|
22
25
|
let nextSequence = 1;
|
|
@@ -28,10 +31,13 @@ let queueDrops = 0;
|
|
|
28
31
|
let invalidDrops = 0;
|
|
29
32
|
let writeDrops = 0;
|
|
30
33
|
let writeFailures = 0;
|
|
34
|
+
let writeRetries = 0;
|
|
31
35
|
let inFlight = 0;
|
|
32
36
|
let queue = [];
|
|
33
37
|
let flushTimer;
|
|
34
38
|
let flushInFlight;
|
|
39
|
+
let nextFlushDelayMs;
|
|
40
|
+
let appendLifecycleFile = appendFile;
|
|
35
41
|
function positiveInteger(value, fallback) {
|
|
36
42
|
return Number.isInteger(value) && (value ?? 0) > 0
|
|
37
43
|
? value
|
|
@@ -123,14 +129,14 @@ function clearScheduledFlush() {
|
|
|
123
129
|
flushTimer = undefined;
|
|
124
130
|
}
|
|
125
131
|
}
|
|
126
|
-
function scheduleFlush() {
|
|
132
|
+
function scheduleFlush(delayMs = flushIntervalMs) {
|
|
127
133
|
if (flushTimer || flushInFlight || queue.length === 0) {
|
|
128
134
|
return;
|
|
129
135
|
}
|
|
130
136
|
flushTimer = setTimeout(() => {
|
|
131
137
|
flushTimer = undefined;
|
|
132
138
|
void startFlush();
|
|
133
|
-
},
|
|
139
|
+
}, delayMs);
|
|
134
140
|
flushTimer.unref?.();
|
|
135
141
|
}
|
|
136
142
|
async function flushBatch() {
|
|
@@ -143,27 +149,53 @@ async function flushBatch() {
|
|
|
143
149
|
const byPath = new Map();
|
|
144
150
|
for (const item of batch) {
|
|
145
151
|
const path = join(item.logDir, `proxy-lifecycle-${item.date}.jsonl`);
|
|
146
|
-
const
|
|
147
|
-
|
|
148
|
-
byPath.set(path,
|
|
152
|
+
const items = byPath.get(path) ?? [];
|
|
153
|
+
items.push(item);
|
|
154
|
+
byPath.set(path, items);
|
|
149
155
|
}
|
|
150
|
-
|
|
156
|
+
const retries = [];
|
|
157
|
+
let retryDelayMs = 0;
|
|
158
|
+
for (const [path, items] of byPath) {
|
|
159
|
+
const lines = items.map((item) => `${JSON.stringify(item.record)}\n`);
|
|
151
160
|
try {
|
|
152
161
|
// This best-effort telemetry sink intentionally avoids fsync so request
|
|
153
162
|
// throughput is not coupled to storage latency. Loss is surfaced by
|
|
154
163
|
// writeDrops/writeFailures rather than delaying proxy responses.
|
|
155
|
-
await withTimeout(
|
|
164
|
+
await withTimeout(appendLifecycleFile(path, lines.join(""), { mode: 0o600 }), LIFECYCLE_APPEND_TIMEOUT_MS, "Timed out writing proxy lifecycle metadata");
|
|
156
165
|
written += lines.length;
|
|
157
166
|
}
|
|
158
167
|
catch (error) {
|
|
159
|
-
dropped += lines.length;
|
|
160
|
-
writeDrops += lines.length;
|
|
161
168
|
writeFailures += 1;
|
|
169
|
+
const retryable = items.filter((item) => item.writeRetries < maxWriteRetries);
|
|
170
|
+
const exhausted = items.length - retryable.length;
|
|
171
|
+
if (retryable.length > 0) {
|
|
172
|
+
const nextRetries = retryable.map((item) => ({
|
|
173
|
+
...item,
|
|
174
|
+
writeRetries: item.writeRetries + 1,
|
|
175
|
+
}));
|
|
176
|
+
retries.push(...nextRetries);
|
|
177
|
+
writeRetries += nextRetries.length;
|
|
178
|
+
retryDelayMs = Math.max(retryDelayMs, Math.min(MAX_WRITE_RETRY_DELAY_MS, flushIntervalMs *
|
|
179
|
+
2 ** Math.max(...nextRetries.map((item) => item.writeRetries))));
|
|
180
|
+
}
|
|
181
|
+
if (exhausted > 0) {
|
|
182
|
+
dropped += exhausted;
|
|
183
|
+
writeDrops += exhausted;
|
|
184
|
+
}
|
|
162
185
|
logger.warn("[proxy] lifecycle metadata write failed", {
|
|
186
|
+
path,
|
|
187
|
+
retrying: retryable.length,
|
|
188
|
+
dropped: exhausted,
|
|
163
189
|
error: error instanceof Error ? error.message : String(error),
|
|
164
190
|
});
|
|
165
191
|
}
|
|
166
192
|
}
|
|
193
|
+
if (retries.length > 0) {
|
|
194
|
+
// Keep retried records ahead of newly admitted records. This preserves
|
|
195
|
+
// per-file sequence order while continuing to keep request paths async.
|
|
196
|
+
queue.unshift(...retries);
|
|
197
|
+
nextFlushDelayMs = Math.max(nextFlushDelayMs ?? 0, retryDelayMs || flushIntervalMs);
|
|
198
|
+
}
|
|
167
199
|
}
|
|
168
200
|
finally {
|
|
169
201
|
inFlight = Math.max(0, inFlight - batch.length);
|
|
@@ -179,21 +211,27 @@ function startFlush() {
|
|
|
179
211
|
if (flushInFlight === currentFlush) {
|
|
180
212
|
flushInFlight = undefined;
|
|
181
213
|
}
|
|
182
|
-
|
|
214
|
+
const delayMs = nextFlushDelayMs;
|
|
215
|
+
nextFlushDelayMs = undefined;
|
|
216
|
+
scheduleFlush(delayMs);
|
|
183
217
|
}, () => {
|
|
184
218
|
if (flushInFlight === currentFlush) {
|
|
185
219
|
flushInFlight = undefined;
|
|
186
220
|
}
|
|
187
|
-
|
|
221
|
+
const delayMs = nextFlushDelayMs;
|
|
222
|
+
nextFlushDelayMs = undefined;
|
|
223
|
+
scheduleFlush(delayMs);
|
|
188
224
|
});
|
|
189
225
|
return currentFlush;
|
|
190
226
|
}
|
|
191
227
|
export function configureProxyLifecycleLogger(options) {
|
|
192
228
|
clearScheduledFlush();
|
|
229
|
+
nextFlushDelayMs = undefined;
|
|
193
230
|
loggerEnabled = false;
|
|
194
231
|
lifecycleLogDir = undefined;
|
|
195
232
|
queueCapacity = positiveInteger(options.queueCapacity, DEFAULT_QUEUE_CAPACITY);
|
|
196
233
|
batchSize = positiveInteger(options.batchSize, DEFAULT_BATCH_SIZE);
|
|
234
|
+
maxWriteRetries = positiveInteger(options.maxWriteRetries, DEFAULT_MAX_WRITE_RETRIES);
|
|
197
235
|
flushIntervalMs = positiveInteger(options.flushIntervalMs, DEFAULT_FLUSH_INTERVAL_MS);
|
|
198
236
|
if (options.enabled && options.logDir) {
|
|
199
237
|
try {
|
|
@@ -268,6 +306,7 @@ export function logProxyLifecycleEvent(input) {
|
|
|
268
306
|
logDir: lifecycleLogDir,
|
|
269
307
|
date: String(record.timestamp).slice(0, 10),
|
|
270
308
|
record,
|
|
309
|
+
writeRetries: 0,
|
|
271
310
|
});
|
|
272
311
|
enqueued += 1;
|
|
273
312
|
scheduleFlush();
|
|
@@ -303,6 +342,7 @@ export function getProxyLifecycleLoggerSnapshot() {
|
|
|
303
342
|
invalidDrops,
|
|
304
343
|
writeDrops,
|
|
305
344
|
writeFailures,
|
|
345
|
+
writeRetries,
|
|
306
346
|
pending: queue.length,
|
|
307
347
|
inFlight,
|
|
308
348
|
flushing: flushInFlight !== undefined,
|
|
@@ -315,6 +355,7 @@ export function resetProxyLifecycleLoggerForTests() {
|
|
|
315
355
|
queueCapacity = DEFAULT_QUEUE_CAPACITY;
|
|
316
356
|
batchSize = DEFAULT_BATCH_SIZE;
|
|
317
357
|
flushIntervalMs = DEFAULT_FLUSH_INTERVAL_MS;
|
|
358
|
+
maxWriteRetries = DEFAULT_MAX_WRITE_RETRIES;
|
|
318
359
|
processInstanceId = randomUUID();
|
|
319
360
|
sessionHashKey = randomBytes(32);
|
|
320
361
|
nextSequence = 1;
|
|
@@ -326,7 +367,16 @@ export function resetProxyLifecycleLoggerForTests() {
|
|
|
326
367
|
invalidDrops = 0;
|
|
327
368
|
writeDrops = 0;
|
|
328
369
|
writeFailures = 0;
|
|
370
|
+
writeRetries = 0;
|
|
329
371
|
inFlight = 0;
|
|
330
372
|
queue = [];
|
|
331
373
|
flushInFlight = undefined;
|
|
374
|
+
nextFlushDelayMs = undefined;
|
|
375
|
+
appendLifecycleFile = appendFile;
|
|
332
376
|
}
|
|
377
|
+
/** Isolated failure injection for lifecycle durability tests. */
|
|
378
|
+
export const __proxyLifecycleTestHooks = {
|
|
379
|
+
setAppendFileForTests(append) {
|
|
380
|
+
appendLifecycleFile = append;
|
|
381
|
+
},
|
|
382
|
+
};
|
|
@@ -77,6 +77,8 @@ export declare function recordFinalSuccess(accountLabel?: string, accountType?:
|
|
|
77
77
|
export declare function recordAttemptError(accountLabel: string, accountType: string, status: number, rateLimitKind?: "transient" | "quota"): void;
|
|
78
78
|
export declare function recordFinalError(status: number, accountLabel?: string, accountType?: string, details?: ProxyTerminalErrorDetails): void;
|
|
79
79
|
export declare function getStats(): ProxyStats;
|
|
80
|
+
/** Return the process-local coherent snapshot without filesystem reconciliation. */
|
|
81
|
+
export declare function getUsageSnapshot(): ProxyUsageStatsSnapshot;
|
|
80
82
|
export declare function getReconciledStats(): Promise<ProxyStats>;
|
|
81
83
|
export declare function getReconciledUsageSnapshot(): Promise<ProxyUsageStatsSnapshot>;
|
|
82
84
|
export declare function getAccountStats(label: string): AccountStats | undefined;
|
package/dist/proxy/usageStats.js
CHANGED
|
@@ -1121,6 +1121,10 @@ export function recordFinalError(status, accountLabel, accountType, details) {
|
|
|
1121
1121
|
export function getStats() {
|
|
1122
1122
|
return defaultStore.getStats();
|
|
1123
1123
|
}
|
|
1124
|
+
/** Return the process-local coherent snapshot without filesystem reconciliation. */
|
|
1125
|
+
export function getUsageSnapshot() {
|
|
1126
|
+
return defaultStore.getUsageSnapshot();
|
|
1127
|
+
}
|
|
1124
1128
|
export async function getReconciledStats() {
|
|
1125
1129
|
return defaultStore.reconcile();
|
|
1126
1130
|
}
|
package/dist/types/proxy.d.ts
CHANGED
|
@@ -1246,6 +1246,8 @@ export type ProxyLifecycleLoggerSnapshot = {
|
|
|
1246
1246
|
invalidDrops: number;
|
|
1247
1247
|
writeDrops: number;
|
|
1248
1248
|
writeFailures: number;
|
|
1249
|
+
/** Events requeued after a transient lifecycle metadata write failure. */
|
|
1250
|
+
writeRetries: number;
|
|
1249
1251
|
pending: number;
|
|
1250
1252
|
inFlight: number;
|
|
1251
1253
|
flushing: boolean;
|
|
@@ -1257,12 +1259,15 @@ export type ProxyLifecycleLoggerOptions = {
|
|
|
1257
1259
|
queueCapacity?: number;
|
|
1258
1260
|
batchSize?: number;
|
|
1259
1261
|
flushIntervalMs?: number;
|
|
1262
|
+
/** Bounded retries for a metadata batch that cannot be appended immediately. */
|
|
1263
|
+
maxWriteRetries?: number;
|
|
1260
1264
|
};
|
|
1261
1265
|
/** Serialized lifecycle line awaiting a bounded batch write. */
|
|
1262
1266
|
export type QueuedProxyLifecycleEvent = {
|
|
1263
1267
|
logDir: string;
|
|
1264
1268
|
date: string;
|
|
1265
1269
|
record: Record<string, unknown>;
|
|
1270
|
+
writeRetries: number;
|
|
1266
1271
|
};
|
|
1267
1272
|
/** Percentile summary used by offline proxy log analysis. */
|
|
1268
1273
|
export type ProxyLatencySummary = {
|
|
@@ -2167,6 +2172,8 @@ export type StatusStats = {
|
|
|
2167
2172
|
terminalErrorDetailsComparable?: boolean;
|
|
2168
2173
|
terminalErrorDetailsMissing?: number;
|
|
2169
2174
|
terminalErrorDetailsExcess?: number;
|
|
2175
|
+
/** Whether this status response reconciled shared state or used local memory. */
|
|
2176
|
+
snapshotSource?: "reconciled" | "memory";
|
|
2170
2177
|
accounts?: {
|
|
2171
2178
|
label: string;
|
|
2172
2179
|
type: string;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@juspay/neurolink",
|
|
3
|
-
"version": "10.9.
|
|
3
|
+
"version": "10.9.1",
|
|
4
4
|
"packageManager": "pnpm@10.15.1",
|
|
5
5
|
"description": "TypeScript AI SDK with 24+ LLM providers behind one consistent API. MCP-native (connect any MCP server), voice TTS/STT/realtime, RAG, agents, memory, context compaction. OpenAI · Anthropic · Gemini · Bedrock · Azure · Ollama · DeepSeek · NVIDIA NIM and more.",
|
|
6
6
|
"author": {
|