@juspay/neurolink 9.92.3 → 9.93.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/dist/browser/neurolink.min.js +287 -287
- package/dist/cli/commands/proxy.d.ts +2 -1
- package/dist/cli/commands/proxy.js +157 -27
- package/dist/lib/proxy/proxyActivity.d.ts +2 -2
- package/dist/lib/proxy/proxyActivity.js +49 -9
- package/dist/lib/proxy/proxyLifecycle.d.ts +8 -0
- package/dist/lib/proxy/proxyLifecycle.js +333 -0
- package/dist/lib/proxy/requestLogger.js +6 -0
- package/dist/lib/server/routes/claudeProxyRoutes.js +23 -18
- package/dist/lib/types/proxy.d.ts +82 -0
- package/dist/proxy/proxyActivity.d.ts +2 -2
- package/dist/proxy/proxyActivity.js +49 -9
- package/dist/proxy/proxyLifecycle.d.ts +8 -0
- package/dist/proxy/proxyLifecycle.js +332 -0
- package/dist/proxy/requestLogger.js +6 -0
- package/dist/server/routes/claudeProxyRoutes.js +23 -18
- package/dist/types/proxy.d.ts +82 -0
- package/package.json +1 -1
|
@@ -0,0 +1,333 @@
|
|
|
1
|
+
import { createHash, createHmac, randomBytes, randomUUID } from "node:crypto";
|
|
2
|
+
import { chmodSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { appendFile } from "node:fs/promises";
|
|
4
|
+
import { join } from "node:path";
|
|
5
|
+
import { performance } from "node:perf_hooks";
|
|
6
|
+
import { withTimeout } from "../utils/async/withTimeout.js";
|
|
7
|
+
import { logger } from "../utils/logger.js";
|
|
8
|
+
const SCHEMA_VERSION = 1;
|
|
9
|
+
const DEFAULT_QUEUE_CAPACITY = 10_000;
|
|
10
|
+
const DEFAULT_BATCH_SIZE = 256;
|
|
11
|
+
const DEFAULT_FLUSH_INTERVAL_MS = 25;
|
|
12
|
+
const LIFECYCLE_APPEND_TIMEOUT_MS = 2_000;
|
|
13
|
+
const MAX_SHORT_FIELD_LENGTH = 256;
|
|
14
|
+
const SESSION_KEY_FILE = ".proxy-lifecycle-session-key";
|
|
15
|
+
let loggerEnabled = false;
|
|
16
|
+
let lifecycleLogDir;
|
|
17
|
+
let queueCapacity = DEFAULT_QUEUE_CAPACITY;
|
|
18
|
+
let batchSize = DEFAULT_BATCH_SIZE;
|
|
19
|
+
let flushIntervalMs = DEFAULT_FLUSH_INTERVAL_MS;
|
|
20
|
+
let processInstanceId = randomUUID();
|
|
21
|
+
let sessionHashKey = randomBytes(32);
|
|
22
|
+
let nextSequence = 1;
|
|
23
|
+
let attempted = 0;
|
|
24
|
+
let enqueued = 0;
|
|
25
|
+
let written = 0;
|
|
26
|
+
let dropped = 0;
|
|
27
|
+
let queueDrops = 0;
|
|
28
|
+
let invalidDrops = 0;
|
|
29
|
+
let writeDrops = 0;
|
|
30
|
+
let writeFailures = 0;
|
|
31
|
+
let inFlight = 0;
|
|
32
|
+
let queue = [];
|
|
33
|
+
let flushTimer;
|
|
34
|
+
let flushInFlight;
|
|
35
|
+
function positiveInteger(value, fallback) {
|
|
36
|
+
return Number.isInteger(value) && (value ?? 0) > 0
|
|
37
|
+
? value
|
|
38
|
+
: fallback;
|
|
39
|
+
}
|
|
40
|
+
function clip(value) {
|
|
41
|
+
if (typeof value !== "string") {
|
|
42
|
+
return undefined;
|
|
43
|
+
}
|
|
44
|
+
return value.length <= MAX_SHORT_FIELD_LENGTH
|
|
45
|
+
? value
|
|
46
|
+
: value.slice(0, MAX_SHORT_FIELD_LENGTH);
|
|
47
|
+
}
|
|
48
|
+
function finiteNonNegative(value) {
|
|
49
|
+
return value !== undefined && Number.isFinite(value) && value >= 0
|
|
50
|
+
? value
|
|
51
|
+
: undefined;
|
|
52
|
+
}
|
|
53
|
+
function nonNegativeInteger(value) {
|
|
54
|
+
const finite = finiteNonNegative(value);
|
|
55
|
+
return finite === undefined ? undefined : Math.floor(finite);
|
|
56
|
+
}
|
|
57
|
+
function formatTimestamp(timestampMs) {
|
|
58
|
+
const candidate = typeof timestampMs === "number" && Number.isFinite(timestampMs)
|
|
59
|
+
? timestampMs
|
|
60
|
+
: Date.now();
|
|
61
|
+
const date = new Date(candidate);
|
|
62
|
+
return Number.isFinite(date.getTime())
|
|
63
|
+
? date.toISOString()
|
|
64
|
+
: new Date().toISOString();
|
|
65
|
+
}
|
|
66
|
+
function readPersistedSessionHashKey(path) {
|
|
67
|
+
try {
|
|
68
|
+
const key = Buffer.from(readFileSync(path, "utf8").trim(), "base64");
|
|
69
|
+
if (key.length !== 32) {
|
|
70
|
+
throw new Error("persisted lifecycle session key must contain 32 bytes");
|
|
71
|
+
}
|
|
72
|
+
chmodSync(path, 0o600);
|
|
73
|
+
return key;
|
|
74
|
+
}
|
|
75
|
+
catch (error) {
|
|
76
|
+
if (error.code === "ENOENT") {
|
|
77
|
+
return undefined;
|
|
78
|
+
}
|
|
79
|
+
throw error;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
function resolveSessionHashKey(logDir) {
|
|
83
|
+
const configuredSecret = process.env.NEUROLINK_PROXY_SESSION_SECRET?.trim();
|
|
84
|
+
if (configuredSecret) {
|
|
85
|
+
return createHash("sha256").update(configuredSecret).digest();
|
|
86
|
+
}
|
|
87
|
+
const keyPath = join(logDir, SESSION_KEY_FILE);
|
|
88
|
+
const existing = readPersistedSessionHashKey(keyPath);
|
|
89
|
+
if (existing) {
|
|
90
|
+
return existing;
|
|
91
|
+
}
|
|
92
|
+
const generated = randomBytes(32);
|
|
93
|
+
try {
|
|
94
|
+
writeFileSync(keyPath, generated.toString("base64"), {
|
|
95
|
+
encoding: "utf8",
|
|
96
|
+
flag: "wx",
|
|
97
|
+
mode: 0o600,
|
|
98
|
+
});
|
|
99
|
+
return generated;
|
|
100
|
+
}
|
|
101
|
+
catch (error) {
|
|
102
|
+
if (error.code === "EEXIST") {
|
|
103
|
+
const concurrentlyCreated = readPersistedSessionHashKey(keyPath);
|
|
104
|
+
if (concurrentlyCreated) {
|
|
105
|
+
return concurrentlyCreated;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
throw error;
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
export function hashProxyLifecycleSessionId(sessionId) {
|
|
112
|
+
if (!sessionId) {
|
|
113
|
+
return undefined;
|
|
114
|
+
}
|
|
115
|
+
return createHmac("sha256", sessionHashKey)
|
|
116
|
+
.update(sessionId)
|
|
117
|
+
.digest("hex")
|
|
118
|
+
.slice(0, 24);
|
|
119
|
+
}
|
|
120
|
+
function clearScheduledFlush() {
|
|
121
|
+
if (flushTimer) {
|
|
122
|
+
clearTimeout(flushTimer);
|
|
123
|
+
flushTimer = undefined;
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
function scheduleFlush() {
|
|
127
|
+
if (flushTimer || flushInFlight || queue.length === 0) {
|
|
128
|
+
return;
|
|
129
|
+
}
|
|
130
|
+
flushTimer = setTimeout(() => {
|
|
131
|
+
flushTimer = undefined;
|
|
132
|
+
void startFlush();
|
|
133
|
+
}, flushIntervalMs);
|
|
134
|
+
flushTimer.unref?.();
|
|
135
|
+
}
|
|
136
|
+
async function flushBatch() {
|
|
137
|
+
if (queue.length === 0) {
|
|
138
|
+
return;
|
|
139
|
+
}
|
|
140
|
+
const batch = queue.splice(0, batchSize);
|
|
141
|
+
inFlight += batch.length;
|
|
142
|
+
try {
|
|
143
|
+
const byPath = new Map();
|
|
144
|
+
for (const item of batch) {
|
|
145
|
+
const path = join(item.logDir, `proxy-lifecycle-${item.date}.jsonl`);
|
|
146
|
+
const lines = byPath.get(path) ?? [];
|
|
147
|
+
lines.push(`${JSON.stringify(item.record)}\n`);
|
|
148
|
+
byPath.set(path, lines);
|
|
149
|
+
}
|
|
150
|
+
for (const [path, lines] of byPath) {
|
|
151
|
+
try {
|
|
152
|
+
// This best-effort telemetry sink intentionally avoids fsync so request
|
|
153
|
+
// throughput is not coupled to storage latency. Loss is surfaced by
|
|
154
|
+
// writeDrops/writeFailures rather than delaying proxy responses.
|
|
155
|
+
await withTimeout(appendFile(path, lines.join(""), { mode: 0o600 }), LIFECYCLE_APPEND_TIMEOUT_MS, "Timed out writing proxy lifecycle metadata");
|
|
156
|
+
written += lines.length;
|
|
157
|
+
}
|
|
158
|
+
catch (error) {
|
|
159
|
+
dropped += lines.length;
|
|
160
|
+
writeDrops += lines.length;
|
|
161
|
+
writeFailures += 1;
|
|
162
|
+
logger.warn("[proxy] lifecycle metadata write failed", {
|
|
163
|
+
error: error instanceof Error ? error.message : String(error),
|
|
164
|
+
});
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
finally {
|
|
169
|
+
inFlight = Math.max(0, inFlight - batch.length);
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
function startFlush() {
|
|
173
|
+
if (flushInFlight) {
|
|
174
|
+
return flushInFlight;
|
|
175
|
+
}
|
|
176
|
+
const currentFlush = flushBatch();
|
|
177
|
+
flushInFlight = currentFlush;
|
|
178
|
+
void currentFlush.then(() => {
|
|
179
|
+
if (flushInFlight === currentFlush) {
|
|
180
|
+
flushInFlight = undefined;
|
|
181
|
+
}
|
|
182
|
+
scheduleFlush();
|
|
183
|
+
}, () => {
|
|
184
|
+
if (flushInFlight === currentFlush) {
|
|
185
|
+
flushInFlight = undefined;
|
|
186
|
+
}
|
|
187
|
+
scheduleFlush();
|
|
188
|
+
});
|
|
189
|
+
return currentFlush;
|
|
190
|
+
}
|
|
191
|
+
export function configureProxyLifecycleLogger(options) {
|
|
192
|
+
clearScheduledFlush();
|
|
193
|
+
loggerEnabled = false;
|
|
194
|
+
lifecycleLogDir = undefined;
|
|
195
|
+
queueCapacity = positiveInteger(options.queueCapacity, DEFAULT_QUEUE_CAPACITY);
|
|
196
|
+
batchSize = positiveInteger(options.batchSize, DEFAULT_BATCH_SIZE);
|
|
197
|
+
flushIntervalMs = positiveInteger(options.flushIntervalMs, DEFAULT_FLUSH_INTERVAL_MS);
|
|
198
|
+
if (options.enabled && options.logDir) {
|
|
199
|
+
try {
|
|
200
|
+
mkdirSync(options.logDir, { recursive: true, mode: 0o700 });
|
|
201
|
+
sessionHashKey = resolveSessionHashKey(options.logDir);
|
|
202
|
+
lifecycleLogDir = options.logDir;
|
|
203
|
+
loggerEnabled = true;
|
|
204
|
+
}
|
|
205
|
+
catch (error) {
|
|
206
|
+
logger.warn("[proxy] lifecycle metadata logging disabled", {
|
|
207
|
+
error: error instanceof Error ? error.message : String(error),
|
|
208
|
+
});
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
// Queued records retain their enqueue-time destination. Reconfiguration is
|
|
212
|
+
// therefore atomic for new events while an in-flight or pending batch safely
|
|
213
|
+
// drains to its original directory.
|
|
214
|
+
scheduleFlush();
|
|
215
|
+
}
|
|
216
|
+
/** Enqueue fixed-size lifecycle metadata without awaiting filesystem work. */
|
|
217
|
+
export function logProxyLifecycleEvent(input) {
|
|
218
|
+
if (!loggerEnabled || !lifecycleLogDir) {
|
|
219
|
+
return;
|
|
220
|
+
}
|
|
221
|
+
try {
|
|
222
|
+
attempted += 1;
|
|
223
|
+
const sequence = nextSequence++;
|
|
224
|
+
if (queue.length >= queueCapacity) {
|
|
225
|
+
dropped += 1;
|
|
226
|
+
queueDrops += 1;
|
|
227
|
+
return;
|
|
228
|
+
}
|
|
229
|
+
const timestamp = formatTimestamp(input.timestampMs);
|
|
230
|
+
const monotonicMs = finiteNonNegative(input.monotonicMs) ?? performance.now();
|
|
231
|
+
const toolCount = nonNegativeInteger(input.toolCount);
|
|
232
|
+
const requestBytes = nonNegativeInteger(input.requestBytes);
|
|
233
|
+
const responseStatus = nonNegativeInteger(input.responseStatus);
|
|
234
|
+
const observedBodyBytes = nonNegativeInteger(input.observedBodyBytes);
|
|
235
|
+
const responseChunks = nonNegativeInteger(input.responseChunks);
|
|
236
|
+
const elapsedMs = finiteNonNegative(input.elapsedMs);
|
|
237
|
+
const model = clip(input.model);
|
|
238
|
+
const sessionHash = clip(input.sessionHash);
|
|
239
|
+
const terminalOutcome = clip(input.terminalOutcome);
|
|
240
|
+
const errorType = clip(input.errorType);
|
|
241
|
+
const errorCode = clip(input.errorCode);
|
|
242
|
+
const record = {
|
|
243
|
+
schemaVersion: SCHEMA_VERSION,
|
|
244
|
+
timestamp,
|
|
245
|
+
monotonicMs: Number(monotonicMs.toFixed(3)),
|
|
246
|
+
processInstanceId,
|
|
247
|
+
sequence,
|
|
248
|
+
event: clip(input.event) ?? "unknown",
|
|
249
|
+
requestId: clip(input.requestId) ?? "unknown",
|
|
250
|
+
method: clip(input.method) ?? "unknown",
|
|
251
|
+
path: clip(input.path) ?? "unknown",
|
|
252
|
+
...(model !== undefined ? { model } : {}),
|
|
253
|
+
...(input.stream !== undefined ? { stream: input.stream } : {}),
|
|
254
|
+
...(toolCount !== undefined ? { toolCount } : {}),
|
|
255
|
+
...(sessionHash !== undefined ? { sessionHash } : {}),
|
|
256
|
+
...(requestBytes !== undefined ? { requestBytes } : {}),
|
|
257
|
+
...(responseStatus !== undefined ? { responseStatus } : {}),
|
|
258
|
+
...(observedBodyBytes !== undefined ? { observedBodyBytes } : {}),
|
|
259
|
+
...(responseChunks !== undefined ? { responseChunks } : {}),
|
|
260
|
+
...(elapsedMs !== undefined
|
|
261
|
+
? { elapsedMs: Number(elapsedMs.toFixed(3)) }
|
|
262
|
+
: {}),
|
|
263
|
+
...(terminalOutcome !== undefined ? { terminalOutcome } : {}),
|
|
264
|
+
...(errorType !== undefined ? { errorType } : {}),
|
|
265
|
+
...(errorCode !== undefined ? { errorCode } : {}),
|
|
266
|
+
};
|
|
267
|
+
queue.push({
|
|
268
|
+
logDir: lifecycleLogDir,
|
|
269
|
+
date: String(record.timestamp).slice(0, 10),
|
|
270
|
+
record,
|
|
271
|
+
});
|
|
272
|
+
enqueued += 1;
|
|
273
|
+
scheduleFlush();
|
|
274
|
+
}
|
|
275
|
+
catch {
|
|
276
|
+
dropped += 1;
|
|
277
|
+
invalidDrops += 1;
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
export async function flushProxyLifecycleEvents() {
|
|
281
|
+
clearScheduledFlush();
|
|
282
|
+
while (queue.length > 0 || flushInFlight) {
|
|
283
|
+
if (flushInFlight) {
|
|
284
|
+
await flushInFlight;
|
|
285
|
+
}
|
|
286
|
+
else {
|
|
287
|
+
await startFlush();
|
|
288
|
+
}
|
|
289
|
+
clearScheduledFlush();
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
export function getProxyLifecycleLoggerSnapshot() {
|
|
293
|
+
return {
|
|
294
|
+
enabled: loggerEnabled,
|
|
295
|
+
schemaVersion: SCHEMA_VERSION,
|
|
296
|
+
processInstanceId,
|
|
297
|
+
nextSequence,
|
|
298
|
+
attempted,
|
|
299
|
+
enqueued,
|
|
300
|
+
written,
|
|
301
|
+
dropped,
|
|
302
|
+
queueDrops,
|
|
303
|
+
invalidDrops,
|
|
304
|
+
writeDrops,
|
|
305
|
+
writeFailures,
|
|
306
|
+
pending: queue.length,
|
|
307
|
+
inFlight,
|
|
308
|
+
flushing: flushInFlight !== undefined,
|
|
309
|
+
};
|
|
310
|
+
}
|
|
311
|
+
export function resetProxyLifecycleLoggerForTests() {
|
|
312
|
+
clearScheduledFlush();
|
|
313
|
+
loggerEnabled = false;
|
|
314
|
+
lifecycleLogDir = undefined;
|
|
315
|
+
queueCapacity = DEFAULT_QUEUE_CAPACITY;
|
|
316
|
+
batchSize = DEFAULT_BATCH_SIZE;
|
|
317
|
+
flushIntervalMs = DEFAULT_FLUSH_INTERVAL_MS;
|
|
318
|
+
processInstanceId = randomUUID();
|
|
319
|
+
sessionHashKey = randomBytes(32);
|
|
320
|
+
nextSequence = 1;
|
|
321
|
+
attempted = 0;
|
|
322
|
+
enqueued = 0;
|
|
323
|
+
written = 0;
|
|
324
|
+
dropped = 0;
|
|
325
|
+
queueDrops = 0;
|
|
326
|
+
invalidDrops = 0;
|
|
327
|
+
writeDrops = 0;
|
|
328
|
+
writeFailures = 0;
|
|
329
|
+
inFlight = 0;
|
|
330
|
+
queue = [];
|
|
331
|
+
flushInFlight = undefined;
|
|
332
|
+
}
|
|
333
|
+
//# sourceMappingURL=proxyLifecycle.js.map
|
|
@@ -15,6 +15,7 @@ import { promisify } from "util";
|
|
|
15
15
|
import { gzip as gzipCallback } from "zlib";
|
|
16
16
|
import { OtelBridge } from "../observability/otelBridge.js";
|
|
17
17
|
import { SeverityNumber } from "@opentelemetry/api-logs";
|
|
18
|
+
import { configureProxyLifecycleLogger } from "./proxyLifecycle.js";
|
|
18
19
|
let logDir = null;
|
|
19
20
|
let logEnabled = false;
|
|
20
21
|
/**
|
|
@@ -45,8 +46,11 @@ const SENSITIVE_HEADER_PATTERN = /token|secret|key|password|credential/i;
|
|
|
45
46
|
/** JSON keys whose values should be redacted in request/response bodies. */
|
|
46
47
|
const SENSITIVE_BODY_KEYS = /("(?:password|access_token|refresh_token|api_key|apiKey|secret|authorization|token|credential|x-api-key)"\s*:\s*)"(?:[^"\\]|\\.)*"/gi;
|
|
47
48
|
export function initRequestLogger(enabled = true, customLogsDir) {
|
|
49
|
+
// Lifecycle metadata deliberately shares the request logger's enablement,
|
|
50
|
+
// directory permissions, retention boundary, and operator privacy control.
|
|
48
51
|
logEnabled = enabled;
|
|
49
52
|
if (!enabled) {
|
|
53
|
+
configureProxyLifecycleLogger({ enabled: false });
|
|
50
54
|
return;
|
|
51
55
|
}
|
|
52
56
|
try {
|
|
@@ -55,10 +59,12 @@ export function initRequestLogger(enabled = true, customLogsDir) {
|
|
|
55
59
|
mkdirSync(logDir, { recursive: true, mode: 0o700 });
|
|
56
60
|
}
|
|
57
61
|
chmodSync(logDir, 0o700);
|
|
62
|
+
configureProxyLifecycleLogger({ enabled: true, logDir });
|
|
58
63
|
}
|
|
59
64
|
catch (err) {
|
|
60
65
|
logEnabled = false;
|
|
61
66
|
logDir = null;
|
|
67
|
+
configureProxyLifecycleLogger({ enabled: false });
|
|
62
68
|
logger.warn(`[proxy] Request logging disabled — failed to create log directory: ${err instanceof Error ? err.message : String(err)}`);
|
|
63
69
|
}
|
|
64
70
|
}
|
|
@@ -3062,7 +3062,10 @@ async function handleAnthropicAuthRetry(args) {
|
|
|
3062
3062
|
: String(retryFetchErr);
|
|
3063
3063
|
authRetryError = `network error on retry ${authRetry + 1}: ${message}`;
|
|
3064
3064
|
currentLastError = authRetryError;
|
|
3065
|
-
retryLogAttempt(502, "network_error", message, {
|
|
3065
|
+
retryLogAttempt(502, "network_error", message, {
|
|
3066
|
+
retryable: isRetryableNetworkError(retryFetchErr),
|
|
3067
|
+
errorCode: getErrorCode(retryFetchErr) ?? "unknown",
|
|
3068
|
+
});
|
|
3066
3069
|
logger.debug(`[proxy] ${authRetryError}`);
|
|
3067
3070
|
break;
|
|
3068
3071
|
}
|
|
@@ -3437,6 +3440,7 @@ function createAnthropicAttemptLogger(args) {
|
|
|
3437
3440
|
responseTimeMs: Date.now() - requestStart,
|
|
3438
3441
|
...(errorType ? { errorType } : {}),
|
|
3439
3442
|
...(errorMessage ? { errorMessage } : {}),
|
|
3443
|
+
...(extra?.errorCode ? { errorCode: extra.errorCode } : {}),
|
|
3440
3444
|
...(extra?.inputTokens !== undefined
|
|
3441
3445
|
? { inputTokens: extra.inputTokens }
|
|
3442
3446
|
: {}),
|
|
@@ -3651,19 +3655,27 @@ async function fetchAnthropicAccountResponse(args) {
|
|
|
3651
3655
|
});
|
|
3652
3656
|
}
|
|
3653
3657
|
catch (fetchErr) {
|
|
3654
|
-
|
|
3655
|
-
|
|
3656
|
-
|
|
3658
|
+
const retryable = isRetryableNetworkError(fetchErr);
|
|
3659
|
+
// Every dispatched upstream request is an attempt, including terminal
|
|
3660
|
+
// transport failures. Record it once before preserving the throw behavior.
|
|
3657
3661
|
sawNetworkError = true;
|
|
3658
3662
|
recordAttemptError(account.label, account.type, 502);
|
|
3659
3663
|
const errorCode = getErrorCode(fetchErr) ?? "unknown";
|
|
3660
3664
|
const errorMessage = fetchErr instanceof Error ? fetchErr.message : String(fetchErr);
|
|
3661
3665
|
lastError = errorMessage;
|
|
3662
|
-
logger.always(`[proxy] fetch error account=${account.label} code=${errorCode} (retryable): ${errorMessage}`);
|
|
3663
|
-
logAttempt(502, "network_error", errorMessage
|
|
3666
|
+
logger.always(`[proxy] fetch error account=${account.label} code=${errorCode} (${retryable ? "retryable" : "terminal"}): ${errorMessage}`);
|
|
3667
|
+
logAttempt(502, "network_error", errorMessage, {
|
|
3668
|
+
retryable,
|
|
3669
|
+
errorCode,
|
|
3670
|
+
});
|
|
3664
3671
|
tracer?.setError("network_error", errorMessage);
|
|
3665
|
-
|
|
3672
|
+
if (retryable) {
|
|
3673
|
+
tracer?.recordRetry(account.label, "network_error");
|
|
3674
|
+
}
|
|
3666
3675
|
currentUpstreamSpan?.end();
|
|
3676
|
+
if (!retryable) {
|
|
3677
|
+
throw fetchErr;
|
|
3678
|
+
}
|
|
3667
3679
|
return {
|
|
3668
3680
|
continueLoop: true,
|
|
3669
3681
|
retrySameAccount: true,
|
|
@@ -4469,16 +4481,13 @@ function describeTransportError(error) {
|
|
|
4469
4481
|
*/
|
|
4470
4482
|
function isRetryableNetworkError(error) {
|
|
4471
4483
|
const code = getErrorCode(error);
|
|
4472
|
-
// Check non-retryable codes FIRST — before the string-based heuristic
|
|
4473
|
-
// which could false-positive on error messages containing these strings.
|
|
4474
|
-
const NON_RETRYABLE_CODES = ["ENOTFOUND"];
|
|
4475
|
-
if (code && NON_RETRYABLE_CODES.includes(code)) {
|
|
4476
|
-
return false;
|
|
4477
|
-
}
|
|
4478
4484
|
if (code &&
|
|
4479
4485
|
[
|
|
4480
4486
|
"ECONNREFUSED",
|
|
4481
4487
|
"ECONNRESET",
|
|
4488
|
+
// The Anthropic host is fixed, so ENOTFOUND can be a transient resolver
|
|
4489
|
+
// outage. Keep it inside the existing bounded same-account retry budget.
|
|
4490
|
+
"ENOTFOUND",
|
|
4482
4491
|
"ETIMEDOUT",
|
|
4483
4492
|
"EHOSTUNREACH",
|
|
4484
4493
|
"UND_ERR_CONNECT_TIMEOUT",
|
|
@@ -4490,13 +4499,9 @@ function isRetryableNetworkError(error) {
|
|
|
4490
4499
|
}
|
|
4491
4500
|
const message = error instanceof Error ? error.message : String(error);
|
|
4492
4501
|
const normalized = message.toLowerCase();
|
|
4493
|
-
// Exclude ENOTFOUND from string-based heuristic — DNS failures are permanent
|
|
4494
|
-
// and rotating accounts won't help since they all hit the same host.
|
|
4495
|
-
if (normalized.includes("enotfound")) {
|
|
4496
|
-
return false;
|
|
4497
|
-
}
|
|
4498
4502
|
return (normalized.includes("econnrefused") ||
|
|
4499
4503
|
normalized.includes("econnreset") ||
|
|
4504
|
+
normalized.includes("enotfound") ||
|
|
4500
4505
|
normalized.includes("etimedout") ||
|
|
4501
4506
|
normalized.includes("timed out") ||
|
|
4502
4507
|
normalized.includes("connection error") ||
|
|
@@ -437,6 +437,8 @@ export type RequestLogEntry = {
|
|
|
437
437
|
responseTimeMs: number;
|
|
438
438
|
errorType?: string;
|
|
439
439
|
errorMessage?: string;
|
|
440
|
+
/** Low-level transport code such as ETIMEDOUT or EADDRNOTAVAIL. */
|
|
441
|
+
errorCode?: string;
|
|
440
442
|
inputTokens?: number;
|
|
441
443
|
outputTokens?: number;
|
|
442
444
|
cacheCreationTokens?: number;
|
|
@@ -461,6 +463,8 @@ export type RequestAttemptLogEntry = {
|
|
|
461
463
|
responseTimeMs: number;
|
|
462
464
|
errorType?: string;
|
|
463
465
|
errorMessage?: string;
|
|
466
|
+
/** Low-level transport code such as ETIMEDOUT or EADDRNOTAVAIL. */
|
|
467
|
+
errorCode?: string;
|
|
464
468
|
/** Whether this failed attempt may be retried without changing the request. */
|
|
465
469
|
retryable?: boolean;
|
|
466
470
|
/** Distinguishes short-lived admission throttles from exhausted quota windows. */
|
|
@@ -514,6 +518,8 @@ export type AnthropicAttemptLogger = (status: number, errorType?: string, errorM
|
|
|
514
518
|
cacheCreationTokens?: number;
|
|
515
519
|
cacheReadTokens?: number;
|
|
516
520
|
retryable?: boolean;
|
|
521
|
+
/** Low-level transport code such as ETIMEDOUT or EADDRNOTAVAIL. */
|
|
522
|
+
errorCode?: string;
|
|
517
523
|
rateLimitKind?: "transient" | "quota";
|
|
518
524
|
cooldownReason?: "transient" | "session" | "weekly" | "unified";
|
|
519
525
|
/** Override used when one account selection performs an OAuth retry fetch. */
|
|
@@ -945,6 +951,80 @@ export type ProxyRuntimeActivity = {
|
|
|
945
951
|
activeRequests: number;
|
|
946
952
|
lastActivityAt: string | null;
|
|
947
953
|
};
|
|
954
|
+
/** Terminal state observed while the HTTP adapter relays a response body. */
|
|
955
|
+
export type ProxyResponseTerminalOutcome = "completed" | "bodyless" | "client_cancelled" | "stream_error";
|
|
956
|
+
/** Non-blocking callbacks for response lifecycle metadata. */
|
|
957
|
+
export type ProxyResponseTrackingObserver = {
|
|
958
|
+
onFirstChunk?: (details: {
|
|
959
|
+
/** Decoded response-body bytes observed by the adapter. */
|
|
960
|
+
observedBodyBytes: number;
|
|
961
|
+
responseChunks: 1;
|
|
962
|
+
}) => void;
|
|
963
|
+
onTerminal?: (details: {
|
|
964
|
+
outcome: ProxyResponseTerminalOutcome;
|
|
965
|
+
/** Decoded response-body bytes observed by the adapter. */
|
|
966
|
+
observedBodyBytes: number;
|
|
967
|
+
responseChunks: number;
|
|
968
|
+
}) => void;
|
|
969
|
+
};
|
|
970
|
+
/** Versioned lifecycle event names persisted by the proxy adapter. */
|
|
971
|
+
export type ProxyLifecycleEventName = "request_accepted" | "response_headers" | "response_first_chunk" | "request_terminal";
|
|
972
|
+
/** Client-facing terminal classifications recorded by lifecycle metadata. */
|
|
973
|
+
export type ProxyLifecycleTerminalOutcome = ProxyResponseTerminalOutcome | "handler_error";
|
|
974
|
+
/** Content-free lifecycle event accepted by the bounded metadata logger. */
|
|
975
|
+
export type ProxyLifecycleEventInput = {
|
|
976
|
+
event: ProxyLifecycleEventName;
|
|
977
|
+
requestId: string;
|
|
978
|
+
method: string;
|
|
979
|
+
path: string;
|
|
980
|
+
model?: string;
|
|
981
|
+
stream?: boolean;
|
|
982
|
+
toolCount?: number;
|
|
983
|
+
sessionHash?: string;
|
|
984
|
+
requestBytes?: number;
|
|
985
|
+
responseStatus?: number;
|
|
986
|
+
/** Decoded response-body bytes observed by the adapter. */
|
|
987
|
+
observedBodyBytes?: number;
|
|
988
|
+
responseChunks?: number;
|
|
989
|
+
elapsedMs?: number;
|
|
990
|
+
terminalOutcome?: ProxyLifecycleTerminalOutcome;
|
|
991
|
+
errorType?: string;
|
|
992
|
+
errorCode?: string;
|
|
993
|
+
timestampMs?: number;
|
|
994
|
+
monotonicMs?: number;
|
|
995
|
+
};
|
|
996
|
+
/** Data-quality counters for the bounded lifecycle metadata sink. */
|
|
997
|
+
export type ProxyLifecycleLoggerSnapshot = {
|
|
998
|
+
enabled: boolean;
|
|
999
|
+
schemaVersion: number;
|
|
1000
|
+
processInstanceId: string;
|
|
1001
|
+
nextSequence: number;
|
|
1002
|
+
attempted: number;
|
|
1003
|
+
enqueued: number;
|
|
1004
|
+
written: number;
|
|
1005
|
+
dropped: number;
|
|
1006
|
+
queueDrops: number;
|
|
1007
|
+
invalidDrops: number;
|
|
1008
|
+
writeDrops: number;
|
|
1009
|
+
writeFailures: number;
|
|
1010
|
+
pending: number;
|
|
1011
|
+
inFlight: number;
|
|
1012
|
+
flushing: boolean;
|
|
1013
|
+
};
|
|
1014
|
+
/** Lifecycle logger configuration. Queue overrides are used by stress tests. */
|
|
1015
|
+
export type ProxyLifecycleLoggerOptions = {
|
|
1016
|
+
enabled: boolean;
|
|
1017
|
+
logDir?: string;
|
|
1018
|
+
queueCapacity?: number;
|
|
1019
|
+
batchSize?: number;
|
|
1020
|
+
flushIntervalMs?: number;
|
|
1021
|
+
};
|
|
1022
|
+
/** Serialized lifecycle line awaiting a bounded batch write. */
|
|
1023
|
+
export type QueuedProxyLifecycleEvent = {
|
|
1024
|
+
logDir: string;
|
|
1025
|
+
date: string;
|
|
1026
|
+
record: Record<string, unknown>;
|
|
1027
|
+
};
|
|
948
1028
|
/** Request metadata retained by the HTTP adapter for terminal error logging. */
|
|
949
1029
|
export type RuntimeRequestMetadata = {
|
|
950
1030
|
requestId: string;
|
|
@@ -954,6 +1034,8 @@ export type RuntimeRequestMetadata = {
|
|
|
954
1034
|
model: string;
|
|
955
1035
|
stream: boolean;
|
|
956
1036
|
toolCount: number;
|
|
1037
|
+
terminalErrorType?: string;
|
|
1038
|
+
terminalErrorCode?: string;
|
|
957
1039
|
};
|
|
958
1040
|
/** Accumulated upstream body capture from a raw stream. */
|
|
959
1041
|
export type RawStreamCapture = {
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import type { ProxyActivitySnapshot } from "../types/index.js";
|
|
1
|
+
import type { ProxyActivitySnapshot, ProxyResponseTrackingObserver } from "../types/index.js";
|
|
2
2
|
/** Track one client-facing proxy request until its response body settles. */
|
|
3
3
|
export declare function beginProxyRequest(): () => void;
|
|
4
4
|
export declare function getProxyActivitySnapshot(): ProxyActivitySnapshot;
|
|
5
5
|
export declare function isProxyActivityQuiet(snapshot: ProxyActivitySnapshot, quietThresholdMs: number, nowMs?: number): boolean;
|
|
6
6
|
/** Keep activity open until the response body completes, errors, or is cancelled. */
|
|
7
|
-
export declare function trackProxyResponse(response: Response, finishRequest: () => void): Response;
|
|
7
|
+
export declare function trackProxyResponse(response: Response, finishRequest: () => void, observer?: ProxyResponseTrackingObserver): Response;
|
|
8
8
|
export declare function resetProxyActivityForTests(): void;
|
|
@@ -1,8 +1,24 @@
|
|
|
1
|
+
import { withTimeout } from "../utils/async/withTimeout.js";
|
|
2
|
+
import { logger } from "../utils/logger.js";
|
|
3
|
+
const PROXY_RESPONSE_CANCEL_TIMEOUT_MS = 1_000;
|
|
1
4
|
let activeRequests = 0;
|
|
2
5
|
let lastActivityAtMs = null;
|
|
3
6
|
function touchActivity() {
|
|
4
7
|
lastActivityAtMs = Date.now();
|
|
5
8
|
}
|
|
9
|
+
function safelyNotifyObserver(callback) {
|
|
10
|
+
try {
|
|
11
|
+
callback?.();
|
|
12
|
+
}
|
|
13
|
+
catch (error) {
|
|
14
|
+
// Observability must never alter response handling.
|
|
15
|
+
if (logger.shouldLog("debug")) {
|
|
16
|
+
logger.debug("[proxy] response lifecycle observer failed", {
|
|
17
|
+
error: error instanceof Error ? error.message : String(error),
|
|
18
|
+
});
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
}
|
|
6
22
|
/** Track one client-facing proxy request until its response body settles. */
|
|
7
23
|
export function beginProxyRequest() {
|
|
8
24
|
activeRequests += 1;
|
|
@@ -33,35 +49,59 @@ export function isProxyActivityQuiet(snapshot, quietThresholdMs, nowMs = Date.no
|
|
|
33
49
|
return nowMs - snapshot.lastActivityAt.getTime() >= quietThresholdMs;
|
|
34
50
|
}
|
|
35
51
|
/** Keep activity open until the response body completes, errors, or is cancelled. */
|
|
36
|
-
export function trackProxyResponse(response, finishRequest) {
|
|
52
|
+
export function trackProxyResponse(response, finishRequest, observer) {
|
|
37
53
|
if (!response.body) {
|
|
38
54
|
finishRequest();
|
|
55
|
+
safelyNotifyObserver(() => observer?.onTerminal?.({
|
|
56
|
+
outcome: "bodyless",
|
|
57
|
+
observedBodyBytes: 0,
|
|
58
|
+
responseChunks: 0,
|
|
59
|
+
}));
|
|
39
60
|
return response;
|
|
40
61
|
}
|
|
41
62
|
const reader = response.body.getReader();
|
|
63
|
+
let observedBodyBytes = 0;
|
|
64
|
+
let responseChunks = 0;
|
|
65
|
+
let settled = false;
|
|
66
|
+
const settle = (outcome) => {
|
|
67
|
+
if (settled) {
|
|
68
|
+
return;
|
|
69
|
+
}
|
|
70
|
+
settled = true;
|
|
71
|
+
finishRequest();
|
|
72
|
+
safelyNotifyObserver(() => observer?.onTerminal?.({
|
|
73
|
+
outcome,
|
|
74
|
+
observedBodyBytes,
|
|
75
|
+
responseChunks,
|
|
76
|
+
}));
|
|
77
|
+
};
|
|
42
78
|
const trackedBody = new ReadableStream({
|
|
43
79
|
async pull(controller) {
|
|
44
80
|
try {
|
|
45
81
|
const { value, done } = await reader.read();
|
|
46
82
|
if (done) {
|
|
47
|
-
|
|
83
|
+
settle("completed");
|
|
48
84
|
controller.close();
|
|
49
85
|
return;
|
|
50
86
|
}
|
|
51
87
|
controller.enqueue(value);
|
|
88
|
+
observedBodyBytes += value.byteLength;
|
|
89
|
+
responseChunks += 1;
|
|
90
|
+
if (responseChunks === 1) {
|
|
91
|
+
safelyNotifyObserver(() => observer?.onFirstChunk?.({
|
|
92
|
+
observedBodyBytes,
|
|
93
|
+
responseChunks: 1,
|
|
94
|
+
}));
|
|
95
|
+
}
|
|
52
96
|
}
|
|
53
97
|
catch (error) {
|
|
54
|
-
|
|
98
|
+
settle("stream_error");
|
|
55
99
|
controller.error(error);
|
|
56
100
|
}
|
|
57
101
|
},
|
|
58
102
|
async cancel(reason) {
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
}
|
|
62
|
-
finally {
|
|
63
|
-
finishRequest();
|
|
64
|
-
}
|
|
103
|
+
settle("client_cancelled");
|
|
104
|
+
await withTimeout(reader.cancel(reason), PROXY_RESPONSE_CANCEL_TIMEOUT_MS, "Timed out cancelling the upstream proxy response");
|
|
65
105
|
},
|
|
66
106
|
});
|
|
67
107
|
return new Response(trackedBody, {
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { ProxyLifecycleEventInput, ProxyLifecycleLoggerOptions, ProxyLifecycleLoggerSnapshot } from "../types/index.js";
|
|
2
|
+
export declare function hashProxyLifecycleSessionId(sessionId: string | undefined): string | undefined;
|
|
3
|
+
export declare function configureProxyLifecycleLogger(options: ProxyLifecycleLoggerOptions): void;
|
|
4
|
+
/** Enqueue fixed-size lifecycle metadata without awaiting filesystem work. */
|
|
5
|
+
export declare function logProxyLifecycleEvent(input: ProxyLifecycleEventInput): void;
|
|
6
|
+
export declare function flushProxyLifecycleEvents(): Promise<void>;
|
|
7
|
+
export declare function getProxyLifecycleLoggerSnapshot(): ProxyLifecycleLoggerSnapshot;
|
|
8
|
+
export declare function resetProxyLifecycleLoggerForTests(): void;
|