@mxf-dev/core 2.0.1 → 2.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/protocols/mcp/providers/OpenRouterMcpClient.d.ts +34 -4
- package/dist/protocols/mcp/providers/OpenRouterMcpClient.d.ts.map +1 -1
- package/dist/protocols/mcp/providers/OpenRouterMcpClient.js +297 -65
- package/dist/protocols/mcp/providers/OpenRouterMcpClient.js.map +1 -1
- package/dist/protocols/mcp/utils/NetworkRecovery.d.ts +21 -0
- package/dist/protocols/mcp/utils/NetworkRecovery.d.ts.map +1 -1
- package/dist/protocols/mcp/utils/NetworkRecovery.js +62 -2
- package/dist/protocols/mcp/utils/NetworkRecovery.js.map +1 -1
- package/dist/services/MemoryService.d.ts +17 -1
- package/dist/services/MemoryService.d.ts.map +1 -1
- package/dist/services/MemoryService.js +33 -13
- package/dist/services/MemoryService.js.map +1 -1
- package/dist/types/NetworkRecoveryTypes.d.ts +1 -0
- package/dist/types/NetworkRecoveryTypes.d.ts.map +1 -1
- package/dist/types/NetworkRecoveryTypes.js +14 -0
- package/dist/types/NetworkRecoveryTypes.js.map +1 -1
- package/package.json +1 -1
- package/src/protocols/mcp/providers/OpenRouterMcpClient.ts +351 -83
- package/src/protocols/mcp/utils/NetworkRecovery.ts +74 -3
- package/src/services/MemoryService.ts +40 -16
- package/src/types/NetworkRecoveryTypes.ts +17 -1
|
@@ -114,6 +114,34 @@ interface OpenRouterResponse {
|
|
|
114
114
|
};
|
|
115
115
|
}
|
|
116
116
|
|
|
117
|
+
/**
|
|
118
|
+
* Read a positive integer from the environment, failing fast on garbage.
|
|
119
|
+
* These values bound how long a hung request can stay silent — a NaN or zero
|
|
120
|
+
* from a typo'd env var must not silently disable that bound.
|
|
121
|
+
*/
|
|
122
|
+
const parsePositiveIntEnv = (name: string, defaultValue: number): number => {
|
|
123
|
+
const raw = process.env[name];
|
|
124
|
+
if (raw === undefined || raw === '') {
|
|
125
|
+
return defaultValue;
|
|
126
|
+
}
|
|
127
|
+
const parsed = parseInt(raw, 10);
|
|
128
|
+
if (!Number.isFinite(parsed) || parsed <= 0) {
|
|
129
|
+
throw new Error(`${name} must be a positive integer, got "${raw}"`);
|
|
130
|
+
}
|
|
131
|
+
return parsed;
|
|
132
|
+
};
|
|
133
|
+
|
|
134
|
+
/**
|
|
135
|
+
* True when an error came from an aborted fetch. AbortSignal.timeout() rejects
|
|
136
|
+
* with a DOMException named 'TimeoutError' (Node and Bun); a manual
|
|
137
|
+
* controller.abort() rejects with 'AbortError'. This client owns every signal it
|
|
138
|
+
* passes to fetch, so either name means our own timeout fired.
|
|
139
|
+
*/
|
|
140
|
+
const isAbortOrTimeoutError = (error: unknown): boolean => {
|
|
141
|
+
const name = (error as any)?.name;
|
|
142
|
+
return name === 'TimeoutError' || name === 'AbortError';
|
|
143
|
+
};
|
|
144
|
+
|
|
117
145
|
/**
|
|
118
146
|
* OpenRouter implementation of the MCP client with network recovery
|
|
119
147
|
*/
|
|
@@ -233,50 +261,69 @@ export class OpenRouterMcpClient extends BaseMcpClient {
|
|
|
233
261
|
}
|
|
234
262
|
}
|
|
235
263
|
|
|
236
|
-
//
|
|
237
|
-
|
|
238
|
-
|
|
264
|
+
// Per-instance request queue: keeps one client's requests ordered and spaced.
|
|
265
|
+
// This used to be static (class-level), which serialized every request from
|
|
266
|
+
// every client instance in the process through one queue — one hung request
|
|
267
|
+
// starved every agent's LLM calls, not just its own. Instance scoping plus the
|
|
268
|
+
// per-request timeout below bounds the damage a single request can do to the
|
|
269
|
+
// agent that issued it.
|
|
270
|
+
private requestQueue: Array<() => Promise<any>> = [];
|
|
271
|
+
private isProcessingQueue = false;
|
|
239
272
|
// Configurable delay between requests - reduced from 500ms to 100ms default for better performance
|
|
240
273
|
// Set OPENROUTER_REQUEST_QUEUE_DELAY_MS=0 to disable queueing delay entirely
|
|
241
274
|
private static readonly REQUEST_DELAY_MS = parseInt(process.env.OPENROUTER_REQUEST_QUEUE_DELAY_MS || '100', 10);
|
|
242
|
-
|
|
275
|
+
|
|
243
276
|
// Network recovery manager
|
|
244
277
|
private networkRecovery: NetworkRecoveryManager | null = null;
|
|
245
|
-
|
|
278
|
+
|
|
246
279
|
// JSON recovery manager
|
|
247
280
|
private jsonRecovery: JsonRecoveryManager;
|
|
248
|
-
|
|
281
|
+
|
|
282
|
+
// Hard cap on a single completion request (fetch + body). Generous because
|
|
283
|
+
// reasoning models legitimately run for minutes; finite because a request
|
|
284
|
+
// with no bound turns a hung connection into permanent silence.
|
|
285
|
+
private requestTimeoutMs = 300000;
|
|
286
|
+
|
|
287
|
+
// Max silence between SSE chunks on the streaming path. OpenRouter emits
|
|
288
|
+
// keepalive comment lines every few seconds while a model is thinking, so a
|
|
289
|
+
// long quiet gap means a dead connection, not a slow model.
|
|
290
|
+
private streamIdleTimeoutMs = 120000;
|
|
291
|
+
|
|
292
|
+
// Threshold for the slow-request WARN that makes slow-vs-hung visible in
|
|
293
|
+
// production logs before any timeout fires.
|
|
294
|
+
private slowRequestWarnMs = 60000;
|
|
295
|
+
|
|
249
296
|
/**
|
|
250
297
|
* Process the request queue sequentially to prevent concurrent requests
|
|
251
298
|
*/
|
|
252
|
-
private
|
|
253
|
-
if (
|
|
299
|
+
private async processQueue(): Promise<void> {
|
|
300
|
+
if (this.isProcessingQueue || this.requestQueue.length === 0) {
|
|
254
301
|
return;
|
|
255
302
|
}
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
while (
|
|
260
|
-
const request =
|
|
303
|
+
|
|
304
|
+
this.isProcessingQueue = true;
|
|
305
|
+
|
|
306
|
+
while (this.requestQueue.length > 0) {
|
|
307
|
+
const request = this.requestQueue.shift()!;
|
|
261
308
|
try {
|
|
262
309
|
await request();
|
|
263
310
|
} catch (error) {
|
|
264
311
|
// Request will handle its own error, just continue processing
|
|
265
312
|
}
|
|
266
|
-
|
|
313
|
+
|
|
267
314
|
// Wait between requests to prevent rate limiting
|
|
268
|
-
if (
|
|
315
|
+
if (this.requestQueue.length > 0) {
|
|
269
316
|
await new Promise(resolve => setTimeout(resolve, OpenRouterMcpClient.REQUEST_DELAY_MS));
|
|
270
317
|
}
|
|
271
318
|
}
|
|
272
|
-
|
|
273
|
-
|
|
319
|
+
|
|
320
|
+
this.isProcessingQueue = false;
|
|
274
321
|
}
|
|
275
|
-
|
|
322
|
+
|
|
276
323
|
/**
|
|
277
324
|
* Add a request to the queue and process it
|
|
278
325
|
*/
|
|
279
|
-
private
|
|
326
|
+
private async queueRequest<T>(requestFn: () => Promise<T>): Promise<T> {
|
|
280
327
|
return new Promise<T>((resolve, reject) => {
|
|
281
328
|
const wrappedRequest = async () => {
|
|
282
329
|
try {
|
|
@@ -286,16 +333,105 @@ export class OpenRouterMcpClient extends BaseMcpClient {
|
|
|
286
333
|
reject(error);
|
|
287
334
|
}
|
|
288
335
|
};
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
336
|
+
|
|
337
|
+
this.requestQueue.push(wrappedRequest);
|
|
338
|
+
this.processQueue();
|
|
292
339
|
});
|
|
293
340
|
}
|
|
341
|
+
|
|
342
|
+
/**
|
|
343
|
+
* Start the slow-request watchdog for one LLM request.
|
|
344
|
+
*
|
|
345
|
+
* Emits a WARN once the request has been in flight for slowRequestWarnMs, and
|
|
346
|
+
* another WARN at completion if the total time crossed the threshold. Together
|
|
347
|
+
* with the timeout ERROR this makes slow-vs-hung distinguishable in production
|
|
348
|
+
* logs: a slow request logs WARN…WARN(completed), a hung one WARN…ERROR(timeout).
|
|
349
|
+
*
|
|
350
|
+
* Returns a finish() that must be called exactly once when the request settles.
|
|
351
|
+
*/
|
|
352
|
+
private startSlowRequestWatch(
|
|
353
|
+
kind: 'completion' | 'streaming',
|
|
354
|
+
model: string,
|
|
355
|
+
agentId: string,
|
|
356
|
+
requestBytes: number
|
|
357
|
+
): { finish: (succeeded?: boolean) => void } {
|
|
358
|
+
const startedAt = Date.now();
|
|
359
|
+
let finished = false;
|
|
360
|
+
const timer = setTimeout(() => {
|
|
361
|
+
this.logger.warn(
|
|
362
|
+
`⏱️ OpenRouter ${kind} request still in flight after ${this.slowRequestWarnMs}ms: ` +
|
|
363
|
+
`model=${model}, agent=${agentId}, request=${(requestBytes / 1024).toFixed(1)}KB`
|
|
364
|
+
);
|
|
365
|
+
}, this.slowRequestWarnMs);
|
|
366
|
+
|
|
367
|
+
return {
|
|
368
|
+
// Idempotent. The completed-WARN is success-only: failures carry their
|
|
369
|
+
// elapsed time in their own ERROR log, and a "completed" line for a
|
|
370
|
+
// request that timed out would be a lie.
|
|
371
|
+
finish: (succeeded: boolean = true) => {
|
|
372
|
+
if (finished) {
|
|
373
|
+
return;
|
|
374
|
+
}
|
|
375
|
+
finished = true;
|
|
376
|
+
clearTimeout(timer);
|
|
377
|
+
const elapsedMs = Date.now() - startedAt;
|
|
378
|
+
if (succeeded && elapsedMs >= this.slowRequestWarnMs) {
|
|
379
|
+
this.logger.warn(
|
|
380
|
+
`⏱️ OpenRouter ${kind} request completed after ${elapsedMs}ms: ` +
|
|
381
|
+
`model=${model}, agent=${agentId}`
|
|
382
|
+
);
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
};
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
/**
|
|
389
|
+
* Build, log, and return the error for a timed-out LLM request.
|
|
390
|
+
*
|
|
391
|
+
* The error is named 'TimeoutError' and flagged isRequestTimeout so
|
|
392
|
+
* classifyNetworkError maps it to the non-retryable REQUEST_TIMEOUT type:
|
|
393
|
+
* the caller sees the failure immediately instead of a silent retry loop.
|
|
394
|
+
* Logged at ERROR here — unconditionally — because this is the line that
|
|
395
|
+
* turns a production stall from invisible into diagnosable.
|
|
396
|
+
*/
|
|
397
|
+
private buildRequestTimeoutError(params: {
|
|
398
|
+
kind: 'completion' | 'streaming';
|
|
399
|
+
detail: string;
|
|
400
|
+
model: string;
|
|
401
|
+
agentId: string;
|
|
402
|
+
elapsedMs: number;
|
|
403
|
+
limitMs: number;
|
|
404
|
+
requestBytes: number;
|
|
405
|
+
messageCount: number;
|
|
406
|
+
}): Error {
|
|
407
|
+
const message =
|
|
408
|
+
`⛔ OpenRouter ${params.kind} request timed out (${params.detail}) after ${params.elapsedMs}ms ` +
|
|
409
|
+
`(limit ${params.limitMs}ms): model=${params.model}, agent=${params.agentId}, ` +
|
|
410
|
+
`request=${(params.requestBytes / 1024).toFixed(1)}KB, messages=${params.messageCount}`;
|
|
411
|
+
this.logger.error(message);
|
|
412
|
+
|
|
413
|
+
const error = new Error(message);
|
|
414
|
+
error.name = 'TimeoutError';
|
|
415
|
+
(error as any).isRequestTimeout = true;
|
|
416
|
+
return error;
|
|
417
|
+
}
|
|
294
418
|
|
|
295
419
|
/**
|
|
296
420
|
* Initialize the OpenRouter provider
|
|
297
421
|
*/
|
|
298
422
|
protected async initializeProvider(): Promise<void> {
|
|
423
|
+
// Per-request bounds. All three must be positive and finite — a missing or
|
|
424
|
+
// disabled bound is how a hung connection becomes permanent silence that
|
|
425
|
+
// only a consumer-side backstop can end.
|
|
426
|
+
//
|
|
427
|
+
// requestTimeoutMs defaults to 5 minutes: reasoning models legitimately run
|
|
428
|
+
// for minutes on large contexts, so this is a hang detector, not a latency
|
|
429
|
+
// budget. It is enforced twice: as an AbortSignal on the fetch itself and
|
|
430
|
+
// as an operation bound inside NetworkRecoveryManager.executeWithRetry.
|
|
431
|
+
this.requestTimeoutMs = parsePositiveIntEnv('OPENROUTER_REQUEST_TIMEOUT_MS', 300000);
|
|
432
|
+
this.streamIdleTimeoutMs = parsePositiveIntEnv('OPENROUTER_STREAM_IDLE_TIMEOUT_MS', 120000);
|
|
433
|
+
this.slowRequestWarnMs = parsePositiveIntEnv('OPENROUTER_SLOW_REQUEST_WARN_MS', 60000);
|
|
434
|
+
|
|
299
435
|
// Initialize network recovery configuration from environment or defaults
|
|
300
436
|
const networkRecoveryConfig: NetworkRecoveryConfig = {
|
|
301
437
|
...DEFAULT_NETWORK_RECOVERY_CONFIG,
|
|
@@ -305,7 +441,7 @@ export class OpenRouterMcpClient extends BaseMcpClient {
|
|
|
305
441
|
retryMultiplier: parseFloat(process.env.OPENROUTER_RETRY_MULTIPLIER || '2'),
|
|
306
442
|
circuitBreakerThreshold: parseInt(process.env.OPENROUTER_CIRCUIT_BREAKER_THRESHOLD || '5'),
|
|
307
443
|
circuitBreakerCooldownMs: parseInt(process.env.OPENROUTER_CIRCUIT_BREAKER_COOLDOWN_MS || '60000'),
|
|
308
|
-
requestTimeoutMs:
|
|
444
|
+
requestTimeoutMs: this.requestTimeoutMs,
|
|
309
445
|
enableGracefulDegradation: process.env.OPENROUTER_ENABLE_GRACEFUL_DEGRADATION !== 'false',
|
|
310
446
|
enableDetailedLogging: process.env.OPENROUTER_ENABLE_DETAILED_LOGGING !== 'false'
|
|
311
447
|
};
|
|
@@ -546,13 +682,18 @@ export class OpenRouterMcpClient extends BaseMcpClient {
|
|
|
546
682
|
}
|
|
547
683
|
|
|
548
684
|
// Messages are already in OpenRouter format - send directly
|
|
549
|
-
return await
|
|
685
|
+
return await this.queueRequest(async () => {
|
|
550
686
|
if (!this.networkRecovery) {
|
|
551
687
|
throw new Error('Network recovery not initialized');
|
|
552
688
|
}
|
|
553
689
|
|
|
554
690
|
const result = await this.networkRecovery.executeWithRetry(
|
|
555
|
-
() => this.executeOpenRouterRequestDirect(
|
|
691
|
+
() => this.executeOpenRouterRequestDirect(
|
|
692
|
+
transformedMessages,
|
|
693
|
+
context.availableTools as any,
|
|
694
|
+
// agentId rides along for the slow-request WARN and timeout logs
|
|
695
|
+
{ ...options, agentId: context.agentId }
|
|
696
|
+
),
|
|
556
697
|
extractStatusCodeFromError
|
|
557
698
|
);
|
|
558
699
|
|
|
@@ -572,6 +713,12 @@ export class OpenRouterMcpClient extends BaseMcpClient {
|
|
|
572
713
|
* Makes the same request but with `stream: true`, parses SSE chunks,
|
|
573
714
|
* calls onChunk for each partial token, and returns the accumulated final response.
|
|
574
715
|
*
|
|
716
|
+
* Deliberately NOT wrapped in networkRecovery.executeWithRetry: by the time a
|
|
717
|
+
* streaming request fails, chunks may already have been delivered to the
|
|
718
|
+
* consumer via onChunk, and a retry would replay them. Failures — including
|
|
719
|
+
* the idle-watchdog timeout inside executeStreamingRequest — propagate to the
|
|
720
|
+
* caller instead.
|
|
721
|
+
*
|
|
575
722
|
* @param context - Complete agent context from SDK
|
|
576
723
|
* @param options - Additional options (must include stream: true)
|
|
577
724
|
* @param onChunk - Callback invoked for each streaming chunk
|
|
@@ -587,8 +734,14 @@ export class OpenRouterMcpClient extends BaseMcpClient {
|
|
|
587
734
|
const converter = getMessageConverter('client');
|
|
588
735
|
const transformedMessages = converter.transform(openRouterMessages, MessageFormat.OPENROUTER);
|
|
589
736
|
|
|
590
|
-
return await
|
|
591
|
-
return this.executeStreamingRequest(
|
|
737
|
+
return await this.queueRequest(async () => {
|
|
738
|
+
return this.executeStreamingRequest(
|
|
739
|
+
transformedMessages,
|
|
740
|
+
context.availableTools as any,
|
|
741
|
+
// agentId rides along for the slow-request WARN and timeout logs
|
|
742
|
+
{ ...options, agentId: context.agentId },
|
|
743
|
+
onChunk
|
|
744
|
+
);
|
|
592
745
|
});
|
|
593
746
|
}
|
|
594
747
|
|
|
@@ -636,22 +789,92 @@ export class OpenRouterMcpClient extends BaseMcpClient {
|
|
|
636
789
|
|
|
637
790
|
const headers = this.buildOpenRouterHeaders(options);
|
|
638
791
|
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
792
|
+
// Serialize once so the logged request size is exactly what went on the wire
|
|
793
|
+
const requestBodyJson = JSON.stringify(requestBody);
|
|
794
|
+
const requestBytes = Buffer.byteLength(requestBodyJson, 'utf8');
|
|
795
|
+
const agentId = options?.agentId || 'unknown';
|
|
796
|
+
const requestStartedAt = Date.now();
|
|
797
|
+
const slowWatch = this.startSlowRequestWatch('streaming', model, agentId, requestBytes);
|
|
798
|
+
|
|
799
|
+
// Idle watchdog for the SSE stream. A healthy stream is never silent for
|
|
800
|
+
// long — OpenRouter emits keepalive comment lines every few seconds while a
|
|
801
|
+
// model is thinking — so silence past streamIdleTimeoutMs means the
|
|
802
|
+
// connection is dead, not that the model is slow. The watchdog is re-armed
|
|
803
|
+
// on every read; there is deliberately NO total-time cap here, because an
|
|
804
|
+
// actively producing stream is healthy no matter how long it runs.
|
|
805
|
+
//
|
|
806
|
+
// Each read (and the initial fetch) races against abortPromise as well as
|
|
807
|
+
// carrying the AbortController signal: the signal cancels the real network
|
|
808
|
+
// request, the race guarantees the await itself resolves even if the
|
|
809
|
+
// underlying stream implementation ignores the abort.
|
|
810
|
+
const controller = new AbortController();
|
|
811
|
+
let headersReceived = false;
|
|
812
|
+
let idleTimer: ReturnType<typeof setTimeout> | undefined;
|
|
813
|
+
const armIdleWatchdog = () => {
|
|
814
|
+
clearTimeout(idleTimer);
|
|
815
|
+
idleTimer = setTimeout(() => controller.abort(), this.streamIdleTimeoutMs);
|
|
816
|
+
};
|
|
817
|
+
// Plain sentinel rejection — enrichment and logging happen exactly once,
|
|
818
|
+
// in the catch blocks below, regardless of whether this promise or the
|
|
819
|
+
// fetch/read rejection wins the race.
|
|
820
|
+
const abortPromise = new Promise<never>((_, reject) => {
|
|
821
|
+
controller.signal.addEventListener('abort', () => {
|
|
822
|
+
const sentinel = new Error('OpenRouter streaming request aborted by idle watchdog');
|
|
823
|
+
sentinel.name = 'AbortError';
|
|
824
|
+
reject(sentinel);
|
|
825
|
+
}, { once: true });
|
|
643
826
|
});
|
|
644
827
|
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
(error as any)
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
828
|
+
// Converts an abort/timeout rejection into the logged, non-retryable
|
|
829
|
+
// request-timeout error; returns any other error unchanged.
|
|
830
|
+
const normalizeStreamError = (error: unknown): unknown => {
|
|
831
|
+
if ((error as any)?.isRequestTimeout || !isAbortOrTimeoutError(error)) {
|
|
832
|
+
return error;
|
|
833
|
+
}
|
|
834
|
+
return this.buildRequestTimeoutError({
|
|
835
|
+
kind: 'streaming',
|
|
836
|
+
detail: headersReceived
|
|
837
|
+
? `no SSE data for ${this.streamIdleTimeoutMs}ms`
|
|
838
|
+
: `no response headers within ${this.streamIdleTimeoutMs}ms`,
|
|
839
|
+
model,
|
|
840
|
+
agentId,
|
|
841
|
+
elapsedMs: Date.now() - requestStartedAt,
|
|
842
|
+
limitMs: this.streamIdleTimeoutMs,
|
|
843
|
+
requestBytes,
|
|
844
|
+
messageCount: openRouterMessages.length
|
|
845
|
+
});
|
|
846
|
+
};
|
|
847
|
+
|
|
848
|
+
armIdleWatchdog();
|
|
849
|
+
let response: Response;
|
|
850
|
+
try {
|
|
851
|
+
response = await Promise.race([
|
|
852
|
+
fetch(`${this.baseUrl}/chat/completions`, {
|
|
853
|
+
method: 'POST',
|
|
854
|
+
headers,
|
|
855
|
+
body: requestBodyJson,
|
|
856
|
+
signal: controller.signal
|
|
857
|
+
}),
|
|
858
|
+
abortPromise
|
|
859
|
+
]);
|
|
860
|
+
headersReceived = true;
|
|
861
|
+
armIdleWatchdog();
|
|
652
862
|
|
|
653
|
-
|
|
654
|
-
|
|
863
|
+
if (!response.ok) {
|
|
864
|
+
const errorText = await response.text();
|
|
865
|
+
const error = new Error(`OpenRouter API error [${response.status}]: ${errorText}`);
|
|
866
|
+
(error as any).status = response.status;
|
|
867
|
+
(error as any).statusCode = response.status;
|
|
868
|
+
throw error;
|
|
869
|
+
}
|
|
870
|
+
|
|
871
|
+
if (!response.body) {
|
|
872
|
+
throw new Error('No response body for streaming request');
|
|
873
|
+
}
|
|
874
|
+
} catch (error) {
|
|
875
|
+
clearTimeout(idleTimer);
|
|
876
|
+
slowWatch.finish(false);
|
|
877
|
+
throw normalizeStreamError(error);
|
|
655
878
|
}
|
|
656
879
|
|
|
657
880
|
this.logger.debug(`📡 OpenRouter SSE: Response received, status=${response.status}, starting stream parse`);
|
|
@@ -673,7 +896,10 @@ export class OpenRouterMcpClient extends BaseMcpClient {
|
|
|
673
896
|
|
|
674
897
|
try {
|
|
675
898
|
while (true) {
|
|
676
|
-
|
|
899
|
+
// Race against the idle watchdog: reader.read() on a dead
|
|
900
|
+
// connection can otherwise pend forever with nothing logged.
|
|
901
|
+
const { done, value } = await Promise.race([reader.read(), abortPromise]);
|
|
902
|
+
armIdleWatchdog();
|
|
677
903
|
if (done) break;
|
|
678
904
|
|
|
679
905
|
buffer += decoder.decode(value, { stream: true });
|
|
@@ -772,7 +998,16 @@ export class OpenRouterMcpClient extends BaseMcpClient {
|
|
|
772
998
|
}
|
|
773
999
|
}
|
|
774
1000
|
}
|
|
1001
|
+
} catch (error) {
|
|
1002
|
+
// Cancel the underlying stream so the connection is torn down; the
|
|
1003
|
+
// losing reader.read() from the race is settled by the cancel/abort
|
|
1004
|
+
// and its rejection is already observed by Promise.race.
|
|
1005
|
+
slowWatch.finish(false);
|
|
1006
|
+
reader.cancel().catch(() => undefined);
|
|
1007
|
+
throw normalizeStreamError(error);
|
|
775
1008
|
} finally {
|
|
1009
|
+
clearTimeout(idleTimer);
|
|
1010
|
+
slowWatch.finish();
|
|
776
1011
|
reader.releaseLock();
|
|
777
1012
|
}
|
|
778
1013
|
|
|
@@ -949,7 +1184,7 @@ export class OpenRouterMcpClient extends BaseMcpClient {
|
|
|
949
1184
|
|
|
950
1185
|
|
|
951
1186
|
// Send directly
|
|
952
|
-
return await
|
|
1187
|
+
return await this.queueRequest(async () => {
|
|
953
1188
|
if (!this.networkRecovery) {
|
|
954
1189
|
throw new Error('Network recovery not initialized');
|
|
955
1190
|
}
|
|
@@ -1081,52 +1316,80 @@ export class OpenRouterMcpClient extends BaseMcpClient {
|
|
|
1081
1316
|
// console.log(` Tools: ${requestBody.tools.map((t: any) => t.function.name).join(', ')}`);
|
|
1082
1317
|
}
|
|
1083
1318
|
|
|
1084
|
-
//
|
|
1085
|
-
const
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1319
|
+
// Serialize once so the logged request size is exactly what went on the wire
|
|
1320
|
+
const requestBodyJson = JSON.stringify(requestBody);
|
|
1321
|
+
const requestBytes = Buffer.byteLength(requestBodyJson, 'utf8');
|
|
1322
|
+
const agentId = options?.agentId || 'unknown';
|
|
1323
|
+
const requestStartedAt = Date.now();
|
|
1324
|
+
const slowWatch = this.startSlowRequestWatch('completion', model, agentId, requestBytes);
|
|
1325
|
+
|
|
1326
|
+
let responseText: string;
|
|
1327
|
+
try {
|
|
1328
|
+
// AbortSignal.timeout bounds the entire request — connect, headers,
|
|
1329
|
+
// and body read — so a hung connection surfaces as an error instead
|
|
1330
|
+
// of indefinite silence. Reasoning models can legitimately take
|
|
1331
|
+
// minutes; the default limit is sized for that (see initializeProvider).
|
|
1332
|
+
const response = await fetch(`${this.baseUrl}/chat/completions`, {
|
|
1333
|
+
method: 'POST',
|
|
1334
|
+
headers,
|
|
1335
|
+
body: requestBodyJson,
|
|
1336
|
+
signal: AbortSignal.timeout(this.requestTimeoutMs)
|
|
1337
|
+
});
|
|
1338
|
+
|
|
1339
|
+
// Check for errors with enhanced error information
|
|
1340
|
+
if (!response.ok) {
|
|
1341
|
+
let errorText = await response.text();
|
|
1342
|
+
this.logger.error(`🔧 DEBUG: Error response text: ${errorText}`);
|
|
1343
|
+
|
|
1344
|
+
let errorMessage = errorText;
|
|
1345
|
+
let rateLimitInfo: Record<string, any> = {};
|
|
1346
|
+
|
|
1347
|
+
try {
|
|
1348
|
+
const errorJson = JSON.parse(errorText);
|
|
1349
|
+
errorMessage = errorJson.error?.message || errorText;
|
|
1350
|
+
|
|
1351
|
+
// Extract rate limit information if available
|
|
1352
|
+
if (response.status === 429) {
|
|
1353
|
+
rateLimitInfo = {
|
|
1354
|
+
retryAfter: response.headers.get('retry-after'),
|
|
1355
|
+
rateLimitLimit: response.headers.get('x-ratelimit-limit'),
|
|
1356
|
+
rateLimitRemaining: response.headers.get('x-ratelimit-remaining'),
|
|
1357
|
+
rateLimitReset: response.headers.get('x-ratelimit-reset')
|
|
1358
|
+
};
|
|
1359
|
+
}
|
|
1360
|
+
} catch (e) {
|
|
1361
|
+
// Use error text as is if not JSON
|
|
1113
1362
|
}
|
|
1114
|
-
|
|
1115
|
-
//
|
|
1363
|
+
|
|
1364
|
+
// Create detailed error with status code
|
|
1365
|
+
const error = new Error(`OpenRouter API error [${response.status}]: ${errorMessage}`);
|
|
1366
|
+
(error as any).status = response.status;
|
|
1367
|
+
(error as any).statusCode = response.status;
|
|
1368
|
+
(error as any).rateLimitInfo = rateLimitInfo;
|
|
1369
|
+
|
|
1370
|
+
throw error;
|
|
1371
|
+
}
|
|
1372
|
+
|
|
1373
|
+
// Get response text for JSON parsing
|
|
1374
|
+
responseText = await response.text();
|
|
1375
|
+
slowWatch.finish();
|
|
1376
|
+
} catch (error) {
|
|
1377
|
+
slowWatch.finish(false);
|
|
1378
|
+
if (isAbortOrTimeoutError(error)) {
|
|
1379
|
+
throw this.buildRequestTimeoutError({
|
|
1380
|
+
kind: 'completion',
|
|
1381
|
+
detail: 'no response within the request timeout',
|
|
1382
|
+
model,
|
|
1383
|
+
agentId,
|
|
1384
|
+
elapsedMs: Date.now() - requestStartedAt,
|
|
1385
|
+
limitMs: this.requestTimeoutMs,
|
|
1386
|
+
requestBytes,
|
|
1387
|
+
messageCount: openRouterMessages.length
|
|
1388
|
+
});
|
|
1116
1389
|
}
|
|
1117
|
-
|
|
1118
|
-
// Create detailed error with status code
|
|
1119
|
-
const error = new Error(`OpenRouter API error [${response.status}]: ${errorMessage}`);
|
|
1120
|
-
(error as any).status = response.status;
|
|
1121
|
-
(error as any).statusCode = response.status;
|
|
1122
|
-
(error as any).rateLimitInfo = rateLimitInfo;
|
|
1123
|
-
|
|
1124
1390
|
throw error;
|
|
1125
1391
|
}
|
|
1126
1392
|
|
|
1127
|
-
// Get response text for JSON parsing
|
|
1128
|
-
const responseText = await response.text();
|
|
1129
|
-
|
|
1130
1393
|
// Check if response is empty
|
|
1131
1394
|
if (!responseText || responseText.length === 0) {
|
|
1132
1395
|
this.logger.error('🔧 ERROR: Empty response from OpenRouter API');
|
|
@@ -1150,6 +1413,11 @@ export class OpenRouterMcpClient extends BaseMcpClient {
|
|
|
1150
1413
|
|
|
1151
1414
|
return this.convertToMcpResponse(openRouterResponse);
|
|
1152
1415
|
} catch (error) {
|
|
1416
|
+
// Request timeouts are already logged with full context and must keep
|
|
1417
|
+
// their name/flags so NetworkRecovery classifies them as non-retryable.
|
|
1418
|
+
if ((error as any)?.isRequestTimeout === true) {
|
|
1419
|
+
throw error;
|
|
1420
|
+
}
|
|
1153
1421
|
this.logger.error(`🔧 ERROR in executeOpenRouterRequest: ${error instanceof Error ? error.message : String(error)}`);
|
|
1154
1422
|
if (error instanceof Error && error.stack) {
|
|
1155
1423
|
this.logger.error(`🔧 ERROR STACK: ${error.stack}`);
|
|
@@ -52,6 +52,13 @@ export class NetworkRecoveryManager {
|
|
|
52
52
|
private config: NetworkRecoveryConfig,
|
|
53
53
|
loggerContext: string
|
|
54
54
|
) {
|
|
55
|
+
// A non-finite or non-positive timeout would silently disable the bound
|
|
56
|
+
// that keeps hung requests from wedging callers — reject it up front.
|
|
57
|
+
if (!Number.isFinite(config.requestTimeoutMs) || config.requestTimeoutMs <= 0) {
|
|
58
|
+
throw new Error(
|
|
59
|
+
`NetworkRecoveryManager (${loggerContext}): requestTimeoutMs must be a positive number, got ${config.requestTimeoutMs}`
|
|
60
|
+
);
|
|
61
|
+
}
|
|
55
62
|
this.circuitBreakerStatus = {
|
|
56
63
|
state: CircuitBreakerState.CLOSED,
|
|
57
64
|
failureCount: 0,
|
|
@@ -60,6 +67,69 @@ export class NetworkRecoveryManager {
|
|
|
60
67
|
};
|
|
61
68
|
this.logger = new Logger('debug', loggerContext, 'client');
|
|
62
69
|
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Grace period between the operation's own timeout and this manager's net.
|
|
73
|
+
* The operation (e.g. a fetch carrying AbortSignal.timeout(requestTimeoutMs))
|
|
74
|
+
* is the primary enforcement and produces the richer error — model, elapsed,
|
|
75
|
+
* request size. The net here exists for operations that never implement their
|
|
76
|
+
* own abort, so it must fire strictly after the primary would have; firing at
|
|
77
|
+
* the same instant races it and masks the better diagnostics.
|
|
78
|
+
*/
|
|
79
|
+
private static readonly REQUEST_TIMEOUT_NET_GRACE_MS = 1000;
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* Bound an operation to the configured requestTimeoutMs (plus a short grace
|
|
83
|
+
* so the operation's own abort fires first — see REQUEST_TIMEOUT_NET_GRACE_MS).
|
|
84
|
+
*
|
|
85
|
+
* A request that never settles is invisible to the retry loop — before this
|
|
86
|
+
* existed, a hung fetch held executeWithRetry (and anything queued behind it)
|
|
87
|
+
* open forever with nothing logged. The returned promise rejects with an error
|
|
88
|
+
* named 'TimeoutError' (isRequestTimeout = true), which classifyNetworkError
|
|
89
|
+
* maps to the non-retryable REQUEST_TIMEOUT type, so the failure surfaces to
|
|
90
|
+
* the caller immediately instead of entering the retry loop.
|
|
91
|
+
*/
|
|
92
|
+
private withRequestTimeout<T>(operation: () => Promise<T>): Promise<T> {
|
|
93
|
+
const timeoutMs = this.config.requestTimeoutMs;
|
|
94
|
+
const startedAt = Date.now();
|
|
95
|
+
|
|
96
|
+
return new Promise<T>((resolve, reject) => {
|
|
97
|
+
const timer = setTimeout(() => {
|
|
98
|
+
const elapsedMs = Date.now() - startedAt;
|
|
99
|
+
const error = new Error(
|
|
100
|
+
`Request exceeded the ${timeoutMs}ms request timeout without aborting on its own ` +
|
|
101
|
+
`(elapsed: ${elapsedMs}ms) and was abandoned`
|
|
102
|
+
);
|
|
103
|
+
error.name = 'TimeoutError';
|
|
104
|
+
(error as any).isRequestTimeout = true;
|
|
105
|
+
reject(error);
|
|
106
|
+
}, timeoutMs + NetworkRecoveryManager.REQUEST_TIMEOUT_NET_GRACE_MS);
|
|
107
|
+
|
|
108
|
+
let operationPromise: Promise<T>;
|
|
109
|
+
try {
|
|
110
|
+
operationPromise = operation();
|
|
111
|
+
} catch (error) {
|
|
112
|
+
clearTimeout(timer);
|
|
113
|
+
reject(error);
|
|
114
|
+
return;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
// The two-argument then() handles settlement in every ordering: if the
|
|
118
|
+
// operation settles after the timeout has already rejected this promise,
|
|
119
|
+
// the late resolve/reject is a no-op and the rejection is still observed
|
|
120
|
+
// here, so it cannot become an unhandled rejection.
|
|
121
|
+
operationPromise.then(
|
|
122
|
+
result => {
|
|
123
|
+
clearTimeout(timer);
|
|
124
|
+
resolve(result);
|
|
125
|
+
},
|
|
126
|
+
error => {
|
|
127
|
+
clearTimeout(timer);
|
|
128
|
+
reject(error);
|
|
129
|
+
}
|
|
130
|
+
);
|
|
131
|
+
});
|
|
132
|
+
}
|
|
63
133
|
|
|
64
134
|
/**
|
|
65
135
|
* Check if circuit breaker allows the request
|
|
@@ -155,9 +225,10 @@ export class NetworkRecoveryManager {
|
|
|
155
225
|
}
|
|
156
226
|
|
|
157
227
|
try {
|
|
158
|
-
// Execute the operation
|
|
159
|
-
|
|
160
|
-
|
|
228
|
+
// Execute the operation, bounded by requestTimeoutMs — a request
|
|
229
|
+
// that never settles must surface as an error, not silence.
|
|
230
|
+
const result = await this.withRequestTimeout(operation);
|
|
231
|
+
|
|
161
232
|
// Success! Record it and return
|
|
162
233
|
this.recordSuccess();
|
|
163
234
|
|