@mxf-dev/core 2.0.1 → 2.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/protocols/mcp/providers/OpenRouterMcpClient.d.ts +34 -4
- package/dist/protocols/mcp/providers/OpenRouterMcpClient.d.ts.map +1 -1
- package/dist/protocols/mcp/providers/OpenRouterMcpClient.js +297 -65
- package/dist/protocols/mcp/providers/OpenRouterMcpClient.js.map +1 -1
- package/dist/protocols/mcp/utils/NetworkRecovery.d.ts +21 -0
- package/dist/protocols/mcp/utils/NetworkRecovery.d.ts.map +1 -1
- package/dist/protocols/mcp/utils/NetworkRecovery.js +62 -2
- package/dist/protocols/mcp/utils/NetworkRecovery.js.map +1 -1
- package/dist/services/MemoryService.d.ts +17 -1
- package/dist/services/MemoryService.d.ts.map +1 -1
- package/dist/services/MemoryService.js +33 -13
- package/dist/services/MemoryService.js.map +1 -1
- package/dist/types/NetworkRecoveryTypes.d.ts +1 -0
- package/dist/types/NetworkRecoveryTypes.d.ts.map +1 -1
- package/dist/types/NetworkRecoveryTypes.js +14 -0
- package/dist/types/NetworkRecoveryTypes.js.map +1 -1
- package/package.json +1 -1
- package/src/protocols/mcp/providers/OpenRouterMcpClient.ts +351 -83
- package/src/protocols/mcp/utils/NetworkRecovery.ts +74 -3
- package/src/services/MemoryService.ts +40 -16
- package/src/types/NetworkRecoveryTypes.ts +17 -1
|
@@ -76,19 +76,43 @@ export declare class OpenRouterMcpClient extends BaseMcpClient {
|
|
|
76
76
|
* - absent/no enabled flag → send nothing and let the provider default apply.
|
|
77
77
|
*/
|
|
78
78
|
private applyReasoningParam;
|
|
79
|
-
private
|
|
80
|
-
private
|
|
79
|
+
private requestQueue;
|
|
80
|
+
private isProcessingQueue;
|
|
81
81
|
private static readonly REQUEST_DELAY_MS;
|
|
82
82
|
private networkRecovery;
|
|
83
83
|
private jsonRecovery;
|
|
84
|
+
private requestTimeoutMs;
|
|
85
|
+
private streamIdleTimeoutMs;
|
|
86
|
+
private slowRequestWarnMs;
|
|
84
87
|
/**
|
|
85
88
|
* Process the request queue sequentially to prevent concurrent requests
|
|
86
89
|
*/
|
|
87
|
-
private
|
|
90
|
+
private processQueue;
|
|
88
91
|
/**
|
|
89
92
|
* Add a request to the queue and process it
|
|
90
93
|
*/
|
|
91
|
-
private
|
|
94
|
+
private queueRequest;
|
|
95
|
+
/**
|
|
96
|
+
* Start the slow-request watchdog for one LLM request.
|
|
97
|
+
*
|
|
98
|
+
* Emits a WARN once the request has been in flight for slowRequestWarnMs, and
|
|
99
|
+
* another WARN at completion if the total time crossed the threshold. Together
|
|
100
|
+
* with the timeout ERROR this makes slow-vs-hung distinguishable in production
|
|
101
|
+
* logs: a slow request logs WARN…WARN(completed), a hung one WARN…ERROR(timeout).
|
|
102
|
+
*
|
|
103
|
+
* Returns a finish() that must be called exactly once when the request settles.
|
|
104
|
+
*/
|
|
105
|
+
private startSlowRequestWatch;
|
|
106
|
+
/**
|
|
107
|
+
* Build, log, and return the error for a timed-out LLM request.
|
|
108
|
+
*
|
|
109
|
+
* The error is named 'TimeoutError' and flagged isRequestTimeout so
|
|
110
|
+
* classifyNetworkError maps it to the non-retryable REQUEST_TIMEOUT type:
|
|
111
|
+
* the caller sees the failure immediately instead of a silent retry loop.
|
|
112
|
+
* Logged at ERROR here — unconditionally — because this is the line that
|
|
113
|
+
* turns a production stall from invisible into diagnosable.
|
|
114
|
+
*/
|
|
115
|
+
private buildRequestTimeoutError;
|
|
92
116
|
/**
|
|
93
117
|
* Initialize the OpenRouter provider
|
|
94
118
|
*/
|
|
@@ -138,6 +162,12 @@ export declare class OpenRouterMcpClient extends BaseMcpClient {
|
|
|
138
162
|
* Makes the same request but with `stream: true`, parses SSE chunks,
|
|
139
163
|
* calls onChunk for each partial token, and returns the accumulated final response.
|
|
140
164
|
*
|
|
165
|
+
* Deliberately NOT wrapped in networkRecovery.executeWithRetry: by the time a
|
|
166
|
+
* streaming request fails, chunks may already have been delivered to the
|
|
167
|
+
* consumer via onChunk, and a retry would replay them. Failures — including
|
|
168
|
+
* the idle-watchdog timeout inside executeStreamingRequest — propagate to the
|
|
169
|
+
* caller instead.
|
|
170
|
+
*
|
|
141
171
|
* @param context - Complete agent context from SDK
|
|
142
172
|
* @param options - Additional options (must include stream: true)
|
|
143
173
|
* @param onChunk - Callback invoked for each streaming chunk
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"OpenRouterMcpClient.d.ts","sourceRoot":"","sources":["../../../../src/protocols/mcp/providers/OpenRouterMcpClient.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH;;;;;;;GAOG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,MAAM,CAAC;AAElC,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EACH,UAAU,EACV,OAAO,EACP,cAAc,EAOjB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,YAAY,EAAE,MAAM,qCAAqC,CAAC;
|
|
1
|
+
{"version":3,"file":"OpenRouterMcpClient.d.ts","sourceRoot":"","sources":["../../../../src/protocols/mcp/providers/OpenRouterMcpClient.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH;;;;;;;GAOG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,MAAM,CAAC;AAElC,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EACH,UAAU,EACV,OAAO,EACP,cAAc,EAOjB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,YAAY,EAAE,MAAM,qCAAqC,CAAC;AAqGnE;;GAEG;AACH,qBAAa,mBAAoB,SAAQ,aAAa;IAElD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAkC;IAG1D,OAAO,CAAC,MAAM,CAAwD;IAItE,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAuB,CAM7C;;IAOF;;;OAGG;IACH,OAAO,CAAC,MAAM,CAAC,iBAAiB;IAMhC;;;;;;OAMG;IACH,OAAO,CAAC,kBAAkB;IA6B1B;;;;;;;;;;OAUG;IACH,OAAO,CAAC,sBAAsB;IAS9B;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,mBAAmB;IAqB3B,OAAO,CAAC,YAAY,CAAiC;IACrD,OAAO,CAAC,iBAAiB,CAAS;IAGlC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,gBAAgB,CAAwE;IAGhH,OAAO,CAAC,eAAe,CAAuC;IAG9D,OAAO,CAAC,YAAY,CAAsB;IAK1C,OAAO,CAAC,gBAAgB,CAAU;IAKlC,OAAO,CAAC,mBAAmB,CAAU;IAIrC,OAAO,CAAC,iBAAiB,CAAS;IAElC;;OAEG;YACW,YAAY;IAwB1B;;OAEG;YACW,YAAY;IAgB1B;;;;;;;;;OASG;IACH,OAAO,CAAC,qBAAqB;IAoC7B;;;;;;;;OAQG;IACH,OAAO,CAAC,wBAAwB;IAsBhC;;OAEG;cACa,kBAAkB,IAAI,OAAO,CAAC,IAAI,CAAC;IAsCnD;;;OAGG;IACH,OAAO,CAAC,+BAA+B;IA8BvC;;;;;OAKG;IACH,OAAO,CAAC,wBAAwB;IAIhC;;;;;OAKG;IACH,OAAO,CAAC,oBAAoB;IAkG5B;;;;;;;;;;;;;;;OAeG;IACI,eAAe,CAClB,OAAO,EAAE,YAAY,EACrB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAC9B,UAAU,CAAC,cAAc,CAAC;IAiC7B;;OAEG;YACW,mBAAmB;IAgDjC;;;;;;;;;;;;;;;OAeG;YACW,wBAAwB;IAqBtC;;;;OAIG;YACW,uBAAuB;IAiTrC;;;;OAIG;IACH,OAAO,CAAC,4BAA4B;IAkGpC;;;;;;;OAOG;cACa,mBAAmB,CAC/B,QAAQ,EAAE,UAAU,EAAE,EACtB,KAAK,CAAC,EAAE,OAAO,EAAE,EACjB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAC9B,OAAO,CAAC,cAAc,CAAC;IAmC1B;;;OAGG;YACW,8BAA8B;IAqB5C;;OAEG;YACW,4BAA4B;CAgM7C"}
|
|
@@ -36,6 +36,32 @@ import { convertToolsToProviderFormat } from '../utils/ToolHandlers.js';
|
|
|
36
36
|
import { getMessageConverter } from '../converters/UnifiedMessageConverter.js';
|
|
37
37
|
import { MessageFormat } from '../converters/IFormatConverter.js';
|
|
38
38
|
import { OpenRouterMessageAdapter } from '../converters/adapters/OpenRouterMessageAdapter.js';
|
|
39
|
+
/**
|
|
40
|
+
* Read a positive integer from the environment, failing fast on garbage.
|
|
41
|
+
* These values bound how long a hung request can stay silent — a NaN or zero
|
|
42
|
+
* from a typo'd env var must not silently disable that bound.
|
|
43
|
+
*/
|
|
44
|
+
const parsePositiveIntEnv = (name, defaultValue) => {
|
|
45
|
+
const raw = process.env[name];
|
|
46
|
+
if (raw === undefined || raw === '') {
|
|
47
|
+
return defaultValue;
|
|
48
|
+
}
|
|
49
|
+
const parsed = parseInt(raw, 10);
|
|
50
|
+
if (!Number.isFinite(parsed) || parsed <= 0) {
|
|
51
|
+
throw new Error(`${name} must be a positive integer, got "${raw}"`);
|
|
52
|
+
}
|
|
53
|
+
return parsed;
|
|
54
|
+
};
|
|
55
|
+
/**
|
|
56
|
+
* True when an error came from an aborted fetch. AbortSignal.timeout() rejects
|
|
57
|
+
* with a DOMException named 'TimeoutError' (Node and Bun); a manual
|
|
58
|
+
* controller.abort() rejects with 'AbortError'. This client owns every signal it
|
|
59
|
+
* passes to fetch, so either name means our own timeout fired.
|
|
60
|
+
*/
|
|
61
|
+
const isAbortOrTimeoutError = (error) => {
|
|
62
|
+
const name = error?.name;
|
|
63
|
+
return name === 'TimeoutError' || name === 'AbortError';
|
|
64
|
+
};
|
|
39
65
|
/**
|
|
40
66
|
* OpenRouter implementation of the MCP client with network recovery
|
|
41
67
|
*/
|
|
@@ -142,9 +168,14 @@ export class OpenRouterMcpClient extends BaseMcpClient {
|
|
|
142
168
|
requestBody.reasoning = { enabled: false };
|
|
143
169
|
}
|
|
144
170
|
}
|
|
145
|
-
//
|
|
146
|
-
static
|
|
147
|
-
|
|
171
|
+
// Per-instance request queue: keeps one client's requests ordered and spaced.
|
|
172
|
+
// This used to be static (class-level), which serialized every request from
|
|
173
|
+
// every client instance in the process through one queue — one hung request
|
|
174
|
+
// starved every agent's LLM calls, not just its own. Instance scoping plus the
|
|
175
|
+
// per-request timeout below bounds the damage a single request can do to the
|
|
176
|
+
// agent that issued it.
|
|
177
|
+
requestQueue = [];
|
|
178
|
+
isProcessingQueue = false;
|
|
148
179
|
// Configurable delay between requests - reduced from 500ms to 100ms default for better performance
|
|
149
180
|
// Set OPENROUTER_REQUEST_QUEUE_DELAY_MS=0 to disable queueing delay entirely
|
|
150
181
|
static REQUEST_DELAY_MS = parseInt(process.env.OPENROUTER_REQUEST_QUEUE_DELAY_MS || '100', 10);
|
|
@@ -152,16 +183,27 @@ export class OpenRouterMcpClient extends BaseMcpClient {
|
|
|
152
183
|
networkRecovery = null;
|
|
153
184
|
// JSON recovery manager
|
|
154
185
|
jsonRecovery;
|
|
186
|
+
// Hard cap on a single completion request (fetch + body). Generous because
|
|
187
|
+
// reasoning models legitimately run for minutes; finite because a request
|
|
188
|
+
// with no bound turns a hung connection into permanent silence.
|
|
189
|
+
requestTimeoutMs = 300000;
|
|
190
|
+
// Max silence between SSE chunks on the streaming path. OpenRouter emits
|
|
191
|
+
// keepalive comment lines every few seconds while a model is thinking, so a
|
|
192
|
+
// long quiet gap means a dead connection, not a slow model.
|
|
193
|
+
streamIdleTimeoutMs = 120000;
|
|
194
|
+
// Threshold for the slow-request WARN that makes slow-vs-hung visible in
|
|
195
|
+
// production logs before any timeout fires.
|
|
196
|
+
slowRequestWarnMs = 60000;
|
|
155
197
|
/**
|
|
156
198
|
* Process the request queue sequentially to prevent concurrent requests
|
|
157
199
|
*/
|
|
158
|
-
|
|
159
|
-
if (
|
|
200
|
+
async processQueue() {
|
|
201
|
+
if (this.isProcessingQueue || this.requestQueue.length === 0) {
|
|
160
202
|
return;
|
|
161
203
|
}
|
|
162
|
-
|
|
163
|
-
while (
|
|
164
|
-
const request =
|
|
204
|
+
this.isProcessingQueue = true;
|
|
205
|
+
while (this.requestQueue.length > 0) {
|
|
206
|
+
const request = this.requestQueue.shift();
|
|
165
207
|
try {
|
|
166
208
|
await request();
|
|
167
209
|
}
|
|
@@ -169,16 +211,16 @@ export class OpenRouterMcpClient extends BaseMcpClient {
|
|
|
169
211
|
// Request will handle its own error, just continue processing
|
|
170
212
|
}
|
|
171
213
|
// Wait between requests to prevent rate limiting
|
|
172
|
-
if (
|
|
214
|
+
if (this.requestQueue.length > 0) {
|
|
173
215
|
await new Promise(resolve => setTimeout(resolve, OpenRouterMcpClient.REQUEST_DELAY_MS));
|
|
174
216
|
}
|
|
175
217
|
}
|
|
176
|
-
|
|
218
|
+
this.isProcessingQueue = false;
|
|
177
219
|
}
|
|
178
220
|
/**
|
|
179
221
|
* Add a request to the queue and process it
|
|
180
222
|
*/
|
|
181
|
-
|
|
223
|
+
async queueRequest(requestFn) {
|
|
182
224
|
return new Promise((resolve, reject) => {
|
|
183
225
|
const wrappedRequest = async () => {
|
|
184
226
|
try {
|
|
@@ -189,14 +231,79 @@ export class OpenRouterMcpClient extends BaseMcpClient {
|
|
|
189
231
|
reject(error);
|
|
190
232
|
}
|
|
191
233
|
};
|
|
192
|
-
|
|
193
|
-
|
|
234
|
+
this.requestQueue.push(wrappedRequest);
|
|
235
|
+
this.processQueue();
|
|
194
236
|
});
|
|
195
237
|
}
|
|
238
|
+
/**
|
|
239
|
+
* Start the slow-request watchdog for one LLM request.
|
|
240
|
+
*
|
|
241
|
+
* Emits a WARN once the request has been in flight for slowRequestWarnMs, and
|
|
242
|
+
* another WARN at completion if the total time crossed the threshold. Together
|
|
243
|
+
* with the timeout ERROR this makes slow-vs-hung distinguishable in production
|
|
244
|
+
* logs: a slow request logs WARN…WARN(completed), a hung one WARN…ERROR(timeout).
|
|
245
|
+
*
|
|
246
|
+
* Returns a finish() that must be called exactly once when the request settles.
|
|
247
|
+
*/
|
|
248
|
+
startSlowRequestWatch(kind, model, agentId, requestBytes) {
|
|
249
|
+
const startedAt = Date.now();
|
|
250
|
+
let finished = false;
|
|
251
|
+
const timer = setTimeout(() => {
|
|
252
|
+
this.logger.warn(`⏱️ OpenRouter ${kind} request still in flight after ${this.slowRequestWarnMs}ms: ` +
|
|
253
|
+
`model=${model}, agent=${agentId}, request=${(requestBytes / 1024).toFixed(1)}KB`);
|
|
254
|
+
}, this.slowRequestWarnMs);
|
|
255
|
+
return {
|
|
256
|
+
// Idempotent. The completed-WARN is success-only: failures carry their
|
|
257
|
+
// elapsed time in their own ERROR log, and a "completed" line for a
|
|
258
|
+
// request that timed out would be a lie.
|
|
259
|
+
finish: (succeeded = true) => {
|
|
260
|
+
if (finished) {
|
|
261
|
+
return;
|
|
262
|
+
}
|
|
263
|
+
finished = true;
|
|
264
|
+
clearTimeout(timer);
|
|
265
|
+
const elapsedMs = Date.now() - startedAt;
|
|
266
|
+
if (succeeded && elapsedMs >= this.slowRequestWarnMs) {
|
|
267
|
+
this.logger.warn(`⏱️ OpenRouter ${kind} request completed after ${elapsedMs}ms: ` +
|
|
268
|
+
`model=${model}, agent=${agentId}`);
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
};
|
|
272
|
+
}
|
|
273
|
+
/**
|
|
274
|
+
* Build, log, and return the error for a timed-out LLM request.
|
|
275
|
+
*
|
|
276
|
+
* The error is named 'TimeoutError' and flagged isRequestTimeout so
|
|
277
|
+
* classifyNetworkError maps it to the non-retryable REQUEST_TIMEOUT type:
|
|
278
|
+
* the caller sees the failure immediately instead of a silent retry loop.
|
|
279
|
+
* Logged at ERROR here — unconditionally — because this is the line that
|
|
280
|
+
* turns a production stall from invisible into diagnosable.
|
|
281
|
+
*/
|
|
282
|
+
buildRequestTimeoutError(params) {
|
|
283
|
+
const message = `⛔ OpenRouter ${params.kind} request timed out (${params.detail}) after ${params.elapsedMs}ms ` +
|
|
284
|
+
`(limit ${params.limitMs}ms): model=${params.model}, agent=${params.agentId}, ` +
|
|
285
|
+
`request=${(params.requestBytes / 1024).toFixed(1)}KB, messages=${params.messageCount}`;
|
|
286
|
+
this.logger.error(message);
|
|
287
|
+
const error = new Error(message);
|
|
288
|
+
error.name = 'TimeoutError';
|
|
289
|
+
error.isRequestTimeout = true;
|
|
290
|
+
return error;
|
|
291
|
+
}
|
|
196
292
|
/**
|
|
197
293
|
* Initialize the OpenRouter provider
|
|
198
294
|
*/
|
|
199
295
|
async initializeProvider() {
|
|
296
|
+
// Per-request bounds. All three must be positive and finite — a missing or
|
|
297
|
+
// disabled bound is how a hung connection becomes permanent silence that
|
|
298
|
+
// only a consumer-side backstop can end.
|
|
299
|
+
//
|
|
300
|
+
// requestTimeoutMs defaults to 5 minutes: reasoning models legitimately run
|
|
301
|
+
// for minutes on large contexts, so this is a hang detector, not a latency
|
|
302
|
+
// budget. It is enforced twice: as an AbortSignal on the fetch itself and
|
|
303
|
+
// as an operation bound inside NetworkRecoveryManager.executeWithRetry.
|
|
304
|
+
this.requestTimeoutMs = parsePositiveIntEnv('OPENROUTER_REQUEST_TIMEOUT_MS', 300000);
|
|
305
|
+
this.streamIdleTimeoutMs = parsePositiveIntEnv('OPENROUTER_STREAM_IDLE_TIMEOUT_MS', 120000);
|
|
306
|
+
this.slowRequestWarnMs = parsePositiveIntEnv('OPENROUTER_SLOW_REQUEST_WARN_MS', 60000);
|
|
200
307
|
// Initialize network recovery configuration from environment or defaults
|
|
201
308
|
const networkRecoveryConfig = {
|
|
202
309
|
...DEFAULT_NETWORK_RECOVERY_CONFIG,
|
|
@@ -206,7 +313,7 @@ export class OpenRouterMcpClient extends BaseMcpClient {
|
|
|
206
313
|
retryMultiplier: parseFloat(process.env.OPENROUTER_RETRY_MULTIPLIER || '2'),
|
|
207
314
|
circuitBreakerThreshold: parseInt(process.env.OPENROUTER_CIRCUIT_BREAKER_THRESHOLD || '5'),
|
|
208
315
|
circuitBreakerCooldownMs: parseInt(process.env.OPENROUTER_CIRCUIT_BREAKER_COOLDOWN_MS || '60000'),
|
|
209
|
-
requestTimeoutMs:
|
|
316
|
+
requestTimeoutMs: this.requestTimeoutMs,
|
|
210
317
|
enableGracefulDegradation: process.env.OPENROUTER_ENABLE_GRACEFUL_DEGRADATION !== 'false',
|
|
211
318
|
enableDetailedLogging: process.env.OPENROUTER_ENABLE_DETAILED_LOGGING !== 'false'
|
|
212
319
|
};
|
|
@@ -414,11 +521,13 @@ export class OpenRouterMcpClient extends BaseMcpClient {
|
|
|
414
521
|
this.logger.debug(`Messages adjusted during transformation: ${openRouterMessages.length} → ${transformedMessages.length} (orphaned tool results dropped)`);
|
|
415
522
|
}
|
|
416
523
|
// Messages are already in OpenRouter format - send directly
|
|
417
|
-
return await
|
|
524
|
+
return await this.queueRequest(async () => {
|
|
418
525
|
if (!this.networkRecovery) {
|
|
419
526
|
throw new Error('Network recovery not initialized');
|
|
420
527
|
}
|
|
421
|
-
const result = await this.networkRecovery.executeWithRetry(() => this.executeOpenRouterRequestDirect(transformedMessages, context.availableTools,
|
|
528
|
+
const result = await this.networkRecovery.executeWithRetry(() => this.executeOpenRouterRequestDirect(transformedMessages, context.availableTools,
|
|
529
|
+
// agentId rides along for the slow-request WARN and timeout logs
|
|
530
|
+
{ ...options, agentId: context.agentId }), extractStatusCodeFromError);
|
|
422
531
|
if (!result.success) {
|
|
423
532
|
if (result.circuitBreakerTriggered) {
|
|
424
533
|
throw new Error(result.error.message);
|
|
@@ -433,6 +542,12 @@ export class OpenRouterMcpClient extends BaseMcpClient {
|
|
|
433
542
|
* Makes the same request but with `stream: true`, parses SSE chunks,
|
|
434
543
|
* calls onChunk for each partial token, and returns the accumulated final response.
|
|
435
544
|
*
|
|
545
|
+
* Deliberately NOT wrapped in networkRecovery.executeWithRetry: by the time a
|
|
546
|
+
* streaming request fails, chunks may already have been delivered to the
|
|
547
|
+
* consumer via onChunk, and a retry would replay them. Failures — including
|
|
548
|
+
* the idle-watchdog timeout inside executeStreamingRequest — propagate to the
|
|
549
|
+
* caller instead.
|
|
550
|
+
*
|
|
436
551
|
* @param context - Complete agent context from SDK
|
|
437
552
|
* @param options - Additional options (must include stream: true)
|
|
438
553
|
* @param onChunk - Callback invoked for each streaming chunk
|
|
@@ -443,8 +558,10 @@ export class OpenRouterMcpClient extends BaseMcpClient {
|
|
|
443
558
|
const openRouterMessages = this.structureMessagesFromContext(context);
|
|
444
559
|
const converter = getMessageConverter('client');
|
|
445
560
|
const transformedMessages = converter.transform(openRouterMessages, MessageFormat.OPENROUTER);
|
|
446
|
-
return await
|
|
447
|
-
return this.executeStreamingRequest(transformedMessages, context.availableTools,
|
|
561
|
+
return await this.queueRequest(async () => {
|
|
562
|
+
return this.executeStreamingRequest(transformedMessages, context.availableTools,
|
|
563
|
+
// agentId rides along for the slow-request WARN and timeout logs
|
|
564
|
+
{ ...options, agentId: context.agentId }, onChunk);
|
|
448
565
|
});
|
|
449
566
|
}
|
|
450
567
|
/**
|
|
@@ -479,20 +596,88 @@ export class OpenRouterMcpClient extends BaseMcpClient {
|
|
|
479
596
|
Object.assign(requestBody, options.providerOptions);
|
|
480
597
|
}
|
|
481
598
|
const headers = this.buildOpenRouterHeaders(options);
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
599
|
+
// Serialize once so the logged request size is exactly what went on the wire
|
|
600
|
+
const requestBodyJson = JSON.stringify(requestBody);
|
|
601
|
+
const requestBytes = Buffer.byteLength(requestBodyJson, 'utf8');
|
|
602
|
+
const agentId = options?.agentId || 'unknown';
|
|
603
|
+
const requestStartedAt = Date.now();
|
|
604
|
+
const slowWatch = this.startSlowRequestWatch('streaming', model, agentId, requestBytes);
|
|
605
|
+
// Idle watchdog for the SSE stream. A healthy stream is never silent for
|
|
606
|
+
// long — OpenRouter emits keepalive comment lines every few seconds while a
|
|
607
|
+
// model is thinking — so silence past streamIdleTimeoutMs means the
|
|
608
|
+
// connection is dead, not that the model is slow. The watchdog is re-armed
|
|
609
|
+
// on every read; there is deliberately NO total-time cap here, because an
|
|
610
|
+
// actively producing stream is healthy no matter how long it runs.
|
|
611
|
+
//
|
|
612
|
+
// Each read (and the initial fetch) races against abortPromise as well as
|
|
613
|
+
// carrying the AbortController signal: the signal cancels the real network
|
|
614
|
+
// request, the race guarantees the await itself resolves even if the
|
|
615
|
+
// underlying stream implementation ignores the abort.
|
|
616
|
+
const controller = new AbortController();
|
|
617
|
+
let headersReceived = false;
|
|
618
|
+
let idleTimer;
|
|
619
|
+
const armIdleWatchdog = () => {
|
|
620
|
+
clearTimeout(idleTimer);
|
|
621
|
+
idleTimer = setTimeout(() => controller.abort(), this.streamIdleTimeoutMs);
|
|
622
|
+
};
|
|
623
|
+
// Plain sentinel rejection — enrichment and logging happen exactly once,
|
|
624
|
+
// in the catch blocks below, regardless of whether this promise or the
|
|
625
|
+
// fetch/read rejection wins the race.
|
|
626
|
+
const abortPromise = new Promise((_, reject) => {
|
|
627
|
+
controller.signal.addEventListener('abort', () => {
|
|
628
|
+
const sentinel = new Error('OpenRouter streaming request aborted by idle watchdog');
|
|
629
|
+
sentinel.name = 'AbortError';
|
|
630
|
+
reject(sentinel);
|
|
631
|
+
}, { once: true });
|
|
486
632
|
});
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
error
|
|
491
|
-
|
|
492
|
-
|
|
633
|
+
// Converts an abort/timeout rejection into the logged, non-retryable
|
|
634
|
+
// request-timeout error; returns any other error unchanged.
|
|
635
|
+
const normalizeStreamError = (error) => {
|
|
636
|
+
if (error?.isRequestTimeout || !isAbortOrTimeoutError(error)) {
|
|
637
|
+
return error;
|
|
638
|
+
}
|
|
639
|
+
return this.buildRequestTimeoutError({
|
|
640
|
+
kind: 'streaming',
|
|
641
|
+
detail: headersReceived
|
|
642
|
+
? `no SSE data for ${this.streamIdleTimeoutMs}ms`
|
|
643
|
+
: `no response headers within ${this.streamIdleTimeoutMs}ms`,
|
|
644
|
+
model,
|
|
645
|
+
agentId,
|
|
646
|
+
elapsedMs: Date.now() - requestStartedAt,
|
|
647
|
+
limitMs: this.streamIdleTimeoutMs,
|
|
648
|
+
requestBytes,
|
|
649
|
+
messageCount: openRouterMessages.length
|
|
650
|
+
});
|
|
651
|
+
};
|
|
652
|
+
armIdleWatchdog();
|
|
653
|
+
let response;
|
|
654
|
+
try {
|
|
655
|
+
response = await Promise.race([
|
|
656
|
+
fetch(`${this.baseUrl}/chat/completions`, {
|
|
657
|
+
method: 'POST',
|
|
658
|
+
headers,
|
|
659
|
+
body: requestBodyJson,
|
|
660
|
+
signal: controller.signal
|
|
661
|
+
}),
|
|
662
|
+
abortPromise
|
|
663
|
+
]);
|
|
664
|
+
headersReceived = true;
|
|
665
|
+
armIdleWatchdog();
|
|
666
|
+
if (!response.ok) {
|
|
667
|
+
const errorText = await response.text();
|
|
668
|
+
const error = new Error(`OpenRouter API error [${response.status}]: ${errorText}`);
|
|
669
|
+
error.status = response.status;
|
|
670
|
+
error.statusCode = response.status;
|
|
671
|
+
throw error;
|
|
672
|
+
}
|
|
673
|
+
if (!response.body) {
|
|
674
|
+
throw new Error('No response body for streaming request');
|
|
675
|
+
}
|
|
493
676
|
}
|
|
494
|
-
|
|
495
|
-
|
|
677
|
+
catch (error) {
|
|
678
|
+
clearTimeout(idleTimer);
|
|
679
|
+
slowWatch.finish(false);
|
|
680
|
+
throw normalizeStreamError(error);
|
|
496
681
|
}
|
|
497
682
|
this.logger.debug(`📡 OpenRouter SSE: Response received, status=${response.status}, starting stream parse`);
|
|
498
683
|
// Parse SSE stream and accumulate the full response
|
|
@@ -511,7 +696,10 @@ export class OpenRouterMcpClient extends BaseMcpClient {
|
|
|
511
696
|
const toolCallAccumulators = new Map();
|
|
512
697
|
try {
|
|
513
698
|
while (true) {
|
|
514
|
-
|
|
699
|
+
// Race against the idle watchdog: reader.read() on a dead
|
|
700
|
+
// connection can otherwise pend forever with nothing logged.
|
|
701
|
+
const { done, value } = await Promise.race([reader.read(), abortPromise]);
|
|
702
|
+
armIdleWatchdog();
|
|
515
703
|
if (done)
|
|
516
704
|
break;
|
|
517
705
|
buffer += decoder.decode(value, { stream: true });
|
|
@@ -605,7 +793,17 @@ export class OpenRouterMcpClient extends BaseMcpClient {
|
|
|
605
793
|
}
|
|
606
794
|
}
|
|
607
795
|
}
|
|
796
|
+
catch (error) {
|
|
797
|
+
// Cancel the underlying stream so the connection is torn down; the
|
|
798
|
+
// losing reader.read() from the race is settled by the cancel/abort
|
|
799
|
+
// and its rejection is already observed by Promise.race.
|
|
800
|
+
slowWatch.finish(false);
|
|
801
|
+
reader.cancel().catch(() => undefined);
|
|
802
|
+
throw normalizeStreamError(error);
|
|
803
|
+
}
|
|
608
804
|
finally {
|
|
805
|
+
clearTimeout(idleTimer);
|
|
806
|
+
slowWatch.finish();
|
|
609
807
|
reader.releaseLock();
|
|
610
808
|
}
|
|
611
809
|
this.logger.debug(`📡 OpenRouter SSE: Stream complete. Content=${accumulatedContent.length}chars, Reasoning=${accumulatedReasoning.length}chars, ToolCalls=${toolCallAccumulators.size}, Finish=${finishReason}`);
|
|
@@ -745,7 +943,7 @@ export class OpenRouterMcpClient extends BaseMcpClient {
|
|
|
745
943
|
// Apply transformations (reordering)
|
|
746
944
|
const transformedMessages = converter.transform(openRouterMessages, MessageFormat.OPENROUTER);
|
|
747
945
|
// Send directly
|
|
748
|
-
return await
|
|
946
|
+
return await this.queueRequest(async () => {
|
|
749
947
|
if (!this.networkRecovery) {
|
|
750
948
|
throw new Error('Network recovery not initialized');
|
|
751
949
|
}
|
|
@@ -845,44 +1043,73 @@ export class OpenRouterMcpClient extends BaseMcpClient {
|
|
|
845
1043
|
if (requestBody.tools && requestBody.tools.length > 0) {
|
|
846
1044
|
// console.log(` Tools: ${requestBody.tools.map((t: any) => t.function.name).join(', ')}`);
|
|
847
1045
|
}
|
|
848
|
-
//
|
|
849
|
-
const
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
1046
|
+
// Serialize once so the logged request size is exactly what went on the wire
|
|
1047
|
+
const requestBodyJson = JSON.stringify(requestBody);
|
|
1048
|
+
const requestBytes = Buffer.byteLength(requestBodyJson, 'utf8');
|
|
1049
|
+
const agentId = options?.agentId || 'unknown';
|
|
1050
|
+
const requestStartedAt = Date.now();
|
|
1051
|
+
const slowWatch = this.startSlowRequestWatch('completion', model, agentId, requestBytes);
|
|
1052
|
+
let responseText;
|
|
1053
|
+
try {
|
|
1054
|
+
// AbortSignal.timeout bounds the entire request — connect, headers,
|
|
1055
|
+
// and body read — so a hung connection surfaces as an error instead
|
|
1056
|
+
// of indefinite silence. Reasoning models can legitimately take
|
|
1057
|
+
// minutes; the default limit is sized for that (see initializeProvider).
|
|
1058
|
+
const response = await fetch(`${this.baseUrl}/chat/completions`, {
|
|
1059
|
+
method: 'POST',
|
|
1060
|
+
headers,
|
|
1061
|
+
body: requestBodyJson,
|
|
1062
|
+
signal: AbortSignal.timeout(this.requestTimeoutMs)
|
|
1063
|
+
});
|
|
1064
|
+
// Check for errors with enhanced error information
|
|
1065
|
+
if (!response.ok) {
|
|
1066
|
+
let errorText = await response.text();
|
|
1067
|
+
this.logger.error(`🔧 DEBUG: Error response text: ${errorText}`);
|
|
1068
|
+
let errorMessage = errorText;
|
|
1069
|
+
let rateLimitInfo = {};
|
|
1070
|
+
try {
|
|
1071
|
+
const errorJson = JSON.parse(errorText);
|
|
1072
|
+
errorMessage = errorJson.error?.message || errorText;
|
|
1073
|
+
// Extract rate limit information if available
|
|
1074
|
+
if (response.status === 429) {
|
|
1075
|
+
rateLimitInfo = {
|
|
1076
|
+
retryAfter: response.headers.get('retry-after'),
|
|
1077
|
+
rateLimitLimit: response.headers.get('x-ratelimit-limit'),
|
|
1078
|
+
rateLimitRemaining: response.headers.get('x-ratelimit-remaining'),
|
|
1079
|
+
rateLimitReset: response.headers.get('x-ratelimit-reset')
|
|
1080
|
+
};
|
|
1081
|
+
}
|
|
1082
|
+
}
|
|
1083
|
+
catch (e) {
|
|
1084
|
+
// Use error text as is if not JSON
|
|
872
1085
|
}
|
|
1086
|
+
// Create detailed error with status code
|
|
1087
|
+
const error = new Error(`OpenRouter API error [${response.status}]: ${errorMessage}`);
|
|
1088
|
+
error.status = response.status;
|
|
1089
|
+
error.statusCode = response.status;
|
|
1090
|
+
error.rateLimitInfo = rateLimitInfo;
|
|
1091
|
+
throw error;
|
|
873
1092
|
}
|
|
874
|
-
|
|
875
|
-
|
|
1093
|
+
// Get response text for JSON parsing
|
|
1094
|
+
responseText = await response.text();
|
|
1095
|
+
slowWatch.finish();
|
|
1096
|
+
}
|
|
1097
|
+
catch (error) {
|
|
1098
|
+
slowWatch.finish(false);
|
|
1099
|
+
if (isAbortOrTimeoutError(error)) {
|
|
1100
|
+
throw this.buildRequestTimeoutError({
|
|
1101
|
+
kind: 'completion',
|
|
1102
|
+
detail: 'no response within the request timeout',
|
|
1103
|
+
model,
|
|
1104
|
+
agentId,
|
|
1105
|
+
elapsedMs: Date.now() - requestStartedAt,
|
|
1106
|
+
limitMs: this.requestTimeoutMs,
|
|
1107
|
+
requestBytes,
|
|
1108
|
+
messageCount: openRouterMessages.length
|
|
1109
|
+
});
|
|
876
1110
|
}
|
|
877
|
-
// Create detailed error with status code
|
|
878
|
-
const error = new Error(`OpenRouter API error [${response.status}]: ${errorMessage}`);
|
|
879
|
-
error.status = response.status;
|
|
880
|
-
error.statusCode = response.status;
|
|
881
|
-
error.rateLimitInfo = rateLimitInfo;
|
|
882
1111
|
throw error;
|
|
883
1112
|
}
|
|
884
|
-
// Get response text for JSON parsing
|
|
885
|
-
const responseText = await response.text();
|
|
886
1113
|
// Check if response is empty
|
|
887
1114
|
if (!responseText || responseText.length === 0) {
|
|
888
1115
|
this.logger.error('🔧 ERROR: Empty response from OpenRouter API');
|
|
@@ -901,6 +1128,11 @@ export class OpenRouterMcpClient extends BaseMcpClient {
|
|
|
901
1128
|
return this.convertToMcpResponse(openRouterResponse);
|
|
902
1129
|
}
|
|
903
1130
|
catch (error) {
|
|
1131
|
+
// Request timeouts are already logged with full context and must keep
|
|
1132
|
+
// their name/flags so NetworkRecovery classifies them as non-retryable.
|
|
1133
|
+
if (error?.isRequestTimeout === true) {
|
|
1134
|
+
throw error;
|
|
1135
|
+
}
|
|
904
1136
|
this.logger.error(`🔧 ERROR in executeOpenRouterRequest: ${error instanceof Error ? error.message : String(error)}`);
|
|
905
1137
|
if (error instanceof Error && error.stack) {
|
|
906
1138
|
this.logger.error(`🔧 ERROR STACK: ${error.stack}`);
|