@salesforce/sfdx-agent-harness-mastra 0.67.0 → 0.69.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +9 -0
- package/dist/mastra-harness.d.ts +8 -0
- package/dist/mastra-harness.js +31 -4
- package/dist/pass-through-anthropic.d.ts +4 -0
- package/dist/pass-through-anthropic.js +4 -4
- package/dist/pass-through-openai-chat-completions.d.ts +2 -0
- package/dist/pass-through-openai-chat-completions.js +3 -3
- package/dist/pass-through-openai.d.ts +2 -0
- package/dist/pass-through-openai.js +3 -3
- package/dist/stream-errors.d.ts +26 -0
- package/dist/stream-errors.js +29 -0
- package/dist/v3-stream-mapping.d.ts +24 -2
- package/dist/v3-stream-mapping.js +145 -5
- package/package.json +3 -3
package/CHANGELOG.md
CHANGED
|
@@ -3,6 +3,15 @@
|
|
|
3
3
|
All notable changes to `@salesforce/sfdx-agent-harness-mastra` are documented in this file.
|
|
4
4
|
Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
|
|
5
5
|
|
|
6
|
+
## [0.69.0] - 2026-09-02
|
|
7
|
+
|
|
8
|
+
_No changes — released alongside dependent packages._
|
|
9
|
+
|
|
10
|
+
## [0.68.0] - 2026-09-02
|
|
11
|
+
|
|
12
|
+
### Fixes
|
|
13
|
+
- **harness-mastra**: per-read idle-read timeout on gateway stream @W-24078276@ ([#791](https://github.com/forcedotcom/agentic-dx/pull/791))
|
|
14
|
+
|
|
6
15
|
## [0.67.0] - 2026-09-02
|
|
7
16
|
|
|
8
17
|
### Fixes
|
package/dist/mastra-harness.d.ts
CHANGED
|
@@ -230,6 +230,13 @@ export interface MastraHarnessOptions {
|
|
|
230
230
|
clock?: Clock;
|
|
231
231
|
idGenerator?: UniqueIDGenerator;
|
|
232
232
|
toolApprovalTimeoutMs?: number;
|
|
233
|
+
/**
|
|
234
|
+
* Per-read idle timeout (ms) for the LLM gateway response body. Defaults to
|
|
235
|
+
* {@link DEFAULT_IDLE_READ_TIMEOUT_MS}. The internal test seam for the
|
|
236
|
+
* idle-read watchdog (6b, W-24078276) — no factory/consumer knob, matching
|
|
237
|
+
* `resumeRpcTimeoutMs`'s posture (6c). `<= 0` disables the watchdog.
|
|
238
|
+
*/
|
|
239
|
+
idleReadTimeoutMs?: number;
|
|
233
240
|
/**
|
|
234
241
|
* Inner fetch the harness threads into every in-process HTTP call site
|
|
235
242
|
* (LLM gateway language-model builders + MCP remote transports). Defaults
|
|
@@ -286,6 +293,7 @@ export declare class MastraHarness implements MastraAgentHarness {
|
|
|
286
293
|
private readonly clock;
|
|
287
294
|
private readonly idGenerator;
|
|
288
295
|
private readonly toolApprovalTimeoutMs;
|
|
296
|
+
private readonly idleReadTimeoutMs;
|
|
289
297
|
private readonly innerFetch;
|
|
290
298
|
private readonly wireObserverAbortController;
|
|
291
299
|
private readonly languageModelBuilders?;
|
package/dist/mastra-harness.js
CHANGED
|
@@ -172,6 +172,24 @@ const MAX_TRANSCRIPT_CHARS = 400_000;
|
|
|
172
172
|
* like deploys. Per-server `timeout` in MCPServerConfig still overrides this.
|
|
173
173
|
*/
|
|
174
174
|
const DEFAULT_MCP_TIMEOUT_MS = 600_000;
|
|
175
|
+
/**
|
|
176
|
+
* Default per-read idle timeout (ms) for the gateway response body (6b,
|
|
177
|
+
* W-24078276) — see the "gateway stream has a per-read idle-read timeout"
|
|
178
|
+
* Critical Invariant. It is a per-read watchdog (resets on every chunk), NOT a
|
|
179
|
+
* wall-clock cap, so a healthy long-running turn making byte-level progress is
|
|
180
|
+
* never bounded; it fires only on genuinely zero forward progress.
|
|
181
|
+
*
|
|
182
|
+
* 2 minutes, backed by an orgfarm-e2e measurement of the SDK-consumed body on
|
|
183
|
+
* the primary Bedrock-Anthropic path: the gateway emits `message_start`
|
|
184
|
+
* immediately and streams frames continuously, so the MAX observed silent gap
|
|
185
|
+
* was ~1.8s even on an 837 KB generation AND on a ~500k-token prefill (Opus 4.8,
|
|
186
|
+
* 1M window) — TTFB was ~0-1ms in every case. 120s therefore clears the worst
|
|
187
|
+
* observed forward-progress gap by ~65x while still being ~15x tighter than the
|
|
188
|
+
* service layer's 30-min net (6a). Injectable via
|
|
189
|
+
* {@link MastraHarnessOptions.idleReadTimeoutMs} (the test seam; no
|
|
190
|
+
* factory/consumer knob — same posture as `resumeRpcTimeoutMs`, 6c).
|
|
191
|
+
*/
|
|
192
|
+
const DEFAULT_IDLE_READ_TIMEOUT_MS = 120_000;
|
|
175
193
|
/**
|
|
176
194
|
* Build the `Map<namespacedName, ToolSource>` index `resolveAlwaysActive`
|
|
177
195
|
* needs to match `{ serverName }` / `{ serverName, toolName }` / `{ toolName }`
|
|
@@ -446,6 +464,7 @@ export class MastraHarness {
|
|
|
446
464
|
clock;
|
|
447
465
|
idGenerator;
|
|
448
466
|
toolApprovalTimeoutMs;
|
|
467
|
+
idleReadTimeoutMs;
|
|
449
468
|
innerFetch;
|
|
450
469
|
wireObserverAbortController = new AbortController();
|
|
451
470
|
languageModelBuilders;
|
|
@@ -454,6 +473,7 @@ export class MastraHarness {
|
|
|
454
473
|
this.clock = opts.clock ?? new RealClock();
|
|
455
474
|
this.idGenerator = opts.idGenerator ?? new UUIDGenerator();
|
|
456
475
|
this.toolApprovalTimeoutMs = opts.toolApprovalTimeoutMs ?? 600_000;
|
|
476
|
+
this.idleReadTimeoutMs = opts.idleReadTimeoutMs ?? DEFAULT_IDLE_READ_TIMEOUT_MS;
|
|
457
477
|
this.innerFetch = opts.innerFetch ?? globalThis.fetch;
|
|
458
478
|
this.languageModelBuilders = opts.languageModelBuilders;
|
|
459
479
|
this.storage = new LibSQLStore({
|
|
@@ -1474,6 +1494,7 @@ export class MastraHarness {
|
|
|
1474
1494
|
}
|
|
1475
1495
|
};
|
|
1476
1496
|
const innerFetch = this.innerFetch;
|
|
1497
|
+
const idleReadTimeoutMs = this.idleReadTimeoutMs;
|
|
1477
1498
|
const wireHooks = {
|
|
1478
1499
|
emitWireCommunication,
|
|
1479
1500
|
getModel: () => getInfo().nativeModelId,
|
|
@@ -1486,22 +1507,28 @@ export class MastraHarness {
|
|
|
1486
1507
|
switch (initial.providerHint) {
|
|
1487
1508
|
case 'bedrock-anthropic':
|
|
1488
1509
|
// Salesforce LLM Gateway's `/invoke-with-response-stream` (Bedrock pass-through).
|
|
1489
|
-
return buildAnthropicLanguageModel({ getInfo, logBus, wireHooks, innerFetch });
|
|
1510
|
+
return buildAnthropicLanguageModel({ getInfo, logBus, wireHooks, innerFetch, idleReadTimeoutMs });
|
|
1490
1511
|
case 'anthropic':
|
|
1491
1512
|
// Direct Anthropic Messages API (`api.anthropic.com/v1/messages`) —
|
|
1492
1513
|
// different wire shape (`x-api-key` auth, body without `anthropic_version`).
|
|
1493
|
-
return buildAnthropicDirectLanguageModel({ getInfo, logBus, wireHooks, innerFetch });
|
|
1514
|
+
return buildAnthropicDirectLanguageModel({ getInfo, logBus, wireHooks, innerFetch, idleReadTimeoutMs });
|
|
1494
1515
|
case 'openai-responses':
|
|
1495
1516
|
case 'openai':
|
|
1496
1517
|
// OpenAI Responses API. Salesforce LLM Gateway's `/responses` pass-through
|
|
1497
1518
|
// for `'openai-responses'`; `api.openai.com/v1/responses` for `'openai'` direct.
|
|
1498
1519
|
// Same wire shape; the only difference is the `baseUrl` the resolver supplies.
|
|
1499
|
-
return buildOpenAILanguageModel({ getInfo, logBus, wireHooks, innerFetch });
|
|
1520
|
+
return buildOpenAILanguageModel({ getInfo, logBus, wireHooks, innerFetch, idleReadTimeoutMs });
|
|
1500
1521
|
case 'openai-compatible':
|
|
1501
1522
|
// OpenAI-compatible Chat Completions API (LLMG Express, LiteLLM, etc.).
|
|
1502
1523
|
// Different wire shape from Responses — `/v1/chat/completions` with
|
|
1503
1524
|
// `choices[0].delta` streaming events.
|
|
1504
|
-
return buildOpenAIChatCompletionsLanguageModel({
|
|
1525
|
+
return buildOpenAIChatCompletionsLanguageModel({
|
|
1526
|
+
getInfo,
|
|
1527
|
+
logBus,
|
|
1528
|
+
wireHooks,
|
|
1529
|
+
innerFetch,
|
|
1530
|
+
idleReadTimeoutMs,
|
|
1531
|
+
});
|
|
1505
1532
|
default: {
|
|
1506
1533
|
// Consult the injected builders map before throwing. This is the
|
|
1507
1534
|
// extension point for consumer-registered wire shapes (e.g. a temporary
|
|
@@ -29,6 +29,8 @@ export declare function buildAnthropicLanguageModel(opts: {
|
|
|
29
29
|
* don't need to thread it.
|
|
30
30
|
*/
|
|
31
31
|
innerFetch?: typeof fetch;
|
|
32
|
+
/** Per-read idle timeout (ms) for the gateway response body; see `createGatewayFetch`. */
|
|
33
|
+
idleReadTimeoutMs?: number;
|
|
32
34
|
}): LanguageModelV3;
|
|
33
35
|
/**
|
|
34
36
|
* Builds the in-process `LanguageModelV3` impl for direct-Anthropic API access
|
|
@@ -56,6 +58,8 @@ export declare function buildAnthropicDirectLanguageModel(opts: {
|
|
|
56
58
|
wireHooks?: GatewayFetchWireHooks;
|
|
57
59
|
/** See {@link buildAnthropicLanguageModel} `innerFetch`. */
|
|
58
60
|
innerFetch?: typeof fetch;
|
|
61
|
+
/** Per-read idle timeout (ms) for the gateway response body; see `createGatewayFetch`. */
|
|
62
|
+
idleReadTimeoutMs?: number;
|
|
59
63
|
}): LanguageModelV3;
|
|
60
64
|
/**
|
|
61
65
|
* Translates a V3 `LanguageModelV3Prompt` into Anthropic's Messages API shape
|
|
@@ -21,7 +21,7 @@ import { TextSpanLifecycle, createGatewayFetch, isEmptyUsage, isParameterExclude
|
|
|
21
21
|
* the next request.
|
|
22
22
|
*/
|
|
23
23
|
export function buildAnthropicLanguageModel(opts) {
|
|
24
|
-
const { getInfo, logBus, emitWireCommunication, wireHooks, innerFetch } = opts;
|
|
24
|
+
const { getInfo, logBus, emitWireCommunication, wireHooks, innerFetch, idleReadTimeoutMs } = opts;
|
|
25
25
|
const initialInfo = getInfo();
|
|
26
26
|
const ctx = {
|
|
27
27
|
getInfo,
|
|
@@ -34,7 +34,7 @@ export function buildAnthropicLanguageModel(opts) {
|
|
|
34
34
|
// The fetch wrapper re-evaluates `getHeaders()` on every outbound call —
|
|
35
35
|
// JWT rotation, premium-rate-limit gate flips, BYOK provider rotation
|
|
36
36
|
// all land on the next request without rebuilding the bridge.
|
|
37
|
-
fetch: createGatewayFetch(() => info.getHeaders(), ctx.wireHooks ?? buildWireHooks(ctx), innerFetch),
|
|
37
|
+
fetch: createGatewayFetch(() => info.getHeaders(), ctx.wireHooks ?? buildWireHooks(ctx), innerFetch, idleReadTimeoutMs, logBus),
|
|
38
38
|
}),
|
|
39
39
|
};
|
|
40
40
|
return {
|
|
@@ -70,7 +70,7 @@ export function buildAnthropicLanguageModel(opts) {
|
|
|
70
70
|
* language model.
|
|
71
71
|
*/
|
|
72
72
|
export function buildAnthropicDirectLanguageModel(opts) {
|
|
73
|
-
const { getInfo, logBus, emitWireCommunication, wireHooks, innerFetch } = opts;
|
|
73
|
+
const { getInfo, logBus, emitWireCommunication, wireHooks, innerFetch, idleReadTimeoutMs } = opts;
|
|
74
74
|
const initialInfo = getInfo();
|
|
75
75
|
const ctx = {
|
|
76
76
|
getInfo,
|
|
@@ -88,7 +88,7 @@ export function buildAnthropicDirectLanguageModel(opts) {
|
|
|
88
88
|
new Anthropic({
|
|
89
89
|
apiKey: 'anthropic-direct-resolver-managed',
|
|
90
90
|
baseURL: info.baseUrl,
|
|
91
|
-
fetch: createGatewayFetch(() => translateBearerToXApiKey(info), ctx.wireHooks ?? buildWireHooks(ctx), innerFetch),
|
|
91
|
+
fetch: createGatewayFetch(() => translateBearerToXApiKey(info), ctx.wireHooks ?? buildWireHooks(ctx), innerFetch, idleReadTimeoutMs, logBus),
|
|
92
92
|
}),
|
|
93
93
|
};
|
|
94
94
|
return {
|
|
@@ -30,6 +30,8 @@ export declare function buildOpenAIChatCompletionsLanguageModel(opts: {
|
|
|
30
30
|
wireHooks?: GatewayFetchWireHooks;
|
|
31
31
|
/** See `buildAnthropicLanguageModel` `innerFetch`. */
|
|
32
32
|
innerFetch?: typeof fetch;
|
|
33
|
+
/** Per-read idle timeout (ms) for the gateway response body; see `createGatewayFetch`. */
|
|
34
|
+
idleReadTimeoutMs?: number;
|
|
33
35
|
}): LanguageModelV3;
|
|
34
36
|
/**
|
|
35
37
|
* Translates a V3 `LanguageModelV3Prompt` into the OpenAI Chat Completions
|
|
@@ -26,9 +26,9 @@ import { TextSpanLifecycle, createGatewayFetch, isEmptyUsage, isParameterExclude
|
|
|
26
26
|
* without rebuilding the language model.
|
|
27
27
|
*/
|
|
28
28
|
export function buildOpenAIChatCompletionsLanguageModel(opts) {
|
|
29
|
-
const { getInfo, logBus, emitWireCommunication, wireHooks, innerFetch } = opts;
|
|
29
|
+
const { getInfo, logBus, emitWireCommunication, wireHooks, innerFetch, idleReadTimeoutMs } = opts;
|
|
30
30
|
const initialInfo = getInfo();
|
|
31
|
-
const ctx = { getInfo, logBus, emitWireCommunication, wireHooks, innerFetch };
|
|
31
|
+
const ctx = { getInfo, logBus, emitWireCommunication, wireHooks, innerFetch, idleReadTimeoutMs };
|
|
32
32
|
return {
|
|
33
33
|
specificationVersion: 'v3',
|
|
34
34
|
provider: 'openai-compatible-chat-completions-pass-through',
|
|
@@ -56,7 +56,7 @@ async function buildRequest(ctx, options) {
|
|
|
56
56
|
// request. The SDK still wants a non-empty `apiKey` at construction;
|
|
57
57
|
// an opaque sentinel keeps it from reading `process.env['OPENAI_API_KEY']`.
|
|
58
58
|
apiKey: 'openai-compatible-resolver-managed',
|
|
59
|
-
fetch: createGatewayFetch(() => info.getHeaders(), wireHooks, ctx.innerFetch),
|
|
59
|
+
fetch: createGatewayFetch(() => info.getHeaders(), wireHooks, ctx.innerFetch, ctx.idleReadTimeoutMs, ctx.logBus),
|
|
60
60
|
});
|
|
61
61
|
const messages = translatePromptToChatCompletions(options.prompt);
|
|
62
62
|
const tools = translateToolsToChatCompletions(options);
|
|
@@ -22,6 +22,8 @@ export declare function buildOpenAILanguageModel(opts: {
|
|
|
22
22
|
wireHooks?: GatewayFetchWireHooks;
|
|
23
23
|
/** See `buildAnthropicLanguageModel` `innerFetch`. */
|
|
24
24
|
innerFetch?: typeof fetch;
|
|
25
|
+
/** Per-read idle timeout (ms) for the gateway response body; see `createGatewayFetch`. */
|
|
26
|
+
idleReadTimeoutMs?: number;
|
|
25
27
|
}): LanguageModelV3;
|
|
26
28
|
/**
|
|
27
29
|
* Translates a V3 `LanguageModelV3Prompt` into OpenAI's Responses input shape:
|
|
@@ -18,9 +18,9 @@ import { TextSpanLifecycle, createGatewayFetch, isEmptyUsage, isParameterExclude
|
|
|
18
18
|
* next request.
|
|
19
19
|
*/
|
|
20
20
|
export function buildOpenAILanguageModel(opts) {
|
|
21
|
-
const { getInfo, logBus, emitWireCommunication, wireHooks, innerFetch } = opts;
|
|
21
|
+
const { getInfo, logBus, emitWireCommunication, wireHooks, innerFetch, idleReadTimeoutMs } = opts;
|
|
22
22
|
const initialInfo = getInfo();
|
|
23
|
-
const ctx = { getInfo, logBus, emitWireCommunication, wireHooks, innerFetch };
|
|
23
|
+
const ctx = { getInfo, logBus, emitWireCommunication, wireHooks, innerFetch, idleReadTimeoutMs };
|
|
24
24
|
return {
|
|
25
25
|
specificationVersion: 'v3',
|
|
26
26
|
provider: 'salesforce-llm-gateway-pass-through-openai',
|
|
@@ -56,7 +56,7 @@ async function buildOpenAIRequest(ctx, options) {
|
|
|
56
56
|
// Re-evaluates `getHeaders()` per outbound call so JWT rotation /
|
|
57
57
|
// premium-rate-limit gate flips / BYOK provider rotation land on the
|
|
58
58
|
// next request without rebuilding this bridge.
|
|
59
|
-
fetch: createGatewayFetch(() => info.getHeaders(), wireHooks, ctx.innerFetch),
|
|
59
|
+
fetch: createGatewayFetch(() => info.getHeaders(), wireHooks, ctx.innerFetch, ctx.idleReadTimeoutMs, ctx.logBus),
|
|
60
60
|
});
|
|
61
61
|
const { input, instructions } = translatePromptToOpenAI(options.prompt);
|
|
62
62
|
const tools = translateToolsToOpenAI(options);
|
package/dist/stream-errors.d.ts
CHANGED
|
@@ -27,6 +27,32 @@ export declare class NetworkStreamError extends ClassifiedStreamError {
|
|
|
27
27
|
readonly code: "network-error";
|
|
28
28
|
constructor(cause: Error);
|
|
29
29
|
}
|
|
30
|
+
/**
|
|
31
|
+
* The gateway response body went idle — no bytes for `idleMs` with no forward
|
|
32
|
+
* progress — so the harness's per-read idle watchdog tore the connection down
|
|
33
|
+
* (6b, spike agent-offline-failure-modes, W-24078276). A half-open socket (TCP
|
|
34
|
+
* open, no bytes, no FIN/RST) would otherwise wedge the turn forever; this error
|
|
35
|
+
* releases it as a transport fault.
|
|
36
|
+
*
|
|
37
|
+
* It classifies as {@link NETWORK_ERROR_CODE} — an idle half-open socket is the
|
|
38
|
+
* same "the transport dropped, a retry could recover" class `NetworkStreamError`
|
|
39
|
+
* covers (downstream retry/backoff keys on the code, 8e). Unlike the Anthropic
|
|
40
|
+
* path's `NetworkStreamError`, this carries the code DIRECTLY on a
|
|
41
|
+
* `ClassifiedStreamError` subtype, so it surfaces `network-error` uniformly on
|
|
42
|
+
* every provider path: the Anthropic iterator catch enqueues it unchanged
|
|
43
|
+
* (`recognizeBedrockValidationError` returns `undefined` and it is NOT re-wrapped),
|
|
44
|
+
* and the OpenAI catches enqueue it raw — in both cases `MastraEventAdapter`
|
|
45
|
+
* copies `.code` via {@link isClassifiedStreamError}. It is deliberately NOT
|
|
46
|
+
* matched by {@link isStreamTransportError} (that would double-wrap it in a
|
|
47
|
+
* `NetworkStreamError` on the Anthropic path for no gain). Its `name` is NOT
|
|
48
|
+
* `AbortError`, so the provider SDK `Stream` iterators do not swallow it as a
|
|
49
|
+
* user cancel (which would resurface the false-success wedge, 8a).
|
|
50
|
+
*/
|
|
51
|
+
export declare class IdleReadTimeoutError extends ClassifiedStreamError {
|
|
52
|
+
readonly name = "IdleReadTimeoutError";
|
|
53
|
+
readonly code: "network-error";
|
|
54
|
+
constructor(idleMs: number);
|
|
55
|
+
}
|
|
30
56
|
/** A stream that closed cleanly with no recognized content/usage/terminal event — the "false-success" case (8a); `empty-stream`. */
|
|
31
57
|
export declare class EmptyStreamError extends ClassifiedStreamError {
|
|
32
58
|
readonly name = "EmptyStreamError";
|
package/dist/stream-errors.js
CHANGED
|
@@ -33,6 +33,35 @@ export class NetworkStreamError extends ClassifiedStreamError {
|
|
|
33
33
|
super(cause.message, { cause });
|
|
34
34
|
}
|
|
35
35
|
}
|
|
36
|
+
/**
|
|
37
|
+
* The gateway response body went idle — no bytes for `idleMs` with no forward
|
|
38
|
+
* progress — so the harness's per-read idle watchdog tore the connection down
|
|
39
|
+
* (6b, spike agent-offline-failure-modes, W-24078276). A half-open socket (TCP
|
|
40
|
+
* open, no bytes, no FIN/RST) would otherwise wedge the turn forever; this error
|
|
41
|
+
* releases it as a transport fault.
|
|
42
|
+
*
|
|
43
|
+
* It classifies as {@link NETWORK_ERROR_CODE} — an idle half-open socket is the
|
|
44
|
+
* same "the transport dropped, a retry could recover" class `NetworkStreamError`
|
|
45
|
+
* covers (downstream retry/backoff keys on the code, 8e). Unlike the Anthropic
|
|
46
|
+
* path's `NetworkStreamError`, this carries the code DIRECTLY on a
|
|
47
|
+
* `ClassifiedStreamError` subtype, so it surfaces `network-error` uniformly on
|
|
48
|
+
* every provider path: the Anthropic iterator catch enqueues it unchanged
|
|
49
|
+
* (`recognizeBedrockValidationError` returns `undefined` and it is NOT re-wrapped),
|
|
50
|
+
* and the OpenAI catches enqueue it raw — in both cases `MastraEventAdapter`
|
|
51
|
+
* copies `.code` via {@link isClassifiedStreamError}. It is deliberately NOT
|
|
52
|
+
* matched by {@link isStreamTransportError} (that would double-wrap it in a
|
|
53
|
+
* `NetworkStreamError` on the Anthropic path for no gain). Its `name` is NOT
|
|
54
|
+
* `AbortError`, so the provider SDK `Stream` iterators do not swallow it as a
|
|
55
|
+
* user cancel (which would resurface the false-success wedge, 8a).
|
|
56
|
+
*/
|
|
57
|
+
export class IdleReadTimeoutError extends ClassifiedStreamError {
|
|
58
|
+
name = 'IdleReadTimeoutError';
|
|
59
|
+
code = NETWORK_ERROR_CODE;
|
|
60
|
+
constructor(idleMs) {
|
|
61
|
+
super(`LLM Gateway stream idle for ${idleMs}ms with no forward progress (no bytes received); ` +
|
|
62
|
+
'treating as a transport fault and tearing down the connection.');
|
|
63
|
+
}
|
|
64
|
+
}
|
|
36
65
|
/** A stream that closed cleanly with no recognized content/usage/terminal event — the "false-success" case (8a); `empty-stream`. */
|
|
37
66
|
export class EmptyStreamError extends ClassifiedStreamError {
|
|
38
67
|
name = 'EmptyStreamError';
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { LanguageModelV3FinishReason, LanguageModelV3StreamPart, LanguageModelV3ToolResultOutput, LanguageModelV3Usage } from '@ai-sdk/provider';
|
|
2
|
-
import { type Clock, type UniqueIDGenerator } from '@salesforce/agentic-common';
|
|
2
|
+
import { type Clock, type LogBus, type UniqueIDGenerator } from '@salesforce/agentic-common';
|
|
3
3
|
import type { Model } from '@salesforce/sfdx-agent-sdk';
|
|
4
4
|
/**
|
|
5
5
|
* Shared V3 helpers for the Mastra harness's `LanguageModelV3` implementations
|
|
@@ -230,7 +230,29 @@ export type GatewayFetchWireHooks = {
|
|
|
230
230
|
* <n> }` so wire-communication captures don't blow the consumer's log file
|
|
231
231
|
* budget.
|
|
232
232
|
*/
|
|
233
|
-
export declare function createGatewayFetch(getHeaders: () => Record<string, string> | Promise<Record<string, string>>, wireHooks?: GatewayFetchWireHooks, innerFetch?: typeof fetch
|
|
233
|
+
export declare function createGatewayFetch(getHeaders: () => Record<string, string> | Promise<Record<string, string>>, wireHooks?: GatewayFetchWireHooks, innerFetch?: typeof fetch,
|
|
234
|
+
/**
|
|
235
|
+
* Per-read idle timeout (ms) for the response body handed to the provider
|
|
236
|
+
* SDK. When set (`> 0`), the SDK-facing body is wrapped in a watchdog that
|
|
237
|
+
* resets on every chunk and, after `idleReadTimeoutMs` of ZERO bytes, tears
|
|
238
|
+
* the connection down and errors the stream with a classified
|
|
239
|
+
* {@link IdleReadTimeoutError} (`network-error`) — so a half-open socket
|
|
240
|
+
* (TCP open, no bytes, no FIN/RST) fails the turn loudly instead of hanging
|
|
241
|
+
* forever (6b, W-24078276). This is a true per-read idle watchdog, NOT a
|
|
242
|
+
* wall-clock cap: a healthy long-running turn that keeps making byte-level
|
|
243
|
+
* progress (Anthropic streams `ping`/thinking deltas during long
|
|
244
|
+
* operations) is never capped. Omitted / `0` disables the watchdog (the
|
|
245
|
+
* default for the many `createGatewayFetch` unit tests that resolve
|
|
246
|
+
* immediately); the harness always supplies a value in production.
|
|
247
|
+
*/
|
|
248
|
+
idleReadTimeoutMs?: number,
|
|
249
|
+
/**
|
|
250
|
+
* Optional log bus. When the idle watchdog fires, it emits a `warn` here so
|
|
251
|
+
* the field can distinguish a real half-open-socket teardown from a healthy
|
|
252
|
+
* turn clipped by a too-tight timeout (a false positive). Silent in steady
|
|
253
|
+
* state — nothing is logged unless the watchdog actually trips.
|
|
254
|
+
*/
|
|
255
|
+
logBus?: LogBus): typeof fetch;
|
|
234
256
|
/**
|
|
235
257
|
* The `WireCommunicationEvent` shape from `@salesforce/sfdx-agent-sdk` minus the
|
|
236
258
|
* harness-injected fields. Kept structural so this module doesn't import
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
*/
|
|
5
5
|
import { RealClock, UUIDGenerator } from '@salesforce/agentic-common';
|
|
6
6
|
import { EventStreamCodec } from '@smithy/eventstream-codec';
|
|
7
|
+
import { IdleReadTimeoutError } from './stream-errors.js';
|
|
7
8
|
/**
|
|
8
9
|
* Shared V3 helpers for the Mastra harness's `LanguageModelV3` implementations
|
|
9
10
|
* — stream-shaping primitives, finish-reason / usage mappers, prompt-translation
|
|
@@ -265,7 +266,29 @@ export function mapOpenAIUsage(usage) {
|
|
|
265
266
|
* <n> }` so wire-communication captures don't blow the consumer's log file
|
|
266
267
|
* budget.
|
|
267
268
|
*/
|
|
268
|
-
export function createGatewayFetch(getHeaders, wireHooks, innerFetch = globalThis.fetch
|
|
269
|
+
export function createGatewayFetch(getHeaders, wireHooks, innerFetch = globalThis.fetch,
|
|
270
|
+
/**
|
|
271
|
+
* Per-read idle timeout (ms) for the response body handed to the provider
|
|
272
|
+
* SDK. When set (`> 0`), the SDK-facing body is wrapped in a watchdog that
|
|
273
|
+
* resets on every chunk and, after `idleReadTimeoutMs` of ZERO bytes, tears
|
|
274
|
+
* the connection down and errors the stream with a classified
|
|
275
|
+
* {@link IdleReadTimeoutError} (`network-error`) — so a half-open socket
|
|
276
|
+
* (TCP open, no bytes, no FIN/RST) fails the turn loudly instead of hanging
|
|
277
|
+
* forever (6b, W-24078276). This is a true per-read idle watchdog, NOT a
|
|
278
|
+
* wall-clock cap: a healthy long-running turn that keeps making byte-level
|
|
279
|
+
* progress (Anthropic streams `ping`/thinking deltas during long
|
|
280
|
+
* operations) is never capped. Omitted / `0` disables the watchdog (the
|
|
281
|
+
* default for the many `createGatewayFetch` unit tests that resolve
|
|
282
|
+
* immediately); the harness always supplies a value in production.
|
|
283
|
+
*/
|
|
284
|
+
idleReadTimeoutMs,
|
|
285
|
+
/**
|
|
286
|
+
* Optional log bus. When the idle watchdog fires, it emits a `warn` here so
|
|
287
|
+
* the field can distinguish a real half-open-socket teardown from a healthy
|
|
288
|
+
* turn clipped by a too-tight timeout (a false positive). Silent in steady
|
|
289
|
+
* state — nothing is logged unless the watchdog actually trips.
|
|
290
|
+
*/
|
|
291
|
+
logBus) {
|
|
269
292
|
return async (input, init) => {
|
|
270
293
|
const resolved = await getHeaders();
|
|
271
294
|
// Per WHATWG Fetch, `init.headers` is `HeadersInit = Headers | Record<string, string> | [string, string][]`.
|
|
@@ -278,9 +301,15 @@ export function createGatewayFetch(getHeaders, wireHooks, innerFetch = globalThi
|
|
|
278
301
|
for (const [k, v] of Object.entries(resolved)) {
|
|
279
302
|
finalHeaders.set(k, v); // resolver wins on conflict — same semantics as the prior spread merge.
|
|
280
303
|
}
|
|
281
|
-
|
|
304
|
+
// Link the caller's AbortSignal (finding 05 / W-23988781 wires the turn's
|
|
305
|
+
// signal into fetch) with an internal idle-timeout controller, so an idle
|
|
306
|
+
// expiry aborts the underlying fetch — tearing down the socket AND both
|
|
307
|
+
// tee branches — while a caller abort still works unchanged.
|
|
308
|
+
const idle = makeIdleReadGuard(init?.signal, idleReadTimeoutMs, logBus);
|
|
309
|
+
const finalInit = { ...init, headers: finalHeaders, ...(idle ? { signal: idle.signal } : {}) };
|
|
282
310
|
if (!wireHooks) {
|
|
283
|
-
|
|
311
|
+
const response = await innerFetch(input, finalInit);
|
|
312
|
+
return idle ? idle.wrapResponse(response) : response;
|
|
284
313
|
}
|
|
285
314
|
const url = typeof input === 'string' ? input : input instanceof URL ? input.toString() : input.url;
|
|
286
315
|
const method = (finalInit.method ?? 'GET').toUpperCase();
|
|
@@ -317,7 +346,11 @@ export function createGatewayFetch(getHeaders, wireHooks, innerFetch = globalThi
|
|
|
317
346
|
};
|
|
318
347
|
if (response.body === null || !(wireHooks.hasWireCommunicationSubscribers?.() ?? true)) {
|
|
319
348
|
emitResponse();
|
|
320
|
-
|
|
349
|
+
// Common production path: wire subscribers are opt-in and usually
|
|
350
|
+
// absent, so the watchdog MUST wrap here too — not only in the tee
|
|
351
|
+
// branch below — or a half-open socket wedges the turn whenever
|
|
352
|
+
// nobody is observing wire communication.
|
|
353
|
+
return idle ? idle.wrapResponse(response) : response;
|
|
321
354
|
}
|
|
322
355
|
const [observerBody, sdkBody] = response.body.tee();
|
|
323
356
|
void observeResponseBody({
|
|
@@ -329,7 +362,7 @@ export function createGatewayFetch(getHeaders, wireHooks, innerFetch = globalThi
|
|
|
329
362
|
clock,
|
|
330
363
|
onComplete: emitResponse,
|
|
331
364
|
}).catch(() => undefined);
|
|
332
|
-
return new Response(sdkBody, response);
|
|
365
|
+
return new Response(idle ? idle.wrap(sdkBody) : sdkBody, response);
|
|
333
366
|
}
|
|
334
367
|
catch (error) {
|
|
335
368
|
wireHooks.emitWireCommunication({
|
|
@@ -345,6 +378,113 @@ export function createGatewayFetch(getHeaders, wireHooks, innerFetch = globalThi
|
|
|
345
378
|
}
|
|
346
379
|
};
|
|
347
380
|
}
|
|
381
|
+
/**
|
|
382
|
+
* Builds the per-request idle-read guard, or `undefined` when the watchdog is
|
|
383
|
+
* disabled (`idleReadTimeoutMs` unset / `<= 0`).
|
|
384
|
+
*
|
|
385
|
+
* The returned `signal` links the caller's abort signal with an internal
|
|
386
|
+
* controller and is passed to the underlying `fetch`, so aborting on idle-timeout
|
|
387
|
+
* tears down the live socket (and, in the wire-hooks path, both tee branches at
|
|
388
|
+
* once). `wrap` wraps the SDK-facing body in the watchdog stream; `wrapResponse`
|
|
389
|
+
* is the convenience form for the non-tee return sites (bodyless responses pass
|
|
390
|
+
* through untouched).
|
|
391
|
+
*/
|
|
392
|
+
function makeIdleReadGuard(callerSignal, idleReadTimeoutMs, logBus) {
|
|
393
|
+
if (idleReadTimeoutMs === undefined || idleReadTimeoutMs <= 0)
|
|
394
|
+
return undefined;
|
|
395
|
+
const controller = new AbortController();
|
|
396
|
+
const signal = callerSignal ? AbortSignal.any([callerSignal, controller.signal]) : controller.signal;
|
|
397
|
+
const wrap = (body) => wrapWithIdleReadTimeout(body, idleReadTimeoutMs, () => {
|
|
398
|
+
// Field signal for the one failure mode this watchdog can cause — a
|
|
399
|
+
// false-positive teardown of a healthy-but-silent turn. Silent unless
|
|
400
|
+
// it actually fires.
|
|
401
|
+
logBus?.warn(`LLM Gateway stream idle for ${idleReadTimeoutMs}ms with no forward progress (no bytes received); ` +
|
|
402
|
+
'tearing down the connection as a network-error.', { idleReadTimeoutMs });
|
|
403
|
+
// Abort the fetch so undici tears down the connection; the wrapped
|
|
404
|
+
// stream is errored separately with the non-abort IdleReadTimeoutError
|
|
405
|
+
// the SDK actually observes (an AbortError would be swallowed).
|
|
406
|
+
controller.abort(new IdleReadTimeoutError(idleReadTimeoutMs));
|
|
407
|
+
});
|
|
408
|
+
return {
|
|
409
|
+
signal,
|
|
410
|
+
wrap,
|
|
411
|
+
wrapResponse: (response) => (response.body ? new Response(wrap(response.body), response) : response),
|
|
412
|
+
};
|
|
413
|
+
}
|
|
414
|
+
/**
|
|
415
|
+
* Wraps a response body in a passthrough stream that resets an idle timer on
|
|
416
|
+
* every chunk (and arms it up front to cover time-to-first-byte). After
|
|
417
|
+
* `idleMs` of ZERO bytes it calls `onTimeout` (tears the socket down) and errors
|
|
418
|
+
* the wrapped stream with {@link IdleReadTimeoutError} — the error the provider
|
|
419
|
+
* SDK's body reader surfaces, classified as `network-error`. A normal
|
|
420
|
+
* completion, a downstream cancel, or a source error all clear the timer, so no
|
|
421
|
+
* timer outlives the read. The timer is `unref`'d where supported so a stray
|
|
422
|
+
* armed watchdog never keeps the process alive.
|
|
423
|
+
*/
|
|
424
|
+
function wrapWithIdleReadTimeout(source, idleMs, onTimeout) {
|
|
425
|
+
const reader = source.getReader();
|
|
426
|
+
let timer;
|
|
427
|
+
let timedOut = false;
|
|
428
|
+
const clearIdle = () => {
|
|
429
|
+
if (timer !== undefined) {
|
|
430
|
+
clearTimeout(timer);
|
|
431
|
+
timer = undefined;
|
|
432
|
+
}
|
|
433
|
+
};
|
|
434
|
+
return new ReadableStream({
|
|
435
|
+
start(controller) {
|
|
436
|
+
const arm = () => {
|
|
437
|
+
clearIdle();
|
|
438
|
+
timer = setTimeout(() => {
|
|
439
|
+
timedOut = true;
|
|
440
|
+
const err = new IdleReadTimeoutError(idleMs);
|
|
441
|
+
onTimeout();
|
|
442
|
+
try {
|
|
443
|
+
controller.error(err);
|
|
444
|
+
}
|
|
445
|
+
catch {
|
|
446
|
+
// Already closed / errored — nothing to do.
|
|
447
|
+
}
|
|
448
|
+
void reader.cancel(err).catch(() => undefined);
|
|
449
|
+
}, idleMs);
|
|
450
|
+
timer.unref?.();
|
|
451
|
+
};
|
|
452
|
+
arm();
|
|
453
|
+
const pump = async () => {
|
|
454
|
+
try {
|
|
455
|
+
for (;;) {
|
|
456
|
+
const { done, value } = await reader.read();
|
|
457
|
+
if (timedOut)
|
|
458
|
+
return; // the timer already errored the stream
|
|
459
|
+
if (done) {
|
|
460
|
+
clearIdle();
|
|
461
|
+
controller.close();
|
|
462
|
+
return;
|
|
463
|
+
}
|
|
464
|
+
arm(); // reset the idle window on every chunk
|
|
465
|
+
controller.enqueue(value);
|
|
466
|
+
}
|
|
467
|
+
}
|
|
468
|
+
catch (err) {
|
|
469
|
+
if (timedOut)
|
|
470
|
+
return;
|
|
471
|
+
clearIdle();
|
|
472
|
+
try {
|
|
473
|
+
controller.error(err);
|
|
474
|
+
}
|
|
475
|
+
catch {
|
|
476
|
+
// Already closed / errored.
|
|
477
|
+
}
|
|
478
|
+
}
|
|
479
|
+
};
|
|
480
|
+
void pump();
|
|
481
|
+
},
|
|
482
|
+
cancel(reason) {
|
|
483
|
+
clearIdle();
|
|
484
|
+
return reader.cancel(reason);
|
|
485
|
+
},
|
|
486
|
+
});
|
|
487
|
+
}
|
|
348
488
|
const TRACE_BODY_BYTE_BUDGET = 256 * 1024;
|
|
349
489
|
/**
|
|
350
490
|
* Observes only the owned tee branch. Cancellation intentionally does not await
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@salesforce/sfdx-agent-harness-mastra",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.69.0",
|
|
4
4
|
"description": "Mastra-backed AgentHarness implementation for @salesforce/sfdx-agent-sdk",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -42,12 +42,12 @@
|
|
|
42
42
|
"openai": "^7.0.0"
|
|
43
43
|
},
|
|
44
44
|
"peerDependencies": {
|
|
45
|
-
"@salesforce/sfdx-agent-sdk": "0.
|
|
45
|
+
"@salesforce/sfdx-agent-sdk": "0.70.0"
|
|
46
46
|
},
|
|
47
47
|
"devDependencies": {
|
|
48
48
|
"@eslint/js": "^10.0.1",
|
|
49
49
|
"@salesforce/harness-conformance": "0.0.1",
|
|
50
|
-
"@salesforce/sfdx-agent-sdk": "0.
|
|
50
|
+
"@salesforce/sfdx-agent-sdk": "0.70.0",
|
|
51
51
|
"@types/node": "^22.20.1",
|
|
52
52
|
"@vitest/coverage-istanbul": "^4.1.10",
|
|
53
53
|
"@vitest/eslint-plugin": "^1.6.27",
|