@openllmsh/sdk 2.10.14 → 2.10.15-alpha.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -64,6 +64,11 @@ export declare const AGENT_HOOK_EVENTS: readonly ["PermissionRequest", "PreToolU
64
64
  export declare const AGENT_MAX_ATTACHMENTS = 8;
65
65
  /** Max base64 characters of one inline attachment. */
66
66
  export declare const AGENT_MAX_ATTACHMENT_BASE64_LENGTH: number;
67
+ /**
68
+ * Max characters of one absolute local-path attachment (same bound as a
69
+ * session cwd). Path attachments are loopback-only and never cross the network.
70
+ */
71
+ export declare const AGENT_MAX_ATTACHMENT_PATH_LENGTH = 1024;
67
72
  /**
68
73
  * Engines an agent session can run. Each id is the `openllm <client>` the
69
74
  * daemon launches the engine through (same overlay, gateway URL, key, model
@@ -100,14 +105,27 @@ export declare const AgentStreamOpenPayload: S.Struct<{
100
105
  readonly kind: S.Literal<"agent">;
101
106
  }>;
102
107
  export type TAgentStreamOpenPayload = S.Schema.Type<typeof AgentStreamOpenPayload>;
108
+ /**
109
+ * One attachment on `send`. Exactly one of `data` (base64, any transport) or
110
+ * `path` (absolute local file, loopback agent socket only — the daemon rejects
111
+ * `path` on mux/RTC). Both together are refused (same XOR pattern as
112
+ * {@link AgentRespondCommand}).
113
+ */
103
114
  export declare const AgentAttachment: S.Struct<{
104
115
  readonly kind: S.Literals<readonly ["image", "file"]>;
105
116
  readonly name: S.optional<S.String>;
106
117
  readonly media_type: S.String;
107
- /** Base64 bytes. */
108
- readonly data: S.String;
118
+ /** Base64 bytes. Mutually exclusive with {@link path}. */
119
+ readonly data: S.optional<S.String>;
120
+ /** Absolute local path; admitted only on the desktop loopback agent socket. */
121
+ readonly path: S.optional<S.String>;
109
122
  }>;
110
123
  export type TAgentAttachment = S.Schema.Type<typeof AgentAttachment>;
124
+ /** True when the attachment carries an absolute local path instead of bytes. */
125
+ export declare const agentAttachmentHasPath: (attachment: TAgentAttachment) => attachment is TAgentAttachment & {
126
+ readonly path: string;
127
+ readonly data?: undefined;
128
+ };
111
129
  /**
112
130
  * Metadata for an attachment that was saved on the device for a turn. The
113
131
  * `name` is the on-disk basename under the session's attachments dir (what
@@ -175,8 +193,10 @@ export declare const AgentSendCommand: S.Struct<{
175
193
  readonly kind: S.Literals<readonly ["image", "file"]>;
176
194
  readonly name: S.optional<S.String>;
177
195
  readonly media_type: S.String;
178
- /** Base64 bytes. */
179
- readonly data: S.String;
196
+ /** Base64 bytes. Mutually exclusive with {@link path}. */
197
+ readonly data: S.optional<S.String>;
198
+ /** Absolute local path; admitted only on the desktop loopback agent socket. */
199
+ readonly path: S.optional<S.String>;
180
200
  }>>>;
181
201
  }>;
182
202
  export type TAgentSendCommand = S.Schema.Type<typeof AgentSendCommand>;
@@ -335,8 +355,10 @@ export declare const AgentCommand: S.Union<readonly [S.Struct<{
335
355
  readonly kind: S.Literals<readonly ["image", "file"]>;
336
356
  readonly name: S.optional<S.String>;
337
357
  readonly media_type: S.String;
338
- /** Base64 bytes. */
339
- readonly data: S.String;
358
+ /** Base64 bytes. Mutually exclusive with {@link path}. */
359
+ readonly data: S.optional<S.String>;
360
+ /** Absolute local path; admitted only on the desktop loopback agent socket. */
361
+ readonly path: S.optional<S.String>;
340
362
  }>>>;
341
363
  }>, S.Struct<{
342
364
  readonly type: S.Literal<"send_now">;
@@ -72,6 +72,11 @@ export const AGENT_HOOK_EVENTS = [
72
72
  export const AGENT_MAX_ATTACHMENTS = 8;
73
73
  /** Max base64 characters of one inline attachment. */
74
74
  export const AGENT_MAX_ATTACHMENT_BASE64_LENGTH = 6 * 1024 * 1024;
75
+ /**
76
+ * Max characters of one absolute local-path attachment (same bound as a
77
+ * session cwd). Path attachments are loopback-only and never cross the network.
78
+ */
79
+ export const AGENT_MAX_ATTACHMENT_PATH_LENGTH = 1024;
75
80
  /**
76
81
  * Engines an agent session can run. Each id is the `openllm <client>` the
77
82
  * daemon launches the engine through (same overlay, gateway URL, key, model
@@ -120,13 +125,25 @@ export const AgentStreamOpenPayload = S.Struct({
120
125
  // ---------------------------------------------------------------------------
121
126
  // Commands (consumer → daemon)
122
127
  // ---------------------------------------------------------------------------
128
+ /**
129
+ * One attachment on `send`. Exactly one of `data` (base64, any transport) or
130
+ * `path` (absolute local file, loopback agent socket only — the daemon rejects
131
+ * `path` on mux/RTC). Both together are refused (same XOR pattern as
132
+ * {@link AgentRespondCommand}).
133
+ */
123
134
  export const AgentAttachment = S.Struct({
124
135
  kind: S.Literals(["image", "file"]),
125
136
  name: S.optional(S.String.check(S.isMinLength(1), S.isMaxLength(256))),
126
137
  media_type: S.String.check(S.isMinLength(1), S.isMaxLength(128)),
127
- /** Base64 bytes. */
128
- data: S.String.check(S.isMaxLength(AGENT_MAX_ATTACHMENT_BASE64_LENGTH)),
129
- });
138
+ /** Base64 bytes. Mutually exclusive with {@link path}. */
139
+ data: S.optional(S.String.check(S.isMaxLength(AGENT_MAX_ATTACHMENT_BASE64_LENGTH))),
140
+ /** Absolute local path; admitted only on the desktop loopback agent socket. */
141
+ path: S.optional(S.String.check(S.isMinLength(1), S.isMaxLength(AGENT_MAX_ATTACHMENT_PATH_LENGTH))),
142
+ }).check(S.makeFilter((attachment) => (attachment.data === undefined) !== (attachment.path === undefined), {
143
+ message: "attachment carries exactly one of data / path",
144
+ }));
145
+ /** True when the attachment carries an absolute local path instead of bytes. */
146
+ export const agentAttachmentHasPath = (attachment) => typeof attachment.path === "string";
130
147
  /**
131
148
  * Metadata for an attachment that was saved on the device for a turn. The
132
149
  * `name` is the on-disk basename under the session's attachments dir (what
@@ -1,5 +1,5 @@
1
1
  import { Schema as S } from "effect";
2
- export declare const CooldownReason: S.Literals<readonly ["network", "timeout", "rate_limit", "quota_exhausted", "server_error", "auth", "payment", "not_found", "payload_too_large", "unprocessable", "request_invalid", "accepted_unreadable", "context_overflow", "content_filter", "upstream_rejection"]>;
2
+ export declare const CooldownReason: S.Literals<readonly ["network", "timeout", "upstream_stalled", "rate_limit", "quota_exhausted", "server_error", "auth", "payment", "not_found", "payload_too_large", "unprocessable", "request_invalid", "accepted_unreadable", "context_overflow", "content_filter", "upstream_rejection"]>;
3
3
  export type TCooldownReason = S.Schema.Type<typeof CooldownReason>;
4
4
  export type THopAction = "retry_in_place" | "cool_and_advance" | "walk_no_cool" | "surface";
5
5
  /**
@@ -2,6 +2,7 @@ import { Schema as S } from "effect";
2
2
  export const CooldownReason = S.Literals([
3
3
  "network",
4
4
  "timeout",
5
+ "upstream_stalled",
5
6
  "rate_limit",
6
7
  "quota_exhausted",
7
8
  "server_error",
@@ -19,6 +20,9 @@ export const CooldownReason = S.Literals([
19
20
  const cooldownPolicies = {
20
21
  network: { action: "retry_in_place" },
21
22
  timeout: { action: "retry_in_place" },
23
+ // Streaming hop went silent before any client byte (headers or body). Cool
24
+ // briefly so a dead provider isn't retried immediately for the same turn.
25
+ upstream_stalled: { action: "cool_and_advance", ttlMs: 60_000 },
22
26
  rate_limit: { action: "cool_and_advance", ttlMs: 180_000 },
23
27
  quota_exhausted: { action: "cool_and_advance", ttlMs: 180_000 },
24
28
  server_error: { action: "retry_in_place" },
@@ -509,7 +509,7 @@ export declare const DaemonRecordRequest: S.Struct<{
509
509
  * cloud charges each one the hold's full reservation.
510
510
  */
511
511
  readonly hold_unknown_attempts: S.optional<S.Number>;
512
- readonly cooldown_reason: S.optional<S.Literals<readonly ["network", "timeout", "rate_limit", "quota_exhausted", "server_error", "auth", "payment", "not_found", "payload_too_large", "unprocessable", "request_invalid", "accepted_unreadable", "context_overflow", "content_filter", "upstream_rejection"]>>;
512
+ readonly cooldown_reason: S.optional<S.Literals<readonly ["network", "timeout", "upstream_stalled", "rate_limit", "quota_exhausted", "server_error", "auth", "payment", "not_found", "payload_too_large", "unprocessable", "request_invalid", "accepted_unreadable", "context_overflow", "content_filter", "upstream_rejection"]>>;
513
513
  /** Reset instant for reset-aware TTL cap; optional so old daemons omit. */
514
514
  readonly reset_at_ms: S.optional<S.Number>;
515
515
  }>;
@@ -1390,6 +1390,8 @@ export declare const DaemonProviderConnection: S.Struct<{
1390
1390
  readonly host_signed_in: S.optional<S.Boolean>;
1391
1391
  readonly detail: S.optional<S.String>;
1392
1392
  readonly last_login_at_ms: S.optional<S.NullOr<S.Number>>;
1393
+ /** Wall-clock time of the latest real provider status observation. */
1394
+ readonly last_probed_at_ms: S.optional<S.Number>;
1393
1395
  /** A LIVE device-code flow awaiting the user (codex/kimi on a remote box):
1394
1396
  * the verification URL + one-time code to surface so the dashboard can
1395
1397
  * render a synced "open this link, enter this code" panel. Present only
@@ -1685,6 +1687,8 @@ export declare const DaemonStatus: S.Struct<{
1685
1687
  readonly host_signed_in: S.optional<S.Boolean>;
1686
1688
  readonly detail: S.optional<S.String>;
1687
1689
  readonly last_login_at_ms: S.optional<S.NullOr<S.Number>>;
1690
+ /** Wall-clock time of the latest real provider status observation. */
1691
+ readonly last_probed_at_ms: S.optional<S.Number>;
1688
1692
  /** A LIVE device-code flow awaiting the user (codex/kimi on a remote box):
1689
1693
  * the verification URL + one-time code to surface so the dashboard can
1690
1694
  * render a synced "open this link, enter this code" panel. Present only
@@ -899,6 +899,8 @@ export const DaemonProviderConnection = S.Struct({
899
899
  host_signed_in: S.optional(S.Boolean),
900
900
  detail: S.optional(S.String),
901
901
  last_login_at_ms: S.optional(S.NullOr(S.Number)),
902
+ /** Wall-clock time of the latest real provider status observation. */
903
+ last_probed_at_ms: S.optional(S.Number),
902
904
  /** A LIVE device-code flow awaiting the user (codex/kimi on a remote box):
903
905
  * the verification URL + one-time code to surface so the dashboard can
904
906
  * render a synced "open this link, enter this code" panel. Present only
@@ -38,6 +38,7 @@ export declare const DeviceState: S.Struct<{
38
38
  readonly host_signed_in: S.optional<S.Boolean>;
39
39
  readonly detail: S.optional<S.String>;
40
40
  readonly last_login_at_ms: S.optional<S.NullOr<S.Number>>;
41
+ readonly last_probed_at_ms: S.optional<S.Number>;
41
42
  readonly pending_auth: S.optional<S.NullOr<S.Struct<{
42
43
  readonly url: S.optional<S.String>;
43
44
  readonly code: S.optional<S.String>;
@@ -147,6 +148,7 @@ export declare const RelayDeviceStateFrame: S.Union<readonly [S.Struct<{
147
148
  readonly host_signed_in: S.optional<S.Boolean>;
148
149
  readonly detail: S.optional<S.String>;
149
150
  readonly last_login_at_ms: S.optional<S.NullOr<S.Number>>;
151
+ readonly last_probed_at_ms: S.optional<S.Number>;
150
152
  readonly pending_auth: S.optional<S.NullOr<S.Struct<{
151
153
  readonly url: S.optional<S.String>;
152
154
  readonly code: S.optional<S.String>;
@@ -21,6 +21,7 @@ const MEDIA_NO_RETRY_REASONS = new Set([
21
21
  "server_error",
22
22
  "timeout",
23
23
  "network",
24
+ "upstream_stalled",
24
25
  ]);
25
26
  /**
26
27
  * Media overlay on the shared hop classifier.
@@ -197,6 +197,14 @@ export declare const TunnelForwardHeaders: S.Struct<{
197
197
  * bound to the key's cap hold. Uncapped keys omit it.
198
198
  */
199
199
  readonly hop_grant: S.optional<S.String>;
200
+ /**
201
+ * The subscription provider a fleet consumer could not serve itself and
202
+ * is asking this peer for. Mapped to {@link FLEET_PROVIDER_HEADER}; a peer
203
+ * that can't serve it answers with {@link FLEET_DECLINE_HEADER} instead of
204
+ * walking on to a later hop.
205
+ */
206
+ readonly fleet_provider: S.optional<S.String>;
207
+ readonly fleet_model: S.optional<S.String>;
200
208
  }>;
201
209
  export type TTunnelForwardHeaders = S.Schema.Type<typeof TunnelForwardHeaders>;
202
210
  /** Response metadata carried by the mux `res_head` CTRL payload.
@@ -206,6 +214,14 @@ export type TTunnelForwardHeaders = S.Schema.Type<typeof TunnelForwardHeaders>;
206
214
  */
207
215
  export declare const TUNNEL_RESOLVED_MODEL_MAX = 256;
208
216
  export declare const TUNNEL_CHAIN_MAX = 4096;
217
+ /** Request header: the provider a fleet consumer wants this peer to serve. */
218
+ export declare const FLEET_PROVIDER_HEADER = "x-openllm-fleet-provider";
219
+ export declare const FLEET_MODEL_HEADER = "x-openllm-fleet-model";
220
+ /** Response header: this peer can't serve the requested fleet provider. */
221
+ export declare const FLEET_DECLINE_HEADER = "x-openllm-fleet-decline";
222
+ export declare const FLEET_DECLINE_CAUSES: readonly ["not_signed_in", "no_quota", "cooling", "unavailable"];
223
+ export type TFleetDeclineCause = (typeof FLEET_DECLINE_CAUSES)[number];
224
+ export declare const isFleetDeclineCause: (value: string | null) => value is TFleetDeclineCause;
209
225
  export declare const TunnelResponseHeaders: S.Struct<{
210
226
  readonly media_url: S.optional<S.String>;
211
227
  readonly media_persistence: S.optional<S.Literal<"browser">>;
@@ -217,6 +233,8 @@ export declare const TunnelResponseHeaders: S.Struct<{
217
233
  readonly resolved_model: S.optional<S.String>;
218
234
  /** Comma-separated attempted hops — maps to {@link OPENLLM_CHAIN_HEADER}. */
219
235
  readonly chain: S.optional<S.String>;
236
+ /** Maps to {@link FLEET_DECLINE_HEADER}. */
237
+ readonly fleet_decline: S.optional<S.Literals<readonly ["not_signed_in", "no_quota", "cooling", "unavailable"]>>;
220
238
  }>;
221
239
  export type TTunnelResponseHeaders = S.Schema.Type<typeof TunnelResponseHeaders>;
222
240
  /** Copy closed mux res_head fields onto an HTTP Headers object. */
@@ -264,6 +282,14 @@ export declare const TunnelStreamOpenPayload: S.Struct<{
264
282
  * bound to the key's cap hold. Uncapped keys omit it.
265
283
  */
266
284
  readonly hop_grant: S.optional<S.String>;
285
+ /**
286
+ * The subscription provider a fleet consumer could not serve itself and
287
+ * is asking this peer for. Mapped to {@link FLEET_PROVIDER_HEADER}; a peer
288
+ * that can't serve it answers with {@link FLEET_DECLINE_HEADER} instead of
289
+ * walking on to a later hop.
290
+ */
291
+ readonly fleet_provider: S.optional<S.String>;
292
+ readonly fleet_model: S.optional<S.String>;
267
293
  }>>;
268
294
  readonly consumer: S.optional<S.Literals<readonly ["browser", "daemon"]>>;
269
295
  }>;
@@ -303,6 +329,14 @@ export declare const StreamOpenPayload: S.Union<readonly [S.Struct<{
303
329
  * bound to the key's cap hold. Uncapped keys omit it.
304
330
  */
305
331
  readonly hop_grant: S.optional<S.String>;
332
+ /**
333
+ * The subscription provider a fleet consumer could not serve itself and
334
+ * is asking this peer for. Mapped to {@link FLEET_PROVIDER_HEADER}; a peer
335
+ * that can't serve it answers with {@link FLEET_DECLINE_HEADER} instead of
336
+ * walking on to a later hop.
337
+ */
338
+ readonly fleet_provider: S.optional<S.String>;
339
+ readonly fleet_model: S.optional<S.String>;
306
340
  }>>;
307
341
  readonly consumer: S.optional<S.Literals<readonly ["browser", "daemon"]>>;
308
342
  }>, S.Struct<{
@@ -341,6 +375,8 @@ export declare const StreamCtrlPayload: S.Union<readonly [S.Struct<{
341
375
  readonly resolved_model: S.optional<S.String>;
342
376
  /** Comma-separated attempted hops — maps to {@link OPENLLM_CHAIN_HEADER}. */
343
377
  readonly chain: S.optional<S.String>;
378
+ /** Maps to {@link FLEET_DECLINE_HEADER}. */
379
+ readonly fleet_decline: S.optional<S.Literals<readonly ["not_signed_in", "no_quota", "cooling", "unavailable"]>>;
344
380
  }>>;
345
381
  }>, S.Struct<{
346
382
  readonly t: S.Literal<"replay_done">;
@@ -361,6 +361,14 @@ export const TunnelForwardHeaders = S.Struct({
361
361
  * bound to the key's cap hold. Uncapped keys omit it.
362
362
  */
363
363
  hop_grant: S.optional(S.String.check(S.isMaxLength(HOP_GRANT_B64_MAX))),
364
+ /**
365
+ * The subscription provider a fleet consumer could not serve itself and
366
+ * is asking this peer for. Mapped to {@link FLEET_PROVIDER_HEADER}; a peer
367
+ * that can't serve it answers with {@link FLEET_DECLINE_HEADER} instead of
368
+ * walking on to a later hop.
369
+ */
370
+ fleet_provider: S.optional(S.String.check(S.isMaxLength(64))),
371
+ fleet_model: S.optional(S.String.check(S.isMaxLength(256))),
364
372
  });
365
373
  /** Response metadata carried by the mux `res_head` CTRL payload.
366
374
  *
@@ -369,6 +377,19 @@ export const TunnelForwardHeaders = S.Struct({
369
377
  */
370
378
  export const TUNNEL_RESOLVED_MODEL_MAX = 256;
371
379
  export const TUNNEL_CHAIN_MAX = 4096;
380
+ /** Request header: the provider a fleet consumer wants this peer to serve. */
381
+ export const FLEET_PROVIDER_HEADER = "x-openllm-fleet-provider";
382
+ export const FLEET_MODEL_HEADER = "x-openllm-fleet-model";
383
+ /** Response header: this peer can't serve the requested fleet provider. */
384
+ export const FLEET_DECLINE_HEADER = "x-openllm-fleet-decline";
385
+ export const FLEET_DECLINE_CAUSES = [
386
+ "not_signed_in",
387
+ "no_quota",
388
+ "cooling",
389
+ "unavailable",
390
+ ];
391
+ export const isFleetDeclineCause = (value) => value !== null &&
392
+ FLEET_DECLINE_CAUSES.includes(value);
372
393
  export const TunnelResponseHeaders = S.Struct({
373
394
  media_url: S.optional(S.String.check(S.isMaxLength(TUNNEL_MEDIA_URL_MAX_LENGTH))),
374
395
  media_persistence: S.optional(S.Literal(MEDIA_PERSISTENCE_BROWSER)),
@@ -380,6 +401,8 @@ export const TunnelResponseHeaders = S.Struct({
380
401
  resolved_model: S.optional(S.String.check(S.isMaxLength(TUNNEL_RESOLVED_MODEL_MAX))),
381
402
  /** Comma-separated attempted hops — maps to {@link OPENLLM_CHAIN_HEADER}. */
382
403
  chain: S.optional(S.String.check(S.isMaxLength(TUNNEL_CHAIN_MAX))),
404
+ /** Maps to {@link FLEET_DECLINE_HEADER}. */
405
+ fleet_decline: S.optional(S.Literals(FLEET_DECLINE_CAUSES)),
383
406
  });
384
407
  const boundedOptional = (value, max) => {
385
408
  if (value === null || value.length === 0 || value.length > max)
@@ -406,6 +429,9 @@ export const applyTunnelResponseHeadersToHttp = (headers, res) => {
406
429
  if (res.chain !== undefined) {
407
430
  headers.set(OPENLLM_CHAIN_HEADER, res.chain);
408
431
  }
432
+ if (res.fleet_decline !== undefined) {
433
+ headers.set(FLEET_DECLINE_HEADER, res.fleet_decline);
434
+ }
409
435
  };
410
436
  /** Project HTTP response headers onto the closed mux res_head struct. */
411
437
  export const tunnelResponseHeadersFromHttp = (headers) => {
@@ -429,6 +455,11 @@ export const tunnelResponseHeadersFromHttp = (headers) => {
429
455
  is_sse: contentType?.includes("text/event-stream") === true,
430
456
  ...(resolved === undefined ? {} : { resolved_model: resolved }),
431
457
  ...(chain === undefined ? {} : { chain }),
458
+ ...(isFleetDeclineCause(headers.get(FLEET_DECLINE_HEADER))
459
+ ? {
460
+ fleet_decline: headers.get(FLEET_DECLINE_HEADER),
461
+ }
462
+ : {}),
432
463
  };
433
464
  };
434
465
  /** A channel-level admission failure. */
@@ -613,6 +644,8 @@ export const parseStreamOpenPayload = (value) => {
613
644
  "video_id",
614
645
  "team_grant",
615
646
  "hop_grant",
647
+ "fleet_provider",
648
+ "fleet_model",
616
649
  ])) {
617
650
  return null;
618
651
  }
@@ -7,10 +7,11 @@ export declare const PEER_LINK_DEFAULTS: {
7
7
  readonly rtcIceMs: 20000;
8
8
  readonly idleTtlMs: 90000;
9
9
  readonly failureTtlMs: 60000;
10
+ readonly presenceFailureTtlMs: 5000;
10
11
  };
11
12
  export type TPeerLinkPath = "rtc" | "mux";
12
13
  export type TPeerLinkRtcMode = "use-if-open+warm" | "probe";
13
- export type TPeerLinkFailureReason = "aborted" | "not_capable" | "daemon_outdated" | "overloaded" | "tunnel_busy" | "transport" | "unknown";
14
+ export type TPeerLinkFailureReason = "aborted" | "presence" | "relay_unavailable" | "timeout" | "nack" | "not_capable" | "daemon_outdated" | "overloaded" | "tunnel_busy" | "transport" | "unknown";
14
15
  export type TPeerLinkEvent = {
15
16
  readonly type: "selected" | "failed" | "fallback" | "released";
16
17
  readonly path: TPeerLinkPath;
@@ -66,6 +67,7 @@ export type TPeerLinkFailureCache = {
66
67
  /** Clear one transport's failures without erasing the independent rung. */
67
68
  readonly clearPath: (path: TPeerLinkPath) => void;
68
69
  readonly has: (target: string, path: TPeerLinkPath, now?: number) => boolean;
70
+ readonly reason: (target: string, path: TPeerLinkPath, now?: number) => TPeerLinkFailureReason | null;
69
71
  readonly until: (target: string, path: TPeerLinkPath, now?: number) => number | null;
70
72
  readonly set: (target: string, path: TPeerLinkPath, reason: TPeerLinkFailureReason, now?: number) => void;
71
73
  };
@@ -6,6 +6,7 @@ export const PEER_LINK_DEFAULTS = {
6
6
  rtcIceMs: 20_000,
7
7
  idleTtlMs: 90_000,
8
8
  failureTtlMs: 60_000,
9
+ presenceFailureTtlMs: 5_000,
9
10
  };
10
11
  const failureTtlMs = (reason) => {
11
12
  switch (reason) {
@@ -14,9 +15,10 @@ const failureTtlMs = (reason) => {
14
15
  case "overloaded":
15
16
  case "tunnel_busy":
16
17
  return 2_000;
18
+ case "presence":
17
19
  case "not_capable":
18
20
  case "daemon_outdated":
19
- return 5_000;
21
+ return PEER_LINK_DEFAULTS.presenceFailureTtlMs;
20
22
  default:
21
23
  return PEER_LINK_DEFAULTS.failureTtlMs;
22
24
  }
@@ -31,8 +33,8 @@ const cacheKeyOf = (target, path) => `${target}\u0000${path}`;
31
33
  export const createPeerLinkFailureCache = () => {
32
34
  const failureCache = new Map();
33
35
  const sweepFailureCache = (now) => {
34
- for (const [key, until] of failureCache) {
35
- if (until <= now)
36
+ for (const [key, entry] of failureCache) {
37
+ if (entry.until <= now)
36
38
  failureCache.delete(key);
37
39
  }
38
40
  while (failureCache.size > MAX_FAILURE_CACHE_ENTRIES) {
@@ -68,18 +70,22 @@ export const createPeerLinkFailureCache = () => {
68
70
  const until = failureCache.get(key);
69
71
  if (until === undefined)
70
72
  return false;
71
- if (until > now)
73
+ if (until.until > now)
72
74
  return true;
73
75
  failureCache.delete(key);
74
76
  return false;
75
77
  },
78
+ reason(target, path, now = Date.now()) {
79
+ const entry = failureCache.get(cacheKeyOf(target, path));
80
+ return entry !== undefined && entry.until > now ? entry.reason : null;
81
+ },
76
82
  until(target, path, now = Date.now()) {
77
83
  const key = cacheKeyOf(target, path);
78
84
  const until = failureCache.get(key);
79
85
  if (until === undefined)
80
86
  return null;
81
- if (until > now)
82
- return until;
87
+ if (until.until > now)
88
+ return until.until;
83
89
  failureCache.delete(key);
84
90
  return null;
85
91
  },
@@ -89,7 +95,7 @@ export const createPeerLinkFailureCache = () => {
89
95
  if (ttl > 0) {
90
96
  const key = cacheKeyOf(target, path);
91
97
  failureCache.delete(key);
92
- failureCache.set(key, now + ttl);
98
+ failureCache.set(key, { until: now + ttl, reason });
93
99
  sweepFailureCache(now);
94
100
  }
95
101
  },
@@ -101,6 +107,8 @@ export const createPeerLinkFailureCache = () => {
101
107
  */
102
108
  export const peerLinkFailureCache = createPeerLinkFailureCache();
103
109
  export const peerLinkFailureReasonOf = (error) => {
110
+ if (error instanceof DOMException && error.name === "TimeoutError")
111
+ return "timeout";
104
112
  if (error instanceof DOMException && error.name === "AbortError") {
105
113
  return "aborted";
106
114
  }
@@ -126,7 +134,9 @@ export const createPeerLink = (options) => {
126
134
  const openBudgetMs = options.policy?.openBudgetMs ?? PEER_LINK_DEFAULTS.muxAckSameUserMs;
127
135
  const rtcProbeMs = options.policy?.rtcProbeMs ?? Math.min(2_000, openBudgetMs);
128
136
  const idleTtlMs = options.policy?.idleTtlMs ?? PEER_LINK_DEFAULTS.idleTtlMs;
129
- const classify = options.policy?.classifyFailure ?? peerLinkFailureReasonOf;
137
+ const classify = (error) => error instanceof DOMException && error.name === "TimeoutError"
138
+ ? "timeout"
139
+ : (options.policy?.classifyFailure ?? peerLinkFailureReasonOf)(error);
130
140
  // Retrying a dispatched inference can duplicate provider work. Callers must
131
141
  // prove that the failure happened before dispatch to opt into a fallback.
132
142
  const retrySafe = options.policy?.retrySafe ?? (() => false);
@@ -168,8 +178,8 @@ export const createPeerLink = (options) => {
168
178
  let timer;
169
179
  const timedOut = new Promise((_, reject) => {
170
180
  timer = setTimeout(() => {
181
+ reject(new DOMException("Timed out", "TimeoutError"));
171
182
  controller.abort();
172
- reject(new DOMException("Timed out", "AbortError"));
173
183
  }, timeoutMs);
174
184
  });
175
185
  try {
@@ -249,8 +259,35 @@ export const createPeerLink = (options) => {
249
259
  cache(resolved.path, error);
250
260
  const dialer = resolved.path === "rtc" ? options.rtc : options.mux;
251
261
  dialer?.drop?.(resolved.channel);
252
- if (resolved.path !== "rtc")
253
- throw error;
262
+ if (resolved.path === "mux") {
263
+ if (dialer?.drop === undefined)
264
+ throw error;
265
+ peerLinkFailureCache.clear(options.target, "mux");
266
+ const startedAt = now();
267
+ let fresh;
268
+ try {
269
+ fresh = await dialWithin(options.mux, openBudgetMs);
270
+ }
271
+ catch (dialError) {
272
+ cache("mux", dialError);
273
+ throw error;
274
+ }
275
+ if (fresh === null || fresh === resolved.channel)
276
+ throw error;
277
+ const reopened = fresh;
278
+ try {
279
+ return await fn(resultOf("mux", options.mux, reopened, false, startedAt));
280
+ }
281
+ catch (retryError) {
282
+ // One reopen only: a second pre-dispatch failure is cached and its
283
+ // channel dropped, so the next request honours the failure TTL.
284
+ if (retrySafe(retryError)) {
285
+ cache("mux", retryError);
286
+ dialer.drop(reopened);
287
+ }
288
+ throw retryError;
289
+ }
290
+ }
254
291
  emit({
255
292
  type: "fallback",
256
293
  path: "rtc",
@@ -36,6 +36,7 @@ export * from "./lib/refusal.js";
36
36
  export * from "./lib/secret-scrub.js";
37
37
  export * from "./lib/streaming/accumulate.js";
38
38
  export * from "./lib/streaming/peek.js";
39
+ export * from "./lib/streaming/silence.js";
39
40
  export * from "./lib/streaming/sse.js";
40
41
  export * from "./lib/streaming/strip-tool-calls.js";
41
42
  export * from "./lib/streaming/upstream-error.js";
@@ -36,6 +36,7 @@ export * from "./lib/refusal.js";
36
36
  export * from "./lib/secret-scrub.js";
37
37
  export * from "./lib/streaming/accumulate.js";
38
38
  export * from "./lib/streaming/peek.js";
39
+ export * from "./lib/streaming/silence.js";
39
40
  export * from "./lib/streaming/sse.js";
40
41
  export * from "./lib/streaming/strip-tool-calls.js";
41
42
  export * from "./lib/streaming/upstream-error.js";
@@ -32,11 +32,18 @@ export const reasonFor = (i, rules = errorRulesFor(i.provider)) => {
32
32
  // A 2xx the runner still couldn't use (an undecodable 200): the provider
33
33
  // ACCEPTED the request and may have billed it, so walking on would send —
34
34
  // and bill — it again. Only an explicit in-band refusal (content_filter,
35
- // above) walks from a 2xx; no body matcher or provider rule may turn an
36
- // accepted request into a walk. 3xx falls through to the normal rules
37
- // (typically `upstream_rejection` → walk).
35
+ // above) walks from a 2xx; no body matcher or provider rule — including a
36
+ // forged `upstream_stalled` code on an accepted response — may turn an
37
+ // accepted request into a walk.
38
38
  if (i.status >= 200 && i.status < 300)
39
39
  return "accepted_unreadable";
40
+ // Pre-commit synthetic stall (headers / raw-byte silence). Our runners and
41
+ // the daemon emit this only on non-2xx outcomes (typically 504). Kept AFTER
42
+ // the 2xx guard so an accepted provider response can never be re-POSTed.
43
+ if (i.envelope?.error?.code === "upstream_stalled" ||
44
+ i.envelope?.error?.type === "upstream_stalled") {
45
+ return "upstream_stalled";
46
+ }
40
47
  const message = i.envelope?.error?.message ?? "";
41
48
  if (CONTEXT_OVERFLOW_BODY.test(message))
42
49
  return "context_overflow";
@@ -45,16 +45,16 @@ export declare const isMeaningfulChunk: (chunk: TChatCompletionChunk) => boolean
45
45
  * the module doc for why), then return a stream that replays what was
46
46
  * buffered ahead of the live tail.
47
47
  *
48
- * The wait is UNBOUNDED by design — bounded only by the request's abort
49
- * signal, which the upstream fetch carries (an aborted client errors the
50
- * read). The rejection we're waiting for arrives after the vendor
51
- * tokenizes the prompt, and a near-cap prompt (the exact overflow case)
52
- * prefills for many seconds; any fixed deadline commits the response
53
- * right before the rejection lands, stranding the client in a committed
54
- * stream that then dies (the pre-fix stall). The client loses nothing
55
- * from waiting: it would receive zero bytes before the first event
56
- * regardless. (Same rule as ref/CLIProxyAPI's stream bootstrap: no
57
- * timeouts after the upstream connection is established.)
48
+ * The wait for a MEANINGFUL decoded event is still unbounded here — the
49
+ * request abort signal is the only hard stop at this layer. Raw-byte
50
+ * silence is enforced BELOW the decoder by `withByteSilenceGuard` /
51
+ * `withStreamingHeadersTimeout` (see `silence.ts`): a generous
52
+ * first-byte budget (default 300s, `OPENLLM_STREAM_FIRST_BYTE_TIMEOUT_MS`)
53
+ * covers near-cap prefills that produce no keepalive, and a shorter gap
54
+ * budget (default 120s, `OPENLLM_STREAM_SILENCE_TIMEOUT_MS`) trips only
55
+ * after the upstream has already spoken. That split keeps the overflow
56
+ * peek working for long prefills without letting a truly silent hop
57
+ * burn the Vercel deadline as a fake `length` success.
58
58
  *
59
59
  * A stream that CLOSES before any event is an error by default (walkable
60
60
  * — the upstream produced nothing), not an empty success. EXCEPT when the
@@ -30,16 +30,16 @@ export const isMeaningfulChunk = (chunk) => {
30
30
  * the module doc for why), then return a stream that replays what was
31
31
  * buffered ahead of the live tail.
32
32
  *
33
- * The wait is UNBOUNDED by design — bounded only by the request's abort
34
- * signal, which the upstream fetch carries (an aborted client errors the
35
- * read). The rejection we're waiting for arrives after the vendor
36
- * tokenizes the prompt, and a near-cap prompt (the exact overflow case)
37
- * prefills for many seconds; any fixed deadline commits the response
38
- * right before the rejection lands, stranding the client in a committed
39
- * stream that then dies (the pre-fix stall). The client loses nothing
40
- * from waiting: it would receive zero bytes before the first event
41
- * regardless. (Same rule as ref/CLIProxyAPI's stream bootstrap: no
42
- * timeouts after the upstream connection is established.)
33
+ * The wait for a MEANINGFUL decoded event is still unbounded here — the
34
+ * request abort signal is the only hard stop at this layer. Raw-byte
35
+ * silence is enforced BELOW the decoder by `withByteSilenceGuard` /
36
+ * `withStreamingHeadersTimeout` (see `silence.ts`): a generous
37
+ * first-byte budget (default 300s, `OPENLLM_STREAM_FIRST_BYTE_TIMEOUT_MS`)
38
+ * covers near-cap prefills that produce no keepalive, and a shorter gap
39
+ * budget (default 120s, `OPENLLM_STREAM_SILENCE_TIMEOUT_MS`) trips only
40
+ * after the upstream has already spoken. That split keeps the overflow
41
+ * peek working for long prefills without letting a truly silent hop
42
+ * burn the Vercel deadline as a fake `length` success.
43
43
  *
44
44
  * A stream that CLOSES before any event is an error by default (walkable
45
45
  * — the upstream produced nothing), not an empty success. EXCEPT when the
@@ -0,0 +1,93 @@
1
+ /**
2
+ * Pre-commit silence budgets for STREAMING hops only.
3
+ *
4
+ * Two clocks, both measured on raw upstream bytes (any enqueued chunk —
5
+ * including empty ones, keepalive comments, and pings — counts as alive):
6
+ *
7
+ * 1. **First-byte** (`OPENLLM_STREAM_FIRST_BYTE_TIMEOUT_MS`, default 300s)
8
+ * — waiting for response headers, and the gap before the first body
9
+ * byte. Near-cap prefills (hundreds of thousands of tokens) can
10
+ * legitimately take minutes with no keepalive; this budget is the
11
+ * one that must accommodate them. See `peek.ts`.
12
+ * 2. **Gap** (`OPENLLM_STREAM_SILENCE_TIMEOUT_MS`, default 120s) — after
13
+ * any byte has arrived, until `release()` (hop commit). A provider
14
+ * that went silent mid-prefill after talking once is treated as
15
+ * stalled.
16
+ *
17
+ * Non-streaming requests get no short timeout — they legitimately send
18
+ * nothing until the full JSON body is ready. A non-positive override
19
+ * disables that budget.
20
+ */
21
+ export declare const DEFAULT_STREAM_FIRST_BYTE_TIMEOUT_MS = 300000;
22
+ export declare const DEFAULT_STREAM_SILENCE_TIMEOUT_MS = 120000;
23
+ export declare const streamFirstByteTimeoutMs: () => number;
24
+ export declare const streamSilenceTimeoutMs: () => number;
25
+ /**
26
+ * Whether a hop should apply the streaming silence budgets on its
27
+ * upstream fetch. ChatGPT's Codex endpoint always streams even when the
28
+ * client asked for JSON, so the gate is `alwaysStreams || clientWantsStream`
29
+ * — not the client flag alone.
30
+ */
31
+ export declare const streamingHopNeedsSilence: (opts: {
32
+ readonly upstreamAlwaysStreams: boolean;
33
+ readonly clientWantsStream: boolean;
34
+ }) => boolean;
35
+ export declare const firstByteDeadlineAt: (firstByteTimeoutMs: number) => number | null;
36
+ /**
37
+ * Time left before `deadlineAt`. `null` (no budget) returns 0, which every
38
+ * guard reads as disabled; an EXPIRED deadline returns 1 so the guard fires
39
+ * at once instead of being read as disabled.
40
+ */
41
+ export declare const remainingTimeoutMs: (deadlineAt: number | null) => number;
42
+ /**
43
+ * Read a (non-2xx) response body with `read`, failing with
44
+ * {@link UpstreamStalledError} if it takes longer than `timeoutMs`. The body
45
+ * is read through a reader this helper owns, so a stall can actually cancel
46
+ * it (a `text()` in flight locks the body, and `body.cancel()` would throw).
47
+ * `timeoutMs <= 0` disables the budget.
48
+ */
49
+ export declare const readResponseWithin: <T>(response: Response, timeoutMs: number, read: (response: Response) => Promise<T>) => Promise<T>;
50
+ export type TStreamingHeadersGuard = {
51
+ readonly signal: AbortSignal;
52
+ /** True when the first-byte headers timer fired and the caller did not abort. */
53
+ readonly didStall: () => boolean;
54
+ /** Clear the headers timer once the Response headers have arrived. */
55
+ readonly clear: () => void;
56
+ };
57
+ /**
58
+ * AbortSignal that fires if response headers take longer than
59
+ * `firstByteTimeoutMs`. Cleared as soon as headers arrive so it cannot
60
+ * abort the body later. Caller abort still wins.
61
+ */
62
+ export declare const withStreamingHeadersTimeout: (caller: AbortSignal, firstByteTimeoutMs: number) => TStreamingHeadersGuard;
63
+ export type TByteSilenceGuard = {
64
+ readonly stream: ReadableStream<Uint8Array>;
65
+ /**
66
+ * Disarm the silence timer. Call once the hop commits (first chunk is
67
+ * about to be forwarded to the client) so mid-stream quiet no longer
68
+ * walks the fallback chain.
69
+ */
70
+ readonly release: () => void;
71
+ };
72
+ export type TByteSilenceGuardOptions = {
73
+ /** Budget waiting for the first body byte (default 300s). */
74
+ readonly firstByteTimeoutMs: number;
75
+ /** Budget for gaps after any byte until `release()` (default 120s). */
76
+ readonly gapTimeoutMs: number;
77
+ };
78
+ /**
79
+ * Wrap a raw upstream byte stream with the two-budget silence policy.
80
+ * Errors the stream with {@link UpstreamStalledError} on a trip before
81
+ * `release()`.
82
+ */
83
+ export declare const withByteSilenceGuard: (source: ReadableStream<Uint8Array>, options: TByteSilenceGuardOptions) => TByteSilenceGuard;
84
+ /** Envelope used when a streaming headers / body wait stalls. */
85
+ export declare const upstreamStalledEnvelope: (message?: string) => {
86
+ readonly error: {
87
+ readonly message: string;
88
+ readonly type: "upstream_stalled";
89
+ readonly code: "upstream_stalled";
90
+ };
91
+ };
92
+ /** Shared HTTP status for a pre-commit upstream stall (cloud + daemon). */
93
+ export declare const UPSTREAM_STALLED_HTTP_STATUS = 504;
@@ -0,0 +1,262 @@
1
+ import { UpstreamStalledError } from "./upstream-error.js";
2
+ /**
3
+ * Pre-commit silence budgets for STREAMING hops only.
4
+ *
5
+ * Two clocks, both measured on raw upstream bytes (any enqueued chunk —
6
+ * including empty ones, keepalive comments, and pings — counts as alive):
7
+ *
8
+ * 1. **First-byte** (`OPENLLM_STREAM_FIRST_BYTE_TIMEOUT_MS`, default 300s)
9
+ * — waiting for response headers, and the gap before the first body
10
+ * byte. Near-cap prefills (hundreds of thousands of tokens) can
11
+ * legitimately take minutes with no keepalive; this budget is the
12
+ * one that must accommodate them. See `peek.ts`.
13
+ * 2. **Gap** (`OPENLLM_STREAM_SILENCE_TIMEOUT_MS`, default 120s) — after
14
+ * any byte has arrived, until `release()` (hop commit). A provider
15
+ * that went silent mid-prefill after talking once is treated as
16
+ * stalled.
17
+ *
18
+ * Non-streaming requests get no short timeout — they legitimately send
19
+ * nothing until the full JSON body is ready. A non-positive override
20
+ * disables that budget.
21
+ */
22
+ export const DEFAULT_STREAM_FIRST_BYTE_TIMEOUT_MS = 300_000;
23
+ export const DEFAULT_STREAM_SILENCE_TIMEOUT_MS = 120_000;
24
+ const readTimeoutMs = (envName, fallback) => {
25
+ const raw = process.env[envName];
26
+ if (raw === undefined || raw === "")
27
+ return fallback;
28
+ const override = Number(raw);
29
+ if (!Number.isFinite(override))
30
+ return fallback;
31
+ return override;
32
+ };
33
+ export const streamFirstByteTimeoutMs = () => readTimeoutMs("OPENLLM_STREAM_FIRST_BYTE_TIMEOUT_MS", DEFAULT_STREAM_FIRST_BYTE_TIMEOUT_MS);
34
+ export const streamSilenceTimeoutMs = () => readTimeoutMs("OPENLLM_STREAM_SILENCE_TIMEOUT_MS", DEFAULT_STREAM_SILENCE_TIMEOUT_MS);
35
+ /**
36
+ * Whether a hop should apply the streaming silence budgets on its
37
+ * upstream fetch. ChatGPT's Codex endpoint always streams even when the
38
+ * client asked for JSON, so the gate is `alwaysStreams || clientWantsStream`
39
+ * — not the client flag alone.
40
+ */
41
+ export const streamingHopNeedsSilence = (opts) => opts.upstreamAlwaysStreams || opts.clientWantsStream;
42
+ /**
43
+ * Absolute deadline for the combined headers + first-body-byte budget.
44
+ * Pass the remaining time into the body silence guard so the two waits
45
+ * share one first-byte window instead of stacking to ~2×.
46
+ */
47
+ const STALL_ABORT_REASON = "openllm-upstream-stalled";
48
+ export const firstByteDeadlineAt = (firstByteTimeoutMs) => firstByteTimeoutMs > 0 ? Date.now() + firstByteTimeoutMs : null;
49
+ /**
50
+ * Time left before `deadlineAt`. `null` (no budget) returns 0, which every
51
+ * guard reads as disabled; an EXPIRED deadline returns 1 so the guard fires
52
+ * at once instead of being read as disabled.
53
+ */
54
+ export const remainingTimeoutMs = (deadlineAt) => deadlineAt === null ? 0 : Math.max(1, deadlineAt - Date.now());
55
+ /**
56
+ * Read a (non-2xx) response body with `read`, failing with
57
+ * {@link UpstreamStalledError} if it takes longer than `timeoutMs`. The body
58
+ * is read through a reader this helper owns, so a stall can actually cancel
59
+ * it (a `text()` in flight locks the body, and `body.cancel()` would throw).
60
+ * `timeoutMs <= 0` disables the budget.
61
+ */
62
+ export const readResponseWithin = async (response, timeoutMs, read) => {
63
+ if (!(timeoutMs > 0) || response.body === null)
64
+ return read(response);
65
+ const reader = response.body.getReader();
66
+ const owned = new Response(new ReadableStream({
67
+ pull: async (controller) => {
68
+ const next = await reader.read();
69
+ if (next.done)
70
+ controller.close();
71
+ else
72
+ controller.enqueue(next.value);
73
+ },
74
+ cancel: async (reason) => {
75
+ await reader.cancel(reason);
76
+ },
77
+ }), {
78
+ status: response.status,
79
+ statusText: response.statusText,
80
+ headers: response.headers,
81
+ });
82
+ let timer;
83
+ const stalled = new Promise((_, reject) => {
84
+ timer = setTimeout(() => {
85
+ reader.cancel(STALL_ABORT_REASON).catch(() => { });
86
+ reject(new UpstreamStalledError("upstream stalled reading error body"));
87
+ }, timeoutMs);
88
+ });
89
+ try {
90
+ return await Promise.race([read(owned), stalled]);
91
+ }
92
+ finally {
93
+ clearTimeout(timer);
94
+ }
95
+ };
96
+ /**
97
+ * AbortSignal that fires if response headers take longer than
98
+ * `firstByteTimeoutMs`. Cleared as soon as headers arrive so it cannot
99
+ * abort the body later. Caller abort still wins.
100
+ */
101
+ export const withStreamingHeadersTimeout = (caller, firstByteTimeoutMs) => {
102
+ if (!(firstByteTimeoutMs > 0)) {
103
+ return {
104
+ signal: caller,
105
+ didStall: () => false,
106
+ clear: () => { },
107
+ };
108
+ }
109
+ const stall = new AbortController();
110
+ let stalled = false;
111
+ const timer = setTimeout(() => {
112
+ stalled = true;
113
+ stall.abort(STALL_ABORT_REASON);
114
+ }, firstByteTimeoutMs);
115
+ const clear = () => {
116
+ clearTimeout(timer);
117
+ };
118
+ if (caller.aborted) {
119
+ clear();
120
+ stall.abort(caller.reason);
121
+ }
122
+ else {
123
+ caller.addEventListener("abort", () => {
124
+ clear();
125
+ stall.abort(caller.reason);
126
+ }, { once: true });
127
+ }
128
+ return {
129
+ signal: stall.signal,
130
+ didStall: () => stalled && !caller.aborted,
131
+ clear,
132
+ };
133
+ };
134
+ /**
135
+ * Wrap a raw upstream byte stream with the two-budget silence policy.
136
+ * Errors the stream with {@link UpstreamStalledError} on a trip before
137
+ * `release()`.
138
+ */
139
+ export const withByteSilenceGuard = (source, options) => {
140
+ const firstMs = options.firstByteTimeoutMs;
141
+ const gapMs = options.gapTimeoutMs;
142
+ if (!(firstMs > 0) && !(gapMs > 0)) {
143
+ return { stream: source, release: () => { } };
144
+ }
145
+ const reader = source.getReader();
146
+ let released = false;
147
+ let finished = false;
148
+ let sawByte = false;
149
+ let timer = null;
150
+ const STALL = Symbol("upstream-silence");
151
+ const RELEASED = Symbol("upstream-silence-released");
152
+ let settleRace = null;
153
+ let racePromise = null;
154
+ const clearTimer = () => {
155
+ if (timer !== null) {
156
+ clearTimeout(timer);
157
+ timer = null;
158
+ }
159
+ };
160
+ const currentBudgetMs = () => (sawByte ? gapMs : firstMs);
161
+ const armTimer = () => {
162
+ clearTimer();
163
+ if (released || finished)
164
+ return;
165
+ const budget = currentBudgetMs();
166
+ if (!(budget > 0)) {
167
+ racePromise = null;
168
+ settleRace = null;
169
+ return;
170
+ }
171
+ racePromise = new Promise((resolve) => {
172
+ settleRace = resolve;
173
+ timer = setTimeout(() => {
174
+ timer = null;
175
+ resolve(STALL);
176
+ }, budget);
177
+ });
178
+ };
179
+ const release = () => {
180
+ released = true;
181
+ clearTimer();
182
+ const settle = settleRace;
183
+ settleRace = null;
184
+ racePromise = null;
185
+ // Resolve any in-flight race so a late STALL is not observed as a stall.
186
+ if (settle !== null)
187
+ settle(RELEASED);
188
+ };
189
+ armTimer();
190
+ const stream = new ReadableStream({
191
+ async pull(controller) {
192
+ if (finished) {
193
+ controller.close();
194
+ return;
195
+ }
196
+ if (released) {
197
+ const read = await reader.read();
198
+ if (read.done) {
199
+ finished = true;
200
+ clearTimer();
201
+ controller.close();
202
+ return;
203
+ }
204
+ controller.enqueue(read.value);
205
+ return;
206
+ }
207
+ const read = reader.read();
208
+ read.catch(() => { });
209
+ if (racePromise === null)
210
+ armTimer();
211
+ const race = racePromise;
212
+ let result;
213
+ try {
214
+ result = race === null ? await read : await Promise.race([read, race]);
215
+ }
216
+ catch (err) {
217
+ finished = true;
218
+ clearTimer();
219
+ throw err;
220
+ }
221
+ if (result === RELEASED || (result === STALL && released)) {
222
+ // Commit raced the timer: wait for the outstanding read without stalling.
223
+ result = await read;
224
+ }
225
+ if (result === STALL) {
226
+ finished = true;
227
+ clearTimer();
228
+ reader.cancel(STALL_ABORT_REASON).catch(() => { });
229
+ controller.error(new UpstreamStalledError("upstream stalled before producing output"));
230
+ return;
231
+ }
232
+ if (result.done) {
233
+ finished = true;
234
+ clearTimer();
235
+ controller.close();
236
+ return;
237
+ }
238
+ // Any enqueued chunk — including empty — counts as alive.
239
+ sawByte = true;
240
+ if (!released) {
241
+ armTimer();
242
+ }
243
+ controller.enqueue(result.value);
244
+ },
245
+ cancel(reason) {
246
+ finished = true;
247
+ clearTimer();
248
+ reader.cancel(reason).catch(() => { });
249
+ },
250
+ });
251
+ return { stream, release };
252
+ };
253
+ /** Envelope used when a streaming headers / body wait stalls. */
254
+ export const upstreamStalledEnvelope = (message = "upstream stalled before producing output") => ({
255
+ error: {
256
+ message,
257
+ type: "upstream_stalled",
258
+ code: "upstream_stalled",
259
+ },
260
+ });
261
+ /** Shared HTTP status for a pre-commit upstream stall (cloud + daemon). */
262
+ export const UPSTREAM_STALLED_HTTP_STATUS = 504;
@@ -78,15 +78,20 @@ export declare const withFrameAlignedHeartbeat: (source: ReadableStream<Uint8Arr
78
78
  *
79
79
  * Wrap BEFORE `.tee()` so the usage-drain branch also terminates.
80
80
  */
81
+ export type TStreamDeadlineTruncateInfo = {
82
+ /** True when at least one content / tool / reasoning delta was forwarded. */
83
+ readonly sawOutput: boolean;
84
+ };
81
85
  export declare const withStreamDeadline: (source: ReadableStream<TChatCompletionChunk>, options: {
82
86
  readonly deadlineMs: number;
83
87
  /**
84
88
  * Fires exactly once if the deadline expires before the source
85
89
  * closes naturally. Callers use this to skip caching a body that
86
90
  * was sealed by the synthetic terminator instead of the upstream's
87
- * own `finish_reason`. Omit to ignore truncation.
91
+ * own `finish_reason`, and to log zero-output truncations as errors.
92
+ * Omit to ignore truncation.
88
93
  */
89
- readonly onTruncate?: () => void;
94
+ readonly onTruncate?: (info: TStreamDeadlineTruncateInfo) => void;
90
95
  }) => ReadableStream<TChatCompletionChunk>;
91
96
  /**
92
97
  * Result of wrapping a raw byte stream with a deadline guard.
@@ -99,6 +104,8 @@ export declare const withStreamDeadline: (source: ReadableStream<TChatCompletion
99
104
  export type TByteStreamDeadlineHandle = {
100
105
  readonly stream: ReadableStream<Uint8Array>;
101
106
  readonly isTruncated: () => boolean;
107
+ /** True when the deadline fired with no non-keepalive output forwarded. */
108
+ readonly isZeroOutputTruncate: () => boolean;
102
109
  };
103
110
  /**
104
111
  * Sibling of `withStreamDeadline` for the Anthropic pass-through:
@@ -106,7 +113,7 @@ export type TByteStreamDeadlineHandle = {
106
113
  * the synthetic terminator must be a pre-encoded SSE byte sequence
107
114
  * because there is no canonical-decode step in the pipeline.
108
115
  *
109
- * On cutoff:
116
+ * On cutoff with partial output:
110
117
  * 1. cancel the source — propagating to the upstream `fetch` body
111
118
  * so we stop paying the provider for tokens we'll never deliver;
112
119
  * 2. enqueue ONE pre-encoded `message_delta` + `message_stop` pair
@@ -115,6 +122,10 @@ export type TByteStreamDeadlineHandle = {
115
122
  * 3. close cleanly so SSE clients see a terminated turn instead of
116
123
  * a severed connection.
117
124
  *
125
+ * On cutoff with ZERO output: emit an Anthropic `event: error` instead
126
+ * of a clean `max_tokens` turn, so the client and the usage log treat
127
+ * it as a failure.
128
+ *
118
129
  * Wrap BEFORE `.tee()` so the usage-drain branch also receives the
119
130
  * terminator and the accumulator can finalise. The caller checks
120
131
  * `isTruncated()` after consumption to skip caching a truncated body.
@@ -1,3 +1,4 @@
1
+ import { UpstreamStreamError } from "./upstream-error.js";
1
2
  const SSE_DONE = "[DONE]";
2
3
  const textEncoder = new TextEncoder();
3
4
  export const encodeSseEvent = (data) => {
@@ -391,33 +392,19 @@ export const withFrameAlignedHeartbeat = (source, options) => {
391
392
  },
392
393
  });
393
394
  };
394
- /**
395
- * Self-imposed graceful deadline for a CANONICAL chunk stream.
396
- *
397
- * Vercel hard-kills a function the instant it hits `maxDuration`
398
- * (clamped to the plan ceiling, which may be well below the declared
399
- * value). Mid-stream that severs the SSE connection with no terminal
400
- * frame: the client SDK sees a truncated stream and throws — it
401
- * cannot distinguish a truncation from a crash, so it cannot
402
- * continue. The idle heartbeat does NOT help here; it keeps the
403
- * socket warm right up until the platform guillotine.
404
- *
405
- * `withStreamDeadline` cuts us off a touch early instead. It forwards
406
- * real chunks until `deadlineMs` elapses, then:
407
- * 1. cancels the source — propagating to the upstream `fetch` body
408
- * so we stop paying the provider for tokens we'll never deliver;
409
- * 2. emits ONE synthetic terminal chunk with
410
- * `finish_reason: "length"` and closes cleanly.
411
- *
412
- * The text deltas already streamed ARE the partial answer (they're in
413
- * the client's hands). The synthetic terminal makes the SSE encoders
414
- * close properly — `data: [DONE]` for OpenAI, `message_delta`
415
- * (`stop_reason: "max_tokens"`) + `message_stop` for Anthropic — so
416
- * the caller gets a clean, truncated turn it can re-prompt to resume,
417
- * instead of a thrown network error.
418
- *
419
- * Wrap BEFORE `.tee()` so the usage-drain branch also terminates.
420
- */
395
+ const chunkHasOutputDelta = (chunk) => {
396
+ const nonEmpty = (s) => typeof s === "string" && s.length > 0;
397
+ const nonEmptyArr = (a) => Array.isArray(a) && a.length > 0;
398
+ return chunk.choices.some((choice) => {
399
+ const d = choice.delta;
400
+ return (nonEmpty(d.content) ||
401
+ nonEmpty(d.reasoning_content) ||
402
+ nonEmpty(d.refusal) ||
403
+ nonEmptyArr(d.tool_calls) ||
404
+ nonEmptyArr(d.reasoning_items) ||
405
+ nonEmptyArr(d.server_search_calls));
406
+ });
407
+ };
421
408
  export const withStreamDeadline = (source, options) => {
422
409
  const reader = source.getReader();
423
410
  let lastId = "chatcmpl-openllm-deadline";
@@ -425,6 +412,7 @@ export const withStreamDeadline = (source, options) => {
425
412
  let lastCreated = 0;
426
413
  let hasCreated = false;
427
414
  let finished = false;
415
+ let sawOutput = false;
428
416
  const DEADLINE = Symbol("deadline");
429
417
  let timer = null;
430
418
  const deadline = new Promise((resolve) => {
@@ -476,12 +464,19 @@ export const withStreamDeadline = (source, options) => {
476
464
  reader.cancel("openllm-stream-deadline").catch(() => { });
477
465
  if (options.onTruncate !== undefined) {
478
466
  try {
479
- options.onTruncate();
467
+ options.onTruncate({ sawOutput });
480
468
  }
481
469
  catch {
482
470
  // Callback errors must not crash the stream.
483
471
  }
484
472
  }
473
+ if (!sawOutput) {
474
+ // Zero-output deadline: a clean `length` turn would look like a
475
+ // successful empty answer and skip the fallback chain's chance to
476
+ // have walked (already committed). Surface a real error frame.
477
+ controller.error(new UpstreamStreamError("upstream_stalled", "stream deadline reached with no output"));
478
+ return;
479
+ }
485
480
  controller.enqueue(terminal());
486
481
  controller.close();
487
482
  return;
@@ -493,6 +488,9 @@ export const withStreamDeadline = (source, options) => {
493
488
  return;
494
489
  }
495
490
  const value = r.value;
491
+ if (chunkHasOutputDelta(value)) {
492
+ sawOutput = true;
493
+ }
496
494
  if (typeof value.id === "string" && value.id.length > 0) {
497
495
  lastId = value.id;
498
496
  }
@@ -512,13 +510,22 @@ export const withStreamDeadline = (source, options) => {
512
510
  },
513
511
  });
514
512
  };
513
+ const byteChunkLooksLikeOutput = (bytes) => {
514
+ // Only content / tool / reasoning deltas count as partial output.
515
+ // Housekeeping (`message_start`, role-only frames) and keepalives must
516
+ // NOT flip a zero-output deadline into a clean `max_tokens` turn.
517
+ const text = new TextDecoder().decode(bytes);
518
+ if (text.length === 0)
519
+ return false;
520
+ return (/text_delta|input_json_delta|thinking_delta|signature_delta|tool_use|reasoning_content|"content":"[^"]/i.test(text));
521
+ };
515
522
  /**
516
523
  * Sibling of `withStreamDeadline` for the Anthropic pass-through:
517
524
  * the source is raw provider SSE bytes (not canonical chunks), and
518
525
  * the synthetic terminator must be a pre-encoded SSE byte sequence
519
526
  * because there is no canonical-decode step in the pipeline.
520
527
  *
521
- * On cutoff:
528
+ * On cutoff with partial output:
522
529
  * 1. cancel the source — propagating to the upstream `fetch` body
523
530
  * so we stop paying the provider for tokens we'll never deliver;
524
531
  * 2. enqueue ONE pre-encoded `message_delta` + `message_stop` pair
@@ -527,6 +534,10 @@ export const withStreamDeadline = (source, options) => {
527
534
  * 3. close cleanly so SSE clients see a terminated turn instead of
528
535
  * a severed connection.
529
536
  *
537
+ * On cutoff with ZERO output: emit an Anthropic `event: error` instead
538
+ * of a clean `max_tokens` turn, so the client and the usage log treat
539
+ * it as a failure.
540
+ *
530
541
  * Wrap BEFORE `.tee()` so the usage-drain branch also receives the
531
542
  * terminator and the accumulator can finalise. The caller checks
532
543
  * `isTruncated()` after consumption to skip caching a truncated body.
@@ -541,6 +552,8 @@ export const withByteStreamDeadline = (source, options) => {
541
552
  const reader = source.getReader();
542
553
  let finished = false;
543
554
  let truncated = false;
555
+ let sawOutput = false;
556
+ let zeroOutputTruncate = false;
544
557
  const DEADLINE = Symbol("byte-stream-deadline");
545
558
  let timer = null;
546
559
  const deadline = new Promise((resolve) => {
@@ -552,8 +565,11 @@ export const withByteStreamDeadline = (source, options) => {
552
565
  timer = null;
553
566
  }
554
567
  };
555
- const terminator = () => textEncoder.encode('event: message_delta\ndata: {"type":"message_delta","delta":{"stop_reason":"max_tokens","stop_sequence":null},"usage":{"output_tokens":0}}\n\n' +
568
+ // Leading `\n\n` resynchronises if the deadline fired mid-frame so the
569
+ // synthetic event always starts a fresh SSE record.
570
+ const lengthTerminator = () => textEncoder.encode('\n\nevent: message_delta\ndata: {"type":"message_delta","delta":{"stop_reason":"max_tokens","stop_sequence":null},"usage":{"output_tokens":0}}\n\n' +
556
571
  'event: message_stop\ndata: {"type":"message_stop"}\n\n');
572
+ const zeroOutputError = () => textEncoder.encode('\n\nevent: error\ndata: {"type":"error","error":{"type":"upstream_stalled","message":"stream deadline reached with no output"}}\n\n');
557
573
  const stream = new ReadableStream({
558
574
  async pull(controller) {
559
575
  if (finished) {
@@ -578,7 +594,13 @@ export const withByteStreamDeadline = (source, options) => {
578
594
  truncated = true;
579
595
  clear();
580
596
  reader.cancel("openllm-byte-stream-deadline").catch(() => { });
581
- controller.enqueue(terminator());
597
+ if (!sawOutput) {
598
+ zeroOutputTruncate = true;
599
+ controller.enqueue(zeroOutputError());
600
+ }
601
+ else {
602
+ controller.enqueue(lengthTerminator());
603
+ }
582
604
  controller.close();
583
605
  return;
584
606
  }
@@ -588,6 +610,9 @@ export const withByteStreamDeadline = (source, options) => {
588
610
  controller.close();
589
611
  return;
590
612
  }
613
+ if (byteChunkLooksLikeOutput(r.value)) {
614
+ sawOutput = true;
615
+ }
591
616
  controller.enqueue(r.value);
592
617
  },
593
618
  cancel(reason) {
@@ -596,5 +621,9 @@ export const withByteStreamDeadline = (source, options) => {
596
621
  reader.cancel(reason).catch(() => { });
597
622
  },
598
623
  });
599
- return { stream, isTruncated: () => truncated };
624
+ return {
625
+ stream,
626
+ isTruncated: () => truncated,
627
+ isZeroOutputTruncate: () => zeroOutputTruncate,
628
+ };
600
629
  };
@@ -11,6 +11,15 @@ export declare class UpstreamStreamError extends Error {
11
11
  readonly upstreamType: string;
12
12
  constructor(upstreamType: string, message: string);
13
13
  }
14
+ /**
15
+ * A streaming hop went silent before any byte was forwarded to the client
16
+ * (headers wait or raw upstream inactivity). Distinct from a mid-stream
17
+ * decode error so the classifier can tag `upstream_stalled` and the
18
+ * fallback chain can walk while the hop is still uncommitted.
19
+ */
20
+ export declare class UpstreamStalledError extends UpstreamStreamError {
21
+ constructor(message?: string);
22
+ }
14
23
  /**
15
24
  * A stream ended before a terminal chunk after its decoder had already observed
16
25
  * usage. The accumulator wraps every failure type, including transport errors,
@@ -30,3 +39,5 @@ export declare const upstreamErrorFrom: (err: unknown) => {
30
39
  type: string;
31
40
  message: string;
32
41
  };
42
+ /** True when `err` is (or wraps) an {@link UpstreamStalledError}. */
43
+ export declare const isUpstreamStalledError: (err: unknown) => boolean;
@@ -14,6 +14,18 @@ export class UpstreamStreamError extends Error {
14
14
  this.upstreamType = upstreamType;
15
15
  }
16
16
  }
17
+ /**
18
+ * A streaming hop went silent before any byte was forwarded to the client
19
+ * (headers wait or raw upstream inactivity). Distinct from a mid-stream
20
+ * decode error so the classifier can tag `upstream_stalled` and the
21
+ * fallback chain can walk while the hop is still uncommitted.
22
+ */
23
+ export class UpstreamStalledError extends UpstreamStreamError {
24
+ constructor(message = "upstream stalled before producing output") {
25
+ super("upstream_stalled", message);
26
+ this.name = "UpstreamStalledError";
27
+ }
28
+ }
17
29
  /**
18
30
  * A stream ended before a terminal chunk after its decoder had already observed
19
31
  * usage. The accumulator wraps every failure type, including transport errors,
@@ -43,3 +55,12 @@ export const upstreamErrorFrom = (err) => {
43
55
  }
44
56
  return { type: "stream_error", message: String(err) };
45
57
  };
58
+ /** True when `err` is (or wraps) an {@link UpstreamStalledError}. */
59
+ export const isUpstreamStalledError = (err) => {
60
+ if (err instanceof UpstreamStalledError)
61
+ return true;
62
+ if (err instanceof Error && err.cause !== undefined) {
63
+ return isUpstreamStalledError(err.cause);
64
+ }
65
+ return false;
66
+ };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@openllmsh/sdk",
3
- "version": "2.10.14",
3
+ "version": "2.10.15-alpha.2",
4
4
  "private": false,
5
5
  "description": "The OpenLLM SDK: typed API contracts, wire-format adapters, the device tunnel, client-side vault crypto, the OpenAPI spec and MCP tools.",
6
6
  "keywords": [