@openllmsh/sdk 2.10.14 → 2.10.15-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/protocol/agent.d.ts +28 -6
- package/dist/protocol/agent.js +20 -3
- package/dist/protocol/cooldown-reason.d.ts +1 -1
- package/dist/protocol/cooldown-reason.js +4 -0
- package/dist/protocol/daemon.d.ts +5 -1
- package/dist/protocol/daemon.js +2 -0
- package/dist/protocol/device-state.d.ts +2 -0
- package/dist/protocol/media-attempt.js +1 -0
- package/dist/protocol/mux.d.ts +36 -0
- package/dist/protocol/mux.js +33 -0
- package/dist/tunnel/peer-link.d.ts +3 -1
- package/dist/tunnel/peer-link.js +48 -11
- package/dist/wire/index.d.ts +1 -0
- package/dist/wire/index.js +1 -0
- package/dist/wire/lib/error-class.js +10 -3
- package/dist/wire/lib/streaming/peek.d.ts +10 -10
- package/dist/wire/lib/streaming/peek.js +10 -10
- package/dist/wire/lib/streaming/silence.d.ts +93 -0
- package/dist/wire/lib/streaming/silence.js +262 -0
- package/dist/wire/lib/streaming/sse.d.ts +14 -3
- package/dist/wire/lib/streaming/sse.js +61 -32
- package/dist/wire/lib/streaming/upstream-error.d.ts +11 -0
- package/dist/wire/lib/streaming/upstream-error.js +21 -0
- package/package.json +1 -1
package/dist/protocol/agent.d.ts
CHANGED
|
@@ -64,6 +64,11 @@ export declare const AGENT_HOOK_EVENTS: readonly ["PermissionRequest", "PreToolU
|
|
|
64
64
|
export declare const AGENT_MAX_ATTACHMENTS = 8;
|
|
65
65
|
/** Max base64 characters of one inline attachment. */
|
|
66
66
|
export declare const AGENT_MAX_ATTACHMENT_BASE64_LENGTH: number;
|
|
67
|
+
/**
|
|
68
|
+
* Max characters of one absolute local-path attachment (same bound as a
|
|
69
|
+
* session cwd). Path attachments are loopback-only and never cross the network.
|
|
70
|
+
*/
|
|
71
|
+
export declare const AGENT_MAX_ATTACHMENT_PATH_LENGTH = 1024;
|
|
67
72
|
/**
|
|
68
73
|
* Engines an agent session can run. Each id is the `openllm <client>` the
|
|
69
74
|
* daemon launches the engine through (same overlay, gateway URL, key, model
|
|
@@ -100,14 +105,27 @@ export declare const AgentStreamOpenPayload: S.Struct<{
|
|
|
100
105
|
readonly kind: S.Literal<"agent">;
|
|
101
106
|
}>;
|
|
102
107
|
export type TAgentStreamOpenPayload = S.Schema.Type<typeof AgentStreamOpenPayload>;
|
|
108
|
+
/**
|
|
109
|
+
* One attachment on `send`. Exactly one of `data` (base64, any transport) or
|
|
110
|
+
* `path` (absolute local file, loopback agent socket only — the daemon rejects
|
|
111
|
+
* `path` on mux/RTC). Both together are refused (same XOR pattern as
|
|
112
|
+
* {@link AgentRespondCommand}).
|
|
113
|
+
*/
|
|
103
114
|
export declare const AgentAttachment: S.Struct<{
|
|
104
115
|
readonly kind: S.Literals<readonly ["image", "file"]>;
|
|
105
116
|
readonly name: S.optional<S.String>;
|
|
106
117
|
readonly media_type: S.String;
|
|
107
|
-
/** Base64 bytes. */
|
|
108
|
-
readonly data: S.String
|
|
118
|
+
/** Base64 bytes. Mutually exclusive with {@link path}. */
|
|
119
|
+
readonly data: S.optional<S.String>;
|
|
120
|
+
/** Absolute local path; admitted only on the desktop loopback agent socket. */
|
|
121
|
+
readonly path: S.optional<S.String>;
|
|
109
122
|
}>;
|
|
110
123
|
export type TAgentAttachment = S.Schema.Type<typeof AgentAttachment>;
|
|
124
|
+
/** True when the attachment carries an absolute local path instead of bytes. */
|
|
125
|
+
export declare const agentAttachmentHasPath: (attachment: TAgentAttachment) => attachment is TAgentAttachment & {
|
|
126
|
+
readonly path: string;
|
|
127
|
+
readonly data?: undefined;
|
|
128
|
+
};
|
|
111
129
|
/**
|
|
112
130
|
* Metadata for an attachment that was saved on the device for a turn. The
|
|
113
131
|
* `name` is the on-disk basename under the session's attachments dir (what
|
|
@@ -175,8 +193,10 @@ export declare const AgentSendCommand: S.Struct<{
|
|
|
175
193
|
readonly kind: S.Literals<readonly ["image", "file"]>;
|
|
176
194
|
readonly name: S.optional<S.String>;
|
|
177
195
|
readonly media_type: S.String;
|
|
178
|
-
/** Base64 bytes. */
|
|
179
|
-
readonly data: S.String
|
|
196
|
+
/** Base64 bytes. Mutually exclusive with {@link path}. */
|
|
197
|
+
readonly data: S.optional<S.String>;
|
|
198
|
+
/** Absolute local path; admitted only on the desktop loopback agent socket. */
|
|
199
|
+
readonly path: S.optional<S.String>;
|
|
180
200
|
}>>>;
|
|
181
201
|
}>;
|
|
182
202
|
export type TAgentSendCommand = S.Schema.Type<typeof AgentSendCommand>;
|
|
@@ -335,8 +355,10 @@ export declare const AgentCommand: S.Union<readonly [S.Struct<{
|
|
|
335
355
|
readonly kind: S.Literals<readonly ["image", "file"]>;
|
|
336
356
|
readonly name: S.optional<S.String>;
|
|
337
357
|
readonly media_type: S.String;
|
|
338
|
-
/** Base64 bytes. */
|
|
339
|
-
readonly data: S.String
|
|
358
|
+
/** Base64 bytes. Mutually exclusive with {@link path}. */
|
|
359
|
+
readonly data: S.optional<S.String>;
|
|
360
|
+
/** Absolute local path; admitted only on the desktop loopback agent socket. */
|
|
361
|
+
readonly path: S.optional<S.String>;
|
|
340
362
|
}>>>;
|
|
341
363
|
}>, S.Struct<{
|
|
342
364
|
readonly type: S.Literal<"send_now">;
|
package/dist/protocol/agent.js
CHANGED
|
@@ -72,6 +72,11 @@ export const AGENT_HOOK_EVENTS = [
|
|
|
72
72
|
export const AGENT_MAX_ATTACHMENTS = 8;
|
|
73
73
|
/** Max base64 characters of one inline attachment. */
|
|
74
74
|
export const AGENT_MAX_ATTACHMENT_BASE64_LENGTH = 6 * 1024 * 1024;
|
|
75
|
+
/**
|
|
76
|
+
* Max characters of one absolute local-path attachment (same bound as a
|
|
77
|
+
* session cwd). Path attachments are loopback-only and never cross the network.
|
|
78
|
+
*/
|
|
79
|
+
export const AGENT_MAX_ATTACHMENT_PATH_LENGTH = 1024;
|
|
75
80
|
/**
|
|
76
81
|
* Engines an agent session can run. Each id is the `openllm <client>` the
|
|
77
82
|
* daemon launches the engine through (same overlay, gateway URL, key, model
|
|
@@ -120,13 +125,25 @@ export const AgentStreamOpenPayload = S.Struct({
|
|
|
120
125
|
// ---------------------------------------------------------------------------
|
|
121
126
|
// Commands (consumer → daemon)
|
|
122
127
|
// ---------------------------------------------------------------------------
|
|
128
|
+
/**
|
|
129
|
+
* One attachment on `send`. Exactly one of `data` (base64, any transport) or
|
|
130
|
+
* `path` (absolute local file, loopback agent socket only — the daemon rejects
|
|
131
|
+
* `path` on mux/RTC). Both together are refused (same XOR pattern as
|
|
132
|
+
* {@link AgentRespondCommand}).
|
|
133
|
+
*/
|
|
123
134
|
export const AgentAttachment = S.Struct({
|
|
124
135
|
kind: S.Literals(["image", "file"]),
|
|
125
136
|
name: S.optional(S.String.check(S.isMinLength(1), S.isMaxLength(256))),
|
|
126
137
|
media_type: S.String.check(S.isMinLength(1), S.isMaxLength(128)),
|
|
127
|
-
/** Base64 bytes. */
|
|
128
|
-
data: S.String.check(S.isMaxLength(AGENT_MAX_ATTACHMENT_BASE64_LENGTH)),
|
|
129
|
-
|
|
138
|
+
/** Base64 bytes. Mutually exclusive with {@link path}. */
|
|
139
|
+
data: S.optional(S.String.check(S.isMaxLength(AGENT_MAX_ATTACHMENT_BASE64_LENGTH))),
|
|
140
|
+
/** Absolute local path; admitted only on the desktop loopback agent socket. */
|
|
141
|
+
path: S.optional(S.String.check(S.isMinLength(1), S.isMaxLength(AGENT_MAX_ATTACHMENT_PATH_LENGTH))),
|
|
142
|
+
}).check(S.makeFilter((attachment) => (attachment.data === undefined) !== (attachment.path === undefined), {
|
|
143
|
+
message: "attachment carries exactly one of data / path",
|
|
144
|
+
}));
|
|
145
|
+
/** True when the attachment carries an absolute local path instead of bytes. */
|
|
146
|
+
export const agentAttachmentHasPath = (attachment) => typeof attachment.path === "string";
|
|
130
147
|
/**
|
|
131
148
|
* Metadata for an attachment that was saved on the device for a turn. The
|
|
132
149
|
* `name` is the on-disk basename under the session's attachments dir (what
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { Schema as S } from "effect";
|
|
2
|
-
export declare const CooldownReason: S.Literals<readonly ["network", "timeout", "rate_limit", "quota_exhausted", "server_error", "auth", "payment", "not_found", "payload_too_large", "unprocessable", "request_invalid", "accepted_unreadable", "context_overflow", "content_filter", "upstream_rejection"]>;
|
|
2
|
+
export declare const CooldownReason: S.Literals<readonly ["network", "timeout", "upstream_stalled", "rate_limit", "quota_exhausted", "server_error", "auth", "payment", "not_found", "payload_too_large", "unprocessable", "request_invalid", "accepted_unreadable", "context_overflow", "content_filter", "upstream_rejection"]>;
|
|
3
3
|
export type TCooldownReason = S.Schema.Type<typeof CooldownReason>;
|
|
4
4
|
export type THopAction = "retry_in_place" | "cool_and_advance" | "walk_no_cool" | "surface";
|
|
5
5
|
/**
|
|
@@ -2,6 +2,7 @@ import { Schema as S } from "effect";
|
|
|
2
2
|
export const CooldownReason = S.Literals([
|
|
3
3
|
"network",
|
|
4
4
|
"timeout",
|
|
5
|
+
"upstream_stalled",
|
|
5
6
|
"rate_limit",
|
|
6
7
|
"quota_exhausted",
|
|
7
8
|
"server_error",
|
|
@@ -19,6 +20,9 @@ export const CooldownReason = S.Literals([
|
|
|
19
20
|
const cooldownPolicies = {
|
|
20
21
|
network: { action: "retry_in_place" },
|
|
21
22
|
timeout: { action: "retry_in_place" },
|
|
23
|
+
// Streaming hop went silent before any client byte (headers or body). Cool
|
|
24
|
+
// briefly so a dead provider isn't retried immediately for the same turn.
|
|
25
|
+
upstream_stalled: { action: "cool_and_advance", ttlMs: 60_000 },
|
|
22
26
|
rate_limit: { action: "cool_and_advance", ttlMs: 180_000 },
|
|
23
27
|
quota_exhausted: { action: "cool_and_advance", ttlMs: 180_000 },
|
|
24
28
|
server_error: { action: "retry_in_place" },
|
|
@@ -509,7 +509,7 @@ export declare const DaemonRecordRequest: S.Struct<{
|
|
|
509
509
|
* cloud charges each one the hold's full reservation.
|
|
510
510
|
*/
|
|
511
511
|
readonly hold_unknown_attempts: S.optional<S.Number>;
|
|
512
|
-
readonly cooldown_reason: S.optional<S.Literals<readonly ["network", "timeout", "rate_limit", "quota_exhausted", "server_error", "auth", "payment", "not_found", "payload_too_large", "unprocessable", "request_invalid", "accepted_unreadable", "context_overflow", "content_filter", "upstream_rejection"]>>;
|
|
512
|
+
readonly cooldown_reason: S.optional<S.Literals<readonly ["network", "timeout", "upstream_stalled", "rate_limit", "quota_exhausted", "server_error", "auth", "payment", "not_found", "payload_too_large", "unprocessable", "request_invalid", "accepted_unreadable", "context_overflow", "content_filter", "upstream_rejection"]>>;
|
|
513
513
|
/** Reset instant for reset-aware TTL cap; optional so old daemons omit. */
|
|
514
514
|
readonly reset_at_ms: S.optional<S.Number>;
|
|
515
515
|
}>;
|
|
@@ -1390,6 +1390,8 @@ export declare const DaemonProviderConnection: S.Struct<{
|
|
|
1390
1390
|
readonly host_signed_in: S.optional<S.Boolean>;
|
|
1391
1391
|
readonly detail: S.optional<S.String>;
|
|
1392
1392
|
readonly last_login_at_ms: S.optional<S.NullOr<S.Number>>;
|
|
1393
|
+
/** Wall-clock time of the latest real provider status observation. */
|
|
1394
|
+
readonly last_probed_at_ms: S.optional<S.Number>;
|
|
1393
1395
|
/** A LIVE device-code flow awaiting the user (codex/kimi on a remote box):
|
|
1394
1396
|
* the verification URL + one-time code to surface so the dashboard can
|
|
1395
1397
|
* render a synced "open this link, enter this code" panel. Present only
|
|
@@ -1685,6 +1687,8 @@ export declare const DaemonStatus: S.Struct<{
|
|
|
1685
1687
|
readonly host_signed_in: S.optional<S.Boolean>;
|
|
1686
1688
|
readonly detail: S.optional<S.String>;
|
|
1687
1689
|
readonly last_login_at_ms: S.optional<S.NullOr<S.Number>>;
|
|
1690
|
+
/** Wall-clock time of the latest real provider status observation. */
|
|
1691
|
+
readonly last_probed_at_ms: S.optional<S.Number>;
|
|
1688
1692
|
/** A LIVE device-code flow awaiting the user (codex/kimi on a remote box):
|
|
1689
1693
|
* the verification URL + one-time code to surface so the dashboard can
|
|
1690
1694
|
* render a synced "open this link, enter this code" panel. Present only
|
package/dist/protocol/daemon.js
CHANGED
|
@@ -899,6 +899,8 @@ export const DaemonProviderConnection = S.Struct({
|
|
|
899
899
|
host_signed_in: S.optional(S.Boolean),
|
|
900
900
|
detail: S.optional(S.String),
|
|
901
901
|
last_login_at_ms: S.optional(S.NullOr(S.Number)),
|
|
902
|
+
/** Wall-clock time of the latest real provider status observation. */
|
|
903
|
+
last_probed_at_ms: S.optional(S.Number),
|
|
902
904
|
/** A LIVE device-code flow awaiting the user (codex/kimi on a remote box):
|
|
903
905
|
* the verification URL + one-time code to surface so the dashboard can
|
|
904
906
|
* render a synced "open this link, enter this code" panel. Present only
|
|
@@ -38,6 +38,7 @@ export declare const DeviceState: S.Struct<{
|
|
|
38
38
|
readonly host_signed_in: S.optional<S.Boolean>;
|
|
39
39
|
readonly detail: S.optional<S.String>;
|
|
40
40
|
readonly last_login_at_ms: S.optional<S.NullOr<S.Number>>;
|
|
41
|
+
readonly last_probed_at_ms: S.optional<S.Number>;
|
|
41
42
|
readonly pending_auth: S.optional<S.NullOr<S.Struct<{
|
|
42
43
|
readonly url: S.optional<S.String>;
|
|
43
44
|
readonly code: S.optional<S.String>;
|
|
@@ -147,6 +148,7 @@ export declare const RelayDeviceStateFrame: S.Union<readonly [S.Struct<{
|
|
|
147
148
|
readonly host_signed_in: S.optional<S.Boolean>;
|
|
148
149
|
readonly detail: S.optional<S.String>;
|
|
149
150
|
readonly last_login_at_ms: S.optional<S.NullOr<S.Number>>;
|
|
151
|
+
readonly last_probed_at_ms: S.optional<S.Number>;
|
|
150
152
|
readonly pending_auth: S.optional<S.NullOr<S.Struct<{
|
|
151
153
|
readonly url: S.optional<S.String>;
|
|
152
154
|
readonly code: S.optional<S.String>;
|
package/dist/protocol/mux.d.ts
CHANGED
|
@@ -197,6 +197,14 @@ export declare const TunnelForwardHeaders: S.Struct<{
|
|
|
197
197
|
* bound to the key's cap hold. Uncapped keys omit it.
|
|
198
198
|
*/
|
|
199
199
|
readonly hop_grant: S.optional<S.String>;
|
|
200
|
+
/**
|
|
201
|
+
* The subscription provider a fleet consumer could not serve itself and
|
|
202
|
+
* is asking this peer for. Mapped to {@link FLEET_PROVIDER_HEADER}; a peer
|
|
203
|
+
* that can't serve it answers with {@link FLEET_DECLINE_HEADER} instead of
|
|
204
|
+
* walking on to a later hop.
|
|
205
|
+
*/
|
|
206
|
+
readonly fleet_provider: S.optional<S.String>;
|
|
207
|
+
readonly fleet_model: S.optional<S.String>;
|
|
200
208
|
}>;
|
|
201
209
|
export type TTunnelForwardHeaders = S.Schema.Type<typeof TunnelForwardHeaders>;
|
|
202
210
|
/** Response metadata carried by the mux `res_head` CTRL payload.
|
|
@@ -206,6 +214,14 @@ export type TTunnelForwardHeaders = S.Schema.Type<typeof TunnelForwardHeaders>;
|
|
|
206
214
|
*/
|
|
207
215
|
export declare const TUNNEL_RESOLVED_MODEL_MAX = 256;
|
|
208
216
|
export declare const TUNNEL_CHAIN_MAX = 4096;
|
|
217
|
+
/** Request header: the provider a fleet consumer wants this peer to serve. */
|
|
218
|
+
export declare const FLEET_PROVIDER_HEADER = "x-openllm-fleet-provider";
|
|
219
|
+
export declare const FLEET_MODEL_HEADER = "x-openllm-fleet-model";
|
|
220
|
+
/** Response header: this peer can't serve the requested fleet provider. */
|
|
221
|
+
export declare const FLEET_DECLINE_HEADER = "x-openllm-fleet-decline";
|
|
222
|
+
export declare const FLEET_DECLINE_CAUSES: readonly ["not_signed_in", "no_quota", "cooling", "unavailable"];
|
|
223
|
+
export type TFleetDeclineCause = (typeof FLEET_DECLINE_CAUSES)[number];
|
|
224
|
+
export declare const isFleetDeclineCause: (value: string | null) => value is TFleetDeclineCause;
|
|
209
225
|
export declare const TunnelResponseHeaders: S.Struct<{
|
|
210
226
|
readonly media_url: S.optional<S.String>;
|
|
211
227
|
readonly media_persistence: S.optional<S.Literal<"browser">>;
|
|
@@ -217,6 +233,8 @@ export declare const TunnelResponseHeaders: S.Struct<{
|
|
|
217
233
|
readonly resolved_model: S.optional<S.String>;
|
|
218
234
|
/** Comma-separated attempted hops — maps to {@link OPENLLM_CHAIN_HEADER}. */
|
|
219
235
|
readonly chain: S.optional<S.String>;
|
|
236
|
+
/** Maps to {@link FLEET_DECLINE_HEADER}. */
|
|
237
|
+
readonly fleet_decline: S.optional<S.Literals<readonly ["not_signed_in", "no_quota", "cooling", "unavailable"]>>;
|
|
220
238
|
}>;
|
|
221
239
|
export type TTunnelResponseHeaders = S.Schema.Type<typeof TunnelResponseHeaders>;
|
|
222
240
|
/** Copy closed mux res_head fields onto an HTTP Headers object. */
|
|
@@ -264,6 +282,14 @@ export declare const TunnelStreamOpenPayload: S.Struct<{
|
|
|
264
282
|
* bound to the key's cap hold. Uncapped keys omit it.
|
|
265
283
|
*/
|
|
266
284
|
readonly hop_grant: S.optional<S.String>;
|
|
285
|
+
/**
|
|
286
|
+
* The subscription provider a fleet consumer could not serve itself and
|
|
287
|
+
* is asking this peer for. Mapped to {@link FLEET_PROVIDER_HEADER}; a peer
|
|
288
|
+
* that can't serve it answers with {@link FLEET_DECLINE_HEADER} instead of
|
|
289
|
+
* walking on to a later hop.
|
|
290
|
+
*/
|
|
291
|
+
readonly fleet_provider: S.optional<S.String>;
|
|
292
|
+
readonly fleet_model: S.optional<S.String>;
|
|
267
293
|
}>>;
|
|
268
294
|
readonly consumer: S.optional<S.Literals<readonly ["browser", "daemon"]>>;
|
|
269
295
|
}>;
|
|
@@ -303,6 +329,14 @@ export declare const StreamOpenPayload: S.Union<readonly [S.Struct<{
|
|
|
303
329
|
* bound to the key's cap hold. Uncapped keys omit it.
|
|
304
330
|
*/
|
|
305
331
|
readonly hop_grant: S.optional<S.String>;
|
|
332
|
+
/**
|
|
333
|
+
* The subscription provider a fleet consumer could not serve itself and
|
|
334
|
+
* is asking this peer for. Mapped to {@link FLEET_PROVIDER_HEADER}; a peer
|
|
335
|
+
* that can't serve it answers with {@link FLEET_DECLINE_HEADER} instead of
|
|
336
|
+
* walking on to a later hop.
|
|
337
|
+
*/
|
|
338
|
+
readonly fleet_provider: S.optional<S.String>;
|
|
339
|
+
readonly fleet_model: S.optional<S.String>;
|
|
306
340
|
}>>;
|
|
307
341
|
readonly consumer: S.optional<S.Literals<readonly ["browser", "daemon"]>>;
|
|
308
342
|
}>, S.Struct<{
|
|
@@ -341,6 +375,8 @@ export declare const StreamCtrlPayload: S.Union<readonly [S.Struct<{
|
|
|
341
375
|
readonly resolved_model: S.optional<S.String>;
|
|
342
376
|
/** Comma-separated attempted hops — maps to {@link OPENLLM_CHAIN_HEADER}. */
|
|
343
377
|
readonly chain: S.optional<S.String>;
|
|
378
|
+
/** Maps to {@link FLEET_DECLINE_HEADER}. */
|
|
379
|
+
readonly fleet_decline: S.optional<S.Literals<readonly ["not_signed_in", "no_quota", "cooling", "unavailable"]>>;
|
|
344
380
|
}>>;
|
|
345
381
|
}>, S.Struct<{
|
|
346
382
|
readonly t: S.Literal<"replay_done">;
|
package/dist/protocol/mux.js
CHANGED
|
@@ -361,6 +361,14 @@ export const TunnelForwardHeaders = S.Struct({
|
|
|
361
361
|
* bound to the key's cap hold. Uncapped keys omit it.
|
|
362
362
|
*/
|
|
363
363
|
hop_grant: S.optional(S.String.check(S.isMaxLength(HOP_GRANT_B64_MAX))),
|
|
364
|
+
/**
|
|
365
|
+
* The subscription provider a fleet consumer could not serve itself and
|
|
366
|
+
* is asking this peer for. Mapped to {@link FLEET_PROVIDER_HEADER}; a peer
|
|
367
|
+
* that can't serve it answers with {@link FLEET_DECLINE_HEADER} instead of
|
|
368
|
+
* walking on to a later hop.
|
|
369
|
+
*/
|
|
370
|
+
fleet_provider: S.optional(S.String.check(S.isMaxLength(64))),
|
|
371
|
+
fleet_model: S.optional(S.String.check(S.isMaxLength(256))),
|
|
364
372
|
});
|
|
365
373
|
/** Response metadata carried by the mux `res_head` CTRL payload.
|
|
366
374
|
*
|
|
@@ -369,6 +377,19 @@ export const TunnelForwardHeaders = S.Struct({
|
|
|
369
377
|
*/
|
|
370
378
|
export const TUNNEL_RESOLVED_MODEL_MAX = 256;
|
|
371
379
|
export const TUNNEL_CHAIN_MAX = 4096;
|
|
380
|
+
/** Request header: the provider a fleet consumer wants this peer to serve. */
|
|
381
|
+
export const FLEET_PROVIDER_HEADER = "x-openllm-fleet-provider";
|
|
382
|
+
export const FLEET_MODEL_HEADER = "x-openllm-fleet-model";
|
|
383
|
+
/** Response header: this peer can't serve the requested fleet provider. */
|
|
384
|
+
export const FLEET_DECLINE_HEADER = "x-openllm-fleet-decline";
|
|
385
|
+
export const FLEET_DECLINE_CAUSES = [
|
|
386
|
+
"not_signed_in",
|
|
387
|
+
"no_quota",
|
|
388
|
+
"cooling",
|
|
389
|
+
"unavailable",
|
|
390
|
+
];
|
|
391
|
+
export const isFleetDeclineCause = (value) => value !== null &&
|
|
392
|
+
FLEET_DECLINE_CAUSES.includes(value);
|
|
372
393
|
export const TunnelResponseHeaders = S.Struct({
|
|
373
394
|
media_url: S.optional(S.String.check(S.isMaxLength(TUNNEL_MEDIA_URL_MAX_LENGTH))),
|
|
374
395
|
media_persistence: S.optional(S.Literal(MEDIA_PERSISTENCE_BROWSER)),
|
|
@@ -380,6 +401,8 @@ export const TunnelResponseHeaders = S.Struct({
|
|
|
380
401
|
resolved_model: S.optional(S.String.check(S.isMaxLength(TUNNEL_RESOLVED_MODEL_MAX))),
|
|
381
402
|
/** Comma-separated attempted hops — maps to {@link OPENLLM_CHAIN_HEADER}. */
|
|
382
403
|
chain: S.optional(S.String.check(S.isMaxLength(TUNNEL_CHAIN_MAX))),
|
|
404
|
+
/** Maps to {@link FLEET_DECLINE_HEADER}. */
|
|
405
|
+
fleet_decline: S.optional(S.Literals(FLEET_DECLINE_CAUSES)),
|
|
383
406
|
});
|
|
384
407
|
const boundedOptional = (value, max) => {
|
|
385
408
|
if (value === null || value.length === 0 || value.length > max)
|
|
@@ -406,6 +429,9 @@ export const applyTunnelResponseHeadersToHttp = (headers, res) => {
|
|
|
406
429
|
if (res.chain !== undefined) {
|
|
407
430
|
headers.set(OPENLLM_CHAIN_HEADER, res.chain);
|
|
408
431
|
}
|
|
432
|
+
if (res.fleet_decline !== undefined) {
|
|
433
|
+
headers.set(FLEET_DECLINE_HEADER, res.fleet_decline);
|
|
434
|
+
}
|
|
409
435
|
};
|
|
410
436
|
/** Project HTTP response headers onto the closed mux res_head struct. */
|
|
411
437
|
export const tunnelResponseHeadersFromHttp = (headers) => {
|
|
@@ -429,6 +455,11 @@ export const tunnelResponseHeadersFromHttp = (headers) => {
|
|
|
429
455
|
is_sse: contentType?.includes("text/event-stream") === true,
|
|
430
456
|
...(resolved === undefined ? {} : { resolved_model: resolved }),
|
|
431
457
|
...(chain === undefined ? {} : { chain }),
|
|
458
|
+
...(isFleetDeclineCause(headers.get(FLEET_DECLINE_HEADER))
|
|
459
|
+
? {
|
|
460
|
+
fleet_decline: headers.get(FLEET_DECLINE_HEADER),
|
|
461
|
+
}
|
|
462
|
+
: {}),
|
|
432
463
|
};
|
|
433
464
|
};
|
|
434
465
|
/** A channel-level admission failure. */
|
|
@@ -613,6 +644,8 @@ export const parseStreamOpenPayload = (value) => {
|
|
|
613
644
|
"video_id",
|
|
614
645
|
"team_grant",
|
|
615
646
|
"hop_grant",
|
|
647
|
+
"fleet_provider",
|
|
648
|
+
"fleet_model",
|
|
616
649
|
])) {
|
|
617
650
|
return null;
|
|
618
651
|
}
|
|
@@ -7,10 +7,11 @@ export declare const PEER_LINK_DEFAULTS: {
|
|
|
7
7
|
readonly rtcIceMs: 20000;
|
|
8
8
|
readonly idleTtlMs: 90000;
|
|
9
9
|
readonly failureTtlMs: 60000;
|
|
10
|
+
readonly presenceFailureTtlMs: 5000;
|
|
10
11
|
};
|
|
11
12
|
export type TPeerLinkPath = "rtc" | "mux";
|
|
12
13
|
export type TPeerLinkRtcMode = "use-if-open+warm" | "probe";
|
|
13
|
-
export type TPeerLinkFailureReason = "aborted" | "not_capable" | "daemon_outdated" | "overloaded" | "tunnel_busy" | "transport" | "unknown";
|
|
14
|
+
export type TPeerLinkFailureReason = "aborted" | "presence" | "relay_unavailable" | "timeout" | "nack" | "not_capable" | "daemon_outdated" | "overloaded" | "tunnel_busy" | "transport" | "unknown";
|
|
14
15
|
export type TPeerLinkEvent = {
|
|
15
16
|
readonly type: "selected" | "failed" | "fallback" | "released";
|
|
16
17
|
readonly path: TPeerLinkPath;
|
|
@@ -66,6 +67,7 @@ export type TPeerLinkFailureCache = {
|
|
|
66
67
|
/** Clear one transport's failures without erasing the independent rung. */
|
|
67
68
|
readonly clearPath: (path: TPeerLinkPath) => void;
|
|
68
69
|
readonly has: (target: string, path: TPeerLinkPath, now?: number) => boolean;
|
|
70
|
+
readonly reason: (target: string, path: TPeerLinkPath, now?: number) => TPeerLinkFailureReason | null;
|
|
69
71
|
readonly until: (target: string, path: TPeerLinkPath, now?: number) => number | null;
|
|
70
72
|
readonly set: (target: string, path: TPeerLinkPath, reason: TPeerLinkFailureReason, now?: number) => void;
|
|
71
73
|
};
|
package/dist/tunnel/peer-link.js
CHANGED
|
@@ -6,6 +6,7 @@ export const PEER_LINK_DEFAULTS = {
|
|
|
6
6
|
rtcIceMs: 20_000,
|
|
7
7
|
idleTtlMs: 90_000,
|
|
8
8
|
failureTtlMs: 60_000,
|
|
9
|
+
presenceFailureTtlMs: 5_000,
|
|
9
10
|
};
|
|
10
11
|
const failureTtlMs = (reason) => {
|
|
11
12
|
switch (reason) {
|
|
@@ -14,9 +15,10 @@ const failureTtlMs = (reason) => {
|
|
|
14
15
|
case "overloaded":
|
|
15
16
|
case "tunnel_busy":
|
|
16
17
|
return 2_000;
|
|
18
|
+
case "presence":
|
|
17
19
|
case "not_capable":
|
|
18
20
|
case "daemon_outdated":
|
|
19
|
-
return
|
|
21
|
+
return PEER_LINK_DEFAULTS.presenceFailureTtlMs;
|
|
20
22
|
default:
|
|
21
23
|
return PEER_LINK_DEFAULTS.failureTtlMs;
|
|
22
24
|
}
|
|
@@ -31,8 +33,8 @@ const cacheKeyOf = (target, path) => `${target}\u0000${path}`;
|
|
|
31
33
|
export const createPeerLinkFailureCache = () => {
|
|
32
34
|
const failureCache = new Map();
|
|
33
35
|
const sweepFailureCache = (now) => {
|
|
34
|
-
for (const [key,
|
|
35
|
-
if (until <= now)
|
|
36
|
+
for (const [key, entry] of failureCache) {
|
|
37
|
+
if (entry.until <= now)
|
|
36
38
|
failureCache.delete(key);
|
|
37
39
|
}
|
|
38
40
|
while (failureCache.size > MAX_FAILURE_CACHE_ENTRIES) {
|
|
@@ -68,18 +70,22 @@ export const createPeerLinkFailureCache = () => {
|
|
|
68
70
|
const until = failureCache.get(key);
|
|
69
71
|
if (until === undefined)
|
|
70
72
|
return false;
|
|
71
|
-
if (until > now)
|
|
73
|
+
if (until.until > now)
|
|
72
74
|
return true;
|
|
73
75
|
failureCache.delete(key);
|
|
74
76
|
return false;
|
|
75
77
|
},
|
|
78
|
+
reason(target, path, now = Date.now()) {
|
|
79
|
+
const entry = failureCache.get(cacheKeyOf(target, path));
|
|
80
|
+
return entry !== undefined && entry.until > now ? entry.reason : null;
|
|
81
|
+
},
|
|
76
82
|
until(target, path, now = Date.now()) {
|
|
77
83
|
const key = cacheKeyOf(target, path);
|
|
78
84
|
const until = failureCache.get(key);
|
|
79
85
|
if (until === undefined)
|
|
80
86
|
return null;
|
|
81
|
-
if (until > now)
|
|
82
|
-
return until;
|
|
87
|
+
if (until.until > now)
|
|
88
|
+
return until.until;
|
|
83
89
|
failureCache.delete(key);
|
|
84
90
|
return null;
|
|
85
91
|
},
|
|
@@ -89,7 +95,7 @@ export const createPeerLinkFailureCache = () => {
|
|
|
89
95
|
if (ttl > 0) {
|
|
90
96
|
const key = cacheKeyOf(target, path);
|
|
91
97
|
failureCache.delete(key);
|
|
92
|
-
failureCache.set(key, now + ttl);
|
|
98
|
+
failureCache.set(key, { until: now + ttl, reason });
|
|
93
99
|
sweepFailureCache(now);
|
|
94
100
|
}
|
|
95
101
|
},
|
|
@@ -101,6 +107,8 @@ export const createPeerLinkFailureCache = () => {
|
|
|
101
107
|
*/
|
|
102
108
|
export const peerLinkFailureCache = createPeerLinkFailureCache();
|
|
103
109
|
export const peerLinkFailureReasonOf = (error) => {
|
|
110
|
+
if (error instanceof DOMException && error.name === "TimeoutError")
|
|
111
|
+
return "timeout";
|
|
104
112
|
if (error instanceof DOMException && error.name === "AbortError") {
|
|
105
113
|
return "aborted";
|
|
106
114
|
}
|
|
@@ -126,7 +134,9 @@ export const createPeerLink = (options) => {
|
|
|
126
134
|
const openBudgetMs = options.policy?.openBudgetMs ?? PEER_LINK_DEFAULTS.muxAckSameUserMs;
|
|
127
135
|
const rtcProbeMs = options.policy?.rtcProbeMs ?? Math.min(2_000, openBudgetMs);
|
|
128
136
|
const idleTtlMs = options.policy?.idleTtlMs ?? PEER_LINK_DEFAULTS.idleTtlMs;
|
|
129
|
-
const classify =
|
|
137
|
+
const classify = (error) => error instanceof DOMException && error.name === "TimeoutError"
|
|
138
|
+
? "timeout"
|
|
139
|
+
: (options.policy?.classifyFailure ?? peerLinkFailureReasonOf)(error);
|
|
130
140
|
// Retrying a dispatched inference can duplicate provider work. Callers must
|
|
131
141
|
// prove that the failure happened before dispatch to opt into a fallback.
|
|
132
142
|
const retrySafe = options.policy?.retrySafe ?? (() => false);
|
|
@@ -168,8 +178,8 @@ export const createPeerLink = (options) => {
|
|
|
168
178
|
let timer;
|
|
169
179
|
const timedOut = new Promise((_, reject) => {
|
|
170
180
|
timer = setTimeout(() => {
|
|
181
|
+
reject(new DOMException("Timed out", "TimeoutError"));
|
|
171
182
|
controller.abort();
|
|
172
|
-
reject(new DOMException("Timed out", "AbortError"));
|
|
173
183
|
}, timeoutMs);
|
|
174
184
|
});
|
|
175
185
|
try {
|
|
@@ -249,8 +259,35 @@ export const createPeerLink = (options) => {
|
|
|
249
259
|
cache(resolved.path, error);
|
|
250
260
|
const dialer = resolved.path === "rtc" ? options.rtc : options.mux;
|
|
251
261
|
dialer?.drop?.(resolved.channel);
|
|
252
|
-
if (resolved.path
|
|
253
|
-
|
|
262
|
+
if (resolved.path === "mux") {
|
|
263
|
+
if (dialer?.drop === undefined)
|
|
264
|
+
throw error;
|
|
265
|
+
peerLinkFailureCache.clear(options.target, "mux");
|
|
266
|
+
const startedAt = now();
|
|
267
|
+
let fresh;
|
|
268
|
+
try {
|
|
269
|
+
fresh = await dialWithin(options.mux, openBudgetMs);
|
|
270
|
+
}
|
|
271
|
+
catch (dialError) {
|
|
272
|
+
cache("mux", dialError);
|
|
273
|
+
throw error;
|
|
274
|
+
}
|
|
275
|
+
if (fresh === null || fresh === resolved.channel)
|
|
276
|
+
throw error;
|
|
277
|
+
const reopened = fresh;
|
|
278
|
+
try {
|
|
279
|
+
return await fn(resultOf("mux", options.mux, reopened, false, startedAt));
|
|
280
|
+
}
|
|
281
|
+
catch (retryError) {
|
|
282
|
+
// One reopen only: a second pre-dispatch failure is cached and its
|
|
283
|
+
// channel dropped, so the next request honours the failure TTL.
|
|
284
|
+
if (retrySafe(retryError)) {
|
|
285
|
+
cache("mux", retryError);
|
|
286
|
+
dialer.drop(reopened);
|
|
287
|
+
}
|
|
288
|
+
throw retryError;
|
|
289
|
+
}
|
|
290
|
+
}
|
|
254
291
|
emit({
|
|
255
292
|
type: "fallback",
|
|
256
293
|
path: "rtc",
|
package/dist/wire/index.d.ts
CHANGED
|
@@ -36,6 +36,7 @@ export * from "./lib/refusal.js";
|
|
|
36
36
|
export * from "./lib/secret-scrub.js";
|
|
37
37
|
export * from "./lib/streaming/accumulate.js";
|
|
38
38
|
export * from "./lib/streaming/peek.js";
|
|
39
|
+
export * from "./lib/streaming/silence.js";
|
|
39
40
|
export * from "./lib/streaming/sse.js";
|
|
40
41
|
export * from "./lib/streaming/strip-tool-calls.js";
|
|
41
42
|
export * from "./lib/streaming/upstream-error.js";
|
package/dist/wire/index.js
CHANGED
|
@@ -36,6 +36,7 @@ export * from "./lib/refusal.js";
|
|
|
36
36
|
export * from "./lib/secret-scrub.js";
|
|
37
37
|
export * from "./lib/streaming/accumulate.js";
|
|
38
38
|
export * from "./lib/streaming/peek.js";
|
|
39
|
+
export * from "./lib/streaming/silence.js";
|
|
39
40
|
export * from "./lib/streaming/sse.js";
|
|
40
41
|
export * from "./lib/streaming/strip-tool-calls.js";
|
|
41
42
|
export * from "./lib/streaming/upstream-error.js";
|
|
@@ -32,11 +32,18 @@ export const reasonFor = (i, rules = errorRulesFor(i.provider)) => {
|
|
|
32
32
|
// A 2xx the runner still couldn't use (an undecodable 200): the provider
|
|
33
33
|
// ACCEPTED the request and may have billed it, so walking on would send —
|
|
34
34
|
// and bill — it again. Only an explicit in-band refusal (content_filter,
|
|
35
|
-
// above) walks from a 2xx; no body matcher or provider rule
|
|
36
|
-
//
|
|
37
|
-
//
|
|
35
|
+
// above) walks from a 2xx; no body matcher or provider rule — including a
|
|
36
|
+
// forged `upstream_stalled` code on an accepted response — may turn an
|
|
37
|
+
// accepted request into a walk.
|
|
38
38
|
if (i.status >= 200 && i.status < 300)
|
|
39
39
|
return "accepted_unreadable";
|
|
40
|
+
// Pre-commit synthetic stall (headers / raw-byte silence). Our runners and
|
|
41
|
+
// the daemon emit this only on non-2xx outcomes (typically 504). Kept AFTER
|
|
42
|
+
// the 2xx guard so an accepted provider response can never be re-POSTed.
|
|
43
|
+
if (i.envelope?.error?.code === "upstream_stalled" ||
|
|
44
|
+
i.envelope?.error?.type === "upstream_stalled") {
|
|
45
|
+
return "upstream_stalled";
|
|
46
|
+
}
|
|
40
47
|
const message = i.envelope?.error?.message ?? "";
|
|
41
48
|
if (CONTEXT_OVERFLOW_BODY.test(message))
|
|
42
49
|
return "context_overflow";
|
|
@@ -45,16 +45,16 @@ export declare const isMeaningfulChunk: (chunk: TChatCompletionChunk) => boolean
|
|
|
45
45
|
* the module doc for why), then return a stream that replays what was
|
|
46
46
|
* buffered ahead of the live tail.
|
|
47
47
|
*
|
|
48
|
-
* The wait
|
|
49
|
-
* signal
|
|
50
|
-
*
|
|
51
|
-
*
|
|
52
|
-
*
|
|
53
|
-
*
|
|
54
|
-
*
|
|
55
|
-
*
|
|
56
|
-
*
|
|
57
|
-
*
|
|
48
|
+
* The wait for a MEANINGFUL decoded event is still unbounded here — the
|
|
49
|
+
* request abort signal is the only hard stop at this layer. Raw-byte
|
|
50
|
+
* silence is enforced BELOW the decoder by `withByteSilenceGuard` /
|
|
51
|
+
* `withStreamingHeadersTimeout` (see `silence.ts`): a generous
|
|
52
|
+
* first-byte budget (default 300s, `OPENLLM_STREAM_FIRST_BYTE_TIMEOUT_MS`)
|
|
53
|
+
* covers near-cap prefills that produce no keepalive, and a shorter gap
|
|
54
|
+
* budget (default 120s, `OPENLLM_STREAM_SILENCE_TIMEOUT_MS`) trips only
|
|
55
|
+
* after the upstream has already spoken. That split keeps the overflow
|
|
56
|
+
* peek working for long prefills without letting a truly silent hop
|
|
57
|
+
* burn the Vercel deadline as a fake `length` success.
|
|
58
58
|
*
|
|
59
59
|
* A stream that CLOSES before any event is an error by default (walkable
|
|
60
60
|
* — the upstream produced nothing), not an empty success. EXCEPT when the
|
|
@@ -30,16 +30,16 @@ export const isMeaningfulChunk = (chunk) => {
|
|
|
30
30
|
* the module doc for why), then return a stream that replays what was
|
|
31
31
|
* buffered ahead of the live tail.
|
|
32
32
|
*
|
|
33
|
-
* The wait
|
|
34
|
-
* signal
|
|
35
|
-
*
|
|
36
|
-
*
|
|
37
|
-
*
|
|
38
|
-
*
|
|
39
|
-
*
|
|
40
|
-
*
|
|
41
|
-
*
|
|
42
|
-
*
|
|
33
|
+
* The wait for a MEANINGFUL decoded event is still unbounded here — the
|
|
34
|
+
* request abort signal is the only hard stop at this layer. Raw-byte
|
|
35
|
+
* silence is enforced BELOW the decoder by `withByteSilenceGuard` /
|
|
36
|
+
* `withStreamingHeadersTimeout` (see `silence.ts`): a generous
|
|
37
|
+
* first-byte budget (default 300s, `OPENLLM_STREAM_FIRST_BYTE_TIMEOUT_MS`)
|
|
38
|
+
* covers near-cap prefills that produce no keepalive, and a shorter gap
|
|
39
|
+
* budget (default 120s, `OPENLLM_STREAM_SILENCE_TIMEOUT_MS`) trips only
|
|
40
|
+
* after the upstream has already spoken. That split keeps the overflow
|
|
41
|
+
* peek working for long prefills without letting a truly silent hop
|
|
42
|
+
* burn the Vercel deadline as a fake `length` success.
|
|
43
43
|
*
|
|
44
44
|
* A stream that CLOSES before any event is an error by default (walkable
|
|
45
45
|
* — the upstream produced nothing), not an empty success. EXCEPT when the
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pre-commit silence budgets for STREAMING hops only.
|
|
3
|
+
*
|
|
4
|
+
* Two clocks, both measured on raw upstream bytes (any enqueued chunk —
|
|
5
|
+
* including empty ones, keepalive comments, and pings — counts as alive):
|
|
6
|
+
*
|
|
7
|
+
* 1. **First-byte** (`OPENLLM_STREAM_FIRST_BYTE_TIMEOUT_MS`, default 300s)
|
|
8
|
+
* — waiting for response headers, and the gap before the first body
|
|
9
|
+
* byte. Near-cap prefills (hundreds of thousands of tokens) can
|
|
10
|
+
* legitimately take minutes with no keepalive; this budget is the
|
|
11
|
+
* one that must accommodate them. See `peek.ts`.
|
|
12
|
+
* 2. **Gap** (`OPENLLM_STREAM_SILENCE_TIMEOUT_MS`, default 120s) — after
|
|
13
|
+
* any byte has arrived, until `release()` (hop commit). A provider
|
|
14
|
+
* that went silent mid-prefill after talking once is treated as
|
|
15
|
+
* stalled.
|
|
16
|
+
*
|
|
17
|
+
* Non-streaming requests get no short timeout — they legitimately send
|
|
18
|
+
* nothing until the full JSON body is ready. A non-positive override
|
|
19
|
+
* disables that budget.
|
|
20
|
+
*/
|
|
21
|
+
export declare const DEFAULT_STREAM_FIRST_BYTE_TIMEOUT_MS = 300000;
|
|
22
|
+
export declare const DEFAULT_STREAM_SILENCE_TIMEOUT_MS = 120000;
|
|
23
|
+
export declare const streamFirstByteTimeoutMs: () => number;
|
|
24
|
+
export declare const streamSilenceTimeoutMs: () => number;
|
|
25
|
+
/**
|
|
26
|
+
* Whether a hop should apply the streaming silence budgets on its
|
|
27
|
+
* upstream fetch. ChatGPT's Codex endpoint always streams even when the
|
|
28
|
+
* client asked for JSON, so the gate is `alwaysStreams || clientWantsStream`
|
|
29
|
+
* — not the client flag alone.
|
|
30
|
+
*/
|
|
31
|
+
export declare const streamingHopNeedsSilence: (opts: {
|
|
32
|
+
readonly upstreamAlwaysStreams: boolean;
|
|
33
|
+
readonly clientWantsStream: boolean;
|
|
34
|
+
}) => boolean;
|
|
35
|
+
export declare const firstByteDeadlineAt: (firstByteTimeoutMs: number) => number | null;
|
|
36
|
+
/**
|
|
37
|
+
* Time left before `deadlineAt`. `null` (no budget) returns 0, which every
|
|
38
|
+
* guard reads as disabled; an EXPIRED deadline returns 1 so the guard fires
|
|
39
|
+
* at once instead of being read as disabled.
|
|
40
|
+
*/
|
|
41
|
+
export declare const remainingTimeoutMs: (deadlineAt: number | null) => number;
|
|
42
|
+
/**
|
|
43
|
+
* Read a (non-2xx) response body with `read`, failing with
|
|
44
|
+
* {@link UpstreamStalledError} if it takes longer than `timeoutMs`. The body
|
|
45
|
+
* is read through a reader this helper owns, so a stall can actually cancel
|
|
46
|
+
* it (a `text()` in flight locks the body, and `body.cancel()` would throw).
|
|
47
|
+
* `timeoutMs <= 0` disables the budget.
|
|
48
|
+
*/
|
|
49
|
+
export declare const readResponseWithin: <T>(response: Response, timeoutMs: number, read: (response: Response) => Promise<T>) => Promise<T>;
|
|
50
|
+
export type TStreamingHeadersGuard = {
|
|
51
|
+
readonly signal: AbortSignal;
|
|
52
|
+
/** True when the first-byte headers timer fired and the caller did not abort. */
|
|
53
|
+
readonly didStall: () => boolean;
|
|
54
|
+
/** Clear the headers timer once the Response headers have arrived. */
|
|
55
|
+
readonly clear: () => void;
|
|
56
|
+
};
|
|
57
|
+
/**
|
|
58
|
+
* AbortSignal that fires if response headers take longer than
|
|
59
|
+
* `firstByteTimeoutMs`. Cleared as soon as headers arrive so it cannot
|
|
60
|
+
* abort the body later. Caller abort still wins.
|
|
61
|
+
*/
|
|
62
|
+
export declare const withStreamingHeadersTimeout: (caller: AbortSignal, firstByteTimeoutMs: number) => TStreamingHeadersGuard;
|
|
63
|
+
export type TByteSilenceGuard = {
|
|
64
|
+
readonly stream: ReadableStream<Uint8Array>;
|
|
65
|
+
/**
|
|
66
|
+
* Disarm the silence timer. Call once the hop commits (first chunk is
|
|
67
|
+
* about to be forwarded to the client) so mid-stream quiet no longer
|
|
68
|
+
* walks the fallback chain.
|
|
69
|
+
*/
|
|
70
|
+
readonly release: () => void;
|
|
71
|
+
};
|
|
72
|
+
export type TByteSilenceGuardOptions = {
|
|
73
|
+
/** Budget waiting for the first body byte (default 300s). */
|
|
74
|
+
readonly firstByteTimeoutMs: number;
|
|
75
|
+
/** Budget for gaps after any byte until `release()` (default 120s). */
|
|
76
|
+
readonly gapTimeoutMs: number;
|
|
77
|
+
};
|
|
78
|
+
/**
|
|
79
|
+
* Wrap a raw upstream byte stream with the two-budget silence policy.
|
|
80
|
+
* Errors the stream with {@link UpstreamStalledError} on a trip before
|
|
81
|
+
* `release()`.
|
|
82
|
+
*/
|
|
83
|
+
export declare const withByteSilenceGuard: (source: ReadableStream<Uint8Array>, options: TByteSilenceGuardOptions) => TByteSilenceGuard;
|
|
84
|
+
/** Envelope used when a streaming headers / body wait stalls. */
|
|
85
|
+
export declare const upstreamStalledEnvelope: (message?: string) => {
|
|
86
|
+
readonly error: {
|
|
87
|
+
readonly message: string;
|
|
88
|
+
readonly type: "upstream_stalled";
|
|
89
|
+
readonly code: "upstream_stalled";
|
|
90
|
+
};
|
|
91
|
+
};
|
|
92
|
+
/** Shared HTTP status for a pre-commit upstream stall (cloud + daemon). */
|
|
93
|
+
export declare const UPSTREAM_STALLED_HTTP_STATUS = 504;
|
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
import { UpstreamStalledError } from "./upstream-error.js";
|
|
2
|
+
/**
|
|
3
|
+
* Pre-commit silence budgets for STREAMING hops only.
|
|
4
|
+
*
|
|
5
|
+
* Two clocks, both measured on raw upstream bytes (any enqueued chunk —
|
|
6
|
+
* including empty ones, keepalive comments, and pings — counts as alive):
|
|
7
|
+
*
|
|
8
|
+
* 1. **First-byte** (`OPENLLM_STREAM_FIRST_BYTE_TIMEOUT_MS`, default 300s)
|
|
9
|
+
* — waiting for response headers, and the gap before the first body
|
|
10
|
+
* byte. Near-cap prefills (hundreds of thousands of tokens) can
|
|
11
|
+
* legitimately take minutes with no keepalive; this budget is the
|
|
12
|
+
* one that must accommodate them. See `peek.ts`.
|
|
13
|
+
* 2. **Gap** (`OPENLLM_STREAM_SILENCE_TIMEOUT_MS`, default 120s) — after
|
|
14
|
+
* any byte has arrived, until `release()` (hop commit). A provider
|
|
15
|
+
* that went silent mid-prefill after talking once is treated as
|
|
16
|
+
* stalled.
|
|
17
|
+
*
|
|
18
|
+
* Non-streaming requests get no short timeout — they legitimately send
|
|
19
|
+
* nothing until the full JSON body is ready. A non-positive override
|
|
20
|
+
* disables that budget.
|
|
21
|
+
*/
|
|
22
|
+
export const DEFAULT_STREAM_FIRST_BYTE_TIMEOUT_MS = 300_000;
|
|
23
|
+
export const DEFAULT_STREAM_SILENCE_TIMEOUT_MS = 120_000;
|
|
24
|
+
const readTimeoutMs = (envName, fallback) => {
|
|
25
|
+
const raw = process.env[envName];
|
|
26
|
+
if (raw === undefined || raw === "")
|
|
27
|
+
return fallback;
|
|
28
|
+
const override = Number(raw);
|
|
29
|
+
if (!Number.isFinite(override))
|
|
30
|
+
return fallback;
|
|
31
|
+
return override;
|
|
32
|
+
};
|
|
33
|
+
export const streamFirstByteTimeoutMs = () => readTimeoutMs("OPENLLM_STREAM_FIRST_BYTE_TIMEOUT_MS", DEFAULT_STREAM_FIRST_BYTE_TIMEOUT_MS);
|
|
34
|
+
export const streamSilenceTimeoutMs = () => readTimeoutMs("OPENLLM_STREAM_SILENCE_TIMEOUT_MS", DEFAULT_STREAM_SILENCE_TIMEOUT_MS);
|
|
35
|
+
/**
|
|
36
|
+
* Whether a hop should apply the streaming silence budgets on its
|
|
37
|
+
* upstream fetch. ChatGPT's Codex endpoint always streams even when the
|
|
38
|
+
* client asked for JSON, so the gate is `alwaysStreams || clientWantsStream`
|
|
39
|
+
* — not the client flag alone.
|
|
40
|
+
*/
|
|
41
|
+
export const streamingHopNeedsSilence = (opts) => opts.upstreamAlwaysStreams || opts.clientWantsStream;
|
|
42
|
+
/**
|
|
43
|
+
* Absolute deadline for the combined headers + first-body-byte budget.
|
|
44
|
+
* Pass the remaining time into the body silence guard so the two waits
|
|
45
|
+
* share one first-byte window instead of stacking to ~2×.
|
|
46
|
+
*/
|
|
47
|
+
const STALL_ABORT_REASON = "openllm-upstream-stalled";
|
|
48
|
+
export const firstByteDeadlineAt = (firstByteTimeoutMs) => firstByteTimeoutMs > 0 ? Date.now() + firstByteTimeoutMs : null;
|
|
49
|
+
/**
|
|
50
|
+
* Time left before `deadlineAt`. `null` (no budget) returns 0, which every
|
|
51
|
+
* guard reads as disabled; an EXPIRED deadline returns 1 so the guard fires
|
|
52
|
+
* at once instead of being read as disabled.
|
|
53
|
+
*/
|
|
54
|
+
export const remainingTimeoutMs = (deadlineAt) => deadlineAt === null ? 0 : Math.max(1, deadlineAt - Date.now());
|
|
55
|
+
/**
|
|
56
|
+
* Read a (non-2xx) response body with `read`, failing with
|
|
57
|
+
* {@link UpstreamStalledError} if it takes longer than `timeoutMs`. The body
|
|
58
|
+
* is read through a reader this helper owns, so a stall can actually cancel
|
|
59
|
+
* it (a `text()` in flight locks the body, and `body.cancel()` would throw).
|
|
60
|
+
* `timeoutMs <= 0` disables the budget.
|
|
61
|
+
*/
|
|
62
|
+
export const readResponseWithin = async (response, timeoutMs, read) => {
|
|
63
|
+
if (!(timeoutMs > 0) || response.body === null)
|
|
64
|
+
return read(response);
|
|
65
|
+
const reader = response.body.getReader();
|
|
66
|
+
const owned = new Response(new ReadableStream({
|
|
67
|
+
pull: async (controller) => {
|
|
68
|
+
const next = await reader.read();
|
|
69
|
+
if (next.done)
|
|
70
|
+
controller.close();
|
|
71
|
+
else
|
|
72
|
+
controller.enqueue(next.value);
|
|
73
|
+
},
|
|
74
|
+
cancel: async (reason) => {
|
|
75
|
+
await reader.cancel(reason);
|
|
76
|
+
},
|
|
77
|
+
}), {
|
|
78
|
+
status: response.status,
|
|
79
|
+
statusText: response.statusText,
|
|
80
|
+
headers: response.headers,
|
|
81
|
+
});
|
|
82
|
+
let timer;
|
|
83
|
+
const stalled = new Promise((_, reject) => {
|
|
84
|
+
timer = setTimeout(() => {
|
|
85
|
+
reader.cancel(STALL_ABORT_REASON).catch(() => { });
|
|
86
|
+
reject(new UpstreamStalledError("upstream stalled reading error body"));
|
|
87
|
+
}, timeoutMs);
|
|
88
|
+
});
|
|
89
|
+
try {
|
|
90
|
+
return await Promise.race([read(owned), stalled]);
|
|
91
|
+
}
|
|
92
|
+
finally {
|
|
93
|
+
clearTimeout(timer);
|
|
94
|
+
}
|
|
95
|
+
};
|
|
96
|
+
/**
|
|
97
|
+
* AbortSignal that fires if response headers take longer than
|
|
98
|
+
* `firstByteTimeoutMs`. Cleared as soon as headers arrive so it cannot
|
|
99
|
+
* abort the body later. Caller abort still wins.
|
|
100
|
+
*/
|
|
101
|
+
export const withStreamingHeadersTimeout = (caller, firstByteTimeoutMs) => {
|
|
102
|
+
if (!(firstByteTimeoutMs > 0)) {
|
|
103
|
+
return {
|
|
104
|
+
signal: caller,
|
|
105
|
+
didStall: () => false,
|
|
106
|
+
clear: () => { },
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
const stall = new AbortController();
|
|
110
|
+
let stalled = false;
|
|
111
|
+
const timer = setTimeout(() => {
|
|
112
|
+
stalled = true;
|
|
113
|
+
stall.abort(STALL_ABORT_REASON);
|
|
114
|
+
}, firstByteTimeoutMs);
|
|
115
|
+
const clear = () => {
|
|
116
|
+
clearTimeout(timer);
|
|
117
|
+
};
|
|
118
|
+
if (caller.aborted) {
|
|
119
|
+
clear();
|
|
120
|
+
stall.abort(caller.reason);
|
|
121
|
+
}
|
|
122
|
+
else {
|
|
123
|
+
caller.addEventListener("abort", () => {
|
|
124
|
+
clear();
|
|
125
|
+
stall.abort(caller.reason);
|
|
126
|
+
}, { once: true });
|
|
127
|
+
}
|
|
128
|
+
return {
|
|
129
|
+
signal: stall.signal,
|
|
130
|
+
didStall: () => stalled && !caller.aborted,
|
|
131
|
+
clear,
|
|
132
|
+
};
|
|
133
|
+
};
|
|
134
|
+
/**
|
|
135
|
+
* Wrap a raw upstream byte stream with the two-budget silence policy.
|
|
136
|
+
* Errors the stream with {@link UpstreamStalledError} on a trip before
|
|
137
|
+
* `release()`.
|
|
138
|
+
*/
|
|
139
|
+
export const withByteSilenceGuard = (source, options) => {
|
|
140
|
+
const firstMs = options.firstByteTimeoutMs;
|
|
141
|
+
const gapMs = options.gapTimeoutMs;
|
|
142
|
+
if (!(firstMs > 0) && !(gapMs > 0)) {
|
|
143
|
+
return { stream: source, release: () => { } };
|
|
144
|
+
}
|
|
145
|
+
const reader = source.getReader();
|
|
146
|
+
let released = false;
|
|
147
|
+
let finished = false;
|
|
148
|
+
let sawByte = false;
|
|
149
|
+
let timer = null;
|
|
150
|
+
const STALL = Symbol("upstream-silence");
|
|
151
|
+
const RELEASED = Symbol("upstream-silence-released");
|
|
152
|
+
let settleRace = null;
|
|
153
|
+
let racePromise = null;
|
|
154
|
+
const clearTimer = () => {
|
|
155
|
+
if (timer !== null) {
|
|
156
|
+
clearTimeout(timer);
|
|
157
|
+
timer = null;
|
|
158
|
+
}
|
|
159
|
+
};
|
|
160
|
+
const currentBudgetMs = () => (sawByte ? gapMs : firstMs);
|
|
161
|
+
const armTimer = () => {
|
|
162
|
+
clearTimer();
|
|
163
|
+
if (released || finished)
|
|
164
|
+
return;
|
|
165
|
+
const budget = currentBudgetMs();
|
|
166
|
+
if (!(budget > 0)) {
|
|
167
|
+
racePromise = null;
|
|
168
|
+
settleRace = null;
|
|
169
|
+
return;
|
|
170
|
+
}
|
|
171
|
+
racePromise = new Promise((resolve) => {
|
|
172
|
+
settleRace = resolve;
|
|
173
|
+
timer = setTimeout(() => {
|
|
174
|
+
timer = null;
|
|
175
|
+
resolve(STALL);
|
|
176
|
+
}, budget);
|
|
177
|
+
});
|
|
178
|
+
};
|
|
179
|
+
const release = () => {
|
|
180
|
+
released = true;
|
|
181
|
+
clearTimer();
|
|
182
|
+
const settle = settleRace;
|
|
183
|
+
settleRace = null;
|
|
184
|
+
racePromise = null;
|
|
185
|
+
// Resolve any in-flight race so a late STALL is not observed as a stall.
|
|
186
|
+
if (settle !== null)
|
|
187
|
+
settle(RELEASED);
|
|
188
|
+
};
|
|
189
|
+
armTimer();
|
|
190
|
+
const stream = new ReadableStream({
|
|
191
|
+
async pull(controller) {
|
|
192
|
+
if (finished) {
|
|
193
|
+
controller.close();
|
|
194
|
+
return;
|
|
195
|
+
}
|
|
196
|
+
if (released) {
|
|
197
|
+
const read = await reader.read();
|
|
198
|
+
if (read.done) {
|
|
199
|
+
finished = true;
|
|
200
|
+
clearTimer();
|
|
201
|
+
controller.close();
|
|
202
|
+
return;
|
|
203
|
+
}
|
|
204
|
+
controller.enqueue(read.value);
|
|
205
|
+
return;
|
|
206
|
+
}
|
|
207
|
+
const read = reader.read();
|
|
208
|
+
read.catch(() => { });
|
|
209
|
+
if (racePromise === null)
|
|
210
|
+
armTimer();
|
|
211
|
+
const race = racePromise;
|
|
212
|
+
let result;
|
|
213
|
+
try {
|
|
214
|
+
result = race === null ? await read : await Promise.race([read, race]);
|
|
215
|
+
}
|
|
216
|
+
catch (err) {
|
|
217
|
+
finished = true;
|
|
218
|
+
clearTimer();
|
|
219
|
+
throw err;
|
|
220
|
+
}
|
|
221
|
+
if (result === RELEASED || (result === STALL && released)) {
|
|
222
|
+
// Commit raced the timer: wait for the outstanding read without stalling.
|
|
223
|
+
result = await read;
|
|
224
|
+
}
|
|
225
|
+
if (result === STALL) {
|
|
226
|
+
finished = true;
|
|
227
|
+
clearTimer();
|
|
228
|
+
reader.cancel(STALL_ABORT_REASON).catch(() => { });
|
|
229
|
+
controller.error(new UpstreamStalledError("upstream stalled before producing output"));
|
|
230
|
+
return;
|
|
231
|
+
}
|
|
232
|
+
if (result.done) {
|
|
233
|
+
finished = true;
|
|
234
|
+
clearTimer();
|
|
235
|
+
controller.close();
|
|
236
|
+
return;
|
|
237
|
+
}
|
|
238
|
+
// Any enqueued chunk — including empty — counts as alive.
|
|
239
|
+
sawByte = true;
|
|
240
|
+
if (!released) {
|
|
241
|
+
armTimer();
|
|
242
|
+
}
|
|
243
|
+
controller.enqueue(result.value);
|
|
244
|
+
},
|
|
245
|
+
cancel(reason) {
|
|
246
|
+
finished = true;
|
|
247
|
+
clearTimer();
|
|
248
|
+
reader.cancel(reason).catch(() => { });
|
|
249
|
+
},
|
|
250
|
+
});
|
|
251
|
+
return { stream, release };
|
|
252
|
+
};
|
|
253
|
+
/** Envelope used when a streaming headers / body wait stalls. */
|
|
254
|
+
export const upstreamStalledEnvelope = (message = "upstream stalled before producing output") => ({
|
|
255
|
+
error: {
|
|
256
|
+
message,
|
|
257
|
+
type: "upstream_stalled",
|
|
258
|
+
code: "upstream_stalled",
|
|
259
|
+
},
|
|
260
|
+
});
|
|
261
|
+
/** Shared HTTP status for a pre-commit upstream stall (cloud + daemon). */
|
|
262
|
+
export const UPSTREAM_STALLED_HTTP_STATUS = 504;
|
|
@@ -78,15 +78,20 @@ export declare const withFrameAlignedHeartbeat: (source: ReadableStream<Uint8Arr
|
|
|
78
78
|
*
|
|
79
79
|
* Wrap BEFORE `.tee()` so the usage-drain branch also terminates.
|
|
80
80
|
*/
|
|
81
|
+
export type TStreamDeadlineTruncateInfo = {
|
|
82
|
+
/** True when at least one content / tool / reasoning delta was forwarded. */
|
|
83
|
+
readonly sawOutput: boolean;
|
|
84
|
+
};
|
|
81
85
|
export declare const withStreamDeadline: (source: ReadableStream<TChatCompletionChunk>, options: {
|
|
82
86
|
readonly deadlineMs: number;
|
|
83
87
|
/**
|
|
84
88
|
* Fires exactly once if the deadline expires before the source
|
|
85
89
|
* closes naturally. Callers use this to skip caching a body that
|
|
86
90
|
* was sealed by the synthetic terminator instead of the upstream's
|
|
87
|
-
* own `finish_reason
|
|
91
|
+
* own `finish_reason`, and to log zero-output truncations as errors.
|
|
92
|
+
* Omit to ignore truncation.
|
|
88
93
|
*/
|
|
89
|
-
readonly onTruncate?: () => void;
|
|
94
|
+
readonly onTruncate?: (info: TStreamDeadlineTruncateInfo) => void;
|
|
90
95
|
}) => ReadableStream<TChatCompletionChunk>;
|
|
91
96
|
/**
|
|
92
97
|
* Result of wrapping a raw byte stream with a deadline guard.
|
|
@@ -99,6 +104,8 @@ export declare const withStreamDeadline: (source: ReadableStream<TChatCompletion
|
|
|
99
104
|
export type TByteStreamDeadlineHandle = {
|
|
100
105
|
readonly stream: ReadableStream<Uint8Array>;
|
|
101
106
|
readonly isTruncated: () => boolean;
|
|
107
|
+
/** True when the deadline fired with no non-keepalive output forwarded. */
|
|
108
|
+
readonly isZeroOutputTruncate: () => boolean;
|
|
102
109
|
};
|
|
103
110
|
/**
|
|
104
111
|
* Sibling of `withStreamDeadline` for the Anthropic pass-through:
|
|
@@ -106,7 +113,7 @@ export type TByteStreamDeadlineHandle = {
|
|
|
106
113
|
* the synthetic terminator must be a pre-encoded SSE byte sequence
|
|
107
114
|
* because there is no canonical-decode step in the pipeline.
|
|
108
115
|
*
|
|
109
|
-
* On cutoff:
|
|
116
|
+
* On cutoff with partial output:
|
|
110
117
|
* 1. cancel the source — propagating to the upstream `fetch` body
|
|
111
118
|
* so we stop paying the provider for tokens we'll never deliver;
|
|
112
119
|
* 2. enqueue ONE pre-encoded `message_delta` + `message_stop` pair
|
|
@@ -115,6 +122,10 @@ export type TByteStreamDeadlineHandle = {
|
|
|
115
122
|
* 3. close cleanly so SSE clients see a terminated turn instead of
|
|
116
123
|
* a severed connection.
|
|
117
124
|
*
|
|
125
|
+
* On cutoff with ZERO output: emit an Anthropic `event: error` instead
|
|
126
|
+
* of a clean `max_tokens` turn, so the client and the usage log treat
|
|
127
|
+
* it as a failure.
|
|
128
|
+
*
|
|
118
129
|
* Wrap BEFORE `.tee()` so the usage-drain branch also receives the
|
|
119
130
|
* terminator and the accumulator can finalise. The caller checks
|
|
120
131
|
* `isTruncated()` after consumption to skip caching a truncated body.
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { UpstreamStreamError } from "./upstream-error.js";
|
|
1
2
|
const SSE_DONE = "[DONE]";
|
|
2
3
|
const textEncoder = new TextEncoder();
|
|
3
4
|
export const encodeSseEvent = (data) => {
|
|
@@ -391,33 +392,19 @@ export const withFrameAlignedHeartbeat = (source, options) => {
|
|
|
391
392
|
},
|
|
392
393
|
});
|
|
393
394
|
};
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
* 1. cancels the source — propagating to the upstream `fetch` body
|
|
408
|
-
* so we stop paying the provider for tokens we'll never deliver;
|
|
409
|
-
* 2. emits ONE synthetic terminal chunk with
|
|
410
|
-
* `finish_reason: "length"` and closes cleanly.
|
|
411
|
-
*
|
|
412
|
-
* The text deltas already streamed ARE the partial answer (they're in
|
|
413
|
-
* the client's hands). The synthetic terminal makes the SSE encoders
|
|
414
|
-
* close properly — `data: [DONE]` for OpenAI, `message_delta`
|
|
415
|
-
* (`stop_reason: "max_tokens"`) + `message_stop` for Anthropic — so
|
|
416
|
-
* the caller gets a clean, truncated turn it can re-prompt to resume,
|
|
417
|
-
* instead of a thrown network error.
|
|
418
|
-
*
|
|
419
|
-
* Wrap BEFORE `.tee()` so the usage-drain branch also terminates.
|
|
420
|
-
*/
|
|
395
|
+
const chunkHasOutputDelta = (chunk) => {
|
|
396
|
+
const nonEmpty = (s) => typeof s === "string" && s.length > 0;
|
|
397
|
+
const nonEmptyArr = (a) => Array.isArray(a) && a.length > 0;
|
|
398
|
+
return chunk.choices.some((choice) => {
|
|
399
|
+
const d = choice.delta;
|
|
400
|
+
return (nonEmpty(d.content) ||
|
|
401
|
+
nonEmpty(d.reasoning_content) ||
|
|
402
|
+
nonEmpty(d.refusal) ||
|
|
403
|
+
nonEmptyArr(d.tool_calls) ||
|
|
404
|
+
nonEmptyArr(d.reasoning_items) ||
|
|
405
|
+
nonEmptyArr(d.server_search_calls));
|
|
406
|
+
});
|
|
407
|
+
};
|
|
421
408
|
export const withStreamDeadline = (source, options) => {
|
|
422
409
|
const reader = source.getReader();
|
|
423
410
|
let lastId = "chatcmpl-openllm-deadline";
|
|
@@ -425,6 +412,7 @@ export const withStreamDeadline = (source, options) => {
|
|
|
425
412
|
let lastCreated = 0;
|
|
426
413
|
let hasCreated = false;
|
|
427
414
|
let finished = false;
|
|
415
|
+
let sawOutput = false;
|
|
428
416
|
const DEADLINE = Symbol("deadline");
|
|
429
417
|
let timer = null;
|
|
430
418
|
const deadline = new Promise((resolve) => {
|
|
@@ -476,12 +464,19 @@ export const withStreamDeadline = (source, options) => {
|
|
|
476
464
|
reader.cancel("openllm-stream-deadline").catch(() => { });
|
|
477
465
|
if (options.onTruncate !== undefined) {
|
|
478
466
|
try {
|
|
479
|
-
options.onTruncate();
|
|
467
|
+
options.onTruncate({ sawOutput });
|
|
480
468
|
}
|
|
481
469
|
catch {
|
|
482
470
|
// Callback errors must not crash the stream.
|
|
483
471
|
}
|
|
484
472
|
}
|
|
473
|
+
if (!sawOutput) {
|
|
474
|
+
// Zero-output deadline: a clean `length` turn would look like a
|
|
475
|
+
// successful empty answer and skip the fallback chain's chance to
|
|
476
|
+
// have walked (already committed). Surface a real error frame.
|
|
477
|
+
controller.error(new UpstreamStreamError("upstream_stalled", "stream deadline reached with no output"));
|
|
478
|
+
return;
|
|
479
|
+
}
|
|
485
480
|
controller.enqueue(terminal());
|
|
486
481
|
controller.close();
|
|
487
482
|
return;
|
|
@@ -493,6 +488,9 @@ export const withStreamDeadline = (source, options) => {
|
|
|
493
488
|
return;
|
|
494
489
|
}
|
|
495
490
|
const value = r.value;
|
|
491
|
+
if (chunkHasOutputDelta(value)) {
|
|
492
|
+
sawOutput = true;
|
|
493
|
+
}
|
|
496
494
|
if (typeof value.id === "string" && value.id.length > 0) {
|
|
497
495
|
lastId = value.id;
|
|
498
496
|
}
|
|
@@ -512,13 +510,22 @@ export const withStreamDeadline = (source, options) => {
|
|
|
512
510
|
},
|
|
513
511
|
});
|
|
514
512
|
};
|
|
513
|
+
const byteChunkLooksLikeOutput = (bytes) => {
|
|
514
|
+
// Only content / tool / reasoning deltas count as partial output.
|
|
515
|
+
// Housekeeping (`message_start`, role-only frames) and keepalives must
|
|
516
|
+
// NOT flip a zero-output deadline into a clean `max_tokens` turn.
|
|
517
|
+
const text = new TextDecoder().decode(bytes);
|
|
518
|
+
if (text.length === 0)
|
|
519
|
+
return false;
|
|
520
|
+
return (/text_delta|input_json_delta|thinking_delta|signature_delta|tool_use|reasoning_content|"content":"[^"]/i.test(text));
|
|
521
|
+
};
|
|
515
522
|
/**
|
|
516
523
|
* Sibling of `withStreamDeadline` for the Anthropic pass-through:
|
|
517
524
|
* the source is raw provider SSE bytes (not canonical chunks), and
|
|
518
525
|
* the synthetic terminator must be a pre-encoded SSE byte sequence
|
|
519
526
|
* because there is no canonical-decode step in the pipeline.
|
|
520
527
|
*
|
|
521
|
-
* On cutoff:
|
|
528
|
+
* On cutoff with partial output:
|
|
522
529
|
* 1. cancel the source — propagating to the upstream `fetch` body
|
|
523
530
|
* so we stop paying the provider for tokens we'll never deliver;
|
|
524
531
|
* 2. enqueue ONE pre-encoded `message_delta` + `message_stop` pair
|
|
@@ -527,6 +534,10 @@ export const withStreamDeadline = (source, options) => {
|
|
|
527
534
|
* 3. close cleanly so SSE clients see a terminated turn instead of
|
|
528
535
|
* a severed connection.
|
|
529
536
|
*
|
|
537
|
+
* On cutoff with ZERO output: emit an Anthropic `event: error` instead
|
|
538
|
+
* of a clean `max_tokens` turn, so the client and the usage log treat
|
|
539
|
+
* it as a failure.
|
|
540
|
+
*
|
|
530
541
|
* Wrap BEFORE `.tee()` so the usage-drain branch also receives the
|
|
531
542
|
* terminator and the accumulator can finalise. The caller checks
|
|
532
543
|
* `isTruncated()` after consumption to skip caching a truncated body.
|
|
@@ -541,6 +552,8 @@ export const withByteStreamDeadline = (source, options) => {
|
|
|
541
552
|
const reader = source.getReader();
|
|
542
553
|
let finished = false;
|
|
543
554
|
let truncated = false;
|
|
555
|
+
let sawOutput = false;
|
|
556
|
+
let zeroOutputTruncate = false;
|
|
544
557
|
const DEADLINE = Symbol("byte-stream-deadline");
|
|
545
558
|
let timer = null;
|
|
546
559
|
const deadline = new Promise((resolve) => {
|
|
@@ -552,8 +565,11 @@ export const withByteStreamDeadline = (source, options) => {
|
|
|
552
565
|
timer = null;
|
|
553
566
|
}
|
|
554
567
|
};
|
|
555
|
-
|
|
568
|
+
// Leading `\n\n` resynchronises if the deadline fired mid-frame so the
|
|
569
|
+
// synthetic event always starts a fresh SSE record.
|
|
570
|
+
const lengthTerminator = () => textEncoder.encode('\n\nevent: message_delta\ndata: {"type":"message_delta","delta":{"stop_reason":"max_tokens","stop_sequence":null},"usage":{"output_tokens":0}}\n\n' +
|
|
556
571
|
'event: message_stop\ndata: {"type":"message_stop"}\n\n');
|
|
572
|
+
const zeroOutputError = () => textEncoder.encode('\n\nevent: error\ndata: {"type":"error","error":{"type":"upstream_stalled","message":"stream deadline reached with no output"}}\n\n');
|
|
557
573
|
const stream = new ReadableStream({
|
|
558
574
|
async pull(controller) {
|
|
559
575
|
if (finished) {
|
|
@@ -578,7 +594,13 @@ export const withByteStreamDeadline = (source, options) => {
|
|
|
578
594
|
truncated = true;
|
|
579
595
|
clear();
|
|
580
596
|
reader.cancel("openllm-byte-stream-deadline").catch(() => { });
|
|
581
|
-
|
|
597
|
+
if (!sawOutput) {
|
|
598
|
+
zeroOutputTruncate = true;
|
|
599
|
+
controller.enqueue(zeroOutputError());
|
|
600
|
+
}
|
|
601
|
+
else {
|
|
602
|
+
controller.enqueue(lengthTerminator());
|
|
603
|
+
}
|
|
582
604
|
controller.close();
|
|
583
605
|
return;
|
|
584
606
|
}
|
|
@@ -588,6 +610,9 @@ export const withByteStreamDeadline = (source, options) => {
|
|
|
588
610
|
controller.close();
|
|
589
611
|
return;
|
|
590
612
|
}
|
|
613
|
+
if (byteChunkLooksLikeOutput(r.value)) {
|
|
614
|
+
sawOutput = true;
|
|
615
|
+
}
|
|
591
616
|
controller.enqueue(r.value);
|
|
592
617
|
},
|
|
593
618
|
cancel(reason) {
|
|
@@ -596,5 +621,9 @@ export const withByteStreamDeadline = (source, options) => {
|
|
|
596
621
|
reader.cancel(reason).catch(() => { });
|
|
597
622
|
},
|
|
598
623
|
});
|
|
599
|
-
return {
|
|
624
|
+
return {
|
|
625
|
+
stream,
|
|
626
|
+
isTruncated: () => truncated,
|
|
627
|
+
isZeroOutputTruncate: () => zeroOutputTruncate,
|
|
628
|
+
};
|
|
600
629
|
};
|
|
@@ -11,6 +11,15 @@ export declare class UpstreamStreamError extends Error {
|
|
|
11
11
|
readonly upstreamType: string;
|
|
12
12
|
constructor(upstreamType: string, message: string);
|
|
13
13
|
}
|
|
14
|
+
/**
|
|
15
|
+
* A streaming hop went silent before any byte was forwarded to the client
|
|
16
|
+
* (headers wait or raw upstream inactivity). Distinct from a mid-stream
|
|
17
|
+
* decode error so the classifier can tag `upstream_stalled` and the
|
|
18
|
+
* fallback chain can walk while the hop is still uncommitted.
|
|
19
|
+
*/
|
|
20
|
+
export declare class UpstreamStalledError extends UpstreamStreamError {
|
|
21
|
+
constructor(message?: string);
|
|
22
|
+
}
|
|
14
23
|
/**
|
|
15
24
|
* A stream ended before a terminal chunk after its decoder had already observed
|
|
16
25
|
* usage. The accumulator wraps every failure type, including transport errors,
|
|
@@ -30,3 +39,5 @@ export declare const upstreamErrorFrom: (err: unknown) => {
|
|
|
30
39
|
type: string;
|
|
31
40
|
message: string;
|
|
32
41
|
};
|
|
42
|
+
/** True when `err` is (or wraps) an {@link UpstreamStalledError}. */
|
|
43
|
+
export declare const isUpstreamStalledError: (err: unknown) => boolean;
|
|
@@ -14,6 +14,18 @@ export class UpstreamStreamError extends Error {
|
|
|
14
14
|
this.upstreamType = upstreamType;
|
|
15
15
|
}
|
|
16
16
|
}
|
|
17
|
+
/**
|
|
18
|
+
* A streaming hop went silent before any byte was forwarded to the client
|
|
19
|
+
* (headers wait or raw upstream inactivity). Distinct from a mid-stream
|
|
20
|
+
* decode error so the classifier can tag `upstream_stalled` and the
|
|
21
|
+
* fallback chain can walk while the hop is still uncommitted.
|
|
22
|
+
*/
|
|
23
|
+
export class UpstreamStalledError extends UpstreamStreamError {
|
|
24
|
+
constructor(message = "upstream stalled before producing output") {
|
|
25
|
+
super("upstream_stalled", message);
|
|
26
|
+
this.name = "UpstreamStalledError";
|
|
27
|
+
}
|
|
28
|
+
}
|
|
17
29
|
/**
|
|
18
30
|
* A stream ended before a terminal chunk after its decoder had already observed
|
|
19
31
|
* usage. The accumulator wraps every failure type, including transport errors,
|
|
@@ -43,3 +55,12 @@ export const upstreamErrorFrom = (err) => {
|
|
|
43
55
|
}
|
|
44
56
|
return { type: "stream_error", message: String(err) };
|
|
45
57
|
};
|
|
58
|
+
/** True when `err` is (or wraps) an {@link UpstreamStalledError}. */
|
|
59
|
+
export const isUpstreamStalledError = (err) => {
|
|
60
|
+
if (err instanceof UpstreamStalledError)
|
|
61
|
+
return true;
|
|
62
|
+
if (err instanceof Error && err.cause !== undefined) {
|
|
63
|
+
return isUpstreamStalledError(err.cause);
|
|
64
|
+
}
|
|
65
|
+
return false;
|
|
66
|
+
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@openllmsh/sdk",
|
|
3
|
-
"version": "2.10.
|
|
3
|
+
"version": "2.10.15-alpha.2",
|
|
4
4
|
"private": false,
|
|
5
5
|
"description": "The OpenLLM SDK: typed API contracts, wire-format adapters, the device tunnel, client-side vault crypto, the OpenAPI spec and MCP tools.",
|
|
6
6
|
"keywords": [
|