@openllmsh/sdk 2.10.15-alpha.1 → 2.10.15-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/protocol/agent.d.ts +28 -6
- package/dist/protocol/agent.js +20 -3
- package/dist/protocol/cooldown-reason.d.ts +1 -1
- package/dist/protocol/cooldown-reason.js +4 -0
- package/dist/protocol/daemon.d.ts +1 -1
- package/dist/protocol/media-attempt.js +1 -0
- package/dist/wire/index.d.ts +1 -0
- package/dist/wire/index.js +1 -0
- package/dist/wire/lib/error-class.js +10 -3
- package/dist/wire/lib/streaming/peek.d.ts +10 -10
- package/dist/wire/lib/streaming/peek.js +10 -10
- package/dist/wire/lib/streaming/silence.d.ts +93 -0
- package/dist/wire/lib/streaming/silence.js +262 -0
- package/dist/wire/lib/streaming/sse.d.ts +14 -3
- package/dist/wire/lib/streaming/sse.js +61 -32
- package/dist/wire/lib/streaming/upstream-error.d.ts +11 -0
- package/dist/wire/lib/streaming/upstream-error.js +21 -0
- package/package.json +1 -1
package/dist/protocol/agent.d.ts
CHANGED
|
@@ -64,6 +64,11 @@ export declare const AGENT_HOOK_EVENTS: readonly ["PermissionRequest", "PreToolU
|
|
|
64
64
|
export declare const AGENT_MAX_ATTACHMENTS = 8;
|
|
65
65
|
/** Max base64 characters of one inline attachment. */
|
|
66
66
|
export declare const AGENT_MAX_ATTACHMENT_BASE64_LENGTH: number;
|
|
67
|
+
/**
|
|
68
|
+
* Max characters of one absolute local-path attachment (same bound as a
|
|
69
|
+
* session cwd). Path attachments are loopback-only and never cross the network.
|
|
70
|
+
*/
|
|
71
|
+
export declare const AGENT_MAX_ATTACHMENT_PATH_LENGTH = 1024;
|
|
67
72
|
/**
|
|
68
73
|
* Engines an agent session can run. Each id is the `openllm <client>` the
|
|
69
74
|
* daemon launches the engine through (same overlay, gateway URL, key, model
|
|
@@ -100,14 +105,27 @@ export declare const AgentStreamOpenPayload: S.Struct<{
|
|
|
100
105
|
readonly kind: S.Literal<"agent">;
|
|
101
106
|
}>;
|
|
102
107
|
export type TAgentStreamOpenPayload = S.Schema.Type<typeof AgentStreamOpenPayload>;
|
|
108
|
+
/**
|
|
109
|
+
* One attachment on `send`. Exactly one of `data` (base64, any transport) or
|
|
110
|
+
* `path` (absolute local file, loopback agent socket only — the daemon rejects
|
|
111
|
+
* `path` on mux/RTC). Both together are refused (same XOR pattern as
|
|
112
|
+
* {@link AgentRespondCommand}).
|
|
113
|
+
*/
|
|
103
114
|
export declare const AgentAttachment: S.Struct<{
|
|
104
115
|
readonly kind: S.Literals<readonly ["image", "file"]>;
|
|
105
116
|
readonly name: S.optional<S.String>;
|
|
106
117
|
readonly media_type: S.String;
|
|
107
|
-
/** Base64 bytes. */
|
|
108
|
-
readonly data: S.String
|
|
118
|
+
/** Base64 bytes. Mutually exclusive with {@link path}. */
|
|
119
|
+
readonly data: S.optional<S.String>;
|
|
120
|
+
/** Absolute local path; admitted only on the desktop loopback agent socket. */
|
|
121
|
+
readonly path: S.optional<S.String>;
|
|
109
122
|
}>;
|
|
110
123
|
export type TAgentAttachment = S.Schema.Type<typeof AgentAttachment>;
|
|
124
|
+
/** True when the attachment carries an absolute local path instead of bytes. */
|
|
125
|
+
export declare const agentAttachmentHasPath: (attachment: TAgentAttachment) => attachment is TAgentAttachment & {
|
|
126
|
+
readonly path: string;
|
|
127
|
+
readonly data?: undefined;
|
|
128
|
+
};
|
|
111
129
|
/**
|
|
112
130
|
* Metadata for an attachment that was saved on the device for a turn. The
|
|
113
131
|
* `name` is the on-disk basename under the session's attachments dir (what
|
|
@@ -175,8 +193,10 @@ export declare const AgentSendCommand: S.Struct<{
|
|
|
175
193
|
readonly kind: S.Literals<readonly ["image", "file"]>;
|
|
176
194
|
readonly name: S.optional<S.String>;
|
|
177
195
|
readonly media_type: S.String;
|
|
178
|
-
/** Base64 bytes. */
|
|
179
|
-
readonly data: S.String
|
|
196
|
+
/** Base64 bytes. Mutually exclusive with {@link path}. */
|
|
197
|
+
readonly data: S.optional<S.String>;
|
|
198
|
+
/** Absolute local path; admitted only on the desktop loopback agent socket. */
|
|
199
|
+
readonly path: S.optional<S.String>;
|
|
180
200
|
}>>>;
|
|
181
201
|
}>;
|
|
182
202
|
export type TAgentSendCommand = S.Schema.Type<typeof AgentSendCommand>;
|
|
@@ -335,8 +355,10 @@ export declare const AgentCommand: S.Union<readonly [S.Struct<{
|
|
|
335
355
|
readonly kind: S.Literals<readonly ["image", "file"]>;
|
|
336
356
|
readonly name: S.optional<S.String>;
|
|
337
357
|
readonly media_type: S.String;
|
|
338
|
-
/** Base64 bytes. */
|
|
339
|
-
readonly data: S.String
|
|
358
|
+
/** Base64 bytes. Mutually exclusive with {@link path}. */
|
|
359
|
+
readonly data: S.optional<S.String>;
|
|
360
|
+
/** Absolute local path; admitted only on the desktop loopback agent socket. */
|
|
361
|
+
readonly path: S.optional<S.String>;
|
|
340
362
|
}>>>;
|
|
341
363
|
}>, S.Struct<{
|
|
342
364
|
readonly type: S.Literal<"send_now">;
|
package/dist/protocol/agent.js
CHANGED
|
@@ -72,6 +72,11 @@ export const AGENT_HOOK_EVENTS = [
|
|
|
72
72
|
export const AGENT_MAX_ATTACHMENTS = 8;
|
|
73
73
|
/** Max base64 characters of one inline attachment. */
|
|
74
74
|
export const AGENT_MAX_ATTACHMENT_BASE64_LENGTH = 6 * 1024 * 1024;
|
|
75
|
+
/**
|
|
76
|
+
* Max characters of one absolute local-path attachment (same bound as a
|
|
77
|
+
* session cwd). Path attachments are loopback-only and never cross the network.
|
|
78
|
+
*/
|
|
79
|
+
export const AGENT_MAX_ATTACHMENT_PATH_LENGTH = 1024;
|
|
75
80
|
/**
|
|
76
81
|
* Engines an agent session can run. Each id is the `openllm <client>` the
|
|
77
82
|
* daemon launches the engine through (same overlay, gateway URL, key, model
|
|
@@ -120,13 +125,25 @@ export const AgentStreamOpenPayload = S.Struct({
|
|
|
120
125
|
// ---------------------------------------------------------------------------
|
|
121
126
|
// Commands (consumer → daemon)
|
|
122
127
|
// ---------------------------------------------------------------------------
|
|
128
|
+
/**
|
|
129
|
+
* One attachment on `send`. Exactly one of `data` (base64, any transport) or
|
|
130
|
+
* `path` (absolute local file, loopback agent socket only — the daemon rejects
|
|
131
|
+
* `path` on mux/RTC). Both together are refused (same XOR pattern as
|
|
132
|
+
* {@link AgentRespondCommand}).
|
|
133
|
+
*/
|
|
123
134
|
export const AgentAttachment = S.Struct({
|
|
124
135
|
kind: S.Literals(["image", "file"]),
|
|
125
136
|
name: S.optional(S.String.check(S.isMinLength(1), S.isMaxLength(256))),
|
|
126
137
|
media_type: S.String.check(S.isMinLength(1), S.isMaxLength(128)),
|
|
127
|
-
/** Base64 bytes. */
|
|
128
|
-
data: S.String.check(S.isMaxLength(AGENT_MAX_ATTACHMENT_BASE64_LENGTH)),
|
|
129
|
-
|
|
138
|
+
/** Base64 bytes. Mutually exclusive with {@link path}. */
|
|
139
|
+
data: S.optional(S.String.check(S.isMaxLength(AGENT_MAX_ATTACHMENT_BASE64_LENGTH))),
|
|
140
|
+
/** Absolute local path; admitted only on the desktop loopback agent socket. */
|
|
141
|
+
path: S.optional(S.String.check(S.isMinLength(1), S.isMaxLength(AGENT_MAX_ATTACHMENT_PATH_LENGTH))),
|
|
142
|
+
}).check(S.makeFilter((attachment) => (attachment.data === undefined) !== (attachment.path === undefined), {
|
|
143
|
+
message: "attachment carries exactly one of data / path",
|
|
144
|
+
}));
|
|
145
|
+
/** True when the attachment carries an absolute local path instead of bytes. */
|
|
146
|
+
export const agentAttachmentHasPath = (attachment) => typeof attachment.path === "string";
|
|
130
147
|
/**
|
|
131
148
|
* Metadata for an attachment that was saved on the device for a turn. The
|
|
132
149
|
* `name` is the on-disk basename under the session's attachments dir (what
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { Schema as S } from "effect";
|
|
2
|
-
export declare const CooldownReason: S.Literals<readonly ["network", "timeout", "rate_limit", "quota_exhausted", "server_error", "auth", "payment", "not_found", "payload_too_large", "unprocessable", "request_invalid", "accepted_unreadable", "context_overflow", "content_filter", "upstream_rejection"]>;
|
|
2
|
+
export declare const CooldownReason: S.Literals<readonly ["network", "timeout", "upstream_stalled", "rate_limit", "quota_exhausted", "server_error", "auth", "payment", "not_found", "payload_too_large", "unprocessable", "request_invalid", "accepted_unreadable", "context_overflow", "content_filter", "upstream_rejection"]>;
|
|
3
3
|
export type TCooldownReason = S.Schema.Type<typeof CooldownReason>;
|
|
4
4
|
export type THopAction = "retry_in_place" | "cool_and_advance" | "walk_no_cool" | "surface";
|
|
5
5
|
/**
|
|
@@ -2,6 +2,7 @@ import { Schema as S } from "effect";
|
|
|
2
2
|
export const CooldownReason = S.Literals([
|
|
3
3
|
"network",
|
|
4
4
|
"timeout",
|
|
5
|
+
"upstream_stalled",
|
|
5
6
|
"rate_limit",
|
|
6
7
|
"quota_exhausted",
|
|
7
8
|
"server_error",
|
|
@@ -19,6 +20,9 @@ export const CooldownReason = S.Literals([
|
|
|
19
20
|
const cooldownPolicies = {
|
|
20
21
|
network: { action: "retry_in_place" },
|
|
21
22
|
timeout: { action: "retry_in_place" },
|
|
23
|
+
// Streaming hop went silent before any client byte (headers or body). Cool
|
|
24
|
+
// briefly so a dead provider isn't retried immediately for the same turn.
|
|
25
|
+
upstream_stalled: { action: "cool_and_advance", ttlMs: 60_000 },
|
|
22
26
|
rate_limit: { action: "cool_and_advance", ttlMs: 180_000 },
|
|
23
27
|
quota_exhausted: { action: "cool_and_advance", ttlMs: 180_000 },
|
|
24
28
|
server_error: { action: "retry_in_place" },
|
|
@@ -509,7 +509,7 @@ export declare const DaemonRecordRequest: S.Struct<{
|
|
|
509
509
|
* cloud charges each one the hold's full reservation.
|
|
510
510
|
*/
|
|
511
511
|
readonly hold_unknown_attempts: S.optional<S.Number>;
|
|
512
|
-
readonly cooldown_reason: S.optional<S.Literals<readonly ["network", "timeout", "rate_limit", "quota_exhausted", "server_error", "auth", "payment", "not_found", "payload_too_large", "unprocessable", "request_invalid", "accepted_unreadable", "context_overflow", "content_filter", "upstream_rejection"]>>;
|
|
512
|
+
readonly cooldown_reason: S.optional<S.Literals<readonly ["network", "timeout", "upstream_stalled", "rate_limit", "quota_exhausted", "server_error", "auth", "payment", "not_found", "payload_too_large", "unprocessable", "request_invalid", "accepted_unreadable", "context_overflow", "content_filter", "upstream_rejection"]>>;
|
|
513
513
|
/** Reset instant for reset-aware TTL cap; optional so old daemons omit. */
|
|
514
514
|
readonly reset_at_ms: S.optional<S.Number>;
|
|
515
515
|
}>;
|
package/dist/wire/index.d.ts
CHANGED
|
@@ -36,6 +36,7 @@ export * from "./lib/refusal.js";
|
|
|
36
36
|
export * from "./lib/secret-scrub.js";
|
|
37
37
|
export * from "./lib/streaming/accumulate.js";
|
|
38
38
|
export * from "./lib/streaming/peek.js";
|
|
39
|
+
export * from "./lib/streaming/silence.js";
|
|
39
40
|
export * from "./lib/streaming/sse.js";
|
|
40
41
|
export * from "./lib/streaming/strip-tool-calls.js";
|
|
41
42
|
export * from "./lib/streaming/upstream-error.js";
|
package/dist/wire/index.js
CHANGED
|
@@ -36,6 +36,7 @@ export * from "./lib/refusal.js";
|
|
|
36
36
|
export * from "./lib/secret-scrub.js";
|
|
37
37
|
export * from "./lib/streaming/accumulate.js";
|
|
38
38
|
export * from "./lib/streaming/peek.js";
|
|
39
|
+
export * from "./lib/streaming/silence.js";
|
|
39
40
|
export * from "./lib/streaming/sse.js";
|
|
40
41
|
export * from "./lib/streaming/strip-tool-calls.js";
|
|
41
42
|
export * from "./lib/streaming/upstream-error.js";
|
|
@@ -32,11 +32,18 @@ export const reasonFor = (i, rules = errorRulesFor(i.provider)) => {
|
|
|
32
32
|
// A 2xx the runner still couldn't use (an undecodable 200): the provider
|
|
33
33
|
// ACCEPTED the request and may have billed it, so walking on would send —
|
|
34
34
|
// and bill — it again. Only an explicit in-band refusal (content_filter,
|
|
35
|
-
// above) walks from a 2xx; no body matcher or provider rule
|
|
36
|
-
//
|
|
37
|
-
//
|
|
35
|
+
// above) walks from a 2xx; no body matcher or provider rule — including a
|
|
36
|
+
// forged `upstream_stalled` code on an accepted response — may turn an
|
|
37
|
+
// accepted request into a walk.
|
|
38
38
|
if (i.status >= 200 && i.status < 300)
|
|
39
39
|
return "accepted_unreadable";
|
|
40
|
+
// Pre-commit synthetic stall (headers / raw-byte silence). Our runners and
|
|
41
|
+
// the daemon emit this only on non-2xx outcomes (typically 504). Kept AFTER
|
|
42
|
+
// the 2xx guard so an accepted provider response can never be re-POSTed.
|
|
43
|
+
if (i.envelope?.error?.code === "upstream_stalled" ||
|
|
44
|
+
i.envelope?.error?.type === "upstream_stalled") {
|
|
45
|
+
return "upstream_stalled";
|
|
46
|
+
}
|
|
40
47
|
const message = i.envelope?.error?.message ?? "";
|
|
41
48
|
if (CONTEXT_OVERFLOW_BODY.test(message))
|
|
42
49
|
return "context_overflow";
|
|
@@ -45,16 +45,16 @@ export declare const isMeaningfulChunk: (chunk: TChatCompletionChunk) => boolean
|
|
|
45
45
|
* the module doc for why), then return a stream that replays what was
|
|
46
46
|
* buffered ahead of the live tail.
|
|
47
47
|
*
|
|
48
|
-
* The wait
|
|
49
|
-
* signal
|
|
50
|
-
*
|
|
51
|
-
*
|
|
52
|
-
*
|
|
53
|
-
*
|
|
54
|
-
*
|
|
55
|
-
*
|
|
56
|
-
*
|
|
57
|
-
*
|
|
48
|
+
* The wait for a MEANINGFUL decoded event is still unbounded here — the
|
|
49
|
+
* request abort signal is the only hard stop at this layer. Raw-byte
|
|
50
|
+
* silence is enforced BELOW the decoder by `withByteSilenceGuard` /
|
|
51
|
+
* `withStreamingHeadersTimeout` (see `silence.ts`): a generous
|
|
52
|
+
* first-byte budget (default 300s, `OPENLLM_STREAM_FIRST_BYTE_TIMEOUT_MS`)
|
|
53
|
+
* covers near-cap prefills that produce no keepalive, and a shorter gap
|
|
54
|
+
* budget (default 120s, `OPENLLM_STREAM_SILENCE_TIMEOUT_MS`) trips only
|
|
55
|
+
* after the upstream has already spoken. That split keeps the overflow
|
|
56
|
+
* peek working for long prefills without letting a truly silent hop
|
|
57
|
+
* burn the Vercel deadline as a fake `length` success.
|
|
58
58
|
*
|
|
59
59
|
* A stream that CLOSES before any event is an error by default (walkable
|
|
60
60
|
* — the upstream produced nothing), not an empty success. EXCEPT when the
|
|
@@ -30,16 +30,16 @@ export const isMeaningfulChunk = (chunk) => {
|
|
|
30
30
|
* the module doc for why), then return a stream that replays what was
|
|
31
31
|
* buffered ahead of the live tail.
|
|
32
32
|
*
|
|
33
|
-
* The wait
|
|
34
|
-
* signal
|
|
35
|
-
*
|
|
36
|
-
*
|
|
37
|
-
*
|
|
38
|
-
*
|
|
39
|
-
*
|
|
40
|
-
*
|
|
41
|
-
*
|
|
42
|
-
*
|
|
33
|
+
* The wait for a MEANINGFUL decoded event is still unbounded here — the
|
|
34
|
+
* request abort signal is the only hard stop at this layer. Raw-byte
|
|
35
|
+
* silence is enforced BELOW the decoder by `withByteSilenceGuard` /
|
|
36
|
+
* `withStreamingHeadersTimeout` (see `silence.ts`): a generous
|
|
37
|
+
* first-byte budget (default 300s, `OPENLLM_STREAM_FIRST_BYTE_TIMEOUT_MS`)
|
|
38
|
+
* covers near-cap prefills that produce no keepalive, and a shorter gap
|
|
39
|
+
* budget (default 120s, `OPENLLM_STREAM_SILENCE_TIMEOUT_MS`) trips only
|
|
40
|
+
* after the upstream has already spoken. That split keeps the overflow
|
|
41
|
+
* peek working for long prefills without letting a truly silent hop
|
|
42
|
+
* burn the Vercel deadline as a fake `length` success.
|
|
43
43
|
*
|
|
44
44
|
* A stream that CLOSES before any event is an error by default (walkable
|
|
45
45
|
* — the upstream produced nothing), not an empty success. EXCEPT when the
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pre-commit silence budgets for STREAMING hops only.
|
|
3
|
+
*
|
|
4
|
+
* Two clocks, both measured on raw upstream bytes (any enqueued chunk —
|
|
5
|
+
* including empty ones, keepalive comments, and pings — counts as alive):
|
|
6
|
+
*
|
|
7
|
+
* 1. **First-byte** (`OPENLLM_STREAM_FIRST_BYTE_TIMEOUT_MS`, default 300s)
|
|
8
|
+
* — waiting for response headers, and the gap before the first body
|
|
9
|
+
* byte. Near-cap prefills (hundreds of thousands of tokens) can
|
|
10
|
+
* legitimately take minutes with no keepalive; this budget is the
|
|
11
|
+
* one that must accommodate them. See `peek.ts`.
|
|
12
|
+
* 2. **Gap** (`OPENLLM_STREAM_SILENCE_TIMEOUT_MS`, default 120s) — after
|
|
13
|
+
* any byte has arrived, until `release()` (hop commit). A provider
|
|
14
|
+
* that went silent mid-prefill after talking once is treated as
|
|
15
|
+
* stalled.
|
|
16
|
+
*
|
|
17
|
+
* Non-streaming requests get no short timeout — they legitimately send
|
|
18
|
+
* nothing until the full JSON body is ready. A non-positive override
|
|
19
|
+
* disables that budget.
|
|
20
|
+
*/
|
|
21
|
+
export declare const DEFAULT_STREAM_FIRST_BYTE_TIMEOUT_MS = 300000;
|
|
22
|
+
export declare const DEFAULT_STREAM_SILENCE_TIMEOUT_MS = 120000;
|
|
23
|
+
export declare const streamFirstByteTimeoutMs: () => number;
|
|
24
|
+
export declare const streamSilenceTimeoutMs: () => number;
|
|
25
|
+
/**
|
|
26
|
+
* Whether a hop should apply the streaming silence budgets on its
|
|
27
|
+
* upstream fetch. ChatGPT's Codex endpoint always streams even when the
|
|
28
|
+
* client asked for JSON, so the gate is `alwaysStreams || clientWantsStream`
|
|
29
|
+
* — not the client flag alone.
|
|
30
|
+
*/
|
|
31
|
+
export declare const streamingHopNeedsSilence: (opts: {
|
|
32
|
+
readonly upstreamAlwaysStreams: boolean;
|
|
33
|
+
readonly clientWantsStream: boolean;
|
|
34
|
+
}) => boolean;
|
|
35
|
+
export declare const firstByteDeadlineAt: (firstByteTimeoutMs: number) => number | null;
|
|
36
|
+
/**
|
|
37
|
+
* Time left before `deadlineAt`. `null` (no budget) returns 0, which every
|
|
38
|
+
* guard reads as disabled; an EXPIRED deadline returns 1 so the guard fires
|
|
39
|
+
* at once instead of being read as disabled.
|
|
40
|
+
*/
|
|
41
|
+
export declare const remainingTimeoutMs: (deadlineAt: number | null) => number;
|
|
42
|
+
/**
|
|
43
|
+
* Read a (non-2xx) response body with `read`, failing with
|
|
44
|
+
* {@link UpstreamStalledError} if it takes longer than `timeoutMs`. The body
|
|
45
|
+
* is read through a reader this helper owns, so a stall can actually cancel
|
|
46
|
+
* it (a `text()` in flight locks the body, and `body.cancel()` would throw).
|
|
47
|
+
* `timeoutMs <= 0` disables the budget.
|
|
48
|
+
*/
|
|
49
|
+
export declare const readResponseWithin: <T>(response: Response, timeoutMs: number, read: (response: Response) => Promise<T>) => Promise<T>;
|
|
50
|
+
export type TStreamingHeadersGuard = {
|
|
51
|
+
readonly signal: AbortSignal;
|
|
52
|
+
/** True when the first-byte headers timer fired and the caller did not abort. */
|
|
53
|
+
readonly didStall: () => boolean;
|
|
54
|
+
/** Clear the headers timer once the Response headers have arrived. */
|
|
55
|
+
readonly clear: () => void;
|
|
56
|
+
};
|
|
57
|
+
/**
|
|
58
|
+
* AbortSignal that fires if response headers take longer than
|
|
59
|
+
* `firstByteTimeoutMs`. Cleared as soon as headers arrive so it cannot
|
|
60
|
+
* abort the body later. Caller abort still wins.
|
|
61
|
+
*/
|
|
62
|
+
export declare const withStreamingHeadersTimeout: (caller: AbortSignal, firstByteTimeoutMs: number) => TStreamingHeadersGuard;
|
|
63
|
+
export type TByteSilenceGuard = {
|
|
64
|
+
readonly stream: ReadableStream<Uint8Array>;
|
|
65
|
+
/**
|
|
66
|
+
* Disarm the silence timer. Call once the hop commits (first chunk is
|
|
67
|
+
* about to be forwarded to the client) so mid-stream quiet no longer
|
|
68
|
+
* walks the fallback chain.
|
|
69
|
+
*/
|
|
70
|
+
readonly release: () => void;
|
|
71
|
+
};
|
|
72
|
+
export type TByteSilenceGuardOptions = {
|
|
73
|
+
/** Budget waiting for the first body byte (default 300s). */
|
|
74
|
+
readonly firstByteTimeoutMs: number;
|
|
75
|
+
/** Budget for gaps after any byte until `release()` (default 120s). */
|
|
76
|
+
readonly gapTimeoutMs: number;
|
|
77
|
+
};
|
|
78
|
+
/**
|
|
79
|
+
* Wrap a raw upstream byte stream with the two-budget silence policy.
|
|
80
|
+
* Errors the stream with {@link UpstreamStalledError} on a trip before
|
|
81
|
+
* `release()`.
|
|
82
|
+
*/
|
|
83
|
+
export declare const withByteSilenceGuard: (source: ReadableStream<Uint8Array>, options: TByteSilenceGuardOptions) => TByteSilenceGuard;
|
|
84
|
+
/** Envelope used when a streaming headers / body wait stalls. */
|
|
85
|
+
export declare const upstreamStalledEnvelope: (message?: string) => {
|
|
86
|
+
readonly error: {
|
|
87
|
+
readonly message: string;
|
|
88
|
+
readonly type: "upstream_stalled";
|
|
89
|
+
readonly code: "upstream_stalled";
|
|
90
|
+
};
|
|
91
|
+
};
|
|
92
|
+
/** Shared HTTP status for a pre-commit upstream stall (cloud + daemon). */
|
|
93
|
+
export declare const UPSTREAM_STALLED_HTTP_STATUS = 504;
|
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
import { UpstreamStalledError } from "./upstream-error.js";
|
|
2
|
+
/**
|
|
3
|
+
* Pre-commit silence budgets for STREAMING hops only.
|
|
4
|
+
*
|
|
5
|
+
* Two clocks, both measured on raw upstream bytes (any enqueued chunk —
|
|
6
|
+
* including empty ones, keepalive comments, and pings — counts as alive):
|
|
7
|
+
*
|
|
8
|
+
* 1. **First-byte** (`OPENLLM_STREAM_FIRST_BYTE_TIMEOUT_MS`, default 300s)
|
|
9
|
+
* — waiting for response headers, and the gap before the first body
|
|
10
|
+
* byte. Near-cap prefills (hundreds of thousands of tokens) can
|
|
11
|
+
* legitimately take minutes with no keepalive; this budget is the
|
|
12
|
+
* one that must accommodate them. See `peek.ts`.
|
|
13
|
+
* 2. **Gap** (`OPENLLM_STREAM_SILENCE_TIMEOUT_MS`, default 120s) — after
|
|
14
|
+
* any byte has arrived, until `release()` (hop commit). A provider
|
|
15
|
+
* that went silent mid-prefill after talking once is treated as
|
|
16
|
+
* stalled.
|
|
17
|
+
*
|
|
18
|
+
* Non-streaming requests get no short timeout — they legitimately send
|
|
19
|
+
* nothing until the full JSON body is ready. A non-positive override
|
|
20
|
+
* disables that budget.
|
|
21
|
+
*/
|
|
22
|
+
export const DEFAULT_STREAM_FIRST_BYTE_TIMEOUT_MS = 300_000;
|
|
23
|
+
export const DEFAULT_STREAM_SILENCE_TIMEOUT_MS = 120_000;
|
|
24
|
+
const readTimeoutMs = (envName, fallback) => {
|
|
25
|
+
const raw = process.env[envName];
|
|
26
|
+
if (raw === undefined || raw === "")
|
|
27
|
+
return fallback;
|
|
28
|
+
const override = Number(raw);
|
|
29
|
+
if (!Number.isFinite(override))
|
|
30
|
+
return fallback;
|
|
31
|
+
return override;
|
|
32
|
+
};
|
|
33
|
+
export const streamFirstByteTimeoutMs = () => readTimeoutMs("OPENLLM_STREAM_FIRST_BYTE_TIMEOUT_MS", DEFAULT_STREAM_FIRST_BYTE_TIMEOUT_MS);
|
|
34
|
+
export const streamSilenceTimeoutMs = () => readTimeoutMs("OPENLLM_STREAM_SILENCE_TIMEOUT_MS", DEFAULT_STREAM_SILENCE_TIMEOUT_MS);
|
|
35
|
+
/**
|
|
36
|
+
* Whether a hop should apply the streaming silence budgets on its
|
|
37
|
+
* upstream fetch. ChatGPT's Codex endpoint always streams even when the
|
|
38
|
+
* client asked for JSON, so the gate is `alwaysStreams || clientWantsStream`
|
|
39
|
+
* — not the client flag alone.
|
|
40
|
+
*/
|
|
41
|
+
export const streamingHopNeedsSilence = (opts) => opts.upstreamAlwaysStreams || opts.clientWantsStream;
|
|
42
|
+
/**
|
|
43
|
+
* Absolute deadline for the combined headers + first-body-byte budget.
|
|
44
|
+
* Pass the remaining time into the body silence guard so the two waits
|
|
45
|
+
* share one first-byte window instead of stacking to ~2×.
|
|
46
|
+
*/
|
|
47
|
+
const STALL_ABORT_REASON = "openllm-upstream-stalled";
|
|
48
|
+
export const firstByteDeadlineAt = (firstByteTimeoutMs) => firstByteTimeoutMs > 0 ? Date.now() + firstByteTimeoutMs : null;
|
|
49
|
+
/**
|
|
50
|
+
* Time left before `deadlineAt`. `null` (no budget) returns 0, which every
|
|
51
|
+
* guard reads as disabled; an EXPIRED deadline returns 1 so the guard fires
|
|
52
|
+
* at once instead of being read as disabled.
|
|
53
|
+
*/
|
|
54
|
+
export const remainingTimeoutMs = (deadlineAt) => deadlineAt === null ? 0 : Math.max(1, deadlineAt - Date.now());
|
|
55
|
+
/**
|
|
56
|
+
* Read a (non-2xx) response body with `read`, failing with
|
|
57
|
+
* {@link UpstreamStalledError} if it takes longer than `timeoutMs`. The body
|
|
58
|
+
* is read through a reader this helper owns, so a stall can actually cancel
|
|
59
|
+
* it (a `text()` in flight locks the body, and `body.cancel()` would throw).
|
|
60
|
+
* `timeoutMs <= 0` disables the budget.
|
|
61
|
+
*/
|
|
62
|
+
export const readResponseWithin = async (response, timeoutMs, read) => {
|
|
63
|
+
if (!(timeoutMs > 0) || response.body === null)
|
|
64
|
+
return read(response);
|
|
65
|
+
const reader = response.body.getReader();
|
|
66
|
+
const owned = new Response(new ReadableStream({
|
|
67
|
+
pull: async (controller) => {
|
|
68
|
+
const next = await reader.read();
|
|
69
|
+
if (next.done)
|
|
70
|
+
controller.close();
|
|
71
|
+
else
|
|
72
|
+
controller.enqueue(next.value);
|
|
73
|
+
},
|
|
74
|
+
cancel: async (reason) => {
|
|
75
|
+
await reader.cancel(reason);
|
|
76
|
+
},
|
|
77
|
+
}), {
|
|
78
|
+
status: response.status,
|
|
79
|
+
statusText: response.statusText,
|
|
80
|
+
headers: response.headers,
|
|
81
|
+
});
|
|
82
|
+
let timer;
|
|
83
|
+
const stalled = new Promise((_, reject) => {
|
|
84
|
+
timer = setTimeout(() => {
|
|
85
|
+
reader.cancel(STALL_ABORT_REASON).catch(() => { });
|
|
86
|
+
reject(new UpstreamStalledError("upstream stalled reading error body"));
|
|
87
|
+
}, timeoutMs);
|
|
88
|
+
});
|
|
89
|
+
try {
|
|
90
|
+
return await Promise.race([read(owned), stalled]);
|
|
91
|
+
}
|
|
92
|
+
finally {
|
|
93
|
+
clearTimeout(timer);
|
|
94
|
+
}
|
|
95
|
+
};
|
|
96
|
+
/**
|
|
97
|
+
* AbortSignal that fires if response headers take longer than
|
|
98
|
+
* `firstByteTimeoutMs`. Cleared as soon as headers arrive so it cannot
|
|
99
|
+
* abort the body later. Caller abort still wins.
|
|
100
|
+
*/
|
|
101
|
+
export const withStreamingHeadersTimeout = (caller, firstByteTimeoutMs) => {
|
|
102
|
+
if (!(firstByteTimeoutMs > 0)) {
|
|
103
|
+
return {
|
|
104
|
+
signal: caller,
|
|
105
|
+
didStall: () => false,
|
|
106
|
+
clear: () => { },
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
const stall = new AbortController();
|
|
110
|
+
let stalled = false;
|
|
111
|
+
const timer = setTimeout(() => {
|
|
112
|
+
stalled = true;
|
|
113
|
+
stall.abort(STALL_ABORT_REASON);
|
|
114
|
+
}, firstByteTimeoutMs);
|
|
115
|
+
const clear = () => {
|
|
116
|
+
clearTimeout(timer);
|
|
117
|
+
};
|
|
118
|
+
if (caller.aborted) {
|
|
119
|
+
clear();
|
|
120
|
+
stall.abort(caller.reason);
|
|
121
|
+
}
|
|
122
|
+
else {
|
|
123
|
+
caller.addEventListener("abort", () => {
|
|
124
|
+
clear();
|
|
125
|
+
stall.abort(caller.reason);
|
|
126
|
+
}, { once: true });
|
|
127
|
+
}
|
|
128
|
+
return {
|
|
129
|
+
signal: stall.signal,
|
|
130
|
+
didStall: () => stalled && !caller.aborted,
|
|
131
|
+
clear,
|
|
132
|
+
};
|
|
133
|
+
};
|
|
134
|
+
/**
|
|
135
|
+
* Wrap a raw upstream byte stream with the two-budget silence policy.
|
|
136
|
+
* Errors the stream with {@link UpstreamStalledError} on a trip before
|
|
137
|
+
* `release()`.
|
|
138
|
+
*/
|
|
139
|
+
export const withByteSilenceGuard = (source, options) => {
|
|
140
|
+
const firstMs = options.firstByteTimeoutMs;
|
|
141
|
+
const gapMs = options.gapTimeoutMs;
|
|
142
|
+
if (!(firstMs > 0) && !(gapMs > 0)) {
|
|
143
|
+
return { stream: source, release: () => { } };
|
|
144
|
+
}
|
|
145
|
+
const reader = source.getReader();
|
|
146
|
+
let released = false;
|
|
147
|
+
let finished = false;
|
|
148
|
+
let sawByte = false;
|
|
149
|
+
let timer = null;
|
|
150
|
+
const STALL = Symbol("upstream-silence");
|
|
151
|
+
const RELEASED = Symbol("upstream-silence-released");
|
|
152
|
+
let settleRace = null;
|
|
153
|
+
let racePromise = null;
|
|
154
|
+
const clearTimer = () => {
|
|
155
|
+
if (timer !== null) {
|
|
156
|
+
clearTimeout(timer);
|
|
157
|
+
timer = null;
|
|
158
|
+
}
|
|
159
|
+
};
|
|
160
|
+
const currentBudgetMs = () => (sawByte ? gapMs : firstMs);
|
|
161
|
+
const armTimer = () => {
|
|
162
|
+
clearTimer();
|
|
163
|
+
if (released || finished)
|
|
164
|
+
return;
|
|
165
|
+
const budget = currentBudgetMs();
|
|
166
|
+
if (!(budget > 0)) {
|
|
167
|
+
racePromise = null;
|
|
168
|
+
settleRace = null;
|
|
169
|
+
return;
|
|
170
|
+
}
|
|
171
|
+
racePromise = new Promise((resolve) => {
|
|
172
|
+
settleRace = resolve;
|
|
173
|
+
timer = setTimeout(() => {
|
|
174
|
+
timer = null;
|
|
175
|
+
resolve(STALL);
|
|
176
|
+
}, budget);
|
|
177
|
+
});
|
|
178
|
+
};
|
|
179
|
+
const release = () => {
|
|
180
|
+
released = true;
|
|
181
|
+
clearTimer();
|
|
182
|
+
const settle = settleRace;
|
|
183
|
+
settleRace = null;
|
|
184
|
+
racePromise = null;
|
|
185
|
+
// Resolve any in-flight race so a late STALL is not observed as a stall.
|
|
186
|
+
if (settle !== null)
|
|
187
|
+
settle(RELEASED);
|
|
188
|
+
};
|
|
189
|
+
armTimer();
|
|
190
|
+
const stream = new ReadableStream({
|
|
191
|
+
async pull(controller) {
|
|
192
|
+
if (finished) {
|
|
193
|
+
controller.close();
|
|
194
|
+
return;
|
|
195
|
+
}
|
|
196
|
+
if (released) {
|
|
197
|
+
const read = await reader.read();
|
|
198
|
+
if (read.done) {
|
|
199
|
+
finished = true;
|
|
200
|
+
clearTimer();
|
|
201
|
+
controller.close();
|
|
202
|
+
return;
|
|
203
|
+
}
|
|
204
|
+
controller.enqueue(read.value);
|
|
205
|
+
return;
|
|
206
|
+
}
|
|
207
|
+
const read = reader.read();
|
|
208
|
+
read.catch(() => { });
|
|
209
|
+
if (racePromise === null)
|
|
210
|
+
armTimer();
|
|
211
|
+
const race = racePromise;
|
|
212
|
+
let result;
|
|
213
|
+
try {
|
|
214
|
+
result = race === null ? await read : await Promise.race([read, race]);
|
|
215
|
+
}
|
|
216
|
+
catch (err) {
|
|
217
|
+
finished = true;
|
|
218
|
+
clearTimer();
|
|
219
|
+
throw err;
|
|
220
|
+
}
|
|
221
|
+
if (result === RELEASED || (result === STALL && released)) {
|
|
222
|
+
// Commit raced the timer: wait for the outstanding read without stalling.
|
|
223
|
+
result = await read;
|
|
224
|
+
}
|
|
225
|
+
if (result === STALL) {
|
|
226
|
+
finished = true;
|
|
227
|
+
clearTimer();
|
|
228
|
+
reader.cancel(STALL_ABORT_REASON).catch(() => { });
|
|
229
|
+
controller.error(new UpstreamStalledError("upstream stalled before producing output"));
|
|
230
|
+
return;
|
|
231
|
+
}
|
|
232
|
+
if (result.done) {
|
|
233
|
+
finished = true;
|
|
234
|
+
clearTimer();
|
|
235
|
+
controller.close();
|
|
236
|
+
return;
|
|
237
|
+
}
|
|
238
|
+
// Any enqueued chunk — including empty — counts as alive.
|
|
239
|
+
sawByte = true;
|
|
240
|
+
if (!released) {
|
|
241
|
+
armTimer();
|
|
242
|
+
}
|
|
243
|
+
controller.enqueue(result.value);
|
|
244
|
+
},
|
|
245
|
+
cancel(reason) {
|
|
246
|
+
finished = true;
|
|
247
|
+
clearTimer();
|
|
248
|
+
reader.cancel(reason).catch(() => { });
|
|
249
|
+
},
|
|
250
|
+
});
|
|
251
|
+
return { stream, release };
|
|
252
|
+
};
|
|
253
|
+
/** Envelope used when a streaming headers / body wait stalls. */
|
|
254
|
+
export const upstreamStalledEnvelope = (message = "upstream stalled before producing output") => ({
|
|
255
|
+
error: {
|
|
256
|
+
message,
|
|
257
|
+
type: "upstream_stalled",
|
|
258
|
+
code: "upstream_stalled",
|
|
259
|
+
},
|
|
260
|
+
});
|
|
261
|
+
/** Shared HTTP status for a pre-commit upstream stall (cloud + daemon). */
|
|
262
|
+
export const UPSTREAM_STALLED_HTTP_STATUS = 504;
|
|
@@ -78,15 +78,20 @@ export declare const withFrameAlignedHeartbeat: (source: ReadableStream<Uint8Arr
|
|
|
78
78
|
*
|
|
79
79
|
* Wrap BEFORE `.tee()` so the usage-drain branch also terminates.
|
|
80
80
|
*/
|
|
81
|
+
export type TStreamDeadlineTruncateInfo = {
|
|
82
|
+
/** True when at least one content / tool / reasoning delta was forwarded. */
|
|
83
|
+
readonly sawOutput: boolean;
|
|
84
|
+
};
|
|
81
85
|
export declare const withStreamDeadline: (source: ReadableStream<TChatCompletionChunk>, options: {
|
|
82
86
|
readonly deadlineMs: number;
|
|
83
87
|
/**
|
|
84
88
|
* Fires exactly once if the deadline expires before the source
|
|
85
89
|
* closes naturally. Callers use this to skip caching a body that
|
|
86
90
|
* was sealed by the synthetic terminator instead of the upstream's
|
|
87
|
-
* own `finish_reason
|
|
91
|
+
* own `finish_reason`, and to log zero-output truncations as errors.
|
|
92
|
+
* Omit to ignore truncation.
|
|
88
93
|
*/
|
|
89
|
-
readonly onTruncate?: () => void;
|
|
94
|
+
readonly onTruncate?: (info: TStreamDeadlineTruncateInfo) => void;
|
|
90
95
|
}) => ReadableStream<TChatCompletionChunk>;
|
|
91
96
|
/**
|
|
92
97
|
* Result of wrapping a raw byte stream with a deadline guard.
|
|
@@ -99,6 +104,8 @@ export declare const withStreamDeadline: (source: ReadableStream<TChatCompletion
|
|
|
99
104
|
export type TByteStreamDeadlineHandle = {
|
|
100
105
|
readonly stream: ReadableStream<Uint8Array>;
|
|
101
106
|
readonly isTruncated: () => boolean;
|
|
107
|
+
/** True when the deadline fired with no non-keepalive output forwarded. */
|
|
108
|
+
readonly isZeroOutputTruncate: () => boolean;
|
|
102
109
|
};
|
|
103
110
|
/**
|
|
104
111
|
* Sibling of `withStreamDeadline` for the Anthropic pass-through:
|
|
@@ -106,7 +113,7 @@ export type TByteStreamDeadlineHandle = {
|
|
|
106
113
|
* the synthetic terminator must be a pre-encoded SSE byte sequence
|
|
107
114
|
* because there is no canonical-decode step in the pipeline.
|
|
108
115
|
*
|
|
109
|
-
* On cutoff:
|
|
116
|
+
* On cutoff with partial output:
|
|
110
117
|
* 1. cancel the source — propagating to the upstream `fetch` body
|
|
111
118
|
* so we stop paying the provider for tokens we'll never deliver;
|
|
112
119
|
* 2. enqueue ONE pre-encoded `message_delta` + `message_stop` pair
|
|
@@ -115,6 +122,10 @@ export type TByteStreamDeadlineHandle = {
|
|
|
115
122
|
* 3. close cleanly so SSE clients see a terminated turn instead of
|
|
116
123
|
* a severed connection.
|
|
117
124
|
*
|
|
125
|
+
* On cutoff with ZERO output: emit an Anthropic `event: error` instead
|
|
126
|
+
* of a clean `max_tokens` turn, so the client and the usage log treat
|
|
127
|
+
* it as a failure.
|
|
128
|
+
*
|
|
118
129
|
* Wrap BEFORE `.tee()` so the usage-drain branch also receives the
|
|
119
130
|
* terminator and the accumulator can finalise. The caller checks
|
|
120
131
|
* `isTruncated()` after consumption to skip caching a truncated body.
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { UpstreamStreamError } from "./upstream-error.js";
|
|
1
2
|
const SSE_DONE = "[DONE]";
|
|
2
3
|
const textEncoder = new TextEncoder();
|
|
3
4
|
export const encodeSseEvent = (data) => {
|
|
@@ -391,33 +392,19 @@ export const withFrameAlignedHeartbeat = (source, options) => {
|
|
|
391
392
|
},
|
|
392
393
|
});
|
|
393
394
|
};
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
* 1. cancels the source — propagating to the upstream `fetch` body
|
|
408
|
-
* so we stop paying the provider for tokens we'll never deliver;
|
|
409
|
-
* 2. emits ONE synthetic terminal chunk with
|
|
410
|
-
* `finish_reason: "length"` and closes cleanly.
|
|
411
|
-
*
|
|
412
|
-
* The text deltas already streamed ARE the partial answer (they're in
|
|
413
|
-
* the client's hands). The synthetic terminal makes the SSE encoders
|
|
414
|
-
* close properly — `data: [DONE]` for OpenAI, `message_delta`
|
|
415
|
-
* (`stop_reason: "max_tokens"`) + `message_stop` for Anthropic — so
|
|
416
|
-
* the caller gets a clean, truncated turn it can re-prompt to resume,
|
|
417
|
-
* instead of a thrown network error.
|
|
418
|
-
*
|
|
419
|
-
* Wrap BEFORE `.tee()` so the usage-drain branch also terminates.
|
|
420
|
-
*/
|
|
395
|
+
const chunkHasOutputDelta = (chunk) => {
|
|
396
|
+
const nonEmpty = (s) => typeof s === "string" && s.length > 0;
|
|
397
|
+
const nonEmptyArr = (a) => Array.isArray(a) && a.length > 0;
|
|
398
|
+
return chunk.choices.some((choice) => {
|
|
399
|
+
const d = choice.delta;
|
|
400
|
+
return (nonEmpty(d.content) ||
|
|
401
|
+
nonEmpty(d.reasoning_content) ||
|
|
402
|
+
nonEmpty(d.refusal) ||
|
|
403
|
+
nonEmptyArr(d.tool_calls) ||
|
|
404
|
+
nonEmptyArr(d.reasoning_items) ||
|
|
405
|
+
nonEmptyArr(d.server_search_calls));
|
|
406
|
+
});
|
|
407
|
+
};
|
|
421
408
|
export const withStreamDeadline = (source, options) => {
|
|
422
409
|
const reader = source.getReader();
|
|
423
410
|
let lastId = "chatcmpl-openllm-deadline";
|
|
@@ -425,6 +412,7 @@ export const withStreamDeadline = (source, options) => {
|
|
|
425
412
|
let lastCreated = 0;
|
|
426
413
|
let hasCreated = false;
|
|
427
414
|
let finished = false;
|
|
415
|
+
let sawOutput = false;
|
|
428
416
|
const DEADLINE = Symbol("deadline");
|
|
429
417
|
let timer = null;
|
|
430
418
|
const deadline = new Promise((resolve) => {
|
|
@@ -476,12 +464,19 @@ export const withStreamDeadline = (source, options) => {
|
|
|
476
464
|
reader.cancel("openllm-stream-deadline").catch(() => { });
|
|
477
465
|
if (options.onTruncate !== undefined) {
|
|
478
466
|
try {
|
|
479
|
-
options.onTruncate();
|
|
467
|
+
options.onTruncate({ sawOutput });
|
|
480
468
|
}
|
|
481
469
|
catch {
|
|
482
470
|
// Callback errors must not crash the stream.
|
|
483
471
|
}
|
|
484
472
|
}
|
|
473
|
+
if (!sawOutput) {
|
|
474
|
+
// Zero-output deadline: a clean `length` turn would look like a
|
|
475
|
+
// successful empty answer and skip the fallback chain's chance to
|
|
476
|
+
// have walked (already committed). Surface a real error frame.
|
|
477
|
+
controller.error(new UpstreamStreamError("upstream_stalled", "stream deadline reached with no output"));
|
|
478
|
+
return;
|
|
479
|
+
}
|
|
485
480
|
controller.enqueue(terminal());
|
|
486
481
|
controller.close();
|
|
487
482
|
return;
|
|
@@ -493,6 +488,9 @@ export const withStreamDeadline = (source, options) => {
|
|
|
493
488
|
return;
|
|
494
489
|
}
|
|
495
490
|
const value = r.value;
|
|
491
|
+
if (chunkHasOutputDelta(value)) {
|
|
492
|
+
sawOutput = true;
|
|
493
|
+
}
|
|
496
494
|
if (typeof value.id === "string" && value.id.length > 0) {
|
|
497
495
|
lastId = value.id;
|
|
498
496
|
}
|
|
@@ -512,13 +510,22 @@ export const withStreamDeadline = (source, options) => {
|
|
|
512
510
|
},
|
|
513
511
|
});
|
|
514
512
|
};
|
|
513
|
+
const byteChunkLooksLikeOutput = (bytes) => {
|
|
514
|
+
// Only content / tool / reasoning deltas count as partial output.
|
|
515
|
+
// Housekeeping (`message_start`, role-only frames) and keepalives must
|
|
516
|
+
// NOT flip a zero-output deadline into a clean `max_tokens` turn.
|
|
517
|
+
const text = new TextDecoder().decode(bytes);
|
|
518
|
+
if (text.length === 0)
|
|
519
|
+
return false;
|
|
520
|
+
return (/text_delta|input_json_delta|thinking_delta|signature_delta|tool_use|reasoning_content|"content":"[^"]/i.test(text));
|
|
521
|
+
};
|
|
515
522
|
/**
|
|
516
523
|
* Sibling of `withStreamDeadline` for the Anthropic pass-through:
|
|
517
524
|
* the source is raw provider SSE bytes (not canonical chunks), and
|
|
518
525
|
* the synthetic terminator must be a pre-encoded SSE byte sequence
|
|
519
526
|
* because there is no canonical-decode step in the pipeline.
|
|
520
527
|
*
|
|
521
|
-
* On cutoff:
|
|
528
|
+
* On cutoff with partial output:
|
|
522
529
|
* 1. cancel the source — propagating to the upstream `fetch` body
|
|
523
530
|
* so we stop paying the provider for tokens we'll never deliver;
|
|
524
531
|
* 2. enqueue ONE pre-encoded `message_delta` + `message_stop` pair
|
|
@@ -527,6 +534,10 @@ export const withStreamDeadline = (source, options) => {
|
|
|
527
534
|
* 3. close cleanly so SSE clients see a terminated turn instead of
|
|
528
535
|
* a severed connection.
|
|
529
536
|
*
|
|
537
|
+
* On cutoff with ZERO output: emit an Anthropic `event: error` instead
|
|
538
|
+
* of a clean `max_tokens` turn, so the client and the usage log treat
|
|
539
|
+
* it as a failure.
|
|
540
|
+
*
|
|
530
541
|
* Wrap BEFORE `.tee()` so the usage-drain branch also receives the
|
|
531
542
|
* terminator and the accumulator can finalise. The caller checks
|
|
532
543
|
* `isTruncated()` after consumption to skip caching a truncated body.
|
|
@@ -541,6 +552,8 @@ export const withByteStreamDeadline = (source, options) => {
|
|
|
541
552
|
const reader = source.getReader();
|
|
542
553
|
let finished = false;
|
|
543
554
|
let truncated = false;
|
|
555
|
+
let sawOutput = false;
|
|
556
|
+
let zeroOutputTruncate = false;
|
|
544
557
|
const DEADLINE = Symbol("byte-stream-deadline");
|
|
545
558
|
let timer = null;
|
|
546
559
|
const deadline = new Promise((resolve) => {
|
|
@@ -552,8 +565,11 @@ export const withByteStreamDeadline = (source, options) => {
|
|
|
552
565
|
timer = null;
|
|
553
566
|
}
|
|
554
567
|
};
|
|
555
|
-
|
|
568
|
+
// Leading `\n\n` resynchronises if the deadline fired mid-frame so the
|
|
569
|
+
// synthetic event always starts a fresh SSE record.
|
|
570
|
+
const lengthTerminator = () => textEncoder.encode('\n\nevent: message_delta\ndata: {"type":"message_delta","delta":{"stop_reason":"max_tokens","stop_sequence":null},"usage":{"output_tokens":0}}\n\n' +
|
|
556
571
|
'event: message_stop\ndata: {"type":"message_stop"}\n\n');
|
|
572
|
+
const zeroOutputError = () => textEncoder.encode('\n\nevent: error\ndata: {"type":"error","error":{"type":"upstream_stalled","message":"stream deadline reached with no output"}}\n\n');
|
|
557
573
|
const stream = new ReadableStream({
|
|
558
574
|
async pull(controller) {
|
|
559
575
|
if (finished) {
|
|
@@ -578,7 +594,13 @@ export const withByteStreamDeadline = (source, options) => {
|
|
|
578
594
|
truncated = true;
|
|
579
595
|
clear();
|
|
580
596
|
reader.cancel("openllm-byte-stream-deadline").catch(() => { });
|
|
581
|
-
|
|
597
|
+
if (!sawOutput) {
|
|
598
|
+
zeroOutputTruncate = true;
|
|
599
|
+
controller.enqueue(zeroOutputError());
|
|
600
|
+
}
|
|
601
|
+
else {
|
|
602
|
+
controller.enqueue(lengthTerminator());
|
|
603
|
+
}
|
|
582
604
|
controller.close();
|
|
583
605
|
return;
|
|
584
606
|
}
|
|
@@ -588,6 +610,9 @@ export const withByteStreamDeadline = (source, options) => {
|
|
|
588
610
|
controller.close();
|
|
589
611
|
return;
|
|
590
612
|
}
|
|
613
|
+
if (byteChunkLooksLikeOutput(r.value)) {
|
|
614
|
+
sawOutput = true;
|
|
615
|
+
}
|
|
591
616
|
controller.enqueue(r.value);
|
|
592
617
|
},
|
|
593
618
|
cancel(reason) {
|
|
@@ -596,5 +621,9 @@ export const withByteStreamDeadline = (source, options) => {
|
|
|
596
621
|
reader.cancel(reason).catch(() => { });
|
|
597
622
|
},
|
|
598
623
|
});
|
|
599
|
-
return {
|
|
624
|
+
return {
|
|
625
|
+
stream,
|
|
626
|
+
isTruncated: () => truncated,
|
|
627
|
+
isZeroOutputTruncate: () => zeroOutputTruncate,
|
|
628
|
+
};
|
|
600
629
|
};
|
|
@@ -11,6 +11,15 @@ export declare class UpstreamStreamError extends Error {
|
|
|
11
11
|
readonly upstreamType: string;
|
|
12
12
|
constructor(upstreamType: string, message: string);
|
|
13
13
|
}
|
|
14
|
+
/**
|
|
15
|
+
* A streaming hop went silent before any byte was forwarded to the client
|
|
16
|
+
* (headers wait or raw upstream inactivity). Distinct from a mid-stream
|
|
17
|
+
* decode error so the classifier can tag `upstream_stalled` and the
|
|
18
|
+
* fallback chain can walk while the hop is still uncommitted.
|
|
19
|
+
*/
|
|
20
|
+
export declare class UpstreamStalledError extends UpstreamStreamError {
|
|
21
|
+
constructor(message?: string);
|
|
22
|
+
}
|
|
14
23
|
/**
|
|
15
24
|
* A stream ended before a terminal chunk after its decoder had already observed
|
|
16
25
|
* usage. The accumulator wraps every failure type, including transport errors,
|
|
@@ -30,3 +39,5 @@ export declare const upstreamErrorFrom: (err: unknown) => {
|
|
|
30
39
|
type: string;
|
|
31
40
|
message: string;
|
|
32
41
|
};
|
|
42
|
+
/** True when `err` is (or wraps) an {@link UpstreamStalledError}. */
|
|
43
|
+
export declare const isUpstreamStalledError: (err: unknown) => boolean;
|
|
@@ -14,6 +14,18 @@ export class UpstreamStreamError extends Error {
|
|
|
14
14
|
this.upstreamType = upstreamType;
|
|
15
15
|
}
|
|
16
16
|
}
|
|
17
|
+
/**
|
|
18
|
+
* A streaming hop went silent before any byte was forwarded to the client
|
|
19
|
+
* (headers wait or raw upstream inactivity). Distinct from a mid-stream
|
|
20
|
+
* decode error so the classifier can tag `upstream_stalled` and the
|
|
21
|
+
* fallback chain can walk while the hop is still uncommitted.
|
|
22
|
+
*/
|
|
23
|
+
export class UpstreamStalledError extends UpstreamStreamError {
|
|
24
|
+
constructor(message = "upstream stalled before producing output") {
|
|
25
|
+
super("upstream_stalled", message);
|
|
26
|
+
this.name = "UpstreamStalledError";
|
|
27
|
+
}
|
|
28
|
+
}
|
|
17
29
|
/**
|
|
18
30
|
* A stream ended before a terminal chunk after its decoder had already observed
|
|
19
31
|
* usage. The accumulator wraps every failure type, including transport errors,
|
|
@@ -43,3 +55,12 @@ export const upstreamErrorFrom = (err) => {
|
|
|
43
55
|
}
|
|
44
56
|
return { type: "stream_error", message: String(err) };
|
|
45
57
|
};
|
|
58
|
+
/** True when `err` is (or wraps) an {@link UpstreamStalledError}. */
|
|
59
|
+
export const isUpstreamStalledError = (err) => {
|
|
60
|
+
if (err instanceof UpstreamStalledError)
|
|
61
|
+
return true;
|
|
62
|
+
if (err instanceof Error && err.cause !== undefined) {
|
|
63
|
+
return isUpstreamStalledError(err.cause);
|
|
64
|
+
}
|
|
65
|
+
return false;
|
|
66
|
+
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@openllmsh/sdk",
|
|
3
|
-
"version": "2.10.15-alpha.
|
|
3
|
+
"version": "2.10.15-alpha.2",
|
|
4
4
|
"private": false,
|
|
5
5
|
"description": "The OpenLLM SDK: typed API contracts, wire-format adapters, the device tunnel, client-side vault crypto, the OpenAPI spec and MCP tools.",
|
|
6
6
|
"keywords": [
|