@openllmsh/sdk 2.10.15-alpha.2 → 2.10.15-alpha.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,5 +1,5 @@
1
1
  import { Schema as S } from "effect";
2
- export declare const CooldownReason: S.Literals<readonly ["network", "timeout", "upstream_stalled", "rate_limit", "quota_exhausted", "server_error", "auth", "payment", "not_found", "payload_too_large", "unprocessable", "request_invalid", "accepted_unreadable", "context_overflow", "content_filter", "upstream_rejection"]>;
2
+ export declare const CooldownReason: S.Literals<readonly ["network", "timeout", "rate_limit", "quota_exhausted", "server_error", "auth", "payment", "not_found", "payload_too_large", "unprocessable", "request_invalid", "accepted_unreadable", "context_overflow", "content_filter", "upstream_rejection"]>;
3
3
  export type TCooldownReason = S.Schema.Type<typeof CooldownReason>;
4
4
  export type THopAction = "retry_in_place" | "cool_and_advance" | "walk_no_cool" | "surface";
5
5
  /**
@@ -2,7 +2,6 @@ import { Schema as S } from "effect";
2
2
  export const CooldownReason = S.Literals([
3
3
  "network",
4
4
  "timeout",
5
- "upstream_stalled",
6
5
  "rate_limit",
7
6
  "quota_exhausted",
8
7
  "server_error",
@@ -20,9 +19,6 @@ export const CooldownReason = S.Literals([
20
19
  const cooldownPolicies = {
21
20
  network: { action: "retry_in_place" },
22
21
  timeout: { action: "retry_in_place" },
23
- // Streaming hop went silent before any client byte (headers or body). Cool
24
- // briefly so a dead provider isn't retried immediately for the same turn.
25
- upstream_stalled: { action: "cool_and_advance", ttlMs: 60_000 },
26
22
  rate_limit: { action: "cool_and_advance", ttlMs: 180_000 },
27
23
  quota_exhausted: { action: "cool_and_advance", ttlMs: 180_000 },
28
24
  server_error: { action: "retry_in_place" },
@@ -509,7 +509,7 @@ export declare const DaemonRecordRequest: S.Struct<{
509
509
  * cloud charges each one the hold's full reservation.
510
510
  */
511
511
  readonly hold_unknown_attempts: S.optional<S.Number>;
512
- readonly cooldown_reason: S.optional<S.Literals<readonly ["network", "timeout", "upstream_stalled", "rate_limit", "quota_exhausted", "server_error", "auth", "payment", "not_found", "payload_too_large", "unprocessable", "request_invalid", "accepted_unreadable", "context_overflow", "content_filter", "upstream_rejection"]>>;
512
+ readonly cooldown_reason: S.optional<S.Literals<readonly ["network", "timeout", "rate_limit", "quota_exhausted", "server_error", "auth", "payment", "not_found", "payload_too_large", "unprocessable", "request_invalid", "accepted_unreadable", "context_overflow", "content_filter", "upstream_rejection"]>>;
513
513
  /** Reset instant for reset-aware TTL cap; optional so old daemons omit. */
514
514
  readonly reset_at_ms: S.optional<S.Number>;
515
515
  }>;
@@ -21,7 +21,6 @@ const MEDIA_NO_RETRY_REASONS = new Set([
21
21
  "server_error",
22
22
  "timeout",
23
23
  "network",
24
- "upstream_stalled",
25
24
  ]);
26
25
  /**
27
26
  * Media overlay on the shared hop classifier.
@@ -36,7 +36,6 @@ export * from "./lib/refusal.js";
36
36
  export * from "./lib/secret-scrub.js";
37
37
  export * from "./lib/streaming/accumulate.js";
38
38
  export * from "./lib/streaming/peek.js";
39
- export * from "./lib/streaming/silence.js";
40
39
  export * from "./lib/streaming/sse.js";
41
40
  export * from "./lib/streaming/strip-tool-calls.js";
42
41
  export * from "./lib/streaming/upstream-error.js";
@@ -36,7 +36,6 @@ export * from "./lib/refusal.js";
36
36
  export * from "./lib/secret-scrub.js";
37
37
  export * from "./lib/streaming/accumulate.js";
38
38
  export * from "./lib/streaming/peek.js";
39
- export * from "./lib/streaming/silence.js";
40
39
  export * from "./lib/streaming/sse.js";
41
40
  export * from "./lib/streaming/strip-tool-calls.js";
42
41
  export * from "./lib/streaming/upstream-error.js";
@@ -32,18 +32,11 @@ export const reasonFor = (i, rules = errorRulesFor(i.provider)) => {
32
32
  // A 2xx the runner still couldn't use (an undecodable 200): the provider
33
33
  // ACCEPTED the request and may have billed it, so walking on would send —
34
34
  // and bill — it again. Only an explicit in-band refusal (content_filter,
35
- // above) walks from a 2xx; no body matcher or provider rule — including a
36
- // forged `upstream_stalled` code on an accepted response — may turn an
37
- // accepted request into a walk.
35
+ // above) walks from a 2xx; no body matcher or provider rule may turn an
36
+ // accepted request into a walk. 3xx falls through to the normal rules
37
+ // (typically `upstream_rejection` → walk).
38
38
  if (i.status >= 200 && i.status < 300)
39
39
  return "accepted_unreadable";
40
- // Pre-commit synthetic stall (headers / raw-byte silence). Our runners and
41
- // the daemon emit this only on non-2xx outcomes (typically 504). Kept AFTER
42
- // the 2xx guard so an accepted provider response can never be re-POSTed.
43
- if (i.envelope?.error?.code === "upstream_stalled" ||
44
- i.envelope?.error?.type === "upstream_stalled") {
45
- return "upstream_stalled";
46
- }
47
40
  const message = i.envelope?.error?.message ?? "";
48
41
  if (CONTEXT_OVERFLOW_BODY.test(message))
49
42
  return "context_overflow";
@@ -45,16 +45,16 @@ export declare const isMeaningfulChunk: (chunk: TChatCompletionChunk) => boolean
45
45
  * the module doc for why), then return a stream that replays what was
46
46
  * buffered ahead of the live tail.
47
47
  *
48
- * The wait for a MEANINGFUL decoded event is still unbounded here — the
49
- * request abort signal is the only hard stop at this layer. Raw-byte
50
- * silence is enforced BELOW the decoder by `withByteSilenceGuard` /
51
- * `withStreamingHeadersTimeout` (see `silence.ts`): a generous
52
- * first-byte budget (default 300s, `OPENLLM_STREAM_FIRST_BYTE_TIMEOUT_MS`)
53
- * covers near-cap prefills that produce no keepalive, and a shorter gap
54
- * budget (default 120s, `OPENLLM_STREAM_SILENCE_TIMEOUT_MS`) trips only
55
- * after the upstream has already spoken. That split keeps the overflow
56
- * peek working for long prefills without letting a truly silent hop
57
- * burn the Vercel deadline as a fake `length` success.
48
+ * The wait is UNBOUNDED by design — bounded only by the request's abort
49
+ * signal, which the upstream fetch carries (an aborted client errors the
50
+ * read). The rejection we're waiting for arrives after the vendor
51
+ * tokenizes the prompt, and a near-cap prompt (the exact overflow case)
52
+ * prefills for many seconds; any fixed deadline commits the response
53
+ * right before the rejection lands, stranding the client in a committed
54
+ * stream that then dies (the pre-fix stall). The client loses nothing
55
+ * from waiting: it would receive zero bytes before the first event
56
+ * regardless. (Same rule as ref/CLIProxyAPI's stream bootstrap: no
57
+ * timeouts after the upstream connection is established.)
58
58
  *
59
59
  * A stream that CLOSES before any event is an error by default (walkable
60
60
  * — the upstream produced nothing), not an empty success. EXCEPT when the
@@ -30,16 +30,16 @@ export const isMeaningfulChunk = (chunk) => {
30
30
  * the module doc for why), then return a stream that replays what was
31
31
  * buffered ahead of the live tail.
32
32
  *
33
- * The wait for a MEANINGFUL decoded event is still unbounded here — the
34
- * request abort signal is the only hard stop at this layer. Raw-byte
35
- * silence is enforced BELOW the decoder by `withByteSilenceGuard` /
36
- * `withStreamingHeadersTimeout` (see `silence.ts`): a generous
37
- * first-byte budget (default 300s, `OPENLLM_STREAM_FIRST_BYTE_TIMEOUT_MS`)
38
- * covers near-cap prefills that produce no keepalive, and a shorter gap
39
- * budget (default 120s, `OPENLLM_STREAM_SILENCE_TIMEOUT_MS`) trips only
40
- * after the upstream has already spoken. That split keeps the overflow
41
- * peek working for long prefills without letting a truly silent hop
42
- * burn the Vercel deadline as a fake `length` success.
33
+ * The wait is UNBOUNDED by design — bounded only by the request's abort
34
+ * signal, which the upstream fetch carries (an aborted client errors the
35
+ * read). The rejection we're waiting for arrives after the vendor
36
+ * tokenizes the prompt, and a near-cap prompt (the exact overflow case)
37
+ * prefills for many seconds; any fixed deadline commits the response
38
+ * right before the rejection lands, stranding the client in a committed
39
+ * stream that then dies (the pre-fix stall). The client loses nothing
40
+ * from waiting: it would receive zero bytes before the first event
41
+ * regardless. (Same rule as ref/CLIProxyAPI's stream bootstrap: no
42
+ * timeouts after the upstream connection is established.)
43
43
  *
44
44
  * A stream that CLOSES before any event is an error by default (walkable
45
45
  * — the upstream produced nothing), not an empty success. EXCEPT when the
@@ -78,20 +78,15 @@ export declare const withFrameAlignedHeartbeat: (source: ReadableStream<Uint8Arr
78
78
  *
79
79
  * Wrap BEFORE `.tee()` so the usage-drain branch also terminates.
80
80
  */
81
- export type TStreamDeadlineTruncateInfo = {
82
- /** True when at least one content / tool / reasoning delta was forwarded. */
83
- readonly sawOutput: boolean;
84
- };
85
81
  export declare const withStreamDeadline: (source: ReadableStream<TChatCompletionChunk>, options: {
86
82
  readonly deadlineMs: number;
87
83
  /**
88
84
  * Fires exactly once if the deadline expires before the source
89
85
  * closes naturally. Callers use this to skip caching a body that
90
86
  * was sealed by the synthetic terminator instead of the upstream's
91
- * own `finish_reason`, and to log zero-output truncations as errors.
92
- * Omit to ignore truncation.
87
+ * own `finish_reason`. Omit to ignore truncation.
93
88
  */
94
- readonly onTruncate?: (info: TStreamDeadlineTruncateInfo) => void;
89
+ readonly onTruncate?: () => void;
95
90
  }) => ReadableStream<TChatCompletionChunk>;
96
91
  /**
97
92
  * Result of wrapping a raw byte stream with a deadline guard.
@@ -104,8 +99,6 @@ export declare const withStreamDeadline: (source: ReadableStream<TChatCompletion
104
99
  export type TByteStreamDeadlineHandle = {
105
100
  readonly stream: ReadableStream<Uint8Array>;
106
101
  readonly isTruncated: () => boolean;
107
- /** True when the deadline fired with no non-keepalive output forwarded. */
108
- readonly isZeroOutputTruncate: () => boolean;
109
102
  };
110
103
  /**
111
104
  * Sibling of `withStreamDeadline` for the Anthropic pass-through:
@@ -113,7 +106,7 @@ export type TByteStreamDeadlineHandle = {
113
106
  * the synthetic terminator must be a pre-encoded SSE byte sequence
114
107
  * because there is no canonical-decode step in the pipeline.
115
108
  *
116
- * On cutoff with partial output:
109
+ * On cutoff:
117
110
  * 1. cancel the source — propagating to the upstream `fetch` body
118
111
  * so we stop paying the provider for tokens we'll never deliver;
119
112
  * 2. enqueue ONE pre-encoded `message_delta` + `message_stop` pair
@@ -122,10 +115,6 @@ export type TByteStreamDeadlineHandle = {
122
115
  * 3. close cleanly so SSE clients see a terminated turn instead of
123
116
  * a severed connection.
124
117
  *
125
- * On cutoff with ZERO output: emit an Anthropic `event: error` instead
126
- * of a clean `max_tokens` turn, so the client and the usage log treat
127
- * it as a failure.
128
- *
129
118
  * Wrap BEFORE `.tee()` so the usage-drain branch also receives the
130
119
  * terminator and the accumulator can finalise. The caller checks
131
120
  * `isTruncated()` after consumption to skip caching a truncated body.
@@ -1,4 +1,3 @@
1
- import { UpstreamStreamError } from "./upstream-error.js";
2
1
  const SSE_DONE = "[DONE]";
3
2
  const textEncoder = new TextEncoder();
4
3
  export const encodeSseEvent = (data) => {
@@ -392,19 +391,33 @@ export const withFrameAlignedHeartbeat = (source, options) => {
392
391
  },
393
392
  });
394
393
  };
395
- const chunkHasOutputDelta = (chunk) => {
396
- const nonEmpty = (s) => typeof s === "string" && s.length > 0;
397
- const nonEmptyArr = (a) => Array.isArray(a) && a.length > 0;
398
- return chunk.choices.some((choice) => {
399
- const d = choice.delta;
400
- return (nonEmpty(d.content) ||
401
- nonEmpty(d.reasoning_content) ||
402
- nonEmpty(d.refusal) ||
403
- nonEmptyArr(d.tool_calls) ||
404
- nonEmptyArr(d.reasoning_items) ||
405
- nonEmptyArr(d.server_search_calls));
406
- });
407
- };
394
+ /**
395
+ * Self-imposed graceful deadline for a CANONICAL chunk stream.
396
+ *
397
+ * Vercel hard-kills a function the instant it hits `maxDuration`
398
+ * (clamped to the plan ceiling, which may be well below the declared
399
+ * value). Mid-stream that severs the SSE connection with no terminal
400
+ * frame: the client SDK sees a truncated stream and throws — it
401
+ * cannot distinguish a truncation from a crash, so it cannot
402
+ * continue. The idle heartbeat does NOT help here; it keeps the
403
+ * socket warm right up until the platform guillotine.
404
+ *
405
+ * `withStreamDeadline` cuts us off a touch early instead. It forwards
406
+ * real chunks until `deadlineMs` elapses, then:
407
+ * 1. cancels the source — propagating to the upstream `fetch` body
408
+ * so we stop paying the provider for tokens we'll never deliver;
409
+ * 2. emits ONE synthetic terminal chunk with
410
+ * `finish_reason: "length"` and closes cleanly.
411
+ *
412
+ * The text deltas already streamed ARE the partial answer (they're in
413
+ * the client's hands). The synthetic terminal makes the SSE encoders
414
+ * close properly — `data: [DONE]` for OpenAI, `message_delta`
415
+ * (`stop_reason: "max_tokens"`) + `message_stop` for Anthropic — so
416
+ * the caller gets a clean, truncated turn it can re-prompt to resume,
417
+ * instead of a thrown network error.
418
+ *
419
+ * Wrap BEFORE `.tee()` so the usage-drain branch also terminates.
420
+ */
408
421
  export const withStreamDeadline = (source, options) => {
409
422
  const reader = source.getReader();
410
423
  let lastId = "chatcmpl-openllm-deadline";
@@ -412,7 +425,6 @@ export const withStreamDeadline = (source, options) => {
412
425
  let lastCreated = 0;
413
426
  let hasCreated = false;
414
427
  let finished = false;
415
- let sawOutput = false;
416
428
  const DEADLINE = Symbol("deadline");
417
429
  let timer = null;
418
430
  const deadline = new Promise((resolve) => {
@@ -464,19 +476,12 @@ export const withStreamDeadline = (source, options) => {
464
476
  reader.cancel("openllm-stream-deadline").catch(() => { });
465
477
  if (options.onTruncate !== undefined) {
466
478
  try {
467
- options.onTruncate({ sawOutput });
479
+ options.onTruncate();
468
480
  }
469
481
  catch {
470
482
  // Callback errors must not crash the stream.
471
483
  }
472
484
  }
473
- if (!sawOutput) {
474
- // Zero-output deadline: a clean `length` turn would look like a
475
- // successful empty answer and skip the fallback chain's chance to
476
- // have walked (already committed). Surface a real error frame.
477
- controller.error(new UpstreamStreamError("upstream_stalled", "stream deadline reached with no output"));
478
- return;
479
- }
480
485
  controller.enqueue(terminal());
481
486
  controller.close();
482
487
  return;
@@ -488,9 +493,6 @@ export const withStreamDeadline = (source, options) => {
488
493
  return;
489
494
  }
490
495
  const value = r.value;
491
- if (chunkHasOutputDelta(value)) {
492
- sawOutput = true;
493
- }
494
496
  if (typeof value.id === "string" && value.id.length > 0) {
495
497
  lastId = value.id;
496
498
  }
@@ -510,22 +512,13 @@ export const withStreamDeadline = (source, options) => {
510
512
  },
511
513
  });
512
514
  };
513
- const byteChunkLooksLikeOutput = (bytes) => {
514
- // Only content / tool / reasoning deltas count as partial output.
515
- // Housekeeping (`message_start`, role-only frames) and keepalives must
516
- // NOT flip a zero-output deadline into a clean `max_tokens` turn.
517
- const text = new TextDecoder().decode(bytes);
518
- if (text.length === 0)
519
- return false;
520
- return (/text_delta|input_json_delta|thinking_delta|signature_delta|tool_use|reasoning_content|"content":"[^"]/i.test(text));
521
- };
522
515
  /**
523
516
  * Sibling of `withStreamDeadline` for the Anthropic pass-through:
524
517
  * the source is raw provider SSE bytes (not canonical chunks), and
525
518
  * the synthetic terminator must be a pre-encoded SSE byte sequence
526
519
  * because there is no canonical-decode step in the pipeline.
527
520
  *
528
- * On cutoff with partial output:
521
+ * On cutoff:
529
522
  * 1. cancel the source — propagating to the upstream `fetch` body
530
523
  * so we stop paying the provider for tokens we'll never deliver;
531
524
  * 2. enqueue ONE pre-encoded `message_delta` + `message_stop` pair
@@ -534,10 +527,6 @@ const byteChunkLooksLikeOutput = (bytes) => {
534
527
  * 3. close cleanly so SSE clients see a terminated turn instead of
535
528
  * a severed connection.
536
529
  *
537
- * On cutoff with ZERO output: emit an Anthropic `event: error` instead
538
- * of a clean `max_tokens` turn, so the client and the usage log treat
539
- * it as a failure.
540
- *
541
530
  * Wrap BEFORE `.tee()` so the usage-drain branch also receives the
542
531
  * terminator and the accumulator can finalise. The caller checks
543
532
  * `isTruncated()` after consumption to skip caching a truncated body.
@@ -552,8 +541,6 @@ export const withByteStreamDeadline = (source, options) => {
552
541
  const reader = source.getReader();
553
542
  let finished = false;
554
543
  let truncated = false;
555
- let sawOutput = false;
556
- let zeroOutputTruncate = false;
557
544
  const DEADLINE = Symbol("byte-stream-deadline");
558
545
  let timer = null;
559
546
  const deadline = new Promise((resolve) => {
@@ -565,11 +552,8 @@ export const withByteStreamDeadline = (source, options) => {
565
552
  timer = null;
566
553
  }
567
554
  };
568
- // Leading `\n\n` resynchronises if the deadline fired mid-frame so the
569
- // synthetic event always starts a fresh SSE record.
570
- const lengthTerminator = () => textEncoder.encode('\n\nevent: message_delta\ndata: {"type":"message_delta","delta":{"stop_reason":"max_tokens","stop_sequence":null},"usage":{"output_tokens":0}}\n\n' +
555
+ const terminator = () => textEncoder.encode('event: message_delta\ndata: {"type":"message_delta","delta":{"stop_reason":"max_tokens","stop_sequence":null},"usage":{"output_tokens":0}}\n\n' +
571
556
  'event: message_stop\ndata: {"type":"message_stop"}\n\n');
572
- const zeroOutputError = () => textEncoder.encode('\n\nevent: error\ndata: {"type":"error","error":{"type":"upstream_stalled","message":"stream deadline reached with no output"}}\n\n');
573
557
  const stream = new ReadableStream({
574
558
  async pull(controller) {
575
559
  if (finished) {
@@ -594,13 +578,7 @@ export const withByteStreamDeadline = (source, options) => {
594
578
  truncated = true;
595
579
  clear();
596
580
  reader.cancel("openllm-byte-stream-deadline").catch(() => { });
597
- if (!sawOutput) {
598
- zeroOutputTruncate = true;
599
- controller.enqueue(zeroOutputError());
600
- }
601
- else {
602
- controller.enqueue(lengthTerminator());
603
- }
581
+ controller.enqueue(terminator());
604
582
  controller.close();
605
583
  return;
606
584
  }
@@ -610,9 +588,6 @@ export const withByteStreamDeadline = (source, options) => {
610
588
  controller.close();
611
589
  return;
612
590
  }
613
- if (byteChunkLooksLikeOutput(r.value)) {
614
- sawOutput = true;
615
- }
616
591
  controller.enqueue(r.value);
617
592
  },
618
593
  cancel(reason) {
@@ -621,9 +596,5 @@ export const withByteStreamDeadline = (source, options) => {
621
596
  reader.cancel(reason).catch(() => { });
622
597
  },
623
598
  });
624
- return {
625
- stream,
626
- isTruncated: () => truncated,
627
- isZeroOutputTruncate: () => zeroOutputTruncate,
628
- };
599
+ return { stream, isTruncated: () => truncated };
629
600
  };
@@ -11,15 +11,6 @@ export declare class UpstreamStreamError extends Error {
11
11
  readonly upstreamType: string;
12
12
  constructor(upstreamType: string, message: string);
13
13
  }
14
- /**
15
- * A streaming hop went silent before any byte was forwarded to the client
16
- * (headers wait or raw upstream inactivity). Distinct from a mid-stream
17
- * decode error so the classifier can tag `upstream_stalled` and the
18
- * fallback chain can walk while the hop is still uncommitted.
19
- */
20
- export declare class UpstreamStalledError extends UpstreamStreamError {
21
- constructor(message?: string);
22
- }
23
14
  /**
24
15
  * A stream ended before a terminal chunk after its decoder had already observed
25
16
  * usage. The accumulator wraps every failure type, including transport errors,
@@ -39,5 +30,3 @@ export declare const upstreamErrorFrom: (err: unknown) => {
39
30
  type: string;
40
31
  message: string;
41
32
  };
42
- /** True when `err` is (or wraps) an {@link UpstreamStalledError}. */
43
- export declare const isUpstreamStalledError: (err: unknown) => boolean;
@@ -14,18 +14,6 @@ export class UpstreamStreamError extends Error {
14
14
  this.upstreamType = upstreamType;
15
15
  }
16
16
  }
17
- /**
18
- * A streaming hop went silent before any byte was forwarded to the client
19
- * (headers wait or raw upstream inactivity). Distinct from a mid-stream
20
- * decode error so the classifier can tag `upstream_stalled` and the
21
- * fallback chain can walk while the hop is still uncommitted.
22
- */
23
- export class UpstreamStalledError extends UpstreamStreamError {
24
- constructor(message = "upstream stalled before producing output") {
25
- super("upstream_stalled", message);
26
- this.name = "UpstreamStalledError";
27
- }
28
- }
29
17
  /**
30
18
  * A stream ended before a terminal chunk after its decoder had already observed
31
19
  * usage. The accumulator wraps every failure type, including transport errors,
@@ -55,12 +43,3 @@ export const upstreamErrorFrom = (err) => {
55
43
  }
56
44
  return { type: "stream_error", message: String(err) };
57
45
  };
58
- /** True when `err` is (or wraps) an {@link UpstreamStalledError}. */
59
- export const isUpstreamStalledError = (err) => {
60
- if (err instanceof UpstreamStalledError)
61
- return true;
62
- if (err instanceof Error && err.cause !== undefined) {
63
- return isUpstreamStalledError(err.cause);
64
- }
65
- return false;
66
- };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@openllmsh/sdk",
3
- "version": "2.10.15-alpha.2",
3
+ "version": "2.10.15-alpha.3",
4
4
  "private": false,
5
5
  "description": "The OpenLLM SDK: typed API contracts, wire-format adapters, the device tunnel, client-side vault crypto, the OpenAPI spec and MCP tools.",
6
6
  "keywords": [
@@ -1,93 +0,0 @@
1
- /**
2
- * Pre-commit silence budgets for STREAMING hops only.
3
- *
4
- * Two clocks, both measured on raw upstream bytes (any enqueued chunk —
5
- * including empty ones, keepalive comments, and pings — counts as alive):
6
- *
7
- * 1. **First-byte** (`OPENLLM_STREAM_FIRST_BYTE_TIMEOUT_MS`, default 300s)
8
- * — waiting for response headers, and the gap before the first body
9
- * byte. Near-cap prefills (hundreds of thousands of tokens) can
10
- * legitimately take minutes with no keepalive; this budget is the
11
- * one that must accommodate them. See `peek.ts`.
12
- * 2. **Gap** (`OPENLLM_STREAM_SILENCE_TIMEOUT_MS`, default 120s) — after
13
- * any byte has arrived, until `release()` (hop commit). A provider
14
- * that went silent mid-prefill after talking once is treated as
15
- * stalled.
16
- *
17
- * Non-streaming requests get no short timeout — they legitimately send
18
- * nothing until the full JSON body is ready. A non-positive override
19
- * disables that budget.
20
- */
21
- export declare const DEFAULT_STREAM_FIRST_BYTE_TIMEOUT_MS = 300000;
22
- export declare const DEFAULT_STREAM_SILENCE_TIMEOUT_MS = 120000;
23
- export declare const streamFirstByteTimeoutMs: () => number;
24
- export declare const streamSilenceTimeoutMs: () => number;
25
- /**
26
- * Whether a hop should apply the streaming silence budgets on its
27
- * upstream fetch. ChatGPT's Codex endpoint always streams even when the
28
- * client asked for JSON, so the gate is `alwaysStreams || clientWantsStream`
29
- * — not the client flag alone.
30
- */
31
- export declare const streamingHopNeedsSilence: (opts: {
32
- readonly upstreamAlwaysStreams: boolean;
33
- readonly clientWantsStream: boolean;
34
- }) => boolean;
35
- export declare const firstByteDeadlineAt: (firstByteTimeoutMs: number) => number | null;
36
- /**
37
- * Time left before `deadlineAt`. `null` (no budget) returns 0, which every
38
- * guard reads as disabled; an EXPIRED deadline returns 1 so the guard fires
39
- * at once instead of being read as disabled.
40
- */
41
- export declare const remainingTimeoutMs: (deadlineAt: number | null) => number;
42
- /**
43
- * Read a (non-2xx) response body with `read`, failing with
44
- * {@link UpstreamStalledError} if it takes longer than `timeoutMs`. The body
45
- * is read through a reader this helper owns, so a stall can actually cancel
46
- * it (a `text()` in flight locks the body, and `body.cancel()` would throw).
47
- * `timeoutMs <= 0` disables the budget.
48
- */
49
- export declare const readResponseWithin: <T>(response: Response, timeoutMs: number, read: (response: Response) => Promise<T>) => Promise<T>;
50
- export type TStreamingHeadersGuard = {
51
- readonly signal: AbortSignal;
52
- /** True when the first-byte headers timer fired and the caller did not abort. */
53
- readonly didStall: () => boolean;
54
- /** Clear the headers timer once the Response headers have arrived. */
55
- readonly clear: () => void;
56
- };
57
- /**
58
- * AbortSignal that fires if response headers take longer than
59
- * `firstByteTimeoutMs`. Cleared as soon as headers arrive so it cannot
60
- * abort the body later. Caller abort still wins.
61
- */
62
- export declare const withStreamingHeadersTimeout: (caller: AbortSignal, firstByteTimeoutMs: number) => TStreamingHeadersGuard;
63
- export type TByteSilenceGuard = {
64
- readonly stream: ReadableStream<Uint8Array>;
65
- /**
66
- * Disarm the silence timer. Call once the hop commits (first chunk is
67
- * about to be forwarded to the client) so mid-stream quiet no longer
68
- * walks the fallback chain.
69
- */
70
- readonly release: () => void;
71
- };
72
- export type TByteSilenceGuardOptions = {
73
- /** Budget waiting for the first body byte (default 300s). */
74
- readonly firstByteTimeoutMs: number;
75
- /** Budget for gaps after any byte until `release()` (default 120s). */
76
- readonly gapTimeoutMs: number;
77
- };
78
- /**
79
- * Wrap a raw upstream byte stream with the two-budget silence policy.
80
- * Errors the stream with {@link UpstreamStalledError} on a trip before
81
- * `release()`.
82
- */
83
- export declare const withByteSilenceGuard: (source: ReadableStream<Uint8Array>, options: TByteSilenceGuardOptions) => TByteSilenceGuard;
84
- /** Envelope used when a streaming headers / body wait stalls. */
85
- export declare const upstreamStalledEnvelope: (message?: string) => {
86
- readonly error: {
87
- readonly message: string;
88
- readonly type: "upstream_stalled";
89
- readonly code: "upstream_stalled";
90
- };
91
- };
92
- /** Shared HTTP status for a pre-commit upstream stall (cloud + daemon). */
93
- export declare const UPSTREAM_STALLED_HTTP_STATUS = 504;
@@ -1,262 +0,0 @@
1
- import { UpstreamStalledError } from "./upstream-error.js";
2
- /**
3
- * Pre-commit silence budgets for STREAMING hops only.
4
- *
5
- * Two clocks, both measured on raw upstream bytes (any enqueued chunk —
6
- * including empty ones, keepalive comments, and pings — counts as alive):
7
- *
8
- * 1. **First-byte** (`OPENLLM_STREAM_FIRST_BYTE_TIMEOUT_MS`, default 300s)
9
- * — waiting for response headers, and the gap before the first body
10
- * byte. Near-cap prefills (hundreds of thousands of tokens) can
11
- * legitimately take minutes with no keepalive; this budget is the
12
- * one that must accommodate them. See `peek.ts`.
13
- * 2. **Gap** (`OPENLLM_STREAM_SILENCE_TIMEOUT_MS`, default 120s) — after
14
- * any byte has arrived, until `release()` (hop commit). A provider
15
- * that went silent mid-prefill after talking once is treated as
16
- * stalled.
17
- *
18
- * Non-streaming requests get no short timeout — they legitimately send
19
- * nothing until the full JSON body is ready. A non-positive override
20
- * disables that budget.
21
- */
22
- export const DEFAULT_STREAM_FIRST_BYTE_TIMEOUT_MS = 300_000;
23
- export const DEFAULT_STREAM_SILENCE_TIMEOUT_MS = 120_000;
24
- const readTimeoutMs = (envName, fallback) => {
25
- const raw = process.env[envName];
26
- if (raw === undefined || raw === "")
27
- return fallback;
28
- const override = Number(raw);
29
- if (!Number.isFinite(override))
30
- return fallback;
31
- return override;
32
- };
33
- export const streamFirstByteTimeoutMs = () => readTimeoutMs("OPENLLM_STREAM_FIRST_BYTE_TIMEOUT_MS", DEFAULT_STREAM_FIRST_BYTE_TIMEOUT_MS);
34
- export const streamSilenceTimeoutMs = () => readTimeoutMs("OPENLLM_STREAM_SILENCE_TIMEOUT_MS", DEFAULT_STREAM_SILENCE_TIMEOUT_MS);
35
- /**
36
- * Whether a hop should apply the streaming silence budgets on its
37
- * upstream fetch. ChatGPT's Codex endpoint always streams even when the
38
- * client asked for JSON, so the gate is `alwaysStreams || clientWantsStream`
39
- * — not the client flag alone.
40
- */
41
- export const streamingHopNeedsSilence = (opts) => opts.upstreamAlwaysStreams || opts.clientWantsStream;
42
- /**
43
- * Absolute deadline for the combined headers + first-body-byte budget.
44
- * Pass the remaining time into the body silence guard so the two waits
45
- * share one first-byte window instead of stacking to ~2×.
46
- */
47
- const STALL_ABORT_REASON = "openllm-upstream-stalled";
48
- export const firstByteDeadlineAt = (firstByteTimeoutMs) => firstByteTimeoutMs > 0 ? Date.now() + firstByteTimeoutMs : null;
49
- /**
50
- * Time left before `deadlineAt`. `null` (no budget) returns 0, which every
51
- * guard reads as disabled; an EXPIRED deadline returns 1 so the guard fires
52
- * at once instead of being read as disabled.
53
- */
54
- export const remainingTimeoutMs = (deadlineAt) => deadlineAt === null ? 0 : Math.max(1, deadlineAt - Date.now());
55
- /**
56
- * Read a (non-2xx) response body with `read`, failing with
57
- * {@link UpstreamStalledError} if it takes longer than `timeoutMs`. The body
58
- * is read through a reader this helper owns, so a stall can actually cancel
59
- * it (a `text()` in flight locks the body, and `body.cancel()` would throw).
60
- * `timeoutMs <= 0` disables the budget.
61
- */
62
- export const readResponseWithin = async (response, timeoutMs, read) => {
63
- if (!(timeoutMs > 0) || response.body === null)
64
- return read(response);
65
- const reader = response.body.getReader();
66
- const owned = new Response(new ReadableStream({
67
- pull: async (controller) => {
68
- const next = await reader.read();
69
- if (next.done)
70
- controller.close();
71
- else
72
- controller.enqueue(next.value);
73
- },
74
- cancel: async (reason) => {
75
- await reader.cancel(reason);
76
- },
77
- }), {
78
- status: response.status,
79
- statusText: response.statusText,
80
- headers: response.headers,
81
- });
82
- let timer;
83
- const stalled = new Promise((_, reject) => {
84
- timer = setTimeout(() => {
85
- reader.cancel(STALL_ABORT_REASON).catch(() => { });
86
- reject(new UpstreamStalledError("upstream stalled reading error body"));
87
- }, timeoutMs);
88
- });
89
- try {
90
- return await Promise.race([read(owned), stalled]);
91
- }
92
- finally {
93
- clearTimeout(timer);
94
- }
95
- };
96
- /**
97
- * AbortSignal that fires if response headers take longer than
98
- * `firstByteTimeoutMs`. Cleared as soon as headers arrive so it cannot
99
- * abort the body later. Caller abort still wins.
100
- */
101
- export const withStreamingHeadersTimeout = (caller, firstByteTimeoutMs) => {
102
- if (!(firstByteTimeoutMs > 0)) {
103
- return {
104
- signal: caller,
105
- didStall: () => false,
106
- clear: () => { },
107
- };
108
- }
109
- const stall = new AbortController();
110
- let stalled = false;
111
- const timer = setTimeout(() => {
112
- stalled = true;
113
- stall.abort(STALL_ABORT_REASON);
114
- }, firstByteTimeoutMs);
115
- const clear = () => {
116
- clearTimeout(timer);
117
- };
118
- if (caller.aborted) {
119
- clear();
120
- stall.abort(caller.reason);
121
- }
122
- else {
123
- caller.addEventListener("abort", () => {
124
- clear();
125
- stall.abort(caller.reason);
126
- }, { once: true });
127
- }
128
- return {
129
- signal: stall.signal,
130
- didStall: () => stalled && !caller.aborted,
131
- clear,
132
- };
133
- };
134
- /**
135
- * Wrap a raw upstream byte stream with the two-budget silence policy.
136
- * Errors the stream with {@link UpstreamStalledError} on a trip before
137
- * `release()`.
138
- */
139
- export const withByteSilenceGuard = (source, options) => {
140
- const firstMs = options.firstByteTimeoutMs;
141
- const gapMs = options.gapTimeoutMs;
142
- if (!(firstMs > 0) && !(gapMs > 0)) {
143
- return { stream: source, release: () => { } };
144
- }
145
- const reader = source.getReader();
146
- let released = false;
147
- let finished = false;
148
- let sawByte = false;
149
- let timer = null;
150
- const STALL = Symbol("upstream-silence");
151
- const RELEASED = Symbol("upstream-silence-released");
152
- let settleRace = null;
153
- let racePromise = null;
154
- const clearTimer = () => {
155
- if (timer !== null) {
156
- clearTimeout(timer);
157
- timer = null;
158
- }
159
- };
160
- const currentBudgetMs = () => (sawByte ? gapMs : firstMs);
161
- const armTimer = () => {
162
- clearTimer();
163
- if (released || finished)
164
- return;
165
- const budget = currentBudgetMs();
166
- if (!(budget > 0)) {
167
- racePromise = null;
168
- settleRace = null;
169
- return;
170
- }
171
- racePromise = new Promise((resolve) => {
172
- settleRace = resolve;
173
- timer = setTimeout(() => {
174
- timer = null;
175
- resolve(STALL);
176
- }, budget);
177
- });
178
- };
179
- const release = () => {
180
- released = true;
181
- clearTimer();
182
- const settle = settleRace;
183
- settleRace = null;
184
- racePromise = null;
185
- // Resolve any in-flight race so a late STALL is not observed as a stall.
186
- if (settle !== null)
187
- settle(RELEASED);
188
- };
189
- armTimer();
190
- const stream = new ReadableStream({
191
- async pull(controller) {
192
- if (finished) {
193
- controller.close();
194
- return;
195
- }
196
- if (released) {
197
- const read = await reader.read();
198
- if (read.done) {
199
- finished = true;
200
- clearTimer();
201
- controller.close();
202
- return;
203
- }
204
- controller.enqueue(read.value);
205
- return;
206
- }
207
- const read = reader.read();
208
- read.catch(() => { });
209
- if (racePromise === null)
210
- armTimer();
211
- const race = racePromise;
212
- let result;
213
- try {
214
- result = race === null ? await read : await Promise.race([read, race]);
215
- }
216
- catch (err) {
217
- finished = true;
218
- clearTimer();
219
- throw err;
220
- }
221
- if (result === RELEASED || (result === STALL && released)) {
222
- // Commit raced the timer: wait for the outstanding read without stalling.
223
- result = await read;
224
- }
225
- if (result === STALL) {
226
- finished = true;
227
- clearTimer();
228
- reader.cancel(STALL_ABORT_REASON).catch(() => { });
229
- controller.error(new UpstreamStalledError("upstream stalled before producing output"));
230
- return;
231
- }
232
- if (result.done) {
233
- finished = true;
234
- clearTimer();
235
- controller.close();
236
- return;
237
- }
238
- // Any enqueued chunk — including empty — counts as alive.
239
- sawByte = true;
240
- if (!released) {
241
- armTimer();
242
- }
243
- controller.enqueue(result.value);
244
- },
245
- cancel(reason) {
246
- finished = true;
247
- clearTimer();
248
- reader.cancel(reason).catch(() => { });
249
- },
250
- });
251
- return { stream, release };
252
- };
253
- /** Envelope used when a streaming headers / body wait stalls. */
254
- export const upstreamStalledEnvelope = (message = "upstream stalled before producing output") => ({
255
- error: {
256
- message,
257
- type: "upstream_stalled",
258
- code: "upstream_stalled",
259
- },
260
- });
261
- /** Shared HTTP status for a pre-commit upstream stall (cloud + daemon). */
262
- export const UPSTREAM_STALLED_HTTP_STATUS = 504;