@openllmsh/sdk 0.0.0-stage → 2.10.12-alpha.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (245) hide show
  1. package/ARCHITECTURE.md +43 -0
  2. package/LICENSE +202 -0
  3. package/README.md +142 -2
  4. package/assets/openllm-light.svg +18 -0
  5. package/assets/openllm.svg +18 -0
  6. package/client/chat.ts +140 -0
  7. package/client/devices.ts +269 -0
  8. package/client/http.ts +72 -0
  9. package/client/index.ts +129 -0
  10. package/client/relay-mux.ts +547 -0
  11. package/client/types.ts +51 -0
  12. package/client/vault.ts +239 -0
  13. package/connect/chat.ts +58 -0
  14. package/connect/client.ts +240 -0
  15. package/connect/config.ts +47 -0
  16. package/connect/errors.ts +121 -0
  17. package/connect/index.ts +23 -0
  18. package/connect/oauth.ts +265 -0
  19. package/connect/routes.ts +234 -0
  20. package/connect/store.ts +107 -0
  21. package/mcp/handle.ts +24 -0
  22. package/mcp/index.ts +76 -0
  23. package/mcp/server.ts +319 -0
  24. package/mcp/tools/marketplace.ts +439 -0
  25. package/mcp/tools/memory-client.ts +297 -0
  26. package/mcp/tools/memory-config.ts +32 -0
  27. package/mcp/tools/memory.ts +369 -0
  28. package/mcp/tools/model-priority.ts +88 -0
  29. package/mcp/tools/openllm.ts +883 -0
  30. package/mcp/types.ts +28 -0
  31. package/openapi/call.ts +90 -0
  32. package/openapi/index.ts +8 -0
  33. package/openapi/openapi.json +80213 -0
  34. package/openapi/operations.ts +2075 -0
  35. package/package.json +39 -4
  36. package/protocol/CONTRIBUTING.md +21 -0
  37. package/protocol/README.md +134 -0
  38. package/protocol/access.ts +186 -0
  39. package/protocol/agent-prompt-text.ts +32 -0
  40. package/protocol/agent.ts +991 -0
  41. package/protocol/api-key-syntax.ts +105 -0
  42. package/protocol/assets/openllm-light.svg +18 -0
  43. package/protocol/assets/openllm.svg +18 -0
  44. package/protocol/audio.ts +29 -0
  45. package/protocol/auth.ts +142 -0
  46. package/protocol/changelog.ts +155 -0
  47. package/protocol/chat.ts +458 -0
  48. package/protocol/client-registry.ts +356 -0
  49. package/protocol/config.ts +345 -0
  50. package/protocol/control-channel.ts +115 -0
  51. package/protocol/cooldown-reason.ts +78 -0
  52. package/protocol/cost.ts +10 -0
  53. package/protocol/daemon-port.ts +31 -0
  54. package/protocol/daemon.ts +1524 -0
  55. package/protocol/default-chains.ts +66 -0
  56. package/protocol/device-state.ts +85 -0
  57. package/protocol/doctor-report-local.ts +443 -0
  58. package/protocol/doctor-report-policy.ts +119 -0
  59. package/protocol/doctor-report.ts +947 -0
  60. package/protocol/errors.ts +14 -0
  61. package/protocol/execution-selection.ts +141 -0
  62. package/protocol/hop-grant.ts +33 -0
  63. package/protocol/image-edit-parse.ts +263 -0
  64. package/protocol/images.ts +298 -0
  65. package/protocol/index.ts +74 -0
  66. package/protocol/inference-headers.ts +3 -0
  67. package/protocol/keepalive.ts +69 -0
  68. package/protocol/local-auth.ts +300 -0
  69. package/protocol/local-device-control.ts +133 -0
  70. package/protocol/local-provider.ts +288 -0
  71. package/protocol/local-session.ts +118 -0
  72. package/protocol/marketplace.ts +187 -0
  73. package/protocol/media-attempt.ts +81 -0
  74. package/protocol/media-bytes.ts +28 -0
  75. package/protocol/media-default.ts +357 -0
  76. package/protocol/media-dimensions.ts +132 -0
  77. package/protocol/media-input.ts +61 -0
  78. package/protocol/media-model.ts +142 -0
  79. package/protocol/media-option-issue.ts +90 -0
  80. package/protocol/media-types.ts +70 -0
  81. package/protocol/model-match.ts +1051 -0
  82. package/protocol/model-settings.ts +327 -0
  83. package/protocol/models.ts +767 -0
  84. package/protocol/mux.ts +722 -0
  85. package/protocol/plan-surface.ts +59 -0
  86. package/protocol/provider-models.ts +16 -0
  87. package/protocol/provider-status.ts +206 -0
  88. package/protocol/provider-tier.ts +67 -0
  89. package/protocol/provider-usage.ts +263 -0
  90. package/protocol/providers.ts +901 -0
  91. package/protocol/public-usage-index.ts +354 -0
  92. package/protocol/realtime-generated.ts +31 -0
  93. package/protocol/realtime.ts +324 -0
  94. package/protocol/registry-pointer.ts +118 -0
  95. package/protocol/relay.ts +702 -0
  96. package/protocol/replay-session.ts +11 -0
  97. package/protocol/responses.ts +280 -0
  98. package/protocol/semver.ts +182 -0
  99. package/protocol/setup-mode.ts +22 -0
  100. package/protocol/sha256-hex.ts +98 -0
  101. package/protocol/stats.ts +406 -0
  102. package/protocol/streaming.ts +19 -0
  103. package/protocol/subscription-daemon-floors.ts +33 -0
  104. package/protocol/subscription-provider-compat.ts +104 -0
  105. package/protocol/subscription-provider.ts +32 -0
  106. package/protocol/team-grant.ts +17 -0
  107. package/protocol/team-shareable.ts +61 -0
  108. package/protocol/transcription-encoding.ts +76 -0
  109. package/protocol/url-locality-dns.ts +37 -0
  110. package/protocol/url-locality.ts +288 -0
  111. package/protocol/usage-snapshots.ts +9 -0
  112. package/protocol/vendor-cli-install.ts +29 -0
  113. package/protocol/video-job-plan.ts +50 -0
  114. package/protocol/videos.ts +227 -0
  115. package/tsconfig.json +14 -0
  116. package/tunnel/ARCHITECTURE.md +111 -0
  117. package/tunnel/LICENSE +33 -0
  118. package/tunnel/README.md +45 -0
  119. package/tunnel/assets/openllm-light.svg +18 -0
  120. package/tunnel/assets/openllm.svg +18 -0
  121. package/tunnel/channel-envelope.ts +139 -0
  122. package/tunnel/codec.ts +191 -0
  123. package/tunnel/device-anchor.ts +229 -0
  124. package/tunnel/device-control.ts +118 -0
  125. package/tunnel/device-grant.ts +173 -0
  126. package/tunnel/fleet.ts +77 -0
  127. package/tunnel/index.ts +22 -0
  128. package/tunnel/mux.ts +580 -0
  129. package/tunnel/peer-link.ts +390 -0
  130. package/tunnel/rtc-auth.ts +415 -0
  131. package/tunnel/rtc-duplex.ts +187 -0
  132. package/tunnel/rtc-ice.ts +136 -0
  133. package/tunnel/secure.ts +1298 -0
  134. package/tunnel/stream-reset-error.ts +14 -0
  135. package/tunnel/streams.ts +1430 -0
  136. package/tunnel/team-grant.ts +302 -0
  137. package/tunnel/wire-bytes.ts +102 -0
  138. package/vault/ARCHITECTURE.md +470 -0
  139. package/vault/LICENSE +33 -0
  140. package/vault/api-key.ts +190 -0
  141. package/vault/argon2.ts +106 -0
  142. package/vault/cli-auth-seal.ts +157 -0
  143. package/vault/cred-seal.ts +146 -0
  144. package/vault/credential.ts +55 -0
  145. package/vault/crypto.ts +210 -0
  146. package/vault/daemon-attest.ts +229 -0
  147. package/vault/derive.ts +113 -0
  148. package/vault/device-access.ts +87 -0
  149. package/vault/device-anchor.ts +185 -0
  150. package/vault/device-control.ts +174 -0
  151. package/vault/handoff.ts +396 -0
  152. package/vault/index.ts +21 -0
  153. package/vault/passkey.ts +243 -0
  154. package/vault/phrase.ts +65 -0
  155. package/vault/pin.ts +153 -0
  156. package/vault/public-key-seal.ts +131 -0
  157. package/vault/seed.ts +272 -0
  158. package/vault/team.ts +355 -0
  159. package/vault/vault-item.ts +131 -0
  160. package/vault/vault-migrate-digest.ts +29 -0
  161. package/vault/wrapped.ts +77 -0
  162. package/wire/ARCHITECTURE.md +90 -0
  163. package/wire/CONTRIBUTING.md +19 -0
  164. package/wire/LICENSE +33 -0
  165. package/wire/README.md +51 -0
  166. package/wire/adapters/messages/anthropic-map.ts +37 -0
  167. package/wire/adapters/messages/index.ts +7 -0
  168. package/wire/adapters/messages/reasoning-from-items.ts +38 -0
  169. package/wire/adapters/messages/reasoning-signature.ts +243 -0
  170. package/wire/adapters/messages/request.ts +574 -0
  171. package/wire/adapters/messages/response.ts +215 -0
  172. package/wire/adapters/messages/streaming.ts +1027 -0
  173. package/wire/adapters/messages/usage.ts +43 -0
  174. package/wire/adapters/responses/index.ts +3 -0
  175. package/wire/adapters/responses/request.ts +395 -0
  176. package/wire/adapters/responses/response.ts +88 -0
  177. package/wire/adapters/responses/streaming.ts +344 -0
  178. package/wire/assets/openllm-light.svg +18 -0
  179. package/wire/assets/openllm.svg +18 -0
  180. package/wire/features/account-rank.ts +166 -0
  181. package/wire/features/compaction/compact-request.ts +862 -0
  182. package/wire/features/compaction/compaction-text.ts +43 -0
  183. package/wire/features/context-demote.ts +47 -0
  184. package/wire/features/context-overflow-strategy.ts +26 -0
  185. package/wire/features/context-skip.ts +170 -0
  186. package/wire/features/conversation-affinity.ts +144 -0
  187. package/wire/features/max-tokens-backfill.ts +141 -0
  188. package/wire/features/model-caps.ts +53 -0
  189. package/wire/features/model-settings.ts +24 -0
  190. package/wire/features/output-limit-clamp.ts +64 -0
  191. package/wire/features/quota-gate.ts +85 -0
  192. package/wire/index.ts +44 -0
  193. package/wire/lib/attempt-outcome.ts +43 -0
  194. package/wire/lib/canonical/content-part.ts +38 -0
  195. package/wire/lib/canonical/count-tokens-capability.ts +138 -0
  196. package/wire/lib/canonical/encoding-select.ts +82 -0
  197. package/wire/lib/canonical/image-signature.ts +202 -0
  198. package/wire/lib/canonical/image-sse.ts +216 -0
  199. package/wire/lib/canonical/message.ts +43 -0
  200. package/wire/lib/canonical/token-estimate.ts +266 -0
  201. package/wire/lib/encrypted-content.ts +66 -0
  202. package/wire/lib/error-class.ts +154 -0
  203. package/wire/lib/error-envelope.ts +56 -0
  204. package/wire/lib/forwarded-headers.ts +180 -0
  205. package/wire/lib/provider-error-rules.ts +48 -0
  206. package/wire/lib/refusal.ts +38 -0
  207. package/wire/lib/responses-apply-patch.ts +104 -0
  208. package/wire/lib/responses-namespace.ts +53 -0
  209. package/wire/lib/secret-scrub.ts +101 -0
  210. package/wire/lib/streaming/accumulate.ts +221 -0
  211. package/wire/lib/streaming/finish-reason.ts +33 -0
  212. package/wire/lib/streaming/peek.ts +154 -0
  213. package/wire/lib/streaming/provider-decode.ts +259 -0
  214. package/wire/lib/streaming/response-stream.ts +71 -0
  215. package/wire/lib/streaming/sse.ts +699 -0
  216. package/wire/lib/streaming/strip-tool-calls.ts +95 -0
  217. package/wire/lib/streaming/upstream-error.ts +58 -0
  218. package/wire/lib/strip-subagent-isolation.ts +284 -0
  219. package/wire/lib/thinking-signature.ts +156 -0
  220. package/wire/lib/token-ruler/bpe-core.ts +426 -0
  221. package/wire/lib/token-ruler/index.ts +2 -0
  222. package/wire/lib/token-ruler/ruler.ts +79 -0
  223. package/wire/lib/token-ruler/vocab/claude-ranks.ts +16 -0
  224. package/wire/lib/token-ruler/vocab/claude.ts +22 -0
  225. package/wire/lib/token-ruler/vocab/o200k-ranks.ts +18 -0
  226. package/wire/lib/token-ruler/vocab/o200k.ts +19 -0
  227. package/wire/lib/tool-name-map.ts +271 -0
  228. package/wire/lib/tool-schema.ts +353 -0
  229. package/wire/lib/usage-observer.ts +125 -0
  230. package/wire/providers/anthropic/adaptive-thinking.ts +291 -0
  231. package/wire/providers/anthropic/beta-headers.ts +165 -0
  232. package/wire/providers/anthropic/index.ts +10 -0
  233. package/wire/providers/anthropic/request.ts +607 -0
  234. package/wire/providers/anthropic/response.ts +155 -0
  235. package/wire/providers/anthropic/streaming.ts +315 -0
  236. package/wire/providers/anthropic/web-search-domains.ts +49 -0
  237. package/wire/providers/chatgpt/common.ts +136 -0
  238. package/wire/providers/chatgpt/index.ts +9 -0
  239. package/wire/providers/chatgpt/request.ts +959 -0
  240. package/wire/providers/chatgpt/streaming.ts +1007 -0
  241. package/wire/providers/chatgpt/web-search.ts +63 -0
  242. package/wire/providers/grok/web-search.ts +87 -0
  243. package/wire/providers/kimi/web-search.ts +145 -0
  244. package/wire/providers/upstream-deny.ts +355 -0
  245. package/wire/providers/upstream-request.ts +619 -0
@@ -0,0 +1,699 @@
1
+ import type {
2
+ TChatCompletionChunk,
3
+ THeartbeatOptions,
4
+ TSseEvent,
5
+ } from "@openllmsh/sdk/protocol";
6
+
7
+ const SSE_DONE = "[DONE]";
8
+
9
+ const textEncoder = new TextEncoder();
10
+
11
+ export const encodeSseEvent = (data: unknown): Uint8Array => {
12
+ const payload = data === SSE_DONE ? SSE_DONE : JSON.stringify(data);
13
+ return textEncoder.encode(`data: ${payload}\n\n`);
14
+ };
15
+
16
+ export const encodeSseDone = (): Uint8Array =>
17
+ textEncoder.encode(`data: ${SSE_DONE}\n\n`);
18
+
19
+ export const encodeSseComment = (comment: string): Uint8Array =>
20
+ textEncoder.encode(`: ${comment}\n\n`);
21
+
22
+ // Anthropic's documented keepalive: a real `ping` event (NOT a bare
23
+ // SSE comment). Claude Code drives the official Anthropic SDK whose
24
+ // event-typed stream switch-cases on `type` — emitting the genuine
25
+ // ping is what its standby loop waits on between content blocks while
26
+ // the model is thinking / running tools. Ref: Anthropic streaming
27
+ // messages spec ("ping events ... periodically").
28
+ export const encodeAnthropicPing = (): Uint8Array =>
29
+ textEncoder.encode('event: ping\ndata: {"type": "ping"}\n\n');
30
+
31
+ /**
32
+ * Length of an SSE line terminator at `i`, or 0 if this is not one.
33
+ * Returns -1 when `buffer[i]` is CR at end-of-buffer: a following LF
34
+ * (the second half of CRLF) may still arrive, so the caller must wait.
35
+ */
36
+ const lineTerminatorLength = (
37
+ buffer: string,
38
+ i: number,
39
+ holdSplitCrlf: boolean,
40
+ ): number => {
41
+ if (i >= buffer.length) return 0;
42
+ if (buffer[i] === "\r") {
43
+ if (i + 1 >= buffer.length) return holdSplitCrlf ? -1 : 1;
44
+ return buffer[i + 1] === "\n" ? 2 : 1;
45
+ }
46
+ if (buffer[i] === "\n") return 1;
47
+ return 0;
48
+ };
49
+
50
+ const stripTrailingLineTerminator = (value: string): string => {
51
+ if (value.endsWith("\r\n")) return value.slice(0, -2);
52
+ if (value.endsWith("\n") || value.endsWith("\r")) return value.slice(0, -1);
53
+ return value;
54
+ };
55
+
56
+ /**
57
+ * Split a buffer into complete SSE events (blank-line terminated) and
58
+ * leftover bytes. Accepts LF, CRLF, CR, and mixed endings. A CR at the
59
+ * end of the buffer is never treated as a terminator on its own so a
60
+ * CRLF pair split across chunks stays one terminator.
61
+ */
62
+ type TSplitFrame = {
63
+ readonly content: string;
64
+ readonly raw: string;
65
+ };
66
+
67
+ const splitFrames = (
68
+ buffer: string,
69
+ ): { frames: TSplitFrame[]; rest: string } => {
70
+ const frames: TSplitFrame[] = [];
71
+ let i = 0;
72
+ let lineStart = 0;
73
+ let eventStart = 0;
74
+ while (i < buffer.length) {
75
+ const term = lineTerminatorLength(buffer, i, true);
76
+ if (term === -1) break;
77
+ if (term === 0) {
78
+ i += 1;
79
+ continue;
80
+ }
81
+ if (i === lineStart) {
82
+ frames.push({
83
+ content: stripTrailingLineTerminator(
84
+ buffer.slice(eventStart, lineStart),
85
+ ),
86
+ raw: buffer.slice(eventStart, i + term),
87
+ });
88
+ i += term;
89
+ eventStart = i;
90
+ lineStart = i;
91
+ continue;
92
+ }
93
+ i += term;
94
+ lineStart = i;
95
+ }
96
+ return { frames, rest: buffer.slice(eventStart) };
97
+ };
98
+
99
+ const sseFieldLines = (raw: string): string[] => {
100
+ const lines: string[] = [];
101
+ let i = 0;
102
+ let lineStart = 0;
103
+ while (i < raw.length) {
104
+ const term = lineTerminatorLength(raw, i, false);
105
+ if (term === -1) break;
106
+ if (term === 0) {
107
+ i += 1;
108
+ continue;
109
+ }
110
+ lines.push(raw.slice(lineStart, i));
111
+ i += term;
112
+ lineStart = i;
113
+ }
114
+ if (lineStart < raw.length) {
115
+ lines.push(raw.slice(lineStart));
116
+ }
117
+ return lines;
118
+ };
119
+
120
+ const parseEvent = (raw: string): TSseEvent | null => {
121
+ const lines = sseFieldLines(raw);
122
+ const dataLines: string[] = [];
123
+ let comment: string | null = null;
124
+ for (const line of lines) {
125
+ if (line === "") continue;
126
+ if (line.startsWith(":")) {
127
+ if (comment === null) comment = line.slice(1).trimStart();
128
+ continue;
129
+ }
130
+ if (line.startsWith("data:")) {
131
+ dataLines.push(line.slice(5).trimStart());
132
+ }
133
+ // event: / id: / retry: are ignored and must not erase data.
134
+ }
135
+ if (dataLines.length > 0) {
136
+ const data = dataLines.join("\n");
137
+ if (data === SSE_DONE) return { kind: "done" };
138
+ return { kind: "data", data };
139
+ }
140
+ if (comment !== null) return { kind: "comment", comment };
141
+ return null;
142
+ };
143
+
144
+ /**
145
+ * Rope of decoded string chunks. Avoids `buffer += chunk` (O(n) copy per
146
+ * tiny TCP read) by keeping parts until a frame boundary forces a join.
147
+ */
148
+ type TStringRope = {
149
+ parts: string[];
150
+ length: number;
151
+ };
152
+
153
+ const emptyRope = (): TStringRope => ({ parts: [], length: 0 });
154
+
155
+ const ropePush = (rope: TStringRope, piece: string): void => {
156
+ if (piece.length === 0) return;
157
+ rope.parts.push(piece);
158
+ rope.length += piece.length;
159
+ };
160
+
161
+ const ropeJoin = (rope: TStringRope): string => {
162
+ if (rope.parts.length === 0) return "";
163
+ if (rope.parts.length === 1) return rope.parts[0] ?? "";
164
+ return rope.parts.join("");
165
+ };
166
+
167
+ const ropeSetRest = (rope: TStringRope, rest: string): void => {
168
+ if (rest.length === 0) {
169
+ rope.parts = [];
170
+ rope.length = 0;
171
+ return;
172
+ }
173
+ rope.parts = [rest];
174
+ rope.length = rest.length;
175
+ };
176
+
177
+ /** True when the latest piece may have completed an SSE frame (blank line). */
178
+ const pieceMayCompleteFrame = (piece: string): boolean => {
179
+ if (piece.length === 0) return false;
180
+ // Blank-line terminators: \n\n, \r\n\r\n, \r\r, and mixed. A lone
181
+ // terminator at either end is enough to bother joining — the previous
182
+ // rest may have ended with `\n`/`\r` and this piece starts with the
183
+ // matching half of the blank-line boundary.
184
+ const first = piece.charCodeAt(0);
185
+ const last = piece.charCodeAt(piece.length - 1);
186
+ return (
187
+ first === 10 /* \n */ ||
188
+ first === 13 /* \r */ ||
189
+ last === 10 ||
190
+ last === 13 ||
191
+ piece.includes("\n\n") ||
192
+ // Mixed endings: LF line + CR blank (`\n\r` / `\n\r\n`) interior to the
193
+ // piece — first/last checks miss these when the piece is padded.
194
+ piece.includes("\n\r") ||
195
+ piece.includes("\r\n\r\n") ||
196
+ piece.includes("\r\r")
197
+ );
198
+ };
199
+
200
+ export const sseEventStream = (
201
+ source: ReadableStream<Uint8Array>,
202
+ ): ReadableStream<TSseEvent> => {
203
+ const reader = source.getReader();
204
+ const decoder = new TextDecoder();
205
+ const rope = emptyRope();
206
+ return new ReadableStream<TSseEvent>({
207
+ async pull(controller) {
208
+ for (;;) {
209
+ const { value, done } = await reader.read();
210
+ if (done) {
211
+ ropePush(rope, decoder.decode());
212
+ const buffer = ropeJoin(rope);
213
+ const { frames, rest } = splitFrames(buffer);
214
+ for (const frame of frames) {
215
+ const parsed = parseEvent(frame.content);
216
+ if (parsed) controller.enqueue(parsed);
217
+ }
218
+ const trailing = parseEvent(rest);
219
+ if (trailing) controller.enqueue(trailing);
220
+ ropeSetRest(rope, "");
221
+ controller.close();
222
+ return;
223
+ }
224
+ const piece = decoder.decode(value, { stream: true });
225
+ ropePush(rope, piece);
226
+ // Skip the join+scan while the new bytes cannot complete a frame.
227
+ if (rope.parts.length > 1 && !pieceMayCompleteFrame(piece)) {
228
+ continue;
229
+ }
230
+ const buffer = ropeJoin(rope);
231
+ const { frames, rest } = splitFrames(buffer);
232
+ ropeSetRest(rope, rest);
233
+ if (frames.length === 0) continue;
234
+ for (const frame of frames) {
235
+ const parsed = parseEvent(frame.content);
236
+ if (parsed) controller.enqueue(parsed);
237
+ }
238
+ return;
239
+ }
240
+ },
241
+ cancel(reason) {
242
+ reader.cancel(reason).catch(() => {});
243
+ },
244
+ });
245
+ };
246
+
247
+ // runtime-only: holds ReadableStream<T> branches from .tee(); cannot be a Schema.
248
+ export type TTeeResult<T> = {
249
+ client: ReadableStream<T>;
250
+ observer: ReadableStream<T>;
251
+ };
252
+
253
+ export const teeStream = <T>(source: ReadableStream<T>): TTeeResult<T> => {
254
+ const [a, b] = source.tee();
255
+ return { client: a, observer: b };
256
+ };
257
+
258
+ /**
259
+ * Inject a keepalive frame whenever the upstream goes quiet so a CLI
260
+ * sitting in its streaming read loop (Claude Code, Codex, …) doesn't
261
+ * hit its socket read timeout — and so idle-connection-reaping
262
+ * intermediaries (CDNs, corporate proxies) don't drop the connection —
263
+ * while a reasoning model is thinking or a tool is running.
264
+ *
265
+ * IMPORTANT: only safe over a source that enqueues COMPLETE SSE frames
266
+ * per chunk (our adapter encoders do). It must NOT wrap a raw upstream
267
+ * byte stream whose chunk boundaries are arbitrary — a keepalive could
268
+ * land mid-frame and corrupt the event. Native pass-through therefore
269
+ * relies on the upstream's own `ping`/`message_start` instead.
270
+ *
271
+ * The first beat fires immediately (not after `intervalMs`): the
272
+ * dangerous window is the silence between HTTP 200 and the first token
273
+ * of a slow reasoning run, which a delayed first beat would leave
274
+ * uncovered for a full interval.
275
+ */
276
+ export const withHeartbeat = (
277
+ source: ReadableStream<Uint8Array>,
278
+ options: THeartbeatOptions,
279
+ ): ReadableStream<Uint8Array> => {
280
+ const reader = source.getReader();
281
+ let timer: ReturnType<typeof setInterval> | null = null;
282
+ let controllerRef: ReadableStreamDefaultController<Uint8Array> | null = null;
283
+ let closed = false;
284
+
285
+ const beat =
286
+ options.kind === "anthropic_ping"
287
+ ? encodeAnthropicPing
288
+ : (): Uint8Array => encodeSseComment("keepalive");
289
+
290
+ const clear = (): void => {
291
+ if (timer !== null) {
292
+ clearInterval(timer);
293
+ timer = null;
294
+ }
295
+ };
296
+
297
+ const safeEnqueue = (chunk: Uint8Array): void => {
298
+ if (closed || controllerRef === null) return;
299
+ try {
300
+ controllerRef.enqueue(chunk);
301
+ } catch {
302
+ // Controller already closed/errored (client hung up between the
303
+ // timer firing and this microtask) — stop beating.
304
+ closed = true;
305
+ clear();
306
+ }
307
+ };
308
+
309
+ return new ReadableStream<Uint8Array>({
310
+ start(controller) {
311
+ controllerRef = controller;
312
+ // Immediate first beat closes the post-200 / pre-first-token gap.
313
+ safeEnqueue(beat());
314
+ timer = setInterval(() => safeEnqueue(beat()), options.intervalMs);
315
+ },
316
+ async pull(controller) {
317
+ const { value, done } = await reader.read();
318
+ if (done) {
319
+ closed = true;
320
+ clear();
321
+ controller.close();
322
+ return;
323
+ }
324
+ if (value) controller.enqueue(value);
325
+ },
326
+ cancel(reason) {
327
+ closed = true;
328
+ clear();
329
+ reader.cancel(reason).catch(() => {});
330
+ },
331
+ });
332
+ };
333
+
334
+ /**
335
+ * Frame-aligned keepalive for a RAW upstream byte stream (native
336
+ * Anthropic pass-through). `withHeartbeat` cannot be used there: the
337
+ * upstream's chunk boundaries are arbitrary, so a keepalive injected
338
+ * between two reads could land in the middle of a half-sent SSE event
339
+ * and corrupt it. That is why pass-through historically injected
340
+ * nothing and relied on the upstream's own `ping` events — which
341
+ * leaves the client stranded the moment an anthropic-wire provider
342
+ * goes quiet for minutes during a long reasoning / tool run (the
343
+ * reported "chat stopped while it was actually doing something").
344
+ *
345
+ * This wrapper makes injection safe by re-aligning the stream to SSE
346
+ * frame boundaries: incoming bytes are buffered and only COMPLETE
347
+ * frames (terminated by a blank line) are forwarded; a partial frame
348
+ * is held back until its terminator arrives. Because the client only
349
+ * ever sees whole frames, a keepalive enqueued on the interval is
350
+ * always emitted at a frame boundary — never spliced inside an event
351
+ * — regardless of how the upstream chunked its bytes.
352
+ *
353
+ * Trailing bytes with no terminator (an upstream that ends mid-frame)
354
+ * are flushed verbatim on close so nothing is silently dropped.
355
+ */
356
+ export const withFrameAlignedHeartbeat = (
357
+ source: ReadableStream<Uint8Array>,
358
+ options: THeartbeatOptions,
359
+ ): ReadableStream<Uint8Array> => {
360
+ const reader = source.getReader();
361
+ const decoder = new TextDecoder();
362
+ let timer: ReturnType<typeof setInterval> | null = null;
363
+ let controllerRef: ReadableStreamDefaultController<Uint8Array> | null = null;
364
+ let closed = false;
365
+ const rope = emptyRope();
366
+
367
+ const beat =
368
+ options.kind === "anthropic_ping"
369
+ ? encodeAnthropicPing
370
+ : (): Uint8Array => encodeSseComment("keepalive");
371
+
372
+ const clear = (): void => {
373
+ if (timer !== null) {
374
+ clearInterval(timer);
375
+ timer = null;
376
+ }
377
+ };
378
+
379
+ const safeEnqueue = (chunk: Uint8Array): void => {
380
+ if (closed || controllerRef === null) return;
381
+ try {
382
+ controllerRef.enqueue(chunk);
383
+ } catch {
384
+ closed = true;
385
+ clear();
386
+ }
387
+ };
388
+
389
+ return new ReadableStream<Uint8Array>({
390
+ start(controller) {
391
+ controllerRef = controller;
392
+ // Immediate first beat closes the post-200 / pre-first-token gap.
393
+ safeEnqueue(beat());
394
+ timer = setInterval(() => safeEnqueue(beat()), options.intervalMs);
395
+ },
396
+ async pull(controller) {
397
+ try {
398
+ for (;;) {
399
+ const { value, done } = await reader.read();
400
+ if (done) {
401
+ // Flush decoder state, then any unterminated trailing bytes
402
+ // verbatim so an upstream that ends mid-frame loses nothing.
403
+ ropePush(rope, decoder.decode());
404
+ const buffer = ropeJoin(rope);
405
+ const flushed = splitFrames(buffer);
406
+ for (const frame of flushed.frames) {
407
+ controller.enqueue(textEncoder.encode(frame.raw));
408
+ }
409
+ if (flushed.rest.length > 0) {
410
+ controller.enqueue(textEncoder.encode(flushed.rest));
411
+ }
412
+ ropeSetRest(rope, "");
413
+ closed = true;
414
+ clear();
415
+ controller.close();
416
+ return;
417
+ }
418
+ const piece = decoder.decode(value, { stream: true });
419
+ ropePush(rope, piece);
420
+ if (rope.parts.length > 1 && !pieceMayCompleteFrame(piece)) {
421
+ continue;
422
+ }
423
+ const buffer = ropeJoin(rope);
424
+ const { frames, rest } = splitFrames(buffer);
425
+ ropeSetRest(rope, rest);
426
+ if (frames.length === 0) continue;
427
+ for (const frame of frames) {
428
+ controller.enqueue(textEncoder.encode(frame.raw));
429
+ }
430
+ return;
431
+ }
432
+ } catch {
433
+ // Raw passthrough has no canonical encoder to translate a rejected
434
+ // reader. Emit Anthropic's native terminal envelope rather than severing
435
+ // the socket; never expose the thrown message because fetch errors often
436
+ // contain credential-bearing upstream URLs.
437
+ closed = true;
438
+ clear();
439
+ controller.enqueue(
440
+ textEncoder.encode(
441
+ 'event: error\ndata: {"type":"error","error":{"type":"stream_error","message":"upstream stream failed"}}\n\n',
442
+ ),
443
+ );
444
+ controller.close();
445
+ }
446
+ },
447
+ cancel(reason) {
448
+ closed = true;
449
+ clear();
450
+ reader.cancel(reason).catch(() => {});
451
+ },
452
+ });
453
+ };
454
+
455
+ /**
456
+ * Self-imposed graceful deadline for a CANONICAL chunk stream.
457
+ *
458
+ * Vercel hard-kills a function the instant it hits `maxDuration`
459
+ * (clamped to the plan ceiling, which may be well below the declared
460
+ * value). Mid-stream that severs the SSE connection with no terminal
461
+ * frame: the client SDK sees a truncated stream and throws — it
462
+ * cannot distinguish a truncation from a crash, so it cannot
463
+ * continue. The idle heartbeat does NOT help here; it keeps the
464
+ * socket warm right up until the platform guillotine.
465
+ *
466
+ * `withStreamDeadline` cuts us off a touch early instead. It forwards
467
+ * real chunks until `deadlineMs` elapses, then:
468
+ * 1. cancels the source — propagating to the upstream `fetch` body
469
+ * so we stop paying the provider for tokens we'll never deliver;
470
+ * 2. emits ONE synthetic terminal chunk with
471
+ * `finish_reason: "length"` and closes cleanly.
472
+ *
473
+ * The text deltas already streamed ARE the partial answer (they're in
474
+ * the client's hands). The synthetic terminal makes the SSE encoders
475
+ * close properly — `data: [DONE]` for OpenAI, `message_delta`
476
+ * (`stop_reason: "max_tokens"`) + `message_stop` for Anthropic — so
477
+ * the caller gets a clean, truncated turn it can re-prompt to resume,
478
+ * instead of a thrown network error.
479
+ *
480
+ * Wrap BEFORE `.tee()` so the usage-drain branch also terminates.
481
+ */
482
+ export const withStreamDeadline = (
483
+ source: ReadableStream<TChatCompletionChunk>,
484
+ options: {
485
+ readonly deadlineMs: number;
486
+ /**
487
+ * Fires exactly once if the deadline expires before the source
488
+ * closes naturally. Callers use this to skip caching a body that
489
+ * was sealed by the synthetic terminator instead of the upstream's
490
+ * own `finish_reason`. Omit to ignore truncation.
491
+ */
492
+ readonly onTruncate?: () => void;
493
+ },
494
+ ): ReadableStream<TChatCompletionChunk> => {
495
+ const reader = source.getReader();
496
+ let lastId = "chatcmpl-openllm-deadline";
497
+ let lastModel = "";
498
+ let lastCreated = 0;
499
+ let hasCreated = false;
500
+ let finished = false;
501
+ const DEADLINE: unique symbol = Symbol("deadline");
502
+ let timer: ReturnType<typeof setTimeout> | null = null;
503
+ const deadline = new Promise<typeof DEADLINE>((resolve) => {
504
+ timer = setTimeout(
505
+ () => resolve(DEADLINE),
506
+ Math.max(0, options.deadlineMs),
507
+ );
508
+ });
509
+ const clear = (): void => {
510
+ if (timer !== null) {
511
+ clearTimeout(timer);
512
+ timer = null;
513
+ }
514
+ };
515
+ // The synthetic terminal preserves the stream's last-seen id/created/model but
516
+ // carries NO usage: OpenAI reports usage once, last-wins, on a `choices: []`
517
+ // chunk, so re-emitting the last usage snapshot on this populated-choice
518
+ // terminal would double-count for clients that sum snapshots. The usage-logging
519
+ // tee accounts partial usage independently.
520
+ const terminal = (): TChatCompletionChunk => ({
521
+ id: lastId,
522
+ object: "chat.completion.chunk",
523
+ created: hasCreated ? lastCreated : Math.floor(Date.now() / 1000),
524
+ model: lastModel,
525
+ choices: [{ index: 0, delta: {}, finish_reason: "length" }],
526
+ });
527
+
528
+ return new ReadableStream<TChatCompletionChunk>({
529
+ async pull(controller) {
530
+ if (finished) {
531
+ controller.close();
532
+ return;
533
+ }
534
+ const read = reader.read();
535
+ // The losing race branch must not surface as an unhandled
536
+ // rejection once we cancel the reader on deadline.
537
+ read.catch(() => {});
538
+ let r: Awaited<typeof read> | typeof DEADLINE;
539
+ try {
540
+ r = await Promise.race([read, deadline]);
541
+ } catch (err) {
542
+ // Source errored before the deadline — preserve existing
543
+ // behaviour (the SSE encoder's catch emits a clean error
544
+ // frame + terminator).
545
+ finished = true;
546
+ clear();
547
+ throw err;
548
+ }
549
+ if (r === DEADLINE) {
550
+ finished = true;
551
+ clear();
552
+ reader.cancel("openllm-stream-deadline").catch(() => {});
553
+ if (options.onTruncate !== undefined) {
554
+ try {
555
+ options.onTruncate();
556
+ } catch {
557
+ // Callback errors must not crash the stream.
558
+ }
559
+ }
560
+ controller.enqueue(terminal());
561
+ controller.close();
562
+ return;
563
+ }
564
+ if (r.done) {
565
+ finished = true;
566
+ clear();
567
+ controller.close();
568
+ return;
569
+ }
570
+ const value = r.value;
571
+ if (typeof value.id === "string" && value.id.length > 0) {
572
+ lastId = value.id;
573
+ }
574
+ if (typeof value.model === "string" && value.model.length > 0) {
575
+ lastModel = value.model;
576
+ }
577
+ if (Number.isFinite(value.created) && value.created >= 0) {
578
+ lastCreated = value.created;
579
+ hasCreated = true;
580
+ }
581
+ controller.enqueue(value);
582
+ },
583
+ cancel(reason) {
584
+ finished = true;
585
+ clear();
586
+ reader.cancel(reason).catch(() => {});
587
+ },
588
+ });
589
+ };
590
+
591
+ /**
592
+ * Result of wrapping a raw byte stream with a deadline guard.
593
+ *
594
+ * `isTruncated()` reflects whether the deadline fired before the
595
+ * source closed naturally — the caller checks this AFTER the stream
596
+ * is consumed to decide whether to cache or discard the accumulated
597
+ * body. A truncated body would be a broken replay.
598
+ */
599
+ export type TByteStreamDeadlineHandle = {
600
+ readonly stream: ReadableStream<Uint8Array>;
601
+ readonly isTruncated: () => boolean;
602
+ };
603
+
604
+ /**
605
+ * Sibling of `withStreamDeadline` for the Anthropic pass-through:
606
+ * the source is raw provider SSE bytes (not canonical chunks), and
607
+ * the synthetic terminator must be a pre-encoded SSE byte sequence
608
+ * because there is no canonical-decode step in the pipeline.
609
+ *
610
+ * On cutoff:
611
+ * 1. cancel the source — propagating to the upstream `fetch` body
612
+ * so we stop paying the provider for tokens we'll never deliver;
613
+ * 2. enqueue ONE pre-encoded `message_delta` + `message_stop` pair
614
+ * with `stop_reason: "max_tokens"` (matching what the canonical
615
+ * adapter already emits for `finish_reason: "length"`);
616
+ * 3. close cleanly so SSE clients see a terminated turn instead of
617
+ * a severed connection.
618
+ *
619
+ * Wrap BEFORE `.tee()` so the usage-drain branch also receives the
620
+ * terminator and the accumulator can finalise. The caller checks
621
+ * `isTruncated()` after consumption to skip caching a truncated body.
622
+ *
623
+ * Best-effort frame alignment: the deadline can fire mid-frame on the
624
+ * raw byte stream, but the synthetic terminator starts with a fresh
625
+ * `event:` line — so the worst case is a torn frame followed by two
626
+ * well-formed events, which the Anthropic SDK parser handles by
627
+ * discarding the unterminated lead bytes.
628
+ */
629
+ export const withByteStreamDeadline = (
630
+ source: ReadableStream<Uint8Array>,
631
+ options: { deadlineMs: number },
632
+ ): TByteStreamDeadlineHandle => {
633
+ const reader = source.getReader();
634
+ let finished = false;
635
+ let truncated = false;
636
+ const DEADLINE: unique symbol = Symbol("byte-stream-deadline");
637
+ let timer: ReturnType<typeof setTimeout> | null = null;
638
+ const deadline = new Promise<typeof DEADLINE>((resolve) => {
639
+ timer = setTimeout(
640
+ () => resolve(DEADLINE),
641
+ Math.max(0, options.deadlineMs),
642
+ );
643
+ });
644
+ const clear = (): void => {
645
+ if (timer !== null) {
646
+ clearTimeout(timer);
647
+ timer = null;
648
+ }
649
+ };
650
+ const terminator = (): Uint8Array =>
651
+ textEncoder.encode(
652
+ 'event: message_delta\ndata: {"type":"message_delta","delta":{"stop_reason":"max_tokens","stop_sequence":null},"usage":{"output_tokens":0}}\n\n' +
653
+ 'event: message_stop\ndata: {"type":"message_stop"}\n\n',
654
+ );
655
+
656
+ const stream = new ReadableStream<Uint8Array>({
657
+ async pull(controller) {
658
+ if (finished) {
659
+ controller.close();
660
+ return;
661
+ }
662
+ const read = reader.read();
663
+ // The losing race branch must not surface as an unhandled
664
+ // rejection once we cancel the reader on deadline.
665
+ read.catch(() => {});
666
+ let r: Awaited<typeof read> | typeof DEADLINE;
667
+ try {
668
+ r = await Promise.race([read, deadline]);
669
+ } catch (err) {
670
+ finished = true;
671
+ clear();
672
+ throw err;
673
+ }
674
+ if (r === DEADLINE) {
675
+ finished = true;
676
+ truncated = true;
677
+ clear();
678
+ reader.cancel("openllm-byte-stream-deadline").catch(() => {});
679
+ controller.enqueue(terminator());
680
+ controller.close();
681
+ return;
682
+ }
683
+ if (r.done) {
684
+ finished = true;
685
+ clear();
686
+ controller.close();
687
+ return;
688
+ }
689
+ controller.enqueue(r.value);
690
+ },
691
+ cancel(reason) {
692
+ finished = true;
693
+ clear();
694
+ reader.cancel(reason).catch(() => {});
695
+ },
696
+ });
697
+
698
+ return { stream, isTruncated: () => truncated };
699
+ };