@openllmsh/sdk 0.0.0-stage → 2.10.12-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (245) hide show
  1. package/ARCHITECTURE.md +43 -0
  2. package/LICENSE +202 -0
  3. package/README.md +142 -2
  4. package/assets/openllm-light.svg +18 -0
  5. package/assets/openllm.svg +18 -0
  6. package/client/chat.ts +140 -0
  7. package/client/devices.ts +269 -0
  8. package/client/http.ts +72 -0
  9. package/client/index.ts +129 -0
  10. package/client/relay-mux.ts +547 -0
  11. package/client/types.ts +51 -0
  12. package/client/vault.ts +239 -0
  13. package/connect/chat.ts +58 -0
  14. package/connect/client.ts +240 -0
  15. package/connect/config.ts +47 -0
  16. package/connect/errors.ts +121 -0
  17. package/connect/index.ts +23 -0
  18. package/connect/oauth.ts +265 -0
  19. package/connect/routes.ts +234 -0
  20. package/connect/store.ts +107 -0
  21. package/mcp/handle.ts +24 -0
  22. package/mcp/index.ts +76 -0
  23. package/mcp/server.ts +319 -0
  24. package/mcp/tools/marketplace.ts +439 -0
  25. package/mcp/tools/memory-client.ts +297 -0
  26. package/mcp/tools/memory-config.ts +32 -0
  27. package/mcp/tools/memory.ts +369 -0
  28. package/mcp/tools/model-priority.ts +88 -0
  29. package/mcp/tools/openllm.ts +883 -0
  30. package/mcp/types.ts +28 -0
  31. package/openapi/call.ts +90 -0
  32. package/openapi/index.ts +8 -0
  33. package/openapi/openapi.json +80213 -0
  34. package/openapi/operations.ts +2075 -0
  35. package/package.json +39 -4
  36. package/protocol/CONTRIBUTING.md +21 -0
  37. package/protocol/README.md +134 -0
  38. package/protocol/access.ts +186 -0
  39. package/protocol/agent-prompt-text.ts +32 -0
  40. package/protocol/agent.ts +991 -0
  41. package/protocol/api-key-syntax.ts +105 -0
  42. package/protocol/assets/openllm-light.svg +18 -0
  43. package/protocol/assets/openllm.svg +18 -0
  44. package/protocol/audio.ts +29 -0
  45. package/protocol/auth.ts +142 -0
  46. package/protocol/changelog.ts +155 -0
  47. package/protocol/chat.ts +458 -0
  48. package/protocol/client-registry.ts +356 -0
  49. package/protocol/config.ts +345 -0
  50. package/protocol/control-channel.ts +115 -0
  51. package/protocol/cooldown-reason.ts +78 -0
  52. package/protocol/cost.ts +10 -0
  53. package/protocol/daemon-port.ts +31 -0
  54. package/protocol/daemon.ts +1524 -0
  55. package/protocol/default-chains.ts +66 -0
  56. package/protocol/device-state.ts +85 -0
  57. package/protocol/doctor-report-local.ts +443 -0
  58. package/protocol/doctor-report-policy.ts +119 -0
  59. package/protocol/doctor-report.ts +947 -0
  60. package/protocol/errors.ts +14 -0
  61. package/protocol/execution-selection.ts +141 -0
  62. package/protocol/hop-grant.ts +33 -0
  63. package/protocol/image-edit-parse.ts +263 -0
  64. package/protocol/images.ts +298 -0
  65. package/protocol/index.ts +74 -0
  66. package/protocol/inference-headers.ts +3 -0
  67. package/protocol/keepalive.ts +69 -0
  68. package/protocol/local-auth.ts +300 -0
  69. package/protocol/local-device-control.ts +133 -0
  70. package/protocol/local-provider.ts +288 -0
  71. package/protocol/local-session.ts +118 -0
  72. package/protocol/marketplace.ts +187 -0
  73. package/protocol/media-attempt.ts +81 -0
  74. package/protocol/media-bytes.ts +28 -0
  75. package/protocol/media-default.ts +357 -0
  76. package/protocol/media-dimensions.ts +132 -0
  77. package/protocol/media-input.ts +61 -0
  78. package/protocol/media-model.ts +142 -0
  79. package/protocol/media-option-issue.ts +90 -0
  80. package/protocol/media-types.ts +70 -0
  81. package/protocol/model-match.ts +1051 -0
  82. package/protocol/model-settings.ts +327 -0
  83. package/protocol/models.ts +767 -0
  84. package/protocol/mux.ts +722 -0
  85. package/protocol/plan-surface.ts +59 -0
  86. package/protocol/provider-models.ts +16 -0
  87. package/protocol/provider-status.ts +206 -0
  88. package/protocol/provider-tier.ts +67 -0
  89. package/protocol/provider-usage.ts +263 -0
  90. package/protocol/providers.ts +901 -0
  91. package/protocol/public-usage-index.ts +354 -0
  92. package/protocol/realtime-generated.ts +31 -0
  93. package/protocol/realtime.ts +324 -0
  94. package/protocol/registry-pointer.ts +118 -0
  95. package/protocol/relay.ts +702 -0
  96. package/protocol/replay-session.ts +11 -0
  97. package/protocol/responses.ts +280 -0
  98. package/protocol/semver.ts +182 -0
  99. package/protocol/setup-mode.ts +22 -0
  100. package/protocol/sha256-hex.ts +98 -0
  101. package/protocol/stats.ts +406 -0
  102. package/protocol/streaming.ts +19 -0
  103. package/protocol/subscription-daemon-floors.ts +33 -0
  104. package/protocol/subscription-provider-compat.ts +104 -0
  105. package/protocol/subscription-provider.ts +32 -0
  106. package/protocol/team-grant.ts +17 -0
  107. package/protocol/team-shareable.ts +61 -0
  108. package/protocol/transcription-encoding.ts +76 -0
  109. package/protocol/url-locality-dns.ts +37 -0
  110. package/protocol/url-locality.ts +288 -0
  111. package/protocol/usage-snapshots.ts +9 -0
  112. package/protocol/vendor-cli-install.ts +29 -0
  113. package/protocol/video-job-plan.ts +50 -0
  114. package/protocol/videos.ts +227 -0
  115. package/tsconfig.json +14 -0
  116. package/tunnel/ARCHITECTURE.md +111 -0
  117. package/tunnel/LICENSE +33 -0
  118. package/tunnel/README.md +45 -0
  119. package/tunnel/assets/openllm-light.svg +18 -0
  120. package/tunnel/assets/openllm.svg +18 -0
  121. package/tunnel/channel-envelope.ts +139 -0
  122. package/tunnel/codec.ts +191 -0
  123. package/tunnel/device-anchor.ts +229 -0
  124. package/tunnel/device-control.ts +118 -0
  125. package/tunnel/device-grant.ts +173 -0
  126. package/tunnel/fleet.ts +77 -0
  127. package/tunnel/index.ts +22 -0
  128. package/tunnel/mux.ts +580 -0
  129. package/tunnel/peer-link.ts +390 -0
  130. package/tunnel/rtc-auth.ts +415 -0
  131. package/tunnel/rtc-duplex.ts +187 -0
  132. package/tunnel/rtc-ice.ts +136 -0
  133. package/tunnel/secure.ts +1298 -0
  134. package/tunnel/stream-reset-error.ts +14 -0
  135. package/tunnel/streams.ts +1430 -0
  136. package/tunnel/team-grant.ts +302 -0
  137. package/tunnel/wire-bytes.ts +102 -0
  138. package/vault/ARCHITECTURE.md +470 -0
  139. package/vault/LICENSE +33 -0
  140. package/vault/api-key.ts +190 -0
  141. package/vault/argon2.ts +106 -0
  142. package/vault/cli-auth-seal.ts +157 -0
  143. package/vault/cred-seal.ts +146 -0
  144. package/vault/credential.ts +55 -0
  145. package/vault/crypto.ts +210 -0
  146. package/vault/daemon-attest.ts +229 -0
  147. package/vault/derive.ts +113 -0
  148. package/vault/device-access.ts +87 -0
  149. package/vault/device-anchor.ts +185 -0
  150. package/vault/device-control.ts +174 -0
  151. package/vault/handoff.ts +396 -0
  152. package/vault/index.ts +21 -0
  153. package/vault/passkey.ts +243 -0
  154. package/vault/phrase.ts +65 -0
  155. package/vault/pin.ts +153 -0
  156. package/vault/public-key-seal.ts +131 -0
  157. package/vault/seed.ts +272 -0
  158. package/vault/team.ts +355 -0
  159. package/vault/vault-item.ts +131 -0
  160. package/vault/vault-migrate-digest.ts +29 -0
  161. package/vault/wrapped.ts +77 -0
  162. package/wire/ARCHITECTURE.md +90 -0
  163. package/wire/CONTRIBUTING.md +19 -0
  164. package/wire/LICENSE +33 -0
  165. package/wire/README.md +51 -0
  166. package/wire/adapters/messages/anthropic-map.ts +37 -0
  167. package/wire/adapters/messages/index.ts +7 -0
  168. package/wire/adapters/messages/reasoning-from-items.ts +38 -0
  169. package/wire/adapters/messages/reasoning-signature.ts +243 -0
  170. package/wire/adapters/messages/request.ts +574 -0
  171. package/wire/adapters/messages/response.ts +215 -0
  172. package/wire/adapters/messages/streaming.ts +1027 -0
  173. package/wire/adapters/messages/usage.ts +43 -0
  174. package/wire/adapters/responses/index.ts +3 -0
  175. package/wire/adapters/responses/request.ts +395 -0
  176. package/wire/adapters/responses/response.ts +88 -0
  177. package/wire/adapters/responses/streaming.ts +344 -0
  178. package/wire/assets/openllm-light.svg +18 -0
  179. package/wire/assets/openllm.svg +18 -0
  180. package/wire/features/account-rank.ts +166 -0
  181. package/wire/features/compaction/compact-request.ts +862 -0
  182. package/wire/features/compaction/compaction-text.ts +43 -0
  183. package/wire/features/context-demote.ts +47 -0
  184. package/wire/features/context-overflow-strategy.ts +26 -0
  185. package/wire/features/context-skip.ts +170 -0
  186. package/wire/features/conversation-affinity.ts +144 -0
  187. package/wire/features/max-tokens-backfill.ts +141 -0
  188. package/wire/features/model-caps.ts +53 -0
  189. package/wire/features/model-settings.ts +24 -0
  190. package/wire/features/output-limit-clamp.ts +64 -0
  191. package/wire/features/quota-gate.ts +85 -0
  192. package/wire/index.ts +44 -0
  193. package/wire/lib/attempt-outcome.ts +43 -0
  194. package/wire/lib/canonical/content-part.ts +38 -0
  195. package/wire/lib/canonical/count-tokens-capability.ts +138 -0
  196. package/wire/lib/canonical/encoding-select.ts +82 -0
  197. package/wire/lib/canonical/image-signature.ts +202 -0
  198. package/wire/lib/canonical/image-sse.ts +216 -0
  199. package/wire/lib/canonical/message.ts +43 -0
  200. package/wire/lib/canonical/token-estimate.ts +266 -0
  201. package/wire/lib/encrypted-content.ts +66 -0
  202. package/wire/lib/error-class.ts +154 -0
  203. package/wire/lib/error-envelope.ts +56 -0
  204. package/wire/lib/forwarded-headers.ts +180 -0
  205. package/wire/lib/provider-error-rules.ts +48 -0
  206. package/wire/lib/refusal.ts +38 -0
  207. package/wire/lib/responses-apply-patch.ts +104 -0
  208. package/wire/lib/responses-namespace.ts +53 -0
  209. package/wire/lib/secret-scrub.ts +101 -0
  210. package/wire/lib/streaming/accumulate.ts +221 -0
  211. package/wire/lib/streaming/finish-reason.ts +33 -0
  212. package/wire/lib/streaming/peek.ts +154 -0
  213. package/wire/lib/streaming/provider-decode.ts +259 -0
  214. package/wire/lib/streaming/response-stream.ts +71 -0
  215. package/wire/lib/streaming/sse.ts +699 -0
  216. package/wire/lib/streaming/strip-tool-calls.ts +95 -0
  217. package/wire/lib/streaming/upstream-error.ts +58 -0
  218. package/wire/lib/strip-subagent-isolation.ts +284 -0
  219. package/wire/lib/thinking-signature.ts +156 -0
  220. package/wire/lib/token-ruler/bpe-core.ts +426 -0
  221. package/wire/lib/token-ruler/index.ts +2 -0
  222. package/wire/lib/token-ruler/ruler.ts +79 -0
  223. package/wire/lib/token-ruler/vocab/claude-ranks.ts +16 -0
  224. package/wire/lib/token-ruler/vocab/claude.ts +22 -0
  225. package/wire/lib/token-ruler/vocab/o200k-ranks.ts +18 -0
  226. package/wire/lib/token-ruler/vocab/o200k.ts +19 -0
  227. package/wire/lib/tool-name-map.ts +271 -0
  228. package/wire/lib/tool-schema.ts +353 -0
  229. package/wire/lib/usage-observer.ts +125 -0
  230. package/wire/providers/anthropic/adaptive-thinking.ts +291 -0
  231. package/wire/providers/anthropic/beta-headers.ts +165 -0
  232. package/wire/providers/anthropic/index.ts +10 -0
  233. package/wire/providers/anthropic/request.ts +607 -0
  234. package/wire/providers/anthropic/response.ts +155 -0
  235. package/wire/providers/anthropic/streaming.ts +315 -0
  236. package/wire/providers/anthropic/web-search-domains.ts +49 -0
  237. package/wire/providers/chatgpt/common.ts +136 -0
  238. package/wire/providers/chatgpt/index.ts +9 -0
  239. package/wire/providers/chatgpt/request.ts +959 -0
  240. package/wire/providers/chatgpt/streaming.ts +1007 -0
  241. package/wire/providers/chatgpt/web-search.ts +63 -0
  242. package/wire/providers/grok/web-search.ts +87 -0
  243. package/wire/providers/kimi/web-search.ts +145 -0
  244. package/wire/providers/upstream-deny.ts +355 -0
  245. package/wire/providers/upstream-request.ts +619 -0
@@ -0,0 +1,221 @@
1
+ import type {
2
+ TChatCompletionChunk,
3
+ TChatCompletionResponse,
4
+ TServerSearchCall,
5
+ TToolCall,
6
+ } from "@openllmsh/sdk/protocol";
7
+
8
+ import { plainTextFromReasoningItems } from "../../adapters/messages/reasoning-from-items";
9
+ import { normalizeToolCallFinishReason } from "./finish-reason";
10
+ import { IncompleteStreamError } from "./upstream-error";
11
+
12
+ const ZERO_USAGE: TChatCompletionResponse["usage"] = {
13
+ prompt_tokens: 0,
14
+ completion_tokens: 0,
15
+ total_tokens: 0,
16
+ };
17
+
18
+ // runtime-only: builder state while we walk a chunk stream. Tool calls
19
+ // arrive as `{index, id?, function:{name?, arguments?}}` deltas that
20
+ // must be reassembled into the final `tool_calls` array.
21
+ type TToolCallBuilder = {
22
+ id: string;
23
+ name: string;
24
+ arguments: string;
25
+ };
26
+
27
+ /**
28
+ * Drain a stream of ChatCompletion chunks into a single non-streaming
29
+ * response. Used for providers that ONLY stream upstream (chatgpt.com
30
+ * `/backend-api/codex/responses`) so non-streaming callers still get a
31
+ * `chat.completion` JSON envelope.
32
+ *
33
+ * Concatenates `delta.content` into a single message string, rebuilds
34
+ * `tool_calls` from indexed deltas, and uses the final chunk's
35
+ * `finish_reason` / `usage` if present.
36
+ */
37
+ export const accumulateChunksToResponse = async (
38
+ chunks: ReadableStream<TChatCompletionChunk>,
39
+ providerModelId: string,
40
+ ): Promise<TChatCompletionResponse> => {
41
+ const reader = chunks.getReader();
42
+ let content = "";
43
+ let finishReason: TChatCompletionResponse["choices"][number]["finish_reason"] =
44
+ null;
45
+ let usage: TChatCompletionResponse["usage"] = ZERO_USAGE;
46
+ let sawUsage = false;
47
+ let id = "";
48
+ let created = 0;
49
+ let hasCreated = false;
50
+ let systemFingerprint: string | undefined;
51
+ let refusal = "";
52
+ let logprobsContent: unknown[] | undefined;
53
+ const toolCalls = new Map<number, TToolCallBuilder>();
54
+ let reasoningContent = "";
55
+ let reasoningItems: ReadonlyArray<unknown> | undefined;
56
+ // Provider-executed hosted searches (Codex `webSearch` items) — each delta
57
+ // carries the completed call(s); collect them all for the final message.
58
+ const serverSearchCalls: TServerSearchCall[] = [];
59
+
60
+ try {
61
+ for (;;) {
62
+ const { value, done } = await reader.read();
63
+ if (done) break;
64
+ if (value.id !== "") id = value.id;
65
+ if (Number.isFinite(value.created) && value.created >= 0) {
66
+ created = value.created;
67
+ hasCreated = true;
68
+ }
69
+ if (
70
+ typeof value.system_fingerprint === "string" &&
71
+ value.system_fingerprint.length > 0
72
+ ) {
73
+ systemFingerprint = value.system_fingerprint;
74
+ }
75
+ const choice = value.choices[0];
76
+ if (choice === undefined) {
77
+ if (value.usage !== undefined && value.usage !== null) {
78
+ usage = value.usage;
79
+ sawUsage = true;
80
+ }
81
+ continue;
82
+ }
83
+ if (value.usage !== undefined && value.usage !== null) {
84
+ usage = value.usage;
85
+ sawUsage = true;
86
+ }
87
+ const delta = choice.delta;
88
+ if (delta !== undefined) {
89
+ if (typeof delta.content === "string") content += delta.content;
90
+ if (typeof delta.refusal === "string") refusal += delta.refusal;
91
+ if (typeof delta.reasoning_content === "string") {
92
+ reasoningContent += delta.reasoning_content;
93
+ }
94
+ if (
95
+ Array.isArray(delta.reasoning_items) &&
96
+ delta.reasoning_items.length > 0
97
+ ) {
98
+ reasoningItems = delta.reasoning_items;
99
+ }
100
+ if (
101
+ Array.isArray(delta.server_search_calls) &&
102
+ delta.server_search_calls.length > 0
103
+ ) {
104
+ serverSearchCalls.push(...delta.server_search_calls);
105
+ }
106
+ if (Array.isArray(delta.tool_calls)) {
107
+ for (const tc of delta.tool_calls) {
108
+ const existing = toolCalls.get(tc.index) ?? {
109
+ id: "",
110
+ name: "",
111
+ arguments: "",
112
+ };
113
+ const next: TToolCallBuilder = {
114
+ id: tc.id ?? existing.id,
115
+ name: tc.function?.name ?? existing.name,
116
+ arguments: existing.arguments + (tc.function?.arguments ?? ""),
117
+ };
118
+ toolCalls.set(tc.index, next);
119
+ }
120
+ }
121
+ }
122
+ if (
123
+ choice.logprobs !== null &&
124
+ typeof choice.logprobs === "object" &&
125
+ "content" in choice.logprobs &&
126
+ Array.isArray(choice.logprobs.content)
127
+ ) {
128
+ logprobsContent ??= [];
129
+ logprobsContent.push(...choice.logprobs.content);
130
+ }
131
+ if (choice.finish_reason !== null && choice.finish_reason !== undefined) {
132
+ finishReason = choice.finish_reason;
133
+ }
134
+ if (value.usage !== undefined && value.usage !== null) {
135
+ usage = value.usage;
136
+ }
137
+ }
138
+ } catch (err) {
139
+ // A stream failure AFTER the terminal finish_reason chunk (a trailing
140
+ // upstream error event, a connection reset while draining the tail)
141
+ // cannot change the already-complete answer — salvage it instead of
142
+ // discarding the whole turn (issue #274). A failure BEFORE the terminal
143
+ // chunk still rejects: the answer is incomplete and the error is the
144
+ // only truthful outcome.
145
+ if (finishReason === null) {
146
+ throw new IncompleteStreamError(err, sawUsage ? usage : null);
147
+ }
148
+ // Same debug gate as provider-decode's dropped-chunk warning: keep the
149
+ // salvage observable without adding a logger dep to the pure wire layer.
150
+ if (process.env.OPENLLM_DEBUG_STREAM === "1") {
151
+ console.warn(
152
+ "[accumulateChunksToResponse] salvaged completed answer after trailing stream error:",
153
+ err instanceof Error ? err.message : String(err),
154
+ );
155
+ }
156
+ } finally {
157
+ // Release the underlying chunk source promptly when the caller
158
+ // bails (deadline cutoff, response abort, …) so the upstream
159
+ // `fetch` body doesn't stay open waiting for our next read.
160
+ reader.cancel().catch(() => {});
161
+ }
162
+
163
+ const finalToolCalls: TToolCall[] = [];
164
+ const indices = [...toolCalls.keys()].sort((a, b) => a - b);
165
+ for (const idx of indices) {
166
+ const tc = toolCalls.get(idx);
167
+ if (tc === undefined) continue;
168
+ finalToolCalls.push({
169
+ id: tc.id,
170
+ type: "function",
171
+ function: { name: tc.name, arguments: tc.arguments },
172
+ });
173
+ }
174
+
175
+ const effectiveFinishReason =
176
+ normalizeToolCallFinishReason({
177
+ message:
178
+ finalToolCalls.length > 0 ? { tool_calls: finalToolCalls } : undefined,
179
+ finish_reason: finishReason,
180
+ }) ?? "stop";
181
+
182
+ const reasoningFromItems = plainTextFromReasoningItems(reasoningItems);
183
+ /** OpenLLM: surface ref `Delta.reasoning_items` text as `reasoning_content` when deltas omitted. */
184
+ const effectiveReasoningContent =
185
+ reasoningContent.length > 0 ? reasoningContent : reasoningFromItems;
186
+
187
+ return {
188
+ id: id !== "" ? id : `chatcmpl-${crypto.randomUUID()}`,
189
+ object: "chat.completion",
190
+ created: hasCreated ? created : Math.floor(Date.now() / 1000),
191
+ model: providerModelId,
192
+ ...(systemFingerprint !== undefined
193
+ ? { system_fingerprint: systemFingerprint }
194
+ : {}),
195
+ choices: [
196
+ {
197
+ index: 0,
198
+ message: {
199
+ role: "assistant",
200
+ content,
201
+ ...(refusal.length > 0 ? { refusal } : {}),
202
+ ...(finalToolCalls.length > 0 ? { tool_calls: finalToolCalls } : {}),
203
+ ...(effectiveReasoningContent.length > 0
204
+ ? { reasoning_content: effectiveReasoningContent }
205
+ : {}),
206
+ ...(reasoningItems !== undefined
207
+ ? { reasoning_items: [...reasoningItems] }
208
+ : {}),
209
+ ...(serverSearchCalls.length > 0
210
+ ? { server_search_calls: serverSearchCalls }
211
+ : {}),
212
+ },
213
+ finish_reason: effectiveFinishReason,
214
+ ...(logprobsContent !== undefined
215
+ ? { logprobs: { content: logprobsContent } }
216
+ : {}),
217
+ },
218
+ ],
219
+ usage,
220
+ };
221
+ };
@@ -0,0 +1,33 @@
1
+ import type { TChatCompletionChunk } from "@openllmsh/sdk/protocol";
2
+
3
+ type TToolCallFinishReasonChoice = {
4
+ readonly finish_reason: TChatCompletionChunk["choices"][number]["finish_reason"];
5
+ /** Internal semantic signal for adapters that have already decoded tool use. */
6
+ readonly hasToolCalls?: boolean;
7
+ readonly message?: {
8
+ readonly tool_calls?: ReadonlyArray<unknown>;
9
+ };
10
+ readonly delta?: {
11
+ readonly tool_calls?: ReadonlyArray<unknown>;
12
+ };
13
+ };
14
+
15
+ /**
16
+ * Correct providers that emit tool calls but terminate their response as a
17
+ * normal stop. Both Chat Completions responses and stream chunks can carry
18
+ * tool calls, so accept either shape. Empty external arrays do not represent
19
+ * a tool call; adapters with prior decoded state provide `hasToolCalls`.
20
+ */
21
+ export const normalizeToolCallFinishReason = (
22
+ choice: TToolCallFinishReasonChoice,
23
+ ): TChatCompletionChunk["choices"][number]["finish_reason"] => {
24
+ const hasToolCalls =
25
+ choice.hasToolCalls === true ||
26
+ (choice.message?.tool_calls?.length ?? 0) > 0 ||
27
+ (choice.delta?.tool_calls?.length ?? 0) > 0;
28
+
29
+ return hasToolCalls &&
30
+ (choice.finish_reason === null || choice.finish_reason === "stop")
31
+ ? "tool_calls"
32
+ : choice.finish_reason;
33
+ };
@@ -0,0 +1,154 @@
1
+ import type { TChatCompletionChunk } from "@openllmsh/sdk/protocol";
2
+
3
+ /**
4
+ * Pre-commit first-event peek — shared by the daemon walker and the cloud
5
+ * dispatch chain (like the context ladder, ONE implementation so the two
6
+ * paths cannot drift). An in-stream rejection that precedes any output —
7
+ * the overflow incident shape: HTTP 200, then (possibly after
8
+ * housekeeping events) `error: Your input exceeds the context window` —
9
+ * must surface while the hop is still an UNCOMMITTED candidate so the
10
+ * chain can walk to the next hop, instead of failing inside an
11
+ * already-committed 200 stream.
12
+ */
13
+
14
+ /** Outcome of peeking a decoded upstream stream's first meaningful event
15
+ * before committing the client response:
16
+ * - `chunks`: the first meaningful event was real output — the
17
+ * (replayable) chunk stream is committed to the client;
18
+ * - `refusal`: the first meaningful event was a STRUCTURED refusal
19
+ * (canonical `finish_reason: "content_filter"`) that preceded any real
20
+ * output — the (replayable) chunk stream is still returned so a FINAL
21
+ * hop can surface the authentic provider refusal, while a non-final hop
22
+ * can discard it and walk;
23
+ * - `error`: the rejection that arrived before any output — including a
24
+ * stream that closed without producing a single event. */
25
+ export type TPeekedChunks<T> =
26
+ | { readonly kind: "chunks"; readonly chunks: ReadableStream<T> }
27
+ | { readonly kind: "refusal"; readonly chunks: ReadableStream<T> }
28
+ | { readonly kind: "error"; readonly error: unknown };
29
+
30
+ /**
31
+ * Does this decoded chunk carry actual model output (content, reasoning,
32
+ * tool calls, a finish, usage)? Housekeeping chunks — the role-only
33
+ * empty-content chunk decoded from `response.created` (chatgpt) or
34
+ * `message_start` (anthropic) — do NOT commit the response: an in-stream
35
+ * rejection can still follow them (`response.failed`), and committing on
36
+ * housekeeping would forfeit the pre-commit walk for exactly the overflow
37
+ * shape the peek exists to catch.
38
+ */
39
+ export const isMeaningfulChunk = (chunk: TChatCompletionChunk): boolean => {
40
+ if (chunk.usage !== undefined && chunk.usage !== null) return true;
41
+ return chunk.choices.some((choice) => {
42
+ if (choice.finish_reason !== undefined && choice.finish_reason !== null) {
43
+ return true;
44
+ }
45
+ const d = choice.delta;
46
+ const nonEmpty = (s: string | null | undefined): boolean =>
47
+ typeof s === "string" && s.length > 0;
48
+ const nonEmptyArr = (
49
+ a: ReadonlyArray<unknown> | null | undefined,
50
+ ): boolean => Array.isArray(a) && a.length > 0;
51
+ return (
52
+ nonEmpty(d.content) ||
53
+ nonEmpty(d.reasoning_content) ||
54
+ nonEmpty(d.refusal) ||
55
+ nonEmptyArr(d.tool_calls) ||
56
+ nonEmptyArr(d.reasoning_items) ||
57
+ nonEmptyArr(d.server_search_calls)
58
+ );
59
+ });
60
+ };
61
+
62
+ /**
63
+ * Wait for the first MEANINGFUL event of a decoded upstream stream (see
64
+ * the module doc for why), then return a stream that replays what was
65
+ * buffered ahead of the live tail.
66
+ *
67
+ * The wait is UNBOUNDED by design — bounded only by the request's abort
68
+ * signal, which the upstream fetch carries (an aborted client errors the
69
+ * read). The rejection we're waiting for arrives after the vendor
70
+ * tokenizes the prompt, and a near-cap prompt (the exact overflow case)
71
+ * prefills for many seconds; any fixed deadline commits the response
72
+ * right before the rejection lands, stranding the client in a committed
73
+ * stream that then dies (the pre-fix stall). The client loses nothing
74
+ * from waiting: it would receive zero bytes before the first event
75
+ * regardless. (Same rule as ref/CLIProxyAPI's stream bootstrap: no
76
+ * timeouts after the upstream connection is established.)
77
+ *
78
+ * A stream that CLOSES before any event is an error by default (walkable
79
+ * — the upstream produced nothing), not an empty success. EXCEPT when the
80
+ * peeked stream is a lossy DECODED VIEW of a byte-verbatim passthrough
81
+ * (`opts.emptyStreamIsError: false`): there, zero decoded chunks can mean
82
+ * the decoder dropped frames the schema doesn't model — not an empty
83
+ * upstream — and the client's bytes ride the other tee branch untouched,
84
+ * so the peek must commit rather than 502 a healthy response. Buffered
85
+ * housekeeping chunks are replayed ahead of the live tail: no event is
86
+ * lost or duplicated, and a rejection that arrives after real output
87
+ * began still propagates mid-stream as before (committed = never
88
+ * re-dispatched).
89
+ */
90
+ export const peekFirstChunk = async <T>(
91
+ source: ReadableStream<T>,
92
+ isMeaningful: (chunk: T) => boolean = () => true,
93
+ opts?: {
94
+ readonly emptyStreamIsError?: boolean;
95
+ /**
96
+ * Classifies the FIRST meaningful chunk as a structured refusal. When
97
+ * it returns true the peek resolves `kind: "refusal"` (still with the
98
+ * replayable stream) instead of `kind: "chunks"`. Evaluated only on
99
+ * the first meaningful event, so a refusal that arrives AFTER real
100
+ * output has already committed the response is never reported here —
101
+ * exactly the pre-output boundary the fallback walk needs. Refusal
102
+ * thus takes precedence over any usage the same terminal chunk
103
+ * carries, because we classify the chunk before treating it as
104
+ * committed output.
105
+ */
106
+ readonly isRefusal?: (chunk: T) => boolean;
107
+ },
108
+ ): Promise<TPeekedChunks<T>> => {
109
+ const reader = source.getReader();
110
+ const buffered: T[] = [];
111
+ const replayed = (): ReadableStream<T> =>
112
+ new ReadableStream<T>({
113
+ pull: async (controller) => {
114
+ const next = buffered.shift();
115
+ if (next !== undefined) {
116
+ controller.enqueue(next);
117
+ return;
118
+ }
119
+ const r = await reader.read();
120
+ if (r.done) {
121
+ controller.close();
122
+ return;
123
+ }
124
+ controller.enqueue(r.value);
125
+ },
126
+ cancel: (reason) => reader.cancel(reason),
127
+ });
128
+ try {
129
+ for (;;) {
130
+ const r = await reader.read();
131
+ if (r.done) {
132
+ if (buffered.length === 0 && opts?.emptyStreamIsError !== false) {
133
+ return {
134
+ kind: "error",
135
+ error: new Error(
136
+ "upstream closed the stream before producing any event",
137
+ ),
138
+ };
139
+ }
140
+ // Housekeeping-only stream that closed cleanly: commit and replay
141
+ // what arrived rather than synthesizing a failure.
142
+ return { kind: "chunks", chunks: replayed() };
143
+ }
144
+ buffered.push(r.value);
145
+ if (isMeaningful(r.value)) {
146
+ return opts?.isRefusal?.(r.value) === true
147
+ ? { kind: "refusal", chunks: replayed() }
148
+ : { kind: "chunks", chunks: replayed() };
149
+ }
150
+ }
151
+ } catch (error) {
152
+ return { kind: "error", error };
153
+ }
154
+ };
@@ -0,0 +1,259 @@
1
+ /**
2
+ * Coreless upstream-SSE decode driver — the `@openllm/core`-free analogue
3
+ * of `core/lib/streaming/event-stream.ts`'s `providerEventStream`. The
4
+ * coreless daemon walker uses it to turn an upstream provider's raw SSE
5
+ * byte stream into canonical `TChatCompletionChunk`s, then re-encodes
6
+ * those to the client's wire.
7
+ *
8
+ * `core`'s version takes a whole `TChatProviderSpec`; this takes just the
9
+ * streaming pieces (`eventSchema` + `initialState` + `eventToChunk` +
10
+ * optional `isTerminalEvent`) so it has no dependency on `core`'s spec
11
+ * machinery — only `effect`'s `Schema` (for decode) + `@openllmsh/sdk/protocol`
12
+ * + the wire SSE primitives.
13
+ */
14
+ import type { TChatCompletionChunk } from "@openllmsh/sdk/protocol";
15
+ import { Effect, Schema } from "effect";
16
+ import { encodeSseDone, encodeSseEvent, sseEventStream } from "./sse";
17
+ import { upstreamErrorFrom } from "./upstream-error";
18
+
19
+ export type TStreamDecodeSpec<TEvent, TState, TOpts> = {
20
+ readonly eventSchema: Schema.Schema<TEvent>;
21
+ readonly onEvent?: (event: unknown) => void;
22
+ readonly initialState: (options: TOpts) => TState;
23
+ readonly eventToChunk: (
24
+ event: TEvent,
25
+ state: TState,
26
+ options: TOpts,
27
+ ) => TChatCompletionChunk | null;
28
+ /**
29
+ * Optional semantic-end predicate. When it returns true for a decoded
30
+ * event, the driver converts the event, closes the canonical stream,
31
+ * then cancels the source without awaiting transport EOF. Providers
32
+ * without this keep reading until EOF so usage trailers can still
33
+ * arrive (generic OpenAI-compat).
34
+ */
35
+ readonly isTerminalEvent?: (event: TEvent) => boolean;
36
+ };
37
+
38
+ /**
39
+ * Decode an upstream provider SSE stream into canonical chunks. Mirrors
40
+ * `core`'s `providerEventStream` exactly, including the DashScope
41
+ * `choices: []` recovery (harmless for non-OpenAI event schemas — it only
42
+ * runs on the already-failed decode path).
43
+ *
44
+ * Alibaba DashScope (and other OpenAI-compatible upstreams) OMIT the
45
+ * `choices` key entirely on the trailing `stream_options.include_usage`
46
+ * chunk — spec-correct providers send `choices: []`. The OpenAI chunk
47
+ * schema makes `choices` required, so that chunk fails decode. Since it
48
+ * is the ONLY chunk carrying token counts, a silent drop means zero
49
+ * usage for Alibaba (no live feed, no final total) while spec-correct
50
+ * providers work. Retry once with an empty `choices` filled in. (For
51
+ * non-OpenAI event schemas the extra key is ignored on decode, so this
52
+ * is provider-safe and only ever runs on the already-failed path.)
53
+ *
54
+ * A genuinely undecodable frame is still dropped so one odd chunk can't
55
+ * kill the stream — but surface it under a debug flag so the next
56
+ * provider divergence isn't invisible.
57
+ */
58
+ export const decodeProviderEventStream = <TEvent, TState, TOpts>(
59
+ raw: ReadableStream<Uint8Array>,
60
+ spec: TStreamDecodeSpec<TEvent, TState, TOpts>,
61
+ options: TOpts,
62
+ ): ReadableStream<TChatCompletionChunk> => {
63
+ const state = spec.initialState(options);
64
+ const events = sseEventStream(raw);
65
+ const decode = (input: unknown): TEvent =>
66
+ Effect.runSync(
67
+ Schema.decodeUnknownEffect(spec.eventSchema)(input) as Effect.Effect<
68
+ TEvent,
69
+ Schema.SchemaError,
70
+ never
71
+ >,
72
+ );
73
+ const reader = events.getReader();
74
+ let settled = false;
75
+ // After the first Schema-validated event, delta frames reuse JSON.parse
76
+ // only (unchecked). Terminal / non-delta frames keep the Schema path.
77
+ let schemaWarmed = false;
78
+
79
+ const isLikelyDeltaFrame = (value: unknown): boolean => {
80
+ if (value === null || typeof value !== "object") return false;
81
+ const record = value as Record<string, unknown>;
82
+ if (record.type === "content_block_delta") return true;
83
+ // OpenAI chunks need Schema when `choices` is absent — DashScope (and
84
+ // others) omit it on the trailing usage frame; Schema defaults `choices`
85
+ // to `[]`. Fast-path only when the key is already present.
86
+ return (
87
+ record.object === "chat.completion.chunk" && Array.isArray(record.choices)
88
+ );
89
+ };
90
+
91
+ const initiateCancel = (reason?: unknown): void => {
92
+ void reader.cancel(reason).catch(() => {});
93
+ };
94
+
95
+ const settleClose = (
96
+ controller: ReadableStreamDefaultController<TChatCompletionChunk>,
97
+ ): void => {
98
+ if (settled) return;
99
+ settled = true;
100
+ controller.close();
101
+ initiateCancel("semantic-terminal");
102
+ };
103
+
104
+ const settleError = (
105
+ controller: ReadableStreamDefaultController<TChatCompletionChunk>,
106
+ err: unknown,
107
+ ): void => {
108
+ if (settled) return;
109
+ settled = true;
110
+ controller.error(err);
111
+ initiateCancel(err);
112
+ };
113
+
114
+ return new ReadableStream<TChatCompletionChunk>({
115
+ async pull(controller) {
116
+ if (settled) {
117
+ controller.close();
118
+ return;
119
+ }
120
+ for (;;) {
121
+ try {
122
+ const read = await reader.read();
123
+ if (settled) return;
124
+ if (read.done) {
125
+ settled = true;
126
+ controller.close();
127
+ return;
128
+ }
129
+ const value = read.value;
130
+ if (value.kind !== "data") continue;
131
+ let parsedJson: unknown;
132
+ try {
133
+ parsedJson = JSON.parse(value.data);
134
+ } catch {
135
+ continue;
136
+ }
137
+ spec.onEvent?.(parsedJson);
138
+ let event: TEvent;
139
+ let usedFastPath = false;
140
+ try {
141
+ usedFastPath = schemaWarmed && isLikelyDeltaFrame(parsedJson);
142
+ if (usedFastPath) {
143
+ event = parsedJson as TEvent;
144
+ } else {
145
+ event = decode(parsedJson);
146
+ schemaWarmed = true;
147
+ }
148
+ } catch (firstErr) {
149
+ let recovered: TEvent | null = null;
150
+ if (
151
+ parsedJson !== null &&
152
+ typeof parsedJson === "object" &&
153
+ !("choices" in parsedJson)
154
+ ) {
155
+ try {
156
+ recovered = decode({ ...parsedJson, choices: [] });
157
+ } catch {
158
+ recovered = null;
159
+ }
160
+ }
161
+ if (recovered === null) {
162
+ if (process.env.OPENLLM_DEBUG_STREAM === "1") {
163
+ console.warn(
164
+ "[decodeProviderEventStream] dropped undecodable SSE chunk:",
165
+ firstErr instanceof Error
166
+ ? firstErr.message
167
+ : String(firstErr),
168
+ value.data.slice(0, 600),
169
+ );
170
+ }
171
+ continue;
172
+ }
173
+ event = recovered;
174
+ }
175
+ let chunk: TChatCompletionChunk | null = null;
176
+ try {
177
+ chunk = spec.eventToChunk(event, state, options);
178
+ } catch (err) {
179
+ // Fast-path skipped Schema; a malformed delta can throw in
180
+ // eventToChunk. Fall back to Schema decode (drop if that fails)
181
+ // so one bad frame cannot error the whole stream.
182
+ if (!usedFastPath) {
183
+ settleError(controller, err);
184
+ return;
185
+ }
186
+ try {
187
+ event = decode(parsedJson);
188
+ } catch {
189
+ continue;
190
+ }
191
+ try {
192
+ chunk = spec.eventToChunk(event, state, options);
193
+ } catch (fallbackErr) {
194
+ settleError(controller, fallbackErr);
195
+ return;
196
+ }
197
+ }
198
+ const terminal = spec.isTerminalEvent?.(event) === true;
199
+ if (chunk !== null && !settled) {
200
+ controller.enqueue(chunk);
201
+ }
202
+ if (terminal) {
203
+ settleClose(controller);
204
+ return;
205
+ }
206
+ if (chunk !== null) return;
207
+ } catch (err) {
208
+ if (settled) return;
209
+ settleError(controller, err);
210
+ return;
211
+ }
212
+ }
213
+ },
214
+ cancel(reason) {
215
+ settled = true;
216
+ initiateCancel(reason);
217
+ },
218
+ });
219
+ };
220
+
221
+ /**
222
+ * Encode canonical chunks as an OpenAI-wire SSE byte stream (the client
223
+ * encoder for the chat-completions surface). Verbatim port of `core`'s
224
+ * `chunksToSseBytes`; on a mid-stream upstream error it emits a final
225
+ * error frame + `[DONE]` so the client sees a clean failure.
226
+ */
227
+ export const chunksToSseBytes = (
228
+ chunks: ReadableStream<TChatCompletionChunk>,
229
+ ): ReadableStream<Uint8Array> => {
230
+ const reader = chunks.getReader();
231
+ let doneSent = false;
232
+ return new ReadableStream<Uint8Array>({
233
+ async pull(controller) {
234
+ try {
235
+ const { value, done } = await reader.read();
236
+ if (done) {
237
+ if (!doneSent) {
238
+ controller.enqueue(encodeSseDone());
239
+ doneSent = true;
240
+ }
241
+ controller.close();
242
+ return;
243
+ }
244
+ controller.enqueue(encodeSseEvent(value));
245
+ } catch (err) {
246
+ const { type, message } = upstreamErrorFrom(err);
247
+ controller.enqueue(encodeSseEvent({ error: { type, message } }));
248
+ if (!doneSent) {
249
+ controller.enqueue(encodeSseDone());
250
+ doneSent = true;
251
+ }
252
+ controller.close();
253
+ }
254
+ },
255
+ cancel(reason) {
256
+ reader.cancel(reason).catch(() => {});
257
+ },
258
+ });
259
+ };