@openllmsh/sdk 0.0.0-stage → 2.10.12-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +43 -0
- package/LICENSE +202 -0
- package/README.md +142 -2
- package/assets/openllm-light.svg +18 -0
- package/assets/openllm.svg +18 -0
- package/client/chat.ts +140 -0
- package/client/devices.ts +269 -0
- package/client/http.ts +72 -0
- package/client/index.ts +129 -0
- package/client/relay-mux.ts +547 -0
- package/client/types.ts +51 -0
- package/client/vault.ts +239 -0
- package/connect/chat.ts +58 -0
- package/connect/client.ts +240 -0
- package/connect/config.ts +47 -0
- package/connect/errors.ts +121 -0
- package/connect/index.ts +23 -0
- package/connect/oauth.ts +265 -0
- package/connect/routes.ts +234 -0
- package/connect/store.ts +107 -0
- package/mcp/handle.ts +24 -0
- package/mcp/index.ts +76 -0
- package/mcp/server.ts +319 -0
- package/mcp/tools/marketplace.ts +439 -0
- package/mcp/tools/memory-client.ts +297 -0
- package/mcp/tools/memory-config.ts +32 -0
- package/mcp/tools/memory.ts +369 -0
- package/mcp/tools/model-priority.ts +88 -0
- package/mcp/tools/openllm.ts +883 -0
- package/mcp/types.ts +28 -0
- package/openapi/call.ts +90 -0
- package/openapi/index.ts +8 -0
- package/openapi/openapi.json +80213 -0
- package/openapi/operations.ts +2075 -0
- package/package.json +39 -4
- package/protocol/CONTRIBUTING.md +21 -0
- package/protocol/README.md +134 -0
- package/protocol/access.ts +186 -0
- package/protocol/agent-prompt-text.ts +32 -0
- package/protocol/agent.ts +991 -0
- package/protocol/api-key-syntax.ts +105 -0
- package/protocol/assets/openllm-light.svg +18 -0
- package/protocol/assets/openllm.svg +18 -0
- package/protocol/audio.ts +29 -0
- package/protocol/auth.ts +142 -0
- package/protocol/changelog.ts +155 -0
- package/protocol/chat.ts +458 -0
- package/protocol/client-registry.ts +356 -0
- package/protocol/config.ts +345 -0
- package/protocol/control-channel.ts +115 -0
- package/protocol/cooldown-reason.ts +78 -0
- package/protocol/cost.ts +10 -0
- package/protocol/daemon-port.ts +31 -0
- package/protocol/daemon.ts +1524 -0
- package/protocol/default-chains.ts +66 -0
- package/protocol/device-state.ts +85 -0
- package/protocol/doctor-report-local.ts +443 -0
- package/protocol/doctor-report-policy.ts +119 -0
- package/protocol/doctor-report.ts +947 -0
- package/protocol/errors.ts +14 -0
- package/protocol/execution-selection.ts +141 -0
- package/protocol/hop-grant.ts +33 -0
- package/protocol/image-edit-parse.ts +263 -0
- package/protocol/images.ts +298 -0
- package/protocol/index.ts +74 -0
- package/protocol/inference-headers.ts +3 -0
- package/protocol/keepalive.ts +69 -0
- package/protocol/local-auth.ts +300 -0
- package/protocol/local-device-control.ts +133 -0
- package/protocol/local-provider.ts +288 -0
- package/protocol/local-session.ts +118 -0
- package/protocol/marketplace.ts +187 -0
- package/protocol/media-attempt.ts +81 -0
- package/protocol/media-bytes.ts +28 -0
- package/protocol/media-default.ts +357 -0
- package/protocol/media-dimensions.ts +132 -0
- package/protocol/media-input.ts +61 -0
- package/protocol/media-model.ts +142 -0
- package/protocol/media-option-issue.ts +90 -0
- package/protocol/media-types.ts +70 -0
- package/protocol/model-match.ts +1051 -0
- package/protocol/model-settings.ts +327 -0
- package/protocol/models.ts +767 -0
- package/protocol/mux.ts +722 -0
- package/protocol/plan-surface.ts +59 -0
- package/protocol/provider-models.ts +16 -0
- package/protocol/provider-status.ts +206 -0
- package/protocol/provider-tier.ts +67 -0
- package/protocol/provider-usage.ts +263 -0
- package/protocol/providers.ts +901 -0
- package/protocol/public-usage-index.ts +354 -0
- package/protocol/realtime-generated.ts +31 -0
- package/protocol/realtime.ts +324 -0
- package/protocol/registry-pointer.ts +118 -0
- package/protocol/relay.ts +702 -0
- package/protocol/replay-session.ts +11 -0
- package/protocol/responses.ts +280 -0
- package/protocol/semver.ts +182 -0
- package/protocol/setup-mode.ts +22 -0
- package/protocol/sha256-hex.ts +98 -0
- package/protocol/stats.ts +406 -0
- package/protocol/streaming.ts +19 -0
- package/protocol/subscription-daemon-floors.ts +33 -0
- package/protocol/subscription-provider-compat.ts +104 -0
- package/protocol/subscription-provider.ts +32 -0
- package/protocol/team-grant.ts +17 -0
- package/protocol/team-shareable.ts +61 -0
- package/protocol/transcription-encoding.ts +76 -0
- package/protocol/url-locality-dns.ts +37 -0
- package/protocol/url-locality.ts +288 -0
- package/protocol/usage-snapshots.ts +9 -0
- package/protocol/vendor-cli-install.ts +29 -0
- package/protocol/video-job-plan.ts +50 -0
- package/protocol/videos.ts +227 -0
- package/tsconfig.json +14 -0
- package/tunnel/ARCHITECTURE.md +111 -0
- package/tunnel/LICENSE +33 -0
- package/tunnel/README.md +45 -0
- package/tunnel/assets/openllm-light.svg +18 -0
- package/tunnel/assets/openllm.svg +18 -0
- package/tunnel/channel-envelope.ts +139 -0
- package/tunnel/codec.ts +191 -0
- package/tunnel/device-anchor.ts +229 -0
- package/tunnel/device-control.ts +118 -0
- package/tunnel/device-grant.ts +173 -0
- package/tunnel/fleet.ts +77 -0
- package/tunnel/index.ts +22 -0
- package/tunnel/mux.ts +580 -0
- package/tunnel/peer-link.ts +390 -0
- package/tunnel/rtc-auth.ts +415 -0
- package/tunnel/rtc-duplex.ts +187 -0
- package/tunnel/rtc-ice.ts +136 -0
- package/tunnel/secure.ts +1298 -0
- package/tunnel/stream-reset-error.ts +14 -0
- package/tunnel/streams.ts +1430 -0
- package/tunnel/team-grant.ts +302 -0
- package/tunnel/wire-bytes.ts +102 -0
- package/vault/ARCHITECTURE.md +470 -0
- package/vault/LICENSE +33 -0
- package/vault/api-key.ts +190 -0
- package/vault/argon2.ts +106 -0
- package/vault/cli-auth-seal.ts +157 -0
- package/vault/cred-seal.ts +146 -0
- package/vault/credential.ts +55 -0
- package/vault/crypto.ts +210 -0
- package/vault/daemon-attest.ts +229 -0
- package/vault/derive.ts +113 -0
- package/vault/device-access.ts +87 -0
- package/vault/device-anchor.ts +185 -0
- package/vault/device-control.ts +174 -0
- package/vault/handoff.ts +396 -0
- package/vault/index.ts +21 -0
- package/vault/passkey.ts +243 -0
- package/vault/phrase.ts +65 -0
- package/vault/pin.ts +153 -0
- package/vault/public-key-seal.ts +131 -0
- package/vault/seed.ts +272 -0
- package/vault/team.ts +355 -0
- package/vault/vault-item.ts +131 -0
- package/vault/vault-migrate-digest.ts +29 -0
- package/vault/wrapped.ts +77 -0
- package/wire/ARCHITECTURE.md +90 -0
- package/wire/CONTRIBUTING.md +19 -0
- package/wire/LICENSE +33 -0
- package/wire/README.md +51 -0
- package/wire/adapters/messages/anthropic-map.ts +37 -0
- package/wire/adapters/messages/index.ts +7 -0
- package/wire/adapters/messages/reasoning-from-items.ts +38 -0
- package/wire/adapters/messages/reasoning-signature.ts +243 -0
- package/wire/adapters/messages/request.ts +574 -0
- package/wire/adapters/messages/response.ts +215 -0
- package/wire/adapters/messages/streaming.ts +1027 -0
- package/wire/adapters/messages/usage.ts +43 -0
- package/wire/adapters/responses/index.ts +3 -0
- package/wire/adapters/responses/request.ts +395 -0
- package/wire/adapters/responses/response.ts +88 -0
- package/wire/adapters/responses/streaming.ts +344 -0
- package/wire/assets/openllm-light.svg +18 -0
- package/wire/assets/openllm.svg +18 -0
- package/wire/features/account-rank.ts +166 -0
- package/wire/features/compaction/compact-request.ts +862 -0
- package/wire/features/compaction/compaction-text.ts +43 -0
- package/wire/features/context-demote.ts +47 -0
- package/wire/features/context-overflow-strategy.ts +26 -0
- package/wire/features/context-skip.ts +170 -0
- package/wire/features/conversation-affinity.ts +144 -0
- package/wire/features/max-tokens-backfill.ts +141 -0
- package/wire/features/model-caps.ts +53 -0
- package/wire/features/model-settings.ts +24 -0
- package/wire/features/output-limit-clamp.ts +64 -0
- package/wire/features/quota-gate.ts +85 -0
- package/wire/index.ts +44 -0
- package/wire/lib/attempt-outcome.ts +43 -0
- package/wire/lib/canonical/content-part.ts +38 -0
- package/wire/lib/canonical/count-tokens-capability.ts +138 -0
- package/wire/lib/canonical/encoding-select.ts +82 -0
- package/wire/lib/canonical/image-signature.ts +202 -0
- package/wire/lib/canonical/image-sse.ts +216 -0
- package/wire/lib/canonical/message.ts +43 -0
- package/wire/lib/canonical/token-estimate.ts +266 -0
- package/wire/lib/encrypted-content.ts +66 -0
- package/wire/lib/error-class.ts +154 -0
- package/wire/lib/error-envelope.ts +56 -0
- package/wire/lib/forwarded-headers.ts +180 -0
- package/wire/lib/provider-error-rules.ts +48 -0
- package/wire/lib/refusal.ts +38 -0
- package/wire/lib/responses-apply-patch.ts +104 -0
- package/wire/lib/responses-namespace.ts +53 -0
- package/wire/lib/secret-scrub.ts +101 -0
- package/wire/lib/streaming/accumulate.ts +221 -0
- package/wire/lib/streaming/finish-reason.ts +33 -0
- package/wire/lib/streaming/peek.ts +154 -0
- package/wire/lib/streaming/provider-decode.ts +259 -0
- package/wire/lib/streaming/response-stream.ts +71 -0
- package/wire/lib/streaming/sse.ts +699 -0
- package/wire/lib/streaming/strip-tool-calls.ts +95 -0
- package/wire/lib/streaming/upstream-error.ts +58 -0
- package/wire/lib/strip-subagent-isolation.ts +284 -0
- package/wire/lib/thinking-signature.ts +156 -0
- package/wire/lib/token-ruler/bpe-core.ts +426 -0
- package/wire/lib/token-ruler/index.ts +2 -0
- package/wire/lib/token-ruler/ruler.ts +79 -0
- package/wire/lib/token-ruler/vocab/claude-ranks.ts +16 -0
- package/wire/lib/token-ruler/vocab/claude.ts +22 -0
- package/wire/lib/token-ruler/vocab/o200k-ranks.ts +18 -0
- package/wire/lib/token-ruler/vocab/o200k.ts +19 -0
- package/wire/lib/tool-name-map.ts +271 -0
- package/wire/lib/tool-schema.ts +353 -0
- package/wire/lib/usage-observer.ts +125 -0
- package/wire/providers/anthropic/adaptive-thinking.ts +291 -0
- package/wire/providers/anthropic/beta-headers.ts +165 -0
- package/wire/providers/anthropic/index.ts +10 -0
- package/wire/providers/anthropic/request.ts +607 -0
- package/wire/providers/anthropic/response.ts +155 -0
- package/wire/providers/anthropic/streaming.ts +315 -0
- package/wire/providers/anthropic/web-search-domains.ts +49 -0
- package/wire/providers/chatgpt/common.ts +136 -0
- package/wire/providers/chatgpt/index.ts +9 -0
- package/wire/providers/chatgpt/request.ts +959 -0
- package/wire/providers/chatgpt/streaming.ts +1007 -0
- package/wire/providers/chatgpt/web-search.ts +63 -0
- package/wire/providers/grok/web-search.ts +87 -0
- package/wire/providers/kimi/web-search.ts +145 -0
- package/wire/providers/upstream-deny.ts +355 -0
- package/wire/providers/upstream-request.ts +619 -0
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
TChatCompletionChunk,
|
|
3
|
+
TChatCompletionResponse,
|
|
4
|
+
TServerSearchCall,
|
|
5
|
+
TToolCall,
|
|
6
|
+
} from "@openllmsh/sdk/protocol";
|
|
7
|
+
|
|
8
|
+
import { plainTextFromReasoningItems } from "../../adapters/messages/reasoning-from-items";
|
|
9
|
+
import { normalizeToolCallFinishReason } from "./finish-reason";
|
|
10
|
+
import { IncompleteStreamError } from "./upstream-error";
|
|
11
|
+
|
|
12
|
+
const ZERO_USAGE: TChatCompletionResponse["usage"] = {
|
|
13
|
+
prompt_tokens: 0,
|
|
14
|
+
completion_tokens: 0,
|
|
15
|
+
total_tokens: 0,
|
|
16
|
+
};
|
|
17
|
+
|
|
18
|
+
// runtime-only: builder state while we walk a chunk stream. Tool calls
|
|
19
|
+
// arrive as `{index, id?, function:{name?, arguments?}}` deltas that
|
|
20
|
+
// must be reassembled into the final `tool_calls` array.
|
|
21
|
+
type TToolCallBuilder = {
|
|
22
|
+
id: string;
|
|
23
|
+
name: string;
|
|
24
|
+
arguments: string;
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Drain a stream of ChatCompletion chunks into a single non-streaming
|
|
29
|
+
* response. Used for providers that ONLY stream upstream (chatgpt.com
|
|
30
|
+
* `/backend-api/codex/responses`) so non-streaming callers still get a
|
|
31
|
+
* `chat.completion` JSON envelope.
|
|
32
|
+
*
|
|
33
|
+
* Concatenates `delta.content` into a single message string, rebuilds
|
|
34
|
+
* `tool_calls` from indexed deltas, and uses the final chunk's
|
|
35
|
+
* `finish_reason` / `usage` if present.
|
|
36
|
+
*/
|
|
37
|
+
export const accumulateChunksToResponse = async (
|
|
38
|
+
chunks: ReadableStream<TChatCompletionChunk>,
|
|
39
|
+
providerModelId: string,
|
|
40
|
+
): Promise<TChatCompletionResponse> => {
|
|
41
|
+
const reader = chunks.getReader();
|
|
42
|
+
let content = "";
|
|
43
|
+
let finishReason: TChatCompletionResponse["choices"][number]["finish_reason"] =
|
|
44
|
+
null;
|
|
45
|
+
let usage: TChatCompletionResponse["usage"] = ZERO_USAGE;
|
|
46
|
+
let sawUsage = false;
|
|
47
|
+
let id = "";
|
|
48
|
+
let created = 0;
|
|
49
|
+
let hasCreated = false;
|
|
50
|
+
let systemFingerprint: string | undefined;
|
|
51
|
+
let refusal = "";
|
|
52
|
+
let logprobsContent: unknown[] | undefined;
|
|
53
|
+
const toolCalls = new Map<number, TToolCallBuilder>();
|
|
54
|
+
let reasoningContent = "";
|
|
55
|
+
let reasoningItems: ReadonlyArray<unknown> | undefined;
|
|
56
|
+
// Provider-executed hosted searches (Codex `webSearch` items) — each delta
|
|
57
|
+
// carries the completed call(s); collect them all for the final message.
|
|
58
|
+
const serverSearchCalls: TServerSearchCall[] = [];
|
|
59
|
+
|
|
60
|
+
try {
|
|
61
|
+
for (;;) {
|
|
62
|
+
const { value, done } = await reader.read();
|
|
63
|
+
if (done) break;
|
|
64
|
+
if (value.id !== "") id = value.id;
|
|
65
|
+
if (Number.isFinite(value.created) && value.created >= 0) {
|
|
66
|
+
created = value.created;
|
|
67
|
+
hasCreated = true;
|
|
68
|
+
}
|
|
69
|
+
if (
|
|
70
|
+
typeof value.system_fingerprint === "string" &&
|
|
71
|
+
value.system_fingerprint.length > 0
|
|
72
|
+
) {
|
|
73
|
+
systemFingerprint = value.system_fingerprint;
|
|
74
|
+
}
|
|
75
|
+
const choice = value.choices[0];
|
|
76
|
+
if (choice === undefined) {
|
|
77
|
+
if (value.usage !== undefined && value.usage !== null) {
|
|
78
|
+
usage = value.usage;
|
|
79
|
+
sawUsage = true;
|
|
80
|
+
}
|
|
81
|
+
continue;
|
|
82
|
+
}
|
|
83
|
+
if (value.usage !== undefined && value.usage !== null) {
|
|
84
|
+
usage = value.usage;
|
|
85
|
+
sawUsage = true;
|
|
86
|
+
}
|
|
87
|
+
const delta = choice.delta;
|
|
88
|
+
if (delta !== undefined) {
|
|
89
|
+
if (typeof delta.content === "string") content += delta.content;
|
|
90
|
+
if (typeof delta.refusal === "string") refusal += delta.refusal;
|
|
91
|
+
if (typeof delta.reasoning_content === "string") {
|
|
92
|
+
reasoningContent += delta.reasoning_content;
|
|
93
|
+
}
|
|
94
|
+
if (
|
|
95
|
+
Array.isArray(delta.reasoning_items) &&
|
|
96
|
+
delta.reasoning_items.length > 0
|
|
97
|
+
) {
|
|
98
|
+
reasoningItems = delta.reasoning_items;
|
|
99
|
+
}
|
|
100
|
+
if (
|
|
101
|
+
Array.isArray(delta.server_search_calls) &&
|
|
102
|
+
delta.server_search_calls.length > 0
|
|
103
|
+
) {
|
|
104
|
+
serverSearchCalls.push(...delta.server_search_calls);
|
|
105
|
+
}
|
|
106
|
+
if (Array.isArray(delta.tool_calls)) {
|
|
107
|
+
for (const tc of delta.tool_calls) {
|
|
108
|
+
const existing = toolCalls.get(tc.index) ?? {
|
|
109
|
+
id: "",
|
|
110
|
+
name: "",
|
|
111
|
+
arguments: "",
|
|
112
|
+
};
|
|
113
|
+
const next: TToolCallBuilder = {
|
|
114
|
+
id: tc.id ?? existing.id,
|
|
115
|
+
name: tc.function?.name ?? existing.name,
|
|
116
|
+
arguments: existing.arguments + (tc.function?.arguments ?? ""),
|
|
117
|
+
};
|
|
118
|
+
toolCalls.set(tc.index, next);
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
if (
|
|
123
|
+
choice.logprobs !== null &&
|
|
124
|
+
typeof choice.logprobs === "object" &&
|
|
125
|
+
"content" in choice.logprobs &&
|
|
126
|
+
Array.isArray(choice.logprobs.content)
|
|
127
|
+
) {
|
|
128
|
+
logprobsContent ??= [];
|
|
129
|
+
logprobsContent.push(...choice.logprobs.content);
|
|
130
|
+
}
|
|
131
|
+
if (choice.finish_reason !== null && choice.finish_reason !== undefined) {
|
|
132
|
+
finishReason = choice.finish_reason;
|
|
133
|
+
}
|
|
134
|
+
if (value.usage !== undefined && value.usage !== null) {
|
|
135
|
+
usage = value.usage;
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
} catch (err) {
|
|
139
|
+
// A stream failure AFTER the terminal finish_reason chunk (a trailing
|
|
140
|
+
// upstream error event, a connection reset while draining the tail)
|
|
141
|
+
// cannot change the already-complete answer — salvage it instead of
|
|
142
|
+
// discarding the whole turn (issue #274). A failure BEFORE the terminal
|
|
143
|
+
// chunk still rejects: the answer is incomplete and the error is the
|
|
144
|
+
// only truthful outcome.
|
|
145
|
+
if (finishReason === null) {
|
|
146
|
+
throw new IncompleteStreamError(err, sawUsage ? usage : null);
|
|
147
|
+
}
|
|
148
|
+
// Same debug gate as provider-decode's dropped-chunk warning: keep the
|
|
149
|
+
// salvage observable without adding a logger dep to the pure wire layer.
|
|
150
|
+
if (process.env.OPENLLM_DEBUG_STREAM === "1") {
|
|
151
|
+
console.warn(
|
|
152
|
+
"[accumulateChunksToResponse] salvaged completed answer after trailing stream error:",
|
|
153
|
+
err instanceof Error ? err.message : String(err),
|
|
154
|
+
);
|
|
155
|
+
}
|
|
156
|
+
} finally {
|
|
157
|
+
// Release the underlying chunk source promptly when the caller
|
|
158
|
+
// bails (deadline cutoff, response abort, …) so the upstream
|
|
159
|
+
// `fetch` body doesn't stay open waiting for our next read.
|
|
160
|
+
reader.cancel().catch(() => {});
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
const finalToolCalls: TToolCall[] = [];
|
|
164
|
+
const indices = [...toolCalls.keys()].sort((a, b) => a - b);
|
|
165
|
+
for (const idx of indices) {
|
|
166
|
+
const tc = toolCalls.get(idx);
|
|
167
|
+
if (tc === undefined) continue;
|
|
168
|
+
finalToolCalls.push({
|
|
169
|
+
id: tc.id,
|
|
170
|
+
type: "function",
|
|
171
|
+
function: { name: tc.name, arguments: tc.arguments },
|
|
172
|
+
});
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
const effectiveFinishReason =
|
|
176
|
+
normalizeToolCallFinishReason({
|
|
177
|
+
message:
|
|
178
|
+
finalToolCalls.length > 0 ? { tool_calls: finalToolCalls } : undefined,
|
|
179
|
+
finish_reason: finishReason,
|
|
180
|
+
}) ?? "stop";
|
|
181
|
+
|
|
182
|
+
const reasoningFromItems = plainTextFromReasoningItems(reasoningItems);
|
|
183
|
+
/** OpenLLM: surface ref `Delta.reasoning_items` text as `reasoning_content` when deltas omitted. */
|
|
184
|
+
const effectiveReasoningContent =
|
|
185
|
+
reasoningContent.length > 0 ? reasoningContent : reasoningFromItems;
|
|
186
|
+
|
|
187
|
+
return {
|
|
188
|
+
id: id !== "" ? id : `chatcmpl-${crypto.randomUUID()}`,
|
|
189
|
+
object: "chat.completion",
|
|
190
|
+
created: hasCreated ? created : Math.floor(Date.now() / 1000),
|
|
191
|
+
model: providerModelId,
|
|
192
|
+
...(systemFingerprint !== undefined
|
|
193
|
+
? { system_fingerprint: systemFingerprint }
|
|
194
|
+
: {}),
|
|
195
|
+
choices: [
|
|
196
|
+
{
|
|
197
|
+
index: 0,
|
|
198
|
+
message: {
|
|
199
|
+
role: "assistant",
|
|
200
|
+
content,
|
|
201
|
+
...(refusal.length > 0 ? { refusal } : {}),
|
|
202
|
+
...(finalToolCalls.length > 0 ? { tool_calls: finalToolCalls } : {}),
|
|
203
|
+
...(effectiveReasoningContent.length > 0
|
|
204
|
+
? { reasoning_content: effectiveReasoningContent }
|
|
205
|
+
: {}),
|
|
206
|
+
...(reasoningItems !== undefined
|
|
207
|
+
? { reasoning_items: [...reasoningItems] }
|
|
208
|
+
: {}),
|
|
209
|
+
...(serverSearchCalls.length > 0
|
|
210
|
+
? { server_search_calls: serverSearchCalls }
|
|
211
|
+
: {}),
|
|
212
|
+
},
|
|
213
|
+
finish_reason: effectiveFinishReason,
|
|
214
|
+
...(logprobsContent !== undefined
|
|
215
|
+
? { logprobs: { content: logprobsContent } }
|
|
216
|
+
: {}),
|
|
217
|
+
},
|
|
218
|
+
],
|
|
219
|
+
usage,
|
|
220
|
+
};
|
|
221
|
+
};
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import type { TChatCompletionChunk } from "@openllmsh/sdk/protocol";
|
|
2
|
+
|
|
3
|
+
type TToolCallFinishReasonChoice = {
|
|
4
|
+
readonly finish_reason: TChatCompletionChunk["choices"][number]["finish_reason"];
|
|
5
|
+
/** Internal semantic signal for adapters that have already decoded tool use. */
|
|
6
|
+
readonly hasToolCalls?: boolean;
|
|
7
|
+
readonly message?: {
|
|
8
|
+
readonly tool_calls?: ReadonlyArray<unknown>;
|
|
9
|
+
};
|
|
10
|
+
readonly delta?: {
|
|
11
|
+
readonly tool_calls?: ReadonlyArray<unknown>;
|
|
12
|
+
};
|
|
13
|
+
};
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Correct providers that emit tool calls but terminate their response as a
|
|
17
|
+
* normal stop. Both Chat Completions responses and stream chunks can carry
|
|
18
|
+
* tool calls, so accept either shape. Empty external arrays do not represent
|
|
19
|
+
* a tool call; adapters with prior decoded state provide `hasToolCalls`.
|
|
20
|
+
*/
|
|
21
|
+
export const normalizeToolCallFinishReason = (
|
|
22
|
+
choice: TToolCallFinishReasonChoice,
|
|
23
|
+
): TChatCompletionChunk["choices"][number]["finish_reason"] => {
|
|
24
|
+
const hasToolCalls =
|
|
25
|
+
choice.hasToolCalls === true ||
|
|
26
|
+
(choice.message?.tool_calls?.length ?? 0) > 0 ||
|
|
27
|
+
(choice.delta?.tool_calls?.length ?? 0) > 0;
|
|
28
|
+
|
|
29
|
+
return hasToolCalls &&
|
|
30
|
+
(choice.finish_reason === null || choice.finish_reason === "stop")
|
|
31
|
+
? "tool_calls"
|
|
32
|
+
: choice.finish_reason;
|
|
33
|
+
};
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
import type { TChatCompletionChunk } from "@openllmsh/sdk/protocol";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Pre-commit first-event peek — shared by the daemon walker and the cloud
|
|
5
|
+
* dispatch chain (like the context ladder, ONE implementation so the two
|
|
6
|
+
* paths cannot drift). An in-stream rejection that precedes any output —
|
|
7
|
+
* the overflow incident shape: HTTP 200, then (possibly after
|
|
8
|
+
* housekeeping events) `error: Your input exceeds the context window` —
|
|
9
|
+
* must surface while the hop is still an UNCOMMITTED candidate so the
|
|
10
|
+
* chain can walk to the next hop, instead of failing inside an
|
|
11
|
+
* already-committed 200 stream.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
/** Outcome of peeking a decoded upstream stream's first meaningful event
|
|
15
|
+
* before committing the client response:
|
|
16
|
+
* - `chunks`: the first meaningful event was real output — the
|
|
17
|
+
* (replayable) chunk stream is committed to the client;
|
|
18
|
+
* - `refusal`: the first meaningful event was a STRUCTURED refusal
|
|
19
|
+
* (canonical `finish_reason: "content_filter"`) that preceded any real
|
|
20
|
+
* output — the (replayable) chunk stream is still returned so a FINAL
|
|
21
|
+
* hop can surface the authentic provider refusal, while a non-final hop
|
|
22
|
+
* can discard it and walk;
|
|
23
|
+
* - `error`: the rejection that arrived before any output — including a
|
|
24
|
+
* stream that closed without producing a single event. */
|
|
25
|
+
export type TPeekedChunks<T> =
|
|
26
|
+
| { readonly kind: "chunks"; readonly chunks: ReadableStream<T> }
|
|
27
|
+
| { readonly kind: "refusal"; readonly chunks: ReadableStream<T> }
|
|
28
|
+
| { readonly kind: "error"; readonly error: unknown };
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Does this decoded chunk carry actual model output (content, reasoning,
|
|
32
|
+
* tool calls, a finish, usage)? Housekeeping chunks — the role-only
|
|
33
|
+
* empty-content chunk decoded from `response.created` (chatgpt) or
|
|
34
|
+
* `message_start` (anthropic) — do NOT commit the response: an in-stream
|
|
35
|
+
* rejection can still follow them (`response.failed`), and committing on
|
|
36
|
+
* housekeeping would forfeit the pre-commit walk for exactly the overflow
|
|
37
|
+
* shape the peek exists to catch.
|
|
38
|
+
*/
|
|
39
|
+
export const isMeaningfulChunk = (chunk: TChatCompletionChunk): boolean => {
|
|
40
|
+
if (chunk.usage !== undefined && chunk.usage !== null) return true;
|
|
41
|
+
return chunk.choices.some((choice) => {
|
|
42
|
+
if (choice.finish_reason !== undefined && choice.finish_reason !== null) {
|
|
43
|
+
return true;
|
|
44
|
+
}
|
|
45
|
+
const d = choice.delta;
|
|
46
|
+
const nonEmpty = (s: string | null | undefined): boolean =>
|
|
47
|
+
typeof s === "string" && s.length > 0;
|
|
48
|
+
const nonEmptyArr = (
|
|
49
|
+
a: ReadonlyArray<unknown> | null | undefined,
|
|
50
|
+
): boolean => Array.isArray(a) && a.length > 0;
|
|
51
|
+
return (
|
|
52
|
+
nonEmpty(d.content) ||
|
|
53
|
+
nonEmpty(d.reasoning_content) ||
|
|
54
|
+
nonEmpty(d.refusal) ||
|
|
55
|
+
nonEmptyArr(d.tool_calls) ||
|
|
56
|
+
nonEmptyArr(d.reasoning_items) ||
|
|
57
|
+
nonEmptyArr(d.server_search_calls)
|
|
58
|
+
);
|
|
59
|
+
});
|
|
60
|
+
};
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Wait for the first MEANINGFUL event of a decoded upstream stream (see
|
|
64
|
+
* the module doc for why), then return a stream that replays what was
|
|
65
|
+
* buffered ahead of the live tail.
|
|
66
|
+
*
|
|
67
|
+
* The wait is UNBOUNDED by design — bounded only by the request's abort
|
|
68
|
+
* signal, which the upstream fetch carries (an aborted client errors the
|
|
69
|
+
* read). The rejection we're waiting for arrives after the vendor
|
|
70
|
+
* tokenizes the prompt, and a near-cap prompt (the exact overflow case)
|
|
71
|
+
* prefills for many seconds; any fixed deadline commits the response
|
|
72
|
+
* right before the rejection lands, stranding the client in a committed
|
|
73
|
+
* stream that then dies (the pre-fix stall). The client loses nothing
|
|
74
|
+
* from waiting: it would receive zero bytes before the first event
|
|
75
|
+
* regardless. (Same rule as ref/CLIProxyAPI's stream bootstrap: no
|
|
76
|
+
* timeouts after the upstream connection is established.)
|
|
77
|
+
*
|
|
78
|
+
* A stream that CLOSES before any event is an error by default (walkable
|
|
79
|
+
* — the upstream produced nothing), not an empty success. EXCEPT when the
|
|
80
|
+
* peeked stream is a lossy DECODED VIEW of a byte-verbatim passthrough
|
|
81
|
+
* (`opts.emptyStreamIsError: false`): there, zero decoded chunks can mean
|
|
82
|
+
* the decoder dropped frames the schema doesn't model — not an empty
|
|
83
|
+
* upstream — and the client's bytes ride the other tee branch untouched,
|
|
84
|
+
* so the peek must commit rather than 502 a healthy response. Buffered
|
|
85
|
+
* housekeeping chunks are replayed ahead of the live tail: no event is
|
|
86
|
+
* lost or duplicated, and a rejection that arrives after real output
|
|
87
|
+
* began still propagates mid-stream as before (committed = never
|
|
88
|
+
* re-dispatched).
|
|
89
|
+
*/
|
|
90
|
+
export const peekFirstChunk = async <T>(
|
|
91
|
+
source: ReadableStream<T>,
|
|
92
|
+
isMeaningful: (chunk: T) => boolean = () => true,
|
|
93
|
+
opts?: {
|
|
94
|
+
readonly emptyStreamIsError?: boolean;
|
|
95
|
+
/**
|
|
96
|
+
* Classifies the FIRST meaningful chunk as a structured refusal. When
|
|
97
|
+
* it returns true the peek resolves `kind: "refusal"` (still with the
|
|
98
|
+
* replayable stream) instead of `kind: "chunks"`. Evaluated only on
|
|
99
|
+
* the first meaningful event, so a refusal that arrives AFTER real
|
|
100
|
+
* output has already committed the response is never reported here —
|
|
101
|
+
* exactly the pre-output boundary the fallback walk needs. Refusal
|
|
102
|
+
* thus takes precedence over any usage the same terminal chunk
|
|
103
|
+
* carries, because we classify the chunk before treating it as
|
|
104
|
+
* committed output.
|
|
105
|
+
*/
|
|
106
|
+
readonly isRefusal?: (chunk: T) => boolean;
|
|
107
|
+
},
|
|
108
|
+
): Promise<TPeekedChunks<T>> => {
|
|
109
|
+
const reader = source.getReader();
|
|
110
|
+
const buffered: T[] = [];
|
|
111
|
+
const replayed = (): ReadableStream<T> =>
|
|
112
|
+
new ReadableStream<T>({
|
|
113
|
+
pull: async (controller) => {
|
|
114
|
+
const next = buffered.shift();
|
|
115
|
+
if (next !== undefined) {
|
|
116
|
+
controller.enqueue(next);
|
|
117
|
+
return;
|
|
118
|
+
}
|
|
119
|
+
const r = await reader.read();
|
|
120
|
+
if (r.done) {
|
|
121
|
+
controller.close();
|
|
122
|
+
return;
|
|
123
|
+
}
|
|
124
|
+
controller.enqueue(r.value);
|
|
125
|
+
},
|
|
126
|
+
cancel: (reason) => reader.cancel(reason),
|
|
127
|
+
});
|
|
128
|
+
try {
|
|
129
|
+
for (;;) {
|
|
130
|
+
const r = await reader.read();
|
|
131
|
+
if (r.done) {
|
|
132
|
+
if (buffered.length === 0 && opts?.emptyStreamIsError !== false) {
|
|
133
|
+
return {
|
|
134
|
+
kind: "error",
|
|
135
|
+
error: new Error(
|
|
136
|
+
"upstream closed the stream before producing any event",
|
|
137
|
+
),
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
// Housekeeping-only stream that closed cleanly: commit and replay
|
|
141
|
+
// what arrived rather than synthesizing a failure.
|
|
142
|
+
return { kind: "chunks", chunks: replayed() };
|
|
143
|
+
}
|
|
144
|
+
buffered.push(r.value);
|
|
145
|
+
if (isMeaningful(r.value)) {
|
|
146
|
+
return opts?.isRefusal?.(r.value) === true
|
|
147
|
+
? { kind: "refusal", chunks: replayed() }
|
|
148
|
+
: { kind: "chunks", chunks: replayed() };
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
} catch (error) {
|
|
152
|
+
return { kind: "error", error };
|
|
153
|
+
}
|
|
154
|
+
};
|
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Coreless upstream-SSE decode driver — the `@openllm/core`-free analogue
|
|
3
|
+
* of `core/lib/streaming/event-stream.ts`'s `providerEventStream`. The
|
|
4
|
+
* coreless daemon walker uses it to turn an upstream provider's raw SSE
|
|
5
|
+
* byte stream into canonical `TChatCompletionChunk`s, then re-encodes
|
|
6
|
+
* those to the client's wire.
|
|
7
|
+
*
|
|
8
|
+
* `core`'s version takes a whole `TChatProviderSpec`; this takes just the
|
|
9
|
+
* streaming pieces (`eventSchema` + `initialState` + `eventToChunk` +
|
|
10
|
+
* optional `isTerminalEvent`) so it has no dependency on `core`'s spec
|
|
11
|
+
* machinery — only `effect`'s `Schema` (for decode) + `@openllmsh/sdk/protocol`
|
|
12
|
+
* + the wire SSE primitives.
|
|
13
|
+
*/
|
|
14
|
+
import type { TChatCompletionChunk } from "@openllmsh/sdk/protocol";
|
|
15
|
+
import { Effect, Schema } from "effect";
|
|
16
|
+
import { encodeSseDone, encodeSseEvent, sseEventStream } from "./sse";
|
|
17
|
+
import { upstreamErrorFrom } from "./upstream-error";
|
|
18
|
+
|
|
19
|
+
export type TStreamDecodeSpec<TEvent, TState, TOpts> = {
|
|
20
|
+
readonly eventSchema: Schema.Schema<TEvent>;
|
|
21
|
+
readonly onEvent?: (event: unknown) => void;
|
|
22
|
+
readonly initialState: (options: TOpts) => TState;
|
|
23
|
+
readonly eventToChunk: (
|
|
24
|
+
event: TEvent,
|
|
25
|
+
state: TState,
|
|
26
|
+
options: TOpts,
|
|
27
|
+
) => TChatCompletionChunk | null;
|
|
28
|
+
/**
|
|
29
|
+
* Optional semantic-end predicate. When it returns true for a decoded
|
|
30
|
+
* event, the driver converts the event, closes the canonical stream,
|
|
31
|
+
* then cancels the source without awaiting transport EOF. Providers
|
|
32
|
+
* without this keep reading until EOF so usage trailers can still
|
|
33
|
+
* arrive (generic OpenAI-compat).
|
|
34
|
+
*/
|
|
35
|
+
readonly isTerminalEvent?: (event: TEvent) => boolean;
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Decode an upstream provider SSE stream into canonical chunks. Mirrors
|
|
40
|
+
* `core`'s `providerEventStream` exactly, including the DashScope
|
|
41
|
+
* `choices: []` recovery (harmless for non-OpenAI event schemas — it only
|
|
42
|
+
* runs on the already-failed decode path).
|
|
43
|
+
*
|
|
44
|
+
* Alibaba DashScope (and other OpenAI-compatible upstreams) OMIT the
|
|
45
|
+
* `choices` key entirely on the trailing `stream_options.include_usage`
|
|
46
|
+
* chunk — spec-correct providers send `choices: []`. The OpenAI chunk
|
|
47
|
+
* schema makes `choices` required, so that chunk fails decode. Since it
|
|
48
|
+
* is the ONLY chunk carrying token counts, a silent drop means zero
|
|
49
|
+
* usage for Alibaba (no live feed, no final total) while spec-correct
|
|
50
|
+
* providers work. Retry once with an empty `choices` filled in. (For
|
|
51
|
+
* non-OpenAI event schemas the extra key is ignored on decode, so this
|
|
52
|
+
* is provider-safe and only ever runs on the already-failed path.)
|
|
53
|
+
*
|
|
54
|
+
* A genuinely undecodable frame is still dropped so one odd chunk can't
|
|
55
|
+
* kill the stream — but surface it under a debug flag so the next
|
|
56
|
+
* provider divergence isn't invisible.
|
|
57
|
+
*/
|
|
58
|
+
export const decodeProviderEventStream = <TEvent, TState, TOpts>(
|
|
59
|
+
raw: ReadableStream<Uint8Array>,
|
|
60
|
+
spec: TStreamDecodeSpec<TEvent, TState, TOpts>,
|
|
61
|
+
options: TOpts,
|
|
62
|
+
): ReadableStream<TChatCompletionChunk> => {
|
|
63
|
+
const state = spec.initialState(options);
|
|
64
|
+
const events = sseEventStream(raw);
|
|
65
|
+
const decode = (input: unknown): TEvent =>
|
|
66
|
+
Effect.runSync(
|
|
67
|
+
Schema.decodeUnknownEffect(spec.eventSchema)(input) as Effect.Effect<
|
|
68
|
+
TEvent,
|
|
69
|
+
Schema.SchemaError,
|
|
70
|
+
never
|
|
71
|
+
>,
|
|
72
|
+
);
|
|
73
|
+
const reader = events.getReader();
|
|
74
|
+
let settled = false;
|
|
75
|
+
// After the first Schema-validated event, delta frames reuse JSON.parse
|
|
76
|
+
// only (unchecked). Terminal / non-delta frames keep the Schema path.
|
|
77
|
+
let schemaWarmed = false;
|
|
78
|
+
|
|
79
|
+
const isLikelyDeltaFrame = (value: unknown): boolean => {
|
|
80
|
+
if (value === null || typeof value !== "object") return false;
|
|
81
|
+
const record = value as Record<string, unknown>;
|
|
82
|
+
if (record.type === "content_block_delta") return true;
|
|
83
|
+
// OpenAI chunks need Schema when `choices` is absent — DashScope (and
|
|
84
|
+
// others) omit it on the trailing usage frame; Schema defaults `choices`
|
|
85
|
+
// to `[]`. Fast-path only when the key is already present.
|
|
86
|
+
return (
|
|
87
|
+
record.object === "chat.completion.chunk" && Array.isArray(record.choices)
|
|
88
|
+
);
|
|
89
|
+
};
|
|
90
|
+
|
|
91
|
+
const initiateCancel = (reason?: unknown): void => {
|
|
92
|
+
void reader.cancel(reason).catch(() => {});
|
|
93
|
+
};
|
|
94
|
+
|
|
95
|
+
const settleClose = (
|
|
96
|
+
controller: ReadableStreamDefaultController<TChatCompletionChunk>,
|
|
97
|
+
): void => {
|
|
98
|
+
if (settled) return;
|
|
99
|
+
settled = true;
|
|
100
|
+
controller.close();
|
|
101
|
+
initiateCancel("semantic-terminal");
|
|
102
|
+
};
|
|
103
|
+
|
|
104
|
+
const settleError = (
|
|
105
|
+
controller: ReadableStreamDefaultController<TChatCompletionChunk>,
|
|
106
|
+
err: unknown,
|
|
107
|
+
): void => {
|
|
108
|
+
if (settled) return;
|
|
109
|
+
settled = true;
|
|
110
|
+
controller.error(err);
|
|
111
|
+
initiateCancel(err);
|
|
112
|
+
};
|
|
113
|
+
|
|
114
|
+
return new ReadableStream<TChatCompletionChunk>({
|
|
115
|
+
async pull(controller) {
|
|
116
|
+
if (settled) {
|
|
117
|
+
controller.close();
|
|
118
|
+
return;
|
|
119
|
+
}
|
|
120
|
+
for (;;) {
|
|
121
|
+
try {
|
|
122
|
+
const read = await reader.read();
|
|
123
|
+
if (settled) return;
|
|
124
|
+
if (read.done) {
|
|
125
|
+
settled = true;
|
|
126
|
+
controller.close();
|
|
127
|
+
return;
|
|
128
|
+
}
|
|
129
|
+
const value = read.value;
|
|
130
|
+
if (value.kind !== "data") continue;
|
|
131
|
+
let parsedJson: unknown;
|
|
132
|
+
try {
|
|
133
|
+
parsedJson = JSON.parse(value.data);
|
|
134
|
+
} catch {
|
|
135
|
+
continue;
|
|
136
|
+
}
|
|
137
|
+
spec.onEvent?.(parsedJson);
|
|
138
|
+
let event: TEvent;
|
|
139
|
+
let usedFastPath = false;
|
|
140
|
+
try {
|
|
141
|
+
usedFastPath = schemaWarmed && isLikelyDeltaFrame(parsedJson);
|
|
142
|
+
if (usedFastPath) {
|
|
143
|
+
event = parsedJson as TEvent;
|
|
144
|
+
} else {
|
|
145
|
+
event = decode(parsedJson);
|
|
146
|
+
schemaWarmed = true;
|
|
147
|
+
}
|
|
148
|
+
} catch (firstErr) {
|
|
149
|
+
let recovered: TEvent | null = null;
|
|
150
|
+
if (
|
|
151
|
+
parsedJson !== null &&
|
|
152
|
+
typeof parsedJson === "object" &&
|
|
153
|
+
!("choices" in parsedJson)
|
|
154
|
+
) {
|
|
155
|
+
try {
|
|
156
|
+
recovered = decode({ ...parsedJson, choices: [] });
|
|
157
|
+
} catch {
|
|
158
|
+
recovered = null;
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
if (recovered === null) {
|
|
162
|
+
if (process.env.OPENLLM_DEBUG_STREAM === "1") {
|
|
163
|
+
console.warn(
|
|
164
|
+
"[decodeProviderEventStream] dropped undecodable SSE chunk:",
|
|
165
|
+
firstErr instanceof Error
|
|
166
|
+
? firstErr.message
|
|
167
|
+
: String(firstErr),
|
|
168
|
+
value.data.slice(0, 600),
|
|
169
|
+
);
|
|
170
|
+
}
|
|
171
|
+
continue;
|
|
172
|
+
}
|
|
173
|
+
event = recovered;
|
|
174
|
+
}
|
|
175
|
+
let chunk: TChatCompletionChunk | null = null;
|
|
176
|
+
try {
|
|
177
|
+
chunk = spec.eventToChunk(event, state, options);
|
|
178
|
+
} catch (err) {
|
|
179
|
+
// Fast-path skipped Schema; a malformed delta can throw in
|
|
180
|
+
// eventToChunk. Fall back to Schema decode (drop if that fails)
|
|
181
|
+
// so one bad frame cannot error the whole stream.
|
|
182
|
+
if (!usedFastPath) {
|
|
183
|
+
settleError(controller, err);
|
|
184
|
+
return;
|
|
185
|
+
}
|
|
186
|
+
try {
|
|
187
|
+
event = decode(parsedJson);
|
|
188
|
+
} catch {
|
|
189
|
+
continue;
|
|
190
|
+
}
|
|
191
|
+
try {
|
|
192
|
+
chunk = spec.eventToChunk(event, state, options);
|
|
193
|
+
} catch (fallbackErr) {
|
|
194
|
+
settleError(controller, fallbackErr);
|
|
195
|
+
return;
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
const terminal = spec.isTerminalEvent?.(event) === true;
|
|
199
|
+
if (chunk !== null && !settled) {
|
|
200
|
+
controller.enqueue(chunk);
|
|
201
|
+
}
|
|
202
|
+
if (terminal) {
|
|
203
|
+
settleClose(controller);
|
|
204
|
+
return;
|
|
205
|
+
}
|
|
206
|
+
if (chunk !== null) return;
|
|
207
|
+
} catch (err) {
|
|
208
|
+
if (settled) return;
|
|
209
|
+
settleError(controller, err);
|
|
210
|
+
return;
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
},
|
|
214
|
+
cancel(reason) {
|
|
215
|
+
settled = true;
|
|
216
|
+
initiateCancel(reason);
|
|
217
|
+
},
|
|
218
|
+
});
|
|
219
|
+
};
|
|
220
|
+
|
|
221
|
+
/**
|
|
222
|
+
* Encode canonical chunks as an OpenAI-wire SSE byte stream (the client
|
|
223
|
+
* encoder for the chat-completions surface). Verbatim port of `core`'s
|
|
224
|
+
* `chunksToSseBytes`; on a mid-stream upstream error it emits a final
|
|
225
|
+
* error frame + `[DONE]` so the client sees a clean failure.
|
|
226
|
+
*/
|
|
227
|
+
export const chunksToSseBytes = (
|
|
228
|
+
chunks: ReadableStream<TChatCompletionChunk>,
|
|
229
|
+
): ReadableStream<Uint8Array> => {
|
|
230
|
+
const reader = chunks.getReader();
|
|
231
|
+
let doneSent = false;
|
|
232
|
+
return new ReadableStream<Uint8Array>({
|
|
233
|
+
async pull(controller) {
|
|
234
|
+
try {
|
|
235
|
+
const { value, done } = await reader.read();
|
|
236
|
+
if (done) {
|
|
237
|
+
if (!doneSent) {
|
|
238
|
+
controller.enqueue(encodeSseDone());
|
|
239
|
+
doneSent = true;
|
|
240
|
+
}
|
|
241
|
+
controller.close();
|
|
242
|
+
return;
|
|
243
|
+
}
|
|
244
|
+
controller.enqueue(encodeSseEvent(value));
|
|
245
|
+
} catch (err) {
|
|
246
|
+
const { type, message } = upstreamErrorFrom(err);
|
|
247
|
+
controller.enqueue(encodeSseEvent({ error: { type, message } }));
|
|
248
|
+
if (!doneSent) {
|
|
249
|
+
controller.enqueue(encodeSseDone());
|
|
250
|
+
doneSent = true;
|
|
251
|
+
}
|
|
252
|
+
controller.close();
|
|
253
|
+
}
|
|
254
|
+
},
|
|
255
|
+
cancel(reason) {
|
|
256
|
+
reader.cancel(reason).catch(() => {});
|
|
257
|
+
},
|
|
258
|
+
});
|
|
259
|
+
};
|