@frockbot/provider-openai-compatible 0.0.0 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +23 -6
- package/src/index.test.ts +361 -0
- package/src/index.ts +396 -0
- package/tsconfig.json +13 -0
- package/README.md +0 -3
package/package.json
CHANGED
|
@@ -1,14 +1,31 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@frockbot/provider-openai-compatible",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"private": false,
|
|
5
|
+
"type": "module",
|
|
6
|
+
"exports": {
|
|
7
|
+
".": "./src/index.ts"
|
|
8
|
+
},
|
|
9
|
+
"scripts": {
|
|
10
|
+
"test": "bun test src",
|
|
11
|
+
"typecheck": "tsc --noEmit -p tsconfig.json"
|
|
12
|
+
},
|
|
13
|
+
"dependencies": {
|
|
14
|
+
"@frockbot/kernel-contracts": "0.1.0",
|
|
15
|
+
"@frockbot/plugin-models": "0.1.0",
|
|
16
|
+
"cordis": "4.0.0-rc.8"
|
|
17
|
+
},
|
|
18
|
+
"devDependencies": {
|
|
19
|
+
"@types/bun": "1.4.0",
|
|
20
|
+
"@types/node": "26.2.0",
|
|
21
|
+
"typescript": "^7.0.2"
|
|
22
|
+
},
|
|
23
|
+
"publishConfig": {
|
|
24
|
+
"access": "public"
|
|
25
|
+
},
|
|
6
26
|
"repository": {
|
|
7
27
|
"type": "git",
|
|
8
28
|
"url": "git+https://github.com/timoconnellaus/frockbot.git",
|
|
9
29
|
"directory": "packages/provider-openai-compatible"
|
|
10
|
-
},
|
|
11
|
-
"publishConfig": {
|
|
12
|
-
"access": "public"
|
|
13
30
|
}
|
|
14
31
|
}
|
|
@@ -0,0 +1,361 @@
|
|
|
1
|
+
import { describe, expect, test } from "bun:test";
|
|
2
|
+
import { type NormalizedModelRequest } from "@frockbot/kernel-contracts";
|
|
3
|
+
import { LlmRegistry } from "@frockbot/plugin-models";
|
|
4
|
+
import { Context } from "cordis";
|
|
5
|
+
import { OpenAICompatibleProvider, requestToWire } from "./index.js";
|
|
6
|
+
|
|
7
|
+
const request: NormalizedModelRequest = {
|
|
8
|
+
requestId: "request-1",
|
|
9
|
+
provider: "openai-compatible",
|
|
10
|
+
model: "test-model",
|
|
11
|
+
system: "Be useful.",
|
|
12
|
+
messages: [
|
|
13
|
+
{ role: "user", content: "What time is it?" },
|
|
14
|
+
{
|
|
15
|
+
role: "assistant",
|
|
16
|
+
content: "",
|
|
17
|
+
toolCalls: [{ id: "previous", name: "lookup", input: { query: "time" } }],
|
|
18
|
+
},
|
|
19
|
+
{
|
|
20
|
+
role: "tool",
|
|
21
|
+
callId: "previous",
|
|
22
|
+
name: "lookup",
|
|
23
|
+
content: "noon",
|
|
24
|
+
isError: false,
|
|
25
|
+
},
|
|
26
|
+
],
|
|
27
|
+
tools: [
|
|
28
|
+
{
|
|
29
|
+
name: "current_time",
|
|
30
|
+
description: "Return the time.",
|
|
31
|
+
inputSchema: { type: "object", properties: {} },
|
|
32
|
+
},
|
|
33
|
+
],
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
describe("OpenAICompatibleProvider", () => {
|
|
37
|
+
test("normalizes FrockBot messages and tools to the wire format", () => {
|
|
38
|
+
expect(requestToWire(request)).toMatchObject({
|
|
39
|
+
model: "test-model",
|
|
40
|
+
stream: true,
|
|
41
|
+
messages: [
|
|
42
|
+
{ role: "system", content: "Be useful." },
|
|
43
|
+
{ role: "user", content: "What time is it?" },
|
|
44
|
+
{
|
|
45
|
+
role: "assistant",
|
|
46
|
+
tool_calls: [
|
|
47
|
+
{
|
|
48
|
+
id: "previous",
|
|
49
|
+
type: "function",
|
|
50
|
+
function: { name: "lookup", arguments: '{"query":"time"}' },
|
|
51
|
+
},
|
|
52
|
+
],
|
|
53
|
+
},
|
|
54
|
+
{ role: "tool", tool_call_id: "previous", content: "noon" },
|
|
55
|
+
],
|
|
56
|
+
tools: [
|
|
57
|
+
{
|
|
58
|
+
type: "function",
|
|
59
|
+
function: { name: "current_time", description: "Return the time." },
|
|
60
|
+
},
|
|
61
|
+
],
|
|
62
|
+
});
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
test("streams text and assembles fragmented tool calls", async () => {
|
|
66
|
+
const encoder = new TextEncoder();
|
|
67
|
+
const payloads = [
|
|
68
|
+
'data: {"choices":[{"delta":{"content":"Checking "}}]}\n\n',
|
|
69
|
+
'data: {"choices":[{"delta":{"tool_calls":[{"index":0,"id":"call-1","function":{"name":"current_","arguments":"{\\"zone\\":"}}]}}]}\n\n',
|
|
70
|
+
'data: {"choices":[{"delta":{"tool_calls":[{"index":0,"function":{"name":"time","arguments":"\\"UTC\\"}"}}]},"finish_reason":"tool_calls"}]}\n\n',
|
|
71
|
+
"data: [DONE]\n\n",
|
|
72
|
+
];
|
|
73
|
+
let capturedUrl = "";
|
|
74
|
+
let capturedAuthorization = "";
|
|
75
|
+
let capturedIdempotencyKey: string | null = null;
|
|
76
|
+
const fetcher = async (
|
|
77
|
+
input: string | URL | Request,
|
|
78
|
+
init?: RequestInit,
|
|
79
|
+
) => {
|
|
80
|
+
capturedUrl = String(input);
|
|
81
|
+
capturedAuthorization =
|
|
82
|
+
new Headers(init?.headers).get("authorization") ?? "";
|
|
83
|
+
capturedIdempotencyKey = new Headers(init?.headers).get(
|
|
84
|
+
"idempotency-key",
|
|
85
|
+
);
|
|
86
|
+
return new Response(
|
|
87
|
+
new ReadableStream<Uint8Array>({
|
|
88
|
+
start(controller) {
|
|
89
|
+
for (const payload of payloads)
|
|
90
|
+
controller.enqueue(encoder.encode(payload));
|
|
91
|
+
controller.close();
|
|
92
|
+
},
|
|
93
|
+
}),
|
|
94
|
+
{ status: 200 },
|
|
95
|
+
);
|
|
96
|
+
};
|
|
97
|
+
const provider = new OpenAICompatibleProvider({
|
|
98
|
+
baseUrl: "https://models.example/v1/",
|
|
99
|
+
apiKey: "secret",
|
|
100
|
+
fetch: fetcher,
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
const events = [];
|
|
104
|
+
for await (const event of provider.stream(
|
|
105
|
+
request,
|
|
106
|
+
new AbortController().signal,
|
|
107
|
+
)) {
|
|
108
|
+
events.push(event);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
expect(capturedUrl).toBe("https://models.example/v1/chat/completions");
|
|
112
|
+
expect(capturedAuthorization).toBe("Bearer secret");
|
|
113
|
+
expect(capturedIdempotencyKey).toBeNull();
|
|
114
|
+
expect(events).toEqual([
|
|
115
|
+
{ type: "text-delta", text: "Checking " },
|
|
116
|
+
{
|
|
117
|
+
type: "tool-call",
|
|
118
|
+
call: {
|
|
119
|
+
id: "call-1",
|
|
120
|
+
name: "current_time",
|
|
121
|
+
input: { zone: "UTC" },
|
|
122
|
+
},
|
|
123
|
+
},
|
|
124
|
+
{ type: "finish", reason: "tool-calls" },
|
|
125
|
+
]);
|
|
126
|
+
});
|
|
127
|
+
|
|
128
|
+
test("rejects a truncated stream without a terminal marker", async () => {
|
|
129
|
+
const provider = new OpenAICompatibleProvider({
|
|
130
|
+
baseUrl: "https://models.example/v1",
|
|
131
|
+
fetch: () =>
|
|
132
|
+
Promise.resolve(
|
|
133
|
+
new Response(
|
|
134
|
+
'data: {"choices":[{"delta":{"content":"partial"}}]}\n\n',
|
|
135
|
+
{ status: 200 },
|
|
136
|
+
),
|
|
137
|
+
),
|
|
138
|
+
});
|
|
139
|
+
const events = [];
|
|
140
|
+
let failure: unknown;
|
|
141
|
+
|
|
142
|
+
try {
|
|
143
|
+
for await (const event of provider.stream(
|
|
144
|
+
request,
|
|
145
|
+
new AbortController().signal,
|
|
146
|
+
)) {
|
|
147
|
+
events.push(event);
|
|
148
|
+
}
|
|
149
|
+
} catch (error) {
|
|
150
|
+
failure = error;
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
expect(events).toEqual([{ type: "text-delta", text: "partial" }]);
|
|
154
|
+
expect(failure).toBeInstanceOf(Error);
|
|
155
|
+
expect(failure instanceof Error ? failure.message : "").toBe(
|
|
156
|
+
"Model response stream ended before a terminal marker",
|
|
157
|
+
);
|
|
158
|
+
});
|
|
159
|
+
|
|
160
|
+
test.each([
|
|
161
|
+
"data: [DONE]\n\n",
|
|
162
|
+
'data: {"model":"test-model"}\n\ndata: [DONE]\n\n',
|
|
163
|
+
])("rejects a terminal stream without a choice", async (body) => {
|
|
164
|
+
const provider = new OpenAICompatibleProvider({
|
|
165
|
+
baseUrl: "https://models.example/v1",
|
|
166
|
+
fetch: () => Promise.resolve(new Response(body, { status: 200 })),
|
|
167
|
+
});
|
|
168
|
+
|
|
169
|
+
let failure: unknown;
|
|
170
|
+
try {
|
|
171
|
+
for await (const _event of provider.stream(
|
|
172
|
+
request,
|
|
173
|
+
new AbortController().signal,
|
|
174
|
+
)) {
|
|
175
|
+
throw new Error("unexpected stream event");
|
|
176
|
+
}
|
|
177
|
+
} catch (error) {
|
|
178
|
+
failure = error;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
expect(failure instanceof Error ? failure.message : "").toBe(
|
|
182
|
+
"Model response stream did not include a valid choice",
|
|
183
|
+
);
|
|
184
|
+
});
|
|
185
|
+
|
|
186
|
+
test("cancels an oversized unterminated stream", async () => {
|
|
187
|
+
const encoder = new TextEncoder();
|
|
188
|
+
let cancelled = false;
|
|
189
|
+
const provider = new OpenAICompatibleProvider({
|
|
190
|
+
baseUrl: "https://models.example/v1",
|
|
191
|
+
fetch: () =>
|
|
192
|
+
Promise.resolve(
|
|
193
|
+
new Response(
|
|
194
|
+
new ReadableStream<Uint8Array>({
|
|
195
|
+
start(controller) {
|
|
196
|
+
controller.enqueue(
|
|
197
|
+
encoder.encode(`data: ${"x".repeat(1_048_577)}`),
|
|
198
|
+
);
|
|
199
|
+
},
|
|
200
|
+
cancel() {
|
|
201
|
+
cancelled = true;
|
|
202
|
+
},
|
|
203
|
+
}),
|
|
204
|
+
{ status: 200 },
|
|
205
|
+
),
|
|
206
|
+
),
|
|
207
|
+
});
|
|
208
|
+
|
|
209
|
+
let failure: unknown;
|
|
210
|
+
try {
|
|
211
|
+
for await (const _event of provider.stream(
|
|
212
|
+
request,
|
|
213
|
+
new AbortController().signal,
|
|
214
|
+
)) {
|
|
215
|
+
throw new Error("unexpected stream event");
|
|
216
|
+
}
|
|
217
|
+
} catch (error) {
|
|
218
|
+
failure = error;
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
expect(failure instanceof Error ? failure.message : "").toBe(
|
|
222
|
+
"Model response stream exceeded its size limit",
|
|
223
|
+
);
|
|
224
|
+
expect(cancelled).toBe(true);
|
|
225
|
+
});
|
|
226
|
+
|
|
227
|
+
test("reports retrieval unavailable without another provider request", async () => {
|
|
228
|
+
let requests = 0;
|
|
229
|
+
const provider = new OpenAICompatibleProvider({
|
|
230
|
+
baseUrl: "https://models.example/v1",
|
|
231
|
+
fetch: () => {
|
|
232
|
+
requests += 1;
|
|
233
|
+
return Promise.resolve(new Response(null, { status: 500 }));
|
|
234
|
+
},
|
|
235
|
+
});
|
|
236
|
+
const root = new Context();
|
|
237
|
+
await root.plugin(LlmRegistry);
|
|
238
|
+
root.llm.register(provider);
|
|
239
|
+
|
|
240
|
+
const outcome = await root.llm.reconcile(
|
|
241
|
+
request,
|
|
242
|
+
new AbortController().signal,
|
|
243
|
+
);
|
|
244
|
+
|
|
245
|
+
expect(outcome).toEqual({
|
|
246
|
+
status: "unavailable",
|
|
247
|
+
reason:
|
|
248
|
+
'LLM provider "openai-compatible" does not support provider-bound retrieval',
|
|
249
|
+
});
|
|
250
|
+
expect(requests).toBe(0);
|
|
251
|
+
await root.fiber.dispose();
|
|
252
|
+
});
|
|
253
|
+
|
|
254
|
+
test("redacts provider response bodies from HTTP errors", async () => {
|
|
255
|
+
const provider = new OpenAICompatibleProvider({
|
|
256
|
+
baseUrl: "https://models.example/v1",
|
|
257
|
+
fetch: () =>
|
|
258
|
+
Promise.resolve(new Response("bad credentials", { status: 401 })),
|
|
259
|
+
});
|
|
260
|
+
|
|
261
|
+
let failure: unknown;
|
|
262
|
+
try {
|
|
263
|
+
for await (const _event of provider.stream(
|
|
264
|
+
request,
|
|
265
|
+
new AbortController().signal,
|
|
266
|
+
)) {
|
|
267
|
+
// No events are expected from a failed response.
|
|
268
|
+
}
|
|
269
|
+
} catch (error) {
|
|
270
|
+
failure = error;
|
|
271
|
+
}
|
|
272
|
+
expect(failure instanceof Error ? failure.message : "").toBe(
|
|
273
|
+
"Model request failed (401)",
|
|
274
|
+
);
|
|
275
|
+
});
|
|
276
|
+
});
|
|
277
|
+
|
|
278
|
+
// An image a tool produced, on the wire.
|
|
279
|
+
//
|
|
280
|
+
// Two behaviours, and the difference between them has to be visible in the
|
|
281
|
+
// request: a model that takes images is shown the picture, and a model that
|
|
282
|
+
// does not is *told* where it is. Silently dropping it would make a Bot that
|
|
283
|
+
// asked for a screenshot indistinguishable from one that got nothing.
|
|
284
|
+
describe("tool result attachments", () => {
|
|
285
|
+
const attachment = {
|
|
286
|
+
kind: "image" as const,
|
|
287
|
+
mediaType: "image/png" as const,
|
|
288
|
+
workspacePath: {
|
|
289
|
+
root: {
|
|
290
|
+
kind: "package-declared" as const,
|
|
291
|
+
userId: "user-1",
|
|
292
|
+
packageId: "computer",
|
|
293
|
+
rootId: "screenshots",
|
|
294
|
+
},
|
|
295
|
+
path: "bot-1/run-9-1.png",
|
|
296
|
+
},
|
|
297
|
+
contentHash: "b".repeat(64),
|
|
298
|
+
bytes: 3,
|
|
299
|
+
};
|
|
300
|
+
|
|
301
|
+
function requestWith(model: string, dataBase64?: string) {
|
|
302
|
+
return {
|
|
303
|
+
requestId: "request-1",
|
|
304
|
+
provider: "openai-compatible",
|
|
305
|
+
model,
|
|
306
|
+
system: "",
|
|
307
|
+
tools: [],
|
|
308
|
+
messages: [
|
|
309
|
+
{
|
|
310
|
+
role: "tool" as const,
|
|
311
|
+
callId: "call-1",
|
|
312
|
+
name: "computer_screenshot",
|
|
313
|
+
content: '{"path":"bot-1/run-9-1.png"}',
|
|
314
|
+
isError: false,
|
|
315
|
+
attachments: [
|
|
316
|
+
{ ...attachment, ...(dataBase64 ? { dataBase64 } : {}) },
|
|
317
|
+
],
|
|
318
|
+
},
|
|
319
|
+
],
|
|
320
|
+
};
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
test("shows a vision model the image as a following user message", () => {
|
|
324
|
+
const wire = requestToWire(requestWith("gpt-4o", "AAAA"));
|
|
325
|
+
const messages = wire.messages as Record<string, unknown>[];
|
|
326
|
+
|
|
327
|
+
expect(messages[0]).toMatchObject({ role: "tool", tool_call_id: "call-1" });
|
|
328
|
+
expect(messages[1]).toMatchObject({ role: "user" });
|
|
329
|
+
expect(JSON.stringify(messages[1])).toContain("data:image/png;base64,AAAA");
|
|
330
|
+
});
|
|
331
|
+
|
|
332
|
+
test("tells a model that takes no image where the image is", () => {
|
|
333
|
+
const wire = requestToWire(requestWith("llama3-8b", "AAAA"));
|
|
334
|
+
const messages = wire.messages as Record<string, unknown>[];
|
|
335
|
+
|
|
336
|
+
expect(messages).toHaveLength(1);
|
|
337
|
+
expect(messages[0]!.content).toContain("not shown to this model");
|
|
338
|
+
expect(messages[0]!.content).toContain("bot-1/run-9-1.png");
|
|
339
|
+
expect(JSON.stringify(wire)).not.toContain("AAAA");
|
|
340
|
+
});
|
|
341
|
+
|
|
342
|
+
test("says so when a vision model's image could not be resolved", () => {
|
|
343
|
+
const wire = requestToWire(requestWith("gpt-4o"));
|
|
344
|
+
const messages = wire.messages as Record<string, unknown>[];
|
|
345
|
+
|
|
346
|
+
expect(messages).toHaveLength(1);
|
|
347
|
+
expect(messages[0]!.content).toContain("not shown to this model");
|
|
348
|
+
});
|
|
349
|
+
|
|
350
|
+
test("lets an explicit acceptsImages override the model-name guess", () => {
|
|
351
|
+
const shown = requestToWire(requestWith("some-local-model", "AAAA"), {
|
|
352
|
+
acceptsImages: true,
|
|
353
|
+
});
|
|
354
|
+
expect((shown.messages as unknown[]).length).toBe(2);
|
|
355
|
+
|
|
356
|
+
const withheld = requestToWire(requestWith("gpt-4o", "AAAA"), {
|
|
357
|
+
acceptsImages: false,
|
|
358
|
+
});
|
|
359
|
+
expect((withheld.messages as unknown[]).length).toBe(1);
|
|
360
|
+
});
|
|
361
|
+
});
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,396 @@
|
|
|
1
|
+
import {
|
|
2
|
+
type LlmMessage,
|
|
3
|
+
type LlmProvider,
|
|
4
|
+
type LlmStreamEvent,
|
|
5
|
+
type NormalizedModelRequest,
|
|
6
|
+
} from "@frockbot/kernel-contracts";
|
|
7
|
+
import type { Plugin } from "cordis";
|
|
8
|
+
|
|
9
|
+
export type JsonValue =
|
|
10
|
+
null | boolean | number | string | JsonValue[] | { [key: string]: JsonValue };
|
|
11
|
+
|
|
12
|
+
export type FetchLike = (
|
|
13
|
+
input: string | URL | Request,
|
|
14
|
+
init?: RequestInit,
|
|
15
|
+
) => Promise<Response>;
|
|
16
|
+
|
|
17
|
+
export class OpenAICompatibleHttpError extends Error {
|
|
18
|
+
constructor(readonly status: number) {
|
|
19
|
+
super(`Model request failed (${status})`);
|
|
20
|
+
this.name = "OpenAICompatibleHttpError";
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export interface OpenAICompatibleConfig {
|
|
25
|
+
baseUrl: string;
|
|
26
|
+
apiKey?: string;
|
|
27
|
+
providerId?: string;
|
|
28
|
+
headers?: Record<string, string>;
|
|
29
|
+
fetch?: FetchLike;
|
|
30
|
+
/**
|
|
31
|
+
* Whether this endpoint's model accepts image content. Absent, and the
|
|
32
|
+
* model id decides through {@link modelAcceptsImagesV1}.
|
|
33
|
+
*/
|
|
34
|
+
acceptsImages?: boolean;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
interface ToolAccumulator {
|
|
38
|
+
index: number;
|
|
39
|
+
id: string;
|
|
40
|
+
name: string;
|
|
41
|
+
arguments: string;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Model families this adapter will hand an image to.
|
|
46
|
+
*
|
|
47
|
+
* A guess, and named as one: there is no capability field on the wire and no
|
|
48
|
+
* catalog this adapter can consult, so the default is a list of families whose
|
|
49
|
+
* documented input includes images. `acceptsImages` overrides it in both
|
|
50
|
+
* directions, which is what a deployment that knows better sets.
|
|
51
|
+
*/
|
|
52
|
+
const VISION_MODEL_PATTERNS = [
|
|
53
|
+
/gpt-4o/i,
|
|
54
|
+
/gpt-4\.1/i,
|
|
55
|
+
/gpt-5/i,
|
|
56
|
+
/o[34]\b/i,
|
|
57
|
+
/claude-/i,
|
|
58
|
+
/gemini-/i,
|
|
59
|
+
/vision/i,
|
|
60
|
+
/-vl\b/i,
|
|
61
|
+
/llava/i,
|
|
62
|
+
/pixtral/i,
|
|
63
|
+
/internvl/i,
|
|
64
|
+
];
|
|
65
|
+
|
|
66
|
+
/** Whether this adapter will show `model` an image attachment. */
|
|
67
|
+
export function modelAcceptsImagesV1(model: string): boolean {
|
|
68
|
+
return VISION_MODEL_PATTERNS.some((pattern) => pattern.test(model));
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function dataUrl(mediaType: string, dataBase64: string): string {
|
|
72
|
+
return `data:${mediaType};base64,${dataBase64}`;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
function messageToWire(
|
|
76
|
+
message: LlmMessage,
|
|
77
|
+
acceptsImages: boolean,
|
|
78
|
+
): Record<string, unknown>[] {
|
|
79
|
+
if (message.role === "user")
|
|
80
|
+
return [{ role: "user", content: message.content }];
|
|
81
|
+
if (message.role === "tool") {
|
|
82
|
+
const attachments = message.attachments ?? [];
|
|
83
|
+
// An attachment this adapter cannot show is said in the text rather than
|
|
84
|
+
// dropped in silence: a Bot that asked for a screenshot has to be able to
|
|
85
|
+
// tell "the model saw it" from "the model was told where it is".
|
|
86
|
+
const shown = acceptsImages
|
|
87
|
+
? attachments.filter((attachment) => attachment.dataBase64 !== undefined)
|
|
88
|
+
: [];
|
|
89
|
+
const withheld = attachments.filter(
|
|
90
|
+
(attachment) => !shown.includes(attachment),
|
|
91
|
+
);
|
|
92
|
+
const notes = withheld.map(
|
|
93
|
+
(attachment) =>
|
|
94
|
+
`[attachment ${attachment.mediaType} not shown to this model; it is at ${attachment.workspacePath.path} (sha256 ${attachment.contentHash})]`,
|
|
95
|
+
);
|
|
96
|
+
const tool = {
|
|
97
|
+
role: "tool",
|
|
98
|
+
tool_call_id: message.callId,
|
|
99
|
+
content: [message.content, ...notes].filter(Boolean).join("\n"),
|
|
100
|
+
};
|
|
101
|
+
if (shown.length === 0) return [tool];
|
|
102
|
+
// The image travels as a following user message rather than inside the
|
|
103
|
+
// tool result: an OpenAI-shaped `tool` message takes text, and a content
|
|
104
|
+
// array there is refused by the very endpoints that accept the image.
|
|
105
|
+
return [
|
|
106
|
+
tool,
|
|
107
|
+
{
|
|
108
|
+
role: "user",
|
|
109
|
+
content: [
|
|
110
|
+
{
|
|
111
|
+
type: "text",
|
|
112
|
+
text: `Attachments from ${message.name}:`,
|
|
113
|
+
},
|
|
114
|
+
...shown.map((attachment) => ({
|
|
115
|
+
type: "image_url",
|
|
116
|
+
image_url: {
|
|
117
|
+
url: dataUrl(attachment.mediaType, attachment.dataBase64!),
|
|
118
|
+
},
|
|
119
|
+
})),
|
|
120
|
+
],
|
|
121
|
+
},
|
|
122
|
+
];
|
|
123
|
+
}
|
|
124
|
+
return [
|
|
125
|
+
{
|
|
126
|
+
role: "assistant",
|
|
127
|
+
content: message.content || null,
|
|
128
|
+
...(message.toolCalls.length > 0
|
|
129
|
+
? {
|
|
130
|
+
tool_calls: message.toolCalls.map((call) => ({
|
|
131
|
+
id: call.id,
|
|
132
|
+
type: "function",
|
|
133
|
+
function: {
|
|
134
|
+
name: call.name,
|
|
135
|
+
arguments: JSON.stringify(call.input),
|
|
136
|
+
},
|
|
137
|
+
})),
|
|
138
|
+
}
|
|
139
|
+
: {}),
|
|
140
|
+
},
|
|
141
|
+
];
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
export function requestToWire(
|
|
145
|
+
request: NormalizedModelRequest,
|
|
146
|
+
options: { acceptsImages?: boolean } = {},
|
|
147
|
+
): Record<string, unknown> {
|
|
148
|
+
const acceptsImages =
|
|
149
|
+
options.acceptsImages ?? modelAcceptsImagesV1(request.model);
|
|
150
|
+
const messages: Record<string, unknown>[] = [];
|
|
151
|
+
if (request.system)
|
|
152
|
+
messages.push({ role: "system", content: request.system });
|
|
153
|
+
for (const message of request.messages) {
|
|
154
|
+
messages.push(...messageToWire(message, acceptsImages));
|
|
155
|
+
}
|
|
156
|
+
return {
|
|
157
|
+
model: request.model,
|
|
158
|
+
stream: true,
|
|
159
|
+
messages,
|
|
160
|
+
...(request.tools.length > 0
|
|
161
|
+
? {
|
|
162
|
+
tools: request.tools.map((tool) => ({
|
|
163
|
+
type: "function",
|
|
164
|
+
function: {
|
|
165
|
+
name: tool.name,
|
|
166
|
+
description: tool.description,
|
|
167
|
+
parameters: tool.inputSchema,
|
|
168
|
+
},
|
|
169
|
+
})),
|
|
170
|
+
}
|
|
171
|
+
: {}),
|
|
172
|
+
};
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
const MAX_SSE_EVENT_CHARACTERS = 1_048_576;
|
|
176
|
+
const MAX_SSE_RESPONSE_BYTES = 16_777_216;
|
|
177
|
+
|
|
178
|
+
async function rejectOversizedSse(
|
|
179
|
+
reader: ReadableStreamDefaultReader<Uint8Array>,
|
|
180
|
+
): Promise<never> {
|
|
181
|
+
await reader.cancel().catch(() => undefined);
|
|
182
|
+
throw new Error("Model response stream exceeded its size limit");
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
async function* readSseData(
|
|
186
|
+
body: ReadableStream<Uint8Array>,
|
|
187
|
+
signal: AbortSignal,
|
|
188
|
+
): AsyncIterable<string> {
|
|
189
|
+
const reader = body.getReader();
|
|
190
|
+
const decoder = new TextDecoder();
|
|
191
|
+
let buffer = "";
|
|
192
|
+
let responseBytes = 0;
|
|
193
|
+
const cancel = (): void => {
|
|
194
|
+
void reader.cancel(signal.reason).catch(() => undefined);
|
|
195
|
+
};
|
|
196
|
+
signal.addEventListener("abort", cancel, { once: true });
|
|
197
|
+
try {
|
|
198
|
+
signal.throwIfAborted();
|
|
199
|
+
while (true) {
|
|
200
|
+
const { done, value } = await reader.read();
|
|
201
|
+
signal.throwIfAborted();
|
|
202
|
+
responseBytes += value?.byteLength ?? 0;
|
|
203
|
+
if (responseBytes > MAX_SSE_RESPONSE_BYTES) {
|
|
204
|
+
await rejectOversizedSse(reader);
|
|
205
|
+
}
|
|
206
|
+
buffer += decoder.decode(value, { stream: !done });
|
|
207
|
+
const blocks = buffer.split(/\r?\n\r?\n/);
|
|
208
|
+
buffer = blocks.pop() ?? "";
|
|
209
|
+
if (
|
|
210
|
+
buffer.length > MAX_SSE_EVENT_CHARACTERS ||
|
|
211
|
+
blocks.some((block) => block.length > MAX_SSE_EVENT_CHARACTERS)
|
|
212
|
+
) {
|
|
213
|
+
await rejectOversizedSse(reader);
|
|
214
|
+
}
|
|
215
|
+
for (const block of blocks) {
|
|
216
|
+
const data = block
|
|
217
|
+
.split(/\r?\n/)
|
|
218
|
+
.filter((line) => line.startsWith("data:"))
|
|
219
|
+
.map((line) => line.slice(5).trimStart())
|
|
220
|
+
.join("\n");
|
|
221
|
+
if (data) yield data;
|
|
222
|
+
}
|
|
223
|
+
if (done) break;
|
|
224
|
+
}
|
|
225
|
+
if (buffer.startsWith("data:")) yield buffer.slice(5).trimStart();
|
|
226
|
+
} finally {
|
|
227
|
+
signal.removeEventListener("abort", cancel);
|
|
228
|
+
reader.releaseLock();
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
function asRecord(value: unknown): Record<string, unknown> | undefined {
|
|
233
|
+
return typeof value === "object" && value !== null
|
|
234
|
+
? (value as Record<string, unknown>)
|
|
235
|
+
: undefined;
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
function applyToolDeltas(
|
|
239
|
+
value: unknown,
|
|
240
|
+
tools: Map<number, ToolAccumulator>,
|
|
241
|
+
): void {
|
|
242
|
+
if (!Array.isArray(value)) return;
|
|
243
|
+
for (const candidate of value) {
|
|
244
|
+
const delta = asRecord(candidate);
|
|
245
|
+
if (!delta || typeof delta.index !== "number") continue;
|
|
246
|
+
const current = tools.get(delta.index) ?? {
|
|
247
|
+
index: delta.index,
|
|
248
|
+
id: "",
|
|
249
|
+
name: "",
|
|
250
|
+
arguments: "",
|
|
251
|
+
};
|
|
252
|
+
if (typeof delta.id === "string") current.id = delta.id;
|
|
253
|
+
const fn = asRecord(delta.function);
|
|
254
|
+
if (typeof fn?.name === "string") current.name += fn.name;
|
|
255
|
+
if (typeof fn?.arguments === "string") current.arguments += fn.arguments;
|
|
256
|
+
tools.set(delta.index, current);
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
function parseJson(value: string, label: string): JsonValue {
|
|
261
|
+
try {
|
|
262
|
+
return JSON.parse(value) as JsonValue;
|
|
263
|
+
} catch (error) {
|
|
264
|
+
throw new Error(`${label}: ${String(error)}`);
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
function parseToolInput(value: string): JsonValue {
|
|
269
|
+
return value ? parseJson(value, "Model returned invalid tool arguments") : {};
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
/**
|
|
273
|
+
* Normalize an OpenAI-compatible SSE body. Native provider bindings can reuse
|
|
274
|
+
* this wire decoder without pretending their in-process call is HTTP.
|
|
275
|
+
*/
|
|
276
|
+
export async function* streamOpenAICompatibleBody(
|
|
277
|
+
body: ReadableStream<Uint8Array>,
|
|
278
|
+
signal: AbortSignal,
|
|
279
|
+
): AsyncIterable<LlmStreamEvent> {
|
|
280
|
+
const tools = new Map<number, ToolAccumulator>();
|
|
281
|
+
let finishReason: string | undefined;
|
|
282
|
+
let terminal = false;
|
|
283
|
+
let sawChoice = false;
|
|
284
|
+
for await (const data of readSseData(body, signal)) {
|
|
285
|
+
if (data === "[DONE]") {
|
|
286
|
+
terminal = true;
|
|
287
|
+
break;
|
|
288
|
+
}
|
|
289
|
+
const payload = asRecord(
|
|
290
|
+
parseJson(data, "Model returned an invalid stream event"),
|
|
291
|
+
);
|
|
292
|
+
const choices = payload?.choices;
|
|
293
|
+
const choice = Array.isArray(choices) ? asRecord(choices[0]) : undefined;
|
|
294
|
+
const delta = asRecord(choice?.delta);
|
|
295
|
+
if (choice && (delta || typeof choice.finish_reason === "string")) {
|
|
296
|
+
sawChoice = true;
|
|
297
|
+
}
|
|
298
|
+
if (typeof delta?.content === "string" && delta.content) {
|
|
299
|
+
yield { type: "text-delta", text: delta.content };
|
|
300
|
+
}
|
|
301
|
+
applyToolDeltas(delta?.tool_calls, tools);
|
|
302
|
+
if (typeof choice?.finish_reason === "string") {
|
|
303
|
+
finishReason = choice.finish_reason;
|
|
304
|
+
terminal = true;
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
if (!terminal) {
|
|
308
|
+
throw new Error("Model response stream ended before a terminal marker");
|
|
309
|
+
}
|
|
310
|
+
if (!sawChoice) {
|
|
311
|
+
throw new Error("Model response stream did not include a valid choice");
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
for (const tool of [...tools.values()].sort(
|
|
315
|
+
(left, right) => left.index - right.index,
|
|
316
|
+
)) {
|
|
317
|
+
if (!tool.name)
|
|
318
|
+
throw new Error("Model returned a tool call without a name");
|
|
319
|
+
yield {
|
|
320
|
+
type: "tool-call",
|
|
321
|
+
call: {
|
|
322
|
+
id: tool.id || crypto.randomUUID(),
|
|
323
|
+
name: tool.name,
|
|
324
|
+
input: parseToolInput(tool.arguments),
|
|
325
|
+
},
|
|
326
|
+
};
|
|
327
|
+
}
|
|
328
|
+
yield {
|
|
329
|
+
type: "finish",
|
|
330
|
+
reason:
|
|
331
|
+
tools.size > 0 || finishReason === "tool_calls"
|
|
332
|
+
? "tool-calls"
|
|
333
|
+
: finishReason === "length"
|
|
334
|
+
? "max-tokens"
|
|
335
|
+
: "completed",
|
|
336
|
+
};
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
export class OpenAICompatibleProvider implements LlmProvider {
|
|
340
|
+
readonly id: string;
|
|
341
|
+
private config: OpenAICompatibleConfig;
|
|
342
|
+
|
|
343
|
+
constructor(config: OpenAICompatibleConfig) {
|
|
344
|
+
if (!config.baseUrl.trim())
|
|
345
|
+
throw new Error("OpenAI-compatible baseUrl is required");
|
|
346
|
+
this.id = config.providerId ?? "openai-compatible";
|
|
347
|
+
this.config = { ...config, baseUrl: config.baseUrl.replace(/\/$/, "") };
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
async *stream(
|
|
351
|
+
request: NormalizedModelRequest,
|
|
352
|
+
signal: AbortSignal,
|
|
353
|
+
): AsyncIterable<LlmStreamEvent> {
|
|
354
|
+
// Workerd rejects a detached global `fetch` ("Illegal invocation"), so the
|
|
355
|
+
// default fetcher forwards through a closure rather than aliasing it.
|
|
356
|
+
const fetcher =
|
|
357
|
+
this.config.fetch ??
|
|
358
|
+
((input: RequestInfo | URL, init?: RequestInit) =>
|
|
359
|
+
globalThis.fetch(input, init));
|
|
360
|
+
const headers: Record<string, string> = {
|
|
361
|
+
"content-type": "application/json",
|
|
362
|
+
...this.config.headers,
|
|
363
|
+
};
|
|
364
|
+
if (this.config.apiKey)
|
|
365
|
+
headers.authorization = `Bearer ${this.config.apiKey}`;
|
|
366
|
+
const response = await fetcher(`${this.config.baseUrl}/chat/completions`, {
|
|
367
|
+
method: "POST",
|
|
368
|
+
headers,
|
|
369
|
+
body: JSON.stringify(
|
|
370
|
+
requestToWire(request, {
|
|
371
|
+
...(this.config.acceptsImages === undefined
|
|
372
|
+
? {}
|
|
373
|
+
: { acceptsImages: this.config.acceptsImages }),
|
|
374
|
+
}),
|
|
375
|
+
),
|
|
376
|
+
signal,
|
|
377
|
+
});
|
|
378
|
+
if (!response.ok) {
|
|
379
|
+
await response.body?.cancel();
|
|
380
|
+
throw new OpenAICompatibleHttpError(response.status);
|
|
381
|
+
}
|
|
382
|
+
if (!response.body)
|
|
383
|
+
throw new Error("Model response did not include a stream");
|
|
384
|
+
|
|
385
|
+
yield* streamOpenAICompatibleBody(response.body, signal);
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
export function createOpenAICompatiblePlugin(
|
|
390
|
+
config: OpenAICompatibleConfig,
|
|
391
|
+
): Plugin.Function {
|
|
392
|
+
const plugin: Plugin.Function = (ctx) =>
|
|
393
|
+
ctx.llm.register(new OpenAICompatibleProvider(config));
|
|
394
|
+
plugin.inject = ["llm"];
|
|
395
|
+
return plugin;
|
|
396
|
+
}
|
package/tsconfig.json
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
{
|
|
2
|
+
"compilerOptions": {
|
|
3
|
+
"target": "ES2023",
|
|
4
|
+
"module": "ESNext",
|
|
5
|
+
"moduleResolution": "Bundler",
|
|
6
|
+
"strict": true,
|
|
7
|
+
"noEmit": true,
|
|
8
|
+
"skipLibCheck": true,
|
|
9
|
+
"lib": ["ES2023", "DOM", "DOM.Iterable"],
|
|
10
|
+
"types": ["bun", "node"]
|
|
11
|
+
},
|
|
12
|
+
"include": ["src/**/*.ts"]
|
|
13
|
+
}
|
package/README.md
DELETED