@providerkit/core 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +35 -158
- package/dist/context.d.ts.map +1 -1
- package/dist/context.js +8 -0
- package/dist/context.js.map +1 -1
- package/dist/errors.d.ts +36 -0
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +75 -2
- package/dist/errors.js.map +1 -1
- package/dist/index.d.ts +4 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -0
- package/dist/index.js.map +1 -1
- package/dist/key-pool.d.ts +60 -0
- package/dist/key-pool.d.ts.map +1 -0
- package/dist/key-pool.js +235 -0
- package/dist/key-pool.js.map +1 -0
- package/dist/providers/anthropic.d.ts +10 -6
- package/dist/providers/anthropic.d.ts.map +1 -1
- package/dist/providers/anthropic.js +30 -14
- package/dist/providers/anthropic.js.map +1 -1
- package/dist/providers/gemini.d.ts +40 -0
- package/dist/providers/gemini.d.ts.map +1 -0
- package/dist/providers/gemini.js +303 -0
- package/dist/providers/gemini.js.map +1 -0
- package/dist/providers/openai.d.ts +38 -1
- package/dist/providers/openai.d.ts.map +1 -1
- package/dist/providers/openai.js +122 -16
- package/dist/providers/openai.js.map +1 -1
- package/dist/providers/responses.d.ts +38 -0
- package/dist/providers/responses.d.ts.map +1 -0
- package/dist/providers/responses.js +341 -0
- package/dist/providers/responses.js.map +1 -0
- package/dist/rate-limit.d.ts +29 -0
- package/dist/rate-limit.d.ts.map +1 -0
- package/dist/rate-limit.js +194 -0
- package/dist/rate-limit.js.map +1 -0
- package/dist/transport.d.ts +18 -5
- package/dist/transport.d.ts.map +1 -1
- package/dist/transport.js +61 -35
- package/dist/transport.js.map +1 -1
- package/dist/types.d.ts +13 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +7 -2
- package/dist/types.js.map +1 -1
- package/dist/zod.d.ts +8 -1
- package/dist/zod.d.ts.map +1 -1
- package/dist/zod.js +9 -2
- package/dist/zod.js.map +1 -1
- package/package.json +2 -2
- package/src/context.ts +7 -0
- package/src/errors.ts +82 -2
- package/src/index.ts +4 -0
- package/src/key-pool.ts +272 -0
- package/src/providers/anthropic.ts +41 -20
- package/src/providers/gemini.ts +386 -0
- package/src/providers/openai.ts +153 -16
- package/src/providers/responses.ts +455 -0
- package/src/rate-limit.ts +217 -0
- package/src/transport.ts +61 -35
- package/src/types.ts +19 -2
- package/src/zod.ts +12 -2
|
@@ -0,0 +1,386 @@
|
|
|
1
|
+
// Gemini adapter — SSE from POST /v1beta/models/{model}:streamGenerateContent.
|
|
2
|
+
//
|
|
3
|
+
// Native Gemini, not its OpenAI-compatible shim: only this endpoint carries
|
|
4
|
+
// thought signatures, and a signature dropped on the way back costs the model
|
|
5
|
+
// its own chain of thought on the next turn. Written straight against the REST
|
|
6
|
+
// wire rather than @google/genai, because this package ships zero dependencies
|
|
7
|
+
// and the SDK is a Node-shaped one.
|
|
8
|
+
import { streamError } from "../errors.ts";
|
|
9
|
+
import { parseToolArgs } from "../tool-args.ts";
|
|
10
|
+
import { streamSse, apiUrl } from "../transport.ts";
|
|
11
|
+
import type {
|
|
12
|
+
ChatMessage,
|
|
13
|
+
Effort,
|
|
14
|
+
FinishReason,
|
|
15
|
+
Provider,
|
|
16
|
+
ProviderChunk,
|
|
17
|
+
StreamOptions,
|
|
18
|
+
ToolCallDelta,
|
|
19
|
+
ToolChoice,
|
|
20
|
+
ToolDefinition,
|
|
21
|
+
} from "../types.ts";
|
|
22
|
+
|
|
23
|
+
export interface GeminiConfig {
|
|
24
|
+
apiKey: string;
|
|
25
|
+
model: string;
|
|
26
|
+
/** Any endpoint speaking the Generative Language REST dialect — a proxy or
|
|
27
|
+
* gateway. Defaults to the Generative Language API.
|
|
28
|
+
*
|
|
29
|
+
* Not Vertex: it serves `/v1/projects/…/locations/…/publishers/google/models`
|
|
30
|
+
* and authenticates with a Bearer token, and both the path and the
|
|
31
|
+
* `x-goog-api-key` header are fixed below. Vertex would be its own adapter. */
|
|
32
|
+
baseUrl?: string;
|
|
33
|
+
/** Names the provider in errors and logs. */
|
|
34
|
+
id?: string;
|
|
35
|
+
/** Bound default; a per-call `effort` overrides it. */
|
|
36
|
+
effort?: Effort;
|
|
37
|
+
maxTokens?: number;
|
|
38
|
+
fetchImpl?: typeof fetch;
|
|
39
|
+
headers?: Record<string, string>;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
const DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com";
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Gemini 3's thinking dial. `none` is MINIMAL rather than a 0 budget: the Pro
|
|
46
|
+
* models reject a hard 0, so the only way to say "think as little as possible"
|
|
47
|
+
* without a 400 is the lowest level.
|
|
48
|
+
*
|
|
49
|
+
* Sent as the REST enum NAMES. The SDK's `ThinkingLevel` members serialize to
|
|
50
|
+
* exactly these strings, so nothing is lost by writing them out.
|
|
51
|
+
*/
|
|
52
|
+
const THINKING_LEVEL: Record<Effort, string> = {
|
|
53
|
+
none: "MINIMAL",
|
|
54
|
+
low: "LOW",
|
|
55
|
+
medium: "MEDIUM",
|
|
56
|
+
high: "HIGH",
|
|
57
|
+
max: "HIGH",
|
|
58
|
+
};
|
|
59
|
+
|
|
60
|
+
/** A turn in Gemini's history. `model` is its word for the assistant. */
|
|
61
|
+
export interface GeminiContent {
|
|
62
|
+
role: "user" | "model";
|
|
63
|
+
parts: unknown[];
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function isJsonObject(value: unknown): value is Record<string, unknown> {
|
|
67
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* Tool-call arguments as the OBJECT Gemini's `functionCall.args` requires.
|
|
72
|
+
*
|
|
73
|
+
* A bare JSON scalar or array is wrapped rather than rejected — the shared
|
|
74
|
+
* `parseToolArgs` drops a non-object as a protocol violation, which is right
|
|
75
|
+
* for reading a fresh tool call and wrong here, where replaying `{}` deletes an
|
|
76
|
+
* argument the model did send. Everything else defers to it, so a truncated or
|
|
77
|
+
* double-escaped argument string is salvaged on replay rather than degraded to
|
|
78
|
+
* `{}`.
|
|
79
|
+
*/
|
|
80
|
+
function parseArgs(json: string): Record<string, unknown> {
|
|
81
|
+
try {
|
|
82
|
+
const parsed: unknown = JSON.parse(json);
|
|
83
|
+
if (!isJsonObject(parsed)) return { value: parsed };
|
|
84
|
+
} catch {
|
|
85
|
+
// Unparseable — the salvage below is the whole point.
|
|
86
|
+
}
|
|
87
|
+
return parseToolArgs(json);
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/** Same rule for a tool RESULT, which `functionResponse.response` also requires
|
|
91
|
+
* as an object. Plain text (the common case) rides under `output`. */
|
|
92
|
+
function toResponseObject(content: string): Record<string, unknown> {
|
|
93
|
+
try {
|
|
94
|
+
const parsed: unknown = JSON.parse(content);
|
|
95
|
+
return isJsonObject(parsed) ? parsed : { output: parsed };
|
|
96
|
+
} catch {
|
|
97
|
+
return { output: content };
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* Our messages → Gemini contents.
|
|
103
|
+
*
|
|
104
|
+
* Three shape differences the rest of the adapter must not have to know about:
|
|
105
|
+
* system text is lifted out and merged into ONE instruction (Gemini takes a
|
|
106
|
+
* single one, not a role in the turn list); assistant turns are role `model`
|
|
107
|
+
* and carry tool calls as `functionCall` parts with the thought signature
|
|
108
|
+
* beside them; tool results are role `user` with a `functionResponse` part
|
|
109
|
+
* naming the FUNCTION, since Gemini pairs a result to its call by name.
|
|
110
|
+
*/
|
|
111
|
+
export function toGeminiContents(messages: readonly ChatMessage[]): {
|
|
112
|
+
system?: string;
|
|
113
|
+
contents: GeminiContent[];
|
|
114
|
+
} {
|
|
115
|
+
let system = "";
|
|
116
|
+
const contents: GeminiContent[] = [];
|
|
117
|
+
|
|
118
|
+
for (const message of messages) {
|
|
119
|
+
switch (message.role) {
|
|
120
|
+
case "system":
|
|
121
|
+
system = system ? `${system}\n\n${message.content}` : message.content;
|
|
122
|
+
break;
|
|
123
|
+
|
|
124
|
+
case "user":
|
|
125
|
+
contents.push({
|
|
126
|
+
role: "user",
|
|
127
|
+
parts:
|
|
128
|
+
typeof message.content === "string"
|
|
129
|
+
? [{ text: message.content }]
|
|
130
|
+
: message.content.map((part) =>
|
|
131
|
+
part.type === "text"
|
|
132
|
+
? { text: part.text }
|
|
133
|
+
: { inlineData: { mimeType: part.mimeType, data: part.data } },
|
|
134
|
+
),
|
|
135
|
+
});
|
|
136
|
+
break;
|
|
137
|
+
|
|
138
|
+
case "assistant": {
|
|
139
|
+
const parts: unknown[] = [];
|
|
140
|
+
if (message.content) parts.push({ text: message.content });
|
|
141
|
+
for (const call of message.toolCalls ?? []) {
|
|
142
|
+
parts.push({
|
|
143
|
+
functionCall: { id: call.id, name: call.name, args: parseArgs(call.arguments) },
|
|
144
|
+
// Replayed verbatim: this is the model's own reasoning token, and
|
|
145
|
+
// without it the next turn starts from a chain of thought that
|
|
146
|
+
// no longer includes the call it just made.
|
|
147
|
+
...(call.thoughtSignature ? { thoughtSignature: call.thoughtSignature } : {}),
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
// An assistant turn with neither text nor calls has no representation
|
|
151
|
+
// here, and an empty `parts` is a 400.
|
|
152
|
+
if (parts.length > 0) contents.push({ role: "model", parts });
|
|
153
|
+
break;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
case "tool":
|
|
157
|
+
contents.push({
|
|
158
|
+
role: "user",
|
|
159
|
+
parts: [
|
|
160
|
+
{
|
|
161
|
+
functionResponse: {
|
|
162
|
+
id: message.toolCallId,
|
|
163
|
+
name: message.name,
|
|
164
|
+
response: toResponseObject(message.content),
|
|
165
|
+
},
|
|
166
|
+
},
|
|
167
|
+
...(message.images ?? []).map((image) => ({
|
|
168
|
+
inlineData: { mimeType: image.mimeType, data: image.data },
|
|
169
|
+
})),
|
|
170
|
+
],
|
|
171
|
+
});
|
|
172
|
+
break;
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
return { ...(system ? { system } : {}), contents };
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/**
|
|
180
|
+
* `auto` and an absent choice are Gemini's own default, so the field is omitted
|
|
181
|
+
* rather than sent as AUTO. `required` and a pinned tool are both mode ANY —
|
|
182
|
+
* the pin is the allow-list, not the mode.
|
|
183
|
+
*/
|
|
184
|
+
function toToolConfig(choice: ToolChoice | undefined): unknown {
|
|
185
|
+
if (choice === undefined || choice === "auto") return undefined;
|
|
186
|
+
if (choice === "none") return { functionCallingConfig: { mode: "NONE" } };
|
|
187
|
+
if (choice === "required") return { functionCallingConfig: { mode: "ANY" } };
|
|
188
|
+
return { functionCallingConfig: { mode: "ANY", allowedFunctionNames: [choice.name] } };
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
function mapFinishReason(reason: string): FinishReason {
|
|
192
|
+
switch (reason) {
|
|
193
|
+
case "MAX_TOKENS":
|
|
194
|
+
return "length";
|
|
195
|
+
case "SAFETY":
|
|
196
|
+
case "PROHIBITED_CONTENT":
|
|
197
|
+
return "content_filter";
|
|
198
|
+
default:
|
|
199
|
+
return "stop";
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
interface GeminiPart {
|
|
204
|
+
text?: string;
|
|
205
|
+
/** Marks the parts that are the model's reasoning. Absent on the answer. */
|
|
206
|
+
thought?: boolean;
|
|
207
|
+
thoughtSignature?: string;
|
|
208
|
+
functionCall?: { id?: string; name?: string; args?: Record<string, unknown> };
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
/** Google's `google.rpc.Status`: an HTTP `code`, the canonical `status` name,
|
|
212
|
+
* and `details` — which is where RetryInfo's `retryDelay` rides. */
|
|
213
|
+
interface GeminiStatus {
|
|
214
|
+
code?: number;
|
|
215
|
+
message?: string;
|
|
216
|
+
status?: string;
|
|
217
|
+
details?: unknown;
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
interface GeminiResponse {
|
|
221
|
+
candidates?: {
|
|
222
|
+
content?: { parts?: GeminiPart[] };
|
|
223
|
+
finishReason?: string;
|
|
224
|
+
}[];
|
|
225
|
+
usageMetadata?: {
|
|
226
|
+
promptTokenCount?: number;
|
|
227
|
+
candidatesTokenCount?: number;
|
|
228
|
+
thoughtsTokenCount?: number;
|
|
229
|
+
cachedContentTokenCount?: number;
|
|
230
|
+
};
|
|
231
|
+
/** Present only on the in-band failure below — never on a real candidate. */
|
|
232
|
+
error?: GeminiStatus;
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
export function createGeminiProvider(config: GeminiConfig): Provider {
|
|
236
|
+
const baseUrl = config.baseUrl ?? DEFAULT_BASE_URL;
|
|
237
|
+
const id = config.id ?? "gemini";
|
|
238
|
+
|
|
239
|
+
return {
|
|
240
|
+
id,
|
|
241
|
+
model: config.model,
|
|
242
|
+
|
|
243
|
+
async *createStream(
|
|
244
|
+
messages: ChatMessage[],
|
|
245
|
+
tools: ToolDefinition[],
|
|
246
|
+
opts: StreamOptions = {},
|
|
247
|
+
): AsyncIterable<ProviderChunk> {
|
|
248
|
+
const effort = opts.effort ?? config.effort;
|
|
249
|
+
const maxTokens = opts.maxTokens ?? config.maxTokens;
|
|
250
|
+
const { system, contents } = toGeminiContents(messages);
|
|
251
|
+
|
|
252
|
+
const generationConfig: Record<string, unknown> = {};
|
|
253
|
+
if (maxTokens !== undefined) generationConfig.maxOutputTokens = maxTokens;
|
|
254
|
+
if (opts.temperature !== undefined) generationConfig.temperature = opts.temperature;
|
|
255
|
+
// No effort means the model's own dynamic thinking. Sending MINIMAL here
|
|
256
|
+
// would switch that off for a caller who never asked, which is the whole
|
|
257
|
+
// reason the seam treats an absent effort as "never sent".
|
|
258
|
+
if (effort) generationConfig.thinkingConfig = { thinkingLevel: THINKING_LEVEL[effort] };
|
|
259
|
+
if (opts.json) {
|
|
260
|
+
generationConfig.responseMimeType = "application/json";
|
|
261
|
+
// `responseJsonSchema` takes JSON Schema as written; `responseSchema`
|
|
262
|
+
// is Gemini's own trimmed dialect and rejects most of what a real
|
|
263
|
+
// schema carries.
|
|
264
|
+
generationConfig.responseJsonSchema = opts.json.schema;
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
const request: Record<string, unknown> = { contents, generationConfig };
|
|
268
|
+
// A Content, not the bare string the SDK accepts — REST rejects a string.
|
|
269
|
+
if (system) request.systemInstruction = { parts: [{ text: system }] };
|
|
270
|
+
if (tools.length > 0) {
|
|
271
|
+
request.tools = [
|
|
272
|
+
{
|
|
273
|
+
functionDeclarations: tools.map((tool) => ({
|
|
274
|
+
name: tool.name,
|
|
275
|
+
description: tool.description,
|
|
276
|
+
// Same rule as the response schema: `parameters` is the trimmed
|
|
277
|
+
// dialect, `parametersJsonSchema` is the schema we actually wrote.
|
|
278
|
+
parametersJsonSchema: tool.inputSchema,
|
|
279
|
+
})),
|
|
280
|
+
},
|
|
281
|
+
];
|
|
282
|
+
// Pointless without declarations, and Gemini 400s on a tool config that
|
|
283
|
+
// names a function it was never given.
|
|
284
|
+
const toolConfig = toToolConfig(opts.toolChoice);
|
|
285
|
+
if (toolConfig) request.toolConfig = toolConfig;
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
// A model id copied out of Gemini's docs is often already `models/…`, and
|
|
289
|
+
// the doubled segment 404s as "model not found" — a confusing way to
|
|
290
|
+
// learn about a prefix.
|
|
291
|
+
const model = (opts.model ?? config.model).replace(/^models\//, "");
|
|
292
|
+
|
|
293
|
+
// Gemini reports the finish reason on a candidate that can arrive AFTER
|
|
294
|
+
// the chunk carrying the function calls, so a turn's tool use has to be
|
|
295
|
+
// remembered rather than read off the final chunk.
|
|
296
|
+
let sawFunctionCalls = false;
|
|
297
|
+
let callIndex = 0;
|
|
298
|
+
|
|
299
|
+
for await (const data of streamSse({
|
|
300
|
+
// Without `?alt=sse` the response is one long JSON array that only
|
|
301
|
+
// parses once complete, which is not a stream.
|
|
302
|
+
url: apiUrl(baseUrl, `/v1beta/models/${model}:streamGenerateContent?alt=sse`),
|
|
303
|
+
headers: { "x-goog-api-key": config.apiKey, ...config.headers },
|
|
304
|
+
body: request,
|
|
305
|
+
provider: id,
|
|
306
|
+
...(opts.signal ? { signal: opts.signal } : {}),
|
|
307
|
+
...(config.fetchImpl ? { fetchImpl: config.fetchImpl } : {}),
|
|
308
|
+
})) {
|
|
309
|
+
let chunk: GeminiResponse;
|
|
310
|
+
try {
|
|
311
|
+
chunk = JSON.parse(data) as GeminiResponse;
|
|
312
|
+
} catch {
|
|
313
|
+
continue;
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
// `?alt=sse` commits to 200 the moment the headers go out, so a
|
|
317
|
+
// throttle or an overload landing after that arrives here as a
|
|
318
|
+
// google.rpc.Status in the body rather than as a status line.
|
|
319
|
+
if (chunk.error) throw streamError(id, chunk.error);
|
|
320
|
+
|
|
321
|
+
if (chunk.usageMetadata) {
|
|
322
|
+
const usage = chunk.usageMetadata;
|
|
323
|
+
yield {
|
|
324
|
+
type: "usage",
|
|
325
|
+
usage: {
|
|
326
|
+
inputTokens: usage.promptTokenCount ?? 0,
|
|
327
|
+
// Thoughts bill as OUTPUT, and Gemini reports them OUTSIDE
|
|
328
|
+
// candidatesTokenCount — leaving them out undercounts a thinking
|
|
329
|
+
// turn by most of what it cost.
|
|
330
|
+
outputTokens: (usage.candidatesTokenCount ?? 0) + (usage.thoughtsTokenCount ?? 0),
|
|
331
|
+
cachedInputTokens: usage.cachedContentTokenCount ?? 0,
|
|
332
|
+
},
|
|
333
|
+
};
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
const candidate = chunk.candidates?.[0];
|
|
337
|
+
if (!candidate) continue;
|
|
338
|
+
|
|
339
|
+
// One chunk can carry several parts of each kind. They are coalesced
|
|
340
|
+
// per kind so the seam sees one reasoning delta and one content delta
|
|
341
|
+
// per chunk, in that order, rather than interleaved fragments.
|
|
342
|
+
let reasoning = "";
|
|
343
|
+
let content = "";
|
|
344
|
+
const toolCalls: ToolCallDelta[] = [];
|
|
345
|
+
for (const part of candidate.content?.parts ?? []) {
|
|
346
|
+
if (part.text) {
|
|
347
|
+
if (part.thought) reasoning += part.text;
|
|
348
|
+
else content += part.text;
|
|
349
|
+
continue;
|
|
350
|
+
}
|
|
351
|
+
const call = part.functionCall;
|
|
352
|
+
if (!call) continue;
|
|
353
|
+
const index = callIndex++;
|
|
354
|
+
toolCalls.push({
|
|
355
|
+
index,
|
|
356
|
+
// Gemini omits the id on a single-call turn, and the seam's
|
|
357
|
+
// consumers pair a result back to its call by id.
|
|
358
|
+
id: call.id ?? `call_${index}`,
|
|
359
|
+
...(call.name ? { name: call.name } : {}),
|
|
360
|
+
// Whole and already assembled — unlike the OpenAI shape, Gemini
|
|
361
|
+
// never fragments an argument object across chunks.
|
|
362
|
+
arguments: JSON.stringify(call.args ?? {}),
|
|
363
|
+
...(part.thoughtSignature ? { thoughtSignature: part.thoughtSignature } : {}),
|
|
364
|
+
});
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
if (reasoning) yield { type: "delta", reasoning };
|
|
368
|
+
if (content) yield { type: "delta", content };
|
|
369
|
+
if (toolCalls.length > 0) {
|
|
370
|
+
sawFunctionCalls = true;
|
|
371
|
+
yield { type: "delta", toolCalls };
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
if (candidate.finishReason) {
|
|
375
|
+
yield {
|
|
376
|
+
type: "finish",
|
|
377
|
+
// A turn that called tools finishes as tool_calls whatever the
|
|
378
|
+
// candidate says — Gemini routinely reports STOP there, and a
|
|
379
|
+
// caller reading that as "done" drops the tool round entirely.
|
|
380
|
+
finishReason: sawFunctionCalls ? "tool_calls" : mapFinishReason(candidate.finishReason),
|
|
381
|
+
};
|
|
382
|
+
}
|
|
383
|
+
}
|
|
384
|
+
},
|
|
385
|
+
};
|
|
386
|
+
}
|
package/src/providers/openai.ts
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
// This is the dialect most gateways speak, so one adapter serves OpenAI,
|
|
4
4
|
// OpenRouter, DeepSeek, GLM, Kimi, Groq, Together, vLLM, Ollama and LM Studio.
|
|
5
5
|
// Their divergences are small and named where they appear.
|
|
6
|
+
import { streamError } from "../errors.ts";
|
|
6
7
|
import { streamSse, apiUrl } from "../transport.ts";
|
|
7
8
|
import type {
|
|
8
9
|
ChatMessage,
|
|
@@ -21,9 +22,22 @@ export interface OpenAIConfig {
|
|
|
21
22
|
model: string;
|
|
22
23
|
/** Any OpenAI-compatible endpoint. Defaults to OpenAI itself. */
|
|
23
24
|
baseUrl?: string;
|
|
24
|
-
/** Names the provider in errors and logs — "openrouter", "deepseek", …
|
|
25
|
+
/** Names the provider in errors and logs — "openrouter", "deepseek", … It
|
|
26
|
+
* also picks the effort dialect below, unless `effortDialect` overrides. */
|
|
25
27
|
id?: string;
|
|
26
28
|
effort?: Effort;
|
|
29
|
+
/**
|
|
30
|
+
* Which spelling of "think this hard" this endpoint accepts. Inferred from
|
|
31
|
+
* `id`; set it when the gateway is not named after its dialect, or to `off`
|
|
32
|
+
* for one that rejects the field outright.
|
|
33
|
+
*/
|
|
34
|
+
effortDialect?: EffortDialect;
|
|
35
|
+
/**
|
|
36
|
+
* `schema` sends a `json_schema` response format, `object` plain JSON mode.
|
|
37
|
+
* Defaults to `schema` for OpenAI itself and `object` everywhere else, which
|
|
38
|
+
* is the only setting every gateway accepts.
|
|
39
|
+
*/
|
|
40
|
+
jsonMode?: "schema" | "object";
|
|
27
41
|
maxTokens?: number;
|
|
28
42
|
fetchImpl?: typeof fetch;
|
|
29
43
|
headers?: Record<string, string>;
|
|
@@ -39,6 +53,64 @@ export interface OpenAIConfig {
|
|
|
39
53
|
|
|
40
54
|
const DEFAULT_BASE_URL = "https://api.openai.com";
|
|
41
55
|
|
|
56
|
+
/** The spellings of "think this hard" across the dialects that share this
|
|
57
|
+
* adapter. `off` sends nothing and leaves the model on its own default. */
|
|
58
|
+
export type EffortDialect = "openai" | "openrouter" | "deepseek" | "off";
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Effort → the request fields THIS endpoint accepts.
|
|
62
|
+
*
|
|
63
|
+
* One knob, three incompatible spellings, and the differences are not cosmetic:
|
|
64
|
+
*
|
|
65
|
+
* - **OpenRouter has no off switch.** `reasoning.enabled: false` is refused by
|
|
66
|
+
* models that always think — GLM 5.3 Flash answers `400 "Reasoning is
|
|
67
|
+
* mandatory for this endpoint and cannot be disabled."` — so `none` is
|
|
68
|
+
* floored at `low` rather than sent. Named rather than omitted, because
|
|
69
|
+
* letting the endpoint pick leaves cost and latency unpinned on exactly the
|
|
70
|
+
* tasks that asked for neither. Measured against the live endpoint, and it
|
|
71
|
+
* cost one app its onboarding read before it was.
|
|
72
|
+
* - **DeepSeek V4 defaults thinking ON**, so `none` has to be an explicit
|
|
73
|
+
* refusal. That is the case proving OpenRouter's floor is a constraint and
|
|
74
|
+
* not a preference for always thinking.
|
|
75
|
+
* - **OpenAI** takes `reasoning_effort` and nothing at all for `none`.
|
|
76
|
+
*
|
|
77
|
+
* An absent effort sends nothing on every dialect: the seam's rule is that a
|
|
78
|
+
* knob the caller never touched is a knob the provider still owns.
|
|
79
|
+
*/
|
|
80
|
+
export function effortParams(
|
|
81
|
+
dialect: EffortDialect,
|
|
82
|
+
effort: Effort | undefined,
|
|
83
|
+
): Record<string, unknown> {
|
|
84
|
+
if (!effort) return {};
|
|
85
|
+
const level = effort === "max" ? "high" : effort === "none" ? null : effort;
|
|
86
|
+
switch (dialect) {
|
|
87
|
+
case "deepseek":
|
|
88
|
+
if (level === null) return { thinking: { type: "disabled" } };
|
|
89
|
+
// A graded level rides only when it asks for LESS. DeepSeek auto-bumps a
|
|
90
|
+
// complex agent or tool request past its own default, and naming the top
|
|
91
|
+
// tier here caps exactly the turns that most need the bump — so `high`
|
|
92
|
+
// and `max` say "on" and leave the ceiling where DeepSeek puts it, while
|
|
93
|
+
// `low` and `medium` mean what they say.
|
|
94
|
+
return level === "high"
|
|
95
|
+
? { thinking: { type: "enabled" } }
|
|
96
|
+
: { thinking: { type: "enabled" }, reasoning_effort: level };
|
|
97
|
+
case "openrouter":
|
|
98
|
+
return { reasoning: { effort: level ?? "low" } };
|
|
99
|
+
case "openai":
|
|
100
|
+
return level === null ? {} : { reasoning_effort: level };
|
|
101
|
+
case "off":
|
|
102
|
+
return {};
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/** Gateways named after their dialect get it for free; everything else keeps
|
|
107
|
+
* the dialect this adapter is named for. */
|
|
108
|
+
function dialectFor(id: string): EffortDialect {
|
|
109
|
+
if (id === "openrouter") return "openrouter";
|
|
110
|
+
if (id === "deepseek") return "deepseek";
|
|
111
|
+
return "openai";
|
|
112
|
+
}
|
|
113
|
+
|
|
42
114
|
function mapFinishReason(reason: string | null | undefined): FinishReason | undefined {
|
|
43
115
|
switch (reason) {
|
|
44
116
|
case "stop":
|
|
@@ -71,33 +143,62 @@ function partsToOpenAI(content: string | ContentPart[]): unknown {
|
|
|
71
143
|
* first (`stripReasoning`); the two cannot be mixed.
|
|
72
144
|
*/
|
|
73
145
|
export function toOpenAIMessages(messages: readonly ChatMessage[]): unknown[] {
|
|
74
|
-
|
|
146
|
+
const out: unknown[] = [];
|
|
147
|
+
for (const message of messages) {
|
|
75
148
|
switch (message.role) {
|
|
76
149
|
case "system":
|
|
77
|
-
|
|
150
|
+
out.push({ role: "system", content: message.content });
|
|
151
|
+
break;
|
|
152
|
+
|
|
78
153
|
case "user":
|
|
79
|
-
|
|
154
|
+
out.push({ role: "user", content: partsToOpenAI(message.content) });
|
|
155
|
+
break;
|
|
156
|
+
|
|
80
157
|
case "tool":
|
|
81
|
-
|
|
158
|
+
out.push({ role: "tool", tool_call_id: message.toolCallId, content: message.content });
|
|
159
|
+
// This dialect has no image slot on a tool message — a `tool` role takes
|
|
160
|
+
// text and nothing else. A screenshot a tool hands back therefore
|
|
161
|
+
// follows as its own user message, which is the only way the model ever
|
|
162
|
+
// sees it. Dropped instead, the turn reads as a tool that returned
|
|
163
|
+
// words about a picture nobody was shown.
|
|
164
|
+
if (message.images?.length) {
|
|
165
|
+
out.push({
|
|
166
|
+
role: "user",
|
|
167
|
+
content: message.images.map((image) => ({
|
|
168
|
+
type: "image_url",
|
|
169
|
+
image_url: { url: toDataUri(image) },
|
|
170
|
+
})),
|
|
171
|
+
});
|
|
172
|
+
}
|
|
173
|
+
break;
|
|
174
|
+
|
|
82
175
|
case "assistant": {
|
|
83
|
-
const
|
|
176
|
+
const assistant: Record<string, unknown> = {
|
|
84
177
|
role: "assistant",
|
|
85
178
|
// Nullable content beside tool_calls is what this shape expects, but
|
|
86
179
|
// several gateways reject a bare null — "" satisfies both.
|
|
87
180
|
content: message.content || "",
|
|
88
181
|
};
|
|
89
|
-
if (message.reasoning)
|
|
182
|
+
if (message.reasoning) assistant.reasoning_content = message.reasoning;
|
|
183
|
+
// Verbatim, under the name the gateway gave it. Reshaped or dropped, the
|
|
184
|
+
// model loses its own record of how it reached the tool round it is
|
|
185
|
+
// being asked to continue.
|
|
186
|
+
if (message.reasoningDetails?.length) {
|
|
187
|
+
assistant.reasoning_details = message.reasoningDetails;
|
|
188
|
+
}
|
|
90
189
|
if (message.toolCalls?.length) {
|
|
91
|
-
|
|
190
|
+
assistant.tool_calls = message.toolCalls.map((call) => ({
|
|
92
191
|
id: call.id,
|
|
93
192
|
type: "function",
|
|
94
193
|
function: { name: call.name, arguments: call.arguments },
|
|
95
194
|
}));
|
|
96
195
|
}
|
|
97
|
-
|
|
196
|
+
out.push(assistant);
|
|
197
|
+
break;
|
|
98
198
|
}
|
|
99
199
|
}
|
|
100
|
-
}
|
|
200
|
+
}
|
|
201
|
+
return out;
|
|
101
202
|
}
|
|
102
203
|
|
|
103
204
|
interface OpenAIChunk {
|
|
@@ -106,6 +207,8 @@ interface OpenAIChunk {
|
|
|
106
207
|
content?: string | null;
|
|
107
208
|
reasoning_content?: string | null;
|
|
108
209
|
reasoning?: string | null;
|
|
210
|
+
/** OpenRouter's normalized reasoning payload, on the final delta. */
|
|
211
|
+
reasoning_details?: unknown[];
|
|
109
212
|
tool_calls?: {
|
|
110
213
|
index?: number;
|
|
111
214
|
id?: string;
|
|
@@ -118,7 +221,13 @@ interface OpenAIChunk {
|
|
|
118
221
|
prompt_tokens?: number;
|
|
119
222
|
completion_tokens?: number;
|
|
120
223
|
prompt_tokens_details?: { cached_tokens?: number };
|
|
224
|
+
/** DeepSeek's native API reports the cache-hit count here instead of in
|
|
225
|
+
* `prompt_tokens_details`, and it is absent from every OpenAI SDK type. */
|
|
226
|
+
prompt_cache_hit_tokens?: number;
|
|
121
227
|
} | null;
|
|
228
|
+
/** Present only on the in-band failure below — never beside a choice.
|
|
229
|
+
* `code` is the numeric HTTP status on the gateways, a slug on OpenAI. */
|
|
230
|
+
error?: { message?: string; code?: string | number; type?: string };
|
|
122
231
|
}
|
|
123
232
|
|
|
124
233
|
export function createOpenAIProvider(config: OpenAIConfig): Provider {
|
|
@@ -147,7 +256,7 @@ export function createOpenAIProvider(config: OpenAIConfig): Provider {
|
|
|
147
256
|
const maxTokens = opts.maxTokens ?? config.maxTokens;
|
|
148
257
|
if (maxTokens !== undefined) request.max_tokens = maxTokens;
|
|
149
258
|
if (opts.temperature !== undefined) request.temperature = opts.temperature;
|
|
150
|
-
|
|
259
|
+
Object.assign(request, effortParams(config.effortDialect ?? dialectFor(id), effort));
|
|
151
260
|
if (tools.length > 0) {
|
|
152
261
|
request.tools = tools.map((tool) => ({
|
|
153
262
|
type: "function",
|
|
@@ -165,10 +274,19 @@ export function createOpenAIProvider(config: OpenAIConfig): Provider {
|
|
|
165
274
|
: { type: "function", function: { name: opts.toolChoice.name } };
|
|
166
275
|
}
|
|
167
276
|
if (opts.json) {
|
|
168
|
-
request.response_format =
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
277
|
+
request.response_format =
|
|
278
|
+
(config.jsonMode ?? (id === "openai" ? "schema" : "object")) === "schema"
|
|
279
|
+
? {
|
|
280
|
+
type: "json_schema",
|
|
281
|
+
json_schema: { name: opts.json.name, schema: opts.json.schema, strict: true },
|
|
282
|
+
}
|
|
283
|
+
: // Everything else gets plain JSON mode. Schema ENFORCEMENT is
|
|
284
|
+
// OpenAI's; the gateways and the vendors behind them offer JSON
|
|
285
|
+
// mode at best, and several answer a flat 400 to a `json_schema`
|
|
286
|
+
// block. The seam's rule makes this safe either way: a provider's
|
|
287
|
+
// "guaranteed" JSON is not one, so the caller validates
|
|
288
|
+
// regardless — this only decides whether the request is accepted.
|
|
289
|
+
{ type: "json_object" };
|
|
172
290
|
}
|
|
173
291
|
if (config.providerOrder?.length) {
|
|
174
292
|
request.provider = { order: config.providerOrder, allow_fallbacks: true };
|
|
@@ -189,10 +307,25 @@ export function createOpenAIProvider(config: OpenAIConfig): Provider {
|
|
|
189
307
|
continue;
|
|
190
308
|
}
|
|
191
309
|
|
|
310
|
+
// A failure the backend reports after its headers went out. The
|
|
311
|
+
// gateways speaking this dialect — OpenRouter above all — report a
|
|
312
|
+
// throttle or an upstream outage this way rather than as a status
|
|
313
|
+
// line, and a frame carrying `error` carries no choices: unread, it
|
|
314
|
+
// falls through both branches below and the turn ends as a successful
|
|
315
|
+
// zero-token completion nobody retries.
|
|
316
|
+
if (chunk.error) throw streamError(id, chunk.error);
|
|
317
|
+
|
|
192
318
|
// A usage-only frame carries no choices — this shape sends it last.
|
|
193
319
|
if (chunk.usage) {
|
|
194
320
|
const input = chunk.usage.prompt_tokens ?? 0;
|
|
195
|
-
|
|
321
|
+
// Two spellings for the same subset. DeepSeek's native endpoint uses
|
|
322
|
+
// its own field, and reading only the standard one bills every cached
|
|
323
|
+
// token at the full input rate — on an agent loop, where the re-sent
|
|
324
|
+
// prefix is overwhelmingly hits, that overstates a run by up to 10×.
|
|
325
|
+
const cached =
|
|
326
|
+
chunk.usage.prompt_tokens_details?.cached_tokens ??
|
|
327
|
+
chunk.usage.prompt_cache_hit_tokens ??
|
|
328
|
+
0;
|
|
196
329
|
yield {
|
|
197
330
|
type: "usage",
|
|
198
331
|
usage: {
|
|
@@ -223,6 +356,10 @@ export function createOpenAIProvider(config: OpenAIConfig): Provider {
|
|
|
223
356
|
out.reasoning = reasoning;
|
|
224
357
|
has = true;
|
|
225
358
|
}
|
|
359
|
+
if (delta.reasoning_details?.length) {
|
|
360
|
+
out.reasoningDetails = delta.reasoning_details;
|
|
361
|
+
has = true;
|
|
362
|
+
}
|
|
226
363
|
if (delta.tool_calls?.length) {
|
|
227
364
|
out.toolCalls = delta.tool_calls.map((call, position) => ({
|
|
228
365
|
// Some gateways omit `index` entirely on single-tool turns.
|