@graphorin/provider 0.6.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +32 -0
- package/README.md +1 -1
- package/dist/adapters/llamacpp-server.d.ts +1 -1
- package/dist/adapters/llamacpp-server.js.map +1 -1
- package/dist/adapters/ollama.d.ts.map +1 -1
- package/dist/adapters/ollama.js +19 -6
- package/dist/adapters/ollama.js.map +1 -1
- package/dist/adapters/vercel.d.ts +1 -1
- package/dist/adapters/vercel.d.ts.map +1 -1
- package/dist/adapters/vercel.js +111 -33
- package/dist/adapters/vercel.js.map +1 -1
- package/dist/errors/errors.d.ts +1 -1
- package/dist/errors/errors.js +1 -1
- package/dist/errors/errors.js.map +1 -1
- package/dist/index.js +0 -6
- package/dist/index.js.map +1 -1
- package/dist/internal/http.js +111 -7
- package/dist/internal/http.js.map +1 -1
- package/dist/internal/openai-shaped.js +21 -4
- package/dist/internal/openai-shaped.js.map +1 -1
- package/dist/middleware/with-fallback.js +1 -1
- package/dist/middleware/with-rate-limit.d.ts +21 -8
- package/dist/middleware/with-rate-limit.d.ts.map +1 -1
- package/dist/middleware/with-rate-limit.js +65 -12
- package/dist/middleware/with-rate-limit.js.map +1 -1
- package/dist/middleware/with-retry.js +1 -1
- package/dist/package.js +1 -1
- package/dist/package.js.map +1 -1
- package/package.json +18 -16
- package/src/adapters/index.ts +14 -0
- package/src/adapters/llamacpp-server.ts +102 -0
- package/src/adapters/ollama.ts +382 -0
- package/src/adapters/openai-compatible.ts +95 -0
- package/src/adapters/vercel-messages.ts +308 -0
- package/src/adapters/vercel.ts +706 -0
- package/src/counters/anthropic-wire.ts +199 -0
- package/src/counters/anthropic.ts +114 -0
- package/src/counters/bedrock.ts +46 -0
- package/src/counters/dispatcher.ts +127 -0
- package/src/counters/global.ts +39 -0
- package/src/counters/google.ts +46 -0
- package/src/counters/heuristic.ts +107 -0
- package/src/counters/index.ts +35 -0
- package/src/counters/js-tiktoken.ts +135 -0
- package/src/counters/serialize.ts +85 -0
- package/src/errors/errors.ts +316 -0
- package/src/errors/index.ts +20 -0
- package/src/index.ts +42 -0
- package/src/internal/abort.ts +30 -0
- package/src/internal/http.ts +388 -0
- package/src/internal/openai-shaped.ts +555 -0
- package/src/internal/sse.ts +112 -0
- package/src/internal/url-utils.ts +20 -0
- package/src/middleware/compose.ts +213 -0
- package/src/middleware/index.ts +37 -0
- package/src/middleware/production-hook.ts +47 -0
- package/src/middleware/with-cost-limit.ts +131 -0
- package/src/middleware/with-cost-tracking.ts +216 -0
- package/src/middleware/with-fallback.ts +157 -0
- package/src/middleware/with-rate-limit.ts +306 -0
- package/src/middleware/with-redaction.ts +671 -0
- package/src/middleware/with-retry.ts +274 -0
- package/src/middleware/with-tracing.ts +117 -0
- package/src/model-tier/classify.ts +125 -0
- package/src/model-tier/index.ts +11 -0
- package/src/provider.ts +121 -0
- package/src/reasoning/apply-policy.ts +89 -0
- package/src/reasoning/classify-contract.ts +120 -0
- package/src/reasoning/index.ts +21 -0
- package/src/reasoning/retention.ts +64 -0
- package/src/tool-examples.ts +54 -0
- package/src/trust/classify-local-provider.ts +254 -0
- package/src/trust/index.ts +14 -0
- package/dist/adapters/index.js +0 -6
|
@@ -0,0 +1,555 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared implementation backing the two OpenAI-compatible adapters
|
|
3
|
+
* (`llamaCppServerAdapter` and `openAICompatibleAdapter`). The wire
|
|
4
|
+
* shape is identical for the `/v1/chat/completions` endpoint, so we
|
|
5
|
+
* factor the streaming + one-shot logic here and the adapters become
|
|
6
|
+
* thin shells declaring trust class, defaults, and capability flags.
|
|
7
|
+
*
|
|
8
|
+
* @internal
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import type {
|
|
12
|
+
FinishReason,
|
|
13
|
+
Provider,
|
|
14
|
+
ProviderCapabilities,
|
|
15
|
+
ProviderEvent,
|
|
16
|
+
ProviderRequest,
|
|
17
|
+
ProviderResponse,
|
|
18
|
+
Sensitivity,
|
|
19
|
+
Usage,
|
|
20
|
+
} from '@graphorin/core';
|
|
21
|
+
|
|
22
|
+
import { LocalProviderInsecureTransportError, ProviderStreamParseError } from '../errors/errors.js';
|
|
23
|
+
import { applyReasoningPolicy } from '../reasoning/apply-policy.js';
|
|
24
|
+
import { resolveReasoningRetention } from '../reasoning/retention.js';
|
|
25
|
+
import { foldToolExamples } from '../tool-examples.js';
|
|
26
|
+
import {
|
|
27
|
+
classifyLocalProvider,
|
|
28
|
+
type LocalProviderClassification,
|
|
29
|
+
} from '../trust/classify-local-provider.js';
|
|
30
|
+
import {
|
|
31
|
+
type ChatMessageConversionOptions,
|
|
32
|
+
callJsonHttp,
|
|
33
|
+
makeStreamStartEvent,
|
|
34
|
+
toOpenAIChatMessages,
|
|
35
|
+
} from './http.js';
|
|
36
|
+
import { parseEventStream } from './sse.js';
|
|
37
|
+
import { stripTrailingSlashes } from './url-utils.js';
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Options shared by every OpenAI-compatible adapter.
|
|
41
|
+
*
|
|
42
|
+
* @internal
|
|
43
|
+
*/
|
|
44
|
+
export interface OpenAIShapedOptions {
|
|
45
|
+
readonly providerName: string;
|
|
46
|
+
readonly model: string;
|
|
47
|
+
readonly baseUrl: string;
|
|
48
|
+
readonly chatPath?: string;
|
|
49
|
+
readonly apiKey?: string;
|
|
50
|
+
readonly headers?: Readonly<Record<string, string>>;
|
|
51
|
+
readonly fetchImpl?: typeof fetch;
|
|
52
|
+
readonly allowInsecureTransport?: boolean;
|
|
53
|
+
readonly capabilities?: Partial<ProviderCapabilities>;
|
|
54
|
+
/**
|
|
55
|
+
* Time-to-response budget per request (PS-24). Default
|
|
56
|
+
* `DEFAULT_REQUEST_TIMEOUT_MS` (120s); `0` disables.
|
|
57
|
+
*/
|
|
58
|
+
readonly timeoutMs?: number;
|
|
59
|
+
/**
|
|
60
|
+
* Overrides the default `acceptsSensitivity` value derived from the
|
|
61
|
+
* trust classifier. Loud users can opt out of the conservative
|
|
62
|
+
* default, but the framework still emits one WARN per process so
|
|
63
|
+
* the override is visible.
|
|
64
|
+
*/
|
|
65
|
+
readonly acceptsSensitivity?: ReadonlyArray<Sensitivity>;
|
|
66
|
+
/**
|
|
67
|
+
* Optional log sink. When unset, the adapter falls back to
|
|
68
|
+
* `console.warn`. Tests pass a fixture-driven sink to silence the
|
|
69
|
+
* console.
|
|
70
|
+
*/
|
|
71
|
+
readonly logger?: (level: 'warn' | 'info', message: string, meta?: object) => void;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
const DEFAULT_CAPABILITIES: ProviderCapabilities = {
|
|
75
|
+
streaming: true,
|
|
76
|
+
toolCalling: true,
|
|
77
|
+
parallelToolCalls: false,
|
|
78
|
+
multimodal: false,
|
|
79
|
+
structuredOutput: true,
|
|
80
|
+
reasoning: false,
|
|
81
|
+
contextWindow: 8_192,
|
|
82
|
+
maxOutput: 4_096,
|
|
83
|
+
reasoningContract: 'optional',
|
|
84
|
+
};
|
|
85
|
+
|
|
86
|
+
const KNOWN_LOOPBACK_OVERRIDES_WARNED = new Set<string>();
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* Build a Graphorin `Provider` backed by an OpenAI-compatible HTTP
|
|
90
|
+
* server. Returns the wrapped object plus the resolved trust
|
|
91
|
+
* classification so the caller can attach it to span attributes if
|
|
92
|
+
* desired.
|
|
93
|
+
*
|
|
94
|
+
* @internal
|
|
95
|
+
*/
|
|
96
|
+
export function buildOpenAIShapedProvider(opts: OpenAIShapedOptions): {
|
|
97
|
+
readonly provider: Provider;
|
|
98
|
+
readonly classification: LocalProviderClassification;
|
|
99
|
+
} {
|
|
100
|
+
const classification = classifyLocalProvider(opts.baseUrl);
|
|
101
|
+
if (classification.trust === 'public-cleartext' && opts.allowInsecureTransport !== true) {
|
|
102
|
+
throw new LocalProviderInsecureTransportError(opts.baseUrl);
|
|
103
|
+
}
|
|
104
|
+
const log = opts.logger ?? defaultLogger;
|
|
105
|
+
if (classification.trust === 'public-cleartext') {
|
|
106
|
+
log('warn', `[${opts.providerName}] allowInsecureTransport=true accepted for ${opts.baseUrl}`, {
|
|
107
|
+
baseUrl: opts.baseUrl,
|
|
108
|
+
});
|
|
109
|
+
} else if (classification.trust === 'public-tls') {
|
|
110
|
+
log('warn', `[${opts.providerName}] public-TLS endpoint; treating as cloud-tier`, {
|
|
111
|
+
baseUrl: opts.baseUrl,
|
|
112
|
+
});
|
|
113
|
+
} else if (classification.trust === 'private') {
|
|
114
|
+
log(
|
|
115
|
+
'warn',
|
|
116
|
+
`[${opts.providerName}] private-network endpoint detected (${classification.reason})`,
|
|
117
|
+
{
|
|
118
|
+
baseUrl: opts.baseUrl,
|
|
119
|
+
acceptsSensitivity: classification.acceptsSensitivity,
|
|
120
|
+
},
|
|
121
|
+
);
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
const acceptsSensitivity = opts.acceptsSensitivity ?? classification.acceptsSensitivity;
|
|
125
|
+
if (
|
|
126
|
+
opts.acceptsSensitivity !== undefined &&
|
|
127
|
+
!sameSensitivity(opts.acceptsSensitivity, classification.acceptsSensitivity) &&
|
|
128
|
+
!KNOWN_LOOPBACK_OVERRIDES_WARNED.has(opts.baseUrl)
|
|
129
|
+
) {
|
|
130
|
+
KNOWN_LOOPBACK_OVERRIDES_WARNED.add(opts.baseUrl);
|
|
131
|
+
log(
|
|
132
|
+
'info',
|
|
133
|
+
`[${opts.providerName}] sensitivity override accepted; default would be [${classification.acceptsSensitivity.join(', ')}]`,
|
|
134
|
+
{
|
|
135
|
+
baseUrl: opts.baseUrl,
|
|
136
|
+
},
|
|
137
|
+
);
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
const capabilities: ProviderCapabilities = {
|
|
141
|
+
...DEFAULT_CAPABILITIES,
|
|
142
|
+
...opts.capabilities,
|
|
143
|
+
};
|
|
144
|
+
const chatPath = opts.chatPath ?? '/v1/chat/completions';
|
|
145
|
+
const url = `${stripTrailingSlashes(opts.baseUrl)}${chatPath}`;
|
|
146
|
+
|
|
147
|
+
const provider: Provider = {
|
|
148
|
+
name: opts.providerName,
|
|
149
|
+
modelId: opts.model,
|
|
150
|
+
capabilities,
|
|
151
|
+
acceptsSensitivity,
|
|
152
|
+
stream(req) {
|
|
153
|
+
return streamOpenAIShaped(opts, url, applyOpenAIShapedPreflight(req, capabilities));
|
|
154
|
+
},
|
|
155
|
+
async generate(req) {
|
|
156
|
+
return generateOpenAIShaped(opts, url, applyOpenAIShapedPreflight(req, capabilities));
|
|
157
|
+
},
|
|
158
|
+
};
|
|
159
|
+
|
|
160
|
+
return { provider, classification };
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
function applyOpenAIShapedPreflight(
|
|
164
|
+
req: ProviderRequest,
|
|
165
|
+
capabilities: ProviderCapabilities,
|
|
166
|
+
): ProviderRequest {
|
|
167
|
+
const retention = resolveReasoningRetention({
|
|
168
|
+
...(req.reasoningRetention !== undefined ? { requested: req.reasoningRetention } : {}),
|
|
169
|
+
...(capabilities.reasoningContract !== undefined
|
|
170
|
+
? { contract: capabilities.reasoningContract }
|
|
171
|
+
: {}),
|
|
172
|
+
});
|
|
173
|
+
if (retention === 'pass-through-all') return req;
|
|
174
|
+
const filtered = applyReasoningPolicy({ messages: req.messages, retention });
|
|
175
|
+
if (filtered === req.messages) return req;
|
|
176
|
+
return { ...req, messages: filtered };
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
async function* streamOpenAIShaped(
|
|
180
|
+
opts: OpenAIShapedOptions,
|
|
181
|
+
url: string,
|
|
182
|
+
req: ProviderRequest,
|
|
183
|
+
): AsyncIterable<ProviderEvent> {
|
|
184
|
+
const body = buildBody(
|
|
185
|
+
opts.model,
|
|
186
|
+
req,
|
|
187
|
+
true,
|
|
188
|
+
opts.capabilities?.structuredOutput ?? true,
|
|
189
|
+
conversionOptionsFor(opts),
|
|
190
|
+
);
|
|
191
|
+
const resp = await callJsonHttp({
|
|
192
|
+
providerName: opts.providerName,
|
|
193
|
+
url,
|
|
194
|
+
headers: buildHeaders(opts),
|
|
195
|
+
body,
|
|
196
|
+
...(req.signal !== undefined ? { signal: req.signal } : {}),
|
|
197
|
+
...(opts.fetchImpl !== undefined ? { fetchImpl: opts.fetchImpl } : {}),
|
|
198
|
+
...(opts.timeoutMs !== undefined ? { timeoutMs: opts.timeoutMs } : {}),
|
|
199
|
+
});
|
|
200
|
+
|
|
201
|
+
yield makeStreamStartEvent({ providerName: opts.providerName, modelId: opts.model });
|
|
202
|
+
|
|
203
|
+
let finishReason: FinishReason = 'stop';
|
|
204
|
+
let usage: Usage | undefined;
|
|
205
|
+
const toolCallBuffer = new Map<
|
|
206
|
+
number,
|
|
207
|
+
{ id: string; name: string; args: string; emittedStart: boolean }
|
|
208
|
+
>();
|
|
209
|
+
|
|
210
|
+
for await (const payload of parseEventStream(
|
|
211
|
+
resp.body,
|
|
212
|
+
req.signal !== undefined ? { signal: req.signal } : {},
|
|
213
|
+
)) {
|
|
214
|
+
if (req.signal?.aborted) {
|
|
215
|
+
// PS-12: report the abort honestly instead of the default 'stop'.
|
|
216
|
+
finishReason = 'aborted';
|
|
217
|
+
break;
|
|
218
|
+
}
|
|
219
|
+
let parsed: OpenAIChunk;
|
|
220
|
+
try {
|
|
221
|
+
parsed = JSON.parse(payload) as OpenAIChunk;
|
|
222
|
+
} catch (cause) {
|
|
223
|
+
throw new ProviderStreamParseError(
|
|
224
|
+
opts.providerName,
|
|
225
|
+
`failed to parse SSE chunk as JSON: ${(cause as Error).message}`,
|
|
226
|
+
cause,
|
|
227
|
+
);
|
|
228
|
+
}
|
|
229
|
+
const choice = parsed.choices?.[0];
|
|
230
|
+
if (choice === undefined) {
|
|
231
|
+
if (parsed.usage !== undefined) usage = mapOpenAIUsage(parsed.usage);
|
|
232
|
+
continue;
|
|
233
|
+
}
|
|
234
|
+
const delta = choice.delta ?? {};
|
|
235
|
+
if (typeof delta.content === 'string' && delta.content.length > 0) {
|
|
236
|
+
yield { type: 'text-delta', delta: delta.content };
|
|
237
|
+
}
|
|
238
|
+
if (typeof (delta as { reasoning_content?: unknown }).reasoning_content === 'string') {
|
|
239
|
+
const r = (delta as { reasoning_content: string }).reasoning_content;
|
|
240
|
+
if (r.length > 0) yield { type: 'reasoning-delta', delta: r };
|
|
241
|
+
}
|
|
242
|
+
if (Array.isArray(delta.tool_calls)) {
|
|
243
|
+
for (const tc of delta.tool_calls) {
|
|
244
|
+
const idx = tc.index ?? 0;
|
|
245
|
+
const slot = toolCallBuffer.get(idx) ?? {
|
|
246
|
+
id: '',
|
|
247
|
+
name: '',
|
|
248
|
+
args: '',
|
|
249
|
+
emittedStart: false,
|
|
250
|
+
};
|
|
251
|
+
if (typeof tc.id === 'string' && tc.id.length > 0) slot.id = tc.id;
|
|
252
|
+
if (tc.function?.name !== undefined) slot.name = tc.function.name;
|
|
253
|
+
if (typeof tc.function?.arguments === 'string') slot.args += tc.function.arguments;
|
|
254
|
+
toolCallBuffer.set(idx, slot);
|
|
255
|
+
if (!slot.emittedStart && slot.id.length > 0 && slot.name.length > 0) {
|
|
256
|
+
yield {
|
|
257
|
+
type: 'tool-call-start',
|
|
258
|
+
toolCallId: slot.id,
|
|
259
|
+
toolName: slot.name,
|
|
260
|
+
};
|
|
261
|
+
slot.emittedStart = true;
|
|
262
|
+
}
|
|
263
|
+
if (typeof tc.function?.arguments === 'string' && tc.function.arguments.length > 0) {
|
|
264
|
+
yield {
|
|
265
|
+
type: 'tool-call-input-delta',
|
|
266
|
+
toolCallId: slot.id,
|
|
267
|
+
argsDelta: tc.function.arguments,
|
|
268
|
+
};
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
if (typeof choice.finish_reason === 'string') {
|
|
273
|
+
finishReason = mapFinishReason(choice.finish_reason);
|
|
274
|
+
for (const slot of toolCallBuffer.values()) {
|
|
275
|
+
const finalArgs = parseFinalArgs(slot.args);
|
|
276
|
+
yield {
|
|
277
|
+
type: 'tool-call-end',
|
|
278
|
+
toolCallId: slot.id,
|
|
279
|
+
finalArgs,
|
|
280
|
+
};
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
if (parsed.usage !== undefined) usage = mapOpenAIUsage(parsed.usage);
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
yield {
|
|
287
|
+
type: 'finish',
|
|
288
|
+
finishReason,
|
|
289
|
+
usage: usage ?? { promptTokens: 0, completionTokens: 0, totalTokens: 0 },
|
|
290
|
+
};
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
async function generateOpenAIShaped(
|
|
294
|
+
opts: OpenAIShapedOptions,
|
|
295
|
+
url: string,
|
|
296
|
+
req: ProviderRequest,
|
|
297
|
+
): Promise<ProviderResponse> {
|
|
298
|
+
const body = buildBody(
|
|
299
|
+
opts.model,
|
|
300
|
+
req,
|
|
301
|
+
false,
|
|
302
|
+
opts.capabilities?.structuredOutput ?? true,
|
|
303
|
+
conversionOptionsFor(opts),
|
|
304
|
+
);
|
|
305
|
+
const resp = await callJsonHttp({
|
|
306
|
+
providerName: opts.providerName,
|
|
307
|
+
url,
|
|
308
|
+
headers: buildHeaders(opts),
|
|
309
|
+
body,
|
|
310
|
+
...(req.signal !== undefined ? { signal: req.signal } : {}),
|
|
311
|
+
...(opts.fetchImpl !== undefined ? { fetchImpl: opts.fetchImpl } : {}),
|
|
312
|
+
...(opts.timeoutMs !== undefined ? { timeoutMs: opts.timeoutMs } : {}),
|
|
313
|
+
});
|
|
314
|
+
let json: OpenAIChunk;
|
|
315
|
+
try {
|
|
316
|
+
json = (await resp.json()) as OpenAIChunk;
|
|
317
|
+
} catch (cause) {
|
|
318
|
+
throw new ProviderStreamParseError(
|
|
319
|
+
opts.providerName,
|
|
320
|
+
'response body was not valid JSON',
|
|
321
|
+
cause,
|
|
322
|
+
);
|
|
323
|
+
}
|
|
324
|
+
const choice = json.choices?.[0];
|
|
325
|
+
const message = choice?.message ?? {};
|
|
326
|
+
const text = typeof message.content === 'string' ? message.content : undefined;
|
|
327
|
+
const toolCalls = Array.isArray(message.tool_calls)
|
|
328
|
+
? message.tool_calls.map((tc) => ({
|
|
329
|
+
toolCallId: tc.id ?? '',
|
|
330
|
+
toolName: tc.function?.name ?? '',
|
|
331
|
+
args: parseFinalArgs(
|
|
332
|
+
typeof tc.function?.arguments === 'string' ? tc.function.arguments : '',
|
|
333
|
+
),
|
|
334
|
+
}))
|
|
335
|
+
: undefined;
|
|
336
|
+
const usage = mapOpenAIUsage(json.usage) ?? {
|
|
337
|
+
promptTokens: 0,
|
|
338
|
+
completionTokens: 0,
|
|
339
|
+
totalTokens: 0,
|
|
340
|
+
};
|
|
341
|
+
return {
|
|
342
|
+
usage,
|
|
343
|
+
finishReason: mapFinishReason(choice?.finish_reason),
|
|
344
|
+
...(text !== undefined ? { text } : {}),
|
|
345
|
+
...(toolCalls !== undefined ? { toolCalls } : {}),
|
|
346
|
+
};
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
function buildBody(
|
|
350
|
+
model: string,
|
|
351
|
+
req: ProviderRequest,
|
|
352
|
+
stream: boolean,
|
|
353
|
+
structuredOutput: boolean,
|
|
354
|
+
conversion: ChatMessageConversionOptions,
|
|
355
|
+
): Record<string, unknown> {
|
|
356
|
+
const messages =
|
|
357
|
+
req.systemMessage !== undefined
|
|
358
|
+
? [{ role: 'system' as const, content: req.systemMessage }, ...req.messages]
|
|
359
|
+
: req.messages;
|
|
360
|
+
const body: Record<string, unknown> = {
|
|
361
|
+
model,
|
|
362
|
+
messages: toOpenAIChatMessages(messages, conversion),
|
|
363
|
+
stream,
|
|
364
|
+
};
|
|
365
|
+
if (stream) {
|
|
366
|
+
// core-provider-09: vanilla OpenAI-shaped servers (vLLM, Together,
|
|
367
|
+
// OpenAI proper) only emit the final usage chunk when asked;
|
|
368
|
+
// without it streamed calls finish with {0,0,0} usage and cost
|
|
369
|
+
// tracking silently zeroes. llama.cpp accepts the flag too.
|
|
370
|
+
// `providerOptions` is merged last, so callers can override it for
|
|
371
|
+
// servers that reject the field.
|
|
372
|
+
body.stream_options = { include_usage: true };
|
|
373
|
+
}
|
|
374
|
+
if (req.temperature !== undefined) body.temperature = req.temperature;
|
|
375
|
+
if (req.maxTokens !== undefined) body.max_tokens = req.maxTokens;
|
|
376
|
+
if (req.tools !== undefined && req.tools.length > 0) {
|
|
377
|
+
// C2: fold worked examples in the adapter itself (idempotent when an
|
|
378
|
+
// upstream createProvider fold already ran).
|
|
379
|
+
body.tools = foldToolExamples(req.tools).map((t) => ({
|
|
380
|
+
type: 'function',
|
|
381
|
+
function: {
|
|
382
|
+
name: t.name,
|
|
383
|
+
...(t.description !== undefined ? { description: t.description } : {}),
|
|
384
|
+
parameters: t.inputSchema,
|
|
385
|
+
},
|
|
386
|
+
}));
|
|
387
|
+
}
|
|
388
|
+
if (req.toolChoice !== undefined) {
|
|
389
|
+
body.tool_choice = mapToolChoice(req.toolChoice);
|
|
390
|
+
}
|
|
391
|
+
// PS-24: structured output finally reaches the wire - gated on the
|
|
392
|
+
// declared capability so a structuredOutput:false override keeps the
|
|
393
|
+
// request clean for servers that reject response_format.
|
|
394
|
+
if (structuredOutput && req.outputType?.kind === 'structured') {
|
|
395
|
+
body.response_format =
|
|
396
|
+
req.outputType.jsonSchema !== undefined
|
|
397
|
+
? {
|
|
398
|
+
type: 'json_schema',
|
|
399
|
+
json_schema: { name: 'output', schema: req.outputType.jsonSchema, strict: true },
|
|
400
|
+
}
|
|
401
|
+
: { type: 'json_object' };
|
|
402
|
+
}
|
|
403
|
+
if (req.providerOptions !== undefined) {
|
|
404
|
+
Object.assign(body, req.providerOptions);
|
|
405
|
+
}
|
|
406
|
+
return body;
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
function buildHeaders(opts: OpenAIShapedOptions): Record<string, string> {
|
|
410
|
+
const headers: Record<string, string> = {
|
|
411
|
+
'content-type': 'application/json',
|
|
412
|
+
accept: 'application/json',
|
|
413
|
+
...opts.headers,
|
|
414
|
+
};
|
|
415
|
+
if (opts.apiKey !== undefined && opts.apiKey.length > 0) {
|
|
416
|
+
headers.authorization = `Bearer ${opts.apiKey}`;
|
|
417
|
+
}
|
|
418
|
+
return headers;
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
function mapToolChoice(value: unknown): unknown {
|
|
422
|
+
if (value === 'auto' || value === 'none' || value === 'required') return value;
|
|
423
|
+
if (typeof value === 'object' && value !== null && 'tool' in value) {
|
|
424
|
+
return { type: 'function', function: { name: (value as { tool: string }).tool } };
|
|
425
|
+
}
|
|
426
|
+
return 'auto';
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
function mapFinishReason(value: string | null | undefined): FinishReason {
|
|
430
|
+
switch (value) {
|
|
431
|
+
case 'stop':
|
|
432
|
+
case 'length':
|
|
433
|
+
case 'content-filter':
|
|
434
|
+
return value;
|
|
435
|
+
// PS-12: the OpenAI wire format spells it with an underscore; the dashed
|
|
436
|
+
// form alone never matched, so content-filtered completions reported 'stop'.
|
|
437
|
+
case 'content_filter':
|
|
438
|
+
return 'content-filter';
|
|
439
|
+
case 'tool_calls':
|
|
440
|
+
case 'function_call':
|
|
441
|
+
return 'tool-calls';
|
|
442
|
+
case null:
|
|
443
|
+
case undefined:
|
|
444
|
+
return 'stop';
|
|
445
|
+
default:
|
|
446
|
+
return 'stop';
|
|
447
|
+
}
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
function mapOpenAIUsage(input: unknown): Usage | undefined {
|
|
451
|
+
if (input === undefined || input === null || typeof input !== 'object') return undefined;
|
|
452
|
+
const u = input as {
|
|
453
|
+
prompt_tokens?: number;
|
|
454
|
+
completion_tokens?: number;
|
|
455
|
+
total_tokens?: number;
|
|
456
|
+
reasoning_tokens?: number;
|
|
457
|
+
// OpenAI reports cache reads as a SUBSET of prompt_tokens
|
|
458
|
+
// (core-provider-02); there is no cache-write leg on this wire.
|
|
459
|
+
prompt_tokens_details?: { cached_tokens?: number };
|
|
460
|
+
};
|
|
461
|
+
const promptTokens = u.prompt_tokens ?? 0;
|
|
462
|
+
const completionTokens = u.completion_tokens ?? 0;
|
|
463
|
+
const totalTokens = u.total_tokens ?? promptTokens + completionTokens;
|
|
464
|
+
const cachedTokens = u.prompt_tokens_details?.cached_tokens;
|
|
465
|
+
const usage: Usage = {
|
|
466
|
+
promptTokens,
|
|
467
|
+
completionTokens,
|
|
468
|
+
totalTokens,
|
|
469
|
+
...(u.reasoning_tokens !== undefined ? { reasoningTokens: u.reasoning_tokens } : {}),
|
|
470
|
+
...(typeof cachedTokens === 'number' && Number.isFinite(cachedTokens) && cachedTokens > 0
|
|
471
|
+
? { cachedReadTokens: cachedTokens }
|
|
472
|
+
: {}),
|
|
473
|
+
};
|
|
474
|
+
return usage;
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
function parseFinalArgs(raw: string): unknown {
|
|
478
|
+
if (raw.length === 0) return {};
|
|
479
|
+
try {
|
|
480
|
+
return JSON.parse(raw);
|
|
481
|
+
} catch {
|
|
482
|
+
return raw;
|
|
483
|
+
}
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
function sameSensitivity(a: ReadonlyArray<Sensitivity>, b: ReadonlyArray<Sensitivity>): boolean {
|
|
487
|
+
if (a.length !== b.length) return false;
|
|
488
|
+
for (let i = 0; i < a.length; i++) {
|
|
489
|
+
if (a[i] !== b[i]) return false;
|
|
490
|
+
}
|
|
491
|
+
return true;
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
function defaultLogger(level: 'warn' | 'info', message: string, meta?: object): void {
|
|
495
|
+
const fn = level === 'warn' ? console.warn : console.info;
|
|
496
|
+
if (meta !== undefined) {
|
|
497
|
+
fn(`[graphorin/provider] ${message}`, meta);
|
|
498
|
+
} else {
|
|
499
|
+
fn(`[graphorin/provider] ${message}`);
|
|
500
|
+
}
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
/**
|
|
504
|
+
* W-095: one dropped-content WARN per adapter INSTANCE (keyed on the
|
|
505
|
+
* factory's options object, following KNOWN_LOOPBACK_OVERRIDES_WARNED),
|
|
506
|
+
* so a chat loop does not repeat it every call.
|
|
507
|
+
*/
|
|
508
|
+
const droppedContentWarned = new WeakSet<object>();
|
|
509
|
+
|
|
510
|
+
function conversionOptionsFor(opts: {
|
|
511
|
+
readonly capabilities?: { readonly multimodal?: boolean };
|
|
512
|
+
readonly logger?: (level: 'warn' | 'info', message: string, meta?: object) => void;
|
|
513
|
+
}): ChatMessageConversionOptions {
|
|
514
|
+
const log = opts.logger ?? defaultLogger;
|
|
515
|
+
return {
|
|
516
|
+
multimodal: opts.capabilities?.multimodal ?? false,
|
|
517
|
+
warn: (message) => {
|
|
518
|
+
if (droppedContentWarned.has(opts)) return;
|
|
519
|
+
droppedContentWarned.add(opts);
|
|
520
|
+
log('warn', message);
|
|
521
|
+
},
|
|
522
|
+
};
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
interface OpenAIChunk {
|
|
526
|
+
readonly id?: string;
|
|
527
|
+
readonly choices?: ReadonlyArray<{
|
|
528
|
+
readonly delta?: {
|
|
529
|
+
readonly content?: string;
|
|
530
|
+
readonly tool_calls?: ReadonlyArray<{
|
|
531
|
+
readonly index?: number;
|
|
532
|
+
readonly id?: string;
|
|
533
|
+
readonly type?: string;
|
|
534
|
+
readonly function?: { readonly name?: string; readonly arguments?: string };
|
|
535
|
+
}>;
|
|
536
|
+
readonly reasoning_content?: string;
|
|
537
|
+
};
|
|
538
|
+
readonly message?: {
|
|
539
|
+
readonly content?: string | null;
|
|
540
|
+
readonly tool_calls?: ReadonlyArray<{
|
|
541
|
+
readonly id?: string;
|
|
542
|
+
readonly type?: string;
|
|
543
|
+
readonly function?: { readonly name?: string; readonly arguments?: string };
|
|
544
|
+
}>;
|
|
545
|
+
};
|
|
546
|
+
readonly finish_reason?: string | null;
|
|
547
|
+
}>;
|
|
548
|
+
readonly usage?: {
|
|
549
|
+
readonly prompt_tokens?: number;
|
|
550
|
+
readonly completion_tokens?: number;
|
|
551
|
+
readonly total_tokens?: number;
|
|
552
|
+
readonly reasoning_tokens?: number;
|
|
553
|
+
readonly prompt_tokens_details?: { readonly cached_tokens?: number };
|
|
554
|
+
};
|
|
555
|
+
}
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Minimal `text/event-stream` chunk parser. Lives here because three
|
|
3
|
+
* adapters consume the same shape (`llamaCppServerAdapter`,
|
|
4
|
+
* `openAICompatibleAdapter`, plus the OpenAI-compatible legacy path
|
|
5
|
+
* inside `ollamaAdapter`). Keeping it local avoids pulling in a
|
|
6
|
+
* dedicated SSE package.
|
|
7
|
+
*
|
|
8
|
+
* @internal
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Parse a Web `Response`'s body as Server-Sent Events. Yields one
|
|
13
|
+
* payload per `data: <payload>` line; ignores `event:` / `id:` / `:`
|
|
14
|
+
* comments lines per the SSE spec. Terminates on `data: [DONE]` per
|
|
15
|
+
* the OpenAI-compatible convention.
|
|
16
|
+
*
|
|
17
|
+
* @internal
|
|
18
|
+
*/
|
|
19
|
+
export async function* parseEventStream(
|
|
20
|
+
body: ReadableStream<Uint8Array> | null,
|
|
21
|
+
options: { readonly signal?: AbortSignal } = {},
|
|
22
|
+
): AsyncIterable<string> {
|
|
23
|
+
if (body === null) return;
|
|
24
|
+
const reader = body.getReader();
|
|
25
|
+
const decoder = new TextDecoder('utf-8');
|
|
26
|
+
let buffer = '';
|
|
27
|
+
try {
|
|
28
|
+
while (true) {
|
|
29
|
+
if (options.signal?.aborted) return;
|
|
30
|
+
const { value, done } = await reader.read();
|
|
31
|
+
if (done) break;
|
|
32
|
+
buffer += decoder.decode(value, { stream: true });
|
|
33
|
+
let separatorIdx = findSeparator(buffer);
|
|
34
|
+
while (separatorIdx !== -1) {
|
|
35
|
+
const eventBlock = buffer.slice(0, separatorIdx);
|
|
36
|
+
buffer = buffer.slice(separatorIdx + matchSeparatorLength(buffer, separatorIdx));
|
|
37
|
+
const dataPayload = extractDataLines(eventBlock);
|
|
38
|
+
if (dataPayload === '[DONE]') return;
|
|
39
|
+
if (dataPayload !== null) yield dataPayload;
|
|
40
|
+
separatorIdx = findSeparator(buffer);
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
const tail = buffer.trim();
|
|
44
|
+
if (tail.length > 0) {
|
|
45
|
+
const dataPayload = extractDataLines(tail);
|
|
46
|
+
if (dataPayload !== null && dataPayload !== '[DONE]') yield dataPayload;
|
|
47
|
+
}
|
|
48
|
+
} finally {
|
|
49
|
+
reader.releaseLock();
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function findSeparator(buffer: string): number {
|
|
54
|
+
const lf = buffer.indexOf('\n\n');
|
|
55
|
+
const crlf = buffer.indexOf('\r\n\r\n');
|
|
56
|
+
if (lf === -1) return crlf;
|
|
57
|
+
if (crlf === -1) return lf;
|
|
58
|
+
return lf < crlf ? lf : crlf;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function matchSeparatorLength(buffer: string, idx: number): number {
|
|
62
|
+
return buffer.startsWith('\r\n\r\n', idx) ? 4 : 2;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function extractDataLines(block: string): string | null {
|
|
66
|
+
const dataLines: string[] = [];
|
|
67
|
+
for (const rawLine of block.split(/\r?\n/)) {
|
|
68
|
+
const line = rawLine;
|
|
69
|
+
if (line.startsWith(':')) continue;
|
|
70
|
+
if (line.startsWith('data:')) {
|
|
71
|
+
dataLines.push(line.slice(5).replace(/^ /, ''));
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
if (dataLines.length === 0) return null;
|
|
75
|
+
return dataLines.join('\n');
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* Parse a newline-delimited JSON stream. Used by the native Ollama
|
|
80
|
+
* `/api/chat` path which emits one JSON object per `\n`-terminated
|
|
81
|
+
* line (NOT SSE).
|
|
82
|
+
*
|
|
83
|
+
* @internal
|
|
84
|
+
*/
|
|
85
|
+
export async function* parseNdJsonStream(
|
|
86
|
+
body: ReadableStream<Uint8Array> | null,
|
|
87
|
+
options: { readonly signal?: AbortSignal } = {},
|
|
88
|
+
): AsyncIterable<string> {
|
|
89
|
+
if (body === null) return;
|
|
90
|
+
const reader = body.getReader();
|
|
91
|
+
const decoder = new TextDecoder('utf-8');
|
|
92
|
+
let buffer = '';
|
|
93
|
+
try {
|
|
94
|
+
while (true) {
|
|
95
|
+
if (options.signal?.aborted) return;
|
|
96
|
+
const { value, done } = await reader.read();
|
|
97
|
+
if (done) break;
|
|
98
|
+
buffer += decoder.decode(value, { stream: true });
|
|
99
|
+
let nlIdx = buffer.indexOf('\n');
|
|
100
|
+
while (nlIdx !== -1) {
|
|
101
|
+
const line = buffer.slice(0, nlIdx).trim();
|
|
102
|
+
buffer = buffer.slice(nlIdx + 1);
|
|
103
|
+
if (line.length > 0) yield line;
|
|
104
|
+
nlIdx = buffer.indexOf('\n');
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
const tail = buffer.trim();
|
|
108
|
+
if (tail.length > 0) yield tail;
|
|
109
|
+
} finally {
|
|
110
|
+
reader.releaseLock();
|
|
111
|
+
}
|
|
112
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tiny URL helpers shared between the provider adapters. Kept regex-free
|
|
3
|
+
* so CodeQL does not flag the (otherwise harmless) `/+` quantifier on
|
|
4
|
+
* adapter base URLs.
|
|
5
|
+
*
|
|
6
|
+
* @internal
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Strip every trailing `/` from a URL string. Operates in `O(n)` time
|
|
11
|
+
* with a single linear scan from the end, so it is safe to call on
|
|
12
|
+
* adversarial inputs without bounding the length first.
|
|
13
|
+
*
|
|
14
|
+
* @internal
|
|
15
|
+
*/
|
|
16
|
+
export function stripTrailingSlashes(url: string): string {
|
|
17
|
+
let end = url.length;
|
|
18
|
+
while (end > 0 && url.charCodeAt(end - 1) === 0x2f /* '/' */) end -= 1;
|
|
19
|
+
return end === url.length ? url : url.slice(0, end);
|
|
20
|
+
}
|