@b2y/ai-service 0.0.0-stage → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +163 -2
- package/package.json +77 -4
- package/src/analytics/Analytics.js +581 -0
- package/src/analytics/Results.js +394 -0
- package/src/constants/AnalyticsConstants.js +6 -0
- package/src/constants/AssistantConstants.js +5 -0
- package/src/constants/ConversationConstants.js +5 -0
- package/src/constants/DocumentConstants.js +4 -0
- package/src/constants/ProviderConstants.js +23 -0
- package/src/constants/ServerConstants.js +15 -0
- package/src/constants/SqlConstants.js +33 -0
- package/src/constants/UiConstants.js +29 -0
- package/src/conversations/ConversationHistory.js +191 -0
- package/src/conversations/FolderConversationStore.js +103 -0
- package/src/copilot/Assistant.js +459 -0
- package/src/copilot/Capabilities.js +343 -0
- package/src/core/AiService.js +520 -0
- package/src/core/Context.js +243 -0
- package/src/core/Errors.js +249 -0
- package/src/core/Tasks.js +190 -0
- package/src/documents/Documents.js +450 -0
- package/src/documents/FolderDocumentStore.js +167 -0
- package/src/documents/PdfText.js +50 -0
- package/src/enums/ChartType.js +1 -0
- package/src/enums/DataClassification.js +2 -0
- package/src/enums/ProviderType.js +1 -0
- package/src/index.js +65 -0
- package/src/providers/Providers.js +733 -0
- package/src/server/Server.js +153 -0
- package/src/sql/PostgresSqlValidator.js +983 -0
- package/src/sql/SqlPolicy.js +483 -0
- package/src/ui/react/AssistantChat.js +594 -0
- package/src/ui/react/styles.css +239 -0
|
@@ -0,0 +1,733 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import { dataClassificationSchema } from "../core/Context.js";
|
|
3
|
+
import { AiServiceError, createCancelledError, ErrorCodes, summarizeZodError, systemErrorCode } from "../core/Errors.js";
|
|
4
|
+
import { APPROVED_PAID_HOSTS, DEFAULT_MAX_REQUEST_BYTES, DEFAULT_MAX_RESPONSE_BYTES, GROQ_BASE_URL, GROQ_DEFAULT_MODEL, GROQ_DEFAULT_TIMEOUT_MS } from "../constants/ProviderConstants.js";
|
|
5
|
+
import { PROVIDER_TYPE } from "../enums/ProviderType.js";
|
|
6
|
+
export { APPROVED_PAID_HOSTS, DEFAULT_MAX_REQUEST_BYTES, DEFAULT_MAX_RESPONSE_BYTES, GROQ_BASE_URL, GROQ_DEFAULT_MODEL, GROQ_DEFAULT_TIMEOUT_MS } from "../constants/ProviderConstants.js";
|
|
7
|
+
export { PROVIDER_TYPE } from "../enums/ProviderType.js";
|
|
8
|
+
|
|
9
|
+
const PROVIDER_TYPES = Object.freeze(Object.values(PROVIDER_TYPE));
|
|
10
|
+
|
|
11
|
+
// Types only: the contract every model backend implements (documented for editors via JSDoc).
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* @typedef {"SELF_HOSTED" | "PAID"} ProviderKind
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* How the provider is asked to constrain output. Support differs between servers and versions (vLLM and others), so this is a per-deployment setting confirmed by integration tests, never assumed. Output is always validated with Zod afterwards regardless of the mode.
|
|
19
|
+
* @typedef {"json_schema" | "json_object" | "none"} StructuredOutputMode
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* @typedef {Object} ProviderInfo
|
|
24
|
+
* @property {string} id
|
|
25
|
+
* @property {ProviderKind} kind
|
|
26
|
+
* @property {string} model
|
|
27
|
+
* @property {string | null} modelRevision
|
|
28
|
+
* @property {readonly DataClassification[]} allowedClassifications
|
|
29
|
+
* @property {StructuredOutputMode} structuredOutput
|
|
30
|
+
*/
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Only system and user roles exist: there is no tool or function-call role.
|
|
34
|
+
* @typedef {Object} ChatMessage
|
|
35
|
+
* @property {"system" | "user"} role
|
|
36
|
+
* @property {string} content
|
|
37
|
+
*/
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* @typedef {Object} GenerationSettings
|
|
41
|
+
* @property {number} temperature
|
|
42
|
+
* @property {number} maxTokens
|
|
43
|
+
* @property {number} [topP]
|
|
44
|
+
* @property {number} [seed]
|
|
45
|
+
*/
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* @typedef {Object} JsonSchemaFormat
|
|
49
|
+
* @property {string} name
|
|
50
|
+
* @property {object} schema
|
|
51
|
+
*/
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Deliberately has no `tools` field: the model cannot be offered tools.
|
|
55
|
+
* @typedef {Object} ChatRequest
|
|
56
|
+
* @property {readonly ChatMessage[]} messages
|
|
57
|
+
* @property {GenerationSettings} generation
|
|
58
|
+
* @property {JsonSchemaFormat} [jsonSchema]
|
|
59
|
+
*/
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* @typedef {"stop" | "length" | "other"} FinishReason
|
|
63
|
+
*/
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* @typedef {Object} ChatResult
|
|
67
|
+
* @property {string} content
|
|
68
|
+
* @property {FinishReason} finishReason
|
|
69
|
+
* @property {TokenUsage} usage
|
|
70
|
+
* @property {string} model
|
|
71
|
+
* @property {number} latencyMs
|
|
72
|
+
*/
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* @typedef {Object} CallOptions
|
|
76
|
+
* @property {AbortSignal} [signal]
|
|
77
|
+
*/
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Contract every model backend implements. Implementations must throw AiServiceError only, and must never put prompt text, model output or raw response bodies into errors.
|
|
81
|
+
* @typedef {Object} ChatProvider
|
|
82
|
+
* @property {ProviderInfo} info
|
|
83
|
+
* @property {(request: ChatRequest, options?: CallOptions) => Promise<ChatResult>} complete
|
|
84
|
+
* @property {(options?: CallOptions) => Promise<void>} healthCheck
|
|
85
|
+
*/
|
|
86
|
+
|
|
87
|
+
export {};
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* @typedef {Object} EndpointPolicyOptions
|
|
91
|
+
* @property {ProviderKind} providerKind
|
|
92
|
+
* @property {boolean} allowInsecurePrivateHttp - Permit plain HTTP to allow-listed private IPs and single-label hosts (for example "vllm" on a Docker network).
|
|
93
|
+
* @property {readonly string[]} allowedHosts - Administrator-approved hosts for SELF_HOSTED providers. Every self-hosted host must be listed. Ignored for PAID.
|
|
94
|
+
*/
|
|
95
|
+
|
|
96
|
+
const HOSTNAME_PATTERN = /^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?(\.[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?)*$/;
|
|
97
|
+
const IPV4_PATTERN = /^\d{1,3}(\.\d{1,3}){3}$/;
|
|
98
|
+
|
|
99
|
+
/** Validates a provider base URL before any request is made. */
|
|
100
|
+
export function resolveProviderBaseUrl(baseUrl, options) {
|
|
101
|
+
const url = parseBaseUrl(baseUrl);
|
|
102
|
+
const host = hostOf(url);
|
|
103
|
+
const hostClass = classifyHost(host);
|
|
104
|
+
|
|
105
|
+
if (hostClass === "forbidden") {
|
|
106
|
+
throw configError("Provider host is not permitted", "forbidden_address");
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
if (options.providerKind === "PAID") {
|
|
110
|
+
if (url.protocol !== "https:" || url.port !== "" || !APPROVED_PAID_HOSTS.includes(host)) {
|
|
111
|
+
throw configError("Paid providers require HTTPS to an approved hostname", "paid_endpoint_not_allowed");
|
|
112
|
+
}
|
|
113
|
+
return withTrailingSlash(url);
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
if (APPROVED_PAID_HOSTS.includes(host)) {
|
|
117
|
+
throw configError("Hosted provider endpoints must be configured as PAID providers", "paid_host_requires_paid_kind");
|
|
118
|
+
}
|
|
119
|
+
const allowedHosts = options.allowedHosts.map(normalizeAllowedHost);
|
|
120
|
+
if (!allowedHosts.includes(host)) {
|
|
121
|
+
throw configError("Self-hosted provider host is not in the administrator allow-list", "host_not_allowlisted");
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
if (url.protocol === "http:") {
|
|
125
|
+
const privateNetwork = hostClass === "private" || hostClass === "internal-name";
|
|
126
|
+
if (hostClass !== "loopback" && !(privateNetwork && options.allowInsecurePrivateHttp)) {
|
|
127
|
+
throw configError(
|
|
128
|
+
"Plain HTTP is only allowed for loopback, or private hosts when allowInsecurePrivateHttp is set",
|
|
129
|
+
"insecure_http",
|
|
130
|
+
);
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
return withTrailingSlash(url);
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* Normalises an allow-list entry the same way URLs are normalised, so "LOCALHOST", "127.1" and "::1" compare
|
|
139
|
+
* correctly.
|
|
140
|
+
*/
|
|
141
|
+
export function normalizeAllowedHost(entry) {
|
|
142
|
+
const bracketed = entry.includes(":") ? `[${entry.replace(/^\[|\]$/g, "")}]` : entry;
|
|
143
|
+
let url;
|
|
144
|
+
try {
|
|
145
|
+
url = new URL(`http://${bracketed}/`);
|
|
146
|
+
} catch {
|
|
147
|
+
throw configError("Allow-listed host is invalid", "invalid_allowed_host");
|
|
148
|
+
}
|
|
149
|
+
if (url.port !== "" || url.pathname !== "/" || url.search !== "" || url.hash !== "" || url.username !== "") {
|
|
150
|
+
throw configError("Allow-listed host must be a bare hostname or IP address", "invalid_allowed_host");
|
|
151
|
+
}
|
|
152
|
+
const host = hostOf(url);
|
|
153
|
+
if (classifyHost(host) === "forbidden") {
|
|
154
|
+
throw configError("Allow-listed host is not permitted", "forbidden_address");
|
|
155
|
+
}
|
|
156
|
+
return host;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/** Returns the base URL's host if it is loopback; otherwise throws CONFIG_INVALID. */
|
|
160
|
+
export function loopbackHostOf(baseUrl) {
|
|
161
|
+
const host = hostOf(parseBaseUrl(baseUrl));
|
|
162
|
+
if (classifyHost(host) !== "loopback") {
|
|
163
|
+
throw configError("This provider preset only allows loopback endpoints", "loopback_required");
|
|
164
|
+
}
|
|
165
|
+
return host;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
function parseBaseUrl(baseUrl) {
|
|
169
|
+
let url;
|
|
170
|
+
try {
|
|
171
|
+
url = new URL(baseUrl);
|
|
172
|
+
} catch {
|
|
173
|
+
throw configError("Provider baseUrl is not a valid URL", "invalid_url");
|
|
174
|
+
}
|
|
175
|
+
if (url.protocol !== "https:" && url.protocol !== "http:") {
|
|
176
|
+
throw configError("Provider baseUrl must use http or https", "unsupported_protocol");
|
|
177
|
+
}
|
|
178
|
+
if (url.username !== "" || url.password !== "") {
|
|
179
|
+
throw configError("Provider baseUrl must not contain credentials", "credentials_in_url");
|
|
180
|
+
}
|
|
181
|
+
if (url.search !== "" || url.hash !== "") {
|
|
182
|
+
throw configError("Provider baseUrl must not contain a query or fragment", "query_or_fragment");
|
|
183
|
+
}
|
|
184
|
+
return url;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
function hostOf(url) {
|
|
188
|
+
return url.hostname.replace(/^\[(.*)\]$/, "$1").toLowerCase();
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
function classifyHost(host) {
|
|
192
|
+
if (host === "localhost") {
|
|
193
|
+
return "loopback";
|
|
194
|
+
}
|
|
195
|
+
if (IPV4_PATTERN.test(host)) {
|
|
196
|
+
return classifyIPv4(host);
|
|
197
|
+
}
|
|
198
|
+
if (host.includes(":")) {
|
|
199
|
+
return classifyIPv6(host);
|
|
200
|
+
}
|
|
201
|
+
if (host.length > 253 || !HOSTNAME_PATTERN.test(host)) {
|
|
202
|
+
return "forbidden";
|
|
203
|
+
}
|
|
204
|
+
return host.includes(".") ? "public" : "internal-name";
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
function classifyIPv4(host) {
|
|
208
|
+
const octets = host.split(".").map(Number);
|
|
209
|
+
const [a = -1, b = -1] = octets;
|
|
210
|
+
if (octets.some((octet) => octet > 255)) {
|
|
211
|
+
return "forbidden";
|
|
212
|
+
}
|
|
213
|
+
if (a === 127) {
|
|
214
|
+
return "loopback";
|
|
215
|
+
}
|
|
216
|
+
if (a === 0 || a >= 224 || (a === 169 && b === 254)) {
|
|
217
|
+
return "forbidden";
|
|
218
|
+
}
|
|
219
|
+
if (a === 10 || (a === 172 && b >= 16 && b <= 31) || (a === 192 && b === 168)) {
|
|
220
|
+
return "private";
|
|
221
|
+
}
|
|
222
|
+
return "public";
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
function classifyIPv6(host) {
|
|
226
|
+
if (host === "::1") {
|
|
227
|
+
return "loopback";
|
|
228
|
+
}
|
|
229
|
+
if (host === "::" || host.startsWith("::ffff:")) {
|
|
230
|
+
return "forbidden";
|
|
231
|
+
}
|
|
232
|
+
const firstHextet = Number.parseInt(host.split(":")[0] ?? "", 16);
|
|
233
|
+
if (Number.isNaN(firstHextet)) {
|
|
234
|
+
return "forbidden";
|
|
235
|
+
}
|
|
236
|
+
if ((firstHextet & 0xffc0) === 0xfe80 || (firstHextet & 0xff00) === 0xff00) {
|
|
237
|
+
return "forbidden";
|
|
238
|
+
}
|
|
239
|
+
if ((firstHextet & 0xfe00) === 0xfc00) {
|
|
240
|
+
return "private";
|
|
241
|
+
}
|
|
242
|
+
return "public";
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
function withTrailingSlash(url) {
|
|
246
|
+
const normalized = new URL(url.href);
|
|
247
|
+
if (!normalized.pathname.endsWith("/")) {
|
|
248
|
+
normalized.pathname = `${normalized.pathname}/`;
|
|
249
|
+
}
|
|
250
|
+
return normalized;
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
function configError(message, reason) {
|
|
254
|
+
return new AiServiceError(ErrorCodes.CONFIG_INVALID, message, { details: { reason } });
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
/**
|
|
258
|
+
* @typedef {Object} ProviderDependencies
|
|
259
|
+
* @property {typeof globalThis.fetch} [fetch] - Injected for tests. Defaults to the global fetch, resolved at call time.
|
|
260
|
+
* @property {() => number} [now]
|
|
261
|
+
*/
|
|
262
|
+
|
|
263
|
+
/** Network failures worth retrying: the server was briefly unreachable or the connection dropped. */
|
|
264
|
+
const TRANSIENT_NETWORK_CODES = new Set([
|
|
265
|
+
"ECONNREFUSED",
|
|
266
|
+
"ECONNRESET",
|
|
267
|
+
"ECONNABORTED",
|
|
268
|
+
"ETIMEDOUT",
|
|
269
|
+
"EPIPE",
|
|
270
|
+
"EAI_AGAIN",
|
|
271
|
+
"EHOSTUNREACH",
|
|
272
|
+
"ENETUNREACH",
|
|
273
|
+
"UND_ERR_SOCKET",
|
|
274
|
+
"UND_ERR_CLOSED",
|
|
275
|
+
"UND_ERR_CONNECT_TIMEOUT",
|
|
276
|
+
"UND_ERR_HEADERS_TIMEOUT",
|
|
277
|
+
"UND_ERR_BODY_TIMEOUT",
|
|
278
|
+
]);
|
|
279
|
+
|
|
280
|
+
const providerConfigSchema = z.strictObject({
|
|
281
|
+
id: z.string().regex(/^[a-z][a-z0-9-]{0,63}$/),
|
|
282
|
+
kind: z.enum(["SELF_HOSTED", "PAID"]),
|
|
283
|
+
baseUrl: z.string().min(1).max(2048),
|
|
284
|
+
model: z.string().min(1).max(200),
|
|
285
|
+
modelRevision: z.string().min(1).max(200).optional(),
|
|
286
|
+
// Visible ASCII only, so the key is a valid header value and can never
|
|
287
|
+
// surface in a header-validation error.
|
|
288
|
+
apiKey: z
|
|
289
|
+
.string()
|
|
290
|
+
.min(1)
|
|
291
|
+
.max(4096)
|
|
292
|
+
.regex(/^[\x21-\x7e]+$/)
|
|
293
|
+
.optional(),
|
|
294
|
+
timeoutMs: z.number().int().min(100).max(600_000),
|
|
295
|
+
allowedClassifications: z.array(dataClassificationSchema).min(1),
|
|
296
|
+
structuredOutput: z.enum(["json_schema", "json_object", "none"]),
|
|
297
|
+
allowInsecurePrivateHttp: z.boolean().default(false),
|
|
298
|
+
allowedHosts: z.array(z.string().min(1).max(253)).max(20).default([]),
|
|
299
|
+
maxRequestBytes: z.number().int().min(1024).max(5_000_000).default(DEFAULT_MAX_REQUEST_BYTES),
|
|
300
|
+
maxResponseBytes: z.number().int().min(1024).max(10_000_000).default(DEFAULT_MAX_RESPONSE_BYTES),
|
|
301
|
+
});
|
|
302
|
+
|
|
303
|
+
const completionResponseSchema = z.object({
|
|
304
|
+
model: z.string().optional(),
|
|
305
|
+
choices: z
|
|
306
|
+
.array(
|
|
307
|
+
z.object({
|
|
308
|
+
finish_reason: z.string().nullable().optional(),
|
|
309
|
+
message: z.object({
|
|
310
|
+
content: z.string().nullable().optional(),
|
|
311
|
+
tool_calls: z.array(z.unknown()).nullable().optional(),
|
|
312
|
+
}),
|
|
313
|
+
}),
|
|
314
|
+
)
|
|
315
|
+
.min(1),
|
|
316
|
+
usage: z
|
|
317
|
+
.object({
|
|
318
|
+
prompt_tokens: z.number().int().nonnegative().optional(),
|
|
319
|
+
completion_tokens: z.number().int().nonnegative().optional(),
|
|
320
|
+
total_tokens: z.number().int().nonnegative().optional(),
|
|
321
|
+
})
|
|
322
|
+
.nullable()
|
|
323
|
+
.optional(),
|
|
324
|
+
});
|
|
325
|
+
|
|
326
|
+
const modelListSchema = z.object({
|
|
327
|
+
data: z.array(z.object({ id: z.string() })),
|
|
328
|
+
});
|
|
329
|
+
|
|
330
|
+
/**
|
|
331
|
+
* Adapter for servers exposing the OpenAI Chat Completions API (`POST {baseUrl}chat/completions`, `GET
|
|
332
|
+
* {baseUrl}models`), such as vLLM.
|
|
333
|
+
*/
|
|
334
|
+
export function createOpenAICompatibleProvider(config, dependencies = {}) {
|
|
335
|
+
const parsed = providerConfigSchema.safeParse(config);
|
|
336
|
+
if (!parsed.success) {
|
|
337
|
+
throw new AiServiceError(ErrorCodes.CONFIG_INVALID, "Provider configuration is invalid", {
|
|
338
|
+
details: summarizeZodError(parsed.error, providerConfigSchema),
|
|
339
|
+
});
|
|
340
|
+
}
|
|
341
|
+
const settings = parsed.data;
|
|
342
|
+
const baseUrl = resolveProviderBaseUrl(settings.baseUrl, {
|
|
343
|
+
providerKind: settings.kind,
|
|
344
|
+
allowInsecurePrivateHttp: settings.allowInsecurePrivateHttp,
|
|
345
|
+
allowedHosts: settings.allowedHosts,
|
|
346
|
+
});
|
|
347
|
+
const completionsUrl = new URL("chat/completions", baseUrl);
|
|
348
|
+
const modelsUrl = new URL("models", baseUrl);
|
|
349
|
+
const now = dependencies.now ?? (() => performance.now());
|
|
350
|
+
|
|
351
|
+
const info = Object.freeze({
|
|
352
|
+
id: settings.id,
|
|
353
|
+
kind: settings.kind,
|
|
354
|
+
model: settings.model,
|
|
355
|
+
modelRevision: settings.modelRevision ?? null,
|
|
356
|
+
allowedClassifications: Object.freeze([...settings.allowedClassifications]),
|
|
357
|
+
structuredOutput: settings.structuredOutput,
|
|
358
|
+
});
|
|
359
|
+
|
|
360
|
+
function buildHeaders() {
|
|
361
|
+
return {
|
|
362
|
+
accept: "application/json",
|
|
363
|
+
"content-type": "application/json",
|
|
364
|
+
...(settings.apiKey === undefined ? {} : { authorization: `Bearer ${settings.apiKey}` }),
|
|
365
|
+
};
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
async function requestJson(url, body, callerSignal) {
|
|
369
|
+
if (body !== undefined && Buffer.byteLength(body, "utf8") > settings.maxRequestBytes) {
|
|
370
|
+
throw new AiServiceError(ErrorCodes.REQUEST_TOO_LARGE, "Provider request exceeds the configured size limit", {
|
|
371
|
+
details: { limitBytes: settings.maxRequestBytes },
|
|
372
|
+
});
|
|
373
|
+
}
|
|
374
|
+
if (callerSignal?.aborted === true) {
|
|
375
|
+
throw createCancelledError();
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
const timeoutSignal = AbortSignal.timeout(settings.timeoutMs);
|
|
379
|
+
const signal = callerSignal === undefined ? timeoutSignal : AbortSignal.any([timeoutSignal, callerSignal]);
|
|
380
|
+
const fetchImpl = dependencies.fetch ?? globalThis.fetch;
|
|
381
|
+
|
|
382
|
+
try {
|
|
383
|
+
const response = await fetchImpl(url, {
|
|
384
|
+
method: body === undefined ? "GET" : "POST",
|
|
385
|
+
headers: buildHeaders(),
|
|
386
|
+
...(body === undefined ? {} : { body }),
|
|
387
|
+
signal,
|
|
388
|
+
// Redirects are refused: following one could send the request,
|
|
389
|
+
// including any Authorization header, to a host the policy never checked.
|
|
390
|
+
redirect: "error",
|
|
391
|
+
});
|
|
392
|
+
|
|
393
|
+
if (!response.ok) {
|
|
394
|
+
// A 400 may be the server refusing the model's own output under constrained decoding
|
|
395
|
+
// (for example Groq's json_validate_failed): read only its error code, from a bounded body.
|
|
396
|
+
const rejectedOutput = response.status === 400 && (await structuredOutputRejected(response));
|
|
397
|
+
if (!rejectedOutput) await discardBody(response);
|
|
398
|
+
if (rejectedOutput) {
|
|
399
|
+
throw new AiServiceError(ErrorCodes.PROVIDER_OUTPUT_REJECTED, "The provider rejected the model output as invalid structured output", {
|
|
400
|
+
details: { status: 400, reason: "structured_output_rejected" },
|
|
401
|
+
});
|
|
402
|
+
}
|
|
403
|
+
throw httpError(response.status, response.headers);
|
|
404
|
+
}
|
|
405
|
+
const contentType = response.headers.get("content-type") ?? "";
|
|
406
|
+
if (!contentType.toLowerCase().startsWith("application/json")) {
|
|
407
|
+
await discardBody(response);
|
|
408
|
+
throw invalidResponse("content_type");
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
const text = await readLimitedText(response, settings.maxResponseBytes);
|
|
412
|
+
try {
|
|
413
|
+
return JSON.parse(text);
|
|
414
|
+
} catch {
|
|
415
|
+
throw invalidResponse("not_json");
|
|
416
|
+
}
|
|
417
|
+
} catch (error) {
|
|
418
|
+
if (error instanceof AiServiceError) {
|
|
419
|
+
throw error;
|
|
420
|
+
}
|
|
421
|
+
if (isAborted(callerSignal)) {
|
|
422
|
+
throw createCancelledError();
|
|
423
|
+
}
|
|
424
|
+
if (isAborted(timeoutSignal)) {
|
|
425
|
+
throw new AiServiceError(ErrorCodes.PROVIDER_TIMEOUT, "Provider request timed out", {
|
|
426
|
+
retryable: true,
|
|
427
|
+
details: { timeoutMs: settings.timeoutMs },
|
|
428
|
+
});
|
|
429
|
+
}
|
|
430
|
+
throw networkError(error);
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
async function complete(request, options = {}) {
|
|
435
|
+
const body = {
|
|
436
|
+
model: settings.model,
|
|
437
|
+
messages: request.messages.map((message) => ({ role: message.role, content: message.content })),
|
|
438
|
+
temperature: request.generation.temperature,
|
|
439
|
+
max_tokens: request.generation.maxTokens,
|
|
440
|
+
stream: false,
|
|
441
|
+
...(request.generation.topP === undefined ? {} : { top_p: request.generation.topP }),
|
|
442
|
+
...(request.generation.seed === undefined ? {} : { seed: request.generation.seed }),
|
|
443
|
+
};
|
|
444
|
+
if (request.jsonSchema !== undefined) {
|
|
445
|
+
if (settings.structuredOutput === "json_schema") {
|
|
446
|
+
body.response_format = {
|
|
447
|
+
type: "json_schema",
|
|
448
|
+
json_schema: { name: request.jsonSchema.name, schema: request.jsonSchema.schema, strict: true },
|
|
449
|
+
};
|
|
450
|
+
} else if (settings.structuredOutput === "json_object") {
|
|
451
|
+
body.response_format = { type: "json_object" };
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
const started = now();
|
|
456
|
+
const raw = await requestJson(completionsUrl, JSON.stringify(body), options.signal);
|
|
457
|
+
const parsed = completionResponseSchema.safeParse(raw);
|
|
458
|
+
if (!parsed.success) {
|
|
459
|
+
throw invalidResponse("shape");
|
|
460
|
+
}
|
|
461
|
+
const choice = parsed.data.choices[0];
|
|
462
|
+
if (choice === undefined) {
|
|
463
|
+
throw invalidResponse("shape");
|
|
464
|
+
}
|
|
465
|
+
if (choice.message.tool_calls != null && choice.message.tool_calls.length > 0) {
|
|
466
|
+
throw invalidResponse("tool_calls");
|
|
467
|
+
}
|
|
468
|
+
if (typeof choice.message.content !== "string") {
|
|
469
|
+
throw invalidResponse("missing_content");
|
|
470
|
+
}
|
|
471
|
+
if (parsed.data.model !== undefined && parsed.data.model !== settings.model) {
|
|
472
|
+
throw invalidResponse("model_mismatch");
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
return {
|
|
476
|
+
content: choice.message.content,
|
|
477
|
+
finishReason: mapFinishReason(choice.finish_reason),
|
|
478
|
+
usage: mapUsage(parsed.data.usage),
|
|
479
|
+
model: settings.model,
|
|
480
|
+
latencyMs: Math.max(0, now() - started),
|
|
481
|
+
};
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
async function healthCheck(options = {}) {
|
|
485
|
+
const raw = await requestJson(modelsUrl, undefined, options.signal);
|
|
486
|
+
const parsed = modelListSchema.safeParse(raw);
|
|
487
|
+
if (!parsed.success) {
|
|
488
|
+
throw invalidResponse("shape");
|
|
489
|
+
}
|
|
490
|
+
if (!parsed.data.data.some((entry) => entry.id === settings.model)) {
|
|
491
|
+
throw new AiServiceError(ErrorCodes.PROVIDER_UNAVAILABLE, "Configured model is not served by the provider", {
|
|
492
|
+
details: { reason: "model_not_served" },
|
|
493
|
+
});
|
|
494
|
+
}
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
return Object.freeze({ info, complete, healthCheck });
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
async function readLimitedText(response, limitBytes) {
|
|
501
|
+
const declaredLength = Number(response.headers.get("content-length") ?? Number.NaN);
|
|
502
|
+
if (Number.isFinite(declaredLength) && declaredLength > limitBytes) {
|
|
503
|
+
await discardBody(response);
|
|
504
|
+
throw responseTooLarge(limitBytes);
|
|
505
|
+
}
|
|
506
|
+
if (response.body === null) {
|
|
507
|
+
return "";
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
const reader = response.body.getReader();
|
|
511
|
+
const chunks = [];
|
|
512
|
+
let received = 0;
|
|
513
|
+
try {
|
|
514
|
+
for (;;) {
|
|
515
|
+
// The stream is typed `any` by the platform types; check each chunk instead of trusting it.
|
|
516
|
+
const result = await reader.read();
|
|
517
|
+
if (result.done) {
|
|
518
|
+
break;
|
|
519
|
+
}
|
|
520
|
+
const value = result.value;
|
|
521
|
+
if (!(value instanceof Uint8Array)) {
|
|
522
|
+
throw invalidResponse("body_chunk");
|
|
523
|
+
}
|
|
524
|
+
received += value.byteLength;
|
|
525
|
+
if (received > limitBytes) {
|
|
526
|
+
throw responseTooLarge(limitBytes);
|
|
527
|
+
}
|
|
528
|
+
chunks.push(value);
|
|
529
|
+
}
|
|
530
|
+
} catch (error) {
|
|
531
|
+
// Any failure while streaming abandons the body: release the connection.
|
|
532
|
+
await cancelReader(reader);
|
|
533
|
+
throw error;
|
|
534
|
+
}
|
|
535
|
+
|
|
536
|
+
try {
|
|
537
|
+
return new TextDecoder("utf-8", { fatal: true }).decode(Buffer.concat(chunks));
|
|
538
|
+
} catch {
|
|
539
|
+
throw invalidResponse("invalid_utf8");
|
|
540
|
+
}
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
/**
|
|
544
|
+
* Signals abort asynchronously, so this is read through a function call: TypeScript would otherwise keep a
|
|
545
|
+
* narrowing from an earlier check.
|
|
546
|
+
*/
|
|
547
|
+
function isAborted(signal) {
|
|
548
|
+
return signal?.aborted === true;
|
|
549
|
+
}
|
|
550
|
+
|
|
551
|
+
async function discardBody(response) {
|
|
552
|
+
try {
|
|
553
|
+
await response.body?.cancel();
|
|
554
|
+
} catch {
|
|
555
|
+
// The body is never read; failing to release it does not change the
|
|
556
|
+
// error already being reported to the caller.
|
|
557
|
+
}
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
async function cancelReader(reader) {
|
|
561
|
+
try {
|
|
562
|
+
await reader.cancel();
|
|
563
|
+
} catch {
|
|
564
|
+
// An already-errored stream rejects cancel(); the error being reported
|
|
565
|
+
// to the caller is the one that matters.
|
|
566
|
+
}
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
/** Classifies a fetch failure that was not a timeout or cancellation. */
|
|
570
|
+
function networkError(error) {
|
|
571
|
+
if (isRedirectRejection(error)) {
|
|
572
|
+
return new AiServiceError(ErrorCodes.PROVIDER_HTTP_ERROR, "Provider attempted a redirect, which is not allowed", {
|
|
573
|
+
details: { reason: "redirect" },
|
|
574
|
+
});
|
|
575
|
+
}
|
|
576
|
+
const code = systemErrorCode(error);
|
|
577
|
+
const transient = code !== undefined && TRANSIENT_NETWORK_CODES.has(code);
|
|
578
|
+
return new AiServiceError(ErrorCodes.PROVIDER_UNAVAILABLE, "Provider request failed", {
|
|
579
|
+
retryable: transient,
|
|
580
|
+
details: { reason: code ?? "request_failed" },
|
|
581
|
+
cause: error,
|
|
582
|
+
});
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
/** Node's fetch rejects with TypeError("fetch failed", { cause: Error("unexpected redirect") }) under redirect: "error". */
|
|
586
|
+
function isRedirectRejection(error) {
|
|
587
|
+
return error instanceof Error && error.cause instanceof Error && error.cause.message === "unexpected redirect";
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
/** Longest wait a provider's Retry-After may ask for (seconds); larger values are capped. */
|
|
591
|
+
const MAX_RETRY_AFTER_SECONDS = 300;
|
|
592
|
+
|
|
593
|
+
/** Error codes by which OpenAI-compatible servers report model output that failed their JSON schema. */
|
|
594
|
+
const OUTPUT_REJECTION_CODES = new Set(["json_validate_failed", "invalid_json_output"]);
|
|
595
|
+
|
|
596
|
+
async function structuredOutputRejected(response) {
|
|
597
|
+
try {
|
|
598
|
+
const body = JSON.parse(await readLimitedText(response, 64 * 1024));
|
|
599
|
+
return typeof body?.error?.code === "string" && OUTPUT_REJECTION_CODES.has(body.error.code);
|
|
600
|
+
} catch {
|
|
601
|
+
return false;
|
|
602
|
+
}
|
|
603
|
+
}
|
|
604
|
+
|
|
605
|
+
function httpError(status, headers) {
|
|
606
|
+
if (status === 429) {
|
|
607
|
+
// Rate limited: its own code, with the provider's Retry-After (seconds, bounded) when it sends one.
|
|
608
|
+
const raw = headers?.get?.("retry-after");
|
|
609
|
+
const header = typeof raw === "string" && /^\s*\d+(\.\d+)?\s*$/.test(raw) ? Number(raw) : NaN;
|
|
610
|
+
const retryAfterSeconds = Number.isFinite(header) ? Math.min(MAX_RETRY_AFTER_SECONDS, Math.ceil(header)) : undefined;
|
|
611
|
+
return new AiServiceError(ErrorCodes.PROVIDER_RATE_LIMITED, "Provider rate limit reached (HTTP 429)", {
|
|
612
|
+
retryable: true,
|
|
613
|
+
details: { status, ...(retryAfterSeconds === undefined ? {} : { retryAfterSeconds }) },
|
|
614
|
+
});
|
|
615
|
+
}
|
|
616
|
+
const retryable = status === 408 || status >= 500;
|
|
617
|
+
const code = status === 503 ? ErrorCodes.PROVIDER_UNAVAILABLE : ErrorCodes.PROVIDER_HTTP_ERROR;
|
|
618
|
+
return new AiServiceError(code, `Provider responded with HTTP ${String(status)}`, {
|
|
619
|
+
retryable,
|
|
620
|
+
details: { status },
|
|
621
|
+
});
|
|
622
|
+
}
|
|
623
|
+
|
|
624
|
+
function invalidResponse(reason) {
|
|
625
|
+
return new AiServiceError(ErrorCodes.PROVIDER_RESPONSE_INVALID, "Provider response is invalid", {
|
|
626
|
+
details: { reason },
|
|
627
|
+
});
|
|
628
|
+
}
|
|
629
|
+
|
|
630
|
+
function responseTooLarge(limitBytes) {
|
|
631
|
+
return new AiServiceError(ErrorCodes.PROVIDER_RESPONSE_INVALID, "Provider response exceeds the configured size limit", {
|
|
632
|
+
details: { reason: "too_large", limitBytes },
|
|
633
|
+
});
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
function mapFinishReason(value) {
|
|
637
|
+
if (value === "stop") {
|
|
638
|
+
return "stop";
|
|
639
|
+
}
|
|
640
|
+
if (value === "length") {
|
|
641
|
+
return "length";
|
|
642
|
+
}
|
|
643
|
+
return "other";
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
function mapUsage(usage) {
|
|
647
|
+
const promptTokens = usage?.prompt_tokens ?? null;
|
|
648
|
+
const completionTokens = usage?.completion_tokens ?? null;
|
|
649
|
+
const reportedTotal = usage?.total_tokens ?? null;
|
|
650
|
+
const derivedTotal = promptTokens !== null && completionTokens !== null ? promptTokens + completionTokens : null;
|
|
651
|
+
return { promptTokens, completionTokens, totalTokens: reportedTotal ?? derivedTotal };
|
|
652
|
+
}
|
|
653
|
+
|
|
654
|
+
/**
|
|
655
|
+
* Builds a ChatProvider from configuration, so a consumer changes provider by changing configuration, not
|
|
656
|
+
* code.
|
|
657
|
+
*/
|
|
658
|
+
|
|
659
|
+
const apiKeySchema = z
|
|
660
|
+
.string()
|
|
661
|
+
.min(1)
|
|
662
|
+
.max(4096)
|
|
663
|
+
.regex(/^[\x21-\x7e]+$/);
|
|
664
|
+
const structuredOutputSchema = z.enum(["json_schema", "json_object", "none"]);
|
|
665
|
+
|
|
666
|
+
const groqConfigSchema = z.strictObject({
|
|
667
|
+
id: z
|
|
668
|
+
.string()
|
|
669
|
+
.regex(/^[a-z][a-z0-9-]{0,63}$/)
|
|
670
|
+
.default(PROVIDER_TYPE.GROQ),
|
|
671
|
+
apiKey: apiKeySchema,
|
|
672
|
+
model: z.string().min(1).max(200).default(GROQ_DEFAULT_MODEL),
|
|
673
|
+
modelRevision: z.string().min(1).max(200).optional(),
|
|
674
|
+
// Required: what may be sent to a paid provider is the consumer's decision, never a default.
|
|
675
|
+
allowedClassifications: z.array(dataClassificationSchema).min(1),
|
|
676
|
+
timeoutMs: z.number().int().min(100).max(600_000).default(GROQ_DEFAULT_TIMEOUT_MS),
|
|
677
|
+
structuredOutput: structuredOutputSchema.default("json_schema"),
|
|
678
|
+
maxRequestBytes: z.number().int().min(1024).max(5_000_000).optional(),
|
|
679
|
+
maxResponseBytes: z.number().int().min(1024).max(10_000_000).optional(),
|
|
680
|
+
});
|
|
681
|
+
|
|
682
|
+
function invalid(error, schema, what) {
|
|
683
|
+
return new AiServiceError(ErrorCodes.CONFIG_INVALID, `${what} configuration is invalid`, {
|
|
684
|
+
details: summarizeZodError(error, schema),
|
|
685
|
+
});
|
|
686
|
+
}
|
|
687
|
+
|
|
688
|
+
/**
|
|
689
|
+
* Groq (paid, api.groq.com) through its OpenAI-compatible API. The service using it must also be
|
|
690
|
+
* created with allowPaidProvider: true.
|
|
691
|
+
* @param {{ apiKey: string, model?: string, allowedClassifications: DataClassification[], id?: string, modelRevision?: string, timeoutMs?: number, structuredOutput?: StructuredOutputMode, maxRequestBytes?: number, maxResponseBytes?: number }} config
|
|
692
|
+
* @param {{ fetch?: typeof fetch, now?: () => number }} [dependencies]
|
|
693
|
+
* @returns {ChatProvider}
|
|
694
|
+
*/
|
|
695
|
+
export function createGroqProvider(config, dependencies = {}) {
|
|
696
|
+
const parsed = groqConfigSchema.safeParse(config);
|
|
697
|
+
if (!parsed.success) throw invalid(parsed.error, groqConfigSchema, "Groq provider");
|
|
698
|
+
const settings = parsed.data;
|
|
699
|
+
return createOpenAICompatibleProvider(
|
|
700
|
+
{
|
|
701
|
+
id: settings.id,
|
|
702
|
+
kind: "PAID",
|
|
703
|
+
baseUrl: GROQ_BASE_URL,
|
|
704
|
+
model: settings.model,
|
|
705
|
+
...(settings.modelRevision === undefined ? {} : { modelRevision: settings.modelRevision }),
|
|
706
|
+
apiKey: settings.apiKey,
|
|
707
|
+
timeoutMs: settings.timeoutMs,
|
|
708
|
+
allowedClassifications: settings.allowedClassifications,
|
|
709
|
+
structuredOutput: settings.structuredOutput,
|
|
710
|
+
...(settings.maxRequestBytes === undefined ? {} : { maxRequestBytes: settings.maxRequestBytes }),
|
|
711
|
+
...(settings.maxResponseBytes === undefined ? {} : { maxResponseBytes: settings.maxResponseBytes }),
|
|
712
|
+
},
|
|
713
|
+
dependencies,
|
|
714
|
+
);
|
|
715
|
+
}
|
|
716
|
+
|
|
717
|
+
/**
|
|
718
|
+
* One entry point for every supported provider:
|
|
719
|
+
* { type: "groq", apiKey, model?, allowedClassifications, ... } see createGroqProvider (model defaults to GROQ_DEFAULT_MODEL)
|
|
720
|
+
* { type: "openai-compatible", id, kind, baseUrl, model, timeoutMs, ... } any other OpenAI-compatible server,
|
|
721
|
+
* under the endpoint policy
|
|
722
|
+
* @param {{ type: "groq" | "openai-compatible" } & Record<string, unknown>} config
|
|
723
|
+
* @param {{ fetch?: typeof fetch, now?: () => number }} [dependencies]
|
|
724
|
+
* @returns {ChatProvider}
|
|
725
|
+
*/
|
|
726
|
+
export function createProvider(config, dependencies = {}) {
|
|
727
|
+
if (typeof config !== "object" || config === null || !PROVIDER_TYPES.includes(config.type)) {
|
|
728
|
+
throw new AiServiceError(ErrorCodes.CONFIG_INVALID, `Provider type must be one of: ${PROVIDER_TYPES.join(", ")}`);
|
|
729
|
+
}
|
|
730
|
+
const { type, ...rest } = config;
|
|
731
|
+
if (type === PROVIDER_TYPE.GROQ) return createGroqProvider(rest, dependencies);
|
|
732
|
+
return createOpenAICompatibleProvider(rest, dependencies);
|
|
733
|
+
}
|