@appweaver/ai 0.0.0-stage → 2.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +62 -2
- package/config.d.ts +49 -0
- package/config.js +58 -0
- package/driver/ai-driver.d.ts +42 -0
- package/driver/ai-driver.js +31 -0
- package/driver/mock-driver.d.ts +27 -0
- package/driver/mock-driver.js +147 -0
- package/driver/model-registry.d.ts +53 -0
- package/driver/model-registry.js +244 -0
- package/driver/presets.d.ts +23 -0
- package/driver/presets.js +126 -0
- package/index.d.ts +14 -0
- package/index.js +30 -0
- package/module.d.ts +7 -0
- package/module.js +48 -0
- package/package.json +54 -3
- package/service/ai-object-stream.d.ts +35 -0
- package/service/ai-object-stream.js +58 -0
- package/service/ai-service.d.ts +118 -0
- package/service/ai-service.js +769 -0
- package/service/ai-stream.d.ts +48 -0
- package/service/ai-stream.js +133 -0
- package/service/prompt.d.ts +100 -0
- package/service/prompt.js +2 -0
- package/tools/create-tool.d.ts +53 -0
- package/tools/create-tool.js +57 -0
- package/tsconfig.build.tsbuildinfo +1 -0
- package/types.d.ts +201 -0
- package/types.js +2 -0
- package/utils/async-queue.d.ts +13 -0
- package/utils/async-queue.js +59 -0
- package/utils/json-util.d.ts +27 -0
- package/utils/json-util.js +127 -0
- package/utils/stream-util.d.ts +53 -0
- package/utils/stream-util.js +90 -0
- package/utils/telemetry.d.ts +38 -0
- package/utils/telemetry.js +136 -0
- package/utils/vector-util.d.ts +24 -0
- package/utils/vector-util.js +43 -0
|
@@ -0,0 +1,769 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.AiService = void 0;
|
|
4
|
+
const typebox_1 = require("@sinclair/typebox");
|
|
5
|
+
const value_1 = require("@sinclair/typebox/value");
|
|
6
|
+
const common_1 = require("@appweaver/common");
|
|
7
|
+
const core_1 = require("@appweaver/core");
|
|
8
|
+
const config_1 = require("../config");
|
|
9
|
+
const model_registry_1 = require("../driver/model-registry");
|
|
10
|
+
const create_tool_1 = require("../tools/create-tool");
|
|
11
|
+
const async_queue_1 = require("../utils/async-queue");
|
|
12
|
+
const json_util_1 = require("../utils/json-util");
|
|
13
|
+
const telemetry_1 = require("../utils/telemetry");
|
|
14
|
+
const ai_object_stream_1 = require("./ai-object-stream");
|
|
15
|
+
const ai_stream_1 = require("./ai-stream");
|
|
16
|
+
const logger = (0, common_1.createLogger)('ai');
|
|
17
|
+
/** The errors another model of an alias may not have. */
|
|
18
|
+
const RETRIABLE = new Set([
|
|
19
|
+
common_1.ErrorCode.AiRateLimited,
|
|
20
|
+
common_1.ErrorCode.AiTimeout,
|
|
21
|
+
common_1.ErrorCode.AiProviderUnavailable
|
|
22
|
+
]);
|
|
23
|
+
/**
|
|
24
|
+
* Calls the models of the endpoints of the `ai` config: a whole response, a stream, an object matching a schema, a
|
|
25
|
+
* classification or embeddings, directly or through the `ai` queue, running the tools the model calls.
|
|
26
|
+
*
|
|
27
|
+
* @example
|
|
28
|
+
* const ai = inject(AiService);
|
|
29
|
+
* const { text } = await ai.generate({ model: 'fast', prompt: 'Summarize: ...' });
|
|
30
|
+
*/
|
|
31
|
+
class AiService {
|
|
32
|
+
static [common_1.LIFECYCLE] = true;
|
|
33
|
+
static [common_1.HEALTH_CHECK] = true;
|
|
34
|
+
/** @internal */
|
|
35
|
+
_registry = new model_registry_1.ModelRegistry();
|
|
36
|
+
/** @internal */
|
|
37
|
+
_templateEngine = (0, core_1.inject)(common_1.TemplateEngine, false);
|
|
38
|
+
/** @internal */
|
|
39
|
+
_pubsub = (0, core_1.inject)(common_1.PubSub, false);
|
|
40
|
+
/** @internal */
|
|
41
|
+
_queue = (0, core_1.inject)(common_1.Queue, false)?.get('ai');
|
|
42
|
+
/** @internal */
|
|
43
|
+
_results = this._queue
|
|
44
|
+
? new core_1.JobResults(this._queue, 'ai', { toError: aiError })
|
|
45
|
+
: undefined;
|
|
46
|
+
constructor() {
|
|
47
|
+
this._queue?.addWorker(this.processJob.bind(this), {
|
|
48
|
+
concurrency: config_1.aiConfig.QUEUE_CONCURRENCY
|
|
49
|
+
});
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Checks that the default models and the aliases name configured endpoints.
|
|
53
|
+
*
|
|
54
|
+
* @throws {ConfigurationError} `CONFIGURATION_INVALID` listing the invalid models.
|
|
55
|
+
*/
|
|
56
|
+
async onInit() {
|
|
57
|
+
const errors = this._registry.validate();
|
|
58
|
+
if (errors.length) {
|
|
59
|
+
throw new common_1.ConfigurationError(common_1.ErrorCode.ConfigurationInvalid, `Invalid AI config: ${errors.join('; ')}`, { errors });
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Calls a model and returns its whole response, after running the tools it calls. The models of an alias are
|
|
64
|
+
* called in turn while one fails with a rate limit, a timeout or an unavailable provider.
|
|
65
|
+
*
|
|
66
|
+
* @param {AiPrompt} prompt - The model, the messages and the parameters.
|
|
67
|
+
* @return {Promise<AiResult>} The response, its steps and the conversation.
|
|
68
|
+
* @throws {AiError} The error of the request or of the driver.
|
|
69
|
+
*/
|
|
70
|
+
async generate(prompt) {
|
|
71
|
+
return drain(this.run(prompt, false));
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Calls a model and streams its response, running the tools it calls between its steps. The models of an alias are
|
|
75
|
+
* called in turn while one fails before streaming anything.
|
|
76
|
+
*
|
|
77
|
+
* @param {AiPrompt} prompt - The model, the messages and the parameters.
|
|
78
|
+
* @return {AiStream} The text deltas, with the result once it ends.
|
|
79
|
+
*/
|
|
80
|
+
stream(prompt) {
|
|
81
|
+
return new ai_stream_1.AiStream((signal) => this.run({ ...prompt, signal }, true), prompt.signal);
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* Calls a model for an object matching a schema, in the JSON schema mode of the endpoint when it has one. An object
|
|
85
|
+
* not matching the schema is sent back with its errors for the model to repair, up to `repairAttempts` times.
|
|
86
|
+
*
|
|
87
|
+
* @param {AiObjectPrompt} prompt - The model, the messages, the parameters and the schema.
|
|
88
|
+
* @return {Promise<AiObjectResult>} The object and the response.
|
|
89
|
+
* @throws {AiError} `AI_OUTPUT_INVALID` when the object still does not match the schema.
|
|
90
|
+
*/
|
|
91
|
+
async generateObject(prompt) {
|
|
92
|
+
const attempts = prompt.repairAttempts ?? config_1.aiConfig.REPAIR_ATTEMPTS;
|
|
93
|
+
let result = await drain(this.run(prompt, false, objectFormat(prompt)));
|
|
94
|
+
for (let attempt = 0;; attempt++) {
|
|
95
|
+
const checked = checkObject(prompt.schema, result.text);
|
|
96
|
+
if (checked.valid) {
|
|
97
|
+
return { ...result, object: checked.value };
|
|
98
|
+
}
|
|
99
|
+
if (attempt >= attempts) {
|
|
100
|
+
throw outputInvalid(result.model, checked.errors);
|
|
101
|
+
}
|
|
102
|
+
logger.debug({ model: result.model, errors: checked.errors }, 'Repairing the AI object');
|
|
103
|
+
const repaired = await drain(this.run({
|
|
104
|
+
...prompt,
|
|
105
|
+
model: result.model,
|
|
106
|
+
system: undefined,
|
|
107
|
+
prompt: undefined,
|
|
108
|
+
messages: [
|
|
109
|
+
...result.messages,
|
|
110
|
+
{
|
|
111
|
+
role: 'user',
|
|
112
|
+
content: `The JSON does not match the schema: ${checked.errors.join('; ')}. Respond with the corrected JSON only.`
|
|
113
|
+
}
|
|
114
|
+
]
|
|
115
|
+
}, false, objectFormat(prompt)));
|
|
116
|
+
const steps = [...result.steps, ...repaired.steps];
|
|
117
|
+
result = { ...repaired, steps, usage: sumUsage(steps) };
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
/**
|
|
121
|
+
* Calls a model for an object matching a schema, streaming the object while it is generated. The object is checked
|
|
122
|
+
* once it ends, without a repair.
|
|
123
|
+
*
|
|
124
|
+
* @param {AiObjectPrompt} prompt - The model, the messages, the parameters and the schema.
|
|
125
|
+
* @return {AiObjectStream} The partial objects, with the object once it ends.
|
|
126
|
+
*/
|
|
127
|
+
streamObject(prompt) {
|
|
128
|
+
const run = (signal) => this.run({ ...prompt, signal }, true, objectFormat(prompt));
|
|
129
|
+
return new ai_object_stream_1.AiObjectStream(new ai_stream_1.AiStream(async function* (signal) {
|
|
130
|
+
const result = yield* run(signal);
|
|
131
|
+
const checked = checkObject(prompt.schema, result.text);
|
|
132
|
+
if (!checked.valid) {
|
|
133
|
+
throw outputInvalid(result.model, checked.errors);
|
|
134
|
+
}
|
|
135
|
+
return { ...result, object: checked.value };
|
|
136
|
+
}, prompt.signal));
|
|
137
|
+
}
|
|
138
|
+
/**
|
|
139
|
+
* Classifies a text into one of the labels.
|
|
140
|
+
*
|
|
141
|
+
* @param {string} input - The text to classify.
|
|
142
|
+
* @param {string[]} labels - The labels, at least two.
|
|
143
|
+
* @param {AiPrompt} [prompt] - The model, the instructions in `system`, and the parameters.
|
|
144
|
+
* @return {Promise<string>} The label.
|
|
145
|
+
* @throws {AiError} `AI_REQUEST_INVALID` for fewer than two labels.
|
|
146
|
+
*/
|
|
147
|
+
async classify(input, labels, prompt = {}) {
|
|
148
|
+
if (labels.length < 2) {
|
|
149
|
+
throw new common_1.AiError(common_1.ErrorCode.AiRequestInvalid, 'Classifying needs at least two labels', { errors: ['labels: expected at least two'] });
|
|
150
|
+
}
|
|
151
|
+
const instruction = `Classify the input into exactly one of the labels: ${labels.join(', ')}.`;
|
|
152
|
+
const { object } = await this.generateObject({
|
|
153
|
+
...prompt,
|
|
154
|
+
system: typeof prompt.system === 'object'
|
|
155
|
+
? prompt.system
|
|
156
|
+
: [prompt.system, instruction].filter(Boolean).join('\n\n'),
|
|
157
|
+
prompt: input,
|
|
158
|
+
schema: typebox_1.Type.Object({
|
|
159
|
+
label: typebox_1.Type.Union(labels.map((label) => typebox_1.Type.Literal(label)))
|
|
160
|
+
})
|
|
161
|
+
});
|
|
162
|
+
return object.label;
|
|
163
|
+
}
|
|
164
|
+
/**
|
|
165
|
+
* Embeds texts into vectors, with `ai.defaultEmbeddingModel` when no model is given.
|
|
166
|
+
*
|
|
167
|
+
* @param {string | string[]} input - A text or the texts.
|
|
168
|
+
* @param {AiEmbedOptions} [options] - The model, the dimensions and the parameters.
|
|
169
|
+
* @return {Promise<AiEmbedResult>} A vector per text.
|
|
170
|
+
* @throws {AiError} `AI_CAPABILITY_UNSUPPORTED` for an endpoint without embeddings.
|
|
171
|
+
*/
|
|
172
|
+
async embed(input, options = {}) {
|
|
173
|
+
const models = this._registry.resolve(options.model, config_1.aiConfig.DEFAULT_EMBEDDING_MODEL);
|
|
174
|
+
for (let i = 0;; i++) {
|
|
175
|
+
const model = models[i];
|
|
176
|
+
try {
|
|
177
|
+
const driver = await this._registry.driver(model.endpoint);
|
|
178
|
+
requireCapability(driver, model, 'embeddings');
|
|
179
|
+
const request = {
|
|
180
|
+
model: model.model,
|
|
181
|
+
input: Array.isArray(input) ? input : [input],
|
|
182
|
+
dimensions: options.dimensions,
|
|
183
|
+
providerOptions: options.providerOptions
|
|
184
|
+
};
|
|
185
|
+
const trace = (0, telemetry_1.traceAiCall)('embeddings', driver.endpoint, request);
|
|
186
|
+
try {
|
|
187
|
+
const response = await trace.run(() => driver.embed(request, {
|
|
188
|
+
signal: options.signal,
|
|
189
|
+
timeout: options.timeout
|
|
190
|
+
}));
|
|
191
|
+
trace.end({
|
|
192
|
+
model: response.model,
|
|
193
|
+
finishReason: 'stop',
|
|
194
|
+
usage: { inputTokens: response.usage?.inputTokens }
|
|
195
|
+
});
|
|
196
|
+
return { ...response, model: model.reference };
|
|
197
|
+
}
|
|
198
|
+
catch (error) {
|
|
199
|
+
trace.fail(error);
|
|
200
|
+
throw error;
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
catch (error) {
|
|
204
|
+
nextModel(models, i, error);
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
/**
|
|
209
|
+
* Tells whether a model has a capability, i.e. to send images only to a model reading them.
|
|
210
|
+
*
|
|
211
|
+
* @param {string | undefined} model - A model reference or an alias, the default model when none is given.
|
|
212
|
+
* @param {AiCapability} capability - The capability.
|
|
213
|
+
* @return {Promise<boolean>} `true` when the first model of the alias has it.
|
|
214
|
+
*/
|
|
215
|
+
async supports(model, capability) {
|
|
216
|
+
const [resolved] = this._registry.resolve(model);
|
|
217
|
+
const driver = await this._registry.driver(resolved.endpoint);
|
|
218
|
+
return driver.supports(capability, resolved.model);
|
|
219
|
+
}
|
|
220
|
+
/**
|
|
221
|
+
* Adds a call to the `ai` queue, run by a worker of any instance with the retries of the queue.
|
|
222
|
+
*
|
|
223
|
+
* @param {AiQueuedPrompt} prompt - The prompt, a JSON value naming its tools.
|
|
224
|
+
* @param {QueueJobOptions} [options] - The options of the job.
|
|
225
|
+
* @return {Promise<string | undefined>} The ID of the job.
|
|
226
|
+
* @throws {ConfigurationError} `CONFIGURATION_INVALID` without a queue.
|
|
227
|
+
*/
|
|
228
|
+
async enqueue(prompt, options) {
|
|
229
|
+
const job = await this.queue().sendJob({ prompt: queuedPrompt(prompt) }, 'generate', options);
|
|
230
|
+
return job.id;
|
|
231
|
+
}
|
|
232
|
+
/**
|
|
233
|
+
* Runs a call through the `ai` queue and waits for its result, on whichever instance its worker runs.
|
|
234
|
+
*
|
|
235
|
+
* @param {AiQueuedPrompt} prompt - The prompt, a JSON value naming its tools.
|
|
236
|
+
* @param {QueueJobOptions} [options] - The options of the job.
|
|
237
|
+
* @return {Promise<AiResult>} The result.
|
|
238
|
+
* @throws {AiError} The error of the last attempt.
|
|
239
|
+
*/
|
|
240
|
+
async generateQueued(prompt, options) {
|
|
241
|
+
const data = { prompt: queuedPrompt(prompt) };
|
|
242
|
+
const queue = this.queue();
|
|
243
|
+
return this._results.wait((jobId) => queue.sendJob(data, 'generate', { ...options, jobId }));
|
|
244
|
+
}
|
|
245
|
+
/**
|
|
246
|
+
* Runs a call through the `ai` queue and streams it from the worker, the parts relayed through the `PubSub` in
|
|
247
|
+
* batches of `ai.streamFlushInterval`. The job is attempted once, and aborting the stream aborts the worker's call.
|
|
248
|
+
*
|
|
249
|
+
* @param {AiQueuedPrompt} prompt - The prompt, a JSON value naming its tools.
|
|
250
|
+
* @param {QueueJobOptions} [options] - The options of the job, and the signal aborting the stream.
|
|
251
|
+
* @return {AiStream} The stream.
|
|
252
|
+
* @throws {ConfigurationError} `CONFIGURATION_INVALID` without a queue or a pub/sub.
|
|
253
|
+
*/
|
|
254
|
+
streamQueued(prompt, options = {}) {
|
|
255
|
+
const { signal: abortSignal, ...jobOptions } = options;
|
|
256
|
+
const data = { prompt: queuedPrompt(prompt), stream: (0, common_1.uuid)() };
|
|
257
|
+
const channel = `ai:stream:${data.stream}`;
|
|
258
|
+
return new ai_stream_1.AiStream(async function* (signal) {
|
|
259
|
+
const queue = this.queue();
|
|
260
|
+
const pubsub = this.pubsub();
|
|
261
|
+
const parts = new async_queue_1.AsyncQueue();
|
|
262
|
+
let result;
|
|
263
|
+
const id = await pubsub.subscribe(channel, (message) => {
|
|
264
|
+
message.parts?.forEach((part) => parts.push(part));
|
|
265
|
+
if (message.result) {
|
|
266
|
+
result = message.result;
|
|
267
|
+
parts.end();
|
|
268
|
+
}
|
|
269
|
+
});
|
|
270
|
+
const cancel = () => void pubsub.publish(`${channel}:cancel`, {});
|
|
271
|
+
signal.addEventListener('abort', cancel, { once: true });
|
|
272
|
+
try {
|
|
273
|
+
this._results.wait((jobId) => queue.sendJob(data, 'stream', { ...jobOptions, attempts: 1, jobId })).then((completed) => {
|
|
274
|
+
result ??= completed;
|
|
275
|
+
parts.end();
|
|
276
|
+
}, (error) => parts.fail(error));
|
|
277
|
+
for await (const part of parts) {
|
|
278
|
+
signal.throwIfAborted();
|
|
279
|
+
yield part;
|
|
280
|
+
}
|
|
281
|
+
signal.throwIfAborted();
|
|
282
|
+
return result;
|
|
283
|
+
}
|
|
284
|
+
finally {
|
|
285
|
+
signal.removeEventListener('abort', cancel);
|
|
286
|
+
await pubsub.unsubscribe(id);
|
|
287
|
+
}
|
|
288
|
+
}.bind(this), abortSignal);
|
|
289
|
+
}
|
|
290
|
+
async checkHealth() {
|
|
291
|
+
const failed = [];
|
|
292
|
+
for (const name of config_1.aiConfig.HEALTH_CHECK_ENDPOINTS) {
|
|
293
|
+
try {
|
|
294
|
+
const result = await (await this._registry.driver(name)).checkHealth();
|
|
295
|
+
if (!result.success) {
|
|
296
|
+
failed.push(`${name}: ${result.message ?? 'unavailable'}`);
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
catch (error) {
|
|
300
|
+
failed.push(`${name}: ${error.message}`);
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
return failed.length
|
|
304
|
+
? { success: false, message: failed.join('; ') }
|
|
305
|
+
: { success: true };
|
|
306
|
+
}
|
|
307
|
+
checkHealthConfig() {
|
|
308
|
+
return { name: 'ai' };
|
|
309
|
+
}
|
|
310
|
+
/**
|
|
311
|
+
* Calls the models of a prompt in turn, while one fails with a retriable error before it streamed anything.
|
|
312
|
+
*
|
|
313
|
+
* @internal
|
|
314
|
+
*/
|
|
315
|
+
async *run(prompt, stream, decorate) {
|
|
316
|
+
const prepared = await this.prepare(prompt);
|
|
317
|
+
const models = this._registry.resolve(prompt.model);
|
|
318
|
+
for (let i = 0;; i++) {
|
|
319
|
+
const model = models[i];
|
|
320
|
+
let streamed = false;
|
|
321
|
+
try {
|
|
322
|
+
const driver = await this._registry.driver(model.endpoint);
|
|
323
|
+
const steps = this.steps(model, driver, decorate ? decorate(prepared, driver, model.model) : prepared, stream);
|
|
324
|
+
while (true) {
|
|
325
|
+
const next = await steps.next();
|
|
326
|
+
if (next.done) {
|
|
327
|
+
return next.value;
|
|
328
|
+
}
|
|
329
|
+
streamed = true;
|
|
330
|
+
yield next.value;
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
catch (error) {
|
|
334
|
+
nextModel(models, i, error, streamed);
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
/**
|
|
339
|
+
* Calls a model, then again with the results of the tools it calls, until it answers or the steps run out.
|
|
340
|
+
*
|
|
341
|
+
* @internal
|
|
342
|
+
*/
|
|
343
|
+
async *steps(model, driver, prepared, stream) {
|
|
344
|
+
const messages = [...prepared.messages];
|
|
345
|
+
const steps = [];
|
|
346
|
+
for (let step = 1;; step++) {
|
|
347
|
+
const request = {
|
|
348
|
+
...prepared.request,
|
|
349
|
+
model: model.model,
|
|
350
|
+
messages: [...messages]
|
|
351
|
+
};
|
|
352
|
+
checkCapabilities(driver, model, request);
|
|
353
|
+
if (config_1.aiConfig.LOG_PROMPTS) {
|
|
354
|
+
logger.debug({ model: model.reference, messages }, 'AI request');
|
|
355
|
+
}
|
|
356
|
+
const trace = (0, telemetry_1.traceAiCall)('chat', driver.endpoint, request);
|
|
357
|
+
let response;
|
|
358
|
+
try {
|
|
359
|
+
response = stream
|
|
360
|
+
? yield* streamResponse(driver.stream(request, prepared.options))
|
|
361
|
+
: await trace.run(() => driver.generate(request, prepared.options));
|
|
362
|
+
trace.end(response);
|
|
363
|
+
}
|
|
364
|
+
catch (error) {
|
|
365
|
+
trace.fail(error);
|
|
366
|
+
throw error;
|
|
367
|
+
}
|
|
368
|
+
messages.push(assistantMessage(response));
|
|
369
|
+
const toolResults = await this.runTools(response, prepared, messages, step);
|
|
370
|
+
steps.push({ response, toolResults: toolResults ?? [] });
|
|
371
|
+
if (stream) {
|
|
372
|
+
for (const part of toolResults ?? []) {
|
|
373
|
+
const toolCall = response.toolCalls.find((call) => call.id === part.id);
|
|
374
|
+
yield {
|
|
375
|
+
type: 'tool-result',
|
|
376
|
+
toolCall,
|
|
377
|
+
result: part.result,
|
|
378
|
+
isError: part.isError
|
|
379
|
+
};
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
if (!toolResults) {
|
|
383
|
+
const result = {
|
|
384
|
+
...response,
|
|
385
|
+
model: model.reference,
|
|
386
|
+
steps,
|
|
387
|
+
messages,
|
|
388
|
+
usage: sumUsage(steps)
|
|
389
|
+
};
|
|
390
|
+
logger.debug({
|
|
391
|
+
model: model.reference,
|
|
392
|
+
steps: steps.length,
|
|
393
|
+
finishReason: result.finishReason,
|
|
394
|
+
usage: result.usage,
|
|
395
|
+
...(config_1.aiConfig.LOG_PROMPTS ? { text: result.text } : {})
|
|
396
|
+
}, 'AI call completed');
|
|
397
|
+
if (stream) {
|
|
398
|
+
const { finishReason, usage, id } = result;
|
|
399
|
+
yield {
|
|
400
|
+
type: 'finish',
|
|
401
|
+
finishReason,
|
|
402
|
+
usage,
|
|
403
|
+
id,
|
|
404
|
+
model: response.model
|
|
405
|
+
};
|
|
406
|
+
}
|
|
407
|
+
return result;
|
|
408
|
+
}
|
|
409
|
+
messages.push({ role: 'tool', content: toolResults });
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
/**
|
|
413
|
+
* Runs the tool calls of a step, `undefined` when the call ends: without tool calls, with a call of a tool without
|
|
414
|
+
* `execute`, or on the last step.
|
|
415
|
+
*
|
|
416
|
+
* @internal
|
|
417
|
+
*/
|
|
418
|
+
async runTools(response, prepared, messages, step) {
|
|
419
|
+
const tools = new Map(prepared.tools.map((tool) => [tool.name, tool]));
|
|
420
|
+
const calls = response.toolCalls;
|
|
421
|
+
if (!calls.length ||
|
|
422
|
+
step >= prepared.maxSteps ||
|
|
423
|
+
calls.some((call) => tools.has(call.name) && !tools.get(call.name).execute)) {
|
|
424
|
+
return undefined;
|
|
425
|
+
}
|
|
426
|
+
const signal = prepared.options.signal;
|
|
427
|
+
return Promise.all(calls.map(async (toolCall) => {
|
|
428
|
+
const tool = tools.get(toolCall.name);
|
|
429
|
+
const part = {
|
|
430
|
+
type: 'tool-result',
|
|
431
|
+
id: toolCall.id,
|
|
432
|
+
name: toolCall.name
|
|
433
|
+
};
|
|
434
|
+
if (!tool) {
|
|
435
|
+
return {
|
|
436
|
+
...part,
|
|
437
|
+
result: `Unknown tool '${toolCall.name}'`,
|
|
438
|
+
isError: true
|
|
439
|
+
};
|
|
440
|
+
}
|
|
441
|
+
const args = value_1.Value.Convert(tool.parameters, value_1.Value.Default(tool.parameters, structuredClone(toolCall.arguments ?? {})));
|
|
442
|
+
const errors = (0, json_util_1.schemaErrors)(tool.parameters, args);
|
|
443
|
+
if (errors.length) {
|
|
444
|
+
return {
|
|
445
|
+
...part,
|
|
446
|
+
result: `Invalid arguments: ${errors.join('; ')}`,
|
|
447
|
+
isError: true
|
|
448
|
+
};
|
|
449
|
+
}
|
|
450
|
+
try {
|
|
451
|
+
const result = await (0, telemetry_1.traceToolCall)(toolCall, async () => tool.execute(args, { toolCall, messages, signal }));
|
|
452
|
+
return { ...part, result: result ?? null };
|
|
453
|
+
}
|
|
454
|
+
catch (error) {
|
|
455
|
+
signal?.throwIfAborted();
|
|
456
|
+
logger.warn({ tool: toolCall.name, error: error.message }, 'AI tool failed');
|
|
457
|
+
return {
|
|
458
|
+
...part,
|
|
459
|
+
result: `The tool failed: ${error.message}`,
|
|
460
|
+
isError: true
|
|
461
|
+
};
|
|
462
|
+
}
|
|
463
|
+
}));
|
|
464
|
+
}
|
|
465
|
+
/** @internal */
|
|
466
|
+
async prepare(prompt) {
|
|
467
|
+
const messages = [];
|
|
468
|
+
if (prompt.system !== undefined) {
|
|
469
|
+
messages.push({
|
|
470
|
+
role: 'system',
|
|
471
|
+
content: await this.renderPrompt(prompt.system)
|
|
472
|
+
});
|
|
473
|
+
}
|
|
474
|
+
messages.push(...(prompt.messages ?? []));
|
|
475
|
+
if (prompt.prompt !== undefined) {
|
|
476
|
+
messages.push({
|
|
477
|
+
role: 'user',
|
|
478
|
+
content: Array.isArray(prompt.prompt)
|
|
479
|
+
? prompt.prompt
|
|
480
|
+
: await this.renderPrompt(prompt.prompt)
|
|
481
|
+
});
|
|
482
|
+
}
|
|
483
|
+
if (!messages.length) {
|
|
484
|
+
throw new common_1.AiError(common_1.ErrorCode.AiRequestInvalid, 'An AI prompt needs a system prompt, a prompt or messages', { errors: ['prompt: expected a prompt or messages'] });
|
|
485
|
+
}
|
|
486
|
+
const tools = (prompt.tools ?? []).map((tool) => {
|
|
487
|
+
const found = typeof tool === 'string' ? (0, create_tool_1.findAiTool)(tool) : tool;
|
|
488
|
+
if (!found) {
|
|
489
|
+
throw new common_1.AiError(common_1.ErrorCode.AiRequestInvalid, `Unknown AI tool '${tool}', create it with createAiTool`, { errors: [`tools: unknown tool '${tool}'`] });
|
|
490
|
+
}
|
|
491
|
+
(0, create_tool_1.validateToolName)(found.name);
|
|
492
|
+
return found;
|
|
493
|
+
});
|
|
494
|
+
return {
|
|
495
|
+
messages,
|
|
496
|
+
tools,
|
|
497
|
+
maxSteps: Math.max(1, prompt.maxSteps ?? config_1.aiConfig.MAX_STEPS),
|
|
498
|
+
options: { signal: prompt.signal, timeout: prompt.timeout },
|
|
499
|
+
request: {
|
|
500
|
+
maxTokens: prompt.maxTokens ?? config_1.aiConfig.MAX_TOKENS,
|
|
501
|
+
temperature: prompt.temperature ?? config_1.aiConfig.TEMPERATURE,
|
|
502
|
+
topP: prompt.topP,
|
|
503
|
+
stop: prompt.stop,
|
|
504
|
+
seed: prompt.seed,
|
|
505
|
+
reasoning: prompt.reasoning ? { effort: prompt.reasoning } : undefined,
|
|
506
|
+
toolChoice: prompt.toolChoice,
|
|
507
|
+
responseFormat: prompt.responseFormat,
|
|
508
|
+
providerOptions: prompt.providerOptions,
|
|
509
|
+
tools: tools.length
|
|
510
|
+
? tools.map(({ name, description, parameters }) => ({
|
|
511
|
+
name,
|
|
512
|
+
description,
|
|
513
|
+
parameters
|
|
514
|
+
}))
|
|
515
|
+
: undefined
|
|
516
|
+
}
|
|
517
|
+
};
|
|
518
|
+
}
|
|
519
|
+
/** @internal */
|
|
520
|
+
async renderPrompt(prompt) {
|
|
521
|
+
if (typeof prompt === 'string') {
|
|
522
|
+
return prompt;
|
|
523
|
+
}
|
|
524
|
+
const engine = this._templateEngine;
|
|
525
|
+
if (!engine) {
|
|
526
|
+
throw new common_1.ConfigurationError(common_1.ErrorCode.ConfigurationInvalid, `AI prompt template '${prompt.template}' requires a template engine`);
|
|
527
|
+
}
|
|
528
|
+
const name = /\.[a-z0-9]+$/i.test(prompt.template)
|
|
529
|
+
? prompt.template
|
|
530
|
+
: `${prompt.template}.md`;
|
|
531
|
+
if (!(await engine.hasTemplate(name, prompt.locale))) {
|
|
532
|
+
throw new common_1.TemplateError(common_1.ErrorCode.TemplateNotFound, `AI prompt template '${name}' not found`, { template: name, locale: prompt.locale });
|
|
533
|
+
}
|
|
534
|
+
return (await engine.render(name, prompt.data ?? {}, { locale: prompt.locale })).trim();
|
|
535
|
+
}
|
|
536
|
+
/** @internal */
|
|
537
|
+
queue() {
|
|
538
|
+
if (!this._queue) {
|
|
539
|
+
throw new common_1.ConfigurationError(common_1.ErrorCode.ConfigurationInvalid, 'The queued AI calls require a queue (queue.provider)');
|
|
540
|
+
}
|
|
541
|
+
return this._queue;
|
|
542
|
+
}
|
|
543
|
+
/** @internal */
|
|
544
|
+
pubsub() {
|
|
545
|
+
if (!this._pubsub) {
|
|
546
|
+
throw new common_1.ConfigurationError(common_1.ErrorCode.ConfigurationInvalid, 'The queued AI streams require a pub/sub (pubsub.provider)');
|
|
547
|
+
}
|
|
548
|
+
return this._pubsub;
|
|
549
|
+
}
|
|
550
|
+
/** @internal */
|
|
551
|
+
async processJob(job) {
|
|
552
|
+
const { prompt, stream } = job.data;
|
|
553
|
+
if (!stream) {
|
|
554
|
+
return this.generate(prompt);
|
|
555
|
+
}
|
|
556
|
+
const pubsub = this.pubsub();
|
|
557
|
+
const channel = `ai:stream:${stream}`;
|
|
558
|
+
const controller = new AbortController();
|
|
559
|
+
const id = await pubsub.subscribe(`${channel}:cancel`, () => controller.abort());
|
|
560
|
+
let buffer = [];
|
|
561
|
+
let flushed = Promise.resolve();
|
|
562
|
+
const flush = () => {
|
|
563
|
+
if (buffer.length) {
|
|
564
|
+
const parts = buffer;
|
|
565
|
+
buffer = [];
|
|
566
|
+
flushed = flushed.then(() => pubsub.publish(channel, { parts }));
|
|
567
|
+
}
|
|
568
|
+
};
|
|
569
|
+
const timer = setInterval(flush, config_1.aiConfig.STREAM_FLUSH_INTERVAL);
|
|
570
|
+
try {
|
|
571
|
+
const aiStream = this.stream({ ...prompt, signal: controller.signal });
|
|
572
|
+
for await (const part of aiStream.parts()) {
|
|
573
|
+
buffer.push(part);
|
|
574
|
+
}
|
|
575
|
+
const result = await aiStream.result;
|
|
576
|
+
flush();
|
|
577
|
+
await flushed;
|
|
578
|
+
await pubsub.publish(channel, { result });
|
|
579
|
+
return result;
|
|
580
|
+
}
|
|
581
|
+
finally {
|
|
582
|
+
clearInterval(timer);
|
|
583
|
+
await pubsub.unsubscribe(id);
|
|
584
|
+
}
|
|
585
|
+
}
|
|
586
|
+
}
|
|
587
|
+
exports.AiService = AiService;
|
|
588
|
+
/** Streams the parts of a model response, returning the response they make up. */
|
|
589
|
+
async function* streamResponse(parts) {
|
|
590
|
+
const response = {
|
|
591
|
+
text: '',
|
|
592
|
+
toolCalls: [],
|
|
593
|
+
finishReason: 'stop'
|
|
594
|
+
};
|
|
595
|
+
for await (const part of parts) {
|
|
596
|
+
if (part.type === 'finish') {
|
|
597
|
+
const { finishReason, usage, id, model } = part;
|
|
598
|
+
Object.assign(response, { finishReason, usage, id, model });
|
|
599
|
+
continue;
|
|
600
|
+
}
|
|
601
|
+
if (part.type === 'text-delta') {
|
|
602
|
+
response.text += part.text;
|
|
603
|
+
}
|
|
604
|
+
else if (part.type === 'reasoning-delta') {
|
|
605
|
+
response.reasoning = (response.reasoning ?? '') + part.text;
|
|
606
|
+
}
|
|
607
|
+
else if (part.type === 'tool-call') {
|
|
608
|
+
response.toolCalls.push(part.toolCall);
|
|
609
|
+
}
|
|
610
|
+
yield part;
|
|
611
|
+
}
|
|
612
|
+
return response;
|
|
613
|
+
}
|
|
614
|
+
/** Reads a generator to its end, returning its value. */
|
|
615
|
+
async function drain(generator) {
|
|
616
|
+
while (true) {
|
|
617
|
+
const next = await generator.next();
|
|
618
|
+
if (next.done) {
|
|
619
|
+
return next.value;
|
|
620
|
+
}
|
|
621
|
+
}
|
|
622
|
+
}
|
|
623
|
+
/** Throws the error of a model unless the next model of the alias may not have it. */
|
|
624
|
+
function nextModel(models, index, error, streamed = false) {
|
|
625
|
+
const retriable = (0, common_1.isAppweaverError)(error) && RETRIABLE.has(error.code);
|
|
626
|
+
if (streamed || !retriable || index === models.length - 1) {
|
|
627
|
+
throw error;
|
|
628
|
+
}
|
|
629
|
+
logger.warn({
|
|
630
|
+
model: models[index].reference,
|
|
631
|
+
next: models[index + 1].reference,
|
|
632
|
+
error: error.message
|
|
633
|
+
}, 'AI model failed, calling the next model');
|
|
634
|
+
}
|
|
635
|
+
/** Asks for an object of the schema: the JSON schema mode, else the JSON mode or the instructions of the prompt. */
|
|
636
|
+
function objectFormat(prompt) {
|
|
637
|
+
return (prepared, driver, model) => {
|
|
638
|
+
if (driver.supports('json-schema', model)) {
|
|
639
|
+
const name = prompt.schemaName ?? 'response';
|
|
640
|
+
const responseFormat = {
|
|
641
|
+
type: 'json-schema',
|
|
642
|
+
schema: prompt.schema,
|
|
643
|
+
name
|
|
644
|
+
};
|
|
645
|
+
return { ...prepared, request: { ...prepared.request, responseFormat } };
|
|
646
|
+
}
|
|
647
|
+
const instruction = `Respond with only a JSON value matching this JSON Schema:\n${JSON.stringify(prompt.schema)}`;
|
|
648
|
+
const instructed = prepared.messages.some((message) => message.content === instruction);
|
|
649
|
+
return {
|
|
650
|
+
...prepared,
|
|
651
|
+
messages: instructed
|
|
652
|
+
? prepared.messages
|
|
653
|
+
: [{ role: 'system', content: instruction }, ...prepared.messages],
|
|
654
|
+
request: {
|
|
655
|
+
...prepared.request,
|
|
656
|
+
responseFormat: driver.supports('json', model)
|
|
657
|
+
? { type: 'json' }
|
|
658
|
+
: undefined
|
|
659
|
+
}
|
|
660
|
+
};
|
|
661
|
+
};
|
|
662
|
+
}
|
|
663
|
+
function outputInvalid(model, errors) {
|
|
664
|
+
return new common_1.AiError(common_1.ErrorCode.AiOutputInvalid, `The output of the AI model '${model}' does not match the schema`, { model, errors });
|
|
665
|
+
}
|
|
666
|
+
/** Checks that the model has the capabilities a request needs. */
|
|
667
|
+
function checkCapabilities(driver, model, request) {
|
|
668
|
+
if (request.tools?.length) {
|
|
669
|
+
requireCapability(driver, model, 'tools');
|
|
670
|
+
}
|
|
671
|
+
const format = request.responseFormat?.type;
|
|
672
|
+
if (format === 'json' || format === 'json-schema') {
|
|
673
|
+
requireCapability(driver, model, format);
|
|
674
|
+
}
|
|
675
|
+
if (request.reasoning) {
|
|
676
|
+
requireCapability(driver, model, 'reasoning');
|
|
677
|
+
}
|
|
678
|
+
for (const message of request.messages) {
|
|
679
|
+
if (message.role === 'user' && Array.isArray(message.content)) {
|
|
680
|
+
for (const part of message.content) {
|
|
681
|
+
if (part.type === 'image') {
|
|
682
|
+
requireCapability(driver, model, 'images');
|
|
683
|
+
}
|
|
684
|
+
else if (part.type === 'file') {
|
|
685
|
+
requireCapability(driver, model, 'files');
|
|
686
|
+
}
|
|
687
|
+
}
|
|
688
|
+
}
|
|
689
|
+
}
|
|
690
|
+
}
|
|
691
|
+
function requireCapability(driver, model, capability) {
|
|
692
|
+
if (!driver.supports(capability, model.model)) {
|
|
693
|
+
throw new common_1.AiError(common_1.ErrorCode.AiCapabilityUnsupported, `The AI model '${model.reference}' does not support ${capability}, set the capabilities of the endpoint '${model.endpoint}' if it does`, { model: model.reference, capability });
|
|
694
|
+
}
|
|
695
|
+
}
|
|
696
|
+
/** The assistant message of a response, without its reasoning. */
|
|
697
|
+
function assistantMessage(response) {
|
|
698
|
+
if (!response.toolCalls.length) {
|
|
699
|
+
return { role: 'assistant', content: response.text };
|
|
700
|
+
}
|
|
701
|
+
return {
|
|
702
|
+
role: 'assistant',
|
|
703
|
+
content: [
|
|
704
|
+
...(response.text
|
|
705
|
+
? [{ type: 'text', text: response.text }]
|
|
706
|
+
: []),
|
|
707
|
+
...response.toolCalls.map((call) => ({
|
|
708
|
+
type: 'tool-call',
|
|
709
|
+
...call
|
|
710
|
+
}))
|
|
711
|
+
]
|
|
712
|
+
};
|
|
713
|
+
}
|
|
714
|
+
function sumUsage(steps) {
|
|
715
|
+
const usages = steps.map((step) => step.response.usage);
|
|
716
|
+
const sum = (key) => {
|
|
717
|
+
const values = usages
|
|
718
|
+
.map((usage) => usage?.[key])
|
|
719
|
+
.filter((value) => value !== undefined);
|
|
720
|
+
return values.length ? values.reduce((a, b) => a + b, 0) : undefined;
|
|
721
|
+
};
|
|
722
|
+
const usage = {
|
|
723
|
+
inputTokens: sum('inputTokens') ?? 0,
|
|
724
|
+
outputTokens: sum('outputTokens') ?? 0,
|
|
725
|
+
totalTokens: sum('totalTokens') ?? 0
|
|
726
|
+
};
|
|
727
|
+
const cached = sum('cachedInputTokens');
|
|
728
|
+
const reasoning = sum('reasoningTokens');
|
|
729
|
+
if (cached !== undefined) {
|
|
730
|
+
usage.cachedInputTokens = cached;
|
|
731
|
+
}
|
|
732
|
+
if (reasoning !== undefined) {
|
|
733
|
+
usage.reasoningTokens = reasoning;
|
|
734
|
+
}
|
|
735
|
+
return usage;
|
|
736
|
+
}
|
|
737
|
+
/** Parses the JSON of an output and checks it against a schema, filling in its defaults. */
|
|
738
|
+
function checkObject(schema, text) {
|
|
739
|
+
let value;
|
|
740
|
+
try {
|
|
741
|
+
value = (0, json_util_1.parseJsonOutput)(text);
|
|
742
|
+
}
|
|
743
|
+
catch (error) {
|
|
744
|
+
return {
|
|
745
|
+
valid: false,
|
|
746
|
+
errors: [`invalid JSON: ${error.message}`]
|
|
747
|
+
};
|
|
748
|
+
}
|
|
749
|
+
value = value_1.Value.Default(schema, value);
|
|
750
|
+
const errors = (0, json_util_1.schemaErrors)(schema, value);
|
|
751
|
+
return errors.length ? { valid: false, errors } : { valid: true, value };
|
|
752
|
+
}
|
|
753
|
+
/** Checks that a queued prompt names its tools, carrying no functions. */
|
|
754
|
+
function queuedPrompt(prompt) {
|
|
755
|
+
const invalid = (prompt.tools ?? []).filter((tool) => typeof tool !== 'string');
|
|
756
|
+
if (invalid.length) {
|
|
757
|
+
throw new common_1.AiError(common_1.ErrorCode.AiRequestInvalid, 'A queued AI prompt names its tools, created with createAiTool', { errors: ['tools: expected the names of the tools'] });
|
|
758
|
+
}
|
|
759
|
+
return prompt;
|
|
760
|
+
}
|
|
761
|
+
/** Creates the error of a failed AI job, published by the instance that ran it. */
|
|
762
|
+
function aiError(data) {
|
|
763
|
+
if (data.code?.startsWith('AI_')) {
|
|
764
|
+
return new common_1.AiError(data.code, data.message, (data.details ?? {}));
|
|
765
|
+
}
|
|
766
|
+
const error = new Error(data.message);
|
|
767
|
+
error.name = data.name ?? 'Error';
|
|
768
|
+
return error;
|
|
769
|
+
}
|