@bratsos/workflow-engine 0.0.11 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +270 -513
- package/dist/chunk-D7RVRRM2.js +3 -0
- package/dist/chunk-D7RVRRM2.js.map +1 -0
- package/dist/chunk-HL3OJG7W.js +1033 -0
- package/dist/chunk-HL3OJG7W.js.map +1 -0
- package/dist/chunk-MUWP5SF2.js +33 -0
- package/dist/chunk-MUWP5SF2.js.map +1 -0
- package/dist/chunk-NYKMT46J.js +1143 -0
- package/dist/chunk-NYKMT46J.js.map +1 -0
- package/dist/chunk-P4KMGCT3.js +2292 -0
- package/dist/chunk-P4KMGCT3.js.map +1 -0
- package/dist/chunk-SPXBCZLB.js +17 -0
- package/dist/chunk-SPXBCZLB.js.map +1 -0
- package/dist/cli/sync-models.d.ts +1 -0
- package/dist/cli/sync-models.js +210 -0
- package/dist/cli/sync-models.js.map +1 -0
- package/dist/client-D4PoxADF.d.ts +798 -0
- package/dist/client.d.ts +5 -0
- package/dist/client.js +4 -0
- package/dist/client.js.map +1 -0
- package/dist/index-DAzCfO1R.d.ts +217 -0
- package/dist/index.d.ts +569 -0
- package/dist/index.js +399 -0
- package/dist/index.js.map +1 -0
- package/dist/interface-MMqhfQQK.d.ts +411 -0
- package/dist/kernel/index.d.ts +26 -0
- package/dist/kernel/index.js +3 -0
- package/dist/kernel/index.js.map +1 -0
- package/dist/kernel/testing/index.d.ts +44 -0
- package/dist/kernel/testing/index.js +85 -0
- package/dist/kernel/testing/index.js.map +1 -0
- package/dist/persistence/index.d.ts +2 -0
- package/dist/persistence/index.js +6 -0
- package/dist/persistence/index.js.map +1 -0
- package/dist/persistence/prisma/index.d.ts +37 -0
- package/dist/persistence/prisma/index.js +5 -0
- package/dist/persistence/prisma/index.js.map +1 -0
- package/dist/plugins-BCnDUwIc.d.ts +415 -0
- package/dist/ports-tU3rzPXJ.d.ts +245 -0
- package/dist/stage-BPw7m9Wx.d.ts +144 -0
- package/dist/testing/index.d.ts +264 -0
- package/dist/testing/index.js +920 -0
- package/dist/testing/index.js.map +1 -0
- package/package.json +11 -1
- package/skills/workflow-engine/SKILL.md +234 -348
- package/skills/workflow-engine/references/03-runtime-setup.md +111 -426
- package/skills/workflow-engine/references/05-persistence-setup.md +32 -0
- package/skills/workflow-engine/references/07-testing-patterns.md +141 -474
- package/skills/workflow-engine/references/08-common-patterns.md +118 -431
|
@@ -0,0 +1,2292 @@
|
|
|
1
|
+
import { createLogger } from './chunk-MUWP5SF2.js';
|
|
2
|
+
import z2, { z } from 'zod';
|
|
3
|
+
import { Anthropic } from '@anthropic-ai/sdk';
|
|
4
|
+
import { jsonSchema, generateText, Output, embed, streamText } from 'ai';
|
|
5
|
+
import { GoogleGenAI, FunctionCallingConfigMode, JobState } from '@google/genai';
|
|
6
|
+
import OpenAI from 'openai';
|
|
7
|
+
import { google } from '@ai-sdk/google';
|
|
8
|
+
import { openrouter } from '@openrouter/ai-sdk-provider';
|
|
9
|
+
|
|
10
|
+
var NoInputSchema = z.object({});
|
|
11
|
+
function requireStageOutput(workflowContext, stageId, field) {
|
|
12
|
+
const stageOutput = workflowContext[stageId];
|
|
13
|
+
if (!stageOutput) {
|
|
14
|
+
throw new Error(
|
|
15
|
+
`Missing output from required stage: ${stageId}. Check that this stage has been executed before the current stage. Available stages: ${Object.keys(workflowContext).join(", ")}`
|
|
16
|
+
);
|
|
17
|
+
}
|
|
18
|
+
if (field) {
|
|
19
|
+
if (typeof stageOutput !== "object" || stageOutput === null) {
|
|
20
|
+
throw new Error(
|
|
21
|
+
`Stage ${stageId} output is not an object, cannot access field '${field}'. Received: ${typeof stageOutput}`
|
|
22
|
+
);
|
|
23
|
+
}
|
|
24
|
+
if (!(field in stageOutput)) {
|
|
25
|
+
const availableFields = Object.keys(stageOutput);
|
|
26
|
+
throw new Error(
|
|
27
|
+
`Missing required field '${field}' in ${stageId} output. Available fields: ${availableFields.join(", ")}`
|
|
28
|
+
);
|
|
29
|
+
}
|
|
30
|
+
return stageOutput[field];
|
|
31
|
+
}
|
|
32
|
+
return stageOutput;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
// src/core/stage-factory.ts
|
|
36
|
+
function defineStage(definition) {
|
|
37
|
+
const inputSchema = definition.schemas.input === "none" ? NoInputSchema : definition.schemas.input;
|
|
38
|
+
const isAsyncBatch = "mode" in definition && definition.mode === "async-batch";
|
|
39
|
+
const stage = {
|
|
40
|
+
id: definition.id,
|
|
41
|
+
name: definition.name,
|
|
42
|
+
description: definition.description,
|
|
43
|
+
mode: isAsyncBatch ? "async-batch" : "sync",
|
|
44
|
+
dependencies: definition.dependencies,
|
|
45
|
+
inputSchema,
|
|
46
|
+
outputSchema: definition.schemas.output,
|
|
47
|
+
configSchema: definition.schemas.config,
|
|
48
|
+
async execute(context) {
|
|
49
|
+
const enhancedContext = createEnhancedContext(
|
|
50
|
+
context
|
|
51
|
+
);
|
|
52
|
+
const result = await definition.execute(enhancedContext);
|
|
53
|
+
if ("suspended" in result && result.suspended === true) {
|
|
54
|
+
const suspendedResult = result;
|
|
55
|
+
return {
|
|
56
|
+
suspended: true,
|
|
57
|
+
state: suspendedResult.state,
|
|
58
|
+
pollConfig: suspendedResult.pollConfig,
|
|
59
|
+
metrics: {
|
|
60
|
+
startTime: 0,
|
|
61
|
+
endTime: 0,
|
|
62
|
+
duration: 0,
|
|
63
|
+
...suspendedResult.customMetrics
|
|
64
|
+
}
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
const simpleResult = result;
|
|
68
|
+
const stageResult = {
|
|
69
|
+
output: simpleResult.output,
|
|
70
|
+
metrics: {
|
|
71
|
+
// Timing is set by executor - we provide placeholders
|
|
72
|
+
startTime: 0,
|
|
73
|
+
endTime: 0,
|
|
74
|
+
duration: 0,
|
|
75
|
+
// Custom metrics from stage (including AI metrics if stage tracked them)
|
|
76
|
+
...simpleResult.customMetrics
|
|
77
|
+
},
|
|
78
|
+
artifacts: simpleResult.artifacts
|
|
79
|
+
};
|
|
80
|
+
return stageResult;
|
|
81
|
+
},
|
|
82
|
+
estimateCost: definition.estimateCost
|
|
83
|
+
};
|
|
84
|
+
if (isAsyncBatch) {
|
|
85
|
+
const asyncDef = definition;
|
|
86
|
+
stage.checkCompletion = asyncDef.checkCompletion;
|
|
87
|
+
}
|
|
88
|
+
return stage;
|
|
89
|
+
}
|
|
90
|
+
function createEnhancedContext(context) {
|
|
91
|
+
return {
|
|
92
|
+
...context,
|
|
93
|
+
require(stageId) {
|
|
94
|
+
const output = context.workflowContext[stageId];
|
|
95
|
+
if (output === void 0) {
|
|
96
|
+
const availableStages = Object.keys(context.workflowContext);
|
|
97
|
+
throw new Error(
|
|
98
|
+
`Missing required stage output: "${String(stageId)}". Available stages: ${availableStages.length > 0 ? availableStages.join(", ") : "(none)"}`
|
|
99
|
+
);
|
|
100
|
+
}
|
|
101
|
+
return output;
|
|
102
|
+
},
|
|
103
|
+
optional(stageId) {
|
|
104
|
+
return context.workflowContext[stageId];
|
|
105
|
+
}
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
function defineAsyncBatchStage(definition) {
|
|
109
|
+
return defineStage(definition);
|
|
110
|
+
}
|
|
111
|
+
var tiktokenPromise = null;
|
|
112
|
+
async function getTiktoken() {
|
|
113
|
+
if (!tiktokenPromise) {
|
|
114
|
+
tiktokenPromise = Promise.all([
|
|
115
|
+
import('js-tiktoken/lite'),
|
|
116
|
+
import('js-tiktoken/ranks/cl100k_base')
|
|
117
|
+
]).then(([{ Tiktoken }, cl100kModule]) => ({
|
|
118
|
+
Tiktoken,
|
|
119
|
+
cl100k_base: cl100kModule.default
|
|
120
|
+
}));
|
|
121
|
+
}
|
|
122
|
+
return tiktokenPromise;
|
|
123
|
+
}
|
|
124
|
+
var MODEL_REGISTRY = {};
|
|
125
|
+
function registerModels(models) {
|
|
126
|
+
Object.assign(MODEL_REGISTRY, models);
|
|
127
|
+
}
|
|
128
|
+
function getRegisteredModel(key) {
|
|
129
|
+
return MODEL_REGISTRY[key];
|
|
130
|
+
}
|
|
131
|
+
function listRegisteredModels() {
|
|
132
|
+
return Object.entries(MODEL_REGISTRY).map(([key, config]) => ({
|
|
133
|
+
key,
|
|
134
|
+
config
|
|
135
|
+
}));
|
|
136
|
+
}
|
|
137
|
+
var ModelKeyEnum = z2.enum(["gemini-2.5-flash"]);
|
|
138
|
+
var ModelKey = z2.string().refine(
|
|
139
|
+
(key) => {
|
|
140
|
+
if (ModelKeyEnum.safeParse(key).success) {
|
|
141
|
+
return true;
|
|
142
|
+
}
|
|
143
|
+
return MODEL_REGISTRY[key] !== void 0;
|
|
144
|
+
},
|
|
145
|
+
{
|
|
146
|
+
message: "Model not found. Make sure to import the generated models file or register the model."
|
|
147
|
+
}
|
|
148
|
+
).transform((key) => key);
|
|
149
|
+
var AVAILABLE_MODELS = {
|
|
150
|
+
[ModelKeyEnum.enum["gemini-2.5-flash"]]: {
|
|
151
|
+
id: "google/gemini-2.5-flash-preview-09-2025",
|
|
152
|
+
name: "Gemini 2.5 Flash Preview",
|
|
153
|
+
inputCostPerMillion: 0.3,
|
|
154
|
+
outputCostPerMillion: 2.5,
|
|
155
|
+
provider: "openrouter",
|
|
156
|
+
description: "Fast, efficient model for general tasks",
|
|
157
|
+
supportsAsyncBatch: true,
|
|
158
|
+
batchDiscountPercent: 50
|
|
159
|
+
}
|
|
160
|
+
};
|
|
161
|
+
var DEFAULT_MODEL_KEY = "gemini-2.5-flash";
|
|
162
|
+
function getModel(key) {
|
|
163
|
+
const builtInModel = AVAILABLE_MODELS[key];
|
|
164
|
+
if (builtInModel) {
|
|
165
|
+
return builtInModel;
|
|
166
|
+
}
|
|
167
|
+
const registeredModel = MODEL_REGISTRY[key];
|
|
168
|
+
if (registeredModel) {
|
|
169
|
+
return registeredModel;
|
|
170
|
+
}
|
|
171
|
+
const allKeys = [
|
|
172
|
+
...Object.keys(AVAILABLE_MODELS),
|
|
173
|
+
...Object.keys(MODEL_REGISTRY)
|
|
174
|
+
];
|
|
175
|
+
throw new Error(
|
|
176
|
+
`Model "${key}" not found. Available models: ${allKeys.join(", ")}`
|
|
177
|
+
);
|
|
178
|
+
}
|
|
179
|
+
function getDefaultModel() {
|
|
180
|
+
return getModel(DEFAULT_MODEL_KEY);
|
|
181
|
+
}
|
|
182
|
+
function listModels(filter) {
|
|
183
|
+
const builtIn = Object.entries(AVAILABLE_MODELS).map(([key, config]) => ({
|
|
184
|
+
key,
|
|
185
|
+
config
|
|
186
|
+
}));
|
|
187
|
+
const registered = Object.entries(MODEL_REGISTRY).map(([key, config]) => ({
|
|
188
|
+
key,
|
|
189
|
+
config
|
|
190
|
+
}));
|
|
191
|
+
const merged = /* @__PURE__ */ new Map();
|
|
192
|
+
for (const item of builtIn) {
|
|
193
|
+
merged.set(item.key, item);
|
|
194
|
+
}
|
|
195
|
+
for (const item of registered) {
|
|
196
|
+
merged.set(item.key, item);
|
|
197
|
+
}
|
|
198
|
+
let models = Array.from(merged.values());
|
|
199
|
+
if (filter) {
|
|
200
|
+
models = models.filter((item) => {
|
|
201
|
+
const { config } = item;
|
|
202
|
+
if (filter.isEmbeddingModel !== void 0) {
|
|
203
|
+
if (filter.isEmbeddingModel && !config.isEmbeddingModel) return false;
|
|
204
|
+
if (!filter.isEmbeddingModel && config.isEmbeddingModel) return false;
|
|
205
|
+
}
|
|
206
|
+
if (filter.supportsTools !== void 0) {
|
|
207
|
+
if (filter.supportsTools && !config.supportsTools) return false;
|
|
208
|
+
if (!filter.supportsTools && config.supportsTools) return false;
|
|
209
|
+
}
|
|
210
|
+
if (filter.supportsStructuredOutputs !== void 0) {
|
|
211
|
+
if (filter.supportsStructuredOutputs && !config.supportsStructuredOutputs)
|
|
212
|
+
return false;
|
|
213
|
+
if (!filter.supportsStructuredOutputs && config.supportsStructuredOutputs)
|
|
214
|
+
return false;
|
|
215
|
+
}
|
|
216
|
+
if (filter.supportsAsyncBatch !== void 0) {
|
|
217
|
+
if (filter.supportsAsyncBatch && !config.supportsAsyncBatch)
|
|
218
|
+
return false;
|
|
219
|
+
if (!filter.supportsAsyncBatch && config.supportsAsyncBatch)
|
|
220
|
+
return false;
|
|
221
|
+
}
|
|
222
|
+
return true;
|
|
223
|
+
});
|
|
224
|
+
}
|
|
225
|
+
return models.sort((a, b) => a.key.localeCompare(b.key));
|
|
226
|
+
}
|
|
227
|
+
function modelSupportsBatch(modelKey) {
|
|
228
|
+
const model = getModel(modelKey);
|
|
229
|
+
return model.supportsAsyncBatch === true;
|
|
230
|
+
}
|
|
231
|
+
function getModelById(modelKey, tracker) {
|
|
232
|
+
const modelConfig = getModel(modelKey);
|
|
233
|
+
return {
|
|
234
|
+
id: modelConfig.id,
|
|
235
|
+
name: modelConfig.name,
|
|
236
|
+
recordCall: (inputTokens, outputTokens) => {
|
|
237
|
+
if (!tracker) {
|
|
238
|
+
throw new Error("ModelStatsTracker required to use recordCall()");
|
|
239
|
+
}
|
|
240
|
+
tracker.recordCall(inputTokens, outputTokens, modelKey);
|
|
241
|
+
}
|
|
242
|
+
};
|
|
243
|
+
}
|
|
244
|
+
function calculateCost(modelKey, inputTokens, outputTokens) {
|
|
245
|
+
const model = getModel(modelKey);
|
|
246
|
+
const inputCost = inputTokens / 1e6 * model.inputCostPerMillion;
|
|
247
|
+
const outputCost = outputTokens / 1e6 * model.outputCostPerMillion;
|
|
248
|
+
const totalCost = inputCost + outputCost;
|
|
249
|
+
return {
|
|
250
|
+
inputCost,
|
|
251
|
+
outputCost,
|
|
252
|
+
totalCost
|
|
253
|
+
};
|
|
254
|
+
}
|
|
255
|
+
var ModelStatsTracker = class _ModelStatsTracker {
|
|
256
|
+
modelKey;
|
|
257
|
+
modelConfig;
|
|
258
|
+
stats;
|
|
259
|
+
perModelStats = /* @__PURE__ */ new Map();
|
|
260
|
+
isAggregating = false;
|
|
261
|
+
constructor(modelKey = DEFAULT_MODEL_KEY) {
|
|
262
|
+
this.modelKey = modelKey;
|
|
263
|
+
this.modelConfig = getModel(modelKey);
|
|
264
|
+
this.stats = {
|
|
265
|
+
apiCalls: 0,
|
|
266
|
+
inputTokens: 0,
|
|
267
|
+
outputTokens: 0
|
|
268
|
+
};
|
|
269
|
+
}
|
|
270
|
+
/**
|
|
271
|
+
* Create an aggregating tracker that combines stats from multiple models
|
|
272
|
+
* Perfect for parallel execution where different calls use different models
|
|
273
|
+
*/
|
|
274
|
+
static createAggregating() {
|
|
275
|
+
const tracker = new _ModelStatsTracker();
|
|
276
|
+
tracker.isAggregating = true;
|
|
277
|
+
tracker.modelKey = void 0;
|
|
278
|
+
tracker.modelConfig = void 0;
|
|
279
|
+
return tracker;
|
|
280
|
+
}
|
|
281
|
+
/**
|
|
282
|
+
* Get the model ID for use with AI SDK
|
|
283
|
+
* @deprecated Use getModelById(modelKey).id instead for parallel execution
|
|
284
|
+
*/
|
|
285
|
+
getModelId() {
|
|
286
|
+
if (!this.modelConfig) {
|
|
287
|
+
throw new Error("Model not set for this tracker");
|
|
288
|
+
}
|
|
289
|
+
return this.modelConfig.id;
|
|
290
|
+
}
|
|
291
|
+
/**
|
|
292
|
+
* Get the model configuration
|
|
293
|
+
* @deprecated Use getModelById(modelKey) instead for parallel execution
|
|
294
|
+
*/
|
|
295
|
+
getModelConfig() {
|
|
296
|
+
if (!this.modelConfig) {
|
|
297
|
+
throw new Error("Model not set for this tracker");
|
|
298
|
+
}
|
|
299
|
+
return this.modelConfig;
|
|
300
|
+
}
|
|
301
|
+
/**
|
|
302
|
+
* Switch model (useful for sequential model switching)
|
|
303
|
+
* @deprecated For parallel execution, pass model key to recordCall() instead
|
|
304
|
+
*/
|
|
305
|
+
switchModel(modelKey) {
|
|
306
|
+
this.modelKey = modelKey;
|
|
307
|
+
this.modelConfig = getModel(modelKey);
|
|
308
|
+
}
|
|
309
|
+
/**
|
|
310
|
+
* Get a model helper with bound recordCall for parallel execution
|
|
311
|
+
* Perfect for running multiple AI calls in parallel with different models
|
|
312
|
+
*
|
|
313
|
+
* Usage:
|
|
314
|
+
* const flashModel = tracker.getModelById("gemini-2.5-flash");
|
|
315
|
+
* const liteModel = tracker.getModelById("gemini-2.5-flash-lite");
|
|
316
|
+
*
|
|
317
|
+
* const [result1, result2] = await Promise.all([
|
|
318
|
+
* generateText({
|
|
319
|
+
* model: openRouter(flashModel.id),
|
|
320
|
+
* prompt: prompt1,
|
|
321
|
+
* }).then(r => { flashModel.recordCall(r.usage.inputTokens, r.usage.outputTokens); return r; }),
|
|
322
|
+
* generateText({
|
|
323
|
+
* model: openRouter(liteModel.id),
|
|
324
|
+
* prompt: prompt2,
|
|
325
|
+
* }).then(r => { liteModel.recordCall(r.usage.inputTokens, r.usage.outputTokens); return r; }),
|
|
326
|
+
* ]);
|
|
327
|
+
*/
|
|
328
|
+
getModelById(modelKey) {
|
|
329
|
+
const modelConfig = getModel(modelKey);
|
|
330
|
+
return {
|
|
331
|
+
id: modelConfig.id,
|
|
332
|
+
name: modelConfig.name,
|
|
333
|
+
recordCall: (inputTokens, outputTokens) => {
|
|
334
|
+
this.recordCall(inputTokens, outputTokens, modelKey);
|
|
335
|
+
}
|
|
336
|
+
};
|
|
337
|
+
}
|
|
338
|
+
/**
|
|
339
|
+
* Record an API call with token usage
|
|
340
|
+
*
|
|
341
|
+
* For sequential execution:
|
|
342
|
+
* tracker.switchModel("gemini-2.5-flash")
|
|
343
|
+
* tracker.recordCall(inputTokens, outputTokens)
|
|
344
|
+
*
|
|
345
|
+
* For parallel execution:
|
|
346
|
+
* tracker.recordCall(inputTokens, outputTokens, "gemini-2.5-flash")
|
|
347
|
+
* tracker.recordCall(inputTokens, outputTokens, "gemini-2.5-pro")
|
|
348
|
+
*/
|
|
349
|
+
recordCall(inputTokens = 0, outputTokens = 0, modelKeyOverride) {
|
|
350
|
+
const modelKeyToUse = modelKeyOverride || this.modelKey;
|
|
351
|
+
if (!modelKeyToUse) {
|
|
352
|
+
throw new Error(
|
|
353
|
+
"Model not set and no modelKeyOverride provided to recordCall()"
|
|
354
|
+
);
|
|
355
|
+
}
|
|
356
|
+
const modelConfig = getModel(modelKeyToUse);
|
|
357
|
+
this.stats.apiCalls += 1;
|
|
358
|
+
this.stats.inputTokens += inputTokens;
|
|
359
|
+
this.stats.outputTokens += outputTokens;
|
|
360
|
+
if (this.isAggregating) {
|
|
361
|
+
const modelId = modelConfig.id;
|
|
362
|
+
const existing = this.perModelStats.get(modelId) || {
|
|
363
|
+
modelName: modelConfig.name,
|
|
364
|
+
apiCalls: 0,
|
|
365
|
+
inputTokens: 0,
|
|
366
|
+
outputTokens: 0,
|
|
367
|
+
totalTokens: 0,
|
|
368
|
+
inputCost: 0,
|
|
369
|
+
outputCost: 0,
|
|
370
|
+
totalCost: 0
|
|
371
|
+
};
|
|
372
|
+
const costs = calculateCost(modelKeyToUse, inputTokens, outputTokens);
|
|
373
|
+
this.perModelStats.set(modelId, {
|
|
374
|
+
modelId,
|
|
375
|
+
modelName: modelConfig.name,
|
|
376
|
+
apiCalls: existing.apiCalls + 1,
|
|
377
|
+
inputTokens: existing.inputTokens + inputTokens,
|
|
378
|
+
outputTokens: existing.outputTokens + outputTokens,
|
|
379
|
+
totalTokens: existing.totalTokens + inputTokens + outputTokens,
|
|
380
|
+
inputCost: existing.inputCost + costs.inputCost,
|
|
381
|
+
outputCost: existing.outputCost + costs.outputCost,
|
|
382
|
+
totalCost: existing.totalCost + costs.totalCost
|
|
383
|
+
});
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
/**
|
|
387
|
+
* Estimate cost for a prompt without making an API call
|
|
388
|
+
* Useful for dry-run mode to preview costs
|
|
389
|
+
*
|
|
390
|
+
* Note: This method is async because it lazy-loads the tiktoken library
|
|
391
|
+
* to avoid bundling 2MB of tokenizer data for browser clients.
|
|
392
|
+
*
|
|
393
|
+
* @param prompt - The prompt text to estimate
|
|
394
|
+
* @param estimatedOutputTokens - Estimated number of output tokens (default: 500)
|
|
395
|
+
* @returns Object with token counts and cost estimates
|
|
396
|
+
*/
|
|
397
|
+
async estimateCost(prompt, estimatedOutputTokens = 500) {
|
|
398
|
+
if (!this.modelKey) {
|
|
399
|
+
throw new Error("Model not set for estimation");
|
|
400
|
+
}
|
|
401
|
+
const { Tiktoken, cl100k_base } = await getTiktoken();
|
|
402
|
+
const encoding = new Tiktoken(cl100k_base);
|
|
403
|
+
const inputTokens = encoding.encode(prompt).length;
|
|
404
|
+
const costs = calculateCost(
|
|
405
|
+
this.modelKey,
|
|
406
|
+
inputTokens,
|
|
407
|
+
estimatedOutputTokens
|
|
408
|
+
);
|
|
409
|
+
return {
|
|
410
|
+
inputTokens,
|
|
411
|
+
outputTokens: estimatedOutputTokens,
|
|
412
|
+
totalTokens: inputTokens + estimatedOutputTokens,
|
|
413
|
+
inputCost: costs.inputCost,
|
|
414
|
+
outputCost: costs.outputCost,
|
|
415
|
+
totalCost: costs.totalCost
|
|
416
|
+
};
|
|
417
|
+
}
|
|
418
|
+
/**
|
|
419
|
+
* Get current statistics (single model or aggregated)
|
|
420
|
+
* Returns null only if tracker is in aggregating mode - use getAggregatedStats() instead
|
|
421
|
+
*/
|
|
422
|
+
getStats() {
|
|
423
|
+
if (this.isAggregating) {
|
|
424
|
+
return null;
|
|
425
|
+
}
|
|
426
|
+
if (!this.modelKey || !this.modelConfig) {
|
|
427
|
+
throw new Error("Model not set for this tracker");
|
|
428
|
+
}
|
|
429
|
+
const totalTokens = this.stats.inputTokens + this.stats.outputTokens;
|
|
430
|
+
const costs = calculateCost(
|
|
431
|
+
this.modelKey,
|
|
432
|
+
this.stats.inputTokens,
|
|
433
|
+
this.stats.outputTokens
|
|
434
|
+
);
|
|
435
|
+
return {
|
|
436
|
+
modelId: this.modelConfig.id,
|
|
437
|
+
modelName: this.modelConfig.name,
|
|
438
|
+
apiCalls: this.stats.apiCalls,
|
|
439
|
+
inputTokens: this.stats.inputTokens,
|
|
440
|
+
outputTokens: this.stats.outputTokens,
|
|
441
|
+
totalTokens,
|
|
442
|
+
inputCost: costs.inputCost,
|
|
443
|
+
outputCost: costs.outputCost,
|
|
444
|
+
totalCost: costs.totalCost
|
|
445
|
+
};
|
|
446
|
+
}
|
|
447
|
+
/**
|
|
448
|
+
* Get aggregated statistics from all models
|
|
449
|
+
*/
|
|
450
|
+
getAggregatedStats() {
|
|
451
|
+
const perModel = Array.from(this.perModelStats.values());
|
|
452
|
+
const totals = {
|
|
453
|
+
totalApiCalls: perModel.reduce((sum, m) => sum + m.apiCalls, 0),
|
|
454
|
+
totalInputTokens: perModel.reduce((sum, m) => sum + m.inputTokens, 0),
|
|
455
|
+
totalOutputTokens: perModel.reduce((sum, m) => sum + m.outputTokens, 0),
|
|
456
|
+
totalTokens: perModel.reduce((sum, m) => sum + m.totalTokens, 0),
|
|
457
|
+
totalInputCost: perModel.reduce((sum, m) => sum + m.inputCost, 0),
|
|
458
|
+
totalOutputCost: perModel.reduce((sum, m) => sum + m.outputCost, 0),
|
|
459
|
+
totalCost: perModel.reduce((sum, m) => sum + m.totalCost, 0)
|
|
460
|
+
};
|
|
461
|
+
return { perModel, totals };
|
|
462
|
+
}
|
|
463
|
+
/**
|
|
464
|
+
* Print statistics to console
|
|
465
|
+
*/
|
|
466
|
+
printStats() {
|
|
467
|
+
if (this.isAggregating) {
|
|
468
|
+
this.printAggregatedStats();
|
|
469
|
+
} else {
|
|
470
|
+
const stats = this.getStats();
|
|
471
|
+
if (!stats) {
|
|
472
|
+
console.log("No statistics available");
|
|
473
|
+
return;
|
|
474
|
+
}
|
|
475
|
+
console.log("\n\u{1F4CA} API Usage Statistics:");
|
|
476
|
+
console.log(` Model: ${stats.modelName} (${stats.modelId})`);
|
|
477
|
+
console.log(` API Calls: ${stats.apiCalls}`);
|
|
478
|
+
console.log(` Input Tokens: ${stats.inputTokens.toLocaleString()}`);
|
|
479
|
+
console.log(` Output Tokens: ${stats.outputTokens.toLocaleString()}`);
|
|
480
|
+
console.log(` Total Tokens: ${stats.totalTokens.toLocaleString()}`);
|
|
481
|
+
if (stats.totalCost > 0) {
|
|
482
|
+
console.log(` Input Cost: $${stats.inputCost.toFixed(4)}`);
|
|
483
|
+
console.log(` Output Cost: $${stats.outputCost.toFixed(4)}`);
|
|
484
|
+
console.log(` Total Cost: $${stats.totalCost.toFixed(4)}`);
|
|
485
|
+
} else {
|
|
486
|
+
console.log(" Cost: Free / Not calculated");
|
|
487
|
+
}
|
|
488
|
+
}
|
|
489
|
+
}
|
|
490
|
+
/**
|
|
491
|
+
* Print aggregated statistics from all models
|
|
492
|
+
*/
|
|
493
|
+
printAggregatedStats() {
|
|
494
|
+
const { perModel, totals } = this.getAggregatedStats();
|
|
495
|
+
console.log("\n\u{1F4CA} Aggregated API Usage Statistics:");
|
|
496
|
+
console.log("\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n");
|
|
497
|
+
console.log("Per-Model Breakdown:");
|
|
498
|
+
for (const stats of perModel) {
|
|
499
|
+
console.log(`
|
|
500
|
+
${stats.modelName}`);
|
|
501
|
+
console.log(` API Calls: ${stats.apiCalls}`);
|
|
502
|
+
console.log(` Input Tokens: ${stats.inputTokens.toLocaleString()}`);
|
|
503
|
+
console.log(` Output Tokens: ${stats.outputTokens.toLocaleString()}`);
|
|
504
|
+
console.log(` Total Tokens: ${stats.totalTokens.toLocaleString()}`);
|
|
505
|
+
if (stats.totalCost > 0) {
|
|
506
|
+
console.log(` Input Cost: $${stats.inputCost.toFixed(4)}`);
|
|
507
|
+
console.log(` Output Cost: $${stats.outputCost.toFixed(4)}`);
|
|
508
|
+
console.log(` Total Cost: $${stats.totalCost.toFixed(4)}`);
|
|
509
|
+
} else {
|
|
510
|
+
console.log(` Cost: Free`);
|
|
511
|
+
}
|
|
512
|
+
}
|
|
513
|
+
console.log("\n\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500");
|
|
514
|
+
console.log("Totals Across All Models:");
|
|
515
|
+
console.log(` Total API Calls: ${totals.totalApiCalls}`);
|
|
516
|
+
console.log(
|
|
517
|
+
` Total Input Tokens: ${totals.totalInputTokens.toLocaleString()}`
|
|
518
|
+
);
|
|
519
|
+
console.log(
|
|
520
|
+
` Total Output Tokens: ${totals.totalOutputTokens.toLocaleString()}`
|
|
521
|
+
);
|
|
522
|
+
console.log(` Total Tokens: ${totals.totalTokens.toLocaleString()}`);
|
|
523
|
+
console.log(` Total Input Cost: $${totals.totalInputCost.toFixed(4)}`);
|
|
524
|
+
console.log(` Total Output Cost: $${totals.totalOutputCost.toFixed(4)}`);
|
|
525
|
+
console.log(` TOTAL COST: $${totals.totalCost.toFixed(4)}`);
|
|
526
|
+
console.log("\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n");
|
|
527
|
+
}
|
|
528
|
+
/**
|
|
529
|
+
* Reset statistics
|
|
530
|
+
*/
|
|
531
|
+
reset() {
|
|
532
|
+
this.stats = {
|
|
533
|
+
apiCalls: 0,
|
|
534
|
+
inputTokens: 0,
|
|
535
|
+
outputTokens: 0
|
|
536
|
+
};
|
|
537
|
+
this.perModelStats.clear();
|
|
538
|
+
}
|
|
539
|
+
};
|
|
540
|
+
function printAvailableModels() {
|
|
541
|
+
console.log("\n\u{1F4CB} Available Models:");
|
|
542
|
+
const models = listModels();
|
|
543
|
+
for (const { key, config } of models) {
|
|
544
|
+
const isDefault = key === DEFAULT_MODEL_KEY;
|
|
545
|
+
const defaultMarker = isDefault ? " (DEFAULT)" : "";
|
|
546
|
+
const costInfo = config.inputCostPerMillion === 0 && config.outputCostPerMillion === 0 ? "Free" : `$${config.inputCostPerMillion}/1M in, $${config.outputCostPerMillion}/1M out`;
|
|
547
|
+
console.log(` ${key}${defaultMarker}`);
|
|
548
|
+
console.log(` Name: ${config.name}`);
|
|
549
|
+
console.log(` ID: ${config.id}`);
|
|
550
|
+
console.log(` Cost: ${costInfo}`);
|
|
551
|
+
if (config.description) {
|
|
552
|
+
console.log(` Description: ${config.description}`);
|
|
553
|
+
}
|
|
554
|
+
console.log("");
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
z2.enum(["google", "anthropic", "openai"]);
|
|
558
|
+
function extractNativeModelId(openRouterId, provider) {
|
|
559
|
+
const prefix = `${provider}/`;
|
|
560
|
+
if (openRouterId.startsWith(prefix)) {
|
|
561
|
+
return openRouterId.slice(prefix.length);
|
|
562
|
+
}
|
|
563
|
+
if (provider === "google" && openRouterId.startsWith("google/")) {
|
|
564
|
+
return openRouterId.slice("google/".length);
|
|
565
|
+
}
|
|
566
|
+
return openRouterId;
|
|
567
|
+
}
|
|
568
|
+
function getProviderFromOpenRouterId(openRouterId) {
|
|
569
|
+
if (openRouterId.startsWith("google/") || openRouterId.includes("gemini")) {
|
|
570
|
+
return "google";
|
|
571
|
+
}
|
|
572
|
+
if (openRouterId.startsWith("anthropic/") || openRouterId.includes("claude")) {
|
|
573
|
+
return "anthropic";
|
|
574
|
+
}
|
|
575
|
+
if (openRouterId.startsWith("openai/") || openRouterId.includes("gpt")) {
|
|
576
|
+
return "openai";
|
|
577
|
+
}
|
|
578
|
+
return void 0;
|
|
579
|
+
}
|
|
580
|
+
function getDefaultModelForProvider(provider) {
|
|
581
|
+
const models = listModels({
|
|
582
|
+
supportsAsyncBatch: true,
|
|
583
|
+
isEmbeddingModel: false
|
|
584
|
+
});
|
|
585
|
+
for (const { config } of models) {
|
|
586
|
+
const modelProvider = getProviderFromOpenRouterId(config.id);
|
|
587
|
+
if (modelProvider === provider) {
|
|
588
|
+
return extractNativeModelId(config.id, provider);
|
|
589
|
+
}
|
|
590
|
+
}
|
|
591
|
+
throw new Error(
|
|
592
|
+
`No batch-compatible models found for ${provider}. Ensure you have models with supportsAsyncBatch: true in your generated models file.`
|
|
593
|
+
);
|
|
594
|
+
}
|
|
595
|
+
function resolveModelForProvider(modelKey, provider) {
|
|
596
|
+
if (!modelKey) {
|
|
597
|
+
return getDefaultModelForProvider(provider);
|
|
598
|
+
}
|
|
599
|
+
const modelConfig = getModel(modelKey);
|
|
600
|
+
if (!modelConfig.supportsAsyncBatch) {
|
|
601
|
+
throw new Error(
|
|
602
|
+
`Model "${modelKey}" does not support async batch processing.`
|
|
603
|
+
);
|
|
604
|
+
}
|
|
605
|
+
const modelProvider = getProviderFromOpenRouterId(modelConfig.id);
|
|
606
|
+
if (modelProvider !== provider) {
|
|
607
|
+
throw new Error(
|
|
608
|
+
`Model "${modelKey}" belongs to ${modelProvider || "unknown"} provider, not ${provider}. Use a ${provider} model or change the batch provider.`
|
|
609
|
+
);
|
|
610
|
+
}
|
|
611
|
+
return extractNativeModelId(modelConfig.id, provider);
|
|
612
|
+
}
|
|
613
|
+
function getBestProviderForModel(modelKey) {
|
|
614
|
+
const modelConfig = getModel(modelKey);
|
|
615
|
+
if (!modelConfig.supportsAsyncBatch) {
|
|
616
|
+
return void 0;
|
|
617
|
+
}
|
|
618
|
+
return getProviderFromOpenRouterId(modelConfig.id);
|
|
619
|
+
}
|
|
620
|
+
var AnthropicBatchProvider = class {
|
|
621
|
+
name = "anthropic";
|
|
622
|
+
supportsBatching = true;
|
|
623
|
+
client;
|
|
624
|
+
logger;
|
|
625
|
+
constructor(config = {}, logger2) {
|
|
626
|
+
const apiKey = config.apiKey || process.env.ANTHROPIC_API_KEY;
|
|
627
|
+
if (!apiKey) {
|
|
628
|
+
throw new Error(
|
|
629
|
+
"Anthropic API key is required. Set ANTHROPIC_API_KEY or pass apiKey in config."
|
|
630
|
+
);
|
|
631
|
+
}
|
|
632
|
+
this.client = new Anthropic({ apiKey });
|
|
633
|
+
this.logger = logger2;
|
|
634
|
+
}
|
|
635
|
+
async submit(requests, options) {
|
|
636
|
+
if (requests.length === 0) {
|
|
637
|
+
throw new Error("Cannot submit empty batch");
|
|
638
|
+
}
|
|
639
|
+
const modelKey = requests[0]?.model;
|
|
640
|
+
const model = resolveModelForProvider(modelKey, "anthropic");
|
|
641
|
+
this.logger?.log("INFO", "Submitting Anthropic batch", {
|
|
642
|
+
requestCount: requests.length,
|
|
643
|
+
modelKey: modelKey || "default",
|
|
644
|
+
model
|
|
645
|
+
});
|
|
646
|
+
const batchRequests = requests.map((req, i) => {
|
|
647
|
+
const reqModel = req.model ? resolveModelForProvider(req.model, "anthropic") : model;
|
|
648
|
+
const anthropicTools = req.tools && Object.keys(req.tools).length > 0 ? Object.entries(req.tools).map(([name, tool]) => ({
|
|
649
|
+
name,
|
|
650
|
+
description: tool.description || "",
|
|
651
|
+
input_schema: tool.inputSchema ? jsonSchema(tool.inputSchema) : {
|
|
652
|
+
type: "object",
|
|
653
|
+
properties: {}
|
|
654
|
+
}
|
|
655
|
+
})) : void 0;
|
|
656
|
+
let toolChoice;
|
|
657
|
+
if (req.toolChoice) {
|
|
658
|
+
if (req.toolChoice === "auto") {
|
|
659
|
+
toolChoice = { type: "auto" };
|
|
660
|
+
} else if (req.toolChoice === "required") {
|
|
661
|
+
toolChoice = { type: "any" };
|
|
662
|
+
} else if (typeof req.toolChoice === "object" && req.toolChoice.type === "tool") {
|
|
663
|
+
toolChoice = { type: "tool", name: req.toolChoice.toolName };
|
|
664
|
+
}
|
|
665
|
+
}
|
|
666
|
+
return {
|
|
667
|
+
custom_id: req.customId || `request-${i}`,
|
|
668
|
+
params: {
|
|
669
|
+
model: reqModel,
|
|
670
|
+
max_tokens: req.maxTokens || 1024,
|
|
671
|
+
messages: [{ role: "user", content: req.prompt }],
|
|
672
|
+
...req.system && { system: req.system },
|
|
673
|
+
...req.temperature !== void 0 && {
|
|
674
|
+
temperature: req.temperature
|
|
675
|
+
},
|
|
676
|
+
...anthropicTools && req.toolChoice !== "none" && { tools: anthropicTools },
|
|
677
|
+
...toolChoice && { tool_choice: toolChoice }
|
|
678
|
+
}
|
|
679
|
+
};
|
|
680
|
+
});
|
|
681
|
+
const response = await this.client.messages.batches.create({
|
|
682
|
+
requests: batchRequests
|
|
683
|
+
});
|
|
684
|
+
this.logger?.log("INFO", "Anthropic batch submitted", {
|
|
685
|
+
batchId: response.id,
|
|
686
|
+
requestCount: requests.length,
|
|
687
|
+
processingStatus: response.processing_status
|
|
688
|
+
});
|
|
689
|
+
return {
|
|
690
|
+
id: response.id,
|
|
691
|
+
provider: this.name,
|
|
692
|
+
requestCount: requests.length,
|
|
693
|
+
createdAt: new Date(response.created_at),
|
|
694
|
+
metadata: {
|
|
695
|
+
model,
|
|
696
|
+
processingStatus: response.processing_status,
|
|
697
|
+
...options?.metadata
|
|
698
|
+
}
|
|
699
|
+
};
|
|
700
|
+
}
|
|
701
|
+
async checkStatus(handle) {
|
|
702
|
+
const batch = await this.client.messages.batches.retrieve(handle.id);
|
|
703
|
+
const succeededCount = batch.request_counts?.succeeded || 0;
|
|
704
|
+
const erroredCount = batch.request_counts?.errored || 0;
|
|
705
|
+
const canceledCount = batch.request_counts?.canceled || 0;
|
|
706
|
+
const expiredCount = batch.request_counts?.expired || 0;
|
|
707
|
+
const processingCount = batch.request_counts?.processing || 0;
|
|
708
|
+
const processedCount = succeededCount + erroredCount + canceledCount + expiredCount;
|
|
709
|
+
const totalCount = processedCount + processingCount;
|
|
710
|
+
const status = {
|
|
711
|
+
state: this.mapStatus(batch.processing_status),
|
|
712
|
+
processedCount,
|
|
713
|
+
totalCount: totalCount || handle.requestCount,
|
|
714
|
+
succeededCount,
|
|
715
|
+
failedCount: erroredCount + canceledCount + expiredCount
|
|
716
|
+
};
|
|
717
|
+
this.logger?.log("DEBUG", "Anthropic batch status", {
|
|
718
|
+
batchId: handle.id,
|
|
719
|
+
state: status.state,
|
|
720
|
+
processed: status.processedCount,
|
|
721
|
+
total: status.totalCount
|
|
722
|
+
});
|
|
723
|
+
return status;
|
|
724
|
+
}
|
|
725
|
+
async getResults(handle) {
|
|
726
|
+
const status = await this.checkStatus(handle);
|
|
727
|
+
if (status.state !== "completed" && status.state !== "failed") {
|
|
728
|
+
throw new Error(`Batch not complete: state=${status.state}`);
|
|
729
|
+
}
|
|
730
|
+
this.logger?.log("INFO", "Retrieving Anthropic batch results", {
|
|
731
|
+
batchId: handle.id
|
|
732
|
+
});
|
|
733
|
+
const results = [];
|
|
734
|
+
let index = 0;
|
|
735
|
+
const resultsIterator = await this.client.messages.batches.results(
|
|
736
|
+
handle.id
|
|
737
|
+
);
|
|
738
|
+
for await (const entry of resultsIterator) {
|
|
739
|
+
if (entry.result.type === "succeeded") {
|
|
740
|
+
const message = entry.result.message;
|
|
741
|
+
const textContent = message.content.find(
|
|
742
|
+
(c) => c.type === "text"
|
|
743
|
+
);
|
|
744
|
+
results.push({
|
|
745
|
+
index,
|
|
746
|
+
customId: entry.custom_id,
|
|
747
|
+
text: textContent?.text || "",
|
|
748
|
+
inputTokens: message.usage?.input_tokens || 0,
|
|
749
|
+
outputTokens: message.usage?.output_tokens || 0
|
|
750
|
+
});
|
|
751
|
+
} else {
|
|
752
|
+
let errorMsg;
|
|
753
|
+
switch (entry.result.type) {
|
|
754
|
+
case "errored":
|
|
755
|
+
errorMsg = entry.result.error?.message || `Error type: ${entry.result.error?.type}` || "Request errored";
|
|
756
|
+
break;
|
|
757
|
+
case "canceled":
|
|
758
|
+
errorMsg = "Request was canceled";
|
|
759
|
+
break;
|
|
760
|
+
case "expired":
|
|
761
|
+
errorMsg = "Request expired";
|
|
762
|
+
break;
|
|
763
|
+
default:
|
|
764
|
+
errorMsg = `Unknown result type: ${entry.result.type}`;
|
|
765
|
+
}
|
|
766
|
+
results.push({
|
|
767
|
+
index,
|
|
768
|
+
customId: entry.custom_id,
|
|
769
|
+
text: "",
|
|
770
|
+
inputTokens: 0,
|
|
771
|
+
outputTokens: 0,
|
|
772
|
+
error: errorMsg
|
|
773
|
+
});
|
|
774
|
+
}
|
|
775
|
+
index++;
|
|
776
|
+
}
|
|
777
|
+
this.logger?.log("INFO", "Anthropic batch results retrieved", {
|
|
778
|
+
batchId: handle.id,
|
|
779
|
+
resultCount: results.length,
|
|
780
|
+
successCount: results.filter((r) => !r.error).length,
|
|
781
|
+
errorCount: results.filter((r) => r.error).length
|
|
782
|
+
});
|
|
783
|
+
return results;
|
|
784
|
+
}
|
|
785
|
+
async cancel(handle) {
|
|
786
|
+
await this.client.messages.batches.cancel(handle.id);
|
|
787
|
+
this.logger?.log("INFO", "Anthropic batch cancelled", {
|
|
788
|
+
batchId: handle.id
|
|
789
|
+
});
|
|
790
|
+
}
|
|
791
|
+
mapStatus(status) {
|
|
792
|
+
switch (status) {
|
|
793
|
+
case "ended":
|
|
794
|
+
return "completed";
|
|
795
|
+
case "canceling":
|
|
796
|
+
case "canceled":
|
|
797
|
+
return "cancelled";
|
|
798
|
+
case "in_progress":
|
|
799
|
+
return "processing";
|
|
800
|
+
default:
|
|
801
|
+
return "pending";
|
|
802
|
+
}
|
|
803
|
+
}
|
|
804
|
+
};
|
|
805
|
+
var GoogleBatchProvider = class {
|
|
806
|
+
name = "google";
|
|
807
|
+
supportsBatching = true;
|
|
808
|
+
ai;
|
|
809
|
+
logger;
|
|
810
|
+
constructor(config = {}, logger2) {
|
|
811
|
+
const apiKey = config.apiKey || process.env.GOOGLE_GENERATIVE_AI_API_KEY;
|
|
812
|
+
if (!apiKey) {
|
|
813
|
+
throw new Error(
|
|
814
|
+
"Google API key is required. Set GOOGLE_GENERATIVE_AI_API_KEY or pass apiKey in config."
|
|
815
|
+
);
|
|
816
|
+
}
|
|
817
|
+
this.ai = new GoogleGenAI({ apiKey });
|
|
818
|
+
this.logger = logger2;
|
|
819
|
+
}
|
|
820
|
+
async submit(requests, options) {
|
|
821
|
+
if (requests.length === 0) {
|
|
822
|
+
throw new Error("Cannot submit empty batch");
|
|
823
|
+
}
|
|
824
|
+
const modelKey = requests[0]?.model;
|
|
825
|
+
const model = resolveModelForProvider(modelKey, "google");
|
|
826
|
+
const customIds = requests.map(
|
|
827
|
+
(req, idx) => req.id || `request-${idx}`
|
|
828
|
+
);
|
|
829
|
+
this.logger?.log("INFO", "Submitting Google batch", {
|
|
830
|
+
requestCount: requests.length,
|
|
831
|
+
modelKey: modelKey || "default",
|
|
832
|
+
model
|
|
833
|
+
});
|
|
834
|
+
const inlinedRequests = requests.map((req) => {
|
|
835
|
+
const parts = [{ text: req.prompt }];
|
|
836
|
+
if (req.schema) {
|
|
837
|
+
parts.push({
|
|
838
|
+
text: `Please respond with a JSON object matching the expected schema structure.`
|
|
839
|
+
});
|
|
840
|
+
}
|
|
841
|
+
if (req.maxTokens) {
|
|
842
|
+
parts.push({
|
|
843
|
+
text: `Limit your response to a maximum of ${req.maxTokens} tokens.`
|
|
844
|
+
});
|
|
845
|
+
}
|
|
846
|
+
if (req.temperature !== void 0) {
|
|
847
|
+
parts.push({
|
|
848
|
+
text: `Use a temperature setting of ${req.temperature} for this response.`
|
|
849
|
+
});
|
|
850
|
+
}
|
|
851
|
+
const customId = req.id || `request-${requests.indexOf(req)}`;
|
|
852
|
+
const response = {
|
|
853
|
+
contents: [
|
|
854
|
+
{
|
|
855
|
+
role: "user",
|
|
856
|
+
parts
|
|
857
|
+
}
|
|
858
|
+
],
|
|
859
|
+
// Store customId in metadata - Google SDK supports this
|
|
860
|
+
metadata: { customId }
|
|
861
|
+
};
|
|
862
|
+
if (req.tools && Object.keys(req.tools).length > 0) {
|
|
863
|
+
const config = {
|
|
864
|
+
tools: [
|
|
865
|
+
{
|
|
866
|
+
functionDeclarations: Object.entries(req.tools).map(
|
|
867
|
+
([name, tool]) => ({
|
|
868
|
+
name,
|
|
869
|
+
description: tool.description,
|
|
870
|
+
// AI SDK uses inputSchema, convert to JSON Schema for Google
|
|
871
|
+
parameters: tool.inputSchema ? jsonSchema(tool.inputSchema) : void 0
|
|
872
|
+
})
|
|
873
|
+
)
|
|
874
|
+
}
|
|
875
|
+
]
|
|
876
|
+
};
|
|
877
|
+
if (req.toolChoice) {
|
|
878
|
+
if (req.toolChoice === "required") {
|
|
879
|
+
config.toolConfig = {
|
|
880
|
+
functionCallingConfig: { mode: FunctionCallingConfigMode.ANY }
|
|
881
|
+
};
|
|
882
|
+
} else if (req.toolChoice === "none") {
|
|
883
|
+
config.toolConfig = {
|
|
884
|
+
functionCallingConfig: { mode: FunctionCallingConfigMode.NONE }
|
|
885
|
+
};
|
|
886
|
+
} else if (typeof req.toolChoice === "object" && req.toolChoice.type === "tool") {
|
|
887
|
+
config.toolConfig = {
|
|
888
|
+
functionCallingConfig: {
|
|
889
|
+
mode: FunctionCallingConfigMode.ANY,
|
|
890
|
+
allowedFunctionNames: [req.toolChoice.toolName]
|
|
891
|
+
}
|
|
892
|
+
};
|
|
893
|
+
}
|
|
894
|
+
}
|
|
895
|
+
response.config = config;
|
|
896
|
+
}
|
|
897
|
+
return response;
|
|
898
|
+
});
|
|
899
|
+
const batchJob = await this.ai.batches.create({
|
|
900
|
+
model,
|
|
901
|
+
src: { inlinedRequests },
|
|
902
|
+
config: {
|
|
903
|
+
displayName: options?.displayName || `batch-${Date.now()}-${requests.length}`
|
|
904
|
+
}
|
|
905
|
+
});
|
|
906
|
+
if (!batchJob.name) {
|
|
907
|
+
throw new Error("Batch job created but no name returned");
|
|
908
|
+
}
|
|
909
|
+
this.logger?.log("INFO", "Google batch submitted", {
|
|
910
|
+
batchName: batchJob.name,
|
|
911
|
+
requestCount: requests.length
|
|
912
|
+
});
|
|
913
|
+
return {
|
|
914
|
+
id: batchJob.name,
|
|
915
|
+
provider: this.name,
|
|
916
|
+
requestCount: requests.length,
|
|
917
|
+
createdAt: /* @__PURE__ */ new Date(),
|
|
918
|
+
metadata: {
|
|
919
|
+
model,
|
|
920
|
+
displayName: options?.displayName,
|
|
921
|
+
customIds
|
|
922
|
+
// Store for getResults to use
|
|
923
|
+
}
|
|
924
|
+
};
|
|
925
|
+
}
|
|
926
|
+
async checkStatus(handle) {
|
|
927
|
+
const batch = await this.ai.batches.get({ name: handle.id });
|
|
928
|
+
const status = {
|
|
929
|
+
state: this.mapState(batch.state),
|
|
930
|
+
processedCount: Number(batch.completionStats?.successfulCount) || 0,
|
|
931
|
+
totalCount: handle.requestCount,
|
|
932
|
+
succeededCount: batch.completionStats?.successfulCount ? Number(batch.completionStats.successfulCount) : void 0,
|
|
933
|
+
failedCount: batch.completionStats?.failedCount ? Number(batch.completionStats.failedCount) : void 0,
|
|
934
|
+
error: batch.error ? String(batch.error) : void 0
|
|
935
|
+
};
|
|
936
|
+
if (batch.state === JobState.JOB_STATE_SUCCEEDED || batch.state === JobState.JOB_STATE_FAILED) {
|
|
937
|
+
const inlinedResponses = batch.dest?.inlinedResponses;
|
|
938
|
+
if (inlinedResponses && Array.isArray(inlinedResponses)) {
|
|
939
|
+
let totalInputTokens = 0;
|
|
940
|
+
let totalOutputTokens = 0;
|
|
941
|
+
for (const inlinedResponse of inlinedResponses) {
|
|
942
|
+
const usageMetadata = inlinedResponse.response?.usageMetadata;
|
|
943
|
+
if (usageMetadata) {
|
|
944
|
+
totalInputTokens += usageMetadata.promptTokenCount ?? 0;
|
|
945
|
+
totalOutputTokens += usageMetadata.candidatesTokenCount ?? 0;
|
|
946
|
+
}
|
|
947
|
+
}
|
|
948
|
+
status.totalInputTokens = totalInputTokens;
|
|
949
|
+
status.totalOutputTokens = totalOutputTokens;
|
|
950
|
+
this.logger?.log("INFO", "Google batch token usage", {
|
|
951
|
+
batchId: handle.id,
|
|
952
|
+
totalInputTokens,
|
|
953
|
+
totalOutputTokens,
|
|
954
|
+
responseCount: inlinedResponses.length
|
|
955
|
+
});
|
|
956
|
+
}
|
|
957
|
+
}
|
|
958
|
+
this.logger?.log("DEBUG", "sdk response", { batch });
|
|
959
|
+
this.logger?.log("DEBUG", "Google batch status", {
|
|
960
|
+
batchId: handle.id,
|
|
961
|
+
state: status.state,
|
|
962
|
+
processed: status.processedCount,
|
|
963
|
+
total: status.totalCount,
|
|
964
|
+
startTime: batch.startTime,
|
|
965
|
+
endTime: batch.endTime,
|
|
966
|
+
totalInputTokens: status.totalInputTokens,
|
|
967
|
+
totalOutputTokens: status.totalOutputTokens
|
|
968
|
+
});
|
|
969
|
+
return status;
|
|
970
|
+
}
|
|
971
|
+
async getResults(handle, customIds) {
|
|
972
|
+
const requestIds = customIds || handle.metadata?.customIds;
|
|
973
|
+
this.logger?.log("DEBUG", "Google batch getResults - customIds lookup", {
|
|
974
|
+
batchId: handle.id,
|
|
975
|
+
hasCustomIdsParam: !!customIds,
|
|
976
|
+
hasHandleMetadata: !!handle.metadata,
|
|
977
|
+
hasMetadataCustomIds: !!handle.metadata?.customIds,
|
|
978
|
+
requestIdsFound: requestIds?.length ?? 0
|
|
979
|
+
});
|
|
980
|
+
const batch = await this.ai.batches.get({ name: handle.id });
|
|
981
|
+
if (batch.state !== JobState.JOB_STATE_SUCCEEDED && batch.state !== JobState.JOB_STATE_FAILED) {
|
|
982
|
+
throw new Error(`Batch not complete: state=${batch.state}`);
|
|
983
|
+
}
|
|
984
|
+
if (batch.state === JobState.JOB_STATE_FAILED) {
|
|
985
|
+
const errorMsg = batch.error || "Unknown error";
|
|
986
|
+
throw new Error(`Batch failed: ${errorMsg}`);
|
|
987
|
+
}
|
|
988
|
+
const maybeInlinedResponses = batch.dest?.inlinedResponses;
|
|
989
|
+
if (!maybeInlinedResponses) {
|
|
990
|
+
throw new Error(
|
|
991
|
+
"Batch response format unexpected - could not find inlinedResponses array"
|
|
992
|
+
);
|
|
993
|
+
}
|
|
994
|
+
if (!maybeInlinedResponses || !Array.isArray(maybeInlinedResponses)) {
|
|
995
|
+
this.logger?.log("ERROR", "Unexpected batch response format", {
|
|
996
|
+
batchId: handle.id,
|
|
997
|
+
hasResponse: !!maybeInlinedResponses,
|
|
998
|
+
destKeys: batch.dest ? Object.keys(batch.dest) : []
|
|
999
|
+
});
|
|
1000
|
+
throw new Error(
|
|
1001
|
+
"Batch response format unexpected - could not find inlinedResponses array"
|
|
1002
|
+
);
|
|
1003
|
+
}
|
|
1004
|
+
this.logger?.log("INFO", "Processing Google batch results", {
|
|
1005
|
+
batchId: handle.id,
|
|
1006
|
+
responseCount: maybeInlinedResponses.length,
|
|
1007
|
+
firstItem: JSON.stringify(maybeInlinedResponses[0])
|
|
1008
|
+
// Log entire first item structure to be sure
|
|
1009
|
+
});
|
|
1010
|
+
return maybeInlinedResponses.map((inlinedResponse, index) => {
|
|
1011
|
+
try {
|
|
1012
|
+
const responseMetadata = inlinedResponse.metadata;
|
|
1013
|
+
const customId = responseMetadata?.customId || requestIds?.[index] || `request-${index}`;
|
|
1014
|
+
if (!requestIds?.[index] && !responseMetadata?.customId) {
|
|
1015
|
+
this.logger?.log(
|
|
1016
|
+
"WARN",
|
|
1017
|
+
`No customId found for index ${index}, using fallback`,
|
|
1018
|
+
{ index }
|
|
1019
|
+
);
|
|
1020
|
+
}
|
|
1021
|
+
if (inlinedResponse.error) {
|
|
1022
|
+
const result2 = {
|
|
1023
|
+
index,
|
|
1024
|
+
customId,
|
|
1025
|
+
text: "",
|
|
1026
|
+
inputTokens: 0,
|
|
1027
|
+
outputTokens: 0,
|
|
1028
|
+
error: inlinedResponse.error.message || "Unknown error"
|
|
1029
|
+
};
|
|
1030
|
+
this.logger?.log("DEBUG", `Response ${index} has error`, {
|
|
1031
|
+
customId,
|
|
1032
|
+
error: result2.error
|
|
1033
|
+
});
|
|
1034
|
+
return result2;
|
|
1035
|
+
}
|
|
1036
|
+
const response = inlinedResponse.response;
|
|
1037
|
+
if (!response) {
|
|
1038
|
+
const result2 = {
|
|
1039
|
+
index,
|
|
1040
|
+
text: "",
|
|
1041
|
+
inputTokens: 0,
|
|
1042
|
+
outputTokens: 0,
|
|
1043
|
+
error: "InlinedResponse missing response field"
|
|
1044
|
+
};
|
|
1045
|
+
this.logger?.log(
|
|
1046
|
+
"DEBUG",
|
|
1047
|
+
`Response ${index} missing response field`,
|
|
1048
|
+
{ customId }
|
|
1049
|
+
);
|
|
1050
|
+
return result2;
|
|
1051
|
+
}
|
|
1052
|
+
const text = response.text || response.candidates?.[0]?.content?.parts?.[0]?.text || "";
|
|
1053
|
+
const usageMetadata = response.usageMetadata;
|
|
1054
|
+
const result = {
|
|
1055
|
+
index,
|
|
1056
|
+
customId,
|
|
1057
|
+
// Use the already-resolved customId from above
|
|
1058
|
+
text,
|
|
1059
|
+
inputTokens: usageMetadata?.promptTokenCount ?? 0,
|
|
1060
|
+
outputTokens: usageMetadata?.candidatesTokenCount ?? 0
|
|
1061
|
+
};
|
|
1062
|
+
this.logger?.log("DEBUG", `Response ${index} parsed successfully`, {
|
|
1063
|
+
customId,
|
|
1064
|
+
textLength: text.length,
|
|
1065
|
+
inputTokens: result.inputTokens,
|
|
1066
|
+
outputTokens: result.outputTokens
|
|
1067
|
+
});
|
|
1068
|
+
return result;
|
|
1069
|
+
} catch (error) {
|
|
1070
|
+
const result = {
|
|
1071
|
+
index,
|
|
1072
|
+
text: "",
|
|
1073
|
+
inputTokens: 0,
|
|
1074
|
+
outputTokens: 0,
|
|
1075
|
+
error: error instanceof Error ? error.message : String(error)
|
|
1076
|
+
};
|
|
1077
|
+
this.logger?.log("ERROR", `Response ${index} threw exception`, {
|
|
1078
|
+
error: result.error
|
|
1079
|
+
});
|
|
1080
|
+
return result;
|
|
1081
|
+
}
|
|
1082
|
+
});
|
|
1083
|
+
}
|
|
1084
|
+
async cancel(handle) {
|
|
1085
|
+
try {
|
|
1086
|
+
await this.ai.batches.delete({ name: handle.id });
|
|
1087
|
+
this.logger?.log("INFO", "Google batch cancelled", {
|
|
1088
|
+
batchId: handle.id
|
|
1089
|
+
});
|
|
1090
|
+
} catch (error) {
|
|
1091
|
+
this.logger?.log("WARN", "Failed to cancel Google batch", {
|
|
1092
|
+
batchId: handle.id,
|
|
1093
|
+
error: error instanceof Error ? error.message : String(error)
|
|
1094
|
+
});
|
|
1095
|
+
throw error;
|
|
1096
|
+
}
|
|
1097
|
+
}
|
|
1098
|
+
mapState(state) {
|
|
1099
|
+
switch (state) {
|
|
1100
|
+
case JobState.JOB_STATE_SUCCEEDED:
|
|
1101
|
+
return "completed";
|
|
1102
|
+
case JobState.JOB_STATE_FAILED:
|
|
1103
|
+
return "failed";
|
|
1104
|
+
case JobState.JOB_STATE_CANCELLED:
|
|
1105
|
+
return "cancelled";
|
|
1106
|
+
case JobState.JOB_STATE_PENDING:
|
|
1107
|
+
return "pending";
|
|
1108
|
+
case JobState.JOB_STATE_RUNNING:
|
|
1109
|
+
return "processing";
|
|
1110
|
+
default:
|
|
1111
|
+
return "processing";
|
|
1112
|
+
}
|
|
1113
|
+
}
|
|
1114
|
+
};
|
|
1115
|
+
var OpenAIBatchProvider = class {
|
|
1116
|
+
name = "openai";
|
|
1117
|
+
supportsBatching = true;
|
|
1118
|
+
client;
|
|
1119
|
+
logger;
|
|
1120
|
+
constructor(config = {}, logger2) {
|
|
1121
|
+
const apiKey = config.apiKey || process.env.OPENAI_API_KEY;
|
|
1122
|
+
if (!apiKey) {
|
|
1123
|
+
throw new Error(
|
|
1124
|
+
"OpenAI API key is required. Set OPENAI_API_KEY or pass apiKey in config."
|
|
1125
|
+
);
|
|
1126
|
+
}
|
|
1127
|
+
this.client = new OpenAI({ apiKey });
|
|
1128
|
+
this.logger = logger2;
|
|
1129
|
+
}
|
|
1130
|
+
async submit(requests, options) {
|
|
1131
|
+
if (requests.length === 0) {
|
|
1132
|
+
throw new Error("Cannot submit empty batch");
|
|
1133
|
+
}
|
|
1134
|
+
const modelKey = requests[0]?.model;
|
|
1135
|
+
const model = resolveModelForProvider(modelKey, "openai");
|
|
1136
|
+
this.logger?.log("INFO", "Submitting OpenAI batch", {
|
|
1137
|
+
requestCount: requests.length,
|
|
1138
|
+
modelKey: modelKey || "default",
|
|
1139
|
+
model
|
|
1140
|
+
});
|
|
1141
|
+
const jsonlContent = requests.map((req, i) => {
|
|
1142
|
+
const reqModel = req.model ? resolveModelForProvider(req.model, "openai") : model;
|
|
1143
|
+
const openaiTools = req.tools && Object.keys(req.tools).length > 0 ? Object.entries(req.tools).map(([name, tool]) => ({
|
|
1144
|
+
type: "function",
|
|
1145
|
+
function: {
|
|
1146
|
+
name,
|
|
1147
|
+
description: tool.description,
|
|
1148
|
+
parameters: tool.inputSchema ? jsonSchema(tool.inputSchema) : void 0
|
|
1149
|
+
}
|
|
1150
|
+
})) : void 0;
|
|
1151
|
+
let toolChoice;
|
|
1152
|
+
if (req.toolChoice) {
|
|
1153
|
+
if (req.toolChoice === "auto") {
|
|
1154
|
+
toolChoice = "auto";
|
|
1155
|
+
} else if (req.toolChoice === "required") {
|
|
1156
|
+
toolChoice = "required";
|
|
1157
|
+
} else if (req.toolChoice === "none") {
|
|
1158
|
+
toolChoice = "none";
|
|
1159
|
+
} else if (typeof req.toolChoice === "object" && req.toolChoice.type === "tool") {
|
|
1160
|
+
toolChoice = {
|
|
1161
|
+
type: "function",
|
|
1162
|
+
function: { name: req.toolChoice.toolName }
|
|
1163
|
+
};
|
|
1164
|
+
}
|
|
1165
|
+
}
|
|
1166
|
+
return JSON.stringify({
|
|
1167
|
+
custom_id: req.customId || `request-${i}`,
|
|
1168
|
+
method: "POST",
|
|
1169
|
+
url: "/v1/chat/completions",
|
|
1170
|
+
body: {
|
|
1171
|
+
model: reqModel,
|
|
1172
|
+
messages: [
|
|
1173
|
+
...req.system ? [{ role: "system", content: req.system }] : [],
|
|
1174
|
+
{ role: "user", content: req.prompt }
|
|
1175
|
+
],
|
|
1176
|
+
max_tokens: req.maxTokens || 1024,
|
|
1177
|
+
...req.temperature !== void 0 && {
|
|
1178
|
+
temperature: req.temperature
|
|
1179
|
+
},
|
|
1180
|
+
...openaiTools && { tools: openaiTools },
|
|
1181
|
+
...toolChoice && { tool_choice: toolChoice }
|
|
1182
|
+
}
|
|
1183
|
+
});
|
|
1184
|
+
}).join("\n");
|
|
1185
|
+
const file = await this.client.files.create({
|
|
1186
|
+
file: new File([jsonlContent], "batch-requests.jsonl", {
|
|
1187
|
+
type: "application/jsonl"
|
|
1188
|
+
}),
|
|
1189
|
+
purpose: "batch"
|
|
1190
|
+
});
|
|
1191
|
+
this.logger?.log("DEBUG", "OpenAI batch file uploaded", {
|
|
1192
|
+
fileId: file.id,
|
|
1193
|
+
filename: file.filename
|
|
1194
|
+
});
|
|
1195
|
+
const batch = await this.client.batches.create({
|
|
1196
|
+
input_file_id: file.id,
|
|
1197
|
+
endpoint: "/v1/chat/completions",
|
|
1198
|
+
completion_window: "24h",
|
|
1199
|
+
// OpenAI only supports 24h
|
|
1200
|
+
metadata: options?.metadata
|
|
1201
|
+
});
|
|
1202
|
+
this.logger?.log("INFO", "OpenAI batch submitted", {
|
|
1203
|
+
batchId: batch.id,
|
|
1204
|
+
requestCount: requests.length,
|
|
1205
|
+
status: batch.status
|
|
1206
|
+
});
|
|
1207
|
+
return {
|
|
1208
|
+
id: batch.id,
|
|
1209
|
+
provider: this.name,
|
|
1210
|
+
requestCount: requests.length,
|
|
1211
|
+
createdAt: new Date(batch.created_at * 1e3),
|
|
1212
|
+
metadata: {
|
|
1213
|
+
model,
|
|
1214
|
+
inputFileId: file.id,
|
|
1215
|
+
outputFileId: batch.output_file_id,
|
|
1216
|
+
errorFileId: batch.error_file_id,
|
|
1217
|
+
...options?.metadata
|
|
1218
|
+
}
|
|
1219
|
+
};
|
|
1220
|
+
}
|
|
1221
|
+
async checkStatus(handle) {
|
|
1222
|
+
const batch = await this.client.batches.retrieve(handle.id);
|
|
1223
|
+
const requestCounts = batch.request_counts;
|
|
1224
|
+
const completedCount = requestCounts?.completed || 0;
|
|
1225
|
+
const failedCount = requestCounts?.failed || 0;
|
|
1226
|
+
const totalCount = requestCounts?.total || handle.requestCount;
|
|
1227
|
+
let state = this.mapStatus(batch.status);
|
|
1228
|
+
if (state === "completed" && !batch.output_file_id) {
|
|
1229
|
+
this.logger?.log(
|
|
1230
|
+
"WARN",
|
|
1231
|
+
"Batch shows completed but output file not ready yet",
|
|
1232
|
+
{
|
|
1233
|
+
batchId: handle.id,
|
|
1234
|
+
status: batch.status
|
|
1235
|
+
}
|
|
1236
|
+
);
|
|
1237
|
+
state = "processing";
|
|
1238
|
+
}
|
|
1239
|
+
const status = {
|
|
1240
|
+
state,
|
|
1241
|
+
processedCount: completedCount + failedCount,
|
|
1242
|
+
totalCount,
|
|
1243
|
+
succeededCount: completedCount,
|
|
1244
|
+
failedCount,
|
|
1245
|
+
error: batch.errors?.data?.[0]?.message
|
|
1246
|
+
};
|
|
1247
|
+
this.logger?.log("INFO", "OpenAI batch status check", {
|
|
1248
|
+
batchId: handle.id,
|
|
1249
|
+
rawStatus: batch.status,
|
|
1250
|
+
mappedState: status.state,
|
|
1251
|
+
processed: status.processedCount,
|
|
1252
|
+
total: status.totalCount,
|
|
1253
|
+
outputFileId: batch.output_file_id
|
|
1254
|
+
});
|
|
1255
|
+
return status;
|
|
1256
|
+
}
|
|
1257
|
+
async getResults(handle) {
|
|
1258
|
+
const batch = await this.client.batches.retrieve(handle.id);
|
|
1259
|
+
this.logger?.log("INFO", "OpenAI batch retrieve for results", {
|
|
1260
|
+
batchId: handle.id,
|
|
1261
|
+
status: batch.status,
|
|
1262
|
+
outputFileId: batch.output_file_id,
|
|
1263
|
+
errorFileId: batch.error_file_id
|
|
1264
|
+
});
|
|
1265
|
+
if (batch.status !== "completed" && batch.status !== "failed") {
|
|
1266
|
+
throw new Error(`Batch not complete: status=${batch.status}`);
|
|
1267
|
+
}
|
|
1268
|
+
if (batch.status === "failed") {
|
|
1269
|
+
const errorMessage = batch.errors?.data?.[0]?.message || "Unknown batch error";
|
|
1270
|
+
throw new Error(`Batch failed: ${errorMessage}`);
|
|
1271
|
+
}
|
|
1272
|
+
if (!batch.output_file_id) {
|
|
1273
|
+
this.logger?.log("ERROR", "Batch completed but no output file", {
|
|
1274
|
+
batchId: handle.id,
|
|
1275
|
+
status: batch.status,
|
|
1276
|
+
requestCounts: batch.request_counts,
|
|
1277
|
+
errors: batch.errors
|
|
1278
|
+
});
|
|
1279
|
+
throw new Error(
|
|
1280
|
+
`Batch output file not available. Status: ${batch.status}, Request counts: ${JSON.stringify(batch.request_counts)}`
|
|
1281
|
+
);
|
|
1282
|
+
}
|
|
1283
|
+
this.logger?.log("INFO", "Retrieving OpenAI batch results", {
|
|
1284
|
+
batchId: handle.id,
|
|
1285
|
+
outputFileId: batch.output_file_id
|
|
1286
|
+
});
|
|
1287
|
+
const fileContent = await this.client.files.content(batch.output_file_id);
|
|
1288
|
+
const text = await fileContent.text();
|
|
1289
|
+
const lines = text.trim().split("\n").filter(Boolean);
|
|
1290
|
+
const results = lines.map(
|
|
1291
|
+
(line, index) => {
|
|
1292
|
+
try {
|
|
1293
|
+
const result = JSON.parse(line);
|
|
1294
|
+
const response = result.response?.body;
|
|
1295
|
+
const choice = response?.choices?.[0];
|
|
1296
|
+
if (result.error) {
|
|
1297
|
+
return {
|
|
1298
|
+
index,
|
|
1299
|
+
customId: result.custom_id,
|
|
1300
|
+
text: "",
|
|
1301
|
+
inputTokens: 0,
|
|
1302
|
+
outputTokens: 0,
|
|
1303
|
+
error: result.error.message || "Unknown error"
|
|
1304
|
+
};
|
|
1305
|
+
}
|
|
1306
|
+
return {
|
|
1307
|
+
index,
|
|
1308
|
+
customId: result.custom_id,
|
|
1309
|
+
text: choice?.message?.content || "",
|
|
1310
|
+
inputTokens: response?.usage?.prompt_tokens || 0,
|
|
1311
|
+
outputTokens: response?.usage?.completion_tokens || 0
|
|
1312
|
+
};
|
|
1313
|
+
} catch (error) {
|
|
1314
|
+
return {
|
|
1315
|
+
index,
|
|
1316
|
+
customId: void 0,
|
|
1317
|
+
text: "",
|
|
1318
|
+
inputTokens: 0,
|
|
1319
|
+
outputTokens: 0,
|
|
1320
|
+
error: error instanceof Error ? error.message : String(error)
|
|
1321
|
+
};
|
|
1322
|
+
}
|
|
1323
|
+
}
|
|
1324
|
+
);
|
|
1325
|
+
this.logger?.log("INFO", "OpenAI batch results retrieved", {
|
|
1326
|
+
batchId: handle.id,
|
|
1327
|
+
resultCount: results.length,
|
|
1328
|
+
successCount: results.filter((r) => !r.error).length,
|
|
1329
|
+
errorCount: results.filter((r) => r.error).length
|
|
1330
|
+
});
|
|
1331
|
+
return results;
|
|
1332
|
+
}
|
|
1333
|
+
async cancel(handle) {
|
|
1334
|
+
await this.client.batches.cancel(handle.id);
|
|
1335
|
+
this.logger?.log("INFO", "OpenAI batch cancelled", { batchId: handle.id });
|
|
1336
|
+
}
|
|
1337
|
+
mapStatus(status) {
|
|
1338
|
+
switch (status) {
|
|
1339
|
+
case "completed":
|
|
1340
|
+
return "completed";
|
|
1341
|
+
case "failed":
|
|
1342
|
+
case "expired":
|
|
1343
|
+
return "failed";
|
|
1344
|
+
case "cancelling":
|
|
1345
|
+
case "cancelled":
|
|
1346
|
+
return "cancelled";
|
|
1347
|
+
case "validating":
|
|
1348
|
+
case "in_progress":
|
|
1349
|
+
case "finalizing":
|
|
1350
|
+
return "processing";
|
|
1351
|
+
default:
|
|
1352
|
+
return "pending";
|
|
1353
|
+
}
|
|
1354
|
+
}
|
|
1355
|
+
};
|
|
1356
|
+
var logger = createLogger("AIHelper");
|
|
1357
|
+
var DEFAULT_EMBEDDING_DIMENSIONS = 768;
|
|
1358
|
+
function getModelProvider(modelConfig) {
|
|
1359
|
+
if (modelConfig.provider === "openrouter") {
|
|
1360
|
+
return openrouter(modelConfig.id, {
|
|
1361
|
+
extraBody: {
|
|
1362
|
+
provider: {
|
|
1363
|
+
sort: "throughput",
|
|
1364
|
+
require_parameters: true,
|
|
1365
|
+
max_price: {
|
|
1366
|
+
prompt: modelConfig.inputCostPerMillion,
|
|
1367
|
+
completion: modelConfig.outputCostPerMillion
|
|
1368
|
+
}
|
|
1369
|
+
}
|
|
1370
|
+
}
|
|
1371
|
+
});
|
|
1372
|
+
}
|
|
1373
|
+
return google(modelConfig.id);
|
|
1374
|
+
}
|
|
1375
|
+
function calculateCostWithDiscount(modelKey, inputTokens, outputTokens, isBatch = false) {
|
|
1376
|
+
const model = getModel(modelKey);
|
|
1377
|
+
const baseCost = calculateCost(modelKey, inputTokens, outputTokens);
|
|
1378
|
+
if (isBatch && model.batchDiscountPercent) {
|
|
1379
|
+
return baseCost.totalCost * (1 - model.batchDiscountPercent / 100);
|
|
1380
|
+
}
|
|
1381
|
+
return baseCost.totalCost;
|
|
1382
|
+
}
|
|
1383
|
+
var AIHelperImpl = class _AIHelperImpl {
|
|
1384
|
+
topic;
|
|
1385
|
+
aiCallLogger;
|
|
1386
|
+
logContext;
|
|
1387
|
+
batchLogFn;
|
|
1388
|
+
constructor(topic, aiCallLogger, logContext) {
|
|
1389
|
+
if (!aiCallLogger) {
|
|
1390
|
+
throw new Error(
|
|
1391
|
+
"AIHelperImpl requires a logger. Create one using createPrismaAICallLogger(prisma)."
|
|
1392
|
+
);
|
|
1393
|
+
}
|
|
1394
|
+
this.topic = topic;
|
|
1395
|
+
this.aiCallLogger = aiCallLogger;
|
|
1396
|
+
this.logContext = logContext;
|
|
1397
|
+
if (logContext) {
|
|
1398
|
+
this.batchLogFn = (level, message, meta) => {
|
|
1399
|
+
logContext.createLog({
|
|
1400
|
+
workflowRunId: logContext.workflowRunId,
|
|
1401
|
+
workflowStageId: logContext.stageRecordId,
|
|
1402
|
+
level,
|
|
1403
|
+
message,
|
|
1404
|
+
metadata: meta
|
|
1405
|
+
}).catch((err) => logger.error("Failed to persist log:", err));
|
|
1406
|
+
logger.debug(`[${level}] ${message}`, meta ? JSON.stringify(meta) : "");
|
|
1407
|
+
};
|
|
1408
|
+
}
|
|
1409
|
+
}
|
|
1410
|
+
async generateText(modelKey, prompt, options = {}) {
|
|
1411
|
+
const modelConfig = getModel(modelKey);
|
|
1412
|
+
const model = getModelProvider(modelConfig);
|
|
1413
|
+
const startTime = Date.now();
|
|
1414
|
+
const isMultimodal = Array.isArray(prompt);
|
|
1415
|
+
const hasTools = options.tools !== void 0;
|
|
1416
|
+
const hasOutputSchema = options.experimental_output !== void 0;
|
|
1417
|
+
const promptForLog = isMultimodal ? prompt.filter((p) => p.type === "text").map((p) => p.text).join("\n") || "[multimodal content]" : prompt;
|
|
1418
|
+
if (hasTools || hasOutputSchema) {
|
|
1419
|
+
logger.debug(
|
|
1420
|
+
`generateText config: hasTools=${hasTools}, hasOutputSchema=${hasOutputSchema}, toolNames=${hasTools ? Object.keys(options.tools || {}).join(", ") : "none"}`
|
|
1421
|
+
);
|
|
1422
|
+
}
|
|
1423
|
+
const wrappedOnStepFinish = options.onStepFinish ? async (stepResult) => {
|
|
1424
|
+
if (stepResult.toolResults && Array.isArray(stepResult.toolResults)) {
|
|
1425
|
+
for (const toolResult of stepResult.toolResults) {
|
|
1426
|
+
const result = toolResult;
|
|
1427
|
+
if (result.toolName) {
|
|
1428
|
+
const childTopic = `${this.topic}.tool.${result.toolName}`;
|
|
1429
|
+
this.aiCallLogger.logCall({
|
|
1430
|
+
topic: childTopic,
|
|
1431
|
+
callType: "text",
|
|
1432
|
+
modelKey,
|
|
1433
|
+
modelId: modelConfig.id,
|
|
1434
|
+
prompt: JSON.stringify(result.input ?? {}, null, 2),
|
|
1435
|
+
response: JSON.stringify(result.output ?? {}, null, 2),
|
|
1436
|
+
inputTokens: stepResult.usage.inputTokens || 0,
|
|
1437
|
+
outputTokens: stepResult.usage.outputTokens || 0,
|
|
1438
|
+
cost: calculateCostWithDiscount(
|
|
1439
|
+
modelKey,
|
|
1440
|
+
stepResult.usage.inputTokens || 0,
|
|
1441
|
+
stepResult.usage.outputTokens || 0
|
|
1442
|
+
),
|
|
1443
|
+
metadata: {
|
|
1444
|
+
toolName: result.toolName,
|
|
1445
|
+
toolCallId: result.toolCallId,
|
|
1446
|
+
finishReason: stepResult.finishReason
|
|
1447
|
+
}
|
|
1448
|
+
});
|
|
1449
|
+
}
|
|
1450
|
+
}
|
|
1451
|
+
}
|
|
1452
|
+
await options.onStepFinish?.(stepResult);
|
|
1453
|
+
} : void 0;
|
|
1454
|
+
const baseOptions = {
|
|
1455
|
+
model,
|
|
1456
|
+
temperature: options.temperature ?? 0.7,
|
|
1457
|
+
maxOutputTokens: options.maxTokens,
|
|
1458
|
+
// Tool-related options (only included if tools are provided)
|
|
1459
|
+
...hasTools && {
|
|
1460
|
+
tools: options.tools,
|
|
1461
|
+
// Cast to any because TTools generic doesn't match NoInfer<ToolSet> at compile time
|
|
1462
|
+
toolChoice: options.toolChoice,
|
|
1463
|
+
stopWhen: options.stopWhen,
|
|
1464
|
+
onStepFinish: wrappedOnStepFinish
|
|
1465
|
+
},
|
|
1466
|
+
// Experimental structured output (for tools + schema)
|
|
1467
|
+
...hasOutputSchema && {
|
|
1468
|
+
experimental_output: options.experimental_output
|
|
1469
|
+
}
|
|
1470
|
+
};
|
|
1471
|
+
logger.debug(`generateText request`, {
|
|
1472
|
+
model: modelKey,
|
|
1473
|
+
modelId: modelConfig.id,
|
|
1474
|
+
prompt: promptForLog.substring(0, 500) + (promptForLog.length > 500 ? "..." : ""),
|
|
1475
|
+
temperature: options.temperature ?? 0.7,
|
|
1476
|
+
maxTokens: options.maxTokens,
|
|
1477
|
+
hasTools,
|
|
1478
|
+
hasOutputSchema,
|
|
1479
|
+
isMultimodal
|
|
1480
|
+
});
|
|
1481
|
+
try {
|
|
1482
|
+
const result = isMultimodal ? await generateText({
|
|
1483
|
+
...baseOptions,
|
|
1484
|
+
messages: [
|
|
1485
|
+
{
|
|
1486
|
+
role: "user",
|
|
1487
|
+
content: prompt.map(
|
|
1488
|
+
(part) => part.type === "text" ? { type: "text", text: part.text } : {
|
|
1489
|
+
type: "file",
|
|
1490
|
+
data: part.data,
|
|
1491
|
+
mediaType: part.mediaType,
|
|
1492
|
+
...part.filename && { filename: part.filename }
|
|
1493
|
+
}
|
|
1494
|
+
)
|
|
1495
|
+
}
|
|
1496
|
+
]
|
|
1497
|
+
}) : await generateText({
|
|
1498
|
+
...baseOptions,
|
|
1499
|
+
prompt
|
|
1500
|
+
});
|
|
1501
|
+
if (hasTools || hasOutputSchema) {
|
|
1502
|
+
const resultAny = result;
|
|
1503
|
+
logger.debug(
|
|
1504
|
+
`generateText result: stepsCount=${resultAny.steps?.length ?? 0}, hasOutput=${resultAny.output !== void 0}, finishReason=${result.finishReason}`
|
|
1505
|
+
);
|
|
1506
|
+
}
|
|
1507
|
+
const inputTokens = result.usage?.inputTokens ?? 0;
|
|
1508
|
+
const outputTokens = result.usage?.outputTokens ?? 0;
|
|
1509
|
+
const cost = calculateCostWithDiscount(
|
|
1510
|
+
modelKey,
|
|
1511
|
+
inputTokens,
|
|
1512
|
+
outputTokens
|
|
1513
|
+
);
|
|
1514
|
+
const durationMs = Date.now() - startTime;
|
|
1515
|
+
this.aiCallLogger.logCall({
|
|
1516
|
+
topic: this.topic,
|
|
1517
|
+
callType: "text",
|
|
1518
|
+
modelKey,
|
|
1519
|
+
modelId: modelConfig.id,
|
|
1520
|
+
prompt: promptForLog,
|
|
1521
|
+
response: result.text,
|
|
1522
|
+
inputTokens,
|
|
1523
|
+
outputTokens,
|
|
1524
|
+
cost,
|
|
1525
|
+
metadata: {
|
|
1526
|
+
temperature: options.temperature,
|
|
1527
|
+
maxTokens: options.maxTokens,
|
|
1528
|
+
finishReason: result.finishReason,
|
|
1529
|
+
durationMs,
|
|
1530
|
+
isMultimodal,
|
|
1531
|
+
...result.finishReason === "error" && { status: "error" },
|
|
1532
|
+
...isMultimodal && {
|
|
1533
|
+
mediaTypes: prompt.filter((p) => p.type === "file").map((p) => p.mediaType)
|
|
1534
|
+
}
|
|
1535
|
+
}
|
|
1536
|
+
});
|
|
1537
|
+
logger.debug(`generateText response`, {
|
|
1538
|
+
model: modelKey,
|
|
1539
|
+
response: result.text.substring(0, 500) + (result.text.length > 500 ? "..." : ""),
|
|
1540
|
+
inputTokens,
|
|
1541
|
+
outputTokens,
|
|
1542
|
+
cost: cost.toFixed(6),
|
|
1543
|
+
durationMs,
|
|
1544
|
+
finishReason: result.finishReason
|
|
1545
|
+
});
|
|
1546
|
+
return {
|
|
1547
|
+
text: result.text,
|
|
1548
|
+
inputTokens,
|
|
1549
|
+
outputTokens,
|
|
1550
|
+
cost,
|
|
1551
|
+
// Include structured output if experimental_output was used
|
|
1552
|
+
...hasOutputSchema && {
|
|
1553
|
+
output: result.output
|
|
1554
|
+
}
|
|
1555
|
+
};
|
|
1556
|
+
} catch (error) {
|
|
1557
|
+
const durationMs = Date.now() - startTime;
|
|
1558
|
+
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
1559
|
+
logger.error(`generateText error`, {
|
|
1560
|
+
model: modelKey,
|
|
1561
|
+
error: errorMessage,
|
|
1562
|
+
durationMs
|
|
1563
|
+
});
|
|
1564
|
+
this.aiCallLogger.logCall({
|
|
1565
|
+
topic: this.topic,
|
|
1566
|
+
callType: "text",
|
|
1567
|
+
modelKey,
|
|
1568
|
+
modelId: modelConfig.id,
|
|
1569
|
+
prompt: promptForLog,
|
|
1570
|
+
response: "",
|
|
1571
|
+
inputTokens: 0,
|
|
1572
|
+
outputTokens: 0,
|
|
1573
|
+
cost: 0,
|
|
1574
|
+
metadata: {
|
|
1575
|
+
temperature: options.temperature,
|
|
1576
|
+
maxTokens: options.maxTokens,
|
|
1577
|
+
finishReason: "error",
|
|
1578
|
+
durationMs,
|
|
1579
|
+
isMultimodal,
|
|
1580
|
+
status: "error",
|
|
1581
|
+
error: errorMessage
|
|
1582
|
+
}
|
|
1583
|
+
});
|
|
1584
|
+
throw error;
|
|
1585
|
+
}
|
|
1586
|
+
}
|
|
1587
|
+
async generateObject(modelKey, prompt, schema, options = {}) {
|
|
1588
|
+
const modelConfig = getModel(modelKey);
|
|
1589
|
+
const model = getModelProvider(modelConfig);
|
|
1590
|
+
const startTime = Date.now();
|
|
1591
|
+
const isMultimodal = Array.isArray(prompt);
|
|
1592
|
+
const hasTools = options.tools !== void 0;
|
|
1593
|
+
const promptForLog = isMultimodal ? prompt.filter((p) => p.type === "text").map((p) => p.text).join("\n") || "[multimodal content]" : prompt;
|
|
1594
|
+
const baseOptions = {
|
|
1595
|
+
model,
|
|
1596
|
+
output: Output.object({ schema }),
|
|
1597
|
+
temperature: options.temperature ?? 0,
|
|
1598
|
+
maxOutputTokens: options.maxTokens,
|
|
1599
|
+
// Tool-related options (only included if tools are provided)
|
|
1600
|
+
...hasTools && {
|
|
1601
|
+
tools: options.tools,
|
|
1602
|
+
stopWhen: options.stopWhen,
|
|
1603
|
+
onStepFinish: options.onStepFinish
|
|
1604
|
+
}
|
|
1605
|
+
};
|
|
1606
|
+
logger.debug(`generateObject request`, {
|
|
1607
|
+
model: modelKey,
|
|
1608
|
+
modelId: modelConfig.id,
|
|
1609
|
+
prompt: promptForLog.substring(0, 500) + (promptForLog.length > 500 ? "..." : ""),
|
|
1610
|
+
temperature: options.temperature ?? 0,
|
|
1611
|
+
maxTokens: options.maxTokens,
|
|
1612
|
+
hasTools,
|
|
1613
|
+
isMultimodal
|
|
1614
|
+
});
|
|
1615
|
+
try {
|
|
1616
|
+
const result = isMultimodal ? await generateText({
|
|
1617
|
+
...baseOptions,
|
|
1618
|
+
messages: [
|
|
1619
|
+
{
|
|
1620
|
+
role: "user",
|
|
1621
|
+
content: prompt.map(
|
|
1622
|
+
(part) => part.type === "text" ? { type: "text", text: part.text } : {
|
|
1623
|
+
type: "file",
|
|
1624
|
+
data: part.data,
|
|
1625
|
+
mediaType: part.mediaType,
|
|
1626
|
+
...part.filename && { filename: part.filename }
|
|
1627
|
+
}
|
|
1628
|
+
)
|
|
1629
|
+
}
|
|
1630
|
+
]
|
|
1631
|
+
}) : await generateText({
|
|
1632
|
+
...baseOptions,
|
|
1633
|
+
prompt
|
|
1634
|
+
});
|
|
1635
|
+
const inputTokens = result.usage?.inputTokens ?? 0;
|
|
1636
|
+
const outputTokens = result.usage?.outputTokens ?? 0;
|
|
1637
|
+
const cost = calculateCostWithDiscount(
|
|
1638
|
+
modelKey,
|
|
1639
|
+
inputTokens,
|
|
1640
|
+
outputTokens
|
|
1641
|
+
);
|
|
1642
|
+
const durationMs = Date.now() - startTime;
|
|
1643
|
+
this.aiCallLogger.logCall({
|
|
1644
|
+
topic: this.topic,
|
|
1645
|
+
callType: "object",
|
|
1646
|
+
modelKey,
|
|
1647
|
+
modelId: modelConfig.id,
|
|
1648
|
+
prompt: promptForLog,
|
|
1649
|
+
response: JSON.stringify(result.output, null, 2),
|
|
1650
|
+
inputTokens,
|
|
1651
|
+
outputTokens,
|
|
1652
|
+
cost,
|
|
1653
|
+
metadata: {
|
|
1654
|
+
temperature: options.temperature,
|
|
1655
|
+
maxTokens: options.maxTokens,
|
|
1656
|
+
finishReason: result.finishReason,
|
|
1657
|
+
durationMs,
|
|
1658
|
+
isMultimodal,
|
|
1659
|
+
...result.finishReason === "error" && { status: "error" },
|
|
1660
|
+
...isMultimodal && {
|
|
1661
|
+
mediaTypes: prompt.filter((p) => p.type === "file").map((p) => p.mediaType)
|
|
1662
|
+
}
|
|
1663
|
+
}
|
|
1664
|
+
});
|
|
1665
|
+
const responseStr = JSON.stringify(result.output);
|
|
1666
|
+
logger.debug(`generateObject response`, {
|
|
1667
|
+
model: modelKey,
|
|
1668
|
+
response: responseStr.substring(0, 500) + (responseStr.length > 500 ? "..." : ""),
|
|
1669
|
+
inputTokens,
|
|
1670
|
+
outputTokens,
|
|
1671
|
+
cost: cost.toFixed(6),
|
|
1672
|
+
durationMs,
|
|
1673
|
+
finishReason: result.finishReason
|
|
1674
|
+
});
|
|
1675
|
+
return {
|
|
1676
|
+
object: result.output,
|
|
1677
|
+
inputTokens,
|
|
1678
|
+
outputTokens,
|
|
1679
|
+
cost
|
|
1680
|
+
};
|
|
1681
|
+
} catch (error) {
|
|
1682
|
+
const durationMs = Date.now() - startTime;
|
|
1683
|
+
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
1684
|
+
logger.error(`generateObject error`, {
|
|
1685
|
+
model: modelKey,
|
|
1686
|
+
error: errorMessage,
|
|
1687
|
+
durationMs
|
|
1688
|
+
});
|
|
1689
|
+
this.aiCallLogger.logCall({
|
|
1690
|
+
topic: this.topic,
|
|
1691
|
+
callType: "object",
|
|
1692
|
+
modelKey,
|
|
1693
|
+
modelId: modelConfig.id,
|
|
1694
|
+
prompt: promptForLog,
|
|
1695
|
+
response: "",
|
|
1696
|
+
inputTokens: 0,
|
|
1697
|
+
outputTokens: 0,
|
|
1698
|
+
cost: 0,
|
|
1699
|
+
metadata: {
|
|
1700
|
+
temperature: options.temperature,
|
|
1701
|
+
maxTokens: options.maxTokens,
|
|
1702
|
+
finishReason: "error",
|
|
1703
|
+
durationMs,
|
|
1704
|
+
isMultimodal,
|
|
1705
|
+
status: "error",
|
|
1706
|
+
error: errorMessage
|
|
1707
|
+
}
|
|
1708
|
+
});
|
|
1709
|
+
throw error;
|
|
1710
|
+
}
|
|
1711
|
+
}
|
|
1712
|
+
async embed(modelKey, text, options = {}) {
|
|
1713
|
+
const modelConfig = getModel(modelKey);
|
|
1714
|
+
const texts = Array.isArray(text) ? text : [text];
|
|
1715
|
+
const startTime = Date.now();
|
|
1716
|
+
const dimensions = options.dimensions ?? DEFAULT_EMBEDDING_DIMENSIONS;
|
|
1717
|
+
const textPreview = texts.length === 1 ? texts[0].substring(0, 200) + (texts[0].length > 200 ? "..." : "") : `[${texts.length} texts]`;
|
|
1718
|
+
logger.debug(`embed request`, {
|
|
1719
|
+
model: modelKey,
|
|
1720
|
+
modelId: modelConfig.id,
|
|
1721
|
+
textCount: texts.length,
|
|
1722
|
+
textPreview,
|
|
1723
|
+
dimensions,
|
|
1724
|
+
taskType: options.taskType ?? "RETRIEVAL_DOCUMENT"
|
|
1725
|
+
});
|
|
1726
|
+
try {
|
|
1727
|
+
const embeddings = [];
|
|
1728
|
+
let totalInputTokens = 0;
|
|
1729
|
+
for (const t of texts) {
|
|
1730
|
+
const embeddingModelId = modelConfig.id.replace(/^google\//, "");
|
|
1731
|
+
const result = await embed({
|
|
1732
|
+
model: google.embeddingModel(embeddingModelId),
|
|
1733
|
+
value: t,
|
|
1734
|
+
providerOptions: {
|
|
1735
|
+
google: {
|
|
1736
|
+
outputDimensionality: dimensions,
|
|
1737
|
+
taskType: options.taskType ?? "RETRIEVAL_DOCUMENT"
|
|
1738
|
+
}
|
|
1739
|
+
}
|
|
1740
|
+
});
|
|
1741
|
+
embeddings.push(result.embedding);
|
|
1742
|
+
totalInputTokens += result.usage?.tokens || 0;
|
|
1743
|
+
}
|
|
1744
|
+
const outputTokens = 0;
|
|
1745
|
+
const cost = calculateCostWithDiscount(
|
|
1746
|
+
modelKey,
|
|
1747
|
+
totalInputTokens,
|
|
1748
|
+
outputTokens
|
|
1749
|
+
);
|
|
1750
|
+
const durationMs = Date.now() - startTime;
|
|
1751
|
+
this.aiCallLogger.logCall({
|
|
1752
|
+
topic: this.topic,
|
|
1753
|
+
callType: "embed",
|
|
1754
|
+
modelKey,
|
|
1755
|
+
modelId: modelConfig.id,
|
|
1756
|
+
prompt: texts.length === 1 ? texts[0] : `[${texts.length} texts]`,
|
|
1757
|
+
response: `[${embeddings.length} embeddings, ${dimensions} dims]`,
|
|
1758
|
+
inputTokens: totalInputTokens,
|
|
1759
|
+
outputTokens,
|
|
1760
|
+
cost,
|
|
1761
|
+
metadata: {
|
|
1762
|
+
taskType: options.taskType,
|
|
1763
|
+
textCount: texts.length,
|
|
1764
|
+
dimensions,
|
|
1765
|
+
durationMs
|
|
1766
|
+
}
|
|
1767
|
+
});
|
|
1768
|
+
logger.debug(`embed response`, {
|
|
1769
|
+
model: modelKey,
|
|
1770
|
+
embeddingsCount: embeddings.length,
|
|
1771
|
+
dimensions,
|
|
1772
|
+
inputTokens: totalInputTokens,
|
|
1773
|
+
cost: cost.toFixed(6),
|
|
1774
|
+
durationMs
|
|
1775
|
+
});
|
|
1776
|
+
return {
|
|
1777
|
+
embedding: embeddings[0],
|
|
1778
|
+
// Convenience: first embedding
|
|
1779
|
+
embeddings,
|
|
1780
|
+
// All embeddings
|
|
1781
|
+
dimensions,
|
|
1782
|
+
// Dimensionality used
|
|
1783
|
+
inputTokens: totalInputTokens,
|
|
1784
|
+
cost
|
|
1785
|
+
};
|
|
1786
|
+
} catch (error) {
|
|
1787
|
+
const durationMs = Date.now() - startTime;
|
|
1788
|
+
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
1789
|
+
logger.error(`embed error`, {
|
|
1790
|
+
model: modelKey,
|
|
1791
|
+
error: errorMessage,
|
|
1792
|
+
durationMs
|
|
1793
|
+
});
|
|
1794
|
+
this.aiCallLogger.logCall({
|
|
1795
|
+
topic: this.topic,
|
|
1796
|
+
callType: "embed",
|
|
1797
|
+
modelKey,
|
|
1798
|
+
modelId: modelConfig.id,
|
|
1799
|
+
prompt: texts.length === 1 ? texts[0] : `[${texts.length} texts]`,
|
|
1800
|
+
response: "",
|
|
1801
|
+
inputTokens: 0,
|
|
1802
|
+
outputTokens: 0,
|
|
1803
|
+
cost: 0,
|
|
1804
|
+
metadata: {
|
|
1805
|
+
taskType: options.taskType,
|
|
1806
|
+
textCount: texts.length,
|
|
1807
|
+
dimensions,
|
|
1808
|
+
durationMs,
|
|
1809
|
+
status: "error",
|
|
1810
|
+
error: errorMessage
|
|
1811
|
+
}
|
|
1812
|
+
});
|
|
1813
|
+
throw error;
|
|
1814
|
+
}
|
|
1815
|
+
}
|
|
1816
|
+
streamText(modelKey, input, options = {}) {
|
|
1817
|
+
const modelConfig = getModel(modelKey);
|
|
1818
|
+
const model = getModelProvider(modelConfig);
|
|
1819
|
+
const startTime = Date.now();
|
|
1820
|
+
const hasTools = options.tools !== void 0;
|
|
1821
|
+
const promptForLog = "prompt" in input && input.prompt ? input.prompt : JSON.stringify(input.messages);
|
|
1822
|
+
let errorLogged = false;
|
|
1823
|
+
const logError = (error) => {
|
|
1824
|
+
if (errorLogged) return;
|
|
1825
|
+
errorLogged = true;
|
|
1826
|
+
const durationMs = Date.now() - startTime;
|
|
1827
|
+
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
1828
|
+
this.aiCallLogger.logCall({
|
|
1829
|
+
topic: this.topic,
|
|
1830
|
+
callType: "stream",
|
|
1831
|
+
modelKey,
|
|
1832
|
+
modelId: modelConfig.id,
|
|
1833
|
+
prompt: promptForLog,
|
|
1834
|
+
response: "",
|
|
1835
|
+
inputTokens: 0,
|
|
1836
|
+
outputTokens: 0,
|
|
1837
|
+
cost: 0,
|
|
1838
|
+
metadata: {
|
|
1839
|
+
temperature: options.temperature,
|
|
1840
|
+
maxTokens: options.maxTokens,
|
|
1841
|
+
durationMs,
|
|
1842
|
+
status: "error",
|
|
1843
|
+
error: errorMessage,
|
|
1844
|
+
...input.system ? { system: input.system } : {}
|
|
1845
|
+
}
|
|
1846
|
+
});
|
|
1847
|
+
};
|
|
1848
|
+
logger.debug(`streamText request`, {
|
|
1849
|
+
model: modelKey,
|
|
1850
|
+
modelId: modelConfig.id,
|
|
1851
|
+
prompt: promptForLog.substring(0, 500) + (promptForLog.length > 500 ? "..." : ""),
|
|
1852
|
+
temperature: options.temperature ?? 0.7,
|
|
1853
|
+
maxTokens: options.maxTokens,
|
|
1854
|
+
hasTools,
|
|
1855
|
+
hasSystem: !!input.system
|
|
1856
|
+
});
|
|
1857
|
+
const baseParams = {
|
|
1858
|
+
model,
|
|
1859
|
+
temperature: options.temperature ?? 0.7,
|
|
1860
|
+
maxOutputTokens: options.maxTokens,
|
|
1861
|
+
...input.system ? { system: input.system } : {},
|
|
1862
|
+
// Tool-related options (only included if tools are provided)
|
|
1863
|
+
...hasTools && {
|
|
1864
|
+
tools: options.tools,
|
|
1865
|
+
stopWhen: options.stopWhen,
|
|
1866
|
+
onStepFinish: options.onStepFinish
|
|
1867
|
+
},
|
|
1868
|
+
// Error callback to log streaming errors
|
|
1869
|
+
onError: ({ error }) => {
|
|
1870
|
+
logError(error);
|
|
1871
|
+
}
|
|
1872
|
+
};
|
|
1873
|
+
const result = "messages" in input && input.messages ? streamText({ ...baseParams, messages: input.messages }) : streamText({
|
|
1874
|
+
...baseParams,
|
|
1875
|
+
prompt: input.prompt
|
|
1876
|
+
});
|
|
1877
|
+
let fullText = "";
|
|
1878
|
+
let chunkCount = 0;
|
|
1879
|
+
let streamConsumed = false;
|
|
1880
|
+
let usageResolved = false;
|
|
1881
|
+
let cachedUsage = null;
|
|
1882
|
+
const streamIterable = {
|
|
1883
|
+
[Symbol.asyncIterator]: () => {
|
|
1884
|
+
const reader = result.textStream[Symbol.asyncIterator]();
|
|
1885
|
+
return {
|
|
1886
|
+
async next() {
|
|
1887
|
+
try {
|
|
1888
|
+
const { done, value } = await reader.next();
|
|
1889
|
+
if (done) {
|
|
1890
|
+
streamConsumed = true;
|
|
1891
|
+
return { done: true, value: void 0 };
|
|
1892
|
+
}
|
|
1893
|
+
fullText += value;
|
|
1894
|
+
chunkCount++;
|
|
1895
|
+
options.onChunk?.(value);
|
|
1896
|
+
return { done: false, value };
|
|
1897
|
+
} catch (error) {
|
|
1898
|
+
logError(error);
|
|
1899
|
+
throw error;
|
|
1900
|
+
}
|
|
1901
|
+
}
|
|
1902
|
+
};
|
|
1903
|
+
}
|
|
1904
|
+
};
|
|
1905
|
+
const getUsage = async () => {
|
|
1906
|
+
if (usageResolved && cachedUsage) {
|
|
1907
|
+
return cachedUsage;
|
|
1908
|
+
}
|
|
1909
|
+
if (!streamConsumed) {
|
|
1910
|
+
for await (const _ of streamIterable) {
|
|
1911
|
+
}
|
|
1912
|
+
}
|
|
1913
|
+
const usage = await result.usage;
|
|
1914
|
+
const inputTokens = usage?.inputTokens ?? 0;
|
|
1915
|
+
const outputTokens = usage?.outputTokens ?? 0;
|
|
1916
|
+
const cost = calculateCostWithDiscount(
|
|
1917
|
+
modelKey,
|
|
1918
|
+
inputTokens,
|
|
1919
|
+
outputTokens
|
|
1920
|
+
);
|
|
1921
|
+
const durationMs = Date.now() - startTime;
|
|
1922
|
+
if (!usageResolved) {
|
|
1923
|
+
usageResolved = true;
|
|
1924
|
+
cachedUsage = { inputTokens, outputTokens, cost };
|
|
1925
|
+
logger.debug(`streamText response`, {
|
|
1926
|
+
model: modelKey,
|
|
1927
|
+
response: fullText.substring(0, 500) + (fullText.length > 500 ? "..." : ""),
|
|
1928
|
+
inputTokens,
|
|
1929
|
+
outputTokens,
|
|
1930
|
+
cost: cost.toFixed(6),
|
|
1931
|
+
durationMs,
|
|
1932
|
+
chunkCount
|
|
1933
|
+
});
|
|
1934
|
+
this.aiCallLogger.logCall({
|
|
1935
|
+
topic: this.topic,
|
|
1936
|
+
callType: "stream",
|
|
1937
|
+
modelKey,
|
|
1938
|
+
modelId: modelConfig.id,
|
|
1939
|
+
prompt: promptForLog,
|
|
1940
|
+
response: fullText,
|
|
1941
|
+
inputTokens,
|
|
1942
|
+
outputTokens,
|
|
1943
|
+
cost,
|
|
1944
|
+
metadata: {
|
|
1945
|
+
temperature: options.temperature,
|
|
1946
|
+
maxTokens: options.maxTokens,
|
|
1947
|
+
streamChunks: chunkCount,
|
|
1948
|
+
durationMs,
|
|
1949
|
+
...input.system ? { system: input.system } : {}
|
|
1950
|
+
}
|
|
1951
|
+
});
|
|
1952
|
+
}
|
|
1953
|
+
return cachedUsage ?? { inputTokens, outputTokens, cost };
|
|
1954
|
+
};
|
|
1955
|
+
return {
|
|
1956
|
+
stream: streamIterable,
|
|
1957
|
+
getUsage,
|
|
1958
|
+
rawResult: result
|
|
1959
|
+
};
|
|
1960
|
+
}
|
|
1961
|
+
batch(modelKey, provider) {
|
|
1962
|
+
const resolvedProvider = provider ?? getBestProviderForModel(modelKey) ?? "google";
|
|
1963
|
+
return new AIBatchImpl(
|
|
1964
|
+
this,
|
|
1965
|
+
modelKey,
|
|
1966
|
+
resolvedProvider,
|
|
1967
|
+
this.batchLogFn
|
|
1968
|
+
);
|
|
1969
|
+
}
|
|
1970
|
+
createChild(segment, id) {
|
|
1971
|
+
const newTopic = id ? `${this.topic}.${segment}.${id}` : `${this.topic}.${segment}`;
|
|
1972
|
+
return new _AIHelperImpl(newTopic, this.aiCallLogger, this.logContext);
|
|
1973
|
+
}
|
|
1974
|
+
/** @internal Get the logger for batch operations */
|
|
1975
|
+
getLogger() {
|
|
1976
|
+
return this.aiCallLogger;
|
|
1977
|
+
}
|
|
1978
|
+
// Overloaded recordCall to support both new object-based API and legacy positional API
|
|
1979
|
+
recordCall(paramsOrModelKey, prompt, response, tokens, options) {
|
|
1980
|
+
let modelKey;
|
|
1981
|
+
let actualPrompt;
|
|
1982
|
+
let actualResponse;
|
|
1983
|
+
let inputTokens;
|
|
1984
|
+
let outputTokens;
|
|
1985
|
+
let callType;
|
|
1986
|
+
let isBatch;
|
|
1987
|
+
let metadata;
|
|
1988
|
+
if (typeof paramsOrModelKey === "object" && "modelKey" in paramsOrModelKey) {
|
|
1989
|
+
const params = paramsOrModelKey;
|
|
1990
|
+
modelKey = params.modelKey;
|
|
1991
|
+
actualPrompt = params.prompt;
|
|
1992
|
+
actualResponse = params.response;
|
|
1993
|
+
inputTokens = params.inputTokens;
|
|
1994
|
+
outputTokens = params.outputTokens;
|
|
1995
|
+
callType = params.callType;
|
|
1996
|
+
isBatch = callType === "batch";
|
|
1997
|
+
metadata = params.metadata;
|
|
1998
|
+
} else {
|
|
1999
|
+
if (!prompt || !response || !tokens) {
|
|
2000
|
+
throw new Error(
|
|
2001
|
+
"recordCall: legacy API requires prompt, response, and tokens"
|
|
2002
|
+
);
|
|
2003
|
+
}
|
|
2004
|
+
modelKey = paramsOrModelKey;
|
|
2005
|
+
actualPrompt = prompt;
|
|
2006
|
+
actualResponse = response;
|
|
2007
|
+
inputTokens = tokens.input;
|
|
2008
|
+
outputTokens = tokens.output;
|
|
2009
|
+
callType = options?.callType ?? "text";
|
|
2010
|
+
isBatch = options?.isBatch ?? false;
|
|
2011
|
+
metadata = options?.metadata;
|
|
2012
|
+
}
|
|
2013
|
+
const modelConfig = getModel(modelKey);
|
|
2014
|
+
const cost = calculateCostWithDiscount(
|
|
2015
|
+
modelKey,
|
|
2016
|
+
inputTokens,
|
|
2017
|
+
outputTokens,
|
|
2018
|
+
isBatch
|
|
2019
|
+
);
|
|
2020
|
+
this.aiCallLogger.logCall({
|
|
2021
|
+
topic: this.topic,
|
|
2022
|
+
callType,
|
|
2023
|
+
modelKey,
|
|
2024
|
+
modelId: modelConfig.id,
|
|
2025
|
+
prompt: actualPrompt,
|
|
2026
|
+
response: actualResponse,
|
|
2027
|
+
inputTokens,
|
|
2028
|
+
outputTokens,
|
|
2029
|
+
cost,
|
|
2030
|
+
metadata: isBatch ? { ...metadata, isBatch: true } : metadata
|
|
2031
|
+
});
|
|
2032
|
+
}
|
|
2033
|
+
async getStats() {
|
|
2034
|
+
return this.aiCallLogger.getStats(this.topic);
|
|
2035
|
+
}
|
|
2036
|
+
};
|
|
2037
|
+
var AIBatchImpl = class {
|
|
2038
|
+
constructor(helper, modelKey, provider, batchLogFn) {
|
|
2039
|
+
this.helper = helper;
|
|
2040
|
+
this.modelKey = modelKey;
|
|
2041
|
+
this.provider = provider;
|
|
2042
|
+
this.batchLogFn = batchLogFn;
|
|
2043
|
+
const batchLogger = {
|
|
2044
|
+
log: (level, message, meta) => {
|
|
2045
|
+
if (this.batchLogFn) {
|
|
2046
|
+
this.batchLogFn(level, `[Batch:${provider}] ${message}`, meta);
|
|
2047
|
+
} else {
|
|
2048
|
+
logger.debug(
|
|
2049
|
+
`[Batch:${provider}] [${level}] ${message}`,
|
|
2050
|
+
meta ? JSON.stringify(meta) : ""
|
|
2051
|
+
);
|
|
2052
|
+
}
|
|
2053
|
+
}
|
|
2054
|
+
};
|
|
2055
|
+
if (provider === "google") {
|
|
2056
|
+
this.googleProvider = new GoogleBatchProvider({}, batchLogger);
|
|
2057
|
+
} else if (provider === "anthropic") {
|
|
2058
|
+
this.anthropicProvider = new AnthropicBatchProvider({}, batchLogger);
|
|
2059
|
+
} else if (provider === "openai") {
|
|
2060
|
+
this.openaiProvider = new OpenAIBatchProvider({}, batchLogger);
|
|
2061
|
+
}
|
|
2062
|
+
}
|
|
2063
|
+
googleProvider = null;
|
|
2064
|
+
anthropicProvider = null;
|
|
2065
|
+
openaiProvider = null;
|
|
2066
|
+
async submit(requests) {
|
|
2067
|
+
logger.debug(`batch submit request`, {
|
|
2068
|
+
provider: this.provider,
|
|
2069
|
+
model: this.modelKey,
|
|
2070
|
+
requestCount: requests.length,
|
|
2071
|
+
requestIds: requests.slice(0, 10).map((r) => r.id),
|
|
2072
|
+
hasMoreRequests: requests.length > 10
|
|
2073
|
+
});
|
|
2074
|
+
const jsonSystemPrompt = "You must respond with valid JSON only. No markdown, no explanation, just the JSON object.";
|
|
2075
|
+
if (this.provider === "google" && this.googleProvider) {
|
|
2076
|
+
const googleRequests = requests.map((req) => ({
|
|
2077
|
+
id: req.id,
|
|
2078
|
+
prompt: req.prompt,
|
|
2079
|
+
model: this.modelKey,
|
|
2080
|
+
...req.schema && { system: jsonSystemPrompt, schema: req.schema }
|
|
2081
|
+
}));
|
|
2082
|
+
const handle = await this.googleProvider.submit(googleRequests);
|
|
2083
|
+
logger.debug(`batch submitted`, {
|
|
2084
|
+
provider: "google",
|
|
2085
|
+
batchId: handle.id,
|
|
2086
|
+
requestCount: requests.length
|
|
2087
|
+
});
|
|
2088
|
+
return { id: handle.id, status: "pending", provider: this.provider };
|
|
2089
|
+
}
|
|
2090
|
+
if (this.provider === "anthropic" && this.anthropicProvider) {
|
|
2091
|
+
const anthropicRequests = requests.map((req) => ({
|
|
2092
|
+
customId: req.id,
|
|
2093
|
+
prompt: req.prompt,
|
|
2094
|
+
model: this.modelKey,
|
|
2095
|
+
...req.schema && { system: jsonSystemPrompt }
|
|
2096
|
+
}));
|
|
2097
|
+
const handle = await this.anthropicProvider.submit(anthropicRequests);
|
|
2098
|
+
logger.debug(`batch submitted`, {
|
|
2099
|
+
provider: "anthropic",
|
|
2100
|
+
batchId: handle.id,
|
|
2101
|
+
requestCount: requests.length
|
|
2102
|
+
});
|
|
2103
|
+
return { id: handle.id, status: "pending", provider: this.provider };
|
|
2104
|
+
}
|
|
2105
|
+
if (this.provider === "openai" && this.openaiProvider) {
|
|
2106
|
+
const openaiRequests = requests.map((req) => ({
|
|
2107
|
+
customId: req.id,
|
|
2108
|
+
prompt: req.prompt,
|
|
2109
|
+
model: this.modelKey,
|
|
2110
|
+
...req.schema && { system: jsonSystemPrompt }
|
|
2111
|
+
}));
|
|
2112
|
+
const handle = await this.openaiProvider.submit(openaiRequests);
|
|
2113
|
+
logger.debug(`batch submitted`, {
|
|
2114
|
+
provider: "openai",
|
|
2115
|
+
batchId: handle.id,
|
|
2116
|
+
requestCount: requests.length
|
|
2117
|
+
});
|
|
2118
|
+
return { id: handle.id, status: "pending", provider: this.provider };
|
|
2119
|
+
}
|
|
2120
|
+
throw new Error(
|
|
2121
|
+
`Batch submission for provider "${this.provider}" not yet implemented. Use recordCall() to manually record batch results.`
|
|
2122
|
+
);
|
|
2123
|
+
}
|
|
2124
|
+
async getStatus(batchId) {
|
|
2125
|
+
const handle = {
|
|
2126
|
+
id: batchId,
|
|
2127
|
+
provider: this.provider,
|
|
2128
|
+
requestCount: 0,
|
|
2129
|
+
createdAt: /* @__PURE__ */ new Date()
|
|
2130
|
+
};
|
|
2131
|
+
let status;
|
|
2132
|
+
if (this.provider === "google" && this.googleProvider) {
|
|
2133
|
+
status = await this.googleProvider.checkStatus(handle);
|
|
2134
|
+
} else if (this.provider === "anthropic" && this.anthropicProvider) {
|
|
2135
|
+
status = await this.anthropicProvider.checkStatus(handle);
|
|
2136
|
+
} else if (this.provider === "openai" && this.openaiProvider) {
|
|
2137
|
+
status = await this.openaiProvider.checkStatus(handle);
|
|
2138
|
+
} else {
|
|
2139
|
+
throw new Error(
|
|
2140
|
+
`Batch status check for provider "${this.provider}" not yet implemented.`
|
|
2141
|
+
);
|
|
2142
|
+
}
|
|
2143
|
+
let batchStatus;
|
|
2144
|
+
switch (status.state) {
|
|
2145
|
+
case "completed":
|
|
2146
|
+
batchStatus = "completed";
|
|
2147
|
+
break;
|
|
2148
|
+
case "failed":
|
|
2149
|
+
batchStatus = "failed";
|
|
2150
|
+
break;
|
|
2151
|
+
case "processing":
|
|
2152
|
+
batchStatus = "processing";
|
|
2153
|
+
break;
|
|
2154
|
+
default:
|
|
2155
|
+
batchStatus = "pending";
|
|
2156
|
+
}
|
|
2157
|
+
return { id: batchId, status: batchStatus, provider: this.provider };
|
|
2158
|
+
}
|
|
2159
|
+
async getResults(batchId, metadata) {
|
|
2160
|
+
if (this.batchLogFn) {
|
|
2161
|
+
this.batchLogFn("DEBUG", `[AIBatch:getResults] Received metadata`, {
|
|
2162
|
+
hasMetadata: !!metadata,
|
|
2163
|
+
metadataKeys: metadata ? Object.keys(metadata) : [],
|
|
2164
|
+
hasCustomIds: !!metadata?.customIds,
|
|
2165
|
+
customIdsCount: Array.isArray(metadata?.customIds) ? metadata.customIds.length : 0
|
|
2166
|
+
});
|
|
2167
|
+
}
|
|
2168
|
+
const handle = {
|
|
2169
|
+
id: batchId,
|
|
2170
|
+
provider: this.provider,
|
|
2171
|
+
requestCount: 0,
|
|
2172
|
+
createdAt: /* @__PURE__ */ new Date(),
|
|
2173
|
+
metadata
|
|
2174
|
+
};
|
|
2175
|
+
let rawResults;
|
|
2176
|
+
if (this.provider === "google" && this.googleProvider) {
|
|
2177
|
+
rawResults = await this.googleProvider.getResults(handle);
|
|
2178
|
+
} else if (this.provider === "anthropic" && this.anthropicProvider) {
|
|
2179
|
+
rawResults = await this.anthropicProvider.getResults(handle);
|
|
2180
|
+
} else if (this.provider === "openai" && this.openaiProvider) {
|
|
2181
|
+
rawResults = await this.openaiProvider.getResults(handle);
|
|
2182
|
+
} else {
|
|
2183
|
+
throw new Error(
|
|
2184
|
+
`Batch results for provider "${this.provider}" not yet implemented. Use recordCall() to manually record batch results.`
|
|
2185
|
+
);
|
|
2186
|
+
}
|
|
2187
|
+
const totalInputTokens = rawResults.reduce(
|
|
2188
|
+
(sum, r) => sum + (r.inputTokens || 0),
|
|
2189
|
+
0
|
|
2190
|
+
);
|
|
2191
|
+
const totalOutputTokens = rawResults.reduce(
|
|
2192
|
+
(sum, r) => sum + (r.outputTokens || 0),
|
|
2193
|
+
0
|
|
2194
|
+
);
|
|
2195
|
+
const failedCount = rawResults.filter((r) => r.error).length;
|
|
2196
|
+
logger.debug(`batch getResults response`, {
|
|
2197
|
+
batchId,
|
|
2198
|
+
provider: this.provider,
|
|
2199
|
+
resultCount: rawResults.length,
|
|
2200
|
+
failedCount,
|
|
2201
|
+
totalInputTokens,
|
|
2202
|
+
totalOutputTokens
|
|
2203
|
+
});
|
|
2204
|
+
const results = rawResults.map((raw, index) => {
|
|
2205
|
+
if (raw.error) {
|
|
2206
|
+
return {
|
|
2207
|
+
id: raw.customId || `result-${index}`,
|
|
2208
|
+
prompt: "",
|
|
2209
|
+
// Not available from raw results
|
|
2210
|
+
result: {},
|
|
2211
|
+
// Empty result for failed requests
|
|
2212
|
+
inputTokens: raw.inputTokens || 0,
|
|
2213
|
+
outputTokens: raw.outputTokens || 0,
|
|
2214
|
+
status: "failed",
|
|
2215
|
+
error: raw.error
|
|
2216
|
+
};
|
|
2217
|
+
}
|
|
2218
|
+
let parsedResult;
|
|
2219
|
+
try {
|
|
2220
|
+
let cleaned = raw.text.trim();
|
|
2221
|
+
if (cleaned.startsWith("```json")) {
|
|
2222
|
+
cleaned = cleaned.slice(7);
|
|
2223
|
+
} else if (cleaned.startsWith("```")) {
|
|
2224
|
+
cleaned = cleaned.slice(3);
|
|
2225
|
+
}
|
|
2226
|
+
if (cleaned.endsWith("```")) {
|
|
2227
|
+
cleaned = cleaned.slice(0, -3);
|
|
2228
|
+
}
|
|
2229
|
+
cleaned = cleaned.trim();
|
|
2230
|
+
parsedResult = JSON.parse(cleaned);
|
|
2231
|
+
} catch {
|
|
2232
|
+
parsedResult = raw.text;
|
|
2233
|
+
}
|
|
2234
|
+
return {
|
|
2235
|
+
id: raw.customId || `result-${index}`,
|
|
2236
|
+
prompt: "",
|
|
2237
|
+
// Not available from raw results
|
|
2238
|
+
result: parsedResult,
|
|
2239
|
+
inputTokens: raw.inputTokens || 0,
|
|
2240
|
+
outputTokens: raw.outputTokens || 0,
|
|
2241
|
+
status: "succeeded"
|
|
2242
|
+
};
|
|
2243
|
+
});
|
|
2244
|
+
await this.recordResults(batchId, results);
|
|
2245
|
+
return results;
|
|
2246
|
+
}
|
|
2247
|
+
async isRecorded(batchId) {
|
|
2248
|
+
return this.helper.getLogger().isRecorded(batchId);
|
|
2249
|
+
}
|
|
2250
|
+
/**
|
|
2251
|
+
* Record batch results manually.
|
|
2252
|
+
* Use this when batch provider integration is not yet implemented.
|
|
2253
|
+
*/
|
|
2254
|
+
async recordResults(batchId, results) {
|
|
2255
|
+
if (await this.isRecorded(batchId)) {
|
|
2256
|
+
logger.debug(`Batch ${batchId} already recorded, skipping.`);
|
|
2257
|
+
return;
|
|
2258
|
+
}
|
|
2259
|
+
const modelConfig = getModel(this.modelKey);
|
|
2260
|
+
const discountPercent = modelConfig.batchDiscountPercent ?? 0;
|
|
2261
|
+
await this.helper.getLogger().logBatchResults(
|
|
2262
|
+
batchId,
|
|
2263
|
+
results.map((r) => {
|
|
2264
|
+
const baseCost = calculateCost(
|
|
2265
|
+
this.modelKey,
|
|
2266
|
+
r.inputTokens,
|
|
2267
|
+
r.outputTokens
|
|
2268
|
+
);
|
|
2269
|
+
const cost = baseCost.totalCost * (1 - discountPercent / 100);
|
|
2270
|
+
return {
|
|
2271
|
+
topic: this.helper.topic,
|
|
2272
|
+
callType: "batch",
|
|
2273
|
+
modelKey: this.modelKey,
|
|
2274
|
+
modelId: modelConfig.id,
|
|
2275
|
+
prompt: r.prompt,
|
|
2276
|
+
response: typeof r.result === "string" ? r.result : JSON.stringify(r.result),
|
|
2277
|
+
inputTokens: r.inputTokens,
|
|
2278
|
+
outputTokens: r.outputTokens,
|
|
2279
|
+
cost,
|
|
2280
|
+
metadata: { batchId, requestId: r.id }
|
|
2281
|
+
};
|
|
2282
|
+
})
|
|
2283
|
+
);
|
|
2284
|
+
}
|
|
2285
|
+
};
|
|
2286
|
+
function createAIHelper(topic, logger2, logContext) {
|
|
2287
|
+
return new AIHelperImpl(topic, logger2, logContext);
|
|
2288
|
+
}
|
|
2289
|
+
|
|
2290
|
+
export { AVAILABLE_MODELS, AnthropicBatchProvider, DEFAULT_MODEL_KEY, GoogleBatchProvider, ModelKey, ModelStatsTracker, NoInputSchema, OpenAIBatchProvider, calculateCost, createAIHelper, defineAsyncBatchStage, defineStage, getBestProviderForModel, getDefaultModel, getModel, getModelById, getRegisteredModel, listModels, listRegisteredModels, modelSupportsBatch, printAvailableModels, registerModels, requireStageOutput, resolveModelForProvider };
|
|
2291
|
+
//# sourceMappingURL=chunk-P4KMGCT3.js.map
|
|
2292
|
+
//# sourceMappingURL=chunk-P4KMGCT3.js.map
|