@axlsdk/axl 0.21.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -0
- package/dist/index.cjs +3062 -183
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +580 -47
- package/dist/index.d.ts +580 -47
- package/dist/index.js +3073 -207
- package/dist/index.js.map +1 -1
- package/package.json +3 -3
package/dist/index.js
CHANGED
|
@@ -14,6 +14,75 @@ var AxlError = class extends Error {
|
|
|
14
14
|
this.code = code;
|
|
15
15
|
}
|
|
16
16
|
};
|
|
17
|
+
var InvalidModelInputError = class extends AxlError {
|
|
18
|
+
constructor(message) {
|
|
19
|
+
super("INVALID_MODEL_INPUT", message);
|
|
20
|
+
this.name = "InvalidModelInputError";
|
|
21
|
+
}
|
|
22
|
+
};
|
|
23
|
+
var UnsupportedModelInputError = class extends AxlError {
|
|
24
|
+
provider;
|
|
25
|
+
model;
|
|
26
|
+
modality;
|
|
27
|
+
source;
|
|
28
|
+
constructor(options) {
|
|
29
|
+
super(
|
|
30
|
+
"UNSUPPORTED_MODEL_INPUT",
|
|
31
|
+
`Provider '${options.provider}' model '${options.model}' does not support ${options.feature ?? options.modality}${options.source ? ` from ${options.source}` : ""}`
|
|
32
|
+
);
|
|
33
|
+
this.name = "UnsupportedModelInputError";
|
|
34
|
+
this.provider = options.provider;
|
|
35
|
+
this.model = options.model;
|
|
36
|
+
this.modality = options.modality;
|
|
37
|
+
this.source = options.source;
|
|
38
|
+
}
|
|
39
|
+
};
|
|
40
|
+
var InvalidTranscriptionInputError = class extends AxlError {
|
|
41
|
+
constructor(message) {
|
|
42
|
+
super("INVALID_TRANSCRIPTION_INPUT", message);
|
|
43
|
+
this.name = "InvalidTranscriptionInputError";
|
|
44
|
+
}
|
|
45
|
+
};
|
|
46
|
+
var UnsupportedTranscriptionInputError = class extends AxlError {
|
|
47
|
+
provider;
|
|
48
|
+
model;
|
|
49
|
+
constructor(options) {
|
|
50
|
+
super(
|
|
51
|
+
"UNSUPPORTED_TRANSCRIPTION_INPUT",
|
|
52
|
+
`Transcription provider '${options.provider}'${options.model ? ` model '${options.model}'` : ""} does not support ${options.feature ?? "this request"}`
|
|
53
|
+
);
|
|
54
|
+
this.name = "UnsupportedTranscriptionInputError";
|
|
55
|
+
this.provider = options.provider;
|
|
56
|
+
this.model = options.model;
|
|
57
|
+
}
|
|
58
|
+
};
|
|
59
|
+
var TranscriptionOperationError = class extends AxlError {
|
|
60
|
+
provider;
|
|
61
|
+
model;
|
|
62
|
+
usage;
|
|
63
|
+
pricingStatus;
|
|
64
|
+
cleanupStatus;
|
|
65
|
+
/** Safe HTTP diagnostics projected from ProviderError when available. */
|
|
66
|
+
status;
|
|
67
|
+
retryable;
|
|
68
|
+
retryAfterMs;
|
|
69
|
+
requestId;
|
|
70
|
+
constructor(options) {
|
|
71
|
+
super("TRANSCRIPTION_PROVIDER_ERROR", "Transcription provider operation failed");
|
|
72
|
+
this.name = "TranscriptionOperationError";
|
|
73
|
+
this.provider = options.provider;
|
|
74
|
+
this.model = options.model;
|
|
75
|
+
this.usage = options.usage;
|
|
76
|
+
this.pricingStatus = options.pricingStatus;
|
|
77
|
+
this.cleanupStatus = options.cleanupStatus;
|
|
78
|
+
this.status = options.status;
|
|
79
|
+
this.retryable = options.retryable;
|
|
80
|
+
this.retryAfterMs = options.retryAfterMs;
|
|
81
|
+
this.requestId = options.requestId;
|
|
82
|
+
if (options.cause !== void 0)
|
|
83
|
+
Object.defineProperty(this, "cause", { enumerable: false, value: options.cause });
|
|
84
|
+
}
|
|
85
|
+
};
|
|
17
86
|
var EventStreamOverflowError = class extends Error {
|
|
18
87
|
maxQueued;
|
|
19
88
|
eventType;
|
|
@@ -171,6 +240,221 @@ var ValidationError = class extends AxlError {
|
|
|
171
240
|
|
|
172
241
|
// src/tool.ts
|
|
173
242
|
var DEFAULT_MAX_STRING_LENGTH = 1e4;
|
|
243
|
+
var MAX_MODEL_ARGUMENT_ISSUES = 8;
|
|
244
|
+
var MAX_MODEL_ARGUMENT_MESSAGE_LENGTH = 2e3;
|
|
245
|
+
var MAX_MODEL_ARGUMENT_LINE_LENGTH = 220;
|
|
246
|
+
var MAX_MODEL_ARGUMENT_PATH_LENGTH = 160;
|
|
247
|
+
var MAX_UNION_DEPTH = 3;
|
|
248
|
+
var MAX_UNION_BRANCHES = 4;
|
|
249
|
+
var MAX_LITERAL_ALTERNATIVES = 10;
|
|
250
|
+
var SAFE_SCHEMA_TYPES = /* @__PURE__ */ new Set([
|
|
251
|
+
"string",
|
|
252
|
+
"number",
|
|
253
|
+
"boolean",
|
|
254
|
+
"null",
|
|
255
|
+
"array",
|
|
256
|
+
"object",
|
|
257
|
+
"integer"
|
|
258
|
+
]);
|
|
259
|
+
var SAFE_STRING_FORMATS = /* @__PURE__ */ new Set([
|
|
260
|
+
"email",
|
|
261
|
+
"url",
|
|
262
|
+
"emoji",
|
|
263
|
+
"uuid",
|
|
264
|
+
"guid",
|
|
265
|
+
"nanoid",
|
|
266
|
+
"cuid",
|
|
267
|
+
"cuid2",
|
|
268
|
+
"ulid",
|
|
269
|
+
"xid",
|
|
270
|
+
"ksuid",
|
|
271
|
+
"datetime",
|
|
272
|
+
"date",
|
|
273
|
+
"time",
|
|
274
|
+
"duration",
|
|
275
|
+
"ipv4",
|
|
276
|
+
"ipv6",
|
|
277
|
+
"cidrv4",
|
|
278
|
+
"cidrv6",
|
|
279
|
+
"base64",
|
|
280
|
+
"base64url",
|
|
281
|
+
"json_string",
|
|
282
|
+
"e164",
|
|
283
|
+
"lowercase",
|
|
284
|
+
"uppercase",
|
|
285
|
+
"jwt"
|
|
286
|
+
]);
|
|
287
|
+
function truncateModelFeedback(value, maximum) {
|
|
288
|
+
return value.length <= maximum ? value : `${value.slice(0, Math.max(0, maximum - 1))}\u2026`;
|
|
289
|
+
}
|
|
290
|
+
var DISPLAY_CONTROL_CHARACTERS = /[\u061C\u200B-\u200F\u202A-\u202E\u2066-\u2069\u2028\u2029]/g;
|
|
291
|
+
function isJsonSchema(value) {
|
|
292
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
293
|
+
}
|
|
294
|
+
function escapeDisplayControls(value) {
|
|
295
|
+
return value.replace(
|
|
296
|
+
DISPLAY_CONTROL_CHARACTERS,
|
|
297
|
+
(character) => `\\u${character.codePointAt(0).toString(16).padStart(4, "0")}`
|
|
298
|
+
);
|
|
299
|
+
}
|
|
300
|
+
function quoteModelText(value) {
|
|
301
|
+
return escapeDisplayControls(JSON.stringify(value));
|
|
302
|
+
}
|
|
303
|
+
function schemaNumber(value) {
|
|
304
|
+
return typeof value === "number" && Number.isFinite(value) ? String(value) : void 0;
|
|
305
|
+
}
|
|
306
|
+
function schemaLiteral(value) {
|
|
307
|
+
if (typeof value === "string") {
|
|
308
|
+
const rendered = quoteModelText(value);
|
|
309
|
+
return rendered.length <= 120 ? rendered : '"<literal omitted>"';
|
|
310
|
+
}
|
|
311
|
+
if (typeof value === "number") return Number.isFinite(value) ? String(value) : void 0;
|
|
312
|
+
if (typeof value === "boolean") return String(value);
|
|
313
|
+
if (value === null) return "null";
|
|
314
|
+
return void 0;
|
|
315
|
+
}
|
|
316
|
+
function schemaType(schema) {
|
|
317
|
+
const type = schema.type;
|
|
318
|
+
return typeof type === "string" && SAFE_SCHEMA_TYPES.has(type) ? type : void 0;
|
|
319
|
+
}
|
|
320
|
+
function compositionBranches(schema) {
|
|
321
|
+
const branches = [];
|
|
322
|
+
for (const key of ["anyOf", "oneOf", "allOf"]) {
|
|
323
|
+
const value = schema[key];
|
|
324
|
+
if (!Array.isArray(value)) continue;
|
|
325
|
+
for (const branch of value.slice(0, MAX_UNION_BRANCHES)) {
|
|
326
|
+
if (isJsonSchema(branch)) branches.push(branch);
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
return branches;
|
|
330
|
+
}
|
|
331
|
+
function unionBranches(schema) {
|
|
332
|
+
const branches = [];
|
|
333
|
+
for (const key of ["anyOf", "oneOf"]) {
|
|
334
|
+
const value = schema[key];
|
|
335
|
+
if (!Array.isArray(value)) continue;
|
|
336
|
+
for (const branch of value.slice(0, MAX_UNION_BRANCHES)) {
|
|
337
|
+
if (isJsonSchema(branch)) branches.push(branch);
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
return branches;
|
|
341
|
+
}
|
|
342
|
+
function resolvePathSegment(schema, segment, depth = 0) {
|
|
343
|
+
if (depth >= MAX_UNION_DEPTH) return void 0;
|
|
344
|
+
if (typeof segment === "string") {
|
|
345
|
+
const properties = schema.properties;
|
|
346
|
+
if (isJsonSchema(properties) && Object.hasOwn(properties, segment)) {
|
|
347
|
+
const child = properties[segment];
|
|
348
|
+
if (isJsonSchema(child)) return { node: child, renderedSegment: segment };
|
|
349
|
+
}
|
|
350
|
+
if (isJsonSchema(schema.additionalProperties)) {
|
|
351
|
+
return { node: schema.additionalProperties, renderedSegment: "<key>" };
|
|
352
|
+
}
|
|
353
|
+
} else if (typeof segment === "number" && Number.isSafeInteger(segment) && segment >= 0) {
|
|
354
|
+
const prefixItems = schema.prefixItems;
|
|
355
|
+
if (Array.isArray(prefixItems) && isJsonSchema(prefixItems[segment])) {
|
|
356
|
+
return { node: prefixItems[segment], renderedSegment: String(segment) };
|
|
357
|
+
}
|
|
358
|
+
if (isJsonSchema(schema.items)) {
|
|
359
|
+
return { node: schema.items, renderedSegment: "<index>" };
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
for (const branch of compositionBranches(schema)) {
|
|
363
|
+
const resolved = resolvePathSegment(branch, segment, depth + 1);
|
|
364
|
+
if (resolved !== void 0) return resolved;
|
|
365
|
+
}
|
|
366
|
+
return void 0;
|
|
367
|
+
}
|
|
368
|
+
function resolveSchemaPath(providerVisibleSchema, issuePath) {
|
|
369
|
+
if (!isJsonSchema(providerVisibleSchema)) return void 0;
|
|
370
|
+
let node = providerVisibleSchema;
|
|
371
|
+
const segments = [];
|
|
372
|
+
for (const segment of issuePath) {
|
|
373
|
+
const resolved = resolvePathSegment(node, segment);
|
|
374
|
+
if (resolved === void 0) {
|
|
375
|
+
segments.push("<path omitted>");
|
|
376
|
+
return { node, segments };
|
|
377
|
+
}
|
|
378
|
+
node = resolved.node;
|
|
379
|
+
segments.push(resolved.renderedSegment);
|
|
380
|
+
}
|
|
381
|
+
return { node, segments };
|
|
382
|
+
}
|
|
383
|
+
function jsonPointerPath(segments) {
|
|
384
|
+
if (segments.length === 0) return "<root>";
|
|
385
|
+
const pointer = `/${segments.map((segment) => segment.replaceAll("~", "~0").replaceAll("/", "~1")).join("/")}`;
|
|
386
|
+
const rendered = quoteModelText(pointer);
|
|
387
|
+
return rendered.length <= MAX_MODEL_ARGUMENT_PATH_LENGTH ? rendered : '"<path omitted>"';
|
|
388
|
+
}
|
|
389
|
+
function schemaAlternatives(schema, depth = 0) {
|
|
390
|
+
if (depth >= MAX_UNION_DEPTH) return [];
|
|
391
|
+
const constant = "const" in schema ? schemaLiteral(schema.const) : void 0;
|
|
392
|
+
if (constant !== void 0) return [constant];
|
|
393
|
+
if (Array.isArray(schema.enum)) {
|
|
394
|
+
const values = schema.enum.slice(0, MAX_LITERAL_ALTERNATIVES).map(schemaLiteral);
|
|
395
|
+
if (values.every((value) => value !== void 0)) return values;
|
|
396
|
+
}
|
|
397
|
+
const type = schemaType(schema);
|
|
398
|
+
if (type !== void 0) return [type];
|
|
399
|
+
const alternatives = unionBranches(schema).flatMap(
|
|
400
|
+
(branch) => schemaAlternatives(branch, depth + 1)
|
|
401
|
+
);
|
|
402
|
+
return alternatives.slice(0, MAX_LITERAL_ALTERNATIVES);
|
|
403
|
+
}
|
|
404
|
+
function schemaEnumExpectation(schema) {
|
|
405
|
+
if ("const" in schema) {
|
|
406
|
+
const constant = schemaLiteral(schema.const);
|
|
407
|
+
return constant === void 0 ? void 0 : `expected ${constant}`;
|
|
408
|
+
}
|
|
409
|
+
if (!Array.isArray(schema.enum)) return void 0;
|
|
410
|
+
const values = schema.enum.slice(0, MAX_LITERAL_ALTERNATIVES).map(schemaLiteral);
|
|
411
|
+
if (values.some((value) => value === void 0)) return void 0;
|
|
412
|
+
const rendered = values;
|
|
413
|
+
if (rendered.length === 0) return void 0;
|
|
414
|
+
const omission = schema.enum.length > rendered.length ? ", \u2026" : "";
|
|
415
|
+
return `expected ${rendered.length === 1 ? rendered[0] : `one of ${rendered.join(", ")}`}${omission}`;
|
|
416
|
+
}
|
|
417
|
+
function schemaTypeExpectation(schema) {
|
|
418
|
+
const type = schemaType(schema);
|
|
419
|
+
if (type !== void 0) return `expected ${type}`;
|
|
420
|
+
const alternatives = schemaAlternatives(schema);
|
|
421
|
+
return alternatives.length === 0 ? void 0 : `expected one of: ${alternatives.join(", ")}`;
|
|
422
|
+
}
|
|
423
|
+
function schemaSizeExpectation(schema, direction) {
|
|
424
|
+
const type = schemaType(schema);
|
|
425
|
+
const keyword = direction === "small" ? type === "string" ? "minLength" : type === "array" ? "minItems" : "minimum" : type === "string" ? "maxLength" : type === "array" ? "maxItems" : "maximum";
|
|
426
|
+
const value = schemaNumber(schema[keyword]);
|
|
427
|
+
if (value === void 0) return void 0;
|
|
428
|
+
const label = type === "string" ? "string length" : type === "array" ? "array length" : type;
|
|
429
|
+
if (label === void 0) return void 0;
|
|
430
|
+
return `expected ${label} ${direction === "small" ? "at least" : "at most"} ${value}`;
|
|
431
|
+
}
|
|
432
|
+
function schemaExpectation(schema, issueCode) {
|
|
433
|
+
switch (issueCode) {
|
|
434
|
+
case "invalid_type":
|
|
435
|
+
return schemaTypeExpectation(schema) ?? "failed structural validation";
|
|
436
|
+
case "invalid_value":
|
|
437
|
+
return schemaEnumExpectation(schema) ?? schemaTypeExpectation(schema) ?? "failed structural validation";
|
|
438
|
+
case "too_small":
|
|
439
|
+
return schemaSizeExpectation(schema, "small") ?? schemaTypeExpectation(schema) ?? "failed structural validation";
|
|
440
|
+
case "too_big":
|
|
441
|
+
return schemaSizeExpectation(schema, "big") ?? schemaTypeExpectation(schema) ?? "failed structural validation";
|
|
442
|
+
case "not_multiple_of": {
|
|
443
|
+
const multipleOf = schemaNumber(schema.multipleOf);
|
|
444
|
+
return multipleOf === void 0 ? schemaTypeExpectation(schema) ?? "failed structural validation" : `expected a multiple of ${multipleOf}`;
|
|
445
|
+
}
|
|
446
|
+
case "invalid_format": {
|
|
447
|
+
const format = schema.format;
|
|
448
|
+
return typeof format === "string" && SAFE_STRING_FORMATS.has(format) ? `expected valid ${format} format` : schemaTypeExpectation(schema) ?? "failed structural validation";
|
|
449
|
+
}
|
|
450
|
+
case "invalid_union":
|
|
451
|
+
return schemaTypeExpectation(schema) ?? "failed structural validation";
|
|
452
|
+
default: {
|
|
453
|
+
const typeExpectation = schemaTypeExpectation(schema);
|
|
454
|
+
return typeExpectation === void 0 ? "failed structural validation" : `failed structural validation; ${typeExpectation}`;
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
}
|
|
174
458
|
var ToolStringLengthError = class extends Error {
|
|
175
459
|
constructor(path, actual, maximum) {
|
|
176
460
|
const renderedPath = path.reduce(
|
|
@@ -186,6 +470,42 @@ var ToolStringLengthError = class extends Error {
|
|
|
186
470
|
this.name = "ToolStringLengthError";
|
|
187
471
|
}
|
|
188
472
|
};
|
|
473
|
+
function toolArgumentModelMessage(error, providerVisibleSchema) {
|
|
474
|
+
try {
|
|
475
|
+
let lines;
|
|
476
|
+
let omitted = false;
|
|
477
|
+
if (error instanceof ZodError) {
|
|
478
|
+
if (error.issues.length === 0) return void 0;
|
|
479
|
+
lines = [];
|
|
480
|
+
for (const issue of error.issues.slice(0, MAX_MODEL_ARGUMENT_ISSUES)) {
|
|
481
|
+
const resolved = resolveSchemaPath(providerVisibleSchema, issue.path);
|
|
482
|
+
if (resolved === void 0) return void 0;
|
|
483
|
+
const path = jsonPointerPath(resolved.segments);
|
|
484
|
+
const expectation = schemaExpectation(resolved.node, issue.code);
|
|
485
|
+
lines.push(
|
|
486
|
+
truncateModelFeedback(`- ${path}: ${expectation}`, MAX_MODEL_ARGUMENT_LINE_LENGTH)
|
|
487
|
+
);
|
|
488
|
+
}
|
|
489
|
+
omitted = error.issues.length > lines.length;
|
|
490
|
+
} else if (error instanceof ToolStringLengthError) {
|
|
491
|
+
const resolved = resolveSchemaPath(providerVisibleSchema, error.path);
|
|
492
|
+
const maximum = schemaNumber(error.maximum);
|
|
493
|
+
if (resolved === void 0 || maximum === void 0) return void 0;
|
|
494
|
+
const path = jsonPointerPath(resolved.segments);
|
|
495
|
+
lines = [`- ${path}: maximum string length is ${maximum}`];
|
|
496
|
+
} else {
|
|
497
|
+
return void 0;
|
|
498
|
+
}
|
|
499
|
+
const header = "Error: Invalid tool arguments:";
|
|
500
|
+
const footer = "Correct the arguments and try again.";
|
|
501
|
+
const omission = omitted ? "Additional validation issues were omitted." : void 0;
|
|
502
|
+
const parts = [header, ...lines, ...omission ? [omission] : [], footer];
|
|
503
|
+
const message = parts.join("\n");
|
|
504
|
+
return message.length <= MAX_MODEL_ARGUMENT_MESSAGE_LENGTH ? message : void 0;
|
|
505
|
+
} catch {
|
|
506
|
+
return void 0;
|
|
507
|
+
}
|
|
508
|
+
}
|
|
189
509
|
function validateStringLengths(value, maxLen, path = []) {
|
|
190
510
|
if (typeof value === "string") {
|
|
191
511
|
if (value.length > maxLen) {
|
|
@@ -735,6 +1055,38 @@ function unsafeTransportError(surface, remediation, exposedData) {
|
|
|
735
1055
|
}
|
|
736
1056
|
|
|
737
1057
|
// src/providers/openai-compatible.ts
|
|
1058
|
+
var OPENROUTER_IMAGE_MODEL = "openai/gpt-4o-mini";
|
|
1059
|
+
function compatibleImageBase64(source) {
|
|
1060
|
+
return source.type === "base64" ? source.data : Buffer.from(source.data.buffer, source.data.byteOffset, source.data.byteLength).toString(
|
|
1061
|
+
"base64"
|
|
1062
|
+
);
|
|
1063
|
+
}
|
|
1064
|
+
function openRouterImageParts(parts) {
|
|
1065
|
+
const content = [];
|
|
1066
|
+
for (const part of parts) {
|
|
1067
|
+
if (part.type === "text") {
|
|
1068
|
+
content.push({ type: "text", text: part.text });
|
|
1069
|
+
continue;
|
|
1070
|
+
}
|
|
1071
|
+
const { source } = part;
|
|
1072
|
+
if (source.type === "provider-file") {
|
|
1073
|
+
throw new UnsupportedModelInputError({
|
|
1074
|
+
provider: "openrouter",
|
|
1075
|
+
model: OPENROUTER_IMAGE_MODEL,
|
|
1076
|
+
modality: "image",
|
|
1077
|
+
source: "provider-file"
|
|
1078
|
+
});
|
|
1079
|
+
}
|
|
1080
|
+
content.push({
|
|
1081
|
+
type: "image_url",
|
|
1082
|
+
image_url: {
|
|
1083
|
+
url: source.type === "url" ? source.url : `data:${source.mediaType};base64,${compatibleImageBase64(source)}`
|
|
1084
|
+
}
|
|
1085
|
+
});
|
|
1086
|
+
if (part.label) content.push({ type: "text", text: `[Image: ${part.label}]` });
|
|
1087
|
+
}
|
|
1088
|
+
return content;
|
|
1089
|
+
}
|
|
738
1090
|
function resolvePerModel(value, model, fallback) {
|
|
739
1091
|
if (value === void 0) return fallback;
|
|
740
1092
|
return typeof value === "function" ? value(model) : value;
|
|
@@ -850,6 +1202,41 @@ function normalizeMessageContent(content) {
|
|
|
850
1202
|
}
|
|
851
1203
|
var OpenAICompatibleProvider = class {
|
|
852
1204
|
name;
|
|
1205
|
+
inputCapabilities(model) {
|
|
1206
|
+
return this.name === "openrouter" && model === OPENROUTER_IMAGE_MODEL ? { image: { sources: ["url", "bytes", "base64"] } } : {};
|
|
1207
|
+
}
|
|
1208
|
+
validateInput(request) {
|
|
1209
|
+
const effectiveModel = typeof request.providerOptions?.model === "string" ? request.providerOptions.model : request.model;
|
|
1210
|
+
const fail = (source, feature) => {
|
|
1211
|
+
throw new UnsupportedModelInputError({
|
|
1212
|
+
provider: this.name,
|
|
1213
|
+
model: effectiveModel || request.model,
|
|
1214
|
+
modality: "image",
|
|
1215
|
+
...source ? { source } : {},
|
|
1216
|
+
...feature ? { feature } : {}
|
|
1217
|
+
});
|
|
1218
|
+
};
|
|
1219
|
+
if (this.name !== "openrouter" || effectiveModel !== OPENROUTER_IMAGE_MODEL) {
|
|
1220
|
+
fail(void 0, "image input for this model");
|
|
1221
|
+
}
|
|
1222
|
+
if (request.providerOptions && "messages" in request.providerOptions) {
|
|
1223
|
+
fail(void 0, "raw messages providerOptions");
|
|
1224
|
+
}
|
|
1225
|
+
if (request.hasTools) fail(void 0, "tools with image input");
|
|
1226
|
+
for (const message of request.history) {
|
|
1227
|
+
if (!Array.isArray(message.content)) continue;
|
|
1228
|
+
if (message.role !== "user") fail(void 0, "rich non-user history");
|
|
1229
|
+
if (message.content.some(
|
|
1230
|
+
(part) => part.type === "image" && part.source.type === "provider-file"
|
|
1231
|
+
)) {
|
|
1232
|
+
fail("provider-file");
|
|
1233
|
+
}
|
|
1234
|
+
}
|
|
1235
|
+
if (Array.isArray(request.input) && request.input.some((part) => part.type === "image" && part.source.type === "provider-file")) {
|
|
1236
|
+
fail("provider-file");
|
|
1237
|
+
}
|
|
1238
|
+
return { effectiveModel };
|
|
1239
|
+
}
|
|
853
1240
|
/** `json_schema` is honored natively when the profile's `supportsJsonSchema`
|
|
854
1241
|
* capability is true for this model (the default); otherwise the engine
|
|
855
1242
|
* downgrades the request to plain `json_object` (see `buildRequestBody`),
|
|
@@ -1023,7 +1410,7 @@ var OpenAICompatibleProvider = class {
|
|
|
1023
1410
|
formatMessage(msg, model) {
|
|
1024
1411
|
const out = {
|
|
1025
1412
|
role: this.profile.roleFor ? this.profile.roleFor(msg.role, model) : msg.role,
|
|
1026
|
-
content: msg.content
|
|
1413
|
+
content: Array.isArray(msg.content) && this.name === "openrouter" && model === OPENROUTER_IMAGE_MODEL ? openRouterImageParts(msg.content) : msg.content
|
|
1027
1414
|
};
|
|
1028
1415
|
if (msg.name && (this.profile.capabilities?.emitsMessageName ?? true)) out.name = msg.name;
|
|
1029
1416
|
if (msg.tool_calls) out.tool_calls = msg.tool_calls;
|
|
@@ -1770,8 +2157,88 @@ var OpenAIProvider = class extends OpenAICompatibleProvider {
|
|
|
1770
2157
|
};
|
|
1771
2158
|
|
|
1772
2159
|
// src/providers/openai-responses.ts
|
|
2160
|
+
var OPENAI_RESPONSES_IMAGE_MODELS = /* @__PURE__ */ new Set([
|
|
2161
|
+
"gpt-4o",
|
|
2162
|
+
"gpt-4o-2024-08-06",
|
|
2163
|
+
"gpt-4o-2024-11-20",
|
|
2164
|
+
"gpt-4o-mini",
|
|
2165
|
+
"gpt-4o-mini-2024-07-18"
|
|
2166
|
+
]);
|
|
2167
|
+
function base64FromSource(source) {
|
|
2168
|
+
return source.type === "base64" ? source.data : Buffer.from(source.data.buffer, source.data.byteOffset, source.data.byteLength).toString(
|
|
2169
|
+
"base64"
|
|
2170
|
+
);
|
|
2171
|
+
}
|
|
2172
|
+
function responseImageParts(parts) {
|
|
2173
|
+
const mapped = [];
|
|
2174
|
+
for (const part of parts) {
|
|
2175
|
+
if (part.type === "text") {
|
|
2176
|
+
mapped.push({ type: "input_text", text: part.text });
|
|
2177
|
+
continue;
|
|
2178
|
+
}
|
|
2179
|
+
const { source } = part;
|
|
2180
|
+
if (source.type === "provider-file") {
|
|
2181
|
+
if (source.provider !== "openai-responses") {
|
|
2182
|
+
throw new UnsupportedModelInputError({
|
|
2183
|
+
provider: "openai-responses",
|
|
2184
|
+
model: "unknown",
|
|
2185
|
+
modality: "image",
|
|
2186
|
+
source: "provider-file"
|
|
2187
|
+
});
|
|
2188
|
+
}
|
|
2189
|
+
mapped.push({ type: "input_image", file_id: source.reference });
|
|
2190
|
+
} else if (source.type === "url") {
|
|
2191
|
+
mapped.push({ type: "input_image", image_url: source.url });
|
|
2192
|
+
} else {
|
|
2193
|
+
mapped.push({
|
|
2194
|
+
type: "input_image",
|
|
2195
|
+
image_url: `data:${source.mediaType};base64,${base64FromSource(source)}`
|
|
2196
|
+
});
|
|
2197
|
+
}
|
|
2198
|
+
if (part.label) mapped.push({ type: "input_text", text: `[Image: ${part.label}]` });
|
|
2199
|
+
}
|
|
2200
|
+
return mapped;
|
|
2201
|
+
}
|
|
1773
2202
|
var OpenAIResponsesProvider = class {
|
|
1774
2203
|
name = "openai-responses";
|
|
2204
|
+
inputCapabilities(model) {
|
|
2205
|
+
return OPENAI_RESPONSES_IMAGE_MODELS.has(model) ? { image: { sources: ["url", "bytes", "base64", "provider-file"] } } : {};
|
|
2206
|
+
}
|
|
2207
|
+
validateInput(request) {
|
|
2208
|
+
const effectiveModel = typeof request.providerOptions?.model === "string" ? request.providerOptions.model : request.model;
|
|
2209
|
+
const fail = (source, feature) => {
|
|
2210
|
+
throw new UnsupportedModelInputError({
|
|
2211
|
+
provider: this.name,
|
|
2212
|
+
model: effectiveModel || request.model,
|
|
2213
|
+
modality: "image",
|
|
2214
|
+
...source ? { source } : {},
|
|
2215
|
+
...feature ? { feature } : {}
|
|
2216
|
+
});
|
|
2217
|
+
};
|
|
2218
|
+
if (!OPENAI_RESPONSES_IMAGE_MODELS.has(effectiveModel)) {
|
|
2219
|
+
fail(void 0, "image input for this model");
|
|
2220
|
+
}
|
|
2221
|
+
if (request.providerOptions && "input" in request.providerOptions) {
|
|
2222
|
+
fail(void 0, "raw input-container providerOptions");
|
|
2223
|
+
}
|
|
2224
|
+
for (const message of request.history) {
|
|
2225
|
+
if (!Array.isArray(message.content)) continue;
|
|
2226
|
+
if (message.role !== "user") fail(void 0, "rich non-user history");
|
|
2227
|
+
for (const part of message.content) {
|
|
2228
|
+
if (part.type === "image" && part.source.type === "provider-file" && part.source.provider !== this.name) {
|
|
2229
|
+
fail("provider-file");
|
|
2230
|
+
}
|
|
2231
|
+
}
|
|
2232
|
+
}
|
|
2233
|
+
if (Array.isArray(request.input)) {
|
|
2234
|
+
for (const part of request.input) {
|
|
2235
|
+
if (part.type === "image" && part.source.type === "provider-file" && part.source.provider !== this.name) {
|
|
2236
|
+
fail("provider-file");
|
|
2237
|
+
}
|
|
2238
|
+
}
|
|
2239
|
+
}
|
|
2240
|
+
return { effectiveModel };
|
|
2241
|
+
}
|
|
1775
2242
|
/** The Responses API honors `json_schema` (structured outputs) natively. */
|
|
1776
2243
|
nativeStructuredOutputSupport() {
|
|
1777
2244
|
return "schema";
|
|
@@ -1936,11 +2403,12 @@ var OpenAIResponsesProvider = class {
|
|
|
1936
2403
|
buildInput(messages) {
|
|
1937
2404
|
const input = [];
|
|
1938
2405
|
for (const msg of messages) {
|
|
2406
|
+
const text = typeof msg.content === "string" ? msg.content : "";
|
|
1939
2407
|
if (msg.role === "tool") {
|
|
1940
2408
|
input.push({
|
|
1941
2409
|
type: "function_call_output",
|
|
1942
2410
|
call_id: msg.tool_call_id ?? "",
|
|
1943
|
-
output:
|
|
2411
|
+
output: text
|
|
1944
2412
|
});
|
|
1945
2413
|
} else if (msg.role === "assistant" && msg.tool_calls && msg.tool_calls.length > 0) {
|
|
1946
2414
|
const reasoningItems = msg.providerMetadata?.openaiReasoningItems;
|
|
@@ -1949,8 +2417,8 @@ var OpenAIResponsesProvider = class {
|
|
|
1949
2417
|
input.push(item);
|
|
1950
2418
|
}
|
|
1951
2419
|
}
|
|
1952
|
-
if (
|
|
1953
|
-
input.push({ type: "message", role: "assistant", content:
|
|
2420
|
+
if (text) {
|
|
2421
|
+
input.push({ type: "message", role: "assistant", content: text });
|
|
1954
2422
|
}
|
|
1955
2423
|
for (const tc of msg.tool_calls) {
|
|
1956
2424
|
input.push({
|
|
@@ -1970,7 +2438,7 @@ var OpenAIResponsesProvider = class {
|
|
|
1970
2438
|
input.push({
|
|
1971
2439
|
type: "message",
|
|
1972
2440
|
role: msg.role,
|
|
1973
|
-
content: msg.content
|
|
2441
|
+
content: typeof msg.content === "string" ? msg.content : responseImageParts(msg.content)
|
|
1974
2442
|
});
|
|
1975
2443
|
}
|
|
1976
2444
|
}
|
|
@@ -2037,7 +2505,7 @@ var OpenAIResponsesProvider = class {
|
|
|
2037
2505
|
cached_tokens: json.usage.input_tokens_details?.cached_tokens,
|
|
2038
2506
|
cache_write_tokens: json.usage.input_tokens_details?.cache_write_tokens
|
|
2039
2507
|
} : void 0;
|
|
2040
|
-
const cost = usage ? estimateDirectOpenAICost(json.model ?? model, usage, {
|
|
2508
|
+
const cost = usage && !this.requestContainsImages(request) ? estimateDirectOpenAICost(json.model ?? model, usage, {
|
|
2041
2509
|
baseUrl: this.baseUrl,
|
|
2042
2510
|
request,
|
|
2043
2511
|
response: json
|
|
@@ -2052,6 +2520,16 @@ var OpenAIResponsesProvider = class {
|
|
|
2052
2520
|
providerMetadata
|
|
2053
2521
|
};
|
|
2054
2522
|
}
|
|
2523
|
+
/** Responses usage does not report a media-inclusive price, so rich calls stay unpriced. */
|
|
2524
|
+
requestContainsImages(request) {
|
|
2525
|
+
return Array.isArray(request?.input) && request.input.some((item) => {
|
|
2526
|
+
if (!item || typeof item !== "object") return false;
|
|
2527
|
+
const content = item.content;
|
|
2528
|
+
return Array.isArray(content) && content.some(
|
|
2529
|
+
(part) => part !== null && typeof part === "object" && part.type === "input_image"
|
|
2530
|
+
);
|
|
2531
|
+
});
|
|
2532
|
+
}
|
|
2055
2533
|
// ---------------------------------------------------------------------------
|
|
2056
2534
|
// Internal: parse SSE stream
|
|
2057
2535
|
// ---------------------------------------------------------------------------
|
|
@@ -2139,7 +2617,7 @@ var OpenAIResponsesProvider = class {
|
|
|
2139
2617
|
return {
|
|
2140
2618
|
type: "done",
|
|
2141
2619
|
usage,
|
|
2142
|
-
cost: usage ? estimateDirectOpenAICost(response?.model ?? model, usage, {
|
|
2620
|
+
cost: usage && !this.requestContainsImages(request) ? estimateDirectOpenAICost(response?.model ?? model, usage, {
|
|
2143
2621
|
baseUrl: this.baseUrl,
|
|
2144
2622
|
request,
|
|
2145
2623
|
response
|
|
@@ -2176,7 +2654,51 @@ var OpenAIResponsesProvider = class {
|
|
|
2176
2654
|
};
|
|
2177
2655
|
|
|
2178
2656
|
// src/providers/anthropic.ts
|
|
2657
|
+
function anthropicBase64(source) {
|
|
2658
|
+
return source.type === "base64" ? source.data : Buffer.from(source.data.buffer, source.data.byteOffset, source.data.byteLength).toString(
|
|
2659
|
+
"base64"
|
|
2660
|
+
);
|
|
2661
|
+
}
|
|
2662
|
+
function anthropicImageBlocks(parts) {
|
|
2663
|
+
const blocks = [];
|
|
2664
|
+
for (const part of parts) {
|
|
2665
|
+
if (part.type === "text") {
|
|
2666
|
+
blocks.push({ type: "text", text: part.text });
|
|
2667
|
+
continue;
|
|
2668
|
+
}
|
|
2669
|
+
const { source } = part;
|
|
2670
|
+
if (source.type === "provider-file") {
|
|
2671
|
+
if (source.provider !== "anthropic") {
|
|
2672
|
+
throw new UnsupportedModelInputError({
|
|
2673
|
+
provider: "anthropic",
|
|
2674
|
+
model: "unknown",
|
|
2675
|
+
modality: "image",
|
|
2676
|
+
source: "provider-file"
|
|
2677
|
+
});
|
|
2678
|
+
}
|
|
2679
|
+
blocks.push({ type: "image", source: { type: "file", file_id: source.reference } });
|
|
2680
|
+
} else if (source.type === "url") {
|
|
2681
|
+
blocks.push({ type: "image", source: { type: "url", url: source.url } });
|
|
2682
|
+
} else {
|
|
2683
|
+
blocks.push({
|
|
2684
|
+
type: "image",
|
|
2685
|
+
source: { type: "base64", media_type: source.mediaType, data: anthropicBase64(source) }
|
|
2686
|
+
});
|
|
2687
|
+
}
|
|
2688
|
+
if (part.label) blocks.push({ type: "text", text: `[Image: ${part.label}]` });
|
|
2689
|
+
}
|
|
2690
|
+
return blocks;
|
|
2691
|
+
}
|
|
2179
2692
|
var ANTHROPIC_API_VERSION = "2023-06-01";
|
|
2693
|
+
var ANTHROPIC_FILES_BETA = "files-api-2025-04-14";
|
|
2694
|
+
var ANTHROPIC_IMAGE_MODELS = /* @__PURE__ */ new Set(["claude-sonnet-4-5", "claude-opus-4-8"]);
|
|
2695
|
+
function hasAnthropicProviderFileImage(messages) {
|
|
2696
|
+
return messages.some(
|
|
2697
|
+
(message) => Array.isArray(message.content) && message.content.some(
|
|
2698
|
+
(part) => part.type === "image" && part.source.type === "provider-file" && part.source.provider === "anthropic"
|
|
2699
|
+
)
|
|
2700
|
+
);
|
|
2701
|
+
}
|
|
2180
2702
|
var CLAUDE_CAPABILITIES = {
|
|
2181
2703
|
"claude-fable-5": {
|
|
2182
2704
|
thinking: "adaptive-always-on",
|
|
@@ -2437,6 +2959,11 @@ function isEffortAtOrBelow(effort, ceiling) {
|
|
|
2437
2959
|
function pricingContextFromBody(body) {
|
|
2438
2960
|
return {
|
|
2439
2961
|
model: typeof body.model === "string" ? body.model : void 0,
|
|
2962
|
+
hasRichInput: Array.isArray(body.messages) && body.messages.some(
|
|
2963
|
+
(message) => message !== null && typeof message === "object" && Array.isArray(message.content) && message.content.some(
|
|
2964
|
+
(part) => part !== null && typeof part === "object" && part.type === "image"
|
|
2965
|
+
)
|
|
2966
|
+
),
|
|
2440
2967
|
unpricedModifier: isAnthropicModifier("inference_geo", body.inference_geo) || isAnthropicModifier("speed", body.speed) || hasAnthropicServerTool(body.tools) || body.fallbacks !== void 0
|
|
2441
2968
|
};
|
|
2442
2969
|
}
|
|
@@ -2531,6 +3058,44 @@ function mergeAnthropicUsage(initial, update) {
|
|
|
2531
3058
|
}
|
|
2532
3059
|
var AnthropicProvider = class {
|
|
2533
3060
|
name = "anthropic";
|
|
3061
|
+
inputCapabilities(model) {
|
|
3062
|
+
return ANTHROPIC_IMAGE_MODELS.has(model) ? { image: { sources: ["url", "bytes", "base64", "provider-file"] } } : {};
|
|
3063
|
+
}
|
|
3064
|
+
validateInput(request) {
|
|
3065
|
+
const effectiveModel = typeof request.providerOptions?.model === "string" ? request.providerOptions.model : request.model;
|
|
3066
|
+
const fail = (source, feature) => {
|
|
3067
|
+
throw new UnsupportedModelInputError({
|
|
3068
|
+
provider: this.name,
|
|
3069
|
+
model: effectiveModel || request.model,
|
|
3070
|
+
modality: "image",
|
|
3071
|
+
...source ? { source } : {},
|
|
3072
|
+
...feature ? { feature } : {}
|
|
3073
|
+
});
|
|
3074
|
+
};
|
|
3075
|
+
if (!ANTHROPIC_IMAGE_MODELS.has(effectiveModel)) {
|
|
3076
|
+
fail(void 0, "image input for this model");
|
|
3077
|
+
}
|
|
3078
|
+
if (request.providerOptions && "messages" in request.providerOptions) {
|
|
3079
|
+
fail(void 0, "raw messages providerOptions");
|
|
3080
|
+
}
|
|
3081
|
+
for (const message of request.history) {
|
|
3082
|
+
if (!Array.isArray(message.content)) continue;
|
|
3083
|
+
if (message.role !== "user") fail(void 0, "rich non-user history");
|
|
3084
|
+
for (const part of message.content) {
|
|
3085
|
+
if (part.type === "image" && part.source.type === "provider-file" && part.source.provider !== this.name) {
|
|
3086
|
+
fail("provider-file");
|
|
3087
|
+
}
|
|
3088
|
+
}
|
|
3089
|
+
}
|
|
3090
|
+
if (Array.isArray(request.input)) {
|
|
3091
|
+
for (const part of request.input) {
|
|
3092
|
+
if (part.type === "image" && part.source.type === "provider-file" && part.source.provider !== this.name) {
|
|
3093
|
+
fail("provider-file");
|
|
3094
|
+
}
|
|
3095
|
+
}
|
|
3096
|
+
}
|
|
3097
|
+
return { effectiveModel };
|
|
3098
|
+
}
|
|
2534
3099
|
/** Anthropic ignores native `json_schema` structurally — Axl uses a system
|
|
2535
3100
|
* prompt JSON instruction + client-side Zod validation (see buildRequest). */
|
|
2536
3101
|
nativeStructuredOutputSupport() {
|
|
@@ -2568,7 +3133,10 @@ var AnthropicProvider = class {
|
|
|
2568
3133
|
// chat - non-streaming completion
|
|
2569
3134
|
// ---------------------------------------------------------------------------
|
|
2570
3135
|
async chat(messages, options) {
|
|
2571
|
-
const headers = this.buildHeaders(
|
|
3136
|
+
const headers = this.buildHeaders(
|
|
3137
|
+
await this.resolveKey(),
|
|
3138
|
+
hasAnthropicProviderFileImage(messages)
|
|
3139
|
+
);
|
|
2572
3140
|
const body = this.buildRequestBody(messages, options, false);
|
|
2573
3141
|
const pricingContext = pricingContextFromBody(body);
|
|
2574
3142
|
const res = await fetchWithRetry(
|
|
@@ -2599,7 +3167,10 @@ var AnthropicProvider = class {
|
|
|
2599
3167
|
// stream - SSE streaming completion
|
|
2600
3168
|
// ---------------------------------------------------------------------------
|
|
2601
3169
|
async *stream(messages, options) {
|
|
2602
|
-
const headers = this.buildHeaders(
|
|
3170
|
+
const headers = this.buildHeaders(
|
|
3171
|
+
await this.resolveKey(),
|
|
3172
|
+
hasAnthropicProviderFileImage(messages)
|
|
3173
|
+
);
|
|
2603
3174
|
const body = this.buildRequestBody(messages, options, true);
|
|
2604
3175
|
const pricingContext = pricingContextFromBody(body);
|
|
2605
3176
|
const res = await fetchWithRetry(
|
|
@@ -2631,11 +3202,12 @@ var AnthropicProvider = class {
|
|
|
2631
3202
|
// ---------------------------------------------------------------------------
|
|
2632
3203
|
// Internal: request building
|
|
2633
3204
|
// ---------------------------------------------------------------------------
|
|
2634
|
-
buildHeaders(apiKey) {
|
|
3205
|
+
buildHeaders(apiKey, requiresFilesBeta = false) {
|
|
2635
3206
|
return {
|
|
2636
3207
|
"Content-Type": "application/json",
|
|
2637
3208
|
"x-api-key": apiKey,
|
|
2638
|
-
"anthropic-version": ANTHROPIC_API_VERSION
|
|
3209
|
+
"anthropic-version": ANTHROPIC_API_VERSION,
|
|
3210
|
+
...requiresFilesBeta ? { "anthropic-beta": ANTHROPIC_FILES_BETA } : {}
|
|
2639
3211
|
};
|
|
2640
3212
|
}
|
|
2641
3213
|
/** Extract a human-readable message from an API error response body. */
|
|
@@ -2705,12 +3277,13 @@ ${jsonInstruction}` : jsonInstruction;
|
|
|
2705
3277
|
mapMessages(messages) {
|
|
2706
3278
|
const result = [];
|
|
2707
3279
|
for (const msg of messages) {
|
|
3280
|
+
const text = typeof msg.content === "string" ? msg.content : "";
|
|
2708
3281
|
if (msg.role === "assistant") {
|
|
2709
3282
|
const replayedThinking = this.getReplayedThinkingBlocks(msg);
|
|
2710
3283
|
if (msg.tool_calls && msg.tool_calls.length > 0) {
|
|
2711
3284
|
const content = [...replayedThinking];
|
|
2712
|
-
if (
|
|
2713
|
-
content.push({ type: "text", text
|
|
3285
|
+
if (text) {
|
|
3286
|
+
content.push({ type: "text", text });
|
|
2714
3287
|
}
|
|
2715
3288
|
for (const tc of msg.tool_calls) {
|
|
2716
3289
|
let parsedArgs;
|
|
@@ -2729,10 +3302,10 @@ ${jsonInstruction}` : jsonInstruction;
|
|
|
2729
3302
|
result.push({ role: "assistant", content });
|
|
2730
3303
|
} else if (replayedThinking.length > 0) {
|
|
2731
3304
|
const content = [...replayedThinking];
|
|
2732
|
-
if (
|
|
3305
|
+
if (text) content.push({ type: "text", text });
|
|
2733
3306
|
result.push({ role: "assistant", content });
|
|
2734
3307
|
} else {
|
|
2735
|
-
result.push({ role: "assistant", content:
|
|
3308
|
+
result.push({ role: "assistant", content: text });
|
|
2736
3309
|
}
|
|
2737
3310
|
} else if (msg.role === "tool") {
|
|
2738
3311
|
result.push({
|
|
@@ -2741,12 +3314,15 @@ ${jsonInstruction}` : jsonInstruction;
|
|
|
2741
3314
|
{
|
|
2742
3315
|
type: "tool_result",
|
|
2743
3316
|
tool_use_id: msg.tool_call_id,
|
|
2744
|
-
content:
|
|
3317
|
+
content: text
|
|
2745
3318
|
}
|
|
2746
3319
|
]
|
|
2747
3320
|
});
|
|
2748
3321
|
} else if (msg.role === "user") {
|
|
2749
|
-
result.push({
|
|
3322
|
+
result.push({
|
|
3323
|
+
role: "user",
|
|
3324
|
+
content: typeof msg.content === "string" ? msg.content : anthropicImageBlocks(msg.content)
|
|
3325
|
+
});
|
|
2750
3326
|
}
|
|
2751
3327
|
}
|
|
2752
3328
|
return this.mergeConsecutiveRoles(result);
|
|
@@ -2851,7 +3427,7 @@ ${jsonInstruction}` : jsonInstruction;
|
|
|
2851
3427
|
"Anthropic fallback responses cannot be continued safely; retry without fallbacks."
|
|
2852
3428
|
);
|
|
2853
3429
|
}
|
|
2854
|
-
const cost = normalized && !pricingContext.unpricedModifier && !isModifiedAnthropicResponse(json, effectiveModel) && !hasFallbackBoundary && !refused ? estimateAnthropicCost(effectiveModel, normalized.pricingUsage) : void 0;
|
|
3430
|
+
const cost = normalized && !pricingContext.hasRichInput && !pricingContext.unpricedModifier && !isModifiedAnthropicResponse(json, effectiveModel) && !hasFallbackBoundary && !refused ? estimateAnthropicCost(effectiveModel, normalized.pricingUsage) : void 0;
|
|
2855
3431
|
return {
|
|
2856
3432
|
content,
|
|
2857
3433
|
thinking_content: thinkingContent || void 0,
|
|
@@ -2888,7 +3464,7 @@ ${jsonInstruction}` : jsonInstruction;
|
|
|
2888
3464
|
const doneChunk = () => ({
|
|
2889
3465
|
type: "done",
|
|
2890
3466
|
usage,
|
|
2891
|
-
cost: pricingUsage && !unpricedModifier && !hasFallbackBoundary && !refused ? estimateAnthropicCost(effectiveModel, pricingUsage) : void 0,
|
|
3467
|
+
cost: pricingUsage && !pricingContext.hasRichInput && !unpricedModifier && !hasFallbackBoundary && !refused ? estimateAnthropicCost(effectiveModel, pricingUsage) : void 0,
|
|
2892
3468
|
providerMetadata: thinkingBlocks.length > 0 && !hasFallbackBoundary && !hasFallbackIterationSignal ? { anthropicThinkingBlocks: thinkingBlocks } : void 0
|
|
2893
3469
|
});
|
|
2894
3470
|
try {
|
|
@@ -3036,6 +3612,60 @@ function lastFallbackIndex(blocks) {
|
|
|
3036
3612
|
}
|
|
3037
3613
|
|
|
3038
3614
|
// src/providers/gemini.ts
|
|
3615
|
+
function hasRichGeminiMessages(messages) {
|
|
3616
|
+
return messages.some((message) => Array.isArray(message.content));
|
|
3617
|
+
}
|
|
3618
|
+
function geminiImageBase64(source) {
|
|
3619
|
+
return source.type === "base64" ? source.data : Buffer.from(source.data.buffer, source.data.byteOffset, source.data.byteLength).toString(
|
|
3620
|
+
"base64"
|
|
3621
|
+
);
|
|
3622
|
+
}
|
|
3623
|
+
function geminiInteractionContent(parts, model) {
|
|
3624
|
+
const content = [];
|
|
3625
|
+
for (const part of parts) {
|
|
3626
|
+
if (part.type === "text") {
|
|
3627
|
+
content.push({ type: "text", text: part.text });
|
|
3628
|
+
continue;
|
|
3629
|
+
}
|
|
3630
|
+
const { source } = part;
|
|
3631
|
+
if (source.type === "provider-file") {
|
|
3632
|
+
if (source.provider !== "google") {
|
|
3633
|
+
throw new UnsupportedModelInputError({
|
|
3634
|
+
provider: "google",
|
|
3635
|
+
model,
|
|
3636
|
+
modality: "image",
|
|
3637
|
+
source: "provider-file"
|
|
3638
|
+
});
|
|
3639
|
+
}
|
|
3640
|
+
if (!source.mediaType) {
|
|
3641
|
+
throw new UnsupportedModelInputError({
|
|
3642
|
+
provider: "google",
|
|
3643
|
+
model,
|
|
3644
|
+
modality: "image",
|
|
3645
|
+
source: "provider-file",
|
|
3646
|
+
feature: "Interactions URI image mediaType"
|
|
3647
|
+
});
|
|
3648
|
+
}
|
|
3649
|
+
content.push({
|
|
3650
|
+
type: "image",
|
|
3651
|
+
uri: source.reference,
|
|
3652
|
+
mime_type: source.mediaType
|
|
3653
|
+
});
|
|
3654
|
+
} else if (source.type === "url") {
|
|
3655
|
+
throw new UnsupportedModelInputError({
|
|
3656
|
+
provider: "google",
|
|
3657
|
+
model,
|
|
3658
|
+
modality: "image",
|
|
3659
|
+
source: "url",
|
|
3660
|
+
feature: "direct URL image input; pass bytes/base64 or a Gemini provider-file"
|
|
3661
|
+
});
|
|
3662
|
+
} else {
|
|
3663
|
+
content.push({ type: "image", data: geminiImageBase64(source), mime_type: source.mediaType });
|
|
3664
|
+
}
|
|
3665
|
+
if (part.label) content.push({ type: "text", text: `[Image: ${part.label}]` });
|
|
3666
|
+
}
|
|
3667
|
+
return content;
|
|
3668
|
+
}
|
|
3039
3669
|
var GEMINI_DISALLOWED_SCHEMA_KEYS = /* @__PURE__ */ new Set([
|
|
3040
3670
|
"additionalProperties",
|
|
3041
3671
|
"$schema",
|
|
@@ -3107,6 +3737,7 @@ var GEMINI_PRICING = {
|
|
|
3107
3737
|
"gemini-3.5-flash-lite": { input: 3e-7, cached: 3e-8, output: 25e-7 },
|
|
3108
3738
|
"gemini-3.6-flash": { input: 15e-7, cached: 15e-8, output: 75e-7 }
|
|
3109
3739
|
};
|
|
3740
|
+
var GEMINI_INTERACTIONS_IMAGE_MODELS = /* @__PURE__ */ new Set(["gemini-3.7-flash"]);
|
|
3110
3741
|
function isValidTokenCount2(value) {
|
|
3111
3742
|
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
|
|
3112
3743
|
}
|
|
@@ -3194,7 +3825,10 @@ var GEMINI_3X_MIN_THINKING_LEVEL = /* @__PURE__ */ new Map([
|
|
|
3194
3825
|
["gemini-3.1-flash-lite-preview", "minimal"],
|
|
3195
3826
|
["gemini-3.5-flash", "minimal"],
|
|
3196
3827
|
["gemini-3.5-flash-lite", "minimal"],
|
|
3197
|
-
["gemini-3.6-flash", "minimal"]
|
|
3828
|
+
["gemini-3.6-flash", "minimal"],
|
|
3829
|
+
// L4 live evidence: Interactions gemini-3.7-flash accepts low/medium/high,
|
|
3830
|
+
// but rejects minimal. Keep this exact entry isolated from legacy aliases.
|
|
3831
|
+
["gemini-3.7-flash", "low"]
|
|
3198
3832
|
]);
|
|
3199
3833
|
var GEMINI_MODELS_WITHOUT_PORTABLE_TEMPERATURE = /* @__PURE__ */ new Set([
|
|
3200
3834
|
"gemini-3.5-flash-lite",
|
|
@@ -3221,6 +3855,61 @@ function warnGemini3xEffortNone(model) {
|
|
|
3221
3855
|
}
|
|
3222
3856
|
var GeminiProvider = class {
|
|
3223
3857
|
name = "google";
|
|
3858
|
+
inputCapabilities(model) {
|
|
3859
|
+
return GEMINI_INTERACTIONS_IMAGE_MODELS.has(model) ? { image: { sources: ["bytes", "base64", "provider-file"] } } : {};
|
|
3860
|
+
}
|
|
3861
|
+
validateInput(request) {
|
|
3862
|
+
const effectiveModel = typeof request.providerOptions?.model === "string" ? request.providerOptions.model : request.model;
|
|
3863
|
+
const fail = (source, feature) => {
|
|
3864
|
+
throw new UnsupportedModelInputError({
|
|
3865
|
+
provider: this.name,
|
|
3866
|
+
model: effectiveModel || request.model,
|
|
3867
|
+
modality: "image",
|
|
3868
|
+
...source ? { source } : {},
|
|
3869
|
+
...feature ? { feature } : {}
|
|
3870
|
+
});
|
|
3871
|
+
};
|
|
3872
|
+
if (!GEMINI_INTERACTIONS_IMAGE_MODELS.has(effectiveModel)) {
|
|
3873
|
+
fail(void 0, "image input for this model");
|
|
3874
|
+
}
|
|
3875
|
+
if (request.providerOptions) {
|
|
3876
|
+
const forbidden = [
|
|
3877
|
+
"input",
|
|
3878
|
+
"contents",
|
|
3879
|
+
"messages",
|
|
3880
|
+
"previous_interaction_id",
|
|
3881
|
+
"background",
|
|
3882
|
+
"stream",
|
|
3883
|
+
"store"
|
|
3884
|
+
].find((key) => key in request.providerOptions);
|
|
3885
|
+
if (forbidden) fail(void 0, `raw ${forbidden} providerOptions`);
|
|
3886
|
+
}
|
|
3887
|
+
for (const message of request.history) {
|
|
3888
|
+
if (!Array.isArray(message.content)) continue;
|
|
3889
|
+
if (message.role !== "user") fail(void 0, "rich non-user history");
|
|
3890
|
+
for (const part of message.content) {
|
|
3891
|
+
if (part.type !== "image") continue;
|
|
3892
|
+
if (part.source.type === "url")
|
|
3893
|
+
fail("url", "direct URL image input; pass bytes/base64 or a Gemini provider-file");
|
|
3894
|
+
if (part.source.type === "provider-file" && part.source.provider !== this.name)
|
|
3895
|
+
fail("provider-file");
|
|
3896
|
+
if (part.source.type === "provider-file" && !part.source.mediaType)
|
|
3897
|
+
fail("provider-file", "Interactions URI image mediaType");
|
|
3898
|
+
}
|
|
3899
|
+
}
|
|
3900
|
+
if (Array.isArray(request.input)) {
|
|
3901
|
+
for (const part of request.input) {
|
|
3902
|
+
if (part.type !== "image") continue;
|
|
3903
|
+
if (part.source.type === "url")
|
|
3904
|
+
fail("url", "direct URL image input; pass bytes/base64 or a Gemini provider-file");
|
|
3905
|
+
if (part.source.type === "provider-file" && part.source.provider !== this.name)
|
|
3906
|
+
fail("provider-file");
|
|
3907
|
+
if (part.source.type === "provider-file" && !part.source.mediaType)
|
|
3908
|
+
fail("provider-file", "Interactions URI image mediaType");
|
|
3909
|
+
}
|
|
3910
|
+
}
|
|
3911
|
+
return { effectiveModel };
|
|
3912
|
+
}
|
|
3224
3913
|
/** Gemini accepts a `responseSchema` but `sanitizeSchemaForGemini` strips
|
|
3225
3914
|
* keywords it doesn't support (`$ref`/`$defs`/`additionalProperties`/…), so a
|
|
3226
3915
|
* derived schema can lose constraints — lossy, not faithful. */
|
|
@@ -3259,6 +3948,7 @@ var GeminiProvider = class {
|
|
|
3259
3948
|
// chat - non-streaming completion
|
|
3260
3949
|
// ---------------------------------------------------------------------------
|
|
3261
3950
|
async chat(messages, options) {
|
|
3951
|
+
if (hasRichGeminiMessages(messages)) return this.chatInteraction(messages, options);
|
|
3262
3952
|
const body = this.buildRequestBody(messages, options);
|
|
3263
3953
|
const pricingContext = this.pricingContext(this.requestModel(options), body);
|
|
3264
3954
|
this.assertSafeGemini3Continuation(messages, pricingContext.model);
|
|
@@ -3291,6 +3981,10 @@ var GeminiProvider = class {
|
|
|
3291
3981
|
// stream - SSE streaming completion
|
|
3292
3982
|
// ---------------------------------------------------------------------------
|
|
3293
3983
|
async *stream(messages, options) {
|
|
3984
|
+
if (hasRichGeminiMessages(messages)) {
|
|
3985
|
+
yield* this.streamInteraction(messages, options);
|
|
3986
|
+
return;
|
|
3987
|
+
}
|
|
3294
3988
|
const body = this.buildRequestBody(messages, options);
|
|
3295
3989
|
const pricingContext = this.pricingContext(this.requestModel(options), body);
|
|
3296
3990
|
this.assertSafeGemini3Continuation(messages, pricingContext.model);
|
|
@@ -3322,50 +4016,383 @@ var GeminiProvider = class {
|
|
|
3322
4016
|
yield* this.parseSSEStream(res.body, pricingContext);
|
|
3323
4017
|
}
|
|
3324
4018
|
// ---------------------------------------------------------------------------
|
|
3325
|
-
//
|
|
4019
|
+
// Rich image requests use Gemini Interactions. It is deliberately stateless:
|
|
4020
|
+
// Axl sends the complete application-owned history and never asks Google to
|
|
4021
|
+
// retain an interaction or follow a previous_interaction_id.
|
|
3326
4022
|
// ---------------------------------------------------------------------------
|
|
3327
|
-
|
|
3328
|
-
|
|
3329
|
-
|
|
3330
|
-
|
|
3331
|
-
|
|
4023
|
+
async chatInteraction(messages, options) {
|
|
4024
|
+
this.assertSafeInteractionOptions(options);
|
|
4025
|
+
const body = this.buildInteractionRequestBody(messages, options, false);
|
|
4026
|
+
const headers = this.buildHeaders(await this.resolveKey());
|
|
4027
|
+
const res = await fetchWithRetry(
|
|
4028
|
+
`${this.baseUrl}/interactions`,
|
|
4029
|
+
{ method: "POST", headers, body: JSON.stringify(body), signal: options.signal },
|
|
4030
|
+
{ governor: this.governor, provider: this.name }
|
|
4031
|
+
);
|
|
4032
|
+
if (!res.ok) {
|
|
4033
|
+
const errorBody = await res.text();
|
|
4034
|
+
throw buildProviderError({
|
|
4035
|
+
provider: this.name,
|
|
4036
|
+
status: res.status,
|
|
4037
|
+
headers: res.headers,
|
|
4038
|
+
message: this.extractErrorMessage(errorBody, res.status),
|
|
4039
|
+
body: errorBody
|
|
4040
|
+
});
|
|
4041
|
+
}
|
|
4042
|
+
return this.parseInteractionResponse(await res.json());
|
|
3332
4043
|
}
|
|
3333
|
-
|
|
3334
|
-
|
|
3335
|
-
|
|
3336
|
-
|
|
3337
|
-
|
|
3338
|
-
}
|
|
3339
|
-
|
|
4044
|
+
async *streamInteraction(messages, options) {
|
|
4045
|
+
this.assertSafeInteractionOptions(options);
|
|
4046
|
+
const body = this.buildInteractionRequestBody(messages, options, true);
|
|
4047
|
+
const headers = this.buildHeaders(await this.resolveKey());
|
|
4048
|
+
const res = await fetchWithRetry(
|
|
4049
|
+
`${this.baseUrl}/interactions?alt=sse`,
|
|
4050
|
+
{ method: "POST", headers, body: JSON.stringify(body), signal: options.signal },
|
|
4051
|
+
{ governor: this.governor, provider: this.name }
|
|
4052
|
+
);
|
|
4053
|
+
if (!res.ok) {
|
|
4054
|
+
const errorBody = await res.text();
|
|
4055
|
+
throw buildProviderError({
|
|
4056
|
+
provider: this.name,
|
|
4057
|
+
status: res.status,
|
|
4058
|
+
headers: res.headers,
|
|
4059
|
+
message: this.extractErrorMessage(errorBody, res.status),
|
|
4060
|
+
body: errorBody
|
|
4061
|
+
});
|
|
3340
4062
|
}
|
|
3341
|
-
|
|
4063
|
+
if (!res.body) throw new Error("Gemini Interactions stream has no body");
|
|
4064
|
+
yield* this.parseInteractionSSEStream(res.body);
|
|
3342
4065
|
}
|
|
3343
|
-
|
|
3344
|
-
|
|
3345
|
-
|
|
3346
|
-
|
|
3347
|
-
|
|
3348
|
-
|
|
3349
|
-
|
|
3350
|
-
|
|
4066
|
+
buildInteractionRequestBody(messages, options, stream) {
|
|
4067
|
+
const model = this.requestModel(options);
|
|
4068
|
+
const systemInstruction = messages.filter((message) => message.role === "system").map((message) => typeof message.content === "string" ? message.content : "").filter(Boolean).join("\n\n");
|
|
4069
|
+
const body = {
|
|
4070
|
+
model,
|
|
4071
|
+
input: this.mapInteractionInput(
|
|
4072
|
+
messages.filter((message) => message.role !== "system"),
|
|
4073
|
+
model
|
|
4074
|
+
),
|
|
4075
|
+
stream
|
|
3351
4076
|
};
|
|
3352
|
-
|
|
3353
|
-
|
|
3354
|
-
|
|
3355
|
-
|
|
3356
|
-
|
|
3357
|
-
|
|
3358
|
-
|
|
3359
|
-
|
|
3360
|
-
|
|
3361
|
-
|
|
3362
|
-
|
|
4077
|
+
if (systemInstruction) body.system_instruction = systemInstruction;
|
|
4078
|
+
if (options.tools?.length) {
|
|
4079
|
+
body.tools = options.tools.map((tool2) => ({
|
|
4080
|
+
type: "function",
|
|
4081
|
+
name: tool2.function.name,
|
|
4082
|
+
description: tool2.function.description,
|
|
4083
|
+
parameters: sanitizeSchemaForGemini(tool2.function.parameters)
|
|
4084
|
+
}));
|
|
4085
|
+
}
|
|
4086
|
+
const generationConfig = {};
|
|
4087
|
+
if (options.maxTokens !== void 0) generationConfig.max_output_tokens = options.maxTokens;
|
|
4088
|
+
if (options.stop) generationConfig.stop_sequences = options.stop;
|
|
4089
|
+
if (options.toolChoice !== void 0) {
|
|
4090
|
+
generationConfig.tool_choice = typeof options.toolChoice === "string" ? options.toolChoice === "required" ? "any" : options.toolChoice : { allowed_tools: { mode: "any", tools: [options.toolChoice.function.name] } };
|
|
4091
|
+
}
|
|
4092
|
+
const thinking = resolveThinkingOptions(options);
|
|
4093
|
+
if (thinking.thinkingDisabled) {
|
|
4094
|
+
generationConfig.thinking_level = isGemini3x(model) ? minThinkingLevel(model) : "minimal";
|
|
4095
|
+
} else if (thinking.hasBudgetOverride) {
|
|
4096
|
+
generationConfig.thinking_level = budgetToThinkingLevel(thinking.thinkingBudget);
|
|
4097
|
+
} else if (thinking.activeEffort) {
|
|
4098
|
+
generationConfig.thinking_level = isGemini3x(model) ? THINKING_LEVELS[thinking.activeEffort] ?? "medium" : budgetToThinkingLevel(THINKING_BUDGETS2[thinking.activeEffort] ?? 5e3);
|
|
4099
|
+
}
|
|
4100
|
+
if (thinking.includeThoughts) generationConfig.thinking_summaries = "auto";
|
|
4101
|
+
if (Object.keys(generationConfig).length) body.generation_config = generationConfig;
|
|
4102
|
+
if (options.responseFormat?.type && options.responseFormat.type !== "text") {
|
|
4103
|
+
body.response_format = {
|
|
4104
|
+
type: "text",
|
|
4105
|
+
mime_type: "application/json",
|
|
4106
|
+
...options.responseFormat.type === "json_schema" ? { schema: sanitizeSchemaForGemini(options.responseFormat.json_schema.schema) } : {}
|
|
4107
|
+
};
|
|
4108
|
+
}
|
|
4109
|
+
if (options.providerOptions) {
|
|
4110
|
+
const overrides = { ...options.providerOptions };
|
|
4111
|
+
delete overrides.model;
|
|
4112
|
+
delete overrides.input;
|
|
4113
|
+
delete overrides.contents;
|
|
4114
|
+
delete overrides.messages;
|
|
4115
|
+
delete overrides.previous_interaction_id;
|
|
4116
|
+
delete overrides.background;
|
|
4117
|
+
delete overrides.stream;
|
|
4118
|
+
delete overrides.store;
|
|
4119
|
+
const rawGenerationConfig = overrides.generation_config;
|
|
4120
|
+
if (rawGenerationConfig && typeof rawGenerationConfig === "object" && !Array.isArray(rawGenerationConfig)) {
|
|
4121
|
+
const generationConfigOverrides = { ...rawGenerationConfig };
|
|
4122
|
+
delete generationConfigOverrides.temperature;
|
|
4123
|
+
if (Object.keys(generationConfigOverrides).length)
|
|
4124
|
+
overrides.generation_config = generationConfigOverrides;
|
|
4125
|
+
else delete overrides.generation_config;
|
|
3363
4126
|
}
|
|
4127
|
+
Object.assign(body, overrides);
|
|
3364
4128
|
}
|
|
3365
|
-
|
|
3366
|
-
|
|
3367
|
-
|
|
3368
|
-
|
|
4129
|
+
body.stream = stream;
|
|
4130
|
+
body.store = false;
|
|
4131
|
+
return body;
|
|
4132
|
+
}
|
|
4133
|
+
assertSafeInteractionOptions(options) {
|
|
4134
|
+
const model = this.requestModel(options);
|
|
4135
|
+
const fail = (feature) => {
|
|
4136
|
+
throw new UnsupportedModelInputError({
|
|
4137
|
+
provider: this.name,
|
|
4138
|
+
model,
|
|
4139
|
+
modality: "image",
|
|
4140
|
+
feature
|
|
4141
|
+
});
|
|
4142
|
+
};
|
|
4143
|
+
if (!GEMINI_INTERACTIONS_IMAGE_MODELS.has(model)) fail("image input for this model");
|
|
4144
|
+
const forbidden = [
|
|
4145
|
+
"input",
|
|
4146
|
+
"contents",
|
|
4147
|
+
"messages",
|
|
4148
|
+
"previous_interaction_id",
|
|
4149
|
+
"background",
|
|
4150
|
+
"stream",
|
|
4151
|
+
"store"
|
|
4152
|
+
].find((key) => key in (options.providerOptions ?? {}));
|
|
4153
|
+
if (forbidden) fail(`raw ${forbidden} providerOptions`);
|
|
4154
|
+
}
|
|
4155
|
+
mapInteractionInput(messages, model) {
|
|
4156
|
+
const steps = [];
|
|
4157
|
+
for (const message of messages) {
|
|
4158
|
+
if (message.role === "user") {
|
|
4159
|
+
steps.push({
|
|
4160
|
+
type: "user_input",
|
|
4161
|
+
content: Array.isArray(message.content) ? geminiInteractionContent(message.content, model) : [{ type: "text", text: message.content }]
|
|
4162
|
+
});
|
|
4163
|
+
} else if (message.role === "assistant") {
|
|
4164
|
+
const nativeSteps = message.providerMetadata?.geminiInteractionSteps;
|
|
4165
|
+
if (Array.isArray(nativeSteps) && nativeSteps.every(isGeminiInteractionStep)) {
|
|
4166
|
+
steps.push(...nativeSteps);
|
|
4167
|
+
continue;
|
|
4168
|
+
}
|
|
4169
|
+
if (typeof message.content === "string" && message.content) {
|
|
4170
|
+
steps.push({ type: "model_output", content: [{ type: "text", text: message.content }] });
|
|
4171
|
+
}
|
|
4172
|
+
for (const call of message.tool_calls ?? []) {
|
|
4173
|
+
steps.push({
|
|
4174
|
+
type: "function_call",
|
|
4175
|
+
id: call.id,
|
|
4176
|
+
name: call.function.name,
|
|
4177
|
+
arguments: safeJsonObject(call.function.arguments)
|
|
4178
|
+
});
|
|
4179
|
+
}
|
|
4180
|
+
} else if (message.role === "tool") {
|
|
4181
|
+
steps.push({
|
|
4182
|
+
type: "function_result",
|
|
4183
|
+
call_id: message.tool_call_id ?? "",
|
|
4184
|
+
result: [
|
|
4185
|
+
{ type: "text", text: typeof message.content === "string" ? message.content : "" }
|
|
4186
|
+
]
|
|
4187
|
+
});
|
|
4188
|
+
}
|
|
4189
|
+
}
|
|
4190
|
+
return steps;
|
|
4191
|
+
}
|
|
4192
|
+
parseInteractionResponse(json) {
|
|
4193
|
+
let content = "";
|
|
4194
|
+
let thinkingContent = "";
|
|
4195
|
+
const toolCalls = [];
|
|
4196
|
+
for (const step of json.steps ?? []) {
|
|
4197
|
+
if (step.type === "model_output") content += interactionText(step.content);
|
|
4198
|
+
else if (step.type === "thought") thinkingContent += interactionText(step.summary);
|
|
4199
|
+
else if (step.type === "function_call" && step.id && step.name) {
|
|
4200
|
+
toolCalls.push({
|
|
4201
|
+
id: step.id,
|
|
4202
|
+
type: "function",
|
|
4203
|
+
function: { name: step.name, arguments: JSON.stringify(step.arguments ?? {}) }
|
|
4204
|
+
});
|
|
4205
|
+
}
|
|
4206
|
+
}
|
|
4207
|
+
this.assertInteractionTerminalStatus(json.status, toolCalls.length);
|
|
4208
|
+
const usage = normalizeInteractionUsage(json.usage);
|
|
4209
|
+
return {
|
|
4210
|
+
content,
|
|
4211
|
+
thinking_content: thinkingContent || void 0,
|
|
4212
|
+
tool_calls: toolCalls.length ? toolCalls : void 0,
|
|
4213
|
+
usage,
|
|
4214
|
+
// Interactions returns modality usage but no authoritative monetary cost.
|
|
4215
|
+
cost: void 0,
|
|
4216
|
+
providerMetadata: json.steps?.length ? { geminiInteractionSteps: json.steps.filter(isGeminiInteractionStep) } : void 0
|
|
4217
|
+
};
|
|
4218
|
+
}
|
|
4219
|
+
async *parseInteractionSSEStream(body) {
|
|
4220
|
+
const reader = body.getReader();
|
|
4221
|
+
const decoder = new TextDecoder();
|
|
4222
|
+
let buffer = "";
|
|
4223
|
+
const steps = /* @__PURE__ */ new Map();
|
|
4224
|
+
const argumentBuffers = /* @__PURE__ */ new Map();
|
|
4225
|
+
const stopped = /* @__PURE__ */ new Set();
|
|
4226
|
+
let completed = false;
|
|
4227
|
+
try {
|
|
4228
|
+
while (true) {
|
|
4229
|
+
const { done, value } = await reader.read();
|
|
4230
|
+
if (done) break;
|
|
4231
|
+
buffer += decoder.decode(value, { stream: true });
|
|
4232
|
+
const lines = buffer.split("\n");
|
|
4233
|
+
buffer = lines.pop() ?? "";
|
|
4234
|
+
for (const line of lines) {
|
|
4235
|
+
const trimmed = line.trim();
|
|
4236
|
+
if (!trimmed.startsWith("data: ")) continue;
|
|
4237
|
+
if (trimmed.slice(6) === "[DONE]") continue;
|
|
4238
|
+
let event;
|
|
4239
|
+
try {
|
|
4240
|
+
event = JSON.parse(trimmed.slice(6));
|
|
4241
|
+
} catch {
|
|
4242
|
+
continue;
|
|
4243
|
+
}
|
|
4244
|
+
if (event.event_type === "error") {
|
|
4245
|
+
throw new Error(event.error?.message ?? "Gemini Interactions stream failed");
|
|
4246
|
+
}
|
|
4247
|
+
if (event.event_type === "step.start" && event.step) {
|
|
4248
|
+
const index = event.index;
|
|
4249
|
+
if (index === void 0 || steps.has(index)) {
|
|
4250
|
+
throw new Error("Gemini Interactions stream contained an invalid step.start index");
|
|
4251
|
+
}
|
|
4252
|
+
const step = cloneInteractionStep(event.step);
|
|
4253
|
+
steps.set(index, step);
|
|
4254
|
+
if (step.type === "model_output") {
|
|
4255
|
+
for (const text of interactionTextChunks(step.content)) {
|
|
4256
|
+
yield { type: "text_delta", content: text };
|
|
4257
|
+
}
|
|
4258
|
+
} else if (step.type === "thought") {
|
|
4259
|
+
for (const text of interactionTextChunks(step.summary)) {
|
|
4260
|
+
yield { type: "thinking_delta", content: text };
|
|
4261
|
+
}
|
|
4262
|
+
}
|
|
4263
|
+
if (step.type === "function_call" && step.id && step.name) {
|
|
4264
|
+
argumentBuffers.set(index, "");
|
|
4265
|
+
yield { type: "tool_call_delta", id: step.id, name: step.name };
|
|
4266
|
+
}
|
|
4267
|
+
} else if (event.event_type === "step.delta" && event.delta) {
|
|
4268
|
+
const delta = event.delta;
|
|
4269
|
+
const index = event.index;
|
|
4270
|
+
if (index === void 0 || stopped.has(index)) {
|
|
4271
|
+
throw new Error("Gemini Interactions stream delta arrived outside an active step");
|
|
4272
|
+
}
|
|
4273
|
+
const step = steps.get(index);
|
|
4274
|
+
if (!step)
|
|
4275
|
+
throw new Error("Gemini Interactions stream delta arrived before step.start");
|
|
4276
|
+
if (delta.type === "text" && typeof delta.text === "string" && step.type === "model_output") {
|
|
4277
|
+
appendInteractionContent(step, "content", { type: "text", text: delta.text });
|
|
4278
|
+
yield { type: "text_delta", content: delta.text };
|
|
4279
|
+
} else if (delta.type === "thought_summary" && step.type === "thought" && delta.content) {
|
|
4280
|
+
appendInteractionContent(step, "summary", delta.content);
|
|
4281
|
+
for (const text of interactionTextChunks([delta.content])) {
|
|
4282
|
+
yield { type: "thinking_delta", content: text };
|
|
4283
|
+
}
|
|
4284
|
+
} else if (delta.type === "thought_signature" && step.type === "thought" && typeof delta.signature === "string") {
|
|
4285
|
+
step.signature = delta.signature;
|
|
4286
|
+
} else if (delta.type === "arguments_delta" && step.type === "function_call" && typeof delta.arguments === "string") {
|
|
4287
|
+
argumentBuffers.set(index, (argumentBuffers.get(index) ?? "") + delta.arguments);
|
|
4288
|
+
if (step.id)
|
|
4289
|
+
yield { type: "tool_call_delta", id: step.id, arguments: delta.arguments };
|
|
4290
|
+
}
|
|
4291
|
+
} else if (event.event_type === "step.stop") {
|
|
4292
|
+
const index = event.index;
|
|
4293
|
+
const step = index === void 0 ? void 0 : steps.get(index);
|
|
4294
|
+
if (step === void 0 || stopped.has(index)) {
|
|
4295
|
+
throw new Error("Gemini Interactions stream contained an invalid step.stop index");
|
|
4296
|
+
}
|
|
4297
|
+
if (step.type === "function_call") {
|
|
4298
|
+
const rawArguments = argumentBuffers.get(index) ?? "";
|
|
4299
|
+
if (rawArguments) {
|
|
4300
|
+
try {
|
|
4301
|
+
const parsed = JSON.parse(rawArguments);
|
|
4302
|
+
if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
4303
|
+
throw new Error("not an object");
|
|
4304
|
+
}
|
|
4305
|
+
step.arguments = parsed;
|
|
4306
|
+
} catch {
|
|
4307
|
+
throw new Error(
|
|
4308
|
+
"Gemini Interactions function_call arguments were not valid JSON"
|
|
4309
|
+
);
|
|
4310
|
+
}
|
|
4311
|
+
}
|
|
4312
|
+
}
|
|
4313
|
+
stopped.add(index);
|
|
4314
|
+
} else if (event.event_type === "interaction.completed") {
|
|
4315
|
+
if (steps.size !== stopped.size) {
|
|
4316
|
+
throw new Error("Gemini Interactions stream completed before all steps stopped");
|
|
4317
|
+
}
|
|
4318
|
+
const toolCallCount = [...steps.values()].filter(
|
|
4319
|
+
(step) => step.type === "function_call" && !!step.id && !!step.name
|
|
4320
|
+
).length;
|
|
4321
|
+
this.assertInteractionTerminalStatus(event.interaction?.status, toolCallCount);
|
|
4322
|
+
completed = true;
|
|
4323
|
+
yield {
|
|
4324
|
+
type: "done",
|
|
4325
|
+
usage: normalizeInteractionUsage(event.interaction?.usage),
|
|
4326
|
+
providerMetadata: steps.size ? {
|
|
4327
|
+
geminiInteractionSteps: [...steps.entries()].sort(([a], [b]) => a - b).map(([, step]) => step)
|
|
4328
|
+
} : void 0
|
|
4329
|
+
};
|
|
4330
|
+
return;
|
|
4331
|
+
}
|
|
4332
|
+
}
|
|
4333
|
+
}
|
|
4334
|
+
if (!completed)
|
|
4335
|
+
throw new Error("Gemini Interactions stream ended before interaction.completed");
|
|
4336
|
+
} finally {
|
|
4337
|
+
reader.releaseLock();
|
|
4338
|
+
}
|
|
4339
|
+
}
|
|
4340
|
+
assertInteractionTerminalStatus(status, toolCallCount) {
|
|
4341
|
+
if (status === "completed") return;
|
|
4342
|
+
if (status === "requires_action" && toolCallCount > 0) return;
|
|
4343
|
+
const normalizedStatus = typeof status === "string" ? status : "unknown";
|
|
4344
|
+
throw new ProviderError({
|
|
4345
|
+
provider: this.name,
|
|
4346
|
+
status: 0,
|
|
4347
|
+
retryable: false,
|
|
4348
|
+
message: `Gemini Interactions ended with non-success status: ${normalizedStatus}`
|
|
4349
|
+
});
|
|
4350
|
+
}
|
|
4351
|
+
// ---------------------------------------------------------------------------
|
|
4352
|
+
// Internal: request building
|
|
4353
|
+
// ---------------------------------------------------------------------------
|
|
4354
|
+
buildHeaders(apiKey) {
|
|
4355
|
+
return {
|
|
4356
|
+
"Content-Type": "application/json",
|
|
4357
|
+
"x-goog-api-key": apiKey
|
|
4358
|
+
};
|
|
4359
|
+
}
|
|
4360
|
+
extractErrorMessage(body, status) {
|
|
4361
|
+
try {
|
|
4362
|
+
const json = JSON.parse(body);
|
|
4363
|
+
if (json.error?.message) {
|
|
4364
|
+
return `Gemini API error (${status}): ${json.error.message}`;
|
|
4365
|
+
}
|
|
4366
|
+
} catch {
|
|
4367
|
+
}
|
|
4368
|
+
return `Gemini API error (${status}): ${body}`;
|
|
4369
|
+
}
|
|
4370
|
+
pricingContext(fallbackModel, body) {
|
|
4371
|
+
return {
|
|
4372
|
+
// Gemini's model is normally selected in the URL. providerOptions is an
|
|
4373
|
+
// explicit raw escape hatch, so honor a post-merge model override for both
|
|
4374
|
+
// the URL and pricing rather than pricing a different request.
|
|
4375
|
+
model: typeof body.model === "string" ? body.model : fallbackModel,
|
|
4376
|
+
serviceTier: body.serviceTier,
|
|
4377
|
+
eligibleRequest: this.baseUrl === "https://generativelanguage.googleapis.com/v1beta" && this.isTextOnlyClientRequest(body)
|
|
4378
|
+
};
|
|
4379
|
+
}
|
|
4380
|
+
requestModel(options) {
|
|
4381
|
+
return typeof options.providerOptions?.model === "string" ? options.providerOptions.model : options.model;
|
|
4382
|
+
}
|
|
4383
|
+
assertSafeGemini3Continuation(messages, model) {
|
|
4384
|
+
if (GEMINI_MODELS_WITHOUT_PORTABLE_TEMPERATURE.has(model)) {
|
|
4385
|
+
const mapped = this.mapMessages(messages.filter((message) => message.role !== "system"));
|
|
4386
|
+
if (mapped.at(-1)?.role === "model") {
|
|
4387
|
+
throw new Error(
|
|
4388
|
+
`${model} does not support a terminal assistant/model prefill; end the request with a user or tool message.`
|
|
4389
|
+
);
|
|
4390
|
+
}
|
|
4391
|
+
}
|
|
4392
|
+
if (!isGemini3x(model)) return;
|
|
4393
|
+
const replayedNativeCallIds = /* @__PURE__ */ new Set();
|
|
4394
|
+
for (const message of messages) {
|
|
4395
|
+
if (message.role !== "assistant" || !Array.isArray(message.providerMetadata?.geminiParts)) {
|
|
3369
4396
|
continue;
|
|
3370
4397
|
}
|
|
3371
4398
|
for (const part of message.providerMetadata.geminiParts) {
|
|
@@ -3576,14 +4603,15 @@ var GeminiProvider = class {
|
|
|
3576
4603
|
}
|
|
3577
4604
|
const result = [];
|
|
3578
4605
|
for (const msg of messages) {
|
|
4606
|
+
const text = typeof msg.content === "string" ? msg.content : "";
|
|
3579
4607
|
if (msg.role === "assistant") {
|
|
3580
4608
|
const rawParts = msg.providerMetadata?.geminiParts;
|
|
3581
4609
|
if (rawParts && rawParts.length > 0) {
|
|
3582
4610
|
result.push({ role: "model", parts: rawParts });
|
|
3583
4611
|
} else {
|
|
3584
4612
|
const parts = [];
|
|
3585
|
-
if (
|
|
3586
|
-
parts.push({ text
|
|
4613
|
+
if (text) {
|
|
4614
|
+
parts.push({ text });
|
|
3587
4615
|
}
|
|
3588
4616
|
if (msg.tool_calls && msg.tool_calls.length > 0) {
|
|
3589
4617
|
for (const tc of msg.tool_calls) {
|
|
@@ -3609,7 +4637,7 @@ var GeminiProvider = class {
|
|
|
3609
4637
|
const functionName = (msg.tool_call_id ? geminiNativeNames.get(msg.tool_call_id) : void 0) ?? toolCallIdToName.get(msg.tool_call_id) ?? "unknown";
|
|
3610
4638
|
const functionResponse = {
|
|
3611
4639
|
name: functionName,
|
|
3612
|
-
response: parseGeminiFunctionResponse(
|
|
4640
|
+
response: parseGeminiFunctionResponse(text)
|
|
3613
4641
|
};
|
|
3614
4642
|
if (msg.tool_call_id && geminiNativeIds.has(msg.tool_call_id)) {
|
|
3615
4643
|
functionResponse.id = msg.tool_call_id;
|
|
@@ -3619,7 +4647,7 @@ var GeminiProvider = class {
|
|
|
3619
4647
|
parts: [{ functionResponse }]
|
|
3620
4648
|
});
|
|
3621
4649
|
} else if (msg.role === "user") {
|
|
3622
|
-
result.push({ role: "user", parts: [{ text
|
|
4650
|
+
result.push({ role: "user", parts: [{ text }] });
|
|
3623
4651
|
}
|
|
3624
4652
|
}
|
|
3625
4653
|
return this.mergeConsecutiveRoles(result);
|
|
@@ -3807,6 +4835,56 @@ var GeminiProvider = class {
|
|
|
3807
4835
|
}
|
|
3808
4836
|
}
|
|
3809
4837
|
};
|
|
4838
|
+
function isGeminiInteractionStep(value) {
|
|
4839
|
+
return value !== null && typeof value === "object" && ["user_input", "model_output", "thought", "function_call", "function_result"].includes(
|
|
4840
|
+
value.type
|
|
4841
|
+
);
|
|
4842
|
+
}
|
|
4843
|
+
function interactionText(content) {
|
|
4844
|
+
return content?.map((part) => part.type === "text" && typeof part.text === "string" ? part.text : "").join("") ?? "";
|
|
4845
|
+
}
|
|
4846
|
+
function interactionTextChunks(content) {
|
|
4847
|
+
return content?.flatMap(
|
|
4848
|
+
(part) => part.type === "text" && typeof part.text === "string" ? [part.text] : []
|
|
4849
|
+
) ?? [];
|
|
4850
|
+
}
|
|
4851
|
+
function cloneInteractionStep(step) {
|
|
4852
|
+
return JSON.parse(JSON.stringify(step));
|
|
4853
|
+
}
|
|
4854
|
+
function appendInteractionContent(step, field, content) {
|
|
4855
|
+
const existing = step[field] ?? [];
|
|
4856
|
+
const last = existing.at(-1);
|
|
4857
|
+
if (content.type === "text" && last?.type === "text" && typeof content.text === "string" && typeof last.text === "string") {
|
|
4858
|
+
last.text += content.text;
|
|
4859
|
+
} else {
|
|
4860
|
+
existing.push(content);
|
|
4861
|
+
}
|
|
4862
|
+
step[field] = existing;
|
|
4863
|
+
}
|
|
4864
|
+
function safeJsonObject(value) {
|
|
4865
|
+
try {
|
|
4866
|
+
const parsed = JSON.parse(value);
|
|
4867
|
+
return parsed !== null && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : {};
|
|
4868
|
+
} catch {
|
|
4869
|
+
return {};
|
|
4870
|
+
}
|
|
4871
|
+
}
|
|
4872
|
+
function normalizeInteractionUsage(usage) {
|
|
4873
|
+
if (!usage || !isValidTokenCount2(usage.total_input_tokens) || !isValidTokenCount2(usage.total_output_tokens) || !isValidTokenCount2(usage.total_tokens)) {
|
|
4874
|
+
return void 0;
|
|
4875
|
+
}
|
|
4876
|
+
const cached = usage.total_cached_tokens;
|
|
4877
|
+
const thinking = usage.total_thought_tokens;
|
|
4878
|
+
if (cached !== void 0 && (!isValidTokenCount2(cached) || cached > usage.total_input_tokens) || thinking !== void 0 && !isValidTokenCount2(thinking))
|
|
4879
|
+
return void 0;
|
|
4880
|
+
return {
|
|
4881
|
+
prompt_tokens: usage.total_input_tokens,
|
|
4882
|
+
completion_tokens: usage.total_output_tokens,
|
|
4883
|
+
total_tokens: usage.total_tokens,
|
|
4884
|
+
...cached ? { cached_tokens: cached } : {},
|
|
4885
|
+
...thinking ? { reasoning_tokens: thinking } : {}
|
|
4886
|
+
};
|
|
4887
|
+
}
|
|
3810
4888
|
|
|
3811
4889
|
// src/providers/profiles/openrouter.ts
|
|
3812
4890
|
var OPENROUTER_PROFILE = {
|
|
@@ -4247,59 +5325,1211 @@ var ProviderRegistry = class {
|
|
|
4247
5325
|
}
|
|
4248
5326
|
throw new Error(`Unknown provider "${name}". Registered providers: ${this.list().join(", ")}`);
|
|
4249
5327
|
}
|
|
4250
|
-
/**
|
|
4251
|
-
* Resolve a "provider:model" URI string into a Provider instance and model name.
|
|
4252
|
-
*
|
|
4253
|
-
* Supported formats:
|
|
4254
|
-
* - "openai:gpt-4o" -> provider=openai, model=gpt-4o
|
|
4255
|
-
* - "anthropic:claude-3" -> provider=anthropic, model=claude-3
|
|
4256
|
-
* - "gpt-4o" -> uses defaultProvider from config, model=gpt-4o
|
|
4257
|
-
* - undefined / empty -> uses defaultProvider and defaultModel from config
|
|
4258
|
-
*
|
|
4259
|
-
* @param uri Provider:model string, or just a model name
|
|
4260
|
-
* @param config Axl configuration for provider options and defaults
|
|
4261
|
-
*/
|
|
5328
|
+
/**
|
|
5329
|
+
* Resolve a "provider:model" URI string into a Provider instance and model name.
|
|
5330
|
+
*
|
|
5331
|
+
* Supported formats:
|
|
5332
|
+
* - "openai:gpt-4o" -> provider=openai, model=gpt-4o
|
|
5333
|
+
* - "anthropic:claude-3" -> provider=anthropic, model=claude-3
|
|
5334
|
+
* - "gpt-4o" -> uses defaultProvider from config, model=gpt-4o
|
|
5335
|
+
* - undefined / empty -> uses defaultProvider and defaultModel from config
|
|
5336
|
+
*
|
|
5337
|
+
* @param uri Provider:model string, or just a model name
|
|
5338
|
+
* @param config Axl configuration for provider options and defaults
|
|
5339
|
+
*/
|
|
5340
|
+
resolve(uri, config = {}) {
|
|
5341
|
+
if (!uri) {
|
|
5342
|
+
const providerName2 = config.defaultProvider ?? "openai";
|
|
5343
|
+
const model2 = config.defaultModel ?? "gpt-4o";
|
|
5344
|
+
return { provider: this.get(providerName2, config), model: model2 };
|
|
5345
|
+
}
|
|
5346
|
+
const colonIndex = uri.indexOf(":");
|
|
5347
|
+
if (colonIndex === -1) {
|
|
5348
|
+
const providerName2 = config.defaultProvider ?? "openai";
|
|
5349
|
+
return { provider: this.get(providerName2, config), model: uri };
|
|
5350
|
+
}
|
|
5351
|
+
const providerName = uri.slice(0, colonIndex);
|
|
5352
|
+
const model = uri.slice(colonIndex + 1);
|
|
5353
|
+
if (!providerName || !model) {
|
|
5354
|
+
throw new Error(
|
|
5355
|
+
`Invalid provider URI "${uri}". Expected format: "provider:model" (e.g. "openai:gpt-4o")`
|
|
5356
|
+
);
|
|
5357
|
+
}
|
|
5358
|
+
return { provider: this.get(providerName, config), model };
|
|
5359
|
+
}
|
|
5360
|
+
/**
|
|
5361
|
+
* Clear all cached provider instances. Useful for testing or reconfiguration.
|
|
5362
|
+
*/
|
|
5363
|
+
clearCache() {
|
|
5364
|
+
this.instances.clear();
|
|
5365
|
+
}
|
|
5366
|
+
/**
|
|
5367
|
+
* Clear all registered factories (including built-ins).
|
|
5368
|
+
* Useful for test runtimes where only explicitly registered instances should be used.
|
|
5369
|
+
*/
|
|
5370
|
+
clearFactories() {
|
|
5371
|
+
this.factories.clear();
|
|
5372
|
+
}
|
|
5373
|
+
};
|
|
5374
|
+
var defaultRegistry = new ProviderRegistry();
|
|
5375
|
+
|
|
5376
|
+
// src/transcription.ts
|
|
5377
|
+
var MAX_INLINE_TRANSCRIPTION_BYTES = 25 * 1024 * 1024;
|
|
5378
|
+
var RESERVED_PROVIDER_OPTION_KEYS = /* @__PURE__ */ new Set([
|
|
5379
|
+
"model",
|
|
5380
|
+
"audio",
|
|
5381
|
+
"signal",
|
|
5382
|
+
"cleanup",
|
|
5383
|
+
"cleanupStatus",
|
|
5384
|
+
"usage",
|
|
5385
|
+
"cost",
|
|
5386
|
+
"pricingStatus",
|
|
5387
|
+
"accounting"
|
|
5388
|
+
]);
|
|
5389
|
+
function nonEmptyString(value) {
|
|
5390
|
+
return typeof value === "string" && value.trim().length > 0;
|
|
5391
|
+
}
|
|
5392
|
+
function cloneValue(value) {
|
|
5393
|
+
try {
|
|
5394
|
+
return structuredClone(value);
|
|
5395
|
+
} catch {
|
|
5396
|
+
throw new InvalidTranscriptionInputError("Transcription options must be structured-cloneable");
|
|
5397
|
+
}
|
|
5398
|
+
}
|
|
5399
|
+
function normalizeTranscriptionRequest(request) {
|
|
5400
|
+
if (!request || typeof request !== "object" || !nonEmptyString(request.model)) {
|
|
5401
|
+
throw new InvalidTranscriptionInputError("A transcription model URI is required");
|
|
5402
|
+
}
|
|
5403
|
+
const audio = request.audio;
|
|
5404
|
+
if (!audio || typeof audio !== "object") {
|
|
5405
|
+
throw new InvalidTranscriptionInputError("A transcription audio source is required");
|
|
5406
|
+
}
|
|
5407
|
+
let normalizedAudio;
|
|
5408
|
+
if (audio.type === "bytes") {
|
|
5409
|
+
if (!(audio.data instanceof Uint8Array) || audio.data.byteLength === 0 || !nonEmptyString(audio.mediaType)) {
|
|
5410
|
+
throw new InvalidTranscriptionInputError(
|
|
5411
|
+
"Inline transcription bytes require nonempty data and mediaType"
|
|
5412
|
+
);
|
|
5413
|
+
}
|
|
5414
|
+
if (audio.data.byteLength > MAX_INLINE_TRANSCRIPTION_BYTES) {
|
|
5415
|
+
throw new InvalidTranscriptionInputError(
|
|
5416
|
+
"Inline transcription audio must not exceed 25 MiB; chunk it or use a supported provider-file source"
|
|
5417
|
+
);
|
|
5418
|
+
}
|
|
5419
|
+
normalizedAudio = {
|
|
5420
|
+
type: "bytes",
|
|
5421
|
+
data: new Uint8Array(audio.data),
|
|
5422
|
+
mediaType: audio.mediaType
|
|
5423
|
+
};
|
|
5424
|
+
} else if (audio.type === "base64") {
|
|
5425
|
+
if (!nonEmptyString(audio.data) || !nonEmptyString(audio.mediaType)) {
|
|
5426
|
+
throw new InvalidTranscriptionInputError(
|
|
5427
|
+
"Inline base64 transcription audio requires data and mediaType"
|
|
5428
|
+
);
|
|
5429
|
+
}
|
|
5430
|
+
if (audio.data.length > 4 * Math.ceil(MAX_INLINE_TRANSCRIPTION_BYTES / 3)) {
|
|
5431
|
+
throw new InvalidTranscriptionInputError(
|
|
5432
|
+
"Inline transcription audio must not exceed 25 MiB; chunk it or use a supported provider-file source"
|
|
5433
|
+
);
|
|
5434
|
+
}
|
|
5435
|
+
let decoded;
|
|
5436
|
+
try {
|
|
5437
|
+
decoded = Uint8Array.from(Buffer.from(audio.data, "base64"));
|
|
5438
|
+
} catch {
|
|
5439
|
+
throw new InvalidTranscriptionInputError(
|
|
5440
|
+
"Transcription base64 must be canonical and decode to audio"
|
|
5441
|
+
);
|
|
5442
|
+
}
|
|
5443
|
+
if (!decoded.byteLength || Buffer.from(decoded).toString("base64") !== audio.data) {
|
|
5444
|
+
throw new InvalidTranscriptionInputError(
|
|
5445
|
+
"Transcription base64 must be canonical and decode to nonempty audio"
|
|
5446
|
+
);
|
|
5447
|
+
}
|
|
5448
|
+
if (decoded.byteLength > MAX_INLINE_TRANSCRIPTION_BYTES) {
|
|
5449
|
+
throw new InvalidTranscriptionInputError(
|
|
5450
|
+
"Inline transcription audio must not exceed 25 MiB; chunk it or use a supported provider-file source"
|
|
5451
|
+
);
|
|
5452
|
+
}
|
|
5453
|
+
normalizedAudio = { type: "base64", data: audio.data, mediaType: audio.mediaType };
|
|
5454
|
+
} else if (audio.type === "provider-file") {
|
|
5455
|
+
if (!nonEmptyString(audio.provider) || !nonEmptyString(audio.reference)) {
|
|
5456
|
+
throw new InvalidTranscriptionInputError(
|
|
5457
|
+
"A provider-file transcription source requires provider and reference"
|
|
5458
|
+
);
|
|
5459
|
+
}
|
|
5460
|
+
if (audio.mediaType !== void 0 && !nonEmptyString(audio.mediaType)) {
|
|
5461
|
+
throw new InvalidTranscriptionInputError(
|
|
5462
|
+
"Transcription provider-file mediaType must be nonempty when supplied"
|
|
5463
|
+
);
|
|
5464
|
+
}
|
|
5465
|
+
normalizedAudio = {
|
|
5466
|
+
type: "provider-file",
|
|
5467
|
+
provider: audio.provider,
|
|
5468
|
+
reference: audio.reference,
|
|
5469
|
+
...audio.mediaType !== void 0 ? { mediaType: audio.mediaType } : {}
|
|
5470
|
+
};
|
|
5471
|
+
} else {
|
|
5472
|
+
throw new InvalidTranscriptionInputError(
|
|
5473
|
+
"Transcription audio must be bytes, base64, or provider-file"
|
|
5474
|
+
);
|
|
5475
|
+
}
|
|
5476
|
+
if (request.language !== void 0 && !nonEmptyString(request.language)) {
|
|
5477
|
+
throw new InvalidTranscriptionInputError(
|
|
5478
|
+
"Transcription language must be nonempty when supplied"
|
|
5479
|
+
);
|
|
5480
|
+
}
|
|
5481
|
+
if (request.timestamps !== void 0 && request.timestamps !== "segment" && request.timestamps !== "word") {
|
|
5482
|
+
throw new InvalidTranscriptionInputError("Transcription timestamps must be segment or word");
|
|
5483
|
+
}
|
|
5484
|
+
if (request.diarization !== void 0 && typeof request.diarization !== "boolean") {
|
|
5485
|
+
throw new InvalidTranscriptionInputError("Transcription diarization must be boolean");
|
|
5486
|
+
}
|
|
5487
|
+
if (request.providerOptions !== void 0) {
|
|
5488
|
+
if (!request.providerOptions || Array.isArray(request.providerOptions) || typeof request.providerOptions !== "object") {
|
|
5489
|
+
throw new InvalidTranscriptionInputError("Transcription providerOptions must be an object");
|
|
5490
|
+
}
|
|
5491
|
+
for (const key of Object.keys(request.providerOptions)) {
|
|
5492
|
+
if (RESERVED_PROVIDER_OPTION_KEYS.has(key)) {
|
|
5493
|
+
throw new InvalidTranscriptionInputError(
|
|
5494
|
+
"Transcription providerOptions cannot override reserved operation fields"
|
|
5495
|
+
);
|
|
5496
|
+
}
|
|
5497
|
+
}
|
|
5498
|
+
}
|
|
5499
|
+
return {
|
|
5500
|
+
model: request.model,
|
|
5501
|
+
audio: normalizedAudio,
|
|
5502
|
+
...request.language !== void 0 ? { language: request.language } : {},
|
|
5503
|
+
...request.timestamps !== void 0 ? { timestamps: request.timestamps } : {},
|
|
5504
|
+
...request.diarization !== void 0 ? { diarization: request.diarization } : {},
|
|
5505
|
+
...request.providerOptions !== void 0 ? { providerOptions: cloneValue(request.providerOptions) } : {}
|
|
5506
|
+
};
|
|
5507
|
+
}
|
|
5508
|
+
function cloneTranscript(transcript) {
|
|
5509
|
+
if (!transcript || typeof transcript !== "object" || typeof transcript.text !== "string") {
|
|
5510
|
+
throw new InvalidTranscriptionInputError(
|
|
5511
|
+
"Transcription provider returned an invalid transcript"
|
|
5512
|
+
);
|
|
5513
|
+
}
|
|
5514
|
+
const cloned = cloneValue(transcript);
|
|
5515
|
+
const accounting = normalizeTranscriptionAccounting(cloned.usage, cloned.pricingStatus);
|
|
5516
|
+
return {
|
|
5517
|
+
text: cloned.text,
|
|
5518
|
+
...cloned.detectedLanguages ? { detectedLanguages: cloned.detectedLanguages } : {},
|
|
5519
|
+
...cloned.segments ? { segments: cloned.segments } : {},
|
|
5520
|
+
...cloned.words ? { words: cloned.words } : {},
|
|
5521
|
+
...accounting.usage ? { usage: accounting.usage } : {},
|
|
5522
|
+
...accounting.pricingStatus ? { pricingStatus: accounting.pricingStatus } : {},
|
|
5523
|
+
...cloned.providerMetadata ? { providerMetadata: cloned.providerMetadata } : {}
|
|
5524
|
+
};
|
|
5525
|
+
}
|
|
5526
|
+
function normalizeTranscriptionAccounting(usageValue, pricingValue) {
|
|
5527
|
+
const value = usageValue && typeof usageValue === "object" ? usageValue : void 0;
|
|
5528
|
+
const number = (key) => {
|
|
5529
|
+
const candidate = value?.[key];
|
|
5530
|
+
return typeof candidate === "number" && Number.isFinite(candidate) && candidate >= 0 ? candidate : void 0;
|
|
5531
|
+
};
|
|
5532
|
+
const audioSeconds = number("audioSeconds");
|
|
5533
|
+
const inputTokens = number("inputTokens");
|
|
5534
|
+
const outputTokens = number("outputTokens");
|
|
5535
|
+
const totalTokens = number("totalTokens");
|
|
5536
|
+
const reportedCost = number("cost");
|
|
5537
|
+
const usage = value && (audioSeconds !== void 0 || inputTokens !== void 0 || outputTokens !== void 0 || totalTokens !== void 0 || reportedCost !== void 0) ? {
|
|
5538
|
+
...audioSeconds !== void 0 ? { audioSeconds } : {},
|
|
5539
|
+
...inputTokens !== void 0 ? { inputTokens } : {},
|
|
5540
|
+
...outputTokens !== void 0 ? { outputTokens } : {},
|
|
5541
|
+
...totalTokens !== void 0 ? { totalTokens } : {},
|
|
5542
|
+
...reportedCost !== void 0 ? { cost: reportedCost } : {}
|
|
5543
|
+
} : void 0;
|
|
5544
|
+
const hasWork = (audioSeconds ?? 0) > 0 || (inputTokens ?? 0) > 0 || (outputTokens ?? 0) > 0 || (totalTokens ?? 0) > 0;
|
|
5545
|
+
const requested = pricingValue === "priced" || pricingValue === "unpriced" || pricingValue === "zero" ? pricingValue : void 0;
|
|
5546
|
+
const cost = requested === "zero" && (reportedCost === void 0 || reportedCost === 0) ? 0 : reportedCost;
|
|
5547
|
+
const pricingStatus = requested === "zero" && (reportedCost === void 0 || reportedCost === 0) ? "zero" : requested === "priced" && cost !== void 0 ? "priced" : requested === "unpriced" && cost === void 0 ? "unpriced" : cost !== void 0 ? "priced" : hasWork || requested === "unpriced" || requested === "priced" ? "unpriced" : void 0;
|
|
5548
|
+
const tokens = inputTokens !== void 0 || outputTokens !== void 0 ? {
|
|
5549
|
+
...inputTokens !== void 0 ? { input: inputTokens } : {},
|
|
5550
|
+
...outputTokens !== void 0 ? { output: outputTokens } : {}
|
|
5551
|
+
} : void 0;
|
|
5552
|
+
return {
|
|
5553
|
+
...usage ? { usage } : {},
|
|
5554
|
+
...pricingStatus ? { pricingStatus } : {},
|
|
5555
|
+
...cost !== void 0 ? { cost } : {},
|
|
5556
|
+
...tokens ? { tokens } : {},
|
|
5557
|
+
hasWork
|
|
5558
|
+
};
|
|
5559
|
+
}
|
|
5560
|
+
|
|
5561
|
+
// src/providers/transcription-utils.ts
|
|
5562
|
+
var OPENAI_TRANSCRIPTION_MODEL = "gpt-transcribe";
|
|
5563
|
+
var GEMINI_TRANSCRIPTION_MODEL = "gemini-3.5-transcribe";
|
|
5564
|
+
var OPENROUTER_TRANSCRIPTION_MODEL = "openai/whisper-1";
|
|
5565
|
+
function requireExactTranscriptionModel(provider, model, expected) {
|
|
5566
|
+
if (model !== expected) {
|
|
5567
|
+
throw new UnsupportedTranscriptionInputError({
|
|
5568
|
+
provider,
|
|
5569
|
+
model,
|
|
5570
|
+
feature: "this transcription model"
|
|
5571
|
+
});
|
|
5572
|
+
}
|
|
5573
|
+
}
|
|
5574
|
+
function inlineAudioBytes(audio) {
|
|
5575
|
+
if (audio.type === "bytes") return new Uint8Array(audio.data);
|
|
5576
|
+
if (audio.type === "base64") return Uint8Array.from(Buffer.from(audio.data, "base64"));
|
|
5577
|
+
throw new UnsupportedTranscriptionInputError({
|
|
5578
|
+
provider: audio.provider,
|
|
5579
|
+
model: "",
|
|
5580
|
+
feature: "provider-file transcription input"
|
|
5581
|
+
});
|
|
5582
|
+
}
|
|
5583
|
+
function requireNoProviderFile(audio, provider, model) {
|
|
5584
|
+
if (audio.type === "provider-file") {
|
|
5585
|
+
throw new UnsupportedTranscriptionInputError({
|
|
5586
|
+
provider,
|
|
5587
|
+
model,
|
|
5588
|
+
feature: "provider-file transcription input"
|
|
5589
|
+
});
|
|
5590
|
+
}
|
|
5591
|
+
}
|
|
5592
|
+
function assertOnlyProviderOptions(request, provider, model, allowed) {
|
|
5593
|
+
const options = request.providerOptions ?? {};
|
|
5594
|
+
for (const key of Object.keys(options)) {
|
|
5595
|
+
if (!allowed.includes(key)) {
|
|
5596
|
+
throw new InvalidTranscriptionInputError(
|
|
5597
|
+
`Unsupported ${provider} transcription provider option for model '${model}'`
|
|
5598
|
+
);
|
|
5599
|
+
}
|
|
5600
|
+
}
|
|
5601
|
+
return options;
|
|
5602
|
+
}
|
|
5603
|
+
function transcriptUsage(value) {
|
|
5604
|
+
const source = value && typeof value === "object" ? value : {};
|
|
5605
|
+
const number = (...keys) => {
|
|
5606
|
+
for (const key of keys) {
|
|
5607
|
+
const candidate = source[key];
|
|
5608
|
+
if (typeof candidate === "number" && Number.isFinite(candidate) && candidate >= 0)
|
|
5609
|
+
return candidate;
|
|
5610
|
+
}
|
|
5611
|
+
return void 0;
|
|
5612
|
+
};
|
|
5613
|
+
const inputTokens = number("total_input_tokens", "input_tokens", "prompt_tokens", "inputTokens");
|
|
5614
|
+
const outputTokens = number(
|
|
5615
|
+
"total_output_tokens",
|
|
5616
|
+
"output_tokens",
|
|
5617
|
+
"completion_tokens",
|
|
5618
|
+
"outputTokens"
|
|
5619
|
+
);
|
|
5620
|
+
const totalTokens = number("total_tokens", "totalTokens");
|
|
5621
|
+
const audioSeconds = number("seconds", "audio_seconds", "audioSeconds", "duration_seconds");
|
|
5622
|
+
const cost = number("cost");
|
|
5623
|
+
const usage = inputTokens !== void 0 || outputTokens !== void 0 || totalTokens !== void 0 || audioSeconds !== void 0 || cost !== void 0 ? {
|
|
5624
|
+
...audioSeconds !== void 0 ? { audioSeconds } : {},
|
|
5625
|
+
...inputTokens !== void 0 ? { inputTokens } : {},
|
|
5626
|
+
...outputTokens !== void 0 ? { outputTokens } : {},
|
|
5627
|
+
...totalTokens !== void 0 ? { totalTokens } : {},
|
|
5628
|
+
...cost !== void 0 ? { cost } : {}
|
|
5629
|
+
} : void 0;
|
|
5630
|
+
const hasWork = (audioSeconds ?? 0) > 0 || (inputTokens ?? 0) > 0 || (outputTokens ?? 0) > 0 || (totalTokens ?? 0) > 0;
|
|
5631
|
+
return {
|
|
5632
|
+
...usage ? { usage } : {},
|
|
5633
|
+
...cost !== void 0 ? { pricingStatus: "priced" } : hasWork ? { pricingStatus: "unpriced" } : {}
|
|
5634
|
+
};
|
|
5635
|
+
}
|
|
5636
|
+
function safeProviderMessage(provider, status, body) {
|
|
5637
|
+
try {
|
|
5638
|
+
const parsed = JSON.parse(body);
|
|
5639
|
+
const message = parsed.error?.message ?? parsed.message;
|
|
5640
|
+
if (typeof message === "string")
|
|
5641
|
+
return `${provider} transcription API error (${status}): ${message}`;
|
|
5642
|
+
} catch {
|
|
5643
|
+
}
|
|
5644
|
+
return `${provider} transcription API error (${status})`;
|
|
5645
|
+
}
|
|
5646
|
+
function attachTranscriptionFailureMetadata(error, metadata) {
|
|
5647
|
+
for (const [key, value] of Object.entries(metadata)) {
|
|
5648
|
+
if (value !== void 0) Object.defineProperty(error, key, { enumerable: true, value });
|
|
5649
|
+
}
|
|
5650
|
+
return error;
|
|
5651
|
+
}
|
|
5652
|
+
|
|
5653
|
+
// src/providers/openai-transcription.ts
|
|
5654
|
+
var OPENAI_FILE_EXTENSIONS = {
|
|
5655
|
+
"audio/flac": "flac",
|
|
5656
|
+
"audio/m4a": "m4a",
|
|
5657
|
+
"audio/mp3": "mp3",
|
|
5658
|
+
"audio/mp4": "mp4",
|
|
5659
|
+
"audio/mpeg": "mp3",
|
|
5660
|
+
"audio/mpga": "mpga",
|
|
5661
|
+
"audio/ogg": "ogg",
|
|
5662
|
+
"audio/wav": "wav",
|
|
5663
|
+
"audio/webm": "webm"
|
|
5664
|
+
};
|
|
5665
|
+
var OpenAITranscriptionProvider = class {
|
|
5666
|
+
name = "openai-transcription";
|
|
5667
|
+
baseUrl;
|
|
5668
|
+
apiKeySource;
|
|
5669
|
+
governor;
|
|
5670
|
+
constructor(options = {}) {
|
|
5671
|
+
this.apiKeySource = options.apiKey ?? process.env.OPENAI_API_KEY ?? "";
|
|
5672
|
+
this.baseUrl = (options.baseUrl ?? process.env.OPENAI_BASE_URL ?? "https://api.openai.com/v1").replace(/\/$/, "");
|
|
5673
|
+
assertSafeProviderBaseUrl(
|
|
5674
|
+
this.baseUrl,
|
|
5675
|
+
"OpenAI transcription provider",
|
|
5676
|
+
options.dangerouslyAllowInsecureHttp
|
|
5677
|
+
);
|
|
5678
|
+
this.governor = options.rateLimit ? new RateLimiter(options.rateLimit) : void 0;
|
|
5679
|
+
}
|
|
5680
|
+
capabilities(model) {
|
|
5681
|
+
return model === OPENAI_TRANSCRIPTION_MODEL ? { sources: ["bytes", "base64"] } : void 0;
|
|
5682
|
+
}
|
|
5683
|
+
async transcribe(request) {
|
|
5684
|
+
requireExactTranscriptionModel(this.name, request.model, OPENAI_TRANSCRIPTION_MODEL);
|
|
5685
|
+
requireNoProviderFile(request.audio, this.name, request.model);
|
|
5686
|
+
if (request.audio.type === "provider-file")
|
|
5687
|
+
throw new UnsupportedTranscriptionInputError({
|
|
5688
|
+
provider: this.name,
|
|
5689
|
+
model: request.model,
|
|
5690
|
+
feature: "provider-file transcription input"
|
|
5691
|
+
});
|
|
5692
|
+
const options = assertOnlyProviderOptions(request, this.name, request.model, [
|
|
5693
|
+
"temperature",
|
|
5694
|
+
"prompt"
|
|
5695
|
+
]);
|
|
5696
|
+
if (options.temperature !== void 0 && (typeof options.temperature !== "number" || !Number.isFinite(options.temperature) || options.temperature < 0 || options.temperature > 1))
|
|
5697
|
+
throw new InvalidTranscriptionInputError(
|
|
5698
|
+
"OpenAI transcription temperature must be a number between 0 and 1"
|
|
5699
|
+
);
|
|
5700
|
+
if (options.prompt !== void 0 && typeof options.prompt !== "string")
|
|
5701
|
+
throw new InvalidTranscriptionInputError("OpenAI transcription prompt must be a string");
|
|
5702
|
+
const mediaType2 = request.audio.mediaType;
|
|
5703
|
+
const extension = OPENAI_FILE_EXTENSIONS[mediaType2];
|
|
5704
|
+
if (!extension)
|
|
5705
|
+
throw new UnsupportedTranscriptionInputError({
|
|
5706
|
+
provider: this.name,
|
|
5707
|
+
model: request.model,
|
|
5708
|
+
feature: "this audio media type"
|
|
5709
|
+
});
|
|
5710
|
+
const bytes = inlineAudioBytes(request.audio);
|
|
5711
|
+
if (bytes.byteLength > MAX_INLINE_TRANSCRIPTION_BYTES)
|
|
5712
|
+
throw new InvalidTranscriptionInputError("OpenAI transcription audio must not exceed 25 MiB");
|
|
5713
|
+
const form = new FormData();
|
|
5714
|
+
form.set("file", new Blob([Buffer.from(bytes)], { type: mediaType2 }), `audio.${extension}`);
|
|
5715
|
+
form.set("model", request.model);
|
|
5716
|
+
form.set("response_format", "json");
|
|
5717
|
+
if (request.language) form.set("language", request.language);
|
|
5718
|
+
if (options.temperature !== void 0) form.set("temperature", String(options.temperature));
|
|
5719
|
+
if (options.prompt !== void 0) form.set("prompt", options.prompt);
|
|
5720
|
+
const key = await this.resolveKey();
|
|
5721
|
+
const res = await fetchWithRetry(
|
|
5722
|
+
`${this.baseUrl}/audio/transcriptions`,
|
|
5723
|
+
{
|
|
5724
|
+
method: "POST",
|
|
5725
|
+
headers: { Authorization: `Bearer ${key}` },
|
|
5726
|
+
body: form,
|
|
5727
|
+
signal: request.signal
|
|
5728
|
+
},
|
|
5729
|
+
{ governor: this.governor, provider: this.name }
|
|
5730
|
+
);
|
|
5731
|
+
if (!res.ok) {
|
|
5732
|
+
const body = await res.text();
|
|
5733
|
+
throw buildProviderError({
|
|
5734
|
+
provider: this.name,
|
|
5735
|
+
status: res.status,
|
|
5736
|
+
headers: res.headers,
|
|
5737
|
+
message: safeProviderMessage("OpenAI", res.status, body),
|
|
5738
|
+
body
|
|
5739
|
+
});
|
|
5740
|
+
}
|
|
5741
|
+
const json = await res.json();
|
|
5742
|
+
if (typeof json.text !== "string")
|
|
5743
|
+
throw new InvalidTranscriptionInputError(
|
|
5744
|
+
"OpenAI transcription response did not contain text"
|
|
5745
|
+
);
|
|
5746
|
+
return {
|
|
5747
|
+
transcript: { text: json.text, ...transcriptUsage(json.usage) },
|
|
5748
|
+
cleanupStatus: "not_required"
|
|
5749
|
+
};
|
|
5750
|
+
}
|
|
5751
|
+
async resolveKey() {
|
|
5752
|
+
const key = await resolveApiKey(this.apiKeySource);
|
|
5753
|
+
if (!key)
|
|
5754
|
+
throw new Error("OpenAI API key is required. Set OPENAI_API_KEY or pass apiKey in options.");
|
|
5755
|
+
return key;
|
|
5756
|
+
}
|
|
5757
|
+
};
|
|
5758
|
+
|
|
5759
|
+
// src/providers/gemini-transcription.ts
|
|
5760
|
+
var CLEANUP_TIMEOUT_MS = 3e3;
|
|
5761
|
+
var READY_TIMEOUT_MS = 3e4;
|
|
5762
|
+
var READY_POLL_MS = 250;
|
|
5763
|
+
var GEMINI_AUDIO_MEDIA_TYPES = /* @__PURE__ */ new Set([
|
|
5764
|
+
"audio/wav",
|
|
5765
|
+
"audio/mp3",
|
|
5766
|
+
"audio/aiff",
|
|
5767
|
+
"audio/aac",
|
|
5768
|
+
"audio/ogg",
|
|
5769
|
+
"audio/flac",
|
|
5770
|
+
"audio/mpeg",
|
|
5771
|
+
"audio/m4a",
|
|
5772
|
+
"audio/l16",
|
|
5773
|
+
"audio/opus",
|
|
5774
|
+
"audio/alaw",
|
|
5775
|
+
"audio/mulaw",
|
|
5776
|
+
"audio/webm"
|
|
5777
|
+
]);
|
|
5778
|
+
function abortError2() {
|
|
5779
|
+
return new DOMException("The transcription operation was aborted", "AbortError");
|
|
5780
|
+
}
|
|
5781
|
+
function audioMime(audio) {
|
|
5782
|
+
if (!audio.mediaType)
|
|
5783
|
+
throw new UnsupportedTranscriptionInputError({
|
|
5784
|
+
provider: "gemini-transcription",
|
|
5785
|
+
model: GEMINI_TRANSCRIPTION_MODEL,
|
|
5786
|
+
feature: "provider-file audio mediaType"
|
|
5787
|
+
});
|
|
5788
|
+
return audio.mediaType;
|
|
5789
|
+
}
|
|
5790
|
+
function assertGeminiAudioMime(mediaType2) {
|
|
5791
|
+
if (!mediaType2 || !GEMINI_AUDIO_MEDIA_TYPES.has(mediaType2)) {
|
|
5792
|
+
throw new UnsupportedTranscriptionInputError({
|
|
5793
|
+
provider: "gemini-transcription",
|
|
5794
|
+
model: GEMINI_TRANSCRIPTION_MODEL,
|
|
5795
|
+
feature: "this audio media type"
|
|
5796
|
+
});
|
|
5797
|
+
}
|
|
5798
|
+
return mediaType2;
|
|
5799
|
+
}
|
|
5800
|
+
function secondsOffset(value) {
|
|
5801
|
+
if (typeof value !== "string") return void 0;
|
|
5802
|
+
const match = /^(\d+(?:\.\d+)?)s$/.exec(value);
|
|
5803
|
+
if (!match) return void 0;
|
|
5804
|
+
const seconds = Number(match[1]);
|
|
5805
|
+
return Number.isFinite(seconds) && seconds >= 0 ? seconds : void 0;
|
|
5806
|
+
}
|
|
5807
|
+
function sleep3(ms, signal) {
|
|
5808
|
+
return new Promise((resolve2, reject) => {
|
|
5809
|
+
if (signal?.aborted) {
|
|
5810
|
+
reject(abortError2());
|
|
5811
|
+
return;
|
|
5812
|
+
}
|
|
5813
|
+
const timer = setTimeout(done, ms);
|
|
5814
|
+
const onAbort = () => {
|
|
5815
|
+
clearTimeout(timer);
|
|
5816
|
+
done(abortError2());
|
|
5817
|
+
};
|
|
5818
|
+
function done(error) {
|
|
5819
|
+
signal?.removeEventListener("abort", onAbort);
|
|
5820
|
+
if (error) reject(error);
|
|
5821
|
+
else resolve2();
|
|
5822
|
+
}
|
|
5823
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
5824
|
+
});
|
|
5825
|
+
}
|
|
5826
|
+
var GeminiTranscriptionProvider = class {
|
|
5827
|
+
name = "gemini-transcription";
|
|
5828
|
+
baseUrl;
|
|
5829
|
+
apiKeySource;
|
|
5830
|
+
governor;
|
|
5831
|
+
constructor(options = {}) {
|
|
5832
|
+
this.apiKeySource = options.apiKey ?? process.env.GOOGLE_API_KEY ?? process.env.GEMINI_API_KEY ?? "";
|
|
5833
|
+
this.baseUrl = (options.baseUrl ?? "https://generativelanguage.googleapis.com/v1beta").replace(
|
|
5834
|
+
/\/$/,
|
|
5835
|
+
""
|
|
5836
|
+
);
|
|
5837
|
+
assertSafeProviderBaseUrl(
|
|
5838
|
+
this.baseUrl,
|
|
5839
|
+
"Gemini transcription provider",
|
|
5840
|
+
options.dangerouslyAllowInsecureHttp
|
|
5841
|
+
);
|
|
5842
|
+
this.governor = options.rateLimit ? new RateLimiter(options.rateLimit) : void 0;
|
|
5843
|
+
}
|
|
5844
|
+
capabilities(model) {
|
|
5845
|
+
return model === GEMINI_TRANSCRIPTION_MODEL ? { sources: ["bytes", "base64", "provider-file"], timestamps: ["word"], diarization: true } : void 0;
|
|
5846
|
+
}
|
|
5847
|
+
async transcribe(request) {
|
|
5848
|
+
requireExactTranscriptionModel(this.name, request.model, GEMINI_TRANSCRIPTION_MODEL);
|
|
5849
|
+
const options = assertOnlyProviderOptions(request, this.name, request.model, [
|
|
5850
|
+
"mode",
|
|
5851
|
+
"customVocabulary"
|
|
5852
|
+
]);
|
|
5853
|
+
const mode = options.mode ?? "verbatim";
|
|
5854
|
+
if (mode !== "smart" && mode !== "verbatim")
|
|
5855
|
+
throw new InvalidTranscriptionInputError(
|
|
5856
|
+
"Gemini transcription mode must be smart or verbatim"
|
|
5857
|
+
);
|
|
5858
|
+
if (options.customVocabulary !== void 0 && (!Array.isArray(options.customVocabulary) || options.customVocabulary.length > 1e3 || options.customVocabulary.some(
|
|
5859
|
+
(value) => typeof value !== "string" || !value.trim() || value.length > 256
|
|
5860
|
+
)))
|
|
5861
|
+
throw new InvalidTranscriptionInputError(
|
|
5862
|
+
"Gemini transcription customVocabulary must contain at most 1000 nonempty strings up to 256 characters"
|
|
5863
|
+
);
|
|
5864
|
+
if (mode === "smart" && (request.timestamps || request.diarization))
|
|
5865
|
+
throw new InvalidTranscriptionInputError(
|
|
5866
|
+
"Gemini smart transcription does not support timestamps or diarization"
|
|
5867
|
+
);
|
|
5868
|
+
if (request.timestamps && options.customVocabulary !== void 0)
|
|
5869
|
+
throw new InvalidTranscriptionInputError(
|
|
5870
|
+
"Gemini transcription customVocabulary cannot be combined with timestamps"
|
|
5871
|
+
);
|
|
5872
|
+
if (request.timestamps && request.timestamps !== "word")
|
|
5873
|
+
throw new UnsupportedTranscriptionInputError({
|
|
5874
|
+
provider: this.name,
|
|
5875
|
+
model: request.model,
|
|
5876
|
+
feature: "segment timestamps"
|
|
5877
|
+
});
|
|
5878
|
+
if (request.diarization && request.timestamps !== "word")
|
|
5879
|
+
throw new InvalidTranscriptionInputError("Gemini diarization requires word timestamps");
|
|
5880
|
+
if (request.audio.type === "provider-file") {
|
|
5881
|
+
if (request.audio.provider !== this.name || typeof request.audio.reference !== "string" || !request.audio.reference.trim()) {
|
|
5882
|
+
throw new UnsupportedTranscriptionInputError({
|
|
5883
|
+
provider: this.name,
|
|
5884
|
+
model: request.model,
|
|
5885
|
+
feature: "a matching nonempty provider-file reference"
|
|
5886
|
+
});
|
|
5887
|
+
}
|
|
5888
|
+
assertGeminiAudioMime(request.audio.mediaType);
|
|
5889
|
+
} else {
|
|
5890
|
+
assertGeminiAudioMime(request.audio.mediaType);
|
|
5891
|
+
}
|
|
5892
|
+
const key = await this.resolveKey();
|
|
5893
|
+
if (request.audio.type === "provider-file")
|
|
5894
|
+
return this.transcribeReference(request, key, mode, options);
|
|
5895
|
+
return this.transcribeUpload(request, key, mode, options);
|
|
5896
|
+
}
|
|
5897
|
+
async transcribeReference(request, key, mode, options) {
|
|
5898
|
+
const audio = request.audio;
|
|
5899
|
+
if (audio.type !== "provider-file" || audio.provider !== this.name)
|
|
5900
|
+
throw new UnsupportedTranscriptionInputError({
|
|
5901
|
+
provider: this.name,
|
|
5902
|
+
model: request.model,
|
|
5903
|
+
feature: "a provider-file owned by another provider"
|
|
5904
|
+
});
|
|
5905
|
+
const mimeType = audioMime(audio);
|
|
5906
|
+
const transcript = await this.infer(request, key, audio.reference, mimeType, mode, options);
|
|
5907
|
+
return { transcript, cleanupStatus: "not_required" };
|
|
5908
|
+
}
|
|
5909
|
+
async transcribeUpload(request, key, mode, options) {
|
|
5910
|
+
let created = false;
|
|
5911
|
+
let fileName;
|
|
5912
|
+
let primaryError;
|
|
5913
|
+
let transcript;
|
|
5914
|
+
let cleanupStatus = "not_required";
|
|
5915
|
+
try {
|
|
5916
|
+
const uploaded = await this.upload(request, key);
|
|
5917
|
+
created = true;
|
|
5918
|
+
fileName = uploaded.name;
|
|
5919
|
+
const ready = await this.waitForReady(
|
|
5920
|
+
this.fileFromJson(uploaded.file, uploaded.mimeType),
|
|
5921
|
+
key,
|
|
5922
|
+
request.signal
|
|
5923
|
+
);
|
|
5924
|
+
transcript = await this.infer(request, key, ready.uri, ready.mimeType, mode, options);
|
|
5925
|
+
} catch (error) {
|
|
5926
|
+
primaryError = error;
|
|
5927
|
+
} finally {
|
|
5928
|
+
if (created && fileName) cleanupStatus = await this.cleanup(fileName, key);
|
|
5929
|
+
}
|
|
5930
|
+
if (primaryError) {
|
|
5931
|
+
throw attachTranscriptionFailureMetadata(
|
|
5932
|
+
primaryError instanceof Error ? primaryError : new Error("Gemini transcription failed"),
|
|
5933
|
+
{ cleanupStatus }
|
|
5934
|
+
);
|
|
5935
|
+
}
|
|
5936
|
+
if (!transcript)
|
|
5937
|
+
throw new InvalidTranscriptionInputError(
|
|
5938
|
+
"Gemini transcription response did not contain text"
|
|
5939
|
+
);
|
|
5940
|
+
return { transcript, cleanupStatus };
|
|
5941
|
+
}
|
|
5942
|
+
/** A finalization response with a name creates a caller-owned cleanup obligation
|
|
5943
|
+
* even when its URI/state is malformed. `transcribeUpload` stores the name before
|
|
5944
|
+
* strictly parsing the rest of the file representation. */
|
|
5945
|
+
async upload(request, key) {
|
|
5946
|
+
if (request.audio.type === "provider-file")
|
|
5947
|
+
throw new UnsupportedTranscriptionInputError({
|
|
5948
|
+
provider: this.name,
|
|
5949
|
+
model: request.model,
|
|
5950
|
+
feature: "provider-file upload"
|
|
5951
|
+
});
|
|
5952
|
+
const mimeType = assertGeminiAudioMime(request.audio.mediaType);
|
|
5953
|
+
const bytes = inlineAudioBytes(request.audio);
|
|
5954
|
+
const start = await fetchWithRetry(
|
|
5955
|
+
`${this.baseUrl.replace(/\/v1beta$/, "")}/upload/v1beta/files`,
|
|
5956
|
+
{
|
|
5957
|
+
method: "POST",
|
|
5958
|
+
headers: {
|
|
5959
|
+
"x-goog-api-key": key,
|
|
5960
|
+
"X-Goog-Upload-Protocol": "resumable",
|
|
5961
|
+
"X-Goog-Upload-Command": "start",
|
|
5962
|
+
"X-Goog-Upload-Header-Content-Length": String(bytes.byteLength),
|
|
5963
|
+
"X-Goog-Upload-Header-Content-Type": mimeType,
|
|
5964
|
+
"Content-Type": "application/json"
|
|
5965
|
+
},
|
|
5966
|
+
body: JSON.stringify({ file: { display_name: "axl-transcription" } }),
|
|
5967
|
+
signal: request.signal
|
|
5968
|
+
},
|
|
5969
|
+
{ governor: this.governor, provider: this.name, maxRetries: 0 }
|
|
5970
|
+
);
|
|
5971
|
+
if (!start.ok) {
|
|
5972
|
+
const body = await start.text();
|
|
5973
|
+
throw buildProviderError({
|
|
5974
|
+
provider: this.name,
|
|
5975
|
+
status: start.status,
|
|
5976
|
+
headers: start.headers,
|
|
5977
|
+
message: safeProviderMessage("Gemini", start.status, body),
|
|
5978
|
+
body
|
|
5979
|
+
});
|
|
5980
|
+
}
|
|
5981
|
+
const uploadUrl = start.headers.get("x-goog-upload-url");
|
|
5982
|
+
if (!uploadUrl)
|
|
5983
|
+
throw new InvalidTranscriptionInputError(
|
|
5984
|
+
"Gemini transcription upload did not return an upload URL"
|
|
5985
|
+
);
|
|
5986
|
+
let parsedUploadUrl;
|
|
5987
|
+
try {
|
|
5988
|
+
parsedUploadUrl = new URL(uploadUrl);
|
|
5989
|
+
} catch {
|
|
5990
|
+
throw new InvalidTranscriptionInputError(
|
|
5991
|
+
"Gemini transcription upload returned an invalid upload URL"
|
|
5992
|
+
);
|
|
5993
|
+
}
|
|
5994
|
+
const baseOrigin = new URL(this.baseUrl).origin;
|
|
5995
|
+
if (parsedUploadUrl.username || parsedUploadUrl.password || parsedUploadUrl.origin !== baseOrigin) {
|
|
5996
|
+
throw new InvalidTranscriptionInputError(
|
|
5997
|
+
"Gemini transcription upload returned an untrusted upload URL"
|
|
5998
|
+
);
|
|
5999
|
+
}
|
|
6000
|
+
assertSafeProviderBaseUrl(
|
|
6001
|
+
parsedUploadUrl.toString(),
|
|
6002
|
+
"Gemini transcription upload endpoint",
|
|
6003
|
+
new URL(this.baseUrl).protocol === "http:"
|
|
6004
|
+
);
|
|
6005
|
+
const finalize = await fetchWithRetry(
|
|
6006
|
+
parsedUploadUrl.toString(),
|
|
6007
|
+
{
|
|
6008
|
+
method: "POST",
|
|
6009
|
+
headers: {
|
|
6010
|
+
"X-Goog-Upload-Command": "upload, finalize",
|
|
6011
|
+
"X-Goog-Upload-Offset": "0",
|
|
6012
|
+
"Content-Length": String(bytes.byteLength)
|
|
6013
|
+
},
|
|
6014
|
+
body: bytes,
|
|
6015
|
+
signal: request.signal
|
|
6016
|
+
},
|
|
6017
|
+
{ governor: this.governor, provider: this.name, maxRetries: 0 }
|
|
6018
|
+
);
|
|
6019
|
+
if (!finalize.ok) {
|
|
6020
|
+
const body = await finalize.text();
|
|
6021
|
+
throw buildProviderError({
|
|
6022
|
+
provider: this.name,
|
|
6023
|
+
status: finalize.status,
|
|
6024
|
+
headers: finalize.headers,
|
|
6025
|
+
message: safeProviderMessage("Gemini", finalize.status, body),
|
|
6026
|
+
body
|
|
6027
|
+
});
|
|
6028
|
+
}
|
|
6029
|
+
const payload = await finalize.json();
|
|
6030
|
+
const file = payload.file;
|
|
6031
|
+
if (!file || typeof file.name !== "string") {
|
|
6032
|
+
throw new InvalidTranscriptionInputError(
|
|
6033
|
+
"Gemini transcription upload did not return a file name"
|
|
6034
|
+
);
|
|
6035
|
+
}
|
|
6036
|
+
return { name: file.name, file, mimeType };
|
|
6037
|
+
}
|
|
6038
|
+
async waitForReady(file, key, signal) {
|
|
6039
|
+
let current = file;
|
|
6040
|
+
const deadline = Date.now() + READY_TIMEOUT_MS;
|
|
6041
|
+
if (!current.state) {
|
|
6042
|
+
throw new InvalidTranscriptionInputError(
|
|
6043
|
+
"Gemini transcription file readiness state is missing"
|
|
6044
|
+
);
|
|
6045
|
+
}
|
|
6046
|
+
while (current.state && current.state !== "ACTIVE") {
|
|
6047
|
+
if (current.state === "FAILED")
|
|
6048
|
+
throw new InvalidTranscriptionInputError("Gemini transcription file processing failed");
|
|
6049
|
+
if (Date.now() >= deadline)
|
|
6050
|
+
throw new InvalidTranscriptionInputError("Gemini transcription file readiness timed out");
|
|
6051
|
+
await sleep3(Math.min(READY_POLL_MS, deadline - Date.now()), signal);
|
|
6052
|
+
const res = await fetchWithRetry(
|
|
6053
|
+
`${this.baseUrl}/${current.name}`,
|
|
6054
|
+
{ method: "GET", headers: { "x-goog-api-key": key }, signal },
|
|
6055
|
+
{ governor: this.governor, provider: this.name, maxRetries: 0 }
|
|
6056
|
+
);
|
|
6057
|
+
if (!res.ok) {
|
|
6058
|
+
const body = await res.text();
|
|
6059
|
+
throw buildProviderError({
|
|
6060
|
+
provider: this.name,
|
|
6061
|
+
status: res.status,
|
|
6062
|
+
headers: res.headers,
|
|
6063
|
+
message: safeProviderMessage("Gemini", res.status, body),
|
|
6064
|
+
body
|
|
6065
|
+
});
|
|
6066
|
+
}
|
|
6067
|
+
current = this.fileFromJson(await res.json(), current.mimeType);
|
|
6068
|
+
}
|
|
6069
|
+
return current;
|
|
6070
|
+
}
|
|
6071
|
+
async infer(request, key, uri, mimeType, mode, options) {
|
|
6072
|
+
const transcriptionConfig = {
|
|
6073
|
+
mode: mode === "smart" ? "smart" : {
|
|
6074
|
+
type: "verbatim",
|
|
6075
|
+
...request.timestamps ? { timestamp_granularities: ["word"] } : {},
|
|
6076
|
+
...request.diarization ? { diarization_mode: "speaker" } : {}
|
|
6077
|
+
},
|
|
6078
|
+
...request.language ? { language_codes: [request.language] } : {},
|
|
6079
|
+
...options.customVocabulary ? { custom_vocabulary: options.customVocabulary } : {}
|
|
6080
|
+
};
|
|
6081
|
+
const body = {
|
|
6082
|
+
model: request.model,
|
|
6083
|
+
input: [{ type: "audio", uri, mime_type: mimeType }],
|
|
6084
|
+
store: false,
|
|
6085
|
+
generation_config: { transcription_config: transcriptionConfig }
|
|
6086
|
+
};
|
|
6087
|
+
const res = await fetchWithRetry(
|
|
6088
|
+
`${this.baseUrl}/interactions`,
|
|
6089
|
+
{
|
|
6090
|
+
method: "POST",
|
|
6091
|
+
headers: { "x-goog-api-key": key, "Content-Type": "application/json" },
|
|
6092
|
+
body: JSON.stringify(body),
|
|
6093
|
+
signal: request.signal
|
|
6094
|
+
},
|
|
6095
|
+
{ governor: this.governor, provider: this.name }
|
|
6096
|
+
);
|
|
6097
|
+
if (!res.ok) {
|
|
6098
|
+
const errorBody = await res.text();
|
|
6099
|
+
throw buildProviderError({
|
|
6100
|
+
provider: this.name,
|
|
6101
|
+
status: res.status,
|
|
6102
|
+
headers: res.headers,
|
|
6103
|
+
message: safeProviderMessage("Gemini", res.status, errorBody),
|
|
6104
|
+
body: errorBody
|
|
6105
|
+
});
|
|
6106
|
+
}
|
|
6107
|
+
const json = await res.json();
|
|
6108
|
+
if (json.status !== "completed") {
|
|
6109
|
+
throw new InvalidTranscriptionInputError("Gemini transcription interaction did not complete");
|
|
6110
|
+
}
|
|
6111
|
+
const words = [];
|
|
6112
|
+
let text = "";
|
|
6113
|
+
let hasModelOutputText = false;
|
|
6114
|
+
if (Array.isArray(json.steps)) {
|
|
6115
|
+
for (const step of json.steps) {
|
|
6116
|
+
if (!step || typeof step !== "object") continue;
|
|
6117
|
+
const modelOutput = step;
|
|
6118
|
+
if (modelOutput.type !== "model_output" || !Array.isArray(modelOutput.content)) continue;
|
|
6119
|
+
for (const block of modelOutput.content) {
|
|
6120
|
+
if (!block || typeof block !== "object") continue;
|
|
6121
|
+
const content = block;
|
|
6122
|
+
if (content.type !== "text" || typeof content.text !== "string") continue;
|
|
6123
|
+
hasModelOutputText = true;
|
|
6124
|
+
text += content.text;
|
|
6125
|
+
if (!Array.isArray(content.annotations)) continue;
|
|
6126
|
+
for (const raw of content.annotations) {
|
|
6127
|
+
if (!raw || typeof raw !== "object") continue;
|
|
6128
|
+
const annotation = raw;
|
|
6129
|
+
if (annotation.type !== "word_info") continue;
|
|
6130
|
+
if (typeof annotation.text !== "string") continue;
|
|
6131
|
+
const start = secondsOffset(annotation.start_offset);
|
|
6132
|
+
const end = secondsOffset(annotation.end_offset);
|
|
6133
|
+
if (start === void 0 || end === void 0 || end < start) continue;
|
|
6134
|
+
words.push({
|
|
6135
|
+
text: annotation.text,
|
|
6136
|
+
start,
|
|
6137
|
+
end,
|
|
6138
|
+
...typeof annotation.speaker === "string" ? { speaker: annotation.speaker } : {}
|
|
6139
|
+
});
|
|
6140
|
+
}
|
|
6141
|
+
}
|
|
6142
|
+
}
|
|
6143
|
+
}
|
|
6144
|
+
if (!hasModelOutputText)
|
|
6145
|
+
throw new InvalidTranscriptionInputError(
|
|
6146
|
+
"Gemini transcription response did not contain model output text"
|
|
6147
|
+
);
|
|
6148
|
+
return { text, ...words.length ? { words } : {}, ...transcriptUsage(json.usage) };
|
|
6149
|
+
}
|
|
6150
|
+
fileFromJson(value, fallbackMimeType) {
|
|
6151
|
+
const file = "file" in value && value.file ? value.file : value;
|
|
6152
|
+
if (typeof file.name !== "string" || typeof file.uri !== "string")
|
|
6153
|
+
throw new InvalidTranscriptionInputError(
|
|
6154
|
+
"Gemini transcription upload did not return a file reference"
|
|
6155
|
+
);
|
|
6156
|
+
return {
|
|
6157
|
+
name: file.name,
|
|
6158
|
+
uri: file.uri,
|
|
6159
|
+
mimeType: typeof file.mimeType === "string" ? file.mimeType : typeof file.mime_type === "string" ? file.mime_type : fallbackMimeType,
|
|
6160
|
+
...typeof file.state === "string" ? { state: file.state } : {}
|
|
6161
|
+
};
|
|
6162
|
+
}
|
|
6163
|
+
async cleanup(fileName, key) {
|
|
6164
|
+
const controller = new AbortController();
|
|
6165
|
+
const timer = setTimeout(() => controller.abort(), CLEANUP_TIMEOUT_MS);
|
|
6166
|
+
try {
|
|
6167
|
+
const res = await fetchWithRetry(
|
|
6168
|
+
`${this.baseUrl}/${fileName}`,
|
|
6169
|
+
{ method: "DELETE", headers: { "x-goog-api-key": key }, signal: controller.signal },
|
|
6170
|
+
{ governor: this.governor, provider: this.name, maxRetries: 0 }
|
|
6171
|
+
);
|
|
6172
|
+
return res.ok ? "deleted" : "failed";
|
|
6173
|
+
} catch {
|
|
6174
|
+
return controller.signal.aborted ? "timed_out" : "failed";
|
|
6175
|
+
} finally {
|
|
6176
|
+
clearTimeout(timer);
|
|
6177
|
+
}
|
|
6178
|
+
}
|
|
6179
|
+
async resolveKey() {
|
|
6180
|
+
const key = await resolveApiKey(this.apiKeySource);
|
|
6181
|
+
if (!key)
|
|
6182
|
+
throw new Error("Google API key is required. Set GOOGLE_API_KEY or pass apiKey in options.");
|
|
6183
|
+
return key;
|
|
6184
|
+
}
|
|
6185
|
+
};
|
|
6186
|
+
|
|
6187
|
+
// src/providers/openrouter-transcription.ts
|
|
6188
|
+
var FORMAT_BY_MIME = {
|
|
6189
|
+
"audio/aac": "aac",
|
|
6190
|
+
"audio/flac": "flac",
|
|
6191
|
+
"audio/m4a": "m4a",
|
|
6192
|
+
"audio/mp3": "mp3",
|
|
6193
|
+
"audio/mp4": "mp4",
|
|
6194
|
+
"audio/mpeg": "mp3",
|
|
6195
|
+
"audio/ogg": "ogg",
|
|
6196
|
+
"audio/wav": "wav",
|
|
6197
|
+
"audio/webm": "webm"
|
|
6198
|
+
};
|
|
6199
|
+
var OpenRouterTranscriptionProvider = class {
|
|
6200
|
+
name = "openrouter-transcription";
|
|
6201
|
+
baseUrl;
|
|
6202
|
+
apiKeySource;
|
|
6203
|
+
governor;
|
|
6204
|
+
constructor(options = {}) {
|
|
6205
|
+
this.apiKeySource = options.apiKey ?? process.env.OPENROUTER_API_KEY ?? "";
|
|
6206
|
+
this.baseUrl = (options.baseUrl ?? process.env.OPENROUTER_BASE_URL ?? "https://openrouter.ai/api/v1").replace(/\/$/, "");
|
|
6207
|
+
assertSafeProviderBaseUrl(
|
|
6208
|
+
this.baseUrl,
|
|
6209
|
+
"OpenRouter transcription provider",
|
|
6210
|
+
options.dangerouslyAllowInsecureHttp
|
|
6211
|
+
);
|
|
6212
|
+
this.governor = options.rateLimit ? new RateLimiter(options.rateLimit) : void 0;
|
|
6213
|
+
}
|
|
6214
|
+
capabilities(model) {
|
|
6215
|
+
return model === OPENROUTER_TRANSCRIPTION_MODEL ? { sources: ["bytes", "base64"] } : void 0;
|
|
6216
|
+
}
|
|
6217
|
+
async transcribe(request) {
|
|
6218
|
+
requireExactTranscriptionModel(this.name, request.model, OPENROUTER_TRANSCRIPTION_MODEL);
|
|
6219
|
+
requireNoProviderFile(request.audio, this.name, request.model);
|
|
6220
|
+
if (request.audio.type === "provider-file")
|
|
6221
|
+
throw new UnsupportedTranscriptionInputError({
|
|
6222
|
+
provider: this.name,
|
|
6223
|
+
model: request.model,
|
|
6224
|
+
feature: "provider-file transcription input"
|
|
6225
|
+
});
|
|
6226
|
+
const options = assertOnlyProviderOptions(request, this.name, request.model, [
|
|
6227
|
+
"temperature",
|
|
6228
|
+
"provider"
|
|
6229
|
+
]);
|
|
6230
|
+
if (options.temperature !== void 0 && (typeof options.temperature !== "number" || !Number.isFinite(options.temperature) || options.temperature < 0 || options.temperature > 1))
|
|
6231
|
+
throw new InvalidTranscriptionInputError(
|
|
6232
|
+
"OpenRouter transcription temperature must be a number between 0 and 1"
|
|
6233
|
+
);
|
|
6234
|
+
if (options.provider !== void 0 && (!options.provider || Array.isArray(options.provider) || typeof options.provider !== "object"))
|
|
6235
|
+
throw new InvalidTranscriptionInputError(
|
|
6236
|
+
"OpenRouter transcription provider routing must be an object"
|
|
6237
|
+
);
|
|
6238
|
+
const format = FORMAT_BY_MIME[request.audio.mediaType];
|
|
6239
|
+
if (!format)
|
|
6240
|
+
throw new UnsupportedTranscriptionInputError({
|
|
6241
|
+
provider: this.name,
|
|
6242
|
+
model: request.model,
|
|
6243
|
+
feature: "this audio media type"
|
|
6244
|
+
});
|
|
6245
|
+
const bytes = inlineAudioBytes(request.audio);
|
|
6246
|
+
const body = {
|
|
6247
|
+
model: request.model,
|
|
6248
|
+
input_audio: { data: Buffer.from(bytes).toString("base64"), format },
|
|
6249
|
+
...request.language ? { language: request.language } : {},
|
|
6250
|
+
...options.temperature !== void 0 ? { temperature: options.temperature } : {},
|
|
6251
|
+
...options.provider !== void 0 ? { provider: options.provider } : {}
|
|
6252
|
+
};
|
|
6253
|
+
const key = await this.resolveKey();
|
|
6254
|
+
const res = await fetchWithRetry(
|
|
6255
|
+
`${this.baseUrl}/audio/transcriptions`,
|
|
6256
|
+
{
|
|
6257
|
+
method: "POST",
|
|
6258
|
+
headers: { Authorization: `Bearer ${key}`, "Content-Type": "application/json" },
|
|
6259
|
+
body: JSON.stringify(body),
|
|
6260
|
+
signal: request.signal
|
|
6261
|
+
},
|
|
6262
|
+
{ governor: this.governor, provider: this.name }
|
|
6263
|
+
);
|
|
6264
|
+
if (!res.ok) {
|
|
6265
|
+
const errorBody = await res.text();
|
|
6266
|
+
throw buildProviderError({
|
|
6267
|
+
provider: this.name,
|
|
6268
|
+
status: res.status,
|
|
6269
|
+
headers: res.headers,
|
|
6270
|
+
message: safeProviderMessage("OpenRouter", res.status, errorBody),
|
|
6271
|
+
body: errorBody
|
|
6272
|
+
});
|
|
6273
|
+
}
|
|
6274
|
+
const json = await res.json();
|
|
6275
|
+
if (typeof json.text !== "string")
|
|
6276
|
+
throw new InvalidTranscriptionInputError(
|
|
6277
|
+
"OpenRouter transcription response did not contain text"
|
|
6278
|
+
);
|
|
6279
|
+
return {
|
|
6280
|
+
transcript: { text: json.text, ...transcriptUsage(json.usage) },
|
|
6281
|
+
cleanupStatus: "not_required"
|
|
6282
|
+
};
|
|
6283
|
+
}
|
|
6284
|
+
async resolveKey() {
|
|
6285
|
+
const key = await resolveApiKey(this.apiKeySource);
|
|
6286
|
+
if (!key)
|
|
6287
|
+
throw new Error(
|
|
6288
|
+
"OpenRouter API key is required. Set OPENROUTER_API_KEY or pass apiKey in options."
|
|
6289
|
+
);
|
|
6290
|
+
return key;
|
|
6291
|
+
}
|
|
6292
|
+
};
|
|
6293
|
+
|
|
6294
|
+
// src/providers/transcription-registry.ts
|
|
6295
|
+
var builtinFactories2 = {
|
|
6296
|
+
"openai-transcription": (config) => {
|
|
6297
|
+
const opts = config.providers?.openai ?? {};
|
|
6298
|
+
return new OpenAITranscriptionProvider({
|
|
6299
|
+
apiKey: opts.apiKey,
|
|
6300
|
+
baseUrl: opts.baseUrl,
|
|
6301
|
+
dangerouslyAllowInsecureHttp: opts.dangerouslyAllowInsecureHttp,
|
|
6302
|
+
rateLimit: opts.rateLimit
|
|
6303
|
+
});
|
|
6304
|
+
},
|
|
6305
|
+
"gemini-transcription": (config) => {
|
|
6306
|
+
const opts = config.providers?.google ?? {};
|
|
6307
|
+
return new GeminiTranscriptionProvider({
|
|
6308
|
+
apiKey: opts.apiKey,
|
|
6309
|
+
baseUrl: opts.baseUrl,
|
|
6310
|
+
dangerouslyAllowInsecureHttp: opts.dangerouslyAllowInsecureHttp,
|
|
6311
|
+
rateLimit: opts.rateLimit
|
|
6312
|
+
});
|
|
6313
|
+
},
|
|
6314
|
+
"openrouter-transcription": (config) => {
|
|
6315
|
+
const opts = config.providers?.openrouter ?? {};
|
|
6316
|
+
return new OpenRouterTranscriptionProvider({
|
|
6317
|
+
apiKey: opts.apiKey,
|
|
6318
|
+
baseUrl: opts.baseUrl,
|
|
6319
|
+
dangerouslyAllowInsecureHttp: opts.dangerouslyAllowInsecureHttp,
|
|
6320
|
+
rateLimit: opts.rateLimit
|
|
6321
|
+
});
|
|
6322
|
+
}
|
|
6323
|
+
};
|
|
6324
|
+
var TranscriptionProviderRegistry = class {
|
|
6325
|
+
instances = /* @__PURE__ */ new Map();
|
|
6326
|
+
factories = /* @__PURE__ */ new Map();
|
|
6327
|
+
constructor() {
|
|
6328
|
+
for (const [name, factory] of Object.entries(builtinFactories2))
|
|
6329
|
+
this.factories.set(name, factory);
|
|
6330
|
+
}
|
|
6331
|
+
register(name, factory) {
|
|
6332
|
+
this.factories.set(name, factory);
|
|
6333
|
+
this.instances.delete(name);
|
|
6334
|
+
}
|
|
6335
|
+
registerInstance(name, provider) {
|
|
6336
|
+
this.instances.set(name, provider);
|
|
6337
|
+
}
|
|
6338
|
+
has(name) {
|
|
6339
|
+
return this.instances.has(name) || this.factories.has(name);
|
|
6340
|
+
}
|
|
6341
|
+
get(name, config = {}) {
|
|
6342
|
+
const instance = this.instances.get(name);
|
|
6343
|
+
if (instance) return instance;
|
|
6344
|
+
const factory = this.factories.get(name);
|
|
6345
|
+
if (factory) {
|
|
6346
|
+
const created = factory(config);
|
|
6347
|
+
this.instances.set(name, created);
|
|
6348
|
+
return created;
|
|
6349
|
+
}
|
|
6350
|
+
throw new UnsupportedTranscriptionInputError({
|
|
6351
|
+
provider: name,
|
|
6352
|
+
model: "",
|
|
6353
|
+
feature: "transcription"
|
|
6354
|
+
});
|
|
6355
|
+
}
|
|
4262
6356
|
resolve(uri, config = {}) {
|
|
4263
|
-
|
|
4264
|
-
|
|
4265
|
-
|
|
4266
|
-
|
|
4267
|
-
|
|
4268
|
-
|
|
4269
|
-
|
|
4270
|
-
const providerName2 = config.defaultProvider ?? "openai";
|
|
4271
|
-
return { provider: this.get(providerName2, config), model: uri };
|
|
4272
|
-
}
|
|
4273
|
-
const providerName = uri.slice(0, colonIndex);
|
|
4274
|
-
const model = uri.slice(colonIndex + 1);
|
|
4275
|
-
if (!providerName || !model) {
|
|
4276
|
-
throw new Error(
|
|
4277
|
-
`Invalid provider URI "${uri}". Expected format: "provider:model" (e.g. "openai:gpt-4o")`
|
|
4278
|
-
);
|
|
6357
|
+
const colon = typeof uri === "string" ? uri.indexOf(":") : -1;
|
|
6358
|
+
if (colon <= 0 || colon === uri.length - 1 || uri.indexOf(":", colon + 1) !== -1) {
|
|
6359
|
+
throw new UnsupportedTranscriptionInputError({
|
|
6360
|
+
provider: "unknown",
|
|
6361
|
+
model: "",
|
|
6362
|
+
feature: "an explicit transcription provider:model URI"
|
|
6363
|
+
});
|
|
4279
6364
|
}
|
|
4280
|
-
|
|
4281
|
-
|
|
4282
|
-
|
|
4283
|
-
* Clear all cached provider instances. Useful for testing or reconfiguration.
|
|
4284
|
-
*/
|
|
4285
|
-
clearCache() {
|
|
4286
|
-
this.instances.clear();
|
|
4287
|
-
}
|
|
4288
|
-
/**
|
|
4289
|
-
* Clear all registered factories (including built-ins).
|
|
4290
|
-
* Useful for test runtimes where only explicitly registered instances should be used.
|
|
4291
|
-
*/
|
|
4292
|
-
clearFactories() {
|
|
4293
|
-
this.factories.clear();
|
|
6365
|
+
const providerName = uri.slice(0, colon);
|
|
6366
|
+
const model = uri.slice(colon + 1);
|
|
6367
|
+
return { provider: this.get(providerName, config), model, providerName };
|
|
4294
6368
|
}
|
|
4295
6369
|
};
|
|
4296
|
-
var defaultRegistry = new ProviderRegistry();
|
|
4297
6370
|
|
|
4298
6371
|
// src/context.ts
|
|
4299
6372
|
import { AsyncLocalStorage } from "async_hooks";
|
|
4300
6373
|
import { randomUUID } from "crypto";
|
|
4301
6374
|
import { z, ZodError as ZodError2 } from "zod";
|
|
4302
6375
|
|
|
6376
|
+
// src/input.ts
|
|
6377
|
+
var MAX_INLINE_MODEL_INPUT_BYTES = 25 * 1024 * 1024;
|
|
6378
|
+
function invalid(message) {
|
|
6379
|
+
throw new InvalidModelInputError(message);
|
|
6380
|
+
}
|
|
6381
|
+
function nonEmptyString2(value, field) {
|
|
6382
|
+
if (typeof value !== "string" || value.length === 0)
|
|
6383
|
+
invalid(`${field} must be a non-empty string`);
|
|
6384
|
+
return value;
|
|
6385
|
+
}
|
|
6386
|
+
function mediaType(value, field) {
|
|
6387
|
+
return nonEmptyString2(value, field);
|
|
6388
|
+
}
|
|
6389
|
+
function validBase64(value) {
|
|
6390
|
+
return value.length > 0 && value.length % 4 === 0 && /^[A-Za-z0-9+/]*={0,2}$/.test(value);
|
|
6391
|
+
}
|
|
6392
|
+
function decodedBase64Bytes(value) {
|
|
6393
|
+
return Math.floor(value.length * 3 / 4) - (value.endsWith("==") ? 2 : value.endsWith("=") ? 1 : 0);
|
|
6394
|
+
}
|
|
6395
|
+
function normalizeModelInput(input) {
|
|
6396
|
+
if (typeof input === "string") return input;
|
|
6397
|
+
if (!Array.isArray(input) || input.length === 0)
|
|
6398
|
+
invalid("ModelInput parts must be a non-empty array");
|
|
6399
|
+
let inlineBytes = 0;
|
|
6400
|
+
const reserveInlineBytes = (bytes) => {
|
|
6401
|
+
if (bytes > MAX_INLINE_MODEL_INPUT_BYTES - inlineBytes) {
|
|
6402
|
+
invalid(
|
|
6403
|
+
"Inline image data must not exceed 25 MiB total; use a URL or provider-file source where supported"
|
|
6404
|
+
);
|
|
6405
|
+
}
|
|
6406
|
+
inlineBytes += bytes;
|
|
6407
|
+
};
|
|
6408
|
+
return input.map((part, index) => {
|
|
6409
|
+
if (!part || typeof part !== "object") invalid(`ModelInput part ${index} must be an object`);
|
|
6410
|
+
if (part.type === "text")
|
|
6411
|
+
return { type: "text", text: nonEmptyString2(part.text, `part ${index}.text`) };
|
|
6412
|
+
if (part.type !== "image") invalid(`part ${index}.type must be 'text' or 'image'`);
|
|
6413
|
+
const source = part.source;
|
|
6414
|
+
if (!source || typeof source !== "object") invalid(`part ${index}.source must be an object`);
|
|
6415
|
+
let clone;
|
|
6416
|
+
switch (source.type) {
|
|
6417
|
+
case "url": {
|
|
6418
|
+
const url = nonEmptyString2(source.url, `part ${index}.source.url`);
|
|
6419
|
+
let parsed;
|
|
6420
|
+
try {
|
|
6421
|
+
parsed = new URL(url);
|
|
6422
|
+
} catch {
|
|
6423
|
+
invalid(`part ${index}.source.url must be an http(s) URL`);
|
|
6424
|
+
}
|
|
6425
|
+
if (parsed.protocol !== "http:" && parsed.protocol !== "https:")
|
|
6426
|
+
invalid(`part ${index}.source.url must be an http(s) URL`);
|
|
6427
|
+
clone = {
|
|
6428
|
+
type: "url",
|
|
6429
|
+
url,
|
|
6430
|
+
...source.mediaType === void 0 ? {} : { mediaType: mediaType(source.mediaType, `part ${index}.source.mediaType`) }
|
|
6431
|
+
};
|
|
6432
|
+
break;
|
|
6433
|
+
}
|
|
6434
|
+
case "bytes":
|
|
6435
|
+
if (!(source.data instanceof Uint8Array) || source.data.byteLength === 0)
|
|
6436
|
+
invalid(`part ${index}.source.data must be a non-empty Uint8Array`);
|
|
6437
|
+
reserveInlineBytes(source.data.byteLength);
|
|
6438
|
+
clone = {
|
|
6439
|
+
type: "bytes",
|
|
6440
|
+
data: source.data.slice(),
|
|
6441
|
+
mediaType: mediaType(source.mediaType, `part ${index}.source.mediaType`)
|
|
6442
|
+
};
|
|
6443
|
+
break;
|
|
6444
|
+
case "base64": {
|
|
6445
|
+
const data = nonEmptyString2(source.data, `part ${index}.source.data`);
|
|
6446
|
+
if (data.length > 4 * Math.ceil(MAX_INLINE_MODEL_INPUT_BYTES / 3)) {
|
|
6447
|
+
invalid(
|
|
6448
|
+
"Inline image data must not exceed 25 MiB total; use a URL or provider-file source where supported"
|
|
6449
|
+
);
|
|
6450
|
+
}
|
|
6451
|
+
if (!validBase64(data)) invalid(`part ${index}.source.data must be valid base64`);
|
|
6452
|
+
reserveInlineBytes(decodedBase64Bytes(data));
|
|
6453
|
+
clone = {
|
|
6454
|
+
type: "base64",
|
|
6455
|
+
data,
|
|
6456
|
+
mediaType: mediaType(source.mediaType, `part ${index}.source.mediaType`)
|
|
6457
|
+
};
|
|
6458
|
+
break;
|
|
6459
|
+
}
|
|
6460
|
+
case "provider-file":
|
|
6461
|
+
clone = {
|
|
6462
|
+
type: "provider-file",
|
|
6463
|
+
provider: nonEmptyString2(source.provider, `part ${index}.source.provider`),
|
|
6464
|
+
reference: nonEmptyString2(source.reference, `part ${index}.source.reference`),
|
|
6465
|
+
...source.mediaType === void 0 ? {} : { mediaType: mediaType(source.mediaType, `part ${index}.source.mediaType`) }
|
|
6466
|
+
};
|
|
6467
|
+
break;
|
|
6468
|
+
default:
|
|
6469
|
+
invalid(`part ${index}.source.type is unsupported`);
|
|
6470
|
+
}
|
|
6471
|
+
return {
|
|
6472
|
+
type: "image",
|
|
6473
|
+
source: clone,
|
|
6474
|
+
...part.label === void 0 ? {} : { label: nonEmptyString2(part.label, `part ${index}.label`) }
|
|
6475
|
+
};
|
|
6476
|
+
});
|
|
6477
|
+
}
|
|
6478
|
+
function inputText(input) {
|
|
6479
|
+
return typeof input === "string" ? input : input.filter((part) => part.type === "text").map((part) => part.text).join("\n");
|
|
6480
|
+
}
|
|
6481
|
+
function cloneModelInput(input) {
|
|
6482
|
+
return typeof input === "string" ? input : normalizeModelInput(input);
|
|
6483
|
+
}
|
|
6484
|
+
function describeModelInput(input) {
|
|
6485
|
+
if (typeof input === "string") return void 0;
|
|
6486
|
+
return {
|
|
6487
|
+
parts: input.map((part) => {
|
|
6488
|
+
if (part.type === "text") return { type: "text", characters: part.text.length };
|
|
6489
|
+
const { source } = part;
|
|
6490
|
+
return {
|
|
6491
|
+
type: "image",
|
|
6492
|
+
source: source.type,
|
|
6493
|
+
...source.mediaType ? { mediaType: source.mediaType } : {},
|
|
6494
|
+
...source.type === "bytes" ? { bytes: source.data.byteLength } : {},
|
|
6495
|
+
...source.type === "base64" ? {
|
|
6496
|
+
bytes: decodedBase64Bytes(source.data)
|
|
6497
|
+
} : {},
|
|
6498
|
+
...(source.type === "url" ? source.url : source.type === "provider-file" ? source.reference : void 0) ? {
|
|
6499
|
+
locator: source.type === "url" ? source.url : source.type === "provider-file" ? source.reference : void 0
|
|
6500
|
+
} : {},
|
|
6501
|
+
...part.label ? { label: part.label } : {}
|
|
6502
|
+
};
|
|
6503
|
+
})
|
|
6504
|
+
};
|
|
6505
|
+
}
|
|
6506
|
+
function summarizeModelInput(input) {
|
|
6507
|
+
if (typeof input === "string") return input;
|
|
6508
|
+
return input.map(
|
|
6509
|
+
(part) => part.type === "text" ? part.text : `[image ${part.source.mediaType ?? "media"}]`
|
|
6510
|
+
).join("\n");
|
|
6511
|
+
}
|
|
6512
|
+
function sanitizeModelInputForTrace(input) {
|
|
6513
|
+
return inputText(input);
|
|
6514
|
+
}
|
|
6515
|
+
function normalizePersistedSessionHistory(history) {
|
|
6516
|
+
return history.map((message, index) => {
|
|
6517
|
+
if (typeof message.content === "string") return { ...message };
|
|
6518
|
+
if (message.role !== "user") {
|
|
6519
|
+
throw new InvalidModelInputError(
|
|
6520
|
+
`History message ${index} has non-text content on a non-user role`
|
|
6521
|
+
);
|
|
6522
|
+
}
|
|
6523
|
+
const content = normalizeModelInput(message.content);
|
|
6524
|
+
if (typeof content !== "string" && content.some((part) => part.type === "image" && part.source.type === "bytes")) {
|
|
6525
|
+
throw new InvalidModelInputError(
|
|
6526
|
+
"Uint8Array image input cannot be persisted in session history"
|
|
6527
|
+
);
|
|
6528
|
+
}
|
|
6529
|
+
return { ...message, content };
|
|
6530
|
+
});
|
|
6531
|
+
}
|
|
6532
|
+
|
|
4303
6533
|
// src/partial-json.ts
|
|
4304
6534
|
var MAX_DEPTH = 256;
|
|
4305
6535
|
function parsePartialJson(input) {
|
|
@@ -4871,7 +7101,8 @@ var COST_BEARING_LEAF_TYPES = [
|
|
|
4871
7101
|
"agent_call_end",
|
|
4872
7102
|
"tool_call_end",
|
|
4873
7103
|
"memory_remember",
|
|
4874
|
-
"memory_recall"
|
|
7104
|
+
"memory_recall",
|
|
7105
|
+
"transcription_end"
|
|
4875
7106
|
];
|
|
4876
7107
|
var COST_LEAF_SET = new Set(COST_BEARING_LEAF_TYPES);
|
|
4877
7108
|
function hasPositiveTokens(event) {
|
|
@@ -4879,6 +7110,12 @@ function hasPositiveTokens(event) {
|
|
|
4879
7110
|
if (!t) return false;
|
|
4880
7111
|
return (t.input ?? 0) + (t.output ?? 0) + (t.reasoning ?? 0) + (t.cached ?? 0) + (t.cacheWrite ?? 0) > 0;
|
|
4881
7112
|
}
|
|
7113
|
+
function hasPositiveBillableWork(event) {
|
|
7114
|
+
if (hasPositiveTokens(event)) return true;
|
|
7115
|
+
const seconds = event.data?.usage?.audioSeconds;
|
|
7116
|
+
const totalTokens = event.data?.usage?.totalTokens;
|
|
7117
|
+
return typeof seconds === "number" && Number.isFinite(seconds) && seconds > 0 || typeof totalTokens === "number" && Number.isFinite(totalTokens) && totalTokens > 0;
|
|
7118
|
+
}
|
|
4882
7119
|
function isUsableCost(cost) {
|
|
4883
7120
|
return typeof cost === "number" && Number.isFinite(cost) && cost >= 0;
|
|
4884
7121
|
}
|
|
@@ -4891,7 +7128,7 @@ function isCostBearingLeaf(event) {
|
|
|
4891
7128
|
return COST_LEAF_SET.has(event.type);
|
|
4892
7129
|
}
|
|
4893
7130
|
function isUnpricedLeaf(event) {
|
|
4894
|
-
return event.type !== void 0 && COST_LEAF_SET.has(event.type) && !isUsableCost(event.cost) &&
|
|
7131
|
+
return event.type !== void 0 && COST_LEAF_SET.has(event.type) && !isUsableCost(event.cost) && hasPositiveBillableWork(event);
|
|
4895
7132
|
}
|
|
4896
7133
|
function isRootLevel(event) {
|
|
4897
7134
|
const d = event.depth;
|
|
@@ -4906,6 +7143,9 @@ var AXL_EVENT_TYPES = [
|
|
|
4906
7143
|
// Workflow lifecycle
|
|
4907
7144
|
"workflow_start",
|
|
4908
7145
|
"workflow_end",
|
|
7146
|
+
// Transcription operation lifecycle (separate from chat/agent turns)
|
|
7147
|
+
"transcription_start",
|
|
7148
|
+
"transcription_end",
|
|
4909
7149
|
// Ask boundary
|
|
4910
7150
|
"ask_start",
|
|
4911
7151
|
"ask_end",
|
|
@@ -4996,6 +7236,8 @@ var LIFECYCLE_TYPES = /* @__PURE__ */ new Set([
|
|
|
4996
7236
|
"schema_diagnostic",
|
|
4997
7237
|
"workflow_start",
|
|
4998
7238
|
"workflow_end",
|
|
7239
|
+
"transcription_start",
|
|
7240
|
+
"transcription_end",
|
|
4999
7241
|
"checkpoint_save",
|
|
5000
7242
|
"checkpoint_replay",
|
|
5001
7243
|
"await_human",
|
|
@@ -5811,6 +8053,21 @@ function redactGate(event) {
|
|
|
5811
8053
|
}
|
|
5812
8054
|
};
|
|
5813
8055
|
}
|
|
8056
|
+
function redactInputDescriptor(input) {
|
|
8057
|
+
return {
|
|
8058
|
+
...input,
|
|
8059
|
+
parts: input.parts.map((part) => {
|
|
8060
|
+
if (!part || typeof part !== "object" || !("type" in part) || part.type === "text")
|
|
8061
|
+
return part;
|
|
8062
|
+
const value = part;
|
|
8063
|
+
return {
|
|
8064
|
+
...part,
|
|
8065
|
+
...value.locator !== void 0 ? { locator: REDACTED } : {},
|
|
8066
|
+
...value.label !== void 0 ? { label: REDACTED } : {}
|
|
8067
|
+
};
|
|
8068
|
+
})
|
|
8069
|
+
};
|
|
8070
|
+
}
|
|
5814
8071
|
function redactMemory(event) {
|
|
5815
8072
|
const d = event.data;
|
|
5816
8073
|
if (!d || typeof d !== "object") return event;
|
|
@@ -5834,7 +8091,11 @@ var LEGACY_REDACTION_RULES = {
|
|
|
5834
8091
|
}
|
|
5835
8092
|
};
|
|
5836
8093
|
},
|
|
5837
|
-
ask_start: (e) => ({
|
|
8094
|
+
ask_start: (e) => ({
|
|
8095
|
+
...e,
|
|
8096
|
+
prompt: REDACTED,
|
|
8097
|
+
...e.input ? { input: redactInputDescriptor(e.input) } : {}
|
|
8098
|
+
}),
|
|
5838
8099
|
ask_end: (e) => ({
|
|
5839
8100
|
...e,
|
|
5840
8101
|
outcome: e.outcome.ok ? { ok: true, result: REDACTED } : { ok: false, error: REDACTED }
|
|
@@ -5842,6 +8103,14 @@ var LEGACY_REDACTION_RULES = {
|
|
|
5842
8103
|
agent_call_start: (e) => {
|
|
5843
8104
|
const d = e.data;
|
|
5844
8105
|
const out = { ...d, prompt: REDACTED };
|
|
8106
|
+
if (d.input) {
|
|
8107
|
+
out.input = redactInputDescriptor(d.input);
|
|
8108
|
+
}
|
|
8109
|
+
if (d.messageInputs)
|
|
8110
|
+
out.messageInputs = d.messageInputs.map((entry) => ({
|
|
8111
|
+
...entry,
|
|
8112
|
+
input: redactInputDescriptor(entry.input)
|
|
8113
|
+
}));
|
|
5845
8114
|
if (d.system !== void 0) out.system = REDACTED;
|
|
5846
8115
|
if (Array.isArray(d.messages)) {
|
|
5847
8116
|
out.messages = [{ role: "system", content: `[${d.messages.length} messages redacted]` }];
|
|
@@ -5997,6 +8266,18 @@ function reuseLegacyRule(type) {
|
|
|
5997
8266
|
var COMMON_V2_REDACTION_RULES = {
|
|
5998
8267
|
workflow_start: reuseLegacyRule("workflow_start"),
|
|
5999
8268
|
workflow_end: reuseLegacyRule("workflow_end"),
|
|
8269
|
+
transcription_start: (e) => e,
|
|
8270
|
+
transcription_end: (e) => {
|
|
8271
|
+
const d = e.data;
|
|
8272
|
+
return {
|
|
8273
|
+
...e,
|
|
8274
|
+
data: {
|
|
8275
|
+
...d,
|
|
8276
|
+
...d.text !== void 0 ? { text: REDACTED } : {},
|
|
8277
|
+
...d.error !== void 0 ? { error: REDACTED } : {}
|
|
8278
|
+
}
|
|
8279
|
+
};
|
|
8280
|
+
},
|
|
6000
8281
|
ask_start: reuseLegacyRule("ask_start"),
|
|
6001
8282
|
ask_end: reuseLegacyRule("ask_end"),
|
|
6002
8283
|
agent_call_start: reuseLegacyRule("agent_call_start"),
|
|
@@ -6191,7 +8472,7 @@ function resolveConfig(config) {
|
|
|
6191
8472
|
|
|
6192
8473
|
// src/tool-model-output.ts
|
|
6193
8474
|
import { types as nodeTypes } from "util";
|
|
6194
|
-
function
|
|
8475
|
+
function invalid2(path, reason) {
|
|
6195
8476
|
return new Error(`Invalid tool model output at ${path}: ${reason}`);
|
|
6196
8477
|
}
|
|
6197
8478
|
function propertyPath(parent, key) {
|
|
@@ -6209,14 +8490,14 @@ function propertyPath(parent, key) {
|
|
|
6209
8490
|
function normalize(value, path, stack) {
|
|
6210
8491
|
if (value === null || typeof value === "string" || typeof value === "boolean") return value;
|
|
6211
8492
|
if (typeof value === "number") {
|
|
6212
|
-
if (!Number.isFinite(value)) throw
|
|
8493
|
+
if (!Number.isFinite(value)) throw invalid2(path, "number must be finite");
|
|
6213
8494
|
return value;
|
|
6214
8495
|
}
|
|
6215
|
-
if (value === void 0) throw
|
|
8496
|
+
if (value === void 0) throw invalid2(path, "undefined is only allowed on object properties");
|
|
6216
8497
|
if (typeof value === "bigint" || typeof value === "function" || typeof value === "symbol") {
|
|
6217
|
-
throw
|
|
8498
|
+
throw invalid2(path, `unsupported ${typeof value}`);
|
|
6218
8499
|
}
|
|
6219
|
-
if (typeof value !== "object") throw
|
|
8500
|
+
if (typeof value !== "object") throw invalid2(path, `unsupported ${typeof value}`);
|
|
6220
8501
|
let prototype;
|
|
6221
8502
|
let descriptors;
|
|
6222
8503
|
try {
|
|
@@ -6224,38 +8505,38 @@ function normalize(value, path, stack) {
|
|
|
6224
8505
|
descriptors = Object.getOwnPropertyDescriptors(value);
|
|
6225
8506
|
Object.setPrototypeOf(descriptors, null);
|
|
6226
8507
|
} catch (cause) {
|
|
6227
|
-
throw
|
|
8508
|
+
throw invalid2(
|
|
6228
8509
|
path,
|
|
6229
8510
|
`object inspection failed (${cause instanceof Error ? cause.name : "error"})`
|
|
6230
8511
|
);
|
|
6231
8512
|
}
|
|
6232
|
-
if (stack.has(value)) throw
|
|
8513
|
+
if (stack.has(value)) throw invalid2(path, "cyclic reference");
|
|
6233
8514
|
for (const key of Reflect.ownKeys(descriptors)) {
|
|
6234
8515
|
const descriptor = descriptors[key];
|
|
6235
8516
|
if (descriptor && !("value" in descriptor)) {
|
|
6236
|
-
throw
|
|
8517
|
+
throw invalid2(
|
|
6237
8518
|
typeof key === "string" ? propertyPath(path, key) : `${path}[symbol]`,
|
|
6238
8519
|
"accessor properties are not supported"
|
|
6239
8520
|
);
|
|
6240
8521
|
}
|
|
6241
8522
|
if (typeof key === "symbol") {
|
|
6242
|
-
throw
|
|
8523
|
+
throw invalid2(`${path}[symbol]`, "symbol keys are not supported");
|
|
6243
8524
|
}
|
|
6244
8525
|
}
|
|
6245
8526
|
if (typeof descriptors.toJSON?.value === "function") {
|
|
6246
|
-
throw
|
|
8527
|
+
throw invalid2(propertyPath(path, "toJSON"), "custom toJSON is not supported");
|
|
6247
8528
|
}
|
|
6248
8529
|
const array = Array.isArray(value);
|
|
6249
8530
|
if (array && prototype !== Array.prototype && prototype !== null) {
|
|
6250
|
-
throw
|
|
8531
|
+
throw invalid2(path, "array subclasses are not supported");
|
|
6251
8532
|
}
|
|
6252
8533
|
const inheritedToJSON = prototype === Array.prototype || prototype === Object.prototype ? Object.getOwnPropertyDescriptor(prototype, "toJSON") : void 0;
|
|
6253
8534
|
if (typeof inheritedToJSON?.value === "function") {
|
|
6254
|
-
throw
|
|
8535
|
+
throw invalid2(propertyPath(path, "toJSON"), "inherited custom toJSON is not supported");
|
|
6255
8536
|
}
|
|
6256
8537
|
const inheritedThen = prototype === Array.prototype || prototype === Object.prototype ? Object.getOwnPropertyDescriptor(prototype, "then") : void 0;
|
|
6257
8538
|
if (typeof descriptors.then?.value === "function" || inheritedThen !== void 0 && (!("value" in inheritedThen) || typeof inheritedThen.value === "function")) {
|
|
6258
|
-
throw
|
|
8539
|
+
throw invalid2(path, "thenable output is not supported");
|
|
6259
8540
|
}
|
|
6260
8541
|
stack.add(value);
|
|
6261
8542
|
try {
|
|
@@ -6263,15 +8544,15 @@ function normalize(value, path, stack) {
|
|
|
6263
8544
|
const normalized2 = [];
|
|
6264
8545
|
const length = descriptors.length?.value;
|
|
6265
8546
|
if (!Number.isSafeInteger(length) || length < 0) {
|
|
6266
|
-
throw
|
|
8547
|
+
throw invalid2(propertyPath(path, "length"), "array length is invalid");
|
|
6267
8548
|
}
|
|
6268
8549
|
for (let index = 0; index < length; index++) {
|
|
6269
8550
|
const descriptor = descriptors[String(index)];
|
|
6270
8551
|
if (!descriptor || !("value" in descriptor)) {
|
|
6271
|
-
throw
|
|
8552
|
+
throw invalid2(`${path}[${index}]`, "arrays must be dense data properties");
|
|
6272
8553
|
}
|
|
6273
8554
|
if (descriptor.value === void 0) {
|
|
6274
|
-
throw
|
|
8555
|
+
throw invalid2(`${path}[${index}]`, "array values cannot be undefined");
|
|
6275
8556
|
}
|
|
6276
8557
|
normalized2.push(normalize(descriptor.value, `${path}[${index}]`, stack));
|
|
6277
8558
|
}
|
|
@@ -6280,21 +8561,21 @@ function normalize(value, path, stack) {
|
|
|
6280
8561
|
if (!descriptor?.enumerable) continue;
|
|
6281
8562
|
const index = Number(key);
|
|
6282
8563
|
if (!Number.isInteger(index) || index < 0 || index >= length || String(index) !== key) {
|
|
6283
|
-
throw
|
|
8564
|
+
throw invalid2(propertyPath(path, key), "non-index array properties are not supported");
|
|
6284
8565
|
}
|
|
6285
8566
|
}
|
|
6286
8567
|
Object.setPrototypeOf(normalized2, null);
|
|
6287
8568
|
return normalized2;
|
|
6288
8569
|
}
|
|
6289
8570
|
if (prototype !== Object.prototype && prototype !== null) {
|
|
6290
|
-
throw
|
|
8571
|
+
throw invalid2(path, "only plain objects and null-prototype records are supported");
|
|
6291
8572
|
}
|
|
6292
8573
|
const normalized = /* @__PURE__ */ Object.create(null);
|
|
6293
8574
|
for (const key of Object.keys(descriptors)) {
|
|
6294
8575
|
const descriptor = descriptors[key];
|
|
6295
8576
|
if (!descriptor?.enumerable) continue;
|
|
6296
8577
|
if (!("value" in descriptor)) {
|
|
6297
|
-
throw
|
|
8578
|
+
throw invalid2(propertyPath(path, key), "accessor properties are not supported");
|
|
6298
8579
|
}
|
|
6299
8580
|
if (descriptor.value === void 0) continue;
|
|
6300
8581
|
normalized[key] = normalize(descriptor.value, propertyPath(path, key), stack);
|
|
@@ -6330,7 +8611,15 @@ function cloneToolArguments(value) {
|
|
|
6330
8611
|
}
|
|
6331
8612
|
}
|
|
6332
8613
|
function parseToolInvocation(options) {
|
|
6333
|
-
const {
|
|
8614
|
+
const {
|
|
8615
|
+
toolCall,
|
|
8616
|
+
configuredTool,
|
|
8617
|
+
override,
|
|
8618
|
+
mcpCall,
|
|
8619
|
+
mcpTraceName,
|
|
8620
|
+
availableTools,
|
|
8621
|
+
providerVisibleSchema
|
|
8622
|
+
} = options;
|
|
6334
8623
|
const requestedTool = toolCall.function.name;
|
|
6335
8624
|
if (!override && !configuredTool && !mcpCall) {
|
|
6336
8625
|
return {
|
|
@@ -6390,7 +8679,7 @@ function parseToolInvocation(options) {
|
|
|
6390
8679
|
args: parsedArguments,
|
|
6391
8680
|
issues: toolArgumentIssues(error)
|
|
6392
8681
|
},
|
|
6393
|
-
modelMessage: INVALID_ARGUMENTS_MODEL_MESSAGE
|
|
8682
|
+
modelMessage: toolArgumentModelMessage(error, providerVisibleSchema) ?? INVALID_ARGUMENTS_MODEL_MESSAGE
|
|
6394
8683
|
};
|
|
6395
8684
|
}
|
|
6396
8685
|
}
|
|
@@ -6794,44 +9083,44 @@ function resolveBranchDrainTimeoutMs(value) {
|
|
|
6794
9083
|
return resolved;
|
|
6795
9084
|
}
|
|
6796
9085
|
function parseHumanDecision(value) {
|
|
6797
|
-
const
|
|
9086
|
+
const invalid3 = () => {
|
|
6798
9087
|
throw new AxlError(
|
|
6799
9088
|
"INVALID_HUMAN_DECISION",
|
|
6800
9089
|
"Human decision must be a plain object with a boolean approved field and only branch-appropriate string data"
|
|
6801
9090
|
);
|
|
6802
9091
|
};
|
|
6803
|
-
if (typeof value !== "object" || value === null || Array.isArray(value))
|
|
9092
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) invalid3();
|
|
6804
9093
|
let prototype;
|
|
6805
9094
|
try {
|
|
6806
9095
|
prototype = Object.getPrototypeOf(value);
|
|
6807
9096
|
} catch {
|
|
6808
|
-
return
|
|
9097
|
+
return invalid3();
|
|
6809
9098
|
}
|
|
6810
|
-
if (prototype !== Object.prototype && prototype !== null)
|
|
9099
|
+
if (prototype !== Object.prototype && prototype !== null) invalid3();
|
|
6811
9100
|
let ownKeys;
|
|
6812
9101
|
try {
|
|
6813
9102
|
ownKeys = Reflect.ownKeys(value);
|
|
6814
9103
|
} catch {
|
|
6815
|
-
return
|
|
9104
|
+
return invalid3();
|
|
6816
9105
|
}
|
|
6817
9106
|
if (ownKeys.some(
|
|
6818
9107
|
(key) => typeof key !== "string" || key !== "approved" && key !== "data" && key !== "reason"
|
|
6819
9108
|
)) {
|
|
6820
|
-
|
|
9109
|
+
invalid3();
|
|
6821
9110
|
}
|
|
6822
9111
|
const readDataProperty = (key) => {
|
|
6823
9112
|
let descriptor;
|
|
6824
9113
|
try {
|
|
6825
9114
|
descriptor = Object.getOwnPropertyDescriptor(value, key);
|
|
6826
9115
|
} catch {
|
|
6827
|
-
return
|
|
9116
|
+
return invalid3();
|
|
6828
9117
|
}
|
|
6829
9118
|
if (!descriptor) {
|
|
6830
|
-
if (ownKeys.includes(key))
|
|
9119
|
+
if (ownKeys.includes(key)) invalid3();
|
|
6831
9120
|
return { present: false, value: void 0 };
|
|
6832
9121
|
}
|
|
6833
|
-
if (!ownKeys.includes(key)) return
|
|
6834
|
-
if (!("value" in descriptor)) return
|
|
9122
|
+
if (!ownKeys.includes(key)) return invalid3();
|
|
9123
|
+
if (!("value" in descriptor)) return invalid3();
|
|
6835
9124
|
return { present: true, value: descriptor.value };
|
|
6836
9125
|
};
|
|
6837
9126
|
const approved = readDataProperty("approved");
|
|
@@ -6850,7 +9139,7 @@ function parseHumanDecision(value) {
|
|
|
6850
9139
|
};
|
|
6851
9140
|
}
|
|
6852
9141
|
}
|
|
6853
|
-
return
|
|
9142
|
+
return invalid3();
|
|
6854
9143
|
}
|
|
6855
9144
|
var askStorage = new AsyncLocalStorage();
|
|
6856
9145
|
var inlineSchemaCache = /* @__PURE__ */ new WeakMap();
|
|
@@ -6958,8 +9247,12 @@ function appendRetryMessages(messages, content, feedbackMessage, providerMetadat
|
|
|
6958
9247
|
}
|
|
6959
9248
|
function estimateMessagesTokens(messages) {
|
|
6960
9249
|
let total = 0;
|
|
9250
|
+
let unmeasured = false;
|
|
6961
9251
|
for (const msg of messages) {
|
|
6962
|
-
total += estimateTokens(msg.content);
|
|
9252
|
+
total += estimateTokens(inputText(msg.content));
|
|
9253
|
+
if (typeof msg.content !== "string") {
|
|
9254
|
+
unmeasured ||= msg.content.some((part) => part.type === "image");
|
|
9255
|
+
}
|
|
6963
9256
|
if (msg.tool_calls) {
|
|
6964
9257
|
for (const tc of msg.tool_calls) {
|
|
6965
9258
|
total += estimateTokens(tc.function.name + tc.function.arguments);
|
|
@@ -6967,7 +9260,73 @@ function estimateMessagesTokens(messages) {
|
|
|
6967
9260
|
}
|
|
6968
9261
|
total += 4;
|
|
6969
9262
|
}
|
|
6970
|
-
return total;
|
|
9263
|
+
return { tokens: total, unmeasured };
|
|
9264
|
+
}
|
|
9265
|
+
function cloneMessagesForProvider(messages) {
|
|
9266
|
+
return messages.map((message) => ({
|
|
9267
|
+
...message,
|
|
9268
|
+
content: cloneModelInput(message.content),
|
|
9269
|
+
...message.tool_calls ? {
|
|
9270
|
+
tool_calls: message.tool_calls.map((call) => ({
|
|
9271
|
+
...call,
|
|
9272
|
+
function: { ...call.function }
|
|
9273
|
+
}))
|
|
9274
|
+
} : {}
|
|
9275
|
+
}));
|
|
9276
|
+
}
|
|
9277
|
+
function assertRichMessagesAreUserOnly(messages) {
|
|
9278
|
+
for (const message of messages) {
|
|
9279
|
+
if (typeof message.content !== "string" && message.role !== "user") {
|
|
9280
|
+
throw new InvalidModelInputError(
|
|
9281
|
+
"Non-text model input parts are only supported on user messages"
|
|
9282
|
+
);
|
|
9283
|
+
}
|
|
9284
|
+
}
|
|
9285
|
+
}
|
|
9286
|
+
function normalizeSessionHistory(messages) {
|
|
9287
|
+
return messages.map((message) => {
|
|
9288
|
+
if (typeof message.content === "string") return { ...message };
|
|
9289
|
+
if (message.role !== "user") {
|
|
9290
|
+
throw new InvalidModelInputError(
|
|
9291
|
+
"Non-text model input parts are only supported on user messages"
|
|
9292
|
+
);
|
|
9293
|
+
}
|
|
9294
|
+
return { ...message, content: normalizeModelInput(message.content) };
|
|
9295
|
+
});
|
|
9296
|
+
}
|
|
9297
|
+
function hasRichMessage(messages) {
|
|
9298
|
+
return messages.some((message) => typeof message.content !== "string");
|
|
9299
|
+
}
|
|
9300
|
+
function firstImageSource(input, history) {
|
|
9301
|
+
const find = (value) => typeof value === "string" ? void 0 : value.find((part) => part.type === "image")?.source.type;
|
|
9302
|
+
return find(input) ?? history.map((message) => find(message.content)).find(Boolean);
|
|
9303
|
+
}
|
|
9304
|
+
function appendHandoffInstruction(input, originalText, instruction) {
|
|
9305
|
+
if (typeof input === "string" || instruction.length === 0 || instruction === originalText)
|
|
9306
|
+
return input;
|
|
9307
|
+
return [...input, { type: "text", text: instruction }];
|
|
9308
|
+
}
|
|
9309
|
+
var RICH_MODEL_INPUT_ERROR_MESSAGE = "Provider request failed while processing rich model input";
|
|
9310
|
+
var RICH_MODEL_INPUT_ERROR_NAME = "RichModelInputError";
|
|
9311
|
+
var MAX_RICH_FAILURE_ASSOCIATIONS = 32;
|
|
9312
|
+
var MAX_RICH_FAILURE_CAUSE_DEPTH = 4;
|
|
9313
|
+
function legacyErrorMessage(error) {
|
|
9314
|
+
return error instanceof Error ? error.message : String(error);
|
|
9315
|
+
}
|
|
9316
|
+
function providerErrorEventMetadata(error, richRequest) {
|
|
9317
|
+
if (!(error instanceof ProviderError)) {
|
|
9318
|
+
return error instanceof AxlError && richRequest ? { code: error.code } : {};
|
|
9319
|
+
}
|
|
9320
|
+
return {
|
|
9321
|
+
status: error.status,
|
|
9322
|
+
retryable: error.retryable,
|
|
9323
|
+
...richRequest ? {
|
|
9324
|
+
code: error.code,
|
|
9325
|
+
provider: error.provider,
|
|
9326
|
+
...error.requestId ? { requestId: error.requestId } : {},
|
|
9327
|
+
...error.retryAfterMs !== void 0 ? { retryAfterMs: error.retryAfterMs } : {}
|
|
9328
|
+
} : {}
|
|
9329
|
+
};
|
|
6971
9330
|
}
|
|
6972
9331
|
var WorkflowContext = class _WorkflowContext {
|
|
6973
9332
|
input;
|
|
@@ -6983,6 +9342,7 @@ var WorkflowContext = class _WorkflowContext {
|
|
|
6983
9342
|
}
|
|
6984
9343
|
config;
|
|
6985
9344
|
providerRegistry;
|
|
9345
|
+
transcriptionProviderRegistry;
|
|
6986
9346
|
sessionHistory;
|
|
6987
9347
|
onTrace;
|
|
6988
9348
|
pendingDecisions;
|
|
@@ -7008,6 +9368,10 @@ var WorkflowContext = class _WorkflowContext {
|
|
|
7008
9368
|
*/
|
|
7009
9369
|
autoCheckpointCounters;
|
|
7010
9370
|
branchDrainState;
|
|
9371
|
+
/** Bounded exact-identity failure association for this execution tree.
|
|
9372
|
+
* Unlike a process-global marker, this cannot affect another context that
|
|
9373
|
+
* happens to reuse the same Error object. */
|
|
9374
|
+
richFailureState;
|
|
7011
9375
|
/** Root-shared idempotency guards for workflow lifecycle emission.
|
|
7012
9376
|
* Both `runtime.execute()` and `runtime.stream()` have paths where a
|
|
7013
9377
|
* post-emit side-effect (checkpoint deletion, state-store persistence)
|
|
@@ -7091,7 +9455,8 @@ var WorkflowContext = class _WorkflowContext {
|
|
|
7091
9455
|
get events() {
|
|
7092
9456
|
if (!this._busRef.current) {
|
|
7093
9457
|
this._busRef.current = new AxlEventBus(this._eventStreamOptions);
|
|
7094
|
-
|
|
9458
|
+
const signal = this.currentSignal;
|
|
9459
|
+
if (signal?.aborted) {
|
|
7095
9460
|
this._busRef.current._finish();
|
|
7096
9461
|
}
|
|
7097
9462
|
}
|
|
@@ -7129,6 +9494,7 @@ var WorkflowContext = class _WorkflowContext {
|
|
|
7129
9494
|
this.metadata = init.metadata ?? {};
|
|
7130
9495
|
this.config = init.config;
|
|
7131
9496
|
this.providerRegistry = init.providerRegistry;
|
|
9497
|
+
this.transcriptionProviderRegistry = init.transcriptionProviderRegistry;
|
|
7132
9498
|
this.sessionHistory = init.sessionHistory ?? [];
|
|
7133
9499
|
this.onTrace = init.onTrace;
|
|
7134
9500
|
this.pendingDecisions = init.pendingDecisions;
|
|
@@ -7162,6 +9528,11 @@ var WorkflowContext = class _WorkflowContext {
|
|
|
7162
9528
|
byAgent: /* @__PURE__ */ new Map(),
|
|
7163
9529
|
root: 0
|
|
7164
9530
|
};
|
|
9531
|
+
this.richFailureState = init.richFailureState ?? {
|
|
9532
|
+
associations: [],
|
|
9533
|
+
overflowed: false,
|
|
9534
|
+
indeterminate: false
|
|
9535
|
+
};
|
|
7165
9536
|
const branchDrainTimeoutMs = resolveBranchDrainTimeoutMs(init.branchDrainTimeoutMs);
|
|
7166
9537
|
this.branchDrainState = init.branchDrainState ?? {
|
|
7167
9538
|
pending: /* @__PURE__ */ new Set(),
|
|
@@ -7194,6 +9565,7 @@ var WorkflowContext = class _WorkflowContext {
|
|
|
7194
9565
|
executionId: this.executionId,
|
|
7195
9566
|
config: this.config,
|
|
7196
9567
|
providerRegistry: this.providerRegistry,
|
|
9568
|
+
transcriptionProviderRegistry: this.transcriptionProviderRegistry,
|
|
7197
9569
|
metadata: { ...this.metadata },
|
|
7198
9570
|
// Shared infrastructure
|
|
7199
9571
|
budgetContext: this.budgetContext,
|
|
@@ -7216,6 +9588,7 @@ var WorkflowContext = class _WorkflowContext {
|
|
|
7216
9588
|
autoCheckpointCounters: this.autoCheckpointCounters,
|
|
7217
9589
|
branchDrainState: this.branchDrainState,
|
|
7218
9590
|
workflowLifecycleState: this.workflowLifecycleState,
|
|
9591
|
+
richFailureState: this.richFailureState,
|
|
7219
9592
|
// Share the parent's bus slot (mutable ref). Late allocation in
|
|
7220
9593
|
// either parent or child propagates because both read the same
|
|
7221
9594
|
// slot. Nested-ask events (agent-as-tool pattern) bubble up to
|
|
@@ -7227,6 +9600,52 @@ var WorkflowContext = class _WorkflowContext {
|
|
|
7227
9600
|
// Isolated: sessionHistory (empty)
|
|
7228
9601
|
});
|
|
7229
9602
|
}
|
|
9603
|
+
/** Associate an exact thrown value with rich input in this execution only. */
|
|
9604
|
+
recordRichFailure(error) {
|
|
9605
|
+
const state = this.richFailureState;
|
|
9606
|
+
if (state.associations.some((known) => Object.is(known, error))) return;
|
|
9607
|
+
if (state.associations.length >= MAX_RICH_FAILURE_ASSOCIATIONS) {
|
|
9608
|
+
state.associations.shift();
|
|
9609
|
+
state.overflowed = true;
|
|
9610
|
+
state.indeterminate = true;
|
|
9611
|
+
}
|
|
9612
|
+
state.associations.push(error);
|
|
9613
|
+
}
|
|
9614
|
+
/** @internal Safe observer projection for a failure associated with rich input. */
|
|
9615
|
+
_observerErrorProjection(error) {
|
|
9616
|
+
const state = this.richFailureState;
|
|
9617
|
+
const safe = () => ({
|
|
9618
|
+
message: RICH_MODEL_INPUT_ERROR_MESSAGE,
|
|
9619
|
+
name: RICH_MODEL_INPUT_ERROR_NAME
|
|
9620
|
+
});
|
|
9621
|
+
const indeterminate = () => {
|
|
9622
|
+
if (state.associations.length === 0) return void 0;
|
|
9623
|
+
state.indeterminate = true;
|
|
9624
|
+
return safe();
|
|
9625
|
+
};
|
|
9626
|
+
let current = error;
|
|
9627
|
+
const seen = /* @__PURE__ */ new WeakSet();
|
|
9628
|
+
for (let depth = 0; depth < MAX_RICH_FAILURE_CAUSE_DEPTH; depth++) {
|
|
9629
|
+
if (state.associations.some((known) => Object.is(known, current))) {
|
|
9630
|
+
return safe();
|
|
9631
|
+
}
|
|
9632
|
+
if ((typeof current !== "object" || current === null) && typeof current !== "function") {
|
|
9633
|
+
return state.overflowed || state.indeterminate ? safe() : void 0;
|
|
9634
|
+
}
|
|
9635
|
+
if (seen.has(current)) return indeterminate();
|
|
9636
|
+
seen.add(current);
|
|
9637
|
+
let descriptor;
|
|
9638
|
+
try {
|
|
9639
|
+
descriptor = Object.getOwnPropertyDescriptor(current, "cause");
|
|
9640
|
+
} catch {
|
|
9641
|
+
return indeterminate();
|
|
9642
|
+
}
|
|
9643
|
+
if (!descriptor) return state.overflowed || state.indeterminate ? safe() : void 0;
|
|
9644
|
+
if (!("value" in descriptor)) return indeterminate();
|
|
9645
|
+
current = descriptor.value;
|
|
9646
|
+
}
|
|
9647
|
+
return indeterminate();
|
|
9648
|
+
}
|
|
7230
9649
|
/**
|
|
7231
9650
|
* Resolve the current abort signal.
|
|
7232
9651
|
* Branch-scoped signals (from race/spawn/map/budget) in AsyncLocalStorage
|
|
@@ -7293,7 +9712,7 @@ var WorkflowContext = class _WorkflowContext {
|
|
|
7293
9712
|
/** Resolve, execute, and observe one non-handoff tool request. Rejections
|
|
7294
9713
|
* never start. Every accepted request emits one terminal event before a
|
|
7295
9714
|
* continuation message or abort escapes this coordinator. */
|
|
7296
|
-
async observeToolInvocation(agent2, toolCall, currentMessages,
|
|
9715
|
+
async observeToolInvocation(agent2, toolCall, currentMessages, toolDefs) {
|
|
7297
9716
|
const signal = this.currentSignal;
|
|
7298
9717
|
signal?.throwIfAborted();
|
|
7299
9718
|
const requestedTool = toolCall.function.name;
|
|
@@ -7303,6 +9722,7 @@ var WorkflowContext = class _WorkflowContext {
|
|
|
7303
9722
|
const override = this.toolOverrides?.get(requestedTool);
|
|
7304
9723
|
const mcpManager = this.mcpManager;
|
|
7305
9724
|
const mcpMatch = !configuredTool ? mcpManager?.getToolsForAgent(agent2._config.mcp, agent2._config.mcpTools).find(({ tool: tool2 }) => tool2.name === requestedTool) : void 0;
|
|
9725
|
+
const providerVisibleSchema = toolDefs.find((tool2) => tool2.function.name === requestedTool)?.function.parameters;
|
|
7306
9726
|
const parsed = parseToolInvocation({
|
|
7307
9727
|
toolCall,
|
|
7308
9728
|
configuredTool,
|
|
@@ -7311,7 +9731,8 @@ var WorkflowContext = class _WorkflowContext {
|
|
|
7311
9731
|
mcpCall: (args) => mcpManager.callTool(requestedTool, args, mcpMatch.server),
|
|
7312
9732
|
mcpTraceName: `${mcpMatch.server}:${requestedTool}`
|
|
7313
9733
|
} : {},
|
|
7314
|
-
availableTools
|
|
9734
|
+
availableTools: toolDefs.map((tool2) => tool2.function.name),
|
|
9735
|
+
...configuredTool ? { providerVisibleSchema } : {}
|
|
7315
9736
|
});
|
|
7316
9737
|
signal?.throwIfAborted();
|
|
7317
9738
|
if ("kind" in parsed) {
|
|
@@ -7426,6 +9847,17 @@ var WorkflowContext = class _WorkflowContext {
|
|
|
7426
9847
|
}
|
|
7427
9848
|
// ── ctx.ask() ─────────────────────────────────────────────────────────
|
|
7428
9849
|
async ask(agent2, prompt, options) {
|
|
9850
|
+
const normalizedInput = normalizeModelInput(prompt);
|
|
9851
|
+
const normalizedHistory = normalizeSessionHistory(this.sessionHistory);
|
|
9852
|
+
const delegateInputHolder = agent2;
|
|
9853
|
+
if (delegateInputHolder._delegateOriginalInput !== void 0) {
|
|
9854
|
+
delegateInputHolder._delegateOriginalInput = normalizeModelInput(
|
|
9855
|
+
delegateInputHolder._delegateOriginalInput
|
|
9856
|
+
);
|
|
9857
|
+
}
|
|
9858
|
+
const richAsk = typeof normalizedInput !== "string" || hasRichMessage(normalizedHistory) || delegateInputHolder._delegateOriginalInput !== void 0 && typeof delegateInputHolder._delegateOriginalInput !== "string";
|
|
9859
|
+
const promptText = inputText(normalizedInput);
|
|
9860
|
+
const inputDescriptor = describeModelInput(normalizedInput);
|
|
7429
9861
|
const agentName = agent2._name;
|
|
7430
9862
|
return this._checkpoint(
|
|
7431
9863
|
this._autoCheckpointName("ask", agentName),
|
|
@@ -7445,7 +9877,11 @@ var WorkflowContext = class _WorkflowContext {
|
|
|
7445
9877
|
};
|
|
7446
9878
|
return askStorage.run(frame, async () => {
|
|
7447
9879
|
const askStart = Date.now();
|
|
7448
|
-
this.emitEvent({
|
|
9880
|
+
this.emitEvent({
|
|
9881
|
+
type: "ask_start",
|
|
9882
|
+
prompt: promptText,
|
|
9883
|
+
...inputDescriptor ? { input: inputDescriptor } : {}
|
|
9884
|
+
});
|
|
7449
9885
|
const costBefore = this.budgetContext?.totalCost ?? 0;
|
|
7450
9886
|
const unpricedCountBefore = this.budgetContext?.unpricedCount ?? 0;
|
|
7451
9887
|
const resolveCtx = options?.metadata ? { metadata: { ...this.metadata, ...options.metadata } } : { metadata: this.metadata };
|
|
@@ -7453,10 +9889,11 @@ var WorkflowContext = class _WorkflowContext {
|
|
|
7453
9889
|
const doCall = async () => {
|
|
7454
9890
|
const result2 = await this.executeAgentCall(
|
|
7455
9891
|
agent2,
|
|
7456
|
-
|
|
9892
|
+
normalizedInput,
|
|
7457
9893
|
options,
|
|
7458
9894
|
void 0,
|
|
7459
|
-
usageCapture
|
|
9895
|
+
usageCapture,
|
|
9896
|
+
normalizedHistory
|
|
7460
9897
|
);
|
|
7461
9898
|
return result2;
|
|
7462
9899
|
};
|
|
@@ -7473,6 +9910,9 @@ var WorkflowContext = class _WorkflowContext {
|
|
|
7473
9910
|
},
|
|
7474
9911
|
async (span) => {
|
|
7475
9912
|
const r = await doCall();
|
|
9913
|
+
if (usageCapture.modelUri) {
|
|
9914
|
+
span.setAttribute("axl.agent.model", usageCapture.modelUri);
|
|
9915
|
+
}
|
|
7476
9916
|
const costAfter2 = this.budgetContext?.totalCost ?? 0;
|
|
7477
9917
|
span.setAttribute("axl.agent.cost", costAfter2 - costBefore);
|
|
7478
9918
|
span.setAttribute("axl.agent.duration", Date.now() - askStart);
|
|
@@ -7506,9 +9946,10 @@ var WorkflowContext = class _WorkflowContext {
|
|
|
7506
9946
|
try {
|
|
7507
9947
|
this.onAgentCallComplete({
|
|
7508
9948
|
agent: agent2._name,
|
|
7509
|
-
prompt,
|
|
9949
|
+
prompt: promptText,
|
|
9950
|
+
...inputDescriptor ? { input: inputDescriptor } : {},
|
|
7510
9951
|
response: typeof result === "string" ? result : JSON.stringify(result),
|
|
7511
|
-
model: agent2.resolveModel(resolveCtx),
|
|
9952
|
+
model: usageCapture.modelUri ?? agent2.resolveModel(resolveCtx),
|
|
7512
9953
|
cost: costAfter - costBefore,
|
|
7513
9954
|
unpriced: frame.askUnpriced || unpricedCountAfter > unpricedCountBefore,
|
|
7514
9955
|
duration: Date.now() - askStart,
|
|
@@ -7532,9 +9973,10 @@ var WorkflowContext = class _WorkflowContext {
|
|
|
7532
9973
|
} catch (err) {
|
|
7533
9974
|
askFailed = true;
|
|
7534
9975
|
askFailure = err;
|
|
9976
|
+
if (richAsk) this.recordRichFailure(err);
|
|
7535
9977
|
outcome = {
|
|
7536
9978
|
ok: false,
|
|
7537
|
-
error: err
|
|
9979
|
+
error: this._observerErrorProjection(err)?.message ?? legacyErrorMessage(err)
|
|
7538
9980
|
};
|
|
7539
9981
|
}
|
|
7540
9982
|
let terminalFailure;
|
|
@@ -7562,7 +10004,7 @@ var WorkflowContext = class _WorkflowContext {
|
|
|
7562
10004
|
{ agent: agentName }
|
|
7563
10005
|
);
|
|
7564
10006
|
}
|
|
7565
|
-
async executeAgentCall(agent2,
|
|
10007
|
+
async executeAgentCall(agent2, input, options, handoffMessages, usageCapture, sessionHistory = normalizeSessionHistory(this.sessionHistory)) {
|
|
7566
10008
|
if (this.budgetContext?.exceeded) {
|
|
7567
10009
|
const { limit, totalCost: spent, policy } = this.budgetContext;
|
|
7568
10010
|
if (policy === "warn") {
|
|
@@ -7579,7 +10021,82 @@ var WorkflowContext = class _WorkflowContext {
|
|
|
7579
10021
|
const resolveCtx = options?.metadata ? { metadata: { ...this.metadata, ...options.metadata } } : { metadata: this.metadata };
|
|
7580
10022
|
const modelUri = agent2.resolveModel(resolveCtx);
|
|
7581
10023
|
const systemPrompt = agent2.resolveSystem(resolveCtx);
|
|
7582
|
-
const { provider, model } = this.providerRegistry.resolve(modelUri, this.config);
|
|
10024
|
+
const { provider, model: resolvedModel } = this.providerRegistry.resolve(modelUri, this.config);
|
|
10025
|
+
let model = resolvedModel;
|
|
10026
|
+
const prompt = inputText(input);
|
|
10027
|
+
const providerOptions = options?.providerOptions ?? agent2._config.providerOptions;
|
|
10028
|
+
const richRequest = typeof input !== "string" || hasRichMessage(sessionHistory);
|
|
10029
|
+
if (richRequest) {
|
|
10030
|
+
if (providerOptions && ("messages" in providerOptions || "input" in providerOptions)) {
|
|
10031
|
+
throw new UnsupportedModelInputError({
|
|
10032
|
+
provider: provider.name ?? modelUri.split(":", 1)[0],
|
|
10033
|
+
model,
|
|
10034
|
+
modality: "image",
|
|
10035
|
+
feature: "raw input-container providerOptions"
|
|
10036
|
+
});
|
|
10037
|
+
}
|
|
10038
|
+
const source = firstImageSource(input, sessionHistory);
|
|
10039
|
+
if (!provider.validateInput) {
|
|
10040
|
+
throw new UnsupportedModelInputError({
|
|
10041
|
+
provider: provider.name ?? modelUri.split(":", 1)[0],
|
|
10042
|
+
model,
|
|
10043
|
+
modality: "image",
|
|
10044
|
+
...source ? { source } : {}
|
|
10045
|
+
});
|
|
10046
|
+
}
|
|
10047
|
+
const validation = provider.validateInput({
|
|
10048
|
+
model,
|
|
10049
|
+
input: cloneModelInput(input),
|
|
10050
|
+
history: cloneMessagesForProvider(sessionHistory),
|
|
10051
|
+
stream: this._streamingEnabled,
|
|
10052
|
+
hasTools: Boolean(
|
|
10053
|
+
agent2._config.tools?.length || agent2._config.mcp?.length || agent2._config.mcpTools?.length || agent2._config.handoffs
|
|
10054
|
+
),
|
|
10055
|
+
responseMode: options?.schema ? "structured" : "text",
|
|
10056
|
+
providerOptions
|
|
10057
|
+
});
|
|
10058
|
+
if (!validation || typeof validation.effectiveModel !== "string" || validation.effectiveModel.length === 0) {
|
|
10059
|
+
throw new UnsupportedModelInputError({
|
|
10060
|
+
provider: provider.name ?? modelUri.split(":", 1)[0],
|
|
10061
|
+
model,
|
|
10062
|
+
modality: "image",
|
|
10063
|
+
...source ? { source } : {}
|
|
10064
|
+
});
|
|
10065
|
+
}
|
|
10066
|
+
model = validation.effectiveModel;
|
|
10067
|
+
}
|
|
10068
|
+
const effectiveModelUri = !richRequest ? modelUri : `${modelUri.slice(0, modelUri.indexOf(":") + 1)}${model}`;
|
|
10069
|
+
if (usageCapture) usageCapture.modelUri ??= effectiveModelUri;
|
|
10070
|
+
if (richRequest) {
|
|
10071
|
+
const descriptors = [
|
|
10072
|
+
describeModelInput(input),
|
|
10073
|
+
...sessionHistory.map((message) => describeModelInput(message.content))
|
|
10074
|
+
].filter(
|
|
10075
|
+
(descriptor) => descriptor !== void 0
|
|
10076
|
+
);
|
|
10077
|
+
const parts = descriptors.flatMap((descriptor) => descriptor.parts);
|
|
10078
|
+
this.spanManager?.addEventToActiveSpan("axl.model_input", {
|
|
10079
|
+
"axl.input.parts": parts.length,
|
|
10080
|
+
"axl.input.images": parts.filter((part) => part.type === "image").length,
|
|
10081
|
+
"axl.input.source.bytes": parts.filter(
|
|
10082
|
+
(part) => part.type === "image" && part.source === "bytes"
|
|
10083
|
+
).length,
|
|
10084
|
+
"axl.input.source.base64": parts.filter(
|
|
10085
|
+
(part) => part.type === "image" && part.source === "base64"
|
|
10086
|
+
).length,
|
|
10087
|
+
"axl.input.source.url": parts.filter(
|
|
10088
|
+
(part) => part.type === "image" && part.source === "url"
|
|
10089
|
+
).length,
|
|
10090
|
+
"axl.input.source.provider_file": parts.filter(
|
|
10091
|
+
(part) => part.type === "image" && part.source === "provider-file"
|
|
10092
|
+
).length,
|
|
10093
|
+
"axl.input.inline_bytes": parts.reduce(
|
|
10094
|
+
(sum, part) => sum + (part.type === "image" ? part.bytes ?? 0 : 0),
|
|
10095
|
+
0
|
|
10096
|
+
)
|
|
10097
|
+
});
|
|
10098
|
+
}
|
|
10099
|
+
assertRichMessagesAreUserOnly(sessionHistory);
|
|
7583
10100
|
let resolvedHandoffs;
|
|
7584
10101
|
if (typeof agent2._config.handoffs === "function") {
|
|
7585
10102
|
try {
|
|
@@ -7600,50 +10117,60 @@ var WorkflowContext = class _WorkflowContext {
|
|
|
7600
10117
|
messages.push({ role: "system", content: systemPrompt });
|
|
7601
10118
|
}
|
|
7602
10119
|
const maxContext = agent2._config.maxContext;
|
|
7603
|
-
if (maxContext &&
|
|
10120
|
+
if (maxContext && sessionHistory.length > 0) {
|
|
7604
10121
|
const reserveTokens = this.config.contextManagement?.reserveTokens ?? 2e3;
|
|
7605
10122
|
const systemTokens = systemPrompt ? estimateTokens(systemPrompt) : 0;
|
|
7606
10123
|
const toolTokens = toolDefs.length > 0 ? estimateTokens(JSON.stringify(toolDefs)) : 0;
|
|
7607
10124
|
const overhead = systemTokens + toolTokens + reserveTokens;
|
|
7608
10125
|
const availableForHistory = maxContext - overhead;
|
|
7609
|
-
const
|
|
7610
|
-
if (
|
|
10126
|
+
const historyEstimate = estimateMessagesTokens(sessionHistory);
|
|
10127
|
+
if (historyEstimate.unmeasured) {
|
|
10128
|
+
this.emitEvent({
|
|
10129
|
+
type: "log",
|
|
10130
|
+
data: {
|
|
10131
|
+
warning: "Context media contribution is unmeasured; provider will enforce its context limit"
|
|
10132
|
+
}
|
|
10133
|
+
});
|
|
10134
|
+
}
|
|
10135
|
+
if (historyEstimate.tokens > availableForHistory) {
|
|
7611
10136
|
const summarizedMessages = await this.summarizeHistory(
|
|
7612
10137
|
provider,
|
|
7613
10138
|
model,
|
|
7614
|
-
|
|
10139
|
+
sessionHistory,
|
|
7615
10140
|
availableForHistory
|
|
7616
10141
|
);
|
|
7617
10142
|
for (const msg of summarizedMessages) {
|
|
7618
10143
|
messages.push(msg);
|
|
7619
10144
|
}
|
|
7620
10145
|
} else {
|
|
7621
|
-
for (const msg of
|
|
10146
|
+
for (const msg of sessionHistory) {
|
|
7622
10147
|
messages.push(msg);
|
|
7623
10148
|
}
|
|
7624
10149
|
}
|
|
7625
10150
|
} else {
|
|
7626
|
-
for (const msg of
|
|
10151
|
+
for (const msg of sessionHistory) {
|
|
7627
10152
|
messages.push(msg);
|
|
7628
10153
|
}
|
|
7629
10154
|
}
|
|
7630
10155
|
const schemaPromptMode = options?.schemaPrompt ?? agent2._config.schemaPrompt ?? "json-schema";
|
|
7631
|
-
let userContent =
|
|
10156
|
+
let userContent = input;
|
|
7632
10157
|
let appendedSchemaText;
|
|
7633
10158
|
if (options?.schema) {
|
|
7634
10159
|
const schema = options.schema;
|
|
7635
10160
|
if (schemaPromptMode === "none") {
|
|
7636
10161
|
} else if (schemaPromptMode === "json-schema") {
|
|
7637
10162
|
appendedSchemaText = renderSchemaForPrompt(schema);
|
|
7638
|
-
|
|
10163
|
+
const guidance = `
|
|
7639
10164
|
|
|
7640
10165
|
Respond with valid JSON matching this schema:
|
|
7641
10166
|
${appendedSchemaText}`;
|
|
10167
|
+
userContent = typeof input === "string" ? input + guidance : [...input, { type: "text", text: guidance }];
|
|
7642
10168
|
} else {
|
|
7643
10169
|
appendedSchemaText = typeof schemaPromptMode.render === "function" ? schemaPromptMode.render(schema) : schemaPromptMode.render;
|
|
7644
|
-
|
|
10170
|
+
const guidance = `
|
|
7645
10171
|
|
|
7646
10172
|
${appendedSchemaText}`;
|
|
10173
|
+
userContent = typeof input === "string" ? input + guidance : [...input, { type: "text", text: guidance }];
|
|
7647
10174
|
}
|
|
7648
10175
|
}
|
|
7649
10176
|
messages.push({ role: "user", content: userContent });
|
|
@@ -7671,7 +10198,10 @@ ${appendedSchemaText}`;
|
|
|
7671
10198
|
}
|
|
7672
10199
|
const guardrails = agent2._config.guardrails;
|
|
7673
10200
|
if (guardrails?.input) {
|
|
7674
|
-
const inputResult = await guardrails.input(prompt, {
|
|
10201
|
+
const inputResult = await guardrails.input(prompt, {
|
|
10202
|
+
metadata: this.metadata,
|
|
10203
|
+
input: cloneModelInput(input)
|
|
10204
|
+
});
|
|
7675
10205
|
this.emitEvent({
|
|
7676
10206
|
type: "guardrail",
|
|
7677
10207
|
agent: agent2._name,
|
|
@@ -7762,7 +10292,7 @@ ${appendedSchemaText}`;
|
|
|
7762
10292
|
includeThoughts: options?.includeThoughts ?? agent2._config.includeThoughts,
|
|
7763
10293
|
toolChoice: options?.toolChoice ?? agent2._config.toolChoice,
|
|
7764
10294
|
stop: options?.stop ?? agent2._config.stop,
|
|
7765
|
-
providerOptions
|
|
10295
|
+
providerOptions,
|
|
7766
10296
|
signal: this.currentSignal
|
|
7767
10297
|
};
|
|
7768
10298
|
if (options?.schema && toolDefs.length === 0) {
|
|
@@ -7788,15 +10318,26 @@ ${appendedSchemaText}`;
|
|
|
7788
10318
|
...chatOptions.stop !== void 0 ? { stop: chatOptions.stop } : {}
|
|
7789
10319
|
};
|
|
7790
10320
|
let messagesSnapshot;
|
|
10321
|
+
let messageInputs;
|
|
7791
10322
|
if (verboseTrace) {
|
|
7792
10323
|
try {
|
|
7793
|
-
|
|
10324
|
+
messageInputs = currentMessages.flatMap((message, index) => {
|
|
10325
|
+
const descriptor = describeModelInput(message.content);
|
|
10326
|
+
return descriptor ? [{ index, input: descriptor }] : [];
|
|
10327
|
+
});
|
|
10328
|
+
messagesSnapshot = currentMessages.map((message) => ({
|
|
10329
|
+
...message,
|
|
10330
|
+
content: sanitizeModelInputForTrace(message.content)
|
|
10331
|
+
}));
|
|
7794
10332
|
} catch (err) {
|
|
7795
10333
|
console.warn(
|
|
7796
10334
|
"[axl] verbose trace messages snapshot failed to clone; emitting shallow copy:",
|
|
7797
10335
|
err instanceof Error ? err.message : String(err)
|
|
7798
10336
|
);
|
|
7799
|
-
messagesSnapshot =
|
|
10337
|
+
messagesSnapshot = currentMessages.map((message) => ({
|
|
10338
|
+
...message,
|
|
10339
|
+
content: sanitizeModelInputForTrace(message.content)
|
|
10340
|
+
}));
|
|
7800
10341
|
}
|
|
7801
10342
|
}
|
|
7802
10343
|
const retryReason = pendingRetryReason;
|
|
@@ -7804,16 +10345,18 @@ ${appendedSchemaText}`;
|
|
|
7804
10345
|
this.emitEvent({
|
|
7805
10346
|
type: "agent_call_start",
|
|
7806
10347
|
agent: agent2._name,
|
|
7807
|
-
model:
|
|
10348
|
+
model: effectiveModelUri,
|
|
7808
10349
|
turn: turns,
|
|
7809
10350
|
data: {
|
|
7810
10351
|
prompt,
|
|
10352
|
+
...typeof input === "string" ? {} : { input: describeModelInput(input) },
|
|
7811
10353
|
...systemPrompt ? { system: systemPrompt } : {},
|
|
7812
10354
|
params: callParams,
|
|
7813
10355
|
turn: turns,
|
|
7814
10356
|
...retryReason ? { retryReason } : {},
|
|
7815
10357
|
...toolDefs.length > 0 ? { toolNames: toolDefs.map((t) => t.function.name) } : {},
|
|
7816
|
-
...messagesSnapshot ? { messages: messagesSnapshot } : {}
|
|
10358
|
+
...messagesSnapshot ? { messages: messagesSnapshot } : {},
|
|
10359
|
+
...messageInputs && messageInputs.length > 0 ? { messageInputs } : {}
|
|
7817
10360
|
}
|
|
7818
10361
|
});
|
|
7819
10362
|
let response;
|
|
@@ -7837,7 +10380,10 @@ ${appendedSchemaText}`;
|
|
|
7837
10380
|
});
|
|
7838
10381
|
}
|
|
7839
10382
|
}) : void 0;
|
|
7840
|
-
for await (const chunk of provider.stream(
|
|
10383
|
+
for await (const chunk of provider.stream(
|
|
10384
|
+
cloneMessagesForProvider(currentMessages),
|
|
10385
|
+
chatOptions
|
|
10386
|
+
)) {
|
|
7841
10387
|
if (chunk.type === "text_delta") {
|
|
7842
10388
|
content2 += chunk.content;
|
|
7843
10389
|
this.emitEvent({ type: "token", data: chunk.content });
|
|
@@ -7903,24 +10449,25 @@ ${appendedSchemaText}`;
|
|
|
7903
10449
|
response.thinking_content = thinkingContent;
|
|
7904
10450
|
}
|
|
7905
10451
|
} else {
|
|
7906
|
-
response = await provider.chat(currentMessages, chatOptions);
|
|
10452
|
+
response = await provider.chat(cloneMessagesForProvider(currentMessages), chatOptions);
|
|
7907
10453
|
}
|
|
7908
10454
|
} catch (err) {
|
|
10455
|
+
if (richRequest) this.recordRichFailure(err);
|
|
7909
10456
|
this.emitEvent({
|
|
7910
10457
|
type: "agent_call_end",
|
|
7911
10458
|
agent: agent2._name,
|
|
7912
|
-
model:
|
|
10459
|
+
model: effectiveModelUri,
|
|
7913
10460
|
promptVersion: agent2._config.version,
|
|
7914
10461
|
duration: Date.now() - turnStart,
|
|
7915
10462
|
data: {
|
|
7916
10463
|
response: "",
|
|
7917
10464
|
turn: turns,
|
|
7918
10465
|
...retryReason ? { retryReason } : {},
|
|
7919
|
-
error: err
|
|
10466
|
+
error: this._observerErrorProjection(err)?.message ?? legacyErrorMessage(err),
|
|
7920
10467
|
// Enrich with typed-error metadata when available. `body` is
|
|
7921
10468
|
// deliberately NOT emitted — it's redaction-eligible (see
|
|
7922
10469
|
// docs/security.md). `data.error` already carries the message.
|
|
7923
|
-
...err
|
|
10470
|
+
...providerErrorEventMetadata(err, richRequest)
|
|
7924
10471
|
}
|
|
7925
10472
|
});
|
|
7926
10473
|
throw err;
|
|
@@ -7934,7 +10481,7 @@ ${appendedSchemaText}`;
|
|
|
7934
10481
|
this.emitEvent({
|
|
7935
10482
|
type: "agent_call_end",
|
|
7936
10483
|
agent: agent2._name,
|
|
7937
|
-
model:
|
|
10484
|
+
model: effectiveModelUri,
|
|
7938
10485
|
promptVersion: agent2._config.version,
|
|
7939
10486
|
cost: response.cost,
|
|
7940
10487
|
tokens: response.usage ? {
|
|
@@ -7968,12 +10515,12 @@ ${appendedSchemaText}`;
|
|
|
7968
10515
|
if (descriptor) {
|
|
7969
10516
|
const mode = descriptor.mode ?? "oneway";
|
|
7970
10517
|
let handoffPrompt = prompt;
|
|
7971
|
-
|
|
7972
|
-
|
|
7973
|
-
|
|
7974
|
-
|
|
7975
|
-
} catch {
|
|
10518
|
+
try {
|
|
10519
|
+
const args = JSON.parse(toolCall.function.arguments);
|
|
10520
|
+
if (typeof args.message === "string" && args.message.length > 0) {
|
|
10521
|
+
handoffPrompt = args.message;
|
|
7976
10522
|
}
|
|
10523
|
+
} catch {
|
|
7977
10524
|
}
|
|
7978
10525
|
const handoffStart = Date.now();
|
|
7979
10526
|
const sourceFrame = askStorage.getStore();
|
|
@@ -8000,15 +10547,27 @@ ${appendedSchemaText}`;
|
|
|
8000
10547
|
validateRetries: options.validateRetries
|
|
8001
10548
|
} : void 0;
|
|
8002
10549
|
const handoffFn = () => askStorage.run(targetFrame, async () => {
|
|
8003
|
-
|
|
10550
|
+
const delegateOriginal = agent2._delegateOriginalInput;
|
|
10551
|
+
const handoffInput = delegateOriginal ?? (typeof input === "string" ? handoffPrompt : input);
|
|
10552
|
+
const enrichedHandoffInput = appendHandoffInstruction(
|
|
10553
|
+
handoffInput,
|
|
10554
|
+
inputText(input),
|
|
10555
|
+
handoffPrompt
|
|
10556
|
+
);
|
|
10557
|
+
this.emitEvent({
|
|
10558
|
+
type: "ask_start",
|
|
10559
|
+
prompt: inputText(enrichedHandoffInput),
|
|
10560
|
+
...typeof enrichedHandoffInput === "string" ? {} : { input: describeModelInput(enrichedHandoffInput) }
|
|
10561
|
+
});
|
|
8004
10562
|
const targetAskStart = Date.now();
|
|
8005
10563
|
try {
|
|
8006
10564
|
const result = await this.executeAgentCall(
|
|
8007
10565
|
descriptor.agent,
|
|
8008
|
-
|
|
10566
|
+
enrichedHandoffInput,
|
|
8009
10567
|
handoffOptions,
|
|
8010
10568
|
currentMessages,
|
|
8011
|
-
usageCapture
|
|
10569
|
+
usageCapture,
|
|
10570
|
+
sessionHistory
|
|
8012
10571
|
);
|
|
8013
10572
|
this.emitEvent({
|
|
8014
10573
|
type: "ask_end",
|
|
@@ -8019,11 +10578,13 @@ ${appendedSchemaText}`;
|
|
|
8019
10578
|
});
|
|
8020
10579
|
return result;
|
|
8021
10580
|
} catch (err) {
|
|
10581
|
+
const richHandoff = typeof enrichedHandoffInput !== "string" || hasRichMessage(sessionHistory);
|
|
10582
|
+
if (richHandoff) this.recordRichFailure(err);
|
|
8022
10583
|
this.emitEvent({
|
|
8023
10584
|
type: "ask_end",
|
|
8024
10585
|
outcome: {
|
|
8025
10586
|
ok: false,
|
|
8026
|
-
error: err
|
|
10587
|
+
error: this._observerErrorProjection(err)?.message ?? legacyErrorMessage(err)
|
|
8027
10588
|
},
|
|
8028
10589
|
cost: targetFrame.askCost.value,
|
|
8029
10590
|
...targetFrame.askUnpriced ? { unpriced: true } : {},
|
|
@@ -8122,12 +10683,7 @@ ${appendedSchemaText}`;
|
|
|
8122
10683
|
return await handoffFn();
|
|
8123
10684
|
}
|
|
8124
10685
|
}
|
|
8125
|
-
await this.observeToolInvocation(
|
|
8126
|
-
agent2,
|
|
8127
|
-
toolCall,
|
|
8128
|
-
currentMessages,
|
|
8129
|
-
toolDefs.map((tool2) => tool2.function.name)
|
|
8130
|
-
);
|
|
10686
|
+
await this.observeToolInvocation(agent2, toolCall, currentMessages, toolDefs);
|
|
8131
10687
|
}
|
|
8132
10688
|
continue;
|
|
8133
10689
|
}
|
|
@@ -8537,12 +11093,12 @@ ${appendedSchemaText}`;
|
|
|
8537
11093
|
content: `Summary of earlier conversation:
|
|
8538
11094
|
${this.summaryCache}`
|
|
8539
11095
|
};
|
|
8540
|
-
const summaryTokens = estimateTokens(summaryMsg2.content) + 4;
|
|
11096
|
+
const summaryTokens = estimateTokens(summarizeModelInput(summaryMsg2.content)) + 4;
|
|
8541
11097
|
const remaining = availableTokens - summaryTokens;
|
|
8542
11098
|
let recentTokens2 = 0;
|
|
8543
11099
|
let splitIdx2 = history.length;
|
|
8544
11100
|
for (let i = history.length - 1; i >= 0; i--) {
|
|
8545
|
-
const msgTokens = estimateTokens(history[i].content) + 4;
|
|
11101
|
+
const msgTokens = estimateTokens(summarizeModelInput(history[i].content)) + 4;
|
|
8546
11102
|
if (recentTokens2 + msgTokens > remaining) break;
|
|
8547
11103
|
recentTokens2 += msgTokens;
|
|
8548
11104
|
splitIdx2 = i;
|
|
@@ -8555,7 +11111,7 @@ ${this.summaryCache}`
|
|
|
8555
11111
|
let splitIdx = history.length;
|
|
8556
11112
|
const targetRecent = Math.floor(availableTokens * 0.6);
|
|
8557
11113
|
for (let i = history.length - 1; i >= 0; i--) {
|
|
8558
|
-
const msgTokens = estimateTokens(history[i].content) + 4;
|
|
11114
|
+
const msgTokens = estimateTokens(summarizeModelInput(history[i].content)) + 4;
|
|
8559
11115
|
if (recentTokens + msgTokens > targetRecent) break;
|
|
8560
11116
|
recentTokens += msgTokens;
|
|
8561
11117
|
splitIdx = i;
|
|
@@ -8573,7 +11129,7 @@ ${this.summaryCache}`
|
|
|
8573
11129
|
summaryProvider = provider;
|
|
8574
11130
|
summaryModel = model;
|
|
8575
11131
|
}
|
|
8576
|
-
const oldContent = oldMessages.map((m) => `${m.role}: ${m.content}`).join("\n");
|
|
11132
|
+
const oldContent = oldMessages.map((m) => `${m.role}: ${summarizeModelInput(m.content)}`).join("\n");
|
|
8577
11133
|
const summaryResponse = await summaryProvider.chat(
|
|
8578
11134
|
[
|
|
8579
11135
|
{
|
|
@@ -9807,7 +12363,9 @@ ${summaryResponse.content}`
|
|
|
9807
12363
|
reason: "routed"
|
|
9808
12364
|
}
|
|
9809
12365
|
});
|
|
9810
|
-
|
|
12366
|
+
const routerInput = options?.routerInput === "text" ? inputText(prompt) : prompt;
|
|
12367
|
+
routerAgent._delegateOriginalInput = prompt;
|
|
12368
|
+
return this.ask(routerAgent, routerInput, {
|
|
9811
12369
|
schema: options?.schema,
|
|
9812
12370
|
schemaPrompt: options?.schemaPrompt,
|
|
9813
12371
|
nativeStructuredOutput: options?.nativeStructuredOutput,
|
|
@@ -9817,6 +12375,278 @@ ${summaryResponse.content}`
|
|
|
9817
12375
|
validateRetries: options?.validateRetries
|
|
9818
12376
|
});
|
|
9819
12377
|
}
|
|
12378
|
+
/** Transcribe recorded audio through the dedicated transcription registry.
|
|
12379
|
+
* This operation never enters the chat history or chat-provider path. */
|
|
12380
|
+
async transcribe(request) {
|
|
12381
|
+
const startedAt = Date.now();
|
|
12382
|
+
const transcriptionId = randomUUID();
|
|
12383
|
+
const signal = this.currentSignal;
|
|
12384
|
+
const rawAudio = request && typeof request === "object" && "audio" in request ? request.audio : void 0;
|
|
12385
|
+
const source = rawAudio?.type;
|
|
12386
|
+
const requestedUri = typeof request?.model === "string" ? request.model : void 0;
|
|
12387
|
+
const colon = requestedUri?.indexOf(":") ?? -1;
|
|
12388
|
+
const requestedProvider = colon > 0 ? requestedUri.slice(0, colon) : void 0;
|
|
12389
|
+
const requestedModel = colon > 0 && colon < requestedUri.length - 1 ? requestedUri.slice(colon + 1) : void 0;
|
|
12390
|
+
const startAudio = {
|
|
12391
|
+
source: typeof source === "string" ? source : "unknown",
|
|
12392
|
+
...rawAudio?.data instanceof Uint8Array ? { bytes: rawAudio.data.byteLength } : {},
|
|
12393
|
+
...typeof rawAudio?.mediaType === "string" ? { mediaType: rawAudio.mediaType } : {}
|
|
12394
|
+
};
|
|
12395
|
+
this.emitEvent({
|
|
12396
|
+
type: "transcription_start",
|
|
12397
|
+
transcriptionId,
|
|
12398
|
+
model: requestedModel,
|
|
12399
|
+
data: {
|
|
12400
|
+
...requestedProvider ? { provider: requestedProvider } : {},
|
|
12401
|
+
...requestedModel ? { model: requestedModel } : {},
|
|
12402
|
+
audio: startAudio
|
|
12403
|
+
}
|
|
12404
|
+
});
|
|
12405
|
+
let terminalEmitted = false;
|
|
12406
|
+
const emitTerminal = (partial) => {
|
|
12407
|
+
if (terminalEmitted) return;
|
|
12408
|
+
terminalEmitted = true;
|
|
12409
|
+
this.emitEvent({
|
|
12410
|
+
type: "transcription_end",
|
|
12411
|
+
transcriptionId,
|
|
12412
|
+
model: partial.model,
|
|
12413
|
+
duration: Date.now() - startedAt,
|
|
12414
|
+
...partial.cost !== void 0 ? { cost: partial.cost } : {},
|
|
12415
|
+
...partial.tokens ? { tokens: partial.tokens } : {},
|
|
12416
|
+
data: partial.data
|
|
12417
|
+
});
|
|
12418
|
+
};
|
|
12419
|
+
const abortError3 = () => new DOMException("The transcription operation was aborted", "AbortError");
|
|
12420
|
+
const cleanupStatus = (value) => value === "not_required" || value === "deleted" || value === "failed" || value === "timed_out" ? value : void 0;
|
|
12421
|
+
const safeProviderFailure = (error, provider, model) => {
|
|
12422
|
+
const details = error && typeof error === "object" ? error : void 0;
|
|
12423
|
+
const accounting = normalizeTranscriptionAccounting(details?.usage, details?.pricingStatus);
|
|
12424
|
+
const cleanup = cleanupStatus(details?.cleanupStatus);
|
|
12425
|
+
const providerError = error instanceof ProviderError ? error : void 0;
|
|
12426
|
+
const diagnostics = providerError ? {
|
|
12427
|
+
status: providerError.status,
|
|
12428
|
+
retryable: providerError.retryable,
|
|
12429
|
+
...providerError.retryAfterMs !== void 0 ? { retryAfterMs: providerError.retryAfterMs } : {},
|
|
12430
|
+
...providerError.requestId ? { requestId: providerError.requestId } : {}
|
|
12431
|
+
} : void 0;
|
|
12432
|
+
return {
|
|
12433
|
+
accounting,
|
|
12434
|
+
cleanup,
|
|
12435
|
+
diagnostics,
|
|
12436
|
+
error: new TranscriptionOperationError({
|
|
12437
|
+
provider,
|
|
12438
|
+
model,
|
|
12439
|
+
...accounting.usage ? { usage: accounting.usage } : {},
|
|
12440
|
+
...accounting.pricingStatus ? { pricingStatus: accounting.pricingStatus } : {},
|
|
12441
|
+
...cleanup ? { cleanupStatus: cleanup } : {},
|
|
12442
|
+
...diagnostics ?? {},
|
|
12443
|
+
cause: error
|
|
12444
|
+
})
|
|
12445
|
+
};
|
|
12446
|
+
};
|
|
12447
|
+
const terminalData = (status, provider, model, audio, accounting, extras = {}) => ({
|
|
12448
|
+
status,
|
|
12449
|
+
...provider ? { provider } : {},
|
|
12450
|
+
...model ? { model } : {},
|
|
12451
|
+
...audio ? { audio } : {},
|
|
12452
|
+
...accounting.usage ? { usage: accounting.usage } : {},
|
|
12453
|
+
...accounting.pricingStatus ? { pricingStatus: accounting.pricingStatus } : {},
|
|
12454
|
+
...extras
|
|
12455
|
+
});
|
|
12456
|
+
try {
|
|
12457
|
+
const normalized = normalizeTranscriptionRequest(request);
|
|
12458
|
+
const descriptor = {
|
|
12459
|
+
source: normalized.audio.type,
|
|
12460
|
+
...normalized.audio.type === "bytes" ? { bytes: normalized.audio.data.byteLength } : {},
|
|
12461
|
+
...normalized.audio.mediaType !== void 0 ? { mediaType: normalized.audio.mediaType } : {}
|
|
12462
|
+
};
|
|
12463
|
+
if (this.currentSignal?.aborted || signal?.aborted) {
|
|
12464
|
+
emitTerminal({
|
|
12465
|
+
model: requestedModel,
|
|
12466
|
+
data: terminalData("aborted", requestedProvider, requestedModel, descriptor, {
|
|
12467
|
+
hasWork: false
|
|
12468
|
+
})
|
|
12469
|
+
});
|
|
12470
|
+
throw abortError3();
|
|
12471
|
+
}
|
|
12472
|
+
if (this.budgetContext?.exceeded) {
|
|
12473
|
+
const { limit, totalCost: spent, policy } = this.budgetContext;
|
|
12474
|
+
if (policy === "warn") {
|
|
12475
|
+
this.emitEvent({
|
|
12476
|
+
type: "log",
|
|
12477
|
+
data: { warning: "Budget exceeded", limit, spent, policy }
|
|
12478
|
+
});
|
|
12479
|
+
} else {
|
|
12480
|
+
throw new BudgetExceededError(limit, spent, policy);
|
|
12481
|
+
}
|
|
12482
|
+
}
|
|
12483
|
+
if (!this.transcriptionProviderRegistry) {
|
|
12484
|
+
throw new InvalidTranscriptionInputError(
|
|
12485
|
+
"No transcription provider registry is configured"
|
|
12486
|
+
);
|
|
12487
|
+
}
|
|
12488
|
+
const resolved = this.transcriptionProviderRegistry.resolve(normalized.model, this.config);
|
|
12489
|
+
if (normalized.audio.type === "provider-file" && normalized.audio.provider !== resolved.providerName) {
|
|
12490
|
+
throw new InvalidTranscriptionInputError(
|
|
12491
|
+
"Transcription provider-file source does not match the requested provider"
|
|
12492
|
+
);
|
|
12493
|
+
}
|
|
12494
|
+
const capabilities = resolved.provider.capabilities(resolved.model);
|
|
12495
|
+
if (!capabilities || !capabilities.sources.includes(normalized.audio.type)) {
|
|
12496
|
+
throw new InvalidTranscriptionInputError(
|
|
12497
|
+
"The requested transcription model does not support this audio source"
|
|
12498
|
+
);
|
|
12499
|
+
}
|
|
12500
|
+
if (normalized.timestamps && !capabilities.timestamps?.includes(normalized.timestamps)) {
|
|
12501
|
+
throw new InvalidTranscriptionInputError(
|
|
12502
|
+
"The requested transcription model does not support requested timestamps"
|
|
12503
|
+
);
|
|
12504
|
+
}
|
|
12505
|
+
if (normalized.diarization && !capabilities.diarization) {
|
|
12506
|
+
throw new InvalidTranscriptionInputError(
|
|
12507
|
+
"The requested transcription model does not support diarization"
|
|
12508
|
+
);
|
|
12509
|
+
}
|
|
12510
|
+
const providerRequest = {
|
|
12511
|
+
...normalized,
|
|
12512
|
+
model: resolved.model,
|
|
12513
|
+
...signal ? { signal } : {}
|
|
12514
|
+
};
|
|
12515
|
+
let result;
|
|
12516
|
+
try {
|
|
12517
|
+
result = await resolved.provider.transcribe(providerRequest);
|
|
12518
|
+
} catch (error) {
|
|
12519
|
+
const failure = safeProviderFailure(error, resolved.providerName, resolved.model);
|
|
12520
|
+
if (this.currentSignal?.aborted || signal?.aborted) {
|
|
12521
|
+
if (failure.accounting.cost !== void 0)
|
|
12522
|
+
this._accumulateBudgetCost(failure.accounting.cost);
|
|
12523
|
+
emitTerminal({
|
|
12524
|
+
model: resolved.model,
|
|
12525
|
+
...failure.accounting.cost !== void 0 ? { cost: failure.accounting.cost } : {},
|
|
12526
|
+
...failure.accounting.tokens ? { tokens: failure.accounting.tokens } : {},
|
|
12527
|
+
data: terminalData(
|
|
12528
|
+
"aborted",
|
|
12529
|
+
resolved.providerName,
|
|
12530
|
+
resolved.model,
|
|
12531
|
+
descriptor,
|
|
12532
|
+
failure.accounting,
|
|
12533
|
+
...failure.cleanup ? [{ cleanupStatus: failure.cleanup }] : []
|
|
12534
|
+
)
|
|
12535
|
+
});
|
|
12536
|
+
throw abortError3();
|
|
12537
|
+
}
|
|
12538
|
+
if (failure.accounting.cost !== void 0)
|
|
12539
|
+
this._accumulateBudgetCost(failure.accounting.cost);
|
|
12540
|
+
emitTerminal({
|
|
12541
|
+
model: resolved.model,
|
|
12542
|
+
...failure.accounting.cost !== void 0 ? { cost: failure.accounting.cost } : {},
|
|
12543
|
+
...failure.accounting.tokens ? { tokens: failure.accounting.tokens } : {},
|
|
12544
|
+
data: terminalData(
|
|
12545
|
+
"failed",
|
|
12546
|
+
resolved.providerName,
|
|
12547
|
+
resolved.model,
|
|
12548
|
+
descriptor,
|
|
12549
|
+
failure.accounting,
|
|
12550
|
+
{
|
|
12551
|
+
error: "Transcription operation failed",
|
|
12552
|
+
errorCode: failure.error.code,
|
|
12553
|
+
...failure.diagnostics ? { providerError: failure.diagnostics } : {},
|
|
12554
|
+
...failure.cleanup ? { cleanupStatus: failure.cleanup } : {}
|
|
12555
|
+
}
|
|
12556
|
+
)
|
|
12557
|
+
});
|
|
12558
|
+
throw failure.error;
|
|
12559
|
+
}
|
|
12560
|
+
const accounting = normalizeTranscriptionAccounting(
|
|
12561
|
+
result.transcript?.usage,
|
|
12562
|
+
result.transcript?.pricingStatus
|
|
12563
|
+
);
|
|
12564
|
+
if (accounting.cost !== void 0) this._accumulateBudgetCost(accounting.cost);
|
|
12565
|
+
if (this.currentSignal?.aborted || signal?.aborted) {
|
|
12566
|
+
emitTerminal({
|
|
12567
|
+
model: resolved.model,
|
|
12568
|
+
...accounting.cost !== void 0 ? { cost: accounting.cost } : {},
|
|
12569
|
+
...accounting.tokens ? { tokens: accounting.tokens } : {},
|
|
12570
|
+
data: terminalData(
|
|
12571
|
+
"aborted",
|
|
12572
|
+
resolved.providerName,
|
|
12573
|
+
resolved.model,
|
|
12574
|
+
descriptor,
|
|
12575
|
+
accounting,
|
|
12576
|
+
...cleanupStatus(result.cleanupStatus) ? [{ cleanupStatus: cleanupStatus(result.cleanupStatus) }] : []
|
|
12577
|
+
)
|
|
12578
|
+
});
|
|
12579
|
+
throw abortError3();
|
|
12580
|
+
}
|
|
12581
|
+
let transcript;
|
|
12582
|
+
try {
|
|
12583
|
+
transcript = cloneTranscript(result.transcript);
|
|
12584
|
+
} catch (error) {
|
|
12585
|
+
const cleanup = cleanupStatus(result.cleanupStatus);
|
|
12586
|
+
const failure = new TranscriptionOperationError({
|
|
12587
|
+
provider: resolved.providerName,
|
|
12588
|
+
model: resolved.model,
|
|
12589
|
+
...accounting.usage ? { usage: accounting.usage } : {},
|
|
12590
|
+
...accounting.pricingStatus ? { pricingStatus: accounting.pricingStatus } : {},
|
|
12591
|
+
...cleanup ? { cleanupStatus: cleanup } : {},
|
|
12592
|
+
cause: error
|
|
12593
|
+
});
|
|
12594
|
+
emitTerminal({
|
|
12595
|
+
model: resolved.model,
|
|
12596
|
+
...accounting.cost !== void 0 ? { cost: accounting.cost } : {},
|
|
12597
|
+
...accounting.tokens ? { tokens: accounting.tokens } : {},
|
|
12598
|
+
data: terminalData(
|
|
12599
|
+
"failed",
|
|
12600
|
+
resolved.providerName,
|
|
12601
|
+
resolved.model,
|
|
12602
|
+
descriptor,
|
|
12603
|
+
accounting,
|
|
12604
|
+
{
|
|
12605
|
+
error: "Transcription operation failed",
|
|
12606
|
+
errorCode: "TRANSCRIPTION_PROVIDER_ERROR",
|
|
12607
|
+
...cleanup ? { cleanupStatus: cleanup } : {}
|
|
12608
|
+
}
|
|
12609
|
+
)
|
|
12610
|
+
});
|
|
12611
|
+
throw failure;
|
|
12612
|
+
}
|
|
12613
|
+
emitTerminal({
|
|
12614
|
+
model: resolved.model,
|
|
12615
|
+
...accounting.cost !== void 0 ? { cost: accounting.cost } : {},
|
|
12616
|
+
...accounting.tokens ? { tokens: accounting.tokens } : {},
|
|
12617
|
+
data: terminalData(
|
|
12618
|
+
"completed",
|
|
12619
|
+
resolved.providerName,
|
|
12620
|
+
resolved.model,
|
|
12621
|
+
descriptor,
|
|
12622
|
+
accounting,
|
|
12623
|
+
{
|
|
12624
|
+
text: transcript.text,
|
|
12625
|
+
...cleanupStatus(result.cleanupStatus) ? { cleanupStatus: cleanupStatus(result.cleanupStatus) } : {}
|
|
12626
|
+
}
|
|
12627
|
+
)
|
|
12628
|
+
});
|
|
12629
|
+
return transcript;
|
|
12630
|
+
} catch (error) {
|
|
12631
|
+
if (!terminalEmitted) {
|
|
12632
|
+
const aborted = signal?.aborted || error instanceof Error && error.name === "AbortError";
|
|
12633
|
+
emitTerminal({
|
|
12634
|
+
model: requestedModel,
|
|
12635
|
+
data: terminalData(
|
|
12636
|
+
aborted ? "aborted" : "failed",
|
|
12637
|
+
requestedProvider,
|
|
12638
|
+
requestedModel,
|
|
12639
|
+
void 0,
|
|
12640
|
+
{ hasWork: false },
|
|
12641
|
+
{
|
|
12642
|
+
error: aborted ? "Transcription operation aborted" : "Transcription operation failed"
|
|
12643
|
+
}
|
|
12644
|
+
)
|
|
12645
|
+
});
|
|
12646
|
+
}
|
|
12647
|
+
throw error;
|
|
12648
|
+
}
|
|
12649
|
+
}
|
|
9820
12650
|
// ── Private ───────────────────────────────────────────────────────────
|
|
9821
12651
|
/**
|
|
9822
12652
|
* Internal emitter input — intentionally loose so call sites don't need to
|
|
@@ -9832,7 +12662,7 @@ ${summaryResponse.content}`
|
|
|
9832
12662
|
const cost = partial.cost;
|
|
9833
12663
|
if (COST_BEARING_LEAF_TYPES.includes(partial.type)) {
|
|
9834
12664
|
const usable = isUsableCost(cost);
|
|
9835
|
-
const unpriced = !usable &&
|
|
12665
|
+
const unpriced = !usable && hasPositiveBillableWork(
|
|
9836
12666
|
partial
|
|
9837
12667
|
);
|
|
9838
12668
|
if (frame) {
|
|
@@ -9937,11 +12767,13 @@ function agent(config) {
|
|
|
9937
12767
|
_name: defaultName,
|
|
9938
12768
|
async ask(prompt, options) {
|
|
9939
12769
|
const registry = new ProviderRegistry();
|
|
12770
|
+
const transcriptionRegistry = new TranscriptionProviderRegistry();
|
|
9940
12771
|
const ctx = new WorkflowContext({
|
|
9941
12772
|
input: prompt,
|
|
9942
12773
|
executionId: randomUUID2(),
|
|
9943
12774
|
config: {},
|
|
9944
|
-
providerRegistry: registry
|
|
12775
|
+
providerRegistry: registry,
|
|
12776
|
+
transcriptionProviderRegistry: transcriptionRegistry
|
|
9945
12777
|
});
|
|
9946
12778
|
return ctx.ask(agentInstance, prompt, options);
|
|
9947
12779
|
},
|
|
@@ -10001,7 +12833,7 @@ var MemoryStore = class {
|
|
|
10001
12833
|
return data !== void 0 ? structuredClone(data) : null;
|
|
10002
12834
|
}
|
|
10003
12835
|
async saveSession(sessionId, history) {
|
|
10004
|
-
this.sessions.set(sessionId, structuredClone(history));
|
|
12836
|
+
this.sessions.set(sessionId, structuredClone(normalizePersistedSessionHistory(history)));
|
|
10005
12837
|
}
|
|
10006
12838
|
async getSession(sessionId) {
|
|
10007
12839
|
const history = this.sessions.get(sessionId);
|
|
@@ -10425,7 +13257,7 @@ var SQLiteStore = class {
|
|
|
10425
13257
|
const stmt = this.db.prepare(
|
|
10426
13258
|
"INSERT OR REPLACE INTO sessions (session_id, history) VALUES (?, ?)"
|
|
10427
13259
|
);
|
|
10428
|
-
stmt.run(sessionId, JSON.stringify(history));
|
|
13260
|
+
stmt.run(sessionId, JSON.stringify(normalizePersistedSessionHistory(history)));
|
|
10429
13261
|
}
|
|
10430
13262
|
async getSession(sessionId) {
|
|
10431
13263
|
const stmt = this.db.prepare("SELECT history FROM sessions WHERE session_id = ?");
|
|
@@ -10993,7 +13825,7 @@ var AxlStream = class extends Readable {
|
|
|
10993
13825
|
/** Signal an error. `executionId` is required for the same reason as
|
|
10994
13826
|
* `_done`: terminal events must carry a real id even when the
|
|
10995
13827
|
* failure happens before any real trace event fires (review S4). */
|
|
10996
|
-
_error(error, executionId) {
|
|
13828
|
+
_error(error, executionId, observerMessage = error.message, observerName = error.name) {
|
|
10997
13829
|
if (this.events.isFinished) return;
|
|
10998
13830
|
const errorEvent = {
|
|
10999
13831
|
schemaVersion: 2,
|
|
@@ -11001,7 +13833,7 @@ var AxlStream = class extends Readable {
|
|
|
11001
13833
|
executionId,
|
|
11002
13834
|
step: Number.MAX_SAFE_INTEGER,
|
|
11003
13835
|
timestamp: Date.now(),
|
|
11004
|
-
data: { message:
|
|
13836
|
+
data: { message: observerMessage, name: observerName }
|
|
11005
13837
|
};
|
|
11006
13838
|
this.events._push(errorEvent);
|
|
11007
13839
|
this.push(errorEvent);
|
|
@@ -11728,6 +14560,10 @@ var StreamingFlusher = class {
|
|
|
11728
14560
|
function isAbortError4(err) {
|
|
11729
14561
|
return typeof err === "object" && err !== null && err.name === "AbortError";
|
|
11730
14562
|
}
|
|
14563
|
+
function executionErrorProjection(ctx, error) {
|
|
14564
|
+
const safe = ctx?._observerErrorProjection(error);
|
|
14565
|
+
return safe ?? { message: error instanceof Error ? error.message : String(error) };
|
|
14566
|
+
}
|
|
11731
14567
|
function pushEventBounded(execInfo, event, cap) {
|
|
11732
14568
|
if (STREAMING_EXCLUDED_TYPES.has(event.type)) return;
|
|
11733
14569
|
if (cap === Infinity || execInfo.events.length < cap) {
|
|
@@ -11822,6 +14658,7 @@ var AxlRuntime = class extends EventEmitter2 {
|
|
|
11822
14658
|
tools = /* @__PURE__ */ new Map();
|
|
11823
14659
|
agents = /* @__PURE__ */ new Map();
|
|
11824
14660
|
providerRegistry;
|
|
14661
|
+
transcriptionProviderRegistry;
|
|
11825
14662
|
stateStore;
|
|
11826
14663
|
executions = /* @__PURE__ */ new Map();
|
|
11827
14664
|
pendingDecisionResolvers = /* @__PURE__ */ new Map();
|
|
@@ -11877,6 +14714,7 @@ var AxlRuntime = class extends EventEmitter2 {
|
|
|
11877
14714
|
super();
|
|
11878
14715
|
this.config = resolveConfig(config ?? {});
|
|
11879
14716
|
this.providerRegistry = new ProviderRegistry();
|
|
14717
|
+
this.transcriptionProviderRegistry = new TranscriptionProviderRegistry();
|
|
11880
14718
|
this.stateStore = this.createStateStore();
|
|
11881
14719
|
const requested = this.config.state?.maxEventsPerExecution;
|
|
11882
14720
|
if (requested === void 0) {
|
|
@@ -12025,7 +14863,7 @@ var AxlRuntime = class extends EventEmitter2 {
|
|
|
12025
14863
|
execInfo.status = "failed";
|
|
12026
14864
|
execInfo.completedAt = Date.now();
|
|
12027
14865
|
execInfo.duration = execInfo.completedAt - execInfo.startedAt;
|
|
12028
|
-
execInfo.error =
|
|
14866
|
+
execInfo.error = executionErrorProjection(ctx, terminalError).message;
|
|
12029
14867
|
execInfo.observation = observation;
|
|
12030
14868
|
ctx._emitWorkflowEnd({
|
|
12031
14869
|
status: "failed",
|
|
@@ -12280,6 +15118,7 @@ var AxlRuntime = class extends EventEmitter2 {
|
|
|
12280
15118
|
metadata: options?.metadata,
|
|
12281
15119
|
config: this.config,
|
|
12282
15120
|
providerRegistry: this.providerRegistry,
|
|
15121
|
+
transcriptionProviderRegistry: this.transcriptionProviderRegistry,
|
|
12283
15122
|
stateStore: this.stateStore,
|
|
12284
15123
|
mcpManager: this.mcpManager,
|
|
12285
15124
|
spanManager: this.spanManager,
|
|
@@ -12308,6 +15147,10 @@ var AxlRuntime = class extends EventEmitter2 {
|
|
|
12308
15147
|
registerProvider(name, provider) {
|
|
12309
15148
|
this.providerRegistry.registerInstance(name, provider);
|
|
12310
15149
|
}
|
|
15150
|
+
/** Register a dedicated transcription adapter. It is never used for chat. */
|
|
15151
|
+
registerTranscriptionProvider(name, provider) {
|
|
15152
|
+
this.transcriptionProviderRegistry.registerInstance(name, provider);
|
|
15153
|
+
}
|
|
12311
15154
|
/** Resolve a provider:model URI to a Provider instance and model name. */
|
|
12312
15155
|
resolveProvider(uri) {
|
|
12313
15156
|
return this.providerRegistry.resolve(uri, this.config);
|
|
@@ -12351,6 +15194,7 @@ var AxlRuntime = class extends EventEmitter2 {
|
|
|
12351
15194
|
metadata: options?.metadata,
|
|
12352
15195
|
config: this.config,
|
|
12353
15196
|
providerRegistry: this.providerRegistry,
|
|
15197
|
+
transcriptionProviderRegistry: this.transcriptionProviderRegistry,
|
|
12354
15198
|
sessionHistory,
|
|
12355
15199
|
signal: controller.signal,
|
|
12356
15200
|
eventStreamOptions: options?.events,
|
|
@@ -12438,6 +15282,7 @@ var AxlRuntime = class extends EventEmitter2 {
|
|
|
12438
15282
|
const executionId = randomUUID3();
|
|
12439
15283
|
this.abortControllers.set(executionId, controller);
|
|
12440
15284
|
let execInfo;
|
|
15285
|
+
let wfCtx;
|
|
12441
15286
|
const run = async () => {
|
|
12442
15287
|
const workflow2 = this.workflows.get(name);
|
|
12443
15288
|
if (!workflow2) throw new Error(`Workflow "${name}" not registered`);
|
|
@@ -12461,12 +15306,13 @@ var AxlRuntime = class extends EventEmitter2 {
|
|
|
12461
15306
|
metadata: liftPersistedMetadata(options?.metadata)
|
|
12462
15307
|
};
|
|
12463
15308
|
this.executions.set(executionId, execInfo);
|
|
12464
|
-
const
|
|
15309
|
+
const context = new WorkflowContext({
|
|
12465
15310
|
input: validated,
|
|
12466
15311
|
executionId,
|
|
12467
15312
|
metadata: options?.metadata,
|
|
12468
15313
|
config: this.config,
|
|
12469
15314
|
providerRegistry: this.providerRegistry,
|
|
15315
|
+
transcriptionProviderRegistry: this.transcriptionProviderRegistry,
|
|
12470
15316
|
sessionHistory,
|
|
12471
15317
|
signal: controller.signal,
|
|
12472
15318
|
eventStreamOptions: options?.events,
|
|
@@ -12531,6 +15377,7 @@ var AxlRuntime = class extends EventEmitter2 {
|
|
|
12531
15377
|
unpricedWarned: false
|
|
12532
15378
|
}
|
|
12533
15379
|
});
|
|
15380
|
+
wfCtx = context;
|
|
12534
15381
|
return this.spanManager.withSpanAsync(
|
|
12535
15382
|
"axl.workflow.execute",
|
|
12536
15383
|
{
|
|
@@ -12540,7 +15387,7 @@ var AxlRuntime = class extends EventEmitter2 {
|
|
|
12540
15387
|
},
|
|
12541
15388
|
(span) => this.runWorkflowBody({
|
|
12542
15389
|
workflow: workflow2,
|
|
12543
|
-
ctx:
|
|
15390
|
+
ctx: context,
|
|
12544
15391
|
execInfo,
|
|
12545
15392
|
validated,
|
|
12546
15393
|
span,
|
|
@@ -12553,12 +15400,14 @@ var AxlRuntime = class extends EventEmitter2 {
|
|
|
12553
15400
|
execInfo.status = "failed";
|
|
12554
15401
|
execInfo.completedAt = Date.now();
|
|
12555
15402
|
execInfo.duration = execInfo.completedAt - execInfo.startedAt;
|
|
12556
|
-
execInfo.error =
|
|
15403
|
+
execInfo.error = executionErrorProjection(wfCtx, err).message;
|
|
12557
15404
|
this.persistExecution(execInfo);
|
|
12558
15405
|
}
|
|
12559
15406
|
this.abortControllers.delete(executionId);
|
|
12560
15407
|
cleanupAbortForwarder();
|
|
12561
|
-
|
|
15408
|
+
const originalError = err instanceof Error ? err : new Error(String(err));
|
|
15409
|
+
const projection = executionErrorProjection(wfCtx, err);
|
|
15410
|
+
axlStream._error(originalError, executionId, projection.message, projection.name);
|
|
12562
15411
|
});
|
|
12563
15412
|
return axlStream;
|
|
12564
15413
|
}
|
|
@@ -12969,7 +15818,7 @@ var AxlRuntime = class extends EventEmitter2 {
|
|
|
12969
15818
|
},
|
|
12970
15819
|
{
|
|
12971
15820
|
role: "user",
|
|
12972
|
-
content: messages.map((m) => `${m.role}: ${m.content}`).join("\n")
|
|
15821
|
+
content: messages.map((m) => `${m.role}: ${summarizeModelInput(m.content)}`).join("\n")
|
|
12973
15822
|
}
|
|
12974
15823
|
],
|
|
12975
15824
|
{ model, maxTokens: 1024 }
|
|
@@ -13416,7 +16265,7 @@ async function registerConditions(conditions) {
|
|
|
13416
16265
|
].join("\n");
|
|
13417
16266
|
nodeModule.register(`data:text/javascript,${encodeURIComponent(hookCode)}`);
|
|
13418
16267
|
} catch {
|
|
13419
|
-
console.warn("Warning:
|
|
16268
|
+
console.warn("Warning: this Node.js runtime does not support --conditions");
|
|
13420
16269
|
}
|
|
13421
16270
|
}
|
|
13422
16271
|
|
|
@@ -13725,7 +16574,11 @@ var RedisStore = class _RedisStore {
|
|
|
13725
16574
|
async saveSession(sessionId, history) {
|
|
13726
16575
|
const ttl = this.ttlFor("session");
|
|
13727
16576
|
const setOptions = ttl !== void 0 ? { EX: ttl } : void 0;
|
|
13728
|
-
await this.client.multi().set(
|
|
16577
|
+
await this.client.multi().set(
|
|
16578
|
+
this.sessionKey(sessionId),
|
|
16579
|
+
JSON.stringify(normalizePersistedSessionHistory(history)),
|
|
16580
|
+
setOptions
|
|
16581
|
+
).sAdd(this.sessionIdsKey(), sessionId).exec();
|
|
13729
16582
|
}
|
|
13730
16583
|
async getSession(sessionId) {
|
|
13731
16584
|
const raw = await this.client.get(this.sessionKey(sessionId));
|
|
@@ -14213,10 +17066,15 @@ export {
|
|
|
14213
17066
|
EventStreamOverflowError,
|
|
14214
17067
|
GROQ_PROFILE,
|
|
14215
17068
|
GeminiProvider,
|
|
17069
|
+
GeminiTranscriptionProvider,
|
|
14216
17070
|
GuardrailError,
|
|
14217
17071
|
InMemoryVectorStore,
|
|
17072
|
+
InvalidModelInputError,
|
|
17073
|
+
InvalidTranscriptionInputError,
|
|
14218
17074
|
LLAMACPP_PROFILE,
|
|
14219
17075
|
LMSTUDIO_PROFILE,
|
|
17076
|
+
MAX_INLINE_MODEL_INPUT_BYTES,
|
|
17077
|
+
MAX_INLINE_TRANSCRIPTION_BYTES,
|
|
14220
17078
|
MISTRAL_PROFILE,
|
|
14221
17079
|
MaxTurnsError,
|
|
14222
17080
|
McpManager,
|
|
@@ -14232,6 +17090,8 @@ export {
|
|
|
14232
17090
|
OpenAIEmbedder,
|
|
14233
17091
|
OpenAIProvider,
|
|
14234
17092
|
OpenAIResponsesProvider,
|
|
17093
|
+
OpenAITranscriptionProvider,
|
|
17094
|
+
OpenRouterTranscriptionProvider,
|
|
14235
17095
|
ProviderError,
|
|
14236
17096
|
ProviderRegistry,
|
|
14237
17097
|
QuorumNotMet,
|
|
@@ -14247,7 +17107,11 @@ export {
|
|
|
14247
17107
|
TimeoutError,
|
|
14248
17108
|
ToolFailure,
|
|
14249
17109
|
ToolModelOutputError,
|
|
17110
|
+
TranscriptionOperationError,
|
|
17111
|
+
TranscriptionProviderRegistry,
|
|
14250
17112
|
UnsupportedEventSchemaVersionError,
|
|
17113
|
+
UnsupportedModelInputError,
|
|
17114
|
+
UnsupportedTranscriptionInputError,
|
|
14251
17115
|
VLLM_PROFILE,
|
|
14252
17116
|
ValidationError,
|
|
14253
17117
|
VerifyError,
|
|
@@ -14264,8 +17128,10 @@ export {
|
|
|
14264
17128
|
findConfig,
|
|
14265
17129
|
getEventSchemaVersion,
|
|
14266
17130
|
getExecutionEventSchemaVersion,
|
|
17131
|
+
hasPositiveBillableWork,
|
|
14267
17132
|
hasPositiveTokens,
|
|
14268
17133
|
importModule,
|
|
17134
|
+
inputText,
|
|
14269
17135
|
isCostBearingLeaf,
|
|
14270
17136
|
isRetryableStatus,
|
|
14271
17137
|
isRootLevel,
|