@axlsdk/axl 0.21.0 → 0.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -14,6 +14,75 @@ var AxlError = class extends Error {
14
14
  this.code = code;
15
15
  }
16
16
  };
17
+ var InvalidModelInputError = class extends AxlError {
18
+ constructor(message) {
19
+ super("INVALID_MODEL_INPUT", message);
20
+ this.name = "InvalidModelInputError";
21
+ }
22
+ };
23
+ var UnsupportedModelInputError = class extends AxlError {
24
+ provider;
25
+ model;
26
+ modality;
27
+ source;
28
+ constructor(options) {
29
+ super(
30
+ "UNSUPPORTED_MODEL_INPUT",
31
+ `Provider '${options.provider}' model '${options.model}' does not support ${options.feature ?? options.modality}${options.source ? ` from ${options.source}` : ""}`
32
+ );
33
+ this.name = "UnsupportedModelInputError";
34
+ this.provider = options.provider;
35
+ this.model = options.model;
36
+ this.modality = options.modality;
37
+ this.source = options.source;
38
+ }
39
+ };
40
+ var InvalidTranscriptionInputError = class extends AxlError {
41
+ constructor(message) {
42
+ super("INVALID_TRANSCRIPTION_INPUT", message);
43
+ this.name = "InvalidTranscriptionInputError";
44
+ }
45
+ };
46
+ var UnsupportedTranscriptionInputError = class extends AxlError {
47
+ provider;
48
+ model;
49
+ constructor(options) {
50
+ super(
51
+ "UNSUPPORTED_TRANSCRIPTION_INPUT",
52
+ `Transcription provider '${options.provider}'${options.model ? ` model '${options.model}'` : ""} does not support ${options.feature ?? "this request"}`
53
+ );
54
+ this.name = "UnsupportedTranscriptionInputError";
55
+ this.provider = options.provider;
56
+ this.model = options.model;
57
+ }
58
+ };
59
+ var TranscriptionOperationError = class extends AxlError {
60
+ provider;
61
+ model;
62
+ usage;
63
+ pricingStatus;
64
+ cleanupStatus;
65
+ /** Safe HTTP diagnostics projected from ProviderError when available. */
66
+ status;
67
+ retryable;
68
+ retryAfterMs;
69
+ requestId;
70
+ constructor(options) {
71
+ super("TRANSCRIPTION_PROVIDER_ERROR", "Transcription provider operation failed");
72
+ this.name = "TranscriptionOperationError";
73
+ this.provider = options.provider;
74
+ this.model = options.model;
75
+ this.usage = options.usage;
76
+ this.pricingStatus = options.pricingStatus;
77
+ this.cleanupStatus = options.cleanupStatus;
78
+ this.status = options.status;
79
+ this.retryable = options.retryable;
80
+ this.retryAfterMs = options.retryAfterMs;
81
+ this.requestId = options.requestId;
82
+ if (options.cause !== void 0)
83
+ Object.defineProperty(this, "cause", { enumerable: false, value: options.cause });
84
+ }
85
+ };
17
86
  var EventStreamOverflowError = class extends Error {
18
87
  maxQueued;
19
88
  eventType;
@@ -171,6 +240,221 @@ var ValidationError = class extends AxlError {
171
240
 
172
241
  // src/tool.ts
173
242
  var DEFAULT_MAX_STRING_LENGTH = 1e4;
243
+ var MAX_MODEL_ARGUMENT_ISSUES = 8;
244
+ var MAX_MODEL_ARGUMENT_MESSAGE_LENGTH = 2e3;
245
+ var MAX_MODEL_ARGUMENT_LINE_LENGTH = 220;
246
+ var MAX_MODEL_ARGUMENT_PATH_LENGTH = 160;
247
+ var MAX_UNION_DEPTH = 3;
248
+ var MAX_UNION_BRANCHES = 4;
249
+ var MAX_LITERAL_ALTERNATIVES = 10;
250
+ var SAFE_SCHEMA_TYPES = /* @__PURE__ */ new Set([
251
+ "string",
252
+ "number",
253
+ "boolean",
254
+ "null",
255
+ "array",
256
+ "object",
257
+ "integer"
258
+ ]);
259
+ var SAFE_STRING_FORMATS = /* @__PURE__ */ new Set([
260
+ "email",
261
+ "url",
262
+ "emoji",
263
+ "uuid",
264
+ "guid",
265
+ "nanoid",
266
+ "cuid",
267
+ "cuid2",
268
+ "ulid",
269
+ "xid",
270
+ "ksuid",
271
+ "datetime",
272
+ "date",
273
+ "time",
274
+ "duration",
275
+ "ipv4",
276
+ "ipv6",
277
+ "cidrv4",
278
+ "cidrv6",
279
+ "base64",
280
+ "base64url",
281
+ "json_string",
282
+ "e164",
283
+ "lowercase",
284
+ "uppercase",
285
+ "jwt"
286
+ ]);
287
+ function truncateModelFeedback(value, maximum) {
288
+ return value.length <= maximum ? value : `${value.slice(0, Math.max(0, maximum - 1))}\u2026`;
289
+ }
290
+ var DISPLAY_CONTROL_CHARACTERS = /[\u061C\u200B-\u200F\u202A-\u202E\u2066-\u2069\u2028\u2029]/g;
291
+ function isJsonSchema(value) {
292
+ return value !== null && typeof value === "object" && !Array.isArray(value);
293
+ }
294
+ function escapeDisplayControls(value) {
295
+ return value.replace(
296
+ DISPLAY_CONTROL_CHARACTERS,
297
+ (character) => `\\u${character.codePointAt(0).toString(16).padStart(4, "0")}`
298
+ );
299
+ }
300
+ function quoteModelText(value) {
301
+ return escapeDisplayControls(JSON.stringify(value));
302
+ }
303
+ function schemaNumber(value) {
304
+ return typeof value === "number" && Number.isFinite(value) ? String(value) : void 0;
305
+ }
306
+ function schemaLiteral(value) {
307
+ if (typeof value === "string") {
308
+ const rendered = quoteModelText(value);
309
+ return rendered.length <= 120 ? rendered : '"<literal omitted>"';
310
+ }
311
+ if (typeof value === "number") return Number.isFinite(value) ? String(value) : void 0;
312
+ if (typeof value === "boolean") return String(value);
313
+ if (value === null) return "null";
314
+ return void 0;
315
+ }
316
+ function schemaType(schema) {
317
+ const type = schema.type;
318
+ return typeof type === "string" && SAFE_SCHEMA_TYPES.has(type) ? type : void 0;
319
+ }
320
+ function compositionBranches(schema) {
321
+ const branches = [];
322
+ for (const key of ["anyOf", "oneOf", "allOf"]) {
323
+ const value = schema[key];
324
+ if (!Array.isArray(value)) continue;
325
+ for (const branch of value.slice(0, MAX_UNION_BRANCHES)) {
326
+ if (isJsonSchema(branch)) branches.push(branch);
327
+ }
328
+ }
329
+ return branches;
330
+ }
331
+ function unionBranches(schema) {
332
+ const branches = [];
333
+ for (const key of ["anyOf", "oneOf"]) {
334
+ const value = schema[key];
335
+ if (!Array.isArray(value)) continue;
336
+ for (const branch of value.slice(0, MAX_UNION_BRANCHES)) {
337
+ if (isJsonSchema(branch)) branches.push(branch);
338
+ }
339
+ }
340
+ return branches;
341
+ }
342
+ function resolvePathSegment(schema, segment, depth = 0) {
343
+ if (depth >= MAX_UNION_DEPTH) return void 0;
344
+ if (typeof segment === "string") {
345
+ const properties = schema.properties;
346
+ if (isJsonSchema(properties) && Object.hasOwn(properties, segment)) {
347
+ const child = properties[segment];
348
+ if (isJsonSchema(child)) return { node: child, renderedSegment: segment };
349
+ }
350
+ if (isJsonSchema(schema.additionalProperties)) {
351
+ return { node: schema.additionalProperties, renderedSegment: "<key>" };
352
+ }
353
+ } else if (typeof segment === "number" && Number.isSafeInteger(segment) && segment >= 0) {
354
+ const prefixItems = schema.prefixItems;
355
+ if (Array.isArray(prefixItems) && isJsonSchema(prefixItems[segment])) {
356
+ return { node: prefixItems[segment], renderedSegment: String(segment) };
357
+ }
358
+ if (isJsonSchema(schema.items)) {
359
+ return { node: schema.items, renderedSegment: "<index>" };
360
+ }
361
+ }
362
+ for (const branch of compositionBranches(schema)) {
363
+ const resolved = resolvePathSegment(branch, segment, depth + 1);
364
+ if (resolved !== void 0) return resolved;
365
+ }
366
+ return void 0;
367
+ }
368
+ function resolveSchemaPath(providerVisibleSchema, issuePath) {
369
+ if (!isJsonSchema(providerVisibleSchema)) return void 0;
370
+ let node = providerVisibleSchema;
371
+ const segments = [];
372
+ for (const segment of issuePath) {
373
+ const resolved = resolvePathSegment(node, segment);
374
+ if (resolved === void 0) {
375
+ segments.push("<path omitted>");
376
+ return { node, segments };
377
+ }
378
+ node = resolved.node;
379
+ segments.push(resolved.renderedSegment);
380
+ }
381
+ return { node, segments };
382
+ }
383
+ function jsonPointerPath(segments) {
384
+ if (segments.length === 0) return "<root>";
385
+ const pointer = `/${segments.map((segment) => segment.replaceAll("~", "~0").replaceAll("/", "~1")).join("/")}`;
386
+ const rendered = quoteModelText(pointer);
387
+ return rendered.length <= MAX_MODEL_ARGUMENT_PATH_LENGTH ? rendered : '"<path omitted>"';
388
+ }
389
+ function schemaAlternatives(schema, depth = 0) {
390
+ if (depth >= MAX_UNION_DEPTH) return [];
391
+ const constant = "const" in schema ? schemaLiteral(schema.const) : void 0;
392
+ if (constant !== void 0) return [constant];
393
+ if (Array.isArray(schema.enum)) {
394
+ const values = schema.enum.slice(0, MAX_LITERAL_ALTERNATIVES).map(schemaLiteral);
395
+ if (values.every((value) => value !== void 0)) return values;
396
+ }
397
+ const type = schemaType(schema);
398
+ if (type !== void 0) return [type];
399
+ const alternatives = unionBranches(schema).flatMap(
400
+ (branch) => schemaAlternatives(branch, depth + 1)
401
+ );
402
+ return alternatives.slice(0, MAX_LITERAL_ALTERNATIVES);
403
+ }
404
+ function schemaEnumExpectation(schema) {
405
+ if ("const" in schema) {
406
+ const constant = schemaLiteral(schema.const);
407
+ return constant === void 0 ? void 0 : `expected ${constant}`;
408
+ }
409
+ if (!Array.isArray(schema.enum)) return void 0;
410
+ const values = schema.enum.slice(0, MAX_LITERAL_ALTERNATIVES).map(schemaLiteral);
411
+ if (values.some((value) => value === void 0)) return void 0;
412
+ const rendered = values;
413
+ if (rendered.length === 0) return void 0;
414
+ const omission = schema.enum.length > rendered.length ? ", \u2026" : "";
415
+ return `expected ${rendered.length === 1 ? rendered[0] : `one of ${rendered.join(", ")}`}${omission}`;
416
+ }
417
+ function schemaTypeExpectation(schema) {
418
+ const type = schemaType(schema);
419
+ if (type !== void 0) return `expected ${type}`;
420
+ const alternatives = schemaAlternatives(schema);
421
+ return alternatives.length === 0 ? void 0 : `expected one of: ${alternatives.join(", ")}`;
422
+ }
423
+ function schemaSizeExpectation(schema, direction) {
424
+ const type = schemaType(schema);
425
+ const keyword = direction === "small" ? type === "string" ? "minLength" : type === "array" ? "minItems" : "minimum" : type === "string" ? "maxLength" : type === "array" ? "maxItems" : "maximum";
426
+ const value = schemaNumber(schema[keyword]);
427
+ if (value === void 0) return void 0;
428
+ const label = type === "string" ? "string length" : type === "array" ? "array length" : type;
429
+ if (label === void 0) return void 0;
430
+ return `expected ${label} ${direction === "small" ? "at least" : "at most"} ${value}`;
431
+ }
432
+ function schemaExpectation(schema, issueCode) {
433
+ switch (issueCode) {
434
+ case "invalid_type":
435
+ return schemaTypeExpectation(schema) ?? "failed structural validation";
436
+ case "invalid_value":
437
+ return schemaEnumExpectation(schema) ?? schemaTypeExpectation(schema) ?? "failed structural validation";
438
+ case "too_small":
439
+ return schemaSizeExpectation(schema, "small") ?? schemaTypeExpectation(schema) ?? "failed structural validation";
440
+ case "too_big":
441
+ return schemaSizeExpectation(schema, "big") ?? schemaTypeExpectation(schema) ?? "failed structural validation";
442
+ case "not_multiple_of": {
443
+ const multipleOf = schemaNumber(schema.multipleOf);
444
+ return multipleOf === void 0 ? schemaTypeExpectation(schema) ?? "failed structural validation" : `expected a multiple of ${multipleOf}`;
445
+ }
446
+ case "invalid_format": {
447
+ const format = schema.format;
448
+ return typeof format === "string" && SAFE_STRING_FORMATS.has(format) ? `expected valid ${format} format` : schemaTypeExpectation(schema) ?? "failed structural validation";
449
+ }
450
+ case "invalid_union":
451
+ return schemaTypeExpectation(schema) ?? "failed structural validation";
452
+ default: {
453
+ const typeExpectation = schemaTypeExpectation(schema);
454
+ return typeExpectation === void 0 ? "failed structural validation" : `failed structural validation; ${typeExpectation}`;
455
+ }
456
+ }
457
+ }
174
458
  var ToolStringLengthError = class extends Error {
175
459
  constructor(path, actual, maximum) {
176
460
  const renderedPath = path.reduce(
@@ -186,6 +470,42 @@ var ToolStringLengthError = class extends Error {
186
470
  this.name = "ToolStringLengthError";
187
471
  }
188
472
  };
473
+ function toolArgumentModelMessage(error, providerVisibleSchema) {
474
+ try {
475
+ let lines;
476
+ let omitted = false;
477
+ if (error instanceof ZodError) {
478
+ if (error.issues.length === 0) return void 0;
479
+ lines = [];
480
+ for (const issue of error.issues.slice(0, MAX_MODEL_ARGUMENT_ISSUES)) {
481
+ const resolved = resolveSchemaPath(providerVisibleSchema, issue.path);
482
+ if (resolved === void 0) return void 0;
483
+ const path = jsonPointerPath(resolved.segments);
484
+ const expectation = schemaExpectation(resolved.node, issue.code);
485
+ lines.push(
486
+ truncateModelFeedback(`- ${path}: ${expectation}`, MAX_MODEL_ARGUMENT_LINE_LENGTH)
487
+ );
488
+ }
489
+ omitted = error.issues.length > lines.length;
490
+ } else if (error instanceof ToolStringLengthError) {
491
+ const resolved = resolveSchemaPath(providerVisibleSchema, error.path);
492
+ const maximum = schemaNumber(error.maximum);
493
+ if (resolved === void 0 || maximum === void 0) return void 0;
494
+ const path = jsonPointerPath(resolved.segments);
495
+ lines = [`- ${path}: maximum string length is ${maximum}`];
496
+ } else {
497
+ return void 0;
498
+ }
499
+ const header = "Error: Invalid tool arguments:";
500
+ const footer = "Correct the arguments and try again.";
501
+ const omission = omitted ? "Additional validation issues were omitted." : void 0;
502
+ const parts = [header, ...lines, ...omission ? [omission] : [], footer];
503
+ const message = parts.join("\n");
504
+ return message.length <= MAX_MODEL_ARGUMENT_MESSAGE_LENGTH ? message : void 0;
505
+ } catch {
506
+ return void 0;
507
+ }
508
+ }
189
509
  function validateStringLengths(value, maxLen, path = []) {
190
510
  if (typeof value === "string") {
191
511
  if (value.length > maxLen) {
@@ -735,6 +1055,38 @@ function unsafeTransportError(surface, remediation, exposedData) {
735
1055
  }
736
1056
 
737
1057
  // src/providers/openai-compatible.ts
1058
+ var OPENROUTER_IMAGE_MODEL = "openai/gpt-4o-mini";
1059
+ function compatibleImageBase64(source) {
1060
+ return source.type === "base64" ? source.data : Buffer.from(source.data.buffer, source.data.byteOffset, source.data.byteLength).toString(
1061
+ "base64"
1062
+ );
1063
+ }
1064
+ function openRouterImageParts(parts) {
1065
+ const content = [];
1066
+ for (const part of parts) {
1067
+ if (part.type === "text") {
1068
+ content.push({ type: "text", text: part.text });
1069
+ continue;
1070
+ }
1071
+ const { source } = part;
1072
+ if (source.type === "provider-file") {
1073
+ throw new UnsupportedModelInputError({
1074
+ provider: "openrouter",
1075
+ model: OPENROUTER_IMAGE_MODEL,
1076
+ modality: "image",
1077
+ source: "provider-file"
1078
+ });
1079
+ }
1080
+ content.push({
1081
+ type: "image_url",
1082
+ image_url: {
1083
+ url: source.type === "url" ? source.url : `data:${source.mediaType};base64,${compatibleImageBase64(source)}`
1084
+ }
1085
+ });
1086
+ if (part.label) content.push({ type: "text", text: `[Image: ${part.label}]` });
1087
+ }
1088
+ return content;
1089
+ }
738
1090
  function resolvePerModel(value, model, fallback) {
739
1091
  if (value === void 0) return fallback;
740
1092
  return typeof value === "function" ? value(model) : value;
@@ -850,6 +1202,41 @@ function normalizeMessageContent(content) {
850
1202
  }
851
1203
  var OpenAICompatibleProvider = class {
852
1204
  name;
1205
+ inputCapabilities(model) {
1206
+ return this.name === "openrouter" && model === OPENROUTER_IMAGE_MODEL ? { image: { sources: ["url", "bytes", "base64"] } } : {};
1207
+ }
1208
+ validateInput(request) {
1209
+ const effectiveModel = typeof request.providerOptions?.model === "string" ? request.providerOptions.model : request.model;
1210
+ const fail = (source, feature) => {
1211
+ throw new UnsupportedModelInputError({
1212
+ provider: this.name,
1213
+ model: effectiveModel || request.model,
1214
+ modality: "image",
1215
+ ...source ? { source } : {},
1216
+ ...feature ? { feature } : {}
1217
+ });
1218
+ };
1219
+ if (this.name !== "openrouter" || effectiveModel !== OPENROUTER_IMAGE_MODEL) {
1220
+ fail(void 0, "image input for this model");
1221
+ }
1222
+ if (request.providerOptions && "messages" in request.providerOptions) {
1223
+ fail(void 0, "raw messages providerOptions");
1224
+ }
1225
+ if (request.hasTools) fail(void 0, "tools with image input");
1226
+ for (const message of request.history) {
1227
+ if (!Array.isArray(message.content)) continue;
1228
+ if (message.role !== "user") fail(void 0, "rich non-user history");
1229
+ if (message.content.some(
1230
+ (part) => part.type === "image" && part.source.type === "provider-file"
1231
+ )) {
1232
+ fail("provider-file");
1233
+ }
1234
+ }
1235
+ if (Array.isArray(request.input) && request.input.some((part) => part.type === "image" && part.source.type === "provider-file")) {
1236
+ fail("provider-file");
1237
+ }
1238
+ return { effectiveModel };
1239
+ }
853
1240
  /** `json_schema` is honored natively when the profile's `supportsJsonSchema`
854
1241
  * capability is true for this model (the default); otherwise the engine
855
1242
  * downgrades the request to plain `json_object` (see `buildRequestBody`),
@@ -1023,7 +1410,7 @@ var OpenAICompatibleProvider = class {
1023
1410
  formatMessage(msg, model) {
1024
1411
  const out = {
1025
1412
  role: this.profile.roleFor ? this.profile.roleFor(msg.role, model) : msg.role,
1026
- content: msg.content
1413
+ content: Array.isArray(msg.content) && this.name === "openrouter" && model === OPENROUTER_IMAGE_MODEL ? openRouterImageParts(msg.content) : msg.content
1027
1414
  };
1028
1415
  if (msg.name && (this.profile.capabilities?.emitsMessageName ?? true)) out.name = msg.name;
1029
1416
  if (msg.tool_calls) out.tool_calls = msg.tool_calls;
@@ -1770,8 +2157,88 @@ var OpenAIProvider = class extends OpenAICompatibleProvider {
1770
2157
  };
1771
2158
 
1772
2159
  // src/providers/openai-responses.ts
2160
+ var OPENAI_RESPONSES_IMAGE_MODELS = /* @__PURE__ */ new Set([
2161
+ "gpt-4o",
2162
+ "gpt-4o-2024-08-06",
2163
+ "gpt-4o-2024-11-20",
2164
+ "gpt-4o-mini",
2165
+ "gpt-4o-mini-2024-07-18"
2166
+ ]);
2167
+ function base64FromSource(source) {
2168
+ return source.type === "base64" ? source.data : Buffer.from(source.data.buffer, source.data.byteOffset, source.data.byteLength).toString(
2169
+ "base64"
2170
+ );
2171
+ }
2172
+ function responseImageParts(parts) {
2173
+ const mapped = [];
2174
+ for (const part of parts) {
2175
+ if (part.type === "text") {
2176
+ mapped.push({ type: "input_text", text: part.text });
2177
+ continue;
2178
+ }
2179
+ const { source } = part;
2180
+ if (source.type === "provider-file") {
2181
+ if (source.provider !== "openai-responses") {
2182
+ throw new UnsupportedModelInputError({
2183
+ provider: "openai-responses",
2184
+ model: "unknown",
2185
+ modality: "image",
2186
+ source: "provider-file"
2187
+ });
2188
+ }
2189
+ mapped.push({ type: "input_image", file_id: source.reference });
2190
+ } else if (source.type === "url") {
2191
+ mapped.push({ type: "input_image", image_url: source.url });
2192
+ } else {
2193
+ mapped.push({
2194
+ type: "input_image",
2195
+ image_url: `data:${source.mediaType};base64,${base64FromSource(source)}`
2196
+ });
2197
+ }
2198
+ if (part.label) mapped.push({ type: "input_text", text: `[Image: ${part.label}]` });
2199
+ }
2200
+ return mapped;
2201
+ }
1773
2202
  var OpenAIResponsesProvider = class {
1774
2203
  name = "openai-responses";
2204
+ inputCapabilities(model) {
2205
+ return OPENAI_RESPONSES_IMAGE_MODELS.has(model) ? { image: { sources: ["url", "bytes", "base64", "provider-file"] } } : {};
2206
+ }
2207
+ validateInput(request) {
2208
+ const effectiveModel = typeof request.providerOptions?.model === "string" ? request.providerOptions.model : request.model;
2209
+ const fail = (source, feature) => {
2210
+ throw new UnsupportedModelInputError({
2211
+ provider: this.name,
2212
+ model: effectiveModel || request.model,
2213
+ modality: "image",
2214
+ ...source ? { source } : {},
2215
+ ...feature ? { feature } : {}
2216
+ });
2217
+ };
2218
+ if (!OPENAI_RESPONSES_IMAGE_MODELS.has(effectiveModel)) {
2219
+ fail(void 0, "image input for this model");
2220
+ }
2221
+ if (request.providerOptions && "input" in request.providerOptions) {
2222
+ fail(void 0, "raw input-container providerOptions");
2223
+ }
2224
+ for (const message of request.history) {
2225
+ if (!Array.isArray(message.content)) continue;
2226
+ if (message.role !== "user") fail(void 0, "rich non-user history");
2227
+ for (const part of message.content) {
2228
+ if (part.type === "image" && part.source.type === "provider-file" && part.source.provider !== this.name) {
2229
+ fail("provider-file");
2230
+ }
2231
+ }
2232
+ }
2233
+ if (Array.isArray(request.input)) {
2234
+ for (const part of request.input) {
2235
+ if (part.type === "image" && part.source.type === "provider-file" && part.source.provider !== this.name) {
2236
+ fail("provider-file");
2237
+ }
2238
+ }
2239
+ }
2240
+ return { effectiveModel };
2241
+ }
1775
2242
  /** The Responses API honors `json_schema` (structured outputs) natively. */
1776
2243
  nativeStructuredOutputSupport() {
1777
2244
  return "schema";
@@ -1936,11 +2403,12 @@ var OpenAIResponsesProvider = class {
1936
2403
  buildInput(messages) {
1937
2404
  const input = [];
1938
2405
  for (const msg of messages) {
2406
+ const text = typeof msg.content === "string" ? msg.content : "";
1939
2407
  if (msg.role === "tool") {
1940
2408
  input.push({
1941
2409
  type: "function_call_output",
1942
2410
  call_id: msg.tool_call_id ?? "",
1943
- output: msg.content
2411
+ output: text
1944
2412
  });
1945
2413
  } else if (msg.role === "assistant" && msg.tool_calls && msg.tool_calls.length > 0) {
1946
2414
  const reasoningItems = msg.providerMetadata?.openaiReasoningItems;
@@ -1949,8 +2417,8 @@ var OpenAIResponsesProvider = class {
1949
2417
  input.push(item);
1950
2418
  }
1951
2419
  }
1952
- if (msg.content) {
1953
- input.push({ type: "message", role: "assistant", content: msg.content });
2420
+ if (text) {
2421
+ input.push({ type: "message", role: "assistant", content: text });
1954
2422
  }
1955
2423
  for (const tc of msg.tool_calls) {
1956
2424
  input.push({
@@ -1970,7 +2438,7 @@ var OpenAIResponsesProvider = class {
1970
2438
  input.push({
1971
2439
  type: "message",
1972
2440
  role: msg.role,
1973
- content: msg.content
2441
+ content: typeof msg.content === "string" ? msg.content : responseImageParts(msg.content)
1974
2442
  });
1975
2443
  }
1976
2444
  }
@@ -2037,7 +2505,7 @@ var OpenAIResponsesProvider = class {
2037
2505
  cached_tokens: json.usage.input_tokens_details?.cached_tokens,
2038
2506
  cache_write_tokens: json.usage.input_tokens_details?.cache_write_tokens
2039
2507
  } : void 0;
2040
- const cost = usage ? estimateDirectOpenAICost(json.model ?? model, usage, {
2508
+ const cost = usage && !this.requestContainsImages(request) ? estimateDirectOpenAICost(json.model ?? model, usage, {
2041
2509
  baseUrl: this.baseUrl,
2042
2510
  request,
2043
2511
  response: json
@@ -2052,6 +2520,16 @@ var OpenAIResponsesProvider = class {
2052
2520
  providerMetadata
2053
2521
  };
2054
2522
  }
2523
+ /** Responses usage does not report a media-inclusive price, so rich calls stay unpriced. */
2524
+ requestContainsImages(request) {
2525
+ return Array.isArray(request?.input) && request.input.some((item) => {
2526
+ if (!item || typeof item !== "object") return false;
2527
+ const content = item.content;
2528
+ return Array.isArray(content) && content.some(
2529
+ (part) => part !== null && typeof part === "object" && part.type === "input_image"
2530
+ );
2531
+ });
2532
+ }
2055
2533
  // ---------------------------------------------------------------------------
2056
2534
  // Internal: parse SSE stream
2057
2535
  // ---------------------------------------------------------------------------
@@ -2139,7 +2617,7 @@ var OpenAIResponsesProvider = class {
2139
2617
  return {
2140
2618
  type: "done",
2141
2619
  usage,
2142
- cost: usage ? estimateDirectOpenAICost(response?.model ?? model, usage, {
2620
+ cost: usage && !this.requestContainsImages(request) ? estimateDirectOpenAICost(response?.model ?? model, usage, {
2143
2621
  baseUrl: this.baseUrl,
2144
2622
  request,
2145
2623
  response
@@ -2176,7 +2654,51 @@ var OpenAIResponsesProvider = class {
2176
2654
  };
2177
2655
 
2178
2656
  // src/providers/anthropic.ts
2657
+ function anthropicBase64(source) {
2658
+ return source.type === "base64" ? source.data : Buffer.from(source.data.buffer, source.data.byteOffset, source.data.byteLength).toString(
2659
+ "base64"
2660
+ );
2661
+ }
2662
+ function anthropicImageBlocks(parts) {
2663
+ const blocks = [];
2664
+ for (const part of parts) {
2665
+ if (part.type === "text") {
2666
+ blocks.push({ type: "text", text: part.text });
2667
+ continue;
2668
+ }
2669
+ const { source } = part;
2670
+ if (source.type === "provider-file") {
2671
+ if (source.provider !== "anthropic") {
2672
+ throw new UnsupportedModelInputError({
2673
+ provider: "anthropic",
2674
+ model: "unknown",
2675
+ modality: "image",
2676
+ source: "provider-file"
2677
+ });
2678
+ }
2679
+ blocks.push({ type: "image", source: { type: "file", file_id: source.reference } });
2680
+ } else if (source.type === "url") {
2681
+ blocks.push({ type: "image", source: { type: "url", url: source.url } });
2682
+ } else {
2683
+ blocks.push({
2684
+ type: "image",
2685
+ source: { type: "base64", media_type: source.mediaType, data: anthropicBase64(source) }
2686
+ });
2687
+ }
2688
+ if (part.label) blocks.push({ type: "text", text: `[Image: ${part.label}]` });
2689
+ }
2690
+ return blocks;
2691
+ }
2179
2692
  var ANTHROPIC_API_VERSION = "2023-06-01";
2693
+ var ANTHROPIC_FILES_BETA = "files-api-2025-04-14";
2694
+ var ANTHROPIC_IMAGE_MODELS = /* @__PURE__ */ new Set(["claude-sonnet-4-5", "claude-opus-4-8"]);
2695
+ function hasAnthropicProviderFileImage(messages) {
2696
+ return messages.some(
2697
+ (message) => Array.isArray(message.content) && message.content.some(
2698
+ (part) => part.type === "image" && part.source.type === "provider-file" && part.source.provider === "anthropic"
2699
+ )
2700
+ );
2701
+ }
2180
2702
  var CLAUDE_CAPABILITIES = {
2181
2703
  "claude-fable-5": {
2182
2704
  thinking: "adaptive-always-on",
@@ -2437,6 +2959,11 @@ function isEffortAtOrBelow(effort, ceiling) {
2437
2959
  function pricingContextFromBody(body) {
2438
2960
  return {
2439
2961
  model: typeof body.model === "string" ? body.model : void 0,
2962
+ hasRichInput: Array.isArray(body.messages) && body.messages.some(
2963
+ (message) => message !== null && typeof message === "object" && Array.isArray(message.content) && message.content.some(
2964
+ (part) => part !== null && typeof part === "object" && part.type === "image"
2965
+ )
2966
+ ),
2440
2967
  unpricedModifier: isAnthropicModifier("inference_geo", body.inference_geo) || isAnthropicModifier("speed", body.speed) || hasAnthropicServerTool(body.tools) || body.fallbacks !== void 0
2441
2968
  };
2442
2969
  }
@@ -2531,6 +3058,44 @@ function mergeAnthropicUsage(initial, update) {
2531
3058
  }
2532
3059
  var AnthropicProvider = class {
2533
3060
  name = "anthropic";
3061
+ inputCapabilities(model) {
3062
+ return ANTHROPIC_IMAGE_MODELS.has(model) ? { image: { sources: ["url", "bytes", "base64", "provider-file"] } } : {};
3063
+ }
3064
+ validateInput(request) {
3065
+ const effectiveModel = typeof request.providerOptions?.model === "string" ? request.providerOptions.model : request.model;
3066
+ const fail = (source, feature) => {
3067
+ throw new UnsupportedModelInputError({
3068
+ provider: this.name,
3069
+ model: effectiveModel || request.model,
3070
+ modality: "image",
3071
+ ...source ? { source } : {},
3072
+ ...feature ? { feature } : {}
3073
+ });
3074
+ };
3075
+ if (!ANTHROPIC_IMAGE_MODELS.has(effectiveModel)) {
3076
+ fail(void 0, "image input for this model");
3077
+ }
3078
+ if (request.providerOptions && "messages" in request.providerOptions) {
3079
+ fail(void 0, "raw messages providerOptions");
3080
+ }
3081
+ for (const message of request.history) {
3082
+ if (!Array.isArray(message.content)) continue;
3083
+ if (message.role !== "user") fail(void 0, "rich non-user history");
3084
+ for (const part of message.content) {
3085
+ if (part.type === "image" && part.source.type === "provider-file" && part.source.provider !== this.name) {
3086
+ fail("provider-file");
3087
+ }
3088
+ }
3089
+ }
3090
+ if (Array.isArray(request.input)) {
3091
+ for (const part of request.input) {
3092
+ if (part.type === "image" && part.source.type === "provider-file" && part.source.provider !== this.name) {
3093
+ fail("provider-file");
3094
+ }
3095
+ }
3096
+ }
3097
+ return { effectiveModel };
3098
+ }
2534
3099
  /** Anthropic ignores native `json_schema` structurally — Axl uses a system
2535
3100
  * prompt JSON instruction + client-side Zod validation (see buildRequest). */
2536
3101
  nativeStructuredOutputSupport() {
@@ -2568,7 +3133,10 @@ var AnthropicProvider = class {
2568
3133
  // chat - non-streaming completion
2569
3134
  // ---------------------------------------------------------------------------
2570
3135
  async chat(messages, options) {
2571
- const headers = this.buildHeaders(await this.resolveKey());
3136
+ const headers = this.buildHeaders(
3137
+ await this.resolveKey(),
3138
+ hasAnthropicProviderFileImage(messages)
3139
+ );
2572
3140
  const body = this.buildRequestBody(messages, options, false);
2573
3141
  const pricingContext = pricingContextFromBody(body);
2574
3142
  const res = await fetchWithRetry(
@@ -2599,7 +3167,10 @@ var AnthropicProvider = class {
2599
3167
  // stream - SSE streaming completion
2600
3168
  // ---------------------------------------------------------------------------
2601
3169
  async *stream(messages, options) {
2602
- const headers = this.buildHeaders(await this.resolveKey());
3170
+ const headers = this.buildHeaders(
3171
+ await this.resolveKey(),
3172
+ hasAnthropicProviderFileImage(messages)
3173
+ );
2603
3174
  const body = this.buildRequestBody(messages, options, true);
2604
3175
  const pricingContext = pricingContextFromBody(body);
2605
3176
  const res = await fetchWithRetry(
@@ -2631,11 +3202,12 @@ var AnthropicProvider = class {
2631
3202
  // ---------------------------------------------------------------------------
2632
3203
  // Internal: request building
2633
3204
  // ---------------------------------------------------------------------------
2634
- buildHeaders(apiKey) {
3205
+ buildHeaders(apiKey, requiresFilesBeta = false) {
2635
3206
  return {
2636
3207
  "Content-Type": "application/json",
2637
3208
  "x-api-key": apiKey,
2638
- "anthropic-version": ANTHROPIC_API_VERSION
3209
+ "anthropic-version": ANTHROPIC_API_VERSION,
3210
+ ...requiresFilesBeta ? { "anthropic-beta": ANTHROPIC_FILES_BETA } : {}
2639
3211
  };
2640
3212
  }
2641
3213
  /** Extract a human-readable message from an API error response body. */
@@ -2705,12 +3277,13 @@ ${jsonInstruction}` : jsonInstruction;
2705
3277
  mapMessages(messages) {
2706
3278
  const result = [];
2707
3279
  for (const msg of messages) {
3280
+ const text = typeof msg.content === "string" ? msg.content : "";
2708
3281
  if (msg.role === "assistant") {
2709
3282
  const replayedThinking = this.getReplayedThinkingBlocks(msg);
2710
3283
  if (msg.tool_calls && msg.tool_calls.length > 0) {
2711
3284
  const content = [...replayedThinking];
2712
- if (msg.content) {
2713
- content.push({ type: "text", text: msg.content });
3285
+ if (text) {
3286
+ content.push({ type: "text", text });
2714
3287
  }
2715
3288
  for (const tc of msg.tool_calls) {
2716
3289
  let parsedArgs;
@@ -2729,10 +3302,10 @@ ${jsonInstruction}` : jsonInstruction;
2729
3302
  result.push({ role: "assistant", content });
2730
3303
  } else if (replayedThinking.length > 0) {
2731
3304
  const content = [...replayedThinking];
2732
- if (msg.content) content.push({ type: "text", text: msg.content });
3305
+ if (text) content.push({ type: "text", text });
2733
3306
  result.push({ role: "assistant", content });
2734
3307
  } else {
2735
- result.push({ role: "assistant", content: msg.content });
3308
+ result.push({ role: "assistant", content: text });
2736
3309
  }
2737
3310
  } else if (msg.role === "tool") {
2738
3311
  result.push({
@@ -2741,12 +3314,15 @@ ${jsonInstruction}` : jsonInstruction;
2741
3314
  {
2742
3315
  type: "tool_result",
2743
3316
  tool_use_id: msg.tool_call_id,
2744
- content: msg.content
3317
+ content: text
2745
3318
  }
2746
3319
  ]
2747
3320
  });
2748
3321
  } else if (msg.role === "user") {
2749
- result.push({ role: "user", content: msg.content });
3322
+ result.push({
3323
+ role: "user",
3324
+ content: typeof msg.content === "string" ? msg.content : anthropicImageBlocks(msg.content)
3325
+ });
2750
3326
  }
2751
3327
  }
2752
3328
  return this.mergeConsecutiveRoles(result);
@@ -2851,7 +3427,7 @@ ${jsonInstruction}` : jsonInstruction;
2851
3427
  "Anthropic fallback responses cannot be continued safely; retry without fallbacks."
2852
3428
  );
2853
3429
  }
2854
- const cost = normalized && !pricingContext.unpricedModifier && !isModifiedAnthropicResponse(json, effectiveModel) && !hasFallbackBoundary && !refused ? estimateAnthropicCost(effectiveModel, normalized.pricingUsage) : void 0;
3430
+ const cost = normalized && !pricingContext.hasRichInput && !pricingContext.unpricedModifier && !isModifiedAnthropicResponse(json, effectiveModel) && !hasFallbackBoundary && !refused ? estimateAnthropicCost(effectiveModel, normalized.pricingUsage) : void 0;
2855
3431
  return {
2856
3432
  content,
2857
3433
  thinking_content: thinkingContent || void 0,
@@ -2888,7 +3464,7 @@ ${jsonInstruction}` : jsonInstruction;
2888
3464
  const doneChunk = () => ({
2889
3465
  type: "done",
2890
3466
  usage,
2891
- cost: pricingUsage && !unpricedModifier && !hasFallbackBoundary && !refused ? estimateAnthropicCost(effectiveModel, pricingUsage) : void 0,
3467
+ cost: pricingUsage && !pricingContext.hasRichInput && !unpricedModifier && !hasFallbackBoundary && !refused ? estimateAnthropicCost(effectiveModel, pricingUsage) : void 0,
2892
3468
  providerMetadata: thinkingBlocks.length > 0 && !hasFallbackBoundary && !hasFallbackIterationSignal ? { anthropicThinkingBlocks: thinkingBlocks } : void 0
2893
3469
  });
2894
3470
  try {
@@ -3036,6 +3612,60 @@ function lastFallbackIndex(blocks) {
3036
3612
  }
3037
3613
 
3038
3614
  // src/providers/gemini.ts
3615
+ function hasRichGeminiMessages(messages) {
3616
+ return messages.some((message) => Array.isArray(message.content));
3617
+ }
3618
+ function geminiImageBase64(source) {
3619
+ return source.type === "base64" ? source.data : Buffer.from(source.data.buffer, source.data.byteOffset, source.data.byteLength).toString(
3620
+ "base64"
3621
+ );
3622
+ }
3623
+ function geminiInteractionContent(parts, model) {
3624
+ const content = [];
3625
+ for (const part of parts) {
3626
+ if (part.type === "text") {
3627
+ content.push({ type: "text", text: part.text });
3628
+ continue;
3629
+ }
3630
+ const { source } = part;
3631
+ if (source.type === "provider-file") {
3632
+ if (source.provider !== "google") {
3633
+ throw new UnsupportedModelInputError({
3634
+ provider: "google",
3635
+ model,
3636
+ modality: "image",
3637
+ source: "provider-file"
3638
+ });
3639
+ }
3640
+ if (!source.mediaType) {
3641
+ throw new UnsupportedModelInputError({
3642
+ provider: "google",
3643
+ model,
3644
+ modality: "image",
3645
+ source: "provider-file",
3646
+ feature: "Interactions URI image mediaType"
3647
+ });
3648
+ }
3649
+ content.push({
3650
+ type: "image",
3651
+ uri: source.reference,
3652
+ mime_type: source.mediaType
3653
+ });
3654
+ } else if (source.type === "url") {
3655
+ throw new UnsupportedModelInputError({
3656
+ provider: "google",
3657
+ model,
3658
+ modality: "image",
3659
+ source: "url",
3660
+ feature: "direct URL image input; pass bytes/base64 or a Gemini provider-file"
3661
+ });
3662
+ } else {
3663
+ content.push({ type: "image", data: geminiImageBase64(source), mime_type: source.mediaType });
3664
+ }
3665
+ if (part.label) content.push({ type: "text", text: `[Image: ${part.label}]` });
3666
+ }
3667
+ return content;
3668
+ }
3039
3669
  var GEMINI_DISALLOWED_SCHEMA_KEYS = /* @__PURE__ */ new Set([
3040
3670
  "additionalProperties",
3041
3671
  "$schema",
@@ -3107,6 +3737,7 @@ var GEMINI_PRICING = {
3107
3737
  "gemini-3.5-flash-lite": { input: 3e-7, cached: 3e-8, output: 25e-7 },
3108
3738
  "gemini-3.6-flash": { input: 15e-7, cached: 15e-8, output: 75e-7 }
3109
3739
  };
3740
+ var GEMINI_INTERACTIONS_IMAGE_MODELS = /* @__PURE__ */ new Set(["gemini-3.7-flash"]);
3110
3741
  function isValidTokenCount2(value) {
3111
3742
  return typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
3112
3743
  }
@@ -3194,7 +3825,10 @@ var GEMINI_3X_MIN_THINKING_LEVEL = /* @__PURE__ */ new Map([
3194
3825
  ["gemini-3.1-flash-lite-preview", "minimal"],
3195
3826
  ["gemini-3.5-flash", "minimal"],
3196
3827
  ["gemini-3.5-flash-lite", "minimal"],
3197
- ["gemini-3.6-flash", "minimal"]
3828
+ ["gemini-3.6-flash", "minimal"],
3829
+ // L4 live evidence: Interactions gemini-3.7-flash accepts low/medium/high,
3830
+ // but rejects minimal. Keep this exact entry isolated from legacy aliases.
3831
+ ["gemini-3.7-flash", "low"]
3198
3832
  ]);
3199
3833
  var GEMINI_MODELS_WITHOUT_PORTABLE_TEMPERATURE = /* @__PURE__ */ new Set([
3200
3834
  "gemini-3.5-flash-lite",
@@ -3221,6 +3855,61 @@ function warnGemini3xEffortNone(model) {
3221
3855
  }
3222
3856
  var GeminiProvider = class {
3223
3857
  name = "google";
3858
+ inputCapabilities(model) {
3859
+ return GEMINI_INTERACTIONS_IMAGE_MODELS.has(model) ? { image: { sources: ["bytes", "base64", "provider-file"] } } : {};
3860
+ }
3861
+ validateInput(request) {
3862
+ const effectiveModel = typeof request.providerOptions?.model === "string" ? request.providerOptions.model : request.model;
3863
+ const fail = (source, feature) => {
3864
+ throw new UnsupportedModelInputError({
3865
+ provider: this.name,
3866
+ model: effectiveModel || request.model,
3867
+ modality: "image",
3868
+ ...source ? { source } : {},
3869
+ ...feature ? { feature } : {}
3870
+ });
3871
+ };
3872
+ if (!GEMINI_INTERACTIONS_IMAGE_MODELS.has(effectiveModel)) {
3873
+ fail(void 0, "image input for this model");
3874
+ }
3875
+ if (request.providerOptions) {
3876
+ const forbidden = [
3877
+ "input",
3878
+ "contents",
3879
+ "messages",
3880
+ "previous_interaction_id",
3881
+ "background",
3882
+ "stream",
3883
+ "store"
3884
+ ].find((key) => key in request.providerOptions);
3885
+ if (forbidden) fail(void 0, `raw ${forbidden} providerOptions`);
3886
+ }
3887
+ for (const message of request.history) {
3888
+ if (!Array.isArray(message.content)) continue;
3889
+ if (message.role !== "user") fail(void 0, "rich non-user history");
3890
+ for (const part of message.content) {
3891
+ if (part.type !== "image") continue;
3892
+ if (part.source.type === "url")
3893
+ fail("url", "direct URL image input; pass bytes/base64 or a Gemini provider-file");
3894
+ if (part.source.type === "provider-file" && part.source.provider !== this.name)
3895
+ fail("provider-file");
3896
+ if (part.source.type === "provider-file" && !part.source.mediaType)
3897
+ fail("provider-file", "Interactions URI image mediaType");
3898
+ }
3899
+ }
3900
+ if (Array.isArray(request.input)) {
3901
+ for (const part of request.input) {
3902
+ if (part.type !== "image") continue;
3903
+ if (part.source.type === "url")
3904
+ fail("url", "direct URL image input; pass bytes/base64 or a Gemini provider-file");
3905
+ if (part.source.type === "provider-file" && part.source.provider !== this.name)
3906
+ fail("provider-file");
3907
+ if (part.source.type === "provider-file" && !part.source.mediaType)
3908
+ fail("provider-file", "Interactions URI image mediaType");
3909
+ }
3910
+ }
3911
+ return { effectiveModel };
3912
+ }
3224
3913
  /** Gemini accepts a `responseSchema` but `sanitizeSchemaForGemini` strips
3225
3914
  * keywords it doesn't support (`$ref`/`$defs`/`additionalProperties`/…), so a
3226
3915
  * derived schema can lose constraints — lossy, not faithful. */
@@ -3259,6 +3948,7 @@ var GeminiProvider = class {
3259
3948
  // chat - non-streaming completion
3260
3949
  // ---------------------------------------------------------------------------
3261
3950
  async chat(messages, options) {
3951
+ if (hasRichGeminiMessages(messages)) return this.chatInteraction(messages, options);
3262
3952
  const body = this.buildRequestBody(messages, options);
3263
3953
  const pricingContext = this.pricingContext(this.requestModel(options), body);
3264
3954
  this.assertSafeGemini3Continuation(messages, pricingContext.model);
@@ -3291,6 +3981,10 @@ var GeminiProvider = class {
3291
3981
  // stream - SSE streaming completion
3292
3982
  // ---------------------------------------------------------------------------
3293
3983
  async *stream(messages, options) {
3984
+ if (hasRichGeminiMessages(messages)) {
3985
+ yield* this.streamInteraction(messages, options);
3986
+ return;
3987
+ }
3294
3988
  const body = this.buildRequestBody(messages, options);
3295
3989
  const pricingContext = this.pricingContext(this.requestModel(options), body);
3296
3990
  this.assertSafeGemini3Continuation(messages, pricingContext.model);
@@ -3322,50 +4016,383 @@ var GeminiProvider = class {
3322
4016
  yield* this.parseSSEStream(res.body, pricingContext);
3323
4017
  }
3324
4018
  // ---------------------------------------------------------------------------
3325
- // Internal: request building
4019
+ // Rich image requests use Gemini Interactions. It is deliberately stateless:
4020
+ // Axl sends the complete application-owned history and never asks Google to
4021
+ // retain an interaction or follow a previous_interaction_id.
3326
4022
  // ---------------------------------------------------------------------------
3327
- buildHeaders(apiKey) {
3328
- return {
3329
- "Content-Type": "application/json",
3330
- "x-goog-api-key": apiKey
3331
- };
4023
+ async chatInteraction(messages, options) {
4024
+ this.assertSafeInteractionOptions(options);
4025
+ const body = this.buildInteractionRequestBody(messages, options, false);
4026
+ const headers = this.buildHeaders(await this.resolveKey());
4027
+ const res = await fetchWithRetry(
4028
+ `${this.baseUrl}/interactions`,
4029
+ { method: "POST", headers, body: JSON.stringify(body), signal: options.signal },
4030
+ { governor: this.governor, provider: this.name }
4031
+ );
4032
+ if (!res.ok) {
4033
+ const errorBody = await res.text();
4034
+ throw buildProviderError({
4035
+ provider: this.name,
4036
+ status: res.status,
4037
+ headers: res.headers,
4038
+ message: this.extractErrorMessage(errorBody, res.status),
4039
+ body: errorBody
4040
+ });
4041
+ }
4042
+ return this.parseInteractionResponse(await res.json());
3332
4043
  }
3333
- extractErrorMessage(body, status) {
3334
- try {
3335
- const json = JSON.parse(body);
3336
- if (json.error?.message) {
3337
- return `Gemini API error (${status}): ${json.error.message}`;
3338
- }
3339
- } catch {
4044
+ async *streamInteraction(messages, options) {
4045
+ this.assertSafeInteractionOptions(options);
4046
+ const body = this.buildInteractionRequestBody(messages, options, true);
4047
+ const headers = this.buildHeaders(await this.resolveKey());
4048
+ const res = await fetchWithRetry(
4049
+ `${this.baseUrl}/interactions?alt=sse`,
4050
+ { method: "POST", headers, body: JSON.stringify(body), signal: options.signal },
4051
+ { governor: this.governor, provider: this.name }
4052
+ );
4053
+ if (!res.ok) {
4054
+ const errorBody = await res.text();
4055
+ throw buildProviderError({
4056
+ provider: this.name,
4057
+ status: res.status,
4058
+ headers: res.headers,
4059
+ message: this.extractErrorMessage(errorBody, res.status),
4060
+ body: errorBody
4061
+ });
3340
4062
  }
3341
- return `Gemini API error (${status}): ${body}`;
4063
+ if (!res.body) throw new Error("Gemini Interactions stream has no body");
4064
+ yield* this.parseInteractionSSEStream(res.body);
3342
4065
  }
3343
- pricingContext(fallbackModel, body) {
3344
- return {
3345
- // Gemini's model is normally selected in the URL. providerOptions is an
3346
- // explicit raw escape hatch, so honor a post-merge model override for both
3347
- // the URL and pricing rather than pricing a different request.
3348
- model: typeof body.model === "string" ? body.model : fallbackModel,
3349
- serviceTier: body.serviceTier,
3350
- eligibleRequest: this.baseUrl === "https://generativelanguage.googleapis.com/v1beta" && this.isTextOnlyClientRequest(body)
4066
+ buildInteractionRequestBody(messages, options, stream) {
4067
+ const model = this.requestModel(options);
4068
+ const systemInstruction = messages.filter((message) => message.role === "system").map((message) => typeof message.content === "string" ? message.content : "").filter(Boolean).join("\n\n");
4069
+ const body = {
4070
+ model,
4071
+ input: this.mapInteractionInput(
4072
+ messages.filter((message) => message.role !== "system"),
4073
+ model
4074
+ ),
4075
+ stream
3351
4076
  };
3352
- }
3353
- requestModel(options) {
3354
- return typeof options.providerOptions?.model === "string" ? options.providerOptions.model : options.model;
3355
- }
3356
- assertSafeGemini3Continuation(messages, model) {
3357
- if (GEMINI_MODELS_WITHOUT_PORTABLE_TEMPERATURE.has(model)) {
3358
- const mapped = this.mapMessages(messages.filter((message) => message.role !== "system"));
3359
- if (mapped.at(-1)?.role === "model") {
3360
- throw new Error(
3361
- `${model} does not support a terminal assistant/model prefill; end the request with a user or tool message.`
3362
- );
4077
+ if (systemInstruction) body.system_instruction = systemInstruction;
4078
+ if (options.tools?.length) {
4079
+ body.tools = options.tools.map((tool2) => ({
4080
+ type: "function",
4081
+ name: tool2.function.name,
4082
+ description: tool2.function.description,
4083
+ parameters: sanitizeSchemaForGemini(tool2.function.parameters)
4084
+ }));
4085
+ }
4086
+ const generationConfig = {};
4087
+ if (options.maxTokens !== void 0) generationConfig.max_output_tokens = options.maxTokens;
4088
+ if (options.stop) generationConfig.stop_sequences = options.stop;
4089
+ if (options.toolChoice !== void 0) {
4090
+ generationConfig.tool_choice = typeof options.toolChoice === "string" ? options.toolChoice === "required" ? "any" : options.toolChoice : { allowed_tools: { mode: "any", tools: [options.toolChoice.function.name] } };
4091
+ }
4092
+ const thinking = resolveThinkingOptions(options);
4093
+ if (thinking.thinkingDisabled) {
4094
+ generationConfig.thinking_level = isGemini3x(model) ? minThinkingLevel(model) : "minimal";
4095
+ } else if (thinking.hasBudgetOverride) {
4096
+ generationConfig.thinking_level = budgetToThinkingLevel(thinking.thinkingBudget);
4097
+ } else if (thinking.activeEffort) {
4098
+ generationConfig.thinking_level = isGemini3x(model) ? THINKING_LEVELS[thinking.activeEffort] ?? "medium" : budgetToThinkingLevel(THINKING_BUDGETS2[thinking.activeEffort] ?? 5e3);
4099
+ }
4100
+ if (thinking.includeThoughts) generationConfig.thinking_summaries = "auto";
4101
+ if (Object.keys(generationConfig).length) body.generation_config = generationConfig;
4102
+ if (options.responseFormat?.type && options.responseFormat.type !== "text") {
4103
+ body.response_format = {
4104
+ type: "text",
4105
+ mime_type: "application/json",
4106
+ ...options.responseFormat.type === "json_schema" ? { schema: sanitizeSchemaForGemini(options.responseFormat.json_schema.schema) } : {}
4107
+ };
4108
+ }
4109
+ if (options.providerOptions) {
4110
+ const overrides = { ...options.providerOptions };
4111
+ delete overrides.model;
4112
+ delete overrides.input;
4113
+ delete overrides.contents;
4114
+ delete overrides.messages;
4115
+ delete overrides.previous_interaction_id;
4116
+ delete overrides.background;
4117
+ delete overrides.stream;
4118
+ delete overrides.store;
4119
+ const rawGenerationConfig = overrides.generation_config;
4120
+ if (rawGenerationConfig && typeof rawGenerationConfig === "object" && !Array.isArray(rawGenerationConfig)) {
4121
+ const generationConfigOverrides = { ...rawGenerationConfig };
4122
+ delete generationConfigOverrides.temperature;
4123
+ if (Object.keys(generationConfigOverrides).length)
4124
+ overrides.generation_config = generationConfigOverrides;
4125
+ else delete overrides.generation_config;
3363
4126
  }
4127
+ Object.assign(body, overrides);
3364
4128
  }
3365
- if (!isGemini3x(model)) return;
3366
- const replayedNativeCallIds = /* @__PURE__ */ new Set();
3367
- for (const message of messages) {
3368
- if (message.role !== "assistant" || !Array.isArray(message.providerMetadata?.geminiParts)) {
4129
+ body.stream = stream;
4130
+ body.store = false;
4131
+ return body;
4132
+ }
4133
+ assertSafeInteractionOptions(options) {
4134
+ const model = this.requestModel(options);
4135
+ const fail = (feature) => {
4136
+ throw new UnsupportedModelInputError({
4137
+ provider: this.name,
4138
+ model,
4139
+ modality: "image",
4140
+ feature
4141
+ });
4142
+ };
4143
+ if (!GEMINI_INTERACTIONS_IMAGE_MODELS.has(model)) fail("image input for this model");
4144
+ const forbidden = [
4145
+ "input",
4146
+ "contents",
4147
+ "messages",
4148
+ "previous_interaction_id",
4149
+ "background",
4150
+ "stream",
4151
+ "store"
4152
+ ].find((key) => key in (options.providerOptions ?? {}));
4153
+ if (forbidden) fail(`raw ${forbidden} providerOptions`);
4154
+ }
4155
+ mapInteractionInput(messages, model) {
4156
+ const steps = [];
4157
+ for (const message of messages) {
4158
+ if (message.role === "user") {
4159
+ steps.push({
4160
+ type: "user_input",
4161
+ content: Array.isArray(message.content) ? geminiInteractionContent(message.content, model) : [{ type: "text", text: message.content }]
4162
+ });
4163
+ } else if (message.role === "assistant") {
4164
+ const nativeSteps = message.providerMetadata?.geminiInteractionSteps;
4165
+ if (Array.isArray(nativeSteps) && nativeSteps.every(isGeminiInteractionStep)) {
4166
+ steps.push(...nativeSteps);
4167
+ continue;
4168
+ }
4169
+ if (typeof message.content === "string" && message.content) {
4170
+ steps.push({ type: "model_output", content: [{ type: "text", text: message.content }] });
4171
+ }
4172
+ for (const call of message.tool_calls ?? []) {
4173
+ steps.push({
4174
+ type: "function_call",
4175
+ id: call.id,
4176
+ name: call.function.name,
4177
+ arguments: safeJsonObject(call.function.arguments)
4178
+ });
4179
+ }
4180
+ } else if (message.role === "tool") {
4181
+ steps.push({
4182
+ type: "function_result",
4183
+ call_id: message.tool_call_id ?? "",
4184
+ result: [
4185
+ { type: "text", text: typeof message.content === "string" ? message.content : "" }
4186
+ ]
4187
+ });
4188
+ }
4189
+ }
4190
+ return steps;
4191
+ }
4192
+ parseInteractionResponse(json) {
4193
+ let content = "";
4194
+ let thinkingContent = "";
4195
+ const toolCalls = [];
4196
+ for (const step of json.steps ?? []) {
4197
+ if (step.type === "model_output") content += interactionText(step.content);
4198
+ else if (step.type === "thought") thinkingContent += interactionText(step.summary);
4199
+ else if (step.type === "function_call" && step.id && step.name) {
4200
+ toolCalls.push({
4201
+ id: step.id,
4202
+ type: "function",
4203
+ function: { name: step.name, arguments: JSON.stringify(step.arguments ?? {}) }
4204
+ });
4205
+ }
4206
+ }
4207
+ this.assertInteractionTerminalStatus(json.status, toolCalls.length);
4208
+ const usage = normalizeInteractionUsage(json.usage);
4209
+ return {
4210
+ content,
4211
+ thinking_content: thinkingContent || void 0,
4212
+ tool_calls: toolCalls.length ? toolCalls : void 0,
4213
+ usage,
4214
+ // Interactions returns modality usage but no authoritative monetary cost.
4215
+ cost: void 0,
4216
+ providerMetadata: json.steps?.length ? { geminiInteractionSteps: json.steps.filter(isGeminiInteractionStep) } : void 0
4217
+ };
4218
+ }
4219
+ async *parseInteractionSSEStream(body) {
4220
+ const reader = body.getReader();
4221
+ const decoder = new TextDecoder();
4222
+ let buffer = "";
4223
+ const steps = /* @__PURE__ */ new Map();
4224
+ const argumentBuffers = /* @__PURE__ */ new Map();
4225
+ const stopped = /* @__PURE__ */ new Set();
4226
+ let completed = false;
4227
+ try {
4228
+ while (true) {
4229
+ const { done, value } = await reader.read();
4230
+ if (done) break;
4231
+ buffer += decoder.decode(value, { stream: true });
4232
+ const lines = buffer.split("\n");
4233
+ buffer = lines.pop() ?? "";
4234
+ for (const line of lines) {
4235
+ const trimmed = line.trim();
4236
+ if (!trimmed.startsWith("data: ")) continue;
4237
+ if (trimmed.slice(6) === "[DONE]") continue;
4238
+ let event;
4239
+ try {
4240
+ event = JSON.parse(trimmed.slice(6));
4241
+ } catch {
4242
+ continue;
4243
+ }
4244
+ if (event.event_type === "error") {
4245
+ throw new Error(event.error?.message ?? "Gemini Interactions stream failed");
4246
+ }
4247
+ if (event.event_type === "step.start" && event.step) {
4248
+ const index = event.index;
4249
+ if (index === void 0 || steps.has(index)) {
4250
+ throw new Error("Gemini Interactions stream contained an invalid step.start index");
4251
+ }
4252
+ const step = cloneInteractionStep(event.step);
4253
+ steps.set(index, step);
4254
+ if (step.type === "model_output") {
4255
+ for (const text of interactionTextChunks(step.content)) {
4256
+ yield { type: "text_delta", content: text };
4257
+ }
4258
+ } else if (step.type === "thought") {
4259
+ for (const text of interactionTextChunks(step.summary)) {
4260
+ yield { type: "thinking_delta", content: text };
4261
+ }
4262
+ }
4263
+ if (step.type === "function_call" && step.id && step.name) {
4264
+ argumentBuffers.set(index, "");
4265
+ yield { type: "tool_call_delta", id: step.id, name: step.name };
4266
+ }
4267
+ } else if (event.event_type === "step.delta" && event.delta) {
4268
+ const delta = event.delta;
4269
+ const index = event.index;
4270
+ if (index === void 0 || stopped.has(index)) {
4271
+ throw new Error("Gemini Interactions stream delta arrived outside an active step");
4272
+ }
4273
+ const step = steps.get(index);
4274
+ if (!step)
4275
+ throw new Error("Gemini Interactions stream delta arrived before step.start");
4276
+ if (delta.type === "text" && typeof delta.text === "string" && step.type === "model_output") {
4277
+ appendInteractionContent(step, "content", { type: "text", text: delta.text });
4278
+ yield { type: "text_delta", content: delta.text };
4279
+ } else if (delta.type === "thought_summary" && step.type === "thought" && delta.content) {
4280
+ appendInteractionContent(step, "summary", delta.content);
4281
+ for (const text of interactionTextChunks([delta.content])) {
4282
+ yield { type: "thinking_delta", content: text };
4283
+ }
4284
+ } else if (delta.type === "thought_signature" && step.type === "thought" && typeof delta.signature === "string") {
4285
+ step.signature = delta.signature;
4286
+ } else if (delta.type === "arguments_delta" && step.type === "function_call" && typeof delta.arguments === "string") {
4287
+ argumentBuffers.set(index, (argumentBuffers.get(index) ?? "") + delta.arguments);
4288
+ if (step.id)
4289
+ yield { type: "tool_call_delta", id: step.id, arguments: delta.arguments };
4290
+ }
4291
+ } else if (event.event_type === "step.stop") {
4292
+ const index = event.index;
4293
+ const step = index === void 0 ? void 0 : steps.get(index);
4294
+ if (step === void 0 || stopped.has(index)) {
4295
+ throw new Error("Gemini Interactions stream contained an invalid step.stop index");
4296
+ }
4297
+ if (step.type === "function_call") {
4298
+ const rawArguments = argumentBuffers.get(index) ?? "";
4299
+ if (rawArguments) {
4300
+ try {
4301
+ const parsed = JSON.parse(rawArguments);
4302
+ if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) {
4303
+ throw new Error("not an object");
4304
+ }
4305
+ step.arguments = parsed;
4306
+ } catch {
4307
+ throw new Error(
4308
+ "Gemini Interactions function_call arguments were not valid JSON"
4309
+ );
4310
+ }
4311
+ }
4312
+ }
4313
+ stopped.add(index);
4314
+ } else if (event.event_type === "interaction.completed") {
4315
+ if (steps.size !== stopped.size) {
4316
+ throw new Error("Gemini Interactions stream completed before all steps stopped");
4317
+ }
4318
+ const toolCallCount = [...steps.values()].filter(
4319
+ (step) => step.type === "function_call" && !!step.id && !!step.name
4320
+ ).length;
4321
+ this.assertInteractionTerminalStatus(event.interaction?.status, toolCallCount);
4322
+ completed = true;
4323
+ yield {
4324
+ type: "done",
4325
+ usage: normalizeInteractionUsage(event.interaction?.usage),
4326
+ providerMetadata: steps.size ? {
4327
+ geminiInteractionSteps: [...steps.entries()].sort(([a], [b]) => a - b).map(([, step]) => step)
4328
+ } : void 0
4329
+ };
4330
+ return;
4331
+ }
4332
+ }
4333
+ }
4334
+ if (!completed)
4335
+ throw new Error("Gemini Interactions stream ended before interaction.completed");
4336
+ } finally {
4337
+ reader.releaseLock();
4338
+ }
4339
+ }
4340
+ assertInteractionTerminalStatus(status, toolCallCount) {
4341
+ if (status === "completed") return;
4342
+ if (status === "requires_action" && toolCallCount > 0) return;
4343
+ const normalizedStatus = typeof status === "string" ? status : "unknown";
4344
+ throw new ProviderError({
4345
+ provider: this.name,
4346
+ status: 0,
4347
+ retryable: false,
4348
+ message: `Gemini Interactions ended with non-success status: ${normalizedStatus}`
4349
+ });
4350
+ }
4351
+ // ---------------------------------------------------------------------------
4352
+ // Internal: request building
4353
+ // ---------------------------------------------------------------------------
4354
+ buildHeaders(apiKey) {
4355
+ return {
4356
+ "Content-Type": "application/json",
4357
+ "x-goog-api-key": apiKey
4358
+ };
4359
+ }
4360
+ extractErrorMessage(body, status) {
4361
+ try {
4362
+ const json = JSON.parse(body);
4363
+ if (json.error?.message) {
4364
+ return `Gemini API error (${status}): ${json.error.message}`;
4365
+ }
4366
+ } catch {
4367
+ }
4368
+ return `Gemini API error (${status}): ${body}`;
4369
+ }
4370
+ pricingContext(fallbackModel, body) {
4371
+ return {
4372
+ // Gemini's model is normally selected in the URL. providerOptions is an
4373
+ // explicit raw escape hatch, so honor a post-merge model override for both
4374
+ // the URL and pricing rather than pricing a different request.
4375
+ model: typeof body.model === "string" ? body.model : fallbackModel,
4376
+ serviceTier: body.serviceTier,
4377
+ eligibleRequest: this.baseUrl === "https://generativelanguage.googleapis.com/v1beta" && this.isTextOnlyClientRequest(body)
4378
+ };
4379
+ }
4380
+ requestModel(options) {
4381
+ return typeof options.providerOptions?.model === "string" ? options.providerOptions.model : options.model;
4382
+ }
4383
+ assertSafeGemini3Continuation(messages, model) {
4384
+ if (GEMINI_MODELS_WITHOUT_PORTABLE_TEMPERATURE.has(model)) {
4385
+ const mapped = this.mapMessages(messages.filter((message) => message.role !== "system"));
4386
+ if (mapped.at(-1)?.role === "model") {
4387
+ throw new Error(
4388
+ `${model} does not support a terminal assistant/model prefill; end the request with a user or tool message.`
4389
+ );
4390
+ }
4391
+ }
4392
+ if (!isGemini3x(model)) return;
4393
+ const replayedNativeCallIds = /* @__PURE__ */ new Set();
4394
+ for (const message of messages) {
4395
+ if (message.role !== "assistant" || !Array.isArray(message.providerMetadata?.geminiParts)) {
3369
4396
  continue;
3370
4397
  }
3371
4398
  for (const part of message.providerMetadata.geminiParts) {
@@ -3576,14 +4603,15 @@ var GeminiProvider = class {
3576
4603
  }
3577
4604
  const result = [];
3578
4605
  for (const msg of messages) {
4606
+ const text = typeof msg.content === "string" ? msg.content : "";
3579
4607
  if (msg.role === "assistant") {
3580
4608
  const rawParts = msg.providerMetadata?.geminiParts;
3581
4609
  if (rawParts && rawParts.length > 0) {
3582
4610
  result.push({ role: "model", parts: rawParts });
3583
4611
  } else {
3584
4612
  const parts = [];
3585
- if (msg.content) {
3586
- parts.push({ text: msg.content });
4613
+ if (text) {
4614
+ parts.push({ text });
3587
4615
  }
3588
4616
  if (msg.tool_calls && msg.tool_calls.length > 0) {
3589
4617
  for (const tc of msg.tool_calls) {
@@ -3609,7 +4637,7 @@ var GeminiProvider = class {
3609
4637
  const functionName = (msg.tool_call_id ? geminiNativeNames.get(msg.tool_call_id) : void 0) ?? toolCallIdToName.get(msg.tool_call_id) ?? "unknown";
3610
4638
  const functionResponse = {
3611
4639
  name: functionName,
3612
- response: parseGeminiFunctionResponse(msg.content)
4640
+ response: parseGeminiFunctionResponse(text)
3613
4641
  };
3614
4642
  if (msg.tool_call_id && geminiNativeIds.has(msg.tool_call_id)) {
3615
4643
  functionResponse.id = msg.tool_call_id;
@@ -3619,7 +4647,7 @@ var GeminiProvider = class {
3619
4647
  parts: [{ functionResponse }]
3620
4648
  });
3621
4649
  } else if (msg.role === "user") {
3622
- result.push({ role: "user", parts: [{ text: msg.content }] });
4650
+ result.push({ role: "user", parts: [{ text }] });
3623
4651
  }
3624
4652
  }
3625
4653
  return this.mergeConsecutiveRoles(result);
@@ -3807,6 +4835,56 @@ var GeminiProvider = class {
3807
4835
  }
3808
4836
  }
3809
4837
  };
4838
+ function isGeminiInteractionStep(value) {
4839
+ return value !== null && typeof value === "object" && ["user_input", "model_output", "thought", "function_call", "function_result"].includes(
4840
+ value.type
4841
+ );
4842
+ }
4843
+ function interactionText(content) {
4844
+ return content?.map((part) => part.type === "text" && typeof part.text === "string" ? part.text : "").join("") ?? "";
4845
+ }
4846
+ function interactionTextChunks(content) {
4847
+ return content?.flatMap(
4848
+ (part) => part.type === "text" && typeof part.text === "string" ? [part.text] : []
4849
+ ) ?? [];
4850
+ }
4851
+ function cloneInteractionStep(step) {
4852
+ return JSON.parse(JSON.stringify(step));
4853
+ }
4854
+ function appendInteractionContent(step, field, content) {
4855
+ const existing = step[field] ?? [];
4856
+ const last = existing.at(-1);
4857
+ if (content.type === "text" && last?.type === "text" && typeof content.text === "string" && typeof last.text === "string") {
4858
+ last.text += content.text;
4859
+ } else {
4860
+ existing.push(content);
4861
+ }
4862
+ step[field] = existing;
4863
+ }
4864
+ function safeJsonObject(value) {
4865
+ try {
4866
+ const parsed = JSON.parse(value);
4867
+ return parsed !== null && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : {};
4868
+ } catch {
4869
+ return {};
4870
+ }
4871
+ }
4872
+ function normalizeInteractionUsage(usage) {
4873
+ if (!usage || !isValidTokenCount2(usage.total_input_tokens) || !isValidTokenCount2(usage.total_output_tokens) || !isValidTokenCount2(usage.total_tokens)) {
4874
+ return void 0;
4875
+ }
4876
+ const cached = usage.total_cached_tokens;
4877
+ const thinking = usage.total_thought_tokens;
4878
+ if (cached !== void 0 && (!isValidTokenCount2(cached) || cached > usage.total_input_tokens) || thinking !== void 0 && !isValidTokenCount2(thinking))
4879
+ return void 0;
4880
+ return {
4881
+ prompt_tokens: usage.total_input_tokens,
4882
+ completion_tokens: usage.total_output_tokens,
4883
+ total_tokens: usage.total_tokens,
4884
+ ...cached ? { cached_tokens: cached } : {},
4885
+ ...thinking ? { reasoning_tokens: thinking } : {}
4886
+ };
4887
+ }
3810
4888
 
3811
4889
  // src/providers/profiles/openrouter.ts
3812
4890
  var OPENROUTER_PROFILE = {
@@ -4247,59 +5325,1211 @@ var ProviderRegistry = class {
4247
5325
  }
4248
5326
  throw new Error(`Unknown provider "${name}". Registered providers: ${this.list().join(", ")}`);
4249
5327
  }
4250
- /**
4251
- * Resolve a "provider:model" URI string into a Provider instance and model name.
4252
- *
4253
- * Supported formats:
4254
- * - "openai:gpt-4o" -> provider=openai, model=gpt-4o
4255
- * - "anthropic:claude-3" -> provider=anthropic, model=claude-3
4256
- * - "gpt-4o" -> uses defaultProvider from config, model=gpt-4o
4257
- * - undefined / empty -> uses defaultProvider and defaultModel from config
4258
- *
4259
- * @param uri Provider:model string, or just a model name
4260
- * @param config Axl configuration for provider options and defaults
4261
- */
5328
+ /**
5329
+ * Resolve a "provider:model" URI string into a Provider instance and model name.
5330
+ *
5331
+ * Supported formats:
5332
+ * - "openai:gpt-4o" -> provider=openai, model=gpt-4o
5333
+ * - "anthropic:claude-3" -> provider=anthropic, model=claude-3
5334
+ * - "gpt-4o" -> uses defaultProvider from config, model=gpt-4o
5335
+ * - undefined / empty -> uses defaultProvider and defaultModel from config
5336
+ *
5337
+ * @param uri Provider:model string, or just a model name
5338
+ * @param config Axl configuration for provider options and defaults
5339
+ */
5340
+ resolve(uri, config = {}) {
5341
+ if (!uri) {
5342
+ const providerName2 = config.defaultProvider ?? "openai";
5343
+ const model2 = config.defaultModel ?? "gpt-4o";
5344
+ return { provider: this.get(providerName2, config), model: model2 };
5345
+ }
5346
+ const colonIndex = uri.indexOf(":");
5347
+ if (colonIndex === -1) {
5348
+ const providerName2 = config.defaultProvider ?? "openai";
5349
+ return { provider: this.get(providerName2, config), model: uri };
5350
+ }
5351
+ const providerName = uri.slice(0, colonIndex);
5352
+ const model = uri.slice(colonIndex + 1);
5353
+ if (!providerName || !model) {
5354
+ throw new Error(
5355
+ `Invalid provider URI "${uri}". Expected format: "provider:model" (e.g. "openai:gpt-4o")`
5356
+ );
5357
+ }
5358
+ return { provider: this.get(providerName, config), model };
5359
+ }
5360
+ /**
5361
+ * Clear all cached provider instances. Useful for testing or reconfiguration.
5362
+ */
5363
+ clearCache() {
5364
+ this.instances.clear();
5365
+ }
5366
+ /**
5367
+ * Clear all registered factories (including built-ins).
5368
+ * Useful for test runtimes where only explicitly registered instances should be used.
5369
+ */
5370
+ clearFactories() {
5371
+ this.factories.clear();
5372
+ }
5373
+ };
5374
+ var defaultRegistry = new ProviderRegistry();
5375
+
5376
+ // src/transcription.ts
5377
+ var MAX_INLINE_TRANSCRIPTION_BYTES = 25 * 1024 * 1024;
5378
+ var RESERVED_PROVIDER_OPTION_KEYS = /* @__PURE__ */ new Set([
5379
+ "model",
5380
+ "audio",
5381
+ "signal",
5382
+ "cleanup",
5383
+ "cleanupStatus",
5384
+ "usage",
5385
+ "cost",
5386
+ "pricingStatus",
5387
+ "accounting"
5388
+ ]);
5389
+ function nonEmptyString(value) {
5390
+ return typeof value === "string" && value.trim().length > 0;
5391
+ }
5392
+ function cloneValue(value) {
5393
+ try {
5394
+ return structuredClone(value);
5395
+ } catch {
5396
+ throw new InvalidTranscriptionInputError("Transcription options must be structured-cloneable");
5397
+ }
5398
+ }
5399
+ function normalizeTranscriptionRequest(request) {
5400
+ if (!request || typeof request !== "object" || !nonEmptyString(request.model)) {
5401
+ throw new InvalidTranscriptionInputError("A transcription model URI is required");
5402
+ }
5403
+ const audio = request.audio;
5404
+ if (!audio || typeof audio !== "object") {
5405
+ throw new InvalidTranscriptionInputError("A transcription audio source is required");
5406
+ }
5407
+ let normalizedAudio;
5408
+ if (audio.type === "bytes") {
5409
+ if (!(audio.data instanceof Uint8Array) || audio.data.byteLength === 0 || !nonEmptyString(audio.mediaType)) {
5410
+ throw new InvalidTranscriptionInputError(
5411
+ "Inline transcription bytes require nonempty data and mediaType"
5412
+ );
5413
+ }
5414
+ if (audio.data.byteLength > MAX_INLINE_TRANSCRIPTION_BYTES) {
5415
+ throw new InvalidTranscriptionInputError(
5416
+ "Inline transcription audio must not exceed 25 MiB; chunk it or use a supported provider-file source"
5417
+ );
5418
+ }
5419
+ normalizedAudio = {
5420
+ type: "bytes",
5421
+ data: new Uint8Array(audio.data),
5422
+ mediaType: audio.mediaType
5423
+ };
5424
+ } else if (audio.type === "base64") {
5425
+ if (!nonEmptyString(audio.data) || !nonEmptyString(audio.mediaType)) {
5426
+ throw new InvalidTranscriptionInputError(
5427
+ "Inline base64 transcription audio requires data and mediaType"
5428
+ );
5429
+ }
5430
+ if (audio.data.length > 4 * Math.ceil(MAX_INLINE_TRANSCRIPTION_BYTES / 3)) {
5431
+ throw new InvalidTranscriptionInputError(
5432
+ "Inline transcription audio must not exceed 25 MiB; chunk it or use a supported provider-file source"
5433
+ );
5434
+ }
5435
+ let decoded;
5436
+ try {
5437
+ decoded = Uint8Array.from(Buffer.from(audio.data, "base64"));
5438
+ } catch {
5439
+ throw new InvalidTranscriptionInputError(
5440
+ "Transcription base64 must be canonical and decode to audio"
5441
+ );
5442
+ }
5443
+ if (!decoded.byteLength || Buffer.from(decoded).toString("base64") !== audio.data) {
5444
+ throw new InvalidTranscriptionInputError(
5445
+ "Transcription base64 must be canonical and decode to nonempty audio"
5446
+ );
5447
+ }
5448
+ if (decoded.byteLength > MAX_INLINE_TRANSCRIPTION_BYTES) {
5449
+ throw new InvalidTranscriptionInputError(
5450
+ "Inline transcription audio must not exceed 25 MiB; chunk it or use a supported provider-file source"
5451
+ );
5452
+ }
5453
+ normalizedAudio = { type: "base64", data: audio.data, mediaType: audio.mediaType };
5454
+ } else if (audio.type === "provider-file") {
5455
+ if (!nonEmptyString(audio.provider) || !nonEmptyString(audio.reference)) {
5456
+ throw new InvalidTranscriptionInputError(
5457
+ "A provider-file transcription source requires provider and reference"
5458
+ );
5459
+ }
5460
+ if (audio.mediaType !== void 0 && !nonEmptyString(audio.mediaType)) {
5461
+ throw new InvalidTranscriptionInputError(
5462
+ "Transcription provider-file mediaType must be nonempty when supplied"
5463
+ );
5464
+ }
5465
+ normalizedAudio = {
5466
+ type: "provider-file",
5467
+ provider: audio.provider,
5468
+ reference: audio.reference,
5469
+ ...audio.mediaType !== void 0 ? { mediaType: audio.mediaType } : {}
5470
+ };
5471
+ } else {
5472
+ throw new InvalidTranscriptionInputError(
5473
+ "Transcription audio must be bytes, base64, or provider-file"
5474
+ );
5475
+ }
5476
+ if (request.language !== void 0 && !nonEmptyString(request.language)) {
5477
+ throw new InvalidTranscriptionInputError(
5478
+ "Transcription language must be nonempty when supplied"
5479
+ );
5480
+ }
5481
+ if (request.timestamps !== void 0 && request.timestamps !== "segment" && request.timestamps !== "word") {
5482
+ throw new InvalidTranscriptionInputError("Transcription timestamps must be segment or word");
5483
+ }
5484
+ if (request.diarization !== void 0 && typeof request.diarization !== "boolean") {
5485
+ throw new InvalidTranscriptionInputError("Transcription diarization must be boolean");
5486
+ }
5487
+ if (request.providerOptions !== void 0) {
5488
+ if (!request.providerOptions || Array.isArray(request.providerOptions) || typeof request.providerOptions !== "object") {
5489
+ throw new InvalidTranscriptionInputError("Transcription providerOptions must be an object");
5490
+ }
5491
+ for (const key of Object.keys(request.providerOptions)) {
5492
+ if (RESERVED_PROVIDER_OPTION_KEYS.has(key)) {
5493
+ throw new InvalidTranscriptionInputError(
5494
+ "Transcription providerOptions cannot override reserved operation fields"
5495
+ );
5496
+ }
5497
+ }
5498
+ }
5499
+ return {
5500
+ model: request.model,
5501
+ audio: normalizedAudio,
5502
+ ...request.language !== void 0 ? { language: request.language } : {},
5503
+ ...request.timestamps !== void 0 ? { timestamps: request.timestamps } : {},
5504
+ ...request.diarization !== void 0 ? { diarization: request.diarization } : {},
5505
+ ...request.providerOptions !== void 0 ? { providerOptions: cloneValue(request.providerOptions) } : {}
5506
+ };
5507
+ }
5508
+ function cloneTranscript(transcript) {
5509
+ if (!transcript || typeof transcript !== "object" || typeof transcript.text !== "string") {
5510
+ throw new InvalidTranscriptionInputError(
5511
+ "Transcription provider returned an invalid transcript"
5512
+ );
5513
+ }
5514
+ const cloned = cloneValue(transcript);
5515
+ const accounting = normalizeTranscriptionAccounting(cloned.usage, cloned.pricingStatus);
5516
+ return {
5517
+ text: cloned.text,
5518
+ ...cloned.detectedLanguages ? { detectedLanguages: cloned.detectedLanguages } : {},
5519
+ ...cloned.segments ? { segments: cloned.segments } : {},
5520
+ ...cloned.words ? { words: cloned.words } : {},
5521
+ ...accounting.usage ? { usage: accounting.usage } : {},
5522
+ ...accounting.pricingStatus ? { pricingStatus: accounting.pricingStatus } : {},
5523
+ ...cloned.providerMetadata ? { providerMetadata: cloned.providerMetadata } : {}
5524
+ };
5525
+ }
5526
+ function normalizeTranscriptionAccounting(usageValue, pricingValue) {
5527
+ const value = usageValue && typeof usageValue === "object" ? usageValue : void 0;
5528
+ const number = (key) => {
5529
+ const candidate = value?.[key];
5530
+ return typeof candidate === "number" && Number.isFinite(candidate) && candidate >= 0 ? candidate : void 0;
5531
+ };
5532
+ const audioSeconds = number("audioSeconds");
5533
+ const inputTokens = number("inputTokens");
5534
+ const outputTokens = number("outputTokens");
5535
+ const totalTokens = number("totalTokens");
5536
+ const reportedCost = number("cost");
5537
+ const usage = value && (audioSeconds !== void 0 || inputTokens !== void 0 || outputTokens !== void 0 || totalTokens !== void 0 || reportedCost !== void 0) ? {
5538
+ ...audioSeconds !== void 0 ? { audioSeconds } : {},
5539
+ ...inputTokens !== void 0 ? { inputTokens } : {},
5540
+ ...outputTokens !== void 0 ? { outputTokens } : {},
5541
+ ...totalTokens !== void 0 ? { totalTokens } : {},
5542
+ ...reportedCost !== void 0 ? { cost: reportedCost } : {}
5543
+ } : void 0;
5544
+ const hasWork = (audioSeconds ?? 0) > 0 || (inputTokens ?? 0) > 0 || (outputTokens ?? 0) > 0 || (totalTokens ?? 0) > 0;
5545
+ const requested = pricingValue === "priced" || pricingValue === "unpriced" || pricingValue === "zero" ? pricingValue : void 0;
5546
+ const cost = requested === "zero" && (reportedCost === void 0 || reportedCost === 0) ? 0 : reportedCost;
5547
+ const pricingStatus = requested === "zero" && (reportedCost === void 0 || reportedCost === 0) ? "zero" : requested === "priced" && cost !== void 0 ? "priced" : requested === "unpriced" && cost === void 0 ? "unpriced" : cost !== void 0 ? "priced" : hasWork || requested === "unpriced" || requested === "priced" ? "unpriced" : void 0;
5548
+ const tokens = inputTokens !== void 0 || outputTokens !== void 0 ? {
5549
+ ...inputTokens !== void 0 ? { input: inputTokens } : {},
5550
+ ...outputTokens !== void 0 ? { output: outputTokens } : {}
5551
+ } : void 0;
5552
+ return {
5553
+ ...usage ? { usage } : {},
5554
+ ...pricingStatus ? { pricingStatus } : {},
5555
+ ...cost !== void 0 ? { cost } : {},
5556
+ ...tokens ? { tokens } : {},
5557
+ hasWork
5558
+ };
5559
+ }
5560
+
5561
+ // src/providers/transcription-utils.ts
5562
+ var OPENAI_TRANSCRIPTION_MODEL = "gpt-transcribe";
5563
+ var GEMINI_TRANSCRIPTION_MODEL = "gemini-3.5-transcribe";
5564
+ var OPENROUTER_TRANSCRIPTION_MODEL = "openai/whisper-1";
5565
+ function requireExactTranscriptionModel(provider, model, expected) {
5566
+ if (model !== expected) {
5567
+ throw new UnsupportedTranscriptionInputError({
5568
+ provider,
5569
+ model,
5570
+ feature: "this transcription model"
5571
+ });
5572
+ }
5573
+ }
5574
+ function inlineAudioBytes(audio) {
5575
+ if (audio.type === "bytes") return new Uint8Array(audio.data);
5576
+ if (audio.type === "base64") return Uint8Array.from(Buffer.from(audio.data, "base64"));
5577
+ throw new UnsupportedTranscriptionInputError({
5578
+ provider: audio.provider,
5579
+ model: "",
5580
+ feature: "provider-file transcription input"
5581
+ });
5582
+ }
5583
+ function requireNoProviderFile(audio, provider, model) {
5584
+ if (audio.type === "provider-file") {
5585
+ throw new UnsupportedTranscriptionInputError({
5586
+ provider,
5587
+ model,
5588
+ feature: "provider-file transcription input"
5589
+ });
5590
+ }
5591
+ }
5592
+ function assertOnlyProviderOptions(request, provider, model, allowed) {
5593
+ const options = request.providerOptions ?? {};
5594
+ for (const key of Object.keys(options)) {
5595
+ if (!allowed.includes(key)) {
5596
+ throw new InvalidTranscriptionInputError(
5597
+ `Unsupported ${provider} transcription provider option for model '${model}'`
5598
+ );
5599
+ }
5600
+ }
5601
+ return options;
5602
+ }
5603
+ function transcriptUsage(value) {
5604
+ const source = value && typeof value === "object" ? value : {};
5605
+ const number = (...keys) => {
5606
+ for (const key of keys) {
5607
+ const candidate = source[key];
5608
+ if (typeof candidate === "number" && Number.isFinite(candidate) && candidate >= 0)
5609
+ return candidate;
5610
+ }
5611
+ return void 0;
5612
+ };
5613
+ const inputTokens = number("total_input_tokens", "input_tokens", "prompt_tokens", "inputTokens");
5614
+ const outputTokens = number(
5615
+ "total_output_tokens",
5616
+ "output_tokens",
5617
+ "completion_tokens",
5618
+ "outputTokens"
5619
+ );
5620
+ const totalTokens = number("total_tokens", "totalTokens");
5621
+ const audioSeconds = number("seconds", "audio_seconds", "audioSeconds", "duration_seconds");
5622
+ const cost = number("cost");
5623
+ const usage = inputTokens !== void 0 || outputTokens !== void 0 || totalTokens !== void 0 || audioSeconds !== void 0 || cost !== void 0 ? {
5624
+ ...audioSeconds !== void 0 ? { audioSeconds } : {},
5625
+ ...inputTokens !== void 0 ? { inputTokens } : {},
5626
+ ...outputTokens !== void 0 ? { outputTokens } : {},
5627
+ ...totalTokens !== void 0 ? { totalTokens } : {},
5628
+ ...cost !== void 0 ? { cost } : {}
5629
+ } : void 0;
5630
+ const hasWork = (audioSeconds ?? 0) > 0 || (inputTokens ?? 0) > 0 || (outputTokens ?? 0) > 0 || (totalTokens ?? 0) > 0;
5631
+ return {
5632
+ ...usage ? { usage } : {},
5633
+ ...cost !== void 0 ? { pricingStatus: "priced" } : hasWork ? { pricingStatus: "unpriced" } : {}
5634
+ };
5635
+ }
5636
+ function safeProviderMessage(provider, status, body) {
5637
+ try {
5638
+ const parsed = JSON.parse(body);
5639
+ const message = parsed.error?.message ?? parsed.message;
5640
+ if (typeof message === "string")
5641
+ return `${provider} transcription API error (${status}): ${message}`;
5642
+ } catch {
5643
+ }
5644
+ return `${provider} transcription API error (${status})`;
5645
+ }
5646
+ function attachTranscriptionFailureMetadata(error, metadata) {
5647
+ for (const [key, value] of Object.entries(metadata)) {
5648
+ if (value !== void 0) Object.defineProperty(error, key, { enumerable: true, value });
5649
+ }
5650
+ return error;
5651
+ }
5652
+
5653
+ // src/providers/openai-transcription.ts
5654
+ var OPENAI_FILE_EXTENSIONS = {
5655
+ "audio/flac": "flac",
5656
+ "audio/m4a": "m4a",
5657
+ "audio/mp3": "mp3",
5658
+ "audio/mp4": "mp4",
5659
+ "audio/mpeg": "mp3",
5660
+ "audio/mpga": "mpga",
5661
+ "audio/ogg": "ogg",
5662
+ "audio/wav": "wav",
5663
+ "audio/webm": "webm"
5664
+ };
5665
+ var OpenAITranscriptionProvider = class {
5666
+ name = "openai-transcription";
5667
+ baseUrl;
5668
+ apiKeySource;
5669
+ governor;
5670
+ constructor(options = {}) {
5671
+ this.apiKeySource = options.apiKey ?? process.env.OPENAI_API_KEY ?? "";
5672
+ this.baseUrl = (options.baseUrl ?? process.env.OPENAI_BASE_URL ?? "https://api.openai.com/v1").replace(/\/$/, "");
5673
+ assertSafeProviderBaseUrl(
5674
+ this.baseUrl,
5675
+ "OpenAI transcription provider",
5676
+ options.dangerouslyAllowInsecureHttp
5677
+ );
5678
+ this.governor = options.rateLimit ? new RateLimiter(options.rateLimit) : void 0;
5679
+ }
5680
+ capabilities(model) {
5681
+ return model === OPENAI_TRANSCRIPTION_MODEL ? { sources: ["bytes", "base64"] } : void 0;
5682
+ }
5683
+ async transcribe(request) {
5684
+ requireExactTranscriptionModel(this.name, request.model, OPENAI_TRANSCRIPTION_MODEL);
5685
+ requireNoProviderFile(request.audio, this.name, request.model);
5686
+ if (request.audio.type === "provider-file")
5687
+ throw new UnsupportedTranscriptionInputError({
5688
+ provider: this.name,
5689
+ model: request.model,
5690
+ feature: "provider-file transcription input"
5691
+ });
5692
+ const options = assertOnlyProviderOptions(request, this.name, request.model, [
5693
+ "temperature",
5694
+ "prompt"
5695
+ ]);
5696
+ if (options.temperature !== void 0 && (typeof options.temperature !== "number" || !Number.isFinite(options.temperature) || options.temperature < 0 || options.temperature > 1))
5697
+ throw new InvalidTranscriptionInputError(
5698
+ "OpenAI transcription temperature must be a number between 0 and 1"
5699
+ );
5700
+ if (options.prompt !== void 0 && typeof options.prompt !== "string")
5701
+ throw new InvalidTranscriptionInputError("OpenAI transcription prompt must be a string");
5702
+ const mediaType2 = request.audio.mediaType;
5703
+ const extension = OPENAI_FILE_EXTENSIONS[mediaType2];
5704
+ if (!extension)
5705
+ throw new UnsupportedTranscriptionInputError({
5706
+ provider: this.name,
5707
+ model: request.model,
5708
+ feature: "this audio media type"
5709
+ });
5710
+ const bytes = inlineAudioBytes(request.audio);
5711
+ if (bytes.byteLength > MAX_INLINE_TRANSCRIPTION_BYTES)
5712
+ throw new InvalidTranscriptionInputError("OpenAI transcription audio must not exceed 25 MiB");
5713
+ const form = new FormData();
5714
+ form.set("file", new Blob([Buffer.from(bytes)], { type: mediaType2 }), `audio.${extension}`);
5715
+ form.set("model", request.model);
5716
+ form.set("response_format", "json");
5717
+ if (request.language) form.set("language", request.language);
5718
+ if (options.temperature !== void 0) form.set("temperature", String(options.temperature));
5719
+ if (options.prompt !== void 0) form.set("prompt", options.prompt);
5720
+ const key = await this.resolveKey();
5721
+ const res = await fetchWithRetry(
5722
+ `${this.baseUrl}/audio/transcriptions`,
5723
+ {
5724
+ method: "POST",
5725
+ headers: { Authorization: `Bearer ${key}` },
5726
+ body: form,
5727
+ signal: request.signal
5728
+ },
5729
+ { governor: this.governor, provider: this.name }
5730
+ );
5731
+ if (!res.ok) {
5732
+ const body = await res.text();
5733
+ throw buildProviderError({
5734
+ provider: this.name,
5735
+ status: res.status,
5736
+ headers: res.headers,
5737
+ message: safeProviderMessage("OpenAI", res.status, body),
5738
+ body
5739
+ });
5740
+ }
5741
+ const json = await res.json();
5742
+ if (typeof json.text !== "string")
5743
+ throw new InvalidTranscriptionInputError(
5744
+ "OpenAI transcription response did not contain text"
5745
+ );
5746
+ return {
5747
+ transcript: { text: json.text, ...transcriptUsage(json.usage) },
5748
+ cleanupStatus: "not_required"
5749
+ };
5750
+ }
5751
+ async resolveKey() {
5752
+ const key = await resolveApiKey(this.apiKeySource);
5753
+ if (!key)
5754
+ throw new Error("OpenAI API key is required. Set OPENAI_API_KEY or pass apiKey in options.");
5755
+ return key;
5756
+ }
5757
+ };
5758
+
5759
+ // src/providers/gemini-transcription.ts
5760
+ var CLEANUP_TIMEOUT_MS = 3e3;
5761
+ var READY_TIMEOUT_MS = 3e4;
5762
+ var READY_POLL_MS = 250;
5763
+ var GEMINI_AUDIO_MEDIA_TYPES = /* @__PURE__ */ new Set([
5764
+ "audio/wav",
5765
+ "audio/mp3",
5766
+ "audio/aiff",
5767
+ "audio/aac",
5768
+ "audio/ogg",
5769
+ "audio/flac",
5770
+ "audio/mpeg",
5771
+ "audio/m4a",
5772
+ "audio/l16",
5773
+ "audio/opus",
5774
+ "audio/alaw",
5775
+ "audio/mulaw",
5776
+ "audio/webm"
5777
+ ]);
5778
+ function abortError2() {
5779
+ return new DOMException("The transcription operation was aborted", "AbortError");
5780
+ }
5781
+ function audioMime(audio) {
5782
+ if (!audio.mediaType)
5783
+ throw new UnsupportedTranscriptionInputError({
5784
+ provider: "gemini-transcription",
5785
+ model: GEMINI_TRANSCRIPTION_MODEL,
5786
+ feature: "provider-file audio mediaType"
5787
+ });
5788
+ return audio.mediaType;
5789
+ }
5790
+ function assertGeminiAudioMime(mediaType2) {
5791
+ if (!mediaType2 || !GEMINI_AUDIO_MEDIA_TYPES.has(mediaType2)) {
5792
+ throw new UnsupportedTranscriptionInputError({
5793
+ provider: "gemini-transcription",
5794
+ model: GEMINI_TRANSCRIPTION_MODEL,
5795
+ feature: "this audio media type"
5796
+ });
5797
+ }
5798
+ return mediaType2;
5799
+ }
5800
+ function secondsOffset(value) {
5801
+ if (typeof value !== "string") return void 0;
5802
+ const match = /^(\d+(?:\.\d+)?)s$/.exec(value);
5803
+ if (!match) return void 0;
5804
+ const seconds = Number(match[1]);
5805
+ return Number.isFinite(seconds) && seconds >= 0 ? seconds : void 0;
5806
+ }
5807
+ function sleep3(ms, signal) {
5808
+ return new Promise((resolve2, reject) => {
5809
+ if (signal?.aborted) {
5810
+ reject(abortError2());
5811
+ return;
5812
+ }
5813
+ const timer = setTimeout(done, ms);
5814
+ const onAbort = () => {
5815
+ clearTimeout(timer);
5816
+ done(abortError2());
5817
+ };
5818
+ function done(error) {
5819
+ signal?.removeEventListener("abort", onAbort);
5820
+ if (error) reject(error);
5821
+ else resolve2();
5822
+ }
5823
+ signal?.addEventListener("abort", onAbort, { once: true });
5824
+ });
5825
+ }
5826
+ var GeminiTranscriptionProvider = class {
5827
+ name = "gemini-transcription";
5828
+ baseUrl;
5829
+ apiKeySource;
5830
+ governor;
5831
+ constructor(options = {}) {
5832
+ this.apiKeySource = options.apiKey ?? process.env.GOOGLE_API_KEY ?? process.env.GEMINI_API_KEY ?? "";
5833
+ this.baseUrl = (options.baseUrl ?? "https://generativelanguage.googleapis.com/v1beta").replace(
5834
+ /\/$/,
5835
+ ""
5836
+ );
5837
+ assertSafeProviderBaseUrl(
5838
+ this.baseUrl,
5839
+ "Gemini transcription provider",
5840
+ options.dangerouslyAllowInsecureHttp
5841
+ );
5842
+ this.governor = options.rateLimit ? new RateLimiter(options.rateLimit) : void 0;
5843
+ }
5844
+ capabilities(model) {
5845
+ return model === GEMINI_TRANSCRIPTION_MODEL ? { sources: ["bytes", "base64", "provider-file"], timestamps: ["word"], diarization: true } : void 0;
5846
+ }
5847
+ async transcribe(request) {
5848
+ requireExactTranscriptionModel(this.name, request.model, GEMINI_TRANSCRIPTION_MODEL);
5849
+ const options = assertOnlyProviderOptions(request, this.name, request.model, [
5850
+ "mode",
5851
+ "customVocabulary"
5852
+ ]);
5853
+ const mode = options.mode ?? "verbatim";
5854
+ if (mode !== "smart" && mode !== "verbatim")
5855
+ throw new InvalidTranscriptionInputError(
5856
+ "Gemini transcription mode must be smart or verbatim"
5857
+ );
5858
+ if (options.customVocabulary !== void 0 && (!Array.isArray(options.customVocabulary) || options.customVocabulary.length > 1e3 || options.customVocabulary.some(
5859
+ (value) => typeof value !== "string" || !value.trim() || value.length > 256
5860
+ )))
5861
+ throw new InvalidTranscriptionInputError(
5862
+ "Gemini transcription customVocabulary must contain at most 1000 nonempty strings up to 256 characters"
5863
+ );
5864
+ if (mode === "smart" && (request.timestamps || request.diarization))
5865
+ throw new InvalidTranscriptionInputError(
5866
+ "Gemini smart transcription does not support timestamps or diarization"
5867
+ );
5868
+ if (request.timestamps && options.customVocabulary !== void 0)
5869
+ throw new InvalidTranscriptionInputError(
5870
+ "Gemini transcription customVocabulary cannot be combined with timestamps"
5871
+ );
5872
+ if (request.timestamps && request.timestamps !== "word")
5873
+ throw new UnsupportedTranscriptionInputError({
5874
+ provider: this.name,
5875
+ model: request.model,
5876
+ feature: "segment timestamps"
5877
+ });
5878
+ if (request.diarization && request.timestamps !== "word")
5879
+ throw new InvalidTranscriptionInputError("Gemini diarization requires word timestamps");
5880
+ if (request.audio.type === "provider-file") {
5881
+ if (request.audio.provider !== this.name || typeof request.audio.reference !== "string" || !request.audio.reference.trim()) {
5882
+ throw new UnsupportedTranscriptionInputError({
5883
+ provider: this.name,
5884
+ model: request.model,
5885
+ feature: "a matching nonempty provider-file reference"
5886
+ });
5887
+ }
5888
+ assertGeminiAudioMime(request.audio.mediaType);
5889
+ } else {
5890
+ assertGeminiAudioMime(request.audio.mediaType);
5891
+ }
5892
+ const key = await this.resolveKey();
5893
+ if (request.audio.type === "provider-file")
5894
+ return this.transcribeReference(request, key, mode, options);
5895
+ return this.transcribeUpload(request, key, mode, options);
5896
+ }
5897
+ async transcribeReference(request, key, mode, options) {
5898
+ const audio = request.audio;
5899
+ if (audio.type !== "provider-file" || audio.provider !== this.name)
5900
+ throw new UnsupportedTranscriptionInputError({
5901
+ provider: this.name,
5902
+ model: request.model,
5903
+ feature: "a provider-file owned by another provider"
5904
+ });
5905
+ const mimeType = audioMime(audio);
5906
+ const transcript = await this.infer(request, key, audio.reference, mimeType, mode, options);
5907
+ return { transcript, cleanupStatus: "not_required" };
5908
+ }
5909
+ async transcribeUpload(request, key, mode, options) {
5910
+ let created = false;
5911
+ let fileName;
5912
+ let primaryError;
5913
+ let transcript;
5914
+ let cleanupStatus = "not_required";
5915
+ try {
5916
+ const uploaded = await this.upload(request, key);
5917
+ created = true;
5918
+ fileName = uploaded.name;
5919
+ const ready = await this.waitForReady(
5920
+ this.fileFromJson(uploaded.file, uploaded.mimeType),
5921
+ key,
5922
+ request.signal
5923
+ );
5924
+ transcript = await this.infer(request, key, ready.uri, ready.mimeType, mode, options);
5925
+ } catch (error) {
5926
+ primaryError = error;
5927
+ } finally {
5928
+ if (created && fileName) cleanupStatus = await this.cleanup(fileName, key);
5929
+ }
5930
+ if (primaryError) {
5931
+ throw attachTranscriptionFailureMetadata(
5932
+ primaryError instanceof Error ? primaryError : new Error("Gemini transcription failed"),
5933
+ { cleanupStatus }
5934
+ );
5935
+ }
5936
+ if (!transcript)
5937
+ throw new InvalidTranscriptionInputError(
5938
+ "Gemini transcription response did not contain text"
5939
+ );
5940
+ return { transcript, cleanupStatus };
5941
+ }
5942
+ /** A finalization response with a name creates a caller-owned cleanup obligation
5943
+ * even when its URI/state is malformed. `transcribeUpload` stores the name before
5944
+ * strictly parsing the rest of the file representation. */
5945
+ async upload(request, key) {
5946
+ if (request.audio.type === "provider-file")
5947
+ throw new UnsupportedTranscriptionInputError({
5948
+ provider: this.name,
5949
+ model: request.model,
5950
+ feature: "provider-file upload"
5951
+ });
5952
+ const mimeType = assertGeminiAudioMime(request.audio.mediaType);
5953
+ const bytes = inlineAudioBytes(request.audio);
5954
+ const start = await fetchWithRetry(
5955
+ `${this.baseUrl.replace(/\/v1beta$/, "")}/upload/v1beta/files`,
5956
+ {
5957
+ method: "POST",
5958
+ headers: {
5959
+ "x-goog-api-key": key,
5960
+ "X-Goog-Upload-Protocol": "resumable",
5961
+ "X-Goog-Upload-Command": "start",
5962
+ "X-Goog-Upload-Header-Content-Length": String(bytes.byteLength),
5963
+ "X-Goog-Upload-Header-Content-Type": mimeType,
5964
+ "Content-Type": "application/json"
5965
+ },
5966
+ body: JSON.stringify({ file: { display_name: "axl-transcription" } }),
5967
+ signal: request.signal
5968
+ },
5969
+ { governor: this.governor, provider: this.name, maxRetries: 0 }
5970
+ );
5971
+ if (!start.ok) {
5972
+ const body = await start.text();
5973
+ throw buildProviderError({
5974
+ provider: this.name,
5975
+ status: start.status,
5976
+ headers: start.headers,
5977
+ message: safeProviderMessage("Gemini", start.status, body),
5978
+ body
5979
+ });
5980
+ }
5981
+ const uploadUrl = start.headers.get("x-goog-upload-url");
5982
+ if (!uploadUrl)
5983
+ throw new InvalidTranscriptionInputError(
5984
+ "Gemini transcription upload did not return an upload URL"
5985
+ );
5986
+ let parsedUploadUrl;
5987
+ try {
5988
+ parsedUploadUrl = new URL(uploadUrl);
5989
+ } catch {
5990
+ throw new InvalidTranscriptionInputError(
5991
+ "Gemini transcription upload returned an invalid upload URL"
5992
+ );
5993
+ }
5994
+ const baseOrigin = new URL(this.baseUrl).origin;
5995
+ if (parsedUploadUrl.username || parsedUploadUrl.password || parsedUploadUrl.origin !== baseOrigin) {
5996
+ throw new InvalidTranscriptionInputError(
5997
+ "Gemini transcription upload returned an untrusted upload URL"
5998
+ );
5999
+ }
6000
+ assertSafeProviderBaseUrl(
6001
+ parsedUploadUrl.toString(),
6002
+ "Gemini transcription upload endpoint",
6003
+ new URL(this.baseUrl).protocol === "http:"
6004
+ );
6005
+ const finalize = await fetchWithRetry(
6006
+ parsedUploadUrl.toString(),
6007
+ {
6008
+ method: "POST",
6009
+ headers: {
6010
+ "X-Goog-Upload-Command": "upload, finalize",
6011
+ "X-Goog-Upload-Offset": "0",
6012
+ "Content-Length": String(bytes.byteLength)
6013
+ },
6014
+ body: bytes,
6015
+ signal: request.signal
6016
+ },
6017
+ { governor: this.governor, provider: this.name, maxRetries: 0 }
6018
+ );
6019
+ if (!finalize.ok) {
6020
+ const body = await finalize.text();
6021
+ throw buildProviderError({
6022
+ provider: this.name,
6023
+ status: finalize.status,
6024
+ headers: finalize.headers,
6025
+ message: safeProviderMessage("Gemini", finalize.status, body),
6026
+ body
6027
+ });
6028
+ }
6029
+ const payload = await finalize.json();
6030
+ const file = payload.file;
6031
+ if (!file || typeof file.name !== "string") {
6032
+ throw new InvalidTranscriptionInputError(
6033
+ "Gemini transcription upload did not return a file name"
6034
+ );
6035
+ }
6036
+ return { name: file.name, file, mimeType };
6037
+ }
6038
+ async waitForReady(file, key, signal) {
6039
+ let current = file;
6040
+ const deadline = Date.now() + READY_TIMEOUT_MS;
6041
+ if (!current.state) {
6042
+ throw new InvalidTranscriptionInputError(
6043
+ "Gemini transcription file readiness state is missing"
6044
+ );
6045
+ }
6046
+ while (current.state && current.state !== "ACTIVE") {
6047
+ if (current.state === "FAILED")
6048
+ throw new InvalidTranscriptionInputError("Gemini transcription file processing failed");
6049
+ if (Date.now() >= deadline)
6050
+ throw new InvalidTranscriptionInputError("Gemini transcription file readiness timed out");
6051
+ await sleep3(Math.min(READY_POLL_MS, deadline - Date.now()), signal);
6052
+ const res = await fetchWithRetry(
6053
+ `${this.baseUrl}/${current.name}`,
6054
+ { method: "GET", headers: { "x-goog-api-key": key }, signal },
6055
+ { governor: this.governor, provider: this.name, maxRetries: 0 }
6056
+ );
6057
+ if (!res.ok) {
6058
+ const body = await res.text();
6059
+ throw buildProviderError({
6060
+ provider: this.name,
6061
+ status: res.status,
6062
+ headers: res.headers,
6063
+ message: safeProviderMessage("Gemini", res.status, body),
6064
+ body
6065
+ });
6066
+ }
6067
+ current = this.fileFromJson(await res.json(), current.mimeType);
6068
+ }
6069
+ return current;
6070
+ }
6071
+ async infer(request, key, uri, mimeType, mode, options) {
6072
+ const transcriptionConfig = {
6073
+ mode: mode === "smart" ? "smart" : {
6074
+ type: "verbatim",
6075
+ ...request.timestamps ? { timestamp_granularities: ["word"] } : {},
6076
+ ...request.diarization ? { diarization_mode: "speaker" } : {}
6077
+ },
6078
+ ...request.language ? { language_codes: [request.language] } : {},
6079
+ ...options.customVocabulary ? { custom_vocabulary: options.customVocabulary } : {}
6080
+ };
6081
+ const body = {
6082
+ model: request.model,
6083
+ input: [{ type: "audio", uri, mime_type: mimeType }],
6084
+ store: false,
6085
+ generation_config: { transcription_config: transcriptionConfig }
6086
+ };
6087
+ const res = await fetchWithRetry(
6088
+ `${this.baseUrl}/interactions`,
6089
+ {
6090
+ method: "POST",
6091
+ headers: { "x-goog-api-key": key, "Content-Type": "application/json" },
6092
+ body: JSON.stringify(body),
6093
+ signal: request.signal
6094
+ },
6095
+ { governor: this.governor, provider: this.name }
6096
+ );
6097
+ if (!res.ok) {
6098
+ const errorBody = await res.text();
6099
+ throw buildProviderError({
6100
+ provider: this.name,
6101
+ status: res.status,
6102
+ headers: res.headers,
6103
+ message: safeProviderMessage("Gemini", res.status, errorBody),
6104
+ body: errorBody
6105
+ });
6106
+ }
6107
+ const json = await res.json();
6108
+ if (json.status !== "completed") {
6109
+ throw new InvalidTranscriptionInputError("Gemini transcription interaction did not complete");
6110
+ }
6111
+ const words = [];
6112
+ let text = "";
6113
+ let hasModelOutputText = false;
6114
+ if (Array.isArray(json.steps)) {
6115
+ for (const step of json.steps) {
6116
+ if (!step || typeof step !== "object") continue;
6117
+ const modelOutput = step;
6118
+ if (modelOutput.type !== "model_output" || !Array.isArray(modelOutput.content)) continue;
6119
+ for (const block of modelOutput.content) {
6120
+ if (!block || typeof block !== "object") continue;
6121
+ const content = block;
6122
+ if (content.type !== "text" || typeof content.text !== "string") continue;
6123
+ hasModelOutputText = true;
6124
+ text += content.text;
6125
+ if (!Array.isArray(content.annotations)) continue;
6126
+ for (const raw of content.annotations) {
6127
+ if (!raw || typeof raw !== "object") continue;
6128
+ const annotation = raw;
6129
+ if (annotation.type !== "word_info") continue;
6130
+ if (typeof annotation.text !== "string") continue;
6131
+ const start = secondsOffset(annotation.start_offset);
6132
+ const end = secondsOffset(annotation.end_offset);
6133
+ if (start === void 0 || end === void 0 || end < start) continue;
6134
+ words.push({
6135
+ text: annotation.text,
6136
+ start,
6137
+ end,
6138
+ ...typeof annotation.speaker === "string" ? { speaker: annotation.speaker } : {}
6139
+ });
6140
+ }
6141
+ }
6142
+ }
6143
+ }
6144
+ if (!hasModelOutputText)
6145
+ throw new InvalidTranscriptionInputError(
6146
+ "Gemini transcription response did not contain model output text"
6147
+ );
6148
+ return { text, ...words.length ? { words } : {}, ...transcriptUsage(json.usage) };
6149
+ }
6150
+ fileFromJson(value, fallbackMimeType) {
6151
+ const file = "file" in value && value.file ? value.file : value;
6152
+ if (typeof file.name !== "string" || typeof file.uri !== "string")
6153
+ throw new InvalidTranscriptionInputError(
6154
+ "Gemini transcription upload did not return a file reference"
6155
+ );
6156
+ return {
6157
+ name: file.name,
6158
+ uri: file.uri,
6159
+ mimeType: typeof file.mimeType === "string" ? file.mimeType : typeof file.mime_type === "string" ? file.mime_type : fallbackMimeType,
6160
+ ...typeof file.state === "string" ? { state: file.state } : {}
6161
+ };
6162
+ }
6163
+ async cleanup(fileName, key) {
6164
+ const controller = new AbortController();
6165
+ const timer = setTimeout(() => controller.abort(), CLEANUP_TIMEOUT_MS);
6166
+ try {
6167
+ const res = await fetchWithRetry(
6168
+ `${this.baseUrl}/${fileName}`,
6169
+ { method: "DELETE", headers: { "x-goog-api-key": key }, signal: controller.signal },
6170
+ { governor: this.governor, provider: this.name, maxRetries: 0 }
6171
+ );
6172
+ return res.ok ? "deleted" : "failed";
6173
+ } catch {
6174
+ return controller.signal.aborted ? "timed_out" : "failed";
6175
+ } finally {
6176
+ clearTimeout(timer);
6177
+ }
6178
+ }
6179
+ async resolveKey() {
6180
+ const key = await resolveApiKey(this.apiKeySource);
6181
+ if (!key)
6182
+ throw new Error("Google API key is required. Set GOOGLE_API_KEY or pass apiKey in options.");
6183
+ return key;
6184
+ }
6185
+ };
6186
+
6187
+ // src/providers/openrouter-transcription.ts
6188
+ var FORMAT_BY_MIME = {
6189
+ "audio/aac": "aac",
6190
+ "audio/flac": "flac",
6191
+ "audio/m4a": "m4a",
6192
+ "audio/mp3": "mp3",
6193
+ "audio/mp4": "mp4",
6194
+ "audio/mpeg": "mp3",
6195
+ "audio/ogg": "ogg",
6196
+ "audio/wav": "wav",
6197
+ "audio/webm": "webm"
6198
+ };
6199
+ var OpenRouterTranscriptionProvider = class {
6200
+ name = "openrouter-transcription";
6201
+ baseUrl;
6202
+ apiKeySource;
6203
+ governor;
6204
+ constructor(options = {}) {
6205
+ this.apiKeySource = options.apiKey ?? process.env.OPENROUTER_API_KEY ?? "";
6206
+ this.baseUrl = (options.baseUrl ?? process.env.OPENROUTER_BASE_URL ?? "https://openrouter.ai/api/v1").replace(/\/$/, "");
6207
+ assertSafeProviderBaseUrl(
6208
+ this.baseUrl,
6209
+ "OpenRouter transcription provider",
6210
+ options.dangerouslyAllowInsecureHttp
6211
+ );
6212
+ this.governor = options.rateLimit ? new RateLimiter(options.rateLimit) : void 0;
6213
+ }
6214
+ capabilities(model) {
6215
+ return model === OPENROUTER_TRANSCRIPTION_MODEL ? { sources: ["bytes", "base64"] } : void 0;
6216
+ }
6217
+ async transcribe(request) {
6218
+ requireExactTranscriptionModel(this.name, request.model, OPENROUTER_TRANSCRIPTION_MODEL);
6219
+ requireNoProviderFile(request.audio, this.name, request.model);
6220
+ if (request.audio.type === "provider-file")
6221
+ throw new UnsupportedTranscriptionInputError({
6222
+ provider: this.name,
6223
+ model: request.model,
6224
+ feature: "provider-file transcription input"
6225
+ });
6226
+ const options = assertOnlyProviderOptions(request, this.name, request.model, [
6227
+ "temperature",
6228
+ "provider"
6229
+ ]);
6230
+ if (options.temperature !== void 0 && (typeof options.temperature !== "number" || !Number.isFinite(options.temperature) || options.temperature < 0 || options.temperature > 1))
6231
+ throw new InvalidTranscriptionInputError(
6232
+ "OpenRouter transcription temperature must be a number between 0 and 1"
6233
+ );
6234
+ if (options.provider !== void 0 && (!options.provider || Array.isArray(options.provider) || typeof options.provider !== "object"))
6235
+ throw new InvalidTranscriptionInputError(
6236
+ "OpenRouter transcription provider routing must be an object"
6237
+ );
6238
+ const format = FORMAT_BY_MIME[request.audio.mediaType];
6239
+ if (!format)
6240
+ throw new UnsupportedTranscriptionInputError({
6241
+ provider: this.name,
6242
+ model: request.model,
6243
+ feature: "this audio media type"
6244
+ });
6245
+ const bytes = inlineAudioBytes(request.audio);
6246
+ const body = {
6247
+ model: request.model,
6248
+ input_audio: { data: Buffer.from(bytes).toString("base64"), format },
6249
+ ...request.language ? { language: request.language } : {},
6250
+ ...options.temperature !== void 0 ? { temperature: options.temperature } : {},
6251
+ ...options.provider !== void 0 ? { provider: options.provider } : {}
6252
+ };
6253
+ const key = await this.resolveKey();
6254
+ const res = await fetchWithRetry(
6255
+ `${this.baseUrl}/audio/transcriptions`,
6256
+ {
6257
+ method: "POST",
6258
+ headers: { Authorization: `Bearer ${key}`, "Content-Type": "application/json" },
6259
+ body: JSON.stringify(body),
6260
+ signal: request.signal
6261
+ },
6262
+ { governor: this.governor, provider: this.name }
6263
+ );
6264
+ if (!res.ok) {
6265
+ const errorBody = await res.text();
6266
+ throw buildProviderError({
6267
+ provider: this.name,
6268
+ status: res.status,
6269
+ headers: res.headers,
6270
+ message: safeProviderMessage("OpenRouter", res.status, errorBody),
6271
+ body: errorBody
6272
+ });
6273
+ }
6274
+ const json = await res.json();
6275
+ if (typeof json.text !== "string")
6276
+ throw new InvalidTranscriptionInputError(
6277
+ "OpenRouter transcription response did not contain text"
6278
+ );
6279
+ return {
6280
+ transcript: { text: json.text, ...transcriptUsage(json.usage) },
6281
+ cleanupStatus: "not_required"
6282
+ };
6283
+ }
6284
+ async resolveKey() {
6285
+ const key = await resolveApiKey(this.apiKeySource);
6286
+ if (!key)
6287
+ throw new Error(
6288
+ "OpenRouter API key is required. Set OPENROUTER_API_KEY or pass apiKey in options."
6289
+ );
6290
+ return key;
6291
+ }
6292
+ };
6293
+
6294
+ // src/providers/transcription-registry.ts
6295
+ var builtinFactories2 = {
6296
+ "openai-transcription": (config) => {
6297
+ const opts = config.providers?.openai ?? {};
6298
+ return new OpenAITranscriptionProvider({
6299
+ apiKey: opts.apiKey,
6300
+ baseUrl: opts.baseUrl,
6301
+ dangerouslyAllowInsecureHttp: opts.dangerouslyAllowInsecureHttp,
6302
+ rateLimit: opts.rateLimit
6303
+ });
6304
+ },
6305
+ "gemini-transcription": (config) => {
6306
+ const opts = config.providers?.google ?? {};
6307
+ return new GeminiTranscriptionProvider({
6308
+ apiKey: opts.apiKey,
6309
+ baseUrl: opts.baseUrl,
6310
+ dangerouslyAllowInsecureHttp: opts.dangerouslyAllowInsecureHttp,
6311
+ rateLimit: opts.rateLimit
6312
+ });
6313
+ },
6314
+ "openrouter-transcription": (config) => {
6315
+ const opts = config.providers?.openrouter ?? {};
6316
+ return new OpenRouterTranscriptionProvider({
6317
+ apiKey: opts.apiKey,
6318
+ baseUrl: opts.baseUrl,
6319
+ dangerouslyAllowInsecureHttp: opts.dangerouslyAllowInsecureHttp,
6320
+ rateLimit: opts.rateLimit
6321
+ });
6322
+ }
6323
+ };
6324
+ var TranscriptionProviderRegistry = class {
6325
+ instances = /* @__PURE__ */ new Map();
6326
+ factories = /* @__PURE__ */ new Map();
6327
+ constructor() {
6328
+ for (const [name, factory] of Object.entries(builtinFactories2))
6329
+ this.factories.set(name, factory);
6330
+ }
6331
+ register(name, factory) {
6332
+ this.factories.set(name, factory);
6333
+ this.instances.delete(name);
6334
+ }
6335
+ registerInstance(name, provider) {
6336
+ this.instances.set(name, provider);
6337
+ }
6338
+ has(name) {
6339
+ return this.instances.has(name) || this.factories.has(name);
6340
+ }
6341
+ get(name, config = {}) {
6342
+ const instance = this.instances.get(name);
6343
+ if (instance) return instance;
6344
+ const factory = this.factories.get(name);
6345
+ if (factory) {
6346
+ const created = factory(config);
6347
+ this.instances.set(name, created);
6348
+ return created;
6349
+ }
6350
+ throw new UnsupportedTranscriptionInputError({
6351
+ provider: name,
6352
+ model: "",
6353
+ feature: "transcription"
6354
+ });
6355
+ }
4262
6356
  resolve(uri, config = {}) {
4263
- if (!uri) {
4264
- const providerName2 = config.defaultProvider ?? "openai";
4265
- const model2 = config.defaultModel ?? "gpt-4o";
4266
- return { provider: this.get(providerName2, config), model: model2 };
4267
- }
4268
- const colonIndex = uri.indexOf(":");
4269
- if (colonIndex === -1) {
4270
- const providerName2 = config.defaultProvider ?? "openai";
4271
- return { provider: this.get(providerName2, config), model: uri };
4272
- }
4273
- const providerName = uri.slice(0, colonIndex);
4274
- const model = uri.slice(colonIndex + 1);
4275
- if (!providerName || !model) {
4276
- throw new Error(
4277
- `Invalid provider URI "${uri}". Expected format: "provider:model" (e.g. "openai:gpt-4o")`
4278
- );
6357
+ const colon = typeof uri === "string" ? uri.indexOf(":") : -1;
6358
+ if (colon <= 0 || colon === uri.length - 1 || uri.indexOf(":", colon + 1) !== -1) {
6359
+ throw new UnsupportedTranscriptionInputError({
6360
+ provider: "unknown",
6361
+ model: "",
6362
+ feature: "an explicit transcription provider:model URI"
6363
+ });
4279
6364
  }
4280
- return { provider: this.get(providerName, config), model };
4281
- }
4282
- /**
4283
- * Clear all cached provider instances. Useful for testing or reconfiguration.
4284
- */
4285
- clearCache() {
4286
- this.instances.clear();
4287
- }
4288
- /**
4289
- * Clear all registered factories (including built-ins).
4290
- * Useful for test runtimes where only explicitly registered instances should be used.
4291
- */
4292
- clearFactories() {
4293
- this.factories.clear();
6365
+ const providerName = uri.slice(0, colon);
6366
+ const model = uri.slice(colon + 1);
6367
+ return { provider: this.get(providerName, config), model, providerName };
4294
6368
  }
4295
6369
  };
4296
- var defaultRegistry = new ProviderRegistry();
4297
6370
 
4298
6371
  // src/context.ts
4299
6372
  import { AsyncLocalStorage } from "async_hooks";
4300
6373
  import { randomUUID } from "crypto";
4301
6374
  import { z, ZodError as ZodError2 } from "zod";
4302
6375
 
6376
+ // src/input.ts
6377
+ var MAX_INLINE_MODEL_INPUT_BYTES = 25 * 1024 * 1024;
6378
+ function invalid(message) {
6379
+ throw new InvalidModelInputError(message);
6380
+ }
6381
+ function nonEmptyString2(value, field) {
6382
+ if (typeof value !== "string" || value.length === 0)
6383
+ invalid(`${field} must be a non-empty string`);
6384
+ return value;
6385
+ }
6386
+ function mediaType(value, field) {
6387
+ return nonEmptyString2(value, field);
6388
+ }
6389
+ function validBase64(value) {
6390
+ return value.length > 0 && value.length % 4 === 0 && /^[A-Za-z0-9+/]*={0,2}$/.test(value);
6391
+ }
6392
+ function decodedBase64Bytes(value) {
6393
+ return Math.floor(value.length * 3 / 4) - (value.endsWith("==") ? 2 : value.endsWith("=") ? 1 : 0);
6394
+ }
6395
+ function normalizeModelInput(input) {
6396
+ if (typeof input === "string") return input;
6397
+ if (!Array.isArray(input) || input.length === 0)
6398
+ invalid("ModelInput parts must be a non-empty array");
6399
+ let inlineBytes = 0;
6400
+ const reserveInlineBytes = (bytes) => {
6401
+ if (bytes > MAX_INLINE_MODEL_INPUT_BYTES - inlineBytes) {
6402
+ invalid(
6403
+ "Inline image data must not exceed 25 MiB total; use a URL or provider-file source where supported"
6404
+ );
6405
+ }
6406
+ inlineBytes += bytes;
6407
+ };
6408
+ return input.map((part, index) => {
6409
+ if (!part || typeof part !== "object") invalid(`ModelInput part ${index} must be an object`);
6410
+ if (part.type === "text")
6411
+ return { type: "text", text: nonEmptyString2(part.text, `part ${index}.text`) };
6412
+ if (part.type !== "image") invalid(`part ${index}.type must be 'text' or 'image'`);
6413
+ const source = part.source;
6414
+ if (!source || typeof source !== "object") invalid(`part ${index}.source must be an object`);
6415
+ let clone;
6416
+ switch (source.type) {
6417
+ case "url": {
6418
+ const url = nonEmptyString2(source.url, `part ${index}.source.url`);
6419
+ let parsed;
6420
+ try {
6421
+ parsed = new URL(url);
6422
+ } catch {
6423
+ invalid(`part ${index}.source.url must be an http(s) URL`);
6424
+ }
6425
+ if (parsed.protocol !== "http:" && parsed.protocol !== "https:")
6426
+ invalid(`part ${index}.source.url must be an http(s) URL`);
6427
+ clone = {
6428
+ type: "url",
6429
+ url,
6430
+ ...source.mediaType === void 0 ? {} : { mediaType: mediaType(source.mediaType, `part ${index}.source.mediaType`) }
6431
+ };
6432
+ break;
6433
+ }
6434
+ case "bytes":
6435
+ if (!(source.data instanceof Uint8Array) || source.data.byteLength === 0)
6436
+ invalid(`part ${index}.source.data must be a non-empty Uint8Array`);
6437
+ reserveInlineBytes(source.data.byteLength);
6438
+ clone = {
6439
+ type: "bytes",
6440
+ data: source.data.slice(),
6441
+ mediaType: mediaType(source.mediaType, `part ${index}.source.mediaType`)
6442
+ };
6443
+ break;
6444
+ case "base64": {
6445
+ const data = nonEmptyString2(source.data, `part ${index}.source.data`);
6446
+ if (data.length > 4 * Math.ceil(MAX_INLINE_MODEL_INPUT_BYTES / 3)) {
6447
+ invalid(
6448
+ "Inline image data must not exceed 25 MiB total; use a URL or provider-file source where supported"
6449
+ );
6450
+ }
6451
+ if (!validBase64(data)) invalid(`part ${index}.source.data must be valid base64`);
6452
+ reserveInlineBytes(decodedBase64Bytes(data));
6453
+ clone = {
6454
+ type: "base64",
6455
+ data,
6456
+ mediaType: mediaType(source.mediaType, `part ${index}.source.mediaType`)
6457
+ };
6458
+ break;
6459
+ }
6460
+ case "provider-file":
6461
+ clone = {
6462
+ type: "provider-file",
6463
+ provider: nonEmptyString2(source.provider, `part ${index}.source.provider`),
6464
+ reference: nonEmptyString2(source.reference, `part ${index}.source.reference`),
6465
+ ...source.mediaType === void 0 ? {} : { mediaType: mediaType(source.mediaType, `part ${index}.source.mediaType`) }
6466
+ };
6467
+ break;
6468
+ default:
6469
+ invalid(`part ${index}.source.type is unsupported`);
6470
+ }
6471
+ return {
6472
+ type: "image",
6473
+ source: clone,
6474
+ ...part.label === void 0 ? {} : { label: nonEmptyString2(part.label, `part ${index}.label`) }
6475
+ };
6476
+ });
6477
+ }
6478
+ function inputText(input) {
6479
+ return typeof input === "string" ? input : input.filter((part) => part.type === "text").map((part) => part.text).join("\n");
6480
+ }
6481
+ function cloneModelInput(input) {
6482
+ return typeof input === "string" ? input : normalizeModelInput(input);
6483
+ }
6484
+ function describeModelInput(input) {
6485
+ if (typeof input === "string") return void 0;
6486
+ return {
6487
+ parts: input.map((part) => {
6488
+ if (part.type === "text") return { type: "text", characters: part.text.length };
6489
+ const { source } = part;
6490
+ return {
6491
+ type: "image",
6492
+ source: source.type,
6493
+ ...source.mediaType ? { mediaType: source.mediaType } : {},
6494
+ ...source.type === "bytes" ? { bytes: source.data.byteLength } : {},
6495
+ ...source.type === "base64" ? {
6496
+ bytes: decodedBase64Bytes(source.data)
6497
+ } : {},
6498
+ ...(source.type === "url" ? source.url : source.type === "provider-file" ? source.reference : void 0) ? {
6499
+ locator: source.type === "url" ? source.url : source.type === "provider-file" ? source.reference : void 0
6500
+ } : {},
6501
+ ...part.label ? { label: part.label } : {}
6502
+ };
6503
+ })
6504
+ };
6505
+ }
6506
+ function summarizeModelInput(input) {
6507
+ if (typeof input === "string") return input;
6508
+ return input.map(
6509
+ (part) => part.type === "text" ? part.text : `[image ${part.source.mediaType ?? "media"}]`
6510
+ ).join("\n");
6511
+ }
6512
+ function sanitizeModelInputForTrace(input) {
6513
+ return inputText(input);
6514
+ }
6515
+ function normalizePersistedSessionHistory(history) {
6516
+ return history.map((message, index) => {
6517
+ if (typeof message.content === "string") return { ...message };
6518
+ if (message.role !== "user") {
6519
+ throw new InvalidModelInputError(
6520
+ `History message ${index} has non-text content on a non-user role`
6521
+ );
6522
+ }
6523
+ const content = normalizeModelInput(message.content);
6524
+ if (typeof content !== "string" && content.some((part) => part.type === "image" && part.source.type === "bytes")) {
6525
+ throw new InvalidModelInputError(
6526
+ "Uint8Array image input cannot be persisted in session history"
6527
+ );
6528
+ }
6529
+ return { ...message, content };
6530
+ });
6531
+ }
6532
+
4303
6533
  // src/partial-json.ts
4304
6534
  var MAX_DEPTH = 256;
4305
6535
  function parsePartialJson(input) {
@@ -4871,7 +7101,8 @@ var COST_BEARING_LEAF_TYPES = [
4871
7101
  "agent_call_end",
4872
7102
  "tool_call_end",
4873
7103
  "memory_remember",
4874
- "memory_recall"
7104
+ "memory_recall",
7105
+ "transcription_end"
4875
7106
  ];
4876
7107
  var COST_LEAF_SET = new Set(COST_BEARING_LEAF_TYPES);
4877
7108
  function hasPositiveTokens(event) {
@@ -4879,6 +7110,12 @@ function hasPositiveTokens(event) {
4879
7110
  if (!t) return false;
4880
7111
  return (t.input ?? 0) + (t.output ?? 0) + (t.reasoning ?? 0) + (t.cached ?? 0) + (t.cacheWrite ?? 0) > 0;
4881
7112
  }
7113
+ function hasPositiveBillableWork(event) {
7114
+ if (hasPositiveTokens(event)) return true;
7115
+ const seconds = event.data?.usage?.audioSeconds;
7116
+ const totalTokens = event.data?.usage?.totalTokens;
7117
+ return typeof seconds === "number" && Number.isFinite(seconds) && seconds > 0 || typeof totalTokens === "number" && Number.isFinite(totalTokens) && totalTokens > 0;
7118
+ }
4882
7119
  function isUsableCost(cost) {
4883
7120
  return typeof cost === "number" && Number.isFinite(cost) && cost >= 0;
4884
7121
  }
@@ -4891,7 +7128,7 @@ function isCostBearingLeaf(event) {
4891
7128
  return COST_LEAF_SET.has(event.type);
4892
7129
  }
4893
7130
  function isUnpricedLeaf(event) {
4894
- return event.type !== void 0 && COST_LEAF_SET.has(event.type) && !isUsableCost(event.cost) && hasPositiveTokens(event);
7131
+ return event.type !== void 0 && COST_LEAF_SET.has(event.type) && !isUsableCost(event.cost) && hasPositiveBillableWork(event);
4895
7132
  }
4896
7133
  function isRootLevel(event) {
4897
7134
  const d = event.depth;
@@ -4906,6 +7143,9 @@ var AXL_EVENT_TYPES = [
4906
7143
  // Workflow lifecycle
4907
7144
  "workflow_start",
4908
7145
  "workflow_end",
7146
+ // Transcription operation lifecycle (separate from chat/agent turns)
7147
+ "transcription_start",
7148
+ "transcription_end",
4909
7149
  // Ask boundary
4910
7150
  "ask_start",
4911
7151
  "ask_end",
@@ -4996,6 +7236,8 @@ var LIFECYCLE_TYPES = /* @__PURE__ */ new Set([
4996
7236
  "schema_diagnostic",
4997
7237
  "workflow_start",
4998
7238
  "workflow_end",
7239
+ "transcription_start",
7240
+ "transcription_end",
4999
7241
  "checkpoint_save",
5000
7242
  "checkpoint_replay",
5001
7243
  "await_human",
@@ -5811,6 +8053,21 @@ function redactGate(event) {
5811
8053
  }
5812
8054
  };
5813
8055
  }
8056
+ function redactInputDescriptor(input) {
8057
+ return {
8058
+ ...input,
8059
+ parts: input.parts.map((part) => {
8060
+ if (!part || typeof part !== "object" || !("type" in part) || part.type === "text")
8061
+ return part;
8062
+ const value = part;
8063
+ return {
8064
+ ...part,
8065
+ ...value.locator !== void 0 ? { locator: REDACTED } : {},
8066
+ ...value.label !== void 0 ? { label: REDACTED } : {}
8067
+ };
8068
+ })
8069
+ };
8070
+ }
5814
8071
  function redactMemory(event) {
5815
8072
  const d = event.data;
5816
8073
  if (!d || typeof d !== "object") return event;
@@ -5834,7 +8091,11 @@ var LEGACY_REDACTION_RULES = {
5834
8091
  }
5835
8092
  };
5836
8093
  },
5837
- ask_start: (e) => ({ ...e, prompt: REDACTED }),
8094
+ ask_start: (e) => ({
8095
+ ...e,
8096
+ prompt: REDACTED,
8097
+ ...e.input ? { input: redactInputDescriptor(e.input) } : {}
8098
+ }),
5838
8099
  ask_end: (e) => ({
5839
8100
  ...e,
5840
8101
  outcome: e.outcome.ok ? { ok: true, result: REDACTED } : { ok: false, error: REDACTED }
@@ -5842,6 +8103,14 @@ var LEGACY_REDACTION_RULES = {
5842
8103
  agent_call_start: (e) => {
5843
8104
  const d = e.data;
5844
8105
  const out = { ...d, prompt: REDACTED };
8106
+ if (d.input) {
8107
+ out.input = redactInputDescriptor(d.input);
8108
+ }
8109
+ if (d.messageInputs)
8110
+ out.messageInputs = d.messageInputs.map((entry) => ({
8111
+ ...entry,
8112
+ input: redactInputDescriptor(entry.input)
8113
+ }));
5845
8114
  if (d.system !== void 0) out.system = REDACTED;
5846
8115
  if (Array.isArray(d.messages)) {
5847
8116
  out.messages = [{ role: "system", content: `[${d.messages.length} messages redacted]` }];
@@ -5997,6 +8266,18 @@ function reuseLegacyRule(type) {
5997
8266
  var COMMON_V2_REDACTION_RULES = {
5998
8267
  workflow_start: reuseLegacyRule("workflow_start"),
5999
8268
  workflow_end: reuseLegacyRule("workflow_end"),
8269
+ transcription_start: (e) => e,
8270
+ transcription_end: (e) => {
8271
+ const d = e.data;
8272
+ return {
8273
+ ...e,
8274
+ data: {
8275
+ ...d,
8276
+ ...d.text !== void 0 ? { text: REDACTED } : {},
8277
+ ...d.error !== void 0 ? { error: REDACTED } : {}
8278
+ }
8279
+ };
8280
+ },
6000
8281
  ask_start: reuseLegacyRule("ask_start"),
6001
8282
  ask_end: reuseLegacyRule("ask_end"),
6002
8283
  agent_call_start: reuseLegacyRule("agent_call_start"),
@@ -6191,7 +8472,7 @@ function resolveConfig(config) {
6191
8472
 
6192
8473
  // src/tool-model-output.ts
6193
8474
  import { types as nodeTypes } from "util";
6194
- function invalid(path, reason) {
8475
+ function invalid2(path, reason) {
6195
8476
  return new Error(`Invalid tool model output at ${path}: ${reason}`);
6196
8477
  }
6197
8478
  function propertyPath(parent, key) {
@@ -6209,14 +8490,14 @@ function propertyPath(parent, key) {
6209
8490
  function normalize(value, path, stack) {
6210
8491
  if (value === null || typeof value === "string" || typeof value === "boolean") return value;
6211
8492
  if (typeof value === "number") {
6212
- if (!Number.isFinite(value)) throw invalid(path, "number must be finite");
8493
+ if (!Number.isFinite(value)) throw invalid2(path, "number must be finite");
6213
8494
  return value;
6214
8495
  }
6215
- if (value === void 0) throw invalid(path, "undefined is only allowed on object properties");
8496
+ if (value === void 0) throw invalid2(path, "undefined is only allowed on object properties");
6216
8497
  if (typeof value === "bigint" || typeof value === "function" || typeof value === "symbol") {
6217
- throw invalid(path, `unsupported ${typeof value}`);
8498
+ throw invalid2(path, `unsupported ${typeof value}`);
6218
8499
  }
6219
- if (typeof value !== "object") throw invalid(path, `unsupported ${typeof value}`);
8500
+ if (typeof value !== "object") throw invalid2(path, `unsupported ${typeof value}`);
6220
8501
  let prototype;
6221
8502
  let descriptors;
6222
8503
  try {
@@ -6224,38 +8505,38 @@ function normalize(value, path, stack) {
6224
8505
  descriptors = Object.getOwnPropertyDescriptors(value);
6225
8506
  Object.setPrototypeOf(descriptors, null);
6226
8507
  } catch (cause) {
6227
- throw invalid(
8508
+ throw invalid2(
6228
8509
  path,
6229
8510
  `object inspection failed (${cause instanceof Error ? cause.name : "error"})`
6230
8511
  );
6231
8512
  }
6232
- if (stack.has(value)) throw invalid(path, "cyclic reference");
8513
+ if (stack.has(value)) throw invalid2(path, "cyclic reference");
6233
8514
  for (const key of Reflect.ownKeys(descriptors)) {
6234
8515
  const descriptor = descriptors[key];
6235
8516
  if (descriptor && !("value" in descriptor)) {
6236
- throw invalid(
8517
+ throw invalid2(
6237
8518
  typeof key === "string" ? propertyPath(path, key) : `${path}[symbol]`,
6238
8519
  "accessor properties are not supported"
6239
8520
  );
6240
8521
  }
6241
8522
  if (typeof key === "symbol") {
6242
- throw invalid(`${path}[symbol]`, "symbol keys are not supported");
8523
+ throw invalid2(`${path}[symbol]`, "symbol keys are not supported");
6243
8524
  }
6244
8525
  }
6245
8526
  if (typeof descriptors.toJSON?.value === "function") {
6246
- throw invalid(propertyPath(path, "toJSON"), "custom toJSON is not supported");
8527
+ throw invalid2(propertyPath(path, "toJSON"), "custom toJSON is not supported");
6247
8528
  }
6248
8529
  const array = Array.isArray(value);
6249
8530
  if (array && prototype !== Array.prototype && prototype !== null) {
6250
- throw invalid(path, "array subclasses are not supported");
8531
+ throw invalid2(path, "array subclasses are not supported");
6251
8532
  }
6252
8533
  const inheritedToJSON = prototype === Array.prototype || prototype === Object.prototype ? Object.getOwnPropertyDescriptor(prototype, "toJSON") : void 0;
6253
8534
  if (typeof inheritedToJSON?.value === "function") {
6254
- throw invalid(propertyPath(path, "toJSON"), "inherited custom toJSON is not supported");
8535
+ throw invalid2(propertyPath(path, "toJSON"), "inherited custom toJSON is not supported");
6255
8536
  }
6256
8537
  const inheritedThen = prototype === Array.prototype || prototype === Object.prototype ? Object.getOwnPropertyDescriptor(prototype, "then") : void 0;
6257
8538
  if (typeof descriptors.then?.value === "function" || inheritedThen !== void 0 && (!("value" in inheritedThen) || typeof inheritedThen.value === "function")) {
6258
- throw invalid(path, "thenable output is not supported");
8539
+ throw invalid2(path, "thenable output is not supported");
6259
8540
  }
6260
8541
  stack.add(value);
6261
8542
  try {
@@ -6263,15 +8544,15 @@ function normalize(value, path, stack) {
6263
8544
  const normalized2 = [];
6264
8545
  const length = descriptors.length?.value;
6265
8546
  if (!Number.isSafeInteger(length) || length < 0) {
6266
- throw invalid(propertyPath(path, "length"), "array length is invalid");
8547
+ throw invalid2(propertyPath(path, "length"), "array length is invalid");
6267
8548
  }
6268
8549
  for (let index = 0; index < length; index++) {
6269
8550
  const descriptor = descriptors[String(index)];
6270
8551
  if (!descriptor || !("value" in descriptor)) {
6271
- throw invalid(`${path}[${index}]`, "arrays must be dense data properties");
8552
+ throw invalid2(`${path}[${index}]`, "arrays must be dense data properties");
6272
8553
  }
6273
8554
  if (descriptor.value === void 0) {
6274
- throw invalid(`${path}[${index}]`, "array values cannot be undefined");
8555
+ throw invalid2(`${path}[${index}]`, "array values cannot be undefined");
6275
8556
  }
6276
8557
  normalized2.push(normalize(descriptor.value, `${path}[${index}]`, stack));
6277
8558
  }
@@ -6280,21 +8561,21 @@ function normalize(value, path, stack) {
6280
8561
  if (!descriptor?.enumerable) continue;
6281
8562
  const index = Number(key);
6282
8563
  if (!Number.isInteger(index) || index < 0 || index >= length || String(index) !== key) {
6283
- throw invalid(propertyPath(path, key), "non-index array properties are not supported");
8564
+ throw invalid2(propertyPath(path, key), "non-index array properties are not supported");
6284
8565
  }
6285
8566
  }
6286
8567
  Object.setPrototypeOf(normalized2, null);
6287
8568
  return normalized2;
6288
8569
  }
6289
8570
  if (prototype !== Object.prototype && prototype !== null) {
6290
- throw invalid(path, "only plain objects and null-prototype records are supported");
8571
+ throw invalid2(path, "only plain objects and null-prototype records are supported");
6291
8572
  }
6292
8573
  const normalized = /* @__PURE__ */ Object.create(null);
6293
8574
  for (const key of Object.keys(descriptors)) {
6294
8575
  const descriptor = descriptors[key];
6295
8576
  if (!descriptor?.enumerable) continue;
6296
8577
  if (!("value" in descriptor)) {
6297
- throw invalid(propertyPath(path, key), "accessor properties are not supported");
8578
+ throw invalid2(propertyPath(path, key), "accessor properties are not supported");
6298
8579
  }
6299
8580
  if (descriptor.value === void 0) continue;
6300
8581
  normalized[key] = normalize(descriptor.value, propertyPath(path, key), stack);
@@ -6330,7 +8611,15 @@ function cloneToolArguments(value) {
6330
8611
  }
6331
8612
  }
6332
8613
  function parseToolInvocation(options) {
6333
- const { toolCall, configuredTool, override, mcpCall, mcpTraceName, availableTools } = options;
8614
+ const {
8615
+ toolCall,
8616
+ configuredTool,
8617
+ override,
8618
+ mcpCall,
8619
+ mcpTraceName,
8620
+ availableTools,
8621
+ providerVisibleSchema
8622
+ } = options;
6334
8623
  const requestedTool = toolCall.function.name;
6335
8624
  if (!override && !configuredTool && !mcpCall) {
6336
8625
  return {
@@ -6390,7 +8679,7 @@ function parseToolInvocation(options) {
6390
8679
  args: parsedArguments,
6391
8680
  issues: toolArgumentIssues(error)
6392
8681
  },
6393
- modelMessage: INVALID_ARGUMENTS_MODEL_MESSAGE
8682
+ modelMessage: toolArgumentModelMessage(error, providerVisibleSchema) ?? INVALID_ARGUMENTS_MODEL_MESSAGE
6394
8683
  };
6395
8684
  }
6396
8685
  }
@@ -6794,44 +9083,44 @@ function resolveBranchDrainTimeoutMs(value) {
6794
9083
  return resolved;
6795
9084
  }
6796
9085
  function parseHumanDecision(value) {
6797
- const invalid2 = () => {
9086
+ const invalid3 = () => {
6798
9087
  throw new AxlError(
6799
9088
  "INVALID_HUMAN_DECISION",
6800
9089
  "Human decision must be a plain object with a boolean approved field and only branch-appropriate string data"
6801
9090
  );
6802
9091
  };
6803
- if (typeof value !== "object" || value === null || Array.isArray(value)) invalid2();
9092
+ if (typeof value !== "object" || value === null || Array.isArray(value)) invalid3();
6804
9093
  let prototype;
6805
9094
  try {
6806
9095
  prototype = Object.getPrototypeOf(value);
6807
9096
  } catch {
6808
- return invalid2();
9097
+ return invalid3();
6809
9098
  }
6810
- if (prototype !== Object.prototype && prototype !== null) invalid2();
9099
+ if (prototype !== Object.prototype && prototype !== null) invalid3();
6811
9100
  let ownKeys;
6812
9101
  try {
6813
9102
  ownKeys = Reflect.ownKeys(value);
6814
9103
  } catch {
6815
- return invalid2();
9104
+ return invalid3();
6816
9105
  }
6817
9106
  if (ownKeys.some(
6818
9107
  (key) => typeof key !== "string" || key !== "approved" && key !== "data" && key !== "reason"
6819
9108
  )) {
6820
- invalid2();
9109
+ invalid3();
6821
9110
  }
6822
9111
  const readDataProperty = (key) => {
6823
9112
  let descriptor;
6824
9113
  try {
6825
9114
  descriptor = Object.getOwnPropertyDescriptor(value, key);
6826
9115
  } catch {
6827
- return invalid2();
9116
+ return invalid3();
6828
9117
  }
6829
9118
  if (!descriptor) {
6830
- if (ownKeys.includes(key)) invalid2();
9119
+ if (ownKeys.includes(key)) invalid3();
6831
9120
  return { present: false, value: void 0 };
6832
9121
  }
6833
- if (!ownKeys.includes(key)) return invalid2();
6834
- if (!("value" in descriptor)) return invalid2();
9122
+ if (!ownKeys.includes(key)) return invalid3();
9123
+ if (!("value" in descriptor)) return invalid3();
6835
9124
  return { present: true, value: descriptor.value };
6836
9125
  };
6837
9126
  const approved = readDataProperty("approved");
@@ -6850,7 +9139,7 @@ function parseHumanDecision(value) {
6850
9139
  };
6851
9140
  }
6852
9141
  }
6853
- return invalid2();
9142
+ return invalid3();
6854
9143
  }
6855
9144
  var askStorage = new AsyncLocalStorage();
6856
9145
  var inlineSchemaCache = /* @__PURE__ */ new WeakMap();
@@ -6958,8 +9247,12 @@ function appendRetryMessages(messages, content, feedbackMessage, providerMetadat
6958
9247
  }
6959
9248
  function estimateMessagesTokens(messages) {
6960
9249
  let total = 0;
9250
+ let unmeasured = false;
6961
9251
  for (const msg of messages) {
6962
- total += estimateTokens(msg.content);
9252
+ total += estimateTokens(inputText(msg.content));
9253
+ if (typeof msg.content !== "string") {
9254
+ unmeasured ||= msg.content.some((part) => part.type === "image");
9255
+ }
6963
9256
  if (msg.tool_calls) {
6964
9257
  for (const tc of msg.tool_calls) {
6965
9258
  total += estimateTokens(tc.function.name + tc.function.arguments);
@@ -6967,7 +9260,73 @@ function estimateMessagesTokens(messages) {
6967
9260
  }
6968
9261
  total += 4;
6969
9262
  }
6970
- return total;
9263
+ return { tokens: total, unmeasured };
9264
+ }
9265
+ function cloneMessagesForProvider(messages) {
9266
+ return messages.map((message) => ({
9267
+ ...message,
9268
+ content: cloneModelInput(message.content),
9269
+ ...message.tool_calls ? {
9270
+ tool_calls: message.tool_calls.map((call) => ({
9271
+ ...call,
9272
+ function: { ...call.function }
9273
+ }))
9274
+ } : {}
9275
+ }));
9276
+ }
9277
+ function assertRichMessagesAreUserOnly(messages) {
9278
+ for (const message of messages) {
9279
+ if (typeof message.content !== "string" && message.role !== "user") {
9280
+ throw new InvalidModelInputError(
9281
+ "Non-text model input parts are only supported on user messages"
9282
+ );
9283
+ }
9284
+ }
9285
+ }
9286
+ function normalizeSessionHistory(messages) {
9287
+ return messages.map((message) => {
9288
+ if (typeof message.content === "string") return { ...message };
9289
+ if (message.role !== "user") {
9290
+ throw new InvalidModelInputError(
9291
+ "Non-text model input parts are only supported on user messages"
9292
+ );
9293
+ }
9294
+ return { ...message, content: normalizeModelInput(message.content) };
9295
+ });
9296
+ }
9297
+ function hasRichMessage(messages) {
9298
+ return messages.some((message) => typeof message.content !== "string");
9299
+ }
9300
+ function firstImageSource(input, history) {
9301
+ const find = (value) => typeof value === "string" ? void 0 : value.find((part) => part.type === "image")?.source.type;
9302
+ return find(input) ?? history.map((message) => find(message.content)).find(Boolean);
9303
+ }
9304
+ function appendHandoffInstruction(input, originalText, instruction) {
9305
+ if (typeof input === "string" || instruction.length === 0 || instruction === originalText)
9306
+ return input;
9307
+ return [...input, { type: "text", text: instruction }];
9308
+ }
9309
+ var RICH_MODEL_INPUT_ERROR_MESSAGE = "Provider request failed while processing rich model input";
9310
+ var RICH_MODEL_INPUT_ERROR_NAME = "RichModelInputError";
9311
+ var MAX_RICH_FAILURE_ASSOCIATIONS = 32;
9312
+ var MAX_RICH_FAILURE_CAUSE_DEPTH = 4;
9313
+ function legacyErrorMessage(error) {
9314
+ return error instanceof Error ? error.message : String(error);
9315
+ }
9316
+ function providerErrorEventMetadata(error, richRequest) {
9317
+ if (!(error instanceof ProviderError)) {
9318
+ return error instanceof AxlError && richRequest ? { code: error.code } : {};
9319
+ }
9320
+ return {
9321
+ status: error.status,
9322
+ retryable: error.retryable,
9323
+ ...richRequest ? {
9324
+ code: error.code,
9325
+ provider: error.provider,
9326
+ ...error.requestId ? { requestId: error.requestId } : {},
9327
+ ...error.retryAfterMs !== void 0 ? { retryAfterMs: error.retryAfterMs } : {}
9328
+ } : {}
9329
+ };
6971
9330
  }
6972
9331
  var WorkflowContext = class _WorkflowContext {
6973
9332
  input;
@@ -6983,6 +9342,7 @@ var WorkflowContext = class _WorkflowContext {
6983
9342
  }
6984
9343
  config;
6985
9344
  providerRegistry;
9345
+ transcriptionProviderRegistry;
6986
9346
  sessionHistory;
6987
9347
  onTrace;
6988
9348
  pendingDecisions;
@@ -7008,6 +9368,10 @@ var WorkflowContext = class _WorkflowContext {
7008
9368
  */
7009
9369
  autoCheckpointCounters;
7010
9370
  branchDrainState;
9371
+ /** Bounded exact-identity failure association for this execution tree.
9372
+ * Unlike a process-global marker, this cannot affect another context that
9373
+ * happens to reuse the same Error object. */
9374
+ richFailureState;
7011
9375
  /** Root-shared idempotency guards for workflow lifecycle emission.
7012
9376
  * Both `runtime.execute()` and `runtime.stream()` have paths where a
7013
9377
  * post-emit side-effect (checkpoint deletion, state-store persistence)
@@ -7091,7 +9455,8 @@ var WorkflowContext = class _WorkflowContext {
7091
9455
  get events() {
7092
9456
  if (!this._busRef.current) {
7093
9457
  this._busRef.current = new AxlEventBus(this._eventStreamOptions);
7094
- if (this.signal?.aborted) {
9458
+ const signal = this.currentSignal;
9459
+ if (signal?.aborted) {
7095
9460
  this._busRef.current._finish();
7096
9461
  }
7097
9462
  }
@@ -7129,6 +9494,7 @@ var WorkflowContext = class _WorkflowContext {
7129
9494
  this.metadata = init.metadata ?? {};
7130
9495
  this.config = init.config;
7131
9496
  this.providerRegistry = init.providerRegistry;
9497
+ this.transcriptionProviderRegistry = init.transcriptionProviderRegistry;
7132
9498
  this.sessionHistory = init.sessionHistory ?? [];
7133
9499
  this.onTrace = init.onTrace;
7134
9500
  this.pendingDecisions = init.pendingDecisions;
@@ -7162,6 +9528,11 @@ var WorkflowContext = class _WorkflowContext {
7162
9528
  byAgent: /* @__PURE__ */ new Map(),
7163
9529
  root: 0
7164
9530
  };
9531
+ this.richFailureState = init.richFailureState ?? {
9532
+ associations: [],
9533
+ overflowed: false,
9534
+ indeterminate: false
9535
+ };
7165
9536
  const branchDrainTimeoutMs = resolveBranchDrainTimeoutMs(init.branchDrainTimeoutMs);
7166
9537
  this.branchDrainState = init.branchDrainState ?? {
7167
9538
  pending: /* @__PURE__ */ new Set(),
@@ -7194,6 +9565,7 @@ var WorkflowContext = class _WorkflowContext {
7194
9565
  executionId: this.executionId,
7195
9566
  config: this.config,
7196
9567
  providerRegistry: this.providerRegistry,
9568
+ transcriptionProviderRegistry: this.transcriptionProviderRegistry,
7197
9569
  metadata: { ...this.metadata },
7198
9570
  // Shared infrastructure
7199
9571
  budgetContext: this.budgetContext,
@@ -7216,6 +9588,7 @@ var WorkflowContext = class _WorkflowContext {
7216
9588
  autoCheckpointCounters: this.autoCheckpointCounters,
7217
9589
  branchDrainState: this.branchDrainState,
7218
9590
  workflowLifecycleState: this.workflowLifecycleState,
9591
+ richFailureState: this.richFailureState,
7219
9592
  // Share the parent's bus slot (mutable ref). Late allocation in
7220
9593
  // either parent or child propagates because both read the same
7221
9594
  // slot. Nested-ask events (agent-as-tool pattern) bubble up to
@@ -7227,6 +9600,52 @@ var WorkflowContext = class _WorkflowContext {
7227
9600
  // Isolated: sessionHistory (empty)
7228
9601
  });
7229
9602
  }
9603
+ /** Associate an exact thrown value with rich input in this execution only. */
9604
+ recordRichFailure(error) {
9605
+ const state = this.richFailureState;
9606
+ if (state.associations.some((known) => Object.is(known, error))) return;
9607
+ if (state.associations.length >= MAX_RICH_FAILURE_ASSOCIATIONS) {
9608
+ state.associations.shift();
9609
+ state.overflowed = true;
9610
+ state.indeterminate = true;
9611
+ }
9612
+ state.associations.push(error);
9613
+ }
9614
+ /** @internal Safe observer projection for a failure associated with rich input. */
9615
+ _observerErrorProjection(error) {
9616
+ const state = this.richFailureState;
9617
+ const safe = () => ({
9618
+ message: RICH_MODEL_INPUT_ERROR_MESSAGE,
9619
+ name: RICH_MODEL_INPUT_ERROR_NAME
9620
+ });
9621
+ const indeterminate = () => {
9622
+ if (state.associations.length === 0) return void 0;
9623
+ state.indeterminate = true;
9624
+ return safe();
9625
+ };
9626
+ let current = error;
9627
+ const seen = /* @__PURE__ */ new WeakSet();
9628
+ for (let depth = 0; depth < MAX_RICH_FAILURE_CAUSE_DEPTH; depth++) {
9629
+ if (state.associations.some((known) => Object.is(known, current))) {
9630
+ return safe();
9631
+ }
9632
+ if ((typeof current !== "object" || current === null) && typeof current !== "function") {
9633
+ return state.overflowed || state.indeterminate ? safe() : void 0;
9634
+ }
9635
+ if (seen.has(current)) return indeterminate();
9636
+ seen.add(current);
9637
+ let descriptor;
9638
+ try {
9639
+ descriptor = Object.getOwnPropertyDescriptor(current, "cause");
9640
+ } catch {
9641
+ return indeterminate();
9642
+ }
9643
+ if (!descriptor) return state.overflowed || state.indeterminate ? safe() : void 0;
9644
+ if (!("value" in descriptor)) return indeterminate();
9645
+ current = descriptor.value;
9646
+ }
9647
+ return indeterminate();
9648
+ }
7230
9649
  /**
7231
9650
  * Resolve the current abort signal.
7232
9651
  * Branch-scoped signals (from race/spawn/map/budget) in AsyncLocalStorage
@@ -7293,7 +9712,7 @@ var WorkflowContext = class _WorkflowContext {
7293
9712
  /** Resolve, execute, and observe one non-handoff tool request. Rejections
7294
9713
  * never start. Every accepted request emits one terminal event before a
7295
9714
  * continuation message or abort escapes this coordinator. */
7296
- async observeToolInvocation(agent2, toolCall, currentMessages, availableTools) {
9715
+ async observeToolInvocation(agent2, toolCall, currentMessages, toolDefs) {
7297
9716
  const signal = this.currentSignal;
7298
9717
  signal?.throwIfAborted();
7299
9718
  const requestedTool = toolCall.function.name;
@@ -7303,6 +9722,7 @@ var WorkflowContext = class _WorkflowContext {
7303
9722
  const override = this.toolOverrides?.get(requestedTool);
7304
9723
  const mcpManager = this.mcpManager;
7305
9724
  const mcpMatch = !configuredTool ? mcpManager?.getToolsForAgent(agent2._config.mcp, agent2._config.mcpTools).find(({ tool: tool2 }) => tool2.name === requestedTool) : void 0;
9725
+ const providerVisibleSchema = toolDefs.find((tool2) => tool2.function.name === requestedTool)?.function.parameters;
7306
9726
  const parsed = parseToolInvocation({
7307
9727
  toolCall,
7308
9728
  configuredTool,
@@ -7311,7 +9731,8 @@ var WorkflowContext = class _WorkflowContext {
7311
9731
  mcpCall: (args) => mcpManager.callTool(requestedTool, args, mcpMatch.server),
7312
9732
  mcpTraceName: `${mcpMatch.server}:${requestedTool}`
7313
9733
  } : {},
7314
- availableTools
9734
+ availableTools: toolDefs.map((tool2) => tool2.function.name),
9735
+ ...configuredTool ? { providerVisibleSchema } : {}
7315
9736
  });
7316
9737
  signal?.throwIfAborted();
7317
9738
  if ("kind" in parsed) {
@@ -7426,6 +9847,17 @@ var WorkflowContext = class _WorkflowContext {
7426
9847
  }
7427
9848
  // ── ctx.ask() ─────────────────────────────────────────────────────────
7428
9849
  async ask(agent2, prompt, options) {
9850
+ const normalizedInput = normalizeModelInput(prompt);
9851
+ const normalizedHistory = normalizeSessionHistory(this.sessionHistory);
9852
+ const delegateInputHolder = agent2;
9853
+ if (delegateInputHolder._delegateOriginalInput !== void 0) {
9854
+ delegateInputHolder._delegateOriginalInput = normalizeModelInput(
9855
+ delegateInputHolder._delegateOriginalInput
9856
+ );
9857
+ }
9858
+ const richAsk = typeof normalizedInput !== "string" || hasRichMessage(normalizedHistory) || delegateInputHolder._delegateOriginalInput !== void 0 && typeof delegateInputHolder._delegateOriginalInput !== "string";
9859
+ const promptText = inputText(normalizedInput);
9860
+ const inputDescriptor = describeModelInput(normalizedInput);
7429
9861
  const agentName = agent2._name;
7430
9862
  return this._checkpoint(
7431
9863
  this._autoCheckpointName("ask", agentName),
@@ -7445,7 +9877,11 @@ var WorkflowContext = class _WorkflowContext {
7445
9877
  };
7446
9878
  return askStorage.run(frame, async () => {
7447
9879
  const askStart = Date.now();
7448
- this.emitEvent({ type: "ask_start", prompt });
9880
+ this.emitEvent({
9881
+ type: "ask_start",
9882
+ prompt: promptText,
9883
+ ...inputDescriptor ? { input: inputDescriptor } : {}
9884
+ });
7449
9885
  const costBefore = this.budgetContext?.totalCost ?? 0;
7450
9886
  const unpricedCountBefore = this.budgetContext?.unpricedCount ?? 0;
7451
9887
  const resolveCtx = options?.metadata ? { metadata: { ...this.metadata, ...options.metadata } } : { metadata: this.metadata };
@@ -7453,10 +9889,11 @@ var WorkflowContext = class _WorkflowContext {
7453
9889
  const doCall = async () => {
7454
9890
  const result2 = await this.executeAgentCall(
7455
9891
  agent2,
7456
- prompt,
9892
+ normalizedInput,
7457
9893
  options,
7458
9894
  void 0,
7459
- usageCapture
9895
+ usageCapture,
9896
+ normalizedHistory
7460
9897
  );
7461
9898
  return result2;
7462
9899
  };
@@ -7473,6 +9910,9 @@ var WorkflowContext = class _WorkflowContext {
7473
9910
  },
7474
9911
  async (span) => {
7475
9912
  const r = await doCall();
9913
+ if (usageCapture.modelUri) {
9914
+ span.setAttribute("axl.agent.model", usageCapture.modelUri);
9915
+ }
7476
9916
  const costAfter2 = this.budgetContext?.totalCost ?? 0;
7477
9917
  span.setAttribute("axl.agent.cost", costAfter2 - costBefore);
7478
9918
  span.setAttribute("axl.agent.duration", Date.now() - askStart);
@@ -7506,9 +9946,10 @@ var WorkflowContext = class _WorkflowContext {
7506
9946
  try {
7507
9947
  this.onAgentCallComplete({
7508
9948
  agent: agent2._name,
7509
- prompt,
9949
+ prompt: promptText,
9950
+ ...inputDescriptor ? { input: inputDescriptor } : {},
7510
9951
  response: typeof result === "string" ? result : JSON.stringify(result),
7511
- model: agent2.resolveModel(resolveCtx),
9952
+ model: usageCapture.modelUri ?? agent2.resolveModel(resolveCtx),
7512
9953
  cost: costAfter - costBefore,
7513
9954
  unpriced: frame.askUnpriced || unpricedCountAfter > unpricedCountBefore,
7514
9955
  duration: Date.now() - askStart,
@@ -7532,9 +9973,10 @@ var WorkflowContext = class _WorkflowContext {
7532
9973
  } catch (err) {
7533
9974
  askFailed = true;
7534
9975
  askFailure = err;
9976
+ if (richAsk) this.recordRichFailure(err);
7535
9977
  outcome = {
7536
9978
  ok: false,
7537
- error: err instanceof Error ? err.message : String(err)
9979
+ error: this._observerErrorProjection(err)?.message ?? legacyErrorMessage(err)
7538
9980
  };
7539
9981
  }
7540
9982
  let terminalFailure;
@@ -7562,7 +10004,7 @@ var WorkflowContext = class _WorkflowContext {
7562
10004
  { agent: agentName }
7563
10005
  );
7564
10006
  }
7565
- async executeAgentCall(agent2, prompt, options, handoffMessages, usageCapture) {
10007
+ async executeAgentCall(agent2, input, options, handoffMessages, usageCapture, sessionHistory = normalizeSessionHistory(this.sessionHistory)) {
7566
10008
  if (this.budgetContext?.exceeded) {
7567
10009
  const { limit, totalCost: spent, policy } = this.budgetContext;
7568
10010
  if (policy === "warn") {
@@ -7579,7 +10021,82 @@ var WorkflowContext = class _WorkflowContext {
7579
10021
  const resolveCtx = options?.metadata ? { metadata: { ...this.metadata, ...options.metadata } } : { metadata: this.metadata };
7580
10022
  const modelUri = agent2.resolveModel(resolveCtx);
7581
10023
  const systemPrompt = agent2.resolveSystem(resolveCtx);
7582
- const { provider, model } = this.providerRegistry.resolve(modelUri, this.config);
10024
+ const { provider, model: resolvedModel } = this.providerRegistry.resolve(modelUri, this.config);
10025
+ let model = resolvedModel;
10026
+ const prompt = inputText(input);
10027
+ const providerOptions = options?.providerOptions ?? agent2._config.providerOptions;
10028
+ const richRequest = typeof input !== "string" || hasRichMessage(sessionHistory);
10029
+ if (richRequest) {
10030
+ if (providerOptions && ("messages" in providerOptions || "input" in providerOptions)) {
10031
+ throw new UnsupportedModelInputError({
10032
+ provider: provider.name ?? modelUri.split(":", 1)[0],
10033
+ model,
10034
+ modality: "image",
10035
+ feature: "raw input-container providerOptions"
10036
+ });
10037
+ }
10038
+ const source = firstImageSource(input, sessionHistory);
10039
+ if (!provider.validateInput) {
10040
+ throw new UnsupportedModelInputError({
10041
+ provider: provider.name ?? modelUri.split(":", 1)[0],
10042
+ model,
10043
+ modality: "image",
10044
+ ...source ? { source } : {}
10045
+ });
10046
+ }
10047
+ const validation = provider.validateInput({
10048
+ model,
10049
+ input: cloneModelInput(input),
10050
+ history: cloneMessagesForProvider(sessionHistory),
10051
+ stream: this._streamingEnabled,
10052
+ hasTools: Boolean(
10053
+ agent2._config.tools?.length || agent2._config.mcp?.length || agent2._config.mcpTools?.length || agent2._config.handoffs
10054
+ ),
10055
+ responseMode: options?.schema ? "structured" : "text",
10056
+ providerOptions
10057
+ });
10058
+ if (!validation || typeof validation.effectiveModel !== "string" || validation.effectiveModel.length === 0) {
10059
+ throw new UnsupportedModelInputError({
10060
+ provider: provider.name ?? modelUri.split(":", 1)[0],
10061
+ model,
10062
+ modality: "image",
10063
+ ...source ? { source } : {}
10064
+ });
10065
+ }
10066
+ model = validation.effectiveModel;
10067
+ }
10068
+ const effectiveModelUri = !richRequest ? modelUri : `${modelUri.slice(0, modelUri.indexOf(":") + 1)}${model}`;
10069
+ if (usageCapture) usageCapture.modelUri ??= effectiveModelUri;
10070
+ if (richRequest) {
10071
+ const descriptors = [
10072
+ describeModelInput(input),
10073
+ ...sessionHistory.map((message) => describeModelInput(message.content))
10074
+ ].filter(
10075
+ (descriptor) => descriptor !== void 0
10076
+ );
10077
+ const parts = descriptors.flatMap((descriptor) => descriptor.parts);
10078
+ this.spanManager?.addEventToActiveSpan("axl.model_input", {
10079
+ "axl.input.parts": parts.length,
10080
+ "axl.input.images": parts.filter((part) => part.type === "image").length,
10081
+ "axl.input.source.bytes": parts.filter(
10082
+ (part) => part.type === "image" && part.source === "bytes"
10083
+ ).length,
10084
+ "axl.input.source.base64": parts.filter(
10085
+ (part) => part.type === "image" && part.source === "base64"
10086
+ ).length,
10087
+ "axl.input.source.url": parts.filter(
10088
+ (part) => part.type === "image" && part.source === "url"
10089
+ ).length,
10090
+ "axl.input.source.provider_file": parts.filter(
10091
+ (part) => part.type === "image" && part.source === "provider-file"
10092
+ ).length,
10093
+ "axl.input.inline_bytes": parts.reduce(
10094
+ (sum, part) => sum + (part.type === "image" ? part.bytes ?? 0 : 0),
10095
+ 0
10096
+ )
10097
+ });
10098
+ }
10099
+ assertRichMessagesAreUserOnly(sessionHistory);
7583
10100
  let resolvedHandoffs;
7584
10101
  if (typeof agent2._config.handoffs === "function") {
7585
10102
  try {
@@ -7600,50 +10117,60 @@ var WorkflowContext = class _WorkflowContext {
7600
10117
  messages.push({ role: "system", content: systemPrompt });
7601
10118
  }
7602
10119
  const maxContext = agent2._config.maxContext;
7603
- if (maxContext && this.sessionHistory.length > 0) {
10120
+ if (maxContext && sessionHistory.length > 0) {
7604
10121
  const reserveTokens = this.config.contextManagement?.reserveTokens ?? 2e3;
7605
10122
  const systemTokens = systemPrompt ? estimateTokens(systemPrompt) : 0;
7606
10123
  const toolTokens = toolDefs.length > 0 ? estimateTokens(JSON.stringify(toolDefs)) : 0;
7607
10124
  const overhead = systemTokens + toolTokens + reserveTokens;
7608
10125
  const availableForHistory = maxContext - overhead;
7609
- const historyTokens = estimateMessagesTokens(this.sessionHistory);
7610
- if (historyTokens > availableForHistory) {
10126
+ const historyEstimate = estimateMessagesTokens(sessionHistory);
10127
+ if (historyEstimate.unmeasured) {
10128
+ this.emitEvent({
10129
+ type: "log",
10130
+ data: {
10131
+ warning: "Context media contribution is unmeasured; provider will enforce its context limit"
10132
+ }
10133
+ });
10134
+ }
10135
+ if (historyEstimate.tokens > availableForHistory) {
7611
10136
  const summarizedMessages = await this.summarizeHistory(
7612
10137
  provider,
7613
10138
  model,
7614
- this.sessionHistory,
10139
+ sessionHistory,
7615
10140
  availableForHistory
7616
10141
  );
7617
10142
  for (const msg of summarizedMessages) {
7618
10143
  messages.push(msg);
7619
10144
  }
7620
10145
  } else {
7621
- for (const msg of this.sessionHistory) {
10146
+ for (const msg of sessionHistory) {
7622
10147
  messages.push(msg);
7623
10148
  }
7624
10149
  }
7625
10150
  } else {
7626
- for (const msg of this.sessionHistory) {
10151
+ for (const msg of sessionHistory) {
7627
10152
  messages.push(msg);
7628
10153
  }
7629
10154
  }
7630
10155
  const schemaPromptMode = options?.schemaPrompt ?? agent2._config.schemaPrompt ?? "json-schema";
7631
- let userContent = prompt;
10156
+ let userContent = input;
7632
10157
  let appendedSchemaText;
7633
10158
  if (options?.schema) {
7634
10159
  const schema = options.schema;
7635
10160
  if (schemaPromptMode === "none") {
7636
10161
  } else if (schemaPromptMode === "json-schema") {
7637
10162
  appendedSchemaText = renderSchemaForPrompt(schema);
7638
- userContent += `
10163
+ const guidance = `
7639
10164
 
7640
10165
  Respond with valid JSON matching this schema:
7641
10166
  ${appendedSchemaText}`;
10167
+ userContent = typeof input === "string" ? input + guidance : [...input, { type: "text", text: guidance }];
7642
10168
  } else {
7643
10169
  appendedSchemaText = typeof schemaPromptMode.render === "function" ? schemaPromptMode.render(schema) : schemaPromptMode.render;
7644
- userContent += `
10170
+ const guidance = `
7645
10171
 
7646
10172
  ${appendedSchemaText}`;
10173
+ userContent = typeof input === "string" ? input + guidance : [...input, { type: "text", text: guidance }];
7647
10174
  }
7648
10175
  }
7649
10176
  messages.push({ role: "user", content: userContent });
@@ -7671,7 +10198,10 @@ ${appendedSchemaText}`;
7671
10198
  }
7672
10199
  const guardrails = agent2._config.guardrails;
7673
10200
  if (guardrails?.input) {
7674
- const inputResult = await guardrails.input(prompt, { metadata: this.metadata });
10201
+ const inputResult = await guardrails.input(prompt, {
10202
+ metadata: this.metadata,
10203
+ input: cloneModelInput(input)
10204
+ });
7675
10205
  this.emitEvent({
7676
10206
  type: "guardrail",
7677
10207
  agent: agent2._name,
@@ -7762,7 +10292,7 @@ ${appendedSchemaText}`;
7762
10292
  includeThoughts: options?.includeThoughts ?? agent2._config.includeThoughts,
7763
10293
  toolChoice: options?.toolChoice ?? agent2._config.toolChoice,
7764
10294
  stop: options?.stop ?? agent2._config.stop,
7765
- providerOptions: options?.providerOptions ?? agent2._config.providerOptions,
10295
+ providerOptions,
7766
10296
  signal: this.currentSignal
7767
10297
  };
7768
10298
  if (options?.schema && toolDefs.length === 0) {
@@ -7788,15 +10318,26 @@ ${appendedSchemaText}`;
7788
10318
  ...chatOptions.stop !== void 0 ? { stop: chatOptions.stop } : {}
7789
10319
  };
7790
10320
  let messagesSnapshot;
10321
+ let messageInputs;
7791
10322
  if (verboseTrace) {
7792
10323
  try {
7793
- messagesSnapshot = structuredClone(currentMessages);
10324
+ messageInputs = currentMessages.flatMap((message, index) => {
10325
+ const descriptor = describeModelInput(message.content);
10326
+ return descriptor ? [{ index, input: descriptor }] : [];
10327
+ });
10328
+ messagesSnapshot = currentMessages.map((message) => ({
10329
+ ...message,
10330
+ content: sanitizeModelInputForTrace(message.content)
10331
+ }));
7794
10332
  } catch (err) {
7795
10333
  console.warn(
7796
10334
  "[axl] verbose trace messages snapshot failed to clone; emitting shallow copy:",
7797
10335
  err instanceof Error ? err.message : String(err)
7798
10336
  );
7799
- messagesSnapshot = [...currentMessages];
10337
+ messagesSnapshot = currentMessages.map((message) => ({
10338
+ ...message,
10339
+ content: sanitizeModelInputForTrace(message.content)
10340
+ }));
7800
10341
  }
7801
10342
  }
7802
10343
  const retryReason = pendingRetryReason;
@@ -7804,16 +10345,18 @@ ${appendedSchemaText}`;
7804
10345
  this.emitEvent({
7805
10346
  type: "agent_call_start",
7806
10347
  agent: agent2._name,
7807
- model: modelUri,
10348
+ model: effectiveModelUri,
7808
10349
  turn: turns,
7809
10350
  data: {
7810
10351
  prompt,
10352
+ ...typeof input === "string" ? {} : { input: describeModelInput(input) },
7811
10353
  ...systemPrompt ? { system: systemPrompt } : {},
7812
10354
  params: callParams,
7813
10355
  turn: turns,
7814
10356
  ...retryReason ? { retryReason } : {},
7815
10357
  ...toolDefs.length > 0 ? { toolNames: toolDefs.map((t) => t.function.name) } : {},
7816
- ...messagesSnapshot ? { messages: messagesSnapshot } : {}
10358
+ ...messagesSnapshot ? { messages: messagesSnapshot } : {},
10359
+ ...messageInputs && messageInputs.length > 0 ? { messageInputs } : {}
7817
10360
  }
7818
10361
  });
7819
10362
  let response;
@@ -7837,7 +10380,10 @@ ${appendedSchemaText}`;
7837
10380
  });
7838
10381
  }
7839
10382
  }) : void 0;
7840
- for await (const chunk of provider.stream(currentMessages, chatOptions)) {
10383
+ for await (const chunk of provider.stream(
10384
+ cloneMessagesForProvider(currentMessages),
10385
+ chatOptions
10386
+ )) {
7841
10387
  if (chunk.type === "text_delta") {
7842
10388
  content2 += chunk.content;
7843
10389
  this.emitEvent({ type: "token", data: chunk.content });
@@ -7903,24 +10449,25 @@ ${appendedSchemaText}`;
7903
10449
  response.thinking_content = thinkingContent;
7904
10450
  }
7905
10451
  } else {
7906
- response = await provider.chat(currentMessages, chatOptions);
10452
+ response = await provider.chat(cloneMessagesForProvider(currentMessages), chatOptions);
7907
10453
  }
7908
10454
  } catch (err) {
10455
+ if (richRequest) this.recordRichFailure(err);
7909
10456
  this.emitEvent({
7910
10457
  type: "agent_call_end",
7911
10458
  agent: agent2._name,
7912
- model: modelUri,
10459
+ model: effectiveModelUri,
7913
10460
  promptVersion: agent2._config.version,
7914
10461
  duration: Date.now() - turnStart,
7915
10462
  data: {
7916
10463
  response: "",
7917
10464
  turn: turns,
7918
10465
  ...retryReason ? { retryReason } : {},
7919
- error: err instanceof Error ? err.message : String(err),
10466
+ error: this._observerErrorProjection(err)?.message ?? legacyErrorMessage(err),
7920
10467
  // Enrich with typed-error metadata when available. `body` is
7921
10468
  // deliberately NOT emitted — it's redaction-eligible (see
7922
10469
  // docs/security.md). `data.error` already carries the message.
7923
- ...err instanceof ProviderError ? { status: err.status, retryable: err.retryable } : {}
10470
+ ...providerErrorEventMetadata(err, richRequest)
7924
10471
  }
7925
10472
  });
7926
10473
  throw err;
@@ -7934,7 +10481,7 @@ ${appendedSchemaText}`;
7934
10481
  this.emitEvent({
7935
10482
  type: "agent_call_end",
7936
10483
  agent: agent2._name,
7937
- model: modelUri,
10484
+ model: effectiveModelUri,
7938
10485
  promptVersion: agent2._config.version,
7939
10486
  cost: response.cost,
7940
10487
  tokens: response.usage ? {
@@ -7968,12 +10515,12 @@ ${appendedSchemaText}`;
7968
10515
  if (descriptor) {
7969
10516
  const mode = descriptor.mode ?? "oneway";
7970
10517
  let handoffPrompt = prompt;
7971
- if (mode === "roundtrip") {
7972
- try {
7973
- const args = JSON.parse(toolCall.function.arguments);
7974
- if (args.message) handoffPrompt = args.message;
7975
- } catch {
10518
+ try {
10519
+ const args = JSON.parse(toolCall.function.arguments);
10520
+ if (typeof args.message === "string" && args.message.length > 0) {
10521
+ handoffPrompt = args.message;
7976
10522
  }
10523
+ } catch {
7977
10524
  }
7978
10525
  const handoffStart = Date.now();
7979
10526
  const sourceFrame = askStorage.getStore();
@@ -8000,15 +10547,27 @@ ${appendedSchemaText}`;
8000
10547
  validateRetries: options.validateRetries
8001
10548
  } : void 0;
8002
10549
  const handoffFn = () => askStorage.run(targetFrame, async () => {
8003
- this.emitEvent({ type: "ask_start", prompt: handoffPrompt });
10550
+ const delegateOriginal = agent2._delegateOriginalInput;
10551
+ const handoffInput = delegateOriginal ?? (typeof input === "string" ? handoffPrompt : input);
10552
+ const enrichedHandoffInput = appendHandoffInstruction(
10553
+ handoffInput,
10554
+ inputText(input),
10555
+ handoffPrompt
10556
+ );
10557
+ this.emitEvent({
10558
+ type: "ask_start",
10559
+ prompt: inputText(enrichedHandoffInput),
10560
+ ...typeof enrichedHandoffInput === "string" ? {} : { input: describeModelInput(enrichedHandoffInput) }
10561
+ });
8004
10562
  const targetAskStart = Date.now();
8005
10563
  try {
8006
10564
  const result = await this.executeAgentCall(
8007
10565
  descriptor.agent,
8008
- handoffPrompt,
10566
+ enrichedHandoffInput,
8009
10567
  handoffOptions,
8010
10568
  currentMessages,
8011
- usageCapture
10569
+ usageCapture,
10570
+ sessionHistory
8012
10571
  );
8013
10572
  this.emitEvent({
8014
10573
  type: "ask_end",
@@ -8019,11 +10578,13 @@ ${appendedSchemaText}`;
8019
10578
  });
8020
10579
  return result;
8021
10580
  } catch (err) {
10581
+ const richHandoff = typeof enrichedHandoffInput !== "string" || hasRichMessage(sessionHistory);
10582
+ if (richHandoff) this.recordRichFailure(err);
8022
10583
  this.emitEvent({
8023
10584
  type: "ask_end",
8024
10585
  outcome: {
8025
10586
  ok: false,
8026
- error: err instanceof Error ? err.message : String(err)
10587
+ error: this._observerErrorProjection(err)?.message ?? legacyErrorMessage(err)
8027
10588
  },
8028
10589
  cost: targetFrame.askCost.value,
8029
10590
  ...targetFrame.askUnpriced ? { unpriced: true } : {},
@@ -8122,12 +10683,7 @@ ${appendedSchemaText}`;
8122
10683
  return await handoffFn();
8123
10684
  }
8124
10685
  }
8125
- await this.observeToolInvocation(
8126
- agent2,
8127
- toolCall,
8128
- currentMessages,
8129
- toolDefs.map((tool2) => tool2.function.name)
8130
- );
10686
+ await this.observeToolInvocation(agent2, toolCall, currentMessages, toolDefs);
8131
10687
  }
8132
10688
  continue;
8133
10689
  }
@@ -8537,12 +11093,12 @@ ${appendedSchemaText}`;
8537
11093
  content: `Summary of earlier conversation:
8538
11094
  ${this.summaryCache}`
8539
11095
  };
8540
- const summaryTokens = estimateTokens(summaryMsg2.content) + 4;
11096
+ const summaryTokens = estimateTokens(summarizeModelInput(summaryMsg2.content)) + 4;
8541
11097
  const remaining = availableTokens - summaryTokens;
8542
11098
  let recentTokens2 = 0;
8543
11099
  let splitIdx2 = history.length;
8544
11100
  for (let i = history.length - 1; i >= 0; i--) {
8545
- const msgTokens = estimateTokens(history[i].content) + 4;
11101
+ const msgTokens = estimateTokens(summarizeModelInput(history[i].content)) + 4;
8546
11102
  if (recentTokens2 + msgTokens > remaining) break;
8547
11103
  recentTokens2 += msgTokens;
8548
11104
  splitIdx2 = i;
@@ -8555,7 +11111,7 @@ ${this.summaryCache}`
8555
11111
  let splitIdx = history.length;
8556
11112
  const targetRecent = Math.floor(availableTokens * 0.6);
8557
11113
  for (let i = history.length - 1; i >= 0; i--) {
8558
- const msgTokens = estimateTokens(history[i].content) + 4;
11114
+ const msgTokens = estimateTokens(summarizeModelInput(history[i].content)) + 4;
8559
11115
  if (recentTokens + msgTokens > targetRecent) break;
8560
11116
  recentTokens += msgTokens;
8561
11117
  splitIdx = i;
@@ -8573,7 +11129,7 @@ ${this.summaryCache}`
8573
11129
  summaryProvider = provider;
8574
11130
  summaryModel = model;
8575
11131
  }
8576
- const oldContent = oldMessages.map((m) => `${m.role}: ${m.content}`).join("\n");
11132
+ const oldContent = oldMessages.map((m) => `${m.role}: ${summarizeModelInput(m.content)}`).join("\n");
8577
11133
  const summaryResponse = await summaryProvider.chat(
8578
11134
  [
8579
11135
  {
@@ -9807,7 +12363,9 @@ ${summaryResponse.content}`
9807
12363
  reason: "routed"
9808
12364
  }
9809
12365
  });
9810
- return this.ask(routerAgent, prompt, {
12366
+ const routerInput = options?.routerInput === "text" ? inputText(prompt) : prompt;
12367
+ routerAgent._delegateOriginalInput = prompt;
12368
+ return this.ask(routerAgent, routerInput, {
9811
12369
  schema: options?.schema,
9812
12370
  schemaPrompt: options?.schemaPrompt,
9813
12371
  nativeStructuredOutput: options?.nativeStructuredOutput,
@@ -9817,6 +12375,278 @@ ${summaryResponse.content}`
9817
12375
  validateRetries: options?.validateRetries
9818
12376
  });
9819
12377
  }
12378
+ /** Transcribe recorded audio through the dedicated transcription registry.
12379
+ * This operation never enters the chat history or chat-provider path. */
12380
+ async transcribe(request) {
12381
+ const startedAt = Date.now();
12382
+ const transcriptionId = randomUUID();
12383
+ const signal = this.currentSignal;
12384
+ const rawAudio = request && typeof request === "object" && "audio" in request ? request.audio : void 0;
12385
+ const source = rawAudio?.type;
12386
+ const requestedUri = typeof request?.model === "string" ? request.model : void 0;
12387
+ const colon = requestedUri?.indexOf(":") ?? -1;
12388
+ const requestedProvider = colon > 0 ? requestedUri.slice(0, colon) : void 0;
12389
+ const requestedModel = colon > 0 && colon < requestedUri.length - 1 ? requestedUri.slice(colon + 1) : void 0;
12390
+ const startAudio = {
12391
+ source: typeof source === "string" ? source : "unknown",
12392
+ ...rawAudio?.data instanceof Uint8Array ? { bytes: rawAudio.data.byteLength } : {},
12393
+ ...typeof rawAudio?.mediaType === "string" ? { mediaType: rawAudio.mediaType } : {}
12394
+ };
12395
+ this.emitEvent({
12396
+ type: "transcription_start",
12397
+ transcriptionId,
12398
+ model: requestedModel,
12399
+ data: {
12400
+ ...requestedProvider ? { provider: requestedProvider } : {},
12401
+ ...requestedModel ? { model: requestedModel } : {},
12402
+ audio: startAudio
12403
+ }
12404
+ });
12405
+ let terminalEmitted = false;
12406
+ const emitTerminal = (partial) => {
12407
+ if (terminalEmitted) return;
12408
+ terminalEmitted = true;
12409
+ this.emitEvent({
12410
+ type: "transcription_end",
12411
+ transcriptionId,
12412
+ model: partial.model,
12413
+ duration: Date.now() - startedAt,
12414
+ ...partial.cost !== void 0 ? { cost: partial.cost } : {},
12415
+ ...partial.tokens ? { tokens: partial.tokens } : {},
12416
+ data: partial.data
12417
+ });
12418
+ };
12419
+ const abortError3 = () => new DOMException("The transcription operation was aborted", "AbortError");
12420
+ const cleanupStatus = (value) => value === "not_required" || value === "deleted" || value === "failed" || value === "timed_out" ? value : void 0;
12421
+ const safeProviderFailure = (error, provider, model) => {
12422
+ const details = error && typeof error === "object" ? error : void 0;
12423
+ const accounting = normalizeTranscriptionAccounting(details?.usage, details?.pricingStatus);
12424
+ const cleanup = cleanupStatus(details?.cleanupStatus);
12425
+ const providerError = error instanceof ProviderError ? error : void 0;
12426
+ const diagnostics = providerError ? {
12427
+ status: providerError.status,
12428
+ retryable: providerError.retryable,
12429
+ ...providerError.retryAfterMs !== void 0 ? { retryAfterMs: providerError.retryAfterMs } : {},
12430
+ ...providerError.requestId ? { requestId: providerError.requestId } : {}
12431
+ } : void 0;
12432
+ return {
12433
+ accounting,
12434
+ cleanup,
12435
+ diagnostics,
12436
+ error: new TranscriptionOperationError({
12437
+ provider,
12438
+ model,
12439
+ ...accounting.usage ? { usage: accounting.usage } : {},
12440
+ ...accounting.pricingStatus ? { pricingStatus: accounting.pricingStatus } : {},
12441
+ ...cleanup ? { cleanupStatus: cleanup } : {},
12442
+ ...diagnostics ?? {},
12443
+ cause: error
12444
+ })
12445
+ };
12446
+ };
12447
+ const terminalData = (status, provider, model, audio, accounting, extras = {}) => ({
12448
+ status,
12449
+ ...provider ? { provider } : {},
12450
+ ...model ? { model } : {},
12451
+ ...audio ? { audio } : {},
12452
+ ...accounting.usage ? { usage: accounting.usage } : {},
12453
+ ...accounting.pricingStatus ? { pricingStatus: accounting.pricingStatus } : {},
12454
+ ...extras
12455
+ });
12456
+ try {
12457
+ const normalized = normalizeTranscriptionRequest(request);
12458
+ const descriptor = {
12459
+ source: normalized.audio.type,
12460
+ ...normalized.audio.type === "bytes" ? { bytes: normalized.audio.data.byteLength } : {},
12461
+ ...normalized.audio.mediaType !== void 0 ? { mediaType: normalized.audio.mediaType } : {}
12462
+ };
12463
+ if (this.currentSignal?.aborted || signal?.aborted) {
12464
+ emitTerminal({
12465
+ model: requestedModel,
12466
+ data: terminalData("aborted", requestedProvider, requestedModel, descriptor, {
12467
+ hasWork: false
12468
+ })
12469
+ });
12470
+ throw abortError3();
12471
+ }
12472
+ if (this.budgetContext?.exceeded) {
12473
+ const { limit, totalCost: spent, policy } = this.budgetContext;
12474
+ if (policy === "warn") {
12475
+ this.emitEvent({
12476
+ type: "log",
12477
+ data: { warning: "Budget exceeded", limit, spent, policy }
12478
+ });
12479
+ } else {
12480
+ throw new BudgetExceededError(limit, spent, policy);
12481
+ }
12482
+ }
12483
+ if (!this.transcriptionProviderRegistry) {
12484
+ throw new InvalidTranscriptionInputError(
12485
+ "No transcription provider registry is configured"
12486
+ );
12487
+ }
12488
+ const resolved = this.transcriptionProviderRegistry.resolve(normalized.model, this.config);
12489
+ if (normalized.audio.type === "provider-file" && normalized.audio.provider !== resolved.providerName) {
12490
+ throw new InvalidTranscriptionInputError(
12491
+ "Transcription provider-file source does not match the requested provider"
12492
+ );
12493
+ }
12494
+ const capabilities = resolved.provider.capabilities(resolved.model);
12495
+ if (!capabilities || !capabilities.sources.includes(normalized.audio.type)) {
12496
+ throw new InvalidTranscriptionInputError(
12497
+ "The requested transcription model does not support this audio source"
12498
+ );
12499
+ }
12500
+ if (normalized.timestamps && !capabilities.timestamps?.includes(normalized.timestamps)) {
12501
+ throw new InvalidTranscriptionInputError(
12502
+ "The requested transcription model does not support requested timestamps"
12503
+ );
12504
+ }
12505
+ if (normalized.diarization && !capabilities.diarization) {
12506
+ throw new InvalidTranscriptionInputError(
12507
+ "The requested transcription model does not support diarization"
12508
+ );
12509
+ }
12510
+ const providerRequest = {
12511
+ ...normalized,
12512
+ model: resolved.model,
12513
+ ...signal ? { signal } : {}
12514
+ };
12515
+ let result;
12516
+ try {
12517
+ result = await resolved.provider.transcribe(providerRequest);
12518
+ } catch (error) {
12519
+ const failure = safeProviderFailure(error, resolved.providerName, resolved.model);
12520
+ if (this.currentSignal?.aborted || signal?.aborted) {
12521
+ if (failure.accounting.cost !== void 0)
12522
+ this._accumulateBudgetCost(failure.accounting.cost);
12523
+ emitTerminal({
12524
+ model: resolved.model,
12525
+ ...failure.accounting.cost !== void 0 ? { cost: failure.accounting.cost } : {},
12526
+ ...failure.accounting.tokens ? { tokens: failure.accounting.tokens } : {},
12527
+ data: terminalData(
12528
+ "aborted",
12529
+ resolved.providerName,
12530
+ resolved.model,
12531
+ descriptor,
12532
+ failure.accounting,
12533
+ ...failure.cleanup ? [{ cleanupStatus: failure.cleanup }] : []
12534
+ )
12535
+ });
12536
+ throw abortError3();
12537
+ }
12538
+ if (failure.accounting.cost !== void 0)
12539
+ this._accumulateBudgetCost(failure.accounting.cost);
12540
+ emitTerminal({
12541
+ model: resolved.model,
12542
+ ...failure.accounting.cost !== void 0 ? { cost: failure.accounting.cost } : {},
12543
+ ...failure.accounting.tokens ? { tokens: failure.accounting.tokens } : {},
12544
+ data: terminalData(
12545
+ "failed",
12546
+ resolved.providerName,
12547
+ resolved.model,
12548
+ descriptor,
12549
+ failure.accounting,
12550
+ {
12551
+ error: "Transcription operation failed",
12552
+ errorCode: failure.error.code,
12553
+ ...failure.diagnostics ? { providerError: failure.diagnostics } : {},
12554
+ ...failure.cleanup ? { cleanupStatus: failure.cleanup } : {}
12555
+ }
12556
+ )
12557
+ });
12558
+ throw failure.error;
12559
+ }
12560
+ const accounting = normalizeTranscriptionAccounting(
12561
+ result.transcript?.usage,
12562
+ result.transcript?.pricingStatus
12563
+ );
12564
+ if (accounting.cost !== void 0) this._accumulateBudgetCost(accounting.cost);
12565
+ if (this.currentSignal?.aborted || signal?.aborted) {
12566
+ emitTerminal({
12567
+ model: resolved.model,
12568
+ ...accounting.cost !== void 0 ? { cost: accounting.cost } : {},
12569
+ ...accounting.tokens ? { tokens: accounting.tokens } : {},
12570
+ data: terminalData(
12571
+ "aborted",
12572
+ resolved.providerName,
12573
+ resolved.model,
12574
+ descriptor,
12575
+ accounting,
12576
+ ...cleanupStatus(result.cleanupStatus) ? [{ cleanupStatus: cleanupStatus(result.cleanupStatus) }] : []
12577
+ )
12578
+ });
12579
+ throw abortError3();
12580
+ }
12581
+ let transcript;
12582
+ try {
12583
+ transcript = cloneTranscript(result.transcript);
12584
+ } catch (error) {
12585
+ const cleanup = cleanupStatus(result.cleanupStatus);
12586
+ const failure = new TranscriptionOperationError({
12587
+ provider: resolved.providerName,
12588
+ model: resolved.model,
12589
+ ...accounting.usage ? { usage: accounting.usage } : {},
12590
+ ...accounting.pricingStatus ? { pricingStatus: accounting.pricingStatus } : {},
12591
+ ...cleanup ? { cleanupStatus: cleanup } : {},
12592
+ cause: error
12593
+ });
12594
+ emitTerminal({
12595
+ model: resolved.model,
12596
+ ...accounting.cost !== void 0 ? { cost: accounting.cost } : {},
12597
+ ...accounting.tokens ? { tokens: accounting.tokens } : {},
12598
+ data: terminalData(
12599
+ "failed",
12600
+ resolved.providerName,
12601
+ resolved.model,
12602
+ descriptor,
12603
+ accounting,
12604
+ {
12605
+ error: "Transcription operation failed",
12606
+ errorCode: "TRANSCRIPTION_PROVIDER_ERROR",
12607
+ ...cleanup ? { cleanupStatus: cleanup } : {}
12608
+ }
12609
+ )
12610
+ });
12611
+ throw failure;
12612
+ }
12613
+ emitTerminal({
12614
+ model: resolved.model,
12615
+ ...accounting.cost !== void 0 ? { cost: accounting.cost } : {},
12616
+ ...accounting.tokens ? { tokens: accounting.tokens } : {},
12617
+ data: terminalData(
12618
+ "completed",
12619
+ resolved.providerName,
12620
+ resolved.model,
12621
+ descriptor,
12622
+ accounting,
12623
+ {
12624
+ text: transcript.text,
12625
+ ...cleanupStatus(result.cleanupStatus) ? { cleanupStatus: cleanupStatus(result.cleanupStatus) } : {}
12626
+ }
12627
+ )
12628
+ });
12629
+ return transcript;
12630
+ } catch (error) {
12631
+ if (!terminalEmitted) {
12632
+ const aborted = signal?.aborted || error instanceof Error && error.name === "AbortError";
12633
+ emitTerminal({
12634
+ model: requestedModel,
12635
+ data: terminalData(
12636
+ aborted ? "aborted" : "failed",
12637
+ requestedProvider,
12638
+ requestedModel,
12639
+ void 0,
12640
+ { hasWork: false },
12641
+ {
12642
+ error: aborted ? "Transcription operation aborted" : "Transcription operation failed"
12643
+ }
12644
+ )
12645
+ });
12646
+ }
12647
+ throw error;
12648
+ }
12649
+ }
9820
12650
  // ── Private ───────────────────────────────────────────────────────────
9821
12651
  /**
9822
12652
  * Internal emitter input — intentionally loose so call sites don't need to
@@ -9832,7 +12662,7 @@ ${summaryResponse.content}`
9832
12662
  const cost = partial.cost;
9833
12663
  if (COST_BEARING_LEAF_TYPES.includes(partial.type)) {
9834
12664
  const usable = isUsableCost(cost);
9835
- const unpriced = !usable && hasPositiveTokens(
12665
+ const unpriced = !usable && hasPositiveBillableWork(
9836
12666
  partial
9837
12667
  );
9838
12668
  if (frame) {
@@ -9937,11 +12767,13 @@ function agent(config) {
9937
12767
  _name: defaultName,
9938
12768
  async ask(prompt, options) {
9939
12769
  const registry = new ProviderRegistry();
12770
+ const transcriptionRegistry = new TranscriptionProviderRegistry();
9940
12771
  const ctx = new WorkflowContext({
9941
12772
  input: prompt,
9942
12773
  executionId: randomUUID2(),
9943
12774
  config: {},
9944
- providerRegistry: registry
12775
+ providerRegistry: registry,
12776
+ transcriptionProviderRegistry: transcriptionRegistry
9945
12777
  });
9946
12778
  return ctx.ask(agentInstance, prompt, options);
9947
12779
  },
@@ -10001,7 +12833,7 @@ var MemoryStore = class {
10001
12833
  return data !== void 0 ? structuredClone(data) : null;
10002
12834
  }
10003
12835
  async saveSession(sessionId, history) {
10004
- this.sessions.set(sessionId, structuredClone(history));
12836
+ this.sessions.set(sessionId, structuredClone(normalizePersistedSessionHistory(history)));
10005
12837
  }
10006
12838
  async getSession(sessionId) {
10007
12839
  const history = this.sessions.get(sessionId);
@@ -10425,7 +13257,7 @@ var SQLiteStore = class {
10425
13257
  const stmt = this.db.prepare(
10426
13258
  "INSERT OR REPLACE INTO sessions (session_id, history) VALUES (?, ?)"
10427
13259
  );
10428
- stmt.run(sessionId, JSON.stringify(history));
13260
+ stmt.run(sessionId, JSON.stringify(normalizePersistedSessionHistory(history)));
10429
13261
  }
10430
13262
  async getSession(sessionId) {
10431
13263
  const stmt = this.db.prepare("SELECT history FROM sessions WHERE session_id = ?");
@@ -10993,7 +13825,7 @@ var AxlStream = class extends Readable {
10993
13825
  /** Signal an error. `executionId` is required for the same reason as
10994
13826
  * `_done`: terminal events must carry a real id even when the
10995
13827
  * failure happens before any real trace event fires (review S4). */
10996
- _error(error, executionId) {
13828
+ _error(error, executionId, observerMessage = error.message, observerName = error.name) {
10997
13829
  if (this.events.isFinished) return;
10998
13830
  const errorEvent = {
10999
13831
  schemaVersion: 2,
@@ -11001,7 +13833,7 @@ var AxlStream = class extends Readable {
11001
13833
  executionId,
11002
13834
  step: Number.MAX_SAFE_INTEGER,
11003
13835
  timestamp: Date.now(),
11004
- data: { message: error.message, name: error.name }
13836
+ data: { message: observerMessage, name: observerName }
11005
13837
  };
11006
13838
  this.events._push(errorEvent);
11007
13839
  this.push(errorEvent);
@@ -11728,6 +14560,10 @@ var StreamingFlusher = class {
11728
14560
  function isAbortError4(err) {
11729
14561
  return typeof err === "object" && err !== null && err.name === "AbortError";
11730
14562
  }
14563
+ function executionErrorProjection(ctx, error) {
14564
+ const safe = ctx?._observerErrorProjection(error);
14565
+ return safe ?? { message: error instanceof Error ? error.message : String(error) };
14566
+ }
11731
14567
  function pushEventBounded(execInfo, event, cap) {
11732
14568
  if (STREAMING_EXCLUDED_TYPES.has(event.type)) return;
11733
14569
  if (cap === Infinity || execInfo.events.length < cap) {
@@ -11822,6 +14658,7 @@ var AxlRuntime = class extends EventEmitter2 {
11822
14658
  tools = /* @__PURE__ */ new Map();
11823
14659
  agents = /* @__PURE__ */ new Map();
11824
14660
  providerRegistry;
14661
+ transcriptionProviderRegistry;
11825
14662
  stateStore;
11826
14663
  executions = /* @__PURE__ */ new Map();
11827
14664
  pendingDecisionResolvers = /* @__PURE__ */ new Map();
@@ -11877,6 +14714,7 @@ var AxlRuntime = class extends EventEmitter2 {
11877
14714
  super();
11878
14715
  this.config = resolveConfig(config ?? {});
11879
14716
  this.providerRegistry = new ProviderRegistry();
14717
+ this.transcriptionProviderRegistry = new TranscriptionProviderRegistry();
11880
14718
  this.stateStore = this.createStateStore();
11881
14719
  const requested = this.config.state?.maxEventsPerExecution;
11882
14720
  if (requested === void 0) {
@@ -12025,7 +14863,7 @@ var AxlRuntime = class extends EventEmitter2 {
12025
14863
  execInfo.status = "failed";
12026
14864
  execInfo.completedAt = Date.now();
12027
14865
  execInfo.duration = execInfo.completedAt - execInfo.startedAt;
12028
- execInfo.error = terminalError instanceof Error ? terminalError.message : String(terminalError);
14866
+ execInfo.error = executionErrorProjection(ctx, terminalError).message;
12029
14867
  execInfo.observation = observation;
12030
14868
  ctx._emitWorkflowEnd({
12031
14869
  status: "failed",
@@ -12280,6 +15118,7 @@ var AxlRuntime = class extends EventEmitter2 {
12280
15118
  metadata: options?.metadata,
12281
15119
  config: this.config,
12282
15120
  providerRegistry: this.providerRegistry,
15121
+ transcriptionProviderRegistry: this.transcriptionProviderRegistry,
12283
15122
  stateStore: this.stateStore,
12284
15123
  mcpManager: this.mcpManager,
12285
15124
  spanManager: this.spanManager,
@@ -12308,6 +15147,10 @@ var AxlRuntime = class extends EventEmitter2 {
12308
15147
  registerProvider(name, provider) {
12309
15148
  this.providerRegistry.registerInstance(name, provider);
12310
15149
  }
15150
+ /** Register a dedicated transcription adapter. It is never used for chat. */
15151
+ registerTranscriptionProvider(name, provider) {
15152
+ this.transcriptionProviderRegistry.registerInstance(name, provider);
15153
+ }
12311
15154
  /** Resolve a provider:model URI to a Provider instance and model name. */
12312
15155
  resolveProvider(uri) {
12313
15156
  return this.providerRegistry.resolve(uri, this.config);
@@ -12351,6 +15194,7 @@ var AxlRuntime = class extends EventEmitter2 {
12351
15194
  metadata: options?.metadata,
12352
15195
  config: this.config,
12353
15196
  providerRegistry: this.providerRegistry,
15197
+ transcriptionProviderRegistry: this.transcriptionProviderRegistry,
12354
15198
  sessionHistory,
12355
15199
  signal: controller.signal,
12356
15200
  eventStreamOptions: options?.events,
@@ -12438,6 +15282,7 @@ var AxlRuntime = class extends EventEmitter2 {
12438
15282
  const executionId = randomUUID3();
12439
15283
  this.abortControllers.set(executionId, controller);
12440
15284
  let execInfo;
15285
+ let wfCtx;
12441
15286
  const run = async () => {
12442
15287
  const workflow2 = this.workflows.get(name);
12443
15288
  if (!workflow2) throw new Error(`Workflow "${name}" not registered`);
@@ -12461,12 +15306,13 @@ var AxlRuntime = class extends EventEmitter2 {
12461
15306
  metadata: liftPersistedMetadata(options?.metadata)
12462
15307
  };
12463
15308
  this.executions.set(executionId, execInfo);
12464
- const wfCtx = new WorkflowContext({
15309
+ const context = new WorkflowContext({
12465
15310
  input: validated,
12466
15311
  executionId,
12467
15312
  metadata: options?.metadata,
12468
15313
  config: this.config,
12469
15314
  providerRegistry: this.providerRegistry,
15315
+ transcriptionProviderRegistry: this.transcriptionProviderRegistry,
12470
15316
  sessionHistory,
12471
15317
  signal: controller.signal,
12472
15318
  eventStreamOptions: options?.events,
@@ -12531,6 +15377,7 @@ var AxlRuntime = class extends EventEmitter2 {
12531
15377
  unpricedWarned: false
12532
15378
  }
12533
15379
  });
15380
+ wfCtx = context;
12534
15381
  return this.spanManager.withSpanAsync(
12535
15382
  "axl.workflow.execute",
12536
15383
  {
@@ -12540,7 +15387,7 @@ var AxlRuntime = class extends EventEmitter2 {
12540
15387
  },
12541
15388
  (span) => this.runWorkflowBody({
12542
15389
  workflow: workflow2,
12543
- ctx: wfCtx,
15390
+ ctx: context,
12544
15391
  execInfo,
12545
15392
  validated,
12546
15393
  span,
@@ -12553,12 +15400,14 @@ var AxlRuntime = class extends EventEmitter2 {
12553
15400
  execInfo.status = "failed";
12554
15401
  execInfo.completedAt = Date.now();
12555
15402
  execInfo.duration = execInfo.completedAt - execInfo.startedAt;
12556
- execInfo.error = err instanceof Error ? err.message : String(err);
15403
+ execInfo.error = executionErrorProjection(wfCtx, err).message;
12557
15404
  this.persistExecution(execInfo);
12558
15405
  }
12559
15406
  this.abortControllers.delete(executionId);
12560
15407
  cleanupAbortForwarder();
12561
- axlStream._error(err instanceof Error ? err : new Error(String(err)), executionId);
15408
+ const originalError = err instanceof Error ? err : new Error(String(err));
15409
+ const projection = executionErrorProjection(wfCtx, err);
15410
+ axlStream._error(originalError, executionId, projection.message, projection.name);
12562
15411
  });
12563
15412
  return axlStream;
12564
15413
  }
@@ -12969,7 +15818,7 @@ var AxlRuntime = class extends EventEmitter2 {
12969
15818
  },
12970
15819
  {
12971
15820
  role: "user",
12972
- content: messages.map((m) => `${m.role}: ${m.content}`).join("\n")
15821
+ content: messages.map((m) => `${m.role}: ${summarizeModelInput(m.content)}`).join("\n")
12973
15822
  }
12974
15823
  ],
12975
15824
  { model, maxTokens: 1024 }
@@ -13416,7 +16265,7 @@ async function registerConditions(conditions) {
13416
16265
  ].join("\n");
13417
16266
  nodeModule.register(`data:text/javascript,${encodeURIComponent(hookCode)}`);
13418
16267
  } catch {
13419
- console.warn("Warning: --conditions requires Node.js 20.6+");
16268
+ console.warn("Warning: this Node.js runtime does not support --conditions");
13420
16269
  }
13421
16270
  }
13422
16271
 
@@ -13725,7 +16574,11 @@ var RedisStore = class _RedisStore {
13725
16574
  async saveSession(sessionId, history) {
13726
16575
  const ttl = this.ttlFor("session");
13727
16576
  const setOptions = ttl !== void 0 ? { EX: ttl } : void 0;
13728
- await this.client.multi().set(this.sessionKey(sessionId), JSON.stringify(history), setOptions).sAdd(this.sessionIdsKey(), sessionId).exec();
16577
+ await this.client.multi().set(
16578
+ this.sessionKey(sessionId),
16579
+ JSON.stringify(normalizePersistedSessionHistory(history)),
16580
+ setOptions
16581
+ ).sAdd(this.sessionIdsKey(), sessionId).exec();
13729
16582
  }
13730
16583
  async getSession(sessionId) {
13731
16584
  const raw = await this.client.get(this.sessionKey(sessionId));
@@ -14213,10 +17066,15 @@ export {
14213
17066
  EventStreamOverflowError,
14214
17067
  GROQ_PROFILE,
14215
17068
  GeminiProvider,
17069
+ GeminiTranscriptionProvider,
14216
17070
  GuardrailError,
14217
17071
  InMemoryVectorStore,
17072
+ InvalidModelInputError,
17073
+ InvalidTranscriptionInputError,
14218
17074
  LLAMACPP_PROFILE,
14219
17075
  LMSTUDIO_PROFILE,
17076
+ MAX_INLINE_MODEL_INPUT_BYTES,
17077
+ MAX_INLINE_TRANSCRIPTION_BYTES,
14220
17078
  MISTRAL_PROFILE,
14221
17079
  MaxTurnsError,
14222
17080
  McpManager,
@@ -14232,6 +17090,8 @@ export {
14232
17090
  OpenAIEmbedder,
14233
17091
  OpenAIProvider,
14234
17092
  OpenAIResponsesProvider,
17093
+ OpenAITranscriptionProvider,
17094
+ OpenRouterTranscriptionProvider,
14235
17095
  ProviderError,
14236
17096
  ProviderRegistry,
14237
17097
  QuorumNotMet,
@@ -14247,7 +17107,11 @@ export {
14247
17107
  TimeoutError,
14248
17108
  ToolFailure,
14249
17109
  ToolModelOutputError,
17110
+ TranscriptionOperationError,
17111
+ TranscriptionProviderRegistry,
14250
17112
  UnsupportedEventSchemaVersionError,
17113
+ UnsupportedModelInputError,
17114
+ UnsupportedTranscriptionInputError,
14251
17115
  VLLM_PROFILE,
14252
17116
  ValidationError,
14253
17117
  VerifyError,
@@ -14264,8 +17128,10 @@ export {
14264
17128
  findConfig,
14265
17129
  getEventSchemaVersion,
14266
17130
  getExecutionEventSchemaVersion,
17131
+ hasPositiveBillableWork,
14267
17132
  hasPositiveTokens,
14268
17133
  importModule,
17134
+ inputText,
14269
17135
  isCostBearingLeaf,
14270
17136
  isRetryableStatus,
14271
17137
  isRootLevel,