@zhivex-ai/openai 0.13.6 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,10 +1,14 @@
1
+ import { jsonHeaders, parseJson, RESERVED_REQUEST_HEADERS } from "./http.js";
2
+ import { OpenAIRealtimeModel, openAIRealtimeMcpMetadataKey, resolveOpenAIRealtimeHeaders } from "./realtime.js";
3
+ export { openAIRealtimeMcpMetadataKey } from "./realtime.js";
4
+ import { OpenAIEmbeddingModel, OpenAITranscriptionModel, OpenAISpeechModel, toUint8Array } from "./resource-models.js";
1
5
  import { OpenAIAgentsClient } from "./agents.js";
2
6
  export { OpenAIAgentsClient } from "./agents.js";
3
7
  import { toJSONSchema, z } from "zod";
4
8
  import { OpenAILiveModel, isOpenAILiveModel } from "./live.js";
5
9
  import { createOpenAIImageGenerationModel, normalizeOpenAIImageGenerationCall, normalizeOpenAIImageGenerationPartialImage } from "./image-generation.js";
6
- import { capabilities, groundedCapabilities, inferOpenAIRealtimeMode, supportsOpenAIModernResponses, isOpenAIAstraModel, isOpenAISolLunaModel, modelCapabilities, openAIRealtimeSupportsImageInput, realtimeCapabilities, speechCapabilities, supportsOpenAIApplyPatchAndSkills, supportsOpenAIShell, transcriptionCapabilities, } from "./capabilities.js";
7
- import { imageInputToDataUrl, CallbackRealtimeSession, ConfigurationError, ProviderHTTPError, ProviderResponseTooLargeError, ProviderToolCallError, assertTrustedEndpoint, decodeBase64WithLimit, readBodyWithLimit, readErrorBodyWithLimit, readJsonWithLimit, resolveAudioResponseLimits, openWebSocketConnection, hostedTool, providerDataPart, UnsupportedFeatureError, createProviderAdapter, encodeAudioFrame, encodeMediaFrame, isCallableToolDefinition, isHostedToolDefinition, normalizeFinishReason, streamSSE, toToolSet, toolResultPayload, withRetry, withResponseRetry, withTimeoutSignal } from "@zhivex-ai/core/provider";
10
+ import { groundedCapabilities, supportsOpenAIModernResponses, assertOpenAIModelRequestSupported, resolveOpenAIModelProfile, modelCapabilities } from "./capabilities.js";
11
+ import { imageInputToDataUrl, ConfigurationError, ProviderHTTPError, ProviderResponseTooLargeError, ProviderToolCallError, assertTrustedEndpoint, resolveAudioResponseLimits, openWebSocketConnection, hostedTool, providerDataPart, UnsupportedFeatureError, createProviderAdapter, isCallableToolDefinition, isHostedToolDefinition, normalizeFinishReason, streamSSE, streamChatCompletions, toolResultPayload, withRetry, withResponseRetry, withTimeoutSignal } from "@zhivex-ai/core/provider";
8
12
  const MIB = 1024 * 1024;
9
13
  const DEFAULT_HOSTED_IMAGE_EVENT_BYTES = 32 * MIB;
10
14
  const DEFAULT_HOSTED_IMAGE_TOTAL_BYTES = 128 * MIB;
@@ -59,42 +63,6 @@ const assertOpenAIToolPathInsideRoot = async (rootDir, targetPath, label) => {
59
63
  }
60
64
  return target;
61
65
  };
62
- export const openAIRealtimeMcpMetadataKey = "openai.realtime_mcp";
63
- const jsonHeaders = (apiKey) => ({
64
- "content-type": "application/json",
65
- authorization: `Bearer ${apiKey}`
66
- });
67
- const OPENAI_JSON_RESPONSE_MAX_BYTES = 128 * 1024 * 1024;
68
- const OPENAI_REALTIME_AUDIO_MAX_BYTES = 16 * 1024 * 1024;
69
- const RESERVED_REQUEST_HEADERS = new Set([
70
- "authorization",
71
- "content-type",
72
- "content-length",
73
- "host",
74
- "connection",
75
- "transfer-encoding"
76
- ]);
77
- const resolveOpenAIRealtimeHeaders = (apiKey, providerOptions, includeContentType = false) => {
78
- const rawHeaders = providerOptions?.headers;
79
- const customHeaders = rawHeaders && typeof rawHeaders === "object" && !Array.isArray(rawHeaders)
80
- ? { ...rawHeaders }
81
- : {};
82
- const safetyIdentifier = providerOptions?.safety_identifier;
83
- for (const key of Object.keys(customHeaders)) {
84
- const normalizedKey = key.toLowerCase();
85
- if (RESERVED_REQUEST_HEADERS.has(normalizedKey) ||
86
- (typeof safetyIdentifier === "string" && safetyIdentifier && normalizedKey === "openai-safety-identifier")) {
87
- delete customHeaders[key];
88
- }
89
- }
90
- return {
91
- ...(includeContentType ? jsonHeaders(apiKey) : { authorization: `Bearer ${apiKey}` }),
92
- ...customHeaders,
93
- ...(typeof safetyIdentifier === "string" && safetyIdentifier
94
- ? { "OpenAI-Safety-Identifier": safetyIdentifier }
95
- : {})
96
- };
97
- };
98
66
  const resolveOpenAILanguageRequestOptions = (providerOptions, apiKey) => {
99
67
  const bodyOptions = { ...(providerOptions ?? {}) };
100
68
  const apiMode = bodyOptions.apiMode ?? "auto";
@@ -139,22 +107,7 @@ const openAIResponsesBodyOptions = (options, modelId) => options.store === false
139
107
  }
140
108
  : options;
141
109
  const getRequestOptions = (input) => withTimeoutSignal(input);
142
- const toUint8Array = (data) => {
143
- if (data instanceof Uint8Array) {
144
- return data;
145
- }
146
- if (data instanceof ArrayBuffer) {
147
- return new Uint8Array(data);
148
- }
149
- return Uint8Array.from(Buffer.from(data, "base64"));
150
- };
151
110
  const toBase64 = (data) => typeof data === "string" ? data : Buffer.from(data instanceof Uint8Array ? data : new Uint8Array(data)).toString("base64");
152
- const createAudioFile = (audio) => {
153
- const bytes = toUint8Array(audio.data);
154
- return new File([bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength)], audio.filename ?? "audio", {
155
- type: audio.mediaType
156
- });
157
- };
158
111
  const inferOpenAIAudioFormat = (mediaType, explicitFormat) => {
159
112
  if (explicitFormat) {
160
113
  return explicitFormat;
@@ -180,20 +133,6 @@ const inferOpenAIAudioFormat = (mediaType, explicitFormat) => {
180
133
  }
181
134
  return normalized?.replace(/^audio\//, "") || mediaType;
182
135
  };
183
- const parseJson = async (response, options = {}) => {
184
- if (!response.ok) {
185
- const body = await readErrorBodyWithLimit(response, options.errorBodyBytes);
186
- throw new ProviderHTTPError(`OpenAI request failed with status ${response.status}.`, response.status, {
187
- responseBody: body
188
- });
189
- }
190
- return readJsonWithLimit(response, {
191
- maxBytes: options.maxBytes ?? OPENAI_JSON_RESPONSE_MAX_BYTES,
192
- provider: options.provider ?? "openai",
193
- endpoint: options.endpoint,
194
- abort: options.abort
195
- });
196
- };
197
136
  const isOpenAIPromptCacheBreakpointPart = (part) => part.type === "provider-data" &&
198
137
  part.provider === "openai" &&
199
138
  part.data !== null &&
@@ -331,8 +270,7 @@ const responsesImageGenerationConfig = (tools, limits) => {
331
270
  : {})
332
271
  };
333
272
  };
334
- const assertResponsesToolsSupported = (modelId, tools) => {
335
- const currentCapabilities = modelCapabilities(modelId).agentCapabilities;
273
+ const assertResponsesToolsSupported = (modelId, tools, currentCapabilities) => {
336
274
  for (const definition of Object.values(tools ?? {})) {
337
275
  const type = isCallableToolDefinition(definition) ? openAILocalResponsesToolType(definition) : definition.type;
338
276
  if (type === "tool_search" && !currentCapabilities?.toolSearch) {
@@ -344,22 +282,22 @@ const assertResponsesToolsSupported = (modelId, tools) => {
344
282
  if (type === "computer" && !currentCapabilities?.computerUse) {
345
283
  throw new UnsupportedFeatureError(`Provider "openai" model "${modelId}" does not support the Responses computer tool.`);
346
284
  }
347
- if (type === "shell" && !supportsOpenAIShell(modelId)) {
285
+ if (type === "shell" && !currentCapabilities?.shell) {
348
286
  throw new UnsupportedFeatureError(`Provider "openai" model "${modelId}" does not support the Responses ${type} tool.`);
349
287
  }
350
- if (type === "apply_patch" && !supportsOpenAIApplyPatchAndSkills(modelId)) {
288
+ if (type === "apply_patch" && !currentCapabilities?.applyPatch) {
351
289
  throw new UnsupportedFeatureError(`Provider "openai" model "${modelId}" does not support the Responses ${type} tool.`);
352
290
  }
353
291
  if (type === "skill" && !currentCapabilities?.skills) {
354
292
  throw new UnsupportedFeatureError(`Provider "openai" model "${modelId}" does not support the Responses skills tool.`);
355
293
  }
356
- if (type === "programmatic_tool_calling" && !supportsOpenAIModernResponses(modelId)) {
294
+ if (type === "programmatic_tool_calling" && !currentCapabilities?.programmaticToolCalling) {
357
295
  throw new UnsupportedFeatureError(`Provider "openai" model "${modelId}" does not support Programmatic Tool Calling.`);
358
296
  }
359
297
  }
360
298
  };
361
- const assertOpenAIResponsesOptionsSupported = (modelId, options) => {
362
- if (options.multi_agent?.enabled && !supportsOpenAIModernResponses(modelId)) {
299
+ const assertOpenAIResponsesOptionsSupported = (modelId, options, currentCapabilities) => {
300
+ if (options.multi_agent?.enabled && !currentCapabilities?.multiAgent) {
363
301
  throw new UnsupportedFeatureError(`Provider "openai" model "${modelId}" does not support Multi-agent.`);
364
302
  }
365
303
  };
@@ -389,28 +327,6 @@ const mapTools = (input) => input
389
327
  };
390
328
  })
391
329
  : undefined;
392
- const mapRealtimeTools = (input) => input
393
- ? Object.values(input).map((tool) => {
394
- if (isCallableToolDefinition(tool)) {
395
- return {
396
- type: "function",
397
- name: tool.name,
398
- description: tool.description,
399
- parameters: toJSONSchema(tool.schema)
400
- };
401
- }
402
- if (tool.provider && tool.provider !== "openai") {
403
- throw new UnsupportedFeatureError(`Provider "openai" does not support hosted tools declared for provider "${tool.provider}".`);
404
- }
405
- if (tool.type !== "mcp") {
406
- throw new UnsupportedFeatureError(`Provider "openai" Realtime does not support the hosted tool type "${tool.type}". Use function or MCP tools.`);
407
- }
408
- return {
409
- type: tool.type,
410
- ...(tool.config && typeof tool.config === "object" ? tool.config : {})
411
- };
412
- })
413
- : undefined;
414
330
  const mapResponsesTools = (input) => input
415
331
  ? Object.values(input).map((tool) => {
416
332
  if (isCallableToolDefinition(tool)) {
@@ -461,359 +377,6 @@ const mapResponsesToolChoice = (toolChoice) => {
461
377
  name: toolChoice.toolName
462
378
  };
463
379
  };
464
- const mapRealtimeToolChoice = (toolChoice) => {
465
- if (!toolChoice || typeof toolChoice === "string") {
466
- return toolChoice;
467
- }
468
- return {
469
- type: "function",
470
- name: toolChoice.toolName
471
- };
472
- };
473
- const mapRealtimeProviderOptions = (providerOptions) => {
474
- if (!providerOptions) {
475
- return {};
476
- }
477
- return Object.fromEntries(Object.entries(providerOptions).filter(([key]) => !["headers", "safety_identifier", "realtime_url", "realtime_query", "expires_after"].includes(key)));
478
- };
479
- const mapRealtimeAudioFormat = (mediaType, sampleRateHz) => mediaType || sampleRateHz
480
- ? {
481
- ...(mediaType ? { type: mediaType } : {}),
482
- ...(sampleRateHz ? { rate: sampleRateHz } : {})
483
- }
484
- : undefined;
485
- const mapRealtimeSessionConfig = (config, modelId) => {
486
- const mode = inferOpenAIRealtimeMode(modelId ?? "", config.mode);
487
- const tools = mapRealtimeTools(toToolSet(config.tools));
488
- const audio = {
489
- input: {
490
- format: mapRealtimeAudioFormat(config.inputAudioMediaType, config.inputSampleRateHz),
491
- transcription: mode === "transcription" || config.inputTranscription
492
- ? {
493
- model: config.inputTranscription?.model ?? (mode === "transcription" ? modelId : undefined),
494
- language: config.inputTranscription?.language,
495
- prompt: config.inputTranscription?.prompt,
496
- delay: config.inputTranscription?.delay
497
- }
498
- : undefined,
499
- noise_reduction: config.noiseReduction ?? undefined,
500
- turn_detection: config.turnDetection ?? undefined
501
- },
502
- ...(mode === "transcription"
503
- ? {}
504
- : {
505
- output: {
506
- format: mapRealtimeAudioFormat(config.outputAudioMediaType, config.outputSampleRateHz),
507
- voice: config.voice
508
- }
509
- })
510
- };
511
- return {
512
- type: mode === "transcription" ? "transcription" : "realtime",
513
- model: modelId,
514
- instructions: config.translation?.instructions ?? config.instructions,
515
- output_modalities: mode === "transcription" ? undefined : config.outputAudioMediaType || config.voice || mode === "translation" ? ["audio"] : ["text"],
516
- tools: mode === "conversation" ? tools : undefined,
517
- tool_choice: mode === "conversation" && config.toolChoice ? mapRealtimeToolChoice(config.toolChoice) : undefined,
518
- reasoning: mode === "conversation" && config.reasoning?.effort
519
- ? {
520
- effort: config.reasoning.effort
521
- }
522
- : undefined,
523
- include: config.inputTranscription?.includeLogprobs && mode === "transcription" ? ["item.input_audio_transcription.logprobs"] : undefined,
524
- translation: mode === "translation"
525
- ? {
526
- target_language: config.translation?.targetLanguage,
527
- source_language: config.translation?.sourceLanguage
528
- }
529
- : undefined,
530
- audio,
531
- ...mapRealtimeProviderOptions(config.providerOptions)
532
- };
533
- };
534
- const openAIRealtimeURL = (baseURL, modelId, mode, providerOptions) => {
535
- const override = providerOptions?.realtime_url;
536
- if (typeof override === "string" && override) {
537
- return override;
538
- }
539
- const url = new URL(baseURL);
540
- url.protocol = url.protocol === "https:" ? "wss:" : url.protocol === "http:" ? "ws:" : url.protocol;
541
- const endpoint = mode === "translation" ? "realtime/translations" : mode === "transcription" ? "realtime/transcription_sessions" : "realtime";
542
- url.pathname = `${url.pathname.replace(/\/+$/, "")}/${endpoint}`;
543
- url.searchParams.set("model", modelId);
544
- const extraQuery = providerOptions?.realtime_query;
545
- if (extraQuery && typeof extraQuery === "object" && !Array.isArray(extraQuery)) {
546
- for (const [key, value] of Object.entries(extraQuery)) {
547
- if (value != null) {
548
- url.searchParams.set(key, String(value));
549
- }
550
- }
551
- }
552
- return url.toString();
553
- };
554
- const parseRealtimeProviderMetadata = (payload) => payload;
555
- const parseRealtimeJsonValue = (value) => {
556
- if (typeof value === "string") {
557
- try {
558
- return JSON.parse(value);
559
- }
560
- catch {
561
- return value;
562
- }
563
- }
564
- return value == null ? null : value;
565
- };
566
- const openAIRealtimeMcpProviderDataEvent = (payload, data) => [
567
- {
568
- type: "realtime-provider-data",
569
- provider: "openai",
570
- data: {
571
- ...data,
572
- raw_event: payload
573
- }
574
- }
575
- ];
576
- const openAIRealtimeMcpCommonData = (payload, item) => {
577
- const itemId = typeof payload.item_id === "string" ? payload.item_id : typeof item?.id === "string" ? item.id : undefined;
578
- const serverLabel = typeof payload.server_label === "string"
579
- ? payload.server_label
580
- : typeof item?.server_label === "string"
581
- ? item.server_label
582
- : undefined;
583
- const name = typeof payload.name === "string" ? payload.name : typeof item?.name === "string" ? item.name : undefined;
584
- return {
585
- ...(itemId ? { item_id: itemId } : {}),
586
- ...(serverLabel ? { server_label: serverLabel } : {}),
587
- ...(name ? { name } : {})
588
- };
589
- };
590
- const parseOpenAIRealtimeEvent = (payload) => {
591
- const type = String(payload.type ?? "");
592
- if (type === "session.created" || type === "session.updated") {
593
- return [];
594
- }
595
- if (type === "mcp_list_tools.in_progress" || type === "mcp_list_tools.completed" || type === "mcp_list_tools.failed") {
596
- const status = type.slice("mcp_list_tools.".length);
597
- return openAIRealtimeMcpProviderDataEvent(payload, {
598
- type: "mcp_list_tools",
599
- status,
600
- ...openAIRealtimeMcpCommonData(payload),
601
- ...(payload.error !== undefined ? { error: parseRealtimeJsonValue(payload.error) } : {})
602
- });
603
- }
604
- if (type === "conversation.item.done") {
605
- const item = payload.item && typeof payload.item === "object" ? payload.item : undefined;
606
- if (item?.type === "mcp_list_tools") {
607
- return openAIRealtimeMcpProviderDataEvent(payload, {
608
- type: "mcp_list_tools",
609
- status: "completed",
610
- ...openAIRealtimeMcpCommonData(payload, item),
611
- ...(item.tools !== undefined ? { tools: parseRealtimeJsonValue(item.tools) } : {})
612
- });
613
- }
614
- if (item?.type === "mcp_approval_request") {
615
- const approvalRequestId = typeof item.id === "string" ? item.id : undefined;
616
- return openAIRealtimeMcpProviderDataEvent(payload, {
617
- type: "mcp_approval_request",
618
- status: "approval_required",
619
- ...openAIRealtimeMcpCommonData(payload, item),
620
- ...(approvalRequestId ? { approval_request_id: approvalRequestId } : {}),
621
- ...(item.arguments !== undefined ? { arguments: parseRealtimeJsonValue(item.arguments) } : {})
622
- });
623
- }
624
- if (item?.type === "mcp_approval_response") {
625
- const approve = item.approve === true;
626
- return openAIRealtimeMcpProviderDataEvent(payload, {
627
- type: "mcp_approval_response",
628
- status: approve ? "approved" : "rejected",
629
- approve,
630
- ...openAIRealtimeMcpCommonData(payload, item),
631
- ...(typeof item.approval_request_id === "string" ? { approval_request_id: item.approval_request_id } : {}),
632
- ...(typeof item.reason === "string" ? { reason: item.reason } : {})
633
- });
634
- }
635
- }
636
- if (type === "response.mcp_call_arguments.delta" || type === "response.mcp_call_arguments.done") {
637
- const isDone = type.endsWith(".done");
638
- return openAIRealtimeMcpProviderDataEvent(payload, {
639
- type: "mcp_call",
640
- status: isDone ? "arguments_done" : "arguments_delta",
641
- ...openAIRealtimeMcpCommonData(payload),
642
- ...(isDone && payload.arguments !== undefined
643
- ? { arguments: parseRealtimeJsonValue(payload.arguments) }
644
- : payload.delta !== undefined
645
- ? { delta: parseRealtimeJsonValue(payload.delta) }
646
- : {})
647
- });
648
- }
649
- if (type === "response.mcp_call.in_progress" || type === "response.mcp_call.completed" || type === "response.mcp_call.failed") {
650
- const status = type.slice("response.mcp_call.".length);
651
- return openAIRealtimeMcpProviderDataEvent(payload, {
652
- type: "mcp_call",
653
- status,
654
- ...openAIRealtimeMcpCommonData(payload),
655
- ...(payload.output !== undefined ? { output: parseRealtimeJsonValue(payload.output) } : {}),
656
- ...(payload.error !== undefined ? { error: parseRealtimeJsonValue(payload.error) } : {})
657
- });
658
- }
659
- if (type === "response.output_item.done") {
660
- const item = payload.item && typeof payload.item === "object" ? payload.item : undefined;
661
- if (item?.type === "mcp_call") {
662
- return openAIRealtimeMcpProviderDataEvent(payload, {
663
- type: "mcp_call",
664
- status: item.status === "failed" ? "failed" : "completed",
665
- ...openAIRealtimeMcpCommonData(payload, item),
666
- ...(item.arguments !== undefined ? { arguments: parseRealtimeJsonValue(item.arguments) } : {}),
667
- ...(item.output !== undefined ? { output: parseRealtimeJsonValue(item.output) } : {}),
668
- ...(item.error !== undefined ? { error: parseRealtimeJsonValue(item.error) } : {}),
669
- ...(typeof item.approval_request_id === "string" ? { approval_request_id: item.approval_request_id } : {})
670
- });
671
- }
672
- }
673
- if (type === "response.text.delta" || type === "response.output_text.delta") {
674
- return [
675
- {
676
- type: "realtime-text-delta",
677
- textDelta: String(payload.delta ?? ""),
678
- itemId: typeof payload.item_id === "string" ? payload.item_id : undefined,
679
- responseId: typeof payload.response_id === "string" ? payload.response_id : undefined,
680
- role: "assistant",
681
- providerMetadata: parseRealtimeProviderMetadata(payload)
682
- }
683
- ];
684
- }
685
- if (type === "response.audio.delta" || type === "response.output_audio.delta") {
686
- return [
687
- {
688
- type: "realtime-audio-output",
689
- audio: decodeBase64WithLimit(String(payload.delta ?? ""), {
690
- maxBytes: OPENAI_REALTIME_AUDIO_MAX_BYTES,
691
- provider: "openai",
692
- endpoint: "realtime"
693
- }),
694
- mediaType: typeof payload.media_type === "string" ? payload.media_type : "audio/pcm",
695
- sampleRateHz: typeof payload.sample_rate_hz === "number" ? payload.sample_rate_hz : undefined,
696
- channels: typeof payload.channels === "number" ? payload.channels : undefined,
697
- itemId: typeof payload.item_id === "string" ? payload.item_id : undefined,
698
- responseId: typeof payload.response_id === "string" ? payload.response_id : undefined,
699
- providerMetadata: parseRealtimeProviderMetadata(payload)
700
- }
701
- ];
702
- }
703
- if (type === "conversation.item.input_audio_transcription.delta" || type === "input_audio_buffer.transcription.delta") {
704
- return [
705
- {
706
- type: "realtime-transcript",
707
- text: String(payload.delta ?? ""),
708
- role: "user",
709
- isFinal: false,
710
- itemId: typeof payload.item_id === "string" ? payload.item_id : undefined,
711
- providerMetadata: parseRealtimeProviderMetadata(payload)
712
- }
713
- ];
714
- }
715
- if (type === "conversation.item.input_audio_transcription.completed" || type === "input_audio_buffer.transcription.completed") {
716
- return [
717
- {
718
- type: "realtime-transcript",
719
- text: String(payload.transcript ?? ""),
720
- role: "user",
721
- isFinal: true,
722
- itemId: typeof payload.item_id === "string" ? payload.item_id : undefined,
723
- providerMetadata: parseRealtimeProviderMetadata(payload)
724
- }
725
- ];
726
- }
727
- if (type === "response.audio_transcript.delta" || type === "response.audio_transcription.delta" || type === "response.output_audio_transcript.delta") {
728
- return [
729
- {
730
- type: "realtime-transcript",
731
- text: String(payload.delta ?? ""),
732
- role: "assistant",
733
- isFinal: false,
734
- itemId: typeof payload.item_id === "string" ? payload.item_id : undefined,
735
- responseId: typeof payload.response_id === "string" ? payload.response_id : undefined,
736
- providerMetadata: parseRealtimeProviderMetadata(payload)
737
- }
738
- ];
739
- }
740
- if (type === "response.audio_transcript.done" || type === "response.audio_transcription.done" || type === "response.output_audio_transcript.done") {
741
- return [
742
- {
743
- type: "realtime-transcript",
744
- text: String(payload.transcript ?? ""),
745
- role: "assistant",
746
- isFinal: true,
747
- itemId: typeof payload.item_id === "string" ? payload.item_id : undefined,
748
- responseId: typeof payload.response_id === "string" ? payload.response_id : undefined,
749
- providerMetadata: parseRealtimeProviderMetadata(payload)
750
- }
751
- ];
752
- }
753
- if (type === "response.function_call_arguments.done" || type === "response.output_item.done") {
754
- const name = typeof payload.name === "string"
755
- ? payload.name
756
- : payload.item && typeof payload.item === "object" && typeof payload.item.name === "string"
757
- ? String(payload.item.name)
758
- : undefined;
759
- const callId = typeof payload.call_id === "string"
760
- ? payload.call_id
761
- : payload.item && typeof payload.item === "object" && typeof payload.item.call_id === "string"
762
- ? String(payload.item.call_id)
763
- : undefined;
764
- const rawArgs = typeof payload.arguments === "string"
765
- ? payload.arguments
766
- : payload.item && typeof payload.item === "object" && typeof payload.item.arguments === "string"
767
- ? String(payload.item.arguments)
768
- : "{}";
769
- if (!name || !callId) {
770
- return [];
771
- }
772
- return [
773
- {
774
- type: "realtime-tool-call",
775
- toolCall: {
776
- id: callId,
777
- name,
778
- input: JSON.parse(rawArgs || "{}")
779
- }
780
- }
781
- ];
782
- }
783
- if (type === "response.done") {
784
- return [
785
- {
786
- type: "realtime-response-complete",
787
- reason: typeof payload.status === "string" ? payload.status : undefined,
788
- providerMetadata: parseRealtimeProviderMetadata(payload)
789
- }
790
- ];
791
- }
792
- if (type === "error") {
793
- const error = payload.error && typeof payload.error === "object" ? payload.error : undefined;
794
- return [
795
- {
796
- type: "realtime-error",
797
- message: typeof payload.message === "string"
798
- ? payload.message
799
- : typeof error?.message === "string"
800
- ? error.message
801
- : "Realtime API error.",
802
- providerMetadata: parseRealtimeProviderMetadata(payload)
803
- }
804
- ];
805
- }
806
- if (type === "session.end") {
807
- return [
808
- {
809
- type: "realtime-end",
810
- reason: "session-end",
811
- providerMetadata: parseRealtimeProviderMetadata(payload)
812
- }
813
- ];
814
- }
815
- return [];
816
- };
817
380
  const mapStructuredOutput = (input) => {
818
381
  if (!input.structuredOutput || input.structuredOutput.mode !== "native") {
819
382
  return undefined;
@@ -1855,45 +1418,43 @@ class OpenAILanguageModel {
1855
1418
  responseLimits;
1856
1419
  provider = "openai";
1857
1420
  capabilities;
1858
- constructor(modelId, apiKey, baseURL, fetcher, responseLimits) {
1421
+ constructor(modelId, apiKey, baseURL, fetcher, responseLimits, capabilityOptions) {
1859
1422
  this.modelId = modelId;
1860
1423
  this.apiKey = apiKey;
1861
1424
  this.baseURL = baseURL;
1862
1425
  this.fetcher = fetcher;
1863
1426
  this.responseLimits = responseLimits;
1864
- this.capabilities = { ...modelCapabilities(modelId), toolHistory: "json" };
1427
+ const base = modelCapabilities(modelId, capabilityOptions.unknownModelCapabilities);
1428
+ const override = (capabilityOptions.modelCapabilities && Object.hasOwn(capabilityOptions.modelCapabilities, modelId) ? capabilityOptions.modelCapabilities[modelId] : undefined);
1429
+ this.capabilities = { ...base, ...override,
1430
+ agentCapabilities: { ...base.agentCapabilities, ...override?.agentCapabilities }, toolHistory: "json" };
1865
1431
  }
1866
1432
  usesResponsesAPI(input, options) {
1867
- if (isOpenAISolLunaModel(this.modelId)) {
1868
- const raw = options.bodyOptions;
1869
- const effort = input.reasoning?.effort ?? raw.reasoning?.effort ?? raw.reasoning_effort;
1870
- if (effort !== undefined && !["none", "low", "medium", "high", "xhigh", "max"].includes(String(effort))) {
1871
- throw new UnsupportedFeatureError(`GPT-6 Sol/Luna does not support reasoning effort "${effort}".`);
1872
- }
1873
- if (effort !== "none") {
1874
- if (input.temperature !== undefined || raw.temperature !== undefined || raw.top_p !== undefined || raw.top_logprobs !== undefined ||
1875
- (options.apiMode === "chat" && raw.logprobs !== undefined) || raw.include?.includes("message.output_text.logprobs")) {
1876
- throw new UnsupportedFeatureError('GPT-6 Sol/Luna sampling and logprobs controls require reasoning effort "none".');
1877
- }
1878
- if (options.apiMode === "chat" && Object.keys(input.tools ?? {}).length) {
1879
- throw new UnsupportedFeatureError('GPT-6 Sol/Luna tool calling with reasoning requires the Responses API; Chat Completions requires effort "none".');
1880
- }
1881
- }
1433
+ if (!this.capabilities.tools && (Object.keys(input.tools ?? {}).length || input.messages.some(message => message.parts.some(part => part.type === "tool-call" || part.type === "tool-result")))) {
1434
+ throw new UnsupportedFeatureError(`OpenAI model "${this.modelId}" has no declared tool support.`);
1882
1435
  }
1883
- if (isOpenAIAstraModel(this.modelId)) {
1884
- const raw = options.bodyOptions;
1885
- const effort = input.reasoning?.effort ?? raw.reasoning?.effort ?? raw.reasoning_effort;
1886
- if (effort !== undefined && !["low", "medium", "high", "xhigh", "max"].includes(String(effort))) {
1887
- throw new UnsupportedFeatureError(`GPT-6 Astra does not support reasoning effort "${effort}".`);
1888
- }
1889
- if (input.temperature !== undefined || raw.temperature !== undefined || raw.top_p !== undefined || raw.top_logprobs !== undefined ||
1890
- (options.apiMode === "chat" && raw.logprobs !== undefined) || raw.include?.includes("message.output_text.logprobs")) {
1891
- throw new UnsupportedFeatureError("GPT-6 Astra does not support the requested sampling or logprobs controls.");
1892
- }
1893
- if (options.apiMode === "chat" && Object.keys(input.tools ?? {}).length) {
1894
- throw new UnsupportedFeatureError("GPT-6 Astra tool calling requires the Responses API.");
1895
- }
1436
+ if (!this.capabilities.vision && input.messages.some(message => message.parts.some(part => part.type === "image"))) {
1437
+ throw new UnsupportedFeatureError(`OpenAI model "${this.modelId}" has no declared vision support.`);
1438
+ }
1439
+ const rawFormat = options.bodyOptions.response_format;
1440
+ const rawText = options.bodyOptions.text;
1441
+ const formatTypes = [rawFormat?.type, rawText?.format?.type];
1442
+ if (!this.capabilities.structuredOutput && (input.structuredOutput?.mode === "native" || formatTypes.includes("json_schema"))) {
1443
+ throw new UnsupportedFeatureError(`OpenAI model "${this.modelId}" has no declared native structured output support.`);
1896
1444
  }
1445
+ if (!this.capabilities.jsonMode && formatTypes.includes("json_object")) {
1446
+ throw new UnsupportedFeatureError(`OpenAI model "${this.modelId}" has no declared JSON mode support.`);
1447
+ }
1448
+ if (!this.capabilities.parallelToolCalls && options.bodyOptions.parallel_tool_calls === true) {
1449
+ throw new UnsupportedFeatureError(`OpenAI model "${this.modelId}" has no declared parallel tool calling support.`);
1450
+ }
1451
+ if (!this.capabilities.reasoning && (input.reasoning || options.bodyOptions.reasoning || options.bodyOptions.reasoning_effort)) {
1452
+ throw new UnsupportedFeatureError(`OpenAI model "${this.modelId}" has no declared reasoning support.`);
1453
+ }
1454
+ if (!this.capabilities.toolChoice && (input.toolChoice !== undefined || options.bodyOptions.tool_choice !== undefined)) {
1455
+ throw new UnsupportedFeatureError(`OpenAI model "${this.modelId}" has no declared tool choice support.`);
1456
+ }
1457
+ assertOpenAIModelRequestSupported(this.modelId, input, options);
1897
1458
  const requiresResponses = hasResponsesOnlyTools(input.tools) || options.multiAgentEnabled;
1898
1459
  if (options.apiMode === "chat") {
1899
1460
  if (requiresResponses) {
@@ -1901,12 +1462,12 @@ class OpenAILanguageModel {
1901
1462
  }
1902
1463
  return false;
1903
1464
  }
1904
- return options.apiMode === "responses" || requiresResponses || supportsOpenAIModernResponses(this.modelId);
1465
+ return options.apiMode === "responses" || requiresResponses || resolveOpenAIModelProfile(this.modelId).defaultApi === "responses";
1905
1466
  }
1906
1467
  async generateViaResponses(input, signal, options) {
1907
1468
  const responseBodyOptions = openAIResponsesBodyOptions(options.bodyOptions, this.modelId);
1908
- assertResponsesToolsSupported(this.modelId, input.tools);
1909
- assertOpenAIResponsesOptionsSupported(this.modelId, responseBodyOptions);
1469
+ assertResponsesToolsSupported(this.modelId, input.tools, this.capabilities.agentCapabilities);
1470
+ assertOpenAIResponsesOptionsSupported(this.modelId, responseBodyOptions, this.capabilities.agentCapabilities);
1910
1471
  const previousResponse = responseBodyOptions.store === false ? undefined : getProviderResponseId(input.messages);
1911
1472
  const messages = previousResponse && previousResponse.index < input.messages.length - 1
1912
1473
  ? input.messages.slice(previousResponse.index + 1)
@@ -1927,7 +1488,7 @@ class OpenAILanguageModel {
1927
1488
  ...(nextInput.length ? { input: nextInput } : {}),
1928
1489
  tools: mapResponsesTools(input.tools),
1929
1490
  ...(input.toolChoice ? { tool_choice: mapResponsesToolChoice(input.toolChoice) } : {}),
1930
- text: mapResponsesStructuredOutput(input),
1491
+ text: mapResponsesStructuredOutput(input) ?? responseBodyOptions.text,
1931
1492
  temperature: input.temperature,
1932
1493
  max_output_tokens: input.maxTokens,
1933
1494
  ...mapResponsesReasoning(input, responseBodyOptions.reasoning)
@@ -2001,7 +1562,7 @@ class OpenAILanguageModel {
2001
1562
  messages: mapMessages(input.messages, input.toolResultFormat),
2002
1563
  tools: mapTools(input.tools),
2003
1564
  ...(input.toolChoice ? { tool_choice: mapToolChoice(input.toolChoice) } : {}),
2004
- response_format: mapStructuredOutput(input),
1565
+ response_format: mapStructuredOutput(input) ?? options.bodyOptions.response_format,
2005
1566
  temperature: input.temperature,
2006
1567
  ...(input.reasoning ? {} : { max_tokens: input.maxTokens }),
2007
1568
  ...mapChatReasoning(input),
@@ -2030,8 +1591,8 @@ class OpenAILanguageModel {
2030
1591
  const options = resolveOpenAILanguageRequestOptions(input.providerOptions, this.apiKey);
2031
1592
  if (this.usesResponsesAPI(input, options)) {
2032
1593
  const responseBodyOptions = openAIResponsesBodyOptions(options.bodyOptions, this.modelId);
2033
- assertResponsesToolsSupported(this.modelId, input.tools);
2034
- assertOpenAIResponsesOptionsSupported(this.modelId, responseBodyOptions);
1594
+ assertResponsesToolsSupported(this.modelId, input.tools, this.capabilities.agentCapabilities);
1595
+ assertOpenAIResponsesOptionsSupported(this.modelId, responseBodyOptions, this.capabilities.agentCapabilities);
2035
1596
  const { signal, cleanup } = getRequestOptions(input);
2036
1597
  if (hasProgrammaticToolCalling(input.tools)) {
2037
1598
  const result = await this.generateViaResponses(input, signal, options);
@@ -2059,7 +1620,7 @@ class OpenAILanguageModel {
2059
1620
  ...(messages.length ? { input: toResponsesInput(messages, input.toolResultFormat, input.messages) } : {}),
2060
1621
  tools: mapResponsesTools(input.tools),
2061
1622
  ...(input.toolChoice ? { tool_choice: mapResponsesToolChoice(input.toolChoice) } : {}),
2062
- text: mapResponsesStructuredOutput(input),
1623
+ text: mapResponsesStructuredOutput(input) ?? responseBodyOptions.text,
2063
1624
  temperature: input.temperature,
2064
1625
  max_output_tokens: input.maxTokens,
2065
1626
  ...mapResponsesReasoning(input, responseBodyOptions.reasoning),
@@ -2088,7 +1649,7 @@ class OpenAILanguageModel {
2088
1649
  messages: mapMessages(input.messages, input.toolResultFormat),
2089
1650
  tools: mapTools(input.tools),
2090
1651
  ...(input.toolChoice ? { tool_choice: mapToolChoice(input.toolChoice) } : {}),
2091
- response_format: mapStructuredOutput(input),
1652
+ response_format: mapStructuredOutput(input) ?? options.bodyOptions.response_format,
2092
1653
  temperature: input.temperature,
2093
1654
  ...(input.reasoning ? {} : { max_tokens: input.maxTokens }),
2094
1655
  ...mapChatReasoning(input),
@@ -2098,61 +1659,7 @@ class OpenAILanguageModel {
2098
1659
  }), { ...input, abortSignal: signal }, "OpenAI").catch((error) => { cleanup(); throw error; });
2099
1660
  return (async function* () {
2100
1661
  try {
2101
- const toolBuffers = new Map();
2102
- let lastFinishReason;
2103
- let lastUsage;
2104
- for await (const event of streamSSE(response)) {
2105
- if (event.data === "[DONE]") {
2106
- break;
2107
- }
2108
- const json = JSON.parse(event.data);
2109
- const choice = json.choices?.[0];
2110
- const delta = choice?.delta;
2111
- if (delta?.content) {
2112
- yield { type: "text-delta", textDelta: delta.content };
2113
- }
2114
- for (const toolCall of delta?.tool_calls ?? []) {
2115
- const index = typeof toolCall.index === "number" && Number.isSafeInteger(toolCall.index) && toolCall.index >= 0
2116
- ? toolCall.index
2117
- : 0;
2118
- const existing = toolBuffers.get(index) ?? {
2119
- id: toolCall.id ?? `${index}`,
2120
- name: toolCall.function?.name ?? "",
2121
- args: ""
2122
- };
2123
- existing.id = toolCall.id ?? existing.id;
2124
- existing.name ||= toolCall.function?.name ?? "";
2125
- existing.args += toolCall.function?.arguments ?? "";
2126
- toolBuffers.set(index, existing);
2127
- }
2128
- if (choice?.finish_reason === "tool_calls") {
2129
- for (const [, toolCall] of [...toolBuffers.entries()].sort(([left], [right]) => left - right)) {
2130
- yield {
2131
- type: "tool-call",
2132
- toolCall: {
2133
- id: toolCall.id,
2134
- name: toolCall.name,
2135
- input: JSON.parse(toolCall.args || "{}")
2136
- }
2137
- };
2138
- }
2139
- toolBuffers.clear();
2140
- }
2141
- if (choice?.finish_reason) {
2142
- lastFinishReason = choice.finish_reason;
2143
- }
2144
- if (json.usage) {
2145
- lastUsage = json.usage;
2146
- }
2147
- }
2148
- if (lastFinishReason || lastUsage) {
2149
- yield {
2150
- type: "finish",
2151
- finishReason: normalizeFinishReason(lastFinishReason),
2152
- providerFinishReason: lastFinishReason,
2153
- usage: mapChatUsage(lastUsage)
2154
- };
2155
- }
1662
+ yield* streamChatCompletions(response, "openai");
2156
1663
  }
2157
1664
  finally {
2158
1665
  cleanup();
@@ -2160,159 +1667,6 @@ class OpenAILanguageModel {
2160
1667
  })();
2161
1668
  }
2162
1669
  }
2163
- class OpenAIEmbeddingModel {
2164
- modelId;
2165
- apiKey;
2166
- baseURL;
2167
- fetcher;
2168
- provider = "openai";
2169
- capabilities = capabilities;
2170
- constructor(modelId, apiKey, baseURL, fetcher) {
2171
- this.modelId = modelId;
2172
- this.apiKey = apiKey;
2173
- this.baseURL = baseURL;
2174
- this.fetcher = fetcher;
2175
- }
2176
- async embed(input) {
2177
- const { signal, cleanup } = withTimeoutSignal(input);
2178
- const values = input.values.map((value) => {
2179
- if (typeof value !== "string") {
2180
- throw new UnsupportedFeatureError('Provider "openai" does not support multimodal embedding values.');
2181
- }
2182
- return value;
2183
- });
2184
- try {
2185
- const response = await withRetry(() => this.fetcher(`${this.baseURL}/embeddings`, {
2186
- method: "POST",
2187
- headers: jsonHeaders(this.apiKey),
2188
- signal,
2189
- body: JSON.stringify({
2190
- model: this.modelId,
2191
- input: values
2192
- })
2193
- }), input);
2194
- const json = await parseJson(response);
2195
- return {
2196
- embeddings: json.data.map((entry) => entry.embedding),
2197
- usage: {
2198
- inputTokens: json.usage?.prompt_tokens,
2199
- totalTokens: json.usage?.total_tokens
2200
- },
2201
- rawResponse: json
2202
- };
2203
- }
2204
- finally {
2205
- cleanup();
2206
- }
2207
- }
2208
- }
2209
- class OpenAITranscriptionModel {
2210
- modelId;
2211
- apiKey;
2212
- baseURL;
2213
- fetcher;
2214
- responseLimits;
2215
- provider = "openai";
2216
- capabilities = transcriptionCapabilities;
2217
- constructor(modelId, apiKey, baseURL, fetcher, responseLimits = resolveAudioResponseLimits()) {
2218
- this.modelId = modelId;
2219
- this.apiKey = apiKey;
2220
- this.baseURL = baseURL;
2221
- this.fetcher = fetcher;
2222
- this.responseLimits = responseLimits;
2223
- }
2224
- async transcribe(input) {
2225
- const { signal, cleanup, abort } = withTimeoutSignal(input);
2226
- const form = new FormData();
2227
- for (const [key, value] of Object.entries(input.providerOptions ?? {})) {
2228
- if (["model", "file", "prompt", "language"].includes(key)) {
2229
- continue;
2230
- }
2231
- form.set(key, typeof value === "string" ? value : JSON.stringify(value));
2232
- }
2233
- form.set("model", this.modelId);
2234
- form.set("file", createAudioFile(input.audio));
2235
- if (input.prompt) {
2236
- form.set("prompt", input.prompt);
2237
- }
2238
- if (input.language) {
2239
- form.set("language", input.language);
2240
- }
2241
- try {
2242
- const response = await withRetry(() => this.fetcher(`${this.baseURL}/audio/transcriptions`, {
2243
- method: "POST",
2244
- redirect: "error",
2245
- headers: { authorization: `Bearer ${this.apiKey}` },
2246
- signal,
2247
- body: form
2248
- }), input);
2249
- const json = await parseJson(response, {
2250
- maxBytes: this.responseLimits.transcriptionBytes,
2251
- errorBodyBytes: this.responseLimits.errorBodyBytes,
2252
- endpoint: "audio/transcriptions",
2253
- abort
2254
- });
2255
- return {
2256
- text: json.text ?? "",
2257
- rawResponse: json
2258
- };
2259
- }
2260
- finally {
2261
- cleanup();
2262
- }
2263
- }
2264
- }
2265
- class OpenAISpeechModel {
2266
- modelId;
2267
- apiKey;
2268
- baseURL;
2269
- fetcher;
2270
- responseLimits;
2271
- provider = "openai";
2272
- capabilities = speechCapabilities;
2273
- constructor(modelId, apiKey, baseURL, fetcher, responseLimits = resolveAudioResponseLimits()) {
2274
- this.modelId = modelId;
2275
- this.apiKey = apiKey;
2276
- this.baseURL = baseURL;
2277
- this.fetcher = fetcher;
2278
- this.responseLimits = responseLimits;
2279
- }
2280
- async generateSpeech(input) {
2281
- const { signal, cleanup, abort } = withTimeoutSignal(input);
2282
- try {
2283
- const response = await withRetry(() => this.fetcher(`${this.baseURL}/audio/speech`, {
2284
- method: "POST",
2285
- headers: jsonHeaders(this.apiKey),
2286
- signal,
2287
- body: JSON.stringify({
2288
- ...input.providerOptions,
2289
- model: this.modelId,
2290
- input: input.input,
2291
- voice: input.voice ?? "alloy"
2292
- })
2293
- }), input);
2294
- if (!response.ok) {
2295
- const body = await readErrorBodyWithLimit(response, this.responseLimits.errorBodyBytes);
2296
- throw new ProviderHTTPError(`OpenAI request failed with status ${response.status}.`, response.status, {
2297
- responseBody: body
2298
- });
2299
- }
2300
- return {
2301
- audio: await readBodyWithLimit(response, {
2302
- maxBytes: this.responseLimits.speechBytes,
2303
- provider: "openai",
2304
- endpoint: "audio/speech",
2305
- abort
2306
- }),
2307
- mediaType: response.headers.get("content-type") ?? "audio/mpeg",
2308
- rawResponse: undefined
2309
- };
2310
- }
2311
- finally {
2312
- cleanup();
2313
- }
2314
- }
2315
- }
2316
1670
  class OpenAIGroundedLanguageModel {
2317
1671
  modelId;
2318
1672
  apiKey;
@@ -2360,234 +1714,6 @@ class OpenAIGroundedLanguageModel {
2360
1714
  }
2361
1715
  }
2362
1716
  }
2363
- class OpenAIRealtimeModel {
2364
- modelId;
2365
- apiKey;
2366
- baseURL;
2367
- fetcher;
2368
- connectionFactory;
2369
- realtimeURL;
2370
- browserTokenURL;
2371
- allowUnsafeEndpoints;
2372
- provider = "openai";
2373
- capabilities;
2374
- constructor(modelId, apiKey, baseURL, fetcher, connectionFactory, realtimeURL, browserTokenURL, allowUnsafeEndpoints = false) {
2375
- this.modelId = modelId;
2376
- this.apiKey = apiKey;
2377
- this.baseURL = baseURL;
2378
- this.fetcher = fetcher;
2379
- this.connectionFactory = connectionFactory;
2380
- this.realtimeURL = realtimeURL;
2381
- this.browserTokenURL = browserTokenURL;
2382
- this.allowUnsafeEndpoints = allowUnsafeEndpoints;
2383
- this.capabilities = realtimeCapabilities(modelId);
2384
- }
2385
- resolveConfig(config) {
2386
- if (config.delegation)
2387
- throw new UnsupportedFeatureError("Client delegation requires a GPT-Live model.");
2388
- const mode = inferOpenAIRealtimeMode(this.modelId, config.mode);
2389
- if (mode !== "conversation" && (config.tools || config.toolChoice)) {
2390
- throw new UnsupportedFeatureError(`Provider "openai" model "${this.modelId}" does not support realtime tools in ${mode} mode.`);
2391
- }
2392
- if (mode === "translation" && !config.translation?.targetLanguage) {
2393
- throw new ConfigurationError('OpenAI realtime translation sessions require "translation.targetLanguage".');
2394
- }
2395
- if (mode === "transcription" && config.voice) {
2396
- throw new UnsupportedFeatureError(`Provider "openai" model "${this.modelId}" does not support realtime audio output in transcription mode.`);
2397
- }
2398
- return {
2399
- mode,
2400
- autoResponse: mode === "conversation",
2401
- ...config
2402
- };
2403
- }
2404
- async connect(config = {}, options) {
2405
- const initialConfig = this.resolveConfig(config);
2406
- const providerOptions = initialConfig.providerOptions;
2407
- const headers = resolveOpenAIRealtimeHeaders(this.apiKey, providerOptions);
2408
- const trustedRealtimeHost = new URL(this.realtimeURL ?? this.baseURL).hostname;
2409
- const realtimeEndpoint = assertTrustedEndpoint(this.realtimeURL ??
2410
- openAIRealtimeURL(this.baseURL, this.modelId, inferOpenAIRealtimeMode(this.modelId, initialConfig.mode), providerOptions), {
2411
- label: "OpenAI realtime endpoint",
2412
- protocols: ["wss"],
2413
- allowedHosts: [trustedRealtimeHost],
2414
- allowUnsafe: this.allowUnsafeEndpoints
2415
- }).toString();
2416
- const connection = await (this.connectionFactory ?? openWebSocketConnection)(realtimeEndpoint, headers, options);
2417
- const session = new CallbackRealtimeSession({
2418
- provider: this.provider,
2419
- modelId: this.modelId,
2420
- capabilities: this.capabilities,
2421
- config: initialConfig,
2422
- connection,
2423
- callbacks: {
2424
- parseEvent: parseOpenAIRealtimeEvent,
2425
- buildAudioPayloads: (frame, sessionConfig) => {
2426
- const mode = inferOpenAIRealtimeMode(this.modelId, sessionConfig.mode);
2427
- const payloads = [
2428
- {
2429
- type: "input_audio_buffer.append",
2430
- audio: encodeAudioFrame(frame)
2431
- }
2432
- ];
2433
- if (frame.isFinal) {
2434
- payloads.push({ type: "input_audio_buffer.commit" });
2435
- if (mode === "conversation" && (sessionConfig.autoResponse ?? true)) {
2436
- payloads.push({ type: "response.create" });
2437
- }
2438
- }
2439
- return payloads;
2440
- },
2441
- buildMediaPayloads: (frame) => {
2442
- if (!openAIRealtimeSupportsImageInput(this.modelId)) {
2443
- throw new UnsupportedFeatureError(`Provider "openai" model "${this.modelId}" does not support realtime image input.`);
2444
- }
2445
- if (!frame.mediaType.startsWith("image/")) {
2446
- throw new UnsupportedFeatureError(`Provider "openai" only supports realtime image media input, but received "${frame.mediaType}".`);
2447
- }
2448
- return [
2449
- {
2450
- type: "conversation.item.create",
2451
- item: {
2452
- type: "message",
2453
- role: "user",
2454
- content: [
2455
- {
2456
- type: "input_image",
2457
- image_url: `data:${frame.mediaType};base64,${encodeMediaFrame(frame)}`
2458
- }
2459
- ]
2460
- }
2461
- }
2462
- ];
2463
- },
2464
- buildTextPayloads: (text, sessionConfig) => {
2465
- const mode = inferOpenAIRealtimeMode(this.modelId, sessionConfig.mode);
2466
- if (mode !== "conversation") {
2467
- throw new UnsupportedFeatureError(`Provider "openai" model "${this.modelId}" does not support realtime text input in ${mode} mode.`);
2468
- }
2469
- const payloads = [
2470
- {
2471
- type: "conversation.item.create",
2472
- item: {
2473
- type: "message",
2474
- role: "user",
2475
- content: [{ type: "input_text", text }]
2476
- }
2477
- }
2478
- ];
2479
- if (sessionConfig.autoResponse ?? true) {
2480
- payloads.push({ type: "response.create" });
2481
- }
2482
- return payloads;
2483
- },
2484
- buildToolResultPayloads: (result, sessionConfig) => {
2485
- const mode = inferOpenAIRealtimeMode(this.modelId, sessionConfig.mode);
2486
- if (mode !== "conversation") {
2487
- throw new UnsupportedFeatureError(`Provider "openai" model "${this.modelId}" does not support realtime tools in ${mode} mode.`);
2488
- }
2489
- const mcpMetadata = result.providerMetadata?.[openAIRealtimeMcpMetadataKey];
2490
- if (mcpMetadata && typeof mcpMetadata === "object" && !Array.isArray(mcpMetadata)) {
2491
- const approval = mcpMetadata;
2492
- if (approval.type === "mcp_approval_response" &&
2493
- typeof approval.approval_request_id === "string" &&
2494
- typeof approval.approve === "boolean") {
2495
- return [
2496
- {
2497
- type: "conversation.item.create",
2498
- item: {
2499
- ...(typeof approval.id === "string" ? { id: approval.id } : {}),
2500
- type: "mcp_approval_response",
2501
- approval_request_id: approval.approval_request_id,
2502
- approve: approval.approve,
2503
- ...(typeof approval.reason === "string" ? { reason: approval.reason } : {})
2504
- }
2505
- }
2506
- ];
2507
- }
2508
- }
2509
- const payloads = [
2510
- {
2511
- type: "conversation.item.create",
2512
- item: {
2513
- type: "function_call_output",
2514
- call_id: result.toolCallId,
2515
- output: JSON.stringify(toolResultPayload(result))
2516
- }
2517
- }
2518
- ];
2519
- if (sessionConfig.autoResponse ?? true) {
2520
- payloads.push({ type: "response.create" });
2521
- }
2522
- return payloads;
2523
- },
2524
- buildUpdatePayloads: (sessionConfig) => [
2525
- {
2526
- type: "session.update",
2527
- session: mapRealtimeSessionConfig(this.resolveConfig(sessionConfig), this.modelId)
2528
- }
2529
- ],
2530
- buildInitialPayloads: (sessionConfig) => [
2531
- {
2532
- type: "session.update",
2533
- session: mapRealtimeSessionConfig(sessionConfig, this.modelId)
2534
- }
2535
- ]
2536
- }
2537
- });
2538
- await session.initialize();
2539
- return session;
2540
- }
2541
- async createBrowserToken(config = {}, options) {
2542
- const resolvedConfig = this.resolveConfig(config);
2543
- const providerOptions = { ...(resolvedConfig.providerOptions ?? {}) };
2544
- const expiresAfter = providerOptions.expires_after;
2545
- delete providerOptions.expires_after;
2546
- const { signal, cleanup } = withTimeoutSignal({
2547
- abortSignal: options?.signal,
2548
- timeoutMs: options?.timeoutMs
2549
- });
2550
- try {
2551
- const response = await withRetry(() => this.fetcher(this.browserTokenURL ?? `${this.baseURL}/realtime/client_secrets`, {
2552
- method: "POST",
2553
- redirect: "error",
2554
- headers: resolveOpenAIRealtimeHeaders(this.apiKey, providerOptions, true),
2555
- body: JSON.stringify({
2556
- ...(expiresAfter ? { expires_after: expiresAfter } : {}),
2557
- session: mapRealtimeSessionConfig({
2558
- ...resolvedConfig,
2559
- providerOptions
2560
- }, this.modelId)
2561
- }),
2562
- signal
2563
- }));
2564
- const payload = await parseJson(response);
2565
- const secret = payload.client_secret;
2566
- const value = secret && typeof secret === "object" && typeof secret.value === "string"
2567
- ? secret.value
2568
- : typeof payload.token === "string"
2569
- ? payload.token
2570
- : typeof payload.value === "string"
2571
- ? payload.value
2572
- : "";
2573
- const expiresAtMs = secret && typeof secret === "object" && typeof secret.expires_at_ms === "number"
2574
- ? secret.expires_at_ms
2575
- : typeof payload.expires_at_ms === "number"
2576
- ? payload.expires_at_ms
2577
- : typeof payload.expires_at === "number"
2578
- ? payload.expires_at * 1000
2579
- : undefined;
2580
- return {
2581
- value,
2582
- expiresAtMs,
2583
- rawResponse: payload
2584
- };
2585
- }
2586
- finally {
2587
- cleanup();
2588
- }
2589
- }
2590
- }
2591
1717
  export const createOpenAI = (options = {}) => {
2592
1718
  const apiKey = options.apiKey ?? process.env.OPENAI_API_KEY;
2593
1719
  if (!apiKey) {
@@ -2622,7 +1748,7 @@ export const createOpenAI = (options = {}) => {
2622
1748
  languageModel: (modelId) => {
2623
1749
  if (modelId.startsWith("gpt-live-"))
2624
1750
  throw new UnsupportedFeatureError("GPT-Live is a voice model; use openai.realtimeModel(modelId).");
2625
- return new OpenAILanguageModel(modelId, apiKey, baseURL, fetcher, responseLimits);
1751
+ return new OpenAILanguageModel(modelId, apiKey, baseURL, fetcher, responseLimits, options);
2626
1752
  },
2627
1753
  embeddingModel: (modelId) => new OpenAIEmbeddingModel(modelId, apiKey, baseURL, fetcher),
2628
1754
  transcriptionModel: (modelId) => new OpenAITranscriptionModel(modelId, apiKey, baseURL, fetcher, responseLimits),