@ai-sdk/openai 4.0.6 → 4.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/dist/index.d.ts +11 -2
- package/dist/index.js +262 -4
- package/dist/index.js.map +1 -1
- package/dist/internal/index.d.ts +15 -4
- package/dist/internal/index.js +272 -15
- package/dist/internal/index.js.map +1 -1
- package/docs/03-openai.mdx +15 -5
- package/package.json +3 -3
- package/src/chat/convert-to-openai-chat-messages.ts +1 -1
- package/src/openai-config.ts +5 -1
- package/src/openai-provider.ts +8 -0
- package/src/transcription/openai-transcription-model-options.ts +20 -0
- package/src/transcription/openai-transcription-model.ts +328 -4
package/dist/internal/index.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import * as _ai_sdk_provider from '@ai-sdk/provider';
|
|
2
|
-
import { LanguageModelV4, LanguageModelV4CallOptions, LanguageModelV4GenerateResult, LanguageModelV4StreamResult, EmbeddingModelV4, ImageModelV4, TranscriptionModelV4CallOptions, TranscriptionModelV4, SpeechModelV4, JSONValue } from '@ai-sdk/provider';
|
|
2
|
+
import { LanguageModelV4, LanguageModelV4CallOptions, LanguageModelV4GenerateResult, LanguageModelV4StreamResult, EmbeddingModelV4, ImageModelV4, TranscriptionModelV4CallOptions, TranscriptionModelV4, Experimental_TranscriptionModelV4StreamOptions, SpeechModelV4, JSONValue } from '@ai-sdk/provider';
|
|
3
3
|
import * as _ai_sdk_provider_utils from '@ai-sdk/provider-utils';
|
|
4
|
-
import { InferSchema, WORKFLOW_SERIALIZE, WORKFLOW_DESERIALIZE, FetchFunction } from '@ai-sdk/provider-utils';
|
|
4
|
+
import { InferSchema, WORKFLOW_SERIALIZE, WORKFLOW_DESERIALIZE, FetchFunction, WebSocketConstructor } from '@ai-sdk/provider-utils';
|
|
5
5
|
|
|
6
6
|
type OpenAIChatModelId = 'o1' | 'o1-2024-12-17' | 'o3-mini' | 'o3-mini-2025-01-31' | 'o3' | 'o3-2025-04-16' | 'o4-mini' | 'o4-mini-2025-04-16' | 'gpt-4.1' | 'gpt-4.1-2025-04-14' | 'gpt-4.1-mini' | 'gpt-4.1-mini-2025-04-14' | 'gpt-4.1-nano' | 'gpt-4.1-nano-2025-04-14' | 'gpt-4o' | 'gpt-4o-2024-05-13' | 'gpt-4o-2024-08-06' | 'gpt-4o-2024-11-20' | 'gpt-4o-audio-preview' | 'gpt-4o-audio-preview-2024-12-17' | 'gpt-4o-audio-preview-2025-06-03' | 'gpt-4o-mini' | 'gpt-4o-mini-2024-07-18' | 'gpt-4o-mini-audio-preview' | 'gpt-4o-mini-audio-preview-2024-12-17' | 'gpt-4o-search-preview' | 'gpt-4o-search-preview-2025-03-11' | 'gpt-4o-mini-search-preview' | 'gpt-4o-mini-search-preview-2025-03-11' | 'gpt-3.5-turbo-0125' | 'gpt-3.5-turbo' | 'gpt-3.5-turbo-1106' | 'gpt-3.5-turbo-16k' | 'gpt-5' | 'gpt-5-2025-08-07' | 'gpt-5-mini' | 'gpt-5-mini-2025-08-07' | 'gpt-5-nano' | 'gpt-5-nano-2025-08-07' | 'gpt-5-chat-latest' | 'gpt-5.1' | 'gpt-5.1-2025-11-13' | 'gpt-5.1-chat-latest' | 'gpt-5.2' | 'gpt-5.2-2025-12-11' | 'gpt-5.2-chat-latest' | 'gpt-5.2-pro' | 'gpt-5.2-pro-2025-12-11' | 'gpt-5.3-chat-latest' | 'gpt-5.4' | 'gpt-5.4-2026-03-05' | 'gpt-5.4-mini' | 'gpt-5.4-mini-2026-03-17' | 'gpt-5.4-nano' | 'gpt-5.4-nano-2026-03-17' | 'gpt-5.4-pro' | 'gpt-5.4-pro-2026-03-05' | 'gpt-5.5' | 'gpt-5.5-2026-04-23' | (string & {});
|
|
7
7
|
declare const openaiLanguageModelChatOptions: _ai_sdk_provider_utils.LazySchema<{
|
|
@@ -104,6 +104,7 @@ type OpenAIConfig = {
|
|
|
104
104
|
}) => string;
|
|
105
105
|
headers?: () => Record<string, string | undefined>;
|
|
106
106
|
fetch?: FetchFunction;
|
|
107
|
+
webSocket?: WebSocketConstructor;
|
|
107
108
|
generateId?: () => string;
|
|
108
109
|
/**
|
|
109
110
|
* This is soft-deprecated. Use provider references (e.g. `{ openai: 'file-abc123' }`)
|
|
@@ -196,13 +197,17 @@ declare class OpenAIImageModel implements ImageModelV4 {
|
|
|
196
197
|
doGenerate({ prompt, files, mask, n, size, aspectRatio, seed, providerOptions, headers, abortSignal, }: Parameters<ImageModelV4['doGenerate']>[0]): Promise<Awaited<ReturnType<ImageModelV4['doGenerate']>>>;
|
|
197
198
|
}
|
|
198
199
|
|
|
199
|
-
type OpenAITranscriptionModelId = 'whisper-1' | 'gpt-4o-mini-transcribe' | 'gpt-4o-mini-transcribe-2025-03-20' | 'gpt-4o-mini-transcribe-2025-12-15' | 'gpt-4o-transcribe' | 'gpt-4o-transcribe-diarize' | (string & {});
|
|
200
|
+
type OpenAITranscriptionModelId = 'whisper-1' | 'gpt-4o-mini-transcribe' | 'gpt-4o-mini-transcribe-2025-03-20' | 'gpt-4o-mini-transcribe-2025-12-15' | 'gpt-4o-transcribe' | 'gpt-4o-transcribe-diarize' | 'gpt-realtime-whisper' | (string & {});
|
|
200
201
|
declare const openAITranscriptionModelOptions: _ai_sdk_provider_utils.LazySchema<{
|
|
201
202
|
include?: string[] | undefined;
|
|
202
203
|
language?: string | undefined;
|
|
203
204
|
prompt?: string | undefined;
|
|
204
205
|
temperature?: number | undefined;
|
|
205
206
|
timestampGranularities?: ("word" | "segment")[] | undefined;
|
|
207
|
+
streaming?: {
|
|
208
|
+
delay?: "minimal" | "low" | "medium" | "high" | "xhigh" | undefined;
|
|
209
|
+
include?: string[] | undefined;
|
|
210
|
+
} | undefined;
|
|
206
211
|
}>;
|
|
207
212
|
type OpenAITranscriptionModelOptions = InferSchema<typeof openAITranscriptionModelOptions>;
|
|
208
213
|
|
|
@@ -211,6 +216,11 @@ type OpenAITranscriptionCallOptions = Omit<TranscriptionModelV4CallOptions, 'pro
|
|
|
211
216
|
openai?: OpenAITranscriptionModelOptions;
|
|
212
217
|
};
|
|
213
218
|
};
|
|
219
|
+
type OpenAITranscriptionStreamOptions = Omit<Experimental_TranscriptionModelV4StreamOptions, 'providerOptions'> & {
|
|
220
|
+
providerOptions?: {
|
|
221
|
+
openai?: OpenAITranscriptionModelOptions;
|
|
222
|
+
};
|
|
223
|
+
};
|
|
214
224
|
interface OpenAITranscriptionModelConfig extends OpenAIConfig {
|
|
215
225
|
_internal?: {
|
|
216
226
|
currentDate?: () => Date;
|
|
@@ -232,6 +242,7 @@ declare class OpenAITranscriptionModel implements TranscriptionModelV4 {
|
|
|
232
242
|
constructor(modelId: OpenAITranscriptionModelId, config: OpenAITranscriptionModelConfig);
|
|
233
243
|
private getArgs;
|
|
234
244
|
doGenerate(options: OpenAITranscriptionCallOptions): Promise<Awaited<ReturnType<TranscriptionModelV4['doGenerate']>>>;
|
|
245
|
+
doStream(options: OpenAITranscriptionStreamOptions): Promise<Awaited<ReturnType<NonNullable<TranscriptionModelV4['doStream']>>>>;
|
|
235
246
|
}
|
|
236
247
|
|
|
237
248
|
type OpenAISpeechModelId = 'tts-1' | 'tts-1-1106' | 'tts-1-hd' | 'tts-1-hd-1106' | 'gpt-4o-mini-tts' | 'gpt-4o-mini-tts-2025-03-20' | 'gpt-4o-mini-tts-2025-12-15' | (string & {});
|
|
@@ -1494,4 +1505,4 @@ declare const webSearchPreview: _ai_sdk_provider_utils.ProviderExecutedToolFacto
|
|
|
1494
1505
|
};
|
|
1495
1506
|
}, {}>;
|
|
1496
1507
|
|
|
1497
|
-
export { type ApplyPatchOperation, OpenAIChatLanguageModel, type OpenAIChatModelId, OpenAICompletionLanguageModel, type OpenAICompletionModelId, OpenAIEmbeddingModel, type OpenAIEmbeddingModelId, type OpenAIEmbeddingModelOptions, OpenAIImageModel, type OpenAIImageModelEditOptions, type OpenAIImageModelGenerationOptions, type OpenAIImageModelId, type OpenAIImageModelOptions, type OpenAILanguageModelChatOptions, type OpenAILanguageModelCompletionOptions, OpenAIResponsesLanguageModel, OpenAISpeechModel, type OpenAISpeechModelId, type OpenAISpeechModelOptions, type OpenAITranscriptionCallOptions, OpenAITranscriptionModel, type OpenAITranscriptionModelId, type OpenAITranscriptionModelOptions, type OpenaiResponsesCompactionProviderMetadata, type OpenaiResponsesProviderMetadata, type OpenaiResponsesReasoningProviderMetadata, type OpenaiResponsesSourceDocumentProviderMetadata, type OpenaiResponsesTextProviderMetadata, type ResponsesCompactionProviderMetadata, type ResponsesProviderMetadata, type ResponsesReasoningProviderMetadata, type ResponsesSourceDocumentProviderMetadata, type ResponsesTextProviderMetadata, applyPatch, applyPatchArgsSchema, applyPatchInputSchema, applyPatchOutputSchema, applyPatchToolFactory, codeInterpreter, codeInterpreterArgsSchema, codeInterpreterInputSchema, codeInterpreterOutputSchema, codeInterpreterToolFactory, fileSearch, fileSearchArgsSchema, fileSearchOutputSchema, hasDefaultResponseFormat, imageGeneration, imageGenerationArgsSchema, imageGenerationOutputSchema, modelMaxImagesPerCall, openAITranscriptionModelOptions, openaiEmbeddingModelOptions, openaiImageModelEditOptions, openaiImageModelGenerationOptions, openaiImageModelOptions, openaiLanguageModelChatOptions, openaiLanguageModelCompletionOptions, openaiSpeechModelOptionsSchema, webSearch, webSearchArgsSchema, webSearchOutputSchema, webSearchPreview, webSearchPreviewArgsSchema, webSearchPreviewInputSchema, webSearchToolFactory };
|
|
1508
|
+
export { type ApplyPatchOperation, OpenAIChatLanguageModel, type OpenAIChatModelId, OpenAICompletionLanguageModel, type OpenAICompletionModelId, OpenAIEmbeddingModel, type OpenAIEmbeddingModelId, type OpenAIEmbeddingModelOptions, OpenAIImageModel, type OpenAIImageModelEditOptions, type OpenAIImageModelGenerationOptions, type OpenAIImageModelId, type OpenAIImageModelOptions, type OpenAILanguageModelChatOptions, type OpenAILanguageModelCompletionOptions, OpenAIResponsesLanguageModel, OpenAISpeechModel, type OpenAISpeechModelId, type OpenAISpeechModelOptions, type OpenAITranscriptionCallOptions, OpenAITranscriptionModel, type OpenAITranscriptionModelId, type OpenAITranscriptionModelOptions, type OpenAITranscriptionStreamOptions, type OpenaiResponsesCompactionProviderMetadata, type OpenaiResponsesProviderMetadata, type OpenaiResponsesReasoningProviderMetadata, type OpenaiResponsesSourceDocumentProviderMetadata, type OpenaiResponsesTextProviderMetadata, type ResponsesCompactionProviderMetadata, type ResponsesProviderMetadata, type ResponsesReasoningProviderMetadata, type ResponsesSourceDocumentProviderMetadata, type ResponsesTextProviderMetadata, applyPatch, applyPatchArgsSchema, applyPatchInputSchema, applyPatchOutputSchema, applyPatchToolFactory, codeInterpreter, codeInterpreterArgsSchema, codeInterpreterInputSchema, codeInterpreterOutputSchema, codeInterpreterToolFactory, fileSearch, fileSearchArgsSchema, fileSearchOutputSchema, hasDefaultResponseFormat, imageGeneration, imageGenerationArgsSchema, imageGenerationOutputSchema, modelMaxImagesPerCall, openAITranscriptionModelOptions, openaiEmbeddingModelOptions, openaiImageModelEditOptions, openaiImageModelGenerationOptions, openaiImageModelOptions, openaiLanguageModelChatOptions, openaiLanguageModelCompletionOptions, openaiSpeechModelOptionsSchema, webSearch, webSearchArgsSchema, webSearchOutputSchema, webSearchPreview, webSearchPreviewArgsSchema, webSearchPreviewInputSchema, webSearchToolFactory };
|
package/dist/internal/index.js
CHANGED
|
@@ -296,7 +296,7 @@ function convertToOpenAIChatMessages({
|
|
|
296
296
|
return {
|
|
297
297
|
type: "image_url",
|
|
298
298
|
image_url: {
|
|
299
|
-
url: part.data.type === "url" ? part.data.url.toString() : convertToBase64(part.data.data)
|
|
299
|
+
url: part.data.type === "url" ? part.data.url.toString() : `data:${resolveFullMediaType({ part })};base64,${convertToBase64(part.data.data)}`,
|
|
300
300
|
detail: (_b = (_a2 = part.providerOptions) == null ? void 0 : _a2.openai) == null ? void 0 : _b.imageDetail
|
|
301
301
|
}
|
|
302
302
|
};
|
|
@@ -2226,14 +2226,22 @@ async function fileToBlob(file) {
|
|
|
2226
2226
|
}
|
|
2227
2227
|
|
|
2228
2228
|
// src/transcription/openai-transcription-model.ts
|
|
2229
|
+
import {
|
|
2230
|
+
UnsupportedFunctionalityError as UnsupportedFunctionalityError4
|
|
2231
|
+
} from "@ai-sdk/provider";
|
|
2229
2232
|
import {
|
|
2230
2233
|
combineHeaders as combineHeaders5,
|
|
2231
2234
|
convertBase64ToUint8Array as convertBase64ToUint8Array2,
|
|
2235
|
+
convertToBase64 as convertToBase642,
|
|
2232
2236
|
createJsonResponseHandler as createJsonResponseHandler5,
|
|
2237
|
+
getWebSocketConstructor,
|
|
2233
2238
|
mediaTypeToExtension,
|
|
2234
2239
|
parseProviderOptions as parseProviderOptions5,
|
|
2235
2240
|
postFormDataToApi as postFormDataToApi2,
|
|
2241
|
+
readWebSocketMessageText,
|
|
2242
|
+
safeParseJSON,
|
|
2236
2243
|
serializeModelOptions as serializeModelOptions5,
|
|
2244
|
+
toWebSocketUrl,
|
|
2237
2245
|
WORKFLOW_DESERIALIZE as WORKFLOW_DESERIALIZE5,
|
|
2238
2246
|
WORKFLOW_SERIALIZE as WORKFLOW_SERIALIZE5
|
|
2239
2247
|
} from "@ai-sdk/provider-utils";
|
|
@@ -2302,12 +2310,28 @@ var openAITranscriptionModelOptions = lazySchema10(
|
|
|
2302
2310
|
* The timestamp granularities to populate for this transcription.
|
|
2303
2311
|
* @default ['segment']
|
|
2304
2312
|
*/
|
|
2305
|
-
timestampGranularities: z11.array(z11.enum(["word", "segment"])).default(["segment"]).optional()
|
|
2313
|
+
timestampGranularities: z11.array(z11.enum(["word", "segment"])).default(["segment"]).optional(),
|
|
2314
|
+
/**
|
|
2315
|
+
* Options for streaming transcription models such as `gpt-realtime-whisper`.
|
|
2316
|
+
*/
|
|
2317
|
+
streaming: z11.object({
|
|
2318
|
+
/**
|
|
2319
|
+
* Latency/accuracy tradeoff for realtime transcription.
|
|
2320
|
+
*/
|
|
2321
|
+
delay: z11.enum(["minimal", "low", "medium", "high", "xhigh"]).optional(),
|
|
2322
|
+
/**
|
|
2323
|
+
* Additional fields to include in realtime transcription events.
|
|
2324
|
+
*/
|
|
2325
|
+
include: z11.array(z11.string()).optional()
|
|
2326
|
+
}).optional()
|
|
2306
2327
|
})
|
|
2307
2328
|
)
|
|
2308
2329
|
);
|
|
2309
2330
|
|
|
2310
2331
|
// src/transcription/openai-transcription-model.ts
|
|
2332
|
+
function isRealtimeTranscriptionModelId(modelId) {
|
|
2333
|
+
return modelId === "gpt-realtime-whisper" || modelId.startsWith("gpt-realtime-whisper-");
|
|
2334
|
+
}
|
|
2311
2335
|
var languageMap = {
|
|
2312
2336
|
afrikaans: "af",
|
|
2313
2337
|
arabic: "ar",
|
|
@@ -2444,6 +2468,11 @@ var OpenAITranscriptionModel = class _OpenAITranscriptionModel {
|
|
|
2444
2468
|
}
|
|
2445
2469
|
async doGenerate(options) {
|
|
2446
2470
|
var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j;
|
|
2471
|
+
if (isRealtimeTranscriptionModelId(this.modelId)) {
|
|
2472
|
+
throw new UnsupportedFunctionalityError4({
|
|
2473
|
+
functionality: `non-streaming transcription with ${this.modelId}`
|
|
2474
|
+
});
|
|
2475
|
+
}
|
|
2447
2476
|
const currentDate = (_c = (_b = (_a = this.config._internal) == null ? void 0 : _a.currentDate) == null ? void 0 : _b.call(_a)) != null ? _c : /* @__PURE__ */ new Date();
|
|
2448
2477
|
const { formData, warnings } = await this.getArgs(options);
|
|
2449
2478
|
const {
|
|
@@ -2487,7 +2516,235 @@ var OpenAITranscriptionModel = class _OpenAITranscriptionModel {
|
|
|
2487
2516
|
}
|
|
2488
2517
|
};
|
|
2489
2518
|
}
|
|
2519
|
+
async doStream(options) {
|
|
2520
|
+
var _a, _b, _c, _d, _e, _f, _g;
|
|
2521
|
+
if (!isRealtimeTranscriptionModelId(this.modelId)) {
|
|
2522
|
+
throw new UnsupportedFunctionalityError4({
|
|
2523
|
+
functionality: `streaming transcription with ${this.modelId}`
|
|
2524
|
+
});
|
|
2525
|
+
}
|
|
2526
|
+
const currentDate = (_c = (_b = (_a = this.config._internal) == null ? void 0 : _a.currentDate) == null ? void 0 : _b.call(_a)) != null ? _c : /* @__PURE__ */ new Date();
|
|
2527
|
+
const openAIOptions = await parseProviderOptions5({
|
|
2528
|
+
provider: "openai",
|
|
2529
|
+
providerOptions: options.providerOptions,
|
|
2530
|
+
schema: openAITranscriptionModelOptions
|
|
2531
|
+
});
|
|
2532
|
+
const warnings = [];
|
|
2533
|
+
const rawOpenAIOptions = (_e = (_d = options.providerOptions) == null ? void 0 : _d.openai) != null ? _e : {};
|
|
2534
|
+
for (const option of [
|
|
2535
|
+
"include",
|
|
2536
|
+
"prompt",
|
|
2537
|
+
"temperature",
|
|
2538
|
+
"timestampGranularities"
|
|
2539
|
+
]) {
|
|
2540
|
+
if (rawOpenAIOptions[option] != null) {
|
|
2541
|
+
warnings.push({
|
|
2542
|
+
type: "unsupported",
|
|
2543
|
+
feature: `providerOptions.openai.${option}`,
|
|
2544
|
+
details: `OpenAI streaming transcription does not support ${option}.`
|
|
2545
|
+
});
|
|
2546
|
+
}
|
|
2547
|
+
}
|
|
2548
|
+
const headers = combineHeaders5((_g = (_f = this.config).headers) == null ? void 0 : _g.call(_f), options.headers);
|
|
2549
|
+
const sessionUpdate = buildOpenAIRealtimeTranscriptionSession({
|
|
2550
|
+
modelId: this.modelId,
|
|
2551
|
+
inputAudioFormat: options.inputAudioFormat,
|
|
2552
|
+
providerOptions: openAIOptions
|
|
2553
|
+
});
|
|
2554
|
+
return {
|
|
2555
|
+
request: { body: sessionUpdate },
|
|
2556
|
+
response: {
|
|
2557
|
+
timestamp: currentDate,
|
|
2558
|
+
modelId: this.modelId
|
|
2559
|
+
},
|
|
2560
|
+
stream: createOpenAIRealtimeTranscriptionStream({
|
|
2561
|
+
webSocket: this.config.webSocket,
|
|
2562
|
+
url: toWebSocketUrl(
|
|
2563
|
+
this.config.url({
|
|
2564
|
+
path: "/realtime?intent=transcription",
|
|
2565
|
+
modelId: this.modelId
|
|
2566
|
+
})
|
|
2567
|
+
),
|
|
2568
|
+
headers,
|
|
2569
|
+
sessionUpdate,
|
|
2570
|
+
language: openAIOptions == null ? void 0 : openAIOptions.language,
|
|
2571
|
+
warnings,
|
|
2572
|
+
audio: options.audio,
|
|
2573
|
+
abortSignal: options.abortSignal,
|
|
2574
|
+
includeRawChunks: options.includeRawChunks
|
|
2575
|
+
})
|
|
2576
|
+
};
|
|
2577
|
+
}
|
|
2490
2578
|
};
|
|
2579
|
+
function createOpenAIRealtimeTranscriptionStream({
|
|
2580
|
+
webSocket,
|
|
2581
|
+
url,
|
|
2582
|
+
headers,
|
|
2583
|
+
sessionUpdate,
|
|
2584
|
+
language,
|
|
2585
|
+
warnings,
|
|
2586
|
+
audio,
|
|
2587
|
+
abortSignal,
|
|
2588
|
+
includeRawChunks
|
|
2589
|
+
}) {
|
|
2590
|
+
let finished = false;
|
|
2591
|
+
let cleanup = () => {
|
|
2592
|
+
};
|
|
2593
|
+
return new ReadableStream({
|
|
2594
|
+
start: (controller) => {
|
|
2595
|
+
const WebSocketConstructor = getWebSocketConstructor(webSocket);
|
|
2596
|
+
const ws = new WebSocketConstructor(
|
|
2597
|
+
url,
|
|
2598
|
+
getOpenAIRealtimeProtocols(headers),
|
|
2599
|
+
{ headers }
|
|
2600
|
+
);
|
|
2601
|
+
let audioReader;
|
|
2602
|
+
cleanup = (closeCode) => {
|
|
2603
|
+
abortSignal == null ? void 0 : abortSignal.removeEventListener("abort", abort);
|
|
2604
|
+
void (audioReader == null ? void 0 : audioReader.cancel().catch(() => {
|
|
2605
|
+
}));
|
|
2606
|
+
try {
|
|
2607
|
+
ws.close(closeCode);
|
|
2608
|
+
} catch (e) {
|
|
2609
|
+
}
|
|
2610
|
+
};
|
|
2611
|
+
const finishWithError = (error) => {
|
|
2612
|
+
if (finished) return;
|
|
2613
|
+
finished = true;
|
|
2614
|
+
cleanup();
|
|
2615
|
+
controller.error(error);
|
|
2616
|
+
};
|
|
2617
|
+
const finish = (text, id) => {
|
|
2618
|
+
if (finished) return;
|
|
2619
|
+
finished = true;
|
|
2620
|
+
if (id != null) {
|
|
2621
|
+
controller.enqueue({ type: "transcript-final", id, text });
|
|
2622
|
+
}
|
|
2623
|
+
controller.enqueue({
|
|
2624
|
+
type: "finish",
|
|
2625
|
+
text,
|
|
2626
|
+
segments: [],
|
|
2627
|
+
language
|
|
2628
|
+
});
|
|
2629
|
+
controller.close();
|
|
2630
|
+
cleanup(1e3);
|
|
2631
|
+
};
|
|
2632
|
+
const abort = () => {
|
|
2633
|
+
var _a;
|
|
2634
|
+
finishWithError((_a = abortSignal == null ? void 0 : abortSignal.reason) != null ? _a : new Error("Aborted"));
|
|
2635
|
+
};
|
|
2636
|
+
if (abortSignal == null ? void 0 : abortSignal.aborted) {
|
|
2637
|
+
abort();
|
|
2638
|
+
return;
|
|
2639
|
+
}
|
|
2640
|
+
abortSignal == null ? void 0 : abortSignal.addEventListener("abort", abort, { once: true });
|
|
2641
|
+
const sendAudio = async () => {
|
|
2642
|
+
audioReader = audio.getReader();
|
|
2643
|
+
try {
|
|
2644
|
+
while (true) {
|
|
2645
|
+
const { done, value } = await audioReader.read();
|
|
2646
|
+
if (done || finished) break;
|
|
2647
|
+
ws.send(
|
|
2648
|
+
JSON.stringify({
|
|
2649
|
+
type: "input_audio_buffer.append",
|
|
2650
|
+
audio: convertToBase642(value)
|
|
2651
|
+
})
|
|
2652
|
+
);
|
|
2653
|
+
}
|
|
2654
|
+
} finally {
|
|
2655
|
+
audioReader.releaseLock();
|
|
2656
|
+
}
|
|
2657
|
+
if (!finished) {
|
|
2658
|
+
ws.send(JSON.stringify({ type: "input_audio_buffer.commit" }));
|
|
2659
|
+
}
|
|
2660
|
+
};
|
|
2661
|
+
ws.onopen = () => {
|
|
2662
|
+
controller.enqueue({ type: "stream-start", warnings });
|
|
2663
|
+
ws.send(JSON.stringify(sessionUpdate));
|
|
2664
|
+
void sendAudio().catch(finishWithError);
|
|
2665
|
+
};
|
|
2666
|
+
ws.onmessage = (event) => {
|
|
2667
|
+
void readWebSocketMessageText(event.data).then(async (text) => {
|
|
2668
|
+
var _a, _b, _c, _d;
|
|
2669
|
+
const parsed = await safeParseJSON({ text });
|
|
2670
|
+
if (!parsed.success) return;
|
|
2671
|
+
const raw = parsed.value;
|
|
2672
|
+
if (includeRawChunks) {
|
|
2673
|
+
controller.enqueue({ type: "raw", rawValue: raw });
|
|
2674
|
+
}
|
|
2675
|
+
switch (raw.type) {
|
|
2676
|
+
case "conversation.item.input_audio_transcription.delta": {
|
|
2677
|
+
controller.enqueue({
|
|
2678
|
+
type: "transcript-delta",
|
|
2679
|
+
id: raw.item_id,
|
|
2680
|
+
delta: (_a = raw.delta) != null ? _a : ""
|
|
2681
|
+
});
|
|
2682
|
+
break;
|
|
2683
|
+
}
|
|
2684
|
+
case "conversation.item.input_audio_transcription.completed": {
|
|
2685
|
+
finish((_b = raw.transcript) != null ? _b : "", raw.item_id);
|
|
2686
|
+
break;
|
|
2687
|
+
}
|
|
2688
|
+
case "error": {
|
|
2689
|
+
finishWithError(
|
|
2690
|
+
new Error((_d = (_c = raw.error) == null ? void 0 : _c.message) != null ? _d : "OpenAI realtime error")
|
|
2691
|
+
);
|
|
2692
|
+
break;
|
|
2693
|
+
}
|
|
2694
|
+
}
|
|
2695
|
+
}).catch(finishWithError);
|
|
2696
|
+
};
|
|
2697
|
+
ws.onerror = () => {
|
|
2698
|
+
finishWithError(new Error("OpenAI realtime transcription error"));
|
|
2699
|
+
};
|
|
2700
|
+
ws.onclose = () => {
|
|
2701
|
+
if (finished) return;
|
|
2702
|
+
finished = true;
|
|
2703
|
+
cleanup();
|
|
2704
|
+
controller.close();
|
|
2705
|
+
};
|
|
2706
|
+
},
|
|
2707
|
+
cancel: () => {
|
|
2708
|
+
if (finished) return;
|
|
2709
|
+
finished = true;
|
|
2710
|
+
cleanup();
|
|
2711
|
+
}
|
|
2712
|
+
});
|
|
2713
|
+
}
|
|
2714
|
+
function buildOpenAIRealtimeTranscriptionSession({
|
|
2715
|
+
modelId,
|
|
2716
|
+
inputAudioFormat,
|
|
2717
|
+
providerOptions
|
|
2718
|
+
}) {
|
|
2719
|
+
var _a, _b;
|
|
2720
|
+
return {
|
|
2721
|
+
type: "session.update",
|
|
2722
|
+
session: {
|
|
2723
|
+
type: "transcription",
|
|
2724
|
+
audio: {
|
|
2725
|
+
input: {
|
|
2726
|
+
format: {
|
|
2727
|
+
type: inputAudioFormat.type,
|
|
2728
|
+
...inputAudioFormat.rate != null ? { rate: inputAudioFormat.rate } : {}
|
|
2729
|
+
},
|
|
2730
|
+
transcription: {
|
|
2731
|
+
model: modelId,
|
|
2732
|
+
...(providerOptions == null ? void 0 : providerOptions.language) != null ? { language: providerOptions.language } : {},
|
|
2733
|
+
...((_a = providerOptions == null ? void 0 : providerOptions.streaming) == null ? void 0 : _a.delay) != null ? { delay: providerOptions.streaming.delay } : {}
|
|
2734
|
+
},
|
|
2735
|
+
turn_detection: null
|
|
2736
|
+
}
|
|
2737
|
+
},
|
|
2738
|
+
...((_b = providerOptions == null ? void 0 : providerOptions.streaming) == null ? void 0 : _b.include) != null ? { include: providerOptions.streaming.include } : {}
|
|
2739
|
+
}
|
|
2740
|
+
};
|
|
2741
|
+
}
|
|
2742
|
+
function getOpenAIRealtimeProtocols(headers) {
|
|
2743
|
+
var _a;
|
|
2744
|
+
const authorization = (_a = headers.Authorization) != null ? _a : headers.authorization;
|
|
2745
|
+
const token = (authorization == null ? void 0 : authorization.startsWith("Bearer ")) ? authorization.slice("Bearer ".length) : void 0;
|
|
2746
|
+
return token == null ? ["realtime"] : ["realtime", `openai-insecure-api-key.${token}`];
|
|
2747
|
+
}
|
|
2491
2748
|
|
|
2492
2749
|
// src/speech/openai-speech-model.ts
|
|
2493
2750
|
import {
|
|
@@ -2684,10 +2941,10 @@ function convertOpenAIResponsesUsage(usage) {
|
|
|
2684
2941
|
|
|
2685
2942
|
// src/responses/convert-to-openai-responses-input.ts
|
|
2686
2943
|
import {
|
|
2687
|
-
UnsupportedFunctionalityError as
|
|
2944
|
+
UnsupportedFunctionalityError as UnsupportedFunctionalityError5
|
|
2688
2945
|
} from "@ai-sdk/provider";
|
|
2689
2946
|
import {
|
|
2690
|
-
convertToBase64 as
|
|
2947
|
+
convertToBase64 as convertToBase643,
|
|
2691
2948
|
getTopLevelMediaType as getTopLevelMediaType2,
|
|
2692
2949
|
isNonNullable,
|
|
2693
2950
|
parseJSON,
|
|
@@ -2996,7 +3253,7 @@ async function convertToOpenAIResponsesInput({
|
|
|
2996
3253
|
};
|
|
2997
3254
|
}
|
|
2998
3255
|
case "text": {
|
|
2999
|
-
throw new
|
|
3256
|
+
throw new UnsupportedFunctionalityError5({
|
|
3000
3257
|
functionality: "text file parts"
|
|
3001
3258
|
});
|
|
3002
3259
|
}
|
|
@@ -3007,7 +3264,7 @@ async function convertToOpenAIResponsesInput({
|
|
|
3007
3264
|
return {
|
|
3008
3265
|
type: "input_image",
|
|
3009
3266
|
...part.data.type === "url" ? { image_url: part.data.url.toString() } : typeof part.data.data === "string" && isFileId(part.data.data, fileIdPrefixes) ? { file_id: part.data.data } : {
|
|
3010
|
-
image_url: `data:${resolveFullMediaType2({ part })};base64,${
|
|
3267
|
+
image_url: `data:${resolveFullMediaType2({ part })};base64,${convertToBase643(part.data.data)}`
|
|
3011
3268
|
},
|
|
3012
3269
|
detail: (_d2 = (_c2 = part.providerOptions) == null ? void 0 : _c2[providerOptionsName]) == null ? void 0 : _d2.imageDetail
|
|
3013
3270
|
};
|
|
@@ -3020,7 +3277,7 @@ async function convertToOpenAIResponsesInput({
|
|
|
3020
3277
|
}
|
|
3021
3278
|
const fullMediaType = resolveFullMediaType2({ part });
|
|
3022
3279
|
if (fullMediaType !== "application/pdf" && !passThroughUnsupportedFiles) {
|
|
3023
|
-
throw new
|
|
3280
|
+
throw new UnsupportedFunctionalityError5({
|
|
3024
3281
|
functionality: `file part media type ${fullMediaType}`
|
|
3025
3282
|
});
|
|
3026
3283
|
}
|
|
@@ -3028,7 +3285,7 @@ async function convertToOpenAIResponsesInput({
|
|
|
3028
3285
|
type: "input_file",
|
|
3029
3286
|
...typeof part.data.data === "string" && isFileId(part.data.data, fileIdPrefixes) ? { file_id: part.data.data } : {
|
|
3030
3287
|
filename: (_e2 = part.filename) != null ? _e2 : fullMediaType === "application/pdf" ? `part-${index}.pdf` : `part-${index}`,
|
|
3031
|
-
file_data: `data:${fullMediaType};base64,${
|
|
3288
|
+
file_data: `data:${fullMediaType};base64,${convertToBase643(part.data.data)}`
|
|
3032
3289
|
}
|
|
3033
3290
|
};
|
|
3034
3291
|
}
|
|
@@ -3461,14 +3718,14 @@ async function convertToOpenAIResponsesInput({
|
|
|
3461
3718
|
if (topLevel === "image") {
|
|
3462
3719
|
return {
|
|
3463
3720
|
type: "input_image",
|
|
3464
|
-
image_url: `data:${fullMediaType};base64,${
|
|
3721
|
+
image_url: `data:${fullMediaType};base64,${convertToBase643(item.data.data)}`,
|
|
3465
3722
|
detail: imageDetail
|
|
3466
3723
|
};
|
|
3467
3724
|
}
|
|
3468
3725
|
return {
|
|
3469
3726
|
type: "input_file",
|
|
3470
3727
|
filename: (_c2 = item.filename) != null ? _c2 : "data",
|
|
3471
|
-
file_data: `data:${fullMediaType};base64,${
|
|
3728
|
+
file_data: `data:${fullMediaType};base64,${convertToBase643(item.data.data)}`
|
|
3472
3729
|
};
|
|
3473
3730
|
}
|
|
3474
3731
|
if (item.data.type === "url") {
|
|
@@ -3539,14 +3796,14 @@ async function convertToOpenAIResponsesInput({
|
|
|
3539
3796
|
if (topLevel === "image") {
|
|
3540
3797
|
return {
|
|
3541
3798
|
type: "input_image",
|
|
3542
|
-
image_url: `data:${fullMediaType};base64,${
|
|
3799
|
+
image_url: `data:${fullMediaType};base64,${convertToBase643(item.data.data)}`,
|
|
3543
3800
|
detail: imageDetail
|
|
3544
3801
|
};
|
|
3545
3802
|
}
|
|
3546
3803
|
return {
|
|
3547
3804
|
type: "input_file",
|
|
3548
3805
|
filename: (_c2 = item.filename) != null ? _c2 : "data",
|
|
3549
|
-
file_data: `data:${fullMediaType};base64,${
|
|
3806
|
+
file_data: `data:${fullMediaType};base64,${convertToBase643(item.data.data)}`
|
|
3550
3807
|
};
|
|
3551
3808
|
}
|
|
3552
3809
|
if (item.data.type === "url") {
|
|
@@ -4789,7 +5046,7 @@ var openaiLanguageModelResponsesOptionsSchema = lazySchema17(
|
|
|
4789
5046
|
|
|
4790
5047
|
// src/responses/openai-responses-prepare-tools.ts
|
|
4791
5048
|
import {
|
|
4792
|
-
UnsupportedFunctionalityError as
|
|
5049
|
+
UnsupportedFunctionalityError as UnsupportedFunctionalityError6
|
|
4793
5050
|
} from "@ai-sdk/provider";
|
|
4794
5051
|
import {
|
|
4795
5052
|
resolveProviderReference as resolveProviderReference3,
|
|
@@ -5182,7 +5439,7 @@ async function prepareResponsesTools({
|
|
|
5182
5439
|
namespaceTools.set(namespace.name, namespaceTool);
|
|
5183
5440
|
openaiTools.push(namespaceTool);
|
|
5184
5441
|
} else if (namespaceTool.description !== namespace.description) {
|
|
5185
|
-
throw new
|
|
5442
|
+
throw new UnsupportedFunctionalityError6({
|
|
5186
5443
|
functionality: `conflicting descriptions for OpenAI tool namespace "${namespace.name}"`
|
|
5187
5444
|
});
|
|
5188
5445
|
}
|
|
@@ -5394,7 +5651,7 @@ async function prepareResponsesTools({
|
|
|
5394
5651
|
}
|
|
5395
5652
|
default: {
|
|
5396
5653
|
const _exhaustiveCheck = type;
|
|
5397
|
-
throw new
|
|
5654
|
+
throw new UnsupportedFunctionalityError6({
|
|
5398
5655
|
functionality: `tool choice type: ${_exhaustiveCheck}`
|
|
5399
5656
|
});
|
|
5400
5657
|
}
|