@ai-sdk/openai 4.0.107 → 4.0.109
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +38 -0
- package/dist/index.d.ts +67 -15
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +620 -378
- package/dist/index.js.map +1 -1
- package/dist/internal/index.d.ts +244 -4
- package/dist/internal/index.d.ts.map +1 -1
- package/dist/internal/index.js +481 -345
- package/dist/internal/index.js.map +1 -1
- package/docs/03-openai.mdx +156 -18
- package/package.json +3 -3
- package/src/chat/convert-to-openai-chat-messages.ts +29 -4
- package/src/chat/openai-chat-language-model-options.ts +13 -0
- package/src/chat/openai-chat-language-model.ts +10 -10
- package/src/chat/openai-chat-prompt.ts +4 -0
- package/src/image/openai-image-model-options.ts +48 -0
- package/src/image/openai-image-model.ts +49 -4
- package/src/index.ts +1 -0
- package/src/internal/index.ts +2 -0
- package/src/live/openai-realtime-model-live.ts +21 -2
- package/src/openai-model-catalog.ts +3 -1
- package/src/openai-tools.ts +4 -3
- package/src/realtime/openai-realtime-event-mapper.ts +172 -11
- package/src/realtime/openai-realtime-model.ts +57 -3
- package/src/responses/openai-responses-api.ts +19 -1
- package/src/responses/openai-responses-prepare-tools.ts +18 -1
- package/src/tool/code-interpreter.ts +38 -2
- package/src/tool/mcp.ts +27 -8
- package/src/transcription/openai-transcription-model.ts +43 -8
package/dist/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { EXPERIMENTAL_EMBEDDING_MODEL_MAX_INPUT_BYTES_PER_CALL, StreamingToolCallTracker, WORKFLOW_DESERIALIZE, WORKFLOW_SERIALIZE, combineHeaders, connectToWebSocket, convertAsyncIteratorToReadableStream, convertBase64ToUint8Array, convertInlineFileDataToUint8Array, convertToBase64, convertToFormData, convertUint8ArrayToBase64, createBinaryResponseHandler, createBinaryStreamResponseHandler, createEventSourceResponseHandler, createJsonErrorResponseHandler, createJsonLinesResponseHandler, createJsonResponseHandler, createLanguageModelResponseMetadata, createLanguageModelResponseMetadata as getResponseMetadata, createNullLanguageModelUsage, createProviderDefinedToolFactory, createProviderDefinedToolFactoryWithOutputSchema, createProviderExecutedToolFactory, createProviderStreamError, createToolNameMapping, deleteFromApi, detectMediaType, downloadBlob, encodePathSegment, experimental_toolCaller, generateId, getFromApi, getTopLevelMediaType, isCustomReasoning, isFullMediaType, isNonNullable, isRecord, lazySchema, loadApiKey, loadOptionalSetting, mapReasoningToProviderEffort, mediaTypeToExtension, normalizeBatchRequestCounts, parseJSON, parseProviderOptions, postFormDataToApi, postJsonToApi, postMultipartStreamToApi, postToApi, resolve, resolveFullMediaType, resolveProviderReference, resolveWith, safeParseJSON, safeValidateTypes, secureJsonParse, serializeModelOptions, stringifyWithBigIntAndCycles, toWebSocketUrl, validateBaseURL, validateTypes, waitForWebSocketBufferDrain, withUserAgentSuffix, withoutTrailingSlash, zodSchema } from "@ai-sdk/provider-utils";
|
|
1
|
+
import { EXPERIMENTAL_EMBEDDING_MODEL_MAX_INPUT_BYTES_PER_CALL, StreamingToolCallTracker, WORKFLOW_DESERIALIZE, WORKFLOW_SERIALIZE, combineHeaders, connectToWebSocket, convertAsyncIteratorToReadableStream, convertBase64ToUint8Array, convertInlineFileDataToUint8Array, convertToBase64, convertToFormData, convertUint8ArrayToBase64, createBinaryResponseHandler, createBinaryStreamResponseHandler, createEventSourceResponseHandler, createJsonErrorResponseHandler, createJsonLinesResponseHandler, createJsonResponseHandler, createLanguageModelResponseMetadata, createLanguageModelResponseMetadata as getResponseMetadata, createNullLanguageModelUsage, createProviderDefinedToolFactory, createProviderDefinedToolFactoryWithOutputSchema, createProviderExecutedToolFactory, createProviderStreamError, createToolNameMapping, deleteFromApi, detectMediaType, downloadBlob, encodePathSegment, experimental_toolCaller, generateId, getFromApi, getTopLevelMediaType, isCustomReasoning, isFullMediaType, isNonNullable, isRecord, lazySchema, loadApiKey, loadOptionalSetting, mapReasoningToProviderEffort, mediaTypeToExtension, normalizeBatchRequestCounts, parseJSON, parseProviderOptions, postFormDataToApi, postJsonToApi, postMultipartStreamToApi, postToApi, removeUndefinedEntries, resolve, resolveFullMediaType, resolveProviderReference, resolveWith, safeParseJSON, safeValidateTypes, secureJsonParse, serializeModelOptions, stringifyWithBigIntAndCycles, toWebSocketUrl, validateBaseURL, validateTypes, waitForWebSocketBufferDrain, withUserAgentSuffix, withoutTrailingSlash, zodSchema } from "@ai-sdk/provider-utils";
|
|
2
2
|
import { APICallError, InvalidArgumentError, InvalidPromptError, InvalidResponseDataError, TooManyEmbeddingValuesForCallError, UnsupportedFunctionalityError, isJSONObject } from "@ai-sdk/provider";
|
|
3
3
|
import { z } from "zod/v4";
|
|
4
4
|
//#region src/openai-config.ts
|
|
@@ -537,6 +537,8 @@ function createGptImageEntry(sizes) {
|
|
|
537
537
|
};
|
|
538
538
|
}
|
|
539
539
|
const openaiImageModelEntries = {
|
|
540
|
+
"gpt-image-2.6-flare": createGptImageEntry(customImageSizes),
|
|
541
|
+
"gpt-image-2.6-sunburst": createGptImageEntry(customImageSizes),
|
|
540
542
|
"gpt-image-2.5-flare": createGptImageEntry(customImageSizes),
|
|
541
543
|
"gpt-image-2.5-sunburst": createGptImageEntry(customImageSizes),
|
|
542
544
|
"gpt-image-2": createGptImageEntry(gptImageSizes),
|
|
@@ -725,7 +727,7 @@ const realtimeModelEntry = {
|
|
|
725
727
|
capabilities: { realtime: {
|
|
726
728
|
conversation: "turn-based",
|
|
727
729
|
transports: ["websocket"],
|
|
728
|
-
connections: ["client-secret-websocket"]
|
|
730
|
+
connections: ["client-secret-websocket", "server-websocket"]
|
|
729
731
|
} },
|
|
730
732
|
requestConfig: { realtime: {
|
|
731
733
|
api: "realtime",
|
|
@@ -1004,6 +1006,158 @@ function convertOpenAIChatUsage(usage) {
|
|
|
1004
1006
|
};
|
|
1005
1007
|
}
|
|
1006
1008
|
//#endregion
|
|
1009
|
+
//#region src/chat/openai-chat-language-model-options.ts
|
|
1010
|
+
const openaiLanguageModelChatOptions = lazySchema(() => zodSchema(z.object({
|
|
1011
|
+
/**
|
|
1012
|
+
* Modify the likelihood of specified tokens appearing in the completion.
|
|
1013
|
+
*
|
|
1014
|
+
* Accepts a JSON object that maps tokens (specified by their token ID in
|
|
1015
|
+
* the GPT tokenizer) to an associated bias value from -100 to 100.
|
|
1016
|
+
*/
|
|
1017
|
+
logitBias: z.record(z.coerce.number(), z.number()).optional(),
|
|
1018
|
+
/**
|
|
1019
|
+
* Return the log probabilities of the tokens.
|
|
1020
|
+
*
|
|
1021
|
+
* Setting to true will return the log probabilities of the tokens that
|
|
1022
|
+
* were generated.
|
|
1023
|
+
*
|
|
1024
|
+
* Setting to a number will return the log probabilities of the top n
|
|
1025
|
+
* tokens that were generated.
|
|
1026
|
+
*/
|
|
1027
|
+
logprobs: z.union([z.boolean(), z.number()]).optional(),
|
|
1028
|
+
/**
|
|
1029
|
+
* Whether to enable parallel function calling during tool use. Default to true.
|
|
1030
|
+
*/
|
|
1031
|
+
parallelToolCalls: z.boolean().optional(),
|
|
1032
|
+
/**
|
|
1033
|
+
* A unique identifier representing your end-user, which can help OpenAI to
|
|
1034
|
+
* monitor and detect abuse.
|
|
1035
|
+
*/
|
|
1036
|
+
user: z.string().optional(),
|
|
1037
|
+
/**
|
|
1038
|
+
* Reasoning effort for reasoning models. Defaults to `medium`.
|
|
1039
|
+
*/
|
|
1040
|
+
reasoningEffort: z.enum([
|
|
1041
|
+
"none",
|
|
1042
|
+
"minimal",
|
|
1043
|
+
"low",
|
|
1044
|
+
"medium",
|
|
1045
|
+
"high",
|
|
1046
|
+
"xhigh",
|
|
1047
|
+
"max"
|
|
1048
|
+
]).optional(),
|
|
1049
|
+
/**
|
|
1050
|
+
* Maximum number of completion tokens to generate. Useful for reasoning models.
|
|
1051
|
+
*/
|
|
1052
|
+
maxCompletionTokens: z.number().optional(),
|
|
1053
|
+
/**
|
|
1054
|
+
* Whether to enable persistence in responses API.
|
|
1055
|
+
*/
|
|
1056
|
+
store: z.boolean().optional(),
|
|
1057
|
+
/**
|
|
1058
|
+
* Metadata to associate with the request.
|
|
1059
|
+
*/
|
|
1060
|
+
metadata: z.record(z.string().max(64), z.string().max(512)).optional(),
|
|
1061
|
+
/**
|
|
1062
|
+
* Parameters for prediction mode.
|
|
1063
|
+
*/
|
|
1064
|
+
prediction: z.record(z.string(), z.any()).optional(),
|
|
1065
|
+
/**
|
|
1066
|
+
* Service tier for the request.
|
|
1067
|
+
* - 'auto': Default service tier. The request will be processed with the service tier configured in the
|
|
1068
|
+
* Project settings. Unless otherwise configured, the Project will use 'default'.
|
|
1069
|
+
* - 'flex': 50% cheaper processing at the cost of increased latency. Only available for o3 and o4-mini models.
|
|
1070
|
+
* - 'priority': Higher-speed processing with predictably low latency at premium cost. Available for Enterprise customers.
|
|
1071
|
+
* - 'fast': OpenAI's newer name for the 'priority' tier. Interchangeable with it.
|
|
1072
|
+
* - 'ultrafast': Access-controlled Ultrafast processing. Only available for gpt-5.6-sol.
|
|
1073
|
+
* - 'default': The request will be processed with the standard pricing and performance for the selected model.
|
|
1074
|
+
*
|
|
1075
|
+
* @default 'auto'
|
|
1076
|
+
*/
|
|
1077
|
+
serviceTier: z.enum([
|
|
1078
|
+
"auto",
|
|
1079
|
+
"flex",
|
|
1080
|
+
"priority",
|
|
1081
|
+
"fast",
|
|
1082
|
+
"ultrafast",
|
|
1083
|
+
"default"
|
|
1084
|
+
]).optional(),
|
|
1085
|
+
/**
|
|
1086
|
+
* Whether to use strict JSON schema validation.
|
|
1087
|
+
*
|
|
1088
|
+
* @default true
|
|
1089
|
+
*/
|
|
1090
|
+
strictJsonSchema: z.boolean().optional(),
|
|
1091
|
+
/**
|
|
1092
|
+
* Controls the verbosity of the model's responses.
|
|
1093
|
+
* Lower values will result in more concise responses, while higher values will result in more verbose responses.
|
|
1094
|
+
*/
|
|
1095
|
+
textVerbosity: z.enum([
|
|
1096
|
+
"low",
|
|
1097
|
+
"medium",
|
|
1098
|
+
"high"
|
|
1099
|
+
]).optional(),
|
|
1100
|
+
/**
|
|
1101
|
+
* A cache key for prompt caching. Allows manual control over prompt caching behavior.
|
|
1102
|
+
* Useful for improving cache hit rates and working around automatic caching issues.
|
|
1103
|
+
*/
|
|
1104
|
+
promptCacheKey: z.string().optional(),
|
|
1105
|
+
/**
|
|
1106
|
+
* Prompt cache behavior for GPT-5.6 and later models.
|
|
1107
|
+
* `mode` controls whether OpenAI also places an implicit breakpoint.
|
|
1108
|
+
* `ttl` sets the minimum cache lifetime and currently only supports 30 minutes.
|
|
1109
|
+
*/
|
|
1110
|
+
promptCacheOptions: z.object({
|
|
1111
|
+
mode: z.enum(["implicit", "explicit"]).optional(),
|
|
1112
|
+
ttl: z.literal("30m").optional()
|
|
1113
|
+
}).optional(),
|
|
1114
|
+
/**
|
|
1115
|
+
* The retention policy for the prompt cache.
|
|
1116
|
+
* - 'in_memory': Default. Standard prompt caching behavior.
|
|
1117
|
+
* - '24h': Extended prompt caching that keeps cached prefixes active for up to 24 hours.
|
|
1118
|
+
* Available for models before GPT-5.6 that support extended caching.
|
|
1119
|
+
*
|
|
1120
|
+
* @deprecated For GPT-5.6 and later models, use `promptCacheOptions.ttl`.
|
|
1121
|
+
*
|
|
1122
|
+
* @default 'in_memory'
|
|
1123
|
+
*/
|
|
1124
|
+
promptCacheRetention: z.enum(["in_memory", "24h"]).optional(),
|
|
1125
|
+
/**
|
|
1126
|
+
* A stable identifier used to help detect users of your application
|
|
1127
|
+
* that may be violating OpenAI's usage policies. The IDs should be a
|
|
1128
|
+
* string that uniquely identifies each user. We recommend hashing their
|
|
1129
|
+
* username or email address, in order to avoid sending us any identifying
|
|
1130
|
+
* information.
|
|
1131
|
+
*/
|
|
1132
|
+
safetyIdentifier: z.string().optional(),
|
|
1133
|
+
/**
|
|
1134
|
+
* Override the system message mode for this model.
|
|
1135
|
+
* - 'system': Use the 'system' role for system messages (default for most models)
|
|
1136
|
+
* - 'developer': Use the 'developer' role for system messages (used by reasoning models)
|
|
1137
|
+
* - 'remove': Remove system messages entirely
|
|
1138
|
+
*
|
|
1139
|
+
* If not specified, the mode is automatically determined based on the model.
|
|
1140
|
+
*/
|
|
1141
|
+
systemMessageMode: z.enum([
|
|
1142
|
+
"system",
|
|
1143
|
+
"developer",
|
|
1144
|
+
"remove"
|
|
1145
|
+
]).optional(),
|
|
1146
|
+
/**
|
|
1147
|
+
* Force treating this model as a reasoning model.
|
|
1148
|
+
*
|
|
1149
|
+
* This is useful for "stealth" reasoning models (e.g. via a custom baseURL)
|
|
1150
|
+
* where the model ID is not recognized by the SDK's allowlist.
|
|
1151
|
+
*
|
|
1152
|
+
* When enabled, the SDK applies reasoning-model parameter compatibility rules
|
|
1153
|
+
* and defaults `systemMessageMode` to `developer` unless overridden.
|
|
1154
|
+
*/
|
|
1155
|
+
forceReasoning: z.boolean().optional()
|
|
1156
|
+
})));
|
|
1157
|
+
const openaiChatMessageProviderOptions = lazySchema(() => zodSchema(z.object({
|
|
1158
|
+
/** The participant name on a system, developer, user, or assistant message. */
|
|
1159
|
+
name: z.string().optional() })));
|
|
1160
|
+
//#endregion
|
|
1007
1161
|
//#region src/chat/convert-to-openai-chat-messages.ts
|
|
1008
1162
|
function serializeToolCallArguments$1(input) {
|
|
1009
1163
|
if (typeof input === "string") try {
|
|
@@ -1018,197 +1172,213 @@ function serializeToolCallArguments$1(input) {
|
|
|
1018
1172
|
function getPromptCacheBreakpoint$1(providerOptions) {
|
|
1019
1173
|
return providerOptions?.openai?.promptCacheBreakpoint;
|
|
1020
1174
|
}
|
|
1021
|
-
function convertToOpenAIChatMessages({ prompt, systemMessageMode = "system" }) {
|
|
1175
|
+
async function convertToOpenAIChatMessages({ prompt, systemMessageMode = "system" }) {
|
|
1022
1176
|
const messages = [];
|
|
1023
1177
|
const warnings = [];
|
|
1024
|
-
for (const { role, content, providerOptions } of prompt)
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1178
|
+
for (const { role, content, providerOptions } of prompt) {
|
|
1179
|
+
const name = (await parseProviderOptions({
|
|
1180
|
+
provider: "openai",
|
|
1181
|
+
providerOptions,
|
|
1182
|
+
schema: openaiChatMessageProviderOptions
|
|
1183
|
+
}))?.name;
|
|
1184
|
+
switch (role) {
|
|
1185
|
+
case "system":
|
|
1186
|
+
switch (systemMessageMode) {
|
|
1187
|
+
case "system": {
|
|
1188
|
+
const promptCacheBreakpoint = getPromptCacheBreakpoint$1(providerOptions);
|
|
1189
|
+
messages.push({
|
|
1190
|
+
role: "system",
|
|
1191
|
+
...name != null && { name },
|
|
1192
|
+
content: promptCacheBreakpoint == null ? content : [{
|
|
1193
|
+
type: "text",
|
|
1194
|
+
text: content,
|
|
1195
|
+
prompt_cache_breakpoint: promptCacheBreakpoint
|
|
1196
|
+
}]
|
|
1197
|
+
});
|
|
1198
|
+
break;
|
|
1199
|
+
}
|
|
1200
|
+
case "developer": {
|
|
1201
|
+
const promptCacheBreakpoint = getPromptCacheBreakpoint$1(providerOptions);
|
|
1202
|
+
messages.push({
|
|
1203
|
+
role: "developer",
|
|
1204
|
+
...name != null && { name },
|
|
1205
|
+
content: promptCacheBreakpoint == null ? content : [{
|
|
1206
|
+
type: "text",
|
|
1207
|
+
text: content,
|
|
1208
|
+
prompt_cache_breakpoint: promptCacheBreakpoint
|
|
1209
|
+
}]
|
|
1210
|
+
});
|
|
1211
|
+
break;
|
|
1212
|
+
}
|
|
1213
|
+
case "remove":
|
|
1214
|
+
warnings.push({
|
|
1215
|
+
type: "other",
|
|
1216
|
+
message: "system messages are removed for this model"
|
|
1217
|
+
});
|
|
1218
|
+
break;
|
|
1219
|
+
default: throw new Error(`Unsupported system message mode: ${systemMessageMode}`);
|
|
1038
1220
|
}
|
|
1039
|
-
|
|
1040
|
-
|
|
1221
|
+
break;
|
|
1222
|
+
case "user":
|
|
1223
|
+
if (content.length === 1 && content[0].type === "text" && getPromptCacheBreakpoint$1(content[0].providerOptions) == null) {
|
|
1041
1224
|
messages.push({
|
|
1042
|
-
role: "
|
|
1043
|
-
content:
|
|
1044
|
-
|
|
1045
|
-
text: content,
|
|
1046
|
-
prompt_cache_breakpoint: promptCacheBreakpoint
|
|
1047
|
-
}]
|
|
1225
|
+
role: "user",
|
|
1226
|
+
content: content[0].text,
|
|
1227
|
+
...name != null && { name }
|
|
1048
1228
|
});
|
|
1049
1229
|
break;
|
|
1050
1230
|
}
|
|
1051
|
-
case "remove":
|
|
1052
|
-
warnings.push({
|
|
1053
|
-
type: "other",
|
|
1054
|
-
message: "system messages are removed for this model"
|
|
1055
|
-
});
|
|
1056
|
-
break;
|
|
1057
|
-
default: throw new Error(`Unsupported system message mode: ${systemMessageMode}`);
|
|
1058
|
-
}
|
|
1059
|
-
break;
|
|
1060
|
-
case "user":
|
|
1061
|
-
if (content.length === 1 && content[0].type === "text" && getPromptCacheBreakpoint$1(content[0].providerOptions) == null) {
|
|
1062
1231
|
messages.push({
|
|
1063
1232
|
role: "user",
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
case "text": {
|
|
1073
|
-
const promptCacheBreakpoint = getPromptCacheBreakpoint$1(part.providerOptions);
|
|
1074
|
-
return {
|
|
1075
|
-
type: "text",
|
|
1076
|
-
text: part.text,
|
|
1077
|
-
...promptCacheBreakpoint != null && { prompt_cache_breakpoint: promptCacheBreakpoint }
|
|
1078
|
-
};
|
|
1079
|
-
}
|
|
1080
|
-
case "file": {
|
|
1081
|
-
const promptCacheBreakpoint = getPromptCacheBreakpoint$1(part.providerOptions);
|
|
1082
|
-
switch (part.data.type) {
|
|
1083
|
-
case "reference": return {
|
|
1084
|
-
type: "file",
|
|
1085
|
-
file: { file_id: resolveProviderReference({
|
|
1086
|
-
reference: part.data.reference,
|
|
1087
|
-
provider: "openai"
|
|
1088
|
-
}) },
|
|
1233
|
+
...name != null && { name },
|
|
1234
|
+
content: content.map((part, index) => {
|
|
1235
|
+
switch (part.type) {
|
|
1236
|
+
case "text": {
|
|
1237
|
+
const promptCacheBreakpoint = getPromptCacheBreakpoint$1(part.providerOptions);
|
|
1238
|
+
return {
|
|
1239
|
+
type: "text",
|
|
1240
|
+
text: part.text,
|
|
1089
1241
|
...promptCacheBreakpoint != null && { prompt_cache_breakpoint: promptCacheBreakpoint }
|
|
1090
1242
|
};
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
type: "
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
},
|
|
1243
|
+
}
|
|
1244
|
+
case "file": {
|
|
1245
|
+
const promptCacheBreakpoint = getPromptCacheBreakpoint$1(part.providerOptions);
|
|
1246
|
+
switch (part.data.type) {
|
|
1247
|
+
case "reference": return {
|
|
1248
|
+
type: "file",
|
|
1249
|
+
file: { file_id: resolveProviderReference({
|
|
1250
|
+
reference: part.data.reference,
|
|
1251
|
+
provider: "openai"
|
|
1252
|
+
}) },
|
|
1101
1253
|
...promptCacheBreakpoint != null && { prompt_cache_breakpoint: promptCacheBreakpoint }
|
|
1102
1254
|
};
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1255
|
+
case "text": throw new UnsupportedFunctionalityError({ functionality: "text file parts" });
|
|
1256
|
+
case "url":
|
|
1257
|
+
case "data": {
|
|
1258
|
+
const topLevel = getTopLevelMediaType(part.mediaType);
|
|
1259
|
+
if (topLevel === "image") return {
|
|
1260
|
+
type: "image_url",
|
|
1261
|
+
image_url: {
|
|
1262
|
+
url: part.data.type === "url" ? part.data.url.toString() : `data:${resolveFullMediaType({ part })};base64,${convertToBase64(part.data.data)}`,
|
|
1263
|
+
detail: part.providerOptions?.openai?.imageDetail
|
|
1264
|
+
},
|
|
1265
|
+
...promptCacheBreakpoint != null && { prompt_cache_breakpoint: promptCacheBreakpoint }
|
|
1266
|
+
};
|
|
1267
|
+
else if (topLevel === "audio") {
|
|
1268
|
+
if (part.data.type === "url") throw new UnsupportedFunctionalityError({ functionality: "audio file parts with URLs" });
|
|
1269
|
+
const fullMediaType = resolveFullMediaType({ part });
|
|
1270
|
+
switch (fullMediaType) {
|
|
1271
|
+
case "audio/wav": return {
|
|
1272
|
+
type: "input_audio",
|
|
1273
|
+
input_audio: {
|
|
1274
|
+
data: convertToBase64(part.data.data),
|
|
1275
|
+
format: "wav"
|
|
1276
|
+
},
|
|
1277
|
+
...promptCacheBreakpoint != null && { prompt_cache_breakpoint: promptCacheBreakpoint }
|
|
1278
|
+
};
|
|
1279
|
+
case "audio/mp3":
|
|
1280
|
+
case "audio/mpeg": return {
|
|
1281
|
+
type: "input_audio",
|
|
1282
|
+
input_audio: {
|
|
1283
|
+
data: convertToBase64(part.data.data),
|
|
1284
|
+
format: "mp3"
|
|
1285
|
+
},
|
|
1286
|
+
...promptCacheBreakpoint != null && { prompt_cache_breakpoint: promptCacheBreakpoint }
|
|
1287
|
+
};
|
|
1288
|
+
default: throw new UnsupportedFunctionalityError({ functionality: `audio content parts with media type ${fullMediaType}` });
|
|
1289
|
+
}
|
|
1290
|
+
}
|
|
1291
|
+
{
|
|
1292
|
+
const fullMediaType = resolveFullMediaType({ part });
|
|
1293
|
+
if (fullMediaType !== "application/pdf") throw new UnsupportedFunctionalityError({ functionality: `file part media type ${fullMediaType}` });
|
|
1294
|
+
if (part.data.type === "url") throw new UnsupportedFunctionalityError({ functionality: "PDF file parts with URLs" });
|
|
1295
|
+
return {
|
|
1296
|
+
type: "file",
|
|
1297
|
+
file: {
|
|
1298
|
+
filename: part.filename ?? `part-${index}.pdf`,
|
|
1299
|
+
file_data: `data:application/pdf;base64,${convertToBase64(part.data.data)}`
|
|
1121
1300
|
},
|
|
1122
1301
|
...promptCacheBreakpoint != null && { prompt_cache_breakpoint: promptCacheBreakpoint }
|
|
1123
1302
|
};
|
|
1124
|
-
default: throw new UnsupportedFunctionalityError({ functionality: `audio content parts with media type ${fullMediaType}` });
|
|
1125
1303
|
}
|
|
1126
1304
|
}
|
|
1127
|
-
{
|
|
1128
|
-
const fullMediaType = resolveFullMediaType({ part });
|
|
1129
|
-
if (fullMediaType !== "application/pdf") throw new UnsupportedFunctionalityError({ functionality: `file part media type ${fullMediaType}` });
|
|
1130
|
-
if (part.data.type === "url") throw new UnsupportedFunctionalityError({ functionality: "PDF file parts with URLs" });
|
|
1131
|
-
return {
|
|
1132
|
-
type: "file",
|
|
1133
|
-
file: {
|
|
1134
|
-
filename: part.filename ?? `part-${index}.pdf`,
|
|
1135
|
-
file_data: `data:application/pdf;base64,${convertToBase64(part.data.data)}`
|
|
1136
|
-
},
|
|
1137
|
-
...promptCacheBreakpoint != null && { prompt_cache_breakpoint: promptCacheBreakpoint }
|
|
1138
|
-
};
|
|
1139
|
-
}
|
|
1140
1305
|
}
|
|
1141
1306
|
}
|
|
1142
1307
|
}
|
|
1308
|
+
})
|
|
1309
|
+
});
|
|
1310
|
+
break;
|
|
1311
|
+
case "assistant": {
|
|
1312
|
+
let text = "";
|
|
1313
|
+
const textParts = [];
|
|
1314
|
+
let hasPromptCacheBreakpoint = false;
|
|
1315
|
+
const toolCalls = [];
|
|
1316
|
+
for (const part of content) switch (part.type) {
|
|
1317
|
+
case "text": {
|
|
1318
|
+
const promptCacheBreakpoint = getPromptCacheBreakpoint$1(part.providerOptions);
|
|
1319
|
+
text += part.text;
|
|
1320
|
+
textParts.push({
|
|
1321
|
+
type: "text",
|
|
1322
|
+
text: part.text,
|
|
1323
|
+
...promptCacheBreakpoint != null && { prompt_cache_breakpoint: promptCacheBreakpoint }
|
|
1324
|
+
});
|
|
1325
|
+
hasPromptCacheBreakpoint ||= promptCacheBreakpoint != null;
|
|
1326
|
+
break;
|
|
1143
1327
|
}
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
textParts.
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1328
|
+
case "tool-call": toolCalls.push({
|
|
1329
|
+
id: part.toolCallId,
|
|
1330
|
+
type: "function",
|
|
1331
|
+
function: {
|
|
1332
|
+
name: part.toolName,
|
|
1333
|
+
arguments: serializeToolCallArguments$1(part.input)
|
|
1334
|
+
}
|
|
1335
|
+
});
|
|
1336
|
+
}
|
|
1337
|
+
messages.push({
|
|
1338
|
+
role: "assistant",
|
|
1339
|
+
...name != null && { name },
|
|
1340
|
+
content: hasPromptCacheBreakpoint ? textParts : toolCalls.length > 0 ? text || null : text,
|
|
1341
|
+
tool_calls: toolCalls.length > 0 ? toolCalls : void 0
|
|
1342
|
+
});
|
|
1343
|
+
break;
|
|
1344
|
+
}
|
|
1345
|
+
case "tool":
|
|
1346
|
+
if (name != null) warnings.push({
|
|
1347
|
+
type: "unsupported",
|
|
1348
|
+
feature: "message name on tool messages"
|
|
1349
|
+
});
|
|
1350
|
+
for (const toolResponse of content) {
|
|
1351
|
+
if (toolResponse.type === "tool-approval-response") continue;
|
|
1352
|
+
const output = toolResponse.output;
|
|
1353
|
+
const promptCacheBreakpoint = (output.type === "content" ? output.value.map((part) => getPromptCacheBreakpoint$1(part.providerOptions)).find((breakpoint) => breakpoint != null) : getPromptCacheBreakpoint$1(output.providerOptions)) ?? getPromptCacheBreakpoint$1(toolResponse.providerOptions);
|
|
1354
|
+
let contentValue;
|
|
1355
|
+
switch (output.type) {
|
|
1356
|
+
case "text":
|
|
1357
|
+
contentValue = output.value;
|
|
1358
|
+
break;
|
|
1359
|
+
case "error-text":
|
|
1360
|
+
case "error-json":
|
|
1361
|
+
contentValue = stringifyWithBigIntAndCycles({ error: output.value });
|
|
1362
|
+
break;
|
|
1363
|
+
case "execution-denied":
|
|
1364
|
+
contentValue = output.reason ?? "Tool call execution denied.";
|
|
1365
|
+
break;
|
|
1366
|
+
case "content":
|
|
1367
|
+
case "json": contentValue = stringifyWithBigIntAndCycles(output.value);
|
|
1368
|
+
}
|
|
1369
|
+
messages.push({
|
|
1370
|
+
role: "tool",
|
|
1371
|
+
tool_call_id: toolResponse.toolCallId,
|
|
1372
|
+
content: promptCacheBreakpoint == null ? contentValue : [{
|
|
1373
|
+
type: "text",
|
|
1374
|
+
text: contentValue,
|
|
1375
|
+
prompt_cache_breakpoint: promptCacheBreakpoint
|
|
1376
|
+
}]
|
|
1160
1377
|
});
|
|
1161
|
-
hasPromptCacheBreakpoint ||= promptCacheBreakpoint != null;
|
|
1162
|
-
break;
|
|
1163
1378
|
}
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
type: "function",
|
|
1167
|
-
function: {
|
|
1168
|
-
name: part.toolName,
|
|
1169
|
-
arguments: serializeToolCallArguments$1(part.input)
|
|
1170
|
-
}
|
|
1171
|
-
});
|
|
1172
|
-
}
|
|
1173
|
-
messages.push({
|
|
1174
|
-
role: "assistant",
|
|
1175
|
-
content: hasPromptCacheBreakpoint ? textParts : toolCalls.length > 0 ? text || null : text,
|
|
1176
|
-
tool_calls: toolCalls.length > 0 ? toolCalls : void 0
|
|
1177
|
-
});
|
|
1178
|
-
break;
|
|
1379
|
+
break;
|
|
1380
|
+
default: throw new Error(`Unsupported role: ${role}`);
|
|
1179
1381
|
}
|
|
1180
|
-
case "tool":
|
|
1181
|
-
for (const toolResponse of content) {
|
|
1182
|
-
if (toolResponse.type === "tool-approval-response") continue;
|
|
1183
|
-
const output = toolResponse.output;
|
|
1184
|
-
const promptCacheBreakpoint = (output.type === "content" ? output.value.map((part) => getPromptCacheBreakpoint$1(part.providerOptions)).find((breakpoint) => breakpoint != null) : getPromptCacheBreakpoint$1(output.providerOptions)) ?? getPromptCacheBreakpoint$1(toolResponse.providerOptions);
|
|
1185
|
-
let contentValue;
|
|
1186
|
-
switch (output.type) {
|
|
1187
|
-
case "text":
|
|
1188
|
-
contentValue = output.value;
|
|
1189
|
-
break;
|
|
1190
|
-
case "error-text":
|
|
1191
|
-
case "error-json":
|
|
1192
|
-
contentValue = stringifyWithBigIntAndCycles({ error: output.value });
|
|
1193
|
-
break;
|
|
1194
|
-
case "execution-denied":
|
|
1195
|
-
contentValue = output.reason ?? "Tool call execution denied.";
|
|
1196
|
-
break;
|
|
1197
|
-
case "content":
|
|
1198
|
-
case "json": contentValue = stringifyWithBigIntAndCycles(output.value);
|
|
1199
|
-
}
|
|
1200
|
-
messages.push({
|
|
1201
|
-
role: "tool",
|
|
1202
|
-
tool_call_id: toolResponse.toolCallId,
|
|
1203
|
-
content: promptCacheBreakpoint == null ? contentValue : [{
|
|
1204
|
-
type: "text",
|
|
1205
|
-
text: contentValue,
|
|
1206
|
-
prompt_cache_breakpoint: promptCacheBreakpoint
|
|
1207
|
-
}]
|
|
1208
|
-
});
|
|
1209
|
-
}
|
|
1210
|
-
break;
|
|
1211
|
-
default: throw new Error(`Unsupported role: ${role}`);
|
|
1212
1382
|
}
|
|
1213
1383
|
return {
|
|
1214
1384
|
messages,
|
|
@@ -1345,155 +1515,6 @@ const openaiChatChunkSchema = lazySchema(() => zodSchema(z.union([z.object({
|
|
|
1345
1515
|
}).nullish()
|
|
1346
1516
|
}), openaiErrorDataSchema()])));
|
|
1347
1517
|
//#endregion
|
|
1348
|
-
//#region src/chat/openai-chat-language-model-options.ts
|
|
1349
|
-
const openaiLanguageModelChatOptions = lazySchema(() => zodSchema(z.object({
|
|
1350
|
-
/**
|
|
1351
|
-
* Modify the likelihood of specified tokens appearing in the completion.
|
|
1352
|
-
*
|
|
1353
|
-
* Accepts a JSON object that maps tokens (specified by their token ID in
|
|
1354
|
-
* the GPT tokenizer) to an associated bias value from -100 to 100.
|
|
1355
|
-
*/
|
|
1356
|
-
logitBias: z.record(z.coerce.number(), z.number()).optional(),
|
|
1357
|
-
/**
|
|
1358
|
-
* Return the log probabilities of the tokens.
|
|
1359
|
-
*
|
|
1360
|
-
* Setting to true will return the log probabilities of the tokens that
|
|
1361
|
-
* were generated.
|
|
1362
|
-
*
|
|
1363
|
-
* Setting to a number will return the log probabilities of the top n
|
|
1364
|
-
* tokens that were generated.
|
|
1365
|
-
*/
|
|
1366
|
-
logprobs: z.union([z.boolean(), z.number()]).optional(),
|
|
1367
|
-
/**
|
|
1368
|
-
* Whether to enable parallel function calling during tool use. Default to true.
|
|
1369
|
-
*/
|
|
1370
|
-
parallelToolCalls: z.boolean().optional(),
|
|
1371
|
-
/**
|
|
1372
|
-
* A unique identifier representing your end-user, which can help OpenAI to
|
|
1373
|
-
* monitor and detect abuse.
|
|
1374
|
-
*/
|
|
1375
|
-
user: z.string().optional(),
|
|
1376
|
-
/**
|
|
1377
|
-
* Reasoning effort for reasoning models. Defaults to `medium`.
|
|
1378
|
-
*/
|
|
1379
|
-
reasoningEffort: z.enum([
|
|
1380
|
-
"none",
|
|
1381
|
-
"minimal",
|
|
1382
|
-
"low",
|
|
1383
|
-
"medium",
|
|
1384
|
-
"high",
|
|
1385
|
-
"xhigh",
|
|
1386
|
-
"max"
|
|
1387
|
-
]).optional(),
|
|
1388
|
-
/**
|
|
1389
|
-
* Maximum number of completion tokens to generate. Useful for reasoning models.
|
|
1390
|
-
*/
|
|
1391
|
-
maxCompletionTokens: z.number().optional(),
|
|
1392
|
-
/**
|
|
1393
|
-
* Whether to enable persistence in responses API.
|
|
1394
|
-
*/
|
|
1395
|
-
store: z.boolean().optional(),
|
|
1396
|
-
/**
|
|
1397
|
-
* Metadata to associate with the request.
|
|
1398
|
-
*/
|
|
1399
|
-
metadata: z.record(z.string().max(64), z.string().max(512)).optional(),
|
|
1400
|
-
/**
|
|
1401
|
-
* Parameters for prediction mode.
|
|
1402
|
-
*/
|
|
1403
|
-
prediction: z.record(z.string(), z.any()).optional(),
|
|
1404
|
-
/**
|
|
1405
|
-
* Service tier for the request.
|
|
1406
|
-
* - 'auto': Default service tier. The request will be processed with the service tier configured in the
|
|
1407
|
-
* Project settings. Unless otherwise configured, the Project will use 'default'.
|
|
1408
|
-
* - 'flex': 50% cheaper processing at the cost of increased latency. Only available for o3 and o4-mini models.
|
|
1409
|
-
* - 'priority': Higher-speed processing with predictably low latency at premium cost. Available for Enterprise customers.
|
|
1410
|
-
* - 'fast': OpenAI's newer name for the 'priority' tier. Interchangeable with it.
|
|
1411
|
-
* - 'ultrafast': Access-controlled Ultrafast processing. Only available for gpt-5.6-sol.
|
|
1412
|
-
* - 'default': The request will be processed with the standard pricing and performance for the selected model.
|
|
1413
|
-
*
|
|
1414
|
-
* @default 'auto'
|
|
1415
|
-
*/
|
|
1416
|
-
serviceTier: z.enum([
|
|
1417
|
-
"auto",
|
|
1418
|
-
"flex",
|
|
1419
|
-
"priority",
|
|
1420
|
-
"fast",
|
|
1421
|
-
"ultrafast",
|
|
1422
|
-
"default"
|
|
1423
|
-
]).optional(),
|
|
1424
|
-
/**
|
|
1425
|
-
* Whether to use strict JSON schema validation.
|
|
1426
|
-
*
|
|
1427
|
-
* @default true
|
|
1428
|
-
*/
|
|
1429
|
-
strictJsonSchema: z.boolean().optional(),
|
|
1430
|
-
/**
|
|
1431
|
-
* Controls the verbosity of the model's responses.
|
|
1432
|
-
* Lower values will result in more concise responses, while higher values will result in more verbose responses.
|
|
1433
|
-
*/
|
|
1434
|
-
textVerbosity: z.enum([
|
|
1435
|
-
"low",
|
|
1436
|
-
"medium",
|
|
1437
|
-
"high"
|
|
1438
|
-
]).optional(),
|
|
1439
|
-
/**
|
|
1440
|
-
* A cache key for prompt caching. Allows manual control over prompt caching behavior.
|
|
1441
|
-
* Useful for improving cache hit rates and working around automatic caching issues.
|
|
1442
|
-
*/
|
|
1443
|
-
promptCacheKey: z.string().optional(),
|
|
1444
|
-
/**
|
|
1445
|
-
* Prompt cache behavior for GPT-5.6 and later models.
|
|
1446
|
-
* `mode` controls whether OpenAI also places an implicit breakpoint.
|
|
1447
|
-
* `ttl` sets the minimum cache lifetime and currently only supports 30 minutes.
|
|
1448
|
-
*/
|
|
1449
|
-
promptCacheOptions: z.object({
|
|
1450
|
-
mode: z.enum(["implicit", "explicit"]).optional(),
|
|
1451
|
-
ttl: z.literal("30m").optional()
|
|
1452
|
-
}).optional(),
|
|
1453
|
-
/**
|
|
1454
|
-
* The retention policy for the prompt cache.
|
|
1455
|
-
* - 'in_memory': Default. Standard prompt caching behavior.
|
|
1456
|
-
* - '24h': Extended prompt caching that keeps cached prefixes active for up to 24 hours.
|
|
1457
|
-
* Available for models before GPT-5.6 that support extended caching.
|
|
1458
|
-
*
|
|
1459
|
-
* @deprecated For GPT-5.6 and later models, use `promptCacheOptions.ttl`.
|
|
1460
|
-
*
|
|
1461
|
-
* @default 'in_memory'
|
|
1462
|
-
*/
|
|
1463
|
-
promptCacheRetention: z.enum(["in_memory", "24h"]).optional(),
|
|
1464
|
-
/**
|
|
1465
|
-
* A stable identifier used to help detect users of your application
|
|
1466
|
-
* that may be violating OpenAI's usage policies. The IDs should be a
|
|
1467
|
-
* string that uniquely identifies each user. We recommend hashing their
|
|
1468
|
-
* username or email address, in order to avoid sending us any identifying
|
|
1469
|
-
* information.
|
|
1470
|
-
*/
|
|
1471
|
-
safetyIdentifier: z.string().optional(),
|
|
1472
|
-
/**
|
|
1473
|
-
* Override the system message mode for this model.
|
|
1474
|
-
* - 'system': Use the 'system' role for system messages (default for most models)
|
|
1475
|
-
* - 'developer': Use the 'developer' role for system messages (used by reasoning models)
|
|
1476
|
-
* - 'remove': Remove system messages entirely
|
|
1477
|
-
*
|
|
1478
|
-
* If not specified, the mode is automatically determined based on the model.
|
|
1479
|
-
*/
|
|
1480
|
-
systemMessageMode: z.enum([
|
|
1481
|
-
"system",
|
|
1482
|
-
"developer",
|
|
1483
|
-
"remove"
|
|
1484
|
-
]).optional(),
|
|
1485
|
-
/**
|
|
1486
|
-
* Force treating this model as a reasoning model.
|
|
1487
|
-
*
|
|
1488
|
-
* This is useful for "stealth" reasoning models (e.g. via a custom baseURL)
|
|
1489
|
-
* where the model ID is not recognized by the SDK's allowlist.
|
|
1490
|
-
*
|
|
1491
|
-
* When enabled, the SDK applies reasoning-model parameter compatibility rules
|
|
1492
|
-
* and defaults `systemMessageMode` to `developer` unless overridden.
|
|
1493
|
-
*/
|
|
1494
|
-
forceReasoning: z.boolean().optional()
|
|
1495
|
-
})));
|
|
1496
|
-
//#endregion
|
|
1497
1518
|
//#region src/normalize-openai-json-schema.ts
|
|
1498
1519
|
/**
|
|
1499
1520
|
* Normalizes JSON Schema for OpenAI structured outputs.
|
|
@@ -1732,7 +1753,7 @@ var OpenAIChatLanguageModel = class OpenAIChatLanguageModel {
|
|
|
1732
1753
|
feature: "reasoningSummary",
|
|
1733
1754
|
details: "reasoningSummary is only supported by the Responses API, not the Chat Completions API"
|
|
1734
1755
|
});
|
|
1735
|
-
const { messages, warnings: messageWarnings } = convertToOpenAIChatMessages({
|
|
1756
|
+
const { messages, warnings: messageWarnings } = await convertToOpenAIChatMessages({
|
|
1736
1757
|
prompt,
|
|
1737
1758
|
systemMessageMode: openaiOptions.systemMessageMode ?? (isReasoningModel ? "developer" : modelRequestConfig.systemMessageMode)
|
|
1738
1759
|
});
|
|
@@ -1885,11 +1906,12 @@ var OpenAIChatLanguageModel = class OpenAIChatLanguageModel {
|
|
|
1885
1906
|
}
|
|
1886
1907
|
async doGenerate(options) {
|
|
1887
1908
|
const { args: body, warnings } = await this.getArgs(options);
|
|
1909
|
+
const url = this.config.url({
|
|
1910
|
+
path: "/chat/completions",
|
|
1911
|
+
modelId: this.config.requestModelId ?? this.modelId
|
|
1912
|
+
});
|
|
1888
1913
|
const { responseHeaders, value: response, rawValue: rawResponse } = await postJsonToApi({
|
|
1889
|
-
url
|
|
1890
|
-
path: "/chat/completions",
|
|
1891
|
-
modelId: this.config.requestModelId ?? this.modelId
|
|
1892
|
-
}),
|
|
1914
|
+
url,
|
|
1893
1915
|
headers: combineHeaders(await resolve(this.config.headers), options.headers),
|
|
1894
1916
|
body,
|
|
1895
1917
|
failedResponseHandler: openaiFailedResponseHandler,
|
|
@@ -1933,7 +1955,10 @@ var OpenAIChatLanguageModel = class OpenAIChatLanguageModel {
|
|
|
1933
1955
|
raw: choice.finish_reason ?? void 0
|
|
1934
1956
|
},
|
|
1935
1957
|
usage: convertOpenAIChatUsage(response.usage),
|
|
1936
|
-
request: {
|
|
1958
|
+
request: {
|
|
1959
|
+
body,
|
|
1960
|
+
url
|
|
1961
|
+
},
|
|
1937
1962
|
response: {
|
|
1938
1963
|
...getResponseMetadata$1(response),
|
|
1939
1964
|
headers: responseHeaders,
|
|
@@ -2080,7 +2105,10 @@ var OpenAIChatLanguageModel = class OpenAIChatLanguageModel {
|
|
|
2080
2105
|
});
|
|
2081
2106
|
}
|
|
2082
2107
|
})),
|
|
2083
|
-
request: {
|
|
2108
|
+
request: {
|
|
2109
|
+
body,
|
|
2110
|
+
url
|
|
2111
|
+
},
|
|
2084
2112
|
response: { headers: responseHeaders }
|
|
2085
2113
|
};
|
|
2086
2114
|
}
|
|
@@ -2819,6 +2847,15 @@ const baseImageModelOptionsObject = () => z.object({
|
|
|
2819
2847
|
"webp"
|
|
2820
2848
|
]).optional(),
|
|
2821
2849
|
/**
|
|
2850
|
+
* How much detail the model puts into the image. Supported by GPT Image
|
|
2851
|
+
* 2.6 models. The API defaults to `high`.
|
|
2852
|
+
*/
|
|
2853
|
+
complexity: z.enum([
|
|
2854
|
+
"low",
|
|
2855
|
+
"medium",
|
|
2856
|
+
"high"
|
|
2857
|
+
]).optional(),
|
|
2858
|
+
/**
|
|
2822
2859
|
* Compression level (0-100) for the generated image(s). Applies to the
|
|
2823
2860
|
* `jpeg` and `webp` output formats.
|
|
2824
2861
|
*/
|
|
@@ -2827,7 +2864,13 @@ const baseImageModelOptionsObject = () => z.object({
|
|
|
2827
2864
|
* A unique identifier representing your end-user, which can help OpenAI
|
|
2828
2865
|
* to monitor and detect abuse.
|
|
2829
2866
|
*/
|
|
2830
|
-
user: z.string().optional()
|
|
2867
|
+
user: z.string().optional(),
|
|
2868
|
+
/**
|
|
2869
|
+
* A stable identifier used to help detect users of your application that
|
|
2870
|
+
* may be violating OpenAI's usage policies. Should be a hashed username
|
|
2871
|
+
* or email address.
|
|
2872
|
+
*/
|
|
2873
|
+
safetyIdentifier: z.string().max(128).optional()
|
|
2831
2874
|
});
|
|
2832
2875
|
lazySchema(() => zodSchema(baseImageModelOptionsObject()));
|
|
2833
2876
|
const openaiImageModelGenerationOptions = lazySchema(() => zodSchema(baseImageModelOptionsObject().extend({
|
|
@@ -2838,16 +2881,50 @@ const openaiImageModelGenerationOptions = lazySchema(() => zodSchema(baseImageMo
|
|
|
2838
2881
|
*/
|
|
2839
2882
|
style: z.enum(["vivid", "natural"]).optional(),
|
|
2840
2883
|
/**
|
|
2884
|
+
* Format in which the generated image(s) are returned. `gif` produces
|
|
2885
|
+
* an animated GIF and is supported by GPT Image 2.6 models.
|
|
2886
|
+
*/
|
|
2887
|
+
outputFormat: z.enum([
|
|
2888
|
+
"png",
|
|
2889
|
+
"jpeg",
|
|
2890
|
+
"webp",
|
|
2891
|
+
"gif"
|
|
2892
|
+
]).optional(),
|
|
2893
|
+
/**
|
|
2894
|
+
* Number of frames (2-64) of an animated GIF. Only applies to the `gif`
|
|
2895
|
+
* output format. The API defaults to 9.
|
|
2896
|
+
*/
|
|
2897
|
+
frameCount: z.number().int().min(2).max(64).optional(),
|
|
2898
|
+
/**
|
|
2899
|
+
* Duration of each frame in milliseconds (20-2000) of an animated GIF.
|
|
2900
|
+
* The API rounds it to the nearest 10 ms. Only applies to the `gif`
|
|
2901
|
+
* output format. The API defaults to 200.
|
|
2902
|
+
*/
|
|
2903
|
+
frameDurationMs: z.number().int().min(20).max(2e3).optional(),
|
|
2904
|
+
/**
|
|
2841
2905
|
* Content moderation level for the generated image(s). `low` applies
|
|
2842
2906
|
* less restrictive filtering.
|
|
2843
2907
|
*/
|
|
2844
2908
|
moderation: z.enum(["auto", "low"]).optional()
|
|
2845
2909
|
})));
|
|
2846
|
-
const openaiImageModelEditOptions = lazySchema(() => zodSchema(baseImageModelOptionsObject().extend({
|
|
2847
|
-
/**
|
|
2848
|
-
* Fidelity of the output image(s) to the input image(s).
|
|
2849
|
-
*/
|
|
2850
|
-
inputFidelity: z.enum(["high", "low"]).optional()
|
|
2910
|
+
const openaiImageModelEditOptions = lazySchema(() => zodSchema(baseImageModelOptionsObject().extend({
|
|
2911
|
+
/**
|
|
2912
|
+
* Fidelity of the output image(s) to the input image(s).
|
|
2913
|
+
*/
|
|
2914
|
+
inputFidelity: z.enum(["high", "low"]).optional(),
|
|
2915
|
+
/**
|
|
2916
|
+
* A transparent PNG overlaid on the first input image to mark up the
|
|
2917
|
+
* regions to change, as an HTTPS URL or a base64 data URL. It must match
|
|
2918
|
+
* the dimensions of the first image. Supported by GPT Image 2.5 and 2.6
|
|
2919
|
+
* models. Cannot be combined with a `mask`.
|
|
2920
|
+
*/
|
|
2921
|
+
markup: z.string().optional(),
|
|
2922
|
+
/**
|
|
2923
|
+
* Content moderation level for the edited image(s). `low` applies
|
|
2924
|
+
* less restrictive filtering. Supported by GPT Image models.
|
|
2925
|
+
*/
|
|
2926
|
+
moderation: z.enum(["auto", "low"]).optional()
|
|
2927
|
+
})));
|
|
2851
2928
|
//#endregion
|
|
2852
2929
|
//#region src/image/openai-image-model.ts
|
|
2853
2930
|
var OpenAIImageModel = class OpenAIImageModel {
|
|
@@ -2899,6 +2976,10 @@ var OpenAIImageModel = class OpenAIImageModel {
|
|
|
2899
2976
|
providerOptions,
|
|
2900
2977
|
schema: openaiImageModelEditOptions
|
|
2901
2978
|
}) ?? {};
|
|
2979
|
+
if (mask != null && openaiOptions.markup != null) throw new InvalidArgumentError({
|
|
2980
|
+
argument: "providerOptions.openai.markup",
|
|
2981
|
+
message: "`markup` cannot be combined with `mask`."
|
|
2982
|
+
});
|
|
2902
2983
|
const { value: response, responseHeaders } = await postFormDataToApi({
|
|
2903
2984
|
url: this.config.url({
|
|
2904
2985
|
path: "/images/edits",
|
|
@@ -2910,14 +2991,18 @@ var OpenAIImageModel = class OpenAIImageModel {
|
|
|
2910
2991
|
prompt,
|
|
2911
2992
|
image: await Promise.all(files.map((file) => file.type === "file" ? new Blob([file.data instanceof Uint8Array ? new Blob([file.data], { type: file.mediaType }) : new Blob([convertBase64ToUint8Array(file.data)], { type: file.mediaType })], { type: file.mediaType }) : downloadBlob(file.url, { abortSignal }))),
|
|
2912
2993
|
mask: mask != null ? await fileToBlob(mask, abortSignal) : void 0,
|
|
2994
|
+
markup: openaiOptions.markup != null ? await markupToBlob(openaiOptions.markup, abortSignal) : void 0,
|
|
2913
2995
|
n,
|
|
2914
2996
|
size,
|
|
2915
2997
|
quality: openaiOptions.quality,
|
|
2916
2998
|
background: openaiOptions.background,
|
|
2917
2999
|
output_format: openaiOptions.outputFormat,
|
|
2918
3000
|
output_compression: openaiOptions.outputCompression,
|
|
3001
|
+
complexity: openaiOptions.complexity,
|
|
2919
3002
|
input_fidelity: openaiOptions.inputFidelity,
|
|
2920
|
-
|
|
3003
|
+
moderation: openaiOptions.moderation,
|
|
3004
|
+
user: openaiOptions.user,
|
|
3005
|
+
safety_identifier: openaiOptions.safetyIdentifier
|
|
2921
3006
|
}),
|
|
2922
3007
|
failedResponseHandler: openaiFailedResponseHandler,
|
|
2923
3008
|
successfulResponseHandler: createJsonResponseHandler(openaiImageResponseSchema),
|
|
@@ -2970,7 +3055,11 @@ var OpenAIImageModel = class OpenAIImageModel {
|
|
|
2970
3055
|
moderation: openaiOptions.moderation,
|
|
2971
3056
|
output_format: openaiOptions.outputFormat,
|
|
2972
3057
|
output_compression: openaiOptions.outputCompression,
|
|
3058
|
+
complexity: openaiOptions.complexity,
|
|
3059
|
+
frame_count: openaiOptions.frameCount,
|
|
3060
|
+
frame_duration_ms: openaiOptions.frameDurationMs,
|
|
2973
3061
|
user: openaiOptions.user,
|
|
3062
|
+
safety_identifier: openaiOptions.safetyIdentifier,
|
|
2974
3063
|
...!hasDefaultResponseFormat(this.modelId) ? { response_format: "b64_json" } : {}
|
|
2975
3064
|
},
|
|
2976
3065
|
failedResponseHandler: openaiFailedResponseHandler,
|
|
@@ -3023,6 +3112,15 @@ function distributeTokenDetails(details, index, total) {
|
|
|
3023
3112
|
}
|
|
3024
3113
|
return result;
|
|
3025
3114
|
}
|
|
3115
|
+
async function markupToBlob(markup, abortSignal) {
|
|
3116
|
+
const dataUrl = /^data:([^;,]*)(;base64)?,(.*)$/s.exec(markup);
|
|
3117
|
+
if (dataUrl == null) return downloadBlob(markup, { abortSignal });
|
|
3118
|
+
if (dataUrl[2] == null) throw new InvalidArgumentError({
|
|
3119
|
+
argument: "providerOptions.openai.markup",
|
|
3120
|
+
message: "`markup` data URLs must be base64 encoded."
|
|
3121
|
+
});
|
|
3122
|
+
return new Blob([convertBase64ToUint8Array(dataUrl[3])], { type: dataUrl[1] || "image/png" });
|
|
3123
|
+
}
|
|
3026
3124
|
async function fileToBlob(file, abortSignal) {
|
|
3027
3125
|
if (!file) return void 0;
|
|
3028
3126
|
if (file.type === "url") return downloadBlob(file.url, { abortSignal });
|
|
@@ -3088,7 +3186,24 @@ const codeInterpreterOutputSchema = lazySchema(() => zodSchema(z.object({ output
|
|
|
3088
3186
|
type: z.literal("image"),
|
|
3089
3187
|
url: z.string()
|
|
3090
3188
|
})])).nullish() })));
|
|
3091
|
-
const codeInterpreterArgsSchema = lazySchema(() => zodSchema(z.object({ container: z.union([z.string(), z.object({
|
|
3189
|
+
const codeInterpreterArgsSchema = lazySchema(() => zodSchema(z.object({ container: z.union([z.string(), z.object({
|
|
3190
|
+
fileIds: z.array(z.string()).optional(),
|
|
3191
|
+
memoryLimit: z.enum([
|
|
3192
|
+
"1g",
|
|
3193
|
+
"4g",
|
|
3194
|
+
"16g",
|
|
3195
|
+
"64g"
|
|
3196
|
+
]).optional(),
|
|
3197
|
+
networkPolicy: z.discriminatedUnion("type", [z.object({ type: z.literal("disabled") }), z.object({
|
|
3198
|
+
type: z.literal("allowlist"),
|
|
3199
|
+
allowedDomains: z.array(z.string()),
|
|
3200
|
+
domainSecrets: z.array(z.object({
|
|
3201
|
+
domain: z.string(),
|
|
3202
|
+
name: z.string(),
|
|
3203
|
+
value: z.string()
|
|
3204
|
+
})).optional()
|
|
3205
|
+
})]).optional()
|
|
3206
|
+
})]).optional() })));
|
|
3092
3207
|
const codeInterpreterToolFactory = createProviderExecutedToolFactory({
|
|
3093
3208
|
id: "openai.code_interpreter",
|
|
3094
3209
|
inputSchema: codeInterpreterInputSchema,
|
|
@@ -3532,8 +3647,9 @@ const mcpArgsSchema = lazySchema(() => zodSchema(z.object({
|
|
|
3532
3647
|
headers: z.record(z.string(), z.string()).optional(),
|
|
3533
3648
|
requireApproval: z.union([z.enum(["always", "never"]), z.object({ never: z.object({ toolNames: z.array(z.string()).optional() }).optional() })]).optional(),
|
|
3534
3649
|
serverDescription: z.string().optional(),
|
|
3535
|
-
serverUrl: z.string().optional()
|
|
3536
|
-
|
|
3650
|
+
serverUrl: z.string().optional(),
|
|
3651
|
+
tunnelId: z.string().optional()
|
|
3652
|
+
}).refine((v) => v.serverUrl != null || v.connectorId != null || v.tunnelId != null, "One of serverUrl, connectorId, or tunnelId must be provided.").refine((v) => v.tunnelId == null || v.serverUrl == null && v.connectorId == null, "tunnelId cannot be combined with serverUrl or connectorId.")));
|
|
3537
3653
|
const mcpInputSchema = lazySchema(() => zodSchema(z.object({})));
|
|
3538
3654
|
const mcpOutputSchema = lazySchema(() => zodSchema(z.object({
|
|
3539
3655
|
type: z.literal("call"),
|
|
@@ -3684,16 +3800,17 @@ const openaiTools = {
|
|
|
3684
3800
|
webSearch,
|
|
3685
3801
|
/**
|
|
3686
3802
|
* MCP (Model Context Protocol) allows models to call tools exposed by
|
|
3687
|
-
* remote MCP servers or
|
|
3803
|
+
* remote MCP servers, service connectors, or Secure MCP tunnels.
|
|
3688
3804
|
*
|
|
3689
3805
|
* @param serverLabel - Label to identify the MCP server.
|
|
3690
3806
|
* @param allowedTools - Allowed tool names or filter object.
|
|
3691
|
-
* @param authorization - OAuth access token for the MCP server
|
|
3807
|
+
* @param authorization - OAuth access token for the MCP server or connector (including tunneled servers).
|
|
3692
3808
|
* @param connectorId - Identifier for a service connector.
|
|
3693
3809
|
* @param headers - Optional headers to include in MCP requests.
|
|
3694
|
-
*
|
|
3810
|
+
* @param requireApproval - Approval policy ('always'|'never'|filter object).
|
|
3695
3811
|
* @param serverDescription - Optional description of the server.
|
|
3696
3812
|
* @param serverUrl - URL for the MCP server.
|
|
3813
|
+
* @param tunnelId - Identifier for an OpenAI Secure MCP tunnel.
|
|
3697
3814
|
*/
|
|
3698
3815
|
mcp,
|
|
3699
3816
|
/**
|
|
@@ -6353,7 +6470,13 @@ async function prepareResponsesTools({ tools, toolChoice, allowedTools, toolName
|
|
|
6353
6470
|
file_ids: void 0
|
|
6354
6471
|
} : typeof args.container === "string" ? args.container : {
|
|
6355
6472
|
type: "auto",
|
|
6356
|
-
file_ids: args.container.fileIds
|
|
6473
|
+
file_ids: args.container.fileIds,
|
|
6474
|
+
memory_limit: args.container.memoryLimit,
|
|
6475
|
+
network_policy: args.container.networkPolicy == null ? void 0 : args.container.networkPolicy.type === "disabled" ? { type: "disabled" } : {
|
|
6476
|
+
type: "allowlist",
|
|
6477
|
+
allowed_domains: args.container.networkPolicy.allowedDomains,
|
|
6478
|
+
domain_secrets: args.container.networkPolicy.domainSecrets
|
|
6479
|
+
}
|
|
6357
6480
|
}
|
|
6358
6481
|
});
|
|
6359
6482
|
break;
|
|
@@ -6402,7 +6525,8 @@ async function prepareResponsesTools({ tools, toolChoice, allowedTools, toolName
|
|
|
6402
6525
|
headers: args.headers,
|
|
6403
6526
|
require_approval: requireApprovalParam ?? "never",
|
|
6404
6527
|
server_description: args.serverDescription,
|
|
6405
|
-
server_url: args.serverUrl
|
|
6528
|
+
server_url: args.serverUrl,
|
|
6529
|
+
tunnel_id: args.tunnelId
|
|
6406
6530
|
});
|
|
6407
6531
|
break;
|
|
6408
6532
|
}
|
|
@@ -9471,7 +9595,8 @@ var OpenAIRealtimeModelLive = class {
|
|
|
9471
9595
|
getWebRTCConfig() {
|
|
9472
9596
|
return { dataChannelLabel: "oai-events" };
|
|
9473
9597
|
}
|
|
9474
|
-
getServerWebSocketConfig() {
|
|
9598
|
+
getServerWebSocketConfig(options) {
|
|
9599
|
+
this.assertConversationIntent(options);
|
|
9475
9600
|
return resolveWith(this.config.headers, (configHeaders) => {
|
|
9476
9601
|
const url = new URL(`${this.config.baseURL}/live/sessions`);
|
|
9477
9602
|
url.protocol = url.protocol === "http:" ? "ws:" : "wss:";
|
|
@@ -9511,10 +9636,15 @@ var OpenAIRealtimeModelLive = class {
|
|
|
9511
9636
|
createServerEventParser() {
|
|
9512
9637
|
return createOpenAILiveServerEventParser();
|
|
9513
9638
|
}
|
|
9514
|
-
|
|
9639
|
+
assertConversationIntent(options) {
|
|
9640
|
+
if (options?.intent != null && options.intent !== "conversation") throw new Error(`OpenAI Live session intent '${options.intent}' is not supported.`);
|
|
9641
|
+
}
|
|
9642
|
+
serializeClientEvent(event, options) {
|
|
9643
|
+
this.assertConversationIntent(options);
|
|
9515
9644
|
return serializeOpenAILiveClientEvent(event, this.modelId);
|
|
9516
9645
|
}
|
|
9517
|
-
buildSessionConfig(config) {
|
|
9646
|
+
buildSessionConfig(config, options) {
|
|
9647
|
+
this.assertConversationIntent(options);
|
|
9518
9648
|
return buildOpenAILiveSessionConfig(config, this.modelId);
|
|
9519
9649
|
}
|
|
9520
9650
|
};
|
|
@@ -9558,23 +9688,31 @@ function parseOpenAIRealtimeServerEvent(raw) {
|
|
|
9558
9688
|
item: event.item,
|
|
9559
9689
|
raw
|
|
9560
9690
|
};
|
|
9561
|
-
case "conversation.item.input_audio_transcription.completed":
|
|
9562
|
-
|
|
9563
|
-
|
|
9564
|
-
|
|
9565
|
-
|
|
9566
|
-
|
|
9691
|
+
case "conversation.item.input_audio_transcription.completed": {
|
|
9692
|
+
const usage = mapTranscriptionUsage(event.usage);
|
|
9693
|
+
return {
|
|
9694
|
+
type: "input-transcription-completed",
|
|
9695
|
+
itemId: event.item_id,
|
|
9696
|
+
transcript: event.transcript ?? "",
|
|
9697
|
+
...usage != null ? { usage } : {},
|
|
9698
|
+
raw
|
|
9699
|
+
};
|
|
9700
|
+
}
|
|
9567
9701
|
case "response.created": return {
|
|
9568
9702
|
type: "response-created",
|
|
9569
9703
|
responseId: event.response?.id ?? event.response_id,
|
|
9570
9704
|
raw
|
|
9571
9705
|
};
|
|
9572
|
-
case "response.done":
|
|
9573
|
-
|
|
9574
|
-
|
|
9575
|
-
|
|
9576
|
-
|
|
9577
|
-
|
|
9706
|
+
case "response.done": {
|
|
9707
|
+
const usage = mapResponseUsage(event.response?.usage);
|
|
9708
|
+
return {
|
|
9709
|
+
type: "response-done",
|
|
9710
|
+
responseId: event.response?.id ?? event.response_id,
|
|
9711
|
+
status: event.response?.status ?? "completed",
|
|
9712
|
+
...usage != null ? { usage } : {},
|
|
9713
|
+
raw
|
|
9714
|
+
};
|
|
9715
|
+
}
|
|
9578
9716
|
case "response.output_item.added": return {
|
|
9579
9717
|
type: "output-item-added",
|
|
9580
9718
|
responseId: event.response_id,
|
|
@@ -9672,13 +9810,60 @@ function parseOpenAIRealtimeServerEvent(raw) {
|
|
|
9672
9810
|
}
|
|
9673
9811
|
}
|
|
9674
9812
|
/**
|
|
9813
|
+
* Maps OpenAI realtime `response.done` usage to normalized usage.
|
|
9814
|
+
*
|
|
9815
|
+
* Input buckets are reported gross (cache-inclusive) with the cached portion
|
|
9816
|
+
* surfaced separately, so consumers can apply their own cached-vs-uncached
|
|
9817
|
+
* billing split without re-reading `raw`.
|
|
9818
|
+
*/
|
|
9819
|
+
function mapResponseUsage(usage) {
|
|
9820
|
+
if (usage == null) return void 0;
|
|
9821
|
+
const input = usage.input_token_details;
|
|
9822
|
+
const output = usage.output_token_details;
|
|
9823
|
+
const cached = input?.cached_tokens_details;
|
|
9824
|
+
return compactUsage({
|
|
9825
|
+
inputTokens: usage.input_tokens,
|
|
9826
|
+
outputTokens: usage.output_tokens,
|
|
9827
|
+
cachedInputTokens: input?.cached_tokens,
|
|
9828
|
+
inputAudioTokens: input?.audio_tokens,
|
|
9829
|
+
inputTextTokens: input?.text_tokens,
|
|
9830
|
+
outputAudioTokens: output?.audio_tokens,
|
|
9831
|
+
outputTextTokens: output?.text_tokens,
|
|
9832
|
+
cachedInputAudioTokens: cached?.audio_tokens,
|
|
9833
|
+
cachedInputTextTokens: cached?.text_tokens
|
|
9834
|
+
});
|
|
9835
|
+
}
|
|
9836
|
+
/**
|
|
9837
|
+
* Maps OpenAI realtime transcription-completed usage to normalized usage.
|
|
9838
|
+
* Duration-billed transcription reports `seconds`; token-billed reports tokens.
|
|
9839
|
+
*/
|
|
9840
|
+
function mapTranscriptionUsage(usage) {
|
|
9841
|
+
if (usage == null) return void 0;
|
|
9842
|
+
if (usage.type === "duration") return usage.seconds != null ? { seconds: usage.seconds } : void 0;
|
|
9843
|
+
const input = usage.input_token_details;
|
|
9844
|
+
return compactUsage({
|
|
9845
|
+
inputTokens: usage.input_tokens,
|
|
9846
|
+
outputTokens: usage.output_tokens,
|
|
9847
|
+
inputAudioTokens: input?.audio_tokens,
|
|
9848
|
+
inputTextTokens: input?.text_tokens,
|
|
9849
|
+
outputTextTokens: usage.output_tokens
|
|
9850
|
+
});
|
|
9851
|
+
}
|
|
9852
|
+
/** Drop `undefined` fields; return `undefined` when nothing is observable. */
|
|
9853
|
+
function compactUsage(usage) {
|
|
9854
|
+
const result = {};
|
|
9855
|
+
for (const [key, value] of Object.entries(usage)) if (value != null) result[key] = value;
|
|
9856
|
+
return Object.keys(result).length > 0 ? result : void 0;
|
|
9857
|
+
}
|
|
9858
|
+
/**
|
|
9675
9859
|
* Serializes a normalized client event into OpenAI's Realtime API format.
|
|
9676
9860
|
*/
|
|
9677
|
-
function serializeOpenAIRealtimeClientEvent(event, modelId) {
|
|
9861
|
+
function serializeOpenAIRealtimeClientEvent(event, modelId, options) {
|
|
9862
|
+
assertSupportedOpenAIIntent(options?.intent);
|
|
9678
9863
|
switch (event.type) {
|
|
9679
9864
|
case "session-update": return {
|
|
9680
9865
|
type: "session.update",
|
|
9681
|
-
session: buildOpenAISessionConfig(event.config, modelId),
|
|
9866
|
+
session: buildOpenAISessionConfig(event.config, modelId, options),
|
|
9682
9867
|
...event.eventId != null ? { event_id: event.eventId } : {}
|
|
9683
9868
|
};
|
|
9684
9869
|
case "input-audio-append": return {
|
|
@@ -9744,15 +9929,27 @@ function serializeOpenAIRealtimeClientEvent(event, modelId) {
|
|
|
9744
9929
|
/**
|
|
9745
9930
|
* Builds an OpenAI-specific session configuration from a normalized config.
|
|
9746
9931
|
*/
|
|
9747
|
-
function buildOpenAISessionConfig(config, modelId) {
|
|
9932
|
+
function buildOpenAISessionConfig(config, modelId, options) {
|
|
9933
|
+
const intent = options?.intent ?? "conversation";
|
|
9934
|
+
assertSupportedOpenAIIntent(intent);
|
|
9935
|
+
if (intent === "transcription") {
|
|
9936
|
+
for (const key of [
|
|
9937
|
+
"instructions",
|
|
9938
|
+
"voice",
|
|
9939
|
+
"outputModalities",
|
|
9940
|
+
"outputAudioFormat",
|
|
9941
|
+
"outputAudioTranscription",
|
|
9942
|
+
"tools"
|
|
9943
|
+
]) if (config[key] != null) throw new Error(`OpenAI realtime transcription sessions do not support '${key}'.`);
|
|
9944
|
+
}
|
|
9748
9945
|
const session = {
|
|
9749
|
-
type: "realtime",
|
|
9750
|
-
model: modelId
|
|
9946
|
+
type: intent === "transcription" ? "transcription" : "realtime",
|
|
9947
|
+
...intent === "conversation" ? { model: modelId } : {}
|
|
9751
9948
|
};
|
|
9752
|
-
if (config.instructions != null) session.instructions = config.instructions;
|
|
9753
|
-
if (config.outputModalities != null) session.output_modalities = config.outputModalities;
|
|
9949
|
+
if (intent === "conversation" && config.instructions != null) session.instructions = config.instructions;
|
|
9950
|
+
if (intent === "conversation" && config.outputModalities != null) session.output_modalities = config.outputModalities;
|
|
9754
9951
|
const audio = {};
|
|
9755
|
-
if (config.inputAudioFormat != null || config.inputAudioTranscription != null || config.turnDetection != null) {
|
|
9952
|
+
if (intent === "transcription" || config.inputAudioFormat != null || config.inputAudioTranscription != null || config.turnDetection != null) {
|
|
9756
9953
|
const input = {};
|
|
9757
9954
|
if (config.inputAudioFormat != null) input.format = {
|
|
9758
9955
|
type: config.inputAudioFormat.type,
|
|
@@ -9768,14 +9965,19 @@ function buildOpenAISessionConfig(config, modelId) {
|
|
|
9768
9965
|
input.turn_detection = td;
|
|
9769
9966
|
}
|
|
9770
9967
|
}
|
|
9771
|
-
if (
|
|
9968
|
+
if (intent === "transcription") input.transcription = {
|
|
9969
|
+
model: config.inputAudioTranscription?.model ?? modelId,
|
|
9970
|
+
...config.inputAudioTranscription?.language != null ? { language: config.inputAudioTranscription.language } : {},
|
|
9971
|
+
...config.inputAudioTranscription?.prompt != null ? { prompt: config.inputAudioTranscription.prompt } : {}
|
|
9972
|
+
};
|
|
9973
|
+
else if (config.inputAudioTranscription != null) input.transcription = {
|
|
9772
9974
|
model: config.inputAudioTranscription.model ?? "gpt-realtime-whisper",
|
|
9773
9975
|
...config.inputAudioTranscription.language != null ? { language: config.inputAudioTranscription.language } : {},
|
|
9774
9976
|
...config.inputAudioTranscription.prompt != null ? { prompt: config.inputAudioTranscription.prompt } : {}
|
|
9775
9977
|
};
|
|
9776
9978
|
audio.input = input;
|
|
9777
9979
|
}
|
|
9778
|
-
if (config.outputAudioFormat != null || config.voice != null) {
|
|
9980
|
+
if (intent === "conversation" && (config.outputAudioFormat != null || config.voice != null)) {
|
|
9779
9981
|
const output = {};
|
|
9780
9982
|
if (config.outputAudioFormat != null) output.format = {
|
|
9781
9983
|
type: config.outputAudioFormat.type,
|
|
@@ -9785,7 +9987,7 @@ function buildOpenAISessionConfig(config, modelId) {
|
|
|
9785
9987
|
audio.output = output;
|
|
9786
9988
|
}
|
|
9787
9989
|
if (Object.keys(audio).length > 0) session.audio = audio;
|
|
9788
|
-
if (config.tools != null && config.tools.length > 0) {
|
|
9990
|
+
if (intent === "conversation" && config.tools != null && config.tools.length > 0) {
|
|
9789
9991
|
session.tools = config.tools.map((tool) => ({
|
|
9790
9992
|
type: tool.type,
|
|
9791
9993
|
name: tool.name,
|
|
@@ -9797,11 +9999,19 @@ function buildOpenAISessionConfig(config, modelId) {
|
|
|
9797
9999
|
if (config.providerOptions != null) Object.assign(session, config.providerOptions);
|
|
9798
10000
|
return session;
|
|
9799
10001
|
}
|
|
10002
|
+
function assertSupportedOpenAIIntent(intent) {
|
|
10003
|
+
if (intent != null && intent !== "conversation" && intent !== "transcription") throw new Error(`OpenAI realtime session intent '${intent}' is not supported.`);
|
|
10004
|
+
}
|
|
9800
10005
|
//#endregion
|
|
9801
10006
|
//#region src/realtime/openai-realtime-model.ts
|
|
9802
10007
|
var OpenAIRealtimeModel = class {
|
|
9803
10008
|
constructor(modelId, config) {
|
|
9804
10009
|
this.specificationVersion = "v4";
|
|
10010
|
+
this.capabilities = {
|
|
10011
|
+
conversation: "turn-based",
|
|
10012
|
+
transports: ["websocket"],
|
|
10013
|
+
connections: ["client-secret-websocket", "server-websocket"]
|
|
10014
|
+
};
|
|
9805
10015
|
this.modelId = modelId;
|
|
9806
10016
|
this.provider = config.provider;
|
|
9807
10017
|
this.config = config;
|
|
@@ -9844,16 +10054,36 @@ var OpenAIRealtimeModel = class {
|
|
|
9844
10054
|
protocols: ["realtime", `openai-insecure-api-key.${options.token}`]
|
|
9845
10055
|
};
|
|
9846
10056
|
}
|
|
10057
|
+
getServerWebSocketConfig(options) {
|
|
10058
|
+
return resolveWith(this.config.headers, (headers) => ({
|
|
10059
|
+
url: buildServerUrl(this.config.baseURL, this.modelId, options?.intent ?? "conversation"),
|
|
10060
|
+
headers: removeUndefinedEntries(headers)
|
|
10061
|
+
}));
|
|
10062
|
+
}
|
|
9847
10063
|
parseServerEvent(raw) {
|
|
9848
10064
|
return parseOpenAIRealtimeServerEvent(raw);
|
|
9849
10065
|
}
|
|
9850
|
-
serializeClientEvent(event) {
|
|
9851
|
-
return serializeOpenAIRealtimeClientEvent(event, this.modelId);
|
|
10066
|
+
serializeClientEvent(event, options) {
|
|
10067
|
+
return serializeOpenAIRealtimeClientEvent(event, this.modelId, options);
|
|
9852
10068
|
}
|
|
9853
|
-
buildSessionConfig(config) {
|
|
9854
|
-
return buildOpenAISessionConfig(config, this.modelId);
|
|
10069
|
+
buildSessionConfig(config, options) {
|
|
10070
|
+
return buildOpenAISessionConfig(config, this.modelId, options);
|
|
9855
10071
|
}
|
|
9856
10072
|
};
|
|
10073
|
+
/**
|
|
10074
|
+
* Builds the upstream realtime WebSocket URL for a server-initiated connection.
|
|
10075
|
+
* Transcription keeps the model in `session.update`; a `?model=` here would
|
|
10076
|
+
* open a voice session instead.
|
|
10077
|
+
*/
|
|
10078
|
+
function buildServerUrl(baseURL, modelId, intent) {
|
|
10079
|
+
const url = new URL(baseURL);
|
|
10080
|
+
const base = `${url.protocol === "http:" ? "ws:" : "wss:"}//${url.host}${url.pathname.replace(/\/$/, "")}`;
|
|
10081
|
+
switch (intent) {
|
|
10082
|
+
case "transcription": return `${base}/realtime?intent=transcription`;
|
|
10083
|
+
case "conversation": return `${base}/realtime?model=${encodeURIComponent(modelId)}`;
|
|
10084
|
+
default: throw new Error(`OpenAI realtime session intent '${intent}' is not supported.`);
|
|
10085
|
+
}
|
|
10086
|
+
}
|
|
9857
10087
|
//#endregion
|
|
9858
10088
|
//#region src/realtime/openai-realtime-factory.ts
|
|
9859
10089
|
function resolveRealtimeApi(modelId, { api } = {}) {
|
|
@@ -10180,7 +10410,7 @@ var OpenAITranscriptionModel = class OpenAITranscriptionModel {
|
|
|
10180
10410
|
this.config = config;
|
|
10181
10411
|
this.specificationVersion = "v4";
|
|
10182
10412
|
}
|
|
10183
|
-
async getArgs({ audio, mediaType, providerOptions }) {
|
|
10413
|
+
async getArgs({ audio, mediaType, providerOptions, timestampGranularities: requestedTimestampGranularities }) {
|
|
10184
10414
|
const warnings = [];
|
|
10185
10415
|
const openAIOptions = await parseProviderOptions({
|
|
10186
10416
|
provider: "openai",
|
|
@@ -10194,7 +10424,17 @@ var OpenAITranscriptionModel = class OpenAITranscriptionModel {
|
|
|
10194
10424
|
formData.append("file", new File([blob], "audio", { type: mediaType }), `audio.${fileExtension}`);
|
|
10195
10425
|
const { transcription: requestConfig } = getOpenAITranscriptionModelEntry(this.modelId).requestConfig;
|
|
10196
10426
|
const responseFormat = openAIOptions?.responseFormat ?? requestConfig.responseFormat;
|
|
10197
|
-
const timestampGranularities = openAIOptions?.timestampGranularities ?? (responseFormat === "verbose_json" ? ["segment"] : void 0);
|
|
10427
|
+
const timestampGranularities = openAIOptions?.timestampGranularities ?? requestedTimestampGranularities ?? (responseFormat === "verbose_json" ? ["segment"] : void 0);
|
|
10428
|
+
for (const granularity of requestedTimestampGranularities ?? []) if (openAIOptions?.timestampGranularities != null && !openAIOptions.timestampGranularities.includes(granularity)) warnings.push({
|
|
10429
|
+
type: "unsupported",
|
|
10430
|
+
feature: "timestampGranularities",
|
|
10431
|
+
details: `The "${granularity}" timestamp granularity is excluded by providerOptions.openai.timestampGranularities.`
|
|
10432
|
+
});
|
|
10433
|
+
for (const granularity of timestampGranularities ?? []) if (responseFormat !== "verbose_json" && !(responseFormat === "diarized_json" && granularity === "segment")) warnings.push({
|
|
10434
|
+
type: "unsupported",
|
|
10435
|
+
feature: openAIOptions?.timestampGranularities != null ? "providerOptions.openai.timestampGranularities" : "timestampGranularities",
|
|
10436
|
+
details: `OpenAI transcription with the ${responseFormat ?? "default JSON"} response format does not support ${granularity} timestamps.`
|
|
10437
|
+
});
|
|
10198
10438
|
const chunkingStrategy = openAIOptions?.chunkingStrategy ?? requestConfig.chunkingStrategy;
|
|
10199
10439
|
const transcriptionModelOptions = {
|
|
10200
10440
|
response_format: responseFormat,
|
|
@@ -10204,7 +10444,7 @@ var OpenAITranscriptionModel = class OpenAITranscriptionModel {
|
|
|
10204
10444
|
keywords: openAIOptions?.keywords,
|
|
10205
10445
|
prompt: openAIOptions?.prompt,
|
|
10206
10446
|
temperature: openAIOptions?.temperature,
|
|
10207
|
-
timestamp_granularities: timestampGranularities
|
|
10447
|
+
timestamp_granularities: responseFormat === "verbose_json" ? timestampGranularities : void 0
|
|
10208
10448
|
};
|
|
10209
10449
|
for (const [key, value] of Object.entries(transcriptionModelOptions)) if (value != null) {
|
|
10210
10450
|
if (Array.isArray(value)) for (const item of value) formData.append(`${key}[]`, String(item));
|
|
@@ -10244,17 +10484,19 @@ var OpenAITranscriptionModel = class OpenAITranscriptionModel {
|
|
|
10244
10484
|
endSecond: segment.end,
|
|
10245
10485
|
speaker: segment.speaker
|
|
10246
10486
|
}] : []);
|
|
10487
|
+
const words = response.words?.map((word) => ({
|
|
10488
|
+
text: word.word,
|
|
10489
|
+
startSecond: word.start,
|
|
10490
|
+
endSecond: word.end
|
|
10491
|
+
}));
|
|
10247
10492
|
return {
|
|
10248
10493
|
text: response.text,
|
|
10249
10494
|
segments: response.segments?.map((segment) => ({
|
|
10250
10495
|
text: segment.text,
|
|
10251
10496
|
startSecond: segment.start,
|
|
10252
10497
|
endSecond: segment.end
|
|
10253
|
-
})) ??
|
|
10254
|
-
|
|
10255
|
-
startSecond: word.start,
|
|
10256
|
-
endSecond: word.end
|
|
10257
|
-
})) ?? [],
|
|
10498
|
+
})) ?? words ?? [],
|
|
10499
|
+
...words != null && words.length > 0 ? { words } : {},
|
|
10258
10500
|
language,
|
|
10259
10501
|
durationInSeconds: response.duration ?? (response.usage?.type === "duration" && typeof response.usage.seconds === "number" ? response.usage.seconds : void 0),
|
|
10260
10502
|
warnings,
|
|
@@ -10767,7 +11009,7 @@ var OpenAISkills = class {
|
|
|
10767
11009
|
};
|
|
10768
11010
|
//#endregion
|
|
10769
11011
|
//#region src/version.ts
|
|
10770
|
-
const VERSION = "4.0.
|
|
11012
|
+
const VERSION = "4.0.109";
|
|
10771
11013
|
//#endregion
|
|
10772
11014
|
//#region src/openai-provider.ts
|
|
10773
11015
|
/**
|