@yourgpt/llm-sdk 2.6.0-beta.1 → 2.6.0-beta.3

package/dist/providers/openai/index.js

@@ -1,5 +1,108 @@
 'use strict';
 
+// src/core/tool-output.ts
+var imageTypes = /* @__PURE__ */ new Set([
+  "image/png",
+  "image/jpeg",
+  "image/webp",
+  "image/gif"
+]);
+function normalizeToolOutput(value) {
+  const output = value;
+  if (output?.type !== "content" || !Array.isArray(output.content) || !output.content.length) {
+    throw new Error("Tool model output must contain a nonempty content array");
+  }
+  return {
+    type: "content",
+    content: output.content.map((part) => {
+      if (part?.type === "text" && typeof part.text === "string")
+        return { type: "text", text: part.text };
+      if (part?.type !== "image")
+        throw new Error("Unsupported tool output content type");
+      const sources = ["data", "url", "fileId"].filter((key) => key in part);
+      if (sources.length !== 1)
+        throw new Error(
+          "Tool image requires exactly one of data, url, or fileId"
+        );
+      if ("fileId" in part) {
+        if (part.provider !== "openai" || typeof part.fileId !== "string" || !part.fileId.trim())
+          throw new Error(
+            "Tool image file references must name the openai provider and fileId"
+          );
+        return { type: "image", fileId: part.fileId, provider: "openai" };
+      }
+      if ("url" in part) {
+        let url;
+        try {
+          url = new URL(part.url);
+        } catch {
+          throw new Error("Invalid tool image URL");
+        }
+        if (!["https:", "http:"].includes(url.protocol) || url.username || url.password)
+          throw new Error(
+            "Tool image URL must use HTTP(S) without credentials"
+          );
+        return { type: "image", url: url.href };
+      }
+      if (!("data" in part) || !imageTypes.has(part.mediaType))
+        throw new Error("Unsupported tool image media type");
+      let data = part.data instanceof Uint8Array ? Buffer.from(part.data).toString("base64") : part.data;
+      if (typeof data !== "string")
+        throw new Error("Tool image data must be base64 or bytes");
+      if (data.startsWith("data:")) {
+        const match = /^data:([^;,]+);base64,(.+)$/.exec(data);
+        if (!match || match[1] !== part.mediaType)
+          throw new Error(
+            "Tool image data URL media type does not match mediaType"
+          );
+        data = match[2];
+      }
+      if (!data.length || data.length % 4 !== 0 || !/^[A-Za-z0-9+/]+={0,2}$/.test(data))
+        throw new Error("Invalid tool image base64 data");
+      if (Buffer.byteLength(data, "base64") > 20 * 1024 * 1024)
+        throw new Error("Tool image exceeds 20 MiB");
+      if (Buffer.from(data, "base64").toString("base64") !== data)
+        throw new Error("Invalid tool image base64 data");
+      return { type: "image", data, mediaType: part.mediaType };
+    })
+  };
+}
+function toolOutputForProvider(value, provider) {
+  const output = normalizeToolOutput(value);
+  return output.content.map((part) => {
+    if (part.type === "text")
+      return {
+        type: provider === "responses" ? "input_text" : "text",
+        text: part.text
+      };
+    if (provider === "unsupported")
+      throw new Error(
+        "Image tool results require OpenAI Responses or Anthropic; this endpoint is not supported"
+      );
+    if ("fileId" in part) {
+      if (provider !== "responses")
+        throw new Error(
+          "OpenAI image file references cannot be sent to another provider"
+        );
+      return { type: "input_image", file_id: part.fileId, detail: "auto" };
+    }
+    if (provider === "responses")
+      return {
+        type: "input_image",
+        image_url: "url" in part ? part.url : `data:${part.mediaType};base64,${part.data}`,
+        detail: "auto"
+      };
+    return {
+      type: "image",
+      source: "url" in part ? { type: "url", url: part.url } : { type: "base64", media_type: part.mediaType, data: part.data }
+    };
+  });
+}
+function textToolOutput(value, fallback) {
+  if (value === void 0) return fallback;
+  return toolOutputForProvider(value, "unsupported").map((part) => part.text).join("\n");
+}
+
 // src/core/openai-parameters.ts
 function capabilities(model) {
   const gpt = /^gpt-(\d+)(?:\.(\d+))?(?:-(.+))?$/i.exec(model);
@@ -51,7 +154,120 @@ function buildOpenAIParameters(model, maxTokens, temperature, options = {}, api
   return result;
 }
 
+// src/core/openai-responses.ts
+function resolveOpenAITransport(defaults, overrides) {
+  return {
+    api: overrides?.api ?? defaults.api ?? "auto",
+    responses: defaults.responses !== void 0 || overrides?.responses !== void 0 ? { ...defaults.responses, ...overrides?.responses } : void 0
+  };
+}
+function useOpenAIResponses(model, baseURL, parameters, transport2, hasTools, hasReplay, nativeToolSearch = false) {
+  const api = transport2.api ?? "auto";
+  if (!["auto", "chat", "responses"].includes(api)) {
+    throw new Error(`Unknown OpenAI API selection: ${api}`);
+  }
+  if (api === "chat") {
+    if (transport2.responses !== void 0 || nativeToolSearch) {
+      throw new Error(
+        "Responses options and native tool search require the Responses API"
+      );
+    }
+    return false;
+  }
+  let officialEndpoint = !baseURL;
+  try {
+    officialEndpoint = !baseURL || new URL(baseURL).hostname === "api.openai.com";
+  } catch {
+  }
+  return api === "responses" || transport2.responses !== void 0 || hasReplay && officialEndpoint || nativeToolSearch || hasTools && requiresOpenAIResponses(model, baseURL, parameters);
+}
+function responseMetadata(response) {
+  const output = Array.isArray(response?.output) ? response.output : [];
+  return {
+    openai: {
+      response: {
+        id: response?.id,
+        model: response?.model,
+        status: response?.status,
+        output,
+        usage: response?.usage,
+        store: response?.store,
+        refusals: output.flatMap(
+          (item) => item?.type === "message" && Array.isArray(item.content) ? item.content.filter(
+            (part) => part?.type === "refusal" && typeof part.refusal === "string"
+          ).map((part) => part.refusal) : []
+        )
+      }
+    }
+  };
+}
+function responseReplayMetadata(metadata) {
+  const response = metadata?.openai?.response;
+  if (!response) return metadata;
+  const output = response.output.map((item) => {
+    const fields = ["type", "id", "status"];
+    switch (item.type) {
+      case "reasoning":
+        fields.push("encrypted_content");
+        break;
+      case "message":
+        fields.push("role", "phase");
+        break;
+      case "function_call":
+        fields.push("call_id", "name", "namespace", "arguments");
+        break;
+      case "tool_search_call":
+        fields.push("call_id", "execution", "arguments");
+        break;
+      case "tool_search_output":
+        fields.push("call_id", "tools");
+        break;
+      case "web_search_call":
+        fields.push("action");
+        break;
+      default:
+        return stripItemDiagnostics(item);
+    }
+    const replay = Object.fromEntries(
+      fields.filter((key) => item[key] !== void 0).map((key) => [key, item[key]])
+    );
+    if (item.type === "reasoning") replay.summary = [];
+    if (item.type === "message")
+      replay.content = Array.isArray(item.content) ? item.content.flatMap(
+        (part) => part?.type === "output_text" && typeof part.text === "string" ? [{ type: "output_text", text: part.text, annotations: [] }] : part?.type === "refusal" && typeof part.refusal === "string" ? [{ type: "refusal", refusal: part.refusal }] : []
+      ) : [];
+    return stripItemDiagnostics(replay);
+  });
+  return { openai: { response: { output, refusals: [] } } };
+}
+function stripItemDiagnostics(item) {
+  const {
+    usage: _usage,
+    logprobs: _logprobs,
+    metadata: _metadata,
+    ...state
+  } = item;
+  return state;
+}
+function getResponseReplay(metadata) {
+  const response = metadata?.openai?.response;
+  if (response === void 0) return void 0;
+  if (!Array.isArray(response.output) || response.output.some((item) => !item || typeof item.type !== "string")) {
+    throw new Error(
+      "Invalid OpenAI Responses replay metadata: output must contain typed items"
+    );
+  }
+  return responseReplayMetadata(metadata)?.openai?.response?.output;
+}
+
 // src/core/provider-error.ts
+function invalidProviderResponse(message) {
+  return Object.assign(new Error(message), {
+    name: "InvalidProviderResponseError",
+    status: 502,
+    code: "INVALID_PROVIDER_RESPONSE"
+  });
+}
 function providerErrorEvent(error, code) {
   const details = error ?? {};
   const event = {
@@ -283,7 +499,7 @@ function messageToOpenAIContent(message) {
   }
   return blocks;
 }
-function formatMessagesForOpenAI(messages, systemPrompt) {
+function formatMessagesForOpenAI(messages, systemPrompt, preserveProviderMetadata = false) {
   const formatted = [];
   if (systemPrompt) {
     formatted.push({ role: "system", content: systemPrompt });
@@ -301,7 +517,8 @@ function formatMessagesForOpenAI(messages, systemPrompt) {
       const assistantMsg = {
         role: "assistant",
         // Gemini/xAI (OpenAI-compatible) reject content: "" on assistant messages with tool_calls
-        content: hasToolCalls ? msg.content || null : msg.content
+        content: hasToolCalls ? msg.content || null : msg.content,
+        ...preserveProviderMetadata && msg.providerMetadata ? { providerMetadata: msg.providerMetadata } : {}
       };
       if (hasToolCalls) {
         assistantMsg.tool_calls = msg.tool_calls;
@@ -310,7 +527,8 @@ function formatMessagesForOpenAI(messages, systemPrompt) {
     } else if (msg.role === "tool" && msg.tool_call_id) {
       formatted.push({
         role: "tool",
-        content: msg.content ?? "",
+        content: preserveProviderMetadata ? msg.content ?? "" : textToolOutput(msg.modelOutput, msg.content ?? ""),
+        ...preserveProviderMetadata && msg.modelOutput ? { modelOutput: msg.modelOutput } : {},
         tool_call_id: msg.tool_call_id
       });
     }
@@ -361,13 +579,16 @@ var OpenAIAdapter = class _OpenAIAdapter {
     }));
   }
   shouldUseResponsesApi(request) {
-    if (this.toolDefinitions(request).length === 0) return false;
-    if (request.providerToolOptions?.openai?.nativeToolSearch?.enabled === true && request.providerToolOptions.openai.nativeToolSearch.useResponsesApi !== false)
-      return true;
-    return requiresOpenAIResponses(
+    return useOpenAIResponses(
       request.config?.model || this.model,
       this.config.baseUrl,
-      this.parameterOptions(request)
+      this.parameterOptions(request),
+      resolveOpenAITransport(this.config, request.config?.openai),
+      this.toolDefinitions(request).length > 0,
+      !!(request.rawMessages ?? request.messages)?.some(
+        (message) => message.providerMetadata?.openai?.response
+      ),
+      request.providerToolOptions?.openai?.nativeToolSearch?.enabled === true && request.providerToolOptions.openai.nativeToolSearch.useResponsesApi !== false
     );
   }
   /**
@@ -400,11 +621,11 @@ var OpenAIAdapter = class _OpenAIAdapter {
     });
   }
   buildResponsesInput(request) {
-    const sourceMessages = request.rawMessages && request.rawMessages.length > 0 ? this.formatRawMessages(request.rawMessages) : formatMessagesForOpenAI(request.messages, void 0);
+    const sourceMessages = request.rawMessages && request.rawMessages.length > 0 ? this.formatRawMessages(request.rawMessages, true) : formatMessagesForOpenAI(request.messages, void 0, true);
     const input = [];
     for (const message of sourceMessages) {
       if (message.role === "system") {
-        if (!request.systemPrompt && message.content) {
+        if (message.content && message.content !== request.systemPrompt) {
           input.push({
             type: "message",
             role: "developer",
@@ -414,6 +635,11 @@ var OpenAIAdapter = class _OpenAIAdapter {
         continue;
       }
       if (message.role === "assistant") {
+        const replay = getResponseReplay(message.providerMetadata);
+        if (replay) {
+          input.push(...replay);
+          continue;
+        }
         const content = this.toResponsesContent(message.content, "assistant");
         if (typeof content === "string" ? content : Array.isArray(content) && content.length > 0) {
           input.push({
@@ -437,7 +663,7 @@ var OpenAIAdapter = class _OpenAIAdapter {
         input.push({
           type: "function_call_output",
           call_id: message.tool_call_id,
-          output: typeof message.content === "string" ? message.content : JSON.stringify(message.content ?? null)
+          output: message.modelOutput ? toolOutputForProvider(message.modelOutput, "responses") : typeof message.content === "string" ? message.content : JSON.stringify(message.content ?? null)
         });
         continue;
       }
@@ -451,10 +677,12 @@ var OpenAIAdapter = class _OpenAIAdapter {
     return input;
   }
   /** Preserve Copilot screenshot attachments on every OpenAI transport. */
-  formatRawMessages(messages) {
+  formatRawMessages(messages, preserveProviderMetadata = false) {
     return messages.map((message) => {
-      const { attachments, ...rest } = message;
-      let content = message.content;
+      const { attachments, providerMetadata, modelOutput, ...rest } = message;
+      let content = message.role === "tool" && !preserveProviderMetadata && modelOutput !== void 0 ? textToolOutput(modelOutput, "") : message.content;
+      if (preserveProviderMetadata && modelOutput)
+        rest.modelOutput = modelOutput;
       if (message.role === "assistant" && Array.isArray(message.tool_calls) && message.tool_calls.length > 0 && content === "") {
         content = null;
       }
@@ -467,7 +695,11 @@ var OpenAIAdapter = class _OpenAIAdapter {
           content = [...parts, ...images];
         }
       }
-      return { ...rest, content };
+      return {
+        ...rest,
+        content,
+        ...preserveProviderMetadata && providerMetadata ? { providerMetadata } : {}
+      };
     });
   }
   /**
@@ -548,14 +780,40 @@ var OpenAIAdapter = class _OpenAIAdapter {
       ).map((part) => part.text) : []
     ).join("") : "";
     const content = textFromOutput || (typeof response?.output_text === "string" ? response.output_text : "");
-    const toolCalls = Array.isArray(response?.output) ? response.output.filter((item) => item?.type === "function_call").map((item) => ({
-      id: item.call_id ?? item.id ?? generateToolCallId(),
-      name: item.name,
-      args: JSON.parse(item.arguments ?? "{}")
-    })) : [];
+    const seenCallIds = /* @__PURE__ */ new Set();
+    const toolCalls = Array.isArray(response?.output) ? response.output.filter((item) => item?.type === "function_call").map((item) => {
+      if (typeof item.call_id !== "string" || !item.call_id || typeof item.name !== "string" || !item.name || typeof item.arguments !== "string" || item.status && item.status !== "completed") {
+        throw invalidProviderResponse(
+          "Responses contains an incomplete function call"
+        );
+      }
+      if (seenCallIds.has(item.call_id))
+        throw invalidProviderResponse(
+          "Responses contains duplicate function call IDs"
+        );
+      seenCallIds.add(item.call_id);
+      let args;
+      try {
+        args = JSON.parse(item.arguments);
+      } catch {
+        throw invalidProviderResponse(
+          "Responses function arguments must be valid JSON"
+        );
+      }
+      if (!args || typeof args !== "object" || Array.isArray(args))
+        throw invalidProviderResponse(
+          "Responses function arguments must be a JSON object"
+        );
+      return {
+        id: item.call_id,
+        name: item.name,
+        args
+      };
+    }) : [];
     return {
       content,
       toolCalls,
+      providerMetadata: responseMetadata(response),
       usage: response?.usage ? {
         promptTokens: response.usage.input_tokens ?? 0,
         completionTokens: response.usage.output_tokens ?? 0,
@@ -566,6 +824,17 @@ var OpenAIAdapter = class _OpenAIAdapter {
   }
   buildResponsesPayload(request, stream) {
     const openaiToolOptions = request.providerToolOptions?.openai;
+    const options = resolveOpenAITransport(
+      this.config,
+      request.config?.openai
+    ).responses;
+    const parameters = buildOpenAIParameters(
+      request.config?.model || this.model,
+      request.config?.maxTokens ?? this.config.maxTokens,
+      request.config?.temperature ?? this.config.temperature,
+      this.parameterOptions(request),
+      "responses"
+    );
     const responsesTextFormat = toOpenAIResponsesTextFormat(
       request.config?.responseFormat
     );
@@ -581,13 +850,26 @@ var OpenAIAdapter = class _OpenAIAdapter {
       ),
       tool_choice: openaiToolOptions?.toolChoice,
       parallel_tool_calls: openaiToolOptions?.parallelToolCalls,
-      ...buildOpenAIParameters(
-        modelIdForPayload,
-        request.config?.maxTokens ?? this.config.maxTokens,
-        request.config?.temperature ?? this.config.temperature,
-        this.parameterOptions(request),
-        "responses"
-      ),
+      ...parameters,
+      ...options?.reasoningSummary ? {
+        reasoning: {
+          ...parameters.reasoning,
+          summary: options.reasoningSummary
+        }
+      } : {},
+      ...options?.store !== void 0 ? { store: options.store } : {},
+      ...options?.previousResponseId !== void 0 ? { previous_response_id: options.previousResponseId } : {},
+      ...options?.promptCacheKey !== void 0 ? { prompt_cache_key: options.promptCacheKey } : {},
+      ...options?.promptCacheOptions !== void 0 ? { prompt_cache_options: options.promptCacheOptions } : {},
+      ...options?.promptCacheRetention !== void 0 ? { prompt_cache_retention: options.promptCacheRetention } : {},
+      ...options?.store === false || options?.include ? {
+        include: [
+          .../* @__PURE__ */ new Set([
+            ...options?.include ?? [],
+            ...options?.store === false ? ["reasoning.encrypted_content"] : []
+          ])
+        ]
+      } : {},
       ...responsesTextFormat ? { text: { format: responsesTextFormat } } : {},
       stream
     };
@@ -646,6 +928,10 @@ var OpenAIAdapter = class _OpenAIAdapter {
     let usage;
     const citations = [];
     let completed = false;
+    let terminalResponse;
+    let responseIdentity = {};
+    const outputItems = /* @__PURE__ */ new Map();
+    let streamedText = "";
     function* flushToolCalls() {
       for (const call of toolCalls.values()) {
         if (!call.name)
@@ -662,17 +948,36 @@ var OpenAIAdapter = class _OpenAIAdapter {
     for await (const event of stream) {
       throwIfAborted(request.signal);
       logProviderPayload("openai", "stream chunk", event, request.debug);
+      if (completed)
+        throw new Error("Responses stream emitted an event after completion");
       const index = event?.output_index ?? 0;
       switch (event?.type) {
+        case "response.created":
+          responseIdentity = {
+            id: event.response?.id,
+            model: event.response?.model
+          };
+          break;
+        case "response.output_item.done":
+          if (event.item) outputItems.set(index, event.item);
+          if (event.item?.type === "function_call") {
+            toolCalls.set(index, {
+              id: event.item.call_id,
+              name: event.item.name,
+              arguments: event.item.arguments
+            });
+          }
+          break;
         case "response.output_text.delta":
           if (event.delta) {
+            streamedText += event.delta;
             yield { type: "message:delta", content: event.delta };
           }
           break;
         case "response.output_item.added":
           if (event.item?.type === "function_call") {
             toolCalls.set(index, {
-              id: event.item.call_id ?? event.item.id ?? generateToolCallId(),
+              id: event.item.call_id ?? "",
               name: event.item.name ?? "",
               arguments: event.item.arguments ?? ""
             });
@@ -691,7 +996,10 @@ var OpenAIAdapter = class _OpenAIAdapter {
           break;
         }
         case "response.completed": {
+          if (event.response?.status && event.response.status !== "completed")
+            throw new Error(`Responses stream ${event.response.status}`);
           completed = true;
+          terminalResponse = { ...responseIdentity, ...event.response };
           const found = this.collectResponsesCitations(
             event.response?.output,
             citations.length
@@ -728,12 +1036,66 @@ var OpenAIAdapter = class _OpenAIAdapter {
     throwIfAborted(request.signal);
     if (!completed)
       throw new Error("Responses stream ended without a completed response");
+    const output = terminalResponse.output?.length ? terminalResponse.output : [...outputItems.entries()].sort(([a], [b]) => a - b).map(([, item]) => item);
+    if (output.some((item) => item.type === "function_call"))
+      toolCalls.clear();
+    for (const [index, item] of output.entries()) {
+      if (item.type === "function_call") {
+        toolCalls.set(index, {
+          id: item.call_id,
+          name: item.name,
+          arguments: item.arguments
+        });
+      }
+    }
+    const uniqueCalls = /* @__PURE__ */ new Map();
+    for (const call of toolCalls.values()) {
+      if (!call.id || !call.name)
+        throw invalidProviderResponse(
+          "Responses stream contains an incomplete tool call"
+        );
+      uniqueCalls.set(call.id, call);
+    }
+    toolCalls.clear();
+    [...uniqueCalls.values()].forEach(
+      (call, index) => toolCalls.set(index, call)
+    );
+    if (!output.length && toolCalls.size) {
+      for (const call of toolCalls.values())
+        output.push({
+          type: "function_call",
+          call_id: call.id,
+          name: call.name,
+          arguments: call.arguments
+        });
+    }
+    if (!output.length && streamedText)
+      output.push({
+        type: "message",
+        role: "assistant",
+        content: [{ type: "output_text", text: streamedText, annotations: [] }]
+      });
+    terminalResponse.output = output;
+    const finalText = this.parseResponsesResult(terminalResponse).content;
+    if (finalText.startsWith(streamedText) && finalText.length > streamedText.length) {
+      yield {
+        type: "message:delta",
+        content: finalText.slice(streamedText.length)
+      };
+    }
     yield* flushToolCalls();
     if (citations.length > 0) {
       yield { type: "citation", citations: deduplicateCitations(citations) };
     }
-    yield { type: "message:end" };
-    yield { type: "done", usage };
+    yield {
+      type: "message:end",
+      providerMetadata: responseMetadata(terminalResponse)
+    };
+    yield {
+      type: "done",
+      usage,
+      providerMetadata: responseMetadata(terminalResponse)
+    };
   }
   async *stream(request) {
     throwIfAborted(request.signal);
@@ -1101,7 +1463,16 @@ function openai(modelId, options = {}) {
     };
   }
   function useResponses(params) {
-    return !!params.tools?.length && requiresOpenAIResponses(modelId, baseURL, parameterOptions(params));
+    return useOpenAIResponses(
+      modelId,
+      baseURL,
+      parameterOptions(params),
+      resolveOpenAITransport(options, params.openai),
+      !!params.tools?.length,
+      params.messages.some(
+        (message) => message.role === "assistant" && !!message.providerMetadata?.openai?.response
+      )
+    );
   }
   const responsesAdapter = new OpenAIAdapter({
     ...options,
@@ -1112,7 +1483,7 @@ function openai(modelId, options = {}) {
   function responsesRequest(params) {
     return {
       messages: [],
-      rawMessages: formatMessagesForOpenAI2(params.messages),
+      rawMessages: formatMessagesForOpenAI2(params.messages, true),
       toolDefinitions: params.tools?.map((raw) => {
         const tool = raw;
         if (tool.type !== "function")
@@ -1127,6 +1498,10 @@ function openai(modelId, options = {}) {
         };
       }),
       config: {
+        openai: {
+          ...resolveOpenAITransport(options, params.openai),
+          api: "responses"
+        },
         temperature: params.temperature,
         maxTokens: params.maxTokens,
         responseFormat: params.responseFormat,
@@ -1158,13 +1533,14 @@ function openai(modelId, options = {}) {
         return {
           text: result.content,
           toolCalls: result.toolCalls,
-          finishReason: result.toolCalls.length ? "tool-calls" : "stop",
+          finishReason: result.toolCalls.length ? "tool-calls" : result.providerMetadata?.openai?.response?.refusals.length ? "content-filter" : "stop",
           usage: result.usage ?? {
             promptTokens: 0,
             completionTokens: 0,
             totalTokens: 0
           },
-          rawResponse: result.rawResponse
+          rawResponse: result.rawResponse,
+          providerMetadata: result.providerMetadata
         };
       }
       const client2 = await getClient();
@@ -1246,7 +1622,8 @@ function openai(modelId, options = {}) {
             case "done":
               yield {
                 type: "finish",
-                finishReason: hasTools ? "tool-calls" : "stop",
+                finishReason: hasTools ? "tool-calls" : event.providerMetadata?.openai?.response?.refusals.length ? "content-filter" : "stop",
+                providerMetadata: event.providerMetadata,
                 usage: {
                   promptTokens: event.usage?.prompt_tokens ?? 0,
                   completionTokens: event.usage?.completion_tokens ?? 0,
@@ -1342,7 +1719,7 @@ function mapFinishReason(reason) {
       return "unknown";
   }
 }
-function formatMessagesForOpenAI2(messages) {
+function formatMessagesForOpenAI2(messages, responses = false) {
   return messages.map((msg) => {
     switch (msg.role) {
       case "system":
@@ -1359,16 +1736,28 @@ function formatMessagesForOpenAI2(messages) {
             }
             if (part.type === "image") {
               const imageData = typeof part.image === "string" ? part.image : Buffer.from(part.image).toString("base64");
-              const url = imageData.startsWith("data:") ? imageData : `data:${part.mimeType ?? "image/png"};base64,${imageData}`;
+              const url = /^(data:|https?:\/\/)/.test(imageData) ? imageData : `data:${part.mimeType ?? "image/png"};base64,${imageData}`;
               return { type: "image_url", image_url: { url, detail: "auto" } };
             }
-            return { type: "text", text: "" };
+            if (part.type === "file") {
+              if (!responses)
+                throw new Error(
+                  "File inputs require the Responses API on this OpenAI path"
+                );
+              return /^https?:\/\//.test(part.data) ? { type: "input_file", file_url: part.data } : {
+                type: "input_file",
+                filename: part.filename ?? "document.pdf",
+                file_data: part.data.startsWith("data:") ? part.data : `data:${part.mimeType};base64,${part.data}`
+              };
+            }
+            throw new Error("Unsupported OpenAI message content");
           })
         };
       case "assistant":
         const assistantMsg = {
           role: "assistant",
-          content: msg.content
+          content: msg.content,
+          ...responses && msg.providerMetadata ? { providerMetadata: msg.providerMetadata } : {}
         };
         if (msg.toolCalls && msg.toolCalls.length > 0) {
           assistantMsg.tool_calls = msg.toolCalls.map((tc) => ({
@@ -1385,7 +1774,8 @@ function formatMessagesForOpenAI2(messages) {
         return {
           role: "tool",
           tool_call_id: msg.toolCallId,
-          content: msg.content
+          content: responses ? msg.content : textToolOutput(msg.modelOutput, msg.content),
+          ...responses && msg.modelOutput ? { modelOutput: msg.modelOutput } : {}
         };
       default:
         return msg;
@@ -1409,6 +1799,394 @@ function createCallableProvider(providerFn, properties) {
   return providerFn;
 }
 
+// src/core/audio.ts
+var audioErrorMarker = /* @__PURE__ */ Symbol.for("@yourgpt/llm-sdk/AudioAPIError");
+var _a;
+var AudioAPIError = class extends Error {
+  constructor(message, code, status, requestId) {
+    super(message);
+    this.code = code;
+    this.status = status;
+    this.requestId = requestId;
+    this[_a] = true;
+    this.name = "AudioAPIError";
+  }
+  static [(_a = audioErrorMarker, Symbol.hasInstance)](value) {
+    return !!value && typeof value === "object" && value[audioErrorMarker] === true;
+  }
+};
+
+// src/providers/openai/audio.ts
+var formats = {
+  mp3: "audio/mpeg",
+  opus: "audio/ogg",
+  aac: "audio/aac",
+  flac: "audio/flac",
+  wav: "audio/wav",
+  pcm: "audio/pcm"
+};
+var extensions = {
+  "audio/mpeg": "mp3",
+  "audio/mp3": "mp3",
+  "audio/mp4": "m4a",
+  "video/mp4": "mp4",
+  "audio/wav": "wav",
+  "audio/x-wav": "wav",
+  "audio/webm": "webm",
+  "video/webm": "webm",
+  "audio/ogg": "ogg",
+  "audio/flac": "flac",
+  "audio/x-flac": "flac"
+};
+var MAX_AUDIO = 25e6;
+function hasAudioHeader(data, format) {
+  const head = Buffer.from(data.subarray(0, 12)).toString("latin1");
+  if (format === "wav")
+    return head.startsWith("RIFF") && head.slice(8, 12) === "WAVE";
+  if (format === "mp3" || format === "mpeg" || format === "mpga")
+    return head.startsWith("ID3") || data[0] === 255 && (data[1] & 224) === 224;
+  if (format === "opus" || format === "ogg") return head.startsWith("OggS");
+  if (format === "flac") return head.startsWith("fLaC");
+  if (format === "aac") return data[0] === 255 && (data[1] & 246) === 240;
+  if (format === "webm")
+    return data[0] === 26 && data[1] === 69 && data[2] === 223 && data[3] === 163;
+  if (format === "m4a" || format === "mp4") return head.slice(4, 8) === "ftyp";
+  return format === "pcm" && data.length > 0 && data.length % 2 === 0;
+}
+function invalid(message) {
+  throw new AudioAPIError(message, "invalid_input");
+}
+function unsupported(message) {
+  throw new AudioAPIError(message, "unsupported_option");
+}
+function providerOptions(value, allowed) {
+  for (const [provider, options] of Object.entries(value ?? {})) {
+    if (provider !== "openai")
+      unsupported(
+        `Provider options for ${provider} cannot be used with OpenAI audio`
+      );
+    for (const key of Object.keys(options))
+      if (!allowed.includes(key))
+        unsupported(`Unsupported OpenAI audio option: ${key}`);
+  }
+  return value?.openai ?? {};
+}
+async function readBody(response, limit) {
+  if (Number(response.headers.get("content-length")) > limit) {
+    await response.body?.cancel();
+    throw new AudioAPIError(
+      "Audio API response exceeds the size limit",
+      "invalid_response"
+    );
+  }
+  const reader = response.body?.getReader();
+  if (!reader) return new Uint8Array();
+  const chunks = [];
+  let length = 0;
+  try {
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      length += value.length;
+      if (length > limit) {
+        await reader.cancel();
+        throw new AudioAPIError(
+          "Audio API response exceeds the size limit",
+          "invalid_response"
+        );
+      }
+      chunks.push(value);
+    }
+  } finally {
+    reader.releaseLock();
+  }
+  const data = new Uint8Array(length);
+  let offset = 0;
+  for (const chunk of chunks) {
+    data.set(chunk, offset);
+    offset += chunk.length;
+  }
+  return data;
+}
+function transport(config) {
+  return async (path, body, signal) => {
+    signal?.throwIfAborted();
+    const apiKey = config.apiKey ?? process.env.OPENAI_API_KEY;
+    if (!apiKey) invalid("OpenAI audio requires an API key on the server");
+    const headers = {
+      Authorization: `Bearer ${apiKey}`
+    };
+    if (typeof body === "string") headers["Content-Type"] = "application/json";
+    if (config.organization)
+      headers["OpenAI-Organization"] = config.organization;
+    if (config.project) headers["OpenAI-Project"] = config.project;
+    let response;
+    try {
+      response = await (config.fetch ?? globalThis.fetch)(
+        `${(config.baseUrl ?? "https://api.openai.com/v1").replace(/\/$/, "")}${path}`,
+        { method: "POST", headers, body, signal, redirect: "error" }
+      );
+    } catch {
+      signal?.throwIfAborted();
+      throw new AudioAPIError("OpenAI audio request failed", "network_error");
+    }
+    if (!response.ok) {
+      await response.body?.cancel();
+      throw new AudioAPIError(
+        `OpenAI audio request failed (HTTP ${response.status})`,
+        "provider_error",
+        response.status,
+        response.headers.get("x-request-id") ?? void 0
+      );
+    }
+    return response;
+  };
+}
+function openaiSpeech(modelId = "gpt-4o-mini-tts", config = {}) {
+  const request = transport(config);
+  return {
+    provider: "openai",
+    modelId,
+    async doGenerate(options) {
+      const {
+        text,
+        voice = "alloy",
+        outputFormat = "mp3",
+        speed,
+        instructions,
+        signal
+      } = options;
+      signal?.throwIfAborted();
+      providerOptions(options.providerOptions, []);
+      if (typeof text !== "string" || !text.trim() || text.length > 4096)
+        invalid("Speech text must contain 1\u20134096 characters");
+      if (!Object.prototype.hasOwnProperty.call(formats, outputFormat))
+        unsupported("Unsupported OpenAI speech output format");
+      if (typeof voice !== "string" || !voice.trim())
+        invalid("Speech voice must not be empty");
+      if (speed !== void 0 && (!Number.isFinite(speed) || speed < 0.25 || speed > 4))
+        invalid("Speech speed must be between 0.25 and 4");
+      if (instructions !== void 0) {
+        if (typeof instructions !== "string" || instructions.length > 4096)
+          invalid(
+            "Speech instructions must be a string of at most 4096 characters"
+          );
+        if (modelId === "tts-1" || modelId === "tts-1-hd")
+          unsupported("Instructions are not supported by tts-1 or tts-1-hd");
+      }
+      const response = await request(
+        "/audio/speech",
+        JSON.stringify({
+          model: modelId,
+          input: text,
+          voice,
+          response_format: outputFormat,
+          speed,
+          instructions
+        }),
+        signal
+      );
+      const mediaType = response.headers.get("content-type")?.split(";")[0].trim();
+      if (mediaType && !mediaType.startsWith("audio/") && mediaType !== "application/octet-stream") {
+        await response.body?.cancel();
+        throw new AudioAPIError(
+          "Speech endpoint returned a non-audio response",
+          "invalid_response"
+        );
+      }
+      const data = await readBody(response, MAX_AUDIO);
+      signal?.throwIfAborted();
+      if (!data.length || !hasAudioHeader(data, outputFormat))
+        throw new AudioAPIError(
+          "Speech endpoint returned empty or invalid audio",
+          "invalid_response"
+        );
+      return {
+        audio: {
+          uint8Array: data,
+          get base64() {
+            return Buffer.from(data).toString("base64");
+          },
+          mediaType: formats[outputFormat],
+          format: outputFormat
+        },
+        warnings: [],
+        response: {
+          provider: "openai",
+          modelId,
+          requestId: response.headers.get("x-request-id") ?? void 0,
+          status: response.status
+        }
+      };
+    }
+  };
+}
+async function audioBytes(options) {
+  const blob = typeof Blob !== "undefined" && options.audio instanceof Blob ? options.audio : void 0;
+  if (blob && blob.size > MAX_AUDIO)
+    invalid("Transcription file exceeds 25 MB");
+  const declared = options.mediaType?.split(";")[0].trim();
+  const blobType = blob?.type.split(";")[0].trim();
+  if (declared && blobType && declared !== blobType)
+    invalid("Transcription mediaType does not match the Blob type");
+  const mediaType = declared || blobType || "";
+  const extension = extensions[mediaType];
+  if (!extension)
+    unsupported(
+      "Unsupported transcription media type; provide an encoded audio file"
+    );
+  let bytes;
+  if (blob) bytes = new Uint8Array(await blob.arrayBuffer());
+  else if (options.audio instanceof Uint8Array) bytes = options.audio;
+  else if (options.audio instanceof ArrayBuffer)
+    bytes = new Uint8Array(options.audio);
+  else if (typeof options.audio === "string") {
+    const base64 = options.audio;
+    if (base64.length > MAX_AUDIO * 4 / 3 + 4)
+      invalid("Transcription file exceeds 25 MB");
+    if (!base64 || base64.length % 4 !== 0 || !/^[A-Za-z0-9+/]+={0,2}$/.test(base64))
+      invalid(
+        "Audio must be bytes, a Blob, or valid base64; URLs are not fetched"
+      );
+    bytes = Buffer.from(base64, "base64");
+    if (Buffer.from(bytes).toString("base64") !== base64)
+      invalid("Invalid audio base64");
+  } else invalid("Audio must be bytes, a Blob, or base64");
+  if (!bytes.length || bytes.length > MAX_AUDIO)
+    invalid("Transcription file must contain 1 byte to 25 MB");
+  if (!hasAudioHeader(bytes, extension))
+    invalid("Audio header does not match the declared media type");
+  return { bytes, mediaType, extension };
+}
+function openaiTranscription(modelId = "gpt-4o-mini-transcribe", config = {}) {
+  const request = transport(config);
+  return {
+    provider: "openai",
+    modelId,
+    async doTranscribe(options) {
+      options.signal?.throwIfAborted();
+      if (modelId.includes("realtime"))
+        unsupported(
+          "Realtime models require a realtime session; use a file transcription model"
+        );
+      const opts = providerOptions(options.providerOptions, [
+        "timestampGranularities",
+        "includeLogprobs",
+        "chunkingStrategy"
+      ]);
+      const timestamps = opts.timestampGranularities;
+      if (timestamps !== void 0 && (!Array.isArray(timestamps) || !timestamps.length || timestamps.some((x) => x !== "word" && x !== "segment")))
+        invalid("timestampGranularities must contain word or segment");
+      if (timestamps !== void 0 && modelId !== "whisper-1")
+        unsupported("Timestamp granularities require whisper-1");
+      if (opts.includeLogprobs !== void 0 && typeof opts.includeLogprobs !== "boolean")
+        invalid("includeLogprobs must be boolean");
+      if (opts.includeLogprobs && !/^gpt-4o-(mini-)?transcribe(?:-\d{4}-\d{2}-\d{2})?$/.test(modelId))
+        unsupported(
+          "Log probabilities require a supported gpt-4o transcription model"
+        );
+      const diarize = modelId === "gpt-4o-transcribe-diarize";
+      if (opts.chunkingStrategy !== void 0 && opts.chunkingStrategy !== "auto")
+        unsupported("Only automatic audio chunking is supported");
+      if (diarize && options.prompt !== void 0)
+        unsupported("Prompts are not supported by the diarization model");
+      if (options.language !== void 0 && !/^[a-z]{2}$/.test(options.language))
+        invalid("language must be a two-letter ISO-639-1 code");
+      if (options.prompt !== void 0 && typeof options.prompt !== "string")
+        invalid("Transcription prompt must be a string");
+      const { bytes, mediaType, extension } = await audioBytes(options);
+      options.signal?.throwIfAborted();
+      const fileName = options.filename ?? `audio.${extension}`;
+      if (!/^[^/\\\r\n\x00]+\.[a-z0-9]+$/i.test(fileName))
+        invalid("Transcription filename must be a filename with an extension");
+      const allowedExt = mediaType === "audio/mpeg" ? ["mp3", "mpeg", "mpga"] : mediaType === "audio/mp4" ? ["m4a", "mp4"] : [extension];
+      if (!allowedExt.includes(fileName.split(".").at(-1).toLowerCase()))
+        invalid("Transcription filename extension does not match mediaType");
+      const form = new FormData();
+      form.append(
+        "file",
+        new Blob([new Uint8Array(bytes)], { type: mediaType }),
+        fileName
+      );
+      form.append("model", modelId);
+      form.append(
+        "response_format",
+        diarize ? "diarized_json" : timestamps ? "verbose_json" : "json"
+      );
+      if (options.language) form.append("language", options.language);
+      if (options.prompt) form.append("prompt", options.prompt);
+      if (timestamps)
+        for (const value2 of timestamps)
+          form.append("timestamp_granularities[]", value2);
+      if (opts.includeLogprobs) form.append("include[]", "logprobs");
+      if (diarize || opts.chunkingStrategy)
+        form.append("chunking_strategy", "auto");
+      const response = await request(
+        "/audio/transcriptions",
+        form,
+        options.signal
+      );
+      let value;
+      try {
+        value = JSON.parse(
+          Buffer.from(await readBody(response, 5 * 1024 * 1024)).toString(
+            "utf8"
+          )
+        );
+      } catch (error) {
+        options.signal?.throwIfAborted();
+        if (error instanceof AudioAPIError) throw error;
+        throw new AudioAPIError(
+          "Transcription endpoint returned invalid JSON",
+          "invalid_response"
+        );
+      }
+      options.signal?.throwIfAborted();
+      if (!value || typeof value.text !== "string")
+        throw new AudioAPIError(
+          "Transcription response is missing text",
+          "invalid_response"
+        );
+      function timed(items, word = false) {
+        if (items === void 0) return void 0;
+        if (!Array.isArray(items) || items.some(
+          (x) => !x || typeof x[word ? "word" : "text"] !== "string" || !Number.isFinite(x.start) || !Number.isFinite(x.end) || x.start < 0 || x.end < x.start
+        ))
+          throw new AudioAPIError(
+            "Invalid transcription timestamps",
+            "invalid_response"
+          );
+        return items.map(
+          (x) => word ? { word: x.word, start: x.start, end: x.end } : {
+            text: x.text,
+            start: x.start,
+            end: x.end,
+            ...typeof x.speaker === "string" ? { speaker: x.speaker } : {}
+          }
+        );
+      }
+      return {
+        text: value.text,
+        language: typeof value.language === "string" ? value.language : void 0,
+        durationSeconds: typeof value.duration === "number" ? value.duration : void 0,
+        segments: timed(value.segments),
+        words: timed(value.words, true),
+        usage: value.usage,
+        providerMetadata: {
+          openai: { ...value.logprobs ? { logprobs: value.logprobs } : {} }
+        },
+        warnings: [],
+        response: {
+          provider: "openai",
+          modelId,
+          status: response.status,
+          requestId: response.headers.get("x-request-id") ?? void 0
+        }
+      };
+    }
+  };
+}
+
 // src/providers/openai/index.ts
 var OPENAI_MODELS2 = {
   // GPT-4o series
@@ -1525,6 +2303,8 @@ function createOpenAI(config = {}) {
       model: modelId,
       baseUrl: config.baseUrl,
       reasoningEffort: config.reasoningEffort,
+      api: config.api,
+      responses: config.responses,
       parameterPolicy: config.parameterPolicy,
       onWarning: config.onWarning
     });
@@ -1558,5 +2338,7 @@ exports.createOpenAI = createOpenAI;
 exports.createOpenAIModel = openai;
 exports.createOpenAIProvider = createOpenAIProvider;
 exports.openai = openai;
+exports.openaiSpeech = openaiSpeech;
+exports.openaiTranscription = openaiTranscription;
 //# sourceMappingURL=index.js.map
 //# sourceMappingURL=index.js.map