@yourgpt/llm-sdk 2.6.0-beta.1 → 2.6.0-beta.2

package/dist/index.js

@@ -3,13 +3,87 @@
 var hono = require('hono');
 var cors = require('hono/cors');
 
+// src/core/tool-output.ts
+var imageTypes = /* @__PURE__ */ new Set([
+  "image/png",
+  "image/jpeg",
+  "image/webp",
+  "image/gif"
+]);
+function normalizeToolOutput(value) {
+  const output = value;
+  if (output?.type !== "content" || !Array.isArray(output.content) || !output.content.length) {
+    throw new Error("Tool model output must contain a nonempty content array");
+  }
+  return {
+    type: "content",
+    content: output.content.map((part) => {
+      if (part?.type === "text" && typeof part.text === "string")
+        return { type: "text", text: part.text };
+      if (part?.type !== "image")
+        throw new Error("Unsupported tool output content type");
+      const sources = ["data", "url", "fileId"].filter((key) => key in part);
+      if (sources.length !== 1)
+        throw new Error(
+          "Tool image requires exactly one of data, url, or fileId"
+        );
+      if ("fileId" in part) {
+        if (part.provider !== "openai" || typeof part.fileId !== "string" || !part.fileId.trim())
+          throw new Error(
+            "Tool image file references must name the openai provider and fileId"
+          );
+        return { type: "image", fileId: part.fileId, provider: "openai" };
+      }
+      if ("url" in part) {
+        let url;
+        try {
+          url = new URL(part.url);
+        } catch {
+          throw new Error("Invalid tool image URL");
+        }
+        if (!["https:", "http:"].includes(url.protocol) || url.username || url.password)
+          throw new Error(
+            "Tool image URL must use HTTP(S) without credentials"
+          );
+        return { type: "image", url: url.href };
+      }
+      if (!("data" in part) || !imageTypes.has(part.mediaType))
+        throw new Error("Unsupported tool image media type");
+      let data = part.data instanceof Uint8Array ? Buffer.from(part.data).toString("base64") : part.data;
+      if (typeof data !== "string")
+        throw new Error("Tool image data must be base64 or bytes");
+      if (data.startsWith("data:")) {
+        const match = /^data:([^;,]+);base64,(.+)$/.exec(data);
+        if (!match || match[1] !== part.mediaType)
+          throw new Error(
+            "Tool image data URL media type does not match mediaType"
+          );
+        data = match[2];
+      }
+      if (!data.length || data.length % 4 !== 0 || !/^[A-Za-z0-9+/]+={0,2}$/.test(data))
+        throw new Error("Invalid tool image base64 data");
+      if (Buffer.byteLength(data, "base64") > 20 * 1024 * 1024)
+        throw new Error("Tool image exceeds 20 MiB");
+      if (Buffer.from(data, "base64").toString("base64") !== data)
+        throw new Error("Invalid tool image base64 data");
+      return { type: "image", data, mediaType: part.mediaType };
+    })
+  };
+}
+function resolveToolOutput(result, mapped) {
+  if (mapped !== void 0) return normalizeToolOutput(mapped);
+  const content = result?._aiContent;
+  return content !== void 0 ? normalizeToolOutput({ type: "content", content }) : void 0;
+}
+
 // src/core/tool.ts
 function tool(config) {
   return {
     description: config.description,
     parameters: config.parameters,
     execute: config.execute,
-    hidden: config.hidden
+    hidden: config.hidden,
+    toModelOutput: config.toModelOutput
   };
 }
 function toolToJsonSchema(toolDef) {
@@ -157,6 +231,7 @@ async function generateText(params) {
     const formattedTools = tools ? formatToolsForProvider(tools, model.provider) : void 0;
     const result = await model.doGenerate({
       messages,
+      openai: params.openai,
       tools: formattedTools,
       temperature: params.temperature,
       reasoningEffort: params.reasoningEffort,
@@ -190,7 +265,11 @@ async function generateText(params) {
           throwIfAborted(signal);
           const result2 = {
             toolCallId: call.id,
-            result: toolResult
+            result: toolResult,
+            modelOutput: resolveToolOutput(
+              toolResult,
+              toolDef.toModelOutput?.(toolResult)
+            )
           };
           stepToolResults.push(result2);
           allToolResults.push(result2);
@@ -212,22 +291,23 @@ async function generateText(params) {
       toolCalls: result.toolCalls,
       toolResults: stepToolResults,
       finishReason: result.finishReason,
-      usage: result.usage
+      usage: result.usage,
+      providerMetadata: result.providerMetadata
     });
-    if (!result.toolCalls || result.toolCalls.length === 0) {
-      break;
-    }
     const assistantMessage = {
       role: "assistant",
       content: result.text || null,
-      toolCalls: result.toolCalls
+      ...result.toolCalls.length ? { toolCalls: result.toolCalls } : {},
+      ...result.providerMetadata ? { providerMetadata: result.providerMetadata } : {}
     };
     messages = [...messages, assistantMessage];
+    if (!result.toolCalls || result.toolCalls.length === 0) break;
     for (const tr of stepToolResults) {
       const toolMessage = {
         role: "tool",
         toolCallId: tr.toolCallId,
-        content: JSON.stringify(tr.result)
+        content: tr.modelOutput ? "[Tool returned content]" : JSON.stringify(tr.result),
+        ...tr.modelOutput ? { modelOutput: tr.modelOutput } : {}
       };
       messages = [...messages, toolMessage];
     }
@@ -235,6 +315,7 @@ async function generateText(params) {
   const lastStep = steps[steps.length - 1];
   return {
     text: lastStep?.text ?? "",
+    providerMetadata: lastStep?.providerMetadata,
     usage: sumUsage(steps),
     finishReason: lastStep?.finishReason ?? "stop",
     steps,
@@ -288,6 +369,8 @@ async function streamText(params) {
       `[llm-sdk] ${model.provider}/${model.modelId} does not support structured output (responseFormat); the request will be sent but the provider may ignore it.`
     );
   }
+  let messages = buildMessages2(params);
+  let finalProviderMetadata;
   let fullText = "";
   let finalUsage = {
     promptTokens: 0,
@@ -296,7 +379,6 @@ async function streamText(params) {
   };
   let finalFinishReason = "stop";
   async function* createFullStream() {
-    let messages = buildMessages2(params);
     for (let step = 0; step < maxSteps; step++) {
       yield { type: "step-start", step };
       if (signal?.aborted) {
@@ -305,11 +387,13 @@ async function streamText(params) {
       }
       const formattedTools = tools ? formatToolsForProvider2(tools, model.provider) : void 0;
       let stepText = "";
+      let stepProviderMetadata;
       const toolCalls = [];
       let stepFinishReason = "stop";
       try {
         for await (const chunk of model.doStream({
           messages,
+          openai: params.openai,
           tools: formattedTools,
           temperature: params.temperature,
           reasoningEffort: params.reasoningEffort,
@@ -332,6 +416,8 @@ async function streamText(params) {
               };
               break;
             case "finish":
+              stepProviderMetadata = chunk.providerMetadata;
+              finalProviderMetadata = chunk.providerMetadata;
               stepFinishReason = chunk.finishReason;
               finalFinishReason = chunk.finishReason;
               if (chunk.usage) {
@@ -354,16 +440,22 @@ async function streamText(params) {
         };
         return;
       }
-      yield { type: "step-finish", step, finishReason: stepFinishReason };
-      if (toolCalls.length === 0 || !tools) {
-        break;
-      }
-      const assistantMessage = {
-        role: "assistant",
-        content: stepText || null,
-        toolCalls
+      yield {
+        type: "step-finish",
+        step,
+        finishReason: stepFinishReason,
+        providerMetadata: stepProviderMetadata
       };
-      messages = [...messages, assistantMessage];
+      messages = [
+        ...messages,
+        {
+          role: "assistant",
+          content: stepText || null,
+          ...toolCalls.length ? { toolCalls } : {},
+          ...stepProviderMetadata ? { providerMetadata: stepProviderMetadata } : {}
+        }
+      ];
+      if (toolCalls.length === 0 || !tools) break;
       for (const call of toolCalls) {
         throwIfAborted(signal);
         const toolDef = tools[call.name];
@@ -392,13 +484,23 @@ async function streamText(params) {
             messages
           });
           throwIfAborted(signal);
-          yield { type: "tool-result", toolCallId: call.id, result };
+          const modelOutput = resolveToolOutput(
+            result,
+            toolDef.toModelOutput?.(result)
+          );
+          yield {
+            type: "tool-result",
+            toolCallId: call.id,
+            result,
+            modelOutput
+          };
           messages = [
             ...messages,
             {
               role: "tool",
               toolCallId: call.id,
-              content: JSON.stringify(result)
+              content: modelOutput ? "[Tool returned content]" : JSON.stringify(result),
+              ...modelOutput ? { modelOutput } : {}
             }
           ];
         } catch (error) {
@@ -425,7 +527,8 @@ async function streamText(params) {
     yield {
       type: "finish",
       finishReason: finalFinishReason,
-      usage: finalUsage
+      usage: finalUsage,
+      providerMetadata: finalProviderMetadata
     };
   }
   const source = createFullStream();
@@ -477,6 +580,12 @@ async function streamText(params) {
   return {
     textStream: createTextStream(),
     fullStream: sharedStream(),
+    get response() {
+      return consumeStream().then(() => ({ messages }));
+    },
+    get providerMetadata() {
+      return consumeStream().then(() => finalProviderMetadata);
+    },
     get text() {
       return consumeStream().then(() => fullText);
     },
@@ -600,6 +709,7 @@ function createMessage(partial) {
     tool_calls: partial.tool_calls,
     tool_call_id: partial.tool_call_id,
     metadata: partial.metadata,
+    providerMetadata: partial.providerMetadata,
     created_at: partial.created_at ?? /* @__PURE__ */ new Date()
   };
 }
@@ -648,7 +758,13 @@ function mapOutputMessages(resultMessages) {
     role: m.role,
     content: m.content ?? "",
     toolCalls: m.tool_calls,
-    toolCallId: m.tool_call_id
+    toolCallId: m.tool_call_id,
+    ...m.providerMetadata || m.modelOutput ? {
+      metadata: {
+        ...m.providerMetadata ? { providerMetadata: m.providerMetadata } : {},
+        ...m.modelOutput ? { modelOutput: m.modelOutput } : {}
+      }
+    } : {}
   }));
 }
 
@@ -1538,9 +1654,6 @@ function buildProviderToolOptions(params) {
 function buildToolResultForAI(tool2, result, args) {
   const typedResult = result;
   const responseMode = typedResult?._aiResponseMode ?? tool2?.aiResponseMode ?? "full";
-  if (typedResult?._aiContent && typedResult._aiContent.length > 0) {
-    return typedResult._aiContent;
-  }
   let aiContext;
   if (typedResult?._aiContext) {
     aiContext = typedResult._aiContext;
@@ -1560,25 +1673,6 @@ function buildToolResultForAI(tool2, result, args) {
 Full data: ${fullData}` : fullData;
   }
 }
-function serializeToolResultContent(content) {
-  if (typeof content === "string") {
-    return content;
-  }
-  return content.map((item) => {
-    if (item.type === "image") {
-      return {
-        type: "image_url",
-        image_url: {
-          url: `data:${item.mediaType};base64,${item.data}`
-        }
-      };
-    }
-    return {
-      type: "text",
-      text: item.text
-    };
-  });
-}
 function extractHeaders(request) {
   if (!request) return {};
   const headers = {};
@@ -1635,7 +1729,11 @@ var Runtime = class {
     const messages = request.messages.map(
       (m, i) => createMessage({
         role: m.role,
-        content: m.content
+        content: m.content,
+        providerMetadata: m.providerMetadata,
+        modelOutput: m.modelOutput,
+        tool_calls: m.tool_calls,
+        tool_call_id: m.tool_call_id
       })
     );
     const allActions = [...this.actions.values()];
@@ -2252,6 +2350,7 @@ var Runtime = class {
     let currentToolCall = null;
     const serverToolResults = [];
     const toolContextData = "toolContext" in this.config ? this.config.toolContext : void 0;
+    let adapterProviderMetadata;
     let adapterUsage;
     const completionRequest = {
       messages: [],
@@ -2350,18 +2449,24 @@ var Runtime = class {
             );
             try {
               const result = await tool2.handler(args, toolContext);
+              const modelOutput = resolveToolOutput(
+                result,
+                tool2.toModelOutput?.(result)
+              );
               serverToolResults.push({
                 id: event.id,
                 name: toolName,
                 args,
                 result,
+                modelOutput,
                 tool: tool2
               });
               yield {
                 type: "action:end",
                 id: event.id,
                 name: toolName,
-                result
+                result,
+                modelOutput
               };
             } catch (error) {
               const errorResult = {
@@ -2395,6 +2500,7 @@ var Runtime = class {
           return;
         // Exit on error
         case "done":
+          adapterProviderMetadata = event.providerMetadata;
           if (event.usage) {
             adapterUsage = event.usage;
           }
@@ -2415,11 +2521,11 @@ var Runtime = class {
         (tc) => !serverToolIds.has(tc.id)
       );
       const serverToolResultMessages = serverToolResults.map((tr) => {
-        const aiContent = buildToolResultForAI(tr.tool, tr.result, tr.args);
-        const content = typeof aiContent === "string" ? aiContent : JSON.stringify(serializeToolResultContent(aiContent));
+        const content = tr.modelOutput ? "[Tool returned content]" : buildToolResultForAI(tr.tool, tr.result, tr.args);
         return {
           role: "tool",
           content,
+          ...tr.modelOutput ? { modelOutput: tr.modelOutput } : {},
           tool_call_id: tr.id
         };
       });
@@ -2432,6 +2538,7 @@ var Runtime = class {
         const assistantMessage = {
           role: "assistant",
           content: accumulatedText || null,
+          ...adapterProviderMetadata ? { providerMetadata: adapterProviderMetadata } : {},
           tool_calls: toolCalls.map((tc) => ({
             id: tc.id,
             type: "function",
@@ -2466,6 +2573,7 @@ var Runtime = class {
         const assistantWithToolCalls = {
           role: "assistant",
           content: accumulatedText || null,
+          ...adapterProviderMetadata ? { providerMetadata: adapterProviderMetadata } : {},
           tool_calls: serverToolResults.map((tr) => {
             const tc = toolCalls.find((t) => t.id === tr.id);
             return {
@@ -2516,6 +2624,7 @@ var Runtime = class {
         const assistantMessage = {
           role: "assistant",
           content: accumulatedText || null,
+          ...adapterProviderMetadata ? { providerMetadata: adapterProviderMetadata } : {},
           tool_calls: clientToolCalls.map((tc) => ({
             id: tc.id,
             type: "function",
@@ -2541,10 +2650,11 @@ var Runtime = class {
         return;
       }
     }
-    if (accumulatedText) {
+    if (accumulatedText || adapterProviderMetadata) {
       newMessages.push({
         role: "assistant",
-        content: accumulatedText
+        content: accumulatedText,
+        ...adapterProviderMetadata ? { providerMetadata: adapterProviderMetadata } : {}
       });
     }
     if (debug) {
@@ -2659,7 +2769,10 @@ var Runtime = class {
             content: result.content
           };
         }
-        yield { type: "message:end" };
+        yield {
+          type: "message:end",
+          providerMetadata: result.providerMetadata
+        };
         if (result.toolCalls.length > 0) {
           const serverToolCalls = [];
           const clientToolCalls = [];
@@ -2706,18 +2819,24 @@ var Runtime = class {
               );
               try {
                 const toolResult = await tool2.handler(tc.args, toolContext);
+                const modelOutput = resolveToolOutput(
+                  toolResult,
+                  tool2.toModelOutput?.(toolResult)
+                );
                 serverToolResults.push({
                   id: tc.id,
                   name: tc.name,
                   args: tc.args,
                   result: toolResult,
+                  modelOutput,
                   tool: tool2
                 });
                 yield {
                   type: "action:end",
                   id: tc.id,
                   name: tc.name,
-                  result: toolResult
+                  result: toolResult,
+                  modelOutput
                 };
               } catch (error) {
                 const errorResult = {
@@ -2741,15 +2860,11 @@ var Runtime = class {
             }
           }
           const serverToolResultMessages = serverToolResults.map((tr) => {
-            const aiContent = buildToolResultForAI(
-              tr.tool,
-              tr.result,
-              tr.args
-            );
-            const content = typeof aiContent === "string" ? aiContent : JSON.stringify(serializeToolResultContent(aiContent));
+            const content = tr.modelOutput ? "[Tool returned content]" : buildToolResultForAI(tr.tool, tr.result, tr.args);
             return {
               role: "tool",
               content,
+              ...tr.modelOutput ? { modelOutput: tr.modelOutput } : {},
               tool_call_id: tr.id
             };
           });
@@ -2757,6 +2872,7 @@ var Runtime = class {
             const assistantMessage = {
               role: "assistant",
               content: result.content || null,
+              ...result.providerMetadata ? { providerMetadata: result.providerMetadata } : {},
               tool_calls: result.toolCalls.map((tc) => ({
                 id: tc.id,
                 type: "function",
@@ -2786,6 +2902,7 @@ var Runtime = class {
             const assistantWithToolCalls = {
               role: "assistant",
               content: result.content || null,
+              ...result.providerMetadata ? { providerMetadata: result.providerMetadata } : {},
               tool_calls: result.toolCalls.map((tc) => ({
                 id: tc.id,
                 type: "function",
@@ -2817,6 +2934,7 @@ var Runtime = class {
             const assistantMessage = {
               role: "assistant",
               content: result.content || null,
+              ...result.providerMetadata ? { providerMetadata: result.providerMetadata } : {},
               tool_calls: clientToolCalls.map((tc) => ({
                 id: tc.id,
                 type: "function",
@@ -2842,10 +2960,11 @@ var Runtime = class {
             return;
           }
         }
-        if (result.content) {
+        if (result.content || result.providerMetadata) {
           newMessages.push({
             role: "assistant",
-            content: result.content
+            content: result.content,
+            ...result.providerMetadata ? { providerMetadata: result.providerMetadata } : {}
           });
         }
         if (debug) {
@@ -3997,6 +4116,54 @@ async function executeToolCalls(toolCalls, tools, executeServerTool, waitForClie
   return results;
 }
 
+// src/core/audio.ts
+var audioErrorMarker = /* @__PURE__ */ Symbol.for("@yourgpt/llm-sdk/AudioAPIError");
+var _a;
+var AudioAPIError = class extends Error {
+  constructor(message, code, status, requestId) {
+    super(message);
+    this.code = code;
+    this.status = status;
+    this.requestId = requestId;
+    this[_a] = true;
+    this.name = "AudioAPIError";
+  }
+  static [(_a = audioErrorMarker, Symbol.hasInstance)](value) {
+    return !!value && typeof value === "object" && value[audioErrorMarker] === true;
+  }
+};
+async function generateSpeech({
+  model,
+  ...options
+}) {
+  options.signal?.throwIfAborted();
+  if (typeof options.text !== "string" || !options.text.trim())
+    throw new AudioAPIError("Speech text must not be empty", "invalid_input");
+  const result = await model.doGenerate(options);
+  options.signal?.throwIfAborted();
+  if (!(result.audio?.uint8Array instanceof Uint8Array) || !result.audio.uint8Array.length)
+    throw new AudioAPIError(
+      "Speech provider returned empty audio",
+      "invalid_response"
+    );
+  return result;
+}
+async function transcribe({
+  model,
+  ...options
+}) {
+  options.signal?.throwIfAborted();
+  const result = await model.doTranscribe(options);
+  options.signal?.throwIfAborted();
+  if (typeof result.text !== "string")
+    throw new AudioAPIError(
+      "Transcription provider returned an invalid transcript",
+      "invalid_response"
+    );
+  return result;
+}
+
+exports.AudioAPIError = AudioAPIError;
 exports.DEFAULT_CAPABILITIES = DEFAULT_CAPABILITIES;
 exports.DEFAULT_MAX_ITERATIONS = DEFAULT_MAX_ITERATIONS;
 exports.GenerateResult = GenerateResult;
@@ -4021,6 +4188,7 @@ exports.formatToolsForAnthropic = formatToolsForAnthropic;
 exports.formatToolsForGoogle = formatToolsForGoogle;
 exports.formatToolsForOpenAI = formatToolsForOpenAI;
 exports.generateMessageId = generateMessageId;
+exports.generateSpeech = generateSpeech;
 exports.generateText = generateText;
 exports.generateThreadId = generateThreadId;
 exports.generateToolCallId = generateToolCallId;
@@ -4033,5 +4201,6 @@ exports.selectTools = selectTools;
 exports.shouldExposeToolSearch = shouldExposeToolSearch;
 exports.streamText = streamText;
 exports.tool = tool;
+exports.transcribe = transcribe;
 //# sourceMappingURL=index.js.map
 //# sourceMappingURL=index.js.map