@codeproxy/core 0.1.12 → 0.1.14

package/dist/index.d.cts

@@ -5,7 +5,7 @@
  */
 type ResponsesInputItem = string | ResponsesMessageItem | ResponsesReasoningItem | ResponsesFunctionCallItem | ResponsesFunctionCallOutputItem | ResponsesLocalShellCallItem | ResponsesCommandExecutionItem | ResponsesCommandExecutionOutputItem | ResponsesCustomToolCallItem | ResponsesCustomToolCallOutputItem | ResponsesFileChangeItem | ResponsesFileChangeOutputItem | ResponsesWebSearchCallItem | Record<string, unknown>;
 interface ResponsesContentPart {
-    type: 'input_text' | 'text' | 'output_text' | 'reasoning_text' | 'input_image' | 'image' | 'image_url' | 'input_file' | 'file' | 'tool_result' | string;
+    type: 'input_text' | 'text' | 'output_text' | 'reasoning_text' | 'input_image' | 'image' | 'image_url' | 'input_file' | 'file' | 'input_audio' | 'tool_result' | string;
     text?: string;
     image_url?: string | {
         url: string;
@@ -19,6 +19,11 @@ interface ResponsesContentPart {
     file_url?: string | {
         url: string;
     };
+    input_audio?: {
+        data?: string;
+        format?: string;
+    };
+    format?: string;
     tool_use_id?: string;
     call_id?: string;
     content?: unknown;

package/dist/index.d.ts

@@ -5,7 +5,7 @@
  */
 type ResponsesInputItem = string | ResponsesMessageItem | ResponsesReasoningItem | ResponsesFunctionCallItem | ResponsesFunctionCallOutputItem | ResponsesLocalShellCallItem | ResponsesCommandExecutionItem | ResponsesCommandExecutionOutputItem | ResponsesCustomToolCallItem | ResponsesCustomToolCallOutputItem | ResponsesFileChangeItem | ResponsesFileChangeOutputItem | ResponsesWebSearchCallItem | Record<string, unknown>;
 interface ResponsesContentPart {
-    type: 'input_text' | 'text' | 'output_text' | 'reasoning_text' | 'input_image' | 'image' | 'image_url' | 'input_file' | 'file' | 'tool_result' | string;
+    type: 'input_text' | 'text' | 'output_text' | 'reasoning_text' | 'input_image' | 'image' | 'image_url' | 'input_file' | 'file' | 'input_audio' | 'tool_result' | string;
     text?: string;
     image_url?: string | {
         url: string;
@@ -19,6 +19,11 @@ interface ResponsesContentPart {
     file_url?: string | {
         url: string;
     };
+    input_audio?: {
+        data?: string;
+        format?: string;
+    };
+    format?: string;
     tool_use_id?: string;
     call_id?: string;
     content?: unknown;

package/dist/index.js

@@ -1209,41 +1209,30 @@ function processInputItem(item, messages, options) {
               contentBlocks.push({ type: "text", text: String(contentPart.text ?? "") });
             } else if (contentPart.type === "reasoning_text") {
               reasoningContent += String(contentPart.text ?? "");
-            } else if (contentPart.type === "input_image" || contentPart.type === "image" || contentPart.type === "image_url") {
+            } else if (isImagePart(contentPart)) {
               if (options.dropImages) {
                 continue;
               }
-              let url = "";
-              const partWithImage = part;
-              const imgUrl = partWithImage.image_url;
-              if (typeof imgUrl === "string") {
-                url = imgUrl;
-              } else if (imgUrl && typeof imgUrl === "object" && imgUrl.url) {
-                url = imgUrl.url;
-              } else {
-                const partWithData = part;
-                const imgData = String(partWithData.data ?? partWithData.base64 ?? "");
-                if (imgData) {
-                  const partWithMime = part;
-                  const mimeType = String(
-                    partWithMime.mime_type ?? partWithMime.media_type ?? "image/png"
-                  );
-                  url = imgData.startsWith("data:") ? imgData : `data:${mimeType};base64,${imgData}`;
-                }
-              }
+              const url = imagePartToUrl(part);
               if (url) {
                 contentBlocks.push({ type: "image_url", image_url: { url } });
               }
             } else if (part.type === "input_file" || part.type === "file") {
-              const partFile = part;
-              const fileData = String(partFile.file_data ?? partFile.data ?? "");
+              const fileData = String(contentPart.file_data ?? contentPart.data ?? "");
               const mimeType = String(
-                partFile.mime_type ?? partFile.media_type ?? "application/pdf"
+                contentPart.mime_type ?? contentPart.media_type ?? "application/pdf"
               );
               if (fileData) {
                 const url = fileData.startsWith("data:") ? fileData : `data:${mimeType};base64,${fileData}`;
                 contentBlocks.push({ type: "image_url", image_url: { url } });
               }
+            } else if (contentPart.type === "input_audio") {
+              const ia = contentPart.input_audio;
+              const data = String(ia?.data ?? contentPart.data ?? "");
+              const format = String(ia?.format ?? contentPart.format ?? "mp3");
+              if (data) {
+                contentBlocks.push({ type: "input_audio", input_audio: { data, format } });
+              }
             }
           }
         }
@@ -1289,7 +1278,7 @@ function processInputItem(item, messages, options) {
     return;
   }
   if (itemType === "function_call_output" || itemType === "commandExecutionOutput" || itemType === "fileChangeOutput" || itemType === "custom_tool_call_output") {
-    processToolOutput(item, messages);
+    processToolOutput(item, messages, options);
     return;
   }
 }
@@ -1355,10 +1344,29 @@ function processToolCall(item, messages, getLastAssistant, fallbackThoughtSignat
     amsg.reasoning_content = (amsg.reasoning_content ?? "") + thought;
   }
 }
-function processToolOutput(item, messages) {
+function isImagePart(part) {
+  return part.type === "input_image" || part.type === "image" || part.type === "image_url";
+}
+function imagePartToUrl(part) {
+  const imgUrl = part.image_url;
+  if (typeof imgUrl === "string") {
+    return imgUrl;
+  }
+  if (imgUrl && typeof imgUrl === "object" && imgUrl.url) {
+    return imgUrl.url;
+  }
+  const imgData = String(part.data ?? part.base64 ?? "");
+  if (imgData) {
+    const mimeType = String(part.mime_type ?? part.media_type ?? "image/png");
+    return imgData.startsWith("data:") ? imgData : `data:${mimeType};base64,${imgData}`;
+  }
+  return "";
+}
+function processToolOutput(item, messages, options) {
   const callId = item.call_id === void 0 ? void 0 : String(item.call_id);
   const outputRaw = item.output ?? item.content ?? item.stdout ?? "";
   let content = "";
+  const imageBlocks = [];
   if (typeof outputRaw === "string") {
     content = outputRaw;
   } else if (Array.isArray(outputRaw)) {
@@ -1369,6 +1377,11 @@ function processToolOutput(item, messages) {
         const partItem = part;
         if (partItem.type === "input_text" || partItem.type === "text") {
           content += String(partItem.text ?? "");
+        } else if (!options.dropImages && isImagePart(partItem)) {
+          const url = imagePartToUrl(part);
+          if (url) {
+            imageBlocks.push({ type: "image_url", image_url: { url } });
+          }
         }
       }
     }
@@ -1387,6 +1400,15 @@ function processToolOutput(item, messages) {
     tool_call_id: callId,
     content
   });
+  if (imageBlocks.length > 0) {
+    messages.push({
+      role: "user",
+      content: [
+        { type: "text", text: "[Image output returned by the preceding tool call]" },
+        ...imageBlocks
+      ]
+    });
+  }
 }
 function mapTools2(tools) {
   const out = [];