@opencow-ai/opencow-agent-sdk 0.4.11 → 0.4.12

package/dist/client.js

@@ -31267,6 +31267,27 @@ var init_types3 = __esm(() => {
   CanonicalUserAbortError = APIUserAbortError;
 });
 
+// src/session/canonical/imageSource.ts
+function normalizeCanonicalImageSource(source) {
+  if (!source || typeof source !== "object")
+    return null;
+  const s = source;
+  if (s.type === "base64" && typeof s.data === "string") {
+    return {
+      kind: "base64",
+      mediaType: typeof s.media_type === "string" ? s.media_type : "image/png",
+      data: s.data
+    };
+  }
+  if (s.type === "url" && typeof s.url === "string") {
+    return { kind: "url", url: s.url };
+  }
+  return null;
+}
+function imageSourceToDataUri(img) {
+  return img.kind === "base64" ? `data:${img.mediaType};base64,${img.data}` : img.url;
+}
+
 // src/session/canonical/index.ts
 var init_canonical = __esm(() => {
   init_types3();
@@ -95407,19 +95428,9 @@ function splitToolResultMedia(content) {
       continue;
     }
     if (block?.type === "image") {
-      const src = block.source;
-      if (src?.type === "base64" && typeof src.data === "string") {
-        images.push({
-          kind: "base64",
-          mediaType: src.media_type ?? "image/png",
-          data: src.data
-        });
-        continue;
-      }
-      if (src?.type === "url" && typeof src.url === "string") {
-        images.push({ kind: "url", url: src.url });
-        continue;
-      }
+      const img = normalizeCanonicalImageSource(block.source);
+      if (img)
+        images.push(img);
       continue;
     }
     if (typeof block?.text === "string") {
@@ -95430,8 +95441,7 @@ function splitToolResultMedia(content) {
 `), images };
 }
 function toResponsesInputImagePart(img) {
-  const url3 = img.kind === "base64" ? `data:${img.mediaType};base64,${img.data}` : img.url;
-  return { type: "input_image", image_url: url3 };
+  return { type: "input_image", image_url: imageSourceToDataUri(img) };
 }
 function convertContentBlocksToResponsesParts(content, role) {
   const textType = role === "assistant" ? "output_text" : "input_text";
@@ -95450,18 +95460,9 @@ function convertContentBlocksToResponsesParts(content, role) {
       case "image": {
         if (role === "assistant")
           break;
-        const source = block.source;
-        if (source?.type === "base64") {
-          parts.push({
-            type: "input_image",
-            image_url: `data:${source.media_type};base64,${source.data}`
-          });
-        } else if (source?.type === "url" && source.url) {
-          parts.push({
-            type: "input_image",
-            image_url: source.url
-          });
-        }
+        const img = normalizeCanonicalImageSource(block.source);
+        if (img)
+          parts.push(toResponsesInputImagePart(img));
         break;
       }
       case "thinking":
@@ -96049,6 +96050,7 @@ var init_shim = __esm(() => {
   init_sdk();
   init_schema();
   init_capabilities2();
+  init_canonical();
 });
 
 // src/providers/shared/providerRecommendation.ts
@@ -96191,19 +96193,9 @@ function splitToolResultMedia2(content) {
       continue;
     }
     if (block?.type === "image") {
-      const source = block.source;
-      if (source?.type === "base64" && typeof source.data === "string") {
-        images.push({
-          kind: "base64",
-          mediaType: source.media_type ?? "image/png",
-          data: source.data
-        });
-        continue;
-      }
-      if (source?.type === "url" && typeof source.url === "string") {
-        images.push({ kind: "url", url: source.url });
-        continue;
-      }
+      const img = normalizeCanonicalImageSource(block.source);
+      if (img)
+        images.push(img);
       continue;
     }
     if (block?.type === "tool_reference" && typeof block.tool_name === "string") {
@@ -96218,8 +96210,7 @@ function splitToolResultMedia2(content) {
 `), images };
 }
 function toOpenAIImageUrl(img) {
-  const url3 = img.kind === "base64" ? `data:${img.mediaType};base64,${img.data}` : img.url;
-  return { type: "image_url", image_url: { url: url3 } };
+  return { type: "image_url", image_url: { url: imageSourceToDataUri(img) } };
 }
 function convertContentBlocks(content) {
   if (typeof content === "string")
@@ -96233,17 +96224,9 @@ function convertContentBlocks(content) {
         parts.push({ type: "text", text: block.text ?? "" });
         break;
       case "image": {
-        const src = block.source;
-        if (src?.type === "base64") {
-          parts.push({
-            type: "image_url",
-            image_url: {
-              url: `data:${src.media_type};base64,${src.data}`
-            }
-          });
-        } else if (src?.type === "url") {
-          parts.push({ type: "image_url", image_url: { url: src.url } });
-        }
+        const img = normalizeCanonicalImageSource(block.source);
+        if (img)
+          parts.push(toOpenAIImageUrl(img));
         break;
       }
       case "tool_use":
@@ -97120,6 +97103,7 @@ var init_shim2 = __esm(() => {
   init_schemaSanitizer();
   init_providerProfile();
   init_capabilities2();
+  init_canonical();
   OpenAIShimStream = class OpenAIShimStream {
     generator;
     controller = new AbortController;
@@ -273622,9 +273606,21 @@ var init_mcpWebSocketTransport = __esm(() => {
 });
 
 // src/capabilities/adapters/callToolResultAdapter.ts
+async function buildImageBlock(bytes, mediaType, uploadMedia) {
+  let url3 = null;
+  if (uploadMedia) {
+    try {
+      url3 = await uploadMedia({ bytes, mediaType });
+    } catch (e) {
+      logError(e);
+    }
+  }
+  const source = url3 ? { type: "url", url: url3 } : { type: "base64", media_type: mediaType, data: bytes.toString("base64") };
+  return { type: "image", source };
+}
 async function translateMcpContentItem(input) {
   const { item, options: options2 } = input;
-  const { sourceName } = options2;
+  const { sourceName, uploadMedia } = options2;
   switch (item.type) {
     case "text":
       return [{ type: "text", text: item.text }];
@@ -273642,14 +273638,7 @@ async function translateMcpContentItem(input) {
       const ext = item.mimeType?.split("/")[1] || "png";
       const resized = await maybeResizeAndDownsampleImageBuffer(imageBuffer, imageBuffer.length, ext);
       return [
-        {
-          type: "image",
-          source: {
-            data: resized.buffer.toString("base64"),
-            media_type: `image/${resized.mediaType}`,
-            type: "base64"
-          }
-        }
+        await buildImageBlock(resized.buffer, `image/${resized.mediaType}`, uploadMedia)
       ];
     }
     case "resource": {
@@ -273666,14 +273655,7 @@ async function translateMcpContentItem(input) {
           const resized = await maybeResizeAndDownsampleImageBuffer(imageBuffer, imageBuffer.length, ext);
           return [
             { type: "text", text: prefix },
-            {
-              type: "image",
-              source: {
-                data: resized.buffer.toString("base64"),
-                media_type: `image/${resized.mediaType}`,
-                type: "base64"
-              }
-            }
+            await buildImageBlock(resized.buffer, `image/${resized.mediaType}`, uploadMedia)
           ];
         }
         return await persistBlobToTextBlock({
@@ -273727,6 +273709,7 @@ var IMAGE_MIME_TYPES;
 var init_callToolResultAdapter = __esm(() => {
   init_imageResizer();
   init_mcpOutputStorage();
+  init_log2();
   IMAGE_MIME_TYPES = new Set([
     "image/jpeg",
     "image/png",
@@ -282692,7 +282675,7 @@ function getAnthropicEnvMetadata() {
 function getBuildAgeMinutes() {
   if (false)
     ;
-  const buildTime = new Date("2026-06-24T03:20:56.844Z").getTime();
+  const buildTime = new Date("2026-06-24T10:02:32.669Z").getTime();
   if (isNaN(buildTime))
     return;
   return Math.floor((Date.now() - buildTime) / 60000);
@@ -286016,6 +285999,16 @@ async function validateContentTokens(content, ext, maxTokens) {
     throw new MaxFileReadTokenExceededError(effectiveCount, effectiveMaxTokens);
   }
 }
+async function maybeUploadBytes(uploadMedia, bytes, mediaType) {
+  if (!uploadMedia)
+    return null;
+  try {
+    return await uploadMedia({ bytes, mediaType });
+  } catch (e) {
+    logError(e);
+    return null;
+  }
+}
 function createImageResponse(buffer, mediaType, originalSize, dimensions) {
   return {
     type: "image",
@@ -286063,6 +286056,10 @@ async function callInner(file_path, fullFilePath, resolvedFilePath, ext, offset,
   if (IMAGE_EXTENSIONS.has(ext)) {
     const data2 = await readImageWithTokenBudget(resolvedFilePath, maxTokens);
     context4.nestedMemoryAttachmentTriggers?.add(fullFilePath);
+    const uploadedUrl = await maybeUploadBytes(context4.uploadMedia, Buffer.from(data2.file.base64, "base64"), data2.file.type);
+    if (uploadedUrl) {
+      data2.file.url = uploadedUrl;
+    }
     logFileOperation({
       operation: "read",
       tool: "FileReadTool",
@@ -286104,14 +286101,14 @@ async function callInner(file_path, fullFilePath, resolvedFilePath, ext, offset,
         const imgPath = path11.join(extractResult.data.file.outputDir, f);
         const imgBuffer = await getFsImplementation().readFile(imgPath);
         const resized = await maybeResizeAndDownsampleImageBuffer(imgBuffer, imgBuffer.length, "jpeg");
-        return {
-          type: "image",
-          source: {
-            type: "base64",
-            media_type: `image/${resized.mediaType}`,
-            data: resized.buffer.toString("base64")
-          }
+        const mediaType = `image/${resized.mediaType}`;
+        const uploadedUrl2 = await maybeUploadBytes(context4.uploadMedia, resized.buffer, mediaType);
+        const source = uploadedUrl2 ? { type: "url", url: uploadedUrl2 } : {
+          type: "base64",
+          media_type: mediaType,
+          data: resized.buffer.toString("base64")
         };
+        return { type: "image", source };
       }));
       return {
         data: extractResult.data,
@@ -286159,20 +286156,17 @@ async function callInner(file_path, fullFilePath, resolvedFilePath, ext, offset,
       filePath: fullFilePath,
       content: pdfData.file.base64
     });
+    const uploadedUrl = await maybeUploadBytes(context4.uploadMedia, Buffer.from(pdfData.file.base64, "base64"), "application/pdf");
+    const documentSource = uploadedUrl ? { type: "url", url: uploadedUrl } : {
+      type: "base64",
+      media_type: "application/pdf",
+      data: pdfData.file.base64
+    };
     return {
       data: pdfData,
       newMessages: [
         createUserMessage({
-          content: [
-            {
-              type: "document",
-              source: {
-                type: "base64",
-                media_type: "application/pdf",
-                data: pdfData.file.base64
-              }
-            }
-          ],
+          content: [{ type: "document", source: documentSource }],
           isMeta: true
         })
       ]
@@ -286379,6 +286373,7 @@ var init_FileReadTool = __esm(() => {
           base64: exports_external2.string().describe("Base64-encoded image data"),
           type: imageMediaTypes.describe("The MIME type of the image"),
           originalSize: exports_external2.number().describe("Original file size in bytes"),
+          url: exports_external2.string().optional().describe("Fetchable URL for the (compressed) image when a host uploader is configured; takes precedence over base64 for rendering"),
           dimensions: exports_external2.object({
             originalWidth: exports_external2.number().optional().describe("Original image width in pixels"),
             originalHeight: exports_external2.number().optional().describe("Original image height in pixels"),
@@ -286607,17 +286602,18 @@ var init_FileReadTool = __esm(() => {
     mapToolResultToToolResultBlockParam(data, toolUseID) {
       switch (data.type) {
         case "image": {
+          const source = data.file.url ? { type: "url", url: data.file.url } : {
+            type: "base64",
+            data: data.file.base64,
+            media_type: data.file.type
+          };
           return {
             tool_use_id: toolUseID,
             type: "tool_result",
             content: [
               {
                 type: "image",
-                source: {
-                  type: "base64",
-                  data: data.file.base64,
-                  media_type: data.file.type
-                }
+                source
               }
             ]
           };
@@ -300013,6 +300009,7 @@ class QueryEngine {
       agents = [],
       setSDKStatus,
       onToolInvoke,
+      uploadMedia,
       orphanedPermission
     } = this.config;
     this.discoveredSkillNames.clear();
@@ -300116,7 +300113,8 @@ class QueryEngine {
         });
       },
       setSDKStatus,
-      onToolInvoke
+      onToolInvoke,
+      uploadMedia
     };
     if (orphanedPermission && !this.hasHandledOrphanedPermission) {
       this.hasHandledOrphanedPermission = true;
@@ -300205,7 +300203,8 @@ class QueryEngine {
       setResponseLength: () => {},
       updateFileHistoryState: processUserInputContext.updateFileHistoryState,
       updateAttributionState: processUserInputContext.updateAttributionState,
-      setSDKStatus
+      setSDKStatus,
+      uploadMedia
     };
     headlessProfilerCheckpoint("before_skills_plugins");
     const [skills, { enabled: enabledPlugins }] = await Promise.all([
@@ -300700,6 +300699,7 @@ async function* ask({
   agents = [],
   setSDKStatus,
   onToolInvoke,
+  uploadMedia,
   orphanedPermission
 }) {
   const engine = new QueryEngine({
@@ -300732,6 +300732,7 @@ async function* ask({
     includePartialMessages,
     setSDKStatus,
     onToolInvoke,
+    uploadMedia,
     abortController,
     orphanedPermission,
     ...{}
@@ -329479,7 +329480,7 @@ function toInternalTool(sdkTool) {
       });
       const content = await translateCallToolResultContent({
         content: result.content,
-        options: { sourceName: sdkTool.name }
+        options: { sourceName: sdkTool.name, uploadMedia: context4.uploadMedia }
       });
       return { data: content };
     }
@@ -330002,6 +330003,7 @@ function runSdkQueryRuntime(params) {
           includePartialMessages: Boolean(options2.includePartialMessages),
           agents: mergedAgents,
           onToolInvoke: typeof options2.onToolInvoke === "function" ? options2.onToolInvoke : undefined,
+          uploadMedia: typeof options2.uploadMedia === "function" ? options2.uploadMedia : undefined,
           setSDKStatus: (status) => {
             turnStatusStream.enqueue({
               type: "system",
@@ -336055,4 +336057,4 @@ export {
   AbortError2 as AbortError
 };
 
-//# debugId=7376AE95E91B9BFD64756E2164756E21
+//# debugId=C79EE43EDCAB4CD864756E2164756E21

package/dist/entrypoints/sdk/runtimeTypes.d.ts

@@ -1,5 +1,6 @@
 import type { ChildProcessWithoutNullStreams } from 'node:child_process';
 import type { ModelProviders } from '../../providers/shared/routing.js';
+import type { UploadMediaFn } from '../../types/toolRuntime.js';
 export type { ModelProviders, ModelProviderConfig } from '../../providers/shared/routing.js';
 import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 import type { CallToolResult, ToolAnnotations } from '@modelcontextprotocol/sdk/types.js';
@@ -278,6 +279,21 @@ export type Options = {
         toolInput: Record<string, unknown>;
         toolUseId: string;
     }) => void | Promise<void>;
+    /**
+     * Host-injected media uploader. When supplied, media-producing built-in
+     * tools (FileReadTool reading an image) upload the **compressed** bytes
+     * (post token-budget) and emit `{type:'image', source:{type:'url', url}}`
+     * in the tool_result instead of inline base64 — keeping the bytes out of
+     * every subsequent turn's request and letting the host materialize them.
+     * Media-neutral: the same port serves image / PDF / page-image tool output,
+     * driven by `mediaType`.
+     *
+     * Returns a fetchable URL, or `null` when upload is unavailable; the SDK
+     * also treats a thrown error as "unavailable". Either way it falls back to
+     * inline base64, so standalone SDK/CLI runs (no uploader) are byte-for-byte
+     * unchanged.
+     */
+    uploadMedia?: UploadMediaFn;
     jsonSchema?: Record<string, unknown>;
     betas?: string[];
     settingSources?: SettingSource[];