@codemation/core-nodes 0.12.0 → 0.13.0

package/dist/metadata.json

@@ -1,7 +1,7 @@
 {
   "schemaVersion": 1,
   "packageName": "@codemation/core-nodes",
-  "packageVersion": "0.12.0",
+  "packageVersion": "0.13.0",
   "description": "",
   "kind": "nodes",
   "nodes": [

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@codemation/core-nodes",
-  "version": "0.12.0",
+  "version": "0.13.0",
   "publishConfig": {
     "access": "public"
   },

package/src/nodes/AIAgentConfig.ts

@@ -58,6 +58,12 @@ export interface AIAgentOptions<TInputJson = unknown, _TOutputJson = unknown> {
    * entirely (the node then behaves as if no binaries were present).
    */
   readonly passBinariesToModel?: boolean;
+  /**
+   * Whether binaries returned by a tool (e.g. an MCP tool returning a PDF or image) are passed to
+   * the chat model as native multimodal tool-result blocks. Defaults to `true`. Set to `false` to
+   * keep tool results as inert JSON text (the model then never "sees" the document).
+   */
+  readonly passToolBinariesToModel?: boolean;
   /**
    * Explicit binaries to pass to the chat model, instead of the ones on the current item.
    * Either a static array or a function resolved per item (so an author can forward binaries
@@ -96,6 +102,7 @@ export class AIAgent<TInputJson = unknown, TOutputJson = unknown>
   readonly pinnedMcpTools?: readonly string[];
   readonly untrustedSources?: ReadonlyArray<string>;
   readonly passBinariesToModel?: boolean;
+  readonly passToolBinariesToModel?: boolean;
   readonly binaries?:
     | ReadonlyArray<BinaryAttachment>
     | ((args: AgentMessageBuildArgs<TInputJson>) => ReadonlyArray<BinaryAttachment>);
@@ -115,6 +122,7 @@ export class AIAgent<TInputJson = unknown, TOutputJson = unknown>
     this.pinnedMcpTools = options.pinnedMcpTools;
     this.untrustedSources = options.untrustedSources;
     this.passBinariesToModel = options.passBinariesToModel;
+    this.passToolBinariesToModel = options.passToolBinariesToModel;
     this.binaries = options.binaries;
   }
 

package/src/nodes/AIAgentNode.ts

@@ -210,7 +210,7 @@ export class AIAgentNode implements RunnableNode<AIAgent<any, any>> {
     };
     const conversation: ModelMessage[] = [
       ...checkpoint.conversation,
-      AgentMessageFactory.createToolResultsMessage([toolResultEntry]),
+      AgentMessageFactory.createToolResultsMessage([toolResultEntry], ctx.config.passToolBinariesToModel !== false),
     ];
 
     const loopResult = await this.runTurnLoopUntilFinalAnswer({
@@ -490,6 +490,7 @@ export class AIAgentNode implements RunnableNode<AIAgent<any, any>> {
         result.text,
         result.toolCalls,
         allExecutedCalls,
+        ctx.config.passToolBinariesToModel !== false,
       );
     }
 
@@ -522,10 +523,11 @@ export class AIAgentNode implements RunnableNode<AIAgent<any, any>> {
     text: string,
     toolCalls: ReadonlyArray<AgentToolCall>,
     executedToolCalls: ReadonlyArray<ExecutedToolCall>,
+    passToolBinariesToModel: boolean,
   ): void {
     conversation.push(
       assistantMessage ?? AgentMessageFactory.createAssistantWithToolCalls(text, toolCalls),
-      AgentMessageFactory.createToolResultsMessage(executedToolCalls),
+      AgentMessageFactory.createToolResultsMessage(executedToolCalls, passToolBinariesToModel),
     );
   }
 

package/src/nodes/AgentMessageFactory.ts

@@ -1,7 +1,8 @@
 import type { AgentMessageDto, AgentToolCall } from "@codemation/core";
 
-import type { AssistantModelMessage, ModelMessage, ToolModelMessage } from "ai";
+import type { AssistantModelMessage, ModelMessage, ToolModelMessage, ToolResultPart } from "ai";
 
+import { AgentToolResultContentFactory } from "./AgentToolResultContentFactory";
 import type { ExecutedToolCall } from "./aiAgentSupport.types";
 
 /**
@@ -41,21 +42,36 @@ export class AgentMessageFactory {
    * Builds the `{ role: "tool", content: [{ type: "tool-result", ... }, ...] }` message returned
    * to the model after each tool round.
    */
-  static createToolResultsMessage(executedToolCalls: ReadonlyArray<ExecutedToolCall>): ToolModelMessage {
+  static createToolResultsMessage(
+    executedToolCalls: ReadonlyArray<ExecutedToolCall>,
+    passToolBinariesToModel = true,
+  ): ToolModelMessage {
     return {
       role: "tool",
       content: executedToolCalls.map((executed) => ({
         type: "tool-result",
         toolCallId: executed.toolCallId,
         toolName: executed.toolName,
-        output: {
-          type: "json",
-          value: AgentMessageFactory.toToolResultJson(executed.result),
-        },
+        output: AgentMessageFactory.toToolResultOutput(executed.result, passToolBinariesToModel),
       })),
     };
   }
 
+  /**
+   * Routes a tool result to a native multimodal `{ type: "content" }` output when it is
+   * content-block-shaped (an MCP `CallToolResult`) and binary passdown is enabled; otherwise keeps
+   * the inert `{ type: "json" }` path.
+   */
+  private static toToolResultOutput(result: unknown, passToolBinariesToModel: boolean): ToolResultPart["output"] {
+    if (passToolBinariesToModel) {
+      const content = AgentToolResultContentFactory.tryMapToContentOutput(result);
+      if (content !== undefined) {
+        return { type: "content", value: content };
+      }
+    }
+    return { type: "json", value: AgentMessageFactory.toToolResultJson(result) };
+  }
+
   private static toToolResultJson(value: unknown): import("ai").JSONValue {
     if (value === undefined) return null;
     try {

package/src/nodes/AgentToolResultContentFactory.ts

@@ -0,0 +1,155 @@
+import type { ToolResultPart } from "ai";
+
+/** The `value` array of a `{ type: "content" }` tool-result output, as accepted by the AI SDK. */
+type ContentOutputValue = Extract<ToolResultPart["output"], { type: "content" }>["value"];
+type ContentOutputPart = ContentOutputValue[number];
+
+/**
+ * Cap on raw (pre-base64) bytes inlined from a single tool result. Base64 inflates ~33% and every
+ * inlined byte eats model context, so oversize binaries are replaced with a text placeholder.
+ */
+const MAX_INLINE_BYTES = 8 * 1024 * 1024;
+
+/** MCP content-block discriminators. At least one must appear for a result to be treated as MCP-shaped. */
+const KNOWN_MCP_BLOCK_TYPES: ReadonlySet<string> = new Set(["text", "image", "audio", "resource", "resource_link"]);
+
+/**
+ * MCP-style content block, as returned verbatim by a `CallToolResult` (`@ai-sdk/mcp` tool `execute`).
+ * Only the fields this factory reads are modelled; unknown shapes fall through to a text marker.
+ */
+type McpContentBlock = Readonly<{
+  type?: unknown;
+  text?: unknown;
+  data?: unknown;
+  mimeType?: unknown;
+  uri?: unknown;
+  name?: unknown;
+  resource?: Readonly<{ blob?: unknown; text?: unknown; mimeType?: unknown; uri?: unknown; name?: unknown }>;
+}>;
+
+/**
+ * Maps a tool result that is **content-block-shaped** (an MCP `CallToolResult` with a `content`
+ * array) into AI SDK `{ type: "content" }` tool-result output, so binaries reach the chat model as
+ * native multimodal tool-result blocks instead of being flattened to inert JSON text.
+ *
+ * The `@ai-sdk/anthropic` provider maps a `content`-output part as follows:
+ * `text` → text block, `image-data` → image block, `file-data` (only `application/pdf`) → document
+ * block. Non-PDF `file-data` is dropped by the provider, so this factory emits `image-data` for
+ * images, `file-data` only for PDFs, and a text marker for every other binary type.
+ *
+ * Returns `undefined` when the result is not content-block-shaped — callers keep the existing
+ * `{ type: "json" }` path, so plain string/object tool results are unaffected.
+ */
+export class AgentToolResultContentFactory {
+  static tryMapToContentOutput(result: unknown): ContentOutputValue | undefined {
+    const blocks = AgentToolResultContentFactory.contentBlocks(result);
+    if (blocks === undefined) return undefined;
+
+    const parts: ContentOutputPart[] = [];
+    let inlinedBytes = 0;
+    for (const block of blocks) {
+      const mapped = AgentToolResultContentFactory.mapBlock(block, inlinedBytes);
+      parts.push(mapped.part);
+      inlinedBytes += mapped.bytes;
+    }
+    return parts;
+  }
+
+  /**
+   * Returns the `content` array iff `result` is an object whose `content` is an array of typed
+   * blocks AND at least one block carries a known MCP discriminator. A plain JSON result that merely
+   * has a `content` key of some other shape (e.g. Notion/Slack rich-text blocks) is rejected,
+   * preserving the `{ type: "json" }` path so its payload is never lost.
+   */
+  private static contentBlocks(result: unknown): ReadonlyArray<McpContentBlock> | undefined {
+    if (result === null || typeof result !== "object") return undefined;
+    const content = (result as { content?: unknown }).content;
+    if (!Array.isArray(content) || content.length === 0) return undefined;
+    const allTyped = content.every(
+      (block) => block !== null && typeof block === "object" && typeof (block as { type?: unknown }).type === "string",
+    );
+    if (!allTyped) return undefined;
+    const looksMcp = content.some((block) => KNOWN_MCP_BLOCK_TYPES.has((block as { type: string }).type));
+    return looksMcp ? (content as ReadonlyArray<McpContentBlock>) : undefined;
+  }
+
+  private static mapBlock(
+    block: McpContentBlock,
+    inlinedBytesSoFar: number,
+  ): Readonly<{ part: ContentOutputPart; bytes: number }> {
+    const type = block.type;
+    if (type === "text" && typeof block.text === "string") {
+      return { part: { type: "text", text: block.text }, bytes: 0 };
+    }
+    if (type === "image" && typeof block.data === "string" && typeof block.mimeType === "string") {
+      return AgentToolResultContentFactory.mapBinary({
+        base64: block.data,
+        mediaType: block.mimeType,
+        inlinedBytesSoFar,
+      });
+    }
+    if (type === "resource" && block.resource) {
+      return AgentToolResultContentFactory.mapEmbeddedResource(block.resource, inlinedBytesSoFar);
+    }
+    if (type === "resource_link" && typeof block.uri === "string") {
+      const mime = typeof block.mimeType === "string" ? ` (${block.mimeType})` : "";
+      return { part: { type: "text", text: `[linked resource: ${block.uri}${mime}]` }, bytes: 0 };
+    }
+    return { part: { type: "text", text: `[unsupported tool content block: ${String(type)}]` }, bytes: 0 };
+  }
+
+  private static mapEmbeddedResource(
+    resource: NonNullable<McpContentBlock["resource"]>,
+    inlinedBytesSoFar: number,
+  ): Readonly<{ part: ContentOutputPart; bytes: number }> {
+    if (typeof resource.text === "string") {
+      return { part: { type: "text", text: resource.text }, bytes: 0 };
+    }
+    if (typeof resource.blob === "string" && typeof resource.mimeType === "string") {
+      return AgentToolResultContentFactory.mapBinary({
+        base64: resource.blob,
+        mediaType: resource.mimeType,
+        filename: typeof resource.name === "string" ? resource.name : undefined,
+        inlinedBytesSoFar,
+      });
+    }
+    const uri = typeof resource.uri === "string" ? resource.uri : "unknown";
+    return { part: { type: "text", text: `[embedded resource: ${uri}]` }, bytes: 0 };
+  }
+
+  private static mapBinary(
+    args: Readonly<{ base64: string; mediaType: string; filename?: string; inlinedBytesSoFar: number }>,
+  ): Readonly<{ part: ContentOutputPart; bytes: number }> {
+    const rawBytes = Math.floor((args.base64.length * 3) / 4);
+    if (args.inlinedBytesSoFar + rawBytes > MAX_INLINE_BYTES) {
+      const name = args.filename ? ` "${args.filename}"` : "";
+      const kb = Math.round(rawBytes / 1024);
+      return {
+        part: {
+          type: "text",
+          text: `[binary${name} (${args.mediaType}, ~${kb} KB) omitted: exceeds the per-tool-result inline limit]`,
+        },
+        bytes: 0,
+      };
+    }
+    if (args.mediaType.startsWith("image/")) {
+      return { part: { type: "image-data", data: args.base64, mediaType: args.mediaType }, bytes: rawBytes };
+    }
+    if (args.mediaType === "application/pdf") {
+      return {
+        part: {
+          type: "file-data",
+          data: args.base64,
+          mediaType: args.mediaType,
+          ...(args.filename ? { filename: args.filename } : {}),
+        },
+        bytes: rawBytes,
+      };
+    }
+    const name = args.filename ? ` "${args.filename}"` : "";
+    return {
+      part: { type: "text", text: `[binary${name} (${args.mediaType}) not inlined: unsupported by the model]` },
+      bytes: 0,
+    };
+  }
+}