@codemation/core-nodes 0.12.0 → 0.13.0

package/CHANGELOG.md

@@ -1,5 +1,22 @@
 # @codemation/core-nodes
 
+## 0.13.0
+
+### Minor Changes
+
+- [#240](https://github.com/MadeRelevant/codemation/pull/240) [`d786fc6`](https://github.com/MadeRelevant/codemation/commit/d786fc6ff8684a9c29a6b343b64f2d813d745a5e) Thanks [@cblokland90](https://github.com/cblokland90)! - feat(core-nodes): agent nodes pass tool-returned image/PDF binaries to the chat model
+
+  When an agent tool (including MCP tools like Gmail/Drive) returns a binary — an image or
+  a PDF — inside a `CallToolResult` content array, the agent now forwards it to the chat model
+  as a native multimodal tool-result block instead of flattening it to inert JSON text. Images
+  become `image-data` parts (image blocks on the wire); PDFs become `file-data` document parts.
+  `resource_link`s, embedded text resources, and unsupported binary types surface as text
+  markers rather than being dropped, and a per-result size cap replaces oversize binaries with a
+  placeholder. This is the tool-result analogue of input-time `item.binary` passdown.
+
+  A new `passToolBinariesToModel` option on the AI agent (default `true`) keeps the old inert
+  JSON behavior when set to `false`.
+
 ## 0.12.0
 
 ### Minor Changes

package/dist/index.cjs

@@ -4486,6 +4486,155 @@ var CodemationChatModelConfig = class {
 	}
 };
 
+//#endregion
+//#region src/nodes/AgentToolResultContentFactory.ts
+/**
+* Cap on raw (pre-base64) bytes inlined from a single tool result. Base64 inflates ~33% and every
+* inlined byte eats model context, so oversize binaries are replaced with a text placeholder.
+*/
+const MAX_INLINE_BYTES = 8 * 1024 * 1024;
+/** MCP content-block discriminators. At least one must appear for a result to be treated as MCP-shaped. */
+const KNOWN_MCP_BLOCK_TYPES = new Set([
+	"text",
+	"image",
+	"audio",
+	"resource",
+	"resource_link"
+]);
+/**
+* Maps a tool result that is **content-block-shaped** (an MCP `CallToolResult` with a `content`
+* array) into AI SDK `{ type: "content" }` tool-result output, so binaries reach the chat model as
+* native multimodal tool-result blocks instead of being flattened to inert JSON text.
+*
+* The `@ai-sdk/anthropic` provider maps a `content`-output part as follows:
+* `text` → text block, `image-data` → image block, `file-data` (only `application/pdf`) → document
+* block. Non-PDF `file-data` is dropped by the provider, so this factory emits `image-data` for
+* images, `file-data` only for PDFs, and a text marker for every other binary type.
+*
+* Returns `undefined` when the result is not content-block-shaped — callers keep the existing
+* `{ type: "json" }` path, so plain string/object tool results are unaffected.
+*/
+var AgentToolResultContentFactory = class AgentToolResultContentFactory {
+	static tryMapToContentOutput(result) {
+		const blocks = AgentToolResultContentFactory.contentBlocks(result);
+		if (blocks === void 0) return void 0;
+		const parts = [];
+		let inlinedBytes = 0;
+		for (const block of blocks) {
+			const mapped = AgentToolResultContentFactory.mapBlock(block, inlinedBytes);
+			parts.push(mapped.part);
+			inlinedBytes += mapped.bytes;
+		}
+		return parts;
+	}
+	/**
+	* Returns the `content` array iff `result` is an object whose `content` is an array of typed
+	* blocks AND at least one block carries a known MCP discriminator. A plain JSON result that merely
+	* has a `content` key of some other shape (e.g. Notion/Slack rich-text blocks) is rejected,
+	* preserving the `{ type: "json" }` path so its payload is never lost.
+	*/
+	static contentBlocks(result) {
+		if (result === null || typeof result !== "object") return void 0;
+		const content = result.content;
+		if (!Array.isArray(content) || content.length === 0) return void 0;
+		if (!content.every((block) => block !== null && typeof block === "object" && typeof block.type === "string")) return void 0;
+		return content.some((block) => KNOWN_MCP_BLOCK_TYPES.has(block.type)) ? content : void 0;
+	}
+	static mapBlock(block, inlinedBytesSoFar) {
+		const type = block.type;
+		if (type === "text" && typeof block.text === "string") return {
+			part: {
+				type: "text",
+				text: block.text
+			},
+			bytes: 0
+		};
+		if (type === "image" && typeof block.data === "string" && typeof block.mimeType === "string") return AgentToolResultContentFactory.mapBinary({
+			base64: block.data,
+			mediaType: block.mimeType,
+			inlinedBytesSoFar
+		});
+		if (type === "resource" && block.resource) return AgentToolResultContentFactory.mapEmbeddedResource(block.resource, inlinedBytesSoFar);
+		if (type === "resource_link" && typeof block.uri === "string") {
+			const mime = typeof block.mimeType === "string" ? ` (${block.mimeType})` : "";
+			return {
+				part: {
+					type: "text",
+					text: `[linked resource: ${block.uri}${mime}]`
+				},
+				bytes: 0
+			};
+		}
+		return {
+			part: {
+				type: "text",
+				text: `[unsupported tool content block: ${String(type)}]`
+			},
+			bytes: 0
+		};
+	}
+	static mapEmbeddedResource(resource, inlinedBytesSoFar) {
+		if (typeof resource.text === "string") return {
+			part: {
+				type: "text",
+				text: resource.text
+			},
+			bytes: 0
+		};
+		if (typeof resource.blob === "string" && typeof resource.mimeType === "string") return AgentToolResultContentFactory.mapBinary({
+			base64: resource.blob,
+			mediaType: resource.mimeType,
+			filename: typeof resource.name === "string" ? resource.name : void 0,
+			inlinedBytesSoFar
+		});
+		return {
+			part: {
+				type: "text",
+				text: `[embedded resource: ${typeof resource.uri === "string" ? resource.uri : "unknown"}]`
+			},
+			bytes: 0
+		};
+	}
+	static mapBinary(args) {
+		const rawBytes = Math.floor(args.base64.length * 3 / 4);
+		if (args.inlinedBytesSoFar + rawBytes > MAX_INLINE_BYTES) {
+			const name = args.filename ? ` "${args.filename}"` : "";
+			const kb = Math.round(rawBytes / 1024);
+			return {
+				part: {
+					type: "text",
+					text: `[binary${name} (${args.mediaType}, ~${kb} KB) omitted: exceeds the per-tool-result inline limit]`
+				},
+				bytes: 0
+			};
+		}
+		if (args.mediaType.startsWith("image/")) return {
+			part: {
+				type: "image-data",
+				data: args.base64,
+				mediaType: args.mediaType
+			},
+			bytes: rawBytes
+		};
+		if (args.mediaType === "application/pdf") return {
+			part: {
+				type: "file-data",
+				data: args.base64,
+				mediaType: args.mediaType,
+				...args.filename ? { filename: args.filename } : {}
+			},
+			bytes: rawBytes
+		};
+		return {
+			part: {
+				type: "text",
+				text: `[binary${args.filename ? ` "${args.filename}"` : ""} (${args.mediaType}) not inlined: unsupported by the model]`
+			},
+			bytes: 0
+		};
+	}
+};
+
 //#endregion
 //#region src/nodes/AgentMessageFactory.ts
 /**
@@ -4522,20 +4671,35 @@ var AgentMessageFactory = class AgentMessageFactory {
 	* Builds the `{ role: "tool", content: [{ type: "tool-result", ... }, ...] }` message returned
 	* to the model after each tool round.
 	*/
-	static createToolResultsMessage(executedToolCalls) {
+	static createToolResultsMessage(executedToolCalls, passToolBinariesToModel = true) {
 		return {
 			role: "tool",
 			content: executedToolCalls.map((executed) => ({
 				type: "tool-result",
 				toolCallId: executed.toolCallId,
 				toolName: executed.toolName,
-				output: {
-					type: "json",
-					value: AgentMessageFactory.toToolResultJson(executed.result)
-				}
+				output: AgentMessageFactory.toToolResultOutput(executed.result, passToolBinariesToModel)
 			}))
 		};
 	}
+	/**
+	* Routes a tool result to a native multimodal `{ type: "content" }` output when it is
+	* content-block-shaped (an MCP `CallToolResult`) and binary passdown is enabled; otherwise keeps
+	* the inert `{ type: "json" }` path.
+	*/
+	static toToolResultOutput(result, passToolBinariesToModel) {
+		if (passToolBinariesToModel) {
+			const content = AgentToolResultContentFactory.tryMapToContentOutput(result);
+			if (content !== void 0) return {
+				type: "content",
+				value: content
+			};
+		}
+		return {
+			type: "json",
+			value: AgentMessageFactory.toToolResultJson(result)
+		};
+	}
 	static toToolResultJson(value) {
 		if (value === void 0) return null;
 		try {
@@ -6324,7 +6488,7 @@ let AIAgentNode = class AIAgentNode$1 {
 			result: decision,
 			serialized: JSON.stringify(decision)
 		};
-		const conversation = [...checkpoint.conversation, AgentMessageFactory.createToolResultsMessage([toolResultEntry])];
+		const conversation = [...checkpoint.conversation, AgentMessageFactory.createToolResultsMessage([toolResultEntry], ctx.config.passToolBinariesToModel !== false)];
 		const loopResult = await this.runTurnLoopUntilFinalAnswer({
 			prepared,
 			itemInputsByPort,
@@ -6544,7 +6708,7 @@ let AIAgentNode = class AIAgentNode$1 {
 				coordinatorExecutedCalls.push(...executed);
 			}
 			const allExecutedCalls = [...strategyExecutedCalls, ...coordinatorExecutedCalls];
-			this.appendAssistantAndToolMessages(conversation, result.assistantMessage, result.text, result.toolCalls, allExecutedCalls);
+			this.appendAssistantAndToolMessages(conversation, result.assistantMessage, result.text, result.toolCalls, allExecutedCalls, ctx.config.passToolBinariesToModel !== false);
 		}
 		return {
 			finalText,
@@ -6559,8 +6723,8 @@ let AIAgentNode = class AIAgentNode$1 {
 		if (guardrails.onTurnLimitReached === "respondWithLastMessage") return;
 		throw new Error(`AIAgent "${ctx.config.name ?? ctx.nodeId}" reached maxTurns=${guardrails.maxTurns} before producing a final response.`);
 	}
-	appendAssistantAndToolMessages(conversation, assistantMessage, text, toolCalls, executedToolCalls) {
-		conversation.push(assistantMessage ?? AgentMessageFactory.createAssistantWithToolCalls(text, toolCalls), AgentMessageFactory.createToolResultsMessage(executedToolCalls));
+	appendAssistantAndToolMessages(conversation, assistantMessage, text, toolCalls, executedToolCalls, passToolBinariesToModel) {
+		conversation.push(assistantMessage ?? AgentMessageFactory.createAssistantWithToolCalls(text, toolCalls), AgentMessageFactory.createToolResultsMessage(executedToolCalls, passToolBinariesToModel));
 	}
 	async resolveFinalOutputJson(prepared, itemInputsByPort, conversation, finalText, wasToolEnabledRun) {
 		if (!prepared.ctx.config.outputSchema) return AgentOutputFactory.fromAgentContent(finalText);
@@ -7324,6 +7488,7 @@ var AIAgent = class {
 	pinnedMcpTools;
 	untrustedSources;
 	passBinariesToModel;
+	passToolBinariesToModel;
 	binaries;
 	constructor(options) {
 		this.name = options.name;
@@ -7340,6 +7505,7 @@ var AIAgent = class {
 		this.pinnedMcpTools = options.pinnedMcpTools;
 		this.untrustedSources = options.untrustedSources;
 		this.passBinariesToModel = options.passBinariesToModel;
+		this.passToolBinariesToModel = options.passToolBinariesToModel;
 		this.binaries = options.binaries;
 	}
 	inspectorSummary() {