@zerowidth/workbench-sdk 2.1.1 → 2.1.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zerowidth/workbench-sdk",
-  "version": "2.1.1",
+  "version": "2.1.2",
   "dependencies": {
     "adm-zip": "^0.5.16",
     "ajv": "^8.17.1",

package/src/index.js

@@ -20,6 +20,92 @@ import { sanitizeAPICallEvent } from "./utilities/sanitizeAPICall.js";
  * Workbench - Core class for executing node-based flows
  * Handles node loading, input/output validation, and flow execution
  */
+
+/**
+ * The single `role:"tool"` message for one tool result. MCP-style image
+ * content blocks ({type:'image', data, mimeType} with string data) are
+ * replaced in place by a short text note — the pixels are delivered to
+ * the model separately as an ephemeral vision input (see
+ * toolResultToMessages / callLLMWithTools). This same message is used on
+ * the wire, in the accumulated multi-round history, and therefore in the
+ * final conversation output, so all three carry an identical record with
+ * no base64 payload. Image blocks whose data is not a string are left
+ * untouched and stringify as before.
+ */
+function toolResultToDurableMessage(toolResult) {
+  const result = toolResult.result;
+  let durable = result;
+  if (result && Array.isArray(result.content)) {
+    const hasImage = result.content.some(
+      (c) => c && c.type === 'image' && typeof c.data === 'string'
+    );
+    if (hasImage) {
+      durable = {
+        ...result,
+        content: result.content.map((c) =>
+          c && c.type === 'image' && typeof c.data === 'string'
+            ? {
+                type: 'text',
+                text: `[image (${c.mimeType || 'image/png'}) — delivered to the model as a vision input]`,
+              }
+            : c
+        ),
+      };
+    }
+  }
+  return {
+    role: 'tool',
+    tool_call_id: toolResult.tool_call_id,
+    name: toolResult.name,
+    content: typeof durable === 'string' ? durable : JSON.stringify(durable),
+  };
+}
+
+/**
+ * Split one tool result into the wire messages for the next LLM round:
+ * the durable `role:"tool"` message plus, when the result carries image
+ * content blocks, an ephemeral `role:"user"` vision message with
+ * `image_url` data-URI parts. An OpenAI-style tool message is text-only —
+ * stringifying image blocks hands the model base64 soup and it
+ * confabulates what the image "probably" shows.
+ *
+ * The caller must insert vision messages ABOVE the entire trailing run of
+ * tool-cycle messages, not adjacent to this round's cycle: LLM nodes
+ * rebuild their `conversation` output by walking the wire messages
+ * backwards and stopping at the first message that is neither a tool
+ * result nor an assistant tool_calls turn. Placed above the whole run,
+ * a vision message is naturally excluded from the durable output while
+ * every tool cycle below it survives the walk.
+ */
+function toolResultToMessages(toolResult) {
+  const toolMessage = toolResultToDurableMessage(toolResult);
+  const result = toolResult.result;
+  let visionMessage = null;
+  if (result && Array.isArray(result.content)) {
+    const images = result.content.filter(
+      (c) => c && c.type === 'image' && typeof c.data === 'string'
+    );
+    if (images.length > 0) {
+      visionMessage = {
+        role: 'user',
+        content: [
+          {
+            type: 'text',
+            text: `[Image${images.length > 1 ? 's' : ''} returned by the ${toolResult.name} tool call (${toolResult.tool_call_id}) — this is the actual content:]`,
+          },
+          ...images.map((img) => ({
+            type: 'image_url',
+            image_url: {
+              url: `data:${img.mimeType || 'image/png'};base64,${img.data}`,
+            },
+          })),
+        ],
+      };
+    }
+  }
+  return { toolMessage, visionMessage };
+}
+
 export default class Workbench {
 
 
@@ -2171,14 +2257,7 @@ export default class Workbench {
       if (toolCallMessage && tool_results.length > 0) {
         accumulatedMessages = [...accumulatedMessages, toolCallMessage];
         for (const toolResult of tool_results) {
-          accumulatedMessages.push({
-            role: "tool",
-            tool_call_id: toolResult.tool_call_id,
-            name: toolResult.name,
-            content: typeof toolResult.result === "string"
-              ? toolResult.result
-              : JSON.stringify(toolResult.result)
-          });
+          accumulatedMessages.push(toolResultToDurableMessage(toolResult));
         }
       }
 
@@ -2851,27 +2930,34 @@ export default class Workbench {
 
     // If this is a tool call response, append it to the messages array (OpenAI style)
     if (toolCallMessage && toolResults && Array.isArray(llmInputs.messages)) {
-
-      // 
-      llmInputs.messages = [
-        ...llmInputs.messages,
-        toolCallMessage
-      ];
-
-      for(const toolResult of toolResults) {
+      const toolMessages = [];
+      const visionMessages = [];
+      for (const toolResult of toolResults) {
         // You may need to adapt this for other LLMs
-          llmInputs.messages = [
-            ...llmInputs.messages,
-            {
-              role: "tool",
-              tool_call_id: toolResult.tool_call_id,
-              name: toolResult.name,
-              content: typeof toolResult.result === "string"
-                ? toolResult.result
-                : JSON.stringify(toolResult.result)
-            }
-          ];
+        const { toolMessage, visionMessage } = toolResultToMessages(toolResult);
+        toolMessages.push(toolMessage);
+        if (visionMessage) visionMessages.push(visionMessage);
+      }
+      // Vision messages go ABOVE the entire trailing tool-cycle run, not
+      // just this round's cycle — a vision message wedged between two
+      // cycles would stop the conversation walk early and cut every
+      // earlier cycle out of the final output (see toolResultToMessages).
+      const base = llmInputs.messages;
+      let insertAt = base.length;
+      while (insertAt > 0) {
+        const m = base[insertAt - 1];
+        const inCycle = m && typeof m === 'object' &&
+          (m.role === 'tool' || (Array.isArray(m.tool_calls) && m.tool_calls.length > 0));
+        if (!inCycle) break;
+        insertAt--;
       }
+      llmInputs.messages = [
+        ...base.slice(0, insertAt),
+        ...visionMessages,
+        ...base.slice(insertAt),
+        toolCallMessage,
+        ...toolMessages
+      ];
     }
 
     // Execute using the shared core logic