@theokit/sdk 4.6.1 → 4.7.0

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## 4.7.0
+
+### Minor Changes
+
+- 6871152: M35 (multimodal) — implement image input end-to-end. `agent.send({ text, images })` (the `SDKUserMessage` form) previously carried the `images` TYPE but the runtime dropped them (only `.text` was used). Now `prepareRunContext` carries the images, the agent loop attaches them as `image` content parts (new `LlmImagePart`), and the provider adapters serialize them: OpenAI/OpenRouter to a content-array with an `image_url` data URL, Anthropic to a native base64 image block. Text-only turns are byte-unchanged (back-compat). Zero new dependencies.
+
 ## 4.6.1
 
 ### Patch Changes

package/dist/cron.cjs

@@ -4370,7 +4370,8 @@ function applyExtraRunFields(base, script) {
 // src/internal/runtime/fixtures/fixture-run-base.ts
 function prepareRunContext(message) {
   const userText = typeof message === "string" ? message : message.text;
-  return { userText, id: generateRunId(), startTime: Date.now() };
+  const userImages = typeof message === "string" ? void 0 : message.images;
+  return { userText, userImages, id: generateRunId(), startTime: Date.now() };
 }
 var FixtureRunBase = class {
   id;
@@ -9491,6 +9492,18 @@ function buildAssistantTurn(text, toolCalls) {
 }
 
 // src/internal/agent-loop/loop-context-init.ts
+function buildUserContent(text, images) {
+  const content = [{ type: "text", text }];
+  for (const img of images ?? []) {
+    if ("data" in img) {
+      content.push({
+        type: "image",
+        source: { type: "base64", media_type: img.mimeType, data: img.data }
+      });
+    }
+  }
+  return content;
+}
 function buildAgentRef(inputs) {
   return {
     agentId: inputs.agentId,
@@ -9578,7 +9591,7 @@ async function initLoopContext(inputs) {
     conversation: [],
     messages: [
       ...priorMessages,
-      { role: "user", content: [{ type: "text", text: inputs.userMessage }] }
+      { role: "user", content: buildUserContent(inputs.userMessage, inputs.userImages) }
     ],
     tools,
     finalText: "",
@@ -11599,6 +11612,9 @@ function toAnthropicWireMessage(message) {
     if (part.type === "tool_use") {
       return { type: "tool_use", id: part.id, name: part.name, input: part.input };
     }
+    if (part.type === "image") {
+      return { type: "image", source: part.source };
+    }
     return {
       type: "tool_result",
       tool_use_id: part.toolUseId,
@@ -12900,7 +12916,20 @@ function userOrToolMessages(message) {
     }
   }
   const userText = joinTextParts2(message);
-  if (userText.length > 0) out.push({ role: "user", content: userText });
+  const imageParts = message.content.filter((p) => p.type === "image");
+  if (imageParts.length > 0) {
+    const content = [];
+    if (userText.length > 0) content.push({ type: "text", text: userText });
+    for (const img of imageParts) {
+      content.push({
+        type: "image_url",
+        image_url: { url: `data:${img.source.media_type};base64,${img.source.data}` }
+      });
+    }
+    out.push({ role: "user", content });
+  } else if (userText.length > 0) {
+    out.push({ role: "user", content: userText });
+  }
   return out;
 }
 function assistantMessage(message) {
@@ -14086,7 +14115,7 @@ function buildCustomToolsInput(agentOptions, sendOptions, pluginManager, persona
 
 // src/internal/local-agent/real-local-run.ts
 function createRealLocalRun(options) {
-  const { userText, id, startTime } = prepareRunContext(options.message);
+  const { userText, userImages, id, startTime } = prepareRunContext(options.message);
   const supported = /* @__PURE__ */ new Set(["stream", "wait", "cancel", "conversation"]);
   const runScript = {
     events: [],
@@ -14105,7 +14134,7 @@ function createRealLocalRun(options) {
       supportedOps: supported,
       startTime
     },
-    () => buildLoopInputs(options, id, userText)
+    () => buildLoopInputs(options, id, userText, userImages)
   );
   handle.bootstrap();
   registerRun(handle);
@@ -14138,7 +14167,7 @@ function mergeExplicitApiKey(pools, primary, apiKey) {
   if (existing !== void 0 && existing.length > 0) return pools;
   return { ...pools ?? {}, [primary]: [apiKey] };
 }
-function buildLoopInputs(options, runId, userText) {
+function buildLoopInputs(options, runId, userText, userImages) {
   const maxIterations = options.sendOptions.maxIterations;
   if (maxIterations !== void 0 && (!Number.isInteger(maxIterations) || maxIterations < 1)) {
     throw new ConfigurationError(
@@ -14177,6 +14206,7 @@ function buildLoopInputs(options, runId, userText) {
       ...options.model?.params !== void 0 ? { params: options.model.params } : {}
     },
     userMessage: userText,
+    ...userImages !== void 0 ? { userImages } : {},
     llm,
     mcp: buildMcpMap(options),
     hooks: options.hooks,