@absolutejs/ai 0.4.1 → 0.6.0

package/CHANGELOG.md

@@ -6,6 +6,23 @@ This file is generated by `absolute-changelog` from the entries in
 `changelog/`. Edit an entry, not this file — and add new ones under
 `changelog/unreleased/`.
 
+## 0.6.0 — 2026-10-20
+
+### Added
+
+- **Load tools on demand (deferLoading, through Anthropic tool search) and clear old tool results past a threshold (toolResultClearing); both cross the provider proxy** (`AIToolResultClearing`, `AIToolDefinition`, `AIProviderToolDefinition`, `AIProviderStreamParams`, `StreamAIOptions`)
+
+## 0.5.0 — 2026-10-19
+
+### Breaking
+
+- **AIToolDefinition.handler may return { text, images } as well as a string** (`AIToolDefinition`, `toolOutputText`)
+  _Migration:_ Code that calls a tool's handler itself (a wrapper, a test) now receives AIToolOutput, a string or { text, images }. Read its text with toolOutputText(output). Handlers that return a string need no change.
+
+### Added
+
+- **Tools can return images for the model to see, such as a screenshot: a handler returns { text, images } and the tool loops send the images inside the tool result, mapped for Anthropic, OpenAI Responses, OpenAI Chat Completions, Gemini and Ollama** (`AIToolResult`, `AIToolOutput`, `toolOutputText`, `AIProviderContentBlock`, `anthropic`, `openai`, `openaiResponses`, `gemini`, `ollama`, `streamAIToSSE`, `streamAIWithTools`, `generateAIWithTools`, `hardenUntrustedTool`, `createAIStoredToolResultRecovery`)
+
 ## 0.4.1 — 2026-10-17
 
 ### Fixed

package/changelog.json

@@ -2,6 +2,61 @@
 	"contract": 1,
 	"name": "@absolutejs/ai",
 	"releases": [
+		{
+			"changes": [
+				{
+					"kind": "added",
+					"summary": "Load tools on demand (deferLoading, through Anthropic tool search) and clear old tool results past a threshold (toolResultClearing); both cross the provider proxy",
+					"symbols": [
+						"AIToolResultClearing",
+						"AIToolDefinition",
+						"AIProviderToolDefinition",
+						"AIProviderStreamParams",
+						"StreamAIOptions"
+					]
+				}
+			],
+			"date": "2026-10-20",
+			"version": "0.6.0"
+		},
+		{
+			"changes": [
+				{
+					"kind": "breaking",
+					"migration": {
+						"instruction": "Code that calls a tool's handler itself (a wrapper, a test) now receives AIToolOutput, a string or { text, images }. Read its text with toolOutputText(output). Handlers that return a string need no change.",
+						"manual": true
+					},
+					"summary": "AIToolDefinition.handler may return { text, images } as well as a string",
+					"symbols": [
+						"AIToolDefinition",
+						"toolOutputText"
+					]
+				},
+				{
+					"kind": "added",
+					"summary": "Tools can return images for the model to see, such as a screenshot: a handler returns { text, images } and the tool loops send the images inside the tool result, mapped for Anthropic, OpenAI Responses, OpenAI Chat Completions, Gemini and Ollama",
+					"symbols": [
+						"AIToolResult",
+						"AIToolOutput",
+						"toolOutputText",
+						"AIProviderContentBlock",
+						"anthropic",
+						"openai",
+						"openaiResponses",
+						"gemini",
+						"ollama",
+						"streamAIToSSE",
+						"streamAIWithTools",
+						"generateAIWithTools",
+						"hardenUntrustedTool",
+						"createAIStoredToolResultRecovery"
+					]
+				}
+			],
+			"date": "2026-10-19",
+			"version": "0.5.0"
+		},
 		{
 			"changes": [
 				{

package/dist/ai/index.js

@@ -258,6 +258,14 @@ var parseAIMessage = (raw) => {
 };
 var serializeAIMessage = (message) => JSON.stringify(message);
 
+// src/ai/toolOutput.ts
+var toolOutputText = (output) => typeof output === "string" ? output : output.text;
+var toolOutputImages = (output) => typeof output === "string" || !output.images?.length ? undefined : output.images;
+var toolResultBlock = (toolUseId, output, content = toolOutputText(output)) => {
+  const images = toolOutputImages(output);
+  return images ? { content, images, tool_use_id: toolUseId, type: "tool_result" } : { content, tool_use_id: toolUseId, type: "tool_result" };
+};
+
 // src/ai/inputCapacity.ts
 class AIInputError extends Error {
   code;
@@ -470,6 +478,7 @@ var sendMessage = async (socket, message) => {
   return true;
 };
 var buildToolDefinitions = (tools) => Object.entries(tools).map(([name, def]) => ({
+  ...def.deferLoading ? { deferLoading: true } : {},
   description: def.description,
   input_schema: def.input,
   name
@@ -643,14 +652,11 @@ var processToolTurn = async (socket, options, state, messageId, conversationId,
   const toolResultBlocks = [];
   for (const toolCall of toolCalls) {
     await sendToolRunning(socket, toolCall.name, toolCall.input, messageId, conversationId);
-    const result = await executeTool(options, toolCall.name, toolCall.input);
+    const output = await executeTool(options, toolCall.name, toolCall.input);
+    const result = toolOutputText(output);
     await sendToolComplete(socket, toolCall.name, result, messageId, conversationId);
     options.onToolUse?.(toolCall.name, toolCall.input, result);
-    toolResultBlocks.push({
-      content: result,
-      tool_use_id: toolCall.id,
-      type: "tool_result"
-    });
+    toolResultBlocks.push(toolResultBlock(toolCall.id, output));
     state.executedToolKeys.add(serializeToolCall(toolCall.name, toolCall.input));
   }
   state.currentMessages.push({
@@ -665,6 +671,7 @@ var processToolTurn = async (socket, options, state, messageId, conversationId,
     model: options.model,
     promptCaching: options.promptCaching,
     providerOptions: options.providerOptions,
+    toolResultClearing: options.toolResultClearing,
     reasoning: options.reasoning,
     signal,
     systemPrompt: options.systemPrompt,
@@ -755,6 +762,7 @@ var processStream = async (socket, options, messages, messageId, conversationId,
     model: options.model,
     promptCaching: options.promptCaching,
     providerOptions: options.providerOptions,
+    toolResultClearing: options.toolResultClearing,
     reasoning: options.reasoning,
     signal,
     systemPrompt: options.systemPrompt,
@@ -924,6 +932,7 @@ var withHeartbeat = async function* (source, signal, intervalMs) {
   }
 };
 var buildToolDefinitions2 = (tools) => Object.entries(tools).map(([name, def]) => ({
+  ...def.deferLoading ? { deferLoading: true } : {},
   description: def.description,
   input_schema: def.input,
   name
@@ -1112,7 +1121,8 @@ var executeToolCalls = async function* (pendingToolCalls, options, renderers, tu
       turnState.allToolsHtml += renderers.toolRunning(toolCall.name, toolCall.input);
       yield { data: turnState.allToolsHtml, event: "tools" };
     }
-    const result = await executeTool2(options, toolCall.name, toolCall.input);
+    const output = await executeTool2(options, toolCall.name, toolCall.input);
+    const result = toolOutputText(output);
     if (options.structuredEvents) {
       yield {
         data: JSON.stringify({
@@ -1129,11 +1139,7 @@ var executeToolCalls = async function* (pendingToolCalls, options, renderers, tu
     }
     options.onToolUse?.(toolCall.name, toolCall.input, result);
     const resultContent = options.maxToolResultChars !== undefined ? truncateToolResult(result, options.maxToolResultChars) : result;
-    toolResultBlocks.push({
-      content: resultContent,
-      tool_use_id: toolCall.id,
-      type: "tool_result"
-    });
+    toolResultBlocks.push(toolResultBlock(toolCall.id, output, resultContent));
   }
   return toolResultBlocks;
 };
@@ -1191,6 +1197,7 @@ var streamTurns = async function* (options, renderers, messages, signal, startTi
         reasoning: options.reasoning,
         signal,
         systemPrompt: options.systemPrompt,
+        toolResultClearing: options.toolResultClearing,
         tools: toolDefs
       }, options.contextPolicy, (prepared) => {
         turnState.currentMessages = prepared.messages;
@@ -1803,6 +1810,7 @@ var mergeUsage = (left, right) => {
   };
 };
 var toProviderTools = (tools) => Object.entries(tools).map(([name, definition]) => ({
+  ...definition.deferLoading ? { deferLoading: true } : {},
   description: definition.description,
   input_schema: definition.input,
   name
@@ -1833,12 +1841,8 @@ var generateAIWithTools = async (options) => {
     const resultBlocks = await Promise.all(result.toolCalls.map(async (call) => {
       const definition = tools[call.name];
       const output = definition ? await Promise.resolve(definition.handler(call.input)).catch((err) => `Error: ${err instanceof Error ? err.message : String(err)}`) : `Error: unknown tool "${call.name}"`;
-      onToolUse?.(call.name, call.input, output);
-      return {
-        content: output,
-        tool_use_id: call.id,
-        type: "tool_result"
-      };
+      onToolUse?.(call.name, call.input, toolOutputText(output));
+      return toolResultBlock(call.id, output);
     }));
     return [
       ...result.requestMessages ?? messages,
@@ -1988,14 +1992,15 @@ var flushThinking3 = (blocks, thinking) => {
 var executeToolHandler = async (tools, call) => {
   const definition = tools[call.name];
   if (!definition) {
-    return { ok: false, result: `Error: unknown tool "${call.name}"` };
+    return { ok: false, output: `Error: unknown tool "${call.name}"` };
   }
   try {
-    return { ok: true, result: await definition.handler(call.input) };
+    const output = await definition.handler(call.input);
+    return { ok: true, output };
   } catch (err) {
     return {
       ok: false,
-      result: `Error: ${err instanceof Error ? err.message : String(err)}`
+      output: `Error: ${err instanceof Error ? err.message : String(err)}`
     };
   }
 };
@@ -2142,7 +2147,8 @@ var streamAIWithTools = async function* (options) {
         type: "tool_start"
       };
       const startedAt = Date.now();
-      const { ok, result } = await executeToolHandler(tools, call);
+      const { ok, output } = await executeToolHandler(tools, call);
+      const result = toolOutputText(output);
       executedKeys.add(toolCallKey(call));
       yield {
         id: call.id,
@@ -2153,11 +2159,7 @@ var streamAIWithTools = async function* (options) {
         result,
         type: "tool_result"
       };
-      resultBlocks.push({
-        content: result,
-        tool_use_id: call.id,
-        type: "tool_result"
-      });
+      resultBlocks.push(toolResultBlock(call.id, output));
       if (ok && stopAfterTools?.includes(call.name)) {
         terminalToolSucceeded = true;
         break;
@@ -2300,6 +2302,7 @@ var wireParams = (params) => ({
   ...params.systemPrompt === undefined ? {} : { systemPrompt: params.systemPrompt },
   ...params.temperature === undefined ? {} : { temperature: params.temperature },
   ...params.toolChoice === undefined ? {} : { toolChoice: params.toolChoice },
+  ...params.toolResultClearing === undefined ? {} : { toolResultClearing: params.toolResultClearing },
   ...params.tools === undefined ? {} : { tools: params.tools },
   ...params.topP === undefined ? {} : { topP: params.topP }
 });
@@ -5179,7 +5182,10 @@ var convertToolBlock = (block) => {
   if (block.type === "tool_result") {
     return {
       call_id: block.tool_use_id,
-      output: typeof block.content === "string" ? block.content : "",
+      output: block.images?.length ? [
+        { text: block.content, type: "input_text" },
+        ...block.images.map((source) => mapBlockToResponsesFormat({ source, type: "image" }))
+      ] : block.content,
       type: "function_call_output"
     };
   }
@@ -5709,6 +5715,7 @@ var NOT_FOUND4 = -1;
 var isRecord3 = (value) => typeof value === "object" && value !== null;
 var isRecordArray2 = (value) => Array.isArray(value) && value.length > 0 && isRecord3(value[0]);
 var hasArrayContent = (msg) => typeof msg.content !== "string" && Array.isArray(msg.content);
+var imageUrl = (source) => source.type === "url" ? source.url : `data:${source.media_type};base64,${source.data}`;
 var buildToolMessages = (blocks) => {
   const toolUseBlocks = blocks.filter((block) => block.type === "tool_use");
   const toolResultBlocks = blocks.filter((block) => block.type === "tool_result");
@@ -5734,6 +5741,18 @@ var buildToolMessages = (blocks) => {
       tool_call_id: result.tool_use_id
     });
   }
+  const images = toolResultBlocks.flatMap((result) => result.images ?? []);
+  if (images.length > 0)
+    messages.push({
+      content: [
+        { text: "Images returned by the tool calls above:", type: "text" },
+        ...images.map((source) => ({
+          image_url: { url: imageUrl(source) },
+          type: "image_url"
+        }))
+      ],
+      role: "user"
+    });
   return messages;
 };
 var mapToolDefinitions = (tools) => tools.map((tool) => ({
@@ -5747,9 +5766,7 @@ var mapToolDefinitions = (tools) => tools.map((tool) => ({
 var mapContentBlockToOpenAI = (block) => {
   if (block.type === "image") {
     return {
-      image_url: {
-        url: block.source.type === "url" ? block.source.url : `data:${block.source.media_type};base64,${block.source.data}`
-      },
+      image_url: { url: imageUrl(block.source) },
       type: "image_url"
     };
   }
@@ -6372,7 +6389,10 @@ var convertMessageContent = (content) => {
   if (typeof content === "string") {
     return [{ text: content }];
   }
-  return content.map(mapContentBlock).filter((mapped) => mapped !== null);
+  return content.flatMap((block) => block.type === "tool_result" && block.images?.length ? [
+    mapContentBlock(block),
+    ...block.images.map((source) => mapContentBlock({ source, type: "image" }))
+  ] : [mapContentBlock(block)]).filter((mapped) => mapped !== null);
 };
 var hasFunctionResponse = (content) => content.some((block) => block.type === "tool_result");
 var convertSingleMessage = (msg) => {
@@ -6672,7 +6692,10 @@ var mapContentBlock2 = (block) => {
   }
   if (block.type === "tool_result") {
     return {
-      content: block.content,
+      content: block.images?.length ? [
+        { text: block.content, type: "text" },
+        ...block.images.map((source) => ({ source, type: "image" }))
+      ] : block.content,
       tool_use_id: block.tool_use_id,
       type: "tool_result"
     };
@@ -6700,10 +6723,32 @@ var mapMessage = (msg) => ({
   role: msg.role === "system" ? "user" : msg.role
 });
 var mapToolDefinition2 = (tool) => ({
+  ...tool.deferLoading ? { defer_loading: true } : {},
   description: tool.description,
   input_schema: tool.input_schema,
   name: tool.name
 });
+var TOOL_SEARCH_TOOL = {
+  name: "tool_search_tool_bm25",
+  type: "tool_search_tool_bm25_20251119"
+};
+var CONTEXT_MANAGEMENT_BETA = "context-management-2025-06-27";
+var contextManagementFor = (clearing) => ({
+  edits: [
+    {
+      ...clearing.clearAtLeastInputTokens === undefined ? {} : {
+        clear_at_least: {
+          type: "input_tokens",
+          value: clearing.clearAtLeastInputTokens
+        }
+      },
+      ...clearing.excludeTools?.length ? { exclude_tools: [...clearing.excludeTools] } : {},
+      ...clearing.keepToolUses === undefined ? {} : { keep: { type: "tool_uses", value: clearing.keepToolUses } },
+      trigger: { type: "input_tokens", value: clearing.triggerInputTokens },
+      type: "clear_tool_uses_20250919"
+    }
+  ]
+});
 var requestOptionsFor = (params) => {
   const raw = params.providerOptions?.anthropic;
   if (raw === undefined)
@@ -6812,16 +6857,20 @@ var buildRequestBody3 = (params, configuredMax, configCaching) => {
     ] : params.systemPrompt;
   }
   const clientTools = params.tools?.map(mapToolDefinition2) ?? [];
-  if (clientTools.length > 0) {
-    if (caching) {
-      clientTools[clientTools.length - 1] = {
-        ...clientTools[clientTools.length - 1],
-        cache_control: { type: "ephemeral" }
-      };
-    }
+  const deferring = clientTools.some((tool) => tool.defer_loading === true);
+  const lastLoaded = clientTools.findLastIndex((tool) => tool.defer_loading !== true);
+  if (caching && lastLoaded >= 0) {
+    clientTools[lastLoaded] = {
+      ...clientTools[lastLoaded],
+      cache_control: { type: "ephemeral" }
+    };
   }
   const serverTools = (options.serverTools ?? []).map(mapServerTool);
-  const tools = [...clientTools, ...serverTools];
+  const tools = [
+    ...clientTools,
+    ...deferring ? [TOOL_SEARCH_TOOL] : [],
+    ...serverTools
+  ];
   if (tools.length > 0) {
     body.tools = tools;
     if (params.toolChoice === "auto" || params.toolChoice === "none") {
@@ -6835,6 +6884,9 @@ var buildRequestBody3 = (params, configuredMax, configCaching) => {
   if (params.stopSequences && params.stopSequences.length > 0) {
     body.stop_sequences = params.stopSequences;
   }
+  if (params.toolResultClearing) {
+    body.context_management = contextManagementFor(params.toolResultClearing);
+  }
   const mode = params.reasoning && params.reasoning.effort !== "none" ? anthropicReasoningMode(params.model) : "none";
   if (params.reasoning?.effort === "none")
     body.thinking = { type: "disabled" };
@@ -7281,6 +7333,11 @@ var fetchAndStream = async function* (baseUrl, config, params, configuredMax, pr
   const suppliedHeaders = typeof config.headers === "function" ? await config.headers(params) : config.headers ?? {};
   const requestHeaders = new Headers(suppliedHeaders);
   requestHeaders.set("Content-Type", "application/json");
+  if (body.context_management !== undefined) {
+    const betas = (requestHeaders.get("anthropic-beta") ?? "").split(",").map((beta) => beta.trim()).filter(Boolean);
+    if (!betas.includes(CONTEXT_MANAGEMENT_BETA))
+      requestHeaders.set("anthropic-beta", [...betas, CONTEXT_MANAGEMENT_BETA].join(","));
+  }
   if (config.authStyle === "bearer") {
     requestHeaders.set("Authorization", `Bearer ${token}`);
   } else {
@@ -7349,7 +7406,12 @@ var anthropic = (config) => {
     const hosted = tools.filter((tool) => isRecord5(tool) && typeof tool.type === "string" && /^(web_search_|web_fetch_|code_execution_|bash_code_execution_|text_editor_code_execution_|tool_search_tool_)/u.test(tool.type));
     const scope = hosted.length > 0 || body.mcp_servers !== undefined ? "caller-input" : "request";
     if (hosted.length > 0) {
-      const clientTools = tools.filter((tool) => !hosted.includes(tool));
+      const clientTools = tools.filter((tool) => !hosted.includes(tool)).map((tool) => {
+        if (!isRecord5(tool) || tool.defer_loading === undefined)
+          return tool;
+        const { defer_loading: _deferred, ...loaded } = tool;
+        return loaded;
+      });
       if (clientTools.length)
         counted.tools = clientTools;
       else
@@ -7418,8 +7480,10 @@ var convertMessage2 = (msg) => {
     });
   }
   for (const block of toolResultBlocks) {
+    const images = (block.images ?? []).flatMap((source) => source.type === "base64" ? [source.data] : []);
     results.push({
       content: typeof block.content === "string" ? block.content : "",
+      ...images.length > 0 ? { images } : {},
       role: "tool"
     });
   }
@@ -9026,6 +9090,12 @@ var createAIStoredToolResultRecovery = (toolNames) => async ({ params }) => {
       if (block.type !== "tool_result" || !eligible.has(block.tool_use_id) || block === newest)
         return block;
       const marker = "[Earlier lookup removed from working context. Its immutable original remains saved. The original tool call and source/version arguments above are unchanged: repeat that lookup if needed to verify facts. Do not infer facts from this marker.]";
+      if (block.images?.length)
+        return {
+          content: marker,
+          tool_use_id: block.tool_use_id,
+          type: block.type
+        };
       return block.content.length > marker.length ? { ...block, content: marker } : block;
     })
   }));
@@ -9167,10 +9237,11 @@ export {
   streamAIWithTools,
   streamWithAIContext,
   tableCard,
+  toolOutputText,
   verifyOpenRouterWebhookSignature,
   withResilience,
   xai
 };
 
-//# debugId=A58174807DA8A73664756E2164756E21
+//# debugId=9A889A121344253764756E2164756E21
 //# sourceMappingURL=index.js.map