@absolutejs/ai 0.4.0 → 0.5.0

package/CHANGELOG.md

@@ -6,6 +6,23 @@ This file is generated by `absolute-changelog` from the entries in
 `changelog/`. Edit an entry, not this file — and add new ones under
 `changelog/unreleased/`.
 
+## 0.5.0 — 2026-10-19
+
+### Breaking
+
+- **AIToolDefinition.handler may return { text, images } as well as a string** (`AIToolDefinition`, `toolOutputText`)
+  _Migration:_ Code that calls a tool's handler itself (a wrapper, a test) now receives AIToolOutput, a string or { text, images }. Read its text with toolOutputText(output). Handlers that return a string need no change.
+
+### Added
+
+- **Tools can return images for the model to see, such as a screenshot: a handler returns { text, images } and the tool loops send the images inside the tool result, mapped for Anthropic, OpenAI Responses, OpenAI Chat Completions, Gemini and Ollama** (`AIToolResult`, `AIToolOutput`, `toolOutputText`, `AIProviderContentBlock`, `anthropic`, `openai`, `openaiResponses`, `gemini`, `ollama`, `streamAIToSSE`, `streamAIWithTools`, `generateAIWithTools`, `hardenUntrustedTool`, `createAIStoredToolResultRecovery`)
+
+## 0.4.1 — 2026-10-17
+
+### Fixed
+
+- **Preserve provider usage snapshots through stream interruptions, distinguish partial from final usage, and retain tool-input start events without application patches** (`AIUsageUpdateChunk`, `anthropic`, `openai`, `openaiResponses`, `AIChunk`, `StreamAIWithToolsEvent`, `AIToolInputStartChunk`)
+
 ## 0.4.0 — 2026-10-16
 
 ### Added

package/README.md

@@ -560,3 +560,18 @@ are documented provider limits, not a substitute for runtime token budgeting.
 Regional features are omitted when they are not consistently supported.
 
 `@absolutejs/ai/research` separates an injected `SearchProvider` from evidence-only synthesis. It returns sources, findings, searches, limitations and explicit availability, and excludes invented source references/quotes. Quotation checks establish provenance; they are not a substitute for claim entailment or entity review. Hosts can set `maxRetries: 0` on one-shot or stream requests when they own attempt accounting, and independently bound `maxRepairAttempts` and disable context recovery.
+
+### Interrupted streams and usage
+
+Provider streams can emit `usage_update` chunks before `done`. Each carries a
+provider-reported cumulative snapshot: replace the previous snapshot for that
+attempt; never sum snapshots. `final: true` means the provider reported complete
+usage. An absent or false `final` means partial evidence, not a final bill.
+Anthropic exposes message-start and message-delta usage; OpenAI Chat exposes its
+usage chunk, and Responses preserves usage even on incomplete responses.
+
+Persist snapshots while consuming the stream. Cancellation or transport failure
+can prevent `done`, and no adapter can reconstruct usage the provider never sent.
+`onUsage` still reports only complete usage, once per instrumented stream. Keep
+credit reservations separate from charges and reconcile unknown outcomes in the
+host application. The package does not invent token counts or decide billing policy.

package/changelog.json

@@ -2,6 +2,63 @@
 	"contract": 1,
 	"name": "@absolutejs/ai",
 	"releases": [
+		{
+			"changes": [
+				{
+					"kind": "breaking",
+					"migration": {
+						"instruction": "Code that calls a tool's handler itself (a wrapper, a test) now receives AIToolOutput, a string or { text, images }. Read its text with toolOutputText(output). Handlers that return a string need no change.",
+						"manual": true
+					},
+					"summary": "AIToolDefinition.handler may return { text, images } as well as a string",
+					"symbols": [
+						"AIToolDefinition",
+						"toolOutputText"
+					]
+				},
+				{
+					"kind": "added",
+					"summary": "Tools can return images for the model to see, such as a screenshot: a handler returns { text, images } and the tool loops send the images inside the tool result, mapped for Anthropic, OpenAI Responses, OpenAI Chat Completions, Gemini and Ollama",
+					"symbols": [
+						"AIToolResult",
+						"AIToolOutput",
+						"toolOutputText",
+						"AIProviderContentBlock",
+						"anthropic",
+						"openai",
+						"openaiResponses",
+						"gemini",
+						"ollama",
+						"streamAIToSSE",
+						"streamAIWithTools",
+						"generateAIWithTools",
+						"hardenUntrustedTool",
+						"createAIStoredToolResultRecovery"
+					]
+				}
+			],
+			"date": "2026-10-19",
+			"version": "0.5.0"
+		},
+		{
+			"changes": [
+				{
+					"kind": "fixed",
+					"summary": "Preserve provider usage snapshots through stream interruptions, distinguish partial from final usage, and retain tool-input start events without application patches",
+					"symbols": [
+						"AIUsageUpdateChunk",
+						"anthropic",
+						"openai",
+						"openaiResponses",
+						"AIChunk",
+						"StreamAIWithToolsEvent",
+						"AIToolInputStartChunk"
+					]
+				}
+			],
+			"date": "2026-10-17",
+			"version": "0.4.1"
+		},
 		{
 			"changes": [
 				{

package/dist/ai/index.js

@@ -258,6 +258,14 @@ var parseAIMessage = (raw) => {
 };
 var serializeAIMessage = (message) => JSON.stringify(message);
 
+// src/ai/toolOutput.ts
+var toolOutputText = (output) => typeof output === "string" ? output : output.text;
+var toolOutputImages = (output) => typeof output === "string" || !output.images?.length ? undefined : output.images;
+var toolResultBlock = (toolUseId, output, content = toolOutputText(output)) => {
+  const images = toolOutputImages(output);
+  return images ? { content, images, tool_use_id: toolUseId, type: "tool_result" } : { content, tool_use_id: toolUseId, type: "tool_result" };
+};
+
 // src/ai/inputCapacity.ts
 class AIInputError extends Error {
   code;
@@ -643,14 +651,11 @@ var processToolTurn = async (socket, options, state, messageId, conversationId,
   const toolResultBlocks = [];
   for (const toolCall of toolCalls) {
     await sendToolRunning(socket, toolCall.name, toolCall.input, messageId, conversationId);
-    const result = await executeTool(options, toolCall.name, toolCall.input);
+    const output = await executeTool(options, toolCall.name, toolCall.input);
+    const result = toolOutputText(output);
     await sendToolComplete(socket, toolCall.name, result, messageId, conversationId);
     options.onToolUse?.(toolCall.name, toolCall.input, result);
-    toolResultBlocks.push({
-      content: result,
-      tool_use_id: toolCall.id,
-      type: "tool_result"
-    });
+    toolResultBlocks.push(toolResultBlock(toolCall.id, output));
     state.executedToolKeys.add(serializeToolCall(toolCall.name, toolCall.input));
   }
   state.currentMessages.push({
@@ -1112,7 +1117,8 @@ var executeToolCalls = async function* (pendingToolCalls, options, renderers, tu
       turnState.allToolsHtml += renderers.toolRunning(toolCall.name, toolCall.input);
       yield { data: turnState.allToolsHtml, event: "tools" };
     }
-    const result = await executeTool2(options, toolCall.name, toolCall.input);
+    const output = await executeTool2(options, toolCall.name, toolCall.input);
+    const result = toolOutputText(output);
     if (options.structuredEvents) {
       yield {
         data: JSON.stringify({
@@ -1129,11 +1135,7 @@ var executeToolCalls = async function* (pendingToolCalls, options, renderers, tu
     }
     options.onToolUse?.(toolCall.name, toolCall.input, result);
     const resultContent = options.maxToolResultChars !== undefined ? truncateToolResult(result, options.maxToolResultChars) : result;
-    toolResultBlocks.push({
-      content: resultContent,
-      tool_use_id: toolCall.id,
-      type: "tool_result"
-    });
+    toolResultBlocks.push(toolResultBlock(toolCall.id, output, resultContent));
   }
   return toolResultBlocks;
 };
@@ -1833,12 +1835,8 @@ var generateAIWithTools = async (options) => {
     const resultBlocks = await Promise.all(result.toolCalls.map(async (call) => {
       const definition = tools[call.name];
       const output = definition ? await Promise.resolve(definition.handler(call.input)).catch((err) => `Error: ${err instanceof Error ? err.message : String(err)}`) : `Error: unknown tool "${call.name}"`;
-      onToolUse?.(call.name, call.input, output);
-      return {
-        content: output,
-        tool_use_id: call.id,
-        type: "tool_result"
-      };
+      onToolUse?.(call.name, call.input, toolOutputText(output));
+      return toolResultBlock(call.id, output);
     }));
     return [
       ...result.requestMessages ?? messages,
@@ -1988,14 +1986,15 @@ var flushThinking3 = (blocks, thinking) => {
 var executeToolHandler = async (tools, call) => {
   const definition = tools[call.name];
   if (!definition) {
-    return { ok: false, result: `Error: unknown tool "${call.name}"` };
+    return { ok: false, output: `Error: unknown tool "${call.name}"` };
   }
   try {
-    return { ok: true, result: await definition.handler(call.input) };
+    const output = await definition.handler(call.input);
+    return { ok: true, output };
   } catch (err) {
     return {
       ok: false,
-      result: `Error: ${err instanceof Error ? err.message : String(err)}`
+      output: `Error: ${err instanceof Error ? err.message : String(err)}`
     };
   }
 };
@@ -2074,6 +2073,8 @@ var streamAIWithTools = async function* (options) {
           transcript: chunk.transcript,
           type: "audio"
         };
+      } else if (chunk.type === "tool_input_start") {
+        yield chunk;
       } else if (chunk.type === "tool_use") {
         thinking = flushThinking3(blocks, thinking);
         pending.push({ id: chunk.id, input: chunk.input, name: chunk.name });
@@ -2140,7 +2141,8 @@ var streamAIWithTools = async function* (options) {
         type: "tool_start"
       };
       const startedAt = Date.now();
-      const { ok, result } = await executeToolHandler(tools, call);
+      const { ok, output } = await executeToolHandler(tools, call);
+      const result = toolOutputText(output);
       executedKeys.add(toolCallKey(call));
       yield {
         id: call.id,
@@ -2151,11 +2153,7 @@ var streamAIWithTools = async function* (options) {
         result,
         type: "tool_result"
       };
-      resultBlocks.push({
-        content: result,
-        tool_use_id: call.id,
-        type: "tool_result"
-      });
+      resultBlocks.push(toolResultBlock(call.id, output));
       if (ok && stopAfterTools?.includes(call.name)) {
         terminalToolSucceeded = true;
         break;
@@ -4874,7 +4872,7 @@ async function* tapStream(source, params, providerName) {
   let lastUsage;
   try {
     for await (const chunk of source) {
-      if (chunk.type === "done" && chunk.usage) {
+      if ((chunk.type === "done" || chunk.type === "usage_update" && chunk.final) && chunk.usage) {
         lastUsage = chunk.usage;
       }
       yield chunk;
@@ -5177,7 +5175,10 @@ var convertToolBlock = (block) => {
   if (block.type === "tool_result") {
     return {
       call_id: block.tool_use_id,
-      output: typeof block.content === "string" ? block.content : "",
+      output: block.images?.length ? [
+        { text: block.content, type: "input_text" },
+        ...block.images.map((source) => mapBlockToResponsesFormat({ source, type: "image" }))
+      ] : block.content,
       type: "function_call_output"
     };
   }
@@ -5487,6 +5488,8 @@ var processCompleted = function* (parsed) {
   const { response } = parsed;
   const usage = extractUsage(response);
   const metadata = extractResponseMetadata(response);
+  if (usage)
+    yield { type: "usage_update", usage, metadata, final: true };
   if (isRecordArray(response.output)) {
     yield* extractCitationsFromOutput(response.output);
     yield* extractImageFromOutput(response.output);
@@ -5525,6 +5528,15 @@ var processSSEEvent = function* (eventType, parsed, pendingCalls, providerName)
       break;
     case "response.output_item.added":
       processOutputItemAdded(parsed, pendingCalls);
+      if (isRecord2(parsed.item) && parsed.item.type === "function_call" && typeof parsed.item.name === "string") {
+        const id = typeof parsed.item.call_id === "string" ? parsed.item.call_id : parsed.item.id;
+        if (typeof id === "string")
+          yield {
+            type: "tool_input_start",
+            id,
+            name: parsed.item.name
+          };
+      }
       break;
     case "response.output_item.done":
       yield* processOutputItemDone(parsed);
@@ -5541,8 +5553,19 @@ var processSSEEvent = function* (eventType, parsed, pendingCalls, providerName)
     case "response.failed":
     case "response.incomplete":
     case "response.error":
-    case "error":
+    case "error": {
+      if (isRecord2(parsed.response)) {
+        const usage = extractUsage(parsed.response);
+        if (usage)
+          yield {
+            type: "usage_update",
+            usage,
+            metadata: extractResponseMetadata(parsed.response),
+            final: false
+          };
+      }
       throw responseFailure(eventType, parsed, providerName);
+    }
   }
 };
 var flushSSEBuffer = function* (state) {
@@ -5685,6 +5708,7 @@ var NOT_FOUND4 = -1;
 var isRecord3 = (value) => typeof value === "object" && value !== null;
 var isRecordArray2 = (value) => Array.isArray(value) && value.length > 0 && isRecord3(value[0]);
 var hasArrayContent = (msg) => typeof msg.content !== "string" && Array.isArray(msg.content);
+var imageUrl = (source) => source.type === "url" ? source.url : `data:${source.media_type};base64,${source.data}`;
 var buildToolMessages = (blocks) => {
   const toolUseBlocks = blocks.filter((block) => block.type === "tool_use");
   const toolResultBlocks = blocks.filter((block) => block.type === "tool_result");
@@ -5710,6 +5734,18 @@ var buildToolMessages = (blocks) => {
       tool_call_id: result.tool_use_id
     });
   }
+  const images = toolResultBlocks.flatMap((result) => result.images ?? []);
+  if (images.length > 0)
+    messages.push({
+      content: [
+        { text: "Images returned by the tool calls above:", type: "text" },
+        ...images.map((source) => ({
+          image_url: { url: imageUrl(source) },
+          type: "image_url"
+        }))
+      ],
+      role: "user"
+    });
   return messages;
 };
 var mapToolDefinitions = (tools) => tools.map((tool) => ({
@@ -5723,9 +5759,7 @@ var mapToolDefinitions = (tools) => tools.map((tool) => ({
 var mapContentBlockToOpenAI = (block) => {
   if (block.type === "image") {
     return {
-      image_url: {
-        url: block.source.type === "url" ? block.source.url : `data:${block.source.media_type};base64,${block.source.data}`
-      },
+      image_url: { url: imageUrl(block.source) },
       type: "image_url"
     };
   }
@@ -5964,6 +5998,14 @@ var processDelta = function* (delta, pendingToolCalls) {
   }
   if (isRecordArray2(delta.tool_calls)) {
     processToolCallDeltas(delta.tool_calls, pendingToolCalls);
+    for (const call of delta.tool_calls) {
+      if (isRecord3(call.function) && typeof call.function.name === "string" && call.function.name)
+        yield {
+          type: "tool_input_start",
+          id: typeof call.id === "string" ? call.id : String(call.index),
+          name: call.function.name
+        };
+    }
   }
   if (Array.isArray(delta.annotations)) {
     for (const annotation of delta.annotations) {
@@ -6062,7 +6104,12 @@ var processSSELine2 = function* (line, pendingToolCalls, providerName) {
   }
   const usageUpdate = narrowUsageRecord(parsed);
   if (usageUpdate) {
-    yield { type: "usage_update", usage: usageUpdate };
+    yield {
+      type: "usage_update",
+      usage: usageUpdate,
+      final: Array.isArray(parsed.choices) && parsed.choices.length === 0,
+      metadata: narrowResponseMetadata(parsed)
+    };
   }
   const metadata = narrowResponseMetadata(parsed);
   if (metadata) {
@@ -6091,10 +6138,9 @@ var processSSELine2 = function* (line, pendingToolCalls, providerName) {
   }
   yield* processChoice(firstChoice, pendingToolCalls);
 };
-var isUsageUpdate = (chunk) => chunk.type === "usage_update";
 var collectYieldableChunks = (line, pendingToolCalls, usageRef, metadataRef, providerName) => {
   const allChunks = Array.from(processSSELine2(line, pendingToolCalls, providerName));
-  const usageChunks = allChunks.filter(isUsageUpdate);
+  const usageChunks = allChunks.filter((chunk) => chunk.type === "usage_update");
   const lastUsage = usageChunks.at(NOT_FOUND4);
   if (lastUsage) {
     usageRef.current = lastUsage.usage;
@@ -6111,7 +6157,7 @@ var collectYieldableChunks = (line, pendingToolCalls, usageRef, metadataRef, pro
       }
     };
   }
-  return allChunks.filter((chunk) => !isUsageUpdate(chunk) && chunk.type !== "response_metadata");
+  return allChunks.filter((chunk) => chunk.type !== "response_metadata").map((chunk) => chunk.type === "usage_update" ? { ...chunk, metadata: metadataRef.current } : chunk);
 };
 var processSSELines2 = function* (lines, pendingToolCalls, usageRef, metadataRef, providerName) {
   for (const line of lines) {
@@ -6336,7 +6382,10 @@ var convertMessageContent = (content) => {
   if (typeof content === "string") {
     return [{ text: content }];
   }
-  return content.map(mapContentBlock).filter((mapped) => mapped !== null);
+  return content.flatMap((block) => block.type === "tool_result" && block.images?.length ? [
+    mapContentBlock(block),
+    ...block.images.map((source) => mapContentBlock({ source, type: "image" }))
+  ] : [mapContentBlock(block)]).filter((mapped) => mapped !== null);
 };
 var hasFunctionResponse = (content) => content.some((block) => block.type === "tool_result");
 var convertSingleMessage = (msg) => {
@@ -6613,7 +6662,6 @@ var API_VERSION = "2023-06-01";
 var DEFAULT_MAX_TOKENS = 32000;
 var EVENT_PREFIX_LENGTH2 = 7;
 var DATA_PREFIX_LENGTH2 = 6;
-var EMPTY_CHUNKS = [];
 var isRecord5 = (val) => typeof val === "object" && val !== null;
 var mapContentBlock2 = (block) => {
   if (block.type === "thinking") {
@@ -6637,7 +6685,10 @@ var mapContentBlock2 = (block) => {
   }
   if (block.type === "tool_result") {
     return {
-      content: block.content,
+      content: block.images?.length ? [
+        { text: block.content, type: "text" },
+        ...block.images.map((source) => ({ source, type: "image" }))
+      ] : block.content,
       tool_use_id: block.tool_use_id,
       type: "tool_result"
     };
@@ -7141,11 +7192,21 @@ var processEvent = (eventType, parsed, state) => {
     case "message_delta": {
       handleMessageDelta(parsed, state);
       mergeMetadata(parsed, state);
-      return;
+      return state.usage ? {
+        type: "usage_update",
+        usage: state.usage,
+        metadata: state.metadata,
+        final: false
+      } : undefined;
     }
     case "message_start": {
       handleMessageStart(parsed, state);
-      return;
+      return state.usage ? {
+        type: "usage_update",
+        usage: state.usage,
+        metadata: state.metadata,
+        final: false
+      } : undefined;
     }
     case "message_stop": {
       mergeMetadata(parsed, state);
@@ -7179,49 +7240,6 @@ var processSingleEvent = (event, state) => {
   }
   return processEvent(eventType, parsed, state);
 };
-var collectChunk = (event, state) => {
-  const chunk = processSingleEvent(event, state);
-  return chunk ? [chunk] : [];
-};
-var processBufferedEvents = (eventsText, state) => {
-  const events = eventsText.split(`
-
-`);
-  state.buffer = events.pop() ?? "";
-  return events.flatMap((event) => collectChunk(event, state));
-};
-var readNextChunks = async (reader, decoder, state, signal) => {
-  if (signal?.aborted) {
-    return { chunks: EMPTY_CHUNKS, done: true };
-  }
-  const { done, value } = await reader.read();
-  if (done) {
-    return { chunks: EMPTY_CHUNKS, done: true };
-  }
-  const rawText = state.buffer + decoder.decode(value, { stream: true });
-  const chunks = processBufferedEvents(rawText, state);
-  return { chunks, done: false };
-};
-var findDoneChunk = (chunks) => chunks.findIndex((c) => c.type === "done");
-var sseStreamLoop = async (reader, decoder, state, signal) => {
-  const result = await readNextChunks(reader, decoder, state, signal);
-  if (result.done) {
-    return { chunks: result.chunks, finished: true };
-  }
-  const doneIdx = findDoneChunk(result.chunks);
-  if (doneIdx >= 0) {
-    return { chunks: result.chunks.slice(0, doneIdx + 1), finished: true };
-  }
-  return { chunks: result.chunks, finished: false };
-};
-async function* streamChunks(reader, decoder, state, signal) {
-  let finished = false;
-  while (!finished) {
-    const result = await sseStreamLoop(reader, decoder, state, signal);
-    ({ finished } = result);
-    yield* result.chunks;
-  }
-}
 async function* parseSSEStream4(body, providerName, signal) {
   const reader = body.getReader();
   const decoder = new TextDecoder;
@@ -7239,8 +7257,34 @@ async function* parseSSEStream4(body, providerName, signal) {
     providerBlockInputJson: ""
   };
   try {
-    yield* streamChunks(reader, decoder, state, signal);
+    while (true) {
+      signal?.throwIfAborted();
+      const { done, value } = await reader.read();
+      if (done)
+        return;
+      const events = (state.buffer + decoder.decode(value, { stream: true })).split(`
+
+`);
+      state.buffer = events.pop() ?? "";
+      for (const event of events) {
+        const chunk = processSingleEvent(event, state);
+        if (!chunk)
+          continue;
+        if (chunk.type === "tool_input" && chunk.receivedChars === 0)
+          yield {
+            type: "tool_input_start",
+            id: chunk.id,
+            name: chunk.name
+          };
+        yield chunk;
+        if (chunk.type === "done")
+          return;
+      }
+    }
   } finally {
+    await reader.cancel().catch(() => {
+      return;
+    });
     reader.releaseLock();
   }
 }
@@ -7390,8 +7434,10 @@ var convertMessage2 = (msg) => {
     });
   }
   for (const block of toolResultBlocks) {
+    const images = (block.images ?? []).flatMap((source) => source.type === "base64" ? [source.data] : []);
     results.push({
       content: typeof block.content === "string" ? block.content : "",
+      ...images.length > 0 ? { images } : {},
       role: "tool"
     });
   }
@@ -8998,6 +9044,12 @@ var createAIStoredToolResultRecovery = (toolNames) => async ({ params }) => {
       if (block.type !== "tool_result" || !eligible.has(block.tool_use_id) || block === newest)
         return block;
       const marker = "[Earlier lookup removed from working context. Its immutable original remains saved. The original tool call and source/version arguments above are unchanged: repeat that lookup if needed to verify facts. Do not infer facts from this marker.]";
+      if (block.images?.length)
+        return {
+          content: marker,
+          tool_use_id: block.tool_use_id,
+          type: block.type
+        };
       return block.content.length > marker.length ? { ...block, content: marker } : block;
     })
   }));
@@ -9139,10 +9191,11 @@ export {
   streamAIWithTools,
   streamWithAIContext,
   tableCard,
+  toolOutputText,
   verifyOpenRouterWebhookSignature,
   withResilience,
   xai
 };
 
-//# debugId=4ED2ED351363579D64756E2164756E21
+//# debugId=94F48F9A5CBF872A64756E2164756E21
 //# sourceMappingURL=index.js.map