@inferencesh/sdk 0.23.0 → 0.24.0

package/CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- Client tools can return files. A `ClientToolHandlerFn` may return `{ result: string; attachments?: FileRef[] }` (new `ClientToolResult` type) besides a string; the files are uploaded first with POST /files (`client.files.upload`, the chat's `uploadFile`) and sent as `attachments` with the result. Images reach the model as image input when it supports images. `submitToolResult` (agent api, `Agent`, `AgentsAPI`, the React chat actions) takes optional `attachments`, sent only when non-empty. `ToolResultRequest.attachments` is added by hand to the generated types until the next `make types` picks it up from the API. String returns work as before.
+
 ## [0.23.0] - 2026-10-11
 
 ### Changed

package/dist/agent/actions.js

@@ -16,6 +16,15 @@ import { isInferenceError } from '../http/errors.js';
 // =============================================================================
 // Action Creators
 // =============================================================================
+/** Up to about ten minutes of backoff (1s doubling to 30s, jittered) before a tool result is given up. */
+const TOOL_RESULT_MAX_ATTEMPTS = 25;
+/** A failure a retry can get past: no response at all, a 5xx, a 408 or a 429. */
+function isTransient(err) {
+    const status = err?.statusCode;
+    if (typeof status !== 'number')
+        return true;
+    return status >= 500 || status === 408 || status === 429;
+}
 export function createActions(ctx) {
     const dispatchedToolInvocations = new Set();
     const { client, dispatch, getState, getConfig, getChatId, getClientToolHandlers, getStreamManager, setStreamManager, getStreamEnabled, getPollIntervalMs, callbacks } = ctx;
@@ -24,6 +33,28 @@ export function createActions(ctx) {
     // is no longer current (stopped, reset, switched to another chat) leaves
     // state alone, including work it started before that.
     let streamGeneration = 0;
+    // A client tool's result is the only way its turn continues: the server
+    // waits on it. A result lost to a network drop left the turn stuck on
+    // "running" for good, so it is sent again until it lands or is refused.
+    const deliverToolResult = async (invocationId, result, attachments) => {
+        for (let attempt = 1;; attempt++) {
+            try {
+                if (attachments?.length)
+                    await api.submitToolResult(client, invocationId, result, attachments);
+                else
+                    await api.submitToolResult(client, invocationId, result);
+                return;
+            }
+            catch (err) {
+                if (!isTransient(err) || attempt >= TOOL_RESULT_MAX_ATTEMPTS) {
+                    console.error(`[AgentSDK] Failed to submit tool result for ${invocationId}:`, err);
+                    return;
+                }
+                const step = Math.min(30000, 1000 * 2 ** (attempt - 1));
+                await new Promise((r) => setTimeout(r, step / 2 + Math.random() * (step / 2)));
+            }
+        }
+    };
     const checkTurnEnd = (chat) => {
         const busy = isChatBusy(chat);
         if (prevChatWasBusy && !busy) {
@@ -63,7 +94,7 @@ export function createActions(ctx) {
                     const handler = clientToolHandlers.get(functionName);
                     if (!handler) {
                         console.warn(`[AgentSDK] No handler for client tool: ${functionName}`);
-                        api.submitToolResult(client, invocation.id, JSON.stringify({
+                        deliverToolResult(invocation.id, JSON.stringify({
                             status: 'not_available',
                             message: `Client tool "${functionName}" is not available in this environment`,
                         }));
@@ -73,12 +104,12 @@ export function createActions(ctx) {
                     // Use Promise.resolve to handle both sync and async handlers
                     const args = invocation.function?.arguments || {};
                     Promise.resolve(handler(args))
-                        .then((result) => {
-                        api.submitToolResult(client, invocation.id, result);
-                    })
+                        .then((output) => typeof output === 'string'
+                        ? deliverToolResult(invocation.id, output)
+                        : deliverToolResult(invocation.id, output.result, output.attachments))
                         .catch((error) => {
                         console.error(`[AgentSDK] Client tool ${functionName} error:`, error);
-                        api.submitToolResult(client, invocation.id, JSON.stringify({
+                        return deliverToolResult(invocation.id, JSON.stringify({
                             error: String(error)
                         }));
                     });
@@ -381,9 +412,9 @@ export function createActions(ctx) {
             dispatch({ type: 'SET_ERROR', payload: undefined });
             dispatch({ type: 'SET_CONNECTION_STATUS', payload: 'idle' });
         },
-        submitToolResult: async (toolInvocationId, result) => {
+        submitToolResult: async (toolInvocationId, result, attachments) => {
             try {
-                await api.submitToolResult(client, toolInvocationId, result);
+                await api.submitToolResult(client, toolInvocationId, result, attachments);
             }
             catch (error) {
                 console.error('[AgentSDK] Failed to submit tool result:', error);

package/dist/agent/api.d.ts

@@ -45,7 +45,7 @@ export declare function submitToolResult(client: AgentClient, toolInvocationId:
         payload?: Record<string, unknown>;
     };
     form_data?: Record<string, unknown>;
-}): Promise<void>;
+}, attachments?: FileRef[]): Promise<void>;
 /**
  * Approve a call awaiting approval; the api then runs it.
  *

package/dist/agent/api.js

@@ -144,8 +144,11 @@ export async function setAgent(client, chatId, agentRef) {
 // =========================================================================
 // Tool operations
 // =========================================================================
-export async function submitToolResult(client, toolInvocationId, resultOrAction) {
-    const data = typeof resultOrAction === 'string' ? { result: resultOrAction } : resultOrAction;
+export async function submitToolResult(client, toolInvocationId, resultOrAction, attachments) {
+    const data = typeof resultOrAction === 'string' ? { result: resultOrAction } : { ...resultOrAction };
+    if (attachments && attachments.length > 0) {
+        data.attachments = attachments;
+    }
     await client.http.request('post', `/tools/${toolInvocationId}`, { data });
 }
 /**

package/dist/agent/index.d.ts

@@ -46,6 +46,6 @@
 export { AgentChatProvider } from './provider.js';
 export { useAgentChat, useAgentActions, useAgentClient, useAgentChatContext, useMessage, } from './hooks.js';
 export { AgentChatContext, type AgentChatContextValue } from './context.js';
-export type { AgentClient, FileRef, AdHocAgentConfig, TemplateAgentConfig, AgentOptions, AgentChatState, AgentChatActions, AlwaysAllowChoice, ChatStatus, AgentChatProviderProps, ClientTool, ClientToolHandlerFn, } from './types.js';
+export type { AgentClient, FileRef, AdHocAgentConfig, TemplateAgentConfig, AgentOptions, AgentChatState, AgentChatActions, AlwaysAllowChoice, ChatStatus, AgentChatProviderProps, ClientTool, ClientToolHandlerFn, ClientToolResult, } from './types.js';
 export { isAdHocConfig, isTemplateConfig, isClientTool, extractToolSchemas, extractClientToolHandlers, } from './types.js';
 export { MCPMethodElicitationCreate, parseMCPInputState, elicitParams, isURLElicitation, buildMCPInputResult, type MCPInputState, type ElicitRequestParams, type ElicitRequestedSchema, type ElicitPropertySchema, } from './mcp-input.js';

package/dist/agent/types.d.ts

@@ -129,7 +129,7 @@ export interface AgentChatActions {
     /** Clear the current error */
     clearError: () => void;
     /** Submit a tool result (for widgets/awaiting input) - usually called automatically by SDK */
-    submitToolResult: (toolInvocationId: string, result: string) => Promise<void>;
+    submitToolResult: (toolInvocationId: string, result: string, attachments?: FileRef[]) => Promise<void>;
     /** Answer an MCP tool call's input requests (AwaitingInput status with MCPInputState data) */
     submitMCPInput: (toolInvocationId: string, responses: Record<string, ElicitResult>) => Promise<void>;
     /**
@@ -221,10 +221,23 @@ export interface AgentChatProviderProps {
     /** Children */
     children: React.ReactNode;
 }
+/**
+ * What a client tool handler returns: the result text, or the text plus files.
+ *
+ * `attachments` are files already uploaded with POST /files (for example with
+ * `client.files.upload(dataUrlOrBlob)` or the chat's `uploadFile`), passed as
+ * `FileRef`s, the same as message attachments. An image attachment reaches the
+ * model as image input when the model supports images; other files are listed
+ * on the tool message.
+ */
+export type ClientToolResult = string | {
+    result: string;
+    attachments?: FileRef[];
+};
 /**
  * Client tool handler function - takes args and returns a result (sync or async)
  */
-export type ClientToolHandlerFn = (args: Record<string, unknown>) => Promise<string> | string;
+export type ClientToolHandlerFn = (args: Record<string, unknown>) => Promise<ClientToolResult> | ClientToolResult;
 /**
  * Client tool definition - combines schema with handler
  */

package/dist/api/agents.d.ts

@@ -2,7 +2,7 @@ import { HttpClient } from '../http/client.js';
 import type { Response } from '../http/response.js';
 import type { AgentUpdateBody } from './write-bodies.js';
 import { FilesAPI } from './files.js';
-import { ChannelContext, ChatDTO, ChatMessageDTO, LLMDelta, LLMOutput, AgentConfigInput as AgentConfig, AgentDTO, AgentVersionDTO, CreateAgentRequest, FileDTO as File, InterruptDTO, CursorListRequest, CursorListResponse } from '../types.js';
+import { ChannelContext, ChatDTO, ChatMessageDTO, LLMDelta, LLMOutput, AgentConfigInput as AgentConfig, AgentDTO, AgentVersionDTO, CreateAgentRequest, FileDTO as File, InterruptDTO, CursorListRequest, CursorListResponse, FileRef } from '../types.js';
 /** Internal tool definition returned by getInternalTools */
 export interface InternalToolDefinition {
     id: string;
@@ -108,7 +108,9 @@ export declare class Agent {
     /** Stop the current chat generation */
     stopChat(): Promise<void>;
     /**
-     * Submit a tool result
+     * Submit a tool result. `attachments` are files uploaded with POST /files
+     * (`files.upload`) that the tool returns; images reach the model as image
+     * input when it supports images.
      */
     submitToolResult(toolInvocationId: string, resultOrAction: string | {
         action: {
@@ -116,7 +118,7 @@ export declare class Agent {
             payload?: Record<string, unknown>;
         };
         form_data?: Record<string, unknown>;
-    }): Promise<void>;
+    }, attachments?: FileRef[]): Promise<void>;
     /** Stop streaming/polling and cleanup */
     disconnect(): void;
     /**
@@ -205,7 +207,9 @@ export declare class AgentsAPI {
      */
     create(config: string | AgentConfig, options?: AgentOptions): Agent;
     /**
-     * Submit a tool result
+     * Submit a tool result. `attachments` are files uploaded with POST /files
+     * (`files.upload`) that the tool returns; images reach the model as image
+     * input when it supports images.
      */
     submitToolResult(toolInvocationId: string, resultOrAction: string | {
         action: {
@@ -213,7 +217,7 @@ export declare class AgentsAPI {
             payload?: Record<string, unknown>;
         };
         form_data?: Record<string, unknown>;
-    }): Promise<void>;
+    }, attachments?: FileRef[]): Promise<void>;
     /**
      * Resolve an interrupt gate on an agent run.
      *

package/dist/api/agents.js

@@ -172,11 +172,17 @@ export class Agent {
         await this.http.request('post', `/chats/${this.chatId}/stop`);
     }
     /**
-     * Submit a tool result
+     * Submit a tool result. `attachments` are files uploaded with POST /files
+     * (`files.upload`) that the tool returns; images reach the model as image
+     * input when it supports images.
      */
-    async submitToolResult(toolInvocationId, resultOrAction) {
+    async submitToolResult(toolInvocationId, resultOrAction, attachments) {
         const result = typeof resultOrAction === 'string' ? resultOrAction : JSON.stringify(resultOrAction);
-        await this.http.request('post', `/tools/${toolInvocationId}`, { data: { result } });
+        const data = { result };
+        if (attachments && attachments.length > 0) {
+            data.attachments = attachments;
+        }
+        await this.http.request('post', `/tools/${toolInvocationId}`, { data });
     }
     /** Stop streaming/polling and cleanup */
     disconnect() {
@@ -496,11 +502,17 @@ export class AgentsAPI {
         return new Agent(this.http, this.files, config, options);
     }
     /**
-     * Submit a tool result
+     * Submit a tool result. `attachments` are files uploaded with POST /files
+     * (`files.upload`) that the tool returns; images reach the model as image
+     * input when it supports images.
      */
-    async submitToolResult(toolInvocationId, resultOrAction) {
+    async submitToolResult(toolInvocationId, resultOrAction, attachments) {
         const result = typeof resultOrAction === 'string' ? resultOrAction : JSON.stringify(resultOrAction);
-        await this.http.request('post', `/tools/${toolInvocationId}`, { data: { result } });
+        const data = { result };
+        if (attachments && attachments.length > 0) {
+            data.attachments = attachments;
+        }
+        await this.http.request('post', `/tools/${toolInvocationId}`, { data });
     }
     /**
      * Resolve an interrupt gate on an agent run.

package/dist/types.d.ts

@@ -502,10 +502,17 @@ export interface CreateAgentMessageResponse {
     assistant_message?: ChatMessageDTO;
 }
 /**
- * ToolResultRequest represents a tool result submission
+ * ToolResultRequest is the body of POST /tools/{id}: a client's result for a
+ * tool call it ran.
  */
 export interface ToolResultRequest {
     result: string;
+    /**
+     * Attachments are files uploaded with POST /files, as in MessageInput.
+     * The tool message carries them; images reach the model as image input
+     * when it accepts images.
+     */
+    attachments?: FileRef[];
 }
 /**
  * PartialFile is the clean DTO version (no gorm tags).
@@ -5640,6 +5647,16 @@ export interface DecisionVisionInput extends DecisionInput {
      */
     images?: string[];
 }
+/**
+ * DecisionVideoInput is the input of a decision model that also watches
+ * videos: a DecisionVisionInput plus the videos the questions are about.
+ */
+export interface DecisionVideoInput extends DecisionVisionInput {
+    /**
+     * Videos are file URIs. Every question sees them, after the images.
+     */
+    videos?: string[];
+}
 /**
  * DecisionChoiceAnswer is the answer to a choice question.
  */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@inferencesh/sdk",
-  "version": "0.23.0",
+  "version": "0.24.0",
   "description": "Official JavaScript/TypeScript SDK for inference.sh - Run AI models with a simple API",
   "type": "module",
   "main": "dist/index.js",