@theoremjs/agents 0.3.4-next.387.g8bea7ff → 0.3.4-next.389.gecc7333

package/esm/src/kernel/tools/execute.js

@@ -14,6 +14,7 @@ import { AgentCallFailed, agentToolOrigin } from './agent.js';
 import { coordinateTool } from './coordinate.js';
 import { dispatchToolBody, failedActivity, failedActivityOf, failureEvent, messageOf, networkBlocked, networkBlockedEvent, requestChecks, sourceEvents, startToolExecution, toolCallArguments, toolEvent, } from './events.js';
 import { HandlerEvents } from './handler-events.js';
+import { fetchToolMedia } from './media-fetch.js';
 import { formatToolFailureForModel, formatToolResult, frameToolFailure } from './model-text.js';
 import { operationUnavailable, operationWireValue } from './operation.js';
 import { checkPermission, gateDetails, isGateResumeDenied, isGateResumeGranted, isResumeContinuation, } from './permission.js';
@@ -665,6 +666,18 @@ function authorizeSavedDisclosure(args, tool, outcome) {
     if (guard.event?.action === 'block' || guard.event?.action === 'redact')
         throw blocked();
 }
+/** A successful remote call's result with the pictures its tool's `media` names, fetched and added beside any the server sent. */
+async function* withNamedMedia(tool, outcome, ctx, base) {
+    if (outcome.kind !== 'ok' || !tool.media)
+        return outcome;
+    const fetched = await fetchToolMedia(tool.media, outcome.outputRaw, ctx);
+    if (fetched.warning)
+        yield toolEvent(base, { phase: 'warning', warning: fetched.warning });
+    if (fetched.parts.length === 0)
+        return outcome;
+    const parts = [...(outcome.modelResult.parts ?? []), ...fetched.parts];
+    return { ...outcome, modelResult: { ...outcome.modelResult, parts } };
+}
 /** Settles an outcome the kernel reached without a handler's own result: a remote body, or a sign-in. */
 async function* settleBodyOutcome(args) {
     const { outcome, tool, base, callId, safeInput, stages, guard, labels } = args;
@@ -1097,9 +1110,10 @@ async function* settleByType(tools, tool, safeInput, fullCtx, base, snapshot, st
         });
     }
     const guard = resultGuard(tools, tool, fullCtx, snapshot, stages?.span);
-    const remoteOutcome = tool.type === 'http'
+    const body = tool.type === 'http'
         ? yield* executeHttpTool(tool, safeInput, fullCtx, base, stages, beginBody, new HandlerEvents(base, guard.handlerEvents(stages?.drafts === true)))
         : yield* executeMcpTool(tool, safeInput, fullCtx, base, stages, tools.mcpSessions, beginBody);
+    const remoteOutcome = yield* withNamedMedia(tool, body, fullCtx, base);
     return yield* settleBodyOutcome({
         outcome: remoteOutcome,
         tool,

package/esm/src/kernel/tools/media-fetch.d.ts

@@ -0,0 +1,14 @@
+import type { InteractionPart, ToolWarning } from '../turn-events.js';
+import type { ToolContext } from './types.js';
+/** The most pictures one call hands the model. */
+export declare const MAX_TOOL_MEDIA_FILES = 4;
+/**
+ * The pictures a tool's `media(output)` names, fetched through the profile's network policy (every
+ * hop, redirects included) and handed to the model as image parts. A picture that is blocked, slow,
+ * too big or not an image is left out and reported in one `media_unfetched` warning: the call
+ * itself stays whole.
+ */
+export declare function fetchToolMedia(media: (output: never) => string[], output: unknown, ctx: ToolContext): Promise<{
+    parts: InteractionPart[];
+    warning?: ToolWarning;
+}>;

package/esm/src/kernel/tools/media-fetch.js

@@ -0,0 +1,94 @@
+import { fetchGuarded } from '../../guardrails/network.js';
+import { bytesToBase64 } from '../util/base64.js';
+import { toolNetworkPolicy } from './events.js';
+/** The most pictures one call hands the model. */
+export const MAX_TOOL_MEDIA_FILES = 4;
+/** The largest picture fetched; a larger one is left out, not cut. */
+const MAX_TOOL_MEDIA_BYTES = 5 * 1024 * 1024;
+/** How long each picture may take, so a slow host cannot hold the call. */
+const MEDIA_FETCH_TIMEOUT_MS = 15_000;
+const IMAGE_TYPES = new Set(['image/png', 'image/jpeg', 'image/gif', 'image/webp']);
+/** The body, read up to `maxBytes`; `undefined` when it runs past. */
+async function readCappedBytes(response, maxBytes) {
+    const declared = Number(response.headers.get('content-length'));
+    if (Number.isFinite(declared) && declared > maxBytes) {
+        await response.body?.cancel().catch(() => { });
+        return undefined;
+    }
+    if (!response.body)
+        return new Uint8Array(await response.arrayBuffer());
+    const reader = response.body.getReader();
+    const chunks = [];
+    let total = 0;
+    for (;;) {
+        const { done, value } = await reader.read();
+        if (done)
+            break;
+        total += value.byteLength;
+        if (total > maxBytes) {
+            await reader.cancel().catch(() => { });
+            return undefined;
+        }
+        chunks.push(value);
+    }
+    const bytes = new Uint8Array(total);
+    let at = 0;
+    for (const chunk of chunks) {
+        bytes.set(chunk, at);
+        at += chunk.byteLength;
+    }
+    return bytes;
+}
+async function fetchOne(url, ctx) {
+    const timeout = AbortSignal.timeout(MEDIA_FETCH_TIMEOUT_MS);
+    const signal = ctx.signal ? AbortSignal.any([ctx.signal, timeout]) : timeout;
+    const response = await fetchGuarded(url, { method: 'GET', headers: { Accept: 'image/*' }, signal }, {
+        policy: toolNetworkPolicy(ctx),
+        followRedirects: true,
+        resolveHost: ctx.resolveHost,
+    });
+    if (!response.ok)
+        throw new Error(`HTTP ${response.status}`); // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
+    const mimeType = (response.headers.get('content-type') ?? '').split(';')[0]?.trim().toLowerCase();
+    if (!mimeType || !IMAGE_TYPES.has(mimeType)) {
+        await response.body?.cancel().catch(() => { });
+        throw new Error(`not a png, jpeg, gif or webp image (${mimeType || 'no type'})`); // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
+    }
+    const bytes = await readCappedBytes(response, MAX_TOOL_MEDIA_BYTES);
+    if (!bytes)
+        throw new Error(`larger than ${MAX_TOOL_MEDIA_BYTES} bytes`); // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
+    return { type: 'image', mimeType, data: bytesToBase64(bytes) };
+}
+/**
+ * The pictures a tool's `media(output)` names, fetched through the profile's network policy (every
+ * hop, redirects included) and handed to the model as image parts. A picture that is blocked, slow,
+ * too big or not an image is left out and reported in one `media_unfetched` warning: the call
+ * itself stays whole.
+ */
+export async function fetchToolMedia(media, output, ctx) {
+    let urls;
+    try {
+        urls = [...new Set(media(output))].slice(0, MAX_TOOL_MEDIA_FILES);
+    }
+    catch (err) {
+        // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
+        return { parts: [], warning: unfetched([`the media hook threw: ${String(err)}`]) };
+    }
+    const settled = await Promise.allSettled(urls.map((url) => fetchOne(url, ctx)));
+    const parts = [];
+    const reasons = [];
+    settled.forEach((result, index) => {
+        if (result.status === 'fulfilled')
+            parts.push(result.value);
+        else
+            reasons.push(`${new URL(urls[index] ?? 'https://invalid.invalid').hostname}: ${String(result.reason instanceof Error ? result.reason.message : result.reason)}`);
+    });
+    return { parts, ...(reasons.length ? { warning: unfetched(reasons) } : {}) };
+}
+function unfetched(reasons) {
+    return {
+        code: 'media_unfetched',
+        severity: 'warning',
+        message: `Some pictures the tool named were left out. ${reasons.join('; ')}`, // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
+    };
+}

package/esm/src/kernel/tools/types.d.ts

@@ -160,6 +160,15 @@ export interface ToolOutputHooks<TOut = unknown> {
     /** A source that fails `sourceSchema`, or a throw, is a `sources_invalid` warning and is not cited. */
     sources?: (output: TOut) => Source[];
 }
+/** For a tool whose result names pictures by address; function tools return the bytes themselves as `parts`. */
+export interface ToolMediaHooks<TOut = unknown> {
+    /**
+     * The https addresses of pictures in `output` the model should see, at most `MAX_TOOL_MEDIA_FILES`
+     * of them. The kernel fetches each one through the profile's network policy and hands the model
+     * the image as a part; one it cannot fetch is a `media_unfetched` warning, never a failed call.
+     */
+    media?: (output: TOut) => string[];
+}
 export interface FunctionToolDef<TIn = unknown, TOut = unknown> extends ToolBase, ToolHostHooks<TIn>, ToolOutputHooks<TOut> {
     type: 'function';
     input: z.ZodType<TIn>;
@@ -201,7 +210,7 @@ export interface ToolAuthConfig {
     clientId?: string;
     redirectUri?: string;
 }
-export interface HttpToolDef<TIn = unknown, TOut = unknown> extends ToolBase, ToolHostHooks<TIn>, ToolOutputHooks<TOut> {
+export interface HttpToolDef<TIn = unknown, TOut = unknown> extends ToolBase, ToolHostHooks<TIn>, ToolOutputHooks<TOut>, ToolMediaHooks<TOut> {
     type: 'http';
     input: z.ZodType<TIn>;
     output: z.ZodType<TOut>;
@@ -228,7 +237,7 @@ export interface HttpToolLimit {
     requests: number;
     windowMs: number;
 }
-export interface McpToolDef<TIn = unknown, TOut = unknown> extends ToolBase, ToolHostHooks<TIn>, ToolOutputHooks<TOut> {
+export interface McpToolDef<TIn = unknown, TOut = unknown> extends ToolBase, ToolHostHooks<TIn>, ToolOutputHooks<TOut>, ToolMediaHooks<TOut> {
     type: 'mcp';
     input: z.ZodType<TIn>;
     output: z.ZodType<TOut>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@theoremjs/agents",
-  "version": "0.3.4-next.387.g8bea7ff",
+  "version": "0.3.4-next.389.gecc7333",
   "description": "A flat TypeScript agent kernel for typed profiles, deterministic turn execution, registered tools, provider adapters, guardrails, and host-injected traces.",
   "keywords": [
     "agent",