@theoremjs/agents 0.3.4-next.387.g8bea7ff → 0.3.4-next.401.gec4c582

package/esm/src/kernel/tools/execute.js

@@ -14,6 +14,7 @@ import { AgentCallFailed, agentToolOrigin } from './agent.js';
 import { coordinateTool } from './coordinate.js';
 import { dispatchToolBody, failedActivity, failedActivityOf, failureEvent, messageOf, networkBlocked, networkBlockedEvent, requestChecks, sourceEvents, startToolExecution, toolCallArguments, toolEvent, } from './events.js';
 import { HandlerEvents } from './handler-events.js';
+import { fetchToolImages } from './image-fetch.js';
 import { formatToolFailureForModel, formatToolResult, frameToolFailure } from './model-text.js';
 import { operationUnavailable, operationWireValue } from './operation.js';
 import { checkPermission, gateDetails, isGateResumeDenied, isGateResumeGranted, isResumeContinuation, } from './permission.js';
@@ -275,8 +276,15 @@ async function* settleToolCall(args) {
                 denied = true;
         }
     }
-    if (!failure && sources)
-        yield* sourceEvents(base, sources, outputRaw);
+    if (!failure) {
+        modelResult = yield* citeAndShow({
+            base,
+            sources,
+            images: args.images,
+            outputRaw,
+            modelResult,
+        });
+    }
     yield failure
         ? failureEvent(base, { ...failure, message: String(yield* guard.shown(failure.message)) }, formatToolResult(modelResult))
         : completeEvent(base, {
@@ -295,6 +303,35 @@ async function* settleToolCall(args) {
         ...stageSettlement(post),
     };
 }
+/** The settled output's cited sources, then the pictures its tool names; the result carrying them. */
+async function* citeAndShow(args) {
+    const { base, sources, images, outputRaw, modelResult } = args;
+    if (sources)
+        yield* sourceEvents(base, sources, outputRaw);
+    if (!images || outputRaw === undefined)
+        return modelResult;
+    return yield* withFetchedImages(base, modelResult, images, outputRaw);
+}
+/**
+ * The result with the pictures its tool names, fetched after the guard and the stages passed it.
+ * The addresses fetched are written into the result's text, so the egress rule on pictures the
+ * model shows counts them as given by a tool.
+ */
+async function* withFetchedImages(base, result, images, outputRaw) {
+    const fetched = await images(outputRaw);
+    if (fetched.warning)
+        yield toolEvent(base, { phase: 'warning', warning: fetched.warning });
+    if (fetched.parts.length === 0)
+        return result;
+    const note = `\n[Pictures shown with this result: ${fetched.urls.join(' ')}]`; // labels-exempt: a marker the egress rule reads the addresses from, not prose (P2)
+    return {
+        ...result,
+        parts: [...(result.parts ?? []), ...fetched.parts],
+        ...(result.modelText !== undefined
+            ? { modelText: result.modelText + note }
+            : { finding: result.finding + note }),
+    };
+}
 /** What the after-call stages add to a settlement: a turn to stop, input to hand the model. */
 function stageSettlement(post) {
     return {
@@ -474,6 +511,7 @@ agent, beginBody) {
     throwIfAborted(ctx.signal);
     const settleOutcome = (outcome) => settleBodyOutcome({
         outcome,
+        ctx,
         tool,
         base,
         callId,
@@ -667,7 +705,7 @@ function authorizeSavedDisclosure(args, tool, outcome) {
 }
 /** Settles an outcome the kernel reached without a handler's own result: a remote body, or a sign-in. */
 async function* settleBodyOutcome(args) {
-    const { outcome, tool, base, callId, safeInput, stages, guard, labels } = args;
+    const { outcome, tool, base, callId, safeInput, stages, guard, labels, ctx } = args;
     const name = tool.name;
     if (outcome.kind === 'gated') {
         if (outcome.gate.kind === 'confirmation') {
@@ -703,6 +741,9 @@ async function* settleBodyOutcome(args) {
         guard,
         provisional: { outputRaw: outcome.outputRaw, modelResult: outcome.modelResult },
         sources: tool.sources,
+        ...(tool.type !== 'function' && tool.images
+            ? { images: (output) => fetchToolImages(tool.images, output, ctx) }
+            : {}),
         labels: tool.labels,
         reproject: makeReproject((v) => parseToolOutput(tool.output, v), 
         // why: A hook's edit replaces the value; the media the tool returned stays.
@@ -1097,11 +1138,12 @@ async function* settleByType(tools, tool, safeInput, fullCtx, base, snapshot, st
         });
     }
     const guard = resultGuard(tools, tool, fullCtx, snapshot, stages?.span);
-    const remoteOutcome = tool.type === 'http'
+    const body = tool.type === 'http'
         ? yield* executeHttpTool(tool, safeInput, fullCtx, base, stages, beginBody, new HandlerEvents(base, guard.handlerEvents(stages?.drafts === true)))
         : yield* executeMcpTool(tool, safeInput, fullCtx, base, stages, tools.mcpSessions, beginBody);
     return yield* settleBodyOutcome({
-        outcome: remoteOutcome,
+        outcome: body,
+        ctx: fullCtx,
         tool,
         base,
         callId: base.callId,

package/esm/src/kernel/tools/image-fetch.d.ts

@@ -0,0 +1,15 @@
+import type { InteractionPart, ToolWarning } from '../turn-events.js';
+import type { ToolContext } from './types.js';
+/** The most pictures one call hands the model. */
+export declare const MAX_TOOL_IMAGES = 4;
+/**
+ * The pictures a tool's `images(output)` names, fetched through the profile's network policy (every
+ * hop, redirects included, no tool credentials) and handed to the model as image parts. A picture
+ * that is not an https address, blocked, slow, too big or not an image is left out and reported in
+ * one `images_unfetched` warning: the call itself stays whole. `urls` are the ones fetched.
+ */
+export declare function fetchToolImages(images: (output: never) => string[], output: unknown, ctx: ToolContext): Promise<{
+    parts: InteractionPart[];
+    urls: string[];
+    warning?: ToolWarning;
+}>;

package/esm/src/kernel/tools/image-fetch.js

@@ -0,0 +1,136 @@
+import { fetchGuarded } from '../../guardrails/network.js';
+import { bytesToBase64 } from '../util/base64.js';
+import { toolNetworkPolicy } from './events.js';
+/** The most pictures one call hands the model. */
+export const MAX_TOOL_IMAGES = 4;
+/** The largest picture fetched; a larger one is left out, not cut. */
+const MAX_TOOL_IMAGE_BYTES = 5 * 1024 * 1024;
+/** The most one call hands the model in all, so pictures cannot crowd a request past a provider's limit. */
+const MAX_TOOL_IMAGES_TOTAL_BYTES = 10 * 1024 * 1024;
+/** How long each picture may take, so a slow host cannot hold the call. */
+const IMAGE_FETCH_TIMEOUT_MS = 15_000;
+const IMAGE_TYPES = new Set(['image/png', 'image/jpeg', 'image/gif', 'image/webp']);
+/** The body, read up to `maxBytes`; `undefined` when it runs past. */
+async function readCappedBytes(response, maxBytes) {
+    const declared = Number(response.headers.get('content-length'));
+    if (Number.isFinite(declared) && declared > maxBytes) {
+        await response.body?.cancel().catch(() => { });
+        return undefined;
+    }
+    if (!response.body)
+        return new Uint8Array(await response.arrayBuffer());
+    const reader = response.body.getReader();
+    const chunks = [];
+    let total = 0;
+    for (;;) {
+        const { done, value } = await reader.read();
+        if (done)
+            break;
+        total += value.byteLength;
+        if (total > maxBytes) {
+            await reader.cancel().catch(() => { });
+            return undefined;
+        }
+        chunks.push(value);
+    }
+    const bytes = new Uint8Array(total);
+    let at = 0;
+    for (const chunk of chunks) {
+        bytes.set(chunk, at);
+        at += chunk.byteLength;
+    }
+    return bytes;
+}
+async function fetchOne(url, ctx) {
+    const timeout = AbortSignal.timeout(IMAGE_FETCH_TIMEOUT_MS);
+    const signal = ctx.signal ? AbortSignal.any([ctx.signal, timeout]) : timeout;
+    const response = await fetchGuarded(url, { method: 'GET', headers: { Accept: 'image/*' }, signal }, {
+        policy: toolNetworkPolicy(ctx),
+        followRedirects: true,
+        resolveHost: ctx.resolveHost,
+    });
+    if (!response.ok)
+        throw new Error(`HTTP ${response.status}`); // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
+    const mimeType = (response.headers.get('content-type') ?? '').split(';')[0]?.trim().toLowerCase();
+    if (!mimeType || !IMAGE_TYPES.has(mimeType)) {
+        await response.body?.cancel().catch(() => { });
+        throw new Error(`not a png, jpeg, gif or webp image (${mimeType || 'no type'})`); // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
+    }
+    const bytes = await readCappedBytes(response, MAX_TOOL_IMAGE_BYTES);
+    if (!bytes)
+        throw new Error(`larger than ${MAX_TOOL_IMAGE_BYTES} bytes`); // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
+    return { type: 'image', mimeType, data: bytesToBase64(bytes) };
+}
+/** `url` when it is an absolute https address; otherwise the reason it is not. */
+function httpsAddress(url) {
+    if (typeof url !== 'string')
+        return { reason: 'an address that is not text' }; // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
+    try {
+        if (new URL(url).protocol === 'https:')
+            return { url };
+    }
+    catch {
+        // why: an unparseable address is reported below as a reason, not thrown.
+    }
+    return { reason: `${url.slice(0, 80)}: not an absolute https address` }; // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
+}
+/**
+ * The pictures a tool's `images(output)` names, fetched through the profile's network policy (every
+ * hop, redirects included, no tool credentials) and handed to the model as image parts. A picture
+ * that is not an https address, blocked, slow, too big or not an image is left out and reported in
+ * one `images_unfetched` warning: the call itself stays whole. `urls` are the ones fetched.
+ */
+export async function fetchToolImages(images, output, ctx) {
+    const reasons = [];
+    const named = [];
+    try {
+        for (const raw of new Set(images(output))) {
+            const address = httpsAddress(raw);
+            if ('url' in address)
+                named.push(address.url);
+            else
+                reasons.push(address.reason);
+        }
+    }
+    catch (err) {
+        // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
+        return {
+            parts: [],
+            urls: [],
+            warning: unfetched([`the images hook threw: ${String(err)}`]), // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
+        };
+    }
+    const wanted = named.slice(0, MAX_TOOL_IMAGES);
+    const settled = await Promise.allSettled(wanted.map((url) => fetchOne(url, ctx)));
+    const parts = [];
+    const urls = [];
+    let total = 0;
+    settled.forEach((result, index) => {
+        const url = wanted[index];
+        if (result.status === 'rejected') {
+            const why = result.reason instanceof Error ? result.reason.message : String(result.reason);
+            reasons.push(`${new URL(url).hostname}: ${why}`);
+            return;
+        }
+        const size = 'data' in result.value ? result.value.data.length : 0;
+        if (total + size > MAX_TOOL_IMAGES_TOTAL_BYTES) {
+            reasons.push(`${new URL(url).hostname}: past the ${MAX_TOOL_IMAGES_TOTAL_BYTES} byte total`); // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
+            return;
+        }
+        total += size;
+        parts.push(result.value);
+        urls.push(url);
+    });
+    return {
+        parts,
+        urls,
+        ...(reasons.length ? { warning: unfetched(reasons) } : {}),
+    };
+}
+function unfetched(reasons) {
+    return {
+        code: 'images_unfetched',
+        severity: 'warning',
+        message: `Some pictures the tool named were left out. ${reasons.join('; ')}`, // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
+    };
+}

package/esm/src/kernel/tools/types.d.ts

@@ -160,6 +160,16 @@ export interface ToolOutputHooks<TOut = unknown> {
     /** A source that fails `sourceSchema`, or a throw, is a `sources_invalid` warning and is not cited. */
     sources?: (output: TOut) => Source[];
 }
+/** For a tool whose result names pictures by address; function tools return the bytes themselves as `parts`. */
+export interface ToolImageHooks<TOut = unknown> {
+    /**
+     * The https addresses of pictures in `output` the model should see, at most `MAX_TOOL_IMAGES`
+     * of them. After the result passes the output guard, the kernel fetches each one through the
+     * profile's network policy, without the tool's credentials, and hands the model the image as a
+     * part; one it cannot fetch is an `images_unfetched` warning, never a failed call.
+     */
+    images?: (output: TOut) => string[];
+}
 export interface FunctionToolDef<TIn = unknown, TOut = unknown> extends ToolBase, ToolHostHooks<TIn>, ToolOutputHooks<TOut> {
     type: 'function';
     input: z.ZodType<TIn>;
@@ -201,7 +211,7 @@ export interface ToolAuthConfig {
     clientId?: string;
     redirectUri?: string;
 }
-export interface HttpToolDef<TIn = unknown, TOut = unknown> extends ToolBase, ToolHostHooks<TIn>, ToolOutputHooks<TOut> {
+export interface HttpToolDef<TIn = unknown, TOut = unknown> extends ToolBase, ToolHostHooks<TIn>, ToolOutputHooks<TOut>, ToolImageHooks<TOut> {
     type: 'http';
     input: z.ZodType<TIn>;
     output: z.ZodType<TOut>;
@@ -228,7 +238,7 @@ export interface HttpToolLimit {
     requests: number;
     windowMs: number;
 }
-export interface McpToolDef<TIn = unknown, TOut = unknown> extends ToolBase, ToolHostHooks<TIn>, ToolOutputHooks<TOut> {
+export interface McpToolDef<TIn = unknown, TOut = unknown> extends ToolBase, ToolHostHooks<TIn>, ToolOutputHooks<TOut>, ToolImageHooks<TOut> {
     type: 'mcp';
     input: z.ZodType<TIn>;
     output: z.ZodType<TOut>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@theoremjs/agents",
-  "version": "0.3.4-next.387.g8bea7ff",
+  "version": "0.3.4-next.401.gec4c582",
   "description": "A flat TypeScript agent kernel for typed profiles, deterministic turn execution, registered tools, provider adapters, guardrails, and host-injected traces.",
   "keywords": [
     "agent",