@theoremjs/agents 0.3.4-next.389.gecc7333 → 0.3.4-next.401.gec4c582

package/esm/src/kernel/tools/execute.js

@@ -14,7 +14,7 @@ import { AgentCallFailed, agentToolOrigin } from './agent.js';
 import { coordinateTool } from './coordinate.js';
 import { dispatchToolBody, failedActivity, failedActivityOf, failureEvent, messageOf, networkBlocked, networkBlockedEvent, requestChecks, sourceEvents, startToolExecution, toolCallArguments, toolEvent, } from './events.js';
 import { HandlerEvents } from './handler-events.js';
-import { fetchToolMedia } from './media-fetch.js';
+import { fetchToolImages } from './image-fetch.js';
 import { formatToolFailureForModel, formatToolResult, frameToolFailure } from './model-text.js';
 import { operationUnavailable, operationWireValue } from './operation.js';
 import { checkPermission, gateDetails, isGateResumeDenied, isGateResumeGranted, isResumeContinuation, } from './permission.js';
@@ -276,8 +276,15 @@ async function* settleToolCall(args) {
                 denied = true;
         }
     }
-    if (!failure && sources)
-        yield* sourceEvents(base, sources, outputRaw);
+    if (!failure) {
+        modelResult = yield* citeAndShow({
+            base,
+            sources,
+            images: args.images,
+            outputRaw,
+            modelResult,
+        });
+    }
     yield failure
         ? failureEvent(base, { ...failure, message: String(yield* guard.shown(failure.message)) }, formatToolResult(modelResult))
         : completeEvent(base, {
@@ -296,6 +303,35 @@ async function* settleToolCall(args) {
         ...stageSettlement(post),
     };
 }
+/** The settled output's cited sources, then the pictures its tool names; the result carrying them. */
+async function* citeAndShow(args) {
+    const { base, sources, images, outputRaw, modelResult } = args;
+    if (sources)
+        yield* sourceEvents(base, sources, outputRaw);
+    if (!images || outputRaw === undefined)
+        return modelResult;
+    return yield* withFetchedImages(base, modelResult, images, outputRaw);
+}
+/**
+ * The result with the pictures its tool names, fetched after the guard and the stages passed it.
+ * The addresses fetched are written into the result's text, so the egress rule on pictures the
+ * model shows counts them as given by a tool.
+ */
+async function* withFetchedImages(base, result, images, outputRaw) {
+    const fetched = await images(outputRaw);
+    if (fetched.warning)
+        yield toolEvent(base, { phase: 'warning', warning: fetched.warning });
+    if (fetched.parts.length === 0)
+        return result;
+    const note = `\n[Pictures shown with this result: ${fetched.urls.join(' ')}]`; // labels-exempt: a marker the egress rule reads the addresses from, not prose (P2)
+    return {
+        ...result,
+        parts: [...(result.parts ?? []), ...fetched.parts],
+        ...(result.modelText !== undefined
+            ? { modelText: result.modelText + note }
+            : { finding: result.finding + note }),
+    };
+}
 /** What the after-call stages add to a settlement: a turn to stop, input to hand the model. */
 function stageSettlement(post) {
     return {
@@ -475,6 +511,7 @@ agent, beginBody) {
     throwIfAborted(ctx.signal);
     const settleOutcome = (outcome) => settleBodyOutcome({
         outcome,
+        ctx,
         tool,
         base,
         callId,
@@ -666,21 +703,9 @@ function authorizeSavedDisclosure(args, tool, outcome) {
     if (guard.event?.action === 'block' || guard.event?.action === 'redact')
         throw blocked();
 }
-/** A successful remote call's result with the pictures its tool's `media` names, fetched and added beside any the server sent. */
-async function* withNamedMedia(tool, outcome, ctx, base) {
-    if (outcome.kind !== 'ok' || !tool.media)
-        return outcome;
-    const fetched = await fetchToolMedia(tool.media, outcome.outputRaw, ctx);
-    if (fetched.warning)
-        yield toolEvent(base, { phase: 'warning', warning: fetched.warning });
-    if (fetched.parts.length === 0)
-        return outcome;
-    const parts = [...(outcome.modelResult.parts ?? []), ...fetched.parts];
-    return { ...outcome, modelResult: { ...outcome.modelResult, parts } };
-}
 /** Settles an outcome the kernel reached without a handler's own result: a remote body, or a sign-in. */
 async function* settleBodyOutcome(args) {
-    const { outcome, tool, base, callId, safeInput, stages, guard, labels } = args;
+    const { outcome, tool, base, callId, safeInput, stages, guard, labels, ctx } = args;
     const name = tool.name;
     if (outcome.kind === 'gated') {
         if (outcome.gate.kind === 'confirmation') {
@@ -716,6 +741,9 @@ async function* settleBodyOutcome(args) {
         guard,
         provisional: { outputRaw: outcome.outputRaw, modelResult: outcome.modelResult },
         sources: tool.sources,
+        ...(tool.type !== 'function' && tool.images
+            ? { images: (output) => fetchToolImages(tool.images, output, ctx) }
+            : {}),
         labels: tool.labels,
         reproject: makeReproject((v) => parseToolOutput(tool.output, v), 
         // why: A hook's edit replaces the value; the media the tool returned stays.
@@ -1113,9 +1141,9 @@ async function* settleByType(tools, tool, safeInput, fullCtx, base, snapshot, st
     const body = tool.type === 'http'
         ? yield* executeHttpTool(tool, safeInput, fullCtx, base, stages, beginBody, new HandlerEvents(base, guard.handlerEvents(stages?.drafts === true)))
         : yield* executeMcpTool(tool, safeInput, fullCtx, base, stages, tools.mcpSessions, beginBody);
-    const remoteOutcome = yield* withNamedMedia(tool, body, fullCtx, base);
     return yield* settleBodyOutcome({
-        outcome: remoteOutcome,
+        outcome: body,
+        ctx: fullCtx,
         tool,
         base,
         callId: base.callId,

package/esm/src/kernel/tools/image-fetch.d.ts

@@ -0,0 +1,15 @@
+import type { InteractionPart, ToolWarning } from '../turn-events.js';
+import type { ToolContext } from './types.js';
+/** The most pictures one call hands the model. */
+export declare const MAX_TOOL_IMAGES = 4;
+/**
+ * The pictures a tool's `images(output)` names, fetched through the profile's network policy (every
+ * hop, redirects included, no tool credentials) and handed to the model as image parts. A picture
+ * that is not an https address, blocked, slow, too big or not an image is left out and reported in
+ * one `images_unfetched` warning: the call itself stays whole. `urls` are the ones fetched.
+ */
+export declare function fetchToolImages(images: (output: never) => string[], output: unknown, ctx: ToolContext): Promise<{
+    parts: InteractionPart[];
+    urls: string[];
+    warning?: ToolWarning;
+}>;

package/esm/src/kernel/tools/image-fetch.js

@@ -0,0 +1,136 @@
+import { fetchGuarded } from '../../guardrails/network.js';
+import { bytesToBase64 } from '../util/base64.js';
+import { toolNetworkPolicy } from './events.js';
+/** The most pictures one call hands the model. */
+export const MAX_TOOL_IMAGES = 4;
+/** The largest picture fetched; a larger one is left out, not cut. */
+const MAX_TOOL_IMAGE_BYTES = 5 * 1024 * 1024;
+/** The most one call hands the model in all, so pictures cannot crowd a request past a provider's limit. */
+const MAX_TOOL_IMAGES_TOTAL_BYTES = 10 * 1024 * 1024;
+/** How long each picture may take, so a slow host cannot hold the call. */
+const IMAGE_FETCH_TIMEOUT_MS = 15_000;
+const IMAGE_TYPES = new Set(['image/png', 'image/jpeg', 'image/gif', 'image/webp']);
+/** The body, read up to `maxBytes`; `undefined` when it runs past. */
+async function readCappedBytes(response, maxBytes) {
+    const declared = Number(response.headers.get('content-length'));
+    if (Number.isFinite(declared) && declared > maxBytes) {
+        await response.body?.cancel().catch(() => { });
+        return undefined;
+    }
+    if (!response.body)
+        return new Uint8Array(await response.arrayBuffer());
+    const reader = response.body.getReader();
+    const chunks = [];
+    let total = 0;
+    for (;;) {
+        const { done, value } = await reader.read();
+        if (done)
+            break;
+        total += value.byteLength;
+        if (total > maxBytes) {
+            await reader.cancel().catch(() => { });
+            return undefined;
+        }
+        chunks.push(value);
+    }
+    const bytes = new Uint8Array(total);
+    let at = 0;
+    for (const chunk of chunks) {
+        bytes.set(chunk, at);
+        at += chunk.byteLength;
+    }
+    return bytes;
+}
+async function fetchOne(url, ctx) {
+    const timeout = AbortSignal.timeout(IMAGE_FETCH_TIMEOUT_MS);
+    const signal = ctx.signal ? AbortSignal.any([ctx.signal, timeout]) : timeout;
+    const response = await fetchGuarded(url, { method: 'GET', headers: { Accept: 'image/*' }, signal }, {
+        policy: toolNetworkPolicy(ctx),
+        followRedirects: true,
+        resolveHost: ctx.resolveHost,
+    });
+    if (!response.ok)
+        throw new Error(`HTTP ${response.status}`); // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
+    const mimeType = (response.headers.get('content-type') ?? '').split(';')[0]?.trim().toLowerCase();
+    if (!mimeType || !IMAGE_TYPES.has(mimeType)) {
+        await response.body?.cancel().catch(() => { });
+        throw new Error(`not a png, jpeg, gif or webp image (${mimeType || 'no type'})`); // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
+    }
+    const bytes = await readCappedBytes(response, MAX_TOOL_IMAGE_BYTES);
+    if (!bytes)
+        throw new Error(`larger than ${MAX_TOOL_IMAGE_BYTES} bytes`); // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
+    return { type: 'image', mimeType, data: bytesToBase64(bytes) };
+}
+/** `url` when it is an absolute https address; otherwise the reason it is not. */
+function httpsAddress(url) {
+    if (typeof url !== 'string')
+        return { reason: 'an address that is not text' }; // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
+    try {
+        if (new URL(url).protocol === 'https:')
+            return { url };
+    }
+    catch {
+        // why: an unparseable address is reported below as a reason, not thrown.
+    }
+    return { reason: `${url.slice(0, 80)}: not an absolute https address` }; // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
+}
+/**
+ * The pictures a tool's `images(output)` names, fetched through the profile's network policy (every
+ * hop, redirects included, no tool credentials) and handed to the model as image parts. A picture
+ * that is not an https address, blocked, slow, too big or not an image is left out and reported in
+ * one `images_unfetched` warning: the call itself stays whole. `urls` are the ones fetched.
+ */
+export async function fetchToolImages(images, output, ctx) {
+    const reasons = [];
+    const named = [];
+    try {
+        for (const raw of new Set(images(output))) {
+            const address = httpsAddress(raw);
+            if ('url' in address)
+                named.push(address.url);
+            else
+                reasons.push(address.reason);
+        }
+    }
+    catch (err) {
+        // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
+        return {
+            parts: [],
+            urls: [],
+            warning: unfetched([`the images hook threw: ${String(err)}`]), // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
+        };
+    }
+    const wanted = named.slice(0, MAX_TOOL_IMAGES);
+    const settled = await Promise.allSettled(wanted.map((url) => fetchOne(url, ctx)));
+    const parts = [];
+    const urls = [];
+    let total = 0;
+    settled.forEach((result, index) => {
+        const url = wanted[index];
+        if (result.status === 'rejected') {
+            const why = result.reason instanceof Error ? result.reason.message : String(result.reason);
+            reasons.push(`${new URL(url).hostname}: ${why}`);
+            return;
+        }
+        const size = 'data' in result.value ? result.value.data.length : 0;
+        if (total + size > MAX_TOOL_IMAGES_TOTAL_BYTES) {
+            reasons.push(`${new URL(url).hostname}: past the ${MAX_TOOL_IMAGES_TOTAL_BYTES} byte total`); // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
+            return;
+        }
+        total += size;
+        parts.push(result.value);
+        urls.push(url);
+    });
+    return {
+        parts,
+        urls,
+        ...(reasons.length ? { warning: unfetched(reasons) } : {}),
+    };
+}
+function unfetched(reasons) {
+    return {
+        code: 'images_unfetched',
+        severity: 'warning',
+        message: `Some pictures the tool named were left out. ${reasons.join('; ')}`, // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
+    };
+}

package/esm/src/kernel/tools/types.d.ts

@@ -161,13 +161,14 @@ export interface ToolOutputHooks<TOut = unknown> {
     sources?: (output: TOut) => Source[];
 }
 /** For a tool whose result names pictures by address; function tools return the bytes themselves as `parts`. */
-export interface ToolMediaHooks<TOut = unknown> {
+export interface ToolImageHooks<TOut = unknown> {
     /**
-     * The https addresses of pictures in `output` the model should see, at most `MAX_TOOL_MEDIA_FILES`
-     * of them. The kernel fetches each one through the profile's network policy and hands the model
-     * the image as a part; one it cannot fetch is a `media_unfetched` warning, never a failed call.
+     * The https addresses of pictures in `output` the model should see, at most `MAX_TOOL_IMAGES`
+     * of them. After the result passes the output guard, the kernel fetches each one through the
+     * profile's network policy, without the tool's credentials, and hands the model the image as a
+     * part; one it cannot fetch is an `images_unfetched` warning, never a failed call.
      */
-    media?: (output: TOut) => string[];
+    images?: (output: TOut) => string[];
 }
 export interface FunctionToolDef<TIn = unknown, TOut = unknown> extends ToolBase, ToolHostHooks<TIn>, ToolOutputHooks<TOut> {
     type: 'function';
@@ -210,7 +211,7 @@ export interface ToolAuthConfig {
     clientId?: string;
     redirectUri?: string;
 }
-export interface HttpToolDef<TIn = unknown, TOut = unknown> extends ToolBase, ToolHostHooks<TIn>, ToolOutputHooks<TOut>, ToolMediaHooks<TOut> {
+export interface HttpToolDef<TIn = unknown, TOut = unknown> extends ToolBase, ToolHostHooks<TIn>, ToolOutputHooks<TOut>, ToolImageHooks<TOut> {
     type: 'http';
     input: z.ZodType<TIn>;
     output: z.ZodType<TOut>;
@@ -237,7 +238,7 @@ export interface HttpToolLimit {
     requests: number;
     windowMs: number;
 }
-export interface McpToolDef<TIn = unknown, TOut = unknown> extends ToolBase, ToolHostHooks<TIn>, ToolOutputHooks<TOut>, ToolMediaHooks<TOut> {
+export interface McpToolDef<TIn = unknown, TOut = unknown> extends ToolBase, ToolHostHooks<TIn>, ToolOutputHooks<TOut>, ToolImageHooks<TOut> {
     type: 'mcp';
     input: z.ZodType<TIn>;
     output: z.ZodType<TOut>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@theoremjs/agents",
-  "version": "0.3.4-next.389.gecc7333",
+  "version": "0.3.4-next.401.gec4c582",
   "description": "A flat TypeScript agent kernel for typed profiles, deterministic turn execution, registered tools, provider adapters, guardrails, and host-injected traces.",
   "keywords": [
     "agent",

package/esm/src/kernel/tools/media-fetch.d.ts

@@ -1,14 +0,0 @@
-import type { InteractionPart, ToolWarning } from '../turn-events.js';
-import type { ToolContext } from './types.js';
-/** The most pictures one call hands the model. */
-export declare const MAX_TOOL_MEDIA_FILES = 4;
-/**
- * The pictures a tool's `media(output)` names, fetched through the profile's network policy (every
- * hop, redirects included) and handed to the model as image parts. A picture that is blocked, slow,
- * too big or not an image is left out and reported in one `media_unfetched` warning: the call
- * itself stays whole.
- */
-export declare function fetchToolMedia(media: (output: never) => string[], output: unknown, ctx: ToolContext): Promise<{
-    parts: InteractionPart[];
-    warning?: ToolWarning;
-}>;

package/esm/src/kernel/tools/media-fetch.js

@@ -1,94 +0,0 @@
-import { fetchGuarded } from '../../guardrails/network.js';
-import { bytesToBase64 } from '../util/base64.js';
-import { toolNetworkPolicy } from './events.js';
-/** The most pictures one call hands the model. */
-export const MAX_TOOL_MEDIA_FILES = 4;
-/** The largest picture fetched; a larger one is left out, not cut. */
-const MAX_TOOL_MEDIA_BYTES = 5 * 1024 * 1024;
-/** How long each picture may take, so a slow host cannot hold the call. */
-const MEDIA_FETCH_TIMEOUT_MS = 15_000;
-const IMAGE_TYPES = new Set(['image/png', 'image/jpeg', 'image/gif', 'image/webp']);
-/** The body, read up to `maxBytes`; `undefined` when it runs past. */
-async function readCappedBytes(response, maxBytes) {
-    const declared = Number(response.headers.get('content-length'));
-    if (Number.isFinite(declared) && declared > maxBytes) {
-        await response.body?.cancel().catch(() => { });
-        return undefined;
-    }
-    if (!response.body)
-        return new Uint8Array(await response.arrayBuffer());
-    const reader = response.body.getReader();
-    const chunks = [];
-    let total = 0;
-    for (;;) {
-        const { done, value } = await reader.read();
-        if (done)
-            break;
-        total += value.byteLength;
-        if (total > maxBytes) {
-            await reader.cancel().catch(() => { });
-            return undefined;
-        }
-        chunks.push(value);
-    }
-    const bytes = new Uint8Array(total);
-    let at = 0;
-    for (const chunk of chunks) {
-        bytes.set(chunk, at);
-        at += chunk.byteLength;
-    }
-    return bytes;
-}
-async function fetchOne(url, ctx) {
-    const timeout = AbortSignal.timeout(MEDIA_FETCH_TIMEOUT_MS);
-    const signal = ctx.signal ? AbortSignal.any([ctx.signal, timeout]) : timeout;
-    const response = await fetchGuarded(url, { method: 'GET', headers: { Accept: 'image/*' }, signal }, {
-        policy: toolNetworkPolicy(ctx),
-        followRedirects: true,
-        resolveHost: ctx.resolveHost,
-    });
-    if (!response.ok)
-        throw new Error(`HTTP ${response.status}`); // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
-    const mimeType = (response.headers.get('content-type') ?? '').split(';')[0]?.trim().toLowerCase();
-    if (!mimeType || !IMAGE_TYPES.has(mimeType)) {
-        await response.body?.cancel().catch(() => { });
-        throw new Error(`not a png, jpeg, gif or webp image (${mimeType || 'no type'})`); // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
-    }
-    const bytes = await readCappedBytes(response, MAX_TOOL_MEDIA_BYTES);
-    if (!bytes)
-        throw new Error(`larger than ${MAX_TOOL_MEDIA_BYTES} bytes`); // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
-    return { type: 'image', mimeType, data: bytesToBase64(bytes) };
-}
-/**
- * The pictures a tool's `media(output)` names, fetched through the profile's network policy (every
- * hop, redirects included) and handed to the model as image parts. A picture that is blocked, slow,
- * too big or not an image is left out and reported in one `media_unfetched` warning: the call
- * itself stays whole.
- */
-export async function fetchToolMedia(media, output, ctx) {
-    let urls;
-    try {
-        urls = [...new Set(media(output))].slice(0, MAX_TOOL_MEDIA_FILES);
-    }
-    catch (err) {
-        // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
-        return { parts: [], warning: unfetched([`the media hook threw: ${String(err)}`]) };
-    }
-    const settled = await Promise.allSettled(urls.map((url) => fetchOne(url, ctx)));
-    const parts = [];
-    const reasons = [];
-    settled.forEach((result, index) => {
-        if (result.status === 'fulfilled')
-            parts.push(result.value);
-        else
-            reasons.push(`${new URL(urls[index] ?? 'https://invalid.invalid').hostname}: ${String(result.reason instanceof Error ? result.reason.message : result.reason)}`);
-    });
-    return { parts, ...(reasons.length ? { warning: unfetched(reasons) } : {}) };
-}
-function unfetched(reasons) {
-    return {
-        code: 'media_unfetched',
-        severity: 'warning',
-        message: `Some pictures the tool named were left out. ${reasons.join('; ')}`, // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
-    };
-}