@theoremjs/agents 0.3.4-next.389.gecc7333 → 0.3.4-next.403.g5de27f3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -682,7 +682,7 @@ export const LABEL_NOTES = {
682
682
  'stop.cancelled': 'Status on a reply or message the person stopped.',
683
683
  'stop.interrupted': 'Status on a Live reply the person spoke over.',
684
684
  'stop.length': 'Status on a reply the model ended at its output limit.',
685
- 'stop.filtered': "Status on a reply the provider's content filter ended. A reply Theorem's own guardrails stopped shows its error instead.",
685
+ 'stop.filtered': "Status on a reply the provider's content filter ended. A reply Theorem's own guardrails stopped shows its error or refusal instead.",
686
686
  'stop.provider_error': 'Status on a reply the provider ended with an error and no error message.',
687
687
  'stop.stream_incomplete': 'Status on a reply whose stream ended before the model finished.',
688
688
  'stop.step_limit': 'Status on a turn that made its last allowed model call and got no answer.',
@@ -7,7 +7,7 @@ export declare const WORDED_STOP_KINDS: readonly ["cancelled", "interrupted", "l
7
7
  export type WordedStopKind = (typeof WORDED_STOP_KINDS)[number];
8
8
  /**
9
9
  * Why a turn ended short, in the person's words: its `stop.<kind>` label, the profile's own when it
10
- * sets one. `undefined` when the turn ended whole, or when an error in `blocks` (the turn's
11
- * transcript blocks) already says why.
10
+ * sets one. `undefined` when the turn ended whole, when Theorem's own guardrail stopped it, or when
11
+ * an error in `blocks` (the turn's transcript blocks) already says why.
12
12
  */
13
13
  export declare function stopText(stop: TurnStop | undefined, blocks: readonly TranscriptBlock[], labels?: LabelOverrides): string | undefined;
@@ -1,4 +1,5 @@
1
1
  import { labelText } from '../guardrails/labels.js';
2
+ import { isGuardrailStop } from '../kernel/stop.js';
2
3
  /** The stops that end a reply short of a whole answer. Each is worded by its `stop.<kind>` label. */
3
4
  export const WORDED_STOP_KINDS = [
4
5
  'cancelled',
@@ -15,11 +16,11 @@ function isWorded(kind) {
15
16
  }
16
17
  /**
17
18
  * Why a turn ended short, in the person's words: its `stop.<kind>` label, the profile's own when it
18
- * sets one. `undefined` when the turn ended whole, or when an error in `blocks` (the turn's
19
- * transcript blocks) already says why.
19
+ * sets one. `undefined` when the turn ended whole, when Theorem's own guardrail stopped it, or when
20
+ * an error in `blocks` (the turn's transcript blocks) already says why.
20
21
  */
21
22
  export function stopText(stop, blocks, labels) {
22
- if (!stop || !isWorded(stop.kind))
23
+ if (!stop || !isWorded(stop.kind) || isGuardrailStop(stop))
23
24
  return undefined;
24
25
  if (blocks.some((block) => block.kind === 'error'))
25
26
  return undefined;
@@ -10,6 +10,8 @@ export declare const CONTINUE_INSTRUCTION_TYPES: readonly ProfileType[];
10
10
  export declare const DEFAULT_ALLOW_CONTINUE: readonly ContinueStopKind[];
11
11
  /** Hosts wait briefly, then continue once on their own. */
12
12
  export declare const DEFAULT_AUTO_CONTINUE: readonly ContinueStopKind[];
13
+ /** Whether Theorem's own guardrail ended the turn, so its error or refusal already says why. */
14
+ export declare function isGuardrailStop(stop: TurnStop): boolean;
13
15
  /** Pause before an auto-continue, the host's or the turn's own, so a flaky tunnel can settle. */
14
16
  export declare const AUTO_CONTINUE_DELAY_MS = 1500;
15
17
  /** Which stops a profile offers Continue for, and which the host continues on its own. */
@@ -8,6 +8,17 @@ export const CONTINUE_INSTRUCTION_TYPES = ['text'];
8
8
  export const DEFAULT_ALLOW_CONTINUE = CONTINUE_STOP_KINDS;
9
9
  /** Hosts wait briefly, then continue once on their own. */
10
10
  export const DEFAULT_AUTO_CONTINUE = ['length', 'stream_incomplete'];
11
+ /** The `native` of a `filtered` stop that Theorem's own guardrail made, not the provider's filter. */
12
+ const GUARDRAIL_STOP_NATIVES = new Set([
13
+ 'egress',
14
+ 'canary',
15
+ 'prompt_echo',
16
+ 'provider_tool_leak',
17
+ ]);
18
+ /** Whether Theorem's own guardrail ended the turn, so its error or refusal already says why. */
19
+ export function isGuardrailStop(stop) {
20
+ return stop.kind === 'filtered' && GUARDRAIL_STOP_NATIVES.has(stop.native ?? '');
21
+ }
11
22
  /** Pause before an auto-continue, the host's or the turn's own, so a flaky tunnel can settle. */
12
23
  export const AUTO_CONTINUE_DELAY_MS = 1_500;
13
24
  const CONTINUE_KIND_SET = new Set(CONTINUE_STOP_KINDS);
@@ -14,7 +14,7 @@ import { AgentCallFailed, agentToolOrigin } from './agent.js';
14
14
  import { coordinateTool } from './coordinate.js';
15
15
  import { dispatchToolBody, failedActivity, failedActivityOf, failureEvent, messageOf, networkBlocked, networkBlockedEvent, requestChecks, sourceEvents, startToolExecution, toolCallArguments, toolEvent, } from './events.js';
16
16
  import { HandlerEvents } from './handler-events.js';
17
- import { fetchToolMedia } from './media-fetch.js';
17
+ import { fetchToolImages } from './image-fetch.js';
18
18
  import { formatToolFailureForModel, formatToolResult, frameToolFailure } from './model-text.js';
19
19
  import { operationUnavailable, operationWireValue } from './operation.js';
20
20
  import { checkPermission, gateDetails, isGateResumeDenied, isGateResumeGranted, isResumeContinuation, } from './permission.js';
@@ -276,8 +276,15 @@ async function* settleToolCall(args) {
276
276
  denied = true;
277
277
  }
278
278
  }
279
- if (!failure && sources)
280
- yield* sourceEvents(base, sources, outputRaw);
279
+ if (!failure) {
280
+ modelResult = yield* citeAndShow({
281
+ base,
282
+ sources,
283
+ images: args.images,
284
+ outputRaw,
285
+ modelResult,
286
+ });
287
+ }
281
288
  yield failure
282
289
  ? failureEvent(base, { ...failure, message: String(yield* guard.shown(failure.message)) }, formatToolResult(modelResult))
283
290
  : completeEvent(base, {
@@ -296,6 +303,35 @@ async function* settleToolCall(args) {
296
303
  ...stageSettlement(post),
297
304
  };
298
305
  }
306
+ /** The settled output's cited sources, then the pictures its tool names; the result carrying them. */
307
+ async function* citeAndShow(args) {
308
+ const { base, sources, images, outputRaw, modelResult } = args;
309
+ if (sources)
310
+ yield* sourceEvents(base, sources, outputRaw);
311
+ if (!images || outputRaw === undefined)
312
+ return modelResult;
313
+ return yield* withFetchedImages(base, modelResult, images, outputRaw);
314
+ }
315
+ /**
316
+ * The result with the pictures its tool names, fetched after the guard and the stages passed it.
317
+ * The addresses fetched are written into the result's text, so the egress rule on pictures the
318
+ * model shows counts them as given by a tool.
319
+ */
320
+ async function* withFetchedImages(base, result, images, outputRaw) {
321
+ const fetched = await images(outputRaw);
322
+ if (fetched.warning)
323
+ yield toolEvent(base, { phase: 'warning', warning: fetched.warning });
324
+ if (fetched.parts.length === 0)
325
+ return result;
326
+ const note = `\n[Pictures shown with this result: ${fetched.urls.join(' ')}]`; // labels-exempt: a marker the egress rule reads the addresses from, not prose (P2)
327
+ return {
328
+ ...result,
329
+ parts: [...(result.parts ?? []), ...fetched.parts],
330
+ ...(result.modelText !== undefined
331
+ ? { modelText: result.modelText + note }
332
+ : { finding: result.finding + note }),
333
+ };
334
+ }
299
335
  /** What the after-call stages add to a settlement: a turn to stop, input to hand the model. */
300
336
  function stageSettlement(post) {
301
337
  return {
@@ -475,6 +511,7 @@ agent, beginBody) {
475
511
  throwIfAborted(ctx.signal);
476
512
  const settleOutcome = (outcome) => settleBodyOutcome({
477
513
  outcome,
514
+ ctx,
478
515
  tool,
479
516
  base,
480
517
  callId,
@@ -666,21 +703,9 @@ function authorizeSavedDisclosure(args, tool, outcome) {
666
703
  if (guard.event?.action === 'block' || guard.event?.action === 'redact')
667
704
  throw blocked();
668
705
  }
669
- /** A successful remote call's result with the pictures its tool's `media` names, fetched and added beside any the server sent. */
670
- async function* withNamedMedia(tool, outcome, ctx, base) {
671
- if (outcome.kind !== 'ok' || !tool.media)
672
- return outcome;
673
- const fetched = await fetchToolMedia(tool.media, outcome.outputRaw, ctx);
674
- if (fetched.warning)
675
- yield toolEvent(base, { phase: 'warning', warning: fetched.warning });
676
- if (fetched.parts.length === 0)
677
- return outcome;
678
- const parts = [...(outcome.modelResult.parts ?? []), ...fetched.parts];
679
- return { ...outcome, modelResult: { ...outcome.modelResult, parts } };
680
- }
681
706
  /** Settles an outcome the kernel reached without a handler's own result: a remote body, or a sign-in. */
682
707
  async function* settleBodyOutcome(args) {
683
- const { outcome, tool, base, callId, safeInput, stages, guard, labels } = args;
708
+ const { outcome, tool, base, callId, safeInput, stages, guard, labels, ctx } = args;
684
709
  const name = tool.name;
685
710
  if (outcome.kind === 'gated') {
686
711
  if (outcome.gate.kind === 'confirmation') {
@@ -716,6 +741,9 @@ async function* settleBodyOutcome(args) {
716
741
  guard,
717
742
  provisional: { outputRaw: outcome.outputRaw, modelResult: outcome.modelResult },
718
743
  sources: tool.sources,
744
+ ...(tool.type !== 'function' && tool.images
745
+ ? { images: (output) => fetchToolImages(tool.images, output, ctx) }
746
+ : {}),
719
747
  labels: tool.labels,
720
748
  reproject: makeReproject((v) => parseToolOutput(tool.output, v),
721
749
  // why: A hook's edit replaces the value; the media the tool returned stays.
@@ -1113,9 +1141,9 @@ async function* settleByType(tools, tool, safeInput, fullCtx, base, snapshot, st
1113
1141
  const body = tool.type === 'http'
1114
1142
  ? yield* executeHttpTool(tool, safeInput, fullCtx, base, stages, beginBody, new HandlerEvents(base, guard.handlerEvents(stages?.drafts === true)))
1115
1143
  : yield* executeMcpTool(tool, safeInput, fullCtx, base, stages, tools.mcpSessions, beginBody);
1116
- const remoteOutcome = yield* withNamedMedia(tool, body, fullCtx, base);
1117
1144
  return yield* settleBodyOutcome({
1118
- outcome: remoteOutcome,
1145
+ outcome: body,
1146
+ ctx: fullCtx,
1119
1147
  tool,
1120
1148
  base,
1121
1149
  callId: base.callId,
@@ -0,0 +1,15 @@
1
+ import type { InteractionPart, ToolWarning } from '../turn-events.js';
2
+ import type { ToolContext } from './types.js';
3
+ /** The most pictures one call hands the model. */
4
+ export declare const MAX_TOOL_IMAGES = 4;
5
+ /**
6
+ * The pictures a tool's `images(output)` names, fetched through the profile's network policy (every
7
+ * hop, redirects included, no tool credentials) and handed to the model as image parts. A picture
8
+ * that is not an https address, blocked, slow, too big or not an image is left out and reported in
9
+ * one `images_unfetched` warning: the call itself stays whole. `urls` are the ones fetched.
10
+ */
11
+ export declare function fetchToolImages(images: (output: never) => string[], output: unknown, ctx: ToolContext): Promise<{
12
+ parts: InteractionPart[];
13
+ urls: string[];
14
+ warning?: ToolWarning;
15
+ }>;
@@ -0,0 +1,136 @@
1
+ import { fetchGuarded } from '../../guardrails/network.js';
2
+ import { bytesToBase64 } from '../util/base64.js';
3
+ import { toolNetworkPolicy } from './events.js';
4
+ /** The most pictures one call hands the model. */
5
+ export const MAX_TOOL_IMAGES = 4;
6
+ /** The largest picture fetched; a larger one is left out, not cut. */
7
+ const MAX_TOOL_IMAGE_BYTES = 5 * 1024 * 1024;
8
+ /** The most one call hands the model in all, so pictures cannot crowd a request past a provider's limit. */
9
+ const MAX_TOOL_IMAGES_TOTAL_BYTES = 10 * 1024 * 1024;
10
+ /** How long each picture may take, so a slow host cannot hold the call. */
11
+ const IMAGE_FETCH_TIMEOUT_MS = 15_000;
12
+ const IMAGE_TYPES = new Set(['image/png', 'image/jpeg', 'image/gif', 'image/webp']);
13
+ /** The body, read up to `maxBytes`; `undefined` when it runs past. */
14
+ async function readCappedBytes(response, maxBytes) {
15
+ const declared = Number(response.headers.get('content-length'));
16
+ if (Number.isFinite(declared) && declared > maxBytes) {
17
+ await response.body?.cancel().catch(() => { });
18
+ return undefined;
19
+ }
20
+ if (!response.body)
21
+ return new Uint8Array(await response.arrayBuffer());
22
+ const reader = response.body.getReader();
23
+ const chunks = [];
24
+ let total = 0;
25
+ for (;;) {
26
+ const { done, value } = await reader.read();
27
+ if (done)
28
+ break;
29
+ total += value.byteLength;
30
+ if (total > maxBytes) {
31
+ await reader.cancel().catch(() => { });
32
+ return undefined;
33
+ }
34
+ chunks.push(value);
35
+ }
36
+ const bytes = new Uint8Array(total);
37
+ let at = 0;
38
+ for (const chunk of chunks) {
39
+ bytes.set(chunk, at);
40
+ at += chunk.byteLength;
41
+ }
42
+ return bytes;
43
+ }
44
+ async function fetchOne(url, ctx) {
45
+ const timeout = AbortSignal.timeout(IMAGE_FETCH_TIMEOUT_MS);
46
+ const signal = ctx.signal ? AbortSignal.any([ctx.signal, timeout]) : timeout;
47
+ const response = await fetchGuarded(url, { method: 'GET', headers: { Accept: 'image/*' }, signal }, {
48
+ policy: toolNetworkPolicy(ctx),
49
+ followRedirects: true,
50
+ resolveHost: ctx.resolveHost,
51
+ });
52
+ if (!response.ok)
53
+ throw new Error(`HTTP ${response.status}`); // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
54
+ const mimeType = (response.headers.get('content-type') ?? '').split(';')[0]?.trim().toLowerCase();
55
+ if (!mimeType || !IMAGE_TYPES.has(mimeType)) {
56
+ await response.body?.cancel().catch(() => { });
57
+ throw new Error(`not a png, jpeg, gif or webp image (${mimeType || 'no type'})`); // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
58
+ }
59
+ const bytes = await readCappedBytes(response, MAX_TOOL_IMAGE_BYTES);
60
+ if (!bytes)
61
+ throw new Error(`larger than ${MAX_TOOL_IMAGE_BYTES} bytes`); // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
62
+ return { type: 'image', mimeType, data: bytesToBase64(bytes) };
63
+ }
64
+ /** `url` when it is an absolute https address; otherwise the reason it is not. */
65
+ function httpsAddress(url) {
66
+ if (typeof url !== 'string')
67
+ return { reason: 'an address that is not text' }; // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
68
+ try {
69
+ if (new URL(url).protocol === 'https:')
70
+ return { url };
71
+ }
72
+ catch {
73
+ // why: an unparseable address is reported below as a reason, not thrown.
74
+ }
75
+ return { reason: `${url.slice(0, 80)}: not an absolute https address` }; // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
76
+ }
77
+ /**
78
+ * The pictures a tool's `images(output)` names, fetched through the profile's network policy (every
79
+ * hop, redirects included, no tool credentials) and handed to the model as image parts. A picture
80
+ * that is not an https address, blocked, slow, too big or not an image is left out and reported in
81
+ * one `images_unfetched` warning: the call itself stays whole. `urls` are the ones fetched.
82
+ */
83
+ export async function fetchToolImages(images, output, ctx) {
84
+ const reasons = [];
85
+ const named = [];
86
+ try {
87
+ for (const raw of new Set(images(output))) {
88
+ const address = httpsAddress(raw);
89
+ if ('url' in address)
90
+ named.push(address.url);
91
+ else
92
+ reasons.push(address.reason);
93
+ }
94
+ }
95
+ catch (err) {
96
+ // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
97
+ return {
98
+ parts: [],
99
+ urls: [],
100
+ warning: unfetched([`the images hook threw: ${String(err)}`]), // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
101
+ };
102
+ }
103
+ const wanted = named.slice(0, MAX_TOOL_IMAGES);
104
+ const settled = await Promise.allSettled(wanted.map((url) => fetchOne(url, ctx)));
105
+ const parts = [];
106
+ const urls = [];
107
+ let total = 0;
108
+ settled.forEach((result, index) => {
109
+ const url = wanted[index];
110
+ if (result.status === 'rejected') {
111
+ const why = result.reason instanceof Error ? result.reason.message : String(result.reason);
112
+ reasons.push(`${new URL(url).hostname}: ${why}`);
113
+ return;
114
+ }
115
+ const size = 'data' in result.value ? result.value.data.length : 0;
116
+ if (total + size > MAX_TOOL_IMAGES_TOTAL_BYTES) {
117
+ reasons.push(`${new URL(url).hostname}: past the ${MAX_TOOL_IMAGES_TOTAL_BYTES} byte total`); // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
118
+ return;
119
+ }
120
+ total += size;
121
+ parts.push(result.value);
122
+ urls.push(url);
123
+ });
124
+ return {
125
+ parts,
126
+ urls,
127
+ ...(reasons.length ? { warning: unfetched(reasons) } : {}),
128
+ };
129
+ }
130
+ function unfetched(reasons) {
131
+ return {
132
+ code: 'images_unfetched',
133
+ severity: 'warning',
134
+ message: `Some pictures the tool named were left out. ${reasons.join('; ')}`, // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
135
+ };
136
+ }
@@ -161,13 +161,14 @@ export interface ToolOutputHooks<TOut = unknown> {
161
161
  sources?: (output: TOut) => Source[];
162
162
  }
163
163
  /** For a tool whose result names pictures by address; function tools return the bytes themselves as `parts`. */
164
- export interface ToolMediaHooks<TOut = unknown> {
164
+ export interface ToolImageHooks<TOut = unknown> {
165
165
  /**
166
- * The https addresses of pictures in `output` the model should see, at most `MAX_TOOL_MEDIA_FILES`
167
- * of them. The kernel fetches each one through the profile's network policy and hands the model
168
- * the image as a part; one it cannot fetch is a `media_unfetched` warning, never a failed call.
166
+ * The https addresses of pictures in `output` the model should see, at most `MAX_TOOL_IMAGES`
167
+ * of them. After the result passes the output guard, the kernel fetches each one through the
168
+ * profile's network policy, without the tool's credentials, and hands the model the image as a
169
+ * part; one it cannot fetch is an `images_unfetched` warning, never a failed call.
169
170
  */
170
- media?: (output: TOut) => string[];
171
+ images?: (output: TOut) => string[];
171
172
  }
172
173
  export interface FunctionToolDef<TIn = unknown, TOut = unknown> extends ToolBase, ToolHostHooks<TIn>, ToolOutputHooks<TOut> {
173
174
  type: 'function';
@@ -210,7 +211,7 @@ export interface ToolAuthConfig {
210
211
  clientId?: string;
211
212
  redirectUri?: string;
212
213
  }
213
- export interface HttpToolDef<TIn = unknown, TOut = unknown> extends ToolBase, ToolHostHooks<TIn>, ToolOutputHooks<TOut>, ToolMediaHooks<TOut> {
214
+ export interface HttpToolDef<TIn = unknown, TOut = unknown> extends ToolBase, ToolHostHooks<TIn>, ToolOutputHooks<TOut>, ToolImageHooks<TOut> {
214
215
  type: 'http';
215
216
  input: z.ZodType<TIn>;
216
217
  output: z.ZodType<TOut>;
@@ -237,7 +238,7 @@ export interface HttpToolLimit {
237
238
  requests: number;
238
239
  windowMs: number;
239
240
  }
240
- export interface McpToolDef<TIn = unknown, TOut = unknown> extends ToolBase, ToolHostHooks<TIn>, ToolOutputHooks<TOut>, ToolMediaHooks<TOut> {
241
+ export interface McpToolDef<TIn = unknown, TOut = unknown> extends ToolBase, ToolHostHooks<TIn>, ToolOutputHooks<TOut>, ToolImageHooks<TOut> {
241
242
  type: 'mcp';
242
243
  input: z.ZodType<TIn>;
243
244
  output: z.ZodType<TOut>;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@theoremjs/agents",
3
- "version": "0.3.4-next.389.gecc7333",
3
+ "version": "0.3.4-next.403.g5de27f3",
4
4
  "description": "A flat TypeScript agent kernel for typed profiles, deterministic turn execution, registered tools, provider adapters, guardrails, and host-injected traces.",
5
5
  "keywords": [
6
6
  "agent",
@@ -1,14 +0,0 @@
1
- import type { InteractionPart, ToolWarning } from '../turn-events.js';
2
- import type { ToolContext } from './types.js';
3
- /** The most pictures one call hands the model. */
4
- export declare const MAX_TOOL_MEDIA_FILES = 4;
5
- /**
6
- * The pictures a tool's `media(output)` names, fetched through the profile's network policy (every
7
- * hop, redirects included) and handed to the model as image parts. A picture that is blocked, slow,
8
- * too big or not an image is left out and reported in one `media_unfetched` warning: the call
9
- * itself stays whole.
10
- */
11
- export declare function fetchToolMedia(media: (output: never) => string[], output: unknown, ctx: ToolContext): Promise<{
12
- parts: InteractionPart[];
13
- warning?: ToolWarning;
14
- }>;
@@ -1,94 +0,0 @@
1
- import { fetchGuarded } from '../../guardrails/network.js';
2
- import { bytesToBase64 } from '../util/base64.js';
3
- import { toolNetworkPolicy } from './events.js';
4
- /** The most pictures one call hands the model. */
5
- export const MAX_TOOL_MEDIA_FILES = 4;
6
- /** The largest picture fetched; a larger one is left out, not cut. */
7
- const MAX_TOOL_MEDIA_BYTES = 5 * 1024 * 1024;
8
- /** How long each picture may take, so a slow host cannot hold the call. */
9
- const MEDIA_FETCH_TIMEOUT_MS = 15_000;
10
- const IMAGE_TYPES = new Set(['image/png', 'image/jpeg', 'image/gif', 'image/webp']);
11
- /** The body, read up to `maxBytes`; `undefined` when it runs past. */
12
- async function readCappedBytes(response, maxBytes) {
13
- const declared = Number(response.headers.get('content-length'));
14
- if (Number.isFinite(declared) && declared > maxBytes) {
15
- await response.body?.cancel().catch(() => { });
16
- return undefined;
17
- }
18
- if (!response.body)
19
- return new Uint8Array(await response.arrayBuffer());
20
- const reader = response.body.getReader();
21
- const chunks = [];
22
- let total = 0;
23
- for (;;) {
24
- const { done, value } = await reader.read();
25
- if (done)
26
- break;
27
- total += value.byteLength;
28
- if (total > maxBytes) {
29
- await reader.cancel().catch(() => { });
30
- return undefined;
31
- }
32
- chunks.push(value);
33
- }
34
- const bytes = new Uint8Array(total);
35
- let at = 0;
36
- for (const chunk of chunks) {
37
- bytes.set(chunk, at);
38
- at += chunk.byteLength;
39
- }
40
- return bytes;
41
- }
42
- async function fetchOne(url, ctx) {
43
- const timeout = AbortSignal.timeout(MEDIA_FETCH_TIMEOUT_MS);
44
- const signal = ctx.signal ? AbortSignal.any([ctx.signal, timeout]) : timeout;
45
- const response = await fetchGuarded(url, { method: 'GET', headers: { Accept: 'image/*' }, signal }, {
46
- policy: toolNetworkPolicy(ctx),
47
- followRedirects: true,
48
- resolveHost: ctx.resolveHost,
49
- });
50
- if (!response.ok)
51
- throw new Error(`HTTP ${response.status}`); // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
52
- const mimeType = (response.headers.get('content-type') ?? '').split(';')[0]?.trim().toLowerCase();
53
- if (!mimeType || !IMAGE_TYPES.has(mimeType)) {
54
- await response.body?.cancel().catch(() => { });
55
- throw new Error(`not a png, jpeg, gif or webp image (${mimeType || 'no type'})`); // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
56
- }
57
- const bytes = await readCappedBytes(response, MAX_TOOL_MEDIA_BYTES);
58
- if (!bytes)
59
- throw new Error(`larger than ${MAX_TOOL_MEDIA_BYTES} bytes`); // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
60
- return { type: 'image', mimeType, data: bytesToBase64(bytes) };
61
- }
62
- /**
63
- * The pictures a tool's `media(output)` names, fetched through the profile's network policy (every
64
- * hop, redirects included) and handed to the model as image parts. A picture that is blocked, slow,
65
- * too big or not an image is left out and reported in one `media_unfetched` warning: the call
66
- * itself stays whole.
67
- */
68
- export async function fetchToolMedia(media, output, ctx) {
69
- let urls;
70
- try {
71
- urls = [...new Set(media(output))].slice(0, MAX_TOOL_MEDIA_FILES);
72
- }
73
- catch (err) {
74
- // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
75
- return { parts: [], warning: unfetched([`the media hook threw: ${String(err)}`]) };
76
- }
77
- const settled = await Promise.allSettled(urls.map((url) => fetchOne(url, ctx)));
78
- const parts = [];
79
- const reasons = [];
80
- settled.forEach((result, index) => {
81
- if (result.status === 'fulfilled')
82
- parts.push(result.value);
83
- else
84
- reasons.push(`${new URL(urls[index] ?? 'https://invalid.invalid').hostname}: ${String(result.reason instanceof Error ? result.reason.message : result.reason)}`);
85
- });
86
- return { parts, ...(reasons.length ? { warning: unfetched(reasons) } : {}) };
87
- }
88
- function unfetched(reasons) {
89
- return {
90
- code: 'media_unfetched',
91
- severity: 'warning',
92
- message: `Some pictures the tool named were left out. ${reasons.join('; ')}`, // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
93
- };
94
- }