@theoremjs/agents 0.3.4-next.389.gecc7333 → 0.3.4-next.403.g5de27f3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm/src/guardrails/labels.js +1 -1
- package/esm/src/interface/stop.d.ts +2 -2
- package/esm/src/interface/stop.js +4 -3
- package/esm/src/kernel/stop.d.ts +2 -0
- package/esm/src/kernel/stop.js +11 -0
- package/esm/src/kernel/tools/execute.js +46 -18
- package/esm/src/kernel/tools/image-fetch.d.ts +15 -0
- package/esm/src/kernel/tools/image-fetch.js +136 -0
- package/esm/src/kernel/tools/types.d.ts +8 -7
- package/package.json +1 -1
- package/esm/src/kernel/tools/media-fetch.d.ts +0 -14
- package/esm/src/kernel/tools/media-fetch.js +0 -94
|
@@ -682,7 +682,7 @@ export const LABEL_NOTES = {
|
|
|
682
682
|
'stop.cancelled': 'Status on a reply or message the person stopped.',
|
|
683
683
|
'stop.interrupted': 'Status on a Live reply the person spoke over.',
|
|
684
684
|
'stop.length': 'Status on a reply the model ended at its output limit.',
|
|
685
|
-
'stop.filtered': "Status on a reply the provider's content filter ended. A reply Theorem's own guardrails stopped shows its error instead.",
|
|
685
|
+
'stop.filtered': "Status on a reply the provider's content filter ended. A reply Theorem's own guardrails stopped shows its error or refusal instead.",
|
|
686
686
|
'stop.provider_error': 'Status on a reply the provider ended with an error and no error message.',
|
|
687
687
|
'stop.stream_incomplete': 'Status on a reply whose stream ended before the model finished.',
|
|
688
688
|
'stop.step_limit': 'Status on a turn that made its last allowed model call and got no answer.',
|
|
@@ -7,7 +7,7 @@ export declare const WORDED_STOP_KINDS: readonly ["cancelled", "interrupted", "l
|
|
|
7
7
|
export type WordedStopKind = (typeof WORDED_STOP_KINDS)[number];
|
|
8
8
|
/**
|
|
9
9
|
* Why a turn ended short, in the person's words: its `stop.<kind>` label, the profile's own when it
|
|
10
|
-
* sets one. `undefined` when the turn ended whole,
|
|
11
|
-
* transcript blocks) already says why.
|
|
10
|
+
* sets one. `undefined` when the turn ended whole, when Theorem's own guardrail stopped it, or when
|
|
11
|
+
* an error in `blocks` (the turn's transcript blocks) already says why.
|
|
12
12
|
*/
|
|
13
13
|
export declare function stopText(stop: TurnStop | undefined, blocks: readonly TranscriptBlock[], labels?: LabelOverrides): string | undefined;
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { labelText } from '../guardrails/labels.js';
|
|
2
|
+
import { isGuardrailStop } from '../kernel/stop.js';
|
|
2
3
|
/** The stops that end a reply short of a whole answer. Each is worded by its `stop.<kind>` label. */
|
|
3
4
|
export const WORDED_STOP_KINDS = [
|
|
4
5
|
'cancelled',
|
|
@@ -15,11 +16,11 @@ function isWorded(kind) {
|
|
|
15
16
|
}
|
|
16
17
|
/**
|
|
17
18
|
* Why a turn ended short, in the person's words: its `stop.<kind>` label, the profile's own when it
|
|
18
|
-
* sets one. `undefined` when the turn ended whole,
|
|
19
|
-
* transcript blocks) already says why.
|
|
19
|
+
* sets one. `undefined` when the turn ended whole, when Theorem's own guardrail stopped it, or when
|
|
20
|
+
* an error in `blocks` (the turn's transcript blocks) already says why.
|
|
20
21
|
*/
|
|
21
22
|
export function stopText(stop, blocks, labels) {
|
|
22
|
-
if (!stop || !isWorded(stop.kind))
|
|
23
|
+
if (!stop || !isWorded(stop.kind) || isGuardrailStop(stop))
|
|
23
24
|
return undefined;
|
|
24
25
|
if (blocks.some((block) => block.kind === 'error'))
|
|
25
26
|
return undefined;
|
package/esm/src/kernel/stop.d.ts
CHANGED
|
@@ -10,6 +10,8 @@ export declare const CONTINUE_INSTRUCTION_TYPES: readonly ProfileType[];
|
|
|
10
10
|
export declare const DEFAULT_ALLOW_CONTINUE: readonly ContinueStopKind[];
|
|
11
11
|
/** Hosts wait briefly, then continue once on their own. */
|
|
12
12
|
export declare const DEFAULT_AUTO_CONTINUE: readonly ContinueStopKind[];
|
|
13
|
+
/** Whether Theorem's own guardrail ended the turn, so its error or refusal already says why. */
|
|
14
|
+
export declare function isGuardrailStop(stop: TurnStop): boolean;
|
|
13
15
|
/** Pause before an auto-continue, the host's or the turn's own, so a flaky tunnel can settle. */
|
|
14
16
|
export declare const AUTO_CONTINUE_DELAY_MS = 1500;
|
|
15
17
|
/** Which stops a profile offers Continue for, and which the host continues on its own. */
|
package/esm/src/kernel/stop.js
CHANGED
|
@@ -8,6 +8,17 @@ export const CONTINUE_INSTRUCTION_TYPES = ['text'];
|
|
|
8
8
|
export const DEFAULT_ALLOW_CONTINUE = CONTINUE_STOP_KINDS;
|
|
9
9
|
/** Hosts wait briefly, then continue once on their own. */
|
|
10
10
|
export const DEFAULT_AUTO_CONTINUE = ['length', 'stream_incomplete'];
|
|
11
|
+
/** The `native` of a `filtered` stop that Theorem's own guardrail made, not the provider's filter. */
|
|
12
|
+
const GUARDRAIL_STOP_NATIVES = new Set([
|
|
13
|
+
'egress',
|
|
14
|
+
'canary',
|
|
15
|
+
'prompt_echo',
|
|
16
|
+
'provider_tool_leak',
|
|
17
|
+
]);
|
|
18
|
+
/** Whether Theorem's own guardrail ended the turn, so its error or refusal already says why. */
|
|
19
|
+
export function isGuardrailStop(stop) {
|
|
20
|
+
return stop.kind === 'filtered' && GUARDRAIL_STOP_NATIVES.has(stop.native ?? '');
|
|
21
|
+
}
|
|
11
22
|
/** Pause before an auto-continue, the host's or the turn's own, so a flaky tunnel can settle. */
|
|
12
23
|
export const AUTO_CONTINUE_DELAY_MS = 1_500;
|
|
13
24
|
const CONTINUE_KIND_SET = new Set(CONTINUE_STOP_KINDS);
|
|
@@ -14,7 +14,7 @@ import { AgentCallFailed, agentToolOrigin } from './agent.js';
|
|
|
14
14
|
import { coordinateTool } from './coordinate.js';
|
|
15
15
|
import { dispatchToolBody, failedActivity, failedActivityOf, failureEvent, messageOf, networkBlocked, networkBlockedEvent, requestChecks, sourceEvents, startToolExecution, toolCallArguments, toolEvent, } from './events.js';
|
|
16
16
|
import { HandlerEvents } from './handler-events.js';
|
|
17
|
-
import {
|
|
17
|
+
import { fetchToolImages } from './image-fetch.js';
|
|
18
18
|
import { formatToolFailureForModel, formatToolResult, frameToolFailure } from './model-text.js';
|
|
19
19
|
import { operationUnavailable, operationWireValue } from './operation.js';
|
|
20
20
|
import { checkPermission, gateDetails, isGateResumeDenied, isGateResumeGranted, isResumeContinuation, } from './permission.js';
|
|
@@ -276,8 +276,15 @@ async function* settleToolCall(args) {
|
|
|
276
276
|
denied = true;
|
|
277
277
|
}
|
|
278
278
|
}
|
|
279
|
-
if (!failure
|
|
280
|
-
yield*
|
|
279
|
+
if (!failure) {
|
|
280
|
+
modelResult = yield* citeAndShow({
|
|
281
|
+
base,
|
|
282
|
+
sources,
|
|
283
|
+
images: args.images,
|
|
284
|
+
outputRaw,
|
|
285
|
+
modelResult,
|
|
286
|
+
});
|
|
287
|
+
}
|
|
281
288
|
yield failure
|
|
282
289
|
? failureEvent(base, { ...failure, message: String(yield* guard.shown(failure.message)) }, formatToolResult(modelResult))
|
|
283
290
|
: completeEvent(base, {
|
|
@@ -296,6 +303,35 @@ async function* settleToolCall(args) {
|
|
|
296
303
|
...stageSettlement(post),
|
|
297
304
|
};
|
|
298
305
|
}
|
|
306
|
+
/** The settled output's cited sources, then the pictures its tool names; the result carrying them. */
|
|
307
|
+
async function* citeAndShow(args) {
|
|
308
|
+
const { base, sources, images, outputRaw, modelResult } = args;
|
|
309
|
+
if (sources)
|
|
310
|
+
yield* sourceEvents(base, sources, outputRaw);
|
|
311
|
+
if (!images || outputRaw === undefined)
|
|
312
|
+
return modelResult;
|
|
313
|
+
return yield* withFetchedImages(base, modelResult, images, outputRaw);
|
|
314
|
+
}
|
|
315
|
+
/**
|
|
316
|
+
* The result with the pictures its tool names, fetched after the guard and the stages passed it.
|
|
317
|
+
* The addresses fetched are written into the result's text, so the egress rule on pictures the
|
|
318
|
+
* model shows counts them as given by a tool.
|
|
319
|
+
*/
|
|
320
|
+
async function* withFetchedImages(base, result, images, outputRaw) {
|
|
321
|
+
const fetched = await images(outputRaw);
|
|
322
|
+
if (fetched.warning)
|
|
323
|
+
yield toolEvent(base, { phase: 'warning', warning: fetched.warning });
|
|
324
|
+
if (fetched.parts.length === 0)
|
|
325
|
+
return result;
|
|
326
|
+
const note = `\n[Pictures shown with this result: ${fetched.urls.join(' ')}]`; // labels-exempt: a marker the egress rule reads the addresses from, not prose (P2)
|
|
327
|
+
return {
|
|
328
|
+
...result,
|
|
329
|
+
parts: [...(result.parts ?? []), ...fetched.parts],
|
|
330
|
+
...(result.modelText !== undefined
|
|
331
|
+
? { modelText: result.modelText + note }
|
|
332
|
+
: { finding: result.finding + note }),
|
|
333
|
+
};
|
|
334
|
+
}
|
|
299
335
|
/** What the after-call stages add to a settlement: a turn to stop, input to hand the model. */
|
|
300
336
|
function stageSettlement(post) {
|
|
301
337
|
return {
|
|
@@ -475,6 +511,7 @@ agent, beginBody) {
|
|
|
475
511
|
throwIfAborted(ctx.signal);
|
|
476
512
|
const settleOutcome = (outcome) => settleBodyOutcome({
|
|
477
513
|
outcome,
|
|
514
|
+
ctx,
|
|
478
515
|
tool,
|
|
479
516
|
base,
|
|
480
517
|
callId,
|
|
@@ -666,21 +703,9 @@ function authorizeSavedDisclosure(args, tool, outcome) {
|
|
|
666
703
|
if (guard.event?.action === 'block' || guard.event?.action === 'redact')
|
|
667
704
|
throw blocked();
|
|
668
705
|
}
|
|
669
|
-
/** A successful remote call's result with the pictures its tool's `media` names, fetched and added beside any the server sent. */
|
|
670
|
-
async function* withNamedMedia(tool, outcome, ctx, base) {
|
|
671
|
-
if (outcome.kind !== 'ok' || !tool.media)
|
|
672
|
-
return outcome;
|
|
673
|
-
const fetched = await fetchToolMedia(tool.media, outcome.outputRaw, ctx);
|
|
674
|
-
if (fetched.warning)
|
|
675
|
-
yield toolEvent(base, { phase: 'warning', warning: fetched.warning });
|
|
676
|
-
if (fetched.parts.length === 0)
|
|
677
|
-
return outcome;
|
|
678
|
-
const parts = [...(outcome.modelResult.parts ?? []), ...fetched.parts];
|
|
679
|
-
return { ...outcome, modelResult: { ...outcome.modelResult, parts } };
|
|
680
|
-
}
|
|
681
706
|
/** Settles an outcome the kernel reached without a handler's own result: a remote body, or a sign-in. */
|
|
682
707
|
async function* settleBodyOutcome(args) {
|
|
683
|
-
const { outcome, tool, base, callId, safeInput, stages, guard, labels } = args;
|
|
708
|
+
const { outcome, tool, base, callId, safeInput, stages, guard, labels, ctx } = args;
|
|
684
709
|
const name = tool.name;
|
|
685
710
|
if (outcome.kind === 'gated') {
|
|
686
711
|
if (outcome.gate.kind === 'confirmation') {
|
|
@@ -716,6 +741,9 @@ async function* settleBodyOutcome(args) {
|
|
|
716
741
|
guard,
|
|
717
742
|
provisional: { outputRaw: outcome.outputRaw, modelResult: outcome.modelResult },
|
|
718
743
|
sources: tool.sources,
|
|
744
|
+
...(tool.type !== 'function' && tool.images
|
|
745
|
+
? { images: (output) => fetchToolImages(tool.images, output, ctx) }
|
|
746
|
+
: {}),
|
|
719
747
|
labels: tool.labels,
|
|
720
748
|
reproject: makeReproject((v) => parseToolOutput(tool.output, v),
|
|
721
749
|
// why: A hook's edit replaces the value; the media the tool returned stays.
|
|
@@ -1113,9 +1141,9 @@ async function* settleByType(tools, tool, safeInput, fullCtx, base, snapshot, st
|
|
|
1113
1141
|
const body = tool.type === 'http'
|
|
1114
1142
|
? yield* executeHttpTool(tool, safeInput, fullCtx, base, stages, beginBody, new HandlerEvents(base, guard.handlerEvents(stages?.drafts === true)))
|
|
1115
1143
|
: yield* executeMcpTool(tool, safeInput, fullCtx, base, stages, tools.mcpSessions, beginBody);
|
|
1116
|
-
const remoteOutcome = yield* withNamedMedia(tool, body, fullCtx, base);
|
|
1117
1144
|
return yield* settleBodyOutcome({
|
|
1118
|
-
outcome:
|
|
1145
|
+
outcome: body,
|
|
1146
|
+
ctx: fullCtx,
|
|
1119
1147
|
tool,
|
|
1120
1148
|
base,
|
|
1121
1149
|
callId: base.callId,
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { InteractionPart, ToolWarning } from '../turn-events.js';
|
|
2
|
+
import type { ToolContext } from './types.js';
|
|
3
|
+
/** The most pictures one call hands the model. */
|
|
4
|
+
export declare const MAX_TOOL_IMAGES = 4;
|
|
5
|
+
/**
|
|
6
|
+
* The pictures a tool's `images(output)` names, fetched through the profile's network policy (every
|
|
7
|
+
* hop, redirects included, no tool credentials) and handed to the model as image parts. A picture
|
|
8
|
+
* that is not an https address, blocked, slow, too big or not an image is left out and reported in
|
|
9
|
+
* one `images_unfetched` warning: the call itself stays whole. `urls` are the ones fetched.
|
|
10
|
+
*/
|
|
11
|
+
export declare function fetchToolImages(images: (output: never) => string[], output: unknown, ctx: ToolContext): Promise<{
|
|
12
|
+
parts: InteractionPart[];
|
|
13
|
+
urls: string[];
|
|
14
|
+
warning?: ToolWarning;
|
|
15
|
+
}>;
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
import { fetchGuarded } from '../../guardrails/network.js';
|
|
2
|
+
import { bytesToBase64 } from '../util/base64.js';
|
|
3
|
+
import { toolNetworkPolicy } from './events.js';
|
|
4
|
+
/** The most pictures one call hands the model. */
|
|
5
|
+
export const MAX_TOOL_IMAGES = 4;
|
|
6
|
+
/** The largest picture fetched; a larger one is left out, not cut. */
|
|
7
|
+
const MAX_TOOL_IMAGE_BYTES = 5 * 1024 * 1024;
|
|
8
|
+
/** The most one call hands the model in all, so pictures cannot crowd a request past a provider's limit. */
|
|
9
|
+
const MAX_TOOL_IMAGES_TOTAL_BYTES = 10 * 1024 * 1024;
|
|
10
|
+
/** How long each picture may take, so a slow host cannot hold the call. */
|
|
11
|
+
const IMAGE_FETCH_TIMEOUT_MS = 15_000;
|
|
12
|
+
const IMAGE_TYPES = new Set(['image/png', 'image/jpeg', 'image/gif', 'image/webp']);
|
|
13
|
+
/** The body, read up to `maxBytes`; `undefined` when it runs past. */
|
|
14
|
+
async function readCappedBytes(response, maxBytes) {
|
|
15
|
+
const declared = Number(response.headers.get('content-length'));
|
|
16
|
+
if (Number.isFinite(declared) && declared > maxBytes) {
|
|
17
|
+
await response.body?.cancel().catch(() => { });
|
|
18
|
+
return undefined;
|
|
19
|
+
}
|
|
20
|
+
if (!response.body)
|
|
21
|
+
return new Uint8Array(await response.arrayBuffer());
|
|
22
|
+
const reader = response.body.getReader();
|
|
23
|
+
const chunks = [];
|
|
24
|
+
let total = 0;
|
|
25
|
+
for (;;) {
|
|
26
|
+
const { done, value } = await reader.read();
|
|
27
|
+
if (done)
|
|
28
|
+
break;
|
|
29
|
+
total += value.byteLength;
|
|
30
|
+
if (total > maxBytes) {
|
|
31
|
+
await reader.cancel().catch(() => { });
|
|
32
|
+
return undefined;
|
|
33
|
+
}
|
|
34
|
+
chunks.push(value);
|
|
35
|
+
}
|
|
36
|
+
const bytes = new Uint8Array(total);
|
|
37
|
+
let at = 0;
|
|
38
|
+
for (const chunk of chunks) {
|
|
39
|
+
bytes.set(chunk, at);
|
|
40
|
+
at += chunk.byteLength;
|
|
41
|
+
}
|
|
42
|
+
return bytes;
|
|
43
|
+
}
|
|
44
|
+
async function fetchOne(url, ctx) {
|
|
45
|
+
const timeout = AbortSignal.timeout(IMAGE_FETCH_TIMEOUT_MS);
|
|
46
|
+
const signal = ctx.signal ? AbortSignal.any([ctx.signal, timeout]) : timeout;
|
|
47
|
+
const response = await fetchGuarded(url, { method: 'GET', headers: { Accept: 'image/*' }, signal }, {
|
|
48
|
+
policy: toolNetworkPolicy(ctx),
|
|
49
|
+
followRedirects: true,
|
|
50
|
+
resolveHost: ctx.resolveHost,
|
|
51
|
+
});
|
|
52
|
+
if (!response.ok)
|
|
53
|
+
throw new Error(`HTTP ${response.status}`); // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
|
|
54
|
+
const mimeType = (response.headers.get('content-type') ?? '').split(';')[0]?.trim().toLowerCase();
|
|
55
|
+
if (!mimeType || !IMAGE_TYPES.has(mimeType)) {
|
|
56
|
+
await response.body?.cancel().catch(() => { });
|
|
57
|
+
throw new Error(`not a png, jpeg, gif or webp image (${mimeType || 'no type'})`); // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
|
|
58
|
+
}
|
|
59
|
+
const bytes = await readCappedBytes(response, MAX_TOOL_IMAGE_BYTES);
|
|
60
|
+
if (!bytes)
|
|
61
|
+
throw new Error(`larger than ${MAX_TOOL_IMAGE_BYTES} bytes`); // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
|
|
62
|
+
return { type: 'image', mimeType, data: bytesToBase64(bytes) };
|
|
63
|
+
}
|
|
64
|
+
/** `url` when it is an absolute https address; otherwise the reason it is not. */
|
|
65
|
+
function httpsAddress(url) {
|
|
66
|
+
if (typeof url !== 'string')
|
|
67
|
+
return { reason: 'an address that is not text' }; // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
|
|
68
|
+
try {
|
|
69
|
+
if (new URL(url).protocol === 'https:')
|
|
70
|
+
return { url };
|
|
71
|
+
}
|
|
72
|
+
catch {
|
|
73
|
+
// why: an unparseable address is reported below as a reason, not thrown.
|
|
74
|
+
}
|
|
75
|
+
return { reason: `${url.slice(0, 80)}: not an absolute https address` }; // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* The pictures a tool's `images(output)` names, fetched through the profile's network policy (every
|
|
79
|
+
* hop, redirects included, no tool credentials) and handed to the model as image parts. A picture
|
|
80
|
+
* that is not an https address, blocked, slow, too big or not an image is left out and reported in
|
|
81
|
+
* one `images_unfetched` warning: the call itself stays whole. `urls` are the ones fetched.
|
|
82
|
+
*/
|
|
83
|
+
export async function fetchToolImages(images, output, ctx) {
|
|
84
|
+
const reasons = [];
|
|
85
|
+
const named = [];
|
|
86
|
+
try {
|
|
87
|
+
for (const raw of new Set(images(output))) {
|
|
88
|
+
const address = httpsAddress(raw);
|
|
89
|
+
if ('url' in address)
|
|
90
|
+
named.push(address.url);
|
|
91
|
+
else
|
|
92
|
+
reasons.push(address.reason);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
catch (err) {
|
|
96
|
+
// labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
|
|
97
|
+
return {
|
|
98
|
+
parts: [],
|
|
99
|
+
urls: [],
|
|
100
|
+
warning: unfetched([`the images hook threw: ${String(err)}`]), // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
const wanted = named.slice(0, MAX_TOOL_IMAGES);
|
|
104
|
+
const settled = await Promise.allSettled(wanted.map((url) => fetchOne(url, ctx)));
|
|
105
|
+
const parts = [];
|
|
106
|
+
const urls = [];
|
|
107
|
+
let total = 0;
|
|
108
|
+
settled.forEach((result, index) => {
|
|
109
|
+
const url = wanted[index];
|
|
110
|
+
if (result.status === 'rejected') {
|
|
111
|
+
const why = result.reason instanceof Error ? result.reason.message : String(result.reason);
|
|
112
|
+
reasons.push(`${new URL(url).hostname}: ${why}`);
|
|
113
|
+
return;
|
|
114
|
+
}
|
|
115
|
+
const size = 'data' in result.value ? result.value.data.length : 0;
|
|
116
|
+
if (total + size > MAX_TOOL_IMAGES_TOTAL_BYTES) {
|
|
117
|
+
reasons.push(`${new URL(url).hostname}: past the ${MAX_TOOL_IMAGES_TOTAL_BYTES} byte total`); // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
|
|
118
|
+
return;
|
|
119
|
+
}
|
|
120
|
+
total += size;
|
|
121
|
+
parts.push(result.value);
|
|
122
|
+
urls.push(url);
|
|
123
|
+
});
|
|
124
|
+
return {
|
|
125
|
+
parts,
|
|
126
|
+
urls,
|
|
127
|
+
...(reasons.length ? { warning: unfetched(reasons) } : {}),
|
|
128
|
+
};
|
|
129
|
+
}
|
|
130
|
+
function unfetched(reasons) {
|
|
131
|
+
return {
|
|
132
|
+
code: 'images_unfetched',
|
|
133
|
+
severity: 'warning',
|
|
134
|
+
message: `Some pictures the tool named were left out. ${reasons.join('; ')}`, // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
|
|
135
|
+
};
|
|
136
|
+
}
|
|
@@ -161,13 +161,14 @@ export interface ToolOutputHooks<TOut = unknown> {
|
|
|
161
161
|
sources?: (output: TOut) => Source[];
|
|
162
162
|
}
|
|
163
163
|
/** For a tool whose result names pictures by address; function tools return the bytes themselves as `parts`. */
|
|
164
|
-
export interface
|
|
164
|
+
export interface ToolImageHooks<TOut = unknown> {
|
|
165
165
|
/**
|
|
166
|
-
* The https addresses of pictures in `output` the model should see, at most `
|
|
167
|
-
* of them.
|
|
168
|
-
*
|
|
166
|
+
* The https addresses of pictures in `output` the model should see, at most `MAX_TOOL_IMAGES`
|
|
167
|
+
* of them. After the result passes the output guard, the kernel fetches each one through the
|
|
168
|
+
* profile's network policy, without the tool's credentials, and hands the model the image as a
|
|
169
|
+
* part; one it cannot fetch is an `images_unfetched` warning, never a failed call.
|
|
169
170
|
*/
|
|
170
|
-
|
|
171
|
+
images?: (output: TOut) => string[];
|
|
171
172
|
}
|
|
172
173
|
export interface FunctionToolDef<TIn = unknown, TOut = unknown> extends ToolBase, ToolHostHooks<TIn>, ToolOutputHooks<TOut> {
|
|
173
174
|
type: 'function';
|
|
@@ -210,7 +211,7 @@ export interface ToolAuthConfig {
|
|
|
210
211
|
clientId?: string;
|
|
211
212
|
redirectUri?: string;
|
|
212
213
|
}
|
|
213
|
-
export interface HttpToolDef<TIn = unknown, TOut = unknown> extends ToolBase, ToolHostHooks<TIn>, ToolOutputHooks<TOut>,
|
|
214
|
+
export interface HttpToolDef<TIn = unknown, TOut = unknown> extends ToolBase, ToolHostHooks<TIn>, ToolOutputHooks<TOut>, ToolImageHooks<TOut> {
|
|
214
215
|
type: 'http';
|
|
215
216
|
input: z.ZodType<TIn>;
|
|
216
217
|
output: z.ZodType<TOut>;
|
|
@@ -237,7 +238,7 @@ export interface HttpToolLimit {
|
|
|
237
238
|
requests: number;
|
|
238
239
|
windowMs: number;
|
|
239
240
|
}
|
|
240
|
-
export interface McpToolDef<TIn = unknown, TOut = unknown> extends ToolBase, ToolHostHooks<TIn>, ToolOutputHooks<TOut>,
|
|
241
|
+
export interface McpToolDef<TIn = unknown, TOut = unknown> extends ToolBase, ToolHostHooks<TIn>, ToolOutputHooks<TOut>, ToolImageHooks<TOut> {
|
|
241
242
|
type: 'mcp';
|
|
242
243
|
input: z.ZodType<TIn>;
|
|
243
244
|
output: z.ZodType<TOut>;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@theoremjs/agents",
|
|
3
|
-
"version": "0.3.4-next.
|
|
3
|
+
"version": "0.3.4-next.403.g5de27f3",
|
|
4
4
|
"description": "A flat TypeScript agent kernel for typed profiles, deterministic turn execution, registered tools, provider adapters, guardrails, and host-injected traces.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"agent",
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
import type { InteractionPart, ToolWarning } from '../turn-events.js';
|
|
2
|
-
import type { ToolContext } from './types.js';
|
|
3
|
-
/** The most pictures one call hands the model. */
|
|
4
|
-
export declare const MAX_TOOL_MEDIA_FILES = 4;
|
|
5
|
-
/**
|
|
6
|
-
* The pictures a tool's `media(output)` names, fetched through the profile's network policy (every
|
|
7
|
-
* hop, redirects included) and handed to the model as image parts. A picture that is blocked, slow,
|
|
8
|
-
* too big or not an image is left out and reported in one `media_unfetched` warning: the call
|
|
9
|
-
* itself stays whole.
|
|
10
|
-
*/
|
|
11
|
-
export declare function fetchToolMedia(media: (output: never) => string[], output: unknown, ctx: ToolContext): Promise<{
|
|
12
|
-
parts: InteractionPart[];
|
|
13
|
-
warning?: ToolWarning;
|
|
14
|
-
}>;
|
|
@@ -1,94 +0,0 @@
|
|
|
1
|
-
import { fetchGuarded } from '../../guardrails/network.js';
|
|
2
|
-
import { bytesToBase64 } from '../util/base64.js';
|
|
3
|
-
import { toolNetworkPolicy } from './events.js';
|
|
4
|
-
/** The most pictures one call hands the model. */
|
|
5
|
-
export const MAX_TOOL_MEDIA_FILES = 4;
|
|
6
|
-
/** The largest picture fetched; a larger one is left out, not cut. */
|
|
7
|
-
const MAX_TOOL_MEDIA_BYTES = 5 * 1024 * 1024;
|
|
8
|
-
/** How long each picture may take, so a slow host cannot hold the call. */
|
|
9
|
-
const MEDIA_FETCH_TIMEOUT_MS = 15_000;
|
|
10
|
-
const IMAGE_TYPES = new Set(['image/png', 'image/jpeg', 'image/gif', 'image/webp']);
|
|
11
|
-
/** The body, read up to `maxBytes`; `undefined` when it runs past. */
|
|
12
|
-
async function readCappedBytes(response, maxBytes) {
|
|
13
|
-
const declared = Number(response.headers.get('content-length'));
|
|
14
|
-
if (Number.isFinite(declared) && declared > maxBytes) {
|
|
15
|
-
await response.body?.cancel().catch(() => { });
|
|
16
|
-
return undefined;
|
|
17
|
-
}
|
|
18
|
-
if (!response.body)
|
|
19
|
-
return new Uint8Array(await response.arrayBuffer());
|
|
20
|
-
const reader = response.body.getReader();
|
|
21
|
-
const chunks = [];
|
|
22
|
-
let total = 0;
|
|
23
|
-
for (;;) {
|
|
24
|
-
const { done, value } = await reader.read();
|
|
25
|
-
if (done)
|
|
26
|
-
break;
|
|
27
|
-
total += value.byteLength;
|
|
28
|
-
if (total > maxBytes) {
|
|
29
|
-
await reader.cancel().catch(() => { });
|
|
30
|
-
return undefined;
|
|
31
|
-
}
|
|
32
|
-
chunks.push(value);
|
|
33
|
-
}
|
|
34
|
-
const bytes = new Uint8Array(total);
|
|
35
|
-
let at = 0;
|
|
36
|
-
for (const chunk of chunks) {
|
|
37
|
-
bytes.set(chunk, at);
|
|
38
|
-
at += chunk.byteLength;
|
|
39
|
-
}
|
|
40
|
-
return bytes;
|
|
41
|
-
}
|
|
42
|
-
async function fetchOne(url, ctx) {
|
|
43
|
-
const timeout = AbortSignal.timeout(MEDIA_FETCH_TIMEOUT_MS);
|
|
44
|
-
const signal = ctx.signal ? AbortSignal.any([ctx.signal, timeout]) : timeout;
|
|
45
|
-
const response = await fetchGuarded(url, { method: 'GET', headers: { Accept: 'image/*' }, signal }, {
|
|
46
|
-
policy: toolNetworkPolicy(ctx),
|
|
47
|
-
followRedirects: true,
|
|
48
|
-
resolveHost: ctx.resolveHost,
|
|
49
|
-
});
|
|
50
|
-
if (!response.ok)
|
|
51
|
-
throw new Error(`HTTP ${response.status}`); // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
|
|
52
|
-
const mimeType = (response.headers.get('content-type') ?? '').split(';')[0]?.trim().toLowerCase();
|
|
53
|
-
if (!mimeType || !IMAGE_TYPES.has(mimeType)) {
|
|
54
|
-
await response.body?.cancel().catch(() => { });
|
|
55
|
-
throw new Error(`not a png, jpeg, gif or webp image (${mimeType || 'no type'})`); // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
|
|
56
|
-
}
|
|
57
|
-
const bytes = await readCappedBytes(response, MAX_TOOL_MEDIA_BYTES);
|
|
58
|
-
if (!bytes)
|
|
59
|
-
throw new Error(`larger than ${MAX_TOOL_MEDIA_BYTES} bytes`); // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
|
|
60
|
-
return { type: 'image', mimeType, data: bytesToBase64(bytes) };
|
|
61
|
-
}
|
|
62
|
-
/**
|
|
63
|
-
* The pictures a tool's `media(output)` names, fetched through the profile's network policy (every
|
|
64
|
-
* hop, redirects included) and handed to the model as image parts. A picture that is blocked, slow,
|
|
65
|
-
* too big or not an image is left out and reported in one `media_unfetched` warning: the call
|
|
66
|
-
* itself stays whole.
|
|
67
|
-
*/
|
|
68
|
-
export async function fetchToolMedia(media, output, ctx) {
|
|
69
|
-
let urls;
|
|
70
|
-
try {
|
|
71
|
-
urls = [...new Set(media(output))].slice(0, MAX_TOOL_MEDIA_FILES);
|
|
72
|
-
}
|
|
73
|
-
catch (err) {
|
|
74
|
-
// labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
|
|
75
|
-
return { parts: [], warning: unfetched([`the media hook threw: ${String(err)}`]) };
|
|
76
|
-
}
|
|
77
|
-
const settled = await Promise.allSettled(urls.map((url) => fetchOne(url, ctx)));
|
|
78
|
-
const parts = [];
|
|
79
|
-
const reasons = [];
|
|
80
|
-
settled.forEach((result, index) => {
|
|
81
|
-
if (result.status === 'fulfilled')
|
|
82
|
-
parts.push(result.value);
|
|
83
|
-
else
|
|
84
|
-
reasons.push(`${new URL(urls[index] ?? 'https://invalid.invalid').hostname}: ${String(result.reason instanceof Error ? result.reason.message : result.reason)}`);
|
|
85
|
-
});
|
|
86
|
-
return { parts, ...(reasons.length ? { warning: unfetched(reasons) } : {}) };
|
|
87
|
-
}
|
|
88
|
-
function unfetched(reasons) {
|
|
89
|
-
return {
|
|
90
|
-
code: 'media_unfetched',
|
|
91
|
-
severity: 'warning',
|
|
92
|
-
message: `Some pictures the tool named were left out. ${reasons.join('; ')}`, // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
|
|
93
|
-
};
|
|
94
|
-
}
|