akanjs 3.1.0-rc.12 → 3.1.0-rc.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. package/package.json +1 -1
  2. package/service/agent.service.ts +21 -1
  3. package/service/predefinedAdaptor/anthropicLlm.ts +36 -16
  4. package/service/predefinedAdaptor/llm.adaptor.ts +2 -0
  5. package/service/predefinedAdaptor/openaiDialect.ts +53 -17
  6. package/store/action.ts +10 -4
  7. package/store/agent/ScreenImage.ts +166 -0
  8. package/store/agent/StoreSurfaceSource.ts +36 -2
  9. package/store/agent/index.ts +2 -1
  10. package/types/service/agent.service.d.ts +2 -0
  11. package/types/service/predefinedAdaptor/anthropicLlm.d.ts +9 -4
  12. package/types/service/predefinedAdaptor/llm.adaptor.d.ts +2 -0
  13. package/types/service/predefinedAdaptor/openaiDialect.d.ts +20 -5
  14. package/types/store/action.d.ts +2 -0
  15. package/types/store/agent/ScreenImage.d.ts +27 -0
  16. package/types/store/agent/StoreSurfaceSource.d.ts +1 -1
  17. package/types/store/agent/index.d.ts +2 -1
  18. package/types/vendor/use-agentic/Compaction.d.ts +3 -1
  19. package/types/vendor/use-agentic/ToolOutput.d.ts +6 -1
  20. package/types/vendor/use-agentic/ToolReturn.d.ts +18 -0
  21. package/types/vendor/use-agentic/Transcript.d.ts +7 -0
  22. package/types/vendor/use-agentic/index.d.ts +1 -0
  23. package/types/vendor/use-agentic/types.d.ts +6 -0
  24. package/ui/Agent/agentSessionOf.ts +15 -8
  25. package/ui/Load/Units.tsx +4 -2
  26. package/vendor/use-agentic/Compaction.ts +38 -7
  27. package/vendor/use-agentic/ToolOutput.ts +25 -3
  28. package/vendor/use-agentic/ToolReturn.ts +32 -0
  29. package/vendor/use-agentic/ToolRunner.ts +15 -13
  30. package/vendor/use-agentic/Transcript.ts +38 -2
  31. package/vendor/use-agentic/WIRE.md +20 -0
  32. package/vendor/use-agentic/index.ts +1 -0
  33. package/vendor/use-agentic/types.ts +6 -0
  34. package/webkit/McpAppAgent.ts +12 -2
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "akanjs",
3
- "version": "3.1.0-rc.12",
3
+ "version": "3.1.0-rc.14",
4
4
  "sourceType": "module",
5
5
  "type": "module",
6
6
  "publishConfig": {
@@ -3,6 +3,7 @@ import type {
3
3
  AgentWireAttachment,
4
4
  AgentWireMessage,
5
5
  AgentWireReference,
6
+ AgentWireToolResult,
6
7
  LlmAccepts,
7
8
  LlmTurnRequest,
8
9
  } from "./predefinedAdaptor/llm.adaptor";
@@ -104,12 +105,21 @@ export class AgentService extends serve("agent" as const, ({ plug }) => ({
104
105
 
105
106
  /** Swaps an attachment the provider cannot read for a note; dropped silently, it is guessed at from its name. */
106
107
  static readable(request: LlmTurnRequest, accepts: LlmAccepts | undefined): LlmTurnRequest {
107
- if (!request.messages.some((message) => message.attachments?.length)) return request;
108
+ if (!request.messages.some(AgentService.carriesFiles)) return request;
108
109
  const messages = request.messages.map((message) => AgentService.readableMessage(message, accepts ?? {}));
109
110
  return { ...request, messages };
110
111
  }
111
112
 
113
+ private static carriesFiles(message: AgentWireMessage): boolean {
114
+ return !!message.attachments?.length || !!message.toolResults?.some((result) => result.attachments?.length);
115
+ }
116
+
112
117
  private static readableMessage(message: AgentWireMessage, accepts: LlmAccepts): AgentWireMessage {
118
+ if (message.role === "tool")
119
+ return {
120
+ ...message,
121
+ toolResults: (message.toolResults ?? []).map((result) => AgentService.readableResult(result, accepts)),
122
+ };
113
123
  const { attachments = [], ...rest } = message;
114
124
  if (!attachments.length) return message;
115
125
  const kept = attachments.filter((attachment) => AgentService.isReadable(attachment, accepts));
@@ -122,6 +132,16 @@ export class AgentService extends serve("agent" as const, ({ plug }) => ({
122
132
  };
123
133
  }
124
134
 
135
+ private static readableResult(result: AgentWireToolResult, accepts: LlmAccepts): AgentWireToolResult {
136
+ if (!result.attachments?.length) return result;
137
+ const attachments = result.attachments.map((attachment) =>
138
+ AgentService.isReadable(attachment, accepts)
139
+ ? attachment
140
+ : { name: attachment.name, mimeType: attachment.mimeType, text: AgentService.note(attachment) },
141
+ );
142
+ return { ...result, attachments };
143
+ }
144
+
125
145
  private static isReadable(attachment: AgentWireAttachment, accepts: LlmAccepts): boolean {
126
146
  if (attachment.text) return true;
127
147
  if (!attachment.data && !attachment.url) return false;
@@ -2,6 +2,7 @@ import { adapt } from "../adapt";
2
2
  import type {
3
3
  AgentWireAttachment,
4
4
  AgentWireMessage,
5
+ AgentWireToolResult,
5
6
  LlmAccepts,
6
7
  LlmAdaptor,
7
8
  LlmLimits,
@@ -14,12 +15,14 @@ import { LlmOverflow } from "./llmOverflow";
14
15
  import { eachSseData } from "./sseData";
15
16
 
16
17
  type AnthropicSource = { type: "base64"; media_type: string; data: string } | { type: "url"; url: string };
17
- type AnthropicBlock =
18
+ type AnthropicContentBlock =
18
19
  | { type: "text"; text: string }
19
20
  | { type: "image"; source: AnthropicSource }
20
- | { type: "document"; source: AnthropicSource }
21
+ | { type: "document"; source: AnthropicSource };
22
+ type AnthropicBlock =
23
+ | AnthropicContentBlock
21
24
  | { type: "tool_use"; id: string; name: string; input: Record<string, unknown> }
22
- | { type: "tool_result"; tool_use_id: string; content: string };
25
+ | { type: "tool_result"; tool_use_id: string; content: string | AnthropicContentBlock[] };
23
26
  interface AnthropicMessage {
24
27
  role: "user" | "assistant";
25
28
  content: AnthropicBlock[];
@@ -221,15 +224,7 @@ export class AnthropicLlm
221
224
  if (message.role === "tool")
222
225
  return {
223
226
  role: "user",
224
- content: (message.toolResults ?? []).map((result) => ({
225
- type: "tool_result" as const,
226
- tool_use_id: result.id,
227
- content: JSON.stringify({
228
- ...(result.result !== undefined ? { result: result.result } : {}),
229
- ...(result.changes?.length ? { changes: result.changes } : {}),
230
- ...(result.error ? { error: result.error } : {}),
231
- }),
232
- })),
227
+ content: (message.toolResults ?? []).map((result) => AnthropicLlm.toolResult(result, accepts)),
233
228
  };
234
229
  if (message.role === "assistant")
235
230
  return {
@@ -248,9 +243,36 @@ export class AnthropicLlm
248
243
  }
249
244
 
250
245
  static userContent(message: AgentWireMessage, accepts?: LlmAccepts): AnthropicBlock[] {
251
- const attachments = message.attachments ?? [];
252
246
  const notes: string[] = [];
253
- const blocks = attachments.flatMap((attachment): AnthropicBlock[] => {
247
+ const blocks = AnthropicLlm.attachmentBlocks(message.attachments ?? [], accepts, notes);
248
+ const text = [message.text, ...notes].filter(Boolean).join("\n\n");
249
+ return [...(text ? [{ type: "text" as const, text }] : []), ...blocks];
250
+ }
251
+
252
+ /** A string while the call returned no file, so a transcript without one posts what it always did. */
253
+ static toolResult(result: AgentWireToolResult, accepts?: LlmAccepts): AnthropicBlock {
254
+ const payload = JSON.stringify({
255
+ ...(result.result !== undefined ? { result: result.result } : {}),
256
+ ...(result.changes?.length ? { changes: result.changes } : {}),
257
+ ...(result.error ? { error: result.error } : {}),
258
+ });
259
+ if (!result.attachments?.length) return { type: "tool_result", tool_use_id: result.id, content: payload };
260
+ const notes: string[] = [];
261
+ const blocks = AnthropicLlm.attachmentBlocks(result.attachments, accepts, notes);
262
+ return {
263
+ type: "tool_result",
264
+ tool_use_id: result.id,
265
+ content: [{ type: "text", text: [payload, ...notes].join("\n\n") }, ...blocks],
266
+ };
267
+ }
268
+
269
+ /** Pushes a note for each file no block can carry, for the caller to fold into the text beside them. */
270
+ static attachmentBlocks(
271
+ attachments: AgentWireAttachment[],
272
+ accepts: LlmAccepts | undefined,
273
+ notes: string[],
274
+ ): AnthropicContentBlock[] {
275
+ return attachments.flatMap((attachment): AnthropicContentBlock[] => {
254
276
  if (attachment.text)
255
277
  return [
256
278
  {
@@ -271,8 +293,6 @@ export class AnthropicLlm
271
293
  notes.push(`[Attachment not read: ${attachment.name} (${attachment.mimeType}) — this API has no block for it.]`);
272
294
  return [];
273
295
  });
274
- const text = [message.text, ...notes].filter(Boolean).join("\n\n");
275
- return [...(text ? [{ type: "text" as const, text }] : []), ...blocks];
276
296
  }
277
297
 
278
298
  static typed(source: AnthropicSource, mimeType: string): AnthropicSource {
@@ -10,6 +10,8 @@ export interface AgentWireToolResult {
10
10
  result?: unknown;
11
11
  changes?: unknown[];
12
12
  error?: string;
13
+ /** Files the call handed the model, as content beside the JSON; one this model cannot read arrives as a `text` note. */
14
+ attachments?: AgentWireAttachment[];
13
15
  }
14
16
 
15
17
  /** Exactly one carrier reaches the model: `data` (inlined bytes), `url` (fetched by the provider) or `text`. */
@@ -1,4 +1,11 @@
1
- import type { AgentWireMessage, LlmAccepts, LlmTurnAnswer, LlmTurnRequest, LlmUsage } from "./llm.adaptor";
1
+ import type {
2
+ AgentWireAttachment,
3
+ AgentWireMessage,
4
+ LlmAccepts,
5
+ LlmTurnAnswer,
6
+ LlmTurnRequest,
7
+ LlmUsage,
8
+ } from "./llm.adaptor";
2
9
  import { eachSseData } from "./sseData";
3
10
 
4
11
  export interface OpenaiToolCall {
@@ -25,7 +32,8 @@ interface OpenaiStreamChunk {
25
32
  finish_reason?: string | null;
26
33
  }[];
27
34
  }
28
- type OpenaiContentPart = { type: "text"; text: string } | { type: "image_url"; image_url: { url: string } };
35
+ type OpenaiImagePart = { type: "image_url"; image_url: { url: string } };
36
+ type OpenaiContentPart = { type: "text"; text: string } | OpenaiImagePart;
29
37
  export interface OpenaiMessage {
30
38
  role: "system" | "user" | "assistant" | "tool";
31
39
  content: string | OpenaiContentPart[];
@@ -86,16 +94,7 @@ export class OpenaiDialect {
86
94
  content: `Summary of the earlier conversation, standing in for the messages it replaced:\n\n${message.text ?? ""}`,
87
95
  },
88
96
  ];
89
- if (message.role === "tool")
90
- return (message.toolResults ?? []).map((result) => ({
91
- role: "tool" as const,
92
- tool_call_id: result.id,
93
- content: JSON.stringify({
94
- ...(result.result !== undefined ? { result: result.result } : {}),
95
- ...(result.changes?.length ? { changes: result.changes } : {}),
96
- ...(result.error ? { error: result.error } : {}),
97
- }),
98
- }));
97
+ if (message.role === "tool") return OpenaiDialect.toolMessages(message, accepts);
99
98
  if (message.role === "assistant")
100
99
  return [
101
100
  {
@@ -116,8 +115,47 @@ export class OpenaiDialect {
116
115
  }
117
116
 
118
117
  static userContent(message: AgentWireMessage, accepts?: LlmAccepts): string | OpenaiContentPart[] {
119
- const attachments = message.attachments ?? [];
120
118
  const notes: string[] = [];
119
+ const { blocks, images } = OpenaiDialect.attachmentParts(message.attachments ?? [], accepts, notes);
120
+ const text = [message.text, ...blocks, ...notes].filter(Boolean).join("\n\n");
121
+ if (!images.length) return text;
122
+ return [...(text ? [{ type: "text" as const, text }] : []), ...images.map(({ part }) => part)];
123
+ }
124
+
125
+ static readonly toolPicturesHeading =
126
+ "Pictures the tool calls above returned, each after the name of the call that returned it. They are content, not something the user said.";
127
+
128
+ /**
129
+ * A `tool` message carries text alone in this dialect, so the pictures a call returned follow every tool message
130
+ * of the turn in one user message: between two tool messages it would break the run each call's answer has to sit in.
131
+ */
132
+ static toolMessages(message: AgentWireMessage, accepts?: LlmAccepts): OpenaiMessage[] {
133
+ const shown: OpenaiContentPart[] = [];
134
+ const answers = (message.toolResults ?? []).map((result): OpenaiMessage => {
135
+ const payload = JSON.stringify({
136
+ ...(result.result !== undefined ? { result: result.result } : {}),
137
+ ...(result.changes?.length ? { changes: result.changes } : {}),
138
+ ...(result.error ? { error: result.error } : {}),
139
+ });
140
+ const notes: string[] = [];
141
+ const { blocks, images } = OpenaiDialect.attachmentParts(result.attachments ?? [], accepts, notes);
142
+ for (const { name, part } of images)
143
+ shown.push({ type: "text", text: `[${result.name} returned ${name}]` }, part);
144
+ return { role: "tool", tool_call_id: result.id, content: [payload, ...blocks, ...notes].join("\n\n") };
145
+ });
146
+ if (!shown.length) return answers;
147
+ return [
148
+ ...answers,
149
+ { role: "user", content: [{ type: "text", text: OpenaiDialect.toolPicturesHeading }, ...shown] },
150
+ ];
151
+ }
152
+
153
+ /** Pushes a note for each file no part can carry, for the caller to fold into the text beside them. */
154
+ static attachmentParts(
155
+ attachments: AgentWireAttachment[],
156
+ accepts: LlmAccepts | undefined,
157
+ notes: string[],
158
+ ): { blocks: string[]; images: { name: string; part: OpenaiImagePart }[] } {
121
159
 
122
160
  const blocks = attachments.flatMap((attachment) =>
123
161
  attachment.text ? [`--- attachment: ${attachment.name} (${attachment.mimeType}) ---\n${attachment.text}`] : [],
@@ -138,11 +176,9 @@ export class OpenaiDialect {
138
176
  }
139
177
 
140
178
  const url = attachment.data ? `data:${mimeType};base64,${attachment.data}` : (attachment.url ?? "");
141
- return url ? [{ type: "image_url" as const, image_url: { url } }] : [];
179
+ return url ? [{ name: attachment.name, part: { type: "image_url" as const, image_url: { url } } }] : [];
142
180
  });
143
- const text = [message.text, ...blocks, ...notes].filter(Boolean).join("\n\n");
144
- if (!images.length) return text;
145
- return [...(text ? [{ type: "text" as const, text }] : []), ...images];
181
+ return { blocks, images };
146
182
  }
147
183
 
148
184
  static async consumeStream(
package/store/action.ts CHANGED
@@ -136,7 +136,7 @@ type BaseAction<
136
136
  ) => Promise<void>;
137
137
  } & { [K in `set${_CapitalizedRefName}`]: (...models: (Full | Light)[]) => void } & {
138
138
  [K in `reset${_CapitalizedRefName}`]: (model?: Full) => void;
139
- };
139
+ } & { [K in `invalidate${_CapitalizedRefName}`]: () => void };
140
140
 
141
141
  export type SliceAction<
142
142
  RefName extends string,
@@ -543,6 +543,7 @@ export const makeActions = (refName: string, slice: { [key: string]: SerializedS
543
543
  viewModel: `view${className}`,
544
544
  setModel: `set${className}`,
545
545
  resetModel: `reset${className}`,
546
+ invalidateModel: `invalidate${className}`,
546
547
  loadModelDraft: `load${className}FormDraft`,
547
548
  restoreModelDraft: `restore${className}FormDraft`,
548
549
  discardModelDraft: `discard${className}FormDraft`,
@@ -905,6 +906,9 @@ export const makeActions = (refName: string, slice: { [key: string]: SerializedS
905
906
  });
906
907
  return model ?? null;
907
908
  },
909
+ [names.invalidateModel]: function (this: SetGet) {
910
+ this.set(staleAtOfSlices());
911
+ },
908
912
 
909
913
  [names.loadModelDraft]: async function (this: SetGet, draftScope?: string) {
910
914
  const currentState = this.get() as { [key: string]: any };
@@ -1033,6 +1037,8 @@ export const makeActions = (refName: string, slice: { [key: string]: SerializedS
1033
1037
  isQueryEqual(sort as unknown as object, sortOfModel as unknown as object)
1034
1038
  )
1035
1039
  return;
1040
+
1041
+ const requestedAt = new Date();
1036
1042
  await loadList(
1037
1043
  this,
1038
1044
  () =>
@@ -1049,7 +1055,7 @@ export const makeActions = (refName: string, slice: { [key: string]: SerializedS
1049
1055
  [namesOfSlice.modelList]: modelList,
1050
1056
  [namesOfSlice.modelInsight]: modelInsight,
1051
1057
  [namesOfSlice.modelInitList]: modelList,
1052
- [namesOfSlice.modelInitAt]: new Date(),
1058
+ [namesOfSlice.modelInitAt]: requestedAt,
1053
1059
  ...lastPageFrom(modelInsight.count, limit),
1054
1060
  ...hasMoreFrom(modelDataList, fetchLimit),
1055
1061
  [namesOfSlice.limitOfModel]: limit,
@@ -1209,10 +1215,10 @@ export const makeActions = (refName: string, slice: { [key: string]: SerializedS
1209
1215
 
1210
1216
  [namesOfSlice.applyLiveModel]: function (this: SetGet, event: LiveEventPayload) {
1211
1217
  const currentState = this.get() as { [key: string]: any };
1212
- if (currentState[namesOfSlice.modelListLoading] as boolean) return;
1213
1218
  const modelList = currentState[namesOfSlice.modelList] as DataList<Light>;
1214
1219
  const staleAt = { [namesOfSlice.modelStaleAt]: new Date() };
1215
- if (event.op === "invalidate") {
1220
+
1221
+ if (event.op === "invalidate" || (currentState[namesOfSlice.modelListLoading] as boolean)) {
1216
1222
  this.set(staleAt);
1217
1223
  return;
1218
1224
  }
@@ -0,0 +1,166 @@
1
+ import { Binary } from "akanjs/base";
2
+ import { type MessageAttachment, ToolOutput } from "../../vendor/use-agentic";
3
+ import { ScreenTarget } from "./ScreenTarget";
4
+
5
+ export interface ScreenImageView {
6
+ attachment: MessageAttachment;
7
+ /** The picture's size and how it was sent, for a model asked about its proportions or resolution. */
8
+ size: string;
9
+ }
10
+
11
+ /**
12
+ * A picture on the screen, handed to the model as the browser drew it: redrawn into a JPEG no longer than
13
+ * `maxEdge`, inlined as bytes. Redrawing turns HEIC, AVIF and SVG into a type every provider reads, bounds what one
14
+ * picture costs, and leaves the provider nothing to download — a storage path or a `blob:` address it cannot reach,
15
+ * or a signed URL that expires while the conversation still carries it.
16
+ */
17
+ export class ScreenImage {
18
+ /** The long edge Anthropic downscales past; a bigger picture spends upload and latency on pixels nobody reads. */
19
+ static readonly maxEdge = 1568;
20
+ static readonly quality = 0.85;
21
+ static readonly loadMs = 10_000;
22
+
23
+ static readonly #typeByExtension = new Map([
24
+ ["jpg", "image/jpeg"],
25
+ ["jpeg", "image/jpeg"],
26
+ ["png", "image/png"],
27
+ ["gif", "image/gif"],
28
+ ["webp", "image/webp"],
29
+ ]);
30
+
31
+ /** The `<img>` showing `src` in `root`, matched on what readScreen prints, its absolute form, or the attribute. */
32
+ static find(src: string, root?: HTMLElement): HTMLImageElement | null {
33
+ const scope = root ?? (typeof document === "undefined" ? null : document.body);
34
+ if (!scope) return null;
35
+ for (const image of scope.querySelectorAll("img"))
36
+ if (
37
+ (image.currentSrc === src || image.src === src || image.getAttribute("src") === src) &&
38
+ ScreenTarget.visible(image)
39
+ )
40
+ return image;
41
+ return null;
42
+ }
43
+
44
+ static async view(image: HTMLImageElement): Promise<ScreenImageView> {
45
+ const src = image.currentSrc || image.src;
46
+ const name = ScreenImage.#nameOf(image, src);
47
+
48
+ const loaded = image.complete && image.naturalWidth > 0 ? image : await ScreenImage.#load(src, false);
49
+ const drawn =
50
+ (loaded ? await ScreenImage.#draw(loaded, name) : null) ??
51
+ (await ScreenImage.#drawFrom(await ScreenImage.#load(src, true), name));
52
+ if (drawn) return drawn;
53
+ const mimeType = ScreenImage.#typeOf(src);
54
+ if (mimeType && ScreenImage.publiclyFetchable(src))
55
+ return {
56
+ attachment: { name, mimeType, url: src },
57
+ size: `${ScreenImage.#sizeOf(image)}, sent by address because this page may not read its pixels`,
58
+ };
59
+ throw new Error(
60
+ `${name} cannot be viewed: its host does not let this page read the picture, and the model's provider cannot download it from ${src}. Tell the user you cannot see it.`,
61
+ );
62
+ }
63
+
64
+ /**
65
+ * Whether a provider could download `url` itself — an http(s) address off loopback, private, link-local and CGNAT
66
+ * ranges. It decides between an address and no picture at all; it is not an SSRF guard and must not become one.
67
+ */
68
+ static publiclyFetchable(url: string): boolean {
69
+ if (!/^https?:\/\//i.test(url)) return false;
70
+ let hostname: string;
71
+ try {
72
+ hostname = new URL(url).hostname.toLowerCase();
73
+ } catch {
74
+ return false;
75
+ }
76
+ if (hostname === "localhost" || hostname.endsWith(".localhost") || hostname.endsWith(".local")) return false;
77
+ if (hostname === "internal" || hostname.endsWith(".internal")) return false;
78
+ if (hostname.startsWith("[")) return !ScreenImage.#privateV6(hostname.slice(1, -1));
79
+ return !/^(?:0\.|10\.|127\.|169\.254\.|192\.168\.|172\.(?:1[6-9]|2\d|3[01])\.|100\.(?:6[4-9]|[7-9]\d|1[01]\d|12[0-7])\.)/.test(
80
+ hostname,
81
+ );
82
+ }
83
+
84
+ static #privateV6(address: string): boolean {
85
+ if (address === "::" || address === "::1" || address.startsWith("::ffff:")) return true;
86
+ const first = Number.parseInt(address.split(":")[0] || "0", 16);
87
+ return (first & 0xfe00) === 0xfc00 || (first & 0xffc0) === 0xfe80;
88
+ }
89
+
90
+ static async #drawFrom(image: HTMLImageElement | null, name: string): Promise<ScreenImageView | null> {
91
+ return image ? await ScreenImage.#draw(image, name) : null;
92
+ }
93
+
94
+ static async #draw(image: HTMLImageElement, name: string): Promise<ScreenImageView | null> {
95
+ const width = image.naturalWidth || image.width;
96
+ const height = image.naturalHeight || image.height;
97
+ if (!width || !height) return null;
98
+ const scale = Math.min(1, ScreenImage.maxEdge / Math.max(width, height));
99
+ const canvas = document.createElement("canvas");
100
+ canvas.width = Math.max(1, Math.round(width * scale));
101
+ canvas.height = Math.max(1, Math.round(height * scale));
102
+ const context = canvas.getContext("2d");
103
+ if (!context) return null;
104
+
105
+ context.fillStyle = "#ffffff";
106
+ context.fillRect(0, 0, canvas.width, canvas.height);
107
+ context.drawImage(image, 0, 0, canvas.width, canvas.height);
108
+ let blob: Blob | null;
109
+ try {
110
+ blob = await new Promise<Blob | null>((resolve) => canvas.toBlob(resolve, "image/jpeg", ScreenImage.quality));
111
+ } catch {
112
+
113
+ return null;
114
+ }
115
+ if (!blob) return null;
116
+ const bytes = new Uint8Array(await blob.arrayBuffer());
117
+ if (bytes.length > ToolOutput.attachmentBytes)
118
+ throw new Error(`${name} is ${bytes.length} bytes even as a JPEG, past what one tool result may carry.`);
119
+ return {
120
+ attachment: { name, mimeType: "image/jpeg", data: Binary.toBase64(bytes) },
121
+ size: `${width}×${height}, sent as ${canvas.width}×${canvas.height} JPEG`,
122
+ };
123
+ }
124
+
125
+ static #load(src: string, cors: boolean): Promise<HTMLImageElement | null> {
126
+ return new Promise((resolve) => {
127
+ const image = new Image();
128
+ if (cors) image.crossOrigin = "anonymous";
129
+ const timer = setTimeout(() => resolve(null), ScreenImage.loadMs);
130
+ image.onload = () => {
131
+ clearTimeout(timer);
132
+ resolve(image);
133
+ };
134
+ image.onerror = () => {
135
+ clearTimeout(timer);
136
+ resolve(null);
137
+ };
138
+ image.src = src;
139
+ });
140
+ }
141
+
142
+ static #nameOf(image: HTMLImageElement, src: string): string {
143
+ const alt = image.getAttribute("alt")?.trim();
144
+ if (alt) return alt;
145
+ if (!/^https?:/i.test(src)) return "image";
146
+ try {
147
+ return decodeURIComponent(new URL(src).pathname.split("/").pop() ?? "") || "image";
148
+ } catch {
149
+ return "image";
150
+ }
151
+ }
152
+
153
+ static #typeOf(src: string): string | null {
154
+ let pathname: string;
155
+ try {
156
+ pathname = new URL(src).pathname;
157
+ } catch {
158
+ return null;
159
+ }
160
+ return ScreenImage.#typeByExtension.get(pathname.split(".").pop()?.toLowerCase() ?? "") ?? null;
161
+ }
162
+
163
+ static #sizeOf(image: HTMLImageElement): string {
164
+ return image.naturalWidth ? `${image.naturalWidth}×${image.naturalHeight}` : "size unknown";
165
+ }
166
+ }
@@ -1,7 +1,8 @@
1
1
  import { router } from "akanjs/client";
2
- import { AgenticSurface, type SurfaceSource, type ToolEntry } from "../../vendor/use-agentic";
2
+ import { AgenticSurface, type SurfaceSource, type ToolEntry, ToolReturn } from "../../vendor/use-agentic";
3
3
  import { AgentBridge } from "./AgentBridge";
4
4
  import { ScreenFlash } from "./ScreenFlash";
5
+ import { ScreenImage } from "./ScreenImage";
5
6
  import { ScreenReader } from "./ScreenReader";
6
7
  import { ScreenSettle } from "./ScreenSettle";
7
8
  import { ScreenTarget } from "./ScreenTarget";
@@ -9,7 +10,7 @@ import { ScreenTarget } from "./ScreenTarget";
9
10
  /** Every screen's built-in tools. A same-named hook tool shadows one at root scope only; a zone uses `builtins`. */
10
11
  export class StoreSurfaceSource implements SurfaceSource {
11
12
  /** What a session's `builtins` option selects from, in build order. */
12
- static readonly builtins = ["navigate", "goBack", "readScreen", "readState", "highlight"] as const;
13
+ static readonly builtins = ["navigate", "goBack", "readScreen", "readState", "highlight", "viewImage"] as const;
13
14
 
14
15
  #bridge: AgentBridge | null;
15
16
  readonly #builtins = new Map<string, ToolEntry[]>();
@@ -29,6 +30,7 @@ export class StoreSurfaceSource implements SurfaceSource {
29
30
  StoreSurfaceSource.#readScreen(viewKey),
30
31
  this.#readState(viewKey),
31
32
  StoreSurfaceSource.#highlight(viewKey),
33
+ StoreSurfaceSource.#viewImage(viewKey),
32
34
  ];
33
35
  this.#builtins.set(viewKey, builtins);
34
36
  }
@@ -158,6 +160,38 @@ export class StoreSurfaceSource implements SurfaceSource {
158
160
  };
159
161
  }
160
162
 
163
+ static #viewImage(viewKey: string): ToolEntry {
164
+ const absent = (src: string) =>
165
+ `No picture at ${src} is on screen now. Call readScreen with images: true and pass an address exactly as it prints one; a picture that has left the screen cannot be viewed.`;
166
+ return {
167
+ name: "viewImage",
168
+ description:
169
+ "Look at one picture on screen, to answer from what it shows rather than from its caption. Pass an [image] address readScreen printed with `images: true`. Each picture costs this turn over a thousand tokens, so view only the ones the answer needs.",
170
+ parameters: {
171
+ type: "object",
172
+ properties: { src: { type: "string", description: "The image address exactly as readScreen printed it." } },
173
+ required: ["src"],
174
+ additionalProperties: false,
175
+ },
176
+ settle: false,
177
+
178
+ guard: (args) => {
179
+ const src = typeof args.src === "string" ? args.src.trim() : "";
180
+ if (!src) return "viewImage needs a src.";
181
+ if (src.startsWith("data:"))
182
+ return "A data: address is the picture's bytes, not an address readScreen prints. Pass one readScreen printed.";
183
+ return ScreenImage.find(src, StoreSurfaceSource.#zoneRoot(viewKey)) ? true : absent(src);
184
+ },
185
+ run: async (args) => {
186
+ const src = String(args.src).trim();
187
+ const image = ScreenImage.find(src, StoreSurfaceSource.#zoneRoot(viewKey));
188
+ if (!image) throw new Error(absent(src));
189
+ const { attachment, size } = await ScreenImage.view(image);
190
+ return ToolReturn.attach(`Attached ${attachment.name}: ${size}.`, [attachment]);
191
+ },
192
+ };
193
+ }
194
+
161
195
  #readState(viewKey: string): ToolEntry {
162
196
  return {
163
197
  name: "readState",
@@ -1,11 +1,12 @@
1
1
 
2
- export { AgentAbort, AgentProgress, type AgentProgressStep } from "../../vendor/use-agentic";
2
+ export { AgentAbort, AgentProgress, type AgentProgressStep, ToolReturn } from "../../vendor/use-agentic";
3
3
  export * from "./AgentBridge";
4
4
  export * from "./AgentContext";
5
5
  export * from "./AgentCursor";
6
6
  export * from "./AgentPrompts";
7
7
  export * from "./AgentVisual";
8
8
  export * from "./ScreenFlash";
9
+ export * from "./ScreenImage";
9
10
  export * from "./ScreenReader";
10
11
  export * from "./ScreenSettle";
11
12
  export * from "./ScreenTarget";
@@ -31,7 +31,9 @@ export declare class AgentService extends AgentService_base {
31
31
  private static fenced;
32
32
  /** Swaps an attachment the provider cannot read for a note; dropped silently, it is guessed at from its name. */
33
33
  static readable(request: LlmTurnRequest, accepts: LlmAccepts | undefined): LlmTurnRequest;
34
+ private static carriesFiles;
34
35
  private static readableMessage;
36
+ private static readableResult;
35
37
  private static isReadable;
36
38
  private static note;
37
39
  }
@@ -1,4 +1,4 @@
1
- import type { AgentWireAttachment, AgentWireMessage, LlmAccepts, LlmAdaptor, LlmLimits, LlmOption, LlmTurnAnswer, LlmTurnRequest, LlmUsage } from "./llm.adaptor";
1
+ import type { AgentWireAttachment, AgentWireMessage, AgentWireToolResult, LlmAccepts, LlmAdaptor, LlmLimits, LlmOption, LlmTurnAnswer, LlmTurnRequest, LlmUsage } from "./llm.adaptor";
2
2
  type AnthropicSource = {
3
3
  type: "base64";
4
4
  media_type: string;
@@ -7,7 +7,7 @@ type AnthropicSource = {
7
7
  type: "url";
8
8
  url: string;
9
9
  };
10
- type AnthropicBlock = {
10
+ type AnthropicContentBlock = {
11
11
  type: "text";
12
12
  text: string;
13
13
  } | {
@@ -16,7 +16,8 @@ type AnthropicBlock = {
16
16
  } | {
17
17
  type: "document";
18
18
  source: AnthropicSource;
19
- } | {
19
+ };
20
+ type AnthropicBlock = AnthropicContentBlock | {
20
21
  type: "tool_use";
21
22
  id: string;
22
23
  name: string;
@@ -24,7 +25,7 @@ type AnthropicBlock = {
24
25
  } | {
25
26
  type: "tool_result";
26
27
  tool_use_id: string;
27
- content: string;
28
+ content: string | AnthropicContentBlock[];
28
29
  };
29
30
  interface AnthropicMessage {
30
31
  role: "user" | "assistant";
@@ -90,6 +91,10 @@ export declare class AnthropicLlm extends AnthropicLlm_base implements LlmAdapto
90
91
  static providerMessages(messages: AgentWireMessage[], accepts?: LlmAccepts): AnthropicMessage[];
91
92
  static providerMessage(message: AgentWireMessage, accepts?: LlmAccepts): AnthropicMessage;
92
93
  static userContent(message: AgentWireMessage, accepts?: LlmAccepts): AnthropicBlock[];
94
+ /** A string while the call returned no file, so a transcript without one posts what it always did. */
95
+ static toolResult(result: AgentWireToolResult, accepts?: LlmAccepts): AnthropicBlock;
96
+ /** Pushes a note for each file no block can carry, for the caller to fold into the text beside them. */
97
+ static attachmentBlocks(attachments: AgentWireAttachment[], accepts: LlmAccepts | undefined, notes: string[]): AnthropicContentBlock[];
93
98
  static typed(source: AnthropicSource, mimeType: string): AnthropicSource;
94
99
  /** Bytes beat a URL: the default storage serves a path only the app can resolve, so the provider cannot fetch it. */
95
100
  static sourceOf(attachment: AgentWireAttachment): AnthropicSource | null;
@@ -9,6 +9,8 @@ export interface AgentWireToolResult {
9
9
  result?: unknown;
10
10
  changes?: unknown[];
11
11
  error?: string;
12
+ /** Files the call handed the model, as content beside the JSON; one this model cannot read arrives as a `text` note. */
13
+ attachments?: AgentWireAttachment[];
12
14
  }
13
15
  /** Exactly one carrier reaches the model: `data` (inlined bytes), `url` (fetched by the provider) or `text`. */
14
16
  export interface AgentWireAttachment {