akanjs 3.1.0-rc.12 → 3.1.0-rc.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/service/agent.service.ts +21 -1
- package/service/predefinedAdaptor/anthropicLlm.ts +36 -16
- package/service/predefinedAdaptor/llm.adaptor.ts +2 -0
- package/service/predefinedAdaptor/openaiDialect.ts +53 -17
- package/store/action.ts +10 -4
- package/store/agent/ScreenImage.ts +166 -0
- package/store/agent/StoreSurfaceSource.ts +36 -2
- package/store/agent/index.ts +2 -1
- package/types/service/agent.service.d.ts +2 -0
- package/types/service/predefinedAdaptor/anthropicLlm.d.ts +9 -4
- package/types/service/predefinedAdaptor/llm.adaptor.d.ts +2 -0
- package/types/service/predefinedAdaptor/openaiDialect.d.ts +20 -5
- package/types/store/action.d.ts +2 -0
- package/types/store/agent/ScreenImage.d.ts +27 -0
- package/types/store/agent/StoreSurfaceSource.d.ts +1 -1
- package/types/store/agent/index.d.ts +2 -1
- package/types/vendor/use-agentic/Compaction.d.ts +3 -1
- package/types/vendor/use-agentic/ToolOutput.d.ts +6 -1
- package/types/vendor/use-agentic/ToolReturn.d.ts +18 -0
- package/types/vendor/use-agentic/Transcript.d.ts +7 -0
- package/types/vendor/use-agentic/index.d.ts +1 -0
- package/types/vendor/use-agentic/types.d.ts +6 -0
- package/ui/Agent/agentSessionOf.ts +15 -8
- package/ui/Load/Units.tsx +4 -2
- package/vendor/use-agentic/Compaction.ts +38 -7
- package/vendor/use-agentic/ToolOutput.ts +25 -3
- package/vendor/use-agentic/ToolReturn.ts +32 -0
- package/vendor/use-agentic/ToolRunner.ts +15 -13
- package/vendor/use-agentic/Transcript.ts +38 -2
- package/vendor/use-agentic/WIRE.md +20 -0
- package/vendor/use-agentic/index.ts +1 -0
- package/vendor/use-agentic/types.ts +6 -0
- package/webkit/McpAppAgent.ts +12 -2
package/package.json
CHANGED
package/service/agent.service.ts
CHANGED
|
@@ -3,6 +3,7 @@ import type {
|
|
|
3
3
|
AgentWireAttachment,
|
|
4
4
|
AgentWireMessage,
|
|
5
5
|
AgentWireReference,
|
|
6
|
+
AgentWireToolResult,
|
|
6
7
|
LlmAccepts,
|
|
7
8
|
LlmTurnRequest,
|
|
8
9
|
} from "./predefinedAdaptor/llm.adaptor";
|
|
@@ -104,12 +105,21 @@ export class AgentService extends serve("agent" as const, ({ plug }) => ({
|
|
|
104
105
|
|
|
105
106
|
/** Swaps an attachment the provider cannot read for a note; dropped silently, it is guessed at from its name. */
|
|
106
107
|
static readable(request: LlmTurnRequest, accepts: LlmAccepts | undefined): LlmTurnRequest {
|
|
107
|
-
if (!request.messages.some(
|
|
108
|
+
if (!request.messages.some(AgentService.carriesFiles)) return request;
|
|
108
109
|
const messages = request.messages.map((message) => AgentService.readableMessage(message, accepts ?? {}));
|
|
109
110
|
return { ...request, messages };
|
|
110
111
|
}
|
|
111
112
|
|
|
113
|
+
private static carriesFiles(message: AgentWireMessage): boolean {
|
|
114
|
+
return !!message.attachments?.length || !!message.toolResults?.some((result) => result.attachments?.length);
|
|
115
|
+
}
|
|
116
|
+
|
|
112
117
|
private static readableMessage(message: AgentWireMessage, accepts: LlmAccepts): AgentWireMessage {
|
|
118
|
+
if (message.role === "tool")
|
|
119
|
+
return {
|
|
120
|
+
...message,
|
|
121
|
+
toolResults: (message.toolResults ?? []).map((result) => AgentService.readableResult(result, accepts)),
|
|
122
|
+
};
|
|
113
123
|
const { attachments = [], ...rest } = message;
|
|
114
124
|
if (!attachments.length) return message;
|
|
115
125
|
const kept = attachments.filter((attachment) => AgentService.isReadable(attachment, accepts));
|
|
@@ -122,6 +132,16 @@ export class AgentService extends serve("agent" as const, ({ plug }) => ({
|
|
|
122
132
|
};
|
|
123
133
|
}
|
|
124
134
|
|
|
135
|
+
private static readableResult(result: AgentWireToolResult, accepts: LlmAccepts): AgentWireToolResult {
|
|
136
|
+
if (!result.attachments?.length) return result;
|
|
137
|
+
const attachments = result.attachments.map((attachment) =>
|
|
138
|
+
AgentService.isReadable(attachment, accepts)
|
|
139
|
+
? attachment
|
|
140
|
+
: { name: attachment.name, mimeType: attachment.mimeType, text: AgentService.note(attachment) },
|
|
141
|
+
);
|
|
142
|
+
return { ...result, attachments };
|
|
143
|
+
}
|
|
144
|
+
|
|
125
145
|
private static isReadable(attachment: AgentWireAttachment, accepts: LlmAccepts): boolean {
|
|
126
146
|
if (attachment.text) return true;
|
|
127
147
|
if (!attachment.data && !attachment.url) return false;
|
|
@@ -2,6 +2,7 @@ import { adapt } from "../adapt";
|
|
|
2
2
|
import type {
|
|
3
3
|
AgentWireAttachment,
|
|
4
4
|
AgentWireMessage,
|
|
5
|
+
AgentWireToolResult,
|
|
5
6
|
LlmAccepts,
|
|
6
7
|
LlmAdaptor,
|
|
7
8
|
LlmLimits,
|
|
@@ -14,12 +15,14 @@ import { LlmOverflow } from "./llmOverflow";
|
|
|
14
15
|
import { eachSseData } from "./sseData";
|
|
15
16
|
|
|
16
17
|
type AnthropicSource = { type: "base64"; media_type: string; data: string } | { type: "url"; url: string };
|
|
17
|
-
type
|
|
18
|
+
type AnthropicContentBlock =
|
|
18
19
|
| { type: "text"; text: string }
|
|
19
20
|
| { type: "image"; source: AnthropicSource }
|
|
20
|
-
| { type: "document"; source: AnthropicSource }
|
|
21
|
+
| { type: "document"; source: AnthropicSource };
|
|
22
|
+
type AnthropicBlock =
|
|
23
|
+
| AnthropicContentBlock
|
|
21
24
|
| { type: "tool_use"; id: string; name: string; input: Record<string, unknown> }
|
|
22
|
-
| { type: "tool_result"; tool_use_id: string; content: string };
|
|
25
|
+
| { type: "tool_result"; tool_use_id: string; content: string | AnthropicContentBlock[] };
|
|
23
26
|
interface AnthropicMessage {
|
|
24
27
|
role: "user" | "assistant";
|
|
25
28
|
content: AnthropicBlock[];
|
|
@@ -221,15 +224,7 @@ export class AnthropicLlm
|
|
|
221
224
|
if (message.role === "tool")
|
|
222
225
|
return {
|
|
223
226
|
role: "user",
|
|
224
|
-
content: (message.toolResults ?? []).map((result) => (
|
|
225
|
-
type: "tool_result" as const,
|
|
226
|
-
tool_use_id: result.id,
|
|
227
|
-
content: JSON.stringify({
|
|
228
|
-
...(result.result !== undefined ? { result: result.result } : {}),
|
|
229
|
-
...(result.changes?.length ? { changes: result.changes } : {}),
|
|
230
|
-
...(result.error ? { error: result.error } : {}),
|
|
231
|
-
}),
|
|
232
|
-
})),
|
|
227
|
+
content: (message.toolResults ?? []).map((result) => AnthropicLlm.toolResult(result, accepts)),
|
|
233
228
|
};
|
|
234
229
|
if (message.role === "assistant")
|
|
235
230
|
return {
|
|
@@ -248,9 +243,36 @@ export class AnthropicLlm
|
|
|
248
243
|
}
|
|
249
244
|
|
|
250
245
|
static userContent(message: AgentWireMessage, accepts?: LlmAccepts): AnthropicBlock[] {
|
|
251
|
-
const attachments = message.attachments ?? [];
|
|
252
246
|
const notes: string[] = [];
|
|
253
|
-
const blocks =
|
|
247
|
+
const blocks = AnthropicLlm.attachmentBlocks(message.attachments ?? [], accepts, notes);
|
|
248
|
+
const text = [message.text, ...notes].filter(Boolean).join("\n\n");
|
|
249
|
+
return [...(text ? [{ type: "text" as const, text }] : []), ...blocks];
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
/** A string while the call returned no file, so a transcript without one posts what it always did. */
|
|
253
|
+
static toolResult(result: AgentWireToolResult, accepts?: LlmAccepts): AnthropicBlock {
|
|
254
|
+
const payload = JSON.stringify({
|
|
255
|
+
...(result.result !== undefined ? { result: result.result } : {}),
|
|
256
|
+
...(result.changes?.length ? { changes: result.changes } : {}),
|
|
257
|
+
...(result.error ? { error: result.error } : {}),
|
|
258
|
+
});
|
|
259
|
+
if (!result.attachments?.length) return { type: "tool_result", tool_use_id: result.id, content: payload };
|
|
260
|
+
const notes: string[] = [];
|
|
261
|
+
const blocks = AnthropicLlm.attachmentBlocks(result.attachments, accepts, notes);
|
|
262
|
+
return {
|
|
263
|
+
type: "tool_result",
|
|
264
|
+
tool_use_id: result.id,
|
|
265
|
+
content: [{ type: "text", text: [payload, ...notes].join("\n\n") }, ...blocks],
|
|
266
|
+
};
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
/** Pushes a note for each file no block can carry, for the caller to fold into the text beside them. */
|
|
270
|
+
static attachmentBlocks(
|
|
271
|
+
attachments: AgentWireAttachment[],
|
|
272
|
+
accepts: LlmAccepts | undefined,
|
|
273
|
+
notes: string[],
|
|
274
|
+
): AnthropicContentBlock[] {
|
|
275
|
+
return attachments.flatMap((attachment): AnthropicContentBlock[] => {
|
|
254
276
|
if (attachment.text)
|
|
255
277
|
return [
|
|
256
278
|
{
|
|
@@ -271,8 +293,6 @@ export class AnthropicLlm
|
|
|
271
293
|
notes.push(`[Attachment not read: ${attachment.name} (${attachment.mimeType}) — this API has no block for it.]`);
|
|
272
294
|
return [];
|
|
273
295
|
});
|
|
274
|
-
const text = [message.text, ...notes].filter(Boolean).join("\n\n");
|
|
275
|
-
return [...(text ? [{ type: "text" as const, text }] : []), ...blocks];
|
|
276
296
|
}
|
|
277
297
|
|
|
278
298
|
static typed(source: AnthropicSource, mimeType: string): AnthropicSource {
|
|
@@ -10,6 +10,8 @@ export interface AgentWireToolResult {
|
|
|
10
10
|
result?: unknown;
|
|
11
11
|
changes?: unknown[];
|
|
12
12
|
error?: string;
|
|
13
|
+
/** Files the call handed the model, as content beside the JSON; one this model cannot read arrives as a `text` note. */
|
|
14
|
+
attachments?: AgentWireAttachment[];
|
|
13
15
|
}
|
|
14
16
|
|
|
15
17
|
/** Exactly one carrier reaches the model: `data` (inlined bytes), `url` (fetched by the provider) or `text`. */
|
|
@@ -1,4 +1,11 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type {
|
|
2
|
+
AgentWireAttachment,
|
|
3
|
+
AgentWireMessage,
|
|
4
|
+
LlmAccepts,
|
|
5
|
+
LlmTurnAnswer,
|
|
6
|
+
LlmTurnRequest,
|
|
7
|
+
LlmUsage,
|
|
8
|
+
} from "./llm.adaptor";
|
|
2
9
|
import { eachSseData } from "./sseData";
|
|
3
10
|
|
|
4
11
|
export interface OpenaiToolCall {
|
|
@@ -25,7 +32,8 @@ interface OpenaiStreamChunk {
|
|
|
25
32
|
finish_reason?: string | null;
|
|
26
33
|
}[];
|
|
27
34
|
}
|
|
28
|
-
type
|
|
35
|
+
type OpenaiImagePart = { type: "image_url"; image_url: { url: string } };
|
|
36
|
+
type OpenaiContentPart = { type: "text"; text: string } | OpenaiImagePart;
|
|
29
37
|
export interface OpenaiMessage {
|
|
30
38
|
role: "system" | "user" | "assistant" | "tool";
|
|
31
39
|
content: string | OpenaiContentPart[];
|
|
@@ -86,16 +94,7 @@ export class OpenaiDialect {
|
|
|
86
94
|
content: `Summary of the earlier conversation, standing in for the messages it replaced:\n\n${message.text ?? ""}`,
|
|
87
95
|
},
|
|
88
96
|
];
|
|
89
|
-
if (message.role === "tool")
|
|
90
|
-
return (message.toolResults ?? []).map((result) => ({
|
|
91
|
-
role: "tool" as const,
|
|
92
|
-
tool_call_id: result.id,
|
|
93
|
-
content: JSON.stringify({
|
|
94
|
-
...(result.result !== undefined ? { result: result.result } : {}),
|
|
95
|
-
...(result.changes?.length ? { changes: result.changes } : {}),
|
|
96
|
-
...(result.error ? { error: result.error } : {}),
|
|
97
|
-
}),
|
|
98
|
-
}));
|
|
97
|
+
if (message.role === "tool") return OpenaiDialect.toolMessages(message, accepts);
|
|
99
98
|
if (message.role === "assistant")
|
|
100
99
|
return [
|
|
101
100
|
{
|
|
@@ -116,8 +115,47 @@ export class OpenaiDialect {
|
|
|
116
115
|
}
|
|
117
116
|
|
|
118
117
|
static userContent(message: AgentWireMessage, accepts?: LlmAccepts): string | OpenaiContentPart[] {
|
|
119
|
-
const attachments = message.attachments ?? [];
|
|
120
118
|
const notes: string[] = [];
|
|
119
|
+
const { blocks, images } = OpenaiDialect.attachmentParts(message.attachments ?? [], accepts, notes);
|
|
120
|
+
const text = [message.text, ...blocks, ...notes].filter(Boolean).join("\n\n");
|
|
121
|
+
if (!images.length) return text;
|
|
122
|
+
return [...(text ? [{ type: "text" as const, text }] : []), ...images.map(({ part }) => part)];
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
static readonly toolPicturesHeading =
|
|
126
|
+
"Pictures the tool calls above returned, each after the name of the call that returned it. They are content, not something the user said.";
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* A `tool` message carries text alone in this dialect, so the pictures a call returned follow every tool message
|
|
130
|
+
* of the turn in one user message: between two tool messages it would break the run each call's answer has to sit in.
|
|
131
|
+
*/
|
|
132
|
+
static toolMessages(message: AgentWireMessage, accepts?: LlmAccepts): OpenaiMessage[] {
|
|
133
|
+
const shown: OpenaiContentPart[] = [];
|
|
134
|
+
const answers = (message.toolResults ?? []).map((result): OpenaiMessage => {
|
|
135
|
+
const payload = JSON.stringify({
|
|
136
|
+
...(result.result !== undefined ? { result: result.result } : {}),
|
|
137
|
+
...(result.changes?.length ? { changes: result.changes } : {}),
|
|
138
|
+
...(result.error ? { error: result.error } : {}),
|
|
139
|
+
});
|
|
140
|
+
const notes: string[] = [];
|
|
141
|
+
const { blocks, images } = OpenaiDialect.attachmentParts(result.attachments ?? [], accepts, notes);
|
|
142
|
+
for (const { name, part } of images)
|
|
143
|
+
shown.push({ type: "text", text: `[${result.name} returned ${name}]` }, part);
|
|
144
|
+
return { role: "tool", tool_call_id: result.id, content: [payload, ...blocks, ...notes].join("\n\n") };
|
|
145
|
+
});
|
|
146
|
+
if (!shown.length) return answers;
|
|
147
|
+
return [
|
|
148
|
+
...answers,
|
|
149
|
+
{ role: "user", content: [{ type: "text", text: OpenaiDialect.toolPicturesHeading }, ...shown] },
|
|
150
|
+
];
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
/** Pushes a note for each file no part can carry, for the caller to fold into the text beside them. */
|
|
154
|
+
static attachmentParts(
|
|
155
|
+
attachments: AgentWireAttachment[],
|
|
156
|
+
accepts: LlmAccepts | undefined,
|
|
157
|
+
notes: string[],
|
|
158
|
+
): { blocks: string[]; images: { name: string; part: OpenaiImagePart }[] } {
|
|
121
159
|
|
|
122
160
|
const blocks = attachments.flatMap((attachment) =>
|
|
123
161
|
attachment.text ? [`--- attachment: ${attachment.name} (${attachment.mimeType}) ---\n${attachment.text}`] : [],
|
|
@@ -138,11 +176,9 @@ export class OpenaiDialect {
|
|
|
138
176
|
}
|
|
139
177
|
|
|
140
178
|
const url = attachment.data ? `data:${mimeType};base64,${attachment.data}` : (attachment.url ?? "");
|
|
141
|
-
return url ? [{ type: "image_url" as const, image_url: { url } }] : [];
|
|
179
|
+
return url ? [{ name: attachment.name, part: { type: "image_url" as const, image_url: { url } } }] : [];
|
|
142
180
|
});
|
|
143
|
-
|
|
144
|
-
if (!images.length) return text;
|
|
145
|
-
return [...(text ? [{ type: "text" as const, text }] : []), ...images];
|
|
181
|
+
return { blocks, images };
|
|
146
182
|
}
|
|
147
183
|
|
|
148
184
|
static async consumeStream(
|
package/store/action.ts
CHANGED
|
@@ -136,7 +136,7 @@ type BaseAction<
|
|
|
136
136
|
) => Promise<void>;
|
|
137
137
|
} & { [K in `set${_CapitalizedRefName}`]: (...models: (Full | Light)[]) => void } & {
|
|
138
138
|
[K in `reset${_CapitalizedRefName}`]: (model?: Full) => void;
|
|
139
|
-
};
|
|
139
|
+
} & { [K in `invalidate${_CapitalizedRefName}`]: () => void };
|
|
140
140
|
|
|
141
141
|
export type SliceAction<
|
|
142
142
|
RefName extends string,
|
|
@@ -543,6 +543,7 @@ export const makeActions = (refName: string, slice: { [key: string]: SerializedS
|
|
|
543
543
|
viewModel: `view${className}`,
|
|
544
544
|
setModel: `set${className}`,
|
|
545
545
|
resetModel: `reset${className}`,
|
|
546
|
+
invalidateModel: `invalidate${className}`,
|
|
546
547
|
loadModelDraft: `load${className}FormDraft`,
|
|
547
548
|
restoreModelDraft: `restore${className}FormDraft`,
|
|
548
549
|
discardModelDraft: `discard${className}FormDraft`,
|
|
@@ -905,6 +906,9 @@ export const makeActions = (refName: string, slice: { [key: string]: SerializedS
|
|
|
905
906
|
});
|
|
906
907
|
return model ?? null;
|
|
907
908
|
},
|
|
909
|
+
[names.invalidateModel]: function (this: SetGet) {
|
|
910
|
+
this.set(staleAtOfSlices());
|
|
911
|
+
},
|
|
908
912
|
|
|
909
913
|
[names.loadModelDraft]: async function (this: SetGet, draftScope?: string) {
|
|
910
914
|
const currentState = this.get() as { [key: string]: any };
|
|
@@ -1033,6 +1037,8 @@ export const makeActions = (refName: string, slice: { [key: string]: SerializedS
|
|
|
1033
1037
|
isQueryEqual(sort as unknown as object, sortOfModel as unknown as object)
|
|
1034
1038
|
)
|
|
1035
1039
|
return;
|
|
1040
|
+
|
|
1041
|
+
const requestedAt = new Date();
|
|
1036
1042
|
await loadList(
|
|
1037
1043
|
this,
|
|
1038
1044
|
() =>
|
|
@@ -1049,7 +1055,7 @@ export const makeActions = (refName: string, slice: { [key: string]: SerializedS
|
|
|
1049
1055
|
[namesOfSlice.modelList]: modelList,
|
|
1050
1056
|
[namesOfSlice.modelInsight]: modelInsight,
|
|
1051
1057
|
[namesOfSlice.modelInitList]: modelList,
|
|
1052
|
-
[namesOfSlice.modelInitAt]:
|
|
1058
|
+
[namesOfSlice.modelInitAt]: requestedAt,
|
|
1053
1059
|
...lastPageFrom(modelInsight.count, limit),
|
|
1054
1060
|
...hasMoreFrom(modelDataList, fetchLimit),
|
|
1055
1061
|
[namesOfSlice.limitOfModel]: limit,
|
|
@@ -1209,10 +1215,10 @@ export const makeActions = (refName: string, slice: { [key: string]: SerializedS
|
|
|
1209
1215
|
|
|
1210
1216
|
[namesOfSlice.applyLiveModel]: function (this: SetGet, event: LiveEventPayload) {
|
|
1211
1217
|
const currentState = this.get() as { [key: string]: any };
|
|
1212
|
-
if (currentState[namesOfSlice.modelListLoading] as boolean) return;
|
|
1213
1218
|
const modelList = currentState[namesOfSlice.modelList] as DataList<Light>;
|
|
1214
1219
|
const staleAt = { [namesOfSlice.modelStaleAt]: new Date() };
|
|
1215
|
-
|
|
1220
|
+
|
|
1221
|
+
if (event.op === "invalidate" || (currentState[namesOfSlice.modelListLoading] as boolean)) {
|
|
1216
1222
|
this.set(staleAt);
|
|
1217
1223
|
return;
|
|
1218
1224
|
}
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
import { Binary } from "akanjs/base";
|
|
2
|
+
import { type MessageAttachment, ToolOutput } from "../../vendor/use-agentic";
|
|
3
|
+
import { ScreenTarget } from "./ScreenTarget";
|
|
4
|
+
|
|
5
|
+
export interface ScreenImageView {
|
|
6
|
+
attachment: MessageAttachment;
|
|
7
|
+
/** The picture's size and how it was sent, for a model asked about its proportions or resolution. */
|
|
8
|
+
size: string;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* A picture on the screen, handed to the model as the browser drew it: redrawn into a JPEG no longer than
|
|
13
|
+
* `maxEdge`, inlined as bytes. Redrawing turns HEIC, AVIF and SVG into a type every provider reads, bounds what one
|
|
14
|
+
* picture costs, and leaves the provider nothing to download — a storage path or a `blob:` address it cannot reach,
|
|
15
|
+
* or a signed URL that expires while the conversation still carries it.
|
|
16
|
+
*/
|
|
17
|
+
export class ScreenImage {
|
|
18
|
+
/** The long edge Anthropic downscales past; a bigger picture spends upload and latency on pixels nobody reads. */
|
|
19
|
+
static readonly maxEdge = 1568;
|
|
20
|
+
static readonly quality = 0.85;
|
|
21
|
+
static readonly loadMs = 10_000;
|
|
22
|
+
|
|
23
|
+
static readonly #typeByExtension = new Map([
|
|
24
|
+
["jpg", "image/jpeg"],
|
|
25
|
+
["jpeg", "image/jpeg"],
|
|
26
|
+
["png", "image/png"],
|
|
27
|
+
["gif", "image/gif"],
|
|
28
|
+
["webp", "image/webp"],
|
|
29
|
+
]);
|
|
30
|
+
|
|
31
|
+
/** The `<img>` showing `src` in `root`, matched on what readScreen prints, its absolute form, or the attribute. */
|
|
32
|
+
static find(src: string, root?: HTMLElement): HTMLImageElement | null {
|
|
33
|
+
const scope = root ?? (typeof document === "undefined" ? null : document.body);
|
|
34
|
+
if (!scope) return null;
|
|
35
|
+
for (const image of scope.querySelectorAll("img"))
|
|
36
|
+
if (
|
|
37
|
+
(image.currentSrc === src || image.src === src || image.getAttribute("src") === src) &&
|
|
38
|
+
ScreenTarget.visible(image)
|
|
39
|
+
)
|
|
40
|
+
return image;
|
|
41
|
+
return null;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
static async view(image: HTMLImageElement): Promise<ScreenImageView> {
|
|
45
|
+
const src = image.currentSrc || image.src;
|
|
46
|
+
const name = ScreenImage.#nameOf(image, src);
|
|
47
|
+
|
|
48
|
+
const loaded = image.complete && image.naturalWidth > 0 ? image : await ScreenImage.#load(src, false);
|
|
49
|
+
const drawn =
|
|
50
|
+
(loaded ? await ScreenImage.#draw(loaded, name) : null) ??
|
|
51
|
+
(await ScreenImage.#drawFrom(await ScreenImage.#load(src, true), name));
|
|
52
|
+
if (drawn) return drawn;
|
|
53
|
+
const mimeType = ScreenImage.#typeOf(src);
|
|
54
|
+
if (mimeType && ScreenImage.publiclyFetchable(src))
|
|
55
|
+
return {
|
|
56
|
+
attachment: { name, mimeType, url: src },
|
|
57
|
+
size: `${ScreenImage.#sizeOf(image)}, sent by address because this page may not read its pixels`,
|
|
58
|
+
};
|
|
59
|
+
throw new Error(
|
|
60
|
+
`${name} cannot be viewed: its host does not let this page read the picture, and the model's provider cannot download it from ${src}. Tell the user you cannot see it.`,
|
|
61
|
+
);
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Whether a provider could download `url` itself — an http(s) address off loopback, private, link-local and CGNAT
|
|
66
|
+
* ranges. It decides between an address and no picture at all; it is not an SSRF guard and must not become one.
|
|
67
|
+
*/
|
|
68
|
+
static publiclyFetchable(url: string): boolean {
|
|
69
|
+
if (!/^https?:\/\//i.test(url)) return false;
|
|
70
|
+
let hostname: string;
|
|
71
|
+
try {
|
|
72
|
+
hostname = new URL(url).hostname.toLowerCase();
|
|
73
|
+
} catch {
|
|
74
|
+
return false;
|
|
75
|
+
}
|
|
76
|
+
if (hostname === "localhost" || hostname.endsWith(".localhost") || hostname.endsWith(".local")) return false;
|
|
77
|
+
if (hostname === "internal" || hostname.endsWith(".internal")) return false;
|
|
78
|
+
if (hostname.startsWith("[")) return !ScreenImage.#privateV6(hostname.slice(1, -1));
|
|
79
|
+
return !/^(?:0\.|10\.|127\.|169\.254\.|192\.168\.|172\.(?:1[6-9]|2\d|3[01])\.|100\.(?:6[4-9]|[7-9]\d|1[01]\d|12[0-7])\.)/.test(
|
|
80
|
+
hostname,
|
|
81
|
+
);
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
static #privateV6(address: string): boolean {
|
|
85
|
+
if (address === "::" || address === "::1" || address.startsWith("::ffff:")) return true;
|
|
86
|
+
const first = Number.parseInt(address.split(":")[0] || "0", 16);
|
|
87
|
+
return (first & 0xfe00) === 0xfc00 || (first & 0xffc0) === 0xfe80;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
static async #drawFrom(image: HTMLImageElement | null, name: string): Promise<ScreenImageView | null> {
|
|
91
|
+
return image ? await ScreenImage.#draw(image, name) : null;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
static async #draw(image: HTMLImageElement, name: string): Promise<ScreenImageView | null> {
|
|
95
|
+
const width = image.naturalWidth || image.width;
|
|
96
|
+
const height = image.naturalHeight || image.height;
|
|
97
|
+
if (!width || !height) return null;
|
|
98
|
+
const scale = Math.min(1, ScreenImage.maxEdge / Math.max(width, height));
|
|
99
|
+
const canvas = document.createElement("canvas");
|
|
100
|
+
canvas.width = Math.max(1, Math.round(width * scale));
|
|
101
|
+
canvas.height = Math.max(1, Math.round(height * scale));
|
|
102
|
+
const context = canvas.getContext("2d");
|
|
103
|
+
if (!context) return null;
|
|
104
|
+
|
|
105
|
+
context.fillStyle = "#ffffff";
|
|
106
|
+
context.fillRect(0, 0, canvas.width, canvas.height);
|
|
107
|
+
context.drawImage(image, 0, 0, canvas.width, canvas.height);
|
|
108
|
+
let blob: Blob | null;
|
|
109
|
+
try {
|
|
110
|
+
blob = await new Promise<Blob | null>((resolve) => canvas.toBlob(resolve, "image/jpeg", ScreenImage.quality));
|
|
111
|
+
} catch {
|
|
112
|
+
|
|
113
|
+
return null;
|
|
114
|
+
}
|
|
115
|
+
if (!blob) return null;
|
|
116
|
+
const bytes = new Uint8Array(await blob.arrayBuffer());
|
|
117
|
+
if (bytes.length > ToolOutput.attachmentBytes)
|
|
118
|
+
throw new Error(`${name} is ${bytes.length} bytes even as a JPEG, past what one tool result may carry.`);
|
|
119
|
+
return {
|
|
120
|
+
attachment: { name, mimeType: "image/jpeg", data: Binary.toBase64(bytes) },
|
|
121
|
+
size: `${width}×${height}, sent as ${canvas.width}×${canvas.height} JPEG`,
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
static #load(src: string, cors: boolean): Promise<HTMLImageElement | null> {
|
|
126
|
+
return new Promise((resolve) => {
|
|
127
|
+
const image = new Image();
|
|
128
|
+
if (cors) image.crossOrigin = "anonymous";
|
|
129
|
+
const timer = setTimeout(() => resolve(null), ScreenImage.loadMs);
|
|
130
|
+
image.onload = () => {
|
|
131
|
+
clearTimeout(timer);
|
|
132
|
+
resolve(image);
|
|
133
|
+
};
|
|
134
|
+
image.onerror = () => {
|
|
135
|
+
clearTimeout(timer);
|
|
136
|
+
resolve(null);
|
|
137
|
+
};
|
|
138
|
+
image.src = src;
|
|
139
|
+
});
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
static #nameOf(image: HTMLImageElement, src: string): string {
|
|
143
|
+
const alt = image.getAttribute("alt")?.trim();
|
|
144
|
+
if (alt) return alt;
|
|
145
|
+
if (!/^https?:/i.test(src)) return "image";
|
|
146
|
+
try {
|
|
147
|
+
return decodeURIComponent(new URL(src).pathname.split("/").pop() ?? "") || "image";
|
|
148
|
+
} catch {
|
|
149
|
+
return "image";
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
static #typeOf(src: string): string | null {
|
|
154
|
+
let pathname: string;
|
|
155
|
+
try {
|
|
156
|
+
pathname = new URL(src).pathname;
|
|
157
|
+
} catch {
|
|
158
|
+
return null;
|
|
159
|
+
}
|
|
160
|
+
return ScreenImage.#typeByExtension.get(pathname.split(".").pop()?.toLowerCase() ?? "") ?? null;
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
static #sizeOf(image: HTMLImageElement): string {
|
|
164
|
+
return image.naturalWidth ? `${image.naturalWidth}×${image.naturalHeight}` : "size unknown";
|
|
165
|
+
}
|
|
166
|
+
}
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { router } from "akanjs/client";
|
|
2
|
-
import { AgenticSurface, type SurfaceSource, type ToolEntry } from "../../vendor/use-agentic";
|
|
2
|
+
import { AgenticSurface, type SurfaceSource, type ToolEntry, ToolReturn } from "../../vendor/use-agentic";
|
|
3
3
|
import { AgentBridge } from "./AgentBridge";
|
|
4
4
|
import { ScreenFlash } from "./ScreenFlash";
|
|
5
|
+
import { ScreenImage } from "./ScreenImage";
|
|
5
6
|
import { ScreenReader } from "./ScreenReader";
|
|
6
7
|
import { ScreenSettle } from "./ScreenSettle";
|
|
7
8
|
import { ScreenTarget } from "./ScreenTarget";
|
|
@@ -9,7 +10,7 @@ import { ScreenTarget } from "./ScreenTarget";
|
|
|
9
10
|
/** Every screen's built-in tools. A same-named hook tool shadows one at root scope only; a zone uses `builtins`. */
|
|
10
11
|
export class StoreSurfaceSource implements SurfaceSource {
|
|
11
12
|
/** What a session's `builtins` option selects from, in build order. */
|
|
12
|
-
static readonly builtins = ["navigate", "goBack", "readScreen", "readState", "highlight"] as const;
|
|
13
|
+
static readonly builtins = ["navigate", "goBack", "readScreen", "readState", "highlight", "viewImage"] as const;
|
|
13
14
|
|
|
14
15
|
#bridge: AgentBridge | null;
|
|
15
16
|
readonly #builtins = new Map<string, ToolEntry[]>();
|
|
@@ -29,6 +30,7 @@ export class StoreSurfaceSource implements SurfaceSource {
|
|
|
29
30
|
StoreSurfaceSource.#readScreen(viewKey),
|
|
30
31
|
this.#readState(viewKey),
|
|
31
32
|
StoreSurfaceSource.#highlight(viewKey),
|
|
33
|
+
StoreSurfaceSource.#viewImage(viewKey),
|
|
32
34
|
];
|
|
33
35
|
this.#builtins.set(viewKey, builtins);
|
|
34
36
|
}
|
|
@@ -158,6 +160,38 @@ export class StoreSurfaceSource implements SurfaceSource {
|
|
|
158
160
|
};
|
|
159
161
|
}
|
|
160
162
|
|
|
163
|
+
static #viewImage(viewKey: string): ToolEntry {
|
|
164
|
+
const absent = (src: string) =>
|
|
165
|
+
`No picture at ${src} is on screen now. Call readScreen with images: true and pass an address exactly as it prints one; a picture that has left the screen cannot be viewed.`;
|
|
166
|
+
return {
|
|
167
|
+
name: "viewImage",
|
|
168
|
+
description:
|
|
169
|
+
"Look at one picture on screen, to answer from what it shows rather than from its caption. Pass an [image] address readScreen printed with `images: true`. Each picture costs this turn over a thousand tokens, so view only the ones the answer needs.",
|
|
170
|
+
parameters: {
|
|
171
|
+
type: "object",
|
|
172
|
+
properties: { src: { type: "string", description: "The image address exactly as readScreen printed it." } },
|
|
173
|
+
required: ["src"],
|
|
174
|
+
additionalProperties: false,
|
|
175
|
+
},
|
|
176
|
+
settle: false,
|
|
177
|
+
|
|
178
|
+
guard: (args) => {
|
|
179
|
+
const src = typeof args.src === "string" ? args.src.trim() : "";
|
|
180
|
+
if (!src) return "viewImage needs a src.";
|
|
181
|
+
if (src.startsWith("data:"))
|
|
182
|
+
return "A data: address is the picture's bytes, not an address readScreen prints. Pass one readScreen printed.";
|
|
183
|
+
return ScreenImage.find(src, StoreSurfaceSource.#zoneRoot(viewKey)) ? true : absent(src);
|
|
184
|
+
},
|
|
185
|
+
run: async (args) => {
|
|
186
|
+
const src = String(args.src).trim();
|
|
187
|
+
const image = ScreenImage.find(src, StoreSurfaceSource.#zoneRoot(viewKey));
|
|
188
|
+
if (!image) throw new Error(absent(src));
|
|
189
|
+
const { attachment, size } = await ScreenImage.view(image);
|
|
190
|
+
return ToolReturn.attach(`Attached ${attachment.name}: ${size}.`, [attachment]);
|
|
191
|
+
},
|
|
192
|
+
};
|
|
193
|
+
}
|
|
194
|
+
|
|
161
195
|
#readState(viewKey: string): ToolEntry {
|
|
162
196
|
return {
|
|
163
197
|
name: "readState",
|
package/store/agent/index.ts
CHANGED
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
|
|
2
|
-
export { AgentAbort, AgentProgress, type AgentProgressStep } from "../../vendor/use-agentic";
|
|
2
|
+
export { AgentAbort, AgentProgress, type AgentProgressStep, ToolReturn } from "../../vendor/use-agentic";
|
|
3
3
|
export * from "./AgentBridge";
|
|
4
4
|
export * from "./AgentContext";
|
|
5
5
|
export * from "./AgentCursor";
|
|
6
6
|
export * from "./AgentPrompts";
|
|
7
7
|
export * from "./AgentVisual";
|
|
8
8
|
export * from "./ScreenFlash";
|
|
9
|
+
export * from "./ScreenImage";
|
|
9
10
|
export * from "./ScreenReader";
|
|
10
11
|
export * from "./ScreenSettle";
|
|
11
12
|
export * from "./ScreenTarget";
|
|
@@ -31,7 +31,9 @@ export declare class AgentService extends AgentService_base {
|
|
|
31
31
|
private static fenced;
|
|
32
32
|
/** Swaps an attachment the provider cannot read for a note; dropped silently, it is guessed at from its name. */
|
|
33
33
|
static readable(request: LlmTurnRequest, accepts: LlmAccepts | undefined): LlmTurnRequest;
|
|
34
|
+
private static carriesFiles;
|
|
34
35
|
private static readableMessage;
|
|
36
|
+
private static readableResult;
|
|
35
37
|
private static isReadable;
|
|
36
38
|
private static note;
|
|
37
39
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { AgentWireAttachment, AgentWireMessage, LlmAccepts, LlmAdaptor, LlmLimits, LlmOption, LlmTurnAnswer, LlmTurnRequest, LlmUsage } from "./llm.adaptor";
|
|
1
|
+
import type { AgentWireAttachment, AgentWireMessage, AgentWireToolResult, LlmAccepts, LlmAdaptor, LlmLimits, LlmOption, LlmTurnAnswer, LlmTurnRequest, LlmUsage } from "./llm.adaptor";
|
|
2
2
|
type AnthropicSource = {
|
|
3
3
|
type: "base64";
|
|
4
4
|
media_type: string;
|
|
@@ -7,7 +7,7 @@ type AnthropicSource = {
|
|
|
7
7
|
type: "url";
|
|
8
8
|
url: string;
|
|
9
9
|
};
|
|
10
|
-
type
|
|
10
|
+
type AnthropicContentBlock = {
|
|
11
11
|
type: "text";
|
|
12
12
|
text: string;
|
|
13
13
|
} | {
|
|
@@ -16,7 +16,8 @@ type AnthropicBlock = {
|
|
|
16
16
|
} | {
|
|
17
17
|
type: "document";
|
|
18
18
|
source: AnthropicSource;
|
|
19
|
-
}
|
|
19
|
+
};
|
|
20
|
+
type AnthropicBlock = AnthropicContentBlock | {
|
|
20
21
|
type: "tool_use";
|
|
21
22
|
id: string;
|
|
22
23
|
name: string;
|
|
@@ -24,7 +25,7 @@ type AnthropicBlock = {
|
|
|
24
25
|
} | {
|
|
25
26
|
type: "tool_result";
|
|
26
27
|
tool_use_id: string;
|
|
27
|
-
content: string;
|
|
28
|
+
content: string | AnthropicContentBlock[];
|
|
28
29
|
};
|
|
29
30
|
interface AnthropicMessage {
|
|
30
31
|
role: "user" | "assistant";
|
|
@@ -90,6 +91,10 @@ export declare class AnthropicLlm extends AnthropicLlm_base implements LlmAdapto
|
|
|
90
91
|
static providerMessages(messages: AgentWireMessage[], accepts?: LlmAccepts): AnthropicMessage[];
|
|
91
92
|
static providerMessage(message: AgentWireMessage, accepts?: LlmAccepts): AnthropicMessage;
|
|
92
93
|
static userContent(message: AgentWireMessage, accepts?: LlmAccepts): AnthropicBlock[];
|
|
94
|
+
/** A string while the call returned no file, so a transcript without one posts what it always did. */
|
|
95
|
+
static toolResult(result: AgentWireToolResult, accepts?: LlmAccepts): AnthropicBlock;
|
|
96
|
+
/** Pushes a note for each file no block can carry, for the caller to fold into the text beside them. */
|
|
97
|
+
static attachmentBlocks(attachments: AgentWireAttachment[], accepts: LlmAccepts | undefined, notes: string[]): AnthropicContentBlock[];
|
|
93
98
|
static typed(source: AnthropicSource, mimeType: string): AnthropicSource;
|
|
94
99
|
/** Bytes beat a URL: the default storage serves a path only the app can resolve, so the provider cannot fetch it. */
|
|
95
100
|
static sourceOf(attachment: AgentWireAttachment): AnthropicSource | null;
|
|
@@ -9,6 +9,8 @@ export interface AgentWireToolResult {
|
|
|
9
9
|
result?: unknown;
|
|
10
10
|
changes?: unknown[];
|
|
11
11
|
error?: string;
|
|
12
|
+
/** Files the call handed the model, as content beside the JSON; one this model cannot read arrives as a `text` note. */
|
|
13
|
+
attachments?: AgentWireAttachment[];
|
|
12
14
|
}
|
|
13
15
|
/** Exactly one carrier reaches the model: `data` (inlined bytes), `url` (fetched by the provider) or `text`. */
|
|
14
16
|
export interface AgentWireAttachment {
|