@coolkiller007/my-page-agent 0.2.7 → 0.2.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +43 -12
- package/dist/index.js +214 -34
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.d.ts
CHANGED
|
@@ -44,7 +44,7 @@ declare interface AgentConfig extends LLMConfig {
|
|
|
44
44
|
maxSteps?: number;
|
|
45
45
|
/**
|
|
46
46
|
* 拼装 LLM 提示词时,最近保留完整详情的历史步数;更早的步骤会被压缩成一行。
|
|
47
|
-
* @default
|
|
47
|
+
* @default 5
|
|
48
48
|
*/
|
|
49
49
|
historyWindow?: number;
|
|
50
50
|
/**
|
|
@@ -264,17 +264,14 @@ declare class AgentRuntime extends EventTarget {
|
|
|
264
264
|
attachments?: Attachment[];
|
|
265
265
|
}>;
|
|
266
266
|
/**
|
|
267
|
-
*
|
|
268
|
-
*
|
|
269
|
-
*/
|
|
270
|
-
pendingAnswerAttachments: Attachment[];
|
|
271
|
-
/**
|
|
272
|
-
* 图片附件缓存(按附件名索引):所有出现过的图片附件在此登记原始 dataUrl,
|
|
273
|
-
* 使原图在 prompt 里只出现一次之后,仍可被 `image_understand` 工具按名称"按需重传"重新查看,
|
|
267
|
+
* 图片附件缓存(按稳定附件 ID 索引):所有出现过的图片附件在此登记原始 dataUrl,
|
|
268
|
+
* 使原图在 prompt 里只出现一次之后,仍可被 `image_understand` 工具按稳定 ID“按需重传”重新查看,
|
|
274
269
|
* 而不必每步都把原图重新塞进主对话历史(节省 token)。
|
|
275
|
-
*
|
|
270
|
+
* @note 跨任务持久保留(不随 `execute()` 清空):缓存本身是纯内存 Map,不进 prompt 就不占 token,
|
|
271
|
+
* 只有模型主动调用 `image_understand` 时才会产生一次性 token 成本。唯一真实开销是内存,
|
|
272
|
+
* 因此按 `MAX_CACHED_IMAGES` 做容量上限,超出时淘汰最久未读取的一张。
|
|
276
273
|
*/
|
|
277
|
-
imageAttachmentCache: Map<string,
|
|
274
|
+
readonly imageAttachmentCache: Map<string, StoredImageAttachment>;
|
|
278
275
|
/** 构造函数:初始化配置、LLM、工具集,并注册重试事件监听与注入自定义工具 */
|
|
279
276
|
constructor(config: AgentRuntimeConfig);
|
|
280
277
|
/** 获取当前 Agent 状态 */
|
|
@@ -292,15 +289,17 @@ declare class AgentRuntime extends EventTarget {
|
|
|
292
289
|
* Agent 内部错误会被捕获并加入历史,同时返回失败结果
|
|
293
290
|
*/
|
|
294
291
|
execute(task: string, attachments?: Attachment[]): Promise<ExecutionResult>;
|
|
292
|
+
/* Excluded from this release type: setPendingAnswerAttachments */
|
|
293
|
+
/* Excluded from this release type: listAvailableAttachments */
|
|
295
294
|
/**
|
|
296
295
|
* 针对已缓存的图片附件发起一次独立的视觉问答,不进入主对话历史、不携带主链路的历史上下文。
|
|
297
296
|
* 供 `image_understand` 工具使用:模型需要复看图片细节时按需调用,而不是每步都重发原图。
|
|
298
|
-
* @param
|
|
297
|
+
* @param attachmentIdOrName - 优先使用稳定附件 ID;为兼容旧调用也接受文件名
|
|
299
298
|
* @param query - 需要从图片中确认的具体问题
|
|
300
299
|
* @param signal - 取消信号
|
|
301
300
|
* @returns 视觉模型针对 query 给出的文字答案;附件不存在时返回错误提示
|
|
302
301
|
*/
|
|
303
|
-
describeImage(
|
|
302
|
+
describeImage(attachmentIdOrName: string, query: string, signal: AbortSignal): Promise<string>;
|
|
304
303
|
/** 销毁 Agent:中止任务、释放浏览器控制器,并触发 dispose 事件(一次性) */
|
|
305
304
|
dispose(): void;
|
|
306
305
|
}
|
|
@@ -325,6 +324,8 @@ declare interface AgentStepEvent {
|
|
|
325
324
|
name: string;
|
|
326
325
|
input: any;
|
|
327
326
|
output: string;
|
|
327
|
+
attachmentNames?: string[];
|
|
328
|
+
attachments?: AttachmentReference[];
|
|
328
329
|
};
|
|
329
330
|
usage: {
|
|
330
331
|
promptTokens: number;
|
|
@@ -367,6 +368,13 @@ declare interface Attachment_2 {
|
|
|
367
368
|
dataUrl: string;
|
|
368
369
|
}
|
|
369
370
|
|
|
371
|
+
/** 历史与 prompt 中使用的轻量图片引用,不包含原图数据 */
|
|
372
|
+
declare interface AttachmentReference {
|
|
373
|
+
id: string;
|
|
374
|
+
name: string;
|
|
375
|
+
mimeType: string;
|
|
376
|
+
}
|
|
377
|
+
|
|
370
378
|
/**
|
|
371
379
|
* BrowserController 管理 DOM 状态和元素交互。
|
|
372
380
|
* 它为所有 DOM 操作提供异步方法,并保持状态隔离。
|
|
@@ -795,6 +803,16 @@ declare interface RetryEvent {
|
|
|
795
803
|
maxAttempts: number;
|
|
796
804
|
}
|
|
797
805
|
|
|
806
|
+
/** 会话级图片仓库记录;原图仅保存在应用内存中,不进入普通任务 prompt */
|
|
807
|
+
declare interface StoredImageAttachment extends AttachmentReference {
|
|
808
|
+
dataUrl: string;
|
|
809
|
+
taskId: string;
|
|
810
|
+
task: string;
|
|
811
|
+
source: 'task' | 'answer';
|
|
812
|
+
createdAt: number;
|
|
813
|
+
lastAccessedAt: number;
|
|
814
|
+
}
|
|
815
|
+
|
|
798
816
|
/** 支持的 UI 语言 */
|
|
799
817
|
declare type SupportedLanguage = 'en-US' | 'zh-CN';
|
|
800
818
|
|
|
@@ -810,6 +828,7 @@ declare interface TaskStartEvent {
|
|
|
810
828
|
type: 'task_start';
|
|
811
829
|
task: string;
|
|
812
830
|
attachmentNames?: string[];
|
|
831
|
+
attachments?: AttachmentReference[];
|
|
813
832
|
}
|
|
814
833
|
|
|
815
834
|
declare interface TextDomNode {
|
|
@@ -923,6 +942,12 @@ declare interface UIAdapter extends EventTarget {
|
|
|
923
942
|
name: string;
|
|
924
943
|
input: unknown;
|
|
925
944
|
output: string;
|
|
945
|
+
attachmentNames?: string[];
|
|
946
|
+
attachments?: {
|
|
947
|
+
id: string;
|
|
948
|
+
name: string;
|
|
949
|
+
mimeType: string;
|
|
950
|
+
}[];
|
|
926
951
|
};
|
|
927
952
|
/** 仅用于 'observation' 类型 */
|
|
928
953
|
content?: string;
|
|
@@ -930,6 +955,12 @@ declare interface UIAdapter extends EventTarget {
|
|
|
930
955
|
task?: string;
|
|
931
956
|
/** 仅用于 'task_start' 类型 */
|
|
932
957
|
attachmentNames?: string[];
|
|
958
|
+
/** 仅用于 'task_start' 类型 */
|
|
959
|
+
attachments?: {
|
|
960
|
+
id: string;
|
|
961
|
+
name: string;
|
|
962
|
+
mimeType: string;
|
|
963
|
+
}[];
|
|
933
964
|
/** 仅用于 'retry' 类型 */
|
|
934
965
|
attempt?: number;
|
|
935
966
|
maxAttempts?: number;
|
package/dist/index.js
CHANGED
|
@@ -3494,6 +3494,7 @@ var UI = class {
|
|
|
3494
3494
|
} else if (action.name === "ask_user") {
|
|
3495
3495
|
const input = action.input;
|
|
3496
3496
|
const answer = action.output.replace(/^User answered:\s*/i, "");
|
|
3497
|
+
const answerContent = action.attachmentNames?.length ? [`Answer: ${answer}`, this.#i18n.t("ui.attachmentsLabel", { names: action.attachmentNames.join(", ") })] : `Answer: ${answer}`;
|
|
3497
3498
|
cards.push(createCard({
|
|
3498
3499
|
icon: "❓",
|
|
3499
3500
|
content: `Question: ${input.question || ""}`,
|
|
@@ -3502,7 +3503,7 @@ var UI = class {
|
|
|
3502
3503
|
}));
|
|
3503
3504
|
cards.push(createCard({
|
|
3504
3505
|
icon: "💬",
|
|
3505
|
-
content:
|
|
3506
|
+
content: answerContent,
|
|
3506
3507
|
meta,
|
|
3507
3508
|
type: "input"
|
|
3508
3509
|
}));
|
|
@@ -4315,18 +4316,28 @@ tools.set("ask_user", tool({
|
|
|
4315
4316
|
const raw = await this.onAskUser(input.question, { signal });
|
|
4316
4317
|
const answer = typeof raw === "string" ? raw : raw.text;
|
|
4317
4318
|
const attachments = typeof raw === "string" ? void 0 : raw.attachments;
|
|
4318
|
-
if (attachments?.length) this.
|
|
4319
|
+
if (attachments?.length) this.setPendingAnswerAttachments(attachments);
|
|
4319
4320
|
return `User answered: ${answer}`;
|
|
4320
4321
|
}
|
|
4321
4322
|
}));
|
|
4323
|
+
tools.set("list_attachments", tool({
|
|
4324
|
+
description: "List image attachments still available in this agent session without loading their image data. Use this when the relevant attachment ID is not present in the current context.",
|
|
4325
|
+
inputSchema: z.object({ query: z.string().optional() }),
|
|
4326
|
+
execute: async function(input) {
|
|
4327
|
+
return this.listAvailableAttachments(input.query);
|
|
4328
|
+
}
|
|
4329
|
+
}));
|
|
4322
4330
|
tools.set("image_understand", tool({
|
|
4323
|
-
description: "Re-examine a previously shown image attachment to answer a specific visual question about it. Images are only
|
|
4331
|
+
description: "Re-examine a previously shown image attachment to answer a specific visual question about it. Prefer attachment_id from available_attachments/list_attachments; attachment_name remains supported for compatibility. Images are loaded only by this tool — use it instead of assuming you can still \"see\" an image from an earlier step or task. Only call this when the info is NOT already recorded in your memory/history; do not call it repeatedly for the same question.",
|
|
4324
4332
|
inputSchema: z.object({
|
|
4325
|
-
|
|
4326
|
-
|
|
4333
|
+
attachment_id: z.string().optional(),
|
|
4334
|
+
attachment_name: z.string().optional(),
|
|
4335
|
+
query: z.string().min(1)
|
|
4327
4336
|
}),
|
|
4328
4337
|
execute: async function(input, { signal }) {
|
|
4329
|
-
|
|
4338
|
+
const identifier = input.attachment_id ?? input.attachment_name;
|
|
4339
|
+
if (!identifier) return "❌ attachment_id or attachment_name is required.";
|
|
4340
|
+
return await this.describeImage(identifier, input.query, signal);
|
|
4330
4341
|
}
|
|
4331
4342
|
}));
|
|
4332
4343
|
tools.set("click_element_by_index", tool({
|
|
@@ -4407,6 +4418,10 @@ var ATTACHMENT_IMAGE_MAX_DIMENSION = 1568;
|
|
|
4407
4418
|
var ATTACHMENT_IMAGE_JPEG_QUALITY = .85;
|
|
4408
4419
|
/** 单次注入里最多内联发送的图片张数,超出的仅登记缓存,需模型主动调用 image_understand 查看 */
|
|
4409
4420
|
var MAX_INLINE_IMAGES = 4;
|
|
4421
|
+
/** 图片仓库跨任务最多保留的张数,超出时按 LRU 淘汰(仓库本身不占 prompt token) */
|
|
4422
|
+
var MAX_CACHED_IMAGES = 20;
|
|
4423
|
+
/** 相同图片、相同问题的视觉结果缓存上限,避免重复视觉调用同时控制内存 */
|
|
4424
|
+
var MAX_IMAGE_UNDERSTANDING_CACHE_ENTRIES = 100;
|
|
4410
4425
|
/** 历史任务 done 文本注入 prompt 时的最大字符数 */
|
|
4411
4426
|
var PREVIOUS_TASK_TEXT_MAX_LENGTH = 300;
|
|
4412
4427
|
/**
|
|
@@ -4478,18 +4493,27 @@ var AgentRuntime = class extends EventTarget {
|
|
|
4478
4493
|
#observations = [];
|
|
4479
4494
|
/** 本次任务上传的附件;仅在任务第一步注入 LLM prompt,之后步骤不重复携带原文件以节省 token */
|
|
4480
4495
|
#taskAttachments = [];
|
|
4496
|
+
/** 本次任务图片在会话级仓库中的轻量引用 */
|
|
4497
|
+
#taskAttachmentReferences = [];
|
|
4481
4498
|
/**
|
|
4482
4499
|
* `ask_user` 工具收到带附件的回答后暂存于此,供下一步 `#assembleUserPrompt` 注入 LLM prompt 后清空。
|
|
4483
|
-
* 非私有字段:需要被 `tools/index.ts` 中的 `ask_user` 工具从外部写入。
|
|
4484
4500
|
*/
|
|
4485
|
-
pendingAnswerAttachments = [];
|
|
4501
|
+
#pendingAnswerAttachments = [];
|
|
4502
|
+
/** 追问回答中的图片在会话级仓库中的轻量引用 */
|
|
4503
|
+
#pendingAnswerAttachmentReferences = [];
|
|
4486
4504
|
/**
|
|
4487
|
-
*
|
|
4488
|
-
* 使原图在 prompt 里只出现一次之后,仍可被 `image_understand`
|
|
4505
|
+
* 图片附件缓存(按稳定附件 ID 索引):所有出现过的图片附件在此登记原始 dataUrl,
|
|
4506
|
+
* 使原图在 prompt 里只出现一次之后,仍可被 `image_understand` 工具按稳定 ID“按需重传”重新查看,
|
|
4489
4507
|
* 而不必每步都把原图重新塞进主对话历史(节省 token)。
|
|
4490
|
-
*
|
|
4508
|
+
* @note 跨任务持久保留(不随 `execute()` 清空):缓存本身是纯内存 Map,不进 prompt 就不占 token,
|
|
4509
|
+
* 只有模型主动调用 `image_understand` 时才会产生一次性 token 成本。唯一真实开销是内存,
|
|
4510
|
+
* 因此按 `MAX_CACHED_IMAGES` 做容量上限,超出时淘汰最久未读取的一张。
|
|
4491
4511
|
*/
|
|
4492
4512
|
imageAttachmentCache = /* @__PURE__ */ new Map();
|
|
4513
|
+
/** 同一个附件对象只注册一次,避免首次注入 prompt 时重复生成 ID */
|
|
4514
|
+
#attachmentObjectIds = /* @__PURE__ */ new WeakMap();
|
|
4515
|
+
/** 独立视觉问答结果缓存,key 由附件 ID 和规范化后的问题组成 */
|
|
4516
|
+
#imageUnderstandingCache = /* @__PURE__ */ new Map();
|
|
4493
4517
|
/** 当前任务在 history 数组中的起始下标(用于跨任务保留历史时,把 prompt/循环检测限定在当前任务范围内) */
|
|
4494
4518
|
#taskHistoryStart = 0;
|
|
4495
4519
|
/** 当前一次运行完全结束时 resolve。由 `stop()` 等待。 */
|
|
@@ -4608,13 +4632,15 @@ var AgentRuntime = class extends EventTarget {
|
|
|
4608
4632
|
this.task = task;
|
|
4609
4633
|
this.taskId = uid();
|
|
4610
4634
|
this.#taskAttachments = attachments;
|
|
4611
|
-
this
|
|
4612
|
-
this
|
|
4635
|
+
this.#taskAttachmentReferences = this.#registerImageAttachments(attachments, "task");
|
|
4636
|
+
this.#pendingAnswerAttachments = [];
|
|
4637
|
+
this.#pendingAnswerAttachmentReferences = [];
|
|
4613
4638
|
this.#taskHistoryStart = this.history.length;
|
|
4614
4639
|
this.#emitHistoryChange({
|
|
4615
4640
|
type: "task_start",
|
|
4616
4641
|
task,
|
|
4617
|
-
...attachments.length > 0 ? { attachmentNames: attachments.map((a) => a.name) } : {}
|
|
4642
|
+
...attachments.length > 0 ? { attachmentNames: attachments.map((a) => a.name) } : {},
|
|
4643
|
+
...this.#taskAttachmentReferences.length > 0 ? { attachments: this.#taskAttachmentReferences } : {}
|
|
4618
4644
|
});
|
|
4619
4645
|
this.#observations = [];
|
|
4620
4646
|
this.#states = {
|
|
@@ -4696,7 +4722,9 @@ var AgentRuntime = class extends EventTarget {
|
|
|
4696
4722
|
const action = {
|
|
4697
4723
|
name: actionName,
|
|
4698
4724
|
input: input.action[actionName],
|
|
4699
|
-
output
|
|
4725
|
+
output,
|
|
4726
|
+
...macroResult.attachmentNames?.length ? { attachmentNames: macroResult.attachmentNames } : {},
|
|
4727
|
+
...macroResult.attachments?.length ? { attachments: macroResult.attachments } : {}
|
|
4700
4728
|
};
|
|
4701
4729
|
this.#emitHistoryChange({
|
|
4702
4730
|
type: "step",
|
|
@@ -4822,9 +4850,13 @@ var AgentRuntime = class extends EventTarget {
|
|
|
4822
4850
|
});
|
|
4823
4851
|
if (toolName === "wait") this.#states.totalWaitTime += toolInput?.seconds || 0;
|
|
4824
4852
|
else this.#states.totalWaitTime = 0;
|
|
4853
|
+
const attachmentNames = toolName === "ask_user" && this.#pendingAnswerAttachments.length > 0 ? this.#pendingAnswerAttachments.map((attachment) => attachment.name) : void 0;
|
|
4854
|
+
const attachments = toolName === "ask_user" && this.#pendingAnswerAttachmentReferences.length > 0 ? this.#pendingAnswerAttachmentReferences : void 0;
|
|
4825
4855
|
return {
|
|
4826
4856
|
input,
|
|
4827
|
-
output: result
|
|
4857
|
+
output: result,
|
|
4858
|
+
...attachmentNames ? { attachmentNames } : {},
|
|
4859
|
+
...attachments ? { attachments } : {}
|
|
4828
4860
|
};
|
|
4829
4861
|
}
|
|
4830
4862
|
};
|
|
@@ -4862,7 +4894,8 @@ var AgentRuntime = class extends EventTarget {
|
|
|
4862
4894
|
/**
|
|
4863
4895
|
* 汇总当前任务之前、同一 AgentRuntime 实例已完成过的历史任务,生成一段纯文字摘要。
|
|
4864
4896
|
* 只带文字结论(任务描述 + done 文本 + 涉及的附件文件名),不携带图片原始数据 ——
|
|
4865
|
-
*
|
|
4897
|
+
* 图片附件是否仍可复看取决于是否还在 `imageAttachmentCache`(跨任务持久、有容量上限)里:
|
|
4898
|
+
* 还在则提示模型改用 `image_understand` 复看;已被淘汰(或本就是非图片附件)则提示用户重新上传。
|
|
4866
4899
|
* 只取最近 `PREVIOUS_TASKS_SUMMARY_LIMIT` 个任务,避免任务数增多后 prompt 无限增长。
|
|
4867
4900
|
*/
|
|
4868
4901
|
#buildPreviousTasksSummary() {
|
|
@@ -4876,19 +4909,51 @@ var AgentRuntime = class extends EventTarget {
|
|
|
4876
4909
|
attachmentNames: event.attachmentNames
|
|
4877
4910
|
};
|
|
4878
4911
|
summaries.push(current);
|
|
4879
|
-
} else if (event.type === "step" &&
|
|
4880
|
-
|
|
4881
|
-
|
|
4912
|
+
} else if (event.type === "step" && current) {
|
|
4913
|
+
if (event.action.name === "ask_user" && event.action.attachmentNames?.length) current.attachmentNames = [.../* @__PURE__ */ new Set([...current.attachmentNames ?? [], ...event.action.attachmentNames])];
|
|
4914
|
+
if (event.action.name === "done") {
|
|
4915
|
+
const input = event.action.input;
|
|
4916
|
+
current.result = `${input?.success === false ? "❌" : "✅"} ${input?.text ?? ""}`;
|
|
4917
|
+
}
|
|
4882
4918
|
}
|
|
4883
4919
|
const recent = summaries.slice(-5);
|
|
4884
4920
|
if (recent.length === 0) return "";
|
|
4885
4921
|
return `<previous_tasks_summary>\n${recent.map((s) => {
|
|
4886
|
-
const attachmentNote = s.attachmentNames?.length ? `
|
|
4922
|
+
const attachmentNote = s.attachmentNames?.length ? ` ${this.#describeAttachmentAvailability(s.attachmentNames)}` : "";
|
|
4887
4923
|
const result = s.result ? truncate(s.result, PREVIOUS_TASK_TEXT_MAX_LENGTH) : "(no result recorded — task may have been interrupted)";
|
|
4888
4924
|
return `- Task: "${s.task}"${attachmentNote}\n Result: ${result}`;
|
|
4889
4925
|
}).join("\n")}\n</previous_tasks_summary>\n\n`;
|
|
4890
4926
|
}
|
|
4891
4927
|
/**
|
|
4928
|
+
* 按附件名分成"仍在 `imageAttachmentCache` 里、可 `image_understand` 复看"与
|
|
4929
|
+
* "已不可用、需用户重新上传"两组,拼成一句注入 prompt 的提示。
|
|
4930
|
+
*/
|
|
4931
|
+
#describeAttachmentAvailability(attachmentNames) {
|
|
4932
|
+
const viewable = [];
|
|
4933
|
+
const unavailable = [];
|
|
4934
|
+
for (const name of attachmentNames) (this.#resolveImageAttachment(name, false) ? viewable : unavailable).push(name);
|
|
4935
|
+
const parts = [];
|
|
4936
|
+
if (viewable.length > 0) parts.push(`still viewable via image_understand: ${viewable.join(", ")}`);
|
|
4937
|
+
if (unavailable.length > 0) parts.push(`NOT available in this task, ask the user to re-upload if you need to see them again: ${unavailable.join(", ")}`);
|
|
4938
|
+
return `[attachments uploaded then — ${parts.join("; ")}]`;
|
|
4939
|
+
}
|
|
4940
|
+
/** 生成新任务首步使用的轻量图片清单;不包含 dataUrl。 */
|
|
4941
|
+
#buildAvailableAttachmentsManifest() {
|
|
4942
|
+
if (this.imageAttachmentCache.size === 0) return "";
|
|
4943
|
+
const attachments = [...this.imageAttachmentCache.values()].reverse().map((attachment) => ({
|
|
4944
|
+
id: attachment.id,
|
|
4945
|
+
name: attachment.name,
|
|
4946
|
+
mimeType: attachment.mimeType,
|
|
4947
|
+
sourceTask: truncate(attachment.task, 100),
|
|
4948
|
+
source: attachment.source
|
|
4949
|
+
}));
|
|
4950
|
+
return `<available_attachments>
|
|
4951
|
+
These session images are not included inline. Use image_understand with attachment_id only when visual details are needed.
|
|
4952
|
+
${JSON.stringify(attachments)}\n</available_attachments>
|
|
4953
|
+
|
|
4954
|
+
`;
|
|
4955
|
+
}
|
|
4956
|
+
/**
|
|
4892
4957
|
* 在每一步之前生成系统观察
|
|
4893
4958
|
* @todo 控制台错误
|
|
4894
4959
|
*/
|
|
@@ -4937,7 +5002,10 @@ var AgentRuntime = class extends EventTarget {
|
|
|
4937
5002
|
prompt += await this.#getInstructions();
|
|
4938
5003
|
const currentTaskHistory = this.#currentTaskHistory;
|
|
4939
5004
|
const stepCount = currentTaskHistory.filter((e) => e.type === "step").length;
|
|
4940
|
-
if (stepCount === 0)
|
|
5005
|
+
if (stepCount === 0) {
|
|
5006
|
+
prompt += this.#buildPreviousTasksSummary();
|
|
5007
|
+
prompt += this.#buildAvailableAttachmentsManifest();
|
|
5008
|
+
}
|
|
4941
5009
|
prompt += "<agent_state>\n";
|
|
4942
5010
|
prompt += "<user_request>\n";
|
|
4943
5011
|
prompt += `${this.task}\n`;
|
|
@@ -4962,6 +5030,8 @@ var AgentRuntime = class extends EventTarget {
|
|
|
4962
5030
|
prompt += `Next Goal: ${event.reflection.next_goal}\n`;
|
|
4963
5031
|
prompt += `Action: ${event.action.name}(${JSON.stringify(event.action.input)})\n`;
|
|
4964
5032
|
prompt += `Action Results: ${event.action.output}\n`;
|
|
5033
|
+
if (event.action.attachments?.length) prompt += `User-uploaded attachments: ${JSON.stringify(event.action.attachments)}\n`;
|
|
5034
|
+
else if (event.action.attachmentNames?.length) prompt += `User-uploaded attachment names: ${JSON.stringify(event.action.attachmentNames)}\n`;
|
|
4965
5035
|
prompt += `</step_${stepIndex}>\n`;
|
|
4966
5036
|
} else if (event.type === "observation") prompt += `<sys>${event.content}</sys>\n`;
|
|
4967
5037
|
else if (event.type === "user_takeover") prompt += `<sys>User took over control and made changes to the page</sys>\n`;
|
|
@@ -4975,17 +5045,107 @@ var AgentRuntime = class extends EventTarget {
|
|
|
4975
5045
|
prompt += browserState.footer + "\n\n";
|
|
4976
5046
|
prompt += "</browser_state>\n\n";
|
|
4977
5047
|
if (stepCount === 0 && this.#taskAttachments.length > 0) return await this.#injectAttachments(prompt, this.#taskAttachments);
|
|
4978
|
-
if (this
|
|
4979
|
-
const attachments = this
|
|
4980
|
-
this
|
|
5048
|
+
if (this.#pendingAnswerAttachments.length > 0) {
|
|
5049
|
+
const attachments = this.#pendingAnswerAttachments;
|
|
5050
|
+
this.#pendingAnswerAttachments = [];
|
|
5051
|
+
this.#pendingAnswerAttachmentReferences = [];
|
|
4981
5052
|
return await this.#injectAttachments(prompt, attachments);
|
|
4982
5053
|
}
|
|
4983
5054
|
return prompt;
|
|
4984
5055
|
}
|
|
4985
5056
|
/**
|
|
5057
|
+
* 保存 ask_user 回答附件,供下一个主循环步骤注入;图片会同时登记进跨任务仓库。
|
|
5058
|
+
* @internal 供内置 ask_user 工具调用。
|
|
5059
|
+
*/
|
|
5060
|
+
setPendingAnswerAttachments(attachments) {
|
|
5061
|
+
this.#pendingAnswerAttachments = attachments;
|
|
5062
|
+
this.#pendingAnswerAttachmentReferences = this.#registerImageAttachments(attachments, "answer");
|
|
5063
|
+
}
|
|
5064
|
+
/** 把图片登记进按稳定 ID 索引的会话级 LRU 仓库,并返回不含原图的引用。 */
|
|
5065
|
+
#registerImageAttachments(attachments, source) {
|
|
5066
|
+
const references = [];
|
|
5067
|
+
for (const attachment of attachments) {
|
|
5068
|
+
if (!attachment.mimeType.startsWith("image/")) continue;
|
|
5069
|
+
const existingId = this.#attachmentObjectIds.get(attachment);
|
|
5070
|
+
const existing = existingId ? this.imageAttachmentCache.get(existingId) : void 0;
|
|
5071
|
+
if (existing) {
|
|
5072
|
+
this.#touchImageAttachment(existing.id);
|
|
5073
|
+
references.push(this.#toAttachmentReference(existing));
|
|
5074
|
+
continue;
|
|
5075
|
+
}
|
|
5076
|
+
while (this.imageAttachmentCache.size >= MAX_CACHED_IMAGES) {
|
|
5077
|
+
const oldestId = this.imageAttachmentCache.keys().next().value;
|
|
5078
|
+
if (oldestId === void 0) break;
|
|
5079
|
+
this.#evictImageAttachment(oldestId);
|
|
5080
|
+
}
|
|
5081
|
+
const id = `att_${uid()}`;
|
|
5082
|
+
const now = Date.now();
|
|
5083
|
+
const stored = {
|
|
5084
|
+
id,
|
|
5085
|
+
name: attachment.name,
|
|
5086
|
+
mimeType: attachment.mimeType,
|
|
5087
|
+
dataUrl: attachment.dataUrl,
|
|
5088
|
+
taskId: this.taskId,
|
|
5089
|
+
task: this.task,
|
|
5090
|
+
source,
|
|
5091
|
+
createdAt: now,
|
|
5092
|
+
lastAccessedAt: now
|
|
5093
|
+
};
|
|
5094
|
+
this.imageAttachmentCache.set(id, stored);
|
|
5095
|
+
this.#attachmentObjectIds.set(attachment, id);
|
|
5096
|
+
references.push(this.#toAttachmentReference(stored));
|
|
5097
|
+
}
|
|
5098
|
+
return references;
|
|
5099
|
+
}
|
|
5100
|
+
#toAttachmentReference(attachment) {
|
|
5101
|
+
return {
|
|
5102
|
+
id: attachment.id,
|
|
5103
|
+
name: attachment.name,
|
|
5104
|
+
mimeType: attachment.mimeType
|
|
5105
|
+
};
|
|
5106
|
+
}
|
|
5107
|
+
/** 按 ID 精确查找;旧文件名调用兼容为最近上传的同名图片。 */
|
|
5108
|
+
#resolveImageAttachment(attachmentIdOrName, touch = true) {
|
|
5109
|
+
let attachment = this.imageAttachmentCache.get(attachmentIdOrName);
|
|
5110
|
+
if (!attachment) attachment = [...this.imageAttachmentCache.values()].reverse().find((candidate) => candidate.name === attachmentIdOrName);
|
|
5111
|
+
if (attachment && touch) this.#touchImageAttachment(attachment.id);
|
|
5112
|
+
return attachment;
|
|
5113
|
+
}
|
|
5114
|
+
/** 读取图片时移动到 Map 末尾,使容量淘汰遵循 LRU 而非单纯 FIFO。 */
|
|
5115
|
+
#touchImageAttachment(attachmentId) {
|
|
5116
|
+
const attachment = this.imageAttachmentCache.get(attachmentId);
|
|
5117
|
+
if (!attachment) return;
|
|
5118
|
+
attachment.lastAccessedAt = Date.now();
|
|
5119
|
+
this.imageAttachmentCache.delete(attachmentId);
|
|
5120
|
+
this.imageAttachmentCache.set(attachmentId, attachment);
|
|
5121
|
+
}
|
|
5122
|
+
#evictImageAttachment(attachmentId) {
|
|
5123
|
+
this.imageAttachmentCache.delete(attachmentId);
|
|
5124
|
+
for (const key of this.#imageUnderstandingCache.keys()) if (key.startsWith(`${attachmentId}\n`)) this.#imageUnderstandingCache.delete(key);
|
|
5125
|
+
}
|
|
5126
|
+
/**
|
|
5127
|
+
* 返回不含原图数据的附件列表,供 list_attachments 工具在任务中途重新发现历史图片。
|
|
5128
|
+
* @internal
|
|
5129
|
+
*/
|
|
5130
|
+
listAvailableAttachments(query) {
|
|
5131
|
+
const needle = query?.trim().toLowerCase();
|
|
5132
|
+
const attachments = [...this.imageAttachmentCache.values()].reverse().filter((attachment) => {
|
|
5133
|
+
if (!needle) return true;
|
|
5134
|
+
return attachment.id.toLowerCase().includes(needle) || attachment.name.toLowerCase().includes(needle) || attachment.task.toLowerCase().includes(needle);
|
|
5135
|
+
}).map((attachment) => ({
|
|
5136
|
+
id: attachment.id,
|
|
5137
|
+
name: attachment.name,
|
|
5138
|
+
mimeType: attachment.mimeType,
|
|
5139
|
+
sourceTask: truncate(attachment.task, 100),
|
|
5140
|
+
source: attachment.source
|
|
5141
|
+
}));
|
|
5142
|
+
if (attachments.length === 0) return "No matching image attachments are available.";
|
|
5143
|
+
return `Available image attachments (metadata only): ${JSON.stringify(attachments)}`;
|
|
5144
|
+
}
|
|
5145
|
+
/**
|
|
4986
5146
|
* 把附件内容注入 prompt:图片转为 image_url 片段;文本类文件解码后截断拼进文字部分。
|
|
4987
5147
|
* 仅当存在图片附件时才返回多模态数组,纯文本附件保持返回字符串(不影响不支持 vision 的模型)。
|
|
4988
|
-
*
|
|
5148
|
+
* 图片附件已在接收时登记进 `imageAttachmentCache`(存原图,供 `image_understand` 复看细节用),
|
|
4989
5149
|
* 内联发给模型的图片先按 `ATTACHMENT_IMAGE_MAX_DIMENSION` 压缩降 token;
|
|
4990
5150
|
* 单次超过 `MAX_INLINE_IMAGES` 张时,多出的图片只登记缓存 + 文字提示名称,不内联发送。
|
|
4991
5151
|
* 之后步骤不再重复下发原图,需要复看细节时改由模型调用 `image_understand` 按需重传。
|
|
@@ -4996,14 +5156,14 @@ var AgentRuntime = class extends EventTarget {
|
|
|
4996
5156
|
let inlinedImageCount = 0;
|
|
4997
5157
|
for (const attachment of attachments) {
|
|
4998
5158
|
if (attachment.mimeType.startsWith("image/")) {
|
|
4999
|
-
this
|
|
5159
|
+
const attachmentId = this.#registerImageAttachments([attachment], "task")[0].id;
|
|
5000
5160
|
if (inlinedImageCount >= MAX_INLINE_IMAGES) {
|
|
5001
|
-
textSegments.push(`[Image attachment "${attachment.name}" was NOT shown inline (too many images in this batch). Call image_understand with
|
|
5161
|
+
textSegments.push(`[Image attachment "${attachment.name}" was NOT shown inline (too many images in this batch). Call image_understand with attachment_id="${attachmentId}" if you need to see it.]`);
|
|
5002
5162
|
continue;
|
|
5003
5163
|
}
|
|
5004
5164
|
inlinedImageCount++;
|
|
5005
5165
|
const resizedDataUrl = await resizeImageDataUrl(attachment.dataUrl, ATTACHMENT_IMAGE_MAX_DIMENSION, ATTACHMENT_IMAGE_JPEG_QUALITY);
|
|
5006
|
-
textSegments.push(`[Image attachment "${attachment.name}" is shown below. It will NOT be resent in later steps — note only the key facts you need into memory, don't transcribe it verbatim. If you need to re-examine its visual details (e.g. exact small text) in a later step, call image_understand with
|
|
5166
|
+
textSegments.push(`[Image attachment "${attachment.name}" (attachment_id="${attachmentId}") is shown below. It will NOT be resent in later steps — note only the key facts you need into memory, don't transcribe it verbatim. If you need to re-examine its visual details (e.g. exact small text) in a later step, call image_understand with attachment_id="${attachmentId}".]`);
|
|
5007
5167
|
imageParts.push({
|
|
5008
5168
|
type: "image_url",
|
|
5009
5169
|
image_url: { url: resizedDataUrl }
|
|
@@ -5022,20 +5182,25 @@ var AgentRuntime = class extends EventTarget {
|
|
|
5022
5182
|
/**
|
|
5023
5183
|
* 针对已缓存的图片附件发起一次独立的视觉问答,不进入主对话历史、不携带主链路的历史上下文。
|
|
5024
5184
|
* 供 `image_understand` 工具使用:模型需要复看图片细节时按需调用,而不是每步都重发原图。
|
|
5025
|
-
* @param
|
|
5185
|
+
* @param attachmentIdOrName - 优先使用稳定附件 ID;为兼容旧调用也接受文件名
|
|
5026
5186
|
* @param query - 需要从图片中确认的具体问题
|
|
5027
5187
|
* @param signal - 取消信号
|
|
5028
5188
|
* @returns 视觉模型针对 query 给出的文字答案;附件不存在时返回错误提示
|
|
5029
5189
|
*/
|
|
5030
|
-
async describeImage(
|
|
5031
|
-
|
|
5032
|
-
|
|
5190
|
+
async describeImage(attachmentIdOrName, query, signal) {
|
|
5191
|
+
signal.throwIfAborted();
|
|
5192
|
+
const attachment = this.#resolveImageAttachment(attachmentIdOrName);
|
|
5193
|
+
if (!attachment) return `❌ Attachment "${attachmentIdOrName}" not found. ${this.listAvailableAttachments()}`;
|
|
5194
|
+
const normalizedQuery = query.trim().replace(/\s+/g, " ").toLowerCase();
|
|
5195
|
+
const cacheKey = `${attachment.id}\n${normalizedQuery}`;
|
|
5196
|
+
const cached = this.#imageUnderstandingCache.get(cacheKey);
|
|
5197
|
+
if (cached !== void 0) return cached;
|
|
5033
5198
|
const answerTool = {
|
|
5034
5199
|
description: "Answer the question about the image as precisely and completely as possible.",
|
|
5035
5200
|
inputSchema: z.object({ answer: z.string() }),
|
|
5036
5201
|
execute: async (args) => args
|
|
5037
5202
|
};
|
|
5038
|
-
|
|
5203
|
+
const answer = (await this.#llm.invoke([{
|
|
5039
5204
|
role: "user",
|
|
5040
5205
|
content: [{
|
|
5041
5206
|
type: "text",
|
|
@@ -5045,6 +5210,12 @@ var AgentRuntime = class extends EventTarget {
|
|
|
5045
5210
|
image_url: { url: attachment.dataUrl }
|
|
5046
5211
|
}]
|
|
5047
5212
|
}], { answer_image_question: answerTool }, signal, { toolChoiceName: "answer_image_question" })).toolResult.answer;
|
|
5213
|
+
if (this.#imageUnderstandingCache.size >= MAX_IMAGE_UNDERSTANDING_CACHE_ENTRIES) {
|
|
5214
|
+
const oldestKey = this.#imageUnderstandingCache.keys().next().value;
|
|
5215
|
+
if (oldestKey !== void 0) this.#imageUnderstandingCache.delete(oldestKey);
|
|
5216
|
+
}
|
|
5217
|
+
this.#imageUnderstandingCache.set(cacheKey, answer);
|
|
5218
|
+
return answer;
|
|
5048
5219
|
}
|
|
5049
5220
|
/** 销毁 Agent:中止任务、释放浏览器控制器,并触发 dispose 事件(一次性) */
|
|
5050
5221
|
dispose() {
|
|
@@ -5061,6 +5232,15 @@ var AgentRuntime = class extends EventTarget {
|
|
|
5061
5232
|
};
|
|
5062
5233
|
cleanUp(() => this.#abortController.abort());
|
|
5063
5234
|
cleanUp(() => this.browserController.dispose());
|
|
5235
|
+
cleanUp(() => this.imageAttachmentCache.clear());
|
|
5236
|
+
cleanUp(() => this.#imageUnderstandingCache.clear());
|
|
5237
|
+
cleanUp(() => {
|
|
5238
|
+
this.#attachmentObjectIds = /* @__PURE__ */ new WeakMap();
|
|
5239
|
+
this.#taskAttachments = [];
|
|
5240
|
+
this.#taskAttachmentReferences = [];
|
|
5241
|
+
this.#pendingAnswerAttachments = [];
|
|
5242
|
+
this.#pendingAnswerAttachmentReferences = [];
|
|
5243
|
+
});
|
|
5064
5244
|
cleanUp(() => this.dispatchEvent(new Event("dispose")));
|
|
5065
5245
|
cleanUp(() => this.config.onDispose?.(this));
|
|
5066
5246
|
if (errors.length === 1) throw errors[0];
|