agentpage 0.0.32 → 0.0.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +81 -92
- package/dist/index.d.mts +4 -14
- package/dist/index.d.mts.map +1 -1
- package/dist/index.mjs +508 -417
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/index.mjs
CHANGED
|
@@ -154,13 +154,11 @@ function deriveNextInstruction(text, currentInstruction) {
|
|
|
154
154
|
*/
|
|
155
155
|
function reduceRemainingHeuristically(currentInstruction, executedCount) {
|
|
156
156
|
if (!currentInstruction.trim() || executedCount <= 0) return currentInstruction;
|
|
157
|
-
|
|
158
|
-
const parts = currentInstruction.replace(/\s+/g, " ").replace(/(->|=>|→)/g, " 然后 ").split(/\s*(?:then|and then|next|after that|然后|接着|随后|之后|再)\s*/gi).map((part) => part.trim()).filter(Boolean);
|
|
157
|
+
const parts = currentInstruction.replace(/\s+/g, " ").replace(/(->|=>|→)/g, " 然后 ").replace(/[,,。;;]/g, " 然后 ").split(/\s*(?:然后|再|并且|并|接着|随后|之后)\s*/g).map((part) => part.trim()).filter(Boolean);
|
|
159
158
|
if (parts.length <= 1) return currentInstruction;
|
|
160
|
-
const
|
|
161
|
-
const nextParts = parts.slice(Math.min(consumedSteps, parts.length));
|
|
159
|
+
const nextParts = parts.slice(Math.min(executedCount, parts.length));
|
|
162
160
|
if (nextParts.length === 0) return "";
|
|
163
|
-
return nextParts.join("
|
|
161
|
+
return nextParts.join(" -> ");
|
|
164
162
|
}
|
|
165
163
|
/**
|
|
166
164
|
* 判定是否强制断轮。
|
|
@@ -297,7 +295,7 @@ function hasToolError(result) {
|
|
|
297
295
|
* 4) 返回稳定字符串给 loop,供后续注入消息与统计。
|
|
298
296
|
*
|
|
299
297
|
* 默认参数意图:
|
|
300
|
-
* - `maxDepth=
|
|
298
|
+
* - `maxDepth=12`: 保留更深层级,减少深层组件控件被截断。
|
|
301
299
|
* - `viewportOnly=false`: 优先完整性,避免误判“元素不存在”。
|
|
302
300
|
* - `pruneLayout=true`: 抑制纯布局噪声,降低 token 压力。
|
|
303
301
|
* - `maxNodes=500` / `maxChildren=30`: 控制体积上限,兼顾可读性。
|
|
@@ -308,12 +306,6 @@ function hasToolError(result) {
|
|
|
308
306
|
* - `pruneLayout=true` 时:无 id/无语义/无交互/无直接文本的布局容器会被“折叠”,
|
|
309
307
|
* 子节点直接提升输出,减少无意义层级;当同一折叠容器提升出多个相邻节点时,
|
|
310
308
|
* 快照会用括号分组块标记其关联来源(collapsed-group)。
|
|
311
|
-
* - 布局主干保留:浅层结构优先保留(避免页面主骨架被过早折叠导致业务区域缺失)。
|
|
312
|
-
* - 事件信号保留:节点自身存在事件绑定(inline/on* 或 addEventListener 追踪)时优先保留;
|
|
313
|
-
* 中浅层会做受预算约束的子树事件探测,尽量保留潜在可操作链路。
|
|
314
|
-
* - 语义文本保留:包含语义文本的容器优先保留,避免“有意义但非控件”信息被误删。
|
|
315
|
-
* - 噪音过滤:跳过 `svg` 等装饰节点及 `__SVG_SPRITE_NODE__` sprite 容器,
|
|
316
|
-
* 避免图标定义树挤占节点预算。
|
|
317
309
|
* - `maxNodes`:全局节点预算,超限后停止继续遍历并追加 truncation 提示。
|
|
318
310
|
* - `maxChildren`:每个父节点只保留前 N 个子元素,其余用 `... (n children omitted)` 汇总。
|
|
319
311
|
* - `maxTextLength`:节点文本按长度截断,避免长段文案占满上下文。
|
|
@@ -328,7 +320,7 @@ function hasToolError(result) {
|
|
|
328
320
|
async function readPageSnapshot(registry, options) {
|
|
329
321
|
return toContentString((await registry.dispatch("page_info", {
|
|
330
322
|
action: "snapshot",
|
|
331
|
-
maxDepth: options?.maxDepth ??
|
|
323
|
+
maxDepth: options?.maxDepth ?? 12,
|
|
332
324
|
viewportOnly: options?.viewportOnly ?? false,
|
|
333
325
|
pruneLayout: options?.pruneLayout ?? true,
|
|
334
326
|
maxNodes: options?.maxNodes ?? 500,
|
|
@@ -485,10 +477,26 @@ function buildCompactMessages(userMessage, trace, latestSnapshot, currentUrl, hi
|
|
|
485
477
|
content: `Done steps (do NOT repeat):\n${traceParts.join("\n")}`
|
|
486
478
|
});
|
|
487
479
|
const hasErrors = trace.some((e) => hasToolError(e.result));
|
|
488
|
-
const
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
480
|
+
const contextParts = [
|
|
481
|
+
"## Execution context",
|
|
482
|
+
"Current remaining instruction:",
|
|
483
|
+
activeInstruction,
|
|
484
|
+
"",
|
|
485
|
+
"Task-reduction model:",
|
|
486
|
+
"Input: current remaining instruction + previous round executed actions + this-round actions.",
|
|
487
|
+
"Output: new remaining instruction after removing this-round actions.",
|
|
488
|
+
"Start from visible page state directly. Do NOT restate task. Do NOT output planning text.",
|
|
489
|
+
"Execute all independent visible sub-tasks in one round.",
|
|
490
|
+
"Do NOT act on elements not present in this snapshot yet.",
|
|
491
|
+
"If action changes DOM (open modal/navigate), stop after that batch and continue next round.",
|
|
492
|
+
"Do NOT call page_info (get_url/get_title/query_all/snapshot).",
|
|
493
|
+
"For dropdown/select fields, use dom with action=select_option (or fill on a select).",
|
|
494
|
+
"If a needed list shows `... (N children omitted)` under a specific container, output `SNAPSHOT_HINT: EXPAND_CHILDREN #<containerRef>` and wait for next round snapshot.",
|
|
495
|
+
"Build the minimal action array from current snapshot to finish this remaining instruction in one round whenever possible.",
|
|
496
|
+
"For deterministic increase/decrease controls, compute delta from current visible value and issue exactly that many clicks in one round (e.g., +2 => two increase clicks). Do not overshoot then undo.",
|
|
497
|
+
"Stop rule: once requested state is reached, stop tool calls. If verification is needed, verify once and then output REMAINING: DONE.",
|
|
498
|
+
allowAgentUiInteraction ? "User explicitly asked to operate AutoPilot UI. You may interact with chat input/send/dock only as requested." : "Do NOT interact with any AI chat UI elements (chat input, send button, dock). Only operate on the actual page content."
|
|
499
|
+
];
|
|
492
500
|
if (hasErrors) contextParts.push("", "The last step failed. Retry with a different approach, or skip and continue with other visible targets.");
|
|
493
501
|
else contextParts.push("", "If the goal is fully done, reply with a short summary (no tool calls).");
|
|
494
502
|
if (previousRoundTasks && previousRoundTasks.length > 0) contextParts.push("", "Previous round planned task array (already executed):", ...previousRoundTasks.map((task, index) => `${index + 1}. ${task}`));
|
|
@@ -980,8 +988,7 @@ async function executeAgentLoop(params) {
|
|
|
980
988
|
else pendingNotFoundRetry = void 0;
|
|
981
989
|
if (parsedInstructionState.hasRemainingProtocol) remainingInstruction = parsedInstructionState.nextInstruction;
|
|
982
990
|
else {
|
|
983
|
-
const
|
|
984
|
-
const nextByHeuristic = reduceRemainingHeuristically(remainingInstruction, heuristicProgressUnits);
|
|
991
|
+
const nextByHeuristic = reduceRemainingHeuristically(remainingInstruction, executedTaskCalls.length);
|
|
985
992
|
if (nextByHeuristic !== remainingInstruction) remainingInstruction = nextByHeuristic;
|
|
986
993
|
else roundHasError = true;
|
|
987
994
|
}
|
|
@@ -989,6 +996,11 @@ async function executeAgentLoop(params) {
|
|
|
989
996
|
lastRoundHadError = roundHasError;
|
|
990
997
|
previousRoundTasks = buildTaskArray(executedTaskCalls);
|
|
991
998
|
previousRoundPlannedTasks = plannedTasksCurrentRound;
|
|
999
|
+
if (parsedInstructionState.hasRemainingProtocol && remainingInstruction.trim().length === 0 && !roundHasError) {
|
|
1000
|
+
finalReply = response.text?.trim() || "任务已完成。";
|
|
1001
|
+
if (finalReply) callbacks?.onText?.(finalReply);
|
|
1002
|
+
break;
|
|
1003
|
+
}
|
|
992
1004
|
const idleResult = detectIdleLoop(response.toolCalls.map((tc) => ({
|
|
993
1005
|
name: tc.name,
|
|
994
1006
|
input: tc.input
|
|
@@ -1649,7 +1661,13 @@ var ToolRegistry = class {
|
|
|
1649
1661
|
|
|
1650
1662
|
//#endregion
|
|
1651
1663
|
//#region src/core/system-prompt.ts
|
|
1652
|
-
/**
|
|
1664
|
+
/**
|
|
1665
|
+
* 规范化额外指令:统一转为非空字符串数组。
|
|
1666
|
+
*
|
|
1667
|
+
* - 单字符串 → 单元素数组
|
|
1668
|
+
* - 字符串数组 → 过滤空值
|
|
1669
|
+
* - undefined → 空数组
|
|
1670
|
+
*/
|
|
1653
1671
|
function normalizeExtraInstructions(input) {
|
|
1654
1672
|
if (!input) return [];
|
|
1655
1673
|
return (Array.isArray(input) ? input : [input]).map((s) => s.trim()).filter(Boolean);
|
|
@@ -1657,9 +1675,32 @@ function normalizeExtraInstructions(input) {
|
|
|
1657
1675
|
/**
|
|
1658
1676
|
* 构建系统提示词。
|
|
1659
1677
|
*
|
|
1660
|
-
*
|
|
1661
|
-
*
|
|
1662
|
-
*
|
|
1678
|
+
* 输出结构(按章节顺序):
|
|
1679
|
+
* 1. **Core Rules** — Agent 核心行为规则
|
|
1680
|
+
* - 快照驱动决策:仅基于当前快照 + 剩余任务工作
|
|
1681
|
+
* - 增量消费模型:每轮执行后输出 REMAINING 推进任务
|
|
1682
|
+
* - hash ID 定位:仅交互元素携带 #hashID,非交互元素为上下文
|
|
1683
|
+
* - 事件信号:listeners="..." 标注运行时事件绑定
|
|
1684
|
+
* - 批量执行:同轮完成所有独立可见操作
|
|
1685
|
+
* - 输入顺序:fill/type 前必须先 focus/click 同一目标
|
|
1686
|
+
* - DOM 变化断轮:会改变 DOM 的动作执行后等待下一轮新快照
|
|
1687
|
+
* - 停机规则:任务完成后输出 REMAINING: DONE
|
|
1688
|
+
*
|
|
1689
|
+
* 2. **Listener Abbrevs** — 事件简写对照表
|
|
1690
|
+
* - 快照中 listeners="clk,inp,chg" 的简写含义
|
|
1691
|
+
* - 与 page-info-tool.ts 的 EVENT_ABBREV 映射一致
|
|
1692
|
+
*
|
|
1693
|
+
* 3. **Output Contract** — 输出协议
|
|
1694
|
+
* - 每轮返回工具调用 + REMAINING 文本行
|
|
1695
|
+
*
|
|
1696
|
+
* 4. **Available Tools**(可选) — 当前注册的工具及描述
|
|
1697
|
+
*
|
|
1698
|
+
* 5. **Reasoning Profile**(可选) — 思考深度配置
|
|
1699
|
+
*
|
|
1700
|
+
* 6. **Extra Instructions**(可选) — 用户自定义额外指令
|
|
1701
|
+
*
|
|
1702
|
+
* @param params - 构建参数(工具列表、思考深度、额外指令)
|
|
1703
|
+
* @returns 完整的系统提示词字符串(英文)
|
|
1663
1704
|
*/
|
|
1664
1705
|
function buildSystemPrompt(params = {}) {
|
|
1665
1706
|
const sections = [];
|
|
@@ -1672,6 +1713,9 @@ function buildSystemPrompt(params = {}) {
|
|
|
1672
1713
|
" Input: (1) current remaining task, (2) previous round executed actions, (3) actions you execute this round.",
|
|
1673
1714
|
" Output: new remaining task after removing this-round actions.",
|
|
1674
1715
|
"- Use only visible targets from snapshot. Use #hashID as selector. Do not guess CSS selectors.",
|
|
1716
|
+
"- Only interactive elements (with events, inputs, buttons, links, etc.) carry #hashID. Elements without #hashID are context-only (labels, headings, text) and cannot be targeted.",
|
|
1717
|
+
"- Snapshot tag in brackets may show ARIA role instead of HTML tag when it better describes the interaction pattern (e.g. [combobox] for input with role=\"combobox\", [slider] for div with role=\"slider\"). Treat the bracket tag as the primary interaction hint.",
|
|
1718
|
+
"- listeners=\"...\" on snapshot indicates bound event handlers (see Listener Abbrevs below). Prefer targets with relevant listeners when multiple candidates look similar.",
|
|
1675
1719
|
"- Batch independent visible actions in one round. Do not split one form into many rounds unnecessarily.",
|
|
1676
1720
|
"- Strict input order (MANDATORY): before every fill/type/select_option, click or focus the SAME target immediately in the SAME round.",
|
|
1677
1721
|
"- Multi-field rule (MANDATORY): execute alternating pairs in one batch: focus/click field A -> fill/type A -> focus/click field B -> fill/type B.",
|
|
@@ -1684,7 +1728,6 @@ function buildSystemPrompt(params = {}) {
|
|
|
1684
1728
|
"- If an action will change DOM (open modal, navigate), stop after that action batch and continue next round with new snapshot.",
|
|
1685
1729
|
"- Do NOT call page_info (snapshot/query/get_url/get_title). Snapshot is already provided every round.",
|
|
1686
1730
|
"- For dropdown/select, use dom action=select_option (or fill on select).",
|
|
1687
|
-
"- Always cross-check planned actions against the original goal to avoid task drift (e.g., do not confuse create issue vs create repository).",
|
|
1688
1731
|
"- If a required list shows `... (N children omitted)` under a specific container, request focused expansion by outputting `SNAPSHOT_HINT: EXPAND_CHILDREN #<containerRef>`.",
|
|
1689
1732
|
"- After outputting snapshot expansion hint, wait for the next refreshed snapshot before further scrolling/clicking on that list.",
|
|
1690
1733
|
"- Verification whitelist: do NOT use get_text/get_attr to verify input/select values unless the user explicitly asks for verification.",
|
|
@@ -1796,13 +1839,24 @@ function hasTrackedElementEvents(el) {
|
|
|
1796
1839
|
}
|
|
1797
1840
|
|
|
1798
1841
|
//#endregion
|
|
1799
|
-
//#region src/web/tools/dom-tool
|
|
1842
|
+
//#region src/web/tools/dom-tool.ts
|
|
1800
1843
|
/**
|
|
1801
|
-
* DOM Tool
|
|
1844
|
+
* DOM Tool — 浏览器 DOM 操作工具(结合 Playwright 核心交互模式增强)。
|
|
1802
1845
|
*
|
|
1803
|
-
*
|
|
1846
|
+
* 关键改进(参考 Playwright):
|
|
1847
|
+
* 1. retarget — 点击时自动重定向到 button/link/label.control
|
|
1848
|
+
* 2. scrollIntoView 多策略 — 4 种 block 对齐轮换,解决 sticky 遮挡
|
|
1849
|
+
* 3. stable 检查 — rAF 逐帧检测元素位置稳定后再操作
|
|
1850
|
+
* 4. hit-target 验证 — elementsFromPoint 检查是否被遮挡
|
|
1851
|
+
* 5. 完整点击事件链 — pointermove→pointerdown→mousedown→pointerup→mouseup→click
|
|
1852
|
+
* 6. check/uncheck 通过 click — 先检查→click 切换→验证状态
|
|
1853
|
+
* 7. press 组合键 — 支持 Control+a, Shift+Enter 等修饰键
|
|
1854
|
+
* 8. fill 分类型 — date/color/range 走 setValue,text 类走 selectAll+原生写入
|
|
1855
|
+
* 9. 自定义下拉增强 — 更广泛的 option 选择器 + 等待弹出
|
|
1856
|
+
* 10. ARIA disabled — 检查祖先链 aria-disabled
|
|
1857
|
+
*
|
|
1858
|
+
* 运行环境:浏览器 Content Script(直接访问 DOM,无 CDP)。
|
|
1804
1859
|
*/
|
|
1805
|
-
/** 默认等待超时(ms) */
|
|
1806
1860
|
const DEFAULT_WAIT_MS = 1200;
|
|
1807
1861
|
/** scrollIntoView 轮换策略(参考 Playwright dom.ts) */
|
|
1808
1862
|
const SCROLL_OPTIONS = [
|
|
@@ -1863,9 +1917,16 @@ const KEY_CODE_MAP = {
|
|
|
1863
1917
|
Alt: "AltLeft",
|
|
1864
1918
|
Meta: "MetaLeft"
|
|
1865
1919
|
};
|
|
1866
|
-
|
|
1867
|
-
|
|
1868
|
-
|
|
1920
|
+
const FILL_RELEVANT_EVENTS = new Set([
|
|
1921
|
+
"input",
|
|
1922
|
+
"change",
|
|
1923
|
+
"focus",
|
|
1924
|
+
"blur",
|
|
1925
|
+
"keydown",
|
|
1926
|
+
"click",
|
|
1927
|
+
"mousedown",
|
|
1928
|
+
"pointerdown"
|
|
1929
|
+
]);
|
|
1869
1930
|
let activeRefStore;
|
|
1870
1931
|
function setActiveRefStore(store) {
|
|
1871
1932
|
activeRefStore = store;
|
|
@@ -1876,26 +1937,15 @@ function getActiveRefStore() {
|
|
|
1876
1937
|
function sleep(ms) {
|
|
1877
1938
|
return new Promise((r) => setTimeout(r, ms));
|
|
1878
1939
|
}
|
|
1879
|
-
/**
|
|
1880
|
-
* 查询元素:优先 RefStore hash,回退 CSS 选择器。
|
|
1881
|
-
* 支持复合 hash 选择器(如 "#hashID .child-class")——先解析 hash 根,再在其子树内 querySelector。
|
|
1882
|
-
*/
|
|
1940
|
+
/** 查询元素:优先 RefStore hash,回退 CSS 选择器 */
|
|
1883
1941
|
function queryElement(selector) {
|
|
1884
1942
|
try {
|
|
1885
1943
|
if (selector.startsWith("#") && activeRefStore) {
|
|
1886
|
-
const
|
|
1887
|
-
|
|
1888
|
-
|
|
1889
|
-
|
|
1890
|
-
|
|
1891
|
-
if (!root || !root.isConnected) {
|
|
1892
|
-
activeRefStore.delete(hashPart);
|
|
1893
|
-
return `未找到 ref "#${hashPart}" 对应的元素(可能已被移除或快照已过期)`;
|
|
1894
|
-
}
|
|
1895
|
-
if (!rest) return root;
|
|
1896
|
-
const child = root.querySelector(rest);
|
|
1897
|
-
if (!child) return `在 #${hashPart} 内未找到匹配 "${rest}" 的子元素`;
|
|
1898
|
-
return child;
|
|
1944
|
+
const id = selector.slice(1);
|
|
1945
|
+
if (activeRefStore.has(id)) {
|
|
1946
|
+
const el = activeRefStore.get(id);
|
|
1947
|
+
if (!el) return `未找到 ref "${selector}" 对应的元素(可能已被移除或快照已过期)`;
|
|
1948
|
+
return el;
|
|
1899
1949
|
}
|
|
1900
1950
|
}
|
|
1901
1951
|
const el = document.querySelector(selector);
|
|
@@ -1923,30 +1973,6 @@ function resolveWaitMs(params) {
|
|
|
1923
1973
|
if (typeof waitSeconds === "number" && Number.isFinite(waitSeconds)) return Math.max(0, Math.floor(waitSeconds * 1e3));
|
|
1924
1974
|
return DEFAULT_WAIT_MS;
|
|
1925
1975
|
}
|
|
1926
|
-
/** 生成元素的简洁描述字符串,用于工具调用结果的可读输出。 */
|
|
1927
|
-
function describeElement(el) {
|
|
1928
|
-
const tag = el.tagName.toLowerCase();
|
|
1929
|
-
const id = el.id ? `#${el.id}` : "";
|
|
1930
|
-
const cls = el.className && typeof el.className === "string" ? el.className.trim().split(/\s+/).filter(Boolean).slice(0, 3).map((c) => `.${c}`).join("") : "";
|
|
1931
|
-
const text = el instanceof HTMLSelectElement ? el.selectedOptions[0]?.textContent?.trim().slice(0, 40) ?? "" : el.textContent?.trim().slice(0, 40) ?? "";
|
|
1932
|
-
const textHint = text ? ` "${text}"` : "";
|
|
1933
|
-
const hints = [];
|
|
1934
|
-
for (const attr of [
|
|
1935
|
-
"type",
|
|
1936
|
-
"name",
|
|
1937
|
-
"placeholder",
|
|
1938
|
-
"href",
|
|
1939
|
-
"role"
|
|
1940
|
-
]) {
|
|
1941
|
-
const v = el.getAttribute(attr);
|
|
1942
|
-
if (v) hints.push(`${attr}=${v}`);
|
|
1943
|
-
}
|
|
1944
|
-
if (el instanceof HTMLSelectElement && el.value) hints.push(`val=${el.value}`);
|
|
1945
|
-
return `<${tag}${id}${cls}>${textHint}${hints.length > 0 ? ` [${hints.join(", ")}]` : ""}`;
|
|
1946
|
-
}
|
|
1947
|
-
|
|
1948
|
-
//#endregion
|
|
1949
|
-
//#region src/web/tools/dom-tool/actionability.ts
|
|
1950
1976
|
/** 检查元素样式可见性(处理 checkVisibility / details 折叠 / visibility) */
|
|
1951
1977
|
function isStyleVisible(el, style) {
|
|
1952
1978
|
style = style ?? window.getComputedStyle(el);
|
|
@@ -2027,6 +2053,23 @@ function checkElementStable(el, timeoutMs = 800) {
|
|
|
2027
2053
|
requestAnimationFrame(check);
|
|
2028
2054
|
});
|
|
2029
2055
|
}
|
|
2056
|
+
/**
|
|
2057
|
+
* 将目标重定向到关联的交互控件。
|
|
2058
|
+
* - button-link:非交互元素→最近 button/[role=button]/a/[role=link]
|
|
2059
|
+
* - follow-label:label→control + 非交互→button/[role=button]/[role=checkbox]/[role=radio]
|
|
2060
|
+
*/
|
|
2061
|
+
function retarget(el, mode) {
|
|
2062
|
+
if (mode === "none") return el;
|
|
2063
|
+
if (!el.matches("input, textarea, select") && !el.isContentEditable) if (mode === "button-link") el = el.closest("button, [role=button], a, [role=link]") || el;
|
|
2064
|
+
else el = el.closest("button, [role=button], [role=checkbox], [role=radio]") || el;
|
|
2065
|
+
if (mode === "follow-label") {
|
|
2066
|
+
if (!el.matches("a, input, textarea, button, select, [role=link], [role=button], [role=checkbox], [role=radio]") && !el.isContentEditable) {
|
|
2067
|
+
const label = el.closest("label");
|
|
2068
|
+
if (label?.control) el = label.control;
|
|
2069
|
+
}
|
|
2070
|
+
}
|
|
2071
|
+
return el;
|
|
2072
|
+
}
|
|
2030
2073
|
function scrollIntoViewIfNeeded(el, retry = 0) {
|
|
2031
2074
|
if (retry === 0 && "scrollIntoViewIfNeeded" in el) {
|
|
2032
2075
|
el.scrollIntoViewIfNeeded(true);
|
|
@@ -2048,7 +2091,7 @@ function checkHitTarget(el) {
|
|
|
2048
2091
|
if (topEl === el || el.contains(topEl) || topEl.contains(el)) return null;
|
|
2049
2092
|
const sharedLabel = topEl.closest("label");
|
|
2050
2093
|
if (sharedLabel && sharedLabel.contains(el)) return null;
|
|
2051
|
-
return
|
|
2094
|
+
return describeElement(topEl);
|
|
2052
2095
|
}
|
|
2053
2096
|
function ensureActionable(el, action, selector, force) {
|
|
2054
2097
|
if (force) return null;
|
|
@@ -2092,26 +2135,36 @@ function ensureActionable(el, action, selector, force) {
|
|
|
2092
2135
|
"fill",
|
|
2093
2136
|
"type",
|
|
2094
2137
|
"clear"
|
|
2095
|
-
].includes(action) && !isEditableElement(el))
|
|
2096
|
-
|
|
2097
|
-
|
|
2098
|
-
|
|
2099
|
-
|
|
2100
|
-
|
|
2101
|
-
|
|
2102
|
-
|
|
2103
|
-
|
|
2138
|
+
].includes(action) && !isEditableElement(el)) {
|
|
2139
|
+
if (action === "fill" && el.getAttribute("role") === "slider") return null;
|
|
2140
|
+
return {
|
|
2141
|
+
content: `"${selector}" 不是可编辑元素,无法执行 ${action}`,
|
|
2142
|
+
details: {
|
|
2143
|
+
error: true,
|
|
2144
|
+
code: "UNSUPPORTED_FILL_TARGET",
|
|
2145
|
+
action,
|
|
2146
|
+
selector
|
|
2147
|
+
}
|
|
2148
|
+
};
|
|
2149
|
+
}
|
|
2104
2150
|
return null;
|
|
2105
2151
|
}
|
|
2106
|
-
|
|
2107
|
-
//#endregion
|
|
2108
|
-
//#region src/web/tools/dom-tool/events.ts
|
|
2109
2152
|
/**
|
|
2110
|
-
*
|
|
2111
|
-
*
|
|
2112
|
-
* 包含:完整点击事件链、hover 事件链、input/change 派发、
|
|
2113
|
-
* 原生 setter 写入、selectText、组合键 press。
|
|
2153
|
+
* 为 role=slider 查找关联的数值输入框。
|
|
2154
|
+
* 典型场景:Element Plus slider + input-number 同属一个 form-item。
|
|
2114
2155
|
*/
|
|
2156
|
+
function findAssociatedSliderInput(slider) {
|
|
2157
|
+
const candidates = [];
|
|
2158
|
+
const formItem = slider.closest(".el-form-item");
|
|
2159
|
+
if (formItem) candidates.push(formItem);
|
|
2160
|
+
let cursor = slider.parentElement;
|
|
2161
|
+
for (let depth = 0; cursor && depth < 4; depth++, cursor = cursor.parentElement) candidates.push(cursor);
|
|
2162
|
+
for (const scope of candidates) {
|
|
2163
|
+
const input = scope.querySelector("input[type=\"number\"], input[role=\"spinbutton\"], .el-input-number input:not([type=\"hidden\"])");
|
|
2164
|
+
if (input instanceof HTMLInputElement && isEditableElement(input) && isElementVisible(input)) return input;
|
|
2165
|
+
}
|
|
2166
|
+
return null;
|
|
2167
|
+
}
|
|
2115
2168
|
function getClickPoint(el) {
|
|
2116
2169
|
const r = el.getBoundingClientRect();
|
|
2117
2170
|
return {
|
|
@@ -2120,7 +2173,7 @@ function getClickPoint(el) {
|
|
|
2120
2173
|
};
|
|
2121
2174
|
}
|
|
2122
2175
|
/**
|
|
2123
|
-
*
|
|
2176
|
+
* 完整点击事件链(参考 Playwright Mouse.click):
|
|
2124
2177
|
* pointermove → mousemove → (per clickCount) pointerdown → mousedown → focus → pointerup → mouseup → click
|
|
2125
2178
|
*/
|
|
2126
2179
|
function dispatchClickEvents(el, clickCount = 1) {
|
|
@@ -2206,6 +2259,163 @@ function setNativeValue(el, value) {
|
|
|
2206
2259
|
if (desc?.set) desc.set.call(el, value);
|
|
2207
2260
|
else el.value = value;
|
|
2208
2261
|
}
|
|
2262
|
+
function getFillEventSupportScore(el) {
|
|
2263
|
+
let score = 0;
|
|
2264
|
+
if (el.hasAttribute("oninput") || el.hasAttribute("onchange")) score += 80;
|
|
2265
|
+
if (el.hasAttribute("onfocus") || el.hasAttribute("onblur")) score += 60;
|
|
2266
|
+
if (el.hasAttribute("onclick")) score += 40;
|
|
2267
|
+
const tracked = getTrackedElementEvents(el);
|
|
2268
|
+
for (const eventName of tracked) {
|
|
2269
|
+
if (!FILL_RELEVANT_EVENTS.has(eventName)) continue;
|
|
2270
|
+
if (eventName === "input") score += 40;
|
|
2271
|
+
else if (eventName === "change") score += 35;
|
|
2272
|
+
else if (eventName === "focus" || eventName === "blur") score += 28;
|
|
2273
|
+
else if (eventName === "keydown") score += 24;
|
|
2274
|
+
else score += 14;
|
|
2275
|
+
}
|
|
2276
|
+
return score;
|
|
2277
|
+
}
|
|
2278
|
+
function isCandidateFillTarget(el) {
|
|
2279
|
+
if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement || el instanceof HTMLSelectElement) return !isElementDisabled(el);
|
|
2280
|
+
if (el instanceof HTMLElement && el.isContentEditable) return true;
|
|
2281
|
+
return false;
|
|
2282
|
+
}
|
|
2283
|
+
function executeFillOnResolvedTarget(target, value, selector, action, sourceHint) {
|
|
2284
|
+
if (target instanceof HTMLInputElement) {
|
|
2285
|
+
const type = target.type.toLowerCase();
|
|
2286
|
+
if (INPUT_BLOCKED_TYPES.has(type)) return {
|
|
2287
|
+
content: `"${selector}" 为 input[type=${type}],不支持 fill;请使用 click/check 等动作。`,
|
|
2288
|
+
details: {
|
|
2289
|
+
error: true,
|
|
2290
|
+
code: "UNSUPPORTED_FILL_TARGET",
|
|
2291
|
+
action,
|
|
2292
|
+
selector
|
|
2293
|
+
}
|
|
2294
|
+
};
|
|
2295
|
+
if (INPUT_SET_VALUE_TYPES.has(type)) {
|
|
2296
|
+
const finalVal = type === "color" ? value.toLowerCase().trim() : value.trim();
|
|
2297
|
+
target.focus();
|
|
2298
|
+
target.value = finalVal;
|
|
2299
|
+
if (target.value !== finalVal) return {
|
|
2300
|
+
content: `"${selector}" 填写格式不匹配(type=${type})`,
|
|
2301
|
+
details: {
|
|
2302
|
+
error: true,
|
|
2303
|
+
code: "MALFORMED_VALUE",
|
|
2304
|
+
action,
|
|
2305
|
+
selector
|
|
2306
|
+
}
|
|
2307
|
+
};
|
|
2308
|
+
dispatchInputEvents(target);
|
|
2309
|
+
const suffix = sourceHint ? `(${sourceHint})` : "";
|
|
2310
|
+
return { content: `已填写 ${describeElement(target)}: "${finalVal}"${suffix}` };
|
|
2311
|
+
}
|
|
2312
|
+
if (type === "number" && Number.isNaN(Number(value.trim()))) return {
|
|
2313
|
+
content: `"${selector}" 为 input[type=number],无法填写非数字 "${value}"`,
|
|
2314
|
+
details: {
|
|
2315
|
+
error: true,
|
|
2316
|
+
code: "INVALID_NUMBER",
|
|
2317
|
+
action,
|
|
2318
|
+
selector
|
|
2319
|
+
}
|
|
2320
|
+
};
|
|
2321
|
+
scrollIntoViewIfNeeded(target);
|
|
2322
|
+
target.focus();
|
|
2323
|
+
selectText(target);
|
|
2324
|
+
setNativeValue(target, value);
|
|
2325
|
+
dispatchInputEvents(target);
|
|
2326
|
+
if (target.value !== value) return {
|
|
2327
|
+
content: `"${selector}" 填写后值不一致:期望 "${value}",实际 "${target.value}"`,
|
|
2328
|
+
details: {
|
|
2329
|
+
error: true,
|
|
2330
|
+
code: "FILL_NOT_APPLIED",
|
|
2331
|
+
action,
|
|
2332
|
+
selector
|
|
2333
|
+
}
|
|
2334
|
+
};
|
|
2335
|
+
const suffix = sourceHint ? `(${sourceHint})` : "";
|
|
2336
|
+
return { content: `已填写 ${describeElement(target)}: "${value}"${suffix}` };
|
|
2337
|
+
}
|
|
2338
|
+
if (target instanceof HTMLTextAreaElement) {
|
|
2339
|
+
scrollIntoViewIfNeeded(target);
|
|
2340
|
+
target.focus();
|
|
2341
|
+
selectText(target);
|
|
2342
|
+
setNativeValue(target, value);
|
|
2343
|
+
dispatchInputEvents(target);
|
|
2344
|
+
const suffix = sourceHint ? `(${sourceHint})` : "";
|
|
2345
|
+
return { content: `已填写 ${describeElement(target)}: "${value}"${suffix}` };
|
|
2346
|
+
}
|
|
2347
|
+
if (target instanceof HTMLSelectElement) {
|
|
2348
|
+
target.focus();
|
|
2349
|
+
const options = Array.from(target.options);
|
|
2350
|
+
let matched = options.find((o) => o.value === value);
|
|
2351
|
+
if (!matched) {
|
|
2352
|
+
const normalized = value.trim().toLowerCase();
|
|
2353
|
+
matched = options.find((o) => o.text.trim().toLowerCase() === normalized);
|
|
2354
|
+
}
|
|
2355
|
+
if (!matched) return { content: `"${selector}" 下拉框中不存在选项 "${value}"` };
|
|
2356
|
+
target.value = matched.value;
|
|
2357
|
+
dispatchInputEvents(target);
|
|
2358
|
+
const suffix = sourceHint ? `(${sourceHint})` : "";
|
|
2359
|
+
return { content: `已填写 ${describeElement(target)}: "${value}"${suffix}` };
|
|
2360
|
+
}
|
|
2361
|
+
if (target instanceof HTMLElement && target.isContentEditable) {
|
|
2362
|
+
target.focus();
|
|
2363
|
+
selectText(target);
|
|
2364
|
+
if (value) document.execCommand("insertText", false, value);
|
|
2365
|
+
else document.execCommand("delete", false, void 0);
|
|
2366
|
+
const suffix = sourceHint ? `(${sourceHint})` : "";
|
|
2367
|
+
return { content: `已填写 ${describeElement(target)}: "${value}"${suffix}` };
|
|
2368
|
+
}
|
|
2369
|
+
return null;
|
|
2370
|
+
}
|
|
2371
|
+
function guessNearbyFillTarget(anchor, value) {
|
|
2372
|
+
const preferNumeric = Number.isFinite(Number(value));
|
|
2373
|
+
const scopeEntries = [];
|
|
2374
|
+
const formItem = anchor.closest(".el-form-item");
|
|
2375
|
+
if (formItem) scopeEntries.push({
|
|
2376
|
+
scope: formItem,
|
|
2377
|
+
level: 0
|
|
2378
|
+
});
|
|
2379
|
+
let cursor = anchor.parentElement;
|
|
2380
|
+
for (let level = 1; cursor && level <= 4; level++, cursor = cursor.parentElement) scopeEntries.push({
|
|
2381
|
+
scope: cursor,
|
|
2382
|
+
level
|
|
2383
|
+
});
|
|
2384
|
+
const visited = /* @__PURE__ */ new Set();
|
|
2385
|
+
let best = null;
|
|
2386
|
+
for (const { scope, level } of scopeEntries) {
|
|
2387
|
+
const candidates = Array.from(scope.querySelectorAll("input:not([type=\"hidden\"]), textarea, select, [contenteditable=\"true\"], [role=\"spinbutton\"]"));
|
|
2388
|
+
for (const candidate of candidates) {
|
|
2389
|
+
if (!(candidate instanceof Element)) continue;
|
|
2390
|
+
if (visited.has(candidate)) continue;
|
|
2391
|
+
visited.add(candidate);
|
|
2392
|
+
if (!isCandidateFillTarget(candidate)) continue;
|
|
2393
|
+
if (!isElementVisible(candidate)) continue;
|
|
2394
|
+
let score = 100 - level * 18;
|
|
2395
|
+
score += getFillEventSupportScore(candidate);
|
|
2396
|
+
if (candidate instanceof HTMLInputElement) {
|
|
2397
|
+
const type = candidate.type.toLowerCase();
|
|
2398
|
+
if (preferNumeric && (type === "number" || candidate.getAttribute("role") === "spinbutton")) score += 80;
|
|
2399
|
+
if (!preferNumeric && [
|
|
2400
|
+
"text",
|
|
2401
|
+
"",
|
|
2402
|
+
"search",
|
|
2403
|
+
"email",
|
|
2404
|
+
"tel",
|
|
2405
|
+
"url",
|
|
2406
|
+
"password"
|
|
2407
|
+
].includes(type)) score += 36;
|
|
2408
|
+
}
|
|
2409
|
+
if (candidate.getAttribute("placeholder")) score += 8;
|
|
2410
|
+
if (candidate.getAttribute("aria-label")) score += 8;
|
|
2411
|
+
if (!best || score > best.score) best = {
|
|
2412
|
+
el: candidate,
|
|
2413
|
+
score
|
|
2414
|
+
};
|
|
2415
|
+
}
|
|
2416
|
+
}
|
|
2417
|
+
return best?.el ?? null;
|
|
2418
|
+
}
|
|
2209
2419
|
function selectText(el) {
|
|
2210
2420
|
if (el instanceof HTMLInputElement) {
|
|
2211
2421
|
el.select();
|
|
@@ -2288,31 +2498,25 @@ function executePress(el, key) {
|
|
|
2288
2498
|
...modState
|
|
2289
2499
|
}));
|
|
2290
2500
|
}
|
|
2291
|
-
|
|
2292
|
-
|
|
2293
|
-
|
|
2294
|
-
|
|
2295
|
-
|
|
2296
|
-
|
|
2297
|
-
|
|
2298
|
-
|
|
2299
|
-
|
|
2300
|
-
|
|
2301
|
-
|
|
2302
|
-
|
|
2303
|
-
|
|
2304
|
-
|
|
2305
|
-
|
|
2306
|
-
|
|
2307
|
-
if (!el.matches("input, textarea, select") && !el.isContentEditable) if (mode === "button-link") el = el.closest("button, [role=button], a, [role=link]") || el;
|
|
2308
|
-
else el = el.closest("button, [role=button], [role=checkbox], [role=radio]") || el;
|
|
2309
|
-
if (mode === "follow-label") {
|
|
2310
|
-
if (!el.matches("a, input, textarea, button, select, [role=link], [role=button], [role=checkbox], [role=radio]") && !el.isContentEditable) {
|
|
2311
|
-
const label = el.closest("label");
|
|
2312
|
-
if (label?.control) el = label.control;
|
|
2313
|
-
}
|
|
2501
|
+
function describeElement(el) {
|
|
2502
|
+
const tag = el.tagName.toLowerCase();
|
|
2503
|
+
const id = el.id ? `#${el.id}` : "";
|
|
2504
|
+
const cls = el.className && typeof el.className === "string" ? el.className.trim().split(/\s+/).filter(Boolean).slice(0, 3).map((c) => `.${c}`).join("") : "";
|
|
2505
|
+
const text = el instanceof HTMLSelectElement ? el.selectedOptions[0]?.textContent?.trim().slice(0, 40) ?? "" : el.textContent?.trim().slice(0, 40) ?? "";
|
|
2506
|
+
const textHint = text ? ` "${text}"` : "";
|
|
2507
|
+
const hints = [];
|
|
2508
|
+
for (const attr of [
|
|
2509
|
+
"type",
|
|
2510
|
+
"name",
|
|
2511
|
+
"placeholder",
|
|
2512
|
+
"href",
|
|
2513
|
+
"role"
|
|
2514
|
+
]) {
|
|
2515
|
+
const v = el.getAttribute(attr);
|
|
2516
|
+
if (v) hints.push(`${attr}=${v}`);
|
|
2314
2517
|
}
|
|
2315
|
-
|
|
2518
|
+
if (el instanceof HTMLSelectElement && el.value) hints.push(`val=${el.value}`);
|
|
2519
|
+
return `<${tag}${id}${cls}>${textHint}${hints.length > 0 ? ` [${hints.join(", ")}]` : ""}`;
|
|
2316
2520
|
}
|
|
2317
2521
|
function getChecked(el) {
|
|
2318
2522
|
if (el instanceof HTMLInputElement && (el.type === "checkbox" || el.type === "radio")) return el.checked;
|
|
@@ -2359,30 +2563,6 @@ function resolvePointerActionTarget(el) {
|
|
|
2359
2563
|
return el;
|
|
2360
2564
|
}
|
|
2361
2565
|
/**
|
|
2362
|
-
* 点击目标上卷:当命中文本/装饰子节点时,优先上卷到最近可点击祖先。
|
|
2363
|
-
*
|
|
2364
|
-
* 典型场景:
|
|
2365
|
-
* - 列表项文本本身无 click,但父级容器(如 .g-pointer)有点击语义
|
|
2366
|
-
* - 事件委托绑定在祖先,子节点点击命中不稳定
|
|
2367
|
-
*/
|
|
2368
|
-
function resolveClickableAncestorTarget(el) {
|
|
2369
|
-
const isSelfClickable = (node) => {
|
|
2370
|
-
if (node.matches("a, button, input, textarea, select, summary, [role=button], [role=link], [role=menuitem]")) return true;
|
|
2371
|
-
if (node.hasAttribute("onclick")) return true;
|
|
2372
|
-
const tabIndexAttr = node.getAttribute("tabindex");
|
|
2373
|
-
if (tabIndexAttr !== null && tabIndexAttr !== "-1") return true;
|
|
2374
|
-
if (getTrackedElementEvents(node).some((name) => name === "click" || name === "pointerdown" || name === "mousedown")) return true;
|
|
2375
|
-
return false;
|
|
2376
|
-
};
|
|
2377
|
-
if (isSelfClickable(el)) return el;
|
|
2378
|
-
let ancestor = el.parentElement;
|
|
2379
|
-
for (let depth = 0; ancestor && depth < 6; depth++, ancestor = ancestor.parentElement) {
|
|
2380
|
-
if (!isElementVisible(ancestor)) continue;
|
|
2381
|
-
if (isSelfClickable(ancestor)) return ancestor;
|
|
2382
|
-
}
|
|
2383
|
-
return el;
|
|
2384
|
-
}
|
|
2385
|
-
/**
|
|
2386
2566
|
* 当命中表单项说明 label(如 Element Plus el-form-item__label)时,
|
|
2387
2567
|
* 自动重定向到同一表单项中的首个可交互控件。
|
|
2388
2568
|
*/
|
|
@@ -2397,35 +2577,6 @@ function resolveFormItemControlTarget(el) {
|
|
|
2397
2577
|
if (control && isElementVisible(control)) return control;
|
|
2398
2578
|
return el;
|
|
2399
2579
|
}
|
|
2400
|
-
/**
|
|
2401
|
-
* 穿透包裹容器,查找内部可编辑子元素。
|
|
2402
|
-
* 覆盖 UI 框架常见模式:wrapper div 包裹真实 input/textarea。
|
|
2403
|
-
* 若自身已可编辑则直接返回;否则在子树中搜索第一个可编辑且可见的控件。
|
|
2404
|
-
* 对 role=slider/spinbutton 等 ARIA widget:向上逐级查找最近容器中的关联 input。
|
|
2405
|
-
*/
|
|
2406
|
-
function resolveEditableTarget(el) {
|
|
2407
|
-
if (isEditableElement(el)) return el;
|
|
2408
|
-
const inner = el.querySelector("input:not([type=\"hidden\"]), textarea, select, [contenteditable=\"true\"]");
|
|
2409
|
-
if (inner && isEditableElement(inner) && isElementVisible(inner)) return inner;
|
|
2410
|
-
const role = el.getAttribute("role");
|
|
2411
|
-
if (role === "slider" || role === "spinbutton") {
|
|
2412
|
-
let ancestor = el.parentElement;
|
|
2413
|
-
for (let depth = 0; ancestor && depth < 5; depth++, ancestor = ancestor.parentElement) {
|
|
2414
|
-
const input = ancestor.querySelector("input[type=\"number\"], input[role=\"spinbutton\"], input:not([type=\"hidden\"])");
|
|
2415
|
-
if (input instanceof HTMLInputElement && isEditableElement(input) && isElementVisible(input)) return input;
|
|
2416
|
-
}
|
|
2417
|
-
}
|
|
2418
|
-
return el;
|
|
2419
|
-
}
|
|
2420
|
-
|
|
2421
|
-
//#endregion
|
|
2422
|
-
//#region src/web/tools/dom-tool/dropdown.ts
|
|
2423
|
-
/**
|
|
2424
|
-
* DOM Tool — 自定义下拉增强。
|
|
2425
|
-
*
|
|
2426
|
-
* 包含:全局可见 option 查找、下拉弹出等待。
|
|
2427
|
-
*/
|
|
2428
|
-
/** 在全局可见 option 节点中按文本匹配(精确 → 包含) */
|
|
2429
2580
|
function findVisibleOptionByText(text) {
|
|
2430
2581
|
const target = text.trim().toLowerCase();
|
|
2431
2582
|
if (!target) return null;
|
|
@@ -2446,7 +2597,6 @@ function findVisibleOptionByText(text) {
|
|
|
2446
2597
|
for (const n of visible) if (n.textContent?.trim().toLowerCase().includes(target)) return n;
|
|
2447
2598
|
return null;
|
|
2448
2599
|
}
|
|
2449
|
-
/** 轮询等待下拉弹出层出现 */
|
|
2450
2600
|
async function waitForDropdownPopup(maxWait = 500) {
|
|
2451
2601
|
const start = Date.now();
|
|
2452
2602
|
while (Date.now() - start < maxWait) {
|
|
@@ -2455,33 +2605,22 @@ async function waitForDropdownPopup(maxWait = 500) {
|
|
|
2455
2605
|
await sleep(50);
|
|
2456
2606
|
}
|
|
2457
2607
|
}
|
|
2458
|
-
|
|
2459
|
-
//#endregion
|
|
2460
|
-
//#region src/web/tools/dom-tool/index.ts
|
|
2461
|
-
/**
|
|
2462
|
-
* DOM Tool — 浏览器 DOM 操作工具入口(结合 Playwright 核心交互模式增强)。
|
|
2463
|
-
*
|
|
2464
|
-
* 关键能力:
|
|
2465
|
-
* 1. retarget — 点击时自动重定向到 button/link/label.control
|
|
2466
|
-
* 2. scrollIntoView 多策略 — 4 种 block 对齐轮换,解决 sticky 遮挡
|
|
2467
|
-
* 3. stable 检查 — rAF 逐帧检测元素位置稳定后再操作
|
|
2468
|
-
* 4. hit-target 验证 — elementsFromPoint 检查是否被遮挡
|
|
2469
|
-
* 5. 完整点击事件链 — pointermove→pointerdown→mousedown→pointerup→mouseup→click
|
|
2470
|
-
* 6. check/uncheck 通过 click — 先检查→click 切换→验证状态
|
|
2471
|
-
* 7. press 组合键 — 支持 Control+a, Shift+Enter 等修饰键
|
|
2472
|
-
* 8. fill 分类型 — date/color/range 走 setValue,text 类走 selectAll+原生写入
|
|
2473
|
-
* 9. 自定义下拉增强 — 更广泛的 option 选择器 + 等待弹出
|
|
2474
|
-
* 10. ARIA disabled — 检查祖先链 aria-disabled
|
|
2475
|
-
*
|
|
2476
|
-
* 运行环境:浏览器 Content Script(直接访问 DOM,无 CDP)。
|
|
2477
|
-
*/
|
|
2478
2608
|
function createDomTool() {
|
|
2479
2609
|
return {
|
|
2480
2610
|
name: "dom",
|
|
2481
2611
|
description: [
|
|
2482
2612
|
"Perform DOM operations on the current page.",
|
|
2483
2613
|
"Actions: click, fill, select_option, clear, check, uncheck, type, focus, hover, scroll, press, get_text, get_attr, set_attr, add_class, remove_class.",
|
|
2484
|
-
"
|
|
2614
|
+
"Input/Select rule: before each fill/type/select_option, click or focus the same target immediately in the same round.",
|
|
2615
|
+
"For multiple fields, use alternating pairs in one batch: focus/click A -> fill/type A -> focus/click B -> fill/type B.",
|
|
2616
|
+
"Use the hash ID from DOM snapshot (e.g. #a1b2c) as selector.",
|
|
2617
|
+
"press supports combo keys like 'Control+a', 'Shift+Enter'.",
|
|
2618
|
+
"check/uncheck is done via click — state change is verified after action.",
|
|
2619
|
+
"Ordinal/index rule: treat visual order as 1-based when the instruction says 'the Nth item' (e.g. 4th star = 4th visible icon from left to right), and avoid off-by-one mistakes.",
|
|
2620
|
+
"Disambiguation rule: distinguish descriptive text/labels from actionable options. Do not click nearby label/help text; click the actual interactive option/control item (icon/button/option) that changes state.",
|
|
2621
|
+
"Unknown/complex components: if a container element (e.g. role=slider, rating, custom widget) has multiple child icons/items in the snapshot but you don't know how to operate it directly, try clicking the appropriate child element instead. For example, a rating component with 5 star icon children — click the 4th icon child to set 4 stars. A slider with a runway — clicking the runway at the right position may work. Always prefer interacting with visible children when the parent container doesn't respond to fill/click as expected.",
|
|
2622
|
+
"fill supports role=slider elements: use fill with a numeric value on a role=slider container (rating/slider) to set its value programmatically.",
|
|
2623
|
+
"For wheel/virtualized pickers where target option is not visible yet, use scroll on the picker column first, then click/select the newly visible option. scroll supports steps for repeated scrolling in one call."
|
|
2485
2624
|
].join(" "),
|
|
2486
2625
|
schema: Type.Object({
|
|
2487
2626
|
action: Type.String({ description: "DOM action: click | fill | select_option | clear | check | uncheck | type | focus | hover | scroll | press | get_text | get_attr | set_attr | add_class | remove_class." }),
|
|
@@ -2544,18 +2683,13 @@ function createDomTool() {
|
|
|
2544
2683
|
el = r;
|
|
2545
2684
|
}
|
|
2546
2685
|
if (action === "check" || action === "uncheck") el = resolveCheckableTarget(el);
|
|
2547
|
-
|
|
2548
|
-
"fill",
|
|
2549
|
-
"type",
|
|
2550
|
-
"clear"
|
|
2551
|
-
].includes(action)) el = resolveEditableTarget(retarget(el, "follow-label"));
|
|
2552
|
-
const actionabilityTarget = action === "click" || action === "check" || action === "uncheck" ? resolvePointerActionTarget(resolveClickableAncestorTarget(resolveFormItemControlTarget(el))) : el;
|
|
2686
|
+
const actionabilityTarget = action === "click" || action === "check" || action === "uncheck" ? resolvePointerActionTarget(resolveFormItemControlTarget(el)) : el;
|
|
2553
2687
|
try {
|
|
2554
2688
|
const checkResult = ensureActionable(actionabilityTarget, action, selector, force);
|
|
2555
2689
|
if (checkResult) return checkResult;
|
|
2556
2690
|
switch (action) {
|
|
2557
2691
|
case "click": {
|
|
2558
|
-
const target = resolvePointerActionTarget(
|
|
2692
|
+
const target = resolvePointerActionTarget(resolveFormItemControlTarget(retarget(el, force ? "none" : "button-link")));
|
|
2559
2693
|
const clickCount = typeof params.clickCount === "number" ? params.clickCount : 1;
|
|
2560
2694
|
if (target instanceof HTMLOptionElement) {
|
|
2561
2695
|
const parent = target.parentElement;
|
|
@@ -2582,88 +2716,72 @@ function createDomTool() {
|
|
|
2582
2716
|
case "fill": {
|
|
2583
2717
|
const value = params.value;
|
|
2584
2718
|
if (value === void 0) return { content: "缺少 value 参数" };
|
|
2585
|
-
const target = el;
|
|
2586
|
-
if (target instanceof
|
|
2587
|
-
const
|
|
2588
|
-
if (
|
|
2589
|
-
|
|
2590
|
-
|
|
2591
|
-
|
|
2592
|
-
|
|
2593
|
-
action,
|
|
2594
|
-
selector
|
|
2719
|
+
const target = retarget(el, "follow-label");
|
|
2720
|
+
if (target instanceof HTMLElement && target.getAttribute("role") === "slider") {
|
|
2721
|
+
const numericValue = Number(value);
|
|
2722
|
+
if (!Number.isFinite(numericValue)) {
|
|
2723
|
+
const guessed = guessNearbyFillTarget(target, value);
|
|
2724
|
+
if (guessed) {
|
|
2725
|
+
const guessedResult = executeFillOnResolvedTarget(guessed, value, selector, action, "heuristic-nearby-target");
|
|
2726
|
+
if (guessedResult) return guessedResult;
|
|
2595
2727
|
}
|
|
2596
|
-
|
|
2597
|
-
|
|
2598
|
-
const finalVal = type === "color" ? value.toLowerCase().trim() : value.trim();
|
|
2599
|
-
target.focus();
|
|
2600
|
-
target.value = finalVal;
|
|
2601
|
-
if (target.value !== finalVal) return {
|
|
2602
|
-
content: `"${selector}" 填写格式不匹配(type=${type})`,
|
|
2728
|
+
return {
|
|
2729
|
+
content: `"${selector}" 为 role=slider,未找到可推断填写目标`,
|
|
2603
2730
|
details: {
|
|
2604
2731
|
error: true,
|
|
2605
|
-
code: "
|
|
2732
|
+
code: "UNSUPPORTED_FILL_TARGET",
|
|
2606
2733
|
action,
|
|
2607
2734
|
selector
|
|
2608
2735
|
}
|
|
2609
2736
|
};
|
|
2610
|
-
dispatchInputEvents(target);
|
|
2611
|
-
return { content: `已填写 ${describeElement(target)}: "${finalVal}"` };
|
|
2612
2737
|
}
|
|
2613
|
-
|
|
2614
|
-
|
|
2615
|
-
|
|
2616
|
-
|
|
2617
|
-
|
|
2618
|
-
|
|
2619
|
-
|
|
2620
|
-
|
|
2621
|
-
|
|
2622
|
-
|
|
2623
|
-
|
|
2624
|
-
|
|
2625
|
-
|
|
2626
|
-
|
|
2627
|
-
|
|
2628
|
-
|
|
2738
|
+
const linkedInput = findAssociatedSliderInput(target);
|
|
2739
|
+
if (linkedInput) {
|
|
2740
|
+
const filled = executeFillOnResolvedTarget(linkedInput, String(numericValue), selector, action, `from ${describeElement(target)}`);
|
|
2741
|
+
if (filled) return filled;
|
|
2742
|
+
}
|
|
2743
|
+
const min = Number(target.getAttribute("aria-valuemin") ?? "1");
|
|
2744
|
+
const max = Number(target.getAttribute("aria-valuemax") ?? String(target.children.length || 5));
|
|
2745
|
+
const discreteCount = Number.isFinite(max - min + 1) ? Math.max(1, Math.round(max - min + 1)) : target.children.length;
|
|
2746
|
+
const desiredIndex = Math.round(numericValue - min);
|
|
2747
|
+
const children = Array.from(target.children).filter((node) => node instanceof HTMLElement);
|
|
2748
|
+
if (children.length >= discreteCount && desiredIndex >= 0 && desiredIndex < children.length) {
|
|
2749
|
+
const item = children[desiredIndex];
|
|
2750
|
+
scrollIntoViewIfNeeded(item);
|
|
2751
|
+
dispatchClickEvents(item);
|
|
2752
|
+
return { content: `已点击 ${describeElement(item)},设置 ${describeElement(target)} 值为 ${numericValue}` };
|
|
2753
|
+
}
|
|
2754
|
+
const guessed = guessNearbyFillTarget(target, String(numericValue));
|
|
2755
|
+
if (guessed) {
|
|
2756
|
+
const guessedResult = executeFillOnResolvedTarget(guessed, String(numericValue), selector, action, "heuristic-nearby-target");
|
|
2757
|
+
if (guessedResult) return guessedResult;
|
|
2758
|
+
}
|
|
2759
|
+
return {
|
|
2760
|
+
content: `"${selector}" 为 role=slider,但未找到可写入输入框或可点击离散子项`,
|
|
2629
2761
|
details: {
|
|
2630
2762
|
error: true,
|
|
2631
|
-
code: "
|
|
2763
|
+
code: "UNSUPPORTED_FILL_TARGET",
|
|
2632
2764
|
action,
|
|
2633
2765
|
selector
|
|
2634
2766
|
}
|
|
2635
2767
|
};
|
|
2636
|
-
return { content: `已填写 ${describeElement(target)}: "${value}"` };
|
|
2637
2768
|
}
|
|
2638
|
-
|
|
2639
|
-
|
|
2640
|
-
|
|
2641
|
-
|
|
2642
|
-
|
|
2643
|
-
|
|
2644
|
-
return { content: `已填写 ${describeElement(target)}: "${value}"` };
|
|
2769
|
+
const directFilled = executeFillOnResolvedTarget(target, value, selector, action);
|
|
2770
|
+
if (directFilled) return directFilled;
|
|
2771
|
+
const guessed = guessNearbyFillTarget(target, value);
|
|
2772
|
+
if (guessed) {
|
|
2773
|
+
const guessedResult = executeFillOnResolvedTarget(guessed, value, selector, action, "heuristic-nearby-target");
|
|
2774
|
+
if (guessedResult) return guessedResult;
|
|
2645
2775
|
}
|
|
2646
|
-
|
|
2647
|
-
|
|
2648
|
-
|
|
2649
|
-
|
|
2650
|
-
|
|
2651
|
-
|
|
2652
|
-
|
|
2776
|
+
return {
|
|
2777
|
+
content: `"${selector}" 不是可编辑元素,且未在附近找到可推断填写目标`,
|
|
2778
|
+
details: {
|
|
2779
|
+
error: true,
|
|
2780
|
+
code: "UNSUPPORTED_FILL_TARGET",
|
|
2781
|
+
action,
|
|
2782
|
+
selector
|
|
2653
2783
|
}
|
|
2654
|
-
|
|
2655
|
-
target.value = matched.value;
|
|
2656
|
-
dispatchInputEvents(target);
|
|
2657
|
-
return { content: `已填写 ${describeElement(target)}: "${value}"` };
|
|
2658
|
-
}
|
|
2659
|
-
if (target instanceof HTMLElement && target.isContentEditable) {
|
|
2660
|
-
target.focus();
|
|
2661
|
-
selectText(target);
|
|
2662
|
-
if (value) document.execCommand("insertText", false, value);
|
|
2663
|
-
else document.execCommand("delete", false, void 0);
|
|
2664
|
-
return { content: `已填写 ${describeElement(target)}: "${value}"` };
|
|
2665
|
-
}
|
|
2666
|
-
return { content: `"${selector}" 不是可编辑元素` };
|
|
2784
|
+
};
|
|
2667
2785
|
}
|
|
2668
2786
|
case "select_option": {
|
|
2669
2787
|
const value = params.value;
|
|
@@ -2725,7 +2843,7 @@ function createDomTool() {
|
|
|
2725
2843
|
return { content: `已选择 ${describeElement(target)}: value="${selected.value}", label="${selected.text.trim()}"` };
|
|
2726
2844
|
}
|
|
2727
2845
|
case "clear": {
|
|
2728
|
-
const target = el;
|
|
2846
|
+
const target = retarget(el, "follow-label");
|
|
2729
2847
|
if (target instanceof HTMLInputElement || target instanceof HTMLTextAreaElement) {
|
|
2730
2848
|
scrollIntoViewIfNeeded(target);
|
|
2731
2849
|
target.focus();
|
|
@@ -2785,7 +2903,7 @@ function createDomTool() {
|
|
|
2785
2903
|
case "type": {
|
|
2786
2904
|
const value = params.value;
|
|
2787
2905
|
if (value === void 0) return { content: "缺少 value 参数" };
|
|
2788
|
-
const target = el;
|
|
2906
|
+
const target = retarget(el, "follow-label");
|
|
2789
2907
|
scrollIntoViewIfNeeded(target);
|
|
2790
2908
|
if (target instanceof HTMLElement) target.focus();
|
|
2791
2909
|
for (const char of value) {
|
|
@@ -2951,11 +3069,7 @@ const MAX_SNAPSHOT_ATTR_VALUE_LENGTH = 120;
|
|
|
2951
3069
|
const MAX_EXPANDED_LIST_CHILDREN = 120;
|
|
2952
3070
|
/** 定向放宽 children 的硬上限。 */
|
|
2953
3071
|
const MAX_EXPANDED_CHILDREN_LIMIT = 300;
|
|
2954
|
-
/**
|
|
2955
|
-
* 事件名 → 快照简写映射。
|
|
2956
|
-
* 目的:大幅压缩 listeners="..." 占用的 token,同时保留可读性。
|
|
2957
|
-
* 简写规则在 system-prompt 中向模型说明。
|
|
2958
|
-
*/
|
|
3072
|
+
/** 事件名 → 快照简写映射(压缩 token)。 */
|
|
2959
3073
|
const EVENT_ABBREV = {
|
|
2960
3074
|
click: "clk",
|
|
2961
3075
|
dblclick: "dbl",
|
|
@@ -2969,14 +3083,10 @@ const EVENT_ABBREV = {
|
|
|
2969
3083
|
pointerdown: "pdn",
|
|
2970
3084
|
pointerup: "pup",
|
|
2971
3085
|
pointermove: "pmv",
|
|
2972
|
-
pointerenter: "pen",
|
|
2973
|
-
pointerleave: "plv",
|
|
2974
3086
|
touchstart: "tst",
|
|
2975
3087
|
touchend: "ted",
|
|
2976
|
-
touchmove: "tmv",
|
|
2977
3088
|
keydown: "kdn",
|
|
2978
3089
|
keyup: "kup",
|
|
2979
|
-
keypress: "kpr",
|
|
2980
3090
|
input: "inp",
|
|
2981
3091
|
change: "chg",
|
|
2982
3092
|
submit: "sub",
|
|
@@ -2988,10 +3098,8 @@ const EVENT_ABBREV = {
|
|
|
2988
3098
|
dragstart: "drs",
|
|
2989
3099
|
dragend: "dre",
|
|
2990
3100
|
drop: "drp",
|
|
2991
|
-
contextmenu: "ctx"
|
|
2992
|
-
resize: "rsz"
|
|
3101
|
+
contextmenu: "ctx"
|
|
2993
3102
|
};
|
|
2994
|
-
/** 将完整事件名转为快照简写(未收录的取前 3 字符)。 */
|
|
2995
3103
|
function abbrevEvent(name) {
|
|
2996
3104
|
return EVENT_ABBREV[name] ?? name.slice(0, 3);
|
|
2997
3105
|
}
|
|
@@ -3036,18 +3144,17 @@ function sanitizeSnapshotAttrValue(value) {
|
|
|
3036
3144
|
*/
|
|
3037
3145
|
function generateSnapshot(root = document.body, options = {}) {
|
|
3038
3146
|
const opts = typeof options === "number" ? { maxDepth: options } : options;
|
|
3039
|
-
const maxDepth = opts.maxDepth ??
|
|
3147
|
+
const maxDepth = opts.maxDepth ?? 12;
|
|
3040
3148
|
const viewportOnly = opts.viewportOnly ?? true;
|
|
3041
3149
|
const pruneLayout = opts.pruneLayout ?? true;
|
|
3042
|
-
const maxNodes = opts.maxNodes ??
|
|
3043
|
-
const maxChildren = opts.maxChildren ??
|
|
3150
|
+
const maxNodes = opts.maxNodes ?? 220;
|
|
3151
|
+
const maxChildren = opts.maxChildren ?? 25;
|
|
3044
3152
|
const maxTextLength = opts.maxTextLength ?? 40;
|
|
3045
3153
|
const expandOptionLists = opts.expandOptionLists ?? false;
|
|
3046
3154
|
const expandedChildrenLimit = Math.min(MAX_EXPANDED_CHILDREN_LIMIT, Math.max(1, opts.expandedChildrenLimit ?? MAX_EXPANDED_LIST_CHILDREN));
|
|
3047
3155
|
const expandChildrenRefSet = new Set((opts.expandChildrenRefs ?? []).map((ref) => ref.trim().replace(/^#/, "")).filter(Boolean));
|
|
3048
3156
|
let emittedNodes = 0;
|
|
3049
3157
|
let truncatedByNodeBudget = false;
|
|
3050
|
-
const emittedRefIds = /* @__PURE__ */ new Set();
|
|
3051
3158
|
const refStore = opts.refStore;
|
|
3052
3159
|
const SKIP_TAGS = new Set([
|
|
3053
3160
|
"SCRIPT",
|
|
@@ -3083,11 +3190,7 @@ function generateSnapshot(root = document.body, options = {}) {
|
|
|
3083
3190
|
"value",
|
|
3084
3191
|
"name",
|
|
3085
3192
|
"role",
|
|
3086
|
-
"tabindex",
|
|
3087
3193
|
"aria-label",
|
|
3088
|
-
"aria-valuenow",
|
|
3089
|
-
"aria-valuemin",
|
|
3090
|
-
"aria-valuemax",
|
|
3091
3194
|
"src",
|
|
3092
3195
|
"alt",
|
|
3093
3196
|
"title",
|
|
@@ -3105,7 +3208,6 @@ function generateSnapshot(root = document.body, options = {}) {
|
|
|
3105
3208
|
"LABEL",
|
|
3106
3209
|
"SUMMARY"
|
|
3107
3210
|
]);
|
|
3108
|
-
/** 常见可交互事件(用于提升元素交互优先级)。 */
|
|
3109
3211
|
const INTERACTIVE_EVENTS = new Set([
|
|
3110
3212
|
"click",
|
|
3111
3213
|
"dblclick",
|
|
@@ -3119,11 +3221,51 @@ function generateSnapshot(root = document.body, options = {}) {
|
|
|
3119
3221
|
"change",
|
|
3120
3222
|
"keydown",
|
|
3121
3223
|
"keyup",
|
|
3122
|
-
"keypress",
|
|
3123
3224
|
"submit",
|
|
3124
3225
|
"focus",
|
|
3125
3226
|
"blur"
|
|
3126
3227
|
]);
|
|
3228
|
+
/** 交互性 ARIA role — 需要分配 hash ID 的角色集合 */
|
|
3229
|
+
const INTERACTIVE_ROLES = new Set([
|
|
3230
|
+
"button",
|
|
3231
|
+
"link",
|
|
3232
|
+
"tab",
|
|
3233
|
+
"switch",
|
|
3234
|
+
"slider",
|
|
3235
|
+
"checkbox",
|
|
3236
|
+
"radio",
|
|
3237
|
+
"combobox",
|
|
3238
|
+
"listbox",
|
|
3239
|
+
"option",
|
|
3240
|
+
"menuitem",
|
|
3241
|
+
"textbox",
|
|
3242
|
+
"spinbutton",
|
|
3243
|
+
"searchbox",
|
|
3244
|
+
"treeitem",
|
|
3245
|
+
"gridcell",
|
|
3246
|
+
"scrollbar"
|
|
3247
|
+
]);
|
|
3248
|
+
/**
|
|
3249
|
+
* 事件优先级(值越大越优先):
|
|
3250
|
+
* 输入链路(input/change/focus/blur) > 点击链路(click/pointer) > 其他事件。
|
|
3251
|
+
*/
|
|
3252
|
+
const EVENT_PRIORITY = {
|
|
3253
|
+
input: 140,
|
|
3254
|
+
change: 130,
|
|
3255
|
+
focus: 120,
|
|
3256
|
+
blur: 110,
|
|
3257
|
+
keydown: 100,
|
|
3258
|
+
keyup: 90,
|
|
3259
|
+
click: 80,
|
|
3260
|
+
dblclick: 70,
|
|
3261
|
+
pointerdown: 60,
|
|
3262
|
+
pointerup: 55,
|
|
3263
|
+
mousedown: 50,
|
|
3264
|
+
mouseup: 45,
|
|
3265
|
+
touchstart: 40,
|
|
3266
|
+
touchend: 35,
|
|
3267
|
+
submit: 30
|
|
3268
|
+
};
|
|
3127
3269
|
/** 布尔状态属性 — 只在存在时输出(无值),如 disabled、checked */
|
|
3128
3270
|
const BOOLEAN_ATTRS = [
|
|
3129
3271
|
"disabled",
|
|
@@ -3158,70 +3300,6 @@ function generateSnapshot(root = document.body, options = {}) {
|
|
|
3158
3300
|
if (rect.width === 0 && rect.height === 0) return false;
|
|
3159
3301
|
return true;
|
|
3160
3302
|
}
|
|
3161
|
-
/** 统一标签名键值(HTML/SVG 在不同环境可能大小写不一致)。 */
|
|
3162
|
-
function getTagKey(el) {
|
|
3163
|
-
return (el.tagName || "").toUpperCase();
|
|
3164
|
-
}
|
|
3165
|
-
/** 判断元素是否存在绑定事件(inline 或 addEventListener 追踪)。 */
|
|
3166
|
-
function hasBoundEvents(el) {
|
|
3167
|
-
if (hasTrackedElementEvents(el)) return true;
|
|
3168
|
-
for (const attr of Array.from(el.attributes)) if (attr.name.startsWith("on")) return true;
|
|
3169
|
-
return false;
|
|
3170
|
-
}
|
|
3171
|
-
/**
|
|
3172
|
-
* 轻量检测:当前容器浅层子树里是否出现事件绑定节点。
|
|
3173
|
-
* 仅用于是否保留布局容器,预算受控避免再次吞掉整页层级。
|
|
3174
|
-
*/
|
|
3175
|
-
function hasBoundEventsInShallowSubtree(el, scanBudget = 48, maxTreeDepth = 2) {
|
|
3176
|
-
const queue = Array.from(el.children).map((node) => ({
|
|
3177
|
-
node,
|
|
3178
|
-
depth: 1
|
|
3179
|
-
}));
|
|
3180
|
-
let scanned = 0;
|
|
3181
|
-
while (queue.length > 0) {
|
|
3182
|
-
const current = queue.shift();
|
|
3183
|
-
if (!current) continue;
|
|
3184
|
-
if (hasBoundEvents(current.node)) return true;
|
|
3185
|
-
scanned += 1;
|
|
3186
|
-
if (scanned >= scanBudget) return false;
|
|
3187
|
-
if (current.depth >= maxTreeDepth) continue;
|
|
3188
|
-
for (const child of Array.from(current.node.children)) queue.push({
|
|
3189
|
-
node: child,
|
|
3190
|
-
depth: current.depth + 1
|
|
3191
|
-
});
|
|
3192
|
-
}
|
|
3193
|
-
return false;
|
|
3194
|
-
}
|
|
3195
|
-
/** 判断文本是否具有语义信息(过滤纯符号/超短噪音)。 */
|
|
3196
|
-
function isSemanticText(text) {
|
|
3197
|
-
const normalized = text.replace(/\s+/g, "").trim();
|
|
3198
|
-
if (!normalized) return false;
|
|
3199
|
-
if (normalized.length < 2) return false;
|
|
3200
|
-
return /[\p{Script=Han}A-Za-z0-9]/u.test(normalized);
|
|
3201
|
-
}
|
|
3202
|
-
/** 在子树内查找语义文本(浅层限流,避免额外大开销)。 */
|
|
3203
|
-
function hasSemanticTextInSubtree(el, scanBudget = 180) {
|
|
3204
|
-
const stack = Array.from(el.children);
|
|
3205
|
-
let scanned = 0;
|
|
3206
|
-
while (stack.length > 0) {
|
|
3207
|
-
const current = stack.pop();
|
|
3208
|
-
if (!current) continue;
|
|
3209
|
-
let directText = "";
|
|
3210
|
-
for (let i = 0; i < current.childNodes.length; i++) {
|
|
3211
|
-
const node = current.childNodes[i];
|
|
3212
|
-
if (node.nodeType === Node.TEXT_NODE) {
|
|
3213
|
-
const t = node.textContent?.trim();
|
|
3214
|
-
if (t) directText += t + " ";
|
|
3215
|
-
}
|
|
3216
|
-
}
|
|
3217
|
-
if (isSemanticText(directText.trim())) return true;
|
|
3218
|
-
scanned += 1;
|
|
3219
|
-
if (scanned >= scanBudget) return false;
|
|
3220
|
-
const children = Array.from(current.children);
|
|
3221
|
-
for (let i = children.length - 1; i >= 0; i--) stack.push(children[i]);
|
|
3222
|
-
}
|
|
3223
|
-
return false;
|
|
3224
|
-
}
|
|
3225
3303
|
/**
|
|
3226
3304
|
* 判断元素是否为「无意义布局容器」(智能剪枝候选)。
|
|
3227
3305
|
* 满足所有条件时返回 true:
|
|
@@ -3230,25 +3308,60 @@ function generateSnapshot(root = document.body, options = {}) {
|
|
|
3230
3308
|
* 3. 没有交互属性(href/role/aria-label/onclick 等)
|
|
3231
3309
|
* 4. 没有直接文本内容
|
|
3232
3310
|
*/
|
|
3233
|
-
function isEmptyLayoutContainer(el, directText
|
|
3311
|
+
function isEmptyLayoutContainer(el, directText) {
|
|
3234
3312
|
if (!pruneLayout) return false;
|
|
3235
|
-
if (
|
|
3236
|
-
if (!LAYOUT_TAGS.has(getTagKey(el))) return false;
|
|
3313
|
+
if (!LAYOUT_TAGS.has(el.tagName)) return false;
|
|
3237
3314
|
if (el.getAttribute("id")) return false;
|
|
3238
3315
|
if (el.getAttribute("role") || el.getAttribute("aria-label")) return false;
|
|
3239
|
-
|
|
3240
|
-
if (
|
|
3241
|
-
if (isSemanticText(directText) || hasSemanticTextInSubtree(el)) return false;
|
|
3316
|
+
for (const attr of Array.from(el.attributes)) if (attr.name.startsWith("on")) return false;
|
|
3317
|
+
if (hasTrackedElementEvents(el)) return false;
|
|
3242
3318
|
if (directText) return false;
|
|
3243
3319
|
return true;
|
|
3244
3320
|
}
|
|
3245
3321
|
function hasInteractiveTrackedEvents(el) {
|
|
3246
|
-
const
|
|
3247
|
-
if (
|
|
3248
|
-
return
|
|
3322
|
+
const tracked = getTrackedElementEvents(el);
|
|
3323
|
+
if (tracked.length === 0) return false;
|
|
3324
|
+
return tracked.some((name) => INTERACTIVE_EVENTS.has(name));
|
|
3325
|
+
}
|
|
3326
|
+
function getTrackedEventPriorityScore(el) {
|
|
3327
|
+
const tracked = getTrackedElementEvents(el);
|
|
3328
|
+
if (tracked.length === 0) return 0;
|
|
3329
|
+
let score = 0;
|
|
3330
|
+
for (const name of tracked) score += EVENT_PRIORITY[name] ?? 8;
|
|
3331
|
+
return score;
|
|
3332
|
+
}
|
|
3333
|
+
/**
|
|
3334
|
+
* 元素优先级:
|
|
3335
|
+
* 1) 输入控件/按钮等语义控件
|
|
3336
|
+
* 2) 事件追踪优先级(输入、点击、失焦等)
|
|
3337
|
+
* 3) inline 事件与可聚焦能力补充加分
|
|
3338
|
+
*/
|
|
3339
|
+
function getElementPriorityScore(el) {
|
|
3340
|
+
let score = 0;
|
|
3341
|
+
if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement || el instanceof HTMLSelectElement) score += 200;
|
|
3342
|
+
else if (el instanceof HTMLButtonElement || el instanceof HTMLAnchorElement) score += 180;
|
|
3343
|
+
else if (el.getAttribute("role") === "button" || el.getAttribute("role") === "switch" || el.getAttribute("role") === "slider") score += 160;
|
|
3344
|
+
score += getTrackedEventPriorityScore(el);
|
|
3345
|
+
if (el.hasAttribute("onclick")) score += 60;
|
|
3346
|
+
if (el.hasAttribute("oninput") || el.hasAttribute("onchange")) score += 80;
|
|
3347
|
+
if (el.hasAttribute("onfocus") || el.hasAttribute("onblur")) score += 70;
|
|
3348
|
+
if (el.hasAttribute("tabindex")) score += 20;
|
|
3349
|
+
return score;
|
|
3350
|
+
}
|
|
3351
|
+
function orderChildrenByPriority(children) {
|
|
3352
|
+
return children.map((child, index) => ({
|
|
3353
|
+
child,
|
|
3354
|
+
index,
|
|
3355
|
+
interactive: isInteractiveElement(child),
|
|
3356
|
+
score: getElementPriorityScore(child)
|
|
3357
|
+
})).sort((a, b) => {
|
|
3358
|
+
if (a.interactive !== b.interactive) return a.interactive ? -1 : 1;
|
|
3359
|
+
if (b.score !== a.score) return b.score - a.score;
|
|
3360
|
+
return a.index - b.index;
|
|
3361
|
+
}).map((entry) => entry.child);
|
|
3249
3362
|
}
|
|
3250
3363
|
function isInteractiveElement(el) {
|
|
3251
|
-
if (INTERACTIVE_TAGS.has(
|
|
3364
|
+
if (INTERACTIVE_TAGS.has(el.tagName)) return true;
|
|
3252
3365
|
if (el.hasAttribute("onclick")) return true;
|
|
3253
3366
|
if (el.hasAttribute("role")) return true;
|
|
3254
3367
|
if (el.hasAttribute("tabindex")) return true;
|
|
@@ -3256,6 +3369,22 @@ function generateSnapshot(root = document.body, options = {}) {
|
|
|
3256
3369
|
if (hasInteractiveTrackedEvents(el)) return true;
|
|
3257
3370
|
return false;
|
|
3258
3371
|
}
|
|
3372
|
+
/**
|
|
3373
|
+
* 判断元素是否需要分配 hash ID(仅交互节点分配,节省 token)。
|
|
3374
|
+
*
|
|
3375
|
+
* 核心依据:元素是否绑定了交互事件(INTERACTIVE_EVENTS 集合)。
|
|
3376
|
+
* 辅助依据:语义交互标签、内联事件、ARIA role、tabindex 等兜底。
|
|
3377
|
+
*/
|
|
3378
|
+
function needsHashId(el) {
|
|
3379
|
+
if (hasInteractiveTrackedEvents(el)) return true;
|
|
3380
|
+
for (const attr of Array.from(el.attributes)) if (attr.name.startsWith("on")) return true;
|
|
3381
|
+
if (INTERACTIVE_TAGS.has(el.tagName)) return true;
|
|
3382
|
+
const role = el.getAttribute("role");
|
|
3383
|
+
if (role && INTERACTIVE_ROLES.has(role)) return true;
|
|
3384
|
+
if (el.hasAttribute("tabindex")) return true;
|
|
3385
|
+
if (el.isContentEditable && el.getAttribute("contenteditable") !== "inherit") return true;
|
|
3386
|
+
return false;
|
|
3387
|
+
}
|
|
3259
3388
|
/** 判断是否为“选项列表”容器(时间/下拉/listbox 等)。 */
|
|
3260
3389
|
function isOptionListContainer(el) {
|
|
3261
3390
|
if (el.getAttribute("role") === "listbox") return true;
|
|
@@ -3282,17 +3411,18 @@ function generateSnapshot(root = document.body, options = {}) {
|
|
|
3282
3411
|
return "";
|
|
3283
3412
|
}
|
|
3284
3413
|
if (depth > maxDepth) return "";
|
|
3285
|
-
|
|
3286
|
-
if (SKIP_TAGS.has(tagKey)) return "";
|
|
3287
|
-
if (el.getAttribute("id") === "__SVG_SPRITE_NODE__") return "";
|
|
3414
|
+
if (SKIP_TAGS.has(el.tagName)) return "";
|
|
3288
3415
|
if (el.hasAttribute("data-autopilot-ignore")) return "";
|
|
3289
3416
|
const style = window.getComputedStyle(el);
|
|
3290
3417
|
if (style.display === "none" || style.visibility === "hidden") return "";
|
|
3291
3418
|
if (!isInViewport(el, depth)) return "";
|
|
3292
3419
|
const indent = " ".repeat(depth);
|
|
3293
3420
|
const tag = el.tagName.toLowerCase();
|
|
3421
|
+
const rawRole = el.getAttribute("role");
|
|
3422
|
+
const useRoleAsTag = !!(rawRole && INTERACTIVE_ROLES.has(rawRole) && rawRole !== tag);
|
|
3423
|
+
const displayTag = useRoleAsTag ? rawRole : tag;
|
|
3294
3424
|
const currentPath = `${parentPath}/${tag}${getSiblingIndex(el)}`;
|
|
3295
|
-
const hashId = refStore ? refStore.set(el, currentPath) : void 0;
|
|
3425
|
+
const hashId = refStore && needsHashId(el) ? refStore.set(el, currentPath) : void 0;
|
|
3296
3426
|
const attrs = [];
|
|
3297
3427
|
const elId = el.getAttribute("id");
|
|
3298
3428
|
if (elId) attrs.push(`id="${elId}"`);
|
|
@@ -3302,6 +3432,7 @@ function generateSnapshot(root = document.body, options = {}) {
|
|
|
3302
3432
|
if (cls) attrs.push(`class="${cls}"`);
|
|
3303
3433
|
}
|
|
3304
3434
|
for (const attr of INTERACTIVE_ATTRS) {
|
|
3435
|
+
if (attr === "role" && useRoleAsTag) continue;
|
|
3305
3436
|
const val = el.getAttribute(attr);
|
|
3306
3437
|
if (val) {
|
|
3307
3438
|
const safeVal = sanitizeSnapshotAttrValue(val);
|
|
@@ -3347,11 +3478,8 @@ function generateSnapshot(root = document.body, options = {}) {
|
|
|
3347
3478
|
}
|
|
3348
3479
|
}
|
|
3349
3480
|
directText = directText.trim();
|
|
3350
|
-
if (isEmptyLayoutContainer(el, directText
|
|
3351
|
-
const
|
|
3352
|
-
const interactiveChildren = allChildren.filter(isInteractiveElement);
|
|
3353
|
-
const nonInteractiveChildren = allChildren.filter((child) => !isInteractiveElement(child));
|
|
3354
|
-
const orderedChildren = [...interactiveChildren, ...nonInteractiveChildren];
|
|
3481
|
+
if (isEmptyLayoutContainer(el, directText)) {
|
|
3482
|
+
const orderedChildren = orderChildrenByPriority(Array.from(el.children));
|
|
3355
3483
|
const childLimit = resolveChildLimit(el, maxChildren, hashId);
|
|
3356
3484
|
const selectedChildren = orderedChildren.slice(0, childLimit);
|
|
3357
3485
|
const omittedChildren = orderedChildren.length - selectedChildren.length;
|
|
@@ -3362,25 +3490,19 @@ function generateSnapshot(root = document.body, options = {}) {
|
|
|
3362
3490
|
}
|
|
3363
3491
|
if (childBlocks.length === 0 && omittedChildren <= 0) return "";
|
|
3364
3492
|
if (!(childBlocks.length >= 2 || omittedChildren > 0)) return childBlocks.join("\n");
|
|
3365
|
-
const groupLines = [`${" ".repeat(depth)}([${
|
|
3493
|
+
const groupLines = [`${" ".repeat(depth)}([${displayTag}] collapsed-group`];
|
|
3366
3494
|
for (const block of childBlocks) groupLines.push(indentMultiline(block, 1));
|
|
3367
3495
|
if (omittedChildren > 0) groupLines.push(`${" ".repeat(depth + 1)}... (${omittedChildren} children omitted)`);
|
|
3368
3496
|
groupLines.push(`${" ".repeat(depth)})`);
|
|
3369
3497
|
return groupLines.join("\n");
|
|
3370
3498
|
}
|
|
3371
|
-
let line = `${indent}[${
|
|
3499
|
+
let line = `${indent}[${displayTag}]`;
|
|
3372
3500
|
if (directText) line += ` "${directText.slice(0, maxTextLength)}"`;
|
|
3373
3501
|
if (attrs.length) line += ` ${attrs.join(" ")}`;
|
|
3374
|
-
if (hashId) {
|
|
3375
|
-
line += ` #${hashId}`;
|
|
3376
|
-
emittedRefIds.add(hashId);
|
|
3377
|
-
} else line += ` ref="${currentPath}"`;
|
|
3502
|
+
if (hashId) line += ` #${hashId}`;
|
|
3378
3503
|
const lines = [line];
|
|
3379
3504
|
emittedNodes++;
|
|
3380
|
-
const
|
|
3381
|
-
const interactiveChildren = allChildren.filter(isInteractiveElement);
|
|
3382
|
-
const nonInteractiveChildren = allChildren.filter((child) => !isInteractiveElement(child));
|
|
3383
|
-
const orderedChildren = [...interactiveChildren, ...nonInteractiveChildren];
|
|
3505
|
+
const orderedChildren = orderChildrenByPriority(Array.from(el.children));
|
|
3384
3506
|
const childLimit = resolveChildLimit(el, maxChildren, hashId);
|
|
3385
3507
|
const selectedChildren = orderedChildren.slice(0, childLimit);
|
|
3386
3508
|
const omittedChildren = orderedChildren.length - selectedChildren.length;
|
|
@@ -3392,7 +3514,6 @@ function generateSnapshot(root = document.body, options = {}) {
|
|
|
3392
3514
|
return lines.join("\n");
|
|
3393
3515
|
}
|
|
3394
3516
|
const output = walk(root, 0, "") || "(空页面)";
|
|
3395
|
-
refStore?.prune(emittedRefIds);
|
|
3396
3517
|
if (!truncatedByNodeBudget) return output;
|
|
3397
3518
|
return `${output}\n... (snapshot truncated: maxNodes=${maxNodes})`;
|
|
3398
3519
|
}
|
|
@@ -3437,11 +3558,11 @@ function createPageInfoTool() {
|
|
|
3437
3558
|
schema: Type.Object({
|
|
3438
3559
|
action: Type.String({ description: "Info action: get_url | get_title | get_selection | get_viewport | snapshot | query_all" }),
|
|
3439
3560
|
selector: Type.Optional(Type.String({ description: "CSS selector for query_all action" })),
|
|
3440
|
-
maxDepth: Type.Optional(Type.Number({ description: "Max depth for snapshot (default:
|
|
3561
|
+
maxDepth: Type.Optional(Type.Number({ description: "Max depth for snapshot (default: 12)" })),
|
|
3441
3562
|
viewportOnly: Type.Optional(Type.Boolean({ description: "Only snapshot elements visible in viewport (default: true)" })),
|
|
3442
3563
|
pruneLayout: Type.Optional(Type.Boolean({ description: "Collapse empty layout containers like div/span (default: true)" })),
|
|
3443
|
-
maxNodes: Type.Optional(Type.Number({ description: "Maximum nodes to include in snapshot (default:
|
|
3444
|
-
maxChildren: Type.Optional(Type.Number({ description: "Maximum children per element (default:
|
|
3564
|
+
maxNodes: Type.Optional(Type.Number({ description: "Maximum nodes to include in snapshot (default: 220)" })),
|
|
3565
|
+
maxChildren: Type.Optional(Type.Number({ description: "Maximum children per element (default: 25)" })),
|
|
3445
3566
|
maxTextLength: Type.Optional(Type.Number({ description: "Maximum text length per node (default: 40)" })),
|
|
3446
3567
|
expandOptionLists: Type.Optional(Type.Boolean({ description: "Expand option-list containers to avoid child truncation (default: false)" })),
|
|
3447
3568
|
expandChildrenRefs: Type.Optional(Type.Array(Type.String({ description: "Hash refs to expand child truncation for (e.g. #abc123)" }))),
|
|
@@ -3466,11 +3587,11 @@ function createPageInfoTool() {
|
|
|
3466
3587
|
return { content: JSON.stringify(info, null, 2) };
|
|
3467
3588
|
}
|
|
3468
3589
|
case "snapshot": {
|
|
3469
|
-
const maxDepth = params.maxDepth ??
|
|
3590
|
+
const maxDepth = params.maxDepth ?? 12;
|
|
3470
3591
|
const viewportOnly = params.viewportOnly ?? true;
|
|
3471
3592
|
const pruneLayout = params.pruneLayout ?? true;
|
|
3472
|
-
const maxNodes = params.maxNodes ??
|
|
3473
|
-
const maxChildren = params.maxChildren ??
|
|
3593
|
+
const maxNodes = params.maxNodes ?? 220;
|
|
3594
|
+
const maxChildren = params.maxChildren ?? 25;
|
|
3474
3595
|
const maxTextLength = params.maxTextLength ?? 40;
|
|
3475
3596
|
const expandOptionLists = params.expandOptionLists ?? false;
|
|
3476
3597
|
const expandChildrenRefs = Array.isArray(params.expandChildrenRefs) ? params.expandChildrenRefs.filter((ref) => typeof ref === "string") : void 0;
|
|
@@ -3674,14 +3795,7 @@ function resolveSelector(selector) {
|
|
|
3674
3795
|
const store = getActiveRefStore();
|
|
3675
3796
|
if (store) {
|
|
3676
3797
|
const id = selector.slice(1);
|
|
3677
|
-
if (store.has(id))
|
|
3678
|
-
const el = store.get(id);
|
|
3679
|
-
if (!el || !el.isConnected) {
|
|
3680
|
-
store.delete(id);
|
|
3681
|
-
return null;
|
|
3682
|
-
}
|
|
3683
|
-
return el;
|
|
3684
|
-
}
|
|
3798
|
+
if (store.has(id)) return store.get(id) ?? null;
|
|
3685
3799
|
}
|
|
3686
3800
|
}
|
|
3687
3801
|
try {
|
|
@@ -4025,29 +4139,6 @@ var RefStore = class {
|
|
|
4025
4139
|
has(id) {
|
|
4026
4140
|
return this.map.has(id);
|
|
4027
4141
|
}
|
|
4028
|
-
/** 删除指定 hash ID 映射,返回是否删除成功。 */
|
|
4029
|
-
delete(id) {
|
|
4030
|
-
return this.map.delete(id);
|
|
4031
|
-
}
|
|
4032
|
-
/**
|
|
4033
|
-
* 清理失效引用:
|
|
4034
|
-
* - 仅保留 keepIds 中的映射(若提供)
|
|
4035
|
-
* - 自动移除已脱离文档(isConnected=false)的元素
|
|
4036
|
-
*
|
|
4037
|
-
* @returns 被移除的映射数量
|
|
4038
|
-
*/
|
|
4039
|
-
prune(keepIds) {
|
|
4040
|
-
let removed = 0;
|
|
4041
|
-
for (const [id, el] of this.map.entries()) {
|
|
4042
|
-
const shouldKeepById = keepIds ? keepIds.has(id) : true;
|
|
4043
|
-
const isConnected = el.isConnected;
|
|
4044
|
-
if (!shouldKeepById || !isConnected) {
|
|
4045
|
-
this.map.delete(id);
|
|
4046
|
-
removed++;
|
|
4047
|
-
}
|
|
4048
|
-
}
|
|
4049
|
-
return removed;
|
|
4050
|
-
}
|
|
4051
4142
|
/** 清空所有映射 */
|
|
4052
4143
|
clear() {
|
|
4053
4144
|
this.map.clear();
|
|
@@ -4395,7 +4486,7 @@ var WebAgent = class WebAgent {
|
|
|
4395
4486
|
let initialSnapshot;
|
|
4396
4487
|
try {
|
|
4397
4488
|
const snapshot = generateSnapshot(document.body, {
|
|
4398
|
-
maxDepth:
|
|
4489
|
+
maxDepth: 12,
|
|
4399
4490
|
viewportOnly: false,
|
|
4400
4491
|
maxNodes: 500,
|
|
4401
4492
|
maxChildren: 30,
|