agentpage 0.0.33 → 0.0.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +87 -27
- package/dist/index.d.mts +12 -2
- package/dist/index.d.mts.map +1 -1
- package/dist/index.mjs +627 -101
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/index.mjs
CHANGED
|
@@ -10,7 +10,25 @@ const DEFAULT_MAX_ROUNDS = 40;
|
|
|
10
10
|
const DEFAULT_RECOVERY_WAIT_MS = 100;
|
|
11
11
|
const DEFAULT_ACTION_RECOVERY_ROUNDS = 2;
|
|
12
12
|
const DEFAULT_NOT_FOUND_RETRY_ROUNDS = 2;
|
|
13
|
-
const DEFAULT_NOT_FOUND_RETRY_WAIT_MS =
|
|
13
|
+
const DEFAULT_NOT_FOUND_RETRY_WAIT_MS = 1e3;
|
|
14
|
+
const DEFAULT_ROUND_STABILITY_WAIT_TIMEOUT_MS = 4e3;
|
|
15
|
+
const DEFAULT_ROUND_STABILITY_WAIT_QUIET_MS = 200;
|
|
16
|
+
const DEFAULT_ROUND_STABILITY_WAIT_LOADING_SELECTORS = [
|
|
17
|
+
".ant-spin",
|
|
18
|
+
".ant-spin-spinning",
|
|
19
|
+
".ant-skeleton",
|
|
20
|
+
".el-loading-mask",
|
|
21
|
+
".bk-loading",
|
|
22
|
+
".bk-spin-loading",
|
|
23
|
+
".bk-skeleton",
|
|
24
|
+
".bk-sideslider-loading",
|
|
25
|
+
".t-loading",
|
|
26
|
+
".t-skeleton",
|
|
27
|
+
".t-skeleton__row",
|
|
28
|
+
"[aria-busy=\"true\"]",
|
|
29
|
+
".skeleton",
|
|
30
|
+
".loading"
|
|
31
|
+
];
|
|
14
32
|
/** 快照起始标记 — 用于在消息中识别快照边界 */
|
|
15
33
|
const SNAPSHOT_START = "<!-- SNAPSHOT_START -->";
|
|
16
34
|
/** 快照结束标记 */
|
|
@@ -163,6 +181,36 @@ function shouldForceRoundBreak(toolName, toolInput) {
|
|
|
163
181
|
return toolName === "evaluate";
|
|
164
182
|
}
|
|
165
183
|
/**
|
|
184
|
+
* 判定动作是否可能引发页面结构或状态变化。
|
|
185
|
+
*
|
|
186
|
+
* 用于“轮次后稳定等待”触发条件:
|
|
187
|
+
* - 命中 true:本轮结束后执行加载态 + DOM 静默双重等待
|
|
188
|
+
* - 命中 false:跳过等待,直接进入下一轮
|
|
189
|
+
*/
|
|
190
|
+
function isPotentialDomMutation(toolName, toolInput) {
|
|
191
|
+
const action = getToolAction(toolInput);
|
|
192
|
+
if (toolName === "navigate") return true;
|
|
193
|
+
if (toolName === "evaluate") return true;
|
|
194
|
+
if (toolName !== "dom") return false;
|
|
195
|
+
if (!action) return false;
|
|
196
|
+
return [
|
|
197
|
+
"click",
|
|
198
|
+
"fill",
|
|
199
|
+
"select_option",
|
|
200
|
+
"clear",
|
|
201
|
+
"check",
|
|
202
|
+
"uncheck",
|
|
203
|
+
"type",
|
|
204
|
+
"focus",
|
|
205
|
+
"hover",
|
|
206
|
+
"scroll",
|
|
207
|
+
"press",
|
|
208
|
+
"set_attr",
|
|
209
|
+
"add_class",
|
|
210
|
+
"remove_class"
|
|
211
|
+
].includes(action);
|
|
212
|
+
}
|
|
213
|
+
/**
|
|
166
214
|
* 采集找不到元素任务。
|
|
167
215
|
*
|
|
168
216
|
* 返回 null 表示当前结果不属于“元素未找到”,
|
|
@@ -247,7 +295,7 @@ function hasToolError(result) {
|
|
|
247
295
|
* 4) 返回稳定字符串给 loop,供后续注入消息与统计。
|
|
248
296
|
*
|
|
249
297
|
* 默认参数意图:
|
|
250
|
-
* - `maxDepth=
|
|
298
|
+
* - `maxDepth=12`: 保留更深层级,减少深层组件控件被截断。
|
|
251
299
|
* - `viewportOnly=false`: 优先完整性,避免误判“元素不存在”。
|
|
252
300
|
* - `pruneLayout=true`: 抑制纯布局噪声,降低 token 压力。
|
|
253
301
|
* - `maxNodes=500` / `maxChildren=30`: 控制体积上限,兼顾可读性。
|
|
@@ -272,7 +320,7 @@ function hasToolError(result) {
|
|
|
272
320
|
async function readPageSnapshot(registry, options) {
|
|
273
321
|
return toContentString((await registry.dispatch("page_info", {
|
|
274
322
|
action: "snapshot",
|
|
275
|
-
maxDepth: options?.maxDepth ??
|
|
323
|
+
maxDepth: options?.maxDepth ?? 12,
|
|
276
324
|
viewportOnly: options?.viewportOnly ?? false,
|
|
277
325
|
pruneLayout: options?.pruneLayout ?? true,
|
|
278
326
|
maxNodes: options?.maxNodes ?? 500,
|
|
@@ -696,7 +744,7 @@ function detectIdleLoop(toolCalls, consecutiveReadOnlyRounds) {
|
|
|
696
744
|
* - 达到 `maxRounds`
|
|
697
745
|
*/
|
|
698
746
|
async function executeAgentLoop(params) {
|
|
699
|
-
const { client, registry, systemPrompt, message, initialSnapshot, history, dryRun = false, maxRounds = DEFAULT_MAX_ROUNDS, callbacks } = params;
|
|
747
|
+
const { client, registry, systemPrompt, message, initialSnapshot, history, dryRun = false, maxRounds = DEFAULT_MAX_ROUNDS, roundStabilityWait, callbacks } = params;
|
|
700
748
|
const tools = registry.getDefinitions();
|
|
701
749
|
const allToolCalls = [];
|
|
702
750
|
const fullToolTrace = [];
|
|
@@ -717,6 +765,12 @@ async function executeAgentLoop(params) {
|
|
|
717
765
|
let lastRoundHadError = false;
|
|
718
766
|
let protocolViolationHint;
|
|
719
767
|
const snapshotExpandRefIds = /* @__PURE__ */ new Set();
|
|
768
|
+
const effectiveRoundStabilityWait = {
|
|
769
|
+
enabled: roundStabilityWait?.enabled ?? true,
|
|
770
|
+
timeoutMs: Math.max(200, Math.floor(roundStabilityWait?.timeoutMs ?? DEFAULT_ROUND_STABILITY_WAIT_TIMEOUT_MS)),
|
|
771
|
+
quietMs: Math.max(50, Math.floor(roundStabilityWait?.quietMs ?? DEFAULT_ROUND_STABILITY_WAIT_QUIET_MS)),
|
|
772
|
+
loadingSelectors: [...new Set([...DEFAULT_ROUND_STABILITY_WAIT_LOADING_SELECTORS, ...roundStabilityWait?.loadingSelectors ?? []].map((selector) => selector.trim()).filter(Boolean))]
|
|
773
|
+
};
|
|
720
774
|
let recoveryCount = 0;
|
|
721
775
|
let redundantInterceptCount = 0;
|
|
722
776
|
let pendingNotFoundRetry;
|
|
@@ -748,6 +802,30 @@ async function executeAgentLoop(params) {
|
|
|
748
802
|
} : void 0);
|
|
749
803
|
recordSnapshotStats(pageContext.latestSnapshot);
|
|
750
804
|
};
|
|
805
|
+
/**
|
|
806
|
+
* 轮次后稳定等待(双重等待)。
|
|
807
|
+
*
|
|
808
|
+
* 顺序固定为:
|
|
809
|
+
* 1) 等待 loading 指示器隐藏
|
|
810
|
+
* 2) 等待 DOM quiet window
|
|
811
|
+
*/
|
|
812
|
+
const runRoundStabilityBarrier = async () => {
|
|
813
|
+
if (!effectiveRoundStabilityWait.enabled) return;
|
|
814
|
+
if (!registry.has("wait")) return;
|
|
815
|
+
const timeout = effectiveRoundStabilityWait.timeoutMs;
|
|
816
|
+
const loadingSelector = effectiveRoundStabilityWait.loadingSelectors.join(", ");
|
|
817
|
+
if (loadingSelector) await registry.dispatch("wait", {
|
|
818
|
+
action: "wait_for_selector",
|
|
819
|
+
selector: loadingSelector,
|
|
820
|
+
state: "hidden",
|
|
821
|
+
timeout
|
|
822
|
+
});
|
|
823
|
+
await registry.dispatch("wait", {
|
|
824
|
+
action: "wait_for_stable",
|
|
825
|
+
timeout,
|
|
826
|
+
quietMs: effectiveRoundStabilityWait.quietMs
|
|
827
|
+
});
|
|
828
|
+
};
|
|
751
829
|
if (pageContext.latestSnapshot) recordSnapshotStats(pageContext.latestSnapshot);
|
|
752
830
|
/**
|
|
753
831
|
* 追加工具轨迹。
|
|
@@ -863,6 +941,7 @@ async function executeAgentLoop(params) {
|
|
|
863
941
|
break;
|
|
864
942
|
}
|
|
865
943
|
let roundHasError = false;
|
|
944
|
+
let roundHasPotentialDomMutation = false;
|
|
866
945
|
const executedTaskCalls = [];
|
|
867
946
|
const roundMissingTasks = [];
|
|
868
947
|
for (const tc of response.toolCalls) {
|
|
@@ -893,6 +972,7 @@ async function executeAgentLoop(params) {
|
|
|
893
972
|
const missingTask = collectMissingTask(tc.name, tc.input, result);
|
|
894
973
|
if (missingTask) roundMissingTasks.push(missingTask);
|
|
895
974
|
if (result.details && typeof result.details === "object") roundHasError = roundHasError || Boolean(result.details.error);
|
|
975
|
+
if (!hasToolError(result) && isPotentialDomMutation(tc.name, tc.input)) roundHasPotentialDomMutation = true;
|
|
896
976
|
if (tc.name === "page_info" && getToolAction(tc.input) === "snapshot") {
|
|
897
977
|
pageContext.latestSnapshot = toContentString(result.content);
|
|
898
978
|
recordSnapshotStats(pageContext.latestSnapshot);
|
|
@@ -916,6 +996,11 @@ async function executeAgentLoop(params) {
|
|
|
916
996
|
lastRoundHadError = roundHasError;
|
|
917
997
|
previousRoundTasks = buildTaskArray(executedTaskCalls);
|
|
918
998
|
previousRoundPlannedTasks = plannedTasksCurrentRound;
|
|
999
|
+
if (parsedInstructionState.hasRemainingProtocol && remainingInstruction.trim().length === 0 && !roundHasError) {
|
|
1000
|
+
finalReply = response.text?.trim() || "任务已完成。";
|
|
1001
|
+
if (finalReply) callbacks?.onText?.(finalReply);
|
|
1002
|
+
break;
|
|
1003
|
+
}
|
|
919
1004
|
const idleResult = detectIdleLoop(response.toolCalls.map((tc) => ({
|
|
920
1005
|
name: tc.name,
|
|
921
1006
|
input: tc.input
|
|
@@ -926,6 +1011,7 @@ async function executeAgentLoop(params) {
|
|
|
926
1011
|
break;
|
|
927
1012
|
}
|
|
928
1013
|
consecutiveReadOnlyRounds = idleResult;
|
|
1014
|
+
if (roundHasPotentialDomMutation) await runRoundStabilityBarrier();
|
|
929
1015
|
await refreshSnapshot();
|
|
930
1016
|
}
|
|
931
1017
|
const resultMessages = [...history ?? [], {
|
|
@@ -1575,7 +1661,13 @@ var ToolRegistry = class {
|
|
|
1575
1661
|
|
|
1576
1662
|
//#endregion
|
|
1577
1663
|
//#region src/core/system-prompt.ts
|
|
1578
|
-
/**
|
|
1664
|
+
/**
|
|
1665
|
+
* 规范化额外指令:统一转为非空字符串数组。
|
|
1666
|
+
*
|
|
1667
|
+
* - 单字符串 → 单元素数组
|
|
1668
|
+
* - 字符串数组 → 过滤空值
|
|
1669
|
+
* - undefined → 空数组
|
|
1670
|
+
*/
|
|
1579
1671
|
function normalizeExtraInstructions(input) {
|
|
1580
1672
|
if (!input) return [];
|
|
1581
1673
|
return (Array.isArray(input) ? input : [input]).map((s) => s.trim()).filter(Boolean);
|
|
@@ -1583,9 +1675,32 @@ function normalizeExtraInstructions(input) {
|
|
|
1583
1675
|
/**
|
|
1584
1676
|
* 构建系统提示词。
|
|
1585
1677
|
*
|
|
1586
|
-
*
|
|
1587
|
-
*
|
|
1588
|
-
*
|
|
1678
|
+
* 输出结构(按章节顺序):
|
|
1679
|
+
* 1. **Core Rules** — Agent 核心行为规则
|
|
1680
|
+
* - 快照驱动决策:仅基于当前快照 + 剩余任务工作
|
|
1681
|
+
* - 增量消费模型:每轮执行后输出 REMAINING 推进任务
|
|
1682
|
+
* - hash ID 定位:仅交互元素携带 #hashID,非交互元素为上下文
|
|
1683
|
+
* - 事件信号:listeners="..." 标注运行时事件绑定
|
|
1684
|
+
* - 批量执行:同轮完成所有独立可见操作
|
|
1685
|
+
* - 输入顺序:fill/type 前必须先 focus/click 同一目标
|
|
1686
|
+
* - DOM 变化断轮:会改变 DOM 的动作执行后等待下一轮新快照
|
|
1687
|
+
* - 停机规则:任务完成后输出 REMAINING: DONE
|
|
1688
|
+
*
|
|
1689
|
+
* 2. **Listener Abbrevs** — 事件简写对照表
|
|
1690
|
+
* - 快照中 listeners="clk,inp,chg" 的简写含义
|
|
1691
|
+
* - 与 page-info-tool.ts 的 EVENT_ABBREV 映射一致
|
|
1692
|
+
*
|
|
1693
|
+
* 3. **Output Contract** — 输出协议
|
|
1694
|
+
* - 每轮返回工具调用 + REMAINING 文本行
|
|
1695
|
+
*
|
|
1696
|
+
* 4. **Available Tools**(可选) — 当前注册的工具及描述
|
|
1697
|
+
*
|
|
1698
|
+
* 5. **Reasoning Profile**(可选) — 思考深度配置
|
|
1699
|
+
*
|
|
1700
|
+
* 6. **Extra Instructions**(可选) — 用户自定义额外指令
|
|
1701
|
+
*
|
|
1702
|
+
* @param params - 构建参数(工具列表、思考深度、额外指令)
|
|
1703
|
+
* @returns 完整的系统提示词字符串(英文)
|
|
1589
1704
|
*/
|
|
1590
1705
|
function buildSystemPrompt(params = {}) {
|
|
1591
1706
|
const sections = [];
|
|
@@ -1598,6 +1713,9 @@ function buildSystemPrompt(params = {}) {
|
|
|
1598
1713
|
" Input: (1) current remaining task, (2) previous round executed actions, (3) actions you execute this round.",
|
|
1599
1714
|
" Output: new remaining task after removing this-round actions.",
|
|
1600
1715
|
"- Use only visible targets from snapshot. Use #hashID as selector. Do not guess CSS selectors.",
|
|
1716
|
+
"- Only interactive elements (with events, inputs, buttons, links, etc.) carry #hashID. Elements without #hashID are context-only (labels, headings, text) and cannot be targeted.",
|
|
1717
|
+
"- Snapshot tag in brackets may show ARIA role instead of HTML tag when it better describes the interaction pattern (e.g. [combobox] for input with role=\"combobox\", [slider] for div with role=\"slider\"). Treat the bracket tag as the primary interaction hint.",
|
|
1718
|
+
"- listeners=\"...\" on snapshot indicates bound event handlers (see Listener Abbrevs below). Prefer targets with relevant listeners when multiple candidates look similar.",
|
|
1601
1719
|
"- Batch independent visible actions in one round. Do not split one form into many rounds unnecessarily.",
|
|
1602
1720
|
"- Strict input order (MANDATORY): before every fill/type/select_option, click or focus the SAME target immediately in the SAME round.",
|
|
1603
1721
|
"- Multi-field rule (MANDATORY): execute alternating pairs in one batch: focus/click field A -> fill/type A -> focus/click field B -> fill/type B.",
|
|
@@ -1616,6 +1734,9 @@ function buildSystemPrompt(params = {}) {
|
|
|
1616
1734
|
"- Stop rule: when the requested state is achieved, stop calling tools. If verification is requested, verify once and then return REMAINING: DONE (no repeated get_text/get_attr on the same target).",
|
|
1617
1735
|
"- Do NOT interact with AutoPilot UI unless user explicitly asks.",
|
|
1618
1736
|
"",
|
|
1737
|
+
"## Listener Abbrevs",
|
|
1738
|
+
"clk=click dbl=dblclick mdn=mousedown mup=mouseup mmv=mousemove mov=mouseover mot=mouseout men=mouseenter mlv=mouseleave pdn=pointerdown pup=pointerup pmv=pointermove tst=touchstart ted=touchend kdn=keydown kup=keyup inp=input chg=change sub=submit fcs=focus blr=blur scl=scroll whl=wheel drg=drag drs=dragstart dre=dragend drp=drop ctx=contextmenu",
|
|
1739
|
+
"",
|
|
1619
1740
|
"## Output Contract",
|
|
1620
1741
|
"- Return tool calls for this round.",
|
|
1621
1742
|
"- Also include one plain text line:",
|
|
@@ -1642,6 +1763,81 @@ function buildSystemPrompt(params = {}) {
|
|
|
1642
1763
|
return sections.join("\n\n");
|
|
1643
1764
|
}
|
|
1644
1765
|
|
|
1766
|
+
//#endregion
|
|
1767
|
+
//#region src/web/event-listener-tracker.ts
|
|
1768
|
+
const elementEventMap = /* @__PURE__ */ new WeakMap();
|
|
1769
|
+
let installed = false;
|
|
1770
|
+
let originalAddEventListener;
|
|
1771
|
+
let originalRemoveEventListener;
|
|
1772
|
+
function normalizeEventType(type) {
|
|
1773
|
+
if (typeof type !== "string") return null;
|
|
1774
|
+
return type.trim().toLowerCase() || null;
|
|
1775
|
+
}
|
|
1776
|
+
function canTrackElementTarget(target) {
|
|
1777
|
+
if (typeof Element === "undefined") return false;
|
|
1778
|
+
return target instanceof Element;
|
|
1779
|
+
}
|
|
1780
|
+
function trackElementEvent(target, type) {
|
|
1781
|
+
if (!canTrackElementTarget(target)) return;
|
|
1782
|
+
const prev = elementEventMap.get(target);
|
|
1783
|
+
if (prev) {
|
|
1784
|
+
prev.add(type);
|
|
1785
|
+
return;
|
|
1786
|
+
}
|
|
1787
|
+
elementEventMap.set(target, new Set([type]));
|
|
1788
|
+
}
|
|
1789
|
+
function untrackElementEvent(target, type) {
|
|
1790
|
+
if (!canTrackElementTarget(target)) return;
|
|
1791
|
+
const prev = elementEventMap.get(target);
|
|
1792
|
+
if (!prev) return;
|
|
1793
|
+
prev.delete(type);
|
|
1794
|
+
if (prev.size === 0) elementEventMap.delete(target);
|
|
1795
|
+
}
|
|
1796
|
+
/**
|
|
1797
|
+
* 安装全局监听追踪补丁(幂等)。
|
|
1798
|
+
*/
|
|
1799
|
+
function installEventListenerTracking() {
|
|
1800
|
+
if (installed) return;
|
|
1801
|
+
if (typeof EventTarget === "undefined") return;
|
|
1802
|
+
const proto = EventTarget.prototype;
|
|
1803
|
+
const nativeAdd = proto.addEventListener;
|
|
1804
|
+
const nativeRemove = proto.removeEventListener;
|
|
1805
|
+
if (typeof nativeAdd !== "function" || typeof nativeRemove !== "function") return;
|
|
1806
|
+
originalAddEventListener = nativeAdd;
|
|
1807
|
+
originalRemoveEventListener = nativeRemove;
|
|
1808
|
+
proto.addEventListener = function patchedAddEventListener(type, listener, options) {
|
|
1809
|
+
originalAddEventListener?.call(this, type, listener, options);
|
|
1810
|
+
try {
|
|
1811
|
+
const normalizedType = normalizeEventType(type);
|
|
1812
|
+
if (!normalizedType || listener == null) return;
|
|
1813
|
+
trackElementEvent(this, normalizedType);
|
|
1814
|
+
} catch {}
|
|
1815
|
+
};
|
|
1816
|
+
proto.removeEventListener = function patchedRemoveEventListener(type, listener, options) {
|
|
1817
|
+
originalRemoveEventListener?.call(this, type, listener, options);
|
|
1818
|
+
try {
|
|
1819
|
+
const normalizedType = normalizeEventType(type);
|
|
1820
|
+
if (!normalizedType || listener == null) return;
|
|
1821
|
+
untrackElementEvent(this, normalizedType);
|
|
1822
|
+
} catch {}
|
|
1823
|
+
};
|
|
1824
|
+
installed = true;
|
|
1825
|
+
}
|
|
1826
|
+
/**
|
|
1827
|
+
* 读取元素已记录的事件名(排序后返回,便于稳定输出)。
|
|
1828
|
+
*/
|
|
1829
|
+
function getTrackedElementEvents(el) {
|
|
1830
|
+
const set = elementEventMap.get(el);
|
|
1831
|
+
if (!set || set.size === 0) return [];
|
|
1832
|
+
return Array.from(set).sort();
|
|
1833
|
+
}
|
|
1834
|
+
/**
|
|
1835
|
+
* 判断元素是否存在至少一个被追踪到的事件绑定。
|
|
1836
|
+
*/
|
|
1837
|
+
function hasTrackedElementEvents(el) {
|
|
1838
|
+
return (elementEventMap.get(el)?.size ?? 0) > 0;
|
|
1839
|
+
}
|
|
1840
|
+
|
|
1645
1841
|
//#endregion
|
|
1646
1842
|
//#region src/web/tools/dom-tool.ts
|
|
1647
1843
|
/**
|
|
@@ -1661,7 +1857,7 @@ function buildSystemPrompt(params = {}) {
|
|
|
1661
1857
|
*
|
|
1662
1858
|
* 运行环境:浏览器 Content Script(直接访问 DOM,无 CDP)。
|
|
1663
1859
|
*/
|
|
1664
|
-
const DEFAULT_WAIT_MS =
|
|
1860
|
+
const DEFAULT_WAIT_MS = 1200;
|
|
1665
1861
|
/** scrollIntoView 轮换策略(参考 Playwright dom.ts) */
|
|
1666
1862
|
const SCROLL_OPTIONS = [
|
|
1667
1863
|
void 0,
|
|
@@ -1721,6 +1917,16 @@ const KEY_CODE_MAP = {
|
|
|
1721
1917
|
Alt: "AltLeft",
|
|
1722
1918
|
Meta: "MetaLeft"
|
|
1723
1919
|
};
|
|
1920
|
+
const FILL_RELEVANT_EVENTS = new Set([
|
|
1921
|
+
"input",
|
|
1922
|
+
"change",
|
|
1923
|
+
"focus",
|
|
1924
|
+
"blur",
|
|
1925
|
+
"keydown",
|
|
1926
|
+
"click",
|
|
1927
|
+
"mousedown",
|
|
1928
|
+
"pointerdown"
|
|
1929
|
+
]);
|
|
1724
1930
|
let activeRefStore;
|
|
1725
1931
|
function setActiveRefStore(store) {
|
|
1726
1932
|
activeRefStore = store;
|
|
@@ -1929,15 +2135,34 @@ function ensureActionable(el, action, selector, force) {
|
|
|
1929
2135
|
"fill",
|
|
1930
2136
|
"type",
|
|
1931
2137
|
"clear"
|
|
1932
|
-
].includes(action) && !isEditableElement(el))
|
|
1933
|
-
|
|
1934
|
-
|
|
1935
|
-
|
|
1936
|
-
|
|
1937
|
-
|
|
1938
|
-
|
|
1939
|
-
|
|
1940
|
-
|
|
2138
|
+
].includes(action) && !isEditableElement(el)) {
|
|
2139
|
+
if (action === "fill" && el.getAttribute("role") === "slider") return null;
|
|
2140
|
+
return {
|
|
2141
|
+
content: `"${selector}" 不是可编辑元素,无法执行 ${action}`,
|
|
2142
|
+
details: {
|
|
2143
|
+
error: true,
|
|
2144
|
+
code: "UNSUPPORTED_FILL_TARGET",
|
|
2145
|
+
action,
|
|
2146
|
+
selector
|
|
2147
|
+
}
|
|
2148
|
+
};
|
|
2149
|
+
}
|
|
2150
|
+
return null;
|
|
2151
|
+
}
|
|
2152
|
+
/**
|
|
2153
|
+
* 为 role=slider 查找关联的数值输入框。
|
|
2154
|
+
* 典型场景:Element Plus slider + input-number 同属一个 form-item。
|
|
2155
|
+
*/
|
|
2156
|
+
function findAssociatedSliderInput(slider) {
|
|
2157
|
+
const candidates = [];
|
|
2158
|
+
const formItem = slider.closest(".el-form-item");
|
|
2159
|
+
if (formItem) candidates.push(formItem);
|
|
2160
|
+
let cursor = slider.parentElement;
|
|
2161
|
+
for (let depth = 0; cursor && depth < 4; depth++, cursor = cursor.parentElement) candidates.push(cursor);
|
|
2162
|
+
for (const scope of candidates) {
|
|
2163
|
+
const input = scope.querySelector("input[type=\"number\"], input[role=\"spinbutton\"], .el-input-number input:not([type=\"hidden\"])");
|
|
2164
|
+
if (input instanceof HTMLInputElement && isEditableElement(input) && isElementVisible(input)) return input;
|
|
2165
|
+
}
|
|
1941
2166
|
return null;
|
|
1942
2167
|
}
|
|
1943
2168
|
function getClickPoint(el) {
|
|
@@ -2034,6 +2259,163 @@ function setNativeValue(el, value) {
|
|
|
2034
2259
|
if (desc?.set) desc.set.call(el, value);
|
|
2035
2260
|
else el.value = value;
|
|
2036
2261
|
}
|
|
2262
|
+
function getFillEventSupportScore(el) {
|
|
2263
|
+
let score = 0;
|
|
2264
|
+
if (el.hasAttribute("oninput") || el.hasAttribute("onchange")) score += 80;
|
|
2265
|
+
if (el.hasAttribute("onfocus") || el.hasAttribute("onblur")) score += 60;
|
|
2266
|
+
if (el.hasAttribute("onclick")) score += 40;
|
|
2267
|
+
const tracked = getTrackedElementEvents(el);
|
|
2268
|
+
for (const eventName of tracked) {
|
|
2269
|
+
if (!FILL_RELEVANT_EVENTS.has(eventName)) continue;
|
|
2270
|
+
if (eventName === "input") score += 40;
|
|
2271
|
+
else if (eventName === "change") score += 35;
|
|
2272
|
+
else if (eventName === "focus" || eventName === "blur") score += 28;
|
|
2273
|
+
else if (eventName === "keydown") score += 24;
|
|
2274
|
+
else score += 14;
|
|
2275
|
+
}
|
|
2276
|
+
return score;
|
|
2277
|
+
}
|
|
2278
|
+
function isCandidateFillTarget(el) {
|
|
2279
|
+
if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement || el instanceof HTMLSelectElement) return !isElementDisabled(el);
|
|
2280
|
+
if (el instanceof HTMLElement && el.isContentEditable) return true;
|
|
2281
|
+
return false;
|
|
2282
|
+
}
|
|
2283
|
+
function executeFillOnResolvedTarget(target, value, selector, action, sourceHint) {
|
|
2284
|
+
if (target instanceof HTMLInputElement) {
|
|
2285
|
+
const type = target.type.toLowerCase();
|
|
2286
|
+
if (INPUT_BLOCKED_TYPES.has(type)) return {
|
|
2287
|
+
content: `"${selector}" 为 input[type=${type}],不支持 fill;请使用 click/check 等动作。`,
|
|
2288
|
+
details: {
|
|
2289
|
+
error: true,
|
|
2290
|
+
code: "UNSUPPORTED_FILL_TARGET",
|
|
2291
|
+
action,
|
|
2292
|
+
selector
|
|
2293
|
+
}
|
|
2294
|
+
};
|
|
2295
|
+
if (INPUT_SET_VALUE_TYPES.has(type)) {
|
|
2296
|
+
const finalVal = type === "color" ? value.toLowerCase().trim() : value.trim();
|
|
2297
|
+
target.focus();
|
|
2298
|
+
target.value = finalVal;
|
|
2299
|
+
if (target.value !== finalVal) return {
|
|
2300
|
+
content: `"${selector}" 填写格式不匹配(type=${type})`,
|
|
2301
|
+
details: {
|
|
2302
|
+
error: true,
|
|
2303
|
+
code: "MALFORMED_VALUE",
|
|
2304
|
+
action,
|
|
2305
|
+
selector
|
|
2306
|
+
}
|
|
2307
|
+
};
|
|
2308
|
+
dispatchInputEvents(target);
|
|
2309
|
+
const suffix = sourceHint ? `(${sourceHint})` : "";
|
|
2310
|
+
return { content: `已填写 ${describeElement(target)}: "${finalVal}"${suffix}` };
|
|
2311
|
+
}
|
|
2312
|
+
if (type === "number" && Number.isNaN(Number(value.trim()))) return {
|
|
2313
|
+
content: `"${selector}" 为 input[type=number],无法填写非数字 "${value}"`,
|
|
2314
|
+
details: {
|
|
2315
|
+
error: true,
|
|
2316
|
+
code: "INVALID_NUMBER",
|
|
2317
|
+
action,
|
|
2318
|
+
selector
|
|
2319
|
+
}
|
|
2320
|
+
};
|
|
2321
|
+
scrollIntoViewIfNeeded(target);
|
|
2322
|
+
target.focus();
|
|
2323
|
+
selectText(target);
|
|
2324
|
+
setNativeValue(target, value);
|
|
2325
|
+
dispatchInputEvents(target);
|
|
2326
|
+
if (target.value !== value) return {
|
|
2327
|
+
content: `"${selector}" 填写后值不一致:期望 "${value}",实际 "${target.value}"`,
|
|
2328
|
+
details: {
|
|
2329
|
+
error: true,
|
|
2330
|
+
code: "FILL_NOT_APPLIED",
|
|
2331
|
+
action,
|
|
2332
|
+
selector
|
|
2333
|
+
}
|
|
2334
|
+
};
|
|
2335
|
+
const suffix = sourceHint ? `(${sourceHint})` : "";
|
|
2336
|
+
return { content: `已填写 ${describeElement(target)}: "${value}"${suffix}` };
|
|
2337
|
+
}
|
|
2338
|
+
if (target instanceof HTMLTextAreaElement) {
|
|
2339
|
+
scrollIntoViewIfNeeded(target);
|
|
2340
|
+
target.focus();
|
|
2341
|
+
selectText(target);
|
|
2342
|
+
setNativeValue(target, value);
|
|
2343
|
+
dispatchInputEvents(target);
|
|
2344
|
+
const suffix = sourceHint ? `(${sourceHint})` : "";
|
|
2345
|
+
return { content: `已填写 ${describeElement(target)}: "${value}"${suffix}` };
|
|
2346
|
+
}
|
|
2347
|
+
if (target instanceof HTMLSelectElement) {
|
|
2348
|
+
target.focus();
|
|
2349
|
+
const options = Array.from(target.options);
|
|
2350
|
+
let matched = options.find((o) => o.value === value);
|
|
2351
|
+
if (!matched) {
|
|
2352
|
+
const normalized = value.trim().toLowerCase();
|
|
2353
|
+
matched = options.find((o) => o.text.trim().toLowerCase() === normalized);
|
|
2354
|
+
}
|
|
2355
|
+
if (!matched) return { content: `"${selector}" 下拉框中不存在选项 "${value}"` };
|
|
2356
|
+
target.value = matched.value;
|
|
2357
|
+
dispatchInputEvents(target);
|
|
2358
|
+
const suffix = sourceHint ? `(${sourceHint})` : "";
|
|
2359
|
+
return { content: `已填写 ${describeElement(target)}: "${value}"${suffix}` };
|
|
2360
|
+
}
|
|
2361
|
+
if (target instanceof HTMLElement && target.isContentEditable) {
|
|
2362
|
+
target.focus();
|
|
2363
|
+
selectText(target);
|
|
2364
|
+
if (value) document.execCommand("insertText", false, value);
|
|
2365
|
+
else document.execCommand("delete", false, void 0);
|
|
2366
|
+
const suffix = sourceHint ? `(${sourceHint})` : "";
|
|
2367
|
+
return { content: `已填写 ${describeElement(target)}: "${value}"${suffix}` };
|
|
2368
|
+
}
|
|
2369
|
+
return null;
|
|
2370
|
+
}
|
|
2371
|
+
function guessNearbyFillTarget(anchor, value) {
|
|
2372
|
+
const preferNumeric = Number.isFinite(Number(value));
|
|
2373
|
+
const scopeEntries = [];
|
|
2374
|
+
const formItem = anchor.closest(".el-form-item");
|
|
2375
|
+
if (formItem) scopeEntries.push({
|
|
2376
|
+
scope: formItem,
|
|
2377
|
+
level: 0
|
|
2378
|
+
});
|
|
2379
|
+
let cursor = anchor.parentElement;
|
|
2380
|
+
for (let level = 1; cursor && level <= 4; level++, cursor = cursor.parentElement) scopeEntries.push({
|
|
2381
|
+
scope: cursor,
|
|
2382
|
+
level
|
|
2383
|
+
});
|
|
2384
|
+
const visited = /* @__PURE__ */ new Set();
|
|
2385
|
+
let best = null;
|
|
2386
|
+
for (const { scope, level } of scopeEntries) {
|
|
2387
|
+
const candidates = Array.from(scope.querySelectorAll("input:not([type=\"hidden\"]), textarea, select, [contenteditable=\"true\"], [role=\"spinbutton\"]"));
|
|
2388
|
+
for (const candidate of candidates) {
|
|
2389
|
+
if (!(candidate instanceof Element)) continue;
|
|
2390
|
+
if (visited.has(candidate)) continue;
|
|
2391
|
+
visited.add(candidate);
|
|
2392
|
+
if (!isCandidateFillTarget(candidate)) continue;
|
|
2393
|
+
if (!isElementVisible(candidate)) continue;
|
|
2394
|
+
let score = 100 - level * 18;
|
|
2395
|
+
score += getFillEventSupportScore(candidate);
|
|
2396
|
+
if (candidate instanceof HTMLInputElement) {
|
|
2397
|
+
const type = candidate.type.toLowerCase();
|
|
2398
|
+
if (preferNumeric && (type === "number" || candidate.getAttribute("role") === "spinbutton")) score += 80;
|
|
2399
|
+
if (!preferNumeric && [
|
|
2400
|
+
"text",
|
|
2401
|
+
"",
|
|
2402
|
+
"search",
|
|
2403
|
+
"email",
|
|
2404
|
+
"tel",
|
|
2405
|
+
"url",
|
|
2406
|
+
"password"
|
|
2407
|
+
].includes(type)) score += 36;
|
|
2408
|
+
}
|
|
2409
|
+
if (candidate.getAttribute("placeholder")) score += 8;
|
|
2410
|
+
if (candidate.getAttribute("aria-label")) score += 8;
|
|
2411
|
+
if (!best || score > best.score) best = {
|
|
2412
|
+
el: candidate,
|
|
2413
|
+
score
|
|
2414
|
+
};
|
|
2415
|
+
}
|
|
2416
|
+
}
|
|
2417
|
+
return best?.el ?? null;
|
|
2418
|
+
}
|
|
2037
2419
|
function selectText(el) {
|
|
2038
2420
|
if (el instanceof HTMLInputElement) {
|
|
2039
2421
|
el.select();
|
|
@@ -2253,7 +2635,7 @@ function createDomTool() {
|
|
|
2253
2635
|
deltaY: Type.Optional(Type.Number({ description: "Vertical scroll delta for scroll action. Positive = down, negative = up." })),
|
|
2254
2636
|
deltaX: Type.Optional(Type.Number({ description: "Horizontal scroll delta for scroll action." })),
|
|
2255
2637
|
steps: Type.Optional(Type.Number({ description: "Repeat count for scroll action (default 1, max 20)." })),
|
|
2256
|
-
waitMs: Type.Optional(Type.Number({ description: "Wait timeout in ms before action (default:
|
|
2638
|
+
waitMs: Type.Optional(Type.Number({ description: "Wait timeout in ms before action (default: 1200)." })),
|
|
2257
2639
|
waitSeconds: Type.Optional(Type.Number({ description: "Wait timeout in seconds (fallback for waitMs)." })),
|
|
2258
2640
|
force: Type.Optional(Type.Boolean({ description: "Skip actionability checks (default false)." }))
|
|
2259
2641
|
}),
|
|
@@ -2335,87 +2717,71 @@ function createDomTool() {
|
|
|
2335
2717
|
const value = params.value;
|
|
2336
2718
|
if (value === void 0) return { content: "缺少 value 参数" };
|
|
2337
2719
|
const target = retarget(el, "follow-label");
|
|
2338
|
-
if (target instanceof
|
|
2339
|
-
const
|
|
2340
|
-
if (
|
|
2341
|
-
|
|
2342
|
-
|
|
2343
|
-
|
|
2344
|
-
|
|
2345
|
-
action,
|
|
2346
|
-
selector
|
|
2720
|
+
if (target instanceof HTMLElement && target.getAttribute("role") === "slider") {
|
|
2721
|
+
const numericValue = Number(value);
|
|
2722
|
+
if (!Number.isFinite(numericValue)) {
|
|
2723
|
+
const guessed = guessNearbyFillTarget(target, value);
|
|
2724
|
+
if (guessed) {
|
|
2725
|
+
const guessedResult = executeFillOnResolvedTarget(guessed, value, selector, action, "heuristic-nearby-target");
|
|
2726
|
+
if (guessedResult) return guessedResult;
|
|
2347
2727
|
}
|
|
2348
|
-
|
|
2349
|
-
|
|
2350
|
-
const finalVal = type === "color" ? value.toLowerCase().trim() : value.trim();
|
|
2351
|
-
target.focus();
|
|
2352
|
-
target.value = finalVal;
|
|
2353
|
-
if (target.value !== finalVal) return {
|
|
2354
|
-
content: `"${selector}" 填写格式不匹配(type=${type})`,
|
|
2728
|
+
return {
|
|
2729
|
+
content: `"${selector}" 为 role=slider,未找到可推断填写目标`,
|
|
2355
2730
|
details: {
|
|
2356
2731
|
error: true,
|
|
2357
|
-
code: "
|
|
2732
|
+
code: "UNSUPPORTED_FILL_TARGET",
|
|
2358
2733
|
action,
|
|
2359
2734
|
selector
|
|
2360
2735
|
}
|
|
2361
2736
|
};
|
|
2362
|
-
dispatchInputEvents(target);
|
|
2363
|
-
return { content: `已填写 ${describeElement(target)}: "${finalVal}"` };
|
|
2364
2737
|
}
|
|
2365
|
-
|
|
2366
|
-
|
|
2367
|
-
|
|
2368
|
-
|
|
2369
|
-
|
|
2370
|
-
|
|
2371
|
-
|
|
2372
|
-
|
|
2373
|
-
|
|
2374
|
-
|
|
2375
|
-
|
|
2376
|
-
|
|
2377
|
-
|
|
2378
|
-
|
|
2379
|
-
|
|
2380
|
-
|
|
2738
|
+
const linkedInput = findAssociatedSliderInput(target);
|
|
2739
|
+
if (linkedInput) {
|
|
2740
|
+
const filled = executeFillOnResolvedTarget(linkedInput, String(numericValue), selector, action, `from ${describeElement(target)}`);
|
|
2741
|
+
if (filled) return filled;
|
|
2742
|
+
}
|
|
2743
|
+
const min = Number(target.getAttribute("aria-valuemin") ?? "1");
|
|
2744
|
+
const max = Number(target.getAttribute("aria-valuemax") ?? String(target.children.length || 5));
|
|
2745
|
+
const discreteCount = Number.isFinite(max - min + 1) ? Math.max(1, Math.round(max - min + 1)) : target.children.length;
|
|
2746
|
+
const desiredIndex = Math.round(numericValue - min);
|
|
2747
|
+
const children = Array.from(target.children).filter((node) => node instanceof HTMLElement);
|
|
2748
|
+
if (children.length >= discreteCount && desiredIndex >= 0 && desiredIndex < children.length) {
|
|
2749
|
+
const item = children[desiredIndex];
|
|
2750
|
+
scrollIntoViewIfNeeded(item);
|
|
2751
|
+
dispatchClickEvents(item);
|
|
2752
|
+
return { content: `已点击 ${describeElement(item)},设置 ${describeElement(target)} 值为 ${numericValue}` };
|
|
2753
|
+
}
|
|
2754
|
+
const guessed = guessNearbyFillTarget(target, String(numericValue));
|
|
2755
|
+
if (guessed) {
|
|
2756
|
+
const guessedResult = executeFillOnResolvedTarget(guessed, String(numericValue), selector, action, "heuristic-nearby-target");
|
|
2757
|
+
if (guessedResult) return guessedResult;
|
|
2758
|
+
}
|
|
2759
|
+
return {
|
|
2760
|
+
content: `"${selector}" 为 role=slider,但未找到可写入输入框或可点击离散子项`,
|
|
2381
2761
|
details: {
|
|
2382
2762
|
error: true,
|
|
2383
|
-
code: "
|
|
2763
|
+
code: "UNSUPPORTED_FILL_TARGET",
|
|
2384
2764
|
action,
|
|
2385
2765
|
selector
|
|
2386
2766
|
}
|
|
2387
2767
|
};
|
|
2388
|
-
return { content: `已填写 ${describeElement(target)}: "${value}"` };
|
|
2389
2768
|
}
|
|
2390
|
-
|
|
2391
|
-
|
|
2392
|
-
|
|
2393
|
-
|
|
2394
|
-
|
|
2395
|
-
|
|
2396
|
-
return { content: `已填写 ${describeElement(target)}: "${value}"` };
|
|
2769
|
+
const directFilled = executeFillOnResolvedTarget(target, value, selector, action);
|
|
2770
|
+
if (directFilled) return directFilled;
|
|
2771
|
+
const guessed = guessNearbyFillTarget(target, value);
|
|
2772
|
+
if (guessed) {
|
|
2773
|
+
const guessedResult = executeFillOnResolvedTarget(guessed, value, selector, action, "heuristic-nearby-target");
|
|
2774
|
+
if (guessedResult) return guessedResult;
|
|
2397
2775
|
}
|
|
2398
|
-
|
|
2399
|
-
|
|
2400
|
-
|
|
2401
|
-
|
|
2402
|
-
|
|
2403
|
-
|
|
2404
|
-
|
|
2776
|
+
return {
|
|
2777
|
+
content: `"${selector}" 不是可编辑元素,且未在附近找到可推断填写目标`,
|
|
2778
|
+
details: {
|
|
2779
|
+
error: true,
|
|
2780
|
+
code: "UNSUPPORTED_FILL_TARGET",
|
|
2781
|
+
action,
|
|
2782
|
+
selector
|
|
2405
2783
|
}
|
|
2406
|
-
|
|
2407
|
-
target.value = matched.value;
|
|
2408
|
-
dispatchInputEvents(target);
|
|
2409
|
-
return { content: `已填写 ${describeElement(target)}: "${value}"` };
|
|
2410
|
-
}
|
|
2411
|
-
if (target instanceof HTMLElement && target.isContentEditable) {
|
|
2412
|
-
target.focus();
|
|
2413
|
-
selectText(target);
|
|
2414
|
-
if (value) document.execCommand("insertText", false, value);
|
|
2415
|
-
else document.execCommand("delete", false, void 0);
|
|
2416
|
-
return { content: `已填写 ${describeElement(target)}: "${value}"` };
|
|
2417
|
-
}
|
|
2418
|
-
return { content: `"${selector}" 不是可编辑元素` };
|
|
2784
|
+
};
|
|
2419
2785
|
}
|
|
2420
2786
|
case "select_option": {
|
|
2421
2787
|
const value = params.value;
|
|
@@ -2703,6 +3069,40 @@ const MAX_SNAPSHOT_ATTR_VALUE_LENGTH = 120;
|
|
|
2703
3069
|
const MAX_EXPANDED_LIST_CHILDREN = 120;
|
|
2704
3070
|
/** 定向放宽 children 的硬上限。 */
|
|
2705
3071
|
const MAX_EXPANDED_CHILDREN_LIMIT = 300;
|
|
3072
|
+
/** 事件名 → 快照简写映射(压缩 token)。 */
|
|
3073
|
+
const EVENT_ABBREV = {
|
|
3074
|
+
click: "clk",
|
|
3075
|
+
dblclick: "dbl",
|
|
3076
|
+
mousedown: "mdn",
|
|
3077
|
+
mouseup: "mup",
|
|
3078
|
+
mousemove: "mmv",
|
|
3079
|
+
mouseover: "mov",
|
|
3080
|
+
mouseout: "mot",
|
|
3081
|
+
mouseenter: "men",
|
|
3082
|
+
mouseleave: "mlv",
|
|
3083
|
+
pointerdown: "pdn",
|
|
3084
|
+
pointerup: "pup",
|
|
3085
|
+
pointermove: "pmv",
|
|
3086
|
+
touchstart: "tst",
|
|
3087
|
+
touchend: "ted",
|
|
3088
|
+
keydown: "kdn",
|
|
3089
|
+
keyup: "kup",
|
|
3090
|
+
input: "inp",
|
|
3091
|
+
change: "chg",
|
|
3092
|
+
submit: "sub",
|
|
3093
|
+
focus: "fcs",
|
|
3094
|
+
blur: "blr",
|
|
3095
|
+
scroll: "scl",
|
|
3096
|
+
wheel: "whl",
|
|
3097
|
+
drag: "drg",
|
|
3098
|
+
dragstart: "drs",
|
|
3099
|
+
dragend: "dre",
|
|
3100
|
+
drop: "drp",
|
|
3101
|
+
contextmenu: "ctx"
|
|
3102
|
+
};
|
|
3103
|
+
function abbrevEvent(name) {
|
|
3104
|
+
return EVENT_ABBREV[name] ?? name.slice(0, 3);
|
|
3105
|
+
}
|
|
2706
3106
|
/**
|
|
2707
3107
|
* 规整快照属性值,避免把长 base64/data URL 原样注入快照。
|
|
2708
3108
|
*/
|
|
@@ -2744,7 +3144,7 @@ function sanitizeSnapshotAttrValue(value) {
|
|
|
2744
3144
|
*/
|
|
2745
3145
|
function generateSnapshot(root = document.body, options = {}) {
|
|
2746
3146
|
const opts = typeof options === "number" ? { maxDepth: options } : options;
|
|
2747
|
-
const maxDepth = opts.maxDepth ??
|
|
3147
|
+
const maxDepth = opts.maxDepth ?? 12;
|
|
2748
3148
|
const viewportOnly = opts.viewportOnly ?? true;
|
|
2749
3149
|
const pruneLayout = opts.pruneLayout ?? true;
|
|
2750
3150
|
const maxNodes = opts.maxNodes ?? 220;
|
|
@@ -2808,6 +3208,64 @@ function generateSnapshot(root = document.body, options = {}) {
|
|
|
2808
3208
|
"LABEL",
|
|
2809
3209
|
"SUMMARY"
|
|
2810
3210
|
]);
|
|
3211
|
+
const INTERACTIVE_EVENTS = new Set([
|
|
3212
|
+
"click",
|
|
3213
|
+
"dblclick",
|
|
3214
|
+
"mousedown",
|
|
3215
|
+
"mouseup",
|
|
3216
|
+
"pointerdown",
|
|
3217
|
+
"pointerup",
|
|
3218
|
+
"touchstart",
|
|
3219
|
+
"touchend",
|
|
3220
|
+
"input",
|
|
3221
|
+
"change",
|
|
3222
|
+
"keydown",
|
|
3223
|
+
"keyup",
|
|
3224
|
+
"submit",
|
|
3225
|
+
"focus",
|
|
3226
|
+
"blur"
|
|
3227
|
+
]);
|
|
3228
|
+
/** 交互性 ARIA role — 需要分配 hash ID 的角色集合 */
|
|
3229
|
+
const INTERACTIVE_ROLES = new Set([
|
|
3230
|
+
"button",
|
|
3231
|
+
"link",
|
|
3232
|
+
"tab",
|
|
3233
|
+
"switch",
|
|
3234
|
+
"slider",
|
|
3235
|
+
"checkbox",
|
|
3236
|
+
"radio",
|
|
3237
|
+
"combobox",
|
|
3238
|
+
"listbox",
|
|
3239
|
+
"option",
|
|
3240
|
+
"menuitem",
|
|
3241
|
+
"textbox",
|
|
3242
|
+
"spinbutton",
|
|
3243
|
+
"searchbox",
|
|
3244
|
+
"treeitem",
|
|
3245
|
+
"gridcell",
|
|
3246
|
+
"scrollbar"
|
|
3247
|
+
]);
|
|
3248
|
+
/**
|
|
3249
|
+
* 事件优先级(值越大越优先):
|
|
3250
|
+
* 输入链路(input/change/focus/blur) > 点击链路(click/pointer) > 其他事件。
|
|
3251
|
+
*/
|
|
3252
|
+
const EVENT_PRIORITY = {
|
|
3253
|
+
input: 140,
|
|
3254
|
+
change: 130,
|
|
3255
|
+
focus: 120,
|
|
3256
|
+
blur: 110,
|
|
3257
|
+
keydown: 100,
|
|
3258
|
+
keyup: 90,
|
|
3259
|
+
click: 80,
|
|
3260
|
+
dblclick: 70,
|
|
3261
|
+
pointerdown: 60,
|
|
3262
|
+
pointerup: 55,
|
|
3263
|
+
mousedown: 50,
|
|
3264
|
+
mouseup: 45,
|
|
3265
|
+
touchstart: 40,
|
|
3266
|
+
touchend: 35,
|
|
3267
|
+
submit: 30
|
|
3268
|
+
};
|
|
2811
3269
|
/** 布尔状态属性 — 只在存在时输出(无值),如 disabled、checked */
|
|
2812
3270
|
const BOOLEAN_ATTRS = [
|
|
2813
3271
|
"disabled",
|
|
@@ -2856,15 +3314,75 @@ function generateSnapshot(root = document.body, options = {}) {
|
|
|
2856
3314
|
if (el.getAttribute("id")) return false;
|
|
2857
3315
|
if (el.getAttribute("role") || el.getAttribute("aria-label")) return false;
|
|
2858
3316
|
for (const attr of Array.from(el.attributes)) if (attr.name.startsWith("on")) return false;
|
|
3317
|
+
if (hasTrackedElementEvents(el)) return false;
|
|
2859
3318
|
if (directText) return false;
|
|
2860
3319
|
return true;
|
|
2861
3320
|
}
|
|
3321
|
+
function hasInteractiveTrackedEvents(el) {
|
|
3322
|
+
const tracked = getTrackedElementEvents(el);
|
|
3323
|
+
if (tracked.length === 0) return false;
|
|
3324
|
+
return tracked.some((name) => INTERACTIVE_EVENTS.has(name));
|
|
3325
|
+
}
|
|
3326
|
+
function getTrackedEventPriorityScore(el) {
|
|
3327
|
+
const tracked = getTrackedElementEvents(el);
|
|
3328
|
+
if (tracked.length === 0) return 0;
|
|
3329
|
+
let score = 0;
|
|
3330
|
+
for (const name of tracked) score += EVENT_PRIORITY[name] ?? 8;
|
|
3331
|
+
return score;
|
|
3332
|
+
}
|
|
3333
|
+
/**
|
|
3334
|
+
* 元素优先级:
|
|
3335
|
+
* 1) 输入控件/按钮等语义控件
|
|
3336
|
+
* 2) 事件追踪优先级(输入、点击、失焦等)
|
|
3337
|
+
* 3) inline 事件与可聚焦能力补充加分
|
|
3338
|
+
*/
|
|
3339
|
+
function getElementPriorityScore(el) {
|
|
3340
|
+
let score = 0;
|
|
3341
|
+
if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement || el instanceof HTMLSelectElement) score += 200;
|
|
3342
|
+
else if (el instanceof HTMLButtonElement || el instanceof HTMLAnchorElement) score += 180;
|
|
3343
|
+
else if (el.getAttribute("role") === "button" || el.getAttribute("role") === "switch" || el.getAttribute("role") === "slider") score += 160;
|
|
3344
|
+
score += getTrackedEventPriorityScore(el);
|
|
3345
|
+
if (el.hasAttribute("onclick")) score += 60;
|
|
3346
|
+
if (el.hasAttribute("oninput") || el.hasAttribute("onchange")) score += 80;
|
|
3347
|
+
if (el.hasAttribute("onfocus") || el.hasAttribute("onblur")) score += 70;
|
|
3348
|
+
if (el.hasAttribute("tabindex")) score += 20;
|
|
3349
|
+
return score;
|
|
3350
|
+
}
|
|
3351
|
+
function orderChildrenByPriority(children) {
|
|
3352
|
+
return children.map((child, index) => ({
|
|
3353
|
+
child,
|
|
3354
|
+
index,
|
|
3355
|
+
interactive: isInteractiveElement(child),
|
|
3356
|
+
score: getElementPriorityScore(child)
|
|
3357
|
+
})).sort((a, b) => {
|
|
3358
|
+
if (a.interactive !== b.interactive) return a.interactive ? -1 : 1;
|
|
3359
|
+
if (b.score !== a.score) return b.score - a.score;
|
|
3360
|
+
return a.index - b.index;
|
|
3361
|
+
}).map((entry) => entry.child);
|
|
3362
|
+
}
|
|
2862
3363
|
function isInteractiveElement(el) {
|
|
2863
3364
|
if (INTERACTIVE_TAGS.has(el.tagName)) return true;
|
|
2864
3365
|
if (el.hasAttribute("onclick")) return true;
|
|
2865
3366
|
if (el.hasAttribute("role")) return true;
|
|
2866
3367
|
if (el.hasAttribute("tabindex")) return true;
|
|
2867
3368
|
if (el.hasAttribute("aria-label")) return true;
|
|
3369
|
+
if (hasInteractiveTrackedEvents(el)) return true;
|
|
3370
|
+
return false;
|
|
3371
|
+
}
|
|
3372
|
+
/**
|
|
3373
|
+
* 判断元素是否需要分配 hash ID(仅交互节点分配,节省 token)。
|
|
3374
|
+
*
|
|
3375
|
+
* 核心依据:元素是否绑定了交互事件(INTERACTIVE_EVENTS 集合)。
|
|
3376
|
+
* 辅助依据:语义交互标签、内联事件、ARIA role、tabindex 等兜底。
|
|
3377
|
+
*/
|
|
3378
|
+
function needsHashId(el) {
|
|
3379
|
+
if (hasInteractiveTrackedEvents(el)) return true;
|
|
3380
|
+
for (const attr of Array.from(el.attributes)) if (attr.name.startsWith("on")) return true;
|
|
3381
|
+
if (INTERACTIVE_TAGS.has(el.tagName)) return true;
|
|
3382
|
+
const role = el.getAttribute("role");
|
|
3383
|
+
if (role && INTERACTIVE_ROLES.has(role)) return true;
|
|
3384
|
+
if (el.hasAttribute("tabindex")) return true;
|
|
3385
|
+
if (el.isContentEditable && el.getAttribute("contenteditable") !== "inherit") return true;
|
|
2868
3386
|
return false;
|
|
2869
3387
|
}
|
|
2870
3388
|
/** 判断是否为“选项列表”容器(时间/下拉/listbox 等)。 */
|
|
@@ -2900,8 +3418,11 @@ function generateSnapshot(root = document.body, options = {}) {
|
|
|
2900
3418
|
if (!isInViewport(el, depth)) return "";
|
|
2901
3419
|
const indent = " ".repeat(depth);
|
|
2902
3420
|
const tag = el.tagName.toLowerCase();
|
|
3421
|
+
const rawRole = el.getAttribute("role");
|
|
3422
|
+
const useRoleAsTag = !!(rawRole && INTERACTIVE_ROLES.has(rawRole) && rawRole !== tag);
|
|
3423
|
+
const displayTag = useRoleAsTag ? rawRole : tag;
|
|
2903
3424
|
const currentPath = `${parentPath}/${tag}${getSiblingIndex(el)}`;
|
|
2904
|
-
const hashId = refStore ? refStore.set(el, currentPath) : void 0;
|
|
3425
|
+
const hashId = refStore && needsHashId(el) ? refStore.set(el, currentPath) : void 0;
|
|
2905
3426
|
const attrs = [];
|
|
2906
3427
|
const elId = el.getAttribute("id");
|
|
2907
3428
|
if (elId) attrs.push(`id="${elId}"`);
|
|
@@ -2911,6 +3432,7 @@ function generateSnapshot(root = document.body, options = {}) {
|
|
|
2911
3432
|
if (cls) attrs.push(`class="${cls}"`);
|
|
2912
3433
|
}
|
|
2913
3434
|
for (const attr of INTERACTIVE_ATTRS) {
|
|
3435
|
+
if (attr === "role" && useRoleAsTag) continue;
|
|
2914
3436
|
const val = el.getAttribute(attr);
|
|
2915
3437
|
if (val) {
|
|
2916
3438
|
const safeVal = sanitizeSnapshotAttrValue(val);
|
|
@@ -2925,6 +3447,12 @@ function generateSnapshot(root = document.body, options = {}) {
|
|
|
2925
3447
|
if (!attrs.includes("readonly")) attrs.push("readonly");
|
|
2926
3448
|
}
|
|
2927
3449
|
if (el.hasAttribute("onclick")) attrs.push("onclick");
|
|
3450
|
+
const trackedEvents = getTrackedElementEvents(el);
|
|
3451
|
+
if (trackedEvents.length > 0) {
|
|
3452
|
+
const preview = trackedEvents.slice(0, 6).map(abbrevEvent).join(",");
|
|
3453
|
+
const suffix = trackedEvents.length > 6 ? ",..." : "";
|
|
3454
|
+
attrs.push(`listeners="${preview}${suffix}"`);
|
|
3455
|
+
}
|
|
2928
3456
|
const testId = el.getAttribute("data-testid") || el.getAttribute("data-test-id");
|
|
2929
3457
|
if (testId) {
|
|
2930
3458
|
const safeTestId = sanitizeSnapshotAttrValue(testId).slice(0, 25);
|
|
@@ -2951,10 +3479,7 @@ function generateSnapshot(root = document.body, options = {}) {
|
|
|
2951
3479
|
}
|
|
2952
3480
|
directText = directText.trim();
|
|
2953
3481
|
if (isEmptyLayoutContainer(el, directText)) {
|
|
2954
|
-
const
|
|
2955
|
-
const interactiveChildren = allChildren.filter(isInteractiveElement);
|
|
2956
|
-
const nonInteractiveChildren = allChildren.filter((child) => !isInteractiveElement(child));
|
|
2957
|
-
const orderedChildren = [...interactiveChildren, ...nonInteractiveChildren];
|
|
3482
|
+
const orderedChildren = orderChildrenByPriority(Array.from(el.children));
|
|
2958
3483
|
const childLimit = resolveChildLimit(el, maxChildren, hashId);
|
|
2959
3484
|
const selectedChildren = orderedChildren.slice(0, childLimit);
|
|
2960
3485
|
const omittedChildren = orderedChildren.length - selectedChildren.length;
|
|
@@ -2965,23 +3490,19 @@ function generateSnapshot(root = document.body, options = {}) {
|
|
|
2965
3490
|
}
|
|
2966
3491
|
if (childBlocks.length === 0 && omittedChildren <= 0) return "";
|
|
2967
3492
|
if (!(childBlocks.length >= 2 || omittedChildren > 0)) return childBlocks.join("\n");
|
|
2968
|
-
const groupLines = [`${" ".repeat(depth)}([${
|
|
3493
|
+
const groupLines = [`${" ".repeat(depth)}([${displayTag}] collapsed-group`];
|
|
2969
3494
|
for (const block of childBlocks) groupLines.push(indentMultiline(block, 1));
|
|
2970
3495
|
if (omittedChildren > 0) groupLines.push(`${" ".repeat(depth + 1)}... (${omittedChildren} children omitted)`);
|
|
2971
3496
|
groupLines.push(`${" ".repeat(depth)})`);
|
|
2972
3497
|
return groupLines.join("\n");
|
|
2973
3498
|
}
|
|
2974
|
-
let line = `${indent}[${
|
|
3499
|
+
let line = `${indent}[${displayTag}]`;
|
|
2975
3500
|
if (directText) line += ` "${directText.slice(0, maxTextLength)}"`;
|
|
2976
3501
|
if (attrs.length) line += ` ${attrs.join(" ")}`;
|
|
2977
3502
|
if (hashId) line += ` #${hashId}`;
|
|
2978
|
-
else line += ` ref="${currentPath}"`;
|
|
2979
3503
|
const lines = [line];
|
|
2980
3504
|
emittedNodes++;
|
|
2981
|
-
const
|
|
2982
|
-
const interactiveChildren = allChildren.filter(isInteractiveElement);
|
|
2983
|
-
const nonInteractiveChildren = allChildren.filter((child) => !isInteractiveElement(child));
|
|
2984
|
-
const orderedChildren = [...interactiveChildren, ...nonInteractiveChildren];
|
|
3505
|
+
const orderedChildren = orderChildrenByPriority(Array.from(el.children));
|
|
2985
3506
|
const childLimit = resolveChildLimit(el, maxChildren, hashId);
|
|
2986
3507
|
const selectedChildren = orderedChildren.slice(0, childLimit);
|
|
2987
3508
|
const omittedChildren = orderedChildren.length - selectedChildren.length;
|
|
@@ -3037,7 +3558,7 @@ function createPageInfoTool() {
|
|
|
3037
3558
|
schema: Type.Object({
|
|
3038
3559
|
action: Type.String({ description: "Info action: get_url | get_title | get_selection | get_viewport | snapshot | query_all" }),
|
|
3039
3560
|
selector: Type.Optional(Type.String({ description: "CSS selector for query_all action" })),
|
|
3040
|
-
maxDepth: Type.Optional(Type.Number({ description: "Max depth for snapshot (default:
|
|
3561
|
+
maxDepth: Type.Optional(Type.Number({ description: "Max depth for snapshot (default: 12)" })),
|
|
3041
3562
|
viewportOnly: Type.Optional(Type.Boolean({ description: "Only snapshot elements visible in viewport (default: true)" })),
|
|
3042
3563
|
pruneLayout: Type.Optional(Type.Boolean({ description: "Collapse empty layout containers like div/span (default: true)" })),
|
|
3043
3564
|
maxNodes: Type.Optional(Type.Number({ description: "Maximum nodes to include in snapshot (default: 220)" })),
|
|
@@ -3066,7 +3587,7 @@ function createPageInfoTool() {
|
|
|
3066
3587
|
return { content: JSON.stringify(info, null, 2) };
|
|
3067
3588
|
}
|
|
3068
3589
|
case "snapshot": {
|
|
3069
|
-
const maxDepth = params.maxDepth ??
|
|
3590
|
+
const maxDepth = params.maxDepth ?? 12;
|
|
3070
3591
|
const viewportOnly = params.viewportOnly ?? true;
|
|
3071
3592
|
const pruneLayout = params.pruneLayout ?? true;
|
|
3072
3593
|
const maxNodes = params.maxNodes ?? 220;
|
|
@@ -3220,7 +3741,7 @@ function createNavigateTool() {
|
|
|
3220
3741
|
* - hash selector(如 #abc123)优先通过 RefStore 解析。
|
|
3221
3742
|
* - 可见性语义与 dom-tool 保持一致(参考 Playwright 风格)。
|
|
3222
3743
|
*/
|
|
3223
|
-
const DEFAULT_TIMEOUT =
|
|
3744
|
+
const DEFAULT_TIMEOUT = 6e3;
|
|
3224
3745
|
const POLL_INTERVAL_MS = 80;
|
|
3225
3746
|
const STABLE_TICK_MS = 50;
|
|
3226
3747
|
const OBSERVER_OPTIONS = {
|
|
@@ -3412,7 +3933,7 @@ function createWaitTool() {
|
|
|
3412
3933
|
selector: Type.Optional(Type.String({ description: "CSS selector for wait_for_selector/wait_for_hidden" })),
|
|
3413
3934
|
state: Type.Optional(Type.String({ description: "Selector state for wait_for_selector: attached | visible | hidden | detached (default: attached)" })),
|
|
3414
3935
|
text: Type.Optional(Type.String({ description: "Text to wait for in wait_for_text" })),
|
|
3415
|
-
timeout: Type.Optional(Type.Number({ description: "Timeout in milliseconds (default:
|
|
3936
|
+
timeout: Type.Optional(Type.Number({ description: "Timeout in milliseconds (default: 6000)" })),
|
|
3416
3937
|
quietMs: Type.Optional(Type.Number({ description: "Quiet window for wait_for_stable in milliseconds (default: 300)" }))
|
|
3417
3938
|
}),
|
|
3418
3939
|
execute: async (params) => {
|
|
@@ -3749,6 +4270,7 @@ function registerToolHandler(executors) {
|
|
|
3749
4270
|
* │ └──────────┘ └────────────┘ └──────────────┘ │
|
|
3750
4271
|
* └──────────────────────────────────────────────────┘
|
|
3751
4272
|
*/
|
|
4273
|
+
installEventListenerTracking();
|
|
3752
4274
|
var WebAgent = class WebAgent {
|
|
3753
4275
|
/** 默认系统提示词 key(兼容旧版 setSystemPrompt(prompt))。 */
|
|
3754
4276
|
static DEFAULT_SYSTEM_PROMPT_KEY = "default";
|
|
@@ -3781,6 +4303,8 @@ var WebAgent = class WebAgent {
|
|
|
3781
4303
|
autoSnapshot;
|
|
3782
4304
|
/** 快照选项 */
|
|
3783
4305
|
snapshotOptions;
|
|
4306
|
+
/** 轮次后稳定等待配置 */
|
|
4307
|
+
roundStabilityWait;
|
|
3784
4308
|
/** 工具注册表实例 — 每个 WebAgent 拥有独立的工具集 */
|
|
3785
4309
|
registry = new ToolRegistry();
|
|
3786
4310
|
/** 事件回调 — 绑定后可实时获取 Agent 进度,用于 UI 展示 */
|
|
@@ -3797,6 +4321,7 @@ var WebAgent = class WebAgent {
|
|
|
3797
4321
|
this.memory = options.memory ?? false;
|
|
3798
4322
|
this.autoSnapshot = options.autoSnapshot ?? true;
|
|
3799
4323
|
this.snapshotOptions = options.snapshotOptions ?? {};
|
|
4324
|
+
this.roundStabilityWait = options.roundStabilityWait;
|
|
3800
4325
|
if (typeof options.systemPrompt === "string") this.setSystemPrompt(options.systemPrompt);
|
|
3801
4326
|
else if (options.systemPrompt && typeof options.systemPrompt === "object") this.setSystemPrompts(options.systemPrompt);
|
|
3802
4327
|
}
|
|
@@ -3961,7 +4486,7 @@ var WebAgent = class WebAgent {
|
|
|
3961
4486
|
let initialSnapshot;
|
|
3962
4487
|
try {
|
|
3963
4488
|
const snapshot = generateSnapshot(document.body, {
|
|
3964
|
-
maxDepth:
|
|
4489
|
+
maxDepth: 12,
|
|
3965
4490
|
viewportOnly: false,
|
|
3966
4491
|
maxNodes: 500,
|
|
3967
4492
|
maxChildren: 30,
|
|
@@ -3989,6 +4514,7 @@ var WebAgent = class WebAgent {
|
|
|
3989
4514
|
history: this.memory ? this.history : void 0,
|
|
3990
4515
|
dryRun: this.dryRun,
|
|
3991
4516
|
maxRounds: this.maxRounds,
|
|
4517
|
+
roundStabilityWait: this.roundStabilityWait,
|
|
3992
4518
|
callbacks: wrappedCallbacks
|
|
3993
4519
|
});
|
|
3994
4520
|
if (this.memory) this.history = result.messages;
|