@glittering0717/ai-browser 0.0.0-stage → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +784 -2
- package/bin/ai-browser.cjs +78 -0
- package/examples/AGENT_INSTRUCTIONS.md +166 -0
- package/examples/mcp-config.json +12 -0
- package/llms-install.md +158 -0
- package/package.json +78 -4
- package/scripts/electron_resolve.cjs +241 -0
- package/scripts/start.cjs +60 -0
- package/src/main/axExtractor.js +436 -0
- package/src/main/canvas_hook_source.js +654 -0
- package/src/main/canvas_monitor.js +269 -0
- package/src/main/capability_probe.js +415 -0
- package/src/main/download_monitor.js +137 -0
- package/src/main/index.js +359 -0
- package/src/main/mcp_server.js +408 -0
- package/src/main/mcp_tools.js +289 -0
- package/src/main/mcp_ws.js +178 -0
- package/src/main/network_monitor.js +805 -0
- package/src/main/page_manager.js +2743 -0
- package/src/main/snapshot_diff.js +371 -0
- package/src/main/ws_server.js +440 -0
- package/src/preload/actions.cjs +190 -0
- package/src/preload/bridge.cjs +75 -0
- package/src/preload/extractor.cjs +936 -0
- package/src/preload/watcher.cjs +241 -0
- package/src/renderer/tab_bar.html +115 -0
- package/src/shared/config.js +32 -0
- package/src/shared/deadline.js +89 -0
- package/src/shared/guards.js +17 -0
- package/src/shared/protocol.js +116 -0
- package/src/shared/value_contract.cjs +132 -0
- package/src/shared/version.js +5 -0
- package/src/shared/watchdog.js +57 -0
|
@@ -0,0 +1,936 @@
|
|
|
1
|
+
// preload/extractor.cjs — Semantic extraction (CommonJS)
|
|
2
|
+
// Split out of bridge.js v6.9. Pure DOM functions, no Electron dependency,
|
|
3
|
+
// so it can be unit-tested under jsdom.
|
|
4
|
+
// Exports: extractTree(), extractPageContext()
|
|
5
|
+
var valueContract = require("../shared/value_contract.cjs");
|
|
6
|
+
var truncateNodeValue = valueContract.truncateNodeValue;
|
|
7
|
+
var sensitiveInputKind = valueContract.sensitiveInputKind;
|
|
8
|
+
|
|
9
|
+
// ============================================================
|
|
10
|
+
// 1. EXTRACT TREE — DOM → semantic tree with bounds + states
|
|
11
|
+
// ============================================================
|
|
12
|
+
|
|
13
|
+
// 自动编号的序号。**故意放在模块级**:每次抽取都归零会让"新旧元素拿到同一个 id"
|
|
14
|
+
// (旧元素沿用自身 data-ai-id,新元素却可能生成出同名的号),所以它只增不减。
|
|
15
|
+
var AI_ID_SEQ = 0;
|
|
16
|
+
|
|
17
|
+
function extractTree() {
|
|
18
|
+
var seen = new WeakSet();
|
|
19
|
+
var counter = 0;
|
|
20
|
+
var VERSION_TAG = "v6.9";
|
|
21
|
+
|
|
22
|
+
// 元素自己所在的 window。跨文档时必须用它:
|
|
23
|
+
// `el instanceof HTMLElement` 里 HTMLElement 是**顶层 window**的构造函数,
|
|
24
|
+
// 而同源 iframe 里的元素属于 iframe 自己的 window —— instanceof 恒为 false,
|
|
25
|
+
// 于是 iframe 里每个元素都被当成"不是 HTML 元素"丢掉。
|
|
26
|
+
// 实测:ComfyUI 编辑器装在同源 iframe 里(2024 个元素 / 106 个按钮 / 42 个输入框),
|
|
27
|
+
// 修复前语义树里一个都看不见。
|
|
28
|
+
function winOf(el) {
|
|
29
|
+
try {
|
|
30
|
+
var d = el && el.ownerDocument;
|
|
31
|
+
var w = d && d.defaultView;
|
|
32
|
+
return w || window;
|
|
33
|
+
} catch (e) { return window; }
|
|
34
|
+
}
|
|
35
|
+
function isHtmlElement(el) {
|
|
36
|
+
if (!el || typeof el !== "object") return false;
|
|
37
|
+
var w = winOf(el);
|
|
38
|
+
try { return el instanceof w.HTMLElement; } catch (e) {
|
|
39
|
+
return !!(el.tagName && typeof el.getAttribute === "function");
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
function isHidden(el) {
|
|
44
|
+
if (!isHtmlElement(el)) return true;
|
|
45
|
+
var tag = el.tagName.toLowerCase();
|
|
46
|
+
if (tag === "dialog") return false;
|
|
47
|
+
var style = winOf(el).getComputedStyle(el);
|
|
48
|
+
if (style.display === "none") return true;
|
|
49
|
+
if (style.visibility === "hidden") return true;
|
|
50
|
+
if (el.hasAttribute("aria-hidden") && el.getAttribute("aria-hidden") === "true") return true;
|
|
51
|
+
if (el.hidden === true) return true;
|
|
52
|
+
var p = el.parentElement;
|
|
53
|
+
while (p && p !== document.body) {
|
|
54
|
+
if (p.hidden === true) return true;
|
|
55
|
+
if (p.hasAttribute("aria-hidden") && p.getAttribute("aria-hidden") === "true") return true;
|
|
56
|
+
var ps = winOf(p).getComputedStyle(p);
|
|
57
|
+
if (ps.display === "none") return true;
|
|
58
|
+
p = p.parentElement;
|
|
59
|
+
}
|
|
60
|
+
return false;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function isPureLayout(el) {
|
|
64
|
+
if (!isHtmlElement(el)) return false;
|
|
65
|
+
var tag = el.tagName.toLowerCase();
|
|
66
|
+
if (tag !== "div" && tag !== "span") return false;
|
|
67
|
+
if (el.contentEditable === "true" || el.getAttribute("contenteditable") === "true") return false;
|
|
68
|
+
var cls = (el.className || "").toLowerCase();
|
|
69
|
+
if (cls.indexOf("codemirror") >= 0 || cls.indexOf("prosemirror") >= 0 || cls.indexOf("monaco") >= 0) return false;
|
|
70
|
+
if (el.hasAttribute("role")) return false;
|
|
71
|
+
if (typeof el.onclick === "function") return false;
|
|
72
|
+
var hasAria = false;
|
|
73
|
+
try { var attrs = el.attributes; for (var ai = 0; ai < attrs.length; ai++) { if (attrs[ai].name.indexOf("aria-") === 0) { hasAria = true; break; } } } catch(e) {}
|
|
74
|
+
if (hasAria) return false;
|
|
75
|
+
// Check if any child is semantic — if so, keep as container
|
|
76
|
+
var children = el.children;
|
|
77
|
+
for (var ci = 0; ci < children.length; ci++) {
|
|
78
|
+
if (children[ci].tagName && INTERACTIVE_TAGS.indexOf(children[ci].tagName.toLowerCase()) >= 0) return false;
|
|
79
|
+
if (children[ci].hasAttribute && children[ci].hasAttribute("role")) return false;
|
|
80
|
+
}
|
|
81
|
+
return true;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
var INTERACTIVE_TAGS = [
|
|
85
|
+
"a","button","input","select","textarea","option","details","dialog",
|
|
86
|
+
"summary","video","audio","canvas","svg","img",
|
|
87
|
+
"table","ul","ol","li","h1","h2","h3","h4","h5","h6","p","label","form",
|
|
88
|
+
"nav","main","header","footer","section","article","aside","iframe"
|
|
89
|
+
];
|
|
90
|
+
|
|
91
|
+
function isSemantic(el) {
|
|
92
|
+
if (!isHtmlElement(el)) return false;
|
|
93
|
+
if (el === document.body || el === document.documentElement) return true;
|
|
94
|
+
if (isHidden(el)) return false;
|
|
95
|
+
var tag = el.tagName.toLowerCase();
|
|
96
|
+
if (INTERACTIVE_TAGS.indexOf(tag) >= 0) return true;
|
|
97
|
+
if (el.hasAttribute("role")) return true;
|
|
98
|
+
if (typeof el.onclick === "function") return true;
|
|
99
|
+
if (el.contentEditable === "true" || el.getAttribute("contenteditable") === "true") return true;
|
|
100
|
+
if (isPureLayout(el)) return false;
|
|
101
|
+
if (tag === "div" || tag === "span") return false;
|
|
102
|
+
// Non-interactive/non-visible tags — never semantic
|
|
103
|
+
var EXCLUDED_TAGS = ["script", "link", "style", "meta", "noscript", "br", "wbr", "param", "source", "track"];
|
|
104
|
+
if (EXCLUDED_TAGS.indexOf(tag) >= 0) return false;
|
|
105
|
+
return true;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
function inferRole(el) {
|
|
109
|
+
var tag = el.tagName.toLowerCase();
|
|
110
|
+
if (tag === "button") return "button";
|
|
111
|
+
if (tag === "input") {
|
|
112
|
+
var type = (el.getAttribute("type") || "").toLowerCase();
|
|
113
|
+
if (type === "checkbox") return "checkbox";
|
|
114
|
+
if (type === "radio") return "radio";
|
|
115
|
+
if (type === "range" || type === "number") return "slider";
|
|
116
|
+
// file 输入框必须单独一个 role。此前落进 textbox,于是 actions 报
|
|
117
|
+
// [type,setContent,clear,focus] —— agent 会去 type 一个文件选择器,
|
|
118
|
+
// 而真正的动作是 upload(ui.act action=upload file=...)。
|
|
119
|
+
// 实测(tools/_oneoff/role_gap_probe.cjs):input[type=file] 报的是
|
|
120
|
+
// role=textbox actions=["type","setContent","clear","focus"]。
|
|
121
|
+
// 这是"不崩不报错、只是把 agent 引向错误动作"的典型,所以必须有回归钉住。
|
|
122
|
+
if (type === "file") return "fileinput";
|
|
123
|
+
return "textbox";
|
|
124
|
+
}
|
|
125
|
+
if (tag === "textarea") return "textbox";
|
|
126
|
+
if (tag === "select") return "select";
|
|
127
|
+
if (tag === "a" && (el.hasAttribute("href") || typeof el.onclick === "function")) return "link";
|
|
128
|
+
if (tag === "details" || tag === "summary") return "button";
|
|
129
|
+
if (tag === "dialog") return "dialog";
|
|
130
|
+
if (tag === "option") return "option";
|
|
131
|
+
if (tag === "img") return "image";
|
|
132
|
+
if (tag === "canvas") return "canvas";
|
|
133
|
+
var ariaRole = (el.getAttribute("role") || "").toLowerCase();
|
|
134
|
+
if (ariaRole) return ariaRole;
|
|
135
|
+
if (el.contentEditable === "true" || el.getAttribute("contenteditable") === "true") return "textbox";
|
|
136
|
+
if (typeof el.onclick === "function") return "button";
|
|
137
|
+
return "generic";
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
// ARIA 控件角色 → 自述动作。
|
|
141
|
+
//
|
|
142
|
+
// 为什么需要这张表:这些 role 是 W3C 定义的,**可点击是角色语义的一部分**,
|
|
143
|
+
// 不需要任何启发式推断。而此前它们全部落到 default 分支的 inferContainerActions,
|
|
144
|
+
// 只有当元素自己带 cursor:pointer / onclick / data-ai-clickable 时才报 click ——
|
|
145
|
+
// 一个老老实实写 <div role="combobox" tabindex="0"> 的自定义下拉,
|
|
146
|
+
// 在语义树里就是 role=combobox actions=["focus"],listbox/option/menu/menuitem
|
|
147
|
+
// 更是 actions=[]。实测(tools/_oneoff/role_gap_probe.cjs)正是如此。
|
|
148
|
+
//
|
|
149
|
+
// 后果:agent 从树里看不出这些控件可点,只能猜或者放弃。这与下面
|
|
150
|
+
// inferContainerActions 补的是同一类问题(可交互面不可见),只不过那里靠行为信号,
|
|
151
|
+
// 这里靠角色定义 —— 角色是权威来源,优先用它。
|
|
152
|
+
var ARIA_WIDGET_ACTIONS = {
|
|
153
|
+
combobox: ["click", "focus"],
|
|
154
|
+
listbox: ["click", "focus"],
|
|
155
|
+
option: ["click", "focus"],
|
|
156
|
+
menu: ["click", "focus"],
|
|
157
|
+
menubar: ["click", "focus"],
|
|
158
|
+
menuitem: ["click", "focus"],
|
|
159
|
+
menuitemcheckbox: ["click", "focus"],
|
|
160
|
+
menuitemradio: ["click", "focus"],
|
|
161
|
+
tab: ["click", "focus"],
|
|
162
|
+
tablist: ["click", "focus"],
|
|
163
|
+
treeitem: ["click", "focus"],
|
|
164
|
+
gridcell: ["click", "focus"],
|
|
165
|
+
row: ["click", "focus"],
|
|
166
|
+
switch: ["click", "focus"],
|
|
167
|
+
spinbutton: ["type", "focus"],
|
|
168
|
+
searchbox: ["type", "setContent", "clear", "focus"],
|
|
169
|
+
};
|
|
170
|
+
|
|
171
|
+
function extractActions(el, role) {
|
|
172
|
+
switch (role) {
|
|
173
|
+
case "button": case "link": return ["click", "focus", "hover"];
|
|
174
|
+
case "textbox": return ["type", "setContent", "clear", "focus"];
|
|
175
|
+
// 文件输入框:真正的动作是 upload,不是 type。给 focus 是因为
|
|
176
|
+
// 有些站点的自定义上传按钮要先 focus 才渲染。
|
|
177
|
+
case "fileinput": return ["upload", "focus"];
|
|
178
|
+
case "select": return ["select", "focus"];
|
|
179
|
+
case "slider": return ["scroll_to", "focus"];
|
|
180
|
+
case "checkbox": case "radio": return ["click", "focus"];
|
|
181
|
+
// dialog 曾经广告 ["open","close"] —— 但 page_manager.performAction 与
|
|
182
|
+
// preload actions.cjs 都没有这两个 case,调用只会拿到
|
|
183
|
+
// {success:false,error:"Unknown action: open"}。语义树自述了做不到的动作,
|
|
184
|
+
// agent 照着做必然失败(e2e/audit_ws.cjs AU-A3/A4 抓到的)。
|
|
185
|
+
// "打开/关闭一个对话框"也没有通用实现:真正的 <dialog> 有 showModal/close,
|
|
186
|
+
// 而 <div role=dialog> 只能靠点它里面的确定/取消按钮 —— 那是猜。
|
|
187
|
+
// 弹窗里真正能点的按钮本身就带 ["click","focus"],所以这里给空数组不损失能力。
|
|
188
|
+
// 与 inferContainerActions 同一条原则:宁可少报,绝不猜一个假动作。
|
|
189
|
+
case "dialog": return [];
|
|
190
|
+
default:
|
|
191
|
+
if (Object.prototype.hasOwnProperty.call(ARIA_WIDGET_ACTIONS, role)) {
|
|
192
|
+
return ARIA_WIDGET_ACTIONS[role];
|
|
193
|
+
}
|
|
194
|
+
return inferContainerActions(el);
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
// 容器类节点(group / generic / image …)原先一律返回空动作。但**很多真实可交互
|
|
199
|
+
// 元素恰恰落在这里**:画布节点的 <div>、自定义下拉、可拖卡片。只按 role 判定,
|
|
200
|
+
// agent 就完全看不出它们可点/可拖 —— 实测在 React Flow 工作流编辑器上,66 个
|
|
201
|
+
// 画布节点全是 `role:group, actions:[]`,"选中一个节点"这个最基本的操作因此无从
|
|
202
|
+
// 下手(只能靠试)。
|
|
203
|
+
//
|
|
204
|
+
// 这里改用**行为信号**推断,而不是猜类名(类名是各框架私有的,猜不通用):
|
|
205
|
+
// cursor 拽取态 / draggable 属性 → 可拖(并且按下松开即可点选,故同时给 click)
|
|
206
|
+
// cursor:pointer / 真实 onclick → 可点
|
|
207
|
+
// 显式 tabindex>=0 → 可 focus
|
|
208
|
+
// 只补充**已有节点**的动作清单,不新增节点,所以不会撑大语义树体积。
|
|
209
|
+
//
|
|
210
|
+
// 刻意**不给推断节点加 hover**:hover 只有在"悬停会揭示东西"时才有意义,而这一点
|
|
211
|
+
// 从行为信号看不出来,加了就是猜。实测加上后 177 个节点里 172 个都报 hover,
|
|
212
|
+
// 纯噪声、白花 token,却什么都没告诉 agent。
|
|
213
|
+
function inferContainerActions(el) {
|
|
214
|
+
var acts = [];
|
|
215
|
+
try {
|
|
216
|
+
var cursor = "";
|
|
217
|
+
try { cursor = (window.getComputedStyle(el).cursor || "").toLowerCase(); } catch (e) {}
|
|
218
|
+
var draggable =
|
|
219
|
+
(el.getAttribute && el.getAttribute("draggable") === "true") ||
|
|
220
|
+
/^(grab|grabbing|move|col-resize|row-resize|ew-resize|ns-resize|nesw-resize|nwse-resize|crosshair)$/.test(cursor);
|
|
221
|
+
var clickable =
|
|
222
|
+
draggable ||
|
|
223
|
+
cursor === "pointer" ||
|
|
224
|
+
(typeof el.onclick === "function") ||
|
|
225
|
+
(el.hasAttribute && el.hasAttribute("data-ai-clickable"));
|
|
226
|
+
var ti = el.getAttribute ? el.getAttribute("tabindex") : null;
|
|
227
|
+
var focusable = ti !== null && ti !== undefined && Number(ti) >= 0;
|
|
228
|
+
if (clickable) acts.push("click");
|
|
229
|
+
if (draggable) acts.push("drag");
|
|
230
|
+
if (focusable) acts.push("focus");
|
|
231
|
+
} catch (e) { /* 推断失败就不给动作 —— 宁可少报,绝不猜一个假动作 */ }
|
|
232
|
+
return acts;
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
function detectEditorType(el) {
|
|
236
|
+
if (el.contentEditable === "true" || el.getAttribute("contenteditable") === "true") return "contenteditable";
|
|
237
|
+
var cls = (el.className || "").toLowerCase();
|
|
238
|
+
if (cls.indexOf("codemirror") >= 0) return "codemirror";
|
|
239
|
+
if (cls.indexOf("prosemirror") >= 0 || (el.querySelector && el.querySelector("[class*=ProseMirror]"))) return "prosemirror";
|
|
240
|
+
if (cls.indexOf("monaco") >= 0) return "monaco";
|
|
241
|
+
if (cls.indexOf("ql-editor") >= 0) return "quill";
|
|
242
|
+
if (cls.indexOf("draft") >= 0 || (el.querySelector && el.querySelector(".public-DraftEditor-content"))) return "draft";
|
|
243
|
+
// Also check parent for Draft.js (DraftEditor-content is nested)
|
|
244
|
+
if (el.querySelector && el.querySelector("[class*=DraftEditor]")) return "draft";
|
|
245
|
+
if (el.tagName.toLowerCase() === "textarea") return "textarea";
|
|
246
|
+
return null;
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
function extractLabel(el) {
|
|
250
|
+
var ariaLabel = el.getAttribute("aria-label");
|
|
251
|
+
if (ariaLabel) return ariaLabel;
|
|
252
|
+
var title = el.getAttribute("title");
|
|
253
|
+
if (title) return title;
|
|
254
|
+
var text = (el.textContent || "").trim();
|
|
255
|
+
if (text.length > 60) text = text.slice(0, 60);
|
|
256
|
+
if (text) return text;
|
|
257
|
+
var placeholder = el.getAttribute("placeholder");
|
|
258
|
+
if (placeholder) return placeholder;
|
|
259
|
+
return "";
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
function extractBounds(el) {
|
|
263
|
+
try {
|
|
264
|
+
var r = el.getBoundingClientRect();
|
|
265
|
+
return { x: Math.round(r.x), y: Math.round(r.y), width: Math.round(r.width), height: Math.round(r.height) };
|
|
266
|
+
} catch(e) {
|
|
267
|
+
return { x: 0, y: 0, width: 0, height: 0 };
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
// 富文本块级读取:对任何 contenteditable / 富文本输出段落边界,而非把全文
|
|
272
|
+
// 揉成一段(否则换行被吞、段落结构丢失)。纯结构化,不识别任何编辑器框架:
|
|
273
|
+
// 先定位真正的可编辑元素(contenteditable=true,若自身不是则向下找),
|
|
274
|
+
// 再取其顶层块级子元素;无块时按换行拆分。
|
|
275
|
+
function extractEditorBlocks(editor) {
|
|
276
|
+
try {
|
|
277
|
+
if (!editor) return null;
|
|
278
|
+
var isEditable = editor.contentEditable === 'true' ||
|
|
279
|
+
(editor.getAttribute && editor.getAttribute('contenteditable') === 'true');
|
|
280
|
+
var editable = isEditable
|
|
281
|
+
? editor
|
|
282
|
+
: (editor.querySelector ? editor.querySelector('[contenteditable=true]') : null) || editor;
|
|
283
|
+
var one = function(b, i) {
|
|
284
|
+
var t = (b.innerText || (b.textContent ? b.textContent : '') || b.value || '').trim();
|
|
285
|
+
if (!t) return null;
|
|
286
|
+
return { i: i, role: 'paragraph', text: t.slice(0, 500) };
|
|
287
|
+
};
|
|
288
|
+
// 通用:取可编辑元素顶层块级子元素(Draft 的 [data-block] 即 DIV,天然覆盖)
|
|
289
|
+
var kids = Array.prototype.slice.call(editable.children || [])
|
|
290
|
+
.filter(function(c) { return /^(P|H1|H2|H3|H4|H5|H6|LI|PRE|BLOCKQUOTE|DIV)$/.test(c.tagName); });
|
|
291
|
+
var items = kids.map(function(c, i) {
|
|
292
|
+
var t = (c.innerText || (c.textContent ? c.textContent : '') || '').trim();
|
|
293
|
+
if (!t) return null;
|
|
294
|
+
return { i: i, role: c.tagName.toLowerCase(), text: t.slice(0, 500) };
|
|
295
|
+
}).filter(Boolean);
|
|
296
|
+
if (items.length === 0) {
|
|
297
|
+
// 无显式块时按换行拆分(textarea 语义)
|
|
298
|
+
var itx = editable.innerText || (editable.textContent ? editable.textContent : '');
|
|
299
|
+
var tx = itx.split('\n').map(function(s) { return s.trim(); }).filter(Boolean);
|
|
300
|
+
return tx.map(function(s, i) { return { i: i, role: 'paragraph', text: s.slice(0, 500) }; });
|
|
301
|
+
}
|
|
302
|
+
return items;
|
|
303
|
+
} catch(e) { return null; }
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
function extractAttributes(el) {
|
|
307
|
+
var attrs = {};
|
|
308
|
+
try {
|
|
309
|
+
for (var ai = 0; ai < el.attributes.length; ai++) {
|
|
310
|
+
var a = el.attributes[ai];
|
|
311
|
+
if (a.name === "data-ai-id") continue;
|
|
312
|
+
if (a.name.indexOf("data-") === 0 || a.name.indexOf("aria-") === 0 || a.name === "name" || a.name === "id" || a.name === "class") {
|
|
313
|
+
attrs[a.name] = a.value;
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
} catch(e) {}
|
|
317
|
+
return attrs;
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
function process(el) {
|
|
321
|
+
if (seen.has(el)) return null;
|
|
322
|
+
seen.add(el);
|
|
323
|
+
if (!el.isConnected) return null;
|
|
324
|
+
var tag = el.tagName.toLowerCase();
|
|
325
|
+
// 同源 iframe 必须**在 isSemantic 判定之前**处理:iframe 自己不带语义(无 role/无 label),
|
|
326
|
+
// 若放在后面,会被下面"非语义元素只递归子节点"的分支吞掉,iframe 里的内容一个都进不了树。
|
|
327
|
+
// 实测:ComfyUI 编辑器装在同源 iframe 里,里面有 2024 个元素(106 个按钮 / 42 个输入框),
|
|
328
|
+
// 修复前语义树只有 17 个元素、可操作面 6 个 click,agent 根本看不见编辑器。
|
|
329
|
+
if (tag === "iframe") {
|
|
330
|
+
try {
|
|
331
|
+
var ifrDoc = el.contentDocument || (el.contentWindow && el.contentWindow.document);
|
|
332
|
+
if (ifrDoc && ifrDoc.body) {
|
|
333
|
+
var sub = process(ifrDoc.body);
|
|
334
|
+
var subKids = Array.isArray(sub) ? sub : (sub ? [sub] : []);
|
|
335
|
+
return {
|
|
336
|
+
id: "iframe-" + (el.id || counter++),
|
|
337
|
+
role: "iframe_body",
|
|
338
|
+
label: el.title || el.name || "iframe content",
|
|
339
|
+
children: subKids
|
|
340
|
+
};
|
|
341
|
+
}
|
|
342
|
+
} catch (e) { /* cross-origin:读不到就算了,交给后面的兜底 */ }
|
|
343
|
+
}
|
|
344
|
+
if (!isSemantic(el)) {
|
|
345
|
+
var elChildren = el.children;
|
|
346
|
+
if (elChildren && elChildren.length > 0) {
|
|
347
|
+
var results = [];
|
|
348
|
+
for (var ci = 0; ci < elChildren.length; ci++) {
|
|
349
|
+
var childNode = process(elChildren[ci]);
|
|
350
|
+
if (childNode) {
|
|
351
|
+
if (Array.isArray(childNode)) results.push.apply(results, childNode);
|
|
352
|
+
else results.push(childNode);
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
return results.length ? results : null;
|
|
356
|
+
}
|
|
357
|
+
return null;
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
var label = extractLabel(el);
|
|
361
|
+
var role = inferRole(el);
|
|
362
|
+
var actions = extractActions(el, role);
|
|
363
|
+
var bounds = extractBounds(el);
|
|
364
|
+
var attributes = extractAttributes(el);
|
|
365
|
+
var nativeId = el.id || "";
|
|
366
|
+
// editor_type 仅作"解读"元数据提示(agent 可读懂这是什么编辑器),
|
|
367
|
+
// 操作路径已不依赖它——CDP 受信输入对任何框架通用。
|
|
368
|
+
var editorType = detectEditorType(el);
|
|
369
|
+
var editorBlocks = editorType ? extractEditorBlocks(el) : null;
|
|
370
|
+
|
|
371
|
+
// Assign ref ID — use native id, fallback to generated.
|
|
372
|
+
//
|
|
373
|
+
// ai-id 必须跨"读树 → 操作"稳定。agent 的典型用法是先 get_tree 拿 id、再按 id
|
|
374
|
+
// 操作,而页面在这中间自更新(画布增删节点、列表插入一行)是常态。
|
|
375
|
+
// 旧实现用**每次抽取都归零**的 counter 按遍历顺序编号:在最前面插入 1 个元素,
|
|
376
|
+
// 后面**所有**元素都被重新编号,旧 id 就指向了另一个元素 —— 操作照常返回
|
|
377
|
+
// success,却打在了别的东西上(静默错打;实测插入 1 个元素后 4/4 元素 id 全变)。
|
|
378
|
+
// 修法两条:① 元素已有 data-ai-id 就沿用,id 不再随遍历顺序变化;
|
|
379
|
+
// ② 新元素用模块级只增不减的序号,保证生成出来的 id 不重号。
|
|
380
|
+
var existingAiId = el.getAttribute("data-ai-id");
|
|
381
|
+
var aiId = nativeId || existingAiId || "e:" + el.tagName.toLowerCase() + "-" + (AI_ID_SEQ++);
|
|
382
|
+
// ③ 只在值**真的变了**时才写。
|
|
383
|
+
//
|
|
384
|
+
// 实测(tools/_oneoff/mutation_microtest.cjs,真实 Chromium):setAttribute 把属性
|
|
385
|
+
// 设成**完全相同的值**,MutationObserver 照样收到记录({"name":"data-ai-id",
|
|
386
|
+
// "value":"a","old":"a"})。所以无条件写 = **每一次**读取都向页面自己的 observer
|
|
387
|
+
// 灌 N 条属性变更 —— 4000 个语义元素的页面上实测每次 4001 条。
|
|
388
|
+
// 页面里任何监听属性的脚本(埋点、框架插件、站点自己的 observer)都会被惊动,
|
|
389
|
+
// 而我们只是想"读"一下。加个值比较,热读就归零;冷读仍要写,那是 id 稳定性的
|
|
390
|
+
// 代价,不能省。
|
|
391
|
+
if (existingAiId !== aiId) el.setAttribute("data-ai-id", aiId);
|
|
392
|
+
|
|
393
|
+
var states = [];
|
|
394
|
+
if (el.disabled === true) states.push("disabled");
|
|
395
|
+
if (el.readOnly === true) states.push("readonly");
|
|
396
|
+
if (el.checked === true) states.push("checked");
|
|
397
|
+
if (el === document.activeElement) states.push("focused");
|
|
398
|
+
if (!isHidden(el)) states.push("visible");
|
|
399
|
+
|
|
400
|
+
var node = {
|
|
401
|
+
id: aiId,
|
|
402
|
+
role: role,
|
|
403
|
+
label: label,
|
|
404
|
+
states: states.length ? states : undefined,
|
|
405
|
+
actions: actions,
|
|
406
|
+
bounds: bounds
|
|
407
|
+
};
|
|
408
|
+
// Attach current value for input/textarea/contenteditable.
|
|
409
|
+
// 截断走共享契约(src/shared/value_contract.cjs):按 Unicode code point
|
|
410
|
+
// 计数,截断时带 value_truncated / value_full_length / value_fetch_ref,
|
|
411
|
+
// 未截断只给 value(不给普通节点加字段)。敏感控件不泄露值。
|
|
412
|
+
if (tag === "input" || tag === "textarea" || tag === "select") {
|
|
413
|
+
var sensitive = sensitiveInputKind(el);
|
|
414
|
+
if (sensitive) {
|
|
415
|
+
node.value_sensitive = true;
|
|
416
|
+
node.value_sensitive_reason = sensitive;
|
|
417
|
+
} else {
|
|
418
|
+
var v = el.value;
|
|
419
|
+
if (v !== undefined && v !== "" && v !== null) {
|
|
420
|
+
var cut = truncateNodeValue(v, aiId, 200);
|
|
421
|
+
node.value = cut.value;
|
|
422
|
+
if (cut.value_truncated) {
|
|
423
|
+
node.value_truncated = true;
|
|
424
|
+
node.value_full_length = cut.value_full_length;
|
|
425
|
+
node.value_length_unit = cut.value_length_unit;
|
|
426
|
+
node.value_fetch_ref = cut.value_fetch_ref;
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
} else if (el.contentEditable === "true" || el.getAttribute("contenteditable") === "true") {
|
|
431
|
+
var innerText = el.innerText;
|
|
432
|
+
if (innerText && innerText.trim()) {
|
|
433
|
+
var cutCe = truncateNodeValue(innerText, aiId, 200);
|
|
434
|
+
node.value = cutCe.value;
|
|
435
|
+
if (cutCe.value_truncated) {
|
|
436
|
+
node.value_truncated = true;
|
|
437
|
+
node.value_full_length = cutCe.value_full_length;
|
|
438
|
+
node.value_length_unit = cutCe.value_length_unit;
|
|
439
|
+
node.value_fetch_ref = cutCe.value_fetch_ref;
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
}
|
|
443
|
+
if (Object.keys(attributes).length) node.attributes = attributes;
|
|
444
|
+
if (editorType) node.editor_type = editorType;
|
|
445
|
+
if (editorBlocks) node.editor_blocks = editorBlocks;
|
|
446
|
+
|
|
447
|
+
// Children — including iframe content if same-origin
|
|
448
|
+
var elChildren = el.children;
|
|
449
|
+
var childNodes = [];
|
|
450
|
+
|
|
451
|
+
// 同源 iframe 的递归已在 process() 开头完成(必须在 isSemantic 之前),此处不再重复。
|
|
452
|
+
|
|
453
|
+
if (elChildren && elChildren.length) {
|
|
454
|
+
for (var ci = 0; ci < elChildren.length; ci++) {
|
|
455
|
+
var childNode = process(elChildren[ci]);
|
|
456
|
+
if (childNode) {
|
|
457
|
+
if (Array.isArray(childNode)) childNodes.push.apply(childNodes, childNode);
|
|
458
|
+
else childNodes.push(childNode);
|
|
459
|
+
}
|
|
460
|
+
}
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
if (childNodes.length) node.children = childNodes;
|
|
464
|
+
|
|
465
|
+
return node;
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
var root = process(document.body);
|
|
469
|
+
// Fallback: explicitly scan for iframes missed by process() traversal
|
|
470
|
+
var allIframes = document.querySelectorAll("iframe");
|
|
471
|
+
var extraIframes = [];
|
|
472
|
+
for (var ii = 0; ii < allIframes.length; ii++) {
|
|
473
|
+
var ifr = allIframes[ii];
|
|
474
|
+
var iframeBody = null;
|
|
475
|
+
var iframeText = "";
|
|
476
|
+
try {
|
|
477
|
+
var ifrDoc = ifr.contentDocument || ifr.contentWindow.document;
|
|
478
|
+
// 已由 process() 处理过的同源 iframe 直接跳过。否则会再发一个 iframe-text-N /
|
|
479
|
+
// iframe-empty-N 空壳节点,看起来像"这里有个 iframe 但里面是空的",反而误导调用方。
|
|
480
|
+
if (ifrDoc && ifrDoc.body && seen.has(ifrDoc.body)) continue;
|
|
481
|
+
if (ifrDoc && ifrDoc.body) {
|
|
482
|
+
iframeText = (ifrDoc.body.innerText || "").slice(0, 200);
|
|
483
|
+
// Use a fresh sub-process for iframe body
|
|
484
|
+
var cb = function(el) {
|
|
485
|
+
if (!el || seen.has(el)) return null;
|
|
486
|
+
seen.add(el);
|
|
487
|
+
if (!el.isConnected) return null;
|
|
488
|
+
var tg = el.tagName.toLowerCase();
|
|
489
|
+
var lbl = el.getAttribute("aria-label") || el.getAttribute("title") || (el.textContent || "").slice(0, 40) || el.getAttribute("placeholder") || "";
|
|
490
|
+
var rl = tg;
|
|
491
|
+
if (tg === "button") rl = "button";
|
|
492
|
+
else if (tg === "input") { var ty = (el.getAttribute("type") || "").toLowerCase(); rl = (ty === "checkbox") ? "checkbox" : (ty === "radio") ? "radio" : "textbox"; }
|
|
493
|
+
else if (tg === "textarea") rl = "textbox";
|
|
494
|
+
else if (tg === "a") rl = "link";
|
|
495
|
+
else if (tg === "img") rl = "image";
|
|
496
|
+
else if (tg === "iframe") rl = "iframe";
|
|
497
|
+
else if (el.contentEditable === "true" || el.getAttribute("contenteditable") === "true") rl = "textbox";
|
|
498
|
+
var ariaRole = (el.getAttribute("role") || "").toLowerCase();
|
|
499
|
+
if (ariaRole) rl = ariaRole;
|
|
500
|
+
// Skip pure layout divs/spans inside iframe too
|
|
501
|
+
if (tg === "div" || tg === "span") {
|
|
502
|
+
if (!el.hasAttribute("role") && !(el.contentEditable === "true" || el.getAttribute("contenteditable") === "true")) {
|
|
503
|
+
// recurse into children but don't create a node
|
|
504
|
+
var ch = el.children;
|
|
505
|
+
if (ch && ch.length) {
|
|
506
|
+
var results = [];
|
|
507
|
+
for (var ci = 0; ci < ch.length; ci++) { var cn = cb(ch[ci]); if (cn) { if (Array.isArray(cn)) results.push.apply(results, cn); else results.push(cn); } }
|
|
508
|
+
return results.length ? results : null;
|
|
509
|
+
}
|
|
510
|
+
return null;
|
|
511
|
+
}
|
|
512
|
+
}
|
|
513
|
+
var nd = { id: "iframe-el-" + counter++, role: rl, label: lbl };
|
|
514
|
+
var ch2 = el.children;
|
|
515
|
+
if (ch2 && ch2.length) {
|
|
516
|
+
var cc = [];
|
|
517
|
+
for (var ci2 = 0; ci2 < ch2.length; ci2++) { var cn2 = cb(ch2[ci2]); if (cn2) { if (Array.isArray(cn2)) cc.push.apply(cc, cn2); else cc.push(cn2); } }
|
|
518
|
+
if (cc.length) nd.children = cc;
|
|
519
|
+
}
|
|
520
|
+
return nd;
|
|
521
|
+
};
|
|
522
|
+
iframeBody = cb(ifrDoc.body);
|
|
523
|
+
}
|
|
524
|
+
} catch(e) {}
|
|
525
|
+
if (iframeBody) {
|
|
526
|
+
extraIframes.push({ id: "iframe-" + counter++, role: "iframe_body", label: (ifr.title || ifr.name || "iframe content"), children: iframeBody.children || (Array.isArray(iframeBody) ? iframeBody : [iframeBody]), text: iframeText });
|
|
527
|
+
} else if (iframeText) {
|
|
528
|
+
extraIframes.push({ id: "iframe-text-" + counter++, role: "iframe_body", label: ifr.title || ifr.name || "iframe content", text: iframeText });
|
|
529
|
+
} else {
|
|
530
|
+
// At least add the iframe node itself
|
|
531
|
+
extraIframes.push({ id: "iframe-empty-" + counter++, role: "iframe", label: ifr.title || ifr.name || "iframe" });
|
|
532
|
+
}
|
|
533
|
+
}
|
|
534
|
+
if (extraIframes.length && root) {
|
|
535
|
+
if (!root.children) root.children = [];
|
|
536
|
+
root.children.push.apply(root.children, extraIframes);
|
|
537
|
+
}
|
|
538
|
+
if (!root) {
|
|
539
|
+
return { id: "root", role: "generic", label: "", states: ["visible"], actions: [], bounds: { x: 0, y: 0, width: 0, height: 0 } };
|
|
540
|
+
}
|
|
541
|
+
// 当前焦点元素的 data-ai-id,挂根上。ui.get_focused 一直在读这个字段,而
|
|
542
|
+
// extractor 从不产出它 —— 于是那个方法恒返回 null(e2e/audit_ws.cjs AU-B2 抓到)。
|
|
543
|
+
// 焦点信息本来就在每个节点的 states:[...,"focused"] 里,但那是 O(n) 扫树;
|
|
544
|
+
// 挂根上 O(1),且与节点状态同源(都是 document.activeElement)。
|
|
545
|
+
//
|
|
546
|
+
// 三种情况给 null,不猜:焦点在 body/documentElement(没有对应语义节点)、
|
|
547
|
+
// 焦点在iframe 里(activeElement 是 iframe 元素本身,不是里面那个输入框)、
|
|
548
|
+
// 焦点元素不是语义节点(没有 data-ai-id,比如被 isPureLayout 剪掉的容器)。
|
|
549
|
+
var focusedEl = null;
|
|
550
|
+
try { focusedEl = document.activeElement; } catch (e) {}
|
|
551
|
+
var focusedId = null;
|
|
552
|
+
if (focusedEl && focusedEl !== document.body && focusedEl !== document.documentElement &&
|
|
553
|
+
typeof focusedEl.getAttribute === "function") {
|
|
554
|
+
focusedId = focusedEl.getAttribute("data-ai-id") || null;
|
|
555
|
+
}
|
|
556
|
+
root.focused_element_id = focusedId;
|
|
557
|
+
root._debug = { iframe_count: extraIframes.length, version: VERSION_TAG };
|
|
558
|
+
return root;
|
|
559
|
+
}
|
|
560
|
+
|
|
561
|
+
// ============================================================
|
|
562
|
+
// 2. EXTRACT PAGE CONTEXT — modals, forms, iframes, session
|
|
563
|
+
// ============================================================
|
|
564
|
+
|
|
565
|
+
function extractPageContext() {
|
|
566
|
+
// MODALS — unified search across ALL patterns.
|
|
567
|
+
// Visibility rule: do NOT use `el.offsetParent === null` to skip hidden nodes —
|
|
568
|
+
// position:fixed overlays (masks/dialogs/popups) always have offsetParent===null
|
|
569
|
+
// even when fully visible, so that check silently drops every modal. Use a
|
|
570
|
+
// layout-rect + computed-style test that tolerates fixed positioning instead.
|
|
571
|
+
function isShown(el) {
|
|
572
|
+
try {
|
|
573
|
+
if (el.getClientRects().length === 0) return false;
|
|
574
|
+
var cs = window.getComputedStyle(el);
|
|
575
|
+
if (cs.display === "none" || cs.visibility === "hidden") return false;
|
|
576
|
+
var r = el.getBoundingClientRect();
|
|
577
|
+
if (r.width === 0 && r.height === 0) return false;
|
|
578
|
+
} catch (e) { return false; }
|
|
579
|
+
return true;
|
|
580
|
+
}
|
|
581
|
+
var modalSelectors = [
|
|
582
|
+
"[class*=modal]", "[class*=dialog]", "[class*=popup]", "[class*=drawer]", "[class*=overlay]", "[class*=mask]",
|
|
583
|
+
"[role=dialog]", "[role=alertdialog]"
|
|
584
|
+
];
|
|
585
|
+
// 字段级报错选择器。提到函数开头:describeField 在 modal 循环里就要用它,
|
|
586
|
+
// 而顶层扫描在后面 —— var 提升会让它在那时还是 undefined。
|
|
587
|
+
var ERROR_SEL = [
|
|
588
|
+
"[class*=error]", "[class*=err]", "[class*=invalid]", "[class*=fail]",
|
|
589
|
+
"[class*=warning]", "[class*=warn]", "[class*=toast]", "[class*=snackbar]",
|
|
590
|
+
"[role=alert]"
|
|
591
|
+
];
|
|
592
|
+
|
|
593
|
+
// 字段描述的唯一实现。modals[].fields[] 与 forms[].fields[] 曾经各写一份,
|
|
594
|
+
// 结果弹窗字段少了 id —— 弹窗里有多个同类输入框时,agent 无法把读到的
|
|
595
|
+
// error/required 对应到具体元素,只能靠 placeholder 猜。两份实现必然漂移,
|
|
596
|
+
// 所以收成一处,两边形状完全一致。
|
|
597
|
+
//
|
|
598
|
+
// 字段级错误只认两条路,都没有就宁可空着也不猜(猜出来的对应关系比没有更糟):
|
|
599
|
+
// 1) ARIA 承诺:aria-describedby / aria-errormessage 指向。属性值可以是
|
|
600
|
+
// 空格分隔的多个 id,逐个看,而不是整串塞进 getElementById(那样永远
|
|
601
|
+
// 查不到)。
|
|
602
|
+
// 2) 同一父容器内恰好一个输入控件 —— 结构上只可能对应它。
|
|
603
|
+
// 老实现没有第 2 条的数量约束:容器里有 3 个字段、3 条错误时,它取容器内
|
|
604
|
+
// 第一条错误,于是**每个字段都报同一条错误**。这是错报对应关系,比不报更糟。
|
|
605
|
+
function fieldErrorOf(inp) {
|
|
606
|
+
var ids = (inp.getAttribute("aria-describedby") || "") + " " +
|
|
607
|
+
(inp.getAttribute("aria-errormessage") || "");
|
|
608
|
+
var list = ids.split(/\s+/);
|
|
609
|
+
for (var i = 0; i < list.length; i++) {
|
|
610
|
+
if (!list[i]) continue;
|
|
611
|
+
var el = document.getElementById(list[i]);
|
|
612
|
+
if (el && isShown(el)) {
|
|
613
|
+
var t = (el.textContent || "").trim();
|
|
614
|
+
if (t && t.length < 100) return t;
|
|
615
|
+
}
|
|
616
|
+
}
|
|
617
|
+
var p = inp.parentElement;
|
|
618
|
+
if (!p) return null;
|
|
619
|
+
if (p.querySelectorAll("input:not([type=hidden]),textarea,select").length !== 1) return null;
|
|
620
|
+
var errs = p.querySelectorAll(ERROR_SEL.join(","));
|
|
621
|
+
for (var j = 0; j < errs.length; j++) {
|
|
622
|
+
var et = (errs[j].textContent || "").trim();
|
|
623
|
+
if (et && et.length > 0 && et.length < 100) return et;
|
|
624
|
+
}
|
|
625
|
+
return null;
|
|
626
|
+
}
|
|
627
|
+
|
|
628
|
+
function describeField(inp) {
|
|
629
|
+
var entry = {
|
|
630
|
+
id: inp.id || inp.name || "",
|
|
631
|
+
type: inp.type || inp.tagName.toLowerCase(),
|
|
632
|
+
placeholder: (inp.placeholder || "").slice(0, 40),
|
|
633
|
+
// 只读原生 required 会把只用 aria-required 标必填的站点(不少组件库如此)
|
|
634
|
+
// 整片当成选填 —— agent 于是跳过必填项、提交必失败。ARIA 的 "true"/"false"
|
|
635
|
+
// 都按字面认,其它值(含缺失)不认。
|
|
636
|
+
required: isRequired(inp)
|
|
637
|
+
};
|
|
638
|
+
var err = fieldErrorOf(inp);
|
|
639
|
+
if (err) entry.error = err;
|
|
640
|
+
return entry;
|
|
641
|
+
}
|
|
642
|
+
|
|
643
|
+
var modals = [];
|
|
644
|
+
for (var si = 0; si < modalSelectors.length; si++) {
|
|
645
|
+
var els = document.querySelectorAll(modalSelectors[si]);
|
|
646
|
+
for (var ei = 0; ei < els.length; ei++) {
|
|
647
|
+
var m = els[ei];
|
|
648
|
+
if (!isShown(m)) continue;
|
|
649
|
+
// Buttons may be real <button> or ARIA/JS-styled fakes (div/span[role=button]).
|
|
650
|
+
var btns = m.querySelectorAll("button,[role=button],input[type=submit],input[type=button]");
|
|
651
|
+
var inps = m.querySelectorAll("input:not([type=hidden]),textarea,select");
|
|
652
|
+
if (btns.length === 0 && inps.length === 0) continue;
|
|
653
|
+
|
|
654
|
+
var btnList = [];
|
|
655
|
+
for (var bi = 0; bi < btns.length; bi++) {
|
|
656
|
+
var t = (btns[bi].textContent || btns[bi].value || "").trim();
|
|
657
|
+
if (!t) continue;
|
|
658
|
+
btnList.push({ text: t.slice(0, 30), disabled: btns[bi].disabled || false });
|
|
659
|
+
}
|
|
660
|
+
|
|
661
|
+
var fieldList = [];
|
|
662
|
+
for (var fi = 0; fi < inps.length; fi++) fieldList.push(describeField(inps[fi]));
|
|
663
|
+
|
|
664
|
+
// Errors — 14 selectors + key phrase scan
|
|
665
|
+
var errSel = ["[class*=error]","[class*=err]","[class*=invalid]","[class*=toast]","[class*=message]",
|
|
666
|
+
"[class*=notification]","[class*=alert]","[class*=warning]","[class*=fail]","[class*=tip]",
|
|
667
|
+
"[role=alert]","[role=status]","[class*=notice]","[class*=snackbar]"];
|
|
668
|
+
var errors = [];
|
|
669
|
+
for (var esi = 0; esi < errSel.length; esi++) {
|
|
670
|
+
var errEls = m.querySelectorAll(errSel[esi]);
|
|
671
|
+
for (var eei = 0; eei < errEls.length; eei++) {
|
|
672
|
+
var et = (errEls[eei].textContent || "").trim();
|
|
673
|
+
if (et && et.length < 200 && errors.indexOf(et) < 0) errors.push(et);
|
|
674
|
+
}
|
|
675
|
+
}
|
|
676
|
+
// Key phrase scan
|
|
677
|
+
var mText = m.innerText || "";
|
|
678
|
+
var phrases = ["不存在","失败","错误","不能为空","请选择","请填写","not found","failed","error","required","cannot be empty","博主","验证","字数"];
|
|
679
|
+
for (var pi = 0; pi < phrases.length; pi++) {
|
|
680
|
+
var idx = mText.indexOf(phrases[pi]);
|
|
681
|
+
if (idx >= 0) {
|
|
682
|
+
var snippet = mText.slice(Math.max(0, idx - 10), idx + phrases[pi].length + 40);
|
|
683
|
+
if (errors.indexOf(snippet) < 0) errors.push(snippet);
|
|
684
|
+
}
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
// 必填提示不在这里扫。这些扫描原先全写在 modal 循环里,于是"红星能否被
|
|
688
|
+
// 读到"取决于它所在容器的 class 里有没有 modal/dialog/popup/drawer/
|
|
689
|
+
// overlay/mask 关键词 —— 普通页面表单里的红星,context.modals[].required_hints
|
|
690
|
+
// 读不到,顶层压根没有这个字段,forms[].fields 里也没有。报错扫描早已改成
|
|
691
|
+
// 整文档 + 顶层,必填扫描只是漏迁;现在统一扫整份文档、落顶层
|
|
692
|
+
// context.required_hints(见本函数下方的 scanRequiredHints)。
|
|
693
|
+
modals.push({
|
|
694
|
+
buttons: btnList,
|
|
695
|
+
fields: fieldList,
|
|
696
|
+
errors: errors.length ? errors : null
|
|
697
|
+
});
|
|
698
|
+
}
|
|
699
|
+
}
|
|
700
|
+
|
|
701
|
+
// FORMS
|
|
702
|
+
var forms = document.querySelectorAll("form");
|
|
703
|
+
var formList = [];
|
|
704
|
+
for (var fi = 0; fi < forms.length; fi++) {
|
|
705
|
+
var f = forms[fi];
|
|
706
|
+
if (!isShown(f)) continue;
|
|
707
|
+
var inps = f.querySelectorAll("input:not([type=hidden]),textarea,select");
|
|
708
|
+
var fields = [];
|
|
709
|
+
for (var ii = 0; ii < inps.length; ii++) fields.push(describeField(inps[ii]));
|
|
710
|
+
if (fields.length) formList.push({ fields: fields });
|
|
711
|
+
}
|
|
712
|
+
|
|
713
|
+
// 必填判定:原生 required 或 aria-required="true"。站点/组件库常只用 ARIA 标必填
|
|
714
|
+
// (原生 required 会触发浏览器自带的气泡校验,很多站点刻意绕开它)。
|
|
715
|
+
function isRequired(el) {
|
|
716
|
+
if (el.required === true) return true;
|
|
717
|
+
var ar = el.getAttribute ? el.getAttribute("aria-required") : null;
|
|
718
|
+
return ar === "true";
|
|
719
|
+
}
|
|
720
|
+
|
|
721
|
+
// ---- 顶层错误 / 提示扫描(P0-2 / P1-6)----
|
|
722
|
+
// context.modals[] 的 errSel 扫描只在 class 含 modal/dialog/popup/drawer/
|
|
723
|
+
// overlay/mask 的容器内部生效,而"页面即表单"的站点根本没有这类容器 ——
|
|
724
|
+
// 字段级小红字(无 role 的 <span class="field-error">)、表单级汇总、toast、
|
|
725
|
+
// 服务端报错统统读不到:agent 提交了表单却看不到哪里错了。这里把同类扫描
|
|
726
|
+
// 扩到整个 document,并落到**顶层** context。
|
|
727
|
+
//
|
|
728
|
+
// 刻意**不**把这些元素提升为语义树节点:站点用裸 span/div 承载提示文案是常态,
|
|
729
|
+
// isPureLayout 剪掉它们正是为了压住语义树的 token 体积(真实页面上量极大)。
|
|
730
|
+
// 只读文本、只进 context —— 主树一个节点都不加(实测节点数与改动前完全一致)。
|
|
731
|
+
// ERROR_SEL 已在本函数开头声明(字段描述要用它,见 fieldErrorOf)。
|
|
732
|
+
var HINT_SEL = [
|
|
733
|
+
"[class*=tip]", "[class*=hint]", "[class*=notice]", "[class*=help]",
|
|
734
|
+
"[class*=counter]", "[class*=remaining]", "[role=status]", "[aria-live]"
|
|
735
|
+
];
|
|
736
|
+
// 字数/计数类文案("0/256")常被塞进连 class 都没有的裸 span —— 无 class 可依,
|
|
737
|
+
// 只能认文本形态。限定"叶子元素 + 文本很短"来降噪,避免整页文本被收进来。
|
|
738
|
+
var LIMIT_RE = /^[^\d]{0,10}\d+\s*\/\s*\d+[^\d]{0,10}$/;
|
|
739
|
+
|
|
740
|
+
function textOf(el) {
|
|
741
|
+
if (!isShown(el)) return "";
|
|
742
|
+
var t = (el.textContent || "").trim();
|
|
743
|
+
if (!t || t.length > 120) return "";
|
|
744
|
+
return t;
|
|
745
|
+
}
|
|
746
|
+
// 报错对应到具体字段。两条可靠路径,都没有就宁可空着也不猜:
|
|
747
|
+
// 1) 标准 ARIA 关联:哪个输入用 aria-describedby / aria-errormessage 指到这个
|
|
748
|
+
// 报错元素(mTitle[aria-describedby=mErr] → mErr 的 field 就是 mTitle)。
|
|
749
|
+
// 2) 同一容器内恰好一个输入控件(#tagWrap 里的 tagDup → tagInput)。
|
|
750
|
+
// 刻意不做"找最近的 input"这类位置猜测:报错常集中渲染在表单底部/顶部,
|
|
751
|
+
// 离它最近的 input 往往不是它对应的那个字段,猜出来的对应关系比没有更糟。
|
|
752
|
+
function fieldOf(el) {
|
|
753
|
+
var eid = el.id;
|
|
754
|
+
if (eid) {
|
|
755
|
+
try {
|
|
756
|
+
var ref = document.querySelector(
|
|
757
|
+
'[aria-errormessage~="' + eid + '"],[aria-describedby~="' + eid + '"]');
|
|
758
|
+
if (ref) return ref.id || ref.name || "";
|
|
759
|
+
} catch (e) {}
|
|
760
|
+
}
|
|
761
|
+
var p = el.parentElement;
|
|
762
|
+
if (!p) return "";
|
|
763
|
+
var inps = p.querySelectorAll("input:not([type=hidden]),textarea,select");
|
|
764
|
+
if (inps.length !== 1) return "";
|
|
765
|
+
return inps[0].id || inps[0].name || "";
|
|
766
|
+
}
|
|
767
|
+
function scanSelectors(sels) {
|
|
768
|
+
var out = [];
|
|
769
|
+
for (var si = 0; si < sels.length; si++) {
|
|
770
|
+
var els;
|
|
771
|
+
try { els = document.querySelectorAll(sels[si]); } catch (e) { continue; }
|
|
772
|
+
for (var ei = 0; ei < els.length && out.length < 40; ei++) {
|
|
773
|
+
var t = textOf(els[ei]);
|
|
774
|
+
if (t && out.indexOf(t) < 0) out.push(t);
|
|
775
|
+
}
|
|
776
|
+
}
|
|
777
|
+
return out;
|
|
778
|
+
}
|
|
779
|
+
|
|
780
|
+
var topErrors = scanSelectors(ERROR_SEL);
|
|
781
|
+
var fieldErrors = [];
|
|
782
|
+
var seenFieldErr = {};
|
|
783
|
+
for (var fes = 0; fes < ERROR_SEL.length && fieldErrors.length < 40; fes++) {
|
|
784
|
+
var feEls;
|
|
785
|
+
try { feEls = document.querySelectorAll(ERROR_SEL[fes]); } catch (e) { continue; }
|
|
786
|
+
for (var fei = 0; fei < feEls.length && fieldErrors.length < 40; fei++) {
|
|
787
|
+
var fet = textOf(feEls[fei]);
|
|
788
|
+
if (!fet) continue;
|
|
789
|
+
var fid = fieldOf(feEls[fei]);
|
|
790
|
+
if (!fid) continue;
|
|
791
|
+
var fkey = fid + "\u0000" + fet;
|
|
792
|
+
if (seenFieldErr[fkey]) continue;
|
|
793
|
+
seenFieldErr[fkey] = 1;
|
|
794
|
+
fieldErrors.push({ field: fid, text: fet });
|
|
795
|
+
}
|
|
796
|
+
}
|
|
797
|
+
var topHints = scanSelectors(HINT_SEL);
|
|
798
|
+
try {
|
|
799
|
+
var cand = document.querySelectorAll("span,div,p,small,em,i,label,li,strong,b");
|
|
800
|
+
for (var ci2 = 0; ci2 < cand.length && topHints.length < 40; ci2++) {
|
|
801
|
+
if (cand[ci2].children && cand[ci2].children.length) continue; // 只认叶子,避免整页文本
|
|
802
|
+
var ht = textOf(cand[ci2]);
|
|
803
|
+
if (!ht || !LIMIT_RE.test(ht)) continue;
|
|
804
|
+
if (topHints.indexOf(ht) < 0) topHints.push(ht);
|
|
805
|
+
}
|
|
806
|
+
} catch (e) {}
|
|
807
|
+
// 上面两条都要求"有 class 提示"或"文本形状是 N/M"。但站点普遍用裸 span/div 放
|
|
808
|
+
// 这一组的说明文案(格式要求、隐私承诺、快捷提示……),既没有 class 可依,形状也
|
|
809
|
+
// 未必是计数 —— 于是语义树里完全看不见:isPureLayout 剪掉裸 div/span 是为了压
|
|
810
|
+
// token 体积,那个取舍本身没错,但代价是 agent 只能靠 ui.evaluate 兜底才能读到。
|
|
811
|
+
//
|
|
812
|
+
// 收窄条件:只收"和表单控件待在同一个紧凑容器里"的短叶子文本。同一个父容器是
|
|
813
|
+
// **结构关系**,不是"离得近"这种位置猜测(项目对位置猜测有明确禁令);再要求
|
|
814
|
+
// 容器元素数很少,避免顺着 body 把整页文本灌进来。字段自己的 label 文案跳过 ——
|
|
815
|
+
// 那部分语义树已经有了,重复会挤占名额。仍然只进 context,主树一个节点都不加。
|
|
816
|
+
try {
|
|
817
|
+
var ctrls = document.querySelectorAll("input:not([type=hidden]),textarea,select");
|
|
818
|
+
for (var ci3 = 0; ci3 < ctrls.length && topHints.length < 60; ci3++) {
|
|
819
|
+
var fbox = ctrls[ci3].parentElement;
|
|
820
|
+
if (!fbox || fbox === document.body || fbox === document.documentElement) continue;
|
|
821
|
+
if (fbox.querySelectorAll("*").length > 24) continue; // 只认紧凑的字段组
|
|
822
|
+
var labs = fbox.querySelectorAll("label");
|
|
823
|
+
var labTexts = [];
|
|
824
|
+
for (var la = 0; la < labs.length; la++) labTexts.push((labs[la].textContent || "").trim());
|
|
825
|
+
var fleaves = fbox.querySelectorAll("span,div,small,em,i,b,strong,p,li");
|
|
826
|
+
for (var fli = 0; fli < fleaves.length && topHints.length < 60; fli++) {
|
|
827
|
+
if (fleaves[fli].children && fleaves[fli].children.length) continue;
|
|
828
|
+
var fht = textOf(fleaves[fli]);
|
|
829
|
+
if (!fht || fht.length < 2 || fht.length > 60) continue;
|
|
830
|
+
if (!/[一-龥A-Za-z0-9]/.test(fht)) continue; // 纯符号(*、:、—)没有信息量
|
|
831
|
+
if (labTexts.indexOf(fht) >= 0) continue; // 字段自己的文案,树里已经有了
|
|
832
|
+
if (topHints.indexOf(fht) < 0) topHints.push(fht);
|
|
833
|
+
}
|
|
834
|
+
}
|
|
835
|
+
} catch (e) {}
|
|
836
|
+
|
|
837
|
+
// ---- 必填提示(整文档,落顶层)----
|
|
838
|
+
// 这些扫描原先全部写在 modal 循环里,于是"红星能否被读到"取决于它所在容器的 class
|
|
839
|
+
// 里有没有 modal/dialog/popup/drawer/overlay/mask 关键词。报错扫描早已改成整文档 +
|
|
840
|
+
// 顶层,必填扫描只是漏迁;这里应用同一范式,弹窗里那份删掉(它是前者的子集,留着
|
|
841
|
+
// 就是两份会漂移的实现)。
|
|
842
|
+
//
|
|
843
|
+
// 这是**启发式补充**,不是权威判定:字段到底必填不必填看 fields[].required(原生
|
|
844
|
+
// required / aria-required,那是承诺)。这里抓的是"只用红星标注、什么都不写"的站点。
|
|
845
|
+
//
|
|
846
|
+
// 刻意不收字数上限 N/M 和「字数」提示:那不是必填标记,而且 hints 已经收过了
|
|
847
|
+
// (形状扫描 + 紧凑容器扫描两条路)。老代码在 modal 容器里顺带收了它们,搬到整文档
|
|
848
|
+
// 之后这个顺手就变成了噪声 —— 实测编辑器页(一个表单控件都没有)会报出
|
|
849
|
+
// ["字数统计测试","字数 6/256","6/256"],agent 会以为这页有必填项。按字段名办事。
|
|
850
|
+
function scanRequiredHints() {
|
|
851
|
+
var out = [];
|
|
852
|
+
var push = function (t) {
|
|
853
|
+
t = (t || "").trim();
|
|
854
|
+
// 纯符号(*、*、:)没有信息量:红星本身不说明"哪个字段必填",说明它的是
|
|
855
|
+
// 它所在的那行文案(由下面的信号 4 收)。老代码把 .req-star 的 textContent
|
|
856
|
+
// 原样推进来,于是数组里永远躺一个孤零零的 "*"。
|
|
857
|
+
if (!/[一-龥A-Za-z0-9]/.test(t)) return;
|
|
858
|
+
if (t.length < 50 && out.indexOf(t) < 0 && out.length < 40) out.push(t);
|
|
859
|
+
};
|
|
860
|
+
// 1) CSS 必填标记
|
|
861
|
+
var reqEls = document.querySelectorAll("[class*=required],[class*=req],[class*=mandatory],[class*=asterisk]");
|
|
862
|
+
for (var rq = 0; rq < reqEls.length; rq++) push(reqEls[rq].textContent);
|
|
863
|
+
// 2) 中文提示("请填写""请选择"——这是真的在要求 agent 填东西)
|
|
864
|
+
var rhm = (document.body ? (document.body.innerText || "") : "").match(/请\s*[选择填写输入].{0,20}/g);
|
|
865
|
+
if (rhm) for (var rh = 0; rh < rhm.length; rh++) push(rhm[rh]);
|
|
866
|
+
// 3) 红星字符(含全角)→ 收它所在那一行的文案,红星本身没有信息量
|
|
867
|
+
var rstars = document.querySelectorAll("span,em,i,label,sup");
|
|
868
|
+
for (var rs = 0; rs < rstars.length; rs++) {
|
|
869
|
+
var rst = (rstars[rs].textContent || "").trim();
|
|
870
|
+
if (rst === "*" || rst === "*" || rst === "✱") {
|
|
871
|
+
var rsp = rstars[rs].parentElement;
|
|
872
|
+
if (rsp) push((rsp.textContent || "").slice(0, 30).replace(/\*/g, ""));
|
|
873
|
+
}
|
|
874
|
+
}
|
|
875
|
+
// 5) ::before 伪元素红星。getComputedStyle 不便宜,所以:候选集按标签收窄
|
|
876
|
+
// (红星伪元素只可能落在这些标签上),宿主必须带短文本,且按**实际发起的
|
|
877
|
+
// 样式查询次数**设上限。老代码按"找到了几个"计数,等于没有上限。
|
|
878
|
+
var rcand = document.querySelectorAll("label,span,div,p,em,i,sup,b,strong,small,td,th");
|
|
879
|
+
var rchecked = 0;
|
|
880
|
+
for (var ra = 0; ra < rcand.length && rchecked < 200 && out.length < 40; ra++) {
|
|
881
|
+
var rel = rcand[ra];
|
|
882
|
+
var rown = (rel.textContent || "").trim();
|
|
883
|
+
if (!rown || rown.length > 30) continue;
|
|
884
|
+
rchecked++;
|
|
885
|
+
try {
|
|
886
|
+
var rbefore = window.getComputedStyle(rel, "::before");
|
|
887
|
+
var rcontent = rbefore.getPropertyValue("content");
|
|
888
|
+
if (!rcontent || rcontent === "none" || rcontent === "normal" ||
|
|
889
|
+
rcontent === '""' || rcontent === "''") continue;
|
|
890
|
+
var rcolor = rbefore.getPropertyValue("color");
|
|
891
|
+
if (rcolor && (rcolor.indexOf("rgb(255") >= 0 || rcolor.indexOf("red") >= 0 ||
|
|
892
|
+
rcolor.indexOf("#f") >= 0 || rcolor.indexOf("#F") >= 0 || rcolor.indexOf("#e") >= 0)) {
|
|
893
|
+
push(rown);
|
|
894
|
+
}
|
|
895
|
+
} catch (e) {}
|
|
896
|
+
}
|
|
897
|
+
return out;
|
|
898
|
+
}
|
|
899
|
+
var requiredHints = scanRequiredHints();
|
|
900
|
+
|
|
901
|
+
// SESSION — more robust detection
|
|
902
|
+
var hasPwdInput = document.querySelector("input[type=password]");
|
|
903
|
+
var bodyText = document.body.innerText || "";
|
|
904
|
+
var isLoginPage = bodyText.indexOf("登录") >= 0 || bodyText.indexOf("Sign In") >= 0 ||
|
|
905
|
+
document.title.indexOf("登录") >= 0 || document.URL.indexOf("login") >= 0 || document.URL.indexOf("signin") >= 0;
|
|
906
|
+
var hasAvatar = document.querySelector("[class*=avatar],img[class*=avatar],img[class*=user]");
|
|
907
|
+
// Cookie-based check
|
|
908
|
+
var hasLoginCookie = document.cookie.indexOf("token") >= 0 || document.cookie.indexOf("session") >= 0
|
|
909
|
+
|| document.cookie.indexOf("auth") >= 0;
|
|
910
|
+
var loggedIn = !hasPwdInput && !isLoginPage && (hasAvatar !== null || hasLoginCookie || bodyText.length > 1000);
|
|
911
|
+
|
|
912
|
+
return {
|
|
913
|
+
title: document.title,
|
|
914
|
+
url: document.URL,
|
|
915
|
+
forms: formList.length ? formList : null,
|
|
916
|
+
modals: modals.length ? modals : null,
|
|
917
|
+
// 顶层报错/提示/必填:都不依赖容器 class 是否含 modal 关键词,普通页面也能读到。
|
|
918
|
+
// errors —— 全文档扫描到的可见错误文案;field_errors —— 能对应到具体输入
|
|
919
|
+
// 框的那些({field, text});hints —— 字数/提示/轻通知类辅助文案;
|
|
920
|
+
// required_hints —— 必填标记(红星/CSS 标记/「请填写」类文案),启发式补充,
|
|
921
|
+
// 权威判定看 fields[].required。
|
|
922
|
+
errors: topErrors.length ? topErrors : null,
|
|
923
|
+
field_errors: fieldErrors.length ? fieldErrors : null,
|
|
924
|
+
hints: topHints.length ? topHints : null,
|
|
925
|
+
required_hints: requiredHints.length ? requiredHints : null,
|
|
926
|
+
session: { logged_in: loggedIn },
|
|
927
|
+
stats: {
|
|
928
|
+
inputs: document.querySelectorAll("input:not([type=hidden])").length,
|
|
929
|
+
buttons: document.querySelectorAll("button").length,
|
|
930
|
+
links: document.querySelectorAll("a[href]").length,
|
|
931
|
+
iframes: document.querySelectorAll("iframe").length
|
|
932
|
+
}
|
|
933
|
+
};
|
|
934
|
+
}
|
|
935
|
+
|
|
936
|
+
module.exports = { extractTree, extractPageContext };
|