@yuzc-001/grasp 0.6.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +327 -0
- package/README.zh-CN.md +324 -0
- package/examples/README.md +31 -0
- package/examples/claude-desktop.json +8 -0
- package/examples/codex-config.toml +4 -0
- package/grasp.skill +0 -0
- package/index.js +87 -0
- package/package.json +48 -0
- package/scripts/grasp_openclaw_ctl.sh +122 -0
- package/scripts/run-search-benchmark.mjs +287 -0
- package/scripts/update-star-history.mjs +274 -0
- package/skill/SKILL.md +61 -0
- package/skill/references/tools.md +306 -0
- package/src/cli/auto-configure.js +116 -0
- package/src/cli/cmd-connect.js +148 -0
- package/src/cli/cmd-explain.js +42 -0
- package/src/cli/cmd-logs.js +55 -0
- package/src/cli/cmd-status.js +119 -0
- package/src/cli/config.js +27 -0
- package/src/cli/detect-chrome.js +58 -0
- package/src/grasp/handoff/events.js +67 -0
- package/src/grasp/handoff/persist.js +48 -0
- package/src/grasp/handoff/state.js +28 -0
- package/src/grasp/page/capture.js +34 -0
- package/src/grasp/page/state.js +273 -0
- package/src/grasp/verify/evidence.js +40 -0
- package/src/grasp/verify/pipeline.js +52 -0
- package/src/layer1-bridge/chrome.js +416 -0
- package/src/layer1-bridge/webmcp.js +143 -0
- package/src/layer2-perception/hints.js +284 -0
- package/src/layer3-action/actions.js +400 -0
- package/src/runtime/browser-instance.js +65 -0
- package/src/runtime/truth/model.js +94 -0
- package/src/runtime/truth/snapshot.js +51 -0
- package/src/server/affordances.js +47 -0
- package/src/server/audit.js +122 -0
- package/src/server/boss-fast-path.js +164 -0
- package/src/server/boundary-guard.js +53 -0
- package/src/server/content.js +97 -0
- package/src/server/continuity.js +256 -0
- package/src/server/engine-selection.js +29 -0
- package/src/server/entry-orchestrator.js +115 -0
- package/src/server/error-codes.js +7 -0
- package/src/server/explain-share-card.js +113 -0
- package/src/server/fast-path-router.js +134 -0
- package/src/server/form-runtime.js +602 -0
- package/src/server/form-tasks.js +254 -0
- package/src/server/gateway-response.js +62 -0
- package/src/server/index.js +22 -0
- package/src/server/observe.js +52 -0
- package/src/server/page-projection.js +31 -0
- package/src/server/page-state.js +27 -0
- package/src/server/postconditions.js +128 -0
- package/src/server/prompt-assembly.js +148 -0
- package/src/server/responses.js +44 -0
- package/src/server/route-boundary.js +174 -0
- package/src/server/route-policy.js +168 -0
- package/src/server/runtime-confirmation.js +87 -0
- package/src/server/runtime-status.js +7 -0
- package/src/server/share-artifacts.js +284 -0
- package/src/server/state.js +132 -0
- package/src/server/structured-extraction.js +131 -0
- package/src/server/surface-prompts.js +166 -0
- package/src/server/task-frame.js +11 -0
- package/src/server/tasks/search-task.js +321 -0
- package/src/server/tools.actions.js +1361 -0
- package/src/server/tools.form.js +526 -0
- package/src/server/tools.gateway.js +757 -0
- package/src/server/tools.handoff.js +210 -0
- package/src/server/tools.js +20 -0
- package/src/server/tools.legacy.js +983 -0
- package/src/server/tools.strategy.js +250 -0
- package/src/server/tools.task-surface.js +66 -0
- package/src/server/tools.workspace.js +873 -0
- package/src/server/workspace-runtime.js +1138 -0
- package/src/server/workspace-tasks.js +735 -0
- package/start-chrome.bat +84 -0
|
@@ -0,0 +1,284 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Layer 2 - 感知层:hints.js
|
|
3
|
+
* 将当前视口内可交互元素转化为极简语义地图(HintMap)
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* 在浏览器页面中采集可交互元素,构建 HintMap。
|
|
8
|
+
* registry 和 counters 由调用方持有,跨调用复用,保证同一元素 ID 稳定。
|
|
9
|
+
* @param {import('playwright').Page} page
|
|
10
|
+
* @param {Map<string, string>} registry fingerprint → id 映射表
|
|
11
|
+
* @param {{ B: number, I: number, L: number, S: number }} counters 各前缀已用计数
|
|
12
|
+
* @returns {Promise<Array<{id: string, type: string, label: string, x: number, y: number, meta: Record<string, string> }>>}
|
|
13
|
+
*/
|
|
14
|
+
export async function buildHintMap(page, registry = new Map(), counters = { B: 0, I: 0, L: 0, S: 0 }) {
|
|
15
|
+
// 将 registry 序列化传入 browser context(Map 不可直接传)
|
|
16
|
+
const registryEntries = [...registry.entries()];
|
|
17
|
+
|
|
18
|
+
const result = await page.evaluate(({ registryEntries, counters }) => {
|
|
19
|
+
const reg = new Map(registryEntries);
|
|
20
|
+
const INTERACTIVE_TAGS = new Set(['button', 'a', 'input', 'select', 'textarea']);
|
|
21
|
+
const INTERACTIVE_ROLES = new Set([
|
|
22
|
+
'button', 'link', 'textbox', 'searchbox', 'combobox',
|
|
23
|
+
'checkbox', 'radio', 'menuitem', 'tab', 'option',
|
|
24
|
+
'slider', 'spinbutton', 'switch',
|
|
25
|
+
]);
|
|
26
|
+
|
|
27
|
+
const vw = window.innerWidth;
|
|
28
|
+
const vh = window.innerHeight;
|
|
29
|
+
|
|
30
|
+
function getCandidatePriority(el, tag, role, isContentEditable) {
|
|
31
|
+
const inputType = (el.getAttribute('type') || '').toLowerCase();
|
|
32
|
+
if (isContentEditable || role === 'textbox' || role === 'searchbox' || role === 'combobox') return 5;
|
|
33
|
+
if (tag === 'textarea') return 4;
|
|
34
|
+
if (tag === 'input') {
|
|
35
|
+
if (['text', 'search', 'email', 'url', 'password', 'number', 'tel', ''].includes(inputType)) return 4;
|
|
36
|
+
if (['file', 'hidden'].includes(inputType)) return 1;
|
|
37
|
+
return 3;
|
|
38
|
+
}
|
|
39
|
+
if (tag === 'button' || role === 'button') return 2;
|
|
40
|
+
if (tag === 'a' || role === 'link') return 1;
|
|
41
|
+
return 0;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function isSelected(el) {
|
|
45
|
+
const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase();
|
|
46
|
+
const classAttr = String(el.getAttribute('class') || '');
|
|
47
|
+
const hasStateClass = classAttr
|
|
48
|
+
.split(/\s+/)
|
|
49
|
+
.some((token) => /(^|[-_])(selected|current)($|[-_])/i.test(token));
|
|
50
|
+
return el.getAttribute('aria-selected') === 'true'
|
|
51
|
+
|| el.getAttribute('data-selected') === 'true'
|
|
52
|
+
|| ['page', 'step', 'location', 'date', 'time', 'true'].includes(ariaCurrent)
|
|
53
|
+
|| hasStateClass
|
|
54
|
+
|| el.classList.contains('selected')
|
|
55
|
+
|| el.classList.contains('is-selected')
|
|
56
|
+
|| el.classList.contains('workspace-item--selected');
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
// 1. 遍历 document.body 下所有元素
|
|
60
|
+
const walker = document.createTreeWalker(
|
|
61
|
+
document.body,
|
|
62
|
+
NodeFilter.SHOW_ELEMENT,
|
|
63
|
+
null,
|
|
64
|
+
);
|
|
65
|
+
|
|
66
|
+
const candidates = [];
|
|
67
|
+
let node;
|
|
68
|
+
while ((node = walker.nextNode())) {
|
|
69
|
+
const el = /** @type {HTMLElement} */ (node);
|
|
70
|
+
const tag = el.tagName.toLowerCase();
|
|
71
|
+
const role = (el.getAttribute('role') || '').toLowerCase();
|
|
72
|
+
|
|
73
|
+
// 2. 只处理 INTERACTIVE_TAGS 或 INTERACTIVE_ROLES 或 contenteditable
|
|
74
|
+
const isContentEditable = el.getAttribute('contenteditable') === 'true' ||
|
|
75
|
+
el.getAttribute('contenteditable') === '';
|
|
76
|
+
if (!INTERACTIVE_TAGS.has(tag) && !INTERACTIVE_ROLES.has(role) && !isContentEditable) continue;
|
|
77
|
+
|
|
78
|
+
// 3. 过滤不可见元素
|
|
79
|
+
const rect = el.getBoundingClientRect();
|
|
80
|
+
if (rect.width === 0 || rect.height === 0) continue;
|
|
81
|
+
if (rect.bottom < 0 || rect.top > vh || rect.right < 0 || rect.left > vw) continue;
|
|
82
|
+
|
|
83
|
+
const style = window.getComputedStyle(el);
|
|
84
|
+
if (
|
|
85
|
+
style.visibility === 'hidden' ||
|
|
86
|
+
style.display === 'none' ||
|
|
87
|
+
style.opacity === '0'
|
|
88
|
+
) continue;
|
|
89
|
+
|
|
90
|
+
const cx = Math.round(rect.left + rect.width / 2);
|
|
91
|
+
const cy = Math.round(rect.top + rect.height / 2);
|
|
92
|
+
|
|
93
|
+
candidates.push({
|
|
94
|
+
el,
|
|
95
|
+
tag,
|
|
96
|
+
role,
|
|
97
|
+
cx,
|
|
98
|
+
cy,
|
|
99
|
+
priority: getCandidatePriority(el, tag, role, isContentEditable),
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// 4. 排序:y 优先,x 其次
|
|
104
|
+
candidates.sort((a, b) => {
|
|
105
|
+
if (a.cy !== b.cy) return a.cy - b.cy;
|
|
106
|
+
if (a.cx !== b.cx) return a.cx - b.cx;
|
|
107
|
+
return b.priority - a.priority;
|
|
108
|
+
});
|
|
109
|
+
|
|
110
|
+
// 5. 去重:中心坐标差 < 5px 视为同一交互点,保留更可操作的候选
|
|
111
|
+
const filtered = [];
|
|
112
|
+
for (const c of candidates) {
|
|
113
|
+
const dupIndex = filtered.findIndex((s) => Math.abs(s.cx - c.cx) < 5 && Math.abs(s.cy - c.cy) < 5);
|
|
114
|
+
if (dupIndex === -1) {
|
|
115
|
+
filtered.push(c);
|
|
116
|
+
continue;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
const previous = filtered[dupIndex];
|
|
120
|
+
if (c.priority > previous.priority) {
|
|
121
|
+
filtered[dupIndex] = c;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
// 6. 分配 ID 并构建 Hint 列表
|
|
126
|
+
|
|
127
|
+
/**
|
|
128
|
+
* 根据元素信息确定 ID 前缀。
|
|
129
|
+
* button → B, input/textarea/textbox/searchbox/combobox → I, link → L, 其他 → S
|
|
130
|
+
*/
|
|
131
|
+
function getPrefix(tag, role, el) {
|
|
132
|
+
if (tag === 'button' || role === 'button') return 'B';
|
|
133
|
+
if (
|
|
134
|
+
tag === 'input' || tag === 'textarea' ||
|
|
135
|
+
role === 'textbox' || role === 'searchbox' || role === 'combobox' ||
|
|
136
|
+
el.getAttribute('contenteditable') === 'true' ||
|
|
137
|
+
el.getAttribute('contenteditable') === ''
|
|
138
|
+
) return 'I';
|
|
139
|
+
if (tag === 'a' || role === 'link') return 'L';
|
|
140
|
+
return 'S';
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/**
|
|
144
|
+
* 从元素上提取最优 label。
|
|
145
|
+
* 优先级:aria-label > placeholder > title > alt > innerText前40字 > name > id > tagName
|
|
146
|
+
*/
|
|
147
|
+
function getLabel(el, tag) {
|
|
148
|
+
const labelledBy = el.getAttribute('aria-labelledby');
|
|
149
|
+
if (labelledBy) {
|
|
150
|
+
const text = labelledBy.trim().split(/\s+/)
|
|
151
|
+
.map(id => document.getElementById(id)?.textContent?.trim() ?? '')
|
|
152
|
+
.filter(Boolean).join(' ');
|
|
153
|
+
if (text) return text;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
const ariaLabel = el.getAttribute('aria-label');
|
|
157
|
+
if (ariaLabel && ariaLabel.trim()) return ariaLabel.trim();
|
|
158
|
+
|
|
159
|
+
const placeholder = el.getAttribute('placeholder');
|
|
160
|
+
if (placeholder && placeholder.trim()) return placeholder.trim();
|
|
161
|
+
|
|
162
|
+
const title = el.getAttribute('title');
|
|
163
|
+
if (title && title.trim()) return title.trim();
|
|
164
|
+
|
|
165
|
+
const alt = el.getAttribute('alt');
|
|
166
|
+
if (alt && alt.trim()) return alt.trim();
|
|
167
|
+
|
|
168
|
+
const text = (el.innerText || '').trim().slice(0, 40);
|
|
169
|
+
if (text) return text;
|
|
170
|
+
|
|
171
|
+
const name = el.getAttribute('name');
|
|
172
|
+
if (name && name.trim()) return name.trim();
|
|
173
|
+
|
|
174
|
+
const id = el.getAttribute('id');
|
|
175
|
+
if (id && id.trim()) return id.trim();
|
|
176
|
+
|
|
177
|
+
return tag;
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
/**
|
|
181
|
+
* 生成元素指纹:tag | label前8字(空格转下划线)| 位置取整到20px格
|
|
182
|
+
* 同一元素在多次扫描中指纹相同,用于稳定 ID 复用。
|
|
183
|
+
*/
|
|
184
|
+
function fingerprint(tag, label, cx, cy) {
|
|
185
|
+
const gx = Math.round(cx / 20) * 20;
|
|
186
|
+
const gy = Math.round(cy / 20) * 20;
|
|
187
|
+
const text = label.slice(0, 8).replace(/\s+/g, '_');
|
|
188
|
+
return `${tag}|${text}|${gx}|${gy}`;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
const newEntries = []; // 本次新分配的 [fp, id] 对,用于更新 Node.js 侧 registry
|
|
192
|
+
const results = [];
|
|
193
|
+
|
|
194
|
+
for (const { el, tag, role, cx, cy } of filtered) {
|
|
195
|
+
const isContentEditable = el.getAttribute('contenteditable') === 'true' ||
|
|
196
|
+
el.getAttribute('contenteditable') === '';
|
|
197
|
+
// 8. 构建 label(先于指纹计算)
|
|
198
|
+
const label = getLabel(el, tag);
|
|
199
|
+
const type = isContentEditable && !role ? 'textbox' : (role || tag);
|
|
200
|
+
const prefix = getPrefix(tag, role, el);
|
|
201
|
+
|
|
202
|
+
// 9. 查指纹注册表,命中则复用 ID,未命中则分配新 ID
|
|
203
|
+
const fp = fingerprint(tag, label, cx, cy);
|
|
204
|
+
let id = reg.get(fp);
|
|
205
|
+
if (!id) {
|
|
206
|
+
counters[prefix] = (counters[prefix] || 0) + 1;
|
|
207
|
+
id = `${prefix}${counters[prefix]}`;
|
|
208
|
+
reg.set(fp, id);
|
|
209
|
+
newEntries.push([fp, id]);
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
// 10. 写入 data-grasp-id
|
|
213
|
+
el.setAttribute('data-grasp-id', id);
|
|
214
|
+
const meta = {
|
|
215
|
+
name: el.getAttribute('name') ?? '',
|
|
216
|
+
idAttr: el.getAttribute('id') ?? '',
|
|
217
|
+
ariaLabel: el.getAttribute('aria-label') ?? '',
|
|
218
|
+
ariaCurrent: el.getAttribute('aria-current') ?? '',
|
|
219
|
+
placeholder: el.getAttribute('placeholder') ?? '',
|
|
220
|
+
contenteditable: isContentEditable,
|
|
221
|
+
selected: isSelected(el),
|
|
222
|
+
role,
|
|
223
|
+
tag,
|
|
224
|
+
};
|
|
225
|
+
results.push({ id, type, label, x: cx, y: cy, meta });
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
return { hints: results, newEntries, counters };
|
|
229
|
+
}, { registryEntries, counters: { ...counters } });
|
|
230
|
+
|
|
231
|
+
// 将新分配的指纹 → ID 写回 Node.js 侧 registry
|
|
232
|
+
for (const [fp, id] of result.newEntries) {
|
|
233
|
+
registry.set(fp, id);
|
|
234
|
+
}
|
|
235
|
+
// 同步更新计数器
|
|
236
|
+
Object.assign(counters, result.counters);
|
|
237
|
+
|
|
238
|
+
return result.hints;
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
const INPUT_LIKE_TYPES = new Set(['input', 'textarea', 'textbox', 'searchbox', 'combobox']);
|
|
242
|
+
|
|
243
|
+
function isInputLikeHint(hint = {}) {
|
|
244
|
+
const type = String(hint.type ?? '').toLowerCase();
|
|
245
|
+
if (INPUT_LIKE_TYPES.has(type)) return true;
|
|
246
|
+
if (hint.meta?.contenteditable === true || hint.meta?.contenteditable === 'true') return true;
|
|
247
|
+
return false;
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
function normalizeHintLabel(hint = {}) {
|
|
251
|
+
const candidates = [
|
|
252
|
+
hint.label,
|
|
253
|
+
hint.meta?.ariaLabel,
|
|
254
|
+
hint.meta?.placeholder,
|
|
255
|
+
hint.meta?.name,
|
|
256
|
+
hint.meta?.idAttr,
|
|
257
|
+
];
|
|
258
|
+
return candidates
|
|
259
|
+
.map((value) => String(value ?? '').trim().toLowerCase())
|
|
260
|
+
.find(Boolean) ?? '';
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
export function rebindHintCandidate(previous, nextHints) {
|
|
264
|
+
if (!previous) return null;
|
|
265
|
+
const candidates = nextHints.filter(
|
|
266
|
+
(hint) => {
|
|
267
|
+
const sameType = hint.type === previous.type;
|
|
268
|
+
const sameInputFamily = isInputLikeHint(previous) && isInputLikeHint(hint);
|
|
269
|
+
if (!sameType && !sameInputFamily) return false;
|
|
270
|
+
return normalizeHintLabel(hint) === normalizeHintLabel(previous);
|
|
271
|
+
}
|
|
272
|
+
);
|
|
273
|
+
if (candidates.length === 0) return null;
|
|
274
|
+
|
|
275
|
+
const distance = (hint) =>
|
|
276
|
+
Math.abs(hint.x - previous.x) + Math.abs(hint.y - previous.y);
|
|
277
|
+
|
|
278
|
+
return candidates.reduce((best, current) => {
|
|
279
|
+
if (!best) return current;
|
|
280
|
+
const bestDistance = distance(best);
|
|
281
|
+
const currentDistance = distance(current);
|
|
282
|
+
return currentDistance < bestDistance ? current : best;
|
|
283
|
+
}, null);
|
|
284
|
+
}
|
|
@@ -0,0 +1,400 @@
|
|
|
1
|
+
// Layer 3 - Action Layer
|
|
2
|
+
// Pointer and keyboard interactions use real CDP mouse/keyboard events to
|
|
3
|
+
// minimize anti-bot fingerprints. Targeted nested-container scrolling is the
|
|
4
|
+
// one deliberate exception: it may use DOM scrollBy for precision when wheel
|
|
5
|
+
// routing to a specific overflow container is unreliable.
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Find the nearest scrollable ancestor of an element via browser-side evaluation.
|
|
9
|
+
* Returns a unique CSS selector for the scrollable container, or null if none found.
|
|
10
|
+
*/
|
|
11
|
+
export async function findScrollableAncestor(page, selector) {
|
|
12
|
+
return page.evaluate((sel) => {
|
|
13
|
+
const el = document.querySelector(sel);
|
|
14
|
+
if (!el) return null;
|
|
15
|
+
|
|
16
|
+
let current = el;
|
|
17
|
+
while (current && current !== document.documentElement) {
|
|
18
|
+
const style = window.getComputedStyle(current);
|
|
19
|
+
const overflowY = style.overflowY;
|
|
20
|
+
const overflowX = style.overflowX;
|
|
21
|
+
const isNativeScrollable = current.tagName === 'TEXTAREA' ||
|
|
22
|
+
(current.tagName === 'DIV' && current.contentEditable === 'true');
|
|
23
|
+
const isCSSScrollable =
|
|
24
|
+
overflowY === 'auto' || overflowY === 'scroll' || overflowY === 'overlay' ||
|
|
25
|
+
overflowX === 'auto' || overflowX === 'scroll' || overflowX === 'overlay';
|
|
26
|
+
const hasOverflow = current.scrollHeight > current.clientHeight || current.scrollWidth > current.clientWidth;
|
|
27
|
+
|
|
28
|
+
if ((isNativeScrollable || isCSSScrollable) && hasOverflow) {
|
|
29
|
+
if (current.id) return `#${CSS.escape(current.id)}`;
|
|
30
|
+
const graspId = current.getAttribute('data-grasp-id');
|
|
31
|
+
if (graspId) return `[data-grasp-id="${graspId}"]`;
|
|
32
|
+
const classes = [...current.classList].map((cls) => `.${CSS.escape(cls)}`).join('');
|
|
33
|
+
if (classes) {
|
|
34
|
+
const candidateSelector = `${current.tagName.toLowerCase()}${classes}`;
|
|
35
|
+
const matches = document.querySelectorAll(candidateSelector);
|
|
36
|
+
if (matches.length === 1 && matches[0] === current) {
|
|
37
|
+
return candidateSelector;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
current = current.parentElement;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
return null;
|
|
46
|
+
}, selector);
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function randomInt(min, max) {
|
|
50
|
+
return Math.floor(Math.random() * (max - min + 1)) + min;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* 在元素 boundingBox 内生成随机点击坐标(避免总在正中心)。
|
|
55
|
+
* @param {{ x: number, y: number, width: number, height: number }} box
|
|
56
|
+
* @returns {{ x: number, y: number }}
|
|
57
|
+
*/
|
|
58
|
+
function randomPointInBox(box) {
|
|
59
|
+
// 极小元素(宽或高 < 10px)直接取中心,避免 margin 使可用区域退化为零
|
|
60
|
+
if (box.width < 10 || box.height < 10) {
|
|
61
|
+
return { x: Math.round(box.x + box.width / 2), y: Math.round(box.y + box.height / 2) };
|
|
62
|
+
}
|
|
63
|
+
const margin = 0.2;
|
|
64
|
+
const x = box.x + box.width * (margin + Math.random() * (1 - 2 * margin));
|
|
65
|
+
const y = box.y + box.height * (margin + Math.random() * (1 - 2 * margin));
|
|
66
|
+
return { x: Math.round(x), y: Math.round(y) };
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// 记录每个 page 是否已初始化鼠标位置,避免 session 首次点击从 (0,0) 出发
|
|
70
|
+
const _warmedUpPages = new WeakSet();
|
|
71
|
+
|
|
72
|
+
async function warmupMouseIfNeeded(page) {
|
|
73
|
+
if (_warmedUpPages.has(page)) return;
|
|
74
|
+
_warmedUpPages.add(page);
|
|
75
|
+
const vp = await page.evaluate(() => ({ w: window.innerWidth, h: window.innerHeight }));
|
|
76
|
+
// 随机初始位置:水平 30%~70%,垂直 30%~70%,模拟真人屏幕中央区域习惯
|
|
77
|
+
const x = Math.round(vp.w * (0.3 + Math.random() * 0.4));
|
|
78
|
+
const y = Math.round(vp.h * (0.3 + Math.random() * 0.4));
|
|
79
|
+
await page.mouse.move(x, y);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Scroll the page or container by a given amount.
|
|
84
|
+
* Page scrolling uses real CDP wheel events; targeted container scrolling uses
|
|
85
|
+
* DOM scrollBy to avoid sending wheel input to the wrong overflow region.
|
|
86
|
+
* @param {import('playwright').Page} page
|
|
87
|
+
* @param {'up'|'down'|'left'|'right'} direction
|
|
88
|
+
* @param {number} [amount=600]
|
|
89
|
+
* @param {{ selector?: string }} [options]
|
|
90
|
+
*/
|
|
91
|
+
export async function scroll(page, direction, amount = 600, options = {}) {
|
|
92
|
+
const validDirections = ['up', 'down', 'left', 'right'];
|
|
93
|
+
if (!validDirections.includes(direction)) {
|
|
94
|
+
throw new Error(`Invalid scroll direction: "${direction}". Expected one of: ${validDirections.join(', ')}.`);
|
|
95
|
+
}
|
|
96
|
+
if (amount === 0) return;
|
|
97
|
+
|
|
98
|
+
const isVertical = direction === 'up' || direction === 'down';
|
|
99
|
+
const delta = (direction === 'down' || direction === 'right') ? amount : -amount;
|
|
100
|
+
const dx = isVertical ? 0 : delta;
|
|
101
|
+
const dy = isVertical ? delta : 0;
|
|
102
|
+
|
|
103
|
+
if (options.selector) {
|
|
104
|
+
const scrolled = await page.evaluate(({ selector, dx: scrollX, dy: scrollY }) => {
|
|
105
|
+
const el = document.querySelector(selector);
|
|
106
|
+
if (!el) return { ok: false, reason: 'not_found' };
|
|
107
|
+
el.scrollBy(scrollX, scrollY);
|
|
108
|
+
return { ok: true, tag: el.tagName.toLowerCase() };
|
|
109
|
+
}, { selector: options.selector, dx, dy });
|
|
110
|
+
|
|
111
|
+
if (!scrolled.ok) {
|
|
112
|
+
throw new Error(`Scroll target not found: "${options.selector}".`);
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
await page.evaluate(() => new Promise((r) => requestAnimationFrame(() => requestAnimationFrame(r))));
|
|
116
|
+
return;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
const steps = 5;
|
|
120
|
+
const stepDx = dx / steps;
|
|
121
|
+
const stepDy = dy / steps;
|
|
122
|
+
|
|
123
|
+
for (let i = 0; i < steps; i++) {
|
|
124
|
+
await page.mouse.wheel(stepDx, stepDy);
|
|
125
|
+
await new Promise((r) => setTimeout(r, 20 + Math.random() * 40));
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
await page.evaluate(() => new Promise((r) => requestAnimationFrame(() => requestAnimationFrame(r))));
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/**
|
|
132
|
+
* Locate an element by hint ID.
|
|
133
|
+
* If the element is not in the viewport, naturally scrolls to bring it into view.
|
|
134
|
+
* @param {import('playwright').Page} page
|
|
135
|
+
* @param {string} hintId
|
|
136
|
+
* @returns {Promise<{ info: { tag: string, label: string }, el: import('playwright').ElementHandle }>}
|
|
137
|
+
*/
|
|
138
|
+
async function locateElement(page, hintId, options = {}) {
|
|
139
|
+
// 检查元素是否存在以及是否在视口内
|
|
140
|
+
const defaultEvaluateHint = (id) => page.evaluate((targetId) => {
|
|
141
|
+
const el = document.querySelector(`[data-grasp-id="${targetId}"]`);
|
|
142
|
+
if (!el) return null;
|
|
143
|
+
const rect = el.getBoundingClientRect();
|
|
144
|
+
const centerX = rect.left + rect.width / 2;
|
|
145
|
+
const centerY = rect.top + rect.height / 2;
|
|
146
|
+
const inView = (
|
|
147
|
+
centerX >= 0 && centerX <= window.innerWidth &&
|
|
148
|
+
centerY >= 0 && centerY <= window.innerHeight
|
|
149
|
+
);
|
|
150
|
+
return {
|
|
151
|
+
inView,
|
|
152
|
+
centerY,
|
|
153
|
+
tag: el.tagName.toLowerCase(),
|
|
154
|
+
label:
|
|
155
|
+
el.getAttribute('aria-label') ||
|
|
156
|
+
el.getAttribute('placeholder') ||
|
|
157
|
+
el.innerText?.trim() ||
|
|
158
|
+
'',
|
|
159
|
+
};
|
|
160
|
+
}, id);
|
|
161
|
+
|
|
162
|
+
const evaluateHint = options.evaluateHint ?? defaultEvaluateHint;
|
|
163
|
+
const fetchHandle = async (id) => page.$(`[data-grasp-id="${id}"]`);
|
|
164
|
+
const ensureActionableHandle = async (id, currentInfo) => {
|
|
165
|
+
let handle = await fetchHandle(id);
|
|
166
|
+
let box = handle && typeof handle.boundingBox === 'function'
|
|
167
|
+
? await handle.boundingBox()
|
|
168
|
+
: null;
|
|
169
|
+
|
|
170
|
+
const actionable = !!(box && box.width > 0 && box.height > 0);
|
|
171
|
+
if (actionable || typeof options.rebuildHints !== 'function') {
|
|
172
|
+
return { handle, reboundId: id, reboundInfo: currentInfo };
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
const rebound = await options.rebuildHints(id);
|
|
176
|
+
if (!rebound?.id || rebound.id === id) {
|
|
177
|
+
return { handle, reboundId: id, reboundInfo: currentInfo };
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
const reboundInfo = await evaluateHint(rebound.id);
|
|
181
|
+
const reboundHandle = await fetchHandle(rebound.id);
|
|
182
|
+
return { handle: reboundHandle, reboundId: rebound.id, reboundInfo };
|
|
183
|
+
};
|
|
184
|
+
|
|
185
|
+
let viewportCheck = await evaluateHint(hintId);
|
|
186
|
+
if (viewportCheck === null && typeof options.rebuildHints === 'function') {
|
|
187
|
+
const rebound = await options.rebuildHints(hintId);
|
|
188
|
+
if (rebound?.id) {
|
|
189
|
+
hintId = rebound.id;
|
|
190
|
+
}
|
|
191
|
+
viewportCheck = await evaluateHint(hintId);
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
if (viewportCheck === null) {
|
|
195
|
+
throw new Error(`No element with hint ID "${hintId}". Call get_hint_map first.`);
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
// 如果不在视口内,用真实滚动将其带入视野
|
|
199
|
+
if (!viewportCheck.inView) {
|
|
200
|
+
const vh = await page.evaluate(() => window.innerHeight);
|
|
201
|
+
const scrollNeeded = viewportCheck.centerY - vh / 2;
|
|
202
|
+
const direction = scrollNeeded > 0 ? 'down' : 'up';
|
|
203
|
+
const amount = Math.abs(scrollNeeded);
|
|
204
|
+
await scroll(page, direction, amount);
|
|
205
|
+
// 等待滚动动画完成
|
|
206
|
+
await new Promise((r) => setTimeout(r, 300));
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
const actionable = await ensureActionableHandle(hintId, viewportCheck);
|
|
210
|
+
hintId = actionable.reboundId;
|
|
211
|
+
viewportCheck = actionable.reboundInfo ?? viewportCheck;
|
|
212
|
+
const el = actionable.handle;
|
|
213
|
+
if (el === null) {
|
|
214
|
+
throw new Error(`Element "${hintId}" disappeared after scrolling.`);
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
return {
|
|
218
|
+
info: { tag: viewportCheck.tag, label: viewportCheck.label },
|
|
219
|
+
el,
|
|
220
|
+
};
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
/**
|
|
224
|
+
* Click an element identified by its hint ID.
|
|
225
|
+
* Moves the mouse naturally to the element before clicking.
|
|
226
|
+
* @param {import('playwright').Page} page
|
|
227
|
+
* @param {string} hintId
|
|
228
|
+
* @returns {Promise<{ tag: string, label: string }>}
|
|
229
|
+
*/
|
|
230
|
+
export async function clickByHintId(page, hintId, options = {}) {
|
|
231
|
+
const { info, el } = await locateElement(page, hintId, options);
|
|
232
|
+
const clickCount = Math.max(1, Number(options.clickCount ?? 1));
|
|
233
|
+
|
|
234
|
+
// 获取元素真实坐标
|
|
235
|
+
const box = await el.boundingBox();
|
|
236
|
+
if (!box) throw new Error(`Element "${hintId}" (<${info.tag}> "${info.label}") has no bounding box (may be hidden).`);
|
|
237
|
+
|
|
238
|
+
// 在元素内随机取一个点
|
|
239
|
+
const target = randomPointInBox(box);
|
|
240
|
+
|
|
241
|
+
// 先把鼠标从当前位置移动过来(分 15 步,模拟自然移动轨迹)
|
|
242
|
+
await warmupMouseIfNeeded(page);
|
|
243
|
+
await page.mouse.move(target.x, target.y, { steps: 15 });
|
|
244
|
+
|
|
245
|
+
if (clickCount > 1) {
|
|
246
|
+
await page.mouse.click(target.x, target.y, { clickCount });
|
|
247
|
+
} else {
|
|
248
|
+
// 按下 + 随机持续时间 + 抬起,模拟人类按键
|
|
249
|
+
await page.mouse.down();
|
|
250
|
+
await new Promise((r) => setTimeout(r, randomInt(40, 120)));
|
|
251
|
+
await page.mouse.up();
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
// 等待页面响应(networkidle 或超时)
|
|
255
|
+
await page.waitForLoadState('networkidle', { timeout: 5000 }).catch(() => {});
|
|
256
|
+
|
|
257
|
+
return info;
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
/**
|
|
261
|
+
* Type text into an element identified by its hint ID.
|
|
262
|
+
* @param {import('playwright').Page} page
|
|
263
|
+
* @param {string} hintId
|
|
264
|
+
* @param {string} text
|
|
265
|
+
* @param {boolean} [pressEnter=false]
|
|
266
|
+
*/
|
|
267
|
+
export async function typeByHintId(page, hintId, text, pressEnter = false, options = {}) {
|
|
268
|
+
const { el } = await locateElement(page, hintId, options);
|
|
269
|
+
|
|
270
|
+
// 三连击全选:优先用真实鼠标坐标事件,兼容 React 受控输入
|
|
271
|
+
const box = await el.boundingBox();
|
|
272
|
+
if (box) {
|
|
273
|
+
const target = randomPointInBox(box);
|
|
274
|
+
await warmupMouseIfNeeded(page);
|
|
275
|
+
await page.mouse.move(target.x, target.y, { steps: 15 });
|
|
276
|
+
await page.mouse.click(target.x, target.y, { clickCount: 3 });
|
|
277
|
+
} else {
|
|
278
|
+
// 极少数情况元素无坐标,回退到 Playwright 高层 API
|
|
279
|
+
await el.click({ clickCount: 3 });
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
// 清除已选中文本
|
|
283
|
+
await page.keyboard.press('Backspace');
|
|
284
|
+
|
|
285
|
+
// Step 4: type character by character with random human-like delay
|
|
286
|
+
await el.type(text, { delay: randomInt(30, 80) });
|
|
287
|
+
|
|
288
|
+
// Step 5: optionally press Enter
|
|
289
|
+
if (pressEnter) {
|
|
290
|
+
await page.keyboard.press('Enter');
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
/**
|
|
295
|
+
* Watch a DOM element for a specific condition using MutationObserver.
|
|
296
|
+
* @param {import('playwright').Page} page
|
|
297
|
+
* @param {string} selector - CSS selector to watch
|
|
298
|
+
* @param {'appears'|'disappears'|'changes'} [condition='appears']
|
|
299
|
+
* @param {number} [timeout=30000]
|
|
300
|
+
* @returns {Promise<{ met?: true, text?: string, timeout?: true }>}
|
|
301
|
+
*/
|
|
302
|
+
export async function watchElement(page, selector, condition = 'appears', timeout = 30000) {
|
|
303
|
+
return page.evaluate(
|
|
304
|
+
({ selector: sel, condition: cond, timeout: ms }) => {
|
|
305
|
+
return new Promise((resolve) => {
|
|
306
|
+
let settled = false;
|
|
307
|
+
let observer;
|
|
308
|
+
let timer;
|
|
309
|
+
let initialText;
|
|
310
|
+
|
|
311
|
+
function done(result) {
|
|
312
|
+
if (settled) return;
|
|
313
|
+
settled = true;
|
|
314
|
+
observer?.disconnect();
|
|
315
|
+
clearTimeout(timer);
|
|
316
|
+
resolve(result);
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
function check() {
|
|
320
|
+
const el = document.querySelector(sel);
|
|
321
|
+
if (cond === 'appears') {
|
|
322
|
+
if (el) {
|
|
323
|
+
done({ met: true, text: el.innerText?.trim() });
|
|
324
|
+
return true;
|
|
325
|
+
}
|
|
326
|
+
} else if (cond === 'disappears') {
|
|
327
|
+
if (!el) {
|
|
328
|
+
done({ met: true });
|
|
329
|
+
return true;
|
|
330
|
+
}
|
|
331
|
+
} else if (cond === 'changes') {
|
|
332
|
+
if (el) {
|
|
333
|
+
initialText = el.innerText?.trim();
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
return false;
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
const immediatelyMet = check();
|
|
340
|
+
if (immediatelyMet) return;
|
|
341
|
+
|
|
342
|
+
timer = setTimeout(() => {
|
|
343
|
+
done({ timeout: true });
|
|
344
|
+
}, ms);
|
|
345
|
+
|
|
346
|
+
observer = new MutationObserver(() => {
|
|
347
|
+
const el = document.querySelector(sel);
|
|
348
|
+
if (cond === 'appears') {
|
|
349
|
+
if (el) {
|
|
350
|
+
done({ met: true, text: el.innerText?.trim() });
|
|
351
|
+
}
|
|
352
|
+
} else if (cond === 'disappears') {
|
|
353
|
+
if (!el) {
|
|
354
|
+
done({ met: true });
|
|
355
|
+
}
|
|
356
|
+
} else if (cond === 'changes') {
|
|
357
|
+
if (el) {
|
|
358
|
+
const current = el.innerText?.trim();
|
|
359
|
+
if (current !== initialText) {
|
|
360
|
+
done({ met: true, text: current });
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
});
|
|
365
|
+
|
|
366
|
+
const target = document.querySelector(sel);
|
|
367
|
+
if (target) {
|
|
368
|
+
observer.observe(target, { childList: true, subtree: true, characterData: true });
|
|
369
|
+
} else {
|
|
370
|
+
observer.observe(document.body, { childList: true, subtree: true });
|
|
371
|
+
}
|
|
372
|
+
});
|
|
373
|
+
},
|
|
374
|
+
{ selector, condition, timeout }
|
|
375
|
+
);
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
/**
|
|
379
|
+
* Press a keyboard key or shortcut (e.g. 'Enter', 'Escape', 'Control+Enter').
|
|
380
|
+
* @param {import('playwright').Page} page
|
|
381
|
+
* @param {string} key
|
|
382
|
+
*/
|
|
383
|
+
export async function pressKey(page, key) {
|
|
384
|
+
await page.keyboard.press(key);
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
/**
|
|
388
|
+
* Hover over an element by its Hint Map ID to trigger dropdowns or tooltips.
|
|
389
|
+
* @param {import('playwright').Page} page
|
|
390
|
+
* @param {string} hintId
|
|
391
|
+
*/
|
|
392
|
+
export async function hoverByHintId(page, hintId, options = {}) {
|
|
393
|
+
const { info, el } = await locateElement(page, hintId, options);
|
|
394
|
+
await el.hover();
|
|
395
|
+
// Allow hover-triggered animations/menus to settle
|
|
396
|
+
await new Promise((r) => setTimeout(r, 200));
|
|
397
|
+
return info;
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
export { locateElement };
|