@yuzc-001/grasp 0.6.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +327 -0
- package/README.zh-CN.md +324 -0
- package/examples/README.md +31 -0
- package/examples/claude-desktop.json +8 -0
- package/examples/codex-config.toml +4 -0
- package/grasp.skill +0 -0
- package/index.js +87 -0
- package/package.json +48 -0
- package/scripts/grasp_openclaw_ctl.sh +122 -0
- package/scripts/run-search-benchmark.mjs +287 -0
- package/scripts/update-star-history.mjs +274 -0
- package/skill/SKILL.md +61 -0
- package/skill/references/tools.md +306 -0
- package/src/cli/auto-configure.js +116 -0
- package/src/cli/cmd-connect.js +148 -0
- package/src/cli/cmd-explain.js +42 -0
- package/src/cli/cmd-logs.js +55 -0
- package/src/cli/cmd-status.js +119 -0
- package/src/cli/config.js +27 -0
- package/src/cli/detect-chrome.js +58 -0
- package/src/grasp/handoff/events.js +67 -0
- package/src/grasp/handoff/persist.js +48 -0
- package/src/grasp/handoff/state.js +28 -0
- package/src/grasp/page/capture.js +34 -0
- package/src/grasp/page/state.js +273 -0
- package/src/grasp/verify/evidence.js +40 -0
- package/src/grasp/verify/pipeline.js +52 -0
- package/src/layer1-bridge/chrome.js +416 -0
- package/src/layer1-bridge/webmcp.js +143 -0
- package/src/layer2-perception/hints.js +284 -0
- package/src/layer3-action/actions.js +400 -0
- package/src/runtime/browser-instance.js +65 -0
- package/src/runtime/truth/model.js +94 -0
- package/src/runtime/truth/snapshot.js +51 -0
- package/src/server/affordances.js +47 -0
- package/src/server/audit.js +122 -0
- package/src/server/boss-fast-path.js +164 -0
- package/src/server/boundary-guard.js +53 -0
- package/src/server/content.js +97 -0
- package/src/server/continuity.js +256 -0
- package/src/server/engine-selection.js +29 -0
- package/src/server/entry-orchestrator.js +115 -0
- package/src/server/error-codes.js +7 -0
- package/src/server/explain-share-card.js +113 -0
- package/src/server/fast-path-router.js +134 -0
- package/src/server/form-runtime.js +602 -0
- package/src/server/form-tasks.js +254 -0
- package/src/server/gateway-response.js +62 -0
- package/src/server/index.js +22 -0
- package/src/server/observe.js +52 -0
- package/src/server/page-projection.js +31 -0
- package/src/server/page-state.js +27 -0
- package/src/server/postconditions.js +128 -0
- package/src/server/prompt-assembly.js +148 -0
- package/src/server/responses.js +44 -0
- package/src/server/route-boundary.js +174 -0
- package/src/server/route-policy.js +168 -0
- package/src/server/runtime-confirmation.js +87 -0
- package/src/server/runtime-status.js +7 -0
- package/src/server/share-artifacts.js +284 -0
- package/src/server/state.js +132 -0
- package/src/server/structured-extraction.js +131 -0
- package/src/server/surface-prompts.js +166 -0
- package/src/server/task-frame.js +11 -0
- package/src/server/tasks/search-task.js +321 -0
- package/src/server/tools.actions.js +1361 -0
- package/src/server/tools.form.js +526 -0
- package/src/server/tools.gateway.js +757 -0
- package/src/server/tools.handoff.js +210 -0
- package/src/server/tools.js +20 -0
- package/src/server/tools.legacy.js +983 -0
- package/src/server/tools.strategy.js +250 -0
- package/src/server/tools.task-surface.js +66 -0
- package/src/server/tools.workspace.js +873 -0
- package/src/server/workspace-runtime.js +1138 -0
- package/src/server/workspace-tasks.js +735 -0
- package/start-chrome.bat +84 -0
|
@@ -0,0 +1,735 @@
|
|
|
1
|
+
function compactText(value) {
|
|
2
|
+
return String(value ?? '').replace(/\s+/g, ' ').trim();
|
|
3
|
+
}
|
|
4
|
+
|
|
5
|
+
function normalizeLabel(value) {
|
|
6
|
+
return compactText(value).toLowerCase();
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
export function deriveWorkspaceHintItems(hintMap = []) {
|
|
10
|
+
const hints = Array.isArray(hintMap) ? hintMap : [];
|
|
11
|
+
const candidates = hints
|
|
12
|
+
.map((hint) => {
|
|
13
|
+
const label = compactText(hint?.label);
|
|
14
|
+
const type = compactText(hint?.type ?? hint?.meta?.tag).toLowerCase();
|
|
15
|
+
const ariaCurrent = compactText(hint?.meta?.ariaCurrent).toLowerCase();
|
|
16
|
+
return {
|
|
17
|
+
label,
|
|
18
|
+
normalized_label: normalizeLabel(label),
|
|
19
|
+
hint_id: compactText(hint?.id) || null,
|
|
20
|
+
type,
|
|
21
|
+
x: Number(hint?.x),
|
|
22
|
+
y: Number(hint?.y),
|
|
23
|
+
selected: hint?.meta?.selected === true
|
|
24
|
+
|| ['page', 'step', 'location', 'date', 'time', 'true'].includes(ariaCurrent),
|
|
25
|
+
};
|
|
26
|
+
})
|
|
27
|
+
.filter((hint) => hint.label
|
|
28
|
+
&& hint.label.length <= 24
|
|
29
|
+
&& !/^\d+$/.test(hint.label)
|
|
30
|
+
&& (hint.type === 'a' || hint.type === 'button')
|
|
31
|
+
&& Number.isFinite(hint.x)
|
|
32
|
+
&& Number.isFinite(hint.y)
|
|
33
|
+
&& hint.x <= 240)
|
|
34
|
+
.sort((left, right) => left.y - right.y || left.x - right.x);
|
|
35
|
+
|
|
36
|
+
if (candidates.length < 2) {
|
|
37
|
+
return [];
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
const xValues = candidates.map((hint) => hint.x);
|
|
41
|
+
const yValues = candidates.map((hint) => hint.y);
|
|
42
|
+
const xSpread = Math.max(...xValues) - Math.min(...xValues);
|
|
43
|
+
const ySpread = Math.max(...yValues) - Math.min(...yValues);
|
|
44
|
+
|
|
45
|
+
if (xSpread > 120 || ySpread < 40) {
|
|
46
|
+
return [];
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
return candidates
|
|
50
|
+
.filter((hint, index, items) => items.findIndex((candidate) => candidate.normalized_label === hint.normalized_label) === index)
|
|
51
|
+
.map(({ label, normalized_label, hint_id, selected }) => ({
|
|
52
|
+
label,
|
|
53
|
+
normalized_label,
|
|
54
|
+
hint_id,
|
|
55
|
+
selected: selected === true,
|
|
56
|
+
}));
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function pick(snapshot, camelKey, snakeKey, fallback = null) {
|
|
60
|
+
if (snapshot?.[camelKey] !== undefined) return snapshot[camelKey];
|
|
61
|
+
if (snapshot?.[snakeKey] !== undefined) return snapshot[snakeKey];
|
|
62
|
+
return fallback;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function pickText(snapshot, camelKey, snakeKey, fallback = '') {
|
|
66
|
+
return compactText(pick(snapshot, camelKey, snakeKey, fallback));
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function getLiveItems(snapshot) {
|
|
70
|
+
const items = pick(snapshot, 'liveItems', 'live_items', []);
|
|
71
|
+
return Array.isArray(items) ? items : [];
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
function getComposer(snapshot) {
|
|
75
|
+
const composer = pick(snapshot, 'composer', 'composer', null);
|
|
76
|
+
return composer && typeof composer === 'object' ? composer : null;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function getActionControls(snapshot) {
|
|
80
|
+
const controls = pick(snapshot, 'actionControls', 'action_controls', []);
|
|
81
|
+
return Array.isArray(controls) ? controls : [];
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function getBlockingModals(snapshot) {
|
|
85
|
+
const modals = pick(snapshot, 'blockingModals', 'blocking_modals', []);
|
|
86
|
+
return Array.isArray(modals) ? modals : [];
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function getDetailPanel(snapshot) {
|
|
90
|
+
const detailPanel = pick(snapshot, 'detailPanel', 'detail_panel', null);
|
|
91
|
+
return detailPanel && typeof detailPanel === 'object' ? detailPanel : null;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
function getLoadingShell(snapshot) {
|
|
95
|
+
const loadingShell = pick(snapshot, 'loadingShell', 'loading_shell', false);
|
|
96
|
+
return loadingShell === true;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
function isSelectedItem(item) {
|
|
100
|
+
return item?.selected === true;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
function hasExactLoadingShellText(text) {
|
|
104
|
+
return text.includes('加载中,请稍候')
|
|
105
|
+
|| (text.includes('加载中') && text.includes('请稍候'))
|
|
106
|
+
|| text.includes('正在加载');
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
function hasThreadPromptText(text) {
|
|
110
|
+
return text.includes('按enter键发送')
|
|
111
|
+
|| text.includes('发送消息')
|
|
112
|
+
|| text.includes('发消息')
|
|
113
|
+
|| text.includes('输入消息');
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
function hasThreadContextText(text) {
|
|
117
|
+
return text.includes('消息')
|
|
118
|
+
|| text.includes('聊天')
|
|
119
|
+
|| text.includes('对话');
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
function isComposerSurface(workspaceSurface) {
|
|
123
|
+
return workspaceSurface === 'thread' || workspaceSurface === 'composer';
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
function hasSendActionControl(actionControls) {
|
|
127
|
+
return actionControls.some((control) => {
|
|
128
|
+
const label = normalizeLabel(control?.label);
|
|
129
|
+
return label.includes('发送')
|
|
130
|
+
|| label.includes('send')
|
|
131
|
+
|| label.includes('回复')
|
|
132
|
+
|| label.includes('提交');
|
|
133
|
+
});
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
function hasEnglishSuccessSignal(text) {
|
|
137
|
+
return /\b(delivered|sent)\b/i.test(text);
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
function hasThreadEvidence(snapshot) {
|
|
141
|
+
const bodyText = pickText(snapshot, 'bodyText', 'body_text').toLowerCase();
|
|
142
|
+
const actionControls = getActionControls(snapshot);
|
|
143
|
+
const liveItems = getLiveItems(snapshot);
|
|
144
|
+
|
|
145
|
+
if (hasThreadPromptText(bodyText)) {
|
|
146
|
+
return true;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
if (hasThreadContextText(bodyText) && liveItems.some(isSelectedItem) && hasSendActionControl(actionControls)) {
|
|
150
|
+
return true;
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
return false;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
function hasComposerEvidence(snapshot) {
|
|
157
|
+
const composer = getComposer(snapshot);
|
|
158
|
+
if (!composer) return false;
|
|
159
|
+
return composer.kind === 'chat_composer';
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
export function classifyWorkspaceSurface(snapshot = {}) {
|
|
163
|
+
if (getLoadingShell(snapshot)) {
|
|
164
|
+
return 'loading_shell';
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
const bodyText = pickText(snapshot, 'bodyText', 'body_text').toLowerCase();
|
|
168
|
+
if (hasExactLoadingShellText(bodyText)) {
|
|
169
|
+
return 'loading_shell';
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
if (hasThreadEvidence(snapshot)) {
|
|
173
|
+
return 'thread';
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
if (hasComposerEvidence(snapshot)) {
|
|
177
|
+
return 'composer';
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
const detailPanel = getDetailPanel(snapshot);
|
|
181
|
+
if (detailPanel) {
|
|
182
|
+
return 'detail';
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
if (getLiveItems(snapshot).length > 0) {
|
|
186
|
+
return 'list';
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
return pick(snapshot, 'workspaceSurface', 'workspace_surface', null);
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
function getVisibleItemLabel(item) {
|
|
193
|
+
return compactText(item?.label || item?.normalized_label || item?.text || '');
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
function getSelectedLiveItem(liveItems) {
|
|
197
|
+
const selected = liveItems.filter(isSelectedItem);
|
|
198
|
+
if (selected.length !== 1) return null;
|
|
199
|
+
return selected[0];
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
function getActiveItem(snapshot, liveItems, detailPanel) {
|
|
203
|
+
const selectedLiveItem = getSelectedLiveItem(liveItems);
|
|
204
|
+
if (selectedLiveItem) {
|
|
205
|
+
return {
|
|
206
|
+
label: getVisibleItemLabel(selectedLiveItem),
|
|
207
|
+
normalized_label: normalizeLabel(getVisibleItemLabel(selectedLiveItem)),
|
|
208
|
+
hint_id: selectedLiveItem.hint_id ?? selectedLiveItem.hintId ?? null,
|
|
209
|
+
selected: true,
|
|
210
|
+
};
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
return null;
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
function getDetailAlignment(activeItem, detailPanel) {
|
|
217
|
+
const detailLabel = getVisibleItemLabel(detailPanel);
|
|
218
|
+
if (!activeItem || !detailLabel) {
|
|
219
|
+
return 'unknown';
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
return normalizeLabel(activeItem.label) === normalizeLabel(detailLabel) ? 'aligned' : 'mismatch';
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
function getSelectionWindow(activeItem, detailPanel, liveItems) {
|
|
226
|
+
if (!activeItem) {
|
|
227
|
+
if (detailPanel && liveItems.length > 0) {
|
|
228
|
+
return 'virtualized';
|
|
229
|
+
}
|
|
230
|
+
return 'not_found';
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
const hasVisibleMatch = liveItems.some((item) => normalizeLabel(getVisibleItemLabel(item)) === normalizeLabel(activeItem.label));
|
|
234
|
+
if (hasVisibleMatch && isSelectedItem(liveItems.find((item) => normalizeLabel(getVisibleItemLabel(item)) === normalizeLabel(activeItem.label)))) {
|
|
235
|
+
return 'visible';
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
if (detailPanel) {
|
|
239
|
+
return 'virtualized';
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
return 'not_found';
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
function getRecoveryHint(selectionWindow, liveItems, detailPanel) {
|
|
246
|
+
if (selectionWindow === 'virtualized') {
|
|
247
|
+
return 'scroll_list';
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
if (selectionWindow === 'not_found') {
|
|
251
|
+
if (liveItems.length > 0) return 'scroll_list';
|
|
252
|
+
if (detailPanel) return 'reinspect_workspace';
|
|
253
|
+
return 'reinspect_workspace';
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
return null;
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
function getOutcomeSignals(snapshot, composer, activeItem, detailAlignment, selectionWindow) {
|
|
260
|
+
const bodyText = pickText(snapshot, 'bodyText', 'body_text').toLowerCase();
|
|
261
|
+
const delivered = bodyText.includes('已发送') || bodyText.includes('发送成功') || hasEnglishSuccessSignal(bodyText);
|
|
262
|
+
const composerCleared = delivered;
|
|
263
|
+
const activeItemStable = detailAlignment !== undefined
|
|
264
|
+
? detailAlignment === 'aligned' && selectionWindow === 'visible'
|
|
265
|
+
: !!activeItem && getDetailAlignment(activeItem, getDetailPanel(snapshot)) === 'aligned';
|
|
266
|
+
|
|
267
|
+
return {
|
|
268
|
+
delivered,
|
|
269
|
+
composer_cleared: composerCleared,
|
|
270
|
+
active_item_stable: activeItemStable,
|
|
271
|
+
};
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
function isBlockedHandoffState(handoffState) {
|
|
275
|
+
return handoffState === 'handoff_required'
|
|
276
|
+
|| handoffState === 'handoff_in_progress'
|
|
277
|
+
|| handoffState === 'awaiting_reacquisition';
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
export function getWorkspaceStatus(state) {
|
|
281
|
+
const handoffState = state.handoff?.state ?? 'idle';
|
|
282
|
+
if (isBlockedHandoffState(handoffState)) {
|
|
283
|
+
return 'handoff_required';
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
return state.pageState?.riskGateDetected ? 'gated' : 'direct';
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
export function getWorkspaceContinuation(state, suggestedNextAction) {
|
|
290
|
+
const handoffState = state.handoff?.state ?? 'idle';
|
|
291
|
+
if (getWorkspaceStatus(state) !== 'direct') {
|
|
292
|
+
return {
|
|
293
|
+
can_continue: false,
|
|
294
|
+
suggested_next_action: 'request_handoff',
|
|
295
|
+
handoff_state: handoffState,
|
|
296
|
+
};
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
return {
|
|
300
|
+
can_continue: true,
|
|
301
|
+
suggested_next_action: suggestedNextAction,
|
|
302
|
+
handoff_state: handoffState,
|
|
303
|
+
};
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
function getSummaryString({ workspaceSurface, activeItem, composer, blockingModals, loadingShell, detailAlignment, selectionWindow }) {
|
|
307
|
+
const activeLabel = activeItem?.label ?? 'none';
|
|
308
|
+
const draftState = composer?.draft_present ? 'draft' : 'empty';
|
|
309
|
+
const blockerCount = blockingModals.length;
|
|
310
|
+
return `surface=${workspaceSurface ?? 'unknown'} active=${activeLabel} draft=${draftState} blockers=${blockerCount} loading=${loadingShell ? 'yes' : 'no'} detail=${detailAlignment} selection=${selectionWindow}`;
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
export function summarizeWorkspaceSnapshot(snapshot = {}) {
|
|
314
|
+
const liveItems = getLiveItems(snapshot);
|
|
315
|
+
const composer = getComposer(snapshot);
|
|
316
|
+
const detailPanel = getDetailPanel(snapshot);
|
|
317
|
+
const blockingModals = getBlockingModals(snapshot);
|
|
318
|
+
const loadingShell = snapshot.loading_shell !== undefined ? snapshot.loading_shell : snapshot.loadingShell;
|
|
319
|
+
const rawActiveItem = snapshot.active_item !== undefined ? snapshot.active_item : snapshot.activeItem;
|
|
320
|
+
const rawDetailAlignment = snapshot.detail_alignment !== undefined ? snapshot.detail_alignment : snapshot.detailAlignment;
|
|
321
|
+
const rawSelectionWindow = snapshot.selection_window !== undefined ? snapshot.selection_window : snapshot.selectionWindow;
|
|
322
|
+
const rawRecoveryHint = snapshot.recovery_hint !== undefined ? snapshot.recovery_hint : snapshot.recoveryHint;
|
|
323
|
+
const rawOutcomeSignals = snapshot.outcome_signals !== undefined ? snapshot.outcome_signals : snapshot.outcomeSignals;
|
|
324
|
+
const derivedActiveItem = getActiveItem(snapshot, liveItems, detailPanel);
|
|
325
|
+
const activeItem = rawActiveItem ?? derivedActiveItem;
|
|
326
|
+
const shouldPreferDerivedSelection = rawActiveItem == null && derivedActiveItem != null;
|
|
327
|
+
const workspaceSurface = pick(snapshot, 'workspaceSurface', 'workspace_surface', null) ?? classifyWorkspaceSurface(snapshot);
|
|
328
|
+
const detailAlignment = shouldPreferDerivedSelection
|
|
329
|
+
? getDetailAlignment(activeItem, detailPanel)
|
|
330
|
+
: rawDetailAlignment !== undefined ? rawDetailAlignment : getDetailAlignment(activeItem, detailPanel);
|
|
331
|
+
const selectionWindow = shouldPreferDerivedSelection
|
|
332
|
+
? getSelectionWindow(activeItem, detailPanel, liveItems)
|
|
333
|
+
: rawSelectionWindow !== undefined ? rawSelectionWindow : getSelectionWindow(activeItem, detailPanel, liveItems);
|
|
334
|
+
const recoveryHint = shouldPreferDerivedSelection
|
|
335
|
+
? getRecoveryHint(selectionWindow, liveItems, detailPanel)
|
|
336
|
+
: rawRecoveryHint !== undefined ? rawRecoveryHint : getRecoveryHint(selectionWindow, liveItems, detailPanel);
|
|
337
|
+
const outcomeSignals = shouldPreferDerivedSelection
|
|
338
|
+
? getOutcomeSignals(snapshot, composer, activeItem, detailAlignment, selectionWindow)
|
|
339
|
+
: rawOutcomeSignals !== undefined ? rawOutcomeSignals : getOutcomeSignals(snapshot, composer, activeItem, detailAlignment, selectionWindow);
|
|
340
|
+
const summary = getSummaryString({
|
|
341
|
+
workspaceSurface,
|
|
342
|
+
activeItem,
|
|
343
|
+
composer,
|
|
344
|
+
blockingModals,
|
|
345
|
+
loadingShell,
|
|
346
|
+
detailAlignment,
|
|
347
|
+
selectionWindow,
|
|
348
|
+
});
|
|
349
|
+
|
|
350
|
+
return {
|
|
351
|
+
workspace_surface: workspaceSurface,
|
|
352
|
+
active_item_label: activeItem?.label ?? null,
|
|
353
|
+
draft_present: composer?.draft_present === true,
|
|
354
|
+
loading_shell: loadingShell !== undefined ? loadingShell : getLoadingShell(snapshot),
|
|
355
|
+
blocking_modals: blockingModals,
|
|
356
|
+
blocking_modal_count: blockingModals.length,
|
|
357
|
+
blocking_modal_labels: blockingModals.map((modal) => compactText(modal?.label)).filter(Boolean),
|
|
358
|
+
detail_alignment: detailAlignment,
|
|
359
|
+
selection_window: selectionWindow,
|
|
360
|
+
recovery_hint: recoveryHint,
|
|
361
|
+
outcome_signals: outcomeSignals,
|
|
362
|
+
summary,
|
|
363
|
+
};
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
export function buildWorkspaceVerification(snapshot = {}) {
|
|
367
|
+
const summary = summarizeWorkspaceSnapshot(snapshot);
|
|
368
|
+
const activeItemLabel = summary.active_item_label ?? null;
|
|
369
|
+
const draftPresent = summary.draft_present === true;
|
|
370
|
+
const delivered = summary.outcome_signals?.delivered === true;
|
|
371
|
+
const loadingShell = summary.loading_shell === true;
|
|
372
|
+
const blockingModalPresent = summary.blocking_modal_count > 0;
|
|
373
|
+
const detailAlignment = summary.detail_alignment ?? 'unknown';
|
|
374
|
+
const outcomeSignals = summary.outcome_signals ?? {
|
|
375
|
+
delivered: false,
|
|
376
|
+
composer_cleared: false,
|
|
377
|
+
active_item_stable: false,
|
|
378
|
+
};
|
|
379
|
+
const workspaceSurface = summary.workspace_surface ?? pick(snapshot, 'workspaceSurface', 'workspace_surface', null);
|
|
380
|
+
const composer = getComposer(snapshot);
|
|
381
|
+
const actionControls = getActionControls(snapshot);
|
|
382
|
+
const activeItemStable = outcomeSignals?.active_item_stable === true;
|
|
383
|
+
const hasReliableSendControl = actionControls.some((control) => control?.action_kind === 'send' && compactText(control?.label));
|
|
384
|
+
const readyForNextAction = loadingShell || blockingModalPresent
|
|
385
|
+
? 'workspace_inspect'
|
|
386
|
+
: !activeItemLabel
|
|
387
|
+
? 'select_live_item'
|
|
388
|
+
: detailAlignment === 'mismatch'
|
|
389
|
+
? 'select_live_item'
|
|
390
|
+
: !activeItemStable
|
|
391
|
+
? 'workspace_inspect'
|
|
392
|
+
: !composer
|
|
393
|
+
? 'workspace_inspect'
|
|
394
|
+
: draftPresent
|
|
395
|
+
? (hasReliableSendControl ? 'execute_action' : 'workspace_inspect')
|
|
396
|
+
: isComposerSurface(workspaceSurface)
|
|
397
|
+
? 'draft_action'
|
|
398
|
+
: 'workspace_inspect';
|
|
399
|
+
|
|
400
|
+
return {
|
|
401
|
+
active_item_label: activeItemLabel,
|
|
402
|
+
draft_present: draftPresent,
|
|
403
|
+
delivered,
|
|
404
|
+
loading_shell: loadingShell,
|
|
405
|
+
blocking_modal_present: blockingModalPresent,
|
|
406
|
+
detail_alignment: detailAlignment,
|
|
407
|
+
outcome_signals: outcomeSignals,
|
|
408
|
+
ready_for_next_action: readyForNextAction,
|
|
409
|
+
};
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
export async function collectVisibleWorkspaceSnapshot(page, state) {
|
|
413
|
+
const rawSnapshot = await page.evaluate(() => {
|
|
414
|
+
if (typeof document === 'undefined') {
|
|
415
|
+
return {
|
|
416
|
+
bodyText: '',
|
|
417
|
+
live_items: [],
|
|
418
|
+
active_item: null,
|
|
419
|
+
detail_panel: null,
|
|
420
|
+
detail_alignment: 'unknown',
|
|
421
|
+
composer: null,
|
|
422
|
+
action_controls: [],
|
|
423
|
+
outcome_signals: {
|
|
424
|
+
delivered: false,
|
|
425
|
+
composer_cleared: false,
|
|
426
|
+
active_item_stable: false,
|
|
427
|
+
},
|
|
428
|
+
blocking_modals: [],
|
|
429
|
+
loading_shell: false,
|
|
430
|
+
};
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
function compactText(value) {
|
|
434
|
+
return String(value ?? '').replace(/\s+/g, ' ').trim();
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
function normalizeLabel(value) {
|
|
438
|
+
return compactText(value).toLowerCase();
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
function hasEnglishSuccessSignal(text) {
|
|
442
|
+
return /\b(delivered|sent)\b/i.test(text);
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
function hasEnglishComposerPromptText(text) {
|
|
446
|
+
return /\b(type|write|send)?\s*(a\s*)?(message|reply)\b/i.test(text)
|
|
447
|
+
|| /\bchat\b/i.test(text);
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
function getHintId(el) {
|
|
451
|
+
return el.getAttribute('data-grasp-id') || null;
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
function isVisible(el) {
|
|
455
|
+
const rect = el.getBoundingClientRect();
|
|
456
|
+
const style = window.getComputedStyle(el);
|
|
457
|
+
return rect.width > 0 && rect.height > 0 && style.visibility !== 'hidden' && style.display !== 'none';
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
function getText(el) {
|
|
461
|
+
return compactText(el.getAttribute('aria-label') || el.textContent || el.value || '');
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
const bodyText = compactText(document.body?.innerText);
|
|
465
|
+
const hasThreadPromptBodyText = bodyText.includes('按enter键发送')
|
|
466
|
+
|| bodyText.includes('发送消息')
|
|
467
|
+
|| bodyText.includes('发消息')
|
|
468
|
+
|| bodyText.includes('输入消息');
|
|
469
|
+
const hasExactLoadingShellBodyText = bodyText.includes('加载中,请稍候')
|
|
470
|
+
|| (bodyText.includes('加载中') && bodyText.includes('请稍候'))
|
|
471
|
+
|| bodyText.includes('正在加载');
|
|
472
|
+
|
|
473
|
+
function isSelected(el) {
|
|
474
|
+
const ariaCurrent = el.getAttribute('aria-current');
|
|
475
|
+
const classAttr = String(el.getAttribute('class') || '');
|
|
476
|
+
const hasStateClass = classAttr
|
|
477
|
+
.split(/\s+/)
|
|
478
|
+
.some((token) => /(^|[-_])(selected|current)($|[-_])/i.test(token));
|
|
479
|
+
return el.getAttribute('aria-selected') === 'true'
|
|
480
|
+
|| el.getAttribute('data-selected') === 'true'
|
|
481
|
+
|| ariaCurrent === 'true'
|
|
482
|
+
|| ariaCurrent === 'page'
|
|
483
|
+
|| ariaCurrent === 'step'
|
|
484
|
+
|| ariaCurrent === 'location'
|
|
485
|
+
|| hasStateClass
|
|
486
|
+
|| el.classList.contains('selected')
|
|
487
|
+
|| el.classList.contains('is-selected')
|
|
488
|
+
|| el.classList.contains('workspace-item--selected');
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
function isButtonLike(el) {
|
|
492
|
+
return el.matches('button, [role="button"], input[type="submit"], input[type="button"]');
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
function isActionLikeLabel(label) {
|
|
496
|
+
const text = normalizeLabel(label);
|
|
497
|
+
return text.includes('发送')
|
|
498
|
+
|| text.includes('send')
|
|
499
|
+
|| text.includes('回复')
|
|
500
|
+
|| text.includes('reply')
|
|
501
|
+
|| text.includes('提交')
|
|
502
|
+
|| text.includes('submit')
|
|
503
|
+
|| text.includes('取消')
|
|
504
|
+
|| text.includes('cancel')
|
|
505
|
+
|| text.includes('关闭')
|
|
506
|
+
|| text.includes('close');
|
|
507
|
+
}
|
|
508
|
+
|
|
509
|
+
const structuredItemSelector = 'li, [role="option"], [role="row"], [role="treeitem"], [data-list-item], [data-thread-item], [data-conversation-item]';
|
|
510
|
+
const navLeafSelector = 'a, [role="link"], [role="menuitem"], [role="tab"], button, [role="button"]';
|
|
511
|
+
|
|
512
|
+
function hasNestedNavLeaf(el) {
|
|
513
|
+
return Boolean(el.querySelector(navLeafSelector));
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
function isNavLeafCandidate(el) {
|
|
517
|
+
if (!isVisible(el)) return false;
|
|
518
|
+
if (!el.matches(navLeafSelector)) return false;
|
|
519
|
+
const label = getText(el);
|
|
520
|
+
if (!label || label.length > 60) return false;
|
|
521
|
+
const selectedState = el.getAttribute('aria-current') || el.getAttribute('aria-selected') === 'true';
|
|
522
|
+
const structuredNav = Boolean(el.closest('nav, [role="navigation"], [role="menu"], [role="tablist"]'));
|
|
523
|
+
const looseNav = structuredNav || Boolean(el.closest('aside, header'));
|
|
524
|
+
|
|
525
|
+
if (isButtonLike(el)) {
|
|
526
|
+
if (isActionLikeLabel(label)) return false;
|
|
527
|
+
return Boolean(structuredNav || selectedState);
|
|
528
|
+
}
|
|
529
|
+
|
|
530
|
+
return Boolean(
|
|
531
|
+
looseNav
|
|
532
|
+
|| selectedState
|
|
533
|
+
);
|
|
534
|
+
}
|
|
535
|
+
|
|
536
|
+
function isWorkspaceItemCandidate(el) {
|
|
537
|
+
if (!isVisible(el)) return false;
|
|
538
|
+
if (isNavLeafCandidate(el)) return true;
|
|
539
|
+
if (el.closest('button, a, [role="button"], [role="link"], [role="menuitem"], [role="tab"], input, textarea, select')) return false;
|
|
540
|
+
if (hasNestedNavLeaf(el)) return false;
|
|
541
|
+
return el.matches(structuredItemSelector);
|
|
542
|
+
}
|
|
543
|
+
|
|
544
|
+
function readLiveItem(el) {
|
|
545
|
+
const label = getText(el);
|
|
546
|
+
if (!label || label.length > 120) return null;
|
|
547
|
+
return {
|
|
548
|
+
label,
|
|
549
|
+
normalized_label: normalizeLabel(label),
|
|
550
|
+
hint_id: getHintId(el),
|
|
551
|
+
selected: isSelected(el),
|
|
552
|
+
};
|
|
553
|
+
}
|
|
554
|
+
|
|
555
|
+
function readDetailPanel() {
|
|
556
|
+
const candidates = [...document.querySelectorAll('[data-detail-panel], [role="complementary"], .detail-panel, aside')];
|
|
557
|
+
const visible = candidates.find(isVisible);
|
|
558
|
+
if (!visible) return null;
|
|
559
|
+
const label = getText(visible.querySelector('h1, h2, h3, h4, h5, h6') || visible);
|
|
560
|
+
return label ? {
|
|
561
|
+
label,
|
|
562
|
+
normalized_label: normalizeLabel(label),
|
|
563
|
+
hint_id: getHintId(visible),
|
|
564
|
+
selected: false,
|
|
565
|
+
} : null;
|
|
566
|
+
}
|
|
567
|
+
|
|
568
|
+
function readComposer() {
|
|
569
|
+
const candidates = [...document.querySelectorAll('textarea, input:not([type="hidden"]), [contenteditable="true"], [role="textbox"]')];
|
|
570
|
+
const visible = candidates.find(isVisible);
|
|
571
|
+
if (!visible) return null;
|
|
572
|
+
const draftText = compactText('value' in visible ? visible.value : visible.textContent);
|
|
573
|
+
const hintText = compactText([
|
|
574
|
+
visible.getAttribute('placeholder'),
|
|
575
|
+
visible.getAttribute('aria-label'),
|
|
576
|
+
visible.getAttribute('title'),
|
|
577
|
+
].filter(Boolean).join(' ')).toLowerCase();
|
|
578
|
+
const messageHints = ['输入消息', '发消息', '发送消息', '回复', '说点什么', '写点什么', '输入内容', '按enter键发送', '聊天', 'type a message', 'write a message', 'send a message', 'write a reply', 'type your reply'];
|
|
579
|
+
const hasVisibleSendActionControl = [...document.querySelectorAll('button, [role="button"], input[type="submit"], input[type="button"]')]
|
|
580
|
+
.filter(isVisible)
|
|
581
|
+
.some((el) => {
|
|
582
|
+
const label = normalizeLabel(getText(el));
|
|
583
|
+
return label.includes('发送') || label.includes('send') || label.includes('回复') || label.includes('reply') || label.includes('提交') || label.includes('submit');
|
|
584
|
+
});
|
|
585
|
+
const hasHintText = messageHints.some((hint) => hintText.includes(hint))
|
|
586
|
+
|| (hasVisibleSendActionControl && hasEnglishComposerPromptText(hintText));
|
|
587
|
+
const hasPromptAndSend = (
|
|
588
|
+
(hasThreadPromptBodyText || hasEnglishComposerPromptText(bodyText))
|
|
589
|
+
&& hasVisibleSendActionControl
|
|
590
|
+
);
|
|
591
|
+
if (!hasHintText && !hasPromptAndSend) {
|
|
592
|
+
return null;
|
|
593
|
+
}
|
|
594
|
+
return {
|
|
595
|
+
kind: 'chat_composer',
|
|
596
|
+
hint_id: getHintId(visible),
|
|
597
|
+
draft_present: draftText.length > 0,
|
|
598
|
+
draft_text: draftText,
|
|
599
|
+
};
|
|
600
|
+
}
|
|
601
|
+
|
|
602
|
+
function readActionControls() {
|
|
603
|
+
return [...document.querySelectorAll('button, [role="button"], input[type="submit"], input[type="button"]')]
|
|
604
|
+
.filter(isVisible)
|
|
605
|
+
.map((el) => {
|
|
606
|
+
const label = getText(el);
|
|
607
|
+
return label ? {
|
|
608
|
+
label,
|
|
609
|
+
action_kind: (() => {
|
|
610
|
+
const text = normalizeLabel(label);
|
|
611
|
+
if (text.includes('发送') || text.includes('send') || text.includes('提交') || text.includes('回复')) {
|
|
612
|
+
return 'send';
|
|
613
|
+
}
|
|
614
|
+
if (text.includes('取消') || text.includes('关闭') || text.includes('close')) {
|
|
615
|
+
return 'dismiss';
|
|
616
|
+
}
|
|
617
|
+
return 'action';
|
|
618
|
+
})(),
|
|
619
|
+
hint_id: getHintId(el),
|
|
620
|
+
} : null;
|
|
621
|
+
})
|
|
622
|
+
.filter(Boolean);
|
|
623
|
+
}
|
|
624
|
+
|
|
625
|
+
function readBlockingModals() {
|
|
626
|
+
return [...document.querySelectorAll('[role="dialog"], [aria-modal="true"], dialog[open]')]
|
|
627
|
+
.filter(isVisible)
|
|
628
|
+
.map((el) => {
|
|
629
|
+
const label = getText(el.querySelector('h1, h2, h3, h4, h5, h6') || el);
|
|
630
|
+
return label ? {
|
|
631
|
+
label,
|
|
632
|
+
normalized_label: normalizeLabel(label),
|
|
633
|
+
hint_id: getHintId(el),
|
|
634
|
+
} : null;
|
|
635
|
+
})
|
|
636
|
+
.filter(Boolean);
|
|
637
|
+
}
|
|
638
|
+
|
|
639
|
+
const live_items = [...document.querySelectorAll(structuredItemSelector), ...document.querySelectorAll(navLeafSelector)]
|
|
640
|
+
.filter(isWorkspaceItemCandidate)
|
|
641
|
+
.map(readLiveItem)
|
|
642
|
+
.filter(Boolean)
|
|
643
|
+
.filter((item, index, items) => {
|
|
644
|
+
const key = `${item.hint_id ?? ''}|${item.normalized_label}`;
|
|
645
|
+
return items.findIndex((candidate) => `${candidate.hint_id ?? ''}|${candidate.normalized_label}` === key) === index;
|
|
646
|
+
});
|
|
647
|
+
const detail_panel = readDetailPanel();
|
|
648
|
+
const active_item = (() => {
|
|
649
|
+
const selectedLiveItems = live_items.filter((item) => item.selected);
|
|
650
|
+
if (selectedLiveItems.length === 1) return selectedLiveItems[0];
|
|
651
|
+
return null;
|
|
652
|
+
})();
|
|
653
|
+
const detail_alignment = active_item && detail_panel
|
|
654
|
+
? (active_item.normalized_label === detail_panel.normalized_label ? 'aligned' : 'mismatch')
|
|
655
|
+
: 'unknown';
|
|
656
|
+
const selection_window = active_item
|
|
657
|
+
? (live_items.some((item) => item.selected && item.normalized_label === active_item.normalized_label) ? 'visible' : detail_panel ? 'virtualized' : 'not_found')
|
|
658
|
+
: (detail_panel && live_items.length > 0 ? 'virtualized' : 'not_found');
|
|
659
|
+
const recovery_hint = selection_window === 'virtualized'
|
|
660
|
+
? 'scroll_list'
|
|
661
|
+
: (selection_window === 'not_found' ? (live_items.length > 0 ? 'scroll_list' : 'reinspect_workspace') : null);
|
|
662
|
+
const composer = readComposer();
|
|
663
|
+
const action_controls = readActionControls();
|
|
664
|
+
const blocking_modals = readBlockingModals();
|
|
665
|
+
const loadingIndicator = [...document.querySelectorAll('[aria-busy="true"], .loading, .skeleton, .spinner')]
|
|
666
|
+
.find(isVisible);
|
|
667
|
+
const loading_shell = !!(hasExactLoadingShellBodyText
|
|
668
|
+
|| (loadingIndicator && /加载中|请稍候|正在加载/.test(bodyText)));
|
|
669
|
+
const outcome_signals = {
|
|
670
|
+
delivered: /已发送|发送成功/i.test(bodyText) || hasEnglishSuccessSignal(bodyText),
|
|
671
|
+
composer_cleared: /已发送|发送成功/i.test(bodyText) || hasEnglishSuccessSignal(bodyText),
|
|
672
|
+
active_item_stable: !!active_item && detail_alignment === 'aligned' && selection_window === 'visible',
|
|
673
|
+
};
|
|
674
|
+
|
|
675
|
+
return {
|
|
676
|
+
bodyText,
|
|
677
|
+
live_items,
|
|
678
|
+
active_item,
|
|
679
|
+
detail_panel,
|
|
680
|
+
detail_alignment,
|
|
681
|
+
composer,
|
|
682
|
+
action_controls,
|
|
683
|
+
outcome_signals,
|
|
684
|
+
blocking_modals,
|
|
685
|
+
loading_shell,
|
|
686
|
+
selection_window,
|
|
687
|
+
recovery_hint,
|
|
688
|
+
};
|
|
689
|
+
});
|
|
690
|
+
|
|
691
|
+
const hintLiveItems = deriveWorkspaceHintItems(state?.hintMap ?? []);
|
|
692
|
+
const mergedLiveItems = [...hintLiveItems, ...getLiveItems(rawSnapshot)]
|
|
693
|
+
.filter((item, index, items) => {
|
|
694
|
+
const key = `${compactText(item?.hint_id)}|${normalizeLabel(item?.normalized_label ?? item?.label)}`;
|
|
695
|
+
return items.findIndex((candidate) => `${compactText(candidate?.hint_id)}|${normalizeLabel(candidate?.normalized_label ?? candidate?.label)}` === key) === index;
|
|
696
|
+
});
|
|
697
|
+
const detailPanel = getDetailPanel(rawSnapshot);
|
|
698
|
+
const rawActiveItem = rawSnapshot?.active_item !== undefined ? rawSnapshot.active_item : rawSnapshot?.activeItem;
|
|
699
|
+
const mergedActiveItem = rawActiveItem ?? getActiveItem({}, mergedLiveItems, detailPanel);
|
|
700
|
+
const shouldReconcileSelection = rawActiveItem == null && mergedActiveItem != null;
|
|
701
|
+
const detailAlignment = shouldReconcileSelection
|
|
702
|
+
? getDetailAlignment(mergedActiveItem, detailPanel)
|
|
703
|
+
: pick(rawSnapshot, 'detailAlignment', 'detail_alignment', undefined);
|
|
704
|
+
const selectionWindow = shouldReconcileSelection
|
|
705
|
+
? getSelectionWindow(mergedActiveItem, detailPanel, mergedLiveItems)
|
|
706
|
+
: pick(rawSnapshot, 'selectionWindow', 'selection_window', undefined);
|
|
707
|
+
const recoveryHint = shouldReconcileSelection
|
|
708
|
+
? getRecoveryHint(selectionWindow, mergedLiveItems, detailPanel)
|
|
709
|
+
: pick(rawSnapshot, 'recoveryHint', 'recovery_hint', undefined);
|
|
710
|
+
const outcomeSignals = shouldReconcileSelection
|
|
711
|
+
? getOutcomeSignals(rawSnapshot, getComposer(rawSnapshot), mergedActiveItem, detailAlignment, selectionWindow)
|
|
712
|
+
: pick(rawSnapshot, 'outcomeSignals', 'outcome_signals', undefined);
|
|
713
|
+
const snapshot = {
|
|
714
|
+
...rawSnapshot,
|
|
715
|
+
live_items: mergedLiveItems,
|
|
716
|
+
...(shouldReconcileSelection
|
|
717
|
+
? {
|
|
718
|
+
active_item: mergedActiveItem,
|
|
719
|
+
detail_alignment: detailAlignment,
|
|
720
|
+
selection_window: selectionWindow,
|
|
721
|
+
recovery_hint: recoveryHint,
|
|
722
|
+
outcome_signals: outcomeSignals,
|
|
723
|
+
}
|
|
724
|
+
: {}),
|
|
725
|
+
workspace_surface: classifyWorkspaceSurface({
|
|
726
|
+
...rawSnapshot,
|
|
727
|
+
live_items: mergedLiveItems,
|
|
728
|
+
}),
|
|
729
|
+
};
|
|
730
|
+
|
|
731
|
+
return {
|
|
732
|
+
...snapshot,
|
|
733
|
+
summary: summarizeWorkspaceSnapshot(snapshot),
|
|
734
|
+
};
|
|
735
|
+
}
|