@yuzc-001/grasp 0.6.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +327 -0
- package/README.zh-CN.md +324 -0
- package/examples/README.md +31 -0
- package/examples/claude-desktop.json +8 -0
- package/examples/codex-config.toml +4 -0
- package/grasp.skill +0 -0
- package/index.js +87 -0
- package/package.json +48 -0
- package/scripts/grasp_openclaw_ctl.sh +122 -0
- package/scripts/run-search-benchmark.mjs +287 -0
- package/scripts/update-star-history.mjs +274 -0
- package/skill/SKILL.md +61 -0
- package/skill/references/tools.md +306 -0
- package/src/cli/auto-configure.js +116 -0
- package/src/cli/cmd-connect.js +148 -0
- package/src/cli/cmd-explain.js +42 -0
- package/src/cli/cmd-logs.js +55 -0
- package/src/cli/cmd-status.js +119 -0
- package/src/cli/config.js +27 -0
- package/src/cli/detect-chrome.js +58 -0
- package/src/grasp/handoff/events.js +67 -0
- package/src/grasp/handoff/persist.js +48 -0
- package/src/grasp/handoff/state.js +28 -0
- package/src/grasp/page/capture.js +34 -0
- package/src/grasp/page/state.js +273 -0
- package/src/grasp/verify/evidence.js +40 -0
- package/src/grasp/verify/pipeline.js +52 -0
- package/src/layer1-bridge/chrome.js +416 -0
- package/src/layer1-bridge/webmcp.js +143 -0
- package/src/layer2-perception/hints.js +284 -0
- package/src/layer3-action/actions.js +400 -0
- package/src/runtime/browser-instance.js +65 -0
- package/src/runtime/truth/model.js +94 -0
- package/src/runtime/truth/snapshot.js +51 -0
- package/src/server/affordances.js +47 -0
- package/src/server/audit.js +122 -0
- package/src/server/boss-fast-path.js +164 -0
- package/src/server/boundary-guard.js +53 -0
- package/src/server/content.js +97 -0
- package/src/server/continuity.js +256 -0
- package/src/server/engine-selection.js +29 -0
- package/src/server/entry-orchestrator.js +115 -0
- package/src/server/error-codes.js +7 -0
- package/src/server/explain-share-card.js +113 -0
- package/src/server/fast-path-router.js +134 -0
- package/src/server/form-runtime.js +602 -0
- package/src/server/form-tasks.js +254 -0
- package/src/server/gateway-response.js +62 -0
- package/src/server/index.js +22 -0
- package/src/server/observe.js +52 -0
- package/src/server/page-projection.js +31 -0
- package/src/server/page-state.js +27 -0
- package/src/server/postconditions.js +128 -0
- package/src/server/prompt-assembly.js +148 -0
- package/src/server/responses.js +44 -0
- package/src/server/route-boundary.js +174 -0
- package/src/server/route-policy.js +168 -0
- package/src/server/runtime-confirmation.js +87 -0
- package/src/server/runtime-status.js +7 -0
- package/src/server/share-artifacts.js +284 -0
- package/src/server/state.js +132 -0
- package/src/server/structured-extraction.js +131 -0
- package/src/server/surface-prompts.js +166 -0
- package/src/server/task-frame.js +11 -0
- package/src/server/tasks/search-task.js +321 -0
- package/src/server/tools.actions.js +1361 -0
- package/src/server/tools.form.js +526 -0
- package/src/server/tools.gateway.js +757 -0
- package/src/server/tools.handoff.js +210 -0
- package/src/server/tools.js +20 -0
- package/src/server/tools.legacy.js +983 -0
- package/src/server/tools.strategy.js +250 -0
- package/src/server/tools.task-surface.js +66 -0
- package/src/server/tools.workspace.js +873 -0
- package/src/server/workspace-runtime.js +1138 -0
- package/src/server/workspace-tasks.js +735 -0
- package/start-chrome.bat +84 -0
|
@@ -0,0 +1,1361 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
|
|
3
|
+
import { getActivePage, getTabs, navigateTo, switchTab, newTab, closeTab } from '../layer1-bridge/chrome.js';
|
|
4
|
+
import { clickByHintId, typeByHintId, hoverByHintId, pressKey, watchElement, scroll, findScrollableAncestor } from '../layer3-action/actions.js';
|
|
5
|
+
import { errorResponse, imageResponse, textResponse } from './responses.js';
|
|
6
|
+
import { describeMode, syncPageState } from './state.js';
|
|
7
|
+
import { audit } from './audit.js';
|
|
8
|
+
import { verifyTypeResult, verifyGenericAction } from './postconditions.js';
|
|
9
|
+
import { TYPE_FAILED } from './error-codes.js';
|
|
10
|
+
import { runVerifiedAction } from '../grasp/verify/pipeline.js';
|
|
11
|
+
import { readHandoffState } from '../grasp/handoff/persist.js';
|
|
12
|
+
import { extractMainContent } from './content.js';
|
|
13
|
+
import { readFastPath } from './fast-path-router.js';
|
|
14
|
+
import { buildPageProjection } from './page-projection.js';
|
|
15
|
+
import { selectEngine } from './engine-selection.js';
|
|
16
|
+
import { readLatestRouteDecision } from './audit.js';
|
|
17
|
+
import { readBrowserInstance } from '../runtime/browser-instance.js';
|
|
18
|
+
import { buildRuntimeConfirmationSuccessResponse, getRuntimeConfirmationSummary, requireConfirmedRuntimeInstance, storeRuntimeConfirmation } from './runtime-confirmation.js';
|
|
19
|
+
|
|
20
|
+
function buildStructuredError(message, normalizedHintId, verdict) {
|
|
21
|
+
const meta = {
|
|
22
|
+
error_code: verdict?.error_code ?? TYPE_FAILED,
|
|
23
|
+
retryable: verdict?.retryable ?? true,
|
|
24
|
+
suggested_next_step: verdict?.suggested_next_step ?? 'retry',
|
|
25
|
+
evidence: verdict?.evidence ?? { hint_id: normalizedHintId },
|
|
26
|
+
};
|
|
27
|
+
return errorResponse(message, meta);
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
function createRebuildHints(page, state) {
|
|
31
|
+
return async () => {
|
|
32
|
+
await syncPageState(page, state, { force: true });
|
|
33
|
+
return null;
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function normalizeQuery(value) {
|
|
38
|
+
return String(value ?? '').trim().toLowerCase();
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function listUserTabs(tabs = [], activeUrl) {
|
|
42
|
+
return tabs
|
|
43
|
+
.filter((tab) => tab.isUser)
|
|
44
|
+
.map((tab) => ({
|
|
45
|
+
...tab,
|
|
46
|
+
active: tab.url === activeUrl,
|
|
47
|
+
}));
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function matchVisibleTabs(tabs = [], { query = '', title_contains = '', url_contains = '' } = {}) {
|
|
51
|
+
const normalizedQuery = normalizeQuery(query);
|
|
52
|
+
const normalizedTitle = normalizeQuery(title_contains);
|
|
53
|
+
const normalizedUrl = normalizeQuery(url_contains);
|
|
54
|
+
|
|
55
|
+
return tabs.filter((tab) => {
|
|
56
|
+
const title = normalizeQuery(tab.title);
|
|
57
|
+
const url = normalizeQuery(tab.url);
|
|
58
|
+
if (normalizedTitle && !title.includes(normalizedTitle)) return false;
|
|
59
|
+
if (normalizedUrl && !url.includes(normalizedUrl)) return false;
|
|
60
|
+
if (!normalizedQuery) return true;
|
|
61
|
+
return title.includes(normalizedQuery) || url.includes(normalizedQuery);
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export function registerActionTools(server, state, deps = {}) {
|
|
66
|
+
const getPage = deps.getActivePage ?? getActivePage;
|
|
67
|
+
const listTabs = deps.getTabs ?? getTabs;
|
|
68
|
+
const activateTab = deps.switchTab ?? switchTab;
|
|
69
|
+
const openTab = deps.newTab ?? newTab;
|
|
70
|
+
const closeBrowserTab = deps.closeTab ?? closeTab;
|
|
71
|
+
const syncState = deps.syncPageState ?? syncPageState;
|
|
72
|
+
const extractContent = deps.extractMainContent ?? extractMainContent;
|
|
73
|
+
const navigate = deps.navigateTo ?? navigateTo;
|
|
74
|
+
const getBrowserInstance = deps.getBrowserInstance ?? (() => readBrowserInstance(process.env.CHROME_CDP_URL || 'http://localhost:9222'));
|
|
75
|
+
const readFastPathContent = deps.readFastPath ?? readFastPath;
|
|
76
|
+
|
|
77
|
+
const dialogListeners = new WeakSet();
|
|
78
|
+
const consoleListeners = new WeakSet();
|
|
79
|
+
state.pendingDialog = state.pendingDialog ?? null;
|
|
80
|
+
async function ensureDialogListener(page) {
|
|
81
|
+
if (!page || typeof page.on !== 'function') return;
|
|
82
|
+
if (dialogListeners.has(page)) return;
|
|
83
|
+
dialogListeners.add(page);
|
|
84
|
+
page.on('dialog', (dialog) => {
|
|
85
|
+
state.pendingDialog = {
|
|
86
|
+
type: dialog.type(),
|
|
87
|
+
message: dialog.message(),
|
|
88
|
+
defaultValue: dialog.defaultValue(),
|
|
89
|
+
ref: dialog,
|
|
90
|
+
};
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
if (!state.consoleLogs) state.consoleLogs = [];
|
|
95
|
+
function ensureConsoleListener(page) {
|
|
96
|
+
if (!page || typeof page.on !== 'function') return;
|
|
97
|
+
if (consoleListeners.has(page)) return;
|
|
98
|
+
consoleListeners.add(page);
|
|
99
|
+
page.on('console', (msg) => {
|
|
100
|
+
state.consoleLogs.push({
|
|
101
|
+
level: msg.type(),
|
|
102
|
+
text: msg.text(),
|
|
103
|
+
url: msg.location?.()?.url ?? '',
|
|
104
|
+
lineNumber: msg.location?.()?.lineNumber ?? 0,
|
|
105
|
+
timestamp: Date.now(),
|
|
106
|
+
});
|
|
107
|
+
if (state.consoleLogs.length > 200) state.consoleLogs.shift();
|
|
108
|
+
});
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
async function getPageWithListeners(opts) {
|
|
112
|
+
const page = await getPage(opts);
|
|
113
|
+
await ensureDialogListener(page);
|
|
114
|
+
ensureConsoleListener(page);
|
|
115
|
+
return page;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
async function requireActionConfirmation(toolName) {
|
|
119
|
+
const instance = await getBrowserInstance();
|
|
120
|
+
return requireConfirmedRuntimeInstance(state, instance, toolName);
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
server.registerTool(
|
|
124
|
+
'list_visible_tabs',
|
|
125
|
+
{
|
|
126
|
+
description: 'List user-visible tabs in the current runtime and mark which one is active.',
|
|
127
|
+
inputSchema: {},
|
|
128
|
+
},
|
|
129
|
+
async () => {
|
|
130
|
+
const page = await getPage({ state });
|
|
131
|
+
const tabs = listUserTabs(await listTabs(), page.url());
|
|
132
|
+
|
|
133
|
+
return textResponse([
|
|
134
|
+
`Visible tabs: ${tabs.length}`,
|
|
135
|
+
'',
|
|
136
|
+
...tabs.map((tab) => `[${tab.index}] ${tab.title}${tab.active ? ' (active)' : ''}\n${tab.url}`),
|
|
137
|
+
], { tabs });
|
|
138
|
+
}
|
|
139
|
+
);
|
|
140
|
+
|
|
141
|
+
server.registerTool(
|
|
142
|
+
'select_visible_tab',
|
|
143
|
+
{
|
|
144
|
+
description: 'Bring a visible runtime tab to the front by matching its title or URL fragment.',
|
|
145
|
+
inputSchema: {
|
|
146
|
+
query: z.string().optional().describe('Text fragment that may match either the tab title or the tab URL'),
|
|
147
|
+
title_contains: z.string().optional().describe('Optional title fragment to narrow tab selection'),
|
|
148
|
+
url_contains: z.string().optional().describe('Optional URL fragment to narrow tab selection'),
|
|
149
|
+
},
|
|
150
|
+
},
|
|
151
|
+
async ({ query = '', title_contains = '', url_contains = '' } = {}) => {
|
|
152
|
+
const instance = await getBrowserInstance();
|
|
153
|
+
const confirmationError = requireConfirmedRuntimeInstance(state, instance, 'select_visible_tab');
|
|
154
|
+
if (confirmationError) return confirmationError;
|
|
155
|
+
|
|
156
|
+
const tabs = listUserTabs(await listTabs());
|
|
157
|
+
const matches = matchVisibleTabs(tabs, { query, title_contains, url_contains });
|
|
158
|
+
|
|
159
|
+
if (matches.length === 0) {
|
|
160
|
+
return errorResponse('No visible tab matched the provided query.', {
|
|
161
|
+
error_code: 'TAB_NOT_FOUND',
|
|
162
|
+
retryable: true,
|
|
163
|
+
suggested_next_step: 'list_visible_tabs',
|
|
164
|
+
tabs,
|
|
165
|
+
});
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
if (matches.length > 1) {
|
|
169
|
+
return errorResponse('Multiple visible tabs matched the provided query.', {
|
|
170
|
+
error_code: 'TAB_AMBIGUOUS',
|
|
171
|
+
retryable: true,
|
|
172
|
+
suggested_next_step: 'list_visible_tabs',
|
|
173
|
+
candidates: matches,
|
|
174
|
+
});
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
const selectedTab = matches[0];
|
|
178
|
+
const page = await activateTab(selectedTab.index);
|
|
179
|
+
await syncState(page, state, { force: true });
|
|
180
|
+
|
|
181
|
+
return textResponse([
|
|
182
|
+
`Selected tab [${selectedTab.index}]: ${selectedTab.title}`,
|
|
183
|
+
`URL: ${selectedTab.url}`,
|
|
184
|
+
`Page role: ${state.pageState?.currentRole ?? 'unknown'}`,
|
|
185
|
+
], { tab: selectedTab });
|
|
186
|
+
}
|
|
187
|
+
);
|
|
188
|
+
|
|
189
|
+
server.registerTool(
|
|
190
|
+
'confirm_runtime_instance',
|
|
191
|
+
{
|
|
192
|
+
description: 'Confirm the current runtime browser instance before performing page-changing actions.',
|
|
193
|
+
inputSchema: {
|
|
194
|
+
display: z.enum(['windowed', 'headless', 'unknown']).describe('The runtime instance mode you expect to act against'),
|
|
195
|
+
},
|
|
196
|
+
},
|
|
197
|
+
async ({ display }) => {
|
|
198
|
+
const instance = await getBrowserInstance();
|
|
199
|
+
if (!instance) {
|
|
200
|
+
return errorResponse('Runtime instance unavailable. Call get_status and try again.');
|
|
201
|
+
}
|
|
202
|
+
if ((instance.display ?? 'unknown') !== display) {
|
|
203
|
+
return errorResponse([
|
|
204
|
+
'Runtime instance mismatch.',
|
|
205
|
+
`Expected: ${display}`,
|
|
206
|
+
`Actual: ${instance.display ?? 'unknown'}`,
|
|
207
|
+
...(instance.browser ? [`Browser: ${instance.browser}`] : []),
|
|
208
|
+
], {
|
|
209
|
+
error_code: 'INSTANCE_CONFIRMATION_MISMATCH',
|
|
210
|
+
retryable: true,
|
|
211
|
+
suggested_next_step: 'get_status',
|
|
212
|
+
instance,
|
|
213
|
+
});
|
|
214
|
+
}
|
|
215
|
+
const confirmation = storeRuntimeConfirmation(state, instance);
|
|
216
|
+
return buildRuntimeConfirmationSuccessResponse(confirmation, instance);
|
|
217
|
+
}
|
|
218
|
+
);
|
|
219
|
+
|
|
220
|
+
server.registerTool(
|
|
221
|
+
'navigate',
|
|
222
|
+
{
|
|
223
|
+
description: 'Navigate the browser to a URL and refresh Grasp page state.',
|
|
224
|
+
inputSchema: { url: z.string().url().describe('Full URL to navigate to') },
|
|
225
|
+
},
|
|
226
|
+
async ({ url }) => {
|
|
227
|
+
try {
|
|
228
|
+
const instance = await getBrowserInstance();
|
|
229
|
+
const confirmationError = requireConfirmedRuntimeInstance(state, instance, 'navigate');
|
|
230
|
+
if (confirmationError) return confirmationError;
|
|
231
|
+
const page = await navigate(url, { state });
|
|
232
|
+
await syncState(page, state, { force: true });
|
|
233
|
+
await audit('navigate', url, null, state);
|
|
234
|
+
return textResponse([
|
|
235
|
+
`Navigated to: ${url}`,
|
|
236
|
+
`Page title: ${await page.title()}`,
|
|
237
|
+
`CDP mode - ${state.hintMap.length} interactive elements found.`,
|
|
238
|
+
'Use get_hint_map to inspect the current interaction map.',
|
|
239
|
+
]);
|
|
240
|
+
} catch (err) {
|
|
241
|
+
return errorResponse(`Navigation failed: ${err.message}`);
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
);
|
|
245
|
+
|
|
246
|
+
server.registerTool(
|
|
247
|
+
'get_status',
|
|
248
|
+
{
|
|
249
|
+
description: 'Get current Grasp status, including page grasp and handoff state.',
|
|
250
|
+
inputSchema: {},
|
|
251
|
+
},
|
|
252
|
+
async () => {
|
|
253
|
+
try {
|
|
254
|
+
const page = await getPage({ state });
|
|
255
|
+
await syncState(page, state);
|
|
256
|
+
state.handoff = await readHandoffState();
|
|
257
|
+
const handoff = state.handoff ?? { state: 'idle' };
|
|
258
|
+
const pageState = state.pageState ?? {};
|
|
259
|
+
const route = state.lastRouteTrace ?? await readLatestRouteDecision();
|
|
260
|
+
const { mode, detail } = describeMode(state);
|
|
261
|
+
const instance = await getBrowserInstance();
|
|
262
|
+
const confirmation = getRuntimeConfirmationSummary(state, instance);
|
|
263
|
+
|
|
264
|
+
return textResponse([
|
|
265
|
+
'Grasp is connected',
|
|
266
|
+
'',
|
|
267
|
+
`Page: ${await page.title()}`,
|
|
268
|
+
`URL: ${page.url()}`,
|
|
269
|
+
`Mode: ${mode}`,
|
|
270
|
+
` ${detail}`,
|
|
271
|
+
...(instance?.browser ? [`Browser instance: ${instance.browser}`] : []),
|
|
272
|
+
...(instance?.display ? [`Instance mode: ${instance.display}`] : []),
|
|
273
|
+
...(instance?.warning ? [`Instance warning: ${instance.warning}`] : []),
|
|
274
|
+
`Instance confirmed: ${confirmation.confirmed ? 'yes' : 'no'}`,
|
|
275
|
+
`Hint Map: ${state.hintMap?.length ?? 0} elements cached`,
|
|
276
|
+
`Page role: ${pageState.currentRole ?? 'unknown'}`,
|
|
277
|
+
`Grasp confidence: ${pageState.graspConfidence ?? 'unknown'}`,
|
|
278
|
+
`Reacquired: ${pageState.reacquired ? 'yes' : 'no'}`,
|
|
279
|
+
`Risk gate detected: ${pageState.riskGateDetected ? 'yes' : 'no'}`,
|
|
280
|
+
...(pageState.checkpointKind ? [`Checkpoint kind: ${pageState.checkpointKind}`] : []),
|
|
281
|
+
...(pageState.checkpointSignals?.length ? [`Checkpoint signals: ${pageState.checkpointSignals.join(', ')}`] : []),
|
|
282
|
+
...(pageState.suggestedNextAction ? [`Suggested next action: ${pageState.suggestedNextAction}`] : []),
|
|
283
|
+
...(route?.selected_mode ? [`Last route: ${route.selected_mode}`] : []),
|
|
284
|
+
...(route?.next_step ? [`Route next step: ${route.next_step}`] : []),
|
|
285
|
+
`Handoff: ${handoff.state}`,
|
|
286
|
+
...(handoff.reason ? [` reason: ${handoff.reason}`] : []),
|
|
287
|
+
], { handoff, pageState, ...(instance ? { instance } : {}), ...(route ? { route } : {}) });
|
|
288
|
+
} catch (err) {
|
|
289
|
+
return errorResponse(`Grasp is NOT connected.\n${err.message}`);
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
);
|
|
293
|
+
|
|
294
|
+
server.registerTool(
|
|
295
|
+
'get_page_summary',
|
|
296
|
+
{
|
|
297
|
+
description: 'Extract a concise summary of the current page.',
|
|
298
|
+
inputSchema: {},
|
|
299
|
+
},
|
|
300
|
+
async () => {
|
|
301
|
+
const page = await getPage({ state });
|
|
302
|
+
const selection = selectEngine({ tool: 'get_page_summary', url: page.url() });
|
|
303
|
+
let fastPath = null;
|
|
304
|
+
|
|
305
|
+
if (selection.engine === 'runtime') {
|
|
306
|
+
await syncState(page, state, { force: true });
|
|
307
|
+
fastPath = await readFastPathContent(page);
|
|
308
|
+
} else {
|
|
309
|
+
await syncState(page, state);
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
const main = fastPath ?? await extractContent(page);
|
|
313
|
+
const result = buildPageProjection({
|
|
314
|
+
...selection,
|
|
315
|
+
surface: fastPath?.surface ?? 'content',
|
|
316
|
+
title: main.title,
|
|
317
|
+
url: fastPath?.url ?? page.url(),
|
|
318
|
+
mainText: fastPath?.mainText ?? main.text,
|
|
319
|
+
});
|
|
320
|
+
const { summary } = describeMode(state);
|
|
321
|
+
return textResponse([
|
|
322
|
+
`Title: ${result.title}`,
|
|
323
|
+
`URL: ${result.url}`,
|
|
324
|
+
`Mode: ${summary}`,
|
|
325
|
+
'',
|
|
326
|
+
'Visible content (truncated):',
|
|
327
|
+
result.main_text.slice(0, 2000),
|
|
328
|
+
], { result });
|
|
329
|
+
}
|
|
330
|
+
);
|
|
331
|
+
|
|
332
|
+
server.registerTool(
|
|
333
|
+
'get_hint_map',
|
|
334
|
+
{
|
|
335
|
+
description: 'Return the current Hint Map for interactive elements on the page.',
|
|
336
|
+
inputSchema: {
|
|
337
|
+
filter: z.string().optional().describe('Optional text filter for hint labels'),
|
|
338
|
+
},
|
|
339
|
+
},
|
|
340
|
+
async ({ filter } = {}) => {
|
|
341
|
+
const page = await getPage({ state });
|
|
342
|
+
await syncPageState(page, state, { force: true });
|
|
343
|
+
const query = (filter ?? '').trim().toLowerCase();
|
|
344
|
+
const hints = query
|
|
345
|
+
? state.hintMap.filter((hint) => hint.label.toLowerCase().includes(query))
|
|
346
|
+
: state.hintMap;
|
|
347
|
+
|
|
348
|
+
if (hints.length === 0) {
|
|
349
|
+
return textResponse(query ? `No hints matched filter: ${filter}` : 'No interactive elements found.');
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
return textResponse([
|
|
353
|
+
`Hint Map (${hints.length} elements):`,
|
|
354
|
+
'',
|
|
355
|
+
...hints.map((hint) => `[${hint.id}] ${hint.label} (${hint.type}, pos:${hint.x},${hint.y})`),
|
|
356
|
+
], { hints });
|
|
357
|
+
}
|
|
358
|
+
);
|
|
359
|
+
|
|
360
|
+
server.registerTool(
|
|
361
|
+
'click',
|
|
362
|
+
{
|
|
363
|
+
description: 'Click an element by hint ID and verify the result.',
|
|
364
|
+
inputSchema: {
|
|
365
|
+
hint_id: z.string().describe('Hint ID from get_hint_map'),
|
|
366
|
+
},
|
|
367
|
+
},
|
|
368
|
+
async ({ hint_id }) => {
|
|
369
|
+
const normalizedHintId = String(hint_id).trim();
|
|
370
|
+
const instance = await getBrowserInstance();
|
|
371
|
+
const confirmationError = requireConfirmedRuntimeInstance(state, instance, 'click');
|
|
372
|
+
if (confirmationError) return confirmationError;
|
|
373
|
+
const page = await getPage({ state });
|
|
374
|
+
await syncPageState(page, state);
|
|
375
|
+
const prevDomRevision = state.pageState?.domRevision ?? 0;
|
|
376
|
+
const prevUrl = page.url();
|
|
377
|
+
const prevActiveId = await page.evaluate(() => document.activeElement?.getAttribute('data-grasp-id') ?? null);
|
|
378
|
+
const rebuildHints = createRebuildHints(page, state);
|
|
379
|
+
|
|
380
|
+
return runVerifiedAction({
|
|
381
|
+
action: 'click',
|
|
382
|
+
page,
|
|
383
|
+
state,
|
|
384
|
+
baseEvidence: { hint_id: normalizedHintId },
|
|
385
|
+
execute: async () => {
|
|
386
|
+
const result = await clickByHintId(page, normalizedHintId, { rebuildHints });
|
|
387
|
+
await syncPageState(page, state, { force: true });
|
|
388
|
+
return result;
|
|
389
|
+
},
|
|
390
|
+
verify: async () => verifyGenericAction({
|
|
391
|
+
page,
|
|
392
|
+
hintId: normalizedHintId,
|
|
393
|
+
prevDomRevision,
|
|
394
|
+
prevUrl,
|
|
395
|
+
prevActiveId,
|
|
396
|
+
newDomRevision: state.pageState?.domRevision ?? 0,
|
|
397
|
+
}),
|
|
398
|
+
onFailure: async (failure) => {
|
|
399
|
+
await audit('click_failed', `[${normalizedHintId}] ${failure.error_code}`, null, state);
|
|
400
|
+
return buildStructuredError(`Click verification failed for [${normalizedHintId}]`, normalizedHintId, failure);
|
|
401
|
+
},
|
|
402
|
+
onSuccess: async ({ executionResult, evidence }) => {
|
|
403
|
+
await audit('click', `[${normalizedHintId}] "${executionResult.label}"`, null, state);
|
|
404
|
+
const urlAfter = page.url();
|
|
405
|
+
const nav = urlAfter !== prevUrl ? `\nNavigated to: ${urlAfter}` : '';
|
|
406
|
+
return textResponse(
|
|
407
|
+
`Clicked [${normalizedHintId}]: "${executionResult.label}"${nav}\nPage now has ${state.hintMap.length} elements. Call get_hint_map to inspect the new state.`,
|
|
408
|
+
{ evidence }
|
|
409
|
+
);
|
|
410
|
+
},
|
|
411
|
+
});
|
|
412
|
+
}
|
|
413
|
+
);
|
|
414
|
+
|
|
415
|
+
server.registerTool(
|
|
416
|
+
'type',
|
|
417
|
+
{
|
|
418
|
+
description: 'Type text into an element by hint ID and verify the result.',
|
|
419
|
+
inputSchema: {
|
|
420
|
+
hint_id: z.string().describe('Hint ID from get_hint_map'),
|
|
421
|
+
text: z.string().describe('Text to type'),
|
|
422
|
+
press_enter: z.boolean().optional().describe('Press Enter after typing'),
|
|
423
|
+
},
|
|
424
|
+
},
|
|
425
|
+
async ({ hint_id, text, press_enter = false }) => {
|
|
426
|
+
const normalizedHintId = String(hint_id).trim();
|
|
427
|
+
const instance = await getBrowserInstance();
|
|
428
|
+
const confirmationError = requireConfirmedRuntimeInstance(state, instance, 'type');
|
|
429
|
+
if (confirmationError) return confirmationError;
|
|
430
|
+
const page = await getPage({ state });
|
|
431
|
+
await syncPageState(page, state);
|
|
432
|
+
const prevDomRevision = state.pageState?.domRevision ?? 0;
|
|
433
|
+
const prevUrl = page.url();
|
|
434
|
+
const rebuildHints = createRebuildHints(page, state);
|
|
435
|
+
|
|
436
|
+
return runVerifiedAction({
|
|
437
|
+
action: 'type',
|
|
438
|
+
page,
|
|
439
|
+
state,
|
|
440
|
+
baseEvidence: { hint_id: normalizedHintId },
|
|
441
|
+
execute: async () => {
|
|
442
|
+
await typeByHintId(page, normalizedHintId, text, press_enter, { rebuildHints });
|
|
443
|
+
await syncPageState(page, state, { force: true });
|
|
444
|
+
return { text, press_enter };
|
|
445
|
+
},
|
|
446
|
+
verify: async () => {
|
|
447
|
+
const newDomRevision = state.pageState?.domRevision ?? prevDomRevision;
|
|
448
|
+
return verifyTypeResult({
|
|
449
|
+
page,
|
|
450
|
+
expectedText: text,
|
|
451
|
+
allowPageChange: press_enter,
|
|
452
|
+
prevUrl,
|
|
453
|
+
prevDomRevision,
|
|
454
|
+
newDomRevision,
|
|
455
|
+
});
|
|
456
|
+
},
|
|
457
|
+
onFailure: async (failure) => {
|
|
458
|
+
await audit('type_failed', `[${normalizedHintId}] ${failure.error_code}`, null, state);
|
|
459
|
+
return buildStructuredError(`Type verification failed for [${normalizedHintId}]`, normalizedHintId, failure);
|
|
460
|
+
},
|
|
461
|
+
onSuccess: async ({ executionResult, evidence }) => {
|
|
462
|
+
await audit('type', `[${normalizedHintId}] "${executionResult.text.slice(0, 20)}${executionResult.text.length > 20 ? '...' : ''}"`, null, state);
|
|
463
|
+
return textResponse(
|
|
464
|
+
`Typed "${executionResult.text}" into [${normalizedHintId}]${executionResult.press_enter ? ' and pressed Enter' : ''}.`,
|
|
465
|
+
{ evidence }
|
|
466
|
+
);
|
|
467
|
+
},
|
|
468
|
+
});
|
|
469
|
+
}
|
|
470
|
+
);
|
|
471
|
+
|
|
472
|
+
server.registerTool(
|
|
473
|
+
'hover',
|
|
474
|
+
{
|
|
475
|
+
description: 'Hover an element by hint ID and refresh page state.',
|
|
476
|
+
inputSchema: {
|
|
477
|
+
hint_id: z.string().describe('Hint ID from get_hint_map'),
|
|
478
|
+
},
|
|
479
|
+
},
|
|
480
|
+
async ({ hint_id }) => {
|
|
481
|
+
const normalizedHintId = String(hint_id).trim();
|
|
482
|
+
const instance = await getBrowserInstance();
|
|
483
|
+
const confirmationError = requireConfirmedRuntimeInstance(state, instance, 'hover');
|
|
484
|
+
if (confirmationError) return confirmationError;
|
|
485
|
+
const page = await getPage({ state });
|
|
486
|
+
await syncPageState(page, state);
|
|
487
|
+
const prevUrl = page.url();
|
|
488
|
+
const rebuildHints = createRebuildHints(page, state);
|
|
489
|
+
|
|
490
|
+
try {
|
|
491
|
+
const result = await hoverByHintId(page, normalizedHintId, { rebuildHints });
|
|
492
|
+
await syncPageState(page, state, { force: true });
|
|
493
|
+
await audit('hover', `[${normalizedHintId}] "${result.label}"`, null, state);
|
|
494
|
+
const urlAfter = page.url();
|
|
495
|
+
const nav = urlAfter !== prevUrl ? `\nNavigated to: ${urlAfter}` : '';
|
|
496
|
+
return textResponse(
|
|
497
|
+
`Hovered over [${normalizedHintId}]: "${result.label}".${nav}\n${state.hintMap.length} elements now visible.`,
|
|
498
|
+
{ hint_id: normalizedHintId }
|
|
499
|
+
);
|
|
500
|
+
} catch (err) {
|
|
501
|
+
await audit('hover_failed', `[${normalizedHintId}] ${err.message}`, null, state);
|
|
502
|
+
await syncPageState(page, state, { force: true });
|
|
503
|
+
return buildStructuredError(`hover failed: ${err.message}`, normalizedHintId, {
|
|
504
|
+
error_code: TYPE_FAILED,
|
|
505
|
+
retryable: true,
|
|
506
|
+
suggested_next_step: 'retry',
|
|
507
|
+
evidence: {
|
|
508
|
+
hint_id: normalizedHintId,
|
|
509
|
+
reason: err.message,
|
|
510
|
+
},
|
|
511
|
+
});
|
|
512
|
+
}
|
|
513
|
+
}
|
|
514
|
+
);
|
|
515
|
+
|
|
516
|
+
server.registerTool(
|
|
517
|
+
'press_key',
|
|
518
|
+
{
|
|
519
|
+
description: 'Press a keyboard key or shortcut and refresh page state.',
|
|
520
|
+
inputSchema: {
|
|
521
|
+
key: z.string().describe('Keyboard key or shortcut, e.g. Enter, Escape, Control+Enter'),
|
|
522
|
+
},
|
|
523
|
+
},
|
|
524
|
+
async ({ key }) => {
|
|
525
|
+
const instance = await getBrowserInstance();
|
|
526
|
+
const confirmationError = requireConfirmedRuntimeInstance(state, instance, 'press_key');
|
|
527
|
+
if (confirmationError) return confirmationError;
|
|
528
|
+
const page = await getPage({ state });
|
|
529
|
+
await syncPageState(page, state);
|
|
530
|
+
await pressKey(page, key);
|
|
531
|
+
await syncPageState(page, state, { force: true });
|
|
532
|
+
await audit('press_key', key, null, state);
|
|
533
|
+
return textResponse(`Pressed key: ${key}`, {
|
|
534
|
+
key,
|
|
535
|
+
page_role: state.pageState?.currentRole ?? 'unknown',
|
|
536
|
+
grasp_confidence: state.pageState?.graspConfidence ?? 'unknown',
|
|
537
|
+
});
|
|
538
|
+
}
|
|
539
|
+
);
|
|
540
|
+
|
|
541
|
+
server.registerTool(
|
|
542
|
+
'watch_element',
|
|
543
|
+
{
|
|
544
|
+
description: 'Watch a DOM element for appears/disappears/changes and report the result.',
|
|
545
|
+
inputSchema: {
|
|
546
|
+
selector: z.string().describe('CSS selector to watch'),
|
|
547
|
+
condition: z.enum(['appears', 'disappears', 'changes']).optional().describe('Condition to watch for'),
|
|
548
|
+
timeout_ms: z.number().int().positive().optional().describe('Timeout in milliseconds'),
|
|
549
|
+
},
|
|
550
|
+
},
|
|
551
|
+
async ({ selector, condition = 'appears', timeout_ms = 30000 }) => {
|
|
552
|
+
const page = await getPage({ state });
|
|
553
|
+
const result = await watchElement(page, selector, condition, timeout_ms);
|
|
554
|
+
await audit('watch_element', `${condition} ${selector}`, null, state);
|
|
555
|
+
return textResponse([
|
|
556
|
+
`Watch selector: ${selector}`,
|
|
557
|
+
`Condition: ${condition}`,
|
|
558
|
+
`Result: ${result.met ? 'met' : result.timeout ? 'timeout' : 'unknown'}`,
|
|
559
|
+
...(result.text ? [`Text: ${result.text}`] : []),
|
|
560
|
+
], { selector, condition, result });
|
|
561
|
+
}
|
|
562
|
+
);
|
|
563
|
+
|
|
564
|
+
server.registerTool(
|
|
565
|
+
'scroll',
|
|
566
|
+
{
|
|
567
|
+
description: 'Scroll the page or a specific container by pixel amount. Use hint_id to scroll a nested scrollable area (e.g. sidebar, chat list) instead of the whole page. Returns scroll position for the active axis. NOTE: If your goal is to make a known element visible, use scroll_into_view instead — it is a single call and much more accurate.',
|
|
568
|
+
inputSchema: {
|
|
569
|
+
direction: z.enum(['up', 'down', 'left', 'right']).describe('Scroll direction'),
|
|
570
|
+
amount: z.number().int().positive().optional().describe('Scroll distance in pixels (default: 600). Use small values like 50-150 for precise scrolling.'),
|
|
571
|
+
hint_id: z.string().optional().describe('Hint ID of an element inside the scrollable container to target'),
|
|
572
|
+
},
|
|
573
|
+
},
|
|
574
|
+
async ({ direction, amount = 600, hint_id }) => {
|
|
575
|
+
const instance = await getBrowserInstance();
|
|
576
|
+
const confirmationError = requireConfirmedRuntimeInstance(state, instance, 'scroll');
|
|
577
|
+
if (confirmationError) return confirmationError;
|
|
578
|
+
const page = await getPage({ state });
|
|
579
|
+
await syncState(page, state);
|
|
580
|
+
|
|
581
|
+
let scrollTarget = null;
|
|
582
|
+
let scrollOptions = {};
|
|
583
|
+
|
|
584
|
+
if (hint_id) {
|
|
585
|
+
const normalizedId = String(hint_id).trim();
|
|
586
|
+
const selector = `[data-grasp-id="${normalizedId}"]`;
|
|
587
|
+
const ancestorSelector = await findScrollableAncestor(page, selector);
|
|
588
|
+
if (ancestorSelector) {
|
|
589
|
+
scrollOptions.selector = ancestorSelector;
|
|
590
|
+
scrollTarget = ancestorSelector;
|
|
591
|
+
}
|
|
592
|
+
}
|
|
593
|
+
|
|
594
|
+
await scroll(page, direction, amount, scrollOptions);
|
|
595
|
+
await syncState(page, state, { force: true });
|
|
596
|
+
|
|
597
|
+
const scrollInfo = await page.evaluate((sel) => {
|
|
598
|
+
const target = sel ? document.querySelector(sel) : document.documentElement;
|
|
599
|
+
if (!target) return null;
|
|
600
|
+
return {
|
|
601
|
+
scrollTop: Math.round(target.scrollTop),
|
|
602
|
+
scrollHeight: Math.round(target.scrollHeight),
|
|
603
|
+
clientHeight: Math.round(target.clientHeight),
|
|
604
|
+
scrollLeft: Math.round(target.scrollLeft),
|
|
605
|
+
scrollWidth: Math.round(target.scrollWidth),
|
|
606
|
+
clientWidth: Math.round(target.clientWidth),
|
|
607
|
+
atTop: target.scrollTop <= 0,
|
|
608
|
+
atBottom: target.scrollTop + target.clientHeight >= target.scrollHeight - 1,
|
|
609
|
+
atLeft: target.scrollLeft <= 0,
|
|
610
|
+
atRight: target.scrollLeft + target.clientWidth >= target.scrollWidth - 1,
|
|
611
|
+
};
|
|
612
|
+
}, scrollTarget);
|
|
613
|
+
|
|
614
|
+
const targetLabel = scrollTarget ? `container ${scrollTarget}` : 'page';
|
|
615
|
+
await audit('scroll', `${direction} ${amount} target=${targetLabel}`, null, state);
|
|
616
|
+
|
|
617
|
+
const isVertical = direction === 'up' || direction === 'down';
|
|
618
|
+
const posInfo = scrollInfo
|
|
619
|
+
? ` Position: ${isVertical ? scrollInfo.scrollTop : scrollInfo.scrollLeft}/${isVertical ? scrollInfo.scrollHeight : scrollInfo.scrollWidth}px.` +
|
|
620
|
+
`${isVertical
|
|
621
|
+
? `${scrollInfo.atTop ? ' [AT TOP]' : ''}${scrollInfo.atBottom ? ' [AT BOTTOM]' : ''}`
|
|
622
|
+
: `${scrollInfo.atLeft ? ' [AT LEFT]' : ''}${scrollInfo.atRight ? ' [AT RIGHT]' : ''}`}`
|
|
623
|
+
: '';
|
|
624
|
+
return textResponse(
|
|
625
|
+
`Scrolled ${targetLabel} ${direction} by ${amount}px.${posInfo}`,
|
|
626
|
+
{
|
|
627
|
+
direction,
|
|
628
|
+
amount,
|
|
629
|
+
target: scrollTarget ?? 'page',
|
|
630
|
+
...(scrollInfo ?? {}),
|
|
631
|
+
page_role: state.pageState?.currentRole ?? 'unknown',
|
|
632
|
+
grasp_confidence: state.pageState?.graspConfidence ?? 'unknown',
|
|
633
|
+
dom_revision: state.pageState?.domRevision ?? 0,
|
|
634
|
+
}
|
|
635
|
+
);
|
|
636
|
+
}
|
|
637
|
+
);
|
|
638
|
+
|
|
639
|
+
server.registerTool(
|
|
640
|
+
'scroll_into_view',
|
|
641
|
+
{
|
|
642
|
+
description: 'Scroll the page or container so that a specific element becomes visible in the viewport. Uses browser-native scrollIntoView which automatically handles arbitrarily nested scrollable containers in one call. PREFERRED over scroll() when you need to locate a known element — no pixel estimation or evaluate() needed.',
|
|
643
|
+
inputSchema: {
|
|
644
|
+
hint_id: z.string().describe('Hint ID of the element to scroll into view'),
|
|
645
|
+
position: z.enum(['center', 'start', 'end', 'nearest']).optional().describe('Where to place the element in the viewport (default: center)'),
|
|
646
|
+
},
|
|
647
|
+
},
|
|
648
|
+
async ({ hint_id, position = 'center' }) => {
|
|
649
|
+
const instance = await getBrowserInstance();
|
|
650
|
+
const confirmationError = requireConfirmedRuntimeInstance(state, instance, 'scroll_into_view');
|
|
651
|
+
if (confirmationError) return confirmationError;
|
|
652
|
+
const page = await getPage({ state });
|
|
653
|
+
await syncState(page, state, { force: true });
|
|
654
|
+
const normalizedId = String(hint_id).trim();
|
|
655
|
+
const selector = `[data-grasp-id="${normalizedId}"]`;
|
|
656
|
+
|
|
657
|
+
const result = await page.evaluate(({ sel, pos }) => {
|
|
658
|
+
const el = document.querySelector(sel);
|
|
659
|
+
if (!el) return { ok: false, reason: 'not_found' };
|
|
660
|
+
|
|
661
|
+
const before = el.getBoundingClientRect();
|
|
662
|
+
el.scrollIntoView({ behavior: 'instant', block: pos, inline: 'nearest' });
|
|
663
|
+
const after = el.getBoundingClientRect();
|
|
664
|
+
|
|
665
|
+
return {
|
|
666
|
+
ok: true,
|
|
667
|
+
tag: el.tagName.toLowerCase(),
|
|
668
|
+
label: el.getAttribute('aria-label') || el.innerText?.trim()?.substring(0, 60) || '',
|
|
669
|
+
moved: Math.abs(after.top - before.top) > 1 || Math.abs(after.left - before.left) > 1,
|
|
670
|
+
rect: {
|
|
671
|
+
top: Math.round(after.top),
|
|
672
|
+
left: Math.round(after.left),
|
|
673
|
+
width: Math.round(after.width),
|
|
674
|
+
height: Math.round(after.height),
|
|
675
|
+
},
|
|
676
|
+
};
|
|
677
|
+
}, { sel: selector, pos: position });
|
|
678
|
+
|
|
679
|
+
if (!result.ok) {
|
|
680
|
+
return errorResponse(`Element [${normalizedId}] not found. Call get_hint_map to refresh.`);
|
|
681
|
+
}
|
|
682
|
+
|
|
683
|
+
await page.evaluate(() => new Promise((resolve) => requestAnimationFrame(() => requestAnimationFrame(resolve))));
|
|
684
|
+
await syncState(page, state, { force: true });
|
|
685
|
+
await audit('scroll_into_view', `[${normalizedId}] position=${position}`, null, state);
|
|
686
|
+
|
|
687
|
+
const movedLabel = result.moved ? 'Scrolled to' : 'Already visible:';
|
|
688
|
+
return textResponse(
|
|
689
|
+
`${movedLabel} [${normalizedId}] (${result.tag}: "${result.label}"). Position: top=${result.rect.top}px, left=${result.rect.left}px.`,
|
|
690
|
+
{ hint_id: normalizedId, position, moved: result.moved, rect: result.rect }
|
|
691
|
+
);
|
|
692
|
+
}
|
|
693
|
+
);
|
|
694
|
+
|
|
695
|
+
server.registerTool(
|
|
696
|
+
'screenshot',
|
|
697
|
+
{
|
|
698
|
+
description: 'Take a screenshot of the current browser viewport, or a specific element by hint ID. Returns base64-encoded PNG image. Use annotate=true to overlay HintMap element labels on the screenshot for visual identification.',
|
|
699
|
+
inputSchema: {
|
|
700
|
+
fullPage: z.boolean().optional().describe('Capture the full scrollable page instead of just the viewport'),
|
|
701
|
+
annotate: z.boolean().optional().describe('Overlay HintMap element IDs on the screenshot (e.g. [B0], [I1], [L2])'),
|
|
702
|
+
hint_id: z.string().optional().describe('Capture only this element (mutually exclusive with fullPage and annotate)'),
|
|
703
|
+
},
|
|
704
|
+
},
|
|
705
|
+
async ({ fullPage = false, annotate = false, hint_id } = {}) => {
|
|
706
|
+
try {
|
|
707
|
+
if (hint_id && (fullPage || annotate)) {
|
|
708
|
+
return errorResponse('hint_id is mutually exclusive with fullPage and annotate. Use hint_id alone for element screenshots.');
|
|
709
|
+
}
|
|
710
|
+
|
|
711
|
+
const page = await getPage({ state });
|
|
712
|
+
await page.waitForFunction(
|
|
713
|
+
() => document.body && document.body.getBoundingClientRect().height > 100,
|
|
714
|
+
{ timeout: 3000 }
|
|
715
|
+
).catch(() => {});
|
|
716
|
+
|
|
717
|
+
if (annotate) {
|
|
718
|
+
await syncState(page, state, { force: true });
|
|
719
|
+
const hints = state.hintMap ?? [];
|
|
720
|
+
if (hints.length > 0) {
|
|
721
|
+
await page.evaluate((hintItems) => {
|
|
722
|
+
const container = document.createElement('div');
|
|
723
|
+
container.id = '__grasp_annotations__';
|
|
724
|
+
container.style.cssText = 'position:fixed;top:0;left:0;width:100%;height:100%;pointer-events:none;z-index:2147483647;';
|
|
725
|
+
|
|
726
|
+
for (const h of hintItems) {
|
|
727
|
+
const el = document.elementFromPoint(h.x, h.y);
|
|
728
|
+
if (!el) continue;
|
|
729
|
+
const rect = el.getBoundingClientRect();
|
|
730
|
+
if (rect.width === 0 || rect.height === 0) continue;
|
|
731
|
+
|
|
732
|
+
const box = document.createElement('div');
|
|
733
|
+
box.style.cssText = `position:fixed;left:${rect.left}px;top:${rect.top}px;width:${rect.width}px;height:${rect.height}px;border:2px solid rgba(255,0,0,0.7);box-sizing:border-box;pointer-events:none;`;
|
|
734
|
+
|
|
735
|
+
const tag = document.createElement('div');
|
|
736
|
+
const labelTop = rect.top > 16 ? '-16px' : `${rect.height + 2}px`;
|
|
737
|
+
tag.style.cssText = `position:absolute;left:0;top:${labelTop};background:rgba(255,0,0,0.85);color:#fff;font:bold 11px/14px monospace;padding:0 4px;border-radius:2px;white-space:nowrap;`;
|
|
738
|
+
tag.textContent = h.id;
|
|
739
|
+
box.appendChild(tag);
|
|
740
|
+
container.appendChild(box);
|
|
741
|
+
}
|
|
742
|
+
document.body.appendChild(container);
|
|
743
|
+
}, hints);
|
|
744
|
+
}
|
|
745
|
+
}
|
|
746
|
+
|
|
747
|
+
if (hint_id) {
|
|
748
|
+
const normalizedId = String(hint_id).trim();
|
|
749
|
+
await syncState(page, state, { force: true });
|
|
750
|
+
const selector = `[data-grasp-id="${normalizedId}"]`;
|
|
751
|
+
const el = page.locator(selector);
|
|
752
|
+
const count = await el.count();
|
|
753
|
+
if (count === 0) {
|
|
754
|
+
return errorResponse(`Element [${normalizedId}] not found for screenshot. Call get_hint_map to refresh available IDs.`);
|
|
755
|
+
}
|
|
756
|
+
const box = await el.first().boundingBox();
|
|
757
|
+
if (!box || box.width === 0 || box.height === 0) {
|
|
758
|
+
return errorResponse(`Element [${normalizedId}] is not visible (zero bounds). Scroll it into view first.`);
|
|
759
|
+
}
|
|
760
|
+
const viewport = page.viewportSize() || await page.evaluate(() => ({
|
|
761
|
+
width: window.innerWidth,
|
|
762
|
+
height: window.innerHeight,
|
|
763
|
+
}));
|
|
764
|
+
const clip = {
|
|
765
|
+
x: Math.max(0, box.x),
|
|
766
|
+
y: Math.max(0, box.y),
|
|
767
|
+
width: Math.min(box.width, viewport.width - Math.max(0, box.x)),
|
|
768
|
+
height: Math.min(box.height, viewport.height - Math.max(0, box.y)),
|
|
769
|
+
};
|
|
770
|
+
if (clip.width <= 0 || clip.height <= 0) {
|
|
771
|
+
return errorResponse(`Element [${normalizedId}] is outside the viewport. Use scroll_into_view first.`);
|
|
772
|
+
}
|
|
773
|
+
const base64 = await page.screenshot({ encoding: 'base64', clip });
|
|
774
|
+
return imageResponse(base64);
|
|
775
|
+
}
|
|
776
|
+
|
|
777
|
+
const base64 = await page.screenshot({ encoding: 'base64', fullPage });
|
|
778
|
+
|
|
779
|
+
if (annotate) {
|
|
780
|
+
await page.evaluate(() => {
|
|
781
|
+
const overlay = document.getElementById('__grasp_annotations__');
|
|
782
|
+
if (overlay) overlay.remove();
|
|
783
|
+
});
|
|
784
|
+
}
|
|
785
|
+
|
|
786
|
+
return imageResponse(base64);
|
|
787
|
+
} catch (err) {
|
|
788
|
+
try {
|
|
789
|
+
const p = await getPage({ state });
|
|
790
|
+
await p.evaluate(() => {
|
|
791
|
+
const overlay = document.getElementById('__grasp_annotations__');
|
|
792
|
+
if (overlay) overlay.remove();
|
|
793
|
+
});
|
|
794
|
+
} catch {
|
|
795
|
+
// ignore cleanup errors
|
|
796
|
+
}
|
|
797
|
+
return errorResponse(`Screenshot failed: ${err.message}`);
|
|
798
|
+
}
|
|
799
|
+
}
|
|
800
|
+
);
|
|
801
|
+
|
|
802
|
+
server.registerTool(
|
|
803
|
+
'get_tabs',
|
|
804
|
+
{
|
|
805
|
+
description: 'List all open browser tabs with index, title, and URL.',
|
|
806
|
+
inputSchema: {},
|
|
807
|
+
},
|
|
808
|
+
async () => {
|
|
809
|
+
const tabs = await listTabs();
|
|
810
|
+
await audit('get_tabs', `${tabs.length} tabs`, null, state);
|
|
811
|
+
return textResponse(
|
|
812
|
+
tabs.map((tab, index) => `[${tab.index ?? index}] ${tab.title} — ${tab.url}`).join('\n'),
|
|
813
|
+
{ tabs }
|
|
814
|
+
);
|
|
815
|
+
}
|
|
816
|
+
);
|
|
817
|
+
|
|
818
|
+
server.registerTool(
|
|
819
|
+
'switch_tab',
|
|
820
|
+
{
|
|
821
|
+
description: 'Switch to a browser tab by its index (from get_tabs).',
|
|
822
|
+
inputSchema: {
|
|
823
|
+
index: z.number().int().min(0).describe('Tab index to switch to'),
|
|
824
|
+
},
|
|
825
|
+
},
|
|
826
|
+
async ({ index }) => {
|
|
827
|
+
const confirmationError = await requireActionConfirmation('switch_tab');
|
|
828
|
+
if (confirmationError) return confirmationError;
|
|
829
|
+
const page = await activateTab(index);
|
|
830
|
+
await syncState(page, state, { force: true });
|
|
831
|
+
await audit('switch_tab', `index=${index}`, null, state);
|
|
832
|
+
return textResponse(
|
|
833
|
+
`Switched to tab [${index}]: ${page.url()}`,
|
|
834
|
+
{ index, url: page.url() }
|
|
835
|
+
);
|
|
836
|
+
}
|
|
837
|
+
);
|
|
838
|
+
|
|
839
|
+
server.registerTool(
|
|
840
|
+
'new_tab',
|
|
841
|
+
{
|
|
842
|
+
description: 'Open a new browser tab and navigate to the given URL.',
|
|
843
|
+
inputSchema: {
|
|
844
|
+
url: z.string().url().describe('URL to open in new tab'),
|
|
845
|
+
},
|
|
846
|
+
},
|
|
847
|
+
async ({ url }) => {
|
|
848
|
+
const confirmationError = await requireActionConfirmation('new_tab');
|
|
849
|
+
if (confirmationError) return confirmationError;
|
|
850
|
+
const page = await openTab(url);
|
|
851
|
+
await syncState(page, state, { force: true });
|
|
852
|
+
await audit('new_tab', url, null, state);
|
|
853
|
+
return textResponse(
|
|
854
|
+
`Opened new tab: ${page.url()}`,
|
|
855
|
+
{ url: page.url() }
|
|
856
|
+
);
|
|
857
|
+
}
|
|
858
|
+
);
|
|
859
|
+
|
|
860
|
+
server.registerTool(
|
|
861
|
+
'close_tab',
|
|
862
|
+
{
|
|
863
|
+
description: 'Close a browser tab by its index.',
|
|
864
|
+
inputSchema: {
|
|
865
|
+
index: z.number().int().min(0).describe('Tab index to close'),
|
|
866
|
+
},
|
|
867
|
+
},
|
|
868
|
+
async ({ index }) => {
|
|
869
|
+
const confirmationError = await requireActionConfirmation('close_tab');
|
|
870
|
+
if (confirmationError) return confirmationError;
|
|
871
|
+
await closeBrowserTab(index);
|
|
872
|
+
await audit('close_tab', `index=${index}`, null, state);
|
|
873
|
+
const remaining = await listTabs();
|
|
874
|
+
return textResponse(
|
|
875
|
+
`Closed tab [${index}]. ${remaining.length} tabs remaining.`,
|
|
876
|
+
{ closedIndex: index, remainingTabs: remaining.length }
|
|
877
|
+
);
|
|
878
|
+
}
|
|
879
|
+
);
|
|
880
|
+
|
|
881
|
+
server.registerTool(
|
|
882
|
+
'evaluate',
|
|
883
|
+
{
|
|
884
|
+
description: 'Execute JavaScript in the browser page. Use specialized tools (click, type, etc.) when possible — this is a low-level escape hatch.',
|
|
885
|
+
inputSchema: {
|
|
886
|
+
expression: z.string().describe('JavaScript expression to evaluate (can be async)'),
|
|
887
|
+
},
|
|
888
|
+
},
|
|
889
|
+
async ({ expression }) => {
|
|
890
|
+
const confirmationError = await requireActionConfirmation('evaluate');
|
|
891
|
+
if (confirmationError) return confirmationError;
|
|
892
|
+
const page = await getPageWithListeners({ state });
|
|
893
|
+
try {
|
|
894
|
+
const result = await page.evaluate(expression);
|
|
895
|
+
const serialized = result === undefined ? null : result;
|
|
896
|
+
const output = typeof serialized === 'string' ? serialized : JSON.stringify(serialized, null, 2);
|
|
897
|
+
const truncated = output && output.length > 10240 ? `${output.slice(0, 10240)}\n...(truncated)` : output;
|
|
898
|
+
await audit('evaluate', expression.slice(0, 100), null, state);
|
|
899
|
+
return textResponse(truncated ?? 'undefined', { type: typeof result });
|
|
900
|
+
} catch (err) {
|
|
901
|
+
await audit('evaluate_error', err.message.slice(0, 100), null, state);
|
|
902
|
+
return errorResponse(`Evaluate failed: ${err.message}`);
|
|
903
|
+
}
|
|
904
|
+
}
|
|
905
|
+
);
|
|
906
|
+
|
|
907
|
+
server.registerTool(
|
|
908
|
+
'handle_dialog',
|
|
909
|
+
{
|
|
910
|
+
description: 'Handle a browser dialog (alert/confirm/prompt). The dialog must already be open.',
|
|
911
|
+
inputSchema: {
|
|
912
|
+
action: z.enum(['accept', 'dismiss']).describe('Whether to accept or dismiss the dialog'),
|
|
913
|
+
text: z.string().optional().describe('Text to enter for prompt dialogs (only used with accept)'),
|
|
914
|
+
},
|
|
915
|
+
},
|
|
916
|
+
async ({ action, text }) => {
|
|
917
|
+
const confirmationError = await requireActionConfirmation('handle_dialog');
|
|
918
|
+
if (confirmationError) return confirmationError;
|
|
919
|
+
if (!state.pendingDialog) {
|
|
920
|
+
return errorResponse('No dialog is currently open. Dialogs are captured automatically when they appear.');
|
|
921
|
+
}
|
|
922
|
+
const dialog = state.pendingDialog;
|
|
923
|
+
try {
|
|
924
|
+
if (action === 'accept') {
|
|
925
|
+
await dialog.ref.accept(text ?? '');
|
|
926
|
+
} else {
|
|
927
|
+
await dialog.ref.dismiss();
|
|
928
|
+
}
|
|
929
|
+
const info = { type: dialog.type, message: dialog.message, action };
|
|
930
|
+
state.pendingDialog = null;
|
|
931
|
+
await audit('handle_dialog', `${action} ${dialog.type}: "${dialog.message}"`, null, state);
|
|
932
|
+
return textResponse(`Dialog ${action}ed. Type: ${dialog.type}, Message: "${dialog.message}"`, info);
|
|
933
|
+
} catch (err) {
|
|
934
|
+
state.pendingDialog = null;
|
|
935
|
+
return errorResponse(`Dialog handling failed: ${err.message}`);
|
|
936
|
+
}
|
|
937
|
+
}
|
|
938
|
+
);
|
|
939
|
+
|
|
940
|
+
server.registerTool(
|
|
941
|
+
'upload_file',
|
|
942
|
+
{
|
|
943
|
+
description: 'Upload file(s) to a file input element identified by hint ID.',
|
|
944
|
+
inputSchema: {
|
|
945
|
+
hint_id: z.string().describe('Hint ID of the file input element'),
|
|
946
|
+
file_paths: z.array(z.string()).min(1).describe('Array of absolute file paths to upload'),
|
|
947
|
+
},
|
|
948
|
+
},
|
|
949
|
+
async ({ hint_id, file_paths }) => {
|
|
950
|
+
const confirmationError = await requireActionConfirmation('upload_file');
|
|
951
|
+
if (confirmationError) return confirmationError;
|
|
952
|
+
const page = await getPageWithListeners({ state });
|
|
953
|
+
await syncState(page, state);
|
|
954
|
+
const normalizedId = String(hint_id).trim();
|
|
955
|
+
const selector = `[data-grasp-id="${normalizedId}"]`;
|
|
956
|
+
|
|
957
|
+
const elInfo = await page.evaluate((sel) => {
|
|
958
|
+
const el = document.querySelector(sel);
|
|
959
|
+
if (!el) return { found: false };
|
|
960
|
+
return { found: true, tag: el.tagName, type: el.type || null };
|
|
961
|
+
}, selector);
|
|
962
|
+
|
|
963
|
+
if (!elInfo.found) {
|
|
964
|
+
return errorResponse(`Element [${normalizedId}] not found.`);
|
|
965
|
+
}
|
|
966
|
+
if (elInfo.tag !== 'INPUT' || elInfo.type !== 'file') {
|
|
967
|
+
return errorResponse(`Element [${normalizedId}] is not a file input (found: <${elInfo.tag} type="${elInfo.type}">).`);
|
|
968
|
+
}
|
|
969
|
+
|
|
970
|
+
const locator = page.locator(selector);
|
|
971
|
+
await locator.setInputFiles(file_paths);
|
|
972
|
+
await syncState(page, state, { force: true });
|
|
973
|
+
await audit('upload_file', `[${normalizedId}] ${file_paths.length} file(s)`, null, state);
|
|
974
|
+
return textResponse(
|
|
975
|
+
`Uploaded ${file_paths.length} file(s) to [${normalizedId}]: ${file_paths.map((p) => p.split(/[/\\]/).pop()).join(', ')}`,
|
|
976
|
+
{ hint_id: normalizedId, files: file_paths }
|
|
977
|
+
);
|
|
978
|
+
}
|
|
979
|
+
);
|
|
980
|
+
|
|
981
|
+
server.registerTool(
|
|
982
|
+
'drag',
|
|
983
|
+
{
|
|
984
|
+
description: 'Drag an element and drop it onto another element, both identified by hint ID.',
|
|
985
|
+
inputSchema: {
|
|
986
|
+
from_hint_id: z.string().describe('Hint ID of the element to drag'),
|
|
987
|
+
to_hint_id: z.string().describe('Hint ID of the drop target'),
|
|
988
|
+
},
|
|
989
|
+
},
|
|
990
|
+
async ({ from_hint_id, to_hint_id }) => {
|
|
991
|
+
const confirmationError = await requireActionConfirmation('drag');
|
|
992
|
+
if (confirmationError) return confirmationError;
|
|
993
|
+
const page = await getPageWithListeners({ state });
|
|
994
|
+
await syncState(page, state);
|
|
995
|
+
const fromId = String(from_hint_id).trim();
|
|
996
|
+
const toId = String(to_hint_id).trim();
|
|
997
|
+
const fromSel = `[data-grasp-id="${fromId}"]`;
|
|
998
|
+
const toSel = `[data-grasp-id="${toId}"]`;
|
|
999
|
+
|
|
1000
|
+
const boxes = await page.evaluate(({ fs, ts }) => {
|
|
1001
|
+
const from = document.querySelector(fs);
|
|
1002
|
+
const to = document.querySelector(ts);
|
|
1003
|
+
if (!from) return { error: 'from_not_found' };
|
|
1004
|
+
if (!to) return { error: 'to_not_found' };
|
|
1005
|
+
const fb = from.getBoundingClientRect();
|
|
1006
|
+
const tb = to.getBoundingClientRect();
|
|
1007
|
+
return {
|
|
1008
|
+
from: { x: fb.x + (fb.width / 2), y: fb.y + (fb.height / 2), label: from.textContent?.slice(0, 30) },
|
|
1009
|
+
to: { x: tb.x + (tb.width / 2), y: tb.y + (tb.height / 2), label: to.textContent?.slice(0, 30) },
|
|
1010
|
+
};
|
|
1011
|
+
}, { fs: fromSel, ts: toSel });
|
|
1012
|
+
|
|
1013
|
+
if (boxes.error === 'from_not_found') return errorResponse(`Source element [${fromId}] not found.`);
|
|
1014
|
+
if (boxes.error === 'to_not_found') return errorResponse(`Target element [${toId}] not found.`);
|
|
1015
|
+
|
|
1016
|
+
const steps = 8;
|
|
1017
|
+
const dx = (boxes.to.x - boxes.from.x) / steps;
|
|
1018
|
+
const dy = (boxes.to.y - boxes.from.y) / steps;
|
|
1019
|
+
|
|
1020
|
+
await page.mouse.move(boxes.from.x, boxes.from.y);
|
|
1021
|
+
await page.mouse.down();
|
|
1022
|
+
for (let i = 1; i <= steps; i += 1) {
|
|
1023
|
+
await page.mouse.move(
|
|
1024
|
+
boxes.from.x + (dx * i),
|
|
1025
|
+
boxes.from.y + (dy * i),
|
|
1026
|
+
);
|
|
1027
|
+
await new Promise((resolve) => setTimeout(resolve, 30 + (Math.random() * 50)));
|
|
1028
|
+
}
|
|
1029
|
+
await page.mouse.up();
|
|
1030
|
+
|
|
1031
|
+
await syncState(page, state, { force: true });
|
|
1032
|
+
await audit('drag', `[${fromId}] → [${toId}]`, null, state);
|
|
1033
|
+
return textResponse(
|
|
1034
|
+
`Dragged [${fromId}] "${boxes.from.label}" → [${toId}] "${boxes.to.label}"`,
|
|
1035
|
+
{ from: fromId, to: toId }
|
|
1036
|
+
);
|
|
1037
|
+
}
|
|
1038
|
+
);
|
|
1039
|
+
|
|
1040
|
+
server.registerTool(
|
|
1041
|
+
'go_back',
|
|
1042
|
+
{
|
|
1043
|
+
description: 'Navigate back in browser history (like pressing the Back button).',
|
|
1044
|
+
inputSchema: {},
|
|
1045
|
+
},
|
|
1046
|
+
async () => {
|
|
1047
|
+
const confirmationError = await requireActionConfirmation('go_back');
|
|
1048
|
+
if (confirmationError) return confirmationError;
|
|
1049
|
+
const page = await getPageWithListeners({ state });
|
|
1050
|
+
const prevUrl = page.url();
|
|
1051
|
+
const resp = await page.goBack({ waitUntil: 'domcontentloaded', timeout: 10000 }).catch(() => null);
|
|
1052
|
+
await syncState(page, state, { force: true });
|
|
1053
|
+
const newUrl = page.url();
|
|
1054
|
+
await audit('go_back', `${prevUrl} → ${newUrl}`, null, state);
|
|
1055
|
+
if (!resp && newUrl === prevUrl) {
|
|
1056
|
+
return textResponse('No previous page in history.', { url: newUrl, changed: false });
|
|
1057
|
+
}
|
|
1058
|
+
return textResponse(`Navigated back: ${newUrl}`, { url: newUrl, changed: newUrl !== prevUrl });
|
|
1059
|
+
}
|
|
1060
|
+
);
|
|
1061
|
+
|
|
1062
|
+
server.registerTool(
|
|
1063
|
+
'go_forward',
|
|
1064
|
+
{
|
|
1065
|
+
description: 'Navigate forward in browser history.',
|
|
1066
|
+
inputSchema: {},
|
|
1067
|
+
},
|
|
1068
|
+
async () => {
|
|
1069
|
+
const confirmationError = await requireActionConfirmation('go_forward');
|
|
1070
|
+
if (confirmationError) return confirmationError;
|
|
1071
|
+
const page = await getPageWithListeners({ state });
|
|
1072
|
+
const prevUrl = page.url();
|
|
1073
|
+
const resp = await page.goForward({ waitUntil: 'domcontentloaded', timeout: 10000 }).catch(() => null);
|
|
1074
|
+
await syncState(page, state, { force: true });
|
|
1075
|
+
const newUrl = page.url();
|
|
1076
|
+
await audit('go_forward', `${prevUrl} → ${newUrl}`, null, state);
|
|
1077
|
+
if (!resp && newUrl === prevUrl) {
|
|
1078
|
+
return textResponse('No forward page in history.', { url: newUrl, changed: false });
|
|
1079
|
+
}
|
|
1080
|
+
return textResponse(`Navigated forward: ${newUrl}`, { url: newUrl, changed: newUrl !== prevUrl });
|
|
1081
|
+
}
|
|
1082
|
+
);
|
|
1083
|
+
|
|
1084
|
+
server.registerTool(
|
|
1085
|
+
'reload',
|
|
1086
|
+
{
|
|
1087
|
+
description: 'Reload the current page.',
|
|
1088
|
+
inputSchema: {},
|
|
1089
|
+
},
|
|
1090
|
+
async () => {
|
|
1091
|
+
const confirmationError = await requireActionConfirmation('reload');
|
|
1092
|
+
if (confirmationError) return confirmationError;
|
|
1093
|
+
const page = await getPageWithListeners({ state });
|
|
1094
|
+
const url = page.url();
|
|
1095
|
+
await page.reload({ waitUntil: 'domcontentloaded', timeout: 15000 });
|
|
1096
|
+
await syncState(page, state, { force: true });
|
|
1097
|
+
await audit('reload', url, null, state);
|
|
1098
|
+
return textResponse(`Reloaded: ${url}`, { url });
|
|
1099
|
+
}
|
|
1100
|
+
);
|
|
1101
|
+
|
|
1102
|
+
server.registerTool(
|
|
1103
|
+
'get_console_logs',
|
|
1104
|
+
{
|
|
1105
|
+
description: 'Get captured browser console messages. Logs are captured automatically in the background.',
|
|
1106
|
+
inputSchema: {
|
|
1107
|
+
level: z.enum(['all', 'error', 'warning', 'info', 'log', 'debug']).optional().describe('Filter by log level (default: all)'),
|
|
1108
|
+
clear: z.boolean().optional().describe('Clear the log buffer after returning (default: false)'),
|
|
1109
|
+
},
|
|
1110
|
+
},
|
|
1111
|
+
async ({ level = 'all', clear = false }) => {
|
|
1112
|
+
try {
|
|
1113
|
+
await getPageWithListeners({ state });
|
|
1114
|
+
} catch {
|
|
1115
|
+
// ok if no page
|
|
1116
|
+
}
|
|
1117
|
+
let logs = state.consoleLogs || [];
|
|
1118
|
+
if (level !== 'all') {
|
|
1119
|
+
logs = logs.filter((log) => log.level === level);
|
|
1120
|
+
}
|
|
1121
|
+
const result = logs.map((log) => `[${log.level}] ${log.text}`).join('\n') || '(no logs)';
|
|
1122
|
+
const meta = { count: logs.length, total: state.consoleLogs?.length ?? 0 };
|
|
1123
|
+
if (clear) {
|
|
1124
|
+
state.consoleLogs = [];
|
|
1125
|
+
}
|
|
1126
|
+
await audit('get_console_logs', `${logs.length} entries (level=${level}, clear=${clear})`, null, state);
|
|
1127
|
+
return textResponse(result, meta);
|
|
1128
|
+
}
|
|
1129
|
+
);
|
|
1130
|
+
|
|
1131
|
+
server.registerTool(
|
|
1132
|
+
'get_cookies',
|
|
1133
|
+
{
|
|
1134
|
+
description: 'Get browser cookies, optionally filtered by URL/domain.',
|
|
1135
|
+
inputSchema: {
|
|
1136
|
+
url: z.string().optional().describe('URL to filter cookies for (e.g., "https://example.com")'),
|
|
1137
|
+
},
|
|
1138
|
+
},
|
|
1139
|
+
async ({ url } = {}) => {
|
|
1140
|
+
const page = await getPageWithListeners({ state });
|
|
1141
|
+
const context = page.context();
|
|
1142
|
+
const filterUrl = url || page.url();
|
|
1143
|
+
const cookies = await context.cookies(filterUrl);
|
|
1144
|
+
await audit('get_cookies', `${cookies.length} cookies for ${filterUrl}`, null, state);
|
|
1145
|
+
const summary = cookies.map((cookie) => `${cookie.name}=${cookie.value.slice(0, 40)}${cookie.value.length > 40 ? '...' : ''} (domain=${cookie.domain})`).join('\n') || '(no cookies)';
|
|
1146
|
+
return textResponse(summary, { count: cookies.length, cookies });
|
|
1147
|
+
}
|
|
1148
|
+
);
|
|
1149
|
+
|
|
1150
|
+
server.registerTool(
|
|
1151
|
+
'set_cookie',
|
|
1152
|
+
{
|
|
1153
|
+
description: 'Set a browser cookie.',
|
|
1154
|
+
inputSchema: {
|
|
1155
|
+
name: z.string().describe('Cookie name'),
|
|
1156
|
+
value: z.string().describe('Cookie value'),
|
|
1157
|
+
domain: z.string().optional().describe('Cookie domain (e.g., ".example.com")'),
|
|
1158
|
+
path: z.string().optional().describe('Cookie path (default: "/")'),
|
|
1159
|
+
httpOnly: z.boolean().optional().describe('HTTP only flag'),
|
|
1160
|
+
secure: z.boolean().optional().describe('Secure flag'),
|
|
1161
|
+
sameSite: z.enum(['Strict', 'Lax', 'None']).optional().describe('SameSite attribute'),
|
|
1162
|
+
},
|
|
1163
|
+
},
|
|
1164
|
+
async ({ name, value, domain, path = '/', httpOnly, secure, sameSite }) => {
|
|
1165
|
+
const confirmationError = await requireActionConfirmation('set_cookie');
|
|
1166
|
+
if (confirmationError) return confirmationError;
|
|
1167
|
+
const page = await getPageWithListeners({ state });
|
|
1168
|
+
const context = page.context();
|
|
1169
|
+
const pageUrl = new URL(page.url());
|
|
1170
|
+
const cookie = {
|
|
1171
|
+
name,
|
|
1172
|
+
value,
|
|
1173
|
+
domain: domain || pageUrl.hostname,
|
|
1174
|
+
path,
|
|
1175
|
+
...(httpOnly !== undefined && { httpOnly }),
|
|
1176
|
+
...(secure !== undefined && { secure }),
|
|
1177
|
+
...(sameSite && { sameSite }),
|
|
1178
|
+
};
|
|
1179
|
+
await context.addCookies([cookie]);
|
|
1180
|
+
await audit('set_cookie', `${name}=${value.slice(0, 20)} domain=${cookie.domain}`, null, state);
|
|
1181
|
+
return textResponse(`Cookie set: ${name}=${value.slice(0, 40)} (domain=${cookie.domain})`, { cookie });
|
|
1182
|
+
}
|
|
1183
|
+
);
|
|
1184
|
+
|
|
1185
|
+
server.registerTool(
|
|
1186
|
+
'clear_cookies',
|
|
1187
|
+
{
|
|
1188
|
+
description: 'Clear all browser cookies for the current context.',
|
|
1189
|
+
inputSchema: {},
|
|
1190
|
+
},
|
|
1191
|
+
async () => {
|
|
1192
|
+
const confirmationError = await requireActionConfirmation('clear_cookies');
|
|
1193
|
+
if (confirmationError) return confirmationError;
|
|
1194
|
+
const page = await getPageWithListeners({ state });
|
|
1195
|
+
const context = page.context();
|
|
1196
|
+
await context.clearCookies();
|
|
1197
|
+
await audit('clear_cookies', 'all cookies cleared', null, state);
|
|
1198
|
+
return textResponse('All cookies cleared.');
|
|
1199
|
+
}
|
|
1200
|
+
);
|
|
1201
|
+
|
|
1202
|
+
server.registerTool(
|
|
1203
|
+
'wait_for',
|
|
1204
|
+
{
|
|
1205
|
+
description: 'Wait for a condition: text to appear, text to disappear, or URL to contain a string.',
|
|
1206
|
+
inputSchema: {
|
|
1207
|
+
text: z.string().optional().describe('Wait for this text to appear on the page'),
|
|
1208
|
+
text_gone: z.string().optional().describe('Wait for this text to disappear from the page'),
|
|
1209
|
+
url_contains: z.string().optional().describe('Wait for the page URL to contain this string'),
|
|
1210
|
+
timeout: z.number().optional().describe('Timeout in ms (default: 10000)'),
|
|
1211
|
+
},
|
|
1212
|
+
},
|
|
1213
|
+
async ({ text, text_gone, url_contains, timeout = 10000 }) => {
|
|
1214
|
+
const provided = [text, text_gone, url_contains].filter((value) => value !== undefined);
|
|
1215
|
+
if (provided.length === 0) {
|
|
1216
|
+
return errorResponse('Provide exactly one condition: text, text_gone, or url_contains.');
|
|
1217
|
+
}
|
|
1218
|
+
if (provided.length > 1) {
|
|
1219
|
+
return errorResponse('Provide only one condition at a time (text, text_gone, or url_contains).');
|
|
1220
|
+
}
|
|
1221
|
+
|
|
1222
|
+
const page = await getPageWithListeners({ state });
|
|
1223
|
+
|
|
1224
|
+
try {
|
|
1225
|
+
if (text) {
|
|
1226
|
+
await page.getByText(text).first().waitFor({ state: 'visible', timeout });
|
|
1227
|
+
await audit('wait_for', `text "${text}" appeared`, null, state);
|
|
1228
|
+
return textResponse(`Text "${text}" appeared on the page.`);
|
|
1229
|
+
}
|
|
1230
|
+
if (text_gone) {
|
|
1231
|
+
await page.getByText(text_gone).first().waitFor({ state: 'hidden', timeout });
|
|
1232
|
+
await audit('wait_for', `text "${text_gone}" gone`, null, state);
|
|
1233
|
+
return textResponse(`Text "${text_gone}" is no longer visible.`);
|
|
1234
|
+
}
|
|
1235
|
+
if (url_contains) {
|
|
1236
|
+
await page.waitForURL(`**/*${url_contains}*`, { timeout });
|
|
1237
|
+
await syncState(page, state, { force: true });
|
|
1238
|
+
await audit('wait_for', `url contains "${url_contains}"`, null, state);
|
|
1239
|
+
return textResponse(`URL now contains "${url_contains}": ${page.url()}`);
|
|
1240
|
+
}
|
|
1241
|
+
} catch {
|
|
1242
|
+
const condition = text ? `text "${text}"` : text_gone ? `text_gone "${text_gone}"` : `url "${url_contains}"`;
|
|
1243
|
+
return errorResponse(`Timeout waiting for ${condition} after ${timeout}ms.`);
|
|
1244
|
+
}
|
|
1245
|
+
}
|
|
1246
|
+
);
|
|
1247
|
+
|
|
1248
|
+
server.registerTool(
|
|
1249
|
+
'double_click',
|
|
1250
|
+
{
|
|
1251
|
+
description: 'Double-click an element by hint ID.',
|
|
1252
|
+
inputSchema: {
|
|
1253
|
+
hint_id: z.string().describe('Hint ID from get_hint_map'),
|
|
1254
|
+
},
|
|
1255
|
+
},
|
|
1256
|
+
async ({ hint_id }) => {
|
|
1257
|
+
const confirmationError = await requireActionConfirmation('double_click');
|
|
1258
|
+
if (confirmationError) return confirmationError;
|
|
1259
|
+
const normalizedHintId = String(hint_id).trim();
|
|
1260
|
+
const page = await getPageWithListeners({ state });
|
|
1261
|
+
await syncState(page, state);
|
|
1262
|
+
const rebuildHints = createRebuildHints(page, state);
|
|
1263
|
+
|
|
1264
|
+
const result = await clickByHintId(page, normalizedHintId, { rebuildHints, clickCount: 2 });
|
|
1265
|
+
await syncState(page, state, { force: true });
|
|
1266
|
+
await audit('double_click', `[${normalizedHintId}] "${result.label}"`, null, state);
|
|
1267
|
+
return textResponse(
|
|
1268
|
+
`Double-clicked [${normalizedHintId}]: "${result.label}"\nPage now has ${state.hintMap?.length ?? 0} elements.`,
|
|
1269
|
+
{ hint_id: normalizedHintId, label: result.label }
|
|
1270
|
+
);
|
|
1271
|
+
}
|
|
1272
|
+
);
|
|
1273
|
+
|
|
1274
|
+
server.registerTool(
|
|
1275
|
+
'check',
|
|
1276
|
+
{
|
|
1277
|
+
description: 'Set the checked state of a checkbox or radio button by hint ID.',
|
|
1278
|
+
inputSchema: {
|
|
1279
|
+
hint_id: z.string().describe('Hint ID of the checkbox or radio element'),
|
|
1280
|
+
checked: z.boolean().optional().describe('Desired state: true to check, false to uncheck (default: true)'),
|
|
1281
|
+
},
|
|
1282
|
+
},
|
|
1283
|
+
async ({ hint_id, checked = true }) => {
|
|
1284
|
+
const confirmationError = await requireActionConfirmation('check');
|
|
1285
|
+
if (confirmationError) return confirmationError;
|
|
1286
|
+
const normalizedId = String(hint_id).trim();
|
|
1287
|
+
const page = await getPageWithListeners({ state });
|
|
1288
|
+
await syncState(page, state);
|
|
1289
|
+
const selector = `[data-grasp-id="${normalizedId}"]`;
|
|
1290
|
+
|
|
1291
|
+
const elInfo = await page.evaluate((sel) => {
|
|
1292
|
+
const el = document.querySelector(sel);
|
|
1293
|
+
if (!el) return { found: false };
|
|
1294
|
+
return {
|
|
1295
|
+
found: true,
|
|
1296
|
+
tag: el.tagName,
|
|
1297
|
+
type: (el.type || '').toLowerCase(),
|
|
1298
|
+
isChecked: el.checked ?? false,
|
|
1299
|
+
};
|
|
1300
|
+
}, selector);
|
|
1301
|
+
|
|
1302
|
+
if (!elInfo.found) {
|
|
1303
|
+
return errorResponse(`Element [${normalizedId}] not found.`);
|
|
1304
|
+
}
|
|
1305
|
+
if (elInfo.tag !== 'INPUT' || !['checkbox', 'radio'].includes(elInfo.type)) {
|
|
1306
|
+
return errorResponse(`Element [${normalizedId}] is not a checkbox or radio (found: <${elInfo.tag} type="${elInfo.type}">).`);
|
|
1307
|
+
}
|
|
1308
|
+
|
|
1309
|
+
if (elInfo.isChecked === checked) {
|
|
1310
|
+
await audit('check', `[${normalizedId}] already ${checked ? 'checked' : 'unchecked'}`, null, state);
|
|
1311
|
+
return textResponse(`[${normalizedId}] is already ${checked ? 'checked' : 'unchecked'}. No action taken.`, { hint_id: normalizedId, checked, changed: false });
|
|
1312
|
+
}
|
|
1313
|
+
|
|
1314
|
+
const locator = page.locator(selector);
|
|
1315
|
+
await locator.click();
|
|
1316
|
+
await syncState(page, state, { force: true });
|
|
1317
|
+
|
|
1318
|
+
await audit('check', `[${normalizedId}] → ${checked ? 'checked' : 'unchecked'}`, null, state);
|
|
1319
|
+
return textResponse(
|
|
1320
|
+
`[${normalizedId}] is now ${checked ? 'checked' : 'unchecked'}.`,
|
|
1321
|
+
{ hint_id: normalizedId, checked, changed: true }
|
|
1322
|
+
);
|
|
1323
|
+
}
|
|
1324
|
+
);
|
|
1325
|
+
|
|
1326
|
+
server.registerTool(
|
|
1327
|
+
'key_down',
|
|
1328
|
+
{
|
|
1329
|
+
description: 'Press and hold a key without releasing. Use key_up to release. Useful for Shift+Click, Ctrl+Drag, etc.',
|
|
1330
|
+
inputSchema: {
|
|
1331
|
+
key: z.string().describe('Key name (e.g., "Shift", "Control", "Alt", "Meta", "a")'),
|
|
1332
|
+
},
|
|
1333
|
+
},
|
|
1334
|
+
async ({ key }) => {
|
|
1335
|
+
const confirmationError = await requireActionConfirmation('key_down');
|
|
1336
|
+
if (confirmationError) return confirmationError;
|
|
1337
|
+
const page = await getPageWithListeners({ state });
|
|
1338
|
+
await page.keyboard.down(key);
|
|
1339
|
+
await audit('key_down', key, null, state);
|
|
1340
|
+
return textResponse(`Key "${key}" pressed and held. Use key_up to release.`);
|
|
1341
|
+
}
|
|
1342
|
+
);
|
|
1343
|
+
|
|
1344
|
+
server.registerTool(
|
|
1345
|
+
'key_up',
|
|
1346
|
+
{
|
|
1347
|
+
description: 'Release a previously held key.',
|
|
1348
|
+
inputSchema: {
|
|
1349
|
+
key: z.string().describe('Key name to release (e.g., "Shift", "Control", "Alt", "Meta", "a")'),
|
|
1350
|
+
},
|
|
1351
|
+
},
|
|
1352
|
+
async ({ key }) => {
|
|
1353
|
+
const confirmationError = await requireActionConfirmation('key_up');
|
|
1354
|
+
if (confirmationError) return confirmationError;
|
|
1355
|
+
const page = await getPageWithListeners({ state });
|
|
1356
|
+
await page.keyboard.up(key);
|
|
1357
|
+
await audit('key_up', key, null, state);
|
|
1358
|
+
return textResponse(`Key "${key}" released.`);
|
|
1359
|
+
}
|
|
1360
|
+
);
|
|
1361
|
+
}
|