dsh-browser-plus 0.0.0-stage → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +166 -0
- package/LICENSE +22 -0
- package/NOTICE.md +7 -0
- package/README.en.md +100 -0
- package/README.md +99 -2
- package/assets/dsh-browser-plus-256.png +0 -0
- package/assets/dsh-browser-plus-512.png +0 -0
- package/assets/dsh-browser-plus-small.svg +9 -0
- package/assets/dsh-browser-plus.ico +0 -0
- package/assets/dsh-browser-plus.svg +11 -0
- package/assets/readme-workspace.png +0 -0
- package/cordis.patch.yml +17 -0
- package/docs/MIGRATION.md +48 -0
- package/docs/README.md +22 -0
- package/docs/SOAK-CHECKLIST.md +98 -0
- package/docs/architecture.md +88 -0
- package/docs/tool-reference.md +124 -0
- package/docs/user-guide.md +121 -0
- package/docs/why-browser.md +45 -0
- package/lib/browser/runtime.d.ts +225 -0
- package/lib/browser/runtime.js +302 -0
- package/lib/browser/types.d.ts +668 -0
- package/lib/browser/types.js +18 -0
- package/lib/browser-electron/auth-cookies.d.ts +54 -0
- package/lib/browser-electron/auth-cookies.js +83 -0
- package/lib/browser-electron/chrome-state.d.ts +187 -0
- package/lib/browser-electron/chrome-state.js +12 -0
- package/lib/browser-electron/entry.d.ts +66 -0
- package/lib/browser-electron/entry.js +62 -0
- package/lib/browser-electron/fingerprint.d.ts +29 -0
- package/lib/browser-electron/fingerprint.js +42 -0
- package/lib/browser-electron/host-main.d.ts +18 -0
- package/lib/browser-electron/host-main.js +2494 -0
- package/lib/browser-electron/icon.d.ts +11 -0
- package/lib/browser-electron/icon.js +23 -0
- package/lib/browser-electron/page-chrome.d.ts +21 -0
- package/lib/browser-electron/page-chrome.js +2034 -0
- package/lib/browser-electron/provider.d.ts +709 -0
- package/lib/browser-electron/provider.js +2575 -0
- package/lib/browser-electron/remote-host.d.ts +143 -0
- package/lib/browser-electron/remote-host.js +952 -0
- package/lib/browser-electron/task-summary.d.ts +2 -0
- package/lib/browser-electron/task-summary.js +12 -0
- package/lib/browser-electron/task-thumbnail.d.ts +11 -0
- package/lib/browser-electron/task-thumbnail.js +9 -0
- package/lib/browser-electron/write-guard.d.ts +41 -0
- package/lib/browser-electron/write-guard.js +123 -0
- package/lib/index.d.ts +16 -0
- package/lib/index.js +14 -0
- package/lib/tool-browser/index.d.ts +31 -0
- package/lib/tool-browser/index.js +1931 -0
- package/package.json +95 -4
- package/screenshots.json +3 -0
- package/scripts/build-icons.mjs +80 -0
- package/scripts/capture-window.ps1 +79 -0
- package/scripts/crop-image.ps1 +20 -0
- package/scripts/smoke-browser-tools.mjs +1968 -0
- package/scripts/smoke-chrome-world.mjs +63 -0
- package/scripts/smoke-electron-host.mjs +50 -0
- package/src/browser/runtime.ts +470 -0
- package/src/browser/types.ts +649 -0
- package/src/browser-electron/auth-cookies.ts +125 -0
- package/src/browser-electron/chrome-state.ts +174 -0
- package/src/browser-electron/entry.ts +115 -0
- package/src/browser-electron/fingerprint.ts +45 -0
- package/src/browser-electron/host-main.ts +2330 -0
- package/src/browser-electron/icon.ts +26 -0
- package/src/browser-electron/page-chrome.ts +2046 -0
- package/src/browser-electron/provider.ts +3088 -0
- package/src/browser-electron/remote-host.ts +1004 -0
- package/src/browser-electron/task-summary.ts +10 -0
- package/src/browser-electron/task-thumbnail.ts +17 -0
- package/src/browser-electron/write-guard.ts +134 -0
- package/src/index.ts +52 -0
- package/src/tool-browser/index.ts +1974 -0
- package/src/types/electron-shim.d.ts +143 -0
|
@@ -0,0 +1,1931 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Model-facing browser tools over `ctx.browser`: `browser_open`,
|
|
3
|
+
* `browser_snapshot`, `browser_execute`, `browser_content`,
|
|
4
|
+
* `browser_screenshot`, and tab management (`browser_list_tabs`,
|
|
5
|
+
* `browser_switch_tab`, `browser_close_tab`, `browser_reset`).
|
|
6
|
+
*
|
|
7
|
+
* The tool layer owns only the model-facing schema, argument validation, and
|
|
8
|
+
* result formatting — never provider selection or page driving, which belong
|
|
9
|
+
* to the seam. Session lifecycle is owned here at the plugin level: each
|
|
10
|
+
* calling task (a DSH session) gets its own browser session — the first
|
|
11
|
+
* `browser_open` (or any tool when no session exists) opens it, and later
|
|
12
|
+
* tools in the same task reuse it. Concurrent tasks therefore never fight
|
|
13
|
+
* over tabs, history, or navigation state.
|
|
14
|
+
* @module dsh-browser-plus/tool-browser
|
|
15
|
+
*/
|
|
16
|
+
import { defineTool } from '@deepseek-ai/dsh-tools';
|
|
17
|
+
/** Plugin name used by loader diagnostics. */
|
|
18
|
+
export const name = 'tool-browser';
|
|
19
|
+
/** The tool registry, browser seam, and system-prompt registry this tool layer consumes. */
|
|
20
|
+
export const inject = ['tools', 'browser', 'systemPrompt'];
|
|
21
|
+
/** Per-task browser sessions, keyed by the calling DSH session id. */
|
|
22
|
+
const sessionsByTask = new Map();
|
|
23
|
+
/** In-flight first-open per task key, so concurrent first calls share one session. */
|
|
24
|
+
const pendingOpens = new Map();
|
|
25
|
+
/**
|
|
26
|
+
* Tail promise for queued operations in one browser task. Entries are dropped
|
|
27
|
+
* as soon as a queue drains (see {@link queueTaskOperation}), so this stays
|
|
28
|
+
* bounded by the number of tasks with work in flight instead of growing one
|
|
29
|
+
* permanent entry per task key.
|
|
30
|
+
*/
|
|
31
|
+
const operationTails = new Map();
|
|
32
|
+
/** In-flight identical read requests, keyed by task and canonical request shape. */
|
|
33
|
+
const pendingReads = new Map();
|
|
34
|
+
/**
|
|
35
|
+
* Queue one operation after prior work for the same task (per-task FIFO).
|
|
36
|
+
*
|
|
37
|
+
* Both page-changing actions and read-only operations go through here, so a
|
|
38
|
+
* read issued while a navigation is in flight resolves against the post-write
|
|
39
|
+
* page instead of racing it. The entry is removed once its queue is idle: the
|
|
40
|
+
* identity check means a newer operation that already chained onto this tail
|
|
41
|
+
* keeps its ordering, because the map no longer points at this tail.
|
|
42
|
+
*/
|
|
43
|
+
function queueTaskOperation(key, operation) {
|
|
44
|
+
const prior = operationTails.get(key) ?? Promise.resolve();
|
|
45
|
+
const result = prior.catch(() => undefined).then(operation);
|
|
46
|
+
const tail = result.then(() => undefined, () => undefined);
|
|
47
|
+
operationTails.set(key, tail);
|
|
48
|
+
void tail.then(() => {
|
|
49
|
+
if (operationTails.get(key) === tail)
|
|
50
|
+
operationTails.delete(key);
|
|
51
|
+
});
|
|
52
|
+
return result;
|
|
53
|
+
}
|
|
54
|
+
/** Keep one identical read in flight per task instead of repeating CDP work. */
|
|
55
|
+
function coalesceTaskRead(key, readKey, operation) {
|
|
56
|
+
const cacheKey = key + '\u0000' + readKey;
|
|
57
|
+
const existing = pendingReads.get(cacheKey);
|
|
58
|
+
if (existing !== undefined)
|
|
59
|
+
return existing;
|
|
60
|
+
const result = operation().finally(() => pendingReads.delete(cacheKey));
|
|
61
|
+
pendingReads.set(cacheKey, result);
|
|
62
|
+
return result;
|
|
63
|
+
}
|
|
64
|
+
/** True when the provider no longer knows the cached session id. */
|
|
65
|
+
function isUnknownSession(error) {
|
|
66
|
+
return error instanceof Error && error.code === 'BROWSER_SESSION_UNKNOWN';
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Run one operation against the task's session, reopening it once when the
|
|
70
|
+
* provider has forgotten the cached id — for example when the browser row
|
|
71
|
+
* reloaded and the tool layer still holds a session from the previous provider.
|
|
72
|
+
* Without this the task fails on every later call until someone happens to run
|
|
73
|
+
* browser_reset_session.
|
|
74
|
+
*/
|
|
75
|
+
async function withRecoveredSession(browser, key, run) {
|
|
76
|
+
try {
|
|
77
|
+
return await run(await ensureSession(browser, key));
|
|
78
|
+
}
|
|
79
|
+
catch (error) {
|
|
80
|
+
if (!isUnknownSession(error))
|
|
81
|
+
throw error;
|
|
82
|
+
sessionsByTask.delete(key);
|
|
83
|
+
return run(await ensureSession(browser, key));
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
/** Run a page-changing action with visible task status and FIFO ordering. */
|
|
87
|
+
async function withTaskAction(browser, key, action, exec, operation, label) {
|
|
88
|
+
await ensureSession(browser, key, label);
|
|
89
|
+
return queueTaskOperation(key, () => withRecoveredSession(browser, key, async (session) => {
|
|
90
|
+
const task = await browser.getTask(session);
|
|
91
|
+
if (task.control === 'human') {
|
|
92
|
+
await browser.updateTask(session, { status: 'waiting-user', latestAction: 'waiting for user' });
|
|
93
|
+
throw new Error('browser action is paused while the human controls this task');
|
|
94
|
+
}
|
|
95
|
+
await browser.updateTask(session, { status: 'running', latestAction: action });
|
|
96
|
+
try {
|
|
97
|
+
const result = await operation(session);
|
|
98
|
+
const current = await browser.getTask(session).catch(() => undefined);
|
|
99
|
+
await browser.updateTask(session, current?.control === 'human'
|
|
100
|
+
? { status: 'waiting-user', latestAction: 'waiting for user' }
|
|
101
|
+
: { status: 'idle', latestAction: action });
|
|
102
|
+
return result;
|
|
103
|
+
}
|
|
104
|
+
catch (error) {
|
|
105
|
+
const message = String(error);
|
|
106
|
+
const current = await browser.getTask(session).catch(() => undefined);
|
|
107
|
+
await browser.updateTask(session, current?.control === 'human'
|
|
108
|
+
? { status: 'waiting-user', latestAction: 'waiting for user' }
|
|
109
|
+
: { status: 'failed', latestAction: action, error: message });
|
|
110
|
+
throw error;
|
|
111
|
+
}
|
|
112
|
+
}));
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* Run and coalesce a read-only operation for the current task.
|
|
116
|
+
*
|
|
117
|
+
* The read joins the task's FIFO queue, so a read issued while a write is in
|
|
118
|
+
* flight (e.g. browser_open + browser_snapshot, both marked concurrency-safe)
|
|
119
|
+
* resolves against the post-write page instead of racing the navigation.
|
|
120
|
+
*
|
|
121
|
+
* Dedup deliberately wraps the queue: if the queue wrapped the dedup, a second
|
|
122
|
+
* identical read would start only after the first settled, miss the in-flight
|
|
123
|
+
* entry, and issue a second CDP call. The queued closure never re-enters the
|
|
124
|
+
* queue, so this cannot deadlock.
|
|
125
|
+
*/
|
|
126
|
+
async function withTaskRead(browser, key, readKey, operation) {
|
|
127
|
+
await ensureSession(browser, key);
|
|
128
|
+
return coalesceTaskRead(key, readKey, () => queueTaskOperation(key, () => withRecoveredSession(browser, key, operation)));
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* Per-task action restriction. `browser_restrict` writes an entry keyed by the
|
|
132
|
+
* calling task, so one task's allow-list never restricts another task's tools.
|
|
133
|
+
* An entry holding an empty array means that task explicitly lifted the
|
|
134
|
+
* restriction — distinct from having no entry at all, which inherits the
|
|
135
|
+
* plugin-level default.
|
|
136
|
+
*/
|
|
137
|
+
const restrictedToByTask = new Map();
|
|
138
|
+
/** Plugin-level default from config; applies to tasks that set no rule of their own. */
|
|
139
|
+
let defaultRestriction;
|
|
140
|
+
/** The allow-list in force for one task, or undefined when unrestricted. */
|
|
141
|
+
function restrictionFor(key) {
|
|
142
|
+
const explicit = restrictedToByTask.get(key);
|
|
143
|
+
if (explicit === undefined)
|
|
144
|
+
return defaultRestriction;
|
|
145
|
+
return explicit.length === 0 ? undefined : explicit;
|
|
146
|
+
}
|
|
147
|
+
/**
|
|
148
|
+
* Guard one browser tool call against the calling task's restriction. Refuses
|
|
149
|
+
* calls that are not on that task's allow-list when a restriction is in effect.
|
|
150
|
+
* @param toolName - the browser tool about to run.
|
|
151
|
+
* @param exec - the tool-execution context; only its agent id (task key) is read.
|
|
152
|
+
*/
|
|
153
|
+
function assertAllowed(toolName, exec) {
|
|
154
|
+
const restrictedTo = restrictionFor(taskKey(exec));
|
|
155
|
+
if (restrictedTo === undefined)
|
|
156
|
+
return;
|
|
157
|
+
if (restrictedTo.includes(toolName))
|
|
158
|
+
return;
|
|
159
|
+
throw new Error(`browser action "${toolName}" is restricted for this task (allow-list: ${restrictedTo.join(', ')})`);
|
|
160
|
+
}
|
|
161
|
+
/**
|
|
162
|
+
* The task key for a tool call: the calling DSH session id, or the shared
|
|
163
|
+
* default key when the call carries no agent context (CLI probes, tests).
|
|
164
|
+
* @param exec - the tool-execution context; only its optional agent id is read.
|
|
165
|
+
*/
|
|
166
|
+
function taskKey(exec) {
|
|
167
|
+
return exec?.agent?.id ?? 'default';
|
|
168
|
+
}
|
|
169
|
+
/**
|
|
170
|
+
* Resolve the calling task's browser session, opening one on first use.
|
|
171
|
+
* The Provider also recovers a keyed session if this tool-layer cache was lost.
|
|
172
|
+
* Concurrent first calls for the same key share a single open.
|
|
173
|
+
* @param browser - the seam service.
|
|
174
|
+
* @param key - the task key (see {@link taskKey}).
|
|
175
|
+
* @param label - optional space name applied when the session is first opened.
|
|
176
|
+
* @returns the task's session id.
|
|
177
|
+
*/
|
|
178
|
+
async function ensureSession(browser, key, label) {
|
|
179
|
+
const existing = sessionsByTask.get(key);
|
|
180
|
+
if (existing !== undefined)
|
|
181
|
+
return existing;
|
|
182
|
+
const pending = pendingOpens.get(key);
|
|
183
|
+
if (pending !== undefined)
|
|
184
|
+
return pending;
|
|
185
|
+
const opening = browser.open({
|
|
186
|
+
key,
|
|
187
|
+
...label !== undefined && label !== '' ? { label } : {},
|
|
188
|
+
}).then(session => { sessionsByTask.set(key, session); pendingOpens.delete(key); return session; }, error => { pendingOpens.delete(key); throw error; });
|
|
189
|
+
pendingOpens.set(key, opening);
|
|
190
|
+
return opening;
|
|
191
|
+
}
|
|
192
|
+
/** Coerce a tool-provided string value back to boolean/number only when lossless. */
|
|
193
|
+
function parseFillValue(v) {
|
|
194
|
+
if (v === 'true')
|
|
195
|
+
return true;
|
|
196
|
+
if (v === 'false')
|
|
197
|
+
return false;
|
|
198
|
+
if (v !== undefined && /^-?\d+(\.\d+)?$/.test(v) && String(Number(v)) === v)
|
|
199
|
+
return Number(v);
|
|
200
|
+
return v ?? '';
|
|
201
|
+
}
|
|
202
|
+
/** Longest string kept in one history row handed to the model. */
|
|
203
|
+
const HISTORY_OUTPUT_MAX_CHARS = 500;
|
|
204
|
+
/**
|
|
205
|
+
* A shallow, bounded copy of one entry's params. The previous deep clone
|
|
206
|
+
* (JSON.parse(JSON.stringify(...))) copied every stored script and typed text in
|
|
207
|
+
* full on every browser_history call, and the row is only ever rendered.
|
|
208
|
+
*/
|
|
209
|
+
function summarizeHistoryParams(params) {
|
|
210
|
+
const out = {};
|
|
211
|
+
for (const [key, value] of Object.entries(params)) {
|
|
212
|
+
// Dropping undefined keeps the row lossless-JSON, which the deep clone used
|
|
213
|
+
// to guarantee.
|
|
214
|
+
if (value === undefined)
|
|
215
|
+
continue;
|
|
216
|
+
out[key] = typeof value === 'string' && value.length > HISTORY_OUTPUT_MAX_CHARS
|
|
217
|
+
? `${value.slice(0, HISTORY_OUTPUT_MAX_CHARS)}…(${value.length - HISTORY_OUTPUT_MAX_CHARS} more)`
|
|
218
|
+
: value;
|
|
219
|
+
}
|
|
220
|
+
return out;
|
|
221
|
+
}
|
|
222
|
+
/** Format a snapshot element list for the model. */
|
|
223
|
+
function formatSnapshot(snapshot) {
|
|
224
|
+
const lines = snapshot.elements.map(el => `[${el.ref}] ${el.kind}: ${el.label} (${el.x},${el.y}) loc=${el.loc}`);
|
|
225
|
+
const header = `URL: ${snapshot.url}${snapshot.title !== undefined ? `\nTitle: ${snapshot.title}` : ''}${snapshot.snapshotId !== undefined ? `\nSnapshot: ${snapshot.snapshotId}` : ''}`;
|
|
226
|
+
const body = lines.length > 0 ? lines.join('\n') : '(no interactive elements found)';
|
|
227
|
+
const tail = snapshot.truncated === true ? '\n(snapshot truncated)' : '';
|
|
228
|
+
const banner = snapshot.challenge?.blocked === true
|
|
229
|
+
? `\n\nCHALLENGE: ${snapshot.challenge.reason ?? 'human-verification'}. Do NOT keep retrying — ask the human to complete it in the shared browser window, then re-snapshot.`
|
|
230
|
+
: '';
|
|
231
|
+
const user = snapshot.userControlling === true
|
|
232
|
+
? '\n\nUSER CONTROL: the human is using this page. Do not operate the browser until they hand it back.'
|
|
233
|
+
: '';
|
|
234
|
+
return `${header}\n\n${body}${tail}${banner}${user}`;
|
|
235
|
+
}
|
|
236
|
+
/** Register all browser tools with `ctx.tools`. */
|
|
237
|
+
export function apply(ctx, config = {}) {
|
|
238
|
+
const timeoutMs = config.timeoutMs ?? 60_000;
|
|
239
|
+
/**
|
|
240
|
+
* A caller-supplied budget is capped below the tool's own deadline, so the
|
|
241
|
+
* provider reports a clean timeout instead of the runtime aborting the call.
|
|
242
|
+
*/
|
|
243
|
+
const withinToolBudget = (requested, fallback) => Math.min(requested ?? fallback, Math.max(timeoutMs - 5_000, 1_000));
|
|
244
|
+
// Re-apply clears task-scoped rules and re-seeds the plugin-level default;
|
|
245
|
+
// an omitted allowedActions lifts the default.
|
|
246
|
+
restrictedToByTask.clear();
|
|
247
|
+
defaultRestriction = config.allowedActions !== undefined ? [...config.allowedActions] : undefined;
|
|
248
|
+
ctx.systemPrompt.section({
|
|
249
|
+
name: 'tool:browser',
|
|
250
|
+
// Tool guidance band is 100-199; 150 keeps clear of the common 110/120
|
|
251
|
+
// tool sections so ordering does not depend on plugin load sequence.
|
|
252
|
+
order: 150,
|
|
253
|
+
text: 'Use the browser_* tools to operate a real shared browser the human can see and take over. Start with browser_snapshot, then use browser_click_ref or browser_scroll_into_view with its snapshotId and reference number whenever possible; re-snapshot if a reference is stale. Use browser_fill for forms and browser_back/browser_forward/browser_reload/browser_stop/browser_scroll for normal browser controls before resorting to browser_execute. browser_screenshot is for visual confirmation, not primary targeting. Keep the human informed of what you are doing on the page. Each task gets its own browser session: tabs and history are isolated from other tasks. browser_handoff state="waiting-user" marks a task for human action; do not operate page-changing tools while the user owns the task. If a snapshot or browser_challenge reports a CAPTCHA, stop retrying, hand off to the human, then re-check.',
|
|
254
|
+
});
|
|
255
|
+
ctx.tools.register(defineTool({
|
|
256
|
+
name: 'browser_open',
|
|
257
|
+
description: 'Open a URL in the shared browser window. Opens this task\'s browser session on first use; optionally opens in a new tab. Returns the resulting page snapshot.',
|
|
258
|
+
parameters: {
|
|
259
|
+
url: { type: 'string', required: true, description: 'The URL to open (HTTP/HTTPS).' },
|
|
260
|
+
newTab: { type: 'boolean', description: 'Open in a new tab instead of the active one.' },
|
|
261
|
+
space: { type: 'string', description: 'Optional browser-task label shown in the task manager and active window title.' },
|
|
262
|
+
},
|
|
263
|
+
output: {
|
|
264
|
+
schema: {
|
|
265
|
+
type: 'object',
|
|
266
|
+
additionalProperties: false,
|
|
267
|
+
properties: {
|
|
268
|
+
snapshotId: { type: 'string', required: true },
|
|
269
|
+
url: { type: 'string', required: true },
|
|
270
|
+
title: { type: 'string' },
|
|
271
|
+
truncated: { type: 'boolean' },
|
|
272
|
+
elements: {
|
|
273
|
+
type: 'array',
|
|
274
|
+
required: true,
|
|
275
|
+
items: {
|
|
276
|
+
type: 'object',
|
|
277
|
+
additionalProperties: false,
|
|
278
|
+
properties: {
|
|
279
|
+
ref: { type: 'number', required: true },
|
|
280
|
+
kind: { type: 'string', required: true },
|
|
281
|
+
label: { type: 'string', required: true },
|
|
282
|
+
x: { type: 'number', required: true },
|
|
283
|
+
y: { type: 'number', required: true },
|
|
284
|
+
loc: { type: 'string', required: true },
|
|
285
|
+
},
|
|
286
|
+
},
|
|
287
|
+
},
|
|
288
|
+
challenge: {
|
|
289
|
+
type: 'object',
|
|
290
|
+
additionalProperties: false,
|
|
291
|
+
properties: {
|
|
292
|
+
blocked: { type: 'boolean', required: true },
|
|
293
|
+
kind: { type: 'string' },
|
|
294
|
+
reason: { type: 'string' },
|
|
295
|
+
},
|
|
296
|
+
},
|
|
297
|
+
userControlling: { type: 'boolean' },
|
|
298
|
+
},
|
|
299
|
+
},
|
|
300
|
+
render: (_args, value) => [{ type: 'text', text: formatSnapshot(value) }],
|
|
301
|
+
},
|
|
302
|
+
timeoutMs,
|
|
303
|
+
isConcurrencySafe: () => true,
|
|
304
|
+
async execute(args, exec) {
|
|
305
|
+
assertAllowed('browser_open', exec);
|
|
306
|
+
const browser = ctx.get('browser');
|
|
307
|
+
if (browser === undefined)
|
|
308
|
+
throw new Error('tool-browser: browser service unavailable');
|
|
309
|
+
const key = taskKey(exec);
|
|
310
|
+
return withTaskAction(browser, key, 'open page', exec, async (session) => {
|
|
311
|
+
await browser.openUrl(session, {
|
|
312
|
+
url: args.url,
|
|
313
|
+
...args.newTab === true ? { newTab: true } : {},
|
|
314
|
+
}, exec.signal);
|
|
315
|
+
const snapshot = await browser.snapshot(session, {}, exec.signal);
|
|
316
|
+
return {
|
|
317
|
+
snapshotId: snapshot.snapshotId,
|
|
318
|
+
url: snapshot.url,
|
|
319
|
+
...snapshot.title !== undefined ? { title: snapshot.title } : {},
|
|
320
|
+
elements: snapshot.elements.map(el => ({ ref: el.ref, kind: el.kind, label: el.label, x: el.x, y: el.y, loc: el.loc })),
|
|
321
|
+
truncated: snapshot.truncated,
|
|
322
|
+
...snapshot.challenge !== undefined ? { challenge: snapshot.challenge } : {},
|
|
323
|
+
...snapshot.userControlling !== undefined ? { userControlling: snapshot.userControlling } : {},
|
|
324
|
+
};
|
|
325
|
+
}, args.space);
|
|
326
|
+
},
|
|
327
|
+
}));
|
|
328
|
+
ctx.tools.register(defineTool({
|
|
329
|
+
name: 'browser_back',
|
|
330
|
+
description: 'Navigate the active tab to the previous history entry when one exists.',
|
|
331
|
+
parameters: {},
|
|
332
|
+
output: {
|
|
333
|
+
schema: { type: 'object', additionalProperties: false, properties: { navigated: { type: 'boolean', required: true } } },
|
|
334
|
+
render: (_args, value) => [{ type: 'text', text: value.navigated ? 'Navigated back.' : 'No previous page in this tab.' }],
|
|
335
|
+
},
|
|
336
|
+
timeoutMs,
|
|
337
|
+
isConcurrencySafe: () => false,
|
|
338
|
+
async execute(_args, exec) {
|
|
339
|
+
assertAllowed('browser_back', exec);
|
|
340
|
+
const browser = ctx.get('browser');
|
|
341
|
+
if (browser === undefined)
|
|
342
|
+
throw new Error('tool-browser: browser service unavailable');
|
|
343
|
+
const navigated = await withTaskAction(browser, taskKey(exec), 'go back', exec, session => browser.back(session, exec.signal));
|
|
344
|
+
return { navigated };
|
|
345
|
+
},
|
|
346
|
+
}));
|
|
347
|
+
ctx.tools.register(defineTool({
|
|
348
|
+
name: 'browser_forward',
|
|
349
|
+
description: 'Navigate the active tab to the next history entry when one exists.',
|
|
350
|
+
parameters: {},
|
|
351
|
+
output: {
|
|
352
|
+
schema: { type: 'object', additionalProperties: false, properties: { navigated: { type: 'boolean', required: true } } },
|
|
353
|
+
render: (_args, value) => [{ type: 'text', text: value.navigated ? 'Navigated forward.' : 'No next page in this tab.' }],
|
|
354
|
+
},
|
|
355
|
+
timeoutMs,
|
|
356
|
+
isConcurrencySafe: () => false,
|
|
357
|
+
async execute(_args, exec) {
|
|
358
|
+
assertAllowed('browser_forward', exec);
|
|
359
|
+
const browser = ctx.get('browser');
|
|
360
|
+
if (browser === undefined)
|
|
361
|
+
throw new Error('tool-browser: browser service unavailable');
|
|
362
|
+
const navigated = await withTaskAction(browser, taskKey(exec), 'go forward', exec, session => browser.forward(session, exec.signal));
|
|
363
|
+
return { navigated };
|
|
364
|
+
},
|
|
365
|
+
}));
|
|
366
|
+
ctx.tools.register(defineTool({
|
|
367
|
+
name: 'browser_reload',
|
|
368
|
+
description: 'Reload the active page in the shared browser.',
|
|
369
|
+
parameters: {},
|
|
370
|
+
output: {
|
|
371
|
+
schema: { type: 'object', additionalProperties: false, properties: { reloaded: { type: 'boolean', required: true } } },
|
|
372
|
+
render: () => [{ type: 'text', text: 'Reloaded the active page.' }],
|
|
373
|
+
},
|
|
374
|
+
timeoutMs,
|
|
375
|
+
isConcurrencySafe: () => false,
|
|
376
|
+
async execute(_args, exec) {
|
|
377
|
+
assertAllowed('browser_reload', exec);
|
|
378
|
+
const browser = ctx.get('browser');
|
|
379
|
+
if (browser === undefined)
|
|
380
|
+
throw new Error('tool-browser: browser service unavailable');
|
|
381
|
+
await withTaskAction(browser, taskKey(exec), 'reload page', exec, session => browser.reload(session, exec.signal));
|
|
382
|
+
return { reloaded: true };
|
|
383
|
+
},
|
|
384
|
+
}));
|
|
385
|
+
ctx.tools.register(defineTool({
|
|
386
|
+
name: 'browser_stop',
|
|
387
|
+
description: 'Stop the active page from loading.',
|
|
388
|
+
parameters: {},
|
|
389
|
+
output: {
|
|
390
|
+
schema: { type: 'object', additionalProperties: false, properties: { stopped: { type: 'boolean', required: true } } },
|
|
391
|
+
render: () => [{ type: 'text', text: 'Stopped page loading.' }],
|
|
392
|
+
},
|
|
393
|
+
timeoutMs,
|
|
394
|
+
isConcurrencySafe: () => false,
|
|
395
|
+
async execute(_args, exec) {
|
|
396
|
+
assertAllowed('browser_stop', exec);
|
|
397
|
+
const browser = ctx.get('browser');
|
|
398
|
+
if (browser === undefined)
|
|
399
|
+
throw new Error('tool-browser: browser service unavailable');
|
|
400
|
+
await withTaskAction(browser, taskKey(exec), 'stop loading', exec, session => browser.stopLoading(session, exec.signal));
|
|
401
|
+
return { stopped: true };
|
|
402
|
+
},
|
|
403
|
+
}));
|
|
404
|
+
ctx.tools.register(defineTool({
|
|
405
|
+
name: 'browser_space',
|
|
406
|
+
description: 'Name this browser task or list browser tasks. The task manager controls which isolated task view is visible in the shared window.',
|
|
407
|
+
parameters: {
|
|
408
|
+
label: { type: 'string', description: 'New display name for this browser task. Omit to list tasks.' },
|
|
409
|
+
},
|
|
410
|
+
output: {
|
|
411
|
+
schema: {
|
|
412
|
+
type: 'object',
|
|
413
|
+
additionalProperties: false,
|
|
414
|
+
properties: {
|
|
415
|
+
label: { type: 'string' },
|
|
416
|
+
spaces: {
|
|
417
|
+
type: 'array',
|
|
418
|
+
items: {
|
|
419
|
+
type: 'object',
|
|
420
|
+
additionalProperties: false,
|
|
421
|
+
properties: {
|
|
422
|
+
key: { type: 'string', required: true },
|
|
423
|
+
label: { type: 'string', required: true },
|
|
424
|
+
},
|
|
425
|
+
},
|
|
426
|
+
},
|
|
427
|
+
},
|
|
428
|
+
},
|
|
429
|
+
render: (_args, value) => {
|
|
430
|
+
if (value.label !== undefined)
|
|
431
|
+
return [{ type: 'text', text: `Browser task named "${value.label}".` }];
|
|
432
|
+
const spaces = value.spaces;
|
|
433
|
+
const lines = spaces.length === 0 ? '(no browser tasks open)' : spaces.map(s => `${s.key}${s.label !== '' ? ` — ${s.label}` : ''}`).join('\n');
|
|
434
|
+
return [{ type: 'text', text: `Browser tasks:\n${lines}` }];
|
|
435
|
+
},
|
|
436
|
+
},
|
|
437
|
+
timeoutMs,
|
|
438
|
+
isConcurrencySafe: () => true,
|
|
439
|
+
async execute(args, exec) {
|
|
440
|
+
assertAllowed('browser_space', exec);
|
|
441
|
+
const browser = ctx.get('browser');
|
|
442
|
+
if (browser === undefined)
|
|
443
|
+
throw new Error('tool-browser: browser service unavailable');
|
|
444
|
+
const key = taskKey(exec);
|
|
445
|
+
const label = args.label;
|
|
446
|
+
if (label === undefined && sessionsByTask.get(key) === undefined) {
|
|
447
|
+
// LIST mode must not force-open a visible window just to enumerate
|
|
448
|
+
// spaces: consult this task's session map before opening anything.
|
|
449
|
+
// This reads the task registry rather than page state, so it stays
|
|
450
|
+
// outside the page FIFO.
|
|
451
|
+
const spaces = await browser.listSpaces();
|
|
452
|
+
return { spaces: spaces.map(s => ({ key: s.key, label: s.label ?? '' })) };
|
|
453
|
+
}
|
|
454
|
+
const session = await ensureSession(browser, key);
|
|
455
|
+
if (label !== undefined) {
|
|
456
|
+
// Naming a task mutates shared task-manager state: keep FIFO order.
|
|
457
|
+
await queueTaskOperation(key, () => browser.setSpace(session, label));
|
|
458
|
+
return { label };
|
|
459
|
+
}
|
|
460
|
+
const spaces = await withTaskRead(browser, key, 'spaces', () => browser.listSpaces());
|
|
461
|
+
return { spaces: spaces.map(s => ({ key: s.key, label: s.label ?? '' })) };
|
|
462
|
+
},
|
|
463
|
+
}));
|
|
464
|
+
ctx.tools.register(defineTool({
|
|
465
|
+
name: 'browser_tasks',
|
|
466
|
+
description: 'List browser tasks with their visible activity, collaboration owner, tab count, and latest action.',
|
|
467
|
+
parameters: {},
|
|
468
|
+
output: {
|
|
469
|
+
schema: {
|
|
470
|
+
type: 'object', additionalProperties: false, properties: {
|
|
471
|
+
tasks: {
|
|
472
|
+
type: 'array', required: true, items: {
|
|
473
|
+
type: 'object', additionalProperties: false, properties: {
|
|
474
|
+
key: { type: 'string', required: true },
|
|
475
|
+
label: { type: 'string', required: true },
|
|
476
|
+
active: { type: 'boolean', required: true },
|
|
477
|
+
tabs: { type: 'number', required: true },
|
|
478
|
+
status: { type: 'string', required: true },
|
|
479
|
+
control: { type: 'string', required: true },
|
|
480
|
+
latestAction: { type: 'string' },
|
|
481
|
+
updatedAt: { type: 'number', required: true },
|
|
482
|
+
error: { type: 'string' },
|
|
483
|
+
},
|
|
484
|
+
},
|
|
485
|
+
},
|
|
486
|
+
},
|
|
487
|
+
},
|
|
488
|
+
render: (_args, value) => [{ type: 'text', text: value.tasks.map(task => `${task.active ? '*' : ' '} ${task.label || task.key} — ${task.status}, ${task.control}, ${task.tabs} tabs${task.latestAction !== undefined ? ` — ${task.latestAction}` : ''}`).join('\n') || '(no browser tasks open)' }],
|
|
489
|
+
},
|
|
490
|
+
timeoutMs,
|
|
491
|
+
isConcurrencySafe: () => true,
|
|
492
|
+
async execute() {
|
|
493
|
+
const browser = ctx.get('browser');
|
|
494
|
+
if (browser === undefined)
|
|
495
|
+
throw new Error('tool-browser: browser service unavailable');
|
|
496
|
+
const tasks = await browser.listTasks();
|
|
497
|
+
return { tasks: tasks.map(task => ({
|
|
498
|
+
key: task.key,
|
|
499
|
+
label: task.label,
|
|
500
|
+
active: task.active,
|
|
501
|
+
tabs: task.tabs,
|
|
502
|
+
status: task.status,
|
|
503
|
+
control: task.control,
|
|
504
|
+
...task.latestAction !== undefined ? { latestAction: task.latestAction } : {},
|
|
505
|
+
updatedAt: task.updatedAt,
|
|
506
|
+
...task.error !== undefined ? { error: task.error } : {},
|
|
507
|
+
})) };
|
|
508
|
+
},
|
|
509
|
+
}));
|
|
510
|
+
ctx.tools.register(defineTool({
|
|
511
|
+
name: 'browser_handoff',
|
|
512
|
+
description: 'Mark this browser task as waiting for the human, or return it to Agent control after a handoff.',
|
|
513
|
+
parameters: {
|
|
514
|
+
state: { type: 'string', required: true, enum: ['waiting-user', 'agent'], description: 'waiting-user pauses Agent page changes; agent returns control to the Agent.' },
|
|
515
|
+
},
|
|
516
|
+
output: {
|
|
517
|
+
schema: {
|
|
518
|
+
type: 'object', additionalProperties: false, properties: {
|
|
519
|
+
key: { type: 'string', required: true },
|
|
520
|
+
label: { type: 'string', required: true },
|
|
521
|
+
status: { type: 'string', required: true },
|
|
522
|
+
control: { type: 'string', required: true },
|
|
523
|
+
},
|
|
524
|
+
},
|
|
525
|
+
render: (_args, value) => [{ type: 'text', text: value.status === 'waiting-user' ? 'Waiting for the human in the shared browser.' : 'Agent browser control resumed.' }],
|
|
526
|
+
},
|
|
527
|
+
timeoutMs,
|
|
528
|
+
isConcurrencySafe: () => false,
|
|
529
|
+
async execute(args, exec) {
|
|
530
|
+
// Handoff is not read-only: it changes the task's control state.
|
|
531
|
+
assertAllowed('browser_handoff', exec);
|
|
532
|
+
const browser = ctx.get('browser');
|
|
533
|
+
if (browser === undefined)
|
|
534
|
+
throw new Error('tool-browser: browser service unavailable');
|
|
535
|
+
const key = taskKey(exec);
|
|
536
|
+
const session = await ensureSession(browser, key);
|
|
537
|
+
const task = await queueTaskOperation(key, () => browser.setHandoff(session, args.state));
|
|
538
|
+
return { key: task.key, label: task.label, status: task.status, control: task.control };
|
|
539
|
+
},
|
|
540
|
+
}));
|
|
541
|
+
ctx.tools.register(defineTool({
|
|
542
|
+
name: 'browser_snapshot',
|
|
543
|
+
description: 'Return an AI-friendly snapshot of the current shared-browser page (optionally filtered): numbered interactive elements (inputs, buttons, links) the model can cite. Use this to understand an interactive page before driving it.',
|
|
544
|
+
parameters: {
|
|
545
|
+
query: { type: 'string', description: 'Case-insensitive substring matched against each element\'s kind and label (e.g. "sign in", "email", "submit"). Filtering happens before the cap, so it can reach elements past the default limit.' },
|
|
546
|
+
limit: { type: 'number', description: 'Maximum number of elements to return (1-1000; default 60).' },
|
|
547
|
+
},
|
|
548
|
+
output: {
|
|
549
|
+
schema: {
|
|
550
|
+
type: 'object',
|
|
551
|
+
additionalProperties: false,
|
|
552
|
+
properties: {
|
|
553
|
+
snapshotId: { type: 'string', required: true },
|
|
554
|
+
url: { type: 'string', required: true },
|
|
555
|
+
title: { type: 'string' },
|
|
556
|
+
truncated: { type: 'boolean' },
|
|
557
|
+
elements: {
|
|
558
|
+
type: 'array',
|
|
559
|
+
required: true,
|
|
560
|
+
items: {
|
|
561
|
+
type: 'object',
|
|
562
|
+
additionalProperties: false,
|
|
563
|
+
properties: {
|
|
564
|
+
ref: { type: 'number', required: true },
|
|
565
|
+
kind: { type: 'string', required: true },
|
|
566
|
+
label: { type: 'string', required: true },
|
|
567
|
+
x: { type: 'number', required: true },
|
|
568
|
+
y: { type: 'number', required: true },
|
|
569
|
+
loc: { type: 'string', required: true },
|
|
570
|
+
},
|
|
571
|
+
},
|
|
572
|
+
},
|
|
573
|
+
challenge: {
|
|
574
|
+
type: 'object',
|
|
575
|
+
additionalProperties: false,
|
|
576
|
+
properties: {
|
|
577
|
+
blocked: { type: 'boolean', required: true },
|
|
578
|
+
kind: { type: 'string' },
|
|
579
|
+
reason: { type: 'string' },
|
|
580
|
+
},
|
|
581
|
+
},
|
|
582
|
+
userControlling: { type: 'boolean' },
|
|
583
|
+
},
|
|
584
|
+
},
|
|
585
|
+
render: (_args, value) => [{ type: 'text', text: formatSnapshot(value) }],
|
|
586
|
+
},
|
|
587
|
+
timeoutMs,
|
|
588
|
+
isConcurrencySafe: () => true,
|
|
589
|
+
async execute(args, exec) {
|
|
590
|
+
const browser = ctx.get('browser');
|
|
591
|
+
if (browser === undefined)
|
|
592
|
+
throw new Error('tool-browser: browser service unavailable');
|
|
593
|
+
const key = taskKey(exec);
|
|
594
|
+
const snapshot = await withTaskRead(browser, key, 'snapshot', session => browser.snapshot(session, {
|
|
595
|
+
...args.query !== undefined ? { query: args.query } : {},
|
|
596
|
+
...args.limit !== undefined ? { limit: args.limit } : {},
|
|
597
|
+
}, exec.signal));
|
|
598
|
+
return {
|
|
599
|
+
snapshotId: snapshot.snapshotId,
|
|
600
|
+
url: snapshot.url,
|
|
601
|
+
...snapshot.title !== undefined ? { title: snapshot.title } : {},
|
|
602
|
+
elements: snapshot.elements.map(el => ({ ref: el.ref, kind: el.kind, label: el.label, x: el.x, y: el.y, loc: el.loc })),
|
|
603
|
+
truncated: snapshot.truncated,
|
|
604
|
+
...snapshot.challenge !== undefined ? { challenge: snapshot.challenge } : {},
|
|
605
|
+
...snapshot.userControlling !== undefined ? { userControlling: snapshot.userControlling } : {},
|
|
606
|
+
};
|
|
607
|
+
},
|
|
608
|
+
}));
|
|
609
|
+
ctx.tools.register(defineTool({
|
|
610
|
+
name: 'browser_click_ref',
|
|
611
|
+
description: 'Click an element from a specific browser_snapshot by its snapshotId and ref. Re-snapshot if the page has changed.',
|
|
612
|
+
parameters: {
|
|
613
|
+
snapshotId: { type: 'string', required: true, description: 'Opaque snapshot id returned by browser_open or browser_snapshot.' },
|
|
614
|
+
ref: { type: 'number', required: true, description: 'Element reference number from that snapshot.' },
|
|
615
|
+
},
|
|
616
|
+
output: {
|
|
617
|
+
schema: { type: 'object', additionalProperties: false, properties: { clicked: { type: 'boolean', required: true } } },
|
|
618
|
+
render: () => [{ type: 'text', text: 'Clicked the referenced element.' }],
|
|
619
|
+
},
|
|
620
|
+
timeoutMs,
|
|
621
|
+
isConcurrencySafe: () => false,
|
|
622
|
+
async execute(args, exec) {
|
|
623
|
+
assertAllowed('browser_click_ref', exec);
|
|
624
|
+
const browser = ctx.get('browser');
|
|
625
|
+
if (browser === undefined)
|
|
626
|
+
throw new Error('tool-browser: browser service unavailable');
|
|
627
|
+
await withTaskAction(browser, taskKey(exec), 'click referenced element', exec, session => browser.clickRef(session, { snapshotId: args.snapshotId, ref: args.ref }, exec.signal));
|
|
628
|
+
return { clicked: true };
|
|
629
|
+
},
|
|
630
|
+
}));
|
|
631
|
+
ctx.tools.register(defineTool({
|
|
632
|
+
name: 'browser_scroll_into_view',
|
|
633
|
+
description: 'Scroll an element from a specific browser_snapshot into view. Re-snapshot if the page has changed.',
|
|
634
|
+
parameters: {
|
|
635
|
+
snapshotId: { type: 'string', required: true, description: 'Opaque snapshot id returned by browser_open or browser_snapshot.' },
|
|
636
|
+
ref: { type: 'number', required: true, description: 'Element reference number from that snapshot.' },
|
|
637
|
+
block: { type: 'string', enum: ['start', 'center', 'end', 'nearest'], description: 'Vertical alignment after scrolling. Default center.' },
|
|
638
|
+
},
|
|
639
|
+
output: {
|
|
640
|
+
schema: {
|
|
641
|
+
type: 'object', additionalProperties: false, properties: {
|
|
642
|
+
scrolled: { type: 'boolean', required: true },
|
|
643
|
+
x: { type: 'number', required: true }, y: { type: 'number', required: true },
|
|
644
|
+
maxX: { type: 'number', required: true }, maxY: { type: 'number', required: true },
|
|
645
|
+
},
|
|
646
|
+
},
|
|
647
|
+
render: (_args, value) => [{ type: 'text', text: `Scrolled to (${value.x}, ${value.y}).` }],
|
|
648
|
+
},
|
|
649
|
+
timeoutMs,
|
|
650
|
+
isConcurrencySafe: () => false,
|
|
651
|
+
async execute(args, exec) {
|
|
652
|
+
assertAllowed('browser_scroll_into_view', exec);
|
|
653
|
+
const browser = ctx.get('browser');
|
|
654
|
+
if (browser === undefined)
|
|
655
|
+
throw new Error('tool-browser: browser service unavailable');
|
|
656
|
+
const result = await withTaskAction(browser, taskKey(exec), 'scroll referenced element into view', exec, session => browser.scrollIntoView(session, {
|
|
657
|
+
snapshotId: args.snapshotId,
|
|
658
|
+
ref: args.ref,
|
|
659
|
+
...args.block !== undefined ? { block: args.block } : {},
|
|
660
|
+
}, exec.signal));
|
|
661
|
+
return { scrolled: true, x: result.x, y: result.y, maxX: result.maxX, maxY: result.maxY };
|
|
662
|
+
},
|
|
663
|
+
}));
|
|
664
|
+
ctx.tools.register(defineTool({
|
|
665
|
+
name: 'browser_challenge',
|
|
666
|
+
description: 'Check whether a human-verification challenge (CAPTCHA / bot detection: Cloudflare "Just a moment", reCAPTCHA, hCaptcha, Turnstile) is blocking the current page. When blocked, do NOT keep retrying automated steps — ask the human to complete the verification in the shared browser window, then re-check with browser_snapshot.',
|
|
667
|
+
parameters: {},
|
|
668
|
+
output: {
|
|
669
|
+
schema: {
|
|
670
|
+
type: 'object',
|
|
671
|
+
additionalProperties: false,
|
|
672
|
+
properties: {
|
|
673
|
+
blocked: { type: 'boolean', required: true },
|
|
674
|
+
kind: { type: 'string' },
|
|
675
|
+
reason: { type: 'string' },
|
|
676
|
+
hint: { type: 'string' },
|
|
677
|
+
},
|
|
678
|
+
},
|
|
679
|
+
render: (_args, value) => [{
|
|
680
|
+
type: 'text',
|
|
681
|
+
text: value.blocked
|
|
682
|
+
? `Challenge detected: ${value.reason ?? value.kind ?? 'human-verification'}. ${value.hint ?? ''}`
|
|
683
|
+
: 'No human-verification challenge detected.',
|
|
684
|
+
}],
|
|
685
|
+
},
|
|
686
|
+
timeoutMs,
|
|
687
|
+
isConcurrencySafe: () => true,
|
|
688
|
+
async execute(_args, exec) {
|
|
689
|
+
const browser = ctx.get('browser');
|
|
690
|
+
if (browser === undefined)
|
|
691
|
+
throw new Error('tool-browser: browser service unavailable');
|
|
692
|
+
const key = taskKey(exec);
|
|
693
|
+
const challenge = await withTaskRead(browser, key, 'challenge', session => browser.detectChallenge(session, exec.signal));
|
|
694
|
+
return {
|
|
695
|
+
blocked: challenge.blocked,
|
|
696
|
+
...challenge.kind !== undefined ? { kind: challenge.kind } : {},
|
|
697
|
+
...challenge.reason !== undefined ? { reason: challenge.reason } : {},
|
|
698
|
+
hint: challenge.blocked
|
|
699
|
+
? 'Ask the human to complete the verification in the shared browser window (the page is visible to them), then re-check with browser_snapshot.'
|
|
700
|
+
: '',
|
|
701
|
+
};
|
|
702
|
+
},
|
|
703
|
+
}));
|
|
704
|
+
ctx.tools.register(defineTool({
|
|
705
|
+
name: 'browser_dialog',
|
|
706
|
+
description: 'Inspect or steer the next JavaScript dialog (alert / confirm / prompt). A dialog freezes the page until it is answered, so the host accepts it immediately by default and keeps a record of what the page asked; `action: "inspect"` reports that record. To drive a page that confirms a destructive action, set the answer FIRST with `action: "dismiss"` (or "accept", plus `promptText` for a prompt()) and then trigger it - the policy applies to the next dialog on this tab. Use browser_handoff when a dialog needs a human decision.',
|
|
707
|
+
parameters: {
|
|
708
|
+
action: { type: 'string', enum: ['inspect', 'accept', 'dismiss'], required: true, description: 'inspect reports the last dialog and the current policy; accept/dismiss set how the NEXT dialog is answered.' },
|
|
709
|
+
promptText: { type: 'string', description: 'With action=accept: the text to type into a prompt(). Ignored by alert/confirm.' },
|
|
710
|
+
},
|
|
711
|
+
output: {
|
|
712
|
+
schema: {
|
|
713
|
+
type: 'object',
|
|
714
|
+
additionalProperties: false,
|
|
715
|
+
properties: {
|
|
716
|
+
dialog: {
|
|
717
|
+
type: 'object',
|
|
718
|
+
additionalProperties: false,
|
|
719
|
+
properties: {
|
|
720
|
+
type: { type: 'string', required: true },
|
|
721
|
+
message: { type: 'string', required: true },
|
|
722
|
+
prompt: { type: 'string' },
|
|
723
|
+
answered: { type: 'string' },
|
|
724
|
+
promptText: { type: 'string' },
|
|
725
|
+
},
|
|
726
|
+
},
|
|
727
|
+
policy: {
|
|
728
|
+
type: 'object',
|
|
729
|
+
additionalProperties: false,
|
|
730
|
+
properties: {
|
|
731
|
+
behavior: { type: 'string', required: true },
|
|
732
|
+
promptText: { type: 'string' },
|
|
733
|
+
},
|
|
734
|
+
},
|
|
735
|
+
},
|
|
736
|
+
},
|
|
737
|
+
render: (_args, value) => [{ type: 'text', text: value.dialog === undefined
|
|
738
|
+
? 'No dialog has been raised on this tab; the next one will be ' + String(value.policy?.behavior ?? 'accept') + 'ed.'
|
|
739
|
+
: 'Last dialog (' + String((value.dialog).answered ?? 'answered') + 'ed): ' + String((value.dialog).type) + ' - ' + String((value.dialog).message).slice(0, 120) }],
|
|
740
|
+
},
|
|
741
|
+
timeoutMs,
|
|
742
|
+
isConcurrencySafe: () => false,
|
|
743
|
+
async execute(args, exec) {
|
|
744
|
+
assertAllowed('browser_dialog', exec);
|
|
745
|
+
const browser = ctx.get('browser');
|
|
746
|
+
if (browser === undefined)
|
|
747
|
+
throw new Error('tool-browser: browser service unavailable');
|
|
748
|
+
const key = taskKey(exec);
|
|
749
|
+
const state = args.action === 'inspect'
|
|
750
|
+
? await withTaskRead(browser, key, 'dialog', async (session) => browser.inspectDialog(session))
|
|
751
|
+
: await withTaskAction(browser, key, 'dialog ' + args.action, exec, session => browser.setDialogPolicy(session, {
|
|
752
|
+
behavior: args.action === 'dismiss' ? 'dismiss' : 'accept',
|
|
753
|
+
...args.promptText === undefined ? {} : { promptText: args.promptText },
|
|
754
|
+
}));
|
|
755
|
+
const dialog = state.dialog;
|
|
756
|
+
return {
|
|
757
|
+
...dialog === null || dialog === undefined ? {} : { dialog },
|
|
758
|
+
policy: { ...state.policy },
|
|
759
|
+
};
|
|
760
|
+
},
|
|
761
|
+
}));
|
|
762
|
+
ctx.tools.register(defineTool({
|
|
763
|
+
name: 'browser_console',
|
|
764
|
+
description: 'Read the console messages and uncaught exceptions the browser captured for the active tab (a bounded ring, newest last). Use it to find out WHY a page misbehaved: a failed script, a rejected promise, a 404 the page logged. Reading does not clear - pass clear: true when you want a fresh window (e.g. before triggering the action you are debugging).',
|
|
765
|
+
parameters: {
|
|
766
|
+
level: { type: 'string', description: 'Only this level: log, info, warning, error, debug.' },
|
|
767
|
+
limit: { type: 'number', description: 'Maximum messages to return (1-200; default 50, newest last).' },
|
|
768
|
+
clear: { type: 'boolean', description: 'Drop what has been captured so far (default false).' },
|
|
769
|
+
},
|
|
770
|
+
output: {
|
|
771
|
+
schema: {
|
|
772
|
+
type: 'object',
|
|
773
|
+
additionalProperties: false,
|
|
774
|
+
properties: {
|
|
775
|
+
messages: {
|
|
776
|
+
type: 'array',
|
|
777
|
+
required: true,
|
|
778
|
+
items: {
|
|
779
|
+
type: 'object',
|
|
780
|
+
additionalProperties: false,
|
|
781
|
+
properties: {
|
|
782
|
+
level: { type: 'string', required: true },
|
|
783
|
+
text: { type: 'string', required: true },
|
|
784
|
+
at: { type: 'string', required: true },
|
|
785
|
+
},
|
|
786
|
+
},
|
|
787
|
+
},
|
|
788
|
+
},
|
|
789
|
+
},
|
|
790
|
+
render: (_args, value) => [{ type: 'text', text: value.messages.length === 0
|
|
791
|
+
? 'No console messages captured for this tab.'
|
|
792
|
+
: value.messages.length + ' message(s), newest last:\n' + value.messages.map(m => '[' + m.level + '] ' + String(m.text).slice(0, 300)).join('\n') }],
|
|
793
|
+
},
|
|
794
|
+
timeoutMs,
|
|
795
|
+
isConcurrencySafe: () => true,
|
|
796
|
+
async execute(args, exec) {
|
|
797
|
+
assertAllowed('browser_console', exec);
|
|
798
|
+
const browser = ctx.get('browser');
|
|
799
|
+
if (browser === undefined)
|
|
800
|
+
throw new Error('tool-browser: browser service unavailable');
|
|
801
|
+
const key = taskKey(exec);
|
|
802
|
+
return withTaskRead(browser, key, 'console', async (session) => browser.consoleMessages(session, {
|
|
803
|
+
...args.level === undefined ? {} : { level: args.level },
|
|
804
|
+
...args.limit === undefined ? {} : { limit: args.limit },
|
|
805
|
+
...args.clear === true ? { clear: true } : {},
|
|
806
|
+
}));
|
|
807
|
+
},
|
|
808
|
+
}));
|
|
809
|
+
ctx.tools.register(defineTool({
|
|
810
|
+
name: 'browser_network',
|
|
811
|
+
description: 'Read the network requests the browser captured for the active tab (a bounded ring, newest last): method, url, status, mime type, duration, and the failure text when one did not complete. Use it to check whether an API call actually happened and what it returned. Reading does not clear - pass clear: true for a fresh window.',
|
|
812
|
+
parameters: {
|
|
813
|
+
urlContains: { type: 'string', description: 'Only requests whose url contains this text (case-insensitive).' },
|
|
814
|
+
failedOnly: { type: 'boolean', description: 'Only requests that failed to complete.' },
|
|
815
|
+
limit: { type: 'number', description: 'Maximum requests to return (1-200; default 50, newest last).' },
|
|
816
|
+
clear: { type: 'boolean', description: 'Drop what has been captured so far (default false).' },
|
|
817
|
+
},
|
|
818
|
+
output: {
|
|
819
|
+
schema: {
|
|
820
|
+
type: 'object',
|
|
821
|
+
additionalProperties: false,
|
|
822
|
+
properties: {
|
|
823
|
+
requests: {
|
|
824
|
+
type: 'array',
|
|
825
|
+
required: true,
|
|
826
|
+
items: {
|
|
827
|
+
type: 'object',
|
|
828
|
+
additionalProperties: false,
|
|
829
|
+
properties: {
|
|
830
|
+
method: { type: 'string', required: true },
|
|
831
|
+
url: { type: 'string', required: true },
|
|
832
|
+
status: { type: 'number' },
|
|
833
|
+
mime: { type: 'string' },
|
|
834
|
+
kind: { type: 'string' },
|
|
835
|
+
failed: { type: 'string' },
|
|
836
|
+
ms: { type: 'number' },
|
|
837
|
+
at: { type: 'string', required: true },
|
|
838
|
+
},
|
|
839
|
+
},
|
|
840
|
+
},
|
|
841
|
+
},
|
|
842
|
+
},
|
|
843
|
+
render: (_args, value) => [{ type: 'text', text: value.requests.length === 0
|
|
844
|
+
? 'No network requests captured for this tab.'
|
|
845
|
+
: value.requests.length + ' request(s), newest last:\n' + value.requests.map(r => [r.method, r.status ?? (r.failed !== undefined ? 'FAILED' : '?'), String(r.url).slice(0, 160)].join(' ')).join('\n') }],
|
|
846
|
+
},
|
|
847
|
+
timeoutMs,
|
|
848
|
+
isConcurrencySafe: () => true,
|
|
849
|
+
async execute(args, exec) {
|
|
850
|
+
assertAllowed('browser_network', exec);
|
|
851
|
+
const browser = ctx.get('browser');
|
|
852
|
+
if (browser === undefined)
|
|
853
|
+
throw new Error('tool-browser: browser service unavailable');
|
|
854
|
+
const key = taskKey(exec);
|
|
855
|
+
return withTaskRead(browser, key, 'network', async (session) => browser.networkRequests(session, {
|
|
856
|
+
...args.urlContains === undefined ? {} : { urlContains: args.urlContains },
|
|
857
|
+
...args.failedOnly === true ? { failedOnly: true } : {},
|
|
858
|
+
...args.limit === undefined ? {} : { limit: args.limit },
|
|
859
|
+
...args.clear === true ? { clear: true } : {},
|
|
860
|
+
}));
|
|
861
|
+
},
|
|
862
|
+
}));
|
|
863
|
+
ctx.tools.register(defineTool({
|
|
864
|
+
name: 'browser_emulate',
|
|
865
|
+
description: 'Emulate a device on the active tab: viewport size (with optional mobile mode and device pixel ratio), a custom user agent, or prefers-color-scheme. Use it to check a responsive layout, to take a screenshot at a fixed size, or to see the dark theme. Pass clear: true to undo all three and go back to the real window. Note the emulated viewport is a rendering override - the window itself does not resize.',
|
|
866
|
+
parameters: {
|
|
867
|
+
width: { type: 'number', description: 'Viewport width in CSS px (give with height).' },
|
|
868
|
+
height: { type: 'number', description: 'Viewport height in CSS px (give with width).' },
|
|
869
|
+
deviceScaleFactor: { type: 'number', description: 'Device pixel ratio, e.g. 2 for a retina phone. Default keeps the real one.' },
|
|
870
|
+
mobile: { type: 'boolean', description: 'Emulate a mobile device (touch + mobile viewport behaviour).' },
|
|
871
|
+
userAgent: { type: 'string', description: 'User agent string to send instead of the real one.' },
|
|
872
|
+
colorScheme: { type: 'string', enum: ['light', 'dark', 'no-preference'], description: 'Value for prefers-color-scheme.' },
|
|
873
|
+
clear: { type: 'boolean', description: 'Undo viewport, user agent and color-scheme emulation on this tab.' },
|
|
874
|
+
},
|
|
875
|
+
output: {
|
|
876
|
+
schema: {
|
|
877
|
+
type: 'object',
|
|
878
|
+
additionalProperties: false,
|
|
879
|
+
properties: {
|
|
880
|
+
applied: { type: 'array', required: true, items: { type: 'string' } },
|
|
881
|
+
},
|
|
882
|
+
},
|
|
883
|
+
render: (_args, value) => [{ type: 'text', text: value.applied.length === 0
|
|
884
|
+
? 'Nothing to emulate: give width+height, userAgent, colorScheme, or clear.'
|
|
885
|
+
: 'Applied: ' + value.applied.join(', ') }],
|
|
886
|
+
},
|
|
887
|
+
timeoutMs,
|
|
888
|
+
isConcurrencySafe: () => false,
|
|
889
|
+
async execute(args, exec) {
|
|
890
|
+
assertAllowed('browser_emulate', exec);
|
|
891
|
+
const browser = ctx.get('browser');
|
|
892
|
+
if (browser === undefined)
|
|
893
|
+
throw new Error('tool-browser: browser service unavailable');
|
|
894
|
+
const key = taskKey(exec);
|
|
895
|
+
return withTaskAction(browser, key, 'emulate', exec, session => browser.emulate(session, {
|
|
896
|
+
...args.width === undefined ? {} : { width: args.width },
|
|
897
|
+
...args.height === undefined ? {} : { height: args.height },
|
|
898
|
+
...args.deviceScaleFactor === undefined ? {} : { deviceScaleFactor: args.deviceScaleFactor },
|
|
899
|
+
...args.mobile === true ? { mobile: true } : {},
|
|
900
|
+
...args.userAgent === undefined ? {} : { userAgent: args.userAgent },
|
|
901
|
+
...args.colorScheme === undefined ? {} : { colorScheme: args.colorScheme },
|
|
902
|
+
...args.clear === true ? { clear: true } : {},
|
|
903
|
+
}));
|
|
904
|
+
},
|
|
905
|
+
}));
|
|
906
|
+
ctx.tools.register(defineTool({
|
|
907
|
+
name: 'browser_execute',
|
|
908
|
+
description: 'Execute JavaScript in the shared-browser page context. This is the primary way to interact with page elements: focus, fill inputs (use the native value setter for framework-controlled inputs, then dispatch an input event), click buttons (element.click() or a constructed MouseEvent). The script may be a single expression (its value is returned) or statements with an explicit `return` - `const el = document.querySelector("#x"); return el.textContent` works. Promises are awaited. Returns the evaluation result by value, or the exception text.',
|
|
909
|
+
parameters: {
|
|
910
|
+
script: { type: 'string', required: true, description: 'The JavaScript to evaluate: an expression, or statements with a `return`.' },
|
|
911
|
+
args: { type: 'array', items: { type: 'string' }, description: 'Optional arguments injected into the script scope as arguments[0..n].' },
|
|
912
|
+
},
|
|
913
|
+
output: {
|
|
914
|
+
schema: {
|
|
915
|
+
type: 'object',
|
|
916
|
+
additionalProperties: false,
|
|
917
|
+
properties: {
|
|
918
|
+
ok: { type: 'boolean', required: true },
|
|
919
|
+
value: { type: 'string' },
|
|
920
|
+
exception: { type: 'string' },
|
|
921
|
+
},
|
|
922
|
+
},
|
|
923
|
+
render: (_args, value) => [{ type: 'text', text: value.ok ? `Result: ${String(value.value)}` : `Exception: ${value.exception}` }],
|
|
924
|
+
},
|
|
925
|
+
timeoutMs,
|
|
926
|
+
isConcurrencySafe: () => false, // page JS can be stateful
|
|
927
|
+
async execute(args, exec) {
|
|
928
|
+
assertAllowed('browser_execute', exec);
|
|
929
|
+
const browser = ctx.get('browser');
|
|
930
|
+
if (browser === undefined)
|
|
931
|
+
throw new Error('tool-browser: browser service unavailable');
|
|
932
|
+
const result = await withTaskAction(browser, taskKey(exec), 'execute page script', exec, session => browser.execute(session, {
|
|
933
|
+
script: args.script,
|
|
934
|
+
args: args.args ?? [],
|
|
935
|
+
}, exec.signal));
|
|
936
|
+
if (result.ok) {
|
|
937
|
+
const raw = result.value;
|
|
938
|
+
const value = typeof raw === 'string' ? raw : JSON.stringify(raw ?? null);
|
|
939
|
+
return { ok: true, value };
|
|
940
|
+
}
|
|
941
|
+
return { ok: false, exception: result.exception };
|
|
942
|
+
},
|
|
943
|
+
}));
|
|
944
|
+
ctx.tools.register(defineTool({
|
|
945
|
+
name: 'browser_content',
|
|
946
|
+
description: 'Fetch the current shared-browser page content in a chosen format: html (raw DOM), markdown (structured reading), txt (plain text), or json. Optionally scope to a CSS selector and cap the length. Use this to read page content, not to interact.',
|
|
947
|
+
parameters: {
|
|
948
|
+
format: { type: 'string', required: true, enum: ['html', 'markdown', 'txt', 'json'], description: 'Output format.' },
|
|
949
|
+
selector: { type: 'string', description: 'CSS selector limiting the fetch to one region (e.g. #main).' },
|
|
950
|
+
maxChars: { type: 'number', description: 'Maximum characters of returned content.' },
|
|
951
|
+
timeoutMs: { type: 'number', description: `Evaluation timeout in ms (default 30000), capped below this tool's ${String(timeoutMs)}ms budget.` },
|
|
952
|
+
},
|
|
953
|
+
output: {
|
|
954
|
+
schema: {
|
|
955
|
+
type: 'object',
|
|
956
|
+
additionalProperties: false,
|
|
957
|
+
properties: {
|
|
958
|
+
content: { type: 'string', required: true },
|
|
959
|
+
truncated: { type: 'boolean', required: true },
|
|
960
|
+
},
|
|
961
|
+
},
|
|
962
|
+
render: (_args, value) => [{ type: 'text', text: value.content + (value.truncated ? '\n(truncated)' : '') }],
|
|
963
|
+
},
|
|
964
|
+
timeoutMs,
|
|
965
|
+
isConcurrencySafe: () => true,
|
|
966
|
+
async execute(args, exec) {
|
|
967
|
+
const browser = ctx.get('browser');
|
|
968
|
+
if (browser === undefined)
|
|
969
|
+
throw new Error('tool-browser: browser service unavailable');
|
|
970
|
+
const key = taskKey(exec);
|
|
971
|
+
const budgetMs = withinToolBudget(args.timeoutMs, 30_000);
|
|
972
|
+
const readKey = 'content:' + JSON.stringify({ format: args.format, selector: args.selector, maxChars: args.maxChars, timeoutMs: budgetMs });
|
|
973
|
+
const result = await withTaskRead(browser, key, readKey, session => browser.content(session, {
|
|
974
|
+
format: args.format,
|
|
975
|
+
...args.selector !== undefined ? { selector: args.selector } : {},
|
|
976
|
+
...args.maxChars !== undefined ? { maxChars: args.maxChars } : {},
|
|
977
|
+
timeoutMs: budgetMs,
|
|
978
|
+
}, exec.signal));
|
|
979
|
+
return { content: result.content, truncated: result.truncated };
|
|
980
|
+
},
|
|
981
|
+
}));
|
|
982
|
+
ctx.tools.register(defineTool({
|
|
983
|
+
name: 'browser_click',
|
|
984
|
+
description: 'Click an element in the shared browser. Address it three ways: x and y (use with browser_screenshot when a vision model located it — this covers icons, image buttons and canvas that DOM snapshots cannot target), a CSS selector, or visible text. Selector and text are resolved in the page and scrolled into view first, so you can click "the sign-in button" without spending a browser_snapshot round-trip on its ref; the reply names the element it actually hit. Coordinates are relative to the visible viewport, same as a screenshot.',
|
|
985
|
+
parameters: {
|
|
986
|
+
x: { type: 'number', description: 'Viewport x coordinate (CSS px), inside the visible viewport. Give with y, or give selector/text instead - a coordinate target is NOT scrolled into view (selector/text are), so an off-screen point is refused.' },
|
|
987
|
+
y: { type: 'number', description: 'Viewport y coordinate (CSS px), inside the visible viewport.' },
|
|
988
|
+
button: { type: 'string', enum: ['left', 'right', 'middle'], description: 'Mouse button, default left. A right-click reaches the page own context-menu handler: Electron installs no native menu, so whatever the page shows is what you interact with next.' },
|
|
989
|
+
modifiers: { type: 'array', items: { type: 'string', enum: ['alt', 'ctrl', 'meta', 'shift'] }, description: 'Modifiers held during the action. ctrl/meta-click opens a link in a new tab (check browser_list_tabs afterwards); shift-click extends a selection.' },
|
|
990
|
+
selector: { type: 'string', description: 'CSS selector to click instead of coordinates; the first visible match is used and scrolled into view.' },
|
|
991
|
+
text: { type: 'string', description: 'Visible text (or aria-label/value, case-insensitive) to click instead of coordinates; the innermost visible match wins. Saves a browser_snapshot round-trip when you know the label.' },
|
|
992
|
+
},
|
|
993
|
+
output: {
|
|
994
|
+
schema: {
|
|
995
|
+
type: 'object',
|
|
996
|
+
additionalProperties: false,
|
|
997
|
+
properties: {
|
|
998
|
+
clicked: { type: 'boolean', required: true },
|
|
999
|
+
x: { type: 'number' },
|
|
1000
|
+
y: { type: 'number' },
|
|
1001
|
+
target: { type: 'string' },
|
|
1002
|
+
},
|
|
1003
|
+
},
|
|
1004
|
+
render: (_args, value) => [{ type: 'text', text: value.clicked === true
|
|
1005
|
+
? (value.target !== undefined ? `Clicked "${value.target}" at ${value.x},${value.y}.` : `Clicked at ${value.x},${value.y}.`)
|
|
1006
|
+
: 'Clicked failed.' }],
|
|
1007
|
+
},
|
|
1008
|
+
timeoutMs,
|
|
1009
|
+
isConcurrencySafe: () => false,
|
|
1010
|
+
async execute(args, exec) {
|
|
1011
|
+
assertAllowed('browser_click', exec);
|
|
1012
|
+
const browser = ctx.get('browser');
|
|
1013
|
+
if (browser === undefined)
|
|
1014
|
+
throw new Error('tool-browser: browser service unavailable');
|
|
1015
|
+
const target = {
|
|
1016
|
+
...args.x !== undefined ? { x: args.x } : {},
|
|
1017
|
+
...args.y !== undefined ? { y: args.y } : {},
|
|
1018
|
+
...args.selector !== undefined ? { selector: args.selector } : {},
|
|
1019
|
+
...args.text !== undefined ? { text: args.text } : {},
|
|
1020
|
+
...args.button !== undefined ? { button: args.button } : {},
|
|
1021
|
+
...args.modifiers !== undefined ? { modifiers: args.modifiers } : {},
|
|
1022
|
+
};
|
|
1023
|
+
const point = await withTaskAction(browser, taskKey(exec), 'click page', exec, session => browser.click(session, target, exec.signal));
|
|
1024
|
+
return { clicked: true, ...point };
|
|
1025
|
+
},
|
|
1026
|
+
}));
|
|
1027
|
+
ctx.tools.register(defineTool({
|
|
1028
|
+
name: 'browser_double_click',
|
|
1029
|
+
description: 'Double-click an element in the shared browser. Address it with x and y, a CSS selector, or visible text (resolved and scrolled into view first). Use for opening links, selecting text, or expanding UI that ignores single clicks.',
|
|
1030
|
+
parameters: {
|
|
1031
|
+
x: { type: 'number', description: 'Viewport x coordinate (CSS px), inside the visible viewport. Give with y, or give selector/text instead - a coordinate target is NOT scrolled into view (selector/text are), so an off-screen point is refused.' },
|
|
1032
|
+
y: { type: 'number', description: 'Viewport y coordinate (CSS px), inside the visible viewport.' },
|
|
1033
|
+
button: { type: 'string', enum: ['left', 'right', 'middle'], description: 'Mouse button, default left. A right-click reaches the page own context-menu handler: Electron installs no native menu, so whatever the page shows is what you interact with next.' },
|
|
1034
|
+
modifiers: { type: 'array', items: { type: 'string', enum: ['alt', 'ctrl', 'meta', 'shift'] }, description: 'Modifiers held during the action. ctrl/meta-click opens a link in a new tab (check browser_list_tabs afterwards); shift-click extends a selection.' },
|
|
1035
|
+
selector: { type: 'string', description: 'CSS selector to double-click instead of coordinates; the first visible match is used and scrolled into view.' },
|
|
1036
|
+
text: { type: 'string', description: 'Visible text (or aria-label/value, case-insensitive) to double-click instead of coordinates; the innermost visible match wins. Saves a browser_snapshot round-trip when you know the label.' },
|
|
1037
|
+
},
|
|
1038
|
+
output: {
|
|
1039
|
+
schema: {
|
|
1040
|
+
type: 'object',
|
|
1041
|
+
additionalProperties: false,
|
|
1042
|
+
properties: {
|
|
1043
|
+
clicked: { type: 'boolean', required: true },
|
|
1044
|
+
x: { type: 'number' },
|
|
1045
|
+
y: { type: 'number' },
|
|
1046
|
+
target: { type: 'string' },
|
|
1047
|
+
},
|
|
1048
|
+
},
|
|
1049
|
+
render: (_args, value) => [{ type: 'text', text: value.clicked === true
|
|
1050
|
+
? (value.target !== undefined ? `Double-clicked "${value.target}" at ${value.x},${value.y}.` : `Double-clicked at ${value.x},${value.y}.`)
|
|
1051
|
+
: 'Double-clicked failed.' }],
|
|
1052
|
+
},
|
|
1053
|
+
timeoutMs,
|
|
1054
|
+
isConcurrencySafe: () => false,
|
|
1055
|
+
async execute(args, exec) {
|
|
1056
|
+
assertAllowed('browser_double_click', exec);
|
|
1057
|
+
const browser = ctx.get('browser');
|
|
1058
|
+
if (browser === undefined)
|
|
1059
|
+
throw new Error('tool-browser: browser service unavailable');
|
|
1060
|
+
const target = {
|
|
1061
|
+
...args.x !== undefined ? { x: args.x } : {},
|
|
1062
|
+
...args.y !== undefined ? { y: args.y } : {},
|
|
1063
|
+
...args.selector !== undefined ? { selector: args.selector } : {},
|
|
1064
|
+
...args.text !== undefined ? { text: args.text } : {},
|
|
1065
|
+
...args.button !== undefined ? { button: args.button } : {},
|
|
1066
|
+
...args.modifiers !== undefined ? { modifiers: args.modifiers } : {},
|
|
1067
|
+
};
|
|
1068
|
+
const point = await withTaskAction(browser, taskKey(exec), 'double-click page', exec, session => browser.doubleClick(session, target, exec.signal));
|
|
1069
|
+
return { clicked: true, ...point };
|
|
1070
|
+
},
|
|
1071
|
+
}));
|
|
1072
|
+
ctx.tools.register(defineTool({
|
|
1073
|
+
name: 'browser_hover',
|
|
1074
|
+
description: 'Move the pointer over an element without clicking. Address it with x and y, a CSS selector, or visible text (resolved and scrolled into view first). Triggers hover states, tooltips, and dropdown menus.',
|
|
1075
|
+
parameters: {
|
|
1076
|
+
x: { type: 'number', description: 'Viewport x coordinate (CSS px), inside the visible viewport. Give with y, or give selector/text instead - a coordinate target is NOT scrolled into view (selector/text are), so an off-screen point is refused.' },
|
|
1077
|
+
y: { type: 'number', description: 'Viewport y coordinate (CSS px), inside the visible viewport.' },
|
|
1078
|
+
button: { type: 'string', enum: ['left', 'right', 'middle'], description: 'Mouse button, default left. A right-click reaches the page own context-menu handler: Electron installs no native menu, so whatever the page shows is what you interact with next.' },
|
|
1079
|
+
modifiers: { type: 'array', items: { type: 'string', enum: ['alt', 'ctrl', 'meta', 'shift'] }, description: 'Modifiers held during the action. ctrl/meta-click opens a link in a new tab (check browser_list_tabs afterwards); shift-click extends a selection.' },
|
|
1080
|
+
selector: { type: 'string', description: 'CSS selector to hover over instead of coordinates; the first visible match is used and scrolled into view.' },
|
|
1081
|
+
text: { type: 'string', description: 'Visible text (or aria-label/value, case-insensitive) to hover over instead of coordinates; the innermost visible match wins. Saves a browser_snapshot round-trip when you know the label.' },
|
|
1082
|
+
},
|
|
1083
|
+
output: {
|
|
1084
|
+
schema: {
|
|
1085
|
+
type: 'object',
|
|
1086
|
+
additionalProperties: false,
|
|
1087
|
+
properties: {
|
|
1088
|
+
hovered: { type: 'boolean', required: true },
|
|
1089
|
+
x: { type: 'number' },
|
|
1090
|
+
y: { type: 'number' },
|
|
1091
|
+
target: { type: 'string' },
|
|
1092
|
+
},
|
|
1093
|
+
},
|
|
1094
|
+
render: (_args, value) => [{ type: 'text', text: value.hovered === true
|
|
1095
|
+
? (value.target !== undefined ? `Hovered "${value.target}" at ${value.x},${value.y}.` : `Hovered at ${value.x},${value.y}.`)
|
|
1096
|
+
: 'Hovered failed.' }],
|
|
1097
|
+
},
|
|
1098
|
+
timeoutMs,
|
|
1099
|
+
isConcurrencySafe: () => false,
|
|
1100
|
+
async execute(args, exec) {
|
|
1101
|
+
assertAllowed('browser_hover', exec);
|
|
1102
|
+
const browser = ctx.get('browser');
|
|
1103
|
+
if (browser === undefined)
|
|
1104
|
+
throw new Error('tool-browser: browser service unavailable');
|
|
1105
|
+
const target = {
|
|
1106
|
+
...args.x !== undefined ? { x: args.x } : {},
|
|
1107
|
+
...args.y !== undefined ? { y: args.y } : {},
|
|
1108
|
+
...args.selector !== undefined ? { selector: args.selector } : {},
|
|
1109
|
+
...args.text !== undefined ? { text: args.text } : {},
|
|
1110
|
+
...args.button !== undefined ? { button: args.button } : {},
|
|
1111
|
+
...args.modifiers !== undefined ? { modifiers: args.modifiers } : {},
|
|
1112
|
+
};
|
|
1113
|
+
const point = await withTaskAction(browser, taskKey(exec), 'hover page', exec, session => browser.hover(session, target, exec.signal));
|
|
1114
|
+
return { hovered: true, ...point };
|
|
1115
|
+
},
|
|
1116
|
+
}));
|
|
1117
|
+
ctx.tools.register(defineTool({
|
|
1118
|
+
name: 'browser_scroll',
|
|
1119
|
+
description: 'Scroll the active page by CSS-pixel deltas. With no deltas it scrolls downward by about one viewport (80% of the viewport height, at least 480px).',
|
|
1120
|
+
parameters: {
|
|
1121
|
+
deltaX: { type: 'number', description: 'Horizontal CSS-pixel delta. Default 0.' },
|
|
1122
|
+
deltaY: { type: 'number', description: 'Vertical CSS-pixel delta. Default one viewport downward.' },
|
|
1123
|
+
},
|
|
1124
|
+
output: {
|
|
1125
|
+
schema: {
|
|
1126
|
+
type: 'object', additionalProperties: false, properties: {
|
|
1127
|
+
x: { type: 'number', required: true }, y: { type: 'number', required: true },
|
|
1128
|
+
maxX: { type: 'number', required: true }, maxY: { type: 'number', required: true },
|
|
1129
|
+
},
|
|
1130
|
+
},
|
|
1131
|
+
render: (_args, value) => [{ type: 'text', text: `Scrolled to (${value.x}, ${value.y}).` }],
|
|
1132
|
+
},
|
|
1133
|
+
timeoutMs,
|
|
1134
|
+
isConcurrencySafe: () => false,
|
|
1135
|
+
async execute(args, exec) {
|
|
1136
|
+
assertAllowed('browser_scroll', exec);
|
|
1137
|
+
const browser = ctx.get('browser');
|
|
1138
|
+
if (browser === undefined)
|
|
1139
|
+
throw new Error('tool-browser: browser service unavailable');
|
|
1140
|
+
const result = await withTaskAction(browser, taskKey(exec), 'scroll page', exec, session => browser.scroll(session, {
|
|
1141
|
+
...args.deltaX !== undefined ? { deltaX: args.deltaX } : {},
|
|
1142
|
+
...args.deltaY !== undefined ? { deltaY: args.deltaY } : {},
|
|
1143
|
+
}, exec.signal));
|
|
1144
|
+
return { x: result.x, y: result.y, maxX: result.maxX, maxY: result.maxY };
|
|
1145
|
+
},
|
|
1146
|
+
}));
|
|
1147
|
+
ctx.tools.register(defineTool({
|
|
1148
|
+
name: 'browser_upload_file',
|
|
1149
|
+
description: 'Attach a local file to a file input in the shared browser (CDP DOM.setFileInputFiles, so the page sees a real file selection). Use for avatar uploads, attachments, and import dialogs.',
|
|
1150
|
+
parameters: {
|
|
1151
|
+
filePath: { type: 'string', required: true, description: 'Absolute path of the file to attach.' },
|
|
1152
|
+
selector: { type: 'string', description: 'CSS selector of the file input; defaults to the first input[type="file"] on the page.' },
|
|
1153
|
+
},
|
|
1154
|
+
output: {
|
|
1155
|
+
schema: { type: 'object', additionalProperties: false, properties: { path: { type: 'string', required: true } } },
|
|
1156
|
+
render: (_args, value) => [{ type: 'text', text: `Attached ${value.path}.` }],
|
|
1157
|
+
},
|
|
1158
|
+
timeoutMs,
|
|
1159
|
+
isConcurrencySafe: () => false,
|
|
1160
|
+
async execute(args, exec) {
|
|
1161
|
+
assertAllowed('browser_upload_file', exec);
|
|
1162
|
+
const browser = ctx.get('browser');
|
|
1163
|
+
if (browser === undefined)
|
|
1164
|
+
throw new Error('tool-browser: browser service unavailable');
|
|
1165
|
+
const result = await withTaskAction(browser, taskKey(exec), 'attach file', exec, session => browser.uploadFile(session, {
|
|
1166
|
+
filePath: args.filePath,
|
|
1167
|
+
...args.selector !== undefined ? { selector: args.selector } : {},
|
|
1168
|
+
}, exec.signal));
|
|
1169
|
+
return { path: result.path };
|
|
1170
|
+
},
|
|
1171
|
+
}));
|
|
1172
|
+
ctx.tools.register(defineTool({
|
|
1173
|
+
name: 'browser_wait_for',
|
|
1174
|
+
description: 'Wait until an element matching a CSS selector appears (and is visible), polling every 250ms. Use before interacting with dynamically-loaded content (SPA views, toasts, menus).',
|
|
1175
|
+
parameters: {
|
|
1176
|
+
selector: { type: 'string', required: true, description: 'CSS selector to wait for.' },
|
|
1177
|
+
timeoutMs: { type: 'number', description: `Total budget in ms (default 15000), capped below this tool's ${String(timeoutMs)}ms budget.` },
|
|
1178
|
+
visible: { type: 'boolean', description: 'Require visibility (>4x4 px, not display:none). Default true.' },
|
|
1179
|
+
},
|
|
1180
|
+
output: {
|
|
1181
|
+
schema: {
|
|
1182
|
+
type: 'object',
|
|
1183
|
+
additionalProperties: false,
|
|
1184
|
+
properties: {
|
|
1185
|
+
found: { type: 'boolean', required: true },
|
|
1186
|
+
selector: { type: 'string', required: true },
|
|
1187
|
+
tag: { type: 'string', required: true },
|
|
1188
|
+
text: { type: 'string' },
|
|
1189
|
+
},
|
|
1190
|
+
},
|
|
1191
|
+
render: (_args, value) => [{ type: 'text', text: `Found <${value.tag}> ${value.selector}${value.text !== undefined ? ` — "${value.text.slice(0, 80)}"` : ''}.` }],
|
|
1192
|
+
},
|
|
1193
|
+
timeoutMs,
|
|
1194
|
+
isConcurrencySafe: () => true,
|
|
1195
|
+
async execute(args, exec) {
|
|
1196
|
+
assertAllowed('browser_wait_for', exec);
|
|
1197
|
+
const browser = ctx.get('browser');
|
|
1198
|
+
if (browser === undefined)
|
|
1199
|
+
throw new Error('tool-browser: browser service unavailable');
|
|
1200
|
+
const result = await withTaskAction(browser, taskKey(exec), 'wait for element', exec, session => browser.waitForElement(session, {
|
|
1201
|
+
selector: args.selector,
|
|
1202
|
+
timeoutMs: withinToolBudget(args.timeoutMs, 15_000),
|
|
1203
|
+
...args.visible !== undefined ? { visible: args.visible } : {},
|
|
1204
|
+
}, exec.signal));
|
|
1205
|
+
return { found: true, selector: result.selector, tag: result.tag, text: result.text };
|
|
1206
|
+
},
|
|
1207
|
+
}));
|
|
1208
|
+
ctx.tools.register(defineTool({
|
|
1209
|
+
name: 'browser_type',
|
|
1210
|
+
description: 'Type text into the focused element of the shared browser. Use after browser_execute focuses an input (e.g. el.focus()), or after a click lands in a field. Text is inserted at the current focus via CDP Input.insertText.',
|
|
1211
|
+
parameters: {
|
|
1212
|
+
text: { type: 'string', required: true, description: 'The text to insert.' },
|
|
1213
|
+
},
|
|
1214
|
+
output: {
|
|
1215
|
+
schema: { type: 'object', additionalProperties: false, properties: { typed: { type: 'boolean', required: true } } },
|
|
1216
|
+
render: (_args, value) => [{ type: 'text', text: value.typed ? `Typed ${String(_args.text).length} chars.` : 'Type failed.' }],
|
|
1217
|
+
},
|
|
1218
|
+
timeoutMs,
|
|
1219
|
+
isConcurrencySafe: () => false,
|
|
1220
|
+
async execute(args, exec) {
|
|
1221
|
+
assertAllowed('browser_type', exec);
|
|
1222
|
+
const browser = ctx.get('browser');
|
|
1223
|
+
if (browser === undefined)
|
|
1224
|
+
throw new Error('tool-browser: browser service unavailable');
|
|
1225
|
+
await withTaskAction(browser, taskKey(exec), 'type text', exec, session => browser.type(session, { text: args.text }, exec.signal));
|
|
1226
|
+
return { typed: true };
|
|
1227
|
+
},
|
|
1228
|
+
}));
|
|
1229
|
+
ctx.tools.register(defineTool({
|
|
1230
|
+
name: 'browser_press_key',
|
|
1231
|
+
description: 'Press a key into the focused element of the shared browser (keyDown + keyUp, physical input). Supports single characters, Enter/Tab/Escape/Backspace/Delete, arrow keys, Home/End/PageUp/PageDown, F1-F12, and modifier combos (e.g. key="a" modifiers=["ctrl"] for Ctrl+A). Use after focusing an input or for keyboard navigation.',
|
|
1232
|
+
parameters: {
|
|
1233
|
+
key: { type: 'string', required: true, description: 'The key to press: a character, Enter, Tab, Escape, ArrowDown, Home, F5, etc.' },
|
|
1234
|
+
modifiers: { type: 'array', items: { type: 'string', enum: ['alt', 'ctrl', 'meta', 'shift'] }, description: 'Modifier keys held during the press.' },
|
|
1235
|
+
},
|
|
1236
|
+
output: {
|
|
1237
|
+
schema: { type: 'object', additionalProperties: false, properties: { pressed: { type: 'boolean', required: true } } },
|
|
1238
|
+
render: (_args, value) => [{ type: 'text', text: value.pressed ? `Pressed ${String(_args.key)}.` : 'Press failed.' }],
|
|
1239
|
+
},
|
|
1240
|
+
timeoutMs,
|
|
1241
|
+
isConcurrencySafe: () => false,
|
|
1242
|
+
async execute(args, exec) {
|
|
1243
|
+
assertAllowed('browser_press_key', exec);
|
|
1244
|
+
const browser = ctx.get('browser');
|
|
1245
|
+
if (browser === undefined)
|
|
1246
|
+
throw new Error('tool-browser: browser service unavailable');
|
|
1247
|
+
await withTaskAction(browser, taskKey(exec), 'press key', exec, session => browser.pressKey(session, {
|
|
1248
|
+
key: args.key,
|
|
1249
|
+
...args.modifiers !== undefined ? { modifiers: args.modifiers } : {},
|
|
1250
|
+
}, exec.signal));
|
|
1251
|
+
return { pressed: true };
|
|
1252
|
+
},
|
|
1253
|
+
}));
|
|
1254
|
+
ctx.tools.register(defineTool({
|
|
1255
|
+
name: 'browser_fill',
|
|
1256
|
+
description: 'Fill a form in one batch: pass fields with a CSS selector or name/label/placeholder text and the value to set (string, number, or boolean for checkbox/radio; for selects or radio groups pass the option value or visible text). Values are applied with the native setter plus input/change events, so React/Vue controlled inputs update correctly. Optionally submit the containing form. Prefer this over hand-written browser_execute for form filling; per-field failures are reported instead of throwing.',
|
|
1257
|
+
parameters: {
|
|
1258
|
+
fields: {
|
|
1259
|
+
type: 'array',
|
|
1260
|
+
required: true,
|
|
1261
|
+
items: {
|
|
1262
|
+
type: 'object',
|
|
1263
|
+
additionalProperties: true,
|
|
1264
|
+
properties: {
|
|
1265
|
+
selector: { type: 'string', description: 'CSS selector; when present, candidates are scoped to it.' },
|
|
1266
|
+
name: { type: 'string', description: 'Match by the field\'s name attribute.' },
|
|
1267
|
+
label: { type: 'string', description: 'Match by associated <label> text or aria-label.' },
|
|
1268
|
+
placeholder: { type: 'string', description: 'Match by placeholder text.' },
|
|
1269
|
+
kind: { type: 'string', enum: ['text', 'textarea', 'checkbox', 'radio', 'select'], description: 'Field kind; defaults to text.' },
|
|
1270
|
+
value: { type: 'string', description: 'Value to set (string form; booleans/numbers accepted as strings).' },
|
|
1271
|
+
},
|
|
1272
|
+
},
|
|
1273
|
+
},
|
|
1274
|
+
submit: { type: 'boolean', description: 'Submit the containing form after filling (default false).' },
|
|
1275
|
+
},
|
|
1276
|
+
output: {
|
|
1277
|
+
schema: {
|
|
1278
|
+
type: 'object',
|
|
1279
|
+
additionalProperties: false,
|
|
1280
|
+
properties: {
|
|
1281
|
+
fields: {
|
|
1282
|
+
type: 'array',
|
|
1283
|
+
required: true,
|
|
1284
|
+
items: {
|
|
1285
|
+
type: 'object',
|
|
1286
|
+
additionalProperties: false,
|
|
1287
|
+
properties: {
|
|
1288
|
+
ok: { type: 'boolean', required: true },
|
|
1289
|
+
target: { type: 'string', required: true },
|
|
1290
|
+
method: { type: 'string' },
|
|
1291
|
+
error: { type: 'string' },
|
|
1292
|
+
},
|
|
1293
|
+
},
|
|
1294
|
+
},
|
|
1295
|
+
submitted: { type: 'boolean', required: true },
|
|
1296
|
+
},
|
|
1297
|
+
},
|
|
1298
|
+
render: (_args, value) => [{
|
|
1299
|
+
type: 'text',
|
|
1300
|
+
text: (() => {
|
|
1301
|
+
const fields = value.fields;
|
|
1302
|
+
const failed = fields.filter(f => !f.ok);
|
|
1303
|
+
const lines = fields.map(f => `${f.ok ? 'OK' : 'FAIL'} ${f.target}${f.ok ? ` (${f.method ?? 'input'})` : `: ${f.error ?? 'unknown error'}`}`);
|
|
1304
|
+
const head = failed.length === 0
|
|
1305
|
+
? `Filled ${fields.length}/${fields.length} fields${value.submitted ? ' and submitted the form' : ''}.`
|
|
1306
|
+
: `Filled ${fields.length - failed.length}/${fields.length} fields; ${failed.length} failed:`;
|
|
1307
|
+
return head + '\n' + lines.join('\n');
|
|
1308
|
+
})(),
|
|
1309
|
+
}],
|
|
1310
|
+
},
|
|
1311
|
+
timeoutMs,
|
|
1312
|
+
isConcurrencySafe: () => false,
|
|
1313
|
+
async execute(args, exec) {
|
|
1314
|
+
assertAllowed('browser_fill', exec);
|
|
1315
|
+
const browser = ctx.get('browser');
|
|
1316
|
+
if (browser === undefined)
|
|
1317
|
+
throw new Error('tool-browser: browser service unavailable');
|
|
1318
|
+
const fields = (args.fields ?? []).map((f) => ({
|
|
1319
|
+
...f.selector !== undefined ? { selector: f.selector } : {},
|
|
1320
|
+
...f.name !== undefined ? { name: f.name } : {},
|
|
1321
|
+
...f.label !== undefined ? { label: f.label } : {},
|
|
1322
|
+
...f.placeholder !== undefined ? { placeholder: f.placeholder } : {},
|
|
1323
|
+
...f.kind !== undefined ? { kind: f.kind } : {},
|
|
1324
|
+
value: parseFillValue(f.value),
|
|
1325
|
+
}));
|
|
1326
|
+
const result = await withTaskAction(browser, taskKey(exec), 'fill form', exec, session => browser.fillForm(session, {
|
|
1327
|
+
fields,
|
|
1328
|
+
...args.submit === true ? { submit: true } : {},
|
|
1329
|
+
}, exec.signal));
|
|
1330
|
+
return {
|
|
1331
|
+
fields: result.fields.map(f => ({ ok: f.ok, target: f.target, ...f.method !== undefined ? { method: f.method } : {}, ...f.error !== undefined ? { error: f.error } : {} })),
|
|
1332
|
+
submitted: result.submitted,
|
|
1333
|
+
};
|
|
1334
|
+
},
|
|
1335
|
+
}));
|
|
1336
|
+
ctx.tools.register(defineTool({
|
|
1337
|
+
name: 'browser_drag',
|
|
1338
|
+
description: 'Press on one element, move to another, and release: a drag. Both ends are addressed like browser_click (x and y, a CSS selector, or visible text) and scrolled into view first. Use it for sliders, sortable lists and canvas editors. It drives pointer-based drags; a page that relies on HTML5 drag-and-drop (dragstart/drop) will not respond to it, so use that page own controls instead.',
|
|
1339
|
+
parameters: {
|
|
1340
|
+
from: {
|
|
1341
|
+
type: 'object',
|
|
1342
|
+
required: true,
|
|
1343
|
+
additionalProperties: false,
|
|
1344
|
+
description: 'Where the drag starts.',
|
|
1345
|
+
properties: {
|
|
1346
|
+
x: { type: 'number', description: 'Viewport x coordinate (CSS px).' },
|
|
1347
|
+
y: { type: 'number', description: 'Viewport y coordinate (CSS px), inside the visible viewport.' },
|
|
1348
|
+
selector: { type: 'string', description: 'CSS selector; the first visible match is used and scrolled into view.' },
|
|
1349
|
+
text: { type: 'string', description: 'Visible text (or aria-label/value, case-insensitive); the innermost visible match wins.' },
|
|
1350
|
+
},
|
|
1351
|
+
},
|
|
1352
|
+
to: {
|
|
1353
|
+
type: 'object',
|
|
1354
|
+
required: true,
|
|
1355
|
+
additionalProperties: false,
|
|
1356
|
+
description: 'Where it ends.',
|
|
1357
|
+
properties: {
|
|
1358
|
+
x: { type: 'number', description: 'Viewport x coordinate (CSS px).' },
|
|
1359
|
+
y: { type: 'number', description: 'Viewport y coordinate (CSS px), inside the visible viewport.' },
|
|
1360
|
+
selector: { type: 'string', description: 'CSS selector; the first visible match is used and scrolled into view.' },
|
|
1361
|
+
text: { type: 'string', description: 'Visible text (or aria-label/value, case-insensitive); the innermost visible match wins.' },
|
|
1362
|
+
},
|
|
1363
|
+
},
|
|
1364
|
+
steps: { type: 'number', description: 'Intermediate move events (default 12, max 60). A hand does not teleport, and a listener that reads positions per frame needs more than one.' },
|
|
1365
|
+
},
|
|
1366
|
+
output: {
|
|
1367
|
+
schema: {
|
|
1368
|
+
type: 'object',
|
|
1369
|
+
additionalProperties: false,
|
|
1370
|
+
properties: {
|
|
1371
|
+
dragged: { type: 'boolean', required: true },
|
|
1372
|
+
from: { type: 'string' },
|
|
1373
|
+
to: { type: 'string' },
|
|
1374
|
+
},
|
|
1375
|
+
},
|
|
1376
|
+
render: (_args, value) => [{ type: 'text', text: value.dragged
|
|
1377
|
+
? `Dragged ${value.from ?? '(point)'} to ${value.to ?? '(point)'}.`
|
|
1378
|
+
: 'Drag failed.' }],
|
|
1379
|
+
},
|
|
1380
|
+
timeoutMs,
|
|
1381
|
+
isConcurrencySafe: () => false,
|
|
1382
|
+
async execute(args, exec) {
|
|
1383
|
+
assertAllowed('browser_drag', exec);
|
|
1384
|
+
const browser = ctx.get('browser');
|
|
1385
|
+
if (browser === undefined)
|
|
1386
|
+
throw new Error('tool-browser: browser service unavailable');
|
|
1387
|
+
const target = (point) => ({
|
|
1388
|
+
...point.x !== undefined ? { x: point.x } : {},
|
|
1389
|
+
...point.y !== undefined ? { y: point.y } : {},
|
|
1390
|
+
...point.selector !== undefined ? { selector: point.selector } : {},
|
|
1391
|
+
...point.text !== undefined ? { text: point.text } : {},
|
|
1392
|
+
});
|
|
1393
|
+
const result = await withTaskAction(browser, taskKey(exec), 'drag page', exec, session => browser.drag(session, {
|
|
1394
|
+
from: target(args.from),
|
|
1395
|
+
to: target(args.to),
|
|
1396
|
+
...args.steps !== undefined ? { steps: args.steps } : {},
|
|
1397
|
+
}, exec.signal));
|
|
1398
|
+
return { dragged: true, ...result.from.target === undefined ? {} : { from: result.from.target }, ...result.to.target === undefined ? {} : { to: result.to.target } };
|
|
1399
|
+
},
|
|
1400
|
+
}));
|
|
1401
|
+
ctx.tools.register(defineTool({
|
|
1402
|
+
name: 'browser_screenshot',
|
|
1403
|
+
description: 'Capture the current shared-browser page as a PNG screenshot. Use for visual confirmation of layout, charts, designs, or CAPTCHAs, or to feed a vision tool (read_image) that locates elements visually. Supports optional full-page capture and optional save-to-file (the saved path can be passed to read_image for vision-based element location).',
|
|
1404
|
+
parameters: {
|
|
1405
|
+
fullPage: { type: 'boolean', description: 'Capture the full scrollable page instead of the viewport (default false).' },
|
|
1406
|
+
savePath: { type: 'string', description: 'Absolute file path to also save the PNG to (e.g. for read_image vision location). It must be inside the browser-electron write roots; anything else is refused with "outside the allowed roots".' },
|
|
1407
|
+
},
|
|
1408
|
+
output: {
|
|
1409
|
+
schema: {
|
|
1410
|
+
type: 'object',
|
|
1411
|
+
additionalProperties: false,
|
|
1412
|
+
properties: {
|
|
1413
|
+
dataUrl: { type: 'string', required: true, description: 'Base64 PNG data URL of the screenshot.' },
|
|
1414
|
+
path: { type: 'string', description: 'The file path the screenshot was saved to, when savePath was given.' },
|
|
1415
|
+
},
|
|
1416
|
+
},
|
|
1417
|
+
render: (_args, value) => [{ type: 'text', text: `Screenshot captured (${Math.round(value.dataUrl.length * 3 / 4 / 1024)} KiB)${value.path !== undefined ? ` saved to ${value.path}` : ''}.` }],
|
|
1418
|
+
},
|
|
1419
|
+
timeoutMs,
|
|
1420
|
+
isConcurrencySafe: () => true,
|
|
1421
|
+
async execute(args, exec) {
|
|
1422
|
+
// An unsaved capture is read-only and stays outside the allow-list; the
|
|
1423
|
+
// variant that writes a file is guarded like the other writing tools.
|
|
1424
|
+
if (args.savePath !== undefined)
|
|
1425
|
+
assertAllowed('browser_screenshot', exec);
|
|
1426
|
+
const browser = ctx.get('browser');
|
|
1427
|
+
if (browser === undefined)
|
|
1428
|
+
throw new Error('tool-browser: browser service unavailable');
|
|
1429
|
+
const key = taskKey(exec);
|
|
1430
|
+
const capture = (session) => browser.screenshot(session, {
|
|
1431
|
+
...args.fullPage === true ? { fullPage: true } : {},
|
|
1432
|
+
...args.savePath !== undefined ? { savePath: args.savePath } : {},
|
|
1433
|
+
}, exec.signal);
|
|
1434
|
+
// A saved capture must not share a dedup key with an unsaved one, and
|
|
1435
|
+
// two different save paths must not either; both variants join the FIFO.
|
|
1436
|
+
const readKey = 'screenshot:' + JSON.stringify({ fullPage: args.fullPage === true, savePath: args.savePath });
|
|
1437
|
+
const shot = await withTaskRead(browser, key, readKey, capture);
|
|
1438
|
+
return {
|
|
1439
|
+
dataUrl: shot.dataUrl,
|
|
1440
|
+
...shot.path !== undefined ? { path: shot.path } : {},
|
|
1441
|
+
};
|
|
1442
|
+
},
|
|
1443
|
+
}));
|
|
1444
|
+
if (config.tabTools !== false) {
|
|
1445
|
+
ctx.tools.register(defineTool({
|
|
1446
|
+
name: 'browser_list_tabs',
|
|
1447
|
+
description: 'List the shared-browser session\'s tabs with their URLs and which is active.',
|
|
1448
|
+
parameters: {},
|
|
1449
|
+
output: {
|
|
1450
|
+
schema: {
|
|
1451
|
+
type: 'object',
|
|
1452
|
+
additionalProperties: false,
|
|
1453
|
+
properties: {
|
|
1454
|
+
tabs: {
|
|
1455
|
+
type: 'array',
|
|
1456
|
+
required: true,
|
|
1457
|
+
items: {
|
|
1458
|
+
type: 'object',
|
|
1459
|
+
additionalProperties: false,
|
|
1460
|
+
properties: {
|
|
1461
|
+
id: { type: 'string', required: true },
|
|
1462
|
+
url: { type: 'string', required: true },
|
|
1463
|
+
active: { type: 'boolean', required: true },
|
|
1464
|
+
},
|
|
1465
|
+
},
|
|
1466
|
+
},
|
|
1467
|
+
},
|
|
1468
|
+
},
|
|
1469
|
+
render: (_args, value) => [{
|
|
1470
|
+
type: 'text',
|
|
1471
|
+
text: value.tabs
|
|
1472
|
+
.map(t => `${t.active ? '*' : ' '} ${t.id} ${t.url}`).join('\n'),
|
|
1473
|
+
}],
|
|
1474
|
+
},
|
|
1475
|
+
timeoutMs,
|
|
1476
|
+
isConcurrencySafe: () => true,
|
|
1477
|
+
async execute(_args, exec) {
|
|
1478
|
+
const browser = ctx.get('browser');
|
|
1479
|
+
if (browser === undefined)
|
|
1480
|
+
throw new Error('tool-browser: browser service unavailable');
|
|
1481
|
+
const key = taskKey(exec);
|
|
1482
|
+
const tabs = await withTaskRead(browser, key, 'list_tabs', session => browser.listTabs(session));
|
|
1483
|
+
return { tabs: tabs.map(t => ({ id: t.id, url: t.url, active: t.active })) };
|
|
1484
|
+
},
|
|
1485
|
+
}));
|
|
1486
|
+
ctx.tools.register(defineTool({
|
|
1487
|
+
name: 'browser_switch_tab',
|
|
1488
|
+
description: 'Switch the shared browser to a tab by id (from browser_list_tabs).',
|
|
1489
|
+
parameters: {
|
|
1490
|
+
tabId: { type: 'string', required: true, description: 'The tab id to switch to.' },
|
|
1491
|
+
},
|
|
1492
|
+
output: {
|
|
1493
|
+
schema: { type: 'object', additionalProperties: false, properties: { switched: { type: 'boolean', required: true } } },
|
|
1494
|
+
render: (_args, value) => [{ type: 'text', text: value.switched ? 'Switched.' : 'Tab not found.' }],
|
|
1495
|
+
},
|
|
1496
|
+
timeoutMs,
|
|
1497
|
+
isConcurrencySafe: () => true,
|
|
1498
|
+
async execute(args, exec) {
|
|
1499
|
+
assertAllowed('browser_switch_tab', exec);
|
|
1500
|
+
const browser = ctx.get('browser');
|
|
1501
|
+
if (browser === undefined)
|
|
1502
|
+
throw new Error('tool-browser: browser service unavailable');
|
|
1503
|
+
await withTaskAction(browser, taskKey(exec), 'switch tab', exec, session => browser.switchTab(session, args.tabId));
|
|
1504
|
+
return { switched: true };
|
|
1505
|
+
},
|
|
1506
|
+
}));
|
|
1507
|
+
ctx.tools.register(defineTool({
|
|
1508
|
+
name: 'browser_close_tab',
|
|
1509
|
+
description: 'Close a tab in the shared browser by id. Closing the active tab activates the next.',
|
|
1510
|
+
parameters: {
|
|
1511
|
+
tabId: { type: 'string', required: true, description: 'The tab id to close.' },
|
|
1512
|
+
},
|
|
1513
|
+
output: {
|
|
1514
|
+
schema: { type: 'object', additionalProperties: false, properties: { closed: { type: 'boolean', required: true } } },
|
|
1515
|
+
render: (_args, value) => [{ type: 'text', text: value.closed ? 'Closed.' : 'Tab not found.' }],
|
|
1516
|
+
},
|
|
1517
|
+
timeoutMs,
|
|
1518
|
+
isConcurrencySafe: () => true,
|
|
1519
|
+
async execute(args, exec) {
|
|
1520
|
+
assertAllowed('browser_close_tab', exec);
|
|
1521
|
+
const browser = ctx.get('browser');
|
|
1522
|
+
if (browser === undefined)
|
|
1523
|
+
throw new Error('tool-browser: browser service unavailable');
|
|
1524
|
+
const closed = await withTaskAction(browser, taskKey(exec), 'close tab', exec, session => browser.closeTab(session, args.tabId));
|
|
1525
|
+
return { closed };
|
|
1526
|
+
},
|
|
1527
|
+
}));
|
|
1528
|
+
ctx.tools.register(defineTool({
|
|
1529
|
+
name: 'browser_reset',
|
|
1530
|
+
description: 'Close every tab in the shared browser and start fresh with one blank tab.',
|
|
1531
|
+
parameters: {},
|
|
1532
|
+
output: {
|
|
1533
|
+
schema: { type: 'object', additionalProperties: false, properties: { reset: { type: 'boolean', required: true } } },
|
|
1534
|
+
render: (_args, value) => [{ type: 'text', text: value.reset ? 'Browser reset.' : 'Failed.' }],
|
|
1535
|
+
},
|
|
1536
|
+
timeoutMs,
|
|
1537
|
+
isConcurrencySafe: () => true,
|
|
1538
|
+
async execute(_args, exec) {
|
|
1539
|
+
assertAllowed('browser_reset', exec);
|
|
1540
|
+
const browser = ctx.get('browser');
|
|
1541
|
+
if (browser === undefined)
|
|
1542
|
+
throw new Error('tool-browser: browser service unavailable');
|
|
1543
|
+
await withTaskAction(browser, taskKey(exec), 'reset tabs', exec, session => browser.reset(session));
|
|
1544
|
+
return { reset: true };
|
|
1545
|
+
},
|
|
1546
|
+
}));
|
|
1547
|
+
}
|
|
1548
|
+
ctx.tools.register(defineTool({
|
|
1549
|
+
name: 'browser_history',
|
|
1550
|
+
description: 'List the shared browser session\'s recorded operation history (navigate/execute/click/type/pressKey), newest last, with per-step success/error. Use to understand what the agent did and to pick a step to replay.',
|
|
1551
|
+
parameters: {},
|
|
1552
|
+
output: {
|
|
1553
|
+
schema: {
|
|
1554
|
+
type: 'object',
|
|
1555
|
+
additionalProperties: false,
|
|
1556
|
+
properties: {
|
|
1557
|
+
entries: {
|
|
1558
|
+
type: 'array',
|
|
1559
|
+
required: true,
|
|
1560
|
+
items: {
|
|
1561
|
+
type: 'object',
|
|
1562
|
+
additionalProperties: false,
|
|
1563
|
+
properties: {
|
|
1564
|
+
seq: { type: 'number', required: true },
|
|
1565
|
+
action: { type: 'string', required: true },
|
|
1566
|
+
ok: { type: 'boolean', required: true },
|
|
1567
|
+
params: { type: 'object', additionalProperties: true, required: true },
|
|
1568
|
+
result: { type: 'string' },
|
|
1569
|
+
error: { type: 'string' },
|
|
1570
|
+
},
|
|
1571
|
+
},
|
|
1572
|
+
},
|
|
1573
|
+
},
|
|
1574
|
+
},
|
|
1575
|
+
render: (_args, value) => {
|
|
1576
|
+
const entries = value.entries;
|
|
1577
|
+
if (entries.length === 0)
|
|
1578
|
+
return [{ type: 'text', text: '(no recorded operations yet)' }];
|
|
1579
|
+
return [{
|
|
1580
|
+
type: 'text',
|
|
1581
|
+
text: entries.map(e => {
|
|
1582
|
+
const rawParams = JSON.stringify(e.params);
|
|
1583
|
+
const shownParams = rawParams.length > 300 ? rawParams.slice(0, 300) + '…' : rawParams;
|
|
1584
|
+
return `#${e.seq} ${e.action} ${e.ok ? 'ok' : 'FAIL'} ${shownParams}${e.result !== undefined ? ` -> ${e.result}` : ''}${e.error !== undefined ? ` !! ${e.error}` : ''}`;
|
|
1585
|
+
}).join('\n'),
|
|
1586
|
+
}];
|
|
1587
|
+
},
|
|
1588
|
+
},
|
|
1589
|
+
timeoutMs,
|
|
1590
|
+
isConcurrencySafe: () => true,
|
|
1591
|
+
async execute(_args, exec) {
|
|
1592
|
+
const browser = ctx.get('browser');
|
|
1593
|
+
if (browser === undefined)
|
|
1594
|
+
throw new Error('tool-browser: browser service unavailable');
|
|
1595
|
+
const key = taskKey(exec);
|
|
1596
|
+
const entries = await withTaskRead(browser, key, 'history', session => browser.history(session));
|
|
1597
|
+
const rendered = entries.map(e => {
|
|
1598
|
+
const row = {
|
|
1599
|
+
seq: e.seq,
|
|
1600
|
+
action: e.action,
|
|
1601
|
+
ok: e.ok,
|
|
1602
|
+
params: summarizeHistoryParams(e.params),
|
|
1603
|
+
};
|
|
1604
|
+
if (e.result !== undefined)
|
|
1605
|
+
row.result = e.result;
|
|
1606
|
+
if (e.error !== undefined)
|
|
1607
|
+
row.error = e.error;
|
|
1608
|
+
return row;
|
|
1609
|
+
});
|
|
1610
|
+
return { entries: rendered };
|
|
1611
|
+
},
|
|
1612
|
+
}));
|
|
1613
|
+
ctx.tools.register(defineTool({
|
|
1614
|
+
name: 'browser_replay',
|
|
1615
|
+
description: 'Replay one recorded browser operation by its history sequence number (from browser_history). Navigate/click/type/pressKey are re-issued against the current page; execute re-runs its script. The replayed step is appended to history as a new entry.',
|
|
1616
|
+
parameters: {
|
|
1617
|
+
seq: { type: 'number', required: true, description: 'The history entry sequence number to replay.' },
|
|
1618
|
+
},
|
|
1619
|
+
output: {
|
|
1620
|
+
schema: { type: 'object', additionalProperties: false, properties: { replayed: { type: 'boolean', required: true } } },
|
|
1621
|
+
render: (_args, value) => [{ type: 'text', text: value.replayed ? 'Replayed.' : 'Replay failed.' }],
|
|
1622
|
+
},
|
|
1623
|
+
timeoutMs,
|
|
1624
|
+
isConcurrencySafe: () => false,
|
|
1625
|
+
async execute(args, exec) {
|
|
1626
|
+
assertAllowed('browser_replay', exec);
|
|
1627
|
+
const browser = ctx.get('browser');
|
|
1628
|
+
if (browser === undefined)
|
|
1629
|
+
throw new Error('tool-browser: browser service unavailable');
|
|
1630
|
+
await withTaskAction(browser, taskKey(exec), 'replay browser action', exec, session => browser.replay(session, args.seq));
|
|
1631
|
+
return { replayed: true };
|
|
1632
|
+
},
|
|
1633
|
+
}));
|
|
1634
|
+
ctx.tools.register(defineTool({
|
|
1635
|
+
name: 'browser_download',
|
|
1636
|
+
description: 'Download a URL to a local file, keeping the browser session\'s cookies and login state. Use for fetching files behind authentication or from the current page context. Available on the self-hosted browser; the desktop shell delegates downloads to the real browser UI.',
|
|
1637
|
+
parameters: {
|
|
1638
|
+
url: { type: 'string', required: true, description: 'The URL to download.' },
|
|
1639
|
+
savePath: { type: 'string', required: true, description: 'Absolute path of the file to write.' },
|
|
1640
|
+
},
|
|
1641
|
+
output: {
|
|
1642
|
+
schema: { type: 'object', additionalProperties: false, properties: { path: { type: 'string', required: true } } },
|
|
1643
|
+
render: (_args, value) => [{ type: 'text', text: `Downloaded to ${value.path}.` }],
|
|
1644
|
+
},
|
|
1645
|
+
timeoutMs,
|
|
1646
|
+
isConcurrencySafe: () => false,
|
|
1647
|
+
async execute(args, exec) {
|
|
1648
|
+
assertAllowed('browser_download', exec);
|
|
1649
|
+
const browser = ctx.get('browser');
|
|
1650
|
+
if (browser === undefined)
|
|
1651
|
+
throw new Error('tool-browser: browser service unavailable');
|
|
1652
|
+
const result = await withTaskAction(browser, taskKey(exec), 'download file', exec, session => browser.download(session, { url: args.url, savePath: args.savePath }, exec.signal));
|
|
1653
|
+
return { path: result.path };
|
|
1654
|
+
},
|
|
1655
|
+
}));
|
|
1656
|
+
ctx.tools.register(defineTool({
|
|
1657
|
+
name: 'browser_session',
|
|
1658
|
+
description: 'Show THIS task\'s browser session: its id and open tabs. Each task (DSH session) has its own browser session, so this reflects what your task drives. The window is shared with the human and other tasks, but tab sets and history are isolated per task.',
|
|
1659
|
+
parameters: {},
|
|
1660
|
+
output: {
|
|
1661
|
+
schema: {
|
|
1662
|
+
type: 'object',
|
|
1663
|
+
additionalProperties: false,
|
|
1664
|
+
properties: {
|
|
1665
|
+
session: { type: 'string', required: true },
|
|
1666
|
+
tabs: {
|
|
1667
|
+
type: 'array',
|
|
1668
|
+
required: true,
|
|
1669
|
+
items: {
|
|
1670
|
+
type: 'object',
|
|
1671
|
+
additionalProperties: false,
|
|
1672
|
+
properties: {
|
|
1673
|
+
id: { type: 'string', required: true },
|
|
1674
|
+
url: { type: 'string', required: true },
|
|
1675
|
+
active: { type: 'boolean', required: true },
|
|
1676
|
+
},
|
|
1677
|
+
},
|
|
1678
|
+
},
|
|
1679
|
+
},
|
|
1680
|
+
},
|
|
1681
|
+
render: (_args, value) => [{
|
|
1682
|
+
type: 'text',
|
|
1683
|
+
text: `Session ${value.session}\n${value.tabs.map(t => `${t.active ? '*' : ' '} ${t.id} ${t.url}`).join('\n')}`,
|
|
1684
|
+
}],
|
|
1685
|
+
},
|
|
1686
|
+
timeoutMs,
|
|
1687
|
+
isConcurrencySafe: () => true,
|
|
1688
|
+
async execute(_args, exec) {
|
|
1689
|
+
const browser = ctx.get('browser');
|
|
1690
|
+
if (browser === undefined)
|
|
1691
|
+
throw new Error('tool-browser: browser service unavailable');
|
|
1692
|
+
const key = taskKey(exec);
|
|
1693
|
+
const session = await ensureSession(browser, key);
|
|
1694
|
+
const tabs = await withTaskRead(browser, key, 'session', () => browser.listTabs(session));
|
|
1695
|
+
return { session, tabs: tabs.map(t => ({ id: t.id, url: t.url, active: t.active })) };
|
|
1696
|
+
},
|
|
1697
|
+
}));
|
|
1698
|
+
ctx.tools.register(defineTool({
|
|
1699
|
+
name: 'browser_reset_session',
|
|
1700
|
+
description: 'Reset THIS task\'s browser session: close it entirely so the next browser_* call starts a fresh session with one blank tab. Other tasks\' sessions are untouched. Use when a session is in a bad state or you want a clean slate.',
|
|
1701
|
+
parameters: {},
|
|
1702
|
+
output: {
|
|
1703
|
+
schema: { type: 'object', additionalProperties: false, properties: { reset: { type: 'boolean', required: true } } },
|
|
1704
|
+
render: (_args, value) => [{ type: 'text', text: value.reset ? 'This task\'s browser session was closed; the next call starts fresh.' : 'Failed.' }],
|
|
1705
|
+
},
|
|
1706
|
+
timeoutMs,
|
|
1707
|
+
isConcurrencySafe: () => true,
|
|
1708
|
+
async execute(_args, exec) {
|
|
1709
|
+
assertAllowed('browser_reset_session', exec);
|
|
1710
|
+
const browser = ctx.get('browser');
|
|
1711
|
+
if (browser === undefined)
|
|
1712
|
+
throw new Error('tool-browser: browser service unavailable');
|
|
1713
|
+
const key = taskKey(exec);
|
|
1714
|
+
// Join the task's FIFO: closing the session while a queued operation is
|
|
1715
|
+
// still using it would tear the view out from under that call.
|
|
1716
|
+
await queueTaskOperation(key, async () => {
|
|
1717
|
+
const session = sessionsByTask.get(key);
|
|
1718
|
+
if (session === undefined)
|
|
1719
|
+
return;
|
|
1720
|
+
try {
|
|
1721
|
+
await browser.close(session);
|
|
1722
|
+
}
|
|
1723
|
+
finally {
|
|
1724
|
+
// Always forget the mapping so the next call opens a fresh session,
|
|
1725
|
+
// even if the provider close threw (the session is half-closed).
|
|
1726
|
+
sessionsByTask.delete(key);
|
|
1727
|
+
}
|
|
1728
|
+
});
|
|
1729
|
+
return { reset: true };
|
|
1730
|
+
},
|
|
1731
|
+
}));
|
|
1732
|
+
ctx.tools.register(defineTool({
|
|
1733
|
+
name: 'browser_restrict',
|
|
1734
|
+
description: 'Restrict which browser actions are allowed for THIS task, to prevent stray clicks/navigation. Pass a list of browser tool names (e.g. ["browser_snapshot","browser_content","browser_click"]) — any other browser_* call from this task is refused. The rule is scoped to the calling task; other tasks keep their own rules. Pass an empty list or omit to lift this task\'s restriction. Read-only tools (snapshot/content/screenshot/session/history/list_tabs/challenge) are never blocked.',
|
|
1735
|
+
parameters: {
|
|
1736
|
+
allowed: {
|
|
1737
|
+
type: 'array',
|
|
1738
|
+
items: { type: 'string' },
|
|
1739
|
+
description: 'Allow-list of browser tool names; empty clears the restriction.',
|
|
1740
|
+
},
|
|
1741
|
+
},
|
|
1742
|
+
output: {
|
|
1743
|
+
schema: { type: 'object', additionalProperties: false, properties: { restrictedTo: { type: 'array', required: true, items: { type: 'string' } } } },
|
|
1744
|
+
render: (_args, value) => [{ type: 'text', text: value.restrictedTo.length > 0 ? `Restricted to: ${value.restrictedTo.join(', ')}` : 'Restriction lifted.' }],
|
|
1745
|
+
},
|
|
1746
|
+
timeoutMs,
|
|
1747
|
+
isConcurrencySafe: () => false,
|
|
1748
|
+
async execute(args, exec) {
|
|
1749
|
+
// Always allowed so the guard can be lifted.
|
|
1750
|
+
const allowed = args.allowed ?? [];
|
|
1751
|
+
const unknown = allowed.filter((t) => !t.startsWith('browser_'));
|
|
1752
|
+
if (unknown.length > 0) {
|
|
1753
|
+
throw new Error(`browser_restrict: unknown tool name(s) ${unknown.map(t => `"${t}"`).join(', ')} (must start with "browser_")`);
|
|
1754
|
+
}
|
|
1755
|
+
// A misspelled name would otherwise be accepted and silently refuse every
|
|
1756
|
+
// guarded action for this task, so check it against the real registry.
|
|
1757
|
+
const schemas = ctx.tools.schemas;
|
|
1758
|
+
const registered = typeof schemas === 'function' ? schemas.call(ctx.tools).map(schema => schema.name) : [];
|
|
1759
|
+
const misspelled = registered.length === 0 ? [] : allowed.filter((t) => !registered.includes(t));
|
|
1760
|
+
if (misspelled.length > 0) {
|
|
1761
|
+
throw new Error(`browser_restrict: no such browser tool ${misspelled.map(t => `"${t}"`).join(', ')}`);
|
|
1762
|
+
}
|
|
1763
|
+
// Empty list (or omitted) lifts THIS task's restriction; a non-empty list
|
|
1764
|
+
// becomes this task's allow-list. Either way the rule is task-scoped.
|
|
1765
|
+
const key = taskKey(exec);
|
|
1766
|
+
restrictedToByTask.set(key, allowed.length === 0 ? [] : [...allowed]);
|
|
1767
|
+
const restrictedTo = restrictionFor(key);
|
|
1768
|
+
return { restrictedTo: restrictedTo === undefined ? [] : [...restrictedTo] };
|
|
1769
|
+
},
|
|
1770
|
+
}));
|
|
1771
|
+
ctx.tools.register(defineTool({
|
|
1772
|
+
name: 'browser_auth',
|
|
1773
|
+
description: 'Export or restore the browser session\'s cookies (login state). Use "flush" to get a JSON cookie list, "restore" with that list (or with file: a JSON export on disk) to put logins back, or "clear" with a domain and/or name filter to drop the cookies of that site (e.g. stale WAF challenge generations). Logging in once in this browser is usually easier than importing: the profile is persistent, so a human can use browser_handoff and log in by hand. A browser\'s own cookie store cannot be read automatically — Chrome and Edge 127+ encrypt cookie values with App-Bound Encryption — so importing means a JSON export the user produced. Available on the self-hosted browser.',
|
|
1774
|
+
parameters: {
|
|
1775
|
+
action: { type: 'string', required: true, enum: ['flush', 'restore', 'clear'], description: 'flush = export cookies; restore = import cookies; clear = remove cookies for a domain/name scope.' },
|
|
1776
|
+
cookies: { type: 'array', items: { type: 'object', additionalProperties: true }, description: 'Cookie list to restore (action=restore).' },
|
|
1777
|
+
file: { type: 'string', description: 'restore only: read the cookie list from this JSON file (an array, or {"cookies": [...]}) instead of passing it inline. Takes precedence over cookies. The path must be inside browser-electron.readRoots.' },
|
|
1778
|
+
domain: { type: 'string', description: 'clear only: remove cookies for this domain and its subdomains (e.g. "example.com").' },
|
|
1779
|
+
name: { type: 'string', description: 'clear only: remove only this exact cookie name within the scope.' },
|
|
1780
|
+
all: { type: 'boolean', description: 'clear only: remove every cookie in the profile. Required when neither domain nor name is given; destructive.' },
|
|
1781
|
+
},
|
|
1782
|
+
output: {
|
|
1783
|
+
schema: {
|
|
1784
|
+
type: 'object',
|
|
1785
|
+
additionalProperties: false,
|
|
1786
|
+
properties: {
|
|
1787
|
+
cookies: { type: 'array', items: { type: 'object', additionalProperties: true } },
|
|
1788
|
+
restored: { type: 'number' },
|
|
1789
|
+
failed: { type: 'number' },
|
|
1790
|
+
removed: { type: 'number' },
|
|
1791
|
+
names: { type: 'array', items: { type: 'string' } },
|
|
1792
|
+
},
|
|
1793
|
+
},
|
|
1794
|
+
render: (_args, value) => [{
|
|
1795
|
+
type: 'text',
|
|
1796
|
+
text: value.removed !== undefined
|
|
1797
|
+
? `Removed ${value.removed} cookie(s)` + (value.names.length > 0 ? ': ' + value.names.join(', ') : '.')
|
|
1798
|
+
: value.cookies !== undefined
|
|
1799
|
+
? `Exported ${value.cookies.length} cookies.`
|
|
1800
|
+
: `Restored ${value.restored} cookies` + (typeof value.failed === 'number' && value.failed > 0 ? ` (${value.failed} skipped)` : '') + '.',
|
|
1801
|
+
}],
|
|
1802
|
+
},
|
|
1803
|
+
timeoutMs,
|
|
1804
|
+
isConcurrencySafe: () => false,
|
|
1805
|
+
async execute(args, exec) {
|
|
1806
|
+
assertAllowed('browser_auth', exec);
|
|
1807
|
+
const browser = ctx.get('browser');
|
|
1808
|
+
if (browser === undefined)
|
|
1809
|
+
throw new Error('tool-browser: browser service unavailable');
|
|
1810
|
+
const session = await ensureSession(browser, taskKey(exec));
|
|
1811
|
+
if (args.action === 'flush') {
|
|
1812
|
+
const cookies = await browser.flushAuth(session);
|
|
1813
|
+
return { cookies: cookies.map(c => ({ ...c })) };
|
|
1814
|
+
}
|
|
1815
|
+
if (args.action === 'clear') {
|
|
1816
|
+
const domain = typeof args.domain === 'string' && args.domain.trim() !== '' ? args.domain : undefined;
|
|
1817
|
+
const name = typeof args.name === 'string' && args.name !== '' ? args.name : undefined;
|
|
1818
|
+
if (domain === undefined && name === undefined && args.all !== true) {
|
|
1819
|
+
throw new Error('browser_auth clear requires a domain or name filter; pass all: true to remove every cookie in the profile');
|
|
1820
|
+
}
|
|
1821
|
+
const cleared = await browser.clearAuth(session, {
|
|
1822
|
+
...domain !== undefined ? { domain } : {},
|
|
1823
|
+
...name !== undefined ? { name } : {},
|
|
1824
|
+
...args.all === true ? { all: true } : {},
|
|
1825
|
+
});
|
|
1826
|
+
return { removed: cleared.removed, names: [...cleared.names] };
|
|
1827
|
+
}
|
|
1828
|
+
// A file source exists because a real cookie export runs to hundreds of
|
|
1829
|
+
// entries, which is impractical to pass through the model inline.
|
|
1830
|
+
const file = typeof args.file === 'string' && args.file.trim() !== '' ? args.file : undefined;
|
|
1831
|
+
if (file !== undefined) {
|
|
1832
|
+
const imported = await browser.importAuth(session, file);
|
|
1833
|
+
return { restored: imported.restored, failed: imported.failed };
|
|
1834
|
+
}
|
|
1835
|
+
const list = (args.cookies ?? []);
|
|
1836
|
+
const restored = await browser.restoreAuth(session, list);
|
|
1837
|
+
return { restored };
|
|
1838
|
+
},
|
|
1839
|
+
}));
|
|
1840
|
+
ctx.tools.register(defineTool({
|
|
1841
|
+
name: 'browser_scrape',
|
|
1842
|
+
description: 'Visit many URLs in the background and append one JSON line per page to a file, so the results never travel back through the model: a batch of a thousand costs the same number of tokens as a batch of one. Start with action=start, then poll action=status. Each row is { seq, url, ok, data } or { seq, url, ok, error }, where seq is the index of that URL in the input. Rows are appended the moment they are produced, so a stopped batch keeps everything it managed. Raise concurrency to load several pages at once; rows then arrive in completion order. The batch drives the task\'s active tab, so the visible page changes while it runs.',
|
|
1843
|
+
parameters: {
|
|
1844
|
+
action: { type: 'string', enum: ['start', 'status', 'stop', 'list'], description: 'start (default) begins a batch; status and stop need id; list shows every batch this process knows.' },
|
|
1845
|
+
urls: { type: 'array', items: { type: 'string' }, description: 'start: URLs to visit, in order.' },
|
|
1846
|
+
script: { type: 'string', description: 'start: expression evaluated on each page once it is ready; its JSON value becomes the row\'s data. An async IIFE is fine (promises are awaited).' },
|
|
1847
|
+
outPath: { type: 'string', description: 'start: JSONL destination, inside browser-electron.writeRoots. Truncated when the batch starts.' },
|
|
1848
|
+
waitFor: { type: 'string', description: 'start: optional CSS selector awaited on each page before the script runs.' },
|
|
1849
|
+
timeoutMs: { type: 'number', description: 'start: per-URL budget in ms for the wait and the extraction (default 30000).' },
|
|
1850
|
+
concurrency: { type: 'number', description: 'start: how many pages to load at once (default 1, max 8). Each worker costs a tab of its own. Rows then land in completion order; every row carries its URL index as seq, so sort by seq to restore order.' },
|
|
1851
|
+
id: { type: 'string', description: 'status/stop: the batch id returned by start.' },
|
|
1852
|
+
},
|
|
1853
|
+
output: {
|
|
1854
|
+
schema: {
|
|
1855
|
+
type: 'object',
|
|
1856
|
+
additionalProperties: false,
|
|
1857
|
+
properties: {
|
|
1858
|
+
id: { type: 'string' },
|
|
1859
|
+
state: { type: 'string' },
|
|
1860
|
+
total: { type: 'number' },
|
|
1861
|
+
done: { type: 'number' },
|
|
1862
|
+
failed: { type: 'number' },
|
|
1863
|
+
path: { type: 'string' },
|
|
1864
|
+
error: { type: 'string' },
|
|
1865
|
+
jobs: { type: 'array', items: { type: 'object', additionalProperties: true } },
|
|
1866
|
+
},
|
|
1867
|
+
},
|
|
1868
|
+
render: (_args, value) => {
|
|
1869
|
+
if (Array.isArray(value.jobs)) {
|
|
1870
|
+
return [{ type: 'text', text: value.jobs.length === 0 ? 'No scrape batches.' : `${value.jobs.length} scrape batch(es).` }];
|
|
1871
|
+
}
|
|
1872
|
+
const parts = [`Scrape ${value.state}: ${value.done}/${value.total} rows`];
|
|
1873
|
+
if (typeof value.failed === 'number' && value.failed > 0)
|
|
1874
|
+
parts.push(`${value.failed} failed`);
|
|
1875
|
+
if (typeof value.path === 'string')
|
|
1876
|
+
parts.push(value.path);
|
|
1877
|
+
if (typeof value.error === 'string')
|
|
1878
|
+
parts.push(`error: ${value.error}`);
|
|
1879
|
+
return [{ type: 'text', text: parts.join(' — ') + '.' }];
|
|
1880
|
+
},
|
|
1881
|
+
},
|
|
1882
|
+
timeoutMs,
|
|
1883
|
+
isConcurrencySafe: () => false,
|
|
1884
|
+
async execute(args, exec) {
|
|
1885
|
+
assertAllowed('browser_scrape', exec);
|
|
1886
|
+
const browser = ctx.get('browser');
|
|
1887
|
+
if (browser === undefined)
|
|
1888
|
+
throw new Error('tool-browser: browser service unavailable');
|
|
1889
|
+
const action = typeof args.action === 'string' ? args.action : 'start';
|
|
1890
|
+
if (action === 'list') {
|
|
1891
|
+
return { jobs: (await browser.listScrapes()).map(job => ({ ...job })) };
|
|
1892
|
+
}
|
|
1893
|
+
if (action === 'status' || action === 'stop') {
|
|
1894
|
+
const id = typeof args.id === 'string' && args.id !== '' ? args.id : undefined;
|
|
1895
|
+
if (id === undefined)
|
|
1896
|
+
throw new Error(`browser_scrape ${action} requires id`);
|
|
1897
|
+
const status = action === 'stop' ? await browser.stopScrape(id) : await browser.scrapeStatus(id);
|
|
1898
|
+
return { ...status };
|
|
1899
|
+
}
|
|
1900
|
+
const outPath = typeof args.outPath === 'string' && args.outPath !== '' ? args.outPath : undefined;
|
|
1901
|
+
if (outPath === undefined)
|
|
1902
|
+
throw new Error('browser_scrape start requires outPath');
|
|
1903
|
+
const session = await ensureSession(browser, taskKey(exec));
|
|
1904
|
+
const status = await browser.startScrape(session, {
|
|
1905
|
+
urls: (args.urls ?? []),
|
|
1906
|
+
script: typeof args.script === 'string' ? args.script : '',
|
|
1907
|
+
outPath,
|
|
1908
|
+
...args.waitFor !== undefined ? { waitFor: args.waitFor } : {},
|
|
1909
|
+
...args.timeoutMs !== undefined ? { timeoutMs: args.timeoutMs } : {},
|
|
1910
|
+
...args.concurrency !== undefined ? { concurrency: args.concurrency } : {},
|
|
1911
|
+
});
|
|
1912
|
+
return { ...status };
|
|
1913
|
+
},
|
|
1914
|
+
}));
|
|
1915
|
+
}
|
|
1916
|
+
/**
|
|
1917
|
+
* Test hook: inspect and reset plugin-level state (used by tests).
|
|
1918
|
+
*
|
|
1919
|
+
* @internal This is a test seam, not part of the supported tool API. It stays
|
|
1920
|
+
* exported because `test/tool-browser-session.test.mjs` imports
|
|
1921
|
+
* `internals.clearSession` by name to simulate a lost tool-layer cache; treat
|
|
1922
|
+
* everything reachable here as unstable and internal to this package.
|
|
1923
|
+
*/
|
|
1924
|
+
export const internals = {
|
|
1925
|
+
/** A copy of the per-task session map (task key -> provider session id). */
|
|
1926
|
+
get sessions() { return new Map(sessionsByTask); },
|
|
1927
|
+
/** Drop one task's mapping without closing the provider session. */
|
|
1928
|
+
clearSession(key = 'default') { sessionsByTask.delete(key); },
|
|
1929
|
+
/** Number of task queues still holding a tail; drains back to 0 when idle. */
|
|
1930
|
+
get operationTailCount() { return operationTails.size; },
|
|
1931
|
+
};
|