dsh-builtin-browser 0.1.6 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +21 -3
- package/README.md +30 -4
- package/lib/browser/runtime.d.ts +24 -8
- package/lib/browser/runtime.js +35 -7
- package/lib/browser/types.d.ts +120 -7
- package/lib/browser/types.js +1 -1
- package/lib/browser-electron/host-main.js +85 -0
- package/lib/browser-electron/provider.d.ts +49 -8
- package/lib/browser-electron/provider.js +355 -34
- package/lib/browser-electron/remote-host.js +29 -1
- package/lib/tool-browser/index.d.ts +13 -7
- package/lib/tool-browser/index.js +509 -34
- package/package.json +5 -3
- package/src/browser/runtime.ts +260 -0
- package/src/browser/types.ts +338 -0
- package/src/browser-electron/entry.ts +58 -0
- package/src/browser-electron/host-main.ts +229 -0
- package/src/browser-electron/provider.ts +922 -0
- package/src/browser-electron/remote-host.ts +421 -0
- package/src/index.ts +33 -0
- package/src/tool-browser/index.ts +862 -0
- package/src/types/electron-shim.d.ts +82 -0
|
@@ -5,10 +5,12 @@
|
|
|
5
5
|
* `browser_switch_tab`, `browser_close_tab`, `browser_reset`).
|
|
6
6
|
*
|
|
7
7
|
* The tool layer owns only the model-facing schema, argument validation, and
|
|
8
|
-
* result formatting
|
|
9
|
-
* to the seam. Session lifecycle is owned here at the plugin level:
|
|
10
|
-
*
|
|
11
|
-
*
|
|
8
|
+
* result formatting — never provider selection or page driving, which belong
|
|
9
|
+
* to the seam. Session lifecycle is owned here at the plugin level: each
|
|
10
|
+
* calling task (a DSH session) gets its own browser session — the first
|
|
11
|
+
* `browser_open` (or any tool when no session exists) opens it, and later
|
|
12
|
+
* tools in the same task reuse it. Concurrent tasks therefore never fight
|
|
13
|
+
* over tabs, history, or navigation state.
|
|
12
14
|
* @module dsh-browser/tool-browser
|
|
13
15
|
*/
|
|
14
16
|
import { defineTool } from '@deepseek-ai/dsh-tools';
|
|
@@ -16,18 +18,57 @@ import { defineTool } from '@deepseek-ai/dsh-tools';
|
|
|
16
18
|
export const name = 'tool-browser';
|
|
17
19
|
/** The tool registry, browser seam, and system-prompt registry this tool layer consumes. */
|
|
18
20
|
export const inject = ['tools', 'browser', 'systemPrompt'];
|
|
19
|
-
/**
|
|
20
|
-
|
|
21
|
+
/** Per-task browser sessions, keyed by the calling DSH session id. */
|
|
22
|
+
const sessionsByTask = new Map();
|
|
21
23
|
/**
|
|
22
|
-
*
|
|
24
|
+
* Action restriction state: an allow-list of browser tool names, or undefined
|
|
25
|
+
* for unrestricted. When set, any browser_* tool not in the list is refused
|
|
26
|
+
* (browser_restrict itself is always allowed so the guard can be lifted).
|
|
27
|
+
*/
|
|
28
|
+
let restrictedTo;
|
|
29
|
+
/**
|
|
30
|
+
* Guard one browser tool call against the active restriction. Refuses calls
|
|
31
|
+
* not on the allow-list when a restriction is in effect.
|
|
32
|
+
* @param toolName - the browser tool about to run.
|
|
33
|
+
*/
|
|
34
|
+
function assertAllowed(toolName) {
|
|
35
|
+
if (restrictedTo === undefined)
|
|
36
|
+
return;
|
|
37
|
+
if (restrictedTo.includes(toolName))
|
|
38
|
+
return;
|
|
39
|
+
throw new Error(`browser action "${toolName}" is restricted (allow-list: ${restrictedTo.join(', ')})`);
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* The task key for a tool call: the calling DSH session id, or the shared
|
|
43
|
+
* default key when the call carries no agent context (CLI probes, tests).
|
|
44
|
+
* @param exec - the tool-execution context; only its optional agent id is read.
|
|
45
|
+
*/
|
|
46
|
+
function taskKey(exec) {
|
|
47
|
+
return exec?.agent?.id ?? 'default';
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Resolve the calling task's browser session, opening one on first use.
|
|
23
51
|
* @param browser - the seam service.
|
|
24
|
-
* @
|
|
52
|
+
* @param key - the task key (see {@link taskKey}).
|
|
53
|
+
* @returns the task's session id.
|
|
25
54
|
*/
|
|
26
|
-
async function ensureSession(browser) {
|
|
27
|
-
|
|
28
|
-
|
|
55
|
+
async function ensureSession(browser, key) {
|
|
56
|
+
let session = sessionsByTask.get(key);
|
|
57
|
+
if (session === undefined) {
|
|
58
|
+
session = await browser.open();
|
|
59
|
+
sessionsByTask.set(key, session);
|
|
29
60
|
}
|
|
30
|
-
return
|
|
61
|
+
return session;
|
|
62
|
+
}
|
|
63
|
+
/** Coerce a tool-provided string value back to boolean/number when it looks like one. */
|
|
64
|
+
function parseFillValue(v) {
|
|
65
|
+
if (v === 'true')
|
|
66
|
+
return true;
|
|
67
|
+
if (v === 'false')
|
|
68
|
+
return false;
|
|
69
|
+
if (v !== undefined && /^-?\d+(\.\d+)?$/.test(v))
|
|
70
|
+
return Number(v);
|
|
71
|
+
return v ?? '';
|
|
31
72
|
}
|
|
32
73
|
/** Format a snapshot element list for the model. */
|
|
33
74
|
function formatSnapshot(snapshot) {
|
|
@@ -35,21 +76,26 @@ function formatSnapshot(snapshot) {
|
|
|
35
76
|
const header = `URL: ${snapshot.url}${snapshot.title !== undefined ? `\nTitle: ${snapshot.title}` : ''}`;
|
|
36
77
|
const body = lines.length > 0 ? lines.join('\n') : '(no interactive elements found)';
|
|
37
78
|
const tail = snapshot.truncated === true ? '\n(snapshot truncated)' : '';
|
|
38
|
-
|
|
79
|
+
const banner = snapshot.challenge?.blocked === true
|
|
80
|
+
? `\n\nCHALLENGE: ${snapshot.challenge.reason ?? 'human-verification'}. Do NOT keep retrying — ask the human to complete it in the shared browser window, then re-snapshot.`
|
|
81
|
+
: '';
|
|
82
|
+
return `${header}\n\n${body}${tail}${banner}`;
|
|
39
83
|
}
|
|
40
84
|
/** Register all browser tools with `ctx.tools`. */
|
|
41
85
|
export function apply(ctx, config = {}) {
|
|
42
86
|
const timeoutMs = config.timeoutMs ?? 60_000;
|
|
87
|
+
if (config.allowedActions !== undefined)
|
|
88
|
+
restrictedTo = [...config.allowedActions];
|
|
43
89
|
ctx.systemPrompt.section({
|
|
44
90
|
name: 'tool:browser',
|
|
45
91
|
// Tool guidance band is 100-199; 150 keeps clear of the common 110/120
|
|
46
92
|
// tool sections so ordering does not depend on plugin load sequence.
|
|
47
93
|
order: 150,
|
|
48
|
-
text: 'Use the browser_* tools to operate a real shared browser the human can see and take over. Locate elements by snapshot reference numbers (browser_snapshot) and drive them with browser_execute (DOM-referenced JS, native setters for framework inputs). browser_screenshot is for visual confirmation, not primary targeting. Keep the human informed of what you are doing on the page.',
|
|
94
|
+
text: 'Use the browser_* tools to operate a real shared browser the human can see and take over. Locate elements by snapshot reference numbers (browser_snapshot) and drive them with browser_execute (DOM-referenced JS, native setters for framework inputs). For form filling prefer browser_fill, which handles controlled inputs, selects, checkboxes and radio groups in one batch. browser_screenshot is for visual confirmation, not primary targeting. Keep the human informed of what you are doing on the page. Each task gets its own browser session: your tabs and history are isolated from other tasks, so do not assume another task\'s navigation state is visible to you. If a snapshot or browser_challenge reports a human-verification challenge (CAPTCHA), stop retrying and ask the human to complete it in the shared browser window, then re-check.',
|
|
49
95
|
});
|
|
50
96
|
ctx.tools.register(defineTool({
|
|
51
97
|
name: 'browser_open',
|
|
52
|
-
description: 'Open a URL in the shared browser. Opens
|
|
98
|
+
description: 'Open a URL in the shared browser window. Opens this task\'s browser session on first use; optionally opens in a new tab. Returns the resulting page snapshot.',
|
|
53
99
|
parameters: {
|
|
54
100
|
url: { type: 'string', required: true, description: 'The URL to open (HTTP/HTTPS).' },
|
|
55
101
|
newTab: { type: 'boolean', description: 'Open in a new tab instead of the active one.' },
|
|
@@ -76,6 +122,15 @@ export function apply(ctx, config = {}) {
|
|
|
76
122
|
},
|
|
77
123
|
},
|
|
78
124
|
},
|
|
125
|
+
challenge: {
|
|
126
|
+
type: 'object',
|
|
127
|
+
additionalProperties: false,
|
|
128
|
+
properties: {
|
|
129
|
+
blocked: { type: 'boolean', required: true },
|
|
130
|
+
kind: { type: 'string' },
|
|
131
|
+
reason: { type: 'string' },
|
|
132
|
+
},
|
|
133
|
+
},
|
|
79
134
|
},
|
|
80
135
|
},
|
|
81
136
|
render: (_args, value) => [{ type: 'text', text: formatSnapshot(value) }],
|
|
@@ -83,10 +138,11 @@ export function apply(ctx, config = {}) {
|
|
|
83
138
|
timeoutMs,
|
|
84
139
|
isConcurrencySafe: () => true,
|
|
85
140
|
async execute(args, exec) {
|
|
141
|
+
assertAllowed('browser_open');
|
|
86
142
|
const browser = ctx.get('browser');
|
|
87
143
|
if (browser === undefined)
|
|
88
144
|
throw new Error('tool-browser: browser service unavailable');
|
|
89
|
-
const session = await ensureSession(browser);
|
|
145
|
+
const session = await ensureSession(browser, taskKey(exec));
|
|
90
146
|
await browser.openUrl(session, {
|
|
91
147
|
url: args.url,
|
|
92
148
|
...args.newTab === true ? { newTab: true } : {},
|
|
@@ -97,6 +153,7 @@ export function apply(ctx, config = {}) {
|
|
|
97
153
|
...snapshot.title !== undefined ? { title: snapshot.title } : {},
|
|
98
154
|
elements: snapshot.elements.map(el => ({ ref: el.ref, kind: el.kind, label: el.label, x: el.x, y: el.y })),
|
|
99
155
|
truncated: snapshot.truncated,
|
|
156
|
+
...snapshot.challenge !== undefined ? { challenge: snapshot.challenge } : {},
|
|
100
157
|
};
|
|
101
158
|
},
|
|
102
159
|
}));
|
|
@@ -126,6 +183,15 @@ export function apply(ctx, config = {}) {
|
|
|
126
183
|
},
|
|
127
184
|
},
|
|
128
185
|
},
|
|
186
|
+
challenge: {
|
|
187
|
+
type: 'object',
|
|
188
|
+
additionalProperties: false,
|
|
189
|
+
properties: {
|
|
190
|
+
blocked: { type: 'boolean', required: true },
|
|
191
|
+
kind: { type: 'string' },
|
|
192
|
+
reason: { type: 'string' },
|
|
193
|
+
},
|
|
194
|
+
},
|
|
129
195
|
},
|
|
130
196
|
},
|
|
131
197
|
render: (_args, value) => [{ type: 'text', text: formatSnapshot(value) }],
|
|
@@ -136,13 +202,54 @@ export function apply(ctx, config = {}) {
|
|
|
136
202
|
const browser = ctx.get('browser');
|
|
137
203
|
if (browser === undefined)
|
|
138
204
|
throw new Error('tool-browser: browser service unavailable');
|
|
139
|
-
const session = await ensureSession(browser);
|
|
205
|
+
const session = await ensureSession(browser, taskKey(exec));
|
|
140
206
|
const snapshot = await browser.snapshot(session, exec.signal);
|
|
141
207
|
return {
|
|
142
208
|
url: snapshot.url,
|
|
143
209
|
...snapshot.title !== undefined ? { title: snapshot.title } : {},
|
|
144
210
|
elements: snapshot.elements.map(el => ({ ref: el.ref, kind: el.kind, label: el.label, x: el.x, y: el.y })),
|
|
145
211
|
truncated: snapshot.truncated,
|
|
212
|
+
...snapshot.challenge !== undefined ? { challenge: snapshot.challenge } : {},
|
|
213
|
+
};
|
|
214
|
+
},
|
|
215
|
+
}));
|
|
216
|
+
ctx.tools.register(defineTool({
|
|
217
|
+
name: 'browser_challenge',
|
|
218
|
+
description: 'Check whether a human-verification challenge (CAPTCHA / bot detection: Cloudflare "Just a moment", reCAPTCHA, hCaptcha, Turnstile) is blocking the current page. When blocked, do NOT keep retrying automated steps — ask the human to complete the verification in the shared browser window, then re-check with browser_snapshot.',
|
|
219
|
+
parameters: {},
|
|
220
|
+
output: {
|
|
221
|
+
schema: {
|
|
222
|
+
type: 'object',
|
|
223
|
+
additionalProperties: false,
|
|
224
|
+
properties: {
|
|
225
|
+
blocked: { type: 'boolean', required: true },
|
|
226
|
+
kind: { type: 'string' },
|
|
227
|
+
reason: { type: 'string' },
|
|
228
|
+
hint: { type: 'string' },
|
|
229
|
+
},
|
|
230
|
+
},
|
|
231
|
+
render: (_args, value) => [{
|
|
232
|
+
type: 'text',
|
|
233
|
+
text: value.blocked
|
|
234
|
+
? `Challenge detected: ${value.reason ?? value.kind ?? 'human-verification'}. ${value.hint ?? ''}`
|
|
235
|
+
: 'No human-verification challenge detected.',
|
|
236
|
+
}],
|
|
237
|
+
},
|
|
238
|
+
timeoutMs,
|
|
239
|
+
isConcurrencySafe: () => true,
|
|
240
|
+
async execute(_args, exec) {
|
|
241
|
+
const browser = ctx.get('browser');
|
|
242
|
+
if (browser === undefined)
|
|
243
|
+
throw new Error('tool-browser: browser service unavailable');
|
|
244
|
+
const session = await ensureSession(browser, taskKey(exec));
|
|
245
|
+
const challenge = await browser.detectChallenge(session, exec.signal);
|
|
246
|
+
return {
|
|
247
|
+
blocked: challenge.blocked,
|
|
248
|
+
...challenge.kind !== undefined ? { kind: challenge.kind } : {},
|
|
249
|
+
...challenge.reason !== undefined ? { reason: challenge.reason } : {},
|
|
250
|
+
hint: challenge.blocked
|
|
251
|
+
? 'Ask the human to complete the verification in the shared browser window (the page is visible to them), then re-check with browser_snapshot.'
|
|
252
|
+
: '',
|
|
146
253
|
};
|
|
147
254
|
},
|
|
148
255
|
}));
|
|
@@ -168,10 +275,11 @@ export function apply(ctx, config = {}) {
|
|
|
168
275
|
timeoutMs,
|
|
169
276
|
isConcurrencySafe: () => false, // page JS can be stateful
|
|
170
277
|
async execute(args, exec) {
|
|
278
|
+
assertAllowed('browser_execute');
|
|
171
279
|
const browser = ctx.get('browser');
|
|
172
280
|
if (browser === undefined)
|
|
173
281
|
throw new Error('tool-browser: browser service unavailable');
|
|
174
|
-
const session = await ensureSession(browser);
|
|
282
|
+
const session = await ensureSession(browser, taskKey(exec));
|
|
175
283
|
const result = await browser.execute(session, {
|
|
176
284
|
script: args.script,
|
|
177
285
|
args: args.args ?? [],
|
|
@@ -210,7 +318,7 @@ export function apply(ctx, config = {}) {
|
|
|
210
318
|
const browser = ctx.get('browser');
|
|
211
319
|
if (browser === undefined)
|
|
212
320
|
throw new Error('tool-browser: browser service unavailable');
|
|
213
|
-
const session = await ensureSession(browser);
|
|
321
|
+
const session = await ensureSession(browser, taskKey(exec));
|
|
214
322
|
const result = await browser.content(session, {
|
|
215
323
|
format: args.format,
|
|
216
324
|
...args.selector !== undefined ? { selector: args.selector } : {},
|
|
@@ -220,11 +328,140 @@ export function apply(ctx, config = {}) {
|
|
|
220
328
|
return { content: result.content, truncated: result.truncated };
|
|
221
329
|
},
|
|
222
330
|
}));
|
|
331
|
+
ctx.tools.register(defineTool({
|
|
332
|
+
name: 'browser_click',
|
|
333
|
+
description: 'Click at viewport coordinates (CSS pixels) in the shared browser. Use with browser_screenshot: have a vision model locate an element on the screenshot, then click its coordinates — this covers icons, image buttons, and canvas elements that DOM snapshots cannot target. Coordinates are relative to the visible viewport, same as the screenshot.',
|
|
334
|
+
parameters: {
|
|
335
|
+
x: { type: 'number', required: true, description: 'Viewport x coordinate (CSS px), e.g. from a vision model reading the screenshot.' },
|
|
336
|
+
y: { type: 'number', required: true, description: 'Viewport y coordinate (CSS px).' },
|
|
337
|
+
},
|
|
338
|
+
output: {
|
|
339
|
+
schema: { type: 'object', additionalProperties: false, properties: { clicked: { type: 'boolean', required: true } } },
|
|
340
|
+
render: (_args, value) => [{ type: 'text', text: value.clicked ? 'Clicked.' : 'Click failed.' }],
|
|
341
|
+
},
|
|
342
|
+
timeoutMs,
|
|
343
|
+
isConcurrencySafe: () => false,
|
|
344
|
+
async execute(args, exec) {
|
|
345
|
+
assertAllowed('browser_click');
|
|
346
|
+
const browser = ctx.get('browser');
|
|
347
|
+
if (browser === undefined)
|
|
348
|
+
throw new Error('tool-browser: browser service unavailable');
|
|
349
|
+
const session = await ensureSession(browser, taskKey(exec));
|
|
350
|
+
await browser.click(session, { x: args.x, y: args.y }, exec.signal);
|
|
351
|
+
return { clicked: true };
|
|
352
|
+
},
|
|
353
|
+
}));
|
|
354
|
+
ctx.tools.register(defineTool({
|
|
355
|
+
name: 'browser_type',
|
|
356
|
+
description: 'Type text into the focused element of the shared browser. Use after browser_execute focuses an input (e.g. el.focus()), or after a click lands in a field. Text is inserted at the current focus via CDP Input.insertText.',
|
|
357
|
+
parameters: {
|
|
358
|
+
text: { type: 'string', required: true, description: 'The text to insert.' },
|
|
359
|
+
},
|
|
360
|
+
output: {
|
|
361
|
+
schema: { type: 'object', additionalProperties: false, properties: { typed: { type: 'boolean', required: true } } },
|
|
362
|
+
render: (_args, value) => [{ type: 'text', text: value.typed ? `Typed ${String(_args.text).length} chars.` : 'Type failed.' }],
|
|
363
|
+
},
|
|
364
|
+
timeoutMs,
|
|
365
|
+
isConcurrencySafe: () => false,
|
|
366
|
+
async execute(args, exec) {
|
|
367
|
+
assertAllowed('browser_type');
|
|
368
|
+
const browser = ctx.get('browser');
|
|
369
|
+
if (browser === undefined)
|
|
370
|
+
throw new Error('tool-browser: browser service unavailable');
|
|
371
|
+
const session = await ensureSession(browser, taskKey(exec));
|
|
372
|
+
await browser.type(session, { text: args.text }, exec.signal);
|
|
373
|
+
return { typed: true };
|
|
374
|
+
},
|
|
375
|
+
}));
|
|
376
|
+
ctx.tools.register(defineTool({
|
|
377
|
+
name: 'browser_fill',
|
|
378
|
+
description: 'Fill a form in one batch: pass fields with a CSS selector or name/label/placeholder text and the value to set (string, number, or boolean for checkbox/radio; for selects or radio groups pass the option value or visible text). Values are applied with the native setter plus input/change events, so React/Vue controlled inputs update correctly. Optionally submit the containing form. Prefer this over hand-written browser_execute for form filling; per-field failures are reported instead of throwing.',
|
|
379
|
+
parameters: {
|
|
380
|
+
fields: {
|
|
381
|
+
type: 'array',
|
|
382
|
+
required: true,
|
|
383
|
+
items: {
|
|
384
|
+
type: 'object',
|
|
385
|
+
additionalProperties: true,
|
|
386
|
+
properties: {
|
|
387
|
+
selector: { type: 'string', description: 'CSS selector; when present, candidates are scoped to it.' },
|
|
388
|
+
name: { type: 'string', description: 'Match by the field\'s name attribute.' },
|
|
389
|
+
label: { type: 'string', description: 'Match by associated <label> text or aria-label.' },
|
|
390
|
+
placeholder: { type: 'string', description: 'Match by placeholder text.' },
|
|
391
|
+
kind: { type: 'string', enum: ['text', 'textarea', 'checkbox', 'radio', 'select'], description: 'Field kind; defaults to text.' },
|
|
392
|
+
value: { type: 'string', description: 'Value to set (string form; booleans/numbers accepted as strings).' },
|
|
393
|
+
},
|
|
394
|
+
},
|
|
395
|
+
},
|
|
396
|
+
submit: { type: 'boolean', description: 'Submit the containing form after filling (default false).' },
|
|
397
|
+
},
|
|
398
|
+
output: {
|
|
399
|
+
schema: {
|
|
400
|
+
type: 'object',
|
|
401
|
+
additionalProperties: false,
|
|
402
|
+
properties: {
|
|
403
|
+
fields: {
|
|
404
|
+
type: 'array',
|
|
405
|
+
required: true,
|
|
406
|
+
items: {
|
|
407
|
+
type: 'object',
|
|
408
|
+
additionalProperties: false,
|
|
409
|
+
properties: {
|
|
410
|
+
ok: { type: 'boolean', required: true },
|
|
411
|
+
target: { type: 'string', required: true },
|
|
412
|
+
method: { type: 'string' },
|
|
413
|
+
error: { type: 'string' },
|
|
414
|
+
},
|
|
415
|
+
},
|
|
416
|
+
},
|
|
417
|
+
submitted: { type: 'boolean', required: true },
|
|
418
|
+
},
|
|
419
|
+
},
|
|
420
|
+
render: (_args, value) => [{
|
|
421
|
+
type: 'text',
|
|
422
|
+
text: (() => {
|
|
423
|
+
const fields = value.fields;
|
|
424
|
+
const failed = fields.filter(f => !f.ok);
|
|
425
|
+
const lines = fields.map(f => `${f.ok ? 'OK' : 'FAIL'} ${f.target}${f.ok ? ` (${f.method ?? 'input'})` : `: ${f.error ?? 'unknown error'}`}`);
|
|
426
|
+
const head = failed.length === 0
|
|
427
|
+
? `Filled ${fields.length}/${fields.length} fields${value.submitted ? ' and submitted the form' : ''}.`
|
|
428
|
+
: `Filled ${fields.length - failed.length}/${fields.length} fields; ${failed.length} failed:`;
|
|
429
|
+
return head + '\n' + lines.join('\n');
|
|
430
|
+
})(),
|
|
431
|
+
}],
|
|
432
|
+
},
|
|
433
|
+
timeoutMs,
|
|
434
|
+
isConcurrencySafe: () => false,
|
|
435
|
+
async execute(args, exec) {
|
|
436
|
+
assertAllowed('browser_fill');
|
|
437
|
+
const browser = ctx.get('browser');
|
|
438
|
+
if (browser === undefined)
|
|
439
|
+
throw new Error('tool-browser: browser service unavailable');
|
|
440
|
+
const session = await ensureSession(browser, taskKey(exec));
|
|
441
|
+
const fields = (args.fields ?? []).map((f) => ({
|
|
442
|
+
...f.selector !== undefined ? { selector: f.selector } : {},
|
|
443
|
+
...f.name !== undefined ? { name: f.name } : {},
|
|
444
|
+
...f.label !== undefined ? { label: f.label } : {},
|
|
445
|
+
...f.placeholder !== undefined ? { placeholder: f.placeholder } : {},
|
|
446
|
+
...f.kind !== undefined ? { kind: f.kind } : {},
|
|
447
|
+
value: parseFillValue(f.value),
|
|
448
|
+
}));
|
|
449
|
+
const result = await browser.fillForm(session, {
|
|
450
|
+
fields,
|
|
451
|
+
...args.submit === true ? { submit: true } : {},
|
|
452
|
+
}, exec.signal);
|
|
453
|
+
return {
|
|
454
|
+
fields: result.fields.map(f => ({ ok: f.ok, target: f.target, ...f.method !== undefined ? { method: f.method } : {}, ...f.error !== undefined ? { error: f.error } : {} })),
|
|
455
|
+
submitted: result.submitted,
|
|
456
|
+
};
|
|
457
|
+
},
|
|
458
|
+
}));
|
|
223
459
|
ctx.tools.register(defineTool({
|
|
224
460
|
name: 'browser_screenshot',
|
|
225
|
-
description: 'Capture the current shared-browser page as a PNG screenshot. Use for visual confirmation of layout, charts, designs, or CAPTCHAs
|
|
461
|
+
description: 'Capture the current shared-browser page as a PNG screenshot. Use for visual confirmation of layout, charts, designs, or CAPTCHAs, or to feed a vision tool (read_image) that locates elements visually. Supports optional full-page capture and optional save-to-file (the saved path can be passed to read_image for vision-based element location).',
|
|
226
462
|
parameters: {
|
|
227
463
|
fullPage: { type: 'boolean', description: 'Capture the full scrollable page instead of the viewport (default false).' },
|
|
464
|
+
savePath: { type: 'string', description: 'Absolute file path to also save the PNG to (e.g. for read_image vision location).' },
|
|
228
465
|
},
|
|
229
466
|
output: {
|
|
230
467
|
schema: {
|
|
@@ -232,9 +469,10 @@ export function apply(ctx, config = {}) {
|
|
|
232
469
|
additionalProperties: false,
|
|
233
470
|
properties: {
|
|
234
471
|
dataUrl: { type: 'string', required: true, description: 'Base64 PNG data URL of the screenshot.' },
|
|
472
|
+
path: { type: 'string', description: 'The file path the screenshot was saved to, when savePath was given.' },
|
|
235
473
|
},
|
|
236
474
|
},
|
|
237
|
-
render: (_args, value) => [{ type: 'text', text: `Screenshot captured (${Math.round(value.dataUrl.length * 3 / 4 / 1024)} KiB).
|
|
475
|
+
render: (_args, value) => [{ type: 'text', text: `Screenshot captured (${Math.round(value.dataUrl.length * 3 / 4 / 1024)} KiB)${value.path !== undefined ? ` saved to ${value.path}` : ''}.` }],
|
|
238
476
|
},
|
|
239
477
|
timeoutMs,
|
|
240
478
|
isConcurrencySafe: () => true,
|
|
@@ -242,11 +480,15 @@ export function apply(ctx, config = {}) {
|
|
|
242
480
|
const browser = ctx.get('browser');
|
|
243
481
|
if (browser === undefined)
|
|
244
482
|
throw new Error('tool-browser: browser service unavailable');
|
|
245
|
-
const session = await ensureSession(browser);
|
|
483
|
+
const session = await ensureSession(browser, taskKey(exec));
|
|
246
484
|
const shot = await browser.screenshot(session, {
|
|
247
485
|
...args.fullPage === true ? { fullPage: true } : {},
|
|
486
|
+
...args.savePath !== undefined ? { savePath: args.savePath } : {},
|
|
248
487
|
}, exec.signal);
|
|
249
|
-
return {
|
|
488
|
+
return {
|
|
489
|
+
dataUrl: shot.dataUrl,
|
|
490
|
+
...shot.path !== undefined ? { path: shot.path } : {},
|
|
491
|
+
};
|
|
250
492
|
},
|
|
251
493
|
}));
|
|
252
494
|
if (config.tabTools !== false) {
|
|
@@ -282,11 +524,11 @@ export function apply(ctx, config = {}) {
|
|
|
282
524
|
},
|
|
283
525
|
timeoutMs,
|
|
284
526
|
isConcurrencySafe: () => true,
|
|
285
|
-
async execute(_args,
|
|
527
|
+
async execute(_args, exec) {
|
|
286
528
|
const browser = ctx.get('browser');
|
|
287
529
|
if (browser === undefined)
|
|
288
530
|
throw new Error('tool-browser: browser service unavailable');
|
|
289
|
-
const session = await ensureSession(browser);
|
|
531
|
+
const session = await ensureSession(browser, taskKey(exec));
|
|
290
532
|
const tabs = await browser.listTabs(session);
|
|
291
533
|
return { tabs: tabs.map(t => ({ id: t.id, url: t.url, active: t.active })) };
|
|
292
534
|
},
|
|
@@ -303,11 +545,11 @@ export function apply(ctx, config = {}) {
|
|
|
303
545
|
},
|
|
304
546
|
timeoutMs,
|
|
305
547
|
isConcurrencySafe: () => true,
|
|
306
|
-
async execute(args,
|
|
548
|
+
async execute(args, exec) {
|
|
307
549
|
const browser = ctx.get('browser');
|
|
308
550
|
if (browser === undefined)
|
|
309
551
|
throw new Error('tool-browser: browser service unavailable');
|
|
310
|
-
const session = await ensureSession(browser);
|
|
552
|
+
const session = await ensureSession(browser, taskKey(exec));
|
|
311
553
|
await browser.switchTab(session, args.tabId);
|
|
312
554
|
return { switched: true };
|
|
313
555
|
},
|
|
@@ -324,11 +566,11 @@ export function apply(ctx, config = {}) {
|
|
|
324
566
|
},
|
|
325
567
|
timeoutMs,
|
|
326
568
|
isConcurrencySafe: () => true,
|
|
327
|
-
async execute(args,
|
|
569
|
+
async execute(args, exec) {
|
|
328
570
|
const browser = ctx.get('browser');
|
|
329
571
|
if (browser === undefined)
|
|
330
572
|
throw new Error('tool-browser: browser service unavailable');
|
|
331
|
-
const session = await ensureSession(browser);
|
|
573
|
+
const session = await ensureSession(browser, taskKey(exec));
|
|
332
574
|
await browser.closeTab(session, args.tabId);
|
|
333
575
|
return { closed: true };
|
|
334
576
|
},
|
|
@@ -343,19 +585,252 @@ export function apply(ctx, config = {}) {
|
|
|
343
585
|
},
|
|
344
586
|
timeoutMs,
|
|
345
587
|
isConcurrencySafe: () => true,
|
|
346
|
-
async execute(_args,
|
|
588
|
+
async execute(_args, exec) {
|
|
589
|
+
assertAllowed('browser_reset');
|
|
347
590
|
const browser = ctx.get('browser');
|
|
348
591
|
if (browser === undefined)
|
|
349
592
|
throw new Error('tool-browser: browser service unavailable');
|
|
350
|
-
const session = await ensureSession(browser);
|
|
593
|
+
const session = await ensureSession(browser, taskKey(exec));
|
|
351
594
|
await browser.reset(session);
|
|
352
595
|
return { reset: true };
|
|
353
596
|
},
|
|
354
597
|
}));
|
|
355
598
|
}
|
|
599
|
+
ctx.tools.register(defineTool({
|
|
600
|
+
name: 'browser_history',
|
|
601
|
+
description: 'List the shared browser session\'s recorded operation history (navigate/execute/click/type), newest last, with per-step success/error. Use to understand what the agent did and to pick a step to replay.',
|
|
602
|
+
parameters: {},
|
|
603
|
+
output: {
|
|
604
|
+
schema: {
|
|
605
|
+
type: 'object',
|
|
606
|
+
additionalProperties: false,
|
|
607
|
+
properties: {
|
|
608
|
+
entries: {
|
|
609
|
+
type: 'array',
|
|
610
|
+
required: true,
|
|
611
|
+
items: {
|
|
612
|
+
type: 'object',
|
|
613
|
+
additionalProperties: false,
|
|
614
|
+
properties: {
|
|
615
|
+
seq: { type: 'number', required: true },
|
|
616
|
+
action: { type: 'string', required: true },
|
|
617
|
+
ok: { type: 'boolean', required: true },
|
|
618
|
+
params: { type: 'object', additionalProperties: true, required: true },
|
|
619
|
+
result: { type: 'string' },
|
|
620
|
+
error: { type: 'string' },
|
|
621
|
+
},
|
|
622
|
+
},
|
|
623
|
+
},
|
|
624
|
+
},
|
|
625
|
+
},
|
|
626
|
+
render: (_args, value) => {
|
|
627
|
+
const entries = value.entries;
|
|
628
|
+
if (entries.length === 0)
|
|
629
|
+
return [{ type: 'text', text: '(no recorded operations yet)' }];
|
|
630
|
+
return [{
|
|
631
|
+
type: 'text',
|
|
632
|
+
text: entries.map(e => `#${e.seq} ${e.action} ${e.ok ? 'ok' : 'FAIL'} ${JSON.stringify(e.params)}${e.result !== undefined ? ` -> ${e.result}` : ''}${e.error !== undefined ? ` !! ${e.error}` : ''}`).join('\n'),
|
|
633
|
+
}];
|
|
634
|
+
},
|
|
635
|
+
},
|
|
636
|
+
timeoutMs,
|
|
637
|
+
isConcurrencySafe: () => true,
|
|
638
|
+
async execute(_args, exec) {
|
|
639
|
+
const browser = ctx.get('browser');
|
|
640
|
+
if (browser === undefined)
|
|
641
|
+
throw new Error('tool-browser: browser service unavailable');
|
|
642
|
+
const session = await ensureSession(browser, taskKey(exec));
|
|
643
|
+
const entries = await browser.history(session);
|
|
644
|
+
const rendered = entries.map(e => {
|
|
645
|
+
const row = {
|
|
646
|
+
seq: e.seq,
|
|
647
|
+
action: e.action,
|
|
648
|
+
ok: e.ok,
|
|
649
|
+
params: JSON.parse(JSON.stringify(e.params)),
|
|
650
|
+
};
|
|
651
|
+
if (e.result !== undefined)
|
|
652
|
+
row.result = e.result;
|
|
653
|
+
if (e.error !== undefined)
|
|
654
|
+
row.error = e.error;
|
|
655
|
+
return row;
|
|
656
|
+
});
|
|
657
|
+
return { entries: rendered };
|
|
658
|
+
},
|
|
659
|
+
}));
|
|
660
|
+
ctx.tools.register(defineTool({
|
|
661
|
+
name: 'browser_replay',
|
|
662
|
+
description: 'Replay one recorded browser operation by its history sequence number (from browser_history). Navigate/click/type are re-issued against the current page; execute re-runs its script. The replayed step is appended to history as a new entry.',
|
|
663
|
+
parameters: {
|
|
664
|
+
seq: { type: 'number', required: true, description: 'The history entry sequence number to replay.' },
|
|
665
|
+
},
|
|
666
|
+
output: {
|
|
667
|
+
schema: { type: 'object', additionalProperties: false, properties: { replayed: { type: 'boolean', required: true } } },
|
|
668
|
+
render: (_args, value) => [{ type: 'text', text: value.replayed ? 'Replayed.' : 'Replay failed.' }],
|
|
669
|
+
},
|
|
670
|
+
timeoutMs,
|
|
671
|
+
isConcurrencySafe: () => false,
|
|
672
|
+
async execute(args, exec) {
|
|
673
|
+
assertAllowed('browser_replay');
|
|
674
|
+
const browser = ctx.get('browser');
|
|
675
|
+
if (browser === undefined)
|
|
676
|
+
throw new Error('tool-browser: browser service unavailable');
|
|
677
|
+
const session = await ensureSession(browser, taskKey(exec));
|
|
678
|
+
await browser.replay(session, args.seq);
|
|
679
|
+
return { replayed: true };
|
|
680
|
+
},
|
|
681
|
+
}));
|
|
682
|
+
ctx.tools.register(defineTool({
|
|
683
|
+
name: 'browser_download',
|
|
684
|
+
description: 'Download a URL to a local file, keeping the browser session\'s cookies and login state. Use for fetching files behind authentication or from the current page context. Available on the self-hosted browser; the desktop shell delegates downloads to the real browser UI.',
|
|
685
|
+
parameters: {
|
|
686
|
+
url: { type: 'string', required: true, description: 'The URL to download.' },
|
|
687
|
+
savePath: { type: 'string', required: true, description: 'Absolute path of the file to write.' },
|
|
688
|
+
},
|
|
689
|
+
output: {
|
|
690
|
+
schema: { type: 'object', additionalProperties: false, properties: { path: { type: 'string', required: true } } },
|
|
691
|
+
render: (_args, value) => [{ type: 'text', text: `Downloaded to ${value.path}.` }],
|
|
692
|
+
},
|
|
693
|
+
timeoutMs,
|
|
694
|
+
isConcurrencySafe: () => false,
|
|
695
|
+
async execute(args, exec) {
|
|
696
|
+
assertAllowed('browser_download');
|
|
697
|
+
const browser = ctx.get('browser');
|
|
698
|
+
if (browser === undefined)
|
|
699
|
+
throw new Error('tool-browser: browser service unavailable');
|
|
700
|
+
const session = await ensureSession(browser, taskKey(exec));
|
|
701
|
+
const result = await browser.download(session, { url: args.url, savePath: args.savePath }, exec.signal);
|
|
702
|
+
return { path: result.path };
|
|
703
|
+
},
|
|
704
|
+
}));
|
|
705
|
+
ctx.tools.register(defineTool({
|
|
706
|
+
name: 'browser_session',
|
|
707
|
+
description: 'Show THIS task\'s browser session: its id and open tabs. Each task (DSH session) has its own browser session, so this reflects what your task drives. The window is shared with the human and other tasks, but tab sets and history are isolated per task.',
|
|
708
|
+
parameters: {},
|
|
709
|
+
output: {
|
|
710
|
+
schema: {
|
|
711
|
+
type: 'object',
|
|
712
|
+
additionalProperties: false,
|
|
713
|
+
properties: {
|
|
714
|
+
session: { type: 'string', required: true },
|
|
715
|
+
tabs: {
|
|
716
|
+
type: 'array',
|
|
717
|
+
required: true,
|
|
718
|
+
items: {
|
|
719
|
+
type: 'object',
|
|
720
|
+
additionalProperties: false,
|
|
721
|
+
properties: {
|
|
722
|
+
id: { type: 'string', required: true },
|
|
723
|
+
url: { type: 'string', required: true },
|
|
724
|
+
active: { type: 'boolean', required: true },
|
|
725
|
+
},
|
|
726
|
+
},
|
|
727
|
+
},
|
|
728
|
+
},
|
|
729
|
+
},
|
|
730
|
+
render: (_args, value) => [{
|
|
731
|
+
type: 'text',
|
|
732
|
+
text: `Session ${value.session}\n${value.tabs.map(t => `${t.active ? '*' : ' '} ${t.id} ${t.url}`).join('\n')}`,
|
|
733
|
+
}],
|
|
734
|
+
},
|
|
735
|
+
timeoutMs,
|
|
736
|
+
isConcurrencySafe: () => true,
|
|
737
|
+
async execute(_args, exec) {
|
|
738
|
+
const browser = ctx.get('browser');
|
|
739
|
+
if (browser === undefined)
|
|
740
|
+
throw new Error('tool-browser: browser service unavailable');
|
|
741
|
+
const key = taskKey(exec);
|
|
742
|
+
const session = sessionsByTask.get(key) ?? await browser.open();
|
|
743
|
+
sessionsByTask.set(key, session);
|
|
744
|
+
const tabs = await browser.listTabs(session);
|
|
745
|
+
return { session, tabs: tabs.map(t => ({ id: t.id, url: t.url, active: t.active })) };
|
|
746
|
+
},
|
|
747
|
+
}));
|
|
748
|
+
ctx.tools.register(defineTool({
|
|
749
|
+
name: 'browser_reset_session',
|
|
750
|
+
description: 'Reset THIS task\'s browser session: close it entirely so the next browser_* call starts a fresh session with one blank tab. Other tasks\' sessions are untouched. Use when a session is in a bad state or you want a clean slate.',
|
|
751
|
+
parameters: {},
|
|
752
|
+
output: {
|
|
753
|
+
schema: { type: 'object', additionalProperties: false, properties: { reset: { type: 'boolean', required: true } } },
|
|
754
|
+
render: (_args, value) => [{ type: 'text', text: value.reset ? 'This task\'s browser session was closed; the next call starts fresh.' : 'Failed.' }],
|
|
755
|
+
},
|
|
756
|
+
timeoutMs,
|
|
757
|
+
isConcurrencySafe: () => true,
|
|
758
|
+
async execute(_args, exec) {
|
|
759
|
+
assertAllowed('browser_reset_session');
|
|
760
|
+
const browser = ctx.get('browser');
|
|
761
|
+
if (browser === undefined)
|
|
762
|
+
throw new Error('tool-browser: browser service unavailable');
|
|
763
|
+
const key = taskKey(exec);
|
|
764
|
+
const session = sessionsByTask.get(key);
|
|
765
|
+
if (session !== undefined) {
|
|
766
|
+
await browser.close(session);
|
|
767
|
+
sessionsByTask.delete(key);
|
|
768
|
+
}
|
|
769
|
+
return { reset: true };
|
|
770
|
+
},
|
|
771
|
+
}));
|
|
772
|
+
ctx.tools.register(defineTool({
|
|
773
|
+
name: 'browser_restrict',
|
|
774
|
+
description: 'Restrict which browser actions are allowed, to prevent stray clicks/navigation. Pass a list of browser tool names (e.g. ["browser_snapshot","browser_content","browser_click"]) — any other browser_* call is refused. Pass an empty list or omit to lift the restriction. Read-only tools (snapshot/content/screenshot/session) are never blocked.',
|
|
775
|
+
parameters: {
|
|
776
|
+
allowed: {
|
|
777
|
+
type: 'array',
|
|
778
|
+
items: { type: 'string' },
|
|
779
|
+
description: 'Allow-list of browser tool names; empty clears the restriction.',
|
|
780
|
+
},
|
|
781
|
+
},
|
|
782
|
+
output: {
|
|
783
|
+
schema: { type: 'object', additionalProperties: false, properties: { restrictedTo: { type: 'array', required: true, items: { type: 'string' } } } },
|
|
784
|
+
render: (_args, value) => [{ type: 'text', text: value.restrictedTo.length > 0 ? `Restricted to: ${value.restrictedTo.join(', ')}` : 'Restriction lifted.' }],
|
|
785
|
+
},
|
|
786
|
+
timeoutMs,
|
|
787
|
+
isConcurrencySafe: () => false,
|
|
788
|
+
async execute(args, exec) {
|
|
789
|
+
// Always allowed so the guard can be lifted.
|
|
790
|
+
restrictedTo = (args.allowed ?? []).filter((t) => t.startsWith('browser_'));
|
|
791
|
+
return { restrictedTo: [...restrictedTo] };
|
|
792
|
+
},
|
|
793
|
+
}));
|
|
794
|
+
ctx.tools.register(defineTool({
|
|
795
|
+
name: 'browser_auth',
|
|
796
|
+
description: 'Export or restore the browser session\'s cookies (login state). Use "flush" to get a JSON cookie list (save it to a file to persist logins), or "restore" with that list to put logins back (e.g. after the browser host restarted). Available on the self-hosted browser.',
|
|
797
|
+
parameters: {
|
|
798
|
+
action: { type: 'string', required: true, enum: ['flush', 'restore'], description: 'flush = export cookies; restore = import cookies.' },
|
|
799
|
+
cookies: { type: 'array', items: { type: 'object', additionalProperties: true }, description: 'Cookie list to restore (required when action=restore).' },
|
|
800
|
+
},
|
|
801
|
+
output: {
|
|
802
|
+
schema: {
|
|
803
|
+
type: 'object',
|
|
804
|
+
additionalProperties: false,
|
|
805
|
+
properties: {
|
|
806
|
+
cookies: { type: 'array', items: { type: 'object', additionalProperties: true } },
|
|
807
|
+
restored: { type: 'number' },
|
|
808
|
+
},
|
|
809
|
+
},
|
|
810
|
+
render: (_args, value) => [{ type: 'text', text: value.cookies !== undefined ? `Exported ${value.cookies.length} cookies.` : `Restored ${value.restored} cookies.` }],
|
|
811
|
+
},
|
|
812
|
+
timeoutMs,
|
|
813
|
+
isConcurrencySafe: () => false,
|
|
814
|
+
async execute(args, exec) {
|
|
815
|
+
assertAllowed('browser_auth');
|
|
816
|
+
const browser = ctx.get('browser');
|
|
817
|
+
if (browser === undefined)
|
|
818
|
+
throw new Error('tool-browser: browser service unavailable');
|
|
819
|
+
const session = await ensureSession(browser, taskKey(exec));
|
|
820
|
+
if (args.action === 'flush') {
|
|
821
|
+
const cookies = await browser.flushAuth(session);
|
|
822
|
+
return { cookies: cookies.map(c => ({ ...c })) };
|
|
823
|
+
}
|
|
824
|
+
const list = (args.cookies ?? []);
|
|
825
|
+
const restored = await browser.restoreAuth(session, list);
|
|
826
|
+
return { restored };
|
|
827
|
+
},
|
|
828
|
+
}));
|
|
356
829
|
}
|
|
357
|
-
/** Test hook:
|
|
830
|
+
/** Test hook: inspect and reset the plugin-level session map (used by tests). */
|
|
358
831
|
export const internals = {
|
|
359
|
-
|
|
360
|
-
|
|
832
|
+
/** The per-task session map (task key -> provider session id). */
|
|
833
|
+
get sessions() { return sessionsByTask; },
|
|
834
|
+
/** Drop one task's mapping without closing the provider session. */
|
|
835
|
+
clearSession(key = 'default') { sessionsByTask.delete(key); },
|
|
361
836
|
};
|