dsh-builtin-browser 0.4.7 → 0.4.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +74 -0
- package/README.en.md +62 -491
- package/README.md +44 -470
- package/client.js +10 -10
- package/cordis.patch.yml +4 -4
- package/desktop-bridge/plugin-browser-bridge.js +1 -1
- package/docs/README.md +8 -4
- package/docs/architecture.md +82 -11
- package/docs/development.md +38 -0
- package/docs/limitations.md +49 -0
- package/docs/tool-reference.md +42 -42
- package/docs/user-guide.md +93 -37
- package/docs/why-browser.md +3 -3
- package/lib/browser/types.d.ts +2 -2
- package/lib/browser-electron/entry.d.ts +2 -2
- package/lib/browser-electron/entry.js +1 -1
- package/lib/browser-electron/history-store.d.ts +1 -1
- package/lib/browser-electron/history-store.js +1 -1
- package/lib/browser-electron/host-main.js +3 -3
- package/lib/browser-electron/provider.d.ts +9 -9
- package/lib/browser-electron/provider.js +14 -14
- package/lib/browser-electron/remote-host.js +1 -1
- package/lib/browser-electron/settings-store.d.ts +4 -4
- package/lib/browser-electron/system-browser.d.ts +60 -0
- package/lib/browser-electron/system-browser.js +257 -23
- package/lib/index.d.ts +1 -1
- package/lib/index.js +1 -1
- package/lib/tool-browser/element-fields.d.ts +4 -4
- package/lib/tool-browser/element-fields.js +6 -6
- package/lib/tool-browser/index.d.ts +6 -6
- package/lib/tool-browser/index.js +161 -139
- package/package.json +2 -2
- package/scripts/verify-host-compat.mjs +6 -6
- package/src/browser/types.ts +2 -2
- package/src/browser-electron/entry.ts +2 -2
- package/src/browser-electron/history-store.ts +1 -1
- package/src/browser-electron/host-main.ts +3 -3
- package/src/browser-electron/provider.ts +15 -15
- package/src/browser-electron/remote-host.ts +1 -1
- package/src/browser-electron/settings-store.ts +4 -4
- package/src/browser-electron/system-browser.ts +244 -16
- package/src/index.ts +1 -1
- package/src/tool-browser/element-fields.ts +6 -6
- package/src/tool-browser/index.ts +164 -140
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Model-facing browser tools over `ctx.browser`: `
|
|
3
|
-
* `
|
|
4
|
-
* `
|
|
5
|
-
* `
|
|
2
|
+
* Model-facing browser tools over `ctx.browser`: `dsh_browser_open`,
|
|
3
|
+
* `dsh_browser_snapshot`, `dsh_browser_execute`, `dsh_browser_content`,
|
|
4
|
+
* `dsh_browser_screenshot`, and tab management (`dsh_browser_list_tabs`,
|
|
5
|
+
* `dsh_browser_switch_tab`, `dsh_browser_close_tab`, `dsh_browser_reset`).
|
|
6
6
|
*
|
|
7
7
|
* The tool layer owns only the model-facing schema, argument validation, and
|
|
8
8
|
* result formatting — never provider selection or page driving, which belong
|
|
9
9
|
* to the seam. Session lifecycle is owned here at the plugin level: each
|
|
10
10
|
* calling task (a DSH session) gets its own browser session — the first
|
|
11
|
-
* `
|
|
11
|
+
* `dsh_browser_open` (or any tool when no session exists) opens it, and later
|
|
12
12
|
* tools in the same task reuse it. Concurrent tasks therefore never fight
|
|
13
13
|
* over tabs, history, or navigation state.
|
|
14
14
|
* @module dsh-browser/tool-browser
|
|
@@ -72,7 +72,7 @@ interface ToolBrowserState {
|
|
|
72
72
|
*
|
|
73
73
|
* Two different things share this policy. A restriction from `cordis.patch.yml` is an
|
|
74
74
|
* operator decision about the whole plugin, so it applies to everyone and lives in
|
|
75
|
-
* `configRestrictedTo`. One set by `
|
|
75
|
+
* `configRestrictedTo`. One set by `dsh_browser_restrict` is a decision by one task about
|
|
76
76
|
* its own work, so it is keyed by task — a single shared value let one task lock the
|
|
77
77
|
* browser tools of every other task, which is not what any caller asks for. Sessions
|
|
78
78
|
* beside it were already keyed this way; this was not.
|
|
@@ -98,33 +98,33 @@ const liveStates = new Set<ToolBrowserState>()
|
|
|
98
98
|
* Tools that only observe, plus the ones that undo a restriction.
|
|
99
99
|
*
|
|
100
100
|
* Never subject to an allow-list: restricting what the agent may *do* must not also
|
|
101
|
-
* blind it or trap it, and the tool descriptions promise as much.
|
|
101
|
+
* blind it or trap it, and the tool descriptions promise as much. dsh_browser_restrict
|
|
102
102
|
* is in here so a task that restricted everything can still lift it.
|
|
103
103
|
*/
|
|
104
104
|
export const READ_ONLY_TOOLS: ReadonlySet<string> = new Set([
|
|
105
105
|
// Tools that only observe.
|
|
106
|
-
'
|
|
107
|
-
'
|
|
108
|
-
'
|
|
109
|
-
//
|
|
106
|
+
'dsh_browser_snapshot', 'dsh_browser_a11y', 'dsh_browser_content', 'dsh_browser_scrape',
|
|
107
|
+
'dsh_browser_screenshot', 'dsh_browser_get_value', 'dsh_browser_wait', 'dsh_browser_challenge',
|
|
108
|
+
'dsh_browser_list_tabs', 'dsh_browser_session', 'dsh_browser_history', 'dsh_browser_visited',
|
|
109
|
+
// dsh_browser_auth reads cookies out of the browser: an observation, and one the
|
|
110
110
|
// credentials switch gates on its own (see credentials.allowRead in the settings).
|
|
111
|
-
//
|
|
111
|
+
// dsh_browser_auth is deliberately NOT here. It reads cookies on "flush" but WRITES them on
|
|
112
112
|
// "restore", to arbitrary domains, which is an action like any other — exempting the tool
|
|
113
113
|
// would let a task that restricted its own actions rewrite the shared browser's logins.
|
|
114
114
|
//
|
|
115
115
|
// Getting out of a bad state, or lifting the restriction itself. Without these a task
|
|
116
116
|
// that restricted everything could neither recover nor be released.
|
|
117
|
-
'
|
|
117
|
+
'dsh_browser_restrict', 'dsh_browser_reset_session', 'dsh_browser_reset',
|
|
118
118
|
])
|
|
119
119
|
function assertAllowed(state: ToolBrowserState, toolName: string, task: string): void {
|
|
120
|
-
//
|
|
120
|
+
// dsh_browser_restrict must stay usable, or a task that restricted everything
|
|
121
121
|
// could never lift it again.
|
|
122
122
|
// An allow-list is a statement about ACTIONS, so it must not take away the ability to
|
|
123
123
|
// look, or to get out of a bad state. The tool description and both READMEs promise
|
|
124
124
|
// that read-only tools are never blocked; before this they were, because only
|
|
125
|
-
//
|
|
126
|
-
//
|
|
127
|
-
if (READ_ONLY_TOOLS.has(toolName) || toolName === '
|
|
125
|
+
// dsh_browser_restrict itself was exempt — a narrow list then locked dsh_browser_history,
|
|
126
|
+
// dsh_browser_reset_session and dsh_browser_auth away with no way back.
|
|
127
|
+
if (READ_ONLY_TOOLS.has(toolName) || toolName === 'dsh_browser_restrict') return
|
|
128
128
|
const config = state.configRestrictedTo
|
|
129
129
|
if (config !== undefined && !config.includes(toolName)) {
|
|
130
130
|
throw new Error(`browser action "${toolName}" is restricted by the plugin configuration (allow-list: ${config.join(', ')})`)
|
|
@@ -172,7 +172,7 @@ function assertNotBareDshWebUrl(rawUrl: string): void {
|
|
|
172
172
|
// The authenticated URL printed by `dsh web` carries a token; the bare root does not.
|
|
173
173
|
if (target.searchParams.has('token')) return
|
|
174
174
|
throw new Error(
|
|
175
|
-
'
|
|
175
|
+
'dsh_browser_open: refusing the bare DSH web root (' + dsh.origin + '/) because it carries no '
|
|
176
176
|
+ 'authentication token, so it answers with a 401 page that reads like a real site. Open the '
|
|
177
177
|
+ 'target site instead; to open DSH itself, use the authenticated URL printed by `dsh web`.',
|
|
178
178
|
)
|
|
@@ -314,17 +314,17 @@ function formatSnapshot(snapshot: {
|
|
|
314
314
|
export function apply(ctx: Context, config: Config = {}): void {
|
|
315
315
|
const timeoutMs = config.timeoutMs ?? 60_000
|
|
316
316
|
/**
|
|
317
|
-
* Longest wait `
|
|
317
|
+
* Longest wait `dsh_browser_wait` may poll for, leaving room for the tool call itself to
|
|
318
318
|
* finish and report.
|
|
319
319
|
*
|
|
320
|
-
* `
|
|
320
|
+
* `dsh_browser_wait` promises a VERDICT — `{ready:false, reason}` — and the enclosing tool
|
|
321
321
|
* call aborts at `timeoutMs`. A caller-supplied budget longer than that aborted the call
|
|
322
322
|
* instead, so the documented verdict never arrived. Clamping to just inside the budget is
|
|
323
323
|
* what keeps the promise: the polling stops, the reason is returned.
|
|
324
324
|
*/
|
|
325
325
|
const maxWaitMs = Math.max(250, timeoutMs - 5_000)
|
|
326
326
|
/**
|
|
327
|
-
* Most fields `
|
|
327
|
+
* Most fields `dsh_browser_fill` accepts in one batch.
|
|
328
328
|
*
|
|
329
329
|
* The batch runs as ONE page evaluation, so hundreds of fields do not produce hundreds of
|
|
330
330
|
* per-field verdicts — they produce a single evaluate that exceeds the tool budget and
|
|
@@ -346,11 +346,35 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
346
346
|
// Tool guidance band is 100-199; 150 keeps clear of the common 110/120
|
|
347
347
|
// tool sections so ordering does not depend on plugin load sequence.
|
|
348
348
|
order: 150,
|
|
349
|
-
text: 'Use the
|
|
349
|
+
text: 'Use the dsh_browser_* tools to operate the built-in browser. Each task gets its own browser session AND its own window (with a real toolbar: address bar, back/forward/reload, tab strip) — the human can see it, use it like any browser, and take over at any time; your tabs and history are isolated from other tasks, so do not assume another task\'s navigation state is visible to you. Understand a page with dsh_browser_a11y (semantic roles/names/states — the best structure map) or dsh_browser_snapshot (numbered interactive elements), then drive it: dsh_browser_click/dsh_browser_type accept a target {by: css|text|xpath, value} for semantic locating, or coordinates from dsh_browser_screenshot for visual targeting. For form filling prefer dsh_browser_fill (batch) or dsh_browser_set_value/dsh_browser_check/dsh_browser_select/dsh_browser_clear (single control, target-based); verify with dsh_browser_get_value. Use dsh_browser_scrape for structured extraction from list pages instead of hand-written dsh_browser_execute. After navigating on slow sites, dsh_browser_wait for the page. Keep the human informed of what you are doing on the page. If a snapshot or dsh_browser_challenge reports a human-verification challenge (CAPTCHA), stop retrying and ask the human to complete it in the browser window, then re-check.',
|
|
350
350
|
})
|
|
351
351
|
|
|
352
|
-
|
|
353
|
-
|
|
352
|
+
/**
|
|
353
|
+
* Register one tool, turning a name collision into a warning instead of a dead apply.
|
|
354
|
+
*
|
|
355
|
+
* Registering the tools in sequence without this is how ONE taken name becomes thirty-two
|
|
356
|
+
* missing ones: the throw aborts the rest of the apply, and what the user sees is a plugin that
|
|
357
|
+
* half-exists. That is precisely the shape of the failure reported against a sibling plugin,
|
|
358
|
+
* where the first spec collided and all six vanished behind a `console.warn` a desktop user
|
|
359
|
+
* never sees.
|
|
360
|
+
*
|
|
361
|
+
* A name this plugin cannot have is worth a sentence in the log, not the toolset.
|
|
362
|
+
* @param definition - the tool to register.
|
|
363
|
+
*/
|
|
364
|
+
const registerSafely = (definition: ReturnType<typeof defineTool>): void => {
|
|
365
|
+
try {
|
|
366
|
+
ctx.tools.register(definition)
|
|
367
|
+
} catch (error) {
|
|
368
|
+
ctx.logger?.warn?.(
|
|
369
|
+
`dsh-builtin-browser: could not register ${definition.name} (${String(error)}). Another `
|
|
370
|
+
+ 'plugin has probably taken that name — browser tool names are a shared namespace. The '
|
|
371
|
+
+ 'rest of this plugin\'s tools are unaffected.',
|
|
372
|
+
)
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
registerSafely(defineTool({
|
|
377
|
+
name: 'dsh_browser_open',
|
|
354
378
|
description: 'Open a URL in the shared browser window. Opens this task\'s browser session on first use; optionally opens in a new tab. Returns the resulting page snapshot. A fresh session starts on a blank, uncommitted tab: call this with the REAL target URL before reading, because reading that tab returns an empty page instead of waiting. Never treat the bare DSH host (e.g. http://127.0.0.1:3080/) as the target website — it requires the authenticated URL printed by `dsh web`, and the bare root is refused outright.',
|
|
355
379
|
parameters: {
|
|
356
380
|
url: { type: 'string', required: true, description: 'The URL to open (HTTP/HTTPS).' },
|
|
@@ -367,7 +391,7 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
367
391
|
elements: {
|
|
368
392
|
type: 'array',
|
|
369
393
|
required: true,
|
|
370
|
-
// Same shared item schema as
|
|
394
|
+
// Same shared item schema as dsh_browser_open's, so the two cannot drift apart.
|
|
371
395
|
items: ELEMENT_ITEM_SCHEMA,
|
|
372
396
|
},
|
|
373
397
|
challenge: {
|
|
@@ -381,15 +405,15 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
381
405
|
},
|
|
382
406
|
},
|
|
383
407
|
},
|
|
384
|
-
//
|
|
385
|
-
// `coords` parameter, so coordinates stay off here (
|
|
408
|
+
// dsh_browser_open is an overview of what just loaded, not a locator: it has no
|
|
409
|
+
// `coords` parameter, so coordinates stay off here (dsh_browser_snapshot owns
|
|
386
410
|
// that switch for callers that actually target pixels).
|
|
387
411
|
render: (args, value) => [{ type: 'text', text: formatSnapshot(value, {}) }],
|
|
388
412
|
},
|
|
389
413
|
timeoutMs,
|
|
390
414
|
isConcurrencySafe: () => false, // opens tabs / navigates; exclusive within a task
|
|
391
415
|
async execute(args, exec) {
|
|
392
|
-
assertAllowed(state, '
|
|
416
|
+
assertAllowed(state, 'dsh_browser_open', taskKey(exec))
|
|
393
417
|
assertNotBareDshWebUrl(args.url)
|
|
394
418
|
const browser = ctx.get('browser')
|
|
395
419
|
if (browser === undefined) throw new Error('tool-browser: browser service unavailable')
|
|
@@ -409,9 +433,9 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
409
433
|
},
|
|
410
434
|
}))
|
|
411
435
|
|
|
412
|
-
|
|
413
|
-
name: '
|
|
414
|
-
description: 'Return an AI-friendly snapshot of the current shared-browser page: numbered interactive elements (inputs, buttons, links) the model can cite. This is the primary way to understand a page, and it needs no image input — every listed element can afterwards be addressed by a semantic target (
|
|
436
|
+
registerSafely(defineTool({
|
|
437
|
+
name: 'dsh_browser_snapshot',
|
|
438
|
+
description: 'Return an AI-friendly snapshot of the current shared-browser page: numbered interactive elements (inputs, buttons, links) the model can cite. This is the primary way to understand a page, and it needs no image input — every listed element can afterwards be addressed by a semantic target (dsh_browser_click, dsh_browser_type, dsh_browser_check, …) rather than by coordinates.',
|
|
415
439
|
parameters: {
|
|
416
440
|
coords: { type: 'boolean', description: 'Also print each element\'s viewport coordinates. Off by default; only useful when a vision pass is going to click a pixel position.' },
|
|
417
441
|
},
|
|
@@ -475,9 +499,9 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
475
499
|
},
|
|
476
500
|
}))
|
|
477
501
|
|
|
478
|
-
|
|
479
|
-
name: '
|
|
480
|
-
description: 'Read the page\'s accessibility tree: every interactive node with its semantic role (button/link/textbox/checkbox/…), accessible name, current value, and states (enabled/disabled/checked/expanded/…), (coordinates only when coords: true). Prefer this over
|
|
502
|
+
registerSafely(defineTool({
|
|
503
|
+
name: 'dsh_browser_a11y',
|
|
504
|
+
description: 'Read the page\'s accessibility tree: every interactive node with its semantic role (button/link/textbox/checkbox/…), accessible name, current value, and states (enabled/disabled/checked/expanded/…), (coordinates only when coords: true). Prefer this over dsh_browser_snapshot to understand a page\'s structure and find the right element: roles and names tell you WHAT each node is, and the name is what dsh_browser_click matches when you target by text. Penetrates same-origin iframes and shadow roots.',
|
|
481
505
|
parameters: {
|
|
482
506
|
includeHidden: { type: 'boolean', description: 'Include hidden elements (default false).' },
|
|
483
507
|
maxNodes: { type: 'number', description: 'Maximum nodes (default 150, range 10-5000).' },
|
|
@@ -560,9 +584,9 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
560
584
|
},
|
|
561
585
|
}))
|
|
562
586
|
|
|
563
|
-
|
|
564
|
-
name: '
|
|
565
|
-
description: 'Check whether a human-verification challenge (CAPTCHA / bot detection: Cloudflare "Just a moment", reCAPTCHA, hCaptcha, Turnstile) is blocking the current page. When blocked, do NOT keep retrying automated steps — ask the human to complete the verification in the shared browser window, then re-check with
|
|
587
|
+
registerSafely(defineTool({
|
|
588
|
+
name: 'dsh_browser_challenge',
|
|
589
|
+
description: 'Check whether a human-verification challenge (CAPTCHA / bot detection: Cloudflare "Just a moment", reCAPTCHA, hCaptcha, Turnstile) is blocking the current page. When blocked, do NOT keep retrying automated steps — ask the human to complete the verification in the shared browser window, then re-check with dsh_browser_snapshot.',
|
|
566
590
|
parameters: {},
|
|
567
591
|
output: {
|
|
568
592
|
schema: {
|
|
@@ -594,15 +618,15 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
594
618
|
...challenge.kind !== undefined ? { kind: challenge.kind } : {},
|
|
595
619
|
...challenge.reason !== undefined ? { reason: challenge.reason } : {},
|
|
596
620
|
hint: challenge.blocked
|
|
597
|
-
? 'Ask the human to complete the verification in the shared browser window (the page is visible to them), then re-check with
|
|
621
|
+
? 'Ask the human to complete the verification in the shared browser window (the page is visible to them), then re-check with dsh_browser_snapshot.'
|
|
598
622
|
: '',
|
|
599
623
|
}
|
|
600
624
|
},
|
|
601
625
|
}))
|
|
602
626
|
|
|
603
|
-
|
|
604
|
-
name: '
|
|
605
|
-
description: 'Wait until the shared-browser page is ready: load complete, and optionally the expected URL and/or a CSS selector present (top document or same-origin iframes). Use after
|
|
627
|
+
registerSafely(defineTool({
|
|
628
|
+
name: 'dsh_browser_wait',
|
|
629
|
+
description: 'Wait until the shared-browser page is ready: load complete, and optionally the expected URL and/or a CSS selector present (top document or same-origin iframes). Use after dsh_browser_open on slow sites instead of snapshotting a white page — wait for the URL you navigated to first. Returns ready=true/false with a short reason; a miss is not an error.',
|
|
606
630
|
parameters: {
|
|
607
631
|
timeoutMs: { type: 'number', description: 'Maximum wait in ms (default 30000).' },
|
|
608
632
|
url: { type: 'string', description: 'Expected page URL (exact or prefix), e.g. the URL you opened.' },
|
|
@@ -636,8 +660,8 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
636
660
|
},
|
|
637
661
|
}))
|
|
638
662
|
|
|
639
|
-
|
|
640
|
-
name: '
|
|
663
|
+
registerSafely(defineTool({
|
|
664
|
+
name: 'dsh_browser_execute',
|
|
641
665
|
description: 'Execute JavaScript in the shared-browser page context. This is the primary way to interact with page elements: focus, fill inputs (use the native value setter for framework-controlled inputs, then dispatch an input event), click buttons (element.click() or a constructed MouseEvent). Returns the evaluation result by value, or the exception text.',
|
|
642
666
|
parameters: {
|
|
643
667
|
script: { type: 'string', required: true, description: 'The JavaScript expression to evaluate in the page context.' },
|
|
@@ -661,7 +685,7 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
661
685
|
const rendered = String(value.value ?? '')
|
|
662
686
|
const cap = 50_000
|
|
663
687
|
const text = rendered.length > cap
|
|
664
|
-
? `Result: ${rendered.slice(0, cap)}\n(result truncated at ${cap} of ${rendered.length} characters — return a smaller value, or write it with
|
|
688
|
+
? `Result: ${rendered.slice(0, cap)}\n(result truncated at ${cap} of ${rendered.length} characters — return a smaller value, or write it with dsh_browser_execute + a variable)`
|
|
665
689
|
: `Result: ${rendered}`
|
|
666
690
|
return [{ type: 'text', text }]
|
|
667
691
|
},
|
|
@@ -669,7 +693,7 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
669
693
|
timeoutMs,
|
|
670
694
|
isConcurrencySafe: () => false, // page JS can be stateful
|
|
671
695
|
async execute(args, exec) {
|
|
672
|
-
assertAllowed(state, '
|
|
696
|
+
assertAllowed(state, 'dsh_browser_execute', taskKey(exec))
|
|
673
697
|
const browser = ctx.get('browser')
|
|
674
698
|
if (browser === undefined) throw new Error('tool-browser: browser service unavailable')
|
|
675
699
|
const session = await ensureSession(browser, state, taskKey(exec), agentOf(exec))
|
|
@@ -686,8 +710,8 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
686
710
|
},
|
|
687
711
|
}))
|
|
688
712
|
|
|
689
|
-
|
|
690
|
-
name: '
|
|
713
|
+
registerSafely(defineTool({
|
|
714
|
+
name: 'dsh_browser_content',
|
|
691
715
|
description: 'Fetch the current shared-browser page content in a chosen format: html (raw DOM), markdown (structured reading), txt (plain text), or json. Optionally scope to a CSS selector and cap the length. Use this to read page content, not to interact.',
|
|
692
716
|
parameters: {
|
|
693
717
|
format: { type: 'string', required: true, enum: ['html', 'markdown', 'txt', 'json'], description: 'Output format.' },
|
|
@@ -722,9 +746,9 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
722
746
|
},
|
|
723
747
|
}))
|
|
724
748
|
|
|
725
|
-
|
|
726
|
-
name: '
|
|
727
|
-
description: 'Extract structured data from a page: give the container CSS selector (item) and a field map (name -> selector, optionally selector@attr to take an attribute; a@href yields the absolute URL). Returns one object per item. Static CSS queries only — no arbitrary code runs — so it is safe on any page. Use for list pages (search results, cards, tables) instead of hand-writing
|
|
749
|
+
registerSafely(defineTool({
|
|
750
|
+
name: 'dsh_browser_scrape',
|
|
751
|
+
description: 'Extract structured data from a page: give the container CSS selector (item) and a field map (name -> selector, optionally selector@attr to take an attribute; a@href yields the absolute URL). Returns one object per item. Static CSS queries only — no arbitrary code runs — so it is safe on any page. Use for list pages (search results, cards, tables) instead of hand-writing dsh_browser_execute.',
|
|
728
752
|
parameters: {
|
|
729
753
|
item: { type: 'string', required: true, description: 'CSS selector of each result container (e.g. "div.card").' },
|
|
730
754
|
fields: {
|
|
@@ -759,7 +783,7 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
759
783
|
timeoutMs,
|
|
760
784
|
isConcurrencySafe: () => true,
|
|
761
785
|
async execute(args, exec) {
|
|
762
|
-
assertAllowed(state, '
|
|
786
|
+
assertAllowed(state, 'dsh_browser_scrape', taskKey(exec))
|
|
763
787
|
const browser = ctx.get('browser')
|
|
764
788
|
if (browser === undefined) throw new Error('tool-browser: browser service unavailable')
|
|
765
789
|
const session = await ensureSession(browser, state, taskKey(exec), agentOf(exec))
|
|
@@ -773,9 +797,9 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
773
797
|
},
|
|
774
798
|
}))
|
|
775
799
|
|
|
776
|
-
|
|
777
|
-
name: '
|
|
778
|
-
description: 'Click in the shared browser. Prefer the semantic target: pass target {by: css|text|xpath, value, index?} and the element is located from the DOM, scrolled into view and clicked at its centre — this needs no image input and is the only form that works without vision. The alternative is viewport coordinates (x/y), which are meaningful only after a vision model has located the element on a
|
|
800
|
+
registerSafely(defineTool({
|
|
801
|
+
name: 'dsh_browser_click',
|
|
802
|
+
description: 'Click in the shared browser. Prefer the semantic target: pass target {by: css|text|xpath, value, index?} and the element is located from the DOM, scrolled into view and clicked at its centre — this needs no image input and is the only form that works without vision. The alternative is viewport coordinates (x/y), which are meaningful only after a vision model has located the element on a dsh_browser_screenshot; coordinate clicks are refused outright under the non-visual strategy. Provide exactly one of target or x/y.',
|
|
779
803
|
parameters: {
|
|
780
804
|
target: {
|
|
781
805
|
type: 'object',
|
|
@@ -787,7 +811,7 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
787
811
|
},
|
|
788
812
|
description: 'Locate the element semantically and click it (css/text/xpath).',
|
|
789
813
|
},
|
|
790
|
-
x: { type: 'number', description: 'Viewport x coordinate (CSS px), taken from a
|
|
814
|
+
x: { type: 'number', description: 'Viewport x coordinate (CSS px), taken from a dsh_browser_screenshot read by a vision model. Prefer a semantic target — it works without image input.' },
|
|
791
815
|
y: { type: 'number', description: 'Viewport y coordinate (CSS px), taken from the same screenshot as x.' },
|
|
792
816
|
},
|
|
793
817
|
output: {
|
|
@@ -797,7 +821,7 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
797
821
|
timeoutMs,
|
|
798
822
|
isConcurrencySafe: () => false,
|
|
799
823
|
async execute(args, exec) {
|
|
800
|
-
assertAllowed(state, '
|
|
824
|
+
assertAllowed(state, 'dsh_browser_click', taskKey(exec))
|
|
801
825
|
const browser = ctx.get('browser')
|
|
802
826
|
if (browser === undefined) throw new Error('tool-browser: browser service unavailable')
|
|
803
827
|
const session = await ensureSession(browser, state, taskKey(exec), agentOf(exec))
|
|
@@ -819,7 +843,7 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
819
843
|
}, exec.signal)
|
|
820
844
|
} else {
|
|
821
845
|
if (typeof args.x !== 'number' || typeof args.y !== 'number') {
|
|
822
|
-
throw new Error('
|
|
846
|
+
throw new Error('dsh_browser_click: provide a target (css/text/xpath) or x/y coordinates')
|
|
823
847
|
}
|
|
824
848
|
await browser.click(session, { x: args.x, y: args.y }, exec.signal)
|
|
825
849
|
}
|
|
@@ -827,9 +851,9 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
827
851
|
},
|
|
828
852
|
}))
|
|
829
853
|
|
|
830
|
-
|
|
831
|
-
name: '
|
|
832
|
-
description: 'Type text into the focused element of the shared browser, or into a located element (pass target {by: css|text|xpath, value, index?} to focus it first). Text is inserted at the focus via CDP Input.insertText. For setting whole field values (and React-controlled inputs) prefer
|
|
854
|
+
registerSafely(defineTool({
|
|
855
|
+
name: 'dsh_browser_type',
|
|
856
|
+
description: 'Type text into the focused element of the shared browser, or into a located element (pass target {by: css|text|xpath, value, index?} to focus it first). Text is inserted at the focus via CDP Input.insertText. For setting whole field values (and React-controlled inputs) prefer dsh_browser_set_value; use dsh_browser_key for Enter/Tab/arrows.',
|
|
833
857
|
parameters: {
|
|
834
858
|
text: { type: 'string', required: true, description: 'The text to insert.' },
|
|
835
859
|
target: {
|
|
@@ -850,7 +874,7 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
850
874
|
timeoutMs,
|
|
851
875
|
isConcurrencySafe: () => false,
|
|
852
876
|
async execute(args, exec) {
|
|
853
|
-
assertAllowed(state, '
|
|
877
|
+
assertAllowed(state, 'dsh_browser_type', taskKey(exec))
|
|
854
878
|
const browser = ctx.get('browser')
|
|
855
879
|
if (browser === undefined) throw new Error('tool-browser: browser service unavailable')
|
|
856
880
|
const session = await ensureSession(browser, state, taskKey(exec), agentOf(exec))
|
|
@@ -878,8 +902,8 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
878
902
|
},
|
|
879
903
|
}))
|
|
880
904
|
|
|
881
|
-
|
|
882
|
-
name: '
|
|
905
|
+
registerSafely(defineTool({
|
|
906
|
+
name: 'dsh_browser_scroll',
|
|
883
907
|
description: 'Scroll the shared-browser page: by pixel deltas (deltaX/deltaY), to a CSS selector\'s element, or to the top/bottom. Use to reveal below-the-fold content before snapshotting or clicking.',
|
|
884
908
|
parameters: {
|
|
885
909
|
deltaX: { type: 'number', description: 'Horizontal scroll delta in CSS pixels.' },
|
|
@@ -895,7 +919,7 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
895
919
|
timeoutMs,
|
|
896
920
|
isConcurrencySafe: () => false, // mutates page scroll state; exclusive within a task
|
|
897
921
|
async execute(args, exec) {
|
|
898
|
-
assertAllowed(state, '
|
|
922
|
+
assertAllowed(state, 'dsh_browser_scroll', taskKey(exec))
|
|
899
923
|
const browser = ctx.get('browser')
|
|
900
924
|
if (browser === undefined) throw new Error('tool-browser: browser service unavailable')
|
|
901
925
|
const session = await ensureSession(browser, state, taskKey(exec), agentOf(exec))
|
|
@@ -910,8 +934,8 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
910
934
|
},
|
|
911
935
|
}))
|
|
912
936
|
|
|
913
|
-
|
|
914
|
-
name: '
|
|
937
|
+
registerSafely(defineTool({
|
|
938
|
+
name: 'dsh_browser_back',
|
|
915
939
|
description: 'Go back one step in the shared-browser page history. A no-op when there is no previous entry.',
|
|
916
940
|
parameters: {},
|
|
917
941
|
output: {
|
|
@@ -921,7 +945,7 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
921
945
|
timeoutMs,
|
|
922
946
|
isConcurrencySafe: () => false, // mutates page state; exclusive within a task
|
|
923
947
|
async execute(_args, exec) {
|
|
924
|
-
assertAllowed(state, '
|
|
948
|
+
assertAllowed(state, 'dsh_browser_back', taskKey(exec))
|
|
925
949
|
const browser = ctx.get('browser')
|
|
926
950
|
if (browser === undefined) throw new Error('tool-browser: browser service unavailable')
|
|
927
951
|
const session = await ensureSession(browser, state, taskKey(exec), agentOf(exec))
|
|
@@ -930,8 +954,8 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
930
954
|
},
|
|
931
955
|
}))
|
|
932
956
|
|
|
933
|
-
|
|
934
|
-
name: '
|
|
957
|
+
registerSafely(defineTool({
|
|
958
|
+
name: 'dsh_browser_forward',
|
|
935
959
|
description: 'Go forward one step in the shared-browser page history. A no-op when there is no next entry.',
|
|
936
960
|
parameters: {},
|
|
937
961
|
output: {
|
|
@@ -941,7 +965,7 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
941
965
|
timeoutMs,
|
|
942
966
|
isConcurrencySafe: () => false, // mutates page state; exclusive within a task
|
|
943
967
|
async execute(_args, exec) {
|
|
944
|
-
assertAllowed(state, '
|
|
968
|
+
assertAllowed(state, 'dsh_browser_forward', taskKey(exec))
|
|
945
969
|
const browser = ctx.get('browser')
|
|
946
970
|
if (browser === undefined) throw new Error('tool-browser: browser service unavailable')
|
|
947
971
|
const session = await ensureSession(browser, state, taskKey(exec), agentOf(exec))
|
|
@@ -950,8 +974,8 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
950
974
|
},
|
|
951
975
|
}))
|
|
952
976
|
|
|
953
|
-
|
|
954
|
-
name: '
|
|
977
|
+
registerSafely(defineTool({
|
|
978
|
+
name: 'dsh_browser_refresh',
|
|
955
979
|
description: 'Reload the current page in the shared browser (like a browser\'s refresh button). Use after a page got stuck, to apply script changes, or to re-fetch a page.',
|
|
956
980
|
parameters: {},
|
|
957
981
|
output: {
|
|
@@ -961,7 +985,7 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
961
985
|
timeoutMs,
|
|
962
986
|
isConcurrencySafe: () => false, // reloads the page; exclusive within a task
|
|
963
987
|
async execute(_args, exec) {
|
|
964
|
-
assertAllowed(state, '
|
|
988
|
+
assertAllowed(state, 'dsh_browser_refresh', taskKey(exec))
|
|
965
989
|
const browser = ctx.get('browser')
|
|
966
990
|
if (browser === undefined) throw new Error('tool-browser: browser service unavailable')
|
|
967
991
|
const session = await ensureSession(browser, state, taskKey(exec), agentOf(exec))
|
|
@@ -970,8 +994,8 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
970
994
|
},
|
|
971
995
|
}))
|
|
972
996
|
|
|
973
|
-
|
|
974
|
-
name: '
|
|
997
|
+
registerSafely(defineTool({
|
|
998
|
+
name: 'dsh_browser_key',
|
|
975
999
|
description: 'Press one named key in the shared-browser page (Enter, Tab, Escape, Backspace, Delete, ArrowUp/Down/Left/Right, Home, End, PageUp, PageDown, Space). Use after focusing an input to submit a chat box (Enter), move focus (Tab), or navigate a list (arrows).',
|
|
976
1000
|
parameters: {
|
|
977
1001
|
key: { type: 'string', required: true, enum: ['Enter', 'Tab', 'Escape', 'Backspace', 'Delete', 'ArrowUp', 'ArrowDown', 'ArrowLeft', 'ArrowRight', 'Home', 'End', 'PageUp', 'PageDown', 'Space'], description: 'The key to press.' },
|
|
@@ -983,7 +1007,7 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
983
1007
|
timeoutMs,
|
|
984
1008
|
isConcurrencySafe: () => false, // mutates page state; exclusive within a task
|
|
985
1009
|
async execute(args, exec) {
|
|
986
|
-
assertAllowed(state, '
|
|
1010
|
+
assertAllowed(state, 'dsh_browser_key', taskKey(exec))
|
|
987
1011
|
const browser = ctx.get('browser')
|
|
988
1012
|
if (browser === undefined) throw new Error('tool-browser: browser service unavailable')
|
|
989
1013
|
const session = await ensureSession(browser, state, taskKey(exec), agentOf(exec))
|
|
@@ -992,9 +1016,9 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
992
1016
|
},
|
|
993
1017
|
}))
|
|
994
1018
|
|
|
995
|
-
|
|
996
|
-
name: '
|
|
997
|
-
description: 'Fill a form in one batch: pass fields with a CSS selector or name/label/placeholder text and the value to set (string, number, or boolean for checkbox/radio; for selects or radio groups pass the option value or visible text). Values are applied with the native setter plus input/change events, so React/Vue controlled inputs update correctly. Optionally submit the containing form. Prefer this over hand-written
|
|
1019
|
+
registerSafely(defineTool({
|
|
1020
|
+
name: 'dsh_browser_fill',
|
|
1021
|
+
description: 'Fill a form in one batch: pass fields with a CSS selector or name/label/placeholder text and the value to set (string, number, or boolean for checkbox/radio; for selects or radio groups pass the option value or visible text). Values are applied with the native setter plus input/change events, so React/Vue controlled inputs update correctly. Optionally submit the containing form. Prefer this over hand-written dsh_browser_execute for form filling; per-field failures are reported instead of throwing.',
|
|
998
1022
|
parameters: {
|
|
999
1023
|
fields: {
|
|
1000
1024
|
type: 'array',
|
|
@@ -1052,13 +1076,13 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
1052
1076
|
timeoutMs,
|
|
1053
1077
|
isConcurrencySafe: () => false,
|
|
1054
1078
|
async execute(args, exec) {
|
|
1055
|
-
assertAllowed(state, '
|
|
1079
|
+
assertAllowed(state, 'dsh_browser_fill', taskKey(exec))
|
|
1056
1080
|
const browser = ctx.get('browser')
|
|
1057
1081
|
if (browser === undefined) throw new Error('tool-browser: browser service unavailable')
|
|
1058
1082
|
const session = await ensureSession(browser, state, taskKey(exec), agentOf(exec))
|
|
1059
1083
|
const requested = args.fields ?? []
|
|
1060
1084
|
if (requested.length > maxFillFields) {
|
|
1061
|
-
throw new Error(`
|
|
1085
|
+
throw new Error(`dsh_browser_fill: ${requested.length} fields in one call exceeds the ${maxFillFields}-field limit; split it into several calls so each one can report per-field results instead of timing out as a batch`)
|
|
1062
1086
|
}
|
|
1063
1087
|
const fields = requested.map((f: { selector?: string; name?: string; label?: string; placeholder?: string; kind?: string; value?: string }) => {
|
|
1064
1088
|
// A missing value is a caller mistake, not a request to clear the field. Turning
|
|
@@ -1067,7 +1091,7 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
1067
1091
|
// a call can still arrive without it.
|
|
1068
1092
|
if (f.value === undefined) {
|
|
1069
1093
|
const matcher = f.selector ?? f.name ?? f.label ?? f.placeholder ?? '(no matcher)'
|
|
1070
|
-
throw new Error(`
|
|
1094
|
+
throw new Error(`dsh_browser_fill: field "${matcher}" has no "value"; pass one (use "false" to uncheck, or "" to clear deliberately)`)
|
|
1071
1095
|
}
|
|
1072
1096
|
return {
|
|
1073
1097
|
...f.selector !== undefined ? { selector: f.selector } : {},
|
|
@@ -1104,9 +1128,9 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
1104
1128
|
description: 'Locate the element (css/text/xpath).',
|
|
1105
1129
|
}
|
|
1106
1130
|
|
|
1107
|
-
|
|
1108
|
-
name: '
|
|
1109
|
-
description: 'Set the value of ONE input/textarea/select/contenteditable, located by css/text/xpath. Uses the native setter plus input/change events, so React/Vue controlled inputs update correctly. For selects pass the option value or visible text. For a whole form at once prefer
|
|
1131
|
+
registerSafely(defineTool({
|
|
1132
|
+
name: 'dsh_browser_set_value',
|
|
1133
|
+
description: 'Set the value of ONE input/textarea/select/contenteditable, located by css/text/xpath. Uses the native setter plus input/change events, so React/Vue controlled inputs update correctly. For selects pass the option value or visible text. For a whole form at once prefer dsh_browser_fill.',
|
|
1110
1134
|
parameters: {
|
|
1111
1135
|
target: targetParam,
|
|
1112
1136
|
value: { type: 'string', required: true, description: 'The value to set (string form; numbers/booleans accepted).' },
|
|
@@ -1125,7 +1149,7 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
1125
1149
|
timeoutMs,
|
|
1126
1150
|
isConcurrencySafe: () => false, // mutates a field; exclusive within a task
|
|
1127
1151
|
async execute(args, exec) {
|
|
1128
|
-
assertAllowed(state, '
|
|
1152
|
+
assertAllowed(state, 'dsh_browser_set_value', taskKey(exec))
|
|
1129
1153
|
const browser = ctx.get('browser')
|
|
1130
1154
|
if (browser === undefined) throw new Error('tool-browser: browser service unavailable')
|
|
1131
1155
|
const session = await ensureSession(browser, state, taskKey(exec), agentOf(exec))
|
|
@@ -1142,8 +1166,8 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
1142
1166
|
},
|
|
1143
1167
|
}))
|
|
1144
1168
|
|
|
1145
|
-
|
|
1146
|
-
name: '
|
|
1169
|
+
registerSafely(defineTool({
|
|
1170
|
+
name: 'dsh_browser_check',
|
|
1147
1171
|
description: 'Check (or uncheck, with checked=false) a checkbox or radio button, located by css/text/xpath.',
|
|
1148
1172
|
parameters: {
|
|
1149
1173
|
target: targetParam,
|
|
@@ -1156,7 +1180,7 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
1156
1180
|
timeoutMs,
|
|
1157
1181
|
isConcurrencySafe: () => false, // mutates a field; exclusive within a task
|
|
1158
1182
|
async execute(args, exec) {
|
|
1159
|
-
assertAllowed(state, '
|
|
1183
|
+
assertAllowed(state, 'dsh_browser_check', taskKey(exec))
|
|
1160
1184
|
const browser = ctx.get('browser')
|
|
1161
1185
|
if (browser === undefined) throw new Error('tool-browser: browser service unavailable')
|
|
1162
1186
|
const session = await ensureSession(browser, state, taskKey(exec), agentOf(exec))
|
|
@@ -1173,8 +1197,8 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
1173
1197
|
},
|
|
1174
1198
|
}))
|
|
1175
1199
|
|
|
1176
|
-
|
|
1177
|
-
name: '
|
|
1200
|
+
registerSafely(defineTool({
|
|
1201
|
+
name: 'dsh_browser_select',
|
|
1178
1202
|
description: 'Select one option of a <select>, located by css/text/xpath — by option value, visible text, or 0-based index (provide exactly one).',
|
|
1179
1203
|
parameters: {
|
|
1180
1204
|
target: targetParam,
|
|
@@ -1196,7 +1220,7 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
1196
1220
|
timeoutMs,
|
|
1197
1221
|
isConcurrencySafe: () => false, // mutates a field; exclusive within a task
|
|
1198
1222
|
async execute(args, exec) {
|
|
1199
|
-
assertAllowed(state, '
|
|
1223
|
+
assertAllowed(state, 'dsh_browser_select', taskKey(exec))
|
|
1200
1224
|
const browser = ctx.get('browser')
|
|
1201
1225
|
if (browser === undefined) throw new Error('tool-browser: browser service unavailable')
|
|
1202
1226
|
const session = await ensureSession(browser, state, taskKey(exec), agentOf(exec))
|
|
@@ -1215,8 +1239,8 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
1215
1239
|
},
|
|
1216
1240
|
}))
|
|
1217
1241
|
|
|
1218
|
-
|
|
1219
|
-
name: '
|
|
1242
|
+
registerSafely(defineTool({
|
|
1243
|
+
name: 'dsh_browser_clear',
|
|
1220
1244
|
description: 'Clear an input/textarea/contenteditable, or uncheck a checkbox/radio, located by css/text/xpath.',
|
|
1221
1245
|
parameters: {
|
|
1222
1246
|
target: targetParam,
|
|
@@ -1228,7 +1252,7 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
1228
1252
|
timeoutMs,
|
|
1229
1253
|
isConcurrencySafe: () => false, // mutates a field; exclusive within a task
|
|
1230
1254
|
async execute(args, exec) {
|
|
1231
|
-
assertAllowed(state, '
|
|
1255
|
+
assertAllowed(state, 'dsh_browser_clear', taskKey(exec))
|
|
1232
1256
|
const browser = ctx.get('browser')
|
|
1233
1257
|
if (browser === undefined) throw new Error('tool-browser: browser service unavailable')
|
|
1234
1258
|
const session = await ensureSession(browser, state, taskKey(exec), agentOf(exec))
|
|
@@ -1244,8 +1268,8 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
1244
1268
|
},
|
|
1245
1269
|
}))
|
|
1246
1270
|
|
|
1247
|
-
|
|
1248
|
-
name: '
|
|
1271
|
+
registerSafely(defineTool({
|
|
1272
|
+
name: 'dsh_browser_get_value',
|
|
1249
1273
|
description: 'Read the current value of one input/textarea/select/contenteditable, located by css/text/xpath. Use to VERIFY that a fill worked (e.g. before submitting).',
|
|
1250
1274
|
parameters: {
|
|
1251
1275
|
target: targetParam,
|
|
@@ -1269,7 +1293,7 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
1269
1293
|
timeoutMs,
|
|
1270
1294
|
isConcurrencySafe: () => true,
|
|
1271
1295
|
async execute(args, exec) {
|
|
1272
|
-
assertAllowed(state, '
|
|
1296
|
+
assertAllowed(state, 'dsh_browser_get_value', taskKey(exec))
|
|
1273
1297
|
const browser = ctx.get('browser')
|
|
1274
1298
|
if (browser === undefined) throw new Error('tool-browser: browser service unavailable')
|
|
1275
1299
|
const session = await ensureSession(browser, state, taskKey(exec), agentOf(exec))
|
|
@@ -1289,9 +1313,9 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
1289
1313
|
},
|
|
1290
1314
|
}))
|
|
1291
1315
|
|
|
1292
|
-
|
|
1293
|
-
name: '
|
|
1294
|
-
description: 'Capture the current shared-browser page as a screenshot (PNG default, JPEG optional). This is for models that can read images: layout checks, charts, designs, CAPTCHAs, or locating an element by eye before clicking its coordinates. A model without image input gains nothing from it —
|
|
1316
|
+
registerSafely(defineTool({
|
|
1317
|
+
name: 'dsh_browser_screenshot',
|
|
1318
|
+
description: 'Capture the current shared-browser page as a screenshot (PNG default, JPEG optional). This is for models that can read images: layout checks, charts, designs, CAPTCHAs, or locating an element by eye before clicking its coordinates. A model without image input gains nothing from it — dsh_browser_snapshot, dsh_browser_a11y and dsh_browser_content carry the same page as text, and dsh_browser_scrape extracts structured data. Supports full-page capture, save-to-file, JPEG encoding, and downscaling (maxWidth/maxHeight) to cut vision-tool token cost. JPEG needs a browser whose CDP encoder works — the self-hosted browser and an installed Chrome/Edge; some carriers (the desktop sidebar) return PNG instead. Downscaling works on every carrier.',
|
|
1295
1319
|
parameters: {
|
|
1296
1320
|
fullPage: { type: 'boolean', description: 'Capture the full scrollable page instead of the viewport (default false).' },
|
|
1297
1321
|
savePath: { type: 'string', description: 'Absolute file path to also save the image to (e.g. for read_image vision location). Must resolve inside the configured downloadDir (default: the system Downloads folder, localized names such as ~/下载 included); an existing file is never overwritten.' },
|
|
@@ -1341,8 +1365,8 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
1341
1365
|
}))
|
|
1342
1366
|
|
|
1343
1367
|
if (config.tabTools !== false) {
|
|
1344
|
-
|
|
1345
|
-
name: '
|
|
1368
|
+
registerSafely(defineTool({
|
|
1369
|
+
name: 'dsh_browser_list_tabs',
|
|
1346
1370
|
description: 'List the shared-browser session\'s tabs with their URLs and which is active.',
|
|
1347
1371
|
parameters: {},
|
|
1348
1372
|
output: {
|
|
@@ -1382,9 +1406,9 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
1382
1406
|
},
|
|
1383
1407
|
}))
|
|
1384
1408
|
|
|
1385
|
-
|
|
1386
|
-
name: '
|
|
1387
|
-
description: 'Switch the shared browser to a tab by id (from
|
|
1409
|
+
registerSafely(defineTool({
|
|
1410
|
+
name: 'dsh_browser_switch_tab',
|
|
1411
|
+
description: 'Switch the shared browser to a tab by id (from dsh_browser_list_tabs).',
|
|
1388
1412
|
parameters: {
|
|
1389
1413
|
tabId: { type: 'string', required: true, description: 'The tab id to switch to.' },
|
|
1390
1414
|
},
|
|
@@ -1395,7 +1419,7 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
1395
1419
|
timeoutMs,
|
|
1396
1420
|
isConcurrencySafe: () => false, // mutates the active tab; exclusive within a task
|
|
1397
1421
|
async execute(args, exec) {
|
|
1398
|
-
assertAllowed(state, '
|
|
1422
|
+
assertAllowed(state, 'dsh_browser_switch_tab', taskKey(exec))
|
|
1399
1423
|
const browser = ctx.get('browser')
|
|
1400
1424
|
if (browser === undefined) throw new Error('tool-browser: browser service unavailable')
|
|
1401
1425
|
const session = await ensureSession(browser, state, taskKey(exec), agentOf(exec))
|
|
@@ -1404,8 +1428,8 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
1404
1428
|
},
|
|
1405
1429
|
}))
|
|
1406
1430
|
|
|
1407
|
-
|
|
1408
|
-
name: '
|
|
1431
|
+
registerSafely(defineTool({
|
|
1432
|
+
name: 'dsh_browser_close_tab',
|
|
1409
1433
|
description: 'Close a tab in the shared browser by id. Closing the active tab activates the next.',
|
|
1410
1434
|
parameters: {
|
|
1411
1435
|
tabId: { type: 'string', required: true, description: 'The tab id to close.' },
|
|
@@ -1417,7 +1441,7 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
1417
1441
|
timeoutMs,
|
|
1418
1442
|
isConcurrencySafe: () => false, // mutates the tab list; exclusive within a task
|
|
1419
1443
|
async execute(args, exec) {
|
|
1420
|
-
assertAllowed(state, '
|
|
1444
|
+
assertAllowed(state, 'dsh_browser_close_tab', taskKey(exec))
|
|
1421
1445
|
const browser = ctx.get('browser')
|
|
1422
1446
|
if (browser === undefined) throw new Error('tool-browser: browser service unavailable')
|
|
1423
1447
|
const session = await ensureSession(browser, state, taskKey(exec), agentOf(exec))
|
|
@@ -1426,8 +1450,8 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
1426
1450
|
},
|
|
1427
1451
|
}))
|
|
1428
1452
|
|
|
1429
|
-
|
|
1430
|
-
name: '
|
|
1453
|
+
registerSafely(defineTool({
|
|
1454
|
+
name: 'dsh_browser_reset',
|
|
1431
1455
|
description: 'Close every tab in the shared browser and start fresh with one blank tab.',
|
|
1432
1456
|
parameters: {},
|
|
1433
1457
|
output: {
|
|
@@ -1437,7 +1461,7 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
1437
1461
|
timeoutMs,
|
|
1438
1462
|
isConcurrencySafe: () => false, // closes every tab; exclusive within a task
|
|
1439
1463
|
async execute(_args, exec) {
|
|
1440
|
-
assertAllowed(state, '
|
|
1464
|
+
assertAllowed(state, 'dsh_browser_reset', taskKey(exec))
|
|
1441
1465
|
const browser = ctx.get('browser')
|
|
1442
1466
|
if (browser === undefined) throw new Error('tool-browser: browser service unavailable')
|
|
1443
1467
|
const session = await ensureSession(browser, state, taskKey(exec), agentOf(exec))
|
|
@@ -1447,8 +1471,8 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
1447
1471
|
}))
|
|
1448
1472
|
}
|
|
1449
1473
|
|
|
1450
|
-
|
|
1451
|
-
name: '
|
|
1474
|
+
registerSafely(defineTool({
|
|
1475
|
+
name: 'dsh_browser_history',
|
|
1452
1476
|
description: 'List the shared browser session\'s recorded operation history (navigate/execute/click/type), newest last, with per-step success/error. Use to understand what the agent did and to pick a step to replay.',
|
|
1453
1477
|
parameters: {
|
|
1454
1478
|
verbose: { type: 'boolean', description: "Include each operation's parameters (default false). They are what you just sent, so they are omitted unless asked for." },
|
|
@@ -1544,9 +1568,9 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
1544
1568
|
},
|
|
1545
1569
|
}))
|
|
1546
1570
|
|
|
1547
|
-
|
|
1548
|
-
name: '
|
|
1549
|
-
description: 'List pages the shared browser has visited, newest first, from the PERSISTENT browsing history: it survives closing the browser and restarting DSH, so it is how you (or the human) find a page again afterwards. Reopen one by passing its url to
|
|
1571
|
+
registerSafely(defineTool({
|
|
1572
|
+
name: 'dsh_browser_visited',
|
|
1573
|
+
description: 'List pages the shared browser has visited, newest first, from the PERSISTENT browsing history: it survives closing the browser and restarting DSH, so it is how you (or the human) find a page again afterwards. Reopen one by passing its url to dsh_browser_open. Not to be confused with dsh_browser_history, which is only the current session\'s operation log (navigate/click/type) and disappears with the session.',
|
|
1550
1574
|
parameters: {
|
|
1551
1575
|
limit: { type: 'number', description: 'Maximum entries to return (default 30, capped at 200).' },
|
|
1552
1576
|
domain: { type: 'string', description: 'Only visits whose hostname contains this text (case-insensitive).' },
|
|
@@ -1623,9 +1647,9 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
1623
1647
|
},
|
|
1624
1648
|
}))
|
|
1625
1649
|
|
|
1626
|
-
|
|
1627
|
-
name: '
|
|
1628
|
-
description: 'Replay one recorded browser operation by its history sequence number (from
|
|
1650
|
+
registerSafely(defineTool({
|
|
1651
|
+
name: 'dsh_browser_replay',
|
|
1652
|
+
description: 'Replay one recorded browser operation by its history sequence number (from dsh_browser_history). Navigate/click/type are re-issued against the current page; execute re-runs its script. The replayed step is appended to history as a new entry.',
|
|
1629
1653
|
parameters: {
|
|
1630
1654
|
seq: { type: 'number', required: true, description: 'The history entry sequence number to replay.' },
|
|
1631
1655
|
},
|
|
@@ -1636,7 +1660,7 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
1636
1660
|
timeoutMs,
|
|
1637
1661
|
isConcurrencySafe: () => false,
|
|
1638
1662
|
async execute(args, exec) {
|
|
1639
|
-
assertAllowed(state, '
|
|
1663
|
+
assertAllowed(state, 'dsh_browser_replay', taskKey(exec))
|
|
1640
1664
|
const browser = ctx.get('browser')
|
|
1641
1665
|
if (browser === undefined) throw new Error('tool-browser: browser service unavailable')
|
|
1642
1666
|
const session = await ensureSession(browser, state, taskKey(exec), agentOf(exec))
|
|
@@ -1645,8 +1669,8 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
1645
1669
|
},
|
|
1646
1670
|
}))
|
|
1647
1671
|
|
|
1648
|
-
|
|
1649
|
-
name: '
|
|
1672
|
+
registerSafely(defineTool({
|
|
1673
|
+
name: 'dsh_browser_download',
|
|
1650
1674
|
description: 'Download a URL to a local file, keeping the browser session\'s cookies and login state. Use for fetching files behind authentication or from the current page context. Available on the self-hosted browser; the desktop shell delegates downloads to the real browser UI.',
|
|
1651
1675
|
parameters: {
|
|
1652
1676
|
url: { type: 'string', required: true, description: 'The URL to download.' },
|
|
@@ -1659,7 +1683,7 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
1659
1683
|
timeoutMs,
|
|
1660
1684
|
isConcurrencySafe: () => false,
|
|
1661
1685
|
async execute(args, exec) {
|
|
1662
|
-
assertAllowed(state, '
|
|
1686
|
+
assertAllowed(state, 'dsh_browser_download', taskKey(exec))
|
|
1663
1687
|
const browser = ctx.get('browser')
|
|
1664
1688
|
if (browser === undefined) throw new Error('tool-browser: browser service unavailable')
|
|
1665
1689
|
const session = await ensureSession(browser, state, taskKey(exec), agentOf(exec))
|
|
@@ -1668,8 +1692,8 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
1668
1692
|
},
|
|
1669
1693
|
}))
|
|
1670
1694
|
|
|
1671
|
-
|
|
1672
|
-
name: '
|
|
1695
|
+
registerSafely(defineTool({
|
|
1696
|
+
name: 'dsh_browser_session',
|
|
1673
1697
|
description: 'Show THIS task\'s browser session: its id and open tabs. Each task (DSH session) has its own browser session, so this reflects what your task drives. The window is shared with the human and other tasks, but tab sets and history are isolated per task.',
|
|
1674
1698
|
parameters: {},
|
|
1675
1699
|
output: {
|
|
@@ -1709,9 +1733,9 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
1709
1733
|
},
|
|
1710
1734
|
}))
|
|
1711
1735
|
|
|
1712
|
-
|
|
1713
|
-
name: '
|
|
1714
|
-
description: 'Reset THIS task\'s browser session: close it entirely so the next
|
|
1736
|
+
registerSafely(defineTool({
|
|
1737
|
+
name: 'dsh_browser_reset_session',
|
|
1738
|
+
description: 'Reset THIS task\'s browser session: close it entirely so the next dsh_browser_* call starts a fresh session with one blank tab. Other tasks\' sessions are untouched. Use when a session is in a bad state or you want a clean slate.',
|
|
1715
1739
|
parameters: {},
|
|
1716
1740
|
output: {
|
|
1717
1741
|
schema: { type: 'object', additionalProperties: false, properties: { reset: { type: 'boolean', required: true } } },
|
|
@@ -1720,7 +1744,7 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
1720
1744
|
timeoutMs,
|
|
1721
1745
|
isConcurrencySafe: () => false, // closes the whole session; exclusive within a task
|
|
1722
1746
|
async execute(_args, exec) {
|
|
1723
|
-
assertAllowed(state, '
|
|
1747
|
+
assertAllowed(state, 'dsh_browser_reset_session', taskKey(exec))
|
|
1724
1748
|
const browser = ctx.get('browser')
|
|
1725
1749
|
if (browser === undefined) throw new Error('tool-browser: browser service unavailable')
|
|
1726
1750
|
const key = taskKey(exec)
|
|
@@ -1738,9 +1762,9 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
1738
1762
|
},
|
|
1739
1763
|
}))
|
|
1740
1764
|
|
|
1741
|
-
|
|
1742
|
-
name: '
|
|
1743
|
-
description: 'Restrict which browser actions are allowed, to prevent stray clicks/navigation. Pass a list of browser tool names (e.g. ["
|
|
1765
|
+
registerSafely(defineTool({
|
|
1766
|
+
name: 'dsh_browser_restrict',
|
|
1767
|
+
description: 'Restrict which browser actions are allowed, to prevent stray clicks/navigation. Pass a list of browser tool names (e.g. ["dsh_browser_snapshot","dsh_browser_content","dsh_browser_click"]) — any other dsh_browser_* call is refused. Pass an empty list or omit to lift the restriction. Never blocked, whatever the list says: the observing tools (snapshot/a11y/content/scrape/screenshot/get_value/wait/challenge/list_tabs/session/history/visited) and the three that lift a restriction or recover from a bad state (restrict/reset/reset_session) — so a restriction can always be read around and undone. IMPORTANT: this is a SOFT guardrail against accidental actions, NOT a security boundary — you (the model) can lift it yourself with an empty list.',
|
|
1744
1768
|
parameters: {
|
|
1745
1769
|
allowed: {
|
|
1746
1770
|
type: 'array',
|
|
@@ -1757,9 +1781,9 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
1757
1781
|
async execute(args, exec) {
|
|
1758
1782
|
// Always allowed so the guard can be lifted.
|
|
1759
1783
|
const allowed = args.allowed ?? []
|
|
1760
|
-
const unknown = allowed.filter((t: string) => !t.startsWith('
|
|
1784
|
+
const unknown = allowed.filter((t: string) => !t.startsWith('dsh_browser_'))
|
|
1761
1785
|
if (unknown.length > 0) {
|
|
1762
|
-
throw new Error(`
|
|
1786
|
+
throw new Error(`dsh_browser_restrict: unknown tool name(s) ${unknown.map(t => `"${t}"`).join(', ')} (must start with "dsh_browser_")`)
|
|
1763
1787
|
}
|
|
1764
1788
|
// Empty list (or omitted) lifts this task's restriction; a non-empty list is its
|
|
1765
1789
|
// new allow-list. Per task, not per plugin: this state object is shared by every
|
|
@@ -1772,8 +1796,8 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
1772
1796
|
},
|
|
1773
1797
|
}))
|
|
1774
1798
|
|
|
1775
|
-
|
|
1776
|
-
name: '
|
|
1799
|
+
registerSafely(defineTool({
|
|
1800
|
+
name: 'dsh_browser_auth',
|
|
1777
1801
|
description: 'Export or restore the browser session\'s cookies (login state). Use "flush" to get a JSON cookie list (save it to a private file to persist logins), or "restore" with that list to put logins back (e.g. after the browser host restarted). Available on all three carriers: the self-hosted browser reads its own session, while the desktop sidebar and an installed Chrome/Edge go through CDP. Refused entirely when the credential switch is off. Exported cookies are LIVE CREDENTIALS: treat them as secrets — do not echo them into the conversation, keep them out of logs, and store the list in a private file.',
|
|
1778
1802
|
parameters: {
|
|
1779
1803
|
action: { type: 'string', required: true, enum: ['flush', 'restore'], description: 'flush = export cookies; restore = import cookies.' },
|
|
@@ -1793,7 +1817,7 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
1793
1817
|
timeoutMs,
|
|
1794
1818
|
isConcurrencySafe: () => false,
|
|
1795
1819
|
async execute(args, exec) {
|
|
1796
|
-
assertAllowed(state, '
|
|
1820
|
+
assertAllowed(state, 'dsh_browser_auth', taskKey(exec))
|
|
1797
1821
|
const browser = ctx.get('browser')
|
|
1798
1822
|
if (browser === undefined) throw new Error('tool-browser: browser service unavailable')
|
|
1799
1823
|
const session = await ensureSession(browser, state, taskKey(exec), agentOf(exec))
|
|
@@ -1805,7 +1829,7 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
1805
1829
|
// empty call returned {restored: 0} and rendered 'Restored 0 cookies.', which
|
|
1806
1830
|
// reads as success and would leave the caller believing the login was restored.
|
|
1807
1831
|
if (!Array.isArray(args.cookies) || args.cookies.length === 0) {
|
|
1808
|
-
throw new Error('
|
|
1832
|
+
throw new Error('dsh_browser_auth: action "restore" needs a non-empty "cookies" array (get one from action "flush" on the same profile)')
|
|
1809
1833
|
}
|
|
1810
1834
|
const list = args.cookies as unknown[]
|
|
1811
1835
|
const restored = await browser.restoreAuth(session, list as never)
|