dsh-builtin-browser 0.4.0 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +68 -30
- package/README.md +66 -30
- package/client.js +60 -6
- package/cordis.patch.yml +21 -0
- package/desktop-bridge/plugin-browser-bridge.js +12 -4
- package/docs/architecture.md +12 -2
- package/docs/tool-reference.md +17 -17
- package/docs/user-guide.md +13 -4
- package/docs/why-browser.md +2 -2
- package/lib/browser/types.d.ts +25 -1
- package/lib/browser-electron/bridge-connection.d.ts +31 -5
- package/lib/browser-electron/bridge-connection.js +143 -21
- package/lib/browser-electron/entry.d.ts +1 -1
- package/lib/browser-electron/entry.js +61 -35
- package/lib/browser-electron/ephemeral-profile.d.ts +37 -0
- package/lib/browser-electron/ephemeral-profile.js +146 -0
- package/lib/browser-electron/history-store.d.ts +25 -0
- package/lib/browser-electron/history-store.js +51 -4
- package/lib/browser-electron/host-main.js +75 -27
- package/lib/browser-electron/provider.d.ts +44 -1
- package/lib/browser-electron/provider.js +379 -48
- package/lib/browser-electron/remote-host.d.ts +52 -3
- package/lib/browser-electron/remote-host.js +244 -23
- package/lib/browser-electron/settings-route.d.ts +28 -0
- package/lib/browser-electron/settings-route.js +28 -0
- package/lib/browser-electron/settings-store.d.ts +40 -0
- package/lib/browser-electron/settings-store.js +107 -11
- package/lib/browser-electron/system-browser.d.ts +71 -3
- package/lib/browser-electron/system-browser.js +355 -38
- package/lib/tool-browser/index.js +143 -34
- package/package.json +5 -3
- package/scripts/verify-host-compat.mjs +195 -0
- package/src/browser/types.ts +25 -1
- package/src/browser-electron/bridge-connection.ts +142 -21
- package/src/browser-electron/entry.ts +58 -33
- package/src/browser-electron/ephemeral-profile.ts +152 -0
- package/src/browser-electron/history-store.ts +51 -4
- package/src/browser-electron/host-main.ts +105 -35
- package/src/browser-electron/provider.ts +401 -51
- package/src/browser-electron/remote-host.ts +261 -36
- package/src/browser-electron/settings-route.ts +46 -0
- package/src/browser-electron/settings-store.ts +121 -11
- package/src/browser-electron/system-browser.ts +342 -21
- package/src/tool-browser/index.ts +144 -36
|
@@ -102,7 +102,10 @@ export const READ_ONLY_TOOLS: ReadonlySet<string> = new Set([
|
|
|
102
102
|
'browser_list_tabs', 'browser_session', 'browser_history', 'browser_visited',
|
|
103
103
|
// browser_auth reads cookies out of the browser: an observation, and one the
|
|
104
104
|
// credentials switch gates on its own (see credentials.allowRead in the settings).
|
|
105
|
-
|
|
105
|
+
// browser_auth is deliberately NOT here. It reads cookies on "flush" but WRITES them on
|
|
106
|
+
// "restore", to arbitrary domains, which is an action like any other — exempting the tool
|
|
107
|
+
// would let a task that restricted its own actions rewrite the shared browser's logins.
|
|
108
|
+
//
|
|
106
109
|
// Getting out of a bad state, or lifting the restriction itself. Without these a task
|
|
107
110
|
// that restricted everything could neither recover nor be released.
|
|
108
111
|
'browser_restrict', 'browser_reset_session', 'browser_reset',
|
|
@@ -137,6 +140,26 @@ function agentOf(exec: unknown): ExecAgent | undefined {
|
|
|
137
140
|
return (exec as { agent?: ExecAgent } | undefined)?.agent
|
|
138
141
|
}
|
|
139
142
|
|
|
143
|
+
/**
|
|
144
|
+
* Read a target's `index`, refusing the values that cannot mean anything.
|
|
145
|
+
*
|
|
146
|
+
* The index is looked up in the page (`els[index] ?? null`), so a negative one — `-1`, the
|
|
147
|
+
* common "give me the last one" shorthand — matches nothing. The page script then treated
|
|
148
|
+
* that as "not on the page yet" and polled for its whole 10s budget before reporting
|
|
149
|
+
* "element not found", which reads like a slow page rather than a bad argument. Refusing it
|
|
150
|
+
* here names the real problem immediately.
|
|
151
|
+
* @param target - the caller's target, or undefined when there is none.
|
|
152
|
+
* @returns the index, or undefined to let the page default to 0.
|
|
153
|
+
*/
|
|
154
|
+
function targetIndex(target: { readonly index?: unknown } | undefined): number | undefined {
|
|
155
|
+
const index = target?.index
|
|
156
|
+
if (index === undefined) return undefined
|
|
157
|
+
if (typeof index !== 'number' || !Number.isSafeInteger(index) || index < 0) {
|
|
158
|
+
throw new Error(`target.index must be a non-negative integer (got ${JSON.stringify(index)}); it is a 0-based position among the matches, and negative values such as -1 match nothing — pass the position you want, or narrow the selector`)
|
|
159
|
+
}
|
|
160
|
+
return index
|
|
161
|
+
}
|
|
162
|
+
|
|
140
163
|
/**
|
|
141
164
|
* The task key for a tool call: the calling DSH session id, or the shared
|
|
142
165
|
* default key when the call carries no agent context (CLI probes, tests).
|
|
@@ -234,12 +257,12 @@ function parseFillValue(v: string | undefined): string | number | boolean {
|
|
|
234
257
|
function formatSnapshot(snapshot: {
|
|
235
258
|
url: string
|
|
236
259
|
title?: string
|
|
237
|
-
elements: readonly { ref: number; kind: string; label: string; x: number; y: number; frame?: boolean }[]
|
|
260
|
+
elements: readonly { ref: number; kind: string; label: string; x: number; y: number; frame?: boolean; selector?: string }[]
|
|
238
261
|
truncated?: boolean
|
|
239
262
|
challenge?: { blocked: boolean; kind?: string; reason?: string }
|
|
240
263
|
}, options: { coords?: boolean } = {}): string {
|
|
241
264
|
const showCoords = options.coords === true
|
|
242
|
-
const lines = snapshot.elements.map(el => `[${el.ref}] ${el.kind}: ${el.label}${el.frame === true ? ' (iframe)' : ''}${showCoords ? ` (${el.x},${el.y})` : ''}`)
|
|
265
|
+
const lines = snapshot.elements.map(el => `[${el.ref}] ${el.kind}: ${el.label}${el.selector !== undefined && el.selector !== '' ? ` {${el.selector}}` : ''}${el.frame === true ? ' (iframe)' : ''}${showCoords ? ` (${el.x},${el.y})` : ''}`)
|
|
243
266
|
const header = `URL: ${snapshot.url}${snapshot.title !== undefined ? `\nTitle: ${snapshot.title}` : ''}`
|
|
244
267
|
const body = lines.length > 0 ? lines.join('\n') : '(no interactive elements found)'
|
|
245
268
|
const tail = snapshot.truncated === true ? '\n(snapshot truncated)' : ''
|
|
@@ -252,6 +275,25 @@ function formatSnapshot(snapshot: {
|
|
|
252
275
|
/** Register all browser tools with `ctx.tools`. */
|
|
253
276
|
export function apply(ctx: Context, config: Config = {}): void {
|
|
254
277
|
const timeoutMs = config.timeoutMs ?? 60_000
|
|
278
|
+
/**
|
|
279
|
+
* Longest wait `browser_wait` may poll for, leaving room for the tool call itself to
|
|
280
|
+
* finish and report.
|
|
281
|
+
*
|
|
282
|
+
* `browser_wait` promises a VERDICT — `{ready:false, reason}` — and the enclosing tool
|
|
283
|
+
* call aborts at `timeoutMs`. A caller-supplied budget longer than that aborted the call
|
|
284
|
+
* instead, so the documented verdict never arrived. Clamping to just inside the budget is
|
|
285
|
+
* what keeps the promise: the polling stops, the reason is returned.
|
|
286
|
+
*/
|
|
287
|
+
const maxWaitMs = Math.max(250, timeoutMs - 5_000)
|
|
288
|
+
/**
|
|
289
|
+
* Most fields `browser_fill` accepts in one batch.
|
|
290
|
+
*
|
|
291
|
+
* The batch runs as ONE page evaluation, so hundreds of fields do not produce hundreds of
|
|
292
|
+
* per-field verdicts — they produce a single evaluate that exceeds the tool budget and
|
|
293
|
+
* throws, which is the opposite of the documented "per-field failures are reported instead
|
|
294
|
+
* of throwing". Real forms are far smaller than this; anything larger should be split.
|
|
295
|
+
*/
|
|
296
|
+
const maxFillFields = 200
|
|
255
297
|
// Per-context state: sessions, in-flight opens, and the restriction are
|
|
256
298
|
// scoped to THIS plugin apply, so parallel contexts never share sessions
|
|
257
299
|
// or leak restrictions into each other.
|
|
@@ -297,6 +339,12 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
297
339
|
x: { type: 'number', required: true },
|
|
298
340
|
y: { type: 'number', required: true },
|
|
299
341
|
frame: { type: 'boolean' },
|
|
342
|
+
// The page-side script computes this for every element and the tool layer
|
|
343
|
+
// passes it through. Without it in the schema, a declared
|
|
344
|
+
// additionalProperties: false rejects the whole result at run time — which is
|
|
345
|
+
// how this was found, on the very first real call, after a static review had
|
|
346
|
+
// called the change verified.
|
|
347
|
+
selector: { type: 'string' },
|
|
300
348
|
},
|
|
301
349
|
},
|
|
302
350
|
},
|
|
@@ -331,7 +379,7 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
331
379
|
return {
|
|
332
380
|
url: snapshot.url,
|
|
333
381
|
...snapshot.title !== undefined ? { title: snapshot.title } : {},
|
|
334
|
-
elements: snapshot.elements.map(el => ({ ref: el.ref, kind: el.kind, label: el.label, x: el.x, y: el.y, ...el.frame === true ? { frame: true } : {} })),
|
|
382
|
+
elements: snapshot.elements.map(el => ({ ref: el.ref, kind: el.kind, label: el.label, selector: el.selector, x: el.x, y: el.y, ...el.frame === true ? { frame: true } : {} })),
|
|
335
383
|
truncated: snapshot.truncated,
|
|
336
384
|
...snapshot.challenge !== undefined ? { challenge: snapshot.challenge } : {},
|
|
337
385
|
}
|
|
@@ -365,6 +413,12 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
365
413
|
x: { type: 'number', required: true },
|
|
366
414
|
y: { type: 'number', required: true },
|
|
367
415
|
frame: { type: 'boolean' },
|
|
416
|
+
// The page-side script computes this for every element and the tool layer
|
|
417
|
+
// passes it through. Without it in the schema, a declared
|
|
418
|
+
// additionalProperties: false rejects the whole result at run time — which is
|
|
419
|
+
// how this was found, on the very first real call, after a static review had
|
|
420
|
+
// called the change verified.
|
|
421
|
+
selector: { type: 'string' },
|
|
368
422
|
},
|
|
369
423
|
},
|
|
370
424
|
},
|
|
@@ -391,7 +445,7 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
391
445
|
return {
|
|
392
446
|
url: snapshot.url,
|
|
393
447
|
...snapshot.title !== undefined ? { title: snapshot.title } : {},
|
|
394
|
-
elements: snapshot.elements.map(el => ({ ref: el.ref, kind: el.kind, label: el.label, x: el.x, y: el.y, ...el.frame === true ? { frame: true } : {} })),
|
|
448
|
+
elements: snapshot.elements.map(el => ({ ref: el.ref, kind: el.kind, label: el.label, selector: el.selector, x: el.x, y: el.y, ...el.frame === true ? { frame: true } : {} })),
|
|
395
449
|
truncated: snapshot.truncated,
|
|
396
450
|
...snapshot.challenge !== undefined ? { challenge: snapshot.challenge } : {},
|
|
397
451
|
}
|
|
@@ -400,10 +454,10 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
400
454
|
|
|
401
455
|
ctx.tools.register(defineTool({
|
|
402
456
|
name: 'browser_a11y',
|
|
403
|
-
description: 'Read the page\'s accessibility tree: every interactive node with its semantic role (button/link/textbox/checkbox/…), accessible name, current value, and states (enabled/disabled/checked/expanded/…),
|
|
457
|
+
description: 'Read the page\'s accessibility tree: every interactive node with its semantic role (button/link/textbox/checkbox/…), accessible name, current value, and states (enabled/disabled/checked/expanded/…), (coordinates only when coords: true). Prefer this over browser_snapshot to understand a page\'s structure and find the right element: roles and names tell you WHAT each node is, and the name is what browser_click matches when you target by text. Penetrates same-origin iframes and shadow roots.',
|
|
404
458
|
parameters: {
|
|
405
459
|
includeHidden: { type: 'boolean', description: 'Include hidden elements (default false).' },
|
|
406
|
-
maxNodes: { type: 'number', description: 'Maximum nodes (default
|
|
460
|
+
maxNodes: { type: 'number', description: 'Maximum nodes (default 150, range 10-5000).' },
|
|
407
461
|
coords: { type: 'boolean', description: 'Include each element\'s viewport coordinates. Off by default: a semantic target needs no coordinates, and they are pure noise unless a vision pass is going to click a pixel position.' },
|
|
408
462
|
},
|
|
409
463
|
output: {
|
|
@@ -432,13 +486,19 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
432
486
|
x: { type: 'number', required: true },
|
|
433
487
|
y: { type: 'number', required: true },
|
|
434
488
|
frame: { type: 'boolean' },
|
|
489
|
+
// The page-side script computes this for every element and the tool layer
|
|
490
|
+
// passes it through. Without it in the schema, a declared
|
|
491
|
+
// additionalProperties: false rejects the whole result at run time — which is
|
|
492
|
+
// how this was found, on the very first real call, after a static review had
|
|
493
|
+
// called the change verified.
|
|
494
|
+
selector: { type: 'string' },
|
|
435
495
|
},
|
|
436
496
|
},
|
|
437
497
|
},
|
|
438
498
|
},
|
|
439
499
|
},
|
|
440
500
|
render: (args, value) => {
|
|
441
|
-
const nodes = value.nodes as Array<{ ref: number; role: string; name: string; value?: string | null; states: string[]; depth: number; tag: string; x: number; y: number; frame?: boolean }>
|
|
501
|
+
const nodes = value.nodes as Array<{ ref: number; role: string; name: string; value?: string | null; states: string[]; depth: number; tag: string; x: number; y: number; frame?: boolean; selector?: string }>
|
|
442
502
|
// Built for reading, not for imaging: containment is shown as indentation
|
|
443
503
|
// (the depth was already collected and simply thrown away), an empty state
|
|
444
504
|
// list adds nothing, and coordinates appear only when asked for — a model
|
|
@@ -447,13 +507,22 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
447
507
|
const showCoords = args.coords === true
|
|
448
508
|
const lines = nodes.map(n => {
|
|
449
509
|
const valuePart = n.value !== undefined && n.value !== null ? ` value="${String(n.value).slice(0, 60)}"` : ''
|
|
450
|
-
|
|
510
|
+
// One space per level, not two: indentation was 19.7% of a measured 38,953-character
|
|
511
|
+
// result, and DOM depth is not a11y depth — deep wrapper divs push it to the cap.
|
|
512
|
+
const indent = ' '.repeat(Math.max(0, Math.min(8, Number(n.depth) || 0)))
|
|
451
513
|
const coordsPart = showCoords ? ` (${n.x},${n.y})` : ''
|
|
452
|
-
|
|
453
|
-
|
|
514
|
+
// Only states that are not the default. `states=[enabled]` was printed on 405 of 500
|
|
515
|
+
// nodes whose state WAS the default — 22% of the output for no information. Absence
|
|
516
|
+
// now means "enabled", which the description states.
|
|
517
|
+
const notableStates = Array.isArray(n.states) ? n.states.filter((state: string) => state !== 'enabled') : []
|
|
518
|
+
const statePart = notableStates.length > 0 ? ` states=[${notableStates.join(',')}]` : ''
|
|
519
|
+
// The provider computes this for a node with an id or a name; printing it is what
|
|
520
|
+
// makes the node something the caller can point at instead of guessing at its text.
|
|
521
|
+
const selectorPart = typeof n.selector === 'string' && n.selector !== '' ? ` {${n.selector}}` : ''
|
|
522
|
+
return `${indent}[${n.ref}] ${n.role} "${n.name}"${valuePart}${coordsPart}${statePart}${selectorPart}${n.frame === true ? ' (iframe)' : ''}`
|
|
454
523
|
})
|
|
455
524
|
const header = `URL: ${value.url}${value.title !== undefined ? `\nTitle: ${value.title}` : ''}`
|
|
456
|
-
const hint = showCoords ? '' : '\n(
|
|
525
|
+
const hint = showCoords ? '' : '\n(no coords; pass coords: true)'
|
|
457
526
|
return [{ type: 'text', text: `${header}\n\n${lines.length > 0 ? lines.join('\n') : '(no accessible interactive nodes)'}${value.truncated === true ? '\n(truncated)' : ''}${hint}` }]
|
|
458
527
|
},
|
|
459
528
|
},
|
|
@@ -480,6 +549,11 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
480
549
|
states: [...n.states],
|
|
481
550
|
depth: n.depth,
|
|
482
551
|
tag: n.tag,
|
|
552
|
+
// The provider computes this and the renderer prints it, but this mapping is an
|
|
553
|
+
// explicit allow-list rather than a pass-through — so leaving it out here silently
|
|
554
|
+
// dropped the field between the two, and the tool promised a reference it never
|
|
555
|
+
// sent. The snapshot mapping lists its selector; this one did not.
|
|
556
|
+
selector: n.selector,
|
|
483
557
|
x: n.x,
|
|
484
558
|
y: n.y,
|
|
485
559
|
...n.frame === true ? { frame: true } : {},
|
|
@@ -554,7 +628,9 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
554
628
|
if (browser === undefined) throw new Error('tool-browser: browser service unavailable')
|
|
555
629
|
const session = await ensureSession(browser, state, taskKey(exec), agentOf(exec))
|
|
556
630
|
const result = await browser.waitFor(session, {
|
|
557
|
-
|
|
631
|
+
// Capped so an over-budget argument returns the documented verdict (see maxWaitMs)
|
|
632
|
+
// rather than aborting the whole tool call.
|
|
633
|
+
...args.timeoutMs !== undefined ? { timeoutMs: Math.min(args.timeoutMs, maxWaitMs) } : {},
|
|
558
634
|
...args.url !== undefined ? { url: args.url } : {},
|
|
559
635
|
...args.selector !== undefined ? { selector: args.selector } : {},
|
|
560
636
|
}, exec.signal)
|
|
@@ -579,7 +655,18 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
579
655
|
exception: { type: 'string' },
|
|
580
656
|
},
|
|
581
657
|
},
|
|
582
|
-
|
|
658
|
+
// A return value is whatever the page produced, and `document.body.outerHTML` measured
|
|
659
|
+
// 157,350 characters — ~39-52k tokens from one call. Bounded, with the cut stated so the
|
|
660
|
+
// caller knows to narrow the expression rather than the value being silently short.
|
|
661
|
+
render: (_args, value) => {
|
|
662
|
+
if (!value.ok) return [{ type: 'text', text: `Exception: ${value.exception}` }]
|
|
663
|
+
const rendered = String(value.value ?? '')
|
|
664
|
+
const cap = 50_000
|
|
665
|
+
const text = rendered.length > cap
|
|
666
|
+
? `Result: ${rendered.slice(0, cap)}\n(result truncated at ${cap} of ${rendered.length} characters — return a smaller value, or write it with browser_execute + a variable)`
|
|
667
|
+
: `Result: ${rendered}`
|
|
668
|
+
return [{ type: 'text', text }]
|
|
669
|
+
},
|
|
583
670
|
},
|
|
584
671
|
timeoutMs,
|
|
585
672
|
isConcurrencySafe: () => false, // page JS can be stateful
|
|
@@ -619,7 +706,7 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
619
706
|
truncated: { type: 'boolean', required: true },
|
|
620
707
|
},
|
|
621
708
|
},
|
|
622
|
-
render: (_args, value) => [{ type: 'text', text: value.content + (value.truncated ?
|
|
709
|
+
render: (_args, value) => [{ type: 'text', text: value.content + (value.truncated ? `\n(content truncated — narrow it with selector= or raise maxChars)` : '') }],
|
|
623
710
|
},
|
|
624
711
|
timeoutMs,
|
|
625
712
|
isConcurrencySafe: () => true,
|
|
@@ -665,7 +752,10 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
665
752
|
},
|
|
666
753
|
render: (_args, value) => [{
|
|
667
754
|
type: 'text',
|
|
668
|
-
|
|
755
|
+
// Compact, unlike the pretty-printed form this used to emit: two-space indentation
|
|
756
|
+
// carries no information a model can use and cost 13.7% of a measured 39,559-character
|
|
757
|
+
// result. The count leads so the caller can tell a short list from a capped one.
|
|
758
|
+
text: `${value.count} item(s):\n${JSON.stringify(value.items)}`,
|
|
669
759
|
}],
|
|
670
760
|
},
|
|
671
761
|
timeoutMs,
|
|
@@ -720,12 +810,13 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
720
810
|
if (target !== undefined && typeof target.value === 'string' && target.value === '') {
|
|
721
811
|
throw new Error('target.value is empty; pass a css selector, visible text or XPath, or omit target entirely to use x/y')
|
|
722
812
|
}
|
|
813
|
+
const index = targetIndex(target)
|
|
723
814
|
if (target !== undefined && typeof target.value === 'string' && target.value !== '') {
|
|
724
815
|
await browser.click(session, {
|
|
725
816
|
target: {
|
|
726
817
|
by: (target.by === 'text' || target.by === 'xpath' ? target.by : 'css') as 'css' | 'text' | 'xpath',
|
|
727
818
|
value: target.value,
|
|
728
|
-
...
|
|
819
|
+
...index !== undefined ? { index } : {},
|
|
729
820
|
},
|
|
730
821
|
}, exec.signal)
|
|
731
822
|
} else {
|
|
@@ -772,13 +863,14 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
772
863
|
if (target !== undefined && typeof target.value === 'string' && target.value === '') {
|
|
773
864
|
throw new Error('target.value is empty; pass a css selector, visible text or XPath, or omit target entirely to use x/y')
|
|
774
865
|
}
|
|
866
|
+
const index = targetIndex(target)
|
|
775
867
|
if (target !== undefined && typeof target.value === 'string' && target.value !== '') {
|
|
776
868
|
await browser.type(session, {
|
|
777
869
|
text: args.text,
|
|
778
870
|
target: {
|
|
779
871
|
by: (target.by === 'text' || target.by === 'xpath' ? target.by : 'css') as 'css' | 'text' | 'xpath',
|
|
780
872
|
value: target.value,
|
|
781
|
-
...
|
|
873
|
+
...index !== undefined ? { index } : {},
|
|
782
874
|
},
|
|
783
875
|
}, exec.signal)
|
|
784
876
|
} else {
|
|
@@ -966,7 +1058,11 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
966
1058
|
const browser = ctx.get('browser')
|
|
967
1059
|
if (browser === undefined) throw new Error('tool-browser: browser service unavailable')
|
|
968
1060
|
const session = await ensureSession(browser, state, taskKey(exec), agentOf(exec))
|
|
969
|
-
const
|
|
1061
|
+
const requested = args.fields ?? []
|
|
1062
|
+
if (requested.length > maxFillFields) {
|
|
1063
|
+
throw new Error(`browser_fill: ${requested.length} fields in one call exceeds the ${maxFillFields}-field limit; split it into several calls so each one can report per-field results instead of timing out as a batch`)
|
|
1064
|
+
}
|
|
1065
|
+
const fields = requested.map((f: { selector?: string; name?: string; label?: string; placeholder?: string; kind?: string; value?: string }) => {
|
|
970
1066
|
// A missing value is a caller mistake, not a request to clear the field. Turning
|
|
971
1067
|
// it into '' silently unchecks a checkbox or empties a text input while the tool
|
|
972
1068
|
// still reports success, so it fails loudly. The schema marks value required, but
|
|
@@ -1197,7 +1293,7 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
1197
1293
|
|
|
1198
1294
|
ctx.tools.register(defineTool({
|
|
1199
1295
|
name: 'browser_screenshot',
|
|
1200
|
-
description: 'Capture the current shared-browser page as a screenshot (PNG default, JPEG optional). This is for models that can read images: layout checks, charts, designs, CAPTCHAs, or locating an element by eye before clicking its coordinates. A model without image input gains nothing from it — browser_snapshot, browser_a11y and browser_content carry the same page as text, and browser_scrape extracts structured data. Supports full-page capture, save-to-file, JPEG encoding, and downscaling (maxWidth/maxHeight) to cut vision-tool token cost. JPEG needs a browser whose CDP encoder works — the self-hosted browser and an installed Chrome/Edge; the desktop sidebar
|
|
1296
|
+
description: 'Capture the current shared-browser page as a screenshot (PNG default, JPEG optional). This is for models that can read images: layout checks, charts, designs, CAPTCHAs, or locating an element by eye before clicking its coordinates. A model without image input gains nothing from it — browser_snapshot, browser_a11y and browser_content carry the same page as text, and browser_scrape extracts structured data. Supports full-page capture, save-to-file, JPEG encoding, and downscaling (maxWidth/maxHeight) to cut vision-tool token cost. JPEG needs a browser whose CDP encoder works — the self-hosted browser and an installed Chrome/Edge; some carriers (the desktop sidebar) return PNG instead. Downscaling works on every carrier.',
|
|
1201
1297
|
parameters: {
|
|
1202
1298
|
fullPage: { type: 'boolean', description: 'Capture the full scrollable page instead of the viewport (default false).' },
|
|
1203
1299
|
savePath: { type: 'string', description: 'Absolute file path to also save the image to (e.g. for read_image vision location). Must resolve inside the configured downloadDir (default: the system Downloads folder, localized names such as ~/下载 included); an existing file is never overwritten.' },
|
|
@@ -1211,7 +1307,9 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
1211
1307
|
type: 'object',
|
|
1212
1308
|
additionalProperties: false,
|
|
1213
1309
|
properties: {
|
|
1214
|
-
dataUrl: { type: 'string', required: true, description: 'Base64
|
|
1310
|
+
dataUrl: { type: 'string', required: true, description: 'Base64 data URL of the screenshot (PNG, or JPEG when the carrier uses CDP JPEG).' },
|
|
1311
|
+
width: { type: 'number', description: 'Pixel width of the returned image, when the capture was measured (present with maxWidth/maxHeight).' },
|
|
1312
|
+
height: { type: 'number', description: 'Pixel height of the returned image, when the capture was measured.' },
|
|
1215
1313
|
path: { type: 'string', description: 'The file path the screenshot was saved to, when savePath was given.' },
|
|
1216
1314
|
},
|
|
1217
1315
|
},
|
|
@@ -1348,7 +1446,9 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
1348
1446
|
ctx.tools.register(defineTool({
|
|
1349
1447
|
name: 'browser_history',
|
|
1350
1448
|
description: 'List the shared browser session\'s recorded operation history (navigate/execute/click/type), newest last, with per-step success/error. Use to understand what the agent did and to pick a step to replay.',
|
|
1351
|
-
parameters: {
|
|
1449
|
+
parameters: {
|
|
1450
|
+
verbose: { type: 'boolean', description: "Include each operation's parameters (default false). They are what you just sent, so they are omitted unless asked for." },
|
|
1451
|
+
},
|
|
1352
1452
|
output: {
|
|
1353
1453
|
schema: {
|
|
1354
1454
|
type: 'object',
|
|
@@ -1372,17 +1472,20 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
1372
1472
|
},
|
|
1373
1473
|
},
|
|
1374
1474
|
},
|
|
1375
|
-
render: (
|
|
1475
|
+
render: (args, value) => {
|
|
1376
1476
|
const entries = value.entries as Array<{ seq: number; action: string; ok: boolean; params: Record<string, unknown>; result?: string; error?: string }>
|
|
1377
1477
|
if (entries.length === 0) return [{ type: 'text', text: '(no recorded operations yet)' }]
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1478
|
+
// The parameters are what the caller itself just sent, so echoing them back is cost
|
|
1479
|
+
// without information — measured at 18,162 characters for 200 operations, of which the
|
|
1480
|
+
// per-entry params dominated. Only the tail is shown by default: the recent steps are
|
|
1481
|
+
// what a caller reasons about, and the count leads so the tail is not mistaken for all.
|
|
1482
|
+
const recent = entries.slice(-20)
|
|
1483
|
+
const omitted = entries.length - recent.length
|
|
1484
|
+
const lines = recent.map(e => `#${e.seq} ${e.action} ${e.ok ? 'ok' : 'FAIL'}${e.result !== undefined ? ` -> ${e.result}` : ''}${e.error !== undefined ? ` !! ${e.error}` : ''}${args.verbose === true ? ` ${JSON.stringify(e.params)}` : ''}`)
|
|
1485
|
+
const header = omitted > 0
|
|
1486
|
+
? `${entries.length} operations recorded; showing the last ${recent.length} (${args.verbose === true ? '' : ' (pass verbose: true for parameters)'}):`
|
|
1487
|
+
: `${entries.length} operation(s):`
|
|
1488
|
+
return [{ type: 'text', text: `${header}\n${lines.join('\n')}` }]
|
|
1386
1489
|
},
|
|
1387
1490
|
},
|
|
1388
1491
|
timeoutMs,
|
|
@@ -1470,15 +1573,20 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
1470
1573
|
},
|
|
1471
1574
|
render: (_args, value) => {
|
|
1472
1575
|
const entries = value.entries as Array<{ at: number; url: string; title?: string; session?: string }>
|
|
1576
|
+
let lastSession: string | undefined
|
|
1473
1577
|
if (entries.length === 0) {
|
|
1474
1578
|
return [{ type: 'text', text: '(no recorded visits — nothing has been browsed yet, or history recording is turned off in settings)' }]
|
|
1475
1579
|
}
|
|
1476
1580
|
return [{
|
|
1477
1581
|
type: 'text',
|
|
1478
1582
|
text: entries.map(entry => {
|
|
1479
|
-
const when = new Date(entry.at).toISOString().replace('T', ' ').slice(0,
|
|
1583
|
+
const when = new Date(entry.at).toISOString().replace('T', ' ').slice(0, 16)
|
|
1480
1584
|
const title = entry.title !== undefined ? `${entry.title} — ` : ''
|
|
1481
|
-
|
|
1585
|
+
const tag = entry.session !== undefined && entry.session !== lastSession
|
|
1586
|
+
? ` [session ${entry.session.slice(0, 8)}]`
|
|
1587
|
+
: ''
|
|
1588
|
+
lastSession = entry.session
|
|
1589
|
+
return `${when} ${title}${entry.url}${tag}`
|
|
1482
1590
|
}).join('\n'),
|
|
1483
1591
|
}]
|
|
1484
1592
|
},
|
|
@@ -1628,7 +1736,7 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
1628
1736
|
|
|
1629
1737
|
ctx.tools.register(defineTool({
|
|
1630
1738
|
name: 'browser_restrict',
|
|
1631
|
-
description: 'Restrict which browser actions are allowed, to prevent stray clicks/navigation. Pass a list of browser tool names (e.g. ["browser_snapshot","browser_content","browser_click"]) — any other browser_* call is refused. Pass an empty list or omit to lift the restriction.
|
|
1739
|
+
description: 'Restrict which browser actions are allowed, to prevent stray clicks/navigation. Pass a list of browser tool names (e.g. ["browser_snapshot","browser_content","browser_click"]) — any other browser_* call is refused. Pass an empty list or omit to lift the restriction. Never blocked, whatever the list says: the observing tools (snapshot/a11y/content/scrape/screenshot/get_value/wait/challenge/list_tabs/session/history/visited) and the three that lift a restriction or recover from a bad state (restrict/reset/reset_session) — so a restriction can always be read around and undone. IMPORTANT: this is a SOFT guardrail against accidental actions, NOT a security boundary — you (the model) can lift it yourself with an empty list.',
|
|
1632
1740
|
parameters: {
|
|
1633
1741
|
allowed: {
|
|
1634
1742
|
type: 'array',
|
|
@@ -1662,7 +1770,7 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
1662
1770
|
|
|
1663
1771
|
ctx.tools.register(defineTool({
|
|
1664
1772
|
name: 'browser_auth',
|
|
1665
|
-
description: 'Export or restore the browser session\'s cookies (login state). Use "flush" to get a JSON cookie list (save it to a private file to persist logins), or "restore" with that list to put logins back (e.g. after the browser host restarted). Available on the self-hosted browser. Exported cookies are LIVE CREDENTIALS: treat them as secrets — do not echo them into the conversation, keep them out of logs, and store the list in a private file.',
|
|
1773
|
+
description: 'Export or restore the browser session\'s cookies (login state). Use "flush" to get a JSON cookie list (save it to a private file to persist logins), or "restore" with that list to put logins back (e.g. after the browser host restarted). Available on all three carriers: the self-hosted browser reads its own session, while the desktop sidebar and an installed Chrome/Edge go through CDP. Refused entirely when the credential switch is off. Exported cookies are LIVE CREDENTIALS: treat them as secrets — do not echo them into the conversation, keep them out of logs, and store the list in a private file.',
|
|
1666
1774
|
parameters: {
|
|
1667
1775
|
action: { type: 'string', required: true, enum: ['flush', 'restore'], description: 'flush = export cookies; restore = import cookies.' },
|
|
1668
1776
|
cookies: { type: 'array', items: { type: 'object', additionalProperties: true }, description: 'Cookie list to restore (required when action=restore).' },
|
|
@@ -1693,7 +1801,7 @@ export function apply(ctx: Context, config: Config = {}): void {
|
|
|
1693
1801
|
// empty call returned {restored: 0} and rendered 'Restored 0 cookies.', which
|
|
1694
1802
|
// reads as success and would leave the caller believing the login was restored.
|
|
1695
1803
|
if (!Array.isArray(args.cookies) || args.cookies.length === 0) {
|
|
1696
|
-
throw new Error('browser_auth: action "restore" needs a non-empty "cookies" array (get one from action "
|
|
1804
|
+
throw new Error('browser_auth: action "restore" needs a non-empty "cookies" array (get one from action "flush" on the same profile)')
|
|
1697
1805
|
}
|
|
1698
1806
|
const list = args.cookies as unknown[]
|
|
1699
1807
|
const restored = await browser.restoreAuth(session, list as never)
|