dsh-builtin-browser 0.4.1 → 0.4.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1699 -0
- package/README.en.md +114 -33
- package/README.md +108 -32
- package/client.js +478 -29
- package/cordis.patch.yml +21 -0
- package/desktop-bridge/install.mjs +66 -6
- package/desktop-bridge/plugin-browser-bridge.js +426 -167
- package/docs/architecture.md +14 -4
- package/docs/tool-reference.md +13 -13
- package/docs/user-guide.md +58 -3
- package/docs/why-browser.md +1 -1
- package/lib/browser/types.d.ts +34 -3
- package/lib/browser-electron/bridge-connection.d.ts +49 -26
- package/lib/browser-electron/bridge-connection.js +185 -22
- package/lib/browser-electron/desktop-bridge-host.d.ts +69 -36
- package/lib/browser-electron/desktop-bridge-host.js +151 -86
- package/lib/browser-electron/entry.d.ts +1 -1
- package/lib/browser-electron/entry.js +133 -40
- package/lib/browser-electron/ephemeral-profile.d.ts +37 -0
- package/lib/browser-electron/ephemeral-profile.js +146 -0
- package/lib/browser-electron/history-store.d.ts +36 -2
- package/lib/browser-electron/history-store.js +76 -7
- package/lib/browser-electron/host-main.js +86 -28
- package/lib/browser-electron/provider.d.ts +106 -3
- package/lib/browser-electron/provider.js +407 -60
- package/lib/browser-electron/remote-host.d.ts +52 -3
- package/lib/browser-electron/remote-host.js +259 -25
- package/lib/browser-electron/rpc-ops.d.ts +18 -0
- package/lib/browser-electron/rpc-ops.js +36 -0
- package/lib/browser-electron/settings-route.d.ts +28 -0
- package/lib/browser-electron/settings-route.js +68 -0
- package/lib/browser-electron/settings-store.d.ts +40 -0
- package/lib/browser-electron/settings-store.js +160 -23
- package/lib/browser-electron/system-browser.d.ts +102 -12
- package/lib/browser-electron/system-browser.js +411 -51
- package/lib/index.js +0 -1
- package/lib/tool-browser/element-fields.d.ts +145 -0
- package/lib/tool-browser/element-fields.js +152 -0
- package/lib/tool-browser/index.js +149 -74
- package/package.json +10 -4
- package/scripts/verify-host-compat.mjs +195 -0
- package/src/browser/types.ts +34 -3
- package/src/browser-electron/bridge-connection.ts +179 -22
- package/src/browser-electron/desktop-bridge-host.ts +146 -84
- package/src/browser-electron/entry.ts +132 -39
- package/src/browser-electron/ephemeral-profile.ts +152 -0
- package/src/browser-electron/history-store.ts +81 -9
- package/src/browser-electron/host-main.ts +115 -36
- package/src/browser-electron/provider.ts +492 -69
- package/src/browser-electron/remote-host.ts +276 -38
- package/src/browser-electron/rpc-ops.ts +41 -0
- package/src/browser-electron/settings-route.ts +87 -0
- package/src/browser-electron/settings-store.ts +173 -24
- package/src/browser-electron/system-browser.ts +408 -39
- package/src/index.ts +3 -2
- package/src/tool-browser/element-fields.ts +167 -0
- package/src/tool-browser/index.ts +155 -76
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
* @module dsh-browser/tool-browser
|
|
15
15
|
*/
|
|
16
16
|
import { defineTool } from '@deepseek-ai/dsh-tools';
|
|
17
|
+
import { ELEMENT_ITEM_SCHEMA, NODE_ITEM_SCHEMA, projectElement, projectNode, } from './element-fields.js';
|
|
17
18
|
/**
|
|
18
19
|
* Register a cleanup that runs once when the process exits.
|
|
19
20
|
*
|
|
@@ -66,7 +67,10 @@ export const READ_ONLY_TOOLS = new Set([
|
|
|
66
67
|
'browser_list_tabs', 'browser_session', 'browser_history', 'browser_visited',
|
|
67
68
|
// browser_auth reads cookies out of the browser: an observation, and one the
|
|
68
69
|
// credentials switch gates on its own (see credentials.allowRead in the settings).
|
|
69
|
-
|
|
70
|
+
// browser_auth is deliberately NOT here. It reads cookies on "flush" but WRITES them on
|
|
71
|
+
// "restore", to arbitrary domains, which is an action like any other — exempting the tool
|
|
72
|
+
// would let a task that restricted its own actions rewrite the shared browser's logins.
|
|
73
|
+
//
|
|
70
74
|
// Getting out of a bad state, or lifting the restriction itself. Without these a task
|
|
71
75
|
// that restricted everything could neither recover nor be released.
|
|
72
76
|
'browser_restrict', 'browser_reset_session', 'browser_reset',
|
|
@@ -94,6 +98,26 @@ function assertAllowed(state, toolName, task) {
|
|
|
94
98
|
function agentOf(exec) {
|
|
95
99
|
return exec?.agent;
|
|
96
100
|
}
|
|
101
|
+
/**
|
|
102
|
+
* Read a target's `index`, refusing the values that cannot mean anything.
|
|
103
|
+
*
|
|
104
|
+
* The index is looked up in the page (`els[index] ?? null`), so a negative one — `-1`, the
|
|
105
|
+
* common "give me the last one" shorthand — matches nothing. The page script then treated
|
|
106
|
+
* that as "not on the page yet" and polled for its whole 10s budget before reporting
|
|
107
|
+
* "element not found", which reads like a slow page rather than a bad argument. Refusing it
|
|
108
|
+
* here names the real problem immediately.
|
|
109
|
+
* @param target - the caller's target, or undefined when there is none.
|
|
110
|
+
* @returns the index, or undefined to let the page default to 0.
|
|
111
|
+
*/
|
|
112
|
+
function targetIndex(target) {
|
|
113
|
+
const index = target?.index;
|
|
114
|
+
if (index === undefined)
|
|
115
|
+
return undefined;
|
|
116
|
+
if (typeof index !== 'number' || !Number.isSafeInteger(index) || index < 0) {
|
|
117
|
+
throw new Error(`target.index must be a non-negative integer (got ${JSON.stringify(index)}); it is a 0-based position among the matches, and negative values such as -1 match nothing — pass the position you want, or narrow the selector`);
|
|
118
|
+
}
|
|
119
|
+
return index;
|
|
120
|
+
}
|
|
97
121
|
/**
|
|
98
122
|
* The task key for a tool call: the calling DSH session id, or the shared
|
|
99
123
|
* default key when the call carries no agent context (CLI probes, tests).
|
|
@@ -191,7 +215,7 @@ function parseFillValue(v) {
|
|
|
191
215
|
*/
|
|
192
216
|
function formatSnapshot(snapshot, options = {}) {
|
|
193
217
|
const showCoords = options.coords === true;
|
|
194
|
-
const lines = snapshot.elements.map(el => `[${el.ref}] ${el.kind}: ${el.label}${el.frame === true ? ' (iframe)' : ''}${showCoords ? ` (${el.x},${el.y})` : ''}`);
|
|
218
|
+
const lines = snapshot.elements.map(el => `[${el.ref}] ${el.kind}: ${el.label}${el.selector !== undefined && el.selector !== '' ? ` {${el.selector}}` : ''}${el.frame === true ? ' (iframe)' : ''}${showCoords ? ` (${el.x},${el.y})` : ''}`);
|
|
195
219
|
const header = `URL: ${snapshot.url}${snapshot.title !== undefined ? `\nTitle: ${snapshot.title}` : ''}`;
|
|
196
220
|
const body = lines.length > 0 ? lines.join('\n') : '(no interactive elements found)';
|
|
197
221
|
const tail = snapshot.truncated === true ? '\n(snapshot truncated)' : '';
|
|
@@ -203,6 +227,25 @@ function formatSnapshot(snapshot, options = {}) {
|
|
|
203
227
|
/** Register all browser tools with `ctx.tools`. */
|
|
204
228
|
export function apply(ctx, config = {}) {
|
|
205
229
|
const timeoutMs = config.timeoutMs ?? 60_000;
|
|
230
|
+
/**
|
|
231
|
+
* Longest wait `browser_wait` may poll for, leaving room for the tool call itself to
|
|
232
|
+
* finish and report.
|
|
233
|
+
*
|
|
234
|
+
* `browser_wait` promises a VERDICT — `{ready:false, reason}` — and the enclosing tool
|
|
235
|
+
* call aborts at `timeoutMs`. A caller-supplied budget longer than that aborted the call
|
|
236
|
+
* instead, so the documented verdict never arrived. Clamping to just inside the budget is
|
|
237
|
+
* what keeps the promise: the polling stops, the reason is returned.
|
|
238
|
+
*/
|
|
239
|
+
const maxWaitMs = Math.max(250, timeoutMs - 5_000);
|
|
240
|
+
/**
|
|
241
|
+
* Most fields `browser_fill` accepts in one batch.
|
|
242
|
+
*
|
|
243
|
+
* The batch runs as ONE page evaluation, so hundreds of fields do not produce hundreds of
|
|
244
|
+
* per-field verdicts — they produce a single evaluate that exceeds the tool budget and
|
|
245
|
+
* throws, which is the opposite of the documented "per-field failures are reported instead
|
|
246
|
+
* of throwing". Real forms are far smaller than this; anything larger should be split.
|
|
247
|
+
*/
|
|
248
|
+
const maxFillFields = 200;
|
|
206
249
|
// Per-context state: sessions, in-flight opens, and the restriction are
|
|
207
250
|
// scoped to THIS plugin apply, so parallel contexts never share sessions
|
|
208
251
|
// or leak restrictions into each other.
|
|
@@ -236,18 +279,8 @@ export function apply(ctx, config = {}) {
|
|
|
236
279
|
elements: {
|
|
237
280
|
type: 'array',
|
|
238
281
|
required: true,
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
additionalProperties: false,
|
|
242
|
-
properties: {
|
|
243
|
-
ref: { type: 'number', required: true },
|
|
244
|
-
kind: { type: 'string', required: true },
|
|
245
|
-
label: { type: 'string', required: true },
|
|
246
|
-
x: { type: 'number', required: true },
|
|
247
|
-
y: { type: 'number', required: true },
|
|
248
|
-
frame: { type: 'boolean' },
|
|
249
|
-
},
|
|
250
|
-
},
|
|
282
|
+
// Same shared item schema as browser_open's, so the two cannot drift apart.
|
|
283
|
+
items: ELEMENT_ITEM_SCHEMA,
|
|
251
284
|
},
|
|
252
285
|
challenge: {
|
|
253
286
|
type: 'object',
|
|
@@ -281,7 +314,7 @@ export function apply(ctx, config = {}) {
|
|
|
281
314
|
return {
|
|
282
315
|
url: snapshot.url,
|
|
283
316
|
...snapshot.title !== undefined ? { title: snapshot.title } : {},
|
|
284
|
-
elements: snapshot.elements.map(el => (
|
|
317
|
+
elements: snapshot.elements.map(el => projectElement(el)),
|
|
285
318
|
truncated: snapshot.truncated,
|
|
286
319
|
...snapshot.challenge !== undefined ? { challenge: snapshot.challenge } : {},
|
|
287
320
|
};
|
|
@@ -314,6 +347,12 @@ export function apply(ctx, config = {}) {
|
|
|
314
347
|
x: { type: 'number', required: true },
|
|
315
348
|
y: { type: 'number', required: true },
|
|
316
349
|
frame: { type: 'boolean' },
|
|
350
|
+
// The page-side script computes this for every element and the tool layer
|
|
351
|
+
// passes it through. Without it in the schema, a declared
|
|
352
|
+
// additionalProperties: false rejects the whole result at run time — which is
|
|
353
|
+
// how this was found, on the very first real call, after a static review had
|
|
354
|
+
// called the change verified.
|
|
355
|
+
selector: { type: 'string' },
|
|
317
356
|
},
|
|
318
357
|
},
|
|
319
358
|
},
|
|
@@ -341,7 +380,7 @@ export function apply(ctx, config = {}) {
|
|
|
341
380
|
return {
|
|
342
381
|
url: snapshot.url,
|
|
343
382
|
...snapshot.title !== undefined ? { title: snapshot.title } : {},
|
|
344
|
-
elements: snapshot.elements.map(el => (
|
|
383
|
+
elements: snapshot.elements.map(el => projectElement(el)),
|
|
345
384
|
truncated: snapshot.truncated,
|
|
346
385
|
...snapshot.challenge !== undefined ? { challenge: snapshot.challenge } : {},
|
|
347
386
|
};
|
|
@@ -349,10 +388,10 @@ export function apply(ctx, config = {}) {
|
|
|
349
388
|
}));
|
|
350
389
|
ctx.tools.register(defineTool({
|
|
351
390
|
name: 'browser_a11y',
|
|
352
|
-
description: 'Read the page\'s accessibility tree: every interactive node with its semantic role (button/link/textbox/checkbox/…), accessible name, current value, and states (enabled/disabled/checked/expanded/…),
|
|
391
|
+
description: 'Read the page\'s accessibility tree: every interactive node with its semantic role (button/link/textbox/checkbox/…), accessible name, current value, and states (enabled/disabled/checked/expanded/…), (coordinates only when coords: true). Prefer this over browser_snapshot to understand a page\'s structure and find the right element: roles and names tell you WHAT each node is, and the name is what browser_click matches when you target by text. Penetrates same-origin iframes and shadow roots.',
|
|
353
392
|
parameters: {
|
|
354
393
|
includeHidden: { type: 'boolean', description: 'Include hidden elements (default false).' },
|
|
355
|
-
maxNodes: { type: 'number', description: 'Maximum nodes (default
|
|
394
|
+
maxNodes: { type: 'number', description: 'Maximum nodes (default 150, range 10-5000).' },
|
|
356
395
|
coords: { type: 'boolean', description: 'Include each element\'s viewport coordinates. Off by default: a semantic target needs no coordinates, and they are pure noise unless a vision pass is going to click a pixel position.' },
|
|
357
396
|
},
|
|
358
397
|
output: {
|
|
@@ -367,22 +406,8 @@ export function apply(ctx, config = {}) {
|
|
|
367
406
|
nodes: {
|
|
368
407
|
type: 'array',
|
|
369
408
|
required: true,
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
additionalProperties: false,
|
|
373
|
-
properties: {
|
|
374
|
-
ref: { type: 'number', required: true },
|
|
375
|
-
role: { type: 'string', required: true },
|
|
376
|
-
name: { type: 'string', required: true },
|
|
377
|
-
value: { type: 'string' },
|
|
378
|
-
states: { type: 'array', required: true, items: { type: 'string' } },
|
|
379
|
-
depth: { type: 'number', required: true },
|
|
380
|
-
tag: { type: 'string', required: true },
|
|
381
|
-
x: { type: 'number', required: true },
|
|
382
|
-
y: { type: 'number', required: true },
|
|
383
|
-
frame: { type: 'boolean' },
|
|
384
|
-
},
|
|
385
|
-
},
|
|
409
|
+
// Same shared table as the node projection.
|
|
410
|
+
items: NODE_ITEM_SCHEMA,
|
|
386
411
|
},
|
|
387
412
|
},
|
|
388
413
|
},
|
|
@@ -396,13 +421,22 @@ export function apply(ctx, config = {}) {
|
|
|
396
421
|
const showCoords = args.coords === true;
|
|
397
422
|
const lines = nodes.map(n => {
|
|
398
423
|
const valuePart = n.value !== undefined && n.value !== null ? ` value="${String(n.value).slice(0, 60)}"` : '';
|
|
399
|
-
|
|
424
|
+
// One space per level, not two: indentation was 19.7% of a measured 38,953-character
|
|
425
|
+
// result, and DOM depth is not a11y depth — deep wrapper divs push it to the cap.
|
|
426
|
+
const indent = ' '.repeat(Math.max(0, Math.min(8, Number(n.depth) || 0)));
|
|
400
427
|
const coordsPart = showCoords ? ` (${n.x},${n.y})` : '';
|
|
401
|
-
|
|
402
|
-
|
|
428
|
+
// Only states that are not the default. `states=[enabled]` was printed on 405 of 500
|
|
429
|
+
// nodes whose state WAS the default — 22% of the output for no information. Absence
|
|
430
|
+
// now means "enabled", which the description states.
|
|
431
|
+
const notableStates = Array.isArray(n.states) ? n.states.filter((state) => state !== 'enabled') : [];
|
|
432
|
+
const statePart = notableStates.length > 0 ? ` states=[${notableStates.join(',')}]` : '';
|
|
433
|
+
// The provider computes this for a node with an id or a name; printing it is what
|
|
434
|
+
// makes the node something the caller can point at instead of guessing at its text.
|
|
435
|
+
const selectorPart = typeof n.selector === 'string' && n.selector !== '' ? ` {${n.selector}}` : '';
|
|
436
|
+
return `${indent}[${n.ref}] ${n.role} "${n.name}"${valuePart}${coordsPart}${statePart}${selectorPart}${n.frame === true ? ' (iframe)' : ''}`;
|
|
403
437
|
});
|
|
404
438
|
const header = `URL: ${value.url}${value.title !== undefined ? `\nTitle: ${value.title}` : ''}`;
|
|
405
|
-
const hint = showCoords ? '' : '\n(
|
|
439
|
+
const hint = showCoords ? '' : '\n(no coords; pass coords: true)';
|
|
406
440
|
return [{ type: 'text', text: `${header}\n\n${lines.length > 0 ? lines.join('\n') : '(no accessible interactive nodes)'}${value.truncated === true ? '\n(truncated)' : ''}${hint}` }];
|
|
407
441
|
},
|
|
408
442
|
},
|
|
@@ -422,18 +456,18 @@ export function apply(ctx, config = {}) {
|
|
|
422
456
|
...result.title !== undefined ? { title: result.title } : {},
|
|
423
457
|
count: result.count,
|
|
424
458
|
truncated: result.truncated,
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
459
|
+
// The projection drops `value: null` (the schema declares a string, and the contract is
|
|
460
|
+
// that an absent value means "not applicable"), so the run-time shape and the inferred
|
|
461
|
+
// type differ by exactly that. The field table is what keeps the schema and the
|
|
462
|
+
// projection in agreement; this assertion is the type-level echo of that.
|
|
463
|
+
// The projection and the schema are now generated from one field table, so the shape
|
|
464
|
+
// they agree on is guaranteed at run time. TypeScript cannot see that agreement — it
|
|
465
|
+
// infers the output type from the schema object and compares it against the provider's
|
|
466
|
+
// interface, which describes the provider's internal shape rather than the contract
|
|
467
|
+
// (`value` nullable internally, absent on the wire; `selector` required by the
|
|
468
|
+
// interface, optional in the inferred schema). The assertion bridges that last gap; the
|
|
469
|
+
// test that walks every registered tool is what checks the agreement for real.
|
|
470
|
+
nodes: result.nodes.map(n => projectNode(n)),
|
|
437
471
|
};
|
|
438
472
|
},
|
|
439
473
|
}));
|
|
@@ -504,7 +538,9 @@ export function apply(ctx, config = {}) {
|
|
|
504
538
|
throw new Error('tool-browser: browser service unavailable');
|
|
505
539
|
const session = await ensureSession(browser, state, taskKey(exec), agentOf(exec));
|
|
506
540
|
const result = await browser.waitFor(session, {
|
|
507
|
-
|
|
541
|
+
// Capped so an over-budget argument returns the documented verdict (see maxWaitMs)
|
|
542
|
+
// rather than aborting the whole tool call.
|
|
543
|
+
...args.timeoutMs !== undefined ? { timeoutMs: Math.min(args.timeoutMs, maxWaitMs) } : {},
|
|
508
544
|
...args.url !== undefined ? { url: args.url } : {},
|
|
509
545
|
...args.selector !== undefined ? { selector: args.selector } : {},
|
|
510
546
|
}, exec.signal);
|
|
@@ -528,7 +564,19 @@ export function apply(ctx, config = {}) {
|
|
|
528
564
|
exception: { type: 'string' },
|
|
529
565
|
},
|
|
530
566
|
},
|
|
531
|
-
|
|
567
|
+
// A return value is whatever the page produced, and `document.body.outerHTML` measured
|
|
568
|
+
// 157,350 characters — ~39-52k tokens from one call. Bounded, with the cut stated so the
|
|
569
|
+
// caller knows to narrow the expression rather than the value being silently short.
|
|
570
|
+
render: (_args, value) => {
|
|
571
|
+
if (!value.ok)
|
|
572
|
+
return [{ type: 'text', text: `Exception: ${value.exception}` }];
|
|
573
|
+
const rendered = String(value.value ?? '');
|
|
574
|
+
const cap = 50_000;
|
|
575
|
+
const text = rendered.length > cap
|
|
576
|
+
? `Result: ${rendered.slice(0, cap)}\n(result truncated at ${cap} of ${rendered.length} characters — return a smaller value, or write it with browser_execute + a variable)`
|
|
577
|
+
: `Result: ${rendered}`;
|
|
578
|
+
return [{ type: 'text', text }];
|
|
579
|
+
},
|
|
532
580
|
},
|
|
533
581
|
timeoutMs,
|
|
534
582
|
isConcurrencySafe: () => false, // page JS can be stateful
|
|
@@ -568,7 +616,7 @@ export function apply(ctx, config = {}) {
|
|
|
568
616
|
truncated: { type: 'boolean', required: true },
|
|
569
617
|
},
|
|
570
618
|
},
|
|
571
|
-
render: (_args, value) => [{ type: 'text', text: value.content + (value.truncated ?
|
|
619
|
+
render: (_args, value) => [{ type: 'text', text: value.content + (value.truncated ? `\n(content truncated — narrow it with selector= or raise maxChars)` : '') }],
|
|
572
620
|
},
|
|
573
621
|
timeoutMs,
|
|
574
622
|
isConcurrencySafe: () => true,
|
|
@@ -614,7 +662,10 @@ export function apply(ctx, config = {}) {
|
|
|
614
662
|
},
|
|
615
663
|
render: (_args, value) => [{
|
|
616
664
|
type: 'text',
|
|
617
|
-
|
|
665
|
+
// Compact, unlike the pretty-printed form this used to emit: two-space indentation
|
|
666
|
+
// carries no information a model can use and cost 13.7% of a measured 39,559-character
|
|
667
|
+
// result. The count leads so the caller can tell a short list from a capped one.
|
|
668
|
+
text: `${value.count} item(s):\n${JSON.stringify(value.items)}`,
|
|
618
669
|
}],
|
|
619
670
|
},
|
|
620
671
|
timeoutMs,
|
|
@@ -670,12 +721,13 @@ export function apply(ctx, config = {}) {
|
|
|
670
721
|
if (target !== undefined && typeof target.value === 'string' && target.value === '') {
|
|
671
722
|
throw new Error('target.value is empty; pass a css selector, visible text or XPath, or omit target entirely to use x/y');
|
|
672
723
|
}
|
|
724
|
+
const index = targetIndex(target);
|
|
673
725
|
if (target !== undefined && typeof target.value === 'string' && target.value !== '') {
|
|
674
726
|
await browser.click(session, {
|
|
675
727
|
target: {
|
|
676
728
|
by: (target.by === 'text' || target.by === 'xpath' ? target.by : 'css'),
|
|
677
729
|
value: target.value,
|
|
678
|
-
...
|
|
730
|
+
...index !== undefined ? { index } : {},
|
|
679
731
|
},
|
|
680
732
|
}, exec.signal);
|
|
681
733
|
}
|
|
@@ -723,13 +775,14 @@ export function apply(ctx, config = {}) {
|
|
|
723
775
|
if (target !== undefined && typeof target.value === 'string' && target.value === '') {
|
|
724
776
|
throw new Error('target.value is empty; pass a css selector, visible text or XPath, or omit target entirely to use x/y');
|
|
725
777
|
}
|
|
778
|
+
const index = targetIndex(target);
|
|
726
779
|
if (target !== undefined && typeof target.value === 'string' && target.value !== '') {
|
|
727
780
|
await browser.type(session, {
|
|
728
781
|
text: args.text,
|
|
729
782
|
target: {
|
|
730
783
|
by: (target.by === 'text' || target.by === 'xpath' ? target.by : 'css'),
|
|
731
784
|
value: target.value,
|
|
732
|
-
...
|
|
785
|
+
...index !== undefined ? { index } : {},
|
|
733
786
|
},
|
|
734
787
|
}, exec.signal);
|
|
735
788
|
}
|
|
@@ -918,7 +971,11 @@ export function apply(ctx, config = {}) {
|
|
|
918
971
|
if (browser === undefined)
|
|
919
972
|
throw new Error('tool-browser: browser service unavailable');
|
|
920
973
|
const session = await ensureSession(browser, state, taskKey(exec), agentOf(exec));
|
|
921
|
-
const
|
|
974
|
+
const requested = args.fields ?? [];
|
|
975
|
+
if (requested.length > maxFillFields) {
|
|
976
|
+
throw new Error(`browser_fill: ${requested.length} fields in one call exceeds the ${maxFillFields}-field limit; split it into several calls so each one can report per-field results instead of timing out as a batch`);
|
|
977
|
+
}
|
|
978
|
+
const fields = requested.map((f) => {
|
|
922
979
|
// A missing value is a caller mistake, not a request to clear the field. Turning
|
|
923
980
|
// it into '' silently unchecks a checkbox or empties a text input while the tool
|
|
924
981
|
// still reports success, so it fails loudly. The schema marks value required, but
|
|
@@ -1149,7 +1206,7 @@ export function apply(ctx, config = {}) {
|
|
|
1149
1206
|
}));
|
|
1150
1207
|
ctx.tools.register(defineTool({
|
|
1151
1208
|
name: 'browser_screenshot',
|
|
1152
|
-
description: 'Capture the current shared-browser page as a screenshot (PNG default, JPEG optional). This is for models that can read images: layout checks, charts, designs, CAPTCHAs, or locating an element by eye before clicking its coordinates. A model without image input gains nothing from it — browser_snapshot, browser_a11y and browser_content carry the same page as text, and browser_scrape extracts structured data. Supports full-page capture, save-to-file, JPEG encoding, and downscaling (maxWidth/maxHeight) to cut vision-tool token cost. JPEG needs a browser whose CDP encoder works — the self-hosted browser and an installed Chrome/Edge; the desktop sidebar
|
|
1209
|
+
description: 'Capture the current shared-browser page as a screenshot (PNG default, JPEG optional). This is for models that can read images: layout checks, charts, designs, CAPTCHAs, or locating an element by eye before clicking its coordinates. A model without image input gains nothing from it — browser_snapshot, browser_a11y and browser_content carry the same page as text, and browser_scrape extracts structured data. Supports full-page capture, save-to-file, JPEG encoding, and downscaling (maxWidth/maxHeight) to cut vision-tool token cost. JPEG needs a browser whose CDP encoder works — the self-hosted browser and an installed Chrome/Edge; some carriers (the desktop sidebar) return PNG instead. Downscaling works on every carrier.',
|
|
1153
1210
|
parameters: {
|
|
1154
1211
|
fullPage: { type: 'boolean', description: 'Capture the full scrollable page instead of the viewport (default false).' },
|
|
1155
1212
|
savePath: { type: 'string', description: 'Absolute file path to also save the image to (e.g. for read_image vision location). Must resolve inside the configured downloadDir (default: the system Downloads folder, localized names such as ~/下载 included); an existing file is never overwritten.' },
|
|
@@ -1163,11 +1220,13 @@ export function apply(ctx, config = {}) {
|
|
|
1163
1220
|
type: 'object',
|
|
1164
1221
|
additionalProperties: false,
|
|
1165
1222
|
properties: {
|
|
1166
|
-
dataUrl: { type: 'string', required: true, description: 'Base64
|
|
1223
|
+
dataUrl: { type: 'string', required: true, description: 'Base64 data URL of the screenshot (PNG, or JPEG when the carrier uses CDP JPEG).' },
|
|
1224
|
+
width: { type: 'number', description: 'Pixel width of the returned image, when the capture was measured (present with maxWidth/maxHeight).' },
|
|
1225
|
+
height: { type: 'number', description: 'Pixel height of the returned image, when the capture was measured.' },
|
|
1167
1226
|
path: { type: 'string', description: 'The file path the screenshot was saved to, when savePath was given.' },
|
|
1168
1227
|
},
|
|
1169
1228
|
},
|
|
1170
|
-
render: (_args, value) => [{ type: 'text', text: `Screenshot captured (${Math.round(value.dataUrl.length * 3 / 4 / 1024)} KiB)${value.path !== undefined ? ` saved to ${value.path}` : ''}.` }],
|
|
1229
|
+
render: (_args, value) => [{ type: 'text', text: `Screenshot captured (${Math.round(value.dataUrl.length * 3 / 4 / 1024)} KiB${value.width !== undefined && value.height !== undefined ? `, ${value.width}x${value.height}` : ''})${value.path !== undefined ? ` saved to ${value.path}` : ''}.` }],
|
|
1171
1230
|
},
|
|
1172
1231
|
timeoutMs,
|
|
1173
1232
|
isConcurrencySafe: () => true,
|
|
@@ -1187,6 +1246,12 @@ export function apply(ctx, config = {}) {
|
|
|
1187
1246
|
return {
|
|
1188
1247
|
dataUrl: shot.dataUrl,
|
|
1189
1248
|
...shot.path !== undefined ? { path: shot.path } : {},
|
|
1249
|
+
// The provider measures these when it scales, the schema declares them, and the seam
|
|
1250
|
+
// type carries them — but this mapping is an allow-list, so leaving them out dropped
|
|
1251
|
+
// them silently. Same shape of bug as the a11y selector, found the same way: by
|
|
1252
|
+
// reading the three places that mention a field and noticing this one did not.
|
|
1253
|
+
...shot.width !== undefined ? { width: shot.width } : {},
|
|
1254
|
+
...shot.height !== undefined ? { height: shot.height } : {},
|
|
1190
1255
|
};
|
|
1191
1256
|
},
|
|
1192
1257
|
}));
|
|
@@ -1300,7 +1365,9 @@ export function apply(ctx, config = {}) {
|
|
|
1300
1365
|
ctx.tools.register(defineTool({
|
|
1301
1366
|
name: 'browser_history',
|
|
1302
1367
|
description: 'List the shared browser session\'s recorded operation history (navigate/execute/click/type), newest last, with per-step success/error. Use to understand what the agent did and to pick a step to replay.',
|
|
1303
|
-
parameters: {
|
|
1368
|
+
parameters: {
|
|
1369
|
+
verbose: { type: 'boolean', description: "Include each operation's parameters (default false). They are what you just sent, so they are omitted unless asked for." },
|
|
1370
|
+
},
|
|
1304
1371
|
output: {
|
|
1305
1372
|
schema: {
|
|
1306
1373
|
type: 'object',
|
|
@@ -1324,18 +1391,21 @@ export function apply(ctx, config = {}) {
|
|
|
1324
1391
|
},
|
|
1325
1392
|
},
|
|
1326
1393
|
},
|
|
1327
|
-
render: (
|
|
1394
|
+
render: (args, value) => {
|
|
1328
1395
|
const entries = value.entries;
|
|
1329
1396
|
if (entries.length === 0)
|
|
1330
1397
|
return [{ type: 'text', text: '(no recorded operations yet)' }];
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
1398
|
+
// The parameters are what the caller itself just sent, so echoing them back is cost
|
|
1399
|
+
// without information — measured at 18,162 characters for 200 operations, of which the
|
|
1400
|
+
// per-entry params dominated. Only the tail is shown by default: the recent steps are
|
|
1401
|
+
// what a caller reasons about, and the count leads so the tail is not mistaken for all.
|
|
1402
|
+
const recent = entries.slice(-20);
|
|
1403
|
+
const omitted = entries.length - recent.length;
|
|
1404
|
+
const lines = recent.map(e => `#${e.seq} ${e.action} ${e.ok ? 'ok' : 'FAIL'}${e.result !== undefined ? ` -> ${e.result}` : ''}${e.error !== undefined ? ` !! ${e.error}` : ''}${args.verbose === true ? ` ${JSON.stringify(e.params)}` : ''}`);
|
|
1405
|
+
const header = omitted > 0
|
|
1406
|
+
? `${entries.length} operations recorded; showing the last ${recent.length} (${args.verbose === true ? '' : ' (pass verbose: true for parameters)'}):`
|
|
1407
|
+
: `${entries.length} operation(s):`;
|
|
1408
|
+
return [{ type: 'text', text: `${header}\n${lines.join('\n')}` }];
|
|
1339
1409
|
},
|
|
1340
1410
|
},
|
|
1341
1411
|
timeoutMs,
|
|
@@ -1425,15 +1495,20 @@ export function apply(ctx, config = {}) {
|
|
|
1425
1495
|
},
|
|
1426
1496
|
render: (_args, value) => {
|
|
1427
1497
|
const entries = value.entries;
|
|
1498
|
+
let lastSession;
|
|
1428
1499
|
if (entries.length === 0) {
|
|
1429
1500
|
return [{ type: 'text', text: '(no recorded visits — nothing has been browsed yet, or history recording is turned off in settings)' }];
|
|
1430
1501
|
}
|
|
1431
1502
|
return [{
|
|
1432
1503
|
type: 'text',
|
|
1433
1504
|
text: entries.map(entry => {
|
|
1434
|
-
const when = new Date(entry.at).toISOString().replace('T', ' ').slice(0,
|
|
1505
|
+
const when = new Date(entry.at).toISOString().replace('T', ' ').slice(0, 16);
|
|
1435
1506
|
const title = entry.title !== undefined ? `${entry.title} — ` : '';
|
|
1436
|
-
|
|
1507
|
+
const tag = entry.session !== undefined && entry.session !== lastSession
|
|
1508
|
+
? ` [session ${entry.session.slice(0, 8)}]`
|
|
1509
|
+
: '';
|
|
1510
|
+
lastSession = entry.session;
|
|
1511
|
+
return `${when} ${title}${entry.url}${tag}`;
|
|
1437
1512
|
}).join('\n'),
|
|
1438
1513
|
}];
|
|
1439
1514
|
},
|
|
@@ -1584,7 +1659,7 @@ export function apply(ctx, config = {}) {
|
|
|
1584
1659
|
}));
|
|
1585
1660
|
ctx.tools.register(defineTool({
|
|
1586
1661
|
name: 'browser_restrict',
|
|
1587
|
-
description: 'Restrict which browser actions are allowed, to prevent stray clicks/navigation. Pass a list of browser tool names (e.g. ["browser_snapshot","browser_content","browser_click"]) — any other browser_* call is refused. Pass an empty list or omit to lift the restriction.
|
|
1662
|
+
description: 'Restrict which browser actions are allowed, to prevent stray clicks/navigation. Pass a list of browser tool names (e.g. ["browser_snapshot","browser_content","browser_click"]) — any other browser_* call is refused. Pass an empty list or omit to lift the restriction. Never blocked, whatever the list says: the observing tools (snapshot/a11y/content/scrape/screenshot/get_value/wait/challenge/list_tabs/session/history/visited) and the three that lift a restriction or recover from a bad state (restrict/reset/reset_session) — so a restriction can always be read around and undone. IMPORTANT: this is a SOFT guardrail against accidental actions, NOT a security boundary — you (the model) can lift it yourself with an empty list.',
|
|
1588
1663
|
parameters: {
|
|
1589
1664
|
allowed: {
|
|
1590
1665
|
type: 'array',
|
|
@@ -1651,7 +1726,7 @@ export function apply(ctx, config = {}) {
|
|
|
1651
1726
|
// empty call returned {restored: 0} and rendered 'Restored 0 cookies.', which
|
|
1652
1727
|
// reads as success and would leave the caller believing the login was restored.
|
|
1653
1728
|
if (!Array.isArray(args.cookies) || args.cookies.length === 0) {
|
|
1654
|
-
throw new Error('browser_auth: action "restore" needs a non-empty "cookies" array (get one from action "
|
|
1729
|
+
throw new Error('browser_auth: action "restore" needs a non-empty "cookies" array (get one from action "flush" on the same profile)');
|
|
1655
1730
|
}
|
|
1656
1731
|
const list = args.cookies;
|
|
1657
1732
|
const restored = await browser.restoreAuth(session, list);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dsh-builtin-browser",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.5",
|
|
4
4
|
"description": "Shared real browser plugin for DeepSeek Harness: install-and-use — a visible native browser the human can take over, driven by the agent over CDP. DOM-level interaction (React/Vue safe), per-task session isolation, cookie persistence (browser_auth), CAPTCHA detection, batch form fill (browser_fill), authenticated download, history/replay, action restriction",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "lib/index.js",
|
|
@@ -40,7 +40,9 @@
|
|
|
40
40
|
"cordis.patch.yml",
|
|
41
41
|
"README.md",
|
|
42
42
|
"README.en.md",
|
|
43
|
-
"
|
|
43
|
+
"CHANGELOG.md",
|
|
44
|
+
"LICENSE",
|
|
45
|
+
"scripts"
|
|
44
46
|
],
|
|
45
47
|
"scripts": {
|
|
46
48
|
"build": "tsc -p tsconfig.json",
|
|
@@ -73,7 +75,10 @@
|
|
|
73
75
|
"@deepseek-ai/dsh-llm": ">=0.1.1-rc.2 <0.3.0",
|
|
74
76
|
"@deepseek-ai/dsh-tools": ">=0.1.1-rc.2 <0.3.0",
|
|
75
77
|
"@deepseek-ai/dsh-system-prompt": ">=0.1.1-rc.2 <0.3.0",
|
|
76
|
-
"@deepseek-ai/schemastery": "^3.18.1"
|
|
78
|
+
"@deepseek-ai/schemastery": "^3.18.1",
|
|
79
|
+
"@deepseek-ai/dsh-client-locale": ">=0.1.1-rc.2 <0.3.0",
|
|
80
|
+
"@deepseek-ai/dsh-client-ui-settings": ">=0.1.1-rc.2 <0.3.0",
|
|
81
|
+
"@deepseek-ai/dsh-client-ui-slots": ">=0.1.1-rc.2 <0.3.0"
|
|
77
82
|
},
|
|
78
83
|
"devDependencies": {
|
|
79
84
|
"@types/node": "^22.0.0",
|
|
@@ -102,7 +107,8 @@
|
|
|
102
107
|
"0.1.1-rc.1": "compatible",
|
|
103
108
|
"0.1.1-rc.2": "compatible",
|
|
104
109
|
"0.2.0-rc.1": "compatible",
|
|
105
|
-
"0.2.0-rc.2": "compatible"
|
|
110
|
+
"0.2.0-rc.2": "compatible",
|
|
111
|
+
"0.2.1-alpha.1": "compatible"
|
|
106
112
|
}
|
|
107
113
|
}
|
|
108
114
|
}
|