dsh-builtin-browser 0.3.1 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +84 -39
- package/README.md +78 -38
- package/client.js +2 -3
- package/cordis.patch.yml +13 -7
- package/desktop-bridge/install.mjs +274 -12
- package/desktop-bridge/plugin-browser-bridge.js +55 -15
- package/docs/architecture.md +1 -1
- package/docs/tool-reference.md +8 -8
- package/docs/user-guide.md +4 -2
- package/docs/why-browser.md +1 -1
- package/lib/browser-electron/desktop-bridge-host.d.ts +10 -1
- package/lib/browser-electron/desktop-bridge-host.js +10 -2
- package/lib/browser-electron/entry.js +22 -2
- package/lib/browser-electron/history-store.js +5 -1
- package/lib/browser-electron/missing-system-browser.d.ts +54 -0
- package/lib/browser-electron/missing-system-browser.js +89 -0
- package/lib/browser-electron/provider.d.ts +54 -1
- package/lib/browser-electron/provider.js +241 -37
- package/lib/browser-electron/remote-host.js +6 -3
- package/lib/browser-electron/settings-store.js +10 -4
- package/lib/browser-electron/system-browser.d.ts +27 -6
- package/lib/browser-electron/system-browser.js +274 -37
- package/lib/tool-browser/index.d.ts +14 -0
- package/lib/tool-browser/index.js +143 -58
- package/package.json +1 -1
- package/src/browser-electron/desktop-bridge-host.ts +11 -2
- package/src/browser-electron/entry.ts +22 -2
- package/src/browser-electron/history-store.ts +5 -1
- package/src/browser-electron/missing-system-browser.ts +113 -0
- package/src/browser-electron/provider.ts +267 -36
- package/src/browser-electron/remote-host.ts +6 -3
- package/src/browser-electron/settings-store.ts +11 -4
- package/src/browser-electron/system-browser.ts +256 -31
- package/src/tool-browser/index.ts +149 -63
|
@@ -14,12 +14,35 @@
|
|
|
14
14
|
* @module dsh-browser/tool-browser
|
|
15
15
|
*/
|
|
16
16
|
import { defineTool } from '@deepseek-ai/dsh-tools';
|
|
17
|
+
/**
|
|
18
|
+
* Register a cleanup that runs once when the process exits.
|
|
19
|
+
*
|
|
20
|
+
* A single hook per process, holding a list of callbacks, rather than one listener per
|
|
21
|
+
* session: sessions come and go, listeners do not, and Node warns when a listener count
|
|
22
|
+
* passes ten.
|
|
23
|
+
*/
|
|
24
|
+
const exitCleanups = new Set();
|
|
25
|
+
let exitHookInstalled = false;
|
|
26
|
+
function registerExitCleanup(cleanup) {
|
|
27
|
+
exitCleanups.add(cleanup);
|
|
28
|
+
if (exitHookInstalled)
|
|
29
|
+
return;
|
|
30
|
+
exitHookInstalled = true;
|
|
31
|
+
process.on('exit', () => {
|
|
32
|
+
for (const run of exitCleanups) {
|
|
33
|
+
try {
|
|
34
|
+
run();
|
|
35
|
+
}
|
|
36
|
+
catch { /* exiting; nothing useful to do */ }
|
|
37
|
+
}
|
|
38
|
+
});
|
|
39
|
+
}
|
|
17
40
|
/** Plugin name used by loader diagnostics. */
|
|
18
41
|
export const name = 'tool-browser';
|
|
19
42
|
/** The tool registry, browser seam, and system-prompt registry this tool layer consumes. */
|
|
20
43
|
export const inject = ['tools', 'browser', 'systemPrompt'];
|
|
21
44
|
function createState() {
|
|
22
|
-
return { sessionsByTask: new Map(), pendingOpens: new Map(),
|
|
45
|
+
return { sessionsByTask: new Map(), pendingOpens: new Map(), configRestrictedTo: undefined, restrictedByTask: new Map() };
|
|
23
46
|
}
|
|
24
47
|
/** Every live state, for the test hook (introspection only, never shared). */
|
|
25
48
|
const liveStates = new Set();
|
|
@@ -29,13 +52,43 @@ const liveStates = new Set();
|
|
|
29
52
|
* @param state - the calling context's tool state.
|
|
30
53
|
* @param toolName - the browser tool about to run.
|
|
31
54
|
*/
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
55
|
+
/**
|
|
56
|
+
* Tools that only observe, plus the ones that undo a restriction.
|
|
57
|
+
*
|
|
58
|
+
* Never subject to an allow-list: restricting what the agent may *do* must not also
|
|
59
|
+
* blind it or trap it, and the tool descriptions promise as much. browser_restrict
|
|
60
|
+
* is in here so a task that restricted everything can still lift it.
|
|
61
|
+
*/
|
|
62
|
+
export const READ_ONLY_TOOLS = new Set([
|
|
63
|
+
// Tools that only observe.
|
|
64
|
+
'browser_snapshot', 'browser_a11y', 'browser_content', 'browser_scrape',
|
|
65
|
+
'browser_screenshot', 'browser_get_value', 'browser_wait', 'browser_challenge',
|
|
66
|
+
'browser_list_tabs', 'browser_session', 'browser_history', 'browser_visited',
|
|
67
|
+
// browser_auth reads cookies out of the browser: an observation, and one the
|
|
68
|
+
// credentials switch gates on its own (see credentials.allowRead in the settings).
|
|
69
|
+
'browser_auth',
|
|
70
|
+
// Getting out of a bad state, or lifting the restriction itself. Without these a task
|
|
71
|
+
// that restricted everything could neither recover nor be released.
|
|
72
|
+
'browser_restrict', 'browser_reset_session', 'browser_reset',
|
|
73
|
+
]);
|
|
74
|
+
function assertAllowed(state, toolName, task) {
|
|
75
|
+
// browser_restrict must stay usable, or a task that restricted everything
|
|
76
|
+
// could never lift it again.
|
|
77
|
+
// An allow-list is a statement about ACTIONS, so it must not take away the ability to
|
|
78
|
+
// look, or to get out of a bad state. The tool description and both READMEs promise
|
|
79
|
+
// that read-only tools are never blocked; before this they were, because only
|
|
80
|
+
// browser_restrict itself was exempt — a narrow list then locked browser_history,
|
|
81
|
+
// browser_reset_session and browser_auth away with no way back.
|
|
82
|
+
if (READ_ONLY_TOOLS.has(toolName) || toolName === 'browser_restrict')
|
|
35
83
|
return;
|
|
36
|
-
|
|
84
|
+
const config = state.configRestrictedTo;
|
|
85
|
+
if (config !== undefined && !config.includes(toolName)) {
|
|
86
|
+
throw new Error(`browser action "${toolName}" is restricted by the plugin configuration (allow-list: ${config.join(', ')})`);
|
|
87
|
+
}
|
|
88
|
+
const allowed = state.restrictedByTask.get(task);
|
|
89
|
+
if (allowed === undefined || allowed.includes(toolName))
|
|
37
90
|
return;
|
|
38
|
-
throw new Error(`browser action "${toolName}" is restricted (allow-list: ${
|
|
91
|
+
throw new Error(`browser action "${toolName}" is restricted (allow-list: ${allowed.join(', ')})`);
|
|
39
92
|
}
|
|
40
93
|
/** Extract the agent from a tool-execution context, when one exists. */
|
|
41
94
|
function agentOf(exec) {
|
|
@@ -96,9 +149,10 @@ async function ensureSession(browser, state, key, agent) {
|
|
|
96
149
|
else if (key === 'default') {
|
|
97
150
|
// Agentless / CLI probe: close the session on process exit so the
|
|
98
151
|
// window is not orphaned.
|
|
99
|
-
process
|
|
100
|
-
|
|
101
|
-
|
|
152
|
+
// One hook for the whole process, not one listener per session: a default session can
|
|
153
|
+
// be created repeatedly (a CLI probe does), and each registration lived for the life of
|
|
154
|
+
// the process — so the listener list grew without bound and Node warned at eleven.
|
|
155
|
+
registerExitCleanup(() => browser.close(session).catch(() => { }));
|
|
102
156
|
}
|
|
103
157
|
}
|
|
104
158
|
catch (error) {
|
|
@@ -118,8 +172,10 @@ function parseFillValue(v) {
|
|
|
118
172
|
return true;
|
|
119
173
|
if (v === 'false')
|
|
120
174
|
return false;
|
|
121
|
-
|
|
122
|
-
|
|
175
|
+
// Deliberately NOT coerced to a number. The description says numbers are accepted in
|
|
176
|
+
// string form, and callers pass account numbers, postcodes and zero-padded ids through
|
|
177
|
+
// here: "007" became "7" and "1e3" became "1000", which silently rewrote the value they
|
|
178
|
+
// meant to type. A numeric-looking string is a string; the DOM decides how to store it.
|
|
123
179
|
return v ?? '';
|
|
124
180
|
}
|
|
125
181
|
/**
|
|
@@ -154,7 +210,7 @@ export function apply(ctx, config = {}) {
|
|
|
154
210
|
liveStates.add(state);
|
|
155
211
|
ctx.effect(() => () => { liveStates.delete(state); });
|
|
156
212
|
// Re-apply resets the restriction: an omitted allowedActions lifts it.
|
|
157
|
-
state.
|
|
213
|
+
state.configRestrictedTo = config.allowedActions !== undefined ? [...config.allowedActions] : undefined;
|
|
158
214
|
ctx.systemPrompt.section({
|
|
159
215
|
name: 'tool:browser',
|
|
160
216
|
// Tool guidance band is 100-199; 150 keeps clear of the common 110/120
|
|
@@ -207,12 +263,12 @@ export function apply(ctx, config = {}) {
|
|
|
207
263
|
// browser_open is an overview of what just loaded, not a locator: it has no
|
|
208
264
|
// `coords` parameter, so coordinates stay off here (browser_snapshot owns
|
|
209
265
|
// that switch for callers that actually target pixels).
|
|
210
|
-
render: (args, value) => [{ type: 'text', text: formatSnapshot(value, {
|
|
266
|
+
render: (args, value) => [{ type: 'text', text: formatSnapshot(value, {}) }],
|
|
211
267
|
},
|
|
212
268
|
timeoutMs,
|
|
213
269
|
isConcurrencySafe: () => false, // opens tabs / navigates; exclusive within a task
|
|
214
270
|
async execute(args, exec) {
|
|
215
|
-
assertAllowed(state, 'browser_open');
|
|
271
|
+
assertAllowed(state, 'browser_open', taskKey(exec));
|
|
216
272
|
const browser = ctx.get('browser');
|
|
217
273
|
if (browser === undefined)
|
|
218
274
|
throw new Error('tool-browser: browser service unavailable');
|
|
@@ -477,7 +533,7 @@ export function apply(ctx, config = {}) {
|
|
|
477
533
|
timeoutMs,
|
|
478
534
|
isConcurrencySafe: () => false, // page JS can be stateful
|
|
479
535
|
async execute(args, exec) {
|
|
480
|
-
assertAllowed(state, 'browser_execute');
|
|
536
|
+
assertAllowed(state, 'browser_execute', taskKey(exec));
|
|
481
537
|
const browser = ctx.get('browser');
|
|
482
538
|
if (browser === undefined)
|
|
483
539
|
throw new Error('tool-browser: browser service unavailable');
|
|
@@ -564,7 +620,7 @@ export function apply(ctx, config = {}) {
|
|
|
564
620
|
timeoutMs,
|
|
565
621
|
isConcurrencySafe: () => true,
|
|
566
622
|
async execute(args, exec) {
|
|
567
|
-
assertAllowed(state, 'browser_scrape');
|
|
623
|
+
assertAllowed(state, 'browser_scrape', taskKey(exec));
|
|
568
624
|
const browser = ctx.get('browser');
|
|
569
625
|
if (browser === undefined)
|
|
570
626
|
throw new Error('tool-browser: browser service unavailable');
|
|
@@ -602,12 +658,18 @@ export function apply(ctx, config = {}) {
|
|
|
602
658
|
timeoutMs,
|
|
603
659
|
isConcurrencySafe: () => false,
|
|
604
660
|
async execute(args, exec) {
|
|
605
|
-
assertAllowed(state, 'browser_click');
|
|
661
|
+
assertAllowed(state, 'browser_click', taskKey(exec));
|
|
606
662
|
const browser = ctx.get('browser');
|
|
607
663
|
if (browser === undefined)
|
|
608
664
|
throw new Error('tool-browser: browser service unavailable');
|
|
609
665
|
const session = await ensureSession(browser, state, taskKey(exec), agentOf(exec));
|
|
610
666
|
const target = args.target;
|
|
667
|
+
// A target whose value is empty is a caller mistake, not a request to fall back.
|
|
668
|
+
// Degrading silently sent typed text to whatever held focus — possibly a password
|
|
669
|
+
// field — or dropped the semantic target in favour of coordinates.
|
|
670
|
+
if (target !== undefined && typeof target.value === 'string' && target.value === '') {
|
|
671
|
+
throw new Error('target.value is empty; pass a css selector, visible text or XPath, or omit target entirely to use x/y');
|
|
672
|
+
}
|
|
611
673
|
if (target !== undefined && typeof target.value === 'string' && target.value !== '') {
|
|
612
674
|
await browser.click(session, {
|
|
613
675
|
target: {
|
|
@@ -649,12 +711,18 @@ export function apply(ctx, config = {}) {
|
|
|
649
711
|
timeoutMs,
|
|
650
712
|
isConcurrencySafe: () => false,
|
|
651
713
|
async execute(args, exec) {
|
|
652
|
-
assertAllowed(state, 'browser_type');
|
|
714
|
+
assertAllowed(state, 'browser_type', taskKey(exec));
|
|
653
715
|
const browser = ctx.get('browser');
|
|
654
716
|
if (browser === undefined)
|
|
655
717
|
throw new Error('tool-browser: browser service unavailable');
|
|
656
718
|
const session = await ensureSession(browser, state, taskKey(exec), agentOf(exec));
|
|
657
719
|
const target = args.target;
|
|
720
|
+
// A target whose value is empty is a caller mistake, not a request to fall back.
|
|
721
|
+
// Degrading silently sent typed text to whatever held focus — possibly a password
|
|
722
|
+
// field — or dropped the semantic target in favour of coordinates.
|
|
723
|
+
if (target !== undefined && typeof target.value === 'string' && target.value === '') {
|
|
724
|
+
throw new Error('target.value is empty; pass a css selector, visible text or XPath, or omit target entirely to use x/y');
|
|
725
|
+
}
|
|
658
726
|
if (target !== undefined && typeof target.value === 'string' && target.value !== '') {
|
|
659
727
|
await browser.type(session, {
|
|
660
728
|
text: args.text,
|
|
@@ -688,7 +756,7 @@ export function apply(ctx, config = {}) {
|
|
|
688
756
|
timeoutMs,
|
|
689
757
|
isConcurrencySafe: () => false, // mutates page scroll state; exclusive within a task
|
|
690
758
|
async execute(args, exec) {
|
|
691
|
-
assertAllowed(state, 'browser_scroll');
|
|
759
|
+
assertAllowed(state, 'browser_scroll', taskKey(exec));
|
|
692
760
|
const browser = ctx.get('browser');
|
|
693
761
|
if (browser === undefined)
|
|
694
762
|
throw new Error('tool-browser: browser service unavailable');
|
|
@@ -714,7 +782,7 @@ export function apply(ctx, config = {}) {
|
|
|
714
782
|
timeoutMs,
|
|
715
783
|
isConcurrencySafe: () => false, // mutates page state; exclusive within a task
|
|
716
784
|
async execute(_args, exec) {
|
|
717
|
-
assertAllowed(state, 'browser_back');
|
|
785
|
+
assertAllowed(state, 'browser_back', taskKey(exec));
|
|
718
786
|
const browser = ctx.get('browser');
|
|
719
787
|
if (browser === undefined)
|
|
720
788
|
throw new Error('tool-browser: browser service unavailable');
|
|
@@ -734,7 +802,7 @@ export function apply(ctx, config = {}) {
|
|
|
734
802
|
timeoutMs,
|
|
735
803
|
isConcurrencySafe: () => false, // mutates page state; exclusive within a task
|
|
736
804
|
async execute(_args, exec) {
|
|
737
|
-
assertAllowed(state, 'browser_forward');
|
|
805
|
+
assertAllowed(state, 'browser_forward', taskKey(exec));
|
|
738
806
|
const browser = ctx.get('browser');
|
|
739
807
|
if (browser === undefined)
|
|
740
808
|
throw new Error('tool-browser: browser service unavailable');
|
|
@@ -754,7 +822,7 @@ export function apply(ctx, config = {}) {
|
|
|
754
822
|
timeoutMs,
|
|
755
823
|
isConcurrencySafe: () => false, // reloads the page; exclusive within a task
|
|
756
824
|
async execute(_args, exec) {
|
|
757
|
-
assertAllowed(state, 'browser_refresh');
|
|
825
|
+
assertAllowed(state, 'browser_refresh', taskKey(exec));
|
|
758
826
|
const browser = ctx.get('browser');
|
|
759
827
|
if (browser === undefined)
|
|
760
828
|
throw new Error('tool-browser: browser service unavailable');
|
|
@@ -776,7 +844,7 @@ export function apply(ctx, config = {}) {
|
|
|
776
844
|
timeoutMs,
|
|
777
845
|
isConcurrencySafe: () => false, // mutates page state; exclusive within a task
|
|
778
846
|
async execute(args, exec) {
|
|
779
|
-
assertAllowed(state, 'browser_key');
|
|
847
|
+
assertAllowed(state, 'browser_key', taskKey(exec));
|
|
780
848
|
const browser = ctx.get('browser');
|
|
781
849
|
if (browser === undefined)
|
|
782
850
|
throw new Error('tool-browser: browser service unavailable');
|
|
@@ -801,7 +869,7 @@ export function apply(ctx, config = {}) {
|
|
|
801
869
|
label: { type: 'string', description: 'Match by associated <label> text or aria-label.' },
|
|
802
870
|
placeholder: { type: 'string', description: 'Match by placeholder text.' },
|
|
803
871
|
kind: { type: 'string', enum: ['text', 'textarea', 'checkbox', 'radio', 'select'], description: 'Field kind; defaults to text.' },
|
|
804
|
-
value: { type: 'string', description: 'Value to set (string form; booleans/numbers accepted as strings).' },
|
|
872
|
+
value: { type: 'string', required: true, description: 'Value to set (string form; booleans/numbers accepted as strings). Required: omitting it would be read as "set empty", which unchecks a checkbox and clears a text field while still reporting success.' },
|
|
805
873
|
},
|
|
806
874
|
},
|
|
807
875
|
},
|
|
@@ -845,19 +913,29 @@ export function apply(ctx, config = {}) {
|
|
|
845
913
|
timeoutMs,
|
|
846
914
|
isConcurrencySafe: () => false,
|
|
847
915
|
async execute(args, exec) {
|
|
848
|
-
assertAllowed(state, 'browser_fill');
|
|
916
|
+
assertAllowed(state, 'browser_fill', taskKey(exec));
|
|
849
917
|
const browser = ctx.get('browser');
|
|
850
918
|
if (browser === undefined)
|
|
851
919
|
throw new Error('tool-browser: browser service unavailable');
|
|
852
920
|
const session = await ensureSession(browser, state, taskKey(exec), agentOf(exec));
|
|
853
|
-
const fields = (args.fields ?? []).map((f) =>
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
921
|
+
const fields = (args.fields ?? []).map((f) => {
|
|
922
|
+
// A missing value is a caller mistake, not a request to clear the field. Turning
|
|
923
|
+
// it into '' silently unchecks a checkbox or empties a text input while the tool
|
|
924
|
+
// still reports success, so it fails loudly. The schema marks value required, but
|
|
925
|
+
// a call can still arrive without it.
|
|
926
|
+
if (f.value === undefined) {
|
|
927
|
+
const matcher = f.selector ?? f.name ?? f.label ?? f.placeholder ?? '(no matcher)';
|
|
928
|
+
throw new Error(`browser_fill: field "${matcher}" has no "value"; pass one (use "false" to uncheck, or "" to clear deliberately)`);
|
|
929
|
+
}
|
|
930
|
+
return {
|
|
931
|
+
...f.selector !== undefined ? { selector: f.selector } : {},
|
|
932
|
+
...f.name !== undefined ? { name: f.name } : {},
|
|
933
|
+
...f.label !== undefined ? { label: f.label } : {},
|
|
934
|
+
...f.placeholder !== undefined ? { placeholder: f.placeholder } : {},
|
|
935
|
+
...f.kind !== undefined ? { kind: f.kind } : {},
|
|
936
|
+
value: parseFillValue(f.value),
|
|
937
|
+
};
|
|
938
|
+
});
|
|
861
939
|
const result = await browser.fillForm(session, {
|
|
862
940
|
fields,
|
|
863
941
|
...args.submit === true ? { submit: true } : {},
|
|
@@ -903,7 +981,7 @@ export function apply(ctx, config = {}) {
|
|
|
903
981
|
timeoutMs,
|
|
904
982
|
isConcurrencySafe: () => false, // mutates a field; exclusive within a task
|
|
905
983
|
async execute(args, exec) {
|
|
906
|
-
assertAllowed(state, 'browser_set_value');
|
|
984
|
+
assertAllowed(state, 'browser_set_value', taskKey(exec));
|
|
907
985
|
const browser = ctx.get('browser');
|
|
908
986
|
if (browser === undefined)
|
|
909
987
|
throw new Error('tool-browser: browser service unavailable');
|
|
@@ -934,7 +1012,7 @@ export function apply(ctx, config = {}) {
|
|
|
934
1012
|
timeoutMs,
|
|
935
1013
|
isConcurrencySafe: () => false, // mutates a field; exclusive within a task
|
|
936
1014
|
async execute(args, exec) {
|
|
937
|
-
assertAllowed(state, 'browser_check');
|
|
1015
|
+
assertAllowed(state, 'browser_check', taskKey(exec));
|
|
938
1016
|
const browser = ctx.get('browser');
|
|
939
1017
|
if (browser === undefined)
|
|
940
1018
|
throw new Error('tool-browser: browser service unavailable');
|
|
@@ -974,7 +1052,7 @@ export function apply(ctx, config = {}) {
|
|
|
974
1052
|
timeoutMs,
|
|
975
1053
|
isConcurrencySafe: () => false, // mutates a field; exclusive within a task
|
|
976
1054
|
async execute(args, exec) {
|
|
977
|
-
assertAllowed(state, 'browser_select');
|
|
1055
|
+
assertAllowed(state, 'browser_select', taskKey(exec));
|
|
978
1056
|
const browser = ctx.get('browser');
|
|
979
1057
|
if (browser === undefined)
|
|
980
1058
|
throw new Error('tool-browser: browser service unavailable');
|
|
@@ -1006,7 +1084,7 @@ export function apply(ctx, config = {}) {
|
|
|
1006
1084
|
timeoutMs,
|
|
1007
1085
|
isConcurrencySafe: () => false, // mutates a field; exclusive within a task
|
|
1008
1086
|
async execute(args, exec) {
|
|
1009
|
-
assertAllowed(state, 'browser_clear');
|
|
1087
|
+
assertAllowed(state, 'browser_clear', taskKey(exec));
|
|
1010
1088
|
const browser = ctx.get('browser');
|
|
1011
1089
|
if (browser === undefined)
|
|
1012
1090
|
throw new Error('tool-browser: browser service unavailable');
|
|
@@ -1049,7 +1127,7 @@ export function apply(ctx, config = {}) {
|
|
|
1049
1127
|
timeoutMs,
|
|
1050
1128
|
isConcurrencySafe: () => true,
|
|
1051
1129
|
async execute(args, exec) {
|
|
1052
|
-
assertAllowed(state, 'browser_get_value');
|
|
1130
|
+
assertAllowed(state, 'browser_get_value', taskKey(exec));
|
|
1053
1131
|
const browser = ctx.get('browser');
|
|
1054
1132
|
if (browser === undefined)
|
|
1055
1133
|
throw new Error('tool-browser: browser service unavailable');
|
|
@@ -1071,11 +1149,11 @@ export function apply(ctx, config = {}) {
|
|
|
1071
1149
|
}));
|
|
1072
1150
|
ctx.tools.register(defineTool({
|
|
1073
1151
|
name: 'browser_screenshot',
|
|
1074
|
-
description: 'Capture the current shared-browser page as a screenshot (PNG default, JPEG optional). This is for models that can read images: layout checks, charts, designs, CAPTCHAs, or locating an element by eye before clicking its coordinates. A model without image input gains nothing from it — browser_snapshot, browser_a11y and browser_content carry the same page as text, and browser_scrape extracts structured data. Supports full-page capture, save-to-file, JPEG encoding, and downscaling (maxWidth/maxHeight) to cut vision-tool token cost. JPEG
|
|
1152
|
+
description: 'Capture the current shared-browser page as a screenshot (PNG default, JPEG optional). This is for models that can read images: layout checks, charts, designs, CAPTCHAs, or locating an element by eye before clicking its coordinates. A model without image input gains nothing from it — browser_snapshot, browser_a11y and browser_content carry the same page as text, and browser_scrape extracts structured data. Supports full-page capture, save-to-file, JPEG encoding, and downscaling (maxWidth/maxHeight) to cut vision-tool token cost. JPEG needs a browser whose CDP encoder works — the self-hosted browser and an installed Chrome/Edge; the desktop sidebar falls back to PNG because the Electron CDP JPEG encoder hangs. Downscaling works on every carrier.',
|
|
1075
1153
|
parameters: {
|
|
1076
1154
|
fullPage: { type: 'boolean', description: 'Capture the full scrollable page instead of the viewport (default false).' },
|
|
1077
1155
|
savePath: { type: 'string', description: 'Absolute file path to also save the image to (e.g. for read_image vision location). Must resolve inside the configured downloadDir (default: the system Downloads folder, localized names such as ~/下载 included); an existing file is never overwritten.' },
|
|
1078
|
-
format: { type: 'string', enum: ['png', 'jpeg'], description: 'Image format (default png; jpeg
|
|
1156
|
+
format: { type: 'string', enum: ['png', 'jpeg'], description: 'Image format (default png; jpeg needs the self-hosted browser or an installed Chrome/Edge; the desktop sidebar returns PNG because the Electron CDP JPEG encoder hangs).' },
|
|
1079
1157
|
quality: { type: 'number', description: 'JPEG quality 1-100 (default 80); ignored for PNG.' },
|
|
1080
1158
|
maxWidth: { type: 'number', description: 'Downscale to fit within this width (aspect preserved).' },
|
|
1081
1159
|
maxHeight: { type: 'number', description: 'Downscale to fit within this height (aspect preserved).' },
|
|
@@ -1167,7 +1245,7 @@ export function apply(ctx, config = {}) {
|
|
|
1167
1245
|
timeoutMs,
|
|
1168
1246
|
isConcurrencySafe: () => false, // mutates the active tab; exclusive within a task
|
|
1169
1247
|
async execute(args, exec) {
|
|
1170
|
-
assertAllowed(state, 'browser_switch_tab');
|
|
1248
|
+
assertAllowed(state, 'browser_switch_tab', taskKey(exec));
|
|
1171
1249
|
const browser = ctx.get('browser');
|
|
1172
1250
|
if (browser === undefined)
|
|
1173
1251
|
throw new Error('tool-browser: browser service unavailable');
|
|
@@ -1189,7 +1267,7 @@ export function apply(ctx, config = {}) {
|
|
|
1189
1267
|
timeoutMs,
|
|
1190
1268
|
isConcurrencySafe: () => false, // mutates the tab list; exclusive within a task
|
|
1191
1269
|
async execute(args, exec) {
|
|
1192
|
-
assertAllowed(state, 'browser_close_tab');
|
|
1270
|
+
assertAllowed(state, 'browser_close_tab', taskKey(exec));
|
|
1193
1271
|
const browser = ctx.get('browser');
|
|
1194
1272
|
if (browser === undefined)
|
|
1195
1273
|
throw new Error('tool-browser: browser service unavailable');
|
|
@@ -1209,7 +1287,7 @@ export function apply(ctx, config = {}) {
|
|
|
1209
1287
|
timeoutMs,
|
|
1210
1288
|
isConcurrencySafe: () => false, // closes every tab; exclusive within a task
|
|
1211
1289
|
async execute(_args, exec) {
|
|
1212
|
-
assertAllowed(state, 'browser_reset');
|
|
1290
|
+
assertAllowed(state, 'browser_reset', taskKey(exec));
|
|
1213
1291
|
const browser = ctx.get('browser');
|
|
1214
1292
|
if (browser === undefined)
|
|
1215
1293
|
throw new Error('tool-browser: browser service unavailable');
|
|
@@ -1284,9 +1362,6 @@ export function apply(ctx, config = {}) {
|
|
|
1284
1362
|
if (e.action === 'replay' && params.of === 'type' && typeof params.text === 'string') {
|
|
1285
1363
|
params.text = '*'.repeat(Math.min(params.text.length, 64)) + ` (${params.text.length} chars)`;
|
|
1286
1364
|
}
|
|
1287
|
-
if (e.action === 'replay' && params.of === 'setValue' && typeof params.value === 'string') {
|
|
1288
|
-
params.value = '*'.repeat(Math.min(params.value.length, 64)) + ` (${params.value.length} chars)`;
|
|
1289
|
-
}
|
|
1290
1365
|
// Execute scripts may embed form tokens / credentials; mask script,
|
|
1291
1366
|
// args, and result for both direct execute and replay-of-execute.
|
|
1292
1367
|
const isExecute = e.action === 'execute' || (e.action === 'replay' && params.of === 'execute');
|
|
@@ -1404,7 +1479,7 @@ export function apply(ctx, config = {}) {
|
|
|
1404
1479
|
timeoutMs,
|
|
1405
1480
|
isConcurrencySafe: () => false,
|
|
1406
1481
|
async execute(args, exec) {
|
|
1407
|
-
assertAllowed(state, 'browser_replay');
|
|
1482
|
+
assertAllowed(state, 'browser_replay', taskKey(exec));
|
|
1408
1483
|
const browser = ctx.get('browser');
|
|
1409
1484
|
if (browser === undefined)
|
|
1410
1485
|
throw new Error('tool-browser: browser service unavailable');
|
|
@@ -1427,7 +1502,7 @@ export function apply(ctx, config = {}) {
|
|
|
1427
1502
|
timeoutMs,
|
|
1428
1503
|
isConcurrencySafe: () => false,
|
|
1429
1504
|
async execute(args, exec) {
|
|
1430
|
-
assertAllowed(state, 'browser_download');
|
|
1505
|
+
assertAllowed(state, 'browser_download', taskKey(exec));
|
|
1431
1506
|
const browser = ctx.get('browser');
|
|
1432
1507
|
if (browser === undefined)
|
|
1433
1508
|
throw new Error('tool-browser: browser service unavailable');
|
|
@@ -1488,7 +1563,7 @@ export function apply(ctx, config = {}) {
|
|
|
1488
1563
|
timeoutMs,
|
|
1489
1564
|
isConcurrencySafe: () => false, // closes the whole session; exclusive within a task
|
|
1490
1565
|
async execute(_args, exec) {
|
|
1491
|
-
assertAllowed(state, 'browser_reset_session');
|
|
1566
|
+
assertAllowed(state, 'browser_reset_session', taskKey(exec));
|
|
1492
1567
|
const browser = ctx.get('browser');
|
|
1493
1568
|
if (browser === undefined)
|
|
1494
1569
|
throw new Error('tool-browser: browser service unavailable');
|
|
@@ -1530,17 +1605,21 @@ export function apply(ctx, config = {}) {
|
|
|
1530
1605
|
if (unknown.length > 0) {
|
|
1531
1606
|
throw new Error(`browser_restrict: unknown tool name(s) ${unknown.map(t => `"${t}"`).join(', ')} (must start with "browser_")`);
|
|
1532
1607
|
}
|
|
1533
|
-
// Empty list (or omitted) lifts
|
|
1534
|
-
// new allow-list.
|
|
1535
|
-
//
|
|
1536
|
-
//
|
|
1537
|
-
|
|
1538
|
-
|
|
1608
|
+
// Empty list (or omitted) lifts this task's restriction; a non-empty list is its
|
|
1609
|
+
// new allow-list. Per task, not per plugin: this state object is shared by every
|
|
1610
|
+
// agent, so one value here used to lock every other task's browser tools.
|
|
1611
|
+
//
|
|
1612
|
+
if (allowed.length === 0)
|
|
1613
|
+
state.restrictedByTask.delete(taskKey(exec));
|
|
1614
|
+
else
|
|
1615
|
+
state.restrictedByTask.set(taskKey(exec), [...allowed]);
|
|
1616
|
+
const current = state.restrictedByTask.get(taskKey(exec));
|
|
1617
|
+
return { restrictedTo: current === undefined ? [] : [...current] };
|
|
1539
1618
|
},
|
|
1540
1619
|
}));
|
|
1541
1620
|
ctx.tools.register(defineTool({
|
|
1542
1621
|
name: 'browser_auth',
|
|
1543
|
-
description: 'Export or restore the browser session\'s cookies (login state). Use "flush" to get a JSON cookie list (save it to a private file to persist logins), or "restore" with that list to put logins back (e.g. after the browser host restarted). Available on the self-hosted browser. Exported cookies are LIVE CREDENTIALS: treat them as secrets — do not echo them into the conversation, keep them out of logs, and store the list in a private file.',
|
|
1622
|
+
description: 'Export or restore the browser session\'s cookies (login state). Use "flush" to get a JSON cookie list (save it to a private file to persist logins), or "restore" with that list to put logins back (e.g. after the browser host restarted). Available on all three carriers: the self-hosted browser reads its own session, while the desktop sidebar and an installed Chrome/Edge go through CDP. Refused entirely when the credential switch is off. Exported cookies are LIVE CREDENTIALS: treat them as secrets — do not echo them into the conversation, keep them out of logs, and store the list in a private file.',
|
|
1544
1623
|
parameters: {
|
|
1545
1624
|
action: { type: 'string', required: true, enum: ['flush', 'restore'], description: 'flush = export cookies; restore = import cookies.' },
|
|
1546
1625
|
cookies: { type: 'array', items: { type: 'object', additionalProperties: true }, description: 'Cookie list to restore (required when action=restore).' },
|
|
@@ -1559,7 +1638,7 @@ export function apply(ctx, config = {}) {
|
|
|
1559
1638
|
timeoutMs,
|
|
1560
1639
|
isConcurrencySafe: () => false,
|
|
1561
1640
|
async execute(args, exec) {
|
|
1562
|
-
assertAllowed(state, 'browser_auth');
|
|
1641
|
+
assertAllowed(state, 'browser_auth', taskKey(exec));
|
|
1563
1642
|
const browser = ctx.get('browser');
|
|
1564
1643
|
if (browser === undefined)
|
|
1565
1644
|
throw new Error('tool-browser: browser service unavailable');
|
|
@@ -1568,7 +1647,13 @@ export function apply(ctx, config = {}) {
|
|
|
1568
1647
|
const cookies = await browser.flushAuth(session);
|
|
1569
1648
|
return { cookies: cookies.map(c => ({ ...c })) };
|
|
1570
1649
|
}
|
|
1571
|
-
|
|
1650
|
+
// The description says cookies are required for restore; without this check an
|
|
1651
|
+
// empty call returned {restored: 0} and rendered 'Restored 0 cookies.', which
|
|
1652
|
+
// reads as success and would leave the caller believing the login was restored.
|
|
1653
|
+
if (!Array.isArray(args.cookies) || args.cookies.length === 0) {
|
|
1654
|
+
throw new Error('browser_auth: action "restore" needs a non-empty "cookies" array (get one from action "export" on the same profile)');
|
|
1655
|
+
}
|
|
1656
|
+
const list = args.cookies;
|
|
1572
1657
|
const restored = await browser.restoreAuth(session, list);
|
|
1573
1658
|
return { restored };
|
|
1574
1659
|
},
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dsh-builtin-browser",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.4.1",
|
|
4
4
|
"description": "Shared real browser plugin for DeepSeek Harness: install-and-use — a visible native browser the human can take over, driven by the agent over CDP. DOM-level interaction (React/Vue safe), per-task session isolation, cookie persistence (browser_auth), CAPTCHA detection, batch form fill (browser_fill), authenticated download, history/replay, action restriction",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "lib/index.js",
|
|
@@ -25,7 +25,6 @@
|
|
|
25
25
|
|
|
26
26
|
import { BridgeConnection, type BridgeEndpoint } from './bridge-connection.js'
|
|
27
27
|
import { readFileSync } from 'node:fs'
|
|
28
|
-
import { connect, type Socket } from 'node:net'
|
|
29
28
|
import { homedir } from 'node:os'
|
|
30
29
|
import { join } from 'node:path'
|
|
31
30
|
import { randomUUID } from 'node:crypto'
|
|
@@ -69,12 +68,22 @@ export class DesktopBridgeViewHost implements ElectronBrowserViewHost {
|
|
|
69
68
|
* session gets its own page).
|
|
70
69
|
*/
|
|
71
70
|
private readonly orphaned = new Set<number>()
|
|
71
|
+
/**
|
|
72
|
+
* Registered by the provider, and deliberately never invoked on this carrier.
|
|
73
|
+
*
|
|
74
|
+
* The sidebar is the shell's own interface: a human clicking inside that page is an
|
|
75
|
+
* event the shell owns, and the bridge exposes no operation that reports it. The
|
|
76
|
+
* handler is stored anyway so the interface contract holds (a caller may register
|
|
77
|
+
* one and must not be surprised) and so a future bridge operation can deliver it
|
|
78
|
+
* without changing the provider. Consequently no user-action event is emitted while
|
|
79
|
+
* the desktop sidebar is the carrier — by design, not by omission.
|
|
80
|
+
*/
|
|
72
81
|
private userActionHandler: ((action: BrowserUserAction) => void) | undefined
|
|
73
82
|
|
|
74
83
|
/**
|
|
75
84
|
* @param endpoint - the shell's published bridge endpoint.
|
|
76
85
|
*/
|
|
77
|
-
private constructor(
|
|
86
|
+
private constructor(endpoint: BridgeEndpoint) {
|
|
78
87
|
this.connection = new BridgeConnection(endpoint)
|
|
79
88
|
}
|
|
80
89
|
|
|
@@ -21,7 +21,8 @@ import { homedir } from 'node:os'
|
|
|
21
21
|
import { join } from 'node:path'
|
|
22
22
|
import { defaultHostMainPath, RemoteElectronViewHost } from './remote-host.js'
|
|
23
23
|
import { DesktopBridgeViewHost } from './desktop-bridge-host.js'
|
|
24
|
-
import { detectBrowser, SystemBrowserViewHost } from './system-browser.js'
|
|
24
|
+
import { detectBrowser, searchSummary, SystemBrowserViewHost } from './system-browser.js'
|
|
25
|
+
import { MissingSystemBrowserHost } from './missing-system-browser.js'
|
|
25
26
|
import { SettingsStore } from './settings-store.js'
|
|
26
27
|
|
|
27
28
|
export {
|
|
@@ -131,10 +132,25 @@ export function apply(ctx: Context & { browser: BrowserRuntime }, config: Config
|
|
|
131
132
|
// spawns a browser window. (An earlier version launched it here, which meant a
|
|
132
133
|
// browser appeared the moment the plugin loaded — reported and fixed.)
|
|
133
134
|
const channel = settings.get().browser.channel
|
|
135
|
+
// Naming a browser is a decision about which one to use, so it also decides whether the
|
|
136
|
+
// desktop sidebar may take over. Skipping discovery for an explicit choice is what makes
|
|
137
|
+
// that choice real — otherwise a user who picked Chrome would silently get the sidebar,
|
|
138
|
+
// and a user who picked a browser that is not installed would never see the explanation
|
|
139
|
+
// written for exactly that case, because the sidebar would quietly replace it.
|
|
140
|
+
const explicitChoice = channel !== 'bundled' && channel !== 'auto'
|
|
134
141
|
if (channel !== 'bundled') {
|
|
135
142
|
const detected = detectBrowser(channel)
|
|
136
143
|
if (detected === undefined) {
|
|
137
|
-
|
|
144
|
+
// `auto` promises to take whatever is available, so falling back is what it
|
|
145
|
+
// asked for. Naming a browser is a specific request, and quietly satisfying it
|
|
146
|
+
// with a different one turns the real problem ("no Chrome here") into a
|
|
147
|
+
// confusing error about Electron later on. Report it instead.
|
|
148
|
+
if (channel === 'auto') {
|
|
149
|
+
ctx.logger?.warn?.('dsh-builtin-browser: no installed browser was found; using the bundled browser')
|
|
150
|
+
} else {
|
|
151
|
+
adopt(new MissingSystemBrowserHost(channel, searchSummary(channel)),
|
|
152
|
+
`the selected ${channel} is not installed — commands will explain how to fix it`)
|
|
153
|
+
}
|
|
138
154
|
} else {
|
|
139
155
|
const home = process.env.DSH_HOME ?? homedir()
|
|
140
156
|
// Login state lives in the browser profile, so the cookies setting decides where
|
|
@@ -150,6 +166,10 @@ export function apply(ctx: Context & { browser: BrowserRuntime }, config: Config
|
|
|
150
166
|
}
|
|
151
167
|
}
|
|
152
168
|
|
|
169
|
+
// An explicit choice was already adopted above; discovering the sidebar here would
|
|
170
|
+
// replace it, which is exactly what this guard prevents.
|
|
171
|
+
if (explicitChoice) return
|
|
172
|
+
|
|
153
173
|
void DesktopBridgeViewHost.discover().then(sidebar => {
|
|
154
174
|
// No bridge (plain `dsh web`, an older desktop build, or a shell that already
|
|
155
175
|
// exited): keep self-hosting, which is what every surface has always done.
|
|
@@ -149,7 +149,11 @@ export class HistoryStore {
|
|
|
149
149
|
return []
|
|
150
150
|
}
|
|
151
151
|
const pages: VisitedPage[] = []
|
|
152
|
-
|
|
152
|
+
// Strip a BOM first: trim() does not remove U+FEFF (it is not whitespace in
|
|
153
|
+
// ECMAScript), so a file written by Notepad or PowerShell made the first line fail to
|
|
154
|
+
// parse and be discarded as corrupt — silently losing one visit, or the whole file when
|
|
155
|
+
// it holds one. settings-store strips one for the same reason.
|
|
156
|
+
for (const line of raw.replace(/^\uFEFF/, '').split('\n')) {
|
|
153
157
|
const trimmed = line.trim()
|
|
154
158
|
if (trimmed === '') continue
|
|
155
159
|
try {
|