@xingular/harness 1.5.0-unstable → 1.6.0-unstable
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +93 -0
- package/browser-extension/README.md +32 -0
- package/browser-extension/background.js +612 -0
- package/browser-extension/manifest.json +16 -0
- package/browser-extension/overlay.js +97 -0
- package/browser-extension/popup.css +19 -0
- package/browser-extension/popup.html +30 -0
- package/browser-extension/popup.js +101 -0
- package/browser-extension/site-access.js +63 -0
- package/dist/cli.js +1489 -680
- package/dist/sdk/browser/config.d.ts +3 -0
- package/dist/sdk/browser/config.js +2 -1
- package/dist/sdk/browser/extension/bridge.d.ts +55 -0
- package/dist/sdk/browser/extension/bridge.js +336 -0
- package/dist/sdk/browser/extension/session.d.ts +50 -0
- package/dist/sdk/browser/extension/session.js +321 -0
- package/dist/sdk/browser/index.d.ts +7 -2
- package/dist/sdk/browser/index.js +58 -25
- package/dist/sdk/browser/serverMethods.js +4 -1
- package/dist/sdk/browser/tools.d.ts +99 -2
- package/dist/sdk/browser/tools.js +6 -1
- package/dist/sdk/commands/registry.d.ts +5 -0
- package/dist/sdk/commands/registry.js +1 -0
- package/dist/sdk/harness.d.ts +4 -0
- package/dist/sdk/harness.js +1 -1
- package/dist/sdk/pi/retryPolicy.d.ts +19 -0
- package/dist/sdk/pi/retryPolicy.js +11 -0
- package/dist/sdk/prompt/compile.d.ts +0 -2
- package/dist/sdk/prompt/compile.js +2 -6
- package/dist/sdk/prompt/reporting.d.ts +1 -1
- package/dist/sdk/prompt/reporting.js +30 -13
- package/dist/sdk/prompt/standard.d.ts +1 -1
- package/dist/sdk/prompt/standard.js +1 -1
- package/dist/sdk/remote/webClient.js +6 -2
- package/dist/sdk/sessions/contracts.d.ts +2 -0
- package/dist/sdk/sessions/contracts.js +5 -1
- package/dist/sdk/sessions/planMode.d.ts +16 -6
- package/dist/sdk/sessions/planMode.js +59 -23
- package/dist/sdk/sessions/runner.js +14 -5
- package/dist/sdk/sessions/workSlice.d.ts +15 -1
- package/dist/sdk/sessions/workSlice.js +17 -9
- package/dist/sdk/terminal/controller.d.ts +6 -0
- package/dist/sdk/terminal/controller.js +44 -5
- package/dist/sdk/terminal/opentui/model.js +3 -1
- package/dist/sdk/terminal/piExtension.js +7 -2
- package/dist/sdk/terminal/stallWatch.d.ts +18 -0
- package/dist/sdk/terminal/stallWatch.js +20 -0
- package/dist/sdk/terminal/subagentMonitor.d.ts +2 -0
- package/dist/sdk/terminal/subagentMonitor.js +4 -1
- package/dist/sdk/terminal/terminals.js +35 -4
- package/dist/sdk/terminal/toolName.d.ts +6 -0
- package/dist/sdk/terminal/toolName.js +12 -0
- package/dist/sdk/terminal/tui/commandPages.d.ts +1 -1
- package/dist/sdk/terminal/tui/commandPages.js +3 -0
- package/dist/sdk/terminal/tui/commands.d.ts +14 -0
- package/dist/sdk/terminal/tui/commands.js +4 -1
- package/dist/sdk/terminal/tui/pages/followUpQueuePage.d.ts +34 -0
- package/dist/sdk/terminal/tui/pages/followUpQueuePage.js +130 -0
- package/dist/sdk/terminal/tui/pages/trajectoryPage.js +37 -7
- package/dist/sdk/terminal/tui/runtime.js +2 -1
- package/dist/sdk/terminal/tui/settingsModel.js +88 -7
- package/dist/sdk/tools/askUser.js +2 -2
- package/dist/sdk/world/world.d.ts +12 -2
- package/dist/sdk/world/world.js +42 -4
- package/dist/terminal-opentui.js +1456 -756
- package/dist/terminal-pi.js +62 -31
- package/docs/README.md +1 -0
- package/docs/architecture/02-sessions-and-runs.md +43 -25
- package/docs/architecture/04-prompt.md +23 -22
- package/docs/architecture/10-server-protocol.md +23 -3
- package/docs/architecture/11-terminal.md +41 -29
- package/docs/architecture/12-storage.md +2 -1
- package/docs/architecture/13-world-tools.md +35 -22
- package/docs/architecture/14-surfaces-and-parity.md +24 -20
- package/docs/architecture/README.md +4 -1
- package/docs/architecture/proposals/PROP-0026-connected-sketch-display.md +3 -1
- package/docs/decisions/ADR-0027-connected-sketch-display.md +4 -2
- package/docs/decisions/ADR-0076-codex-final-answer-verbatim.md +6 -1
- package/docs/decisions/ADR-0077-plan-runs-may-conclude-no-plan-needed.md +120 -0
- package/docs/decisions/ADR-0078-child-aware-stall-watch.md +46 -0
- package/docs/decisions/ADR-0079-phase-led-reporting-guidance.md +72 -0
- package/docs/decisions/ADR-0080-fixed-width-question-sketch-canvas.md +50 -0
- package/docs/decisions/ADR-0081-no-work-slice-budget-by-default.md +36 -0
- package/docs/decisions/ADR-0082-connected-brave-browser.md +72 -0
- package/docs/parity.md +7 -5
- package/docs/releases/1.5.0-unstable.md +12 -3
- package/docs/releases/1.5.1-unstable.md +24 -0
- package/docs/releases/1.6.0-unstable.md +59 -0
- package/docs/study/00-index.md +17 -1
- package/docs/study/STUDY-0007-claude-code-prompt.md +162 -0
- package/docs/study/STUDY-0008-browser-control.md +281 -0
- package/docs/study/STUDY-0009-claude-code-reporting.md +205 -0
- package/docs/study/sources/claude-code.md +51 -39
- package/docs/terminal-tui.md +46 -18
- package/package.json +2 -1
|
@@ -0,0 +1,321 @@
|
|
|
1
|
+
const isRecord = (value) => Boolean(value && typeof value === 'object' && !Array.isArray(value));
|
|
2
|
+
const escaped = (value) => value.replace(/[&<>]/g, char => ({ '&': '&', '<': '<', '>': '>' })[char]);
|
|
3
|
+
const quote = (value) => JSON.stringify(value.slice(0, 240));
|
|
4
|
+
const ACTIONABLE = new Set(['button', 'link', 'textbox', 'combobox', 'checkbox', 'radio', 'menuitem', 'tab', 'switch', 'option']);
|
|
5
|
+
export class BraveBrowserSession {
|
|
6
|
+
options;
|
|
7
|
+
tabs = new Map();
|
|
8
|
+
current = null;
|
|
9
|
+
queue = Promise.resolve();
|
|
10
|
+
idle = null;
|
|
11
|
+
idleMs;
|
|
12
|
+
downloads = [];
|
|
13
|
+
profileNote = 'Brave is using your existing profile. Only normal tabs on sites you allowed in the extension are available; Incognito tabs are never exposed.';
|
|
14
|
+
constructor(options) {
|
|
15
|
+
this.options = options;
|
|
16
|
+
this.idleMs = options.idleMs;
|
|
17
|
+
}
|
|
18
|
+
run(work) {
|
|
19
|
+
const next = this.queue.then(work, work);
|
|
20
|
+
this.queue = next.catch(() => undefined);
|
|
21
|
+
this.touch();
|
|
22
|
+
return next;
|
|
23
|
+
}
|
|
24
|
+
touch() {
|
|
25
|
+
if (this.idle)
|
|
26
|
+
clearTimeout(this.idle);
|
|
27
|
+
this.idle = setTimeout(this.options.onIdle, this.idleMs);
|
|
28
|
+
this.idle.unref?.();
|
|
29
|
+
}
|
|
30
|
+
async invoke(operation) {
|
|
31
|
+
const response = await this.options.bridge.request(operation);
|
|
32
|
+
if (!isRecord(response) || typeof response.ok !== 'boolean')
|
|
33
|
+
throw new Error('The Brave extension returned an invalid response.');
|
|
34
|
+
if (!response.ok)
|
|
35
|
+
throw new Error(typeof response.error === 'string' ? response.error : 'The Brave extension could not complete the operation.');
|
|
36
|
+
return response.value;
|
|
37
|
+
}
|
|
38
|
+
idFor(tabId) {
|
|
39
|
+
return `b${tabId.toString(36)}`;
|
|
40
|
+
}
|
|
41
|
+
async refreshTabs() {
|
|
42
|
+
const remote = await this.invoke({ op: 'tabs.list' });
|
|
43
|
+
if (!Array.isArray(remote))
|
|
44
|
+
throw new Error('The Brave extension returned an invalid tab list.');
|
|
45
|
+
const live = new Set();
|
|
46
|
+
this.current = null;
|
|
47
|
+
for (const entry of remote) {
|
|
48
|
+
if (!isRecord(entry) || !Number.isInteger(entry.tabId) || typeof entry.url !== 'string' || typeof entry.title !== 'string' || !Number.isInteger(entry.generation))
|
|
49
|
+
continue;
|
|
50
|
+
if (await this.options.network.refusal(entry.url))
|
|
51
|
+
continue;
|
|
52
|
+
const tabId = Number(entry.tabId);
|
|
53
|
+
const id = this.idFor(tabId);
|
|
54
|
+
live.add(id);
|
|
55
|
+
const old = this.tabs.get(id);
|
|
56
|
+
if (old) {
|
|
57
|
+
old.remote = { tabId, url: entry.url, title: entry.title, generation: Number(entry.generation), active: entry.active === true };
|
|
58
|
+
old.generation = Number(entry.generation);
|
|
59
|
+
}
|
|
60
|
+
else {
|
|
61
|
+
this.tabs.set(id, {
|
|
62
|
+
id,
|
|
63
|
+
remote: { tabId, url: entry.url, title: entry.title, generation: Number(entry.generation), active: entry.active === true },
|
|
64
|
+
generation: Number(entry.generation), refs: new Map(), refsGeneration: -1,
|
|
65
|
+
page: new BravePage(this, id)
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
if (entry.active === true)
|
|
69
|
+
this.current = id;
|
|
70
|
+
}
|
|
71
|
+
for (const id of this.tabs.keys())
|
|
72
|
+
if (!live.has(id))
|
|
73
|
+
this.tabs.delete(id);
|
|
74
|
+
if (this.current && !this.tabs.has(this.current))
|
|
75
|
+
this.current = null;
|
|
76
|
+
}
|
|
77
|
+
async refresh() {
|
|
78
|
+
await this.refreshTabs();
|
|
79
|
+
}
|
|
80
|
+
async refreshOne(id) {
|
|
81
|
+
await this.refreshTabs();
|
|
82
|
+
const tab = this.tabs.get(id);
|
|
83
|
+
if (!tab)
|
|
84
|
+
throw new Error(`Brave tab ${id} is no longer available. The tab may have closed, or the page may have moved to a site you have not allowed.`);
|
|
85
|
+
return tab;
|
|
86
|
+
}
|
|
87
|
+
get(id) {
|
|
88
|
+
const tab = this.tabs.get(id);
|
|
89
|
+
if (!tab)
|
|
90
|
+
throw new Error(`There is no tab ${id}. browser_tabs lists the available tabs.`);
|
|
91
|
+
return tab;
|
|
92
|
+
}
|
|
93
|
+
async tab(id) {
|
|
94
|
+
await this.refreshTabs();
|
|
95
|
+
if (id) {
|
|
96
|
+
const tab = this.tabs.get(id);
|
|
97
|
+
if (!tab)
|
|
98
|
+
throw new Error(`There is no tab ${id}. browser_tabs lists the available tabs.`);
|
|
99
|
+
return tab;
|
|
100
|
+
}
|
|
101
|
+
if (this.current && this.tabs.has(this.current))
|
|
102
|
+
return this.tabs.get(this.current);
|
|
103
|
+
return this.open();
|
|
104
|
+
}
|
|
105
|
+
async open() {
|
|
106
|
+
const tabId = await this.invoke({ op: 'tabs.create' });
|
|
107
|
+
if (!Number.isInteger(tabId))
|
|
108
|
+
throw new Error('The Brave extension did not return the new tab.');
|
|
109
|
+
await this.refreshTabs();
|
|
110
|
+
const tab = this.tabs.get(this.idFor(tabId));
|
|
111
|
+
if (!tab)
|
|
112
|
+
throw new Error('Brave created a tab but it is not available yet. Try browser_tabs again.');
|
|
113
|
+
this.current = tab.id;
|
|
114
|
+
return tab;
|
|
115
|
+
}
|
|
116
|
+
async select(id) {
|
|
117
|
+
const tab = this.get(id);
|
|
118
|
+
await this.invoke({ op: 'tabs.activate', tabId: tab.remote.tabId });
|
|
119
|
+
this.current = id;
|
|
120
|
+
await this.refreshTabs();
|
|
121
|
+
return this.get(id);
|
|
122
|
+
}
|
|
123
|
+
list() {
|
|
124
|
+
return [...this.tabs.entries()].map(([id, tab]) => ({ id, url: tab.remote.url, current: id === this.current }));
|
|
125
|
+
}
|
|
126
|
+
get network() {
|
|
127
|
+
return this.options.network;
|
|
128
|
+
}
|
|
129
|
+
get started() {
|
|
130
|
+
return true;
|
|
131
|
+
}
|
|
132
|
+
async close() {
|
|
133
|
+
if (this.idle)
|
|
134
|
+
clearTimeout(this.idle);
|
|
135
|
+
this.tabs.clear();
|
|
136
|
+
this.current = null;
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
class BravePage {
|
|
140
|
+
session;
|
|
141
|
+
id;
|
|
142
|
+
refs = new Map();
|
|
143
|
+
constructor(session, id) {
|
|
144
|
+
this.session = session;
|
|
145
|
+
this.id = id;
|
|
146
|
+
}
|
|
147
|
+
get tab() {
|
|
148
|
+
return this.session.get(this.id);
|
|
149
|
+
}
|
|
150
|
+
async refresh() {
|
|
151
|
+
const tab = await this.session.refreshOne(this.id);
|
|
152
|
+
tab.generation = tab.remote.generation;
|
|
153
|
+
}
|
|
154
|
+
url() {
|
|
155
|
+
return this.tab.remote.url;
|
|
156
|
+
}
|
|
157
|
+
async title() {
|
|
158
|
+
await this.refresh();
|
|
159
|
+
return this.tab.remote.title;
|
|
160
|
+
}
|
|
161
|
+
async goto(url) {
|
|
162
|
+
await this.session.invoke({ op: 'page.navigate', tabId: this.tab.remote.tabId, url });
|
|
163
|
+
await this.refresh();
|
|
164
|
+
return null;
|
|
165
|
+
}
|
|
166
|
+
async goBack() {
|
|
167
|
+
await this.session.invoke({ op: 'page.back', tabId: this.tab.remote.tabId });
|
|
168
|
+
await this.refresh();
|
|
169
|
+
return null;
|
|
170
|
+
}
|
|
171
|
+
async goForward() {
|
|
172
|
+
await this.session.invoke({ op: 'page.forward', tabId: this.tab.remote.tabId });
|
|
173
|
+
await this.refresh();
|
|
174
|
+
return null;
|
|
175
|
+
}
|
|
176
|
+
async reload() {
|
|
177
|
+
await this.session.invoke({ op: 'page.reload', tabId: this.tab.remote.tabId });
|
|
178
|
+
await this.refresh();
|
|
179
|
+
return null;
|
|
180
|
+
}
|
|
181
|
+
async close() {
|
|
182
|
+
await this.session.invoke({ op: 'tabs.close', tabId: this.tab.remote.tabId });
|
|
183
|
+
}
|
|
184
|
+
async ariaSnapshot(options = { mode: 'ai' }) {
|
|
185
|
+
const response = await this.session.invoke({
|
|
186
|
+
op: 'page.snapshot', tabId: this.tab.remote.tabId,
|
|
187
|
+
...(options.depth ? { depth: Math.max(1, Math.min(40, options.depth)) } : {})
|
|
188
|
+
});
|
|
189
|
+
const tab = this.tab;
|
|
190
|
+
tab.generation = response.generation;
|
|
191
|
+
tab.remote.generation = response.generation;
|
|
192
|
+
this.refs.clear();
|
|
193
|
+
const nodes = Array.isArray(response.nodes) ? response.nodes : [];
|
|
194
|
+
const byId = new Map(nodes.filter(node => node && typeof node.nodeId === 'string').map(node => [node.nodeId, node]));
|
|
195
|
+
const lines = [];
|
|
196
|
+
let nextRef = 1;
|
|
197
|
+
const walk = (node, depth) => {
|
|
198
|
+
if (depth > (options.depth ?? 12))
|
|
199
|
+
return;
|
|
200
|
+
const role = node.role || 'text';
|
|
201
|
+
const name = node.name?.trim() ?? '';
|
|
202
|
+
const actionable = ACTIONABLE.has(role) && Number.isInteger(node.backendDOMNodeId);
|
|
203
|
+
if (!node.ignored && role !== 'rootWebArea' && role !== 'none' && (name || actionable)) {
|
|
204
|
+
const ref = actionable ? `e${nextRef++}` : null;
|
|
205
|
+
if (ref)
|
|
206
|
+
this.refs.set(ref, { backendDOMNodeId: node.backendDOMNodeId, generation: response.generation });
|
|
207
|
+
const suffix = ref ? ` [ref=${ref}]` : '';
|
|
208
|
+
const level = role === 'heading' && Number.isInteger(node.level) ? ` [level=${node.level}]` : '';
|
|
209
|
+
lines.push(`${' '.repeat(depth)}- ${role}${name ? ` ${quote(name)}` : ''}${level}${suffix}`);
|
|
210
|
+
}
|
|
211
|
+
for (const childId of node.childIds ?? []) {
|
|
212
|
+
const child = byId.get(childId);
|
|
213
|
+
if (child)
|
|
214
|
+
walk(child, depth + 1);
|
|
215
|
+
}
|
|
216
|
+
};
|
|
217
|
+
const roots = nodes.filter(node => !node.parentId || !byId.has(node.parentId));
|
|
218
|
+
for (const root of roots)
|
|
219
|
+
walk(root, 0);
|
|
220
|
+
return lines.join('\n');
|
|
221
|
+
}
|
|
222
|
+
async content() {
|
|
223
|
+
await this.refresh();
|
|
224
|
+
const value = await this.session.invoke({ op: 'page.text', tabId: this.tab.remote.tabId });
|
|
225
|
+
this.tab.remote.title = value.title;
|
|
226
|
+
return `<html><head><title>${escaped(value.title)}</title></head><body>${escaped(value.text)}</body></html>`;
|
|
227
|
+
}
|
|
228
|
+
async screenshot(options) {
|
|
229
|
+
return this.capture(options);
|
|
230
|
+
}
|
|
231
|
+
getByRef(ref) {
|
|
232
|
+
const target = this.refs.get(ref);
|
|
233
|
+
if (!target)
|
|
234
|
+
throw new Error(`There is no live element for ref ${ref}. Take browser_snapshot again.`);
|
|
235
|
+
return new BraveLocator(this, target);
|
|
236
|
+
}
|
|
237
|
+
async evaluate(expression) {
|
|
238
|
+
await this.refresh();
|
|
239
|
+
return this.session.invoke({ op: 'page.evaluate', tabId: this.tab.remote.tabId, expression });
|
|
240
|
+
}
|
|
241
|
+
keyboard = {
|
|
242
|
+
press: async (key) => {
|
|
243
|
+
await this.session.invoke({ op: 'page.press', tabId: this.tab.remote.tabId, key });
|
|
244
|
+
}
|
|
245
|
+
};
|
|
246
|
+
mouse = {
|
|
247
|
+
wheel: async (x, y) => {
|
|
248
|
+
await this.session.invoke({ op: 'page.scroll', tabId: this.tab.remote.tabId, x, y });
|
|
249
|
+
}
|
|
250
|
+
};
|
|
251
|
+
async waitForTimeout(ms) {
|
|
252
|
+
await new Promise(resolve => setTimeout(resolve, Math.min(10_000, Math.max(0, ms))));
|
|
253
|
+
}
|
|
254
|
+
getByText(text) {
|
|
255
|
+
return { first: () => ({ waitFor: async ({ timeout }) => {
|
|
256
|
+
const end = Date.now() + timeout;
|
|
257
|
+
while (Date.now() < end) {
|
|
258
|
+
const found = await this.session.invoke({ op: 'page.containsText', tabId: this.tab.remote.tabId, text });
|
|
259
|
+
if (found)
|
|
260
|
+
return;
|
|
261
|
+
await new Promise(resolve => setTimeout(resolve, 250));
|
|
262
|
+
}
|
|
263
|
+
const error = new Error(`Timeout ${timeout}ms exceeded while waiting for text`);
|
|
264
|
+
error.name = 'TimeoutError';
|
|
265
|
+
throw error;
|
|
266
|
+
} }) };
|
|
267
|
+
}
|
|
268
|
+
getNode(target) {
|
|
269
|
+
if (target.generation !== this.tab.generation || this.tab.refsGeneration !== this.tab.generation)
|
|
270
|
+
throw new Error(`The page in ${this.id} changed since its last snapshot. Take browser_snapshot again and use a ref from it.`);
|
|
271
|
+
return target.backendDOMNodeId;
|
|
272
|
+
}
|
|
273
|
+
async locatorCall(target, method, args = {}) {
|
|
274
|
+
await this.refresh();
|
|
275
|
+
const backendDOMNodeId = this.getNode(target);
|
|
276
|
+
return this.session.invoke({ op: 'page.element', tabId: this.tab.remote.tabId, backendDOMNodeId, generation: target.generation, method, ...args });
|
|
277
|
+
}
|
|
278
|
+
async capture(options, target) {
|
|
279
|
+
await this.refresh();
|
|
280
|
+
const args = { op: 'page.screenshot', tabId: this.tab.remote.tabId, quality: Math.max(10, Math.min(95, options.quality)) };
|
|
281
|
+
if (target) {
|
|
282
|
+
args.backendDOMNodeId = this.getNode(target);
|
|
283
|
+
args.generation = target.generation;
|
|
284
|
+
}
|
|
285
|
+
const result = await this.session.invoke(args);
|
|
286
|
+
return Buffer.from(result.data, 'base64');
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
class BraveLocator {
|
|
290
|
+
page;
|
|
291
|
+
target;
|
|
292
|
+
constructor(page, target) {
|
|
293
|
+
this.page = page;
|
|
294
|
+
this.target = target;
|
|
295
|
+
}
|
|
296
|
+
async click(options = {}) {
|
|
297
|
+
await this.page.locatorCall({ ...this.target }, 'click', { button: options.button ?? 'left', double: false });
|
|
298
|
+
}
|
|
299
|
+
async dblclick(options = {}) {
|
|
300
|
+
await this.page.locatorCall({ ...this.target }, 'click', { button: options.button ?? 'left', double: true });
|
|
301
|
+
}
|
|
302
|
+
async screenshot(options) {
|
|
303
|
+
return this.page.capture(options, this.target);
|
|
304
|
+
}
|
|
305
|
+
async evaluate(callback) {
|
|
306
|
+
const declaration = callback.toString();
|
|
307
|
+
return this.page.locatorCall({ ...this.target }, 'evaluate', { functionDeclaration: `function(){ return (${declaration})(this) }` });
|
|
308
|
+
}
|
|
309
|
+
async fill(value) {
|
|
310
|
+
await this.page.locatorCall({ ...this.target }, 'fill', { text: value });
|
|
311
|
+
}
|
|
312
|
+
async press(key) {
|
|
313
|
+
await this.page.locatorCall({ ...this.target }, 'press', { key });
|
|
314
|
+
}
|
|
315
|
+
async selectOption(values) {
|
|
316
|
+
return this.page.locatorCall({ ...this.target }, 'select', { values });
|
|
317
|
+
}
|
|
318
|
+
async scrollIntoViewIfNeeded() {
|
|
319
|
+
await this.page.locatorCall({ ...this.target }, 'scrollIntoView');
|
|
320
|
+
}
|
|
321
|
+
}
|
|
@@ -2,7 +2,9 @@ import type { PluginDescriptor } from "../plugins/types.js";
|
|
|
2
2
|
import type { WorldContext, WorldPart } from "../world/world.js";
|
|
3
3
|
import type { ToolSpec } from "../tools/types.js";
|
|
4
4
|
import { NetworkPolicy } from "./network.js";
|
|
5
|
-
import {
|
|
5
|
+
import { type Launcher } from "./session.js";
|
|
6
|
+
import type { BrowserToolSession } from "./tools.js";
|
|
7
|
+
import { BrowserExtensionBridge } from "./extension/bridge.js";
|
|
6
8
|
export declare const BROWSER_PLUGIN: PluginDescriptor;
|
|
7
9
|
/** A browser closes after this long without a call. */
|
|
8
10
|
export declare const IDLE_MS: number;
|
|
@@ -12,12 +14,15 @@ export interface BrowserPartOptions {
|
|
|
12
14
|
/** Replaces the address check in tests. */
|
|
13
15
|
network?: NetworkPolicy;
|
|
14
16
|
idleMs?: number;
|
|
17
|
+
extensionPort?: number;
|
|
15
18
|
}
|
|
16
19
|
/** Foundation-owned isolated browser engine. Plugin tools receive sessions through this interface. */
|
|
17
20
|
export interface BrowserEngine {
|
|
18
|
-
sessions: Map<string,
|
|
21
|
+
sessions: Map<string, BrowserToolSession>;
|
|
22
|
+
extension: BrowserExtensionBridge;
|
|
19
23
|
diagnostics: NonNullable<WorldPart['diagnostics']>;
|
|
20
24
|
tools(context: WorldContext): ToolSpec[];
|
|
25
|
+
toolsForSession(session: import("../tools/types.js").ToolSession, context: WorldContext): ToolSpec[];
|
|
21
26
|
release(sessionKey: string): Promise<void>;
|
|
22
27
|
close(): Promise<void>;
|
|
23
28
|
}
|
|
@@ -5,11 +5,13 @@ import { readBrowserConfig } from "./config.js";
|
|
|
5
5
|
import { browserDiagnostic } from "./diagnostics.js";
|
|
6
6
|
import { NetworkPolicy } from "./network.js";
|
|
7
7
|
import { BrowserSession, fileName, launchBrowser } from "./session.js";
|
|
8
|
+
import { BrowserExtensionBridge } from "./extension/bridge.js";
|
|
9
|
+
import { BraveBrowserSession } from "./extension/session.js";
|
|
8
10
|
import { BROWSER_TOOL_NAMES, browserTools } from "./tools.js";
|
|
9
11
|
export const BROWSER_PLUGIN = {
|
|
10
12
|
id: 'browser',
|
|
11
13
|
title: 'Browser',
|
|
12
|
-
description: '
|
|
14
|
+
description: 'The browser backend selected by the person: an isolated Harness profile or the connected Brave profile. Brave exposes only normal tabs on sites explicitly allowed in the extension, never Incognito; a disconnect refuses calls and never switches backends. Sending, buying, deleting, account changes and granting access always ask you.',
|
|
13
15
|
manage: 'tools',
|
|
14
16
|
switchable: { on: true },
|
|
15
17
|
// Its tools are listed in the World group, which the Web plugin owns.
|
|
@@ -19,45 +21,76 @@ export const BROWSER_PLUGIN = {
|
|
|
19
21
|
export const IDLE_MS = 15 * 60_000;
|
|
20
22
|
export function createBrowserEngine(options) {
|
|
21
23
|
const sessions = new Map();
|
|
24
|
+
const owners = new Map();
|
|
22
25
|
const diagnosticKeys = new Set();
|
|
23
26
|
const config = () => readBrowserConfig(options.stateDir);
|
|
24
27
|
const network = options.network ?? new NetworkPolicy(config);
|
|
25
|
-
const
|
|
26
|
-
|
|
28
|
+
const extension = new BrowserExtensionBridge({ stateDir: options.stateDir, port: options.extensionPort });
|
|
29
|
+
const releaseExact = async (key) => {
|
|
30
|
+
const session = sessions.get(key);
|
|
31
|
+
sessions.delete(key);
|
|
32
|
+
owners.delete(key);
|
|
33
|
+
await session?.close();
|
|
34
|
+
if (diagnosticKeys.delete(key))
|
|
35
|
+
rmSync(join(options.stateDir, 'browser', 'downloads', fileName(key)), { recursive: true, force: true });
|
|
36
|
+
};
|
|
37
|
+
const sessionFor = (context, backend) => {
|
|
38
|
+
const key = `${backend}:${context.sessionKey}`;
|
|
39
|
+
let session = sessions.get(key);
|
|
27
40
|
if (!session) {
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
const diagnostic = context.caller.surface === 'doctor';
|
|
31
|
-
if (diagnostic)
|
|
41
|
+
owners.set(key, context.sessionKey);
|
|
42
|
+
if (context.caller.surface === 'doctor') {
|
|
32
43
|
diagnosticKeys.add(key);
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
44
|
+
session = new BrowserSession({
|
|
45
|
+
key,
|
|
46
|
+
profileDir: null,
|
|
47
|
+
downloadsDir: join(options.stateDir, 'browser', 'downloads', fileName(key)),
|
|
48
|
+
config: () => ({ ...config(), backend: 'isolated', cdpUrl: null, headed: false }),
|
|
49
|
+
network,
|
|
50
|
+
launch: options.launch ?? launchBrowser,
|
|
51
|
+
idleMs: options.idleMs ?? IDLE_MS,
|
|
52
|
+
onIdle: () => void releaseExact(key)
|
|
53
|
+
});
|
|
54
|
+
}
|
|
55
|
+
else if (backend === 'brave') {
|
|
56
|
+
session = new BraveBrowserSession({ bridge: extension, network, idleMs: options.idleMs ?? IDLE_MS, onIdle: () => void releaseExact(key) });
|
|
57
|
+
}
|
|
58
|
+
else {
|
|
59
|
+
const owner = context.caller.agentId ?? context.caller.projectId ?? 'default';
|
|
60
|
+
session = new BrowserSession({
|
|
61
|
+
key,
|
|
62
|
+
profileDir: join(options.stateDir, 'browser', 'profiles', fileName(owner)),
|
|
63
|
+
downloadsDir: join(options.stateDir, 'browser', 'downloads', fileName(key)),
|
|
64
|
+
config,
|
|
65
|
+
network,
|
|
66
|
+
launch: options.launch ?? launchBrowser,
|
|
67
|
+
idleMs: options.idleMs ?? IDLE_MS,
|
|
68
|
+
onIdle: () => void releaseExact(key)
|
|
69
|
+
});
|
|
70
|
+
}
|
|
43
71
|
sessions.set(key, session);
|
|
44
72
|
}
|
|
45
73
|
return session;
|
|
46
74
|
};
|
|
47
|
-
const
|
|
48
|
-
const
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
75
|
+
const toolsForContext = (context) => {
|
|
76
|
+
const backend = context.caller.surface !== 'doctor' && config().backend === 'brave' ? 'brave' : 'isolated';
|
|
77
|
+
const session = sessionFor(context, backend);
|
|
78
|
+
return browserTools(context, () => session);
|
|
79
|
+
};
|
|
80
|
+
const release = async (sessionKey) => {
|
|
81
|
+
const keys = [...owners.entries()].filter(([, owner]) => owner === sessionKey).map(([key]) => key);
|
|
82
|
+
await Promise.all(keys.map(releaseExact));
|
|
53
83
|
};
|
|
54
84
|
return {
|
|
55
85
|
diagnostics: () => [browserDiagnostic(config)],
|
|
56
86
|
sessions,
|
|
57
|
-
|
|
87
|
+
extension,
|
|
88
|
+
tools: toolsForContext,
|
|
89
|
+
toolsForSession: (_session, context) => toolsForContext(context),
|
|
58
90
|
release,
|
|
59
91
|
close: async () => {
|
|
60
|
-
await Promise.all([...sessions.keys()].map(
|
|
92
|
+
await Promise.all([...sessions.keys()].map(releaseExact));
|
|
93
|
+
await extension.close();
|
|
61
94
|
}
|
|
62
95
|
};
|
|
63
96
|
}
|
|
@@ -2,6 +2,9 @@ import { pick } from "../server/params.js";
|
|
|
2
2
|
export function browserServerMethods(harness) {
|
|
3
3
|
return {
|
|
4
4
|
'browser.config': () => harness.world.browserConfig(),
|
|
5
|
-
'browser.configure': (params) => harness.world.configureBrowser(pick(params, ['headed', 'cdpUrl', 'executablePath', 'allowPrivate', 'allowedDomains']))
|
|
5
|
+
'browser.configure': (params) => harness.world.configureBrowser(pick(params, ['backend', 'headed', 'cdpUrl', 'executablePath', 'allowPrivate', 'allowedDomains'])),
|
|
6
|
+
'browser.extension.status': () => harness.world.browserExtensionStatus(),
|
|
7
|
+
'browser.extension.start': () => harness.world.browserExtensionStart(),
|
|
8
|
+
'browser.extension.stop': () => harness.world.browserExtensionStop()
|
|
6
9
|
};
|
|
7
10
|
}
|
|
@@ -1,5 +1,102 @@
|
|
|
1
1
|
import type { ToolSpec } from "../tools/types.js";
|
|
2
|
+
import type { NetworkPolicy } from "./network.js";
|
|
2
3
|
import type { WorldContext } from "../world/world.js";
|
|
3
|
-
import type {
|
|
4
|
+
import type { Download } from "./session.js";
|
|
4
5
|
export { BROWSER_TOOL_NAMES, SNAPSHOT_LINES, SNAPSHOT_CHARS } from "./contracts.js";
|
|
5
|
-
export
|
|
6
|
+
export interface BrowserToolElement {
|
|
7
|
+
getAttribute(name: string): string | null;
|
|
8
|
+
}
|
|
9
|
+
export interface BrowserToolLocator {
|
|
10
|
+
click(options?: {
|
|
11
|
+
button?: 'left' | 'right' | 'middle';
|
|
12
|
+
}): Promise<void>;
|
|
13
|
+
dblclick(options?: {
|
|
14
|
+
button?: 'left' | 'right' | 'middle';
|
|
15
|
+
}): Promise<void>;
|
|
16
|
+
screenshot(options: {
|
|
17
|
+
type: 'jpeg';
|
|
18
|
+
quality: number;
|
|
19
|
+
}): Promise<Buffer>;
|
|
20
|
+
evaluate<T>(callback: (element: BrowserToolElement) => T): Promise<T>;
|
|
21
|
+
fill(value: string): Promise<void>;
|
|
22
|
+
press(key: string): Promise<void>;
|
|
23
|
+
selectOption(values: string[]): Promise<string[]>;
|
|
24
|
+
scrollIntoViewIfNeeded(): Promise<void>;
|
|
25
|
+
}
|
|
26
|
+
export interface BrowserToolPage {
|
|
27
|
+
url(): string;
|
|
28
|
+
title(): Promise<string>;
|
|
29
|
+
refresh?(): Promise<void>;
|
|
30
|
+
goto(url: string, options?: {
|
|
31
|
+
waitUntil?: 'domcontentloaded';
|
|
32
|
+
}): Promise<{
|
|
33
|
+
status(): number;
|
|
34
|
+
} | null>;
|
|
35
|
+
goBack(options?: {
|
|
36
|
+
waitUntil?: 'domcontentloaded';
|
|
37
|
+
}): Promise<{
|
|
38
|
+
status(): number;
|
|
39
|
+
} | null>;
|
|
40
|
+
goForward(options?: {
|
|
41
|
+
waitUntil?: 'domcontentloaded';
|
|
42
|
+
}): Promise<{
|
|
43
|
+
status(): number;
|
|
44
|
+
} | null>;
|
|
45
|
+
reload(options?: {
|
|
46
|
+
waitUntil?: 'domcontentloaded';
|
|
47
|
+
}): Promise<{
|
|
48
|
+
status(): number;
|
|
49
|
+
} | null>;
|
|
50
|
+
close(): Promise<void>;
|
|
51
|
+
ariaSnapshot(options?: {
|
|
52
|
+
mode: 'ai';
|
|
53
|
+
depth?: number;
|
|
54
|
+
timeout?: number;
|
|
55
|
+
}): Promise<string>;
|
|
56
|
+
content(): Promise<string>;
|
|
57
|
+
screenshot(options: {
|
|
58
|
+
type: 'jpeg';
|
|
59
|
+
quality: number;
|
|
60
|
+
}): Promise<Buffer>;
|
|
61
|
+
getByRef(ref: string): BrowserToolLocator;
|
|
62
|
+
evaluate<T = unknown>(expression: string): Promise<T>;
|
|
63
|
+
keyboard: {
|
|
64
|
+
press(key: string): Promise<void>;
|
|
65
|
+
};
|
|
66
|
+
mouse: {
|
|
67
|
+
wheel(x: number, y: number): Promise<void>;
|
|
68
|
+
};
|
|
69
|
+
waitForTimeout(ms: number): Promise<void>;
|
|
70
|
+
getByText(text: string): {
|
|
71
|
+
first(): {
|
|
72
|
+
waitFor(options: {
|
|
73
|
+
timeout: number;
|
|
74
|
+
}): Promise<void>;
|
|
75
|
+
};
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
export interface BrowserToolTab {
|
|
79
|
+
id: string;
|
|
80
|
+
page: BrowserToolPage;
|
|
81
|
+
generation: number;
|
|
82
|
+
refs: Map<string, string>;
|
|
83
|
+
refsGeneration: number;
|
|
84
|
+
}
|
|
85
|
+
export interface BrowserToolSession {
|
|
86
|
+
run<T>(work: () => Promise<T>): Promise<T>;
|
|
87
|
+
tab(id?: string | null): Promise<BrowserToolTab>;
|
|
88
|
+
open(): Promise<BrowserToolTab>;
|
|
89
|
+
select(id: string): BrowserToolTab | Promise<BrowserToolTab>;
|
|
90
|
+
list(): {
|
|
91
|
+
id: string;
|
|
92
|
+
url: string;
|
|
93
|
+
current: boolean;
|
|
94
|
+
}[];
|
|
95
|
+
refresh?(): Promise<void>;
|
|
96
|
+
readonly network: NetworkPolicy;
|
|
97
|
+
readonly downloads: Download[];
|
|
98
|
+
close(): Promise<void>;
|
|
99
|
+
readonly profileNote: string | null;
|
|
100
|
+
readonly started: boolean;
|
|
101
|
+
}
|
|
102
|
+
export declare function browserTools(context: WorldContext, session: () => BrowserToolSession): ToolSpec[];
|
|
@@ -10,6 +10,7 @@ const isTimeout = (error) => error instanceof Error && (error.name === 'TimeoutE
|
|
|
10
10
|
const UNKNOWN = 'Its outcome is unknown: take browser_snapshot before trying again, so nothing is clicked, sent or bought twice.';
|
|
11
11
|
const text = (value) => (typeof value === 'string' ? value : '');
|
|
12
12
|
async function where(tab) {
|
|
13
|
+
await tab.page.refresh?.();
|
|
13
14
|
const title = await tab.page.title().catch(() => '');
|
|
14
15
|
return `Tab ${tab.id}: ${tab.page.url()}${title ? ` — ${title}` : ''}`;
|
|
15
16
|
}
|
|
@@ -30,6 +31,7 @@ async function approve(context, tool, tab, target, args) {
|
|
|
30
31
|
return null;
|
|
31
32
|
if (!context.attended || !context.run)
|
|
32
33
|
return `Refused: this would ${verdict.reason}, and nobody can approve it in this run.`;
|
|
34
|
+
await tab.page.refresh?.();
|
|
33
35
|
const before = { url: tab.page.url(), generation: tab.generation };
|
|
34
36
|
let host = '';
|
|
35
37
|
try {
|
|
@@ -41,6 +43,7 @@ async function approve(context, tool, tab, target, args) {
|
|
|
41
43
|
const refusal = await askForAction({ run: context.run, tool, target: `${target}${host ? ` on ${host}` : ''}`, args, reason: verdict.reason });
|
|
42
44
|
if (refusal)
|
|
43
45
|
return refusal;
|
|
46
|
+
await tab.page.refresh?.();
|
|
44
47
|
if (tab.page.url() !== before.url || tab.generation !== before.generation)
|
|
45
48
|
return 'The page changed while waiting for approval, so nothing was done. Take browser_snapshot and decide again.';
|
|
46
49
|
return null;
|
|
@@ -113,6 +116,7 @@ export function browserTools(context, session) {
|
|
|
113
116
|
...browserToolSpec('browser_read'),
|
|
114
117
|
execute: (input) => act('browser_read', async () => {
|
|
115
118
|
const tab = await session().tab(text(input.tab) || null);
|
|
119
|
+
await tab.page.refresh?.();
|
|
116
120
|
const { title, text: markdown } = htmlToMarkdown(await tab.page.content());
|
|
117
121
|
const offset = Math.max(0, Math.floor(Number(input.offset)) || 0);
|
|
118
122
|
const end = Math.min(markdown.length, offset + PAGE_CHARS);
|
|
@@ -237,7 +241,7 @@ export function browserTools(context, session) {
|
|
|
237
241
|
return untrusted(`Opened ${await where(tab)}`);
|
|
238
242
|
}
|
|
239
243
|
if (input.op === 'switch')
|
|
240
|
-
return untrusted(`Now in ${await where(browser.select(text(input.tab)))}`);
|
|
244
|
+
return untrusted(`Now in ${await where(await browser.select(text(input.tab)))}`);
|
|
241
245
|
if (input.op === 'close') {
|
|
242
246
|
const tab = await browser.tab(text(input.tab) || null);
|
|
243
247
|
await tab.page.close();
|
|
@@ -245,6 +249,7 @@ export function browserTools(context, session) {
|
|
|
245
249
|
}
|
|
246
250
|
if (!browser.started)
|
|
247
251
|
return 'No tabs are open yet. browser_navigate opens one.';
|
|
252
|
+
await browser.refresh?.();
|
|
248
253
|
return untrusted(browser.list().map((tab) => `${tab.current ? '*' : ' '} ${tab.id} ${tab.url}`).join('\n'));
|
|
249
254
|
})
|
|
250
255
|
}
|
|
@@ -143,6 +143,11 @@ export declare const CORE_COMMANDS: readonly [{
|
|
|
143
143
|
readonly group: "Inspect";
|
|
144
144
|
readonly kind: "session";
|
|
145
145
|
readonly methods: readonly ["runs.todo"];
|
|
146
|
+
}, {
|
|
147
|
+
readonly name: "follow-up";
|
|
148
|
+
readonly description: "Manage queued follow-up and steering messages";
|
|
149
|
+
readonly group: "Run";
|
|
150
|
+
readonly kind: "page";
|
|
146
151
|
}, {
|
|
147
152
|
readonly name: "trajectory";
|
|
148
153
|
readonly args: "[run-id]";
|
|
@@ -21,6 +21,7 @@ export const CORE_COMMANDS = [
|
|
|
21
21
|
{ name: 'copy', description: 'Copy the last reply', group: 'Inspect', kind: 'local', methods: ['sessions.lastReply'] },
|
|
22
22
|
{ name: 'export', args: '[path.html|path.jsonl]', description: 'Export this branch to HTML or JSONL', group: 'Inspect', kind: 'session', methods: ['sessions.export', 'sessions.run'] },
|
|
23
23
|
{ name: 'todo', aliases: ['checklist'], description: 'Show the current todo list', group: 'Inspect', kind: 'session', methods: ['runs.todo'] },
|
|
24
|
+
{ name: 'follow-up', description: 'Manage queued follow-up and steering messages', group: 'Run', kind: 'page' },
|
|
24
25
|
{ name: 'trajectory', args: '[run-id]', description: 'Inspect recorded runs for this session', group: 'Inspect', kind: 'page', methods: ['runs.list', 'runs.events'] },
|
|
25
26
|
{ name: 'reload', description: 'Reload trusted resources and keybindings', group: 'Harness', kind: 'session', methods: ['sessions.reload'] },
|
|
26
27
|
{ name: 'accounts', description: 'Sign in, switch, check and remove Pi accounts', group: 'Harness', kind: 'page', methods: ['accounts.list'] },
|
package/dist/sdk/harness.d.ts
CHANGED
|
@@ -33,6 +33,10 @@ import type { WorktreesOptions } from "./worktrees/types.js";
|
|
|
33
33
|
export interface HarnessOptions {
|
|
34
34
|
/** Disable the generic backend when a host supplies its own browser tools. */
|
|
35
35
|
browser?: boolean;
|
|
36
|
+
/** Enables the Brave extension bridge when a host supplies its own isolated browser. */
|
|
37
|
+
browserExtension?: {
|
|
38
|
+
port?: number;
|
|
39
|
+
};
|
|
36
40
|
hub?: HubOptions;
|
|
37
41
|
/** Where knowledge and history live. May be set later with `open`. */
|
|
38
42
|
dataRoot?: string | null;
|