@celljs/ta-workspace 4.7.0 → 4.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,440 @@
1
+ import type { WebTool } from '@celljs/ta-web/lib/browser/extension/web-extension-api';
2
+ import { PACKAGE, pi } from './host';
3
+ import { type AXNode, BrowserSnapshot } from './util/browser-snapshot';
4
+ import { WorkspaceUtil } from './util/workspace';
5
+
6
+ /** The part of Electron's `<webview>` the browser uses (Ta Desktop; the app guards it, embedded-browser.ts). */
7
+ export interface WebviewElement extends HTMLElement {
8
+ loadURL(url: string): Promise<void>;
9
+ reload(): void;
10
+ stop(): void;
11
+ goBack(): void;
12
+ goForward(): void;
13
+ canGoBack(): boolean;
14
+ canGoForward(): boolean;
15
+ isLoading(): boolean;
16
+ getURL(): string;
17
+ getTitle(): string;
18
+ getWebContentsId(): number;
19
+ }
20
+
21
+ /** What the browser view gives the agent while it is open (browser-webview.tsx). */
22
+ export interface BrowserHandle {
23
+ /** Opens an address as the bar does. */
24
+ go(url: string): void;
25
+ /** The webview, once its page can be worked. */
26
+ view(): WebviewElement | undefined;
27
+ /** Shows the page rather than the start page, and that the agent works it. */
28
+ drive(): void;
29
+ /** Whether the view is on screen: a tab the user does not look at is not drawn. */
30
+ shown(): boolean;
31
+ }
32
+
33
+ const ACTIONS = ['navigate', 'back', 'forward', 'reload', 'snapshot', 'click', 'type', 'press', 'scroll', 'screenshot', 'evaluate', 'wait', 'cdp'] as const;
34
+ type Action = (typeof ACTIONS)[number];
35
+ const MAX_OUTPUT = 20_000;
36
+ /** Runs on the element of a field: replaces its text as typing would, or sets it where the page cannot type. */
37
+ const TYPE_INTO = `function (text) {
38
+ this.focus();
39
+ if (typeof this.select === 'function') {
40
+ this.select();
41
+ } else {
42
+ const range = document.createRange();
43
+ range.selectNodeContents(this);
44
+ getSelection().removeAllRanges();
45
+ getSelection().addRange(range);
46
+ }
47
+ const typed = text ? document.execCommand('insertText', false, text) : document.execCommand('delete');
48
+ if (!typed && 'value' in this) {
49
+ const setter = Object.getOwnPropertyDescriptor(Object.getPrototypeOf(this), 'value')?.set;
50
+ setter ? setter.call(this, text) : (this.value = text);
51
+ this.dispatchEvent(new Event('input', { bubbles: true }));
52
+ this.dispatchEvent(new Event('change', { bubbles: true }));
53
+ }
54
+ }`;
55
+ const MAX_WAIT_MS = 30_000;
56
+ /** How long the webview keeps the app's focus around a key, for the focus to reach its page first. */
57
+ const KEY_FOCUS_MS = 50;
58
+
59
+ type Send = (method: string, args?: Record<string, unknown>) => Promise<unknown>;
60
+
61
+ const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
62
+
63
+ /** Waits for `ready` to give a value, every 50 ms, at most `ms`. */
64
+ async function until<T>(ready: () => T | undefined, ms: number): Promise<T | undefined> {
65
+ const end = Date.now() + ms;
66
+ for (;;) {
67
+ const value = ready();
68
+ if (value !== undefined || Date.now() >= end) {
69
+ return value;
70
+ }
71
+ await sleep(50);
72
+ }
73
+ }
74
+
75
+ function clip(text: string): string {
76
+ return text.length > MAX_OUTPUT ? `${text.slice(0, MAX_OUTPUT)}\n… cut at ${MAX_OUTPUT} characters` : text;
77
+ }
78
+
79
+ /**
80
+ * The browser of the workbench as a tool of the agent, in Ta Desktop: the page the user sees, worked
81
+ * through the DevTools protocol (pi.ui.debugWebview). The view is shown while the agent works it.
82
+ */
83
+ export namespace AgentBrowser {
84
+ let handle: BrowserHandle | undefined;
85
+ /** The document of the last snapshot (its loader) and its refs: a ref of another document may name anything. */
86
+ let snapshot: { document: string; refs: Set<number> } | undefined;
87
+
88
+ /** The browser view opened; returns how to forget it. */
89
+ export function attach(next: BrowserHandle): () => void {
90
+ handle = next;
91
+ return () => {
92
+ if (handle === next) {
93
+ handle = undefined;
94
+ }
95
+ };
96
+ }
97
+
98
+ export const TOOL: WebTool = {
99
+ name: 'browser',
100
+ description: [
101
+ "Works the browser in Ta's workbench, on the user's machine: the user sees it, and it keeps the user's sign-ins to sites.",
102
+ 'Actions:',
103
+ '- navigate {url}: open a page (http or https); back, forward, reload',
104
+ '- snapshot: the page as an accessibility tree, one element a line; what one acts on starts with its ref in brackets, [12]',
105
+ '- click {ref} or {x, y}: click it; x and y are CSS pixels of the screenshot',
106
+ '- type {ref, text, submit?}: replace the text of a field with text; submit presses Enter after',
107
+ '- press {key}: a key or a shortcut: "Enter", "Escape", "Tab", "ArrowDown", "a", "Control+A", "Meta+Shift+Z"',
108
+ '- scroll {direction: "down" | "up", amount?}: by amount pixels, a screen by default; or {ref}: until it is shown',
109
+ '- screenshot: what is shown, as a data URL of a JPEG (in codemode, give it to image() to look at it)',
110
+ '- evaluate {expression}: run JavaScript in the page; the JSON of its value, awaited',
111
+ '- wait {text?, ms?}: until the page shows text, or for ms milliseconds; at most 30000',
112
+ '- cdp {method, params?}: a command of the Chrome DevTools Protocol to the page (Accessibility, CSS, DOM, DOMSnapshot, Emulation, Input, Log, Overlay, Page, Runtime)',
113
+ 'After an action it says the title and address of the page; refs change when the page does, so take a snapshot again before acting on a new page.',
114
+ ].join('\n'),
115
+ parameters: {
116
+ type: 'object',
117
+ properties: {
118
+ action: { type: 'string', enum: [...ACTIONS] },
119
+ url: { type: 'string' },
120
+ ref: { type: 'integer', description: 'The number in brackets in the last snapshot.' },
121
+ x: { type: 'number' },
122
+ y: { type: 'number' },
123
+ text: { type: 'string' },
124
+ submit: { type: 'boolean' },
125
+ key: { type: 'string' },
126
+ direction: { type: 'string', enum: ['down', 'up'] },
127
+ amount: { type: 'number' },
128
+ expression: { type: 'string' },
129
+ ms: { type: 'number' },
130
+ method: { type: 'string' },
131
+ params: { type: 'object' },
132
+ },
133
+ required: ['action'],
134
+ },
135
+ execute: (_toolCallId, params) => run(params),
136
+ };
137
+
138
+ async function run(params: Record<string, unknown>): Promise<string> {
139
+ const action = params.action as Action;
140
+ if (!ACTIONS.includes(action)) {
141
+ throw new Error(`Unknown action ${JSON.stringify(params.action)}: use one of ${ACTIONS.join(', ')}.`);
142
+ }
143
+ const browser = await open();
144
+ browser.drive();
145
+ if (action === 'navigate') {
146
+ const url = WorkspaceUtil.toUrl(String(params.url ?? ''));
147
+ if (!url || !/^https?:\/\//i.test(url)) {
148
+ throw new Error('navigate opens pages of the web: give an http or https url.');
149
+ }
150
+ const before = browser.view()?.getURL();
151
+ browser.go(url);
152
+ await settle(browser, before);
153
+ return status(page(browser));
154
+ }
155
+ if (action === 'wait' && !params.text) {
156
+ await sleep(Math.min(MAX_WAIT_MS, Number(params.ms) > 0 ? Number(params.ms) : 1000));
157
+ const shown = browser.view();
158
+ return shown ? status(shown) : 'No page is open.';
159
+ }
160
+ const view = page(browser);
161
+ const send = (method: string, args: Record<string, unknown> = {}) => command(view, method, args);
162
+ switch (action) {
163
+ case 'back':
164
+ case 'forward':
165
+ case 'reload': {
166
+ const before = view.getURL();
167
+ if (action === 'back' && !view.canGoBack()) {
168
+ throw new Error('There is no page to go back to.');
169
+ }
170
+ if (action === 'forward' && !view.canGoForward()) {
171
+ throw new Error('There is no page to go forward to.');
172
+ }
173
+ if (action === 'back') {
174
+ view.goBack();
175
+ } else if (action === 'forward') {
176
+ view.goForward();
177
+ } else {
178
+ view.reload();
179
+ }
180
+ await settle(browser, action === 'reload' ? undefined : before);
181
+ return status(view);
182
+ }
183
+ case 'snapshot': {
184
+ const document = await documentOf(send);
185
+ // Enabled, the tree is kept for the page; a page shown again from the back-forward cache builds it anew.
186
+ await send('Accessibility.enable');
187
+ let text = '';
188
+ for (let attempt = 0; attempt < 10; attempt++) {
189
+ const { nodes } = (await send('Accessibility.getFullAXTree')) as { nodes: AXNode[] };
190
+ text = BrowserSnapshot.render(nodes);
191
+ if (
192
+ text.includes('\n') ||
193
+ !(await evaluate(send, 'Boolean(document.body && document.body.childElementCount)').catch(() => 'false')).includes('true')
194
+ ) {
195
+ break;
196
+ }
197
+ await sleep(200);
198
+ }
199
+ snapshot = { document, refs: new Set([...text.matchAll(/^\s*\[(\d+)\]/gm)].map(match => Number(match[1]))) };
200
+ return clip(`${status(view)}\n\n${text}`);
201
+ }
202
+ case 'click': {
203
+ const point = params.ref === undefined ? { x: Number(params.x), y: Number(params.y) } : await locate(send, await ref(send, params.ref));
204
+ if (!Number.isFinite(point.x) || !Number.isFinite(point.y)) {
205
+ throw new Error('click needs a ref, or x and y.');
206
+ }
207
+ const before = view.getURL();
208
+ // A click gives the webview the focus of the app: it goes back to where the user was typing,
209
+ // while the page, its focus emulated by the app, still takes keys.
210
+ const focused = document.activeElement;
211
+ await send('Input.dispatchMouseEvent', { type: 'mouseMoved', ...point });
212
+ await send('Input.dispatchMouseEvent', { type: 'mousePressed', button: 'left', clickCount: 1, ...point });
213
+ await send('Input.dispatchMouseEvent', { type: 'mouseReleased', button: 'left', clickCount: 1, ...point });
214
+ if (focused instanceof HTMLElement && focused !== view && document.activeElement === view) {
215
+ focused.focus({ preventScroll: true });
216
+ }
217
+ await settle(browser, before, 300);
218
+ return status(view);
219
+ }
220
+ case 'type': {
221
+ const backendNodeId = await ref(send, params.ref);
222
+ await send('DOM.focus', { backendNodeId }).catch(error => stale(error, backendNodeId));
223
+ const { object } = (await send('DOM.resolveNode', { backendNodeId })) as { object: { objectId: string } };
224
+ // As typing does (input events a page listens to), without the keyboard focus of the app, which
225
+ // Input.insertText needs and the user may be using elsewhere.
226
+ await send('Runtime.callFunctionOn', {
227
+ objectId: object.objectId,
228
+ functionDeclaration: TYPE_INTO,
229
+ arguments: [{ value: String(params.text ?? '') }],
230
+ userGesture: true,
231
+ });
232
+ const before = view.getURL();
233
+ if (params.submit === true) {
234
+ await press(view, send, 'Enter');
235
+ }
236
+ await settle(browser, before, 300);
237
+ return status(view);
238
+ }
239
+ case 'press': {
240
+ const before = view.getURL();
241
+ await press(view, send, String(params.key ?? ''));
242
+ await settle(browser, before, 300);
243
+ return status(view);
244
+ }
245
+ case 'scroll': {
246
+ if (params.ref !== undefined) {
247
+ const backendNodeId = await ref(send, params.ref);
248
+ await send('DOM.scrollIntoViewIfNeeded', { backendNodeId }).catch(error => stale(error, backendNodeId));
249
+ } else {
250
+ const { cssLayoutViewport: viewport } = (await send('Page.getLayoutMetrics')) as {
251
+ cssLayoutViewport: { clientWidth: number; clientHeight: number };
252
+ };
253
+ const amount = Number(params.amount) > 0 ? Number(params.amount) : Math.round(viewport.clientHeight * 0.8);
254
+ await send('Input.dispatchMouseEvent', {
255
+ type: 'mouseWheel',
256
+ x: viewport.clientWidth / 2,
257
+ y: viewport.clientHeight / 2,
258
+ deltaX: 0,
259
+ deltaY: params.direction === 'up' ? -amount : amount,
260
+ });
261
+ await sleep(250);
262
+ }
263
+ const where = await evaluate(send, '[Math.round(scrollY), document.documentElement.scrollHeight, innerHeight]');
264
+ const [top, height, shown] = JSON.parse(where) as number[];
265
+ return `${status(view)}\nScrolled to ${top} of ${height} pixels, ${shown} shown.`;
266
+ }
267
+ case 'screenshot': {
268
+ const { cssVisualViewport: viewport } = (await send('Page.getLayoutMetrics')) as {
269
+ cssVisualViewport: { pageX: number; pageY: number; clientWidth: number; clientHeight: number };
270
+ };
271
+ const ratio = Number(await evaluate(send, 'devicePixelRatio')) || 1;
272
+ const { data } = (await send('Page.captureScreenshot', {
273
+ format: 'jpeg',
274
+ quality: 70,
275
+ clip: { x: viewport.pageX, y: viewport.pageY, width: viewport.clientWidth, height: viewport.clientHeight, scale: 1 / ratio },
276
+ })) as { data: string };
277
+ return `data:image/jpeg;base64,${data}`;
278
+ }
279
+ case 'evaluate':
280
+ return clip(await evaluate(send, String(params.expression ?? '')));
281
+ case 'wait': {
282
+ const ms = Math.min(MAX_WAIT_MS, Number(params.ms) > 0 ? Number(params.ms) : 10_000);
283
+ const probe = `Boolean(document.body && document.body.innerText.includes(${JSON.stringify(params.text)}))`;
284
+ const end = Date.now() + ms;
285
+ while (Date.now() < end) {
286
+ if ((await evaluate(send, probe).catch(() => 'false')) === 'true') {
287
+ return status(view);
288
+ }
289
+ await sleep(250);
290
+ }
291
+ throw new Error(`The page did not show ${JSON.stringify(params.text)} in ${ms} ms.`);
292
+ }
293
+ case 'cdp': {
294
+ const method = String(params.method ?? '');
295
+ const args = params.params && typeof params.params === 'object' ? (params.params as Record<string, unknown>) : {};
296
+ return clip(JSON.stringify((await send(method, args)) ?? null));
297
+ }
298
+ }
299
+ throw new Error(`Unknown action ${action}.`);
300
+ }
301
+
302
+ /** The browser view, opened and shown: a page in a tab the user does not look at is not drawn. */
303
+ async function open(): Promise<BrowserHandle> {
304
+ if (!pi.ui.debugWebview) {
305
+ throw new Error('The browser of Ta can be worked in Ta Desktop only.');
306
+ }
307
+ // Shown only when it is not: showing it again would take the focus from where the user types.
308
+ if (!handle?.shown()) {
309
+ pi.ui.showView(`${PACKAGE}/browser`);
310
+ }
311
+ const opened = await until(() => handle, 5000);
312
+ if (!opened) {
313
+ throw new Error('The browser did not open.');
314
+ }
315
+ return opened;
316
+ }
317
+
318
+ function page(browser: BrowserHandle): WebviewElement {
319
+ const view = browser.view();
320
+ if (!view) {
321
+ throw new Error('No page is open: navigate to one first.');
322
+ }
323
+ return view;
324
+ }
325
+
326
+ async function command(view: WebviewElement, method: string, args: Record<string, unknown>): Promise<unknown> {
327
+ const debug = pi.ui.debugWebview;
328
+ if (!debug) {
329
+ throw new Error('The browser of Ta can be worked in Ta Desktop only.');
330
+ }
331
+ return debug(view.getWebContentsId(), method, args);
332
+ }
333
+
334
+ /**
335
+ * Waits for what an action started: a page to load (`before` is the address before, when the action
336
+ * may leave it), at most 15 s. `grace` gives a click the time to start a navigation.
337
+ */
338
+ async function settle(browser: BrowserHandle, before: string | undefined, grace = 150): Promise<void> {
339
+ await sleep(grace);
340
+ const view = await until(() => browser.view(), 15_000);
341
+ if (!view) {
342
+ return;
343
+ }
344
+ const end = Date.now() + 15_000;
345
+ while (view.isLoading() && Date.now() < end) {
346
+ await sleep(100);
347
+ }
348
+ if (before !== undefined && view.getURL() === before) {
349
+ return;
350
+ }
351
+ // A new page: until its document is there, the accessibility tree is the old one's.
352
+ while (Date.now() < end) {
353
+ const state = await command(view, 'Runtime.evaluate', { expression: 'document.readyState', returnByValue: true }).catch(() => undefined);
354
+ if ((state as { result?: { value?: string } } | undefined)?.result?.value !== 'loading') {
355
+ return;
356
+ }
357
+ await sleep(100);
358
+ }
359
+ }
360
+
361
+ function status(view: WebviewElement): string {
362
+ const title = view.getTitle();
363
+ const url = view.getURL();
364
+ return title && title !== url ? `Page: ${title} — ${url}` : `Page: ${url}`;
365
+ }
366
+
367
+ async function documentOf(send: Send): Promise<string> {
368
+ const { frameTree } = (await send('Page.getFrameTree')) as { frameTree: { frame: { loaderId: string } } };
369
+ return frameTree.frame.loaderId;
370
+ }
371
+
372
+ /** A ref of the last snapshot, while its page is shown: refs are numbers the page gives again to other elements. */
373
+ async function ref(send: Send, value: unknown): Promise<number> {
374
+ const id = Number(value);
375
+ if (!Number.isInteger(id) || id <= 0) {
376
+ throw new Error('ref is the number in brackets of an element in the last snapshot.');
377
+ }
378
+ if (!snapshot || snapshot.document !== (await documentOf(send))) {
379
+ throw new Error('The page changed since the last snapshot: take a snapshot, then use its refs.');
380
+ }
381
+ if (!snapshot.refs.has(id)) {
382
+ throw new Error(`[${id}] is not a ref of the last snapshot.`);
383
+ }
384
+ return id;
385
+ }
386
+
387
+ function stale(error: unknown, id: number): never {
388
+ const message = error instanceof Error ? error.message : String(error);
389
+ throw new Error(/node/i.test(message) ? `The element [${id}] is not on the page any more: take a snapshot again.` : message);
390
+ }
391
+
392
+ async function locate(send: Send, backendNodeId: number): Promise<{ x: number; y: number }> {
393
+ await send('DOM.scrollIntoViewIfNeeded', { backendNodeId }).catch(error => stale(error, backendNodeId));
394
+ const { quads } = (await send('DOM.getContentQuads', { backendNodeId }).catch(error => stale(error, backendNodeId))) as { quads: number[][] };
395
+ if (!quads?.length) {
396
+ throw new Error(`The element [${backendNodeId}] is not shown: scroll to it, or take a snapshot again.`);
397
+ }
398
+ return BrowserSnapshot.center(quads[0]);
399
+ }
400
+
401
+ /**
402
+ * Presses a key in the page. Keys go where the app's focus is, which is not the page while the user
403
+ * types elsewhere in Ta: the webview has it for the key, then it goes back.
404
+ */
405
+ async function press(view: WebviewElement, send: Send, name: string): Promise<void> {
406
+ const key = BrowserSnapshot.key(name);
407
+ if (!key) {
408
+ throw new Error(`${JSON.stringify(name)} is not a key: use names such as Enter, Tab, Escape, ArrowDown, a, Control+A.`);
409
+ }
410
+ const { text, ...event } = key;
411
+ const focused = document.activeElement;
412
+ if (focused !== view) {
413
+ view.focus();
414
+ await sleep(KEY_FOCUS_MS);
415
+ }
416
+ try {
417
+ await send('Input.dispatchKeyEvent', { type: text ? 'keyDown' : 'rawKeyDown', ...event, ...(text ? { text, unmodifiedText: text } : {}) });
418
+ await send('Input.dispatchKeyEvent', { type: 'keyUp', ...event });
419
+ } finally {
420
+ if (focused instanceof HTMLElement && focused !== view) {
421
+ await sleep(KEY_FOCUS_MS);
422
+ focused.focus({ preventScroll: true });
423
+ }
424
+ }
425
+ }
426
+
427
+ async function evaluate(send: Send, expression: string): Promise<string> {
428
+ const { result, exceptionDetails } = (await send('Runtime.evaluate', { expression, returnByValue: true, awaitPromise: true, userGesture: true })) as {
429
+ result?: { value?: unknown; type?: string; description?: string };
430
+ exceptionDetails?: { text?: string; exception?: { description?: string } };
431
+ };
432
+ if (exceptionDetails) {
433
+ throw new Error(exceptionDetails.exception?.description ?? exceptionDetails.text ?? 'The expression threw.');
434
+ }
435
+ if (result?.type === 'undefined') {
436
+ return 'undefined';
437
+ }
438
+ return result && 'value' in result ? (JSON.stringify(result.value) ?? String(result.value)) : (result?.description ?? 'undefined');
439
+ }
440
+ }