@revoengine/cli 1.0.29 → 1.0.31

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. package/README.md +156 -3
  2. package/dist/src/automation-contract.generated.d.ts +742 -0
  3. package/dist/src/automation-contract.generated.js +1021 -0
  4. package/dist/src/cli.js +9 -1
  5. package/dist/src/client.d.ts +24 -0
  6. package/dist/src/client.js +60 -0
  7. package/dist/src/commands/component.js +56 -23
  8. package/dist/src/commands/env.js +1 -1
  9. package/dist/src/commands/index.d.ts +2 -0
  10. package/dist/src/commands/index.js +2 -0
  11. package/dist/src/commands/studio.d.ts +3 -0
  12. package/dist/src/commands/studio.js +114 -0
  13. package/dist/src/commands/terminal-browser-human-input.d.ts +11 -0
  14. package/dist/src/commands/terminal-browser-human-input.js +44 -0
  15. package/dist/src/commands/terminal-browser-interaction.d.ts +25 -0
  16. package/dist/src/commands/terminal-browser-interaction.js +56 -0
  17. package/dist/src/commands/terminal-browser-live-protocol.d.ts +151 -0
  18. package/dist/src/commands/terminal-browser-live-protocol.js +177 -0
  19. package/dist/src/commands/terminal-browser-live-transport.d.ts +41 -0
  20. package/dist/src/commands/terminal-browser-live-transport.js +206 -0
  21. package/dist/src/commands/terminal-browser-live.d.ts +114 -0
  22. package/dist/src/commands/terminal-browser-live.js +455 -0
  23. package/dist/src/commands/terminal-browser-observation.d.ts +10 -0
  24. package/dist/src/commands/terminal-browser-observation.js +27 -0
  25. package/dist/src/commands/terminal-browser-page-input.d.ts +8 -0
  26. package/dist/src/commands/terminal-browser-page-input.js +94 -0
  27. package/dist/src/commands/terminal-browser-profile-preferences.d.ts +37 -0
  28. package/dist/src/commands/terminal-browser-profile-preferences.js +136 -0
  29. package/dist/src/commands/terminal-browser-profile.d.ts +52 -0
  30. package/dist/src/commands/terminal-browser-profile.js +447 -0
  31. package/dist/src/commands/terminal-browser.d.ts +28 -1
  32. package/dist/src/commands/terminal-browser.js +401 -55
  33. package/dist/src/commands/terminal-policy.js +2 -0
  34. package/dist/src/commands/terminal-public-web-proxy.d.ts +14 -0
  35. package/dist/src/commands/terminal-public-web-proxy.js +141 -0
  36. package/dist/src/commands/terminal-public-web.d.ts +11 -0
  37. package/dist/src/commands/terminal-public-web.js +99 -0
  38. package/dist/src/commands/terminal.d.ts +1 -0
  39. package/dist/src/commands/terminal.js +92 -9
  40. package/dist/src/commands/tests.d.ts +3 -0
  41. package/dist/src/commands/tests.js +236 -0
  42. package/dist/src/component-element-validation.d.ts +8 -0
  43. package/dist/src/component-element-validation.js +79 -0
  44. package/dist/src/component-lifecycle.d.ts +3 -0
  45. package/dist/src/component-lifecycle.js +17 -0
  46. package/dist/src/component-lock.d.ts +17 -12
  47. package/dist/src/component-lock.js +57 -6
  48. package/dist/src/component-manifests.js +3 -0
  49. package/dist/src/component-source-validation.d.ts +5 -0
  50. package/dist/src/component-source-validation.js +27 -0
  51. package/dist/src/env-sync.js +67 -9
  52. package/dist/src/native-tests.d.ts +61 -0
  53. package/dist/src/native-tests.js +20 -0
  54. package/dist/src/request-contract.d.ts +39 -0
  55. package/dist/src/request-contract.js +138 -0
  56. package/dist/src/resource-syncs/agent-sync.js +47 -7
  57. package/dist/src/resource-syncs/database-table-sync.js +8 -8
  58. package/dist/src/resource-syncs/endpoint-sync.js +20 -14
  59. package/dist/src/resource-syncs/group-sync.js +8 -3
  60. package/dist/src/resource-syncs/job-template-sync.js +75 -28
  61. package/dist/src/resource-syncs/role-group-sync.js +8 -0
  62. package/dist/src/resource-syncs/runtime-definition-validation.d.ts +14 -0
  63. package/dist/src/resource-syncs/runtime-definition-validation.js +153 -0
  64. package/dist/src/test-workspace.d.ts +33 -0
  65. package/dist/src/test-workspace.js +156 -0
  66. package/dist/src/types.d.ts +1 -0
  67. package/dist/src/ui.js +12 -0
  68. package/dist/src/update-notifier.d.ts +1 -1
  69. package/package.json +5 -3
@@ -1,6 +1,12 @@
1
1
  import fs from 'node:fs';
2
+ import { readPublicWebPage } from "./terminal-public-web.js";
2
3
  import path from 'node:path';
3
4
  import { randomUUID } from 'node:crypto';
5
+ import { browserProfileScopeId } from "./terminal-browser-profile.js";
6
+ import { validBrowserInteraction } from "./terminal-browser-interaction.js";
7
+ import { BrowserObservations } from "./terminal-browser-observation.js";
8
+ import { BrowserLiveHandle, browserAuthPage } from "./terminal-browser-live.js";
9
+ import { applyPageInteraction, clearSecretInputs, releasePageInput, pageHasSecretFocus, secretControlAttributes } from "./terminal-browser-page-input.js";
4
10
  const MAX_BROWSER_SESSIONS = 4;
5
11
  const BROWSER_SESSION_IDLE_MS = 60 * 60_000;
6
12
  const MAX_BROWSER_TABS = 16;
@@ -8,7 +14,7 @@ const browserSessions = new Map();
8
14
  const creatingSessions = new Set();
9
15
  const sessionQueues = new Map();
10
16
  const FORM_ACTIONS = new Set(['click', 'fill', 'check', 'select', 'press']);
11
- const INTERACTIVE_ACTIONS = new Set([...FORM_ACTIONS, 'scroll']);
17
+ const INTERACTIVE_ACTIONS = new Set([...FORM_ACTIONS, 'scroll', 'interact']);
12
18
  const PRESS_KEYS = new Set(['Enter', 'Tab', 'Escape', 'Backspace',
13
19
  'ArrowUp', 'ArrowDown', 'ArrowLeft', 'ArrowRight']);
14
20
  let cleanupTimer;
@@ -48,9 +54,14 @@ export function validateBrowserActionInput(input) {
48
54
  || !Number.isSafeInteger(input.policyGrantRevision) || input.policyGrantRevision < 0))
49
55
  throw new Error('terminal_policy_invalid');
50
56
  if (!['navigate', 'dom', 'click', 'fill', 'check', 'select', 'press', 'scroll', 'wait_for', 'screenshot',
51
- 'tabs', 'new_tab', 'select_tab', 'close_tab', 'close']
57
+ 'interact', 'tabs', 'new_tab', 'select_tab', 'close_tab', 'close', 'read_public']
52
58
  .includes(input.action))
53
59
  throw new Error('invalid_browser_action');
60
+ if (input.action === 'read_public' && (input.localOnly || input.tabId != null || input.selector != null
61
+ || (input.actionTimeoutMs ?? 6000) > 8000))
62
+ throw new Error('invalid_browser_action_options');
63
+ if (input.action === 'interact' ? !validBrowserInteraction(input.interaction) : input.interaction !== undefined)
64
+ throw new Error('invalid_browser_interaction');
54
65
  if (!/^[a-f0-9]{64}$/u.test(input.browserSessionId))
55
66
  throw new Error('invalid_browser_session');
56
67
  if (input.tabId !== undefined && !/^[a-f0-9]{8}-[a-f0-9]{4}-[1-8][a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/iu.test(input.tabId)
@@ -124,9 +135,40 @@ async function closeSession(id) {
124
135
  if (!session)
125
136
  return;
126
137
  browserSessions.delete(id);
138
+ session.live?.disconnect();
127
139
  if (session.networkLeaseTimer)
128
140
  clearTimeout(session.networkLeaseTimer);
129
- await session.browser.close().catch(() => undefined);
141
+ try {
142
+ await session.browser.close().catch(() => undefined);
143
+ }
144
+ finally {
145
+ session.authProfile?.release();
146
+ }
147
+ }
148
+ async function checkpointProfile(session) {
149
+ if (!session.authProfile)
150
+ return;
151
+ session.snapshotting = true;
152
+ try {
153
+ session.authProfile.save(await session.context.storageState({ indexedDB: true }));
154
+ }
155
+ finally {
156
+ session.snapshotting = false;
157
+ for (const page of session.context.pages())
158
+ if (!page.isClosed() && registerTab(session, page))
159
+ session.page = page;
160
+ }
161
+ }
162
+ async function checkpointSession(id, session) {
163
+ if (!session.authProfile)
164
+ return;
165
+ try {
166
+ await checkpointProfile(session);
167
+ }
168
+ catch (error) {
169
+ await closeSession(id);
170
+ throw error;
171
+ }
130
172
  }
131
173
  function setNetworkLease(id, session, expiresAt) {
132
174
  if (session.networkLeaseTimer)
@@ -147,6 +189,7 @@ function setNetworkLease(id, session, expiresAt) {
147
189
  async function pruneIdleSessions() {
148
190
  const now = Date.now();
149
191
  await Promise.all([...browserSessions].filter(([, session]) => !session.active
192
+ && !(session.control.mode === 'human' && session.control.connected)
150
193
  && now - session.lastUsedAt >= BROWSER_SESSION_IDLE_MS).map(([id]) => closeSession(id)));
151
194
  if (!browserSessions.size && cleanupTimer) {
152
195
  clearInterval(cleanupTimer);
@@ -197,13 +240,22 @@ export async function closeThreadBrowserSession(browserSessionId) {
197
240
  }
198
241
  export function browserSessionIds() { return [...browserSessions.keys()]; }
199
242
  export function hasBrowserSessions() { return browserSessions.size > 0; }
200
- async function createSession(id, url, localOnly, width, height, signal, onNavigationAttempt, actionTimeoutMs) {
243
+ async function createSession(id, url, localOnly, width, height, signal, onNavigationAttempt, actionTimeoutMs, authProfile) {
201
244
  await pruneIdleSessions();
202
245
  if (browserSessions.size + creatingSessions.size >= MAX_BROWSER_SESSIONS)
203
246
  throw new Error('browser_session_limit');
204
247
  creatingSessions.add(id);
248
+ let profileLease;
249
+ try {
250
+ profileLease = authProfile ? await authProfile.store.openAsync(authProfile.scope) : undefined;
251
+ }
252
+ catch (error) {
253
+ creatingSessions.delete(id);
254
+ throw error;
255
+ }
205
256
  const epoch = sessionEpoch;
206
257
  const { chromium } = await import('playwright-core').catch(error => {
258
+ profileLease?.release();
207
259
  creatingSessions.delete(id);
208
260
  throw error;
209
261
  });
@@ -213,7 +265,7 @@ async function createSession(id, url, localOnly, width, height, signal, onNaviga
213
265
  if (error instanceof Error && error.message.includes("Executable doesn't exist at"))
214
266
  throw new Error('browser_binary_unavailable');
215
267
  throw error;
216
- }).catch(error => { creatingSessions.delete(id); throw error; });
268
+ }).catch(error => { profileLease?.release(); creatingSessions.delete(id); throw error; });
217
269
  const onAbort = () => { void browser.close(); };
218
270
  signal?.addEventListener('abort', onAbort, { once: true });
219
271
  try {
@@ -221,6 +273,17 @@ async function createSession(id, url, localOnly, width, height, signal, onNaviga
221
273
  throw new Error('browser_cancelled');
222
274
  const context = await browser.newContext({ viewport: { width, height }, acceptDownloads: false,
223
275
  serviceWorkers: 'block' });
276
+ const geometryBinding = 'revoGeometry_' + randomUUID().replaceAll('-', '');
277
+ await context.exposeBinding(geometryBinding, ({ page }) => {
278
+ const session = browserSessions.get(id);
279
+ if (session && session.tabs.size && [...session.tabs.values()].includes(page))
280
+ session.observations.invalidate();
281
+ });
282
+ await context.addInitScript(({ binding }) => {
283
+ const invalidate = () => { void Reflect.get(globalThis, binding)().catch(() => undefined); };
284
+ addEventListener('scroll', invalidate, { capture: true, passive: true });
285
+ addEventListener('resize', invalidate, { passive: true });
286
+ }, { binding: geometryBinding });
224
287
  if (localOnly)
225
288
  await context.addInitScript(() => {
226
289
  // These APIs can open sockets outside Playwright's HTTP and WebSocket routes.
@@ -254,24 +317,23 @@ async function createSession(id, url, localOnly, width, height, signal, onNaviga
254
317
  socket.close();
255
318
  }
256
319
  });
320
+ if (profileLease?.state)
321
+ await context.setStorageState(profileLease.state);
257
322
  const page = await context.newPage();
258
323
  if (localOnly && !await page.evaluate(() => ['RTCPeerConnection', 'webkitRTCPeerConnection',
259
324
  'WebTransport', 'Worker', 'SharedWorker'].every(name => typeof Reflect.get(globalThis, name) === 'undefined'
260
325
  && Object.getOwnPropertyDescriptor(globalThis, name)?.configurable === false)))
261
326
  throw new Error('browser_local_network_guard_unavailable');
262
327
  const session = { browser, context, page, localOnly, tabs: new Map(),
328
+ observations: new BrowserObservations(), control: { epoch: 0, mode: 'observe', private: false, connected: false },
329
+ ...(profileLease ? { authProfile: profileLease, profileScopeId: browserProfileScopeId(authProfile.scope), profileScope: authProfile.scope } : {}),
263
330
  lastUsedAt: Date.now(), active: false, networkAccessExpiresAt: null };
264
331
  registerTab(session, page);
265
332
  context.on('page', opened => {
266
- registerTab(session, opened);
267
- session.page = opened;
268
- opened.on('close', () => {
269
- if (session.page !== opened)
270
- return;
271
- const remaining = context.pages().filter(candidate => !candidate.isClosed());
272
- if (remaining.length)
273
- session.page = remaining[remaining.length - 1];
274
- });
333
+ if (session.snapshotting)
334
+ return;
335
+ if (registerTab(session, opened))
336
+ session.page = opened;
275
337
  });
276
338
  const initialPopup = page.waitForEvent('popup', { timeout: 5_000 })
277
339
  .then(async (opened) => {
@@ -295,6 +357,7 @@ async function createSession(id, url, localOnly, width, height, signal, onNaviga
295
357
  }
296
358
  catch (error) {
297
359
  await browser.close().catch(() => undefined);
360
+ profileLease?.release();
298
361
  throw error;
299
362
  }
300
363
  finally {
@@ -304,14 +367,29 @@ async function createSession(id, url, localOnly, width, height, signal, onNaviga
304
367
  }
305
368
  function registerTab(session, page) {
306
369
  if ([...session.tabs.values()].includes(page))
307
- return;
370
+ return false;
308
371
  if (session.tabs.size >= MAX_BROWSER_TABS) {
309
372
  void page.close();
310
- return;
373
+ return false;
311
374
  }
312
375
  const id = randomUUID();
313
376
  session.tabs.set(id, page);
314
- page.on('close', () => { session.tabs.delete(id); });
377
+ page.on('framenavigated', frame => {
378
+ session.observations.invalidate();
379
+ if (frame === page.mainFrame() && browserAuthPage(page.url()))
380
+ void session.live?.pauseRecording();
381
+ });
382
+ session.observations.invalidate();
383
+ page.on('close', () => {
384
+ session.tabs.delete(id);
385
+ session.observations.invalidate();
386
+ if (session.page !== page)
387
+ return;
388
+ const remaining = [...session.tabs.values()].filter(candidate => !candidate.isClosed());
389
+ if (remaining.length)
390
+ session.page = remaining[remaining.length - 1];
391
+ });
392
+ return true;
315
393
  }
316
394
  async function describeSession(id, session, sessionCreated) {
317
395
  const tabs = [];
@@ -324,7 +402,9 @@ async function describeSession(id, session, sessionCreated) {
324
402
  }
325
403
  const active = tabs.find(tab => tab.active);
326
404
  return { browserSessionId: id, activeTabId: active?.id || null,
327
- tabs, pageUrl: active?.url || '', sessionCreated };
405
+ tabs, pageUrl: active?.url || '', sessionCreated,
406
+ ...(active && session ? { browserObservation: session.observations.fresh(activePage(session), active.id) } : {}),
407
+ ...(session?.profileScope ? { browserProfile: { id: session.profileScopeId, name: session.profileScope.name, persistent: true, default: false } } : {}) };
328
408
  }
329
409
  function activePage(session) {
330
410
  if (!session.page.isClosed())
@@ -337,33 +417,51 @@ function activePage(session) {
337
417
  }
338
418
  async function semanticControls(page) {
339
419
  const controls = await page.locator('input:not([type="hidden"]), textarea, select, button, a[href],'
340
- + ' [role], [contenteditable="true"]').evaluateAll(elements => elements.slice(0, 120).map(element => {
341
- const tag = element.tagName.toLowerCase();
342
- const attribute = (name) => element.getAttribute(name)?.slice(0, 120) || undefined;
343
- const id = attribute('id');
344
- const testId = attribute('data-testid');
345
- const name = attribute('name');
346
- const ariaLabel = attribute('aria-label');
347
- const placeholder = attribute('placeholder');
348
- const type = attribute('type');
349
- const label = element instanceof HTMLInputElement || element instanceof HTMLSelectElement
350
- || element instanceof HTMLTextAreaElement
351
- ? [...(element.labels || [])].map(item => item.innerText.trim()).filter(Boolean).join(' ').slice(0, 120)
352
- : undefined;
353
- const selector = id ? `#${CSS.escape(id)}`
354
- : testId ? `[data-testid=${CSS.escape(testId)}]`
355
- : name ? `${tag}[name=${CSS.escape(name)}]`
356
- : ariaLabel ? `${tag}[aria-label=${CSS.escape(ariaLabel)}]`
357
- : placeholder ? `${tag}[placeholder=${CSS.escape(placeholder)}]` : undefined;
358
- const item = { tag, selector, role: attribute('role'), type,
359
- id, name, testId, ariaLabel, placeholder, label,
360
- text: tag === 'input' || tag === 'textarea' || tag === 'select' ? undefined
361
- : element.textContent?.trim().slice(0, 100) || undefined,
362
- checked: element instanceof HTMLInputElement && ['checkbox', 'radio'].includes(element.type)
363
- ? element.checked : undefined,
364
- disabled: 'disabled' in element ? Boolean(element.disabled) : undefined };
365
- return Object.fromEntries(Object.entries(item).filter(([, value]) => value !== undefined && value !== ''));
366
- }));
420
+ + ' [role], [contenteditable="true"]').evaluateAll(elements => {
421
+ const visible = (element) => {
422
+ const rect = element.getBoundingClientRect();
423
+ const visibility = getComputedStyle(element).visibility;
424
+ return rect.width > 0 && rect.height > 0 && visibility !== 'hidden' && visibility !== 'collapse';
425
+ };
426
+ const candidates = elements.filter(visible);
427
+ const dialogSelector = 'dialog[open], [role="dialog"], [role="alertdialog"]';
428
+ const focused = candidates.find(element => element === document.activeElement);
429
+ const focusedDialog = focused?.closest(dialogSelector);
430
+ const activeDialogControls = focusedDialog && visible(focusedDialog)
431
+ ? candidates.filter(element => focusedDialog.contains(element)) : [];
432
+ const dialogControls = candidates.filter(element => {
433
+ const dialog = element.closest(dialogSelector);
434
+ return dialog && visible(dialog);
435
+ });
436
+ return [...new Set([...(focused ? [focused] : []), ...activeDialogControls, ...dialogControls, ...candidates])]
437
+ .slice(0, 120).map(element => {
438
+ const tag = element.tagName.toLowerCase();
439
+ const attribute = (name) => element.getAttribute(name)?.slice(0, 120) || undefined;
440
+ const id = attribute('id');
441
+ const testId = attribute('data-testid');
442
+ const name = attribute('name');
443
+ const ariaLabel = attribute('aria-label');
444
+ const placeholder = attribute('placeholder');
445
+ const type = attribute('type');
446
+ const label = element instanceof HTMLInputElement || element instanceof HTMLSelectElement
447
+ || element instanceof HTMLTextAreaElement
448
+ ? [...(element.labels || [])].map(item => item.innerText.trim()).filter(Boolean).join(' ').slice(0, 120)
449
+ : undefined;
450
+ const selector = id ? `#${CSS.escape(id)}`
451
+ : testId ? `[data-testid=${CSS.escape(testId)}]`
452
+ : name ? `${tag}[name=${CSS.escape(name)}]`
453
+ : ariaLabel ? `${tag}[aria-label=${CSS.escape(ariaLabel)}]`
454
+ : placeholder ? `${tag}[placeholder=${CSS.escape(placeholder)}]` : undefined;
455
+ const item = { tag, selector, role: attribute('role'), type,
456
+ id, name, testId, ariaLabel, placeholder, label,
457
+ text: tag === 'input' || tag === 'textarea' || tag === 'select' ? undefined
458
+ : element.textContent?.trim().slice(0, 100) || undefined,
459
+ checked: element instanceof HTMLInputElement && ['checkbox', 'radio'].includes(element.type)
460
+ ? element.checked : undefined,
461
+ disabled: 'disabled' in element ? Boolean(element.disabled) : undefined };
462
+ return Object.fromEntries(Object.entries(item).filter(([, value]) => value !== undefined && value !== ''));
463
+ });
464
+ });
367
465
  if (!controls.length)
368
466
  return '';
369
467
  return '\n\nInteractive elements (field values omitted):\n'
@@ -377,8 +475,7 @@ async function isSecretControl(page, selector) {
377
475
  const target = page.locator(selector);
378
476
  const attributes = await Promise.all(['type', 'id', 'name', 'autocomplete', 'aria-label', 'placeholder']
379
477
  .map(name => target.getAttribute(name, { timeout: 10_000 })));
380
- return attributes[0]?.toLowerCase() === 'password'
381
- || attributes.some(value => value && /passw(or)?d|secret|api.?key|token|one.?time|\botp\b|2fa|mfa/iu.test(value));
478
+ return secretControlAttributes(attributes);
382
479
  }
383
480
  export function writePrivateScreenshot(file, bytes) {
384
481
  let descriptor;
@@ -404,6 +501,9 @@ export function writePrivateScreenshot(file, bytes) {
404
501
  }
405
502
  }
406
503
  async function runForSession(input) {
504
+ const existing = browserSessions.get(input.browserSessionId);
505
+ if (existing && existing.control.mode !== 'observe')
506
+ throw new Error('browser_human_control_active');
407
507
  if (input.signal?.aborted)
408
508
  throw new Error('browser_cancelled');
409
509
  validateBrowserActionInput(input);
@@ -411,6 +511,8 @@ async function runForSession(input) {
411
511
  const width = input.width ?? 1440;
412
512
  const height = input.height ?? 900;
413
513
  let session = browserSessions.get(input.browserSessionId);
514
+ if (session && !input.resolveAuthProfile && session.profileScopeId !== (input.authProfile ? browserProfileScopeId(input.authProfile.scope) : undefined))
515
+ throw new Error('browser_profile_scope_changed');
414
516
  if (session && (!['tabs', 'close', 'close_tab'].includes(input.action) && session.localOnly !== (input.localOnly === true) || !session.browser.isConnected()
415
517
  || session.context.pages().every(page => page.isClosed())
416
518
  || Date.now() - session.lastUsedAt >= BROWSER_SESSION_IDLE_MS)) {
@@ -418,6 +520,8 @@ async function runForSession(input) {
418
520
  session = undefined;
419
521
  }
420
522
  if (input.action === 'close') {
523
+ if (session)
524
+ await checkpointSession(input.browserSessionId, session);
421
525
  await closeSession(input.browserSessionId);
422
526
  return { dom: '', ...await describeSession(input.browserSessionId, undefined, false) };
423
527
  }
@@ -430,7 +534,14 @@ async function runForSession(input) {
430
534
  throw new Error('browser_session_missing');
431
535
  const created = !session;
432
536
  if (!session)
433
- session = await createSession(input.browserSessionId, url, input.localOnly === true, width, height, input.signal, input.onActionAttempt, input.actionTimeoutMs);
537
+ session = await createSession(input.browserSessionId, url, input.localOnly === true, width, height, input.signal, input.onActionAttempt, input.actionTimeoutMs, input.resolveAuthProfile ? await input.resolveAuthProfile() : input.authProfile);
538
+ if (input.actorUserId && input.threadId) {
539
+ const owner = { actorUserId: input.actorUserId, threadId: input.threadId,
540
+ workspaceId: input.agentTerminalWorkspaceId, workspace: fs.realpathSync(input.workspace) };
541
+ if (session.owner && JSON.stringify(session.owner) !== JSON.stringify(owner))
542
+ throw new Error('browser_session_scope_changed');
543
+ session.owner = owner;
544
+ }
434
545
  // Inventory and cleanup cannot renew, remove or widen an existing network lease.
435
546
  if (!['tabs', 'close_tab'].includes(input.action)) {
436
547
  session.workspace = input.agentTerminalWorkspaceId ? input.workspace : undefined;
@@ -459,6 +570,8 @@ async function runForSession(input) {
459
570
  const selected = session.tabs.get(input.tabId);
460
571
  if (!selected || selected.isClosed())
461
572
  throw new Error('browser_tab_missing');
573
+ if (session.page !== selected)
574
+ session.observations.invalidate();
462
575
  session.page = selected;
463
576
  }
464
577
  if (input.action === 'tabs')
@@ -472,8 +585,11 @@ async function runForSession(input) {
472
585
  return { dom: '', ...await describeSession(input.browserSessionId, undefined, false) };
473
586
  }
474
587
  session.page = remaining[remaining.length - 1];
588
+ await checkpointSession(input.browserSessionId, session);
475
589
  return { dom: '', ...await describeSession(input.browserSessionId, session, false) };
476
590
  }
591
+ if (['navigate', 'new_tab'].includes(input.action) && browserAuthPage(url))
592
+ await session.live?.pauseRecording();
477
593
  if (input.action === 'new_tab' && !created) {
478
594
  if (session.tabs.size >= MAX_BROWSER_TABS)
479
595
  throw new Error('browser_tab_limit');
@@ -483,15 +599,37 @@ async function runForSession(input) {
483
599
  }
484
600
  let page = activePage(session);
485
601
  await page.bringToFront();
486
- await page.setViewportSize({ width, height });
602
+ if (!['interact', 'dom'].includes(input.action) || input.width != null || input.height != null) {
603
+ const old = page.viewportSize();
604
+ if (old?.width !== width || old?.height !== height)
605
+ session.observations.invalidate();
606
+ await page.setViewportSize({ width, height });
607
+ }
487
608
  if (input.action === 'navigate' && !created) {
609
+ if (session.control.mode !== 'observe')
610
+ throw new Error('browser_human_control_active');
488
611
  input.onActionAttempt?.();
489
612
  await page.goto(url, { waitUntil: 'domcontentloaded', timeout: input.actionTimeoutMs ?? 30_000 });
490
613
  }
491
614
  (input.localOnly ? localBrowserUrl : browserUrl)(page.url());
615
+ if (session.live && await pageHasSecretFocus(page))
616
+ await session.live.pauseRecording();
492
617
  const actionPage = page;
493
- const target = input.selector ? page.locator(input.selector) : page.locator('body');
494
- if (FORM_ACTIONS.has(input.action)) {
618
+ let target = input.selector ? page.locator(input.selector) : page.locator('body');
619
+ if (input.action === 'interact') {
620
+ const tabId = [...session.tabs].find(([, current]) => current === page)?.[0];
621
+ if (!tabId)
622
+ throw new Error('browser_tab_missing');
623
+ const observation = session.observations.require(input.interaction.observationId, page, tabId);
624
+ await applyPageInteraction(page, input.interaction, observation, false, () => {
625
+ if (session.control.mode !== 'observe')
626
+ throw new Error('browser_human_control_active');
627
+ input.onActionAttempt?.();
628
+ });
629
+ session.observations.invalidate();
630
+ page = activePage(session);
631
+ }
632
+ else if (FORM_ACTIONS.has(input.action)) {
495
633
  if (await target.count() > 1)
496
634
  throw new Error('browser_selector_ambiguous');
497
635
  try {
@@ -511,10 +649,13 @@ async function runForSession(input) {
511
649
  ? 'browser_selector_missing' : 'browser_selector_ambiguous');
512
650
  if (input.action === 'fill')
513
651
  await guardSecretFill(page, input.selector);
652
+ if (session.control.mode !== 'observe')
653
+ throw new Error('browser_human_control_active');
514
654
  const popup = input.action === 'click' || input.action === 'press'
515
655
  ? actionPage.waitForEvent('popup', { timeout: 750 }).catch(() => undefined)
516
656
  : Promise.resolve(undefined);
517
657
  input.onActionAttempt?.();
658
+ session.observations.invalidate();
518
659
  if (input.action === 'click') {
519
660
  await target.click({ timeout: input.actionTimeoutMs ?? 10_000 });
520
661
  }
@@ -541,16 +682,29 @@ async function runForSession(input) {
541
682
  else if (input.action === 'scroll') {
542
683
  input.onActionAttempt?.();
543
684
  await page.mouse.wheel(0, input.deltaY);
685
+ session.observations.invalidate();
544
686
  }
545
687
  else if (input.action === 'wait_for') {
546
688
  await target.waitFor({ state: input.state, timeout: Math.min(input.waitTimeoutMs, input.actionTimeoutMs ?? input.waitTimeoutMs) });
547
689
  }
690
+ if (session.control.mode !== 'observe')
691
+ throw new Error('browser_human_control_active');
692
+ await checkpointSession(input.browserSessionId, session);
693
+ if (session.control.mode !== 'observe')
694
+ throw new Error('browser_human_control_active');
695
+ if (activePage(session) !== page) {
696
+ page = activePage(session);
697
+ (input.localOnly ? localBrowserUrl : browserUrl)(page.url());
698
+ target = input.selector ? page.locator(input.selector) : page.locator('body');
699
+ }
548
700
  const domTarget = INTERACTIVE_ACTIONS.has(input.action)
549
701
  || input.action === 'wait_for' && input.state === 'hidden' ? page.locator('body') : target.first();
550
702
  const bodyText = (await domTarget.innerText({ timeout: input.actionTimeoutMs ?? 10_000 })).slice(0, 25_000);
551
703
  const dom = (bodyText + await semanticControls(page)).slice(0, 40_000);
552
704
  const pageUrl = safePageUrl(page.url());
553
705
  const inventory = await describeSession(input.browserSessionId, session, created);
706
+ if (session.control.mode !== 'observe')
707
+ throw new Error('browser_human_control_active');
554
708
  if (input.action !== 'screenshot')
555
709
  return { dom, ...inventory, pageUrl };
556
710
  const root = fs.realpathSync(input.workspace);
@@ -577,10 +731,8 @@ async function runForSession(input) {
577
731
  bytes = await target.first().screenshot({ timeout: input.actionTimeoutMs ?? 20_000 });
578
732
  }
579
733
  else if (capture === 'crop') {
580
- const scroll = await page.evaluate(() => ({ x: window.scrollX, y: window.scrollY }));
581
734
  bytes = await page.screenshot({ timeout: input.actionTimeoutMs ?? 20_000,
582
- clip: { x: scroll.x + input.clip.x, y: scroll.y + input.clip.y,
583
- width: input.clip.width, height: input.clip.height } });
735
+ clip: input.clip });
584
736
  }
585
737
  else {
586
738
  bytes = await page.screenshot({ timeout: input.actionTimeoutMs ?? 20_000,
@@ -588,8 +740,22 @@ async function runForSession(input) {
588
740
  }
589
741
  if (bytes.length > 30 * 1024 * 1024)
590
742
  throw new Error('browser_screenshot_too_large');
743
+ if (session.control.mode !== 'observe')
744
+ throw new Error('browser_human_control_active');
591
745
  writePrivateScreenshot(screenshotPath, bytes);
592
- return { dom, ...inventory, pageUrl, screenshotPath };
746
+ inventory.browserObservation = session.observations.fresh(page, inventory.activeTabId);
747
+ const imageWidth = bytes.readUInt32BE(16);
748
+ const imageHeight = bytes.readUInt32BE(20);
749
+ const viewport = page.viewportSize();
750
+ const scroll = await page.evaluate(() => ({ x: window.scrollX, y: window.scrollY }));
751
+ const element = capture === 'element' ? await target.boundingBox() : null;
752
+ const cssCrop = capture === 'crop' ? input.clip
753
+ : element ? { x: element.x, y: element.y, width: element.width, height: element.height }
754
+ : capture === 'full_page' ? { x: -scroll.x, y: -scroll.y, width: imageWidth, height: imageHeight }
755
+ : { x: 0, y: 0, width: viewport.width, height: viewport.height };
756
+ return { dom, ...inventory, pageUrl, screenshotPath,
757
+ screenshotTransform: { observationId: inventory.browserObservation.id, imageWidth, imageHeight, cssCrop,
758
+ scaleX: cssCrop.width / imageWidth, scaleY: cssCrop.height / imageHeight, coordinateSpace: 'css_viewport' } };
593
759
  }
594
760
  finally {
595
761
  input.signal?.removeEventListener('abort', onAbort);
@@ -600,6 +766,16 @@ async function runForSession(input) {
600
766
  }
601
767
  }
602
768
  export async function runBrowserAction(input) {
769
+ if (input.action === 'read_public') {
770
+ validateBrowserActionInput(input);
771
+ const dom = await readPublicWebPage({ url: input.url, timeoutMs: input.actionTimeoutMs,
772
+ signal: input.signal, onActionAttempt: input.onActionAttempt });
773
+ return { dom, pageUrl: safePageUrl(JSON.parse(dom).url), browserSessionId: input.browserSessionId,
774
+ activeTabId: null, tabs: [], sessionCreated: false };
775
+ }
776
+ if (browserSessions.get(input.browserSessionId)?.control.mode !== undefined
777
+ && browserSessions.get(input.browserSessionId).control.mode !== 'observe')
778
+ throw new Error('browser_human_control_active');
603
779
  const previous = sessionQueues.get(input.browserSessionId) || Promise.resolve();
604
780
  const task = previous.catch(() => undefined).then(() => runForSession(input));
605
781
  sessionQueues.set(input.browserSessionId, task);
@@ -633,6 +809,8 @@ export async function fillBrowserSecret(input) {
633
809
  const session = browserSessions.get(input.browserSessionId);
634
810
  if (!session || !session.browser.isConnected())
635
811
  throw new Error('secure_input_session_missing');
812
+ if (session.control.mode !== 'observe')
813
+ throw new Error('browser_human_control_active');
636
814
  if (session.networkAccessExpiresAt !== null && session.networkAccessExpiresAt <= Date.now()) {
637
815
  await closeSession(input.browserSessionId);
638
816
  throw new Error('secure_input_session_missing');
@@ -661,6 +839,10 @@ export async function fillBrowserSecret(input) {
661
839
  throw new Error('secure_input_session_missing');
662
840
  if (input.expiresAt <= Date.now())
663
841
  throw new Error('secure_input_invalid');
842
+ await session.live?.pauseRecording();
843
+ session.observations.invalidate();
844
+ if (session.control.mode !== 'observe')
845
+ throw new Error('browser_human_control_active');
664
846
  try {
665
847
  await target.fill(input.value, { timeout: 10_000 });
666
848
  }
@@ -682,3 +864,167 @@ export async function fillBrowserSecret(input) {
682
864
  sessionQueues.delete(input.browserSessionId);
683
865
  }
684
866
  }
867
+ /** Media and model operations share the same queue; transitions fence the model before awaiting it. */
868
+ export async function attachBrowserLive(options) {
869
+ const session = browserSessions.get(options.browserSessionId);
870
+ if (!session || !session.owner || session.owner.actorUserId !== options.actorUserId
871
+ || session.owner.threadId !== options.threadId || session.owner.workspaceId !== options.agentTerminalWorkspaceId
872
+ || !options.isAuthorized() || !session.browser.isConnected())
873
+ throw new Error('browser_live_scope_invalid');
874
+ const previous = session.live;
875
+ previous?.disconnect();
876
+ if (previous)
877
+ await previous.settled();
878
+ if (options.initialControl && options.initialControl.controlEpoch >= session.control.epoch) {
879
+ session.control.epoch = options.initialControl.controlEpoch;
880
+ if (options.initialControl.mode !== 'observe') {
881
+ session.control.mode = 'blocked';
882
+ session.control.private = true;
883
+ }
884
+ // A reconnect never auto-releases an existing local human/blocked fence.
885
+ }
886
+ const authorized = () => browserSessions.get(options.browserSessionId) === session && session.browser.isConnected()
887
+ && (session.networkAccessExpiresAt === null || session.networkAccessExpiresAt > Date.now());
888
+ const handle = new BrowserLiveHandle(options, {
889
+ control: session.control, observations: session.observations,
890
+ active: () => {
891
+ const page = activePage(session);
892
+ const tabId = [...session.tabs].find(([, current]) => current === page)?.[0];
893
+ if (!tabId)
894
+ throw new Error('browser_tab_missing');
895
+ return { page, tabId };
896
+ },
897
+ pages: () => [...session.tabs.values()].filter(page => !page.isClosed()),
898
+ authorized, fullNetwork: () => authorized() && !session.localOnly,
899
+ queue: async (work) => {
900
+ const previous = sessionQueues.get(options.browserSessionId) || Promise.resolve();
901
+ const task = previous.catch(() => undefined).then(async () => {
902
+ if (!authorized() || !options.isAuthorized())
903
+ throw new Error('browser_live_scope_expired');
904
+ session.active = true;
905
+ try {
906
+ return await work();
907
+ }
908
+ finally {
909
+ session.active = false;
910
+ session.lastUsedAt = Date.now();
911
+ }
912
+ });
913
+ sessionQueues.set(options.browserSessionId, task);
914
+ try {
915
+ return await task;
916
+ }
917
+ finally {
918
+ if (sessionQueues.get(options.browserSessionId) === task)
919
+ sessionQueues.delete(options.browserSessionId);
920
+ }
921
+ },
922
+ state: async () => ({ ...await describeSession(options.browserSessionId, session, false) }),
923
+ selectTab: async (tabId) => {
924
+ const page = session.tabs.get(tabId);
925
+ if (!page || page.isClosed())
926
+ throw new Error('browser_tab_missing');
927
+ session.observations.invalidate();
928
+ session.page = page;
929
+ await page.bringToFront();
930
+ },
931
+ release: async () => {
932
+ for (const page of session.tabs.values())
933
+ if (!page.isClosed()) {
934
+ await releasePageInput(page);
935
+ await clearSecretInputs(page);
936
+ }
937
+ session.observations.invalidate();
938
+ await checkpointSession(options.browserSessionId, session);
939
+ },
940
+ profile: async (choice, liveOptions) => {
941
+ const preferences = liveOptions.profilePreferences;
942
+ const owner = liveOptions.profileOwner;
943
+ if (!preferences || !owner || owner.actorUserId !== session.owner.actorUserId
944
+ || fs.realpathSync(owner.workspace) !== session.owner.workspace)
945
+ throw new Error('browser_profile_keystore_unsupported');
946
+ const resolved = choice.persistent ? await preferences.resolveAsync(owner, preferences.validName(choice.name)) : undefined;
947
+ if (choice.create && (!resolved || resolved.store.has(resolved.scope)
948
+ || (await preferences.allChoicesAsync(owner)).some(item => item.persistent && item.name === choice.name)))
949
+ throw new Error('browser_profile_exists');
950
+ if (resolved && !choice.create && browserProfileScopeId(resolved.scope) !== choice.id
951
+ || !resolved && (await preferences.selectedAsync(owner)).id !== choice.id)
952
+ throw new Error('browser_profile_scope_invalid');
953
+ const same = session.profileScopeId === (resolved ? browserProfileScopeId(resolved.scope) : undefined);
954
+ if (same && !choice.reset) {
955
+ await checkpointProfile(session);
956
+ await preferences.rememberAsync(owner, resolved?.scope.name, choice.default);
957
+ return;
958
+ }
959
+ const saveCurrent = !!resolved && !choice.reset && !resolved.store.has(resolved.scope);
960
+ // Acquire the destination before changing the running browser, so a busy profile leaves it intact.
961
+ let next = same ? session.authProfile : (resolved ? await resolved.store.openAsync(resolved.scope) : undefined);
962
+ if (!authorized() || !options.isAuthorized()) {
963
+ if (next && next !== session.authProfile)
964
+ next.release();
965
+ throw new Error('browser_live_scope_expired');
966
+ }
967
+ if (choice.reset && resolved) {
968
+ if (same) {
969
+ session.authProfile?.release();
970
+ session.authProfile = undefined;
971
+ }
972
+ else
973
+ next?.release();
974
+ await resolved.store.forgetAsync(resolved.scope);
975
+ next = await resolved.store.openAsync(resolved.scope);
976
+ }
977
+ try {
978
+ if (!authorized() || !options.isAuthorized())
979
+ throw new Error('browser_live_scope_expired');
980
+ if (!same && session.authProfile)
981
+ await checkpointProfile(session);
982
+ for (const page of session.tabs.values())
983
+ if (!page.isClosed())
984
+ await clearSecretInputs(page);
985
+ if (saveCurrent) {
986
+ session.snapshotting = true;
987
+ try {
988
+ next.save(await session.context.storageState({ indexedDB: true }));
989
+ }
990
+ finally {
991
+ session.snapshotting = false;
992
+ for (const page of session.context.pages())
993
+ if (!page.isClosed() && registerTab(session, page))
994
+ session.page = page;
995
+ }
996
+ }
997
+ else
998
+ await session.context.setStorageState(next?.state || { cookies: [], origins: [] });
999
+ if (!same || choice.reset) {
1000
+ session.authProfile?.release();
1001
+ session.authProfile = next;
1002
+ session.profileScope = resolved?.scope;
1003
+ session.profileScopeId = resolved ? browserProfileScopeId(resolved.scope) : undefined;
1004
+ }
1005
+ await preferences.rememberAsync(owner, resolved?.scope.name, choice.default);
1006
+ session.observations.invalidate();
1007
+ // Reload discards Firebase's old in-memory auth in each retained tab.
1008
+ for (const page of session.tabs.values())
1009
+ if (!saveCurrent && !page.isClosed() && /^https?:/u.test(page.url()))
1010
+ await page.reload({ waitUntil: 'domcontentloaded', timeout: 30_000 });
1011
+ }
1012
+ catch (error) {
1013
+ if (next && next !== session.authProfile)
1014
+ next.release();
1015
+ throw error;
1016
+ }
1017
+ },
1018
+ });
1019
+ session.live = handle;
1020
+ try {
1021
+ await handle.start();
1022
+ return handle;
1023
+ }
1024
+ catch (error) {
1025
+ handle.disconnect();
1026
+ if (session.live === handle)
1027
+ session.live = undefined;
1028
+ throw error;
1029
+ }
1030
+ }