@nexum-hub/browser 0.0.3-next.8 → 0.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/browser-extension/build.mjs +4 -1
  2. package/browser-extension/connection.js +41 -5
  3. package/browser-extension/dist/development/connection.js +41 -5
  4. package/browser-extension/dist/development/frames.js +11 -0
  5. package/browser-extension/dist/development/manifest.json +1 -1
  6. package/browser-extension/dist/development/page.js +115 -8
  7. package/browser-extension/dist/development/router.js +468 -46
  8. package/browser-extension/dist/development/visual.js +145 -88
  9. package/browser-extension/dist/production/connection.js +41 -5
  10. package/browser-extension/dist/production/frames.js +11 -0
  11. package/browser-extension/dist/production/manifest.json +1 -1
  12. package/browser-extension/dist/production/page.js +115 -8
  13. package/browser-extension/dist/production/router.js +468 -46
  14. package/browser-extension/dist/production/visual.js +145 -88
  15. package/browser-extension/frames.js +11 -0
  16. package/browser-extension/page.js +115 -8
  17. package/browser-extension/release-info.json +5 -3
  18. package/browser-extension/router.js +468 -46
  19. package/browser-extension/visual.js +145 -88
  20. package/dist/client/index.js +25 -1
  21. package/dist/client/routes.js +5 -1
  22. package/dist/client/types.d.ts +20 -0
  23. package/dist/extension/assets.d.ts +1 -0
  24. package/dist/extension/assets.js +3 -1
  25. package/dist/extension/index.js +6 -6
  26. package/dist/extension/scope.d.ts +3 -1
  27. package/dist/extension/scope.js +19 -3
  28. package/dist/extension/service.d.ts +2 -0
  29. package/dist/extension/service.js +18 -8
  30. package/docs/README.md +6 -8
  31. package/docs/connection.md +1 -1
  32. package/docs/usage.md +51 -6
  33. package/implementation.json +1 -1
  34. package/instructions/README.md +5 -7
  35. package/package.json +6 -5
  36. package/skills/browser-control/SKILL.md +11 -5
  37. package/dist/client/index.js.map +0 -1
  38. package/dist/client/routes.js.map +0 -1
  39. package/dist/client/serialize.js.map +0 -1
  40. package/dist/client/types.js.map +0 -1
  41. package/dist/extension/artifacts.js.map +0 -1
  42. package/dist/extension/assets.js.map +0 -1
  43. package/dist/extension/batch.js.map +0 -1
  44. package/dist/extension/files.js.map +0 -1
  45. package/dist/extension/index.js.map +0 -1
  46. package/dist/extension/protocol.js.map +0 -1
  47. package/dist/extension/scope.js.map +0 -1
  48. package/dist/extension/service.js.map +0 -1
@@ -371,14 +371,15 @@ route("tabs.get", async (ctx, args) =>
371
371
  );
372
372
  route(
373
373
  "tabs.new",
374
- async (ctx, args) =>
375
- tabInfo(
376
- ctx,
377
- await ctx.chrome.tabs.create({
378
- ...(args.url === undefined ? {} : { url: httpUrl(args.url) }),
379
- active: options(args).active ?? true,
380
- }),
381
- ),
374
+ async (ctx, args) => {
375
+ const tab = await ctx.chrome.tabs.create({
376
+ ...(args.url === undefined ? {} : { url: httpUrl(args.url) }),
377
+ active: options(args).active ?? true,
378
+ });
379
+ const info = tabInfo(ctx, tab);
380
+ await ctx.router.trackVisualTab({ ...ctx, target: info.target });
381
+ return info;
382
+ },
382
383
  { mutating: true },
383
384
  );
384
385
  route("tabs.selected", async (ctx) => {
@@ -394,6 +395,10 @@ route(
394
395
  const id = tabIdOf(ctx, args);
395
396
  ctx.router.claim(ctx.lease, id);
396
397
  await ctx.chrome.tabs.update(id, { active: true });
398
+ await ctx.router.trackVisualTab({
399
+ ...ctx,
400
+ target: ctx.router.target(String(id), ctx.lease),
401
+ });
397
402
  return ctx.router.actionResult(ctx, id);
398
403
  },
399
404
  { mutating: true },
@@ -1121,33 +1126,202 @@ async function point(ctx, args) {
1121
1126
  requireValue(Number.isFinite(args.x) && Number.isFinite(args.y));
1122
1127
  return { x: args.x, y: args.y };
1123
1128
  }
1129
+
1130
+ async function readyForInput(ctx, framePath = []) {
1131
+ const ready = await ctx.router.page(ctx, {
1132
+ command: "readyState",
1133
+ framePath,
1134
+ });
1135
+ if (ready === "loading" || ready === "interactive") {
1136
+ const remaining = ctx.budgetMs - (performance.now() - ctx.started);
1137
+ requireValue(remaining > 0, "TIMEOUT", "The page did not become ready.");
1138
+ if (framePath.length)
1139
+ await ctx.router.poll(ctx, async () =>
1140
+ (await ctx.router.page(ctx, { command: "readyState", framePath })) ===
1141
+ "complete",
1142
+ );
1143
+ else
1144
+ await ctx.router.waitForLoadState(ctx, {
1145
+ state: "load",
1146
+ timeoutMs: remaining,
1147
+ });
1148
+ }
1149
+ }
1150
+
1151
+ // Build the points for real Chrome Input.dispatchMouseEvent calls, not a
1152
+ // decorative animation that runs independently of the actual input.
1153
+ export function humanPointerPath(from, to, maxSteps = 28) {
1154
+ const dx = to.x - from.x,
1155
+ dy = to.y - from.y,
1156
+ distance = Math.hypot(dx, dy);
1157
+ if (distance < 12) return [{ x: to.x, y: to.y, delayMs: 0 }];
1158
+ const steps = Math.min(maxSteps, Math.max(3, Math.ceil(distance / 28))),
1159
+ durationMs = Math.min(420, Math.max(110, 85 + distance * 0.35)),
1160
+ bend = Math.min(18, distance * 0.08) *
1161
+ (Math.sin(from.x * 0.013 + to.y * 0.017) < 0 ? -1 : 1);
1162
+ const points = [];
1163
+ for (let i = 1; i <= steps; i++) {
1164
+ const t = i / steps,
1165
+ eased = t * t * (3 - 2 * t),
1166
+ arc = Math.sin(Math.PI * t) * bend;
1167
+ points.push({
1168
+ x: i === steps ? to.x : from.x + dx * eased - (dy / distance) * arc,
1169
+ y: i === steps ? to.y : from.y + dy * eased + (dx / distance) * arc,
1170
+ delayMs: i === steps ? 0 : durationMs / steps,
1171
+ });
1172
+ }
1173
+ return points;
1174
+ }
1124
1175
  async function click(ctx, args, double = false) {
1125
- const p = await point(ctx, args),
1126
- button = buttons[args.button ?? "left"];
1176
+ if (!args.resolvedTarget) await readyForInput(ctx);
1177
+ let p = await point(ctx, args);
1178
+ const button = buttons[args.button ?? "left"];
1127
1179
  requireValue(button);
1128
1180
  const count = double ? 2 : (args.clickCount ?? 1);
1129
1181
  requireValue(Number.isInteger(count) && count >= 1 && count <= 3);
1130
- await ctx.router.send(ctx, "Input.dispatchMouseEvent", {
1131
- type: "mouseMoved",
1132
- ...p,
1133
- button: "none",
1134
- });
1135
- for (let i = 1; i <= count; i++) {
1182
+ const framePath = args.framePath ?? [];
1183
+ const originalVersion = ctx.router.version(ctx.target.tabId);
1184
+ let receipt;
1185
+ try {
1136
1186
  await ctx.router.send(ctx, "Input.dispatchMouseEvent", {
1137
- type: "mousePressed",
1187
+ type: "mouseMoved",
1138
1188
  ...p,
1139
- button,
1140
- clickCount: i,
1141
- modifiers: modifierMask(args.modifiers ?? args.keypress),
1189
+ button: "none",
1142
1190
  });
1143
- if (args.duration > 0) await ctx.router.delay(ctx, args.duration * 1000);
1144
- await ctx.router.send(ctx, "Input.dispatchMouseEvent", {
1145
- type: "mouseReleased",
1146
- ...p,
1147
- button,
1148
- clickCount: i,
1149
- modifiers: modifierMask(args.modifiers ?? args.keypress),
1191
+ let framePoint = args.framePoint;
1192
+ if (args.expectedRef) {
1193
+ // The first native pointer movement can cause Chromium's visible
1194
+ // viewport to change. Re-observe the same element after that movement,
1195
+ // before the first mouse press; no click is ever replayed.
1196
+ let stable = false;
1197
+ for (let attempt = 0; attempt < 4; attempt++) {
1198
+ const current = await ctx.router.page(ctx, {
1199
+ command: "resolve",
1200
+ ref: args.expectedRef,
1201
+ framePath,
1202
+ noScroll: true,
1203
+ ...(args.position ? { position: args.position } : {}),
1204
+ });
1205
+ requireValue(
1206
+ current && (!args.frameId || current.frameId === args.frameId),
1207
+ "INVALID_TARGET",
1208
+ "The original click element is no longer at a controllable point.",
1209
+ );
1210
+ if (
1211
+ Math.abs(current.point.x - p.x) <= 1 &&
1212
+ Math.abs(current.point.y - p.y) <= 1
1213
+ ) {
1214
+ framePoint = current.framePoint ?? framePoint;
1215
+ stable = true;
1216
+ break;
1217
+ }
1218
+ p = current.point;
1219
+ framePoint = current.framePoint ?? framePoint;
1220
+ await ctx.router.send(ctx, "Input.dispatchMouseEvent", {
1221
+ type: "mouseMoved",
1222
+ ...p,
1223
+ button: "none",
1224
+ });
1225
+ }
1226
+ requireValue(
1227
+ stable,
1228
+ "INVALID_TARGET",
1229
+ "The click target continued moving after native pointer preparation.",
1230
+ );
1231
+ }
1232
+ const receiptInput = {
1233
+ command: "inputReceipt",
1234
+ token: ctx.actionId,
1235
+ x: framePoint?.x ?? p.x,
1236
+ y: framePoint?.y ?? p.y,
1237
+ button: { left: 0, middle: 1, right: 2 }[args.button ?? "left"],
1238
+ expiresAt: Date.now() + Math.min(3000, ctx.budgetMs),
1239
+ framePath,
1240
+ ...(args.frameId ? { expectedFrameId: args.frameId } : {}),
1241
+ ...(args.expectedRef ? { expectedRef: args.expectedRef } : {}),
1242
+ };
1243
+ const armed = await ctx.router.page(ctx, {
1244
+ ...receiptInput,
1245
+ action: "arm",
1150
1246
  });
1247
+ requireValue(
1248
+ armed?.armed === true && typeof armed.documentToken === "string",
1249
+ "INVALID_TARGET",
1250
+ "The browser could not arm a receipt for the exact input document.",
1251
+ );
1252
+ receipt = {
1253
+ ...receiptInput,
1254
+ expectedDocumentToken: armed.documentToken,
1255
+ };
1256
+ for (let i = 1; i <= count; i++) {
1257
+ await ctx.router.send(ctx, "Input.dispatchMouseEvent", {
1258
+ type: "mousePressed",
1259
+ ...p,
1260
+ button,
1261
+ clickCount: i,
1262
+ modifiers: modifierMask(args.modifiers ?? args.keypress),
1263
+ });
1264
+ if (args.duration > 0) await ctx.router.delay(ctx, args.duration * 1000);
1265
+ await ctx.router.send(ctx, "Input.dispatchMouseEvent", {
1266
+ type: "mouseReleased",
1267
+ ...p,
1268
+ button,
1269
+ clickCount: i,
1270
+ modifiers: modifierMask(args.modifiers ?? args.keypress),
1271
+ });
1272
+ }
1273
+ let observed = false;
1274
+ const expires = Math.min(
1275
+ performance.now() + 450,
1276
+ ctx.started + ctx.budgetMs,
1277
+ );
1278
+ while (performance.now() < expires) {
1279
+ if (
1280
+ !framePath.length &&
1281
+ ctx.router.version(ctx.target.tabId) !== originalVersion
1282
+ ) {
1283
+ observed = true; // The main document navigated after input.
1284
+ break;
1285
+ }
1286
+ try {
1287
+ const result = await ctx.router.page(ctx, {
1288
+ ...receipt,
1289
+ action: "peek",
1290
+ });
1291
+ if (result.observed >= count) {
1292
+ observed = true;
1293
+ break;
1294
+ }
1295
+ } catch {
1296
+ ctx.signal?.throwIfAborted();
1297
+ if (
1298
+ !framePath.length &&
1299
+ ctx.router.version(ctx.target.tabId) !== originalVersion
1300
+ ) {
1301
+ observed = true;
1302
+ break;
1303
+ }
1304
+ // A detached or replaced frame does not prove the target received
1305
+ // input. The action may still have happened, so never replay it.
1306
+ break;
1307
+ }
1308
+ await ctx.router.delay(ctx, Math.min(40, expires - performance.now()));
1309
+ }
1310
+ if (!observed)
1311
+ throw new BrowserActionError(
1312
+ "INPUT_UNCONFIRMED",
1313
+ "Native pointer delivery returned, but the target document did not confirm the expected trusted click event.",
1314
+ { dispatched: true, effect: "unknown" },
1315
+ );
1316
+ // Let the browser process the resulting task/render turn before the next
1317
+ // input, without misrepresenting arbitrary asynchronous business work.
1318
+ const remaining = ctx.budgetMs - (performance.now() - ctx.started);
1319
+ if (remaining > 120) await ctx.router.delay(ctx, 120);
1320
+ } finally {
1321
+ if (receipt)
1322
+ await ctx.router
1323
+ .page(ctx, { ...receipt, action: "drop" })
1324
+ .catch(() => {});
1151
1325
  }
1152
1326
  return null;
1153
1327
  }
@@ -1179,6 +1353,13 @@ route(
1179
1353
  requireValue(
1180
1354
  Number.isFinite(input.scrollX) && Number.isFinite(input.scrollY),
1181
1355
  );
1356
+ // Position the real pointer along the same human trajectory before
1357
+ // applying the wheel at that point; never teleport the visual cursor.
1358
+ await ctx.router.send(ctx, "Input.dispatchMouseEvent", {
1359
+ type: "mouseMoved",
1360
+ ...(await point(ctx, input)),
1361
+ modifiers: modifierMask(input.keypress),
1362
+ });
1182
1363
  await ctx.router.send(ctx, "Input.dispatchMouseEvent", {
1183
1364
  type: "mouseWheel",
1184
1365
  ...(await point(ctx, input)),
@@ -1298,22 +1479,122 @@ route(
1298
1479
  { needsTab: true, mutating: true },
1299
1480
  );
1300
1481
  async function resolveActionTarget(ctx, args) {
1301
- let found;
1302
- // Only repeat the pre-input observation. Native pointer dispatch happens once
1303
- // after an unobstructed current target is observed, never after a failed action.
1482
+ let found, previous;
1483
+ let scrollRequested = false;
1484
+ // Chromium can change the page viewport when its native debugger attaches
1485
+ // (notably by adding a debugging infobar). Attach before measuring or
1486
+ // scrolling the target; attaching only on the first mouse event can shift
1487
+ // the page after preflight and send a trusted click to the wrong element.
1488
+ await ctx.router.attach(ctx);
1489
+ await readyForInput(ctx, args.locator?.framePath ?? args.framePath ?? []);
1490
+ const sameLayout = (left, right) =>
1491
+ left && right &&
1492
+ ["x", "y"].every((key) =>
1493
+ Math.abs(left.point[key] - right.point[key]) <= 1,
1494
+ ) &&
1495
+ ["x", "y", "width", "height"].every((key) =>
1496
+ Math.abs(left.rect[key] - right.rect[key]) <= 1,
1497
+ ) &&
1498
+ left.ref === right.ref &&
1499
+ left.documentToken === right.documentToken &&
1500
+ left.frameId === right.frameId;
1501
+ // Two matching native observations give the layout a chance to settle
1502
+ // after scrolling. Only pre-input observations may repeat, never the click.
1304
1503
  await ctx.router.poll(ctx, async () => {
1305
- found = await ctx.router.page(ctx, {
1504
+ const next = await ctx.router.page(ctx, {
1306
1505
  ...args,
1307
1506
  ...options(args),
1308
1507
  command: "resolve",
1508
+ ...(scrollRequested ? { noScroll: true } : {}),
1309
1509
  });
1310
- return found !== null;
1510
+ if (!next) {
1511
+ previous = undefined;
1512
+ return false;
1513
+ }
1514
+ // A missing target did not establish a usable scroll position. If it
1515
+ // appears later below the viewport, its first observation must still be
1516
+ // allowed to scroll it into view.
1517
+ scrollRequested = true;
1518
+ const stable = sameLayout(previous, next);
1519
+ previous = next;
1520
+ if (!stable) return false;
1521
+ // Native mouse movement later re-resolves this exact ref before pressing,
1522
+ // so another pre-move snapshot cannot prove that the final point is safe.
1523
+ found = next;
1524
+ return true;
1311
1525
  });
1312
1526
  return found;
1313
1527
  }
1314
1528
  async function locatorClick(ctx, args, double = false) {
1315
1529
  const found = await resolveActionTarget(ctx, args);
1316
- return click(ctx, { ...options(args), ...found.point }, double);
1530
+ const input = options(args),
1531
+ expect = input.expect,
1532
+ framePath = args.locator?.framePath ?? args.framePath ?? [];
1533
+ // The native navigation event must be armed before input; top-level only.
1534
+ requireValue(
1535
+ !(expect?.kind === "navigation" && framePath.length),
1536
+ "INVALID_ARGUMENT",
1537
+ "For a nested frame, wait for its URL or a frame-local page condition.",
1538
+ );
1539
+ let navigation;
1540
+ if (expect?.kind === "navigation")
1541
+ navigation = await ctx.router.armWaiter(ctx, {
1542
+ event: "navigation",
1543
+ url: expect.url,
1544
+ waitUntil: expect.waitUntil ?? "load",
1545
+ timeoutMs:
1546
+ expect.timeoutMs ??
1547
+ Math.min(15000, ctx.budgetMs - (performance.now() - ctx.started)),
1548
+ });
1549
+ try {
1550
+ await click(
1551
+ ctx,
1552
+ {
1553
+ ...input,
1554
+ ...found.point,
1555
+ resolvedTarget: true,
1556
+ framePath,
1557
+ framePoint: found.framePoint,
1558
+ frameId: found.frameId,
1559
+ expectedRef: found.ref,
1560
+ },
1561
+ double,
1562
+ );
1563
+ if (!expect) return null;
1564
+ try {
1565
+ if (navigation) {
1566
+ await ctx.router.getWaiter(ctx, navigation.id);
1567
+ return {
1568
+ matched: true,
1569
+ snapshot: {
1570
+ ...ctx.router.target(ctx.target.tabId, ctx.lease),
1571
+ ...(await ctx.router.page(ctx, { command: "snapshot" })),
1572
+ },
1573
+ };
1574
+ }
1575
+ return await ctx.router.waitForCondition(ctx, {
1576
+ condition: expect,
1577
+ framePath,
1578
+ });
1579
+ } catch (cause) {
1580
+ ctx.signal?.throwIfAborted();
1581
+ if (
1582
+ cause instanceof BrowserActionError &&
1583
+ ["TIMEOUT", "STALE_REF", "INVALID_TARGET", "WAITER_GONE"].includes(
1584
+ cause.code,
1585
+ )
1586
+ )
1587
+ throw new BrowserActionError(
1588
+ "EXPECTATION_UNCONFIRMED",
1589
+ "The page did not confirm the expected result. Input was already sent; observe before taking further action.",
1590
+ { dispatched: true, effect: "unknown" },
1591
+ );
1592
+ throw cause;
1593
+ }
1594
+ } finally {
1595
+ if (navigation)
1596
+ await ctx.router.cancelWaiter(ctx, navigation.id).catch(() => {});
1597
+ }
1317
1598
  }
1318
1599
  route("locator.click", (ctx, args) => locatorClick(ctx, args), {
1319
1600
  needsTab: true,
@@ -1684,6 +1965,7 @@ export function createBrowserRouter(
1684
1965
  journalTail = Promise.resolve();
1685
1966
  const versions = new Map(),
1686
1967
  documents = new Map(),
1968
+ pointerPositions = new Map(),
1687
1969
  leases = new Map(),
1688
1970
  closedLeases = new Map(),
1689
1971
  owners = new Map(),
@@ -2255,11 +2537,51 @@ export function createBrowserRouter(
2255
2537
  throw cause;
2256
2538
  }
2257
2539
  };
2258
- const releaseControl = async (lease, tabId, { detach = true } = {}) => {
2540
+ // Chrome may hold a native command indefinitely while a JavaScript dialog
2541
+ // pauses its renderer. A cancelled action must not hold the run's pending
2542
+ // queue forever; the native command may still have an unknown page effect,
2543
+ // so actual debugger detach remains a separate required cleanup step.
2544
+ const nativeWithStop = async (signal, invoke) => {
2545
+ signal?.throwIfAborted();
2546
+ const native = Promise.resolve().then(() => {
2547
+ signal?.throwIfAborted();
2548
+ return invoke();
2549
+ });
2550
+ if (!signal) return native;
2551
+ let onAbort;
2552
+ const cancelled = new Promise((_, reject) => {
2553
+ onAbort = () => reject(signal.reason);
2554
+ signal.addEventListener("abort", onAbort, { once: true });
2555
+ });
2556
+ try {
2557
+ if (signal.aborted) onAbort();
2558
+ return await Promise.race([native, cancelled]);
2559
+ } finally {
2560
+ signal.removeEventListener("abort", onAbort);
2561
+ }
2562
+ };
2563
+ const releaseControl = async (
2564
+ lease,
2565
+ tabId,
2566
+ { detach = true, dismissDialog = false } = {},
2567
+ ) => {
2259
2568
  if (!lease.attached.has(tabId) && !lease.intents.has(tabId)) return;
2260
2569
  for (const waiter of waiters.values())
2261
2570
  if (waiter.leaseId === lease.id && waiter.tabId === tabId)
2262
2571
  cancelWaiter(waiter);
2572
+ // An observed JavaScript dialog pauses its renderer. Dismiss the dialog
2573
+ // for a terminating run before other native cleanup commands; explicit
2574
+ // handoff keeps the dialog available for the user instead.
2575
+ const dialog = dialogs.get(String(tabId));
2576
+ if (dismissDialog && dialog && lease.attached.has(tabId)) {
2577
+ await chrome.debugger.sendCommand(
2578
+ { tabId },
2579
+ "Page.handleJavaScriptDialog",
2580
+ { accept: false },
2581
+ );
2582
+ if (dialogs.get(String(tabId)) === dialog)
2583
+ dialogs.delete(String(tabId));
2584
+ }
2263
2585
  await stopLocalDownloadClick(lease, tabId);
2264
2586
  if (lease.attached.has(tabId)) await syncDownloadIntercept(lease, tabId);
2265
2587
  if (!lease.attached.has(tabId) && !lease.externallyDetached.has(tabId)) {
@@ -2522,6 +2844,7 @@ export function createBrowserRouter(
2522
2844
  finishNavigation(details, "load"),
2523
2845
  );
2524
2846
  on(chrome.tabs.onRemoved, (id) => {
2847
+ pointerPositions.delete(id);
2525
2848
  versions.set(String(id), version(id) + 1);
2526
2849
  dialogs.delete(String(id));
2527
2850
  emit({
@@ -3081,7 +3404,7 @@ export function createBrowserRouter(
3081
3404
  .filter((id) => id !== undefined),
3082
3405
  ))
3083
3406
  try {
3084
- await releaseControl(lease, tabId);
3407
+ await releaseControl(lease, tabId, { dismissDialog: true });
3085
3408
  } catch (cause) {
3086
3409
  tempErrors.push(cause);
3087
3410
  }
@@ -3100,7 +3423,7 @@ export function createBrowserRouter(
3100
3423
  }
3101
3424
  for (const tabId of new Set([...lease.attached, ...lease.intents])) {
3102
3425
  try {
3103
- await releaseControl(lease, tabId);
3426
+ await releaseControl(lease, tabId, { dismissDialog: true });
3104
3427
  } catch (cause) {
3105
3428
  errors.push(cause);
3106
3429
  }
@@ -3331,6 +3654,14 @@ export function createBrowserRouter(
3331
3654
  ].includes(input.method)
3332
3655
  )
3333
3656
  await router.prepareVisual(ctx);
3657
+ if (
3658
+ visual &&
3659
+ definition.needsTab &&
3660
+ !definition.mutating &&
3661
+ input.target.tabId !== undefined &&
3662
+ !lease.temporaryTabs?.has(nativeId(input.target.tabId))
3663
+ )
3664
+ await router.trackVisualTab(ctx);
3334
3665
  actionSignal.throwIfAborted();
3335
3666
  managementHandling = true;
3336
3667
  value = await definition.handle(ctx, input.args ?? {});
@@ -3367,9 +3698,14 @@ export function createBrowserRouter(
3367
3698
  "tabs.closeTemporary",
3368
3699
  "tab.getJsDialog",
3369
3700
  ].includes(input.method);
3701
+ // Native mutations can activate a tab and send shared browser input.
3702
+ // A per-tab queue cannot protect two simultaneous actions that switch
3703
+ // the active tab underneath one another. Reads and event waiters remain
3704
+ // independently schedulable.
3370
3705
  const queueKey =
3371
- input.target.tabId ??
3372
- (definition.mutating ? "browser-management" : undefined);
3706
+ definition.mutating && !nonBlocking
3707
+ ? `native-mutation:${input.target.browserId}:${input.target.browserSessionId}`
3708
+ : input.target.tabId;
3373
3709
  let task;
3374
3710
  if (queueKey && !nonBlocking) {
3375
3711
  task = (queues.get(queueKey) ?? Promise.resolve()).then(perform);
@@ -3671,12 +4007,14 @@ export function createBrowserRouter(
3671
4007
  const framePath = payload.locator?.framePath ?? payload.framePath ?? [];
3672
4008
  if (framePath.length > 0) return framePage(ctx, payload, framePath);
3673
4009
  ctx.dispatched();
3674
- const result = await chrome.scripting.executeScript({
3675
- target: { tabId: nativeId(ctx.target.tabId) },
3676
- world: "ISOLATED",
3677
- func: pageCommand,
3678
- args: [input],
3679
- });
4010
+ const result = await nativeWithStop(ctx.signal, () =>
4011
+ chrome.scripting.executeScript({
4012
+ target: { tabId: nativeId(ctx.target.tabId) },
4013
+ world: "ISOLATED",
4014
+ func: pageCommand,
4015
+ args: [input],
4016
+ }),
4017
+ );
3680
4018
  // A main-frame replacement can finish an injection with no result. Only
3681
4019
  // a readiness observation may wait for a browser-confirmed transition;
3682
4020
  // actions and empty results from settled pages must still fail.
@@ -3865,6 +4203,17 @@ export function createBrowserRouter(
3865
4203
  ctx.dispatched();
3866
4204
  await chrome.debugger.attach({ tabId: id }, "1.3");
3867
4205
  ctx.lease.attached.add(id);
4206
+ // Observe Page.javascriptDialogOpening before *any* page action can
4207
+ // synchronously open a prompt/confirm. Waiting until getJsDialog is
4208
+ // called loses the event while Runtime.evaluate may remain blocked.
4209
+ try {
4210
+ await nativeWithStop(ctx.signal, () =>
4211
+ chrome.debugger.sendCommand({ tabId: id }, "Page.enable", {}),
4212
+ );
4213
+ } catch (cause) {
4214
+ await releaseControl(ctx.lease, id).catch(() => {});
4215
+ throw cause;
4216
+ }
3868
4217
  if (ctx.lease.ended) {
3869
4218
  await releaseControl(ctx.lease, id);
3870
4219
  throw new BrowserActionError(
@@ -3880,9 +4229,42 @@ export function createBrowserRouter(
3880
4229
  ctx.lease.attaching.delete(id);
3881
4230
  }
3882
4231
  },
3883
- async send(ctx, method, params, child) {
4232
+ async send(ctx, method, params, child, nativePointerStep = false) {
3884
4233
  active(ctx.lease);
3885
4234
  ctx.signal?.throwIfAborted();
4235
+ // Every high-level mouse move, click, hover and drag shares the same
4236
+ // physical trajectory. Raw cdp.send remains exact and unmodified.
4237
+ if (
4238
+ !nativePointerStep &&
4239
+ !child &&
4240
+ ctx.method !== "cdp.send" &&
4241
+ method === "Input.dispatchMouseEvent" &&
4242
+ params.type === "mouseMoved" &&
4243
+ Number.isFinite(params.x) &&
4244
+ Number.isFinite(params.y)
4245
+ ) {
4246
+ const tabId = nativeId(ctx.target.tabId),
4247
+ previous = pointerPositions.get(tabId) ?? { x: 32, y: 72 };
4248
+ // Chromium can heavily throttle input acknowledgement in an inactive
4249
+ // tab. Keep a real multi-point native trajectory, but bound its steps
4250
+ // so one background move cannot outlive the original SDK request.
4251
+ const tab = await chrome.tabs.get(tabId);
4252
+ const clickMove = /(^|\.)(click|dblclick|doubleClick)$/.test(ctx.method);
4253
+ const maxSteps = tab.active ? 28 : clickMove ? 2 : 3;
4254
+ const path = humanPointerPath(previous, { x: params.x, y: params.y }, maxSteps);
4255
+ let result;
4256
+ for (const step of path) {
4257
+ result = await router.send(
4258
+ ctx,
4259
+ method,
4260
+ { ...params, x: step.x, y: step.y },
4261
+ child,
4262
+ true,
4263
+ );
4264
+ if (step.delayMs > 0) await router.delay(ctx, step.delayMs);
4265
+ }
4266
+ return result;
4267
+ }
3886
4268
  await router.attach(ctx);
3887
4269
  ctx.signal?.throwIfAborted();
3888
4270
  if (
@@ -4035,7 +4417,20 @@ export function createBrowserRouter(
4035
4417
  ctx.pendingInfo.nativeNetworkTabId = tabId;
4036
4418
  let result;
4037
4419
  try {
4038
- result = await chrome.debugger.sendCommand(source, method, params);
4420
+ // A native Runtime.evaluate can stay suspended behind a modal dialog.
4421
+ // A raw cdp.send Network.loadNetworkResource can also remain pending
4422
+ // after debugger detach (notably on Windows). The lease still confirms
4423
+ // native input release and debugger detach before reporting closure.
4424
+ // Internal document exports must await the native response so any
4425
+ // returned stream can be closed; IO.read and IO.close likewise retain
4426
+ // their actual settlement.
4427
+ result =
4428
+ method === "Runtime.evaluate" ||
4429
+ (method === "Network.loadNetworkResource" && ctx.method === "cdp.send")
4430
+ ? await nativeWithStop(ctx.signal, () =>
4431
+ chrome.debugger.sendCommand(source, method, params),
4432
+ )
4433
+ : await chrome.debugger.sendCommand(source, method, params);
4039
4434
  } catch (cause) {
4040
4435
  if (armedLocalDownload) await stopLocalDownloadClick(ctx.lease, tabId);
4041
4436
  throw cause;
@@ -4044,6 +4439,14 @@ export function createBrowserRouter(
4044
4439
  delete ctx.pendingInfo.nativeNetworkTabId;
4045
4440
  if (watchEnd) await stopLocalDownloadClick(ctx.lease, tabId);
4046
4441
  }
4442
+ if (
4443
+ !child &&
4444
+ method === "Input.dispatchMouseEvent" &&
4445
+ params.type === "mouseMoved" &&
4446
+ Number.isFinite(params.x) &&
4447
+ Number.isFinite(params.y)
4448
+ )
4449
+ pointerPositions.set(tabId, { x: params.x, y: params.y });
4047
4450
  if (rawFetch && method === "Fetch.disable") {
4048
4451
  ctx.lease.rawFetch?.delete(tabId);
4049
4452
  await syncDownloadIntercept(ctx.lease, tabId);
@@ -4874,6 +5277,9 @@ export function createBrowserRouter(
4874
5277
  },
4875
5278
  async screenshot(ctx, input) {
4876
5279
  requireValue(!(input.fullPage && input.clip));
5280
+ await readyForInput(ctx);
5281
+ const readyBudget = ctx.budgetMs - (performance.now() - ctx.started);
5282
+ if (readyBudget > 130) await router.delay(ctx, 120);
4877
5283
  const quality = input.quality ?? 80,
4878
5284
  maxEdgePx = input.maxEdgePx ?? 1920;
4879
5285
  requireValue(
@@ -5308,6 +5714,22 @@ export function createBrowserRouter(
5308
5714
  });
5309
5715
  }
5310
5716
  },
5717
+ async trackVisualTab(ctx) {
5718
+ try {
5719
+ await visual?.trackTab(router.visualContext(ctx));
5720
+ } catch {
5721
+ ctx.signal?.throwIfAborted();
5722
+ emit({
5723
+ type: "event",
5724
+ event: "visual_degraded",
5725
+ leaseId: ctx.lease.id,
5726
+ runId: ctx.runId,
5727
+ operationId: ctx.operationId,
5728
+ target: ctx.target,
5729
+ reason: "VISUAL_UNAVAILABLE",
5730
+ });
5731
+ }
5732
+ },
5311
5733
  async preparePointer(ctx, point) {
5312
5734
  if (
5313
5735
  ctx.lastVisualPoint?.x === point.x &&