@nexum-hub/browser 0.0.3-next.16 → 0.0.3-next.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/browser-extension/dist/development/frames.js +11 -0
- package/browser-extension/dist/development/page.js +93 -3
- package/browser-extension/dist/development/router.js +395 -44
- package/browser-extension/dist/development/visual.js +145 -88
- package/browser-extension/dist/production/frames.js +11 -0
- package/browser-extension/dist/production/page.js +93 -3
- package/browser-extension/dist/production/router.js +395 -44
- package/browser-extension/dist/production/visual.js +145 -88
- package/browser-extension/frames.js +11 -0
- package/browser-extension/page.js +93 -3
- package/browser-extension/release-info.json +1 -1
- package/browser-extension/router.js +395 -44
- package/browser-extension/visual.js +145 -88
- package/dist/client/index.js +25 -1
- package/dist/client/index.js.map +1 -1
- package/dist/client/routes.js +5 -1
- package/dist/client/routes.js.map +1 -1
- package/dist/client/types.d.ts +20 -0
- package/docs/usage.md +22 -1
- package/implementation.json +1 -1
- package/package.json +3 -3
- package/skills/browser-control/SKILL.md +15 -0
|
@@ -371,14 +371,15 @@ route("tabs.get", async (ctx, args) =>
|
|
|
371
371
|
);
|
|
372
372
|
route(
|
|
373
373
|
"tabs.new",
|
|
374
|
-
async (ctx, args) =>
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
374
|
+
async (ctx, args) => {
|
|
375
|
+
const tab = await ctx.chrome.tabs.create({
|
|
376
|
+
...(args.url === undefined ? {} : { url: httpUrl(args.url) }),
|
|
377
|
+
active: options(args).active ?? true,
|
|
378
|
+
});
|
|
379
|
+
const info = tabInfo(ctx, tab);
|
|
380
|
+
await ctx.router.trackVisualTab({ ...ctx, target: info.target });
|
|
381
|
+
return info;
|
|
382
|
+
},
|
|
382
383
|
{ mutating: true },
|
|
383
384
|
);
|
|
384
385
|
route("tabs.selected", async (ctx) => {
|
|
@@ -394,6 +395,10 @@ route(
|
|
|
394
395
|
const id = tabIdOf(ctx, args);
|
|
395
396
|
ctx.router.claim(ctx.lease, id);
|
|
396
397
|
await ctx.chrome.tabs.update(id, { active: true });
|
|
398
|
+
await ctx.router.trackVisualTab({
|
|
399
|
+
...ctx,
|
|
400
|
+
target: ctx.router.target(String(id), ctx.lease),
|
|
401
|
+
});
|
|
397
402
|
return ctx.router.actionResult(ctx, id);
|
|
398
403
|
},
|
|
399
404
|
{ mutating: true },
|
|
@@ -1121,33 +1126,158 @@ async function point(ctx, args) {
|
|
|
1121
1126
|
requireValue(Number.isFinite(args.x) && Number.isFinite(args.y));
|
|
1122
1127
|
return { x: args.x, y: args.y };
|
|
1123
1128
|
}
|
|
1129
|
+
|
|
1130
|
+
async function readyForInput(ctx, framePath = []) {
|
|
1131
|
+
const ready = await ctx.router.page(ctx, {
|
|
1132
|
+
command: "readyState",
|
|
1133
|
+
framePath,
|
|
1134
|
+
});
|
|
1135
|
+
if (ready === "loading" || ready === "interactive") {
|
|
1136
|
+
const remaining = ctx.budgetMs - (performance.now() - ctx.started);
|
|
1137
|
+
requireValue(remaining > 0, "TIMEOUT", "The page did not become ready.");
|
|
1138
|
+
if (framePath.length)
|
|
1139
|
+
await ctx.router.poll(ctx, async () =>
|
|
1140
|
+
(await ctx.router.page(ctx, { command: "readyState", framePath })) ===
|
|
1141
|
+
"complete",
|
|
1142
|
+
);
|
|
1143
|
+
else
|
|
1144
|
+
await ctx.router.waitForLoadState(ctx, {
|
|
1145
|
+
state: "load",
|
|
1146
|
+
timeoutMs: remaining,
|
|
1147
|
+
});
|
|
1148
|
+
}
|
|
1149
|
+
}
|
|
1150
|
+
|
|
1151
|
+
// Build the points for real Chrome Input.dispatchMouseEvent calls, not a
|
|
1152
|
+
// decorative animation that runs independently of the actual input.
|
|
1153
|
+
export function humanPointerPath(from, to) {
|
|
1154
|
+
const dx = to.x - from.x,
|
|
1155
|
+
dy = to.y - from.y,
|
|
1156
|
+
distance = Math.hypot(dx, dy);
|
|
1157
|
+
if (distance < 12) return [{ x: to.x, y: to.y, delayMs: 0 }];
|
|
1158
|
+
const steps = Math.min(28, Math.max(3, Math.ceil(distance / 28))),
|
|
1159
|
+
durationMs = Math.min(420, Math.max(110, 85 + distance * 0.35)),
|
|
1160
|
+
bend = Math.min(18, distance * 0.08) *
|
|
1161
|
+
(Math.sin(from.x * 0.013 + to.y * 0.017) < 0 ? -1 : 1);
|
|
1162
|
+
const points = [];
|
|
1163
|
+
for (let i = 1; i <= steps; i++) {
|
|
1164
|
+
const t = i / steps,
|
|
1165
|
+
eased = t * t * (3 - 2 * t),
|
|
1166
|
+
arc = Math.sin(Math.PI * t) * bend;
|
|
1167
|
+
points.push({
|
|
1168
|
+
x: i === steps ? to.x : from.x + dx * eased - (dy / distance) * arc,
|
|
1169
|
+
y: i === steps ? to.y : from.y + dy * eased + (dx / distance) * arc,
|
|
1170
|
+
delayMs: i === steps ? 0 : durationMs / steps,
|
|
1171
|
+
});
|
|
1172
|
+
}
|
|
1173
|
+
return points;
|
|
1174
|
+
}
|
|
1124
1175
|
async function click(ctx, args, double = false) {
|
|
1176
|
+
if (!args.resolvedTarget) await readyForInput(ctx);
|
|
1125
1177
|
const p = await point(ctx, args),
|
|
1126
1178
|
button = buttons[args.button ?? "left"];
|
|
1127
1179
|
requireValue(button);
|
|
1128
1180
|
const count = double ? 2 : (args.clickCount ?? 1);
|
|
1129
1181
|
requireValue(Number.isInteger(count) && count >= 1 && count <= 3);
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1182
|
+
const framePath = args.framePath ?? [];
|
|
1183
|
+
const originalVersion = ctx.router.version(ctx.target.tabId);
|
|
1184
|
+
const receiptInput = {
|
|
1185
|
+
command: "inputReceipt",
|
|
1186
|
+
token: ctx.actionId,
|
|
1187
|
+
x: args.framePoint?.x ?? p.x,
|
|
1188
|
+
y: args.framePoint?.y ?? p.y,
|
|
1189
|
+
button: { left: 0, middle: 1, right: 2 }[args.button ?? "left"],
|
|
1190
|
+
expiresAt: Date.now() + Math.min(3000, ctx.budgetMs),
|
|
1191
|
+
framePath,
|
|
1192
|
+
...(args.frameId ? { expectedFrameId: args.frameId } : {}),
|
|
1193
|
+
};
|
|
1194
|
+
const armed = await ctx.router.page(ctx, {
|
|
1195
|
+
...receiptInput,
|
|
1196
|
+
action: "arm",
|
|
1134
1197
|
});
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1198
|
+
requireValue(
|
|
1199
|
+
armed?.armed === true && typeof armed.documentToken === "string",
|
|
1200
|
+
"INVALID_TARGET",
|
|
1201
|
+
"The browser could not arm a receipt for the exact input document.",
|
|
1202
|
+
);
|
|
1203
|
+
const receipt = {
|
|
1204
|
+
...receiptInput,
|
|
1205
|
+
expectedDocumentToken: armed.documentToken,
|
|
1206
|
+
};
|
|
1207
|
+
try {
|
|
1144
1208
|
await ctx.router.send(ctx, "Input.dispatchMouseEvent", {
|
|
1145
|
-
type: "
|
|
1209
|
+
type: "mouseMoved",
|
|
1146
1210
|
...p,
|
|
1147
|
-
button,
|
|
1148
|
-
clickCount: i,
|
|
1149
|
-
modifiers: modifierMask(args.modifiers ?? args.keypress),
|
|
1211
|
+
button: "none",
|
|
1150
1212
|
});
|
|
1213
|
+
for (let i = 1; i <= count; i++) {
|
|
1214
|
+
await ctx.router.send(ctx, "Input.dispatchMouseEvent", {
|
|
1215
|
+
type: "mousePressed",
|
|
1216
|
+
...p,
|
|
1217
|
+
button,
|
|
1218
|
+
clickCount: i,
|
|
1219
|
+
modifiers: modifierMask(args.modifiers ?? args.keypress),
|
|
1220
|
+
});
|
|
1221
|
+
if (args.duration > 0) await ctx.router.delay(ctx, args.duration * 1000);
|
|
1222
|
+
await ctx.router.send(ctx, "Input.dispatchMouseEvent", {
|
|
1223
|
+
type: "mouseReleased",
|
|
1224
|
+
...p,
|
|
1225
|
+
button,
|
|
1226
|
+
clickCount: i,
|
|
1227
|
+
modifiers: modifierMask(args.modifiers ?? args.keypress),
|
|
1228
|
+
});
|
|
1229
|
+
}
|
|
1230
|
+
let observed = false;
|
|
1231
|
+
const expires = Math.min(
|
|
1232
|
+
performance.now() + 450,
|
|
1233
|
+
ctx.started + ctx.budgetMs,
|
|
1234
|
+
);
|
|
1235
|
+
while (performance.now() < expires) {
|
|
1236
|
+
if (
|
|
1237
|
+
!framePath.length &&
|
|
1238
|
+
ctx.router.version(ctx.target.tabId) !== originalVersion
|
|
1239
|
+
) {
|
|
1240
|
+
observed = true; // The main document navigated after input.
|
|
1241
|
+
break;
|
|
1242
|
+
}
|
|
1243
|
+
try {
|
|
1244
|
+
const result = await ctx.router.page(ctx, {
|
|
1245
|
+
...receipt,
|
|
1246
|
+
action: "peek",
|
|
1247
|
+
});
|
|
1248
|
+
if (result.observed >= count) {
|
|
1249
|
+
observed = true;
|
|
1250
|
+
break;
|
|
1251
|
+
}
|
|
1252
|
+
} catch {
|
|
1253
|
+
ctx.signal?.throwIfAborted();
|
|
1254
|
+
if (
|
|
1255
|
+
!framePath.length &&
|
|
1256
|
+
ctx.router.version(ctx.target.tabId) !== originalVersion
|
|
1257
|
+
) {
|
|
1258
|
+
observed = true;
|
|
1259
|
+
break;
|
|
1260
|
+
}
|
|
1261
|
+
// A detached or replaced frame does not prove the target received
|
|
1262
|
+
// input. The action may still have happened, so never replay it.
|
|
1263
|
+
break;
|
|
1264
|
+
}
|
|
1265
|
+
await ctx.router.delay(ctx, Math.min(40, expires - performance.now()));
|
|
1266
|
+
}
|
|
1267
|
+
if (!observed)
|
|
1268
|
+
throw new BrowserActionError(
|
|
1269
|
+
"INPUT_UNCONFIRMED",
|
|
1270
|
+
"Native pointer delivery returned, but the target document did not confirm the expected trusted click event.",
|
|
1271
|
+
{ dispatched: true, effect: "unknown" },
|
|
1272
|
+
);
|
|
1273
|
+
// Let the browser process the resulting task/render turn before the next
|
|
1274
|
+
// input, without misrepresenting arbitrary asynchronous business work.
|
|
1275
|
+
const remaining = ctx.budgetMs - (performance.now() - ctx.started);
|
|
1276
|
+
if (remaining > 120) await ctx.router.delay(ctx, 120);
|
|
1277
|
+
} finally {
|
|
1278
|
+
await ctx.router
|
|
1279
|
+
.page(ctx, { ...receipt, action: "drop" })
|
|
1280
|
+
.catch(() => {});
|
|
1151
1281
|
}
|
|
1152
1282
|
return null;
|
|
1153
1283
|
}
|
|
@@ -1179,6 +1309,13 @@ route(
|
|
|
1179
1309
|
requireValue(
|
|
1180
1310
|
Number.isFinite(input.scrollX) && Number.isFinite(input.scrollY),
|
|
1181
1311
|
);
|
|
1312
|
+
// Position the real pointer along the same human trajectory before
|
|
1313
|
+
// applying the wheel at that point; never teleport the visual cursor.
|
|
1314
|
+
await ctx.router.send(ctx, "Input.dispatchMouseEvent", {
|
|
1315
|
+
type: "mouseMoved",
|
|
1316
|
+
...(await point(ctx, input)),
|
|
1317
|
+
modifiers: modifierMask(input.keypress),
|
|
1318
|
+
});
|
|
1182
1319
|
await ctx.router.send(ctx, "Input.dispatchMouseEvent", {
|
|
1183
1320
|
type: "mouseWheel",
|
|
1184
1321
|
...(await point(ctx, input)),
|
|
@@ -1298,22 +1435,105 @@ route(
|
|
|
1298
1435
|
{ needsTab: true, mutating: true },
|
|
1299
1436
|
);
|
|
1300
1437
|
async function resolveActionTarget(ctx, args) {
|
|
1301
|
-
let found;
|
|
1302
|
-
|
|
1303
|
-
//
|
|
1438
|
+
let found, previous;
|
|
1439
|
+
await readyForInput(ctx, args.locator?.framePath ?? args.framePath ?? []);
|
|
1440
|
+
// Two matching native observations give the layout a chance to settle
|
|
1441
|
+
// after scrolling. Only pre-input observations may repeat, never the click.
|
|
1304
1442
|
await ctx.router.poll(ctx, async () => {
|
|
1305
|
-
|
|
1443
|
+
const next = await ctx.router.page(ctx, {
|
|
1306
1444
|
...args,
|
|
1307
1445
|
...options(args),
|
|
1308
1446
|
command: "resolve",
|
|
1309
1447
|
});
|
|
1310
|
-
|
|
1448
|
+
if (!next) {
|
|
1449
|
+
previous = undefined;
|
|
1450
|
+
return false;
|
|
1451
|
+
}
|
|
1452
|
+
const stable =
|
|
1453
|
+
previous &&
|
|
1454
|
+
["x", "y"].every((key) =>
|
|
1455
|
+
Math.abs(previous.point[key] - next.point[key]) <= 1,
|
|
1456
|
+
) &&
|
|
1457
|
+
["x", "y", "width", "height"].every((key) =>
|
|
1458
|
+
Math.abs(previous.rect[key] - next.rect[key]) <= 1,
|
|
1459
|
+
) &&
|
|
1460
|
+
previous.documentToken === next.documentToken &&
|
|
1461
|
+
previous.frameId === next.frameId;
|
|
1462
|
+
previous = next;
|
|
1463
|
+
if (stable) found = next;
|
|
1464
|
+
return !!stable;
|
|
1311
1465
|
});
|
|
1312
1466
|
return found;
|
|
1313
1467
|
}
|
|
1314
1468
|
async function locatorClick(ctx, args, double = false) {
|
|
1315
1469
|
const found = await resolveActionTarget(ctx, args);
|
|
1316
|
-
|
|
1470
|
+
const input = options(args),
|
|
1471
|
+
expect = input.expect,
|
|
1472
|
+
framePath = args.locator?.framePath ?? args.framePath ?? [];
|
|
1473
|
+
// The native navigation event must be armed before input; top-level only.
|
|
1474
|
+
requireValue(
|
|
1475
|
+
!(expect?.kind === "navigation" && framePath.length),
|
|
1476
|
+
"INVALID_ARGUMENT",
|
|
1477
|
+
"For a nested frame, wait for its URL or a frame-local page condition.",
|
|
1478
|
+
);
|
|
1479
|
+
let navigation;
|
|
1480
|
+
if (expect?.kind === "navigation")
|
|
1481
|
+
navigation = await ctx.router.armWaiter(ctx, {
|
|
1482
|
+
event: "navigation",
|
|
1483
|
+
url: expect.url,
|
|
1484
|
+
waitUntil: expect.waitUntil ?? "load",
|
|
1485
|
+
timeoutMs:
|
|
1486
|
+
expect.timeoutMs ??
|
|
1487
|
+
Math.min(15000, ctx.budgetMs - (performance.now() - ctx.started)),
|
|
1488
|
+
});
|
|
1489
|
+
try {
|
|
1490
|
+
await click(
|
|
1491
|
+
ctx,
|
|
1492
|
+
{
|
|
1493
|
+
...input,
|
|
1494
|
+
...found.point,
|
|
1495
|
+
resolvedTarget: true,
|
|
1496
|
+
framePath,
|
|
1497
|
+
framePoint: found.framePoint,
|
|
1498
|
+
frameId: found.frameId,
|
|
1499
|
+
},
|
|
1500
|
+
double,
|
|
1501
|
+
);
|
|
1502
|
+
if (!expect) return null;
|
|
1503
|
+
try {
|
|
1504
|
+
if (navigation) {
|
|
1505
|
+
await ctx.router.getWaiter(ctx, navigation.id);
|
|
1506
|
+
return {
|
|
1507
|
+
matched: true,
|
|
1508
|
+
snapshot: {
|
|
1509
|
+
...ctx.router.target(ctx.target.tabId, ctx.lease),
|
|
1510
|
+
...(await ctx.router.page(ctx, { command: "snapshot" })),
|
|
1511
|
+
},
|
|
1512
|
+
};
|
|
1513
|
+
}
|
|
1514
|
+
return await ctx.router.waitForCondition(ctx, {
|
|
1515
|
+
condition: expect,
|
|
1516
|
+
framePath,
|
|
1517
|
+
});
|
|
1518
|
+
} catch (cause) {
|
|
1519
|
+
ctx.signal?.throwIfAborted();
|
|
1520
|
+
if (
|
|
1521
|
+
cause instanceof BrowserActionError &&
|
|
1522
|
+
["TIMEOUT", "STALE_REF", "INVALID_TARGET", "WAITER_GONE"].includes(
|
|
1523
|
+
cause.code,
|
|
1524
|
+
)
|
|
1525
|
+
)
|
|
1526
|
+
throw new BrowserActionError(
|
|
1527
|
+
"EXPECTATION_UNCONFIRMED",
|
|
1528
|
+
"The page did not confirm the expected result. Input was already sent; observe before taking further action.",
|
|
1529
|
+
{ dispatched: true, effect: "unknown" },
|
|
1530
|
+
);
|
|
1531
|
+
throw cause;
|
|
1532
|
+
}
|
|
1533
|
+
} finally {
|
|
1534
|
+
if (navigation)
|
|
1535
|
+
await ctx.router.cancelWaiter(ctx, navigation.id).catch(() => {});
|
|
1536
|
+
}
|
|
1317
1537
|
}
|
|
1318
1538
|
route("locator.click", (ctx, args) => locatorClick(ctx, args), {
|
|
1319
1539
|
needsTab: true,
|
|
@@ -1684,6 +1904,7 @@ export function createBrowserRouter(
|
|
|
1684
1904
|
journalTail = Promise.resolve();
|
|
1685
1905
|
const versions = new Map(),
|
|
1686
1906
|
documents = new Map(),
|
|
1907
|
+
pointerPositions = new Map(),
|
|
1687
1908
|
leases = new Map(),
|
|
1688
1909
|
closedLeases = new Map(),
|
|
1689
1910
|
owners = new Map(),
|
|
@@ -2255,11 +2476,51 @@ export function createBrowserRouter(
|
|
|
2255
2476
|
throw cause;
|
|
2256
2477
|
}
|
|
2257
2478
|
};
|
|
2258
|
-
|
|
2479
|
+
// Chrome may hold a native command indefinitely while a JavaScript dialog
|
|
2480
|
+
// pauses its renderer. A cancelled action must not hold the run's pending
|
|
2481
|
+
// queue forever; the native command may still have an unknown page effect,
|
|
2482
|
+
// so actual debugger detach remains a separate required cleanup step.
|
|
2483
|
+
const nativeWithStop = async (signal, invoke) => {
|
|
2484
|
+
signal?.throwIfAborted();
|
|
2485
|
+
const native = Promise.resolve().then(() => {
|
|
2486
|
+
signal?.throwIfAborted();
|
|
2487
|
+
return invoke();
|
|
2488
|
+
});
|
|
2489
|
+
if (!signal) return native;
|
|
2490
|
+
let onAbort;
|
|
2491
|
+
const cancelled = new Promise((_, reject) => {
|
|
2492
|
+
onAbort = () => reject(signal.reason);
|
|
2493
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
2494
|
+
});
|
|
2495
|
+
try {
|
|
2496
|
+
if (signal.aborted) onAbort();
|
|
2497
|
+
return await Promise.race([native, cancelled]);
|
|
2498
|
+
} finally {
|
|
2499
|
+
signal.removeEventListener("abort", onAbort);
|
|
2500
|
+
}
|
|
2501
|
+
};
|
|
2502
|
+
const releaseControl = async (
|
|
2503
|
+
lease,
|
|
2504
|
+
tabId,
|
|
2505
|
+
{ detach = true, dismissDialog = false } = {},
|
|
2506
|
+
) => {
|
|
2259
2507
|
if (!lease.attached.has(tabId) && !lease.intents.has(tabId)) return;
|
|
2260
2508
|
for (const waiter of waiters.values())
|
|
2261
2509
|
if (waiter.leaseId === lease.id && waiter.tabId === tabId)
|
|
2262
2510
|
cancelWaiter(waiter);
|
|
2511
|
+
// An observed JavaScript dialog pauses its renderer. Dismiss the dialog
|
|
2512
|
+
// for a terminating run before other native cleanup commands; explicit
|
|
2513
|
+
// handoff keeps the dialog available for the user instead.
|
|
2514
|
+
const dialog = dialogs.get(String(tabId));
|
|
2515
|
+
if (dismissDialog && dialog && lease.attached.has(tabId)) {
|
|
2516
|
+
await chrome.debugger.sendCommand(
|
|
2517
|
+
{ tabId },
|
|
2518
|
+
"Page.handleJavaScriptDialog",
|
|
2519
|
+
{ accept: false },
|
|
2520
|
+
);
|
|
2521
|
+
if (dialogs.get(String(tabId)) === dialog)
|
|
2522
|
+
dialogs.delete(String(tabId));
|
|
2523
|
+
}
|
|
2263
2524
|
await stopLocalDownloadClick(lease, tabId);
|
|
2264
2525
|
if (lease.attached.has(tabId)) await syncDownloadIntercept(lease, tabId);
|
|
2265
2526
|
if (!lease.attached.has(tabId) && !lease.externallyDetached.has(tabId)) {
|
|
@@ -2522,6 +2783,7 @@ export function createBrowserRouter(
|
|
|
2522
2783
|
finishNavigation(details, "load"),
|
|
2523
2784
|
);
|
|
2524
2785
|
on(chrome.tabs.onRemoved, (id) => {
|
|
2786
|
+
pointerPositions.delete(id);
|
|
2525
2787
|
versions.set(String(id), version(id) + 1);
|
|
2526
2788
|
dialogs.delete(String(id));
|
|
2527
2789
|
emit({
|
|
@@ -3081,7 +3343,7 @@ export function createBrowserRouter(
|
|
|
3081
3343
|
.filter((id) => id !== undefined),
|
|
3082
3344
|
))
|
|
3083
3345
|
try {
|
|
3084
|
-
await releaseControl(lease, tabId);
|
|
3346
|
+
await releaseControl(lease, tabId, { dismissDialog: true });
|
|
3085
3347
|
} catch (cause) {
|
|
3086
3348
|
tempErrors.push(cause);
|
|
3087
3349
|
}
|
|
@@ -3100,7 +3362,7 @@ export function createBrowserRouter(
|
|
|
3100
3362
|
}
|
|
3101
3363
|
for (const tabId of new Set([...lease.attached, ...lease.intents])) {
|
|
3102
3364
|
try {
|
|
3103
|
-
await releaseControl(lease, tabId);
|
|
3365
|
+
await releaseControl(lease, tabId, { dismissDialog: true });
|
|
3104
3366
|
} catch (cause) {
|
|
3105
3367
|
errors.push(cause);
|
|
3106
3368
|
}
|
|
@@ -3331,6 +3593,14 @@ export function createBrowserRouter(
|
|
|
3331
3593
|
].includes(input.method)
|
|
3332
3594
|
)
|
|
3333
3595
|
await router.prepareVisual(ctx);
|
|
3596
|
+
if (
|
|
3597
|
+
visual &&
|
|
3598
|
+
definition.needsTab &&
|
|
3599
|
+
!definition.mutating &&
|
|
3600
|
+
input.target.tabId !== undefined &&
|
|
3601
|
+
!lease.temporaryTabs?.has(nativeId(input.target.tabId))
|
|
3602
|
+
)
|
|
3603
|
+
await router.trackVisualTab(ctx);
|
|
3334
3604
|
actionSignal.throwIfAborted();
|
|
3335
3605
|
managementHandling = true;
|
|
3336
3606
|
value = await definition.handle(ctx, input.args ?? {});
|
|
@@ -3367,9 +3637,14 @@ export function createBrowserRouter(
|
|
|
3367
3637
|
"tabs.closeTemporary",
|
|
3368
3638
|
"tab.getJsDialog",
|
|
3369
3639
|
].includes(input.method);
|
|
3640
|
+
// Native mutations can activate a tab and send shared browser input.
|
|
3641
|
+
// A per-tab queue cannot protect two simultaneous actions that switch
|
|
3642
|
+
// the active tab underneath one another. Reads and event waiters remain
|
|
3643
|
+
// independently schedulable.
|
|
3370
3644
|
const queueKey =
|
|
3371
|
-
|
|
3372
|
-
|
|
3645
|
+
definition.mutating && !nonBlocking
|
|
3646
|
+
? `native-mutation:${input.target.browserId}:${input.target.browserSessionId}`
|
|
3647
|
+
: input.target.tabId;
|
|
3373
3648
|
let task;
|
|
3374
3649
|
if (queueKey && !nonBlocking) {
|
|
3375
3650
|
task = (queues.get(queueKey) ?? Promise.resolve()).then(perform);
|
|
@@ -3671,12 +3946,14 @@ export function createBrowserRouter(
|
|
|
3671
3946
|
const framePath = payload.locator?.framePath ?? payload.framePath ?? [];
|
|
3672
3947
|
if (framePath.length > 0) return framePage(ctx, payload, framePath);
|
|
3673
3948
|
ctx.dispatched();
|
|
3674
|
-
const result = await
|
|
3675
|
-
|
|
3676
|
-
|
|
3677
|
-
|
|
3678
|
-
|
|
3679
|
-
|
|
3949
|
+
const result = await nativeWithStop(ctx.signal, () =>
|
|
3950
|
+
chrome.scripting.executeScript({
|
|
3951
|
+
target: { tabId: nativeId(ctx.target.tabId) },
|
|
3952
|
+
world: "ISOLATED",
|
|
3953
|
+
func: pageCommand,
|
|
3954
|
+
args: [input],
|
|
3955
|
+
}),
|
|
3956
|
+
);
|
|
3680
3957
|
// A main-frame replacement can finish an injection with no result. Only
|
|
3681
3958
|
// a readiness observation may wait for a browser-confirmed transition;
|
|
3682
3959
|
// actions and empty results from settled pages must still fail.
|
|
@@ -3865,6 +4142,17 @@ export function createBrowserRouter(
|
|
|
3865
4142
|
ctx.dispatched();
|
|
3866
4143
|
await chrome.debugger.attach({ tabId: id }, "1.3");
|
|
3867
4144
|
ctx.lease.attached.add(id);
|
|
4145
|
+
// Observe Page.javascriptDialogOpening before *any* page action can
|
|
4146
|
+
// synchronously open a prompt/confirm. Waiting until getJsDialog is
|
|
4147
|
+
// called loses the event while Runtime.evaluate may remain blocked.
|
|
4148
|
+
try {
|
|
4149
|
+
await nativeWithStop(ctx.signal, () =>
|
|
4150
|
+
chrome.debugger.sendCommand({ tabId: id }, "Page.enable", {}),
|
|
4151
|
+
);
|
|
4152
|
+
} catch (cause) {
|
|
4153
|
+
await releaseControl(ctx.lease, id).catch(() => {});
|
|
4154
|
+
throw cause;
|
|
4155
|
+
}
|
|
3868
4156
|
if (ctx.lease.ended) {
|
|
3869
4157
|
await releaseControl(ctx.lease, id);
|
|
3870
4158
|
throw new BrowserActionError(
|
|
@@ -3880,9 +4168,36 @@ export function createBrowserRouter(
|
|
|
3880
4168
|
ctx.lease.attaching.delete(id);
|
|
3881
4169
|
}
|
|
3882
4170
|
},
|
|
3883
|
-
async send(ctx, method, params, child) {
|
|
4171
|
+
async send(ctx, method, params, child, nativePointerStep = false) {
|
|
3884
4172
|
active(ctx.lease);
|
|
3885
4173
|
ctx.signal?.throwIfAborted();
|
|
4174
|
+
// Every high-level mouse move, click, hover and drag shares the same
|
|
4175
|
+
// physical trajectory. Raw cdp.send remains exact and unmodified.
|
|
4176
|
+
if (
|
|
4177
|
+
!nativePointerStep &&
|
|
4178
|
+
!child &&
|
|
4179
|
+
ctx.method !== "cdp.send" &&
|
|
4180
|
+
method === "Input.dispatchMouseEvent" &&
|
|
4181
|
+
params.type === "mouseMoved" &&
|
|
4182
|
+
Number.isFinite(params.x) &&
|
|
4183
|
+
Number.isFinite(params.y)
|
|
4184
|
+
) {
|
|
4185
|
+
const tabId = nativeId(ctx.target.tabId),
|
|
4186
|
+
previous = pointerPositions.get(tabId) ?? { x: 32, y: 72 },
|
|
4187
|
+
path = humanPointerPath(previous, { x: params.x, y: params.y });
|
|
4188
|
+
let result;
|
|
4189
|
+
for (const step of path) {
|
|
4190
|
+
result = await router.send(
|
|
4191
|
+
ctx,
|
|
4192
|
+
method,
|
|
4193
|
+
{ ...params, x: step.x, y: step.y },
|
|
4194
|
+
child,
|
|
4195
|
+
true,
|
|
4196
|
+
);
|
|
4197
|
+
if (step.delayMs > 0) await router.delay(ctx, step.delayMs);
|
|
4198
|
+
}
|
|
4199
|
+
return result;
|
|
4200
|
+
}
|
|
3886
4201
|
await router.attach(ctx);
|
|
3887
4202
|
ctx.signal?.throwIfAborted();
|
|
3888
4203
|
if (
|
|
@@ -4035,7 +4350,16 @@ export function createBrowserRouter(
|
|
|
4035
4350
|
ctx.pendingInfo.nativeNetworkTabId = tabId;
|
|
4036
4351
|
let result;
|
|
4037
4352
|
try {
|
|
4038
|
-
|
|
4353
|
+
// A native Runtime.evaluate can stay suspended behind a modal dialog
|
|
4354
|
+
// after its action is cancelled. Other native commands (notably IO.read
|
|
4355
|
+
// and IO.close) must retain their actual settlement: dropping their
|
|
4356
|
+
// replies early can abandon a native stream or misreport cleanup.
|
|
4357
|
+
result =
|
|
4358
|
+
method === "Runtime.evaluate"
|
|
4359
|
+
? await nativeWithStop(ctx.signal, () =>
|
|
4360
|
+
chrome.debugger.sendCommand(source, method, params),
|
|
4361
|
+
)
|
|
4362
|
+
: await chrome.debugger.sendCommand(source, method, params);
|
|
4039
4363
|
} catch (cause) {
|
|
4040
4364
|
if (armedLocalDownload) await stopLocalDownloadClick(ctx.lease, tabId);
|
|
4041
4365
|
throw cause;
|
|
@@ -4044,6 +4368,14 @@ export function createBrowserRouter(
|
|
|
4044
4368
|
delete ctx.pendingInfo.nativeNetworkTabId;
|
|
4045
4369
|
if (watchEnd) await stopLocalDownloadClick(ctx.lease, tabId);
|
|
4046
4370
|
}
|
|
4371
|
+
if (
|
|
4372
|
+
!child &&
|
|
4373
|
+
method === "Input.dispatchMouseEvent" &&
|
|
4374
|
+
params.type === "mouseMoved" &&
|
|
4375
|
+
Number.isFinite(params.x) &&
|
|
4376
|
+
Number.isFinite(params.y)
|
|
4377
|
+
)
|
|
4378
|
+
pointerPositions.set(tabId, { x: params.x, y: params.y });
|
|
4047
4379
|
if (rawFetch && method === "Fetch.disable") {
|
|
4048
4380
|
ctx.lease.rawFetch?.delete(tabId);
|
|
4049
4381
|
await syncDownloadIntercept(ctx.lease, tabId);
|
|
@@ -4874,6 +5206,9 @@ export function createBrowserRouter(
|
|
|
4874
5206
|
},
|
|
4875
5207
|
async screenshot(ctx, input) {
|
|
4876
5208
|
requireValue(!(input.fullPage && input.clip));
|
|
5209
|
+
await readyForInput(ctx);
|
|
5210
|
+
const readyBudget = ctx.budgetMs - (performance.now() - ctx.started);
|
|
5211
|
+
if (readyBudget > 130) await router.delay(ctx, 120);
|
|
4877
5212
|
const quality = input.quality ?? 80,
|
|
4878
5213
|
maxEdgePx = input.maxEdgePx ?? 1920;
|
|
4879
5214
|
requireValue(
|
|
@@ -5308,6 +5643,22 @@ export function createBrowserRouter(
|
|
|
5308
5643
|
});
|
|
5309
5644
|
}
|
|
5310
5645
|
},
|
|
5646
|
+
async trackVisualTab(ctx) {
|
|
5647
|
+
try {
|
|
5648
|
+
await visual?.trackTab(router.visualContext(ctx));
|
|
5649
|
+
} catch {
|
|
5650
|
+
ctx.signal?.throwIfAborted();
|
|
5651
|
+
emit({
|
|
5652
|
+
type: "event",
|
|
5653
|
+
event: "visual_degraded",
|
|
5654
|
+
leaseId: ctx.lease.id,
|
|
5655
|
+
runId: ctx.runId,
|
|
5656
|
+
operationId: ctx.operationId,
|
|
5657
|
+
target: ctx.target,
|
|
5658
|
+
reason: "VISUAL_UNAVAILABLE",
|
|
5659
|
+
});
|
|
5660
|
+
}
|
|
5661
|
+
},
|
|
5311
5662
|
async preparePointer(ctx, point) {
|
|
5312
5663
|
if (
|
|
5313
5664
|
ctx.lastVisualPoint?.x === point.x &&
|