@nexum-hub/browser 0.0.3-next.8 → 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/browser-extension/build.mjs +4 -1
- package/browser-extension/connection.js +41 -5
- package/browser-extension/dist/development/connection.js +41 -5
- package/browser-extension/dist/development/frames.js +11 -0
- package/browser-extension/dist/development/manifest.json +1 -1
- package/browser-extension/dist/development/page.js +115 -8
- package/browser-extension/dist/development/router.js +468 -46
- package/browser-extension/dist/development/visual.js +145 -88
- package/browser-extension/dist/production/connection.js +41 -5
- package/browser-extension/dist/production/frames.js +11 -0
- package/browser-extension/dist/production/manifest.json +1 -1
- package/browser-extension/dist/production/page.js +115 -8
- package/browser-extension/dist/production/router.js +468 -46
- package/browser-extension/dist/production/visual.js +145 -88
- package/browser-extension/frames.js +11 -0
- package/browser-extension/page.js +115 -8
- package/browser-extension/release-info.json +5 -3
- package/browser-extension/router.js +468 -46
- package/browser-extension/visual.js +145 -88
- package/dist/client/index.js +25 -1
- package/dist/client/routes.js +5 -1
- package/dist/client/types.d.ts +20 -0
- package/dist/extension/assets.d.ts +1 -0
- package/dist/extension/assets.js +3 -1
- package/dist/extension/index.js +6 -6
- package/dist/extension/scope.d.ts +3 -1
- package/dist/extension/scope.js +19 -3
- package/dist/extension/service.d.ts +2 -0
- package/dist/extension/service.js +18 -8
- package/docs/README.md +6 -8
- package/docs/connection.md +1 -1
- package/docs/usage.md +51 -6
- package/implementation.json +1 -1
- package/instructions/README.md +5 -7
- package/package.json +6 -5
- package/skills/browser-control/SKILL.md +11 -5
- package/dist/client/index.js.map +0 -1
- package/dist/client/routes.js.map +0 -1
- package/dist/client/serialize.js.map +0 -1
- package/dist/client/types.js.map +0 -1
- package/dist/extension/artifacts.js.map +0 -1
- package/dist/extension/assets.js.map +0 -1
- package/dist/extension/batch.js.map +0 -1
- package/dist/extension/files.js.map +0 -1
- package/dist/extension/index.js.map +0 -1
- package/dist/extension/protocol.js.map +0 -1
- package/dist/extension/scope.js.map +0 -1
- package/dist/extension/service.js.map +0 -1
|
@@ -371,14 +371,15 @@ route("tabs.get", async (ctx, args) =>
|
|
|
371
371
|
);
|
|
372
372
|
route(
|
|
373
373
|
"tabs.new",
|
|
374
|
-
async (ctx, args) =>
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
374
|
+
async (ctx, args) => {
|
|
375
|
+
const tab = await ctx.chrome.tabs.create({
|
|
376
|
+
...(args.url === undefined ? {} : { url: httpUrl(args.url) }),
|
|
377
|
+
active: options(args).active ?? true,
|
|
378
|
+
});
|
|
379
|
+
const info = tabInfo(ctx, tab);
|
|
380
|
+
await ctx.router.trackVisualTab({ ...ctx, target: info.target });
|
|
381
|
+
return info;
|
|
382
|
+
},
|
|
382
383
|
{ mutating: true },
|
|
383
384
|
);
|
|
384
385
|
route("tabs.selected", async (ctx) => {
|
|
@@ -394,6 +395,10 @@ route(
|
|
|
394
395
|
const id = tabIdOf(ctx, args);
|
|
395
396
|
ctx.router.claim(ctx.lease, id);
|
|
396
397
|
await ctx.chrome.tabs.update(id, { active: true });
|
|
398
|
+
await ctx.router.trackVisualTab({
|
|
399
|
+
...ctx,
|
|
400
|
+
target: ctx.router.target(String(id), ctx.lease),
|
|
401
|
+
});
|
|
397
402
|
return ctx.router.actionResult(ctx, id);
|
|
398
403
|
},
|
|
399
404
|
{ mutating: true },
|
|
@@ -1121,33 +1126,202 @@ async function point(ctx, args) {
|
|
|
1121
1126
|
requireValue(Number.isFinite(args.x) && Number.isFinite(args.y));
|
|
1122
1127
|
return { x: args.x, y: args.y };
|
|
1123
1128
|
}
|
|
1129
|
+
|
|
1130
|
+
async function readyForInput(ctx, framePath = []) {
|
|
1131
|
+
const ready = await ctx.router.page(ctx, {
|
|
1132
|
+
command: "readyState",
|
|
1133
|
+
framePath,
|
|
1134
|
+
});
|
|
1135
|
+
if (ready === "loading" || ready === "interactive") {
|
|
1136
|
+
const remaining = ctx.budgetMs - (performance.now() - ctx.started);
|
|
1137
|
+
requireValue(remaining > 0, "TIMEOUT", "The page did not become ready.");
|
|
1138
|
+
if (framePath.length)
|
|
1139
|
+
await ctx.router.poll(ctx, async () =>
|
|
1140
|
+
(await ctx.router.page(ctx, { command: "readyState", framePath })) ===
|
|
1141
|
+
"complete",
|
|
1142
|
+
);
|
|
1143
|
+
else
|
|
1144
|
+
await ctx.router.waitForLoadState(ctx, {
|
|
1145
|
+
state: "load",
|
|
1146
|
+
timeoutMs: remaining,
|
|
1147
|
+
});
|
|
1148
|
+
}
|
|
1149
|
+
}
|
|
1150
|
+
|
|
1151
|
+
// Build the points for real Chrome Input.dispatchMouseEvent calls, not a
|
|
1152
|
+
// decorative animation that runs independently of the actual input.
|
|
1153
|
+
export function humanPointerPath(from, to, maxSteps = 28) {
|
|
1154
|
+
const dx = to.x - from.x,
|
|
1155
|
+
dy = to.y - from.y,
|
|
1156
|
+
distance = Math.hypot(dx, dy);
|
|
1157
|
+
if (distance < 12) return [{ x: to.x, y: to.y, delayMs: 0 }];
|
|
1158
|
+
const steps = Math.min(maxSteps, Math.max(3, Math.ceil(distance / 28))),
|
|
1159
|
+
durationMs = Math.min(420, Math.max(110, 85 + distance * 0.35)),
|
|
1160
|
+
bend = Math.min(18, distance * 0.08) *
|
|
1161
|
+
(Math.sin(from.x * 0.013 + to.y * 0.017) < 0 ? -1 : 1);
|
|
1162
|
+
const points = [];
|
|
1163
|
+
for (let i = 1; i <= steps; i++) {
|
|
1164
|
+
const t = i / steps,
|
|
1165
|
+
eased = t * t * (3 - 2 * t),
|
|
1166
|
+
arc = Math.sin(Math.PI * t) * bend;
|
|
1167
|
+
points.push({
|
|
1168
|
+
x: i === steps ? to.x : from.x + dx * eased - (dy / distance) * arc,
|
|
1169
|
+
y: i === steps ? to.y : from.y + dy * eased + (dx / distance) * arc,
|
|
1170
|
+
delayMs: i === steps ? 0 : durationMs / steps,
|
|
1171
|
+
});
|
|
1172
|
+
}
|
|
1173
|
+
return points;
|
|
1174
|
+
}
|
|
1124
1175
|
async function click(ctx, args, double = false) {
|
|
1125
|
-
|
|
1126
|
-
|
|
1176
|
+
if (!args.resolvedTarget) await readyForInput(ctx);
|
|
1177
|
+
let p = await point(ctx, args);
|
|
1178
|
+
const button = buttons[args.button ?? "left"];
|
|
1127
1179
|
requireValue(button);
|
|
1128
1180
|
const count = double ? 2 : (args.clickCount ?? 1);
|
|
1129
1181
|
requireValue(Number.isInteger(count) && count >= 1 && count <= 3);
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
});
|
|
1135
|
-
for (let i = 1; i <= count; i++) {
|
|
1182
|
+
const framePath = args.framePath ?? [];
|
|
1183
|
+
const originalVersion = ctx.router.version(ctx.target.tabId);
|
|
1184
|
+
let receipt;
|
|
1185
|
+
try {
|
|
1136
1186
|
await ctx.router.send(ctx, "Input.dispatchMouseEvent", {
|
|
1137
|
-
type: "
|
|
1187
|
+
type: "mouseMoved",
|
|
1138
1188
|
...p,
|
|
1139
|
-
button,
|
|
1140
|
-
clickCount: i,
|
|
1141
|
-
modifiers: modifierMask(args.modifiers ?? args.keypress),
|
|
1189
|
+
button: "none",
|
|
1142
1190
|
});
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1191
|
+
let framePoint = args.framePoint;
|
|
1192
|
+
if (args.expectedRef) {
|
|
1193
|
+
// The first native pointer movement can cause Chromium's visible
|
|
1194
|
+
// viewport to change. Re-observe the same element after that movement,
|
|
1195
|
+
// before the first mouse press; no click is ever replayed.
|
|
1196
|
+
let stable = false;
|
|
1197
|
+
for (let attempt = 0; attempt < 4; attempt++) {
|
|
1198
|
+
const current = await ctx.router.page(ctx, {
|
|
1199
|
+
command: "resolve",
|
|
1200
|
+
ref: args.expectedRef,
|
|
1201
|
+
framePath,
|
|
1202
|
+
noScroll: true,
|
|
1203
|
+
...(args.position ? { position: args.position } : {}),
|
|
1204
|
+
});
|
|
1205
|
+
requireValue(
|
|
1206
|
+
current && (!args.frameId || current.frameId === args.frameId),
|
|
1207
|
+
"INVALID_TARGET",
|
|
1208
|
+
"The original click element is no longer at a controllable point.",
|
|
1209
|
+
);
|
|
1210
|
+
if (
|
|
1211
|
+
Math.abs(current.point.x - p.x) <= 1 &&
|
|
1212
|
+
Math.abs(current.point.y - p.y) <= 1
|
|
1213
|
+
) {
|
|
1214
|
+
framePoint = current.framePoint ?? framePoint;
|
|
1215
|
+
stable = true;
|
|
1216
|
+
break;
|
|
1217
|
+
}
|
|
1218
|
+
p = current.point;
|
|
1219
|
+
framePoint = current.framePoint ?? framePoint;
|
|
1220
|
+
await ctx.router.send(ctx, "Input.dispatchMouseEvent", {
|
|
1221
|
+
type: "mouseMoved",
|
|
1222
|
+
...p,
|
|
1223
|
+
button: "none",
|
|
1224
|
+
});
|
|
1225
|
+
}
|
|
1226
|
+
requireValue(
|
|
1227
|
+
stable,
|
|
1228
|
+
"INVALID_TARGET",
|
|
1229
|
+
"The click target continued moving after native pointer preparation.",
|
|
1230
|
+
);
|
|
1231
|
+
}
|
|
1232
|
+
const receiptInput = {
|
|
1233
|
+
command: "inputReceipt",
|
|
1234
|
+
token: ctx.actionId,
|
|
1235
|
+
x: framePoint?.x ?? p.x,
|
|
1236
|
+
y: framePoint?.y ?? p.y,
|
|
1237
|
+
button: { left: 0, middle: 1, right: 2 }[args.button ?? "left"],
|
|
1238
|
+
expiresAt: Date.now() + Math.min(3000, ctx.budgetMs),
|
|
1239
|
+
framePath,
|
|
1240
|
+
...(args.frameId ? { expectedFrameId: args.frameId } : {}),
|
|
1241
|
+
...(args.expectedRef ? { expectedRef: args.expectedRef } : {}),
|
|
1242
|
+
};
|
|
1243
|
+
const armed = await ctx.router.page(ctx, {
|
|
1244
|
+
...receiptInput,
|
|
1245
|
+
action: "arm",
|
|
1150
1246
|
});
|
|
1247
|
+
requireValue(
|
|
1248
|
+
armed?.armed === true && typeof armed.documentToken === "string",
|
|
1249
|
+
"INVALID_TARGET",
|
|
1250
|
+
"The browser could not arm a receipt for the exact input document.",
|
|
1251
|
+
);
|
|
1252
|
+
receipt = {
|
|
1253
|
+
...receiptInput,
|
|
1254
|
+
expectedDocumentToken: armed.documentToken,
|
|
1255
|
+
};
|
|
1256
|
+
for (let i = 1; i <= count; i++) {
|
|
1257
|
+
await ctx.router.send(ctx, "Input.dispatchMouseEvent", {
|
|
1258
|
+
type: "mousePressed",
|
|
1259
|
+
...p,
|
|
1260
|
+
button,
|
|
1261
|
+
clickCount: i,
|
|
1262
|
+
modifiers: modifierMask(args.modifiers ?? args.keypress),
|
|
1263
|
+
});
|
|
1264
|
+
if (args.duration > 0) await ctx.router.delay(ctx, args.duration * 1000);
|
|
1265
|
+
await ctx.router.send(ctx, "Input.dispatchMouseEvent", {
|
|
1266
|
+
type: "mouseReleased",
|
|
1267
|
+
...p,
|
|
1268
|
+
button,
|
|
1269
|
+
clickCount: i,
|
|
1270
|
+
modifiers: modifierMask(args.modifiers ?? args.keypress),
|
|
1271
|
+
});
|
|
1272
|
+
}
|
|
1273
|
+
let observed = false;
|
|
1274
|
+
const expires = Math.min(
|
|
1275
|
+
performance.now() + 450,
|
|
1276
|
+
ctx.started + ctx.budgetMs,
|
|
1277
|
+
);
|
|
1278
|
+
while (performance.now() < expires) {
|
|
1279
|
+
if (
|
|
1280
|
+
!framePath.length &&
|
|
1281
|
+
ctx.router.version(ctx.target.tabId) !== originalVersion
|
|
1282
|
+
) {
|
|
1283
|
+
observed = true; // The main document navigated after input.
|
|
1284
|
+
break;
|
|
1285
|
+
}
|
|
1286
|
+
try {
|
|
1287
|
+
const result = await ctx.router.page(ctx, {
|
|
1288
|
+
...receipt,
|
|
1289
|
+
action: "peek",
|
|
1290
|
+
});
|
|
1291
|
+
if (result.observed >= count) {
|
|
1292
|
+
observed = true;
|
|
1293
|
+
break;
|
|
1294
|
+
}
|
|
1295
|
+
} catch {
|
|
1296
|
+
ctx.signal?.throwIfAborted();
|
|
1297
|
+
if (
|
|
1298
|
+
!framePath.length &&
|
|
1299
|
+
ctx.router.version(ctx.target.tabId) !== originalVersion
|
|
1300
|
+
) {
|
|
1301
|
+
observed = true;
|
|
1302
|
+
break;
|
|
1303
|
+
}
|
|
1304
|
+
// A detached or replaced frame does not prove the target received
|
|
1305
|
+
// input. The action may still have happened, so never replay it.
|
|
1306
|
+
break;
|
|
1307
|
+
}
|
|
1308
|
+
await ctx.router.delay(ctx, Math.min(40, expires - performance.now()));
|
|
1309
|
+
}
|
|
1310
|
+
if (!observed)
|
|
1311
|
+
throw new BrowserActionError(
|
|
1312
|
+
"INPUT_UNCONFIRMED",
|
|
1313
|
+
"Native pointer delivery returned, but the target document did not confirm the expected trusted click event.",
|
|
1314
|
+
{ dispatched: true, effect: "unknown" },
|
|
1315
|
+
);
|
|
1316
|
+
// Let the browser process the resulting task/render turn before the next
|
|
1317
|
+
// input, without misrepresenting arbitrary asynchronous business work.
|
|
1318
|
+
const remaining = ctx.budgetMs - (performance.now() - ctx.started);
|
|
1319
|
+
if (remaining > 120) await ctx.router.delay(ctx, 120);
|
|
1320
|
+
} finally {
|
|
1321
|
+
if (receipt)
|
|
1322
|
+
await ctx.router
|
|
1323
|
+
.page(ctx, { ...receipt, action: "drop" })
|
|
1324
|
+
.catch(() => {});
|
|
1151
1325
|
}
|
|
1152
1326
|
return null;
|
|
1153
1327
|
}
|
|
@@ -1179,6 +1353,13 @@ route(
|
|
|
1179
1353
|
requireValue(
|
|
1180
1354
|
Number.isFinite(input.scrollX) && Number.isFinite(input.scrollY),
|
|
1181
1355
|
);
|
|
1356
|
+
// Position the real pointer along the same human trajectory before
|
|
1357
|
+
// applying the wheel at that point; never teleport the visual cursor.
|
|
1358
|
+
await ctx.router.send(ctx, "Input.dispatchMouseEvent", {
|
|
1359
|
+
type: "mouseMoved",
|
|
1360
|
+
...(await point(ctx, input)),
|
|
1361
|
+
modifiers: modifierMask(input.keypress),
|
|
1362
|
+
});
|
|
1182
1363
|
await ctx.router.send(ctx, "Input.dispatchMouseEvent", {
|
|
1183
1364
|
type: "mouseWheel",
|
|
1184
1365
|
...(await point(ctx, input)),
|
|
@@ -1298,22 +1479,122 @@ route(
|
|
|
1298
1479
|
{ needsTab: true, mutating: true },
|
|
1299
1480
|
);
|
|
1300
1481
|
async function resolveActionTarget(ctx, args) {
|
|
1301
|
-
let found;
|
|
1302
|
-
|
|
1303
|
-
//
|
|
1482
|
+
let found, previous;
|
|
1483
|
+
let scrollRequested = false;
|
|
1484
|
+
// Chromium can change the page viewport when its native debugger attaches
|
|
1485
|
+
// (notably by adding a debugging infobar). Attach before measuring or
|
|
1486
|
+
// scrolling the target; attaching only on the first mouse event can shift
|
|
1487
|
+
// the page after preflight and send a trusted click to the wrong element.
|
|
1488
|
+
await ctx.router.attach(ctx);
|
|
1489
|
+
await readyForInput(ctx, args.locator?.framePath ?? args.framePath ?? []);
|
|
1490
|
+
const sameLayout = (left, right) =>
|
|
1491
|
+
left && right &&
|
|
1492
|
+
["x", "y"].every((key) =>
|
|
1493
|
+
Math.abs(left.point[key] - right.point[key]) <= 1,
|
|
1494
|
+
) &&
|
|
1495
|
+
["x", "y", "width", "height"].every((key) =>
|
|
1496
|
+
Math.abs(left.rect[key] - right.rect[key]) <= 1,
|
|
1497
|
+
) &&
|
|
1498
|
+
left.ref === right.ref &&
|
|
1499
|
+
left.documentToken === right.documentToken &&
|
|
1500
|
+
left.frameId === right.frameId;
|
|
1501
|
+
// Two matching native observations give the layout a chance to settle
|
|
1502
|
+
// after scrolling. Only pre-input observations may repeat, never the click.
|
|
1304
1503
|
await ctx.router.poll(ctx, async () => {
|
|
1305
|
-
|
|
1504
|
+
const next = await ctx.router.page(ctx, {
|
|
1306
1505
|
...args,
|
|
1307
1506
|
...options(args),
|
|
1308
1507
|
command: "resolve",
|
|
1508
|
+
...(scrollRequested ? { noScroll: true } : {}),
|
|
1309
1509
|
});
|
|
1310
|
-
|
|
1510
|
+
if (!next) {
|
|
1511
|
+
previous = undefined;
|
|
1512
|
+
return false;
|
|
1513
|
+
}
|
|
1514
|
+
// A missing target did not establish a usable scroll position. If it
|
|
1515
|
+
// appears later below the viewport, its first observation must still be
|
|
1516
|
+
// allowed to scroll it into view.
|
|
1517
|
+
scrollRequested = true;
|
|
1518
|
+
const stable = sameLayout(previous, next);
|
|
1519
|
+
previous = next;
|
|
1520
|
+
if (!stable) return false;
|
|
1521
|
+
// Native mouse movement later re-resolves this exact ref before pressing,
|
|
1522
|
+
// so another pre-move snapshot cannot prove that the final point is safe.
|
|
1523
|
+
found = next;
|
|
1524
|
+
return true;
|
|
1311
1525
|
});
|
|
1312
1526
|
return found;
|
|
1313
1527
|
}
|
|
1314
1528
|
async function locatorClick(ctx, args, double = false) {
|
|
1315
1529
|
const found = await resolveActionTarget(ctx, args);
|
|
1316
|
-
|
|
1530
|
+
const input = options(args),
|
|
1531
|
+
expect = input.expect,
|
|
1532
|
+
framePath = args.locator?.framePath ?? args.framePath ?? [];
|
|
1533
|
+
// The native navigation event must be armed before input; top-level only.
|
|
1534
|
+
requireValue(
|
|
1535
|
+
!(expect?.kind === "navigation" && framePath.length),
|
|
1536
|
+
"INVALID_ARGUMENT",
|
|
1537
|
+
"For a nested frame, wait for its URL or a frame-local page condition.",
|
|
1538
|
+
);
|
|
1539
|
+
let navigation;
|
|
1540
|
+
if (expect?.kind === "navigation")
|
|
1541
|
+
navigation = await ctx.router.armWaiter(ctx, {
|
|
1542
|
+
event: "navigation",
|
|
1543
|
+
url: expect.url,
|
|
1544
|
+
waitUntil: expect.waitUntil ?? "load",
|
|
1545
|
+
timeoutMs:
|
|
1546
|
+
expect.timeoutMs ??
|
|
1547
|
+
Math.min(15000, ctx.budgetMs - (performance.now() - ctx.started)),
|
|
1548
|
+
});
|
|
1549
|
+
try {
|
|
1550
|
+
await click(
|
|
1551
|
+
ctx,
|
|
1552
|
+
{
|
|
1553
|
+
...input,
|
|
1554
|
+
...found.point,
|
|
1555
|
+
resolvedTarget: true,
|
|
1556
|
+
framePath,
|
|
1557
|
+
framePoint: found.framePoint,
|
|
1558
|
+
frameId: found.frameId,
|
|
1559
|
+
expectedRef: found.ref,
|
|
1560
|
+
},
|
|
1561
|
+
double,
|
|
1562
|
+
);
|
|
1563
|
+
if (!expect) return null;
|
|
1564
|
+
try {
|
|
1565
|
+
if (navigation) {
|
|
1566
|
+
await ctx.router.getWaiter(ctx, navigation.id);
|
|
1567
|
+
return {
|
|
1568
|
+
matched: true,
|
|
1569
|
+
snapshot: {
|
|
1570
|
+
...ctx.router.target(ctx.target.tabId, ctx.lease),
|
|
1571
|
+
...(await ctx.router.page(ctx, { command: "snapshot" })),
|
|
1572
|
+
},
|
|
1573
|
+
};
|
|
1574
|
+
}
|
|
1575
|
+
return await ctx.router.waitForCondition(ctx, {
|
|
1576
|
+
condition: expect,
|
|
1577
|
+
framePath,
|
|
1578
|
+
});
|
|
1579
|
+
} catch (cause) {
|
|
1580
|
+
ctx.signal?.throwIfAborted();
|
|
1581
|
+
if (
|
|
1582
|
+
cause instanceof BrowserActionError &&
|
|
1583
|
+
["TIMEOUT", "STALE_REF", "INVALID_TARGET", "WAITER_GONE"].includes(
|
|
1584
|
+
cause.code,
|
|
1585
|
+
)
|
|
1586
|
+
)
|
|
1587
|
+
throw new BrowserActionError(
|
|
1588
|
+
"EXPECTATION_UNCONFIRMED",
|
|
1589
|
+
"The page did not confirm the expected result. Input was already sent; observe before taking further action.",
|
|
1590
|
+
{ dispatched: true, effect: "unknown" },
|
|
1591
|
+
);
|
|
1592
|
+
throw cause;
|
|
1593
|
+
}
|
|
1594
|
+
} finally {
|
|
1595
|
+
if (navigation)
|
|
1596
|
+
await ctx.router.cancelWaiter(ctx, navigation.id).catch(() => {});
|
|
1597
|
+
}
|
|
1317
1598
|
}
|
|
1318
1599
|
route("locator.click", (ctx, args) => locatorClick(ctx, args), {
|
|
1319
1600
|
needsTab: true,
|
|
@@ -1684,6 +1965,7 @@ export function createBrowserRouter(
|
|
|
1684
1965
|
journalTail = Promise.resolve();
|
|
1685
1966
|
const versions = new Map(),
|
|
1686
1967
|
documents = new Map(),
|
|
1968
|
+
pointerPositions = new Map(),
|
|
1687
1969
|
leases = new Map(),
|
|
1688
1970
|
closedLeases = new Map(),
|
|
1689
1971
|
owners = new Map(),
|
|
@@ -2255,11 +2537,51 @@ export function createBrowserRouter(
|
|
|
2255
2537
|
throw cause;
|
|
2256
2538
|
}
|
|
2257
2539
|
};
|
|
2258
|
-
|
|
2540
|
+
// Chrome may hold a native command indefinitely while a JavaScript dialog
|
|
2541
|
+
// pauses its renderer. A cancelled action must not hold the run's pending
|
|
2542
|
+
// queue forever; the native command may still have an unknown page effect,
|
|
2543
|
+
// so actual debugger detach remains a separate required cleanup step.
|
|
2544
|
+
const nativeWithStop = async (signal, invoke) => {
|
|
2545
|
+
signal?.throwIfAborted();
|
|
2546
|
+
const native = Promise.resolve().then(() => {
|
|
2547
|
+
signal?.throwIfAborted();
|
|
2548
|
+
return invoke();
|
|
2549
|
+
});
|
|
2550
|
+
if (!signal) return native;
|
|
2551
|
+
let onAbort;
|
|
2552
|
+
const cancelled = new Promise((_, reject) => {
|
|
2553
|
+
onAbort = () => reject(signal.reason);
|
|
2554
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
2555
|
+
});
|
|
2556
|
+
try {
|
|
2557
|
+
if (signal.aborted) onAbort();
|
|
2558
|
+
return await Promise.race([native, cancelled]);
|
|
2559
|
+
} finally {
|
|
2560
|
+
signal.removeEventListener("abort", onAbort);
|
|
2561
|
+
}
|
|
2562
|
+
};
|
|
2563
|
+
const releaseControl = async (
|
|
2564
|
+
lease,
|
|
2565
|
+
tabId,
|
|
2566
|
+
{ detach = true, dismissDialog = false } = {},
|
|
2567
|
+
) => {
|
|
2259
2568
|
if (!lease.attached.has(tabId) && !lease.intents.has(tabId)) return;
|
|
2260
2569
|
for (const waiter of waiters.values())
|
|
2261
2570
|
if (waiter.leaseId === lease.id && waiter.tabId === tabId)
|
|
2262
2571
|
cancelWaiter(waiter);
|
|
2572
|
+
// An observed JavaScript dialog pauses its renderer. Dismiss the dialog
|
|
2573
|
+
// for a terminating run before other native cleanup commands; explicit
|
|
2574
|
+
// handoff keeps the dialog available for the user instead.
|
|
2575
|
+
const dialog = dialogs.get(String(tabId));
|
|
2576
|
+
if (dismissDialog && dialog && lease.attached.has(tabId)) {
|
|
2577
|
+
await chrome.debugger.sendCommand(
|
|
2578
|
+
{ tabId },
|
|
2579
|
+
"Page.handleJavaScriptDialog",
|
|
2580
|
+
{ accept: false },
|
|
2581
|
+
);
|
|
2582
|
+
if (dialogs.get(String(tabId)) === dialog)
|
|
2583
|
+
dialogs.delete(String(tabId));
|
|
2584
|
+
}
|
|
2263
2585
|
await stopLocalDownloadClick(lease, tabId);
|
|
2264
2586
|
if (lease.attached.has(tabId)) await syncDownloadIntercept(lease, tabId);
|
|
2265
2587
|
if (!lease.attached.has(tabId) && !lease.externallyDetached.has(tabId)) {
|
|
@@ -2522,6 +2844,7 @@ export function createBrowserRouter(
|
|
|
2522
2844
|
finishNavigation(details, "load"),
|
|
2523
2845
|
);
|
|
2524
2846
|
on(chrome.tabs.onRemoved, (id) => {
|
|
2847
|
+
pointerPositions.delete(id);
|
|
2525
2848
|
versions.set(String(id), version(id) + 1);
|
|
2526
2849
|
dialogs.delete(String(id));
|
|
2527
2850
|
emit({
|
|
@@ -3081,7 +3404,7 @@ export function createBrowserRouter(
|
|
|
3081
3404
|
.filter((id) => id !== undefined),
|
|
3082
3405
|
))
|
|
3083
3406
|
try {
|
|
3084
|
-
await releaseControl(lease, tabId);
|
|
3407
|
+
await releaseControl(lease, tabId, { dismissDialog: true });
|
|
3085
3408
|
} catch (cause) {
|
|
3086
3409
|
tempErrors.push(cause);
|
|
3087
3410
|
}
|
|
@@ -3100,7 +3423,7 @@ export function createBrowserRouter(
|
|
|
3100
3423
|
}
|
|
3101
3424
|
for (const tabId of new Set([...lease.attached, ...lease.intents])) {
|
|
3102
3425
|
try {
|
|
3103
|
-
await releaseControl(lease, tabId);
|
|
3426
|
+
await releaseControl(lease, tabId, { dismissDialog: true });
|
|
3104
3427
|
} catch (cause) {
|
|
3105
3428
|
errors.push(cause);
|
|
3106
3429
|
}
|
|
@@ -3331,6 +3654,14 @@ export function createBrowserRouter(
|
|
|
3331
3654
|
].includes(input.method)
|
|
3332
3655
|
)
|
|
3333
3656
|
await router.prepareVisual(ctx);
|
|
3657
|
+
if (
|
|
3658
|
+
visual &&
|
|
3659
|
+
definition.needsTab &&
|
|
3660
|
+
!definition.mutating &&
|
|
3661
|
+
input.target.tabId !== undefined &&
|
|
3662
|
+
!lease.temporaryTabs?.has(nativeId(input.target.tabId))
|
|
3663
|
+
)
|
|
3664
|
+
await router.trackVisualTab(ctx);
|
|
3334
3665
|
actionSignal.throwIfAborted();
|
|
3335
3666
|
managementHandling = true;
|
|
3336
3667
|
value = await definition.handle(ctx, input.args ?? {});
|
|
@@ -3367,9 +3698,14 @@ export function createBrowserRouter(
|
|
|
3367
3698
|
"tabs.closeTemporary",
|
|
3368
3699
|
"tab.getJsDialog",
|
|
3369
3700
|
].includes(input.method);
|
|
3701
|
+
// Native mutations can activate a tab and send shared browser input.
|
|
3702
|
+
// A per-tab queue cannot protect two simultaneous actions that switch
|
|
3703
|
+
// the active tab underneath one another. Reads and event waiters remain
|
|
3704
|
+
// independently schedulable.
|
|
3370
3705
|
const queueKey =
|
|
3371
|
-
|
|
3372
|
-
|
|
3706
|
+
definition.mutating && !nonBlocking
|
|
3707
|
+
? `native-mutation:${input.target.browserId}:${input.target.browserSessionId}`
|
|
3708
|
+
: input.target.tabId;
|
|
3373
3709
|
let task;
|
|
3374
3710
|
if (queueKey && !nonBlocking) {
|
|
3375
3711
|
task = (queues.get(queueKey) ?? Promise.resolve()).then(perform);
|
|
@@ -3671,12 +4007,14 @@ export function createBrowserRouter(
|
|
|
3671
4007
|
const framePath = payload.locator?.framePath ?? payload.framePath ?? [];
|
|
3672
4008
|
if (framePath.length > 0) return framePage(ctx, payload, framePath);
|
|
3673
4009
|
ctx.dispatched();
|
|
3674
|
-
const result = await
|
|
3675
|
-
|
|
3676
|
-
|
|
3677
|
-
|
|
3678
|
-
|
|
3679
|
-
|
|
4010
|
+
const result = await nativeWithStop(ctx.signal, () =>
|
|
4011
|
+
chrome.scripting.executeScript({
|
|
4012
|
+
target: { tabId: nativeId(ctx.target.tabId) },
|
|
4013
|
+
world: "ISOLATED",
|
|
4014
|
+
func: pageCommand,
|
|
4015
|
+
args: [input],
|
|
4016
|
+
}),
|
|
4017
|
+
);
|
|
3680
4018
|
// A main-frame replacement can finish an injection with no result. Only
|
|
3681
4019
|
// a readiness observation may wait for a browser-confirmed transition;
|
|
3682
4020
|
// actions and empty results from settled pages must still fail.
|
|
@@ -3865,6 +4203,17 @@ export function createBrowserRouter(
|
|
|
3865
4203
|
ctx.dispatched();
|
|
3866
4204
|
await chrome.debugger.attach({ tabId: id }, "1.3");
|
|
3867
4205
|
ctx.lease.attached.add(id);
|
|
4206
|
+
// Observe Page.javascriptDialogOpening before *any* page action can
|
|
4207
|
+
// synchronously open a prompt/confirm. Waiting until getJsDialog is
|
|
4208
|
+
// called loses the event while Runtime.evaluate may remain blocked.
|
|
4209
|
+
try {
|
|
4210
|
+
await nativeWithStop(ctx.signal, () =>
|
|
4211
|
+
chrome.debugger.sendCommand({ tabId: id }, "Page.enable", {}),
|
|
4212
|
+
);
|
|
4213
|
+
} catch (cause) {
|
|
4214
|
+
await releaseControl(ctx.lease, id).catch(() => {});
|
|
4215
|
+
throw cause;
|
|
4216
|
+
}
|
|
3868
4217
|
if (ctx.lease.ended) {
|
|
3869
4218
|
await releaseControl(ctx.lease, id);
|
|
3870
4219
|
throw new BrowserActionError(
|
|
@@ -3880,9 +4229,42 @@ export function createBrowserRouter(
|
|
|
3880
4229
|
ctx.lease.attaching.delete(id);
|
|
3881
4230
|
}
|
|
3882
4231
|
},
|
|
3883
|
-
async send(ctx, method, params, child) {
|
|
4232
|
+
async send(ctx, method, params, child, nativePointerStep = false) {
|
|
3884
4233
|
active(ctx.lease);
|
|
3885
4234
|
ctx.signal?.throwIfAborted();
|
|
4235
|
+
// Every high-level mouse move, click, hover and drag shares the same
|
|
4236
|
+
// physical trajectory. Raw cdp.send remains exact and unmodified.
|
|
4237
|
+
if (
|
|
4238
|
+
!nativePointerStep &&
|
|
4239
|
+
!child &&
|
|
4240
|
+
ctx.method !== "cdp.send" &&
|
|
4241
|
+
method === "Input.dispatchMouseEvent" &&
|
|
4242
|
+
params.type === "mouseMoved" &&
|
|
4243
|
+
Number.isFinite(params.x) &&
|
|
4244
|
+
Number.isFinite(params.y)
|
|
4245
|
+
) {
|
|
4246
|
+
const tabId = nativeId(ctx.target.tabId),
|
|
4247
|
+
previous = pointerPositions.get(tabId) ?? { x: 32, y: 72 };
|
|
4248
|
+
// Chromium can heavily throttle input acknowledgement in an inactive
|
|
4249
|
+
// tab. Keep a real multi-point native trajectory, but bound its steps
|
|
4250
|
+
// so one background move cannot outlive the original SDK request.
|
|
4251
|
+
const tab = await chrome.tabs.get(tabId);
|
|
4252
|
+
const clickMove = /(^|\.)(click|dblclick|doubleClick)$/.test(ctx.method);
|
|
4253
|
+
const maxSteps = tab.active ? 28 : clickMove ? 2 : 3;
|
|
4254
|
+
const path = humanPointerPath(previous, { x: params.x, y: params.y }, maxSteps);
|
|
4255
|
+
let result;
|
|
4256
|
+
for (const step of path) {
|
|
4257
|
+
result = await router.send(
|
|
4258
|
+
ctx,
|
|
4259
|
+
method,
|
|
4260
|
+
{ ...params, x: step.x, y: step.y },
|
|
4261
|
+
child,
|
|
4262
|
+
true,
|
|
4263
|
+
);
|
|
4264
|
+
if (step.delayMs > 0) await router.delay(ctx, step.delayMs);
|
|
4265
|
+
}
|
|
4266
|
+
return result;
|
|
4267
|
+
}
|
|
3886
4268
|
await router.attach(ctx);
|
|
3887
4269
|
ctx.signal?.throwIfAborted();
|
|
3888
4270
|
if (
|
|
@@ -4035,7 +4417,20 @@ export function createBrowserRouter(
|
|
|
4035
4417
|
ctx.pendingInfo.nativeNetworkTabId = tabId;
|
|
4036
4418
|
let result;
|
|
4037
4419
|
try {
|
|
4038
|
-
|
|
4420
|
+
// A native Runtime.evaluate can stay suspended behind a modal dialog.
|
|
4421
|
+
// A raw cdp.send Network.loadNetworkResource can also remain pending
|
|
4422
|
+
// after debugger detach (notably on Windows). The lease still confirms
|
|
4423
|
+
// native input release and debugger detach before reporting closure.
|
|
4424
|
+
// Internal document exports must await the native response so any
|
|
4425
|
+
// returned stream can be closed; IO.read and IO.close likewise retain
|
|
4426
|
+
// their actual settlement.
|
|
4427
|
+
result =
|
|
4428
|
+
method === "Runtime.evaluate" ||
|
|
4429
|
+
(method === "Network.loadNetworkResource" && ctx.method === "cdp.send")
|
|
4430
|
+
? await nativeWithStop(ctx.signal, () =>
|
|
4431
|
+
chrome.debugger.sendCommand(source, method, params),
|
|
4432
|
+
)
|
|
4433
|
+
: await chrome.debugger.sendCommand(source, method, params);
|
|
4039
4434
|
} catch (cause) {
|
|
4040
4435
|
if (armedLocalDownload) await stopLocalDownloadClick(ctx.lease, tabId);
|
|
4041
4436
|
throw cause;
|
|
@@ -4044,6 +4439,14 @@ export function createBrowserRouter(
|
|
|
4044
4439
|
delete ctx.pendingInfo.nativeNetworkTabId;
|
|
4045
4440
|
if (watchEnd) await stopLocalDownloadClick(ctx.lease, tabId);
|
|
4046
4441
|
}
|
|
4442
|
+
if (
|
|
4443
|
+
!child &&
|
|
4444
|
+
method === "Input.dispatchMouseEvent" &&
|
|
4445
|
+
params.type === "mouseMoved" &&
|
|
4446
|
+
Number.isFinite(params.x) &&
|
|
4447
|
+
Number.isFinite(params.y)
|
|
4448
|
+
)
|
|
4449
|
+
pointerPositions.set(tabId, { x: params.x, y: params.y });
|
|
4047
4450
|
if (rawFetch && method === "Fetch.disable") {
|
|
4048
4451
|
ctx.lease.rawFetch?.delete(tabId);
|
|
4049
4452
|
await syncDownloadIntercept(ctx.lease, tabId);
|
|
@@ -4874,6 +5277,9 @@ export function createBrowserRouter(
|
|
|
4874
5277
|
},
|
|
4875
5278
|
async screenshot(ctx, input) {
|
|
4876
5279
|
requireValue(!(input.fullPage && input.clip));
|
|
5280
|
+
await readyForInput(ctx);
|
|
5281
|
+
const readyBudget = ctx.budgetMs - (performance.now() - ctx.started);
|
|
5282
|
+
if (readyBudget > 130) await router.delay(ctx, 120);
|
|
4877
5283
|
const quality = input.quality ?? 80,
|
|
4878
5284
|
maxEdgePx = input.maxEdgePx ?? 1920;
|
|
4879
5285
|
requireValue(
|
|
@@ -5308,6 +5714,22 @@ export function createBrowserRouter(
|
|
|
5308
5714
|
});
|
|
5309
5715
|
}
|
|
5310
5716
|
},
|
|
5717
|
+
async trackVisualTab(ctx) {
|
|
5718
|
+
try {
|
|
5719
|
+
await visual?.trackTab(router.visualContext(ctx));
|
|
5720
|
+
} catch {
|
|
5721
|
+
ctx.signal?.throwIfAborted();
|
|
5722
|
+
emit({
|
|
5723
|
+
type: "event",
|
|
5724
|
+
event: "visual_degraded",
|
|
5725
|
+
leaseId: ctx.lease.id,
|
|
5726
|
+
runId: ctx.runId,
|
|
5727
|
+
operationId: ctx.operationId,
|
|
5728
|
+
target: ctx.target,
|
|
5729
|
+
reason: "VISUAL_UNAVAILABLE",
|
|
5730
|
+
});
|
|
5731
|
+
}
|
|
5732
|
+
},
|
|
5311
5733
|
async preparePointer(ctx, point) {
|
|
5312
5734
|
if (
|
|
5313
5735
|
ctx.lastVisualPoint?.x === point.x &&
|