@jobshimo/browser-link 0.23.3 → 0.23.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -862,6 +862,458 @@ async function performWaitFor(tabId, state, params) {
862
862
  return { ok: false, error: err instanceof Error ? err.message : String(err) };
863
863
  }
864
864
  }
865
+ /**
866
+ * `handlePing` through `handleSetPermission` — the extracted bodies of the
867
+ * remaining standalone tool cases in `handleTool` below that never got the
868
+ * `perform*` treatment above. Each takes exactly the locals its original
869
+ * `case` block closed over (`tabId`, `state`, `msg`, the raw params `p`,
870
+ * and/or the `str`/`optStr` readers) and returns the same wire-level
871
+ * `tool.response` envelope the case used to build inline — pure code
872
+ * motion, not a rewrite of tool behavior.
873
+ */
874
+ async function handlePing(tabId, msg) {
875
+ const tab = await chrome.tabs.get(tabId);
876
+ return {
877
+ kind: 'tool.response',
878
+ id: msg.id,
879
+ ok: true,
880
+ result: { title: tab.title ?? '', url: tab.url ?? '' },
881
+ };
882
+ }
883
+ async function handleNavigate(tabId, msg, p, str) {
884
+ const url = str('url');
885
+ const waitForLoadFlag = p.wait_for_load !== false;
886
+ await cdp(tabId, 'Page.navigate', { url });
887
+ if (waitForLoadFlag)
888
+ await waitForLoad(tabId);
889
+ const tab = await chrome.tabs.get(tabId);
890
+ return {
891
+ kind: 'tool.response',
892
+ id: msg.id,
893
+ ok: true,
894
+ result: { url: tab.url ?? '', title: tab.title ?? '' },
895
+ };
896
+ }
897
+ async function handleSnapshot(tabId, msg, p, optStr) {
898
+ const value = await evaluateInTab(tabId, buildSnapshotJs({
899
+ within_selector: optStr('within_selector'),
900
+ only_interactive: p.only_interactive === true,
901
+ exclude: Array.isArray(p.exclude)
902
+ ? p.exclude.filter((x) => typeof x === 'string')
903
+ : undefined,
904
+ max_interactive: typeof p.max_interactive === 'number' ? p.max_interactive : undefined,
905
+ }));
906
+ return { kind: 'tool.response', id: msg.id, ok: true, result: value };
907
+ }
908
+ async function handleCanvasScreenshot(tabId, msg, p, optStr) {
909
+ const selector = optStr('selector');
910
+ const format = optStr('format') === 'jpeg' ? 'jpeg' : 'png';
911
+ const regionRaw = p.region;
912
+ let region;
913
+ if (regionRaw && typeof regionRaw === 'object') {
914
+ const r = regionRaw;
915
+ if (typeof r.x === 'number' &&
916
+ typeof r.y === 'number' &&
917
+ typeof r.w === 'number' &&
918
+ typeof r.h === 'number') {
919
+ region = { x: r.x, y: r.y, w: r.w, h: r.h };
920
+ }
921
+ }
922
+ const value = await evaluateInTab(tabId, buildCanvasScreenshotJs({ selector, region, format }));
923
+ return { kind: 'tool.response', id: msg.id, ok: true, result: value };
924
+ }
925
+ function handleConsole(state, msg, optStr) {
926
+ const level = optStr('level');
927
+ const entries = level
928
+ ? state.consoleBuffer.filter((e) => e.level === level)
929
+ : state.consoleBuffer;
930
+ return { kind: 'tool.response', id: msg.id, ok: true, result: entries };
931
+ }
932
+ function handleNetwork(state, msg, optStr) {
933
+ const filter = optStr('url_filter')?.toLowerCase();
934
+ const list = state.networkOrder
935
+ .map((id) => state.networkBuffer.get(id))
936
+ .filter((e) => e !== undefined)
937
+ .filter((e) => (filter ? e.url.toLowerCase().includes(filter) : true));
938
+ return { kind: 'tool.response', id: msg.id, ok: true, result: list };
939
+ }
940
+ async function handleNetworkBody(tabId, msg, str) {
941
+ const requestId = str('request_id');
942
+ const body = await cdp(tabId, 'Network.getResponseBody', {
943
+ requestId,
944
+ });
945
+ return { kind: 'tool.response', id: msg.id, ok: true, result: body };
946
+ }
947
+ async function handleFlow(tabId, state, msg, p) {
948
+ // Wire-boundary narrowing: `p.steps` is untrusted JSON. Keep only
949
+ // object-shaped entries here — `runFlow`'s own `stepKind()` guard
950
+ // re-validates each one at runtime regardless, so a malformed
951
+ // entry fails the flow cleanly instead of throwing.
952
+ const rawSteps = Array.isArray(p.steps)
953
+ ? p.steps.filter((s) => typeof s === 'object' && s !== null)
954
+ : [];
955
+ const flowResult = await runFlow(rawSteps, {
956
+ performFind: (params) => performFind(tabId, params),
957
+ performClick: (params) => performClick(tabId, params),
958
+ performType: (params) => performType(tabId, params),
959
+ performPress: (params) => performPress(tabId, params),
960
+ performWaitFor: (params) => performWaitFor(tabId, state, params),
961
+ buildRecoverySnapshot: () => evaluateInTab(tabId, buildSnapshotJs({ only_interactive: true, max_interactive: 40 })),
962
+ });
963
+ // The wire-level response is ok:true whenever the flow RAN (even a
964
+ // failed step is a legitimate business outcome, same pattern as
965
+ // wait_for's matched:false) — flowResult itself carries the
966
+ // ok:true/false the agent reads.
967
+ return { kind: 'tool.response', id: msg.id, ok: true, result: flowResult };
968
+ }
969
+ async function handleDrag(tabId, msg, p, optStr) {
970
+ const optNum = (key) => {
971
+ const v = p[key];
972
+ return typeof v === 'number' && Number.isFinite(v) && v >= 0 ? v : undefined;
973
+ };
974
+ const clamp = (v, max) => Math.min(v, max);
975
+ const fromSelector = optStr('from_selector');
976
+ const toSelector = optStr('to_selector');
977
+ const fromXRaw = optNum('from_x');
978
+ const fromYRaw = optNum('from_y');
979
+ const toXRaw = optNum('to_x');
980
+ const toYRaw = optNum('to_y');
981
+ // Cap every duration that ends up in a setTimeout to keep CodeQL's
982
+ // "user-controlled timer duration" check happy and to prevent a
983
+ // misbehaving agent from parking the bridge for hours on a typo.
984
+ const durationMs = clamp(optNum('duration_ms') ?? 1500, MAX_DRAG_DURATION_MS);
985
+ const holdBeforeMoveMs = clamp(optNum('hold_before_move_ms') ?? 0, MAX_DRAG_HOLD_MS);
986
+ const holdBeforeReleaseMs = clamp(optNum('hold_before_release_ms') ?? 0, MAX_DRAG_HOLD_MS);
987
+ const hasFromCoords = fromXRaw !== undefined && fromYRaw !== undefined;
988
+ const hasToCoords = toXRaw !== undefined && toYRaw !== undefined;
989
+ if (!fromSelector && !hasFromCoords) {
990
+ return {
991
+ kind: 'tool.response',
992
+ id: msg.id,
993
+ ok: false,
994
+ error: 'drag: provide from_selector or both from_x and from_y',
995
+ };
996
+ }
997
+ if (!toSelector && !hasToCoords) {
998
+ return {
999
+ kind: 'tool.response',
1000
+ id: msg.id,
1001
+ ok: false,
1002
+ error: 'drag: provide to_selector or both to_x and to_y',
1003
+ };
1004
+ }
1005
+ const probeExpr = buildDragProbeJs({
1006
+ from_selector: fromSelector,
1007
+ to_selector: toSelector,
1008
+ from_x: fromXRaw,
1009
+ from_y: fromYRaw,
1010
+ to_x: toXRaw,
1011
+ to_y: toYRaw,
1012
+ });
1013
+ const probe = await evaluateInTab(tabId, probeExpr);
1014
+ if (probe.err) {
1015
+ return { kind: 'tool.response', id: msg.id, ok: false, error: `drag: ${probe.err}` };
1016
+ }
1017
+ if (!probe.from || !probe.to) {
1018
+ return {
1019
+ kind: 'tool.response',
1020
+ id: msg.id,
1021
+ ok: false,
1022
+ error: 'drag: could not resolve coordinates',
1023
+ };
1024
+ }
1025
+ if (!probe.from.in_viewport) {
1026
+ return {
1027
+ kind: 'tool.response',
1028
+ id: msg.id,
1029
+ ok: false,
1030
+ error: 'drag: source point is offscreen — scroll first or pass viewport coords',
1031
+ };
1032
+ }
1033
+ if (!probe.to.in_viewport) {
1034
+ return {
1035
+ kind: 'tool.response',
1036
+ id: msg.id,
1037
+ ok: false,
1038
+ error: 'drag: destination point is offscreen — scroll first or pass viewport coords',
1039
+ };
1040
+ }
1041
+ const fromX = probe.from.x;
1042
+ const fromY = probe.from.y;
1043
+ const toX = probe.to.x;
1044
+ const toY = probe.to.y;
1045
+ const isDraggable = !!probe.draggable;
1046
+ // ~30fps interpolation; minimum 2 steps so the path actually has a midpoint.
1047
+ const steps = durationMs > 0 ? Math.max(2, Math.round(durationMs / 33)) : 1;
1048
+ const stepDelayMs = steps > 0 ? durationMs / steps : 0;
1049
+ const eventsFired = [];
1050
+ // Every branch below reassigns `dragMode` before the final `return` reads it.
1051
+ let dragMode = 'pointer'; // eslint-disable-line no-useless-assignment -- see comment above
1052
+ let interceptReceived = false;
1053
+ const dragStart = Date.now();
1054
+ let interceptionEnabled = false;
1055
+ const interpolate = (t) => ({
1056
+ x: fromX + (toX - fromX) * t,
1057
+ y: fromY + (toY - fromY) * t,
1058
+ });
1059
+ // Mouse move WITH the left button held down. Chrome's HTML5 drag
1060
+ // system only treats movement as a drag when the button state is
1061
+ // signalled on every move after mousePressed; omitting it makes
1062
+ // Blink ignore the move for drag-detection purposes.
1063
+ const moveHeld = (x, y) => cdp(tabId, 'Input.dispatchMouseEvent', {
1064
+ type: 'mouseMoved',
1065
+ x,
1066
+ y,
1067
+ button: 'left',
1068
+ buttons: 1,
1069
+ });
1070
+ // Wiggle distance has to comfortably clear `kDragThreshold` (~5px on
1071
+ // Mac, 4px on Linux/Windows). 5px landed *right* at the boundary and
1072
+ // Chrome did not start the drag — 20px crosses it on every platform.
1073
+ const WIGGLE_PX = 20;
1074
+ // Time to wait for Input.dragIntercepted after the wiggle. 120ms was
1075
+ // enough on local-only synthetic tests but flaky in real Chrome —
1076
+ // 250ms is comfortable without dragging out the overall latency.
1077
+ const INTERCEPT_TIMEOUT_MS = 250;
1078
+ try {
1079
+ if (isDraggable) {
1080
+ try {
1081
+ await cdp(tabId, 'Input.setInterceptDrags', { enabled: true });
1082
+ interceptionEnabled = true;
1083
+ }
1084
+ catch {
1085
+ // setInterceptDrags is experimental — fall back to pointer mode silently.
1086
+ }
1087
+ }
1088
+ if (interceptionEnabled) {
1089
+ // Arm the listener BEFORE the press+wiggle that may trigger it.
1090
+ const interceptPromise = waitForCdpEvent(tabId, 'Input.dragIntercepted', INTERCEPT_TIMEOUT_MS);
1091
+ await cdp(tabId, 'Input.dispatchMouseEvent', {
1092
+ type: 'mouseMoved',
1093
+ x: fromX,
1094
+ y: fromY,
1095
+ });
1096
+ await cdp(tabId, 'Input.dispatchMouseEvent', {
1097
+ type: 'mousePressed',
1098
+ x: fromX,
1099
+ y: fromY,
1100
+ button: 'left',
1101
+ clickCount: 1,
1102
+ });
1103
+ if (holdBeforeMoveMs > 0)
1104
+ await sleep(holdBeforeMoveMs);
1105
+ // Wiggle toward the destination so Chrome's native drag system
1106
+ // crosses its activation threshold. Direction matters for libs
1107
+ // that have direction-sensitive activation constraints.
1108
+ const dx = toX - fromX;
1109
+ const dy = toY - fromY;
1110
+ const len = Math.hypot(dx, dy) || 1;
1111
+ const wx = fromX + (dx / len) * WIGGLE_PX;
1112
+ const wy = fromY + (dy / len) * WIGGLE_PX;
1113
+ await moveHeld(wx, wy);
1114
+ const intercepted = await interceptPromise;
1115
+ if (intercepted) {
1116
+ dragMode = 'html5';
1117
+ interceptReceived = true;
1118
+ eventsFired.push('Input.dragIntercepted');
1119
+ const dragData = intercepted.data;
1120
+ for (let i = 1; i <= steps; i++) {
1121
+ const t = i / steps;
1122
+ const { x, y } = interpolate(t);
1123
+ await moveHeld(x, y);
1124
+ await cdp(tabId, 'Input.dispatchDragEvent', {
1125
+ type: 'dragOver',
1126
+ x,
1127
+ y,
1128
+ data: dragData,
1129
+ });
1130
+ if (i === 1)
1131
+ eventsFired.push('dragOver');
1132
+ if (stepDelayMs > 0)
1133
+ await sleep(stepDelayMs);
1134
+ }
1135
+ await cdp(tabId, 'Input.dispatchDragEvent', {
1136
+ type: 'dragEnter',
1137
+ x: toX,
1138
+ y: toY,
1139
+ data: dragData,
1140
+ });
1141
+ eventsFired.push('dragEnter');
1142
+ await cdp(tabId, 'Input.dispatchDragEvent', {
1143
+ type: 'dragOver',
1144
+ x: toX,
1145
+ y: toY,
1146
+ data: dragData,
1147
+ });
1148
+ if (holdBeforeReleaseMs > 0)
1149
+ await sleep(holdBeforeReleaseMs);
1150
+ await cdp(tabId, 'Input.dispatchDragEvent', {
1151
+ type: 'drop',
1152
+ x: toX,
1153
+ y: toY,
1154
+ data: dragData,
1155
+ });
1156
+ eventsFired.push('drop');
1157
+ await cdp(tabId, 'Input.dispatchMouseEvent', {
1158
+ type: 'mouseReleased',
1159
+ x: toX,
1160
+ y: toY,
1161
+ button: 'left',
1162
+ clickCount: 1,
1163
+ });
1164
+ }
1165
+ else {
1166
+ // Element was tagged draggable but no native drag fired — the page
1167
+ // either preventDefault'd dragstart or wires its own pointer logic.
1168
+ // Continue with pointer-only events from where we already pressed.
1169
+ dragMode = 'pointer';
1170
+ for (let i = 1; i <= steps; i++) {
1171
+ const t = i / steps;
1172
+ const { x, y } = interpolate(t);
1173
+ await moveHeld(x, y);
1174
+ if (stepDelayMs > 0)
1175
+ await sleep(stepDelayMs);
1176
+ }
1177
+ if (holdBeforeReleaseMs > 0)
1178
+ await sleep(holdBeforeReleaseMs);
1179
+ await cdp(tabId, 'Input.dispatchMouseEvent', {
1180
+ type: 'mouseReleased',
1181
+ x: toX,
1182
+ y: toY,
1183
+ button: 'left',
1184
+ clickCount: 1,
1185
+ });
1186
+ }
1187
+ }
1188
+ else {
1189
+ // Pointer-only branch: no native drag involvement at all.
1190
+ dragMode = 'pointer';
1191
+ await cdp(tabId, 'Input.dispatchMouseEvent', {
1192
+ type: 'mouseMoved',
1193
+ x: fromX,
1194
+ y: fromY,
1195
+ });
1196
+ await cdp(tabId, 'Input.dispatchMouseEvent', {
1197
+ type: 'mousePressed',
1198
+ x: fromX,
1199
+ y: fromY,
1200
+ button: 'left',
1201
+ clickCount: 1,
1202
+ });
1203
+ if (holdBeforeMoveMs > 0)
1204
+ await sleep(holdBeforeMoveMs);
1205
+ for (let i = 1; i <= steps; i++) {
1206
+ const t = i / steps;
1207
+ const { x, y } = interpolate(t);
1208
+ await moveHeld(x, y);
1209
+ if (stepDelayMs > 0)
1210
+ await sleep(stepDelayMs);
1211
+ }
1212
+ if (holdBeforeReleaseMs > 0)
1213
+ await sleep(holdBeforeReleaseMs);
1214
+ await cdp(tabId, 'Input.dispatchMouseEvent', {
1215
+ type: 'mouseReleased',
1216
+ x: toX,
1217
+ y: toY,
1218
+ button: 'left',
1219
+ clickCount: 1,
1220
+ });
1221
+ }
1222
+ }
1223
+ finally {
1224
+ // Critical: leaving interception on would block the human user from
1225
+ // dragging anything in this tab. Best-effort cleanup, swallow errors.
1226
+ if (interceptionEnabled) {
1227
+ try {
1228
+ await cdp(tabId, 'Input.setInterceptDrags', { enabled: false });
1229
+ }
1230
+ catch {
1231
+ // ignore
1232
+ }
1233
+ }
1234
+ }
1235
+ return {
1236
+ kind: 'tool.response',
1237
+ id: msg.id,
1238
+ ok: true,
1239
+ result: {
1240
+ from: { x: fromX, y: fromY, selector: fromSelector ?? null },
1241
+ to: { x: toX, y: toY, selector: toSelector ?? null },
1242
+ duration_ms_actual: Date.now() - dragStart,
1243
+ drag_mode: dragMode,
1244
+ interception_attempted: interceptionEnabled,
1245
+ intercept_received: interceptReceived,
1246
+ events_fired: eventsFired,
1247
+ },
1248
+ };
1249
+ }
1250
+ async function handleDialogRespond(tabId, msg, p, optStr) {
1251
+ const accept = p.accept === true;
1252
+ const promptText = optStr('prompt_text');
1253
+ // No probing — if there is no dialog open, CDP returns an error.
1254
+ // We propagate it; the caller decides if "no dialog to respond to"
1255
+ // is a problem or a race they can ignore.
1256
+ const params = { accept };
1257
+ if (promptText !== undefined)
1258
+ params.promptText = promptText;
1259
+ await cdp(tabId, 'Page.handleJavaScriptDialog', params);
1260
+ return {
1261
+ kind: 'tool.response',
1262
+ id: msg.id,
1263
+ ok: true,
1264
+ result: { accepted: accept },
1265
+ };
1266
+ }
1267
+ async function handleSetPermission(msg, str) {
1268
+ const origin = str('origin');
1269
+ const name = str('name');
1270
+ const state_ = str('state');
1271
+ // CDP `Browser.setPermission` requires a browser-level target,
1272
+ // which `chrome.debugger.attach({ tabId })` does NOT give us in
1273
+ // MV3. We use `chrome.contentSettings` instead — that surface IS
1274
+ // available to extensions and covers the realistic permissions
1275
+ // an agent needs to pre-set (geolocation, notifications, camera,
1276
+ // microphone, clipboard, sensors).
1277
+ const settingKey = CONTENT_SETTING_BY_PERMISSION[name];
1278
+ if (!settingKey) {
1279
+ return {
1280
+ kind: 'tool.response',
1281
+ id: msg.id,
1282
+ ok: false,
1283
+ error: `set_permission: '${name}' is not supported by chrome.contentSettings in MV3. Supported names: ${Object.keys(CONTENT_SETTING_BY_PERMISSION).join(', ')}.`,
1284
+ };
1285
+ }
1286
+ const setting = STATE_TO_CONTENT_SETTING[state_];
1287
+ if (!setting) {
1288
+ return {
1289
+ kind: 'tool.response',
1290
+ id: msg.id,
1291
+ ok: false,
1292
+ error: `set_permission: unknown state '${state_}' (expected granted | denied | prompt).`,
1293
+ };
1294
+ }
1295
+ // `chrome.contentSettings.<name>.set({ primaryPattern, setting })`
1296
+ // requires a URL pattern, not a bare origin. Append `/*` so the
1297
+ // pattern covers every path on the origin.
1298
+ const primaryPattern = origin.endsWith('/*') ? origin : `${origin}/*`;
1299
+ try {
1300
+ await applyContentSetting(settingKey, primaryPattern, setting);
1301
+ }
1302
+ catch (err) {
1303
+ return {
1304
+ kind: 'tool.response',
1305
+ id: msg.id,
1306
+ ok: false,
1307
+ error: `set_permission: ${err instanceof Error ? err.message : String(err)}`,
1308
+ };
1309
+ }
1310
+ return {
1311
+ kind: 'tool.response',
1312
+ id: msg.id,
1313
+ ok: true,
1314
+ result: { origin, name, state: state_, applied_as: settingKey },
1315
+ };
1316
+ }
865
1317
  async function handleTool(state, msg) {
866
1318
  const tabId = state.tabId;
867
1319
  // Params come over the wire as JSON: each field is unknown until we
@@ -872,40 +1324,12 @@ async function handleTool(state, msg) {
872
1324
  const optStr = (key) => typeof p[key] === 'string' ? p[key] : undefined;
873
1325
  try {
874
1326
  switch (msg.tool) {
875
- case 'ping': {
876
- const tab = await chrome.tabs.get(tabId);
877
- return {
878
- kind: 'tool.response',
879
- id: msg.id,
880
- ok: true,
881
- result: { title: tab.title ?? '', url: tab.url ?? '' },
882
- };
883
- }
884
- case 'navigate': {
885
- const url = str('url');
886
- const waitForLoadFlag = p.wait_for_load !== false;
887
- await cdp(tabId, 'Page.navigate', { url });
888
- if (waitForLoadFlag)
889
- await waitForLoad(tabId);
890
- const tab = await chrome.tabs.get(tabId);
891
- return {
892
- kind: 'tool.response',
893
- id: msg.id,
894
- ok: true,
895
- result: { url: tab.url ?? '', title: tab.title ?? '' },
896
- };
897
- }
898
- case 'snapshot': {
899
- const value = await evaluateInTab(tabId, buildSnapshotJs({
900
- within_selector: optStr('within_selector'),
901
- only_interactive: p.only_interactive === true,
902
- exclude: Array.isArray(p.exclude)
903
- ? p.exclude.filter((x) => typeof x === 'string')
904
- : undefined,
905
- max_interactive: typeof p.max_interactive === 'number' ? p.max_interactive : undefined,
906
- }));
907
- return { kind: 'tool.response', id: msg.id, ok: true, result: value };
908
- }
1327
+ case 'ping':
1328
+ return await handlePing(tabId, msg);
1329
+ case 'navigate':
1330
+ return await handleNavigate(tabId, msg, p, str);
1331
+ case 'snapshot':
1332
+ return await handleSnapshot(tabId, msg, p, optStr);
909
1333
  case 'state': {
910
1334
  const value = await evaluateInTab(tabId, buildStateJs());
911
1335
  return { kind: 'tool.response', id: msg.id, ok: true, result: value };
@@ -921,45 +1345,14 @@ async function handleTool(state, msg) {
921
1345
  }
922
1346
  return { kind: 'tool.response', id: msg.id, ok: true, result: outcome.result };
923
1347
  }
924
- case 'canvas_screenshot': {
925
- const selector = optStr('selector');
926
- const format = optStr('format') === 'jpeg' ? 'jpeg' : 'png';
927
- const regionRaw = p.region;
928
- let region;
929
- if (regionRaw && typeof regionRaw === 'object') {
930
- const r = regionRaw;
931
- if (typeof r.x === 'number' &&
932
- typeof r.y === 'number' &&
933
- typeof r.w === 'number' &&
934
- typeof r.h === 'number') {
935
- region = { x: r.x, y: r.y, w: r.w, h: r.h };
936
- }
937
- }
938
- const value = await evaluateInTab(tabId, buildCanvasScreenshotJs({ selector, region, format }));
939
- return { kind: 'tool.response', id: msg.id, ok: true, result: value };
940
- }
941
- case 'console': {
942
- const level = optStr('level');
943
- const entries = level
944
- ? state.consoleBuffer.filter((e) => e.level === level)
945
- : state.consoleBuffer;
946
- return { kind: 'tool.response', id: msg.id, ok: true, result: entries };
947
- }
948
- case 'network': {
949
- const filter = optStr('url_filter')?.toLowerCase();
950
- const list = state.networkOrder
951
- .map((id) => state.networkBuffer.get(id))
952
- .filter((e) => e !== undefined)
953
- .filter((e) => (filter ? e.url.toLowerCase().includes(filter) : true));
954
- return { kind: 'tool.response', id: msg.id, ok: true, result: list };
955
- }
956
- case 'network_body': {
957
- const requestId = str('request_id');
958
- const body = await cdp(tabId, 'Network.getResponseBody', {
959
- requestId,
960
- });
961
- return { kind: 'tool.response', id: msg.id, ok: true, result: body };
962
- }
1348
+ case 'canvas_screenshot':
1349
+ return await handleCanvasScreenshot(tabId, msg, p, optStr);
1350
+ case 'console':
1351
+ return handleConsole(state, msg, optStr);
1352
+ case 'network':
1353
+ return handleNetwork(state, msg, optStr);
1354
+ case 'network_body':
1355
+ return await handleNetworkBody(tabId, msg, str);
963
1356
  case 'click': {
964
1357
  const outcome = await performClick(tabId, {
965
1358
  selector: str('selector'),
@@ -1001,308 +1394,10 @@ async function handleTool(state, msg) {
1001
1394
  }
1002
1395
  return { kind: 'tool.response', id: msg.id, ok: true, result: outcome.result };
1003
1396
  }
1004
- case 'flow': {
1005
- // Wire-boundary narrowing: `p.steps` is untrusted JSON. Keep only
1006
- // object-shaped entries here — `runFlow`'s own `stepKind()` guard
1007
- // re-validates each one at runtime regardless, so a malformed
1008
- // entry fails the flow cleanly instead of throwing.
1009
- const rawSteps = Array.isArray(p.steps)
1010
- ? p.steps.filter((s) => typeof s === 'object' && s !== null)
1011
- : [];
1012
- const flowResult = await runFlow(rawSteps, {
1013
- performFind: (params) => performFind(tabId, params),
1014
- performClick: (params) => performClick(tabId, params),
1015
- performType: (params) => performType(tabId, params),
1016
- performPress: (params) => performPress(tabId, params),
1017
- performWaitFor: (params) => performWaitFor(tabId, state, params),
1018
- buildRecoverySnapshot: () => evaluateInTab(tabId, buildSnapshotJs({ only_interactive: true, max_interactive: 40 })),
1019
- });
1020
- // The wire-level response is ok:true whenever the flow RAN (even a
1021
- // failed step is a legitimate business outcome, same pattern as
1022
- // wait_for's matched:false) — flowResult itself carries the
1023
- // ok:true/false the agent reads.
1024
- return { kind: 'tool.response', id: msg.id, ok: true, result: flowResult };
1025
- }
1026
- case 'drag': {
1027
- const optNum = (key) => {
1028
- const v = p[key];
1029
- return typeof v === 'number' && Number.isFinite(v) && v >= 0 ? v : undefined;
1030
- };
1031
- const clamp = (v, max) => Math.min(v, max);
1032
- const fromSelector = optStr('from_selector');
1033
- const toSelector = optStr('to_selector');
1034
- const fromXRaw = optNum('from_x');
1035
- const fromYRaw = optNum('from_y');
1036
- const toXRaw = optNum('to_x');
1037
- const toYRaw = optNum('to_y');
1038
- // Cap every duration that ends up in a setTimeout to keep CodeQL's
1039
- // "user-controlled timer duration" check happy and to prevent a
1040
- // misbehaving agent from parking the bridge for hours on a typo.
1041
- const durationMs = clamp(optNum('duration_ms') ?? 1500, MAX_DRAG_DURATION_MS);
1042
- const holdBeforeMoveMs = clamp(optNum('hold_before_move_ms') ?? 0, MAX_DRAG_HOLD_MS);
1043
- const holdBeforeReleaseMs = clamp(optNum('hold_before_release_ms') ?? 0, MAX_DRAG_HOLD_MS);
1044
- const hasFromCoords = fromXRaw !== undefined && fromYRaw !== undefined;
1045
- const hasToCoords = toXRaw !== undefined && toYRaw !== undefined;
1046
- if (!fromSelector && !hasFromCoords) {
1047
- return {
1048
- kind: 'tool.response',
1049
- id: msg.id,
1050
- ok: false,
1051
- error: 'drag: provide from_selector or both from_x and from_y',
1052
- };
1053
- }
1054
- if (!toSelector && !hasToCoords) {
1055
- return {
1056
- kind: 'tool.response',
1057
- id: msg.id,
1058
- ok: false,
1059
- error: 'drag: provide to_selector or both to_x and to_y',
1060
- };
1061
- }
1062
- const probeExpr = buildDragProbeJs({
1063
- from_selector: fromSelector,
1064
- to_selector: toSelector,
1065
- from_x: fromXRaw,
1066
- from_y: fromYRaw,
1067
- to_x: toXRaw,
1068
- to_y: toYRaw,
1069
- });
1070
- const probe = await evaluateInTab(tabId, probeExpr);
1071
- if (probe.err) {
1072
- return { kind: 'tool.response', id: msg.id, ok: false, error: `drag: ${probe.err}` };
1073
- }
1074
- if (!probe.from || !probe.to) {
1075
- return {
1076
- kind: 'tool.response',
1077
- id: msg.id,
1078
- ok: false,
1079
- error: 'drag: could not resolve coordinates',
1080
- };
1081
- }
1082
- if (!probe.from.in_viewport) {
1083
- return {
1084
- kind: 'tool.response',
1085
- id: msg.id,
1086
- ok: false,
1087
- error: 'drag: source point is offscreen — scroll first or pass viewport coords',
1088
- };
1089
- }
1090
- if (!probe.to.in_viewport) {
1091
- return {
1092
- kind: 'tool.response',
1093
- id: msg.id,
1094
- ok: false,
1095
- error: 'drag: destination point is offscreen — scroll first or pass viewport coords',
1096
- };
1097
- }
1098
- const fromX = probe.from.x;
1099
- const fromY = probe.from.y;
1100
- const toX = probe.to.x;
1101
- const toY = probe.to.y;
1102
- const isDraggable = !!probe.draggable;
1103
- // ~30fps interpolation; minimum 2 steps so the path actually has a midpoint.
1104
- const steps = durationMs > 0 ? Math.max(2, Math.round(durationMs / 33)) : 1;
1105
- const stepDelayMs = steps > 0 ? durationMs / steps : 0;
1106
- const eventsFired = [];
1107
- let dragMode = 'pointer';
1108
- let interceptReceived = false;
1109
- const dragStart = Date.now();
1110
- let interceptionEnabled = false;
1111
- const interpolate = (t) => ({
1112
- x: fromX + (toX - fromX) * t,
1113
- y: fromY + (toY - fromY) * t,
1114
- });
1115
- // Mouse move WITH the left button held down. Chrome's HTML5 drag
1116
- // system only treats movement as a drag when the button state is
1117
- // signalled on every move after mousePressed; omitting it makes
1118
- // Blink ignore the move for drag-detection purposes.
1119
- const moveHeld = (x, y) => cdp(tabId, 'Input.dispatchMouseEvent', {
1120
- type: 'mouseMoved',
1121
- x,
1122
- y,
1123
- button: 'left',
1124
- buttons: 1,
1125
- });
1126
- // Wiggle distance has to comfortably clear `kDragThreshold` (~5px on
1127
- // Mac, 4px on Linux/Windows). 5px landed *right* at the boundary and
1128
- // Chrome did not start the drag — 20px crosses it on every platform.
1129
- const WIGGLE_PX = 20;
1130
- // Time to wait for Input.dragIntercepted after the wiggle. 120ms was
1131
- // enough on local-only synthetic tests but flaky in real Chrome —
1132
- // 250ms is comfortable without dragging out the overall latency.
1133
- const INTERCEPT_TIMEOUT_MS = 250;
1134
- try {
1135
- if (isDraggable) {
1136
- try {
1137
- await cdp(tabId, 'Input.setInterceptDrags', { enabled: true });
1138
- interceptionEnabled = true;
1139
- }
1140
- catch {
1141
- // setInterceptDrags is experimental — fall back to pointer mode silently.
1142
- }
1143
- }
1144
- if (interceptionEnabled) {
1145
- // Arm the listener BEFORE the press+wiggle that may trigger it.
1146
- const interceptPromise = waitForCdpEvent(tabId, 'Input.dragIntercepted', INTERCEPT_TIMEOUT_MS);
1147
- await cdp(tabId, 'Input.dispatchMouseEvent', {
1148
- type: 'mouseMoved',
1149
- x: fromX,
1150
- y: fromY,
1151
- });
1152
- await cdp(tabId, 'Input.dispatchMouseEvent', {
1153
- type: 'mousePressed',
1154
- x: fromX,
1155
- y: fromY,
1156
- button: 'left',
1157
- clickCount: 1,
1158
- });
1159
- if (holdBeforeMoveMs > 0)
1160
- await sleep(holdBeforeMoveMs);
1161
- // Wiggle toward the destination so Chrome's native drag system
1162
- // crosses its activation threshold. Direction matters for libs
1163
- // that have direction-sensitive activation constraints.
1164
- const dx = toX - fromX;
1165
- const dy = toY - fromY;
1166
- const len = Math.hypot(dx, dy) || 1;
1167
- const wx = fromX + (dx / len) * WIGGLE_PX;
1168
- const wy = fromY + (dy / len) * WIGGLE_PX;
1169
- await moveHeld(wx, wy);
1170
- const intercepted = await interceptPromise;
1171
- if (intercepted) {
1172
- dragMode = 'html5';
1173
- interceptReceived = true;
1174
- eventsFired.push('Input.dragIntercepted');
1175
- const dragData = intercepted.data;
1176
- for (let i = 1; i <= steps; i++) {
1177
- const t = i / steps;
1178
- const { x, y } = interpolate(t);
1179
- await moveHeld(x, y);
1180
- await cdp(tabId, 'Input.dispatchDragEvent', {
1181
- type: 'dragOver',
1182
- x,
1183
- y,
1184
- data: dragData,
1185
- });
1186
- if (i === 1)
1187
- eventsFired.push('dragOver');
1188
- if (stepDelayMs > 0)
1189
- await sleep(stepDelayMs);
1190
- }
1191
- await cdp(tabId, 'Input.dispatchDragEvent', {
1192
- type: 'dragEnter',
1193
- x: toX,
1194
- y: toY,
1195
- data: dragData,
1196
- });
1197
- eventsFired.push('dragEnter');
1198
- await cdp(tabId, 'Input.dispatchDragEvent', {
1199
- type: 'dragOver',
1200
- x: toX,
1201
- y: toY,
1202
- data: dragData,
1203
- });
1204
- if (holdBeforeReleaseMs > 0)
1205
- await sleep(holdBeforeReleaseMs);
1206
- await cdp(tabId, 'Input.dispatchDragEvent', {
1207
- type: 'drop',
1208
- x: toX,
1209
- y: toY,
1210
- data: dragData,
1211
- });
1212
- eventsFired.push('drop');
1213
- await cdp(tabId, 'Input.dispatchMouseEvent', {
1214
- type: 'mouseReleased',
1215
- x: toX,
1216
- y: toY,
1217
- button: 'left',
1218
- clickCount: 1,
1219
- });
1220
- }
1221
- else {
1222
- // Element was tagged draggable but no native drag fired — the page
1223
- // either preventDefault'd dragstart or wires its own pointer logic.
1224
- // Continue with pointer-only events from where we already pressed.
1225
- dragMode = 'pointer';
1226
- for (let i = 1; i <= steps; i++) {
1227
- const t = i / steps;
1228
- const { x, y } = interpolate(t);
1229
- await moveHeld(x, y);
1230
- if (stepDelayMs > 0)
1231
- await sleep(stepDelayMs);
1232
- }
1233
- if (holdBeforeReleaseMs > 0)
1234
- await sleep(holdBeforeReleaseMs);
1235
- await cdp(tabId, 'Input.dispatchMouseEvent', {
1236
- type: 'mouseReleased',
1237
- x: toX,
1238
- y: toY,
1239
- button: 'left',
1240
- clickCount: 1,
1241
- });
1242
- }
1243
- }
1244
- else {
1245
- // Pointer-only branch: no native drag involvement at all.
1246
- dragMode = 'pointer';
1247
- await cdp(tabId, 'Input.dispatchMouseEvent', {
1248
- type: 'mouseMoved',
1249
- x: fromX,
1250
- y: fromY,
1251
- });
1252
- await cdp(tabId, 'Input.dispatchMouseEvent', {
1253
- type: 'mousePressed',
1254
- x: fromX,
1255
- y: fromY,
1256
- button: 'left',
1257
- clickCount: 1,
1258
- });
1259
- if (holdBeforeMoveMs > 0)
1260
- await sleep(holdBeforeMoveMs);
1261
- for (let i = 1; i <= steps; i++) {
1262
- const t = i / steps;
1263
- const { x, y } = interpolate(t);
1264
- await moveHeld(x, y);
1265
- if (stepDelayMs > 0)
1266
- await sleep(stepDelayMs);
1267
- }
1268
- if (holdBeforeReleaseMs > 0)
1269
- await sleep(holdBeforeReleaseMs);
1270
- await cdp(tabId, 'Input.dispatchMouseEvent', {
1271
- type: 'mouseReleased',
1272
- x: toX,
1273
- y: toY,
1274
- button: 'left',
1275
- clickCount: 1,
1276
- });
1277
- }
1278
- }
1279
- finally {
1280
- // Critical: leaving interception on would block the human user from
1281
- // dragging anything in this tab. Best-effort cleanup, swallow errors.
1282
- if (interceptionEnabled) {
1283
- try {
1284
- await cdp(tabId, 'Input.setInterceptDrags', { enabled: false });
1285
- }
1286
- catch {
1287
- // ignore
1288
- }
1289
- }
1290
- }
1291
- return {
1292
- kind: 'tool.response',
1293
- id: msg.id,
1294
- ok: true,
1295
- result: {
1296
- from: { x: fromX, y: fromY, selector: fromSelector ?? null },
1297
- to: { x: toX, y: toY, selector: toSelector ?? null },
1298
- duration_ms_actual: Date.now() - dragStart,
1299
- drag_mode: dragMode,
1300
- interception_attempted: interceptionEnabled,
1301
- intercept_received: interceptReceived,
1302
- events_fired: eventsFired,
1303
- },
1304
- };
1305
- }
1397
+ case 'flow':
1398
+ return await handleFlow(tabId, state, msg, p);
1399
+ case 'drag':
1400
+ return await handleDrag(tabId, msg, p, optStr);
1306
1401
  case 'evaluate': {
1307
1402
  const expression = str('expression');
1308
1403
  const value = await evaluateInTab(tabId, expression);
@@ -1322,73 +1417,10 @@ async function handleTool(state, msg) {
1322
1417
  }
1323
1418
  return { kind: 'tool.response', id: msg.id, ok: true, result: outcome.result };
1324
1419
  }
1325
- case 'dialog_respond': {
1326
- const accept = p.accept === true;
1327
- const promptText = optStr('prompt_text');
1328
- // No probing — if there is no dialog open, CDP returns an error.
1329
- // We propagate it; the caller decides if "no dialog to respond to"
1330
- // is a problem or a race they can ignore.
1331
- const params = { accept };
1332
- if (promptText !== undefined)
1333
- params.promptText = promptText;
1334
- await cdp(tabId, 'Page.handleJavaScriptDialog', params);
1335
- return {
1336
- kind: 'tool.response',
1337
- id: msg.id,
1338
- ok: true,
1339
- result: { accepted: accept },
1340
- };
1341
- }
1342
- case 'set_permission': {
1343
- const origin = str('origin');
1344
- const name = str('name');
1345
- const state_ = str('state');
1346
- // CDP `Browser.setPermission` requires a browser-level target,
1347
- // which `chrome.debugger.attach({ tabId })` does NOT give us in
1348
- // MV3. We use `chrome.contentSettings` instead — that surface IS
1349
- // available to extensions and covers the realistic permissions
1350
- // an agent needs to pre-set (geolocation, notifications, camera,
1351
- // microphone, clipboard, sensors).
1352
- const settingKey = CONTENT_SETTING_BY_PERMISSION[name];
1353
- if (!settingKey) {
1354
- return {
1355
- kind: 'tool.response',
1356
- id: msg.id,
1357
- ok: false,
1358
- error: `set_permission: '${name}' is not supported by chrome.contentSettings in MV3. Supported names: ${Object.keys(CONTENT_SETTING_BY_PERMISSION).join(', ')}.`,
1359
- };
1360
- }
1361
- const setting = STATE_TO_CONTENT_SETTING[state_];
1362
- if (!setting) {
1363
- return {
1364
- kind: 'tool.response',
1365
- id: msg.id,
1366
- ok: false,
1367
- error: `set_permission: unknown state '${state_}' (expected granted | denied | prompt).`,
1368
- };
1369
- }
1370
- // `chrome.contentSettings.<name>.set({ primaryPattern, setting })`
1371
- // requires a URL pattern, not a bare origin. Append `/*` so the
1372
- // pattern covers every path on the origin.
1373
- const primaryPattern = origin.endsWith('/*') ? origin : `${origin}/*`;
1374
- try {
1375
- await applyContentSetting(settingKey, primaryPattern, setting);
1376
- }
1377
- catch (err) {
1378
- return {
1379
- kind: 'tool.response',
1380
- id: msg.id,
1381
- ok: false,
1382
- error: `set_permission: ${err instanceof Error ? err.message : String(err)}`,
1383
- };
1384
- }
1385
- return {
1386
- kind: 'tool.response',
1387
- id: msg.id,
1388
- ok: true,
1389
- result: { origin, name, state: state_, applied_as: settingKey },
1390
- };
1391
- }
1420
+ case 'dialog_respond':
1421
+ return await handleDialogRespond(tabId, msg, p, optStr);
1422
+ case 'set_permission':
1423
+ return await handleSetPermission(msg, str);
1392
1424
  default:
1393
1425
  return { kind: 'tool.response', id: msg.id, ok: false, error: `Unknown tool: ${msg.tool}` };
1394
1426
  }