@jobshimo/browser-link 0.23.3 → 0.23.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/dist/cdp/flow.d.ts +28 -17
- package/dist/cdp/flow.js +19 -10
- package/dist/cdp/flow.js.map +1 -1
- package/dist/cdp/keymap.d.ts +6 -3
- package/dist/cdp/keymap.js +3 -1
- package/dist/cdp/keymap.js.map +1 -1
- package/dist/cdp/settle.d.ts +5 -2
- package/dist/cdp/settle.js +7 -4
- package/dist/cdp/settle.js.map +1 -1
- package/dist/commands/map.d.ts +5 -0
- package/dist/commands/map.js +7 -1
- package/dist/commands/map.js.map +1 -1
- package/dist/extension/background.js +474 -442
- package/dist/extension/background.js.map +1 -1
- package/dist/extension/flow.d.ts +26 -22
- package/dist/extension/flow.js +18 -16
- package/dist/extension/flow.js.map +1 -1
- package/dist/extension/keymap.d.ts +6 -3
- package/dist/extension/keymap.js +3 -2
- package/dist/extension/keymap.js.map +1 -1
- package/dist/extension/manifest.json +1 -1
- package/dist/extension/settle.d.ts +6 -4
- package/dist/extension/settle.js +8 -6
- package/dist/extension/settle.js.map +1 -1
- package/dist/tools/browser-definitions.js +3 -2
- package/dist/tools/browser-definitions.js.map +1 -1
- package/package.json +1 -1
|
@@ -862,6 +862,458 @@ async function performWaitFor(tabId, state, params) {
|
|
|
862
862
|
return { ok: false, error: err instanceof Error ? err.message : String(err) };
|
|
863
863
|
}
|
|
864
864
|
}
|
|
865
|
+
/**
|
|
866
|
+
* `handlePing` through `handleSetPermission` — the extracted bodies of the
|
|
867
|
+
* remaining standalone tool cases in `handleTool` below that never got the
|
|
868
|
+
* `perform*` treatment above. Each takes exactly the locals its original
|
|
869
|
+
* `case` block closed over (`tabId`, `state`, `msg`, the raw params `p`,
|
|
870
|
+
* and/or the `str`/`optStr` readers) and returns the same wire-level
|
|
871
|
+
* `tool.response` envelope the case used to build inline — pure code
|
|
872
|
+
* motion, not a rewrite of tool behavior.
|
|
873
|
+
*/
|
|
874
|
+
async function handlePing(tabId, msg) {
|
|
875
|
+
const tab = await chrome.tabs.get(tabId);
|
|
876
|
+
return {
|
|
877
|
+
kind: 'tool.response',
|
|
878
|
+
id: msg.id,
|
|
879
|
+
ok: true,
|
|
880
|
+
result: { title: tab.title ?? '', url: tab.url ?? '' },
|
|
881
|
+
};
|
|
882
|
+
}
|
|
883
|
+
async function handleNavigate(tabId, msg, p, str) {
|
|
884
|
+
const url = str('url');
|
|
885
|
+
const waitForLoadFlag = p.wait_for_load !== false;
|
|
886
|
+
await cdp(tabId, 'Page.navigate', { url });
|
|
887
|
+
if (waitForLoadFlag)
|
|
888
|
+
await waitForLoad(tabId);
|
|
889
|
+
const tab = await chrome.tabs.get(tabId);
|
|
890
|
+
return {
|
|
891
|
+
kind: 'tool.response',
|
|
892
|
+
id: msg.id,
|
|
893
|
+
ok: true,
|
|
894
|
+
result: { url: tab.url ?? '', title: tab.title ?? '' },
|
|
895
|
+
};
|
|
896
|
+
}
|
|
897
|
+
async function handleSnapshot(tabId, msg, p, optStr) {
|
|
898
|
+
const value = await evaluateInTab(tabId, buildSnapshotJs({
|
|
899
|
+
within_selector: optStr('within_selector'),
|
|
900
|
+
only_interactive: p.only_interactive === true,
|
|
901
|
+
exclude: Array.isArray(p.exclude)
|
|
902
|
+
? p.exclude.filter((x) => typeof x === 'string')
|
|
903
|
+
: undefined,
|
|
904
|
+
max_interactive: typeof p.max_interactive === 'number' ? p.max_interactive : undefined,
|
|
905
|
+
}));
|
|
906
|
+
return { kind: 'tool.response', id: msg.id, ok: true, result: value };
|
|
907
|
+
}
|
|
908
|
+
async function handleCanvasScreenshot(tabId, msg, p, optStr) {
|
|
909
|
+
const selector = optStr('selector');
|
|
910
|
+
const format = optStr('format') === 'jpeg' ? 'jpeg' : 'png';
|
|
911
|
+
const regionRaw = p.region;
|
|
912
|
+
let region;
|
|
913
|
+
if (regionRaw && typeof regionRaw === 'object') {
|
|
914
|
+
const r = regionRaw;
|
|
915
|
+
if (typeof r.x === 'number' &&
|
|
916
|
+
typeof r.y === 'number' &&
|
|
917
|
+
typeof r.w === 'number' &&
|
|
918
|
+
typeof r.h === 'number') {
|
|
919
|
+
region = { x: r.x, y: r.y, w: r.w, h: r.h };
|
|
920
|
+
}
|
|
921
|
+
}
|
|
922
|
+
const value = await evaluateInTab(tabId, buildCanvasScreenshotJs({ selector, region, format }));
|
|
923
|
+
return { kind: 'tool.response', id: msg.id, ok: true, result: value };
|
|
924
|
+
}
|
|
925
|
+
function handleConsole(state, msg, optStr) {
|
|
926
|
+
const level = optStr('level');
|
|
927
|
+
const entries = level
|
|
928
|
+
? state.consoleBuffer.filter((e) => e.level === level)
|
|
929
|
+
: state.consoleBuffer;
|
|
930
|
+
return { kind: 'tool.response', id: msg.id, ok: true, result: entries };
|
|
931
|
+
}
|
|
932
|
+
function handleNetwork(state, msg, optStr) {
|
|
933
|
+
const filter = optStr('url_filter')?.toLowerCase();
|
|
934
|
+
const list = state.networkOrder
|
|
935
|
+
.map((id) => state.networkBuffer.get(id))
|
|
936
|
+
.filter((e) => e !== undefined)
|
|
937
|
+
.filter((e) => (filter ? e.url.toLowerCase().includes(filter) : true));
|
|
938
|
+
return { kind: 'tool.response', id: msg.id, ok: true, result: list };
|
|
939
|
+
}
|
|
940
|
+
async function handleNetworkBody(tabId, msg, str) {
|
|
941
|
+
const requestId = str('request_id');
|
|
942
|
+
const body = await cdp(tabId, 'Network.getResponseBody', {
|
|
943
|
+
requestId,
|
|
944
|
+
});
|
|
945
|
+
return { kind: 'tool.response', id: msg.id, ok: true, result: body };
|
|
946
|
+
}
|
|
947
|
+
async function handleFlow(tabId, state, msg, p) {
|
|
948
|
+
// Wire-boundary narrowing: `p.steps` is untrusted JSON. Keep only
|
|
949
|
+
// object-shaped entries here — `runFlow`'s own `stepKind()` guard
|
|
950
|
+
// re-validates each one at runtime regardless, so a malformed
|
|
951
|
+
// entry fails the flow cleanly instead of throwing.
|
|
952
|
+
const rawSteps = Array.isArray(p.steps)
|
|
953
|
+
? p.steps.filter((s) => typeof s === 'object' && s !== null)
|
|
954
|
+
: [];
|
|
955
|
+
const flowResult = await runFlow(rawSteps, {
|
|
956
|
+
performFind: (params) => performFind(tabId, params),
|
|
957
|
+
performClick: (params) => performClick(tabId, params),
|
|
958
|
+
performType: (params) => performType(tabId, params),
|
|
959
|
+
performPress: (params) => performPress(tabId, params),
|
|
960
|
+
performWaitFor: (params) => performWaitFor(tabId, state, params),
|
|
961
|
+
buildRecoverySnapshot: () => evaluateInTab(tabId, buildSnapshotJs({ only_interactive: true, max_interactive: 40 })),
|
|
962
|
+
});
|
|
963
|
+
// The wire-level response is ok:true whenever the flow RAN (even a
|
|
964
|
+
// failed step is a legitimate business outcome, same pattern as
|
|
965
|
+
// wait_for's matched:false) — flowResult itself carries the
|
|
966
|
+
// ok:true/false the agent reads.
|
|
967
|
+
return { kind: 'tool.response', id: msg.id, ok: true, result: flowResult };
|
|
968
|
+
}
|
|
969
|
+
async function handleDrag(tabId, msg, p, optStr) {
|
|
970
|
+
const optNum = (key) => {
|
|
971
|
+
const v = p[key];
|
|
972
|
+
return typeof v === 'number' && Number.isFinite(v) && v >= 0 ? v : undefined;
|
|
973
|
+
};
|
|
974
|
+
const clamp = (v, max) => Math.min(v, max);
|
|
975
|
+
const fromSelector = optStr('from_selector');
|
|
976
|
+
const toSelector = optStr('to_selector');
|
|
977
|
+
const fromXRaw = optNum('from_x');
|
|
978
|
+
const fromYRaw = optNum('from_y');
|
|
979
|
+
const toXRaw = optNum('to_x');
|
|
980
|
+
const toYRaw = optNum('to_y');
|
|
981
|
+
// Cap every duration that ends up in a setTimeout to keep CodeQL's
|
|
982
|
+
// "user-controlled timer duration" check happy and to prevent a
|
|
983
|
+
// misbehaving agent from parking the bridge for hours on a typo.
|
|
984
|
+
const durationMs = clamp(optNum('duration_ms') ?? 1500, MAX_DRAG_DURATION_MS);
|
|
985
|
+
const holdBeforeMoveMs = clamp(optNum('hold_before_move_ms') ?? 0, MAX_DRAG_HOLD_MS);
|
|
986
|
+
const holdBeforeReleaseMs = clamp(optNum('hold_before_release_ms') ?? 0, MAX_DRAG_HOLD_MS);
|
|
987
|
+
const hasFromCoords = fromXRaw !== undefined && fromYRaw !== undefined;
|
|
988
|
+
const hasToCoords = toXRaw !== undefined && toYRaw !== undefined;
|
|
989
|
+
if (!fromSelector && !hasFromCoords) {
|
|
990
|
+
return {
|
|
991
|
+
kind: 'tool.response',
|
|
992
|
+
id: msg.id,
|
|
993
|
+
ok: false,
|
|
994
|
+
error: 'drag: provide from_selector or both from_x and from_y',
|
|
995
|
+
};
|
|
996
|
+
}
|
|
997
|
+
if (!toSelector && !hasToCoords) {
|
|
998
|
+
return {
|
|
999
|
+
kind: 'tool.response',
|
|
1000
|
+
id: msg.id,
|
|
1001
|
+
ok: false,
|
|
1002
|
+
error: 'drag: provide to_selector or both to_x and to_y',
|
|
1003
|
+
};
|
|
1004
|
+
}
|
|
1005
|
+
const probeExpr = buildDragProbeJs({
|
|
1006
|
+
from_selector: fromSelector,
|
|
1007
|
+
to_selector: toSelector,
|
|
1008
|
+
from_x: fromXRaw,
|
|
1009
|
+
from_y: fromYRaw,
|
|
1010
|
+
to_x: toXRaw,
|
|
1011
|
+
to_y: toYRaw,
|
|
1012
|
+
});
|
|
1013
|
+
const probe = await evaluateInTab(tabId, probeExpr);
|
|
1014
|
+
if (probe.err) {
|
|
1015
|
+
return { kind: 'tool.response', id: msg.id, ok: false, error: `drag: ${probe.err}` };
|
|
1016
|
+
}
|
|
1017
|
+
if (!probe.from || !probe.to) {
|
|
1018
|
+
return {
|
|
1019
|
+
kind: 'tool.response',
|
|
1020
|
+
id: msg.id,
|
|
1021
|
+
ok: false,
|
|
1022
|
+
error: 'drag: could not resolve coordinates',
|
|
1023
|
+
};
|
|
1024
|
+
}
|
|
1025
|
+
if (!probe.from.in_viewport) {
|
|
1026
|
+
return {
|
|
1027
|
+
kind: 'tool.response',
|
|
1028
|
+
id: msg.id,
|
|
1029
|
+
ok: false,
|
|
1030
|
+
error: 'drag: source point is offscreen — scroll first or pass viewport coords',
|
|
1031
|
+
};
|
|
1032
|
+
}
|
|
1033
|
+
if (!probe.to.in_viewport) {
|
|
1034
|
+
return {
|
|
1035
|
+
kind: 'tool.response',
|
|
1036
|
+
id: msg.id,
|
|
1037
|
+
ok: false,
|
|
1038
|
+
error: 'drag: destination point is offscreen — scroll first or pass viewport coords',
|
|
1039
|
+
};
|
|
1040
|
+
}
|
|
1041
|
+
const fromX = probe.from.x;
|
|
1042
|
+
const fromY = probe.from.y;
|
|
1043
|
+
const toX = probe.to.x;
|
|
1044
|
+
const toY = probe.to.y;
|
|
1045
|
+
const isDraggable = !!probe.draggable;
|
|
1046
|
+
// ~30fps interpolation; minimum 2 steps so the path actually has a midpoint.
|
|
1047
|
+
const steps = durationMs > 0 ? Math.max(2, Math.round(durationMs / 33)) : 1;
|
|
1048
|
+
const stepDelayMs = steps > 0 ? durationMs / steps : 0;
|
|
1049
|
+
const eventsFired = [];
|
|
1050
|
+
// Every branch below reassigns `dragMode` before the final `return` reads it.
|
|
1051
|
+
let dragMode = 'pointer'; // eslint-disable-line no-useless-assignment -- see comment above
|
|
1052
|
+
let interceptReceived = false;
|
|
1053
|
+
const dragStart = Date.now();
|
|
1054
|
+
let interceptionEnabled = false;
|
|
1055
|
+
const interpolate = (t) => ({
|
|
1056
|
+
x: fromX + (toX - fromX) * t,
|
|
1057
|
+
y: fromY + (toY - fromY) * t,
|
|
1058
|
+
});
|
|
1059
|
+
// Mouse move WITH the left button held down. Chrome's HTML5 drag
|
|
1060
|
+
// system only treats movement as a drag when the button state is
|
|
1061
|
+
// signalled on every move after mousePressed; omitting it makes
|
|
1062
|
+
// Blink ignore the move for drag-detection purposes.
|
|
1063
|
+
const moveHeld = (x, y) => cdp(tabId, 'Input.dispatchMouseEvent', {
|
|
1064
|
+
type: 'mouseMoved',
|
|
1065
|
+
x,
|
|
1066
|
+
y,
|
|
1067
|
+
button: 'left',
|
|
1068
|
+
buttons: 1,
|
|
1069
|
+
});
|
|
1070
|
+
// Wiggle distance has to comfortably clear `kDragThreshold` (~5px on
|
|
1071
|
+
// Mac, 4px on Linux/Windows). 5px landed *right* at the boundary and
|
|
1072
|
+
// Chrome did not start the drag — 20px crosses it on every platform.
|
|
1073
|
+
const WIGGLE_PX = 20;
|
|
1074
|
+
// Time to wait for Input.dragIntercepted after the wiggle. 120ms was
|
|
1075
|
+
// enough on local-only synthetic tests but flaky in real Chrome —
|
|
1076
|
+
// 250ms is comfortable without dragging out the overall latency.
|
|
1077
|
+
const INTERCEPT_TIMEOUT_MS = 250;
|
|
1078
|
+
try {
|
|
1079
|
+
if (isDraggable) {
|
|
1080
|
+
try {
|
|
1081
|
+
await cdp(tabId, 'Input.setInterceptDrags', { enabled: true });
|
|
1082
|
+
interceptionEnabled = true;
|
|
1083
|
+
}
|
|
1084
|
+
catch {
|
|
1085
|
+
// setInterceptDrags is experimental — fall back to pointer mode silently.
|
|
1086
|
+
}
|
|
1087
|
+
}
|
|
1088
|
+
if (interceptionEnabled) {
|
|
1089
|
+
// Arm the listener BEFORE the press+wiggle that may trigger it.
|
|
1090
|
+
const interceptPromise = waitForCdpEvent(tabId, 'Input.dragIntercepted', INTERCEPT_TIMEOUT_MS);
|
|
1091
|
+
await cdp(tabId, 'Input.dispatchMouseEvent', {
|
|
1092
|
+
type: 'mouseMoved',
|
|
1093
|
+
x: fromX,
|
|
1094
|
+
y: fromY,
|
|
1095
|
+
});
|
|
1096
|
+
await cdp(tabId, 'Input.dispatchMouseEvent', {
|
|
1097
|
+
type: 'mousePressed',
|
|
1098
|
+
x: fromX,
|
|
1099
|
+
y: fromY,
|
|
1100
|
+
button: 'left',
|
|
1101
|
+
clickCount: 1,
|
|
1102
|
+
});
|
|
1103
|
+
if (holdBeforeMoveMs > 0)
|
|
1104
|
+
await sleep(holdBeforeMoveMs);
|
|
1105
|
+
// Wiggle toward the destination so Chrome's native drag system
|
|
1106
|
+
// crosses its activation threshold. Direction matters for libs
|
|
1107
|
+
// that have direction-sensitive activation constraints.
|
|
1108
|
+
const dx = toX - fromX;
|
|
1109
|
+
const dy = toY - fromY;
|
|
1110
|
+
const len = Math.hypot(dx, dy) || 1;
|
|
1111
|
+
const wx = fromX + (dx / len) * WIGGLE_PX;
|
|
1112
|
+
const wy = fromY + (dy / len) * WIGGLE_PX;
|
|
1113
|
+
await moveHeld(wx, wy);
|
|
1114
|
+
const intercepted = await interceptPromise;
|
|
1115
|
+
if (intercepted) {
|
|
1116
|
+
dragMode = 'html5';
|
|
1117
|
+
interceptReceived = true;
|
|
1118
|
+
eventsFired.push('Input.dragIntercepted');
|
|
1119
|
+
const dragData = intercepted.data;
|
|
1120
|
+
for (let i = 1; i <= steps; i++) {
|
|
1121
|
+
const t = i / steps;
|
|
1122
|
+
const { x, y } = interpolate(t);
|
|
1123
|
+
await moveHeld(x, y);
|
|
1124
|
+
await cdp(tabId, 'Input.dispatchDragEvent', {
|
|
1125
|
+
type: 'dragOver',
|
|
1126
|
+
x,
|
|
1127
|
+
y,
|
|
1128
|
+
data: dragData,
|
|
1129
|
+
});
|
|
1130
|
+
if (i === 1)
|
|
1131
|
+
eventsFired.push('dragOver');
|
|
1132
|
+
if (stepDelayMs > 0)
|
|
1133
|
+
await sleep(stepDelayMs);
|
|
1134
|
+
}
|
|
1135
|
+
await cdp(tabId, 'Input.dispatchDragEvent', {
|
|
1136
|
+
type: 'dragEnter',
|
|
1137
|
+
x: toX,
|
|
1138
|
+
y: toY,
|
|
1139
|
+
data: dragData,
|
|
1140
|
+
});
|
|
1141
|
+
eventsFired.push('dragEnter');
|
|
1142
|
+
await cdp(tabId, 'Input.dispatchDragEvent', {
|
|
1143
|
+
type: 'dragOver',
|
|
1144
|
+
x: toX,
|
|
1145
|
+
y: toY,
|
|
1146
|
+
data: dragData,
|
|
1147
|
+
});
|
|
1148
|
+
if (holdBeforeReleaseMs > 0)
|
|
1149
|
+
await sleep(holdBeforeReleaseMs);
|
|
1150
|
+
await cdp(tabId, 'Input.dispatchDragEvent', {
|
|
1151
|
+
type: 'drop',
|
|
1152
|
+
x: toX,
|
|
1153
|
+
y: toY,
|
|
1154
|
+
data: dragData,
|
|
1155
|
+
});
|
|
1156
|
+
eventsFired.push('drop');
|
|
1157
|
+
await cdp(tabId, 'Input.dispatchMouseEvent', {
|
|
1158
|
+
type: 'mouseReleased',
|
|
1159
|
+
x: toX,
|
|
1160
|
+
y: toY,
|
|
1161
|
+
button: 'left',
|
|
1162
|
+
clickCount: 1,
|
|
1163
|
+
});
|
|
1164
|
+
}
|
|
1165
|
+
else {
|
|
1166
|
+
// Element was tagged draggable but no native drag fired — the page
|
|
1167
|
+
// either preventDefault'd dragstart or wires its own pointer logic.
|
|
1168
|
+
// Continue with pointer-only events from where we already pressed.
|
|
1169
|
+
dragMode = 'pointer';
|
|
1170
|
+
for (let i = 1; i <= steps; i++) {
|
|
1171
|
+
const t = i / steps;
|
|
1172
|
+
const { x, y } = interpolate(t);
|
|
1173
|
+
await moveHeld(x, y);
|
|
1174
|
+
if (stepDelayMs > 0)
|
|
1175
|
+
await sleep(stepDelayMs);
|
|
1176
|
+
}
|
|
1177
|
+
if (holdBeforeReleaseMs > 0)
|
|
1178
|
+
await sleep(holdBeforeReleaseMs);
|
|
1179
|
+
await cdp(tabId, 'Input.dispatchMouseEvent', {
|
|
1180
|
+
type: 'mouseReleased',
|
|
1181
|
+
x: toX,
|
|
1182
|
+
y: toY,
|
|
1183
|
+
button: 'left',
|
|
1184
|
+
clickCount: 1,
|
|
1185
|
+
});
|
|
1186
|
+
}
|
|
1187
|
+
}
|
|
1188
|
+
else {
|
|
1189
|
+
// Pointer-only branch: no native drag involvement at all.
|
|
1190
|
+
dragMode = 'pointer';
|
|
1191
|
+
await cdp(tabId, 'Input.dispatchMouseEvent', {
|
|
1192
|
+
type: 'mouseMoved',
|
|
1193
|
+
x: fromX,
|
|
1194
|
+
y: fromY,
|
|
1195
|
+
});
|
|
1196
|
+
await cdp(tabId, 'Input.dispatchMouseEvent', {
|
|
1197
|
+
type: 'mousePressed',
|
|
1198
|
+
x: fromX,
|
|
1199
|
+
y: fromY,
|
|
1200
|
+
button: 'left',
|
|
1201
|
+
clickCount: 1,
|
|
1202
|
+
});
|
|
1203
|
+
if (holdBeforeMoveMs > 0)
|
|
1204
|
+
await sleep(holdBeforeMoveMs);
|
|
1205
|
+
for (let i = 1; i <= steps; i++) {
|
|
1206
|
+
const t = i / steps;
|
|
1207
|
+
const { x, y } = interpolate(t);
|
|
1208
|
+
await moveHeld(x, y);
|
|
1209
|
+
if (stepDelayMs > 0)
|
|
1210
|
+
await sleep(stepDelayMs);
|
|
1211
|
+
}
|
|
1212
|
+
if (holdBeforeReleaseMs > 0)
|
|
1213
|
+
await sleep(holdBeforeReleaseMs);
|
|
1214
|
+
await cdp(tabId, 'Input.dispatchMouseEvent', {
|
|
1215
|
+
type: 'mouseReleased',
|
|
1216
|
+
x: toX,
|
|
1217
|
+
y: toY,
|
|
1218
|
+
button: 'left',
|
|
1219
|
+
clickCount: 1,
|
|
1220
|
+
});
|
|
1221
|
+
}
|
|
1222
|
+
}
|
|
1223
|
+
finally {
|
|
1224
|
+
// Critical: leaving interception on would block the human user from
|
|
1225
|
+
// dragging anything in this tab. Best-effort cleanup, swallow errors.
|
|
1226
|
+
if (interceptionEnabled) {
|
|
1227
|
+
try {
|
|
1228
|
+
await cdp(tabId, 'Input.setInterceptDrags', { enabled: false });
|
|
1229
|
+
}
|
|
1230
|
+
catch {
|
|
1231
|
+
// ignore
|
|
1232
|
+
}
|
|
1233
|
+
}
|
|
1234
|
+
}
|
|
1235
|
+
return {
|
|
1236
|
+
kind: 'tool.response',
|
|
1237
|
+
id: msg.id,
|
|
1238
|
+
ok: true,
|
|
1239
|
+
result: {
|
|
1240
|
+
from: { x: fromX, y: fromY, selector: fromSelector ?? null },
|
|
1241
|
+
to: { x: toX, y: toY, selector: toSelector ?? null },
|
|
1242
|
+
duration_ms_actual: Date.now() - dragStart,
|
|
1243
|
+
drag_mode: dragMode,
|
|
1244
|
+
interception_attempted: interceptionEnabled,
|
|
1245
|
+
intercept_received: interceptReceived,
|
|
1246
|
+
events_fired: eventsFired,
|
|
1247
|
+
},
|
|
1248
|
+
};
|
|
1249
|
+
}
|
|
1250
|
+
async function handleDialogRespond(tabId, msg, p, optStr) {
|
|
1251
|
+
const accept = p.accept === true;
|
|
1252
|
+
const promptText = optStr('prompt_text');
|
|
1253
|
+
// No probing — if there is no dialog open, CDP returns an error.
|
|
1254
|
+
// We propagate it; the caller decides if "no dialog to respond to"
|
|
1255
|
+
// is a problem or a race they can ignore.
|
|
1256
|
+
const params = { accept };
|
|
1257
|
+
if (promptText !== undefined)
|
|
1258
|
+
params.promptText = promptText;
|
|
1259
|
+
await cdp(tabId, 'Page.handleJavaScriptDialog', params);
|
|
1260
|
+
return {
|
|
1261
|
+
kind: 'tool.response',
|
|
1262
|
+
id: msg.id,
|
|
1263
|
+
ok: true,
|
|
1264
|
+
result: { accepted: accept },
|
|
1265
|
+
};
|
|
1266
|
+
}
|
|
1267
|
+
async function handleSetPermission(msg, str) {
|
|
1268
|
+
const origin = str('origin');
|
|
1269
|
+
const name = str('name');
|
|
1270
|
+
const state_ = str('state');
|
|
1271
|
+
// CDP `Browser.setPermission` requires a browser-level target,
|
|
1272
|
+
// which `chrome.debugger.attach({ tabId })` does NOT give us in
|
|
1273
|
+
// MV3. We use `chrome.contentSettings` instead — that surface IS
|
|
1274
|
+
// available to extensions and covers the realistic permissions
|
|
1275
|
+
// an agent needs to pre-set (geolocation, notifications, camera,
|
|
1276
|
+
// microphone, clipboard, sensors).
|
|
1277
|
+
const settingKey = CONTENT_SETTING_BY_PERMISSION[name];
|
|
1278
|
+
if (!settingKey) {
|
|
1279
|
+
return {
|
|
1280
|
+
kind: 'tool.response',
|
|
1281
|
+
id: msg.id,
|
|
1282
|
+
ok: false,
|
|
1283
|
+
error: `set_permission: '${name}' is not supported by chrome.contentSettings in MV3. Supported names: ${Object.keys(CONTENT_SETTING_BY_PERMISSION).join(', ')}.`,
|
|
1284
|
+
};
|
|
1285
|
+
}
|
|
1286
|
+
const setting = STATE_TO_CONTENT_SETTING[state_];
|
|
1287
|
+
if (!setting) {
|
|
1288
|
+
return {
|
|
1289
|
+
kind: 'tool.response',
|
|
1290
|
+
id: msg.id,
|
|
1291
|
+
ok: false,
|
|
1292
|
+
error: `set_permission: unknown state '${state_}' (expected granted | denied | prompt).`,
|
|
1293
|
+
};
|
|
1294
|
+
}
|
|
1295
|
+
// `chrome.contentSettings.<name>.set({ primaryPattern, setting })`
|
|
1296
|
+
// requires a URL pattern, not a bare origin. Append `/*` so the
|
|
1297
|
+
// pattern covers every path on the origin.
|
|
1298
|
+
const primaryPattern = origin.endsWith('/*') ? origin : `${origin}/*`;
|
|
1299
|
+
try {
|
|
1300
|
+
await applyContentSetting(settingKey, primaryPattern, setting);
|
|
1301
|
+
}
|
|
1302
|
+
catch (err) {
|
|
1303
|
+
return {
|
|
1304
|
+
kind: 'tool.response',
|
|
1305
|
+
id: msg.id,
|
|
1306
|
+
ok: false,
|
|
1307
|
+
error: `set_permission: ${err instanceof Error ? err.message : String(err)}`,
|
|
1308
|
+
};
|
|
1309
|
+
}
|
|
1310
|
+
return {
|
|
1311
|
+
kind: 'tool.response',
|
|
1312
|
+
id: msg.id,
|
|
1313
|
+
ok: true,
|
|
1314
|
+
result: { origin, name, state: state_, applied_as: settingKey },
|
|
1315
|
+
};
|
|
1316
|
+
}
|
|
865
1317
|
async function handleTool(state, msg) {
|
|
866
1318
|
const tabId = state.tabId;
|
|
867
1319
|
// Params come over the wire as JSON: each field is unknown until we
|
|
@@ -872,40 +1324,12 @@ async function handleTool(state, msg) {
|
|
|
872
1324
|
const optStr = (key) => typeof p[key] === 'string' ? p[key] : undefined;
|
|
873
1325
|
try {
|
|
874
1326
|
switch (msg.tool) {
|
|
875
|
-
case 'ping':
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
result: { title: tab.title ?? '', url: tab.url ?? '' },
|
|
882
|
-
};
|
|
883
|
-
}
|
|
884
|
-
case 'navigate': {
|
|
885
|
-
const url = str('url');
|
|
886
|
-
const waitForLoadFlag = p.wait_for_load !== false;
|
|
887
|
-
await cdp(tabId, 'Page.navigate', { url });
|
|
888
|
-
if (waitForLoadFlag)
|
|
889
|
-
await waitForLoad(tabId);
|
|
890
|
-
const tab = await chrome.tabs.get(tabId);
|
|
891
|
-
return {
|
|
892
|
-
kind: 'tool.response',
|
|
893
|
-
id: msg.id,
|
|
894
|
-
ok: true,
|
|
895
|
-
result: { url: tab.url ?? '', title: tab.title ?? '' },
|
|
896
|
-
};
|
|
897
|
-
}
|
|
898
|
-
case 'snapshot': {
|
|
899
|
-
const value = await evaluateInTab(tabId, buildSnapshotJs({
|
|
900
|
-
within_selector: optStr('within_selector'),
|
|
901
|
-
only_interactive: p.only_interactive === true,
|
|
902
|
-
exclude: Array.isArray(p.exclude)
|
|
903
|
-
? p.exclude.filter((x) => typeof x === 'string')
|
|
904
|
-
: undefined,
|
|
905
|
-
max_interactive: typeof p.max_interactive === 'number' ? p.max_interactive : undefined,
|
|
906
|
-
}));
|
|
907
|
-
return { kind: 'tool.response', id: msg.id, ok: true, result: value };
|
|
908
|
-
}
|
|
1327
|
+
case 'ping':
|
|
1328
|
+
return await handlePing(tabId, msg);
|
|
1329
|
+
case 'navigate':
|
|
1330
|
+
return await handleNavigate(tabId, msg, p, str);
|
|
1331
|
+
case 'snapshot':
|
|
1332
|
+
return await handleSnapshot(tabId, msg, p, optStr);
|
|
909
1333
|
case 'state': {
|
|
910
1334
|
const value = await evaluateInTab(tabId, buildStateJs());
|
|
911
1335
|
return { kind: 'tool.response', id: msg.id, ok: true, result: value };
|
|
@@ -921,45 +1345,14 @@ async function handleTool(state, msg) {
|
|
|
921
1345
|
}
|
|
922
1346
|
return { kind: 'tool.response', id: msg.id, ok: true, result: outcome.result };
|
|
923
1347
|
}
|
|
924
|
-
case 'canvas_screenshot':
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
typeof r.y === 'number' &&
|
|
933
|
-
typeof r.w === 'number' &&
|
|
934
|
-
typeof r.h === 'number') {
|
|
935
|
-
region = { x: r.x, y: r.y, w: r.w, h: r.h };
|
|
936
|
-
}
|
|
937
|
-
}
|
|
938
|
-
const value = await evaluateInTab(tabId, buildCanvasScreenshotJs({ selector, region, format }));
|
|
939
|
-
return { kind: 'tool.response', id: msg.id, ok: true, result: value };
|
|
940
|
-
}
|
|
941
|
-
case 'console': {
|
|
942
|
-
const level = optStr('level');
|
|
943
|
-
const entries = level
|
|
944
|
-
? state.consoleBuffer.filter((e) => e.level === level)
|
|
945
|
-
: state.consoleBuffer;
|
|
946
|
-
return { kind: 'tool.response', id: msg.id, ok: true, result: entries };
|
|
947
|
-
}
|
|
948
|
-
case 'network': {
|
|
949
|
-
const filter = optStr('url_filter')?.toLowerCase();
|
|
950
|
-
const list = state.networkOrder
|
|
951
|
-
.map((id) => state.networkBuffer.get(id))
|
|
952
|
-
.filter((e) => e !== undefined)
|
|
953
|
-
.filter((e) => (filter ? e.url.toLowerCase().includes(filter) : true));
|
|
954
|
-
return { kind: 'tool.response', id: msg.id, ok: true, result: list };
|
|
955
|
-
}
|
|
956
|
-
case 'network_body': {
|
|
957
|
-
const requestId = str('request_id');
|
|
958
|
-
const body = await cdp(tabId, 'Network.getResponseBody', {
|
|
959
|
-
requestId,
|
|
960
|
-
});
|
|
961
|
-
return { kind: 'tool.response', id: msg.id, ok: true, result: body };
|
|
962
|
-
}
|
|
1348
|
+
case 'canvas_screenshot':
|
|
1349
|
+
return await handleCanvasScreenshot(tabId, msg, p, optStr);
|
|
1350
|
+
case 'console':
|
|
1351
|
+
return handleConsole(state, msg, optStr);
|
|
1352
|
+
case 'network':
|
|
1353
|
+
return handleNetwork(state, msg, optStr);
|
|
1354
|
+
case 'network_body':
|
|
1355
|
+
return await handleNetworkBody(tabId, msg, str);
|
|
963
1356
|
case 'click': {
|
|
964
1357
|
const outcome = await performClick(tabId, {
|
|
965
1358
|
selector: str('selector'),
|
|
@@ -1001,308 +1394,10 @@ async function handleTool(state, msg) {
|
|
|
1001
1394
|
}
|
|
1002
1395
|
return { kind: 'tool.response', id: msg.id, ok: true, result: outcome.result };
|
|
1003
1396
|
}
|
|
1004
|
-
case 'flow':
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
// entry fails the flow cleanly instead of throwing.
|
|
1009
|
-
const rawSteps = Array.isArray(p.steps)
|
|
1010
|
-
? p.steps.filter((s) => typeof s === 'object' && s !== null)
|
|
1011
|
-
: [];
|
|
1012
|
-
const flowResult = await runFlow(rawSteps, {
|
|
1013
|
-
performFind: (params) => performFind(tabId, params),
|
|
1014
|
-
performClick: (params) => performClick(tabId, params),
|
|
1015
|
-
performType: (params) => performType(tabId, params),
|
|
1016
|
-
performPress: (params) => performPress(tabId, params),
|
|
1017
|
-
performWaitFor: (params) => performWaitFor(tabId, state, params),
|
|
1018
|
-
buildRecoverySnapshot: () => evaluateInTab(tabId, buildSnapshotJs({ only_interactive: true, max_interactive: 40 })),
|
|
1019
|
-
});
|
|
1020
|
-
// The wire-level response is ok:true whenever the flow RAN (even a
|
|
1021
|
-
// failed step is a legitimate business outcome, same pattern as
|
|
1022
|
-
// wait_for's matched:false) — flowResult itself carries the
|
|
1023
|
-
// ok:true/false the agent reads.
|
|
1024
|
-
return { kind: 'tool.response', id: msg.id, ok: true, result: flowResult };
|
|
1025
|
-
}
|
|
1026
|
-
case 'drag': {
|
|
1027
|
-
const optNum = (key) => {
|
|
1028
|
-
const v = p[key];
|
|
1029
|
-
return typeof v === 'number' && Number.isFinite(v) && v >= 0 ? v : undefined;
|
|
1030
|
-
};
|
|
1031
|
-
const clamp = (v, max) => Math.min(v, max);
|
|
1032
|
-
const fromSelector = optStr('from_selector');
|
|
1033
|
-
const toSelector = optStr('to_selector');
|
|
1034
|
-
const fromXRaw = optNum('from_x');
|
|
1035
|
-
const fromYRaw = optNum('from_y');
|
|
1036
|
-
const toXRaw = optNum('to_x');
|
|
1037
|
-
const toYRaw = optNum('to_y');
|
|
1038
|
-
// Cap every duration that ends up in a setTimeout to keep CodeQL's
|
|
1039
|
-
// "user-controlled timer duration" check happy and to prevent a
|
|
1040
|
-
// misbehaving agent from parking the bridge for hours on a typo.
|
|
1041
|
-
const durationMs = clamp(optNum('duration_ms') ?? 1500, MAX_DRAG_DURATION_MS);
|
|
1042
|
-
const holdBeforeMoveMs = clamp(optNum('hold_before_move_ms') ?? 0, MAX_DRAG_HOLD_MS);
|
|
1043
|
-
const holdBeforeReleaseMs = clamp(optNum('hold_before_release_ms') ?? 0, MAX_DRAG_HOLD_MS);
|
|
1044
|
-
const hasFromCoords = fromXRaw !== undefined && fromYRaw !== undefined;
|
|
1045
|
-
const hasToCoords = toXRaw !== undefined && toYRaw !== undefined;
|
|
1046
|
-
if (!fromSelector && !hasFromCoords) {
|
|
1047
|
-
return {
|
|
1048
|
-
kind: 'tool.response',
|
|
1049
|
-
id: msg.id,
|
|
1050
|
-
ok: false,
|
|
1051
|
-
error: 'drag: provide from_selector or both from_x and from_y',
|
|
1052
|
-
};
|
|
1053
|
-
}
|
|
1054
|
-
if (!toSelector && !hasToCoords) {
|
|
1055
|
-
return {
|
|
1056
|
-
kind: 'tool.response',
|
|
1057
|
-
id: msg.id,
|
|
1058
|
-
ok: false,
|
|
1059
|
-
error: 'drag: provide to_selector or both to_x and to_y',
|
|
1060
|
-
};
|
|
1061
|
-
}
|
|
1062
|
-
const probeExpr = buildDragProbeJs({
|
|
1063
|
-
from_selector: fromSelector,
|
|
1064
|
-
to_selector: toSelector,
|
|
1065
|
-
from_x: fromXRaw,
|
|
1066
|
-
from_y: fromYRaw,
|
|
1067
|
-
to_x: toXRaw,
|
|
1068
|
-
to_y: toYRaw,
|
|
1069
|
-
});
|
|
1070
|
-
const probe = await evaluateInTab(tabId, probeExpr);
|
|
1071
|
-
if (probe.err) {
|
|
1072
|
-
return { kind: 'tool.response', id: msg.id, ok: false, error: `drag: ${probe.err}` };
|
|
1073
|
-
}
|
|
1074
|
-
if (!probe.from || !probe.to) {
|
|
1075
|
-
return {
|
|
1076
|
-
kind: 'tool.response',
|
|
1077
|
-
id: msg.id,
|
|
1078
|
-
ok: false,
|
|
1079
|
-
error: 'drag: could not resolve coordinates',
|
|
1080
|
-
};
|
|
1081
|
-
}
|
|
1082
|
-
if (!probe.from.in_viewport) {
|
|
1083
|
-
return {
|
|
1084
|
-
kind: 'tool.response',
|
|
1085
|
-
id: msg.id,
|
|
1086
|
-
ok: false,
|
|
1087
|
-
error: 'drag: source point is offscreen — scroll first or pass viewport coords',
|
|
1088
|
-
};
|
|
1089
|
-
}
|
|
1090
|
-
if (!probe.to.in_viewport) {
|
|
1091
|
-
return {
|
|
1092
|
-
kind: 'tool.response',
|
|
1093
|
-
id: msg.id,
|
|
1094
|
-
ok: false,
|
|
1095
|
-
error: 'drag: destination point is offscreen — scroll first or pass viewport coords',
|
|
1096
|
-
};
|
|
1097
|
-
}
|
|
1098
|
-
const fromX = probe.from.x;
|
|
1099
|
-
const fromY = probe.from.y;
|
|
1100
|
-
const toX = probe.to.x;
|
|
1101
|
-
const toY = probe.to.y;
|
|
1102
|
-
const isDraggable = !!probe.draggable;
|
|
1103
|
-
// ~30fps interpolation; minimum 2 steps so the path actually has a midpoint.
|
|
1104
|
-
const steps = durationMs > 0 ? Math.max(2, Math.round(durationMs / 33)) : 1;
|
|
1105
|
-
const stepDelayMs = steps > 0 ? durationMs / steps : 0;
|
|
1106
|
-
const eventsFired = [];
|
|
1107
|
-
let dragMode = 'pointer';
|
|
1108
|
-
let interceptReceived = false;
|
|
1109
|
-
const dragStart = Date.now();
|
|
1110
|
-
let interceptionEnabled = false;
|
|
1111
|
-
const interpolate = (t) => ({
|
|
1112
|
-
x: fromX + (toX - fromX) * t,
|
|
1113
|
-
y: fromY + (toY - fromY) * t,
|
|
1114
|
-
});
|
|
1115
|
-
// Mouse move WITH the left button held down. Chrome's HTML5 drag
|
|
1116
|
-
// system only treats movement as a drag when the button state is
|
|
1117
|
-
// signalled on every move after mousePressed; omitting it makes
|
|
1118
|
-
// Blink ignore the move for drag-detection purposes.
|
|
1119
|
-
const moveHeld = (x, y) => cdp(tabId, 'Input.dispatchMouseEvent', {
|
|
1120
|
-
type: 'mouseMoved',
|
|
1121
|
-
x,
|
|
1122
|
-
y,
|
|
1123
|
-
button: 'left',
|
|
1124
|
-
buttons: 1,
|
|
1125
|
-
});
|
|
1126
|
-
// Wiggle distance has to comfortably clear `kDragThreshold` (~5px on
|
|
1127
|
-
// Mac, 4px on Linux/Windows). 5px landed *right* at the boundary and
|
|
1128
|
-
// Chrome did not start the drag — 20px crosses it on every platform.
|
|
1129
|
-
const WIGGLE_PX = 20;
|
|
1130
|
-
// Time to wait for Input.dragIntercepted after the wiggle. 120ms was
|
|
1131
|
-
// enough on local-only synthetic tests but flaky in real Chrome —
|
|
1132
|
-
// 250ms is comfortable without dragging out the overall latency.
|
|
1133
|
-
const INTERCEPT_TIMEOUT_MS = 250;
|
|
1134
|
-
try {
|
|
1135
|
-
if (isDraggable) {
|
|
1136
|
-
try {
|
|
1137
|
-
await cdp(tabId, 'Input.setInterceptDrags', { enabled: true });
|
|
1138
|
-
interceptionEnabled = true;
|
|
1139
|
-
}
|
|
1140
|
-
catch {
|
|
1141
|
-
// setInterceptDrags is experimental — fall back to pointer mode silently.
|
|
1142
|
-
}
|
|
1143
|
-
}
|
|
1144
|
-
if (interceptionEnabled) {
|
|
1145
|
-
// Arm the listener BEFORE the press+wiggle that may trigger it.
|
|
1146
|
-
const interceptPromise = waitForCdpEvent(tabId, 'Input.dragIntercepted', INTERCEPT_TIMEOUT_MS);
|
|
1147
|
-
await cdp(tabId, 'Input.dispatchMouseEvent', {
|
|
1148
|
-
type: 'mouseMoved',
|
|
1149
|
-
x: fromX,
|
|
1150
|
-
y: fromY,
|
|
1151
|
-
});
|
|
1152
|
-
await cdp(tabId, 'Input.dispatchMouseEvent', {
|
|
1153
|
-
type: 'mousePressed',
|
|
1154
|
-
x: fromX,
|
|
1155
|
-
y: fromY,
|
|
1156
|
-
button: 'left',
|
|
1157
|
-
clickCount: 1,
|
|
1158
|
-
});
|
|
1159
|
-
if (holdBeforeMoveMs > 0)
|
|
1160
|
-
await sleep(holdBeforeMoveMs);
|
|
1161
|
-
// Wiggle toward the destination so Chrome's native drag system
|
|
1162
|
-
// crosses its activation threshold. Direction matters for libs
|
|
1163
|
-
// that have direction-sensitive activation constraints.
|
|
1164
|
-
const dx = toX - fromX;
|
|
1165
|
-
const dy = toY - fromY;
|
|
1166
|
-
const len = Math.hypot(dx, dy) || 1;
|
|
1167
|
-
const wx = fromX + (dx / len) * WIGGLE_PX;
|
|
1168
|
-
const wy = fromY + (dy / len) * WIGGLE_PX;
|
|
1169
|
-
await moveHeld(wx, wy);
|
|
1170
|
-
const intercepted = await interceptPromise;
|
|
1171
|
-
if (intercepted) {
|
|
1172
|
-
dragMode = 'html5';
|
|
1173
|
-
interceptReceived = true;
|
|
1174
|
-
eventsFired.push('Input.dragIntercepted');
|
|
1175
|
-
const dragData = intercepted.data;
|
|
1176
|
-
for (let i = 1; i <= steps; i++) {
|
|
1177
|
-
const t = i / steps;
|
|
1178
|
-
const { x, y } = interpolate(t);
|
|
1179
|
-
await moveHeld(x, y);
|
|
1180
|
-
await cdp(tabId, 'Input.dispatchDragEvent', {
|
|
1181
|
-
type: 'dragOver',
|
|
1182
|
-
x,
|
|
1183
|
-
y,
|
|
1184
|
-
data: dragData,
|
|
1185
|
-
});
|
|
1186
|
-
if (i === 1)
|
|
1187
|
-
eventsFired.push('dragOver');
|
|
1188
|
-
if (stepDelayMs > 0)
|
|
1189
|
-
await sleep(stepDelayMs);
|
|
1190
|
-
}
|
|
1191
|
-
await cdp(tabId, 'Input.dispatchDragEvent', {
|
|
1192
|
-
type: 'dragEnter',
|
|
1193
|
-
x: toX,
|
|
1194
|
-
y: toY,
|
|
1195
|
-
data: dragData,
|
|
1196
|
-
});
|
|
1197
|
-
eventsFired.push('dragEnter');
|
|
1198
|
-
await cdp(tabId, 'Input.dispatchDragEvent', {
|
|
1199
|
-
type: 'dragOver',
|
|
1200
|
-
x: toX,
|
|
1201
|
-
y: toY,
|
|
1202
|
-
data: dragData,
|
|
1203
|
-
});
|
|
1204
|
-
if (holdBeforeReleaseMs > 0)
|
|
1205
|
-
await sleep(holdBeforeReleaseMs);
|
|
1206
|
-
await cdp(tabId, 'Input.dispatchDragEvent', {
|
|
1207
|
-
type: 'drop',
|
|
1208
|
-
x: toX,
|
|
1209
|
-
y: toY,
|
|
1210
|
-
data: dragData,
|
|
1211
|
-
});
|
|
1212
|
-
eventsFired.push('drop');
|
|
1213
|
-
await cdp(tabId, 'Input.dispatchMouseEvent', {
|
|
1214
|
-
type: 'mouseReleased',
|
|
1215
|
-
x: toX,
|
|
1216
|
-
y: toY,
|
|
1217
|
-
button: 'left',
|
|
1218
|
-
clickCount: 1,
|
|
1219
|
-
});
|
|
1220
|
-
}
|
|
1221
|
-
else {
|
|
1222
|
-
// Element was tagged draggable but no native drag fired — the page
|
|
1223
|
-
// either preventDefault'd dragstart or wires its own pointer logic.
|
|
1224
|
-
// Continue with pointer-only events from where we already pressed.
|
|
1225
|
-
dragMode = 'pointer';
|
|
1226
|
-
for (let i = 1; i <= steps; i++) {
|
|
1227
|
-
const t = i / steps;
|
|
1228
|
-
const { x, y } = interpolate(t);
|
|
1229
|
-
await moveHeld(x, y);
|
|
1230
|
-
if (stepDelayMs > 0)
|
|
1231
|
-
await sleep(stepDelayMs);
|
|
1232
|
-
}
|
|
1233
|
-
if (holdBeforeReleaseMs > 0)
|
|
1234
|
-
await sleep(holdBeforeReleaseMs);
|
|
1235
|
-
await cdp(tabId, 'Input.dispatchMouseEvent', {
|
|
1236
|
-
type: 'mouseReleased',
|
|
1237
|
-
x: toX,
|
|
1238
|
-
y: toY,
|
|
1239
|
-
button: 'left',
|
|
1240
|
-
clickCount: 1,
|
|
1241
|
-
});
|
|
1242
|
-
}
|
|
1243
|
-
}
|
|
1244
|
-
else {
|
|
1245
|
-
// Pointer-only branch: no native drag involvement at all.
|
|
1246
|
-
dragMode = 'pointer';
|
|
1247
|
-
await cdp(tabId, 'Input.dispatchMouseEvent', {
|
|
1248
|
-
type: 'mouseMoved',
|
|
1249
|
-
x: fromX,
|
|
1250
|
-
y: fromY,
|
|
1251
|
-
});
|
|
1252
|
-
await cdp(tabId, 'Input.dispatchMouseEvent', {
|
|
1253
|
-
type: 'mousePressed',
|
|
1254
|
-
x: fromX,
|
|
1255
|
-
y: fromY,
|
|
1256
|
-
button: 'left',
|
|
1257
|
-
clickCount: 1,
|
|
1258
|
-
});
|
|
1259
|
-
if (holdBeforeMoveMs > 0)
|
|
1260
|
-
await sleep(holdBeforeMoveMs);
|
|
1261
|
-
for (let i = 1; i <= steps; i++) {
|
|
1262
|
-
const t = i / steps;
|
|
1263
|
-
const { x, y } = interpolate(t);
|
|
1264
|
-
await moveHeld(x, y);
|
|
1265
|
-
if (stepDelayMs > 0)
|
|
1266
|
-
await sleep(stepDelayMs);
|
|
1267
|
-
}
|
|
1268
|
-
if (holdBeforeReleaseMs > 0)
|
|
1269
|
-
await sleep(holdBeforeReleaseMs);
|
|
1270
|
-
await cdp(tabId, 'Input.dispatchMouseEvent', {
|
|
1271
|
-
type: 'mouseReleased',
|
|
1272
|
-
x: toX,
|
|
1273
|
-
y: toY,
|
|
1274
|
-
button: 'left',
|
|
1275
|
-
clickCount: 1,
|
|
1276
|
-
});
|
|
1277
|
-
}
|
|
1278
|
-
}
|
|
1279
|
-
finally {
|
|
1280
|
-
// Critical: leaving interception on would block the human user from
|
|
1281
|
-
// dragging anything in this tab. Best-effort cleanup, swallow errors.
|
|
1282
|
-
if (interceptionEnabled) {
|
|
1283
|
-
try {
|
|
1284
|
-
await cdp(tabId, 'Input.setInterceptDrags', { enabled: false });
|
|
1285
|
-
}
|
|
1286
|
-
catch {
|
|
1287
|
-
// ignore
|
|
1288
|
-
}
|
|
1289
|
-
}
|
|
1290
|
-
}
|
|
1291
|
-
return {
|
|
1292
|
-
kind: 'tool.response',
|
|
1293
|
-
id: msg.id,
|
|
1294
|
-
ok: true,
|
|
1295
|
-
result: {
|
|
1296
|
-
from: { x: fromX, y: fromY, selector: fromSelector ?? null },
|
|
1297
|
-
to: { x: toX, y: toY, selector: toSelector ?? null },
|
|
1298
|
-
duration_ms_actual: Date.now() - dragStart,
|
|
1299
|
-
drag_mode: dragMode,
|
|
1300
|
-
interception_attempted: interceptionEnabled,
|
|
1301
|
-
intercept_received: interceptReceived,
|
|
1302
|
-
events_fired: eventsFired,
|
|
1303
|
-
},
|
|
1304
|
-
};
|
|
1305
|
-
}
|
|
1397
|
+
case 'flow':
|
|
1398
|
+
return await handleFlow(tabId, state, msg, p);
|
|
1399
|
+
case 'drag':
|
|
1400
|
+
return await handleDrag(tabId, msg, p, optStr);
|
|
1306
1401
|
case 'evaluate': {
|
|
1307
1402
|
const expression = str('expression');
|
|
1308
1403
|
const value = await evaluateInTab(tabId, expression);
|
|
@@ -1322,73 +1417,10 @@ async function handleTool(state, msg) {
|
|
|
1322
1417
|
}
|
|
1323
1418
|
return { kind: 'tool.response', id: msg.id, ok: true, result: outcome.result };
|
|
1324
1419
|
}
|
|
1325
|
-
case 'dialog_respond':
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
// We propagate it; the caller decides if "no dialog to respond to"
|
|
1330
|
-
// is a problem or a race they can ignore.
|
|
1331
|
-
const params = { accept };
|
|
1332
|
-
if (promptText !== undefined)
|
|
1333
|
-
params.promptText = promptText;
|
|
1334
|
-
await cdp(tabId, 'Page.handleJavaScriptDialog', params);
|
|
1335
|
-
return {
|
|
1336
|
-
kind: 'tool.response',
|
|
1337
|
-
id: msg.id,
|
|
1338
|
-
ok: true,
|
|
1339
|
-
result: { accepted: accept },
|
|
1340
|
-
};
|
|
1341
|
-
}
|
|
1342
|
-
case 'set_permission': {
|
|
1343
|
-
const origin = str('origin');
|
|
1344
|
-
const name = str('name');
|
|
1345
|
-
const state_ = str('state');
|
|
1346
|
-
// CDP `Browser.setPermission` requires a browser-level target,
|
|
1347
|
-
// which `chrome.debugger.attach({ tabId })` does NOT give us in
|
|
1348
|
-
// MV3. We use `chrome.contentSettings` instead — that surface IS
|
|
1349
|
-
// available to extensions and covers the realistic permissions
|
|
1350
|
-
// an agent needs to pre-set (geolocation, notifications, camera,
|
|
1351
|
-
// microphone, clipboard, sensors).
|
|
1352
|
-
const settingKey = CONTENT_SETTING_BY_PERMISSION[name];
|
|
1353
|
-
if (!settingKey) {
|
|
1354
|
-
return {
|
|
1355
|
-
kind: 'tool.response',
|
|
1356
|
-
id: msg.id,
|
|
1357
|
-
ok: false,
|
|
1358
|
-
error: `set_permission: '${name}' is not supported by chrome.contentSettings in MV3. Supported names: ${Object.keys(CONTENT_SETTING_BY_PERMISSION).join(', ')}.`,
|
|
1359
|
-
};
|
|
1360
|
-
}
|
|
1361
|
-
const setting = STATE_TO_CONTENT_SETTING[state_];
|
|
1362
|
-
if (!setting) {
|
|
1363
|
-
return {
|
|
1364
|
-
kind: 'tool.response',
|
|
1365
|
-
id: msg.id,
|
|
1366
|
-
ok: false,
|
|
1367
|
-
error: `set_permission: unknown state '${state_}' (expected granted | denied | prompt).`,
|
|
1368
|
-
};
|
|
1369
|
-
}
|
|
1370
|
-
// `chrome.contentSettings.<name>.set({ primaryPattern, setting })`
|
|
1371
|
-
// requires a URL pattern, not a bare origin. Append `/*` so the
|
|
1372
|
-
// pattern covers every path on the origin.
|
|
1373
|
-
const primaryPattern = origin.endsWith('/*') ? origin : `${origin}/*`;
|
|
1374
|
-
try {
|
|
1375
|
-
await applyContentSetting(settingKey, primaryPattern, setting);
|
|
1376
|
-
}
|
|
1377
|
-
catch (err) {
|
|
1378
|
-
return {
|
|
1379
|
-
kind: 'tool.response',
|
|
1380
|
-
id: msg.id,
|
|
1381
|
-
ok: false,
|
|
1382
|
-
error: `set_permission: ${err instanceof Error ? err.message : String(err)}`,
|
|
1383
|
-
};
|
|
1384
|
-
}
|
|
1385
|
-
return {
|
|
1386
|
-
kind: 'tool.response',
|
|
1387
|
-
id: msg.id,
|
|
1388
|
-
ok: true,
|
|
1389
|
-
result: { origin, name, state: state_, applied_as: settingKey },
|
|
1390
|
-
};
|
|
1391
|
-
}
|
|
1420
|
+
case 'dialog_respond':
|
|
1421
|
+
return await handleDialogRespond(tabId, msg, p, optStr);
|
|
1422
|
+
case 'set_permission':
|
|
1423
|
+
return await handleSetPermission(msg, str);
|
|
1392
1424
|
default:
|
|
1393
1425
|
return { kind: 'tool.response', id: msg.id, ok: false, error: `Unknown tool: ${msg.tool}` };
|
|
1394
1426
|
}
|