browserwire 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +113 -0
- package/cli/api/bridge.js +64 -0
- package/cli/api/openapi.js +175 -0
- package/cli/api/router.js +280 -0
- package/cli/api/swagger-ui.js +26 -0
- package/cli/discovery/classify.js +304 -0
- package/cli/discovery/compile.js +392 -0
- package/cli/discovery/enrich.js +376 -0
- package/cli/discovery/entities.js +356 -0
- package/cli/discovery/llm-client.js +352 -0
- package/cli/discovery/locators.js +326 -0
- package/cli/discovery/perceive.js +476 -0
- package/cli/discovery/session.js +930 -0
- package/cli/discovery/synthesize-workflows.js +295 -0
- package/cli/index.js +63 -0
- package/cli/manifest-store.js +140 -0
- package/cli/server.js +539 -0
- package/extension/background.js +1512 -0
- package/extension/content-script.js +491 -0
- package/extension/discovery.js +495 -0
- package/extension/executor.js +392 -0
- package/extension/icons/icon-128.png +0 -0
- package/extension/icons/icon-16.png +0 -0
- package/extension/icons/icon-48.png +0 -0
- package/extension/manifest.json +33 -0
- package/extension/shared/protocol.js +50 -0
- package/extension/sidepanel.html +277 -0
- package/extension/sidepanel.js +211 -0
- package/extension/vendor/LICENSE +22 -0
- package/extension/vendor/rrweb-record.min.js +84 -0
- package/package.json +49 -0
|
@@ -0,0 +1,1512 @@
|
|
|
1
|
+
import {
|
|
2
|
+
createEnvelope,
|
|
3
|
+
MessageType,
|
|
4
|
+
parseEnvelope,
|
|
5
|
+
PROTOCOL_VERSION
|
|
6
|
+
} from "./shared/protocol.js";
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
const DEFAULT_WS_URL = "ws://127.0.0.1:8787";
|
|
10
|
+
const HEARTBEAT_INTERVAL_MS = 20000;
|
|
11
|
+
const MAX_LOG_ENTRIES = 200;
|
|
12
|
+
|
|
13
|
+
let wsUrl = DEFAULT_WS_URL;
|
|
14
|
+
let socket = null;
|
|
15
|
+
let reconnectTimer = null;
|
|
16
|
+
let heartbeatTimer = null;
|
|
17
|
+
let reconnectAttempt = 0;
|
|
18
|
+
let shouldReconnect = false;
|
|
19
|
+
let activeSession = null;
|
|
20
|
+
let currentManifest = null;
|
|
21
|
+
let logs = [];
|
|
22
|
+
let pendingSnapshots = [];
|
|
23
|
+
|
|
24
|
+
// ─── Per-Tab In-Flight Network Request Tracking ─────────────────────
|
|
25
|
+
// Maps tabId → Set of requestIds currently in flight
|
|
26
|
+
const pendingRequests = new Map();
|
|
27
|
+
|
|
28
|
+
const getPendingCount = (tabId) => {
|
|
29
|
+
const set = pendingRequests.get(tabId);
|
|
30
|
+
return set ? set.size : 0;
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
chrome.webRequest.onBeforeRequest.addListener(
|
|
34
|
+
(details) => {
|
|
35
|
+
if (details.tabId < 0) return; // ignore non-tab requests (e.g. service worker)
|
|
36
|
+
if (!pendingRequests.has(details.tabId)) {
|
|
37
|
+
pendingRequests.set(details.tabId, new Set());
|
|
38
|
+
}
|
|
39
|
+
pendingRequests.get(details.tabId).add(details.requestId);
|
|
40
|
+
},
|
|
41
|
+
{ urls: ["<all_urls>"] }
|
|
42
|
+
);
|
|
43
|
+
|
|
44
|
+
const onRequestFinished = (details) => {
|
|
45
|
+
if (details.tabId < 0) return;
|
|
46
|
+
const set = pendingRequests.get(details.tabId);
|
|
47
|
+
if (set) {
|
|
48
|
+
set.delete(details.requestId);
|
|
49
|
+
if (set.size === 0) pendingRequests.delete(details.tabId);
|
|
50
|
+
}
|
|
51
|
+
};
|
|
52
|
+
|
|
53
|
+
chrome.webRequest.onCompleted.addListener(onRequestFinished, { urls: ["<all_urls>"] });
|
|
54
|
+
chrome.webRequest.onErrorOccurred.addListener(onRequestFinished, { urls: ["<all_urls>"] });
|
|
55
|
+
|
|
56
|
+
// Clean up when a tab is closed
|
|
57
|
+
chrome.tabs.onRemoved.addListener((tabId) => { pendingRequests.delete(tabId); });
|
|
58
|
+
|
|
59
|
+
const notifyAllContexts = (payload) => {
|
|
60
|
+
chrome.runtime.sendMessage({ source: "background", ...payload }, () => {
|
|
61
|
+
void chrome.runtime.lastError;
|
|
62
|
+
});
|
|
63
|
+
};
|
|
64
|
+
|
|
65
|
+
const addLog = (message) => {
|
|
66
|
+
const line = `[${new Date().toLocaleTimeString()}] ${message}`;
|
|
67
|
+
logs = [line, ...logs].slice(0, MAX_LOG_ENTRIES);
|
|
68
|
+
notifyAllContexts({ event: "log", line });
|
|
69
|
+
};
|
|
70
|
+
|
|
71
|
+
const getBackendState = () => {
|
|
72
|
+
if (socket && socket.readyState === WebSocket.OPEN) {
|
|
73
|
+
return "connected";
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
if (socket && socket.readyState === WebSocket.CONNECTING) {
|
|
77
|
+
return "connecting";
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
if (shouldReconnect) {
|
|
81
|
+
return "reconnecting";
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
return "disconnected";
|
|
85
|
+
};
|
|
86
|
+
|
|
87
|
+
const getState = () => ({
|
|
88
|
+
wsUrl,
|
|
89
|
+
backendState: getBackendState(),
|
|
90
|
+
reconnectAttempt,
|
|
91
|
+
session: activeSession,
|
|
92
|
+
logs
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
const broadcastState = () => {
|
|
96
|
+
notifyAllContexts({ event: "state", state: getState() });
|
|
97
|
+
};
|
|
98
|
+
|
|
99
|
+
const clearTimers = () => {
|
|
100
|
+
if (reconnectTimer) {
|
|
101
|
+
clearTimeout(reconnectTimer);
|
|
102
|
+
reconnectTimer = null;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
if (heartbeatTimer) {
|
|
106
|
+
clearInterval(heartbeatTimer);
|
|
107
|
+
heartbeatTimer = null;
|
|
108
|
+
}
|
|
109
|
+
};
|
|
110
|
+
|
|
111
|
+
const sendToBackend = (type, payload = {}, requestId = crypto.randomUUID()) => {
|
|
112
|
+
if (!socket || socket.readyState !== WebSocket.OPEN) {
|
|
113
|
+
return false;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
socket.send(JSON.stringify(createEnvelope(type, payload, requestId)));
|
|
117
|
+
return true;
|
|
118
|
+
};
|
|
119
|
+
|
|
120
|
+
const scheduleReconnect = () => {
|
|
121
|
+
reconnectAttempt += 1;
|
|
122
|
+
const delay = Math.min(30000, 500 * 2 ** reconnectAttempt);
|
|
123
|
+
addLog(`backend reconnect scheduled in ${Math.round(delay / 1000)}s`);
|
|
124
|
+
reconnectTimer = setTimeout(() => {
|
|
125
|
+
connectBackend();
|
|
126
|
+
}, delay);
|
|
127
|
+
};
|
|
128
|
+
|
|
129
|
+
const onBackendMessage = (rawMessage) => {
|
|
130
|
+
const message = parseEnvelope(rawMessage);
|
|
131
|
+
|
|
132
|
+
if (!message) {
|
|
133
|
+
addLog("received non-JSON message from backend");
|
|
134
|
+
return;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
if (message.type === MessageType.DISCOVERY_ACK) {
|
|
138
|
+
const elementCount = message.payload?.elementCount ?? 0;
|
|
139
|
+
const a11yCount = message.payload?.a11yCount ?? 0;
|
|
140
|
+
addLog(`backend discovery ack: ${elementCount} elements, ${a11yCount} a11y entries`);
|
|
141
|
+
return;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
if (message.type === MessageType.DISCOVERY_SESSION_STATUS) {
|
|
145
|
+
const stats = message.payload || {};
|
|
146
|
+
addLog(`session status: snapshots=${stats.snapshotCount ?? 0} entities=${stats.entityCount ?? 0} actions=${stats.actionCount ?? 0}`);
|
|
147
|
+
|
|
148
|
+
if (activeSession) {
|
|
149
|
+
activeSession = {
|
|
150
|
+
...activeSession,
|
|
151
|
+
snapshotCount: stats.snapshotCount ?? activeSession.snapshotCount,
|
|
152
|
+
entityCount: stats.entityCount ?? 0,
|
|
153
|
+
actionCount: stats.actionCount ?? 0
|
|
154
|
+
};
|
|
155
|
+
broadcastState();
|
|
156
|
+
}
|
|
157
|
+
return;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
if (message.type === MessageType.MANIFEST_READY) {
|
|
161
|
+
currentManifest = message.payload?.manifest || null;
|
|
162
|
+
const actionCount = currentManifest?.actions?.length ?? 0;
|
|
163
|
+
const viewCount = currentManifest?.views?.length ?? 0;
|
|
164
|
+
addLog(`manifest ready: ${actionCount} actions, ${viewCount} views`);
|
|
165
|
+
if (activeSession) {
|
|
166
|
+
activeSession = { ...activeSession, viewCount };
|
|
167
|
+
}
|
|
168
|
+
notifyAllContexts({ event: "manifest_ready", manifest: currentManifest });
|
|
169
|
+
broadcastState();
|
|
170
|
+
return;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
if (message.type === MessageType.CHECKPOINT_COMPLETE) {
|
|
174
|
+
const { manifest, checkpointIndex } = message.payload || {};
|
|
175
|
+
if (activeSession) {
|
|
176
|
+
activeSession = { ...activeSession, checkpointing: false };
|
|
177
|
+
}
|
|
178
|
+
if (manifest) {
|
|
179
|
+
currentManifest = manifest;
|
|
180
|
+
const actionCount = manifest?.actions?.length ?? 0;
|
|
181
|
+
const viewCount = manifest?.views?.length ?? 0;
|
|
182
|
+
addLog(`checkpoint-${checkpointIndex} complete: ${actionCount} actions, ${viewCount} views`);
|
|
183
|
+
} else {
|
|
184
|
+
addLog(`checkpoint-${checkpointIndex} complete: no manifest produced`);
|
|
185
|
+
}
|
|
186
|
+
notifyAllContexts({ event: "checkpoint_complete", manifest: manifest || null, checkpointIndex });
|
|
187
|
+
broadcastState();
|
|
188
|
+
return;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
if (message.type === MessageType.ERROR) {
|
|
192
|
+
addLog(`backend error: ${message.payload?.message || "unknown"}`);
|
|
193
|
+
return;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
if (message.type === MessageType.HELLO_ACK) {
|
|
197
|
+
addLog("backend handshake acknowledged");
|
|
198
|
+
return;
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
if (message.type === MessageType.EXECUTE_ACTION) {
|
|
202
|
+
handleExecuteAction(message);
|
|
203
|
+
return;
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
if (message.type === MessageType.READ_ENTITY) {
|
|
207
|
+
handleReadEntity(message);
|
|
208
|
+
return;
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
if (message.type === MessageType.EXECUTE_WORKFLOW) {
|
|
212
|
+
handleExecuteWorkflow(message);
|
|
213
|
+
return;
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
if (message.type !== MessageType.STATUS) {
|
|
217
|
+
addLog(`backend message: ${message.type}`);
|
|
218
|
+
}
|
|
219
|
+
};
|
|
220
|
+
|
|
221
|
+
const connectBackend = (nextUrl) => {
|
|
222
|
+
if (nextUrl && typeof nextUrl === "string") {
|
|
223
|
+
wsUrl = nextUrl;
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
shouldReconnect = true;
|
|
227
|
+
|
|
228
|
+
if (socket && (socket.readyState === WebSocket.OPEN || socket.readyState === WebSocket.CONNECTING)) {
|
|
229
|
+
broadcastState();
|
|
230
|
+
return { ok: true, reused: true, state: getState() };
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
if (reconnectTimer) {
|
|
234
|
+
clearTimeout(reconnectTimer);
|
|
235
|
+
reconnectTimer = null;
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
addLog(`connecting to ${wsUrl}`);
|
|
239
|
+
try {
|
|
240
|
+
socket = new WebSocket(wsUrl);
|
|
241
|
+
} catch (error) {
|
|
242
|
+
shouldReconnect = false;
|
|
243
|
+
addLog("invalid backend websocket URL");
|
|
244
|
+
return {
|
|
245
|
+
ok: false,
|
|
246
|
+
error: "invalid_ws_url",
|
|
247
|
+
state: getState()
|
|
248
|
+
};
|
|
249
|
+
}
|
|
250
|
+
broadcastState();
|
|
251
|
+
|
|
252
|
+
socket.addEventListener("open", () => {
|
|
253
|
+
reconnectAttempt = 0;
|
|
254
|
+
addLog("backend connected");
|
|
255
|
+
|
|
256
|
+
sendToBackend(MessageType.HELLO, {
|
|
257
|
+
client: "browserwire-extension-background",
|
|
258
|
+
version: PROTOCOL_VERSION
|
|
259
|
+
});
|
|
260
|
+
|
|
261
|
+
heartbeatTimer = setInterval(() => {
|
|
262
|
+
sendToBackend(MessageType.PING, { source: "background" });
|
|
263
|
+
}, HEARTBEAT_INTERVAL_MS);
|
|
264
|
+
|
|
265
|
+
broadcastState();
|
|
266
|
+
});
|
|
267
|
+
|
|
268
|
+
socket.addEventListener("message", (event) => {
|
|
269
|
+
onBackendMessage(event.data);
|
|
270
|
+
});
|
|
271
|
+
|
|
272
|
+
socket.addEventListener("close", () => {
|
|
273
|
+
clearTimers();
|
|
274
|
+
socket = null;
|
|
275
|
+
|
|
276
|
+
addLog("backend disconnected");
|
|
277
|
+
broadcastState();
|
|
278
|
+
|
|
279
|
+
if (shouldReconnect) {
|
|
280
|
+
scheduleReconnect();
|
|
281
|
+
}
|
|
282
|
+
});
|
|
283
|
+
|
|
284
|
+
socket.addEventListener("error", () => {
|
|
285
|
+
addLog("backend websocket error");
|
|
286
|
+
});
|
|
287
|
+
|
|
288
|
+
return { ok: true, state: getState() };
|
|
289
|
+
};
|
|
290
|
+
|
|
291
|
+
const disconnectBackend = () => {
|
|
292
|
+
shouldReconnect = false;
|
|
293
|
+
reconnectAttempt = 0;
|
|
294
|
+
clearTimers();
|
|
295
|
+
|
|
296
|
+
if (socket) {
|
|
297
|
+
const closingSocket = socket;
|
|
298
|
+
socket = null;
|
|
299
|
+
closingSocket.close();
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
addLog("backend disconnected manually");
|
|
303
|
+
broadcastState();
|
|
304
|
+
};
|
|
305
|
+
|
|
306
|
+
const queryActiveTab = () =>
|
|
307
|
+
new Promise((resolve) => {
|
|
308
|
+
chrome.tabs.query({ active: true, currentWindow: true }, (tabs) => {
|
|
309
|
+
resolve(tabs[0] || null);
|
|
310
|
+
});
|
|
311
|
+
});
|
|
312
|
+
|
|
313
|
+
/** Prefer the explored tab tracked by activeSession; fall back to the active tab. */
|
|
314
|
+
const getTargetTab = async () => {
|
|
315
|
+
if (activeSession && activeSession.tabId != null) {
|
|
316
|
+
const tab = await chrome.tabs.get(activeSession.tabId).catch(() => null);
|
|
317
|
+
if (tab) return tab;
|
|
318
|
+
}
|
|
319
|
+
return queryActiveTab();
|
|
320
|
+
};
|
|
321
|
+
|
|
322
|
+
const sendTabMessage = (tabId, message) =>
|
|
323
|
+
new Promise((resolve, reject) => {
|
|
324
|
+
chrome.tabs.sendMessage(tabId, message, (response) => {
|
|
325
|
+
if (chrome.runtime.lastError) {
|
|
326
|
+
reject(new Error(chrome.runtime.lastError.message));
|
|
327
|
+
return;
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
resolve(response);
|
|
331
|
+
});
|
|
332
|
+
});
|
|
333
|
+
|
|
334
|
+
// ─── Exploration Session ────────────────────────────────────────────
|
|
335
|
+
|
|
336
|
+
const startExploring = async () => {
|
|
337
|
+
if (!socket || socket.readyState !== WebSocket.OPEN) {
|
|
338
|
+
return {
|
|
339
|
+
ok: false,
|
|
340
|
+
error: "backend_not_connected",
|
|
341
|
+
state: getState()
|
|
342
|
+
};
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
const tab = await queryActiveTab();
|
|
346
|
+
|
|
347
|
+
if (!tab || typeof tab.id !== "number") {
|
|
348
|
+
return {
|
|
349
|
+
ok: false,
|
|
350
|
+
error: "no_active_tab",
|
|
351
|
+
state: getState()
|
|
352
|
+
};
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
if (activeSession) {
|
|
356
|
+
await stopExploring();
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
const sessionId = crypto.randomUUID();
|
|
360
|
+
|
|
361
|
+
let response;
|
|
362
|
+
try {
|
|
363
|
+
response = await sendTabMessage(tab.id, {
|
|
364
|
+
source: "background",
|
|
365
|
+
command: "explore_start",
|
|
366
|
+
sessionId
|
|
367
|
+
});
|
|
368
|
+
} catch (error) {
|
|
369
|
+
return {
|
|
370
|
+
ok: false,
|
|
371
|
+
error: "content_script_unavailable",
|
|
372
|
+
state: getState()
|
|
373
|
+
};
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
if (!response || response.ok !== true) {
|
|
377
|
+
return {
|
|
378
|
+
ok: false,
|
|
379
|
+
error: response?.error || "explore_start_failed",
|
|
380
|
+
state: getState()
|
|
381
|
+
};
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
activeSession = {
|
|
385
|
+
sessionId,
|
|
386
|
+
tabId: tab.id,
|
|
387
|
+
url: tab.url || "",
|
|
388
|
+
title: tab.title || "",
|
|
389
|
+
startedAt: new Date().toISOString(),
|
|
390
|
+
snapshotCount: 0,
|
|
391
|
+
entityCount: 0,
|
|
392
|
+
actionCount: 0
|
|
393
|
+
};
|
|
394
|
+
|
|
395
|
+
sendToBackend(MessageType.DISCOVERY_SESSION_START, {
|
|
396
|
+
sessionId,
|
|
397
|
+
tabId: tab.id,
|
|
398
|
+
url: tab.url || "",
|
|
399
|
+
title: tab.title || "",
|
|
400
|
+
startedAt: activeSession.startedAt
|
|
401
|
+
});
|
|
402
|
+
|
|
403
|
+
addLog(`exploration started on tab ${tab.id}`);
|
|
404
|
+
broadcastState();
|
|
405
|
+
|
|
406
|
+
return {
|
|
407
|
+
ok: true,
|
|
408
|
+
sessionId,
|
|
409
|
+
state: getState()
|
|
410
|
+
};
|
|
411
|
+
};
|
|
412
|
+
|
|
413
|
+
const stopExploring = async () => {
|
|
414
|
+
if (!activeSession) {
|
|
415
|
+
return { ok: true, idle: true, state: getState() };
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
const sessionToStop = activeSession;
|
|
419
|
+
const remainingSnapshots = pendingSnapshots.slice();
|
|
420
|
+
activeSession = null;
|
|
421
|
+
currentManifest = null;
|
|
422
|
+
pendingSnapshots = [];
|
|
423
|
+
|
|
424
|
+
try {
|
|
425
|
+
await sendTabMessage(sessionToStop.tabId, {
|
|
426
|
+
source: "background",
|
|
427
|
+
command: "explore_stop",
|
|
428
|
+
sessionId: sessionToStop.sessionId
|
|
429
|
+
});
|
|
430
|
+
} catch (error) {
|
|
431
|
+
addLog(`failed to notify tab ${sessionToStop.tabId} to stop exploring`);
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
sendToBackend(MessageType.DISCOVERY_SESSION_STOP, {
|
|
435
|
+
sessionId: sessionToStop.sessionId,
|
|
436
|
+
stoppedAt: new Date().toISOString(),
|
|
437
|
+
pendingSnapshots: remainingSnapshots
|
|
438
|
+
});
|
|
439
|
+
|
|
440
|
+
if (remainingSnapshots.length > 0) {
|
|
441
|
+
addLog(`exploration stopped (flushed ${remainingSnapshots.length} buffered snapshots)`);
|
|
442
|
+
} else {
|
|
443
|
+
addLog("exploration stopped");
|
|
444
|
+
}
|
|
445
|
+
broadcastState();
|
|
446
|
+
|
|
447
|
+
return { ok: true, state: getState() };
|
|
448
|
+
};
|
|
449
|
+
|
|
450
|
+
/**
|
|
451
|
+
* Annotate a JPEG screenshot data URL with orange boxes around interactable
|
|
452
|
+
* skeleton elements, labeled with their s-ID.
|
|
453
|
+
* Returns base64-encoded annotated JPEG, or null on failure.
|
|
454
|
+
*/
|
|
455
|
+
const annotateScreenshot = async (screenshotDataUrl, skeleton, devicePixelRatio) => {
|
|
456
|
+
try {
|
|
457
|
+
const comma = screenshotDataUrl.indexOf(",");
|
|
458
|
+
if (comma === -1) return null;
|
|
459
|
+
|
|
460
|
+
const b64 = screenshotDataUrl.slice(comma + 1);
|
|
461
|
+
const byteStr = atob(b64);
|
|
462
|
+
const arr = new Uint8Array(byteStr.length);
|
|
463
|
+
for (let i = 0; i < byteStr.length; i++) arr[i] = byteStr.charCodeAt(i);
|
|
464
|
+
const blob = new Blob([arr], { type: "image/jpeg" });
|
|
465
|
+
const bitmap = await createImageBitmap(blob);
|
|
466
|
+
|
|
467
|
+
const dpr = devicePixelRatio || 1;
|
|
468
|
+
const canvas = new OffscreenCanvas(bitmap.width, bitmap.height);
|
|
469
|
+
const ctx = canvas.getContext("2d");
|
|
470
|
+
ctx.drawImage(bitmap, 0, 0);
|
|
471
|
+
|
|
472
|
+
ctx.font = `bold ${Math.round(10 * dpr)}px sans-serif`;
|
|
473
|
+
|
|
474
|
+
for (const entry of skeleton) {
|
|
475
|
+
if (!entry.interactable || !entry.rect) continue;
|
|
476
|
+
const { x, y, width, height } = entry.rect;
|
|
477
|
+
const px = x * dpr;
|
|
478
|
+
const py = y * dpr;
|
|
479
|
+
const pw = width * dpr;
|
|
480
|
+
const ph = height * dpr;
|
|
481
|
+
|
|
482
|
+
ctx.fillStyle = "rgba(255, 165, 0, 0.3)";
|
|
483
|
+
ctx.fillRect(px, py, pw, ph);
|
|
484
|
+
|
|
485
|
+
ctx.strokeStyle = "rgba(255, 140, 0, 0.8)";
|
|
486
|
+
ctx.lineWidth = 1;
|
|
487
|
+
ctx.strokeRect(px, py, pw, ph);
|
|
488
|
+
|
|
489
|
+
ctx.fillStyle = "rgba(255, 165, 0, 0.9)";
|
|
490
|
+
ctx.fillText(`s${entry.scanId}`, px + 2, py + Math.round(11 * dpr));
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
const annotatedBlob = await canvas.convertToBlob({ type: "image/jpeg", quality: 0.5 });
|
|
494
|
+
const arrayBuffer = await annotatedBlob.arrayBuffer();
|
|
495
|
+
const bytes = new Uint8Array(arrayBuffer);
|
|
496
|
+
|
|
497
|
+
// Convert to base64 in chunks to avoid stack overflow on large images
|
|
498
|
+
let binary = "";
|
|
499
|
+
const chunkSize = 8192;
|
|
500
|
+
for (let i = 0; i < bytes.length; i += chunkSize) {
|
|
501
|
+
binary += String.fromCharCode.apply(null, bytes.subarray(i, i + chunkSize));
|
|
502
|
+
}
|
|
503
|
+
return btoa(binary);
|
|
504
|
+
} catch (error) {
|
|
505
|
+
console.error("[browserwire] annotateScreenshot failed:", error);
|
|
506
|
+
return null;
|
|
507
|
+
}
|
|
508
|
+
};
|
|
509
|
+
|
|
510
|
+
/**
|
|
511
|
+
* Handle incremental discovery snapshot from content script.
|
|
512
|
+
* Captures and annotates a screenshot, then buffers locally in pendingSnapshots[].
|
|
513
|
+
* Snapshots are only sent to the backend on a CHECKPOINT or STOP event.
|
|
514
|
+
* Runs async — the content script response is sent before this completes.
|
|
515
|
+
*/
|
|
516
|
+
const handleDiscoveryIncremental = async (message, sender) => {
|
|
517
|
+
const payload = message.payload || {};
|
|
518
|
+
|
|
519
|
+
if (!activeSession || payload.sessionId !== activeSession.sessionId) {
|
|
520
|
+
return;
|
|
521
|
+
}
|
|
522
|
+
|
|
523
|
+
activeSession = {
|
|
524
|
+
...activeSession,
|
|
525
|
+
snapshotCount: activeSession.snapshotCount + 1
|
|
526
|
+
};
|
|
527
|
+
|
|
528
|
+
// Capture and annotate screenshot
|
|
529
|
+
let annotatedScreenshot = null;
|
|
530
|
+
try {
|
|
531
|
+
const screenshotDataUrl = await chrome.tabs.captureVisibleTab({ format: "jpeg", quality: 50 });
|
|
532
|
+
annotatedScreenshot = await annotateScreenshot(
|
|
533
|
+
screenshotDataUrl,
|
|
534
|
+
payload.skeleton || [],
|
|
535
|
+
payload.devicePixelRatio || 1
|
|
536
|
+
);
|
|
537
|
+
} catch (error) {
|
|
538
|
+
addLog(`screenshot capture failed: ${error.message}`);
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
// Buffer locally — do NOT forward to backend yet
|
|
542
|
+
pendingSnapshots.push({
|
|
543
|
+
...payload,
|
|
544
|
+
screenshot: annotatedScreenshot,
|
|
545
|
+
tabId: sender.tab?.id,
|
|
546
|
+
frameId: sender.frameId
|
|
547
|
+
});
|
|
548
|
+
|
|
549
|
+
addLog(`snapshot ${activeSession.snapshotCount} buffered (${pendingSnapshots.length} pending): trigger=${payload.trigger?.kind || "unknown"}, skeleton=${(payload.skeleton || []).length}`);
|
|
550
|
+
notifyAllContexts({ event: "buffered", count: pendingSnapshots.length });
|
|
551
|
+
broadcastState();
|
|
552
|
+
};
|
|
553
|
+
|
|
554
|
+
/**
|
|
555
|
+
* Trigger a checkpoint: send all buffered snapshots to the backend for processing,
|
|
556
|
+
* then clear the local buffer. The backend responds with CHECKPOINT_COMPLETE.
|
|
557
|
+
*/
|
|
558
|
+
const handleCheckpoint = (note) => {
|
|
559
|
+
if (!activeSession) {
|
|
560
|
+
return { ok: false, error: "no_active_session", state: getState() };
|
|
561
|
+
}
|
|
562
|
+
|
|
563
|
+
if (activeSession.checkpointing) {
|
|
564
|
+
return { ok: false, error: "checkpoint_in_progress", state: getState() };
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
const sessionId = activeSession.sessionId;
|
|
568
|
+
const snapshotsToSend = pendingSnapshots.slice();
|
|
569
|
+
const checkpointIndex = activeSession.checkpointIndex || 0;
|
|
570
|
+
|
|
571
|
+
// Mark as checkpointing and clear buffer
|
|
572
|
+
activeSession = { ...activeSession, checkpointing: true, checkpointIndex: checkpointIndex + 1 };
|
|
573
|
+
pendingSnapshots = [];
|
|
574
|
+
|
|
575
|
+
const sent = sendToBackend(MessageType.CHECKPOINT, {
|
|
576
|
+
sessionId,
|
|
577
|
+
snapshots: snapshotsToSend,
|
|
578
|
+
note: note || "",
|
|
579
|
+
checkpointIndex
|
|
580
|
+
});
|
|
581
|
+
|
|
582
|
+
if (!sent) {
|
|
583
|
+
// Restore snapshots if send failed
|
|
584
|
+
pendingSnapshots = snapshotsToSend;
|
|
585
|
+
activeSession = { ...activeSession, checkpointing: false };
|
|
586
|
+
return { ok: false, error: "backend_not_connected", state: getState() };
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
addLog(`checkpoint triggered: ${snapshotsToSend.length} snapshots sent${note ? ` ("${note}")` : ""}`);
|
|
590
|
+
notifyAllContexts({ event: "checkpoint_started", snapshotCount: snapshotsToSend.length });
|
|
591
|
+
broadcastState();
|
|
592
|
+
|
|
593
|
+
return { ok: true, state: getState() };
|
|
594
|
+
};
|
|
595
|
+
|
|
596
|
+
// ─── Action Execution (unchanged) ───────────────────────────────────
|
|
597
|
+
|
|
598
|
+
/**
|
|
599
|
+
* Execute a function in the active tab via chrome.scripting.executeScript.
|
|
600
|
+
*/
|
|
601
|
+
const executeInTab = async (tabId, func, args) => {
|
|
602
|
+
const results = await chrome.scripting.executeScript({
|
|
603
|
+
target: { tabId },
|
|
604
|
+
func,
|
|
605
|
+
args
|
|
606
|
+
});
|
|
607
|
+
|
|
608
|
+
if (!results || results.length === 0) {
|
|
609
|
+
return { ok: false, error: "ERR_SCRIPT_FAILED", message: "No result from executeScript" };
|
|
610
|
+
}
|
|
611
|
+
|
|
612
|
+
return results[0].result;
|
|
613
|
+
};
|
|
614
|
+
|
|
615
|
+
/**
|
|
616
|
+
* Self-contained locator resolver + action executor.
|
|
617
|
+
* Injected directly into the page — no content script dependency.
|
|
618
|
+
*/
|
|
619
|
+
const PAGE_EXECUTE_ACTION = (payload) => {
|
|
620
|
+
const { strategies, interactionKind, inputs } = payload;
|
|
621
|
+
|
|
622
|
+
const isVisible = (el) => {
|
|
623
|
+
if (!(el instanceof HTMLElement)) return false;
|
|
624
|
+
const s = window.getComputedStyle(el);
|
|
625
|
+
if (s.display === "none" || s.visibility === "hidden") return false;
|
|
626
|
+
const r = el.getBoundingClientRect();
|
|
627
|
+
return r.width > 0 && r.height > 0;
|
|
628
|
+
};
|
|
629
|
+
|
|
630
|
+
const tryCSS = (v) => { const m = document.querySelectorAll(v); if (m.length === 1) return m[0]; for (const el of m) { if (isVisible(el)) return el; } return null; };
|
|
631
|
+
|
|
632
|
+
const tryXPath = (v) => {
|
|
633
|
+
const x = v.startsWith("/body") ? `/html${v}` : v;
|
|
634
|
+
const r = document.evaluate(x, document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
|
|
635
|
+
if (r.snapshotLength === 1) return r.snapshotItem(0);
|
|
636
|
+
for (let i = 0; i < r.snapshotLength; i++) { const el = r.snapshotItem(i); if (isVisible(el)) return el; }
|
|
637
|
+
return null;
|
|
638
|
+
};
|
|
639
|
+
|
|
640
|
+
const tryAttr = (v) => {
|
|
641
|
+
const ci = v.indexOf(":");
|
|
642
|
+
if (ci === -1) return null;
|
|
643
|
+
const a = v.slice(0, ci), av = v.slice(ci + 1);
|
|
644
|
+
try {
|
|
645
|
+
const m = document.querySelectorAll(`[${a}="${CSS.escape(av)}"]`);
|
|
646
|
+
if (m.length === 1) return m[0];
|
|
647
|
+
for (const el of m) { if (isVisible(el)) return el; }
|
|
648
|
+
} catch { /* invalid selector */ }
|
|
649
|
+
return null;
|
|
650
|
+
};
|
|
651
|
+
|
|
652
|
+
const tryRoleName = (v) => {
|
|
653
|
+
const match = v.match(/^(\w+)\s+"(.+)"$/);
|
|
654
|
+
if (!match) return null;
|
|
655
|
+
const [, role, name] = match;
|
|
656
|
+
const IMPLICIT = { button:"button",a:"link",nav:"navigation",footer:"contentinfo",header:"banner",main:"main",select:"combobox",textarea:"textbox" };
|
|
657
|
+
let found = null, count = 0;
|
|
658
|
+
for (const el of document.querySelectorAll("*")) {
|
|
659
|
+
const r = el.getAttribute("role") || IMPLICIT[el.tagName.toLowerCase()] || (el.tagName.toLowerCase() === "input" ? "textbox" : null);
|
|
660
|
+
if (r !== role) continue;
|
|
661
|
+
const n = el.getAttribute("aria-label") || el.getAttribute("title") || el.getAttribute("alt") || el.textContent?.trim().slice(0,100) || "";
|
|
662
|
+
if (n === name) { found = el; count++; if (count > 1) return null; }
|
|
663
|
+
}
|
|
664
|
+
return found;
|
|
665
|
+
};
|
|
666
|
+
|
|
667
|
+
const tryText = (v) => {
|
|
668
|
+
const w = document.createTreeWalker(document.body, NodeFilter.SHOW_TEXT, { acceptNode(n) { return (n.textContent||"").trim() === v ? NodeFilter.FILTER_ACCEPT : NodeFilter.FILTER_REJECT; } });
|
|
669
|
+
const t = w.nextNode(); if (!t) return null; if (w.nextNode()) return null;
|
|
670
|
+
return t.parentElement;
|
|
671
|
+
};
|
|
672
|
+
|
|
673
|
+
const sorted = [...(strategies || [])].sort((a, b) => b.confidence - a.confidence);
|
|
674
|
+
let element = null, usedStrategy = null;
|
|
675
|
+
|
|
676
|
+
for (const s of sorted) {
|
|
677
|
+
let el = null;
|
|
678
|
+
try {
|
|
679
|
+
if (s.kind === "css" || s.kind === "dom_path") el = tryCSS(s.value);
|
|
680
|
+
else if (s.kind === "xpath") el = tryXPath(s.value);
|
|
681
|
+
else if (s.kind === "attribute") el = tryAttr(s.value);
|
|
682
|
+
else if (s.kind === "role_name") el = tryRoleName(s.value);
|
|
683
|
+
else if (s.kind === "text") el = tryText(s.value);
|
|
684
|
+
} catch { /* skip */ }
|
|
685
|
+
if (el) { element = el; usedStrategy = { kind: s.kind, value: s.value }; break; }
|
|
686
|
+
}
|
|
687
|
+
|
|
688
|
+
if (!element) return { ok: false, error: "ERR_TARGET_NOT_FOUND", message: "No locator matched" };
|
|
689
|
+
|
|
690
|
+
const kind = (interactionKind || "click").toLowerCase();
|
|
691
|
+
if (kind === "click" || kind === "navigate") {
|
|
692
|
+
element.click();
|
|
693
|
+
return { ok: true, result: { action: "clicked" }, usedStrategy };
|
|
694
|
+
}
|
|
695
|
+
if (kind === "type") {
|
|
696
|
+
const text = inputs?.text || inputs?.value || "";
|
|
697
|
+
element.focus();
|
|
698
|
+
if ("value" in element) element.value = "";
|
|
699
|
+
for (const c of text) {
|
|
700
|
+
element.dispatchEvent(new KeyboardEvent("keydown", { key: c, bubbles: true }));
|
|
701
|
+
if ("value" in element) element.value += c;
|
|
702
|
+
element.dispatchEvent(new InputEvent("input", { data: c, inputType: "insertText", bubbles: true }));
|
|
703
|
+
element.dispatchEvent(new KeyboardEvent("keyup", { key: c, bubbles: true }));
|
|
704
|
+
}
|
|
705
|
+
element.dispatchEvent(new Event("change", { bubbles: true }));
|
|
706
|
+
return { ok: true, result: { action: "typed", length: text.length }, usedStrategy };
|
|
707
|
+
}
|
|
708
|
+
if (kind === "select") {
|
|
709
|
+
const val = inputs?.value || "";
|
|
710
|
+
if ("value" in element) { element.value = val; element.dispatchEvent(new Event("change", { bubbles: true })); }
|
|
711
|
+
return { ok: true, result: { action: "selected", value: val }, usedStrategy };
|
|
712
|
+
}
|
|
713
|
+
element.click();
|
|
714
|
+
return { ok: true, result: { action: "clicked" }, usedStrategy };
|
|
715
|
+
};
|
|
716
|
+
|
|
717
|
+
/**
|
|
718
|
+
* Self-contained entity state reader. Injected directly into the page.
|
|
719
|
+
*/
|
|
720
|
+
const PAGE_READ_ENTITY = (payload) => {
|
|
721
|
+
const { strategies } = payload;
|
|
722
|
+
|
|
723
|
+
const isVisible = (el) => {
|
|
724
|
+
if (!(el instanceof HTMLElement)) return false;
|
|
725
|
+
const s = window.getComputedStyle(el);
|
|
726
|
+
if (s.display === "none" || s.visibility === "hidden") return false;
|
|
727
|
+
const r = el.getBoundingClientRect();
|
|
728
|
+
return r.width > 0 && r.height > 0;
|
|
729
|
+
};
|
|
730
|
+
|
|
731
|
+
const tryCSS = (v) => { const m = document.querySelectorAll(v); if (m.length === 1) return m[0]; for (const el of m) { if (isVisible(el)) return el; } return null; };
|
|
732
|
+
const tryXPath = (v) => {
|
|
733
|
+
const x = v.startsWith("/body") ? `/html${v}` : v;
|
|
734
|
+
const r = document.evaluate(x, document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
|
|
735
|
+
if (r.snapshotLength === 1) return r.snapshotItem(0);
|
|
736
|
+
for (let i = 0; i < r.snapshotLength; i++) { const el = r.snapshotItem(i); if (isVisible(el)) return el; }
|
|
737
|
+
return null;
|
|
738
|
+
};
|
|
739
|
+
const tryAttr = (v) => {
|
|
740
|
+
const ci = v.indexOf(":"); if (ci === -1) return null;
|
|
741
|
+
try { const m = document.querySelectorAll(`[${v.slice(0,ci)}="${CSS.escape(v.slice(ci+1))}"]`); if (m.length === 1) return m[0]; for (const el of m) { if (isVisible(el)) return el; } } catch {}
|
|
742
|
+
return null;
|
|
743
|
+
};
|
|
744
|
+
|
|
745
|
+
const sorted = [...(strategies || [])].sort((a, b) => b.confidence - a.confidence);
|
|
746
|
+
let element = null, usedStrategy = null;
|
|
747
|
+
for (const s of sorted) {
|
|
748
|
+
let el = null;
|
|
749
|
+
try {
|
|
750
|
+
if (s.kind === "css" || s.kind === "dom_path") el = tryCSS(s.value);
|
|
751
|
+
else if (s.kind === "xpath") el = tryXPath(s.value);
|
|
752
|
+
else if (s.kind === "attribute") el = tryAttr(s.value);
|
|
753
|
+
} catch {}
|
|
754
|
+
if (el) { element = el; usedStrategy = { kind: s.kind, value: s.value }; break; }
|
|
755
|
+
}
|
|
756
|
+
|
|
757
|
+
if (!element) return { ok: false, error: "ERR_TARGET_NOT_FOUND", message: "No locator matched" };
|
|
758
|
+
|
|
759
|
+
const rect = element.getBoundingClientRect();
|
|
760
|
+
const IMPLICIT = { button:"button",a:"link",nav:"navigation",footer:"contentinfo",header:"banner",main:"main" };
|
|
761
|
+
|
|
762
|
+
return {
|
|
763
|
+
ok: true,
|
|
764
|
+
usedStrategy,
|
|
765
|
+
state: {
|
|
766
|
+
visible: isVisible(element),
|
|
767
|
+
tag: element.tagName.toLowerCase(),
|
|
768
|
+
text: (element.textContent || "").trim().slice(0, 500),
|
|
769
|
+
value: "value" in element ? element.value : undefined,
|
|
770
|
+
disabled: element.disabled || element.getAttribute("aria-disabled") === "true",
|
|
771
|
+
rect: { x: Math.round(rect.x), y: Math.round(rect.y), width: Math.round(rect.width), height: Math.round(rect.height) },
|
|
772
|
+
role: element.getAttribute("role") || IMPLICIT[element.tagName.toLowerCase()] || null,
|
|
773
|
+
name: element.getAttribute("aria-label") || element.getAttribute("title") || element.textContent?.trim().slice(0,100) || ""
|
|
774
|
+
}
|
|
775
|
+
};
|
|
776
|
+
};
|
|
777
|
+
|
|
778
|
+
/**
|
|
779
|
+
* Self-contained view data extractor. Injected directly into the page.
|
|
780
|
+
* Extracts structured data using CSS selectors — no LLM at runtime.
|
|
781
|
+
*/
|
|
782
|
+
const PAGE_READ_VIEW = (payload) => {
|
|
783
|
+
const { containerLocator, itemLocator, fields, isList } = payload;
|
|
784
|
+
|
|
785
|
+
// ── Locator helpers (same as PAGE_EXECUTE_ACTION) ──
|
|
786
|
+
|
|
787
|
+
const isVisible = (el) => {
|
|
788
|
+
if (!(el instanceof HTMLElement)) return false;
|
|
789
|
+
const s = window.getComputedStyle(el);
|
|
790
|
+
if (s.display === "none" || s.visibility === "hidden") return false;
|
|
791
|
+
const r = el.getBoundingClientRect();
|
|
792
|
+
return r.width > 0 && r.height > 0;
|
|
793
|
+
};
|
|
794
|
+
|
|
795
|
+
const tryCSS = (v) => { const m = document.querySelectorAll(v); if (m.length === 1) return m[0]; for (const el of m) { if (isVisible(el)) return el; } return null; };
|
|
796
|
+
|
|
797
|
+
const tryXPath = (v) => {
|
|
798
|
+
const x = v.startsWith("/body") ? `/html${v}` : v;
|
|
799
|
+
const r = document.evaluate(x, document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
|
|
800
|
+
if (r.snapshotLength === 1) return r.snapshotItem(0);
|
|
801
|
+
for (let i = 0; i < r.snapshotLength; i++) { const el = r.snapshotItem(i); if (isVisible(el)) return el; }
|
|
802
|
+
return null;
|
|
803
|
+
};
|
|
804
|
+
|
|
805
|
+
const tryAttr = (v) => {
|
|
806
|
+
const ci = v.indexOf(":");
|
|
807
|
+
if (ci === -1) return null;
|
|
808
|
+
const a = v.slice(0, ci), av = v.slice(ci + 1);
|
|
809
|
+
try {
|
|
810
|
+
const m = document.querySelectorAll(`[${a}="${CSS.escape(av)}"]`);
|
|
811
|
+
if (m.length === 1) return m[0];
|
|
812
|
+
for (const el of m) { if (isVisible(el)) return el; }
|
|
813
|
+
} catch { /* invalid selector */ }
|
|
814
|
+
return null;
|
|
815
|
+
};
|
|
816
|
+
|
|
817
|
+
const tryRoleName = (v) => {
|
|
818
|
+
const match = v.match(/^(\w+)\s+"(.+)"$/);
|
|
819
|
+
if (!match) return null;
|
|
820
|
+
const [, role, name] = match;
|
|
821
|
+
const IMPLICIT = { button:"button",a:"link",nav:"navigation",footer:"contentinfo",header:"banner",main:"main",select:"combobox",textarea:"textbox" };
|
|
822
|
+
let found = null, count = 0;
|
|
823
|
+
for (const el of document.querySelectorAll("*")) {
|
|
824
|
+
const r = el.getAttribute("role") || IMPLICIT[el.tagName.toLowerCase()] || (el.tagName.toLowerCase() === "input" ? "textbox" : null);
|
|
825
|
+
if (r !== role) continue;
|
|
826
|
+
const n = el.getAttribute("aria-label") || el.getAttribute("title") || el.getAttribute("alt") || el.textContent?.trim().slice(0,100) || "";
|
|
827
|
+
if (n === name) { found = el; count++; if (count > 1) return null; }
|
|
828
|
+
}
|
|
829
|
+
return found;
|
|
830
|
+
};
|
|
831
|
+
|
|
832
|
+
const tryText = (v) => {
|
|
833
|
+
const w = document.createTreeWalker(document.body, NodeFilter.SHOW_TEXT, { acceptNode(n) { return (n.textContent||"").trim() === v ? NodeFilter.FILTER_ACCEPT : NodeFilter.FILTER_REJECT; } });
|
|
834
|
+
const t = w.nextNode(); if (!t) return null; if (w.nextNode()) return null;
|
|
835
|
+
return t.parentElement;
|
|
836
|
+
};
|
|
837
|
+
|
|
838
|
+
const tryLocate = (strategies) => {
|
|
839
|
+
if (!strategies || strategies.length === 0) return null;
|
|
840
|
+
const sorted = [...strategies].sort((a, b) => (b.confidence || 0) - (a.confidence || 0));
|
|
841
|
+
for (const s of sorted) {
|
|
842
|
+
let el = null;
|
|
843
|
+
try {
|
|
844
|
+
if (s.kind === "css" || s.kind === "dom_path") el = tryCSS(s.value);
|
|
845
|
+
else if (s.kind === "xpath") el = tryXPath(s.value);
|
|
846
|
+
else if (s.kind === "attribute") el = tryAttr(s.value);
|
|
847
|
+
else if (s.kind === "role_name") el = tryRoleName(s.value);
|
|
848
|
+
else if (s.kind === "text") el = tryText(s.value);
|
|
849
|
+
} catch { /* skip */ }
|
|
850
|
+
if (el) return el;
|
|
851
|
+
}
|
|
852
|
+
return null;
|
|
853
|
+
};
|
|
854
|
+
|
|
855
|
+
// ── Container fallback chain ──
|
|
856
|
+
|
|
857
|
+
const CONTAINER_FALLBACKS = [
|
|
858
|
+
"main", "[role='main']", "#main-content", "#content", ".content",
|
|
859
|
+
"#app", "#root", "article", "[role='feed']"
|
|
860
|
+
];
|
|
861
|
+
|
|
862
|
+
let container = tryLocate(containerLocator);
|
|
863
|
+
let containerFallback = null;
|
|
864
|
+
if (!container) {
|
|
865
|
+
for (const sel of CONTAINER_FALLBACKS) {
|
|
866
|
+
try {
|
|
867
|
+
const el = document.querySelector(sel);
|
|
868
|
+
if (el && isVisible(el)) { container = el; containerFallback = sel; break; }
|
|
869
|
+
} catch { /* skip */ }
|
|
870
|
+
}
|
|
871
|
+
if (!container) {
|
|
872
|
+
container = document.body;
|
|
873
|
+
containerFallback = "document.body";
|
|
874
|
+
}
|
|
875
|
+
console.warn(`[browserwire] read_view container fallback used: ${containerFallback}`);
|
|
876
|
+
}
|
|
877
|
+
|
|
878
|
+
// ── Field extraction with text fallback ──
|
|
879
|
+
|
|
880
|
+
const coerceValue = (raw, type) => {
|
|
881
|
+
if (!raw) return null;
|
|
882
|
+
if (type === "number") return Number(raw) || null;
|
|
883
|
+
if (type === "boolean") return raw === "true" || raw === "yes";
|
|
884
|
+
return raw || null;
|
|
885
|
+
};
|
|
886
|
+
|
|
887
|
+
const extractField = (root, field) => {
|
|
888
|
+
if (!field || !field.locator) return null;
|
|
889
|
+
const selector = field.locator.value;
|
|
890
|
+
|
|
891
|
+
// 1. Direct CSS selector
|
|
892
|
+
try {
|
|
893
|
+
const el = root.querySelector(selector);
|
|
894
|
+
if (el) return coerceValue((el.textContent || "").trim(), field.type);
|
|
895
|
+
} catch { /* invalid selector */ }
|
|
896
|
+
|
|
897
|
+
// 2. Self-referencing rewrite: if selector starts with a class the root has,
|
|
898
|
+
// rewrite to :scope > rest (e.g. ".card > .title" when root IS .card)
|
|
899
|
+
try {
|
|
900
|
+
const leadClassMatch = selector.match(/^(\.[a-zA-Z0-9_-]+)\s*>\s*(.+)$/);
|
|
901
|
+
if (leadClassMatch) {
|
|
902
|
+
const [, leadClass, rest] = leadClassMatch;
|
|
903
|
+
if (root.matches && root.matches(leadClass)) {
|
|
904
|
+
// Try :scope > rest
|
|
905
|
+
const scopeEl = root.querySelector(`:scope > ${rest}`);
|
|
906
|
+
if (scopeEl) return coerceValue((scopeEl.textContent || "").trim(), field.type);
|
|
907
|
+
// Try just the structural part
|
|
908
|
+
const restEl = root.querySelector(rest);
|
|
909
|
+
if (restEl) return coerceValue((restEl.textContent || "").trim(), field.type);
|
|
910
|
+
}
|
|
911
|
+
}
|
|
912
|
+
} catch { /* skip */ }
|
|
913
|
+
|
|
914
|
+
// 3. aria-label extraction for title/name fields
|
|
915
|
+
// Last-resort: only for fields whose name suggests a title/name
|
|
916
|
+
try {
|
|
917
|
+
if (/title|name/i.test(field.name)) {
|
|
918
|
+
for (const tag of ["a", "button"]) {
|
|
919
|
+
const els = root.querySelectorAll(`${tag}[aria-label]`);
|
|
920
|
+
for (const el of els) {
|
|
921
|
+
const label = (el.getAttribute("aria-label") || "").trim();
|
|
922
|
+
if (label.length > 3) return coerceValue(label, field.type);
|
|
923
|
+
}
|
|
924
|
+
}
|
|
925
|
+
}
|
|
926
|
+
} catch { /* skip */ }
|
|
927
|
+
|
|
928
|
+
// 4. Semantic class matching: fuzzy last-resort — match field name parts
|
|
929
|
+
// against class names of *direct children only* to limit false positives
|
|
930
|
+
try {
|
|
931
|
+
const nameParts = field.name.split("_").filter(p => p.length > 2);
|
|
932
|
+
for (const part of nameParts) {
|
|
933
|
+
const matches = root.querySelectorAll(`:scope > [class*="${part}"], :scope > * > [class*="${part}"]`);
|
|
934
|
+
for (const el of matches) {
|
|
935
|
+
if (el !== root) {
|
|
936
|
+
const raw = (el.textContent || "").trim();
|
|
937
|
+
if (raw.length > 1) return coerceValue(raw, field.type);
|
|
938
|
+
}
|
|
939
|
+
}
|
|
940
|
+
}
|
|
941
|
+
} catch { /* skip */ }
|
|
942
|
+
|
|
943
|
+
return null;
|
|
944
|
+
};
|
|
945
|
+
|
|
946
|
+
/**
|
|
947
|
+
* Text-block fallback: extract the N most distinct text blocks from an element,
|
|
948
|
+
* mapping them positionally to the N field names.
|
|
949
|
+
* Filters out zero-width chars and single-char noise.
|
|
950
|
+
*/
|
|
951
|
+
const ZERO_WIDTH_RE = /[\u200B\u00A0\uFEFF]/g;
|
|
952
|
+
const extractFieldsByTextBlocks = (root, fieldDefs) => {
|
|
953
|
+
if (!fieldDefs || fieldDefs.length === 0) return {};
|
|
954
|
+
const blocks = [];
|
|
955
|
+
const walker = document.createTreeWalker(root, NodeFilter.SHOW_TEXT, {
|
|
956
|
+
acceptNode(n) {
|
|
957
|
+
const t = (n.textContent || "").replace(ZERO_WIDTH_RE, "").trim();
|
|
958
|
+
return t.length > 1 ? NodeFilter.FILTER_ACCEPT : NodeFilter.FILTER_REJECT;
|
|
959
|
+
}
|
|
960
|
+
});
|
|
961
|
+
let node;
|
|
962
|
+
while ((node = walker.nextNode()) && blocks.length < fieldDefs.length + 10) {
|
|
963
|
+
const text = (node.textContent || "").replace(ZERO_WIDTH_RE, "").trim();
|
|
964
|
+
if (text.length > 1 && !blocks.includes(text)) blocks.push(text);
|
|
965
|
+
}
|
|
966
|
+
const row = {};
|
|
967
|
+
for (let i = 0; i < fieldDefs.length; i++) {
|
|
968
|
+
row[fieldDefs[i].name] = i < blocks.length ? blocks[i] : null;
|
|
969
|
+
}
|
|
970
|
+
return row;
|
|
971
|
+
};
|
|
972
|
+
|
|
973
|
+
if (isList && itemLocator) {
|
|
974
|
+
// List view: find all items, extract fields per item
|
|
975
|
+
let items;
|
|
976
|
+
try {
|
|
977
|
+
const selector = itemLocator.value || itemLocator;
|
|
978
|
+
items = container.querySelectorAll(typeof selector === "string" ? selector : selector.value);
|
|
979
|
+
} catch {
|
|
980
|
+
return { ok: false, error: "ERR_TARGET_NOT_FOUND", message: "Item selector invalid" };
|
|
981
|
+
}
|
|
982
|
+
|
|
983
|
+
// If no items found with the item selector, try document-wide before generic fallbacks
|
|
984
|
+
if (!items || items.length === 0) {
|
|
985
|
+
try {
|
|
986
|
+
const selector = itemLocator.value || itemLocator;
|
|
987
|
+
const sel = typeof selector === "string" ? selector : selector.value;
|
|
988
|
+
const docItems = document.querySelectorAll(sel);
|
|
989
|
+
if (docItems.length > 0) {
|
|
990
|
+
items = docItems;
|
|
991
|
+
console.warn(`[browserwire] read_view items found document-wide: ${sel} (${docItems.length} items)`);
|
|
992
|
+
}
|
|
993
|
+
} catch { /* skip */ }
|
|
994
|
+
}
|
|
995
|
+
|
|
996
|
+
// If still no items, try common list patterns within container
|
|
997
|
+
if (!items || items.length === 0) {
|
|
998
|
+
const LIST_ITEM_FALLBACKS = ["li", "[role='listitem']", "[role='row']", "tr", "article", ":scope > div"];
|
|
999
|
+
for (const sel of LIST_ITEM_FALLBACKS) {
|
|
1000
|
+
try {
|
|
1001
|
+
const candidates = container.querySelectorAll(sel);
|
|
1002
|
+
if (candidates.length > 0) { items = candidates; console.warn(`[browserwire] read_view item fallback used: ${sel}`); break; }
|
|
1003
|
+
} catch { /* skip */ }
|
|
1004
|
+
}
|
|
1005
|
+
}
|
|
1006
|
+
|
|
1007
|
+
if (!items || items.length === 0) {
|
|
1008
|
+
return { ok: true, result: [], count: 0, _note: "no items found" };
|
|
1009
|
+
}
|
|
1010
|
+
|
|
1011
|
+
const fieldDefs = fields || [];
|
|
1012
|
+
const result = [];
|
|
1013
|
+
let allNull = true;
|
|
1014
|
+
const seenAncestors = new Set();
|
|
1015
|
+
|
|
1016
|
+
for (const item of items) {
|
|
1017
|
+
const row = {};
|
|
1018
|
+
let extractionRoot = item;
|
|
1019
|
+
for (const field of fieldDefs) {
|
|
1020
|
+
row[field.name] = extractField(item, field);
|
|
1021
|
+
}
|
|
1022
|
+
|
|
1023
|
+
// If all fields null, item may be an overlay/link — escalate to card-like ancestor
|
|
1024
|
+
if (!Object.values(row).some(v => v !== null)) {
|
|
1025
|
+
const CARD_ANCESTORS = '[role="button"], [role="listitem"], [class*="card"], article, li';
|
|
1026
|
+
const ancestor = (item.closest && item.closest(CARD_ANCESTORS)) || item.parentElement;
|
|
1027
|
+
if (ancestor && ancestor !== item && ancestor !== document.body) {
|
|
1028
|
+
// Deduplicate: skip if we already extracted from this ancestor
|
|
1029
|
+
if (seenAncestors.has(ancestor)) continue;
|
|
1030
|
+
seenAncestors.add(ancestor);
|
|
1031
|
+
extractionRoot = ancestor;
|
|
1032
|
+
for (const field of fieldDefs) {
|
|
1033
|
+
row[field.name] = extractField(ancestor, field);
|
|
1034
|
+
}
|
|
1035
|
+
console.warn(`[browserwire] read_view: escalated item to ancestor <${ancestor.tagName.toLowerCase()}>`);
|
|
1036
|
+
}
|
|
1037
|
+
}
|
|
1038
|
+
|
|
1039
|
+
// Check if any field was extracted
|
|
1040
|
+
const hasValue = Object.values(row).some(v => v !== null);
|
|
1041
|
+
if (!hasValue) {
|
|
1042
|
+
// Text-block fallback using best extraction root
|
|
1043
|
+
const textRow = extractFieldsByTextBlocks(extractionRoot, fieldDefs);
|
|
1044
|
+
for (const key of Object.keys(textRow)) row[key] = textRow[key];
|
|
1045
|
+
}
|
|
1046
|
+
if (Object.values(row).some(v => v !== null)) allNull = false;
|
|
1047
|
+
result.push(row);
|
|
1048
|
+
}
|
|
1049
|
+
|
|
1050
|
+
// Last resort: if ALL rows are still entirely null, add _raw_text per item
|
|
1051
|
+
if (allNull && result.length > 0) {
|
|
1052
|
+
for (let i = 0; i < result.length; i++) {
|
|
1053
|
+
const root = items[i];
|
|
1054
|
+
const CARD_ANCESTORS = '[role="button"], [role="listitem"], [class*="card"], article, li';
|
|
1055
|
+
const best = (root.closest && root.closest(CARD_ANCESTORS)) || root;
|
|
1056
|
+
result[i]._raw_text = (best.textContent || "").trim().slice(0, 500);
|
|
1057
|
+
}
|
|
1058
|
+
console.warn("[browserwire] read_view: all fields null, falling back to _raw_text");
|
|
1059
|
+
}
|
|
1060
|
+
|
|
1061
|
+
return { ok: true, result, count: result.length, ...(containerFallback ? { containerFallback } : {}) };
|
|
1062
|
+
} else {
|
|
1063
|
+
// Single/detail view: extract fields from container directly
|
|
1064
|
+
const fieldDefs = fields || [];
|
|
1065
|
+
const row = {};
|
|
1066
|
+
for (const field of fieldDefs) {
|
|
1067
|
+
row[field.name] = extractField(container, field);
|
|
1068
|
+
}
|
|
1069
|
+
// Text-block fallback if all fields are null
|
|
1070
|
+
if (Object.values(row).every(v => v === null) && fieldDefs.length > 0) {
|
|
1071
|
+
const textRow = extractFieldsByTextBlocks(container, fieldDefs);
|
|
1072
|
+
for (const key of Object.keys(textRow)) row[key] = textRow[key];
|
|
1073
|
+
}
|
|
1074
|
+
return { ok: true, result: row, ...(containerFallback ? { containerFallback } : {}) };
|
|
1075
|
+
}
|
|
1076
|
+
};
|
|
1077
|
+
|
|
1078
|
+
/**
|
|
1079
|
+
* Handle EXECUTE_ACTION from the WS server.
|
|
1080
|
+
*/
|
|
1081
|
+
const handleExecuteAction = async (message) => {
|
|
1082
|
+
const tab = await getTargetTab();
|
|
1083
|
+
if (!tab || typeof tab.id !== "number") {
|
|
1084
|
+
sendToBackend(MessageType.EXECUTE_RESULT, {
|
|
1085
|
+
ok: false, error: "no_active_tab", message: "No active tab"
|
|
1086
|
+
}, message.requestId);
|
|
1087
|
+
return;
|
|
1088
|
+
}
|
|
1089
|
+
|
|
1090
|
+
try {
|
|
1091
|
+
const result = await executeInTab(tab.id, PAGE_EXECUTE_ACTION, [message.payload]);
|
|
1092
|
+
sendToBackend(MessageType.EXECUTE_RESULT, result, message.requestId);
|
|
1093
|
+
} catch (error) {
|
|
1094
|
+
sendToBackend(MessageType.EXECUTE_RESULT, {
|
|
1095
|
+
ok: false, error: "ERR_EXECUTION_FAILED", message: error.message || "Script execution failed"
|
|
1096
|
+
}, message.requestId);
|
|
1097
|
+
}
|
|
1098
|
+
};
|
|
1099
|
+
|
|
1100
|
+
/**
|
|
1101
|
+
* Handle READ_ENTITY from the WS server.
|
|
1102
|
+
* Routes to PAGE_READ_VIEW if the payload contains view extraction fields,
|
|
1103
|
+
* otherwise falls back to the simple entity state reader.
|
|
1104
|
+
*/
|
|
1105
|
+
const handleReadEntity = async (message) => {
|
|
1106
|
+
const tab = await getTargetTab();
|
|
1107
|
+
if (!tab || typeof tab.id !== "number") {
|
|
1108
|
+
sendToBackend(MessageType.READ_RESULT, {
|
|
1109
|
+
ok: false, error: "no_active_tab", message: "No active tab"
|
|
1110
|
+
}, message.requestId);
|
|
1111
|
+
return;
|
|
1112
|
+
}
|
|
1113
|
+
|
|
1114
|
+
try {
|
|
1115
|
+
// Route to view extractor if payload has containerLocator + fields
|
|
1116
|
+
const handler = (message.payload?.containerLocator && message.payload?.fields)
|
|
1117
|
+
? PAGE_READ_VIEW
|
|
1118
|
+
: PAGE_READ_ENTITY;
|
|
1119
|
+
const result = await executeInTab(tab.id, handler, [message.payload]);
|
|
1120
|
+
sendToBackend(MessageType.READ_RESULT, result, message.requestId);
|
|
1121
|
+
} catch (error) {
|
|
1122
|
+
sendToBackend(MessageType.READ_RESULT, {
|
|
1123
|
+
ok: false, error: "ERR_READ_FAILED", message: error.message || "Script execution failed"
|
|
1124
|
+
}, message.requestId);
|
|
1125
|
+
}
|
|
1126
|
+
};
|
|
1127
|
+
|
|
1128
|
+
// ─── Workflow Execution ─────────────────────────────────────────────
|
|
1129
|
+
|
|
1130
|
+
/**
|
|
1131
|
+
* Navigate a tab to a URL and wait for the page to finish loading.
|
|
1132
|
+
* Resolves relative URLs against baseOrigin.
|
|
1133
|
+
* After load complete, waits 500ms for SPA hydration.
|
|
1134
|
+
*/
|
|
1135
|
+
const navigateTabAndWait = (tabId, url, baseOrigin, timeoutMs = 10000) =>
|
|
1136
|
+
new Promise((resolve, reject) => {
|
|
1137
|
+
const fullUrl = url.startsWith("http") ? url : `${baseOrigin}${url}`;
|
|
1138
|
+
const timer = setTimeout(() => {
|
|
1139
|
+
chrome.tabs.onUpdated.removeListener(listener);
|
|
1140
|
+
reject(new Error(`Navigation to ${fullUrl} timed out`));
|
|
1141
|
+
}, timeoutMs);
|
|
1142
|
+
|
|
1143
|
+
const listener = (updatedTabId, changeInfo) => {
|
|
1144
|
+
if (updatedTabId !== tabId || changeInfo.status !== "complete") return;
|
|
1145
|
+
chrome.tabs.onUpdated.removeListener(listener);
|
|
1146
|
+
clearTimeout(timer);
|
|
1147
|
+
// 500ms grace for SPA hydration
|
|
1148
|
+
setTimeout(resolve, 500);
|
|
1149
|
+
};
|
|
1150
|
+
|
|
1151
|
+
chrome.tabs.onUpdated.addListener(listener);
|
|
1152
|
+
chrome.tabs.update(tabId, { url: fullUrl });
|
|
1153
|
+
});
|
|
1154
|
+
|
|
1155
|
+
/**
|
|
1156
|
+
* Self-contained outcome evaluator. Injected into the page after workflow steps complete.
|
|
1157
|
+
* Checks outcome signals and returns { outcome: "success" | "failure" | "unknown" }.
|
|
1158
|
+
*/
|
|
1159
|
+
const PAGE_EVALUATE_OUTCOME = (payload) => {
|
|
1160
|
+
const { outcomes } = payload;
|
|
1161
|
+
if (!outcomes) return { outcome: "unknown" };
|
|
1162
|
+
|
|
1163
|
+
const check = (signal) => {
|
|
1164
|
+
if (!signal || !signal.kind || !signal.value) return false;
|
|
1165
|
+
try {
|
|
1166
|
+
if (signal.kind === "url_change") {
|
|
1167
|
+
return new RegExp(signal.value).test(window.location.pathname + window.location.search);
|
|
1168
|
+
}
|
|
1169
|
+
if (signal.kind === "element_appears") {
|
|
1170
|
+
return document.querySelector(signal.value) !== null;
|
|
1171
|
+
}
|
|
1172
|
+
if (signal.kind === "element_disappears") {
|
|
1173
|
+
return document.querySelector(signal.value) === null;
|
|
1174
|
+
}
|
|
1175
|
+
if (signal.kind === "text_contains") {
|
|
1176
|
+
const el = signal.selector ? document.querySelector(signal.selector) : document.body;
|
|
1177
|
+
if (!el) return false;
|
|
1178
|
+
return new RegExp(signal.value, "i").test(el.textContent || "");
|
|
1179
|
+
}
|
|
1180
|
+
} catch { /* invalid regex or selector */ }
|
|
1181
|
+
return false;
|
|
1182
|
+
};
|
|
1183
|
+
|
|
1184
|
+
if (outcomes.success && check(outcomes.success)) return { outcome: "success" };
|
|
1185
|
+
if (outcomes.failure && check(outcomes.failure)) return { outcome: "failure" };
|
|
1186
|
+
return { outcome: "unknown" };
|
|
1187
|
+
};
|
|
1188
|
+
|
|
1189
|
+
/**
|
|
1190
|
+
* Poll/retry for read_view data — handles async-loading DOM and delayed data.
|
|
1191
|
+
* Retries on hard errors (container not found) and empty results.
|
|
1192
|
+
*/
|
|
1193
|
+
const pollReadView = async (tabId, viewConfig, { timeoutMs = 20000, readyQuietMs = 500 } = {}) => {
|
|
1194
|
+
const start = Date.now();
|
|
1195
|
+
let lastActivityTime = start;
|
|
1196
|
+
|
|
1197
|
+
while (Date.now() - start < timeoutMs) {
|
|
1198
|
+
// Wait for DOM to be briefly quiet before reading
|
|
1199
|
+
const domResult = await executeInTab(tabId, PAGE_WAIT_FOR_DOM_IDLE, [{ idleMs: 300, timeoutMs: 3000 }]);
|
|
1200
|
+
const result = await executeInTab(tabId, PAGE_READ_VIEW, [viewConfig]);
|
|
1201
|
+
|
|
1202
|
+
if (!result || result.ok === false) {
|
|
1203
|
+
lastActivityTime = Date.now();
|
|
1204
|
+
await new Promise(r => setTimeout(r, 500));
|
|
1205
|
+
continue;
|
|
1206
|
+
}
|
|
1207
|
+
|
|
1208
|
+
const isEmpty = viewConfig.isList
|
|
1209
|
+
? (Array.isArray(result.result) && result.result.length === 0)
|
|
1210
|
+
: (result.result && Object.values(result.result).every(v => v === null));
|
|
1211
|
+
|
|
1212
|
+
if (!isEmpty) return result;
|
|
1213
|
+
|
|
1214
|
+
// Empty result — check for activity signals
|
|
1215
|
+
const pending = getPendingCount(tabId);
|
|
1216
|
+
const domActive = !domResult?.domIdle;
|
|
1217
|
+
|
|
1218
|
+
if (pending > 0 || domActive) {
|
|
1219
|
+
lastActivityTime = Date.now();
|
|
1220
|
+
await new Promise(r => setTimeout(r, 500));
|
|
1221
|
+
continue;
|
|
1222
|
+
}
|
|
1223
|
+
|
|
1224
|
+
// Everything quiet — but has it been quiet long enough?
|
|
1225
|
+
const quietDuration = Date.now() - lastActivityTime;
|
|
1226
|
+
if (quietDuration < readyQuietMs || (Date.now() - start) < 3000) {
|
|
1227
|
+
await new Promise(r => setTimeout(r, 200));
|
|
1228
|
+
continue;
|
|
1229
|
+
}
|
|
1230
|
+
|
|
1231
|
+
// Truly quiet + empty after reasonable wait → accept as real
|
|
1232
|
+
return result;
|
|
1233
|
+
}
|
|
1234
|
+
|
|
1235
|
+
return await executeInTab(tabId, PAGE_READ_VIEW, [viewConfig]);
|
|
1236
|
+
};
|
|
1237
|
+
|
|
1238
|
+
// Page-injected: watches DOM mutations only.
|
|
1239
|
+
// Network idle is tracked separately in the background via chrome.webRequest.
|
|
1240
|
+
const PAGE_WAIT_FOR_DOM_IDLE = (opts) => {
|
|
1241
|
+
const { idleMs = 500, timeoutMs = 10000 } = opts || {};
|
|
1242
|
+
return new Promise((resolve) => {
|
|
1243
|
+
let lastActivity = Date.now();
|
|
1244
|
+
const start = lastActivity;
|
|
1245
|
+
|
|
1246
|
+
const observer = new MutationObserver(() => { lastActivity = Date.now(); });
|
|
1247
|
+
observer.observe(document.body || document.documentElement, {
|
|
1248
|
+
childList: true, subtree: true, attributes: true, characterData: true
|
|
1249
|
+
});
|
|
1250
|
+
|
|
1251
|
+
const check = () => {
|
|
1252
|
+
const now = Date.now();
|
|
1253
|
+
const elapsed = now - start;
|
|
1254
|
+
if (elapsed >= timeoutMs) {
|
|
1255
|
+
observer.disconnect();
|
|
1256
|
+
resolve({ domIdle: false, elapsed, reason: "timeout" });
|
|
1257
|
+
return;
|
|
1258
|
+
}
|
|
1259
|
+
if (now - lastActivity >= idleMs) {
|
|
1260
|
+
observer.disconnect();
|
|
1261
|
+
resolve({ domIdle: true, elapsed, reason: "idle" });
|
|
1262
|
+
return;
|
|
1263
|
+
}
|
|
1264
|
+
setTimeout(check, 100);
|
|
1265
|
+
};
|
|
1266
|
+
|
|
1267
|
+
// Small initial delay to let the action's immediate effects begin
|
|
1268
|
+
setTimeout(check, 100);
|
|
1269
|
+
});
|
|
1270
|
+
};
|
|
1271
|
+
|
|
1272
|
+
const waitForNavigation = async (tabId, timeoutMs = 10000) => {
|
|
1273
|
+
await new Promise(r => setTimeout(r, 100));
|
|
1274
|
+
const tab = await chrome.tabs.get(tabId);
|
|
1275
|
+
if (tab.status === "loading") {
|
|
1276
|
+
await new Promise((resolve) => {
|
|
1277
|
+
const timeout = setTimeout(() => {
|
|
1278
|
+
chrome.tabs.onUpdated.removeListener(listener);
|
|
1279
|
+
resolve();
|
|
1280
|
+
}, timeoutMs);
|
|
1281
|
+
const listener = (id, info) => {
|
|
1282
|
+
if (id !== tabId || info.status !== "complete") return;
|
|
1283
|
+
chrome.tabs.onUpdated.removeListener(listener);
|
|
1284
|
+
clearTimeout(timeout);
|
|
1285
|
+
resolve();
|
|
1286
|
+
};
|
|
1287
|
+
chrome.tabs.onUpdated.addListener(listener);
|
|
1288
|
+
});
|
|
1289
|
+
}
|
|
1290
|
+
// Grace period for SPA framework to begin post-navigation work
|
|
1291
|
+
await new Promise(r => setTimeout(r, 150));
|
|
1292
|
+
};
|
|
1293
|
+
|
|
1294
|
+
/**
|
|
1295
|
+
* Execute a multi-step workflow in a given tab.
|
|
1296
|
+
* Returns { ok, data?, outcome?, error?, message? } directly.
|
|
1297
|
+
*/
|
|
1298
|
+
const runWorkflowSteps = async (tabId, baseOrigin, steps, outcomes, inputs) => {
|
|
1299
|
+
let lastReadData = null;
|
|
1300
|
+
let hasReadView = false;
|
|
1301
|
+
|
|
1302
|
+
for (const step of (steps || [])) {
|
|
1303
|
+
try {
|
|
1304
|
+
if (step.type === "navigate") {
|
|
1305
|
+
await navigateTabAndWait(tabId, step.url, baseOrigin);
|
|
1306
|
+
continue;
|
|
1307
|
+
}
|
|
1308
|
+
|
|
1309
|
+
if (step.type === "read_view") {
|
|
1310
|
+
hasReadView = true;
|
|
1311
|
+
const viewConfig = {
|
|
1312
|
+
containerLocator: step.viewConfig?.containerLocator || [],
|
|
1313
|
+
itemLocator: step.viewConfig?.itemLocator || null,
|
|
1314
|
+
fields: step.viewConfig?.fields || [],
|
|
1315
|
+
isList: step.viewConfig?.isList || false
|
|
1316
|
+
};
|
|
1317
|
+
console.log("[browserwire] read_view step:", JSON.stringify(viewConfig).slice(0, 300));
|
|
1318
|
+
const result = await pollReadView(tabId, viewConfig);
|
|
1319
|
+
console.log("[browserwire] read_view result:", JSON.stringify(result).slice(0, 300));
|
|
1320
|
+
if (!result || result.ok === false) {
|
|
1321
|
+
return {
|
|
1322
|
+
ok: false,
|
|
1323
|
+
error: result?.error || "ERR_READ_VIEW_FAILED",
|
|
1324
|
+
message: result?.message || "read_view step failed"
|
|
1325
|
+
};
|
|
1326
|
+
}
|
|
1327
|
+
lastReadData = result.result;
|
|
1328
|
+
continue;
|
|
1329
|
+
}
|
|
1330
|
+
|
|
1331
|
+
if (["fill", "select", "click", "submit"].includes(step.type)) {
|
|
1332
|
+
const interactionKind = step.type === "fill" ? "type"
|
|
1333
|
+
: step.type === "submit" ? "click"
|
|
1334
|
+
: step.type;
|
|
1335
|
+
const inputValue = step.inputParam ? (inputs || {})[step.inputParam] : undefined;
|
|
1336
|
+
const stepInputs = inputValue !== undefined
|
|
1337
|
+
? { text: inputValue, value: inputValue }
|
|
1338
|
+
: {};
|
|
1339
|
+
|
|
1340
|
+
const result = await executeInTab(tabId, PAGE_EXECUTE_ACTION, [{
|
|
1341
|
+
strategies: step.strategies || [],
|
|
1342
|
+
interactionKind,
|
|
1343
|
+
inputs: stepInputs
|
|
1344
|
+
}]);
|
|
1345
|
+
|
|
1346
|
+
if (!result || result.ok === false) {
|
|
1347
|
+
return {
|
|
1348
|
+
ok: false,
|
|
1349
|
+
error: result?.error || "ERR_STEP_FAILED",
|
|
1350
|
+
message: result?.message || `${step.type} step failed`
|
|
1351
|
+
};
|
|
1352
|
+
}
|
|
1353
|
+
await waitForNavigation(tabId);
|
|
1354
|
+
continue;
|
|
1355
|
+
}
|
|
1356
|
+
} catch (error) {
|
|
1357
|
+
return {
|
|
1358
|
+
ok: false,
|
|
1359
|
+
error: "ERR_WORKFLOW_STEP_FAILED",
|
|
1360
|
+
message: error.message || `Step ${step.type} threw an error`
|
|
1361
|
+
};
|
|
1362
|
+
}
|
|
1363
|
+
}
|
|
1364
|
+
|
|
1365
|
+
// All steps succeeded
|
|
1366
|
+
if (hasReadView) {
|
|
1367
|
+
return { ok: true, data: lastReadData };
|
|
1368
|
+
}
|
|
1369
|
+
|
|
1370
|
+
// Write workflow: wait for navigation then evaluate outcomes
|
|
1371
|
+
await waitForNavigation(tabId);
|
|
1372
|
+
try {
|
|
1373
|
+
const evalResult = await executeInTab(tabId, PAGE_EVALUATE_OUTCOME, [{ outcomes }]);
|
|
1374
|
+
return { ok: true, outcome: evalResult?.outcome || "unknown" };
|
|
1375
|
+
} catch (error) {
|
|
1376
|
+
return { ok: true, outcome: "unknown" };
|
|
1377
|
+
}
|
|
1378
|
+
};
|
|
1379
|
+
|
|
1380
|
+
/**
|
|
1381
|
+
* Handle EXECUTE_WORKFLOW from the WS server.
|
|
1382
|
+
*/
|
|
1383
|
+
const handleExecuteWorkflow = async (message) => {
|
|
1384
|
+
const { steps, outcomes, inputs } = message.payload || {};
|
|
1385
|
+
const requestId = message.requestId;
|
|
1386
|
+
|
|
1387
|
+
const tab = await getTargetTab();
|
|
1388
|
+
if (!tab || typeof tab.id !== "number") {
|
|
1389
|
+
sendToBackend(MessageType.WORKFLOW_RESULT, {
|
|
1390
|
+
ok: false, error: "no_active_tab", message: "No active tab"
|
|
1391
|
+
}, requestId);
|
|
1392
|
+
return;
|
|
1393
|
+
}
|
|
1394
|
+
|
|
1395
|
+
const baseOrigin = tab.url ? new URL(tab.url).origin : "";
|
|
1396
|
+
const result = await runWorkflowSteps(tab.id, baseOrigin, steps, outcomes, inputs);
|
|
1397
|
+
sendToBackend(MessageType.WORKFLOW_RESULT, result, requestId);
|
|
1398
|
+
};
|
|
1399
|
+
|
|
1400
|
+
// ─── Sidepanel Command Handler ──────────────────────────────────────
|
|
1401
|
+
|
|
1402
|
+
const handleSidepanelCommand = async (message) => {
|
|
1403
|
+
if (message.command === "get_state") {
|
|
1404
|
+
return { ok: true, state: getState() };
|
|
1405
|
+
}
|
|
1406
|
+
|
|
1407
|
+
if (message.command === "connect_backend") {
|
|
1408
|
+
return connectBackend(message.url);
|
|
1409
|
+
}
|
|
1410
|
+
|
|
1411
|
+
if (message.command === "disconnect_backend") {
|
|
1412
|
+
disconnectBackend();
|
|
1413
|
+
return { ok: true, state: getState() };
|
|
1414
|
+
}
|
|
1415
|
+
|
|
1416
|
+
if (message.command === "start_exploring") {
|
|
1417
|
+
return startExploring();
|
|
1418
|
+
}
|
|
1419
|
+
|
|
1420
|
+
if (message.command === "stop_exploring") {
|
|
1421
|
+
return stopExploring();
|
|
1422
|
+
}
|
|
1423
|
+
|
|
1424
|
+
if (message.command === "checkpoint") {
|
|
1425
|
+
return handleCheckpoint(message.note || "");
|
|
1426
|
+
}
|
|
1427
|
+
|
|
1428
|
+
return {
|
|
1429
|
+
ok: false,
|
|
1430
|
+
error: "unsupported_command",
|
|
1431
|
+
state: getState()
|
|
1432
|
+
};
|
|
1433
|
+
};
|
|
1434
|
+
|
|
1435
|
+
// ─── Message Router ─────────────────────────────────────────────────
|
|
1436
|
+
|
|
1437
|
+
chrome.runtime.onMessage.addListener((message, sender, sendResponse) => {
|
|
1438
|
+
if (!message || message.source === "background") {
|
|
1439
|
+
return false;
|
|
1440
|
+
}
|
|
1441
|
+
|
|
1442
|
+
// Content script sends incremental discovery snapshots.
|
|
1443
|
+
// Respond immediately; screenshot capture + backend forwarding runs async.
|
|
1444
|
+
if (message.source === "content" && message.type === "discovery_incremental") {
|
|
1445
|
+
sendResponse({ ok: true });
|
|
1446
|
+
handleDiscoveryIncremental(message, sender).catch((error) => {
|
|
1447
|
+
addLog(`incremental handler error: ${error.message}`);
|
|
1448
|
+
});
|
|
1449
|
+
return false;
|
|
1450
|
+
}
|
|
1451
|
+
|
|
1452
|
+
if (message.source === "sidepanel") {
|
|
1453
|
+
handleSidepanelCommand(message)
|
|
1454
|
+
.then((response) => {
|
|
1455
|
+
sendResponse(response);
|
|
1456
|
+
})
|
|
1457
|
+
.catch((error) => {
|
|
1458
|
+
sendResponse({
|
|
1459
|
+
ok: false,
|
|
1460
|
+
error: error instanceof Error ? error.message : "background_command_failed",
|
|
1461
|
+
state: getState()
|
|
1462
|
+
});
|
|
1463
|
+
});
|
|
1464
|
+
return true;
|
|
1465
|
+
}
|
|
1466
|
+
|
|
1467
|
+
return false;
|
|
1468
|
+
});
|
|
1469
|
+
|
|
1470
|
+
// Notify sidepanel when the active tab URL changes so it can re-filter by route
|
|
1471
|
+
chrome.tabs.onUpdated.addListener((tabId, changeInfo) => {
|
|
1472
|
+
if (changeInfo.url && currentManifest) {
|
|
1473
|
+
notifyAllContexts({ event: "tab_url_changed", url: changeInfo.url });
|
|
1474
|
+
}
|
|
1475
|
+
});
|
|
1476
|
+
|
|
1477
|
+
chrome.tabs.onRemoved.addListener((tabId) => {
|
|
1478
|
+
if (!activeSession || activeSession.tabId !== tabId) {
|
|
1479
|
+
return;
|
|
1480
|
+
}
|
|
1481
|
+
|
|
1482
|
+
const session = activeSession;
|
|
1483
|
+
const remainingSnapshots = pendingSnapshots.slice();
|
|
1484
|
+
activeSession = null;
|
|
1485
|
+
pendingSnapshots = [];
|
|
1486
|
+
|
|
1487
|
+
sendToBackend(MessageType.DISCOVERY_SESSION_STOP, {
|
|
1488
|
+
sessionId: session.sessionId,
|
|
1489
|
+
reason: "tab_closed",
|
|
1490
|
+
stoppedAt: new Date().toISOString(),
|
|
1491
|
+
pendingSnapshots: remainingSnapshots
|
|
1492
|
+
});
|
|
1493
|
+
|
|
1494
|
+
addLog(`exploration stopped because tab ${tabId} closed`);
|
|
1495
|
+
broadcastState();
|
|
1496
|
+
});
|
|
1497
|
+
|
|
1498
|
+
const enableActionSidePanel = () => {
|
|
1499
|
+
chrome.sidePanel.setPanelBehavior({ openPanelOnActionClick: true });
|
|
1500
|
+
};
|
|
1501
|
+
|
|
1502
|
+
chrome.runtime.onInstalled.addListener(() => {
|
|
1503
|
+
enableActionSidePanel();
|
|
1504
|
+
addLog("extension installed");
|
|
1505
|
+
broadcastState();
|
|
1506
|
+
});
|
|
1507
|
+
|
|
1508
|
+
chrome.runtime.onStartup.addListener(() => {
|
|
1509
|
+
enableActionSidePanel();
|
|
1510
|
+
addLog("extension started");
|
|
1511
|
+
broadcastState();
|
|
1512
|
+
});
|