@sable-ai/sdk-core 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +55 -0
- package/dist/esm/index.js +2431 -0
- package/dist/sable.iife.js +1486 -0
- package/dist/types/browser-bridge/actions.d.ts +27 -0
- package/dist/types/browser-bridge/dom-state.d.ts +37 -0
- package/dist/types/browser-bridge/index.d.ts +19 -0
- package/dist/types/connection/index.d.ts +26 -0
- package/dist/types/events/index.d.ts +15 -0
- package/dist/types/global.d.ts +26 -0
- package/dist/types/index.d.ts +23 -0
- package/dist/types/rpc.d.ts +22 -0
- package/dist/types/runtime/clipboard.d.ts +14 -0
- package/dist/types/runtime/index.d.ts +36 -0
- package/dist/types/runtime/video-overlay.d.ts +14 -0
- package/dist/types/session/debug-panel.d.ts +29 -0
- package/dist/types/session/index.d.ts +41 -0
- package/dist/types/types/index.d.ts +131 -0
- package/dist/types/version.d.ts +7 -0
- package/dist/types/vision/frame-source.d.ts +34 -0
- package/dist/types/vision/index.d.ts +29 -0
- package/dist/types/vision/publisher.d.ts +44 -0
- package/dist/types/vision/wireframe.d.ts +22 -0
- package/package.json +61 -0
- package/src/assets/visible-dom.js.txt +764 -0
- package/src/assets/wireframe.js.txt +678 -0
- package/src/assets.d.ts +24 -0
- package/src/browser-bridge/actions.ts +161 -0
- package/src/browser-bridge/dom-state.ts +103 -0
- package/src/browser-bridge/index.ts +99 -0
- package/src/connection/index.ts +49 -0
- package/src/events/index.ts +50 -0
- package/src/global.ts +35 -0
- package/src/index.test.ts +6 -0
- package/src/index.ts +43 -0
- package/src/rpc.ts +31 -0
- package/src/runtime/clipboard.ts +47 -0
- package/src/runtime/index.ts +138 -0
- package/src/runtime/video-overlay.ts +94 -0
- package/src/session/debug-panel.ts +254 -0
- package/src/session/index.ts +375 -0
- package/src/types/index.ts +176 -0
- package/src/version.ts +8 -0
- package/src/vision/frame-source.ts +111 -0
- package/src/vision/index.ts +70 -0
- package/src/vision/publisher.ts +106 -0
- package/src/vision/wireframe.ts +43 -0
|
@@ -0,0 +1,2431 @@
|
|
|
1
|
+
var __require = /* @__PURE__ */ ((x) => typeof require !== "undefined" ? require : typeof Proxy !== "undefined" ? new Proxy(x, {
|
|
2
|
+
get: (a, b) => (typeof require !== "undefined" ? require : a)[b]
|
|
3
|
+
}) : x)(function(x) {
|
|
4
|
+
if (typeof require !== "undefined")
|
|
5
|
+
return require.apply(this, arguments);
|
|
6
|
+
throw Error('Dynamic require of "' + x + '" is not supported');
|
|
7
|
+
});
|
|
8
|
+
|
|
9
|
+
// src/connection/index.ts
|
|
10
|
+
var DEFAULT_API_URL = "https://sable-api-gateway-9dfmhij9.wl.gateway.dev";
|
|
11
|
+
async function fetchConnectionDetails(input) {
|
|
12
|
+
const url = new URL("/connection-details", input.apiUrl);
|
|
13
|
+
url.searchParams.set("agentPublicId", input.publicKey);
|
|
14
|
+
url.searchParams.set("bridge", "user");
|
|
15
|
+
const res = await fetch(url.toString());
|
|
16
|
+
if (!res.ok) {
|
|
17
|
+
const body = await res.text().catch(() => "");
|
|
18
|
+
throw new Error(`connection-details failed: ${res.status} ${body}`);
|
|
19
|
+
}
|
|
20
|
+
return await res.json();
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
// src/events/index.ts
|
|
24
|
+
class SableEventEmitter {
|
|
25
|
+
listeners = new Map;
|
|
26
|
+
on(event, handler) {
|
|
27
|
+
let set = this.listeners.get(event);
|
|
28
|
+
if (!set) {
|
|
29
|
+
set = new Set;
|
|
30
|
+
this.listeners.set(event, set);
|
|
31
|
+
}
|
|
32
|
+
set.add(handler);
|
|
33
|
+
return () => {
|
|
34
|
+
set?.delete(handler);
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
emit(event, payload) {
|
|
38
|
+
const set = this.listeners.get(event);
|
|
39
|
+
if (!set || set.size === 0)
|
|
40
|
+
return;
|
|
41
|
+
for (const handler of set) {
|
|
42
|
+
try {
|
|
43
|
+
handler(payload);
|
|
44
|
+
} catch (err) {
|
|
45
|
+
console.warn(`[Sable] event handler for "${String(event)}" threw`, err);
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
clear() {
|
|
50
|
+
this.listeners.clear();
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
// src/rpc.ts
|
|
55
|
+
function safeParse(payload) {
|
|
56
|
+
try {
|
|
57
|
+
return payload ? JSON.parse(payload) : {};
|
|
58
|
+
} catch {
|
|
59
|
+
return {};
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
// src/runtime/clipboard.ts
|
|
64
|
+
async function handleCopyable(rpcName, payload) {
|
|
65
|
+
const message = typeof payload.message === "string" ? payload.message : "";
|
|
66
|
+
const url = typeof payload.url === "string" ? payload.url : "";
|
|
67
|
+
const toCopy = url || message;
|
|
68
|
+
if (!toCopy) {
|
|
69
|
+
console.warn(`[Sable] ${rpcName}: empty payload, nothing to copy`);
|
|
70
|
+
return { success: false, error: "empty payload" };
|
|
71
|
+
}
|
|
72
|
+
try {
|
|
73
|
+
if (navigator.clipboard?.writeText) {
|
|
74
|
+
await navigator.clipboard.writeText(toCopy);
|
|
75
|
+
return { success: true };
|
|
76
|
+
}
|
|
77
|
+
const ta = document.createElement("textarea");
|
|
78
|
+
ta.value = toCopy;
|
|
79
|
+
ta.style.position = "fixed";
|
|
80
|
+
ta.style.opacity = "0";
|
|
81
|
+
document.body.appendChild(ta);
|
|
82
|
+
ta.select();
|
|
83
|
+
const ok = document.execCommand("copy");
|
|
84
|
+
document.body.removeChild(ta);
|
|
85
|
+
if (!ok)
|
|
86
|
+
throw new Error("execCommand copy returned false");
|
|
87
|
+
return { success: true };
|
|
88
|
+
} catch (err) {
|
|
89
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
90
|
+
console.warn(`[Sable] ${rpcName}: copy failed`, msg);
|
|
91
|
+
return { success: false, error: msg };
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
// src/runtime/video-overlay.ts
|
|
96
|
+
var activeViewOverlay = null;
|
|
97
|
+
function removeViewOverlay() {
|
|
98
|
+
if (activeViewOverlay) {
|
|
99
|
+
activeViewOverlay.remove();
|
|
100
|
+
activeViewOverlay = null;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
function mountVideoOverlay(url) {
|
|
104
|
+
removeViewOverlay();
|
|
105
|
+
const overlay = document.createElement("div");
|
|
106
|
+
overlay.setAttribute("data-sable", "view-overlay");
|
|
107
|
+
Object.assign(overlay.style, {
|
|
108
|
+
position: "fixed",
|
|
109
|
+
top: "50%",
|
|
110
|
+
left: "50%",
|
|
111
|
+
transform: "translate(-50%, -50%)",
|
|
112
|
+
zIndex: "2147483646",
|
|
113
|
+
background: "rgba(0, 0, 0, 0.85)",
|
|
114
|
+
borderRadius: "12px",
|
|
115
|
+
padding: "8px",
|
|
116
|
+
boxShadow: "0 10px 40px rgba(0, 0, 0, 0.5)",
|
|
117
|
+
maxWidth: "min(80vw, 960px)",
|
|
118
|
+
maxHeight: "80vh",
|
|
119
|
+
display: "flex",
|
|
120
|
+
flexDirection: "column",
|
|
121
|
+
gap: "8px"
|
|
122
|
+
});
|
|
123
|
+
const closeBtn = document.createElement("button");
|
|
124
|
+
closeBtn.textContent = "✕";
|
|
125
|
+
closeBtn.setAttribute("aria-label", "Close video");
|
|
126
|
+
Object.assign(closeBtn.style, {
|
|
127
|
+
alignSelf: "flex-end",
|
|
128
|
+
background: "rgba(255,255,255,0.15)",
|
|
129
|
+
color: "white",
|
|
130
|
+
border: "none",
|
|
131
|
+
borderRadius: "999px",
|
|
132
|
+
width: "28px",
|
|
133
|
+
height: "28px",
|
|
134
|
+
cursor: "pointer",
|
|
135
|
+
fontSize: "14px",
|
|
136
|
+
lineHeight: "1"
|
|
137
|
+
});
|
|
138
|
+
closeBtn.addEventListener("click", removeViewOverlay);
|
|
139
|
+
const video = document.createElement("video");
|
|
140
|
+
video.src = url;
|
|
141
|
+
video.controls = false;
|
|
142
|
+
video.autoplay = true;
|
|
143
|
+
video.playsInline = true;
|
|
144
|
+
video.disablePictureInPicture = true;
|
|
145
|
+
video.setAttribute("controlslist", "nodownload nofullscreen noremoteplayback noplaybackrate");
|
|
146
|
+
Object.assign(video.style, {
|
|
147
|
+
maxWidth: "100%",
|
|
148
|
+
maxHeight: "70vh",
|
|
149
|
+
borderRadius: "8px",
|
|
150
|
+
display: "block"
|
|
151
|
+
});
|
|
152
|
+
const onEnded = () => {
|
|
153
|
+
if (activeViewOverlay === overlay) {
|
|
154
|
+
removeViewOverlay();
|
|
155
|
+
}
|
|
156
|
+
};
|
|
157
|
+
video.addEventListener("ended", onEnded);
|
|
158
|
+
overlay.appendChild(closeBtn);
|
|
159
|
+
overlay.appendChild(video);
|
|
160
|
+
document.body.appendChild(overlay);
|
|
161
|
+
activeViewOverlay = overlay;
|
|
162
|
+
video.play().catch((e) => {
|
|
163
|
+
console.warn("[Sable] switchView video autoplay blocked", e);
|
|
164
|
+
});
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
// src/runtime/index.ts
|
|
168
|
+
function noop() {
|
|
169
|
+
return Promise.resolve({ success: true });
|
|
170
|
+
}
|
|
171
|
+
var DEFAULT_RUNTIME = {
|
|
172
|
+
sendToolMessage: (payload) => handleCopyable("sendToolMessage", payload),
|
|
173
|
+
sendCopyableText: (payload) => handleCopyable("sendCopyableText", payload),
|
|
174
|
+
switchView: async (payload) => {
|
|
175
|
+
const mode = typeof payload.mode === "string" ? payload.mode : "";
|
|
176
|
+
const url = typeof payload.url === "string" ? payload.url : "";
|
|
177
|
+
if (mode === "video" && url) {
|
|
178
|
+
try {
|
|
179
|
+
mountVideoOverlay(url);
|
|
180
|
+
return { success: true };
|
|
181
|
+
} catch (err) {
|
|
182
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
183
|
+
console.warn("[Sable] switchView: failed to mount video overlay", msg);
|
|
184
|
+
return { success: false, error: msg };
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
removeViewOverlay();
|
|
188
|
+
return { success: true };
|
|
189
|
+
},
|
|
190
|
+
setCallControlsEnabled: noop,
|
|
191
|
+
setUserInputEnabled: noop,
|
|
192
|
+
setAgentInControl: noop,
|
|
193
|
+
showSuggestedReplies: noop,
|
|
194
|
+
hideSuggestedReplies: noop,
|
|
195
|
+
highlightHangup: noop,
|
|
196
|
+
hideVideo: noop,
|
|
197
|
+
showVideo: noop,
|
|
198
|
+
stopScreenShare: noop,
|
|
199
|
+
showSlide: noop,
|
|
200
|
+
hideSlide: noop,
|
|
201
|
+
responseFailed: noop,
|
|
202
|
+
requestContinue: noop,
|
|
203
|
+
greetingComplete: noop,
|
|
204
|
+
speechComplete: noop,
|
|
205
|
+
enableMicrophone: noop,
|
|
206
|
+
requestDisconnect: noop,
|
|
207
|
+
setNickelSession: noop
|
|
208
|
+
};
|
|
209
|
+
function toRpcHandler(name, method) {
|
|
210
|
+
return async (data) => {
|
|
211
|
+
const payload = safeParse(data.payload);
|
|
212
|
+
try {
|
|
213
|
+
const result = await method(payload);
|
|
214
|
+
return JSON.stringify(result ?? { success: true });
|
|
215
|
+
} catch (err) {
|
|
216
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
217
|
+
console.warn(`[Sable] runtime method "${name}" threw`, msg);
|
|
218
|
+
return JSON.stringify({ success: false, error: msg });
|
|
219
|
+
}
|
|
220
|
+
};
|
|
221
|
+
}
|
|
222
|
+
function installRuntime(room, userRuntime = {}) {
|
|
223
|
+
const merged = { ...DEFAULT_RUNTIME, ...userRuntime };
|
|
224
|
+
for (const [name, method] of Object.entries(merged)) {
|
|
225
|
+
room.registerRpcMethod(name, toRpcHandler(name, method));
|
|
226
|
+
}
|
|
227
|
+
const overrides = Object.keys(userRuntime).filter((k) => (k in DEFAULT_RUNTIME));
|
|
228
|
+
const extensions = Object.keys(userRuntime).filter((k) => !(k in DEFAULT_RUNTIME));
|
|
229
|
+
console.log("[Sable] runtime installed", {
|
|
230
|
+
total: Object.keys(merged).length,
|
|
231
|
+
overrides,
|
|
232
|
+
extensions
|
|
233
|
+
});
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
// src/browser-bridge/actions.ts
|
|
237
|
+
function isCoordinates(p) {
|
|
238
|
+
return typeof p === "object" && p !== null && typeof p.x === "number" && typeof p.y === "number";
|
|
239
|
+
}
|
|
240
|
+
function resolveTarget(payload) {
|
|
241
|
+
if (isCoordinates(payload)) {
|
|
242
|
+
return document.elementFromPoint(payload.x, payload.y);
|
|
243
|
+
}
|
|
244
|
+
if (typeof payload === "string") {
|
|
245
|
+
try {
|
|
246
|
+
return document.querySelector(payload);
|
|
247
|
+
} catch {
|
|
248
|
+
return null;
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
return null;
|
|
252
|
+
}
|
|
253
|
+
async function dispatchAction(action) {
|
|
254
|
+
switch (action.kind) {
|
|
255
|
+
case "click": {
|
|
256
|
+
const el = resolveTarget(action.payload);
|
|
257
|
+
if (!el)
|
|
258
|
+
throw new Error(`click: target not found`);
|
|
259
|
+
el.scrollIntoView({ block: "center", inline: "center" });
|
|
260
|
+
el.click();
|
|
261
|
+
return;
|
|
262
|
+
}
|
|
263
|
+
case "hover": {
|
|
264
|
+
const el = resolveTarget(action.payload);
|
|
265
|
+
if (!el)
|
|
266
|
+
return;
|
|
267
|
+
el.dispatchEvent(new MouseEvent("mouseover", { bubbles: true, cancelable: true }));
|
|
268
|
+
return;
|
|
269
|
+
}
|
|
270
|
+
case "type": {
|
|
271
|
+
const el = resolveTarget(action.payload);
|
|
272
|
+
if (!el)
|
|
273
|
+
throw new Error(`type: target not found`);
|
|
274
|
+
const input = el;
|
|
275
|
+
input.focus();
|
|
276
|
+
if (action.replace) {
|
|
277
|
+
input.value = "";
|
|
278
|
+
}
|
|
279
|
+
const text = action.text ?? "";
|
|
280
|
+
const proto = input instanceof HTMLTextAreaElement ? HTMLTextAreaElement.prototype : HTMLInputElement.prototype;
|
|
281
|
+
const setter = Object.getOwnPropertyDescriptor(proto, "value")?.set;
|
|
282
|
+
if (setter) {
|
|
283
|
+
setter.call(input, (input.value ?? "") + text);
|
|
284
|
+
} else {
|
|
285
|
+
input.value = (input.value ?? "") + text;
|
|
286
|
+
}
|
|
287
|
+
input.dispatchEvent(new Event("input", { bubbles: true }));
|
|
288
|
+
input.dispatchEvent(new Event("change", { bubbles: true }));
|
|
289
|
+
return;
|
|
290
|
+
}
|
|
291
|
+
case "key": {
|
|
292
|
+
const target = document.activeElement ?? document.body;
|
|
293
|
+
const key = action.key ?? "";
|
|
294
|
+
target.dispatchEvent(new KeyboardEvent("keydown", { key, bubbles: true, cancelable: true }));
|
|
295
|
+
target.dispatchEvent(new KeyboardEvent("keyup", { key, bubbles: true, cancelable: true }));
|
|
296
|
+
return;
|
|
297
|
+
}
|
|
298
|
+
case "clear": {
|
|
299
|
+
const el = document.activeElement;
|
|
300
|
+
if (!el || !("value" in el))
|
|
301
|
+
return;
|
|
302
|
+
el.value = "";
|
|
303
|
+
el.dispatchEvent(new Event("input", { bubbles: true }));
|
|
304
|
+
el.dispatchEvent(new Event("change", { bubbles: true }));
|
|
305
|
+
return;
|
|
306
|
+
}
|
|
307
|
+
case "navigate": {
|
|
308
|
+
const url = action.url ?? "";
|
|
309
|
+
if (!url)
|
|
310
|
+
return;
|
|
311
|
+
if (url === window.location.href)
|
|
312
|
+
return;
|
|
313
|
+
console.warn("[Sable] browser.navigate will reload the page; SDK must be re-injected on the new document", { url });
|
|
314
|
+
window.location.assign(url);
|
|
315
|
+
return;
|
|
316
|
+
}
|
|
317
|
+
case "evaluate": {
|
|
318
|
+
const expr = action.expression ?? "";
|
|
319
|
+
(0, eval)(expr);
|
|
320
|
+
return;
|
|
321
|
+
}
|
|
322
|
+
case "highlight_box":
|
|
323
|
+
case "highlight_text":
|
|
324
|
+
case "select_text":
|
|
325
|
+
case "center_scroll":
|
|
326
|
+
case "drag":
|
|
327
|
+
case "hide_cursor":
|
|
328
|
+
case "show_cursor":
|
|
329
|
+
return;
|
|
330
|
+
default:
|
|
331
|
+
throw new Error(`unsupported action kind: ${action.kind}`);
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
// src/assets/visible-dom.js.txt
|
|
336
|
+
var visible_dom_js_default = ` () => {
|
|
337
|
+
const vw = window.innerWidth || document.documentElement.clientWidth || 0;
|
|
338
|
+
const vh = window.innerHeight || document.documentElement.clientHeight || 0;
|
|
339
|
+
const sx = window.scrollX || window.pageXOffset || 0;
|
|
340
|
+
const sy = window.scrollY || window.pageYOffset || 0;
|
|
341
|
+
|
|
342
|
+
const clamp = (v, lo, hi) => Math.max(lo, Math.min(hi, v));
|
|
343
|
+
const norm = (s) => (s || "").trim().replace(/\\s+/g, " ");
|
|
344
|
+
const SKIP_TAGS = new Set(["script","style","noscript","template"]);
|
|
345
|
+
const WRAPPER_TAGS = new Set(["div","span","section","article","main","nav","header","footer"]);
|
|
346
|
+
|
|
347
|
+
const styleCache = new WeakMap();
|
|
348
|
+
const getStyleBits = (el) => {
|
|
349
|
+
const cached = styleCache.get(el);
|
|
350
|
+
if (cached) return cached;
|
|
351
|
+
let cs;
|
|
352
|
+
try { cs = window.getComputedStyle(el); } catch { cs = null; }
|
|
353
|
+
const bits = cs ? {
|
|
354
|
+
display: cs.display || "",
|
|
355
|
+
visibility: cs.visibility || "",
|
|
356
|
+
opacity: cs.opacity || "1",
|
|
357
|
+
} : { display: "", visibility: "", opacity: "1" };
|
|
358
|
+
styleCache.set(el, bits);
|
|
359
|
+
return bits;
|
|
360
|
+
};
|
|
361
|
+
|
|
362
|
+
// Cache for getBoundingClientRect - avoids redundant layout calls
|
|
363
|
+
const rectCache = new WeakMap();
|
|
364
|
+
const getCachedRect = (el) => {
|
|
365
|
+
const cached = rectCache.get(el);
|
|
366
|
+
if (cached !== undefined) return cached;
|
|
367
|
+
let r;
|
|
368
|
+
try { r = el.getBoundingClientRect(); } catch { r = null; }
|
|
369
|
+
rectCache.set(el, r);
|
|
370
|
+
return r;
|
|
371
|
+
};
|
|
372
|
+
|
|
373
|
+
// Cache for isControlLike - called multiple times per element
|
|
374
|
+
const controlCache = new WeakMap();
|
|
375
|
+
|
|
376
|
+
const isSkippableTag = (el) => {
|
|
377
|
+
const tag = (el.tagName || "").toLowerCase();
|
|
378
|
+
return !tag || SKIP_TAGS.has(tag);
|
|
379
|
+
};
|
|
380
|
+
|
|
381
|
+
const intersectViewport = (r) => {
|
|
382
|
+
const left = clamp(r.left, 0, vw);
|
|
383
|
+
const right = clamp(r.right, 0, vw);
|
|
384
|
+
const top = clamp(r.top, 0, vh);
|
|
385
|
+
const bottom = clamp(r.bottom, 0, vh);
|
|
386
|
+
const w = Math.max(0, right - left);
|
|
387
|
+
const h = Math.max(0, bottom - top);
|
|
388
|
+
return { x: left, y: top, width: w, height: h };
|
|
389
|
+
};
|
|
390
|
+
|
|
391
|
+
const roleOf = (el) => el.getAttribute?.("role") || null;
|
|
392
|
+
|
|
393
|
+
const directText = (el) => {
|
|
394
|
+
try {
|
|
395
|
+
let out = "";
|
|
396
|
+
for (const n of el.childNodes || []) {
|
|
397
|
+
if (n && n.nodeType === 3) {
|
|
398
|
+
const t = norm(n.textContent);
|
|
399
|
+
if (t) out += (out ? " " : "") + t;
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
return out;
|
|
403
|
+
} catch { return ""; }
|
|
404
|
+
};
|
|
405
|
+
const descendantText = (el, maxLen = 80) => {
|
|
406
|
+
try {
|
|
407
|
+
let t = norm(el.innerText || el.textContent || "");
|
|
408
|
+
if (!t) return "";
|
|
409
|
+
if (t.length > maxLen) t = t.slice(0, maxLen) + "…";
|
|
410
|
+
return t;
|
|
411
|
+
} catch { return ""; }
|
|
412
|
+
};
|
|
413
|
+
|
|
414
|
+
const isMedia = (el) => {
|
|
415
|
+
const tag = (el.tagName || "").toLowerCase();
|
|
416
|
+
return tag === "svg" || tag === "img" || tag === "canvas" || tag === "video";
|
|
417
|
+
};
|
|
418
|
+
|
|
419
|
+
const isControlLike = (el) => {
|
|
420
|
+
const cached = controlCache.get(el);
|
|
421
|
+
if (cached !== undefined) return cached;
|
|
422
|
+
|
|
423
|
+
const tag = (el.tagName || "").toLowerCase();
|
|
424
|
+
if (tag === "input" || tag === "textarea" || tag === "select" || tag === "button") {
|
|
425
|
+
controlCache.set(el, true); return true;
|
|
426
|
+
}
|
|
427
|
+
if (tag === "a") {
|
|
428
|
+
// treat as interactive even if Gatsby/JS navigation omits href
|
|
429
|
+
const href = (el.getAttribute("href") || "").trim();
|
|
430
|
+
if (href) { controlCache.set(el, true); return true; }
|
|
431
|
+
const role = (roleOf(el) || "").toLowerCase();
|
|
432
|
+
if (role === "link" || role === "button") { controlCache.set(el, true); return true; }
|
|
433
|
+
}
|
|
434
|
+
if (el.isContentEditable) { controlCache.set(el, true); return true; }
|
|
435
|
+
|
|
436
|
+
const role = roleOf(el);
|
|
437
|
+
if (role) {
|
|
438
|
+
const r = role.toLowerCase();
|
|
439
|
+
if ([
|
|
440
|
+
"textbox","searchbox","combobox","listbox","option",
|
|
441
|
+
"button","link","checkbox","radio","switch",
|
|
442
|
+
"tab","menuitem","slider","spinbutton"
|
|
443
|
+
].includes(r)) { controlCache.set(el, true); return true; }
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
const tabindex = el.getAttribute?.("tabindex");
|
|
447
|
+
if (tabindex !== null && tabindex !== "-1") { controlCache.set(el, true); return true; }
|
|
448
|
+
if (el.hasAttribute?.("onclick")) { controlCache.set(el, true); return true; }
|
|
449
|
+
if (typeof el.onclick === "function") { controlCache.set(el, true); return true; }
|
|
450
|
+
|
|
451
|
+
// common misuse: clickable divs/spans with pointer cursor
|
|
452
|
+
try {
|
|
453
|
+
const cs = window.getComputedStyle(el);
|
|
454
|
+
if (cs && cs.cursor === "pointer") { controlCache.set(el, true); return true; }
|
|
455
|
+
} catch {}
|
|
456
|
+
controlCache.set(el, false);
|
|
457
|
+
return false;
|
|
458
|
+
};
|
|
459
|
+
|
|
460
|
+
const hasOwnLabeling = (el) => {
|
|
461
|
+
if (directText(el)) return true;
|
|
462
|
+
const a = norm(el.getAttribute?.("aria-label"));
|
|
463
|
+
const t = norm(el.getAttribute?.("title"));
|
|
464
|
+
const alt = norm(el.getAttribute?.("alt"));
|
|
465
|
+
const ph = norm(el.getAttribute?.("placeholder"));
|
|
466
|
+
if (a || t || alt || ph) return true;
|
|
467
|
+
const cls = norm(el.getAttribute?.("class"));
|
|
468
|
+
if (cls) return true;
|
|
469
|
+
return false;
|
|
470
|
+
};
|
|
471
|
+
|
|
472
|
+
const isImportant = (el) => {
|
|
473
|
+
if (isControlLike(el)) return true;
|
|
474
|
+
if (isMedia(el)) return true;
|
|
475
|
+
if (roleOf(el)) return true;
|
|
476
|
+
if (hasOwnLabeling(el)) return true;
|
|
477
|
+
return false;
|
|
478
|
+
};
|
|
479
|
+
|
|
480
|
+
const cssEscape = (s) => {
|
|
481
|
+
if (window.CSS && typeof window.CSS.escape === "function") return window.CSS.escape(s);
|
|
482
|
+
return s.replace(/[^a-zA-Z0-9_\\-]/g, (c) => "\\\\" + c);
|
|
483
|
+
};
|
|
484
|
+
|
|
485
|
+
const domPath = (el) => {
|
|
486
|
+
const parts = [];
|
|
487
|
+
let cur = el;
|
|
488
|
+
while (cur && cur.nodeType === 1) {
|
|
489
|
+
const tag = cur.tagName.toLowerCase();
|
|
490
|
+
const id = cur.getAttribute?.("id");
|
|
491
|
+
// IDs are unique by HTML spec; skip querySelectorAll validation for speed
|
|
492
|
+
if (id) { parts.push("#" + cssEscape(id)); break; }
|
|
493
|
+
let nth = 1;
|
|
494
|
+
let sib = cur;
|
|
495
|
+
while ((sib = sib.previousElementSibling)) {
|
|
496
|
+
if (sib.tagName.toLowerCase() === tag) nth++;
|
|
497
|
+
}
|
|
498
|
+
parts.push(\`\${tag}:nth-of-type(\${nth})\`);
|
|
499
|
+
cur = cur.parentElement;
|
|
500
|
+
if (parts.length >= 12) break;
|
|
501
|
+
}
|
|
502
|
+
return parts.reverse().join(" > ");
|
|
503
|
+
};
|
|
504
|
+
|
|
505
|
+
const attrsOf = (el) => ({
|
|
506
|
+
"id": el.getAttribute?.("id") || null,
|
|
507
|
+
"class": el.getAttribute?.("class") || null,
|
|
508
|
+
"href": el.getAttribute?.("href") || null,
|
|
509
|
+
"type": el.getAttribute?.("type") || null,
|
|
510
|
+
"name": el.getAttribute?.("name") || null,
|
|
511
|
+
"placeholder": el.getAttribute?.("placeholder") || null,
|
|
512
|
+
"aria-label": el.getAttribute?.("aria-label") || null,
|
|
513
|
+
"aria-labelledby": el.getAttribute?.("aria-labelledby") || null,
|
|
514
|
+
"aria-expanded": el.getAttribute?.("aria-expanded") || null,
|
|
515
|
+
"aria-haspopup": el.getAttribute?.("aria-haspopup") || null,
|
|
516
|
+
"title": el.getAttribute?.("title") || null,
|
|
517
|
+
"alt": el.getAttribute?.("alt") || null,
|
|
518
|
+
"tabindex": el.getAttribute?.("tabindex") || null,
|
|
519
|
+
"contenteditable": el.isContentEditable ? "true" : (el.getAttribute?.("contenteditable") || null),
|
|
520
|
+
});
|
|
521
|
+
|
|
522
|
+
const svgDescriptor = (svg) => {
|
|
523
|
+
try {
|
|
524
|
+
if (!svg || (svg.tagName || "").toLowerCase() !== "svg") return "";
|
|
525
|
+
const viewBox = norm(svg.getAttribute("viewBox"));
|
|
526
|
+
const w = norm(svg.getAttribute("width"));
|
|
527
|
+
const h = norm(svg.getAttribute("height"));
|
|
528
|
+
const fill = norm(svg.getAttribute("fill"));
|
|
529
|
+
const stroke = norm(svg.getAttribute("stroke"));
|
|
530
|
+
const sw = norm(svg.getAttribute("stroke-width"));
|
|
531
|
+
|
|
532
|
+
const title = norm(svg.querySelector?.("title")?.textContent);
|
|
533
|
+
|
|
534
|
+
const cls = norm(svg.getAttribute("class"));
|
|
535
|
+
const useHref = norm(svg.querySelector?.("use")?.getAttribute("href") || svg.querySelector?.("use")?.getAttribute("xlink:href"));
|
|
536
|
+
|
|
537
|
+
const paths = svg.querySelectorAll?.("path") || [];
|
|
538
|
+
const pathCount = paths.length;
|
|
539
|
+
|
|
540
|
+
let dHash = "";
|
|
541
|
+
if (pathCount > 0) {
|
|
542
|
+
const d = paths[0].getAttribute("d") || "";
|
|
543
|
+
let hsh = 0;
|
|
544
|
+
for (let i = 0; i < d.length; i++) hsh = ((hsh << 5) - hsh + d.charCodeAt(i)) | 0;
|
|
545
|
+
dHash = String(hsh);
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
const parts = [];
|
|
549
|
+
if (title) parts.push(\`title=\${title}\`);
|
|
550
|
+
if (useHref) parts.push(\`use=\${useHref}\`);
|
|
551
|
+
if (viewBox) parts.push(\`vb=\${viewBox}\`);
|
|
552
|
+
if (w || h) parts.push(\`wh=\${w||"?"}x\${h||"?"}\`);
|
|
553
|
+
if (stroke || sw) parts.push(\`stroke=\${stroke||""}:\${sw||""}\`);
|
|
554
|
+
if (fill) parts.push(\`fill=\${fill}\`);
|
|
555
|
+
if (cls) parts.push(\`cls=\${cls.split(/\\s+/).slice(0,3).join(".")}\`);
|
|
556
|
+
if (pathCount) parts.push(\`paths=\${pathCount}\`);
|
|
557
|
+
if (dHash) parts.push(\`dhash=\${dHash}\`);
|
|
558
|
+
return parts.join(" ");
|
|
559
|
+
} catch { return ""; }
|
|
560
|
+
};
|
|
561
|
+
|
|
562
|
+
const iconDescriptorFromDescendants = (el, maxSvgs = 2) => {
|
|
563
|
+
try {
|
|
564
|
+
const out = [];
|
|
565
|
+
const svgs = el.querySelectorAll?.("svg") || [];
|
|
566
|
+
for (const svg of svgs) {
|
|
567
|
+
const d = svgDescriptor(svg);
|
|
568
|
+
if (d) out.push(d);
|
|
569
|
+
if (out.length >= maxSvgs) break;
|
|
570
|
+
}
|
|
571
|
+
const imgs = el.querySelectorAll?.("img") || [];
|
|
572
|
+
for (const img of imgs) {
|
|
573
|
+
const alt = norm(img.getAttribute("alt"));
|
|
574
|
+
const title = norm(img.getAttribute("title"));
|
|
575
|
+
const cls = norm(img.getAttribute("class"));
|
|
576
|
+
const parts = [];
|
|
577
|
+
if (alt) parts.push(\`alt=\${alt}\`);
|
|
578
|
+
if (title) parts.push(\`title=\${title}\`);
|
|
579
|
+
if (cls) parts.push(\`cls=\${cls.split(/\\s+/).slice(0,3).join(".")}\`);
|
|
580
|
+
const d = parts.join(" ");
|
|
581
|
+
if (d) out.push("img:" + d);
|
|
582
|
+
if (out.length >= maxSvgs + 1) break;
|
|
583
|
+
}
|
|
584
|
+
return out.join(" | ");
|
|
585
|
+
} catch { return ""; }
|
|
586
|
+
};
|
|
587
|
+
|
|
588
|
+
const baseLabelTextOf = (el) => {
|
|
589
|
+
const chunks = [];
|
|
590
|
+
const own = directText(el) || (isControlLike(el) ? descendantText(el) : "");
|
|
591
|
+
if (own) chunks.push(own);
|
|
592
|
+
|
|
593
|
+
for (const k of ["aria-label","title","alt","placeholder"]) {
|
|
594
|
+
const v = norm(el.getAttribute?.(k));
|
|
595
|
+
if (v) chunks.push(\`\${k}=\${v}\`);
|
|
596
|
+
}
|
|
597
|
+
|
|
598
|
+
const cls = norm(el.getAttribute?.("class"));
|
|
599
|
+
if (cls) chunks.push(\`class=\${cls.split(/\\s+/).slice(0,3).join(" ")}\`);
|
|
600
|
+
|
|
601
|
+
const tag = (el.tagName || "").toLowerCase();
|
|
602
|
+
if (tag === "svg") {
|
|
603
|
+
const sd = svgDescriptor(el);
|
|
604
|
+
if (sd) chunks.push(sd);
|
|
605
|
+
}
|
|
606
|
+
return chunks.join(" | ");
|
|
607
|
+
};
|
|
608
|
+
|
|
609
|
+
const needsIconScan = (el, baseName) => {
|
|
610
|
+
if (!isControlLike(el)) return false;
|
|
611
|
+
|
|
612
|
+
const dt = directText(el);
|
|
613
|
+
const a = norm(el.getAttribute?.("aria-label"));
|
|
614
|
+
const t = norm(el.getAttribute?.("title"));
|
|
615
|
+
const alt = norm(el.getAttribute?.("alt"));
|
|
616
|
+
const ph = norm(el.getAttribute?.("placeholder"));
|
|
617
|
+
const hasStrong = !!(dt || a || t || alt || ph);
|
|
618
|
+
if (!hasStrong) return true;
|
|
619
|
+
|
|
620
|
+
// still scan when label is basically only a class stub / generic tokens
|
|
621
|
+
const s = norm(baseName);
|
|
622
|
+
if (!s) return true;
|
|
623
|
+
if (/^class=/.test(s) && s.length < 40) return true;
|
|
624
|
+
return false;
|
|
625
|
+
};
|
|
626
|
+
|
|
627
|
+
const labelTextOf = (el) => {
|
|
628
|
+
const base = baseLabelTextOf(el);
|
|
629
|
+
if (needsIconScan(el, base)) {
|
|
630
|
+
const desc = iconDescriptorFromDescendants(el);
|
|
631
|
+
return desc ? (base ? \`\${base} | \${desc}\` : desc) : base;
|
|
632
|
+
}
|
|
633
|
+
return base;
|
|
634
|
+
};
|
|
635
|
+
|
|
636
|
+
const mergeIntoName = (entry, extra) => {
|
|
637
|
+
const v = norm(extra);
|
|
638
|
+
if (!v) return;
|
|
639
|
+
if (!entry.name) entry.name = v;
|
|
640
|
+
else if (!entry.name.includes(v)) entry.name = \`\${entry.name} | \${v}\`;
|
|
641
|
+
};
|
|
642
|
+
|
|
643
|
+
const fastIntersectInfo = (el) => {
|
|
644
|
+
if (!el || el.nodeType !== 1) return { ok: false };
|
|
645
|
+
if (isSkippableTag(el)) return { ok: false };
|
|
646
|
+
const r = getCachedRect(el);
|
|
647
|
+
if (!r || r.width < 1 || r.height < 1) return { ok: false };
|
|
648
|
+
const ib = intersectViewport(r);
|
|
649
|
+
if (ib.width < 1 || ib.height < 1) return { ok: false };
|
|
650
|
+
return { ok: true, rect: r, ibox: ib };
|
|
651
|
+
};
|
|
652
|
+
|
|
653
|
+
const isRenderable = (el) => {
|
|
654
|
+
const bits = getStyleBits(el);
|
|
655
|
+
if (bits.display === "none" || bits.visibility === "hidden") return false;
|
|
656
|
+
if (parseFloat(bits.opacity || "1") === 0) return false;
|
|
657
|
+
return true;
|
|
658
|
+
};
|
|
659
|
+
|
|
660
|
+
const samplePointsAdaptive = (ibox, el) => {
|
|
661
|
+
if (ibox.width < 1 || ibox.height < 1) return [];
|
|
662
|
+
const a = ibox.width * ibox.height;
|
|
663
|
+
const inset = 2;
|
|
664
|
+
const cx = ibox.x + ibox.width / 2;
|
|
665
|
+
const cy = ibox.y + ibox.height / 2;
|
|
666
|
+
|
|
667
|
+
// dynamic sampling:
|
|
668
|
+
// - very large regions: center only
|
|
669
|
+
// - medium: center + 2 corners
|
|
670
|
+
// - small: 5 points
|
|
671
|
+
let mode = 5;
|
|
672
|
+
if (a >= 150 * 150) mode = 1;
|
|
673
|
+
else if (a >= 60 * 60) mode = 3;
|
|
674
|
+
|
|
675
|
+
// controls tend to be small/precise; keep stronger checks
|
|
676
|
+
if (isControlLike(el) && mode < 5) mode = 5;
|
|
677
|
+
|
|
678
|
+
const x1 = ibox.x + inset;
|
|
679
|
+
const y1 = ibox.y + inset;
|
|
680
|
+
const x2 = ibox.x + ibox.width - inset;
|
|
681
|
+
const y2 = ibox.y + ibox.height - inset;
|
|
682
|
+
|
|
683
|
+
const pts = [];
|
|
684
|
+
pts.push([cx, cy]);
|
|
685
|
+
if (mode >= 3) {
|
|
686
|
+
pts.push([x1, y1]);
|
|
687
|
+
pts.push([x2, y2]);
|
|
688
|
+
}
|
|
689
|
+
if (mode >= 5) {
|
|
690
|
+
pts.push([x2, y1]);
|
|
691
|
+
pts.push([x1, y2]);
|
|
692
|
+
}
|
|
693
|
+
|
|
694
|
+
const out = [];
|
|
695
|
+
for (const [x, y] of pts) {
|
|
696
|
+
out.push([clamp(x, 0, Math.max(0, vw - 1)), clamp(y, 0, Math.max(0, vh - 1))]);
|
|
697
|
+
}
|
|
698
|
+
return out;
|
|
699
|
+
};
|
|
700
|
+
|
|
701
|
+
const notFullyOccluded = (el, ibox) => {
|
|
702
|
+
const pts = samplePointsAdaptive(ibox, el);
|
|
703
|
+
if (!pts.length) return false;
|
|
704
|
+
for (const [x, y] of pts) {
|
|
705
|
+
const top = document.elementFromPoint(x, y);
|
|
706
|
+
if (!top) continue;
|
|
707
|
+
if (top === el || el.contains(top)) return true;
|
|
708
|
+
}
|
|
709
|
+
return false;
|
|
710
|
+
};
|
|
711
|
+
|
|
712
|
+
const visibleInfo = (el) => {
|
|
713
|
+
const fi = fastIntersectInfo(el);
|
|
714
|
+
if (!fi.ok) return { ok: false };
|
|
715
|
+
|
|
716
|
+
if (!isRenderable(el)) return { ok: false };
|
|
717
|
+
|
|
718
|
+
if (!notFullyOccluded(el, fi.ibox)) return { ok: false };
|
|
719
|
+
|
|
720
|
+
return { ok: true, rect: fi.rect, ibox: fi.ibox };
|
|
721
|
+
};
|
|
722
|
+
|
|
723
|
+
const childrenForTraversal = (el) => {
|
|
724
|
+
const out = [];
|
|
725
|
+
const pushKids = (parent) => {
|
|
726
|
+
try {
|
|
727
|
+
const kids = parent.children ? Array.from(parent.children) : [];
|
|
728
|
+
for (const c of kids) {
|
|
729
|
+
if (isSkippableTag(c)) continue;
|
|
730
|
+
|
|
731
|
+
const fi = fastIntersectInfo(c);
|
|
732
|
+
if (fi.ok) { out.push(c); continue; }
|
|
733
|
+
|
|
734
|
+
// "tunnel" cases: 0×0 wrappers that may contain visible grandchildren
|
|
735
|
+
try {
|
|
736
|
+
const bits = getStyleBits(c);
|
|
737
|
+
const hasKids = (c.children && c.children.length) || (c.shadowRoot && c.shadowRoot.children && c.shadowRoot.children.length);
|
|
738
|
+
if ((bits.display === "contents") || hasKids) out.push(c);
|
|
739
|
+
} catch {
|
|
740
|
+
// if we can't read style, but it has children, still worth tunneling
|
|
741
|
+
if (c.children && c.children.length) out.push(c);
|
|
742
|
+
}
|
|
743
|
+
}
|
|
744
|
+
} catch {}
|
|
745
|
+
};
|
|
746
|
+
|
|
747
|
+
pushKids(el);
|
|
748
|
+
|
|
749
|
+
// include shadow root children too
|
|
750
|
+
try {
|
|
751
|
+
const sr = el.shadowRoot;
|
|
752
|
+
if (sr) pushKids(sr);
|
|
753
|
+
} catch {}
|
|
754
|
+
|
|
755
|
+
return out;
|
|
756
|
+
};
|
|
757
|
+
|
|
758
|
+
const visibleChildrenStrict = (el) => {
|
|
759
|
+
const out = [];
|
|
760
|
+
try {
|
|
761
|
+
const kids = el.children ? Array.from(el.children) : [];
|
|
762
|
+
for (const c of kids) {
|
|
763
|
+
const v = visibleInfo(c);
|
|
764
|
+
if (v.ok) out.push(c);
|
|
765
|
+
}
|
|
766
|
+
} catch {}
|
|
767
|
+
try {
|
|
768
|
+
const sr = el.shadowRoot;
|
|
769
|
+
if (sr && sr.children) {
|
|
770
|
+
for (const c of Array.from(sr.children)) {
|
|
771
|
+
const v = visibleInfo(c);
|
|
772
|
+
if (v.ok) out.push(c);
|
|
773
|
+
}
|
|
774
|
+
}
|
|
775
|
+
} catch {}
|
|
776
|
+
return out;
|
|
777
|
+
};
|
|
778
|
+
|
|
779
|
+
const area = (b) => Math.max(0, b.width) * Math.max(0, b.height);
|
|
780
|
+
const intersectArea = (a, b) => {
|
|
781
|
+
const x1 = Math.max(a.x, b.x);
|
|
782
|
+
const y1 = Math.max(a.y, b.y);
|
|
783
|
+
const x2 = Math.min(a.x + a.width, b.x + b.width);
|
|
784
|
+
const y2 = Math.min(a.y + a.height, b.y + b.height);
|
|
785
|
+
const w = Math.max(0, x2 - x1);
|
|
786
|
+
const h = Math.max(0, y2 - y1);
|
|
787
|
+
return w * h;
|
|
788
|
+
};
|
|
789
|
+
|
|
790
|
+
const isUselessWrapper = (el, child) => {
|
|
791
|
+
const tag = (el.tagName || "").toLowerCase();
|
|
792
|
+
if (!WRAPPER_TAGS.has(tag)) return false;
|
|
793
|
+
if (isImportant(el)) return false;
|
|
794
|
+
|
|
795
|
+
if (el.getAttribute?.("id")) return false;
|
|
796
|
+
if (roleOf(el)) return false;
|
|
797
|
+
const tabindex = el.getAttribute?.("tabindex");
|
|
798
|
+
if (tabindex && tabindex !== "-1") return false;
|
|
799
|
+
if (el.hasAttribute?.("onclick")) return false;
|
|
800
|
+
|
|
801
|
+
const f1 = fastIntersectInfo(el);
|
|
802
|
+
const f2 = fastIntersectInfo(child);
|
|
803
|
+
if (!f1.ok || !f2.ok) return false;
|
|
804
|
+
|
|
805
|
+
const b1 = f1.ibox, b2 = f2.ibox;
|
|
806
|
+
const a1 = area(b1), a2 = area(b2);
|
|
807
|
+
if (a1 < 1 || a2 < 1) return false;
|
|
808
|
+
const ia = intersectArea(b1, b2);
|
|
809
|
+
const overlap = ia / Math.min(a1, a2);
|
|
810
|
+
return overlap >= 0.92;
|
|
811
|
+
};
|
|
812
|
+
|
|
813
|
+
const resolveRepresentative = (el) => {
|
|
814
|
+
let cur = el;
|
|
815
|
+
for (let steps = 0; steps < 50; steps++) {
|
|
816
|
+
const kids = childrenForTraversal(cur);
|
|
817
|
+
if (kids.length !== 1) break;
|
|
818
|
+
const child = kids[0];
|
|
819
|
+
if (isUselessWrapper(cur, child)) { cur = child; continue; }
|
|
820
|
+
break;
|
|
821
|
+
}
|
|
822
|
+
return cur;
|
|
823
|
+
};
|
|
824
|
+
|
|
825
|
+
const nearestVisibleControlAncestor = (el) => {
|
|
826
|
+
let cur = el?.parentElement || null;
|
|
827
|
+
while (cur && cur !== document.body) {
|
|
828
|
+
if (isControlLike(cur)) {
|
|
829
|
+
const v = visibleInfo(cur);
|
|
830
|
+
if (v.ok) return cur;
|
|
831
|
+
}
|
|
832
|
+
cur = cur.parentElement;
|
|
833
|
+
}
|
|
834
|
+
return null;
|
|
835
|
+
};
|
|
836
|
+
|
|
837
|
+
const shouldEmit = (el) => {
|
|
838
|
+
if (isControlLike(el)) return true;
|
|
839
|
+
|
|
840
|
+
// text-heavy, misused divs/spans:
|
|
841
|
+
// emit if it has meaningful direct text and isn't within a visible control
|
|
842
|
+
const dt = directText(el);
|
|
843
|
+
if (dt && dt.length >= 2) {
|
|
844
|
+
const ctrl = nearestVisibleControlAncestor(el);
|
|
845
|
+
if (!ctrl) return true;
|
|
846
|
+
}
|
|
847
|
+
|
|
848
|
+
if (isImportant(el)) {
|
|
849
|
+
const ctrl = nearestVisibleControlAncestor(el);
|
|
850
|
+
return !ctrl;
|
|
851
|
+
}
|
|
852
|
+
return false;
|
|
853
|
+
};
|
|
854
|
+
|
|
855
|
+
const raw = [];
|
|
856
|
+
const byId = new Map();
|
|
857
|
+
const idOf = new WeakMap();
|
|
858
|
+
let nextId = 0;
|
|
859
|
+
|
|
860
|
+
const addNode = (el, parentId, depth, ibox) => {
|
|
861
|
+
const id = nextId++;
|
|
862
|
+
idOf.set(el, id);
|
|
863
|
+
const entry = {
|
|
864
|
+
id,
|
|
865
|
+
el,
|
|
866
|
+
parent_id: parentId,
|
|
867
|
+
depth,
|
|
868
|
+
tag: (el.tagName || "").toLowerCase(),
|
|
869
|
+
role: roleOf(el),
|
|
870
|
+
name: labelTextOf(el),
|
|
871
|
+
bbox_viewport: {
|
|
872
|
+
x: Math.round(ibox.x),
|
|
873
|
+
y: Math.round(ibox.y),
|
|
874
|
+
width: Math.round(ibox.width),
|
|
875
|
+
height: Math.round(ibox.height),
|
|
876
|
+
},
|
|
877
|
+
bbox_page: {
|
|
878
|
+
x: Math.round(ibox.x + sx),
|
|
879
|
+
y: Math.round(ibox.y + sy),
|
|
880
|
+
width: Math.round(ibox.width),
|
|
881
|
+
height: Math.round(ibox.height),
|
|
882
|
+
},
|
|
883
|
+
attributes: attrsOf(el),
|
|
884
|
+
dom_path: domPath(el),
|
|
885
|
+
};
|
|
886
|
+
raw.push(entry);
|
|
887
|
+
byId.set(id, entry);
|
|
888
|
+
return entry;
|
|
889
|
+
};
|
|
890
|
+
|
|
891
|
+
const walk = (startEl, parentEntry, depth) => {
|
|
892
|
+
if (!startEl || startEl.nodeType !== 1) return;
|
|
893
|
+
|
|
894
|
+
// prune early by viewport intersection
|
|
895
|
+
const fi = fastIntersectInfo(startEl);
|
|
896
|
+
if (!fi.ok) {
|
|
897
|
+
// IMPORTANT: wrapper may be 0x0 (e.g. display:contents) but have visible descendants.
|
|
898
|
+
const kids = childrenForTraversal(startEl);
|
|
899
|
+
for (const c of kids) walk(c, parentEntry, depth);
|
|
900
|
+
return;
|
|
901
|
+
}
|
|
902
|
+
|
|
903
|
+
const rep = resolveRepresentative(startEl);
|
|
904
|
+
if (!rep || rep.nodeType !== 1) return;
|
|
905
|
+
|
|
906
|
+
const v = visibleInfo(rep);
|
|
907
|
+
if (!v.ok) {
|
|
908
|
+
// rep itself may fail occlusion / rect tests even though descendants are visible
|
|
909
|
+
const kids = childrenForTraversal(rep);
|
|
910
|
+
for (const c of kids) walk(c, parentEntry, depth);
|
|
911
|
+
return;
|
|
912
|
+
}
|
|
913
|
+
|
|
914
|
+
const ctrl = (!isControlLike(rep)) ? nearestVisibleControlAncestor(rep) : null;
|
|
915
|
+
if (ctrl) {
|
|
916
|
+
const cv = visibleInfo(ctrl);
|
|
917
|
+
if (cv.ok) {
|
|
918
|
+
let ctrlEntry = null;
|
|
919
|
+
const existingId = idOf.get(ctrl);
|
|
920
|
+
if (existingId !== undefined) ctrlEntry = byId.get(existingId) || null;
|
|
921
|
+
if (!ctrlEntry) ctrlEntry = addNode(ctrl, parentEntry ? parentEntry.id : null, depth, cv.ibox);
|
|
922
|
+
|
|
923
|
+
// merge rep's base label always; scan icons only when needed
|
|
924
|
+
const repBase = baseLabelTextOf(rep);
|
|
925
|
+
if (repBase) mergeIntoName(ctrlEntry, repBase);
|
|
926
|
+
|
|
927
|
+
if (needsIconScan(ctrl, baseLabelTextOf(ctrl)) || needsIconScan(rep, repBase)) {
|
|
928
|
+
const repIcon = iconDescriptorFromDescendants(rep);
|
|
929
|
+
if (repIcon) mergeIntoName(ctrlEntry, repIcon);
|
|
930
|
+
}
|
|
931
|
+
|
|
932
|
+
if (!ctrlEntry.attributes) ctrlEntry.attributes = {};
|
|
933
|
+
if (needsIconScan(ctrl, baseLabelTextOf(ctrl))) {
|
|
934
|
+
const iconDesc = norm(iconDescriptorFromDescendants(ctrl));
|
|
935
|
+
if (iconDesc) ctrlEntry.attributes["icon-desc"] = iconDesc;
|
|
936
|
+
}
|
|
937
|
+
|
|
938
|
+
const kids = childrenForTraversal(rep);
|
|
939
|
+
for (const c of kids) walk(c, ctrlEntry, ctrlEntry.depth + 1);
|
|
940
|
+
return;
|
|
941
|
+
}
|
|
942
|
+
}
|
|
943
|
+
|
|
944
|
+
let myEntry = parentEntry;
|
|
945
|
+
let nextDepth = depth;
|
|
946
|
+
|
|
947
|
+
if (shouldEmit(rep)) {
|
|
948
|
+
myEntry = addNode(rep, parentEntry ? parentEntry.id : null, depth, v.ibox);
|
|
949
|
+
nextDepth = depth + 1;
|
|
950
|
+
|
|
951
|
+
if (isControlLike(rep) && needsIconScan(rep, baseLabelTextOf(rep))) {
|
|
952
|
+
const iconDesc = norm(iconDescriptorFromDescendants(rep));
|
|
953
|
+
if (iconDesc) {
|
|
954
|
+
mergeIntoName(myEntry, iconDesc);
|
|
955
|
+
myEntry.attributes["icon-desc"] = iconDesc;
|
|
956
|
+
}
|
|
957
|
+
}
|
|
958
|
+
}
|
|
959
|
+
|
|
960
|
+
const kids = childrenForTraversal(rep);
|
|
961
|
+
for (const c of kids) walk(c, myEntry, nextDepth);
|
|
962
|
+
};
|
|
963
|
+
|
|
964
|
+
const root = document.body || document.documentElement;
|
|
965
|
+
if (!root) return [];
|
|
966
|
+
|
|
967
|
+
const topKids = childrenForTraversal(root);
|
|
968
|
+
for (const c of topKids) walk(c, null, 0);
|
|
969
|
+
|
|
970
|
+
if (!raw.length) return [];
|
|
971
|
+
|
|
972
|
+
// merge labels into controls; remove labels
|
|
973
|
+
const keptIds = new Set(raw.map(e => e.id));
|
|
974
|
+
const domToVisibleId = new WeakMap();
|
|
975
|
+
for (const e of raw) domToVisibleId.set(e.el, e.id);
|
|
976
|
+
|
|
977
|
+
const isLabelTag = (el) => (el?.tagName || "").toLowerCase() === "label";
|
|
978
|
+
|
|
979
|
+
const mergeLabelIntoControl = (controlId, labelId, text) => {
|
|
980
|
+
const c = byId.get(controlId);
|
|
981
|
+
const l = byId.get(labelId);
|
|
982
|
+
if (!c || !l) return;
|
|
983
|
+
if (!keptIds.has(controlId) || !keptIds.has(labelId)) return;
|
|
984
|
+
const t = norm(text || l.name || "");
|
|
985
|
+
if (!t) return;
|
|
986
|
+
mergeIntoName(c, t);
|
|
987
|
+
};
|
|
988
|
+
|
|
989
|
+
const labelRemoval = new Set();
|
|
990
|
+
const controls = raw.filter(e => keptIds.has(e.id) && e.el && isControlLike(e.el));
|
|
991
|
+
|
|
992
|
+
for (const c of controls) {
|
|
993
|
+
try {
|
|
994
|
+
const labs = c.el.labels ? Array.from(c.el.labels) : [];
|
|
995
|
+
for (const labEl of labs) {
|
|
996
|
+
const lid = domToVisibleId.get(labEl);
|
|
997
|
+
if (lid !== undefined && keptIds.has(lid)) {
|
|
998
|
+
mergeLabelIntoControl(c.id, lid, norm(labEl.innerText || labEl.textContent || ""));
|
|
999
|
+
labelRemoval.add(lid);
|
|
1000
|
+
}
|
|
1001
|
+
}
|
|
1002
|
+
} catch {}
|
|
1003
|
+
}
|
|
1004
|
+
|
|
1005
|
+
for (const c of controls) {
|
|
1006
|
+
const v = c.el.getAttribute?.("aria-labelledby");
|
|
1007
|
+
if (!v) continue;
|
|
1008
|
+
const ids = v.split(/\\s+/).map(s => s.trim()).filter(Boolean);
|
|
1009
|
+
for (const domId of ids) {
|
|
1010
|
+
const labEl = document.getElementById(domId);
|
|
1011
|
+
if (!labEl) continue;
|
|
1012
|
+
const lid = domToVisibleId.get(labEl);
|
|
1013
|
+
if (lid !== undefined && keptIds.has(lid)) {
|
|
1014
|
+
mergeLabelIntoControl(c.id, lid, norm(labEl.innerText || labEl.textContent || ""));
|
|
1015
|
+
labelRemoval.add(lid);
|
|
1016
|
+
}
|
|
1017
|
+
}
|
|
1018
|
+
}
|
|
1019
|
+
|
|
1020
|
+
for (const e of raw) {
|
|
1021
|
+
if (!keptIds.has(e.id)) continue;
|
|
1022
|
+
if (!e.el || !isLabelTag(e.el)) continue;
|
|
1023
|
+
const f = e.el.getAttribute?.("for");
|
|
1024
|
+
if (!f) continue;
|
|
1025
|
+
const target = document.getElementById(f);
|
|
1026
|
+
if (!target) continue;
|
|
1027
|
+
const tid = domToVisibleId.get(target);
|
|
1028
|
+
if (tid !== undefined && keptIds.has(tid)) {
|
|
1029
|
+
mergeLabelIntoControl(tid, e.id, norm(e.el.innerText || e.el.textContent || ""));
|
|
1030
|
+
labelRemoval.add(e.id);
|
|
1031
|
+
}
|
|
1032
|
+
}
|
|
1033
|
+
|
|
1034
|
+
for (const lid of labelRemoval) keptIds.delete(lid);
|
|
1035
|
+
|
|
1036
|
+
// recompute parent/depth
|
|
1037
|
+
const findNearestKeptAncestorId = (el) => {
|
|
1038
|
+
let cur = el;
|
|
1039
|
+
while (cur) {
|
|
1040
|
+
let p = cur.parentElement;
|
|
1041
|
+
while (p) {
|
|
1042
|
+
const pid = idOf.get(p);
|
|
1043
|
+
if (pid !== undefined && keptIds.has(pid)) return pid;
|
|
1044
|
+
p = p.parentElement;
|
|
1045
|
+
}
|
|
1046
|
+
const rn = cur.getRootNode?.();
|
|
1047
|
+
if (rn && rn.host) { cur = rn.host; continue; }
|
|
1048
|
+
break;
|
|
1049
|
+
}
|
|
1050
|
+
return null;
|
|
1051
|
+
};
|
|
1052
|
+
|
|
1053
|
+
for (const e of raw) {
|
|
1054
|
+
if (!keptIds.has(e.id)) continue;
|
|
1055
|
+
e.parent_id = findNearestKeptAncestorId(e.el);
|
|
1056
|
+
}
|
|
1057
|
+
|
|
1058
|
+
const depthMemo = new Map();
|
|
1059
|
+
const depthOf = (id) => {
|
|
1060
|
+
if (depthMemo.has(id)) return depthMemo.get(id);
|
|
1061
|
+
const e = byId.get(id);
|
|
1062
|
+
if (!e || !keptIds.has(id)) return 0;
|
|
1063
|
+
const p = e.parent_id;
|
|
1064
|
+
const d = p === null ? 0 : depthOf(p) + 1;
|
|
1065
|
+
depthMemo.set(id, d);
|
|
1066
|
+
return d;
|
|
1067
|
+
};
|
|
1068
|
+
|
|
1069
|
+
for (const e of raw) {
|
|
1070
|
+
if (!keptIds.has(e.id)) continue;
|
|
1071
|
+
e.depth = depthOf(e.id);
|
|
1072
|
+
}
|
|
1073
|
+
|
|
1074
|
+
const kept = raw.filter(e => keptIds.has(e.id));
|
|
1075
|
+
kept.sort((a,b) => {
|
|
1076
|
+
const ay = a.bbox_viewport?.y ?? 0;
|
|
1077
|
+
const by = b.bbox_viewport?.y ?? 0;
|
|
1078
|
+
if (ay !== by) return ay - by;
|
|
1079
|
+
const ax = a.bbox_viewport?.x ?? 0;
|
|
1080
|
+
const bx = b.bbox_viewport?.x ?? 0;
|
|
1081
|
+
if (ax !== bx) return ax - bx;
|
|
1082
|
+
const aa = (a.bbox_viewport?.width ?? 0) * (a.bbox_viewport?.height ?? 0);
|
|
1083
|
+
const ba = (b.bbox_viewport?.width ?? 0) * (b.bbox_viewport?.height ?? 0);
|
|
1084
|
+
return aa - ba;
|
|
1085
|
+
});
|
|
1086
|
+
|
|
1087
|
+
return kept.map(e => ({
|
|
1088
|
+
id: e.id,
|
|
1089
|
+
parent_id: e.parent_id,
|
|
1090
|
+
depth: e.depth,
|
|
1091
|
+
tag: e.tag,
|
|
1092
|
+
role: e.role,
|
|
1093
|
+
name: e.name || "",
|
|
1094
|
+
bbox_viewport: e.bbox_viewport,
|
|
1095
|
+
bbox_page: e.bbox_page,
|
|
1096
|
+
attributes: e.attributes,
|
|
1097
|
+
dom_path: e.dom_path,
|
|
1098
|
+
}));
|
|
1099
|
+
}
|
|
1100
|
+
`;
|
|
1101
|
+
|
|
1102
|
+
// src/assets/wireframe.js.txt
|
|
1103
|
+
var wireframe_js_default = `// wireframe.js — ultra-fast DOM wireframe renderer
|
|
1104
|
+
// No dependencies needed
|
|
1105
|
+
|
|
1106
|
+
class Wireframe {
|
|
1107
|
+
constructor(root = document.body, opts = {}) {
|
|
1108
|
+
this.root = root;
|
|
1109
|
+
this.scale = opts.scale || 1;
|
|
1110
|
+
this.quality = opts.quality || 0.8;
|
|
1111
|
+
this.maxDepth = opts.maxDepth || 30;
|
|
1112
|
+
this.minSize = opts.minSize || 0; // temporarily 0 to debug
|
|
1113
|
+
this.showText = opts.showText !== false;
|
|
1114
|
+
this.showImages = opts.showImages !== false;
|
|
1115
|
+
// When true, image elements (<img> and background-image divs) are
|
|
1116
|
+
// rendered as their actual pixels using a CORS-aware fetch cache.
|
|
1117
|
+
// When false (default), they render as labeled placeholder boxes.
|
|
1118
|
+
// Either way they ALWAYS get a label now — the old behaviour of
|
|
1119
|
+
// drawing a mystery yellow box with no hint is gone.
|
|
1120
|
+
this.images = opts.images === true;
|
|
1121
|
+
// Class-level cache so it persists across captures. \`null\` entries are
|
|
1122
|
+
// negative cache (CORS failure, 404, decode error) — don't retry.
|
|
1123
|
+
if (!Wireframe._imageCache) Wireframe._imageCache = new Map();
|
|
1124
|
+
if (!Wireframe._imagePending) Wireframe._imagePending = new Map();
|
|
1125
|
+
this.colors = {
|
|
1126
|
+
bg: '#ffffff',
|
|
1127
|
+
block: '#e2e8f0',
|
|
1128
|
+
blockStroke: '#94a3b8',
|
|
1129
|
+
text: '#334155',
|
|
1130
|
+
input: '#dbeafe',
|
|
1131
|
+
inputStroke: '#3b82f6',
|
|
1132
|
+
button: '#bfdbfe',
|
|
1133
|
+
buttonStroke: '#2563eb',
|
|
1134
|
+
image: '#fde68a',
|
|
1135
|
+
imageStroke: '#f59e0b',
|
|
1136
|
+
imageCross: '#d97706',
|
|
1137
|
+
link: '#2563eb',
|
|
1138
|
+
heading: '#0f172a',
|
|
1139
|
+
nav: '#e0e7ff',
|
|
1140
|
+
navStroke: '#6366f1',
|
|
1141
|
+
...(opts.colors || {}),
|
|
1142
|
+
};
|
|
1143
|
+
}
|
|
1144
|
+
|
|
1145
|
+
async capture() {
|
|
1146
|
+
const t = performance.now();
|
|
1147
|
+
const rootRect = this.root.getBoundingClientRect();
|
|
1148
|
+
const scrollX = window.scrollX;
|
|
1149
|
+
const scrollY = window.scrollY;
|
|
1150
|
+
|
|
1151
|
+
const canvasW = rootRect.width;
|
|
1152
|
+
const canvasH = rootRect.height;
|
|
1153
|
+
|
|
1154
|
+
const canvas = document.createElement('canvas');
|
|
1155
|
+
canvas.width = canvasW * this.scale;
|
|
1156
|
+
canvas.height = canvasH * this.scale;
|
|
1157
|
+
const ctx = canvas.getContext('2d');
|
|
1158
|
+
ctx.scale(this.scale, this.scale);
|
|
1159
|
+
|
|
1160
|
+
// White background
|
|
1161
|
+
ctx.fillStyle = this.colors.bg;
|
|
1162
|
+
ctx.fillRect(0, 0, canvasW, canvasH);
|
|
1163
|
+
|
|
1164
|
+
// Two-pass: collect elements, draw boxes, then draw text on top
|
|
1165
|
+
this._drawCount = 0;
|
|
1166
|
+
this._skipCount = { tiny: 0, offscreen: 0, hidden: 0, depth: 0 };
|
|
1167
|
+
this._elements = [];
|
|
1168
|
+
this._pendingImageFetches = [];
|
|
1169
|
+
this._collectElements(this.root, rootRect, 0);
|
|
1170
|
+
|
|
1171
|
+
// If image rendering is enabled and the first pass kicked off some
|
|
1172
|
+
// fetches, give them a short window to land before drawing — that way
|
|
1173
|
+
// the first captured frame already has some images instead of only
|
|
1174
|
+
// placeholders. Subsequent frames hit the cache and are instant.
|
|
1175
|
+
if (this.images && this._pendingImageFetches.length > 0) {
|
|
1176
|
+
await Promise.race([
|
|
1177
|
+
Promise.all(this._pendingImageFetches),
|
|
1178
|
+
new Promise((r) => setTimeout(r, 400)),
|
|
1179
|
+
]);
|
|
1180
|
+
}
|
|
1181
|
+
|
|
1182
|
+
// Pass 1: draw all boxes/shapes
|
|
1183
|
+
for (const item of this._elements) {
|
|
1184
|
+
this._drawElementBox(ctx, item);
|
|
1185
|
+
}
|
|
1186
|
+
// Pass 2: find ALL text nodes directly via TreeWalker and draw them
|
|
1187
|
+
this._textCount = 0;
|
|
1188
|
+
this._drawAllText(ctx, rootRect);
|
|
1189
|
+
|
|
1190
|
+
console.log(\`[wireframe] drew \${this._drawCount} elements | skipped: \${JSON.stringify(this._skipCount)} | root: \${rootRect.width.toFixed(0)}x\${rootRect.height.toFixed(0)}\`);
|
|
1191
|
+
|
|
1192
|
+
const elapsed = performance.now() - t;
|
|
1193
|
+
console.log(\`[wireframe] captured: \${elapsed.toFixed(0)}ms | \${canvas.width}x\${canvas.height}\`);
|
|
1194
|
+
|
|
1195
|
+
return { canvas, elapsed };
|
|
1196
|
+
}
|
|
1197
|
+
|
|
1198
|
+
_collectElements(el, rootRect, depth) {
|
|
1199
|
+
if (depth > this.maxDepth) { this._skipCount.depth++; return; }
|
|
1200
|
+
|
|
1201
|
+
const children = el.children;
|
|
1202
|
+
|
|
1203
|
+
if (el.shadowRoot) {
|
|
1204
|
+
this._collectElements(el.shadowRoot, rootRect, depth);
|
|
1205
|
+
}
|
|
1206
|
+
for (let i = 0; i < children.length; i++) {
|
|
1207
|
+
const child = children[i];
|
|
1208
|
+
const rect = child.getBoundingClientRect();
|
|
1209
|
+
|
|
1210
|
+
if (rect.width < this.minSize || rect.height < this.minSize) { this._skipCount.tiny++; continue; }
|
|
1211
|
+
if (rect.bottom < rootRect.top || rect.top > rootRect.bottom) { this._skipCount.offscreen++; continue; }
|
|
1212
|
+
if (rect.right < rootRect.left || rect.left > rootRect.right) { this._skipCount.offscreen++; continue; }
|
|
1213
|
+
|
|
1214
|
+
const style = window.getComputedStyle(child);
|
|
1215
|
+
if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') { this._skipCount.hidden++; continue; }
|
|
1216
|
+
|
|
1217
|
+
const x = rect.left - rootRect.left;
|
|
1218
|
+
const y = rect.top - rootRect.top;
|
|
1219
|
+
const w = rect.width;
|
|
1220
|
+
const h = rect.height;
|
|
1221
|
+
const tag = child.tagName;
|
|
1222
|
+
const type = this._classifyElement(child, tag, style);
|
|
1223
|
+
|
|
1224
|
+
this._drawCount++;
|
|
1225
|
+
|
|
1226
|
+
// For image-like elements, capture a URL and a human label up front so
|
|
1227
|
+
// the draw pass can either render the actual bitmap (opts.images=true
|
|
1228
|
+
// + CORS-allowed fetch) or a labeled placeholder.
|
|
1229
|
+
let url = null;
|
|
1230
|
+
let label = null;
|
|
1231
|
+
if (type === 'image' || (type === 'icon' && tag === 'IMG')) {
|
|
1232
|
+
url = this._getImageUrl(child, style);
|
|
1233
|
+
label = this._getImageLabel(child, url);
|
|
1234
|
+
if (this.images && url) {
|
|
1235
|
+
const p = this._ensureImage(url);
|
|
1236
|
+
if (p && typeof p.then === 'function') {
|
|
1237
|
+
this._pendingImageFetches.push(p);
|
|
1238
|
+
}
|
|
1239
|
+
}
|
|
1240
|
+
}
|
|
1241
|
+
|
|
1242
|
+
this._elements.push({ type, x, y, w, h, el: child, style, tag, url, label });
|
|
1243
|
+
|
|
1244
|
+
this._collectElements(child, rootRect, depth + 1);
|
|
1245
|
+
}
|
|
1246
|
+
}
|
|
1247
|
+
|
|
1248
|
+
_classifyElement(el, tag, style) {
|
|
1249
|
+
if (tag === 'VIDEO' || tag === 'CANVAS') return 'skip';
|
|
1250
|
+
if (tag === 'IMG') return 'icon';
|
|
1251
|
+
if (tag === 'SVG') return 'icon';
|
|
1252
|
+
if (tag === 'INPUT' || tag === 'TEXTAREA' || tag === 'SELECT') return 'input';
|
|
1253
|
+
if (tag === 'BUTTON') return 'button';
|
|
1254
|
+
if (el.getAttribute('role') === 'button') {
|
|
1255
|
+
const bg = style.backgroundColor;
|
|
1256
|
+
if (bg && bg !== 'rgba(0, 0, 0, 0)' && bg !== 'transparent') return 'button';
|
|
1257
|
+
}
|
|
1258
|
+
if (tag === 'A') return 'link';
|
|
1259
|
+
if (tag === 'NAV' || el.getAttribute('role') === 'navigation') return 'nav';
|
|
1260
|
+
if (/^H[1-6]$/.test(tag)) return 'heading';
|
|
1261
|
+
if (tag === 'SPAN' || tag === 'P' || tag === 'LABEL' || tag === 'LI' || tag === 'TD' || tag === 'TH') return 'text';
|
|
1262
|
+
if (style.backgroundImage && style.backgroundImage !== 'none') return 'image';
|
|
1263
|
+
// Check if this element has direct text content (not just from children)
|
|
1264
|
+
if (this._hasDirectText(el)) return 'text';
|
|
1265
|
+
return 'block';
|
|
1266
|
+
}
|
|
1267
|
+
|
|
1268
|
+
// Extract a label for an icon — accessibility first
|
|
1269
|
+
_getIconLabel(el) {
|
|
1270
|
+
// Walk up to find the nearest interactive ancestor (often has the label)
|
|
1271
|
+
const interactive = el.closest('[role="button"], button, a, [role="link"], [aria-label]');
|
|
1272
|
+
|
|
1273
|
+
// 1. aria-label (self or nearest ancestor)
|
|
1274
|
+
const ariaLabel = el.getAttribute('aria-label') || (interactive && interactive.getAttribute('aria-label'));
|
|
1275
|
+
if (ariaLabel) return ariaLabel;
|
|
1276
|
+
|
|
1277
|
+
// 2. aria-labelledby (self or ancestor)
|
|
1278
|
+
const labelledBy = el.getAttribute('aria-labelledby') || (interactive && interactive.getAttribute('aria-labelledby'));
|
|
1279
|
+
if (labelledBy) {
|
|
1280
|
+
const ref = document.getElementById(labelledBy);
|
|
1281
|
+
if (ref) return ref.textContent.trim();
|
|
1282
|
+
}
|
|
1283
|
+
|
|
1284
|
+
// 3. aria-describedby
|
|
1285
|
+
const describedBy = el.getAttribute('aria-describedby') || (interactive && interactive.getAttribute('aria-describedby'));
|
|
1286
|
+
if (describedBy) {
|
|
1287
|
+
const ref = document.getElementById(describedBy);
|
|
1288
|
+
if (ref) return ref.textContent.trim();
|
|
1289
|
+
}
|
|
1290
|
+
|
|
1291
|
+
// 4. alt / title
|
|
1292
|
+
if (el.getAttribute('alt')) return el.getAttribute('alt');
|
|
1293
|
+
if (el.getAttribute('title')) return el.getAttribute('title');
|
|
1294
|
+
if (interactive && interactive.getAttribute('title')) return interactive.getAttribute('title');
|
|
1295
|
+
|
|
1296
|
+
// 5. Visible text content of parent button
|
|
1297
|
+
if (interactive && interactive !== el) {
|
|
1298
|
+
const text = interactive.textContent?.trim();
|
|
1299
|
+
if (text && text.length < 30) return text;
|
|
1300
|
+
}
|
|
1301
|
+
|
|
1302
|
+
return '◆';
|
|
1303
|
+
|
|
1304
|
+
return '◆';
|
|
1305
|
+
}
|
|
1306
|
+
|
|
1307
|
+
// Check if element has its own text nodes (not just inherited from children)
|
|
1308
|
+
_hasDirectText(el) {
|
|
1309
|
+
for (const node of el.childNodes) {
|
|
1310
|
+
if (node.nodeType === 3 && node.textContent.trim().length > 0) return true;
|
|
1311
|
+
}
|
|
1312
|
+
return false;
|
|
1313
|
+
}
|
|
1314
|
+
|
|
1315
|
+
// Resolve the image URL for an <img> or a background-image element.
|
|
1316
|
+
_getImageUrl(el, style) {
|
|
1317
|
+
if (el.tagName === 'IMG') {
|
|
1318
|
+
return el.currentSrc || el.src || null;
|
|
1319
|
+
}
|
|
1320
|
+
const bg = style && style.backgroundImage;
|
|
1321
|
+
if (bg && bg !== 'none') {
|
|
1322
|
+
// backgroundImage may contain multiple layers: url(a), linear-gradient(...)
|
|
1323
|
+
const m = bg.match(/url\\(["']?([^"')]+)["']?\\)/);
|
|
1324
|
+
if (m) return m[1];
|
|
1325
|
+
}
|
|
1326
|
+
return null;
|
|
1327
|
+
}
|
|
1328
|
+
|
|
1329
|
+
// Derive a human-readable label for an image element. Priority:
|
|
1330
|
+
// alt → aria-label → title → figcaption → nearest-ancestor aria-label
|
|
1331
|
+
// → nearest heading → decoded filename → "image"
|
|
1332
|
+
_getImageLabel(el, url) {
|
|
1333
|
+
const get = (attr) => {
|
|
1334
|
+
try { return el.getAttribute && el.getAttribute(attr); } catch { return null; }
|
|
1335
|
+
};
|
|
1336
|
+
if (get('alt')) return get('alt');
|
|
1337
|
+
if (get('aria-label')) return get('aria-label');
|
|
1338
|
+
if (get('title')) return get('title');
|
|
1339
|
+
|
|
1340
|
+
const fig = el.closest && el.closest('figure');
|
|
1341
|
+
if (fig) {
|
|
1342
|
+
const cap = fig.querySelector && fig.querySelector('figcaption');
|
|
1343
|
+
if (cap && cap.textContent) {
|
|
1344
|
+
const t = cap.textContent.trim();
|
|
1345
|
+
if (t) return t;
|
|
1346
|
+
}
|
|
1347
|
+
}
|
|
1348
|
+
|
|
1349
|
+
const labelledAncestor = el.closest && el.closest('[aria-label]');
|
|
1350
|
+
if (labelledAncestor && labelledAncestor !== el) {
|
|
1351
|
+
const a = labelledAncestor.getAttribute('aria-label');
|
|
1352
|
+
if (a) return a;
|
|
1353
|
+
}
|
|
1354
|
+
|
|
1355
|
+
// Notion cover images sit directly above the page H1 — use it as the
|
|
1356
|
+
// label when we have nothing else, so the agent can say "your Notion
|
|
1357
|
+
// page 'hello aidan' has a cover".
|
|
1358
|
+
if (url) {
|
|
1359
|
+
try {
|
|
1360
|
+
const u = new URL(url, window.location.href);
|
|
1361
|
+
const name = u.pathname.split('/').pop();
|
|
1362
|
+
if (name) {
|
|
1363
|
+
const decoded = decodeURIComponent(name).replace(/\\.[a-z0-9]{2,5}$/i, '');
|
|
1364
|
+
if (decoded && decoded.length < 60) return decoded;
|
|
1365
|
+
}
|
|
1366
|
+
} catch { /* ignore */ }
|
|
1367
|
+
}
|
|
1368
|
+
|
|
1369
|
+
return 'image';
|
|
1370
|
+
}
|
|
1371
|
+
|
|
1372
|
+
// Fetch → blob → ImageBitmap with CORS, class-level cache. Returns null
|
|
1373
|
+
// (and negatively caches) on any failure. Concurrent callers for the
|
|
1374
|
+
// same URL share one in-flight promise.
|
|
1375
|
+
_ensureImage(url) {
|
|
1376
|
+
const cache = Wireframe._imageCache;
|
|
1377
|
+
const pending = Wireframe._imagePending;
|
|
1378
|
+
if (cache.has(url)) return cache.get(url); // may be null
|
|
1379
|
+
if (pending.has(url)) return pending.get(url);
|
|
1380
|
+
|
|
1381
|
+
const p = (async () => {
|
|
1382
|
+
try {
|
|
1383
|
+
const res = await fetch(url, { mode: 'cors', cache: 'force-cache' });
|
|
1384
|
+
if (!res.ok) throw new Error(\`HTTP \${res.status}\`);
|
|
1385
|
+
const blob = await res.blob();
|
|
1386
|
+
const bitmap = await createImageBitmap(blob);
|
|
1387
|
+
cache.set(url, bitmap);
|
|
1388
|
+
// LRU bound
|
|
1389
|
+
if (cache.size > 50) {
|
|
1390
|
+
const firstKey = cache.keys().next().value;
|
|
1391
|
+
const evicted = cache.get(firstKey);
|
|
1392
|
+
if (evicted && typeof evicted.close === 'function') {
|
|
1393
|
+
try { evicted.close(); } catch { /* ignore */ }
|
|
1394
|
+
}
|
|
1395
|
+
cache.delete(firstKey);
|
|
1396
|
+
}
|
|
1397
|
+
return bitmap;
|
|
1398
|
+
} catch (e) {
|
|
1399
|
+
cache.set(url, null); // negative cache
|
|
1400
|
+
return null;
|
|
1401
|
+
} finally {
|
|
1402
|
+
pending.delete(url);
|
|
1403
|
+
}
|
|
1404
|
+
})();
|
|
1405
|
+
pending.set(url, p);
|
|
1406
|
+
return p;
|
|
1407
|
+
}
|
|
1408
|
+
|
|
1409
|
+
// Draw a labeled placeholder in the "yellow hatched box" style — used
|
|
1410
|
+
// whenever opts.images is off or the fetch failed. The label is the key
|
|
1411
|
+
// thing: the agent reads it off the frame and can reason about the
|
|
1412
|
+
// image's purpose without seeing the pixels.
|
|
1413
|
+
_drawImagePlaceholder(ctx, label, x, y, w, h, radius) {
|
|
1414
|
+
ctx.fillStyle = this.colors.image;
|
|
1415
|
+
this._roundRect(ctx, x, y, w, h, radius);
|
|
1416
|
+
ctx.fill();
|
|
1417
|
+
ctx.strokeStyle = this.colors.imageStroke;
|
|
1418
|
+
ctx.lineWidth = 1;
|
|
1419
|
+
ctx.stroke();
|
|
1420
|
+
if (this.showImages && w > 20 && h > 20) {
|
|
1421
|
+
ctx.strokeStyle = this.colors.imageCross;
|
|
1422
|
+
ctx.lineWidth = 0.5;
|
|
1423
|
+
ctx.beginPath();
|
|
1424
|
+
ctx.moveTo(x + 4, y + 4);
|
|
1425
|
+
ctx.lineTo(x + w - 4, y + h - 4);
|
|
1426
|
+
ctx.moveTo(x + w - 4, y + 4);
|
|
1427
|
+
ctx.lineTo(x + 4, y + h - 4);
|
|
1428
|
+
ctx.stroke();
|
|
1429
|
+
}
|
|
1430
|
+
// The label band: a semi-transparent stripe across the middle of the
|
|
1431
|
+
// placeholder with the text on top. Sized to remain legible at 1fps
|
|
1432
|
+
// capture resolution while not dominating the frame.
|
|
1433
|
+
if (label && w > 40 && h > 18) {
|
|
1434
|
+
const fontSize = Math.min(Math.max(Math.floor(h * 0.18), 11), 18);
|
|
1435
|
+
ctx.font = \`600 \${fontSize}px -apple-system, system-ui, sans-serif\`;
|
|
1436
|
+
const pad = 6;
|
|
1437
|
+
const bandH = fontSize + pad * 2;
|
|
1438
|
+
const bandY = y + (h - bandH) / 2;
|
|
1439
|
+
// Truncate label to fit
|
|
1440
|
+
let display = label;
|
|
1441
|
+
const maxW = w - pad * 4;
|
|
1442
|
+
while (ctx.measureText(display).width > maxW && display.length > 1) {
|
|
1443
|
+
display = display.slice(0, -2) + '…';
|
|
1444
|
+
}
|
|
1445
|
+
const textW = ctx.measureText(display).width;
|
|
1446
|
+
const bandW = Math.min(textW + pad * 2, w - 8);
|
|
1447
|
+
const bandX = x + (w - bandW) / 2;
|
|
1448
|
+
ctx.fillStyle = 'rgba(0, 0, 0, 0.55)';
|
|
1449
|
+
ctx.fillRect(bandX, bandY, bandW, bandH);
|
|
1450
|
+
ctx.fillStyle = '#ffffff';
|
|
1451
|
+
ctx.textBaseline = 'middle';
|
|
1452
|
+
ctx.textAlign = 'center';
|
|
1453
|
+
ctx.fillText(display, x + w / 2, bandY + bandH / 2);
|
|
1454
|
+
ctx.textAlign = 'start';
|
|
1455
|
+
}
|
|
1456
|
+
}
|
|
1457
|
+
|
|
1458
|
+
// Draw an actual raster image (from the cache) into the given rect.
|
|
1459
|
+
// Clips to the element's rounded-rect so the image respects border-radius.
|
|
1460
|
+
_drawImageBitmap(ctx, bitmap, x, y, w, h, radius) {
|
|
1461
|
+
try {
|
|
1462
|
+
ctx.save();
|
|
1463
|
+
this._roundRect(ctx, x, y, w, h, radius);
|
|
1464
|
+
ctx.clip();
|
|
1465
|
+
ctx.drawImage(bitmap, x, y, w, h);
|
|
1466
|
+
ctx.restore();
|
|
1467
|
+
ctx.strokeStyle = this.colors.imageStroke;
|
|
1468
|
+
ctx.lineWidth = 0.5;
|
|
1469
|
+
this._roundRect(ctx, x, y, w, h, radius);
|
|
1470
|
+
ctx.stroke();
|
|
1471
|
+
return true;
|
|
1472
|
+
} catch (e) {
|
|
1473
|
+
try { ctx.restore(); } catch { /* ignore */ }
|
|
1474
|
+
return false;
|
|
1475
|
+
}
|
|
1476
|
+
}
|
|
1477
|
+
|
|
1478
|
+
_drawElementBox(ctx, { type, x, y, w, h, el, style, url, label }) {
|
|
1479
|
+
const radius = Math.min(parseFloat(style.borderRadius) || 0, w / 2, h / 2, 8);
|
|
1480
|
+
|
|
1481
|
+
switch (type) {
|
|
1482
|
+
case 'skip':
|
|
1483
|
+
return; // don't draw images/video/canvas at all
|
|
1484
|
+
|
|
1485
|
+
case 'image': {
|
|
1486
|
+
// Try the cached bitmap first. If opts.images is off, \`url\` may be
|
|
1487
|
+
// set but the cache won't have it → falls through to placeholder.
|
|
1488
|
+
const cached = url ? Wireframe._imageCache.get(url) : null;
|
|
1489
|
+
if (cached && this._drawImageBitmap(ctx, cached, x, y, w, h, radius)) {
|
|
1490
|
+
break;
|
|
1491
|
+
}
|
|
1492
|
+
this._drawImagePlaceholder(ctx, label || 'image', x, y, w, h, radius);
|
|
1493
|
+
break;
|
|
1494
|
+
}
|
|
1495
|
+
|
|
1496
|
+
case 'icon': {
|
|
1497
|
+
// Large <img> tags are photos/avatars — render as real images if we
|
|
1498
|
+
// can, with a labeled placeholder fallback. Small <img> and <svg>
|
|
1499
|
+
// keep the tight "icon label" treatment.
|
|
1500
|
+
const isBigImg = el && el.tagName === 'IMG' && w >= 48 && h >= 48;
|
|
1501
|
+
if (isBigImg) {
|
|
1502
|
+
const cached = url ? Wireframe._imageCache.get(url) : null;
|
|
1503
|
+
if (cached && this._drawImageBitmap(ctx, cached, x, y, w, h, radius)) {
|
|
1504
|
+
break;
|
|
1505
|
+
}
|
|
1506
|
+
this._drawImagePlaceholder(ctx, label || 'image', x, y, w, h, radius);
|
|
1507
|
+
break;
|
|
1508
|
+
}
|
|
1509
|
+
|
|
1510
|
+
const iconLabel = label || this._getIconLabel(el);
|
|
1511
|
+
|
|
1512
|
+
ctx.fillStyle = '#f1f5f9';
|
|
1513
|
+
this._roundRect(ctx, x, y, w, h, 3);
|
|
1514
|
+
ctx.fill();
|
|
1515
|
+
ctx.strokeStyle = '#94a3b8';
|
|
1516
|
+
ctx.lineWidth = 0.5;
|
|
1517
|
+
ctx.stroke();
|
|
1518
|
+
|
|
1519
|
+
if (iconLabel && w >= 10 && h >= 8) {
|
|
1520
|
+
const fontSize = Math.min(Math.max(h * 0.55, 7), 11);
|
|
1521
|
+
ctx.fillStyle = '#64748b';
|
|
1522
|
+
ctx.font = \`\${fontSize}px -apple-system, sans-serif\`;
|
|
1523
|
+
ctx.textBaseline = 'middle';
|
|
1524
|
+
ctx.textAlign = 'center';
|
|
1525
|
+
const display = iconLabel.length > 6 ? iconLabel.slice(0, 5) + '…' : iconLabel;
|
|
1526
|
+
ctx.fillText(display, x + w / 2, y + h / 2);
|
|
1527
|
+
ctx.textAlign = 'start';
|
|
1528
|
+
}
|
|
1529
|
+
break;
|
|
1530
|
+
}
|
|
1531
|
+
|
|
1532
|
+
case 'input':
|
|
1533
|
+
ctx.fillStyle = this.colors.input;
|
|
1534
|
+
this._roundRect(ctx, x, y, w, h, radius);
|
|
1535
|
+
ctx.fill();
|
|
1536
|
+
ctx.strokeStyle = this.colors.inputStroke;
|
|
1537
|
+
ctx.lineWidth = 1;
|
|
1538
|
+
ctx.stroke();
|
|
1539
|
+
break;
|
|
1540
|
+
|
|
1541
|
+
case 'button': {
|
|
1542
|
+
// Use actual background color if it has one
|
|
1543
|
+
const btnBg = style.backgroundColor;
|
|
1544
|
+
if (btnBg && btnBg !== 'rgba(0, 0, 0, 0)' && btnBg !== 'transparent') {
|
|
1545
|
+
ctx.fillStyle = btnBg;
|
|
1546
|
+
} else {
|
|
1547
|
+
ctx.fillStyle = this.colors.button;
|
|
1548
|
+
}
|
|
1549
|
+
this._roundRect(ctx, x, y, w, h, radius);
|
|
1550
|
+
ctx.fill();
|
|
1551
|
+
ctx.strokeStyle = this.colors.buttonStroke;
|
|
1552
|
+
ctx.lineWidth = 1;
|
|
1553
|
+
ctx.stroke();
|
|
1554
|
+
break;
|
|
1555
|
+
}
|
|
1556
|
+
|
|
1557
|
+
case 'text':
|
|
1558
|
+
// Text elements don't need a box — just drawn in text pass
|
|
1559
|
+
break;
|
|
1560
|
+
|
|
1561
|
+
case 'nav':
|
|
1562
|
+
ctx.fillStyle = this.colors.nav;
|
|
1563
|
+
this._roundRect(ctx, x, y, w, h, radius);
|
|
1564
|
+
ctx.fill();
|
|
1565
|
+
ctx.strokeStyle = this.colors.navStroke;
|
|
1566
|
+
ctx.lineWidth = 1;
|
|
1567
|
+
ctx.stroke();
|
|
1568
|
+
break;
|
|
1569
|
+
|
|
1570
|
+
case 'block':
|
|
1571
|
+
default:
|
|
1572
|
+
ctx.strokeStyle = this.colors.blockStroke;
|
|
1573
|
+
ctx.lineWidth = 0.5;
|
|
1574
|
+
this._roundRect(ctx, x, y, w, h, radius);
|
|
1575
|
+
ctx.stroke();
|
|
1576
|
+
|
|
1577
|
+
const hasBg = style.backgroundColor && style.backgroundColor !== 'rgba(0, 0, 0, 0)' && style.backgroundColor !== 'transparent';
|
|
1578
|
+
if (hasBg) {
|
|
1579
|
+
ctx.fillStyle = this.colors.block;
|
|
1580
|
+
ctx.globalAlpha = 0.3;
|
|
1581
|
+
this._roundRect(ctx, x, y, w, h, radius);
|
|
1582
|
+
ctx.fill();
|
|
1583
|
+
ctx.globalAlpha = 1;
|
|
1584
|
+
}
|
|
1585
|
+
break;
|
|
1586
|
+
}
|
|
1587
|
+
}
|
|
1588
|
+
|
|
1589
|
+
// Use TreeWalker to find ALL visible text nodes, regardless of nesting depth
|
|
1590
|
+
_drawAllText(ctx, rootRect) {
|
|
1591
|
+
if (!this.showText) return;
|
|
1592
|
+
this._drawTextInRoot(ctx, rootRect, this.root);
|
|
1593
|
+
}
|
|
1594
|
+
|
|
1595
|
+
_drawTextInRoot(ctx, rootRect, root) {
|
|
1596
|
+
// Also handle shadow DOM roots
|
|
1597
|
+
const els = root.querySelectorAll('*');
|
|
1598
|
+
for (const el of els) {
|
|
1599
|
+
if (el.shadowRoot) {
|
|
1600
|
+
this._collectElements(el.shadowRoot, rootRect, 0);
|
|
1601
|
+
this._drawTextInRoot(ctx, rootRect, el.shadowRoot);
|
|
1602
|
+
}
|
|
1603
|
+
}
|
|
1604
|
+
|
|
1605
|
+
const walker = document.createTreeWalker(root, NodeFilter.SHOW_TEXT, {
|
|
1606
|
+
acceptNode: (node) => {
|
|
1607
|
+
const text = node.textContent.trim();
|
|
1608
|
+
if (!text) return NodeFilter.FILTER_REJECT;
|
|
1609
|
+
return NodeFilter.FILTER_ACCEPT;
|
|
1610
|
+
}
|
|
1611
|
+
});
|
|
1612
|
+
|
|
1613
|
+
while (walker.nextNode()) {
|
|
1614
|
+
const textNode = walker.currentNode;
|
|
1615
|
+
const text = textNode.textContent.trim();
|
|
1616
|
+
if (!text) continue;
|
|
1617
|
+
|
|
1618
|
+
// Get the parent element for positioning and style
|
|
1619
|
+
const parent = textNode.parentElement;
|
|
1620
|
+
if (!parent) continue;
|
|
1621
|
+
|
|
1622
|
+
// Use Range to get exact bounding rect of the text node
|
|
1623
|
+
const range = document.createRange();
|
|
1624
|
+
range.selectNodeContents(textNode);
|
|
1625
|
+
const rects = range.getClientRects();
|
|
1626
|
+
if (rects.length === 0) continue;
|
|
1627
|
+
|
|
1628
|
+
const style = window.getComputedStyle(parent);
|
|
1629
|
+
if (style.visibility === 'hidden' || style.opacity === '0') continue;
|
|
1630
|
+
|
|
1631
|
+
// Draw text at each rect (text can wrap across lines)
|
|
1632
|
+
for (const rect of rects) {
|
|
1633
|
+
if (rect.width < 4 || rect.height < 4) continue;
|
|
1634
|
+
if (rect.bottom < rootRect.top || rect.top > rootRect.bottom) continue;
|
|
1635
|
+
|
|
1636
|
+
const x = rect.left - rootRect.left;
|
|
1637
|
+
const y = rect.top - rootRect.top;
|
|
1638
|
+
const w = rect.width;
|
|
1639
|
+
const h = rect.height;
|
|
1640
|
+
|
|
1641
|
+
const fontSize = Math.min(Math.max(parseFloat(style.fontSize) || 11, 9), 24);
|
|
1642
|
+
const bold = parseInt(style.fontWeight) >= 600;
|
|
1643
|
+
const isLink = parent.closest('a') !== null;
|
|
1644
|
+
const isButton = parent.closest('button, [role="button"]') !== null;
|
|
1645
|
+
|
|
1646
|
+
// Detect if text is on a colored background (like blue "New" button)
|
|
1647
|
+
let color = this.colors.text;
|
|
1648
|
+
if (isLink) {
|
|
1649
|
+
color = this.colors.link;
|
|
1650
|
+
} else if (isButton) {
|
|
1651
|
+
// Check if parent or ancestor button has a colored bg
|
|
1652
|
+
const btn = parent.closest('button, [role="button"]');
|
|
1653
|
+
if (btn) {
|
|
1654
|
+
const btnStyle = window.getComputedStyle(btn);
|
|
1655
|
+
const bg = btnStyle.backgroundColor;
|
|
1656
|
+
if (bg && bg !== 'rgba(0, 0, 0, 0)' && bg !== 'transparent') {
|
|
1657
|
+
// Dark background → white text
|
|
1658
|
+
const isDarkBg = this._isDarkColor(bg);
|
|
1659
|
+
color = isDarkBg ? '#ffffff' : this.colors.text;
|
|
1660
|
+
}
|
|
1661
|
+
}
|
|
1662
|
+
}
|
|
1663
|
+
|
|
1664
|
+
this._drawText(ctx, text, x, y, w, h, color, fontSize, 'left', isLink, bold);
|
|
1665
|
+
this._textCount++;
|
|
1666
|
+
}
|
|
1667
|
+
}
|
|
1668
|
+
console.log(\`[wireframe] drew \${this._textCount} text nodes\`);
|
|
1669
|
+
}
|
|
1670
|
+
|
|
1671
|
+
// Parse rgb/rgba string and check if it's dark
|
|
1672
|
+
_isDarkColor(colorStr) {
|
|
1673
|
+
const match = colorStr.match(/\\d+/g);
|
|
1674
|
+
if (!match || match.length < 3) return false;
|
|
1675
|
+
const [r, g, b] = match.map(Number);
|
|
1676
|
+
// Luminance formula
|
|
1677
|
+
return (r * 0.299 + g * 0.587 + b * 0.114) < 128;
|
|
1678
|
+
}
|
|
1679
|
+
|
|
1680
|
+
_drawText(ctx, text, x, y, w, h, color, fontSize = 11, align = 'left', underline = false, bold = false) {
|
|
1681
|
+
if (w < 10 || h < 8) return;
|
|
1682
|
+
|
|
1683
|
+
ctx.fillStyle = color;
|
|
1684
|
+
ctx.font = \`\${bold ? 'bold ' : ''}\${fontSize}px -apple-system, sans-serif\`;
|
|
1685
|
+
ctx.textBaseline = 'middle';
|
|
1686
|
+
|
|
1687
|
+
// Truncate text to fit
|
|
1688
|
+
let display = text;
|
|
1689
|
+
while (ctx.measureText(display).width > w && display.length > 1) {
|
|
1690
|
+
display = display.slice(0, -2) + '…';
|
|
1691
|
+
}
|
|
1692
|
+
|
|
1693
|
+
const textY = y + h / 2;
|
|
1694
|
+
let textX = x;
|
|
1695
|
+
if (align === 'center') {
|
|
1696
|
+
textX = x + (w - ctx.measureText(display).width) / 2;
|
|
1697
|
+
}
|
|
1698
|
+
|
|
1699
|
+
ctx.fillText(display, textX, textY);
|
|
1700
|
+
|
|
1701
|
+
if (underline) {
|
|
1702
|
+
const textW = ctx.measureText(display).width;
|
|
1703
|
+
ctx.beginPath();
|
|
1704
|
+
ctx.moveTo(textX, textY + fontSize / 2);
|
|
1705
|
+
ctx.lineTo(textX + textW, textY + fontSize / 2);
|
|
1706
|
+
ctx.strokeStyle = color;
|
|
1707
|
+
ctx.lineWidth = 0.5;
|
|
1708
|
+
ctx.stroke();
|
|
1709
|
+
}
|
|
1710
|
+
}
|
|
1711
|
+
|
|
1712
|
+
_roundRect(ctx, x, y, w, h, r) {
|
|
1713
|
+
ctx.beginPath();
|
|
1714
|
+
ctx.moveTo(x + r, y);
|
|
1715
|
+
ctx.lineTo(x + w - r, y);
|
|
1716
|
+
ctx.quadraticCurveTo(x + w, y, x + w, y + r);
|
|
1717
|
+
ctx.lineTo(x + w, y + h - r);
|
|
1718
|
+
ctx.quadraticCurveTo(x + w, y + h, x + w - r, y + h);
|
|
1719
|
+
ctx.lineTo(x + r, y + h);
|
|
1720
|
+
ctx.quadraticCurveTo(x, y + h, x, y + h - r);
|
|
1721
|
+
ctx.lineTo(x, y + r);
|
|
1722
|
+
ctx.quadraticCurveTo(x, y, x + r, y);
|
|
1723
|
+
ctx.closePath();
|
|
1724
|
+
}
|
|
1725
|
+
|
|
1726
|
+
// --- Export ---
|
|
1727
|
+
async download(filename) {
|
|
1728
|
+
const { canvas } = await this.capture();
|
|
1729
|
+
canvas.toBlob(b => {
|
|
1730
|
+
const a = document.createElement('a');
|
|
1731
|
+
a.href = URL.createObjectURL(b);
|
|
1732
|
+
a.download = filename || \`wireframe-\${Date.now()}.png\`;
|
|
1733
|
+
a.click();
|
|
1734
|
+
URL.revokeObjectURL(a.href);
|
|
1735
|
+
}, 'image/png');
|
|
1736
|
+
}
|
|
1737
|
+
|
|
1738
|
+
async toDataURL() {
|
|
1739
|
+
const { canvas } = await this.capture();
|
|
1740
|
+
return canvas.toDataURL('image/png');
|
|
1741
|
+
}
|
|
1742
|
+
|
|
1743
|
+
async toBlob() {
|
|
1744
|
+
const { canvas } = await this.capture();
|
|
1745
|
+
return new Promise(r => canvas.toBlob(r, 'image/png'));
|
|
1746
|
+
}
|
|
1747
|
+
}
|
|
1748
|
+
|
|
1749
|
+
// --- Quick API ---
|
|
1750
|
+
async function wireframe(el = document.body, opts = {}) {
|
|
1751
|
+
const w = new Wireframe(el, opts);
|
|
1752
|
+
return w.capture();
|
|
1753
|
+
}
|
|
1754
|
+
|
|
1755
|
+
async function demo() {
|
|
1756
|
+
const w = new Wireframe(document.body, { scale: 1 });
|
|
1757
|
+
|
|
1758
|
+
// Benchmark
|
|
1759
|
+
const runs = 5;
|
|
1760
|
+
const times = [];
|
|
1761
|
+
for (let i = 0; i < runs; i++) {
|
|
1762
|
+
const { elapsed } = await w.capture();
|
|
1763
|
+
times.push(elapsed);
|
|
1764
|
+
}
|
|
1765
|
+
const avg = times.reduce((a, b) => a + b) / times.length;
|
|
1766
|
+
console.log(\`\\n=== WIREFRAME BENCHMARK ===\`);
|
|
1767
|
+
console.log(\`Runs: \${times.map(t => t.toFixed(0) + 'ms').join(', ')}\`);
|
|
1768
|
+
console.log(\`Avg: \${avg.toFixed(0)}ms\`);
|
|
1769
|
+
|
|
1770
|
+
// Download
|
|
1771
|
+
await w.download();
|
|
1772
|
+
|
|
1773
|
+
window.__wireframe = w;
|
|
1774
|
+
return w;
|
|
1775
|
+
}
|
|
1776
|
+
|
|
1777
|
+
console.log('Wireframe ready! No dependencies needed.');
|
|
1778
|
+
console.log(' await demo() — benchmark + download');
|
|
1779
|
+
console.log(' await new Wireframe().download() — quick download');
|
|
1780
|
+
console.log(' await new Wireframe().toDataURL() — base64 for LLM');
|
|
1781
|
+
`;
|
|
1782
|
+
|
|
1783
|
+
// src/vision/wireframe.ts
|
|
1784
|
+
var wireframeCtor = null;
|
|
1785
|
+
function getWireframeCtor() {
|
|
1786
|
+
if (!wireframeCtor) {
|
|
1787
|
+
wireframeCtor = (0, eval)(`(function(){
|
|
1788
|
+
var console = Object.assign({}, globalThis.console, {
|
|
1789
|
+
log: function(){}, info: function(){}, debug: function(){}
|
|
1790
|
+
});
|
|
1791
|
+
${wireframe_js_default};
|
|
1792
|
+
return Wireframe;
|
|
1793
|
+
})()`);
|
|
1794
|
+
}
|
|
1795
|
+
return wireframeCtor;
|
|
1796
|
+
}
|
|
1797
|
+
|
|
1798
|
+
// src/browser-bridge/dom-state.ts
|
|
1799
|
+
var visibleDomFn = null;
|
|
1800
|
+
function getVisibleDomFn() {
|
|
1801
|
+
if (!visibleDomFn) {
|
|
1802
|
+
visibleDomFn = (0, eval)(`(${visible_dom_js_default})`);
|
|
1803
|
+
}
|
|
1804
|
+
return visibleDomFn;
|
|
1805
|
+
}
|
|
1806
|
+
async function captureDomState() {
|
|
1807
|
+
const elements = getVisibleDomFn()();
|
|
1808
|
+
const Wireframe = getWireframeCtor();
|
|
1809
|
+
const wf = new Wireframe(document.body, {});
|
|
1810
|
+
const dataUrl = await wf.toDataURL();
|
|
1811
|
+
const b64 = dataUrl.replace(/^data:[^,]+,/, "");
|
|
1812
|
+
return {
|
|
1813
|
+
screenshot_jpeg_b64: b64,
|
|
1814
|
+
elements,
|
|
1815
|
+
viewport: { width: window.innerWidth, height: window.innerHeight },
|
|
1816
|
+
url: window.location.href
|
|
1817
|
+
};
|
|
1818
|
+
}
|
|
1819
|
+
async function settle() {
|
|
1820
|
+
const raf2 = () => new Promise((r) => requestAnimationFrame(() => requestAnimationFrame(() => r())));
|
|
1821
|
+
await raf2();
|
|
1822
|
+
await new Promise((resolve) => {
|
|
1823
|
+
const QUIET_MS = 30;
|
|
1824
|
+
const MAX_MS = 30;
|
|
1825
|
+
const start = performance.now();
|
|
1826
|
+
let lastMut = performance.now();
|
|
1827
|
+
const obs = new MutationObserver(() => {
|
|
1828
|
+
lastMut = performance.now();
|
|
1829
|
+
});
|
|
1830
|
+
obs.observe(document.documentElement, {
|
|
1831
|
+
subtree: true,
|
|
1832
|
+
childList: true,
|
|
1833
|
+
attributes: true,
|
|
1834
|
+
characterData: true
|
|
1835
|
+
});
|
|
1836
|
+
const tick = () => {
|
|
1837
|
+
const now = performance.now();
|
|
1838
|
+
if (now - lastMut >= QUIET_MS) {
|
|
1839
|
+
obs.disconnect();
|
|
1840
|
+
resolve();
|
|
1841
|
+
return;
|
|
1842
|
+
}
|
|
1843
|
+
if (now - start >= MAX_MS) {
|
|
1844
|
+
obs.disconnect();
|
|
1845
|
+
resolve();
|
|
1846
|
+
return;
|
|
1847
|
+
}
|
|
1848
|
+
requestAnimationFrame(tick);
|
|
1849
|
+
};
|
|
1850
|
+
requestAnimationFrame(tick);
|
|
1851
|
+
});
|
|
1852
|
+
await raf2();
|
|
1853
|
+
}
|
|
1854
|
+
|
|
1855
|
+
// src/browser-bridge/index.ts
|
|
1856
|
+
function makeHandler(name, body) {
|
|
1857
|
+
return async (data) => {
|
|
1858
|
+
let req = {};
|
|
1859
|
+
try {
|
|
1860
|
+
req = data.payload ? JSON.parse(data.payload) : {};
|
|
1861
|
+
} catch (e) {
|
|
1862
|
+
console.warn(`[Sable] ${name}: bad JSON payload`, e);
|
|
1863
|
+
}
|
|
1864
|
+
try {
|
|
1865
|
+
const result = await body(req);
|
|
1866
|
+
return JSON.stringify(result ?? {});
|
|
1867
|
+
} catch (err) {
|
|
1868
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1869
|
+
console.warn(`[Sable] ${name}: handler error`, msg);
|
|
1870
|
+
return JSON.stringify({ error: msg });
|
|
1871
|
+
}
|
|
1872
|
+
};
|
|
1873
|
+
}
|
|
1874
|
+
function registerBrowserHandlers(room) {
|
|
1875
|
+
room.registerRpcMethod("browser.execute_action", makeHandler("browser.execute_action", async (req) => {
|
|
1876
|
+
const action = req.action;
|
|
1877
|
+
if (!action || typeof action !== "object") {
|
|
1878
|
+
throw new Error("execute_action: missing action");
|
|
1879
|
+
}
|
|
1880
|
+
await dispatchAction(action);
|
|
1881
|
+
return {};
|
|
1882
|
+
}));
|
|
1883
|
+
room.registerRpcMethod("browser.get_dom_state", makeHandler("browser.get_dom_state", async () => captureDomState()));
|
|
1884
|
+
room.registerRpcMethod("browser.get_url", makeHandler("browser.get_url", async () => ({ url: window.location.href })));
|
|
1885
|
+
room.registerRpcMethod("browser.get_viewport", makeHandler("browser.get_viewport", async () => ({
|
|
1886
|
+
width: window.innerWidth,
|
|
1887
|
+
height: window.innerHeight
|
|
1888
|
+
})));
|
|
1889
|
+
room.registerRpcMethod("browser.verify_selector", makeHandler("browser.verify_selector", async (req) => {
|
|
1890
|
+
const selector = typeof req.selector === "string" ? req.selector : "";
|
|
1891
|
+
let matches = false;
|
|
1892
|
+
try {
|
|
1893
|
+
matches = !!document.querySelector(selector);
|
|
1894
|
+
} catch {
|
|
1895
|
+
matches = false;
|
|
1896
|
+
}
|
|
1897
|
+
return { matches };
|
|
1898
|
+
}));
|
|
1899
|
+
room.registerRpcMethod("browser.settle", makeHandler("browser.settle", async () => {
|
|
1900
|
+
await settle();
|
|
1901
|
+
return {};
|
|
1902
|
+
}));
|
|
1903
|
+
console.log("[Sable] browser bridge RPCs registered");
|
|
1904
|
+
}
|
|
1905
|
+
|
|
1906
|
+
// src/vision/frame-source.ts
|
|
1907
|
+
var DEFAULT_RATE_HZ = 2;
|
|
1908
|
+
function intervalMs(rate) {
|
|
1909
|
+
const r = typeof rate === "number" && rate > 0 ? rate : DEFAULT_RATE_HZ;
|
|
1910
|
+
return Math.max(1, Math.round(1000 / r));
|
|
1911
|
+
}
|
|
1912
|
+
function syncCanvasSize(canvas) {
|
|
1913
|
+
const w = Math.max(1, window.innerWidth);
|
|
1914
|
+
const h = Math.max(1, window.innerHeight);
|
|
1915
|
+
if (canvas.width !== w || canvas.height !== h) {
|
|
1916
|
+
canvas.width = w;
|
|
1917
|
+
canvas.height = h;
|
|
1918
|
+
}
|
|
1919
|
+
}
|
|
1920
|
+
function startFrameSource(source, canvas) {
|
|
1921
|
+
const ctx = canvas.getContext("2d", { alpha: false });
|
|
1922
|
+
if (!ctx) {
|
|
1923
|
+
console.warn("[Sable] frame source: 2d context unavailable");
|
|
1924
|
+
return () => {};
|
|
1925
|
+
}
|
|
1926
|
+
const delayMs = intervalMs(source.rate);
|
|
1927
|
+
let stopped = false;
|
|
1928
|
+
let timer;
|
|
1929
|
+
let inFlight = false;
|
|
1930
|
+
const tick = async () => {
|
|
1931
|
+
if (stopped)
|
|
1932
|
+
return;
|
|
1933
|
+
if (!inFlight) {
|
|
1934
|
+
inFlight = true;
|
|
1935
|
+
try {
|
|
1936
|
+
syncCanvasSize(canvas);
|
|
1937
|
+
if (source.type === "wireframe") {
|
|
1938
|
+
const includeImages = source.features?.includeImages === true;
|
|
1939
|
+
const Wireframe = getWireframeCtor();
|
|
1940
|
+
const wf = new Wireframe(document.body, { images: includeImages });
|
|
1941
|
+
const { canvas: src } = await wf.capture();
|
|
1942
|
+
ctx.fillStyle = "#ffffff";
|
|
1943
|
+
ctx.fillRect(0, 0, canvas.width, canvas.height);
|
|
1944
|
+
ctx.drawImage(src, 0, 0, canvas.width, canvas.height);
|
|
1945
|
+
} else if (source.type === "fn") {
|
|
1946
|
+
const frame = source.captureFn();
|
|
1947
|
+
ctx.fillStyle = "#ffffff";
|
|
1948
|
+
ctx.fillRect(0, 0, canvas.width, canvas.height);
|
|
1949
|
+
ctx.drawImage(frame, 0, 0, canvas.width, canvas.height);
|
|
1950
|
+
}
|
|
1951
|
+
} catch (e) {
|
|
1952
|
+
console.warn("[Sable] frame source tick failed", e);
|
|
1953
|
+
} finally {
|
|
1954
|
+
inFlight = false;
|
|
1955
|
+
}
|
|
1956
|
+
}
|
|
1957
|
+
if (!stopped) {
|
|
1958
|
+
timer = setTimeout(tick, delayMs);
|
|
1959
|
+
}
|
|
1960
|
+
};
|
|
1961
|
+
tick();
|
|
1962
|
+
return () => {
|
|
1963
|
+
stopped = true;
|
|
1964
|
+
if (timer !== undefined)
|
|
1965
|
+
clearTimeout(timer);
|
|
1966
|
+
};
|
|
1967
|
+
}
|
|
1968
|
+
|
|
1969
|
+
// src/vision/publisher.ts
|
|
1970
|
+
var BROWSER_TRACK_NAME = "browser";
|
|
1971
|
+
async function publishCanvasAsVideoTrack(room, lib, canvas, fps) {
|
|
1972
|
+
const mediaStream = canvas.captureStream(fps);
|
|
1973
|
+
const videoTracks = mediaStream.getVideoTracks();
|
|
1974
|
+
if (videoTracks.length === 0) {
|
|
1975
|
+
throw new Error("canvas.captureStream produced no video tracks");
|
|
1976
|
+
}
|
|
1977
|
+
const mediaStreamTrack = videoTracks[0];
|
|
1978
|
+
const localTrack = new lib.LocalVideoTrack(mediaStreamTrack, undefined, true);
|
|
1979
|
+
const publication = await room.localParticipant.publishTrack(localTrack, {
|
|
1980
|
+
source: lib.Track.Source.ScreenShare,
|
|
1981
|
+
name: BROWSER_TRACK_NAME
|
|
1982
|
+
});
|
|
1983
|
+
console.log("[Sable] vision track published", {
|
|
1984
|
+
trackSid: publication.trackSid,
|
|
1985
|
+
fps
|
|
1986
|
+
});
|
|
1987
|
+
return async () => {
|
|
1988
|
+
try {
|
|
1989
|
+
await room.localParticipant.unpublishTrack(localTrack, true);
|
|
1990
|
+
} catch (e) {
|
|
1991
|
+
console.warn("[Sable] vision unpublishTrack failed", e);
|
|
1992
|
+
}
|
|
1993
|
+
try {
|
|
1994
|
+
mediaStreamTrack.stop();
|
|
1995
|
+
} catch {}
|
|
1996
|
+
console.log("[Sable] vision track stopped");
|
|
1997
|
+
};
|
|
1998
|
+
}
|
|
1999
|
+
|
|
2000
|
+
// src/vision/index.ts
|
|
2001
|
+
var DEFAULT_FRAME_SOURCE = {
|
|
2002
|
+
type: "wireframe",
|
|
2003
|
+
rate: 2,
|
|
2004
|
+
features: { includeImages: false }
|
|
2005
|
+
};
|
|
2006
|
+
async function startVision(args) {
|
|
2007
|
+
const source = args.options.frameSource ?? DEFAULT_FRAME_SOURCE;
|
|
2008
|
+
const fps = typeof source.rate === "number" && source.rate > 0 ? source.rate : 2;
|
|
2009
|
+
const canvas = document.createElement("canvas");
|
|
2010
|
+
canvas.width = Math.max(1, window.innerWidth);
|
|
2011
|
+
canvas.height = Math.max(1, window.innerHeight);
|
|
2012
|
+
const stopFrameSource = startFrameSource(source, canvas);
|
|
2013
|
+
let stopPublish;
|
|
2014
|
+
try {
|
|
2015
|
+
stopPublish = await publishCanvasAsVideoTrack(args.room, args.lib, canvas, fps);
|
|
2016
|
+
} catch (err) {
|
|
2017
|
+
stopFrameSource();
|
|
2018
|
+
throw err;
|
|
2019
|
+
}
|
|
2020
|
+
return {
|
|
2021
|
+
canvas,
|
|
2022
|
+
stop: async () => {
|
|
2023
|
+
stopFrameSource();
|
|
2024
|
+
await stopPublish();
|
|
2025
|
+
}
|
|
2026
|
+
};
|
|
2027
|
+
}
|
|
2028
|
+
|
|
2029
|
+
// src/version.ts
|
|
2030
|
+
var VERSION = "0.1.0";
|
|
2031
|
+
|
|
2032
|
+
// src/session/debug-panel.ts
|
|
2033
|
+
var DEBUG_PANEL_STATE_KEY = "sable:debug:panel";
|
|
2034
|
+
function loadState() {
|
|
2035
|
+
try {
|
|
2036
|
+
const raw = window.localStorage?.getItem(DEBUG_PANEL_STATE_KEY);
|
|
2037
|
+
if (!raw)
|
|
2038
|
+
return {};
|
|
2039
|
+
const parsed = JSON.parse(raw);
|
|
2040
|
+
return typeof parsed === "object" && parsed !== null ? parsed : {};
|
|
2041
|
+
} catch {
|
|
2042
|
+
return {};
|
|
2043
|
+
}
|
|
2044
|
+
}
|
|
2045
|
+
function saveState(state) {
|
|
2046
|
+
try {
|
|
2047
|
+
window.localStorage?.setItem(DEBUG_PANEL_STATE_KEY, JSON.stringify(state));
|
|
2048
|
+
} catch {}
|
|
2049
|
+
}
|
|
2050
|
+
function shouldShowDebugPanel(debugOpt) {
|
|
2051
|
+
if (debugOpt)
|
|
2052
|
+
return true;
|
|
2053
|
+
try {
|
|
2054
|
+
if (new URL(window.location.href).searchParams.get("sable-debug") === "1") {
|
|
2055
|
+
return true;
|
|
2056
|
+
}
|
|
2057
|
+
} catch {}
|
|
2058
|
+
try {
|
|
2059
|
+
if (window.localStorage?.getItem("sable:debug") === "1")
|
|
2060
|
+
return true;
|
|
2061
|
+
} catch {}
|
|
2062
|
+
return false;
|
|
2063
|
+
}
|
|
2064
|
+
function mountDebugPanel(canvas) {
|
|
2065
|
+
const state = loadState();
|
|
2066
|
+
const wrap = document.createElement("div");
|
|
2067
|
+
wrap.setAttribute("data-sable-debug", "vision");
|
|
2068
|
+
Object.assign(wrap.style, {
|
|
2069
|
+
position: "fixed",
|
|
2070
|
+
width: "240px",
|
|
2071
|
+
zIndex: "2147483647",
|
|
2072
|
+
background: "#111",
|
|
2073
|
+
color: "#ddd",
|
|
2074
|
+
border: "1px solid #444",
|
|
2075
|
+
borderRadius: "8px",
|
|
2076
|
+
font: "11px/1.3 system-ui, sans-serif",
|
|
2077
|
+
boxShadow: "0 8px 24px rgba(0,0,0,.4)",
|
|
2078
|
+
pointerEvents: "none",
|
|
2079
|
+
userSelect: "none",
|
|
2080
|
+
overflow: "hidden"
|
|
2081
|
+
});
|
|
2082
|
+
if (typeof state.left === "number" && typeof state.top === "number") {
|
|
2083
|
+
wrap.style.left = `${state.left}px`;
|
|
2084
|
+
wrap.style.top = `${state.top}px`;
|
|
2085
|
+
} else {
|
|
2086
|
+
wrap.style.right = "12px";
|
|
2087
|
+
wrap.style.top = "12px";
|
|
2088
|
+
}
|
|
2089
|
+
const header = document.createElement("div");
|
|
2090
|
+
Object.assign(header.style, {
|
|
2091
|
+
display: "flex",
|
|
2092
|
+
alignItems: "center",
|
|
2093
|
+
justifyContent: "space-between",
|
|
2094
|
+
gap: "6px",
|
|
2095
|
+
padding: "6px 8px",
|
|
2096
|
+
background: "#1a1a1a",
|
|
2097
|
+
borderBottom: "1px solid #333",
|
|
2098
|
+
cursor: "move",
|
|
2099
|
+
pointerEvents: "auto"
|
|
2100
|
+
});
|
|
2101
|
+
const title = document.createElement("div");
|
|
2102
|
+
title.textContent = "sable: agent vision";
|
|
2103
|
+
Object.assign(title.style, {
|
|
2104
|
+
opacity: "0.75",
|
|
2105
|
+
fontWeight: "600",
|
|
2106
|
+
flex: "1",
|
|
2107
|
+
pointerEvents: "none"
|
|
2108
|
+
});
|
|
2109
|
+
header.appendChild(title);
|
|
2110
|
+
const minBtn = document.createElement("button");
|
|
2111
|
+
minBtn.setAttribute("aria-label", "Minimize vision panel");
|
|
2112
|
+
Object.assign(minBtn.style, {
|
|
2113
|
+
background: "transparent",
|
|
2114
|
+
color: "#ddd",
|
|
2115
|
+
border: "1px solid #444",
|
|
2116
|
+
borderRadius: "4px",
|
|
2117
|
+
width: "20px",
|
|
2118
|
+
height: "20px",
|
|
2119
|
+
cursor: "pointer",
|
|
2120
|
+
fontSize: "12px",
|
|
2121
|
+
lineHeight: "1",
|
|
2122
|
+
padding: "0",
|
|
2123
|
+
pointerEvents: "auto"
|
|
2124
|
+
});
|
|
2125
|
+
header.appendChild(minBtn);
|
|
2126
|
+
wrap.appendChild(header);
|
|
2127
|
+
const body = document.createElement("div");
|
|
2128
|
+
Object.assign(body.style, {
|
|
2129
|
+
padding: "6px",
|
|
2130
|
+
background: "#111",
|
|
2131
|
+
pointerEvents: "none"
|
|
2132
|
+
});
|
|
2133
|
+
canvas.style.width = "100%";
|
|
2134
|
+
canvas.style.height = "auto";
|
|
2135
|
+
canvas.style.display = "block";
|
|
2136
|
+
canvas.style.background = "#fff";
|
|
2137
|
+
canvas.style.borderRadius = "4px";
|
|
2138
|
+
canvas.style.pointerEvents = "none";
|
|
2139
|
+
body.appendChild(canvas);
|
|
2140
|
+
wrap.appendChild(body);
|
|
2141
|
+
let minimized = !!state.minimized;
|
|
2142
|
+
const applyMinimized = () => {
|
|
2143
|
+
body.style.display = minimized ? "none" : "block";
|
|
2144
|
+
minBtn.textContent = minimized ? "▢" : "–";
|
|
2145
|
+
minBtn.setAttribute("aria-label", minimized ? "Restore vision panel" : "Minimize vision panel");
|
|
2146
|
+
};
|
|
2147
|
+
applyMinimized();
|
|
2148
|
+
minBtn.addEventListener("click", (ev) => {
|
|
2149
|
+
ev.stopPropagation();
|
|
2150
|
+
minimized = !minimized;
|
|
2151
|
+
applyMinimized();
|
|
2152
|
+
saveState({ ...loadState(), minimized });
|
|
2153
|
+
});
|
|
2154
|
+
let dragState = null;
|
|
2155
|
+
const onPointerDown = (ev) => {
|
|
2156
|
+
if (ev.target instanceof HTMLElement && ev.target.closest("button")) {
|
|
2157
|
+
return;
|
|
2158
|
+
}
|
|
2159
|
+
const rect = wrap.getBoundingClientRect();
|
|
2160
|
+
dragState = {
|
|
2161
|
+
offsetX: ev.clientX - rect.left,
|
|
2162
|
+
offsetY: ev.clientY - rect.top
|
|
2163
|
+
};
|
|
2164
|
+
wrap.style.left = `${rect.left}px`;
|
|
2165
|
+
wrap.style.top = `${rect.top}px`;
|
|
2166
|
+
wrap.style.right = "auto";
|
|
2167
|
+
wrap.style.bottom = "auto";
|
|
2168
|
+
header.setPointerCapture(ev.pointerId);
|
|
2169
|
+
ev.preventDefault();
|
|
2170
|
+
};
|
|
2171
|
+
const onPointerMove = (ev) => {
|
|
2172
|
+
if (!dragState)
|
|
2173
|
+
return;
|
|
2174
|
+
const vw = window.innerWidth;
|
|
2175
|
+
const vh = window.innerHeight;
|
|
2176
|
+
const w = wrap.offsetWidth;
|
|
2177
|
+
let left = ev.clientX - dragState.offsetX;
|
|
2178
|
+
let top = ev.clientY - dragState.offsetY;
|
|
2179
|
+
left = Math.min(Math.max(left, -w + 48), vw - 48);
|
|
2180
|
+
top = Math.min(Math.max(top, 0), vh - 24);
|
|
2181
|
+
wrap.style.left = `${left}px`;
|
|
2182
|
+
wrap.style.top = `${top}px`;
|
|
2183
|
+
};
|
|
2184
|
+
const onPointerUp = (ev) => {
|
|
2185
|
+
if (!dragState)
|
|
2186
|
+
return;
|
|
2187
|
+
dragState = null;
|
|
2188
|
+
try {
|
|
2189
|
+
header.releasePointerCapture(ev.pointerId);
|
|
2190
|
+
} catch {}
|
|
2191
|
+
const left = parseFloat(wrap.style.left) || 0;
|
|
2192
|
+
const top = parseFloat(wrap.style.top) || 0;
|
|
2193
|
+
saveState({ ...loadState(), left, top });
|
|
2194
|
+
};
|
|
2195
|
+
header.addEventListener("pointerdown", onPointerDown);
|
|
2196
|
+
header.addEventListener("pointermove", onPointerMove);
|
|
2197
|
+
header.addEventListener("pointerup", onPointerUp);
|
|
2198
|
+
header.addEventListener("pointercancel", onPointerUp);
|
|
2199
|
+
document.body.appendChild(wrap);
|
|
2200
|
+
console.log("[Sable] debug vision panel mounted", {
|
|
2201
|
+
minimized,
|
|
2202
|
+
restoredPosition: typeof state.left === "number" && typeof state.top === "number"
|
|
2203
|
+
});
|
|
2204
|
+
return () => {
|
|
2205
|
+
try {
|
|
2206
|
+
header.removeEventListener("pointerdown", onPointerDown);
|
|
2207
|
+
header.removeEventListener("pointermove", onPointerMove);
|
|
2208
|
+
header.removeEventListener("pointerup", onPointerUp);
|
|
2209
|
+
header.removeEventListener("pointercancel", onPointerUp);
|
|
2210
|
+
wrap.remove();
|
|
2211
|
+
} catch {}
|
|
2212
|
+
};
|
|
2213
|
+
}
|
|
2214
|
+
|
|
2215
|
+
// src/session/index.ts
|
|
2216
|
+
var UI_READY_RETRY_ATTEMPTS = 5;
|
|
2217
|
+
var UI_READY_RETRY_DELAY_MS = 500;
|
|
2218
|
+
function findAgentIdentity(room) {
|
|
2219
|
+
const remotes = room.remoteParticipants ? Array.from(room.remoteParticipants.values()) : [];
|
|
2220
|
+
const agent = remotes.find((p) => typeof p.identity === "string" && p.identity.startsWith("agent"));
|
|
2221
|
+
return agent?.identity ?? remotes[0]?.identity ?? null;
|
|
2222
|
+
}
|
|
2223
|
+
async function sendUiReady(room) {
|
|
2224
|
+
for (let attempt = 1;attempt <= UI_READY_RETRY_ATTEMPTS; attempt++) {
|
|
2225
|
+
const identity = findAgentIdentity(room);
|
|
2226
|
+
if (!identity) {
|
|
2227
|
+
console.warn("[Sable] sendUiReady: no agent participant yet", { attempt });
|
|
2228
|
+
await new Promise((r) => setTimeout(r, UI_READY_RETRY_DELAY_MS));
|
|
2229
|
+
continue;
|
|
2230
|
+
}
|
|
2231
|
+
try {
|
|
2232
|
+
await room.localParticipant.performRpc({
|
|
2233
|
+
destinationIdentity: identity,
|
|
2234
|
+
method: "uiReady",
|
|
2235
|
+
payload: JSON.stringify({ timestamp: Date.now() })
|
|
2236
|
+
});
|
|
2237
|
+
console.log("[Sable] uiReady sent", { identity, attempt });
|
|
2238
|
+
return;
|
|
2239
|
+
} catch (err) {
|
|
2240
|
+
console.warn("[Sable] uiReady RPC failed", { attempt, err });
|
|
2241
|
+
await new Promise((r) => setTimeout(r, UI_READY_RETRY_DELAY_MS));
|
|
2242
|
+
}
|
|
2243
|
+
}
|
|
2244
|
+
console.error("[Sable] uiReady: exhausted retries — agent will not greet");
|
|
2245
|
+
}
|
|
2246
|
+
|
|
2247
|
+
class Session {
|
|
2248
|
+
version = VERSION;
|
|
2249
|
+
emitter = new SableEventEmitter;
|
|
2250
|
+
activeRoom = null;
|
|
2251
|
+
visionHandle = null;
|
|
2252
|
+
unmountDebugPanel = null;
|
|
2253
|
+
on(event, handler) {
|
|
2254
|
+
return this.emitter.on(event, handler);
|
|
2255
|
+
}
|
|
2256
|
+
async start(opts) {
|
|
2257
|
+
if (this.activeRoom) {
|
|
2258
|
+
throw new Error("Sable already started; call stop() first");
|
|
2259
|
+
}
|
|
2260
|
+
const publicKey = opts.publicKey ?? opts.agentPublicId;
|
|
2261
|
+
if (!publicKey) {
|
|
2262
|
+
throw new Error("Sable.start: `publicKey` is required");
|
|
2263
|
+
}
|
|
2264
|
+
const apiUrl = opts.apiUrl ?? DEFAULT_API_URL;
|
|
2265
|
+
console.log("[Sable] fetching connection details", { apiUrl });
|
|
2266
|
+
const details = await fetchConnectionDetails({ apiUrl, publicKey });
|
|
2267
|
+
console.log("[Sable] connection details received", {
|
|
2268
|
+
roomName: details.roomName,
|
|
2269
|
+
participantName: details.participantName
|
|
2270
|
+
});
|
|
2271
|
+
const livekit = await import("livekit-client");
|
|
2272
|
+
const { Room, RoomEvent, LocalVideoTrack, Track } = livekit;
|
|
2273
|
+
const room = new Room;
|
|
2274
|
+
const publishLib = {
|
|
2275
|
+
LocalVideoTrack,
|
|
2276
|
+
Track
|
|
2277
|
+
};
|
|
2278
|
+
room.registerRpcMethod("agentReady", async () => {
|
|
2279
|
+
console.log("[Sable] RPC agentReady received");
|
|
2280
|
+
sendUiReady(room);
|
|
2281
|
+
return JSON.stringify({ success: true });
|
|
2282
|
+
});
|
|
2283
|
+
registerBrowserHandlers(room);
|
|
2284
|
+
installRuntime(room, opts.runtime);
|
|
2285
|
+
this.wireRoomEvents(room, RoomEvent);
|
|
2286
|
+
await room.connect(details.serverUrl, details.participantToken);
|
|
2287
|
+
await room.localParticipant.setMicrophoneEnabled(true);
|
|
2288
|
+
this.activeRoom = room;
|
|
2289
|
+
if (opts.vision?.enabled) {
|
|
2290
|
+
try {
|
|
2291
|
+
this.visionHandle = await startVision({
|
|
2292
|
+
room,
|
|
2293
|
+
lib: publishLib,
|
|
2294
|
+
options: opts.vision
|
|
2295
|
+
});
|
|
2296
|
+
if (shouldShowDebugPanel(opts.debug)) {
|
|
2297
|
+
this.unmountDebugPanel = mountDebugPanel(this.visionHandle.canvas);
|
|
2298
|
+
}
|
|
2299
|
+
} catch (e) {
|
|
2300
|
+
console.warn("[Sable] failed to start vision", e);
|
|
2301
|
+
}
|
|
2302
|
+
}
|
|
2303
|
+
console.log("[Sable] session live", {
|
|
2304
|
+
roomName: details.roomName,
|
|
2305
|
+
participantName: details.participantName
|
|
2306
|
+
});
|
|
2307
|
+
this.emitter.emit("session:started", {
|
|
2308
|
+
roomName: details.roomName,
|
|
2309
|
+
participantName: details.participantName
|
|
2310
|
+
});
|
|
2311
|
+
setTimeout(() => {
|
|
2312
|
+
if (this.activeRoom !== room)
|
|
2313
|
+
return;
|
|
2314
|
+
const r = room;
|
|
2315
|
+
const remotes = r.remoteParticipants ? Array.from(r.remoteParticipants.values()) : [];
|
|
2316
|
+
const summary = remotes.map((p) => ({
|
|
2317
|
+
identity: p.identity,
|
|
2318
|
+
tracks: p.trackPublications ? Array.from(p.trackPublications.values()).map((t) => ({
|
|
2319
|
+
kind: t.kind,
|
|
2320
|
+
subscribed: t.isSubscribed
|
|
2321
|
+
})) : []
|
|
2322
|
+
}));
|
|
2323
|
+
const anyAudio = summary.some((p) => p.tracks.some((t) => t.kind === "audio"));
|
|
2324
|
+
if (!anyAudio) {
|
|
2325
|
+
console.warn("[Sable] no remote audio track after 10s — agent worker probably failed to publish. Remote participants:", summary);
|
|
2326
|
+
}
|
|
2327
|
+
}, 1e4);
|
|
2328
|
+
}
|
|
2329
|
+
async stop() {
|
|
2330
|
+
const room = this.activeRoom;
|
|
2331
|
+
if (!room)
|
|
2332
|
+
return;
|
|
2333
|
+
this.activeRoom = null;
|
|
2334
|
+
if (this.unmountDebugPanel) {
|
|
2335
|
+
try {
|
|
2336
|
+
this.unmountDebugPanel();
|
|
2337
|
+
} catch (e) {
|
|
2338
|
+
console.warn("[Sable] debug panel unmount failed", e);
|
|
2339
|
+
}
|
|
2340
|
+
this.unmountDebugPanel = null;
|
|
2341
|
+
}
|
|
2342
|
+
if (this.visionHandle) {
|
|
2343
|
+
try {
|
|
2344
|
+
await this.visionHandle.stop();
|
|
2345
|
+
} catch (e) {
|
|
2346
|
+
console.warn("[Sable] vision stop failed", e);
|
|
2347
|
+
}
|
|
2348
|
+
this.visionHandle = null;
|
|
2349
|
+
}
|
|
2350
|
+
try {
|
|
2351
|
+
await room.localParticipant.setMicrophoneEnabled(false);
|
|
2352
|
+
} catch (err) {
|
|
2353
|
+
console.warn("[Sable] setMicrophoneEnabled(false) failed", err);
|
|
2354
|
+
}
|
|
2355
|
+
await room.disconnect();
|
|
2356
|
+
console.log("[Sable] session ended");
|
|
2357
|
+
this.emitter.emit("session:ended", {});
|
|
2358
|
+
}
|
|
2359
|
+
wireRoomEvents(room, RoomEvent) {
|
|
2360
|
+
room.on(RoomEvent.ConnectionStateChanged, (state) => {
|
|
2361
|
+
console.log("[Sable] ConnectionStateChanged", state);
|
|
2362
|
+
});
|
|
2363
|
+
room.on(RoomEvent.Disconnected, (reason) => {
|
|
2364
|
+
console.log("[Sable] Disconnected", reason);
|
|
2365
|
+
if (this.activeRoom === room) {
|
|
2366
|
+
this.stop().catch((e) => console.warn("[Sable] stop on disconnect failed", e));
|
|
2367
|
+
}
|
|
2368
|
+
});
|
|
2369
|
+
room.on(RoomEvent.ParticipantConnected, (participant) => {
|
|
2370
|
+
const p = participant;
|
|
2371
|
+
console.log("[Sable] ParticipantConnected", {
|
|
2372
|
+
identity: p.identity,
|
|
2373
|
+
sid: p.sid
|
|
2374
|
+
});
|
|
2375
|
+
});
|
|
2376
|
+
room.on(RoomEvent.ParticipantDisconnected, (participant) => {
|
|
2377
|
+
const p = participant;
|
|
2378
|
+
console.warn("[Sable] ParticipantDisconnected", { identity: p.identity });
|
|
2379
|
+
});
|
|
2380
|
+
room.on(RoomEvent.TrackSubscribed, (track, _pub, participant) => {
|
|
2381
|
+
const t = track;
|
|
2382
|
+
const p = participant;
|
|
2383
|
+
console.log("[Sable] TrackSubscribed", {
|
|
2384
|
+
kind: t.kind,
|
|
2385
|
+
participant: p.identity
|
|
2386
|
+
});
|
|
2387
|
+
if (t.kind === "audio" && typeof t.attach === "function") {
|
|
2388
|
+
const el = t.attach();
|
|
2389
|
+
el.setAttribute("data-sable", "1");
|
|
2390
|
+
el.setAttribute("playsinline", "");
|
|
2391
|
+
el.autoplay = true;
|
|
2392
|
+
document.body.appendChild(el);
|
|
2393
|
+
console.log("[Sable] attached remote audio element");
|
|
2394
|
+
}
|
|
2395
|
+
});
|
|
2396
|
+
room.on(RoomEvent.TrackUnsubscribed, (track) => {
|
|
2397
|
+
const t = track;
|
|
2398
|
+
if (typeof t.detach === "function") {
|
|
2399
|
+
for (const el of t.detach()) {
|
|
2400
|
+
el.remove();
|
|
2401
|
+
}
|
|
2402
|
+
}
|
|
2403
|
+
});
|
|
2404
|
+
room.on(RoomEvent.ActiveSpeakersChanged, (speakers) => {
|
|
2405
|
+
const list = speakers ?? [];
|
|
2406
|
+
const agentTalking = list.some((s) => typeof s.identity === "string" && s.identity.startsWith("agent"));
|
|
2407
|
+
const userTalking = list.some((s) => typeof s.identity === "string" && !s.identity.startsWith("agent"));
|
|
2408
|
+
this.emitter.emit("agent:speaking", agentTalking);
|
|
2409
|
+
this.emitter.emit("user:speaking", userTalking);
|
|
2410
|
+
});
|
|
2411
|
+
}
|
|
2412
|
+
}
|
|
2413
|
+
|
|
2414
|
+
// src/global.ts
|
|
2415
|
+
var Sable = new Session;
|
|
2416
|
+
function installGlobal() {
|
|
2417
|
+
if (typeof window === "undefined")
|
|
2418
|
+
return;
|
|
2419
|
+
if (window.Sable)
|
|
2420
|
+
return;
|
|
2421
|
+
window.Sable = Sable;
|
|
2422
|
+
console.log("[Sable] SDK loaded", Sable.version);
|
|
2423
|
+
}
|
|
2424
|
+
|
|
2425
|
+
// src/index.ts
|
|
2426
|
+
installGlobal();
|
|
2427
|
+
var src_default = Sable;
|
|
2428
|
+
export {
|
|
2429
|
+
src_default as default,
|
|
2430
|
+
VERSION
|
|
2431
|
+
};
|