@velmlabs/unstuck-core 0.1.2 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +251 -5
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -55,6 +55,7 @@ async function verifyToken(key, token, nowMs) {
|
|
|
55
55
|
|
|
56
56
|
// src/session/takeover.ts
|
|
57
57
|
import {
|
|
58
|
+
RESUME_WINDOW_MS,
|
|
58
59
|
b64urlEncode,
|
|
59
60
|
importE2eKey,
|
|
60
61
|
newE2eKey,
|
|
@@ -159,8 +160,18 @@ var VIRTUAL_KEYS = {
|
|
|
159
160
|
ArrowUp: 38,
|
|
160
161
|
ArrowRight: 39,
|
|
161
162
|
ArrowDown: 40,
|
|
162
|
-
Delete: 46
|
|
163
|
+
Delete: 46,
|
|
164
|
+
PageUp: 33,
|
|
165
|
+
PageDown: 34,
|
|
166
|
+
End: 35,
|
|
167
|
+
Home: 36,
|
|
168
|
+
Insert: 45,
|
|
169
|
+
...Object.fromEntries(Array.from({ length: 12 }, (_, i) => [`F${i + 1}`, 112 + i]))
|
|
163
170
|
};
|
|
171
|
+
function virtualKey(key) {
|
|
172
|
+
if (VIRTUAL_KEYS[key] !== void 0) return VIRTUAL_KEYS[key];
|
|
173
|
+
return /^[a-zA-Z0-9]$/.test(key) ? key.toUpperCase().charCodeAt(0) : void 0;
|
|
174
|
+
}
|
|
164
175
|
var MOUSE_TYPES = { down: "mousePressed", up: "mouseReleased", move: "mouseMoved" };
|
|
165
176
|
async function applyInput(cdp, msg, caps) {
|
|
166
177
|
switch (msg.t) {
|
|
@@ -179,7 +190,7 @@ async function applyInput(cdp, msg, caps) {
|
|
|
179
190
|
return;
|
|
180
191
|
case "key": {
|
|
181
192
|
const base = { key: msg.key, code: msg.code, modifiers: msg.modifiers };
|
|
182
|
-
const vk =
|
|
193
|
+
const vk = virtualKey(msg.key);
|
|
183
194
|
const withVk = vk === void 0 ? {} : { windowsVirtualKeyCode: vk };
|
|
184
195
|
if (msg.action === "up") await cdp.send("Input.dispatchKeyEvent", { type: "keyUp", ...base });
|
|
185
196
|
else if (msg.text !== void 0)
|
|
@@ -200,6 +211,227 @@ async function applyInput(cdp, msg, caps) {
|
|
|
200
211
|
}
|
|
201
212
|
}
|
|
202
213
|
|
|
214
|
+
// src/stream/fields.ts
|
|
215
|
+
import { FieldKinds } from "@velmlabs/unstuck-protocol";
|
|
216
|
+
var KIND_OF = `function kindOf(el) {
|
|
217
|
+
if (!el || el.nodeType !== 1 || el.disabled || el.readOnly) return null;
|
|
218
|
+
const tag = el.tagName;
|
|
219
|
+
if (tag === 'TEXTAREA') return 'text';
|
|
220
|
+
if (tag !== 'INPUT') return el.isContentEditable ? 'text' : null;
|
|
221
|
+
const type = (el.getAttribute('type') || 'text').toLowerCase();
|
|
222
|
+
if (!['text', 'tel', 'number', 'email', 'password', 'search', 'url'].includes(type)) return null;
|
|
223
|
+
const im = (el.getAttribute('inputmode') || '').toLowerCase();
|
|
224
|
+
const ac = (el.getAttribute('autocomplete') || '').toLowerCase().split(/\\s+/);
|
|
225
|
+
if (ac.includes('one-time-code')) return 'otp';
|
|
226
|
+
if (type === 'password') return 'password';
|
|
227
|
+
if (type === 'tel' || type === 'number' || ['numeric', 'decimal', 'tel'].includes(im)) return 'numeric';
|
|
228
|
+
if (type === 'email' || im === 'email') return 'email';
|
|
229
|
+
return 'text';
|
|
230
|
+
}`;
|
|
231
|
+
var SCAN_SOURCE = `(() => {
|
|
232
|
+
${KIND_OF}
|
|
233
|
+
const vw = innerWidth, vh = innerHeight, found = [];
|
|
234
|
+
const inside = (a, b) => b.left >= a.left - 1 && b.top >= a.top - 1 && b.right <= a.right + 1 && b.bottom <= a.bottom + 1;
|
|
235
|
+
// Something else on top of the field's visible centre means a tap there doesn't reach it.
|
|
236
|
+
const uncovered = (el, r, root, ox, oy) => {
|
|
237
|
+
const cx = Math.min(Math.max((Math.max(r.left, -ox) + Math.min(r.right, vw - ox)) / 2, 0), vw);
|
|
238
|
+
const cy = Math.min(Math.max((Math.max(r.top, -oy) + Math.min(r.bottom, vh - oy)) / 2, 0), vh);
|
|
239
|
+
const hit = root.elementFromPoint ? root.elementFromPoint(cx, cy) : null;
|
|
240
|
+
if (!hit) return false;
|
|
241
|
+
return hit === el || el.contains(hit) || inside(r, hit.getBoundingClientRect());
|
|
242
|
+
};
|
|
243
|
+
const walk = (root, ox, oy, frameOk) => {
|
|
244
|
+
for (const el of root.querySelectorAll('*')) {
|
|
245
|
+
if (el.shadowRoot) walk(el.shadowRoot, ox, oy, frameOk);
|
|
246
|
+
if (el.tagName === 'IFRAME') {
|
|
247
|
+
let doc = null;
|
|
248
|
+
try { doc = el.contentDocument; } catch (e) { doc = null; }
|
|
249
|
+
if (doc) {
|
|
250
|
+
const fr = el.getBoundingClientRect();
|
|
251
|
+
const nx = ox + fr.left + el.clientLeft, ny = oy + fr.top + el.clientTop;
|
|
252
|
+
walk(doc, nx, ny, (x, y) => frameOk(x, y) && document.elementFromPoint(x, y) === el);
|
|
253
|
+
}
|
|
254
|
+
continue;
|
|
255
|
+
}
|
|
256
|
+
const kind = kindOf(el);
|
|
257
|
+
if (!kind) continue;
|
|
258
|
+
const style = (el.ownerDocument.defaultView || window).getComputedStyle(el);
|
|
259
|
+
if (style.display === 'none' || style.visibility === 'hidden') continue;
|
|
260
|
+
const r = el.getBoundingClientRect();
|
|
261
|
+
if (r.width < 1 || r.height < 1) continue;
|
|
262
|
+
const x = r.left + ox, y = r.top + oy;
|
|
263
|
+
if (x >= vw || y >= vh || x + r.width <= 0 || y + r.height <= 0) continue;
|
|
264
|
+
if (!uncovered(el, r, el.getRootNode(), ox, oy)) continue;
|
|
265
|
+
if (!frameOk(x + r.width / 2, y + r.height / 2)) continue;
|
|
266
|
+
found.push({ x: Math.round(x), y: Math.round(y), w: Math.round(r.width), h: Math.round(r.height), kind });
|
|
267
|
+
}
|
|
268
|
+
};
|
|
269
|
+
walk(document, 0, 0, () => true);
|
|
270
|
+
const d = (f) => Math.hypot(f.x + f.w / 2 - vw / 2, f.y + f.h / 2 - vh / 2);
|
|
271
|
+
return found.sort((a, b) => d(a) - d(b)).slice(0, 30);
|
|
272
|
+
})()`;
|
|
273
|
+
var FOCUS_SOURCE = `(() => {
|
|
274
|
+
${KIND_OF}
|
|
275
|
+
const box = (r, kind, ox, oy) => ({ x: Math.round(r.left + ox), y: Math.round(r.top + oy), w: Math.round(r.width), h: Math.round(r.height), kind });
|
|
276
|
+
let el = document.activeElement, ox = 0, oy = 0;
|
|
277
|
+
for (let i = 0; el && i < 20; i++) {
|
|
278
|
+
if (el.shadowRoot && el.shadowRoot.activeElement) { el = el.shadowRoot.activeElement; continue; }
|
|
279
|
+
if (el.tagName === 'IFRAME') {
|
|
280
|
+
let doc = null;
|
|
281
|
+
try { doc = el.contentDocument; } catch (e) { doc = null; }
|
|
282
|
+
const fr = el.getBoundingClientRect();
|
|
283
|
+
if (!doc) return box(fr, 'text', ox, oy);
|
|
284
|
+
ox += fr.left + el.clientLeft;
|
|
285
|
+
oy += fr.top + el.clientTop;
|
|
286
|
+
el = doc.activeElement;
|
|
287
|
+
continue;
|
|
288
|
+
}
|
|
289
|
+
break;
|
|
290
|
+
}
|
|
291
|
+
const kind = kindOf(el);
|
|
292
|
+
return kind ? box(el.getBoundingClientRect(), kind, ox, oy) : null;
|
|
293
|
+
})()`;
|
|
294
|
+
var isKind = (k) => FieldKinds.includes(k);
|
|
295
|
+
var isField = (f) => {
|
|
296
|
+
const o = f;
|
|
297
|
+
return !!o && typeof o === "object" && ["x", "y", "w", "h"].every((k) => Number.isFinite(o[k])) && isKind(o.kind);
|
|
298
|
+
};
|
|
299
|
+
async function evaluate(cdp, expression) {
|
|
300
|
+
try {
|
|
301
|
+
const r = await cdp.send("Runtime.evaluate", { expression, returnByValue: true });
|
|
302
|
+
return r.exceptionDetails ? void 0 : r.result?.value;
|
|
303
|
+
} catch {
|
|
304
|
+
return void 0;
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
async function scanFields(cdp) {
|
|
308
|
+
const v = await evaluate(cdp, SCAN_SOURCE);
|
|
309
|
+
return Array.isArray(v) ? v.filter(isField).map(({ x, y, w, h, kind }) => ({ x, y, w, h, kind })) : [];
|
|
310
|
+
}
|
|
311
|
+
async function focusedField(cdp) {
|
|
312
|
+
const v = await evaluate(cdp, FOCUS_SOURCE);
|
|
313
|
+
return isField(v) ? { x: v.x, y: v.y, w: v.w, h: v.h, kind: v.kind } : null;
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
// src/stream/shortcuts.ts
|
|
317
|
+
var ALT = 1;
|
|
318
|
+
var CTRL = 2;
|
|
319
|
+
var META = 4;
|
|
320
|
+
var SHIFT = 8;
|
|
321
|
+
var EDIT_LETTERS = ["a", "z", "y"];
|
|
322
|
+
var WORD_KEYS = ["Backspace", "Delete", "ArrowLeft", "ArrowRight"];
|
|
323
|
+
var swap = (m, from, to) => m & ~from | to;
|
|
324
|
+
var only = (m, mod) => (m & ~SHIFT) === mod;
|
|
325
|
+
var withKey = (msg, key, modifiers, code = key) => {
|
|
326
|
+
const { text: _text, ...rest } = msg;
|
|
327
|
+
return { ...rest, key, code, modifiers };
|
|
328
|
+
};
|
|
329
|
+
function translateShortcut(msg, browserIsMac) {
|
|
330
|
+
if (msg.text !== void 0) return [msg];
|
|
331
|
+
const m = msg.modifiers;
|
|
332
|
+
const letter = msg.key.toLowerCase();
|
|
333
|
+
if (!browserIsMac) {
|
|
334
|
+
if (only(m, META) && EDIT_LETTERS.includes(letter)) return [{ ...msg, modifiers: swap(m, META, CTRL) }];
|
|
335
|
+
if (only(m, ALT) && WORD_KEYS.includes(msg.key)) return [{ ...msg, modifiers: swap(m, ALT, CTRL) }];
|
|
336
|
+
if (only(m, META) && (msg.key === "ArrowLeft" || msg.key === "ArrowRight")) return [withKey(msg, msg.key === "ArrowLeft" ? "Home" : "End", m & SHIFT)];
|
|
337
|
+
if (only(m, META) && msg.key === "Backspace") {
|
|
338
|
+
if (msg.action === "up") return [withKey(msg, "Backspace", 0)];
|
|
339
|
+
return [withKey(msg, "Home", SHIFT), { ...withKey(msg, "Home", SHIFT), action: "up" }, withKey(msg, "Backspace", 0)];
|
|
340
|
+
}
|
|
341
|
+
return [msg];
|
|
342
|
+
}
|
|
343
|
+
if (only(m, CTRL) && letter === "y") return [withKey(msg, "z", META | SHIFT, "KeyZ")];
|
|
344
|
+
if (only(m, CTRL) && EDIT_LETTERS.includes(letter)) return [{ ...msg, modifiers: swap(m, CTRL, META) }];
|
|
345
|
+
if (only(m, CTRL) && WORD_KEYS.includes(msg.key)) return [{ ...msg, modifiers: swap(m, CTRL, ALT) }];
|
|
346
|
+
if ((m & ~SHIFT) === 0 && (msg.key === "Home" || msg.key === "End")) return [withKey(msg, msg.key === "Home" ? "ArrowLeft" : "ArrowRight", META | m & SHIFT)];
|
|
347
|
+
return [msg];
|
|
348
|
+
}
|
|
349
|
+
var platforms = /* @__PURE__ */ new WeakMap();
|
|
350
|
+
function browserPlatformIsMac(cdp) {
|
|
351
|
+
let p = platforms.get(cdp);
|
|
352
|
+
if (!p) {
|
|
353
|
+
p = cdp.send("Runtime.evaluate", { expression: "navigator.platform", returnByValue: true }).then((r) => /^mac/i.test(String(r?.result?.value ?? ""))).catch(() => {
|
|
354
|
+
platforms.delete(cdp);
|
|
355
|
+
return false;
|
|
356
|
+
});
|
|
357
|
+
platforms.set(cdp, p);
|
|
358
|
+
}
|
|
359
|
+
return p;
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
// src/session/field-feed.ts
|
|
363
|
+
var FRAME_INTERVAL_MS = 500;
|
|
364
|
+
var RESEND_MS = 5e3;
|
|
365
|
+
function createFieldFeed(deps) {
|
|
366
|
+
let stopped = false;
|
|
367
|
+
let lastScanAt = -Infinity;
|
|
368
|
+
let last = [];
|
|
369
|
+
let lastJson = "[]";
|
|
370
|
+
let inFlight = false;
|
|
371
|
+
let dirty = false;
|
|
372
|
+
let pendingForce = false;
|
|
373
|
+
let trailing = null;
|
|
374
|
+
const run = () => {
|
|
375
|
+
if (stopped) return;
|
|
376
|
+
if (inFlight) return void (dirty = true);
|
|
377
|
+
inFlight = true;
|
|
378
|
+
lastScanAt = Date.now();
|
|
379
|
+
const force = pendingForce;
|
|
380
|
+
pendingForce = false;
|
|
381
|
+
deps.scan().then((fields) => {
|
|
382
|
+
if (stopped) return;
|
|
383
|
+
const json = JSON.stringify(fields);
|
|
384
|
+
if (json === lastJson && !(force && fields.length > 0)) return;
|
|
385
|
+
last = fields;
|
|
386
|
+
lastJson = json;
|
|
387
|
+
deps.send({ t: "fields", fields });
|
|
388
|
+
}).catch(() => {
|
|
389
|
+
}).finally(() => {
|
|
390
|
+
inFlight = false;
|
|
391
|
+
if (dirty) {
|
|
392
|
+
dirty = false;
|
|
393
|
+
run();
|
|
394
|
+
}
|
|
395
|
+
});
|
|
396
|
+
};
|
|
397
|
+
const request = (immediate, force = false) => {
|
|
398
|
+
if (stopped) return;
|
|
399
|
+
if (force) pendingForce = true;
|
|
400
|
+
const wait = lastScanAt + FRAME_INTERVAL_MS - Date.now();
|
|
401
|
+
if (immediate || wait <= 0) return run();
|
|
402
|
+
trailing ??= setTimeout(() => {
|
|
403
|
+
trailing = null;
|
|
404
|
+
run();
|
|
405
|
+
}, wait);
|
|
406
|
+
};
|
|
407
|
+
const resend = setInterval(() => {
|
|
408
|
+
if (!stopped && last.length > 0) deps.send({ t: "fields", fields: last });
|
|
409
|
+
}, RESEND_MS);
|
|
410
|
+
return {
|
|
411
|
+
onFrame() {
|
|
412
|
+
request(false);
|
|
413
|
+
},
|
|
414
|
+
onInput(msg) {
|
|
415
|
+
if (stopped) return;
|
|
416
|
+
const tap = msg.t === "mouse" && msg.action === "up";
|
|
417
|
+
const typed = msg.t === "key" && msg.action === "down" || msg.t === "text";
|
|
418
|
+
if (!(tap || typed || msg.t === "wheel")) return;
|
|
419
|
+
request(true);
|
|
420
|
+
if (tap || typed)
|
|
421
|
+
deps.focus().then((field) => void (stopped || deps.send(field ? { t: "focus", kind: field.kind, field } : { t: "focus", kind: null }))).catch(() => {
|
|
422
|
+
});
|
|
423
|
+
},
|
|
424
|
+
onTargetChanged() {
|
|
425
|
+
request(true, true);
|
|
426
|
+
},
|
|
427
|
+
stop() {
|
|
428
|
+
stopped = true;
|
|
429
|
+
clearInterval(resend);
|
|
430
|
+
if (trailing) clearTimeout(trailing);
|
|
431
|
+
}
|
|
432
|
+
};
|
|
433
|
+
}
|
|
434
|
+
|
|
203
435
|
// src/stream/latest.ts
|
|
204
436
|
function createLatestSender(conn, opts = {}) {
|
|
205
437
|
const mark = opts.highWaterMark ?? 512 * 1024;
|
|
@@ -317,7 +549,6 @@ function sealConnection(conn, key, sessionId) {
|
|
|
317
549
|
|
|
318
550
|
// src/session/takeover.ts
|
|
319
551
|
var HELLO_TIMEOUT_MS = 1e4;
|
|
320
|
-
var RESUME_WINDOW_MS = 6e4;
|
|
321
552
|
var MAX_CODE_ATTEMPTS = 5;
|
|
322
553
|
var DRAIN_INTERVAL_MS = 50;
|
|
323
554
|
var STOP_TIMEOUT_MS = 2e3;
|
|
@@ -374,6 +605,11 @@ function runTakeover(deps, cdp, opts) {
|
|
|
374
605
|
sendJson(c, { t: "error", code: code2, message: errorText[code2] });
|
|
375
606
|
if (close) c.close(1008, code2);
|
|
376
607
|
};
|
|
608
|
+
const fieldFeed = createFieldFeed({
|
|
609
|
+
scan: () => scanFields(active().cdp),
|
|
610
|
+
focus: () => focusedField(active().cdp),
|
|
611
|
+
send: (m) => void (holder && sendJson(holder, m))
|
|
612
|
+
});
|
|
377
613
|
const go = (ev) => {
|
|
378
614
|
status = nextStatus(status, ev);
|
|
379
615
|
emit("status", { status });
|
|
@@ -401,6 +637,7 @@ function runTakeover(deps, cdp, opts) {
|
|
|
401
637
|
helloTimers.clear();
|
|
402
638
|
offClose();
|
|
403
639
|
offPopups();
|
|
640
|
+
fieldFeed.stop();
|
|
404
641
|
for (const t of targets) t.off();
|
|
405
642
|
const viewer = holder;
|
|
406
643
|
holder = null;
|
|
@@ -432,6 +669,7 @@ function runTakeover(deps, cdp, opts) {
|
|
|
432
669
|
sendJson(conn, { t: "ready", reason: opts.reason, site: opts.site ?? "", expiresAt, resumeKey });
|
|
433
670
|
sender = createLatestSender(conn);
|
|
434
671
|
if (lastFrame) sender.push(lastFrame);
|
|
672
|
+
fieldFeed.onTargetChanged();
|
|
435
673
|
if (!stopScreencast) await startStream();
|
|
436
674
|
}
|
|
437
675
|
async function startStream() {
|
|
@@ -444,6 +682,7 @@ function runTakeover(deps, cdp, opts) {
|
|
|
444
682
|
if (gen !== streamGen) return;
|
|
445
683
|
lastFrame = encodeFrame(f);
|
|
446
684
|
sender?.push(lastFrame);
|
|
685
|
+
fieldFeed.onFrame();
|
|
447
686
|
});
|
|
448
687
|
} catch (e) {
|
|
449
688
|
if (gen === streamGen) onTargetError(target, e);
|
|
@@ -464,6 +703,7 @@ function runTakeover(deps, cdp, opts) {
|
|
|
464
703
|
lastFrame = null;
|
|
465
704
|
if (!holder || isTerminal(status)) return;
|
|
466
705
|
sender = createLatestSender(holder);
|
|
706
|
+
fieldFeed.onTargetChanged();
|
|
467
707
|
await startStream();
|
|
468
708
|
}
|
|
469
709
|
async function addPopup(popup) {
|
|
@@ -562,9 +802,15 @@ function runTakeover(deps, cdp, opts) {
|
|
|
562
802
|
return finish("cancelled", msg.reason || "cant_solve");
|
|
563
803
|
default:
|
|
564
804
|
inputCount++;
|
|
565
|
-
inputChain = inputChain.then(() => {
|
|
805
|
+
inputChain = inputChain.then(async () => {
|
|
566
806
|
const target = active();
|
|
567
|
-
|
|
807
|
+
try {
|
|
808
|
+
const msgs = msg.t === "key" ? translateShortcut(msg, await browserPlatformIsMac(target.cdp)) : [msg];
|
|
809
|
+
for (const m of msgs) await applyInput(target.cdp, m, target.caps);
|
|
810
|
+
} catch (e) {
|
|
811
|
+
return onTargetError(target, e);
|
|
812
|
+
}
|
|
813
|
+
fieldFeed.onInput(msg);
|
|
568
814
|
});
|
|
569
815
|
}
|
|
570
816
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@velmlabs/unstuck-core",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.4",
|
|
4
4
|
"description": "The Unstuck takeover engine core: screencast, input, sessions and transports, usable in Node or Cloudflare Workers.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"ai",
|
|
@@ -46,7 +46,7 @@
|
|
|
46
46
|
"node": ">=20"
|
|
47
47
|
},
|
|
48
48
|
"dependencies": {
|
|
49
|
-
"@velmlabs/unstuck-protocol": "0.1.
|
|
49
|
+
"@velmlabs/unstuck-protocol": "0.1.4"
|
|
50
50
|
},
|
|
51
51
|
"publishConfig": {
|
|
52
52
|
"access": "public"
|