@velmlabs/unstuck-core 0.1.1 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +250 -4
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -159,8 +159,18 @@ var VIRTUAL_KEYS = {
|
|
|
159
159
|
ArrowUp: 38,
|
|
160
160
|
ArrowRight: 39,
|
|
161
161
|
ArrowDown: 40,
|
|
162
|
-
Delete: 46
|
|
162
|
+
Delete: 46,
|
|
163
|
+
PageUp: 33,
|
|
164
|
+
PageDown: 34,
|
|
165
|
+
End: 35,
|
|
166
|
+
Home: 36,
|
|
167
|
+
Insert: 45,
|
|
168
|
+
...Object.fromEntries(Array.from({ length: 12 }, (_, i) => [`F${i + 1}`, 112 + i]))
|
|
163
169
|
};
|
|
170
|
+
function virtualKey(key) {
|
|
171
|
+
if (VIRTUAL_KEYS[key] !== void 0) return VIRTUAL_KEYS[key];
|
|
172
|
+
return /^[a-zA-Z0-9]$/.test(key) ? key.toUpperCase().charCodeAt(0) : void 0;
|
|
173
|
+
}
|
|
164
174
|
var MOUSE_TYPES = { down: "mousePressed", up: "mouseReleased", move: "mouseMoved" };
|
|
165
175
|
async function applyInput(cdp, msg, caps) {
|
|
166
176
|
switch (msg.t) {
|
|
@@ -179,7 +189,7 @@ async function applyInput(cdp, msg, caps) {
|
|
|
179
189
|
return;
|
|
180
190
|
case "key": {
|
|
181
191
|
const base = { key: msg.key, code: msg.code, modifiers: msg.modifiers };
|
|
182
|
-
const vk =
|
|
192
|
+
const vk = virtualKey(msg.key);
|
|
183
193
|
const withVk = vk === void 0 ? {} : { windowsVirtualKeyCode: vk };
|
|
184
194
|
if (msg.action === "up") await cdp.send("Input.dispatchKeyEvent", { type: "keyUp", ...base });
|
|
185
195
|
else if (msg.text !== void 0)
|
|
@@ -200,6 +210,227 @@ async function applyInput(cdp, msg, caps) {
|
|
|
200
210
|
}
|
|
201
211
|
}
|
|
202
212
|
|
|
213
|
+
// src/stream/fields.ts
|
|
214
|
+
import { FieldKinds } from "@velmlabs/unstuck-protocol";
|
|
215
|
+
var KIND_OF = `function kindOf(el) {
|
|
216
|
+
if (!el || el.nodeType !== 1 || el.disabled || el.readOnly) return null;
|
|
217
|
+
const tag = el.tagName;
|
|
218
|
+
if (tag === 'TEXTAREA') return 'text';
|
|
219
|
+
if (tag !== 'INPUT') return el.isContentEditable ? 'text' : null;
|
|
220
|
+
const type = (el.getAttribute('type') || 'text').toLowerCase();
|
|
221
|
+
if (!['text', 'tel', 'number', 'email', 'password', 'search', 'url'].includes(type)) return null;
|
|
222
|
+
const im = (el.getAttribute('inputmode') || '').toLowerCase();
|
|
223
|
+
const ac = (el.getAttribute('autocomplete') || '').toLowerCase().split(/\\s+/);
|
|
224
|
+
if (ac.includes('one-time-code')) return 'otp';
|
|
225
|
+
if (type === 'password') return 'password';
|
|
226
|
+
if (type === 'tel' || type === 'number' || ['numeric', 'decimal', 'tel'].includes(im)) return 'numeric';
|
|
227
|
+
if (type === 'email' || im === 'email') return 'email';
|
|
228
|
+
return 'text';
|
|
229
|
+
}`;
|
|
230
|
+
var SCAN_SOURCE = `(() => {
|
|
231
|
+
${KIND_OF}
|
|
232
|
+
const vw = innerWidth, vh = innerHeight, found = [];
|
|
233
|
+
const inside = (a, b) => b.left >= a.left - 1 && b.top >= a.top - 1 && b.right <= a.right + 1 && b.bottom <= a.bottom + 1;
|
|
234
|
+
// Something else on top of the field's visible centre means a tap there doesn't reach it.
|
|
235
|
+
const uncovered = (el, r, root, ox, oy) => {
|
|
236
|
+
const cx = Math.min(Math.max((Math.max(r.left, -ox) + Math.min(r.right, vw - ox)) / 2, 0), vw);
|
|
237
|
+
const cy = Math.min(Math.max((Math.max(r.top, -oy) + Math.min(r.bottom, vh - oy)) / 2, 0), vh);
|
|
238
|
+
const hit = root.elementFromPoint ? root.elementFromPoint(cx, cy) : null;
|
|
239
|
+
if (!hit) return false;
|
|
240
|
+
return hit === el || el.contains(hit) || inside(r, hit.getBoundingClientRect());
|
|
241
|
+
};
|
|
242
|
+
const walk = (root, ox, oy, frameOk) => {
|
|
243
|
+
for (const el of root.querySelectorAll('*')) {
|
|
244
|
+
if (el.shadowRoot) walk(el.shadowRoot, ox, oy, frameOk);
|
|
245
|
+
if (el.tagName === 'IFRAME') {
|
|
246
|
+
let doc = null;
|
|
247
|
+
try { doc = el.contentDocument; } catch (e) { doc = null; }
|
|
248
|
+
if (doc) {
|
|
249
|
+
const fr = el.getBoundingClientRect();
|
|
250
|
+
const nx = ox + fr.left + el.clientLeft, ny = oy + fr.top + el.clientTop;
|
|
251
|
+
walk(doc, nx, ny, (x, y) => frameOk(x, y) && document.elementFromPoint(x, y) === el);
|
|
252
|
+
}
|
|
253
|
+
continue;
|
|
254
|
+
}
|
|
255
|
+
const kind = kindOf(el);
|
|
256
|
+
if (!kind) continue;
|
|
257
|
+
const style = (el.ownerDocument.defaultView || window).getComputedStyle(el);
|
|
258
|
+
if (style.display === 'none' || style.visibility === 'hidden') continue;
|
|
259
|
+
const r = el.getBoundingClientRect();
|
|
260
|
+
if (r.width < 1 || r.height < 1) continue;
|
|
261
|
+
const x = r.left + ox, y = r.top + oy;
|
|
262
|
+
if (x >= vw || y >= vh || x + r.width <= 0 || y + r.height <= 0) continue;
|
|
263
|
+
if (!uncovered(el, r, el.getRootNode(), ox, oy)) continue;
|
|
264
|
+
if (!frameOk(x + r.width / 2, y + r.height / 2)) continue;
|
|
265
|
+
found.push({ x: Math.round(x), y: Math.round(y), w: Math.round(r.width), h: Math.round(r.height), kind });
|
|
266
|
+
}
|
|
267
|
+
};
|
|
268
|
+
walk(document, 0, 0, () => true);
|
|
269
|
+
const d = (f) => Math.hypot(f.x + f.w / 2 - vw / 2, f.y + f.h / 2 - vh / 2);
|
|
270
|
+
return found.sort((a, b) => d(a) - d(b)).slice(0, 30);
|
|
271
|
+
})()`;
|
|
272
|
+
var FOCUS_SOURCE = `(() => {
|
|
273
|
+
${KIND_OF}
|
|
274
|
+
const box = (r, kind, ox, oy) => ({ x: Math.round(r.left + ox), y: Math.round(r.top + oy), w: Math.round(r.width), h: Math.round(r.height), kind });
|
|
275
|
+
let el = document.activeElement, ox = 0, oy = 0;
|
|
276
|
+
for (let i = 0; el && i < 20; i++) {
|
|
277
|
+
if (el.shadowRoot && el.shadowRoot.activeElement) { el = el.shadowRoot.activeElement; continue; }
|
|
278
|
+
if (el.tagName === 'IFRAME') {
|
|
279
|
+
let doc = null;
|
|
280
|
+
try { doc = el.contentDocument; } catch (e) { doc = null; }
|
|
281
|
+
const fr = el.getBoundingClientRect();
|
|
282
|
+
if (!doc) return box(fr, 'text', ox, oy);
|
|
283
|
+
ox += fr.left + el.clientLeft;
|
|
284
|
+
oy += fr.top + el.clientTop;
|
|
285
|
+
el = doc.activeElement;
|
|
286
|
+
continue;
|
|
287
|
+
}
|
|
288
|
+
break;
|
|
289
|
+
}
|
|
290
|
+
const kind = kindOf(el);
|
|
291
|
+
return kind ? box(el.getBoundingClientRect(), kind, ox, oy) : null;
|
|
292
|
+
})()`;
|
|
293
|
+
var isKind = (k) => FieldKinds.includes(k);
|
|
294
|
+
var isField = (f) => {
|
|
295
|
+
const o = f;
|
|
296
|
+
return !!o && typeof o === "object" && ["x", "y", "w", "h"].every((k) => Number.isFinite(o[k])) && isKind(o.kind);
|
|
297
|
+
};
|
|
298
|
+
async function evaluate(cdp, expression) {
|
|
299
|
+
try {
|
|
300
|
+
const r = await cdp.send("Runtime.evaluate", { expression, returnByValue: true });
|
|
301
|
+
return r.exceptionDetails ? void 0 : r.result?.value;
|
|
302
|
+
} catch {
|
|
303
|
+
return void 0;
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
async function scanFields(cdp) {
|
|
307
|
+
const v = await evaluate(cdp, SCAN_SOURCE);
|
|
308
|
+
return Array.isArray(v) ? v.filter(isField).map(({ x, y, w, h, kind }) => ({ x, y, w, h, kind })) : [];
|
|
309
|
+
}
|
|
310
|
+
async function focusedField(cdp) {
|
|
311
|
+
const v = await evaluate(cdp, FOCUS_SOURCE);
|
|
312
|
+
return isField(v) ? { x: v.x, y: v.y, w: v.w, h: v.h, kind: v.kind } : null;
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
// src/stream/shortcuts.ts
|
|
316
|
+
var ALT = 1;
|
|
317
|
+
var CTRL = 2;
|
|
318
|
+
var META = 4;
|
|
319
|
+
var SHIFT = 8;
|
|
320
|
+
var EDIT_LETTERS = ["a", "z", "y"];
|
|
321
|
+
var WORD_KEYS = ["Backspace", "Delete", "ArrowLeft", "ArrowRight"];
|
|
322
|
+
var swap = (m, from, to) => m & ~from | to;
|
|
323
|
+
var only = (m, mod) => (m & ~SHIFT) === mod;
|
|
324
|
+
var withKey = (msg, key, modifiers, code = key) => {
|
|
325
|
+
const { text: _text, ...rest } = msg;
|
|
326
|
+
return { ...rest, key, code, modifiers };
|
|
327
|
+
};
|
|
328
|
+
function translateShortcut(msg, browserIsMac) {
|
|
329
|
+
if (msg.text !== void 0) return [msg];
|
|
330
|
+
const m = msg.modifiers;
|
|
331
|
+
const letter = msg.key.toLowerCase();
|
|
332
|
+
if (!browserIsMac) {
|
|
333
|
+
if (only(m, META) && EDIT_LETTERS.includes(letter)) return [{ ...msg, modifiers: swap(m, META, CTRL) }];
|
|
334
|
+
if (only(m, ALT) && WORD_KEYS.includes(msg.key)) return [{ ...msg, modifiers: swap(m, ALT, CTRL) }];
|
|
335
|
+
if (only(m, META) && (msg.key === "ArrowLeft" || msg.key === "ArrowRight")) return [withKey(msg, msg.key === "ArrowLeft" ? "Home" : "End", m & SHIFT)];
|
|
336
|
+
if (only(m, META) && msg.key === "Backspace") {
|
|
337
|
+
if (msg.action === "up") return [withKey(msg, "Backspace", 0)];
|
|
338
|
+
return [withKey(msg, "Home", SHIFT), { ...withKey(msg, "Home", SHIFT), action: "up" }, withKey(msg, "Backspace", 0)];
|
|
339
|
+
}
|
|
340
|
+
return [msg];
|
|
341
|
+
}
|
|
342
|
+
if (only(m, CTRL) && letter === "y") return [withKey(msg, "z", META | SHIFT, "KeyZ")];
|
|
343
|
+
if (only(m, CTRL) && EDIT_LETTERS.includes(letter)) return [{ ...msg, modifiers: swap(m, CTRL, META) }];
|
|
344
|
+
if (only(m, CTRL) && WORD_KEYS.includes(msg.key)) return [{ ...msg, modifiers: swap(m, CTRL, ALT) }];
|
|
345
|
+
if ((m & ~SHIFT) === 0 && (msg.key === "Home" || msg.key === "End")) return [withKey(msg, msg.key === "Home" ? "ArrowLeft" : "ArrowRight", META | m & SHIFT)];
|
|
346
|
+
return [msg];
|
|
347
|
+
}
|
|
348
|
+
var platforms = /* @__PURE__ */ new WeakMap();
|
|
349
|
+
function browserPlatformIsMac(cdp) {
|
|
350
|
+
let p = platforms.get(cdp);
|
|
351
|
+
if (!p) {
|
|
352
|
+
p = cdp.send("Runtime.evaluate", { expression: "navigator.platform", returnByValue: true }).then((r) => /^mac/i.test(String(r?.result?.value ?? ""))).catch(() => {
|
|
353
|
+
platforms.delete(cdp);
|
|
354
|
+
return false;
|
|
355
|
+
});
|
|
356
|
+
platforms.set(cdp, p);
|
|
357
|
+
}
|
|
358
|
+
return p;
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
// src/session/field-feed.ts
|
|
362
|
+
var FRAME_INTERVAL_MS = 500;
|
|
363
|
+
var RESEND_MS = 5e3;
|
|
364
|
+
function createFieldFeed(deps) {
|
|
365
|
+
let stopped = false;
|
|
366
|
+
let lastScanAt = -Infinity;
|
|
367
|
+
let last = [];
|
|
368
|
+
let lastJson = "[]";
|
|
369
|
+
let inFlight = false;
|
|
370
|
+
let dirty = false;
|
|
371
|
+
let pendingForce = false;
|
|
372
|
+
let trailing = null;
|
|
373
|
+
const run = () => {
|
|
374
|
+
if (stopped) return;
|
|
375
|
+
if (inFlight) return void (dirty = true);
|
|
376
|
+
inFlight = true;
|
|
377
|
+
lastScanAt = Date.now();
|
|
378
|
+
const force = pendingForce;
|
|
379
|
+
pendingForce = false;
|
|
380
|
+
deps.scan().then((fields) => {
|
|
381
|
+
if (stopped) return;
|
|
382
|
+
const json = JSON.stringify(fields);
|
|
383
|
+
if (json === lastJson && !(force && fields.length > 0)) return;
|
|
384
|
+
last = fields;
|
|
385
|
+
lastJson = json;
|
|
386
|
+
deps.send({ t: "fields", fields });
|
|
387
|
+
}).catch(() => {
|
|
388
|
+
}).finally(() => {
|
|
389
|
+
inFlight = false;
|
|
390
|
+
if (dirty) {
|
|
391
|
+
dirty = false;
|
|
392
|
+
run();
|
|
393
|
+
}
|
|
394
|
+
});
|
|
395
|
+
};
|
|
396
|
+
const request = (immediate, force = false) => {
|
|
397
|
+
if (stopped) return;
|
|
398
|
+
if (force) pendingForce = true;
|
|
399
|
+
const wait = lastScanAt + FRAME_INTERVAL_MS - Date.now();
|
|
400
|
+
if (immediate || wait <= 0) return run();
|
|
401
|
+
trailing ??= setTimeout(() => {
|
|
402
|
+
trailing = null;
|
|
403
|
+
run();
|
|
404
|
+
}, wait);
|
|
405
|
+
};
|
|
406
|
+
const resend = setInterval(() => {
|
|
407
|
+
if (!stopped && last.length > 0) deps.send({ t: "fields", fields: last });
|
|
408
|
+
}, RESEND_MS);
|
|
409
|
+
return {
|
|
410
|
+
onFrame() {
|
|
411
|
+
request(false);
|
|
412
|
+
},
|
|
413
|
+
onInput(msg) {
|
|
414
|
+
if (stopped) return;
|
|
415
|
+
const tap = msg.t === "mouse" && msg.action === "up";
|
|
416
|
+
const typed = msg.t === "key" && msg.action === "down" || msg.t === "text";
|
|
417
|
+
if (!(tap || typed || msg.t === "wheel")) return;
|
|
418
|
+
request(true);
|
|
419
|
+
if (tap || typed)
|
|
420
|
+
deps.focus().then((field) => void (stopped || deps.send(field ? { t: "focus", kind: field.kind, field } : { t: "focus", kind: null }))).catch(() => {
|
|
421
|
+
});
|
|
422
|
+
},
|
|
423
|
+
onTargetChanged() {
|
|
424
|
+
request(true, true);
|
|
425
|
+
},
|
|
426
|
+
stop() {
|
|
427
|
+
stopped = true;
|
|
428
|
+
clearInterval(resend);
|
|
429
|
+
if (trailing) clearTimeout(trailing);
|
|
430
|
+
}
|
|
431
|
+
};
|
|
432
|
+
}
|
|
433
|
+
|
|
203
434
|
// src/stream/latest.ts
|
|
204
435
|
function createLatestSender(conn, opts = {}) {
|
|
205
436
|
const mark = opts.highWaterMark ?? 512 * 1024;
|
|
@@ -374,6 +605,11 @@ function runTakeover(deps, cdp, opts) {
|
|
|
374
605
|
sendJson(c, { t: "error", code: code2, message: errorText[code2] });
|
|
375
606
|
if (close) c.close(1008, code2);
|
|
376
607
|
};
|
|
608
|
+
const fieldFeed = createFieldFeed({
|
|
609
|
+
scan: () => scanFields(active().cdp),
|
|
610
|
+
focus: () => focusedField(active().cdp),
|
|
611
|
+
send: (m) => void (holder && sendJson(holder, m))
|
|
612
|
+
});
|
|
377
613
|
const go = (ev) => {
|
|
378
614
|
status = nextStatus(status, ev);
|
|
379
615
|
emit("status", { status });
|
|
@@ -401,6 +637,7 @@ function runTakeover(deps, cdp, opts) {
|
|
|
401
637
|
helloTimers.clear();
|
|
402
638
|
offClose();
|
|
403
639
|
offPopups();
|
|
640
|
+
fieldFeed.stop();
|
|
404
641
|
for (const t of targets) t.off();
|
|
405
642
|
const viewer = holder;
|
|
406
643
|
holder = null;
|
|
@@ -432,6 +669,7 @@ function runTakeover(deps, cdp, opts) {
|
|
|
432
669
|
sendJson(conn, { t: "ready", reason: opts.reason, site: opts.site ?? "", expiresAt, resumeKey });
|
|
433
670
|
sender = createLatestSender(conn);
|
|
434
671
|
if (lastFrame) sender.push(lastFrame);
|
|
672
|
+
fieldFeed.onTargetChanged();
|
|
435
673
|
if (!stopScreencast) await startStream();
|
|
436
674
|
}
|
|
437
675
|
async function startStream() {
|
|
@@ -444,6 +682,7 @@ function runTakeover(deps, cdp, opts) {
|
|
|
444
682
|
if (gen !== streamGen) return;
|
|
445
683
|
lastFrame = encodeFrame(f);
|
|
446
684
|
sender?.push(lastFrame);
|
|
685
|
+
fieldFeed.onFrame();
|
|
447
686
|
});
|
|
448
687
|
} catch (e) {
|
|
449
688
|
if (gen === streamGen) onTargetError(target, e);
|
|
@@ -464,6 +703,7 @@ function runTakeover(deps, cdp, opts) {
|
|
|
464
703
|
lastFrame = null;
|
|
465
704
|
if (!holder || isTerminal(status)) return;
|
|
466
705
|
sender = createLatestSender(holder);
|
|
706
|
+
fieldFeed.onTargetChanged();
|
|
467
707
|
await startStream();
|
|
468
708
|
}
|
|
469
709
|
async function addPopup(popup) {
|
|
@@ -562,9 +802,15 @@ function runTakeover(deps, cdp, opts) {
|
|
|
562
802
|
return finish("cancelled", msg.reason || "cant_solve");
|
|
563
803
|
default:
|
|
564
804
|
inputCount++;
|
|
565
|
-
inputChain = inputChain.then(() => {
|
|
805
|
+
inputChain = inputChain.then(async () => {
|
|
566
806
|
const target = active();
|
|
567
|
-
|
|
807
|
+
try {
|
|
808
|
+
const msgs = msg.t === "key" ? translateShortcut(msg, await browserPlatformIsMac(target.cdp)) : [msg];
|
|
809
|
+
for (const m of msgs) await applyInput(target.cdp, m, target.caps);
|
|
810
|
+
} catch (e) {
|
|
811
|
+
return onTargetError(target, e);
|
|
812
|
+
}
|
|
813
|
+
fieldFeed.onInput(msg);
|
|
568
814
|
});
|
|
569
815
|
}
|
|
570
816
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@velmlabs/unstuck-core",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.3",
|
|
4
4
|
"description": "The Unstuck takeover engine core: screencast, input, sessions and transports, usable in Node or Cloudflare Workers.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"ai",
|
|
@@ -46,7 +46,7 @@
|
|
|
46
46
|
"node": ">=20"
|
|
47
47
|
},
|
|
48
48
|
"dependencies": {
|
|
49
|
-
"@velmlabs/unstuck-protocol": "0.1.
|
|
49
|
+
"@velmlabs/unstuck-protocol": "0.1.3"
|
|
50
50
|
},
|
|
51
51
|
"publishConfig": {
|
|
52
52
|
"access": "public"
|