@velmlabs/unstuck-core 0.1.2 → 0.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (2) hide show
  1. package/dist/index.js +250 -4
  2. package/package.json +2 -2
package/dist/index.js CHANGED
@@ -159,8 +159,18 @@ var VIRTUAL_KEYS = {
159
159
  ArrowUp: 38,
160
160
  ArrowRight: 39,
161
161
  ArrowDown: 40,
162
- Delete: 46
162
+ Delete: 46,
163
+ PageUp: 33,
164
+ PageDown: 34,
165
+ End: 35,
166
+ Home: 36,
167
+ Insert: 45,
168
+ ...Object.fromEntries(Array.from({ length: 12 }, (_, i) => [`F${i + 1}`, 112 + i]))
163
169
  };
170
+ function virtualKey(key) {
171
+ if (VIRTUAL_KEYS[key] !== void 0) return VIRTUAL_KEYS[key];
172
+ return /^[a-zA-Z0-9]$/.test(key) ? key.toUpperCase().charCodeAt(0) : void 0;
173
+ }
164
174
  var MOUSE_TYPES = { down: "mousePressed", up: "mouseReleased", move: "mouseMoved" };
165
175
  async function applyInput(cdp, msg, caps) {
166
176
  switch (msg.t) {
@@ -179,7 +189,7 @@ async function applyInput(cdp, msg, caps) {
179
189
  return;
180
190
  case "key": {
181
191
  const base = { key: msg.key, code: msg.code, modifiers: msg.modifiers };
182
- const vk = VIRTUAL_KEYS[msg.key];
192
+ const vk = virtualKey(msg.key);
183
193
  const withVk = vk === void 0 ? {} : { windowsVirtualKeyCode: vk };
184
194
  if (msg.action === "up") await cdp.send("Input.dispatchKeyEvent", { type: "keyUp", ...base });
185
195
  else if (msg.text !== void 0)
@@ -200,6 +210,227 @@ async function applyInput(cdp, msg, caps) {
200
210
  }
201
211
  }
202
212
 
213
+ // src/stream/fields.ts
214
+ import { FieldKinds } from "@velmlabs/unstuck-protocol";
215
+ var KIND_OF = `function kindOf(el) {
216
+ if (!el || el.nodeType !== 1 || el.disabled || el.readOnly) return null;
217
+ const tag = el.tagName;
218
+ if (tag === 'TEXTAREA') return 'text';
219
+ if (tag !== 'INPUT') return el.isContentEditable ? 'text' : null;
220
+ const type = (el.getAttribute('type') || 'text').toLowerCase();
221
+ if (!['text', 'tel', 'number', 'email', 'password', 'search', 'url'].includes(type)) return null;
222
+ const im = (el.getAttribute('inputmode') || '').toLowerCase();
223
+ const ac = (el.getAttribute('autocomplete') || '').toLowerCase().split(/\\s+/);
224
+ if (ac.includes('one-time-code')) return 'otp';
225
+ if (type === 'password') return 'password';
226
+ if (type === 'tel' || type === 'number' || ['numeric', 'decimal', 'tel'].includes(im)) return 'numeric';
227
+ if (type === 'email' || im === 'email') return 'email';
228
+ return 'text';
229
+ }`;
230
+ var SCAN_SOURCE = `(() => {
231
+ ${KIND_OF}
232
+ const vw = innerWidth, vh = innerHeight, found = [];
233
+ const inside = (a, b) => b.left >= a.left - 1 && b.top >= a.top - 1 && b.right <= a.right + 1 && b.bottom <= a.bottom + 1;
234
+ // Something else on top of the field's visible centre means a tap there doesn't reach it.
235
+ const uncovered = (el, r, root, ox, oy) => {
236
+ const cx = Math.min(Math.max((Math.max(r.left, -ox) + Math.min(r.right, vw - ox)) / 2, 0), vw);
237
+ const cy = Math.min(Math.max((Math.max(r.top, -oy) + Math.min(r.bottom, vh - oy)) / 2, 0), vh);
238
+ const hit = root.elementFromPoint ? root.elementFromPoint(cx, cy) : null;
239
+ if (!hit) return false;
240
+ return hit === el || el.contains(hit) || inside(r, hit.getBoundingClientRect());
241
+ };
242
+ const walk = (root, ox, oy, frameOk) => {
243
+ for (const el of root.querySelectorAll('*')) {
244
+ if (el.shadowRoot) walk(el.shadowRoot, ox, oy, frameOk);
245
+ if (el.tagName === 'IFRAME') {
246
+ let doc = null;
247
+ try { doc = el.contentDocument; } catch (e) { doc = null; }
248
+ if (doc) {
249
+ const fr = el.getBoundingClientRect();
250
+ const nx = ox + fr.left + el.clientLeft, ny = oy + fr.top + el.clientTop;
251
+ walk(doc, nx, ny, (x, y) => frameOk(x, y) && document.elementFromPoint(x, y) === el);
252
+ }
253
+ continue;
254
+ }
255
+ const kind = kindOf(el);
256
+ if (!kind) continue;
257
+ const style = (el.ownerDocument.defaultView || window).getComputedStyle(el);
258
+ if (style.display === 'none' || style.visibility === 'hidden') continue;
259
+ const r = el.getBoundingClientRect();
260
+ if (r.width < 1 || r.height < 1) continue;
261
+ const x = r.left + ox, y = r.top + oy;
262
+ if (x >= vw || y >= vh || x + r.width <= 0 || y + r.height <= 0) continue;
263
+ if (!uncovered(el, r, el.getRootNode(), ox, oy)) continue;
264
+ if (!frameOk(x + r.width / 2, y + r.height / 2)) continue;
265
+ found.push({ x: Math.round(x), y: Math.round(y), w: Math.round(r.width), h: Math.round(r.height), kind });
266
+ }
267
+ };
268
+ walk(document, 0, 0, () => true);
269
+ const d = (f) => Math.hypot(f.x + f.w / 2 - vw / 2, f.y + f.h / 2 - vh / 2);
270
+ return found.sort((a, b) => d(a) - d(b)).slice(0, 30);
271
+ })()`;
272
+ var FOCUS_SOURCE = `(() => {
273
+ ${KIND_OF}
274
+ const box = (r, kind, ox, oy) => ({ x: Math.round(r.left + ox), y: Math.round(r.top + oy), w: Math.round(r.width), h: Math.round(r.height), kind });
275
+ let el = document.activeElement, ox = 0, oy = 0;
276
+ for (let i = 0; el && i < 20; i++) {
277
+ if (el.shadowRoot && el.shadowRoot.activeElement) { el = el.shadowRoot.activeElement; continue; }
278
+ if (el.tagName === 'IFRAME') {
279
+ let doc = null;
280
+ try { doc = el.contentDocument; } catch (e) { doc = null; }
281
+ const fr = el.getBoundingClientRect();
282
+ if (!doc) return box(fr, 'text', ox, oy);
283
+ ox += fr.left + el.clientLeft;
284
+ oy += fr.top + el.clientTop;
285
+ el = doc.activeElement;
286
+ continue;
287
+ }
288
+ break;
289
+ }
290
+ const kind = kindOf(el);
291
+ return kind ? box(el.getBoundingClientRect(), kind, ox, oy) : null;
292
+ })()`;
293
+ var isKind = (k) => FieldKinds.includes(k);
294
+ var isField = (f) => {
295
+ const o = f;
296
+ return !!o && typeof o === "object" && ["x", "y", "w", "h"].every((k) => Number.isFinite(o[k])) && isKind(o.kind);
297
+ };
298
+ async function evaluate(cdp, expression) {
299
+ try {
300
+ const r = await cdp.send("Runtime.evaluate", { expression, returnByValue: true });
301
+ return r.exceptionDetails ? void 0 : r.result?.value;
302
+ } catch {
303
+ return void 0;
304
+ }
305
+ }
306
+ async function scanFields(cdp) {
307
+ const v = await evaluate(cdp, SCAN_SOURCE);
308
+ return Array.isArray(v) ? v.filter(isField).map(({ x, y, w, h, kind }) => ({ x, y, w, h, kind })) : [];
309
+ }
310
+ async function focusedField(cdp) {
311
+ const v = await evaluate(cdp, FOCUS_SOURCE);
312
+ return isField(v) ? { x: v.x, y: v.y, w: v.w, h: v.h, kind: v.kind } : null;
313
+ }
314
+
315
+ // src/stream/shortcuts.ts
316
+ var ALT = 1;
317
+ var CTRL = 2;
318
+ var META = 4;
319
+ var SHIFT = 8;
320
+ var EDIT_LETTERS = ["a", "z", "y"];
321
+ var WORD_KEYS = ["Backspace", "Delete", "ArrowLeft", "ArrowRight"];
322
+ var swap = (m, from, to) => m & ~from | to;
323
+ var only = (m, mod) => (m & ~SHIFT) === mod;
324
+ var withKey = (msg, key, modifiers, code = key) => {
325
+ const { text: _text, ...rest } = msg;
326
+ return { ...rest, key, code, modifiers };
327
+ };
328
+ function translateShortcut(msg, browserIsMac) {
329
+ if (msg.text !== void 0) return [msg];
330
+ const m = msg.modifiers;
331
+ const letter = msg.key.toLowerCase();
332
+ if (!browserIsMac) {
333
+ if (only(m, META) && EDIT_LETTERS.includes(letter)) return [{ ...msg, modifiers: swap(m, META, CTRL) }];
334
+ if (only(m, ALT) && WORD_KEYS.includes(msg.key)) return [{ ...msg, modifiers: swap(m, ALT, CTRL) }];
335
+ if (only(m, META) && (msg.key === "ArrowLeft" || msg.key === "ArrowRight")) return [withKey(msg, msg.key === "ArrowLeft" ? "Home" : "End", m & SHIFT)];
336
+ if (only(m, META) && msg.key === "Backspace") {
337
+ if (msg.action === "up") return [withKey(msg, "Backspace", 0)];
338
+ return [withKey(msg, "Home", SHIFT), { ...withKey(msg, "Home", SHIFT), action: "up" }, withKey(msg, "Backspace", 0)];
339
+ }
340
+ return [msg];
341
+ }
342
+ if (only(m, CTRL) && letter === "y") return [withKey(msg, "z", META | SHIFT, "KeyZ")];
343
+ if (only(m, CTRL) && EDIT_LETTERS.includes(letter)) return [{ ...msg, modifiers: swap(m, CTRL, META) }];
344
+ if (only(m, CTRL) && WORD_KEYS.includes(msg.key)) return [{ ...msg, modifiers: swap(m, CTRL, ALT) }];
345
+ if ((m & ~SHIFT) === 0 && (msg.key === "Home" || msg.key === "End")) return [withKey(msg, msg.key === "Home" ? "ArrowLeft" : "ArrowRight", META | m & SHIFT)];
346
+ return [msg];
347
+ }
348
+ var platforms = /* @__PURE__ */ new WeakMap();
349
+ function browserPlatformIsMac(cdp) {
350
+ let p = platforms.get(cdp);
351
+ if (!p) {
352
+ p = cdp.send("Runtime.evaluate", { expression: "navigator.platform", returnByValue: true }).then((r) => /^mac/i.test(String(r?.result?.value ?? ""))).catch(() => {
353
+ platforms.delete(cdp);
354
+ return false;
355
+ });
356
+ platforms.set(cdp, p);
357
+ }
358
+ return p;
359
+ }
360
+
361
+ // src/session/field-feed.ts
362
+ var FRAME_INTERVAL_MS = 500;
363
+ var RESEND_MS = 5e3;
364
+ function createFieldFeed(deps) {
365
+ let stopped = false;
366
+ let lastScanAt = -Infinity;
367
+ let last = [];
368
+ let lastJson = "[]";
369
+ let inFlight = false;
370
+ let dirty = false;
371
+ let pendingForce = false;
372
+ let trailing = null;
373
+ const run = () => {
374
+ if (stopped) return;
375
+ if (inFlight) return void (dirty = true);
376
+ inFlight = true;
377
+ lastScanAt = Date.now();
378
+ const force = pendingForce;
379
+ pendingForce = false;
380
+ deps.scan().then((fields) => {
381
+ if (stopped) return;
382
+ const json = JSON.stringify(fields);
383
+ if (json === lastJson && !(force && fields.length > 0)) return;
384
+ last = fields;
385
+ lastJson = json;
386
+ deps.send({ t: "fields", fields });
387
+ }).catch(() => {
388
+ }).finally(() => {
389
+ inFlight = false;
390
+ if (dirty) {
391
+ dirty = false;
392
+ run();
393
+ }
394
+ });
395
+ };
396
+ const request = (immediate, force = false) => {
397
+ if (stopped) return;
398
+ if (force) pendingForce = true;
399
+ const wait = lastScanAt + FRAME_INTERVAL_MS - Date.now();
400
+ if (immediate || wait <= 0) return run();
401
+ trailing ??= setTimeout(() => {
402
+ trailing = null;
403
+ run();
404
+ }, wait);
405
+ };
406
+ const resend = setInterval(() => {
407
+ if (!stopped && last.length > 0) deps.send({ t: "fields", fields: last });
408
+ }, RESEND_MS);
409
+ return {
410
+ onFrame() {
411
+ request(false);
412
+ },
413
+ onInput(msg) {
414
+ if (stopped) return;
415
+ const tap = msg.t === "mouse" && msg.action === "up";
416
+ const typed = msg.t === "key" && msg.action === "down" || msg.t === "text";
417
+ if (!(tap || typed || msg.t === "wheel")) return;
418
+ request(true);
419
+ if (tap || typed)
420
+ deps.focus().then((field) => void (stopped || deps.send(field ? { t: "focus", kind: field.kind, field } : { t: "focus", kind: null }))).catch(() => {
421
+ });
422
+ },
423
+ onTargetChanged() {
424
+ request(true, true);
425
+ },
426
+ stop() {
427
+ stopped = true;
428
+ clearInterval(resend);
429
+ if (trailing) clearTimeout(trailing);
430
+ }
431
+ };
432
+ }
433
+
203
434
  // src/stream/latest.ts
204
435
  function createLatestSender(conn, opts = {}) {
205
436
  const mark = opts.highWaterMark ?? 512 * 1024;
@@ -374,6 +605,11 @@ function runTakeover(deps, cdp, opts) {
374
605
  sendJson(c, { t: "error", code: code2, message: errorText[code2] });
375
606
  if (close) c.close(1008, code2);
376
607
  };
608
+ const fieldFeed = createFieldFeed({
609
+ scan: () => scanFields(active().cdp),
610
+ focus: () => focusedField(active().cdp),
611
+ send: (m) => void (holder && sendJson(holder, m))
612
+ });
377
613
  const go = (ev) => {
378
614
  status = nextStatus(status, ev);
379
615
  emit("status", { status });
@@ -401,6 +637,7 @@ function runTakeover(deps, cdp, opts) {
401
637
  helloTimers.clear();
402
638
  offClose();
403
639
  offPopups();
640
+ fieldFeed.stop();
404
641
  for (const t of targets) t.off();
405
642
  const viewer = holder;
406
643
  holder = null;
@@ -432,6 +669,7 @@ function runTakeover(deps, cdp, opts) {
432
669
  sendJson(conn, { t: "ready", reason: opts.reason, site: opts.site ?? "", expiresAt, resumeKey });
433
670
  sender = createLatestSender(conn);
434
671
  if (lastFrame) sender.push(lastFrame);
672
+ fieldFeed.onTargetChanged();
435
673
  if (!stopScreencast) await startStream();
436
674
  }
437
675
  async function startStream() {
@@ -444,6 +682,7 @@ function runTakeover(deps, cdp, opts) {
444
682
  if (gen !== streamGen) return;
445
683
  lastFrame = encodeFrame(f);
446
684
  sender?.push(lastFrame);
685
+ fieldFeed.onFrame();
447
686
  });
448
687
  } catch (e) {
449
688
  if (gen === streamGen) onTargetError(target, e);
@@ -464,6 +703,7 @@ function runTakeover(deps, cdp, opts) {
464
703
  lastFrame = null;
465
704
  if (!holder || isTerminal(status)) return;
466
705
  sender = createLatestSender(holder);
706
+ fieldFeed.onTargetChanged();
467
707
  await startStream();
468
708
  }
469
709
  async function addPopup(popup) {
@@ -562,9 +802,15 @@ function runTakeover(deps, cdp, opts) {
562
802
  return finish("cancelled", msg.reason || "cant_solve");
563
803
  default:
564
804
  inputCount++;
565
- inputChain = inputChain.then(() => {
805
+ inputChain = inputChain.then(async () => {
566
806
  const target = active();
567
- return applyInput(target.cdp, msg, target.caps).catch((e) => onTargetError(target, e));
807
+ try {
808
+ const msgs = msg.t === "key" ? translateShortcut(msg, await browserPlatformIsMac(target.cdp)) : [msg];
809
+ for (const m of msgs) await applyInput(target.cdp, m, target.caps);
810
+ } catch (e) {
811
+ return onTargetError(target, e);
812
+ }
813
+ fieldFeed.onInput(msg);
568
814
  });
569
815
  }
570
816
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@velmlabs/unstuck-core",
3
- "version": "0.1.2",
3
+ "version": "0.1.3",
4
4
  "description": "The Unstuck takeover engine core: screencast, input, sessions and transports, usable in Node or Cloudflare Workers.",
5
5
  "keywords": [
6
6
  "ai",
@@ -46,7 +46,7 @@
46
46
  "node": ">=20"
47
47
  },
48
48
  "dependencies": {
49
- "@velmlabs/unstuck-protocol": "0.1.2"
49
+ "@velmlabs/unstuck-protocol": "0.1.3"
50
50
  },
51
51
  "publishConfig": {
52
52
  "access": "public"