@vosjs/cli 0.33.0 → 0.34.0

package/README.md

@@ -143,7 +143,7 @@ A re-record replaces the footage, the cursor track, the frames and everything de
 }
 ```
 
-Seven verbs: `wait`, `hover` (`ms` 700), `click`, `type` (`delayMs` 40 per key; `focus: false` skips the focusing click, for a submitting Enter), `scroll`, `move`, `drag` (press, move, release: a range input, a canvas element, a timeline clip). Every step takes an optional unique `id`; give steps ids so a span anchored to a step survives script edits and `vos plan --reuse` can follow it, and so a layer can name the element a step touched (`overlays[].pin.step`; the recorder keeps each hover, click, type and drag step's element rect on `meta.steps[].rect`). Because the CLI issues every input itself, the cursor track is synthesized with exact coordinates, exact timing and fresh element rects, which is what powers element-aware auto-zoom and click effects downstream. The schema is [`schema/actions.schema.json`](./schema/actions.schema.json); `vos validate actions.json` checks a script without running anything.
+Seven verbs: `wait`, `hover` (`ms` 700, the dwell), `click` (`ms` the settle after the press, 150), `type` (`delayMs` 40 per key, paced by the clock; `focus: false` skips the focusing click, for a submitting Enter; `ms` the settle after, 150), `scroll` (`ms` the settle, 200), `move`, `drag` (press, move, release: a range input, a canvas element, a timeline clip; `ms` the travel, 700). The recorder's pace is the script's: a gesture is driven by the clock, so a page that answers each pointer sample slowly costs samples, never seconds, and the only pauses are the ones the script names (`wait`, a hover's dwell, a settle). `vos record` ends with a pace line, what the script asked, what the gestures added, what the page cost, and the steps that ran slow, and the `--json` done event carries it as `pace`. Every step takes an optional unique `id`; give steps ids so a span anchored to a step survives script edits and `vos plan --reuse` can follow it, and so a layer can name the element a step touched (`overlays[].pin.step`; the recorder keeps each hover, click, type and drag step's element rect on `meta.steps[].rect`). Because the CLI issues every input itself, the cursor track is synthesized with exact coordinates, exact timing and fresh element rects, which is what powers element-aware auto-zoom and click effects downstream. The schema is [`schema/actions.schema.json`](./schema/actions.schema.json); `vos validate actions.json` checks a script without running anything.
 
 Verified the flow in agent-browser already? `vos actions from-agent-browser steps.jsonl [--out actions.json] [--url] [--viewport WxH]` writes the script from that walk (each command kept beside its `--json` result, the batch record shape; refs resolve through the last `snapshot -i`), and names every step the recorder cannot follow rather than dropping it.
 

package/dist/{chunk-6O55OD6D.js → chunk-43FCXWNS.js}

@@ -156,6 +156,8 @@ function validateActions(value) {
       errors.push(`${at2}: ${s.do} needs a selector`);
     if (s.do === "wait" && typeof s.ms !== "number")
       errors.push(`${at2}: wait needs ms`);
+    if (s.ms !== void 0 && (typeof s.ms !== "number" || !Number.isFinite(s.ms) || s.ms < 0))
+      errors.push(`${at2}: ms must be a number of milliseconds \u2265 0`);
     if (s.do === "type" && typeof s.text !== "string")
       errors.push(`${at2}: type needs text`);
     if (s.do === "type" && s.focus !== void 0 && typeof s.focus !== "boolean")
@@ -6120,9 +6122,99 @@ function cappedLine(capSeconds) {
   return `stopped at ${formatDurationCap(capSeconds)} (--max-duration ${capSeconds}); the remaining steps did not run`;
 }
 
+// src/plugin/pace.ts
+var POINTER_MS_PER_PX = 0.7;
+var POINTER_MIN_MS = 250;
+var POINTER_MAX_MS = 800;
+var MOTION_TICK_MS = 16;
+var SETTLE_MS = {
+  click: 150,
+  type: 150,
+  scroll: 200,
+  drag: 80
+};
+var PRESS_LEAD_MS = 80;
+var PRESS_HOLD_MS = 70;
+var SCROLL_SETTLE_MS = 120;
+var TRAILING_HOLD_MS = 400;
+function pointerTravelMs(dist) {
+  return Math.min(
+    POINTER_MAX_MS,
+    Math.max(POINTER_MIN_MS, Math.round(dist * POINTER_MS_PER_PX))
+  );
+}
+function settleMs(step) {
+  if (typeof step.ms === "number" && step.ms >= 0) return step.ms;
+  const d = step.do;
+  return SETTLE_MS[d] ?? 0;
+}
+var easeInOutCubic = (u) => u < 0.5 ? 4 * u * u * u : 1 - Math.pow(-2 * u + 2, 3) / 2;
+async function clockMotion(dur, at2, clock) {
+  const start = clock.now();
+  let samples = 0;
+  for (; ; ) {
+    const elapsed = clock.now() - start;
+    if (elapsed >= dur) break;
+    await at2(easeInOutCubic(Math.min(1, elapsed / dur)));
+    samples++;
+    const next = start + Math.ceil((clock.now() - start) / MOTION_TICK_MS) * MOTION_TICK_MS;
+    const wait = Math.min(next, start + dur) - clock.now();
+    if (wait > 0) await clock.sleep(wait);
+  }
+  await at2(1);
+  return samples + 1;
+}
+async function clockTyping(chars, delay, type, clock) {
+  const start = clock.now();
+  for (let i = 0; i < chars.length; i++) {
+    await type(chars[i], i);
+    const due = start + (i + 1) * delay;
+    const wait = due - clock.now();
+    if (wait > 0 && i < chars.length - 1) await clock.sleep(wait);
+  }
+}
+function askedMs(step) {
+  switch (step.do) {
+    case "wait":
+      return step.ms ?? 0;
+    case "hover":
+      return step.ms ?? 700;
+    case "drag":
+      return step.ms ?? 700;
+    case "type":
+      return (step.text?.length ?? 0) * (step.delayMs ?? 40);
+    default:
+      return 0;
+  }
+}
+function paceReport(steps) {
+  const askedMs2 = steps.reduce((a, s) => a + s.askedMs, 0);
+  const gestureMs = steps.reduce((a, s) => a + s.gestureMs, 0);
+  const wallMs = steps.reduce((a, s) => a + s.wallMs, 0);
+  const overheadMs = Math.max(0, wallMs - askedMs2 - gestureMs);
+  const slow = steps.filter((s) => s.wallMs > (s.askedMs + s.gestureMs) * 1.34 + 60).map((s) => ({
+    step: s.step,
+    do: s.do,
+    askedMs: s.askedMs,
+    wallMs: s.wallMs
+  }));
+  return {
+    askedMs: askedMs2,
+    gestureMs,
+    wallMs,
+    overheadMs,
+    overheadPct: wallMs > 0 ? Math.round(overheadMs / wallMs * 100) : 0,
+    slow
+  };
+}
+function paceLine(r) {
+  const s = (ms) => `${(ms / 1e3).toFixed(1)} s`;
+  const slow = r.slow.length ? `; slow: ${r.slow.map((x) => `#${x.step} ${x.do} ${s(x.wallMs)} for ${s(x.askedMs)} asked`).join(", ")}` : "";
+  return `pace: the script asked ${s(r.askedMs)}, the gestures added ${s(r.gestureMs)}, the take ran ${s(r.wallMs)} (${r.overheadPct} % overhead${slow})`;
+}
+
 // src/plugin/recorder.ts
 var sleep = (ms) => new Promise((r) => setTimeout(r, ms));
-var easeInOutCubic = (u) => u < 0.5 ? 4 * u * u * u : 1 - Math.pow(-2 * u + 2, 3) / 2;
 function jpegDims(buf) {
   let i = 2;
   while (i < buf.length - 9) {
@@ -6181,26 +6273,30 @@ async function recordTake(browser, url, actions, paths, log, opts = {}) {
   const emit = (e) => events.push({ t: now(), ...e });
   await page.mouse.move(cur.x, cur.y);
   emit({ x: cur.x, y: cur.y, type: "move" });
+  const clock = { now: () => Date.now(), sleep };
   const moveTo = async (tx, ty) => {
     const dist = Math.hypot(tx - cur.x, ty - cur.y);
-    const dur = Math.min(1200, Math.max(350, dist * 1.4));
-    const steps = Math.max(6, Math.round(dur / 16));
     const from = { ...cur };
-    for (let i = 1; i <= steps; i++) {
-      const u = easeInOutCubic(i / steps);
-      cur.x = from.x + (tx - from.x) * u;
-      cur.y = from.y + (ty - from.y) * u;
-      await page.mouse.move(cur.x, cur.y);
-      emit({ x: Math.round(cur.x), y: Math.round(cur.y), type: "move" });
-      await sleep(14);
-    }
+    await clockMotion(
+      pointerTravelMs(dist),
+      async (u) => {
+        cur.x = from.x + (tx - from.x) * u;
+        cur.y = from.y + (ty - from.y) * u;
+        await page.mouse.move(cur.x, cur.y);
+        emit({ x: Math.round(cur.x), y: Math.round(cur.y), type: "move" });
+      },
+      clock
+    );
   };
   const boxOf = async (selector) => {
     const loc = page.locator(selector).first();
     try {
       await loc.waitFor({ state: "visible", timeout: 8e3 });
+      const before = await page.evaluate(() => [window.scrollX, window.scrollY]).catch(() => [0, 0]);
       await loc.scrollIntoViewIfNeeded();
-      await sleep(250);
+      const after = await page.evaluate(() => [window.scrollX, window.scrollY]).catch(() => before);
+      if (before[0] !== after[0] || before[1] !== after[1])
+        await sleep(SCROLL_SETTLE_MS);
       const bb = await loc.boundingBox();
       if (!bb) return null;
       return { x: bb.x, y: bb.y, w: bb.width, h: bb.height };
@@ -6211,7 +6307,7 @@ async function recordTake(browser, url, actions, paths, log, opts = {}) {
   };
   const clickAt = async (rect) => {
     await moveTo(rect.x + rect.w / 2, rect.y + rect.h / 2);
-    await sleep(120);
+    await sleep(PRESS_LEAD_MS);
     emit({
       x: Math.round(cur.x),
       y: Math.round(cur.y),
@@ -6220,7 +6316,7 @@ async function recordTake(browser, url, actions, paths, log, opts = {}) {
       rect
     });
     await page.mouse.down();
-    await sleep(90);
+    await sleep(PRESS_HOLD_MS);
     await page.mouse.up();
     emit({
       x: Math.round(cur.x),
@@ -6232,6 +6328,7 @@ async function recordTake(browser, url, actions, paths, log, opts = {}) {
   };
   const skipped = [];
   const stepSpans = [];
+  const paces = [];
   let capped = false;
   for (const [stepIdx, step] of actions.steps.entries()) {
     if (capReached(now(), maxSeconds)) {
@@ -6239,6 +6336,13 @@ async function recordTake(browser, url, actions, paths, log, opts = {}) {
       break;
     }
     const stepStart = now();
+    let gestureMs = 0;
+    const gesture = async (fn) => {
+      const t = now();
+      const out = await fn();
+      gestureMs += now() - t;
+      return out;
+    };
     const skippedBefore = skipped.length;
     const urlBefore = page.url();
     let stepRect = null;
@@ -6253,7 +6357,7 @@ async function recordTake(browser, url, actions, paths, log, opts = {}) {
         const rect = await boxOf(step.selector);
         stepRect = rect;
         if (rect) {
-          await moveTo(rect.x + rect.w / 2, rect.y + rect.h / 2);
+          await gesture(() => moveTo(rect.x + rect.w / 2, rect.y + rect.h / 2));
           log(`hover ${step.selector}`);
           await sleep(step.ms ?? 700);
         } else
@@ -6264,9 +6368,9 @@ async function recordTake(browser, url, actions, paths, log, opts = {}) {
         const rect = await boxOf(step.selector);
         stepRect = rect;
         if (rect) {
-          await clickAt(rect);
+          await gesture(() => clickAt(rect));
           log(`click ${step.selector}`);
-          await sleep(500);
+          await gesture(() => sleep(settleMs(step)));
         } else
           skipped.push({ step: stepIdx, do: step.do, selector: step.selector });
         break;
@@ -6275,7 +6379,7 @@ async function recordTake(browser, url, actions, paths, log, opts = {}) {
         const rect = await boxOf(step.selector);
         stepRect = rect;
         if (rect) {
-          if (step.focus !== false) await clickAt(rect);
+          if (step.focus !== false) await gesture(() => clickAt(rect));
           const delay = step.delayMs ?? 40;
           const ping = () => emit({
             x: Math.round(rect.x + rect.w / 2),
@@ -6284,17 +6388,21 @@ async function recordTake(browser, url, actions, paths, log, opts = {}) {
             rect
           });
           let lastPing = -Infinity;
-          for (const ch of step.text) {
-            if (now() - lastPing >= 350) {
-              lastPing = now();
-              ping();
-            }
-            await page.keyboard.type(ch);
-            await sleep(delay);
-          }
+          await clockTyping(
+            [...step.text],
+            delay,
+            async (ch) => {
+              if (now() - lastPing >= 350) {
+                lastPing = now();
+                ping();
+              }
+              await page.keyboard.type(ch);
+            },
+            clock
+          );
           ping();
           log(`type "${step.text}" into ${step.selector}`);
-          await sleep(300);
+          await gesture(() => sleep(settleMs(step)));
         } else
           skipped.push({ step: stepIdx, do: step.do, selector: step.selector });
         break;
@@ -6307,7 +6415,7 @@ async function recordTake(browser, url, actions, paths, log, opts = {}) {
           emit({ x: Math.round(cur.x), y: Math.round(cur.y), type: "scroll" });
           await sleep(40);
         }
-        await sleep(400);
+        await gesture(() => sleep(settleMs(step)));
         break;
       }
       case "drag": {
@@ -6331,8 +6439,8 @@ async function recordTake(browser, url, actions, paths, log, opts = {}) {
           start = { x: step.x, y: step.y };
         }
         if (!start) break;
-        await moveTo(start.x, start.y);
-        await sleep(140);
+        await gesture(() => moveTo(start.x, start.y));
+        await gesture(() => sleep(PRESS_LEAD_MS));
         emit({
           x: Math.round(cur.x),
           y: Math.round(cur.y),
@@ -6341,19 +6449,20 @@ async function recordTake(browser, url, actions, paths, log, opts = {}) {
           rect
         });
         await page.mouse.down();
-        await sleep(90);
+        await gesture(() => sleep(PRESS_HOLD_MS));
         const dur = step.ms ?? 700;
-        const steps = Math.max(8, Math.round(dur / 16));
         const from = { ...cur };
-        for (let i = 1; i <= steps; i++) {
-          const u = easeInOutCubic(i / steps);
-          cur.x = from.x + (step.tx - from.x) * u;
-          cur.y = from.y + (step.ty - from.y) * u;
-          await page.mouse.move(cur.x, cur.y);
-          emit({ x: Math.round(cur.x), y: Math.round(cur.y), type: "move" });
-          await sleep(14);
-        }
-        await sleep(100);
+        await clockMotion(
+          dur,
+          async (u) => {
+            cur.x = from.x + (step.tx - from.x) * u;
+            cur.y = from.y + (step.ty - from.y) * u;
+            await page.mouse.move(cur.x, cur.y);
+            emit({ x: Math.round(cur.x), y: Math.round(cur.y), type: "move" });
+          },
+          clock
+        );
+        await gesture(() => sleep(settleMs({ do: "drag" })));
         await page.mouse.up();
         emit({
           x: Math.round(cur.x),
@@ -6388,8 +6497,15 @@ async function recordTake(browser, url, actions, paths, log, opts = {}) {
         }
       } : {}
     });
+    paces.push({
+      step: stepIdx,
+      do: step.do,
+      askedMs: askedMs(step),
+      gestureMs,
+      wallMs: now() - stepStart
+    });
   }
-  if (!capped) await sleep(600);
+  if (!capped) await sleep(TRAILING_HOLD_MS);
   if (capped || capReached(now(), maxSeconds)) {
     capped = true;
     log(`   ${cappedLine(maxSeconds)}`);
@@ -6441,10 +6557,13 @@ async function recordTake(browser, url, actions, paths, log, opts = {}) {
   await writeJson(paths.meta, meta, true);
   await writeJson(paths.framesIndex, frames);
   await writeJson(paths.actions, { ...actions, url }, true);
+  const pace = paceReport(paces);
+  log(`   ${paceLine(pace)}`);
   return {
     events,
     frames,
     meta,
+    pace,
     skipped,
     navTimeout,
     freezes,
@@ -9944,6 +10063,7 @@ async function cmdRecord(argv) {
         freezes: rec.freezes,
         freezePct: rec.freezePct,
         capped: rec.capped,
+        pace: rec.pace,
         ...strictFail ? { strictFailed: true } : {}
       },
       strictFail ? `STRICT: take recorded but incomplete \u2014 ${strictReason(rec)}; fix the flow and re-record.${skippedNote}` : `Take ready: ${outDir}
@@ -11119,4 +11239,4 @@ export {
   convertAgentBrowser,
   run
 };
-//# sourceMappingURL=chunk-6O55OD6D.js.map
+//# sourceMappingURL=chunk-43FCXWNS.js.map