demobites 1.32.0 → 1.34.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "demobites",
-  "version": "1.32.0",
+  "version": "1.34.0",
   "description": "The DemoBites agentic recorder — you prompt, it films a real browser, and DemoBites turns the take into an editable demo bite.",
   "bin": {
     "demobites": "launcher/index.mjs"

package/scripts/calibrate.mjs

@@ -90,7 +90,7 @@ function bboxCrossing(step) {
   return cross;
 }
 
-function roiChanges(bb) {
+function roiChanges(bb, around) {
   // ONSETS of visible change inside the padded bbox crop, in video time.
   // Not threshold crossings: hover styles animate (LinkedIn fades its row
   // background over ~150ms), and a threshold detector fires mid-fade —
@@ -101,14 +101,19 @@ function roiChanges(bb) {
   const pad = 6;
   const x0 = Math.max(0, bb.x - pad), y0 = Math.max(0, bb.y - pad);
   const w = Math.min(FRAME.w - x0, bb.w + 2 * pad), h = Math.min(FRAME.h - y0, bb.h + 2 * pad);
+  // ONLY THE WINDOW THAT MATTERS (2026-10-18: six anchors decoded the whole take six times, 48 s on the founder's
+  // Air). The caller keeps changes within [pred - 3.2, pred + 1.2]; the decode starts 3.6 s before the prediction and
+  // runs 5.2 s. Seeking the mp4 before the input is frame-accurate (decode from the keyframe, drop until the mark),
+  // and the reported pts restart at 0, so the window's start is added back.
+  const from = around == null ? 0 : Math.max(0, around - 3.6);
   const res = spawnSync(FFMPEG(), [
-    "-loglevel", "info", "-i", clean,
+    "-loglevel", "info", ...(around == null ? [] : ["-ss", String(from), "-t", "5.2"]), "-i", clean,
     "-vf", `crop=${w}:${h}:${x0}:${y0},select='gte(scene,0)',metadata=print`, "-f", "null", "-",
   ], { encoding: "utf8", maxBuffer: 256 * 1024 * 1024 });
   const err = `${res.stderr ?? ""}`;
   const rows = [];
   const re = /pts_time:([0-9.]+)[\s\S]*?lavfi\.scene_score=([0-9.]+)/g;
-  for (let m; (m = re.exec(err)); ) rows.push({ t: parseFloat(m[1]), s: parseFloat(m[2]) });
+  for (let m; (m = re.exec(err)); ) rows.push({ t: parseFloat(m[1]) + from, s: parseFloat(m[2]) });
   const onsets = [];
   for (let i = 1; i < rows.length; i++) {
     if (rows[i].s < 0.004) continue;               // not a peak-worthy change
@@ -126,7 +131,7 @@ for (const s of man.steps ?? []) {
   const wallCross = bboxCrossing(s);
   if (wallCross == null) continue;
   const pred = tb.a * wallCross + tb.b;
-  const changes = roiChanges(s.target.bbox).filter((c) => c >= pred - 3.2 && c <= pred + 1.2);
+  const changes = roiChanges(s.target.bbox, pred).filter((c) => c >= pred - 3.2 && c <= pred + 1.2);
   if (changes.length === 0) continue;
   anchors.push({ label: s.label ?? s.action, pred, changes, weight: 1 / Math.sqrt(changes.length) });
 }

package/scripts/trim.mjs

@@ -40,7 +40,7 @@ const clean = path.join(dir, "clean.mp4");
 // tail into a fake 1.027x, and a scene-change fit anchored on a modal that
 // took 0.49s to load tilted 0.897x. Both produced ramps of error that read as
 // a broken cursor.
-const r0 = man.record_from ?? 0;
+let r0 = man.record_from ?? 0;
 
 // Head-beacon phase 0: the identity law above fixes the RATE; the beacon
 // fixes the ANCHOR. record.mjs flashed the blank page full-frame at stamped
@@ -105,7 +105,36 @@ if ((man.beacon?.flips?.length ?? 0) >= 3) {
   }
 }
 
-const cutAt = Math.max(0, r0 - beaconDelta);
+let cutAt = Math.max(0, r0 - beaconDelta);
+
+// THE FIRST FRAME IS A LOADED PAGE (founder 2026-10-18, bite 1236: "you started without the In-app Pulse loaded…
+// after 3 seconds everything is loaded… use the trim at the start"). The camera's still-picture gate waits for the
+// outer page; an inner panel can keep loading after it. So the head of the clean take is scanned for visible change
+// in its first seconds, and when the picture kept changing, the cut moves to the last change plus a breath, and
+// record_from moves with it (the timebase stays video = wall - record_from). Bounded at 3.5 s; a still head is
+// untouched.
+const HEAD_SCAN_S = 4.0;
+const HEAD_MAX_SHIFT_S = 3.5;
+try {
+  const res = spawnSync(FFMPEG(), ["-hide_banner", "-loglevel", "info", "-i", raw, "-vf", `trim=start=${cutAt}:end=${cutAt + HEAD_SCAN_S},setpts=PTS-STARTPTS,scale=480:-2,select='gt(scene,0.02)',showinfo`, "-f", "null", "-"], { encoding: "utf8", maxBuffer: 64 * 1024 * 1024 });
+  const changes = [];
+  for (const m of `${res.stderr ?? ""}`.matchAll(/pts_time:([0-9.]+)/g)) changes.push(parseFloat(m[1]));
+  const late = changes.filter((t) => t > 0.3 && t < HEAD_SCAN_S - 0.3);
+  if (late.length) {
+    const shift = Math.min(HEAD_MAX_SHIFT_S, Math.round((late[late.length - 1] + 0.25) * 100) / 100);
+    console.log(`first frame: the picture kept changing for ${late[late.length - 1].toFixed(2)}s after the cut (${late.length} change${late.length === 1 ? "" : "s"}) — cutting ${shift.toFixed(2)}s later, to a loaded page`);
+    void postEvent("finishing", { text: "Cutting to the first loaded frame…" });
+    cutAt += shift;
+    r0 += shift;
+    man.record_from = r0;
+    man.first_frame_shift_s = shift;
+    // Nothing in the story happens before its first frame.
+    for (const st of man.steps ?? []) { if (Number(st.t_start) < r0) st.t_start = r0; if (Number(st.t_end) < r0) st.t_end = r0; }
+    for (const sh of man.shots ?? []) { if (Number(sh.t_start) < r0) sh.t_start = r0; }
+  } else console.log(`first frame: still (${changes.length} change${changes.length === 1 ? "" : "s"} in the first ${HEAD_SCAN_S}s, none after 0.3s)`);
+} catch (e) {
+  console.log(`first frame: scan skipped (${String(e?.message ?? e).split("\n")[0]})`);
+}
 
 // TAIL law (vanilla-Linux smoke 2026-09-09): raw.webm keeps receiving frames
 // while the browser context tears down — on a headless shell that was a
@@ -166,12 +195,23 @@ man.timebase = {
 fs.writeFileSync(manPath, JSON.stringify(man, null, 2));
 console.log(`timebase: video = (wall - ${r0.toFixed(3)})${speed === 1 ? "" : ` / ${speed}`}${beaconMethod ? ` (cut at raw ${cutAt.toFixed(3)}s)` : ""}`);
 
+// THE MACHINE'S OWN ENCODER (founder 2026-10-18: "saving the take… this should be 5 seconds"; the trim took 39.6 s on
+// his Air, most of it libx264 at preset medium). Apple's VideoToolbox encodes the same h264 several times faster;
+// elsewhere libx264 at a faster preset. The ingest wants h264 at 30 fps in an mp4, which both give.
+const ENCODER = (() => {
+  try {
+    const enc = execFileSync(FFMPEG(), ["-hide_banner", "-encoders"], { stdio: ["ignore", "pipe", "ignore"] }).toString();
+    if (process.platform === "darwin" && /\bh264_videotoolbox\b/.test(enc)) return ["-c:v", "h264_videotoolbox", "-b:v", "3M", "-maxrate", "4M", "-bufsize", "8M", "-profile:v", "high", "-allow_sw", "1", "-g", "60"];
+  } catch { /* libx264 */ }
+  return ["-c:v", "libx264", "-preset", "veryfast", "-crf", "19", "-g", "60"];
+})();
+console.log(`encoder: ${ENCODER[1]}`);
 execFileSync(FFMPEG(), [
   "-y", "-loglevel", "error",
   "-i", raw,
   "-filter_complex", `[0:v]trim=${tailCut ? `start=${cutAt}:end=${tailCut}` : `start=${cutAt}`},setpts=(PTS-STARTPTS)/${speed},fps=30,format=yuv420p[out]`,
   "-map", "[out]",
-  "-c:v", "libx264", "-preset", "medium", "-crf", "19",
+  ...ENCODER,
   "-movflags", "+faststart",
   clean,
 ], { stdio: "inherit" });
@@ -184,7 +224,7 @@ execFileSync(FFMPEG(), [
 try {
   const filters = execFileSync(FFMPEG(), ["-hide_banner", "-filters"], { stdio: ["ignore", "pipe", "ignore"] }).toString();
   if (/\bfreezeframes\b/.test(filters) && /\bsignalstats\b/.test(filters)) {
-    const stats = spawnSync(FFMPEG(), ["-hide_banner", "-loglevel", "error", "-i", clean, "-vf", "crop=iw/2:ih/2:iw/2:ih/2,signalstats,metadata=print:file=-", "-f", "null", "-"], { encoding: "utf8", maxBuffer: 64 * 1024 * 1024 });
+    const stats = spawnSync(FFMPEG(), ["-hide_banner", "-loglevel", "error", "-i", clean, "-vf", "crop=iw/2:ih/2:iw/2:ih/2,scale=240:-2,signalstats,metadata=print:file=-", "-f", "null", "-"], { encoding: "utf8", maxBuffer: 64 * 1024 * 1024 });
     const bad = [];
     let frame = -1, yavg = null, ymin = null, ymax = null;
     const flush = () => { if (frame >= 0 && yavg !== null && yavg >= 118 && yavg <= 138 && ymin !== null && ymax !== null && ymax - ymin <= 24) bad.push(frame); };
@@ -208,7 +248,7 @@ try {
       graph += `[a${chain.length}]format=yuv420p[out]`;
       const fixed = path.join(dir, "clean-fixed.mp4");
       void postEvent("finishing", { text: `Replacing ${bad.length} glitch frame${bad.length === 1 ? "" : "s"}…` });
-      execFileSync(FFMPEG(), ["-y", "-loglevel", "error", "-i", clean, "-filter_complex", graph, "-map", "[out]", "-c:v", "libx264", "-preset", "medium", "-crf", "19", "-movflags", "+faststart", fixed], { stdio: "inherit" });
+      execFileSync(FFMPEG(), ["-y", "-loglevel", "error", "-i", clean, "-filter_complex", graph, "-map", "[out]", ...ENCODER, "-movflags", "+faststart", fixed], { stdio: "inherit" });
       fs.renameSync(fixed, clean);
       man.glitch = { frames: bad.length, windows: chain.map((r) => ({ first: r.first, last: r.last })) };
       fs.writeFileSync(manPath, JSON.stringify(man, null, 2));

package/skill/scripts/plan.mjs

@@ -22,7 +22,7 @@ import { estimateTake } from "./clock.mjs";
 import path from "node:path";
 import { spawnSync } from "node:child_process";
 import { fileURLToPath } from "node:url";
-import { listening, postEvent } from "./listen-event.mjs";
+import { listening, postEvent, poll } from "./listen-event.mjs";
 
 const here = path.dirname(fileURLToPath(import.meta.url));
 const dir = process.argv[2];
@@ -218,7 +218,7 @@ const firstMessage = async () => {
     // storyboard is the base; the note says what to keep an eye on; the probe finds what moved.
     job.retake_of && job.storyboard
       ? `## This is a retake\nThe person wants the same demo filmed again, the same length and the same shots, because the app changed since. Their note: "${(job.retake_note || "no note").replace(/"/g, "'")}". ${job.storyboard.storyboard ? "Start from the previous storyboard below: keep its steps and its lines; probe it; fix only what no longer resolves or what the note names; then submit_storyboard. Never explore when the probe passes." : "Only the previous storyboard's lines survive (below): rebuild the same steps with the same lines, find the controls again, probe, then submit_storyboard."}\n\n## The previous storyboard\n${JSON.stringify(job.storyboard.storyboard ?? job.storyboard.steps ?? job.storyboard)}`
-      : codeOn ? `Write the storyboard from the page's own words, probe it, fix it, probe again, then submit_storyboard. Read a file only when the page's words do not name a control you need; never as a first move. You have at most 4 dry runs and 3 minutes from the first one; after that, hand in what passed and name in 'unsure' what you could not find.` : `There is no code: explore first (explore from the starting address, looking_for = the brief's nouns), then write the storyboard, probe it, fix it, probe again, then submit_storyboard. You have at most 4 dry runs and 3 minutes from the first one; after that, hand in what passed and name in 'unsure' what you could not find.`,
+      : codeOn ? `Write the storyboard from the page's own words, probe it, fix it, probe again, then submit_storyboard. Read a file only when the page's words do not name a control you need; never as a first move. You have three dry runs and two minutes from the first one; when a control of the brief is not where you looked, ask_person at once: the person is there and knows the app.` : `There is no code: explore first (explore from the starting address, looking_for = the brief's nouns), then write the storyboard, probe it, fix it, probe again, then submit_storyboard. You have three dry runs and two minutes from the first one; when a control of the brief is not where you looked, ask_person at once: the person is there and knows the app.`,
   ];
   return parts.join("\n\n");
 };
@@ -230,12 +230,40 @@ const messages = [{ role: "user", content: [{ type: "text", text: await firstMes
 // three minutes from the first one, whichever comes first. After that no probe runs; the next move is the storyboard
 // as it stands, with the steps it could not find named in `unsure`, so the person answers the question instead of
 // waiting while it rewrites and retries (that job: 18 dry runs, 13 model turns).
-const CAP_PROBES = 4;
-const CAP_MS = 3 * 60 * 1000;
+// THE PERSON LEADS (founder 2026-10-18: "if the planner gets stuck, he should put the customer into discussion…
+// I wouldn't go more than 2 minutes"): three dry runs, or two minutes from the first, and the planner ASKS — one
+// short question through the chat (ask_person); the answer comes back as the tool's result and the counters start
+// again. The person is there and knows their app; a storyboard guessed in minute six is worth less than a question
+// in minute two.
+const CAP_PROBES = 3;
+const CAP_MS = 2 * 60 * 1000;
 let probes = 0;
 let firstProbeAt = 0;
+let asked = 0;
 const capped = () => probes >= CAP_PROBES || (firstProbeAt && Date.now() - firstProbeAt > CAP_MS);
-const CAP_TEXT = () => `PLANNING CAP REACHED (${probes} dry run${probes === 1 ? "" : "s"}, ${Math.round((Date.now() - (firstProbeAt || Date.now())) / 1000)} s since the first). No more dry runs, no more reading. Hand in the storyboard NOW with submit_storyboard: keep every step that passed; a step that failed becomes a hover or a settle on the nearest element the page does have, or goes; write ONE short sentence in \`unsure\` naming what you could not find on the page (the step and the control), so the person can tell you where it is. The person is waiting.`;
+const CAP_TEXT = () => asked < 2
+  ? `STOP (${probes} dry run${probes === 1 ? "" : "s"}, ${Math.round((Date.now() - (firstProbeAt || Date.now())) / 1000)} s since the first). No more dry runs, no more reading. Call ask_person NOW with ONE short question in plain words: what you were looking for, where you looked, and how they want to play it (where it is, or what to show instead). The person is there and knows the app. Their answer comes back as the result; then fix the storyboard from their words, probe once, and hand it in.`
+  : `STOP. Hand in the storyboard NOW with submit_storyboard: keep every step that passed; a step that failed becomes a hover or a settle on the nearest element the page does have, or goes; name in \`unsure\` what you could not find.`;
+/** The question goes to the chat; the planner waits here for the person's word (up to 30 minutes). */
+const askPerson = async (text) => {
+  const q = String(text || "").trim().slice(0, 600);
+  if (!q) return "ask_person needs a question.";
+  say("Asking you in the chat…");
+  if (!listening) return "(no chat here: decide yourself and hand in the storyboard)";
+  await postEvent("question", { text: q });
+  const started = Date.now();
+  while (Date.now() - started < 30 * 60 * 1000) {
+    const st = await poll();
+    if (st?.status === "cancelled" || st?.status === "failed") { console.log("  The person cancelled."); process.exit(0); }
+    if (st && st.status !== "waiting_answer") {
+      const a = st.answer && typeof st.answer === "object" ? st.answer : null;
+      asked += 1; probes = 0; firstProbeAt = 0;
+      return a?.note ? `The person says: "${String(a.note).slice(0, 2000)}"` : "The person said to go on as you see fit.";
+    }
+    await new Promise((r) => setTimeout(r, 3000));
+  }
+  return "No answer came in 30 minutes: hand in the storyboard as it stands, with what you could not find in unsure.";
+};
 const plan = async () => {
   say("Writing the storyboard…");
   storyboard = null;
@@ -265,6 +293,7 @@ const plan = async () => {
         else if (u.name === "harvest_words") { say(`Reading the app's own words on ${(input.urls || []).length} page${(input.urls || []).length === 1 ? "" : "s"}…`); out = harvest(input.urls); }
         else if (u.name === "explore") { say(`Walking the app from ${where(input.from)}, looking for ${String(input.looking_for || "the brief's words").slice(0, 60)}…`); out = explore(input.from, input.looking_for, input.depth); }
         else if (u.name === "screenshot") { say(`Looking at ${where(input.url)}…`); out = screenshot(input.url); }
+        else if (u.name === "ask_person") { out = await askPerson(input.question); }
         else if (u.name === "probe") {
           if (capped()) out = CAP_TEXT();
           else {