@vosjs/studio-core 0.41.1 → 0.42.0

package/dist/index.js

@@ -1523,27 +1523,30 @@ function placeFreezes(rated, freezes) {
     acc += Math.max(0, p.out - p.in) / (p.rate ?? 1);
   };
   for (let i = 0; i < rated.length; i++) {
-    let piece = rated[i];
+    const piece = rated[i];
     const next = rated[i + 1];
     const continues = !!next && sameMedia(next.media, piece.media) && Math.abs(next.in - piece.out) < 1e-9 && next.out > piece.out;
     const mine = freezes.filter(
       (f) => f.seconds > 0 && !placed.has(f.id) && sameMedia(f.media, piece.media) && f.at >= piece.in - 1e-9 && (continues ? f.at < piece.out - 1e-9 : f.at <= piece.out + 1e-9)
     ).sort((a, b) => a.at - b.at);
+    const media = piece.media;
+    let rest = piece;
     for (const f of mine) {
-      if (f.at <= piece.in + 1e-9) {
-        freezePiece(f, piece.media);
-      } else if (f.at < piece.out - 1e-9) {
-        push2({ ...piece, out: f.at });
-        freezePiece(f, piece.media);
-        piece = { ...piece, in: f.at };
+      if (!rest) {
+        freezePiece(f, media);
+      } else if (f.at <= rest.in + 1e-9) {
+        freezePiece(f, media);
+      } else if (f.at < rest.out - 1e-9) {
+        push2({ ...rest, out: f.at });
+        freezePiece(f, media);
+        rest = { ...rest, in: f.at };
       } else {
-        const media = piece.media;
-        push2(piece);
-        piece = null;
+        push2(rest);
+        rest = null;
         freezePiece(f, media);
       }
     }
-    if (piece) push2(piece);
+    if (rest) push2(rest);
   }
   return { pieces, marks };
 }
@@ -3400,6 +3403,13 @@ var TYPING_CLICK_ABSORB = 1.5;
 var TYPING_REFOCUS_FRAC = 0.08;
 function groupTrack(track, opts) {
   const { width, height, clusterGap, typingGap, typingZoom, targetFill } = opts;
+  const stepOf = (t) => opts.steps?.find(
+    (st) => st.do !== "wait" && !st.skipped && st.tStart <= t && t <= st.tEnd
+  ) ?? null;
+  const sameStep = (a, b) => {
+    const sa = stepOf(a.t);
+    return sa !== null && sa === stepOf(b.t);
+  };
   const minLevel = opts.minLevel ?? resolveZoomStyle().minLevel;
   const clicks = dedupePresses(
     track.filter((e) => e.type === "down")
@@ -3433,7 +3443,7 @@ function groupTrack(track, opts) {
     for (const c of list) {
       const last = clusters.at(-1);
       const prev = last?.at(-1);
-      const joins = last && prev && c.t - prev.t <= clusterGap && (!spatial || unionFitLevel(unionRect([...last, c]), width, height) >= minLevel);
+      const joins = last && prev && (c.t - prev.t <= clusterGap || sameStep(prev, c)) && (!spatial || unionFitLevel(unionRect([...last, c]), width, height) >= minLevel);
       if (joins) last.push(c);
       else clusters.push([c]);
     }
@@ -3512,7 +3522,8 @@ function planAutoZoom(track, options) {
     typingGap,
     typingZoom,
     targetFill,
-    minLevel
+    minLevel,
+    steps: options.steps
   });
   const { sessions } = grouped;
   const drags = dragsFromTrack(track, width, height);
@@ -3627,10 +3638,11 @@ function planAutoZoom(track, options) {
   const spans = within([...zSpans, ...kSpans, ...dragSpans]).sort(
     (a, b) => a.in - b.in
   );
+  const firstStep = options.steps?.find((st) => !st.skipped)?.tStart;
   const dwells = dwellSpans(track, width, height, maxLevel, [
     ...spans,
     ...dragReserved
-  ]);
+  ]).filter((d) => firstStep === void 0 || d.out > firstStep);
   return within([...spans, ...dwells]).sort((a, b) => a.in - b.in);
 }
 function typingSessions(track, width, height, typingGap) {
@@ -9235,13 +9247,15 @@ var DEFAULT_SPEED_PARAMS = {
   typingMin: 3,
   typingRate: 3,
   scrollMin: 2.5,
-  scrollRate: 2
+  scrollRate: 2,
+  stillRate: 2
 };
 var TYPING_GAP = 1.5;
 var SCROLL_GAP = 0.8;
 var IDLE_PAD = 0.6;
 var MIN_SPAN = 1;
 var PLAYBACK_ACTIVITY = 0.1;
+var STILL_ACTIVITY = 0.02;
 function planAutoSpeed(track, opts) {
   const p = { ...DEFAULT_SPEED_PARAMS, ...opts.params };
   const durS = opts.durationMs / 1e3;
@@ -9260,6 +9274,9 @@ function planAutoSpeed(track, opts) {
     p.scrollMin,
     (start, last) => cands.push({ in: start, out: last, rate: p.scrollRate })
   );
+  for (const [a, b] of stillSteps(opts.steps, opts.activity)) {
+    if (b - a >= MIN_SPAN) cands.push({ in: a, out: b, rate: p.stillRate });
+  }
   for (const [a, b] of idleGaps(evs, durS, p.idleMin)) {
     if (isPlayback(opts.activity, a, b)) continue;
     const start = a + IDLE_PAD;
@@ -9313,6 +9330,29 @@ function idleGaps(track, durationS, idleMin = DEFAULT_SPEED_PARAMS.idleMin) {
   if (durationS - prev >= idleMin) gaps.push([prev, durationS]);
   return gaps;
 }
+function stillSteps(steps, activity) {
+  if (!steps?.length || !activity?.length) return [];
+  const live = steps.filter((st) => !st.skipped);
+  const doing = live.map((st) => st.do !== "wait");
+  const first = doing.indexOf(true);
+  const last = doing.lastIndexOf(true);
+  const out = [];
+  for (const [i, st] of live.entries()) {
+    if (st.do !== "wait" && st.do !== "hover") continue;
+    if (st.do === "wait" && (first < 0 || i < first || i > last)) continue;
+    if (meanActivity(activity, st.tStart, st.tEnd) >= STILL_ACTIVITY) continue;
+    out.push([st.tStart, st.tEnd]);
+  }
+  return out;
+}
+function meanActivity(activity, a, b) {
+  const lo = Math.max(0, Math.floor(a));
+  const hi = Math.min(activity.length, Math.ceil(b));
+  if (hi <= lo) return 0;
+  let sum = 0;
+  for (let i = lo; i < hi; i++) sum += activity[i];
+  return sum / (hi - lo);
+}
 function isPlayback(activity, a, b) {
   if (!activity?.length) return false;
   const lo = Math.max(0, Math.floor(a));
@@ -9351,12 +9391,14 @@ function planForDigest(doc, activity) {
     width: meta.width,
     height: meta.height,
     style: doc.zoomStyle,
-    params: doc.zoomParams
+    params: doc.zoomParams,
+    steps: meta.steps
   });
   const speed = planAutoSpeed(cursor, {
     durationMs: meta.durationMs,
     params: doc.speedParams,
-    activity
+    activity,
+    steps: meta.steps
   });
   const style = resolveZoomStyle(doc.zoomStyle, doc.zoomParams);
   const intensity = doc.tiltStyle ?? style.tilt.intensity;
@@ -9396,7 +9438,8 @@ function momentsFromDoc(doc, plan, opts = {}) {
       typingGap: style.typingGap,
       typingZoom: style.typingZoom,
       targetFill: style.targetFill,
-      minLevel: style.minLevel
+      minLevel: style.minLevel,
+      steps: meta.steps
     });
     const pressed = (c, surface) => {
       const f = clusterFocus(c, width, height);
@@ -9492,15 +9535,30 @@ function momentsFromDoc(doc, plan, opts = {}) {
       rect: null
     });
   }
+  for (const c of opts.changes ?? []) {
+    if (!(c.at >= 0 && c.at < dur)) continue;
+    const peak = Math.min(dur, Math.max(c.at, c.peakAt));
+    drafts.push({
+      kind: "changed",
+      in: c.at,
+      out: peak,
+      at: peak,
+      focus: { cx: c.rect.x + c.rect.w / 2, cy: c.rect.y + c.rect.h / 2 },
+      rect: c.rect,
+      seedAt: c.at,
+      fraction: c.fraction
+    });
+  }
   const order = {
     head: 0,
     click: 1,
     typing: 2,
-    scroll: 3,
-    dwell: 4,
-    scene: 5,
-    idle: 6,
-    tail: 7
+    changed: 3,
+    scroll: 4,
+    dwell: 5,
+    scene: 6,
+    idle: 7,
+    tail: 8
   };
   drafts.sort((a, b) => a.in - b.in || order[a.kind] - order[b.kind]);
   const rated = ratedSegments(doc);
@@ -9509,8 +9567,17 @@ function momentsFromDoc(doc, plan, opts = {}) {
     return spanOutputExtent(rated, lo, Math.max(b, lo + 1e-3));
   };
   const covers = (s, d) => d.in === d.out ? s.in <= d.in && d.in < s.out : s.in < d.out && s.out > d.in;
+  const ids = drafts.map((_, i) => `m${String(i + 1).padStart(2, "0")}`);
+  const seedOf = (d) => {
+    if (d.kind !== "changed" || d.seedAt === void 0) return void 0;
+    const j = drafts.findIndex(
+      (x) => x.kind === "click" && x.at === d.seedAt || x.kind === "typing" && x.out === d.seedAt
+    );
+    return j >= 0 ? ids[j] : void 0;
+  };
   return drafts.map((d, i) => {
     const output = outputOf(d.in, d.out);
+    const seed = seedOf(d);
     const outAt = d.at === null ? null : outputOf(d.at, d.at);
     const proposed = {};
     const z = plan.zoom.find((s) => covers(s, d));
@@ -9520,7 +9587,7 @@ function momentsFromDoc(doc, plan, opts = {}) {
     const tl = plan.tilt.find((s) => covers(s, d));
     if (tl) proposed.tilt = tl.id;
     return {
-      id: `m${String(i + 1).padStart(2, "0")}`,
+      id: ids[i],
       kind: d.kind,
       source: { in: round10(d.in), out: round10(d.out) },
       output: output ? { in: round10(output.start), out: round10(output.end) } : null,
@@ -9536,9 +9603,10 @@ function momentsFromDoc(doc, plan, opts = {}) {
       ...d.clicks !== void 0 ? { clicks: d.clicks } : {},
       ...d.surface ? { surface: true } : {},
       ...d.pings !== void 0 ? { pings: d.pings } : {},
-      activity: activityOf(opts.bins, d),
+      activity: d.kind === "changed" && d.fraction !== void 0 ? round10(d.fraction) : activityOf(opts.bins, d),
       proposed,
-      said: saidOver(opts.transcript, d)
+      said: saidOver(opts.transcript, d),
+      ...seed ? { seed } : {}
     };
   });
 }
@@ -9565,6 +9633,105 @@ function round10(v) {
   return Math.round(v * 1e3) / 1e3;
 }
 
+// src/digest/changes.ts
+var CHANGE_GRID = { w: 128, h: 72 };
+var BLOCK_FILL = 0.5;
+var CHANGE_FLOOR = 0.02;
+var CHANGE_OFFSETS = [0.3, 0.8, 1.5, 2.5];
+var CHANGE_LEAD = 0.5;
+function changedRegion(before, after, delta, minBlocks = 1, ambient = null, block = Math.max(1, Math.round(before.w / 16))) {
+  const { w, h } = before;
+  if (after.w !== w || after.h !== h || w <= 0 || h <= 0) return null;
+  const cols = Math.ceil(w / block);
+  const rows = Math.ceil(h / block);
+  const moved = new Uint16Array(cols * rows);
+  const cells = new Uint16Array(cols * rows);
+  for (let y = 0; y < h; y++) {
+    const by = Math.floor(y / block);
+    for (let x = 0; x < w; x++) {
+      const i = y * w + x;
+      const b = by * cols + Math.floor(x / block);
+      cells[b]++;
+      const drift = ambient ? ambient.rate[i] * ambient.dt : 0;
+      if (Math.abs(after.luma[i] - before.luma[i]) - drift > delta) moved[b]++;
+    }
+  }
+  const filled = new Uint8Array(cols * rows);
+  for (let b = 0; b < filled.length; b++)
+    if (moved[b] >= cells[b] * BLOCK_FILL) filled[b] = 1;
+  const seen = new Uint8Array(cols * rows);
+  let best = null;
+  for (let start = 0; start < filled.length; start++) {
+    if (!filled[start] || seen[start]) continue;
+    const stack = [start];
+    seen[start] = 1;
+    let size = 0;
+    let minX = cols;
+    let minY = rows;
+    let maxX = -1;
+    let maxY = -1;
+    while (stack.length) {
+      const b = stack.pop();
+      size++;
+      const bx = b % cols;
+      const by = Math.floor(b / cols);
+      if (bx < minX) minX = bx;
+      if (bx > maxX) maxX = bx;
+      if (by < minY) minY = by;
+      if (by > maxY) maxY = by;
+      for (const [dx, dy] of [
+        [1, 0],
+        [-1, 0],
+        [0, 1],
+        [0, -1]
+      ]) {
+        const nx = bx + dx;
+        const ny = by + dy;
+        if (nx < 0 || ny < 0 || nx >= cols || ny >= rows) continue;
+        const n = ny * cols + nx;
+        if (filled[n] && !seen[n]) {
+          seen[n] = 1;
+          stack.push(n);
+        }
+      }
+    }
+    if (!best || size > best.size) best = { size, minX, minY, maxX, maxY };
+  }
+  if (!best || best.size < minBlocks) return null;
+  return {
+    fraction: best.size / (cols * rows),
+    rect: {
+      x: best.minX * block / w,
+      y: best.minY * block / h,
+      w: Math.min(w, (best.maxX + 1) * block) / w - best.minX * block / w,
+      h: Math.min(h, (best.maxY + 1) * block) / h - best.minY * block / h
+    }
+  };
+}
+function ambientDrift(earlier, before, lead) {
+  const { w, h } = before;
+  if (earlier.w !== w || earlier.h !== h || !(lead > 0)) return null;
+  const rate = new Array(w * h);
+  for (let i = 0; i < rate.length; i++)
+    rate[i] = Math.abs(before.luma[i] - earlier.luma[i]) / lead;
+  return rate;
+}
+function pickChange(seed, samples, floor = CHANGE_FLOOR) {
+  let best = null;
+  for (const s of samples) {
+    if (!s.region || s.region.fraction < floor) continue;
+    if (!best || s.region.fraction > (best.region?.fraction ?? 0)) best = s;
+  }
+  if (!best?.region) return null;
+  return {
+    seed: seed.id,
+    at: seed.at,
+    peakAt: seed.at + best.dt,
+    fraction: best.region.fraction,
+    rect: best.region.rect
+  };
+}
+
 // src/digest/scenes.ts
 var SCENE_MOTION = 0.25;
 var SCENE_QUIET = 0.1;
@@ -10096,6 +10263,7 @@ function buildDigest(input) {
       activity: "fraction of pixels that changed, per SOURCE second"
     },
     moments: withFiles,
+    steps: digestSteps(doc),
     activity: input.bins,
     plan: input.plan,
     doc: {
@@ -10119,6 +10287,26 @@ function buildDigest(input) {
     }
   };
 }
+function digestSteps(doc) {
+  const steps = doc.source.meta.steps ?? [];
+  if (!steps.length) return [];
+  const rated = ratedSegments(doc);
+  return steps.map((s) => {
+    const out = spanOutputExtent(rated, s.tStart, s.tEnd);
+    return {
+      step: s.step,
+      id: s.id ?? null,
+      do: s.do,
+      selector: s.selector ?? null,
+      source: { in: round11(s.tStart), out: round11(s.tEnd) },
+      output: out ? { in: round11(out.start), out: round11(out.end) } : null,
+      skipped: s.skipped === true,
+      navigated: s.navigated === true,
+      url: s.url ?? null,
+      text: s.text ?? null
+    };
+  });
+}
 function outputDurationOf(doc) {
   return docOutputDuration(doc);
 }
@@ -11953,6 +12141,10 @@ export {
   CARD_FADE_SECONDS,
   CARD_FOV,
   CARD_Z,
+  CHANGE_FLOOR,
+  CHANGE_GRID,
+  CHANGE_LEAD,
+  CHANGE_OFFSETS,
   CHANNEL_SPECS_HASH,
   CHANNEL_SPECS_VERIFIED,
   CLICK_FX_INTENSITY,
@@ -12102,6 +12294,7 @@ export {
   ZOOM_SPAN_MIN,
   ZOOM_STYLES,
   ZOOM_STYLE_OPTIONS,
+  ambientDrift,
   anchorKindOf,
   anchorSourceDuration,
   animStep,
@@ -12134,6 +12327,7 @@ export {
   cardPoseTrack,
   caretAt,
   caretStops,
+  changedRegion,
   clampCamFrac,
   clampCamSize,
   clampFocus,
@@ -12286,6 +12480,7 @@ export {
   pageDisplayUrl,
   parseCssColour,
   parsePoseId,
+  pickChange,
   pickStyle,
   pinBox,
   pinCandidates,