@mineo71/showtime 0.3.0 → 0.4.0

package/src/timeline/build.ts

@@ -1,9 +1,12 @@
 import type { Script, Step } from '../script/schema';
 import { buildCamera, cameraAt, MIN_ZOOM, zoomDepth, type ZoomRegion } from './camera';
-import { BROWSER_BAR, projectRect, windowLayout, type Box } from './layout';
+import { BROWSER_BAR, OUTPUT_SIZES, projectRect, viewLayout, visibleFraction, type View } from './layout';
 import { clamp } from './math';
-import { buildSegments, type SpeedSpan } from './timemap';
-import type { CameraKey, Caption, Rect, Spotlight, StepEvent, TakeEvents, Timeline } from './types';
+import { RIPPLE_MS } from './cursor';
+import { CAPTION_FADE_MS, SPOTLIGHT_FADE_MS } from './fades';
+import { soundEvents, stepKey, subtitleCues, withNarration, type Span, type Typing } from './narration';
+import { buildSegments, sourceToOutput, type SpeedSpan } from './timemap';
+import type { Aspect, CameraKey, Caption, NarrationLine, Rect, Spotlight, StepEvent, TakeEvents, Timeline } from './types';
 
 export const AUTO_SPEED_MIN_IDLE_MS = 800;
 export const AUTO_SPEED_TARGET_MS = 600;
@@ -13,15 +16,17 @@ export const SPOTLIGHT_PAD = 8;
 export const OUTRO_MS = 2600;
 /** Bottom captions own the output below this fraction of its height. */
 export const CAPTION_BAND = 0.78;
-const DEFAULT_OUTPUT = { width: 1920, height: 1080 };
 
 const ZOOMABLE = new Set<Step['do']>(['click', 'type', 'select', 'hover']);
 
 export interface BuildOptions {
   fps?: number;
-  output?: { width: number; height: number };
+  /** 16:9 by default; 9:16 and 1:1 crop the viewport and follow the action. */
+  aspect?: Aspect;
   /** Logo as a data URI. */
   logo?: string;
+  /** Length of each synthesized `show.say` line, by step (`scene#n`); lines without one are left out. */
+  voices?: Record<string, number>;
 }
 
 function stepOf(script: Script, e: StepEvent): Step {
@@ -46,7 +51,7 @@ function captionPosition(
   c: { startMs: number; endMs: number },
   camera: CameraKey[],
   stage: { width: number; height: number },
-  win: Box,
+  win: View,
   output: { width: number; height: number },
 ): Caption['position'] {
   if (!target) return 'bottom';
@@ -63,7 +68,13 @@ const pad = (r: Rect, p: number): Rect => ({ x: r.x - p, y: r.y - p, width: r.wi
 export function buildTimeline(script: Script, ev: TakeEvents, opts: BuildOptions = {}): Timeline {
   const { style } = script;
   const stage = ev.viewport;
-  const output = opts.output ?? DEFAULT_OUTPUT;
+  const aspect = opts.aspect ?? '16:9';
+  const output = OUTPUT_SIZES[aspect];
+  const win = viewLayout(output, stage, style.padding, style.chrome === 'browser' ? BROWSER_BAR : 0, aspect);
+  const visible = visibleFraction(win);
+  // Cropped formats follow every target; a zoom only adds what the crop doesn't already magnify.
+  const crop = aspect !== '16:9';
+  const cropDepth = Math.min(visible.x, visible.y);
   const pairs = ev.steps.map((e) => ({ e, step: stepOf(script, e) }));
 
   // Speed: auto idle first, explicit `show.speed` next, pauses last (later spans win).
@@ -87,9 +98,10 @@ export function buildTimeline(script: Script, ev: TakeEvents, opts: BuildOptions
   for (const { e, step } of pairs) {
     const zoom = step.show?.zoom;
     const wantsZoom = typeof zoom === 'number' || (zoom === undefined && style.autoZoom && ZOOMABLE.has(step.do));
-    if (e.target && wantsZoom) {
-      const scale = typeof zoom === 'number' ? zoom : zoomDepth(e.target, stage, style.maxZoom);
-      if (scale >= MIN_ZOOM) {
+    if (e.target && (wantsZoom || crop)) {
+      const depth = wantsZoom ? (typeof zoom === 'number' ? zoom : zoomDepth(e.target, stage, style.maxZoom)) : 1;
+      const scale = crop ? Math.max(1, depth * cropDepth) : depth;
+      if (scale >= MIN_ZOOM || crop) {
         regions.push({
           startMs: e.actionMs,
           endMs: e.settledMs,
@@ -107,8 +119,14 @@ export function buildTimeline(script: Script, ev: TakeEvents, opts: BuildOptions
     }
   }
 
-  const camera = buildCamera(regions);
-  const win = windowLayout(output, stage, style.padding, style.chrome === 'browser' ? BROWSER_BAR : 0);
+  const lines = pairs.flatMap(({ e, step }) => {
+    const where = stepKey(e.sceneId, e.index);
+    const durationMs = opts.voices?.[where];
+    return step.show?.say && durationMs !== undefined ? [{ where, text: step.show.say, startMs: e.startMs, durationMs }] : [];
+  });
+  const subtitlesBurned = script.audio.subtitles === 'burn' && lines.length > 0;
+
+  const camera = buildCamera(regions, { visible, hold: crop });
   // A caption ends when the next one starts: no overlap, so captions dip instead of colliding.
   drafts.sort((a, b) => a.startMs - b.startMs);
   drafts.forEach((d, i) => {
@@ -117,14 +135,50 @@ export function buildTimeline(script: Script, ev: TakeEvents, opts: BuildOptions
   });
   const captions: Caption[] = drafts
     .filter((d) => d.endMs > d.startMs)
-    .map((d) => ({ startMs: d.startMs, endMs: d.endMs, text: d.text, position: captionPosition(d.target, d, camera, stage, win, output) }));
+    .map((d) => ({
+      startMs: d.startMs,
+      endMs: d.endMs,
+      text: d.text,
+      // 9:16 captions sit in the band under the window, clear of the action.
+      position: subtitlesBurned ? 'top' : aspect === '9:16' ? 'bottom' : captionPosition(d.target, d, camera, stage, win, output),
+    }));
+
+  // Narration holds freeze the frame, so they go where nothing is moving: not during an action,
+  // a cursor move, a ripple, a camera move or a fade.
+  const fades = (x: { startMs: number; endMs: number }, ms: number) => [
+    { startMs: x.startMs, endMs: x.startMs + ms },
+    { startMs: x.endMs - ms, endMs: x.endMs },
+  ];
+  const busy: Span[] = [
+    ...pairs.filter(({ step }) => step.do !== 'pause').map(({ e }) => ({ startMs: e.startMs, endMs: e.endMs })),
+    ...ev.moves,
+    ...ev.clicks.map((c) => ({ startMs: c.tMs, endMs: c.tMs + RIPPLE_MS })),
+    ...camera.slice(1).flatMap((b, i) => {
+      const a = camera[i];
+      return a.scale !== b.scale || a.cx !== b.cx || a.cy !== b.cy ? [{ startMs: a.tMs, endMs: b.tMs }] : [];
+    }),
+    ...captions.flatMap((c) => fades(c, CAPTION_FADE_MS)),
+    ...spotlights.flatMap((s) => fades(s, SPOTLIGHT_FADE_MS)),
+  ];
+  const segments = withNarration(buildSegments(ev.video.durationMs, spans), lines, ev.video.durationMs, busy);
+  const narration: NarrationLine[] = lines.map((l) => ({
+    where: l.where,
+    text: l.text,
+    atMs: Math.round(sourceToOutput(segments, l.startMs)),
+    durationMs: l.durationMs,
+  }));
+  const subtitles = script.audio.subtitles === 'off' ? [] : narration.flatMap((n) => subtitleCues(n.text, n.atMs, n.durationMs));
+  const typing: Typing[] = pairs.flatMap(({ e, step }) =>
+    step.do === 'type' && step.pace !== 'fast' ? [{ startMs: e.actionMs, endMs: e.endMs, chars: [...step.text].length }] : [],
+  );
 
   return {
     version: 1,
     fps: opts.fps ?? 30,
+    aspect,
     output,
     stage,
-    video: { src: ev.video.file, durationMs: ev.video.durationMs },
+    video: { src: `take/${ev.video.file}`, durationMs: ev.video.durationMs },
     intro: script.meta.title ? { title: script.meta.title, subtitle: script.meta.subtitle, durationMs: INTRO_MS } : null,
     chrome:
       style.chrome === 'browser'
@@ -132,12 +186,18 @@ export function buildTimeline(script: Script, ev: TakeEvents, opts: BuildOptions
         : null,
     outro: script.meta.outro ? { ...script.meta.outro, durationMs: OUTRO_MS } : null,
     logo: opts.logo ?? null,
-    segments: buildSegments(ev.video.durationMs, spans),
+    segments,
     camera,
     moves: ev.moves,
     clicks: ev.clicks,
     captions,
     spotlights,
+    narration,
+    subtitles,
+    subtitlesBurned,
+    sounds: script.audio.sounds ? soundEvents(ev, typing, segments) : [],
+    music: script.audio.music ? { volume: script.audio.music.volume } : null,
+    audio: null,
     style,
   };
 }

package/src/timeline/camera.ts

@@ -19,14 +19,33 @@ export function zoomDepth(target: Rect, stage: { width: number; height: number }
   return clamp(fit, 1, maxZoom);
 }
 
-export function clampFocus(scale: number, cx: number, cy: number): { cx: number; cy: number } {
-  const half = 0.5 / scale;
-  return { cx: clamp(cx, half, 1 - half), cy: clamp(cy, half, 1 - half) };
+/** Share of the content visible at scale 1, per axis: 1 unless the format crops the viewport. */
+export interface Visible {
+  x: number;
+  y: number;
+}
+
+const FULL: Visible = { x: 1, y: 1 };
+
+/** Keep the focus where the visible part stays inside the content. */
+export function clampFocus(scale: number, cx: number, cy: number, visible: Visible = FULL): { cx: number; cy: number } {
+  const hx = Math.min(0.5, visible.x / (2 * scale));
+  const hy = Math.min(0.5, visible.y / (2 * scale));
+  return { cx: clamp(cx, hx, 1 - hx), cy: clamp(cy, hy, 1 - hy) };
 }
 
 const rest = (tMs: number): CameraKey => ({ tMs, scale: 1, cx: 0.5, cy: 0.5 });
 
-export function buildCamera(regions: ZoomRegion[]): CameraKey[] {
+export interface CameraOptions {
+  visible?: Visible;
+  /** Between regions keep the last focus (at scale 1) instead of returning to the centre: the follow camera of cropped formats. */
+  hold?: boolean;
+}
+
+export function buildCamera(regions: ZoomRegion[], opts: CameraOptions = {}): CameraKey[] {
+  const visible = opts.visible ?? FULL;
+  let last = { cx: 0.5, cy: 0.5 };
+  const restAt = (tMs: number): CameraKey => (opts.hold ? { tMs, scale: 1, ...clampFocus(1, last.cx, last.cy, visible) } : rest(tMs));
   const rs = [...regions].sort((a, b) => a.startMs - b.startMs);
   const keys: CameraKey[] = [];
   rs.forEach((r, i) => {
@@ -34,14 +53,15 @@ export function buildCamera(regions: ZoomRegion[]): CameraKey[] {
     const next = rs[i + 1];
     const start = prev ? Math.max(r.startMs, prev.endMs) : r.startMs;
     const end = Math.max(start, r.endMs);
-    const focus = clampFocus(r.scale, r.cx, r.cy);
+    const focus = clampFocus(r.scale, r.cx, r.cy, visible);
     if (!prev || start - prev.endMs > CONNECT_GAP_MS) {
       const lastT = keys.length ? keys[keys.length - 1].tMs : 0;
-      keys.push(rest(Math.max(lastT, start - ZOOM_LEAD_MS)));
+      keys.push(restAt(Math.max(lastT, start - ZOOM_LEAD_MS)));
     }
     keys.push({ tMs: start, scale: r.scale, ...focus });
     keys.push({ tMs: end, scale: r.scale, ...focus });
-    if (!next || next.startMs - end > CONNECT_GAP_MS) keys.push(rest(end + ZOOM_LEAD_MS));
+    last = focus;
+    if (!next || next.startMs - end > CONNECT_GAP_MS) keys.push(restAt(end + ZOOM_LEAD_MS));
   });
   return keys;
 }
@@ -60,10 +80,13 @@ export function cameraAt(keys: CameraKey[], tMs: number): CameraKey {
   return { ...keys[keys.length - 1], tMs };
 }
 
-/** CSS transform (transform-origin 0 0) that puts the focus point at the centre of a width×height box. */
-export function cameraTransform(cam: CameraKey, width: number, height: number): string {
-  const tx = width / 2 - cam.cx * width * cam.scale;
-  const ty = height / 2 - cam.cy * height * cam.scale;
+/**
+ * CSS transform (transform-origin 0 0) for content of `content` size that puts the focus point at
+ * the centre of a width×height window (content defaults to the window's size).
+ */
+export function cameraTransform(cam: CameraKey, width: number, height: number, content = { width, height }): string {
+  const tx = width / 2 - cam.cx * content.width * cam.scale;
+  const ty = height / 2 - cam.cy * content.height * cam.scale;
   const r = (v: number) => Math.round(v * 100) / 100 + 0;
   return `translate(${r(tx)}px, ${r(ty)}px) scale(${r(cam.scale * 1000) / 1000})`;
 }

package/src/timeline/fades.ts

@@ -0,0 +1,3 @@
+/** Fade lengths shared by the renderer and the timeline (holds must not freeze a fade). */
+export const CAPTION_FADE_MS = 220;
+export const SPOTLIGHT_FADE_MS = 250;

package/src/timeline/frames.ts

@@ -1,4 +1,4 @@
-import { outputDurationMs, outputToSource, sourceToOutput } from './timemap';
+import { outputDurationMs, outputToSource, segLength, sourceToOutput } from './timemap';
 import type { StepEvent, Timeline } from './types';
 
 export interface SegmentPlan {
@@ -6,6 +6,7 @@ export interface SegmentPlan {
   durationInFrames: number;
   /** source frame where this segment starts */
   trimBefore: number;
+  /** 0: a hold, showing the `trimBefore` frame throughout */
   speed: number;
 }
 
@@ -39,7 +40,7 @@ export function segmentPlan(t: Timeline): SegmentPlan[] {
   let outMs = 0;
   for (const s of t.segments) {
     const startF = toFrames(outMs, t.fps);
-    outMs += (s.srcEndMs - s.srcStartMs) / s.speed;
+    outMs += segLength(s);
     const endF = toFrames(outMs, t.fps);
     if (endF > startF) {
       plan.push({ fromFrame: intro + startF, durationInFrames: endF - startF, trimBefore: toFrames(s.srcStartMs, t.fps), speed: s.speed });

package/src/timeline/layout.ts

@@ -1,4 +1,4 @@
-import type { CameraKey, Rect } from './types';
+import type { Aspect, CameraKey, Rect } from './types';
 
 export interface Box {
   x: number;
@@ -10,6 +10,21 @@ export interface Box {
 /** Height of the browser bar, in stage (viewport) pixels. */
 export const BROWSER_BAR = 44;
 
+export const OUTPUT_SIZES: Record<Aspect, { width: number; height: number }> = {
+  '16:9': { width: 1920, height: 1080 },
+  '9:16': { width: 1080, height: 1920 },
+  '1:1': { width: 1080, height: 1080 },
+};
+
+/** In 9:16 the window (bar included) is this many times taller than wide. */
+const PORTRAIT_WINDOW = 1.3;
+
+/** The window on screen, and the size the viewport is drawn at inside it (larger when cropped). */
+export interface View extends Box {
+  bar: number;
+  content: { width: number; height: number };
+}
+
 /**
  * Where the recorded viewport sits inside the output frame. `bar` (stage px) reserves room for a
  * browser bar above it; the returned box is the viewport, `bar` the bar height in output px.
@@ -30,6 +45,40 @@ export function windowLayout(
   return { x: Math.round((output.width - width) / 2), y: top + barPx, width, height, bar: barPx };
 }
 
+/**
+ * The window for an aspect ratio. 16:9 fits the whole viewport (the 0.3 layout). 1:1 fills the
+ * padded square and 9:16 uses a tall window in the middle; in both the viewport covers the window,
+ * so it is cropped and the camera decides what shows.
+ */
+export function viewLayout(
+  output: { width: number; height: number },
+  stage: { width: number; height: number },
+  padding: number,
+  bar = 0,
+  aspect: Aspect = '16:9',
+): View {
+  if (aspect === '16:9') {
+    const w = windowLayout(output, stage, padding, bar);
+    return { ...w, content: { width: w.width, height: w.height } };
+  }
+  const width = Math.round(output.width * (1 - 2 * padding));
+  const total = aspect === '9:16' ? Math.round(width * PORTRAIT_WINDOW) : Math.round(output.height * (1 - 2 * padding));
+  const k = Math.max(width / stage.width, total / (stage.height + bar));
+  const barPx = Math.round(bar * k);
+  const top = Math.round((output.height - total) / 2);
+  return {
+    x: Math.round((output.width - width) / 2),
+    y: top + barPx,
+    width,
+    height: total - barPx,
+    bar: barPx,
+    content: { width: Math.round(stage.width * k), height: Math.round(stage.height * k) },
+  };
+}
+
+/** How much of the content shows at camera scale 1, per axis (1 = all of it). */
+export const visibleFraction = (v: View) => ({ x: Math.min(1, v.width / v.content.width), y: Math.min(1, v.height / v.content.height) });
+
 const r2 = (v: number) => Math.round(v * 100) / 100;
 
 /** Where a stage rect (CSS px) lands in the output: through the camera, inside the window viewport. */
@@ -37,10 +86,10 @@ export function projectRect(
   r: Rect,
   cam: Pick<CameraKey, 'scale' | 'cx' | 'cy'>,
   stage: { width: number; height: number },
-  win: Box,
+  win: Box & { content?: { width: number; height: number } },
 ): Rect {
-  const w = win.width * cam.scale;
-  const h = win.height * cam.scale;
+  const w = (win.content?.width ?? win.width) * cam.scale;
+  const h = (win.content?.height ?? win.height) * cam.scale;
   return {
     x: r2(win.x + win.width / 2 + (r.x / stage.width - cam.cx) * w),
     y: r2(win.y + win.height / 2 + (r.y / stage.height - cam.cy) * h),

package/src/timeline/narration.ts

@@ -0,0 +1,160 @@
+import { insertHolds, sourceToOutput, type Hold } from './timemap';
+import type { Click, Cue, Segment, Sound } from './types';
+
+export type { Cue, Sound };
+
+/** Silence after a line before the next narrated step starts. */
+export const NARRATION_GAP_MS = 400;
+/** Two subtitle lines of 42 characters. */
+export const CUE_MAX_CHARS = 84;
+const LINE_CHARS = 42;
+/** Keystroke sounds closer than this (output time) are dropped, so sped-up typing doesn't buzz. */
+export const KEY_MIN_GAP_MS = 40;
+
+/** `scene#n`, 1-based: how lint, errors and the voice cache name a step. */
+export const stepKey = (sceneId: string, index: number) => `${sceneId}#${index + 1}`;
+
+export interface Line {
+  /** source ms the step starts */
+  startMs: number;
+  durationMs: number;
+}
+
+/** Something moving on screen over (startMs, endMs), source ms. */
+export interface Span {
+  startMs: number;
+  endMs: number;
+}
+
+/** The latest time in [lo, hi] that no span is moving through, or `hi` when there is none. */
+export function stillPoint(lo: number, hi: number, busy: Span[]): number {
+  let t = hi;
+  for (;;) {
+    const moving = busy.find((b) => b.startMs < t && t < b.endMs);
+    if (!moving) return t;
+    t = moving.startMs;
+    if (t < lo) return hi;
+  }
+}
+
+/**
+ * Holds that let every line finish (plus a gap) before the next narrated step starts, or before
+ * the video ends. `lines` are in source order; each hold sits at the last moment of its window
+ * when nothing in `busy` is moving, so the freeze never stops a camera move or a fade halfway.
+ */
+export function narrationHolds(lines: Line[], segments: Segment[], endMs: number, busy: Span[] = []): Hold[] {
+  const holds: Hold[] = [];
+  lines.forEach((line, i) => {
+    const next = lines[i + 1];
+    const until = next ? next.startMs : endMs;
+    const available = sourceToOutput(segments, until) - sourceToOutput(segments, line.startMs);
+    const short = line.durationMs + NARRATION_GAP_MS - available;
+    if (short <= 0) return;
+    // After the line's own start (a hold at its start would delay the line itself).
+    const lo = line.startMs + 1;
+    const hi = Math.max(lo, next ? next.startMs - 1 : endMs);
+    holds.push({ atMs: stillPoint(lo, hi, busy), holdMs: Math.round(short) });
+  });
+  return holds;
+}
+
+/** Segments with the narration holds in place. */
+export const withNarration = (segments: Segment[], lines: Line[], endMs: number, busy: Span[] = []) =>
+  insertHolds(segments, narrationHolds(lines, segments, endMs, busy));
+
+/** Greedily join pieces with spaces into chunks of at most CUE_MAX_CHARS. */
+function pack(pieces: string[]): string[] {
+  const out: string[] = [];
+  let cur = '';
+  for (const p of pieces) {
+    if (!cur) cur = p;
+    else if (cur.length + 1 + p.length <= CUE_MAX_CHARS) cur += ` ${p}`;
+    else {
+      out.push(cur);
+      cur = p;
+    }
+  }
+  if (cur) out.push(cur);
+  return out;
+}
+
+/** A sentence as cue texts: whole if it fits, else at commas, else between words. */
+function sentenceChunks(sentence: string): string[] {
+  if (sentence.length <= CUE_MAX_CHARS) return [sentence];
+  const out: string[] = [];
+  let clauses: string[] = [];
+  for (const clause of sentence.split(/(?<=[,;:])\s+/)) {
+    if (clause.length <= CUE_MAX_CHARS) {
+      clauses.push(clause);
+      continue;
+    }
+    out.push(...pack(clauses), ...pack(clause.split(' ')));
+    clauses = [];
+  }
+  return [...out, ...pack(clauses)];
+}
+
+/** Subtitle cues for one line, timed in proportion to their length and back to back. */
+export function subtitleCues(text: string, atMs: number, durationMs: number): Cue[] {
+  // Sentences first, long ones split at commas or words; then short neighbours share a cue, so
+  // a two-word sentence doesn't flash by on its own.
+  const chunks = pack(
+    text
+      .trim()
+      .replace(/\s+/g, ' ')
+      .split(/(?<=[.!?…])\s+/)
+      .flatMap(sentenceChunks),
+  );
+  const total = chunks.reduce((n, c) => n + c.length, 0);
+  let done = 0;
+  return chunks.map((chunk) => {
+    const startMs = atMs + Math.round((durationMs * done) / total);
+    done += chunk.length;
+    return { startMs, endMs: atMs + Math.round((durationMs * done) / total), text: chunk };
+  });
+}
+
+const stamp = (ms: number) => {
+  const t = Math.max(0, Math.round(ms));
+  const pad = (n: number, w = 2) => String(n).padStart(w, '0');
+  return `${pad(Math.floor(t / 3_600_000))}:${pad(Math.floor(t / 60_000) % 60)}:${pad(Math.floor(t / 1000) % 60)},${pad(t % 1000, 3)}`;
+};
+
+/** Break a cue over two lines at the space nearest its middle. */
+function wrap(text: string): string {
+  if (text.length <= LINE_CHARS) return text;
+  let best = -1;
+  for (let i = 0; i < text.length; i++) {
+    if (text[i] === ' ' && (best < 0 || Math.abs(i - text.length / 2) < Math.abs(best - text.length / 2))) best = i;
+  }
+  return best < 0 ? text : `${text.slice(0, best)}\n${text.slice(best + 1)}`;
+}
+
+/** SubRip text; `offsetMs` shifts video-relative cues to file time (the intro card). */
+export function toSrt(cues: Cue[], offsetMs: number): string {
+  return cues.map((c, i) => `${i + 1}\n${stamp(c.startMs + offsetMs)} --> ${stamp(c.endMs + offsetMs)}\n${wrap(c.text)}\n`).join('\n');
+}
+
+export interface Typing {
+  startMs: number;
+  endMs: number;
+  chars: number;
+}
+
+/**
+ * Click and keystroke sounds in output time. Takes without a keystroke log (before 0.4) spread
+ * each `type` step's keys evenly over its typing time.
+ */
+export function soundEvents(ev: { clicks: Click[]; keys?: number[] }, typing: Typing[], segments: Segment[]): Sound[] {
+  const keys =
+    ev.keys ?? typing.flatMap((t) => Array.from({ length: t.chars }, (_, i) => t.startMs + ((t.endMs - t.startMs) * (i + 0.5)) / t.chars));
+  const out: Sound[] = ev.clicks.map((c) => ({ kind: 'click', atMs: sourceToOutput(segments, c.tMs) }));
+  let last = -Infinity;
+  for (const k of [...keys].sort((a, b) => a - b)) {
+    const atMs = sourceToOutput(segments, k);
+    if (atMs - last < KEY_MIN_GAP_MS) continue;
+    out.push({ kind: 'key', atMs });
+    last = atMs;
+  }
+  return out.sort((a, b) => a.atMs - b.atMs);
+}

package/src/timeline/timemap.ts

@@ -28,15 +28,50 @@ export function buildSegments(durationMs: number, spans: SpeedSpan[]): Segment[]
   return out;
 }
 
+export interface Hold {
+  /** source ms whose frame stays on screen */
+  atMs: number;
+  holdMs: number;
+}
+
+/** Output length of one segment. */
+export const segLength = (s: Segment): number => (s.speed === 0 ? (s.holdMs ?? 0) : (s.srcEndMs - s.srcStartMs) / s.speed);
+
+/** Insert still-frame holds, splitting the segment each one falls in; holds at one point merge. */
+export function insertHolds(segments: Segment[], holds: Hold[]): Segment[] {
+  const at = new Map<number, number>();
+  for (const h of holds) if (h.holdMs > 0) at.set(h.atMs, (at.get(h.atMs) ?? 0) + h.holdMs);
+  const points = [...at.keys()].sort((a, b) => a - b);
+  const hold = (t: number): Segment => ({ srcStartMs: t, srcEndMs: t, speed: 0, holdMs: at.get(t) });
+  const out: Segment[] = [];
+  let p = 0;
+  for (const s of segments) {
+    if (s.speed === 0) {
+      out.push(s);
+      continue;
+    }
+    let start = s.srcStartMs;
+    for (; p < points.length && points[p] < s.srcEndMs; p++) {
+      const t = Math.max(points[p], start);
+      if (t > start) out.push({ ...s, srcStartMs: start, srcEndMs: t });
+      out.push(hold(points[p]));
+      start = t;
+    }
+    out.push({ ...s, srcStartMs: start });
+  }
+  for (; p < points.length; p++) out.push(hold(points[p]));
+  return out;
+}
+
 export function outputDurationMs(segments: Segment[]): number {
-  return segments.reduce((acc, s) => acc + (s.srcEndMs - s.srcStartMs) / s.speed, 0);
+  return segments.reduce((acc, s) => acc + segLength(s), 0);
 }
 
 export function outputToSource(segments: Segment[], tOutMs: number): number {
   let acc = 0;
   for (const s of segments) {
-    const len = (s.srcEndMs - s.srcStartMs) / s.speed;
-    if (tOutMs < acc + len) return s.srcStartMs + Math.max(0, tOutMs - acc) * s.speed;
+    const len = segLength(s);
+    if (tOutMs < acc + len) return s.speed === 0 ? s.srcStartMs : s.srcStartMs + Math.max(0, tOutMs - acc) * s.speed;
     acc += len;
   }
   return segments.length ? segments[segments.length - 1].srcEndMs : 0;
@@ -45,8 +80,8 @@ export function outputToSource(segments: Segment[], tOutMs: number): number {
 export function sourceToOutput(segments: Segment[], tSrcMs: number): number {
   let acc = 0;
   for (const s of segments) {
-    if (tSrcMs < s.srcEndMs) return acc + Math.max(0, tSrcMs - s.srcStartMs) / s.speed;
-    acc += (s.srcEndMs - s.srcStartMs) / s.speed;
+    if (s.speed !== 0 && tSrcMs < s.srcEndMs) return acc + Math.max(0, tSrcMs - s.srcStartMs) / s.speed;
+    acc += segLength(s);
   }
   return acc;
 }

package/src/timeline/types.ts

@@ -60,12 +60,16 @@ export interface TakeEvents {
   moves: Move[];
   clicks: Click[];
   idle: Interval[];
+  /** keystroke times (human-paced typing); missing in takes recorded before 0.4 */
+  keys?: number[];
 }
 
 export interface Segment {
   srcStartMs: number;
   srcEndMs: number;
+  /** 0 for a hold: the frame at srcStartMs (= srcEndMs) stays on screen for `holdMs` */
   speed: number;
+  holdMs?: number;
 }
 
 export interface CameraKey {
@@ -91,12 +95,37 @@ export interface Spotlight {
   rect: Rect;
 }
 
+/** A subtitle cue, in output ms from the start of the video. */
+export interface Cue {
+  startMs: number;
+  endMs: number;
+  text: string;
+}
+
+export interface Sound {
+  kind: 'click' | 'key';
+  /** output ms from the start of the video */
+  atMs: number;
+}
+
+export interface NarrationLine {
+  /** `scene#n` */
+  where: string;
+  text: string;
+  /** output ms from the start of the video */
+  atMs: number;
+  durationMs: number;
+}
+
 export type ResolvedStyle = Style;
 
-/** The renderer contract. All overlay times are source ms. */
+export type Aspect = '16:9' | '9:16' | '1:1';
+
+/** The renderer contract. Overlay times are source ms; sound and subtitle times are output ms from the video start. */
 export interface Timeline {
   version: 1;
   fps: number;
+  aspect: Aspect;
   output: { width: number; height: number };
   stage: { width: number; height: number };
   video: { src: string; durationMs: number };
@@ -113,5 +142,13 @@ export interface Timeline {
   clicks: Click[];
   captions: Caption[];
   spotlights: Spotlight[];
+  narration: NarrationLine[];
+  subtitles: Cue[];
+  /** Subtitles drawn on the video (captions then sit at the top). */
+  subtitlesBurned: boolean;
+  sounds: Sound[];
+  music: { volume: number } | null;
+  /** The mixed soundtrack, relative to the renderer's public folder; set by the pipeline. */
+  audio: { src: string } | null;
   style: ResolvedStyle;
 }