@erclx/canon 5.76.0 → 5.78.0

package/src/take/log.ts

@@ -31,6 +31,11 @@ export interface Box {
   readonly height: number
 }
 
+/** A position the page was sent, at the clip time it was sent. */
+export interface SentPoint extends Point {
+  readonly t: number
+}
+
 export type LogEvent =
   | {
       readonly t: number
@@ -38,6 +43,15 @@ export type LogEvent =
       readonly from: Point
       readonly to: Point
       readonly ms: number
+      /**
+       * Every position the glide sent, in order. A reader draws the cursor off
+       * these rather than off the eased curve, since a page slower than a frame
+       * skips points the curve would still draw. Absent in a log written
+       * before the field and in a harness log, where the curve stands in.
+       */
+      readonly points?: readonly SentPoint[]
+      /** How long the camera holds aimed at `to` once the glide ends. */
+      readonly hold?: number
     }
   | {
       readonly t: number
@@ -103,6 +117,12 @@ export interface TakeLog {
   readonly clip?: { readonly width: number; readonly height: number }
   /** False when the beat turned the camera's zoom off. */
   readonly push?: false
+  /**
+   * Milliseconds a screencast take shifted every event by, being how long a
+   * move took to show in the clip. Absent on the stepped route, which needs
+   * no shift, and where the page gave no frame to calibrate on.
+   */
+  readonly lag?: number
   readonly frames: number
   readonly events: readonly LogEvent[]
 }
@@ -190,13 +210,34 @@ export function along(from: Point, to: Point, progress: number): Point {
   return { x: from.x + (to.x - from.x) * k, y: from.y + (to.y - from.y) * k }
 }
 
-/** The pointer's position at a time on the clip's clock, read off the moves. */
+/**
+ * The pointer's position at a time on the clip's clock, read off the points
+ * each move sent, or off its eased curve when it carries none.
+ */
 export function cursorAt(events: readonly LogEvent[], ms: number): Point {
   let position = START
   for (const event of events) {
     if (event.kind !== 'move') continue
     if (ms < event.t) break
-    position = along(event.from, event.to, (ms - event.t) / event.ms)
+    position = event.points?.length
+      ? sentAt(event.from, event.points, ms)
+      : along(event.from, event.to, (ms - event.t) / event.ms)
   }
   return position
 }
+
+function sentAt(from: Point, points: readonly SentPoint[], ms: number): Point {
+  let before: SentPoint | undefined
+  for (const point of points) {
+    if (point.t > ms) {
+      if (!before) return from
+      const k = (ms - before.t) / (point.t - before.t)
+      return {
+        x: before.x + (point.x - before.x) * k,
+        y: before.y + (point.y - before.y) * k,
+      }
+    }
+    before = point
+  }
+  return before ? { x: before.x, y: before.y } : from
+}

package/src/take/perform.ts

@@ -3,6 +3,7 @@ import {
   FPS,
   type LogEvent,
   type Point,
+  type SentPoint,
   VIEWPORT,
   along,
   travelMs,
@@ -28,6 +29,16 @@ const SCROLL_MS = 900
 /** How long a drag holds still after its press, so the viewer sees the grab land. */
 const GRAB_MS = 150
 
+/**
+ * The pacing the operator settled by eye in groundwork track
+ * `210-film-craft-and-soundtrack`: a rest on the target before a press, a hold
+ * on its result before the pointer leaves, and a pause between clicking a
+ * field and its first key, so the camera lands before anything changes.
+ */
+const REST_MS = 300
+const RESULT_MS = 500
+const TYPE_MS = 400
+
 export type PointerInput =
   | {
       readonly type: 'move'
@@ -73,6 +84,12 @@ export interface Stage {
   press(input: KeyInput): Promise<void>
   /** Evaluates an expression in the page. */
   evaluate<T>(expression: string): Promise<T>
+  /**
+   * Waits for the page's next frame before a glide stamps a point, on a route
+   * whose clock runs apart from the page's frames. A stage that steps those
+   * frames itself leaves it out.
+   */
+  align?(): Promise<void>
 }
 
 /** A target the pointer cannot reach, which refuses rather than films a guess. */
@@ -125,13 +142,22 @@ export async function performSteps(
 ): Promise<void> {
   for (const step of steps) {
     switch (step.kind) {
-      case 'move':
-        await moveTo(
-          stage,
-          state,
-          'target' in step ? center(await boxOf(stage, step.target)) : step,
-        )
+      case 'move': {
+        const to =
+          'target' in step
+            ? center(await boxOf(stage, step.target))
+            : { x: step.x, y: step.y }
+        if (step.hold === undefined) {
+          await moveTo(stage, state, to)
+          break
+        }
+        const glide =
+          (await moveTo(stage, state, to, { unlogged: true })) ??
+          stillAt(stage, state.pointer)
+        await stage.hold(step.hold)
+        state.log?.push({ ...glide, hold: step.hold })
         break
+      }
       case 'click':
       case 'dblclick':
         await click(
@@ -141,6 +167,7 @@ export async function performSteps(
           step.modifiers ?? [],
           step.kind === 'dblclick' ? 2 : 1,
         )
+        await pause(stage, state, RESULT_MS)
         break
       case 'drag':
         await drag(
@@ -167,6 +194,7 @@ export async function performSteps(
       case 'fill': {
         const box = await boxOf(stage, step.target)
         await click(stage, state, box, [], 1)
+        await pause(stage, state, TYPE_MS)
         const start = stage.now()
         for (const character of step.text) {
           await stage.key(character)
@@ -180,6 +208,7 @@ export async function performSteps(
           chars: step.text.length,
           ms: stage.now() - start,
         })
+        await pause(stage, state, RESULT_MS)
         break
       }
       case 'scroll': {
@@ -214,25 +243,53 @@ export async function performSteps(
   }
 }
 
+type Glide = Extract<LogEvent, { kind: 'move' }>
+
+/**
+ * Glides the pointer to `to` and returns the glide, or undefined when the
+ * pointer is already there. The glide is logged unless `unlogged`, which a
+ * caller sets to log it with more fields itself.
+ */
 async function moveTo(
   stage: Stage,
   state: Performance,
   to: Point,
-  held?: { readonly modifiers: readonly Modifier[] },
-): Promise<void> {
+  options: {
+    readonly held?: { readonly modifiers: readonly Modifier[] }
+    readonly unlogged?: true
+  } = {},
+): Promise<Glide | undefined> {
   const from = state.pointer
-  if (Math.hypot(to.x - from.x, to.y - from.y) < 1) return
+  if (Math.hypot(to.x - from.x, to.y - from.y) < 1) return undefined
   const frames = Math.max(1, Math.round(travelMs(from, to) / FRAME_MS))
-  const pressed = held
-    ? { held: true as const, ...withModifiers(held.modifiers) }
+  const pressed = options.held
+    ? { held: true as const, ...withModifiers(options.held.modifiers) }
     : {}
   const start = stage.now()
+  const points: SentPoint[] = []
   await paced(stage, frames, async (frame) => {
     state.pointer = along(from, to, frame / frames)
+    await stage.align?.()
+    points.push({ t: stage.now(), ...state.pointer })
     await stage.pointer({ type: 'move', ...state.pointer, ...pressed })
   })
   state.pointer = to
-  state.log?.push({ t: start, kind: 'move', from, to, ms: stage.now() - start })
+  const glide: Glide = {
+    t: start,
+    kind: 'move',
+    from,
+    to,
+    ms: stage.now() - start,
+    points,
+  }
+  if (!options.unlogged) state.log?.push(glide)
+  return glide
+}
+
+/** A glide of no length, for a hold aimed where the pointer already rests. */
+function stillAt(stage: Stage, at: Point): Glide {
+  const t = stage.now()
+  return { t, kind: 'move', from: at, to: at, ms: 0, points: [{ t, ...at }] }
 }
 
 /**
@@ -263,6 +320,7 @@ async function click(
 ): Promise<void> {
   const at = center(box)
   await moveTo(stage, state, at)
+  await pause(stage, state, REST_MS)
   await holding(stage, modifiers, async () => {
     state.log?.push({
       t: stage.now(),
@@ -280,6 +338,18 @@ async function click(
   })
 }
 
+/**
+ * Lets `ms` of clip time pass while a beat is filmed, and none in setup, which
+ * no viewer sees.
+ */
+async function pause(
+  stage: Stage,
+  state: Performance,
+  ms: number,
+): Promise<void> {
+  if (state.log) await stage.hold(ms)
+}
+
 /**
  * Presses on the source, holds a moment so the page reads a grab rather than
  * a click, glides held to the target, and releases there. The drop target is
@@ -296,12 +366,13 @@ async function drag(
   const from = center(box)
   const to = center(drop)
   await moveTo(stage, state, from)
+  await pause(stage, state, REST_MS)
   await holding(stage, modifiers, async () => {
     const start = stage.now()
     const press = { clickCount: 1, ...withModifiers(modifiers) }
     await stage.pointer({ type: 'down', ...from, ...press })
     await stage.hold(GRAB_MS)
-    await moveTo(stage, state, to, { modifiers })
+    await moveTo(stage, state, to, { held: { modifiers } })
     await stage.pointer({ type: 'up', ...to, ...press })
     state.log?.push({
       t: start,
@@ -314,6 +385,7 @@ async function drag(
       ms: stage.now() - start,
     })
   })
+  await pause(stage, state, RESULT_MS)
 }
 
 /**

package/src/take/plan.ts

@@ -18,9 +18,17 @@ interface Held {
   readonly modifiers?: readonly Modifier[]
 }
 
+/**
+ * A move's milliseconds at its end with the camera aimed there, absent when the
+ * move aims the camera nowhere.
+ */
+interface Aimed {
+  readonly hold?: number
+}
+
 export type TakeStep =
-  | { readonly kind: 'move'; readonly target: string }
-  | { readonly kind: 'move'; readonly x: number; readonly y: number }
+  | ({ readonly kind: 'move'; readonly target: string } & Aimed)
+  | ({ readonly kind: 'move'; readonly x: number; readonly y: number } & Aimed)
   | ({ readonly kind: 'click'; readonly target: string } & Held)
   | ({ readonly kind: 'dblclick'; readonly target: string } & Held)
   | ({
@@ -196,15 +204,17 @@ function readStep(entry: unknown, at: string): TakeStep {
   }
 
   switch (entry.kind) {
-    case 'move':
-      if (target) return { kind: 'move', target }
+    case 'move': {
+      const aimed = readHold(entry, at)
+      if (target) return { kind: 'move', target, ...aimed }
       if (isNumber(entry.x) && isNumber(entry.y)) {
-        return { kind: 'move', x: entry.x, y: entry.y }
+        return { kind: 'move', x: entry.x, y: entry.y, ...aimed }
       }
       throw new Refusal(
         'bad-step',
         `${at} is a move with neither a target nor an x and y`,
       )
+    }
     case 'click':
     case 'dblclick':
       return {
@@ -258,6 +268,17 @@ function readStep(entry: unknown, at: string): TakeStep {
   }
 }
 
+function readHold(entry: Record<string, unknown>, at: string): Aimed {
+  if (!('hold' in entry)) return {}
+  if (!isNumber(entry.hold) || entry.hold < 0) {
+    throw new Refusal(
+      'bad-step',
+      `${at} is a move whose hold is not a non-negative number of ms`,
+    )
+  }
+  return { hold: entry.hold }
+}
+
 function readModifiers(value: unknown, at: string): Held {
   if (value === undefined) return {}
   if (!Array.isArray(value)) {

package/src/take/reel.ts

@@ -22,8 +22,15 @@ export const REEL_EVERY = 3
 
 export const REEL_FPS = FPS / REEL_EVERY
 
-/** WebP quality, measured at about 1.4 MB for a 9 s reel at this width. */
-const QUALITY = 60
+/** The most GitHub renders inline as an image in a comment. */
+export const REEL_MAX_BYTES = 10_000_000
+
+/**
+ * Quality of the lossy fallback, measured at about 1.4 MB for a 9 s reel at
+ * this width. It skips frames whose change is faint, so a fade-out can leave a
+ * ghost, and the fallback exists only to keep a long reel under the cap.
+ */
+const LOSSY_QUALITY = 60
 
 export interface ReelFrame extends Frame {
   /** Which beat of the take, in plan order. */
@@ -37,6 +44,8 @@ export interface Reel {
   readonly frames: number
   readonly seconds: number
   readonly bytes: number
+  /** False when the reel passed the cap lossless and was encoded again lossy. */
+  readonly lossless?: boolean
 }
 
 /**
@@ -75,11 +84,28 @@ export async function* reelFrames(
   }
 }
 
-/** Renders the reel of a filmed take to `file`, an animated WebP. */
+/**
+ * Renders the reel of a filmed take to `file`, an animated WebP. It encodes
+ * lossless, since the lossy encoder skips a frame whose change is faint and
+ * never repaints the last copy of an element that faded out. A reel past
+ * `maxBytes` is encoded again lossy so a long reel still posts.
+ */
 export async function renderReel(
   out: string,
   beats: readonly FilmedBeat[],
   file: string,
+  maxBytes: number = REEL_MAX_BYTES,
+): Promise<Reel> {
+  const lossless = await encode(out, beats, file, true)
+  if (lossless.bytes <= maxBytes) return { ...lossless, lossless: true }
+  return { ...(await encode(out, beats, file, false)), lossless: false }
+}
+
+async function encode(
+  out: string,
+  beats: readonly FilmedBeat[],
+  file: string,
+  lossless: boolean,
 ): Promise<Reel> {
   const first = beats[0]
   if (!first) throw new Error('a reel needs at least one filmed beat')
@@ -105,8 +131,7 @@ export async function renderReel(
       '-',
       '-c:v',
       'libwebp_anim',
-      '-quality',
-      String(QUALITY),
+      ...(lossless ? ['-lossless', '1'] : ['-quality', String(LOSSY_QUALITY)]),
       '-loop',
       '0',
       '-f',

package/src/take/screencast.ts

@@ -11,7 +11,7 @@ import {
   performSteps,
 } from '@/take/perform'
 import type { Beat } from '@/take/plan'
-import { HOLD_MS } from '@/take/stepper'
+import { HOLD_MS, withinCap } from '@/take/stepper'
 
 /**
  * Films a beat in real time through Playwright's screencast, for a page frame
@@ -27,9 +27,21 @@ const SETTLE_MS = 2500
 /** How long a refusal waits on the screenshot it shows. */
 const STILL_CAP_MS = 2000
 
+/**
+ * How long a move waits on the page's next animation frame before it goes
+ * anyway, since a page whose frames stall is the stepper's case and this route
+ * must still film.
+ */
+const ALIGN_CAP_MS = 100
+
+/** How long the calibration waits on the page to paint its move. */
+const CALIBRATE_CAP_MS = 500
+
 export interface ScreencastTake {
   readonly frames: number
   readonly events: readonly LogEvent[]
+  /** What `events` were shifted by, absent when the page gave no frame to read. */
+  readonly lag?: number
   /** The clip's closing frame as a JPEG. */
   readonly still: Buffer
 }
@@ -56,7 +68,11 @@ export async function filmScreencast(request: {
     const stageOn = (clock: () => number): Stage => ({
       now: clock,
       hold: (ms) => page.waitForTimeout(ms),
+      align: () => nextFrame(page, ALIGN_CAP_MS).then(() => undefined),
       pointer: async (input) => {
+        // A move waited on its frame already, so padding it would land the
+        // next one a frame late.
+        if (input.type === 'move') return send(page.mouse, input)
         const sent = clock()
         await send(page.mouse, input)
         const spent = clock() - sent
@@ -79,7 +95,7 @@ export async function filmScreencast(request: {
     const live = stageOn(() => performance.now())
     await performSteps(live, request.beat.setup, { pointer: START })
     if (request.beat.setup.length > 0) await live.hold(1000)
-    await page.mouse.move(START.x, START.y)
+    const lag = await calibrate(page)
 
     const webm = join(scratch, 'take.webm')
     const events: LogEvent[] = []
@@ -97,7 +113,14 @@ export async function filmScreencast(request: {
 
     await transcode(webm, request.clip, FPS, filmedMs)
     const frames = await probeFrames(request.clip)
-    return { frames, events, still: await lastFrame(request.clip, frames) }
+    const closing = await lastFrame(request.clip, frames)
+    if (lag === undefined) return { frames, events, still: closing }
+    return {
+      frames,
+      events: events.map((event) => shifted(event, lag)),
+      lag: Number(lag.toFixed(1)),
+      still: closing,
+    }
   } catch (error) {
     if (error instanceof TargetRefused && still) error.still = await still()
     throw error
@@ -107,8 +130,92 @@ export async function filmScreencast(request: {
   }
 }
 
+type Page = import('playwright-core').Page
 type Mouse = import('playwright-core').Mouse
 
+/**
+ * The page time of its next animation frame, or undefined when none comes
+ * within `capMs`, as on a hidden page, or the page closed first.
+ */
+function nextFrame(page: Page, capMs: number): Promise<number | undefined> {
+  return withinCap(
+    page
+      .evaluate(
+        () => new Promise<number>((done) => requestAnimationFrame(done)),
+      )
+      .catch(() => undefined),
+    capMs,
+  )
+}
+
+/** Where the calibration's armed listener hands back the frame it painted in. */
+const PAINTED = '__canonTakePainted'
+
+/**
+ * How long a move takes to show in the clip, measured by sending one aligned
+ * move to `START` and reading the animation frame after its handler against
+ * the frame it was aligned to. Both readings are page time, so no clock mapping
+ * is needed. The recorder captures a frame once it is presented, one page
+ * frame after that frame begins, so the lag counts one page frame past the
+ * readings, read off two consecutive animation frames rather than assumed from
+ * the clip's rate. That held a dragged box within 7 px on every run where the
+ * readings alone left 10 to 15. Undefined when the page gives no frame to read,
+ * and the pointer rests on `START` either way.
+ */
+async function calibrate(page: Page): Promise<number | undefined> {
+  await page.mouse.move(START.x - 1, START.y)
+  const interval = await withinCap(
+    page
+      .evaluate(
+        () =>
+          new Promise<number>((done) =>
+            requestAnimationFrame((first) =>
+              requestAnimationFrame((second) => done(second - first)),
+            ),
+          ),
+      )
+      .catch(() => undefined),
+    2 * ALIGN_CAP_MS,
+  )
+  await page.evaluate((key) => {
+    const slot = window as unknown as Record<string, unknown>
+    slot[key] = new Promise<number>((done) => {
+      addEventListener('pointermove', () => requestAnimationFrame(done), {
+        once: true,
+        capture: true,
+      })
+    })
+  }, PAINTED)
+  const aligned = await nextFrame(page, ALIGN_CAP_MS)
+  await page.mouse.move(START.x, START.y)
+  const painted = await withinCap(
+    page
+      .evaluate((key) => {
+        const slot = window as unknown as Record<string, unknown>
+        const armed = slot[key] as Promise<number>
+        delete slot[key]
+        return armed
+      }, PAINTED)
+      .catch(() => undefined),
+    CALIBRATE_CAP_MS,
+  )
+  if (interval === undefined || aligned === undefined) return undefined
+  if (painted === undefined) return undefined
+  return Math.max(0, painted - aligned) + interval
+}
+
+/** An event moved `lag` later on the clip, with every point it sent. */
+function shifted(event: LogEvent, lag: number): LogEvent {
+  if (event.kind === 'move' && event.points) {
+    return {
+      ...event,
+      t: event.t + lag,
+      points: event.points.map((point) => ({ ...point, t: point.t + lag })),
+    }
+  }
+  return { ...event, t: event.t + lag }
+}
+
 async function send(mouse: Mouse, input: PointerInput): Promise<void> {
   switch (input.type) {
     case 'move':

package/src/take/stepper.ts

@@ -187,10 +187,10 @@ export async function filmStepped(
 async function frameStill(
   beginFrame: (screenshot: boolean) => Promise<{ screenshotData?: string }>,
 ): Promise<Buffer | undefined> {
-  const reply = await Promise.race([
+  const reply = await withinCap(
     beginFrame(true).catch(() => undefined),
-    sleep(STILL_CAP_MS),
-  ])
+    STILL_CAP_MS,
+  )
   return reply?.screenshotData
     ? Buffer.from(reply.screenshotData, 'base64')
     : undefined
@@ -214,7 +214,7 @@ async function load(page: ShellPage, url: string): Promise<number> {
   if (navigated.errorText) {
     throw new Error(`${navigated.errorText} at ${url}`)
   }
-  const timestamp = await Promise.race([loaded, sleep(LOAD_CAP_MS)])
+  const timestamp = await withinCap(loaded, LOAD_CAP_MS)
   if (typeof timestamp !== 'number') {
     throw new Error(
       `${url} fired no load event within ${LOAD_CAP_MS / 1000} s, so no frame clock could be read`,
@@ -381,6 +381,26 @@ async function withinStall<T>(
   }
 }
 
+/**
+ * Races `work` against a cap and resolves empty when the cap wins. The timer
+ * is cleared once the race settles, since a pending one keeps the process alive
+ * for the whole cap after filming ends.
+ */
+export async function withinCap<T>(
+  work: Promise<T>,
+  ms: number,
+): Promise<T | undefined> {
+  let timer: ReturnType<typeof setTimeout> | undefined
+  const capped = new Promise<undefined>((done) => {
+    timer = setTimeout(() => done(undefined), ms)
+  })
+  try {
+    return await Promise.race([work, capped])
+  } finally {
+    clearTimeout(timer)
+  }
+}
+
 function sleep(ms: number): Promise<void> {
   return new Promise((done) => setTimeout(done, ms))
 }