@erclx/canon 5.65.0 → 5.67.0

package/src/take/shell.ts

@@ -0,0 +1,287 @@
+import { type ChildProcess, spawn } from 'node:child_process'
+import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'
+import { createRequire } from 'node:module'
+import { homedir, tmpdir } from 'node:os'
+import { dirname, join } from 'node:path'
+
+/**
+ * Launches chrome-headless-shell under begin-frame control and speaks raw CDP
+ * to one page in it. Playwright cannot ask for begin-frame control on a page it
+ * opens, and only the headless shell honors the flag, so this bypasses the
+ * engine and finds the shell Playwright installed beside Chromium.
+ *
+ * `playwright-core` exposes no path for the shell, so the path is rebuilt from
+ * its `browsers.json` revision and its cache layout. A Playwright upgrade that
+ * moves that layout surfaces here as `browser-missing` rather than as a crash.
+ */
+
+export class ShellMissing extends Error {}
+
+/** Each platform's folder and binary inside the shell's revision folder. */
+const SHELL_DIRS: Record<string, readonly [string, string]> = {
+  'linux-x64': ['chrome-headless-shell-linux64', 'chrome-headless-shell'],
+  'linux-arm64': ['chrome-headless-shell-linux-arm64', 'chrome-headless-shell'],
+  'darwin-x64': ['chrome-headless-shell-mac-x64', 'chrome-headless-shell'],
+  'darwin-arm64': ['chrome-headless-shell-mac-arm64', 'chrome-headless-shell'],
+}
+
+const FLAGS = [
+  '--enable-begin-frame-control',
+  '--run-all-compositor-stages-before-draw',
+  '--disable-threaded-animation',
+  '--disable-threaded-scrolling',
+  '--disable-checker-imaging',
+  '--hide-scrollbars',
+  '--force-device-scale-factor=2',
+  '--mute-audio',
+  '--no-first-run',
+]
+
+/** Resolves the installed shell, throwing `ShellMissing` naming where it looked. */
+export function findShell(env: NodeJS.ProcessEnv = process.env): string {
+  const require = createRequire(import.meta.url)
+  const engineRoot = dirname(require.resolve('playwright-core/package.json'))
+  const manifest = JSON.parse(
+    readFileSync(join(engineRoot, 'browsers.json'), 'utf8'),
+  ) as { browsers: { name: string; revision: string }[] }
+  const revision = manifest.browsers.find(
+    (browser) => browser.name === 'chromium-headless-shell',
+  )?.revision
+  const dirs = SHELL_DIRS[`${process.platform}-${process.arch}`]
+  if (!revision || !dirs) {
+    throw new ShellMissing(
+      `no chrome-headless-shell for ${process.platform}-${process.arch} in playwright-core's browsers.json`,
+    )
+  }
+
+  const path = join(
+    registryRoot(env, engineRoot),
+    `chromium_headless_shell-${revision}`,
+    ...dirs,
+  )
+  if (!existsSync(path)) {
+    throw new ShellMissing(
+      `chrome-headless-shell executable doesn't exist at ${path}`,
+    )
+  }
+  return path
+}
+
+/** Where Playwright keeps its browsers, read the way its own registry reads it. */
+function registryRoot(env: NodeJS.ProcessEnv, engineRoot: string): string {
+  const pinned = env.PLAYWRIGHT_BROWSERS_PATH
+  if (pinned === '0') return join(engineRoot, '.local-browsers')
+  if (pinned) return pinned
+  if (process.platform === 'darwin') {
+    return join(homedir(), 'Library', 'Caches', 'ms-playwright')
+  }
+  return join(env.XDG_CACHE_HOME || join(homedir(), '.cache'), 'ms-playwright')
+}
+
+type Listener = (params: Record<string, unknown>) => void
+
+export interface ShellPage {
+  send<T = Record<string, unknown>>(
+    method: string,
+    params?: Record<string, unknown>,
+  ): Promise<T>
+  /** Calls `listener` on every event of `method` from this page. */
+  on(method: string, listener: Listener): void
+  /** Runs an expression in the page and returns its value. */
+  evaluate<T>(expression: string): Promise<T>
+  /** Kills the shell and removes its profile, safe to call more than once. */
+  close(): void
+}
+
+/**
+ * Opens one page at `width` by `height` with begin-frame control. The launch
+ * owns its profile folder and kills the shell on process exit, so a killed take
+ * leaves no shell running and no profile behind.
+ */
+export async function openShell(
+  executable: string,
+  size: { readonly width: number; readonly height: number },
+): Promise<ShellPage> {
+  const profile = mkdtempSync(join(tmpdir(), 'canon-take-'))
+  const proc = spawn(
+    executable,
+    [
+      '--remote-debugging-port=0',
+      `--user-data-dir=${profile}`,
+      ...FLAGS,
+      'about:blank',
+    ],
+    { stdio: ['ignore', 'ignore', 'pipe'] },
+  )
+
+  let closed = false
+  const close = (): void => {
+    if (closed) return
+    closed = true
+    process.off('exit', close)
+    process.off('SIGINT', interrupt)
+    process.off('SIGTERM', interrupt)
+    socket?.close()
+    proc.kill('SIGKILL')
+    rmSync(profile, { recursive: true, force: true })
+  }
+  // A signal skips the exit event, so it closes the shell and then re-raises
+  // itself with the default handler restored, ending the process as it would have.
+  const interrupt = (signal: NodeJS.Signals): void => {
+    close()
+    process.kill(process.pid, signal)
+  }
+  process.on('exit', close)
+  process.once('SIGINT', interrupt)
+  process.once('SIGTERM', interrupt)
+
+  let socket: WebSocket | undefined
+  try {
+    socket = new WebSocket(await devtoolsUrl(proc))
+    await new Promise((done, fail) => {
+      socket?.addEventListener('open', done, { once: true })
+      socket?.addEventListener('error', fail, { once: true })
+    })
+    return await attach(socket, size, close)
+  } catch (error) {
+    close()
+    throw error
+  }
+}
+
+function devtoolsUrl(proc: ChildProcess): Promise<string> {
+  return new Promise((done, fail) => {
+    let output = ''
+    proc.stderr?.on('data', (chunk: Buffer) => {
+      output += chunk.toString()
+      const match = output.match(/DevTools listening on (ws:\S+)/)
+      if (match?.[1]) done(match[1])
+    })
+    proc.on('error', fail)
+    proc.on('exit', (code) =>
+      fail(new Error(`the shell exited ${code}: ${output.slice(-400)}`)),
+    )
+  })
+}
+
+async function attach(
+  socket: WebSocket,
+  size: { readonly width: number; readonly height: number },
+  close: () => void,
+): Promise<ShellPage> {
+  let nextId = 0
+  const pending = new Map<
+    number,
+    { resolve: (value: never) => void; reject: (error: Error) => void }
+  >()
+  const listeners: { method: string; sessionId: string; listener: Listener }[] =
+    []
+
+  socket.addEventListener('message', (event) => {
+    const message = JSON.parse(String(event.data)) as {
+      id?: number
+      method?: string
+      sessionId?: string
+      params?: Record<string, unknown>
+      result?: never
+      error?: { message: string; data?: string }
+    }
+    if (message.id !== undefined) {
+      const waiter = pending.get(message.id)
+      if (!waiter) return
+      pending.delete(message.id)
+      if (message.error) {
+        waiter.reject(
+          new Error(
+            `${message.error.message} ${message.error.data ?? ''}`.trim(),
+          ),
+        )
+      } else {
+        waiter.resolve(message.result as never)
+      }
+      return
+    }
+    for (const entry of listeners) {
+      if (
+        entry.method === message.method &&
+        entry.sessionId === message.sessionId
+      ) {
+        entry.listener(message.params ?? {})
+      }
+    }
+  })
+  socket.addEventListener('close', () => {
+    for (const waiter of pending.values()) {
+      waiter.reject(new Error('the shell closed its connection'))
+    }
+    pending.clear()
+  })
+
+  const raw = <T>(
+    method: string,
+    params: Record<string, unknown> = {},
+    sessionId?: string,
+  ): Promise<T> =>
+    new Promise<T>((resolve, reject) => {
+      nextId += 1
+      pending.set(nextId, {
+        resolve: resolve as (value: never) => void,
+        reject,
+      })
+      socket.send(
+        JSON.stringify({
+          id: nextId,
+          method,
+          params,
+          ...(sessionId ? { sessionId } : {}),
+        }),
+      )
+    })
+
+  const { browserContextId } = await raw<{ browserContextId: string }>(
+    'Target.createBrowserContext',
+  )
+  const { targetId } = await raw<{ targetId: string }>('Target.createTarget', {
+    url: 'about:blank',
+    browserContextId,
+    enableBeginFrameControl: true,
+    width: size.width,
+    height: size.height,
+  })
+  const { sessionId } = await raw<{ sessionId: string }>(
+    'Target.attachToTarget',
+    { targetId, flatten: true },
+  )
+
+  const send = <T>(method: string, params?: Record<string, unknown>) =>
+    raw<T>(method, params, sessionId)
+
+  await send('Page.enable')
+  await send('Runtime.enable')
+
+  return {
+    send,
+    on: (method, listener) => listeners.push({ method, sessionId, listener }),
+    evaluate: async <T>(expression: string): Promise<T> => {
+      const reply = await send<{
+        result: { value: T }
+        exceptionDetails?: {
+          text: string
+          exception?: { description?: string }
+        }
+      }>('Runtime.evaluate', {
+        expression,
+        awaitPromise: true,
+        returnByValue: true,
+      })
+      if (reply.exceptionDetails) {
+        throw new Error(
+          reply.exceptionDetails.exception?.description ??
+            reply.exceptionDetails.text,
+        )
+      }
+      return reply.result.value
+    },
+    close,
+  }
+}

package/src/take/stepper.ts

@@ -0,0 +1,252 @@
+import { openFrameEncoder } from '@/take/encode'
+import { FPS, type LogEvent, START, VIEWPORT } from '@/take/log'
+import { type PointerInput, type Stage, performSteps } from '@/take/perform'
+import type { Beat } from '@/take/plan'
+import { type ShellPage, openShell } from '@/take/shell'
+
+/**
+ * Films a beat by frame stepping. The page's clock is virtual and advances one
+ * 60 fps frame per step, so every transition, animation, and timer moves
+ * exactly one frame between screenshots however long a frame takes to render,
+ * and each screenshot comes back at twice the viewport, 3840x2160.
+ *
+ * Three orderings hold a take together, and each one broken deadlocks or
+ * freezes it rather than failing loudly:
+ *
+ * - An input is sent, a frame is stepped, and only then is the input awaited,
+ *   since under begin-frame control an input is acknowledged by a frame
+ * - Frame time continues Chromium's monotonic clock, since a frame stamped on
+ *   any other base leaves every CSS animation at its first frame
+ * - No frame is begun before the load event, since one sent mid-navigation can
+ *   go unanswered, and each loading frame is capped so a lost one cannot hang
+ */
+
+const FRAME_MS = 1000 / FPS
+
+/** Budgets of 17, 17, and 16 ms average one 60 fps frame exactly. */
+const BUDGETS = [17, 17, 16] as const
+
+/** A frame slower than this means the page will not step, so the beat falls back. */
+export const STALL_MS = 2000
+
+/** The still the edit can cut on, before the beat's first action and after its last. */
+export const HOLD_MS = 600
+
+const LOAD_CAP_MS = 15_000
+const SETTLE_MS = 2500
+const LOADING_FRAME_CAP_MS = 250
+
+/** A frame that took longer than `STALL_MS`, so the beat is filmed another way. */
+export class StepperStalled extends Error {
+  constructor(readonly frame: number) {
+    super(`frame ${frame} took over ${STALL_MS / 1000} s under frame stepping`)
+  }
+}
+
+export interface SteppedTake {
+  readonly frames: number
+  readonly events: readonly LogEvent[]
+}
+
+export interface SteppedRequest {
+  readonly executable: string
+  readonly url: string
+  readonly beat: Beat
+  readonly clip: string
+}
+
+export async function filmStepped(
+  request: SteppedRequest,
+): Promise<SteppedTake> {
+  const page = await openShell(request.executable, VIEWPORT)
+  const encoder = openFrameEncoder(request.clip, FPS)
+  try {
+    let ticks = await load(page, request.url)
+    const beginFrame = (screenshot: boolean) =>
+      page.send<{ screenshotData?: string }>(
+        'HeadlessExperimental.beginFrame',
+        {
+          frameTimeTicks: (ticks += FRAME_MS),
+          interval: FRAME_MS,
+          ...(screenshot
+            ? { screenshot: { format: 'jpeg', quality: 95 } }
+            : {}),
+        },
+      )
+    await pump(beginFrame, SETTLE_MS)
+
+    const live = liveStage(page, beginFrame)
+    await performSteps(live, request.beat.setup, { pointer: START })
+    if (request.beat.setup.length > 0) await live.hold(1000)
+    await live.pointer({ type: 'move', ...START })
+
+    await page.send('Emulation.setVirtualTimePolicy', { policy: 'pause' })
+
+    const events: LogEvent[] = []
+    let frames = 0
+    let budget = 0
+    let last: Buffer | undefined
+    const expired: (() => void)[] = []
+    page.on('Emulation.virtualTimeBudgetExpired', () => expired.shift()?.())
+
+    const frame = async (): Promise<void> => {
+      const advanced = new Promise<void>((done) => expired.push(done))
+      // Plain advance rather than pausing on pending fetches, which held the
+      // clock through every page switch that loads its view, past the stall
+      // limit. A refused advance never expires its budget, which the stall
+      // reports.
+      page
+        .send('Emulation.setVirtualTimePolicy', {
+          policy: 'advance',
+          budget: BUDGETS[budget++ % BUDGETS.length],
+        })
+        .catch(() => {})
+      const reply = await withinStall(
+        advanced.then(() => beginFrame(true)),
+        frames,
+      )
+      const data = reply.screenshotData
+        ? Buffer.from(reply.screenshotData, 'base64')
+        : last
+      if (!data) throw new Error('the first frame carried no screenshot')
+      last = data
+      await encoder.write(data)
+      frames += 1
+    }
+
+    const recording: Stage = {
+      now: () => frames * FRAME_MS,
+      hold: async (ms) => {
+        for (let i = 0; i < Math.round(ms / FRAME_MS); i += 1) await frame()
+      },
+      pointer: async (input) => {
+        const sent = page.send('Input.dispatchMouseEvent', mouseEvent(input))
+        await frame()
+        await sent
+      },
+      key: async (text) => {
+        const sent = page.send('Input.insertText', { text })
+        await frame()
+        await sent
+      },
+      evaluate: (expression) => page.evaluate(expression),
+    }
+
+    await recording.hold(HOLD_MS)
+    await performSteps(recording, request.beat.steps, {
+      pointer: START,
+      log: events,
+    })
+    await recording.hold(HOLD_MS)
+
+    await encoder.close()
+    return { frames, events }
+  } catch (error) {
+    encoder.abort()
+    throw error
+  } finally {
+    page.close()
+  }
+}
+
+/**
+ * Navigates and resolves with Chromium's monotonic clock in milliseconds, read
+ * off the load event's own timestamp. The runtime's clock cannot stand in for
+ * it: Node's `hrtime` shares the base on Linux, Bun's counts from its own
+ * start, and a frame stamped on the wrong base freezes every animation.
+ */
+async function load(page: ShellPage, url: string): Promise<number> {
+  const loaded = new Promise<number>((done) =>
+    page.on('Page.loadEventFired', ({ timestamp }) => {
+      if (typeof timestamp === 'number') done(timestamp * 1000)
+    }),
+  )
+  const navigated = await page.send<{ errorText?: string }>('Page.navigate', {
+    url,
+  })
+  if (navigated.errorText) {
+    throw new Error(`${navigated.errorText} at ${url}`)
+  }
+  const timestamp = await Promise.race([loaded, sleep(LOAD_CAP_MS)])
+  if (typeof timestamp !== 'number') {
+    throw new Error(
+      `${url} fired no load event within ${LOAD_CAP_MS / 1000} s, so no frame clock could be read`,
+    )
+  }
+  return timestamp
+}
+
+/** Begins frames in real time so loading, fonts, and layout settle. */
+async function pump(
+  beginFrame: (screenshot: boolean) => Promise<unknown>,
+  ms: number,
+): Promise<void> {
+  const end = Date.now() + ms
+  while (Date.now() < end) {
+    await Promise.race([
+      beginFrame(false).catch(() => {}),
+      sleep(LOADING_FRAME_CAP_MS),
+    ])
+    await sleep(12)
+  }
+}
+
+/**
+ * The stage setup runs on, before virtual time is paused. Its frames are begun
+ * without screenshots, since nothing before the first held frame is kept.
+ */
+function liveStage(
+  page: ShellPage,
+  beginFrame: (screenshot: boolean) => Promise<unknown>,
+): Stage {
+  const started = Date.now()
+  return {
+    now: () => Date.now() - started,
+    hold: (ms) => pump(beginFrame, ms),
+    pointer: (input) => {
+      const sent = page.send('Input.dispatchMouseEvent', mouseEvent(input))
+      return withinStall(
+        beginFrame(false).then(() => sent),
+        0,
+      ).then(() => {})
+    },
+    key: (text) => {
+      const sent = page.send('Input.insertText', { text })
+      return withinStall(
+        beginFrame(false).then(() => sent),
+        0,
+      ).then(() => {})
+    },
+    evaluate: (expression) => page.evaluate(expression),
+  }
+}
+
+function mouseEvent(input: PointerInput): Record<string, unknown> {
+  const at = { x: input.x, y: input.y }
+  switch (input.type) {
+    case 'move':
+      return { type: 'mouseMoved', ...at, button: 'none' }
+    case 'down':
+      return { type: 'mousePressed', ...at, button: 'left', clickCount: 1 }
+    case 'up':
+      return { type: 'mouseReleased', ...at, button: 'left', clickCount: 1 }
+    case 'wheel':
+      return { type: 'mouseWheel', ...at, deltaX: 0, deltaY: input.dy }
+  }
+}
+
+async function withinStall<T>(work: Promise<T>, frame: number): Promise<T> {
+  let timer: ReturnType<typeof setTimeout> | undefined
+  const stalled = new Promise<never>((_, fail) => {
+    timer = setTimeout(() => fail(new StepperStalled(frame)), STALL_MS)
+  })
+  try {
+    return await Promise.race([work, stalled])
+  } finally {
+    clearTimeout(timer)
+  }
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((done) => setTimeout(done, ms))
+}

package/src/take/zoom/NOTICE.md

@@ -0,0 +1,45 @@
+# Vendored zoom notice
+
+The files in this folder carry the zoom timing of [OpenScreen](https://github.com/siddharthvaddem/openscreen) at commit `ca8104c279da0d4b4481ec992ff103949e22c772`, under the MIT license below.
+
+## What came from where
+
+| File              | Source in OpenScreen, under `src/components/video-editor/videoPlayback/`                                 |
+| ----------------- | -------------------------------------------------------------------------------------------------------- |
+| `math-utils.ts`   | `mathUtils.ts`                                                                                           |
+| `region-utils.ts` | `zoomRegionUtils.ts`, with `TRANSITION_WINDOW_MS` and `ZOOM_IN_TRANSITION_WINDOW_MS` from `constants.ts` |
+| `spring.ts`       | `motionSmoothing.ts` and `zoomSpring.ts`                                                                 |
+
+## What changed
+
+- The file names are kebab-case, as this repository names every TypeScript file, and the code is reformatted to its style
+- `math-utils.ts` keeps `clamp01`, `cubicBezier`, and `easeOutScreenStudio`, and drops the easings nothing here calls
+- `region-utils.ts` keeps `computeRegionStrength`, the connected-pan easing, and the window and gap constants. It drops the dominant-region search, its cache, the 3D rotation, and the cursor-follow focus, since the camera here aims at clicks and holds its focus by its own rule.
+- The zoom-out window is 700 ms rather than 1015.05 ms, and the connected pan runs 600 ms rather than 1000 ms, both on the operator's eye in groundwork track `193-ui-record`. The zoom-in window keeps OpenScreen's 1522.575 ms.
+- `spring.ts` keeps the zoom spring's stiffness 320, damping 40, mass 0.92, its rest thresholds, the delta clamp, and the overshoot clamp from `stepAxis`. OpenScreen steps that spring through the `motion` package, and this solves it in closed form instead, so no dependency comes with it.
+
+## License
+
+```plaintext
+MIT License
+
+Copyright (c) 2025 Siddharth Vaddem
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+```

package/src/take/zoom/math-utils.ts

@@ -0,0 +1,69 @@
+// Vendored from OpenScreen at ca8104c, MIT. See NOTICE.md beside this file.
+// Source: src/components/video-editor/videoPlayback/mathUtils.ts
+
+export function clamp01(value: number) {
+  return Math.max(0, Math.min(1, value))
+}
+
+function sampleCubicBezier(a1: number, a2: number, t: number) {
+  const oneMinusT = 1 - t
+  return (
+    3 * a1 * oneMinusT * oneMinusT * t + 3 * a2 * oneMinusT * t * t + t * t * t
+  )
+}
+
+function sampleCubicBezierDerivative(a1: number, a2: number, t: number) {
+  const oneMinusT = 1 - t
+  return (
+    3 * a1 * oneMinusT * oneMinusT +
+    6 * (a2 - a1) * oneMinusT * t +
+    3 * (1 - a2) * t * t
+  )
+}
+
+export function cubicBezier(
+  x1: number,
+  y1: number,
+  x2: number,
+  y2: number,
+  t: number,
+) {
+  const targetX = clamp01(t)
+  let solvedT = targetX
+
+  for (let i = 0; i < 8; i += 1) {
+    const currentX = sampleCubicBezier(x1, x2, solvedT) - targetX
+    const currentDerivative = sampleCubicBezierDerivative(x1, x2, solvedT)
+
+    if (Math.abs(currentX) < 1e-6 || Math.abs(currentDerivative) < 1e-6) {
+      break
+    }
+
+    solvedT -= currentX / currentDerivative
+  }
+
+  let lower = 0
+  let upper = 1
+  solvedT = clamp01(solvedT)
+
+  for (let i = 0; i < 10; i += 1) {
+    const currentX = sampleCubicBezier(x1, x2, solvedT)
+    if (Math.abs(currentX - targetX) < 1e-6) {
+      break
+    }
+
+    if (currentX < targetX) {
+      lower = solvedT
+    } else {
+      upper = solvedT
+    }
+
+    solvedT = (lower + upper) / 2
+  }
+
+  return sampleCubicBezier(y1, y2, solvedT)
+}
+
+export function easeOutScreenStudio(t: number) {
+  return cubicBezier(0.16, 1, 0.3, 1, t)
+}