@erclx/canon 5.69.0 → 5.70.0

package/src/take/log.ts

@@ -1,4 +1,5 @@
-import type { Route } from '@/take/plan'
+import type { Modifier } from '@/take/keys'
+import { ROUTES, type Route } from '@/take/plan'
 
 /**
  * The click log a take writes beside its clip, and the pointer path both
@@ -43,6 +44,33 @@ export type LogEvent =
       readonly kind: 'click'
       readonly at: Point
       readonly box: Box
+      /** The keys held through the press, absent when none is. */
+      readonly modifiers?: readonly Modifier[]
+      /** 2 for a double click, absent for a single one. */
+      readonly count?: number
+    }
+  | {
+      readonly t: number
+      readonly kind: 'drag'
+      /** Where the button went down and where it came up. */
+      readonly from: Point
+      readonly to: Point
+      /** The box pressed and the box it was dropped on. */
+      readonly box: Box
+      readonly drop: Box
+      readonly modifiers?: readonly Modifier[]
+      /**
+       * From press to release. The travel between is logged as `move` events,
+       * which the cursor is read off.
+       */
+      readonly ms: number
+    }
+  | {
+      readonly t: number
+      readonly kind: 'key'
+      /** The combination as the plan wrote it, such as `Control+K`. */
+      readonly keys: string
+      readonly ms: number
     }
   | {
       readonly t: number
@@ -68,10 +96,75 @@ export interface TakeLog {
   readonly fps: number
   /** The route that filmed the clip, which can differ from the one asked for. */
   readonly route: Route
+  /**
+   * The clip's size in pixels, which sets how deep the camera may zoom before
+   * the clip goes soft. A log written before this field read it off `route`.
+   */
+  readonly clip?: { readonly width: number; readonly height: number }
+  /** False when the beat turned the camera's zoom off. */
+  readonly push?: false
   readonly frames: number
   readonly events: readonly LogEvent[]
 }
 
+/** The size each route films at, the stepper at twice the viewport. */
+export const CLIP_SIZES: Record<
+  Route,
+  { readonly width: number; readonly height: number }
+> = {
+  step: { width: VIEWPORT.width * 2, height: VIEWPORT.height * 2 },
+  screencast: VIEWPORT,
+}
+
+export type TakeLogRead =
+  | { readonly kind: 'read'; readonly log: TakeLog }
+  | { readonly kind: 'refused'; readonly detail: string }
+
+/**
+ * Reads a log back off disk, checking the fields the camera builds on rather
+ * than every event, since a take wrote the events and a reader only ever
+ * points at the wrong file.
+ */
+export function readTakeLog(source: string): TakeLogRead {
+  let document: unknown
+  try {
+    document = JSON.parse(source)
+  } catch (error) {
+    return {
+      kind: 'refused',
+      detail: error instanceof Error ? error.message : String(error),
+    }
+  }
+  if (typeof document !== 'object' || document === null) {
+    return { kind: 'refused', detail: 'the document is not an object' }
+  }
+  const log = document as Partial<Record<keyof TakeLog, unknown>>
+  const missing = [
+    isSize(log.viewport) ? '' : 'viewport',
+    isCount(log.fps) && log.fps > 0 ? '' : 'fps',
+    ROUTES.includes(log.route as Route) ? '' : 'route',
+    isCount(log.frames) ? '' : 'frames',
+    Array.isArray(log.events) ? '' : 'events',
+  ].filter(Boolean)
+  if (missing.length > 0) {
+    return {
+      kind: 'refused',
+      detail: `the document carries no valid ${missing.join(', ')}, so it is not a take's log.json`,
+    }
+  }
+  return { kind: 'read', log: document as TakeLog }
+}
+
+function isCount(value: unknown): value is number {
+  return Number.isInteger(value) && (value as number) >= 0
+}
+
+function isSize(value: unknown): boolean {
+  if (typeof value !== 'object' || value === null) return false
+  const { width, height } = value as Record<string, unknown>
+  return isCount(width) && isCount(height)
+}
+
 /** Where the pointer rests before a take's first move. */
 export const START: Point = { x: VIEWPORT.width / 2, y: VIEWPORT.height / 2 }
 

package/src/take/perform.ts

@@ -6,6 +6,7 @@ import {
   along,
   travelMs,
 } from '@/take/log'
+import { type Chord, type Modifier, isModifier, parseKeys } from '@/take/keys'
 import type { TakeStep } from '@/take/plan'
 
 /**
@@ -23,11 +24,25 @@ const KEY_MS = 90
 /** How long a scroll's wheel travel runs. */
 const SCROLL_MS = 900
 
+/** How long a drag holds still after its press, so the viewer sees the grab land. */
+const GRAB_MS = 150
+
 export type PointerInput =
   | {
-      readonly type: 'move' | 'down' | 'up'
+      readonly type: 'move'
+      readonly x: number
+      readonly y: number
+      /** True while a drag holds the left button down. */
+      readonly held?: true
+      readonly modifiers?: readonly Modifier[]
+    }
+  | {
+      readonly type: 'down' | 'up'
       readonly x: number
       readonly y: number
+      /** Which press of a run this is, 2 being a double click's second. */
+      readonly clickCount: number
+      readonly modifiers?: readonly Modifier[]
     }
   | {
       readonly type: 'wheel'
@@ -36,6 +51,14 @@ export type PointerInput =
       readonly dy: number
     }
 
+/** One key going down or coming up. */
+export interface KeyInput {
+  readonly type: 'down' | 'up'
+  readonly key: string
+  /** The modifiers held once this input lands, so a modifier's own down counts it. */
+  readonly modifiers: readonly Modifier[]
+}
+
 export interface Stage {
   /** Milliseconds on the clip's own clock. */
   now(): number
@@ -45,12 +68,17 @@ export interface Stage {
   pointer(input: PointerInput): Promise<void>
   /** Types one character into the focused field and lets one frame pass. */
   key(text: string): Promise<void>
+  /** Sends one key down or up and lets one frame pass. */
+  press(input: KeyInput): Promise<void>
   /** Evaluates an expression in the page. */
   evaluate<T>(expression: string): Promise<T>
 }
 
 /** A target that resolved to nothing, which refuses rather than films a guess. */
 export class TargetMissing extends Error {
+  /** The page as it stood when the lookup failed, as a JPEG, when the route caught one. */
+  still?: Buffer
+
   constructor(readonly target: string) {
     super(`nothing on the page matches ${target}`)
   }
@@ -77,11 +105,40 @@ export async function performSteps(
         )
         break
       case 'click':
-        await click(stage, state, await boxOf(stage, step.target))
+      case 'dblclick':
+        await click(
+          stage,
+          state,
+          await boxOf(stage, step.target),
+          step.modifiers ?? [],
+          step.kind === 'dblclick' ? 2 : 1,
+        )
         break
+      case 'drag':
+        await drag(
+          stage,
+          state,
+          await boxOf(stage, step.from),
+          await boxOf(stage, step.to),
+          step.modifiers ?? [],
+        )
+        break
+      case 'key': {
+        const read = parseKeys(step.keys)
+        if (read.kind === 'refused') throw new Error(read.detail)
+        const start = stage.now()
+        await pressChord(stage, read.chord)
+        state.log?.push({
+          t: start,
+          kind: 'key',
+          keys: step.keys,
+          ms: stage.now() - start,
+        })
+        break
+      }
       case 'fill': {
         const box = await boxOf(stage, step.target)
-        await click(stage, state, box)
+        await click(stage, state, box, [], 1)
         const start = stage.now()
         for (const character of step.text) {
           await stage.key(character)
@@ -132,6 +189,7 @@ async function moveTo(
   stage: Stage,
   state: Performance,
   to: Point,
+  held?: { readonly modifiers: readonly Modifier[] },
 ): Promise<void> {
   const from = state.pointer
   if (Math.hypot(to.x - from.x, to.y - from.y) < 1) return
@@ -143,9 +201,12 @@ async function moveTo(
     to,
     ms: frames * FRAME_MS,
   })
+  const pressed = held
+    ? { held: true as const, ...withModifiers(held.modifiers) }
+    : {}
   for (let frame = 1; frame <= frames; frame += 1) {
     state.pointer = along(from, to, frame / frames)
-    await stage.pointer({ type: 'move', ...state.pointer })
+    await stage.pointer({ type: 'move', ...state.pointer, ...pressed })
   }
   state.pointer = to
 }
@@ -154,12 +215,100 @@ async function click(
   stage: Stage,
   state: Performance,
   box: Box,
+  modifiers: readonly Modifier[],
+  count: number,
 ): Promise<void> {
   const at = center(box)
   await moveTo(stage, state, at)
-  state.log?.push({ t: stage.now(), kind: 'click', at, box })
-  await stage.pointer({ type: 'down', ...at })
-  await stage.pointer({ type: 'up', ...at })
+  await holding(stage, modifiers, async () => {
+    state.log?.push({
+      t: stage.now(),
+      kind: 'click',
+      at,
+      box,
+      ...withModifiers(modifiers),
+      ...(count > 1 ? { count } : {}),
+    })
+    for (let clickCount = 1; clickCount <= count; clickCount += 1) {
+      const press = { ...at, clickCount, ...withModifiers(modifiers) }
+      await stage.pointer({ type: 'down', ...press })
+      await stage.pointer({ type: 'up', ...press })
+    }
+  })
+}
+
+/**
+ * Presses on the source, holds a moment so the page reads a grab rather than
+ * a click, glides held to the target, and releases there. The drop target is
+ * read before the press, so a page that moves it once a drag starts is
+ * dropped on where it stood.
+ */
+async function drag(
+  stage: Stage,
+  state: Performance,
+  box: Box,
+  drop: Box,
+  modifiers: readonly Modifier[],
+): Promise<void> {
+  const from = center(box)
+  const to = center(drop)
+  await moveTo(stage, state, from)
+  await holding(stage, modifiers, async () => {
+    const start = stage.now()
+    const press = { clickCount: 1, ...withModifiers(modifiers) }
+    await stage.pointer({ type: 'down', ...from, ...press })
+    await stage.hold(GRAB_MS)
+    await moveTo(stage, state, to, { modifiers })
+    await stage.pointer({ type: 'up', ...to, ...press })
+    state.log?.push({
+      t: start,
+      kind: 'drag',
+      from,
+      to,
+      box,
+      drop,
+      ...withModifiers(modifiers),
+      ms: stage.now() - start,
+    })
+  })
+}
+
+/**
+ * Presses each modifier in order, runs the inputs they hold, and releases them
+ * in reverse, so a page reads every input between as modified on both routes.
+ */
+async function holding(
+  stage: Stage,
+  modifiers: readonly Modifier[],
+  inputs: () => Promise<void>,
+): Promise<void> {
+  for (const [i, key] of modifiers.entries()) {
+    await stage.press({
+      type: 'down',
+      key,
+      modifiers: modifiers.slice(0, i + 1),
+    })
+  }
+  await inputs()
+  for (const [i, key] of [...modifiers.entries()].reverse()) {
+    await stage.press({ type: 'up', key, modifiers: modifiers.slice(0, i) })
+  }
+}
+
+/** Presses a chord's key inside its modifiers, counting a modifier key as held while down. */
+async function pressChord(stage: Stage, chord: Chord): Promise<void> {
+  const { key, modifiers } = chord
+  await holding(stage, modifiers, async () => {
+    const down = isModifier(key) ? [...modifiers, key] : modifiers
+    await stage.press({ type: 'down', key, modifiers: down })
+    await stage.press({ type: 'up', key, modifiers })
+  })
+}
+
+function withModifiers(modifiers: readonly Modifier[]): {
+  modifiers?: readonly Modifier[]
+} {
+  return modifiers.length > 0 ? { modifiers } : {}
 }
 
 function center(box: Box): Point {
@@ -173,8 +322,10 @@ async function boxOf(stage: Stage, target: string): Promise<Box> {
 }
 
 /**
- * Finds a target's box in CSS pixels, reading `text=<words>` as the innermost
- * element whose whole text is those words and anything else as a CSS selector.
+ * Finds a target's box in CSS pixels, reading `text=<words>` as the first
+ * innermost element with a size whose whole text is those words, so a hidden
+ * copy earlier in the page cannot shadow a visible one, and anything else as a
+ * CSS selector.
  * It runs inside the page so both routes resolve a target the same way, and a
  * match with no size counts as missing, since a click on it would land on
  * whatever sits behind.
@@ -188,7 +339,11 @@ export function boxExpression(target: string): string {
       const matches = [...document.querySelectorAll('body *')].filter(
         (node) => node.textContent.trim() === want,
       )
-      element = matches.find((node) => !matches.some((other) => other !== node && node.contains(other))) ?? null
+      const innermost = matches.filter((node) => !matches.some((other) => other !== node && node.contains(other)))
+      element = innermost.find((node) => {
+        const rect = node.getBoundingClientRect()
+        return rect.width > 0 && rect.height > 0
+      }) ?? null
     } else {
       element = document.querySelector(target)
     }

package/src/take/plan.ts

@@ -6,19 +6,44 @@
  * keyboard leaves nothing in the log to draw.
  */
 
+import { MODIFIERS, type Modifier, isModifier, parseKeys } from '@/take/keys'
+
 /** How a beat is filmed: frame stepping, or Playwright's real-time screencast. */
 export const ROUTES = ['step', 'screencast'] as const
 
 export type Route = (typeof ROUTES)[number]
 
+/** The keys held through a press, absent when none is. */
+interface Held {
+  readonly modifiers?: readonly Modifier[]
+}
+
 export type TakeStep =
   | { readonly kind: 'move'; readonly target: string }
   | { readonly kind: 'move'; readonly x: number; readonly y: number }
-  | { readonly kind: 'click'; readonly target: string }
+  | ({ readonly kind: 'click'; readonly target: string } & Held)
+  | ({ readonly kind: 'dblclick'; readonly target: string } & Held)
+  | ({
+      readonly kind: 'drag'
+      readonly from: string
+      readonly to: string
+    } & Held)
+  | { readonly kind: 'key'; readonly keys: string }
   | { readonly kind: 'fill'; readonly target: string; readonly text: string }
   | { readonly kind: 'scroll'; readonly target: string; readonly dy: number }
   | { readonly kind: 'wait'; readonly ms: number }
 
+const KINDS = [
+  'move',
+  'click',
+  'dblclick',
+  'drag',
+  'key',
+  'fill',
+  'scroll',
+  'wait',
+] as const
+
 export interface Beat {
   readonly name: string
   /**
@@ -26,6 +51,11 @@ export interface Beat {
    * back to `screencast` on its own, and the take's record says which ran.
    */
   readonly route: Route
+  /**
+   * False turns the camera's zoom off for this beat, while its track still
+   * carries the cursor and the click ring. Absent means the zoom runs.
+   */
+  readonly push?: false
   /** Runs after the page loads and before the first frame is kept. */
   readonly setup: readonly TakeStep[]
   readonly steps: readonly TakeStep[]
@@ -89,16 +119,25 @@ function readPlan(document: unknown): TakePlan {
     throw new Refusal('unreadable-plan', 'the document is not an object')
   }
   const { beats } = document
+  const route = document.route ?? 'step'
+  if (!ROUTES.includes(route as Route)) {
+    throw new Refusal(
+      'unknown-route',
+      `the plan names route ${String(route)}. This build records on: ${ROUTES.join(', ')}.`,
+    )
+  }
   if (!Array.isArray(beats) || beats.length === 0) {
     throw new Refusal(
       'no-beats',
       'beats carries no entry, so nothing is filmed',
     )
   }
-  return { beats: beats.map((entry, index) => readBeat(entry, index)) }
+  return {
+    beats: beats.map((entry, index) => readBeat(entry, index, route as Route)),
+  }
 }
 
-function readBeat(entry: unknown, index: number): Beat {
+function readBeat(entry: unknown, index: number, fallback: Route): Beat {
   const at = `beat ${index + 1}`
   if (!isRecord(entry)) throw new Refusal('bad-beat', `${at} is not an object`)
 
@@ -110,7 +149,7 @@ function readBeat(entry: unknown, index: number): Beat {
     )
   }
 
-  const route = entry.route ?? 'step'
+  const route = entry.route ?? fallback
   if (!ROUTES.includes(route as Route)) {
     throw new Refusal(
       'unknown-route',
@@ -127,10 +166,17 @@ function readBeat(entry: unknown, index: number): Beat {
   if (entry.setup !== undefined && !Array.isArray(entry.setup)) {
     throw new Refusal('bad-beat', `${name} carries a setup that is not a list`)
   }
+  if (entry.push !== undefined && typeof entry.push !== 'boolean') {
+    throw new Refusal(
+      'bad-beat',
+      `${name} carries a push that is not true or false`,
+    )
+  }
 
   return {
     name,
     route: route as Route,
+    ...(entry.push === false ? { push: false as const } : {}),
     setup: (entry.setup ?? []).map((step: unknown, i: number) =>
       readStep(step, `${name}, setup step ${i + 1}`),
     ),
@@ -160,7 +206,35 @@ function readStep(entry: unknown, at: string): TakeStep {
         `${at} is a move with neither a target nor an x and y`,
       )
     case 'click':
-      return { kind: 'click', target: needTarget() }
+    case 'dblclick':
+      return {
+        kind: entry.kind,
+        target: needTarget(),
+        ...readModifiers(entry.modifiers, at),
+      }
+    case 'drag':
+      if (typeof entry.from !== 'string' || !entry.from) {
+        throw new Refusal('bad-step', `${at} is a drag with no from target`)
+      }
+      if (typeof entry.to !== 'string' || !entry.to) {
+        throw new Refusal('bad-step', `${at} is a drag with no to target`)
+      }
+      return {
+        kind: 'drag',
+        from: entry.from,
+        to: entry.to,
+        ...readModifiers(entry.modifiers, at),
+      }
+    case 'key': {
+      if (typeof entry.keys !== 'string') {
+        throw new Refusal('bad-step', `${at} is a key with no keys to press`)
+      }
+      const read = parseKeys(entry.keys)
+      if (read.kind === 'refused') {
+        throw new Refusal('bad-step', `${at} is a key where ${read.detail}`)
+      }
+      return { kind: 'key', keys: entry.keys }
+    }
     case 'fill':
       if (typeof entry.text !== 'string') {
         throw new Refusal('bad-step', `${at} is a fill with no text`)
@@ -179,9 +253,30 @@ function readStep(entry: unknown, at: string): TakeStep {
     default:
       throw new Refusal(
         'bad-step',
-        `${at} names kind ${String(entry.kind)}, which is not move, click, fill, scroll, or wait`,
+        `${at} names kind ${String(entry.kind)}, which is not ${KINDS.join(', ')}`,
+      )
+  }
+}
+
+function readModifiers(value: unknown, at: string): Held {
+  if (value === undefined) return {}
+  if (!Array.isArray(value)) {
+    throw new Refusal('bad-step', `${at} carries modifiers that are not a list`)
+  }
+  const modifiers: Modifier[] = []
+  for (const name of value) {
+    if (!isModifier(name)) {
+      throw new Refusal(
+        'bad-step',
+        `${at} holds ${String(name)}, which is not ${MODIFIERS.join(', ')}`,
       )
+    }
+    if (modifiers.includes(name)) {
+      throw new Refusal('bad-step', `${at} holds ${name} twice`)
+    }
+    modifiers.push(name)
   }
+  return modifiers.length > 0 ? { modifiers } : {}
 }
 
 function isRecord(value: unknown): value is Record<string, unknown> {

package/src/take/screencast.ts

@@ -4,7 +4,12 @@ import { join } from 'node:path'
 import { chromium } from 'playwright-core'
 import { lastFrame, probeFrames, transcode } from '@/take/encode'
 import { FPS, type LogEvent, START, VIEWPORT } from '@/take/log'
-import { type PointerInput, type Stage, performSteps } from '@/take/perform'
+import {
+  type PointerInput,
+  type Stage,
+  TargetMissing,
+  performSteps,
+} from '@/take/perform'
 import type { Beat } from '@/take/plan'
 import { HOLD_MS } from '@/take/stepper'
 
@@ -19,6 +24,9 @@ const FRAME_MS = 1000 / FPS
 
 const SETTLE_MS = 2500
 
+/** How long a refusal waits on the screenshot it shows. */
+const STILL_CAP_MS = 2000
+
 export interface ScreencastTake {
   readonly frames: number
   readonly events: readonly LogEvent[]
@@ -33,8 +41,15 @@ export async function filmScreencast(request: {
 }): Promise<ScreencastTake> {
   const scratch = mkdtempSync(join(tmpdir(), 'canon-take-cast-'))
   const browser = await chromium.launch()
+  let still: (() => Promise<Buffer | undefined>) | undefined
   try {
     const page = await browser.newPage({ viewport: VIEWPORT })
+    // Best effort, so a closed or crashed page leaves the refusal standing
+    // without a still rather than replacing it.
+    still = () =>
+      page
+        .screenshot({ type: 'jpeg', quality: 95, timeout: STILL_CAP_MS })
+        .catch(() => undefined)
     await page.goto(request.url, { waitUntil: 'load' })
     await page.waitForTimeout(SETTLE_MS)
 
@@ -51,6 +66,13 @@ export async function filmScreencast(request: {
         await page.keyboard.insertText(text)
         await page.waitForTimeout(FRAME_MS)
       },
+      // Playwright's mouse reads the modifiers its keyboard holds, so a press
+      // here is all a modified click or drag needs on this route.
+      press: async (input) => {
+        if (input.type === 'down') await page.keyboard.down(input.key)
+        else await page.keyboard.up(input.key)
+        await page.waitForTimeout(FRAME_MS)
+      },
       evaluate: (expression) => page.evaluate(expression),
     })
 
@@ -75,6 +97,9 @@ export async function filmScreencast(request: {
     await transcode(webm, request.clip, FPS)
     const frames = await probeFrames(request.clip)
     return { frames, events, still: await lastFrame(request.clip, frames) }
+  } catch (error) {
+    if (error instanceof TargetMissing && still) error.still = await still()
+    throw error
   } finally {
     await browser.close()
     rmSync(scratch, { recursive: true, force: true })
@@ -88,9 +113,9 @@ async function send(mouse: Mouse, input: PointerInput): Promise<void> {
     case 'move':
       return mouse.move(input.x, input.y)
     case 'down':
-      return mouse.down()
+      return mouse.down({ clickCount: input.clickCount })
     case 'up':
-      return mouse.up()
+      return mouse.up({ clickCount: input.clickCount })
     case 'wheel':
       return mouse.wheel(0, input.dy)
   }