@erclx/canon 5.68.0 → 5.70.0

package/src/take/plan.ts

@@ -6,19 +6,44 @@
  * keyboard leaves nothing in the log to draw.
  */
 
+import { MODIFIERS, type Modifier, isModifier, parseKeys } from '@/take/keys'
+
 /** How a beat is filmed: frame stepping, or Playwright's real-time screencast. */
 export const ROUTES = ['step', 'screencast'] as const
 
 export type Route = (typeof ROUTES)[number]
 
+/** The keys held through a press, absent when none is. */
+interface Held {
+  readonly modifiers?: readonly Modifier[]
+}
+
 export type TakeStep =
   | { readonly kind: 'move'; readonly target: string }
   | { readonly kind: 'move'; readonly x: number; readonly y: number }
-  | { readonly kind: 'click'; readonly target: string }
+  | ({ readonly kind: 'click'; readonly target: string } & Held)
+  | ({ readonly kind: 'dblclick'; readonly target: string } & Held)
+  | ({
+      readonly kind: 'drag'
+      readonly from: string
+      readonly to: string
+    } & Held)
+  | { readonly kind: 'key'; readonly keys: string }
   | { readonly kind: 'fill'; readonly target: string; readonly text: string }
   | { readonly kind: 'scroll'; readonly target: string; readonly dy: number }
   | { readonly kind: 'wait'; readonly ms: number }
 
+const KINDS = [
+  'move',
+  'click',
+  'dblclick',
+  'drag',
+  'key',
+  'fill',
+  'scroll',
+  'wait',
+] as const
+
 export interface Beat {
   readonly name: string
   /**
@@ -26,6 +51,11 @@ export interface Beat {
    * back to `screencast` on its own, and the take's record says which ran.
    */
   readonly route: Route
+  /**
+   * False turns the camera's zoom off for this beat, while its track still
+   * carries the cursor and the click ring. Absent means the zoom runs.
+   */
+  readonly push?: false
   /** Runs after the page loads and before the first frame is kept. */
   readonly setup: readonly TakeStep[]
   readonly steps: readonly TakeStep[]
@@ -89,16 +119,25 @@ function readPlan(document: unknown): TakePlan {
     throw new Refusal('unreadable-plan', 'the document is not an object')
   }
   const { beats } = document
+  const route = document.route ?? 'step'
+  if (!ROUTES.includes(route as Route)) {
+    throw new Refusal(
+      'unknown-route',
+      `the plan names route ${String(route)}. This build records on: ${ROUTES.join(', ')}.`,
+    )
+  }
   if (!Array.isArray(beats) || beats.length === 0) {
     throw new Refusal(
       'no-beats',
       'beats carries no entry, so nothing is filmed',
     )
   }
-  return { beats: beats.map((entry, index) => readBeat(entry, index)) }
+  return {
+    beats: beats.map((entry, index) => readBeat(entry, index, route as Route)),
+  }
 }
 
-function readBeat(entry: unknown, index: number): Beat {
+function readBeat(entry: unknown, index: number, fallback: Route): Beat {
   const at = `beat ${index + 1}`
   if (!isRecord(entry)) throw new Refusal('bad-beat', `${at} is not an object`)
 
@@ -110,7 +149,7 @@ function readBeat(entry: unknown, index: number): Beat {
     )
   }
 
-  const route = entry.route ?? 'step'
+  const route = entry.route ?? fallback
   if (!ROUTES.includes(route as Route)) {
     throw new Refusal(
       'unknown-route',
@@ -127,10 +166,17 @@ function readBeat(entry: unknown, index: number): Beat {
   if (entry.setup !== undefined && !Array.isArray(entry.setup)) {
     throw new Refusal('bad-beat', `${name} carries a setup that is not a list`)
   }
+  if (entry.push !== undefined && typeof entry.push !== 'boolean') {
+    throw new Refusal(
+      'bad-beat',
+      `${name} carries a push that is not true or false`,
+    )
+  }
 
   return {
     name,
     route: route as Route,
+    ...(entry.push === false ? { push: false as const } : {}),
     setup: (entry.setup ?? []).map((step: unknown, i: number) =>
       readStep(step, `${name}, setup step ${i + 1}`),
     ),
@@ -160,7 +206,35 @@ function readStep(entry: unknown, at: string): TakeStep {
         `${at} is a move with neither a target nor an x and y`,
       )
     case 'click':
-      return { kind: 'click', target: needTarget() }
+    case 'dblclick':
+      return {
+        kind: entry.kind,
+        target: needTarget(),
+        ...readModifiers(entry.modifiers, at),
+      }
+    case 'drag':
+      if (typeof entry.from !== 'string' || !entry.from) {
+        throw new Refusal('bad-step', `${at} is a drag with no from target`)
+      }
+      if (typeof entry.to !== 'string' || !entry.to) {
+        throw new Refusal('bad-step', `${at} is a drag with no to target`)
+      }
+      return {
+        kind: 'drag',
+        from: entry.from,
+        to: entry.to,
+        ...readModifiers(entry.modifiers, at),
+      }
+    case 'key': {
+      if (typeof entry.keys !== 'string') {
+        throw new Refusal('bad-step', `${at} is a key with no keys to press`)
+      }
+      const read = parseKeys(entry.keys)
+      if (read.kind === 'refused') {
+        throw new Refusal('bad-step', `${at} is a key where ${read.detail}`)
+      }
+      return { kind: 'key', keys: entry.keys }
+    }
     case 'fill':
       if (typeof entry.text !== 'string') {
         throw new Refusal('bad-step', `${at} is a fill with no text`)
@@ -179,9 +253,30 @@ function readStep(entry: unknown, at: string): TakeStep {
     default:
       throw new Refusal(
         'bad-step',
-        `${at} names kind ${String(entry.kind)}, which is not move, click, fill, scroll, or wait`,
+        `${at} names kind ${String(entry.kind)}, which is not ${KINDS.join(', ')}`,
+      )
+  }
+}
+
+function readModifiers(value: unknown, at: string): Held {
+  if (value === undefined) return {}
+  if (!Array.isArray(value)) {
+    throw new Refusal('bad-step', `${at} carries modifiers that are not a list`)
+  }
+  const modifiers: Modifier[] = []
+  for (const name of value) {
+    if (!isModifier(name)) {
+      throw new Refusal(
+        'bad-step',
+        `${at} holds ${String(name)}, which is not ${MODIFIERS.join(', ')}`,
       )
+    }
+    if (modifiers.includes(name)) {
+      throw new Refusal('bad-step', `${at} holds ${name} twice`)
+    }
+    modifiers.push(name)
   }
+  return modifiers.length > 0 ? { modifiers } : {}
 }
 
 function isRecord(value: unknown): value is Record<string, unknown> {

package/src/take/pointer.ts

@@ -2,7 +2,8 @@ import type { Point } from '@/take/log'
 
 /**
  * Draws the cursor arrow and the click ring onto one decoded frame, in the
- * shape groundwork track `193-ui-record` rendered them: a dark arrow outlined
+ * shape groundwork track `193-ui-record` rendered them (canon-keep-retired):
+ * a dark arrow outlined
  * in white, and a blue ring that grows from its tip and fades out. Both stay
  * one size whatever the frame shows, so the pointer reads the same in every
  * beat of a reel. Drawn in TypeScript rather than a browser, so the reel needs

package/src/take/screencast.ts

@@ -4,7 +4,12 @@ import { join } from 'node:path'
 import { chromium } from 'playwright-core'
 import { lastFrame, probeFrames, transcode } from '@/take/encode'
 import { FPS, type LogEvent, START, VIEWPORT } from '@/take/log'
-import { type PointerInput, type Stage, performSteps } from '@/take/perform'
+import {
+  type PointerInput,
+  type Stage,
+  TargetMissing,
+  performSteps,
+} from '@/take/perform'
 import type { Beat } from '@/take/plan'
 import { HOLD_MS } from '@/take/stepper'
 
@@ -19,6 +24,9 @@ const FRAME_MS = 1000 / FPS
 
 const SETTLE_MS = 2500
 
+/** How long a refusal waits on the screenshot it shows. */
+const STILL_CAP_MS = 2000
+
 export interface ScreencastTake {
   readonly frames: number
   readonly events: readonly LogEvent[]
@@ -33,8 +41,15 @@ export async function filmScreencast(request: {
 }): Promise<ScreencastTake> {
   const scratch = mkdtempSync(join(tmpdir(), 'canon-take-cast-'))
   const browser = await chromium.launch()
+  let still: (() => Promise<Buffer | undefined>) | undefined
   try {
     const page = await browser.newPage({ viewport: VIEWPORT })
+    // Best effort, so a closed or crashed page leaves the refusal standing
+    // without a still rather than replacing it.
+    still = () =>
+      page
+        .screenshot({ type: 'jpeg', quality: 95, timeout: STILL_CAP_MS })
+        .catch(() => undefined)
     await page.goto(request.url, { waitUntil: 'load' })
     await page.waitForTimeout(SETTLE_MS)
 
@@ -51,6 +66,13 @@ export async function filmScreencast(request: {
         await page.keyboard.insertText(text)
         await page.waitForTimeout(FRAME_MS)
       },
+      // Playwright's mouse reads the modifiers its keyboard holds, so a press
+      // here is all a modified click or drag needs on this route.
+      press: async (input) => {
+        if (input.type === 'down') await page.keyboard.down(input.key)
+        else await page.keyboard.up(input.key)
+        await page.waitForTimeout(FRAME_MS)
+      },
       evaluate: (expression) => page.evaluate(expression),
     })
 
@@ -75,6 +97,9 @@ export async function filmScreencast(request: {
     await transcode(webm, request.clip, FPS)
     const frames = await probeFrames(request.clip)
     return { frames, events, still: await lastFrame(request.clip, frames) }
+  } catch (error) {
+    if (error instanceof TargetMissing && still) error.still = await still()
+    throw error
   } finally {
     await browser.close()
     rmSync(scratch, { recursive: true, force: true })
@@ -88,9 +113,9 @@ async function send(mouse: Mouse, input: PointerInput): Promise<void> {
     case 'move':
       return mouse.move(input.x, input.y)
     case 'down':
-      return mouse.down()
+      return mouse.down({ clickCount: input.clickCount })
     case 'up':
-      return mouse.up()
+      return mouse.up({ clickCount: input.clickCount })
     case 'wheel':
       return mouse.wheel(0, input.dy)
   }

package/src/take/stepper.ts

@@ -1,6 +1,13 @@
 import { openFrameEncoder } from '@/take/encode'
+import { describeKey, modifierBits } from '@/take/keys'
 import { FPS, type LogEvent, START, VIEWPORT } from '@/take/log'
-import { type PointerInput, type Stage, performSteps } from '@/take/perform'
+import {
+  type KeyInput,
+  type PointerInput,
+  type Stage,
+  TargetMissing,
+  performSteps,
+} from '@/take/perform'
 import type { Beat } from '@/take/plan'
 import { type ShellPage, openShell } from '@/take/shell'
 
@@ -36,10 +43,16 @@ const LOAD_CAP_MS = 15_000
 const SETTLE_MS = 2500
 const LOADING_FRAME_CAP_MS = 250
 
+/** How long a refusal waits on the frame it shows, since the page may be wedged. */
+const STILL_CAP_MS = 2000
+
 /** A frame that took longer than `STALL_MS`, so the beat is filmed another way. */
 export class StepperStalled extends Error {
-  constructor(readonly frame: number) {
-    super(`frame ${frame} took over ${STALL_MS / 1000} s under frame stepping`)
+  constructor(
+    readonly frame: number,
+    stallMs: number,
+  ) {
+    super(`frame ${frame} took over ${stallMs / 1000} s under frame stepping`)
   }
 }
 
@@ -55,6 +68,8 @@ export interface SteppedRequest {
   readonly url: string
   readonly beat: Beat
   readonly clip: string
+  /** How long one frame may take before the beat falls back, `STALL_MS` by default. */
+  readonly stallMs?: number
 }
 
 export async function filmStepped(
@@ -62,6 +77,8 @@ export async function filmStepped(
 ): Promise<SteppedTake> {
   const page = await openShell(request.executable, VIEWPORT)
   const encoder = openFrameEncoder(request.clip, FPS)
+  const stallMs = request.stallMs ?? STALL_MS
+  let still: (() => Promise<Buffer | undefined>) | undefined
   try {
     let ticks = await load(page, request.url)
     const beginFrame = (screenshot: boolean) =>
@@ -75,9 +92,10 @@ export async function filmStepped(
             : {}),
         },
       )
+    still = () => frameStill(beginFrame)
     await pump(beginFrame, SETTLE_MS)
 
-    const live = liveStage(page, beginFrame)
+    const live = liveStage(page, beginFrame, stallMs)
     await performSteps(live, request.beat.setup, { pointer: START })
     if (request.beat.setup.length > 0) await live.hold(1000)
     await live.pointer({ type: 'move', ...START })
@@ -106,6 +124,7 @@ export async function filmStepped(
       const reply = await withinStall(
         advanced.then(() => beginFrame(true)),
         frames,
+        stallMs,
       )
       const data = reply.screenshotData
         ? Buffer.from(reply.screenshotData, 'base64')
@@ -121,16 +140,14 @@ export async function filmStepped(
       hold: async (ms) => {
         for (let i = 0; i < Math.round(ms / FRAME_MS); i += 1) await frame()
       },
-      pointer: async (input) => {
-        const sent = page.send('Input.dispatchMouseEvent', mouseEvent(input))
-        await frame()
-        await sent
-      },
-      key: async (text) => {
-        const sent = page.send('Input.insertText', { text })
-        await frame()
-        await sent
-      },
+      pointer: (input) =>
+        stepThen(
+          page.send('Input.dispatchMouseEvent', mouseEvent(input)),
+          frame,
+        ),
+      key: (text) => stepThen(page.send('Input.insertText', { text }), frame),
+      press: (input) =>
+        stepThen(page.send('Input.dispatchKeyEvent', keyEvent(input)), frame),
       evaluate: (expression) => page.evaluate(expression),
     }
 
@@ -146,12 +163,31 @@ export async function filmStepped(
     return { frames, events, still: last }
   } catch (error) {
     encoder.abort()
+    if (error instanceof TargetMissing && still) error.still = await still()
     throw error
   } finally {
     page.close()
   }
 }
 
+/**
+ * Begins one frame with a screenshot for a refusal to show, through the same
+ * begin-frame call every kept frame uses, since a separate capture call can
+ * wait on a frame that never begins. Best effort, so a page that answers with
+ * nothing leaves the refusal standing without a still.
+ */
+async function frameStill(
+  beginFrame: (screenshot: boolean) => Promise<{ screenshotData?: string }>,
+): Promise<Buffer | undefined> {
+  const reply = await Promise.race([
+    beginFrame(true).catch(() => undefined),
+    sleep(STILL_CAP_MS),
+  ])
+  return reply?.screenshotData
+    ? Buffer.from(reply.screenshotData, 'base64')
+    : undefined
+}
+
 /**
  * Navigates and resolves with Chromium's monotonic clock in milliseconds, read
  * off the load event's own timestamp. The runtime's clock cannot stand in for
@@ -201,6 +237,7 @@ async function pump(
 function liveStage(
   page: ShellPage,
   beginFrame: (screenshot: boolean) => Promise<unknown>,
+  stallMs: number,
 ): Stage {
   const started = Date.now()
   return {
@@ -209,40 +246,121 @@ function liveStage(
     pointer: (input) => {
       const sent = page.send('Input.dispatchMouseEvent', mouseEvent(input))
       return withinStall(
-        beginFrame(false).then(() => sent),
+        Promise.all([beginFrame(false), sent]),
         0,
+        stallMs,
       ).then(() => {})
     },
     key: (text) => {
       const sent = page.send('Input.insertText', { text })
       return withinStall(
-        beginFrame(false).then(() => sent),
+        Promise.all([beginFrame(false), sent]),
         0,
+        stallMs,
+      ).then(() => {})
+    },
+    press: (input) => {
+      const sent = page.send('Input.dispatchKeyEvent', keyEvent(input))
+      return withinStall(
+        Promise.all([beginFrame(false), sent]),
+        0,
+        stallMs,
       ).then(() => {})
     },
     evaluate: (expression) => page.evaluate(expression),
   }
 }
 
+/**
+ * A pointer input as CDP takes it. `buttons` carries the held button through a
+ * drag's moves, or the page reads them as a hover, and `modifiers` carries the
+ * held keys as a bit field beside the key events that pressed them.
+ */
 function mouseEvent(input: PointerInput): Record<string, unknown> {
   const at = { x: input.x, y: input.y }
+  if (input.type === 'wheel') {
+    return { type: 'mouseWheel', ...at, deltaX: 0, deltaY: input.dy }
+  }
+  const modifiers = modifierBits(input.modifiers ?? [])
   switch (input.type) {
     case 'move':
-      return { type: 'mouseMoved', ...at, button: 'none' }
+      return input.held
+        ? { type: 'mouseMoved', ...at, button: 'left', buttons: 1, modifiers }
+        : { type: 'mouseMoved', ...at, button: 'none', buttons: 0, modifiers }
     case 'down':
-      return { type: 'mousePressed', ...at, button: 'left', clickCount: 1 }
     case 'up':
-      return { type: 'mouseReleased', ...at, button: 'left', clickCount: 1 }
-    case 'wheel':
-      return { type: 'mouseWheel', ...at, deltaX: 0, deltaY: input.dy }
+      return {
+        type: input.type === 'down' ? 'mousePressed' : 'mouseReleased',
+        ...at,
+        button: 'left',
+        buttons: input.type === 'down' ? 1 : 0,
+        clickCount: input.clickCount,
+        modifiers,
+      }
+  }
+}
+
+/**
+ * A key input as CDP takes it. A key that types sends `keyDown` with its text
+ * only when no modifier but Shift is held, as Chromium's own keyboard does, so
+ * `Control+K` in a focused field inserts no K.
+ */
+function keyEvent(input: KeyInput): Record<string, unknown> {
+  const descriptor = describeKey(input.key, {
+    shift: input.modifiers.includes('Shift'),
+  })
+  if (!descriptor) throw new Error(`no key descriptor for ${input.key}`)
+  const event = {
+    key: descriptor.key,
+    code: descriptor.code,
+    windowsVirtualKeyCode: descriptor.keyCode,
+    nativeVirtualKeyCode: descriptor.keyCode,
+    location: descriptor.location,
+    modifiers: modifierBits(input.modifiers),
+  }
+  if (input.type === 'up') return { type: 'keyUp', ...event }
+  const types =
+    descriptor.text !== undefined &&
+    input.modifiers.every((modifier) => modifier === 'Shift')
+  return types
+    ? {
+        type: 'keyDown',
+        ...event,
+        text: descriptor.text,
+        unmodifiedText: descriptor.text,
+      }
+    : { type: 'rawKeyDown', ...event }
+}
+
+/**
+ * Steps a frame, then awaits the input sent ahead of it. When the step fails,
+ * the input is left settling against a shell about to close, so its rejection
+ * is handled here rather than surfacing as an unhandled one.
+ */
+async function stepThen(
+  sent: Promise<unknown>,
+  step: () => Promise<void>,
+): Promise<void> {
+  try {
+    await step()
+  } catch (error) {
+    sent.catch(() => {})
+    throw error
   }
+  await sent
 }
 
-async function withinStall<T>(work: Promise<T>, frame: number): Promise<T> {
+async function withinStall<T>(
+  work: Promise<T>,
+  frame: number,
+  stallMs: number,
+): Promise<T> {
   let timer: ReturnType<typeof setTimeout> | undefined
   const stalled = new Promise<never>((_, fail) => {
-    timer = setTimeout(() => fail(new StepperStalled(frame)), STALL_MS)
+    timer = setTimeout(() => fail(new StepperStalled(frame, stallMs)), stallMs)
   })
+  // A stall leaves the work pending until the shell closes and rejects it.
+  work.catch(() => {})
   try {
     return await Promise.race([work, stalled])
   } finally {

package/src/take/zoom/NOTICE.md

@@ -15,7 +15,7 @@ The files in this folder carry the zoom timing of [OpenScreen](https://github.co
 - The file names are kebab-case, as this repository names every TypeScript file, and the code is reformatted to its style
 - `math-utils.ts` keeps `clamp01`, `cubicBezier`, and `easeOutScreenStudio`, and drops the easings nothing here calls
 - `region-utils.ts` keeps `computeRegionStrength`, the connected-pan easing, and the window and gap constants. It drops the dominant-region search, its cache, the 3D rotation, and the cursor-follow focus, since the camera here aims at clicks and holds its focus by its own rule.
-- The zoom-out window is 700 ms rather than 1015.05 ms, and the connected pan runs 600 ms rather than 1000 ms, both on the operator's eye in groundwork track `193-ui-record`. The zoom-in window keeps OpenScreen's 1522.575 ms.
+- The zoom-out window is 700 ms rather than 1015.05 ms, and the connected pan runs 600 ms rather than 1000 ms, both on the operator's eye in groundwork track `193-ui-record`. <!-- canon-keep-retired --> The zoom-in window keeps OpenScreen's 1522.575 ms.
 - `spring.ts` keeps the zoom spring's stiffness 320, damping 40, mass 0.92, its rest thresholds, the delta clamp, and the overshoot clamp from `stepAxis`. OpenScreen steps that spring through the `motion` package, and this solves it in closed form instead, so no dependency comes with it.
 
 ## License