@erclx/aitk 3.11.2 → 3.12.0

package/claude/.claude-plugin/plugin.json

@@ -1,7 +1,7 @@
 {
   "name": "aitk",
   "description": "Automated governance, versioning, and discovery tools for Claude Code.",
-  "version": "3.11.2",
+  "version": "3.12.0",
   "author": {
     "name": "Eric Le",
     "url": "https://github.com/erclx"

package/docs/agents/demo.md

@@ -34,6 +34,10 @@ The plan is committed, not scratch. Its timing is a starting point tuned by watc
 
 `aitk demo run` reads the plan, refuses if a field is still empty, and drives the application the plan's URL names. It records the whole run to `webm` and writes the still from the beat the draft calls the hero, falling back to the last beat, since a demo's final state is the payoff and a cold open is usually an empty screen.
 
+A step's caption from the draft renders as an overlay while its hold plays, so the narration a person wrote is what shows on screen rather than the name of the action the engine performed.
+
+When `ffmpeg` is on PATH, the run also writes an mp4 beside the webm, since webm plays in a `<video>` tag but nothing else accepts it. A target without `ffmpeg` still gets the webm and a line naming what to install, and the run does not fail over the missing converter.
+
 | Option           | Behavior                                                    |
 | ---------------- | ----------------------------------------------------------- |
 | `--out <dir>`    | Directory to write into, overriding what the plan names     |
@@ -48,7 +52,7 @@ A step waits on its `waitFor` selector becoming visible and then holds for its o
 
 The browser engine's own annotation draws a red dot at the moment of a click and an action label in a corner. It paints no cursor, so a run without more looks like the pointer teleports between targets.
 
-The recorder injects a pointer element before navigation and moves it through the engine's pointer with interpolated steps rather than through the element-clicking helper, which resolves a target and jumps to it. The step count is the whole difference between a cursor that travels and one that appears. The pointer reads the element under it on every move and switches between an arrow, a hand, and a text beam, so it reflects the page the way a real cursor does.
+The recorder injects a pointer element before navigation and moves it through the engine's pointer with interpolated steps rather than through the element-clicking helper, which resolves a target and jumps to it. The step count is the whole difference between a cursor that travels and one that appears, and how many steps a move takes is derived from the machine's own round-trip cost rather than fixed, so the same plan glides at roughly the same pace on a loaded machine as an idle one. It also reads the element under it on every move and switches between an arrow, a hand, and a text beam, so it reflects the page the way a real cursor does.
 
 `--cursor` points at a folder of Windows cursor resources and the browser decodes them directly, with no conversion step and no image tooling. Each resource carries a hotspot per size, and the largest entry's hotspot scaled to the drawn size is what puts the artwork's tip where the click lands. A theme contributes per state, so a folder holding an arrow and no hand still supplies its arrow and the bundled artwork covers the rest.
 
@@ -77,6 +81,6 @@ The cost is stated rather than hidden: the browser binary installs separately, s
 
 ## What it does not do
 
-A generated recording is a raw take. Nothing trims it, scores it, or narrates it, and the beat's caption does not reach the video, since the engine's overlay carries the action it performed rather than text from the draft.
+A generated recording is a raw take. Nothing trims it or scores it beyond the caption each beat already carries.
 
 It also does not replace a narrated screencast. That has a hero moment, a cut list, and a voice, none of which survives being generated. This answers the case where the alternative is recording nothing.

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@erclx/aitk",
   "type": "module",
-  "version": "3.11.2",
+  "version": "3.12.0",
   "description": "Infrastructure and quality tooling for developer workflows",
   "license": "MIT",
   "bin": {

package/src/commands/demo.ts

@@ -3,6 +3,7 @@ import { basename, dirname, extname, join, relative, resolve } from 'node:path'
 import type { Command } from 'commander'
 import { parseDraft } from '@/demo/beats'
 import { compilePlan, parsePlan, unresolved } from '@/demo/compile'
+import { convertToMp4, INSTALL_CONVERTER } from '@/demo/container'
 import { DEFAULT_CURSORS } from '@/demo/cursors'
 import { loadCursorTheme } from '@/demo/theme'
 import { intro, logError, logInfo, logStep, logWarn, outro, plural } from '@/ui'
@@ -89,6 +90,9 @@ export function register(program: Command): void {
         'Needs a browser binary. Install it with:',
         `  ${INSTALL_BROWSER}`,
         '',
+        'Writes mp4 beside the webm when ffmpeg is on PATH, and skips it',
+        `otherwise without failing the run. Install it with: ${INSTALL_CONVERTER}`,
+        '',
         'Exit codes:',
         '  0  the recording and the still were written',
         '  1  refused, with the reason on stderr',
@@ -283,14 +287,35 @@ async function runDrive(planPath: string, opts: RunOptions): Promise<number> {
   logInfo(
     `${result.steps} steps in ${Math.round(result.durationMs / 100) / 10}s`,
   )
+
+  let mp4Path: string | undefined
+  let mp4Reason: string | undefined
+  if (result.videoPath) {
+    logStep('Container')
+    const converted = await convertToMp4(result.videoPath)
+    if (converted.status === 'converted') {
+      mp4Path = converted.mp4Path
+      logInfo(display(mp4Path))
+    } else if (converted.status === 'skipped') {
+      mp4Reason = converted.reason
+      logWarn('ffmpeg is not installed, so no mp4 was written.')
+      logWarn(`Install it with: ${INSTALL_CONVERTER}`)
+    } else {
+      mp4Reason = converted.reason
+      logWarn(`mp4 conversion failed: ${converted.reason}`)
+    }
+  }
+
   outro()
 
   emit(opts.json, {
     plan: source,
     video: result.videoPath ?? null,
+    mp4: mp4Path ?? null,
     still: result.stillPath ?? null,
     steps: result.steps,
     durationMs: result.durationMs,
+    ...(mp4Reason ? { mp4Reason } : {}),
   })
   return 0
 }

package/src/demo/compile.ts

@@ -13,11 +13,42 @@ import type { Beat, Draft } from '@/demo/beats'
  */
 
 /** Hand-tuned in spike 2 against one fixture. Nothing establishes them in general. */
-const POINTER_STEPS = 45
+const POINTER_TRAVEL_MS = 400
 const TYPE_DELAY_MS = 110
 const HOLD_MS = 600
 const FINAL_HOLD_MS = 1200
 
+/**
+ * A step is a round trip to the browser, so a fixed count reproduces the
+ * defect under a new name if it does not respond to the round-trip cost
+ * measured at drive time. The bounds guard the extremes a bad measurement
+ * could produce: too few steps teleports rather than glides, and a
+ * near-zero measured cost cannot be trusted enough to let the count run
+ * away.
+ *
+ * Measured on 2026-08-26 against a served fixture, post-navigation: 10
+ * rounds of `page.mouse.move` averaged 16.6 milliseconds a step, well short
+ * of either bound at the 400 millisecond default (about 24 steps). A
+ * machine landing under `POINTER_TRAVEL_MS / MAX_POINTER_STEPS`, 3.33
+ * milliseconds a step here, hits the cap and stops responding to a faster
+ * one still. Nothing measured here establishes where a real machine sits
+ * relative to that boundary in general.
+ */
+export const MIN_POINTER_STEPS = 6
+export const MAX_POINTER_STEPS = 120
+
+/**
+ * Pure on purpose: the round-trip cost comes from a real browser and can
+ * only be measured at drive time, so this takes it as an argument rather
+ * than measuring it itself, which is what keeps it testable against a
+ * stubbed cost.
+ */
+export function deriveSteps(travelMs: number, roundTripMs: number): number {
+  if (roundTripMs <= 0) return MAX_POINTER_STEPS
+  const steps = Math.round(travelMs / roundTripMs)
+  return Math.min(MAX_POINTER_STEPS, Math.max(MIN_POINTER_STEPS, steps))
+}
+
 const VIEWPORT = { width: 1280, height: 720 } as const
 const ANNOTATIONS = {
   durationMs: 900,
@@ -82,7 +113,7 @@ export interface DemoPlan {
   readonly url: string
   readonly viewport: { readonly width: number; readonly height: number }
   readonly output: { readonly video: string; readonly still: string }
-  readonly pointer: { readonly steps: number; readonly typeDelayMs: number }
+  readonly pointer: { readonly travelMs: number; readonly typeDelayMs: number }
   readonly annotations: typeof ANNOTATIONS
   readonly steps: readonly DemoStep[]
 }
@@ -103,7 +134,7 @@ export function compilePlan(draft: Draft, options: CompileOptions): DemoPlan {
       video: `${options.outDir}/${options.slug}.webm`,
       still: `${options.outDir}/${options.slug}.png`,
     },
-    pointer: { steps: POINTER_STEPS, typeDelayMs: TYPE_DELAY_MS },
+    pointer: { travelMs: POINTER_TRAVEL_MS, typeDelayMs: TYPE_DELAY_MS },
     annotations: ANNOTATIONS,
     steps: draft.beats.map((beat, position) =>
       compileStep(beat, {
@@ -218,7 +249,7 @@ export function parsePlan(text: string): PlanParse {
         still: asText(output.still) || `demos/${slug || 'demo'}.png`,
       },
       pointer: {
-        steps: asNumber(pointer.steps, POINTER_STEPS),
+        travelMs: asNumber(pointer.travelMs, POINTER_TRAVEL_MS),
         typeDelayMs: asNumber(pointer.typeDelayMs, TYPE_DELAY_MS),
       },
       annotations: ANNOTATIONS,

package/src/demo/container.ts

@@ -0,0 +1,63 @@
+import { join, parse } from 'node:path'
+import { execa } from 'execa'
+
+/**
+ * A post-step on the file `@/demo/drive` already wrote. It touches neither
+ * the compiler nor the driving code, which is why deferring it out of the
+ * feature that shipped first cost no rework.
+ */
+
+const CONVERTER_BIN = 'ffmpeg'
+export const INSTALL_CONVERTER = 'ffmpeg (see https://ffmpeg.org/download.html)'
+
+export type ContainerResult =
+  | { status: 'converted'; mp4Path: string }
+  | { status: 'skipped'; reason: 'converter-missing' }
+  | { status: 'failed'; reason: string }
+
+/**
+ * Writes mp4 beside the webm rather than instead of it, since both stated use
+ * cases are a `<video>` tag on a page the operator controls, where webm
+ * already plays, and the social platform case is what mp4 is for.
+ *
+ * A missing binary is reported as skipped rather than failed. The recording
+ * already succeeded, and failing the run over an optional step would discard
+ * a good artifact.
+ */
+export async function convertToMp4(
+  webmPath: string,
+  bin: string = CONVERTER_BIN,
+): Promise<ContainerResult> {
+  const { dir, name } = parse(webmPath)
+  const mp4Path = join(dir, `${name}.mp4`)
+
+  const result = await execa(
+    bin,
+    [
+      '-y',
+      '-i',
+      webmPath,
+      '-c:v',
+      'libx264',
+      '-pix_fmt',
+      'yuv420p',
+      '-c:a',
+      'aac',
+      '-movflags',
+      '+faststart',
+      mp4Path,
+    ],
+    { reject: false },
+  )
+
+  if (result.failed && result.code === 'ENOENT') {
+    return { status: 'skipped', reason: 'converter-missing' }
+  }
+  if (result.exitCode !== 0) {
+    return {
+      status: 'failed',
+      reason: result.stderr?.trim() || `ffmpeg exited ${result.exitCode}`,
+    }
+  }
+  return { status: 'converted', mp4Path }
+}

package/src/demo/drive.ts

@@ -3,10 +3,17 @@ import { tmpdir } from 'node:os'
 import { dirname, join } from 'node:path'
 import { chromium } from 'playwright-core'
 import type { Browser, BrowserContext, Page } from 'playwright-core'
+import { deriveSteps } from '@/demo/compile'
 import type { DemoPlan, DemoStep } from '@/demo/compile'
 import type { CursorSet } from '@/demo/pointer'
 import { pointerSource } from '@/demo/pointer'
 
+declare global {
+  interface Window {
+    __aitk_demo_caption__?: (text: string) => void
+  }
+}
+
 /**
  * Drives a running application and records what it did. Every browser reference
  * the demo feature adds lives here, and `src/commands/demo.ts` reaches it
@@ -34,6 +41,10 @@ const POINTER_SIZE = 32
  */
 const START = { x: 8, y: 8 }
 const SETTLE_MS = 250
+/** Round trips sampled to price one, on the page a step is actually about to move across. */
+const CALIBRATION_STEPS = 8
+/** DOM id the caption bar installs under, read back by `drive.e2e.test.ts`. */
+export const CAPTION_ID = '__aitk_demo_caption_bar__'
 
 /**
  * The two output paths arrive resolved rather than as a root this re-resolves
@@ -118,8 +129,10 @@ export async function drive(options: DriveOptions): Promise<DriveResult> {
     await context.addInitScript({
       content: pointerSource(options.cursors, POINTER_SIZE),
     })
+    await context.addInitScript({ content: captionInitScript() })
     const page = await context.newPage()
     const video = page.video()
+    const pace: PointerPace = {}
 
     // The opening navigate is skipped when the plan already starts with one,
     // because a draft written around an opening verb compiles to a `navigate`
@@ -130,7 +143,7 @@ export async function drive(options: DriveOptions): Promise<DriveResult> {
     }
 
     for (const step of plan.steps) {
-      await runStep(page, plan, step)
+      await runStep(page, plan, step, pace)
       // The first marked step wins. One file holds one frame, so a plan a
       // person edited to mark several would otherwise write each over the last
       // and keep whichever ran last, with nothing saying so.
@@ -187,10 +200,24 @@ async function launch(): Promise<Launch> {
   }
 }
 
-async function runStep(
+/**
+ * Holds the round trip once a step has measured it, so every `moveTo` after
+ * the first reuses the same reading rather than re-timing on every move.
+ */
+export interface PointerPace {
+  roundTripMs?: number
+}
+
+/**
+ * Exported so `drive.e2e.test.ts` can drive one real step against a real
+ * caption and read it back, which is the integration a full `drive()` call
+ * cannot assert without decoding the video it writes.
+ */
+export async function runStep(
   page: Page,
   plan: DemoPlan,
   step: DemoStep,
+  pace: PointerPace,
 ): Promise<void> {
   switch (step.kind) {
     case 'navigate':
@@ -198,33 +225,47 @@ async function runStep(
       await page.mouse.move(START.x, START.y, { steps: 2 })
       break
     case 'click':
-      await moveTo(page, plan, step)
+      await moveTo(page, plan, step, pace)
       await page.mouse.down()
       await page.mouse.up()
       break
     case 'fill':
-      await moveTo(page, plan, step)
+      await moveTo(page, plan, step, pace)
       await page.mouse.down()
       await page.mouse.up()
       await page.keyboard.type(step.text, { delay: plan.pointer.typeDelayMs })
       break
     case 'hover':
-      await moveTo(page, plan, step)
+      await moveTo(page, plan, step, pace)
       break
     case 'scroll':
       await page.locator(step.target).first().scrollIntoViewIfNeeded()
       await page.waitForTimeout(SETTLE_MS)
-      await moveTo(page, plan, step)
+      await moveTo(page, plan, step, pace)
       break
     case 'wait':
     case 'hold':
       break
   }
 
+  // Set after the action rather than before it, so the caption shows for the
+  // hold that follows rather than for the page the action is about to leave.
+  await setCaption(page, step.caption)
   if (step.waitFor) await page.locator(step.waitFor).first().waitFor()
   await page.waitForTimeout(step.holdMs)
 }
 
+/**
+ * Timed on the page a `moveTo` is actually about to move across, never on the
+ * blank page before it, since layout, paint, and page script are what a step
+ * pays the round trip against and a blank page has none of the three.
+ */
+async function calibrateRoundTrip(page: Page): Promise<number> {
+  const startedAt = Date.now()
+  await page.mouse.move(START.x, START.y, { steps: CALIBRATION_STEPS })
+  return (Date.now() - startedAt) / CALIBRATION_STEPS
+}
+
 /**
  * Travel is the whole point of driving the engine's pointer rather than calling
  * the element-clicking helper, which resolves a target and jumps to it. The
@@ -237,16 +278,78 @@ async function moveTo(
   page: Page,
   plan: DemoPlan,
   step: DemoStep,
+  pace: PointerPace,
 ): Promise<void> {
+  pace.roundTripMs ??= await calibrateRoundTrip(page)
   const locator = page.locator(step.target).first()
   await locator.waitFor()
   const box = await locator.boundingBox()
   if (!box) throw new Error(`${step.target} has no box to point at`)
   await page.mouse.move(box.x + box.width / 2, box.y + box.height / 2, {
-    steps: plan.pointer.steps,
+    steps: deriveSteps(plan.pointer.travelMs, pace.roundTripMs),
   })
 }
 
+/**
+ * Playwright's own `showActions` overlay names the API call it made, not the
+ * beat's narration, so a caption needs an element of its own rather than
+ * reusing that annotation. Runs alongside `pointerSource`, guarded the same
+ * way against a page that already carries one.
+ */
+export function captionInitScript(): string {
+  return `(() => {
+  if (window.__aitk_demo_caption__) return;
+
+  let label;
+
+  const install = () => {
+    if (label || !document.body) return;
+    const bar = document.createElement('div');
+    bar.id = '${CAPTION_ID}';
+    bar.setAttribute('aria-hidden', 'true');
+    bar.style.cssText = [
+      'position:fixed',
+      'left:0',
+      'right:0',
+      'bottom:32px',
+      'display:flex',
+      'justify-content:center',
+      'pointer-events:none',
+      'z-index:2147483647',
+    ].join(';');
+    label = document.createElement('span');
+    label.style.cssText = [
+      'background:rgba(16,16,20,0.85)',
+      'color:#f4f4f5',
+      'font:600 20px/1.4 system-ui,sans-serif',
+      'padding:10px 22px',
+      'border-radius:8px',
+      'max-width:80vw',
+      'text-align:center',
+      'display:none',
+    ].join(';');
+    bar.appendChild(label);
+    document.body.appendChild(bar);
+    window.__aitk_demo_caption__ = (text) => {
+      label.textContent = text || '';
+      label.style.display = text ? 'inline-block' : 'none';
+    };
+  };
+
+  if (document.readyState === 'loading') {
+    addEventListener('DOMContentLoaded', install, { once: true });
+  } else {
+    install();
+  }
+})();`
+}
+
+async function setCaption(page: Page, caption: string): Promise<void> {
+  await page.evaluate((text) => {
+    window.__aitk_demo_caption__?.(text)
+  }, caption)
+}
+
 function failed(reason: DriveRefusal, error: unknown): DriveFailure {
   return {
     status: 'failed',