@henols/c64-re-tools 2.0.0-rc.2 → 2.0.0-rc.21

package/skills/c64-testing/scripts/{scenario.ts → lib/scenario.ts}

@@ -3,22 +3,15 @@
 // Observations name semantic symbols that resolve per side, comparisons are
 // explicit, and the result is PASS, FAIL or INCONCLUSIVE.
 
-import { formatC64Address, parseC64Address, textToPetscii } from "#src/c64.ts";
-import {
-  JOYSTICK_DIRECTIONS,
-  MAX_BASELINES,
-  MAX_EXECUTION_COUNT,
-  MAX_KEYBOARD_BYTES,
-  MAX_MASK_RECTANGLES,
-  MAX_MEMORY_READ,
-  MAX_MEMORY_WRITE,
-  MAX_TIMEOUT_FRAMES,
-  WireFailure,
-} from "#src/host-client/tools.ts";
-import { readProjectFile } from "#src/host-client/transfer.ts";
-import type { ViceSessionClient } from "#src/host-client/vice-session.ts";
+import { formatC64Address, formatSprite, formatVicii, parseC64Address } from "#src/c64.ts";
+import { MAX_BASELINES, MAX_MASK_RECTANGLES, maskCoversFrame, WireFailure } from "#src/host-client/tools.ts";
+import { LOAD_LIMIT_FRAMES, RUN_LIMIT_FRAMES, runProgram } from "#src/host-client/autostart.ts";
+import { ACTION_STEP_KEYS, checkActionStep, checkLocation, isInteger, isObject, runActionStep, type ActionStep, type Location } from "#src/host-client/steps.ts";
+import { readProjectFile, writeProjectFile } from "#src/host-client/transfer.ts";
+import { readMemoryInParts, type ViceSessionClient } from "#src/host-client/vice-session.ts";
 import { openForRead } from "#src/knowledge/database.ts";
-import { listSymbols } from "#src/knowledge/read.ts";
+import { listAnalyzerNames, listSymbols } from "#src/knowledge/read.ts";
+import { programImage } from "#src/knowledge/program.ts";
 
 export class ScenarioError extends Error {}
 
@@ -26,14 +19,18 @@ export type Side = "original" | "rebuild";
 export const SIDES: Side[] = ["original", "rebuild"];
 
 /** A "$xxxx" address or a symbol name that each side resolves on its own. */
-export type Location = string;
+export type { Location };
 
 export interface MemoryObservation {
   at: Location;
-  /** Bytes from `at`; 1 when not given. */
+  /** Bytes from `at`; 1 when not given, at most 64 KiB. */
   size?: number;
+  /** cpu (the default) reads as the CPU sees memory; ram reads the RAM under the ROMs and the I/O area (M23.9). */
+  view?: "cpu" | "ram";
   /** The largest allowed difference per byte; 0 when not given. */
   tolerance?: number;
+  /** Bytes in the range that the comparison leaves out, each with the reason (D46). */
+  exclude?: Array<{ at: Location; reason: string }>;
 }
 
 export interface ScreenPolicy {
@@ -43,21 +40,21 @@ export interface ScreenPolicy {
   mask?: Array<{ x: number; y: number; width: number; height: number }>;
 }
 
-export type Step =
+type StepBody =
   | { reset: "hard" | "soft" }
   | { load: true }
   | { autostart: true }
-  | { write: { at: Location; bytes: number[] } }
-  | { registers: Partial<Record<"pc" | "a" | "x" | "y" | "sp", number | Location>> }
-  | { joystick: { port: 1 | 2; direction: JoystickDirection; fire?: boolean } }
-  | { type: string }
-  | { frames: number }
-  | { runUntil: { at: Location } | { memory: { at: Location; equals: number } }; timeoutFrames: number }
+  | { snapshot: string }
+  | { normalize: true }
+  | ActionStep
   | { observe: string; memory?: Array<Location | MemoryObservation>; registers?: ObservedRegister[]; screen?: ScreenPolicy; vicii?: boolean; sprites?: number[] };
 
+/** A step; with side, only that side does it (D46). An observe step has no side: it compares both. */
+export type Step = StepBody & { side?: Side };
+
 export interface Subject {
   program: string;
-  /** "knowledge" for knowledge.db, a project JSON file (c64-assembler output or a name → "$xxxx" map), or "none". */
+  /** "knowledge" for the project knowledge, a project JSON file (c64-assembler output or a name → "$xxxx" map), or "none". */
   symbols: string;
 }
 
@@ -85,26 +82,23 @@ export interface ScenarioResult {
   reason?: string;
   checkpoints: string[];
   differences: Difference[];
+  /** The bytes that the comparison left out, with their reasons (D46). */
+  excluded?: Array<{ checkpoint: string; at: Location; reason: string }>;
+  /** The files that a FAIL wrote with --out. */
+  files?: string[];
 }
 
-const STEP_KINDS = ["reset", "load", "autostart", "write", "registers", "joystick", "type", "frames", "runUntil", "observe"];
+const STEP_KINDS = ["reset", "load", "autostart", "snapshot", "normalize", "write", "registers", "joystick", "type", "frames", "runUntil", "observe"];
 
 /** The keys each kind of step can have. */
 const STEP_KEYS: Record<string, readonly string[]> = {
-  runUntil: ["runUntil", "timeoutFrames"],
+  ...ACTION_STEP_KEYS,
   observe: ["observe", "memory", "registers", "screen", "vicii", "sprites"],
 };
 
 /** The registers an observe step can compare. */
 export const OBSERVED_REGISTERS = ["pc", "a", "x", "y", "sp", "flags"] as const;
 type ObservedRegister = (typeof OBSERVED_REGISTERS)[number];
-/** The registers a registers step can set. */
-const SET_REGISTERS = ["pc", "a", "x", "y", "sp"];
-
-type JoystickDirection = (typeof JOYSTICK_DIRECTIONS)[number];
-
-const isObject = (value: unknown): value is Record<string, unknown> => typeof value === "object" && value !== null && !Array.isArray(value);
-const isInteger = (value: unknown, min: number, max: number): value is number => Number.isInteger(value) && (value as number) >= min && (value as number) <= max;
 
 /** Reads and checks a scenario file. Throws ScenarioError naming the first problem. */
 export function readScenario(path: string): Scenario {
@@ -135,9 +129,10 @@ export function readScenario(path: string): Scenario {
     const kinds = "observe" in step ? ["observe"] : Object.keys(step).filter((key) => STEP_KINDS.includes(key));
     if (kinds.length !== 1) return fail(`${at} must have exactly one of ${STEP_KINDS.join(", ")}`);
     const kind = kinds[0]!;
-    const allowed = STEP_KEYS[kind] ?? [kind];
+    const allowed = [...(STEP_KEYS[kind] ?? [kind]), ...(kind === "observe" ? [] : ["side"])];
     for (const key of Object.keys(step)) if (!allowed.includes(key)) fail(`${at} has the key ${key}, which a ${kind} step does not have`);
-    checkStep(kind, step, at, fail);
+    if (step.side !== undefined && !(SIDES as unknown[]).includes(step.side)) fail(`${at}: side must be original or rebuild`);
+    checkStep(kind, step, at, fail, scenario.videoStandard ?? "pal");
     if (kind === "observe") {
       if (labels.has(step.observe as string)) fail(`${at} needs a new checkpoint name`);
       labels.add(step.observe as string);
@@ -149,20 +144,8 @@ export function readScenario(path: string): Scenario {
   return scenario;
 }
 
-/** Checks a "$xxxx" address or a symbol name. */
-function checkLocation(value: unknown, what: string, fail: (message: string) => never): void {
-  if (typeof value !== "string" || value === "") fail(`${what} must be a $ address or a symbol name`);
-  if ((value as string).startsWith("$")) {
-    try {
-      parseC64Address(value as string);
-    } catch {
-      fail(`${what} must be $ followed by four hex digits, not ${value as string}`);
-    }
-  }
-}
-
 /** Checks the value of one step against its kind. */
-function checkStep(kind: string, step: Record<string, unknown>, at: string, fail: (message: string) => never): void {
+function checkStep(kind: string, step: Record<string, unknown>, at: string, fail: (message: string) => never, videoStandard: "pal" | "ntsc"): void {
   switch (kind) {
     case "reset":
       if (step.reset !== "hard" && step.reset !== "soft") fail(`${at}: reset must be hard or soft`);
@@ -171,66 +154,25 @@ function checkStep(kind: string, step: Record<string, unknown>, at: string, fail
     case "autostart":
       if (step[kind] !== true) fail(`${at}: ${kind} must be true`);
       return;
-    case "write": {
-      const write = step.write;
-      if (!isObject(write) || Object.keys(write).some((key) => key !== "at" && key !== "bytes")) return fail(`${at}: write needs at and bytes, and nothing else`);
-      checkLocation(write.at, `${at}: write.at`, fail);
-      if (!Array.isArray(write.bytes) || write.bytes.length === 0 || write.bytes.length > MAX_MEMORY_WRITE || !write.bytes.every((byte) => isInteger(byte, 0, 0xff))) {
-        fail(`${at}: write.bytes must be a list of 1 to ${MAX_MEMORY_WRITE} numbers from 0 to 255`);
-      }
-      return;
-    }
-    case "registers": {
-      const registers = step.registers;
-      if (!isObject(registers) || Object.keys(registers).length === 0) return fail(`${at}: registers must name at least one of ${SET_REGISTERS.join(", ")}`);
-      for (const [name, value] of Object.entries(registers)) {
-        if (!SET_REGISTERS.includes(name)) fail(`${at}: registers can set only ${SET_REGISTERS.join(", ")}, not ${name}`);
-        if (typeof value === "string") checkLocation(value, `${at}: registers.${name}`, fail);
-        else if (!isInteger(value, 0, name === "pc" ? 0xffff : 0xff)) fail(`${at}: registers.${name} must be a symbol name, a $ address or a number from 0 to ${name === "pc" ? 65535 : 255}`);
-      }
-      return;
-    }
-    case "joystick": {
-      const joystick = step.joystick;
-      if (!isObject(joystick) || Object.keys(joystick).some((key) => !["port", "direction", "fire"].includes(key))) return fail(`${at}: joystick needs port and direction, and fire is optional`);
-      if (joystick.port !== 1 && joystick.port !== 2) fail(`${at}: joystick.port must be 1 or 2`);
-      if (!(JOYSTICK_DIRECTIONS as readonly unknown[]).includes(joystick.direction)) fail(`${at}: joystick.direction must be one of ${JOYSTICK_DIRECTIONS.join(", ")}`);
-      if (joystick.fire !== undefined && typeof joystick.fire !== "boolean") fail(`${at}: joystick.fire must be true or false`);
+    case "snapshot":
+      if (typeof step.snapshot !== "string" || step.snapshot === "") fail(`${at}: snapshot needs a .vsf file in the project`);
       return;
-    }
-    case "type": {
-      if (typeof step.type !== "string" || step.type === "") return fail(`${at}: type needs text`);
-      let bytes: Uint8Array;
-      try {
-        bytes = textToPetscii(step.type);
-      } catch (error) {
-        return fail(`${at}: ${(error as Error).message}`);
-      }
-      if (bytes.length > MAX_KEYBOARD_BYTES) fail(`${at}: type can have at most ${MAX_KEYBOARD_BYTES} keys`);
+    case "normalize":
+      if (step.normalize !== true) fail(`${at}: normalize is true: it writes the entry state of the knowledge`);
       return;
-    }
+    case "write":
+    case "registers":
+    case "joystick":
+    case "type":
     case "frames":
-      if (!isInteger(step.frames, 1, MAX_EXECUTION_COUNT)) fail(`${at} needs a frame count from 1 to ${MAX_EXECUTION_COUNT}`);
-      return;
-    case "runUntil": {
-      const target = step.runUntil;
-      if (!isObject(target) || Object.keys(target).length !== 1 || !("at" in target || "memory" in target)) return fail(`${at}: runUntil needs at or memory`);
-      if ("at" in target) checkLocation(target.at, `${at}: runUntil.at`, fail);
-      else {
-        const memory = target.memory;
-        if (!isObject(memory) || Object.keys(memory).some((key) => key !== "at" && key !== "equals")) return fail(`${at}: runUntil.memory needs at and equals, and nothing else`);
-        checkLocation(memory.at, `${at}: runUntil.memory.at`, fail);
-        if (!isInteger(memory.equals, 0, 0xff)) fail(`${at}: runUntil.memory.equals must be a number from 0 to 255`);
-      }
-      if (!isInteger(step.timeoutFrames, 1, MAX_TIMEOUT_FRAMES)) fail(`${at} needs timeoutFrames from 1 to ${MAX_TIMEOUT_FRAMES}`);
-      return;
-    }
+    case "runUntil":
+      return checkActionStep(kind, step, at, fail);
     default:
-      checkObserve(step, at, fail);
+      checkObserve(step, at, fail, videoStandard);
   }
 }
 
-function checkObserve(step: Record<string, unknown>, at: string, fail: (message: string) => never): void {
+function checkObserve(step: Record<string, unknown>, at: string, fail: (message: string) => never, videoStandard: "pal" | "ntsc"): void {
   if (typeof step.observe !== "string" || step.observe === "") fail(`${at} needs a new checkpoint name`);
   if (step.memory !== undefined) {
     if (!Array.isArray(step.memory) || step.memory.length === 0) fail(`${at}: memory must be a list of names, $ addresses or objects with at`);
@@ -239,9 +181,18 @@ function checkObserve(step: Record<string, unknown>, at: string, fail: (message:
         checkLocation(item, `${at}: memory`, fail);
         continue;
       }
-      if (!isObject(item) || Object.keys(item).some((key) => !["at", "size", "tolerance"].includes(key))) return fail(`${at}: a memory object has at, size and tolerance, and nothing else`);
+      if (!isObject(item) || Object.keys(item).some((key) => !["at", "size", "view", "tolerance", "exclude"].includes(key))) return fail(`${at}: a memory object has at, size, view, tolerance and exclude, and nothing else`);
+      if (item.view !== undefined && item.view !== "cpu" && item.view !== "ram") fail(`${at}: memory.view must be cpu or ram`);
+      if (item.exclude !== undefined) {
+        if (!Array.isArray(item.exclude) || item.exclude.length === 0) fail(`${at}: memory.exclude must be a list of objects with at and reason`);
+        for (const excluded of item.exclude as unknown[]) {
+          if (!isObject(excluded) || Object.keys(excluded).some((key) => key !== "at" && key !== "reason")) return fail(`${at}: an exclude has at and reason, and nothing else`);
+          checkLocation(excluded.at, `${at}: memory.exclude.at`, fail);
+          if (typeof excluded.reason !== "string" || excluded.reason.trim() === "") fail(`${at}: an exclude needs a reason: why this byte may differ`);
+        }
+      }
       checkLocation(item.at, `${at}: memory.at`, fail);
-      if (item.size !== undefined && !isInteger(item.size, 1, MAX_MEMORY_READ)) fail(`${at}: memory.size must be a number from 1 to ${MAX_MEMORY_READ}`);
+      if (item.size !== undefined && !isInteger(item.size, 1, 0x10000)) fail(`${at}: memory.size must be a number from 1 to 65536`);
       if (item.tolerance !== undefined && !isInteger(item.tolerance, 0, 0xff)) fail(`${at}: memory.tolerance must be a number from 0 to 255`);
     }
   }
@@ -269,6 +220,8 @@ function checkObserve(step: Record<string, unknown>, at: string, fail: (message:
           isInteger(rectangle.height, 1, 4096);
         if (!ok) fail(`${at}: a screen.mask rectangle needs whole numbers x and y from 0, and width and height from 1`);
       }
+      // The emulator refuses such a compare too, but only after the original side ran (b2/03).
+      if (maskCoversFrame(screen.mask as Array<{ x: number; y: number; width: number; height: number }>, videoStandard)) fail(`${at}: screen.mask covers the whole screen, so nothing is compared`);
     }
   }
   if (step.vicii !== undefined && typeof step.vicii !== "boolean") fail(`${at}: vicii must be true or false`);
@@ -287,6 +240,10 @@ export function loadSymbols(subject: Subject): Map<string, number> {
   if (subject.symbols === "knowledge") {
     const db = openForRead();
     for (const symbol of listSymbols(db)) symbols.set(symbol.name, symbol.address);
+    // An analyzer name gives an address too, when no symbol has the name and only one analyzer gives it (D74).
+    const second = new Map<string, number[]>();
+    for (const row of listAnalyzerNames(db)) second.set(row.name, [...(second.get(row.name) ?? []), row.address]);
+    for (const [name, addresses] of second) if (!symbols.has(name) && new Set(addresses).size === 1) symbols.set(name, addresses[0]!);
     db?.close();
     return symbols;
   }
@@ -299,11 +256,12 @@ export function loadSymbols(subject: Subject): Map<string, number> {
   // c64-assembler output, or a plain name → "$xxxx" map. A constant in the
   // assembler output counts too: RAM variables such as "counter = $c100" lie
   // outside the program, so the assembler reports them as constants.
-  const assembled = (value as { symbols?: Array<{ name: string; address?: string; value?: number }> }).symbols;
+  const assembled = (value as { symbols?: Array<{ name: string; address?: string; value?: string }> }).symbols;
   const entries: Array<[string, unknown]> = Array.isArray(assembled)
     ? assembled.flatMap((symbol): Array<[string, unknown]> => {
         if (symbol.address !== undefined) return [[symbol.name, symbol.address]];
-        if (Number.isInteger(symbol.value) && symbol.value! >= 0 && symbol.value! <= 0xffff) return [[symbol.name, formatC64Address(symbol.value!)]];
+        // A constant above $ffff has more digits and is no address.
+        if (typeof symbol.value === "string" && /^\$[0-9a-f]{4}$/.test(symbol.value)) return [[symbol.name, symbol.value]];
         return [];
       })
     : Object.entries(value as Record<string, unknown>);
@@ -334,11 +292,35 @@ export function resolve(location: Location, symbols: Map<string, number>, side:
 }
 
 interface Observation {
-  memory: Array<{ label: string; bytes: number[]; tolerance: number }>;
+  memory: Array<{ label: string; address: number; bytes: number[]; tolerance: number; excluded: Array<{ offset: number; at: Location; reason: string }> }>;
   registers: Partial<Record<ObservedRegister, unknown>>;
-  screen?: { match: boolean; mismatchingPixels: number; mismatchRatio: number; bounds?: unknown };
+  screen?: { match: boolean; mismatchingPixels: number; comparedPixels: number; mismatchRatio: number; bounds?: unknown };
   vicii?: unknown;
   sprites?: unknown;
+  /** PNG images for the files of a FAIL (D46): the side's screen, and on the rebuild the difference image. */
+  png?: string;
+  diffPng?: string;
+}
+
+/** Writes pages 0-3 and $00/$01 from the entry state of the knowledge (D43, D46, D83): both sides then start from the same RAM. */
+async function normalize(client: ViceSessionClient): Promise<void> {
+  const db = openForRead();
+  let state: { pages?: Record<string, string>; cpuPort?: Record<string, number> } | undefined;
+  try {
+    state = programImage(db)?.state as typeof state;
+  } finally {
+    db?.close();
+  }
+  if (state === undefined) throw new ScenarioError("normalize needs the entry state of a c64-unpacker capture in the knowledge: put the capture in with knowledge.ts image <image.json> of the c64-knowledge skill");
+  const pages = ["$0000", "$0100", "$0200", "$0300"].map((page) => state.pages?.[page]);
+  const port = [state.cpuPort?.$00, state.cpuPort?.$01];
+  if (pages.some((page) => typeof page !== "string" || !/^[0-9a-f]{512}$/.test(page)) || port.some((value) => !isInteger(value, 0, 0xff))) {
+    throw new ScenarioError("the entry state of the knowledge is not the entry state of a c64-unpacker capture: it needs pages $0000 to $0300 and cpuPort $00 and $01");
+  }
+  if ((await client.status()).state === "running") await client.execution({ action: "pause", space: "c64" });
+  // Pages 0 to 3 in RAM from $0002, then the CPU port: the port bits switch what the CPU sees.
+  await client.memoryWrite({ address: 0x0002, data: pages.join("").slice(4), space: "c64", view: "ram" });
+  await client.memoryWrite({ address: 0x0000, data: Buffer.from(port as number[]).toString("hex"), space: "c64", view: "cpu" });
 }
 
 /**
@@ -350,45 +332,62 @@ export function baselineName(checkpoint: number, label: string): string {
   return `test-${checkpoint}-${label}`.replace(/[^A-Za-z0-9._-]/g, "_").slice(0, 64);
 }
 
-async function runSide(client: ViceSessionClient, scenario: Scenario, side: Side, symbols: Map<string, number>): Promise<Map<string, Observation>> {
+/** Autostart that returns only when the program runs (runProgram), or INCONCLUSIVE with the reason. */
+async function startProgram(client: ViceSessionClient, program: string, side: Side): Promise<void> {
+  const failed = await runProgram(client, program, readProjectFile(program).bytes);
+  if (failed === "not-loaded") throw new Inconclusive(`the ${side} program was not in memory after ${LOAD_LIMIT_FRAMES} frames of autostart`);
+  if (failed === "not-started") throw new Inconclusive(`the ${side} program did not start within ${RUN_LIMIT_FRAMES} frames of RUN; autostart runs only a program with a BASIC start line`);
+}
+
+async function runSide(client: ViceSessionClient, scenario: Scenario, side: Side, symbols: Map<string, number>, files: boolean): Promise<Map<string, Observation>> {
   const subject = scenario[side];
   const observations = new Map<string, Observation>();
   const at = (location: Location) => resolve(location, symbols, side);
-  const stop = async () => {
-    if ((await client.status()).state === "running") await client.execution({ action: "pause", space: "c64" });
-  };
   // Each side starts from a hard reset, stopped at the reset vector, with the control ports released.
   await client.reset({ mode: "hard", run: false });
   await client.joystick({ port: 1, direction: "center", fire: false });
   await client.joystick({ port: 2, direction: "center", fire: false });
   let checkpoint = 0;
   for (const step of scenario.steps) {
+    if (step.side !== undefined && step.side !== side) continue;
     // Observe first: an observe step may also carry "registers" or "memory".
     if ("observe" in step) {
       checkpoint++;
       const observation: Observation = { memory: [], registers: {} };
       for (const item of step.memory ?? []) {
-        const spec = typeof item === "string" ? { at: item } : item;
-        const read = await client.memoryRead({ address: at(spec.at), size: spec.size ?? 1, space: "c64", view: "cpu" });
-        observation.memory.push({ label: spec.size !== undefined && spec.size > 1 ? `${spec.at} (${spec.size} bytes)` : spec.at, bytes: [...Buffer.from(read.data, "hex")], tolerance: spec.tolerance ?? 0 });
+        const spec: MemoryObservation = typeof item === "string" ? { at: item } : item;
+        const address = at(spec.at);
+        const size = spec.size ?? 1;
+        const read = { data: (await readMemoryInParts(client, address, size, spec.view ?? "cpu")).toString("hex") };
+        const excluded = (spec.exclude ?? []).map((exclude) => {
+          const offset = at(exclude.at) - address;
+          if (offset < 0 || offset >= size) throw new ScenarioError(`the exclude ${exclude.at} lies outside the observed ${spec.at} (${size} bytes)`);
+          return { offset, at: exclude.at, reason: exclude.reason };
+        });
+        observation.memory.push({ label: size > 1 ? `${spec.at} (${size} bytes)` : spec.at, address, bytes: [...Buffer.from(read.data, "hex")], tolerance: spec.tolerance ?? 0, excluded });
       }
       if (step.registers !== undefined) {
         const registers = await client.registersGet("c64");
-        for (const name of step.registers) observation.registers[name] = registers[name];
+        for (const name of step.registers) observation.registers[name] = name === "pc" ? formatC64Address(registers.pc) : registers[name];
       }
       if (step.vicii) {
         // The raster line changes from one moment to the next; the comparison leaves it out.
         const { rasterLine: _rasterLine, ...state } = await client.vicii();
-        observation.vicii = state;
+        observation.vicii = formatVicii(state);
       }
-      if (step.sprites !== undefined) observation.sprites = (await client.sprites(step.sprites)).sprites;
+      if (step.sprites !== undefined) observation.sprites = (await client.sprites(step.sprites)).sprites.map(formatSprite);
       if (step.screen !== undefined) {
         const baseline = baselineName(checkpoint, step.observe);
         if (side === "original") {
-          await client.screenCapture(baseline);
+          const shot = await client.screenCapture(baseline);
+          if (files) observation.png = shot.png;
         } else {
-          const compared = await client.screenCompare({ baseline, maxMismatchRatio: step.screen.maxMismatchRatio ?? 0, mask: step.screen.mask ?? [], includeDiff: false });
-          observation.screen = { match: compared.match, mismatchingPixels: compared.mismatchingPixels, mismatchRatio: compared.mismatchRatio, ...(compared.bounds === undefined ? {} : { bounds: compared.bounds }) };
+          const compared = await client.screenCompare({ baseline, maxMismatchRatio: step.screen.maxMismatchRatio ?? 0, mask: step.screen.mask ?? [], includeDiff: files });
+          observation.screen = { match: compared.match, mismatchingPixels: compared.mismatchingPixels, comparedPixels: compared.comparedPixels, mismatchRatio: compared.mismatchRatio, ...(compared.bounds === undefined ? {} : { bounds: compared.bounds }) };
+          if (files) {
+            observation.png = (await client.screenCapture()).png;
+            if (compared.diffPng !== undefined) observation.diffPng = compared.diffPng;
+          }
           await client.screenDiscard(baseline);
         }
       }
@@ -399,30 +398,16 @@ async function runSide(client: ViceSessionClient, scenario: Scenario, side: Side
     } else if ("load" in step) {
       await client.programLoad({ path: subject.program });
     } else if ("autostart" in step) {
-      await client.autostart({ path: subject.program, index: 0, run: true });
-    } else if ("write" in step) {
-      await stop();
-      await client.memoryWrite({ address: at(step.write.at), data: Buffer.from(step.write.bytes).toString("hex"), space: "c64", view: "cpu" });
-    } else if ("registers" in step) {
-      await stop();
-      const values: Record<string, number> = {};
-      for (const [name, value] of Object.entries(step.registers)) values[name] = typeof value === "number" ? value : at(value as Location);
-      await client.registersSet("c64", values);
-    } else if ("joystick" in step) {
-      await client.joystick({ port: step.joystick.port, direction: step.joystick.direction, fire: step.joystick.fire ?? false });
-    } else if ("type" in step) {
-      await client.keyboard(textToPetscii(step.type));
-    } else if ("frames" in step) {
-      await stop();
-      await client.execution({ action: "advance-frames", count: step.frames, space: "c64" });
-    } else if ("runUntil" in step) {
-      const target =
-        "at" in step.runUntil
-          ? { kind: "address" as const, address: at(step.runUntil.at), space: "c64" as const }
-          : { kind: "memory" as const, address: at(step.runUntil.memory.at), operator: "eq" as const, value: step.runUntil.memory.equals, space: "c64" as const, view: "cpu" as const };
-      const reached = await client.runUntil({ target, timeoutFrames: step.timeoutFrames });
-      if (!reached.reached) {
-        throw new Inconclusive(`the ${side} side did not reach the checkpoint before ${step.timeoutFrames} frames (it stopped by ${reached.stopReason} at ${formatC64Address(reached.pc)})`);
+      await startProgram(client, subject.program, side);
+    } else if ("snapshot" in step) {
+      await client.snapshotRestore(readProjectFile(step.snapshot).bytes);
+    } else if ("normalize" in step) {
+      await normalize(client);
+    } else {
+      const { side: _side, ...action } = step;
+      const notReached = await runActionStep(client, action as ActionStep, at);
+      if (notReached !== undefined && "runUntil" in step) {
+        throw new Inconclusive(`the ${side} side did not reach the checkpoint before ${step.timeoutFrames} frames (it stopped by ${notReached.stopReason} at ${formatC64Address(notReached.pc)})`);
       }
     }
   }
@@ -436,8 +421,9 @@ export function compare(original: Map<string, Observation>, rebuild: Map<string,
     const right = rebuild.get(checkpoint)!;
     left.memory.forEach((item, index) => {
       const other = right.memory[index]!;
-      if (item.bytes.some((byte, at) => Math.abs(byte - other.bytes[at]!) > item.tolerance)) {
-        const show = (bytes: number[]) => (bytes.length === 1 ? bytes[0] : Buffer.from(bytes).toString("hex"));
+      const skipped = new Set(item.excluded.map((excluded) => excluded.offset));
+      if (item.bytes.some((byte, at) => !skipped.has(at) && Math.abs(byte - other.bytes[at]!) > item.tolerance)) {
+        const show = (bytes: number[]) => Buffer.from(bytes).toString("hex");
         differences.push({ checkpoint, what: `memory ${item.label}`, original: show(item.bytes), rebuild: show(other.bytes) });
       }
     });
@@ -453,13 +439,47 @@ export function compare(original: Map<string, Observation>, rebuild: Map<string,
   return differences;
 }
 
-/** Runs the scenario on the original, then on the rebuild, in one VICE session. */
-export async function runScenario(client: ViceSessionClient, scenario: Scenario): Promise<ScenarioResult> {
+/**
+ * Writes the files of a FAIL into `out` (D46): for a screen difference both
+ * screens and the difference image, and memory.json with each differing byte.
+ */
+function writeFailFiles(out: string, original: Map<string, Observation>, rebuild: Map<string, Observation>, differences: Difference[]): string[] {
+  const files: string[] = [];
+  const write = (name: string, bytes: Uint8Array) => {
+    writeProjectFile(`${out}/${name}`, bytes, { what: "--out" });
+    files.push(name);
+  };
+  const memory: Array<{ checkpoint: string; what: string; bytes: Array<{ address: string; original: string; rebuild: string }> }> = [];
+  const hex = (byte: number) => byte.toString(16).padStart(2, "0");
+  for (const difference of differences) {
+    const left = original.get(difference.checkpoint)!;
+    const right = rebuild.get(difference.checkpoint)!;
+    const name = difference.checkpoint.replace(/[^A-Za-z0-9._-]/g, "_");
+    if (difference.what === "screen") {
+      if (left.png !== undefined) write(`${name}-original.png`, Buffer.from(left.png, "base64"));
+      if (right.png !== undefined) write(`${name}-rebuild.png`, Buffer.from(right.png, "base64"));
+      if (right.diffPng !== undefined) write(`${name}-diff.png`, Buffer.from(right.diffPng, "base64"));
+    }
+    if (difference.what.startsWith("memory ")) {
+      const index = left.memory.findIndex((item) => `memory ${item.label}` === difference.what);
+      const item = left.memory[index]!;
+      const other = right.memory[index]!;
+      const skipped = new Set(item.excluded.map((excluded) => excluded.offset));
+      const bytes = item.bytes.flatMap((byte, offset) => (!skipped.has(offset) && Math.abs(byte - other.bytes[offset]!) > item.tolerance ? [{ address: formatC64Address(item.address + offset), original: hex(byte), rebuild: hex(other.bytes[offset]!) }] : []));
+      memory.push({ checkpoint: difference.checkpoint, what: difference.what, bytes });
+    }
+  }
+  if (memory.length > 0) write("memory.json", Buffer.from(`${JSON.stringify(memory, null, 2)}\n`));
+  return files;
+}
+
+/** Runs the scenario on the original, then on the rebuild, in one VICE session. With out, a FAIL leaves files there. */
+export async function runScenario(client: ViceSessionClient, scenario: Scenario, options: { out?: string } = {}): Promise<ScenarioResult> {
   const checkpoints = scenario.steps.flatMap((step) => ("observe" in step ? [step.observe] : []));
   const observed = new Map<Side, Map<string, Observation>>();
   for (const side of SIDES) {
     try {
-      observed.set(side, await runSide(client, scenario, side, loadSymbols(scenario[side])));
+      observed.set(side, await runSide(client, scenario, side, loadSymbols(scenario[side]), options.out !== undefined));
     } catch (error) {
       if (error instanceof Inconclusive) return { result: "INCONCLUSIVE", scenario: scenario.name, reason: error.message, checkpoints, differences: [] };
       if (error instanceof WireFailure) {
@@ -468,6 +488,17 @@ export async function runScenario(client: ViceSessionClient, scenario: Scenario)
       throw error;
     }
   }
-  const differences = compare(observed.get("original")!, observed.get("rebuild")!);
-  return { result: differences.length === 0 ? "PASS" : "FAIL", scenario: scenario.name, checkpoints, differences };
+  const original = observed.get("original")!;
+  const rebuild = observed.get("rebuild")!;
+  const differences = compare(original, rebuild);
+  const excluded = [...original].flatMap(([checkpoint, observation]) => observation.memory.flatMap((item) => item.excluded.map((excluded) => ({ checkpoint, at: excluded.at, reason: excluded.reason }))));
+  const files = differences.length > 0 && options.out !== undefined ? writeFailFiles(options.out, original, rebuild, differences) : [];
+  return {
+    result: differences.length === 0 ? "PASS" : "FAIL",
+    scenario: scenario.name,
+    checkpoints,
+    differences,
+    ...(excluded.length === 0 ? {} : { excluded }),
+    ...(files.length === 0 ? {} : { files }),
+  };
 }