@henols/c64-re-tools 2.0.0-rc.2 → 2.0.0-rc.4

package/skills/c64-testing/scripts/scenario.ts

@@ -3,19 +3,11 @@
 // Observations name semantic symbols that resolve per side, comparisons are
 // explicit, and the result is PASS, FAIL or INCONCLUSIVE.
 
-import { formatC64Address, parseC64Address, textToPetscii } from "#src/c64.ts";
-import {
-  JOYSTICK_DIRECTIONS,
-  MAX_BASELINES,
-  MAX_EXECUTION_COUNT,
-  MAX_KEYBOARD_BYTES,
-  MAX_MASK_RECTANGLES,
-  MAX_MEMORY_READ,
-  MAX_MEMORY_WRITE,
-  MAX_TIMEOUT_FRAMES,
-  WireFailure,
-} from "#src/host-client/tools.ts";
-import { readProjectFile } from "#src/host-client/transfer.ts";
+import { formatC64Address, parseC64Address } from "#src/c64.ts";
+import { MAX_BASELINES, MAX_MASK_RECTANGLES, MAX_MEMORY_READ, WireFailure } from "#src/host-client/tools.ts";
+import { LOAD_LIMIT_FRAMES, RUN_LIMIT_FRAMES, runProgram } from "#src/host-client/autostart.ts";
+import { ACTION_STEP_KEYS, checkActionStep, checkLocation, isInteger, isObject, runActionStep, type ActionStep, type Location } from "#src/host-client/steps.ts";
+import { readProjectFile, writeProjectFile } from "#src/host-client/transfer.ts";
 import type { ViceSessionClient } from "#src/host-client/vice-session.ts";
 import { openForRead } from "#src/knowledge/database.ts";
 import { listSymbols } from "#src/knowledge/read.ts";
@@ -26,7 +18,7 @@ export type Side = "original" | "rebuild";
 export const SIDES: Side[] = ["original", "rebuild"];
 
 /** A "$xxxx" address or a symbol name that each side resolves on its own. */
-export type Location = string;
+export type { Location };
 
 export interface MemoryObservation {
   at: Location;
@@ -34,6 +26,8 @@ export interface MemoryObservation {
   size?: number;
   /** The largest allowed difference per byte; 0 when not given. */
   tolerance?: number;
+  /** Bytes in the range that the comparison leaves out, each with the reason (D46). */
+  exclude?: Array<{ at: Location; reason: string }>;
 }
 
 export interface ScreenPolicy {
@@ -43,18 +37,18 @@ export interface ScreenPolicy {
   mask?: Array<{ x: number; y: number; width: number; height: number }>;
 }
 
-export type Step =
+type StepBody =
   | { reset: "hard" | "soft" }
   | { load: true }
   | { autostart: true }
-  | { write: { at: Location; bytes: number[] } }
-  | { registers: Partial<Record<"pc" | "a" | "x" | "y" | "sp", number | Location>> }
-  | { joystick: { port: 1 | 2; direction: JoystickDirection; fire?: boolean } }
-  | { type: string }
-  | { frames: number }
-  | { runUntil: { at: Location } | { memory: { at: Location; equals: number } }; timeoutFrames: number }
+  | { snapshot: string }
+  | { normalize: string }
+  | ActionStep
   | { observe: string; memory?: Array<Location | MemoryObservation>; registers?: ObservedRegister[]; screen?: ScreenPolicy; vicii?: boolean; sprites?: number[] };
 
+/** A step; with side, only that side does it (D46). An observe step has no side: it compares both. */
+export type Step = StepBody & { side?: Side };
+
 export interface Subject {
   program: string;
   /** "knowledge" for knowledge.db, a project JSON file (c64-assembler output or a name → "$xxxx" map), or "none". */
@@ -85,26 +79,23 @@ export interface ScenarioResult {
   reason?: string;
   checkpoints: string[];
   differences: Difference[];
+  /** The bytes that the comparison left out, with their reasons (D46). */
+  excluded?: Array<{ checkpoint: string; at: Location; reason: string }>;
+  /** The files that a FAIL wrote with --out. */
+  files?: string[];
 }
 
-const STEP_KINDS = ["reset", "load", "autostart", "write", "registers", "joystick", "type", "frames", "runUntil", "observe"];
+const STEP_KINDS = ["reset", "load", "autostart", "snapshot", "normalize", "write", "registers", "joystick", "type", "frames", "runUntil", "observe"];
 
 /** The keys each kind of step can have. */
 const STEP_KEYS: Record<string, readonly string[]> = {
-  runUntil: ["runUntil", "timeoutFrames"],
+  ...ACTION_STEP_KEYS,
   observe: ["observe", "memory", "registers", "screen", "vicii", "sprites"],
 };
 
 /** The registers an observe step can compare. */
 export const OBSERVED_REGISTERS = ["pc", "a", "x", "y", "sp", "flags"] as const;
 type ObservedRegister = (typeof OBSERVED_REGISTERS)[number];
-/** The registers a registers step can set. */
-const SET_REGISTERS = ["pc", "a", "x", "y", "sp"];
-
-type JoystickDirection = (typeof JOYSTICK_DIRECTIONS)[number];
-
-const isObject = (value: unknown): value is Record<string, unknown> => typeof value === "object" && value !== null && !Array.isArray(value);
-const isInteger = (value: unknown, min: number, max: number): value is number => Number.isInteger(value) && (value as number) >= min && (value as number) <= max;
 
 /** Reads and checks a scenario file. Throws ScenarioError naming the first problem. */
 export function readScenario(path: string): Scenario {
@@ -135,8 +126,9 @@ export function readScenario(path: string): Scenario {
     const kinds = "observe" in step ? ["observe"] : Object.keys(step).filter((key) => STEP_KINDS.includes(key));
     if (kinds.length !== 1) return fail(`${at} must have exactly one of ${STEP_KINDS.join(", ")}`);
     const kind = kinds[0]!;
-    const allowed = STEP_KEYS[kind] ?? [kind];
+    const allowed = [...(STEP_KEYS[kind] ?? [kind]), ...(kind === "observe" ? [] : ["side"])];
     for (const key of Object.keys(step)) if (!allowed.includes(key)) fail(`${at} has the key ${key}, which a ${kind} step does not have`);
+    if (step.side !== undefined && !(SIDES as unknown[]).includes(step.side)) fail(`${at}: side must be original or rebuild`);
     checkStep(kind, step, at, fail);
     if (kind === "observe") {
       if (labels.has(step.observe as string)) fail(`${at} needs a new checkpoint name`);
@@ -149,18 +141,6 @@ export function readScenario(path: string): Scenario {
   return scenario;
 }
 
-/** Checks a "$xxxx" address or a symbol name. */
-function checkLocation(value: unknown, what: string, fail: (message: string) => never): void {
-  if (typeof value !== "string" || value === "") fail(`${what} must be a $ address or a symbol name`);
-  if ((value as string).startsWith("$")) {
-    try {
-      parseC64Address(value as string);
-    } catch {
-      fail(`${what} must be $ followed by four hex digits, not ${value as string}`);
-    }
-  }
-}
-
 /** Checks the value of one step against its kind. */
 function checkStep(kind: string, step: Record<string, unknown>, at: string, fail: (message: string) => never): void {
   switch (kind) {
@@ -171,60 +151,19 @@ function checkStep(kind: string, step: Record<string, unknown>, at: string, fail
     case "autostart":
       if (step[kind] !== true) fail(`${at}: ${kind} must be true`);
       return;
-    case "write": {
-      const write = step.write;
-      if (!isObject(write) || Object.keys(write).some((key) => key !== "at" && key !== "bytes")) return fail(`${at}: write needs at and bytes, and nothing else`);
-      checkLocation(write.at, `${at}: write.at`, fail);
-      if (!Array.isArray(write.bytes) || write.bytes.length === 0 || write.bytes.length > MAX_MEMORY_WRITE || !write.bytes.every((byte) => isInteger(byte, 0, 0xff))) {
-        fail(`${at}: write.bytes must be a list of 1 to ${MAX_MEMORY_WRITE} numbers from 0 to 255`);
-      }
-      return;
-    }
-    case "registers": {
-      const registers = step.registers;
-      if (!isObject(registers) || Object.keys(registers).length === 0) return fail(`${at}: registers must name at least one of ${SET_REGISTERS.join(", ")}`);
-      for (const [name, value] of Object.entries(registers)) {
-        if (!SET_REGISTERS.includes(name)) fail(`${at}: registers can set only ${SET_REGISTERS.join(", ")}, not ${name}`);
-        if (typeof value === "string") checkLocation(value, `${at}: registers.${name}`, fail);
-        else if (!isInteger(value, 0, name === "pc" ? 0xffff : 0xff)) fail(`${at}: registers.${name} must be a symbol name, a $ address or a number from 0 to ${name === "pc" ? 65535 : 255}`);
-      }
-      return;
-    }
-    case "joystick": {
-      const joystick = step.joystick;
-      if (!isObject(joystick) || Object.keys(joystick).some((key) => !["port", "direction", "fire"].includes(key))) return fail(`${at}: joystick needs port and direction, and fire is optional`);
-      if (joystick.port !== 1 && joystick.port !== 2) fail(`${at}: joystick.port must be 1 or 2`);
-      if (!(JOYSTICK_DIRECTIONS as readonly unknown[]).includes(joystick.direction)) fail(`${at}: joystick.direction must be one of ${JOYSTICK_DIRECTIONS.join(", ")}`);
-      if (joystick.fire !== undefined && typeof joystick.fire !== "boolean") fail(`${at}: joystick.fire must be true or false`);
+    case "snapshot":
+      if (typeof step.snapshot !== "string" || step.snapshot === "") fail(`${at}: snapshot needs a .vsf file in the project`);
       return;
-    }
-    case "type": {
-      if (typeof step.type !== "string" || step.type === "") return fail(`${at}: type needs text`);
-      let bytes: Uint8Array;
-      try {
-        bytes = textToPetscii(step.type);
-      } catch (error) {
-        return fail(`${at}: ${(error as Error).message}`);
-      }
-      if (bytes.length > MAX_KEYBOARD_BYTES) fail(`${at}: type can have at most ${MAX_KEYBOARD_BYTES} keys`);
+    case "normalize":
+      if (typeof step.normalize !== "string" || step.normalize === "") fail(`${at}: normalize needs an entry state file in the project`);
       return;
-    }
+    case "write":
+    case "registers":
+    case "joystick":
+    case "type":
     case "frames":
-      if (!isInteger(step.frames, 1, MAX_EXECUTION_COUNT)) fail(`${at} needs a frame count from 1 to ${MAX_EXECUTION_COUNT}`);
-      return;
-    case "runUntil": {
-      const target = step.runUntil;
-      if (!isObject(target) || Object.keys(target).length !== 1 || !("at" in target || "memory" in target)) return fail(`${at}: runUntil needs at or memory`);
-      if ("at" in target) checkLocation(target.at, `${at}: runUntil.at`, fail);
-      else {
-        const memory = target.memory;
-        if (!isObject(memory) || Object.keys(memory).some((key) => key !== "at" && key !== "equals")) return fail(`${at}: runUntil.memory needs at and equals, and nothing else`);
-        checkLocation(memory.at, `${at}: runUntil.memory.at`, fail);
-        if (!isInteger(memory.equals, 0, 0xff)) fail(`${at}: runUntil.memory.equals must be a number from 0 to 255`);
-      }
-      if (!isInteger(step.timeoutFrames, 1, MAX_TIMEOUT_FRAMES)) fail(`${at} needs timeoutFrames from 1 to ${MAX_TIMEOUT_FRAMES}`);
-      return;
-    }
+    case "runUntil":
+      return checkActionStep(kind, step, at, fail);
     default:
       checkObserve(step, at, fail);
   }
@@ -239,7 +178,15 @@ function checkObserve(step: Record<string, unknown>, at: string, fail: (message:
         checkLocation(item, `${at}: memory`, fail);
         continue;
       }
-      if (!isObject(item) || Object.keys(item).some((key) => !["at", "size", "tolerance"].includes(key))) return fail(`${at}: a memory object has at, size and tolerance, and nothing else`);
+      if (!isObject(item) || Object.keys(item).some((key) => !["at", "size", "tolerance", "exclude"].includes(key))) return fail(`${at}: a memory object has at, size, tolerance and exclude, and nothing else`);
+      if (item.exclude !== undefined) {
+        if (!Array.isArray(item.exclude) || item.exclude.length === 0) fail(`${at}: memory.exclude must be a list of objects with at and reason`);
+        for (const excluded of item.exclude as unknown[]) {
+          if (!isObject(excluded) || Object.keys(excluded).some((key) => key !== "at" && key !== "reason")) return fail(`${at}: an exclude has at and reason, and nothing else`);
+          checkLocation(excluded.at, `${at}: memory.exclude.at`, fail);
+          if (typeof excluded.reason !== "string" || excluded.reason.trim() === "") fail(`${at}: an exclude needs a reason: why this byte may differ`);
+        }
+      }
       checkLocation(item.at, `${at}: memory.at`, fail);
       if (item.size !== undefined && !isInteger(item.size, 1, MAX_MEMORY_READ)) fail(`${at}: memory.size must be a number from 1 to ${MAX_MEMORY_READ}`);
       if (item.tolerance !== undefined && !isInteger(item.tolerance, 0, 0xff)) fail(`${at}: memory.tolerance must be a number from 0 to 255`);
@@ -334,11 +281,34 @@ export function resolve(location: Location, symbols: Map<string, number>, side:
 }
 
 interface Observation {
-  memory: Array<{ label: string; bytes: number[]; tolerance: number }>;
+  memory: Array<{ label: string; address: number; bytes: number[]; tolerance: number; excluded: Array<{ offset: number; at: Location; reason: string }> }>;
   registers: Partial<Record<ObservedRegister, unknown>>;
   screen?: { match: boolean; mismatchingPixels: number; mismatchRatio: number; bounds?: unknown };
   vicii?: unknown;
   sprites?: unknown;
+  /** PNG images for the files of a FAIL (D46): the side's screen, and on the rebuild the difference image. */
+  png?: string;
+  diffPng?: string;
+}
+
+/** Writes pages 0-3 and $00/$01 from an entry state file (D43, D46): both sides then start from the same RAM. */
+async function normalize(client: ViceSessionClient, path: string): Promise<void> {
+  let state: { pages?: Record<string, string>; cpuPort?: Record<string, number> };
+  try {
+    state = JSON.parse(readProjectFile(path).bytes.toString("utf8")) as typeof state;
+  } catch (error) {
+    if (error instanceof WireFailure) throw error;
+    throw new ScenarioError(`${path} is not an entry state file: ${(error as Error).message}`);
+  }
+  const pages = ["$0000", "$0100", "$0200", "$0300"].map((page) => state.pages?.[page]);
+  const port = [state.cpuPort?.$00, state.cpuPort?.$01];
+  if (pages.some((page) => typeof page !== "string" || !/^[0-9a-f]{512}$/.test(page)) || port.some((value) => !isInteger(value, 0, 0xff))) {
+    throw new ScenarioError(`${path} is not an entry state file: it needs pages $0000 to $0300 and cpuPort $00 and $01`);
+  }
+  if ((await client.status()).state === "running") await client.execution({ action: "pause", space: "c64" });
+  // Pages 0 to 3 in RAM from $0002, then the CPU port: the port bits switch what the CPU sees.
+  await client.memoryWrite({ address: 0x0002, data: pages.join("").slice(4), space: "c64", view: "ram" });
+  await client.memoryWrite({ address: 0x0000, data: Buffer.from(port as number[]).toString("hex"), space: "c64", view: "cpu" });
 }
 
 /**
@@ -350,27 +320,39 @@ export function baselineName(checkpoint: number, label: string): string {
   return `test-${checkpoint}-${label}`.replace(/[^A-Za-z0-9._-]/g, "_").slice(0, 64);
 }
 
-async function runSide(client: ViceSessionClient, scenario: Scenario, side: Side, symbols: Map<string, number>): Promise<Map<string, Observation>> {
+/** Autostart that returns only when the program runs (runProgram), or INCONCLUSIVE with the reason. */
+async function startProgram(client: ViceSessionClient, program: string, side: Side): Promise<void> {
+  const failed = await runProgram(client, program, readProjectFile(program).bytes);
+  if (failed === "not-loaded") throw new Inconclusive(`the ${side} program was not in memory after ${LOAD_LIMIT_FRAMES} frames of autostart`);
+  if (failed === "not-started") throw new Inconclusive(`the ${side} program did not start within ${RUN_LIMIT_FRAMES} frames of RUN; autostart runs only a program with a BASIC start line`);
+}
+
+async function runSide(client: ViceSessionClient, scenario: Scenario, side: Side, symbols: Map<string, number>, files: boolean): Promise<Map<string, Observation>> {
   const subject = scenario[side];
   const observations = new Map<string, Observation>();
   const at = (location: Location) => resolve(location, symbols, side);
-  const stop = async () => {
-    if ((await client.status()).state === "running") await client.execution({ action: "pause", space: "c64" });
-  };
   // Each side starts from a hard reset, stopped at the reset vector, with the control ports released.
   await client.reset({ mode: "hard", run: false });
   await client.joystick({ port: 1, direction: "center", fire: false });
   await client.joystick({ port: 2, direction: "center", fire: false });
   let checkpoint = 0;
   for (const step of scenario.steps) {
+    if (step.side !== undefined && step.side !== side) continue;
     // Observe first: an observe step may also carry "registers" or "memory".
     if ("observe" in step) {
       checkpoint++;
       const observation: Observation = { memory: [], registers: {} };
       for (const item of step.memory ?? []) {
-        const spec = typeof item === "string" ? { at: item } : item;
-        const read = await client.memoryRead({ address: at(spec.at), size: spec.size ?? 1, space: "c64", view: "cpu" });
-        observation.memory.push({ label: spec.size !== undefined && spec.size > 1 ? `${spec.at} (${spec.size} bytes)` : spec.at, bytes: [...Buffer.from(read.data, "hex")], tolerance: spec.tolerance ?? 0 });
+        const spec: MemoryObservation = typeof item === "string" ? { at: item } : item;
+        const address = at(spec.at);
+        const size = spec.size ?? 1;
+        const read = await client.memoryRead({ address, size, space: "c64", view: "cpu" });
+        const excluded = (spec.exclude ?? []).map((exclude) => {
+          const offset = at(exclude.at) - address;
+          if (offset < 0 || offset >= size) throw new ScenarioError(`the exclude ${exclude.at} lies outside the observed ${spec.at} (${size} bytes)`);
+          return { offset, at: exclude.at, reason: exclude.reason };
+        });
+        observation.memory.push({ label: size > 1 ? `${spec.at} (${size} bytes)` : spec.at, address, bytes: [...Buffer.from(read.data, "hex")], tolerance: spec.tolerance ?? 0, excluded });
       }
       if (step.registers !== undefined) {
         const registers = await client.registersGet("c64");
@@ -385,10 +367,15 @@ async function runSide(client: ViceSessionClient, scenario: Scenario, side: Side
       if (step.screen !== undefined) {
         const baseline = baselineName(checkpoint, step.observe);
         if (side === "original") {
-          await client.screenCapture(baseline);
+          const shot = await client.screenCapture(baseline);
+          if (files) observation.png = shot.png;
         } else {
-          const compared = await client.screenCompare({ baseline, maxMismatchRatio: step.screen.maxMismatchRatio ?? 0, mask: step.screen.mask ?? [], includeDiff: false });
+          const compared = await client.screenCompare({ baseline, maxMismatchRatio: step.screen.maxMismatchRatio ?? 0, mask: step.screen.mask ?? [], includeDiff: files });
           observation.screen = { match: compared.match, mismatchingPixels: compared.mismatchingPixels, mismatchRatio: compared.mismatchRatio, ...(compared.bounds === undefined ? {} : { bounds: compared.bounds }) };
+          if (files) {
+            observation.png = (await client.screenCapture()).png;
+            if (compared.diffPng !== undefined) observation.diffPng = compared.diffPng;
+          }
           await client.screenDiscard(baseline);
         }
       }
@@ -399,30 +386,16 @@ async function runSide(client: ViceSessionClient, scenario: Scenario, side: Side
     } else if ("load" in step) {
       await client.programLoad({ path: subject.program });
     } else if ("autostart" in step) {
-      await client.autostart({ path: subject.program, index: 0, run: true });
-    } else if ("write" in step) {
-      await stop();
-      await client.memoryWrite({ address: at(step.write.at), data: Buffer.from(step.write.bytes).toString("hex"), space: "c64", view: "cpu" });
-    } else if ("registers" in step) {
-      await stop();
-      const values: Record<string, number> = {};
-      for (const [name, value] of Object.entries(step.registers)) values[name] = typeof value === "number" ? value : at(value as Location);
-      await client.registersSet("c64", values);
-    } else if ("joystick" in step) {
-      await client.joystick({ port: step.joystick.port, direction: step.joystick.direction, fire: step.joystick.fire ?? false });
-    } else if ("type" in step) {
-      await client.keyboard(textToPetscii(step.type));
-    } else if ("frames" in step) {
-      await stop();
-      await client.execution({ action: "advance-frames", count: step.frames, space: "c64" });
-    } else if ("runUntil" in step) {
-      const target =
-        "at" in step.runUntil
-          ? { kind: "address" as const, address: at(step.runUntil.at), space: "c64" as const }
-          : { kind: "memory" as const, address: at(step.runUntil.memory.at), operator: "eq" as const, value: step.runUntil.memory.equals, space: "c64" as const, view: "cpu" as const };
-      const reached = await client.runUntil({ target, timeoutFrames: step.timeoutFrames });
-      if (!reached.reached) {
-        throw new Inconclusive(`the ${side} side did not reach the checkpoint before ${step.timeoutFrames} frames (it stopped by ${reached.stopReason} at ${formatC64Address(reached.pc)})`);
+      await startProgram(client, subject.program, side);
+    } else if ("snapshot" in step) {
+      await client.snapshotRestore(readProjectFile(step.snapshot).bytes);
+    } else if ("normalize" in step) {
+      await normalize(client, step.normalize);
+    } else {
+      const { side: _side, ...action } = step;
+      const notReached = await runActionStep(client, action as ActionStep, at);
+      if (notReached !== undefined && "runUntil" in step) {
+        throw new Inconclusive(`the ${side} side did not reach the checkpoint before ${step.timeoutFrames} frames (it stopped by ${notReached.stopReason} at ${formatC64Address(notReached.pc)})`);
       }
     }
   }
@@ -436,7 +409,8 @@ export function compare(original: Map<string, Observation>, rebuild: Map<string,
     const right = rebuild.get(checkpoint)!;
     left.memory.forEach((item, index) => {
       const other = right.memory[index]!;
-      if (item.bytes.some((byte, at) => Math.abs(byte - other.bytes[at]!) > item.tolerance)) {
+      const skipped = new Set(item.excluded.map((excluded) => excluded.offset));
+      if (item.bytes.some((byte, at) => !skipped.has(at) && Math.abs(byte - other.bytes[at]!) > item.tolerance)) {
         const show = (bytes: number[]) => (bytes.length === 1 ? bytes[0] : Buffer.from(bytes).toString("hex"));
         differences.push({ checkpoint, what: `memory ${item.label}`, original: show(item.bytes), rebuild: show(other.bytes) });
       }
@@ -453,13 +427,46 @@ export function compare(original: Map<string, Observation>, rebuild: Map<string,
   return differences;
 }
 
-/** Runs the scenario on the original, then on the rebuild, in one VICE session. */
-export async function runScenario(client: ViceSessionClient, scenario: Scenario): Promise<ScenarioResult> {
+/**
+ * Writes the files of a FAIL into `out` (D46): for a screen difference both
+ * screens and the difference image, and memory.json with each differing byte.
+ */
+function writeFailFiles(out: string, original: Map<string, Observation>, rebuild: Map<string, Observation>, differences: Difference[]): string[] {
+  const files: string[] = [];
+  const write = (name: string, bytes: Uint8Array) => {
+    writeProjectFile(`${out}/${name}`, bytes, { what: "--out" });
+    files.push(name);
+  };
+  const memory: Array<{ checkpoint: string; what: string; bytes: Array<{ address: string; original: number; rebuild: number }> }> = [];
+  for (const difference of differences) {
+    const left = original.get(difference.checkpoint)!;
+    const right = rebuild.get(difference.checkpoint)!;
+    const name = difference.checkpoint.replace(/[^A-Za-z0-9._-]/g, "_");
+    if (difference.what === "screen") {
+      if (left.png !== undefined) write(`${name}-original.png`, Buffer.from(left.png, "base64"));
+      if (right.png !== undefined) write(`${name}-rebuild.png`, Buffer.from(right.png, "base64"));
+      if (right.diffPng !== undefined) write(`${name}-diff.png`, Buffer.from(right.diffPng, "base64"));
+    }
+    if (difference.what.startsWith("memory ")) {
+      const index = left.memory.findIndex((item) => `memory ${item.label}` === difference.what);
+      const item = left.memory[index]!;
+      const other = right.memory[index]!;
+      const skipped = new Set(item.excluded.map((excluded) => excluded.offset));
+      const bytes = item.bytes.flatMap((byte, offset) => (!skipped.has(offset) && Math.abs(byte - other.bytes[offset]!) > item.tolerance ? [{ address: formatC64Address(item.address + offset), original: byte, rebuild: other.bytes[offset]! }] : []));
+      memory.push({ checkpoint: difference.checkpoint, what: difference.what, bytes });
+    }
+  }
+  if (memory.length > 0) write("memory.json", Buffer.from(`${JSON.stringify(memory, null, 2)}\n`));
+  return files;
+}
+
+/** Runs the scenario on the original, then on the rebuild, in one VICE session. With out, a FAIL leaves files there. */
+export async function runScenario(client: ViceSessionClient, scenario: Scenario, options: { out?: string } = {}): Promise<ScenarioResult> {
   const checkpoints = scenario.steps.flatMap((step) => ("observe" in step ? [step.observe] : []));
   const observed = new Map<Side, Map<string, Observation>>();
   for (const side of SIDES) {
     try {
-      observed.set(side, await runSide(client, scenario, side, loadSymbols(scenario[side])));
+      observed.set(side, await runSide(client, scenario, side, loadSymbols(scenario[side]), options.out !== undefined));
     } catch (error) {
       if (error instanceof Inconclusive) return { result: "INCONCLUSIVE", scenario: scenario.name, reason: error.message, checkpoints, differences: [] };
       if (error instanceof WireFailure) {
@@ -468,6 +475,17 @@ export async function runScenario(client: ViceSessionClient, scenario: Scenario)
       throw error;
     }
   }
-  const differences = compare(observed.get("original")!, observed.get("rebuild")!);
-  return { result: differences.length === 0 ? "PASS" : "FAIL", scenario: scenario.name, checkpoints, differences };
+  const original = observed.get("original")!;
+  const rebuild = observed.get("rebuild")!;
+  const differences = compare(original, rebuild);
+  const excluded = [...original].flatMap(([checkpoint, observation]) => observation.memory.flatMap((item) => item.excluded.map((excluded) => ({ checkpoint, at: excluded.at, reason: excluded.reason }))));
+  const files = differences.length > 0 && options.out !== undefined ? writeFailFiles(options.out, original, rebuild, differences) : [];
+  return {
+    result: differences.length === 0 ? "PASS" : "FAIL",
+    scenario: scenario.name,
+    checkpoints,
+    differences,
+    ...(excluded.length === 0 ? {} : { excluded }),
+    ...(files.length === 0 ? {} : { files }),
+  };
 }

package/skills/c64-testing/scripts/test.ts

@@ -6,22 +6,32 @@ import { parseArgs } from "node:util";
 
 import { ViceSessionClient } from "#src/host-client/vice-session.ts";
 import { WireFailure } from "#src/host-client/tools.ts";
+import { resolveProjectPath } from "#src/project.ts";
 import { runScript, UsageError } from "#src/script.ts";
 import { readScenario, runScenario, ScenarioError, type ScenarioResult } from "./scenario.ts";
 
-const USAGE = `test.ts <scenario.json>
+const USAGE = `test.ts <scenario.json> [--out <dir>]
 
 <scenario.json> is a scenario file relative to the project directory. See
-references/scenario-format.md. The result is one JSON object.`;
+references/scenario-format.md. With --out, a FAIL writes both screens, the
+difference image and memory.json with each differing byte into <dir>. The
+result is one JSON object.`;
 
 const EXIT = { PASS: 0, FAIL: 1, INCONCLUSIVE: 3 } as const;
 
 async function run(argv: string[]): Promise<ScenarioResult> {
-  const { values, positionals } = parseArgs({ args: argv, strict: true, allowPositionals: true, options: { help: { type: "boolean" } } });
+  const { values, positionals } = parseArgs({ args: argv, strict: true, allowPositionals: true, options: { out: { type: "string" }, help: { type: "boolean" } } });
   if (values.help) throw new UsageError("");
   if (positionals.length !== 1) throw new UsageError("give one scenario file");
+  if (values.out !== undefined) {
+    try {
+      resolveProjectPath(values.out);
+    } catch {
+      throw new UsageError(`--out must be a path relative to the project directory that stays inside it, not ${values.out}`);
+    }
+  }
   try {
-    return await runScenarioFile(positionals[0]!);
+    return await runScenarioFile(positionals[0]!, values.out);
   } catch (error) {
     // A scenario or symbol file that is not usable is a bad call.
     if (error instanceof ScenarioError) throw new UsageError(error.message);
@@ -29,7 +39,7 @@ async function run(argv: string[]): Promise<ScenarioResult> {
   }
 }
 
-async function runScenarioFile(path: string): Promise<ScenarioResult> {
+async function runScenarioFile(path: string, out: string | undefined): Promise<ScenarioResult> {
   const scenario = readScenario(path);
   let client: ViceSessionClient;
   try {
@@ -39,7 +49,7 @@ async function runScenarioFile(path: string): Promise<ScenarioResult> {
     return { result: "INCONCLUSIVE", scenario: scenario.name, reason: `no emulator: ${error.message}`, checkpoints: [], differences: [] };
   }
   try {
-    return await runScenario(client, scenario);
+    return await runScenario(client, scenario, out === undefined ? {} : { out });
   } finally {
     await client.close();
   }

package/skills/c64-unpacker/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: c64-unpacker
-description: Use this skill to find out if a packer or a cruncher compressed a C64 program, and to get the unpacked program for analysis. It shows evidence of packing without a guess of the packer name. It runs the program in an emulator until the unpacked code starts, then writes the memory into the project. Do not use it to analyze the unpacked code.
+description: Use this skill to find out if a packer or a cruncher compressed a C64 program, and to get the unpacked program for analysis. It shows evidence of packing without a guess of the packer name. It runs the program in an emulator until the unpacked code starts, then writes the memory into the project. It also captures each part of a multi-load program and the drive RAM. Do not use it to analyze the unpacked code.
 ---
 
 # c64-unpacker
@@ -14,17 +14,19 @@ This skill does not change project knowledge.
 ## Inputs
 
 - The program: a PRG file, relative to the project directory.
-- For a capture: the address where the unpacked program starts, and where to write the result.
+- For a capture: the address where the unpacked program starts, and the directory for the result.
 
 ## Tools / execution path
 
 ```
 node <skill>/scripts/unpack.ts inspect original/game.prg
 node <skill>/scripts/unpack.ts trace original/game.prg
-node <skill>/scripts/unpack.ts capture original/game.prg --until '$c000' --out unpacked/game.prg --range '$c000' '$cfff'
+node <skill>/scripts/unpack.ts capture original/game.prg --until '$c000' --out unpacked
+node <skill>/scripts/unpack.ts stages original/game.prg --steps input.json
+node <skill>/scripts/unpack.ts loads original/game.d64 --steps parts.json --out parts --drive
 ```
 
-`inspect` examines the file locally. `trace` and `capture` start their own emulator through the host runtime and start the program. `trace` runs the program and finds the memory that the program wrote and then executed. `capture` stops the program when the CPU gets to the `--until` address.
+`inspect` examines the file locally. `trace` and `capture` start their own emulator through the host runtime and start the program. `trace` runs the program and finds the memory that the program wrote and then executed. `capture` stops the program when the CPU gets to the `--until` address. `stages` runs the program and finds each stage: a decruncher, an intro, a copier, and then the program. For each stage it gives the entry, the frames, the written memory (`wrote`) and the entry of the next stage (`next`). In `stages`, a `frames` step counts as run time, and a `runUntil` step is not possible.
 
 | Option | Meaning |
 |---|---|
@@ -34,25 +36,71 @@ node <skill>/scripts/unpack.ts capture original/game.prg --until '$c000' --out u
 | `--timeout-frames 3000` | How many frames the program can run before the capture stops (default 3000). |
 | `--frames 3000` | `trace` only: how many frames the program runs (default 3000). |
 | `--entry '$c000'` | `trace` only: start the program at this address. Without it, `trace` types RUN. A program without a BASIC start line needs it. |
+| `--steps input.json` | Input for the program, right after it starts: a JSON list of steps. Use it when an intro waits for a key or a menu waits for a choice. |
+| `--drive` | `loads` only: each capture also writes the 2 KiB RAM of the 1541 drive as `drive8.prg`. |
+| `--min-block 64` | `loads` only: the smallest block of changed memory that becomes a segment (default 64). The result counts smaller changes as `smallChanges`. |
+| `--min-stage 64` | `stages` only: the smallest block of written memory that starts a new stage when the CPU executes it (default 64). |
 
 The capture reads RAM, also the RAM under the ROMs and the I/O area. The script prints one JSON object.
 
+A steps file has the steps `write`, `registers`, `joystick`, `type`, `frames` and `runUntil` of the c64-testing skill. Addresses are `$` and four hex digits. For example, wait 50 frames and press the space bar:
+
+```json
+[{ "frames": 50 }, { "type": " " }]
+```
+
+A `type` step puts keys in the keyboard queue of the KERNAL. A program that reads the keyboard hardware does not see them. A step cannot press one specific key of the keyboard matrix. A wait for any key that reads all keyboard columns can end without a key.
+
 ## Workflow
 
 1. Run `inspect`. Read `basicStart` and `packing`:
    - `likely`: most of the bytes look compressed. Packed data has this property.
    - `unlikely`: the bytes look like code and data. A simple cruncher can also give this result.
-   - `unclear`: the file is short, or the blocks do not give clear evidence.
+   - `unclear`: the file is short, or the blocks do not give clear evidence. Or the code at the SYS address decodes or moves bytes outside the program.
+
+   Also read the evidence that the script finds in the file. The script never names a packer from it:
+   - `basicText`: the BASIC lines as text. A packer or a crack often puts its name there.
+   - `strings`: readable text near the start of the code.
+   - `start`: the code at the SYS address. `jumpsOutside` and `writesOutside` give an address outside the loaded program, where the code jumps or writes.
+   - `decodeLoops`: EOR instructions in loops that store with an index. This is usually decryption or decoding.
 2. Run `trace`. It gives the evidence from the run. Read `packing`:
    - `likely`: the program wrote code and then executed it. It unpacked, decrypted or moved code. `runningRange` is the code that runs now. Its `start` is usually the entry point of the unpacked program.
    - `unclear`: the program executed a small quantity of code that it wrote. This can be code that changes itself.
    - `unlikely`: the program executed no code that it wrote. If the program waits for a key or a joystick, it can move code after that input. Then use the c64-emulator skill.
 3. Find the address where the unpacked program starts:
-   - Use `runningRange.start` from `trace`, or `suggestedUntil`. The CPU gets to `suggestedUntil` after the unpacking.
+   - Run `stages`. The `entry` of the last stage is usually the start of the program.
+   - Or use `runningRange.start` from `trace`, or `suggestedUntil`. The CPU gets to `suggestedUntil` after the unpacking.
    - Or use the c64-emulator skill: set a breakpoint after the unpack loop, or look at the last JMP of the unpacker.
-4. Run `capture` with that address. Use `--range` for the area that holds the unpacked program, if you know it.
-5. Make sure that the result is the application: the code at the start address must not be the unpacker again.
-6. Give the result to the c64-static-analysis skill. For a flat 64 KiB file, use its `--flat64k` option.
+4. Run `capture` with that address and a directory in `--out`. Give the same `--steps` as before. The directory gets:
+   - `image.bin`: all 64 KiB of RAM at the entry.
+   - `image.json`: the entry, the segments, the vectors and the stages. A segment is a block that a stage wrote. Blocks with the code of an earlier stage are not segments. `writtenBy` tells which stage wrote the segment.
+   - One PRG for each segment.
+   - `entry_state.json`: the registers, `$00` and `$01`, pages 0 to 3, the VIC, SID and CIA registers, and the vectors. `differsFromReset` lists the bytes that are not as a clean C64 has them at READY. A rebuild must make these bytes the same, because the program can read them.
+5. Make sure that the result is the application: the code at the entry must not be the unpacker again. A segment can be data that a loader left, for example intro graphics. Keep it until other evidence shows that the program does not use it.
+6. Give `image.bin` to the c64-static-analysis skill with its `--flat64k` option, or give it the segment PRGs.
+7. Use `--range` only when you know the area of the program. Then the capture writes that one segment.
+
+## Multi-load programs
+
+A multi-load program loads more parts from the disk while it runs, for example one part for each level. Use `loads` to capture each part. The program can be a PRG, or a disk, tape or cartridge image that VICE starts.
+
+1. Write a steps file. Put a `capture` step after each point where a part is in memory, for example:
+
+```json
+[
+  { "runUntil": { "at": "$080d" }, "timeoutFrames": 6000 },
+  { "capture": "main" },
+  { "type": " " },
+  { "runUntil": { "at": "$c000" }, "timeoutFrames": 6000 },
+  { "capture": "level1" }
+]
+```
+
+2. Run `loads` with the steps file and a directory in `--out`. Add `--drive` when the program has its own drive code, for example a fast loader.
+3. Each capture writes `<out>/<name>/` with `image.bin`, `image.json`, `entry_state.json` and a PRG for each segment. A segment is a block of RAM from `$0400` that the program changed after the capture before. The first capture also has the program itself.
+4. With `--drive`, `drive8.prg` is the drive RAM at `$0000`-`$07ff`. Analyze it with c64-static-analysis and `--space drive8`, and record its knowledge in space `drive8`.
+
+VICE records no memory access map for the drive CPU. Thus no tool gives coverage of drive code.
 
 ## Knowledge
 
@@ -63,18 +111,21 @@ Do not record the unpacker in the project knowledge, unless the user asks to exa
 
 ## Result
 
-The packing evidence, or the unpacked program in the project with the address where it starts.
+The packing evidence, the stages, or the unpacked program with its image, segments and entry state in a directory of the project.
+
+For `loads`: `captures`, with the name, the PC, the segments and the drive file of each capture.
 
-The exit status is 0 for a result. The exit status is 1 for `traced: false`, for `captured: false` and for an error. The exit status is 2 for a wrong argument.
+The exit status is 0 for a result. The exit status is 1 for `traced: false`, for `captured: false`, for `loaded: false` and for an error. The exit status is 2 for a wrong argument.
 
 ## Failure and conflicts
 
 - `traced: false`: the emulator did not load the program into memory in the time limit. The reason gives the load address. Make sure that the file is a PRG that the C64 can load.
-- `captured: false`: the program did not get to the address in the time limit. The reason tells where it stopped. Find a better address with the c64-emulator skill, or give more frames.
+- `captured: false`: the program did not get to the address in the time limit. The reason tells where it stopped. Find a better address with the c64-emulator skill, or give more frames. If it stopped in a key wait, give `--steps` with a key.
+- A step did not reach its target: the reason names the step. Change the step or give it more frames.
 - Do not name a packer from weak evidence. The script never names one.
-- Some programs unpack in more than one stage. Make sure that the captured code is the application and not a second unpacker.
+- Some programs unpack in more than one stage. Run `stages` to find each stage. Capture at the entry of the last stage. Make sure that it is the application and not a second unpacker.
 - `not-found` or `invalid-input`: a path or an argument is wrong. Read the message.
-- `operation-failed` with a message about the host runtime: the host runtime does not run. Tell the user to start the host runtime with `npx -y --package=@henols/c64-re-tools@latest c64-re-tools-host`.
+- `installation-incomplete`, or a message that the host runtime does not run: stop the task. Give the message to the user and wait for the user. The message gives the remedy. Do not make a replacement for the tool or the host runtime, for example your own disk parser, disassembler or emulator script.
 
 ## Handoffs