@henols/c64-re-tools 2.1.0 → 2.1.2

package/skills/c64-unpacker/scripts/unpacker.ts

@@ -9,17 +9,17 @@
 import { type AddressRange, COVERAGE, EXECUTE, formatC64Address, READ, WRITE } from "#src/c64.ts";
 import { autostartLoaded, LOAD_LIMIT_FRAMES, RUN_LIMIT_FRAMES, typeRun } from "#src/host-client/autostart.ts";
 import { readCoverage, writtenBlocks } from "#src/host-client/coverage.ts";
-import { resetToReady } from "#src/host-client/entry-state.ts";
 import { readActionSteps, runActionStep, type ActionStep } from "#src/host-client/steps.ts";
 import { WireFailure } from "#src/host-client/tools.ts";
-import { readProjectFile, writeReported, type Written } from "#src/host-client/transfer.ts";
+import { readPrg } from "#src/host-client/image.ts";
+import { writeReported, type Written } from "#src/host-client/transfer.ts";
 import { ViceSessionClient } from "#src/host-client/vice-session.ts";
 import { knowledgeAt, knowledgeNames } from "#src/knowledge/names.ts";
-import { OPTIONS, runScript, UsageError, type Call, type CommandSpec, type OptionSpec } from "#src/script.ts";
+import { listed, OPTIONS, runScript, UsageError, type Call, type CommandSpec, type OptionSpec } from "#src/script.ts";
 import { basicStart, inspect, readMemoryMap, underRom, writtenThenExecuted } from "./lib/evidence.ts";
 import { loads, readLoadSteps } from "./lib/loads.ts";
 import { callReturn, captureSegments, executedIn, inRanges, mergeMaps, stageCandidates, stagePoints, withoutRanges } from "./lib/stages.ts";
-import { entryState, IO_AREAS, json, machineState, readRam, vectorsOf } from "./lib/machine.ts";
+import { entryStateFile, machineState, readRam, readyState, vectorsOf } from "./lib/machine.ts";
 
 /** Written-then-executed bytes from this many on are unpacking, decrypting or relocating, not self-changing code. */
 const UNPACKED_BYTES = 256;
@@ -38,10 +38,12 @@ async function runSteps(client: ViceSessionClient, steps: ActionStep[]): Promise
   return undefined;
 }
 
-async function trace(program: string, options: { frames: number; entry?: number; steps: ActionStep[] }): Promise<{ output: Record<string, unknown>; failed: boolean }> {
-  const { bytes } = readProjectFile(program);
-  if (bytes.length < 3) throw new UsageError("the file is too short for a PRG");
-  if (options.entry === undefined && basicStart(bytes) === undefined) throw new UsageError(NO_BASIC);
+/** inspect, trace, stages and capture take a PRG: a disk, tape or cartridge file goes to loads. */
+const NOT_PRG_REMEDY = "Use loads, which runs a disk, tape or cartridge image, or get the PRG out of it with disk.ts read of the c64-disk skill";
+const readProgram = (path: string) => readPrg(path, NOT_PRG_REMEDY).bytes;
+
+async function trace(program: string, options: { frames: number; entry?: number; steps: ActionStep[]; max: number }): Promise<{ output: Record<string, unknown>; failed: boolean }> {
+  const bytes = startable(program, options.entry);
   const client = await ViceSessionClient.open({ videoStandard: "pal" });
   try {
     // Loaded but not started, so the memory map can start empty: the KERNAL start-up writes all of RAM.
@@ -54,7 +56,12 @@ async function trace(program: string, options: { frames: number; entry?: number;
       const step = await client.execution({ action: "advance-frames", count: Math.min(MAX_ADVANCE, options.frames - ran), space: "c64" });
       const advanced = step.advancedFrames ?? 0;
       ran += advanced;
-      if (advanced === 0) break;
+      // A JAM or a point stops the run before its frames: the evidence would be of a run that did not happen.
+      if ((step.stopReason !== undefined && step.stopReason !== "frames") || advanced === 0) {
+        const stopReason = step.stopReason ?? "no-frames";
+        const at = step.pc ?? (await client.registersGet("c64")).pc;
+        return { output: { traced: false, frames: ran, stopReason, pc: formatC64Address(at), reason: stoppedEarly(stopReason, at, ran, options.frames) }, failed: true };
+      }
     }
     const { pc } = await client.registersGet("c64");
     const ranges = await readMemoryMap(async (start) => {
@@ -80,7 +87,7 @@ async function trace(program: string, options: { frames: number; entry?: number;
         traced: true,
         frames: ran,
         pc: formatC64Address(pc),
-        writtenThenExecuted: written.map((range) => ({
+        writtenThenExecuted: listed(written, options.max, (range) => ({
           start: formatC64Address(range.start),
           end: formatC64Address(range.end),
           ...(underRom(range.start) ? { underRom: true } : {}),
@@ -107,6 +114,12 @@ async function trace(program: string, options: { frames: number; entry?: number;
 
 const NO_BASIC = "the program is no BASIC program at $0801, so RUN cannot start it. Give --entry <address>.";
 
+/** Why a run stopped before its frames: what stopped the CPU, where, and after how many frames. */
+function stoppedEarly(stopReason: string, pc: number, ran: number, frames: number): string {
+  const what = stopReason === "jam" ? "a JAM instruction" : stopReason === "no-frames" ? "the emulator, which ran no frame" : `a ${stopReason}`;
+  return `The CPU stopped at ${formatC64Address(pc)} by ${what} after ${ran} of ${frames} frames. Check --entry and the steps: the program must run for the frames.`;
+}
+
 /** The first PC in one of the ranges within SYS_FRAMES frames, found with exec breakpoints over them. */
 async function firstPcIn(client: ViceSessionClient, ranges: Array<{ start: number; end: number }>): Promise<number | undefined> {
   const points = [];
@@ -151,6 +164,13 @@ async function firstPcAfterBasic(client: ViceSessionClient, from: number, to: nu
   return from > to ? undefined : firstPcIn(client, [{ start: from, end: to }]);
 }
 
+/**
+ * What a stage wrote: every written block, also a small one. --min-stage
+ * decides only where a stage starts; the wrote list and the capture segments
+ * keep each byte that a stage wrote.
+ */
+const stageWrote = (map: Uint8Array) => writtenBlocks(map, 1);
+
 /** Frames between two looks at the memory map while stages are followed. */
 const STAGE_CHUNK_FRAMES = 25;
 const MAX_STAGES = 16;
@@ -175,6 +195,8 @@ interface Followed {
   reached: boolean;
   /** Why the program did not start, when it did not. */
   failed?: string;
+  /** What stopped the CPU before the frames ran: a JAM or a point of the program. */
+  halted?: { stopReason: string; pc: number };
 }
 
 type RunUntilStep = Extract<ActionStep, { runUntil: unknown }>;
@@ -197,13 +219,11 @@ function trackedRunUntil(step: ActionStep): step is RunUntilStep & { runUntil: {
 async function follow(
   client: ViceSessionClient,
   program: string,
+  bytes: Uint8Array,
   options: { frames: number; entry?: number; steps: ActionStep[]; minStage: number; until?: number },
 ): Promise<Followed> {
-  const { bytes } = readProjectFile(program);
-  if (bytes.length < 3) throw new UsageError("the file is too short for a PRG");
   const load = bytes[0]! | (bytes[1]! << 8);
   const basic = basicStart(bytes);
-  if (options.entry === undefined && basic === undefined) throw new UsageError(NO_BASIC);
   const notStarted = await startLoaded(client, program, bytes, options.entry);
   if (notStarted !== undefined) return { stages: [], frames: 0, reached: false, failed: notStarted };
   // The frames of each step end where a scenario's do: the first step counts from RUN, a later one from where it starts.
@@ -216,6 +236,7 @@ async function follow(
   let firstFrame = 0;
   let total = 0;
   let ended = false;
+  let halted: { stopReason: string; pc: number } | undefined;
   /** Where the run stops: the target of a runUntil step while it runs, then until. */
   let stopAt: number | undefined;
   const atTarget = (pc: number) => stopAt !== undefined && pc === stopAt;
@@ -223,7 +244,7 @@ async function follow(
   let stage = { map: new Uint8Array(0x10000), own: new Set<number>(), called: [] as AddressRange[] };
   const startStage = async (next: number, last: Uint8Array) => {
     mergeMaps(stage.map, last);
-    found.push({ entry, firstFrame, frames: total - firstFrame, wrote: writtenBlocks(stage.map, options.minStage), next });
+    found.push({ entry, firstFrame, frames: total - firstFrame, wrote: stageWrote(stage.map), next });
     entry = next;
     firstFrame = total;
     stage = { map: new Uint8Array(0x10000), own: new Set<number>(), called: [] };
@@ -235,7 +256,7 @@ async function follow(
    * blocks, and stops at the first opcode that the stage wrote and the CPU
    * then executes. Own code stops the run once; a write watch arms it again.
    */
-  const rerun = async (candidates: AddressRange[], frames: number): Promise<{ used: number; map: Uint8Array; next?: number; atTarget: boolean }> => {
+  const rerun = async (candidates: AddressRange[], frames: number): Promise<{ used: number; map: Uint8Array; next?: number; atTarget: boolean; halted?: { stopReason: string; pc: number } }> => {
     const points = new Map<string, { kind: "exec" | "watch"; id: number }>();
     const arm = async (wanted: { exec: AddressRange[]; watch: AddressRange[] }) => {
       const keys = new Set([...wanted.exec.map((range) => `exec:${range.start}:${range.end}`), ...wanted.watch.map((range) => `watch:${range.start}:${range.end}`)]);
@@ -295,6 +316,7 @@ async function follow(
         left -= advanced;
         const pc = run.pc ?? (await client.registersGet("c64")).pc;
         map = await readCoverage(client);
+        if (run.stopReason === "jam") return { used, map, atTarget: false, halted: { stopReason: "jam", pc } };
         // A frame can end in written code that the stage called before: that stop starts no stage.
         if (inRanges(candidates, pc) && !inRanges(stage.called, pc)) {
           if (written(pc)) {
@@ -341,6 +363,14 @@ async function follow(
       const first = await client.execution({ action: "advance-frames", count: chunk, space: "c64", ...framed() });
       const firstPc = first.pc ?? (await client.registersGet("c64")).pc;
       const map = await readCoverage(client);
+      // A JAM, or a point that is not the target, ends the run before its frames.
+      if (first.stopReason !== undefined && first.stopReason !== "frames" && !atTarget(firstPc)) {
+        total += first.advancedFrames ?? 0;
+        mergeMaps(stage.map, map);
+        halted = { stopReason: first.stopReason, pc: firstPc };
+        ended = true;
+        continue;
+      }
       const candidates = stageCandidates(stage.map, map, stage.own, options.minStage, stage.called);
       if (candidates.length === 0) {
         const advanced = first.advancedFrames ?? 0;
@@ -369,6 +399,10 @@ async function follow(
       if (again.next !== undefined) await startStage(again.next, again.map);
       else mergeMaps(stage.map, again.map);
       if (again.atTarget) stopped = true;
+      if (again.halted !== undefined) {
+        halted = again.halted;
+        ended = true;
+      }
     }
     return stopped;
   };
@@ -389,7 +423,8 @@ async function follow(
     else if (trackedRunUntil(step)) {
       if (!(await trackTo(knowledgeAt()(step.runUntil.target.address), step.runUntil.timeoutFrames))) {
         const { pc } = await client.registersGet("c64");
-        return { stages: found, frames: total, reached: false, failed: `Step ${index + 1} did not reach its target in ${step.runUntil.timeoutFrames} frames. It is at ${formatC64Address(pc)}.` };
+        const why = halted === undefined ? "" : ` ${stoppedEarly(halted.stopReason, halted.pc, total, step.runUntil.timeoutFrames)}`;
+        return { stages: found, frames: total, reached: false, failed: `Step ${index + 1} did not reach its target in ${step.runUntil.timeoutFrames} frames. It is at ${formatC64Address(pc)}.${why}` };
       }
     } else await runActionStep(client, step, knowledgeAt());
   }
@@ -399,8 +434,8 @@ async function follow(
   else reached = await trackTo(options.until, left);
   await client.snapshot({ action: "remove", name: STAGE_SNAPSHOT }).catch(() => undefined);
   mergeMaps(stage.map, await readCoverage(client));
-  found.push({ entry, firstFrame, frames: total - firstFrame, wrote: writtenBlocks(stage.map, options.minStage) });
-  return { stages: found, frames: total, reached };
+  found.push({ entry, firstFrame, frames: total - firstFrame, wrote: stageWrote(stage.map) });
+  return { stages: found, frames: total, reached, ...(halted === undefined ? {} : { halted }) };
 }
 
 const showRange = (range: { start: number; end: number }) => ({ start: formatC64Address(range.start), end: formatC64Address(range.end) });
@@ -413,11 +448,23 @@ const showStages = (stages: Stage[]) =>
     ...(stage.next === undefined ? {} : { next: formatC64Address(stage.next) }),
   }));
 
+/** The bytes of a PRG that the emulator starts: RUN needs a BASIC program at $0801, else --entry gives the start. Checked before the emulator opens. */
+function startable(program: string, entry: number | undefined): Uint8Array {
+  const bytes = readProgram(program);
+  if (entry === undefined && basicStart(bytes) === undefined) throw new UsageError(NO_BASIC);
+  return bytes;
+}
+
 async function stages(program: string, options: { frames: number; entry?: number; steps: ActionStep[]; minStage: number }): Promise<{ output: Record<string, unknown>; failed: boolean }> {
+  const bytes = startable(program, options.entry);
   const client = await ViceSessionClient.open({ videoStandard: "pal" });
   try {
-    const followed = await follow(client, program, options);
+    const followed = await follow(client, program, bytes, options);
     if (followed.failed !== undefined) return { output: { followed: false, reason: followed.failed }, failed: true };
+    if (followed.halted !== undefined) {
+      const { stopReason, pc } = followed.halted;
+      return { output: { followed: false, frames: followed.frames, stopReason, pc: formatC64Address(pc), reason: stoppedEarly(stopReason, pc, followed.frames, options.frames), stages: showStages(followed.stages) }, failed: true };
+    }
     return {
       output: {
         followed: true,
@@ -437,14 +484,20 @@ async function capture(
   program: string,
   options: { until: number; out: string; range?: AddressRange; exclude: AddressRange[]; timeoutFrames: number; entry?: number; steps: ActionStep[]; minStage: number },
 ): Promise<{ output: Record<string, unknown>; failed: boolean }> {
+  const bytes = startable(program, options.entry);
   const client = await ViceSessionClient.open({ videoStandard: "pal" });
   try {
-    const followed = await follow(client, program, { frames: options.timeoutFrames, steps: options.steps, minStage: options.minStage, until: options.until, ...(options.entry === undefined ? {} : { entry: options.entry }) });
+    const followed = await follow(client, program, bytes, { frames: options.timeoutFrames, steps: options.steps, minStage: options.minStage, until: options.until, ...(options.entry === undefined ? {} : { entry: options.entry }) });
     if (followed.failed !== undefined) return { output: { captured: false, reason: followed.failed }, failed: true };
     const registers = await client.registersGet("c64");
     if (!followed.reached) {
       return {
-        output: { captured: false, reason: `The program did not get to ${formatC64Address(options.until)} in ${options.timeoutFrames} frames. It is at ${formatC64Address(registers.pc)}.`, stages: showStages(followed.stages) },
+        output: {
+          captured: false,
+          reason: `The program did not get to ${formatC64Address(options.until)} in ${options.timeoutFrames} frames. It is at ${formatC64Address(registers.pc)}.${followed.halted === undefined ? "" : ` ${stoppedEarly(followed.halted.stopReason, followed.halted.pc, followed.frames, options.timeoutFrames)}`}`,
+          ...(followed.halted === undefined ? {} : { stopReason: followed.halted.stopReason }),
+          stages: showStages(followed.stages),
+        },
         failed: true,
       };
     }
@@ -453,10 +506,6 @@ async function capture(
     const state = await machineState(client);
     // The segments: what the stages wrote, without loader leftovers, with no overlap.
     const found = options.range !== undefined ? [options.range] : captureSegments(followed.stages, registers.pc);
-    const inside = options.exclude.find((range) => registers.pc >= range.start && registers.pc <= range.end);
-    if (inside !== undefined) {
-      return { output: { captured: false, reason: `The entry ${formatC64Address(registers.pc)} is in --exclude ${formatC64Address(inside.start)}-${formatC64Address(inside.end)}. A segment must hold the entry.` }, failed: true };
-    }
     const segments = withoutRanges(found, options.exclude);
     const file = (path: string) => `${options.out}/${path}`;
     const written: Written[] = [];
@@ -486,14 +535,8 @@ async function capture(
       ),
     );
     // A clean machine at the READY prompt shows what the loader and the stages changed.
-    await resetToReady(client);
-    const clean = await machineState(client);
-    const differs: Array<{ address: string; entry: number; reset: number }> = [];
-    const compare = (base: number, entry: Buffer, reset: Buffer) => entry.forEach((value, offset) => value !== reset[offset] && differs.push({ address: formatC64Address(base + offset), entry: value, reset: reset[offset]! }));
-    compare(0x0000, state.port, clean.port);
-    compare(0x0002, state.pages.subarray(2), clean.pages.subarray(2));
-    for (const [name, [address]] of Object.entries(IO_AREAS)) compare(address, state.io[name as keyof typeof IO_AREAS], clean.io[name as keyof typeof IO_AREAS]);
-    write("entry_state.json", json(entryState(registers, state, vectors, differs)));
+    const entry = entryStateFile(registers, state, vectors, await readyState(client));
+    write("entry_state.json", entry.bytes);
     return {
       output: {
         captured: true,
@@ -503,7 +546,7 @@ async function capture(
         segments: segments.map(showRange),
         ...(options.exclude.length === 0 ? {} : { excluded: options.exclude.map(showRange) }),
         stages: followed.stages.length,
-        differsFromReset: differs.length,
+        differsFromReset: entry.differsFromReset,
       },
       failed: false,
     };
@@ -526,7 +569,7 @@ const entryOf = (call: Call) => {
 };
 const stepsOf = (call: Call) => {
   const path = call.string("steps");
-  return path === undefined ? [] : readActionSteps(path, "--steps");
+  return path === undefined ? [] : readActionSteps(path, "--steps", knowledgeAt());
 };
 
 const COMMANDS: CommandSpec[] = [
@@ -534,14 +577,14 @@ const COMMANDS: CommandSpec[] = [
     name: "inspect",
     summary: "Packing evidence: the BASIC start and the entropy of the bytes.",
     subjects: [PROGRAM],
-    run: (call) => inspect(readProjectFile(call.subject(0)).bytes),
+    run: (call) => inspect(readProgram(call.subject(0))),
   },
   {
     name: "trace",
     summary: "Runs the program and reports the memory that it wrote and then executed.",
     subjects: [PROGRAM],
-    options: { frames: FRAMES, entry: ENTRY, steps: STEPS },
-    run: async (call) => outcome(await trace(call.subject(0), { frames: call.count("frames")!, steps: stepsOf(call), ...entryOf(call) })),
+    options: { frames: FRAMES, entry: ENTRY, steps: STEPS, "max-listed": OPTIONS["max-listed"] },
+    run: async (call) => outcome(await trace(call.subject(0), { frames: call.count("frames")!, steps: stepsOf(call), max: call.count("max-listed")!, ...entryOf(call) })),
   },
   {
     name: "stages",
@@ -552,7 +595,8 @@ const COMMANDS: CommandSpec[] = [
     run: async (call) => {
       const steps = stepsOf(call);
       // A runUntil step to a raster line or a memory value cannot be tracked in chunks of frames; capture takes it.
-      if (steps.some((step) => "runUntil" in step && !trackedRunUntil(step))) throw new UsageError("stages takes a runUntil step only with an address target without a condition. Use frames steps, or capture");
+      const untracked = steps.findIndex((step) => "runUntil" in step && !trackedRunUntil(step));
+      if (untracked >= 0) throw new UsageError(`--steps ${call.string("steps")!}: step ${untracked + 1}: stages takes a runUntil step only with an address target without a condition. Use frames steps, or capture`);
       return outcome(await stages(call.subject(0), { frames: call.count("frames")!, steps, minStage: call.count("min-stage")!, ...entryOf(call) }));
     },
   },
@@ -562,7 +606,7 @@ const COMMANDS: CommandSpec[] = [
     subjects: [PROGRAM],
     options: {
       until: { kind: "address", value: "<address>", required: true, help: "the unpacked entry point" },
-      out: { ...OPTIONS.out, value: "<dir>", required: true, help: "the folder for the capture, relative to the project folder" },
+      out: { ...OPTIONS.out, output: "folder", value: "<dir>", required: true, help: "the folder for the capture, relative to the project folder" },
       range: { ...OPTIONS.range, help: "one segment in place of the found ones" },
       exclude: { kind: "range", value: "<start>-<end>", multiple: true, help: "a range to leave out of the segments, for example bytes that a crack left and the program never reads" },
       "timeout-frames": { ...OPTIONS["timeout-frames"], max: 30_000, default: 3000, help: "stop the wait for --until after n frames (default 3000)" },
@@ -573,7 +617,14 @@ const COMMANDS: CommandSpec[] = [
     notes: "entry_state.json has the registers, $00/$01, pages 0-3, the VIC, SID and CIA registers, the vectors, and what differs from a clean reset.",
     run: async (call) => {
       const range = call.range("range");
-      return outcome(await capture(call.subject(0), { until: call.address("until")!, out: call.string("out")!, ...(range === undefined ? {} : { range }), exclude: call.ranges("exclude"), timeoutFrames: call.count("timeout-frames")!, steps: stepsOf(call), minStage: call.count("min-stage")!, ...entryOf(call) }));
+      const until = call.address("until")!;
+      // A segment must hold the entry: refused before the emulator runs.
+      const show = (item: AddressRange) => `${formatC64Address(item.start)}-${formatC64Address(item.end)}`;
+      const holds = (item: AddressRange) => until >= item.start && until <= item.end;
+      const excluded = call.ranges("exclude").find(holds);
+      if (excluded !== undefined) throw new UsageError(`--exclude ${show(excluded)} holds the entry --until ${formatC64Address(until)}. A segment must hold the entry: give an --exclude range without it`);
+      if (range !== undefined && !holds(range)) throw new UsageError(`--range ${show(range)} does not hold the entry --until ${formatC64Address(until)}. Give a --range that holds the entry`);
+      return outcome(await capture(call.subject(0), { until, out: call.string("out")!, ...(range === undefined ? {} : { range }), exclude: call.ranges("exclude"), timeoutFrames: call.count("timeout-frames")!, steps: stepsOf(call), minStage: call.count("min-stage")!, ...entryOf(call) }));
     },
   },
   {
@@ -582,7 +633,7 @@ const COMMANDS: CommandSpec[] = [
     subjects: [{ name: "<program>", help: "a PRG, or a disk, tape or cartridge image, relative to the project folder" }],
     options: {
       steps: { ...STEPS, required: true, help: "action steps and capture steps (JSON)" },
-      out: { ...OPTIONS.out, value: "<dir>", required: true, help: "the folder for the captures, relative to the project folder" },
+      out: { ...OPTIONS.out, output: "folder", value: "<dir>", required: true, help: "the folder for the captures, relative to the project folder" },
       drive: { kind: "flag", help: "also write the 2 KiB drive RAM as drive8.prg, for knowledge space drive8" },
       "min-block": { kind: "count", value: "<bytes>", min: 1, max: 0x10000, default: 64, help: "write a changed block of at least this many bytes (default 64)" },
     },

package/src/host/vice/adapter.ts

@@ -59,7 +59,10 @@ export function conditionExpression(condition: Condition, space: Space): string
     }
     case "memory": {
       if (condition.space !== "c64") {
-        throw new WireFailure("unsupported-in-space", "A memory condition can test only space c64 memory.");
+        throw new WireFailure(
+          "unsupported-in-space",
+          "A memory target or a memory condition tests only space c64 memory. For the drive, use a watchpoint in space drive8, or a c64_run_until address target in space drive8.",
+        );
       }
       return `(@${condition.view}:${formatC64Address(condition.address)} ${operator(condition.operator)} ${formatC64Byte(condition.value)})`;
     }