@henols/c64-re-tools 2.0.0-rc.2 → 2.0.0-rc.4

package/skills/c64-unpacker/scripts/evidence.ts

@@ -1,7 +1,7 @@
 // Local evidence of packing for the c64-unpacker script: where BASIC hands
 // over and how compressed the bytes look. It never names a packer.
 
-import { formatC64Address } from "#src/c64.ts";
+import { basicLineText, formatC64Address, instructionLength, parseTokenized } from "#src/c64.ts";
 
 export class PackingError extends Error {}
 
@@ -42,10 +42,72 @@ export interface PackingEvidence {
   /** The first BASIC line and its constant SYS address, or null when the program has none. */
   basicStart: { line: number; sys: string } | null;
   entropy: { blocks: number; highBlocks: number; median: number | null };
+  /** The BASIC lines as text: a packer or a crack often puts its banner there (D43). */
+  basicText: string[];
+  /** Runs of readable PETSCII near the start of the code, raw. */
+  strings: string[];
+  /** What the code at the SYS address does in its first bytes. */
+  start: { at: string; jumpsOutside?: string; writesOutside?: string } | null;
+  /** EOR loops that store with an index near the start: decryption or decoding. */
+  decodeLoops: string[];
   packing: "likely" | "unlikely" | "unclear";
   evidence: string;
 }
 
+/** How far the start-code signals look from the SYS address. */
+const START_BYTES = 256;
+const MIN_STRING = 8;
+const MAX_STRINGS = 10;
+const EOR_OPCODES = new Set([0x41, 0x45, 0x49, 0x4d, 0x51, 0x55, 0x59, 0x5d]);
+const INDEXED_STORES = new Set([0x81, 0x91, 0x95, 0x99, 0x9d]);
+const BRANCHES = new Set([0x10, 0x30, 0x50, 0x70, 0x90, 0xb0, 0xd0, 0xf0]);
+
+/** Readable PETSCII runs of at least MIN_STRING characters, the first MAX_STRINGS of them. */
+function readableStrings(bytes: Uint8Array): string[] {
+  const found: string[] = [];
+  let run = "";
+  for (const byte of [...bytes, 0]) {
+    if (byte >= 0x20 && byte <= 0x5d) run += String.fromCharCode(byte);
+    else {
+      if (run.trim().length >= MIN_STRING && found.length < MAX_STRINGS) found.push(run.trim().slice(0, 40));
+      run = "";
+    }
+  }
+  return found;
+}
+
+/**
+ * The start-code signals: the first JMP to an address outside the loaded
+ * range, the first indexed store outside it, and EOR loops that store with
+ * an index and branch back.
+ */
+function startSignals(program: Uint8Array, load: number, end: number, sys: number): { start: PackingEvidence["start"]; decodeLoops: string[] } {
+  if (sys < load || sys > end) return { start: null, decodeLoops: [] };
+  const byteAt = (address: number) => (address >= load && address <= end ? program[address - load + 2]! : undefined);
+  const outside = (address: number) => address < load || address > end;
+  const instructions: Array<{ at: number; opcode: number; operand: number; length: number }> = [];
+  for (let at = sys; at <= Math.min(end, sys + START_BYTES - 1); ) {
+    const opcode = byteAt(at)!;
+    const length = instructionLength(opcode);
+    const operand = length === 3 ? (byteAt(at + 1) ?? 0) | ((byteAt(at + 2) ?? 0) << 8) : length === 2 ? (byteAt(at + 1) ?? 0) : 0;
+    instructions.push({ at, opcode, operand, length });
+    at += length;
+  }
+  const jump = instructions.find((instruction) => instruction.opcode === 0x4c && outside(instruction.operand));
+  const store = instructions.find((instruction) => (instruction.opcode === 0x9d || instruction.opcode === 0x99) && outside(instruction.operand));
+  const decodeLoops = instructions
+    .filter((instruction, index) => {
+      if (!EOR_OPCODES.has(instruction.opcode)) return false;
+      const after = instructions.slice(index + 1, index + 9);
+      return after.some((next) => INDEXED_STORES.has(next.opcode)) && after.some((next) => BRANCHES.has(next.opcode) && next.operand >= 0x80);
+    })
+    .map((instruction) => formatC64Address(instruction.at));
+  return {
+    start: { at: formatC64Address(sys), ...(jump === undefined ? {} : { jumpsOutside: formatC64Address(jump.operand) }), ...(store === undefined ? {} : { writesOutside: formatC64Address(store.operand) }) },
+    decodeLoops,
+  };
+}
+
 export function inspect(program: Uint8Array): PackingEvidence {
   if (program.length < 3) throw new PackingError("the file is too short for a PRG");
   const load = program[0]! | (program[1]! << 8);
@@ -58,7 +120,13 @@ export function inspect(program: Uint8Array): PackingEvidence {
   for (let at = 0; at + BLOCK <= body.length; at += BLOCK) blocks.push(entropy(body.subarray(at, at + BLOCK)));
   const high = blocks.filter((value) => value > HIGH_ENTROPY).length;
   const share = blocks.length === 0 ? 0 : high / blocks.length;
-  const packing = blocks.length < 2 ? "unclear" : share >= 0.75 ? "likely" : share <= 0.1 ? "unlikely" : "unclear";
+  const byEntropy = blocks.length < 2 ? "unclear" : share >= 0.75 ? "likely" : share <= 0.1 ? "unlikely" : "unclear";
+  const tokenized = parseTokenized(program);
+  const basicText = "lines" in tokenized && load === 0x0801 ? tokenized.lines.map(basicLineText) : [];
+  const signals = stub === undefined ? { start: null, decodeLoops: [] } : startSignals(program, load, end, stub.address);
+  const startMoves = signals.start?.jumpsOutside !== undefined || signals.start?.writesOutside !== undefined || signals.decodeLoops.length > 0;
+  // Code that decodes or moves itself is a reason to run it, also when the bytes look plain.
+  const packing = byEntropy === "unlikely" && startMoves ? "unclear" : byEntropy;
   const sorted = [...blocks].sort((a, b) => a - b);
   return {
     loadRange: { start: formatC64Address(load), end: formatC64Address(Math.min(end, 0xffff)) },
@@ -69,13 +137,19 @@ export function inspect(program: Uint8Array): PackingEvidence {
       highBlocks: high,
       median: sorted.length === 0 ? null : Number(sorted[sorted.length >> 1]!.toFixed(2)),
     },
+    basicText,
+    strings: readableStrings(body.subarray(0, 1024)),
+    start: signals.start,
+    decodeLoops: signals.decodeLoops,
     packing,
     evidence:
       packing === "likely"
         ? `${high} of ${blocks.length} blocks of 256 bytes look compressed (above ${HIGH_ENTROPY} bits per byte); code and data are usually below.`
         : packing === "unlikely"
           ? `Only ${high} of ${blocks.length} blocks look compressed; the bytes look like code and data. A simple cruncher can look the same: run trace to see if the program writes code and then executes it.`
-          : "The entropy does not show clearly if the program is packed. Run it and look at what it writes to memory.",
+          : startMoves && byEntropy === "unlikely"
+            ? "The bytes look like code and data, but the code at the SYS address decodes or moves bytes outside the program. Run trace or stages to see what it writes and runs."
+            : "The entropy does not show clearly if the program is packed. Run it and look at what it writes to memory.",
   };
 }
 
@@ -88,27 +162,18 @@ export interface AccessRange {
   execute: boolean;
 }
 
-/** The addresses that one memory map read covers before it is split. */
-const MEMMAP_WINDOW = 0x1000;
-
 /**
- * Reads the memory map of $0000-$ffff in windows, in address order. `read`
- * gives at most `maxRanges` ranges for one window; a window that gives that
- * many can have more, so it is read again as two halves.
+ * Reads the memory map of $0000-$ffff in address order. One `read` gives a
+ * limited number of ranges, and `next` when more follow: the next read starts there.
  */
-export async function readMemoryMap(read: (start: number, end: number) => Promise<AccessRange[]>, maxRanges: number): Promise<AccessRange[]> {
+export async function readMemoryMap(read: (start: number) => Promise<{ ranges: AccessRange[]; next?: number }>): Promise<AccessRange[]> {
   const ranges: AccessRange[] = [];
-  const window = async (start: number, end: number): Promise<void> => {
-    const found = await read(start, end);
-    if (found.length < maxRanges || start === end) {
-      ranges.push(...found);
-      return;
-    }
-    const middle = Math.floor((start + end) / 2);
-    await window(start, middle);
-    await window(middle + 1, end);
-  };
-  for (let start = 0; start <= 0xffff; start += MEMMAP_WINDOW) await window(start, start + MEMMAP_WINDOW - 1);
+  for (let start: number | undefined = 0; start !== undefined; ) {
+    const found: { ranges: AccessRange[]; next?: number } = await read(start);
+    ranges.push(...found.ranges);
+    if (found.next !== undefined && found.next <= start) throw new Error(`the memory map did not move on from ${start}`);
+    start = found.next;
+  }
   return ranges;
 }
 

package/skills/c64-unpacker/scripts/loads.ts

@@ -0,0 +1,174 @@
+// The loads command of c64-unpacker (D53): one capture for each part of a
+// multi-load program. The steps file has action steps and capture steps; a
+// capture writes what the C64 side changed since the previous capture as an
+// image with its entry state, and with --drive the drive RAM as a PRG for
+// knowledge space drive8. No tool guesses where a load ends: the steps decide.
+
+import { COVERAGE, formatC64Address, parseC64Address, textToPetscii } from "#src/c64.ts";
+import { autostartLoaded, LOAD_LIMIT_FRAMES } from "#src/host-client/autostart.ts";
+import { readCoverage } from "#src/host-client/coverage.ts";
+import { ACTION_STEP_KEYS, ACTION_STEP_KINDS, checkActionStep, isObject, runActionStep, type ActionStep } from "#src/host-client/steps.ts";
+import { WireFailure } from "#src/host-client/tools.ts";
+import { readProjectFile, writeProjectFile } from "#src/host-client/transfer.ts";
+import { ViceSessionClient } from "#src/host-client/vice-session.ts";
+import { UsageError } from "#src/script.ts";
+import { entryState, json, machineState, readRam, vectorsOf } from "./machine.ts";
+
+export type LoadStep = ActionStep | { capture: string };
+
+/** A capture name: letters, digits, dots, underscores or hyphens. */
+const NAME = /^[A-Za-z0-9._-]{1,64}$/;
+/** The drive RAM of a 1541. */
+const DRIVE_RAM = 0x0800;
+/** Pages 0-3 belong to the entry state, not to the segments. */
+const FIRST_SEGMENT_ADDRESS = 0x0400;
+
+/** Reads and checks the steps file of loads: action steps with $ addresses, and capture steps with new names. */
+export function readLoadSteps(path: string): LoadStep[] {
+  let value: unknown;
+  try {
+    value = JSON.parse(readProjectFile(path).bytes.toString("utf8"));
+  } catch (error) {
+    if (error instanceof WireFailure) throw error;
+    throw new UsageError(`--steps ${path} is not JSON: ${(error as Error).message}`);
+  }
+  const fail = (message: string): never => {
+    throw new UsageError(`--steps ${path}: ${message}`);
+  };
+  if (!Array.isArray(value) || value.length === 0) return fail("the steps are a JSON list with at least one step");
+  const names = new Set<string>();
+  value.forEach((step: unknown, index) => {
+    const at = `step ${index + 1}`;
+    if (!isObject(step)) return fail(`${at} must be an object`);
+    if ("capture" in step) {
+      if (Object.keys(step).length !== 1) fail(`${at}: a capture step has only capture`);
+      if (typeof step.capture !== "string" || !NAME.test(step.capture)) fail(`${at}: capture needs a name of 1 to 64 letters, digits, dots, underscores or hyphens`);
+      if (names.has(step.capture as string)) fail(`${at}: the capture name ${step.capture as string} is used before`);
+      names.add(step.capture as string);
+      return;
+    }
+    const kinds = Object.keys(step).filter((key) => (ACTION_STEP_KINDS as readonly string[]).includes(key));
+    if (kinds.length !== 1) return fail(`${at} must have exactly one of ${[...ACTION_STEP_KINDS, "capture"].join(", ")}`);
+    const kind = kinds[0]!;
+    const allowed = ACTION_STEP_KEYS[kind] ?? [kind];
+    for (const key of Object.keys(step)) if (!allowed.includes(key)) fail(`${at} has the key ${key}, which a ${kind} step does not have`);
+    checkActionStep(kind, step, at, fail, false);
+  });
+  if (names.size === 0) fail("the steps have no capture step");
+  return value as LoadStep[];
+}
+
+/** Ranges from $0400 where the RAM changed and the CPU wrote since the last capture, joined across gaps below `join` bytes. */
+export function changedBlocks(before: Uint8Array, after: Uint8Array, map: Uint8Array, join: number): Array<{ start: number; end: number; bytes: number }> {
+  const blocks: Array<{ start: number; end: number; bytes: number }> = [];
+  for (let address = FIRST_SEGMENT_ADDRESS; address < 0x10000; address++) {
+    if (before[address] === after[address] || (map[address]! & COVERAGE.ramWrite) === 0) continue;
+    const last = blocks.at(-1);
+    if (last !== undefined && address - last.end <= join) {
+      last.end = address;
+      last.bytes++;
+    } else blocks.push({ start: address, end: address, bytes: 1 });
+  }
+  return blocks;
+}
+
+const hex4 = (value: number) => value.toString(16).padStart(4, "0");
+
+/**
+ * Runs the program and its steps. Each capture step writes <out>/<name>/:
+ * image.bin, image.json, entry_state.json, a PRG for each segment, and with
+ * `drive` the drive RAM as drive8.prg. Blocks below `minBlock` bytes are
+ * counted, not written: a game changes its variables all the time.
+ */
+export async function loads(program: string, options: { out: string; steps: LoadStep[]; drive: boolean; minBlock: number }): Promise<{ output: Record<string, unknown>; failed: boolean }> {
+  const client = await ViceSessionClient.open({ videoStandard: "pal" });
+  try {
+    await client.reset({ mode: "hard", run: false });
+    let previous: Buffer;
+    /** A PRG that autostart put into memory without the CPU: its bytes count as written in the first capture. */
+    let loadedProgram: Buffer | undefined;
+    if (program.toLowerCase().endsWith(".prg")) {
+      const { bytes } = readProjectFile(program);
+      if (!(await autostartLoaded(client, program, bytes))) return { output: { loaded: false, reason: `The program was not in memory after ${LOAD_LIMIT_FRAMES} frames.` }, failed: true };
+      // The RAM before RUN, without the program, so the first capture holds the program too.
+      previous = await readRam(client, 0x0000, 0x10000);
+      const load = bytes[0]! | (bytes[1]! << 8);
+      previous.fill(0, load, Math.min(0x10000, load + bytes.length - 2));
+      loadedProgram = bytes;
+      await client.memmap({ action: "clear" });
+      await client.keyboard(textToPetscii("RUN\n"));
+    } else {
+      // A disk, tape or cartridge: VICE loads and starts it. The RAM test of the KERNAL puts back what it writes.
+      previous = await readRam(client, 0x0000, 0x10000);
+      await client.memmap({ action: "clear" });
+      await client.autostart({ path: program, index: 0, run: true });
+    }
+
+    const captures: Array<Record<string, unknown>> = [];
+    for (const [index, step] of options.steps.entries()) {
+      if (!("capture" in step)) {
+        const notReached = await runActionStep(client, step, parseC64Address);
+        if (notReached !== undefined && "runUntil" in step) {
+          const reason = `Step ${index + 1} did not reach its target in ${step.timeoutFrames} frames. It stopped by ${notReached.stopReason} at ${formatC64Address(notReached.pc)}.`;
+          return { output: { loaded: false, reason, captures }, failed: true };
+        }
+        continue;
+      }
+      if ((await client.status()).state === "running") await client.execution({ action: "pause", space: "c64" });
+      const registers = await client.registersGet("c64");
+      const memory = await readRam(client, 0x0000, 0x10000);
+      const map = await readCoverage(client);
+      if (loadedProgram !== undefined) {
+        // The PRG itself is part of the first capture.
+        const load = loadedProgram[0]! | (loadedProgram[1]! << 8);
+        for (let address = load; address < Math.min(0x10000, load + loadedProgram.length - 2); address++) map[address]! |= COVERAGE.ramWrite;
+        loadedProgram = undefined;
+      }
+      const blocks = changedBlocks(previous, memory, map, 16);
+      const segments = blocks.filter((block) => block.end - block.start + 1 >= options.minBlock);
+      const small = blocks.filter((block) => !segments.includes(block)).reduce((sum, block) => sum + block.bytes, 0);
+      const state = await machineState(client);
+      const vectors = vectorsOf(state, memory);
+      const directory = `${options.out}/${step.capture}`;
+      const write = (name: string, bytes: Uint8Array) => writeProjectFile(`${directory}/${name}`, bytes, { what: "--out" });
+      write("image.bin", memory);
+      const files = segments.map((segment) => {
+        const name = `segment-${hex4(segment.start)}-${hex4(segment.end)}.prg`;
+        write(name, Buffer.concat([Buffer.from([segment.start & 0xff, segment.start >> 8]), memory.subarray(segment.start, segment.end + 1)]));
+        return name;
+      });
+      let drive: Record<string, unknown> | undefined;
+      if (options.drive) {
+        const ram = Buffer.from((await client.memoryRead({ address: 0x0000, size: DRIVE_RAM, space: "drive8", view: "cpu" })).data, "hex");
+        write("drive8.prg", Buffer.concat([Buffer.from([0x00, 0x00]), ram]));
+        drive = { file: "drive8.prg", pc: formatC64Address((await client.registersGet("drive8")).pc), space: "drive8" };
+      }
+      const showRange = (range: { start: number; end: number }) => ({ start: formatC64Address(range.start), end: formatC64Address(range.end) });
+      write(
+        "image.json",
+        json({
+          capture: step.capture,
+          entry: formatC64Address(registers.pc),
+          segments: segments.map((segment, at) => ({ ...showRange(segment), file: files[at] })),
+          vectors,
+          image: "image.bin",
+          ...(drive === undefined ? {} : { drive }),
+        }),
+      );
+      write("entry_state.json", json(entryState(registers, state, vectors)));
+      captures.push({
+        capture: step.capture,
+        out: directory,
+        pc: formatC64Address(registers.pc),
+        segments: segments.map(showRange),
+        ...(small === 0 ? {} : { smallChanges: small }),
+        ...(drive === undefined ? {} : { drive: drive.file }),
+      });
+      previous = memory;
+      await client.memmap({ action: "clear" });
+    }
+    return { output: { loaded: true, captures }, failed: false };
+  } finally {
+    await client.close();
+  }
+}

package/skills/c64-unpacker/scripts/machine.ts

@@ -0,0 +1,68 @@
+// The machine state that a capture writes (D43): RAM, the CPU port, pages
+// 0-3, the chip registers read without side effects, and the vectors. The
+// capture and loads commands of the unpacker share it.
+
+import { formatC64Address } from "#src/c64.ts";
+import type { ViceSessionClient } from "#src/host-client/vice-session.ts";
+
+type Registers = Awaited<ReturnType<ViceSessionClient["registersGet"]>>;
+
+const MAX_READ = 4096;
+
+/** The I/O areas of the entry state, read without side effects (D43). */
+export const IO_AREAS = { vic: [0xd000, 0x2f], sid: [0xd400, 0x1d], cia1: [0xdc00, 0x10], cia2: [0xdd00, 0x10] } as const;
+/** Frames from a hard reset to the READY prompt, with margin. */
+export const READY_FRAMES = 200;
+
+export interface MachineState {
+  port: Buffer;
+  pages: Buffer;
+  io: Record<keyof typeof IO_AREAS, Buffer>;
+  romVectors: Buffer;
+}
+
+/** Reads `size` bytes from `address` in the RAM view, in parts. */
+export async function readRam(client: ViceSessionClient, address: number, size: number): Promise<Buffer> {
+  const parts: Buffer[] = [];
+  for (let at = address; at < address + size; at += MAX_READ) {
+    parts.push(Buffer.from((await client.memoryRead({ address: at, size: Math.min(MAX_READ, address + size - at), space: "c64", view: "ram" })).data, "hex"));
+  }
+  return Buffer.concat(parts);
+}
+
+/** The state the program finds at its entry: what a rebuild's loader must recreate. */
+export async function machineState(client: ViceSessionClient): Promise<MachineState> {
+  const port = Buffer.from((await client.memoryRead({ address: 0x0000, size: 2, space: "c64", view: "cpu" })).data, "hex");
+  const pages = await readRam(client, 0x0000, 0x400);
+  const io = {} as Record<keyof typeof IO_AREAS, Buffer>;
+  for (const [name, [address, size]] of Object.entries(IO_AREAS)) io[name as keyof typeof IO_AREAS] = Buffer.from((await client.ioRead(address, size)).data, "hex");
+  const romVectors = Buffer.from((await client.memoryRead({ address: 0xfffa, size: 6, space: "c64", view: "cpu" })).data, "hex");
+  return { port, pages, io, romVectors };
+}
+
+const word = (bytes: Buffer, offset: number) => formatC64Address(bytes[offset]! | (bytes[offset + 1]! << 8));
+
+/** The KERNAL vectors in page 3, the hardware vectors in RAM, and the ROM's own. */
+export function vectorsOf(state: MachineState, memory: Buffer) {
+  return {
+    irq: word(state.pages, 0x314),
+    brk: word(state.pages, 0x316),
+    nmi: word(state.pages, 0x318),
+    ram: { nmi: word(memory, 0xfffa), reset: word(memory, 0xfffc), irq: word(memory, 0xfffe) },
+    rom: { nmi: word(state.romVectors, 0), reset: word(state.romVectors, 2), irq: word(state.romVectors, 4) },
+  };
+}
+
+/** The content of entry_state.json; `differsFromReset` only when the caller compared with a clean reset. */
+export function entryState(registers: Registers, state: MachineState, vectors: ReturnType<typeof vectorsOf>, differsFromReset?: Array<{ address: string; entry: number; reset: number }>) {
+  return {
+    registers: { pc: formatC64Address(registers.pc), a: registers.a, x: registers.x, y: registers.y, sp: registers.sp, flags: registers.flags },
+    cpuPort: { $00: state.port[0], $01: state.port[1] },
+    pages: Object.fromEntries([0, 1, 2, 3].map((page) => [formatC64Address(page << 8), state.pages.subarray(page << 8, (page + 1) << 8).toString("hex")])),
+    io: Object.fromEntries(Object.entries(IO_AREAS).map(([name, [address]]) => [name, { from: formatC64Address(address), bytes: state.io[name as keyof typeof IO_AREAS].toString("hex") }])),
+    vectors,
+    ...(differsFromReset === undefined ? {} : { differsFromReset }),
+  };
+}
+
+export const json = (value: unknown) => Buffer.from(`${JSON.stringify(value, null, 2)}\n`);