@henols/c64-re-tools 1.1.0 → 2.0.0-rc.3

package/skills/c64-unpacker/SKILL.md

@@ -0,0 +1,82 @@
+---
+name: c64-unpacker
+description: Use this skill to find out if a packer or a cruncher compressed a C64 program, and to get the unpacked program for analysis. It shows evidence of packing without a guess of the packer name. It runs the program in an emulator until the unpacked code starts, then writes the memory into the project. Do not use it to analyze the unpacked code.
+---
+
+# c64-unpacker
+
+## Purpose
+
+Find out if a packer compressed a C64 program. Get the unpacked program for analysis.
+
+This skill does not change project knowledge.
+
+## Inputs
+
+- The program: a PRG file, relative to the project directory.
+- For a capture: the address where the unpacked program starts, and where to write the result.
+
+## Tools / execution path
+
+```
+node <skill>/scripts/unpack.ts inspect original/game.prg
+node <skill>/scripts/unpack.ts trace original/game.prg
+node <skill>/scripts/unpack.ts capture original/game.prg --until '$c000' --out unpacked/game.prg --range '$c000' '$cfff'
+```
+
+`inspect` examines the file locally. `trace` and `capture` start their own emulator through the host runtime and start the program. `trace` runs the program and finds the memory that the program wrote and then executed. `capture` stops the program when the CPU gets to the `--until` address.
+
+| Option | Meaning |
+|---|---|
+| `--until '$c000'` | The address where the unpacked program starts. |
+| `--out unpacked/game.prg` | Where to write the result, relative to the project directory. |
+| `--range '$c000' '$cfff'` | Write this range as a PRG. Without it, the script writes all 64 KiB of RAM as a flat file. |
+| `--timeout-frames 3000` | How many frames the program can run before the capture stops (default 3000). |
+| `--frames 3000` | `trace` only: how many frames the program runs (default 3000). |
+| `--entry '$c000'` | `trace` only: start the program at this address. Without it, `trace` types RUN. A program without a BASIC start line needs it. |
+
+The capture reads RAM, also the RAM under the ROMs and the I/O area. The script prints one JSON object.
+
+## Workflow
+
+1. Run `inspect`. Read `basicStart` and `packing`:
+   - `likely`: most of the bytes look compressed. Packed data has this property.
+   - `unlikely`: the bytes look like code and data. A simple cruncher can also give this result.
+   - `unclear`: the file is short, or the blocks do not give clear evidence.
+2. Run `trace`. It gives the evidence from the run. Read `packing`:
+   - `likely`: the program wrote code and then executed it. It unpacked, decrypted or moved code. `runningRange` is the code that runs now. Its `start` is usually the entry point of the unpacked program.
+   - `unclear`: the program executed a small quantity of code that it wrote. This can be code that changes itself.
+   - `unlikely`: the program executed no code that it wrote. If the program waits for a key or a joystick, it can move code after that input. Then use the c64-emulator skill.
+3. Find the address where the unpacked program starts:
+   - Use `runningRange.start` from `trace`, or `suggestedUntil`. The CPU gets to `suggestedUntil` after the unpacking.
+   - Or use the c64-emulator skill: set a breakpoint after the unpack loop, or look at the last JMP of the unpacker.
+4. Run `capture` with that address. Use `--range` for the area that holds the unpacked program, if you know it.
+5. Make sure that the result is the application: the code at the start address must not be the unpacker again.
+6. Give the result to the c64-static-analysis skill. For a flat 64 KiB file, use its `--flat64k` option.
+
+## Knowledge
+
+Automatic write:
+- none.
+
+Do not record the unpacker in the project knowledge, unless the user asks to examine the unpacker.
+
+## Result
+
+The packing evidence, or the unpacked program in the project with the address where it starts.
+
+The exit status is 0 for a result. The exit status is 1 for `traced: false`, for `captured: false` and for an error. The exit status is 2 for a wrong argument.
+
+## Failure and conflicts
+
+- `traced: false`: the emulator did not load the program into memory in the time limit. The reason gives the load address. Make sure that the file is a PRG that the C64 can load.
+- `captured: false`: the program did not get to the address in the time limit. The reason tells where it stopped. Find a better address with the c64-emulator skill, or give more frames.
+- Do not name a packer from weak evidence. The script never names one.
+- Some programs unpack in more than one stage. Make sure that the captured code is the application and not a second unpacker.
+- `not-found` or `invalid-input`: a path or an argument is wrong. Read the message.
+- `operation-failed` with a message about the host runtime: the host runtime does not run. Tell the user to start the host runtime with `npx -y --package=@henols/c64-re-tools@latest c64-re-tools-host`.
+
+## Handoffs
+
+- Unpacked machine code → c64-static-analysis
+- Find the start address at run time → c64-emulator

package/skills/c64-unpacker/scripts/evidence.ts

@@ -0,0 +1,135 @@
+// Local evidence of packing for the c64-unpacker script: where BASIC hands
+// over and how compressed the bytes look. It never names a packer.
+
+import { formatC64Address } from "#src/c64.ts";
+
+export class PackingError extends Error {}
+
+/** A 256-byte block above this many bits per byte looks compressed. */
+const HIGH_ENTROPY = 6.9;
+const BLOCK = 256;
+
+function entropy(bytes: Uint8Array): number {
+  const counts = new Array<number>(256).fill(0);
+  for (const byte of bytes) counts[byte]!++;
+  let bits = 0;
+  for (const count of counts) {
+    if (count === 0) continue;
+    const p = count / bytes.length;
+    bits -= p * Math.log2(p);
+  }
+  return bits;
+}
+
+/** The constant SYS address of the first BASIC line, when the program starts with one. */
+function basicSys(program: Uint8Array): { line: number; address: number; end: number } | undefined {
+  if (program.length < 8 || program[0] !== 0x01 || program[1] !== 0x08) return undefined;
+  const number = program[4]! | (program[5]! << 8);
+  const end = program.indexOf(0, 6);
+  if (end < 0) return undefined;
+  const sys = program.subarray(6, end).indexOf(0x9e);
+  if (sys < 0) return undefined;
+  const digits = Buffer.from(program.subarray(6 + sys + 1, end)).toString("latin1").replace(/[ (]/g, "");
+  const match = /^(\d{1,5})/.exec(digits);
+  if (match === null || Number(match[1]) > 0xffff) return undefined;
+  return { line: number, address: Number(match[1]), end: 0x0801 + end - 2 };
+}
+
+/** The local packing evidence of one PRG. */
+export interface PackingEvidence {
+  loadRange: { start: string; end: string };
+  bytes: number;
+  /** The first BASIC line and its constant SYS address, or null when the program has none. */
+  basicStart: { line: number; sys: string } | null;
+  entropy: { blocks: number; highBlocks: number; median: number | null };
+  packing: "likely" | "unlikely" | "unclear";
+  evidence: string;
+}
+
+export function inspect(program: Uint8Array): PackingEvidence {
+  if (program.length < 3) throw new PackingError("the file is too short for a PRG");
+  const load = program[0]! | (program[1]! << 8);
+  const end = load + program.length - 3;
+  const stub = basicSys(program);
+  // Without a BASIC start, all bytes count; with one, only the bytes after the BASIC line.
+  const from = stub === undefined ? 2 : Math.min(program.length, stub.end - load + 2 + 3);
+  const body = program.subarray(from);
+  const blocks: number[] = [];
+  for (let at = 0; at + BLOCK <= body.length; at += BLOCK) blocks.push(entropy(body.subarray(at, at + BLOCK)));
+  const high = blocks.filter((value) => value > HIGH_ENTROPY).length;
+  const share = blocks.length === 0 ? 0 : high / blocks.length;
+  const packing = blocks.length < 2 ? "unclear" : share >= 0.75 ? "likely" : share <= 0.1 ? "unlikely" : "unclear";
+  const sorted = [...blocks].sort((a, b) => a - b);
+  return {
+    loadRange: { start: formatC64Address(load), end: formatC64Address(Math.min(end, 0xffff)) },
+    bytes: program.length - 2,
+    basicStart: stub === undefined ? null : { line: stub.line, sys: formatC64Address(stub.address) },
+    entropy: {
+      blocks: blocks.length,
+      highBlocks: high,
+      median: sorted.length === 0 ? null : Number(sorted[sorted.length >> 1]!.toFixed(2)),
+    },
+    packing,
+    evidence:
+      packing === "likely"
+        ? `${high} of ${blocks.length} blocks of 256 bytes look compressed (above ${HIGH_ENTROPY} bits per byte); code and data are usually below.`
+        : packing === "unlikely"
+          ? `Only ${high} of ${blocks.length} blocks look compressed; the bytes look like code and data. A simple cruncher can look the same: run trace to see if the program writes code and then executes it.`
+          : "The entropy does not show clearly if the program is packed. Run it and look at what it writes to memory.",
+  };
+}
+
+/** One range of the emulator's memory map: what the program did with these addresses. */
+export interface AccessRange {
+  start: number;
+  end: number;
+  read: boolean;
+  write: boolean;
+  execute: boolean;
+}
+
+/** The addresses that one memory map read covers before it is split. */
+const MEMMAP_WINDOW = 0x1000;
+
+/**
+ * Reads the memory map of $0000-$ffff in windows, in address order. `read`
+ * gives at most `maxRanges` ranges for one window; a window that gives that
+ * many can have more, so it is read again as two halves.
+ */
+export async function readMemoryMap(read: (start: number, end: number) => Promise<AccessRange[]>, maxRanges: number): Promise<AccessRange[]> {
+  const ranges: AccessRange[] = [];
+  const window = async (start: number, end: number): Promise<void> => {
+    const found = await read(start, end);
+    if (found.length < maxRanges || start === end) {
+      ranges.push(...found);
+      return;
+    }
+    const middle = Math.floor((start + end) / 2);
+    await window(start, middle);
+    await window(middle + 1, end);
+  };
+  for (let start = 0; start <= 0xffff; start += MEMMAP_WINDOW) await window(start, start + MEMMAP_WINDOW - 1);
+  return ranges;
+}
+
+/** ROM areas: an address here can be written (RAM below) and executed (ROM) without one being the other. */
+export const underRom = (address: number) => (address >= 0xa000 && address <= 0xbfff) || address >= 0xe000;
+
+/**
+ * The memory that the program wrote and then executed, in pieces of code.
+ * `bytes` and `running` count only RAM outside the ROM areas: a write under
+ * a ROM and code that runs in that ROM are not one piece of code.
+ */
+export function writtenThenExecuted(ranges: AccessRange[], pc: number): { written: Array<{ start: number; end: number }>; bytes: number; running?: { start: number; end: number } } {
+  // VICE marks only opcode bytes as executed; a gap of at most two operand bytes stays inside one piece of code.
+  const written: Array<{ start: number; end: number }> = [];
+  for (const range of ranges.filter((candidate) => candidate.write && candidate.execute)) {
+    const last = written.at(-1);
+    if (last !== undefined && range.start - last.end <= 3 && underRom(range.start) === underRom(last.start)) last.end = range.end;
+    else written.push({ start: range.start, end: range.end });
+  }
+  const counted = written.filter((range) => !underRom(range.start));
+  const bytes = counted.reduce((sum, range) => sum + range.end - range.start + 1, 0);
+  const running = counted.find((range) => pc >= range.start && pc <= range.end);
+  return { written, bytes, ...(running === undefined ? {} : { running }) };
+}

package/skills/c64-unpacker/scripts/unpack.ts

@@ -0,0 +1,216 @@
+// The c64-unpacker script. "inspect" gives local evidence of packing; it never
+// names a packer. "trace" runs the program and reports the memory that it
+// wrote and then executed: the run-time evidence that also finds a simple
+// cruncher whose bytes look like code. "capture" runs the program in an
+// emulator until it reaches a given address (the unpacked entry point) and
+// writes the memory into the project. No native unpack tool is used.
+
+import { mkdirSync, writeFileSync } from "node:fs";
+import { dirname } from "node:path";
+import { parseArgs } from "node:util";
+
+import { formatC64Address, parseC64Address, textToPetscii } from "#src/c64.ts";
+import { WireFailure } from "#src/host-client/tools.ts";
+import { readProjectFile } from "#src/host-client/transfer.ts";
+import { ViceSessionClient } from "#src/host-client/vice-session.ts";
+import { resolveProjectPath } from "#src/project.ts";
+import { runScript, UsageError } from "#src/script.ts";
+import { inspect, PackingError, readMemoryMap, underRom, writtenThenExecuted } from "./evidence.ts";
+
+const USAGE = `unpack.ts inspect <program.prg>
+unpack.ts trace <program.prg> [--frames <n>] [--entry <address>]
+unpack.ts capture <program.prg> --until <address> --out <file> [--range <start> <end>] [--timeout-frames <n>]
+
+inspect   packing evidence: the BASIC start and the entropy of the bytes
+trace     run the program for <n> frames (default 3000) and report the memory
+          that it wrote and then executed; --entry starts it there instead of RUN
+capture   run the program until the CPU gets to <address>, then write memory:
+          with --range, a PRG of that range; without it, all 64 KiB of RAM
+
+Addresses are $ and four hex digits. The result is one JSON object.`;
+
+const MAX_READ = 4096;
+
+function address(text: string, option: string): number {
+  try {
+    return parseC64Address(text);
+  } catch {
+    throw new UsageError(`${option} must be $ followed by four hex digits, not ${text}`);
+  }
+}
+
+/** Written-then-executed bytes from this many on are unpacking, decrypting or relocating, not self-changing code. */
+const UNPACKED_BYTES = 256;
+const LOAD_POLL_FRAMES = 25;
+const LOAD_LIMIT_FRAMES = 1500;
+const MAX_ADVANCE = 10_000;
+/** The most ranges one memory map read gives. */
+const MAX_RANGES = 1000;
+
+async function trace(program: string, options: { frames: number; entry?: number }): Promise<{ output: Record<string, unknown>; failed: boolean }> {
+  const { bytes } = readProjectFile(program);
+  if (bytes.length < 3) throw new UsageError("the file is too short for a PRG");
+  const load = bytes[0]! | (bytes[1]! << 8);
+  const basic = inspect(bytes).basicStart;
+  if (options.entry === undefined && basic === null) throw new UsageError("the program has no BASIC start line; give --entry <address>");
+  const client = await ViceSessionClient.open({ videoStandard: "pal" });
+  try {
+    // Loaded but not started, so the memory map can start empty: the KERNAL start-up writes all of RAM.
+    await client.autostart({ path: program, index: 0, run: false });
+    const head = bytes.subarray(2, Math.min(bytes.length, 2 + 64)).toString("hex");
+    let loaded = false;
+    for (let frames = 0; frames < LOAD_LIMIT_FRAMES && !loaded; frames += LOAD_POLL_FRAMES) {
+      await client.execution({ action: "advance-frames", count: LOAD_POLL_FRAMES, space: "c64" });
+      loaded = (await client.memoryRead({ address: load, size: head.length / 2, space: "c64", view: "ram" })).data === head;
+    }
+    if (!loaded) return { output: { traced: false, reason: `The program was not in memory at ${formatC64Address(load)} after ${LOAD_LIMIT_FRAMES} frames.` }, failed: true };
+    await client.memmap({ action: "clear" });
+    if (options.entry !== undefined) await client.registersSet("c64", { pc: options.entry });
+    else await client.keyboard(textToPetscii("RUN\n"));
+    let ran = 0;
+    while (ran < options.frames) {
+      const step = await client.execution({ action: "advance-frames", count: Math.min(MAX_ADVANCE, options.frames - ran), space: "c64" });
+      const advanced = step.advancedFrames ?? 0;
+      ran += advanced;
+      if (advanced === 0) break;
+    }
+    const { pc } = await client.registersGet("c64");
+    const ranges = await readMemoryMap(async (start, end) => {
+      const map = await client.memmap({ action: "read", start, end, maxRanges: MAX_RANGES });
+      if (!("ranges" in map)) throw new WireFailure("operation-failed", "The emulator did not give the memory map.");
+      return map.ranges;
+    }, MAX_RANGES);
+    const { written, bytes: count, running } = writtenThenExecuted(ranges, pc);
+    const packing = count >= UNPACKED_BYTES || running !== undefined ? "likely" : count > 0 ? "unclear" : "unlikely";
+    return {
+      output: {
+        traced: true,
+        frames: ran,
+        pc: formatC64Address(pc),
+        writtenThenExecuted: written.map((range) => ({
+          start: formatC64Address(range.start),
+          end: formatC64Address(range.end),
+          ...(underRom(range.start) ? { underRom: true } : {}),
+        })),
+        bytes: count,
+        packing,
+        evidence:
+          packing === "likely"
+            ? running !== undefined
+              ? `The CPU now runs code at ${formatC64Address(pc)} that the program wrote: it unpacked, decrypted or relocated it.`
+              : `The program wrote ${count} bytes and then executed them: it unpacked, decrypted or relocated code.`
+            : packing === "unclear"
+              ? `The program executed ${count} bytes that it wrote. This can be code that changes itself or a small loader.`
+              : "The program executed no code that it wrote while it ran.",
+        // A point after the unpacking where the unpacked code runs; the entry point can be at the start of its range.
+        ...(running === undefined ? {} : { suggestedUntil: formatC64Address(pc), runningRange: { start: formatC64Address(running.start), end: formatC64Address(running.end) } }),
+      },
+      failed: false,
+    };
+  } finally {
+    await client.close();
+  }
+}
+
+async function capture(program: string, options: { until: number; out: string; range?: [number, number]; timeoutFrames: number }): Promise<{ output: Record<string, unknown>; failed: boolean }> {
+  const client = await ViceSessionClient.open({ videoStandard: "pal" });
+  try {
+    await client.autostart({ path: program, index: 0, run: true });
+    const reached = await client.runUntil({ target: { kind: "address", address: options.until, space: "c64" }, timeoutFrames: options.timeoutFrames });
+    if (!reached.reached) {
+      return {
+        output: { captured: false, reason: `The program did not get to ${formatC64Address(options.until)} in ${options.timeoutFrames} frames. It stopped by ${reached.stopReason} at ${formatC64Address(reached.pc)}.` },
+        failed: true,
+      };
+    }
+    const [start, end] = options.range ?? [0x0000, 0xffff];
+    const memory = Buffer.alloc(end - start + 1);
+    for (let at = start; at <= end; at += MAX_READ) {
+      const size = Math.min(MAX_READ, end - at + 1);
+      // The RAM view: unpacked code is often under the ROMs or the I/O area.
+      const read = await client.memoryRead({ address: at, size, space: "c64", view: "ram" });
+      Buffer.from(read.data, "hex").copy(memory, at - start);
+    }
+    const bytes = options.range === undefined ? memory : Buffer.concat([Buffer.from([start & 0xff, start >> 8]), memory]);
+    const path = resolveProjectPath(options.out);
+    mkdirSync(dirname(path), { recursive: true });
+    writeFileSync(path, bytes);
+    const registers = await client.registersGet("c64");
+    return {
+      output: {
+        captured: true,
+        pc: formatC64Address(registers.pc),
+        output: options.out,
+        ...(options.range === undefined ? { flat64k: true } : { loadRange: { start: formatC64Address(start), end: formatC64Address(end) } }),
+        registers: { a: registers.a, x: registers.x, y: registers.y, sp: registers.sp },
+      },
+      failed: false,
+    };
+  } finally {
+    await client.close();
+  }
+}
+
+async function run(argv: string[]): Promise<{ output: unknown; failed: boolean }> {
+  const { values, positionals } = parseArgs({
+    args: argv,
+    strict: true,
+    allowPositionals: true,
+    options: {
+      until: { type: "string" },
+      out: { type: "string" },
+      range: { type: "string" },
+      "timeout-frames": { type: "string" },
+      frames: { type: "string" },
+      entry: { type: "string" },
+      help: { type: "boolean" },
+    },
+  });
+  if (values.help) throw new UsageError("");
+  const [command, program, ...rest] = positionals;
+  if (program === undefined) throw new UsageError("give a command and a program");
+  if (command === "inspect") {
+    if (rest.length > 0) throw new UsageError("inspect takes one program");
+    const { bytes } = readProjectFile(program);
+    try {
+      return { output: inspect(bytes), failed: false };
+    } catch (error) {
+      if (error instanceof PackingError) throw new UsageError(error.message);
+      throw error;
+    }
+  }
+  if (command === "trace") {
+    if (rest.length > 0) throw new UsageError("trace takes one program");
+    const frames = values.frames === undefined ? 3000 : Number(values.frames);
+    if (!Number.isInteger(frames) || frames < 1 || frames > 30_000) throw new UsageError("--frames must be from 1 to 30000");
+    return trace(program, { frames, ...(values.entry === undefined ? {} : { entry: address(values.entry, "--entry") }) });
+  }
+  if (command !== "capture") throw new UsageError(`unknown command ${command}`);
+  if (values.until === undefined || values.out === undefined) throw new UsageError("capture needs --until and --out");
+  // --range takes two values: the end is the next positional argument.
+  let range: [number, number] | undefined;
+  if (values.range !== undefined) {
+    if (rest.length !== 1) throw new UsageError("--range takes a start and an end address");
+    range = [address(values.range, "--range"), address(rest[0]!, "--range")];
+    if (range[1] < range[0]) throw new UsageError("the --range end must not be before its start");
+  } else if (rest.length > 0) {
+    throw new UsageError("too many arguments");
+  }
+  try {
+    resolveProjectPath(values.out);
+  } catch {
+    throw new UsageError(`--out must be a path relative to the project directory that stays inside it, not ${values.out}`);
+  }
+  const timeoutFrames = values["timeout-frames"] === undefined ? 3000 : Number(values["timeout-frames"]);
+  if (!Number.isInteger(timeoutFrames) || timeoutFrames < 1 || timeoutFrames > 30_000) throw new UsageError("--timeout-frames must be from 1 to 30000");
+  return capture(program, { until: address(values.until, "--until"), out: values.out, ...(range === undefined ? {} : { range }), timeoutFrames });
+}
+
+await runScript(
+  async () => {
+    const { output, failed } = await run(process.argv.slice(2));
+    if (failed) process.exitCode = 1;
+    return output;
+  },
+  { usage: USAGE },
+);