@henols/c64-re-tools 1.1.0 → 2.0.0-rc.11

package/skills/c64-memory-map/SKILL.md

@@ -0,0 +1,79 @@
+---
+name: c64-memory-map
+description: Use this skill to find what a C64 address means on the platform. Examples are a VIC-II, SID or CIA register, a KERNAL routine, a vector or a system variable. It also decodes the value of a hardware register, for example the memory configuration in $01 or the screen address in $D018. It tells you when an address has no platform meaning and belongs to the application.
+---
+
+# c64-memory-map
+
+## Purpose
+
+Give the C64 platform meaning of an address or of a register value.
+
+This skill does not read the emulator and does not change project knowledge.
+
+## Inputs
+
+- One or more addresses.
+- Or a register and a value to decode.
+
+## Tools / execution path
+
+The script looks up local reference data. It does not use the host runtime.
+
+```
+node <skill>/scripts/memmap.ts at '$d020' '$0314' '$ffd2'
+node <skill>/scripts/memmap.ts decode '$d018' '$15' --bank '$97'
+node <skill>/scripts/memmap.ts list vic
+node <skill>/scripts/memmap.ts export --out src/hw.asm
+```
+
+| Command | Result |
+|---|---|
+| `at <address> ...` | For each address: `kind` (`platform` or `application`), the area, the name and the meaning. |
+| `decode <register> <value>` | The meaning of the value. Use `--bank` with the value of `$DD00` to get the addresses for `$D018`. |
+| `list <vic\|sid\|cia1\|cia2\|kernal\|system>` | All entries of that group. |
+| `export --out <file>` | Writes all platform names to a project file as an ACME include: one `NAME = $xxxx` line for each name, with its meaning as a comment. |
+
+Write an address or a value as `$` and hex digits, as a decimal number, or as `%` and binary digits. Put a `$` value in single quotes in a shell.
+
+The script prints one JSON object.
+
+## Workflow
+
+1. Run `at` for the addresses in the code or the question.
+2. Read `kind`:
+   - `platform`: the C64 gives the address a meaning. Read `name` and `meaning`.
+   - `application`: the address has no platform meaning. The program decides its use. Find its use with the c64-static-analysis or c64-emulator skill.
+3. For an I/O register, `mirrorOf` shows the register that a mirror address repeats. `visible` tells that an address in `$D000`-`$DFFF` is visible only when `$01` shows I/O at `$D000`. This area holds the chip registers, the color RAM and the cartridge I/O areas.
+4. For a byte in a multi-byte entry, for example the high byte of a vector, `entryStart` gives the first address of the entry.
+5. To understand a value in a register, run `decode`. For the memory configuration, decode the value of `$01`.
+6. For source that you write by hand, run `export` and include the file with `!source`. The c64-source skill uses the same names without the file.
+
+The decoders are for `$01`, `$D010`, `$D011`, `$D015` to `$D01F` (the sprite and interrupt bits), `$D016`, `$D018`, `$DC0D`, `$DD00` and `$DD0D`.
+
+## Knowledge
+
+Automatic write:
+- none.
+
+Platform facts are reference data. Do not store them as project knowledge.
+
+A conclusion about the application can be project knowledge. For example: "the program writes its own IRQ handler address to `$0314`". Record it with the c64-knowledge skill.
+
+## Result
+
+The platform meaning of each address, or the decoded meaning of a register value. For `export`: the file, the count of names and the bytes.
+
+The exit status is 0 for a result. The exit status is 2 for a wrong argument.
+
+## Failure and conflicts
+
+- `kind: application`: this is not a failure. The address has no platform meaning.
+- `invalid-input`: the address, the register or the value is not correct, or no decoder exists for the register. Read the message.
+
+Some registers have a different meaning for a read and for a write, for example `$DC0D`. The decoded result shows both.
+
+## Handoffs
+
+- The value of an address at run time → c64-emulator
+- A conclusion about the application → c64-knowledge

package/skills/c64-memory-map/scripts/memmap.ts

@@ -0,0 +1,92 @@
+// The c64-memory-map script: the platform meaning of C64 addresses and of
+// values in platform registers. Local lookup only; nothing is stored. export
+// writes the platform names to a project file as an ACME include (D52).
+
+import { parseArgs } from "node:util";
+
+import { writeProjectFile } from "#src/host-client/transfer.ts";
+import { decode, IO, lookup, platformNames, ROM, SYSTEM, type Entry } from "#src/platform.ts";
+import { runScript, UsageError } from "#src/script.ts";
+
+const USAGE = `memmap.ts at <address> [<address> ...]
+memmap.ts decode <register> <value> [--bank <$dd00 value>]
+memmap.ts list <vic|sid|cia1|cia2|kernal|system>
+memmap.ts export --out <file>     the platform names as an ACME include (NAME = $xxxx)
+
+Addresses and values are $ and hex digits, decimal, or % and binary digits.
+The result is one JSON object.`;
+
+const hex = (value: number, digits = 4) => `$${value.toString(16).padStart(digits, "0")}`;
+
+function number(text: string, max: number, what: string): number {
+  let value = Number.NaN;
+  if (/^\$[0-9a-f]{1,4}$/i.test(text)) value = Number.parseInt(text.slice(1), 16);
+  else if (/^%[01]{1,16}$/.test(text)) value = Number.parseInt(text.slice(1), 2);
+  else if (/^\d{1,5}$/.test(text)) value = Number(text);
+  if (!(value >= 0 && value <= max)) throw new UsageError(`${what} must be from 0 to ${hex(max, max > 0xff ? 4 : 2)}, not ${text}`);
+  return value;
+}
+
+const show = (entry: Entry) => ({ start: hex(entry.start), ...(entry.end !== entry.start ? { end: hex(entry.end) } : {}), name: entry.name, meaning: entry.meaning });
+
+function run(argv: string[]): unknown {
+  const { values, positionals } = parseArgs({ args: argv, strict: true, allowPositionals: true, options: { bank: { type: "string" }, out: { type: "string" }, help: { type: "boolean" } } });
+  if (values.help || positionals.length === 0) throw new UsageError("");
+  const [command, ...rest] = positionals;
+  switch (command) {
+    case "at": {
+      if (rest.length === 0) throw new UsageError("give at least one address");
+      return {
+        addresses: rest.map((text) => {
+          const found = lookup(number(text, 0xffff, "an address"));
+          // The chip registers, color RAM and the cartridge I/O areas show only while $01 maps I/O in.
+          const io = found.address >= 0xd000 && found.address <= 0xdfff;
+          return {
+            ...found,
+            address: hex(found.address),
+            ...(found.mirrorOf === undefined ? {} : { mirrorOf: hex(found.mirrorOf) }),
+            ...(found.entryStart === undefined ? {} : { entryStart: hex(found.entryStart) }),
+            ...(io ? { visible: "when $01 shows I/O at $D000 (the default)" } : {}),
+          };
+        }),
+      };
+    }
+    case "decode": {
+      if (rest.length !== 2) throw new UsageError("decode takes a register and a value");
+      const register = number(rest[0]!, 0xffff, "the register");
+      const value = number(rest[1]!, 0xff, "the value");
+      const bank = values.bank === undefined ? undefined : number(values.bank, 0xff, "--bank");
+      try {
+        return { register: hex(register), value: hex(value, 2), name: lookup(register).name, ...decode(register, value, bank) };
+      } catch (error) {
+        if (error instanceof RangeError) throw new UsageError(error.message);
+        throw error;
+      }
+    }
+    case "list": {
+      const lists: Record<string, Entry[]> = {
+        vic: IO[0]!.registers,
+        sid: IO[1]!.registers,
+        cia1: IO[2]!.registers,
+        cia2: IO[3]!.registers,
+        kernal: ROM,
+        system: SYSTEM,
+      };
+      const list = lists[rest[0] ?? ""];
+      if (list === undefined || rest.length !== 1) throw new UsageError(`list takes one of ${Object.keys(lists).join(", ")}`);
+      return { entries: list.map(show) };
+    }
+    case "export": {
+      if (rest.length !== 0 || values.out === undefined) throw new UsageError("export takes --out <file> and nothing else");
+      const names = platformNames();
+      const lines = ["; C64 platform names from the c64-memory-map skill, for ACME.", ...names.map((item) => `${item.name} = ${hex(item.address, item.address < 0x100 ? 2 : 4)}`.padEnd(24) + `; ${item.meaning}`)];
+      const bytes = Buffer.from(`${lines.join("\n")}\n`, "utf8");
+      writeProjectFile(values.out, bytes, { what: "--out" });
+      return { out: values.out, names: names.length, bytes: bytes.length };
+    }
+    default:
+      throw new UsageError(`unknown command ${command}`);
+  }
+}
+
+await runScript(() => run(process.argv.slice(2)), { usage: USAGE });

package/skills/c64-provenance/SKILL.md

@@ -0,0 +1,110 @@
+---
+name: c64-provenance
+description: Use this skill to compare two or more releases of a C64 program, for example cracked versions from different groups. It finds the bytes where the releases differ and shows which releases agree there. It can run each release and tell how the program uses each difference. With one release, it gives evidence for each range: the unpack stage, the access, and the code that the entry does not reach. Use the result to separate the content of the application from changes that a release added. Do not use it to analyze the code.
+---
+
+# c64-provenance
+
+## Purpose
+
+Compare releases of a C64 program. Find the bytes where they differ and which releases agree. Use this to find which bytes are probably the original application and which are changes of one release.
+
+This skill does not change project knowledge.
+
+## Inputs
+
+- Two or more releases, relative to the project directory. A release is a PRG file or an `image.json` from c64-unpacker capture. Extract releases from disk images with the c64-disk skill. Unpack packed releases with the c64-unpacker skill first.
+- Optional: an address range to compare.
+- Optional: a shift for a release that loads at a different address.
+- Optional: a number of frames to run each release, and a steps file.
+- For one release: the release, coverage files, and the knowledge of the project.
+
+## Tools / execution path
+
+The script compares the files locally. Only `--run` uses the host runtime: each release runs in its own emulator.
+
+```
+node <skill>/scripts/provenance.ts releases/a.prg releases/b.prg releases/c.prg
+node <skill>/scripts/provenance.ts releases/a.prg releases/b.prg --shift 2:-256 --from '$0900' --to '$09ff'
+node <skill>/scripts/provenance.ts releases/a.prg unpacked/b/image.json --run 600 --steps steps/start-game.json
+node <skill>/scripts/provenance.ts separate unpacked/image.json --coverage coverage/attract.bin --coverage coverage/level1.bin --out notes/separate.json
+```
+
+| Option | Meaning |
+|---|---|
+| `--shift 2:-256` | Move release 2 (the first release is 1) by -256 bytes. Use it again for more releases. |
+| `--from '$0900'` | Compare from this address. |
+| `--to '$09ff'` | Compare up to this address. |
+| `--run 600` | Run each release for this many frames, and give each listed difference its runtime evidence. |
+| `--steps steps/start-game.json` | With `--run`: the action steps of the c64-unpacker steps file to run before the frames, for example a key to start the game. |
+
+The script puts each PRG at its load address. An image has bytes only in its segments. The script compares the addresses where all releases have bytes. It prints one JSON object.
+
+With `--shift`, the addresses of `segments`, `common` and `differences` are the release addresses plus the `shift` of the release. The `start` and `end` of `runtime` are the addresses in the release itself, without the shift.
+
+With `--run`, a PRG starts with autostart and RUN. An image starts from its bytes: the script writes the segments and the entry state from `entry_state.json` into the emulator, and starts at the entry. Watchpoints on each difference stop the run at its first access.
+
+## Workflow
+
+1. Run the script on the releases.
+2. Read `identicalShare`. If it is low and `suggestedShifts` is present, the releases are at different addresses. Run the script again with a `--shift` of `suggestedShifts`. `shiftsNotSuggested` tells why a found shift does not fit in `$0000-$ffff`.
+
+   `suggestedShifts` has at most 3 shifts for each release. The shift with the most `matchingSequences` is first. Data that did not move and code that moved give different shifts. Try each shift. `current: true` marks the shift of this run.
+3. Read `differences`. Each one has an address range, the bytes of each release, and `groups`: the releases that have the same bytes there.
+4. To know how the program uses the differences, run the script again with `--run` and the frames that get the program to the part you examine. Read `runtime` of each difference:
+   - `first`: the first access in that release: `read`, `write`, `execute` or `none`. An instruction that reads and then writes the same byte, for example INC, counts as `read`. The operand bytes of an executed instruction count as `execute`, for example the value of `LDA #$03`.
+   - `changed`: `true` when the bytes at the end of the run are not the bytes of the release.
+   - A range that each release writes before it reads is probably runtime state, not a difference of the releases.
+   - A range with `none` in each release is not used in this run. Run more frames or other steps before you decide.
+   - `run.skipped`: the releases have no differences, so no release ran.
+5. Examine each important difference:
+   - Disassemble the bytes of each release with the c64-emulator skill, or analyze them with the c64-static-analysis skill.
+   - Find what the change does, for example a trainer, a removed copy protection, or a changed text.
+6. Decide which bytes are probably the original application. Use more than the byte comparison: the code, the text, and the history of the releases.
+7. Record the conclusion as a comment or a region with the c64-knowledge skill.
+
+## One release
+
+Use `separate` when you have only one release. It gives evidence, not a decision:
+
+- `ranges`: the image in ranges with the same stage and access. `stage` is the last unpack stage that wrote the range. In `image.json`, the last stage is the program. An earlier stage is a depacker, an intro or a loader. Only an image from c64-unpacker `capture` has stages. For an image from `loads`, `noStages` tells this.
+- `access`: from the joined `--coverage` files: `executed`, `read`, `written`, `read-written` or `untouched`. `untouched` bytes did not take part in the runs.
+- `codeRegions.total`: the number of code regions in the knowledge.
+- `codeRegions.unreachable`: code regions of the knowledge that no chain of call and jump references reaches from the entry. Run c64-static-analysis first, so that the knowledge has the code regions and the references.
+- `outside`: RAM outside the image that the program `read` or `executed` but did not write, and code it `generated`. A loader or an earlier stage put the bytes that the program did not write. A `generated` range can have gaps of two bytes or less: these are operand bytes.
+
+Each list has `count` and `listed`. `listed` has the first ranges, at most `--max-listed`. `--out` writes all of them. Run more coverage runs for parts that are `untouched` before you decide.
+
+## Knowledge
+
+Automatic write:
+- none.
+
+Record a confirmed conclusion as a normal comment, region or symbol with the c64-knowledge skill. Do not record the full comparison.
+
+## Result
+
+The differing ranges with the bytes of each release and the groups that agree, and the share of identical bytes. `common` gives the ranges where all releases have bytes. `releases` gives the segments and the `shift` of each release. With `--run`, each difference also has `runtime`, and `run` tells if each release started, or why not.
+
+The runtime evidence is evidence only. The script does not decide which bytes are original.
+
+The exit status is 0 for a result. The exit status is 1 for an error. The exit status is 2 for a wrong argument.
+
+## Failure and conflicts
+
+- Do not call releases independent when you do not know their relation. Two cracks can come from the same source.
+- Do not call a byte original only because one release has it. A majority of releases is also not proof.
+- `common` is empty: the releases do not cover the same addresses. Use `--shift`, or compare a range that all releases cover.
+- `run.releases` shows `started: false`: a PRG without a BASIC start, or an image without `entry_state.json`. The reason tells which. The other releases still give evidence.
+- `installation-incomplete`, or `machine-unavailable` with a message that the host runtime does not run: stop the task. Give the message to the user and wait for the user. The message gives the remedy. Do not make a replacement for the tool or the host runtime, for example your own emulator script.
+- `moreDifferences: true`: the script shows only the first differences. Use `--from` and `--to` to see the others.
+- `not-found`: the project has no file at that path.
+- `invalid-input`: a file, an address or a shift is wrong. A path that goes out of the project directory is also wrong, also through a link. Read the message.
+
+## Handoffs
+
+- Get releases from disk images → c64-disk
+- Unpack a packed release → c64-unpacker
+- Run or examine the different code → c64-emulator
+- Analyze the different code → c64-static-analysis
+- Record a conclusion → c64-knowledge

package/skills/c64-provenance/scripts/compare.ts

@@ -0,0 +1,152 @@
+// Byte comparison of releases for the c64-provenance script. Each release
+// has bytes in one or more segments: a PRG at its load address plus an
+// optional shift, or the segments of an unpacker image (D50). The comparison
+// covers the addresses where every release has bytes, and reports where the
+// releases differ and which releases agree there. It never decides which
+// bytes are original: that needs evidence beyond one comparison.
+
+export interface Range {
+  start: number;
+  end: number;
+}
+
+export interface Release {
+  name: string;
+  /** Where the release has bytes, after the shift, in address order. */
+  segments: Range[];
+  /** The byte at an address inside a segment. */
+  byteAt: (address: number) => number;
+}
+
+export interface DifferingRange {
+  start: number;
+  end: number;
+  /** The bytes of each release in this range, by release name. */
+  values: Record<string, number[]>;
+  /** Releases with the same bytes here, largest group first. */
+  groups: string[][];
+}
+
+export interface Comparison {
+  /** The addresses where every release has bytes. */
+  common: Range[];
+  identicalBytes: number;
+  differingBytes: number;
+  differences: DifferingRange[];
+  /** True when more differing ranges exist than the limit allowed. */
+  moreDifferences: boolean;
+}
+
+/** Joins differing addresses that are at most this far apart into one range. */
+const JOIN_GAP = 4;
+
+export const covers = (release: Release, address: number) => release.segments.some((segment) => address >= segment.start && address <= segment.end);
+
+/** The ranges where every release has bytes, inside from..to. */
+export function commonRanges(releases: Release[], from = 0x0000, to = 0xffff): Range[] {
+  const ranges: Range[] = [];
+  for (let address = from; address <= to; address++) {
+    if (!releases.every((release) => covers(release, address))) continue;
+    const last = ranges.at(-1);
+    if (last !== undefined && last.end + 1 === address) last.end = address;
+    else ranges.push({ start: address, end: address });
+  }
+  return ranges;
+}
+
+export function compareReleases(releases: Release[], options: { from?: number; to?: number; maxRanges?: number } = {}): Comparison {
+  const common = commonRanges(releases, options.from, options.to);
+  let identicalBytes = 0;
+  const differing: number[] = [];
+  for (const range of common) {
+    for (let address = range.start; address <= range.end; address++) {
+      const first = releases[0]!.byteAt(address);
+      if (releases.every((release) => release.byteAt(address) === first)) identicalBytes++;
+      else differing.push(address);
+    }
+  }
+  // A joined range stays inside one common range, so every release has all of its bytes.
+  const inSameCommon = (a: number, b: number) => common.some((range) => a >= range.start && b <= range.end);
+  const ranges: Range[] = [];
+  for (const address of differing) {
+    const last = ranges.at(-1);
+    if (last !== undefined && address - last.end <= JOIN_GAP && inSameCommon(last.start, address)) last.end = address;
+    else ranges.push({ start: address, end: address });
+  }
+  const limit = options.maxRanges ?? 64;
+  const differences = ranges.slice(0, limit).map((range) => {
+    const values: Record<string, number[]> = {};
+    const groups = new Map<string, string[]>();
+    for (const release of releases) {
+      const bytes = Array.from({ length: range.end - range.start + 1 }, (_, offset) => release.byteAt(range.start + offset));
+      values[release.name] = bytes;
+      const key = bytes.join(",");
+      groups.set(key, [...(groups.get(key) ?? []), release.name]);
+    }
+    return { ...range, values, groups: [...groups.values()].sort((a, b) => b.length - a.length) };
+  });
+  return { common, identicalBytes, differingBytes: differing.length, differences, moreDifferences: ranges.length > limit };
+}
+
+/** The addresses of a release in order, so that a window of bytes can run over its segments. */
+function addressesOf(release: Release): number[] {
+  return release.segments.flatMap((segment) => Array.from({ length: segment.end - segment.start + 1 }, (_, offset) => segment.start + offset));
+}
+
+/** Most shifts that suggestShifts gives: data that did not move can outvote code that did (b2/04). */
+export const MAX_SUGGESTED_SHIFTS = 3;
+
+/**
+ * The shifts that put `other` over `base` best: the most common address
+ * differences between equal 8-byte sequences, most votes first. A shift with
+ * less than a tenth of the best one's votes is chance. Empty when nothing matches.
+ */
+export function suggestShifts(base: Release, other: Release): Array<{ shift: number; matches: number }> {
+  const WINDOW = 8;
+  const key = (release: Release, addresses: number[], at: number) =>
+    String.fromCharCode(...addresses.slice(at, at + WINDOW).map((address) => release.byteAt(address)));
+  const baseAddresses = addressesOf(base);
+  const otherAddresses = addressesOf(other);
+  const positions = new Map<string, number[]>();
+  for (let at = 0; at + WINDOW <= baseAddresses.length; at++) {
+    const found = positions.get(key(base, baseAddresses, at));
+    // Very common sequences (fill bytes) say nothing about the alignment.
+    if (found === undefined) positions.set(key(base, baseAddresses, at), [baseAddresses[at]!]);
+    else if (found.length < 8) found.push(baseAddresses[at]!);
+  }
+  const votes = new Map<number, number>();
+  for (let at = 0; at + WINDOW <= otherAddresses.length; at++) {
+    for (const address of positions.get(key(other, otherAddresses, at)) ?? []) {
+      const shift = address - otherAddresses[at]!;
+      votes.set(shift, (votes.get(shift) ?? 0) + 1);
+    }
+  }
+  const ranked = [...votes].map(([shift, matches]) => ({ shift, matches })).sort((a, b) => b.matches - a.matches || Math.abs(a.shift) - Math.abs(b.shift));
+  return ranked.filter((found) => found.matches * 10 >= ranked[0]!.matches).slice(0, MAX_SUGGESTED_SHIFTS);
+}
+
+export type ShiftSuggestion = { release: number; shift: number; matchingSequences: number } | { release: number; matchingSequences: number; reason: string };
+
+/**
+ * The --shift to suggest for release `index` (0 = the first) from `found`,
+ * the best shift of that release over the first one (F-25). The shift is the
+ * full value for --shift: `applied` holds the shifts already given. A shift
+ * that moves a segment out of $0000-$ffff is not suggested: with two releases,
+ * the opposite shift of the first release is suggested when it fits, else the
+ * result gives the reason.
+ */
+export function shiftSuggestion(releases: Release[], applied: number[], index: number, found: { shift: number; matches: number }): ShiftSuggestion {
+  const outside = (release: Release, by: number) => release.segments.find((segment) => segment.start + by < 0 || segment.end + by > 0xffff);
+  const hex = (value: number) => `$${value.toString(16).padStart(4, "0")}`;
+  const why = (number: number, by: number, segment: Range) =>
+    `release ${number} moved by ${by} puts ${hex(segment.start)}-${hex(segment.end)} ${segment.start + by < 0 ? "below $0000" : "past $ffff"}`;
+  const own = outside(releases[index]!, found.shift);
+  if (own === undefined) return { release: index + 1, shift: applied[index]! + found.shift, matchingSequences: found.matches };
+  const reasons = [why(index + 1, found.shift, own)];
+  if (releases.length === 2) {
+    const base = outside(releases[0]!, -found.shift);
+    if (base === undefined) return { release: 1, shift: applied[0]! - found.shift, matchingSequences: found.matches };
+    reasons.push(why(1, -found.shift, base));
+  }
+  return { release: index + 1, matchingSequences: found.matches, reason: `No shift fits: ${reasons.join(", and ")}.` };
+}