@henols/c64-re-tools 2.0.0-rc.8 → 2.0.0

package/skills/c64-knowledge/scripts/lib/listing.ts

@@ -0,0 +1,162 @@
+// One listing of the image with its knowledge: the lines of a range as
+// ACME text with names, operand expressions and comments, the access class of
+// the bytes from coverage files, and the references in and out. Nothing here
+// reads files: knowledge.ts gives the lookups.
+
+import { COVERAGE } from "#src/c64.ts";
+import { MODE_LENGTH, OPCODES, type Mode } from "#src/opcodes.ts";
+
+export interface ListingSource {
+  /** The image byte at an address, or undefined outside the image. */
+  byteAt: (address: number) => number | undefined;
+  /** The name of an address: a symbol, else a platform name. */
+  nameOf: (address: number) => string | undefined;
+  /** The analyzer names of an address, for reference. */
+  secondNames: (address: number) => string[];
+  /** The knowledge regions, in address order. */
+  regions: ReadonlyArray<{ start: number; end: number; type: string }>;
+  /** The line and side comments of an address. */
+  comments: (address: number) => { line: string[]; side?: string };
+  /** The operand expression of the instruction at an address, as ACME text. */
+  operand: (address: number) => string | undefined;
+  /** The references to and from an address. */
+  references: (address: number) => { in: number; out: number };
+  /** Joined coverage files, or undefined without them. */
+  coverage?: Uint8Array;
+}
+
+export interface ListingLine {
+  address: number;
+  /** The bytes as hex, separated by spaces. */
+  bytes: string;
+  label?: string;
+  analyzerNames?: string[];
+  text: string;
+  /** From the coverage: x executed, r read, w written, - none. */
+  access?: string;
+  referencesIn: number;
+  referencesOut: number;
+  lineComments?: string[];
+  comment?: string;
+}
+
+const CODE = new Set(["code", "jumps"]);
+const hex2 = (value: number) => `$${value.toString(16).padStart(2, "0")}`;
+const hex4 = (value: number) => `$${value.toString(16).padStart(4, "0")}`;
+const DATA_PER_LINE = 8;
+
+/** The access class of bytes from the coverage: x, r and w as they occur, or - for none. */
+function accessOf(coverage: Uint8Array | undefined, start: number, length: number): string | undefined {
+  if (coverage === undefined) return undefined;
+  let flags = 0;
+  for (let offset = 0; offset < length; offset++) flags |= coverage[start + offset]!;
+  const text = `${flags & (COVERAGE.ramExecute | COVERAGE.romExecute) ? "x" : ""}${flags & (COVERAGE.ramRead | COVERAGE.romRead) ? "r" : ""}${flags & COVERAGE.ramWrite ? "w" : ""}`;
+  return text === "" ? "-" : text;
+}
+
+/** The operand of an instruction in ACME syntax, with the name of its target where one exists. */
+function operandOf(mode: Mode, address: number, low: number, high: number, nameOf: ListingSource["nameOf"]): string {
+  const word = low | (high << 8);
+  const named = (target: number, digits: 2 | 4) => nameOf(target) ?? (digits === 2 ? hex2(target) : hex4(target));
+  switch (mode) {
+    case "imp":
+      return "";
+    case "imm":
+      return `#${hex2(low)}`;
+    case "zp":
+      return named(low, 2);
+    case "zpx":
+      return `${named(low, 2)},x`;
+    case "zpy":
+      return `${named(low, 2)},y`;
+    case "abs":
+      return named(word, 4);
+    case "abx":
+      return `${named(word, 4)},x`;
+    case "aby":
+      return `${named(word, 4)},y`;
+    case "ind":
+      return `(${named(word, 4)})`;
+    case "izx":
+      return `(${named(low, 2)},x)`;
+    case "izy":
+      return `(${named(low, 2)}),y`;
+    case "rel":
+      return named((address + 2 + (low < 0x80 ? low : low - 0x100)) & 0xffff, 4);
+  }
+}
+
+/** The lines of start..end: instructions in code regions, decoded from the region start, and bytes elsewhere. */
+export function listing(source: ListingSource, start: number, end: number): ListingLine[] {
+  const lines: ListingLine[] = [];
+  const line = (address: number, length: number, text: string): ListingLine => {
+    const bytes = Array.from({ length }, (_, offset) => source.byteAt(address + offset)!.toString(16).padStart(2, "0")).join(" ");
+    const label = source.nameOf(address);
+    const second = source.secondNames(address).filter((name) => name !== label);
+    const { line: lineComments, side } = source.comments(address);
+    const references = source.references(address);
+    const access = accessOf(source.coverage, address, length);
+    return {
+      address,
+      bytes,
+      ...(label === undefined ? {} : { label }),
+      ...(second.length === 0 ? {} : { analyzerNames: second }),
+      text,
+      ...(access === undefined ? {} : { access }),
+      referencesIn: references.in,
+      referencesOut: references.out,
+      ...(lineComments.length === 0 ? {} : { lineComments }),
+      ...(side === undefined ? {} : { comment: side }),
+    };
+  };
+  const regionAt = (address: number) => source.regions.find((region) => address >= region.start && address <= region.end);
+  // Start at the code region that holds the first address, so the instructions keep their boundaries.
+  const first = regionAt(start);
+  let address = first !== undefined && CODE.has(first.type) ? first.start : start;
+  while (address <= end) {
+    const byte = source.byteAt(address);
+    if (byte === undefined) {
+      address++;
+      continue;
+    }
+    const region = regionAt(address);
+    if (region !== undefined && CODE.has(region.type)) {
+      const opcode = OPCODES[byte]!;
+      const length = MODE_LENGTH[opcode.mode];
+      const bytes = Array.from({ length: length - 1 }, (_, offset) => source.byteAt(address + 1 + offset));
+      if (address + length - 1 <= region.end && bytes.every((value) => value !== undefined)) {
+        const operand = source.operand(address) ?? operandOf(opcode.mode, address, bytes[0] ?? 0, bytes[1] ?? 0, source.nameOf);
+        if (address >= start) lines.push(line(address, length, operand === "" ? opcode.mnemonic : `${opcode.mnemonic} ${operand}`));
+        address += length;
+        continue;
+      }
+    }
+    // Bytes up to the next label, region edge or end of the line.
+    const values: number[] = [];
+    let at = address;
+    while (at <= end && values.length < DATA_PER_LINE && source.byteAt(at) !== undefined && (at === address || (source.nameOf(at) === undefined && regionAt(at) === region))) {
+      values.push(source.byteAt(at)!);
+      at++;
+    }
+    if (address >= start) lines.push(line(address, values.length, `!byte ${values.map(hex2).join(",")}`));
+    address = at;
+  }
+  return lines;
+}
+
+/** A listing as text, one line for each line, with its line comments above it. */
+export function listingText(lines: readonly ListingLine[]): string {
+  const out: string[] = [];
+  for (const item of lines) {
+    for (const comment of item.lineComments ?? []) out.push(`                         ; ${comment}`);
+    const head = `${hex4(item.address)}  ${item.bytes.padEnd(9)} ${(item.access ?? "").padEnd(3)}`;
+    const body = (item.label === undefined ? "" : `${item.label} `).padEnd(20) + item.text;
+    const notes = [
+      ...(item.comment === undefined ? [] : [item.comment]),
+      ...(item.referencesIn + item.referencesOut === 0 ? [] : [`refs in ${item.referencesIn} out ${item.referencesOut}`]),
+      ...(item.analyzerNames === undefined ? [] : [`also ${item.analyzerNames.join(", ")}`]),
+    ];
+    out.push(`${head} ${body.padEnd(44)}${notes.length === 0 ? "" : ` ; ${notes.join(" | ")}`}`.trimEnd());
+  }
+  return `${out.join("\n")}\n`;
+}

package/skills/c64-memory-map/SKILL.md

@@ -30,14 +30,23 @@ node <skill>/scripts/memmap.ts export --out src/hw.asm
 | Command | Result |
 |---|---|
 | `at <address> ...` | For each address: `kind` (`platform` or `application`), the area, the name and the meaning. |
-| `decode <register> <value>` | The meaning of the value. Use `--bank` with the value of `$DD00` to get the addresses for `$D018`. |
+| `decode <register> <value>` | The meaning of the value. Use `--bank` with the value of `$DD00` to get the addresses for `$D018`. The decoders are for `$01`, `$D010`, `$D011`, `$D015` to `$D01F`, `$D016`, `$D018`, `$DC0D`, `$DD00` and `$DD0D`. |
 | `list <vic\|sid\|cia1\|cia2\|kernal\|system>` | All entries of that group. |
-| `export --out <file>` | Writes all platform names to a project file as an ACME include: one `NAME = $xxxx` line for each name, with its meaning as a comment. |
+| `export --out <file>` | Writes all platform names to a project file as an ACME include: one line for each name, with its meaning as a comment. A zero-page name is `NAME = $xx`. Other names are `NAME = $xxxx`. |
 
 Write an address or a value as `$` and hex digits, as a decimal number, or as `%` and binary digits. Put a `$` value in single quotes in a shell.
 
 The script prints one JSON object.
 
+### Project folder
+
+- Paths are relative to the project folder. The project folder is the nearest folder at or above the working directory that holds `.c64-re-tools/`.
+- If the working directory is not in a project and has projects below it, the script refuses and lists them. Then give `--project <folder>` to each script.
+- Write each file that you make for your work in `.c64-re-tools/local/` of the project. Git ignores that folder. Examples are results, batches, step files, runs, snapshots and memory dumps.
+- Move a file out of `.c64-re-tools/local/` only when it is the evidence for a finding. Move it before you give it to `--evidence` of c64-knowledge. `--evidence` refuses a file in `.c64-re-tools/local/`.
+- Put a run that is evidence in `runs/<name>/`, with its run file, its results and the snapshot files that it uses. Change the paths in the run file to the new paths. Then git keeps them, and another person can do the run again.
+- A shell redirect (`>`) is relative to the working directory, not to the project folder. From the folder above the project, write `<folder>/.c64-re-tools/local/result.json`.
+
 ## Workflow
 
 1. Run `at` for the addresses in the code or the question.

package/skills/c64-memory-map/scripts/memmap.ts

@@ -1,6 +1,6 @@
 // The c64-memory-map script: the platform meaning of C64 addresses and of
 // values in platform registers. Local lookup only; nothing is stored. export
-// writes the platform names to a project file as an ACME include (D52).
+// writes the platform names to a project file as an ACME include.
 
 import { parseArgs } from "node:util";
 
@@ -89,4 +89,4 @@ function run(argv: string[]): unknown {
   }
 }
 
-await runScript(() => run(process.argv.slice(2)), { usage: USAGE });
+await runScript((argv) => run(argv), { usage: USAGE });

package/skills/c64-provenance/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: c64-provenance
-description: Use this skill to compare two or more releases of a C64 program, for example cracked versions from different groups. It finds the bytes where the releases differ and shows which releases agree there. It can run each release and tell how the program uses each difference. With one release, it gives evidence for each range: the unpack stage, the access, and the code that the entry does not reach. Use the result to separate the content of the application from changes that a release added. Do not use it to analyze the code.
+description: Use this skill to compare two or more releases of a C64 program, for example cracked versions from different groups. It finds the bytes where the releases differ and shows which releases agree there. It can run each release and tell how the program uses each difference. It also gives evidence for each range of the program in the knowledge. The evidence is the unpack stage, the access, and the code that the entry does not reach. Use the result to separate the content of the application from changes that a release added. It also keeps a library of routines that several programs share. It finds them in the program of the knowledge and writes their names as a batch for c64-knowledge. Do not use it to analyze the code.
 ---
 
 # c64-provenance
@@ -9,15 +9,18 @@ description: Use this skill to compare two or more releases of a C64 program, fo
 
 Compare releases of a C64 program. Find the bytes where they differ and which releases agree. Use this to find which bytes are probably the original application and which are changes of one release.
 
+Keep a library of routines that several programs share, for example a music driver or a decruncher. Find the routines of the library in a program, and get their names for that program.
+
 This skill does not change project knowledge.
 
 ## Inputs
 
-- Two or more releases, relative to the project directory. A release is a PRG file or an `image.json` from c64-unpacker capture. Extract releases from disk images with the c64-disk skill. Unpack packed releases with the c64-unpacker skill first.
+- Two or more releases, relative to the project folder. A release is a PRG file or an `image.json` from c64-unpacker capture. Extract releases from disk images with the c64-disk skill. Unpack packed releases with the c64-unpacker skill first.
 - Optional: an address range to compare.
 - Optional: a shift for a release that loads at a different address.
 - Optional: a number of frames to run each release, and a steps file.
-- For one release: the release, coverage files, and the knowledge of the project.
+- For one release: the program, the coverage and the other knowledge of the project. Put the program and the coverage in with `image` and `coverage` of the c64-knowledge skill.
+- For the library: the program and the other knowledge of its project, and the library folder.
 
 ## Tools / execution path
 
@@ -26,8 +29,8 @@ The script compares the files locally. Only `--run` uses the host runtime: each
 ```
 node <skill>/scripts/provenance.ts releases/a.prg releases/b.prg releases/c.prg
 node <skill>/scripts/provenance.ts releases/a.prg releases/b.prg --shift 2:-256 --from '$0900' --to '$09ff'
-node <skill>/scripts/provenance.ts releases/a.prg unpacked/b/image.json --run 600 --steps steps/start-game.json
-node <skill>/scripts/provenance.ts separate unpacked/image.json --coverage coverage/attract.bin --coverage coverage/level1.bin --out notes/separate.json
+node <skill>/scripts/provenance.ts releases/a.prg .c64-re-tools/local/b/image.json --run 600 --steps .c64-re-tools/local/start-game.json
+node <skill>/scripts/provenance.ts separate --out .c64-re-tools/local/separate.json
 ```
 
 | Option | Meaning |
@@ -44,6 +47,15 @@ With `--shift`, the addresses of `segments`, `common` and `differences` are the
 
 With `--run`, a PRG starts with autostart and RUN. An image starts from its bytes: the script writes the segments and the entry state from `entry_state.json` into the emulator, and starts at the entry. Watchpoints on each difference stop the run at its first access.
 
+### Project folder
+
+- Paths are relative to the project folder. The project folder is the nearest folder at or above the working directory that holds `.c64-re-tools/`.
+- If the working directory is not in a project and has projects below it, the script refuses and lists them. Then give `--project <folder>` to each script.
+- Write each file that you make for your work in `.c64-re-tools/local/` of the project. Git ignores that folder. Examples are results, batches, step files, runs, snapshots and memory dumps.
+- Move a file out of `.c64-re-tools/local/` only when it is the evidence for a finding. Move it before you give it to `--evidence` of c64-knowledge. `--evidence` refuses a file in `.c64-re-tools/local/`.
+- Put a run that is evidence in `runs/<name>/`, with its run file, its results and the snapshot files that it uses. Change the paths in the run file to the new paths. Then git keeps them, and another person can do the run again.
+- A shell redirect (`>`) is relative to the working directory, not to the project folder. From the folder above the project, write `<folder>/.c64-re-tools/local/result.json`.
+
 ## Workflow
 
 1. Run the script on the releases.
@@ -65,15 +77,51 @@ With `--run`, a PRG starts with autostart and RUN. An image starts from its byte
 
 ## One release
 
-Use `separate` when you have only one release. It gives evidence, not a decision:
+Use `separate` when you have only one release. It reads the program, its stages and the coverage from the knowledge. It gives evidence, not a decision:
 
-- `ranges`: the image in ranges with the same stage and access. `stage` is the last unpack stage that wrote the range. In `image.json`, the last stage is the program. An earlier stage is a depacker, an intro or a loader. Only an image from c64-unpacker `capture` has stages. For an image from `loads`, `noStages` tells this.
-- `access`: from the joined `--coverage` files: `executed`, `read`, `written`, `read-written` or `untouched`. `untouched` bytes did not take part in the runs.
+- `ranges`: the program in ranges with the same stage and access. `stage` is the last unpack stage that wrote the range. The last stage is the program. An earlier stage is a depacker, an intro or a loader. Only a capture of c64-unpacker `capture` has stages. For a program from `loads` or a PRG, `noStages` tells this.
+- `access`: from the coverage of the knowledge: `executed`, `read`, `written`, `read-written` or `untouched`. `untouched` bytes did not take part in the runs. A `read` byte right after executed code, or at a return address, can be a dummy read of the 6502 that the program does not use.
 - `codeRegions.total`: the number of code regions in the knowledge.
 - `codeRegions.unreachable`: code regions of the knowledge that no chain of call and jump references reaches from the entry. Run c64-static-analysis first, so that the knowledge has the code regions and the references.
-- `outside`: RAM outside the image that the program `read` or `executed` but did not write, and code it `generated`. A loader or an earlier stage put the bytes that the program did not write. A `generated` range can have gaps of two bytes or less: these are operand bytes.
+- `outside`: RAM outside the program that the program `read` or `executed` but did not write, and code it `generated`. A loader or an earlier stage put the bytes that the program did not write. A `generated` range can have gaps of two bytes or less: these are operand bytes.
+
+Each list has `count` and `listed`. `listed` has the first ranges, at most `--max-listed`. `--out` writes all of them. Run more coverage runs for parts that are `untouched`, and add them with `coverage` of the c64-knowledge skill, before you decide.
+
+## Library of routines
+
+A library is a folder of routine files. Each file holds one routine: its bytes and its knowledge. Keep one library beside the project folders, for example `library/` at the root of the workspace. A library is not a project folder. Commit it with the projects.
 
-Each list has `count` and `listed`. `listed` has the first ranges, at most `--max-listed`. `--out` writes all of them. Run more coverage runs for parts that are `untouched` before you decide.
+`--library` is relative to the working directory, as `--project`. All other paths are relative to the project folder.
+
+```
+node <skill>/scripts/provenance.ts routine --module '$1000' --library library --project game-a
+node <skill>/scripts/provenance.ts routine --start '$1000' --end '$1026' --name play --library library --project game-a
+node <skill>/scripts/provenance.ts match --library library --out .c64-re-tools/local/match.json --project game-b
+```
+
+To save a routine:
+
+1. Put the program into the knowledge, and name the routine there first. The file keeps the user and LLM names, comments, regions, references and operand expressions in the range. It also keeps the code regions of each origin.
+2. Run `routine` with the start of a module (`--module`), or with `--start` and `--end`. A module ends at the next module or at the end of its segment.
+   - `--name` is the name of the routine and of its file. When you do not give it, the script uses the user or LLM name at the start.
+   - `--title` is the title of the routine. When you do not give it, the script uses the module title.
+   - A routine has at least 16 bytes. Its last instruction must end in the range.
+3. Read `leftOut`. It lists the rows that the file does not keep, and why. For example, the file does not keep an operand expression that uses a name outside the routine.
+
+To find the routines in a program:
+
+1. Put the program into the knowledge. Then run `match`. The script examines each routine of the library at each address of the program.
+   - The opcodes and the other bytes must be equal.
+   - An address into the routine must point into the routine at its new address. `shift` is the address in the program minus the address in the library.
+   - An operand that addresses something outside the routine can be different. `differing` lists each one with its value in the library and in the program. Examine them. For example, the program can use other variables or another SID voice.
+2. Give `--out` to get a batch for `knowledge.ts apply`. The batch has the names, regions, comments, references and operand expressions of each routine at its addresses in the program. `match` does not change the knowledge.
+3. Read `batch.leftOut` before you apply the batch. The script leaves out a write in these conditions:
+   - The program has a user or LLM row at that place.
+   - The program uses the name at another address.
+   - A removal holds at that place.
+   - The routine occurs two times. Only the first copy gets the names.
+   - The script leaves out the name that an operand expression uses, or the expression does not give the operand in the program.
+4. Apply the batch with the command in `apply`. Add `--project` when you are not in the project folder. The batch is one revision of the LLM. `batch.known` counts the writes that the program has already.
 
 ## Knowledge
 
@@ -82,6 +130,8 @@ Automatic write:
 
 Record a confirmed conclusion as a normal comment, region or symbol with the c64-knowledge skill. Do not record the full comparison.
 
+`match --out` writes a batch file and does not change the knowledge. Apply the batch with the c64-knowledge skill.
+
 ## Result
 
 The differing ranges with the bytes of each release and the groups that agree, and the share of identical bytes. `common` gives the ranges where all releases have bytes. `releases` gives the segments and the `shift` of each release. With `--run`, each difference also has `runtime`, and `run` tells if each release started, or why not.
@@ -99,7 +149,11 @@ The exit status is 0 for a result. The exit status is 1 for an error. The exit s
 - `installation-incomplete`, or `machine-unavailable` with a message that the host runtime does not run: stop the task. Give the message to the user and wait for the user. The message gives the remedy. Do not make a replacement for the tool or the host runtime, for example your own emulator script.
 - `moreDifferences: true`: the script shows only the first differences. Use `--from` and `--to` to see the others.
 - `not-found`: the project has no file at that path.
-- `invalid-input`: a file, an address or a shift is wrong. A path that goes out of the project directory is also wrong, also through a link. Read the message.
+- `not-found` from `match`: the library folder does not exist, or it has no routine files. Save a routine with `routine` first.
+- `conflict` from `routine`: the library has a routine with that name. Give another `--name`, or give `--replace` to save over it.
+- `invalid-input` with "is a project folder": the library folder is a project. Give a folder beside the projects.
+- `match` gives `count: 0`: the routine is not in the program. A packed program can hold the routine in a packed form. Unpack the program with c64-unpacker first.
+- `invalid-input`: a file, an address or a shift is wrong. A path that goes out of the project folder is also wrong, also through a link. Read the message.
 
 ## Handoffs
 
@@ -108,3 +162,4 @@ The exit status is 0 for a result. The exit status is 1 for an error. The exit s
 - Run or examine the different code → c64-emulator
 - Analyze the different code → c64-static-analysis
 - Record a conclusion → c64-knowledge
+- Apply the batch of `match` → c64-knowledge

package/skills/c64-provenance/scripts/{compare.ts → lib/compare.ts}

@@ -1,6 +1,6 @@
 // Byte comparison of releases for the c64-provenance script. Each release
 // has bytes in one or more segments: a PRG at its load address plus an
-// optional shift, or the segments of an unpacker image (D50). The comparison
+// optional shift, or the segments of an unpacker image. The comparison
 // covers the addresses where every release has bytes, and reports where the
 // releases differ and which releases agree there. It never decides which
 // bytes are original: that needs evidence beyond one comparison.
@@ -93,7 +93,7 @@ function addressesOf(release: Release): number[] {
   return release.segments.flatMap((segment) => Array.from({ length: segment.end - segment.start + 1 }, (_, offset) => segment.start + offset));
 }
 
-/** Most shifts that suggestShifts gives: data that did not move can outvote code that did (b2/04). */
+/** Most shifts that suggestShifts gives: data that did not move can outvote code that did. */
 export const MAX_SUGGESTED_SHIFTS = 3;
 
 /**
@@ -129,7 +129,7 @@ export type ShiftSuggestion = { release: number; shift: number; matchingSequence
 
 /**
  * The --shift to suggest for release `index` (0 = the first) from `found`,
- * the best shift of that release over the first one (F-25). The shift is the
+ * the best shift of that release over the first one. The shift is the
  * full value for --shift: `applied` holds the shifts already given. A shift
  * that moves a segment out of $0000-$ffff is not suggested: with two releases,
  * the opposite shift of the first release is suggested when it fits, else the