@henols/c64-re-tools 2.0.0-rc.5 → 2.0.0-rc.7

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@henols/c64-re-tools",
-  "version": "2.0.0-rc.5",
+  "version": "2.0.0-rc.7",
   "description": "Agent-friendly Commodore 64 engineering toolkit for reverse engineering, debugging, development and testing.",
   "license": "MIT",
   "author": "Henrik Olsson",

package/skills/c64-assembler/SKILL.md

@@ -67,7 +67,7 @@ The exit status is 0 for a PRG. The exit status is 1 for `assembled: false`, for
 ## Failure and conflicts
 
 - `assembled: false`: the source has errors. This is a normal result. Correct the source.
-- `installation-incomplete`, or a message that the host runtime does not run: stop the task. Give the message to the user and wait for the user. The message gives the remedy. Do not make a replacement for the tool or the host runtime, for example your own disk parser, disassembler or emulator script.
+- `installation-incomplete`, or `machine-unavailable` with a message that the host runtime does not run: stop the task. Give the message to the user and wait for the user. The message gives the remedy. Do not make a replacement for the tool or the host runtime, for example your own disk parser, disassembler or emulator script.
 - `invalid-input`: an option or a path is wrong. Read the message and correct the call.
 
 ## Handoffs

package/skills/c64-basic/SKILL.md

@@ -56,7 +56,7 @@ The exit status is 0 for a listing. The exit status is 1 for `decoded: false` an
 
 - `decoded: false`: the file is not a BASIC program. This is a normal result. Usually the file is machine code. Give it to the c64-static-analysis skill.
 - An empty `handoffs` list: the BASIC does not start machine code with SYS or USR. Do not make up a handoff. Look for a POKE into a vector or a LOAD of another file in the listing.
-- `installation-incomplete`, or a message that the host runtime does not run: stop the task. Give the message to the user and wait for the user. The message gives the remedy. Do not make a replacement for the tool or the host runtime, for example your own disk parser, disassembler or emulator script.
+- `installation-incomplete`, or `machine-unavailable` with a message that the host runtime does not run: stop the task. Give the message to the user and wait for the user. The message gives the remedy. Do not make a replacement for the tool or the host runtime, for example your own disk parser, disassembler or emulator script.
 - `not-found` or `invalid-input`: the path or an argument is wrong. Read the message and correct the call.
 
 ## Handoffs

package/skills/c64-disk/SKILL.md

@@ -29,7 +29,7 @@ node <skill>/scripts/disk.ts <action> <image> [name] [--out <file>]
 | Need | Command |
 |---|---|
 | The disk name, the free blocks and the files | `directory original/game.d64` |
-| The free and used sectors of each track | `bam original/game.d64` |
+| The free and used sectors of each track, and the used sectors that no file uses | `bam original/game.d64` |
 | The type, size and first block of one file | `entry original/game.d64 GAME` |
 | The track and sector of each block of one file | `chain original/game.d64 GAME` |
 | Extract one file | `read original/game.d64 GAME --out extracted/game.prg` |
@@ -47,7 +47,7 @@ File names show the characters that the C64 shows in a directory list. A byte th
 1. Run `directory`. Find the file in `entries`. Each entry has a `name`, a `type`, a size in `blocks` and the `closed` and `locked` flags.
 2. Run `read` with an output path to extract the file. A PRG file keeps its 2-byte load address at the start.
 3. Run `entry` or `chain` only when the disk structure is important, for example for a loader that reads sectors directly.
-4. Run `bam` only to find used sectors that no file uses.
+4. Run `bam` only to find used sectors that no file uses. `unreferenced` lists them. The list does not include the header, the directory and the BAM blocks. These sectors can hold data that a loader reads directly.
 
 ## Knowledge
 
@@ -79,7 +79,7 @@ The exit status is 0 for a result with the file. The exit status is 1 for `found
 
 - `found: false`: the image has no file with that name. This is a normal result. Read the directory again and compare the names.
 - `media-error`: the image has a defect or is not a disk image. The message tells the problem, for example a sector chain that loops. Tell the user. Do not change the image.
-- `installation-incomplete`, or a message that the host runtime does not run: stop the task. Give the message to the user and wait for the user. The message gives the remedy. Do not make a replacement for the tool or the host runtime, for example your own disk parser, disassembler or emulator script.
+- `installation-incomplete`, or `machine-unavailable` with a message that the host runtime does not run: stop the task. Give the message to the user and wait for the user. The message gives the remedy. Do not make a replacement for the tool or the host runtime, for example your own disk parser, disassembler or emulator script.
 - `invalid-input`: an argument or a path is wrong. Read the message and correct the call.
 
 A file that is not `closed` was not written fully. Its data can be incomplete.

package/skills/c64-disk/scripts/disk.ts

@@ -16,7 +16,8 @@ import { ContainerError, parseCrt, parseT64, tapeFile } from "./containers.ts";
 const USAGE = `disk.ts <action> <image> [name] [--out <file>]
 
   directory <image>                 disk name, id, free blocks and the files
-  bam <image>                       free and used sectors of each track
+  bam <image>                       free and used sectors of each track, and the
+                                    used sectors that no file holds
   entry <image> <name>              the directory entry of one file
   chain <image> <name>              the track and sector of each block of one file
   read <image> <name> --out <file>  write the bytes of one file into the project

package/skills/c64-emulator/SKILL.md

@@ -64,7 +64,9 @@ For a large result, give `out` with a path in the project. `c64_memory_read`, `c
 
 Write operations need a stopped CPU. Stop the CPU with `c64_execution` action `pause` first.
 
-`c64_reset` with `run` false stops the CPU at the first instruction of the reset routine. The result gives that address in `pc`.
+`c64_reset` with `run` false stops the CPU at the first instruction of the reset routine. The result gives that address in `pc`. A reset and `c64_autostart` set `c64_timing` to zero.
+
+A cartridge from `c64_autostart` stays in the C64 through every reset, as on a real C64. `c64_status` gives `cartridge: true` while the C64 has it. `c64_autostart` of a file that is not a cartridge removes the cartridge first.
 
 Step, next and until-return work only in space `c64`. The emulator cannot stop the disk drive CPU after one instruction. To find when the drive executes an address, use `c64_run_until` with an address target in space `drive8`, or a breakpoint in space `drive8`. The drive can execute a few more instructions before it stops. The computer completes its current instruction. The `pc` in the result is the drive address where it stopped.
 

package/skills/c64-knowledge/SKILL.md

@@ -115,7 +115,7 @@ A `line` comment is a line of its own above the address. A `side` comment is at
 
 ## Modules
 
-A module tells the c64-source skill where a file of the generated source starts. The file runs to the next module or to the end of its segment. Each address has one module, and each file name has one module. A file name has letters, digits, underscores or hyphens, and ends in `.asm`. The title becomes the first line of the file. A `module` write without a title keeps the title that the module has. Modules have a history, and `revert` restores them.
+A module tells the c64-source skill where a file of the generated source starts. The file runs to the next module or to the end of its segment. Each address has one module, and each file name has one module. A file name has letters, digits, underscores or hyphens, and ends in `.asm`. The c64-source skill writes `main.asm`, `loader.asm` and `constants.asm` itself. Thus a module cannot have one of these names, in upper or lower case. The title becomes the first line of the file. A `module` write without a title keeps the title that the module has. Modules have a history, and `revert` restores them.
 
 ## Removals
 

package/skills/c64-provenance/SKILL.md

@@ -40,12 +40,14 @@ node <skill>/scripts/provenance.ts separate unpacked/image.json --coverage cover
 
 The script puts each PRG at its load address. An image has bytes only in its segments. The script compares the addresses where all releases have bytes. It prints one JSON object.
 
+With `--shift`, the addresses of `segments`, `common` and `differences` are the release addresses plus the `shift` of the release. The `start` and `end` of `runtime` are the addresses in the release itself, without the shift.
+
 With `--run`, a PRG starts with autostart and RUN. An image starts from its bytes: the script writes the segments and the entry state from `entry_state.json` into the emulator, and starts at the entry. Watchpoints on each difference stop the run at its first access.
 
 ## Workflow
 
 1. Run the script on the releases.
-2. Read `identicalShare`. If it is low and `suggestedShifts` is present, the releases are at different addresses. Run the script again with `--shift`.
+2. Read `identicalShare`. If it is low and `suggestedShifts` is present, the releases are at different addresses. Run the script again with the `--shift` of `suggestedShifts`. `shiftsNotSuggested` tells why a found shift does not fit in `$0000-$ffff`.
 3. Read `differences`. Each one has an address range, the bytes of each release, and `groups`: the releases that have the same bytes there.
 4. To know how the program uses the differences, run the script again with `--run` and the frames that get the program to the part you examine. Read `runtime` of each difference:
    - `first`: the first access in that release: `read`, `write`, `execute` or `none`. An instruction that reads and then writes the same byte, for example INC, counts as `read`. The operand bytes of an executed instruction count as `execute`, for example the value of `LDA #$03`.
@@ -66,9 +68,9 @@ Use `separate` when you have only one release. It gives evidence, not a decision
 - `ranges`: the image in ranges with the same stage and access. `stage` is the last unpack stage that wrote the range. In `image.json`, the last stage is the program. An earlier stage is a depacker, an intro or a loader. Only an image from c64-unpacker `capture` has stages. For an image from `loads`, `noStages` tells this.
 - `access`: from the joined `--coverage` files: `executed`, `read`, `written`, `read-written` or `untouched`. `untouched` bytes did not take part in the runs.
 - `codeRegions.unreachable`: code regions of the knowledge that no chain of call and jump references reaches from the entry. Run c64-static-analysis first, so that the knowledge has the code regions and the references.
-- `outside`: RAM outside the image that the program `read` or `executed` but did not write, and code it `generated`. A loader or an earlier stage put the bytes that the program did not write.
+- `outside`: RAM outside the image that the program `read` or `executed` but did not write, and code it `generated`. A loader or an earlier stage put the bytes that the program did not write. A `generated` range can have gaps of two bytes or less: these are operand bytes.
 
-The result lists the first ranges. `--out` writes all of them. Run more coverage runs for parts that are `untouched` before you decide.
+Each list has `count` and `listed`. `listed` has the first ranges, at most `--max-listed`. `--out` writes all of them. Run more coverage runs for parts that are `untouched` before you decide.
 
 ## Knowledge
 
@@ -79,7 +81,7 @@ Record a confirmed conclusion as a normal comment, region or symbol with the c64
 
 ## Result
 
-The differing ranges with the bytes of each release and the groups that agree, and the share of identical bytes. `common` gives the ranges where all releases have bytes. With `--run`, each difference also has `runtime`, and `run` tells if each release started, or why not.
+The differing ranges with the bytes of each release and the groups that agree, and the share of identical bytes. `common` gives the ranges where all releases have bytes. `releases` gives the segments and the `shift` of each release. With `--run`, each difference also has `runtime`, and `run` tells if each release started, or why not.
 
 The runtime evidence is evidence only. The script does not decide which bytes are original.
 
@@ -91,7 +93,7 @@ The exit status is 0 for a result. The exit status is 1 for an error. The exit s
 - Do not call a byte original only because one release has it. A majority of releases is also not proof.
 - `common` is empty: the releases do not cover the same addresses. Use `--shift`, or compare a range that all releases cover.
 - `run.releases` shows `started: false`: a PRG without a BASIC start, or an image without `entry_state.json`. The reason tells which. The other releases still give evidence.
-- `installation-incomplete`, or a message that the host runtime does not run: stop the task. Give the message to the user and wait for the user. The message gives the remedy. Do not make a replacement for the tool or the host runtime, for example your own emulator script.
+- `installation-incomplete`, or `machine-unavailable` with a message that the host runtime does not run: stop the task. Give the message to the user and wait for the user. The message gives the remedy. Do not make a replacement for the tool or the host runtime, for example your own emulator script.
 - `moreDifferences: true`: the script shows only the first differences. Use `--from` and `--to` to see the others.
 - `not-found`: the project has no file at that path.
 - `invalid-input`: a file, an address or a shift is wrong. A path that goes out of the project directory is also wrong, also through a link. Read the message.

package/skills/c64-provenance/scripts/compare.ts

@@ -121,3 +121,29 @@ export function suggestShift(base: Release, other: Release): { shift: number; ma
   for (const [shift, matches] of votes) if (best === undefined || matches > best.matches) best = { shift, matches };
   return best;
 }
+
+export type ShiftSuggestion = { release: number; shift: number; matchingSequences: number } | { release: number; matchingSequences: number; reason: string };
+
+/**
+ * The --shift to suggest for release `index` (0 = the first) from `found`,
+ * the best shift of that release over the first one (F-25). The shift is the
+ * full value for --shift: `applied` holds the shifts already given. A shift
+ * that moves a segment out of $0000-$ffff is not suggested: with two releases,
+ * the opposite shift of the first release is suggested when it fits, else the
+ * result gives the reason.
+ */
+export function shiftSuggestion(releases: Release[], applied: number[], index: number, found: { shift: number; matches: number }): ShiftSuggestion {
+  const outside = (release: Release, by: number) => release.segments.find((segment) => segment.start + by < 0 || segment.end + by > 0xffff);
+  const hex = (value: number) => `$${value.toString(16).padStart(4, "0")}`;
+  const why = (number: number, by: number, segment: Range) =>
+    `release ${number} moved by ${by} puts ${hex(segment.start)}-${hex(segment.end)} ${segment.start + by < 0 ? "below $0000" : "past $ffff"}`;
+  const own = outside(releases[index]!, found.shift);
+  if (own === undefined) return { release: index + 1, shift: applied[index]! + found.shift, matchingSequences: found.matches };
+  const reasons = [why(index + 1, found.shift, own)];
+  if (releases.length === 2) {
+    const base = outside(releases[0]!, -found.shift);
+    if (base === undefined) return { release: 1, shift: applied[0]! - found.shift, matchingSequences: found.matches };
+    reasons.push(why(1, -found.shift, base));
+  }
+  return { release: index + 1, matchingSequences: found.matches, reason: `No shift fits: ${reasons.join(", and ")}.` };
+}

package/skills/c64-provenance/scripts/provenance.ts

@@ -18,7 +18,7 @@ import { KnowledgeError, openForRead } from "#src/knowledge/database.ts";
 import { listReferences, listRegions } from "#src/knowledge/read.ts";
 import { CODE_REGION_TYPES } from "#src/knowledge/schema.ts";
 import { runScript, UsageError } from "#src/script.ts";
-import { compareReleases, suggestShift, type Range, type Release } from "./compare.ts";
+import { compareReleases, shiftSuggestion, suggestShift, type Range, type Release } from "./compare.ts";
 import { watchRelease, type ImageState, type Start } from "./runtime.ts";
 import { separate } from "./separate.ts";
 
@@ -36,7 +36,7 @@ separate gives evidence from one release: the unpack stage that wrote each range
 its access in the --coverage files (64 KiB each, repeatable), whether call and
 jump references in knowledge reach each code region from the entry, and what the
 program reads or executes outside the image. --out writes all ranges to a file;
-the result lists at most --max-listed of them (default 50).
+the result lists at most --max-listed of each list (default 50) and its count.
 
 A release is a PRG, placed at its load address, or an image.json of the
 c64-unpacker skill, compared inside its segments. Paths are relative to the
@@ -208,12 +208,14 @@ async function run(argv: string[]): Promise<unknown> {
     identicalShare < 0.5
       ? releases.slice(1).flatMap((release, index) => {
           const best = suggestShift(releases[0]!, release);
-          return best === undefined || best.shift === 0 ? [] : [{ release: index + 2, shift: best.shift, matchingSequences: best.matches }];
+          return best === undefined || best.shift === 0 ? [] : [shiftSuggestion(releases, loaded.map((item) => item.shift), index + 1, best)];
         })
       : [];
+  const suggested = suggestions.filter((suggestion) => "shift" in suggestion);
+  const notSuggested = suggestions.filter((suggestion) => "reason" in suggestion);
 
   // The runtime evidence: one emulator for each release, the differences at the release's own addresses.
-  const runtime = new Map<string, Array<{ first: string; changed: boolean }>>();
+  const runtime = new Map<string, Array<Range & { first: string; changed: boolean }>>();
   const runs: Record<string, unknown> = {};
   if (frames !== undefined && comparison.differences.length > 0) {
     for (const item of loaded) {
@@ -229,7 +231,7 @@ async function run(argv: string[]): Promise<unknown> {
         if (!watched.started) runs[name] = { started: false, reason: watched.reason };
         else {
           runs[name] = { started: true };
-          runtime.set(name, watched.accesses);
+          runtime.set(name, watched.accesses.map((access, index) => ({ ...ranges[index]!, ...access })));
         }
       } finally {
         await client.close();
@@ -238,7 +240,7 @@ async function run(argv: string[]): Promise<unknown> {
   }
 
   return {
-    releases: loaded.map((item) => ({ name: item.release.name, segments: item.release.segments.map((segment) => ({ start: formatC64Address(segment.start), end: formatC64Address(segment.end) })) })),
+    releases: loaded.map((item) => ({ name: item.release.name, shift: item.shift, segments: item.release.segments.map((segment) => ({ start: formatC64Address(segment.start), end: formatC64Address(segment.end) })) })),
     common: comparison.common.map((range) => ({ start: formatC64Address(range.start), end: formatC64Address(range.end) })),
     identicalBytes: comparison.identicalBytes,
     differingBytes: comparison.differingBytes,
@@ -248,10 +250,21 @@ async function run(argv: string[]): Promise<unknown> {
       end: formatC64Address(difference.end),
       values: Object.fromEntries(Object.entries(difference.values).map(([name, bytes]) => [name, hexBytes(bytes.slice(0, 32)) + (bytes.length > 32 ? " ..." : "")])),
       groups: difference.groups,
-      ...(runtime.size === 0 ? {} : { runtime: Object.fromEntries([...runtime].map(([name, accesses]) => [name, accesses[index]!])) }),
+      // runtime gives the release's own addresses: with a shift, they are not the addresses of the difference (F-24).
+      ...(runtime.size === 0
+        ? {}
+        : {
+            runtime: Object.fromEntries(
+              [...runtime].map(([name, accesses]) => {
+                const { start, end, first, changed } = accesses[index]!;
+                return [name, { start: formatC64Address(start), end: formatC64Address(end), first, changed }];
+              }),
+            ),
+          }),
     })),
     ...(comparison.moreDifferences ? { moreDifferences: true } : {}),
-    ...(suggestions.length > 0 ? { suggestedShifts: suggestions } : {}),
+    ...(suggested.length > 0 ? { suggestedShifts: suggested } : {}),
+    ...(notSuggested.length > 0 ? { shiftsNotSuggested: notSuggested } : {}),
     ...(frames === undefined
       ? {}
       : { run: { frames, steps: steps.length, releases: runs, ...(comparison.differences.length === 0 ? { skipped: "The releases have no differences, so no release ran." } : {}) } }),
@@ -288,6 +301,7 @@ function separateCommand(path: string, coveragePaths: string[], out: string | un
   const ranges = found.ranges.map((range) => ({ ...show(range), ...(range.stage === undefined ? {} : { stage: range.stage }), ...(range.access === undefined ? {} : { access: range.access }) }));
   const codeRegions = found.codeRegions.map((region) => ({ ...show(region), type: region.type, ...(region.reachable === undefined ? {} : { reachable: region.reachable }), ...(region.executed === undefined ? {} : { executed: region.executed }) }));
   const outside = { read: found.outside.read.map(show), executed: found.outside.executed.map(show), generated: found.outside.generated.map(show) };
+  const listed = <T>(items: T[]) => ({ count: items.length, listed: items.slice(0, Number(max)) });
   if (out !== undefined) writeProjectFile(out, Buffer.from(`${JSON.stringify({ release: path, ranges, codeRegions, outside }, null, 1)}\n`), { what: "--out" });
   const bytes = (filter: (range: (typeof found.ranges)[number]) => boolean) => found.ranges.filter(filter).reduce((sum, range) => sum + range.end - range.start + 1, 0);
   const lastStage = loaded.stages === undefined ? undefined : loaded.stages.length - 1;
@@ -301,12 +315,12 @@ function separateCommand(path: string, coveragePaths: string[], out: string | un
           bytesByStage: Object.fromEntries([...[...Array(lastStage + 1).keys()].map((stage): [string, number] => [String(stage), bytes((range) => range.stage === stage)]), ["none", bytes((range) => range.stage === undefined)]]),
         }),
     ...(coverage === undefined ? {} : { bytesByAccess: Object.fromEntries((["executed", "read", "written", "read-written", "untouched"] as const).map((access) => [access, bytes((range) => range.access === access)])) }),
-    ranges: { count: ranges.length, listed: ranges.slice(0, Number(max)) },
+    ranges: listed(ranges),
     codeRegions: {
       count: codeRegions.length,
       ...(loaded.entry === undefined ? {} : { unreachable: codeRegions.filter((region) => region.reachable === false) }),
     },
-    ...(coverage === undefined ? {} : { outside }),
+    ...(coverage === undefined ? {} : { outside: { read: listed(outside.read), executed: listed(outside.executed), generated: listed(outside.generated) } }),
     ...(out === undefined ? {} : { out }),
   };
 }

package/skills/c64-provenance/scripts/separate.ts

@@ -42,18 +42,20 @@ export interface Separation {
   /**
    * RAM outside the image and outside pages 0-3, from the coverage: `read`
    * and `executed` the program never wrote (a loader or an earlier stage put
-   * those bytes there), and `generated` it wrote and then executed.
+   * those bytes there), and `generated` it wrote and then executed, in
+   * pieces of code with gaps of up to two operand bytes.
    */
   outside: { read: Range[]; executed: Range[]; generated: Range[] };
 }
 
 const SYSTEM_PAGES_END = 0x03ff;
 
-function merge(addresses: number[]): Range[] {
+/** Joins addresses in order into ranges; `gap` is the most bytes between two addresses in one range. */
+function merge(addresses: number[], gap = 0, joins: (from: number, to: number) => boolean = () => true): Range[] {
   const ranges: Range[] = [];
   for (const address of addresses) {
     const last = ranges.at(-1);
-    if (last !== undefined && last.end + 1 === address) last.end = address;
+    if (last !== undefined && address - last.end - 1 <= gap && joins(last.end, address)) last.end = address;
     else ranges.push({ start: address, end: address });
   }
   return ranges;
@@ -128,5 +130,11 @@ export function separate(input: SeparationInput): Separation {
       else if (!written && (input.coverage[address]! & COVERAGE.ramRead) !== 0) read.push(address);
     }
   }
-  return { ranges, codeRegions, outside: { read: merge(read), executed: merge(ranOutside), generated: merge(generated) } };
+  // The image has no bytes of generated code, so only its opcode bytes count as executed. VICE marks only
+  // opcode bytes: a gap of at most two operand bytes outside the image stays inside one piece of code (F-28).
+  const outsideImage = (from: number, to: number) => {
+    for (let address = from + 1; address < to; address++) if (inImage(address)) return false;
+    return true;
+  };
+  return { ranges, codeRegions, outside: { read: merge(read), executed: merge(ranOutside), generated: merge(generated, 2, outsideImage) } };
 }

package/skills/c64-reverse-engineering/SKILL.md

@@ -92,7 +92,7 @@ In stage 5, if you cannot get to a part of the program, ask the user how to get
 1. Acceptance: each criterion of stage 1 and its result, and each exception that the user accepted.
 2. The program: what the program does, in a few sentences.
 3. Modules: the file, the address range and the title of each module.
-4. Untested code: the `notExecuted` ranges from c64-source. Tell why no run executed them, if you know.
+4. Untested code: all ranges in the `not_executed.json` file of c64-source. Tell why no run executed them, if you know.
 5. Open questions: what the evidence does not explain.
 
 Keep observed facts apart from your conclusions.
@@ -133,7 +133,7 @@ A clear answer for the scope, with the durable conclusions in the project knowle
 
 ## Failure and conflicts
 
-- A skill reports `installation-incomplete`, or a message that the host runtime does not run: stop the whole task. Give the message to the user and wait for the user. Do not continue with other skills. Do not make a replacement for the tool or the host runtime, for example your own disk parser, disassembler or emulator script.
+- A skill reports `installation-incomplete`, or `machine-unavailable` with a message that the host runtime does not run: stop the whole task. Give the message to the user and wait for the user. Do not continue with other skills. Do not make a replacement for the tool or the host runtime, for example your own disk parser, disassembler or emulator script.
 - If the evidence does not support a conclusion, keep the unknown and get more evidence. Do not make up a meaning.
 - Before you replace current knowledge from the user or the LLM, read its history and the evidence for it.
 - A conflict from the static analysis: find the correct answer at run time with the c64-emulator skill.

package/skills/c64-source/SKILL.md

@@ -32,7 +32,7 @@ node <skill>/scripts/generate.ts unpacked/image.json --out src --loader
 |---|---|
 | `--out src` | The directory for the source files. The script writes over the files that it generated before. |
 | `--entry '$080d'` | An entry point for Ghidra. Use it again for more entry points. The entry of `image.json`, the routines and the code regions in the knowledge are entry points too. |
-| `--coverage coverage/attract.bin` | A coverage file of 64 KiB from `c64_memmap` with `out`. Use it again for more runs. Code that no run executed gets the line `; not executed`, and `notExecuted` lists it. |
+| `--coverage coverage/attract.bin` | A coverage file of 64 KiB from `c64_memmap` with `out`. Use it again for more runs. Code that no run executed gets the line `; not executed` with its range. `notExecuted` lists it. |
 | `--space drive8` | Generate the source of drive code or a cartridge bank from the knowledge of that space. `--coverage` and `--loader` are only for `c64`, and platform names are only for `c64`. |
 | `--loader` | Also write `loader.asm` and `loader.prg`. This needs `image.json` with an entry, and `entry_state.json`. |
 
@@ -56,10 +56,11 @@ node <skill>/scripts/generate.ts unpacked/image.json --out src --loader
 - Code regions, and the instructions that Ghidra decodes outside of data regions, become instructions.
 - Data regions become data by their type: `!byte`, `!word`, `!pet`, `!scr`, `!fill`, tables of labels, and sprite bytes in binary.
 - `lda+2 $0012` is an absolute operand below $0100. `+1` is a zero-page operand with a label.
+- `sta NMINV+1` is an address inside a platform entry of more than one byte, for example the high byte of a vector.
 - `name = * + 1` is a label inside an instruction, for example for code that changes its own operand.
 - `!byte` with an instruction as its comment: ACME writes this instruction with a different opcode. The bytes keep the source identical.
 - `not decoded` marks bytes in a code region that Ghidra did not decode.
-- `; not executed` starts code that no run of the `--coverage` files executed.
+- `; not executed $xxxx-$yyyy` starts code that no run of the `--coverage` files executed. The range is the same as in `not_executed.json`.
 
 Do not edit the generated files. The next run writes over them.
 
@@ -70,11 +71,12 @@ Do not edit the generated files. The next run writes over them.
 1. `10 SYS 2061` starts it.
 2. It moves its copier to two free pages. The copier runs with interrupts off and with all RAM visible.
 3. It copies each segment and the pages $0002-$03ff of the entry state into place. The script selects an order in which no copy writes over the packed bytes of a copy that has not run yet.
-4. It sets $00/$01, the stack pointer, A, X, Y and the flags, and jumps to the entry.
+4. It sets the display registers of the VIC-II and the VIC bank from `io` in the entry state. Then it waits for the raster line of the entry.
+5. It sets $00/$01, the stack pointer, A, X, Y and the flags, and jumps to the entry.
 
-The flags go through the free stack byte below the stack pointer (`stackByte`). The loader changes this byte. The loader does not set the VIC, SID and CIA registers. They stay as the KERNAL set them.
+The flags go through the free stack byte below the stack pointer (`stackByte`). The loader changes this byte. The loader does not set the SID, the CIA timers or the interrupt registers of the VIC-II. They stay as the KERNAL set them. Thus a scenario can still find a difference in an interrupt or a timer. Then use `runUntil` with `raster` in the c64-testing skill.
 
-The script simulates the copies first. Then it runs `loader.prg` in the emulator to its jump to the entry. It compares the memory, $00/$01 and the registers with the entry state there. `loader.verified: true` tells that all of them are equal. An interrupt can run right at the entry, as in the original program.
+The script simulates the copies first. Then it runs `loader.prg` in the emulator to its jump to the entry. It compares the memory, $00/$01, the registers, the display registers of the VIC-II, the VIC bank and the raster line with the entry state there. `loader.verified: true` tells that all of them are equal. An interrupt can run right at the entry, as in the original program.
 
 ## Knowledge
 
@@ -94,22 +96,26 @@ One JSON object:
 - `files`: the files in the output directory.
 - `instructions`, `dataBytes` and `undecodedBytes`.
 - `labels`: `named` labels from the knowledge and `generated` labels `l_xxxx`.
+- `entriesInData`: each `--entry` in a data region of the knowledge, with the `region` and its `origin`. Ghidra does not decode an entry in a data region.
 - `asBytes`: the instructions that the source has as bytes.
-- `unusedModules`: modules that start outside the image.
-- `notExecuted`: with `--coverage`, the count of instructions that no run executed, and their ranges. Put these ranges in the report.
+- `unusedModules`: modules in the knowledge whose start is in no segment of the image. Often this tells that the image does not hold code that the knowledge describes.
+- `entryOutside`: the entry of `image.json` is in no segment of the image, with a `reason`. The image does not hold the program.
+- `notExecuted`: with `--coverage`, the count of instructions that no run executed, the count of their ranges, and the first ranges. `all` names the file `not_executed.json` in `--out`. It has all of the ranges. Put all of these ranges in the report.
 - `check`: `identical`, and the result for each segment. A different segment gives its first different address.
 - `loader`: the program, the copies in their order, the copier page, `stackByte`, `verified` and `differences`. When the script cannot make the loader, `loader` has `verified: false` and a `reason`. The loader includes the source. Thus a source that does not assemble to the image also stops the loader.
 
-The exit status is 0 when the source is identical and the emulator check of the loader passes. The exit status is 1 for a failed check and for an error. The exit status is 2 for a wrong argument.
+The exit status is 0 when the source is identical and the emulator check of the loader passes. The exit status is 1 for a failed check, for an entry in no segment, and for an error. The exit status is 2 for a wrong argument.
 
 ## Failure and conflicts
 
 - `check.diagnostics`: ACME did not assemble the source. A name in the knowledge can be wrong for ACME. Read the diagnostics and change the name with c64-knowledge.
 - `check.identical: false`: the script has a defect. Give the result to the user.
 - `invalid-input` that names a symbol or a module: change that symbol or module in the knowledge.
+- `entryOutside`: the run fails, but the script still writes and checks the source. With `--loader`, `loader.verified` is `false` with the same `reason`. Capture the image again with c64-unpacker `loads` or `capture`. Make sure that a segment holds the entry.
+- `entriesInData`: an analyzer region of data hides the code at the entry. If the entry is code, record a code region at the entry with the c64-knowledge skill, for example `classify '$0500' '$05ff' code`. A region of the user or the LLM replaces the analyzer region. Then generate again.
 - `loader.verified: false` with a `reason`: the script did not make the loader. The script still writes and checks the source. When no copy order or no free pages exist, or the loader needs too many copies, the reason gives the ranges. Capture the image again with fewer segments, or write the loader by hand.
 - `loader.verified: false` with `differences`: the loader ran, and `differences` gives the bytes and registers that are not equal.
-- `installation-incomplete`, or a message that the host runtime does not run: stop the task. Give the message to the user and wait for the user. The message gives the remedy. Do not make a replacement for the tool or the host runtime, for example your own disassembler or emulator script.
+- `installation-incomplete`, or `machine-unavailable` with a message that the host runtime does not run: stop the task. Give the message to the user and wait for the user. The message gives the remedy. Do not make a replacement for the tool or the host runtime, for example your own disassembler or emulator script.
 
 ## Handoffs
 

package/skills/c64-source/scripts/generate.ts

@@ -25,9 +25,9 @@ import { GHIDRA_LIMITS, type AssembledSymbol, type Diagnostic } from "#src/nativ
 import { platformNames } from "#src/platform.ts";
 import { resolveProjectPath } from "#src/project.ts";
 import { runScript, UsageError } from "#src/script.ts";
-import { LOADER_JUMP, planLoader, readEntryState, simulate, type EntryState, type LoaderPlan } from "./loader.ts";
+import { LOADER_JUMP, planLoader, rasterLine, readEntryState, simulate, VIC_DISPLAY_REGISTERS, type EntryState, type LoaderPlan } from "./loader.ts";
 import { OPCODES } from "./opcodes.ts";
-import { instructionText, layout, literalOperand, probeForms, render, type Layout, type Range } from "./source.ts";
+import { entriesInData, entryOutsideImage, instructionText, layout, literalOperand, probeForms, render, type Layout, type Range } from "./source.ts";
 
 const USAGE = `generate.ts <image> --out <dir> [options]
 
@@ -36,7 +36,8 @@ const USAGE = `generate.ts <image> --out <dir> [options]
   --out <dir>        the directory for the source files, relative to the project
   --entry <address>  an entry point for Ghidra, for example '$080d' (repeatable)
   --coverage <file>  a coverage map of 64 KiB from c64_memmap (repeatable): code
-                     that no run executed is marked and listed
+                     that no run executed is marked and listed; <dir>/not_executed.json
+                     has all of its ranges
   --space <space>    the knowledge space of the image: c64 (the default), drive8, or
                      bank0 to bank255; --coverage and --loader are for c64 only
   --loader           also write loader.asm and loader.prg: the loader copies the
@@ -276,6 +277,15 @@ async function runLoader(path: string, program: Buffer, plan: LoaderPlan, image:
     differ("sp", registers.sp, state.sp);
     const flags = (["n", "v", "d", "i", "z", "c"] as const).reduce((value, name, index) => value | (registers.flags[name] ? [0x80, 0x40, 0x08, 0x04, 0x02, 0x01][index]! : 0), 0);
     differ("flags", flags, state.flags & 0xcf);
+    if (state.vic !== undefined) {
+      // The display registers and the raster line (D60): the loader waited for the line, so the jump comes in it or the next one.
+      const vic = Buffer.from((await client.ioRead(0xd000, 0x2f)).data, "hex");
+      for (const { register, mask } of VIC_DISPLAY_REGISTERS) differ(formatC64Address(0xd000 + register), vic[register]! & mask, state.vic.registers[register]! & mask);
+      differ("the VIC bank", Buffer.from((await client.ioRead(0xdd00, 1)).data, "hex")[0]! & 0x03, state.vic.bank);
+      const line = rasterLine(vic);
+      const wanted = rasterLine(state.vic.registers);
+      if (line !== wanted && line !== wanted + 1) differences.push(`the raster line is ${line}, not ${wanted}`);
+    }
     return { verified: differences.length === 0, differences: differences.slice(0, MAX_LISTED) };
   } finally {
     await client.close();
@@ -284,8 +294,10 @@ async function runLoader(path: string, program: Buffer, plan: LoaderPlan, image:
 
 /** The first words of every file that the script writes. */
 const GENERATED_MARK = "; Generated by c64-source";
+/** The file with every range of instructions that no run executed (D49). */
+const NOT_EXECUTED_FILE = "not_executed.json";
 
-/** Removes the .asm files of an earlier run that this run does not write, and an earlier loader.prg; other files stay. */
+/** Removes the .asm files of an earlier run that this run does not write, and an earlier loader.prg and not_executed.json; other files stay. */
 function removeOldFiles(out: string, keep: ReadonlySet<string>): void {
   const directory = resolveProjectPath(out);
   if (!existsSync(directory)) return;
@@ -293,7 +305,7 @@ function removeOldFiles(out: string, keep: ReadonlySet<string>): void {
     if (!entry.isFile()) continue;
     const name = entry.name;
     const path = join(directory, name);
-    if (name === "loader.prg") rmSync(path, { force: true });
+    if (name === "loader.prg" || name === NOT_EXECUTED_FILE) rmSync(path, { force: true });
     else if (name.endsWith(".asm") && !keep.has(name) && readFileSync(path, "utf8").startsWith(GENERATED_MARK)) rmSync(path, { force: true });
   }
 }
@@ -344,16 +356,18 @@ async function run(argv: string[]): Promise<{ output: Record<string, unknown>; f
     referenceTargets: knowledge.references.map((reference) => reference.to),
     ...(coverage === undefined ? {} : { coverage }),
     // The platform names are C64 names: drive code and banks get none.
-    ...(space === DEFAULT_SPACE ? { platform: new Map(platformNames().map((item) => [item.address, item.name])) } : {}),
+    ...(space === DEFAULT_SPACE ? { platform: platformNames() } : {}),
   });
   const asBytes = await probe(laid);
   const header = [`Generated by c64-source from ${image.path}${space === DEFAULT_SPACE ? "" : ` in space ${space}`} and knowledge revision ${knowledge.revision}.`, "Change names, comments, regions and modules in knowledge, then generate again."];
   const rendered = render(laid, asBytes, header);
   const files = [...rendered.files];
   let loader: LoaderPlan | undefined;
+  // An image without its entry has no program: the source is still written and checked, but the run fails (R2-02).
+  const missingEntry = entryOutsideImage(image.entry, image.segments);
   // A loader that cannot be planned does not stop the source: the source is written and checked, and the result says why.
-  let loaderRefused: string | undefined;
-  if (state !== undefined) {
+  let loaderRefused: string | undefined = state === undefined ? undefined : missingEntry;
+  if (state !== undefined && missingEntry === undefined) {
     const segments = image.segments.map((segment) => ({ ...segment, files: laid.pieces.filter((piece) => piece.start >= segment.start && piece.start <= segment.end).map((piece) => piece.file) }));
     try {
       loader = planLoader({ segments, byteAt: image.byteAt, state, entry: image.entry!, header: [`Loader generated by c64-source from ${image.path}.`, "It copies the segments into place, sets the entry state and jumps to the entry."] });
@@ -367,6 +381,12 @@ async function run(argv: string[]): Promise<{ output: Record<string, unknown>; f
   for (const [index, file] of files.entries()) if (!file.text.startsWith(GENERATED_MARK)) files[index] = { ...file, text: `${GENERATED_MARK}. Do not edit: change the knowledge and generate again.\n${file.text}` };
   removeOldFiles(values.out, new Set(files.map((file) => file.name)));
   for (const file of files) writeProjectFile(`${values.out}/${file.name}`, Buffer.from(file.text, "utf8"), { what: "--out" });
+  const showRange = (range: Range) => ({ start: formatC64Address(range.start), end: formatC64Address(range.end) });
+  const inData = entriesInData(entries, knowledge.regions);
+  if (rendered.notExecuted !== undefined) {
+    const all = { image: image.path, revision: knowledge.revision, instructions: rendered.notExecuted.instructions, ranges: rendered.notExecuted.ranges.map(showRange) };
+    writeProjectFile(`${values.out}/${NOT_EXECUTED_FILE}`, Buffer.from(`${JSON.stringify(all, null, 2)}\n`, "utf8"), { what: "--out" });
+  }
 
   const output: Record<string, unknown> = {
     out: values.out,
@@ -375,7 +395,12 @@ async function run(argv: string[]): Promise<{ output: Record<string, unknown>; f
     instructions: rendered.instructions,
     dataBytes: rendered.dataBytes,
     ...(rendered.undecodedBytes === 0 ? {} : { undecodedBytes: rendered.undecodedBytes }),
+    ...(missingEntry === undefined ? {} : { entryOutside: { entry: formatC64Address(image.entry!), reason: missingEntry } }),
     labels: { named: laid.namedLabels, generated: laid.generatedLabels },
+    // An --entry in a data region decodes nothing: the result says so, and a code region in the knowledge changes it (F-27).
+    ...(inData.length === 0
+      ? {}
+      : { entriesInData: inData.map(({ entry, region }) => ({ entry: formatC64Address(entry), region: { ...showRange(region), type: region.type }, origin: region.origin })) }),
     ...(rendered.asBytes.length === 0
       ? {}
       : { asBytes: { count: rendered.asBytes.length, listed: rendered.asBytes.slice(0, MAX_LISTED).map((item) => ({ address: formatC64Address(item.address), instruction: item.text })) } }),
@@ -385,7 +410,8 @@ async function run(argv: string[]): Promise<{ output: Record<string, unknown>; f
           notExecuted: {
             instructions: rendered.notExecuted.instructions,
             ranges: rendered.notExecuted.ranges.length,
-            listed: rendered.notExecuted.ranges.slice(0, MAX_LISTED).map((range) => ({ start: formatC64Address(range.start), end: formatC64Address(range.end) })),
+            listed: rendered.notExecuted.ranges.slice(0, MAX_LISTED).map(showRange),
+            all: `${values.out}/${NOT_EXECUTED_FILE}`,
           },
         }),
     ...(laid.unusedModules.length === 0 ? {} : { unusedModules: laid.unusedModules.map((module) => ({ start: formatC64Address(module.start), file: module.file })) }),
@@ -400,7 +426,7 @@ async function run(argv: string[]): Promise<{ output: Record<string, unknown>; f
   output.check = checked;
   if (!checked.identical) return { output: { ...output, ...notBuilt("The source does not assemble to the image, and the loader includes it.") }, failed: true };
   if (loaderRefused !== undefined) return { output: { ...output, loader: { verified: false, reason: loaderRefused } }, failed: true };
-  if (loader === undefined || state === undefined) return { output, failed: false };
+  if (loader === undefined || state === undefined) return { output, failed: missingEntry !== undefined };
 
   const loaderBuilt = await assembleFiles(files, "loader.asm");
   if (loaderBuilt.program === undefined) return { output: { ...output, loader: { verified: false, reason: "loader.asm does not assemble.", diagnostics: loaderBuilt.diagnostics.slice(0, MAX_LISTED) } }, failed: true };