@henols/c64-re-tools 2.0.0-rc.3 → 2.0.0-rc.4

package/skills/c64-static-analysis/scripts/analyze.ts

@@ -7,18 +7,23 @@ import { mkdirSync, writeFileSync } from "node:fs";
 import { dirname } from "node:path";
 import { parseArgs } from "node:util";
 
-import { formatC64Address, parseC64Address } from "#src/c64.ts";
+import { formatC64Address, parseC64Address, prgRange } from "#src/c64.ts";
 import { WireFailure } from "#src/host-client/tools.ts";
-import { readProjectFile } from "#src/host-client/transfer.ts";
+import { readProjectFile, writeProjectFile } from "#src/host-client/transfer.ts";
 import { analyze as analyzeWithDxa } from "#src/native/dxa.ts";
 import { analyze as analyzeWithGhidra } from "#src/native/ghidra/analyze.ts";
 import { localToolContext } from "#src/native/local.ts";
+import { GHIDRA_LIMITS } from "#src/native/types.ts";
 import { KnowledgeError, openForRead, withWrite } from "#src/knowledge/database.ts";
 import { importFindings, runProvenance, type Analyzer, type ImportConflict, type ImportContext, type ImportResult, type NormalizedFindings } from "#src/knowledge/import.ts";
 import { currentRevision } from "#src/knowledge/read.ts";
+import { DEFAULT_SPACE, SPACE } from "#src/knowledge/schema.ts";
 import { resolveProjectPath } from "#src/project.ts";
 import { runScript, UsageError } from "#src/script.ts";
+import { codeEntries, coverageConflicts, coverageEntries, imageReader, readCoverageFiles } from "./coverage.ts";
 import { dxaFindings, ghidraFindings, seedsFromKnowledge } from "./findings.ts";
+import { findIdioms } from "./idioms.ts";
+import { instructionList } from "./instructions.ts";
 
 const USAGE = `analyze.ts <image> [options]
 
@@ -28,12 +33,25 @@ const USAGE = `analyze.ts <image> [options]
   --entry <address>       an entry point, for example '$080d' (repeatable)
   --decompile <address>   ghidra: a routine to decompile, at most 32 (repeatable)
   --listing <file>        dxa: write the disassembly listing to this project file
+  --instructions <file>   ghidra: write every instruction (address, bytes, mnemonic,
+                          mode, operand, target, executed) to this project file as JSON
+  --coverage <file>       a coverage map of 64 KiB from c64_memmap (repeatable): its
+                          executed code is seeded, and analyzer data over it is reported
+  --max-listed <n>        list at most n labels or functions (default 50, 0 for none);
+                          the counts are always given, and knowledge has all of them
+  --space <space>         the knowledge space of the image: c64 (the default), drive8
+                          for 1541 drive code, or bank0 to bank255 for a cartridge bank
 
 Routine symbols in knowledge are entry points too. Ghidra needs at least one
 entry point; dxa finds the SYS of a BASIC start by itself. The result is one
 JSON object. Addresses are $ and four hex digits.`;
 
-const MAX_LISTED_FUNCTIONS = 200;
+const DEFAULT_MAX_LISTED = 50;
+
+/** The first `max` items, the count, and where to read the rest when the list is cut. */
+function listed<T, U>(items: T[], max: number, show: (item: T) => U, all: string) {
+  return { count: items.length, listed: items.slice(0, max).map(show), ...(items.length > max ? { rest: all } : {}) };
+}
 
 /** A decompiler that did not start gets the remedy: Ghidra ships no native decompiler for macOS. */
 function withRemedy(reason: string): string {
@@ -103,6 +121,10 @@ async function run(argv: string[]): Promise<unknown> {
       entry: { type: "string", multiple: true },
       decompile: { type: "string", multiple: true },
       listing: { type: "string" },
+      "max-listed": { type: "string" },
+      coverage: { type: "string", multiple: true },
+      instructions: { type: "string" },
+      space: { type: "string" },
       help: { type: "boolean" },
     },
   });
@@ -112,10 +134,14 @@ async function run(argv: string[]): Promise<unknown> {
   const entries = addresses(values.entry, "--entry");
   const decompile = addresses(values.decompile, "--decompile");
   if (decompile.length > 32) throw new UsageError("--decompile takes at most 32 routines");
+  const maxListed = values["max-listed"] ?? String(DEFAULT_MAX_LISTED);
+  if (!/^\d{1,5}$/.test(maxListed)) throw new UsageError(`--max-listed must be a whole number, not ${maxListed}`);
+  const max = Number(maxListed);
   const analyzer = values.analyzer ?? "ghidra";
   if (analyzer !== "dxa" && analyzer !== "ghidra") throw new UsageError(`--analyzer must be dxa or ghidra, not ${analyzer}`);
   if (analyzer === "dxa" && decompile.length > 0) throw new UsageError("--decompile works only with --analyzer ghidra");
   if (analyzer === "ghidra" && values.listing !== undefined) throw new UsageError("--listing works only with --analyzer dxa");
+  if (analyzer === "dxa" && values.instructions !== undefined) throw new UsageError("--instructions works only with --analyzer ghidra");
   let listingPath: string | undefined;
   if (values.listing !== undefined) {
     try {
@@ -125,18 +151,68 @@ async function run(argv: string[]): Promise<unknown> {
     }
   }
 
+  const space = values.space ?? DEFAULT_SPACE;
+  if (!SPACE.test(space)) throw new UsageError("--space must be c64, drive8, or bank0 to bank255");
+  // A coverage map is of the C64 CPU: VICE keeps none for the drive (D53).
+  if (space !== DEFAULT_SPACE && values.coverage !== undefined) throw new UsageError("--coverage works only in space c64");
+  const imageKind = values.flat64k ? "flat64k" : "prg";
+  const bytes = readProjectFile(image).bytes;
+  const coverage = readCoverageFiles(values.coverage ?? []);
+  const byteAt = imageReader(bytes, imageKind);
+
+  let loaded: { start: number; end: number };
+  try {
+    loaded = imageKind === "flat64k" ? { start: 0x0000, end: bytes.length - 1 } : prgRange(bytes);
+  } catch (error) {
+    throw new UsageError(`${image}: ${(error as Error).message}`);
+  }
+  const idioms = findIdioms(byteAt, { start: loaded.start, end: loaded.end }, coverage);
+
   const reader = openForRead();
   const revision = currentRevision(reader);
-  const seeds = seedsFromKnowledge(reader, entries);
+  const seeds = seedsFromKnowledge(reader, entries, space);
+  // Knowledge routines and --entry first, then code regions and JMP tables, then idioms, then coverage, up to the seed bound (D44).
+  const fromCoverage = coverage === undefined ? [] : coverageEntries(coverage, byteAt);
+  const ordered = [...new Set([...seeds.entryPoints, ...codeEntries(reader, byteAt, space), ...idioms.entryPoints, ...fromCoverage])];
+  seeds.entryPoints = ordered.slice(0, GHIDRA_LIMITS.entryPoints).sort((a, b) => a - b);
+  // Idiom tables are data for this run too; semantic data ranges come first.
+  seeds.dataRanges = [...seeds.dataRanges, ...idioms.dataRanges.filter((range) => !seeds.dataRanges.some((seed) => range.start <= seed.end && range.end >= seed.start))].slice(0, GHIDRA_LIMITS.dataRanges);
+  const coverageUsed = fromCoverage.filter((entry) => seeds.entryPoints.includes(entry)).length;
   reader?.close();
-
-  const imageKind = values.flat64k ? "flat64k" : "prg";
-  const bytes = readProjectFile(image).bytes;
+  /** Imports the idioms as their own revision, after the analyzer's, and gives their part of the result. */
+  const idiomsResult = () => {
+    const imported = withWrite((db) => importFindings(db, { ...idioms.findings, space }, { description: `Idioms in ${image}` }));
+    return {
+      idioms: {
+        ...idioms.counts,
+        revision: imported.revision,
+        changes: { regions: imported.regions, references: imported.references },
+        conflicts: imported.conflicts.map((conflict) => showConflict(conflict, "heuristic")),
+        candidates: { dop: idioms.candidates.dop.map(formatC64Address), selfModified: idioms.candidates.selfModified.map(formatC64Address) },
+      },
+    };
+  };
+  /** The coverage part of a result: its seeds, and the analyzer data that it shows to be code. */
+  const coverageResult = () => {
+    if (coverage === undefined) return {};
+    const db = openForRead();
+    const conflicts = coverageConflicts(db, coverage);
+    db?.close();
+    return {
+      coverageSeeds: { entryPoints: fromCoverage.length, used: coverageUsed },
+      coverageConflicts: conflicts.map(({ region, executed }) => ({
+        range: { start: formatC64Address(region.start), end: formatC64Address(region.end) },
+        knowledge: { type: region.type, origin: region.origin },
+        executed,
+        problem: "coverage shows code",
+      })),
+    };
+  };
   if (analyzer === "dxa") {
     const { result, attachments } = await analyzeWithDxa({ imageKind, ...seeds }, bytes, localToolContext());
     const listing = attachments[0]!.toString("utf8");
     // The listing is written only after the import: a refused import leaves no new file.
-    const imported = record(dxaFindings(result), revision, `DXA analysis of ${image}`, runProvenance(bytes, result.toolVersion));
+    const imported = record({ ...dxaFindings(result), space }, revision, `DXA analysis of ${image}`, runProvenance(bytes, result.toolVersion));
     if (listingPath !== undefined) {
       try {
         mkdirSync(dirname(listingPath), { recursive: true });
@@ -146,25 +222,38 @@ async function run(argv: string[]): Promise<unknown> {
         throw new WireFailure("operation-failed", `The script did not write the listing to ${values.listing}: ${(error as Error).message}. ${recorded}`);
       }
     }
+    const found = idiomsResult();
     return {
       analyzer: "dxa",
+      ...(space === DEFAULT_SPACE ? {} : { space }),
+      ...found,
+      ...coverageResult(),
       coverage: result.coverage.map((range) => ({ start: formatC64Address(range.start), end: formatC64Address(range.end) })),
       ...imported,
       regions: result.regions.map((region) => ({ start: formatC64Address(region.start), end: formatC64Address(region.end), classification: region.classification })),
-      labels: result.labels.slice(0, MAX_LISTED_FUNCTIONS).map((label) => ({ address: formatC64Address(label.address), name: label.name })),
-      ...(result.labels.length > MAX_LISTED_FUNCTIONS ? { moreLabels: result.labels.length - MAX_LISTED_FUNCTIONS } : {}),
+      labels: listed(result.labels, max, (label) => ({ address: formatC64Address(label.address), name: label.name }), "c64-knowledge: knowledge.ts symbols"),
       ...(listingPath === undefined ? {} : { listing: values.listing }),
     };
   }
 
   const { result } = await analyzeWithGhidra({ imageKind, ...seeds, decompile }, bytes, localToolContext());
-  const imported = record(ghidraFindings(result), revision, `Ghidra analysis of ${image}`, runProvenance(bytes, result.toolVersion));
+  const imported = record({ ...ghidraFindings(result), space }, revision, `Ghidra analysis of ${image}`, runProvenance(bytes, result.toolVersion));
+  const found = idiomsResult();
+  let instructions: { out: string; count: number } | undefined;
+  if (values.instructions !== undefined) {
+    const list = instructionList(result.instructions, coverage);
+    writeProjectFile(values.instructions, Buffer.from(`${JSON.stringify(list, null, 1)}\n`), { what: "--instructions" });
+    instructions = { out: values.instructions, count: list.length };
+  }
   return {
     analyzer: "ghidra",
+    ...(space === DEFAULT_SPACE ? {} : { space }),
+    ...(instructions === undefined ? {} : { instructions }),
+    ...found,
+    ...coverageResult(),
     coverage: result.coverage.map((range) => ({ start: formatC64Address(range.start), end: formatC64Address(range.end) })),
     ...imported,
-    functions: result.functions.slice(0, MAX_LISTED_FUNCTIONS).map((fn) => ({ entry: formatC64Address(fn.entry), name: fn.name })),
-    ...(result.functions.length > MAX_LISTED_FUNCTIONS ? { moreFunctions: result.functions.length - MAX_LISTED_FUNCTIONS } : {}),
+    functions: listed(result.functions, max, (fn) => ({ entry: formatC64Address(fn.entry), name: fn.name }), "c64-knowledge: knowledge.ts symbols --kind routine"),
     decompilations: result.decompilations.map((item) => ({ entry: formatC64Address(item.entry), text: item.text, ...(item.truncated ? { truncated: true } : {}) })),
     ...((result.notDecompiled ?? []).length === 0
       ? {}

package/skills/c64-static-analysis/scripts/coverage.ts

@@ -0,0 +1,78 @@
+// Runtime evidence for static analysis (D44): coverage files as entry points,
+// user/LLM code regions and JMP tables as entry points, and the check that
+// shows analyzer data over executed code.
+
+import type { DatabaseSync } from "node:sqlite";
+
+import { COVERAGE, instructionLength } from "#src/c64.ts";
+import { FULL_COVERAGE, readCoverageFiles as readFiles } from "#src/host-client/coverage.ts";
+import { listRegions, type RegionRow } from "#src/knowledge/read.ts";
+import { CODE_REGION_TYPES, DEFAULT_SPACE, type Space } from "#src/knowledge/schema.ts";
+import { UsageError } from "#src/script.ts";
+
+const FULL_MAP = FULL_COVERAGE;
+/** Instructions after which the next instruction is not reached by falling through. */
+const FLOW_ENDS = new Set([0x00, 0x40, 0x4c, 0x60, 0x6c]);
+const ANALYZER_ORIGINS = new Set(["dxa", "ghidra", "heuristic"]);
+
+/** Reads coverage files and joins them: an address has every flag that any run gave it. */
+export function readCoverageFiles(paths: string[]): Uint8Array | undefined {
+  return readFiles(paths, (message) => {
+    throw new UsageError(message);
+  });
+}
+
+/** The byte at an address of the analyzed image, or undefined outside it. */
+export type ImageByte = (address: number) => number | undefined;
+
+export function imageReader(bytes: Uint8Array, kind: "prg" | "flat64k"): ImageByte {
+  if (kind === "flat64k") return (address) => bytes[address];
+  const load = bytes[0]! | (bytes[1]! << 8);
+  return (address) => (address >= load && address - load + 2 < bytes.length ? bytes[address - load + 2] : undefined);
+}
+
+/**
+ * The executed instruction starts in RAM that begin a flow: the first after
+ * a gap, or the first after an instruction that does not fall through. The
+ * analyzers follow the rest of a flow themselves.
+ */
+export function coverageEntries(map: Uint8Array, image: ImageByte): number[] {
+  const entries: number[] = [];
+  let falls: number | undefined;
+  for (let address = 0; address < FULL_MAP; address++) {
+    if ((map[address]! & COVERAGE.ramExecute) === 0) continue;
+    const opcode = image(address);
+    if (opcode === undefined) continue;
+    if (falls !== address) entries.push(address);
+    falls = FLOW_ENDS.has(opcode) ? undefined : address + instructionLength(opcode);
+  }
+  return entries;
+}
+
+/** Entry points from semantic knowledge beyond routines: the start of each code region, and each entry of a JMP table with its target. */
+export function codeEntries(db: DatabaseSync | undefined, image: ImageByte, space: Space = DEFAULT_SPACE): number[] {
+  const entries: number[] = [];
+  for (const region of listRegions(db, undefined, undefined, undefined, space).filter((row) => row.origin === "user" || row.origin === "llm")) {
+    if (region.type === "code") entries.push(region.start);
+    if (region.type !== "jumps") continue;
+    for (let at = region.start; at + 2 <= region.end; at += 3) {
+      entries.push(at);
+      const low = image(at + 1);
+      const high = image(at + 2);
+      if (image(at) === 0x4c && low !== undefined && high !== undefined) entries.push(low | (high << 8));
+    }
+  }
+  return entries;
+}
+
+/** Analyzer data regions that hold executed instruction starts: the coverage shows code there. */
+export function coverageConflicts(db: DatabaseSync | undefined, map: Uint8Array): Array<{ region: RegionRow; executed: number }> {
+  const conflicts: Array<{ region: RegionRow; executed: number }> = [];
+  for (const region of listRegions(db)) {
+    if (!ANALYZER_ORIGINS.has(region.origin) || CODE_REGION_TYPES.includes(region.type) || region.type === "dual-use") continue;
+    let executed = 0;
+    for (let address = region.start; address <= region.end; address++) if ((map[address]! & COVERAGE.ramExecute) !== 0) executed++;
+    if (executed > 0) conflicts.push({ region, executed });
+  }
+  return conflicts;
+}

package/skills/c64-static-analysis/scripts/findings.ts

@@ -5,6 +5,7 @@ import type { DatabaseSync } from "node:sqlite";
 
 import type { NormalizedFindings } from "#src/knowledge/import.ts";
 import { listRegions, listSymbols } from "#src/knowledge/read.ts";
+import { CODE_REGION_TYPES, DEFAULT_SPACE, type Space } from "#src/knowledge/schema.ts";
 import type { DxaResult, GhidraParams, GhidraResult } from "#src/native/types.ts";
 
 export type Seeds = Pick<GhidraParams, "entryPoints" | "dataRanges" | "labels">;
@@ -18,13 +19,14 @@ const semantic = (row: { origin: string }) => row.origin === "user" || row.origi
  * earlier guess would force the same answer again, and a re-analysis could
  * not retire it.
  */
-export function seedsFromKnowledge(db: DatabaseSync | undefined, extraEntryPoints: number[] = []): Seeds {
-  const symbols = listSymbols(db).filter(semantic);
+export function seedsFromKnowledge(db: DatabaseSync | undefined, extraEntryPoints: number[] = [], space: Space = DEFAULT_SPACE): Seeds {
+  const symbols = listSymbols(db, undefined, undefined, undefined, space).filter(semantic);
   const entryPoints = [...new Set([...extraEntryPoints, ...symbols.filter((symbol) => symbol.kind === "routine").map((symbol) => symbol.address)])].sort((a, b) => a - b);
   return {
     entryPoints,
-    dataRanges: listRegions(db)
-      .filter((region) => semantic(region) && region.type !== "code")
+    dataRanges: listRegions(db, undefined, undefined, undefined, space)
+      // A JMP table is code and dual-use bytes are both: neither is a data range (D42).
+      .filter((region) => semantic(region) && !CODE_REGION_TYPES.includes(region.type) && region.type !== "dual-use")
       .map((region) => ({ start: region.start, end: region.end })),
     labels: symbols.map((symbol) => ({ address: symbol.address, name: symbol.name })),
   };

package/skills/c64-static-analysis/scripts/idioms.ts

@@ -0,0 +1,144 @@
+// 6502 idioms that hide code from a disassembler (D44): RTS dispatch tables,
+// JMP tables and BIT skips are found by their exact instruction patterns and
+// become heuristic findings and entry points; DOP padding and self-modified
+// opcodes are only reported. A table's length is not known statically:
+// entries count while each target is a plausible instruction in the program.
+
+import { COVERAGE } from "#src/c64.ts";
+import type { NormalizedFindings } from "#src/knowledge/import.ts";
+import type { ImageByte } from "./coverage.ts";
+
+const MAX_TABLE = 64;
+const MIN_JUMPS = 3;
+const LDA_INDEXED = new Set([0xbd, 0xb9]);
+const JAMS = new Set([0x02, 0x12, 0x22, 0x32, 0x42, 0x52, 0x62, 0x72, 0x92, 0xb2, 0xd2, 0xf2]);
+/** Undocumented NOPs with an operand byte (DOP): padding between instructions hides the real ones. */
+const DOPS = new Set([0x04, 0x14, 0x34, 0x44, 0x54, 0x64, 0x74, 0x80, 0x82, 0x89, 0xc2, 0xd4, 0xe2, 0xf4]);
+
+export interface Idioms {
+  findings: NormalizedFindings;
+  /** Entry points for the same analysis run: analyzer facts are never seeds. */
+  entryPoints: number[];
+  /** Table ranges for the same run: they are data. */
+  dataRanges: Array<{ start: number; end: number }>;
+  counts: { rtsTables: number; jumpTables: number; bitSkips: number };
+  /** Seen in the coverage, not written: the LLM decides. */
+  candidates: { dop: number[]; selfModified: number[] };
+}
+
+/** True when an instruction can start at the address: inside the program, and no JAM. */
+function plausible(byteAt: ImageByte, address: number, range: { start: number; end: number }): boolean {
+  const opcode = byteAt(address);
+  return address >= range.start && address <= range.end && opcode !== undefined && !JAMS.has(opcode);
+}
+
+const word = (byteAt: ImageByte, address: number) => {
+  const low = byteAt(address);
+  const high = byteAt(address + 1);
+  return low === undefined || high === undefined ? undefined : low | (high << 8);
+};
+
+export function findIdioms(byteAt: ImageByte, range: { start: number; end: number }, coverage?: Uint8Array): Idioms {
+  const regions: NormalizedFindings["regions"] = [];
+  const references: NormalizedFindings["references"] = [];
+  const entryPoints = new Set<number>();
+  const counts = { rtsTables: 0, jumpTables: 0, bitSkips: 0 };
+  const taken = (start: number, end: number) => regions.some((region) => start <= region.end && end >= region.start);
+  const addRegion = (start: number, end: number, type: NormalizedFindings["regions"][number]["type"]) => {
+    if (start < range.start || end > range.end || taken(start, end)) return false;
+    regions.push({ start, end, type });
+    return true;
+  };
+
+  for (let at = range.start; at + 8 <= range.end; at++) {
+    // RTS dispatch: LDA high,X / PHA / LDA low,X / PHA / RTS.
+    if (LDA_INDEXED.has(byteAt(at)!) && byteAt(at + 3) === 0x48 && LDA_INDEXED.has(byteAt(at + 4)!) && byteAt(at + 7) === 0x48 && byteAt(at + 8) === 0x60) {
+      const high = word(byteAt, at + 1)!;
+      const low = word(byteAt, at + 5)!;
+      const interleaved = high === low + 1;
+      const step = interleaved ? 2 : 1;
+      const targets: number[] = [];
+      for (let index = 0; index < MAX_TABLE; index++) {
+        const lowByte = byteAt(low + index * step);
+        const highByte = byteAt(high + index * step);
+        if (lowByte === undefined || highByte === undefined) break;
+        const target = ((highByte << 8) | lowByte) + 1;
+        if (!plausible(byteAt, target, range)) break;
+        targets.push(target);
+      }
+      if (targets.length === 0) continue;
+      const n = targets.length;
+      const written = interleaved
+        ? addRegion(low, low + 2 * n - 1, "rts-addresses")
+        : low + n === high
+          ? addRegion(low, high + n - 1, "rts-addresses-split")
+          : addRegion(low, low + n - 1, "bytes") && addRegion(high, high + n - 1, "bytes");
+      if (!written) continue;
+      counts.rtsTables++;
+      for (const target of targets) {
+        references.push({ from: at + 8, to: target, kind: "jump" });
+        entryPoints.add(target);
+      }
+    }
+  }
+
+  // JMP tables: at least three JMP abs in a row.
+  for (let at = range.start; at + 3 * MIN_JUMPS - 1 <= range.end; at++) {
+    let n = 0;
+    while (at + 3 * n + 2 <= range.end && byteAt(at + 3 * n) === 0x4c) n++;
+    if (n < MIN_JUMPS || !addRegion(at, at + 3 * n - 1, "jumps")) continue;
+    counts.jumpTables++;
+    for (let index = 0; index < n; index++) {
+      entryPoints.add(at + 3 * index);
+      const target = word(byteAt, at + 3 * index + 1)!;
+      if (plausible(byteAt, target, range)) entryPoints.add(target);
+    }
+    at += 3 * n - 1;
+  }
+
+  // BIT skips: BIT abs ($2c) whose operand bytes are an instruction that a JMP, a JSR or the coverage enters.
+  const entered = new Set<number>();
+  for (let at = range.start; at + 2 <= range.end; at++) {
+    if (byteAt(at) === 0x4c || byteAt(at) === 0x20) entered.add(word(byteAt, at + 1)!);
+  }
+  if (coverage !== undefined) for (let address = range.start; address <= range.end; address++) if ((coverage[address]! & COVERAGE.ramExecute) !== 0) entered.add(address);
+  for (let at = range.start; at + 2 <= range.end; at++) {
+    if (byteAt(at) !== 0x2c || !entered.has(at + 1) || !plausible(byteAt, at + 1, range)) continue;
+    if (!addRegion(at + 1, at + 2, "dual-use")) continue;
+    counts.bitSkips++;
+    entryPoints.add(at);
+    entryPoints.add(at + 1);
+    for (let from = range.start; from + 2 <= range.end; from++) {
+      if ((byteAt(from) === 0x4c || byteAt(from) === 0x20) && word(byteAt, from + 1) === at + 1) references.push({ from, to: at + 1, kind: byteAt(from) === 0x20 ? "call" : "jump" });
+    }
+  }
+
+  // Candidates from the coverage: DOP padding that ran, and opcodes that the program wrote and then ran.
+  const candidates = { dop: [] as number[], selfModified: [] as number[] };
+  if (coverage !== undefined) {
+    for (let address = range.start; address <= range.end; address++) {
+      const flags = coverage[address]!;
+      if ((flags & COVERAGE.ramExecute) === 0) continue;
+      if (DOPS.has(byteAt(address)!) && candidates.dop.length < MAX_TABLE) candidates.dop.push(address);
+      if ((flags & COVERAGE.ramWrite) !== 0 && candidates.selfModified.length < MAX_TABLE) candidates.selfModified.push(address);
+    }
+  }
+
+  regions.sort((a, b) => a.start - b.start);
+  const dataRanges = regions.filter((region) => region.type !== "jumps" && region.type !== "dual-use").map(({ start, end }) => ({ start, end }));
+  return {
+    findings: {
+      analyzer: "heuristic",
+      coverage: [range],
+      authoritative: { symbols: true, regions: true, references: true },
+      symbols: [],
+      regions,
+      // Each reference once: the importer keys them by from, to and kind.
+      references: [...new Map(references.map((reference) => [`${reference.from}|${reference.to}|${reference.kind}`, reference])).values()],
+    },
+    entryPoints: [...entryPoints].sort((a, b) => a - b),
+    dataRanges,
+    counts,
+    candidates,
+  };
+}

package/skills/c64-static-analysis/scripts/instructions.ts

@@ -0,0 +1,105 @@
+// The structured instruction list (D44): Ghidra's decoded instructions with
+// the addressing mode, the operand in 6502 syntax, the flow target and,
+// with a coverage file, whether the instruction ran.
+
+import { COVERAGE, formatC64Address } from "#src/c64.ts";
+import type { GhidraInstruction } from "#src/native/types.ts";
+
+export type Mode =
+  | "implied"
+  | "accumulator"
+  | "immediate"
+  | "zeropage"
+  | "zeropage,x"
+  | "zeropage,y"
+  | "absolute"
+  | "absolute,x"
+  | "absolute,y"
+  | "(indirect)"
+  | "(indirect,x)"
+  | "(indirect),y"
+  | "relative";
+
+const BRANCHES = new Set([0x10, 0x30, 0x50, 0x70, 0x90, 0xb0, 0xd0, 0xf0]);
+const JMP_INDIRECT = 0x6c;
+
+/**
+ * The addressing mode from Ghidra's operand text and the instruction length.
+ * Ghidra prints zero page and absolute alike, and JMP (indirect) as an
+ * address, so the length and the opcode decide those.
+ */
+export function modeOf(opcode: number, length: number, operands: string): Mode {
+  const text = operands.toUpperCase();
+  if (text === "") return "implied";
+  if (text === "A") return "accumulator";
+  if (text.startsWith("#")) return "immediate";
+  if (text.startsWith("(") && text.endsWith(",X)")) return "(indirect,x)";
+  if (text.startsWith("(") && text.endsWith("),Y")) return "(indirect),y";
+  if (text.endsWith(",X")) return length === 2 ? "zeropage,x" : "absolute,x";
+  if (text.endsWith(",Y")) return length === 2 ? "zeropage,y" : "absolute,y";
+  if (BRANCHES.has(opcode)) return "relative";
+  if (opcode === JMP_INDIRECT) return "(indirect)";
+  return length === 2 ? "zeropage" : "absolute";
+}
+
+/** The operand in 6502 syntax, from the instruction bytes; empty for implied. */
+export function operandText(mode: Mode, bytes: Uint8Array, target: number | undefined): string {
+  const byte = `$${(bytes[1] ?? 0).toString(16).padStart(2, "0")}`;
+  const word = formatC64Address((bytes[1] ?? 0) | ((bytes[2] ?? 0) << 8));
+  switch (mode) {
+    case "implied":
+      return "";
+    case "accumulator":
+      return "a";
+    case "immediate":
+      return `#${byte}`;
+    case "zeropage":
+      return byte;
+    case "zeropage,x":
+      return `${byte},x`;
+    case "zeropage,y":
+      return `${byte},y`;
+    case "absolute":
+      return word;
+    case "absolute,x":
+      return `${word},x`;
+    case "absolute,y":
+      return `${word},y`;
+    case "(indirect)":
+      return `(${word})`;
+    case "(indirect,x)":
+      return `(${byte},x)`;
+    case "(indirect),y":
+      return `(${byte}),y`;
+    case "relative":
+      return target === undefined ? byte : formatC64Address(target);
+  }
+}
+
+export interface ListedInstruction {
+  address: string;
+  bytes: string;
+  mnemonic: string;
+  mode: Mode;
+  operand: string;
+  target?: string;
+  executed?: boolean;
+}
+
+/** Ghidra's instructions as the list that --instructions writes. */
+export function instructionList(instructions: GhidraInstruction[], coverage?: Uint8Array): ListedInstruction[] {
+  return instructions.map((instruction) => {
+    const bytes = Buffer.from(instruction.bytes, "hex");
+    const mode = modeOf(bytes[0]!, bytes.length, instruction.operands);
+    const [target] = instruction.flows;
+    return {
+      address: formatC64Address(instruction.address),
+      bytes: (instruction.bytes.match(/../g) ?? []).join(" "),
+      mnemonic: instruction.mnemonic,
+      mode,
+      operand: operandText(mode, bytes, target),
+      ...(target === undefined ? {} : { target: formatC64Address(target) }),
+      ...(coverage === undefined ? {} : { executed: (coverage[instruction.address]! & COVERAGE.ramExecute) !== 0 }),
+    };
+  });
+}

package/skills/c64-testing/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: c64-testing
-description: Use this skill to test if a rebuilt or changed C64 program behaves as the original. It runs one scenario on the original and then on the rebuild in an emulator. It compares memory, registers, chip state and the screen with explicit rules, and gives PASS, FAIL or INCONCLUSIVE. It also makes a checklist for a human playtest. Do not use it to find the cause of a difference.
+description: Use this skill to test if a rebuilt or changed C64 program behaves as the original. It runs one scenario on the original and then on the rebuild in an emulator. It compares memory, registers, chip state and the screen with explicit rules, and gives PASS, FAIL or INCONCLUSIVE. It also makes a checklist for a human playtest. Use its run command to explore a program with many steps, for example to visit each room or level. A run keeps screens, coverage maps and snapshots as files. Do not use it to find the cause of a difference.
 ---
 
 # c64-testing
@@ -22,14 +22,34 @@ A PASS proves only the behavior that the scenario does and observes. It does not
 The script starts one emulator through the host runtime. It runs the scenario on the original and then on the rebuild. Before each side, it does a hard reset of the emulator. The CPU then stops at the reset vector.
 
 ```
-node <skill>/scripts/test.ts tests/moves-left.json
+node <skill>/scripts/test.ts tests/moves-left.json --out results/moves-left
 node <skill>/scripts/checklist.ts tests/moves-left.json --item 'FIRE starts the game.'
 ```
 
-The test script prints one JSON object. The exit status is 0 for PASS, 1 for FAIL, 3 for INCONCLUSIVE and 2 for a bad call.
+The test script prints one JSON object. The exit status is 0 for PASS, 1 for FAIL, 3 for INCONCLUSIVE and 2 for a bad call. With `--out`, a FAIL writes files into that directory: both screens and the difference image for each screen difference, and `memory.json` with each byte that differs. `files` in the result names them.
 
 The checklist script prints a Markdown checklist for a human tester.
 
+### Runs
+
+A run does steps on one program, with no comparison, and keeps the results as files. Use it for many steps. One run replaces many emulator calls, for example a visit to each room of a game.
+
+```
+node <skill>/scripts/run.ts runs/rooms.json --out runs/rooms
+```
+
+The run file gives where to start (a program, or a snapshot file), the steps, and optional symbols. To go on from the live emulator, save its state with `c64_snapshot` action `save` and `out`, and start the run from that file. Besides the steps of a scenario, a run has these steps:
+
+| Step | Effect |
+|---|---|
+| `{ "snapshot": "snapshots/room0.vsf" }` | Restore a snapshot file. |
+| `{ "save": "room3" }` | Write the state now to `room3.vsf` in the output directory. |
+| `{ "screen": "room3" }` | Write the screen to `room3.png`. |
+| `{ "coverage": "room3" }` | Write the coverage map since the start or the last coverage step to `room3.bin`, then clear the map. |
+| `{ "observe": "room3", "memory": ["$c100"], "registers": true }` | Record memory, registers, `vicii` or `sprites` in `results.json`. |
+
+See [references/scenario-format.md](references/scenario-format.md), section Runs. The script prints one JSON object. The exit status is 0 when all steps ran, 1 when a step did not reach its target, and 2 for a bad call. A long run can need more time than one shell call. Then run it in the background.
+
 ## Workflow
 
 1. Write the scenario file. Start the steps with a frame count, for example `{ "frames": 150 }`. Each side starts at the reset vector, and the C64 needs these frames to start.
@@ -66,8 +86,12 @@ PASS, FAIL or INCONCLUSIVE, the checkpoints and the smallest differences that ex
 - INCONCLUSIVE, "did not reach the checkpoint": one side did not reach the checkpoint in the frame limit. Find the reason with the c64-emulator skill. Do not report a FAIL.
 - INCONCLUSIVE, "has no symbol": one side has no symbol with that name. Add the name to the knowledge or to the rebuild source.
 - INCONCLUSIVE, "no emulator" or "could not run": the host runtime or the emulator failed. Tell the user the message.
+- `installation-incomplete`, or a message that the host runtime does not run: stop the task. Give the message to the user and wait for the user. The message gives the remedy. Do not make a replacement for the tool or the host runtime, for example your own disk parser, disassembler or emulator script.
 - A FAIL with only a `screen` difference: add `vicii` or `sprites` to the observation to find which chip state is different.
 - A program that uses random numbers: set the random seed with a `write` step on both sides. If that is not possible, observe only values that do not depend on the random numbers.
+- The two sides start differently, for example the original after a cracked intro. Give the steps that only one side needs a `side`. Then make the memory the same with a `normalize` step from the entry state of the c64-unpacker skill.
+- Frame counts drift by one frame between the sides: the sides got to the same address at different raster positions. Do not sync on an IRQ handler. Sync on a value that the IRQ changes, with `runUntil` `memory`, or on an address and a raster line together.
+- A byte that cannot be the same on both sides, for example a read-only collision register: use `exclude` with the reason. Do not use `exclude` to hide a real difference.
 
 Do not add a mask or a tolerance only to make a test pass. A tolerance must have a reason in the behavior of the program.