@henols/c64-re-tools 2.1.0 → 2.1.2

package/skills/c64-provenance/scripts/provenance.ts

@@ -8,7 +8,7 @@
 // routines that several programs share in a library folder and find them in the
 // program (D71).
 
-import { basename, relative } from "node:path";
+import { basename } from "node:path";
 
 import { type AddressRange, c64AddressOf, formatC64Address } from "#src/c64.ts";
 import { Refusal } from "#src/errors.ts";
@@ -41,7 +41,7 @@ import {
 import { CODE_REGION_TYPES, SEMANTIC_ORIGINS } from "#src/knowledge/schema.ts";
 import { platformNames } from "#src/platform.ts";
 import { projectRoot } from "#src/project.ts";
-import { knowledgeNames } from "#src/knowledge/names.ts";
+import { knowledgeAt, knowledgeNames } from "#src/knowledge/names.ts";
 import { listed, OPTIONS, runScript, UsageError, type Call, type CommandSpec } from "#src/script.ts";
 import { compareReleases, shiftSuggestion, suggestShifts, type Release } from "./lib/compare.ts";
 import { findRoutine, libraryFolder, readLibrary, routineFrom, saveRoutine, suggest } from "./lib/library.ts";
@@ -107,13 +107,17 @@ async function compare(call: Call): Promise<unknown> {
   const shifts = new Map<number, number>();
   for (const text of call.strings("shift")) {
     const match = /^(\d+):([+-]?\d+)$/.exec(text);
-    if (match === null || Number(match[1]) < 1 || Number(match[1]) > positionals.length) throw new UsageError(`--shift must be <release number>:<bytes>, not ${text}`);
+    if (match === null) throw new UsageError(`--shift must be <release number>:<bytes>, for example 2:-256, not ${text}`);
+    if (Number(match[1]) < 1 || Number(match[1]) > positionals.length) {
+      throw new UsageError(`--shift ${text}: there is no release ${match[1]}. This compare has ${positionals.length} releases: give a release number from 1 to ${positionals.length}`);
+    }
+    if (shifts.has(Number(match[1]))) throw new UsageError(`--shift gives release ${match[1]} more than one shift (${shifts.get(Number(match[1]))} and ${match[2]}). Give one --shift for each release`);
     shifts.set(Number(match[1]), Number(match[2]));
   }
   const frames = call.count("frames");
   const stepsFile = call.string("steps");
   if (stepsFile !== undefined && frames === undefined) throw new UsageError("--steps works only with --frames");
-  const steps: ActionStep[] = stepsFile === undefined ? [] : readActionSteps(stepsFile, "--steps");
+  const steps: ActionStep[] = stepsFile === undefined ? [] : readActionSteps(stepsFile, "--steps", knowledgeAt());
   const loaded = positionals.map((path, index) => readRelease(path, shifts.get(index + 1) ?? 0));
   const releases = loaded.map((item) => item.release);
   const range = call.range("range");
@@ -364,10 +368,10 @@ function routineCommand(call: Call): unknown {
       operands: listOperands(db, range, SEMANTIC_ORIGINS),
       platform: platform(),
     });
-    const file = saveRoutine(libraryFolder(library, { create: true, cwd: projectRoot() }), routine, call.flag("replace"));
+    const written = saveRoutine(libraryFolder(library, { create: true, cwd: projectRoot() }), routine, call.flag("replace"), projectRoot());
     return {
       routine: routine.name,
-      file: relative(projectRoot(), file).split("\\").join("/"),
+      ...written,
       start: formatC64Address(start),
       end: formatC64Address(end),
       length: routine.bytes.length,
@@ -460,15 +464,15 @@ const COMMANDS: CommandSpec[] = [
   },
   {
     name: "separate",
-    summary: "Evidence from the program in the knowledge: the unpack stage that wrote each range, its access in the coverage, whether call and jump references reach each code region from the entry, and what the program reads or executes outside its segments.",
-    options: { out: { ...OPTIONS.out, value: "<file>", help: "write all ranges to this file" }, "max-listed": OPTIONS["max-listed"] },
+    summary: "Evidence from the program in the knowledge: the unpack stage that wrote each range, its access in the coverage, whether call and jump references, or the run on from the code before it, reach each code region from the entry, and what the program reads or executes outside its segments.",
+    options: { out: { ...OPTIONS.out, output: "file", value: "<file>", help: "write all ranges to this file" }, "max-listed": OPTIONS["max-listed"] },
     run: separateCommand,
   },
   {
     name: "leftovers",
     summary: "What the program in the knowledge holds besides its working code: texts in PETSCII, screen codes, ASCII or with bit 7 set (Apple II), assembler source, BASIC lines away from the BASIC start, fills, untouched bytes outside code, copies of earlier bytes, and code regions that nothing reaches and no run executed.",
     options: {
-      out: { ...OPTIONS.out, value: "<file>", help: "write all findings to this file" },
+      out: { ...OPTIONS.out, output: "file", value: "<file>", help: "write all findings to this file" },
       "min-text": { kind: "count", value: "<n>", min: 4, max: 999, default: 8, help: "the shortest text to find (default 8)" },
       "max-listed": OPTIONS["max-listed"],
     },
@@ -491,7 +495,7 @@ const COMMANDS: CommandSpec[] = [
     summary: "Finds each routine of the library in the program of the knowledge: the opcodes and other fixed bytes equal, and the addresses into the routine moved by one shift.",
     options: {
       library: { ...LIBRARY, help: "the library folder, relative to the project folder, for example ../library" },
-      out: { ...OPTIONS.out, value: "<file>", help: "write the names of the routines at the found addresses as a batch for knowledge.ts apply" },
+      out: { ...OPTIONS.out, output: "file", value: "<file>", help: "write the names of the routines at the found addresses as a batch for knowledge.ts apply" },
       "max-listed": OPTIONS["max-listed"],
     },
     run: matchCommand,

package/skills/c64-source/SKILL.md

@@ -47,6 +47,7 @@ When the knowledge has coverage, code that no run executed gets the line `; not
 - An address is `$` and 1 to 4 hex digits, for example `'$c000'` or `'$fb'`, or a name. A name is a symbol, else an analyzer name that only one analyzer gives. Quote an address in a shell.
 - A range is `<start>-<end>`, for example `'$c000-$cfff'`. In a JSON file, a range is `start` and `end`.
 - Bytes in a JSON file are hex in a field `data`, for example `"data": "a900"`.
+- The script examines each `--out` path before it starts work. It refuses a path in `.c64-re-tools/knowledge/` and the index of the knowledge, because only c64-knowledge writes them. It refuses a folder where the command writes a file, and a file where the command writes a folder.
 - The result is one JSON object on one line. A file that the script writes shows as `{"out", "bytes", "sha256"}`. A list shows as `{"count", "listed"}`, and `rest` tells where to get the rest of a long list.
 - An error is `{"error": {"code": "...", "message": "..."}}`. The message tells what is wrong and what to do.
 - The exit status is 0 for done or PASS, 1 for FAIL, 2 for an error and 3 for INCONCLUSIVE.
@@ -150,6 +151,7 @@ Do not store:
 - `operandsAsNumbers`: the operand expressions that the source has as a number with a side comment, each with its `address`, `expression` and `reason`. An expression becomes a number when no instruction with an operand value is at the address, or when a name has no address. An expression that ACME refuses or does not assemble to the program also becomes a number. Then the script assembles the source again, at most six times.
 - `unusedModules`: modules in the knowledge whose start is in no segment of the program. Often this tells that the program does not hold code that the knowledge describes.
 - `entryOutside`: the entry is in no segment of the program in the knowledge, with a `reason`. The capture does not hold the program.
+- `dataOnly`: the knowledge has no entry, no routine, no code region and no executed address in the space. Thus the source has only data, and Ghidra does not run. A table or a key page in a cartridge bank is an example.
 - `notExecuted`: when the knowledge has coverage, the count of instructions that no run executed, the count of their ranges, and the first ranges. `rest` names the file `not_executed.json` in `--out` when the list is not complete. This file has all of the ranges. Each range has its source `file` and the `routine` before it. `files` counts the ranges in each file.
 - `macroUses`: the count of macro uses that the source has.
 - `macrosAsInstructions`: the macro uses that the source has as instructions, each with its `start`, `end`, `macro` and `reason`.
@@ -171,6 +173,7 @@ Do not store:
 - `patchesNotWritten`: read each `reason`. Move the start or the end of the patch to an instruction, or give the switch another name.
 - `macrosAsInstructions`: read each `reason`. Correct the macro or the args, or remove the use with `unuse`. The source stays identical, so this is not a failure.
 - `entryOutside`: the run fails, but the script still writes and checks the source. With `--loader`, `loader.verified` is `false` with the same `reason`. Capture the program again with c64-unpacker `loads` or `capture`. Make sure that a segment holds the entry. Then put it into the knowledge again with `image`.
+- `dataOnly`: this is not a failure. If the space has code, give `--entry` or classify the code with the c64-knowledge skill. Then generate again.
 - `entriesInData`: an analyzer region of data hides the code at the entry. If the entry is code, record a code region at the entry with the c64-knowledge skill, for example `classify '$0500-$05ff' code`. A region of the user or the LLM replaces the analyzer region. Then generate again.
 - `loader.verified: false` with a `reason`: the script did not make the loader. The script still writes and checks the source. When no copy order or no free pages exist, or the loader needs too many copies, the reason gives the ranges. Capture the program again with fewer segments, or write the loader by hand.
 - `loader.verified: false` with `differences`: the loader ran, and `differences` gives the bytes and registers that are not equal.

package/skills/c64-source/scripts/lib/source.ts

@@ -13,7 +13,7 @@ import { WireFailure } from "#src/host-client/tools.ts";
 import { identifiers, operandText, replaceIdentifiers, type OperandExpression } from "#src/knowledge/expression.ts";
 import { type CommentPlacement, isAnalyzer, isGeneratedFile, type Origin, type RegionType } from "#src/knowledge/schema.ts";
 import type { Area } from "#src/platform.ts";
-import { hasZeroPageForm, MNEMONICS, MODE_LENGTH, OPCODES, takesAddress, USE_KINDS, useKind, type Mode, type UseKind } from "#src/opcodes.ts";
+import { hasZeroPageForm, instructionWith, labelProblem, MODE_LENGTH, OPCODES, takesAddress, USE_KINDS, useKind, type UseKind } from "#src/opcodes.ts";
 
 export interface SourceInput {
   /** In address order, without overlap. */
@@ -319,7 +319,7 @@ export function layout(input: SourceInput): Layout {
   const outside = new Set<number>();
   // A platform name that knowledge gives to another address, or that reads as an instruction, is not used.
   const knowledgeNames = new Set(input.symbols.map((symbol) => symbol.name));
-  const platform = new Map((input.platform ?? []).filter(({ name }) => !knowledgeNames.has(name) && !MNEMONICS.has(name.toLowerCase())).map(({ address, name }) => [address, name]));
+  const platform = new Map((input.platform ?? []).filter(({ name }) => !knowledgeNames.has(name) && labelProblem(name) === undefined).map(({ address, name }) => [address, name]));
   // An analyzer name (DXA's ld020) gives way to the platform name (EXTCOL); a user or LLM name wins.
   const symbols = new Map(input.symbols.filter((symbol) => !(isAnalyzer(symbol.origin) && platform.has(symbol.address))).map((symbol) => [symbol.address, symbol.name]));
   /** Addresses outside the image without a name of their own, inside a named platform entry of more bytes: the start of that entry. */
@@ -384,7 +384,7 @@ export function layout(input: SourceInput): Layout {
     }
     const missing = [...new Set(use.args.flatMap((arg) => identifiers(arg)))].filter((name) => !byName.has(name));
     if (missing.length > 0) {
-      fallBack(`No symbol and no platform name is ${missing.join(", ")}.`);
+      fallBack(`${missing.join(", ")} ${missing.length === 1 ? "is" : "are"} not a symbol of the knowledge or a platform name. Name the address with knowledge.ts rename, or give other args with knowledge.ts use.`);
       continue;
     }
     candidates.push({ use, items: run as InstructionItem[] });
@@ -403,7 +403,7 @@ export function layout(input: SourceInput): Layout {
     }
     const missing = identifiers(expression).filter((name) => !byName.has(name));
     if (missing.length > 0) {
-      operandsAsNumbers.push({ address: operand.address, expression, reason: `No symbol and no platform name is ${missing.join(", ")}.` });
+      operandsAsNumbers.push({ address: operand.address, expression, reason: `${missing.join(", ")} ${missing.length === 1 ? "is" : "are"} not a symbol of the knowledge or a platform name. Name the address with knowledge.ts rename, or write the operand again with knowledge.ts operand.` });
       continue;
     }
     // An immediate operand builds the address of a name; any other operand uses the address as the instruction does.
@@ -450,7 +450,8 @@ export function layout(input: SourceInput): Layout {
   for (const address of [...wanted, ...outside].sort((a, b) => a - b)) {
     const known = symbols.get(address);
     if (known !== undefined) {
-      if (MNEMONICS.has(known.toLowerCase())) throw new WireFailure("invalid-input", `The symbol ${known} at ${formatC64Address(address)} has the name of an instruction, which ACME cannot read as a label. Rename the symbol with c64-knowledge.`);
+      const problem = labelProblem(known);
+      if (problem !== undefined) throw new WireFailure("invalid-input", `The symbol ${known} at ${formatC64Address(address)}: ${problem} Rename the symbol with c64-knowledge.`);
       names.set(address, known);
       if (inImage(address)) namedLabels++;
       continue;
@@ -574,27 +575,6 @@ export function formOf(layout: Pick<Layout, "inImage" | "byteAt">, item: Instruc
   return { key: `${item.opcode.toString(16).padStart(2, "0")}${suffix}`, opcode: item.opcode, suffix };
 }
 
-/** The instruction text with an operand expression; the expression is a name or a $ number. */
-export function instructionText(opcode: number, suffix: string, expression: string): string {
-  const { mnemonic, mode } = OPCODES[opcode]!;
-  const operand: Record<Mode, string> = {
-    imp: "",
-    imm: `#${expression}`,
-    zp: expression,
-    zpx: `${expression},x`,
-    zpy: `${expression},y`,
-    abs: expression,
-    abx: `${expression},x`,
-    aby: `${expression},y`,
-    ind: `(${expression})`,
-    izx: `(${expression},x)`,
-    izy: `(${expression}),y`,
-    rel: expression,
-  };
-  const text = operand[mode];
-  return text === "" ? `${mnemonic}${suffix}` : `${mnemonic}${suffix} ${text}`;
-}
-
 /** The plain number of an operand: what the probe assembles, and what a comment shows. */
 export function literalOperand(opcode: number, value: number): string {
   const { mode } = OPCODES[opcode]!;
@@ -1046,7 +1026,7 @@ export function render(layout: Layout, asBytes: ReadonlySet<string>, header: str
         const form = formOf(layout, item);
         const { mode } = OPCODES[item.opcode]!;
         const value = mode === "imp" ? 0 : operandValue(item.address, item.opcode, layout.byteAt)!;
-        const literal = instructionText(item.opcode, "", literalOperand(item.opcode, value));
+        const literal = instructionWith(item.opcode, literalOperand(item.opcode, value));
         let written = layout.operands.get(item.address);
         // An expression that is not written stays visible as a side comment.
         const asNumber = (expression: string, reason: string) => {
@@ -1062,13 +1042,11 @@ export function render(layout: Layout, asBytes: ReadonlySet<string>, header: str
           out.push(withSide(`${INDENT}!byte ${bytes.join(",")}`, [literal, ...sides].join(" ")));
           result.asBytes.push({ address: item.address, text: literal });
         } else if (written !== undefined) {
-          // ACME reads an operand in brackets as indirect, so a non-immediate expression does not start with one.
-          const expression = mode !== "imm" && written.startsWith("(") ? `0+${written}` : written;
-          out.push(withSide(`${INDENT}${instructionText(item.opcode, form.suffix, expression)}`, sides.length === 0 ? undefined : sides.join(" ")));
+          out.push(withSide(`${INDENT}${instructionWith(item.opcode, written, form.suffix)}`, sides.length === 0 ? undefined : sides.join(" ")));
           result.expressions.push({ address: item.address, length: item.length, file: piece.file, line: out.length });
         } else {
           const expression = takesAddress(mode) ? (layout.names.get(value) ?? literalOperand(item.opcode, value)) : literalOperand(item.opcode, value);
-          out.push(withSide(`${INDENT}${instructionText(item.opcode, form.suffix, expression)}`, sides.length === 0 ? undefined : sides.join(" ")));
+          out.push(withSide(`${INDENT}${instructionWith(item.opcode, expression, form.suffix)}`, sides.length === 0 ? undefined : sides.join(" ")));
         }
         mapLine(item.address, item.address + item.length - 1);
         result.instructions++;

package/skills/c64-source/scripts/source.ts

@@ -31,8 +31,8 @@ import { resolveProjectPath } from "#src/project.ts";
 import { listed, OPTIONS, runScript, UsageError, type Call } from "#src/script.ts";
 import { knowledgeNames } from "#src/knowledge/names.ts";
 import { COPIER_SOURCE, LOADER_JUMP, loaderLimits, planLoader, rasterLine, loaderState, simulate, stopCause, VIC_DISPLAY_REGISTERS, type EntryState, type LoaderPlan } from "./lib/loader.ts";
-import { OPCODES } from "#src/opcodes.ts";
-import { entriesInData, entryOutsideImage, instructionText, layout, literalOperand, probeForms, render, switchesOf, type Layout, type PatchBlock, type Rendered } from "./lib/source.ts";
+import { instructionWith, OPCODES } from "#src/opcodes.ts";
+import { entriesInData, entryOutsideImage, layout, literalOperand, probeForms, render, switchesOf, type Layout, type PatchBlock, type Rendered } from "./lib/source.ts";
 
 /** At most this many diagnostics or differences of one check. */
 const MAX_DIAGNOSTICS = 20;
@@ -40,6 +40,9 @@ const MAX_DIAGNOSTICS = 20;
 const ROUNDS = 6;
 const PROBE_BASE = 0x1000;
 const PROBE_SLOT = 4;
+/** Why a source has no instructions when the knowledge has no code. */
+const DATA_ONLY = "The knowledge has no entry, routine, code region or executed address in this space. Thus the source has only data. For instructions, give --entry, or classify the code with knowledge.ts classify, then generate again.";
+
 /** The copies of a full 64 KiB take about 140 frames. */
 const LOADER_FRAMES = 1500;
 
@@ -99,9 +102,10 @@ function withAnalyzerNames(symbols: SymbolRow[], second: SymbolRow[]): SymbolRow
  * Ghidra's instruction starts and lengths inside the image, seeded from the
  * entry, from knowledge of any origin, and from the coverage: a run's flow
  * starts, also after a JSR, so code that an analyzer does not reach by itself
- * is in the source.
+ * is in the source. A program without a seed has no known code: it is data
+ * only, and Ghidra does not run.
  */
-async function decode(image: ProgramImage, knowledge: ReturnType<typeof readKnowledge>, entries: number[], coverage: Uint8Array | undefined): Promise<Map<number, number>> {
+async function decode(image: ProgramImage, knowledge: ReturnType<typeof readKnowledge>, entries: number[], coverage: Uint8Array | undefined): Promise<{ instructions: Map<number, number>; dataOnly: boolean }> {
   const inImage = (address: number) => image.segments.some((segment) => address >= segment.start && address <= segment.end);
   const seeds = new Set<number>([...(image.entry === undefined ? [] : [image.entry]), ...entries]);
   for (const address of knowledge.routines) seeds.add(address);
@@ -114,8 +118,7 @@ async function decode(image: ProgramImage, knowledge: ReturnType<typeof readKnow
   const entryPoints = [...seeds].filter(inImage).sort((a, b) => a - b);
   if (entryPoints.length === 0) {
     // An entry in no segment is a capture fault: the image is data only, and the result says entryOutside.
-    if (image.entry !== undefined && !inImage(image.entry)) return new Map();
-    throw new UsageError("Ghidra needs an entry point: give --entry, or record a routine or a code region in knowledge with the c64-knowledge skill");
+    return { instructions: new Map(), dataOnly: image.entry === undefined || inImage(image.entry) };
   }
   const dataRanges = knowledge.regions
     .filter((region) => !CODE_REGION_TYPES.includes(region.type) && region.type !== "dual-use")
@@ -138,7 +141,7 @@ async function decode(image: ProgramImage, knowledge: ReturnType<typeof readKnow
     const inside = Array.from({ length: bytes.length }, (_, offset) => instruction.address + offset).every((address, offset) => inImage(address) && image.byteAt(address) === bytes[offset]);
     if (inside) instructions.set(instruction.address, bytes.length);
   }
-  return instructions;
+  return { instructions, dataOnly: false };
 }
 
 /** Assembles files given as text with ACME; the program or the diagnostics. */
@@ -179,7 +182,7 @@ async function probe(laid: Layout): Promise<Set<string>> {
             ? formatC64Address((slot + 2 + (item.bytes[1]! < 0x80 ? item.bytes[1]! : item.bytes[1]! - 0x100)) & 0xffff)
             : literalOperand(item.form.opcode, item.bytes.length === 3 ? item.bytes[1]! | (item.bytes[2]! << 8) : item.bytes[1]!);
       lines.push(`* = ${formatC64Address(slot)}`);
-      lines.push(`        ${instructionText(item.form.opcode, item.form.suffix, operand)}`);
+      lines.push(`        ${instructionWith(item.form.opcode, operand, item.form.suffix)}`);
       lineOf.set(lines.length, item.form.key);
     });
     const built = await assembleFiles([{ name: "probe.asm", text: `${lines.join("\n")}\n` }], "probe.asm");
@@ -547,7 +550,7 @@ async function generate(call: Call): Promise<{ output: Record<string, unknown>;
   // Read the entry state before the long steps, so a bad state stops the run at once.
   const state = loaderWanted ? loaderState(image.state, "the entry state of the knowledge") : undefined;
   const romUse = romsInUse(image, coverage);
-  const instructions = await decode(image, knowledge, entries, coverage);
+  const { instructions, dataOnly } = await decode(image, knowledge, entries, coverage);
   const laid = layout({
     segments: image.segments,
     byteAt: image.byteAt,
@@ -634,6 +637,7 @@ async function generate(call: Call): Promise<{ output: Record<string, unknown>;
     // The ROM names that the source leaves out, and the evidence.
     ...(space !== DEFAULT_SPACE || (romUse.basic && romUse.kernal) ? {} : { romNamesLeftOut: { roms: [...(romUse.basic ? [] : ["BASIC"]), ...(romUse.kernal ? [] : ["KERNAL"])], ...(romUse.basic || romUse.kernal ? {} : { systemArea: true }), from: romUse.from } }),
     ...(missingEntry === undefined ? {} : { entryOutside: { entry: formatC64Address(image.entry!), reason: missingEntry } }),
+    ...(dataOnly ? { dataOnly: DATA_ONLY } : {}),
     labels: { named: laid.namedLabels, generated: laid.generatedLabels },
     // An --entry in a data region decodes nothing: the result says so, and a code region in the knowledge changes it.
     ...(inData.length === 0
@@ -730,7 +734,7 @@ await runScript({
       name: "generate",
       summary: "Writes the source files, source-map.json and not_executed.json, and checks them. result is PASS when every check passes, else FAIL.",
       options: {
-        out: { ...OPTIONS.out, value: "<dir>", help: "the folder for the source files, relative to the project folder", required: true },
+        out: { ...OPTIONS.out, output: "folder", value: "<dir>", help: "the folder for the source files, relative to the project folder", required: true },
         entry: { ...OPTIONS.entry, multiple: true, help: "an entry point for Ghidra" },
         space: { ...OPTIONS.space, help: "the knowledge space of the program: c64 (the default), drive8, or bank0 to bank255. The coverage and --loader are for c64 only" },
         loader: { kind: "flag", help: "also write loader.asm and loader.prg: the loader copies the segments into place, sets the entry state of the knowledge, and jumps to the entry. It is checked in the emulator" },

package/skills/c64-static-analysis/SKILL.md

@@ -61,6 +61,7 @@ The analyzers take at most 1024 entry points. Knowledge routines and `--entry` c
 - An address is `$` and 1 to 4 hex digits, for example `'$c000'` or `'$fb'`, or a name. A name is a symbol, else an analyzer name that only one analyzer gives. Quote an address in a shell.
 - A range is `<start>-<end>`, for example `'$c000-$cfff'`. In a JSON file, a range is `start` and `end`.
 - Bytes in a JSON file are hex in a field `data`, for example `"data": "a900"`.
+- The script examines each `--out` path before it starts work. It refuses a path in `.c64-re-tools/knowledge/` and the index of the knowledge, because only c64-knowledge writes them. It refuses a folder where the command writes a file, and a file where the command writes a folder.
 - The result is one JSON object on one line. A file that the script writes shows as `{"out", "bytes", "sha256"}`. A list shows as `{"count", "listed"}`, and `rest` tells where to get the rest of a long list.
 - An error is `{"error": {"code": "...", "message": "..."}}`. The message tells what is wrong and what to do.
 - The exit status is 0 for done or PASS, 1 for FAIL, 2 for an error and 3 for INCONCLUSIVE.
@@ -87,6 +88,7 @@ The analyzers take at most 1024 entry points. Knowledge routines and `--entry` c
    - Ghidra: `functions` gives the number of routines that Ghidra found (`count`) and lists the first routines (`listed`). `decompilations` gives C-like code for the requested routines.
    - If a list is not complete, `rest` gives the c64-knowledge command that lists all of them. For `candidates`, `rest` tells you to give a larger `--max-listed`.
    - `conflicts` lists the places where the analyzer and the current knowledge disagree.
+   - DXA or Ghidra can find data where a run executed an instruction start (`executed`), or where a `user` or `llm` symbol of kind `label` or `routine` is (`knowledge`). Then the script does not write that data region, and the earlier region of the analyzer stays. `conflicts` lists each such place. Executed code and your names have priority over the guess of an analyzer.
    - `idioms` tells what the script found by pattern: RTS dispatch tables (`rtsTables`), JMP tables (`jumpTables`) and BIT skips (`bitSkips`). It also finds the `CBM80` header of a cartridge bank at `$8000` (`cartridgeHeaders`). The two start vectors of the header become entry points. The script writes them as knowledge with the origin `heuristic`, and seeds their targets into the same analysis. `candidates` gives DOP padding (`dop`) and opcodes that the program wrote and executed (`selfModified`). For `selfModified`, the write and the execution must be in one coverage file. A coverage file does not record the order. Thus a candidate can also be code that ran first, and that the program then wrote over. To prevent this, save one coverage file for each part of the program, for example the intro and the game. A loader that copies relocated code is also such a part. Clear the memory map between the parts. Each list has a `count`. The script does not write the candidates. Look at them and decide.
    - `candidates.pointers` gives code that makes an address from immediates. A `vector` or a `zero-page pointer` candidate stores a low byte (`low`) and a high byte (`high`) to a vector or to a zero-page pair that `(zp),y` uses. Its `target` is in the program or has a platform name. A `sprite pointer` candidate stores an immediate (`at`) to a sprite pointer of the screen at $0400 in VIC bank 0. Its `target` (the value × 64) is in a `sprite` region. For each candidate that you confirm, write operand expressions with `operand` of the c64-knowledge skill, for example `operand '<low>' --lo <target name>` and `operand '<high>' --hi <target name>`. Then a rebuild can move the target.
    - `candidates.pitfalls` gives code that a C64 runs in a different way than it reads. `raster-bit-8`: the instruction at `at` reads $D011, and the instruction at `store` writes the value back. A read of $D011 gives bit 8 of the raster line in bit 7. Thus the write can move the raster interrupt to line 256 or later, and on PAL past line 311 the interrupt never comes. Examine each candidate. Where the code must not keep bit 7, add a comment and test the program.

package/skills/c64-static-analysis/scripts/lib/idioms.ts

@@ -31,7 +31,7 @@ export interface Idioms {
   candidates: { dop: number[]; selfModified: number[] };
 }
 
-/** True when an instruction can start at the address: inside the program, and no JAM. */
+/** True when an instruction can start at the address: inside the range and the program (`byteAt` gives a byte), and no JAM. */
 function plausible(byteAt: ImageByte, address: number, range: { start: number; end: number }): boolean {
   const opcode = byteAt(address);
   return address >= range.start && address <= range.end && opcode !== undefined && !JAMS.has(opcode);
@@ -43,7 +43,11 @@ const word = (byteAt: ImageByte, address: number) => {
   return low === undefined || high === undefined ? undefined : low | (high << 8);
 };
 
-/** `covered` is the coverage of the knowledge: the joined map, and the addresses that one run wrote and executed. */
+/**
+ * `byteAt` gives the bytes of the program and undefined outside it, so a
+ * table and its targets stay inside the program. `covered` is the coverage of
+ * the knowledge: the joined map, and the addresses that one run wrote and executed.
+ */
 export function findIdioms(byteAt: ImageByte, range: { start: number; end: number }, covered?: { map: Uint8Array; selfModified: Uint8Array }): Idioms {
   const coverage = covered?.map;
   const regions: NormalizedFindings["regions"] = [];

package/skills/c64-static-analysis/scripts/static-analysis.ts

@@ -4,7 +4,7 @@
 // structural findings into .c64-re-tools/knowledge/.
 
 import { COVERAGE, formatC64Address, prgRange } from "#src/c64.ts";
-import { Refusal } from "#src/errors.ts";
+import { isRefusal, Refusal } from "#src/errors.ts";
 import { WireFailure } from "#src/host-client/tools.ts";
 import { writeReported } from "#src/host-client/transfer.ts";
 import { knowledgeNames } from "#src/knowledge/names.ts";
@@ -121,6 +121,15 @@ function withRemedy(reason: string): string {
 
 /** One conflict: what the knowledge has, and what the analyzer found, under the analyzer's name. */
 function showConflict(conflict: ImportConflict, analyzer: Analyzer): Record<string, unknown> {
+  if (conflict.category === "code-evidence") {
+    return {
+      range: { start: formatC64Address(conflict.start), end: formatC64Address(conflict.end) },
+      [analyzer]: conflict.finding,
+      ...(conflict.executed === undefined ? {} : { executed: formatC64Address(conflict.executed) }),
+      ...(conflict.symbol === undefined ? {} : { knowledge: { ...conflict.symbol, address: formatC64Address(conflict.symbol.address) } }),
+      problem: conflict.executed !== undefined ? "a run executed code here: the data finding is not imported" : "the knowledge has a code name here: the data finding is not imported",
+    };
+  }
   if (conflict.category === "symbol") {
     return {
       at: formatC64Address(conflict.address),
@@ -184,7 +193,8 @@ async function analyze(call: Call): Promise<unknown> {
   // With coverage, a code finding over an executed instruction start replaces the data of the other analyzer.
   const executed = coverage === undefined ? {} : { executed: (address: number) => (coverage[address]! & COVERAGE.ramExecute) !== 0 };
   const loaded = imageKind === "flat64k" ? { start: 0x0000, end: 0xffff } : prgRange(bytes);
-  const idioms = findIdioms(byteAt, { start: loaded.start, end: loaded.end }, covered);
+  // The idioms read only the program: a table that runs past a segment gets no targets from the 0 bytes of the 64 KiB.
+  const idioms = findIdioms(inProgram, { start: loaded.start, end: loaded.end }, covered);
 
   const revision = currentRevision(reader);
   const seeds = seedsFromKnowledge(reader, entries, space);
@@ -269,7 +279,7 @@ async function analyze(call: Call): Promise<unknown> {
         written = writeReported(listingPath, listing, { what: "--out-listing" });
       } catch (error) {
         const recorded = imported.revision === null ? "The knowledge did not change." : `The knowledge has the findings as revision ${imported.revision}.`;
-        throw new WireFailure("operation-failed", `The script did not write the listing to ${listingPath}: ${(error as Error).message}. ${recorded}`);
+        throw new WireFailure(isRefusal(error) ? error.code : "operation-failed", `The script did not write the listing to ${listingPath}: ${(error as Error).message.replace(/\.$/, "")}. ${recorded}`);
       }
     }
     const found = idiomsResult();
@@ -281,7 +291,7 @@ async function analyze(call: Call): Promise<unknown> {
       coverage: result.coverage.map((range) => ({ start: formatC64Address(range.start), end: formatC64Address(range.end) })),
       ...imported,
       regions: result.regions.map((region) => ({ start: formatC64Address(region.start), end: formatC64Address(region.end), classification: region.classification })),
-      labels: listed(result.labels, max, (label) => ({ address: formatC64Address(label.address), name: label.name }), "c64-knowledge: knowledge.ts symbols"),
+      labels: listed(result.labels, max, (label) => ({ address: formatC64Address(label.address), name: label.name }), "c64-knowledge: knowledge.ts symbols --analyzer --origin dxa"),
       ...(written === undefined ? {} : { listing: written }),
     };
   }
@@ -311,7 +321,7 @@ async function analyze(call: Call): Promise<unknown> {
     ...(coverage === undefined ? {} : { coverageGaps: listed(gaps, max, formatC64Address, "a larger --max-listed") }),
     coverage: result.coverage.map((range) => ({ start: formatC64Address(range.start), end: formatC64Address(range.end) })),
     ...imported,
-    functions: listed(result.functions, max, (fn) => ({ entry: formatC64Address(fn.entry), name: fn.name }), "c64-knowledge: knowledge.ts symbols --kind routine"),
+    functions: listed(result.functions, max, (fn) => ({ entry: formatC64Address(fn.entry), name: fn.name }), "c64-knowledge: knowledge.ts symbols --analyzer --origin ghidra"),
     decompilations: result.decompilations.map((item) => ({ entry: formatC64Address(item.entry), text: item.text, ...(item.truncated ? { truncated: true } : {}) })),
     ...((result.notDecompiled ?? []).length === 0
       ? {}
@@ -333,9 +343,9 @@ await runScript({
         decompile: { kind: "address", value: "<address>", multiple: true, help: "ghidra: a routine to decompile, at most 32" },
         space: OPTIONS.space,
         "max-listed": { ...OPTIONS["max-listed"], help: "list at most n labels, functions or candidates (default 50, 0 for none). The counts are always given, and the knowledge has all labels and functions" },
-        "out-listing": { kind: "path", value: "<file>", help: "dxa: write the disassembly listing to this project file" },
-        "out-instructions": { kind: "path", value: "<file>", help: "ghidra: write every instruction (address, bytes, mnemonic, mode, operand, target, executed) to this project file as JSON" },
-        "out-macro-batch": { kind: "path", value: "<file>", help: "write the macro candidates as an apply batch of c64-knowledge (the macros and their uses) to this project file" },
+        "out-listing": { kind: "path", output: "file", value: "<file>", help: "dxa: write the disassembly listing to this project file" },
+        "out-instructions": { kind: "path", output: "file", value: "<file>", help: "ghidra: write every instruction (address, bytes, mnemonic, mode, operand, target, executed) to this project file as JSON" },
+        "out-macro-batch": { kind: "path", output: "file", value: "<file>", help: "write the macro candidates as an apply batch of c64-knowledge (the macros and their uses) to this project file" },
       },
       notes: "The program and the coverage come from the knowledge: put them in with knowledge.ts image and coverage of the c64-knowledge skill. The executed code of the coverage is seeded. Routine symbols in the knowledge are entry points too. Ghidra needs at least one entry point. DXA finds the SYS of a BASIC start by itself.",
       run: analyze,

package/skills/c64-testing/SKILL.md

@@ -30,6 +30,8 @@ node <skill>/scripts/testing.ts checklist tests/moves-left.json --item 'FIRE sta
 
 `testing.ts checklist` gives a Markdown checklist for a human tester in `text`.
 
+Before the emulator starts, `testing.ts test` examines each file that the scenario names: the program and the symbols of each side, and each snapshot file. A missing file gives `not-found`. A program must be a PRG. The script refuses a disk, tape or cartridge image.
+
 ### Runs
 
 A run does steps on one program, with no comparison, and keeps the results as files. Use it for many steps. One run replaces many emulator calls, for example a visit to each room of a game.
@@ -50,6 +52,8 @@ The run file gives where to start (a program, or a snapshot file), the steps, an
 | `{ "trace": "d011", "points": [{ "kind": "write", "start": "$d011" }], "frames": 100 }` | Record each hit of the points for 100 frames in `d011.json`: the instruction, its registers and the raster line and cycle. A point has the `kind` `exec`, `read` or `write`, a `start` and an optional `end`. `maxHits` limits the hits. |
 | `{ "observe": "all3", "memory": "all" }` | Write all 64 KiB of RAM to `all3.bin`, also the RAM under the ROMs and the I/O area. |
 
+Each step that writes a file must have its own file name. The script refuses a run where two steps write the same file, for example `{ "coverage": "x" }` and `{ "observe": "x", "memory": "all" }`, which both write `x.bin`. The script also examines the start program and each snapshot file before the emulator starts.
+
 A memory item can have `"view": "ram"`. Then it reads the RAM under the ROMs and the I/O area, for example code at `$e000` under the KERNAL. The default view `cpu` reads as the CPU sees memory. An item larger than 4096 bytes goes into its own `.bin` file, and `results.json` gives the file name.
 
 See [references/scenario-format.md](references/scenario-format.md), section Runs. `result` is PASS when all steps ran, and FAIL when a step did not run. `reason` tells why. A long run can need more time than one shell call. Then run it in the background.
@@ -116,9 +120,13 @@ node <skill>/scripts/testing.ts diff .c64-re-tools/local/runs/rooms/results.json
 
 - Two snapshot files: the script restores each one in its own emulator. It gives the changed ranges of the RAM and of the colour RAM, and the changed VIC-II, CIA, SID and CPU registers.
 - Two memory files of a run: the changed ranges. For a file of less than 64 KiB, `--address` gives the address of its first byte.
-- Two observations of runs, also of one run: the changed memory items and registers.
+- Two observations of runs, also of one run: the changed memory items and registers. `compared` gives the count of items that both observations have. With `compared: 0`, `reason` tells what each observation has. Then the diff compared nothing, and the observations are not equal.
+
+`registers` gives each changed register and chip value. `cpu.pc` is a `$` address. Each CPU flag is `cpu.flags.<flag>`, for example `cpu.flags.c`, with `true` or `false`.
+
+The script refuses a snapshot that does not restore in its emulator. An example is a PAL snapshot with `--ntsc`. Give `--ntsc` only for snapshots of an NTSC machine.
 
-`--gap N` joins ranges across N equal bytes. `--out` writes all ranges to a project file.
+`--gap N` joins ranges across N equal bytes. `--out` writes all ranges and all changed registers to a project file. The file uses the names of the result.
 
 ### Calling the script
 
@@ -127,6 +135,7 @@ node <skill>/scripts/testing.ts diff .c64-re-tools/local/runs/rooms/results.json
 - An address is `$` and 1 to 4 hex digits, for example `'$c000'` or `'$fb'`, or a name. A name is a symbol, else an analyzer name that only one analyzer gives. Quote an address in a shell.
 - A range is `<start>-<end>`, for example `'$c000-$cfff'`. In a JSON file, a range is `start` and `end`.
 - Bytes in a JSON file are hex in a field `data`, for example `"data": "a900"`.
+- The script examines each `--out` path before it starts work. It refuses a path in `.c64-re-tools/knowledge/` and the index of the knowledge, because only c64-knowledge writes them. It refuses a folder where the command writes a file, and a file where the command writes a folder.
 - The result is one JSON object on one line. A file that the script writes shows as `{"out", "bytes", "sha256"}`. A list shows as `{"count", "listed"}`, and `rest` tells where to get the rest of a long list.
 - An error is `{"error": {"code": "...", "message": "..."}}`. The message tells what is wrong and what to do.
 - The exit status is 0 for done or PASS, 1 for FAIL, 2 for an error and 3 for INCONCLUSIVE.
@@ -146,7 +155,7 @@ node <skill>/scripts/testing.ts diff .c64-re-tools/local/runs/rooms/results.json
 1. Write the scenario file. Start the steps with a frame count, for example `{ "frames": 150 }`. Each side starts at the reset vector, and the C64 needs these frames to start.
 2. Use symbol names, for example `player_x`, not fixed addresses. Each side finds the name in its own symbols. Use a `$` address only when both programs use the same address.
 3. Make each checkpoint a condition, for example a `runUntil` step with an `address` target at a routine. Do not wait for a time.
-4. Save the c64-assembler output of the rebuild to the symbol file, for example `build/game.json` in the project folder. Give `--all-symbols`, so that the file also has the constants that the source does not use. From the folder above the project, write `assembler.ts build --project <folder> ... > <folder>/build/game.json`.
+4. Write the symbol file of the rebuild with `--out-symbols build/game.json` of c64-assembler `build`. Give `--all-symbols`, so that the file also has the constants that the source does not use.
 5. Run `testing.ts test`.
 6. Read the result:
    - `PASS`: all observations are equal, or the differences are in the allowed tolerance.
@@ -180,6 +189,8 @@ PASS, FAIL or INCONCLUSIVE, the checkpoints and the smallest differences that ex
 ## Failure and conflicts
 
 - `invalid-input`: the scenario file has an error. The message names the step. Change the step and run the test again.
+- `invalid-input` that names the 2.0 form: a step, a memory item or a recording has a form of version 2.0. An example is `"at"` in place of `"address"`. The message gives the form to write. Change the file. Record a recording of version 2.0 again.
+- `not-found` before the run: a file of the scenario or the run file is missing, for example the rebuild program. Build the rebuild, or change the path.
 - INCONCLUSIVE, "did not reach the checkpoint": one side did not reach the checkpoint in the frame limit. Find the reason with the c64-emulator skill. Do not report a FAIL.
 - INCONCLUSIVE, "has no symbol": one side has no symbol with that name. Add the name to the knowledge or to the rebuild source.
 - INCONCLUSIVE, "no emulator" or "could not run": the host runtime or the emulator failed. Tell the user the message.

package/skills/c64-testing/references/recording-format.md

@@ -101,6 +101,8 @@ The page sends input only between Start and End. The input of the page goes into
 
 A site is the address of the instruction that reads. The values are hex. `<from>` and `<count>` are decimal.
 
+`testing.ts replay` checks `recording.json` before the replay core starts. A recording of version 2.0 can have events or a comparison in the old form, for example `"at"` in place of `"address"`. The script refuses it with `invalid-input`. The message names the event, the old field and the form to write. Record the session again.
+
 ## Replay
 
 `testing.ts replay` replays a recording folder:

package/skills/c64-testing/references/scenario-format.md

@@ -123,3 +123,9 @@ Example: visit two rooms from the same start.
 ## Checks
 
 The script checks the scenario before it starts the emulator. A step with an unknown key or a wrong value stops the script with `invalid-input`. The message names the step.
+
+The script also examines each file before it starts the emulator: the program and the symbols of each side, and each snapshot file. A missing file stops the script with `not-found`. A program that is not a PRG, for example a disk image, stops the script with `invalid-input`.
+
+A step in the form of version 2.0 stops the script with `invalid-input`. The message names the old field and gives the step to write. For example, the script refuses `{ "runUntil": { "at": "main_loop" }, "timeoutFrames": 300 }` and gives `{"runUntil": {"target": {"kind": "address", "address": "main_loop"}, "timeoutFrames": 300}}`. A memory item with `at` gets the same refusal: write `address`.
+
+In a run, two steps must not write the same file name. The script refuses such a run before it starts. Names that differ only in upper and lower case are the same name.