@henols/c64-re-tools 2.0.1 → 2.1.0-rc.2

package/skills/c64-provenance/scripts/provenance.ts

@@ -3,19 +3,22 @@
 // release is a PRG or an unpacker image.json. With --run, each release
 // runs in its own emulator, and each listed difference gets its first access
 // and whether the run changed it. The separate command gives evidence from
-// the program in the knowledge. The routine and match commands keep
+// the program in the knowledge, and the leftovers command lists what its
+// unused bytes hold: text, BASIC lines, fills and copies. The routine and match commands keep
 // routines that several programs share in a library folder and find them in the
 // program (D71).
 
-import { basename, posix, relative } from "node:path";
-import { parseArgs } from "node:util";
+import { basename, relative } from "node:path";
 
-import { formatC64Address, parseC64Address } from "#src/c64.ts";
-import { checkActionSteps, type ActionStep } from "#src/host-client/steps.ts";
-import { MAX_EXECUTION_COUNT, WireFailure } from "#src/host-client/tools.ts";
-import { readProjectFile, writeProjectFile } from "#src/host-client/transfer.ts";
+import { type AddressRange, c64AddressOf, formatC64Address } from "#src/c64.ts";
+import { Refusal } from "#src/errors.ts";
+import { parseEntryState, type EntryState } from "#src/host-client/entry-state.ts";
+import { readImage } from "#src/host-client/image.ts";
+import { readActionSteps, type ActionStep } from "#src/host-client/steps.ts";
+import { MAX_EXECUTION_COUNT } from "#src/host-client/tools.ts";
+import { writeReported } from "#src/host-client/transfer.ts";
 import { ViceSessionClient } from "#src/host-client/vice-session.ts";
-import { KnowledgeError, openForRead } from "#src/knowledge/database.ts";
+import { openForRead } from "#src/knowledge/database.ts";
 import { programCoverage, requireProgram } from "#src/knowledge/program.ts";
 import {
   commentsAt,
@@ -35,59 +38,17 @@ import {
   symbolNamed,
   symbolSuppressed,
 } from "#src/knowledge/read.ts";
-import { CODE_REGION_TYPES, type Origin } from "#src/knowledge/schema.ts";
+import { CODE_REGION_TYPES, SEMANTIC_ORIGINS } from "#src/knowledge/schema.ts";
 import { platformNames } from "#src/platform.ts";
 import { projectRoot } from "#src/project.ts";
-import { runScript, UsageError } from "#src/script.ts";
-import { compareReleases, shiftSuggestion, suggestShifts, type Range, type Release } from "./lib/compare.ts";
-import { findRoutine, libraryFolder, LibraryError, readLibrary, routineFrom, saveRoutine, suggest } from "./lib/library.ts";
-import { watchRelease, type ImageState, type Start } from "./lib/runtime.ts";
+import { knowledgeNames } from "#src/knowledge/names.ts";
+import { listed, OPTIONS, runScript, UsageError, type Call, type CommandSpec } from "#src/script.ts";
+import { compareReleases, shiftSuggestion, suggestShifts, type Release } from "./lib/compare.ts";
+import { findRoutine, libraryFolder, readLibrary, routineFrom, saveRoutine, suggest } from "./lib/library.ts";
+import { watchRelease, type Start } from "./lib/runtime.ts";
+import { findLeftovers } from "./lib/leftovers.ts";
 import { separate } from "./lib/separate.ts";
 
-const USAGE = `provenance.ts <release> <release> [<release> ...] [options]
-provenance.ts separate [--out <file>] [--max-listed <n>]
-provenance.ts routine --library <folder> (--module <address> | --start <address> --end <address>)
-                      [--name <name>] [--title <text>] [--replace]
-provenance.ts match --library <folder> [--out <file>] [--max-listed <n>]
-
-  --shift <n>:<bytes>   move release n (1 = the first) by a number of bytes, for example 2:-256
-  --from <address>      compare from this address
-  --to <address>        compare up to this address
-  --run <frames>        run each release in its own emulator for this many frames, and
-                        give each listed difference its first access and whether it changed
-  --steps <file>        with --run: action steps (JSON) to run before the frames
-
-separate gives evidence from the program in the knowledge: the unpack stage
-that wrote each range, its access in the coverage of the knowledge, whether call
-and jump references in knowledge reach each code region from the entry, and what
-the program reads or executes outside its segments. --out writes all ranges to a file;
-the result lists at most --max-listed of each list (default 50) and its count.
-
-routine saves a routine of the program in the knowledge to a library: a plain folder of routine
-files, beside the projects (--library, relative to the working directory, as
---project). It takes a module of the knowledge, or --start and --end. Its
-code regions give the instructions; the user and LLM names, regions, comments,
-references and operand expressions go into the file at offsets. --name defaults
-to the user or LLM name at the start, --title to the module title.
-
-match finds each routine of the library in the program of the knowledge: the opcodes and other
-fixed bytes equal, and the addresses into the routine moved by one shift.
-Operands that address something outside the routine may differ; the result
-lists them. --out writes the names of the routines at the found addresses as a
-batch for knowledge.ts apply, and the result lists what it left out.
-
-A release to compare is a PRG, placed at its load address, or an image.json of
-the c64-unpacker skill, compared inside its segments. Paths are relative to the
-project folder. The result is one JSON object.`;
-
-function address(text: string, option: string): number {
-  try {
-    return parseC64Address(text);
-  } catch {
-    throw new UsageError(`${option} must be $ followed by four hex digits, not ${text}`);
-  }
-}
-
 const hexBytes = (bytes: number[]) => Buffer.from(bytes).toString("hex").replace(/(..)(?!$)/g, "$1 ");
 
 interface LoadedRelease {
@@ -97,168 +58,66 @@ interface LoadedRelease {
   /** 64 KiB with the release bytes at their own addresses. */
   memory: Uint8Array;
   entry?: number;
-  stages?: Array<{ entry: number; wrote: Range[] }>;
-}
-
-/** Reads entry_state.json of the c64-unpacker skill for a start by writes. */
-function entryState(path: string): ImageState | string {
-  let value: Record<string, any>;
-  try {
-    value = JSON.parse(readProjectFile(path).bytes.toString("utf8")) as Record<string, any>;
-  } catch (error) {
-    if (error instanceof WireFailure && error.code === "not-found") return `It has no ${path}; capture it with c64-unpacker.`;
-    if (error instanceof WireFailure) throw error;
-    return `${path} is not JSON: ${(error as Error).message}`;
-  }
-  const registers = value.registers ?? {};
-  const byte = (item: unknown) => Number.isInteger(item) && (item as number) >= 0 && (item as number) <= 0xff;
-  const pages = ["$0000", "$0100", "$0200", "$0300"].map((page) => value.pages?.[page]);
-  const ok =
-    typeof registers.pc === "string" &&
-    /^\$[0-9a-f]{4}$/i.test(registers.pc) &&
-    ["a", "x", "y", "sp"].every((name) => byte(registers[name])) &&
-    typeof registers.flags === "object" &&
-    byte(value.cpuPort?.$00) &&
-    byte(value.cpuPort?.$01) &&
-    pages.every((page) => typeof page === "string" && /^[0-9a-f]{512}$/i.test(page));
-  if (!ok) return `${path} is not an entry_state.json of the c64-unpacker skill.`;
-  const flag = (name: string) => registers.flags[name] === true;
-  return {
-    pc: Number.parseInt(registers.pc.slice(1), 16),
-    a: registers.a,
-    x: registers.x,
-    y: registers.y,
-    sp: registers.sp,
-    flags: { n: flag("n"), v: flag("v"), d: flag("d"), i: flag("i"), z: flag("z"), c: flag("c") },
-    port: [value.cpuPort.$00, value.cpuPort.$01],
-    pages: Buffer.from(pages.join(""), "hex"),
-  };
+  stages?: Array<{ entry: number; wrote: AddressRange[] }>;
 }
 
-function readRelease(path: string, shift: number): LoadedRelease {
-  const { bytes } = readProjectFile(path);
-  const moved = (range: Range) => ({ start: range.start + shift, end: range.end + shift });
-  const inside = (range: Range) => range.start >= 0 && range.end <= 0xffff;
-  if (!path.endsWith(".json")) {
-    if (bytes.length < 3) throw new UsageError(`${path} is too short for a PRG`);
-    const load = bytes.readUInt16LE(0);
-    const segment = moved({ start: load, end: load + bytes.length - 3 });
-    if (!inside(segment)) throw new UsageError(`the shift moves ${path} outside $0000-$ffff`);
-    const memory = new Uint8Array(0x10000);
-    memory.set(bytes.subarray(2), load);
-    return { release: { name: path, segments: [segment], byteAt: (at) => bytes[2 + at - segment.start]! }, shift, start: { kind: "program", path }, memory };
-  }
-  const fail = (what: string): never => {
-    throw new UsageError(`${path} is not an image.json of the c64-unpacker skill: ${what}`);
-  };
-  let manifest: Record<string, unknown>;
-  try {
-    manifest = JSON.parse(bytes.toString("utf8")) as Record<string, unknown>;
-  } catch (error) {
-    return fail((error as Error).message);
+/**
+ * A release to compare: "knowledge" (the program of the knowledge, with its
+ * entry state and stages), a PRG at its load address, or an image.json of the
+ * c64-unpacker skill with the entry_state.json beside it.
+ */
+function readRelease(name: string, shift: number): LoadedRelease {
+  const moved = (range: AddressRange) => ({ start: range.start + shift, end: range.end + shift });
+  const inside = (range: AddressRange) => range.start >= 0 && range.end <= 0xffff;
+  let image: { segments: AddressRange[]; byteAt: (address: number) => number; memory?: Uint8Array; entry?: number; stages?: Array<Record<string, unknown>>; state?: EntryState };
+  let noState: string;
+  if (name === "knowledge") {
+    const db = openForRead();
+    try {
+      const program = requireProgram(db);
+      image = { ...program, ...(program.state === undefined ? {} : { state: parseEntryState(program.state, "the entry state of the knowledge") }) } as typeof image;
+    } finally {
+      db?.close();
+    }
+    noState = "The knowledge has no entry state: put a c64-unpacker capture in with knowledge.ts image <image.json>.";
+  } else {
+    image = readImage(name);
+    noState = `It has no entry_state.json beside it; capture it with c64-unpacker.`;
   }
-  const parse = (value: unknown) => (typeof value === "string" && /^\$[0-9a-f]{4}$/i.test(value) ? Number.parseInt(value.slice(1), 16) : fail("a segment needs start and end as $ addresses"));
-  if (typeof manifest.image !== "string" || !Array.isArray(manifest.segments) || manifest.segments.length === 0) fail("it needs image and segments");
-  const own = (manifest.segments as Array<Record<string, unknown>>).map((segment) => ({ start: parse(segment.start), end: parse(segment.end) })).sort((a, b) => a.start - b.start);
-  const directory = posix.dirname(path);
-  const memory = readProjectFile(posix.join(directory, manifest.image as string)).bytes;
-  if (memory.length !== 0x10000) fail(`its image has ${memory.length} bytes, not 65536`);
-  const segments = own.map(moved);
-  if (!segments.every(inside)) throw new UsageError(`the shift moves ${path} outside $0000-$ffff`);
-  const state = entryState(posix.join(directory, "entry_state.json"));
-  const stages = Array.isArray(manifest.stages)
-    ? (manifest.stages as Array<Record<string, unknown>>).map((stage) => ({
-        entry: parse(stage.entry),
-        wrote: (Array.isArray(stage.wrote) ? (stage.wrote as Array<Record<string, unknown>>) : []).map((range) => ({ start: parse(range.start), end: parse(range.end) })),
-      }))
-    : undefined;
+  const segments = image.segments.map(moved);
+  if (!segments.every(inside)) throw new UsageError(`the shift moves ${name} outside $0000-$ffff`);
+  const memory = image.memory ?? new Uint8Array(0x10000);
+  if (image.memory === undefined) for (const segment of image.segments) for (let at = segment.start; at <= segment.end; at++) memory[at] = image.byteAt(at);
+  const prg = name !== "knowledge" && !name.endsWith(".json");
+  const stages = stagesOf(image.stages, name);
   return {
-    release: { name: path, segments, byteAt: (at) => memory[at - shift]! },
+    release: { name, segments, byteAt: (at) => memory[at - shift]! },
     shift,
-    start: typeof state === "string" ? { kind: "none", reason: state } : { kind: "image", segments: own, byteAt: (at) => memory[at]!, state },
+    start: prg ? { kind: "program", path: name } : image.state === undefined ? { kind: "none", reason: noState } : { kind: "image", segments: image.segments, byteAt: (at) => memory[at]!, state: image.state },
     memory,
-    ...(manifest.entry === undefined ? {} : { entry: parse(manifest.entry) }),
+    ...(image.entry === undefined ? {} : { entry: image.entry }),
     ...(stages === undefined ? {} : { stages }),
   };
 }
 
-async function run(argv: string[]): Promise<unknown> {
-  const { values, positionals } = parseArgs({
-    args: argv,
-    strict: true,
-    allowPositionals: true,
-    options: {
-      shift: { type: "string", multiple: true },
-      from: { type: "string" },
-      to: { type: "string" },
-      run: { type: "string" },
-      steps: { type: "string" },
-      out: { type: "string" },
-      "max-listed": { type: "string" },
-      library: { type: "string" },
-      module: { type: "string" },
-      start: { type: "string" },
-      end: { type: "string" },
-      name: { type: "string" },
-      title: { type: "string" },
-      replace: { type: "boolean" },
-      help: { type: "boolean" },
-    },
-  });
-  if (values.help) throw new UsageError("");
-  /** Refuses the options that `command` does not take. */
-  const only = (command: string, allowed: readonly string[]) => {
-    const used = Object.keys(values).filter((option) => option !== "help" && !allowed.includes(option));
-    if (used.length > 0) throw new UsageError(`${command} does not take --${used.join(", --")}`);
-  };
-  if (positionals[0] === "separate") {
-    only("separate", ["out", "max-listed"]);
-    if (positionals.length !== 1) throw new UsageError("separate takes no release: it reads the program from the knowledge");
-    return separateCommand(values.out, values["max-listed"]);
-  }
-  if (positionals[0] === "routine") {
-    only("routine", ["library", "module", "start", "end", "name", "title", "replace"]);
-    if (positionals.length !== 1) throw new UsageError("routine takes no image: it reads the program from the knowledge");
-    return routineCommand(values);
-  }
-  if (positionals[0] === "match") {
-    only("match", ["library", "out", "max-listed"]);
-    if (positionals.length !== 1) throw new UsageError("match takes no image: it reads the program from the knowledge");
-    return matchCommand(values.library, values.out, values["max-listed"]);
-  }
-  only("a comparison", ["shift", "from", "to", "run", "steps"]);
-  if (positionals.length < 2) throw new UsageError("give two or more releases");
+/** compare: the releases byte by byte, and with --frames the first access of each difference in each release. */
+async function compare(call: Call): Promise<unknown> {
+  const positionals = call.subjects;
+  if (positionals.length < 2) throw new UsageError("compare needs two or more releases");
   const shifts = new Map<number, number>();
-  for (const text of values.shift ?? []) {
+  for (const text of call.strings("shift")) {
     const match = /^(\d+):([+-]?\d+)$/.exec(text);
     if (match === null || Number(match[1]) < 1 || Number(match[1]) > positionals.length) throw new UsageError(`--shift must be <release number>:<bytes>, not ${text}`);
     shifts.set(Number(match[1]), Number(match[2]));
   }
-  if (values.steps !== undefined && values.run === undefined) throw new UsageError("--steps works only with --run");
-  let frames: number | undefined;
-  if (values.run !== undefined) {
-    if (!/^\d{1,6}$/.test(values.run) || Number(values.run) < 1 || Number(values.run) > MAX_EXECUTION_COUNT) throw new UsageError(`--run needs a frame count from 1 to ${MAX_EXECUTION_COUNT}, not ${values.run}`);
-    frames = Number(values.run);
-  }
-  let steps: ActionStep[] = [];
-  if (values.steps !== undefined) {
-    let parsed: unknown;
-    try {
-      parsed = JSON.parse(readProjectFile(values.steps).bytes.toString("utf8"));
-    } catch (error) {
-      if (error instanceof WireFailure) throw error;
-      throw new UsageError(`the steps file ${values.steps} is not JSON: ${(error as Error).message}`);
-    }
-    steps = checkActionSteps(parsed, (message) => {
-      throw new UsageError(`steps file ${values.steps}: ${message}`);
-    });
-  }
+  const frames = call.count("frames");
+  const stepsFile = call.string("steps");
+  if (stepsFile !== undefined && frames === undefined) throw new UsageError("--steps works only with --frames");
+  const steps: ActionStep[] = stepsFile === undefined ? [] : readActionSteps(stepsFile, "--steps");
   const loaded = positionals.map((path, index) => readRelease(path, shifts.get(index + 1) ?? 0));
   const releases = loaded.map((item) => item.release);
-  const comparison = compareReleases(releases, {
-    ...(values.from === undefined ? {} : { from: address(values.from, "--from") }),
-    ...(values.to === undefined ? {} : { to: address(values.to, "--to") }),
-  });
+  const range = call.range("range");
+  const comparison = compareReleases(releases, range === undefined ? {} : { from: range.start, to: range.end });
   const compared = comparison.identicalBytes + comparison.differingBytes;
   const identicalShare = compared === 0 ? 0 : comparison.identicalBytes / compared;
   // A poor match at the load addresses can mean that the releases are placed differently.
@@ -278,7 +137,7 @@ async function run(argv: string[]): Promise<unknown> {
   const notSuggested = suggestions.filter((suggestion) => "reason" in suggestion);
 
   // The runtime evidence: one emulator for each release, the differences at the release's own addresses.
-  const runtime = new Map<string, Array<Range & { first: string; changed: boolean }>>();
+  const runtime = new Map<string, Array<AddressRange & { first: string; changed: boolean }>>();
   const runs: Record<string, unknown> = {};
   if (frames !== undefined && comparison.differences.length > 0) {
     for (const item of loaded) {
@@ -334,12 +193,13 @@ async function run(argv: string[]): Promise<unknown> {
   };
 }
 
-/** The stages of the capture in the knowledge as ranges; a stage that is not a capture stage is refused. */
-function stagesOf(stages: Array<Record<string, unknown>> | undefined): Array<{ entry: number; wrote: Range[] }> | undefined {
+/** The stages of a capture as ranges; a stage that is not a capture stage is refused. `where` names the capture. */
+function stagesOf(stages: Array<Record<string, unknown>> | undefined, where = "the program in the knowledge"): Array<{ entry: number; wrote: AddressRange[] }> | undefined {
   if (stages === undefined) return undefined;
   const parse = (value: unknown) => {
-    if (typeof value === "string" && /^\$[0-9a-f]{4}$/i.test(value)) return Number.parseInt(value.slice(1), 16);
-    throw new UsageError("the stages of the program in the knowledge are not the stages of a c64-unpacker capture; put the capture in again with knowledge.ts image");
+    const at = typeof value === "string" ? c64AddressOf(value) : undefined;
+    if (at !== undefined) return at;
+    throw new Refusal("invalid-input", `the stages of ${where} are not the stages of a c64-unpacker capture; capture it again with c64-unpacker`);
   };
   return stages.map((stage) => ({
     entry: parse(stage.entry),
@@ -347,10 +207,8 @@ function stagesOf(stages: Array<Record<string, unknown>> | undefined): Array<{ e
   }));
 }
 
-/** separate: evidence from the program in the knowledge, with all ranges in --out and the first ones in the result. */
-function separateCommand(out: string | undefined, maxListed: string | undefined): unknown {
-  const max = maxListed ?? "50";
-  if (!/^\d{1,5}$/.test(max)) throw new UsageError(`--max-listed must be a whole number, not ${max}`);
+/** The program in the knowledge with its stages and coverage, and the separation of its bytes. */
+function programEvidence() {
   const db = openForRead();
   let program: ReturnType<typeof requireProgram>;
   let coverage: Uint8Array | undefined;
@@ -365,7 +223,6 @@ function separateCommand(out: string | undefined, maxListed: string | undefined)
     db?.close();
   }
   const stages = stagesOf(program.stages);
-  const loaded = { entry: program.entry, stages };
   const found = separate({
     segments: program.segments,
     memory: program.memory,
@@ -375,68 +232,121 @@ function separateCommand(out: string | undefined, maxListed: string | undefined)
     codeRegions: regions,
     references,
   });
-  const show = (range: Range) => ({ start: formatC64Address(range.start), end: formatC64Address(range.end) });
+  return { program, stages, coverage, found };
+}
+
+const NO_STAGES = "The program in the knowledge has no stage log. Only a capture of c64-unpacker gives one. A capture of loads holds one part of a program and has none.";
+
+/** separate: evidence from the program in the knowledge, with all ranges in --out and the first ones in the result. */
+function separateCommand(call: Call): unknown {
+  const out = call.string("out");
+  const max = call.count("max-listed")!;
+  const { program, stages, coverage, found } = programEvidence();
+  const loaded = { entry: program.entry, stages };
+  const show = (range: AddressRange) => ({ start: formatC64Address(range.start), end: formatC64Address(range.end) });
   const ranges = found.ranges.map((range) => ({ ...show(range), ...(range.stage === undefined ? {} : { stage: range.stage }), ...(range.access === undefined ? {} : { access: range.access }) }));
   const codeRegions = found.codeRegions.map((region) => ({ ...show(region), type: region.type, ...(region.reachable === undefined ? {} : { reachable: region.reachable }), ...(region.executed === undefined ? {} : { executed: region.executed }) }));
   const outside = { read: found.outside.read.map(show), executed: found.outside.executed.map(show), generated: found.outside.generated.map(show) };
-  const listed = <T>(items: T[]) => ({ count: items.length, listed: items.slice(0, Number(max)) });
-  if (out !== undefined) writeProjectFile(out, Buffer.from(`${JSON.stringify({ ranges, codeRegions, outside }, null, 1)}\n`), { what: "--out" });
+  const written = out === undefined ? undefined : writeReported(out, `${JSON.stringify({ ranges, codeRegions, outside }, null, 1)}\n`, { what: "--out" });
+  const rest = out === undefined ? undefined : out;
+  const list = <T>(items: T[]) => listed(items, max, undefined, rest);
   const bytes = (filter: (range: (typeof found.ranges)[number]) => boolean) => found.ranges.filter(filter).reduce((sum, range) => sum + range.end - range.start + 1, 0);
   const lastStage = loaded.stages === undefined ? undefined : loaded.stages.length - 1;
   return {
     ...(loaded.entry === undefined ? {} : { entry: formatC64Address(loaded.entry) }),
     ...(lastStage === undefined
-      ? { noStages: "The program in the knowledge has no stage log. Only a capture of c64-unpacker gives one. A capture of loads holds one part of a program and has none." }
+      ? { noStages: NO_STAGES }
       : {
           stages: lastStage + 1,
           bytesByStage: Object.fromEntries([...[...Array(lastStage + 1).keys()].map((stage): [string, number] => [String(stage), bytes((range) => range.stage === stage)]), ["none", bytes((range) => range.stage === undefined)]]),
         }),
     ...(coverage === undefined ? {} : { bytesByAccess: Object.fromEntries((["executed", "read", "written", "read-written", "untouched"] as const).map((access) => [access, bytes((range) => range.access === access)])) }),
-    ranges: listed(ranges),
+    ranges: list(ranges),
     // total counts every code region; unreachable is a list like the others.
     codeRegions: {
       total: codeRegions.length,
-      ...(loaded.entry === undefined ? {} : { unreachable: listed(codeRegions.filter((region) => region.reachable === false)) }),
+      ...(loaded.entry === undefined ? {} : { unreachable: list(codeRegions.filter((region) => region.reachable === false)) }),
     },
-    ...(coverage === undefined ? {} : { outside: { read: listed(outside.read), executed: listed(outside.executed), generated: listed(outside.generated) } }),
-    ...(out === undefined ? {} : { out }),
+    ...(coverage === undefined ? {} : { outside: { read: list(outside.read), executed: list(outside.executed), generated: list(outside.generated) } }),
+    ...(written === undefined ? {} : written),
   };
 }
 
-const SEMANTIC: readonly Origin[] = ["user", "llm"];
+/** The longest text that the result shows; --out has all of it. */
+const SHOWN_TEXT = 120;
+
+/** leftovers: what the program in the knowledge holds besides its working code, with all findings in --out and the first ones in the result. */
+function leftoversCommand(call: Call): unknown {
+  const out = call.string("out");
+  const max = call.count("max-listed")!;
+  const min = call.count("min-text")!;
+  const { program, stages, coverage, found } = programEvidence();
+  const leftovers = findLeftovers({
+    segments: program.segments,
+    memory: program.memory,
+    separation: found,
+    ...(program.segments.some((segment) => segment.start === 0x0801) ? { basicStart: 0x0801 } : {}),
+    minText: min,
+  });
+  const show = <T extends AddressRange & { stages?: number[]; access?: string[] }>({ start, end, stages: wrote, access, ...rest }: T) => ({
+    start: formatC64Address(start),
+    end: formatC64Address(end),
+    ...rest,
+    ...(wrote === undefined ? {} : { stages: wrote }),
+    ...(access === undefined ? {} : { access }),
+  });
+  const all = {
+    text: leftovers.text.map(show),
+    basic: leftovers.basic.map(({ linkedAt, ...rest }) => ({ ...show(rest), linkedAt: formatC64Address(linkedAt) })),
+    fill: leftovers.fill.map(({ pattern, ...rest }) => ({ ...show(rest), pattern: hexBytes(pattern) })),
+    residue: leftovers.residue.map((range) => ({ ...show(range), bytes: hexBytes([...program.memory.subarray(range.start, Math.min(range.end, range.start + 15) + 1)]) + (range.end - range.start >= 16 ? " ..." : "") })),
+    copies: leftovers.copies.map(({ copyOf, ...rest }) => ({ ...show(rest), copyOf: formatC64Address(copyOf) })),
+    deadCode: leftovers.deadCode.map(show),
+  };
+  const written = out === undefined ? undefined : writeReported(out, `${JSON.stringify(all, null, 1)}\n`, { what: "--out" });
+  const list = <T>(items: T[]) => listed(items, max, undefined, out ?? "a larger --max-listed, or --out <file>");
+  return {
+    ...(program.entry === undefined ? { noEntry: "The program in the knowledge has no entry, so deadCode is empty." } : { entry: formatC64Address(program.entry) }),
+    ...(stages === undefined ? { noStages: NO_STAGES } : { stages: stages.length }),
+    ...(coverage === undefined ? { noCoverage: "The knowledge has no coverage, so residue is empty and no finding has access. Add coverage with coverage of the c64-knowledge skill." } : {}),
+    text: list(all.text.map((item) => (item.text.length > SHOWN_TEXT ? { ...item, text: `${item.text.slice(0, SHOWN_TEXT)} ...`, length: item.text.length } : item))),
+    basic: list(all.basic),
+    fill: list(all.fill),
+    residue: list(all.residue),
+    copies: list(all.copies),
+    deadCode: list(all.deadCode),
+    ...(written === undefined ? {} : written),
+  };
+}
 
 const platform = () => new Map(platformNames().map((entry) => [entry.name, entry.address]));
 
 /** routine: saves a module or a range of the program in the knowledge, with its knowledge, as a routine of the library (D71). */
-function routineCommand(values: { library?: string; module?: string; start?: string; end?: string; name?: string; title?: string; replace?: boolean }): unknown {
-  if (values.library === undefined) throw new UsageError("routine needs --library <folder>");
-  if (values.module !== undefined && (values.start !== undefined || values.end !== undefined)) throw new UsageError("routine takes --module, or --start and --end, not both");
-  if (values.module === undefined && values.start === undefined && values.end === undefined) throw new UsageError("routine needs --module <address>, or --start and --end");
-  if (values.module === undefined && (values.start === undefined || values.end === undefined)) throw new UsageError("routine needs both --start and --end");
+function routineCommand(call: Call): unknown {
+  const given = call.subject(0);
+  const library = call.string("library")!;
   const db = openForRead();
   try {
     const image = requireProgram(db);
     const segmentOf = (at: number) => image.segments.find((segment) => at >= segment.start && at <= segment.end);
     let start: number;
     let end: number;
-    let title = values.title;
-    if (values.module !== undefined) {
-      start = address(values.module, "--module");
+    let title = call.string("title");
+    if (!given.includes("-")) {
+      start = call.parseAddress(given, "<module>");
       const modules = listModules(db);
       const module = modules.find((item) => item.start === start);
-      if (module === undefined) throw new LibraryError("not-found", `No module starts at ${formatC64Address(start)}. List the modules with knowledge.ts modules, or give --start and --end.`);
+      if (module === undefined) throw new Refusal("not-found", `No module starts at ${formatC64Address(start)}. List the modules with knowledge.ts modules, or give a range <start>-<end>.`);
       const segment = segmentOf(start);
       if (segment === undefined) throw new UsageError(`The module at ${formatC64Address(start)} is outside the segments of the program.`);
       end = (modules.find((item) => item.start > start && item.start <= segment.end)?.start ?? segment.end + 1) - 1;
       title ??= module.title;
     } else {
-      start = address(values.start!, "--start");
-      end = address(values.end!, "--end");
-      if (end < start) throw new UsageError("--end is before --start");
+      ({ start, end } = call.parseRange(given, "<range>"));
       if (segmentOf(start) === undefined || segmentOf(start) !== segmentOf(end)) throw new UsageError(`${formatC64Address(start)}-${formatC64Address(end)} is not inside one segment of the program.`);
     }
     const range = { start, end };
-    const name = values.name ?? listSymbols(db, { start, end: start }, undefined, SEMANTIC)[0]?.name;
+    const name = call.string("name") ?? listSymbols(db, { start, end: start }, undefined, SEMANTIC_ORIGINS)[0]?.name;
     if (name === undefined) throw new UsageError(`The knowledge has no user or LLM name at ${formatC64Address(start)}: give --name.`);
     const { routine, leftOut } = routineFrom({
       name,
@@ -447,17 +357,17 @@ function routineCommand(values: { library?: string; module?: string; start?: str
       end,
       byteAt: image.byteAt,
       analyzed: listRegions(db, range),
-      symbols: listSymbols(db, range, undefined, SEMANTIC),
-      regions: listRegions(db, range, undefined, SEMANTIC),
-      comments: listComments(db, range, SEMANTIC),
-      references: listReferences(db, range, undefined, SEMANTIC),
-      operands: listOperands(db, range, SEMANTIC),
+      symbols: listSymbols(db, range, undefined, SEMANTIC_ORIGINS),
+      regions: listRegions(db, range, undefined, SEMANTIC_ORIGINS),
+      comments: listComments(db, range, SEMANTIC_ORIGINS),
+      references: listReferences(db, range, undefined, SEMANTIC_ORIGINS),
+      operands: listOperands(db, range, SEMANTIC_ORIGINS),
       platform: platform(),
     });
-    const file = saveRoutine(libraryFolder(values.library, { create: true }), routine, values.replace === true);
+    const file = saveRoutine(libraryFolder(library, { create: true, cwd: projectRoot() }), routine, call.flag("replace"));
     return {
       routine: routine.name,
-      file: relative(process.cwd(), file),
+      file: relative(projectRoot(), file).split("\\").join("/"),
       start: formatC64Address(start),
       end: formatC64Address(end),
       length: routine.bytes.length,
@@ -473,13 +383,13 @@ function routineCommand(values: { library?: string; module?: string; start?: str
 }
 
 /** match: finds the routines of the library in the program of the knowledge, and with --out writes their names there as an apply batch (D71). */
-function matchCommand(library: string | undefined, out: string | undefined, maxListed: string | undefined): unknown {
-  if (library === undefined) throw new UsageError("match needs --library <folder>");
-  const max = maxListed ?? "50";
-  if (!/^\d{1,5}$/.test(max)) throw new UsageError(`--max-listed must be a whole number, not ${max}`);
-  const listed = <T>(items: T[]) => ({ count: items.length, listed: items.slice(0, Number(max)) });
-  const routines = readLibrary(libraryFolder(library, { create: false }));
-  if (routines.length === 0) throw new LibraryError("not-found", `The library ${library} has no routine files. Save a routine with routine first.`);
+function matchCommand(call: Call): unknown {
+  const library = call.string("library")!;
+  const out = call.string("out");
+  const max = call.count("max-listed")!;
+  const list = <T>(items: T[]) => listed(items, max);
+  const routines = readLibrary(libraryFolder(library, { create: false, cwd: projectRoot() }));
+  if (routines.length === 0) throw new Refusal("not-found", `The library ${library} has no routine files. Save a routine with routine first.`);
   const programDb = openForRead();
   let image: ReturnType<typeof requireProgram>;
   try {
@@ -500,7 +410,7 @@ function matchCommand(library: string | undefined, out: string | undefined, maxL
       })),
     )
     .sort((a, b) => a.start.localeCompare(b.start));
-  const result: Record<string, unknown> = { library, routines: routines.length, matches: listed(matches) };
+  const result: Record<string, unknown> = { library, routines: routines.length, matches: list(matches) };
   if (out === undefined) return result;
   const db = openForRead();
   let suggestions: ReturnType<typeof suggest>;
@@ -525,12 +435,72 @@ function matchCommand(library: string | undefined, out: string | undefined, maxL
     db?.close();
   }
   const { batch, known, leftOut } = suggestions;
-  writeProjectFile(out, Buffer.from(batch.length === 0 ? "[]\n" : `[\n${batch.map((write) => ` ${JSON.stringify(write)}`).join(",\n")}\n]\n`), { what: "--out" });
+  const written = writeReported(out, batch.length === 0 ? "[]\n" : `[\n${batch.map((write) => ` ${JSON.stringify(write)}`).join(",\n")}\n]\n`, { what: "--out" });
   return {
     ...result,
-    batch: { out, writes: batch.length, known, leftOut: listed(leftOut) },
+    batch: { ...written, writes: batch.length, known, leftOut: list(leftOut) },
     ...(batch.length === 0 ? {} : { apply: `knowledge.ts apply ${out} --reason "names of library routines"` }),
   };
 }
 
-await runScript((argv) => run(argv), { usage: USAGE, refusals: [KnowledgeError, LibraryError] });
+const LIBRARY = { kind: "text", value: "<folder>", required: true, help: "the library: a plain folder of routine files beside the projects, relative to the project folder, for example ../library" } as const;
+
+const COMMANDS: CommandSpec[] = [
+  {
+    name: "compare",
+    summary: "Compares releases byte by byte, in their common ranges, and suggests shifts when they are placed differently.",
+    subjects: [{ name: "<release>", help: "knowledge (the program in the knowledge), a PRG at its load address, or an image.json of the c64-unpacker skill with the entry_state.json beside it. Two or more", list: true }],
+    options: {
+      shift: { kind: "text", value: "<n>:<bytes>", multiple: true, help: "move release n (1 is the first) by a number of bytes, for example 2:-256" },
+      range: { ...OPTIONS.range, help: "compare only the addresses from start to end" },
+      frames: { ...OPTIONS.frames, max: MAX_EXECUTION_COUNT, help: "run each release in its own emulator for n frames, and give each listed difference its first access and whether it changed" },
+      steps: { ...OPTIONS.steps, help: "with --frames: action steps (JSON) to run before the frames" },
+    },
+    run: compare,
+  },
+  {
+    name: "separate",
+    summary: "Evidence from the program in the knowledge: the unpack stage that wrote each range, its access in the coverage, whether call and jump references reach each code region from the entry, and what the program reads or executes outside its segments.",
+    options: { out: { ...OPTIONS.out, value: "<file>", help: "write all ranges to this file" }, "max-listed": OPTIONS["max-listed"] },
+    run: separateCommand,
+  },
+  {
+    name: "leftovers",
+    summary: "What the program in the knowledge holds besides its working code: texts in PETSCII, screen codes, ASCII or with bit 7 set (Apple II), assembler source, BASIC lines away from the BASIC start, fills, untouched bytes outside code, copies of earlier bytes, and code regions that nothing reaches and no run executed.",
+    options: {
+      out: { ...OPTIONS.out, value: "<file>", help: "write all findings to this file" },
+      "min-text": { kind: "count", value: "<n>", min: 4, max: 999, default: 8, help: "the shortest text to find (default 8)" },
+      "max-listed": OPTIONS["max-listed"],
+    },
+    run: leftoversCommand,
+  },
+  {
+    name: "routine",
+    summary: "Saves a routine of the program in the knowledge to a library, with its user and LLM names, regions, comments, references and operand expressions at offsets.",
+    subjects: [{ name: "<module|range>", help: "the start address of a module of the knowledge, or a range <start>-<end>" }],
+    options: {
+      library: LIBRARY,
+      name: { kind: "text", value: "<name>", help: "the routine name. The default is the user or LLM name at the start" },
+      title: { kind: "text", value: "<text>", help: "the title. The default is the module title" },
+      replace: { kind: "flag", help: "replace a routine file of the same name" },
+    },
+    run: routineCommand,
+  },
+  {
+    name: "match",
+    summary: "Finds each routine of the library in the program of the knowledge: the opcodes and other fixed bytes equal, and the addresses into the routine moved by one shift.",
+    options: {
+      library: { ...LIBRARY, help: "the library folder, relative to the project folder, for example ../library" },
+      out: { ...OPTIONS.out, value: "<file>", help: "write the names of the routines at the found addresses as a batch for knowledge.ts apply" },
+      "max-listed": OPTIONS["max-listed"],
+    },
+    run: matchCommand,
+  },
+];
+
+await runScript({
+  name: "provenance",
+  summary: "Where the parts of a program come from: a comparison of releases, the separation of the program in the knowledge, its leftovers, and a library of known routines.",
+  names: knowledgeNames,
+  commands: COMMANDS,
+});