@henols/c64-re-tools 1.1.0 → 2.0.0-rc.11

package/skills/c64-provenance/scripts/provenance.ts

@@ -0,0 +1,334 @@
+// The c64-provenance script: compares two or more releases byte by byte at
+// their addresses and reports where they differ and which releases agree. A
+// release is a PRG or an unpacker image.json (D50). With --run, each release
+// runs in its own emulator, and each listed difference gets its first access
+// and whether the run changed it. The separate command gives evidence from
+// one release (D51).
+
+import { posix } from "node:path";
+import { parseArgs } from "node:util";
+
+import { formatC64Address, parseC64Address } from "#src/c64.ts";
+import { readCoverageFiles } from "#src/host-client/coverage.ts";
+import { checkActionSteps, type ActionStep } from "#src/host-client/steps.ts";
+import { MAX_EXECUTION_COUNT, WireFailure } from "#src/host-client/tools.ts";
+import { readProjectFile, writeProjectFile } from "#src/host-client/transfer.ts";
+import { ViceSessionClient } from "#src/host-client/vice-session.ts";
+import { KnowledgeError, openForRead } from "#src/knowledge/database.ts";
+import { listReferences, listRegions } from "#src/knowledge/read.ts";
+import { CODE_REGION_TYPES } from "#src/knowledge/schema.ts";
+import { runScript, UsageError } from "#src/script.ts";
+import { compareReleases, shiftSuggestion, suggestShifts, type Range, type Release } from "./compare.ts";
+import { watchRelease, type ImageState, type Start } from "./runtime.ts";
+import { separate } from "./separate.ts";
+
+const USAGE = `provenance.ts <release> <release> [<release> ...] [options]
+provenance.ts separate <release> [--coverage <file> ...] [--out <file>] [--max-listed <n>]
+
+  --shift <n>:<bytes>   move release n (1 = the first) by a number of bytes, for example 2:-256
+  --from <address>      compare from this address
+  --to <address>        compare up to this address
+  --run <frames>        run each release in its own emulator for this many frames, and
+                        give each listed difference its first access and whether it changed
+  --steps <file>        with --run: action steps (JSON) to run before the frames
+
+separate gives evidence from one release: the unpack stage that wrote each range,
+its access in the --coverage files (64 KiB each, repeatable), whether call and
+jump references in knowledge reach each code region from the entry, and what the
+program reads or executes outside the image. --out writes all ranges to a file;
+the result lists at most --max-listed of each list (default 50) and its count.
+
+A release is a PRG, placed at its load address, or an image.json of the
+c64-unpacker skill, compared inside its segments. Paths are relative to the
+project directory. The result is one JSON object.`;
+
+function address(text: string, option: string): number {
+  try {
+    return parseC64Address(text);
+  } catch {
+    throw new UsageError(`${option} must be $ followed by four hex digits, not ${text}`);
+  }
+}
+
+const hexBytes = (bytes: number[]) => Buffer.from(bytes).toString("hex").replace(/(..)(?!$)/g, "$1 ");
+
+interface LoadedRelease {
+  release: Release;
+  shift: number;
+  start: Start | { kind: "none"; reason: string };
+  /** 64 KiB with the release bytes at their own addresses. */
+  memory: Uint8Array;
+  entry?: number;
+  stages?: Array<{ entry: number; wrote: Range[] }>;
+}
+
+/** Reads entry_state.json of the c64-unpacker skill for a start by writes. */
+function entryState(path: string): ImageState | string {
+  let value: Record<string, any>;
+  try {
+    value = JSON.parse(readProjectFile(path).bytes.toString("utf8")) as Record<string, any>;
+  } catch (error) {
+    if (error instanceof WireFailure && error.code === "not-found") return `It has no ${path}; capture it with c64-unpacker.`;
+    if (error instanceof WireFailure) throw error;
+    return `${path} is not JSON: ${(error as Error).message}`;
+  }
+  const registers = value.registers ?? {};
+  const byte = (item: unknown) => Number.isInteger(item) && (item as number) >= 0 && (item as number) <= 0xff;
+  const pages = ["$0000", "$0100", "$0200", "$0300"].map((page) => value.pages?.[page]);
+  const ok =
+    typeof registers.pc === "string" &&
+    /^\$[0-9a-f]{4}$/i.test(registers.pc) &&
+    ["a", "x", "y", "sp"].every((name) => byte(registers[name])) &&
+    typeof registers.flags === "object" &&
+    byte(value.cpuPort?.$00) &&
+    byte(value.cpuPort?.$01) &&
+    pages.every((page) => typeof page === "string" && /^[0-9a-f]{512}$/i.test(page));
+  if (!ok) return `${path} is not an entry_state.json of the c64-unpacker skill.`;
+  const flag = (name: string) => registers.flags[name] === true;
+  return {
+    pc: Number.parseInt(registers.pc.slice(1), 16),
+    a: registers.a,
+    x: registers.x,
+    y: registers.y,
+    sp: registers.sp,
+    flags: { n: flag("n"), v: flag("v"), d: flag("d"), i: flag("i"), z: flag("z"), c: flag("c") },
+    port: [value.cpuPort.$00, value.cpuPort.$01],
+    pages: Buffer.from(pages.join(""), "hex"),
+  };
+}
+
+function readRelease(path: string, shift: number): LoadedRelease {
+  const { bytes } = readProjectFile(path);
+  const moved = (range: Range) => ({ start: range.start + shift, end: range.end + shift });
+  const inside = (range: Range) => range.start >= 0 && range.end <= 0xffff;
+  if (!path.endsWith(".json")) {
+    if (bytes.length < 3) throw new UsageError(`${path} is too short for a PRG`);
+    const load = bytes.readUInt16LE(0);
+    const segment = moved({ start: load, end: load + bytes.length - 3 });
+    if (!inside(segment)) throw new UsageError(`the shift moves ${path} outside $0000-$ffff`);
+    const memory = new Uint8Array(0x10000);
+    memory.set(bytes.subarray(2), load);
+    return { release: { name: path, segments: [segment], byteAt: (at) => bytes[2 + at - segment.start]! }, shift, start: { kind: "program", path }, memory };
+  }
+  const fail = (what: string): never => {
+    throw new UsageError(`${path} is not an image.json of the c64-unpacker skill: ${what}`);
+  };
+  let manifest: Record<string, unknown>;
+  try {
+    manifest = JSON.parse(bytes.toString("utf8")) as Record<string, unknown>;
+  } catch (error) {
+    return fail((error as Error).message);
+  }
+  const parse = (value: unknown) => (typeof value === "string" && /^\$[0-9a-f]{4}$/i.test(value) ? Number.parseInt(value.slice(1), 16) : fail("a segment needs start and end as $ addresses"));
+  if (typeof manifest.image !== "string" || !Array.isArray(manifest.segments) || manifest.segments.length === 0) fail("it needs image and segments");
+  const own = (manifest.segments as Array<Record<string, unknown>>).map((segment) => ({ start: parse(segment.start), end: parse(segment.end) })).sort((a, b) => a.start - b.start);
+  const directory = posix.dirname(path);
+  const memory = readProjectFile(posix.join(directory, manifest.image as string)).bytes;
+  if (memory.length !== 0x10000) fail(`its image has ${memory.length} bytes, not 65536`);
+  const segments = own.map(moved);
+  if (!segments.every(inside)) throw new UsageError(`the shift moves ${path} outside $0000-$ffff`);
+  const state = entryState(posix.join(directory, "entry_state.json"));
+  const stages = Array.isArray(manifest.stages)
+    ? (manifest.stages as Array<Record<string, unknown>>).map((stage) => ({
+        entry: parse(stage.entry),
+        wrote: (Array.isArray(stage.wrote) ? (stage.wrote as Array<Record<string, unknown>>) : []).map((range) => ({ start: parse(range.start), end: parse(range.end) })),
+      }))
+    : undefined;
+  return {
+    release: { name: path, segments, byteAt: (at) => memory[at - shift]! },
+    shift,
+    start: typeof state === "string" ? { kind: "none", reason: state } : { kind: "image", segments: own, byteAt: (at) => memory[at]!, state },
+    memory,
+    ...(manifest.entry === undefined ? {} : { entry: parse(manifest.entry) }),
+    ...(stages === undefined ? {} : { stages }),
+  };
+}
+
+async function run(argv: string[]): Promise<unknown> {
+  const { values, positionals } = parseArgs({
+    args: argv,
+    strict: true,
+    allowPositionals: true,
+    options: {
+      shift: { type: "string", multiple: true },
+      from: { type: "string" },
+      to: { type: "string" },
+      run: { type: "string" },
+      steps: { type: "string" },
+      coverage: { type: "string", multiple: true },
+      out: { type: "string" },
+      "max-listed": { type: "string" },
+      help: { type: "boolean" },
+    },
+  });
+  if (values.help) throw new UsageError("");
+  if (positionals[0] === "separate") {
+    const used = ["shift", "from", "to", "run", "steps"].filter((option) => values[option as keyof typeof values] !== undefined);
+    if (used.length > 0) throw new UsageError(`separate does not take --${used.join(", --")}`);
+    if (positionals.length !== 2) throw new UsageError("separate needs one release");
+    return separateCommand(positionals[1]!, values.coverage ?? [], values.out, values["max-listed"]);
+  }
+  if (values.coverage !== undefined || values.out !== undefined || values["max-listed"] !== undefined) throw new UsageError("--coverage, --out and --max-listed work only with separate");
+  if (positionals.length < 2) throw new UsageError("give two or more releases");
+  const shifts = new Map<number, number>();
+  for (const text of values.shift ?? []) {
+    const match = /^(\d+):([+-]?\d+)$/.exec(text);
+    if (match === null || Number(match[1]) < 1 || Number(match[1]) > positionals.length) throw new UsageError(`--shift must be <release number>:<bytes>, not ${text}`);
+    shifts.set(Number(match[1]), Number(match[2]));
+  }
+  if (values.steps !== undefined && values.run === undefined) throw new UsageError("--steps works only with --run");
+  let frames: number | undefined;
+  if (values.run !== undefined) {
+    if (!/^\d{1,6}$/.test(values.run) || Number(values.run) < 1 || Number(values.run) > MAX_EXECUTION_COUNT) throw new UsageError(`--run needs a frame count from 1 to ${MAX_EXECUTION_COUNT}, not ${values.run}`);
+    frames = Number(values.run);
+  }
+  let steps: ActionStep[] = [];
+  if (values.steps !== undefined) {
+    let parsed: unknown;
+    try {
+      parsed = JSON.parse(readProjectFile(values.steps).bytes.toString("utf8"));
+    } catch (error) {
+      if (error instanceof WireFailure) throw error;
+      throw new UsageError(`the steps file ${values.steps} is not JSON: ${(error as Error).message}`);
+    }
+    steps = checkActionSteps(parsed, (message) => {
+      throw new UsageError(`steps file ${values.steps}: ${message}`);
+    });
+  }
+  const loaded = positionals.map((path, index) => readRelease(path, shifts.get(index + 1) ?? 0));
+  const releases = loaded.map((item) => item.release);
+  const comparison = compareReleases(releases, {
+    ...(values.from === undefined ? {} : { from: address(values.from, "--from") }),
+    ...(values.to === undefined ? {} : { to: address(values.to, "--to") }),
+  });
+  const compared = comparison.identicalBytes + comparison.differingBytes;
+  const identicalShare = compared === 0 ? 0 : comparison.identicalBytes / compared;
+  // A poor match at the load addresses can mean that the releases are placed differently.
+  // Each release gets its best shifts: data that stayed can outvote code that moved, so one is not enough (b2/04).
+  const applied = loaded.map((item) => item.shift);
+  const suggestions =
+    identicalShare < 0.5
+      ? releases.slice(1).flatMap((release, index) =>
+          suggestShifts(releases[0]!, release).map((found) => {
+            const suggestion = shiftSuggestion(releases, applied, index + 1, found);
+            // The shift that the run already has: its votes tell how good the present alignment is.
+            return "shift" in suggestion && suggestion.shift === applied[suggestion.release - 1] ? { ...suggestion, current: true } : suggestion;
+          }),
+        )
+      : [];
+  const suggested = suggestions.filter((suggestion) => "shift" in suggestion);
+  const notSuggested = suggestions.filter((suggestion) => "reason" in suggestion);
+
+  // The runtime evidence: one emulator for each release, the differences at the release's own addresses.
+  const runtime = new Map<string, Array<Range & { first: string; changed: boolean }>>();
+  const runs: Record<string, unknown> = {};
+  if (frames !== undefined && comparison.differences.length > 0) {
+    for (const item of loaded) {
+      const name = item.release.name;
+      if (item.start.kind === "none") {
+        runs[name] = { started: false, reason: item.start.reason };
+        continue;
+      }
+      const ranges = comparison.differences.map((difference) => ({ start: difference.start - item.shift, end: difference.end - item.shift }));
+      const client = await ViceSessionClient.open({ videoStandard: "pal" });
+      try {
+        const watched = await watchRelease(client, item.start, ranges, (at) => item.release.byteAt(at + item.shift), { frames, steps });
+        if (!watched.started) runs[name] = { started: false, reason: watched.reason };
+        else {
+          runs[name] = { started: true };
+          runtime.set(name, watched.accesses.map((access, index) => ({ ...ranges[index]!, ...access })));
+        }
+      } finally {
+        await client.close();
+      }
+    }
+  }
+
+  return {
+    releases: loaded.map((item) => ({ name: item.release.name, shift: item.shift, segments: item.release.segments.map((segment) => ({ start: formatC64Address(segment.start), end: formatC64Address(segment.end) })) })),
+    common: comparison.common.map((range) => ({ start: formatC64Address(range.start), end: formatC64Address(range.end) })),
+    identicalBytes: comparison.identicalBytes,
+    differingBytes: comparison.differingBytes,
+    identicalShare: Number(identicalShare.toFixed(3)),
+    differences: comparison.differences.map((difference, index) => ({
+      start: formatC64Address(difference.start),
+      end: formatC64Address(difference.end),
+      values: Object.fromEntries(Object.entries(difference.values).map(([name, bytes]) => [name, hexBytes(bytes.slice(0, 32)) + (bytes.length > 32 ? " ..." : "")])),
+      groups: difference.groups,
+      // runtime gives the release's own addresses: with a shift, they are not the addresses of the difference (F-24).
+      ...(runtime.size === 0
+        ? {}
+        : {
+            runtime: Object.fromEntries(
+              [...runtime].map(([name, accesses]) => {
+                const { start, end, first, changed } = accesses[index]!;
+                return [name, { start: formatC64Address(start), end: formatC64Address(end), first, changed }];
+              }),
+            ),
+          }),
+    })),
+    ...(comparison.moreDifferences ? { moreDifferences: true } : {}),
+    ...(suggested.length > 0 ? { suggestedShifts: suggested } : {}),
+    ...(notSuggested.length > 0 ? { shiftsNotSuggested: notSuggested } : {}),
+    ...(frames === undefined
+      ? {}
+      : { run: { frames, steps: steps.length, releases: runs, ...(comparison.differences.length === 0 ? { skipped: "The releases have no differences, so no release ran." } : {}) } }),
+  };
+}
+
+/** separate: evidence from one release (D51), with all ranges in --out and the first ones in the result. */
+function separateCommand(path: string, coveragePaths: string[], out: string | undefined, maxListed: string | undefined): unknown {
+  const max = maxListed ?? "50";
+  if (!/^\d{1,5}$/.test(max)) throw new UsageError(`--max-listed must be a whole number, not ${max}`);
+  const loaded = readRelease(path, 0);
+  const coverage = readCoverageFiles(coveragePaths, (message) => {
+    throw new UsageError(message);
+  });
+  const db = openForRead();
+  let regions: ReturnType<typeof listRegions>;
+  let references: ReturnType<typeof listReferences>;
+  try {
+    regions = listRegions(db).filter((region) => CODE_REGION_TYPES.includes(region.type));
+    references = listReferences(db);
+  } finally {
+    db?.close();
+  }
+  const found = separate({
+    segments: loaded.release.segments,
+    memory: loaded.memory,
+    ...(loaded.entry === undefined ? {} : { entry: loaded.entry }),
+    ...(loaded.stages === undefined ? {} : { stages: loaded.stages }),
+    ...(coverage === undefined ? {} : { coverage }),
+    codeRegions: regions,
+    references,
+  });
+  const show = (range: Range) => ({ start: formatC64Address(range.start), end: formatC64Address(range.end) });
+  const ranges = found.ranges.map((range) => ({ ...show(range), ...(range.stage === undefined ? {} : { stage: range.stage }), ...(range.access === undefined ? {} : { access: range.access }) }));
+  const codeRegions = found.codeRegions.map((region) => ({ ...show(region), type: region.type, ...(region.reachable === undefined ? {} : { reachable: region.reachable }), ...(region.executed === undefined ? {} : { executed: region.executed }) }));
+  const outside = { read: found.outside.read.map(show), executed: found.outside.executed.map(show), generated: found.outside.generated.map(show) };
+  const listed = <T>(items: T[]) => ({ count: items.length, listed: items.slice(0, Number(max)) });
+  if (out !== undefined) writeProjectFile(out, Buffer.from(`${JSON.stringify({ release: path, ranges, codeRegions, outside }, null, 1)}\n`), { what: "--out" });
+  const bytes = (filter: (range: (typeof found.ranges)[number]) => boolean) => found.ranges.filter(filter).reduce((sum, range) => sum + range.end - range.start + 1, 0);
+  const lastStage = loaded.stages === undefined ? undefined : loaded.stages.length - 1;
+  return {
+    release: path,
+    ...(loaded.entry === undefined ? {} : { entry: formatC64Address(loaded.entry) }),
+    ...(lastStage === undefined
+      ? { noStages: "image.json has no stage log. Only a capture of c64-unpacker writes one. A capture of loads holds one part of a program and has none." }
+      : {
+          stages: lastStage + 1,
+          bytesByStage: Object.fromEntries([...[...Array(lastStage + 1).keys()].map((stage): [string, number] => [String(stage), bytes((range) => range.stage === stage)]), ["none", bytes((range) => range.stage === undefined)]]),
+        }),
+    ...(coverage === undefined ? {} : { bytesByAccess: Object.fromEntries((["executed", "read", "written", "read-written", "untouched"] as const).map((access) => [access, bytes((range) => range.access === access)])) }),
+    ranges: listed(ranges),
+    // total counts every code region; unreachable is a list like the others (b2/08).
+    codeRegions: {
+      total: codeRegions.length,
+      ...(loaded.entry === undefined ? {} : { unreachable: listed(codeRegions.filter((region) => region.reachable === false)) }),
+    },
+    ...(coverage === undefined ? {} : { outside: { read: listed(outside.read), executed: listed(outside.executed), generated: listed(outside.generated) } }),
+    ...(out === undefined ? {} : { out }),
+  };
+}
+
+await runScript(() => run(process.argv.slice(2)), { usage: USAGE, refusals: [KnowledgeError] });

package/skills/c64-provenance/scripts/runtime.ts

@@ -0,0 +1,191 @@
+// Runtime evidence for the differences between releases (D50): each release
+// runs in its own emulator, and each differing range gets its first access
+// (read, write, execute or none) and whether its bytes changed by the end.
+// Watchpoints and range breakpoints stop the run at an access; the coverage
+// map then tells which range and which access it was.
+
+import { COVERAGE, formatC64Address, instructionLength } from "#src/c64.ts";
+import { runProgram } from "#src/host-client/autostart.ts";
+import { readCoverage } from "#src/host-client/coverage.ts";
+import { runActionStep, type ActionStep } from "#src/host-client/steps.ts";
+import { MAX_MEMORY_READ, MAX_MEMORY_WRITE } from "#src/host-client/tools.ts";
+import { readProjectFile } from "#src/host-client/transfer.ts";
+import type { ViceSessionClient } from "#src/host-client/vice-session.ts";
+import type { Range } from "./compare.ts";
+
+export type FirstAccess = "read" | "write" | "execute" | "none";
+
+export interface RangeAccess {
+  first: FirstAccess;
+  /** True when the bytes at the end differ from the bytes of the release. */
+  changed: boolean;
+}
+
+/** How a release starts in the emulator. */
+export type Start =
+  | { kind: "program"; path: string }
+  | { kind: "image"; segments: Range[]; byteAt: (address: number) => number; state: ImageState };
+
+export interface ImageState {
+  pc: number;
+  a: number;
+  x: number;
+  y: number;
+  sp: number;
+  flags: { n: boolean; v: boolean; d: boolean; i: boolean; z: boolean; c: boolean };
+  port: [number, number];
+  /** $0000-$03ff at the entry. */
+  pages: Uint8Array;
+}
+
+const MAX_WATCH = 256;
+/** Frames until the KERNAL is at the READY prompt, before an image is written. */
+const READY_FRAMES = 150;
+
+const READ = COVERAGE.ramRead | COVERAGE.romRead | COVERAGE.ioRead;
+const WRITE = COVERAGE.ramWrite | COVERAGE.ioWrite;
+const EXECUTE = COVERAGE.ramExecute | COVERAGE.romExecute | COVERAGE.ioExecute;
+
+/** The first access that coverage flags show; a read-modify-write reads first. */
+export function accessOf(flags: number): FirstAccess {
+  if ((flags & EXECUTE) !== 0) return "execute";
+  if ((flags & READ) !== 0) return "read";
+  if ((flags & WRITE) !== 0) return "write";
+  return "none";
+}
+
+/** Starts the release; returns why it did not start, or undefined. */
+async function start(client: ViceSessionClient, how: Start): Promise<string | undefined> {
+  if (how.kind === "program") {
+    const failed = await runProgram(client, how.path, readProjectFile(how.path).bytes);
+    if (failed === "not-loaded") return "The program was not in memory after autostart.";
+    if (failed === "not-started") return "The program did not start after RUN; it needs a BASIC start line.";
+    return undefined;
+  }
+  await client.reset({ mode: "hard", run: true });
+  await client.execution({ action: "advance-frames", count: READY_FRAMES, space: "c64" });
+  for (const segment of how.segments) {
+    for (let at = segment.start; at <= segment.end; at += MAX_MEMORY_WRITE) {
+      const last = Math.min(segment.end, at + MAX_MEMORY_WRITE - 1);
+      const bytes = Buffer.from(Array.from({ length: last - at + 1 }, (_, offset) => how.byteAt(at + offset)));
+      await client.memoryWrite({ address: at, data: bytes.toString("hex"), space: "c64", view: "ram" });
+    }
+  }
+  const { state } = how;
+  // Pages 0 to 3 from $0002, then the CPU port: the port bits switch what the CPU sees.
+  await client.memoryWrite({ address: 0x0002, data: Buffer.from(state.pages.subarray(2)).toString("hex"), space: "c64", view: "ram" });
+  await client.memoryWrite({ address: 0x0000, data: Buffer.from(state.port).toString("hex"), space: "c64", view: "cpu" });
+  await client.registersSet("c64", { pc: state.pc, a: state.a, x: state.x, y: state.y, sp: state.sp, flags: state.flags });
+  return undefined;
+}
+
+/**
+ * Runs one release and watches the ranges (addresses of the release, after
+ * any shift is taken away). `release` gives the release's own bytes.
+ */
+export async function watchRelease(
+  client: ViceSessionClient,
+  how: Start,
+  ranges: Range[],
+  release: (address: number) => number,
+  run: { frames: number; steps: ActionStep[] },
+): Promise<{ started: true; accesses: RangeAccess[] } | { started: false; reason: string }> {
+  await client.reset({ mode: "hard", run: false });
+  const failed = await start(client, how);
+  if (failed !== undefined) return { started: false, reason: failed };
+  if ((await client.status()).state === "running") await client.execution({ action: "pause", space: "c64" });
+
+  const first: FirstAccess[] = ranges.map(() => "none");
+  const points = new Map<number, Array<{ kind: "watch" | "break"; id: number }>>();
+  for (const [index, range] of ranges.entries()) {
+    const ids: Array<{ kind: "watch" | "break"; id: number }> = [];
+    for (let at = range.start; at <= range.end; at += MAX_WATCH) {
+      const size = Math.min(MAX_WATCH, range.end - at + 1);
+      ids.push({ kind: "watch", id: (await client.watchpoint({ action: "add", address: at, size, access: "read-write", space: "c64" })).id });
+      ids.push({ kind: "break", id: (await client.breakpoint({ action: "add", address: at, size, space: "c64" })).id });
+    }
+    points.set(index, ids);
+  }
+  await client.memmap({ action: "clear" });
+
+  /**
+   * After a stop: the ranges that the coverage shows accessed get their first
+   * access, and lose their points. VICE marks only the opcode byte of an
+   * executed instruction, so an instruction that starts up to two bytes before
+   * a range, or in it, executes its operand bytes too (F-27).
+   */
+  const settle = async (): Promise<boolean> => {
+    const map = await readCoverage(client);
+    let found = false;
+    for (const [index, ids] of points) {
+      const range = ranges[index]!;
+      let flags = 0;
+      for (let address = range.start; address <= range.end; address++) flags |= map[address]!;
+      const from = Math.max(0, range.start - 2);
+      if ((flags & EXECUTE) === 0 && map.subarray(from, range.start).some((byte) => (byte & COVERAGE.ramExecute) !== 0)) {
+        const code = Buffer.from((await client.memoryRead({ address: from, size: range.start - from, space: "c64", view: "ram" })).data, "hex");
+        for (let address = from; address < range.start; address++) {
+          if ((map[address]! & COVERAGE.ramExecute) !== 0 && address + instructionLength(code[address - from]!) - 1 >= range.start) flags |= COVERAGE.ramExecute;
+        }
+      }
+      if (accessOf(flags) === "none") continue;
+      first[index] = accessOf(flags);
+      for (const point of ids) {
+        if (point.kind === "watch") await client.watchpoint({ action: "remove", id: point.id });
+        else await client.breakpoint({ action: "remove", id: point.id });
+      }
+      points.delete(index);
+      found = true;
+    }
+    return found;
+  };
+
+  // Each stop at a point settles at least one range, so the stops are bounded; the guard ends a run that stops without an access.
+  let guard = 4 * ranges.length + 16;
+  /** Runs frames; a stop at a point settles and goes on with the frames that are left. */
+  const runFrames = async (count: number) => {
+    let left = count;
+    while (left > 0 && guard-- > 0) {
+      const ran = await client.execution({ action: "advance-frames", count: left, space: "c64" });
+      left -= ran.advancedFrames ?? left;
+      if (left > 0 && !(await settle())) left--;
+    }
+  };
+  for (const step of run.steps) {
+    if ("frames" in step) {
+      await runFrames(step.frames);
+      continue;
+    }
+    if ("runUntil" in step) {
+      // A stop at a point is not the target: settle and run to the target again.
+      while (guard-- > 0) {
+        const reached = await runActionStep(client, step, (location) => Number.parseInt(location.slice(1), 16));
+        if (reached === undefined || reached.stopReason !== "watchpoint" && reached.stopReason !== "breakpoint") break;
+        await settle();
+      }
+      continue;
+    }
+    await runActionStep(client, step, (location) => Number.parseInt(location.slice(1), 16));
+  }
+  await runFrames(run.frames);
+  if (points.size > 0) await settle();
+
+  const accesses: RangeAccess[] = [];
+  for (const [index, range] of ranges.entries()) {
+    const parts: Buffer[] = [];
+    for (let at = range.start; at <= range.end; at += MAX_MEMORY_READ) {
+      parts.push(Buffer.from((await client.memoryRead({ address: at, size: Math.min(MAX_MEMORY_READ, range.end - at + 1), space: "c64", view: "ram" })).data, "hex"));
+    }
+    const now = Buffer.concat(parts);
+    accesses.push({ first: first[index]!, changed: now.some((byte, offset) => byte !== release(range.start + offset)) });
+  }
+  for (const ids of points.values()) {
+    for (const point of ids) {
+      if (point.kind === "watch") await client.watchpoint({ action: "remove", id: point.id });
+      else await client.breakpoint({ action: "remove", id: point.id });
+    }
+  }
+  return { started: true, accesses };
+}
+
+export const showRange = (range: Range) => `${formatC64Address(range.start)}-${formatC64Address(range.end)}`;

package/skills/c64-provenance/scripts/separate.ts

@@ -0,0 +1,140 @@
+// Evidence from one release (D51): for each range of the image, the unpack
+// stage that wrote it and the access class from coverage; for each code
+// region of knowledge, whether call and jump references reach it from the
+// entry; and what the program reads or executes outside the image. Evidence
+// only: nothing here decides what is original.
+
+import { COVERAGE, executedRamCode } from "#src/c64.ts";
+import type { Range } from "./compare.ts";
+
+export type Access = "executed" | "read" | "written" | "read-written" | "untouched";
+
+export interface SeparationInput {
+  segments: Range[];
+  /** 64 KiB with the image bytes at their addresses. */
+  memory: Uint8Array;
+  entry?: number;
+  /** The stages of an unpacker capture, in order; the last one is the program. */
+  stages?: Array<{ entry: number; wrote: Range[] }>;
+  /** Joined coverage files (D40). */
+  coverage?: Uint8Array;
+  codeRegions: Array<Range & { type: string }>;
+  references: Array<{ from: number; to: number; kind: string }>;
+}
+
+export interface EvidenceRange extends Range {
+  /** The last stage that wrote the range; absent when no stage wrote it. */
+  stage?: number;
+  access?: Access;
+}
+
+export interface CodeRegionEvidence extends Range {
+  type: string;
+  /** Absent when the image has no entry. */
+  reachable?: boolean;
+  /** Absent without coverage. */
+  executed?: boolean;
+}
+
+export interface Separation {
+  ranges: EvidenceRange[];
+  codeRegions: CodeRegionEvidence[];
+  /**
+   * RAM outside the image and outside pages 0-3, from the coverage: `read`
+   * and `executed` the program never wrote (a loader or an earlier stage put
+   * those bytes there), and `generated` it wrote and then executed, in
+   * pieces of code with gaps of up to two operand bytes.
+   */
+  outside: { read: Range[]; executed: Range[]; generated: Range[] };
+}
+
+const SYSTEM_PAGES_END = 0x03ff;
+
+/** Joins addresses in order into ranges; `gap` is the most bytes between two addresses in one range. */
+function merge(addresses: number[], gap = 0, joins: (from: number, to: number) => boolean = () => true): Range[] {
+  const ranges: Range[] = [];
+  for (const address of addresses) {
+    const last = ranges.at(-1);
+    if (last !== undefined && address - last.end - 1 <= gap && joins(last.end, address)) last.end = address;
+    else ranges.push({ start: address, end: address });
+  }
+  return ranges;
+}
+
+export function separate(input: SeparationInput): Separation {
+  const inImage = (address: number) => input.segments.some((segment) => address >= segment.start && address <= segment.end);
+  const executed = new Uint8Array(0x10000);
+  if (input.coverage !== undefined) for (const range of executedRamCode(input.coverage, 0, input.memory)) executed.fill(1, range.start, range.end + 1);
+  const accessAt = (address: number): Access | undefined => {
+    if (input.coverage === undefined) return undefined;
+    if (executed[address] === 1) return "executed";
+    const flags = input.coverage[address]!;
+    const read = (flags & COVERAGE.ramRead) !== 0;
+    const written = (flags & COVERAGE.ramWrite) !== 0;
+    return read && written ? "read-written" : read ? "read" : written ? "written" : "untouched";
+  };
+  const stageAt = (address: number): number | undefined => {
+    const stages = input.stages ?? [];
+    for (let index = stages.length - 1; index >= 0; index--) if (stages[index]!.wrote.some((range) => address >= range.start && address <= range.end)) return index;
+    return undefined;
+  };
+
+  const ranges: EvidenceRange[] = [];
+  for (const segment of input.segments) {
+    for (let address = segment.start; address <= segment.end; address++) {
+      const stage = stageAt(address);
+      const access = accessAt(address);
+      const last = ranges.at(-1);
+      if (last !== undefined && last.end + 1 === address && last.stage === stage && last.access === access) {
+        last.end = address;
+        continue;
+      }
+      ranges.push({ start: address, end: address, ...(stage === undefined ? {} : { stage }), ...(access === undefined ? {} : { access }) });
+    }
+  }
+
+  // Reachability: from the region that holds the entry, through call and jump references, to a fixed point.
+  const regions = input.codeRegions.filter((region) => input.segments.some((segment) => region.start <= segment.end && segment.start <= region.end));
+  const holding = (address: number) => regions.findIndex((region) => address >= region.start && address <= region.end);
+  const reached = new Set<number>();
+  if (input.entry !== undefined && holding(input.entry) >= 0) reached.add(holding(input.entry));
+  const flows = input.references.filter((reference) => reference.kind === "call" || reference.kind === "jump");
+  for (let grown = true; grown; ) {
+    grown = false;
+    for (const flow of flows) {
+      const from = holding(flow.from);
+      const to = holding(flow.to);
+      if (from >= 0 && to >= 0 && reached.has(from) && !reached.has(to)) {
+        reached.add(to);
+        grown = true;
+      }
+    }
+  }
+  const codeRegions = regions.map((region, index): CodeRegionEvidence => {
+    let ran: boolean | undefined;
+    if (input.coverage !== undefined) {
+      ran = false;
+      for (let address = region.start; address <= region.end && !ran; address++) ran = executed[address] === 1;
+    }
+    return { start: region.start, end: region.end, type: region.type, ...(input.entry === undefined ? {} : { reachable: reached.has(index) }), ...(ran === undefined ? {} : { executed: ran }) };
+  });
+
+  const read: number[] = [];
+  const ranOutside: number[] = [];
+  const generated: number[] = [];
+  if (input.coverage !== undefined) {
+    for (let address = SYSTEM_PAGES_END + 1; address < 0x10000; address++) {
+      if (inImage(address)) continue;
+      const written = (input.coverage[address]! & COVERAGE.ramWrite) !== 0;
+      if (executed[address] === 1) (written ? generated : ranOutside).push(address);
+      else if (!written && (input.coverage[address]! & COVERAGE.ramRead) !== 0) read.push(address);
+    }
+  }
+  // The image has no bytes of generated code, so only its opcode bytes count as executed. VICE marks only
+  // opcode bytes: a gap of at most two operand bytes outside the image stays inside one piece of code (F-28).
+  const outsideImage = (from: number, to: number) => {
+    for (let address = from + 1; address < to; address++) if (inImage(address)) return false;
+    return true;
+  };
+  return { ranges, codeRegions, outside: { read: merge(read), executed: merge(ranOutside), generated: merge(generated, 2, outsideImage) } };
+}