@henols/c64-re-tools 2.0.0-rc.2 → 2.0.0-rc.21

package/skills/c64-provenance/scripts/lib/runtime.ts

@@ -0,0 +1,191 @@
+// Runtime evidence for the differences between releases (D50): each release
+// runs in its own emulator, and each differing range gets its first access
+// (read, write, execute or none) and whether its bytes changed by the end.
+// Watchpoints and range breakpoints stop the run at an access; the coverage
+// map then tells which range and which access it was.
+
+import { COVERAGE, formatC64Address, instructionLength } from "#src/c64.ts";
+import { runProgram } from "#src/host-client/autostart.ts";
+import { readCoverage } from "#src/host-client/coverage.ts";
+import { runActionStep, type ActionStep } from "#src/host-client/steps.ts";
+import { MAX_MEMORY_READ, MAX_MEMORY_WRITE } from "#src/host-client/tools.ts";
+import { readProjectFile } from "#src/host-client/transfer.ts";
+import type { ViceSessionClient } from "#src/host-client/vice-session.ts";
+import type { Range } from "./compare.ts";
+
+export type FirstAccess = "read" | "write" | "execute" | "none";
+
+export interface RangeAccess {
+  first: FirstAccess;
+  /** True when the bytes at the end differ from the bytes of the release. */
+  changed: boolean;
+}
+
+/** How a release starts in the emulator. */
+export type Start =
+  | { kind: "program"; path: string }
+  | { kind: "image"; segments: Range[]; byteAt: (address: number) => number; state: ImageState };
+
+export interface ImageState {
+  pc: number;
+  a: number;
+  x: number;
+  y: number;
+  sp: number;
+  flags: { n: boolean; v: boolean; d: boolean; i: boolean; z: boolean; c: boolean };
+  port: [number, number];
+  /** $0000-$03ff at the entry. */
+  pages: Uint8Array;
+}
+
+const MAX_WATCH = 256;
+/** Frames until the KERNAL is at the READY prompt, before an image is written. */
+const READY_FRAMES = 150;
+
+const READ = COVERAGE.ramRead | COVERAGE.romRead | COVERAGE.ioRead;
+const WRITE = COVERAGE.ramWrite | COVERAGE.ioWrite;
+const EXECUTE = COVERAGE.ramExecute | COVERAGE.romExecute | COVERAGE.ioExecute;
+
+/** The first access that coverage flags show; a read-modify-write reads first. */
+export function accessOf(flags: number): FirstAccess {
+  if ((flags & EXECUTE) !== 0) return "execute";
+  if ((flags & READ) !== 0) return "read";
+  if ((flags & WRITE) !== 0) return "write";
+  return "none";
+}
+
+/** Starts the release; returns why it did not start, or undefined. */
+async function start(client: ViceSessionClient, how: Start): Promise<string | undefined> {
+  if (how.kind === "program") {
+    const failed = await runProgram(client, how.path, readProjectFile(how.path).bytes);
+    if (failed === "not-loaded") return "The program was not in memory after autostart.";
+    if (failed === "not-started") return "The program did not start after RUN; it needs a BASIC start line.";
+    return undefined;
+  }
+  await client.reset({ mode: "hard", run: true });
+  await client.execution({ action: "advance-frames", count: READY_FRAMES, space: "c64" });
+  for (const segment of how.segments) {
+    for (let at = segment.start; at <= segment.end; at += MAX_MEMORY_WRITE) {
+      const last = Math.min(segment.end, at + MAX_MEMORY_WRITE - 1);
+      const bytes = Buffer.from(Array.from({ length: last - at + 1 }, (_, offset) => how.byteAt(at + offset)));
+      await client.memoryWrite({ address: at, data: bytes.toString("hex"), space: "c64", view: "ram" });
+    }
+  }
+  const { state } = how;
+  // Pages 0 to 3 from $0002, then the CPU port: the port bits switch what the CPU sees.
+  await client.memoryWrite({ address: 0x0002, data: Buffer.from(state.pages.subarray(2)).toString("hex"), space: "c64", view: "ram" });
+  await client.memoryWrite({ address: 0x0000, data: Buffer.from(state.port).toString("hex"), space: "c64", view: "cpu" });
+  await client.registersSet("c64", { pc: state.pc, a: state.a, x: state.x, y: state.y, sp: state.sp, flags: state.flags });
+  return undefined;
+}
+
+/**
+ * Runs one release and watches the ranges (addresses of the release, after
+ * any shift is taken away). `release` gives the release's own bytes.
+ */
+export async function watchRelease(
+  client: ViceSessionClient,
+  how: Start,
+  ranges: Range[],
+  release: (address: number) => number,
+  run: { frames: number; steps: ActionStep[] },
+): Promise<{ started: true; accesses: RangeAccess[] } | { started: false; reason: string }> {
+  await client.reset({ mode: "hard", run: false });
+  const failed = await start(client, how);
+  if (failed !== undefined) return { started: false, reason: failed };
+  if ((await client.status()).state === "running") await client.execution({ action: "pause", space: "c64" });
+
+  const first: FirstAccess[] = ranges.map(() => "none");
+  const points = new Map<number, Array<{ kind: "watch" | "break"; id: number }>>();
+  for (const [index, range] of ranges.entries()) {
+    const ids: Array<{ kind: "watch" | "break"; id: number }> = [];
+    for (let at = range.start; at <= range.end; at += MAX_WATCH) {
+      const size = Math.min(MAX_WATCH, range.end - at + 1);
+      ids.push({ kind: "watch", id: (await client.watchpoint({ action: "add", address: at, size, access: "read-write", space: "c64" })).id });
+      ids.push({ kind: "break", id: (await client.breakpoint({ action: "add", address: at, size, space: "c64" })).id });
+    }
+    points.set(index, ids);
+  }
+  await client.memmap({ action: "clear" });
+
+  /**
+   * After a stop: the ranges that the coverage shows accessed get their first
+   * access, and lose their points. VICE marks only the opcode byte of an
+   * executed instruction, so an instruction that starts up to two bytes before
+   * a range, or in it, executes its operand bytes too (F-27).
+   */
+  const settle = async (): Promise<boolean> => {
+    const map = await readCoverage(client);
+    let found = false;
+    for (const [index, ids] of points) {
+      const range = ranges[index]!;
+      let flags = 0;
+      for (let address = range.start; address <= range.end; address++) flags |= map[address]!;
+      const from = Math.max(0, range.start - 2);
+      if ((flags & EXECUTE) === 0 && map.subarray(from, range.start).some((byte) => (byte & COVERAGE.ramExecute) !== 0)) {
+        const code = Buffer.from((await client.memoryRead({ address: from, size: range.start - from, space: "c64", view: "ram" })).data, "hex");
+        for (let address = from; address < range.start; address++) {
+          if ((map[address]! & COVERAGE.ramExecute) !== 0 && address + instructionLength(code[address - from]!) - 1 >= range.start) flags |= COVERAGE.ramExecute;
+        }
+      }
+      if (accessOf(flags) === "none") continue;
+      first[index] = accessOf(flags);
+      for (const point of ids) {
+        if (point.kind === "watch") await client.watchpoint({ action: "remove", id: point.id });
+        else await client.breakpoint({ action: "remove", id: point.id });
+      }
+      points.delete(index);
+      found = true;
+    }
+    return found;
+  };
+
+  // Each stop at a point settles at least one range, so the stops are bounded; the guard ends a run that stops without an access.
+  let guard = 4 * ranges.length + 16;
+  /** Runs frames; a stop at a point settles and goes on with the frames that are left. */
+  const runFrames = async (count: number) => {
+    let left = count;
+    while (left > 0 && guard-- > 0) {
+      const ran = await client.execution({ action: "advance-frames", count: left, space: "c64" });
+      left -= ran.advancedFrames ?? left;
+      if (left > 0 && !(await settle())) left--;
+    }
+  };
+  for (const step of run.steps) {
+    if ("frames" in step) {
+      await runFrames(step.frames);
+      continue;
+    }
+    if ("runUntil" in step) {
+      // A stop at a point is not the target: settle and run to the target again.
+      while (guard-- > 0) {
+        const reached = await runActionStep(client, step, (location) => Number.parseInt(location.slice(1), 16));
+        if (reached === undefined || reached.stopReason !== "watchpoint" && reached.stopReason !== "breakpoint") break;
+        await settle();
+      }
+      continue;
+    }
+    await runActionStep(client, step, (location) => Number.parseInt(location.slice(1), 16));
+  }
+  await runFrames(run.frames);
+  if (points.size > 0) await settle();
+
+  const accesses: RangeAccess[] = [];
+  for (const [index, range] of ranges.entries()) {
+    const parts: Buffer[] = [];
+    for (let at = range.start; at <= range.end; at += MAX_MEMORY_READ) {
+      parts.push(Buffer.from((await client.memoryRead({ address: at, size: Math.min(MAX_MEMORY_READ, range.end - at + 1), space: "c64", view: "ram" })).data, "hex"));
+    }
+    const now = Buffer.concat(parts);
+    accesses.push({ first: first[index]!, changed: now.some((byte, offset) => byte !== release(range.start + offset)) });
+  }
+  for (const ids of points.values()) {
+    for (const point of ids) {
+      if (point.kind === "watch") await client.watchpoint({ action: "remove", id: point.id });
+      else await client.breakpoint({ action: "remove", id: point.id });
+    }
+  }
+  return { started: true, accesses };
+}
+
+export const showRange = (range: Range) => `${formatC64Address(range.start)}-${formatC64Address(range.end)}`;

package/skills/c64-provenance/scripts/lib/separate.ts

@@ -0,0 +1,140 @@
+// Evidence from one release (D51): for each range of the image, the unpack
+// stage that wrote it and the access class from coverage; for each code
+// region of knowledge, whether call and jump references reach it from the
+// entry; and what the program reads or executes outside the image. Evidence
+// only: nothing here decides what is original.
+
+import { COVERAGE, executedRamCode } from "#src/c64.ts";
+import type { Range } from "./compare.ts";
+
+export type Access = "executed" | "read" | "written" | "read-written" | "untouched";
+
+export interface SeparationInput {
+  segments: Range[];
+  /** 64 KiB with the image bytes at their addresses. */
+  memory: Uint8Array;
+  entry?: number;
+  /** The stages of an unpacker capture, in order; the last one is the program. */
+  stages?: Array<{ entry: number; wrote: Range[] }>;
+  /** Joined coverage files (D40). */
+  coverage?: Uint8Array;
+  codeRegions: Array<Range & { type: string }>;
+  references: Array<{ from: number; to: number; kind: string }>;
+}
+
+export interface EvidenceRange extends Range {
+  /** The last stage that wrote the range; absent when no stage wrote it. */
+  stage?: number;
+  access?: Access;
+}
+
+export interface CodeRegionEvidence extends Range {
+  type: string;
+  /** Absent when the image has no entry. */
+  reachable?: boolean;
+  /** Absent without coverage. */
+  executed?: boolean;
+}
+
+export interface Separation {
+  ranges: EvidenceRange[];
+  codeRegions: CodeRegionEvidence[];
+  /**
+   * RAM outside the image and outside pages 0-3, from the coverage: `read`
+   * and `executed` the program never wrote (a loader or an earlier stage put
+   * those bytes there), and `generated` it wrote and then executed, in
+   * pieces of code with gaps of up to two operand bytes.
+   */
+  outside: { read: Range[]; executed: Range[]; generated: Range[] };
+}
+
+const SYSTEM_PAGES_END = 0x03ff;
+
+/** Joins addresses in order into ranges; `gap` is the most bytes between two addresses in one range. */
+function merge(addresses: number[], gap = 0, joins: (from: number, to: number) => boolean = () => true): Range[] {
+  const ranges: Range[] = [];
+  for (const address of addresses) {
+    const last = ranges.at(-1);
+    if (last !== undefined && address - last.end - 1 <= gap && joins(last.end, address)) last.end = address;
+    else ranges.push({ start: address, end: address });
+  }
+  return ranges;
+}
+
+export function separate(input: SeparationInput): Separation {
+  const inImage = (address: number) => input.segments.some((segment) => address >= segment.start && address <= segment.end);
+  const executed = new Uint8Array(0x10000);
+  if (input.coverage !== undefined) for (const range of executedRamCode(input.coverage, 0, input.memory)) executed.fill(1, range.start, range.end + 1);
+  const accessAt = (address: number): Access | undefined => {
+    if (input.coverage === undefined) return undefined;
+    if (executed[address] === 1) return "executed";
+    const flags = input.coverage[address]!;
+    const read = (flags & COVERAGE.ramRead) !== 0;
+    const written = (flags & COVERAGE.ramWrite) !== 0;
+    return read && written ? "read-written" : read ? "read" : written ? "written" : "untouched";
+  };
+  const stageAt = (address: number): number | undefined => {
+    const stages = input.stages ?? [];
+    for (let index = stages.length - 1; index >= 0; index--) if (stages[index]!.wrote.some((range) => address >= range.start && address <= range.end)) return index;
+    return undefined;
+  };
+
+  const ranges: EvidenceRange[] = [];
+  for (const segment of input.segments) {
+    for (let address = segment.start; address <= segment.end; address++) {
+      const stage = stageAt(address);
+      const access = accessAt(address);
+      const last = ranges.at(-1);
+      if (last !== undefined && last.end + 1 === address && last.stage === stage && last.access === access) {
+        last.end = address;
+        continue;
+      }
+      ranges.push({ start: address, end: address, ...(stage === undefined ? {} : { stage }), ...(access === undefined ? {} : { access }) });
+    }
+  }
+
+  // Reachability: from the region that holds the entry, through call and jump references, to a fixed point.
+  const regions = input.codeRegions.filter((region) => input.segments.some((segment) => region.start <= segment.end && segment.start <= region.end));
+  const holding = (address: number) => regions.findIndex((region) => address >= region.start && address <= region.end);
+  const reached = new Set<number>();
+  if (input.entry !== undefined && holding(input.entry) >= 0) reached.add(holding(input.entry));
+  const flows = input.references.filter((reference) => reference.kind === "call" || reference.kind === "jump");
+  for (let grown = true; grown; ) {
+    grown = false;
+    for (const flow of flows) {
+      const from = holding(flow.from);
+      const to = holding(flow.to);
+      if (from >= 0 && to >= 0 && reached.has(from) && !reached.has(to)) {
+        reached.add(to);
+        grown = true;
+      }
+    }
+  }
+  const codeRegions = regions.map((region, index): CodeRegionEvidence => {
+    let ran: boolean | undefined;
+    if (input.coverage !== undefined) {
+      ran = false;
+      for (let address = region.start; address <= region.end && !ran; address++) ran = executed[address] === 1;
+    }
+    return { start: region.start, end: region.end, type: region.type, ...(input.entry === undefined ? {} : { reachable: reached.has(index) }), ...(ran === undefined ? {} : { executed: ran }) };
+  });
+
+  const read: number[] = [];
+  const ranOutside: number[] = [];
+  const generated: number[] = [];
+  if (input.coverage !== undefined) {
+    for (let address = SYSTEM_PAGES_END + 1; address < 0x10000; address++) {
+      if (inImage(address)) continue;
+      const written = (input.coverage[address]! & COVERAGE.ramWrite) !== 0;
+      if (executed[address] === 1) (written ? generated : ranOutside).push(address);
+      else if (!written && (input.coverage[address]! & COVERAGE.ramRead) !== 0) read.push(address);
+    }
+  }
+  // The image has no bytes of generated code, so only its opcode bytes count as executed. VICE marks only
+  // opcode bytes: a gap of at most two operand bytes outside the image stays inside one piece of code (F-28).
+  const outsideImage = (from: number, to: number) => {
+    for (let address = from + 1; address < to; address++) if (inImage(address)) return false;
+    return true;
+  };
+  return { ranges, codeRegions, outside: { read: merge(read), executed: merge(ranOutside), generated: merge(generated, 2, outsideImage) } };
+}