@henols/c64-re-tools 2.0.0-rc.2 → 2.0.0-rc.20

package/src/script.ts

@@ -1,8 +1,9 @@
 // The one error envelope for the skill scripts and the CLI: a script prints one JSON line,
 // a refusal prints {"error":{code,message}} with exit status 1, and a usage error prints the
-// usage text as an invalid-input error with exit status 2.
+// usage text as an invalid-input error with exit status 2. Every script takes --project <folder> (D67).
 
-import { WireFailure } from "./protocol/messages.ts";
+import { ProjectError, useProject } from "./project.ts";
+import { shownMessage, WireFailure } from "./protocol/messages.ts";
 import { elapsedMs, startTrace, trace, type TraceFields } from "./trace.ts";
 
 /** A wrong command line. The message names the problem; an empty message asks for the usage text alone. */
@@ -17,28 +18,67 @@ function isParseArgsError(error: unknown): error is Error {
   return error instanceof Error && typeof code === "string" && code.startsWith("ERR_PARSE_ARGS_");
 }
 
+/** The line that every usage text ends with. */
+export const PROJECT_USAGE = "Every script takes --project <folder>: the project folder, relative to the working directory. Without it, the script uses the nearest folder at or above the working directory that holds .c64-re-tools/.";
+
+/**
+ * Takes --project <folder> and --project=<folder> out of a command line, up to "--".
+ * Returns the folder and the other arguments. Throws UsageError for a --project without a folder or a second --project.
+ */
+export function takeProject(argv: readonly string[]): { project?: string; argv: string[] } {
+  const rest: string[] = [];
+  let project: string | undefined;
+  for (let index = 0; index < argv.length; index++) {
+    const argument = argv[index]!;
+    if (argument === "--") {
+      rest.push(...argv.slice(index));
+      break;
+    }
+    let value: string | undefined;
+    if (argument === "--project") {
+      value = argv[index + 1];
+      if (value === undefined || value.startsWith("-")) throw new UsageError("--project needs a folder");
+      index++;
+    } else if (argument.startsWith("--project=")) {
+      value = argument.slice("--project=".length);
+      if (value === "") throw new UsageError("--project needs a folder");
+    } else {
+      rest.push(argument);
+      continue;
+    }
+    if (project !== undefined) throw new UsageError("give --project only once");
+    project = value;
+  }
+  return project === undefined ? { argv: rest } : { project, argv: rest };
+}
+
 function printLine(value: unknown): void {
   process.stdout.write(`${JSON.stringify(value)}\n`);
 }
 
 /**
- * Runs a script body. Prints its result as one JSON line when it returns a value (undefined prints nothing).
- * A refusal (WireFailure always, plus `refusals`) prints {"error":{code,message}} with exit status 1.
+ * Runs a script body with the command line of the process, without --project: that sets the project first (D67).
+ * Prints its result as one JSON line when it returns a value (undefined prints nothing).
+ * A refusal (WireFailure and ProjectError always, plus `refusals`) prints {"error":{code,message}} with exit status 1.
  * A UsageError or a node:util parseArgs error (code ERR_PARSE_ARGS_*) prints {"error":{"code":"invalid-input",
  * "message": message === "" ? usage : message + "\n\n" + usage}} with exit status 2. Anything else is rethrown.
  * The body may set process.exitCode itself for an extra exit status; a value it returns is printed as usual.
  */
-export async function runScript(body: () => unknown, options: { usage: string; refusals?: readonly RefusalClass[] }): Promise<void> {
+export async function runScript(body: (argv: string[]) => unknown, options: { usage: string; refusals?: readonly RefusalClass[] }): Promise<void> {
+  const usage = `${options.usage}\n\n${PROJECT_USAGE}`;
   // The trace of the script, when C64RT_TRACE is set; its start event carries the script and its arguments.
   startTrace("script", { warn: (line) => process.stderr.write(`${line}\n`) });
   const started = performance.now();
   const end = (fields: TraceFields) => trace().event("script.end", { ms: elapsedMs(started), status: process.exitCode ?? 0, ...fields });
   let result: unknown;
   try {
-    result = await body();
+    const { project, argv } = takeProject(process.argv.slice(2));
+    if (project !== undefined) useProject(project);
+    result = await body(argv);
   } catch (error) {
-    if ([WireFailure, ...(options.refusals ?? [])].some((refusal) => error instanceof refusal)) {
-      const { code, message } = error as Error & { code: string };
+    if ([WireFailure, ProjectError, ...(options.refusals ?? [])].some((refusal) => error instanceof refusal)) {
+      const { code } = error as Error & { code: string };
+      const message = shownMessage(error as Error & { code: string });
       printLine({ error: { code, message } });
       process.exitCode = 1;
       end({ error: { code, message } });
@@ -46,7 +86,7 @@ export async function runScript(body: () => unknown, options: { usage: string; r
     }
     if (error instanceof UsageError || isParseArgsError(error)) {
       const { message } = error;
-      printLine({ error: { code: "invalid-input", message: message === "" ? options.usage : `${message}\n\n${options.usage}` } });
+      printLine({ error: { code: "invalid-input", message: message === "" ? usage : `${message}\n\n${usage}` } });
       process.exitCode = 2;
       end({ error: { code: "invalid-input", message } });
       return;

package/src/trace.ts

@@ -1,7 +1,8 @@
 // An opt-in trace for the person who debugs a run. With C64RT_TRACE=<directory>
-// in the environment, each process appends its events as JSON lines to a file
-// of its own in that directory; unset, every call does nothing. The trace is
-// for people: nothing of it reaches the agent, and no result depends on it.
+// in the environment (the host: --log-dir), each process appends its events as
+// JSON lines to a file of its own in that directory. The host can also echo
+// each event to the console (--log-console). Off, every call does nothing. The
+// trace is for people: nothing of it reaches the agent, and no result depends on it.
 
 import { appendFileSync, copyFileSync, mkdirSync, readFileSync } from "node:fs";
 import { join } from "node:path";
@@ -66,14 +67,16 @@ export class Trace {
   readonly kind: TraceKind;
   /** The directory of the trace, or undefined when the trace is off. */
   readonly dir: string | undefined;
-  /** The file this trace appends to, or undefined when the trace is off. */
+  /** The file this trace appends to, or undefined when the trace writes no file. */
   readonly file: string | undefined;
+  readonly #echo: ((line: string) => void) | undefined;
   #broken = false;
 
-  private constructor(kind: TraceKind, dir: string | undefined, file: string | undefined) {
+  private constructor(kind: TraceKind, dir: string | undefined, file: string | undefined, echo?: (line: string) => void) {
     this.kind = kind;
     this.dir = dir;
     this.file = file;
+    this.#echo = echo;
   }
 
   /** A trace that records nothing. */
@@ -81,31 +84,46 @@ export class Trace {
     return new Trace(kind, undefined, undefined);
   }
 
+  /** A trace that writes no file and gives each event to `echo` as one line. */
+  static echoing(kind: TraceKind, echo: (line: string) => void): Trace {
+    return new Trace(kind, undefined, undefined, echo);
+  }
+
   /**
    * A trace into `dir`, created when it is missing. The file is
    * `<kind>-<CET time>-<pid>.jsonl`, and its first event is `trace.start`.
    * Throws when the directory cannot be used.
    */
-  static open(kind: TraceKind, dir: string, now = new Date()): Trace {
+  static open(kind: TraceKind, dir: string, now = new Date(), echo?: (line: string) => void): Trace {
     mkdirSync(dir, { recursive: true });
-    const trace = new Trace(kind, dir, join(dir, `${kind}-${fileStampCet(now)}-${process.pid}.jsonl`));
+    const trace = new Trace(kind, dir, join(dir, `${kind}-${fileStampCet(now)}-${process.pid}.jsonl`), echo);
     trace.event("trace.start", { version: runningVersion(), pid: process.pid, argv: process.argv.slice(1), cwd: process.cwd(), node: process.version, platform: process.platform });
     return trace;
   }
 
   get on(): boolean {
+    return this.#writes || this.#echo !== undefined;
+  }
+
+  get #writes(): boolean {
     return this.file !== undefined && !this.#broken;
   }
 
-  /** Appends one event. A write that fails turns the trace off; nothing throws. */
+  /**
+   * Records one event: appends it to the file, and gives it to the echo as
+   * "<event> <fields as JSON>". The echo skips `log` events, which the program
+   * prints itself. A write that fails stops the file; nothing throws.
+   */
   event(name: string, fields: TraceFields = {}): void {
     if (!this.on) return;
-    const line: Record<string, unknown> = { t: isoCet(), kind: this.kind, event: name };
+    const summary: Record<string, unknown> = {};
     for (const [key, value] of Object.entries(fields)) {
-      if (!RESERVED.has(key) && value !== undefined) line[key] = summarize(value);
+      if (!RESERVED.has(key) && value !== undefined) summary[key] = summarize(value);
     }
+    if (this.#echo !== undefined && name !== "log") this.#echo(`${name} ${JSON.stringify(summary)}`);
+    if (!this.#writes) return;
     try {
-      appendFileSync(this.file!, `${JSON.stringify(line)}\n`);
+      appendFileSync(this.file!, `${JSON.stringify({ t: isoCet(), kind: this.kind, event: name, ...summary })}\n`);
     } catch {
       this.#broken = true;
     }
@@ -121,7 +139,7 @@ export class Trace {
    * trace, as `<kind>-<pid>-<name>`. A copy that fails is recorded, not thrown.
    */
   keep(source: string, name: string): void {
-    if (!this.on) return;
+    if (!this.#writes) return;
     const target = join(this.dir!, `${this.kind}-${process.pid}-${name}`);
     try {
       copyFileSync(source, target);
@@ -135,23 +153,37 @@ export class Trace {
 // Off until a program calls startTrace; the kind is a placeholder until then.
 let current = Trace.off("script");
 
+/** Where the trace of a process goes: a directory for its file, and an echo for each event. */
+export interface TraceTarget {
+  /** The directory, and how the user gave it (`--log-dir <path>`, for example) for a warning. */
+  dir?: { path: string; given: string };
+  echo?: (line: string) => void;
+}
+
+/** The target that C64RT_TRACE in `env` gives: its directory, or nothing. */
+export function traceTargetFromEnv(env: NodeJS.ProcessEnv = process.env): TraceTarget {
+  const path = env[TRACE_VARIABLE];
+  return path === undefined || path === "" ? {} : { dir: { path, given: `${TRACE_VARIABLE}=${path}` } };
+}
+
 /**
- * Switches the trace of this process on when `env` (the environment by
- * default) has C64RT_TRACE, and off otherwise. A directory that cannot be used
- * leaves the trace off and reports that through `warn`. Called once, at the
- * start of a program.
+ * Switches the trace of this process on for `target` (by default what
+ * C64RT_TRACE in `env` gives), and off when the target is empty. A directory
+ * that cannot be used writes no file and reports that through `warn`; an echo
+ * stays on. Called once, at the start of a program.
  */
-export function startTrace(kind: TraceKind, options: { env?: NodeJS.ProcessEnv; warn?: (line: string) => void } = {}): Trace {
-  const dir = (options.env ?? process.env)[TRACE_VARIABLE];
-  if (dir === undefined || dir === "") {
-    current = Trace.off(kind);
+export function startTrace(kind: TraceKind, options: { env?: NodeJS.ProcessEnv; target?: TraceTarget; warn?: (line: string) => void } = {}): Trace {
+  const { dir, echo } = options.target ?? traceTargetFromEnv(options.env);
+  const noFile = () => (echo === undefined ? Trace.off(kind) : Trace.echoing(kind, echo));
+  if (dir === undefined) {
+    current = noFile();
     return current;
   }
   try {
-    current = Trace.open(kind, dir);
+    current = Trace.open(kind, dir.path, new Date(), echo);
   } catch (error) {
-    current = Trace.off(kind);
-    options.warn?.(`${TRACE_VARIABLE}=${dir} cannot be used (${error instanceof Error ? error.message : String(error)}); the trace is off.`);
+    current = noFile();
+    options.warn?.(`${dir.given} cannot be used (${error instanceof Error ? error.message : String(error)}); no trace file is written.`);
   }
   return current;
 }

package/skills/c64-provenance/scripts/compare.ts

@@ -1,92 +0,0 @@
-// Byte comparison of releases for the c64-provenance script. Each release is
-// placed at its load address plus an optional shift; the comparison reports
-// where the releases differ and which releases agree there. It never decides
-// which bytes are original: that needs evidence beyond one comparison.
-
-export interface Release {
-  name: string;
-  /** Address of the first byte after the shift. */
-  start: number;
-  bytes: Uint8Array;
-}
-
-export interface DifferingRange {
-  start: number;
-  end: number;
-  /** The bytes of each release in this range, by release name. */
-  values: Record<string, number[]>;
-  /** Releases with the same bytes here, largest group first. */
-  groups: string[][];
-}
-
-export interface Comparison {
-  /** The addresses that every release covers. */
-  common: { start: number; end: number } | null;
-  identicalBytes: number;
-  differingBytes: number;
-  differences: DifferingRange[];
-  /** True when more differing ranges exist than the limit allowed. */
-  moreDifferences: boolean;
-}
-
-/** Joins differing addresses that are at most this far apart into one range. */
-const JOIN_GAP = 4;
-
-export function compareReleases(releases: Release[], options: { from?: number; to?: number; maxRanges?: number } = {}): Comparison {
-  const start = Math.max(options.from ?? 0, ...releases.map((release) => release.start));
-  const end = Math.min(options.to ?? 0xffff, ...releases.map((release) => release.start + release.bytes.length - 1));
-  if (start > end) return { common: null, identicalBytes: 0, differingBytes: 0, differences: [], moreDifferences: false };
-  const at = (release: Release, address: number) => release.bytes[address - release.start]!;
-  let identicalBytes = 0;
-  const differing: number[] = [];
-  for (let address = start; address <= end; address++) {
-    const first = at(releases[0]!, address);
-    if (releases.every((release) => at(release, address) === first)) identicalBytes++;
-    else differing.push(address);
-  }
-  const ranges: Array<{ start: number; end: number }> = [];
-  for (const address of differing) {
-    const last = ranges.at(-1);
-    if (last !== undefined && address - last.end <= JOIN_GAP) last.end = address;
-    else ranges.push({ start: address, end: address });
-  }
-  const limit = options.maxRanges ?? 64;
-  const differences = ranges.slice(0, limit).map((range) => {
-    const values: Record<string, number[]> = {};
-    const groups = new Map<string, string[]>();
-    for (const release of releases) {
-      const bytes = Array.from(release.bytes.subarray(range.start - release.start, range.end - release.start + 1));
-      values[release.name] = bytes;
-      const key = bytes.join(",");
-      groups.set(key, [...(groups.get(key) ?? []), release.name]);
-    }
-    return { ...range, values, groups: [...groups.values()].sort((a, b) => b.length - a.length) };
-  });
-  return { common: { start, end }, identicalBytes, differingBytes: differing.length, differences, moreDifferences: ranges.length > limit };
-}
-
-/**
- * The shift that puts `other` over `base` best: the most common address
- * difference between equal 8-byte sequences. Undefined when nothing matches.
- */
-export function suggestShift(base: Release, other: Release): { shift: number; matches: number } | undefined {
-  const WINDOW = 8;
-  const key = (bytes: Uint8Array, at: number) => Buffer.from(bytes.subarray(at, at + WINDOW)).toString("latin1");
-  const positions = new Map<string, number[]>();
-  for (let at = 0; at + WINDOW <= base.bytes.length; at++) {
-    const positionsOfKey = positions.get(key(base.bytes, at));
-    // Very common sequences (fill bytes) say nothing about the alignment.
-    if (positionsOfKey === undefined) positions.set(key(base.bytes, at), [at]);
-    else if (positionsOfKey.length < 8) positionsOfKey.push(at);
-  }
-  const votes = new Map<number, number>();
-  for (let at = 0; at + WINDOW <= other.bytes.length; at++) {
-    for (const position of positions.get(key(other.bytes, at)) ?? []) {
-      const shift = base.start + position - (other.start + at);
-      votes.set(shift, (votes.get(shift) ?? 0) + 1);
-    }
-  }
-  let best: { shift: number; matches: number } | undefined;
-  for (const [shift, matches] of votes) if (best === undefined || matches > best.matches) best = { shift, matches };
-  return best;
-}

package/skills/c64-unpacker/scripts/evidence.ts

@@ -1,135 +0,0 @@
-// Local evidence of packing for the c64-unpacker script: where BASIC hands
-// over and how compressed the bytes look. It never names a packer.
-
-import { formatC64Address } from "#src/c64.ts";
-
-export class PackingError extends Error {}
-
-/** A 256-byte block above this many bits per byte looks compressed. */
-const HIGH_ENTROPY = 6.9;
-const BLOCK = 256;
-
-function entropy(bytes: Uint8Array): number {
-  const counts = new Array<number>(256).fill(0);
-  for (const byte of bytes) counts[byte]!++;
-  let bits = 0;
-  for (const count of counts) {
-    if (count === 0) continue;
-    const p = count / bytes.length;
-    bits -= p * Math.log2(p);
-  }
-  return bits;
-}
-
-/** The constant SYS address of the first BASIC line, when the program starts with one. */
-function basicSys(program: Uint8Array): { line: number; address: number; end: number } | undefined {
-  if (program.length < 8 || program[0] !== 0x01 || program[1] !== 0x08) return undefined;
-  const number = program[4]! | (program[5]! << 8);
-  const end = program.indexOf(0, 6);
-  if (end < 0) return undefined;
-  const sys = program.subarray(6, end).indexOf(0x9e);
-  if (sys < 0) return undefined;
-  const digits = Buffer.from(program.subarray(6 + sys + 1, end)).toString("latin1").replace(/[ (]/g, "");
-  const match = /^(\d{1,5})/.exec(digits);
-  if (match === null || Number(match[1]) > 0xffff) return undefined;
-  return { line: number, address: Number(match[1]), end: 0x0801 + end - 2 };
-}
-
-/** The local packing evidence of one PRG. */
-export interface PackingEvidence {
-  loadRange: { start: string; end: string };
-  bytes: number;
-  /** The first BASIC line and its constant SYS address, or null when the program has none. */
-  basicStart: { line: number; sys: string } | null;
-  entropy: { blocks: number; highBlocks: number; median: number | null };
-  packing: "likely" | "unlikely" | "unclear";
-  evidence: string;
-}
-
-export function inspect(program: Uint8Array): PackingEvidence {
-  if (program.length < 3) throw new PackingError("the file is too short for a PRG");
-  const load = program[0]! | (program[1]! << 8);
-  const end = load + program.length - 3;
-  const stub = basicSys(program);
-  // Without a BASIC start, all bytes count; with one, only the bytes after the BASIC line.
-  const from = stub === undefined ? 2 : Math.min(program.length, stub.end - load + 2 + 3);
-  const body = program.subarray(from);
-  const blocks: number[] = [];
-  for (let at = 0; at + BLOCK <= body.length; at += BLOCK) blocks.push(entropy(body.subarray(at, at + BLOCK)));
-  const high = blocks.filter((value) => value > HIGH_ENTROPY).length;
-  const share = blocks.length === 0 ? 0 : high / blocks.length;
-  const packing = blocks.length < 2 ? "unclear" : share >= 0.75 ? "likely" : share <= 0.1 ? "unlikely" : "unclear";
-  const sorted = [...blocks].sort((a, b) => a - b);
-  return {
-    loadRange: { start: formatC64Address(load), end: formatC64Address(Math.min(end, 0xffff)) },
-    bytes: program.length - 2,
-    basicStart: stub === undefined ? null : { line: stub.line, sys: formatC64Address(stub.address) },
-    entropy: {
-      blocks: blocks.length,
-      highBlocks: high,
-      median: sorted.length === 0 ? null : Number(sorted[sorted.length >> 1]!.toFixed(2)),
-    },
-    packing,
-    evidence:
-      packing === "likely"
-        ? `${high} of ${blocks.length} blocks of 256 bytes look compressed (above ${HIGH_ENTROPY} bits per byte); code and data are usually below.`
-        : packing === "unlikely"
-          ? `Only ${high} of ${blocks.length} blocks look compressed; the bytes look like code and data. A simple cruncher can look the same: run trace to see if the program writes code and then executes it.`
-          : "The entropy does not show clearly if the program is packed. Run it and look at what it writes to memory.",
-  };
-}
-
-/** One range of the emulator's memory map: what the program did with these addresses. */
-export interface AccessRange {
-  start: number;
-  end: number;
-  read: boolean;
-  write: boolean;
-  execute: boolean;
-}
-
-/** The addresses that one memory map read covers before it is split. */
-const MEMMAP_WINDOW = 0x1000;
-
-/**
- * Reads the memory map of $0000-$ffff in windows, in address order. `read`
- * gives at most `maxRanges` ranges for one window; a window that gives that
- * many can have more, so it is read again as two halves.
- */
-export async function readMemoryMap(read: (start: number, end: number) => Promise<AccessRange[]>, maxRanges: number): Promise<AccessRange[]> {
-  const ranges: AccessRange[] = [];
-  const window = async (start: number, end: number): Promise<void> => {
-    const found = await read(start, end);
-    if (found.length < maxRanges || start === end) {
-      ranges.push(...found);
-      return;
-    }
-    const middle = Math.floor((start + end) / 2);
-    await window(start, middle);
-    await window(middle + 1, end);
-  };
-  for (let start = 0; start <= 0xffff; start += MEMMAP_WINDOW) await window(start, start + MEMMAP_WINDOW - 1);
-  return ranges;
-}
-
-/** ROM areas: an address here can be written (RAM below) and executed (ROM) without one being the other. */
-export const underRom = (address: number) => (address >= 0xa000 && address <= 0xbfff) || address >= 0xe000;
-
-/**
- * The memory that the program wrote and then executed, in pieces of code.
- * `bytes` and `running` count only RAM outside the ROM areas: a write under
- * a ROM and code that runs in that ROM are not one piece of code.
- */
-export function writtenThenExecuted(ranges: AccessRange[], pc: number): { written: Array<{ start: number; end: number }>; bytes: number; running?: { start: number; end: number } } {
-  // VICE marks only opcode bytes as executed; a gap of at most two operand bytes stays inside one piece of code.
-  const written: Array<{ start: number; end: number }> = [];
-  for (const range of ranges.filter((candidate) => candidate.write && candidate.execute)) {
-    const last = written.at(-1);
-    if (last !== undefined && range.start - last.end <= 3 && underRom(range.start) === underRom(last.start)) last.end = range.end;
-    else written.push({ start: range.start, end: range.end });
-  }
-  const counted = written.filter((range) => !underRom(range.start));
-  const bytes = counted.reduce((sum, range) => sum + range.end - range.start + 1, 0);
-  const running = counted.find((range) => pc >= range.start && pc <= range.end);
-  return { written, bytes, ...(running === undefined ? {} : { running }) };
-}