@henols/c64-re-tools 1.1.0 → 2.0.0-rc.2

package/src/native/dxa.ts

@@ -0,0 +1,166 @@
+// The DXA adapter. DXA prints an assembler listing only; with
+// "-a dump" each line starts with its address and bytes, so the adapter can
+// tell code from data line by line. The whole listing is checked before the
+// script imports anything: lines in address order from the load address to
+// the end, each line as long as its bytes or data items, so a truncated or
+// changed listing is refused instead of half imported.
+
+import { writeFileSync } from "node:fs";
+
+import { SYMBOL_NAME } from "../c64.ts";
+import { describeError, WireFailure } from "../protocol/messages.ts";
+import { checkDxaResult, checkRequestBounds, imageRange, seedsInside, type DxaParams, type DxaResult } from "./types.ts";
+import { Workspace } from "./staging.ts";
+import { DXA, findTool } from "./discover.ts";
+import { outputTail, runTool, runToolOrFail, type ToolContext } from "./run.ts";
+import { DXA_VERSION, versionLine } from "./status.ts";
+
+const TIMEOUT_MS = 120_000;
+/** The listing is returned as an attachment; larger output is refused. */
+const MAX_LISTING = 8 * 1024 * 1024;
+const VERSION_TIMEOUT_MS = 20_000;
+const MAX_VERSION_OUTPUT = 64 * 1024;
+
+const hex = (value: number) => value.toString(16).padStart(4, "0");
+
+/**
+ * The options for one run; every value comes from the validated request. A
+ * seed file is passed only when it has lines: dxa refuses an empty one.
+ */
+export function dxaArguments(executable: string, params: DxaParams, files: { image: string; routines?: string; blocks?: string; labels?: string }): string[] {
+  const argv = [executable, "-a", "dump", "-p", "all-nmos6502", "-d", "skip-scanning"];
+  if (files.routines !== undefined) argv.push("-R", files.routines);
+  if (files.blocks !== undefined) argv.push("-B", files.blocks);
+  if (files.labels !== undefined) argv.push("-l", files.labels);
+  // A PRG gives its load address and may start with BASIC (-U finds its SYS);
+  // 64 KiB of memory starts at $0000 and has no load address.
+  if (params.imageKind === "prg") argv.push("-U");
+  else argv.push("-g", "0000", "-q");
+  argv.push(files.image);
+  return argv;
+}
+
+/** Number of data bytes in a .byt or .word line, or undefined for an instruction. */
+function dataSize(operation: string): number | undefined {
+  const match = /^\.(byt|word)\s+(.+)$/.exec(operation.trim());
+  if (match === null) return undefined;
+  const items = match[2]!.split(",").length;
+  return match[1] === "word" ? items * 2 : items;
+}
+
+/**
+ * Parses a "-a dump" listing into regions and labels for [start, end].
+ * Throws an Error when the listing is not a complete, consistent listing of
+ * exactly that range.
+ */
+export function parseListing(listing: string, start: number, end: number): Pick<DxaResult, "regions" | "labels"> {
+  const statements: Array<{ address: number; dumped: number; data: number | undefined }> = [];
+  const labels: DxaResult["labels"] = [];
+  for (const line of listing.split(/\r?\n/)) {
+    // dxa prints a seed label as it is, also a local name that starts with a dot.
+    const label = /^([0-9a-f]{4})\s+(\.?[A-Za-z_]\w*):\s*$/.exec(line);
+    if (label !== null) {
+      const address = Number.parseInt(label[1]!, 16);
+      if (address < start || address > end || !SYMBOL_NAME.test(label[2]!)) throw new Error(`label ${label[2]} at $${label[1]} is not usable`);
+      labels.push({ address, name: label[2]! });
+      continue;
+    }
+    // dxa pads the bytes column with a tab, but a two-byte .word line with spaces only.
+    const statement = /^([0-9a-f]{4}) ((?:[0-9a-f]{2} ?){1,3})\s+(\S.*)$/.exec(line);
+    if (statement === null) continue;
+    statements.push({ address: Number.parseInt(statement[1]!, 16), dumped: statement[2]!.trim().split(/\s+/).length, data: dataSize(statement[3]!) });
+  }
+  const first = statements[0];
+  const last = statements.at(-1);
+  if (first === undefined || last === undefined) throw new Error("the listing has no statements");
+  if (first.address !== start) throw new Error(`the listing starts at $${hex(first.address)}, not at $${hex(start)}`);
+  if (last.address > end) throw new Error("the listing runs past the end of the program");
+  const regions: DxaResult["regions"] = [];
+  statements.forEach((statement, index) => {
+    const next = statements[index + 1]?.address ?? end + 1;
+    const size = next - statement.address;
+    if (size <= 0) throw new Error(`the listing goes back at $${hex(next)}`);
+    // Data lines count their items; an instruction line dumps exactly its bytes.
+    const expected = statement.data ?? statement.dumped;
+    if (expected !== size) throw new Error(`the line at $${hex(statement.address)} holds ${expected} bytes, but the next line is ${size} bytes later`);
+    const classification = statement.data === undefined ? "code" : "data";
+    const last = regions.at(-1);
+    if (last?.classification === classification) last.end = next - 1;
+    else regions.push({ start: statement.address, end: next - 1, classification });
+  });
+  return { regions, labels };
+}
+
+/**
+ * The version line of dxa, for example "dxa v0.1.5 -- symbolic 65xx
+ * disassembler". "dxa -V" prints it before its usage and exits with status 1.
+ * On Windows dxa names itself by its full path and ".exe": the line names it
+ * "dxa" on every system. Undefined when dxa prints no version line.
+ */
+export async function dxaVersion(executable: string, cwd: string, context: ToolContext): Promise<string | undefined> {
+  const run = await runTool({ argv: [executable, "-V"], cwd, supervisor: context.supervisor, signal: context.signal, timeoutMs: VERSION_TIMEOUT_MS, outputLimit: MAX_VERSION_OUTPUT, env: context.env });
+  return versionLine(`${run.stdout}\n${run.stderr}`, DXA_VERSION)?.replace(/^(?:.*[\\/])?dxa(?:\.exe)?/i, "dxa");
+}
+
+export async function analyze(
+  params: DxaParams,
+  image: Buffer,
+  context: ToolContext,
+): Promise<{ result: DxaResult; attachments: Buffer[] }> {
+  checkRequestBounds(params);
+  const executable = findTool(DXA, context.env);
+  const { start, end } = imageRange(params.imageKind, image);
+  const { entryPoints, dataRanges } = seedsInside(params, start, end);
+
+  const workspace = Workspace.create(context.supervisor);
+  try {
+    // A cancelled version run gives no version; the analysis run below then refuses the cancelled request.
+    const toolVersion = await dxaVersion(executable, workspace.root, context);
+    workspace.materialize("input", [{ path: "image.bin", size: image.length }], [image]);
+    const files: Parameters<typeof dxaArguments>[2] = { image: "input/image.bin" };
+    const seed = (name: "routines" | "blocks" | "labels", lines: string[]) => {
+      if (lines.length === 0) return;
+      writeFileSync(workspace.path(`${name}.txt`), lines.join(""));
+      files[name] = `${name}.txt`;
+    };
+    seed("routines", entryPoints.map((address) => `${hex(address)}\n`));
+    seed("blocks", dataRanges.map((range) => `${hex(range.start)}-${hex(range.end)}\n`));
+    // The xa label file format: name, address, flags. dxa reads the name up to the first blank, so a leading dot stays.
+    seed("labels", params.labels.filter((label) => SYMBOL_NAME.test(label.name)).map((label) => `${label.name}, 0x${hex(label.address)}, 0x0000\n`));
+    const run = await runToolOrFail(
+      "dxa",
+      "The DXA analysis",
+      {
+        argv: dxaArguments(executable, params, files),
+        cwd: workspace.root,
+        timeoutMs: TIMEOUT_MS,
+        outputLimit: MAX_LISTING,
+        truncated: `dxa printed a listing that is larger than ${MAX_LISTING / 1024 / 1024} MiB. Nothing was imported.`,
+      },
+      context,
+    );
+    if (run.code !== 0) {
+      context.log?.(`dxa exited ${run.code}: ${run.stderr}`);
+      const tail = outputTail("dxa", run, workspace.root);
+      throw new WireFailure("operation-failed", `dxa could not analyze the program${tail === "" ? ` (exit ${run.code}).` : `.${tail}`}`);
+    }
+    const listing = Buffer.from(run.stdout, "utf8");
+    let result: DxaResult;
+    try {
+      const parsed = parseListing(run.stdout, start, end);
+      // DXA classifies every byte, so its regions are complete; it names only referenced addresses.
+      result = checkDxaResult(
+        { coverage: [{ start, end }], ...parsed, listingBytes: listing.length, completeness: { regions: true, labels: false }, ...(toolVersion === undefined ? {} : { toolVersion }) },
+        [listing],
+      );
+    } catch (error) {
+      context.log?.(`dxa listing rejected: ${describeError(error)}`);
+      // The listing is on stdout: only dxa's messages on stderr are quoted.
+      const tail = outputTail("dxa", { stdout: "", stderr: run.stderr }, workspace.root);
+      throw new WireFailure("operation-failed", `dxa returned an incomplete or inconsistent listing. Nothing was imported. The check found: ${describeError(error)}.${tail}`);
+    }
+    return { result, attachments: [listing] };
+  } finally {
+    workspace.remove();
+  }
+}

package/src/native/ghidra/analyze.ts

@@ -0,0 +1,86 @@
+// The Ghidra analysis: one disposable headless run with the NMOS
+// 6510 language, knowledge seeds before analysis, and the structural export
+// after it. The result is checked completely before the script imports it.
+
+import { existsSync, readFileSync, writeFileSync } from "node:fs";
+
+import { describeError, isObject, WireFailure } from "../../protocol/messages.ts";
+import { checkGhidraResult, checkRequestBounds, GHIDRA_LIMITS, imageRange, seedsInside, type GhidraParams, type GhidraResult } from "../types.ts";
+import { outputTail, type ToolContext } from "../run.ts";
+import { Workspace } from "../staging.ts";
+import { findGhidra, ghidraVersion, runHeadless, SCRIPT_DIRECTORY } from "./index.ts";
+
+const TIMEOUT_MS = 600_000;
+/** The part of the run that the decompilations leave for the export, the project removal and the end of Ghidra. */
+const AFTER_DECOMPILE_MS = 30_000;
+
+export async function analyze(
+  params: GhidraParams,
+  image: Buffer,
+  context: ToolContext,
+): Promise<{ result: GhidraResult }> {
+  checkRequestBounds(params);
+  const ghidra = findGhidra(context.env);
+  const { start, end, body } = imageRange(params.imageKind, image);
+  const base = start;
+
+  // Ghidra disassembles and defines data only in the loaded bytes; labels may name any address.
+  const { entryPoints, dataRanges } = seedsInside(params, start, end);
+  if (entryPoints.length === 0) {
+    throw new WireFailure("invalid-input", "Give at least one entry point inside the program, for example the SYS address of its BASIC loader.");
+  }
+
+  const workspace = Workspace.create(context.supervisor);
+  try {
+    workspace.materialize("input", [{ path: "image.bin", size: body.length }], [body]);
+    writeFileSync(workspace.path("seeds.json"), JSON.stringify({ start, end, entryPoints, dataRanges, labels: params.labels }));
+    writeFileSync(
+      workspace.path("request.json"),
+      JSON.stringify({
+        start,
+        end,
+        decompile: params.decompile,
+        maxChars: GHIDRA_LIMITS.decompiledChars,
+        maxTotalChars: GHIDRA_LIMITS.decompiledTotalChars,
+        decompileDeadline: Date.now() + TIMEOUT_MS - AFTER_DECOMPILE_MS,
+      }),
+    );
+    const run = await runHeadless({
+      ghidra,
+      workspace,
+      file: "input/image.bin",
+      baseAddress: base,
+      scriptDirectories: [SCRIPT_DIRECTORY],
+      preScripts: [{ name: "C64Prepare.java", args: [workspace.path("seeds.json")] }],
+      postScripts: [{ name: "C64Export.java", args: [workspace.path("request.json"), workspace.path("out.json")] }],
+      analyze: true,
+      timeoutMs: TIMEOUT_MS,
+      supervisor: context.supervisor,
+      signal: context.signal,
+      ...(context.env === undefined ? {} : { env: context.env }),
+    }).catch((error: unknown) => {
+      context.log?.(`Ghidra failed: ${describeError(error)}`);
+      throw error;
+    });
+    const outPath = workspace.path("out.json");
+    if (!existsSync(outPath)) {
+      context.log?.(`Ghidra wrote no result:\n${run.stdout.slice(-4000)}\n${run.stderr.slice(-4000)}`);
+      // Ghidra's own warnings and errors say why its export script wrote nothing.
+      throw new WireFailure("operation-failed", `Ghidra finished without a result.${outputTail("Ghidra", run, workspace.root, /ERROR|WARN|Exception|Error:|SCRIPT/)}`);
+    }
+    let result: GhidraResult;
+    try {
+      const exported: unknown = JSON.parse(readFileSync(outPath, "utf8"));
+      // The version comes from the installation's application.properties, which findGhidra read.
+      result = checkGhidraResult(isObject(exported) ? { ...exported, toolVersion: ghidraVersion(ghidra) } : exported);
+      const [range] = result.coverage;
+      if (result.coverage.length !== 1 || range === undefined || range.start !== start || range.end !== end) throw new Error("the coverage is not the loaded program");
+    } catch (error) {
+      context.log?.(`Ghidra result rejected: ${describeError(error)}`);
+      throw new WireFailure("operation-failed", `Ghidra returned an incomplete or inconsistent result. Nothing was imported. The check found: ${describeError(error)}.`);
+    }
+    return { result };
+  } finally {
+    workspace.remove();
+  }
+}

package/src/native/ghidra/index.ts

@@ -0,0 +1,157 @@
+// Headless Ghidra, run by the c64-static-analysis script. Ghidra runs with a settings
+// directory of its own per request: the c64-re-tools NMOS 6510 language goes
+// in there as an extension, so the user's Ghidra installation and settings
+// never change. Ghidra compiles the language from its source on first load.
+// Projects are disposable and live in the request workspace.
+
+import { cpSync, mkdirSync, readFileSync, realpathSync, writeFileSync } from "node:fs";
+import { userInfo } from "node:os";
+import { dirname, isAbsolute, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+
+import { WireFailure } from "../../protocol/messages.ts";
+import { findOnPath, isExecutableFile } from "../discover.ts";
+import type { ProcessSupervisor } from "../processes.ts";
+import type { Workspace } from "../staging.ts";
+import { outputTail, runToolOrFail, type ToolRun } from "../run.ts";
+
+export const GHIDRA_LANGUAGE = "C64RT_6510:LE:16:nmos";
+// Next to this module. New URL literals, so an installed skill gets the directories too.
+export const LANGUAGE_DIRECTORY = fileURLToPath(new URL("./language/", import.meta.url));
+export const SCRIPT_DIRECTORY = fileURLToPath(new URL("./scripts/", import.meta.url));
+
+const REMEDY =
+  "Install Ghidra on the machine that runs the skill scripts and set C64RT_GHIDRA to its installation directory " +
+  "(the directory that holds support/analyzeHeadless), or put analyzeHeadless on PATH.";
+
+export interface GhidraInstallation {
+  /** The installation directory. */
+  root: string;
+  analyzeHeadless: string;
+  /** For example ghidra_12.1.3_PUBLIC: the name of Ghidra's settings directory. */
+  settingsName: string;
+  version: string;
+}
+
+/** Ghidra's launcher is a shell script, and a batch file on Windows. */
+const LAUNCHER_SUFFIX = process.platform === "win32" ? ".bat" : "";
+
+function installationAt(root: string, source: string): GhidraInstallation {
+  const analyzeHeadless = join(root, "support", `analyzeHeadless${LAUNCHER_SUFFIX}`);
+  let properties: string;
+  try {
+    if (!isExecutableFile(analyzeHeadless)) throw new Error("no analyzeHeadless");
+    properties = readFileSync(join(root, "Ghidra", "application.properties"), "utf8");
+  } catch {
+    throw new WireFailure("installation-incomplete", `${source} does not point to a Ghidra installation. ${REMEDY}`);
+  }
+  const value = (key: string) => new RegExp(`^${key.replace(/\./g, "\\.")}=(.*)$`, "m").exec(properties)?.[1]?.trim();
+  const version = value("application.version");
+  const release = value("application.release.name");
+  if (version === undefined || release === undefined) throw new WireFailure("installation-incomplete", `The Ghidra installation at ${source} has no version. ${REMEDY}`);
+  return { root, analyzeHeadless, settingsName: `ghidra_${version}_${release}`, version };
+}
+
+/** The version of a Ghidra installation as a result and the tool status name it, for example "Ghidra 12.1.3". */
+export function ghidraVersion(ghidra: GhidraInstallation): string {
+  return `Ghidra ${ghidra.version}`;
+}
+
+/** Finds Ghidra: C64RT_GHIDRA names the installation directory, else analyzeHeadless on PATH. */
+export function findGhidra(env: NodeJS.ProcessEnv = process.env): GhidraInstallation {
+  const configured = env.C64RT_GHIDRA;
+  if (configured !== undefined && configured !== "") {
+    if (!isAbsolute(configured)) throw new WireFailure("installation-incomplete", `C64RT_GHIDRA on this machine is not an absolute path. ${REMEDY}`);
+    return installationAt(configured, "C64RT_GHIDRA");
+  }
+  const found = findOnPath(["analyzeHeadless"], env, [LAUNCHER_SUFFIX]);
+  if (found !== undefined) return installationAt(resolve(dirname(realpathSync(found)), ".."), "analyzeHeadless on PATH");
+  throw new WireFailure("installation-incomplete", `Ghidra is not installed on this machine. ${REMEDY}`);
+}
+
+/**
+ * The user name in the name of Ghidra's settings directory: Java's user.name
+ * without spaces and without a domain before the last backslash or slash. Java gives "?"
+ * for a user without an entry in the password database, where Node throws.
+ */
+export function ghidraUserName(lookup: () => string = () => userInfo().username): string {
+  let name: string;
+  try {
+    name = lookup();
+  } catch {
+    name = "?";
+  }
+  name = name.split(" ").join("");
+  return name.slice(Math.max(name.lastIndexOf("\\"), name.lastIndexOf("/")) + 1);
+}
+
+/**
+ * Makes a Ghidra settings directory inside the workspace with the NMOS 6510
+ * language installed as an extension. Returns the environment that makes
+ * Ghidra use it.
+ */
+export function prepareSettings(workspace: Workspace, ghidra: GhidraInstallation, env: NodeJS.ProcessEnv = process.env): NodeJS.ProcessEnv {
+  const config = workspace.directory("ghidra-config");
+  const cache = workspace.directory("ghidra-cache");
+  // With XDG_CONFIG_HOME set, Ghidra keeps its settings in [<user>-]ghidra/<settings name> below it: with
+  // the user name only when that directory lies outside the user's home. The temporary directory is
+  // outside it on Linux and macOS but inside it on Windows (in AppData), so both names get it.
+  for (const application of [`${ghidraUserName()}-ghidra`, "ghidra"]) {
+    const extension = join(config, application, ghidra.settingsName, "Extensions", "C64RT");
+    mkdirSync(join(extension, "data"), { recursive: true });
+    cpSync(LANGUAGE_DIRECTORY, join(extension, "data", "languages"), { recursive: true });
+    writeFileSync(join(extension, "extension.properties"), `name=C64RT\ndescription=c64-re-tools NMOS 6510 language\nauthor=c64-re-tools\ncreatedOn=\nversion=${ghidra.version}\n`);
+    writeFileSync(join(extension, "Module.manifest"), "");
+  }
+  return { ...env, XDG_CONFIG_HOME: config, XDG_CACHE_HOME: cache };
+}
+
+export interface HeadlessOptions {
+  ghidra: GhidraInstallation;
+  workspace: Workspace;
+  /** Workspace-relative path of the file to import. */
+  file: string;
+  baseAddress: number;
+  /** Scripts from these directories, before and after analysis, with their arguments. */
+  scriptDirectories: string[];
+  preScripts?: Array<{ name: string; args: string[] }>;
+  postScripts?: Array<{ name: string; args: string[] }>;
+  analyze: boolean;
+  timeoutMs: number;
+  supervisor: ProcessSupervisor;
+  signal: AbortSignal;
+  env?: NodeJS.ProcessEnv;
+}
+
+/** Imports one file into a disposable project and runs the scripts. Fails on timeout, abort or a Ghidra error exit. */
+export async function runHeadless(options: HeadlessOptions): Promise<ToolRun> {
+  const env = prepareSettings(options.workspace, options.ghidra, options.env);
+  const project = options.workspace.directory("ghidra-project");
+  const argv = [
+    options.ghidra.analyzeHeadless,
+    project,
+    "request",
+    "-import",
+    options.workspace.path(options.file),
+    "-processor",
+    GHIDRA_LANGUAGE,
+    "-loader",
+    "BinaryLoader",
+    "-loader-baseAddr",
+    `0x${options.baseAddress.toString(16)}`,
+    "-scriptPath",
+    options.scriptDirectories.join(";"),
+    "-deleteProject",
+  ];
+  if (!options.analyze) argv.push("-noanalysis");
+  else argv.push("-analysisTimeoutPerFile", String(Math.ceil(options.timeoutMs / 1000)));
+  for (const script of options.preScripts ?? []) argv.push("-preScript", script.name, ...script.args);
+  for (const script of options.postScripts ?? []) argv.push("-postScript", script.name, ...script.args);
+  const run = await runToolOrFail("Ghidra", "The Ghidra analysis", { argv, cwd: options.workspace.root, timeoutMs: options.timeoutMs }, { supervisor: options.supervisor, signal: options.signal, env });
+  if (run.code !== 0 || /^ERROR REPORT SCRIPT ERROR|Exception in thread "main"/m.test(run.stdout + run.stderr)) {
+    // Ghidra's own error lines tell the user what to fix.
+    const tail = outputTail("Ghidra", run, options.workspace.root, /ERROR|Exception|Error:/);
+    throw new WireFailure("operation-failed", `Ghidra stopped with an error${tail === "" ? ` (exit ${run.code}).` : `.${tail}`}`);
+  }
+  return run;
+}

package/src/native/ghidra/language/c64rt_6510.cspec

@@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Calls pass values in A, X and Y and return them in A. -->
+<compiler_spec>
+  <global>
+    <range space="RAM"/>
+  </global>
+  <stackpointer register="SP" space="RAM" growth="negative"/>
+  <returnaddress>
+    <varnode space="stack" offset="1" size="2"/>
+  </returnaddress>
+  <default_proto>
+    <prototype name="__stdcall" extrapop="2" stackshift="2" strategy="register">
+      <input>
+        <pentry minsize="1" maxsize="1">
+          <register name="A"/>
+        </pentry>
+        <pentry minsize="1" maxsize="1">
+          <register name="X"/>
+        </pentry>
+        <pentry minsize="1" maxsize="1">
+          <register name="Y"/>
+        </pentry>
+      </input>
+      <output>
+        <pentry minsize="1" maxsize="1">
+          <register name="A"/>
+        </pentry>
+      </output>
+      <unaffected>
+        <register name="SP"/>
+      </unaffected>
+    </prototype>
+  </default_proto>
+</compiler_spec>

package/src/native/ghidra/language/c64rt_6510.ldefs

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- The c64-re-tools NMOS 6510 language: all 256 opcodes of the C64 CPU. -->
+<language_definitions>
+  <language processor="C64RT_6510"
+            endian="little"
+            size="16"
+            variant="nmos"
+            version="1.0"
+            slafile="c64rt_6510.sla"
+            processorspec="c64rt_6510.pspec"
+            id="C64RT_6510:LE:16:nmos">
+    <description>NMOS 6510 (C64) with all undocumented opcodes</description>
+    <compiler name="default" spec="c64rt_6510.cspec" id="default"/>
+  </language>
+</language_definitions>

package/src/native/ghidra/language/c64rt_6510.pspec

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  No default memory blocks or vector symbols: the importer makes the blocks
+  for the loaded program and the rest of the 64 KiB address space.
+  D is tracked as 0 at function entry, so the decimal path of ADC and SBC
+  disappears from decompiled code that never sets decimal mode.
+-->
+<processor_spec>
+  <programcounter register="PC"/>
+  <context_data>
+    <tracked_set space="RAM">
+      <set name="D" val="0"/>
+    </tracked_set>
+  </context_data>
+</processor_spec>