@henols/c64-re-tools 1.1.0 → 2.0.0-rc.10

package/src/native/status.ts

@@ -0,0 +1,113 @@
+// Tool status: finds each native tool the way its operations do and runs it
+// once with a version option, so a user can see in advance which tools work.
+// The Host Runtime reports VICE and its tools (host.status); the CLI reports
+// the tools that skill scripts run themselves (ACME, DXA, Ghidra).
+
+import { tmpdir } from "node:os";
+import { stripVTControlCharacters } from "node:util";
+
+import { WireFailure } from "../protocol/messages.ts";
+import { type ToolStatus } from "../protocol/tools.ts";
+import { ACME, DXA, findTool } from "./discover.ts";
+import { findGhidra, ghidraVersion, NO_USABLE_JDK, noJdk } from "./ghidra/index.ts";
+import { runTool, type ToolContext } from "./run.ts";
+import { Workspace } from "./staging.ts";
+
+const PROBE_TIMEOUT_MS = 20_000;
+
+export interface Probe {
+  name: string;
+  /** The executable; throws WireFailure(installation-incomplete) when missing. */
+  find: (env: NodeJS.ProcessEnv) => string;
+  args: string[];
+  /** The version line; no such line means the tool did not run correctly. */
+  pattern: RegExp;
+}
+
+/** dxa's version line, also as "D:\\tools\\dxa.exe v0.1.5 -- symbolic 65xx disassembler" on Windows. */
+export const DXA_VERSION = /^(?:.*[\\/])?dxa(?:\.exe)? v\d/i;
+
+/** The first output line that matches, without terminal color codes (c1541 colors an OpenCBM notice). */
+export function versionLine(output: string, pattern: RegExp): string | undefined {
+  return stripVTControlCharacters(output)
+    .split(/\r?\n/)
+    .map((line) => line.trim())
+    .find((line) => pattern.test(line));
+}
+
+async function run(
+  name: string,
+  path: string,
+  args: string[],
+  version: (output: string) => string | undefined,
+  context: ToolContext,
+  /** A known cause in the output, said in place of the exit status. */
+  cause?: (output: string) => string | undefined,
+): Promise<ToolStatus> {
+  const result = await runTool({ argv: [path, ...args], cwd: tmpdir(), supervisor: context.supervisor, signal: context.signal, timeoutMs: PROBE_TIMEOUT_MS, env: context.env });
+  const output = `${result.stdout}\n${result.stderr}`;
+  const found = version(output);
+  if (found !== undefined) return { name, found: true, path, version: found, runs: true };
+  const known = cause?.(output);
+  if (known !== undefined) return { name, found: true, path, runs: false, problem: known };
+  const problem = result.timedOut ? "It did not answer in time." : result.spawnError !== undefined ? `It did not start (${result.spawnError}).` : `It did not run correctly (exit ${result.code ?? result.signal}).`;
+  return { name, found: true, path, runs: false, problem };
+}
+
+function missing(name: string, error: unknown): ToolStatus {
+  if (!(error instanceof WireFailure)) throw error;
+  return { name, found: false, runs: false, problem: error.message };
+}
+
+/** Finds and runs each probed tool once. */
+export function probeTools(probes: Probe[], context: ToolContext): Promise<ToolStatus[]> {
+  const env = context.env ?? process.env;
+  return Promise.all(
+    probes.map(async (probe) => {
+      let path: string;
+      try {
+        path = probe.find(env);
+      } catch (error) {
+        return missing(probe.name, error);
+      }
+      return run(probe.name, path, probe.args, (output) => versionLine(output, probe.pattern), context);
+    }),
+  );
+}
+
+/** Ghidra: without arguments analyzeHeadless starts Java and prints its usage, which proves that it runs. */
+export async function probeGhidra(context: ToolContext): Promise<ToolStatus> {
+  const env = context.env ?? process.env;
+  try {
+    const ghidra = findGhidra(env);
+    // Its own settings directory keeps the user's Ghidra settings and logs untouched.
+    const workspace = Workspace.create(context.supervisor, "c64-re-tools-ghidra-status-");
+    const settings = workspace.root;
+    try {
+      const status = await run(
+        "Ghidra",
+        ghidra.analyzeHeadless,
+        [],
+        (output) => (/Headless Analyzer Usage/.test(output) ? ghidraVersion(ghidra) : undefined),
+        { ...context, env: { ...env, XDG_CONFIG_HOME: settings, XDG_CACHE_HOME: settings } },
+        (output) => (NO_USABLE_JDK.test(output) ? noJdk(ghidra).message : undefined),
+      );
+      return { ...status, path: ghidra.root };
+    } finally {
+      workspace.remove();
+    }
+  } catch (error) {
+    return missing("Ghidra", error);
+  }
+}
+
+/** The tools that skill scripts run themselves, on this machine. */
+export async function localToolStatus(context: ToolContext): Promise<ToolStatus[]> {
+  const probes: Probe[] = [
+    { name: "ACME", find: (env) => findTool(ACME, env), args: ["--version"], pattern: /^This is ACME, release / },
+    // dxa -V prints its version and exits with status 1. It names itself by argv[0], without the
+    // directory only where that uses "/": on Windows the line starts with the full path.
+    { name: "dxa", find: (env) => findTool(DXA, env), args: ["-V"], pattern: DXA_VERSION },
+  ];
+  return [...(await probeTools(probes, context)), await probeGhidra(context)];
+}

package/src/native/types.ts

@@ -0,0 +1,257 @@
+// Types of the native tools that skill scripts run directly (ACME, DXA,
+// Ghidra), and the checks of their complete results. The skill script runs
+// these tools itself: nothing here crosses the Host Runtime wire. Bytes
+// (a program, an image, a listing) go next to these values as attachments.
+
+import { prgRange, REFERENCE_KINDS, SYMBOL_NAME, type ReferenceKind } from "../c64.ts";
+import { isInteger, isObject, isOneOf, WireFailure } from "../protocol/messages.ts";
+import { type SourceTree } from "../protocol/tools.ts";
+
+export interface Diagnostic {
+  severity: "error" | "warning" | "note";
+  /** Relative to the source root; absent when the message has no source location. */
+  file?: string;
+  line?: number;
+  message: string;
+}
+
+export interface AcmeParams {
+  files: SourceTree;
+  /** Relative to the source root. */
+  entrySource: string;
+  /** Directories relative to the source root. */
+  includeDirs: string[];
+  defines: Record<string, number | boolean>;
+  setPc?: number;
+}
+
+/** A symbol from the assembly: an address inside the program, or a constant value. */
+export type AssembledSymbol = { name: string; kind: "address"; address: number; used: boolean } | { name: string; kind: "constant"; value: number; used: boolean };
+
+export interface AcmeResult {
+  assembled: boolean;
+  /** Present when assembled: where the program loads. The program bytes are the one attachment that assemble returns. */
+  loadRange?: { start: number; end: number; bytes: number };
+  symbols?: AssembledSymbol[];
+  diagnostics: Diagnostic[];
+}
+
+export const IMAGE_KINDS = ["prg", "flat64k"] as const;
+export type ImageKind = (typeof IMAGE_KINDS)[number];
+
+export const GHIDRA_LIMITS = { entryPoints: 1024, dataRanges: 1024, labels: 4096, decompile: 32, decompiledChars: 16_000, decompiledTotalChars: 128_000 } as const;
+
+/** The Ghidra analysis. Its image is given next to these fields: a PRG, or 64 KiB from $0000. */
+export interface GhidraParams {
+  imageKind: ImageKind;
+  /** Seeds from current knowledge. */
+  entryPoints: number[];
+  dataRanges: Array<{ start: number; end: number }>;
+  labels: Array<{ address: number; name: string }>;
+  /** Routine entries to decompile for immediate reasoning; the text is never stored. */
+  decompile: number[];
+}
+
+export interface GhidraInstruction {
+  address: number;
+  /** Lowercase hex, 1 to 3 bytes. */
+  bytes: string;
+  mnemonic: string;
+  /** Ghidra's text of the operands, empty for an implied operand. */
+  operands: string;
+  /** Where the flow goes besides the next instruction: a branch, jump or call target. */
+  flows: number[];
+}
+
+export interface GhidraResult {
+  coverage: Array<{ start: number; end: number }>;
+  functions: Array<{ entry: number; name: string; nameSource: "seed" | "generated" | "native" }>;
+  regions: Array<{ start: number; end: number; classification: "code" | "data" }>;
+  references: Array<{ from: number; to: number; type: ReferenceKind }>;
+  decompilations: Array<{ entry: number; text: string; truncated: boolean }>;
+  /** Requested routines without decompiled code, with the decompiler's reason. */
+  notDecompiled?: Array<{ entry: number; reason: string }>;
+  /** Every instruction in the coverage, as Ghidra decoded it (D44). */
+  instructions: GhidraInstruction[];
+  /** Which categories are complete inside the coverage; private to the importer. */
+  completeness: { functions: boolean; regions: boolean; references: boolean };
+  /** The version of the Ghidra that made the result, for example "Ghidra 12.1.3". */
+  toolVersion?: string;
+}
+
+/** Refuses a DXA or Ghidra request with more seeds or decompilations than the request bounds allow, by name. */
+export function checkRequestBounds(params: DxaParams & { decompile?: number[] }): void {
+  const bounds = [
+    ["entry points", params.entryPoints.length, GHIDRA_LIMITS.entryPoints],
+    ["data ranges", params.dataRanges.length, GHIDRA_LIMITS.dataRanges],
+    ["labels", params.labels.length, GHIDRA_LIMITS.labels],
+    ["routines to decompile", params.decompile?.length ?? 0, GHIDRA_LIMITS.decompile],
+  ] as const;
+  for (const [name, count, limit] of bounds) {
+    if (count > limit) throw new WireFailure("invalid-input", `The request has ${count} ${name}. Give at most ${limit} ${name}.`);
+  }
+}
+
+/** The DXA analysis: the same image and seeds as Ghidra, without decompilation. */
+export type DxaParams = Omit<GhidraParams, "decompile">;
+
+export interface DxaResult {
+  coverage: Array<{ start: number; end: number }>;
+  regions: Array<{ start: number; end: number; classification: "code" | "data" }>;
+  labels: Array<{ address: number; name: string }>;
+  /** The size of the listing, which is the one attachment that analyze returns (UTF-8 text). */
+  listingBytes: number;
+  /** Which categories are complete inside the coverage; private to the importer. */
+  completeness: { regions: boolean; labels: boolean };
+  /** The version line of the dxa that made the result; absent when dxa prints none. */
+  toolVersion?: string;
+}
+
+/** The longest tool version that a result can carry. */
+const MAX_TOOL_VERSION = 200;
+
+/** True for a tool version that a result can carry: absent, or one line of text without control characters, at most MAX_TOOL_VERSION characters. */
+const isToolVersion = (value: unknown) => value === undefined || (typeof value === "string" && value.length <= MAX_TOOL_VERSION && /^[^\p{Cc}]+$/u.test(value));
+
+/** Ends a result check with the Error that names its first problem. */
+function fail(message: string): never {
+  throw new Error(message);
+}
+
+/** Narrows to a list whose items are still unchecked. */
+const isList = (value: unknown): value is unknown[] => Array.isArray(value);
+
+/**
+ * Checks a complete Ghidra result: every field typed and in range, every
+ * function, region and reference source inside the coverage, regions sorted
+ * and without overlap. Throws an Error naming the first problem.
+ */
+export function checkGhidraResult(value: unknown): GhidraResult {
+  if (!isObject(value)) fail("not an object");
+  const { coverage, functions, regions, references, decompilations, notDecompiled, completeness, toolVersion, instructions } = value;
+  if (!isList(coverage) || coverage.length === 0) fail("no coverage");
+  const ranges: Array<{ start: number; end: number }> = [];
+  for (const range of coverage) {
+    if (!isObject(range) || !isInteger(range.start, 0, 0xffff) || !isInteger(range.end, range.start, 0xffff)) fail("bad coverage range");
+    ranges.push({ start: range.start, end: range.end });
+  }
+  const covered = (address: number) => ranges.some((range) => range.start <= address && address <= range.end);
+  if (!isList(functions)) fail("no functions");
+  for (const fn of functions) {
+    if (!isObject(fn) || !isInteger(fn.entry, 0, 0xffff) || typeof fn.name !== "string" || !SYMBOL_NAME.test(fn.name) || !isOneOf(["seed", "generated", "native"] as const, fn.nameSource)) fail("bad function");
+    if (!covered(fn.entry)) fail("a function lies outside the coverage");
+  }
+  if (!isList(regions)) fail("no regions");
+  let previousEnd = -1;
+  for (const region of regions) {
+    if (!isObject(region) || !isInteger(region.start, 0, 0xffff) || !isInteger(region.end, region.start, 0xffff) || (region.classification !== "code" && region.classification !== "data")) fail("bad region");
+    const start = region.start;
+    const end = region.end;
+    if (start <= previousEnd) fail("regions overlap or are out of order");
+    if (!ranges.some((range) => range.start <= start && end <= range.end)) fail("a region lies outside the coverage");
+    previousEnd = end;
+  }
+  if (!isList(instructions)) fail("no instructions");
+  for (const instruction of instructions) {
+    if (
+      !isObject(instruction) ||
+      !isInteger(instruction.address, 0, 0xffff) ||
+      typeof instruction.bytes !== "string" ||
+      !/^(?:[0-9a-f]{2}){1,3}$/.test(instruction.bytes) ||
+      typeof instruction.mnemonic !== "string" ||
+      typeof instruction.operands !== "string" ||
+      !isList(instruction.flows) ||
+      !instruction.flows.every((flow) => isInteger(flow, 0, 0xffff))
+    ) {
+      fail("bad instruction");
+    }
+    if (!covered(instruction.address)) fail("an instruction lies outside the coverage");
+  }
+  if (!isList(references)) fail("no references");
+  for (const reference of references) {
+    if (!isObject(reference) || !isInteger(reference.from, 0, 0xffff) || !isInteger(reference.to, 0, 0xffff) || !isOneOf(REFERENCE_KINDS, reference.type)) fail("bad reference");
+    if (!covered(reference.from)) fail("a reference source lies outside the coverage");
+  }
+  if (!isList(decompilations) || decompilations.length > GHIDRA_LIMITS.decompile) fail("bad decompilations");
+  let total = 0;
+  for (const item of decompilations) {
+    if (!isObject(item) || !isInteger(item.entry, 0, 0xffff) || typeof item.text !== "string" || typeof item.truncated !== "boolean") fail("bad decompilation");
+    if (item.text.length > GHIDRA_LIMITS.decompiledChars) fail("a decompilation is too long");
+    total += item.text.length;
+  }
+  if (total > GHIDRA_LIMITS.decompiledTotalChars) fail("the decompilations are too long together");
+  if (notDecompiled !== undefined) {
+    if (!isList(notDecompiled) || notDecompiled.length > GHIDRA_LIMITS.decompile) fail("bad list of routines that were not decompiled");
+    for (const item of notDecompiled) {
+      if (!isObject(item) || !isInteger(item.entry, 0, 0xffff) || typeof item.reason !== "string" || item.reason.length > 1000) fail("bad routine that was not decompiled");
+    }
+  }
+  if (!isObject(completeness) || typeof completeness.functions !== "boolean" || typeof completeness.regions !== "boolean" || typeof completeness.references !== "boolean") fail("no completeness");
+  if (!isToolVersion(toolVersion)) fail("bad tool version");
+  // Every field is checked: the object is the result as it is.
+  return value as unknown as GhidraResult;
+}
+
+/**
+ * Checks a complete DXA result: typed fields, regions in order, without
+ * overlap and covering the coverage exactly, labels inside the coverage, and
+ * the listing as the one attachment. Throws an Error naming the first problem.
+ */
+export function checkDxaResult(value: unknown, attachments: readonly Uint8Array[]): DxaResult {
+  if (!isObject(value)) fail("not an object");
+  const { coverage, regions, labels, listingBytes, completeness, toolVersion } = value;
+  if (!isList(coverage) || coverage.length !== 1) fail("the coverage is not one range");
+  const range = coverage[0];
+  if (!isObject(range) || !isInteger(range.start, 0, 0xffff) || !isInteger(range.end, range.start, 0xffff)) fail("bad coverage range");
+  const start = range.start;
+  const end = range.end;
+  if (!isList(regions)) fail("no regions");
+  let next = start;
+  for (const region of regions) {
+    if (!isObject(region) || !isInteger(region.start, 0, 0xffff) || !isInteger(region.end, region.start, 0xffff) || (region.classification !== "code" && region.classification !== "data")) fail("bad region");
+    if (region.start !== next) fail("the regions leave a gap or overlap");
+    next = region.end + 1;
+  }
+  if (next !== end + 1) fail("the regions do not reach the end of the coverage");
+  if (!isList(labels)) fail("no labels");
+  for (const label of labels) {
+    if (!isObject(label) || !isInteger(label.address, start, end) || typeof label.name !== "string" || !SYMBOL_NAME.test(label.name)) fail("bad label");
+  }
+  const [listing] = attachments;
+  if (attachments.length !== 1 || listing === undefined || listingBytes !== listing.length) fail("the listing is missing");
+  if (!isObject(completeness) || typeof completeness.regions !== "boolean" || typeof completeness.labels !== "boolean") fail("no completeness");
+  if (!isToolVersion(toolVersion)) fail("bad tool version");
+  // Every field is checked: the object is the result as it is.
+  return value as unknown as DxaResult;
+}
+
+/**
+ * The address range of an analyzer image: a PRG at its load address, or 64 KiB
+ * of memory from $0000. Throws WireFailure(invalid-input) for anything else.
+ */
+export function imageRange(kind: ImageKind, image: Uint8Array): { start: number; end: number; body: Uint8Array } {
+  if (kind === "flat64k") {
+    if (image.length !== 0x10000) throw new WireFailure("invalid-input", "A flat 64 KiB image has exactly 65536 bytes.");
+    return { start: 0, end: 0xffff, body: image };
+  }
+  try {
+    return prgRange(image);
+  } catch (error) {
+    if (!(error instanceof RangeError)) throw error;
+    throw new WireFailure("invalid-input", error.message);
+  }
+}
+
+/**
+ * The seeds that an analysis of [start, end] can use: the entry points inside
+ * it, and the data ranges cut to it. A data range that holds an entry point
+ * is left out, so that the entry point is disassembled.
+ */
+export function seedsInside(params: Pick<DxaParams, "entryPoints" | "dataRanges">, start: number, end: number): Pick<DxaParams, "entryPoints" | "dataRanges"> {
+  const entryPoints = params.entryPoints.filter((address) => address >= start && address <= end);
+  const dataRanges = params.dataRanges
+    .filter((range) => range.end >= start && range.start <= end)
+    .map((range) => ({ start: Math.max(range.start, start), end: Math.min(range.end, end) }))
+    .filter((range) => !entryPoints.some((entry) => entry >= range.start && entry <= range.end));
+  return { entryPoints, dataRanges };
+}

package/src/native/watchdog.ts

@@ -0,0 +1,78 @@
+// The watchdog of a supervisor's process (the Host Runtime or a skill
+// script), started by ProcessSupervisor.startWatchdog.
+// Arguments: <owner pid> <registry file>. Standard input: a pipe whose
+// writing end only the owner holds.
+//
+// It runs in its own process group, so a signal to the owner's group does
+// not reach it. When the owner dies without its exit guard (SIGKILL, a
+// crash), it stops every process group in the registry and removes every
+// path. When the owner exits normally, the exit guard removes the registry
+// and the watchdog ends.
+//
+// The end of the pipe tells it at once that the owner is gone: the system
+// closes the owner's end when the owner ends, however it ends, and a pid that
+// another process now uses cannot keep the pipe open. A poll of the owner's
+// pid is the fallback for a pipe that does not end.
+
+import { existsSync, readFileSync, rmSync } from "node:fs";
+import { dirname } from "node:path";
+
+import { isAlive, killTree, signalGroup, type Registry } from "./processes.ts";
+
+const POLL_MS = 500;
+const GRACE_MS = 2_000;
+
+const [ownerPidText, registryArgument] = process.argv.slice(2);
+const ownerPid = Number(ownerPidText);
+if (!Number.isInteger(ownerPid) || ownerPid <= 0 || registryArgument === undefined) process.exit(2);
+const registryFile: string = registryArgument;
+
+function signal(group: number, name: NodeJS.Signals): void {
+  try {
+    if (process.platform !== "win32") signalGroup(group, name);
+    else if (name === "SIGKILL") killTree(group);
+  } catch {
+    // The group is gone already.
+  }
+}
+
+/** Removes a path. A path that cannot be removed does not stop the removal of the others; nothing reads the watchdog's output. */
+function remove(path: string): void {
+  try {
+    rmSync(path, { recursive: true, force: true });
+  } catch {
+    // The path stays.
+  }
+}
+
+let cleaning = false;
+
+function cleanUp(): void {
+  if (cleaning) return;
+  cleaning = true;
+  clearInterval(timer);
+  let registry: Registry;
+  try {
+    registry = JSON.parse(readFileSync(registryFile, "utf8")) as Registry;
+  } catch {
+    process.exit(0);
+  }
+  for (const group of registry.groups) signal(group, "SIGTERM");
+  setTimeout(() => {
+    for (const group of registry.groups) signal(group, "SIGKILL");
+    for (const path of registry.paths) remove(path);
+    remove(dirname(registryFile));
+    process.exit(0);
+  }, GRACE_MS);
+}
+
+// The owner writes nothing: only the end of the pipe (or an error, which also ends it) counts.
+process.stdin.on("data", () => {});
+process.stdin.once("end", cleanUp);
+process.stdin.once("error", cleanUp);
+
+const timer = setInterval(() => {
+  // A normal exit removed the registry: nothing is left to do.
+  if (!existsSync(registryFile)) process.exit(0);
+  if (!isAlive(ownerPid)) cleanUp();
+}, POLL_MS);

package/src/package.ts

@@ -0,0 +1,21 @@
+// The name, version and npm channel of this copy of c64-re-tools, for the npx
+// commands that refusals and help texts give. The package.json above src/ is
+// the package's own one; in an installed skill it is the skill's one, which
+// has the same name and version (distribution/plugin.ts).
+
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+
+/**
+ * The npm dist-tag that a version is published under: a prerelease
+ * (2.0.0-rc.1) goes to "next", a release to "latest".
+ */
+export function channelOf(version: string): "latest" | "next" {
+  return version.includes("-") ? "next" : "latest";
+}
+
+/** The name, version and dist-tag of this npm package, from its package.json. */
+export const PACKAGE = (() => {
+  const metadata = JSON.parse(readFileSync(join(import.meta.dirname, "..", "package.json"), "utf8")) as { name: string; version: string };
+  return { name: metadata.name, version: metadata.version, tag: channelOf(metadata.version) };
+})();