@henols/c64-re-tools 2.0.0-rc.2 → 2.0.0-rc.20

package/skills/c64-docs/scripts/lib/source-map.ts

@@ -0,0 +1,81 @@
+// The source map that c64-source writes next to the source (D80): the line of
+// each name and of each address. The documentation links to these lines.
+
+import { posix } from "node:path";
+
+import { parseC64Address } from "#src/c64.ts";
+import { WireFailure } from "#src/host-client/tools.ts";
+import { readProjectFile } from "#src/host-client/transfer.ts";
+import type { Space } from "#src/knowledge/schema.ts";
+
+export const SOURCE_MAP_FILE = "source-map.json";
+
+export interface SourceLine {
+  file: string;
+  line: number;
+}
+
+export interface SourceMap {
+  /** The source directory, relative to the project folder. */
+  directory: string;
+  space: Space;
+  revision: string;
+  files: string[];
+  labels: Array<SourceLine & { address: number; name: string }>;
+  /** The label line of a name. */
+  labelOf: (name: string) => (SourceLine & { address: number }) | undefined;
+  /** The line that writes an address: an instruction, a macro use or a data line. */
+  lineOf: (address: number) => SourceLine | undefined;
+  /** The project path of a source file. */
+  pathOf: (file: string) => string;
+}
+
+/** Reads <directory>/source-map.json. A missing map is a refusal with the remedy. */
+export function readSourceMap(directory: string): SourceMap {
+  const path = posix.join(directory, SOURCE_MAP_FILE);
+  let text: string;
+  try {
+    text = readProjectFile(path).bytes.toString("utf8");
+  } catch (error) {
+    if (error instanceof WireFailure && error.code === "not-found") {
+      throw new WireFailure("not-found", `There is no ${path}. Generate the source with c64-source generate.ts --out ${directory}: it writes the source map.`);
+    }
+    throw error;
+  }
+  const fail = (what: string): never => {
+    throw new WireFailure("invalid-input", `${path} is not a source map of c64-source: ${what}. Generate the source again.`);
+  };
+  let raw: Record<string, unknown>;
+  try {
+    raw = JSON.parse(text) as Record<string, unknown>;
+  } catch (error) {
+    return fail((error as Error).message);
+  }
+  if (!Array.isArray(raw.labels) || !Array.isArray(raw.items) || !Array.isArray(raw.files)) fail("it has no labels, items or files");
+  const labels = (raw.labels as Array<Record<string, unknown>>).map((label) => ({ address: parseC64Address(String(label.address)), name: String(label.name), file: String(label.file), line: Number(label.line) }));
+  const items = (raw.items as Array<Record<string, unknown>>)
+    .map((item) => ({ start: parseC64Address(String(item.start)), end: parseC64Address(String(item.end)), file: String(item.file), line: Number(item.line) }))
+    .sort((a, b) => a.start - b.start);
+  const byName = new Map(labels.map((label) => [label.name, label]));
+  return {
+    directory,
+    space: String(raw.space ?? "c64") as Space,
+    revision: String(raw.revision),
+    files: (raw.files as unknown[]).map(String),
+    labels,
+    labelOf: (name) => byName.get(name),
+    lineOf: (address) => {
+      let low = 0;
+      let high = items.length - 1;
+      while (low <= high) {
+        const middle = (low + high) >> 1;
+        const item = items[middle]!;
+        if (address < item.start) high = middle - 1;
+        else if (address > item.end) low = middle + 1;
+        else return { file: item.file, line: item.line };
+      }
+      return undefined;
+    },
+    pathOf: (file) => posix.join(directory, file),
+  };
+}

package/skills/c64-docs/scripts/link.ts

@@ -0,0 +1,193 @@
+// c64-docs link.ts (D80): links the name mentions of the documentation to the
+// source and the reference, and checks every link and image of the pages.
+// With --check it writes nothing and fails when a page is not up to date.
+
+import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
+import { join, posix } from "node:path";
+import { parseArgs } from "node:util";
+
+import { WireFailure } from "#src/host-client/tools.ts";
+import { writeProjectFile } from "#src/host-client/transfer.ts";
+import { KnowledgeError, openForRead } from "#src/knowledge/database.ts";
+import { currentRevision, listSymbols } from "#src/knowledge/read.ts";
+import { resolveProjectFile, resolveProjectPath } from "#src/project.ts";
+import { runScript, UsageError } from "#src/script.ts";
+import { checkLinks, linkPage, type Files, type Problem } from "./lib/links.ts";
+import { GENERATED_MARK } from "./lib/reference.ts";
+import { readSourceMap } from "./lib/source-map.ts";
+
+const USAGE = `link.ts <docs> --source <dir> [--reference <dir>] [--check]
+
+  <docs>             the documentation directory, relative to the project folder;
+                     link.ts reads every .md file below it
+  --source <dir>     the source directory of c64-source, with its source-map.json
+  --reference <dir>  the reference pages of generate.ts; the default is
+                     <docs>/reference when it exists
+  --check            write nothing; fail when a link is not up to date or a page
+                     has a problem
+
+A name in backticks gets a link to its line in the source. An address in
+backticks after the name gets a link to the entry of the name in the reference.
+The result is one JSON object.`;
+
+const MAX_LISTED = 50;
+/** A name outside the image covers the addresses up to the next name, at most this many. */
+const MAX_BODY = 256;
+
+function markdownFiles(directory: string): string[] {
+  const root = resolveProjectPath(directory);
+  if (!existsSync(root) || !statSync(root).isDirectory()) throw new WireFailure("not-found", `There is no directory ${directory} in the project folder.`);
+  const found: string[] = [];
+  const walk = (relative: string) => {
+    for (const entry of readdirSync(join(root, relative), { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
+      const path = posix.join(relative, entry.name);
+      if (entry.isDirectory()) walk(path);
+      else if (entry.name.endsWith(".md")) found.push(posix.join(directory, path));
+    }
+  };
+  walk("");
+  return found;
+}
+
+const files: Files = {
+  inside: (path) => {
+    try {
+      resolveProjectPath(path);
+      return true;
+    } catch {
+      return false;
+    }
+  },
+  exists: (path) => {
+    try {
+      resolveProjectFile(path);
+      return true;
+    } catch {
+      return false;
+    }
+  },
+  text: (path) => {
+    try {
+      const real = resolveProjectFile(path);
+      return statSync(real).isFile() ? readFileSync(real, "utf8") : undefined;
+    } catch {
+      return undefined;
+    }
+  },
+};
+
+function run(argv: string[]): { output: Record<string, unknown>; failed: boolean } {
+  const { values, positionals } = parseArgs({ args: argv, strict: true, allowPositionals: true, options: { source: { type: "string" }, reference: { type: "string" }, check: { type: "boolean" }, help: { type: "boolean" } } });
+  if (values.help) throw new UsageError("");
+  if (positionals.length !== 1) throw new UsageError("give one documentation directory");
+  if (values.source === undefined) throw new UsageError("give --source <dir>: the source directory of c64-source");
+  const docs = posix.normalize(positionals[0]!);
+  for (const [option, path] of [["<docs>", docs], ["--source", values.source], ["--reference", values.reference]] as const) {
+    if (path === undefined) continue;
+    if (!files.inside(path)) throw new UsageError(`${option} must be a path relative to the project folder that stays inside it, not ${path}`);
+  }
+  const map = readSourceMap(values.source);
+  const reference = values.reference === undefined ? (existsSync(resolveProjectPath(posix.join(docs, "reference"))) ? posix.join(docs, "reference") : undefined) : posix.normalize(values.reference);
+  const problems: Problem[] = [];
+  const db = openForRead();
+  let symbols;
+  let revision;
+  try {
+    revision = currentRevision(db);
+    symbols = listSymbols(db, undefined, undefined, undefined, map.space);
+  } finally {
+    db?.close();
+  }
+  if (map.revision !== revision) {
+    problems.push({ page: map.pathOf("source-map.json"), line: 1, kind: "stale-reference", message: `The source is from knowledge revision ${map.revision}, and the knowledge is at ${revision}. Generate the source and the reference again, then run link.ts.` });
+  }
+  const addresses = new Map<string, number>();
+  for (const label of map.labels) addresses.set(label.name, label.address);
+  for (const symbol of symbols) addresses.set(symbol.name, symbol.address);
+  // A routine runs to the next routine; any other name to the next name of the knowledge, in its file, and outside the image at most MAX_BODY bytes.
+  const routines = symbols.filter((symbol) => symbol.kind === "routine").map((symbol) => symbol.address).sort((a, b) => a - b);
+  const named = [...new Set(symbols.map((symbol) => symbol.address))].sort((a, b) => a - b);
+  const kinds = new Map(symbols.map((symbol) => [symbol.name, symbol.kind]));
+  const bodyEnd = (name: string) => {
+    const address = addresses.get(name);
+    if (address === undefined) return undefined;
+    const next = ((kinds.get(name) === "routine" ? routines : named).find((other) => other > address) ?? 0x10000) - 1;
+    const file = map.lineOf(address)?.file;
+    if (file === undefined) return Math.min(next, address + MAX_BODY - 1);
+    let end = address;
+    while (end < next && map.lineOf(end + 1)?.file === file) end++;
+    return end;
+  };
+  const nameAt = new Map(symbols.map((symbol) => [symbol.address, symbol.name]));
+  /** The routine, else the name, whose body holds an address. */
+  const placeOf = (address: number) => {
+    for (const list of [routines, named]) {
+      let start: number | undefined;
+      for (const other of list) if (other <= address) start = other;
+      const name = start === undefined ? undefined : nameAt.get(start);
+      if (name !== undefined && (bodyEnd(name) ?? start!) >= address) return name;
+    }
+    return undefined;
+  };
+
+  // The entries of the reference: each <a id> that is a name.
+  const entries = new Map<string, string>();
+  const referencePages = reference === undefined ? [] : markdownFiles(reference);
+  for (const page of referencePages) {
+    const text = files.text(page)!;
+    const stamp = /^<!-- Generated by c64-docs from knowledge revision (\S+) /.exec(text);
+    if (text.startsWith(GENERATED_MARK) && stamp !== null && stamp[1] !== map.revision) {
+      problems.push({ page, line: 1, kind: "stale-reference", message: `${page} is from knowledge revision ${stamp[1]}, and the source from ${map.revision}. Run generate.ts again.` });
+    }
+    for (const match of text.matchAll(/<a id="([^"]+)"><\/a>/g)) if (addresses.has(match[1]!) && !entries.has(match[1]!)) entries.set(match[1]!, `${page}#${match[1]}`);
+  }
+  const inside = (path: string, directory: string | undefined) => directory !== undefined && (path === directory || path.startsWith(`${directory}/`));
+  const context = {
+    addressOf: (name: string) => addresses.get(name),
+    codeOf: (name: string) => {
+      const label = map.labelOf(name);
+      return label === undefined ? undefined : `${map.pathOf(label.file)}#L${label.line}`;
+    },
+    entryOf: (name: string) => entries.get(name),
+    bodyEnd,
+    placeOf,
+    lineOf: (address: number) => {
+      const line = map.lineOf(address);
+      return line === undefined ? undefined : `${map.pathOf(line.file)}#L${line.line}`;
+    },
+    managed: (path: string) => inside(path, posix.normalize(map.directory)) || inside(path, reference),
+  };
+
+  const pages = markdownFiles(docs);
+  const changedPages: string[] = [];
+  let links = 0;
+  for (const page of pages) {
+    const text = files.text(page)!;
+    // A generated page is right by its making: it is only checked.
+    const generated = text.startsWith(GENERATED_MARK);
+    const linked = generated ? { text, changed: 0, problems: [] } : linkPage(text, { page, ...context });
+    problems.push(...linked.problems, ...checkLinks(page, linked.text, files));
+    if (linked.changed === 0) continue;
+    links += linked.changed;
+    changedPages.push(page);
+    if (!values.check) writeProjectFile(page, Buffer.from(linked.text, "utf8"));
+  }
+  const byKind: Record<string, number> = {};
+  for (const problem of problems) byKind[problem.kind] = (byKind[problem.kind] ?? 0) + 1;
+  const output: Record<string, unknown> = {
+    pages: pages.length,
+    ...(reference === undefined ? { reference: "none: run generate.ts for links to the entries" } : { reference }),
+    ...(values.check ? { notUpToDate: { links, pages: changedPages } } : { linked: { links, pages: changedPages } }),
+    problems: { count: problems.length, byKind, listed: problems.slice(0, MAX_LISTED).map((problem) => ({ at: `${problem.page}:${problem.line}`, kind: problem.kind, message: problem.message })) },
+  };
+  return { output, failed: problems.length > 0 || (values.check === true && links > 0) };
+}
+
+await runScript(
+  (argv) => {
+    const { output, failed } = run(argv);
+    if (failed) process.exitCode = 1;
+    return output;
+  },
+  { usage: USAGE, refusals: [KnowledgeError] },
+);

package/skills/c64-emulator/SKILL.md

@@ -25,18 +25,21 @@ Use only the MCP tools. Each MCP session has its own C64. Nothing else can chang
 
 | Need | Tool |
 |---|---|
-| Run state, video standard, warp mode, window | `c64_status` |
+| Run state, video standard, warp mode, window, version of the MCP server | `c64_status` |
 | Reset, power cycle | `c64_reset` |
 | Fast emulation | `c64_warp` |
 | Show the C64 to the user in a window, hide it again | `c64_window` |
 | Stop, start, step, step over, step out, exact frames | `c64_execution` |
 | Run until an address, a memory value or a raster position | `c64_run_until` |
+| Run to a place and read the machine there in one call | `c64_run_until` with `observe` and `history` |
+| Each write, read or routine entry over many frames, with the instruction and the raster line | `c64_trace` |
 | Stop on an instruction address | `c64_breakpoint` |
 | Stop on a memory read or write | `c64_watchpoint` |
 | Memory | `c64_memory_read`, `c64_memory_write`, `c64_memory_search`, `c64_memory_compare` |
 | CPU registers | `c64_registers` |
 | Code as it is in memory now | `c64_disassemble` |
 | Executed instructions, call chain | `c64_cpu_history`, `c64_backtrace` |
+| Which instruction reads or writes an address | `c64_trace` with a `read` or `write` point |
 | Where the CPU spends cycles, which memory it uses | `c64_profile`, `c64_memmap` |
 | Cycle count between two points | `c64_timing` |
 | Video chip, sprites, CIAs, sound chip | `c64_vicii`, `c64_sprite`, `c64_cia`, `c64_sid` |
@@ -44,27 +47,46 @@ Use only the MCP tools. Each MCP session has its own C64. Nothing else can chang
 | Several of these from one moment | `c64_observe` |
 | Keys, joystick | `c64_keyboard`, `c64_joystick` |
 | Program and disk media | `c64_autostart`, `c64_program_load`, `c64_disk_attach` |
-| Machine state to go back to | `c64_snapshot` |
+| Machine state to go back to, also in a later session | `c64_snapshot` (with `out` and `file`) |
 
 Each memory and register tool has a `space`. Use `c64` for the computer. Use `drive8` for the CPU of the 1541 disk drive. The memory `view` is `cpu` or `ram`. With `cpu`, you see what the CPU sees now, with ROM and I/O where the memory configuration puts them. With `ram`, you see the RAM under them.
 
+For a large result, give `out` with a path in the project. `c64_memory_read`, `c64_disassemble`, `c64_cpu_history` and `c64_screen` then write the result to that file. The answer gives only the file, the number of bytes and a SHA-256. Use it for a memory dump of more than a few hundred bytes, a long listing, or a screen that the user wants to keep. `c64_memmap` action `read` and `c64_snapshot` action `save` also take `out`. Write a file that is only for your work in `.c64-re-tools/local/` of the project. Git ignores that folder.
+
+`c64_memmap` keeps RAM, ROM and I/O apart. Use action `summary` to see which code ran. Use action `read` with `out` to keep a coverage run as a file, for example `.c64-re-tools/local/attract.bin`. Then add it to the knowledge with `coverage` of the c64-knowledge skill. Clear the map before each run that you want to keep apart.
+
+- Only the first byte of an instruction gets execute. Thus `read` without `out` gives a new range at each instruction of code. For code, use `summary`, or `read` with `out`.
+- A read on a byte right after executed code, or at a return address, can be a dummy read of the 6502. The CPU reads such a byte but does not use it.
+
+`c64_profile` gives the routines that used the most cycles since the session started, the last `clear` or the last `c64_window`. Use action `clear` before the part of the program that you want to measure. The profiler gives the cycles to the routine that a JSR called last. Code that the CPU enters with JMP or a branch counts for that routine. After `c64_window` or a snapshot restore, the profiler of VICE keeps the routines that ran before the restore. Then `c64_profile` can give the cycles of the program to such a routine, for example `$fd50`. Do not trust the routine addresses of `c64_profile` after a restore.
+
+### Project folder
+
+Paths (`path`, `file` and `out`) are relative to the project folder. A project folder holds `.c64-re-tools/`. The tools use the nearest project folder at or above the folder where the agent started. If that folder is not in a project and has projects below it, a tool with a path refuses and lists them. Then give `project` with the project folder, for example `"project": "game"`. Give the same folder as the `--project` of the skill scripts. A tool without a path does not need `project`.
+
 ## Workflow
 
 1. Read the project knowledge for the addresses in the question. Use the c64-knowledge skill.
 2. Get the machine state with `c64_status`.
 3. Load the subject if it is not loaded. Use `c64_autostart` to start it as a user does. Use `c64_program_load` to put a PRG in memory with no start.
 4. Set the stop condition. Use a breakpoint, a watchpoint or a `c64_run_until` target. Add a typed condition when the stop must occur only in one state. The `list` result shows the condition of each breakpoint and each watchpoint.
-5. Run, step or give input. Prefer `c64_run_until` and `advance-frames` to waits. They stop at the same emulated time on every run.
+5. Run, step or give input. Prefer `c64_run_until` and `advance-frames` to waits. They stop at the same emulated time on every run. To wait for a breakpoint or a watchpoint, use `advance-frames` with a large count. `stopReason` and `pointId` in the result tell what stopped the CPU. `c64_status` also gives them.
 6. Read only the state that answers the question. To read several things from one moment, use `c64_observe`.
 7. Interpret the evidence. Keep the observed facts apart from your conclusions.
 
 Write operations need a stopped CPU. Stop the CPU with `c64_execution` action `pause` first.
 
-`c64_reset` with `run` false stops the CPU at the first instruction of the reset routine. The result gives that address in `pc`.
+`c64_reset` with `run` false stops the CPU at the first instruction of the reset routine. The result gives that address in `pc`. A reset and `c64_autostart` set `c64_timing` to zero.
+
+A cartridge from `c64_autostart` stays in the C64 through every reset, as on a real C64. `c64_status` gives `cartridge: true` while the C64 has it, and `cartridge: false` when it does not. `c64_autostart` of a file that is not a cartridge removes the cartridge first. `c64_autostart` also sets both joysticks to the center first. `joysticksCentred` gives the ports that it changed.
 
 Step, next and until-return work only in space `c64`. The emulator cannot stop the disk drive CPU after one instruction. To find when the drive executes an address, use `c64_run_until` with an address target in space `drive8`, or a breakpoint in space `drive8`. The drive can execute a few more instructions before it stops. The computer completes its current instruction. The `pc` in the result is the drive address where it stopped.
 
-`c64_keyboard` puts keys in the keyboard queue of the KERNAL. A program that reads the keyboard hardware directly does not see them. Many games do this. For such a program, use `c64_joystick`, or open the window and ask the user to press the key.
+`c64_keyboard` puts keys in the keyboard queue of the KERNAL. A program that reads the keyboard hardware directly does not see them. Many games do this. The emulator has no command that presses one key of the keyboard matrix. For such a program:
+
+- VICE 3.10 and older have a known flaw: bits 5-7 of `$DC00` and `$DC01` are always 0. Then `emulatorFlaw` in the result of `c64_keyboard` and `c64_joystick` gives the flaw. With this flaw, a wait for any key that reads all keyboard columns at one time ends without a key. A wait for one specific key, or a compare of the full byte of a joystick port, never ends. Tell the user about the flaw. The next VICE release and the VICE nightly builds do not have it. Joystick port 1 uses the same lines as the keyboard.
+- For one specific key, for example F1 or RUN/STOP, open the window and ask the user to press the key.
+- Do not change the program to make it accept a key.
 
 Before a risky experiment, save the state with `c64_snapshot`. Restore it to try again from the same point.
 
@@ -106,11 +128,10 @@ If you cannot get the machine into the necessary state, say which state you coul
 
 - `machine-state-lost` and the message says that the VICE window closed: the window closed before `c64_window` action `close`. The emulator has no window again. The machine state is the same as at the time when the window opened. The breakpoints and the watchpoints stay. Do not use results from the window time. Read the machine state again, then continue.
 - `machine-unavailable` and the message says that the VICE window closed: the session starts the emulator again. Wait a few seconds. Then do the operation again.
-- `machine-state-lost` with a different message: the emulator stopped and its state is gone. Do not continue with results from before. Tell the user to restart the MCP server.
+- `machine-state-lost` with a different message: the connection to the host runtime was lost, and the machine state is gone. Do not continue with results from before. The next call starts a new machine when the host runtime runs. Load the program again, and set the breakpoints and the watchpoints again. If the next call gives `machine-unavailable` with a message that the host runtime does not run, obey the rule for that message.
 - `machine-unavailable` and the message names a pause or a dialog in the emulator window: this occurs only while the window is open. Do not send more operations. Tell the user to resume the emulator (Pause, Alt+P) or to close the dialog in the VICE window. Then do the operation again. The machine state does not change.
-- `machine-unavailable` with a different message: the host runtime does not run, or it does not answer. Tell the user to start the host runtime with `npx -y --package=@henols/c64-re-tools@latest c64-re-tools-host`.
 - `machine-running`: stop the CPU first, then try again.
-- `installation-incomplete`: tell the user the message. Do not try to repair the installation.
+- `installation-incomplete`, or `machine-unavailable` with a message that the host runtime does not run: stop the task. Give the message to the user and wait for the user. The message gives the remedy. Do not make a replacement for the tool or the host runtime, for example your own disk parser, disassembler or emulator script. Do not try to repair the installation.
 - `c64_run_until` with `stopReason` `timeout`: the target did not occur in the frame limit. This is evidence too. Report it.
 - `c64_run_until` with `stopReason` `jam`: the CPU executed a JAM opcode and stopped. Report the address.