@henols/c64-re-tools 2.0.0-rc.8 → 2.0.0

package/skills/c64-docs/scripts/lib/source-map.ts

@@ -0,0 +1,81 @@
+// The source map that c64-source writes next to the source: the line of
+// each name and of each address. The documentation links to these lines.
+
+import { posix } from "node:path";
+
+import { parseC64Address } from "#src/c64.ts";
+import { WireFailure } from "#src/host-client/tools.ts";
+import { readProjectFile } from "#src/host-client/transfer.ts";
+import type { Space } from "#src/knowledge/schema.ts";
+
+export const SOURCE_MAP_FILE = "source-map.json";
+
+export interface SourceLine {
+  file: string;
+  line: number;
+}
+
+export interface SourceMap {
+  /** The source directory, relative to the project folder. */
+  directory: string;
+  space: Space;
+  revision: string;
+  files: string[];
+  labels: Array<SourceLine & { address: number; name: string }>;
+  /** The label line of a name. */
+  labelOf: (name: string) => (SourceLine & { address: number }) | undefined;
+  /** The line that writes an address: an instruction, a macro use or a data line. */
+  lineOf: (address: number) => SourceLine | undefined;
+  /** The project path of a source file. */
+  pathOf: (file: string) => string;
+}
+
+/** Reads <directory>/source-map.json. A missing map is a refusal with the remedy. */
+export function readSourceMap(directory: string): SourceMap {
+  const path = posix.join(directory, SOURCE_MAP_FILE);
+  let text: string;
+  try {
+    text = readProjectFile(path).bytes.toString("utf8");
+  } catch (error) {
+    if (error instanceof WireFailure && error.code === "not-found") {
+      throw new WireFailure("not-found", `There is no ${path}. Generate the source with c64-source generate.ts --out ${directory}: it writes the source map.`);
+    }
+    throw error;
+  }
+  const fail = (what: string): never => {
+    throw new WireFailure("invalid-input", `${path} is not a source map of c64-source: ${what}. Generate the source again.`);
+  };
+  let raw: Record<string, unknown>;
+  try {
+    raw = JSON.parse(text) as Record<string, unknown>;
+  } catch (error) {
+    return fail((error as Error).message);
+  }
+  if (!Array.isArray(raw.labels) || !Array.isArray(raw.items) || !Array.isArray(raw.files)) fail("it has no labels, items or files");
+  const labels = (raw.labels as Array<Record<string, unknown>>).map((label) => ({ address: parseC64Address(String(label.address)), name: String(label.name), file: String(label.file), line: Number(label.line) }));
+  const items = (raw.items as Array<Record<string, unknown>>)
+    .map((item) => ({ start: parseC64Address(String(item.start)), end: parseC64Address(String(item.end)), file: String(item.file), line: Number(item.line) }))
+    .sort((a, b) => a.start - b.start);
+  const byName = new Map(labels.map((label) => [label.name, label]));
+  return {
+    directory,
+    space: String(raw.space ?? "c64") as Space,
+    revision: String(raw.revision),
+    files: (raw.files as unknown[]).map(String),
+    labels,
+    labelOf: (name) => byName.get(name),
+    lineOf: (address) => {
+      let low = 0;
+      let high = items.length - 1;
+      while (low <= high) {
+        const middle = (low + high) >> 1;
+        const item = items[middle]!;
+        if (address < item.start) high = middle - 1;
+        else if (address > item.end) low = middle + 1;
+        else return { file: item.file, line: item.line };
+      }
+      return undefined;
+    },
+    pathOf: (file) => posix.join(directory, file),
+  };
+}

package/skills/c64-docs/scripts/link.ts

@@ -0,0 +1,239 @@
+// c64-docs link.ts: links the name mentions of the documentation to the
+// source and the reference, and checks every link and image of the pages.
+// With --check it writes nothing and fails when a page is not up to date.
+
+import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
+import { join, posix } from "node:path";
+import { parseArgs } from "node:util";
+
+import { WireFailure } from "#src/host-client/tools.ts";
+import { writeProjectFile } from "#src/host-client/transfer.ts";
+import { KnowledgeError, openForRead } from "#src/knowledge/database.ts";
+import { currentRevision, listSymbols } from "#src/knowledge/read.ts";
+import { resolveProjectFile, resolveProjectPath } from "#src/project.ts";
+import { runScript, UsageError } from "#src/script.ts";
+import { checkLinks, linkPage, type Files, type Problem } from "./lib/links.ts";
+import { GENERATED_MARK } from "./lib/reference.ts";
+import { readSourceMap } from "./lib/source-map.ts";
+
+const USAGE = `link.ts <pages>... --source <dir> [--reference <dir>] [--check]
+
+  <pages>...         .md files and directories of the documentation, relative to
+                     the project folder, for example README.md docs; link.ts reads
+                     each file and every .md file below each directory
+  --source <dir>     the source directory of c64-source, with its source-map.json
+  --reference <dir>  the reference pages of generate.ts; the default is the
+                     reference/ folder of the first directory that has one
+  --check            write nothing; fail when a link is not up to date or a page
+                     has a problem
+
+A name in backticks gets a link to its line in the source. An address in
+backticks after the name gets a link to the entry of the name in the reference.
+The result is one JSON object.`;
+
+const MAX_LISTED = 50;
+/** A name outside the image covers the addresses up to the next name, at most this many. */
+const MAX_BODY = 256;
+
+const isDirectory = (path: string) => {
+  const real = resolveProjectPath(path);
+  return existsSync(real) && statSync(real).isDirectory();
+};
+
+function markdownFiles(directory: string): string[] {
+  if (!isDirectory(directory)) throw new WireFailure("not-found", `There is no directory ${directory} in the project folder.`);
+  let root: string;
+  try {
+    root = resolveProjectFile(directory);
+  } catch {
+    throw new WireFailure("invalid-input", `${directory} leads out of the project folder through a symbolic link.`);
+  }
+  const found: string[] = [];
+  const walk = (relative: string) => {
+    for (const entry of readdirSync(join(root, relative), { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
+      const path = posix.join(relative, entry.name);
+      if (entry.isDirectory()) walk(path);
+      else if (entry.name.endsWith(".md")) found.push(posix.join(directory, path));
+    }
+  };
+  walk("");
+  return found;
+}
+
+/** The text of each file that the run read: a page, a source file or a reference page is read once. */
+const texts = new Map<string, string | undefined>();
+
+const files: Files = {
+  inside: (path) => {
+    try {
+      resolveProjectPath(path);
+      return true;
+    } catch {
+      return false;
+    }
+  },
+  exists: (path) => {
+    try {
+      resolveProjectFile(path);
+      return true;
+    } catch {
+      return false;
+    }
+  },
+  text: (path) => {
+    if (texts.has(path)) return texts.get(path);
+    let text: string | undefined;
+    try {
+      const real = resolveProjectFile(path);
+      text = statSync(real).isFile() ? readFileSync(real, "utf8") : undefined;
+    } catch {
+      text = undefined;
+    }
+    texts.set(path, text);
+    return text;
+  },
+};
+
+/** A .md entry of a directory that is a symbolic link to no file, or to a file outside the project. */
+const unreadable = (page: string): Problem => ({ page, line: 1, kind: "missing-file", message: `${page} is a symbolic link to no file or to a file outside the project folder.` });
+
+function run(argv: string[]): { output: Record<string, unknown>; failed: boolean } {
+  const { values, positionals } = parseArgs({ args: argv, strict: true, allowPositionals: true, options: { source: { type: "string" }, reference: { type: "string" }, check: { type: "boolean" }, help: { type: "boolean" } } });
+  if (values.help) throw new UsageError("");
+  if (positionals.length === 0) throw new UsageError("give the pages: .md files and directories of the documentation, for example README.md docs");
+  if (values.source === undefined) throw new UsageError("give --source <dir>: the source directory of c64-source");
+  const given = [...new Set(positionals.map((path) => posix.normalize(path)))];
+  for (const [option, path] of [...given.map((path) => ["<pages>", path] as const), ["--source", values.source], ["--reference", values.reference]] as const) {
+    if (path === undefined) continue;
+    if (!files.inside(path)) throw new UsageError(`${option} must be a path relative to the project folder that stays inside it, not ${path}`);
+  }
+  for (const path of given) {
+    if (!path.endsWith(".md") && files.exists(path) && !isDirectory(path)) throw new UsageError(`<pages> takes .md files and directories, not ${path}`);
+  }
+  const directories = given.filter((path) => !path.endsWith(".md") || isDirectory(path));
+  const map = readSourceMap(values.source);
+  const reference = values.reference === undefined ? directories.map((path) => posix.join(path, "reference")).find((path) => isDirectory(path)) : posix.normalize(values.reference);
+  const problems: Problem[] = [];
+  const db = openForRead();
+  let symbols;
+  let revision;
+  try {
+    revision = currentRevision(db);
+    symbols = listSymbols(db, undefined, undefined, undefined, map.space);
+  } finally {
+    db?.close();
+  }
+  if (map.revision !== revision) {
+    problems.push({ page: map.pathOf("source-map.json"), line: 1, kind: "stale-reference", message: `The source is from knowledge revision ${map.revision}, and the knowledge is at ${revision}. Generate the source and the reference again, then run link.ts.` });
+  }
+  const addresses = new Map<string, number>();
+  for (const label of map.labels) addresses.set(label.name, label.address);
+  for (const symbol of symbols) addresses.set(symbol.name, symbol.address);
+  // A routine runs to the next routine; any other name to the next name of the knowledge, in its file, and outside the image at most MAX_BODY bytes.
+  const routines = symbols.filter((symbol) => symbol.kind === "routine").map((symbol) => symbol.address).sort((a, b) => a - b);
+  const named = [...new Set(symbols.map((symbol) => symbol.address))].sort((a, b) => a - b);
+  const kinds = new Map(symbols.map((symbol) => [symbol.name, symbol.kind]));
+  const bodyEnds = new Map<string, number | undefined>();
+  const bodyEnd = (name: string) => {
+    if (bodyEnds.has(name)) return bodyEnds.get(name);
+    const address = addresses.get(name);
+    let end: number | undefined;
+    if (address !== undefined) {
+      const next = ((kinds.get(name) === "routine" ? routines : named).find((other) => other > address) ?? 0x10000) - 1;
+      const file = map.lineOf(address)?.file;
+      if (file === undefined) end = Math.min(next, address + MAX_BODY - 1);
+      else {
+        end = address;
+        while (end < next && map.lineOf(end + 1)?.file === file) end++;
+      }
+    }
+    bodyEnds.set(name, end);
+    return end;
+  };
+  const nameAt = new Map(symbols.map((symbol) => [symbol.address, symbol.name]));
+  /** The routine, else the name, whose body holds an address. */
+  const placeOf = (address: number) => {
+    for (const list of [routines, named]) {
+      let start: number | undefined;
+      for (const other of list) if (other <= address) start = other;
+      const name = start === undefined ? undefined : nameAt.get(start);
+      if (name !== undefined && (bodyEnd(name) ?? start!) >= address) return name;
+    }
+    return undefined;
+  };
+
+  // The entries of the reference: each <a id> that is a name.
+  const entries = new Map<string, string>();
+  const referencePages = reference === undefined ? [] : markdownFiles(reference);
+  for (const page of referencePages) {
+    const text = files.text(page);
+    if (text === undefined) {
+      problems.push(unreadable(page));
+      continue;
+    }
+    const stamp = /^<!-- Generated by c64-docs from knowledge revision (\S+) /.exec(text);
+    if (text.startsWith(GENERATED_MARK) && stamp !== null && stamp[1] !== map.revision) {
+      problems.push({ page, line: 1, kind: "stale-reference", message: `${page} is from knowledge revision ${stamp[1]}, and the source from ${map.revision}. Run generate.ts again.` });
+    }
+    for (const match of text.matchAll(/<a id="([^"]+)"><\/a>/g)) if (addresses.has(match[1]!) && !entries.has(match[1]!)) entries.set(match[1]!, `${page}#${match[1]}`);
+  }
+  const inside = (path: string, directory: string | undefined) => directory !== undefined && (path === directory || path.startsWith(`${directory}/`));
+  const context = {
+    addressOf: (name: string) => addresses.get(name),
+    codeOf: (name: string) => {
+      const label = map.labelOf(name);
+      return label === undefined ? undefined : `${map.pathOf(label.file)}#L${label.line}`;
+    },
+    entryOf: (name: string) => entries.get(name),
+    bodyEnd,
+    placeOf,
+    lineOf: (address: number) => {
+      const line = map.lineOf(address);
+      return line === undefined ? undefined : `${map.pathOf(line.file)}#L${line.line}`;
+    },
+    managed: (path: string) => inside(path, posix.normalize(map.directory)) || inside(path, reference),
+  };
+
+  const pages = [...new Set(given.flatMap((path) => {
+    if (directories.includes(path)) return markdownFiles(path);
+    if (files.text(path) === undefined) throw new WireFailure("not-found", `There is no file ${path} in the project folder.`);
+    return [path];
+  }))];
+  const changedPages: string[] = [];
+  let links = 0;
+  for (const page of pages) {
+    const text = files.text(page);
+    if (text === undefined) {
+      problems.push(unreadable(page));
+      continue;
+    }
+    // A generated page is right by its making: it is only checked.
+    const generated = text.startsWith(GENERATED_MARK);
+    const linked = generated ? { text, changed: 0, problems: [] } : linkPage(text, { page, ...context });
+    problems.push(...linked.problems, ...checkLinks(page, linked.text, files));
+    if (linked.changed === 0) continue;
+    links += linked.changed;
+    changedPages.push(page);
+    if (values.check) continue;
+    writeProjectFile(page, Buffer.from(linked.text, "utf8"));
+    texts.set(page, linked.text);
+  }
+  const byKind: Record<string, number> = {};
+  for (const problem of problems) byKind[problem.kind] = (byKind[problem.kind] ?? 0) + 1;
+  const output: Record<string, unknown> = {
+    pages: pages.length,
+    ...(reference === undefined ? { reference: "none: run generate.ts for links to the entries" } : { reference }),
+    ...(values.check ? { notUpToDate: { links, pages: changedPages } } : { linked: { links, pages: changedPages } }),
+    problems: { count: problems.length, byKind, listed: problems.slice(0, MAX_LISTED).map((problem) => ({ at: `${problem.page}:${problem.line}`, kind: problem.kind, message: problem.message })) },
+  };
+  return { output, failed: problems.length > 0 || (values.check === true && links > 0) };
+}
+
+await runScript(
+  (argv) => {
+    const { output, failed } = run(argv);
+    if (failed) process.exitCode = 1;
+    return output;
+  },
+  { usage: USAGE, refusals: [KnowledgeError] },
+);

package/skills/c64-emulator/SKILL.md

@@ -25,18 +25,21 @@ Use only the MCP tools. Each MCP session has its own C64. Nothing else can chang
 
 | Need | Tool |
 |---|---|
-| Run state, video standard, warp mode, window | `c64_status` |
+| Run state, video standard, warp mode, window, version of the MCP server | `c64_status` |
 | Reset, power cycle | `c64_reset` |
 | Fast emulation | `c64_warp` |
 | Show the C64 to the user in a window, hide it again | `c64_window` |
 | Stop, start, step, step over, step out, exact frames | `c64_execution` |
 | Run until an address, a memory value or a raster position | `c64_run_until` |
+| Run to a place and read the machine there in one call | `c64_run_until` with `observe` and `history` |
+| Each write, read or routine entry over many frames, with the instruction and the raster line | `c64_trace` |
 | Stop on an instruction address | `c64_breakpoint` |
 | Stop on a memory read or write | `c64_watchpoint` |
 | Memory | `c64_memory_read`, `c64_memory_write`, `c64_memory_search`, `c64_memory_compare` |
 | CPU registers | `c64_registers` |
 | Code as it is in memory now | `c64_disassemble` |
 | Executed instructions, call chain | `c64_cpu_history`, `c64_backtrace` |
+| Which instruction reads or writes an address | `c64_trace` with a `read` or `write` point |
 | Where the CPU spends cycles, which memory it uses | `c64_profile`, `c64_memmap` |
 | Cycle count between two points | `c64_timing` |
 | Video chip, sprites, CIAs, sound chip | `c64_vicii`, `c64_sprite`, `c64_cia`, `c64_sid` |
@@ -48,9 +51,18 @@ Use only the MCP tools. Each MCP session has its own C64. Nothing else can chang
 
 Each memory and register tool has a `space`. Use `c64` for the computer. Use `drive8` for the CPU of the 1541 disk drive. The memory `view` is `cpu` or `ram`. With `cpu`, you see what the CPU sees now, with ROM and I/O where the memory configuration puts them. With `ram`, you see the RAM under them.
 
-For a large result, give `out` with a path in the project. `c64_memory_read`, `c64_disassemble`, `c64_cpu_history` and `c64_screen` then write the result to that file. The answer gives only the file, the number of bytes and a SHA-256. Use it for a memory dump of more than a few hundred bytes, a long listing, or a screen that the user wants to keep.
+For a large result, give `out` with a path in the project. `c64_memory_read`, `c64_disassemble`, `c64_cpu_history` and `c64_screen` then write the result to that file. The answer gives only the file, the number of bytes and a SHA-256. Use it for a memory dump of more than a few hundred bytes, a long listing, or a screen that the user wants to keep. `c64_memmap` action `read` and `c64_snapshot` action `save` also take `out`. Write a file that is only for your work in `.c64-re-tools/local/` of the project. Git ignores that folder.
 
-`c64_memmap` keeps RAM, ROM and I/O apart. Use action `summary` to see which code ran. Use action `read` with `out` to keep a coverage run as a file, for example `coverage/attract.bin`. Clear the map before each run that you want to keep apart.
+`c64_memmap` keeps RAM, ROM and I/O apart. Use action `summary` to see which code ran. Use action `read` with `out` to keep a coverage run as a file, for example `.c64-re-tools/local/attract.bin`. Then add it to the knowledge with `coverage` of the c64-knowledge skill. Clear the map before each run that you want to keep apart.
+
+- Only the first byte of an instruction gets execute. Thus `read` without `out` gives a new range at each instruction of code. For code, use `summary`, or `read` with `out`.
+- A read on a byte right after executed code, or at a return address, can be a dummy read of the 6502. The CPU reads such a byte but does not use it.
+
+`c64_profile` gives the routines that used the most cycles since the session started, the last `clear` or the last `c64_window`. Use action `clear` before the part of the program that you want to measure. The profiler gives the cycles to the routine that a JSR called last. Code that the CPU enters with JMP or a branch counts for that routine. After `c64_window` or a snapshot restore, the profiler of VICE keeps the routines that ran before the restore. Then `c64_profile` can give the cycles of the program to such a routine, for example `$fd50`. Do not trust the routine addresses of `c64_profile` after a restore.
+
+### Project folder
+
+Paths (`path`, `file` and `out`) are relative to the project folder. A project folder holds `.c64-re-tools/`. The tools use the nearest project folder at or above the folder where the agent started. If that folder is not in a project and has projects below it, a tool with a path refuses and lists them. Then give `project` with the project folder, for example `"project": "game"`. Give the same folder as the `--project` of the skill scripts. A tool without a path does not need `project`.
 
 ## Workflow
 
@@ -58,7 +70,7 @@ For a large result, give `out` with a path in the project. `c64_memory_read`, `c
 2. Get the machine state with `c64_status`.
 3. Load the subject if it is not loaded. Use `c64_autostart` to start it as a user does. Use `c64_program_load` to put a PRG in memory with no start.
 4. Set the stop condition. Use a breakpoint, a watchpoint or a `c64_run_until` target. Add a typed condition when the stop must occur only in one state. The `list` result shows the condition of each breakpoint and each watchpoint.
-5. Run, step or give input. Prefer `c64_run_until` and `advance-frames` to waits. They stop at the same emulated time on every run.
+5. Run, step or give input. Prefer `c64_run_until` and `advance-frames` to waits. They stop at the same emulated time on every run. To wait for a breakpoint or a watchpoint, use `advance-frames` with a large count. `stopReason` and `pointId` in the result tell what stopped the CPU. `c64_status` also gives them.
 6. Read only the state that answers the question. To read several things from one moment, use `c64_observe`.
 7. Interpret the evidence. Keep the observed facts apart from your conclusions.
 
@@ -72,7 +84,6 @@ Step, next and until-return work only in space `c64`. The emulator cannot stop t
 
 `c64_keyboard` puts keys in the keyboard queue of the KERNAL. A program that reads the keyboard hardware directly does not see them. Many games do this. The emulator has no command that presses one key of the keyboard matrix. For such a program:
 
-- After `c64_window` or a snapshot restore, the profiler of VICE keeps the routines that ran before the restore. Then `c64_profile` can give the cycles of the program to such a routine, for example `$fd50`. Do not trust the routine addresses of `c64_profile` after a restore.
 - VICE 3.10 and older have a known flaw: bits 5-7 of `$DC00` and `$DC01` are always 0. Then `emulatorFlaw` in the result of `c64_keyboard` and `c64_joystick` gives the flaw. With this flaw, a wait for any key that reads all keyboard columns at one time ends without a key. A wait for one specific key, or a compare of the full byte of a joystick port, never ends. Tell the user about the flaw. The next VICE release and the VICE nightly builds do not have it. Joystick port 1 uses the same lines as the keyboard.
 - For one specific key, for example F1 or RUN/STOP, open the window and ask the user to press the key.
 - Do not change the program to make it accept a key.