@henols/c64-re-tools 2.0.0-rc.2 → 2.0.0-rc.21

package/skills/c64-docs/scripts/generate.ts

@@ -0,0 +1,102 @@
+// c64-docs generate.ts (D80): the reference pages of a program from the
+// knowledge, the source map of c64-source and the program in the knowledge. The pages are
+// generated and never edited; the pages of a person link to them.
+
+import { existsSync, readdirSync, readFileSync, rmSync } from "node:fs";
+import { join, posix } from "node:path";
+import { parseArgs } from "node:util";
+
+import { WireFailure } from "#src/host-client/tools.ts";
+import { writeProjectFile } from "#src/host-client/transfer.ts";
+import { conflicts } from "#src/knowledge/conflicts.ts";
+import { KnowledgeError, openForRead } from "#src/knowledge/database.ts";
+import { currentRevision, listComments, listModules, listReferences, listRegions, listSymbols } from "#src/knowledge/read.ts";
+import { resolveProjectPath } from "#src/project.ts";
+import { runScript, UsageError } from "#src/script.ts";
+import { buildReference, GENERATED_MARK } from "./lib/reference.ts";
+import { readSourceMap } from "./lib/source-map.ts";
+import { programImage, type ProgramImage } from "#src/knowledge/program.ts";
+
+const USAGE = `generate.ts --source <dir> --out <dir>
+
+  --source <dir>   the source directory of c64-source, with its source-map.json,
+                   relative to the project folder
+  --out <dir>      the directory for the reference pages, relative to the project
+                   folder, for example docs/reference; generate.ts writes only
+                   there and refuses a page in it that it did not write
+When the knowledge has the program of the source (knowledge.ts image of the
+c64-knowledge skill, D83), the reference decodes the tables and shows the
+graphics. The result is one JSON object.`;
+
+/** The files of an earlier run, or a refusal that names a file of a person. */
+function oldFiles(out: string): string[] {
+  const directory = resolveProjectPath(out);
+  if (!existsSync(directory)) return [];
+  const found: string[] = [];
+  const walk = (relative: string) => {
+    for (const entry of readdirSync(join(directory, relative), { withFileTypes: true })) {
+      const path = posix.join(relative, entry.name);
+      if (entry.isDirectory()) {
+        walk(path);
+        continue;
+      }
+      const generated = path.endsWith(".md") ? readFileSync(join(directory, path), "utf8").startsWith(GENERATED_MARK) : path.startsWith("images/") && path.endsWith(".png");
+      if (!generated) throw new WireFailure("invalid-input", `${posix.join(out, path)} is not a page of generate.ts. Nothing was written. Give --out a directory that holds only the reference, and keep the pages of a person outside it.`);
+      found.push(path);
+    }
+  };
+  walk("");
+  return found;
+}
+
+function run(argv: string[]): Record<string, unknown> {
+  const { values, positionals } = parseArgs({ args: argv, strict: true, allowPositionals: true, options: { source: { type: "string" }, out: { type: "string" }, help: { type: "boolean" } } });
+  if (values.help) throw new UsageError("");
+  if (positionals.length > 0) throw new UsageError(`generate.ts takes no positional arguments, not ${positionals[0]}`);
+  if (values.source === undefined) throw new UsageError("give --source <dir>: the source directory of c64-source");
+  if (values.out === undefined) throw new UsageError("give --out <dir>: the directory for the reference pages");
+  for (const [option, path] of [["--source", values.source], ["--out", values.out]] as const) {
+    try {
+      resolveProjectPath(path);
+    } catch {
+      throw new UsageError(`${option} must be a path relative to the project folder that stays inside it, not ${path}`);
+    }
+  }
+  const map = readSourceMap(values.source);
+  const db = openForRead();
+  let knowledge;
+  let image: ProgramImage | undefined;
+  try {
+    image = programImage(db, map.space);
+    const found = conflicts(db).length;
+    if (found > 0) throw new KnowledgeError("conflict", `The knowledge has ${found} conflict${found === 1 ? "" : "s"} from a git merge. Nothing was written. Run c64-knowledge's knowledge.ts merge, then generate the source and the reference again.`);
+    const revision = currentRevision(db);
+    // A source of another revision has other lines: the links would point to the wrong code.
+    if (map.revision !== revision) {
+      throw new WireFailure("invalid-input", `${map.pathOf("source-map.json")} is from knowledge revision ${map.revision}, and the knowledge is at revision ${revision}. Nothing was written. Generate the source again with c64-source, then run generate.ts.`);
+    }
+    knowledge = {
+      revision,
+      symbols: listSymbols(db, undefined, undefined, undefined, map.space),
+      regions: listRegions(db, undefined, undefined, undefined, map.space),
+      comments: listComments(db, undefined, undefined, map.space),
+      references: listReferences(db, undefined, undefined, undefined, map.space),
+      modules: listModules(db, undefined, undefined, map.space),
+    };
+  } finally {
+    db?.close();
+  }
+  const old = oldFiles(values.out);
+  const built = buildReference({
+    out: posix.normalize(values.out),
+    map,
+    ...knowledge,
+    ...(image === undefined ? {} : { image: { byteAt: image.byteAt, inImage: (address: number) => image.segments.some((segment) => address >= segment.start && address <= segment.end) } }),
+  });
+  const directory = resolveProjectPath(values.out);
+  for (const path of old) rmSync(join(directory, path), { force: true });
+  for (const file of built.files) writeProjectFile(file.path, file.bytes, { what: "--out" });
+  return { out: values.out, revision: knowledge.revision, index: posix.join(posix.normalize(values.out), "index.md"), ...built.counts, program: image === undefined ? "none: put the program into the knowledge with knowledge.ts image to decode the tables and show the graphics" : "the knowledge" };
+}
+
+await runScript(run, { usage: USAGE, refusals: [KnowledgeError] });

package/skills/c64-docs/scripts/lib/links.ts

@@ -0,0 +1,155 @@
+// The links of a page written by a person (D80). A name in backticks links to
+// its line in the source; the address after it links to the entry of the name
+// in the reference, or, for an address inside the routine or the data of the
+// name ("draw_room $0AAF", a place in draw_room), to the line of that address. A link that points into the source or the reference is
+// managed: link.ts writes it again when a line moves. Other links stay as the
+// person wrote them; link.ts only checks that their targets exist.
+
+import { posix } from "node:path";
+
+import { anchorsOf, linksOf, proseLines } from "./markdown.ts";
+
+export interface LinkContext {
+  /** The page, relative to the project folder. */
+  page: string;
+  /** The address of a name of the knowledge or of the source; undefined for a word that is no name. */
+  addressOf: (name: string) => number | undefined;
+  /** The project path and line of the label of a name, as path#Lnn. */
+  codeOf: (name: string) => string | undefined;
+  /** The project path and anchor of the entry of a name, as path#name. */
+  entryOf: (name: string) => string | undefined;
+  /** The last address of the routine or data of a name: an address up to it is a place inside the name. */
+  bodyEnd: (name: string) => number | undefined;
+  /** The project path and line that writes an address, as path#Lnn. */
+  lineOf: (address: number) => string | undefined;
+  /** The name of the routine or data that holds an address, for a message. */
+  placeOf: (address: number) => string | undefined;
+  /** True for a project path that link.ts manages: a file of the source or a page of the reference. */
+  managed: (path: string) => boolean;
+}
+
+export type ProblemKind = "unknown-name" | "wrong-address" | "missing-file" | "missing-anchor" | "missing-line" | "outside-project" | "stale-reference";
+
+export interface Problem {
+  page: string;
+  /** 1 is the first line. */
+  line: number;
+  kind: ProblemKind;
+  message: string;
+}
+
+const NAME = String.raw`([A-Za-z_][A-Za-z0-9_]*)(\+(?:\$[0-9A-Fa-f]{1,4}|[0-9]+))?`;
+const ADDRESS = String.raw`\$([0-9A-Fa-f]{2,4})(?:-\$[0-9A-Fa-f]{2,4})?`;
+const TICK = "`";
+/**
+ * A name in backticks, linked or not, with an optional address in backticks
+ * after it, linked or not. Groups: 1 the [ of a linked name, 2 the name, 3 its
+ * offset, 4 the target of the name, 5 the space or bracket between them, 6 the
+ * address text, 7 its start digits, 8 the target of the address.
+ */
+const MENTION = new RegExp(
+  String.raw`(\[)?${TICK}${NAME}${TICK}(?:\]\(([^)\s]*)\))?(?:(\s*\(\s*|\s+)\[?${TICK}(${ADDRESS})${TICK}(?:\]\(([^)\s]*)\))?)?`,
+  "g",
+);
+
+const relative = (page: string, target: string) => {
+  const [path, fragment] = target.split("#");
+  const shown = posix.relative(posix.dirname(page), path!) || posix.basename(path!);
+  return fragment === undefined ? shown : `${shown}#${fragment}`;
+};
+/** The project path of a link target, without its fragment; undefined for a link out of the project (a URL, or a page anchor only). */
+export function targetPath(page: string, target: string): string | undefined {
+  if (/^[a-z][a-z0-9+.-]*:/i.test(target) || target.startsWith("#")) return undefined;
+  return posix.normalize(posix.join(posix.dirname(page), decodeURIComponent(target.split("#")[0]!)));
+}
+
+/** Writes the links of the name mentions of a page; returns the new text, the number of links changed and the problems. */
+export function linkPage(text: string, context: LinkContext): { text: string; changed: number; problems: Problem[] } {
+  const problems: Problem[] = [];
+  let changed = 0;
+  const lines = proseLines(text).map((line, index) => {
+    if (!line.prose) return line.text;
+    return line.text.replace(MENTION, (whole: string, open: string | undefined, name: string, offset: string | undefined, nameTarget: string | undefined, space: string | undefined, addressText: string | undefined, digits: string | undefined, addressTarget: string | undefined, at: number, all: string) => {
+      // "[`x`]" without a target is no link of ours: leave it.
+      if (open !== undefined && nameTarget === undefined) return whole;
+      const ownLink = (target: string | undefined) => target !== undefined && !context.managed(targetPath(context.page, target) ?? "");
+      const address = context.addressOf(name);
+      const report = (kind: ProblemKind, message: string) => problems.push({ page: context.page, line: index + 1, kind, message });
+      // In a list "`a`/`b` `$1000`/`$1010`" the address belongs to another name: the name gets its link, the address stays as it is.
+      const namePart = open === undefined ? `${TICK}${name}${offset ?? ""}${TICK}` : `[${TICK}${name}${offset ?? ""}${TICK}](${nameTarget})`;
+      const listed = addressText !== undefined && (all[at - 1] === "/" || all[at + whole.length] === "/");
+      const rest = listed ? whole.slice(namePart.length) : "";
+      const claimed = addressText !== undefined;
+      if (listed) addressText = digits = addressTarget = undefined;
+      if (address === undefined) {
+        // A word in backticks is no claim of a name; with an address after it, or with a link of ours, it is.
+        if (claimed || (nameTarget !== undefined && !ownLink(nameTarget))) {
+          report("unknown-name", `\`${name}\` is no name of the knowledge or the source. Apply the name to the knowledge, or correct the page.`);
+        }
+        return whole;
+      }
+      const delta = offset === undefined ? 0 : offset.startsWith("+$") ? Number.parseInt(offset.slice(2), 16) : Number.parseInt(offset.slice(1), 10);
+      const given = digits === undefined ? undefined : Number.parseInt(digits, 16);
+      const place = given !== undefined && given !== address + delta && given > address && given <= (context.bodyEnd(name) ?? address);
+      if (given !== undefined && given !== address + delta && !place) {
+        const expected = `$${(address + delta).toString(16).toUpperCase().padStart(4, "0")}`;
+        const holder = context.placeOf(given!);
+        report("wrong-address", `\`${name}${offset ?? ""}\` is ${expected} in the knowledge, and the page gives ${addressText}${holder === undefined || holder === name ? "" : `, a place in \`${holder}\``}. Correct the name or the address.${digits!.length <= 2 ? ` For a value, write \`${name}\` = \`${addressText}\`.` : ""}`);
+      }
+      const shownName = `${TICK}${name}${offset ?? ""}${TICK}`;
+      const code = context.codeOf(name);
+      const nameLink = ownLink(nameTarget) ? `[${shownName}](${nameTarget})` : code === undefined ? shownName : `[${shownName}](${relative(context.page, code)})`;
+      let result = nameLink;
+      if (addressText !== undefined) {
+        const entry = place ? context.lineOf(given!) : context.entryOf(name);
+        const shownAddress = `${TICK}${addressText}${TICK}`;
+        result += space! + (ownLink(addressTarget) ? `[${shownAddress}](${addressTarget})` : entry === undefined ? shownAddress : `[${shownAddress}](${relative(context.page, entry)})`);
+      }
+      result += rest;
+      if (result !== whole) changed++;
+      return result;
+    });
+  });
+  return { text: lines.join("\n"), changed, problems };
+}
+
+/** The files that a check reads: their text, or undefined when the file does not exist. */
+export interface Files {
+  exists: (path: string) => boolean;
+  /** The text of a project file, or undefined when it is no file. */
+  text: (path: string) => string | undefined;
+  inside: (path: string) => boolean;
+}
+
+/** Checks that each link and image of a page reaches a file, and its anchor or line. */
+export function checkLinks(page: string, text: string, files: Files): Problem[] {
+  const problems: Problem[] = [];
+  const anchors = new Map<string, Set<string>>();
+  const anchorsAt = (path: string) => {
+    if (!anchors.has(path)) anchors.set(path, anchorsOf(path === page ? text : (files.text(path) ?? "")));
+    return anchors.get(path)!;
+  };
+  for (const link of linksOf(text)) {
+    const report = (kind: ProblemKind, message: string) => problems.push({ page, line: link.line, kind, message });
+    const fragment = link.target.includes("#") ? link.target.slice(link.target.indexOf("#") + 1) : undefined;
+    if (/^[a-z][a-z0-9+.-]*:/i.test(link.target)) continue;
+    const path = link.target.startsWith("#") ? page : targetPath(page, link.target)!;
+    if (!files.inside(path)) {
+      report("outside-project", `${link.target} leaves the project folder.`);
+      continue;
+    }
+    if (!files.exists(path)) {
+      report("missing-file", `${link.target}: there is no ${path}.`);
+      continue;
+    }
+    if (fragment === undefined || fragment === "") continue;
+    const line = /^L(\d+)(?:-L?\d+)?$/.exec(fragment);
+    if (path.endsWith(".md")) {
+      if (!anchorsAt(path).has(fragment)) report("missing-anchor", `${link.target}: ${path} has no anchor ${fragment}.`);
+    } else if (line !== null) {
+      const count = (files.text(path) ?? "").split("\n").length;
+      if (Number(line[1]) > count) report("missing-line", `${link.target}: ${path} has ${count} lines.`);
+    }
+  }
+  return problems;
+}

package/skills/c64-docs/scripts/lib/markdown.ts

@@ -0,0 +1,79 @@
+// What the documentation scripts read in a Markdown page: the lines outside
+// code blocks and HTML comments, the anchors that a link can name, and the
+// links and images with their targets.
+
+/** The lines of a page and, for each one, whether it is prose: outside fenced code (also mermaid) and outside HTML comments. */
+export function proseLines(text: string): Array<{ text: string; prose: boolean }> {
+  const lines = text.split("\n");
+  let fence: string | undefined;
+  let comment = false;
+  return lines.map((line) => {
+    const opening = /^\s*(`{3,}|~{3,})/.exec(line);
+    if (fence !== undefined) {
+      if (opening !== null && opening[1]!.startsWith(fence[0]!) && opening[1]!.length >= fence.length && line.trim() === opening[1]) fence = undefined;
+      return { text: line, prose: false };
+    }
+    if (opening !== null) {
+      fence = opening[1]!;
+      return { text: line, prose: false };
+    }
+    if (comment) {
+      if (line.includes("-->")) comment = false;
+      return { text: line, prose: false };
+    }
+    if (line.trimStart().startsWith("<!--")) {
+      comment = !line.includes("-->");
+      return { text: line, prose: false };
+    }
+    return { text: line, prose: true };
+  });
+}
+
+/** The id that GitHub and VS Code give a heading: lower case, without punctuation, spaces as hyphens. */
+export function headingSlug(heading: string): string {
+  const text = heading
+    .replace(/!?\[([^\]]*)\]\([^)]*\)/g, "$1")
+    .replace(/<[^>]+>/g, "")
+    .replace(/`/g, "")
+    .trim()
+    .toLowerCase();
+  return text.replace(/[^\p{L}\p{N}\s_-]/gu, "").replace(/\s/g, "-");
+}
+
+/** The anchors of a page: each <a id> or <a name>, and each heading slug, with -1, -2 for a slug that comes again. */
+export function anchorsOf(text: string): Set<string> {
+  const anchors = new Set<string>();
+  const seen = new Map<string, number>();
+  for (const line of proseLines(text)) {
+    for (const match of line.text.matchAll(/<a\s+(?:id|name)="([^"]+)"/g)) anchors.add(match[1]!);
+    if (!line.prose) continue;
+    const heading = /^#{1,6}\s+(.*?)\s*#*\s*$/.exec(line.text);
+    if (heading === null) continue;
+    const slug = headingSlug(heading[1]!);
+    const count = seen.get(slug) ?? 0;
+    seen.set(slug, count + 1);
+    anchors.add(count === 0 ? slug : `${slug}-${count}`);
+  }
+  return anchors;
+}
+
+export interface Link {
+  /** 1 is the first line. */
+  line: number;
+  image: boolean;
+  target: string;
+}
+
+/** The links and images of the prose lines; links inside inline code are not links. */
+export function linksOf(text: string): Link[] {
+  const links: Link[] = [];
+  proseLines(text).forEach((line, index) => {
+    if (!line.prose) return;
+    // Inline code that holds no link text: `[x](y)` inside one pair of backticks is code, not a link.
+    const outside = line.text.replace(/`[^`]*`/g, (code) => (code.includes("](") ? " ".repeat(code.length) : code));
+    for (const match of outside.matchAll(/(!?)\[(?:[^\]`]|`[^`]*`)*\]\(([^)\s]+)(?:\s+"[^"]*")?\)/g)) {
+      links.push({ line: index + 1, image: match[1] === "!", target: match[2]! });
+    }
+  });
+  return links;
+}