@henols/c64-re-tools 2.0.0-rc.9 → 2.0.0

package/skills/c64-provenance/scripts/provenance.ts

@@ -1,29 +1,54 @@
 // The c64-provenance script: compares two or more releases byte by byte at
 // their addresses and reports where they differ and which releases agree. A
-// release is a PRG or an unpacker image.json (D50). With --run, each release
+// release is a PRG or an unpacker image.json. With --run, each release
 // runs in its own emulator, and each listed difference gets its first access
 // and whether the run changed it. The separate command gives evidence from
-// one release (D51).
+// the program in the knowledge. The routine and match commands keep
+// routines that several programs share in a library folder and find them in the
+// program (D71).
 
-import { posix } from "node:path";
+import { basename, posix, relative } from "node:path";
 import { parseArgs } from "node:util";
 
 import { formatC64Address, parseC64Address } from "#src/c64.ts";
-import { readCoverageFiles } from "#src/host-client/coverage.ts";
 import { checkActionSteps, type ActionStep } from "#src/host-client/steps.ts";
 import { MAX_EXECUTION_COUNT, WireFailure } from "#src/host-client/tools.ts";
 import { readProjectFile, writeProjectFile } from "#src/host-client/transfer.ts";
 import { ViceSessionClient } from "#src/host-client/vice-session.ts";
 import { KnowledgeError, openForRead } from "#src/knowledge/database.ts";
-import { listReferences, listRegions } from "#src/knowledge/read.ts";
-import { CODE_REGION_TYPES } from "#src/knowledge/schema.ts";
+import { programCoverage, requireProgram } from "#src/knowledge/program.ts";
+import {
+  commentsAt,
+  currentRevision,
+  listComments,
+  listModules,
+  listOperands,
+  listReferences,
+  listRegions,
+  listSymbols,
+  operandAt,
+  referencesFrom,
+  referenceSuppressed,
+  regionsOverlapping,
+  suppressionsOverlapping,
+  symbolAt,
+  symbolNamed,
+  symbolSuppressed,
+} from "#src/knowledge/read.ts";
+import { CODE_REGION_TYPES, type Origin } from "#src/knowledge/schema.ts";
+import { platformNames } from "#src/platform.ts";
+import { projectRoot } from "#src/project.ts";
 import { runScript, UsageError } from "#src/script.ts";
-import { compareReleases, shiftSuggestion, suggestShifts, type Range, type Release } from "./compare.ts";
-import { watchRelease, type ImageState, type Start } from "./runtime.ts";
-import { separate } from "./separate.ts";
+import { compareReleases, shiftSuggestion, suggestShifts, type Range, type Release } from "./lib/compare.ts";
+import { findRoutine, libraryFolder, LibraryError, readLibrary, routineFrom, saveRoutine, suggest } from "./lib/library.ts";
+import { watchRelease, type ImageState, type Start } from "./lib/runtime.ts";
+import { separate } from "./lib/separate.ts";
 
 const USAGE = `provenance.ts <release> <release> [<release> ...] [options]
-provenance.ts separate <release> [--coverage <file> ...] [--out <file>] [--max-listed <n>]
+provenance.ts separate [--out <file>] [--max-listed <n>]
+provenance.ts routine --library <folder> (--module <address> | --start <address> --end <address>)
+                      [--name <name>] [--title <text>] [--replace]
+provenance.ts match --library <folder> [--out <file>] [--max-listed <n>]
 
   --shift <n>:<bytes>   move release n (1 = the first) by a number of bytes, for example 2:-256
   --from <address>      compare from this address
@@ -32,15 +57,28 @@ provenance.ts separate <release> [--coverage <file> ...] [--out <file>] [--max-l
                         give each listed difference its first access and whether it changed
   --steps <file>        with --run: action steps (JSON) to run before the frames
 
-separate gives evidence from one release: the unpack stage that wrote each range,
-its access in the --coverage files (64 KiB each, repeatable), whether call and
-jump references in knowledge reach each code region from the entry, and what the
-program reads or executes outside the image. --out writes all ranges to a file;
+separate gives evidence from the program in the knowledge: the unpack stage
+that wrote each range, its access in the coverage of the knowledge, whether call
+and jump references in knowledge reach each code region from the entry, and what
+the program reads or executes outside its segments. --out writes all ranges to a file;
 the result lists at most --max-listed of each list (default 50) and its count.
 
-A release is a PRG, placed at its load address, or an image.json of the
-c64-unpacker skill, compared inside its segments. Paths are relative to the
-project directory. The result is one JSON object.`;
+routine saves a routine of the program in the knowledge to a library: a plain folder of routine
+files, beside the projects (--library, relative to the working directory, as
+--project). It takes a module of the knowledge, or --start and --end. Its
+code regions give the instructions; the user and LLM names, regions, comments,
+references and operand expressions go into the file at offsets. --name defaults
+to the user or LLM name at the start, --title to the module title.
+
+match finds each routine of the library in the program of the knowledge: the opcodes and other
+fixed bytes equal, and the addresses into the routine moved by one shift.
+Operands that address something outside the routine may differ; the result
+lists them. --out writes the names of the routines at the found addresses as a
+batch for knowledge.ts apply, and the result lists what it left out.
+
+A release to compare is a PRG, placed at its load address, or an image.json of
+the c64-unpacker skill, compared inside its segments. Paths are relative to the
+project folder. The result is one JSON object.`;
 
 function address(text: string, option: string): number {
   try {
@@ -155,20 +193,40 @@ async function run(argv: string[]): Promise<unknown> {
       to: { type: "string" },
       run: { type: "string" },
       steps: { type: "string" },
-      coverage: { type: "string", multiple: true },
       out: { type: "string" },
       "max-listed": { type: "string" },
+      library: { type: "string" },
+      module: { type: "string" },
+      start: { type: "string" },
+      end: { type: "string" },
+      name: { type: "string" },
+      title: { type: "string" },
+      replace: { type: "boolean" },
       help: { type: "boolean" },
     },
   });
   if (values.help) throw new UsageError("");
+  /** Refuses the options that `command` does not take. */
+  const only = (command: string, allowed: readonly string[]) => {
+    const used = Object.keys(values).filter((option) => option !== "help" && !allowed.includes(option));
+    if (used.length > 0) throw new UsageError(`${command} does not take --${used.join(", --")}`);
+  };
   if (positionals[0] === "separate") {
-    const used = ["shift", "from", "to", "run", "steps"].filter((option) => values[option as keyof typeof values] !== undefined);
-    if (used.length > 0) throw new UsageError(`separate does not take --${used.join(", --")}`);
-    if (positionals.length !== 2) throw new UsageError("separate needs one release");
-    return separateCommand(positionals[1]!, values.coverage ?? [], values.out, values["max-listed"]);
+    only("separate", ["out", "max-listed"]);
+    if (positionals.length !== 1) throw new UsageError("separate takes no release: it reads the program from the knowledge");
+    return separateCommand(values.out, values["max-listed"]);
+  }
+  if (positionals[0] === "routine") {
+    only("routine", ["library", "module", "start", "end", "name", "title", "replace"]);
+    if (positionals.length !== 1) throw new UsageError("routine takes no image: it reads the program from the knowledge");
+    return routineCommand(values);
+  }
+  if (positionals[0] === "match") {
+    only("match", ["library", "out", "max-listed"]);
+    if (positionals.length !== 1) throw new UsageError("match takes no image: it reads the program from the knowledge");
+    return matchCommand(values.library, values.out, values["max-listed"]);
   }
-  if (values.coverage !== undefined || values.out !== undefined || values["max-listed"] !== undefined) throw new UsageError("--coverage, --out and --max-listed work only with separate");
+  only("a comparison", ["shift", "from", "to", "run", "steps"]);
   if (positionals.length < 2) throw new UsageError("give two or more releases");
   const shifts = new Map<number, number>();
   for (const text of values.shift ?? []) {
@@ -204,7 +262,7 @@ async function run(argv: string[]): Promise<unknown> {
   const compared = comparison.identicalBytes + comparison.differingBytes;
   const identicalShare = compared === 0 ? 0 : comparison.identicalBytes / compared;
   // A poor match at the load addresses can mean that the releases are placed differently.
-  // Each release gets its best shifts: data that stayed can outvote code that moved, so one is not enough (b2/04).
+  // Each release gets its best shifts: data that stayed can outvote code that moved, so one is not enough.
   const applied = loaded.map((item) => item.shift);
   const suggestions =
     identicalShare < 0.5
@@ -255,7 +313,7 @@ async function run(argv: string[]): Promise<unknown> {
       end: formatC64Address(difference.end),
       values: Object.fromEntries(Object.entries(difference.values).map(([name, bytes]) => [name, hexBytes(bytes.slice(0, 32)) + (bytes.length > 32 ? " ..." : "")])),
       groups: difference.groups,
-      // runtime gives the release's own addresses: with a shift, they are not the addresses of the difference (F-24).
+      // runtime gives the release's own addresses: with a shift, they are not the addresses of the difference.
       ...(runtime.size === 0
         ? {}
         : {
@@ -276,28 +334,43 @@ async function run(argv: string[]): Promise<unknown> {
   };
 }
 
-/** separate: evidence from one release (D51), with all ranges in --out and the first ones in the result. */
-function separateCommand(path: string, coveragePaths: string[], out: string | undefined, maxListed: string | undefined): unknown {
+/** The stages of the capture in the knowledge as ranges; a stage that is not a capture stage is refused. */
+function stagesOf(stages: Array<Record<string, unknown>> | undefined): Array<{ entry: number; wrote: Range[] }> | undefined {
+  if (stages === undefined) return undefined;
+  const parse = (value: unknown) => {
+    if (typeof value === "string" && /^\$[0-9a-f]{4}$/i.test(value)) return Number.parseInt(value.slice(1), 16);
+    throw new UsageError("the stages of the program in the knowledge are not the stages of a c64-unpacker capture; put the capture in again with knowledge.ts image");
+  };
+  return stages.map((stage) => ({
+    entry: parse(stage.entry),
+    wrote: (Array.isArray(stage.wrote) ? (stage.wrote as Array<Record<string, unknown>>) : []).map((range) => ({ start: parse(range.start), end: parse(range.end) })),
+  }));
+}
+
+/** separate: evidence from the program in the knowledge, with all ranges in --out and the first ones in the result. */
+function separateCommand(out: string | undefined, maxListed: string | undefined): unknown {
   const max = maxListed ?? "50";
   if (!/^\d{1,5}$/.test(max)) throw new UsageError(`--max-listed must be a whole number, not ${max}`);
-  const loaded = readRelease(path, 0);
-  const coverage = readCoverageFiles(coveragePaths, (message) => {
-    throw new UsageError(message);
-  });
   const db = openForRead();
+  let program: ReturnType<typeof requireProgram>;
+  let coverage: Uint8Array | undefined;
   let regions: ReturnType<typeof listRegions>;
   let references: ReturnType<typeof listReferences>;
   try {
+    program = requireProgram(db);
+    coverage = programCoverage(db)?.map;
     regions = listRegions(db).filter((region) => CODE_REGION_TYPES.includes(region.type));
     references = listReferences(db);
   } finally {
     db?.close();
   }
+  const stages = stagesOf(program.stages);
+  const loaded = { entry: program.entry, stages };
   const found = separate({
-    segments: loaded.release.segments,
-    memory: loaded.memory,
-    ...(loaded.entry === undefined ? {} : { entry: loaded.entry }),
-    ...(loaded.stages === undefined ? {} : { stages: loaded.stages }),
+    segments: program.segments,
+    memory: program.memory,
+    ...(program.entry === undefined ? {} : { entry: program.entry }),
+    ...(stages === undefined ? {} : { stages }),
     ...(coverage === undefined ? {} : { coverage }),
     codeRegions: regions,
     references,
@@ -307,21 +380,20 @@ function separateCommand(path: string, coveragePaths: string[], out: string | un
   const codeRegions = found.codeRegions.map((region) => ({ ...show(region), type: region.type, ...(region.reachable === undefined ? {} : { reachable: region.reachable }), ...(region.executed === undefined ? {} : { executed: region.executed }) }));
   const outside = { read: found.outside.read.map(show), executed: found.outside.executed.map(show), generated: found.outside.generated.map(show) };
   const listed = <T>(items: T[]) => ({ count: items.length, listed: items.slice(0, Number(max)) });
-  if (out !== undefined) writeProjectFile(out, Buffer.from(`${JSON.stringify({ release: path, ranges, codeRegions, outside }, null, 1)}\n`), { what: "--out" });
+  if (out !== undefined) writeProjectFile(out, Buffer.from(`${JSON.stringify({ ranges, codeRegions, outside }, null, 1)}\n`), { what: "--out" });
   const bytes = (filter: (range: (typeof found.ranges)[number]) => boolean) => found.ranges.filter(filter).reduce((sum, range) => sum + range.end - range.start + 1, 0);
   const lastStage = loaded.stages === undefined ? undefined : loaded.stages.length - 1;
   return {
-    release: path,
     ...(loaded.entry === undefined ? {} : { entry: formatC64Address(loaded.entry) }),
     ...(lastStage === undefined
-      ? { noStages: "image.json has no stage log. Only a capture of c64-unpacker writes one. A capture of loads holds one part of a program and has none." }
+      ? { noStages: "The program in the knowledge has no stage log. Only a capture of c64-unpacker gives one. A capture of loads holds one part of a program and has none." }
       : {
           stages: lastStage + 1,
           bytesByStage: Object.fromEntries([...[...Array(lastStage + 1).keys()].map((stage): [string, number] => [String(stage), bytes((range) => range.stage === stage)]), ["none", bytes((range) => range.stage === undefined)]]),
         }),
     ...(coverage === undefined ? {} : { bytesByAccess: Object.fromEntries((["executed", "read", "written", "read-written", "untouched"] as const).map((access) => [access, bytes((range) => range.access === access)])) }),
     ranges: listed(ranges),
-    // total counts every code region; unreachable is a list like the others (b2/08).
+    // total counts every code region; unreachable is a list like the others.
     codeRegions: {
       total: codeRegions.length,
       ...(loaded.entry === undefined ? {} : { unreachable: listed(codeRegions.filter((region) => region.reachable === false)) }),
@@ -331,4 +403,134 @@ function separateCommand(path: string, coveragePaths: string[], out: string | un
   };
 }
 
-await runScript(() => run(process.argv.slice(2)), { usage: USAGE, refusals: [KnowledgeError] });
+const SEMANTIC: readonly Origin[] = ["user", "llm"];
+
+const platform = () => new Map(platformNames().map((entry) => [entry.name, entry.address]));
+
+/** routine: saves a module or a range of the program in the knowledge, with its knowledge, as a routine of the library (D71). */
+function routineCommand(values: { library?: string; module?: string; start?: string; end?: string; name?: string; title?: string; replace?: boolean }): unknown {
+  if (values.library === undefined) throw new UsageError("routine needs --library <folder>");
+  if (values.module !== undefined && (values.start !== undefined || values.end !== undefined)) throw new UsageError("routine takes --module, or --start and --end, not both");
+  if (values.module === undefined && values.start === undefined && values.end === undefined) throw new UsageError("routine needs --module <address>, or --start and --end");
+  if (values.module === undefined && (values.start === undefined || values.end === undefined)) throw new UsageError("routine needs both --start and --end");
+  const db = openForRead();
+  try {
+    const image = requireProgram(db);
+    const segmentOf = (at: number) => image.segments.find((segment) => at >= segment.start && at <= segment.end);
+    let start: number;
+    let end: number;
+    let title = values.title;
+    if (values.module !== undefined) {
+      start = address(values.module, "--module");
+      const modules = listModules(db);
+      const module = modules.find((item) => item.start === start);
+      if (module === undefined) throw new LibraryError("not-found", `No module starts at ${formatC64Address(start)}. List the modules with knowledge.ts modules, or give --start and --end.`);
+      const segment = segmentOf(start);
+      if (segment === undefined) throw new UsageError(`The module at ${formatC64Address(start)} is outside the segments of the program.`);
+      end = (modules.find((item) => item.start > start && item.start <= segment.end)?.start ?? segment.end + 1) - 1;
+      title ??= module.title;
+    } else {
+      start = address(values.start!, "--start");
+      end = address(values.end!, "--end");
+      if (end < start) throw new UsageError("--end is before --start");
+      if (segmentOf(start) === undefined || segmentOf(start) !== segmentOf(end)) throw new UsageError(`${formatC64Address(start)}-${formatC64Address(end)} is not inside one segment of the program.`);
+    }
+    const range = { start, end };
+    const name = values.name ?? listSymbols(db, { start, end: start }, undefined, SEMANTIC)[0]?.name;
+    if (name === undefined) throw new UsageError(`The knowledge has no user or LLM name at ${formatC64Address(start)}: give --name.`);
+    const { routine, leftOut } = routineFrom({
+      name,
+      ...(title === undefined ? {} : { title }),
+      program: basename(projectRoot()),
+      image: `knowledge revision ${currentRevision(db)}`,
+      start,
+      end,
+      byteAt: image.byteAt,
+      analyzed: listRegions(db, range),
+      symbols: listSymbols(db, range, undefined, SEMANTIC),
+      regions: listRegions(db, range, undefined, SEMANTIC),
+      comments: listComments(db, range, SEMANTIC),
+      references: listReferences(db, range, undefined, SEMANTIC),
+      operands: listOperands(db, range, SEMANTIC),
+      platform: platform(),
+    });
+    const file = saveRoutine(libraryFolder(values.library, { create: true }), routine, values.replace === true);
+    return {
+      routine: routine.name,
+      file: relative(process.cwd(), file),
+      start: formatC64Address(start),
+      end: formatC64Address(end),
+      length: routine.bytes.length,
+      instructions: routine.instructions.length,
+      relocations: routine.relocations.length,
+      variable: routine.variable.length,
+      counts: { symbols: routine.symbols.length, regions: routine.regions.length, comments: routine.comments.length, references: routine.references.length, operands: routine.operands.length },
+      ...(leftOut.length === 0 ? {} : { leftOut }),
+    };
+  } finally {
+    db?.close();
+  }
+}
+
+/** match: finds the routines of the library in the program of the knowledge, and with --out writes their names there as an apply batch (D71). */
+function matchCommand(library: string | undefined, out: string | undefined, maxListed: string | undefined): unknown {
+  if (library === undefined) throw new UsageError("match needs --library <folder>");
+  const max = maxListed ?? "50";
+  if (!/^\d{1,5}$/.test(max)) throw new UsageError(`--max-listed must be a whole number, not ${max}`);
+  const listed = <T>(items: T[]) => ({ count: items.length, listed: items.slice(0, Number(max)) });
+  const routines = readLibrary(libraryFolder(library, { create: false }));
+  if (routines.length === 0) throw new LibraryError("not-found", `The library ${library} has no routine files. Save a routine with routine first.`);
+  const programDb = openForRead();
+  let image: ReturnType<typeof requireProgram>;
+  try {
+    image = requireProgram(programDb);
+  } finally {
+    programDb?.close();
+  }
+  const found = routines.map((routine) => ({ routine, matches: findRoutine(routine, image) }));
+  const show = (value: number, size: 1 | 2) => `$${value.toString(16).padStart(2 * size, "0")}`;
+  const matches = found
+    .flatMap(({ routine, matches: each }) =>
+      each.map((match) => ({
+        routine: match.routine,
+        start: formatC64Address(match.start),
+        end: formatC64Address(match.start + routine.bytes.length - 1),
+        shift: match.shift,
+        differing: match.differing.map((operand) => ({ address: formatC64Address(operand.address), library: show(operand.library, operand.size), program: show(operand.program, operand.size) })),
+      })),
+    )
+    .sort((a, b) => a.start.localeCompare(b.start));
+  const result: Record<string, unknown> = { library, routines: routines.length, matches: listed(matches) };
+  if (out === undefined) return result;
+  const db = openForRead();
+  let suggestions: ReturnType<typeof suggest>;
+  try {
+    suggestions = suggest(
+      found,
+      {
+        symbolAt: (at) => symbolAt(db, at),
+        symbolNamed: (name) => symbolNamed(db, name),
+        regionsOverlapping: (start, end) => regionsOverlapping(db, start, end),
+        commentsAt: (at) => commentsAt(db, at),
+        referencesFrom: (at) => referencesFrom(db, at),
+        operandAt: (at) => operandAt(db, at),
+        symbolSuppressed: (at) => symbolSuppressed(db, at),
+        regionSuppressed: (start, end) => suppressionsOverlapping(db, "region", start, end).length > 0,
+        referenceSuppressed: (reference) => referenceSuppressed(db, reference),
+      },
+      image,
+      platform(),
+    );
+  } finally {
+    db?.close();
+  }
+  const { batch, known, leftOut } = suggestions;
+  writeProjectFile(out, Buffer.from(batch.length === 0 ? "[]\n" : `[\n${batch.map((write) => ` ${JSON.stringify(write)}`).join(",\n")}\n]\n`), { what: "--out" });
+  return {
+    ...result,
+    batch: { out, writes: batch.length, known, leftOut: listed(leftOut) },
+    ...(batch.length === 0 ? {} : { apply: `knowledge.ts apply ${out} --reason "names of library routines"` }),
+  };
+}
+
+await runScript((argv) => run(argv), { usage: USAGE, refusals: [KnowledgeError, LibraryError] });

package/skills/c64-reverse-engineering/SKILL.md

@@ -26,6 +26,21 @@ This skill does not do the specialist work itself.
 
 This skill uses the other skills. It does not use the host runtime or the emulator directly.
 
+### Project folder
+
+One repository can hold several programs, with one project folder for each program. A project folder holds `.c64-re-tools/`. Work on one project at a time:
+
+- From the root of the repository, give `--project <folder>` to each skill script, and `project` to each emulator tool with a path.
+- Write all files in the project folder. Do not write files at the root of the repository.
+- Write each file that you make for your work in `.c64-re-tools/local/` of the project. Git ignores that folder. Examples are results, batches, step files, runs, snapshots and memory dumps.
+- Move a file out of `.c64-re-tools/local/` only when it is the evidence for a finding. Move it before you give it to `--evidence` of c64-knowledge. `--evidence` refuses a file in `.c64-re-tools/local/`.
+- Put a run that is evidence in `runs/<name>/`, with its run file, its results and the snapshot files that it uses. Change the paths in the run file to the new paths. Then git keeps them, and another person can do the run again.
+- A subagent can have no emulator tools. Give it its experiments as run files of the c64-testing skill. Each run starts its own emulator, so many runs can go at the same time. The emulator of the emulator tools is one machine, for the main session.
+- To name a large program with more agents at the same time, give each agent one range of addresses. Each agent writes a batch with its owner range. Then join the batches with `join` of the c64-knowledge skill, and apply the result one time.
+- Do not write a build script or a merge script. c64-source writes and assembles the loader with `--loader`, and `join` of c64-knowledge joins the batches.
+- Do not write a script that makes a deliverable again, for example a capture, copy or replay script. Commit the deliverable.
+- Write the knowledge only with the c64-knowledge script. Do not write or edit the files in `.c64-re-tools/knowledge/` yourself.
+
 ## Workflow
 
 Select the path for the scope:
@@ -48,6 +63,7 @@ Select the path for the scope:
 | The routines, code, data and references of the program | c64-static-analysis |
 | The platform meaning of an address or a register | c64-memory-map |
 | Differences between releases of the program | c64-provenance |
+| A routine that another program of the workspace has, for example a music driver | c64-provenance, with the library |
 | Source code from the knowledge | c64-source |
 | Source code to build | c64-assembler |
 | Proof that a rebuild behaves as the original | c64-testing |
@@ -58,61 +74,69 @@ Select the path for the scope:
 
 ### The stages of a full rebuild
 
-Do the stages in this sequence. Go to the next stage only when its exit check is true, or when the user accepts an exception. Write each exception in the report.
+Do the stages in this sequence. Go to the next stage only when its exit check is true, or when the user accepts an exception. Remember each exception for the answer of stage 11.
 
 | Stage | Work | Exit check |
 |---|---|---|
-| 1. Acceptance | Ask the user for the criteria of the rebuild. Offer these: identical source, a verified loader, passed c64-testing scenarios, a list of the code that no run executed. Write the criteria in `report.md`. | The user gave the criteria. |
+| 1. Acceptance | Ask the user for the criteria of the rebuild. Offer these: identical source, a verified loader, passed c64-testing scenarios, and a list of the code that no run executed. Also offer documentation of the flows, tables, formats and graphics. | The user gave the criteria. |
 | 2. Get | c64-disk: list the disk. Extract the start file into `original/`. | The start file is in the project. |
 | 3. Start | c64-basic: find the SYS address of the BASIC start. | You have the entry address. |
-| 4. Unpack | c64-unpacker: examine the program. Capture a packed program into `unpacked/`. | `image.json` and `entry_state.json` exist, or the evidence shows no packer. |
-| 5. Run | c64-emulator and the run of c64-testing: run each part that the scope names. Write a coverage file for each part into `coverage/`. | Each part that the scope names has a coverage file. |
-| 6. Analyze | c64-static-analysis with `--coverage` for each coverage file. | No coverage conflict is open. |
-| 7. Name | c64-knowledge and c64-memory-map: name each routine in the scope and give it a comment. Record the modules. | Each routine in the scope has a name and a comment. |
-| 8. Generate | c64-source with `--coverage` into `src/`. For an unpacked program, add `--loader`. | `check.identical` is true. For an unpacked program, `loader.verified` is true. |
+| 4. Unpack | c64-unpacker: examine the program. Capture a packed program into `.c64-re-tools/local/`. Then c64-knowledge: put the capture, or the PRG of a program without a packer, into the knowledge with `image`. | `program` of c64-knowledge shows the segments and the entry. |
+| 5. Run | c64-emulator and the run of c64-testing: run each part that the scope names. Write a coverage map for each part into `.c64-re-tools/local/`. Then c64-knowledge: add the maps with `coverage`. | `program` of c64-knowledge lists a coverage map for each part that the scope names. |
+| 6. Analyze | c64-static-analysis. It reads the program and the coverage from the knowledge. | No coverage conflict is open. |
+| 7. Name | c64-provenance: if the workspace has a library, find its routines with `match` and apply the batch. Then c64-knowledge and c64-memory-map: name each routine in the scope and give it a comment. Record the modules. When the scope has hundreds of routines, ask the user which routines to name. The source gives the other routines an `l_xxxx` label. | Each routine that the user selected has a name and a comment. |
+| 8. Generate | c64-source into `src/`. For an unpacked program, add `--loader`. | `check.identical` is true. For an unpacked program, `loader.verified` is true. |
 | 9. Build and test | c64-assembler with `--compare` into `build/`. c64-testing scenarios in `tests/`. | The criteria of stage 1 are true. |
-| 10. Report | Write `report.md`. | The report has all its sections. |
+| 10. Document | When the criteria ask for documentation: c64-docs into `README.md` and `docs/`. | `link.ts --check` of c64-docs gives no problem. |
+| 11. Hand over | Find each file that you made outside `.c64-re-tools/local/` and that is not a deliverable. Move it into `.c64-re-tools/local/`. Then give the answer (below). | Each file that you made outside `.c64-re-tools/local/` is a deliverable. |
 
 In stage 5, if you cannot get to a part of the program, ask the user how to get to it. Do stages 7 and 8 again until the source is clear. Each run of c64-source writes the source again.
 
+After stage 7, save each routine that other programs can have to the library with c64-provenance `routine`. Examples are a music driver, a decruncher and a loader. Ask the user before you make a library.
+
 ### Deliverables of a full rebuild
 
-- `.c64-re-tools/knowledge.db` with the names, regions, comments and modules
-- the image and the entry state
-- the coverage files
+- `.c64-re-tools/knowledge/` with the program, its entry state, the coverage, the names, the regions, the comments and the modules
 - the generated source
 - the built program and the loader
-- the c64-testing scenarios and their results
-- `report.md`.
+- the c64-testing scenarios
+- the documentation in `README.md` and `docs/`, when the criteria ask for it
+- the runs in `runs/` that the knowledge gives as evidence
+- the scripts in `tools/` for the formats of this program, when you wrote them.
 
-### The report
+All other files that you made are not deliverables. Keep them in `.c64-re-tools/local/`. Do not delete a file that the user made.
 
-`report.md` is short. It has these sections:
+### The answer
 
-1. Acceptance: each criterion of stage 1 and its result, and each exception that the user accepted.
-2. The program: what the program does, in a few sentences.
-3. Modules: the file, the address range and the title of each module.
-4. Untested code: all ranges in the `not_executed.json` file of c64-source. Tell why no run executed them, if you know.
-5. Open questions: what the evidence does not explain.
+Give the user a short answer at the end. Do not write a report file. Write a report file only when the user asks for it. The knowledge, `docs/` and the scenarios keep the durable results. The answer has:
+
+1. Each criterion of stage 1 and its result, and each exception that the user accepted.
+2. The count of the code ranges that no run executed. The ranges are in the `not_executed.json` file of c64-source.
+3. Open questions: what the evidence does not explain.
 
 Keep observed facts apart from your conclusions.
 
 ### Project layout
 
-If the project has no layout, use this one. It is a proposal. No script checks it. Each script takes the paths that you give.
+If the project has no layout, use this one. No script checks it. Each script takes the paths that you give.
 
 ```
 original/      the disk image or the program as you got it
-unpacked/      image.json, image.bin and entry_state.json from c64-unpacker
-coverage/      coverage files from c64_memmap
 src/           the generated source from c64-source
 build/         programs from c64-assembler
-tests/         c64-testing scenarios and their results
-report.md      the report
+tests/         c64-testing scenarios
+runs/          the runs that the knowledge gives as evidence
+docs/          the documentation from c64-docs, with the reference in docs/reference/
+tools/         scripts for the formats of this program only
+README.md      the entry page of the documentation from c64-docs
 ```
 
+Commit each folder of the layout and `.c64-re-tools/knowledge/`. The knowledge holds the program, the entry state and the coverage, so the project needs no other folder for them. Put all other files that you make in `.c64-re-tools/local/`, also the captures and the coverage maps after you put them into the knowledge. Do not let git ignore a folder of the layout. Do not change the `.gitignore` files of the project.
+
 If the project has a layout, use that layout.
 
+A workspace with several projects keeps its library of routines in `library/` beside the projects. The library is not a project folder.
+
 ## Knowledge
 
 Read:
@@ -129,11 +153,12 @@ Record durable conclusions with the c64-knowledge skill. Do not store:
 
 ## Result
 
-A clear answer for the scope, with the durable conclusions in the project knowledge. For a full rebuild, the deliverables and the report, with the criteria of the user met or each exception in the report.
+A clear answer for the scope, with the durable conclusions in the project knowledge. For a full rebuild, the deliverables and the answer of stage 11, with the criteria of the user met or each exception in the answer.
 
 ## Failure and conflicts
 
 - A skill reports `installation-incomplete`, or `machine-unavailable` with a message that the host runtime does not run: stop the whole task. Give the message to the user and wait for the user. Do not continue with other skills. Do not make a replacement for the tool or the host runtime, for example your own disk parser, disassembler or emulator script.
+- A tool gives a wrong result, and the exit check of a stage cannot be true. Tell the user the defect and the evidence, and wait for the user. Do not make a workaround, for example a change to a generated file. When the user accepts an exception, give the defect and the exception in the answer of stage 11.
 - If the evidence does not support a conclusion, keep the unknown and get more evidence. Do not make up a meaning.
 - Before you replace current knowledge from the user or the LLM, read its history and the evidence for it.
 - A conflict from the static analysis: find the correct answer at run time with the c64-emulator skill.