@henols/c64-re-tools 2.1.0-rc.1 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/package.json +1 -1
  2. package/skills/c64-assembler/SKILL.md +24 -9
  3. package/skills/c64-assembler/scripts/assembler.ts +95 -0
  4. package/skills/c64-basic/SKILL.md +17 -9
  5. package/skills/c64-basic/scripts/basic.ts +26 -36
  6. package/skills/c64-disk/SKILL.md +17 -7
  7. package/skills/c64-disk/scripts/disk.ts +130 -104
  8. package/skills/c64-disk/scripts/lib/containers.ts +10 -12
  9. package/skills/c64-docs/SKILL.md +47 -28
  10. package/skills/c64-docs/references/style.md +6 -6
  11. package/skills/c64-docs/scripts/docs.ts +323 -0
  12. package/skills/c64-docs/scripts/lib/source-map.ts +1 -1
  13. package/skills/c64-emulator/SKILL.md +11 -7
  14. package/skills/c64-knowledge/SKILL.md +34 -21
  15. package/skills/c64-knowledge/scripts/knowledge.ts +597 -718
  16. package/skills/c64-knowledge/scripts/lib/listing.ts +5 -7
  17. package/skills/c64-memory-map/SKILL.md +18 -10
  18. package/skills/c64-memory-map/scripts/memory-map.ts +81 -0
  19. package/skills/c64-provenance/SKILL.md +33 -24
  20. package/skills/c64-provenance/scripts/lib/compare.ts +8 -12
  21. package/skills/c64-provenance/scripts/lib/leftovers.ts +19 -20
  22. package/skills/c64-provenance/scripts/lib/library.ts +15 -25
  23. package/skills/c64-provenance/scripts/lib/runtime.ts +12 -37
  24. package/skills/c64-provenance/scripts/lib/separate.ts +9 -10
  25. package/skills/c64-provenance/scripts/provenance.ts +177 -279
  26. package/skills/c64-reverse-engineering/SKILL.md +13 -12
  27. package/skills/c64-source/SKILL.md +28 -16
  28. package/skills/c64-source/scripts/lib/loader.ts +52 -77
  29. package/skills/c64-source/scripts/lib/source.ts +33 -42
  30. package/skills/c64-source/scripts/{generate.ts → source.ts} +68 -109
  31. package/skills/c64-static-analysis/SKILL.md +24 -13
  32. package/skills/c64-static-analysis/scripts/lib/coverage.ts +3 -4
  33. package/skills/c64-static-analysis/scripts/lib/findings.ts +2 -2
  34. package/skills/c64-static-analysis/scripts/lib/instructions.ts +1 -1
  35. package/skills/c64-static-analysis/scripts/lib/macros.ts +6 -8
  36. package/skills/c64-static-analysis/scripts/{analyze.ts → static-analysis.ts} +58 -118
  37. package/skills/c64-testing/SKILL.md +48 -32
  38. package/skills/c64-testing/references/recording-format.md +48 -19
  39. package/skills/c64-testing/references/scenario-format.md +24 -19
  40. package/skills/c64-testing/scripts/{diff.ts → lib/diff-files.ts} +18 -69
  41. package/skills/c64-testing/scripts/lib/diff.ts +5 -3
  42. package/skills/c64-testing/scripts/lib/play.ts +244 -54
  43. package/skills/c64-testing/scripts/lib/recording.ts +153 -120
  44. package/skills/c64-testing/scripts/{replay.ts → lib/replay.ts} +28 -86
  45. package/skills/c64-testing/scripts/{run.ts → lib/runs.ts} +36 -59
  46. package/skills/c64-testing/scripts/lib/scenario.ts +73 -81
  47. package/skills/c64-testing/scripts/testing.ts +189 -0
  48. package/skills/c64-unpacker/SKILL.md +42 -25
  49. package/skills/c64-unpacker/scripts/lib/evidence.ts +3 -4
  50. package/skills/c64-unpacker/scripts/lib/loads.ts +12 -10
  51. package/skills/c64-unpacker/scripts/lib/machine.ts +0 -2
  52. package/skills/c64-unpacker/scripts/lib/stages.ts +14 -19
  53. package/skills/c64-unpacker/scripts/{unpack.ts → unpacker.ts} +109 -211
  54. package/src/c64.ts +35 -20
  55. package/src/cli/main.ts +2 -2
  56. package/src/errors.ts +38 -0
  57. package/src/graphics.ts +3 -1
  58. package/src/host/server.ts +3 -2
  59. package/src/host/vice/session.ts +4 -4
  60. package/src/host-client/entry-state.ts +106 -0
  61. package/src/host-client/image.ts +21 -10
  62. package/src/host-client/steps.ts +285 -108
  63. package/src/host-client/transfer.ts +21 -15
  64. package/src/host-client/vice-session.ts +1 -1
  65. package/src/knowledge/database.ts +3 -23
  66. package/src/knowledge/expression.ts +8 -11
  67. package/src/knowledge/history.ts +2 -2
  68. package/src/knowledge/import.ts +13 -14
  69. package/src/knowledge/names.ts +50 -0
  70. package/src/knowledge/program.ts +11 -10
  71. package/src/knowledge/read.ts +16 -21
  72. package/src/knowledge/schema.ts +9 -3
  73. package/src/knowledge/text.ts +5 -5
  74. package/src/knowledge/write.ts +93 -137
  75. package/src/mcp/inputs.ts +105 -0
  76. package/src/mcp/output.ts +2 -13
  77. package/src/mcp/server.ts +12 -60
  78. package/src/mcp/tools/debug.ts +17 -12
  79. package/src/mcp/tools/execution.ts +5 -32
  80. package/src/mcp/tools/input.ts +7 -32
  81. package/src/mcp/tools/media.ts +6 -5
  82. package/src/mcp/tools/memory.ts +11 -38
  83. package/src/mcp/tools/video.ts +6 -5
  84. package/src/native/c64run.ts +1 -1
  85. package/src/native/dxa.ts +7 -6
  86. package/src/native/ghidra/analyze.ts +2 -1
  87. package/src/native/ghidra/index.ts +2 -1
  88. package/src/native/processes.ts +4 -3
  89. package/src/native/status.ts +2 -2
  90. package/src/native/types.ts +3 -7
  91. package/src/opcodes.ts +13 -13
  92. package/src/platform.ts +9 -9
  93. package/src/project.ts +16 -26
  94. package/src/protocol/messages.ts +3 -21
  95. package/src/protocol/vice.ts +7 -7
  96. package/src/script.ts +491 -49
  97. package/skills/c64-assembler/scripts/assemble.ts +0 -138
  98. package/skills/c64-docs/scripts/generate.ts +0 -109
  99. package/skills/c64-docs/scripts/link.ts +0 -239
  100. package/skills/c64-memory-map/scripts/memmap.ts +0 -92
  101. package/skills/c64-testing/scripts/checklist.ts +0 -51
  102. package/skills/c64-testing/scripts/record.ts +0 -96
  103. package/skills/c64-testing/scripts/test.ts +0 -65
@@ -21,22 +21,20 @@ Find the static structure of a C64 program. Record the structure in the project
21
21
  The script reads the program, the coverage and the current knowledge, and runs DXA or Ghidra on this machine. The host runtime is not necessary. The script records the result in the project knowledge, `.c64-re-tools/knowledge/`.
22
22
 
23
23
  ```
24
- node <skill>/scripts/analyze.ts --analyzer dxa --listing .c64-re-tools/local/game.lst
25
- node <skill>/scripts/analyze.ts --entry '$080d' --decompile '$080d'
24
+ node <skill>/scripts/static-analysis.ts analyze --analyzer dxa --out-listing .c64-re-tools/local/game.lst
25
+ node <skill>/scripts/static-analysis.ts analyze --entry '$080d' --decompile '$080d'
26
26
  ```
27
27
 
28
28
  | Option | Meaning |
29
29
  |---|---|
30
30
  | `--analyzer dxa` | Use DXA: a fast first pass. Without this option, the script uses Ghidra. |
31
31
  | `--entry '$080d'` | An address where code starts. Use it again for more addresses. |
32
- | `--listing .c64-re-tools/local/game.lst` | DXA only: write the disassembly listing to this project file. |
32
+ | `--out-listing .c64-re-tools/local/game.lst` | DXA only: write the disassembly listing to this project file. |
33
33
  | `--decompile '$0818'` | Ghidra only: decompile the routine at this address. Use it again for more routines, at most 32. |
34
34
  | `--max-listed 20` | List at most this number of labels, functions or candidates. The default is 50. Use 0 to list none. |
35
- | `--instructions .c64-re-tools/local/instructions.json` | Ghidra only: write every instruction to this project file as JSON. Each has `address`, `bytes`, `mnemonic`, `mode` and `operand` in 6502 syntax. A branch, jump or call also has its `target`. When the knowledge has coverage, `executed` tells if it ran. |
35
+ | `--out-instructions .c64-re-tools/local/instructions.json` | Ghidra only: write every instruction to this project file as JSON. Each has `address`, `bytes`, `mnemonic`, `mode` and `operand` in 6502 syntax. A branch, jump or call also has its `target`. When the knowledge has coverage, `executed` tells if it ran. |
36
36
  | `--space drive8` | The knowledge space of the program: `drive8` for code from the 1541 drive, or `bank0` to `bank255` for a cartridge bank. The default is `c64`. The seeds come from that space, and the findings go into it. |
37
- | `--macro-batch .c64-re-tools/local/macros.json` | Write the macro candidates to this project file as a batch for `apply` of the c64-knowledge skill. |
38
-
39
- The script prints one JSON object. Addresses are `$` and four hex digits.
37
+ | `--out-macro-batch .c64-re-tools/local/macros.json` | Write the macro candidates to this project file as a batch for `apply` of the c64-knowledge skill. |
40
38
 
41
39
  Both analyzers use the knowledge from the user and the LLM as seeds:
42
40
 
@@ -51,17 +49,29 @@ More seeds come from evidence:
51
49
  - The start of each `code` region and each entry of a `jumps` table from the user or the LLM is an entry point.
52
50
  - When the knowledge has coverage, each executed instruction of the program that starts a flow is an entry point. A flow starts after a gap, or after RTS, RTI, JMP, BRK or JSR. Then the analyzers find code that only a vector, an RTS table or an interrupt reaches. They also find the code after a JSR to a routine that an analyzer thinks does not return.
53
51
  - For Ghidra, a flow start is a routine only when no executed branch or JMP from another flow gets to it. Also, no JSR can be immediately before it. Another flow start is code in a routine, and Ghidra does not make a function of it.
54
- - When the knowledge has coverage, Ghidra gives `coverageGaps`: the executed instruction starts in the program that are not in its instructions. Normally the count is 0. Each start in the list is missing from `--instructions` and from the source of c64-source. Examine the code at the start. Overlapped instructions and self-modified code can cause a gap. A start in a `dual-use` region is a known overlap, for example a BIT skip, and is not in the list.
52
+ - When the knowledge has coverage, Ghidra gives `coverageGaps`: the executed instruction starts in the program that are not in its instructions. Normally the count is 0. Each start in the list is missing from `--out-instructions` and from the source of c64-source. Examine the code at the start. Overlapped instructions and self-modified code can cause a gap. A start in a `dual-use` region is a known overlap, for example a BIT skip, and is not in the list.
55
53
  - When the knowledge has coverage, some RAM under a ROM is data for the analysis. At these addresses, the runs used the ROM and did not read or execute the RAM. Then a call into the KERNAL does not decode the RAM bytes under it as code. A knowledge routine or an `--entry` in such a range keeps its code. Ghidra does not follow BRK, because the vector at `$fffe` is in ROM unless the program removes the KERNAL.
56
54
 
57
55
  The analyzers take at most 1024 entry points. Knowledge routines and `--entry` come first, then code regions and tables, then coverage. Ghidra also takes at most 8192 flow starts that are code in a routine. `coverageSeeds` gives how many entry points the coverage gave and how many the analysis used. `romOnlyRanges` and `romOnlyUsed` give the same counts for the data ranges under ROM.
58
56
 
57
+ ### Calling the script
58
+
59
+ - Call a script as `node <skill>/scripts/<script>.ts <command> [subjects] [options]`. The command comes first.
60
+ - `<script>.ts --help` lists the commands. `<script>.ts <command> --help` gives the subjects and the options of one command. The help is JSON: `{"text": "..."}`.
61
+ - An address is `$` and 1 to 4 hex digits, for example `'$c000'` or `'$fb'`, or a name. A name is a symbol, else an analyzer name that only one analyzer gives. Quote an address in a shell.
62
+ - A range is `<start>-<end>`, for example `'$c000-$cfff'`. In a JSON file, a range is `start` and `end`.
63
+ - Bytes in a JSON file are hex in a field `data`, for example `"data": "a900"`.
64
+ - The result is one JSON object on one line. A file that the script writes shows as `{"out", "bytes", "sha256"}`. A list shows as `{"count", "listed"}`, and `rest` tells where to get the rest of a long list.
65
+ - An error is `{"error": {"code": "...", "message": "..."}}`. The message tells what is wrong and what to do.
66
+ - The exit status is 0 for done or PASS, 1 for FAIL, 2 for an error and 3 for INCONCLUSIVE.
67
+
59
68
  ### Project folder
60
69
 
61
70
  - Paths are relative to the project folder. The project folder is the nearest folder at or above the working directory that holds `.c64-re-tools/`.
62
71
  - If the working directory is not in a project and has projects below it, the script refuses and lists them. Then give `--project <folder>` to each script.
63
72
  - Write each file that you make for your work in `.c64-re-tools/local/` of the project. Git ignores that folder. Examples are results, batches, step files, runs, snapshots and memory dumps.
64
- - Move a file out of `.c64-re-tools/local/` only when it is the evidence for a finding. Move it before you give it to `--evidence` of c64-knowledge. `--evidence` refuses a file in `.c64-re-tools/local/`.
73
+ - Move a file out of `.c64-re-tools/local/` only when it is the evidence for a finding. Move it before you give it to `--evidence test:<file>` or `--evidence file:<file>` of c64-knowledge. These two refuse a file in `.c64-re-tools/local/`.
74
+ - Do not move a coverage map after `coverage` of c64-knowledge imported it. `--evidence coverage:<map>` names the map by its path at the import. The knowledge keeps the coverage and the SHA-256 of the map, so the evidence stays valid.
65
75
  - Put a run that is evidence in `runs/<name>/`, with its run file, its results and the snapshot files that it uses. Change the paths in the run file to the new paths. Then git keeps them, and another person can do the run again.
66
76
  - A shell redirect (`>`) is relative to the working directory, not to the project folder. From the folder above the project, write `<folder>/.c64-re-tools/local/result.json`.
67
77
 
@@ -80,7 +90,7 @@ The analyzers take at most 1024 entry points. Knowledge routines and `--entry` c
80
90
  - `idioms` tells what the script found by pattern: RTS dispatch tables (`rtsTables`), JMP tables (`jumpTables`) and BIT skips (`bitSkips`). It also finds the `CBM80` header of a cartridge bank at `$8000` (`cartridgeHeaders`). The two start vectors of the header become entry points. The script writes them as knowledge with the origin `heuristic`, and seeds their targets into the same analysis. `candidates` gives DOP padding (`dop`) and opcodes that the program wrote and executed (`selfModified`). For `selfModified`, the write and the execution must be in one coverage file. A coverage file does not record the order. Thus a candidate can also be code that ran first, and that the program then wrote over. To prevent this, save one coverage file for each part of the program, for example the intro and the game. A loader that copies relocated code is also such a part. Clear the memory map between the parts. Each list has a `count`. The script does not write the candidates. Look at them and decide.
81
91
  - `candidates.pointers` gives code that makes an address from immediates. A `vector` or a `zero-page pointer` candidate stores a low byte (`low`) and a high byte (`high`) to a vector or to a zero-page pair that `(zp),y` uses. Its `target` is in the program or has a platform name. A `sprite pointer` candidate stores an immediate (`at`) to a sprite pointer of the screen at $0400 in VIC bank 0. Its `target` (the value × 64) is in a `sprite` region. For each candidate that you confirm, write operand expressions with `operand` of the c64-knowledge skill, for example `operand '<low>' --lo <target name>` and `operand '<high>' --hi <target name>`. Then a rebuild can move the target.
82
92
  - `candidates.pitfalls` gives code that a C64 runs in a different way than it reads. `raster-bit-8`: the instruction at `at` reads $D011, and the instruction at `store` writes the value back. A read of $D011 gives bit 8 of the raster line in bit 7. Thus the write can move the raster interrupt to line 256 or later, and on PAL past line 311 the interrupt never comes. Examine each candidate. Where the code must not keep bit 7, add a comment and test the program.
83
- - `candidates.macros` gives runs of instructions that a macro writes as one line, for example `+inc16 ptr`. The script knows these 6502 idioms: `inc16`, `dec16`, `add16_8`, `add16`, `sub16`, `set16`, `mov16` and `wait_line`. It reads the code regions after the analysis. An arg has the name of a `user` or `llm` symbol or a platform name. Each candidate has `macro`, `start`, `end`, `args` and `text`. `known` counts the uses that the knowledge has. `leftOut` counts the candidates with a label inside, or with a patch or a use over them. The c64-source skill writes such a candidate as instructions. With `--macro-batch`, `batch` gives the file, the count of writes and the `apply` command. The batch holds the macros that the knowledge does not have, and the uses. When the knowledge has a macro with the same name and another body, `otherMacros` tells that the batch does not have its uses. The script does not write the candidates. Apply the batch, then generate the source again.
93
+ - `candidates.macros` gives runs of instructions that a macro writes as one line, for example `+inc16 ptr`. The script knows these 6502 idioms: `inc16`, `dec16`, `add16_8`, `add16`, `sub16`, `set16`, `mov16` and `wait_line`. It reads the code regions after the analysis. An arg has the name of a `user` or `llm` symbol or a platform name. Each candidate has `macro`, `start`, `end`, `args` and `text`. `known` counts the uses that the knowledge has. `leftOut` counts the candidates with a label inside, or with a patch or a use over them. The c64-source skill writes such a candidate as instructions. With `--out-macro-batch`, `batch` gives the file, the count of writes (`writes`) and the `apply` command. The batch holds the macros that the knowledge does not have, and the uses. When the knowledge has a macro with the same name and another body, `otherMacros` tells that the batch does not have its uses. The script does not write the candidates. Apply the batch, then generate the source again.
84
94
  - When the knowledge has coverage, a code finding replaces the DXA or Ghidra data where a run executed an instruction start. `coverageConflicts` lists the analyzer data regions where the coverage still shows executed code. Classify them as code with the c64-knowledge skill, or as `dual-use` when the bytes are also data. For many conflicts, write all classifications in one `apply` batch of the c64-knowledge skill.
85
95
  6. Give each important routine a name with the c64-knowledge skill. Then run the analysis again. The analyzers use the new names.
86
96
 
@@ -109,9 +119,9 @@ Do not store:
109
119
 
110
120
  ## Result
111
121
 
112
- The changes to the knowledge and the conflicts. For DXA, also the regions, the labels and the path of the listing. For Ghidra, also the routines and the decompiled code.
122
+ The changes to the knowledge and the conflicts. For DXA, also the regions, the labels and the file of the listing (`listing`). For Ghidra, also the routines, the decompiled code and the file of the instructions (`instructions`).
113
123
 
114
- The exit status is 0 for a result, also with conflicts. The exit status is 1 for an error. The exit status is 2 for a wrong argument.
124
+ The exit status is 0, also with conflicts.
115
125
 
116
126
  ## Failure and conflicts
117
127
 
@@ -133,4 +143,5 @@ The exit status is 0 for a result, also with conflicts. The exit status is 1 for
133
143
  - Runtime behavior → c64-emulator
134
144
  - Names and classifications → c64-knowledge
135
145
  - The meaning of a C64 hardware address → c64-memory-map
136
- - Source code for a rebuild → c64-assembler
146
+ - Source code from the knowledge → c64-source
147
+ - Build the source → c64-assembler
@@ -9,10 +9,9 @@ import { COVERAGE, coverageEntries, coverageSeedKinds, executedStarts } from "#s
9
9
  export { coverageEntries, coverageSeedKinds, executedStarts };
10
10
  import { FULL_COVERAGE } from "#src/host-client/coverage.ts";
11
11
  import { listRegions, type RegionRow } from "#src/knowledge/read.ts";
12
- import { CODE_REGION_TYPES, DEFAULT_SPACE, type Space } from "#src/knowledge/schema.ts";
12
+ import { CODE_REGION_TYPES, DEFAULT_SPACE, isAnalyzer, isSemantic, type Space } from "#src/knowledge/schema.ts";
13
13
 
14
14
  const FULL_MAP = FULL_COVERAGE;
15
- const ANALYZER_ORIGINS = new Set(["dxa", "ghidra", "heuristic"]);
16
15
 
17
16
  /** The byte at an address of the analyzed image, or undefined outside it. */
18
17
  export type ImageByte = (address: number) => number | undefined;
@@ -46,7 +45,7 @@ export function romOnlyRanges(map: Uint8Array, image: ImageByte): Array<{ start:
46
45
  /** Entry points from semantic knowledge beyond routines: the start of each code region, and each entry of a JMP table with its target. */
47
46
  export function codeEntries(db: DatabaseSync | undefined, image: ImageByte, space: Space = DEFAULT_SPACE): number[] {
48
47
  const entries: number[] = [];
49
- for (const region of listRegions(db, undefined, undefined, undefined, space).filter((row) => row.origin === "user" || row.origin === "llm")) {
48
+ for (const region of listRegions(db, undefined, undefined, undefined, space).filter((row) => isSemantic(row.origin))) {
50
49
  if (region.type === "code") entries.push(region.start);
51
50
  if (region.type !== "jumps") continue;
52
51
  for (let at = region.start; at + 2 <= region.end; at += 3) {
@@ -63,7 +62,7 @@ export function codeEntries(db: DatabaseSync | undefined, image: ImageByte, spac
63
62
  export function coverageConflicts(db: DatabaseSync | undefined, map: Uint8Array): Array<{ region: RegionRow; executed: number }> {
64
63
  const conflicts: Array<{ region: RegionRow; executed: number }> = [];
65
64
  for (const region of listRegions(db)) {
66
- if (!ANALYZER_ORIGINS.has(region.origin) || CODE_REGION_TYPES.includes(region.type) || region.type === "dual-use") continue;
65
+ if (!isAnalyzer(region.origin) || CODE_REGION_TYPES.includes(region.type) || region.type === "dual-use") continue;
67
66
  let executed = 0;
68
67
  for (let address = region.start; address <= region.end; address++) if ((map[address]! & COVERAGE.ramExecute) !== 0) executed++;
69
68
  if (executed > 0) conflicts.push({ region, executed });
@@ -5,12 +5,12 @@ import type { DatabaseSync } from "node:sqlite";
5
5
 
6
6
  import type { NormalizedFindings } from "#src/knowledge/import.ts";
7
7
  import { listRegions, listSymbols } from "#src/knowledge/read.ts";
8
- import { CODE_REGION_TYPES, DEFAULT_SPACE, type Space } from "#src/knowledge/schema.ts";
8
+ import { CODE_REGION_TYPES, DEFAULT_SPACE, isSemantic, type Space } from "#src/knowledge/schema.ts";
9
9
  import type { DxaResult, GhidraParams, GhidraResult } from "#src/native/types.ts";
10
10
 
11
11
  export type Seeds = Pick<GhidraParams, "entryPoints" | "dataRanges" | "labels">;
12
12
 
13
- const semantic = (row: { origin: string }) => row.origin === "user" || row.origin === "llm";
13
+ const semantic = (row: { origin: string }) => isSemantic(row.origin);
14
14
 
15
15
  /**
16
16
  * Seeds from current semantic knowledge: routines from the user or
@@ -86,7 +86,7 @@ export interface ListedInstruction {
86
86
  executed?: boolean;
87
87
  }
88
88
 
89
- /** Ghidra's instructions as the list that --instructions writes. */
89
+ /** Ghidra's instructions as the list that --out-instructions writes. */
90
90
  export function instructionList(instructions: GhidraInstruction[], coverage?: Uint8Array): ListedInstruction[] {
91
91
  return instructions.map((instruction) => {
92
92
  const bytes = Buffer.from(instruction.bytes, "hex");
@@ -8,7 +8,7 @@
8
8
  // known to be correct. A search for any repeated sequence was weighed and not
9
9
  // chosen: it gives many candidates without a name.
10
10
 
11
- import { instructionLength } from "#src/c64.ts";
11
+ import { formatC64Address, formatC64Byte, instructionLength } from "#src/c64.ts";
12
12
  import type { ImageByte } from "./coverage.ts";
13
13
 
14
14
  /** A macro of the catalogue: its params and its ACME body, which names a param with a dot. */
@@ -97,14 +97,12 @@ const CMP_ABSOLUTE = 0xcd;
97
97
  */
98
98
  export type Namer = (address: number) => string | undefined;
99
99
 
100
- const hex2 = (value: number) => `$${value.toString(16).padStart(2, "0")}`;
101
- const hex4 = (value: number) => `$${value.toString(16).padStart(4, "0")}`;
102
100
 
103
101
  function addressArg(operand: Operand, name: Namer): string {
104
102
  const named = name(operand.value);
105
103
  // A name of a zero-page address gives the zero-page form: an absolute operand there keeps its number.
106
104
  if (named !== undefined && (operand.value >= 0x100 || !operand.absolute)) return named;
107
- return operand.absolute ? hex4(operand.value) : hex2(operand.value);
105
+ return operand.absolute ? formatC64Address(operand.value) : formatC64Byte(operand.value);
108
106
  }
109
107
 
110
108
  /** The memory operand of `instruction` when it is one of the opcodes of the mnemonic. */
@@ -141,7 +139,7 @@ const MATCHERS: Record<string, { count: number; match: (run: Instruction[], name
141
139
  const dst = memory(load, "lda");
142
140
  const value = immediate(add, IMMEDIATE.adc);
143
141
  const ok = clc?.opcode === CLC && value !== undefined && same(dst, memory(store, "sta")) && same(dst, memory(high, "inc"), 1) && skips(branch, BCC, high);
144
- return ok ? [addressArg(dst!, name), hex2(value)] : undefined;
142
+ return ok ? [addressArg(dst!, name), formatC64Byte(value)] : undefined;
145
143
  },
146
144
  },
147
145
  add16: { count: 7, match: (run, name) => wide(run, name, CLC, IMMEDIATE.adc) },
@@ -154,7 +152,7 @@ const MATCHERS: Record<string, { count: number; match: (run: Instruction[], name
154
152
  const dst = memory(storeLow, "sta");
155
153
  if (low === undefined || high === undefined || !same(dst, memory(storeHigh, "sta"), 1)) return undefined;
156
154
  const value = low | (high << 8);
157
- return [addressArg(dst!, name), name(value) ?? hex4(value)];
155
+ return [addressArg(dst!, name), name(value) ?? formatC64Address(value)];
158
156
  },
159
157
  },
160
158
  mov16: {
@@ -171,7 +169,7 @@ const MATCHERS: Record<string, { count: number; match: (run: Instruction[], name
171
169
  match: ([load, compare, branch]) => {
172
170
  const line = immediate(load, IMMEDIATE.lda);
173
171
  // BNE back to the CMP: 5 bytes back from the end of the BNE.
174
- return line !== undefined && compare?.opcode === CMP_ABSOLUTE && compare.value === 0xd012 && branch?.opcode === BNE && branch.value === 0xfb ? [hex2(line)] : undefined;
172
+ return line !== undefined && compare?.opcode === CMP_ABSOLUTE && compare.value === 0xd012 && branch?.opcode === BNE && branch.value === 0xfb ? [formatC64Byte(line)] : undefined;
175
173
  },
176
174
  },
177
175
  };
@@ -183,7 +181,7 @@ function wide([carry, loadLow, low, storeLow, loadHigh, high, storeHigh]: Instru
183
181
  const highValue = immediate(high, opcode);
184
182
  const ok =
185
183
  carry?.opcode === carryOpcode && lowValue !== undefined && highValue !== undefined && same(dst, memory(storeLow, "sta")) && same(dst, memory(loadHigh, "lda"), 1) && same(dst, memory(storeHigh, "sta"), 1);
186
- return ok ? [addressArg(dst!, name), hex4(lowValue | (highValue << 8))] : undefined;
184
+ return ok ? [addressArg(dst!, name), formatC64Address(lowValue | (highValue << 8))] : undefined;
187
185
  }
188
186
 
189
187
  /** The instruction at `at`, or undefined when a byte of it is outside the range or the image. */
@@ -3,25 +3,22 @@
3
3
  // from current knowledge and the coverage of the knowledge, and imports the
4
4
  // structural findings into .c64-re-tools/knowledge/.
5
5
 
6
- import { mkdirSync, writeFileSync } from "node:fs";
7
- import { dirname } from "node:path";
8
- import { parseArgs } from "node:util";
9
-
10
- import { COVERAGE, formatC64Address, parseC64Address, prgRange } from "#src/c64.ts";
6
+ import { COVERAGE, formatC64Address, prgRange } from "#src/c64.ts";
7
+ import { Refusal } from "#src/errors.ts";
11
8
  import { WireFailure } from "#src/host-client/tools.ts";
12
- import { writeProjectFile } from "#src/host-client/transfer.ts";
9
+ import { writeReported } from "#src/host-client/transfer.ts";
10
+ import { knowledgeNames } from "#src/knowledge/names.ts";
13
11
  import { analyze as analyzeWithDxa } from "#src/native/dxa.ts";
14
12
  import { analyze as analyzeWithGhidra } from "#src/native/ghidra/analyze.ts";
15
13
  import { localToolContext } from "#src/native/local.ts";
16
14
  import { GHIDRA_LIMITS } from "#src/native/types.ts";
17
- import { KnowledgeError, openForRead, withWrite } from "#src/knowledge/database.ts";
15
+ import { openForRead, withWrite } from "#src/knowledge/database.ts";
18
16
  import { analyzedBytes, programCoverage, requireProgram } from "#src/knowledge/program.ts";
19
- import { importFindings, runProvenance, type Analyzer, type ImportConflict, type ImportContext, type ImportResult, type NormalizedFindings } from "#src/knowledge/import.ts";
17
+ import { importFindings, runProvenance, type ImportConflict, type ImportContext, type ImportResult, type NormalizedFindings } from "#src/knowledge/import.ts";
20
18
  import { currentRevision, listMacros, listMacroUses, listPatches, listReferences, listRegions, listSymbols } from "#src/knowledge/read.ts";
21
- import { DEFAULT_SPACE, SPACE } from "#src/knowledge/schema.ts";
19
+ import { DEFAULT_SPACE, SEMANTIC_ORIGINS, type Analyzer } from "#src/knowledge/schema.ts";
22
20
  import { platformNames } from "#src/platform.ts";
23
- import { resolveProjectPath } from "#src/project.ts";
24
- import { runScript, UsageError } from "#src/script.ts";
21
+ import { listed, OPTIONS, runScript, UsageError, type Call } from "#src/script.ts";
25
22
  import { codeEntries, coverageConflicts, coverageEntries, coverageSeedKinds, executedStarts, imageReader, romOnlyRanges, type ImageByte } from "./lib/coverage.ts";
26
23
  import { dxaFindings, ghidraFindings, seedsFromKnowledge } from "./lib/findings.ts";
27
24
  import { findIdioms } from "./lib/idioms.ts";
@@ -30,37 +27,6 @@ import { CATALOGUE, findMacroUses, type MacroCandidate } from "./lib/macros.ts";
30
27
  import { findPitfalls, type Pitfall } from "./lib/pitfalls.ts";
31
28
  import { findPointers, type PointerCandidate } from "./lib/pointers.ts";
32
29
 
33
- const USAGE = `analyze.ts [options]
34
-
35
- The program and the coverage come from the knowledge: put them in with
36
- knowledge.ts image and coverage of the c64-knowledge skill. The executed code of
37
- the coverage is seeded, and analyzer data over it is reported.
38
-
39
- --analyzer <name> dxa (fast first pass) or ghidra (deeper, the default)
40
- --entry <address> an entry point, for example '$080d' (repeatable)
41
- --decompile <address> ghidra: a routine to decompile, at most 32 (repeatable)
42
- --listing <file> dxa: write the disassembly listing to this project file
43
- --instructions <file> ghidra: write every instruction (address, bytes, mnemonic,
44
- mode, operand, target, executed) to this project file as JSON
45
- --max-listed <n> list at most n labels, functions or candidates (default 50,
46
- 0 for none); the counts are always given, and knowledge has
47
- all labels and functions
48
- --space <space> the knowledge space of the program: c64 (the default), drive8
49
- for 1541 drive code, or bank0 to bank255 for a cartridge bank
50
- --macro-batch <file> write the macro candidates as an apply batch of c64-knowledge
51
- (the macros and their uses) to this project file
52
-
53
- Routine symbols in knowledge are entry points too. Ghidra needs at least one
54
- entry point; dxa finds the SYS of a BASIC start by itself. The result is one
55
- JSON object. Addresses are $ and four hex digits.`;
56
-
57
- const DEFAULT_MAX_LISTED = 50;
58
-
59
- /** The first `max` items, the count, and where to read the rest when the list is cut. */
60
- function listed<T, U>(items: T[], max: number, show: (item: T) => U, all: string) {
61
- return { count: items.length, listed: items.slice(0, max).map(show), ...(items.length > max ? { rest: all } : {}) };
62
- }
63
-
64
30
  /** A pitfall with $ addresses and what goes wrong. */
65
31
  function showPitfall(pitfall: Pitfall) {
66
32
  return {
@@ -73,11 +39,10 @@ function showPitfall(pitfall: Pitfall) {
73
39
 
74
40
  /** A pointer candidate with $ addresses, and the operand writes that would name it. */
75
41
  function showPointer(candidate: PointerCandidate) {
76
- const hex = formatC64Address;
77
42
  if (candidate.kind === "sprite pointer") {
78
- return { kind: candidate.kind, at: hex(candidate.at), to: hex(candidate.to), value: `$${candidate.value.toString(16).padStart(2, "0")}`, target: hex(candidate.target), assumes: "VIC bank 0 and the screen at $0400" };
43
+ return { kind: candidate.kind, at: formatC64Address(candidate.at), to: formatC64Address(candidate.to), value: `$${candidate.value.toString(16).padStart(2, "0")}`, target: formatC64Address(candidate.target), assumes: "VIC bank 0 and the screen at $0400" };
79
44
  }
80
- return { kind: candidate.kind, to: hex(candidate.to), target: hex(candidate.target), low: hex(candidate.low), high: hex(candidate.high) };
45
+ return { kind: candidate.kind, to: formatC64Address(candidate.to), target: formatC64Address(candidate.target), low: formatC64Address(candidate.low), high: formatC64Address(candidate.high) };
81
46
  }
82
47
 
83
48
  /** A macro candidate with $ addresses and its source line. */
@@ -98,7 +63,7 @@ function macroCandidates(space: string, byteAt: ImageByte, batchPath: string | u
98
63
  const db = openForRead();
99
64
  try {
100
65
  const platform = space === DEFAULT_SPACE ? new Map(platformNames().map((entry) => [entry.address, entry.name])) : new Map<number, string>();
101
- const semantic = new Map(listSymbols(db, undefined, undefined, ["user", "llm"], space).map((symbol) => [symbol.address, symbol.name]));
66
+ const semantic = new Map(listSymbols(db, undefined, undefined, [...SEMANTIC_ORIGINS], space).map((symbol) => [symbol.address, symbol.name]));
102
67
  const code = listRegions(db, undefined, "code", undefined, space);
103
68
  const found = findMacroUses(byteAt, code, (address) => semantic.get(address) ?? platform.get(address));
104
69
  const named = new Set([...listSymbols(db, undefined, undefined, undefined, space).map((symbol) => symbol.address), ...listReferences(db, undefined, undefined, undefined, space).map((reference) => reference.to)]);
@@ -130,13 +95,13 @@ function macroCandidates(space: string, byteAt: ImageByte, batchPath: string | u
130
95
  batched.push(candidate);
131
96
  batch.push({ op: "use", start: formatC64Address(candidate.start), end: formatC64Address(candidate.end), macro: candidate.macro, args: candidate.args, ...(space === DEFAULT_SPACE ? {} : { space }) });
132
97
  }
133
- if (batch.length > 0) writeProjectFile(batchPath, Buffer.from(`${JSON.stringify(batch, null, 1)}\n`), { what: "--macro-batch" });
98
+ const written = batch.length > 0 ? writeReported(batchPath, `${JSON.stringify(batch, null, 1)}\n`, { what: "--out-macro-batch" }) : { out: batchPath };
134
99
  return {
135
100
  candidates,
136
101
  known,
137
102
  leftOut,
138
103
  batch: {
139
- out: batchPath,
104
+ ...written,
140
105
  writes: batch.length,
141
106
  ...(other.size === 0 ? {} : { otherMacros: [...other].map((name) => `The knowledge has another macro ${name}: its candidates are not in the batch.`) }),
142
107
  ...(batch.length === 0 ? {} : { apply: `knowledge.ts apply ${batchPath} --reason "macro uses of 6502 idioms"` }),
@@ -154,16 +119,6 @@ function withRemedy(reason: string): string {
154
119
  : reason;
155
120
  }
156
121
 
157
- function addresses(values: string[] | undefined, option: string): number[] {
158
- return (values ?? []).map((value) => {
159
- try {
160
- return parseC64Address(value);
161
- } catch {
162
- throw new UsageError(`${option} must be $ followed by four hex digits, not ${value}`);
163
- }
164
- });
165
- }
166
-
167
122
  /** One conflict: what the knowledge has, and what the analyzer found, under the analyzer's name. */
168
123
  function showConflict(conflict: ImportConflict, analyzer: Analyzer): Record<string, unknown> {
169
124
  if (conflict.category === "symbol") {
@@ -192,8 +147,8 @@ function record(findings: NormalizedFindings, revision: string, description: str
192
147
  try {
193
148
  imported = withWrite((db) => importFindings(db, findings, { expectedRevision: revision, description, ...provenance }));
194
149
  } catch (error) {
195
- if (error instanceof KnowledgeError && error.code === "stale-revision") {
196
- throw new KnowledgeError("stale-revision", "The knowledge changed during the analysis. Nothing was imported. Run the analysis again.");
150
+ if (error instanceof Refusal && error.code === "stale-revision") {
151
+ throw new Refusal("stale-revision", "The knowledge changed during the analysis. Nothing was imported. Run the analysis again.");
197
152
  }
198
153
  throw error;
199
154
  }
@@ -204,55 +159,19 @@ function record(findings: NormalizedFindings, revision: string, description: str
204
159
  };
205
160
  }
206
161
 
207
- async function run(argv: string[]): Promise<unknown> {
208
- const { values, positionals } = parseArgs({
209
- args: argv,
210
- strict: true,
211
- allowPositionals: true,
212
- options: {
213
- analyzer: { type: "string" },
214
- entry: { type: "string", multiple: true },
215
- decompile: { type: "string", multiple: true },
216
- listing: { type: "string" },
217
- "max-listed": { type: "string" },
218
- instructions: { type: "string" },
219
- space: { type: "string" },
220
- "macro-batch": { type: "string" },
221
- help: { type: "boolean" },
222
- },
223
- });
224
- if (values.help) throw new UsageError("");
225
- if (positionals.length !== 0) throw new UsageError("analyze takes no image: it reads the program from the knowledge (knowledge.ts image of the c64-knowledge skill puts it in)");
226
- const entries = addresses(values.entry, "--entry");
227
- const decompile = addresses(values.decompile, "--decompile");
162
+ async function analyze(call: Call): Promise<unknown> {
163
+ const entries = call.addresses("entry");
164
+ const decompile = call.addresses("decompile");
228
165
  if (decompile.length > 32) throw new UsageError("--decompile takes at most 32 routines");
229
- const maxListed = values["max-listed"] ?? String(DEFAULT_MAX_LISTED);
230
- if (!/^\d{1,5}$/.test(maxListed)) throw new UsageError(`--max-listed must be a whole number, not ${maxListed}`);
231
- const max = Number(maxListed);
232
- const analyzer = values.analyzer ?? "ghidra";
233
- if (analyzer !== "dxa" && analyzer !== "ghidra") throw new UsageError(`--analyzer must be dxa or ghidra, not ${analyzer}`);
166
+ const max = call.count("max-listed")!;
167
+ const analyzer = call.string("analyzer") as "dxa" | "ghidra";
168
+ const listingPath = call.string("out-listing");
169
+ const instructionsPath = call.string("out-instructions");
170
+ const batchPath = call.string("out-macro-batch");
234
171
  if (analyzer === "dxa" && decompile.length > 0) throw new UsageError("--decompile works only with --analyzer ghidra");
235
- if (analyzer === "ghidra" && values.listing !== undefined) throw new UsageError("--listing works only with --analyzer dxa");
236
- if (analyzer === "dxa" && values.instructions !== undefined) throw new UsageError("--instructions works only with --analyzer ghidra");
237
- let listingPath: string | undefined;
238
- if (values.listing !== undefined) {
239
- try {
240
- listingPath = resolveProjectPath(values.listing);
241
- } catch {
242
- throw new UsageError(`--listing must be a path relative to the project folder that stays inside it, not ${values.listing}`);
243
- }
244
- }
245
-
246
- if (values["macro-batch"] !== undefined) {
247
- try {
248
- resolveProjectPath(values["macro-batch"]);
249
- } catch {
250
- throw new UsageError(`--macro-batch must be a path relative to the project folder that stays inside it, not ${values["macro-batch"]}`);
251
- }
252
- }
253
-
254
- const space = values.space ?? DEFAULT_SPACE;
255
- if (!SPACE.test(space)) throw new UsageError("--space must be c64, drive8, or bank0 to bank255");
172
+ if (analyzer === "ghidra" && listingPath !== undefined) throw new UsageError("--out-listing works only with --analyzer dxa");
173
+ if (analyzer === "dxa" && instructionsPath !== undefined) throw new UsageError("--out-instructions works only with --analyzer ghidra");
174
+ const space = call.space;
256
175
  const reader = openForRead();
257
176
  const program = requireProgram(reader, space);
258
177
  // A coverage map is of the C64 CPU: VICE keeps none for the drive.
@@ -296,7 +215,7 @@ async function run(argv: string[]): Promise<unknown> {
296
215
  const idiomsResult = () => {
297
216
  const imported = withWrite((db) => importFindings(db, { ...idioms.findings, space }, { description: "Idioms in the program" }));
298
217
  // The macro candidates read the code regions after both imports.
299
- const macros = macroCandidates(space, byteAt, values["macro-batch"]);
218
+ const macros = macroCandidates(space, byteAt, batchPath);
300
219
  // Platform pitfalls are C64 facts: drive code and banks get none.
301
220
  const pitfallDb = openForRead();
302
221
  const pitfalls = space === DEFAULT_SPACE ? findPitfalls(byteAt, listRegions(pitfallDb, undefined, "code", undefined, space)) : [];
@@ -314,7 +233,7 @@ async function run(argv: string[]): Promise<unknown> {
314
233
  pointers: listed(pointers, max, showPointer, "a larger --max-listed"),
315
234
  pitfalls: listed(pitfalls, max, showPitfall, "a larger --max-listed"),
316
235
  macros: {
317
- ...listed(macros.candidates, max, showMacro, "a larger --max-listed, or --macro-batch <file>"),
236
+ ...listed(macros.candidates, max, showMacro, "a larger --max-listed, or --out-macro-batch <file>"),
318
237
  ...(macros.known === 0 ? {} : { known: macros.known }),
319
238
  ...(macros.leftOut === 0 ? {} : { leftOut: macros.leftOut }),
320
239
  ...(macros.batch === undefined ? {} : { batch: macros.batch }),
@@ -344,13 +263,13 @@ async function run(argv: string[]): Promise<unknown> {
344
263
  const listing = attachments[0]!.toString("utf8");
345
264
  // The listing is written only after the import: a refused import leaves no new file.
346
265
  const imported = record({ ...dxaFindings(result), space, ...executed }, revision, "DXA analysis of the program", runProvenance(bytes, result.toolVersion));
266
+ let written: ReturnType<typeof writeReported> | undefined;
347
267
  if (listingPath !== undefined) {
348
268
  try {
349
- mkdirSync(dirname(listingPath), { recursive: true });
350
- writeFileSync(listingPath, listing);
269
+ written = writeReported(listingPath, listing, { what: "--out-listing" });
351
270
  } catch (error) {
352
271
  const recorded = imported.revision === null ? "The knowledge did not change." : `The knowledge has the findings as revision ${imported.revision}.`;
353
- throw new WireFailure("operation-failed", `The script did not write the listing to ${values.listing}: ${(error as Error).message}. ${recorded}`);
272
+ throw new WireFailure("operation-failed", `The script did not write the listing to ${listingPath}: ${(error as Error).message}. ${recorded}`);
354
273
  }
355
274
  }
356
275
  const found = idiomsResult();
@@ -363,7 +282,7 @@ async function run(argv: string[]): Promise<unknown> {
363
282
  ...imported,
364
283
  regions: result.regions.map((region) => ({ start: formatC64Address(region.start), end: formatC64Address(region.end), classification: region.classification })),
365
284
  labels: listed(result.labels, max, (label) => ({ address: formatC64Address(label.address), name: label.name }), "c64-knowledge: knowledge.ts symbols"),
366
- ...(listingPath === undefined ? {} : { listing: values.listing }),
285
+ ...(written === undefined ? {} : { listing: written }),
367
286
  };
368
287
  }
369
288
 
@@ -378,11 +297,10 @@ async function run(argv: string[]): Promise<unknown> {
378
297
  const overlaps = listRegions(overlapDb, undefined, "dual-use", undefined, space);
379
298
  overlapDb?.close();
380
299
  const gaps = coverage === undefined ? [] : executedStarts(coverage, inProgram).filter((address) => address >= loaded.start && address <= loaded.end && !decoded.has(address) && !overlaps.some((region) => address >= region.start && address <= region.end));
381
- let instructions: { out: string; count: number } | undefined;
382
- if (values.instructions !== undefined) {
300
+ let instructions: Record<string, unknown> | undefined;
301
+ if (instructionsPath !== undefined) {
383
302
  const list = instructionList(result.instructions, coverage);
384
- writeProjectFile(values.instructions, Buffer.from(`${JSON.stringify(list, null, 1)}\n`), { what: "--instructions" });
385
- instructions = { out: values.instructions, count: list.length };
303
+ instructions = { count: list.length, ...writeReported(instructionsPath, `${JSON.stringify(list, null, 1)}\n`, { what: "--out-instructions" }) };
386
304
  }
387
305
  return {
388
306
  analyzer: "ghidra",
@@ -401,4 +319,26 @@ async function run(argv: string[]): Promise<unknown> {
401
319
  };
402
320
  }
403
321
 
404
- await runScript((argv) => run(argv), { usage: USAGE, refusals: [KnowledgeError] });
322
+ await runScript({
323
+ name: "static-analysis",
324
+ summary: "Analyzes the program in the knowledge with DXA or Ghidra, seeded from the knowledge and its coverage, and imports the structural findings into the knowledge.",
325
+ names: knowledgeNames,
326
+ commands: [
327
+ {
328
+ name: "analyze",
329
+ summary: "Runs the analyzer over the program in the knowledge and imports what it finds as one revision, then the 6502 idioms as a second one.",
330
+ options: {
331
+ analyzer: { kind: "choice", value: "<dxa|ghidra>", choices: ["dxa", "ghidra"], default: "ghidra", help: "dxa (a fast first pass) or ghidra (deeper, the default)" },
332
+ entry: { ...OPTIONS.entry, multiple: true, help: "an entry point of the program" },
333
+ decompile: { kind: "address", value: "<address>", multiple: true, help: "ghidra: a routine to decompile, at most 32" },
334
+ space: OPTIONS.space,
335
+ "max-listed": { ...OPTIONS["max-listed"], help: "list at most n labels, functions or candidates (default 50, 0 for none). The counts are always given, and the knowledge has all labels and functions" },
336
+ "out-listing": { kind: "path", value: "<file>", help: "dxa: write the disassembly listing to this project file" },
337
+ "out-instructions": { kind: "path", value: "<file>", help: "ghidra: write every instruction (address, bytes, mnemonic, mode, operand, target, executed) to this project file as JSON" },
338
+ "out-macro-batch": { kind: "path", value: "<file>", help: "write the macro candidates as an apply batch of c64-knowledge (the macros and their uses) to this project file" },
339
+ },
340
+ notes: "The program and the coverage come from the knowledge: put them in with knowledge.ts image and coverage of the c64-knowledge skill. The executed code of the coverage is seeded. Routine symbols in the knowledge are entry points too. Ghidra needs at least one entry point. DXA finds the SYS of a BASIC start by itself.",
341
+ run: analyze,
342
+ },
343
+ ],
344
+ });