@henols/c64-re-tools 2.0.1 → 2.1.0-rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -1,5 +1,8 @@
1
1
  # c64-re-tools
2
2
 
3
+ <img src="assets/branding/c64-re-tools-transparent.svg" alt="C64 RE Tools" width="440">
4
+
5
+
3
6
  Let an AI agent work on Commodore 64 programs. The agent drives a real VICE
4
7
  emulator and runs ACME, DXA and Ghidra. Use it to build new C64 apps and games,
5
8
  and to reverse engineer old ones.
@@ -51,7 +54,7 @@ npx -y --package=@henols/c64-re-tools@latest c64-re-tools-host
51
54
 
52
55
  ### Dev container
53
56
 
54
- This is the preferred way. The container gets Node.js 24, ACME, dxa, Java 21 and Ghidra,
57
+ This is the preferred way. The container gets Node.js 24, ACME, dxa, Java 21, Ghidra and c64run,
55
58
  installs the toolkit into the project when it is created, and updates it at
56
59
  each start. Add to `.devcontainer/devcontainer.json`:
57
60
 
@@ -67,7 +70,7 @@ needs a Debian or Ubuntu image. `runArgs` is for Docker on Linux.
67
70
 
68
71
  ### Without a container
69
72
 
70
- Install ACME, dxa and Ghidra as in [docs/install-tools.md](docs/install-tools.md).
73
+ Install ACME, dxa, Ghidra and c64run as in [docs/install-tools.md](docs/install-tools.md).
71
74
  Then, in your project:
72
75
 
73
76
  ```
@@ -119,7 +122,7 @@ program its own folder. A folder with `.c64-re-tools/` is a project.
119
122
  | c64-unpacker | Packing evidence, and capture of the unpacked program. |
120
123
  | c64-static-analysis | Analysis with DXA and Ghidra into the knowledge. |
121
124
  | c64-knowledge | Symbols, regions, comments and references, with history. |
122
- | c64-provenance | Compares releases, and keeps a library of shared routines. |
125
+ | c64-provenance | Compares releases, finds leftovers of the development and of cracks, and keeps a library of shared routines. |
123
126
  | c64-source | ACME source from the knowledge, checked byte for byte. |
124
127
  | c64-docs | Documentation of a program from its knowledge and source. |
125
128
  | c64-reverse-engineering | Leads an investigation and picks the next skill. |
@@ -129,7 +132,8 @@ program its own folder. A folder with `.c64-re-tools/` is a project.
129
132
  | Variable | Meaning |
130
133
  | --- | --- |
131
134
  | `C64RT_VICE`, `C64RT_C1541`, `C64RT_PETCAT` | Full path of the VICE tool, when it is not on `PATH` (host). |
132
- | `C64RT_ACME`, `C64RT_DXA` | Full path of the tool, when it is not on `PATH`. |
135
+ | `C64RT_ACME`, `C64RT_DXA`, `C64RT_C64RUN` | Full path of the tool, when it is not on `PATH`. |
136
+ | `C64RT_C64_ROMS` | The C64 ROM folder of c64run, when it is not `share/c64-re-tools/roms` next to its `bin`. |
133
137
  | `C64RT_GHIDRA` | The Ghidra installation directory, when `analyzeHeadless` is not on `PATH`. |
134
138
  | `C64RT_HOST` | `host:port` of the Host Runtime, when it is not on 127.0.0.1, `host.docker.internal` or `host.containers.internal` at port 6464. |
135
139
  | `C64RT_VIDEO` | `pal` (default) or `ntsc`. |
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@henols/c64-re-tools",
3
- "version": "2.0.1",
3
+ "version": "2.1.0-rc.1",
4
4
  "description": "Agent-friendly Commodore 64 engineering toolkit for reverse engineering, debugging, development and testing.",
5
5
  "license": "MIT",
6
6
  "author": "Henrik Olsson",
@@ -1,6 +1,6 @@
1
1
  ---
2
2
  name: c64-provenance
3
- description: Use this skill to compare two or more releases of a C64 program, for example cracked versions from different groups. It finds the bytes where the releases differ and shows which releases agree there. It can run each release and tell how the program uses each difference. It also gives evidence for each range of the program in the knowledge. The evidence is the unpack stage, the access, and the code that the entry does not reach. Use the result to separate the content of the application from changes that a release added. It also keeps a library of routines that several programs share. It finds them in the program of the knowledge and writes their names as a batch for c64-knowledge. Do not use it to analyze the code.
3
+ description: Use this skill to compare two or more releases of a C64 program, for example cracked versions from different groups. It finds the bytes where the releases differ and shows which releases agree there. It can run each release and tell how the program uses each difference. It also gives evidence for each range of the program in the knowledge. The evidence is the unpack stage, the access, and the code that the entry does not reach. It also finds leftovers in one program: texts, assembler source, BASIC lines, fills, unused bytes, copies and dead code. It finds text of other computers, for example Apple II source. Leftovers tell about the development, the cracks and the recracks. Use the result to separate the content of the application from changes that a release added. It also keeps a library of routines that several programs share. It finds them in the program of the knowledge and writes their names as a batch for c64-knowledge. Do not use it to analyze the code.
4
4
  ---
5
5
 
6
6
  # c64-provenance
@@ -9,6 +9,8 @@ description: Use this skill to compare two or more releases of a C64 program, fo
9
9
 
10
10
  Compare releases of a C64 program. Find the bytes where they differ and which releases agree. Use this to find which bytes are probably the original application and which are changes of one release.
11
11
 
12
+ Find the leftovers in one program. A leftover is a part of the program that the program does not need. Examples are a crack text, an old copy of a routine and a piece of assembler source.
13
+
12
14
  Keep a library of routines that several programs share, for example a music driver or a decruncher. Find the routines of the library in a program, and get their names for that program.
13
15
 
14
16
  This skill does not change project knowledge.
@@ -31,6 +33,7 @@ node <skill>/scripts/provenance.ts releases/a.prg releases/b.prg releases/c.prg
31
33
  node <skill>/scripts/provenance.ts releases/a.prg releases/b.prg --shift 2:-256 --from '$0900' --to '$09ff'
32
34
  node <skill>/scripts/provenance.ts releases/a.prg .c64-re-tools/local/b/image.json --run 600 --steps .c64-re-tools/local/start-game.json
33
35
  node <skill>/scripts/provenance.ts separate --out .c64-re-tools/local/separate.json
36
+ node <skill>/scripts/provenance.ts leftovers --out .c64-re-tools/local/leftovers.json
34
37
  ```
35
38
 
36
39
  | Option | Meaning |
@@ -87,6 +90,48 @@ Use `separate` when you have only one release. It reads the program, its stages
87
90
 
88
91
  Each list has `count` and `listed`. `listed` has the first ranges, at most `--max-listed`. `--out` writes all of them. Run more coverage runs for parts that are `untouched`, and add them with `coverage` of the c64-knowledge skill, before you decide.
89
92
 
93
+ ## Leftovers
94
+
95
+ Use `leftovers` to find what the program holds in addition to its working code. It reads the program, its stages, the coverage and the code regions from the knowledge, as `separate` does. It gives evidence, not a decision. Each list has `count` and `listed`, as in `separate`. `--out` writes all findings.
96
+
97
+ | List | What it holds |
98
+ |---|---|
99
+ | `text` | Text, and lines of assembler source. `encoding` is `petscii`, `screen`, `ascii` or `high-bit`. `reversed: true` marks screen codes with bit 7 set. `source: true` marks lines of assembler source. A line end in `text` is a new line. `mnemonics` counts 6502 instructions in the text. |
100
+ | `basic` | Two or more BASIC lines away from the BASIC start. `linkedAt` is the address where BASIC linked the lines. It can be another address than the address of the bytes now. |
101
+ | `fill` | One byte that repeats, at least 16 times, or a pattern of two to four bytes that repeats, at least 32 bytes in total. `pattern` is the repeated bytes. |
102
+ | `residue` | Bytes that no run touched and that are no text, BASIC, fill or code region. `bytes` shows the first 16 bytes. |
103
+ | `copies` | Bytes that are the same as earlier bytes of the program, at least 32 bytes. `copyOf` is the address of the earlier bytes. |
104
+ | `deadCode` | Code regions of the knowledge that no call or jump reaches from the entry, and that no run executed. |
105
+
106
+ Text from another computer can show the tools of the developer:
107
+
108
+ - `ascii` is text in ASCII, with small letters. A cross assembler on another computer can leave it.
109
+ - `high-bit` is text with bit 7 set. The Apple II keeps text in this form, with `$8d` at the end of a line. Apple II assembler source, for example from Merlin, has `DFB` lines and `*` comments. A C64 text in shifted letters with `$a0` spaces is also `high-bit`.
110
+ - Data of another computer, for example a table of Atari key codes, is not text. The script does not find it. Examine `residue` and the data that no code reads, and use your knowledge of the other computers.
111
+
112
+ Each finding has `stages`: the unpack stages that wrote its bytes. Each finding also has `access`: the accesses of its bytes in the coverage. A stage before the last stage is a depacker, an intro or a loader.
113
+
114
+ | Option | Meaning |
115
+ |---|---|
116
+ | `--min-text 6` | The minimum number of characters in a text. The default is 8. A short text, for example a group name, can need 6. |
117
+
118
+ Do these steps:
119
+
120
+ 1. Unpack the program with `stages` or `capture` of the c64-unpacker skill. Put the capture into the knowledge.
121
+ 2. Add the coverage of runs that use many parts of the program. Without coverage, `residue` is empty and no finding has `access`. `noCoverage` tells this.
122
+ 3. Run c64-static-analysis. Without code regions, `deadCode` is empty, and `residue` can include code.
123
+ 4. Run `leftovers`. When you have two or more releases, also compare them.
124
+ 5. Read `text` first: names, dates, greetings and credits. Then read `copies`, `residue`, `basic` and `deadCode`.
125
+ 6. Examine each important finding with the c64-emulator skill or the c64-static-analysis skill.
126
+ 7. Record each confirmed finding with the c64-knowledge skill. Classify a confirmed PETSCII text as a `petscii` region, and a screen code text as a `screen` region. Classify other text as `bytes`. Write a comment that tells what the leftover is.
127
+
128
+ Read the findings with care:
129
+
130
+ - A text is a run of bytes that looks like text. Some runs are data, for example a character map. A text can include one byte before it or after it that is not part of the text.
131
+ - A copy is not always a leftover. A program can copy its code to another address before it runs the code.
132
+ - A text in an early stage is often a crack intro. A text that no run read can be an old text of an earlier release.
133
+ - Keep the fact apart from the conclusion. "Text at $4770" is a fact. "A recrack by group X" is a conclusion.
134
+
90
135
  ## Library of routines
91
136
 
92
137
  A library is a folder of routine files. Each file holds one routine: its bytes and its knowledge. Keep one library beside the project folders, for example `library/` at the root of the workspace. A library is not a project folder. Commit it with the projects.
@@ -128,7 +173,7 @@ To find the routines in a program:
128
173
  Automatic write:
129
174
  - none.
130
175
 
131
- Record a confirmed conclusion as a normal comment, region or symbol with the c64-knowledge skill. Do not record the full comparison.
176
+ Record a confirmed conclusion as a normal comment, region or symbol with the c64-knowledge skill. Do not record the full comparison or the full list of leftovers.
132
177
 
133
178
  `match --out` writes a batch file and does not change the knowledge. Apply the batch with the c64-knowledge skill.
134
179
 
@@ -136,7 +181,9 @@ Record a confirmed conclusion as a normal comment, region or symbol with the c64
136
181
 
137
182
  The differing ranges with the bytes of each release and the groups that agree, and the share of identical bytes. `common` gives the ranges where all releases have bytes. `releases` gives the segments and the `shift` of each release. With `--run`, each difference also has `runtime`, and `run` tells if each release started, or why not.
138
183
 
139
- The runtime evidence is evidence only. The script does not decide which bytes are original.
184
+ For `leftovers`: the lists of findings, with `entry` and the number of `stages`. `noEntry`, `noStages` and `noCoverage` tell which knowledge is missing.
185
+
186
+ The runtime evidence and the leftovers are evidence only. The script does not decide which bytes are original.
140
187
 
141
188
  The exit status is 0 for a result. The exit status is 1 for an error. The exit status is 2 for a wrong argument.
142
189
 
@@ -0,0 +1,443 @@
1
+ // Leftovers in one program for the c64-provenance script: text in PETSCII,
2
+ // screen codes, ASCII or text with bit 7 set (Apple II), assembler source
3
+ // lines, BASIC lines away from the BASIC start, fills, untouched
4
+ // bytes outside code regions that no other finding explains, exact copies of
5
+ // a byte range, and code that no call or jump reaches and no run executed. Each finding gets the stages and the accesses of
6
+ // its bytes from the separation. Evidence only: nothing here decides what a
7
+ // leftover is or who made it.
8
+
9
+ import { basicLineText } from "#src/c64.ts";
10
+ import { MNEMONICS } from "#src/opcodes.ts";
11
+ import type { Range } from "./compare.ts";
12
+ import type { Access, Separation } from "./separate.ts";
13
+
14
+ export interface LeftoversInput {
15
+ segments: Range[];
16
+ /** 64 KiB with the program bytes at their addresses. */
17
+ memory: Uint8Array;
18
+ separation: Separation;
19
+ /** The first address of the BASIC start, which RUN lists: it is no leftover. */
20
+ basicStart?: number;
21
+ /** The fewest characters of a text (default 8). */
22
+ minText?: number;
23
+ }
24
+
25
+ /** What the separation tells about the bytes of a finding. */
26
+ export interface Placement {
27
+ /** The stages that wrote bytes of the finding, absent when no stage did. */
28
+ stages?: number[];
29
+ /** The accesses of its bytes in the coverage, absent without coverage. */
30
+ access?: Access[];
31
+ }
32
+
33
+ /** high-bit is text with bit 7 set: ASCII as the Apple II keeps it, or C64 shifted letters with $a0 spaces. Its spaces can also be $20. */
34
+ export type TextEncoding = "petscii" | "screen" | "ascii" | "high-bit";
35
+
36
+ export interface TextFinding extends Range, Placement {
37
+ encoding: TextEncoding;
38
+ /** Screen codes with bit 7 set: the characters show reversed. */
39
+ reversed?: boolean;
40
+ /** The text; a line end is "\n". */
41
+ text: string;
42
+ /** Lines of assembler source: most lines are an instruction, a data directive or a comment. */
43
+ source?: true;
44
+ /** Words that are 6502 mnemonics before an operand or a line end. */
45
+ mnemonics?: number;
46
+ }
47
+
48
+ export interface BasicFinding extends Range, Placement {
49
+ /** The address of the first line where the links put it: where the lines were when BASIC linked them. */
50
+ linkedAt: number;
51
+ lines: string[];
52
+ }
53
+
54
+ export interface FillFinding extends Range, Placement {
55
+ pattern: number[];
56
+ }
57
+
58
+ /** Untouched bytes that are no text, BASIC, fill or code region. */
59
+ export type ResidueFinding = Range & Placement;
60
+
61
+ export interface CopyFinding extends Range, Placement {
62
+ /** The first address of the same bytes earlier in the program. */
63
+ copyOf: number;
64
+ }
65
+
66
+ /** A code region that no call or jump reaches from the entry, and that no run executed. */
67
+ export interface DeadCodeFinding extends Range, Placement {
68
+ type: string;
69
+ /** false when the coverage shows that it did not run; absent without coverage. */
70
+ executed?: false;
71
+ }
72
+
73
+ export interface Leftovers {
74
+ text: TextFinding[];
75
+ basic: BasicFinding[];
76
+ fill: FillFinding[];
77
+ residue: ResidueFinding[];
78
+ copies: CopyFinding[];
79
+ deadCode: DeadCodeFinding[];
80
+ }
81
+
82
+ const MIN_TEXT = 8;
83
+ /** A fill of one byte has at least this many bytes; a pattern of two to four bytes twice this many. */
84
+ const MIN_FILL = 16;
85
+ const MIN_RESIDUE = 16;
86
+ const MIN_COPY = 32;
87
+ /** A copy has at least this many different byte values in its first MIN_COPY bytes: no table of a few values. */
88
+ const MIN_COPY_VALUES = 8;
89
+ const MAX_BASIC_LINE = 63999;
90
+
91
+ /** A character of a text, and its letter class: "a" is one case, "b" the other. */
92
+ interface Character {
93
+ /** The character when the class "b" is absent from the run. */
94
+ plain: string;
95
+ /** The character when the run has class "b": the second character set. */
96
+ mixed: string;
97
+ letter?: "a" | "b";
98
+ /** A line end. */
99
+ end?: true;
100
+ }
101
+
102
+ const LINE_END: Character = { plain: "\n", mixed: "\n", end: true };
103
+
104
+ /**
105
+ * PETSCII: $41-$5a is class a, the shifted letters $c1-$da class b. The
106
+ * copies $61-$7a are no letters here: text seldom uses them, and a stray one
107
+ * in front of a text would put the text in the second character set.
108
+ */
109
+ function petsciiCharacter(byte: number): Character | undefined {
110
+ if (byte >= 0x41 && byte <= 0x5a) {
111
+ const upper = String.fromCharCode(byte);
112
+ return { plain: upper, mixed: upper.toLowerCase(), letter: "a" };
113
+ }
114
+ if (byte >= 0xc1 && byte <= 0xda) {
115
+ const upper = String.fromCharCode(byte - 0x80);
116
+ return { plain: upper, mixed: upper, letter: "b" };
117
+ }
118
+ const sign = byte >= 0x20 && byte <= 0x40 ? String.fromCharCode(byte) : { 0x5b: "[", 0x5c: "£", 0x5d: "]", 0x5e: "↑", 0x5f: "←" }[byte];
119
+ return sign === undefined ? undefined : { plain: sign, mixed: sign };
120
+ }
121
+
122
+ /** Screen codes: $01-$1a is class a, $41-$5a (the upper case of the second set) class b. $00 ends a text: it is a common end mark. */
123
+ function screenCharacter(byte: number): Character | undefined {
124
+ if (byte >= 0x01 && byte <= 0x1a) {
125
+ const upper = String.fromCharCode(byte + 0x40);
126
+ return { plain: upper, mixed: upper.toLowerCase(), letter: "a" };
127
+ }
128
+ if (byte >= 0x41 && byte <= 0x5a) {
129
+ const upper = String.fromCharCode(byte);
130
+ return { plain: upper, mixed: upper, letter: "b" };
131
+ }
132
+ const sign = byte >= 0x20 && byte <= 0x3f ? String.fromCharCode(byte) : { 0x1b: "[", 0x1c: "£", 0x1d: "]", 0x1e: "↑", 0x1f: "←" }[byte];
133
+ return sign === undefined ? undefined : { plain: sign, mixed: sign };
134
+ }
135
+
136
+ /** ASCII: small letters are class a, capitals class b, and each shows as it is. $0a and $0d end a line. */
137
+ function asciiCharacter(byte: number): Character | undefined {
138
+ if (byte === 0x0a || byte === 0x0d) return LINE_END;
139
+ if (byte < 0x20 || byte > 0x7e) return undefined;
140
+ const character = String.fromCharCode(byte);
141
+ return { plain: character, mixed: character, ...(byte >= 0x61 && byte <= 0x7a ? { letter: "a" as const } : byte >= 0x41 && byte <= 0x5a ? { letter: "b" as const } : {}) };
142
+ }
143
+
144
+ /** ASCII with bit 7 set: $8d ends a line, and $20 is a space too, as in the comments of Apple II assembler source. */
145
+ function highAsciiCharacter(byte: number): Character | undefined {
146
+ if (byte === 0x8d) return LINE_END;
147
+ if (byte === 0x20) return { plain: " ", mixed: " " };
148
+ return byte >= 0xa0 && byte <= 0xfe ? asciiCharacter(byte - 0x80) : undefined;
149
+ }
150
+
151
+ /** PETSCII with $0d as a line end. */
152
+ function petsciiOrLineEnd(byte: number): Character | undefined {
153
+ return byte === 0x0d ? LINE_END : petsciiCharacter(byte);
154
+ }
155
+
156
+ const COMMON_SIGNS = /[ 0-9.,!?:;'"()\-/&+*=#$%]/;
157
+ const VOWELS = /[AEIOUY]/i;
158
+
159
+ /**
160
+ * Whether a run of characters reads as text: enough letters, at most one odd
161
+ * sign, a word of three letters, each word in one case or a capital and then
162
+ * small letters, and vowels in the words. Bytes of code or data seldom pass
163
+ * all of these. Characters that count up or repeat in a short pattern are a
164
+ * character map of a screen, not text.
165
+ */
166
+ function readsAsText(characters: Character[], min: number): boolean {
167
+ if (characters.length < min) return false;
168
+ const letters = characters.filter((character) => character.letter !== undefined);
169
+ if (letters.length < 4) return false;
170
+ if (characters.filter((character) => character.letter === undefined && !COMMON_SIGNS.test(character.plain)).length > 1) return false;
171
+ if (new Set(characters.map((character) => character.plain)).size < 4) return false;
172
+ if (letters.filter((character) => VOWELS.test(character.plain)).length < letters.length * 0.2) return false;
173
+ const codes = characters.map((character) => character.plain.charCodeAt(0));
174
+ const pairs = codes.length - 1;
175
+ if (codes.slice(1).filter((code, index) => code === codes[index]! + 1).length >= pairs / 2) return false;
176
+ for (let period = 2; period <= 4; period++) if (codes.slice(period).filter((code, index) => code === codes[index]).length >= (codes.length - period) / 2) return false;
177
+ let longest = 0;
178
+ let word: Character[] = [];
179
+ const wordFits = () => {
180
+ const rest = word.slice(1);
181
+ const sameCase = rest.every((character) => character.letter === word[0]!.letter) || (word[0]!.letter === "b" && rest.every((character) => character.letter === "a"));
182
+ return sameCase && (word.length < 4 || word.some((character) => VOWELS.test(character.plain)));
183
+ };
184
+ for (const character of [...characters, { plain: " ", mixed: " " } as Character]) {
185
+ if (character.letter !== undefined) {
186
+ word.push(character);
187
+ continue;
188
+ }
189
+ if (word.length > 0 && !wordFits()) return false;
190
+ longest = Math.max(longest, word.length);
191
+ word = [];
192
+ }
193
+ return longest >= 3;
194
+ }
195
+
196
+ /** The undocumented opcodes: words such as LAX or ISC in a text are seldom source. */
197
+ const UNDOCUMENTED = new Set(["jam", "slo", "rla", "sre", "rra", "sax", "lax", "dcp", "isc", "anc", "asr", "arr", "ane", "sha", "tas", "shy", "shx", "lxa", "las", "sbx"]);
198
+
199
+ /** Counts the words that are a documented 6502 mnemonic followed by an operand sign or the end of the text. */
200
+ function mnemonicCount(text: string): number {
201
+ return [...text.toLowerCase().matchAll(/(?:^|[^a-z])([a-z]{3})(?= ?[#$(<>%]| *$)/g)].filter((match) => MNEMONICS.has(match[1]!) && !UNDOCUMENTED.has(match[1]!)).length;
202
+ }
203
+
204
+ const DIRECTIVES = ["dfb", "db", "dw", "ddb", "dc", "dcb", "ds", "dfs", "byte", "word", "hex", "asc", "dci", "str", "txt", "text", "org", "equ", "epz", "end"];
205
+
206
+ /** A line of assembler source: a comment, or an optional label and then an instruction or a data directive. */
207
+ function sourceLine(line: string): boolean {
208
+ if (/^[*;]/.test(line)) return true;
209
+ const words = line.toLowerCase().split(/\s+/);
210
+ const first = words[0]!.replace(/^[.!]/, "");
211
+ const second = (words[1] ?? "").replace(/^[.!]/, "");
212
+ const known = (word: string) => (MNEMONICS.has(word) && !UNDOCUMENTED.has(word)) || DIRECTIVES.includes(word);
213
+ return known(first) || (/^[a-z_][\w.]*:?$/.test(first) && known(second));
214
+ }
215
+
216
+ /** Whether the bytes of a run show the encoding and no other one: the other scans report the rest. */
217
+ function ownEncoding(encoding: TextEncoding, bytes: number[], characters: Character[]): boolean {
218
+ if (encoding === "screen") return characters.some((character) => character.letter === "a");
219
+ // ASCII text is in capitals, or mostly in small letters: a few small letters are stray bytes at the edges of another text.
220
+ const letters = characters.filter((character) => character.letter !== undefined).length;
221
+ const small = characters.filter((character) => character.letter === "a").length;
222
+ if (small > 0 && small * 2 < letters) return false;
223
+ // Capitals and signs alone are PETSCII too.
224
+ if (encoding === "ascii") return bytes.some((byte) => byte >= 0x61 && byte <= 0x7a);
225
+ // Shifted letters alone are PETSCII too: a space, a digit or a sign with bit 7 shows ASCII.
226
+ if (encoding === "high-bit") return bytes.some((byte) => byte >= 0xa0 && byte <= 0xc0);
227
+ return true;
228
+ }
229
+
230
+ /** The runs of one encoding in a segment that read as text. `reversed` reads the bytes with bit 7 set, without that bit. */
231
+ function textRuns(memory: Uint8Array, segment: Range, encoding: TextEncoding, reversed: boolean, min: number): TextFinding[] {
232
+ const decode = { petscii: petsciiOrLineEnd, screen: screenCharacter, ascii: asciiCharacter, "high-bit": highAsciiCharacter }[encoding];
233
+ const found: TextFinding[] = [];
234
+ let run: Array<{ address: number; character: Character }> = [];
235
+ const solid = (character: Character) => character.letter !== undefined || /[0-9]/.test(character.plain);
236
+ /** Reports a piece of the run as text, from its first to its last solid character, when it reads as text. */
237
+ const report = (piece: typeof run, asSource: boolean) => {
238
+ let first = 0;
239
+ let last = piece.length - 1;
240
+ const edge = asSource ? (character: Character) => character.end !== true && character.plain.trim() !== "" : solid;
241
+ while (first <= last && !edge(piece[first]!.character)) first++;
242
+ while (last >= first && !edge(piece[last]!.character)) last--;
243
+ const kept = piece.slice(first, last + 1);
244
+ const characters = kept.map((item) => item.character);
245
+ if (!(asSource ? characters.length >= min : readsAsText(characters, min))) return;
246
+ if (!ownEncoding(encoding, kept.map((item) => memory[item.address]!), characters)) return;
247
+ const mixed = encoding !== "ascii" && encoding !== "high-bit" && characters.some((character) => character.letter === "b");
248
+ const text = characters.map((character) => (mixed ? character.mixed : character.plain)).join("");
249
+ const mnemonics = mnemonicCount(text);
250
+ found.push({ start: kept[0]!.address, end: kept.at(-1)!.address, encoding, ...(reversed ? { reversed } : {}), text, ...(asSource ? { source: true as const } : {}), ...(mnemonics > 0 ? { mnemonics } : {}) });
251
+ };
252
+ const close = () => {
253
+ const lines = run.map((item) => item.character.plain).join("").split("\n").map((line) => line.trim()).filter((line) => line !== "");
254
+ const source = lines.filter(sourceLine).length;
255
+ if (lines.length >= 2 && source >= 2 && source * 2 >= lines.length) report(run, true);
256
+ else {
257
+ // Not source: each line is a text of its own.
258
+ let piece: typeof run = [];
259
+ for (const item of [...run, { address: -1, character: LINE_END }]) {
260
+ if (item.character.end !== true) piece.push(item);
261
+ else {
262
+ report(piece, false);
263
+ piece = [];
264
+ }
265
+ }
266
+ }
267
+ run = [];
268
+ };
269
+ for (let address = segment.start; address <= segment.end; address++) {
270
+ const byte = memory[address]!;
271
+ const character = reversed ? (byte & 0x80 ? decode(byte & 0x7f) : undefined) : byte & 0x80 && encoding === "screen" ? undefined : decode(byte);
272
+ if (character === undefined) close();
273
+ else run.push({ address, character });
274
+ }
275
+ close();
276
+ return found;
277
+ }
278
+
279
+ export function findText(memory: Uint8Array, segments: Range[], min = MIN_TEXT): TextFinding[] {
280
+ const scan = (encoding: TextEncoding, reversed = false) => segments.flatMap((segment) => textRuns(memory, segment, encoding, reversed, min));
281
+ // A run that is mostly a run of an earlier list is the same text, read in another encoding, with stray bytes at its edges.
282
+ const overlap = (a: Range, b: Range) => Math.max(0, Math.min(a.end, b.end) - Math.max(a.start, b.start) + 1);
283
+ const kept: TextFinding[] = [];
284
+ for (const list of [[...scan("high-bit"), ...scan("ascii")], scan("petscii"), [...scan("screen"), ...scan("screen", true)]]) {
285
+ kept.push(...list.filter((run) => !kept.some((other) => overlap(run, other) * 2 > run.end - run.start + 1)));
286
+ }
287
+ return kept.sort((a, b) => a.start - b.start);
288
+ }
289
+
290
+ /** Whether tokenized BASIC bytes hold only keywords, signs, digits and letters outside quotes. */
291
+ function tokensFit(tokens: Uint8Array): boolean {
292
+ let literal = false;
293
+ for (const byte of tokens) {
294
+ if (byte === 0x22) literal = !literal;
295
+ else if (!literal && !(byte >= 0x20 && byte <= 0x5d) && !(byte >= 0x80 && byte <= 0xcb) && byte !== 0xff) return false;
296
+ }
297
+ return true;
298
+ }
299
+
300
+ /**
301
+ * BASIC lines with links that agree with each other: the distance between two
302
+ * links is the length of a line. Two lines or more, each with a keyword, make
303
+ * a finding. The links give the address where BASIC put the lines, which can
304
+ * be another address than where they are now.
305
+ */
306
+ export function findBasic(memory: Uint8Array, segments: Range[], basicStart?: number): BasicFinding[] {
307
+ const found: BasicFinding[] = [];
308
+ for (const segment of segments) {
309
+ const word = (at: number) => memory[at]! | (memory[at + 1]! << 8);
310
+ /** The line at `at`: its number, its tokens and the address after its end zero, or undefined. */
311
+ const lineAt = (at: number) => {
312
+ if (at + 4 > segment.end) return undefined;
313
+ const number = word(at + 2);
314
+ if (number > MAX_BASIC_LINE) return undefined;
315
+ let end = at + 4;
316
+ while (end <= segment.end && end - at <= 255 && memory[end] !== 0) end++;
317
+ if (end > segment.end || memory[end] !== 0 || end === at + 4) return undefined;
318
+ const tokens = memory.slice(at + 4, end);
319
+ return tokensFit(tokens) ? { number, tokens, next: end + 1 } : undefined;
320
+ };
321
+ for (let at = segment.start; at + 4 <= segment.end; at++) {
322
+ const first = lineAt(at);
323
+ if (first === undefined || (memory[at + 1] ?? 0) === 0) continue;
324
+ const linkedAt = word(at) - (first.next - at);
325
+ if (linkedAt < 0x0400) continue;
326
+ const lines = [first];
327
+ let link = word(at);
328
+ let next = first.next;
329
+ for (;;) {
330
+ if (next + 1 > segment.end || memory[next + 1] === 0) break;
331
+ const line = lineAt(next);
332
+ if (line === undefined || line.number <= lines.at(-1)!.number || word(next) - link !== line.next - next) break;
333
+ lines.push(line);
334
+ link = word(next);
335
+ next = line.next;
336
+ }
337
+ const endMark = next + 1 <= segment.end && memory[next] === 0 && memory[next + 1] === 0;
338
+ if (lines.length < 2 || !lines.every((line) => line.tokens.some((byte) => byte >= 0x80 && byte <= 0xcb))) continue;
339
+ if (at === basicStart && linkedAt === basicStart) {
340
+ at = next - 1;
341
+ continue;
342
+ }
343
+ found.push({ start: at, end: endMark ? next + 1 : next - 1, linkedAt, lines: lines.map(basicLineText) });
344
+ at = endMark ? next + 1 : next - 1;
345
+ }
346
+ }
347
+ return found;
348
+ }
349
+
350
+ /** Runs of one byte of at least MIN_FILL bytes, and runs of a pattern of two to four bytes of at least twice that. */
351
+ export function findFills(memory: Uint8Array, segments: Range[]): FillFinding[] {
352
+ const found: FillFinding[] = [];
353
+ for (const segment of segments) {
354
+ for (let at = segment.start; at <= segment.end; ) {
355
+ let best = { period: 0, length: 0 };
356
+ for (let period = 1; period <= 4; period++) {
357
+ let end = at + period;
358
+ while (end <= segment.end && memory[end] === memory[end - period]) end++;
359
+ const length = end - at;
360
+ const pattern = memory.subarray(at, at + period);
361
+ // A pattern that is one byte repeated is the fill of period 1.
362
+ if (period > 1 && pattern.every((byte) => byte === pattern[0])) continue;
363
+ if (length >= (period === 1 ? MIN_FILL : 2 * MIN_FILL) && length > best.length) best = { period, length };
364
+ }
365
+ if (best.length === 0) {
366
+ at++;
367
+ continue;
368
+ }
369
+ found.push({ start: at, end: at + best.length - 1, pattern: [...memory.subarray(at, at + best.period)] });
370
+ at += best.length;
371
+ }
372
+ }
373
+ return found;
374
+ }
375
+
376
+ /** Exact copies of at least MIN_COPY bytes of an earlier range, with enough different byte values to be no fill. */
377
+ export function findCopies(memory: Uint8Array, segments: Range[]): CopyFinding[] {
378
+ const found: CopyFinding[] = [];
379
+ const seen = new Map<string, number>();
380
+ const inside = (address: number) => segments.some((segment) => address >= segment.start && address <= segment.end);
381
+ for (const segment of segments) {
382
+ for (let at = segment.start; at + MIN_COPY - 1 <= segment.end; at++) {
383
+ const window = memory.subarray(at, at + MIN_COPY);
384
+ if (new Set(window).size < MIN_COPY_VALUES) continue;
385
+ const key = Buffer.from(window).toString("latin1");
386
+ const earlier = seen.get(key);
387
+ if (earlier === undefined) {
388
+ seen.set(key, at);
389
+ continue;
390
+ }
391
+ let length = MIN_COPY;
392
+ while (at + length <= segment.end && inside(earlier + length) && memory[at + length] === memory[earlier + length]) length++;
393
+ found.push({ start: at, end: at + length - 1, copyOf: earlier });
394
+ // The windows of the copy stay unseen: a later copy points to the first range, not to this one.
395
+ at += length - 1;
396
+ }
397
+ }
398
+ return found;
399
+ }
400
+
401
+ /** Splits `range` at the parts that `taken` covers, and keeps the pieces of at least `min` bytes. */
402
+ function without(range: Range, taken: Range[], min: number): Range[] {
403
+ const pieces: Range[] = [];
404
+ let start = range.start;
405
+ for (const part of taken.filter((item) => item.start <= range.end && item.end >= range.start).sort((a, b) => a.start - b.start)) {
406
+ if (part.start > start) pieces.push({ start, end: part.start - 1 });
407
+ start = Math.max(start, part.end + 1);
408
+ }
409
+ if (start <= range.end) pieces.push({ start, end: range.end });
410
+ return pieces.filter((piece) => piece.end - piece.start + 1 >= min);
411
+ }
412
+
413
+ export function findLeftovers(input: LeftoversInput): Leftovers {
414
+ const { memory, segments, separation } = input;
415
+ const ranges = separation.ranges;
416
+ const placement = (range: Range): Placement => {
417
+ const parts = ranges.filter((part) => part.start <= range.end && part.end >= range.start);
418
+ const stages = [...new Set(parts.flatMap((part) => (part.stage === undefined ? [] : [part.stage])))].sort((a, b) => a - b);
419
+ const access = [...new Set(parts.flatMap((part) => (part.access === undefined ? [] : [part.access])))];
420
+ return { ...(stages.length === 0 ? {} : { stages }), ...(access.length === 0 ? {} : { access }) };
421
+ };
422
+ const placed = <T extends Range>(items: T[]) => items.map((item) => ({ ...item, ...placement(item) }));
423
+
424
+ const text = findText(memory, segments, input.minText);
425
+ const basic = findBasic(memory, segments, input.basicStart);
426
+ const fill = findFills(memory, segments);
427
+ // Untouched bytes that no other finding explains.
428
+ const untouched: Range[] = [];
429
+ for (const part of ranges) {
430
+ if (part.access !== "untouched") continue;
431
+ const last = untouched.at(-1);
432
+ if (last !== undefined && last.end + 1 === part.start) last.end = part.end;
433
+ else untouched.push({ start: part.start, end: part.end });
434
+ }
435
+ // Untouched code is code that the runs did not get to, or dead code, which has its own list.
436
+ const explained = [...text, ...basic, ...fill, ...separation.codeRegions];
437
+ const residue = untouched.flatMap((range) => without(range, explained, MIN_RESIDUE));
438
+ // The references miss IRQ vectors and jumps through tables: code that ran is not dead.
439
+ const deadCode: DeadCodeFinding[] = separation.codeRegions
440
+ .filter((region) => region.reachable === false && region.executed !== true)
441
+ .map((region) => ({ start: region.start, end: region.end, type: region.type, ...(region.executed === false ? { executed: false as const } : {}) }));
442
+ return { text: placed(text), basic: placed(basic), fill: placed(fill), residue: placed(residue), copies: placed(findCopies(memory, segments)), deadCode: placed(deadCode) };
443
+ }