@nibling/editor 0.0.0-stage → 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (146) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +53 -2
  3. package/dist/commands/autoclose.d.ts +13 -0
  4. package/dist/commands/autoclose.js +78 -0
  5. package/dist/commands/commands.d.ts +104 -0
  6. package/dist/commands/commands.js +442 -0
  7. package/dist/commands/comments.d.ts +3 -0
  8. package/dist/commands/comments.js +9 -0
  9. package/dist/commands/fold.d.ts +18 -0
  10. package/dist/commands/fold.js +75 -0
  11. package/dist/commands/keymap.d.ts +40 -0
  12. package/dist/commands/keymap.js +118 -0
  13. package/dist/complete/autocomplete.d.ts +35 -0
  14. package/dist/complete/autocomplete.js +386 -0
  15. package/dist/complete/source.d.ts +72 -0
  16. package/dist/complete/source.js +52 -0
  17. package/dist/complete/words.d.ts +53 -0
  18. package/dist/complete/words.js +203 -0
  19. package/dist/element.d.ts +119 -0
  20. package/dist/element.js +1011 -0
  21. package/dist/expand/expansions.d.ts +68 -0
  22. package/dist/expand/expansions.js +143 -0
  23. package/dist/expand/projection.d.ts +73 -0
  24. package/dist/expand/projection.js +189 -0
  25. package/dist/expand/resolver.d.ts +42 -0
  26. package/dist/expand/resolver.js +585 -0
  27. package/dist/expand/slice-tokens.d.ts +25 -0
  28. package/dist/expand/slice-tokens.js +79 -0
  29. package/dist/extension.d.ts +58 -0
  30. package/dist/extension.js +1 -0
  31. package/dist/find/find.d.ts +12 -0
  32. package/dist/find/find.js +469 -0
  33. package/dist/find/search.d.ts +81 -0
  34. package/dist/find/search.js +308 -0
  35. package/dist/headless.d.ts +16 -0
  36. package/dist/headless.js +19 -0
  37. package/dist/index.d.ts +6 -0
  38. package/dist/index.js +6 -0
  39. package/dist/inline-expansion.d.ts +89 -0
  40. package/dist/inline-expansion.js +324 -0
  41. package/dist/input/edit-context.d.ts +23 -0
  42. package/dist/input/edit-context.js +102 -0
  43. package/dist/input/surface.d.ts +101 -0
  44. package/dist/input/surface.js +84 -0
  45. package/dist/input/textarea.d.ts +26 -0
  46. package/dist/input/textarea.js +125 -0
  47. package/dist/sheet.d.ts +6 -0
  48. package/dist/sheet.js +12 -0
  49. package/dist/state/core.d.ts +73 -0
  50. package/dist/state/core.js +86 -0
  51. package/dist/state/fold-ranges.d.ts +41 -0
  52. package/dist/state/fold-ranges.js +167 -0
  53. package/dist/state/folds.d.ts +45 -0
  54. package/dist/state/folds.js +240 -0
  55. package/dist/state/history.d.ts +25 -0
  56. package/dist/state/history.js +127 -0
  57. package/dist/state/indent.d.ts +33 -0
  58. package/dist/state/indent.js +121 -0
  59. package/dist/state/selection.d.ts +45 -0
  60. package/dist/state/selection.js +106 -0
  61. package/dist/state/whitespace.d.ts +16 -0
  62. package/dist/state/whitespace.js +54 -0
  63. package/dist/text/changeset.d.ts +56 -0
  64. package/dist/text/changeset.js +321 -0
  65. package/dist/text/text.d.ts +30 -0
  66. package/dist/text/text.js +321 -0
  67. package/dist/tokens/bridge.d.ts +107 -0
  68. package/dist/tokens/bridge.js +461 -0
  69. package/dist/tokens/engine.d.ts +39 -0
  70. package/dist/tokens/engine.js +360 -0
  71. package/dist/tokens/protocol.d.ts +113 -0
  72. package/dist/tokens/protocol.js +38 -0
  73. package/dist/tokens/textmate/ascii-regex.d.ts +5 -0
  74. package/dist/tokens/textmate/ascii-regex.js +156 -0
  75. package/dist/tokens/textmate/index.d.ts +7 -0
  76. package/dist/tokens/textmate/index.js +19 -0
  77. package/dist/tokens/textmate/linker.d.ts +67 -0
  78. package/dist/tokens/textmate/linker.js +238 -0
  79. package/dist/tokens/textmate/scopes.d.ts +14 -0
  80. package/dist/tokens/textmate/scopes.js +54 -0
  81. package/dist/tokens/textmate/theme.d.ts +54 -0
  82. package/dist/tokens/textmate/theme.js +138 -0
  83. package/dist/tokens/textmate/tokenizer.d.ts +29 -0
  84. package/dist/tokens/textmate/tokenizer.js +684 -0
  85. package/dist/tokens/textmate.d.ts +49 -0
  86. package/dist/tokens/textmate.js +4 -0
  87. package/dist/view/decorations.d.ts +183 -0
  88. package/dist/view/decorations.js +51 -0
  89. package/dist/view/guides.d.ts +58 -0
  90. package/dist/view/guides.js +174 -0
  91. package/dist/view/heightmap.d.ts +97 -0
  92. package/dist/view/heightmap.js +229 -0
  93. package/dist/view/line-dom.d.ts +54 -0
  94. package/dist/view/line-dom.js +168 -0
  95. package/dist/view/measure.d.ts +37 -0
  96. package/dist/view/measure.js +136 -0
  97. package/dist/view/overlays.d.ts +32 -0
  98. package/dist/view/overlays.js +135 -0
  99. package/dist/view/rail.d.ts +27 -0
  100. package/dist/view/rail.js +232 -0
  101. package/dist/view/remote.d.ts +24 -0
  102. package/dist/view/remote.js +143 -0
  103. package/dist/view/scope-focus.d.ts +27 -0
  104. package/dist/view/scope-focus.js +255 -0
  105. package/dist/view/view.d.ts +114 -0
  106. package/dist/view/view.js +1231 -0
  107. package/dist/vite.d.ts +12 -0
  108. package/dist/vite.js +66 -0
  109. package/dist/worker.d.ts +1 -0
  110. package/dist/worker.js +64 -0
  111. package/grammars/astro.mjs +2 -0
  112. package/grammars/chunks/r-90cb351dba.mjs +1 -0
  113. package/grammars/chunks/r-b92a500aab.mjs +1 -0
  114. package/grammars/chunks/r-ced5311992.mjs +1 -0
  115. package/grammars/chunks/r-ebae91d4ec.mjs +1 -0
  116. package/grammars/chunks/s-1821e66170.mjs +1 -0
  117. package/grammars/css.mjs +1 -0
  118. package/grammars/diff.mjs +1 -0
  119. package/grammars/docker.mjs +1 -0
  120. package/grammars/dotenv.mjs +1 -0
  121. package/grammars/graphql.mjs +1 -0
  122. package/grammars/html.mjs +2 -0
  123. package/grammars/ini.mjs +1 -0
  124. package/grammars/javascript.mjs +5 -0
  125. package/grammars/json.mjs +1 -0
  126. package/grammars/json5.mjs +1 -0
  127. package/grammars/jsonc.mjs +1 -0
  128. package/grammars/less.mjs +1 -0
  129. package/grammars/manifest.json +58 -0
  130. package/grammars/markdown.mjs +1 -0
  131. package/grammars/mdx.mjs +1 -0
  132. package/grammars/pug.mjs +1 -0
  133. package/grammars/scss.mjs +1 -0
  134. package/grammars/shellscript.mjs +1 -0
  135. package/grammars/sql.mjs +1 -0
  136. package/grammars/stylus.mjs +1 -0
  137. package/grammars/svelte.mjs +1 -0
  138. package/grammars/toml.mjs +1 -0
  139. package/grammars/tsx.mjs +5 -0
  140. package/grammars/typescript.mjs +2 -0
  141. package/grammars/vue.mjs +1 -0
  142. package/grammars/yaml.mjs +1 -0
  143. package/package.json +71 -3
  144. package/themes/dark-plus.json +1 -0
  145. package/themes/light-plus.json +1 -0
  146. package/themes/monokai.json +1 -0
@@ -0,0 +1,461 @@
1
+ /**
2
+ * Main-thread side of the tokenizer (D6, D10). A follower of EditorCore.
3
+ *
4
+ * It owns the only main-thread copy of syntax tokens, and only for the mounted window
5
+ * [#from, #from + #rows.length). The worker owns tokens for the whole document. That keeps the
6
+ * main thread's per-keystroke work proportional to the window (about 100 lines), never to the
7
+ * document.
8
+ *
9
+ * Staleness rule (the answer to "version N tokens reaching a view at N+k"):
10
+ * A row from a worker batch is only ever attached to a line whose text is identical to the
11
+ * text it was computed for. Batches carry their version; the bridge keeps the Text of every version from
12
+ * the newest batch's version to the present (persistent, so this costs only the edited paths),
13
+ * composes the changes since, and for each line in the batch:
14
+ * - touched by those changes -> dropped (a newer batch is already on its way);
15
+ * - untouched -> moved to its current line number and accepted.
16
+ * Lines below an edit whose *start state* changed keep their old, correct-for-their-text-at-N
17
+ * tokens until the worker's newer batch replaces them. That is bounded staleness, never
18
+ * misalignment.
19
+ *
20
+ * Local edits: on every Update the window is re-keyed. Untouched rows shift to their new line
21
+ * numbers. A touched row is mapped through the change: retained characters keep their runs,
22
+ * inserted characters get style 0. That row is marked provisional; the next batch for that line
23
+ * replaces it. Typing never repaints a colored line as plain text.
24
+ */
25
+ export class TokenBridge {
26
+ #worker;
27
+ #kept = [];
28
+ #from = 1;
29
+ #rows = [];
30
+ /** The range the view mounted; #rows covers it plus #prefetch lines each side. */
31
+ #mounted = { from: 1, to: 1 };
32
+ #prefetch;
33
+ /** Rows that left #rows, by current line number, oldest first. Bounded by #cacheLines. */
34
+ #cache = new Map();
35
+ #cacheLines;
36
+ /** Rows exactly as a worker batch delivered them. Shifting keeps a row's identity; a provisional
37
+ * row mapped through an edit is a new array, so it is never here and never cached. */
38
+ #exact = new WeakSet();
39
+ #epoch = 0;
40
+ /** Epoch of the styles the view currently has. Rows and styles swap together on theme change. */
41
+ #shownEpoch = 0;
42
+ #listeners = [];
43
+ /** Every style of the shown epoch, so a view that subscribes late can build its sheet. */
44
+ #styles = [];
45
+ /** More windows kept besides the mounted one, by owner (an inline expansion's slice). [from, to). */
46
+ #pins = new Map();
47
+ /** Rows for pinned lines outside #rows, by current line number, re-keyed like #rows. */
48
+ #pinned = new Map();
49
+ #errored = false;
50
+ #offCore = () => { };
51
+ /**
52
+ * `prefetch`: lines kept and requested beyond each side of the mounted range, so scrolling lands
53
+ * on colored lines; default, the mounted range's own length. `cacheLines`: how many rows outside
54
+ * that range are kept for scrolling back.
55
+ */
56
+ constructor(worker, core, init, signal) {
57
+ this.#worker = worker;
58
+ this.#prefetch = init.prefetch;
59
+ this.#cacheLines = init.cacheLines ?? 5000;
60
+ this.#worker.addEventListener('message', (e) => this.#onMessage(e.data));
61
+ this.#worker.postMessage({ type: 'theme', epoch: this.#epoch, theme: init.theme });
62
+ this.#worker.postMessage({
63
+ type: 'open',
64
+ version: core.state.version,
65
+ text: core.state.text.toString(),
66
+ language: init.language,
67
+ grammars: init.grammars,
68
+ });
69
+ this.#kept = [{ version: core.state.version, text: core.state.text }];
70
+ this.#offCore = core.listen((u) => this.#onUpdate(u), signal);
71
+ }
72
+ /** Tokens for a line, or undefined to paint it plain. O(1). */
73
+ lineTokens(line) {
74
+ return this.#rows[line - this.#from] ?? this.#pinned.get(line) ?? this.#cache.get(line);
75
+ }
76
+ /** The styles every row of the shown epoch may use. */
77
+ get styles() {
78
+ return this.#styles;
79
+ }
80
+ /**
81
+ * Keep rows for lines [from, to) on behalf of `owner`, wherever the mounted window is: the worker
82
+ * sends them, and edits re-key them provisionally as they do the window, so a line typed into
83
+ * through an expansion never flashes plain. Calling it again moves the pin; `from >= to` removes
84
+ * it. Pins also move themselves through each edit, approximately, until their owner moves them.
85
+ */
86
+ pin(owner, from, to) {
87
+ const cur = this.#pins.get(owner);
88
+ if (from >= to) {
89
+ if (!cur)
90
+ return;
91
+ this.#pins.delete(owner);
92
+ }
93
+ else {
94
+ if (cur && cur.from === from && cur.to === to)
95
+ return;
96
+ this.#pins.set(owner, { from, to });
97
+ }
98
+ const windowTo = this.#from + this.#rows.length;
99
+ for (const [line, row] of this.#pinned) {
100
+ if (this.#inPins(line))
101
+ continue;
102
+ this.#pinned.delete(line);
103
+ if ((line < this.#from || line >= windowTo) && this.#exact.has(row))
104
+ this.#cacheRow(line, row);
105
+ }
106
+ // Rows the cache already has for newly pinned lines are exact; take them.
107
+ for (const { from: a, to: b } of this.#pins.values()) {
108
+ for (let line = a; line < b; line++) {
109
+ const row = this.#cache.get(line);
110
+ if (row === undefined || (line >= this.#from && line < windowTo))
111
+ continue;
112
+ this.#cache.delete(line);
113
+ this.#pinned.set(line, row);
114
+ }
115
+ }
116
+ this.#postViewport();
117
+ }
118
+ #inPins(line) {
119
+ for (const p of this.#pins.values())
120
+ if (line >= p.from && line < p.to)
121
+ return true;
122
+ return false;
123
+ }
124
+ #postViewport() {
125
+ const from = this.#from;
126
+ const to = this.#from + this.#rows.length;
127
+ if (this.#pins.size === 0)
128
+ return this.#worker.postMessage({ type: 'viewport', from, to });
129
+ const extra = [...this.#pins.values()].map((p) => [p.from, p.to]);
130
+ this.#worker.postMessage({ type: 'viewport', from, to, extra });
131
+ }
132
+ /** Called by the view when its mounted line range changes. The worker is asked for the range
133
+ * plus the prefetch margin; rows leaving that span move to the cache, and rows entering it come
134
+ * back from the cache. */
135
+ setViewport(from, to) {
136
+ if (from === this.#mounted.from && to === this.#mounted.to)
137
+ return; // idempotent
138
+ this.#mounted = { from, to };
139
+ const margin = this.#prefetch ?? Math.max(0, to - from);
140
+ const lines = this.#kept[this.#kept.length - 1].text.lines;
141
+ // The mounted range is honored as asked; only the margin is clamped to the document.
142
+ const wantFrom = Math.min(from, Math.max(1, from - margin));
143
+ const wantTo = Math.max(to, Math.min(lines + 1, to + margin));
144
+ const curTo = this.#from + this.#rows.length;
145
+ if (wantFrom === this.#from && wantTo === curTo)
146
+ return;
147
+ for (let i = 0; i < this.#rows.length; i++) {
148
+ const line = this.#from + i;
149
+ const row = this.#rows[i];
150
+ if (row === undefined || (line >= wantFrom && line < wantTo))
151
+ continue;
152
+ // A pinned line keeps even a provisional row; anything else caches only exact rows.
153
+ if (this.#inPins(line))
154
+ this.#pinned.set(line, row);
155
+ else if (this.#exact.has(row))
156
+ this.#cacheRow(line, row);
157
+ }
158
+ const next = new Array(wantTo - wantFrom);
159
+ for (let line = wantFrom; line < wantTo; line++) {
160
+ const idx = line - this.#from;
161
+ let row = idx >= 0 && idx < this.#rows.length ? this.#rows[idx] : undefined;
162
+ row ??= this.#pinned.get(line) ?? this.#cache.get(line);
163
+ this.#pinned.delete(line);
164
+ this.#cache.delete(line);
165
+ next[line - wantFrom] = row;
166
+ }
167
+ this.#from = wantFrom;
168
+ this.#rows = next;
169
+ this.#postViewport();
170
+ }
171
+ #cacheRow(line, row) {
172
+ this.#cache.delete(line);
173
+ this.#cache.set(line, row);
174
+ while (this.#cache.size > this.#cacheLines)
175
+ this.#cache.delete(this.#cache.keys().next().value);
176
+ }
177
+ setTheme(theme) {
178
+ this.#epoch++;
179
+ this.#worker.postMessage({ type: 'theme', epoch: this.#epoch, theme });
180
+ // Current rows and #shownEpoch's styles keep painting until the first batch tagged with the
181
+ // new epoch arrives (#onMessage), which emits reset: true to swap both at once.
182
+ }
183
+ /** Like a theme change, a new epoch: the old grammar's rows keep painting the same text until
184
+ * the first batch of the new epoch replaces them. */
185
+ setLanguage(language, version) {
186
+ this.#epoch++;
187
+ this.#worker.postMessage({ type: 'language', version, epoch: this.#epoch, language });
188
+ }
189
+ onChange(listener) {
190
+ this.#listeners.push(listener);
191
+ return () => {
192
+ const i = this.#listeners.indexOf(listener);
193
+ if (i >= 0)
194
+ this.#listeners.splice(i, 1);
195
+ };
196
+ }
197
+ #emit(c) {
198
+ for (const l of this.#listeners)
199
+ l(c);
200
+ }
201
+ #onUpdate(u) {
202
+ if (!u.docChanged)
203
+ return;
204
+ this.#worker.postMessage({ type: 'change', version: u.after.version, changes: u.changes.toJSON() });
205
+ this.#kept.push({ version: u.after.version, text: u.after.text, changes: u.changes });
206
+ this.#rows = rekeyWindow(this.#rows, this.#from, u.changes, u.before.text, u.after.text);
207
+ this.#cache = rekeyCache(this.#cache, u.changes, u.before.text, u.after.text);
208
+ if (this.#pins.size > 0) {
209
+ this.#pinned = rekeyRows(this.#pinned, u.changes, u.before.text, u.after.text);
210
+ for (const [owner, p] of this.#pins)
211
+ this.#pins.set(owner, mapLineRange(p, u.changes, u.before.text, u.after.text));
212
+ }
213
+ // Lines whose (text, tokens) identity changed: every line the change's new-document span
214
+ // covers, so the view repaints provisional rows on the same frame as the text.
215
+ const windowTo = this.#from + this.#rows.length;
216
+ const touched = new Set();
217
+ // Same reasoning as the view's rekey loop: a remote burst is hundreds of scattered
218
+ // single-character inserts in one ChangeSet, so this runs hundreds of times per Update, and
219
+ // the second `lineAt` (a tree descent) is redundant whenever the inserted text has no line
220
+ // break to possibly land `toB` on a different line than `fromB`.
221
+ u.changes.iterChanges((_fromA, _toA, fromB, toB, inserted) => {
222
+ const lo = u.after.text.lineAt(fromB).number;
223
+ const hi = toB <= fromB || !inserted.includes('\n') ? lo : u.after.text.lineAt(toB).number;
224
+ for (let n = lo; n <= hi; n++)
225
+ if ((n >= this.#from && n < windowTo) || this.#inPins(n))
226
+ touched.add(n);
227
+ });
228
+ if (touched.size > 0)
229
+ this.#emit({ lines: [...touched].sort((a, b) => a - b), styles: [], reset: false });
230
+ }
231
+ #onMessage(m) {
232
+ if (m.type === 'error') {
233
+ if (!this.#errored) {
234
+ this.#errored = true;
235
+ console.error(m.message);
236
+ }
237
+ return;
238
+ }
239
+ if (m.epoch < this.#epoch)
240
+ return; // superseded theme
241
+ const i = this.#kept.findIndex((k) => k.version === m.version);
242
+ if (i < 0)
243
+ return; // older than anything we still track; a later batch already supersedes it
244
+ const base = this.#kept[i];
245
+ let since = null;
246
+ for (let k = i + 1; k < this.#kept.length; k++) {
247
+ const c = this.#kept[k].changes;
248
+ since = since ? since.compose(c) : c;
249
+ }
250
+ const now = this.#kept[this.#kept.length - 1].text;
251
+ const windowTo = this.#from + this.#rows.length;
252
+ const accepted = [];
253
+ for (let j = 0; j < m.lines.length; j++) {
254
+ const L = m.lines[j];
255
+ const { from, to } = base.text.line(L);
256
+ if (since && since.touches(from, to))
257
+ continue; // a newer batch is already on its way
258
+ const cur = since ? now.lineAt(since.mapPos(from, 1)).number : L;
259
+ if (cur >= this.#from && cur < windowTo) {
260
+ const row = m.data.subarray(m.offsets[j], m.offsets[j + 1]);
261
+ this.#exact.add(row);
262
+ this.#rows[cur - this.#from] = row;
263
+ accepted.push(cur);
264
+ }
265
+ else if (this.#inPins(cur)) {
266
+ const row = m.data.subarray(m.offsets[j], m.offsets[j + 1]);
267
+ this.#exact.add(row);
268
+ this.#pinned.set(cur, row);
269
+ accepted.push(cur);
270
+ }
271
+ }
272
+ this.#kept = this.#kept.slice(i); // batches only move forward
273
+ const reset = m.epoch !== this.#shownEpoch;
274
+ if (reset) {
275
+ this.#cache.clear();
276
+ // The new epoch's styles replace every earlier one, so a row this batch did not carry would
277
+ // paint old style ids against new styles. Plain until its own batch arrives.
278
+ const fresh = new Set(accepted);
279
+ for (let r = 0; r < this.#rows.length; r++) {
280
+ if (this.#rows[r] !== undefined && !fresh.has(this.#from + r)) {
281
+ this.#rows[r] = undefined;
282
+ accepted.push(this.#from + r);
283
+ }
284
+ }
285
+ for (const line of this.#pinned.keys()) {
286
+ if (fresh.has(line))
287
+ continue;
288
+ this.#pinned.delete(line);
289
+ accepted.push(line);
290
+ }
291
+ this.#styles = [...m.styles];
292
+ }
293
+ else
294
+ this.#styles.push(...m.styles);
295
+ this.#shownEpoch = m.epoch;
296
+ this.#emit({ lines: accepted, styles: m.styles, reset });
297
+ }
298
+ dispose() {
299
+ this.#offCore();
300
+ this.#worker.terminate?.();
301
+ }
302
+ }
303
+ /**
304
+ * Pure. Re-key a token window through one ChangeSet. Exported for tests.
305
+ * Rows for lines the change does not touch move to their new line numbers unchanged (same
306
+ * Uint32Array). Touched rows are rebuilt by `mapLineTokens`. Lines created by the change get
307
+ * undefined. Cost: O(window + changed sections).
308
+ */
309
+ export function rekeyWindow(rows, from, changes, before, after) {
310
+ const n = rows.length;
311
+ const result = new Array(n).fill(undefined);
312
+ for (let j = 0; j < n; j++) {
313
+ const oldLine = from + j;
314
+ if (oldLine < 1 || oldLine > before.lines)
315
+ continue;
316
+ const moved = mapRow(rows[j], oldLine, changes, before, after);
317
+ if (moved && moved[0] >= from && moved[0] < from + n)
318
+ result[moved[0] - from] = moved[1];
319
+ }
320
+ return result;
321
+ }
322
+ /**
323
+ * Where one row goes through a ChangeSet: an untouched line moves with its row unchanged; a touched
324
+ * line that stays one whole line gets a provisional row (mapLineTokens); anything else is dropped.
325
+ */
326
+ function mapRow(row, oldLine, changes, before, after) {
327
+ const { from: lineFrom, to: lineTo } = before.line(oldLine);
328
+ if (!changes.touches(lineFrom, lineTo))
329
+ return [after.lineAt(changes.mapPos(lineFrom, 1)).number, row];
330
+ if (row === undefined)
331
+ return null;
332
+ // Only rebuild in place when the edit stays inside one line on both sides. A merge or split
333
+ // (the edit reaches across a line break) can't be expressed as a single line's provisional
334
+ // map, so it falls back to undefined until a real batch replaces it.
335
+ // Text typed at either edge of the line joins it, so try both edges mapped outward first. An
336
+ // edge insertion carrying its own line break instead leaves the line's text whole on a line of
337
+ // its own, which the inward mapping finds.
338
+ let nl = exactLine(after, changes.mapPos(lineFrom, -1), changes.mapPos(lineTo, 1));
339
+ nl ??= exactLine(after, changes.mapPos(lineFrom, 1), changes.mapPos(lineTo, -1));
340
+ if (!nl)
341
+ return null;
342
+ return [nl.number, mapLineTokens(row, lineFrom, changes, nl.from, nl.to - nl.from)];
343
+ }
344
+ /** Pure. rekeyWindow for rows kept by line number rather than in a window (the bridge's pins). */
345
+ export function rekeyRows(rows, changes, before, after) {
346
+ const out = new Map();
347
+ for (const [line, row] of rows) {
348
+ if (line > before.lines)
349
+ continue;
350
+ const moved = mapRow(row, line, changes, before, after);
351
+ if (moved?.[1])
352
+ out.set(moved[0], moved[1]);
353
+ }
354
+ return out;
355
+ }
356
+ /** A [from, to) line range carried through a ChangeSet: its first line's start and last line's end. */
357
+ function mapLineRange(r, changes, before, after) {
358
+ const last = Math.min(r.to - 1, before.lines);
359
+ if (last < 1)
360
+ return r;
361
+ const a = after.lineAt(changes.mapPos(before.line(Math.min(r.from, last)).from, -1)).number;
362
+ const b = after.lineAt(changes.mapPos(before.line(last).to, 1)).number;
363
+ return { from: a, to: Math.max(a, b) + 1 };
364
+ }
365
+ /**
366
+ * Pure. Map one line's runs through an in-line edit: retained characters keep their style,
367
+ * inserted characters take style 0, empty runs vanish, equal neighbours merge.
368
+ */
369
+ export function mapLineTokens(tokens, lineFrom, changes, newLineFrom, newLength) {
370
+ const newLineTo = newLineFrom + newLength;
371
+ const starts = [];
372
+ const styles = [];
373
+ // tokens[i] is the last run start <= relOld; style 0 (bare text) covers anything before the
374
+ // first run or when the line had no runs at all.
375
+ const styleAtOld = (relOld) => {
376
+ let style = 0;
377
+ for (let i = 0; i < tokens.length; i += 2) {
378
+ if (tokens[i] <= relOld)
379
+ style = tokens[i + 1];
380
+ else
381
+ break;
382
+ }
383
+ return style;
384
+ };
385
+ const push = (absPos, style) => {
386
+ const rel = absPos - newLineFrom;
387
+ if (rel < 0 || rel >= newLength)
388
+ return;
389
+ if (styles.length > 0 && styles[styles.length - 1] === style)
390
+ return; // equal neighbours merge
391
+ starts.push(rel);
392
+ styles.push(style);
393
+ };
394
+ let posA = 0;
395
+ let posB = 0;
396
+ const sections = changes.sections;
397
+ for (let i = 0; i < sections.length && posB < newLineTo; i += 2) {
398
+ const len = sections[i];
399
+ const ins = sections[i + 1];
400
+ if (ins < 0) {
401
+ // Retained span: oldAbs [posA, posA+len) <-> newAbs [posB, posB+len), a pure shift.
402
+ const segFrom = Math.max(posB, newLineFrom);
403
+ const segTo = Math.min(posB + len, newLineTo);
404
+ if (segTo > segFrom) {
405
+ const offset = posB - posA;
406
+ const oldSegFrom = segFrom - offset;
407
+ const oldSegTo = segTo - offset;
408
+ push(segFrom, styleAtOld(oldSegFrom - lineFrom));
409
+ for (let k = 0; k < tokens.length; k += 2) {
410
+ const boundaryOld = lineFrom + tokens[k];
411
+ if (boundaryOld > oldSegFrom && boundaryOld < oldSegTo)
412
+ push(boundaryOld + offset, tokens[k + 1]);
413
+ }
414
+ }
415
+ posB += len;
416
+ }
417
+ else {
418
+ // Changed span: whatever landed here is new text, not retained from the old line.
419
+ const segFrom = Math.max(posB, newLineFrom);
420
+ const segTo = Math.min(posB + ins, newLineTo);
421
+ if (segTo > segFrom)
422
+ push(segFrom, 0);
423
+ posB += ins;
424
+ }
425
+ posA += len;
426
+ }
427
+ const out = new Uint32Array(starts.length * 2);
428
+ for (let i = 0; i < starts.length; i++) {
429
+ out[i * 2] = starts[i];
430
+ out[i * 2 + 1] = styles[i];
431
+ }
432
+ return out;
433
+ }
434
+ /** The line spanning exactly [from, to], or null when that range is not one whole line. */
435
+ function exactLine(text, from, to) {
436
+ const line = text.lineAt(from);
437
+ return line.from === from && line.to === to ? line : null;
438
+ }
439
+ /**
440
+ * Pure. Re-key cached rows through one ChangeSet with the rule worker batches follow: a row whose
441
+ * line the change touches is dropped, and every other row moves to its line's new number. Cached
442
+ * rows are never mapped provisionally, so each one stays exactly what the worker computed for its
443
+ * text. Order is kept, so the oldest entry stays first. O(cache).
444
+ */
445
+ export function rekeyCache(cache, changes, before, after) {
446
+ const out = new Map();
447
+ if (changes.isEmpty) {
448
+ for (const [line, row] of cache)
449
+ out.set(line, row);
450
+ return out;
451
+ }
452
+ for (const [line, row] of cache) {
453
+ if (line > before.lines)
454
+ continue;
455
+ const { from, to } = before.line(line);
456
+ if (changes.touches(from, to))
457
+ continue;
458
+ out.set(after.lineAt(changes.mapPos(from, 1)).number, row);
459
+ }
460
+ return out;
461
+ }
@@ -0,0 +1,39 @@
1
+ import { type FromWorker, type ToWorker } from './protocol.ts';
2
+ import type { GrammarModule, TextMateRuntime } from './textmate.ts';
3
+ export interface EngineHost {
4
+ post(message: FromWorker, transfer: Transferable[]): void;
5
+ /** Resolve a module URL to a grammar module (dynamic import in the worker). */
6
+ load(url: string): Promise<GrammarModule | null>;
7
+ /** Yield to the worker's event loop so queued 'change' messages run between slices. */
8
+ yieldNow(): Promise<void>;
9
+ now(): number;
10
+ }
11
+ /**
12
+ * Worker-side tokenizer (D6). Owns a mirror Text, a LineStore, the linked grammar, the theme,
13
+ * and a scheduler. One instance per document. Everything here is plain data plus the S3 runtime;
14
+ * no DOM.
15
+ *
16
+ * Invariants:
17
+ * - `#text` and the LineStore always describe document version `#version`.
18
+ * - `#frontier` is the first line whose end state is unknown. Every line above it has a valid
19
+ * state and tokens for that state.
20
+ * - Messages are handled synchronously in arrival order; the only async point is between slices,
21
+ * so an edit never lands in the middle of a line's tokenization.
22
+ * - `#sent` holds the window lines whose current tokens the main thread has in this epoch.
23
+ *
24
+ * Scheduling, per slice (budget 4 ms):
25
+ * 1. Tokenize from #frontier toward the end of the viewport window. Lines in the window go to
26
+ * a pending batch when their tokens differ from what was last sent.
27
+ * 2. Past the window, keep going until a line's new end state equals its old one (converged)
28
+ * or the document ends. These lines are not sent; they will be when the viewport reaches them.
29
+ * On convergence the frontier jumps to the next dirty line, if any.
30
+ * 3. Flush the pending batch at the end of every slice that produced one, so the visible region
31
+ * paints first and an open template literal never blocks it.
32
+ * 4. When converged and nothing is dirty, run the lazy-grammar loop (pre-scan, load, link,
33
+ * dirty the lines whose misses intersect the new bits; see LazyDocument.settle) and repeat.
34
+ */
35
+ export declare class TokenEngine {
36
+ #private;
37
+ constructor(host: EngineHost, runtime: TextMateRuntime);
38
+ handle(m: ToWorker): void;
39
+ }