@nibling/editor 0.0.0-stage → 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (146) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +53 -2
  3. package/dist/commands/autoclose.d.ts +13 -0
  4. package/dist/commands/autoclose.js +78 -0
  5. package/dist/commands/commands.d.ts +104 -0
  6. package/dist/commands/commands.js +442 -0
  7. package/dist/commands/comments.d.ts +3 -0
  8. package/dist/commands/comments.js +9 -0
  9. package/dist/commands/fold.d.ts +18 -0
  10. package/dist/commands/fold.js +75 -0
  11. package/dist/commands/keymap.d.ts +40 -0
  12. package/dist/commands/keymap.js +118 -0
  13. package/dist/complete/autocomplete.d.ts +35 -0
  14. package/dist/complete/autocomplete.js +386 -0
  15. package/dist/complete/source.d.ts +72 -0
  16. package/dist/complete/source.js +52 -0
  17. package/dist/complete/words.d.ts +53 -0
  18. package/dist/complete/words.js +203 -0
  19. package/dist/element.d.ts +119 -0
  20. package/dist/element.js +1011 -0
  21. package/dist/expand/expansions.d.ts +68 -0
  22. package/dist/expand/expansions.js +143 -0
  23. package/dist/expand/projection.d.ts +73 -0
  24. package/dist/expand/projection.js +189 -0
  25. package/dist/expand/resolver.d.ts +42 -0
  26. package/dist/expand/resolver.js +585 -0
  27. package/dist/expand/slice-tokens.d.ts +25 -0
  28. package/dist/expand/slice-tokens.js +79 -0
  29. package/dist/extension.d.ts +58 -0
  30. package/dist/extension.js +1 -0
  31. package/dist/find/find.d.ts +12 -0
  32. package/dist/find/find.js +469 -0
  33. package/dist/find/search.d.ts +81 -0
  34. package/dist/find/search.js +308 -0
  35. package/dist/headless.d.ts +16 -0
  36. package/dist/headless.js +19 -0
  37. package/dist/index.d.ts +6 -0
  38. package/dist/index.js +6 -0
  39. package/dist/inline-expansion.d.ts +89 -0
  40. package/dist/inline-expansion.js +324 -0
  41. package/dist/input/edit-context.d.ts +23 -0
  42. package/dist/input/edit-context.js +102 -0
  43. package/dist/input/surface.d.ts +101 -0
  44. package/dist/input/surface.js +84 -0
  45. package/dist/input/textarea.d.ts +26 -0
  46. package/dist/input/textarea.js +125 -0
  47. package/dist/sheet.d.ts +6 -0
  48. package/dist/sheet.js +12 -0
  49. package/dist/state/core.d.ts +73 -0
  50. package/dist/state/core.js +86 -0
  51. package/dist/state/fold-ranges.d.ts +41 -0
  52. package/dist/state/fold-ranges.js +167 -0
  53. package/dist/state/folds.d.ts +45 -0
  54. package/dist/state/folds.js +240 -0
  55. package/dist/state/history.d.ts +25 -0
  56. package/dist/state/history.js +127 -0
  57. package/dist/state/indent.d.ts +33 -0
  58. package/dist/state/indent.js +121 -0
  59. package/dist/state/selection.d.ts +45 -0
  60. package/dist/state/selection.js +106 -0
  61. package/dist/state/whitespace.d.ts +16 -0
  62. package/dist/state/whitespace.js +54 -0
  63. package/dist/text/changeset.d.ts +56 -0
  64. package/dist/text/changeset.js +321 -0
  65. package/dist/text/text.d.ts +30 -0
  66. package/dist/text/text.js +321 -0
  67. package/dist/tokens/bridge.d.ts +107 -0
  68. package/dist/tokens/bridge.js +461 -0
  69. package/dist/tokens/engine.d.ts +39 -0
  70. package/dist/tokens/engine.js +360 -0
  71. package/dist/tokens/protocol.d.ts +113 -0
  72. package/dist/tokens/protocol.js +38 -0
  73. package/dist/tokens/textmate/ascii-regex.d.ts +5 -0
  74. package/dist/tokens/textmate/ascii-regex.js +156 -0
  75. package/dist/tokens/textmate/index.d.ts +7 -0
  76. package/dist/tokens/textmate/index.js +19 -0
  77. package/dist/tokens/textmate/linker.d.ts +67 -0
  78. package/dist/tokens/textmate/linker.js +238 -0
  79. package/dist/tokens/textmate/scopes.d.ts +14 -0
  80. package/dist/tokens/textmate/scopes.js +54 -0
  81. package/dist/tokens/textmate/theme.d.ts +54 -0
  82. package/dist/tokens/textmate/theme.js +138 -0
  83. package/dist/tokens/textmate/tokenizer.d.ts +29 -0
  84. package/dist/tokens/textmate/tokenizer.js +684 -0
  85. package/dist/tokens/textmate.d.ts +49 -0
  86. package/dist/tokens/textmate.js +4 -0
  87. package/dist/view/decorations.d.ts +183 -0
  88. package/dist/view/decorations.js +51 -0
  89. package/dist/view/guides.d.ts +58 -0
  90. package/dist/view/guides.js +174 -0
  91. package/dist/view/heightmap.d.ts +97 -0
  92. package/dist/view/heightmap.js +229 -0
  93. package/dist/view/line-dom.d.ts +54 -0
  94. package/dist/view/line-dom.js +168 -0
  95. package/dist/view/measure.d.ts +37 -0
  96. package/dist/view/measure.js +136 -0
  97. package/dist/view/overlays.d.ts +32 -0
  98. package/dist/view/overlays.js +135 -0
  99. package/dist/view/rail.d.ts +27 -0
  100. package/dist/view/rail.js +232 -0
  101. package/dist/view/remote.d.ts +24 -0
  102. package/dist/view/remote.js +143 -0
  103. package/dist/view/scope-focus.d.ts +27 -0
  104. package/dist/view/scope-focus.js +255 -0
  105. package/dist/view/view.d.ts +114 -0
  106. package/dist/view/view.js +1231 -0
  107. package/dist/vite.d.ts +12 -0
  108. package/dist/vite.js +66 -0
  109. package/dist/worker.d.ts +1 -0
  110. package/dist/worker.js +64 -0
  111. package/grammars/astro.mjs +2 -0
  112. package/grammars/chunks/r-90cb351dba.mjs +1 -0
  113. package/grammars/chunks/r-b92a500aab.mjs +1 -0
  114. package/grammars/chunks/r-ced5311992.mjs +1 -0
  115. package/grammars/chunks/r-ebae91d4ec.mjs +1 -0
  116. package/grammars/chunks/s-1821e66170.mjs +1 -0
  117. package/grammars/css.mjs +1 -0
  118. package/grammars/diff.mjs +1 -0
  119. package/grammars/docker.mjs +1 -0
  120. package/grammars/dotenv.mjs +1 -0
  121. package/grammars/graphql.mjs +1 -0
  122. package/grammars/html.mjs +2 -0
  123. package/grammars/ini.mjs +1 -0
  124. package/grammars/javascript.mjs +5 -0
  125. package/grammars/json.mjs +1 -0
  126. package/grammars/json5.mjs +1 -0
  127. package/grammars/jsonc.mjs +1 -0
  128. package/grammars/less.mjs +1 -0
  129. package/grammars/manifest.json +58 -0
  130. package/grammars/markdown.mjs +1 -0
  131. package/grammars/mdx.mjs +1 -0
  132. package/grammars/pug.mjs +1 -0
  133. package/grammars/scss.mjs +1 -0
  134. package/grammars/shellscript.mjs +1 -0
  135. package/grammars/sql.mjs +1 -0
  136. package/grammars/stylus.mjs +1 -0
  137. package/grammars/svelte.mjs +1 -0
  138. package/grammars/toml.mjs +1 -0
  139. package/grammars/tsx.mjs +5 -0
  140. package/grammars/typescript.mjs +2 -0
  141. package/grammars/vue.mjs +1 -0
  142. package/grammars/yaml.mjs +1 -0
  143. package/package.json +71 -3
  144. package/themes/dark-plus.json +1 -0
  145. package/themes/light-plus.json +1 -0
  146. package/themes/monokai.json +1 -0
@@ -0,0 +1,360 @@
1
+ import { ChangeSet } from "../text/changeset.js";
2
+ import { Text } from "../text/text.js";
3
+ import { isThemePair, moduleUrl, packBatch } from "./protocol.js";
4
+ const SLICE_MS = 4;
5
+ /**
6
+ * Worker-side tokenizer (D6). Owns a mirror Text, a LineStore, the linked grammar, the theme,
7
+ * and a scheduler. One instance per document. Everything here is plain data plus the S3 runtime;
8
+ * no DOM.
9
+ *
10
+ * Invariants:
11
+ * - `#text` and the LineStore always describe document version `#version`.
12
+ * - `#frontier` is the first line whose end state is unknown. Every line above it has a valid
13
+ * state and tokens for that state.
14
+ * - Messages are handled synchronously in arrival order; the only async point is between slices,
15
+ * so an edit never lands in the middle of a line's tokenization.
16
+ * - `#sent` holds the window lines whose current tokens the main thread has in this epoch.
17
+ *
18
+ * Scheduling, per slice (budget 4 ms):
19
+ * 1. Tokenize from #frontier toward the end of the viewport window. Lines in the window go to
20
+ * a pending batch when their tokens differ from what was last sent.
21
+ * 2. Past the window, keep going until a line's new end state equals its old one (converged)
22
+ * or the document ends. These lines are not sent; they will be when the viewport reaches them.
23
+ * On convergence the frontier jumps to the next dirty line, if any.
24
+ * 3. Flush the pending batch at the end of every slice that produced one, so the visible region
25
+ * paints first and an open template literal never blocks it.
26
+ * 4. When converged and nothing is dirty, run the lazy-grammar loop (pre-scan, load, link,
27
+ * dirty the lines whose misses intersect the new bits; see LazyDocument.settle) and repeat.
28
+ */
29
+ export class TokenEngine {
30
+ #host;
31
+ #rt;
32
+ #version = 0;
33
+ #text = Text.empty;
34
+ #lines = { scopes: [], states: [], misses: new Int32Array(0) };
35
+ #frontier = 1;
36
+ #viewport = { from: 1, to: 1 };
37
+ /** More windows the main thread keeps rows for (inline expansions), [from, to) each. */
38
+ #extra = [];
39
+ #epoch = 0;
40
+ #sentStyles = new Set();
41
+ #sent = new Set();
42
+ #grammar = null;
43
+ #tokenizer = null;
44
+ #scopes;
45
+ #theme = null;
46
+ #manifest = null;
47
+ /** Bumped whenever the root grammar is replaced, so loads for an older one are ignored. */
48
+ #grammarGen = 0;
49
+ #embeds = { unavailable: new Set(), loading: new Set(), fullScan: true, pending: [] };
50
+ #running = false;
51
+ constructor(host, runtime) {
52
+ this.#host = host;
53
+ this.#rt = runtime;
54
+ this.#scopes = runtime.scopeTable();
55
+ }
56
+ handle(m) {
57
+ switch (m.type) {
58
+ case 'open':
59
+ this.#version = m.version;
60
+ this.#text = Text.of(m.text);
61
+ this.#manifest = m.grammars;
62
+ this.#lines = emptyStore(this.#text.lines);
63
+ this.#sent.clear();
64
+ return this.#setLanguage(m.language);
65
+ case 'change':
66
+ return this.#change(m.version, ChangeSet.fromJSON(m.changes));
67
+ case 'viewport':
68
+ this.#viewport = { from: m.from, to: m.to };
69
+ this.#extra = m.extra ?? [];
70
+ for (const l of this.#sent)
71
+ if (!this.#inWindow(l))
72
+ this.#sent.delete(l);
73
+ this.#flush(new Map());
74
+ return this.#kick();
75
+ case 'theme':
76
+ this.#theme = isThemePair(m.theme) ? this.#rt.theme(m.theme.light, m.theme.dark) : this.#rt.theme(m.theme);
77
+ this.#scopes.setTheme(this.#theme);
78
+ this.#epoch = m.epoch;
79
+ this.#sentStyles.clear();
80
+ this.#sent.clear();
81
+ this.#flush(new Map());
82
+ return;
83
+ case 'language':
84
+ this.#epoch = m.epoch;
85
+ this.#sentStyles.clear();
86
+ this.#sent.clear();
87
+ this.#lines = emptyStore(this.#text.lines, this.#lines.scopes);
88
+ return this.#setLanguage(m.language);
89
+ }
90
+ }
91
+ #setLanguage(language) {
92
+ const gen = ++this.#grammarGen;
93
+ this.#grammar = null;
94
+ this.#tokenizer = null;
95
+ this.#frontier = 1;
96
+ this.#embeds = { unavailable: new Set(), loading: new Set(), fullScan: true, pending: [] };
97
+ const manifest = this.#manifest;
98
+ const scope = manifest.languages[language];
99
+ const url = scope === undefined ? undefined : moduleUrl(manifest, scope);
100
+ if (url === undefined)
101
+ return this.#error(`no grammar for language '${language}'`);
102
+ this.#host.load(url).then((mod) => {
103
+ if (gen !== this.#grammarGen)
104
+ return;
105
+ if (!mod)
106
+ return this.#error(`grammar ${scope} did not load`);
107
+ this.#grammar = this.#rt.linkedGrammar(mod, { available: (s) => Object.hasOwn(manifest.modules, s) });
108
+ this.#tokenizer = this.#rt.tokenizer(this.#grammar, this.#scopes);
109
+ this.#kick();
110
+ }, (err) => this.#error(`grammar ${scope} failed to load: ${String(err)}`));
111
+ }
112
+ #change(version, cs) {
113
+ const before = this.#text;
114
+ const after = cs.apply(before);
115
+ const edits = lineEdits(cs, before, after);
116
+ this.#text = after;
117
+ this.#version = version;
118
+ const first = edits[0];
119
+ if (!first)
120
+ return;
121
+ const store = this.#lines;
122
+ // The line at the old frontier holds a state computed from a predecessor that has since
123
+ // changed. Once the frontier moves above it, only an undefined state still marks it dirty.
124
+ if (first.oldFrom < this.#frontier && this.#frontier <= store.states.length)
125
+ store.states[this.#frontier - 1] = undefined;
126
+ for (let k = edits.length - 1; k >= 0; k--) {
127
+ const e = edits[k];
128
+ const del = e.oldTo - e.oldFrom + 1, ins = e.newTo - e.newFrom + 1;
129
+ store.scopes = spliceEmpty(store.scopes, e.oldFrom - 1, del, ins);
130
+ store.states = spliceEmpty(store.states, e.oldFrom - 1, del, ins);
131
+ if (del !== ins)
132
+ store.misses = spliceInt32(store.misses, e.oldFrom - 1, del, ins);
133
+ else
134
+ store.misses.fill(0, e.oldFrom - 1, e.oldFrom - 1 + del);
135
+ }
136
+ this.#frontier = Math.min(this.#frontier, first.newFrom);
137
+ const sent = new Set();
138
+ for (const l of this.#sent) {
139
+ let shift = 0, touched = false;
140
+ for (const e of edits) {
141
+ if (l < e.oldFrom)
142
+ break;
143
+ if (l <= e.oldTo) {
144
+ touched = true;
145
+ break;
146
+ }
147
+ shift += e.newTo - e.newFrom - (e.oldTo - e.oldFrom);
148
+ }
149
+ if (!touched && this.#inWindow(l + shift))
150
+ sent.add(l + shift);
151
+ }
152
+ this.#sent = sent;
153
+ if (!this.#embeds.fullScan)
154
+ for (const e of edits)
155
+ for (let l = e.newFrom; l <= e.newTo; l++)
156
+ this.#embeds.pending.push(this.#lineText(l));
157
+ this.#kick();
158
+ }
159
+ /** Start the slice loop if it is not running. Idempotent. */
160
+ #kick() {
161
+ if (this.#running || !this.#tokenizer)
162
+ return;
163
+ this.#running = true;
164
+ void this.#run();
165
+ }
166
+ async #run() {
167
+ try {
168
+ // Yield first, so a burst of queued messages is applied before any tokenizing.
169
+ await this.#host.yieldNow();
170
+ while (this.#tokenizer && this.#frontier <= this.#text.lines) {
171
+ this.#slice();
172
+ await this.#host.yieldNow();
173
+ }
174
+ }
175
+ catch (err) {
176
+ this.#error(`tokenizer failed: ${String(err)}`);
177
+ return;
178
+ }
179
+ finally {
180
+ this.#running = false;
181
+ }
182
+ this.#settleEmbeds();
183
+ }
184
+ #slice() {
185
+ const tk = this.#tokenizer;
186
+ const { scopes, states, misses } = this.#lines;
187
+ const n = this.#text.lines;
188
+ const deadline = this.#host.now() + SLICE_MS;
189
+ const batch = new Map();
190
+ while (this.#frontier <= n) {
191
+ const i = this.#frontier;
192
+ const r = tk.tokenizeLine(this.#lineText(i), i > 1 ? states[i - 2] : null);
193
+ const old = states[i - 1];
194
+ const prevTokens = scopes[i - 1];
195
+ const tokens = Int32Array.from(r.tokens);
196
+ scopes[i - 1] = tokens;
197
+ states[i - 1] = r.state;
198
+ misses[i - 1] = r.misses;
199
+ if (this.#inWindow(i) && !(this.#sent.has(i) && prevTokens && sameRuns(prevTokens, tokens))) {
200
+ this.#sent.delete(i);
201
+ batch.set(i, tokens);
202
+ }
203
+ this.#frontier = old && r.state.equals(old) ? this.#nextDirty(i + 1) : i + 1;
204
+ if (this.#host.now() >= deadline)
205
+ break;
206
+ }
207
+ this.#flush(batch);
208
+ }
209
+ /** In the mounted window or one of the extra ones. Extras are few (one per open expansion). */
210
+ #inWindow(l) {
211
+ if (l >= this.#viewport.from && l < this.#viewport.to)
212
+ return true;
213
+ for (const [a, b] of this.#extra)
214
+ if (l >= a && l < b)
215
+ return true;
216
+ return false;
217
+ }
218
+ #nextDirty(from) {
219
+ const states = this.#lines.states;
220
+ for (let i = from; i <= states.length; i++)
221
+ if (states[i - 1] === undefined)
222
+ return i;
223
+ return states.length + 1;
224
+ }
225
+ /** Post `batch` plus every tokenized window line the main thread does not have yet. */
226
+ #flush(batch) {
227
+ if (!this.#theme)
228
+ return;
229
+ for (const [from, to] of [[this.#viewport.from, this.#viewport.to], ...this.#extra]) {
230
+ const end = Math.min(to, this.#text.lines + 1);
231
+ for (let l = from; l < end; l++) {
232
+ const s = this.#lines.scopes[l - 1];
233
+ if (s && !this.#sent.has(l))
234
+ batch.set(l, s);
235
+ }
236
+ }
237
+ if (batch.size === 0)
238
+ return;
239
+ const styles = [];
240
+ const entries = [...batch].sort((a, b) => a[0] - b[0]).map(([l, s]) => [l, this.#resolve(s, styles)]);
241
+ for (const [l] of entries)
242
+ this.#sent.add(l);
243
+ const { message, transfer } = packBatch(this.#version, this.#epoch, entries, styles);
244
+ this.#host.post(message, transfer);
245
+ }
246
+ /** Scope runs -> merged style runs, adding first-seen style ids to the batch's StyleDefs.
247
+ * Style 0 is the theme's default (Theme interns its root style first). */
248
+ #resolve(scopeTokens, newStyles) {
249
+ const theme = this.#theme;
250
+ const out = [];
251
+ for (let j = 0; j < scopeTokens.length; j += 2) {
252
+ const style = this.#scopes.styleOf(scopeTokens[j + 1]);
253
+ if (out.length && out[out.length - 1] === style)
254
+ continue;
255
+ out.push(scopeTokens[j], style);
256
+ if (this.#sentStyles.has(style))
257
+ continue;
258
+ this.#sentStyles.add(style);
259
+ newStyles.push({ id: style, ...theme.css(style) });
260
+ }
261
+ return Uint32Array.from(out);
262
+ }
263
+ // The lazy-grammar loop (research/textmate/runtime/lazy.mjs, LazyDocument.settle), run whenever
264
+ // the document has converged.
265
+ #settleEmbeds() {
266
+ const grammar = this.#grammar, tokenizer = this.#tokenizer, manifest = this.#manifest;
267
+ if (!grammar || !tokenizer || !manifest)
268
+ return;
269
+ const embeds = this.#embeds;
270
+ const lines = embeds.fullScan ? this.#text.toString().split('\n') : embeds.pending;
271
+ embeds.fullScan = false;
272
+ embeds.pending = [];
273
+ const wanted = [...grammar.scan(lines)].filter((s) => !embeds.unavailable.has(s) && !embeds.loading.has(s));
274
+ if (!wanted.length)
275
+ return;
276
+ const gen = this.#grammarGen;
277
+ for (const s of wanted)
278
+ embeds.loading.add(s);
279
+ const load = async (s) => {
280
+ const url = moduleUrl(manifest, s);
281
+ return [s, url === undefined ? null : await this.#host.load(url).catch(() => null)];
282
+ };
283
+ void Promise.all(wanted.map(load)).then((mods) => {
284
+ if (gen !== this.#grammarGen)
285
+ return;
286
+ let mask = 0;
287
+ for (const [s, m] of mods) {
288
+ embeds.loading.delete(s);
289
+ if (!m) {
290
+ embeds.unavailable.add(s);
291
+ continue;
292
+ }
293
+ mask |= grammar.bit(s);
294
+ grammar.link(m);
295
+ }
296
+ if (!mask)
297
+ return this.#settleEmbeds();
298
+ tokenizer.invalidate();
299
+ embeds.fullScan = true;
300
+ const { states, misses } = this.#lines;
301
+ for (let i = 0; i < misses.length; i++) {
302
+ if (!(misses[i] & mask))
303
+ continue;
304
+ states[i] = undefined;
305
+ this.#frontier = Math.min(this.#frontier, i + 1);
306
+ }
307
+ if (this.#frontier > this.#text.lines)
308
+ return this.#settleEmbeds();
309
+ this.#kick();
310
+ });
311
+ }
312
+ #lineText(n) {
313
+ const { from, to } = this.#text.line(n);
314
+ return this.#text.slice(from, to);
315
+ }
316
+ #error(message) {
317
+ this.#host.post({ type: 'error', message }, []);
318
+ }
319
+ }
320
+ function emptyStore(lines, keepScopes) {
321
+ return { scopes: keepScopes ?? new Array(lines), states: new Array(lines), misses: new Int32Array(lines) };
322
+ }
323
+ /** Changed line ranges in document order, merged where two changes share a line. */
324
+ function lineEdits(cs, before, after) {
325
+ const out = [];
326
+ cs.iterChanges((fromA, toA, fromB, toB) => {
327
+ const e = { oldFrom: before.lineAt(fromA).number, oldTo: before.lineAt(toA).number, newFrom: after.lineAt(fromB).number, newTo: after.lineAt(toB).number };
328
+ const last = out[out.length - 1];
329
+ if (last && e.oldFrom <= last.oldTo) {
330
+ last.oldTo = e.oldTo;
331
+ last.newTo = e.newTo;
332
+ }
333
+ else
334
+ out.push(e);
335
+ });
336
+ return out;
337
+ }
338
+ /** Replace `del` entries at `start` with `ins` undefined ones. In place unless the insert is too
339
+ * large to pass as spread arguments (a big paste). */
340
+ function spliceEmpty(a, start, del, ins) {
341
+ if (ins <= 4096) {
342
+ a.splice(start, del, ...new Array(ins).fill(undefined));
343
+ return a;
344
+ }
345
+ return [...a.slice(0, start), ...new Array(ins).fill(undefined), ...a.slice(start + del)];
346
+ }
347
+ function spliceInt32(a, start, del, ins) {
348
+ const out = new Int32Array(a.length - del + ins);
349
+ out.set(a.subarray(0, start));
350
+ out.set(a.subarray(start + del), start + ins);
351
+ return out;
352
+ }
353
+ function sameRuns(a, b) {
354
+ if (a.length !== b.length)
355
+ return false;
356
+ for (let i = 0; i < a.length; i++)
357
+ if (a[i] !== b[i])
358
+ return false;
359
+ return true;
360
+ }
@@ -0,0 +1,113 @@
1
+ import type { SerializedChangeSet } from '../text/changeset.ts';
2
+ import type { Version } from '../state/core.ts';
3
+ /**
4
+ * Tokens for one line: [start, styleId, start, styleId, ...], starts strictly increasing from 0,
5
+ * each run extending to the next start or the line's end. Adjacent runs never share a styleId
6
+ * (the engine merges them). styleId 0 is the theme's default style and renders as a bare text
7
+ * node. Starts are UTF-16 offsets within the line.
8
+ */
9
+ export type LineTokens = Uint32Array;
10
+ /** One resolved style, sent once per id per theme epoch. Becomes the CSS class `s<id>`. Colors
11
+ * are CSS colors: a hex color, or `light-dark(light, dark)` under a ThemePair where the two
12
+ * themes disagree. */
13
+ export interface StyleDef {
14
+ readonly id: number;
15
+ readonly fg: string | null;
16
+ readonly bg: string | null;
17
+ /** TextMate fontStyle bits: 1 italic, 2 bold, 4 underline, 8 strikethrough. */
18
+ readonly fontStyle: number;
19
+ }
20
+ /** Compiled VS Code theme (research/textmate/theme output). Opaque to the main thread except for
21
+ * the chrome colors it reads for the editor background, caret, and selection. */
22
+ export interface CompiledTheme {
23
+ readonly colors: readonly string[];
24
+ readonly defaults: readonly [number, number, number];
25
+ readonly trie: unknown;
26
+ readonly chrome: Readonly<Record<string, string>>;
27
+ }
28
+ /**
29
+ * A light and a dark theme used together. Syntax and chrome colors become
30
+ * `light-dark(light, dark)`, so the editor's used `color-scheme` (inherited from the host or an
31
+ * ancestor) picks between them with no retokenize.
32
+ */
33
+ export interface ThemePair {
34
+ readonly light: CompiledTheme;
35
+ readonly dark: CompiledTheme;
36
+ }
37
+ /** What the tokenizer resolves styles against. */
38
+ export type EditorTheme = CompiledTheme | ThemePair;
39
+ export declare function isThemePair(theme: EditorTheme): theme is ThemePair;
40
+ /** `light-dark(a, b)`, or the plain color when both sides agree. A missing side is transparent. */
41
+ export declare function lightDark(a: string | null, b: string | null): string | null;
42
+ /**
43
+ * Grammar manifest. Functions cannot cross postMessage, so the loader is data: root scope per
44
+ * language id, and a module URL per scope. A scope with no URL is unavailable. Module URLs resolve
45
+ * against `base`, normally the manifest's own URL; without it they must be absolute, because
46
+ * the worker would otherwise resolve them against its own location.
47
+ */
48
+ export interface GrammarManifest {
49
+ readonly languages: Readonly<Record<string, string>>;
50
+ readonly modules: Readonly<Record<string, string>>;
51
+ readonly base?: string;
52
+ }
53
+ /** The module URL for `scope`, resolved against the manifest's base, or undefined if unavailable. */
54
+ export declare function moduleUrl(manifest: GrammarManifest, scope: string): string | undefined;
55
+ /**
56
+ * Main -> worker. Messages are processed strictly in order, so a worker message tagged with
57
+ * version N means the worker had applied every change up to N and none after.
58
+ */
59
+ export type ToWorker = {
60
+ readonly type: 'open';
61
+ readonly version: Version;
62
+ readonly text: string;
63
+ readonly language: string;
64
+ readonly grammars: GrammarManifest;
65
+ } | {
66
+ readonly type: 'change';
67
+ readonly version: Version;
68
+ readonly changes: SerializedChangeSet;
69
+ }
70
+ /** Lines [from, to), 1-based, the view has mounted (including overscan). The worker tokenizes
71
+ * through `to` first and only sends tokens for lines in this window. `extra` are more [from, to)
72
+ * windows the main thread keeps rows for, such as an inline expansion's slice of this document;
73
+ * absent when there are none. */
74
+ | {
75
+ readonly type: 'viewport';
76
+ readonly from: number;
77
+ readonly to: number;
78
+ readonly extra?: readonly (readonly [number, number])[];
79
+ } | {
80
+ readonly type: 'theme';
81
+ readonly epoch: number;
82
+ readonly theme: EditorTheme;
83
+ }
84
+ /** A new epoch, shared with theme changes: rows from earlier epochs are never painted again. */
85
+ | {
86
+ readonly type: 'language';
87
+ readonly version: Version;
88
+ readonly epoch: number;
89
+ readonly language: string;
90
+ };
91
+ /**
92
+ * Worker -> main. The three typed arrays are transferred, not copied.
93
+ * `lines[i]` is a 1-based line number in the document at `version`; its tokens are
94
+ * `data.subarray(offsets[i], offsets[i + 1])`.
95
+ */
96
+ export type FromWorker = {
97
+ readonly type: 'tokens';
98
+ readonly version: Version;
99
+ readonly epoch: number;
100
+ readonly lines: Uint32Array;
101
+ readonly offsets: Uint32Array;
102
+ readonly data: Uint32Array;
103
+ /** Styles first referenced in this batch, in this epoch. */
104
+ readonly styles: readonly StyleDef[];
105
+ } | {
106
+ readonly type: 'error';
107
+ readonly message: string;
108
+ };
109
+ /** Worker side: pack per-line token arrays into one transferable batch. */
110
+ export declare function packBatch(version: Version, epoch: number, entries: readonly (readonly [line: number, tokens: ArrayLike<number>])[], styles: readonly StyleDef[]): {
111
+ message: FromWorker;
112
+ transfer: Transferable[];
113
+ };
@@ -0,0 +1,38 @@
1
+ export function isThemePair(theme) {
2
+ return 'light' in theme && 'dark' in theme;
3
+ }
4
+ /** `light-dark(a, b)`, or the plain color when both sides agree. A missing side is transparent. */
5
+ export function lightDark(a, b) {
6
+ if (a === null && b === null)
7
+ return null;
8
+ if (a !== null && b !== null && a.toLowerCase() === b.toLowerCase())
9
+ return a;
10
+ return `light-dark(${a ?? 'transparent'}, ${b ?? 'transparent'})`;
11
+ }
12
+ /** The module URL for `scope`, resolved against the manifest's base, or undefined if unavailable. */
13
+ export function moduleUrl(manifest, scope) {
14
+ const url = manifest.modules[scope];
15
+ if (url === undefined)
16
+ return undefined;
17
+ return manifest.base === undefined ? url : new URL(url, manifest.base).href;
18
+ }
19
+ /** Worker side: pack per-line token arrays into one transferable batch. */
20
+ export function packBatch(version, epoch, entries, styles) {
21
+ const lines = new Uint32Array(entries.length);
22
+ const offsets = new Uint32Array(entries.length + 1);
23
+ let total = 0;
24
+ for (let i = 0; i < entries.length; i++) {
25
+ const [line, tokens] = entries[i];
26
+ lines[i] = line;
27
+ offsets[i] = total;
28
+ total += tokens.length;
29
+ }
30
+ offsets[entries.length] = total;
31
+ const data = new Uint32Array(total);
32
+ for (let i = 0; i < entries.length; i++)
33
+ data.set(entries[i][1], offsets[i]);
34
+ return {
35
+ message: { type: 'tokens', version, epoch, lines, offsets, data, styles },
36
+ transfer: [lines.buffer, offsets.buffer, data.buffer],
37
+ };
38
+ }
@@ -0,0 +1,5 @@
1
+ export interface AsciiRegex {
2
+ readonly source: string;
3
+ readonly flags: string;
4
+ }
5
+ export declare function asciiRegex(source: string, flags: string): AsciiRegex | null;
@@ -0,0 +1,156 @@
1
+ // Rewrite a `v`-flag regex into a regex without `u`/`v` that matches identically on any input
2
+ // made only of ASCII characters. V8 compiles non-Unicode regexes much faster, and the tokenizer
3
+ // picks this variant for a line with no non-ASCII character (see tokenizer.ts, #regex), the first
4
+ // time that regex is needed.
5
+ //
6
+ // On ASCII input every Unicode property has an exact ASCII set, and non-ASCII pattern characters
7
+ // can never match, so the rewrite only has to keep the pattern's structure. It refuses (returns
8
+ // null) anything whose non-Unicode reading could differ: set operations, \q{}, a negated class
9
+ // nested in another class, \P inside a class, \c, \u{} beyond the BMP, non-ASCII characters
10
+ // under `i` (Unicode case folding maps K and ſ onto ASCII letters), a backreference past the last
11
+ // group, and any property it has no table for.
12
+ /** ASCII members of each Unicode property the compiled grammars use. '' never matches. */
13
+ const PROPS = {
14
+ L: 'A-Za-z', Alpha: 'A-Za-z', Upper: 'A-Z', M: '', N: '0-9', Nd: '0-9', Pc: '_',
15
+ AHex: '0-9A-Fa-f', space: '\\t-\\r ',
16
+ };
17
+ export function asciiRegex(source, flags) {
18
+ if (!flags.includes('v'))
19
+ return null;
20
+ const icase = flags.includes('i');
21
+ let groups = 0, maxRef = 0;
22
+ for (let i = 0; i < source.length; i++) {
23
+ if (source[i] === '\\')
24
+ i++;
25
+ else if (source[i] === '(' && (source[i + 1] !== '?' || (source[i + 2] === '<' && source[i + 3] !== '=' && source[i + 3] !== '!')))
26
+ groups++;
27
+ }
28
+ let i = 0;
29
+ const fail = () => { throw FAIL; };
30
+ const hex4 = (cp) => `\\u${cp.toString(16).padStart(4, '0')}`;
31
+ /** One escape at i (pointing at '\'), inside or outside a class. Returns its rewrite. */
32
+ const escape = (inClass) => {
33
+ const c = source[i + 1];
34
+ if (c === undefined)
35
+ fail();
36
+ if (c === 'p' || c === 'P') {
37
+ const m = /^\\[pP]\{([A-Za-z_]+)\}/.exec(source.slice(i));
38
+ if (!m || !(m[1] in PROPS))
39
+ fail();
40
+ i += m[0].length;
41
+ const set = PROPS[m[1]];
42
+ if (inClass) {
43
+ if (c === 'P')
44
+ fail();
45
+ return set;
46
+ }
47
+ return c === 'P' ? `[^${set}]` : `[${set}]`;
48
+ }
49
+ if (c === 'u' && source[i + 2] === '{') {
50
+ const end = source.indexOf('}', i);
51
+ const cp = parseInt(source.slice(i + 3, end), 16);
52
+ if (!(cp <= 0xffff) || (icase && cp > 0x7f))
53
+ fail();
54
+ i = end + 1;
55
+ return hex4(cp);
56
+ }
57
+ if (c === 'u' || c === 'x') {
58
+ const n = c === 'u' ? 4 : 2;
59
+ const cp = parseInt(source.slice(i + 2, i + 2 + n), 16);
60
+ if (icase && cp > 0x7f)
61
+ fail();
62
+ const s = source.slice(i, i + 2 + n);
63
+ i += 2 + n;
64
+ return s;
65
+ }
66
+ if (c === 'q' || c === 'c' || c === 'k')
67
+ fail();
68
+ if (c >= '1' && c <= '9') {
69
+ if (inClass)
70
+ fail();
71
+ const m = /^\d+/.exec(source.slice(i + 1));
72
+ maxRef = Math.max(maxRef, Number(m[0]));
73
+ i += 1 + m[0].length;
74
+ return `\\${m[0]}`;
75
+ }
76
+ if (c === '0' && /[0-9]/.test(source[i + 2] ?? ''))
77
+ fail();
78
+ if (icase && c > '\x7f')
79
+ fail();
80
+ i += 2;
81
+ return `\\${c}`;
82
+ };
83
+ /** A class body starting after '[' (and '^'). Returns members without brackets. */
84
+ const classBody = () => {
85
+ let out = '';
86
+ while (i < source.length && source[i] !== ']') {
87
+ const c = source[i];
88
+ if (c === '[') {
89
+ i++;
90
+ if (source[i] === '^')
91
+ fail();
92
+ out += classBody();
93
+ i++; // ']'
94
+ }
95
+ else if (c === '\\')
96
+ out += escape(true);
97
+ else if ((c === '&' && source[i + 1] === '&') || (c === '-' && source[i + 1] === '-'))
98
+ fail();
99
+ else {
100
+ const cp = source.codePointAt(i);
101
+ if (cp > 0xffff || (icase && cp > 0x7f))
102
+ fail();
103
+ // A '-' or ']' that v mode reads literally must stay literal.
104
+ out += c === '-' && (out === '' || source[i + 1] === ']') ? '\\-' : c;
105
+ i += 1;
106
+ }
107
+ }
108
+ if (source[i] !== ']')
109
+ fail();
110
+ return out;
111
+ };
112
+ let out = '';
113
+ try {
114
+ while (i < source.length) {
115
+ const c = source[i];
116
+ if (c === '\\')
117
+ out += escape(false);
118
+ else if (c === '[') {
119
+ i++;
120
+ const neg = source[i] === '^';
121
+ if (neg)
122
+ i++;
123
+ const body = classBody();
124
+ i++;
125
+ out += `[${neg ? '^' : ''}${body}]`;
126
+ }
127
+ else {
128
+ const cp = source.codePointAt(i);
129
+ if (icase && cp > 0x7f)
130
+ fail();
131
+ // Outside a class an astral character is one unit in v and two without it; neither
132
+ // matches ASCII input, but a quantifier after it would bind differently.
133
+ if (cp > 0xffff)
134
+ fail();
135
+ out += c;
136
+ i += 1;
137
+ }
138
+ }
139
+ }
140
+ catch (e) {
141
+ if (e === FAIL)
142
+ return null;
143
+ throw e;
144
+ }
145
+ if (maxRef > groups)
146
+ return null;
147
+ const outFlags = flags.replace('v', '');
148
+ try {
149
+ new RegExp(out, outFlags);
150
+ }
151
+ catch {
152
+ return null;
153
+ }
154
+ return { source: out, flags: outFlags };
155
+ }
156
+ const FAIL = Symbol('fail');
@@ -0,0 +1,7 @@
1
+ import type { TextMateRuntime } from '../textmate.ts';
2
+ import { LinkedGrammar } from './linker.ts';
3
+ import { ScopeTable } from './scopes.ts';
4
+ import { Theme, ThemePair } from './theme.ts';
5
+ import { Tokenizer } from './tokenizer.ts';
6
+ export { LinkedGrammar, ScopeTable, Theme, ThemePair, Tokenizer };
7
+ export declare const runtime: TextMateRuntime;