@nibling/editor 0.0.0-stage → 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +53 -2
- package/dist/commands/autoclose.d.ts +13 -0
- package/dist/commands/autoclose.js +78 -0
- package/dist/commands/commands.d.ts +104 -0
- package/dist/commands/commands.js +442 -0
- package/dist/commands/comments.d.ts +3 -0
- package/dist/commands/comments.js +9 -0
- package/dist/commands/fold.d.ts +18 -0
- package/dist/commands/fold.js +75 -0
- package/dist/commands/keymap.d.ts +40 -0
- package/dist/commands/keymap.js +118 -0
- package/dist/complete/autocomplete.d.ts +35 -0
- package/dist/complete/autocomplete.js +386 -0
- package/dist/complete/source.d.ts +72 -0
- package/dist/complete/source.js +52 -0
- package/dist/complete/words.d.ts +53 -0
- package/dist/complete/words.js +203 -0
- package/dist/element.d.ts +119 -0
- package/dist/element.js +1011 -0
- package/dist/expand/expansions.d.ts +68 -0
- package/dist/expand/expansions.js +143 -0
- package/dist/expand/projection.d.ts +73 -0
- package/dist/expand/projection.js +189 -0
- package/dist/expand/resolver.d.ts +42 -0
- package/dist/expand/resolver.js +585 -0
- package/dist/expand/slice-tokens.d.ts +25 -0
- package/dist/expand/slice-tokens.js +79 -0
- package/dist/extension.d.ts +58 -0
- package/dist/extension.js +1 -0
- package/dist/find/find.d.ts +12 -0
- package/dist/find/find.js +469 -0
- package/dist/find/search.d.ts +81 -0
- package/dist/find/search.js +308 -0
- package/dist/headless.d.ts +16 -0
- package/dist/headless.js +19 -0
- package/dist/index.d.ts +6 -0
- package/dist/index.js +6 -0
- package/dist/inline-expansion.d.ts +89 -0
- package/dist/inline-expansion.js +324 -0
- package/dist/input/edit-context.d.ts +23 -0
- package/dist/input/edit-context.js +102 -0
- package/dist/input/surface.d.ts +101 -0
- package/dist/input/surface.js +84 -0
- package/dist/input/textarea.d.ts +26 -0
- package/dist/input/textarea.js +125 -0
- package/dist/sheet.d.ts +6 -0
- package/dist/sheet.js +12 -0
- package/dist/state/core.d.ts +73 -0
- package/dist/state/core.js +86 -0
- package/dist/state/fold-ranges.d.ts +41 -0
- package/dist/state/fold-ranges.js +167 -0
- package/dist/state/folds.d.ts +45 -0
- package/dist/state/folds.js +240 -0
- package/dist/state/history.d.ts +25 -0
- package/dist/state/history.js +127 -0
- package/dist/state/indent.d.ts +33 -0
- package/dist/state/indent.js +121 -0
- package/dist/state/selection.d.ts +45 -0
- package/dist/state/selection.js +106 -0
- package/dist/state/whitespace.d.ts +16 -0
- package/dist/state/whitespace.js +54 -0
- package/dist/text/changeset.d.ts +56 -0
- package/dist/text/changeset.js +321 -0
- package/dist/text/text.d.ts +30 -0
- package/dist/text/text.js +321 -0
- package/dist/tokens/bridge.d.ts +107 -0
- package/dist/tokens/bridge.js +461 -0
- package/dist/tokens/engine.d.ts +39 -0
- package/dist/tokens/engine.js +360 -0
- package/dist/tokens/protocol.d.ts +113 -0
- package/dist/tokens/protocol.js +38 -0
- package/dist/tokens/textmate/ascii-regex.d.ts +5 -0
- package/dist/tokens/textmate/ascii-regex.js +156 -0
- package/dist/tokens/textmate/index.d.ts +7 -0
- package/dist/tokens/textmate/index.js +19 -0
- package/dist/tokens/textmate/linker.d.ts +67 -0
- package/dist/tokens/textmate/linker.js +238 -0
- package/dist/tokens/textmate/scopes.d.ts +14 -0
- package/dist/tokens/textmate/scopes.js +54 -0
- package/dist/tokens/textmate/theme.d.ts +54 -0
- package/dist/tokens/textmate/theme.js +138 -0
- package/dist/tokens/textmate/tokenizer.d.ts +29 -0
- package/dist/tokens/textmate/tokenizer.js +684 -0
- package/dist/tokens/textmate.d.ts +49 -0
- package/dist/tokens/textmate.js +4 -0
- package/dist/view/decorations.d.ts +183 -0
- package/dist/view/decorations.js +51 -0
- package/dist/view/guides.d.ts +58 -0
- package/dist/view/guides.js +174 -0
- package/dist/view/heightmap.d.ts +97 -0
- package/dist/view/heightmap.js +229 -0
- package/dist/view/line-dom.d.ts +54 -0
- package/dist/view/line-dom.js +168 -0
- package/dist/view/measure.d.ts +37 -0
- package/dist/view/measure.js +136 -0
- package/dist/view/overlays.d.ts +32 -0
- package/dist/view/overlays.js +135 -0
- package/dist/view/rail.d.ts +27 -0
- package/dist/view/rail.js +232 -0
- package/dist/view/remote.d.ts +24 -0
- package/dist/view/remote.js +143 -0
- package/dist/view/scope-focus.d.ts +27 -0
- package/dist/view/scope-focus.js +255 -0
- package/dist/view/view.d.ts +114 -0
- package/dist/view/view.js +1231 -0
- package/dist/vite.d.ts +12 -0
- package/dist/vite.js +66 -0
- package/dist/worker.d.ts +1 -0
- package/dist/worker.js +64 -0
- package/grammars/astro.mjs +2 -0
- package/grammars/chunks/r-90cb351dba.mjs +1 -0
- package/grammars/chunks/r-b92a500aab.mjs +1 -0
- package/grammars/chunks/r-ced5311992.mjs +1 -0
- package/grammars/chunks/r-ebae91d4ec.mjs +1 -0
- package/grammars/chunks/s-1821e66170.mjs +1 -0
- package/grammars/css.mjs +1 -0
- package/grammars/diff.mjs +1 -0
- package/grammars/docker.mjs +1 -0
- package/grammars/dotenv.mjs +1 -0
- package/grammars/graphql.mjs +1 -0
- package/grammars/html.mjs +2 -0
- package/grammars/ini.mjs +1 -0
- package/grammars/javascript.mjs +5 -0
- package/grammars/json.mjs +1 -0
- package/grammars/json5.mjs +1 -0
- package/grammars/jsonc.mjs +1 -0
- package/grammars/less.mjs +1 -0
- package/grammars/manifest.json +58 -0
- package/grammars/markdown.mjs +1 -0
- package/grammars/mdx.mjs +1 -0
- package/grammars/pug.mjs +1 -0
- package/grammars/scss.mjs +1 -0
- package/grammars/shellscript.mjs +1 -0
- package/grammars/sql.mjs +1 -0
- package/grammars/stylus.mjs +1 -0
- package/grammars/svelte.mjs +1 -0
- package/grammars/toml.mjs +1 -0
- package/grammars/tsx.mjs +5 -0
- package/grammars/typescript.mjs +2 -0
- package/grammars/vue.mjs +1 -0
- package/grammars/yaml.mjs +1 -0
- package/package.json +71 -3
- package/themes/dark-plus.json +1 -0
- package/themes/light-plus.json +1 -0
- package/themes/monokai.json +1 -0
|
@@ -0,0 +1,461 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Main-thread side of the tokenizer (D6, D10). A follower of EditorCore.
|
|
3
|
+
*
|
|
4
|
+
* It owns the only main-thread copy of syntax tokens, and only for the mounted window
|
|
5
|
+
* [#from, #from + #rows.length). The worker owns tokens for the whole document. That keeps the
|
|
6
|
+
* main thread's per-keystroke work proportional to the window (about 100 lines), never to the
|
|
7
|
+
* document.
|
|
8
|
+
*
|
|
9
|
+
* Staleness rule (the answer to "version N tokens reaching a view at N+k"):
|
|
10
|
+
* A row from a worker batch is only ever attached to a line whose text is identical to the
|
|
11
|
+
* text it was computed for. Batches carry their version; the bridge keeps the Text of every version from
|
|
12
|
+
* the newest batch's version to the present (persistent, so this costs only the edited paths),
|
|
13
|
+
* composes the changes since, and for each line in the batch:
|
|
14
|
+
* - touched by those changes -> dropped (a newer batch is already on its way);
|
|
15
|
+
* - untouched -> moved to its current line number and accepted.
|
|
16
|
+
* Lines below an edit whose *start state* changed keep their old, correct-for-their-text-at-N
|
|
17
|
+
* tokens until the worker's newer batch replaces them. That is bounded staleness, never
|
|
18
|
+
* misalignment.
|
|
19
|
+
*
|
|
20
|
+
* Local edits: on every Update the window is re-keyed. Untouched rows shift to their new line
|
|
21
|
+
* numbers. A touched row is mapped through the change: retained characters keep their runs,
|
|
22
|
+
* inserted characters get style 0. That row is marked provisional; the next batch for that line
|
|
23
|
+
* replaces it. Typing never repaints a colored line as plain text.
|
|
24
|
+
*/
|
|
25
|
+
export class TokenBridge {
|
|
26
|
+
#worker;
|
|
27
|
+
#kept = [];
|
|
28
|
+
#from = 1;
|
|
29
|
+
#rows = [];
|
|
30
|
+
/** The range the view mounted; #rows covers it plus #prefetch lines each side. */
|
|
31
|
+
#mounted = { from: 1, to: 1 };
|
|
32
|
+
#prefetch;
|
|
33
|
+
/** Rows that left #rows, by current line number, oldest first. Bounded by #cacheLines. */
|
|
34
|
+
#cache = new Map();
|
|
35
|
+
#cacheLines;
|
|
36
|
+
/** Rows exactly as a worker batch delivered them. Shifting keeps a row's identity; a provisional
|
|
37
|
+
* row mapped through an edit is a new array, so it is never here and never cached. */
|
|
38
|
+
#exact = new WeakSet();
|
|
39
|
+
#epoch = 0;
|
|
40
|
+
/** Epoch of the styles the view currently has. Rows and styles swap together on theme change. */
|
|
41
|
+
#shownEpoch = 0;
|
|
42
|
+
#listeners = [];
|
|
43
|
+
/** Every style of the shown epoch, so a view that subscribes late can build its sheet. */
|
|
44
|
+
#styles = [];
|
|
45
|
+
/** More windows kept besides the mounted one, by owner (an inline expansion's slice). [from, to). */
|
|
46
|
+
#pins = new Map();
|
|
47
|
+
/** Rows for pinned lines outside #rows, by current line number, re-keyed like #rows. */
|
|
48
|
+
#pinned = new Map();
|
|
49
|
+
#errored = false;
|
|
50
|
+
#offCore = () => { };
|
|
51
|
+
/**
|
|
52
|
+
* `prefetch`: lines kept and requested beyond each side of the mounted range, so scrolling lands
|
|
53
|
+
* on colored lines; default, the mounted range's own length. `cacheLines`: how many rows outside
|
|
54
|
+
* that range are kept for scrolling back.
|
|
55
|
+
*/
|
|
56
|
+
constructor(worker, core, init, signal) {
|
|
57
|
+
this.#worker = worker;
|
|
58
|
+
this.#prefetch = init.prefetch;
|
|
59
|
+
this.#cacheLines = init.cacheLines ?? 5000;
|
|
60
|
+
this.#worker.addEventListener('message', (e) => this.#onMessage(e.data));
|
|
61
|
+
this.#worker.postMessage({ type: 'theme', epoch: this.#epoch, theme: init.theme });
|
|
62
|
+
this.#worker.postMessage({
|
|
63
|
+
type: 'open',
|
|
64
|
+
version: core.state.version,
|
|
65
|
+
text: core.state.text.toString(),
|
|
66
|
+
language: init.language,
|
|
67
|
+
grammars: init.grammars,
|
|
68
|
+
});
|
|
69
|
+
this.#kept = [{ version: core.state.version, text: core.state.text }];
|
|
70
|
+
this.#offCore = core.listen((u) => this.#onUpdate(u), signal);
|
|
71
|
+
}
|
|
72
|
+
/** Tokens for a line, or undefined to paint it plain. O(1). */
|
|
73
|
+
lineTokens(line) {
|
|
74
|
+
return this.#rows[line - this.#from] ?? this.#pinned.get(line) ?? this.#cache.get(line);
|
|
75
|
+
}
|
|
76
|
+
/** The styles every row of the shown epoch may use. */
|
|
77
|
+
get styles() {
|
|
78
|
+
return this.#styles;
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Keep rows for lines [from, to) on behalf of `owner`, wherever the mounted window is: the worker
|
|
82
|
+
* sends them, and edits re-key them provisionally as they do the window, so a line typed into
|
|
83
|
+
* through an expansion never flashes plain. Calling it again moves the pin; `from >= to` removes
|
|
84
|
+
* it. Pins also move themselves through each edit, approximately, until their owner moves them.
|
|
85
|
+
*/
|
|
86
|
+
pin(owner, from, to) {
|
|
87
|
+
const cur = this.#pins.get(owner);
|
|
88
|
+
if (from >= to) {
|
|
89
|
+
if (!cur)
|
|
90
|
+
return;
|
|
91
|
+
this.#pins.delete(owner);
|
|
92
|
+
}
|
|
93
|
+
else {
|
|
94
|
+
if (cur && cur.from === from && cur.to === to)
|
|
95
|
+
return;
|
|
96
|
+
this.#pins.set(owner, { from, to });
|
|
97
|
+
}
|
|
98
|
+
const windowTo = this.#from + this.#rows.length;
|
|
99
|
+
for (const [line, row] of this.#pinned) {
|
|
100
|
+
if (this.#inPins(line))
|
|
101
|
+
continue;
|
|
102
|
+
this.#pinned.delete(line);
|
|
103
|
+
if ((line < this.#from || line >= windowTo) && this.#exact.has(row))
|
|
104
|
+
this.#cacheRow(line, row);
|
|
105
|
+
}
|
|
106
|
+
// Rows the cache already has for newly pinned lines are exact; take them.
|
|
107
|
+
for (const { from: a, to: b } of this.#pins.values()) {
|
|
108
|
+
for (let line = a; line < b; line++) {
|
|
109
|
+
const row = this.#cache.get(line);
|
|
110
|
+
if (row === undefined || (line >= this.#from && line < windowTo))
|
|
111
|
+
continue;
|
|
112
|
+
this.#cache.delete(line);
|
|
113
|
+
this.#pinned.set(line, row);
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
this.#postViewport();
|
|
117
|
+
}
|
|
118
|
+
#inPins(line) {
|
|
119
|
+
for (const p of this.#pins.values())
|
|
120
|
+
if (line >= p.from && line < p.to)
|
|
121
|
+
return true;
|
|
122
|
+
return false;
|
|
123
|
+
}
|
|
124
|
+
#postViewport() {
|
|
125
|
+
const from = this.#from;
|
|
126
|
+
const to = this.#from + this.#rows.length;
|
|
127
|
+
if (this.#pins.size === 0)
|
|
128
|
+
return this.#worker.postMessage({ type: 'viewport', from, to });
|
|
129
|
+
const extra = [...this.#pins.values()].map((p) => [p.from, p.to]);
|
|
130
|
+
this.#worker.postMessage({ type: 'viewport', from, to, extra });
|
|
131
|
+
}
|
|
132
|
+
/** Called by the view when its mounted line range changes. The worker is asked for the range
|
|
133
|
+
* plus the prefetch margin; rows leaving that span move to the cache, and rows entering it come
|
|
134
|
+
* back from the cache. */
|
|
135
|
+
setViewport(from, to) {
|
|
136
|
+
if (from === this.#mounted.from && to === this.#mounted.to)
|
|
137
|
+
return; // idempotent
|
|
138
|
+
this.#mounted = { from, to };
|
|
139
|
+
const margin = this.#prefetch ?? Math.max(0, to - from);
|
|
140
|
+
const lines = this.#kept[this.#kept.length - 1].text.lines;
|
|
141
|
+
// The mounted range is honored as asked; only the margin is clamped to the document.
|
|
142
|
+
const wantFrom = Math.min(from, Math.max(1, from - margin));
|
|
143
|
+
const wantTo = Math.max(to, Math.min(lines + 1, to + margin));
|
|
144
|
+
const curTo = this.#from + this.#rows.length;
|
|
145
|
+
if (wantFrom === this.#from && wantTo === curTo)
|
|
146
|
+
return;
|
|
147
|
+
for (let i = 0; i < this.#rows.length; i++) {
|
|
148
|
+
const line = this.#from + i;
|
|
149
|
+
const row = this.#rows[i];
|
|
150
|
+
if (row === undefined || (line >= wantFrom && line < wantTo))
|
|
151
|
+
continue;
|
|
152
|
+
// A pinned line keeps even a provisional row; anything else caches only exact rows.
|
|
153
|
+
if (this.#inPins(line))
|
|
154
|
+
this.#pinned.set(line, row);
|
|
155
|
+
else if (this.#exact.has(row))
|
|
156
|
+
this.#cacheRow(line, row);
|
|
157
|
+
}
|
|
158
|
+
const next = new Array(wantTo - wantFrom);
|
|
159
|
+
for (let line = wantFrom; line < wantTo; line++) {
|
|
160
|
+
const idx = line - this.#from;
|
|
161
|
+
let row = idx >= 0 && idx < this.#rows.length ? this.#rows[idx] : undefined;
|
|
162
|
+
row ??= this.#pinned.get(line) ?? this.#cache.get(line);
|
|
163
|
+
this.#pinned.delete(line);
|
|
164
|
+
this.#cache.delete(line);
|
|
165
|
+
next[line - wantFrom] = row;
|
|
166
|
+
}
|
|
167
|
+
this.#from = wantFrom;
|
|
168
|
+
this.#rows = next;
|
|
169
|
+
this.#postViewport();
|
|
170
|
+
}
|
|
171
|
+
#cacheRow(line, row) {
|
|
172
|
+
this.#cache.delete(line);
|
|
173
|
+
this.#cache.set(line, row);
|
|
174
|
+
while (this.#cache.size > this.#cacheLines)
|
|
175
|
+
this.#cache.delete(this.#cache.keys().next().value);
|
|
176
|
+
}
|
|
177
|
+
setTheme(theme) {
|
|
178
|
+
this.#epoch++;
|
|
179
|
+
this.#worker.postMessage({ type: 'theme', epoch: this.#epoch, theme });
|
|
180
|
+
// Current rows and #shownEpoch's styles keep painting until the first batch tagged with the
|
|
181
|
+
// new epoch arrives (#onMessage), which emits reset: true to swap both at once.
|
|
182
|
+
}
|
|
183
|
+
/** Like a theme change, a new epoch: the old grammar's rows keep painting the same text until
|
|
184
|
+
* the first batch of the new epoch replaces them. */
|
|
185
|
+
setLanguage(language, version) {
|
|
186
|
+
this.#epoch++;
|
|
187
|
+
this.#worker.postMessage({ type: 'language', version, epoch: this.#epoch, language });
|
|
188
|
+
}
|
|
189
|
+
onChange(listener) {
|
|
190
|
+
this.#listeners.push(listener);
|
|
191
|
+
return () => {
|
|
192
|
+
const i = this.#listeners.indexOf(listener);
|
|
193
|
+
if (i >= 0)
|
|
194
|
+
this.#listeners.splice(i, 1);
|
|
195
|
+
};
|
|
196
|
+
}
|
|
197
|
+
#emit(c) {
|
|
198
|
+
for (const l of this.#listeners)
|
|
199
|
+
l(c);
|
|
200
|
+
}
|
|
201
|
+
#onUpdate(u) {
|
|
202
|
+
if (!u.docChanged)
|
|
203
|
+
return;
|
|
204
|
+
this.#worker.postMessage({ type: 'change', version: u.after.version, changes: u.changes.toJSON() });
|
|
205
|
+
this.#kept.push({ version: u.after.version, text: u.after.text, changes: u.changes });
|
|
206
|
+
this.#rows = rekeyWindow(this.#rows, this.#from, u.changes, u.before.text, u.after.text);
|
|
207
|
+
this.#cache = rekeyCache(this.#cache, u.changes, u.before.text, u.after.text);
|
|
208
|
+
if (this.#pins.size > 0) {
|
|
209
|
+
this.#pinned = rekeyRows(this.#pinned, u.changes, u.before.text, u.after.text);
|
|
210
|
+
for (const [owner, p] of this.#pins)
|
|
211
|
+
this.#pins.set(owner, mapLineRange(p, u.changes, u.before.text, u.after.text));
|
|
212
|
+
}
|
|
213
|
+
// Lines whose (text, tokens) identity changed: every line the change's new-document span
|
|
214
|
+
// covers, so the view repaints provisional rows on the same frame as the text.
|
|
215
|
+
const windowTo = this.#from + this.#rows.length;
|
|
216
|
+
const touched = new Set();
|
|
217
|
+
// Same reasoning as the view's rekey loop: a remote burst is hundreds of scattered
|
|
218
|
+
// single-character inserts in one ChangeSet, so this runs hundreds of times per Update, and
|
|
219
|
+
// the second `lineAt` (a tree descent) is redundant whenever the inserted text has no line
|
|
220
|
+
// break to possibly land `toB` on a different line than `fromB`.
|
|
221
|
+
u.changes.iterChanges((_fromA, _toA, fromB, toB, inserted) => {
|
|
222
|
+
const lo = u.after.text.lineAt(fromB).number;
|
|
223
|
+
const hi = toB <= fromB || !inserted.includes('\n') ? lo : u.after.text.lineAt(toB).number;
|
|
224
|
+
for (let n = lo; n <= hi; n++)
|
|
225
|
+
if ((n >= this.#from && n < windowTo) || this.#inPins(n))
|
|
226
|
+
touched.add(n);
|
|
227
|
+
});
|
|
228
|
+
if (touched.size > 0)
|
|
229
|
+
this.#emit({ lines: [...touched].sort((a, b) => a - b), styles: [], reset: false });
|
|
230
|
+
}
|
|
231
|
+
#onMessage(m) {
|
|
232
|
+
if (m.type === 'error') {
|
|
233
|
+
if (!this.#errored) {
|
|
234
|
+
this.#errored = true;
|
|
235
|
+
console.error(m.message);
|
|
236
|
+
}
|
|
237
|
+
return;
|
|
238
|
+
}
|
|
239
|
+
if (m.epoch < this.#epoch)
|
|
240
|
+
return; // superseded theme
|
|
241
|
+
const i = this.#kept.findIndex((k) => k.version === m.version);
|
|
242
|
+
if (i < 0)
|
|
243
|
+
return; // older than anything we still track; a later batch already supersedes it
|
|
244
|
+
const base = this.#kept[i];
|
|
245
|
+
let since = null;
|
|
246
|
+
for (let k = i + 1; k < this.#kept.length; k++) {
|
|
247
|
+
const c = this.#kept[k].changes;
|
|
248
|
+
since = since ? since.compose(c) : c;
|
|
249
|
+
}
|
|
250
|
+
const now = this.#kept[this.#kept.length - 1].text;
|
|
251
|
+
const windowTo = this.#from + this.#rows.length;
|
|
252
|
+
const accepted = [];
|
|
253
|
+
for (let j = 0; j < m.lines.length; j++) {
|
|
254
|
+
const L = m.lines[j];
|
|
255
|
+
const { from, to } = base.text.line(L);
|
|
256
|
+
if (since && since.touches(from, to))
|
|
257
|
+
continue; // a newer batch is already on its way
|
|
258
|
+
const cur = since ? now.lineAt(since.mapPos(from, 1)).number : L;
|
|
259
|
+
if (cur >= this.#from && cur < windowTo) {
|
|
260
|
+
const row = m.data.subarray(m.offsets[j], m.offsets[j + 1]);
|
|
261
|
+
this.#exact.add(row);
|
|
262
|
+
this.#rows[cur - this.#from] = row;
|
|
263
|
+
accepted.push(cur);
|
|
264
|
+
}
|
|
265
|
+
else if (this.#inPins(cur)) {
|
|
266
|
+
const row = m.data.subarray(m.offsets[j], m.offsets[j + 1]);
|
|
267
|
+
this.#exact.add(row);
|
|
268
|
+
this.#pinned.set(cur, row);
|
|
269
|
+
accepted.push(cur);
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
this.#kept = this.#kept.slice(i); // batches only move forward
|
|
273
|
+
const reset = m.epoch !== this.#shownEpoch;
|
|
274
|
+
if (reset) {
|
|
275
|
+
this.#cache.clear();
|
|
276
|
+
// The new epoch's styles replace every earlier one, so a row this batch did not carry would
|
|
277
|
+
// paint old style ids against new styles. Plain until its own batch arrives.
|
|
278
|
+
const fresh = new Set(accepted);
|
|
279
|
+
for (let r = 0; r < this.#rows.length; r++) {
|
|
280
|
+
if (this.#rows[r] !== undefined && !fresh.has(this.#from + r)) {
|
|
281
|
+
this.#rows[r] = undefined;
|
|
282
|
+
accepted.push(this.#from + r);
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
for (const line of this.#pinned.keys()) {
|
|
286
|
+
if (fresh.has(line))
|
|
287
|
+
continue;
|
|
288
|
+
this.#pinned.delete(line);
|
|
289
|
+
accepted.push(line);
|
|
290
|
+
}
|
|
291
|
+
this.#styles = [...m.styles];
|
|
292
|
+
}
|
|
293
|
+
else
|
|
294
|
+
this.#styles.push(...m.styles);
|
|
295
|
+
this.#shownEpoch = m.epoch;
|
|
296
|
+
this.#emit({ lines: accepted, styles: m.styles, reset });
|
|
297
|
+
}
|
|
298
|
+
dispose() {
|
|
299
|
+
this.#offCore();
|
|
300
|
+
this.#worker.terminate?.();
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
/**
|
|
304
|
+
* Pure. Re-key a token window through one ChangeSet. Exported for tests.
|
|
305
|
+
* Rows for lines the change does not touch move to their new line numbers unchanged (same
|
|
306
|
+
* Uint32Array). Touched rows are rebuilt by `mapLineTokens`. Lines created by the change get
|
|
307
|
+
* undefined. Cost: O(window + changed sections).
|
|
308
|
+
*/
|
|
309
|
+
export function rekeyWindow(rows, from, changes, before, after) {
|
|
310
|
+
const n = rows.length;
|
|
311
|
+
const result = new Array(n).fill(undefined);
|
|
312
|
+
for (let j = 0; j < n; j++) {
|
|
313
|
+
const oldLine = from + j;
|
|
314
|
+
if (oldLine < 1 || oldLine > before.lines)
|
|
315
|
+
continue;
|
|
316
|
+
const moved = mapRow(rows[j], oldLine, changes, before, after);
|
|
317
|
+
if (moved && moved[0] >= from && moved[0] < from + n)
|
|
318
|
+
result[moved[0] - from] = moved[1];
|
|
319
|
+
}
|
|
320
|
+
return result;
|
|
321
|
+
}
|
|
322
|
+
/**
|
|
323
|
+
* Where one row goes through a ChangeSet: an untouched line moves with its row unchanged; a touched
|
|
324
|
+
* line that stays one whole line gets a provisional row (mapLineTokens); anything else is dropped.
|
|
325
|
+
*/
|
|
326
|
+
function mapRow(row, oldLine, changes, before, after) {
|
|
327
|
+
const { from: lineFrom, to: lineTo } = before.line(oldLine);
|
|
328
|
+
if (!changes.touches(lineFrom, lineTo))
|
|
329
|
+
return [after.lineAt(changes.mapPos(lineFrom, 1)).number, row];
|
|
330
|
+
if (row === undefined)
|
|
331
|
+
return null;
|
|
332
|
+
// Only rebuild in place when the edit stays inside one line on both sides. A merge or split
|
|
333
|
+
// (the edit reaches across a line break) can't be expressed as a single line's provisional
|
|
334
|
+
// map, so it falls back to undefined until a real batch replaces it.
|
|
335
|
+
// Text typed at either edge of the line joins it, so try both edges mapped outward first. An
|
|
336
|
+
// edge insertion carrying its own line break instead leaves the line's text whole on a line of
|
|
337
|
+
// its own, which the inward mapping finds.
|
|
338
|
+
let nl = exactLine(after, changes.mapPos(lineFrom, -1), changes.mapPos(lineTo, 1));
|
|
339
|
+
nl ??= exactLine(after, changes.mapPos(lineFrom, 1), changes.mapPos(lineTo, -1));
|
|
340
|
+
if (!nl)
|
|
341
|
+
return null;
|
|
342
|
+
return [nl.number, mapLineTokens(row, lineFrom, changes, nl.from, nl.to - nl.from)];
|
|
343
|
+
}
|
|
344
|
+
/** Pure. rekeyWindow for rows kept by line number rather than in a window (the bridge's pins). */
|
|
345
|
+
export function rekeyRows(rows, changes, before, after) {
|
|
346
|
+
const out = new Map();
|
|
347
|
+
for (const [line, row] of rows) {
|
|
348
|
+
if (line > before.lines)
|
|
349
|
+
continue;
|
|
350
|
+
const moved = mapRow(row, line, changes, before, after);
|
|
351
|
+
if (moved?.[1])
|
|
352
|
+
out.set(moved[0], moved[1]);
|
|
353
|
+
}
|
|
354
|
+
return out;
|
|
355
|
+
}
|
|
356
|
+
/** A [from, to) line range carried through a ChangeSet: its first line's start and last line's end. */
|
|
357
|
+
function mapLineRange(r, changes, before, after) {
|
|
358
|
+
const last = Math.min(r.to - 1, before.lines);
|
|
359
|
+
if (last < 1)
|
|
360
|
+
return r;
|
|
361
|
+
const a = after.lineAt(changes.mapPos(before.line(Math.min(r.from, last)).from, -1)).number;
|
|
362
|
+
const b = after.lineAt(changes.mapPos(before.line(last).to, 1)).number;
|
|
363
|
+
return { from: a, to: Math.max(a, b) + 1 };
|
|
364
|
+
}
|
|
365
|
+
/**
|
|
366
|
+
* Pure. Map one line's runs through an in-line edit: retained characters keep their style,
|
|
367
|
+
* inserted characters take style 0, empty runs vanish, equal neighbours merge.
|
|
368
|
+
*/
|
|
369
|
+
export function mapLineTokens(tokens, lineFrom, changes, newLineFrom, newLength) {
|
|
370
|
+
const newLineTo = newLineFrom + newLength;
|
|
371
|
+
const starts = [];
|
|
372
|
+
const styles = [];
|
|
373
|
+
// tokens[i] is the last run start <= relOld; style 0 (bare text) covers anything before the
|
|
374
|
+
// first run or when the line had no runs at all.
|
|
375
|
+
const styleAtOld = (relOld) => {
|
|
376
|
+
let style = 0;
|
|
377
|
+
for (let i = 0; i < tokens.length; i += 2) {
|
|
378
|
+
if (tokens[i] <= relOld)
|
|
379
|
+
style = tokens[i + 1];
|
|
380
|
+
else
|
|
381
|
+
break;
|
|
382
|
+
}
|
|
383
|
+
return style;
|
|
384
|
+
};
|
|
385
|
+
const push = (absPos, style) => {
|
|
386
|
+
const rel = absPos - newLineFrom;
|
|
387
|
+
if (rel < 0 || rel >= newLength)
|
|
388
|
+
return;
|
|
389
|
+
if (styles.length > 0 && styles[styles.length - 1] === style)
|
|
390
|
+
return; // equal neighbours merge
|
|
391
|
+
starts.push(rel);
|
|
392
|
+
styles.push(style);
|
|
393
|
+
};
|
|
394
|
+
let posA = 0;
|
|
395
|
+
let posB = 0;
|
|
396
|
+
const sections = changes.sections;
|
|
397
|
+
for (let i = 0; i < sections.length && posB < newLineTo; i += 2) {
|
|
398
|
+
const len = sections[i];
|
|
399
|
+
const ins = sections[i + 1];
|
|
400
|
+
if (ins < 0) {
|
|
401
|
+
// Retained span: oldAbs [posA, posA+len) <-> newAbs [posB, posB+len), a pure shift.
|
|
402
|
+
const segFrom = Math.max(posB, newLineFrom);
|
|
403
|
+
const segTo = Math.min(posB + len, newLineTo);
|
|
404
|
+
if (segTo > segFrom) {
|
|
405
|
+
const offset = posB - posA;
|
|
406
|
+
const oldSegFrom = segFrom - offset;
|
|
407
|
+
const oldSegTo = segTo - offset;
|
|
408
|
+
push(segFrom, styleAtOld(oldSegFrom - lineFrom));
|
|
409
|
+
for (let k = 0; k < tokens.length; k += 2) {
|
|
410
|
+
const boundaryOld = lineFrom + tokens[k];
|
|
411
|
+
if (boundaryOld > oldSegFrom && boundaryOld < oldSegTo)
|
|
412
|
+
push(boundaryOld + offset, tokens[k + 1]);
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
posB += len;
|
|
416
|
+
}
|
|
417
|
+
else {
|
|
418
|
+
// Changed span: whatever landed here is new text, not retained from the old line.
|
|
419
|
+
const segFrom = Math.max(posB, newLineFrom);
|
|
420
|
+
const segTo = Math.min(posB + ins, newLineTo);
|
|
421
|
+
if (segTo > segFrom)
|
|
422
|
+
push(segFrom, 0);
|
|
423
|
+
posB += ins;
|
|
424
|
+
}
|
|
425
|
+
posA += len;
|
|
426
|
+
}
|
|
427
|
+
const out = new Uint32Array(starts.length * 2);
|
|
428
|
+
for (let i = 0; i < starts.length; i++) {
|
|
429
|
+
out[i * 2] = starts[i];
|
|
430
|
+
out[i * 2 + 1] = styles[i];
|
|
431
|
+
}
|
|
432
|
+
return out;
|
|
433
|
+
}
|
|
434
|
+
/** The line spanning exactly [from, to], or null when that range is not one whole line. */
|
|
435
|
+
function exactLine(text, from, to) {
|
|
436
|
+
const line = text.lineAt(from);
|
|
437
|
+
return line.from === from && line.to === to ? line : null;
|
|
438
|
+
}
|
|
439
|
+
/**
|
|
440
|
+
* Pure. Re-key cached rows through one ChangeSet with the rule worker batches follow: a row whose
|
|
441
|
+
* line the change touches is dropped, and every other row moves to its line's new number. Cached
|
|
442
|
+
* rows are never mapped provisionally, so each one stays exactly what the worker computed for its
|
|
443
|
+
* text. Order is kept, so the oldest entry stays first. O(cache).
|
|
444
|
+
*/
|
|
445
|
+
export function rekeyCache(cache, changes, before, after) {
|
|
446
|
+
const out = new Map();
|
|
447
|
+
if (changes.isEmpty) {
|
|
448
|
+
for (const [line, row] of cache)
|
|
449
|
+
out.set(line, row);
|
|
450
|
+
return out;
|
|
451
|
+
}
|
|
452
|
+
for (const [line, row] of cache) {
|
|
453
|
+
if (line > before.lines)
|
|
454
|
+
continue;
|
|
455
|
+
const { from, to } = before.line(line);
|
|
456
|
+
if (changes.touches(from, to))
|
|
457
|
+
continue;
|
|
458
|
+
out.set(after.lineAt(changes.mapPos(from, 1)).number, row);
|
|
459
|
+
}
|
|
460
|
+
return out;
|
|
461
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { type FromWorker, type ToWorker } from './protocol.ts';
|
|
2
|
+
import type { GrammarModule, TextMateRuntime } from './textmate.ts';
|
|
3
|
+
export interface EngineHost {
|
|
4
|
+
post(message: FromWorker, transfer: Transferable[]): void;
|
|
5
|
+
/** Resolve a module URL to a grammar module (dynamic import in the worker). */
|
|
6
|
+
load(url: string): Promise<GrammarModule | null>;
|
|
7
|
+
/** Yield to the worker's event loop so queued 'change' messages run between slices. */
|
|
8
|
+
yieldNow(): Promise<void>;
|
|
9
|
+
now(): number;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Worker-side tokenizer (D6). Owns a mirror Text, a LineStore, the linked grammar, the theme,
|
|
13
|
+
* and a scheduler. One instance per document. Everything here is plain data plus the S3 runtime;
|
|
14
|
+
* no DOM.
|
|
15
|
+
*
|
|
16
|
+
* Invariants:
|
|
17
|
+
* - `#text` and the LineStore always describe document version `#version`.
|
|
18
|
+
* - `#frontier` is the first line whose end state is unknown. Every line above it has a valid
|
|
19
|
+
* state and tokens for that state.
|
|
20
|
+
* - Messages are handled synchronously in arrival order; the only async point is between slices,
|
|
21
|
+
* so an edit never lands in the middle of a line's tokenization.
|
|
22
|
+
* - `#sent` holds the window lines whose current tokens the main thread has in this epoch.
|
|
23
|
+
*
|
|
24
|
+
* Scheduling, per slice (budget 4 ms):
|
|
25
|
+
* 1. Tokenize from #frontier toward the end of the viewport window. Lines in the window go to
|
|
26
|
+
* a pending batch when their tokens differ from what was last sent.
|
|
27
|
+
* 2. Past the window, keep going until a line's new end state equals its old one (converged)
|
|
28
|
+
* or the document ends. These lines are not sent; they will be when the viewport reaches them.
|
|
29
|
+
* On convergence the frontier jumps to the next dirty line, if any.
|
|
30
|
+
* 3. Flush the pending batch at the end of every slice that produced one, so the visible region
|
|
31
|
+
* paints first and an open template literal never blocks it.
|
|
32
|
+
* 4. When converged and nothing is dirty, run the lazy-grammar loop (pre-scan, load, link,
|
|
33
|
+
* dirty the lines whose misses intersect the new bits; see LazyDocument.settle) and repeat.
|
|
34
|
+
*/
|
|
35
|
+
export declare class TokenEngine {
|
|
36
|
+
#private;
|
|
37
|
+
constructor(host: EngineHost, runtime: TextMateRuntime);
|
|
38
|
+
handle(m: ToWorker): void;
|
|
39
|
+
}
|