@op-ted/efx-syntax-highlighter 0.2.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +63 -22
- package/dist/efx1/efx1Highlighting.js +45 -0
- package/dist/efx1/generated/lexer.js +1772 -0
- package/dist/efx1/lexer.js +26 -0
- package/dist/efx1/parser.js +23 -0
- package/dist/efx1/parser.terms.js +118 -0
- package/dist/efx2/completion/CompletionItems.js +51 -0
- package/dist/efx2/completion/EfxCompletions.js +86 -0
- package/dist/efx2/completion/ExpectedTokens.js +64 -0
- package/dist/efx2/completion/Variables.js +60 -0
- package/dist/efx2/efx2Highlighting.js +82 -0
- package/dist/efx2/generated/completions.js +2318 -0
- package/dist/efx2/generated/lexer.js +3880 -0
- package/dist/efx2/generated/words.js +213 -0
- package/dist/efx2/lexer.js +49 -0
- package/dist/efx2/parser.js +23 -0
- package/dist/efx2/parser.terms.js +318 -0
- package/dist/efxLanguage.js +75 -9
- package/dist/efxSymbols.js +12 -0
- package/dist/efxTheme.js +22 -80
- package/dist/index.js +5 -3
- package/dist/lexer/AntlrLexer.js +427 -0
- package/dist/lexer/EditorLexer.js +134 -0
- package/dist/lexer/EditorParser.js +69 -0
- package/dist/lexer/Lexing.js +238 -0
- package/dist/lexer/ParseRanges.js +22 -0
- package/package.json +19 -8
- package/dist/efx-parser.js +0 -18
- package/dist/efx-parser.terms.js +0 -47
|
@@ -0,0 +1,427 @@
|
|
|
1
|
+
// An ANTLR lexer, run from its table (src/efx<version>/generated/lexer.js, written by `npm run sync`
|
|
2
|
+
// from the lexer antlr-ng generates: scripts/sync/LexerTable.js), as ANTLR's own LexerATNSimulator
|
|
3
|
+
// runs it. In the mode on top of its stack of modes, it follows the network of states of the mode's
|
|
4
|
+
// rules over the text, one character at a time: from every state it is in, each transition the
|
|
5
|
+
// character takes, and on through the transitions that take none (ANTLR's closure). Where a rule
|
|
6
|
+
// reaches its end, a token of that rule could end there. It goes on while some state is left, and
|
|
7
|
+
// keeps the longest token, and of tokens of the same length the one whose rule comes first in the
|
|
8
|
+
// mode, as ANTLR does. The commands of the rule (-> type(...), skip, channel(...), pushMode(...),
|
|
9
|
+
// popMode, mode(...)) are those met on the way, in the rule itself and not in the rules it uses.
|
|
10
|
+
// Where no rule matches, ANTLR's lexer reports an error and goes on after the character where the
|
|
11
|
+
// rules stopped matching; so does this one.
|
|
12
|
+
//
|
|
13
|
+
// The sets of states it reaches are kept, each with where each character leads from it, so that a
|
|
14
|
+
// text is read through states already met (ANTLR's DFA).
|
|
15
|
+
|
|
16
|
+
const EPSILON = 0;
|
|
17
|
+
const RULE = 1;
|
|
18
|
+
const ACTION = 2;
|
|
19
|
+
const ATOM = 3;
|
|
20
|
+
const SET = 4;
|
|
21
|
+
const NOT_SET = 5;
|
|
22
|
+
const WILDCARD = 6;
|
|
23
|
+
|
|
24
|
+
// The end of the text, as ANTLR's EOF and as the reader of the text gives it.
|
|
25
|
+
const END = -1;
|
|
26
|
+
|
|
27
|
+
// The largest character, as ANTLR's lexer reads them.
|
|
28
|
+
const MAX_CHARACTER = 0x10ffff;
|
|
29
|
+
|
|
30
|
+
/** Whether a transition takes a character. */
|
|
31
|
+
function takes(transition, character) {
|
|
32
|
+
const [kind] = transition;
|
|
33
|
+
if (kind === ATOM) {
|
|
34
|
+
return transition[2] === character;
|
|
35
|
+
}
|
|
36
|
+
if (character === END) {
|
|
37
|
+
return false;
|
|
38
|
+
}
|
|
39
|
+
if (kind === WILDCARD) {
|
|
40
|
+
return true;
|
|
41
|
+
}
|
|
42
|
+
let inRanges = false;
|
|
43
|
+
for (let index = 2; index < transition.length; index += 2) {
|
|
44
|
+
inRanges ||= transition[index] <= character && character <= transition[index + 1];
|
|
45
|
+
}
|
|
46
|
+
return kind === SET ? inRanges : kind === NOT_SET && !inRanges && character <= MAX_CHARACTER;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** A list of states to return to, innermost first. */
|
|
50
|
+
class Returns {
|
|
51
|
+
constructor(state, next) {
|
|
52
|
+
this.state = state;
|
|
53
|
+
this.next = next;
|
|
54
|
+
this.key = next === null ? `${state}` : `${state}.${next.key}`;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
// Where a place stands toward a fragment rule whose text is looked for in a token (`spanOf`): before
|
|
59
|
+
// it, inside it, or after it.
|
|
60
|
+
const BEFORE = 0;
|
|
61
|
+
const INSIDE = 1;
|
|
62
|
+
const AFTER = 2;
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* A place in the network: a state, the states the rules it is in return to (null when it is in a
|
|
66
|
+
* token rule itself), the rank of its token rule in the mode, the commands met in that rule so far,
|
|
67
|
+
* and, when a fragment is looked for, where it stands toward it.
|
|
68
|
+
*/
|
|
69
|
+
class Place {
|
|
70
|
+
constructor(state, returns, rank, actions, stage = BEFORE) {
|
|
71
|
+
this.state = state;
|
|
72
|
+
this.returns = returns;
|
|
73
|
+
this.rank = rank;
|
|
74
|
+
this.actions = actions;
|
|
75
|
+
this.stage = stage;
|
|
76
|
+
this.key = `${state}/${returns?.key ?? ''}/${rank}/${actions.join(',')}/${stage}`;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** The same place in another state, in rules that return to other states, with more commands or at another stage. */
|
|
80
|
+
with({ state = this.state, returns = this.returns, actions = this.actions, stage = this.stage }) {
|
|
81
|
+
return new Place(state, returns, this.rank, actions, stage);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* A set of places the lexer can be in, with the token that ends there, if any, and the step each
|
|
87
|
+
* character makes from it: the set it leads to, and for each place there, and then for the token,
|
|
88
|
+
* the place of this set it came from (the first one, in the order ANTLR meets them).
|
|
89
|
+
*/
|
|
90
|
+
class Places {
|
|
91
|
+
constructor(places, accepted, rules, fragmentStart) {
|
|
92
|
+
this.places = places;
|
|
93
|
+
this.accepted = accepted;
|
|
94
|
+
// The rules of the mode, by rank.
|
|
95
|
+
this.rules = rules;
|
|
96
|
+
// The start state of the fragment whose text is looked for (spanOf), if any: the same places
|
|
97
|
+
// step otherwise when it is, as they enter it.
|
|
98
|
+
this.fragmentStart = fragmentStart;
|
|
99
|
+
this.next = new Map();
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// The most sets of places the lexer keeps. Past it, it still reads as ANTLR does, without keeping
|
|
104
|
+
// what it meets: the sets are finite but for rules that call themselves, which make one for each depth.
|
|
105
|
+
const MOST_KEPT = 50000;
|
|
106
|
+
|
|
107
|
+
export class AntlrLexer {
|
|
108
|
+
#table;
|
|
109
|
+
// The modes by name, each with its start state, its rules in their order and its first places.
|
|
110
|
+
#modes;
|
|
111
|
+
// For each rule's stop state, the rule.
|
|
112
|
+
#ruleOfStop = new Map();
|
|
113
|
+
// For each rule's start state, the rule.
|
|
114
|
+
#ruleOfStart = new Map();
|
|
115
|
+
// The places at the start of a token of one rule, where a fragment is looked for, by the rule and
|
|
116
|
+
// the fragment.
|
|
117
|
+
#ruleFirsts = new Map();
|
|
118
|
+
// Every set of places kept, by its contents and the rules it is in, so that one set is kept once.
|
|
119
|
+
#kept = new Map();
|
|
120
|
+
|
|
121
|
+
constructor(table) {
|
|
122
|
+
this.#table = table;
|
|
123
|
+
table.rules.forEach((rule, index) => {
|
|
124
|
+
this.#ruleOfStart.set(rule.start, index);
|
|
125
|
+
this.#ruleOfStop.set(rule.stop, index);
|
|
126
|
+
});
|
|
127
|
+
this.#modes = new Map(table.modes.map(({ name, start }) => {
|
|
128
|
+
const rules = table.states[start].map(([, ruleStart]) => this.#ruleOfStart.get(ruleStart));
|
|
129
|
+
return [name, { start, rules, first: undefined }];
|
|
130
|
+
}));
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* What the lexer reads at the start of a text in a mode, the text read through
|
|
135
|
+
* `characterAt(offset)`, which gives a character code, or -1 past the end: the token, with its
|
|
136
|
+
* rule, its type, whether it is skipped or hidden and the commands it gives the stack of modes, or
|
|
137
|
+
* no token where no rule matches; the length of the token, or of the text ANTLR's lexer skips after
|
|
138
|
+
* an error; and the extent of the text read to decide, from the start.
|
|
139
|
+
*/
|
|
140
|
+
match(mode, characterAt) {
|
|
141
|
+
let places = this.#firstPlaces(mode);
|
|
142
|
+
let token;
|
|
143
|
+
let length = 0;
|
|
144
|
+
let offset = 0;
|
|
145
|
+
let character;
|
|
146
|
+
for (; ; offset += 1) {
|
|
147
|
+
character = characterAt(offset);
|
|
148
|
+
const next = this.#step(places, character);
|
|
149
|
+
if (next === null) {
|
|
150
|
+
break;
|
|
151
|
+
}
|
|
152
|
+
places = next.places;
|
|
153
|
+
if (places.accepted !== undefined) {
|
|
154
|
+
token = places.accepted;
|
|
155
|
+
length = character === END ? offset : offset + 1;
|
|
156
|
+
}
|
|
157
|
+
if (character === END) {
|
|
158
|
+
break;
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
// Where no rule matches, ANTLR's lexer goes on after the character where its rules stopped
|
|
162
|
+
// matching, if any.
|
|
163
|
+
if (token === undefined) {
|
|
164
|
+
length = character === END ? offset : offset + 1;
|
|
165
|
+
}
|
|
166
|
+
return { token, length, extent: offset + 1 };
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
/**
|
|
170
|
+
* Where a token uses a fragment rule itself: the offsets where it enters the fragment the first
|
|
171
|
+
* time and leaves it, or undefined if it does not. The token is the one `match` found at the start
|
|
172
|
+
* of a text, of its rule and length; only its rule is followed. The sets of places are kept as in
|
|
173
|
+
* `match`; each step between them says where each of its places came from, so that the offsets,
|
|
174
|
+
* which the sets do not hold, follow the places through them.
|
|
175
|
+
*/
|
|
176
|
+
spanOf(characterAt, token, fragment) {
|
|
177
|
+
const fragmentStart = this.#table.rules.find(({ name }) => name === fragment).start;
|
|
178
|
+
let places = this.#firstPlacesOf(token.rule, fragmentStart);
|
|
179
|
+
// Where each place, and the token that ends there, entered and left the fragment.
|
|
180
|
+
const spanOfStage = (stage, offset) => (stage === BEFORE ? null : { from: offset, to: stage === AFTER ? offset : undefined });
|
|
181
|
+
let spans = [...places.places, places.accepted].map((place) => (place === undefined ? null : spanOfStage(place.stage, 0)));
|
|
182
|
+
// The token's characters, and the end of the text after them, where `match` took it.
|
|
183
|
+
const steps = characterAt(token.length) === END ? token.length + 1 : token.length;
|
|
184
|
+
for (let offset = 0; offset < steps; offset += 1) {
|
|
185
|
+
const next = this.#step(places, characterAt(offset));
|
|
186
|
+
if (next === null) {
|
|
187
|
+
if (offset === token.length) {
|
|
188
|
+
break;
|
|
189
|
+
}
|
|
190
|
+
return undefined;
|
|
191
|
+
}
|
|
192
|
+
if (offset === token.length && next.places.accepted === undefined) {
|
|
193
|
+
break;
|
|
194
|
+
}
|
|
195
|
+
const before = places.places;
|
|
196
|
+
spans = [...next.places.places, next.places.accepted].map((place, index) => {
|
|
197
|
+
if (place === undefined) {
|
|
198
|
+
return null;
|
|
199
|
+
}
|
|
200
|
+
const origin = next.from[index];
|
|
201
|
+
const span = spans[origin];
|
|
202
|
+
const stage = before[origin].stage;
|
|
203
|
+
if (stage === place.stage) {
|
|
204
|
+
return span;
|
|
205
|
+
}
|
|
206
|
+
const at = Math.min(offset + 1, token.length);
|
|
207
|
+
return { from: stage === BEFORE ? at : span.from, to: place.stage === AFTER ? at : undefined };
|
|
208
|
+
});
|
|
209
|
+
places = next.places;
|
|
210
|
+
}
|
|
211
|
+
const span = places?.accepted === undefined ? undefined : spans.at(-1);
|
|
212
|
+
return span?.to === undefined ? undefined : span;
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
/**
|
|
216
|
+
* Whether a character can start the text of a fragment rule: whether some transition the rule can
|
|
217
|
+
* take first takes it.
|
|
218
|
+
*/
|
|
219
|
+
canStart(fragment, character) {
|
|
220
|
+
const start = this.#table.rules.find(({ name }) => name === fragment).start;
|
|
221
|
+
const seen = new Set();
|
|
222
|
+
const visit = (state) => {
|
|
223
|
+
if (seen.has(state)) {
|
|
224
|
+
return false;
|
|
225
|
+
}
|
|
226
|
+
seen.add(state);
|
|
227
|
+
return this.#table.states[state].some((transition) => {
|
|
228
|
+
const [kind, target] = transition;
|
|
229
|
+
if (kind === EPSILON || kind === ACTION) {
|
|
230
|
+
return visit(target);
|
|
231
|
+
}
|
|
232
|
+
return kind === RULE ? visit(target) : takes(transition, character);
|
|
233
|
+
});
|
|
234
|
+
};
|
|
235
|
+
return visit(start);
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
/** The names of the rules of a mode that use a fragment rule themselves. */
|
|
239
|
+
rulesUsing(mode, fragment) {
|
|
240
|
+
const fragmentStart = this.#table.rules.find(({ name }) => name === fragment)?.start;
|
|
241
|
+
return this.#rulesOf(mode).filter((rule) => this.#statesOf(rule).some((state) =>
|
|
242
|
+
this.#table.states[state].some(([kind, target]) => kind === RULE && target === fragmentStart)))
|
|
243
|
+
.map((rule) => this.#table.rules[rule].name);
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
/** The stack of modes, a list of mode names, after some commands. */
|
|
247
|
+
static modesAfter(stack, commands) {
|
|
248
|
+
let modes = stack;
|
|
249
|
+
for (const [command, mode] of commands) {
|
|
250
|
+
if (command === 'pushMode') {
|
|
251
|
+
modes = [...modes, mode];
|
|
252
|
+
} else if (command === 'popMode') {
|
|
253
|
+
modes = modes.slice(0, -1);
|
|
254
|
+
} else {
|
|
255
|
+
modes = [...modes.slice(0, -1), mode];
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
return modes;
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
#rulesOf(mode) {
|
|
262
|
+
const entry = this.#modes.get(mode);
|
|
263
|
+
if (entry === undefined) {
|
|
264
|
+
throw new RangeError(`The lexer has no mode ${mode}`);
|
|
265
|
+
}
|
|
266
|
+
return entry.rules;
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
/** The states of a rule itself, not those of the rules it uses. */
|
|
270
|
+
#statesOf(rule) {
|
|
271
|
+
const states = new Set();
|
|
272
|
+
const visit = (state) => {
|
|
273
|
+
if (states.has(state)) {
|
|
274
|
+
return;
|
|
275
|
+
}
|
|
276
|
+
states.add(state);
|
|
277
|
+
for (const transition of this.#table.states[state]) {
|
|
278
|
+
visit(transition[0] === RULE ? transition[2] : transition[1]);
|
|
279
|
+
}
|
|
280
|
+
};
|
|
281
|
+
visit(this.#table.rules[rule].start);
|
|
282
|
+
return [...states];
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
/** The places at the start of a token of one rule, where a fragment is looked for, kept once made. */
|
|
286
|
+
#firstPlacesOf(ruleName, fragmentStart) {
|
|
287
|
+
const key = `${ruleName}/${fragmentStart}`;
|
|
288
|
+
if (!this.#ruleFirsts.has(key)) {
|
|
289
|
+
const rule = this.#table.rules.findIndex(({ name }) => name === ruleName);
|
|
290
|
+
const places = [];
|
|
291
|
+
const accepted = [];
|
|
292
|
+
this.#close(new Place(this.#table.rules[rule].start, null, 0, []),
|
|
293
|
+
{ places, accepted, seen: new Set(), rules: [rule], fragmentStart, from: [], acceptedFrom: [], origin: 0 });
|
|
294
|
+
this.#ruleFirsts.set(key, new Places(places, accepted[0], [rule], fragmentStart));
|
|
295
|
+
}
|
|
296
|
+
return this.#ruleFirsts.get(key);
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
/** The places at the start of a token in a mode, kept once made. */
|
|
300
|
+
#firstPlaces(mode) {
|
|
301
|
+
const entry = this.#modes.get(mode);
|
|
302
|
+
if (entry === undefined) {
|
|
303
|
+
throw new RangeError(`The lexer has no mode ${mode}`);
|
|
304
|
+
}
|
|
305
|
+
if (entry.first !== undefined) {
|
|
306
|
+
return entry.first;
|
|
307
|
+
}
|
|
308
|
+
const places = [];
|
|
309
|
+
const accepted = [];
|
|
310
|
+
const seen = new Set();
|
|
311
|
+
this.#table.states[entry.start].forEach(([, ruleStart], rank) => this.#close(new Place(ruleStart, null, rank, []),
|
|
312
|
+
{ places, accepted, seen, rules: entry.rules, from: [], acceptedFrom: [], origin: 0 }));
|
|
313
|
+
entry.first = new Places(places, accepted[0], entry.rules);
|
|
314
|
+
return entry.first;
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
/**
|
|
318
|
+
* The step a character makes from some places: the places it leads to, and where each came from;
|
|
319
|
+
* null if it leads to none. Each step is kept, and each set of places once, to the limit.
|
|
320
|
+
*/
|
|
321
|
+
#step(places, character) {
|
|
322
|
+
const { fragmentStart } = places;
|
|
323
|
+
if (places.next.has(character)) {
|
|
324
|
+
return places.next.get(character);
|
|
325
|
+
}
|
|
326
|
+
const reached = [];
|
|
327
|
+
const accepted = [];
|
|
328
|
+
const closing = { places: reached, accepted, seen: new Set(), rules: places.rules, fragmentStart, from: [],
|
|
329
|
+
acceptedFrom: [], origin: 0 };
|
|
330
|
+
places.places.forEach((place, origin) => {
|
|
331
|
+
closing.origin = origin;
|
|
332
|
+
for (const transition of this.#table.states[place.state]) {
|
|
333
|
+
if (transition[0] >= ATOM && takes(transition, character)) {
|
|
334
|
+
this.#close(place.with({ state: transition[1] }), closing);
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
});
|
|
338
|
+
if (reached.length === 0 && accepted.length === 0) {
|
|
339
|
+
return this.#keep(places, character, null);
|
|
340
|
+
}
|
|
341
|
+
const target = this.#kept.get(this.#keyOf(reached, accepted[0], places.rules, fragmentStart))
|
|
342
|
+
?? new Places(reached, accepted[0], places.rules, fragmentStart);
|
|
343
|
+
return this.#keep(places, character, { places: target, from: [...closing.from, closing.acceptedFrom[0]] });
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
/** Keeps a step, and the set it leads to, while there is room; the step. */
|
|
347
|
+
#keep(places, character, step) {
|
|
348
|
+
if (this.#kept.size < MOST_KEPT) {
|
|
349
|
+
places.next.set(character, step);
|
|
350
|
+
if (step !== null) {
|
|
351
|
+
const { places: kept } = step;
|
|
352
|
+
this.#kept.set(this.#keyOf(kept.places, kept.accepted, kept.rules, kept.fragmentStart), kept);
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
return step;
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
/**
|
|
359
|
+
* What tells a set of places from another: its places in order, its token, the rules it is in, and
|
|
360
|
+
* the fragment looked for, if any.
|
|
361
|
+
*/
|
|
362
|
+
#keyOf(places, accepted, rules, fragmentStart) {
|
|
363
|
+
const token = accepted === undefined ? '' : `${accepted.rule}/${accepted.type}/${accepted.stage}/${JSON.stringify(accepted.commands)}/${accepted.skipped}/${accepted.hidden}`;
|
|
364
|
+
return `${rules.join(',')}|${fragmentStart ?? ''}|${places.map(({ key }) => key).join(';')}|${token}`;
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
/**
|
|
368
|
+
* Adds a place and every place it leads to through transitions that take no character: the places
|
|
369
|
+
* that take one, and the token whose rule ends there, in the order ANTLR meets them.
|
|
370
|
+
*/
|
|
371
|
+
#close(place, closing) {
|
|
372
|
+
const { places, accepted, seen, rules, fragmentStart, from, acceptedFrom, origin } = closing;
|
|
373
|
+
if (seen.has(place.key)) {
|
|
374
|
+
return;
|
|
375
|
+
}
|
|
376
|
+
seen.add(place.key);
|
|
377
|
+
if (this.#ruleOfStop.has(place.state)) {
|
|
378
|
+
if (place.returns === null) {
|
|
379
|
+
accepted.push({ ...this.#token(rules[place.rank], place.actions), stage: place.stage });
|
|
380
|
+
acceptedFrom.push(origin);
|
|
381
|
+
return;
|
|
382
|
+
}
|
|
383
|
+
const leaving = place.stage === INSIDE && place.returns.next === null;
|
|
384
|
+
this.#close(place.with({ state: place.returns.state, returns: place.returns.next,
|
|
385
|
+
stage: leaving ? AFTER : place.stage }), closing);
|
|
386
|
+
return;
|
|
387
|
+
}
|
|
388
|
+
let takesCharacters = false;
|
|
389
|
+
for (const transition of this.#table.states[place.state]) {
|
|
390
|
+
const [kind, target] = transition;
|
|
391
|
+
if (kind === EPSILON) {
|
|
392
|
+
this.#close(place.with({ state: target }), closing);
|
|
393
|
+
} else if (kind === RULE) {
|
|
394
|
+
const entering = place.stage === BEFORE && target === fragmentStart && place.returns === null;
|
|
395
|
+
this.#close(place.with({ state: target, returns: new Returns(transition[2], place.returns),
|
|
396
|
+
stage: entering ? INSIDE : place.stage }), closing);
|
|
397
|
+
} else if (kind === ACTION) {
|
|
398
|
+
const actions = place.returns === null ? [...place.actions, transition[2]] : place.actions;
|
|
399
|
+
this.#close(place.with({ state: target, actions }), closing);
|
|
400
|
+
} else {
|
|
401
|
+
takesCharacters = true;
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
if (takesCharacters) {
|
|
405
|
+
places.push(place);
|
|
406
|
+
from.push(origin);
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
/** The token a rule makes with the commands met on its way. */
|
|
411
|
+
#token(rule, actions) {
|
|
412
|
+
const { name, type } = this.#table.rules[rule];
|
|
413
|
+
const token = { rule: name, type, skipped: false, hidden: false, commands: [] };
|
|
414
|
+
for (const [command, argument] of actions.map((action) => this.#table.actions[action])) {
|
|
415
|
+
if (command === 'type') {
|
|
416
|
+
token.type = argument;
|
|
417
|
+
} else if (command === 'skip') {
|
|
418
|
+
token.skipped = true;
|
|
419
|
+
} else if (command === 'channel') {
|
|
420
|
+
token.hidden = argument !== 0;
|
|
421
|
+
} else {
|
|
422
|
+
token.commands.push(argument === undefined ? [command] : [command, argument]);
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
return token;
|
|
426
|
+
}
|
|
427
|
+
}
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
// An ANTLR lexer (AntlrLexer.js) as the tokenizer of an editor's grammar. The lexer reads the text on
|
|
2
|
+
// its own, from its start, as ANTLR's does (Lexing.js), whatever the parser makes of the tokens: the
|
|
3
|
+
// tokenizer gives the parser the token the lexer read at the parser's position. As a token depends on
|
|
4
|
+
// all the text before it, through the lexer's modes, a parse may reuse an earlier tree only before
|
|
5
|
+
// the first change to the text, and the lexer reads the parts of the document the parse reads as one
|
|
6
|
+
// text (EditorParser.js).
|
|
7
|
+
//
|
|
8
|
+
// A token of a type the grammar declares is that token of the grammar, or the token the grammar
|
|
9
|
+
// names for its type, where a rule of the grammar has the type's name. A token the lexer skips or
|
|
10
|
+
// hides (-> skip, -> channel(...)) is one of the grammar's skip tokens, named for each such rule. A
|
|
11
|
+
// type the grammar takes in no rule is listed as unused: the tokenizer gives no token there, so the
|
|
12
|
+
// parser reports an error, as ANTLR's parser would. A type that is neither declared nor listed is
|
|
13
|
+
// reported when the lexer first makes it. Where the lexer finds no token, the tokenizer gives none
|
|
14
|
+
// either, and the parser reports the error.
|
|
15
|
+
//
|
|
16
|
+
// Where an LR parser must see past a token to know what it is, the grammar may give the token a term
|
|
17
|
+
// of its own for when a given type follows it (`followedBy`): the tokenizer looks at the next token
|
|
18
|
+
// the lexer read, past the ones it skips or hides. A token that holds a comment and more is given as
|
|
19
|
+
// its pieces (Lexing.js): the token, or blanks, before the comment; the comment; the blanks and line
|
|
20
|
+
// ends after it.
|
|
21
|
+
//
|
|
22
|
+
// A bare expression has no lines: in the grammar's `bare` dialect, the lexer starts in the mode of
|
|
23
|
+
// expressions, on top of the default one.
|
|
24
|
+
|
|
25
|
+
import { ExternalTokenizer } from '@lezer/lr';
|
|
26
|
+
import { AntlrLexer } from './AntlrLexer.js';
|
|
27
|
+
import { Lexing } from './Lexing.js';
|
|
28
|
+
import { currentRanges } from './ParseRanges.js';
|
|
29
|
+
|
|
30
|
+
const DEFAULT_MODE = 'DEFAULT_MODE';
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* The tokenizer of a grammar for a lexer's table: `terms`, the grammar's terms;
|
|
34
|
+
* `skipTokens`, the grammar's skip token for each rule the lexer skips or hides, by its name;
|
|
35
|
+
* `typeTokens`, the grammar's token for each type it does not name after the type; `followedBy`, for
|
|
36
|
+
* a type, the grammar's token for it when a token of another type follows, by that type;
|
|
37
|
+
* `unusedTypes`, the types the grammar takes in no rule; `comment`, the lexer's fragment rule of a
|
|
38
|
+
* comment, and `commentToken` and `spaceToken`, the grammar's skip tokens for the comment and the
|
|
39
|
+
* blanks of a token that holds one; `bare`, the dialect of a bare expression and the mode its text starts in.
|
|
40
|
+
*/
|
|
41
|
+
export function editorLexer(table, terms, options) {
|
|
42
|
+
const { skipTokens, typeTokens = {}, followedBy = {}, unusedTypes, comment, commentToken, spaceToken, bare } = options;
|
|
43
|
+
const lexer = new AntlrLexer(table);
|
|
44
|
+
const unused = new Set(unusedTypes);
|
|
45
|
+
// The lexer's reading of each parse's input, by the input and the dialect.
|
|
46
|
+
const readings = new WeakMap();
|
|
47
|
+
|
|
48
|
+
/** The lexer's reading of the input a tokenizer or a context tracker is given, for a parse stack. */
|
|
49
|
+
const lexingOf = (input, stack) => {
|
|
50
|
+
const isBare = bare !== undefined && stack.dialectEnabled(terms[`Dialect_${bare.dialect}`]);
|
|
51
|
+
if (!readings.has(input)) {
|
|
52
|
+
readings.set(input, new Map());
|
|
53
|
+
}
|
|
54
|
+
const byDialect = readings.get(input);
|
|
55
|
+
if (!byDialect.has(isBare)) {
|
|
56
|
+
byDialect.set(isBare, new Lexing(lexer, (from, to) => input.read(from, to), currentRanges(),
|
|
57
|
+
isBare ? [DEFAULT_MODE, bare.mode] : [DEFAULT_MODE], comment));
|
|
58
|
+
}
|
|
59
|
+
return byDialect.get(isBare);
|
|
60
|
+
};
|
|
61
|
+
|
|
62
|
+
/** The grammar's term for a token: a term, or null for a type the grammar takes in no rule. */
|
|
63
|
+
const termOf = (token) => {
|
|
64
|
+
if (token.skipped || token.hidden) {
|
|
65
|
+
const term = terms[skipTokens[token.rule]];
|
|
66
|
+
if (term === undefined) {
|
|
67
|
+
throw new Error(`The lexer skips or hides the tokens of ${token.rule}, which has no skip token in the grammar`);
|
|
68
|
+
}
|
|
69
|
+
return term;
|
|
70
|
+
}
|
|
71
|
+
if (unused.has(token.type)) {
|
|
72
|
+
return null;
|
|
73
|
+
}
|
|
74
|
+
const term = terms[typeTokens[token.type] ?? token.type];
|
|
75
|
+
if (term === undefined) {
|
|
76
|
+
throw new Error(`The lexer makes tokens of type ${token.type}, which the grammar does not declare`);
|
|
77
|
+
}
|
|
78
|
+
return term;
|
|
79
|
+
};
|
|
80
|
+
|
|
81
|
+
/** The type of the token the parser takes after a stretch, past those the lexer skips or hides. */
|
|
82
|
+
const nextType = (lexing, stretch) => {
|
|
83
|
+
let next = lexing.stretchAfter(stretch);
|
|
84
|
+
while (next?.token !== undefined && (next.token.skipped || next.token.hidden)) {
|
|
85
|
+
next = lexing.stretchAfter(next);
|
|
86
|
+
}
|
|
87
|
+
return next?.token?.type;
|
|
88
|
+
};
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* What the tokenizer gives at a position: the stretch the lexer read there, the piece of it that
|
|
92
|
+
* starts there (the whole token, if it has no pieces) and the grammar's term for it; undefined
|
|
93
|
+
* where the lexer read no token, or the type is one the grammar takes in no rule.
|
|
94
|
+
*/
|
|
95
|
+
const tokenAt = (lexing, position) => {
|
|
96
|
+
const { stretch, piece } = lexing.pieceAt(position) ?? {};
|
|
97
|
+
if (stretch?.token === undefined) {
|
|
98
|
+
return undefined;
|
|
99
|
+
}
|
|
100
|
+
if (piece.kind === 'comment') {
|
|
101
|
+
return { stretch, piece, term: terms[commentToken] };
|
|
102
|
+
}
|
|
103
|
+
if (piece.kind === 'space') {
|
|
104
|
+
return { stretch, piece, term: terms[spaceToken] };
|
|
105
|
+
}
|
|
106
|
+
const term = termOf(stretch.token);
|
|
107
|
+
const other = followedBy[stretch.token.type]?.[nextType(lexing, stretch)];
|
|
108
|
+
if (term === null) {
|
|
109
|
+
return undefined;
|
|
110
|
+
}
|
|
111
|
+
return { stretch, piece, term: other === undefined ? term : terms[other] };
|
|
112
|
+
};
|
|
113
|
+
|
|
114
|
+
const lexerModes = new ExternalTokenizer((input, stack) => {
|
|
115
|
+
const lexing = lexingOf(input, stack);
|
|
116
|
+
const offset = lexing.offsetOf(input.pos);
|
|
117
|
+
const found = tokenAt(lexing, offset);
|
|
118
|
+
if (found === undefined) {
|
|
119
|
+
return;
|
|
120
|
+
}
|
|
121
|
+
const { stretch, piece, term } = found;
|
|
122
|
+
// The token depends on the text the lexer read to decide it.
|
|
123
|
+
input.peek(stretch.start + stretch.extent - 1 - offset);
|
|
124
|
+
input.acceptToken(term, piece.end - piece.start);
|
|
125
|
+
}, { contextual: true });
|
|
126
|
+
|
|
127
|
+
/** The name of the mode the lexer is in where a parse stack stands, in the input given. */
|
|
128
|
+
const modeAt = (input, stack) => {
|
|
129
|
+
const lexing = lexingOf(input, stack);
|
|
130
|
+
return lexing.modesAt(lexing.offsetOf(stack.pos))?.at(-1);
|
|
131
|
+
};
|
|
132
|
+
|
|
133
|
+
return { lexerModes, modeAt };
|
|
134
|
+
}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
// The parser of an EFX language in an editor, around the LR parser of its grammar. A token of an
|
|
2
|
+
// ANTLR lexer depends on all the text before it, through the lexer's modes (EditorLexer.js), so:
|
|
3
|
+
//
|
|
4
|
+
// - a parse reuses the parts of an earlier tree only before the first change to the text: the
|
|
5
|
+
// fragments that run on from the start of the text with no change in them. After a change, the same
|
|
6
|
+
// text may be read in other modes, so no part of the earlier tree after it can be trusted;
|
|
7
|
+
// - the lexer must know which parts of the document a parse reads (its ranges), to read them as one
|
|
8
|
+
// text, as ANTLR would: the parse makes them known to it while it runs (ParseRanges.js).
|
|
9
|
+
|
|
10
|
+
import { Parser } from '@lezer/common';
|
|
11
|
+
import { withRanges } from './ParseRanges.js';
|
|
12
|
+
|
|
13
|
+
/** A parse that makes its ranges known to the lexer while it runs. */
|
|
14
|
+
class RangedParse {
|
|
15
|
+
#parse;
|
|
16
|
+
#ranges;
|
|
17
|
+
|
|
18
|
+
constructor(parse, ranges) {
|
|
19
|
+
this.#parse = parse;
|
|
20
|
+
this.#ranges = ranges;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
advance() {
|
|
24
|
+
return withRanges(this.#ranges, () => this.#parse.advance());
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
get parsedPos() {
|
|
28
|
+
return this.#parse.parsedPos;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
stopAt(pos) {
|
|
32
|
+
this.#parse.stopAt(pos);
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
get stoppedAt() {
|
|
36
|
+
return this.#parse.stoppedAt;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export class EditorParser extends Parser {
|
|
41
|
+
#parser;
|
|
42
|
+
|
|
43
|
+
/** The LR parser that does the parsing. */
|
|
44
|
+
constructor(parser) {
|
|
45
|
+
super();
|
|
46
|
+
this.#parser = parser;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
createParse(input, fragments, ranges) {
|
|
50
|
+
return new RangedParse(this.#parser.createParse(input, EditorParser.#unchangedStart(fragments), ranges), ranges);
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* The fragments of earlier trees with no change in or before them: those that run on, one after the
|
|
55
|
+
* other, from the start of the text, where a change would have cut them apart or moved them.
|
|
56
|
+
*/
|
|
57
|
+
static #unchangedStart(fragments) {
|
|
58
|
+
const kept = [];
|
|
59
|
+
let end = 0;
|
|
60
|
+
for (const fragment of fragments) {
|
|
61
|
+
if (fragment.from !== end || fragment.offset !== 0 || fragment.openStart) {
|
|
62
|
+
break;
|
|
63
|
+
}
|
|
64
|
+
kept.push(fragment);
|
|
65
|
+
end = fragment.to;
|
|
66
|
+
}
|
|
67
|
+
return kept;
|
|
68
|
+
}
|
|
69
|
+
}
|