@op-ted/efx-syntax-highlighter 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +286 -286
- package/README.md +77 -41
- package/dist/completion/CompletionItems.js +51 -0
- package/dist/completion/EfxCompletions.js +86 -0
- package/dist/completion/ExpectedTokens.js +64 -0
- package/dist/completion/Variables.js +60 -0
- package/dist/efx-parser.js +20 -12
- package/dist/efx-parser.terms.js +317 -45
- package/dist/efxHighlighting.js +82 -0
- package/dist/efxLanguage.js +29 -12
- package/dist/efxSymbols.js +12 -0
- package/dist/efxTheme.js +46 -104
- package/dist/generated/completions.js +2318 -0
- package/dist/generated/words.js +215 -0
- package/dist/index.js +5 -3
- package/dist/lexer/ExpressionMode.js +44 -0
- package/dist/lexer/FormatMode.js +31 -0
- package/dist/lexer/IdentifierMode.js +12 -0
- package/dist/lexer/LabelMode.js +27 -0
- package/dist/lexer/LexerContext.js +76 -0
- package/dist/lexer/LexerInput.js +188 -0
- package/dist/lexer/LexerMode.js +24 -0
- package/dist/lexer/LexerModes.js +50 -0
- package/dist/lexer/LineMode.js +85 -0
- package/dist/lexer/ReservedWords.js +75 -0
- package/dist/lexer/StageHeaderMode.js +12 -0
- package/dist/lexer/TemplateMode.js +77 -0
- package/dist/lexer/TemplateStartMode.js +25 -0
- package/package.json +63 -43
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
// Generated by `npm run sync` from EfxLexer.g4 at OP-TED/eForms-SDK@b43466c004b0865a979c9ffd2cc84781bf081fe0. Do not edit.
|
|
2
|
+
|
|
3
|
+
// The words the lexer reads as AssetType in a label block.
|
|
4
|
+
export const assetTypes = new Set([
|
|
5
|
+
'auxiliary',
|
|
6
|
+
'business-term',
|
|
7
|
+
'code',
|
|
8
|
+
'codelist',
|
|
9
|
+
'expression',
|
|
10
|
+
'field',
|
|
11
|
+
'group',
|
|
12
|
+
'indicator',
|
|
13
|
+
'interval',
|
|
14
|
+
'node',
|
|
15
|
+
'notice',
|
|
16
|
+
'pattern',
|
|
17
|
+
'rule',
|
|
18
|
+
'view',
|
|
19
|
+
]);
|
|
20
|
+
|
|
21
|
+
// The words the lexer reads as LabelType in a label block.
|
|
22
|
+
export const labelTypes = new Set([
|
|
23
|
+
'description',
|
|
24
|
+
'hint',
|
|
25
|
+
'name',
|
|
26
|
+
'template',
|
|
27
|
+
'text',
|
|
28
|
+
'when-false',
|
|
29
|
+
'when-true',
|
|
30
|
+
]);
|
|
31
|
+
|
|
32
|
+
// The token the lexer makes of each word it reserves at the start of a line or between its parts.
|
|
33
|
+
export const lineTokens = new Map([
|
|
34
|
+
['DISPLAY', 'Display'],
|
|
35
|
+
['ENDPOINT', 'Endpoint'],
|
|
36
|
+
['INVOKE', 'Invoke'],
|
|
37
|
+
['LET', 'Let'],
|
|
38
|
+
['OTHERWISE', 'Otherwise'],
|
|
39
|
+
['WHEN', 'When'],
|
|
40
|
+
['WITH', 'With'],
|
|
41
|
+
['OTHERWISE DISPLAY', 'Otherwise'],
|
|
42
|
+
]);
|
|
43
|
+
|
|
44
|
+
// The token the lexer makes of each word it reads where it skips whitespace and finds none, as directly after a `;`.
|
|
45
|
+
export const skipWhitespaceTokens = new Map([
|
|
46
|
+
['LET', 'Let'],
|
|
47
|
+
['WITH', 'With'],
|
|
48
|
+
]);
|
|
49
|
+
|
|
50
|
+
// The token the lexer makes of each word it reserves in template text.
|
|
51
|
+
export const templateTokens = new Map([
|
|
52
|
+
['INVOKE', 'Invoke'],
|
|
53
|
+
['OTHERWISE', 'Otherwise'],
|
|
54
|
+
['WHEN', 'When'],
|
|
55
|
+
['value', 'ValueKeyword'],
|
|
56
|
+
['OTHERWISE DISPLAY', 'Otherwise'],
|
|
57
|
+
]);
|
|
58
|
+
|
|
59
|
+
// The token the lexer makes of each word of the format options after the pipe of an expression block.
|
|
60
|
+
export const formatTokens = new Map([
|
|
61
|
+
['long', 'FormatLong'],
|
|
62
|
+
['medium', 'FormatMedium'],
|
|
63
|
+
['no-formatting', 'FormatNoFormatting'],
|
|
64
|
+
['no-unit', 'FormatNoUnit'],
|
|
65
|
+
['short', 'FormatShort'],
|
|
66
|
+
]);
|
|
67
|
+
|
|
68
|
+
// The token the lexer makes of each word it reserves in an expression.
|
|
69
|
+
export const expressionTokens = new Map([
|
|
70
|
+
['ALWAYS', 'Always'],
|
|
71
|
+
['ANY', 'Any'],
|
|
72
|
+
['API', 'Api'],
|
|
73
|
+
['AS', 'As'],
|
|
74
|
+
['ASSERT', 'Assert'],
|
|
75
|
+
['AT', 'AT'],
|
|
76
|
+
['BY', 'By'],
|
|
77
|
+
['CALL', 'Call'],
|
|
78
|
+
['COMPUTE', 'Compute'],
|
|
79
|
+
['DISPLAY', 'Display'],
|
|
80
|
+
['ENDPOINT', 'Endpoint'],
|
|
81
|
+
['ERROR', 'Error'],
|
|
82
|
+
['EVERY', 'EVERY'],
|
|
83
|
+
['FALSE', 'False'],
|
|
84
|
+
['FOR', 'FOR'],
|
|
85
|
+
['IN', 'IN'],
|
|
86
|
+
['INDEX', 'Index'],
|
|
87
|
+
['INFO', 'Info'],
|
|
88
|
+
['INVOKE', 'Invoke'],
|
|
89
|
+
['NEVER', 'Never'],
|
|
90
|
+
['ON', 'On'],
|
|
91
|
+
['OTHERWISE', 'Otherwise'],
|
|
92
|
+
['REJECT', 'Reject'],
|
|
93
|
+
['REPORT', 'Report'],
|
|
94
|
+
['SCOPE', 'Scope'],
|
|
95
|
+
['SELECT', 'Select'],
|
|
96
|
+
['TRUE', 'True'],
|
|
97
|
+
['WARN', 'Warn'],
|
|
98
|
+
['WARNING', 'Warning'],
|
|
99
|
+
['WHEN', 'When'],
|
|
100
|
+
['WITH', 'With'],
|
|
101
|
+
['absolute', 'AbsoluteFunction'],
|
|
102
|
+
['add-duration', 'AddDurationFunction'],
|
|
103
|
+
['and', 'And'],
|
|
104
|
+
['at', 'At'],
|
|
105
|
+
['average', 'AverageFunction'],
|
|
106
|
+
['capitalize-first', 'CapitalizeFirstFunction'],
|
|
107
|
+
['code', 'Code'],
|
|
108
|
+
['codelist', 'Codelist'],
|
|
109
|
+
['concat', 'ConcatFunction'],
|
|
110
|
+
['contains', 'ContainsFunction'],
|
|
111
|
+
['context', 'ContextType'],
|
|
112
|
+
['count', 'CountFunction'],
|
|
113
|
+
['count-duplicates', 'CountDuplicatesFunction'],
|
|
114
|
+
['current-date', 'CurrentDateFunction'],
|
|
115
|
+
['current-time', 'CurrentTimeFunction'],
|
|
116
|
+
['date', 'Date'],
|
|
117
|
+
['day', 'DayFunction'],
|
|
118
|
+
['day-time-duration', 'DayTimeDurationFunction'],
|
|
119
|
+
['days', 'DaysFunction'],
|
|
120
|
+
['distinct', 'Distinct'],
|
|
121
|
+
['distinct-values', 'DistinctValuesFunction'],
|
|
122
|
+
['duplicates', 'Duplicates'],
|
|
123
|
+
['duration', 'Duration'],
|
|
124
|
+
['dynamic', 'Dynamic'],
|
|
125
|
+
['else', 'Else'],
|
|
126
|
+
['empty', 'Empty'],
|
|
127
|
+
['ends-with', 'EndsWithFunction'],
|
|
128
|
+
['every', 'Every'],
|
|
129
|
+
['for', 'For'],
|
|
130
|
+
['format-long', 'FormatLongFunction'],
|
|
131
|
+
['format-medium', 'FormatMediumFunction'],
|
|
132
|
+
['format-number', 'FormatNumberFunction'],
|
|
133
|
+
['format-short', 'FormatShortFunction'],
|
|
134
|
+
['get-duplicates', 'GetDuplicatesFunction'],
|
|
135
|
+
['has', 'Has'],
|
|
136
|
+
['hours', 'HoursFunction'],
|
|
137
|
+
['if', 'If'],
|
|
138
|
+
['in', 'In'],
|
|
139
|
+
['index-of', 'IndexOfFunction'],
|
|
140
|
+
['index-of-substring', 'IndexOfSubstringFunction'],
|
|
141
|
+
['indicator', 'Indicator'],
|
|
142
|
+
['is', 'Is'],
|
|
143
|
+
['isDisclosed', 'IsDisclosed'],
|
|
144
|
+
['isMasked', 'IsMasked'],
|
|
145
|
+
['isWithheld', 'IsWithheld'],
|
|
146
|
+
['isWithholdable', 'IsWithholdable'],
|
|
147
|
+
['justificationCode', 'JustificationCode'],
|
|
148
|
+
['justificationDescription', 'JustificationDescription'],
|
|
149
|
+
['like', 'Like'],
|
|
150
|
+
['lower-case', 'LowerCaseFunction'],
|
|
151
|
+
['max', 'MaxFunction'],
|
|
152
|
+
['min', 'MinFunction'],
|
|
153
|
+
['minutes', 'MinutesFunction'],
|
|
154
|
+
['month', 'MonthFunction'],
|
|
155
|
+
['months', 'MonthsFunction'],
|
|
156
|
+
['no', 'No'],
|
|
157
|
+
['normalize-space', 'NormalizeSpaceFunction'],
|
|
158
|
+
['not', 'Not'],
|
|
159
|
+
['number', 'Number'],
|
|
160
|
+
['or', 'Or'],
|
|
161
|
+
['pad-left', 'PadLeftFunction'],
|
|
162
|
+
['pad-right', 'PadRightFunction'],
|
|
163
|
+
['preferred-language', 'PreferredLanguageFunction'],
|
|
164
|
+
['preferred-language-text', 'PreferredLanguageTextFunction'],
|
|
165
|
+
['preferredLanguage', 'PreferredLanguage'],
|
|
166
|
+
['preferredLanguageText', 'PreferredLanguageText'],
|
|
167
|
+
['present', 'Present'],
|
|
168
|
+
['privacyCode', 'PrivacyCode'],
|
|
169
|
+
['publicationDate', 'PublicationDate'],
|
|
170
|
+
['rawValue', 'RawValue'],
|
|
171
|
+
['repeat', 'RepeatFunction'],
|
|
172
|
+
['replace', 'ReplaceFunction'],
|
|
173
|
+
['replace-regex', 'ReplaceRegexFunction'],
|
|
174
|
+
['return', 'Return'],
|
|
175
|
+
['reverse', 'ReverseFunction'],
|
|
176
|
+
['round', 'RoundFunction'],
|
|
177
|
+
['round-down', 'RoundDownFunction'],
|
|
178
|
+
['round-up', 'RoundUpFunction'],
|
|
179
|
+
['satisfies', 'Satisfies'],
|
|
180
|
+
['seconds', 'SecondsFunction'],
|
|
181
|
+
['sequence-equal', 'SequenceEqualFunction'],
|
|
182
|
+
['some', 'Some'],
|
|
183
|
+
['sort', 'SortFunction'],
|
|
184
|
+
['split', 'SplitFunction'],
|
|
185
|
+
['starts-with', 'StartsWithFunction'],
|
|
186
|
+
['string', 'Text'],
|
|
187
|
+
['string-join', 'StringJoinFunction'],
|
|
188
|
+
['string-length', 'StringLengthFunction'],
|
|
189
|
+
['subsequence', 'SubsequenceFunction'],
|
|
190
|
+
['substring', 'SubstringFunction'],
|
|
191
|
+
['substring-after', 'SubstringAfterFunction'],
|
|
192
|
+
['substring-before', 'SubstringBeforeFunction'],
|
|
193
|
+
['subtract-duration', 'SubtractDurationFunction'],
|
|
194
|
+
['sum', 'SumFunction'],
|
|
195
|
+
['template', 'Template'],
|
|
196
|
+
['text', 'Text'],
|
|
197
|
+
['then', 'Then'],
|
|
198
|
+
['time', 'Time'],
|
|
199
|
+
['trim', 'TrimFunction'],
|
|
200
|
+
['trim-left', 'TrimLeftFunction'],
|
|
201
|
+
['trim-right', 'TrimRightFunction'],
|
|
202
|
+
['unique', 'Unique'],
|
|
203
|
+
['upper-case', 'UpperCaseFunction'],
|
|
204
|
+
['url-encode', 'UrlEncodeFunction'],
|
|
205
|
+
['value-except', 'ValueExceptFunction'],
|
|
206
|
+
['value-intersect', 'ValueIntersectFunction'],
|
|
207
|
+
['value-union', 'ValueUnionFunction'],
|
|
208
|
+
['wasWithheld', 'WasWithheld'],
|
|
209
|
+
['year', 'YearFunction'],
|
|
210
|
+
['year-month-duration', 'YearMonthDurationFunction'],
|
|
211
|
+
['years', 'YearsFunction'],
|
|
212
|
+
['OTHERWISE ASSERT', 'OtherwiseAssert'],
|
|
213
|
+
['OTHERWISE DISPLAY', 'Otherwise'],
|
|
214
|
+
['OTHERWISE REPORT', 'OtherwiseReport'],
|
|
215
|
+
]);
|
package/dist/index.js
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
-
//
|
|
2
|
-
|
|
3
|
-
export {
|
|
1
|
+
// The package's public surface: EFX support for a CodeMirror 6 editor (efx), the field and node ids
|
|
2
|
+
// it offers (efxSymbols), the EFX colours (efxHighlightStyle) and a look for the editor (efxTheme).
|
|
3
|
+
export { efx } from './efxLanguage.js';
|
|
4
|
+
export { efxSymbols } from './efxSymbols.js';
|
|
5
|
+
export { efxHighlightStyle, efxTheme } from './efxTheme.js';
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
// EXPRESSION mode, and every place the other modes do not claim. All whitespace, line ends and
|
|
2
|
+
// comments are hidden (WS, ExpressionComment). A reserved word is its token wherever the parser can
|
|
3
|
+
// take that token or a name, as the lexer reads it; any other word is the grammar's own Identifier.
|
|
4
|
+
// A `/` before an `@`, with only hidden text between, is the lexer's AttributeSlash.
|
|
5
|
+
|
|
6
|
+
import * as terms from '../efx-parser.terms.js';
|
|
7
|
+
import { expressionTokens } from '../generated/words.js';
|
|
8
|
+
import { LexerMode } from './LexerMode.js';
|
|
9
|
+
import { ReservedWords } from './ReservedWords.js';
|
|
10
|
+
|
|
11
|
+
export class ExpressionMode extends LexerMode {
|
|
12
|
+
static #words = new ReservedWords(expressionTokens);
|
|
13
|
+
|
|
14
|
+
/** Everywhere: the modes before it in LexerModes claim their places first. */
|
|
15
|
+
isActive() {
|
|
16
|
+
return true;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
hide(input) {
|
|
20
|
+
if (input.isComment(0)) {
|
|
21
|
+
return input.take(terms.Comment, input.commentLength(0));
|
|
22
|
+
}
|
|
23
|
+
return input.take(terms.space, input.whitespace(0));
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
read(input, stack) {
|
|
27
|
+
if (input.isSlash(0) && input.isAt(1 + input.hiddenLength(1))) {
|
|
28
|
+
input.accept(terms.AttributeSlash, 1);
|
|
29
|
+
return;
|
|
30
|
+
}
|
|
31
|
+
if (input.isSemicolon(0)) {
|
|
32
|
+
input.accept(terms.Semicolon, 1);
|
|
33
|
+
return;
|
|
34
|
+
}
|
|
35
|
+
const word = ExpressionMode.#words.wordAt(input, 0);
|
|
36
|
+
if (word !== undefined && (stack.canShift(word.token) || stack.canShift(terms.Identifier))) {
|
|
37
|
+
input.accept(word.token, word.length);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
words() {
|
|
42
|
+
return ExpressionMode.#words.entries();
|
|
43
|
+
}
|
|
44
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
// FORMAT mode, the format options after the pipe of an expression block: blanks are hidden
|
|
2
|
+
// (FormatWhitespace), and a word is one of the format words.
|
|
3
|
+
|
|
4
|
+
import * as terms from '../efx-parser.terms.js';
|
|
5
|
+
import { formatTokens } from '../generated/words.js';
|
|
6
|
+
import { LexerMode } from './LexerMode.js';
|
|
7
|
+
import { ReservedWords } from './ReservedWords.js';
|
|
8
|
+
|
|
9
|
+
export class FormatMode extends LexerMode {
|
|
10
|
+
static #words = new ReservedWords(formatTokens);
|
|
11
|
+
|
|
12
|
+
/** Where the parser can take a number of decimals or `no-unit`. */
|
|
13
|
+
isActive(stack) {
|
|
14
|
+
return stack.canShift(terms.FormatDecimals) || stack.canShift(terms.FormatNoUnit);
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
hide(input) {
|
|
18
|
+
return input.take(terms.space, input.blanks(0));
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
read(input) {
|
|
22
|
+
const word = FormatMode.#words.wordAt(input, 0);
|
|
23
|
+
if (word !== undefined) {
|
|
24
|
+
input.accept(word.token, word.length);
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
words() {
|
|
29
|
+
return FormatMode.#words.entries();
|
|
30
|
+
}
|
|
31
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
// IDENTIFIER mode, after `$`, `@`, `#`, `?` or `...`: the name that follows, never a reserved word,
|
|
2
|
+
// is the grammar's own UnquotedIdentifier. Nothing is hidden.
|
|
3
|
+
|
|
4
|
+
import * as terms from '../efx-parser.terms.js';
|
|
5
|
+
import { LexerMode } from './LexerMode.js';
|
|
6
|
+
|
|
7
|
+
export class IdentifierMode extends LexerMode {
|
|
8
|
+
/** Where the parser can take the name after a prefix. */
|
|
9
|
+
isActive(stack) {
|
|
10
|
+
return stack.canShift(terms.UnquotedIdentifier);
|
|
11
|
+
}
|
|
12
|
+
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
// LABEL mode, in a label block: nothing is hidden. A word is an asset type, a label type or another
|
|
2
|
+
// asset id, OtherAssetId: [a-z]+ ([-.] [a-z0-9]+)*, the longest match, which the lexer lists after
|
|
3
|
+
// the two types.
|
|
4
|
+
|
|
5
|
+
import * as terms from '../efx-parser.terms.js';
|
|
6
|
+
import { assetTypes, labelTypes } from '../generated/words.js';
|
|
7
|
+
import { LexerMode } from './LexerMode.js';
|
|
8
|
+
|
|
9
|
+
const LABEL_TOKENS = [terms.AssetType, terms.LabelType, terms.OtherAssetId];
|
|
10
|
+
|
|
11
|
+
export class LabelMode extends LexerMode {
|
|
12
|
+
/** Where the parser can take an asset type, a label type or an asset id. */
|
|
13
|
+
isActive(stack) {
|
|
14
|
+
return LABEL_TOKENS.some((token) => stack.canShift(token));
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
read(input) {
|
|
18
|
+
const length = input.assetIdLength();
|
|
19
|
+
const word = input.text(0, length);
|
|
20
|
+
const token = assetTypes.has(word) ? terms.AssetType : labelTypes.has(word) ? terms.LabelType : terms.OtherAssetId;
|
|
21
|
+
input.take(token, length);
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
words() {
|
|
25
|
+
return [...[...assetTypes].map((word) => [word, terms.AssetType]), ...[...labelTypes].map((word) => [word, terms.LabelType])];
|
|
26
|
+
}
|
|
27
|
+
}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
// What the parse state cannot tell about the lexer's mode, kept as the parser goes: where the last
|
|
2
|
+
// token leaves the lexer, in the places where the tokens the parser can take do not say it.
|
|
3
|
+
// - LINE_START: after a `;`, a line end, a header or an include, the lexer is at the start of a line
|
|
4
|
+
// (DEFAULT or SKIP_WHITESPACE). After a rule of a rules file the parser can take the same tokens,
|
|
5
|
+
// but the lexer is still in EXPRESSION mode, until a `;`.
|
|
6
|
+
// - TEMPLATE_END: after a template is closed, the lexer is still in TEMPLATE mode, and reads a line
|
|
7
|
+
// end as the CRLF that ends an EFX-1 line (TemplateMode.js).
|
|
8
|
+
// - TEMPLATE_SEMICOLON: after the `;` that ends a template, the lexer is at the start of a line, as
|
|
9
|
+
// at LINE_START, but the `;` takes the blanks and the comment after it on its line (EndTemplate),
|
|
10
|
+
// so that comment needs no line end (LineMode.js).
|
|
11
|
+
// - BEFORE_FIRST_WORD: after the indentation or the outline number of a line, before its first word.
|
|
12
|
+
// - INVOKE_LINE: in a line that starts with INVOKE. At the start of a line the lexer reads INVOKE in
|
|
13
|
+
// DEFAULT mode, which pushes EXPRESSION mode only (Invoke), so the line's `;` takes the lexer back
|
|
14
|
+
// to DEFAULT mode, where any other `;` takes it to SKIP_WHITESPACE.
|
|
15
|
+
// - INVOKE_SEMICOLON: after that `;`, the lexer is at the start of a line in DEFAULT mode
|
|
16
|
+
// (LineMode.js).
|
|
17
|
+
// The skip tokens leave it as it is, and the tokens of a line that starts with INVOKE keep
|
|
18
|
+
// INVOKE_LINE until its `;` or line end; any other token sets none of these values.
|
|
19
|
+
|
|
20
|
+
import { ContextTracker } from '@lezer/lr';
|
|
21
|
+
import * as terms from '../efx-parser.terms.js';
|
|
22
|
+
|
|
23
|
+
export const LINE_START = 1;
|
|
24
|
+
export const TEMPLATE_END = 2;
|
|
25
|
+
export const TEMPLATE_SEMICOLON = 3;
|
|
26
|
+
export const BEFORE_FIRST_WORD = 4;
|
|
27
|
+
export const INVOKE_LINE = 5;
|
|
28
|
+
export const INVOKE_SEMICOLON = 6;
|
|
29
|
+
|
|
30
|
+
// The tokens after which the lexer is at the start of a line.
|
|
31
|
+
const LINE_ENDS = new Set([terms.Semicolon, terms.CRLF, terms.IncludePath, terms.StageHeaderEnd,
|
|
32
|
+
terms.SummarySectionHeader, terms.NavigationSectionHeader]);
|
|
33
|
+
|
|
34
|
+
// The rules that end with a template. The template's own nodes are its fragments, which the
|
|
35
|
+
// grammar defines inside templateFragment, so they have no term of their own to look for.
|
|
36
|
+
const ENDS_WITH_TEMPLATE = new Set([terms.DisplayTemplate, terms.OtherwiseDisplayTemplate]);
|
|
37
|
+
|
|
38
|
+
// The tokens that can come before the first word of a line: its indentation and outline number.
|
|
39
|
+
const LINE_PREFIXES = new Set([terms.Tabs, terms.Spaces, terms.MixedIndent, terms.OutlineNumber]);
|
|
40
|
+
|
|
41
|
+
const SKIPPED = new Set([terms.space, terms.Comment]);
|
|
42
|
+
|
|
43
|
+
/** Whether the lexer is at the start of a line. */
|
|
44
|
+
export function isLineStart(context) {
|
|
45
|
+
return context === LINE_START || context === TEMPLATE_SEMICOLON || context === INVOKE_SEMICOLON;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function shift(context, term) {
|
|
49
|
+
if (SKIPPED.has(term)) {
|
|
50
|
+
return context;
|
|
51
|
+
}
|
|
52
|
+
const lineStart = isLineStart(context) || context === BEFORE_FIRST_WORD;
|
|
53
|
+
if (lineStart && LINE_PREFIXES.has(term)) {
|
|
54
|
+
return BEFORE_FIRST_WORD;
|
|
55
|
+
}
|
|
56
|
+
if (lineStart && term === terms.Invoke) {
|
|
57
|
+
return INVOKE_LINE;
|
|
58
|
+
}
|
|
59
|
+
if (term === terms.Semicolon && context === TEMPLATE_END) {
|
|
60
|
+
return TEMPLATE_SEMICOLON;
|
|
61
|
+
}
|
|
62
|
+
if (term === terms.Semicolon && context === INVOKE_LINE) {
|
|
63
|
+
return INVOKE_SEMICOLON;
|
|
64
|
+
}
|
|
65
|
+
if (LINE_ENDS.has(term)) {
|
|
66
|
+
return LINE_START;
|
|
67
|
+
}
|
|
68
|
+
return context === INVOKE_LINE ? INVOKE_LINE : 0;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export const lexerContext = new ContextTracker({
|
|
72
|
+
start: LINE_START,
|
|
73
|
+
shift,
|
|
74
|
+
reduce: (context, term) => (ENDS_WITH_TEMPLATE.has(term) ? TEMPLATE_END : context),
|
|
75
|
+
hash: (context) => context,
|
|
76
|
+
});
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
// The text at the tokenizer's position, read as EfxLexer.g4 reads it: blanks, line ends, comments
|
|
2
|
+
// and the shapes of its tokens. Offsets count from the position; a negative one looks back.
|
|
3
|
+
|
|
4
|
+
const TAB = 9;
|
|
5
|
+
const LF = 10;
|
|
6
|
+
const FF = 12;
|
|
7
|
+
const CR = 13;
|
|
8
|
+
const SPACE = 32;
|
|
9
|
+
const HYPHEN = 45;
|
|
10
|
+
const DOT = 46;
|
|
11
|
+
const SLASH = 47;
|
|
12
|
+
const SEMICOLON = 59;
|
|
13
|
+
const AT = 64;
|
|
14
|
+
const BACKSLASH = 92;
|
|
15
|
+
|
|
16
|
+
// The text that starts an IncludeDirective.
|
|
17
|
+
const INCLUDE = '#include';
|
|
18
|
+
|
|
19
|
+
// The characters FreeText cannot hold: ~[\r\n\f\t #$@&;\\]
|
|
20
|
+
const NOT_FREE_TEXT = new Set(Array.from('\r\n\f\t #$@&;\\', (c) => c.charCodeAt(0)));
|
|
21
|
+
|
|
22
|
+
export class LexerInput {
|
|
23
|
+
#input;
|
|
24
|
+
|
|
25
|
+
constructor(input) {
|
|
26
|
+
this.#input = input;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** The text of a length at an offset. */
|
|
30
|
+
text(offset, length) {
|
|
31
|
+
const from = this.#input.pos + offset;
|
|
32
|
+
return this.#input.read(from, from + length);
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** Ends the token: the given one, of a length from the position. */
|
|
36
|
+
accept(token, length) {
|
|
37
|
+
this.#input.acceptToken(token, length);
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/** Ends the token if its length is not 0; whether it did. */
|
|
41
|
+
take(token, length) {
|
|
42
|
+
if (length > 0) {
|
|
43
|
+
this.accept(token, length);
|
|
44
|
+
}
|
|
45
|
+
return length > 0;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** TAB | SPACE */
|
|
49
|
+
isBlank(offset) {
|
|
50
|
+
const c = this.#input.peek(offset);
|
|
51
|
+
return c === TAB || c === SPACE;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** Whether the text ends at an offset (EOF). */
|
|
55
|
+
isEnd(offset) {
|
|
56
|
+
return this.#input.peek(offset) < 0;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/** EOL: '\r'? '\n' | '\r' | '\f' */
|
|
60
|
+
isLineEnd(offset) {
|
|
61
|
+
const c = this.#input.peek(offset);
|
|
62
|
+
return c === LF || c === CR || c === FF;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
isSemicolon(offset) {
|
|
66
|
+
return this.#input.peek(offset) === SEMICOLON;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
isSlash(offset) {
|
|
70
|
+
return this.#input.peek(offset) === SLASH;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
isAt(offset) {
|
|
74
|
+
return this.#input.peek(offset) === AT;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
isBackslash(offset) {
|
|
78
|
+
return this.#input.peek(offset) === BACKSLASH;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/** COMMENT: '//' ~[\r\n\f]* */
|
|
82
|
+
isComment(offset) {
|
|
83
|
+
return this.#input.peek(offset) === SLASH && this.#input.peek(offset + 1) === SLASH;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/** Whether a `;` comes before an offset, with only blanks between them. */
|
|
87
|
+
followsSemicolon(offset) {
|
|
88
|
+
let before = offset - 1;
|
|
89
|
+
while (this.isBlank(before)) {
|
|
90
|
+
before -= 1;
|
|
91
|
+
}
|
|
92
|
+
return this.isSemicolon(before);
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/** The number of blanks at an offset. */
|
|
96
|
+
blanks(offset) {
|
|
97
|
+
return this.#count(offset, (o) => this.isBlank(o));
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** The number of line-end characters at an offset. */
|
|
101
|
+
lineEnds(offset) {
|
|
102
|
+
return this.#count(offset, (o) => this.isLineEnd(o));
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/** The number of blanks and line-end characters at an offset. */
|
|
106
|
+
whitespace(offset) {
|
|
107
|
+
return this.#count(offset, (o) => this.isBlank(o) || this.isLineEnd(o));
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/** The length of the whitespace and comments at an offset, which EXPRESSION mode hides (WS, ExpressionComment). */
|
|
111
|
+
hiddenLength(offset) {
|
|
112
|
+
const length = this.whitespace(offset) || this.commentLength(offset);
|
|
113
|
+
return length === 0 ? 0 : length + this.hiddenLength(offset + length);
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/** The length of the comment at an offset, up to the line end. */
|
|
117
|
+
commentLength(offset) {
|
|
118
|
+
return this.isComment(offset) ? this.#count(offset, (o) => this.#input.peek(o) >= 0 && !this.isLineEnd(o)) : 0;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/** The length of the Identifier at an offset: LETTER ('-'? (LETTER | DIGIT)+)* */
|
|
122
|
+
identifierLength(offset) {
|
|
123
|
+
if (!LexerInput.#isLetter(this.#input.peek(offset))) {
|
|
124
|
+
return 0;
|
|
125
|
+
}
|
|
126
|
+
let length = 1;
|
|
127
|
+
for (;;) {
|
|
128
|
+
const hyphen = this.#input.peek(offset + length) === HYPHEN ? 1 : 0;
|
|
129
|
+
const part = this.#count(offset + length + hyphen, (o) => LexerInput.#isLetterOrDigit(this.#input.peek(o)));
|
|
130
|
+
if (part === 0) {
|
|
131
|
+
return length;
|
|
132
|
+
}
|
|
133
|
+
length += hyphen + part;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/** The length of the OtherAssetId at the position: [a-z]+ ([-.] [a-z0-9]+)* */
|
|
138
|
+
assetIdLength() {
|
|
139
|
+
const isPart = (o) => LexerInput.#isLower(this.#input.peek(o)) || LexerInput.#isDigit(this.#input.peek(o));
|
|
140
|
+
let length = this.#count(0, (o) => LexerInput.#isLower(this.#input.peek(o)));
|
|
141
|
+
while (length > 0) {
|
|
142
|
+
const separator = this.#input.peek(length);
|
|
143
|
+
const part = separator === HYPHEN || separator === DOT ? this.#count(length + 1, isPart) : 0;
|
|
144
|
+
if (part === 0) {
|
|
145
|
+
return length;
|
|
146
|
+
}
|
|
147
|
+
length += 1 + part;
|
|
148
|
+
}
|
|
149
|
+
return length;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/** The length of the IncludeDirective at the position: '#include' (TAB | SPACE)+ */
|
|
153
|
+
includeDirectiveLength() {
|
|
154
|
+
const blanks = this.text(0, INCLUDE.length) === INCLUDE ? this.blanks(INCLUDE.length) : 0;
|
|
155
|
+
return blanks > 0 ? INCLUDE.length + blanks : 0;
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/** The length of the FreeText at the position. */
|
|
159
|
+
freeTextLength() {
|
|
160
|
+
return this.#count(0, (o) => this.#input.peek(o) >= 0 && !NOT_FREE_TEXT.has(this.#input.peek(o)));
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
#count(offset, test) {
|
|
164
|
+
let end = offset;
|
|
165
|
+
while (test(end)) {
|
|
166
|
+
end += 1;
|
|
167
|
+
}
|
|
168
|
+
return end - offset;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/** LETTER: [a-zA-Z_] */
|
|
172
|
+
static #isLetter(c) {
|
|
173
|
+
return (c >= 65 && c <= 90) || (c >= 97 && c <= 122) || c === 95;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
static #isLower(c) {
|
|
177
|
+
return c >= 97 && c <= 122;
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
/** DIGIT: [0-9] */
|
|
181
|
+
static #isDigit(c) {
|
|
182
|
+
return c >= 48 && c <= 57;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
static #isLetterOrDigit(c) {
|
|
186
|
+
return LexerInput.#isLetter(c) || LexerInput.#isDigit(c);
|
|
187
|
+
}
|
|
188
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
// A mode of EfxLexer.g4 as the editor finds it. The tokens the parser can take tell whether the
|
|
2
|
+
// lexer would be in the mode there; the mode then hides what the lexer hides in it, as the grammar's
|
|
3
|
+
// skip tokens `space` and `Comment`, and reads the tokens whose reading depends on it. The tokens
|
|
4
|
+
// that read the same in every mode are the grammar's own. Each subclass is named after its mode.
|
|
5
|
+
|
|
6
|
+
export class LexerMode {
|
|
7
|
+
/** Whether the lexer would be in this mode where the parser stands. */
|
|
8
|
+
isActive(stack) {
|
|
9
|
+
throw new Error(`${this.constructor.name} does not say where the lexer is in it`);
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
/** Hides the text the mode hides at the position, if any; whether there was some. */
|
|
13
|
+
hide(input, stack) {
|
|
14
|
+
return false;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/** Reads the token the mode reads at the position, if it reads one there. */
|
|
18
|
+
read(input, stack) {}
|
|
19
|
+
|
|
20
|
+
/** The words the mode reads as tokens of the grammar, each with its token: [word, token] pairs. */
|
|
21
|
+
words() {
|
|
22
|
+
return [];
|
|
23
|
+
}
|
|
24
|
+
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
// The tokens whose reading depends on the mode of EfxLexer.g4. The mode is found from the parse
|
|
2
|
+
// state, and the input is read as the lexer reads it in that mode: first the text the mode hides,
|
|
3
|
+
// then the token the mode reads there, if any; where it reads none, the grammar's own tokens are
|
|
4
|
+
// read.
|
|
5
|
+
//
|
|
6
|
+
// The modes are tried in this order, and the first the parser is in is the one:
|
|
7
|
+
// - where a template can go on, the lexer is in TEMPLATE mode whatever else the parser can take
|
|
8
|
+
// (the tokens that end a template are read in that mode too), so template text comes first, then
|
|
9
|
+
// the start of a template, which takes text but nothing that ends it;
|
|
10
|
+
// - label blocks, the name after a prefix and format options are each found by tokens that no
|
|
11
|
+
// other mode takes;
|
|
12
|
+
// - the name and end of a stage header are tokens that no other mode takes; the mode hides
|
|
13
|
+
// nothing, so the blanks before the end stay with it;
|
|
14
|
+
// - the start of a line takes words that an expression also has, so DEFAULT comes before
|
|
15
|
+
// EXPRESSION, which is the rest. Where the parser has just read a `;`, a line end, a header or an
|
|
16
|
+
// include, the lexer is at the start of a line even when the parser could take more of a rule
|
|
17
|
+
// (LexerContext.js).
|
|
18
|
+
//
|
|
19
|
+
// The completion source asks the same questions where the cursor is: the mode there, and the words
|
|
20
|
+
// it reads (src/completion/ExpectedTokens.js).
|
|
21
|
+
|
|
22
|
+
import { ExternalTokenizer } from '@lezer/lr';
|
|
23
|
+
import { ExpressionMode } from './ExpressionMode.js';
|
|
24
|
+
import { FormatMode } from './FormatMode.js';
|
|
25
|
+
import { IdentifierMode } from './IdentifierMode.js';
|
|
26
|
+
import { LabelMode } from './LabelMode.js';
|
|
27
|
+
import { LexerInput } from './LexerInput.js';
|
|
28
|
+
import { LineMode } from './LineMode.js';
|
|
29
|
+
import { StageHeaderMode } from './StageHeaderMode.js';
|
|
30
|
+
import { TemplateMode } from './TemplateMode.js';
|
|
31
|
+
import { TemplateStartMode } from './TemplateStartMode.js';
|
|
32
|
+
|
|
33
|
+
const MODES = [new TemplateMode(), new TemplateStartMode(), new LabelMode(), new IdentifierMode(), new FormatMode(),
|
|
34
|
+
new StageHeaderMode(), new LineMode(), new ExpressionMode()];
|
|
35
|
+
|
|
36
|
+
/** The mode the lexer is in where the parser stands. */
|
|
37
|
+
export function lexerModeAt(stack) {
|
|
38
|
+
return MODES.find((mode) => mode.isActive(stack));
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** Reads the text at the input's position as the lexer reads it in the mode it is in there. */
|
|
42
|
+
export function readToken(input, stack) {
|
|
43
|
+
const text = new LexerInput(input);
|
|
44
|
+
const mode = lexerModeAt(stack);
|
|
45
|
+
if (!mode.hide(text, stack)) {
|
|
46
|
+
mode.read(text, stack);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export const lexerModes = new ExternalTokenizer(readToken, { contextual: true });
|