@codingame/monaco-vscode-editor-api 25.1.2 → 26.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/vscode/src/vs/editor/editor.api.d.ts +27 -2
- package/vscode/src/vs/editor/standalone/browser/colorizer.js +26 -20
- package/vscode/src/vs/editor/standalone/browser/standaloneEditor.js +46 -23
- package/vscode/src/vs/editor/standalone/browser/standaloneLanguages.js +32 -30
- package/vscode/src/vs/editor/standalone/browser/standaloneWebWorker.js +6 -7
- package/vscode/src/vs/editor/standalone/common/monarch/monarchCommon.js +15 -17
- package/vscode/src/vs/editor/standalone/common/monarch/monarchCompile.js +232 -167
- package/vscode/src/vs/editor/standalone/common/monarch/monarchLexer.js +148 -109
|
@@ -17,13 +17,13 @@ function isArrayOf(elemType, obj) {
|
|
|
17
17
|
return true;
|
|
18
18
|
}
|
|
19
19
|
function bool(prop, defValue) {
|
|
20
|
-
if (typeof prop ===
|
|
20
|
+
if (typeof prop === "boolean") {
|
|
21
21
|
return prop;
|
|
22
22
|
}
|
|
23
23
|
return defValue;
|
|
24
24
|
}
|
|
25
25
|
function string(prop, defValue) {
|
|
26
|
-
if (typeof
|
|
26
|
+
if (typeof prop === "string") {
|
|
27
27
|
return prop;
|
|
28
28
|
}
|
|
29
29
|
return defValue;
|
|
@@ -37,16 +37,17 @@ function arrayToHash(array) {
|
|
|
37
37
|
}
|
|
38
38
|
function createKeywordMatcher(arr, caseInsensitive = false) {
|
|
39
39
|
if (caseInsensitive) {
|
|
40
|
-
arr = ( arr.map(function
|
|
40
|
+
arr = ( arr.map(function(x) {
|
|
41
|
+
return x.toLowerCase();
|
|
42
|
+
}));
|
|
41
43
|
}
|
|
42
44
|
const hash = arrayToHash(arr);
|
|
43
45
|
if (caseInsensitive) {
|
|
44
|
-
return function
|
|
46
|
+
return function(word) {
|
|
45
47
|
return hash[word.toLowerCase()] !== undefined && hash.hasOwnProperty(word.toLowerCase());
|
|
46
48
|
};
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
return function (word) {
|
|
49
|
+
} else {
|
|
50
|
+
return function(word) {
|
|
50
51
|
return hash[word] !== undefined && hash.hasOwnProperty(word);
|
|
51
52
|
};
|
|
52
53
|
}
|
|
@@ -57,35 +58,38 @@ function compileRegExp(lexer, str, handleSn) {
|
|
|
57
58
|
let hadExpansion;
|
|
58
59
|
do {
|
|
59
60
|
hadExpansion = false;
|
|
60
|
-
str = str.replace(/@(\w+)/g, function
|
|
61
|
+
str = str.replace(/@(\w+)/g, function(s, attr) {
|
|
61
62
|
hadExpansion = true;
|
|
62
|
-
let sub =
|
|
63
|
-
if (typeof (lexer[attr]) ===
|
|
63
|
+
let sub = "";
|
|
64
|
+
if (typeof (lexer[attr]) === "string") {
|
|
64
65
|
sub = lexer[attr];
|
|
65
|
-
}
|
|
66
|
-
else if (lexer[attr] && lexer[attr] instanceof RegExp) {
|
|
66
|
+
} else if (lexer[attr] && lexer[attr] instanceof RegExp) {
|
|
67
67
|
sub = lexer[attr].source;
|
|
68
|
-
}
|
|
69
|
-
else {
|
|
68
|
+
} else {
|
|
70
69
|
if (lexer[attr] === undefined) {
|
|
71
|
-
throw createError(
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
70
|
+
throw createError(
|
|
71
|
+
lexer,
|
|
72
|
+
"language definition does not contain attribute '" + attr + "', used at: " + str
|
|
73
|
+
);
|
|
74
|
+
} else {
|
|
75
|
+
throw createError(
|
|
76
|
+
lexer,
|
|
77
|
+
"attribute reference '" + attr + "' must be a string, used at: " + str
|
|
78
|
+
);
|
|
75
79
|
}
|
|
76
80
|
}
|
|
77
|
-
return (empty(sub) ?
|
|
81
|
+
return (empty(sub) ? "" : "(?:" + sub + ")");
|
|
78
82
|
});
|
|
79
83
|
n++;
|
|
80
84
|
} while (hadExpansion && n < 5);
|
|
81
|
-
str = str.replace(/\x01/g,
|
|
82
|
-
const flags = (lexer.ignoreCase ?
|
|
85
|
+
str = str.replace(/\x01/g, "@");
|
|
86
|
+
const flags = (lexer.ignoreCase ? "i" : "") + (lexer.unicode ? "u" : "");
|
|
83
87
|
if (handleSn) {
|
|
84
88
|
const match = str.match(/\$[sS](\d\d?)/g);
|
|
85
89
|
if (match) {
|
|
86
90
|
let lastState = null;
|
|
87
91
|
let lastRegEx = null;
|
|
88
|
-
return
|
|
92
|
+
return state => {
|
|
89
93
|
if (lastRegEx && lastState === state) {
|
|
90
94
|
return lastRegEx;
|
|
91
95
|
}
|
|
@@ -106,7 +110,7 @@ function selectScrutinee(id, matches, state, num) {
|
|
|
106
110
|
}
|
|
107
111
|
if (num >= 100) {
|
|
108
112
|
num = num - 100;
|
|
109
|
-
const parts = state.split(
|
|
113
|
+
const parts = state.split(".");
|
|
110
114
|
parts.unshift(state);
|
|
111
115
|
if (num < parts.length) {
|
|
112
116
|
return parts[num];
|
|
@@ -127,16 +131,14 @@ function createGuard(lexer, ruleName, tkey, val) {
|
|
|
127
131
|
}
|
|
128
132
|
oppat = matches[4];
|
|
129
133
|
}
|
|
130
|
-
let op =
|
|
134
|
+
let op = "~";
|
|
131
135
|
let pat = oppat;
|
|
132
136
|
if (!oppat || oppat.length === 0) {
|
|
133
|
-
op =
|
|
134
|
-
pat =
|
|
135
|
-
}
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
}
|
|
139
|
-
else {
|
|
137
|
+
op = "!=";
|
|
138
|
+
pat = "";
|
|
139
|
+
} else if (/^\w*$/.test(pat)) {
|
|
140
|
+
op = "==";
|
|
141
|
+
} else {
|
|
140
142
|
matches = oppat.match(/^(@|!@|~|!~|==|!=)(.*)$/);
|
|
141
143
|
if (matches) {
|
|
142
144
|
op = matches[1];
|
|
@@ -144,155 +146,185 @@ function createGuard(lexer, ruleName, tkey, val) {
|
|
|
144
146
|
}
|
|
145
147
|
}
|
|
146
148
|
let tester;
|
|
147
|
-
if ((op ===
|
|
148
|
-
const inWords = createKeywordMatcher(pat.split(
|
|
149
|
-
tester = function
|
|
150
|
-
|
|
151
|
-
|
|
149
|
+
if ((op === "~" || op === "!~") && /^(\w|\|)*$/.test(pat)) {
|
|
150
|
+
const inWords = createKeywordMatcher(pat.split("|"), lexer.ignoreCase);
|
|
151
|
+
tester = function(s) {
|
|
152
|
+
return (op === "~" ? inWords(s) : !inWords(s));
|
|
153
|
+
};
|
|
154
|
+
} else if (op === "@" || op === "!@") {
|
|
152
155
|
const words = lexer[pat];
|
|
153
156
|
if (!words) {
|
|
154
|
-
throw createError(
|
|
157
|
+
throw createError(
|
|
158
|
+
lexer,
|
|
159
|
+
"the @ match target '" + pat + "' is not defined, in rule: " + ruleName
|
|
160
|
+
);
|
|
155
161
|
}
|
|
156
|
-
if (!(isArrayOf(function
|
|
157
|
-
|
|
162
|
+
if (!(isArrayOf(function(elem) {
|
|
163
|
+
return (typeof elem === "string");
|
|
164
|
+
}, words))) {
|
|
165
|
+
throw createError(
|
|
166
|
+
lexer,
|
|
167
|
+
"the @ match target '" + pat + "' must be an array of strings, in rule: " + ruleName
|
|
168
|
+
);
|
|
158
169
|
}
|
|
159
170
|
const inWords = createKeywordMatcher(words, lexer.ignoreCase);
|
|
160
|
-
tester = function
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
171
|
+
tester = function(s) {
|
|
172
|
+
return (op === "@" ? inWords(s) : !inWords(s));
|
|
173
|
+
};
|
|
174
|
+
} else if (op === "~" || op === "!~") {
|
|
175
|
+
if (pat.indexOf("$") < 0) {
|
|
176
|
+
const re = compileRegExp(lexer, "^" + pat + "$", false);
|
|
177
|
+
tester = function(s) {
|
|
178
|
+
return (op === "~" ? re.test(s) : !re.test(s));
|
|
179
|
+
};
|
|
180
|
+
} else {
|
|
181
|
+
tester = function(s, id, matches, state) {
|
|
182
|
+
const re = compileRegExp(
|
|
183
|
+
lexer,
|
|
184
|
+
"^" + substituteMatches(lexer, pat, id, matches, state) + "$",
|
|
185
|
+
false
|
|
186
|
+
);
|
|
170
187
|
return re.test(s);
|
|
171
188
|
};
|
|
172
189
|
}
|
|
173
|
-
}
|
|
174
|
-
|
|
175
|
-
if (pat.indexOf('$') < 0) {
|
|
190
|
+
} else {
|
|
191
|
+
if (pat.indexOf("$") < 0) {
|
|
176
192
|
const patx = fixCase(lexer, pat);
|
|
177
|
-
tester = function
|
|
178
|
-
|
|
179
|
-
|
|
193
|
+
tester = function(s) {
|
|
194
|
+
return (op === "==" ? s === patx : s !== patx);
|
|
195
|
+
};
|
|
196
|
+
} else {
|
|
180
197
|
const patx = fixCase(lexer, pat);
|
|
181
|
-
tester = function
|
|
198
|
+
tester = function(s, id, matches, state, eos) {
|
|
182
199
|
const patexp = substituteMatches(lexer, patx, id, matches, state);
|
|
183
|
-
return (op ===
|
|
200
|
+
return (op === "==" ? s === patexp : s !== patexp);
|
|
184
201
|
};
|
|
185
202
|
}
|
|
186
203
|
}
|
|
187
204
|
if (scrut === -1) {
|
|
188
205
|
return {
|
|
189
|
-
name: tkey,
|
|
206
|
+
name: tkey,
|
|
207
|
+
value: val,
|
|
208
|
+
test: function(id, matches, state, eos) {
|
|
190
209
|
return tester(id, id, matches, state, eos);
|
|
191
210
|
}
|
|
192
211
|
};
|
|
193
|
-
}
|
|
194
|
-
else {
|
|
212
|
+
} else {
|
|
195
213
|
return {
|
|
196
|
-
name: tkey,
|
|
214
|
+
name: tkey,
|
|
215
|
+
value: val,
|
|
216
|
+
test: function(id, matches, state, eos) {
|
|
197
217
|
const scrutinee = selectScrutinee(id, matches, state, scrut);
|
|
198
|
-
return tester(!scrutinee ?
|
|
218
|
+
return tester(!scrutinee ? "" : scrutinee, id, matches, state, eos);
|
|
199
219
|
}
|
|
200
220
|
};
|
|
201
221
|
}
|
|
202
222
|
}
|
|
203
223
|
function compileAction(lexer, ruleName, action) {
|
|
204
224
|
if (!action) {
|
|
205
|
-
return {
|
|
206
|
-
|
|
207
|
-
|
|
225
|
+
return {
|
|
226
|
+
token: ""
|
|
227
|
+
};
|
|
228
|
+
} else if (typeof action === "string") {
|
|
208
229
|
return action;
|
|
209
|
-
}
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
if (action.token.indexOf(
|
|
230
|
+
} else if (action.token || action.token === "") {
|
|
231
|
+
if (typeof (action.token) !== "string") {
|
|
232
|
+
throw createError(lexer, "a 'token' attribute must be of type string, in rule: " + ruleName);
|
|
233
|
+
} else {
|
|
234
|
+
const newAction = {
|
|
235
|
+
token: action.token
|
|
236
|
+
};
|
|
237
|
+
if (action.token.indexOf("$") >= 0) {
|
|
217
238
|
newAction.tokenSubst = true;
|
|
218
239
|
}
|
|
219
|
-
if (typeof (action.bracket) ===
|
|
220
|
-
if (action.bracket ===
|
|
240
|
+
if (typeof (action.bracket) === "string") {
|
|
241
|
+
if (action.bracket === "@open") {
|
|
221
242
|
newAction.bracket = MonarchBracket.Open;
|
|
222
|
-
}
|
|
223
|
-
else if (action.bracket === '@close') {
|
|
243
|
+
} else if (action.bracket === "@close") {
|
|
224
244
|
newAction.bracket = MonarchBracket.Close;
|
|
225
|
-
}
|
|
226
|
-
|
|
227
|
-
|
|
245
|
+
} else {
|
|
246
|
+
throw createError(
|
|
247
|
+
lexer,
|
|
248
|
+
"a 'bracket' attribute must be either '@open' or '@close', in rule: " + ruleName
|
|
249
|
+
);
|
|
228
250
|
}
|
|
229
251
|
}
|
|
230
252
|
if (action.next) {
|
|
231
|
-
if (typeof (action.next) !==
|
|
232
|
-
throw createError(lexer,
|
|
233
|
-
}
|
|
234
|
-
else {
|
|
253
|
+
if (typeof (action.next) !== "string") {
|
|
254
|
+
throw createError(lexer, "the next state must be a string value in rule: " + ruleName);
|
|
255
|
+
} else {
|
|
235
256
|
let next = action.next;
|
|
236
257
|
if (!/^(@pop|@push|@popall)$/.test(next)) {
|
|
237
|
-
if (next[0] ===
|
|
258
|
+
if (next[0] === "@") {
|
|
238
259
|
next = next.substr(1);
|
|
239
260
|
}
|
|
240
|
-
if (next.indexOf(
|
|
241
|
-
if (!stateExists(lexer, substituteMatches(lexer, next,
|
|
242
|
-
throw createError(
|
|
261
|
+
if (next.indexOf("$") < 0) {
|
|
262
|
+
if (!stateExists(lexer, substituteMatches(lexer, next, "", [], ""))) {
|
|
263
|
+
throw createError(
|
|
264
|
+
lexer,
|
|
265
|
+
"the next state '" + action.next + "' is not defined in rule: " + ruleName
|
|
266
|
+
);
|
|
243
267
|
}
|
|
244
268
|
}
|
|
245
269
|
}
|
|
246
270
|
newAction.next = next;
|
|
247
271
|
}
|
|
248
272
|
}
|
|
249
|
-
if (typeof (action.goBack) ===
|
|
273
|
+
if (typeof (action.goBack) === "number") {
|
|
250
274
|
newAction.goBack = action.goBack;
|
|
251
275
|
}
|
|
252
|
-
if (typeof (action.switchTo) ===
|
|
276
|
+
if (typeof (action.switchTo) === "string") {
|
|
253
277
|
newAction.switchTo = action.switchTo;
|
|
254
278
|
}
|
|
255
|
-
if (typeof (action.log) ===
|
|
279
|
+
if (typeof (action.log) === "string") {
|
|
256
280
|
newAction.log = action.log;
|
|
257
281
|
}
|
|
258
|
-
if (typeof (action.nextEmbedded) ===
|
|
282
|
+
if (typeof (action.nextEmbedded) === "string") {
|
|
259
283
|
newAction.nextEmbedded = action.nextEmbedded;
|
|
260
284
|
lexer.usesEmbedded = true;
|
|
261
285
|
}
|
|
262
286
|
return newAction;
|
|
263
287
|
}
|
|
264
|
-
}
|
|
265
|
-
else if (Array.isArray(action)) {
|
|
288
|
+
} else if (Array.isArray(action)) {
|
|
266
289
|
const results = [];
|
|
267
290
|
for (let i = 0, len = action.length; i < len; i++) {
|
|
268
291
|
results[i] = compileAction(lexer, ruleName, action[i]);
|
|
269
292
|
}
|
|
270
|
-
return {
|
|
271
|
-
|
|
272
|
-
|
|
293
|
+
return {
|
|
294
|
+
group: results
|
|
295
|
+
};
|
|
296
|
+
} else if (action.cases) {
|
|
273
297
|
const cases = [];
|
|
274
298
|
let hasEmbeddedEndInCases = false;
|
|
275
299
|
for (const tkey in action.cases) {
|
|
276
300
|
if (action.cases.hasOwnProperty(tkey)) {
|
|
277
301
|
const val = compileAction(lexer, ruleName, action.cases[tkey]);
|
|
278
|
-
if (tkey ===
|
|
279
|
-
cases.push({
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
else {
|
|
302
|
+
if (tkey === "@default" || tkey === "@" || tkey === "") {
|
|
303
|
+
cases.push({
|
|
304
|
+
test: undefined,
|
|
305
|
+
value: val,
|
|
306
|
+
name: tkey
|
|
307
|
+
});
|
|
308
|
+
} else if (tkey === "@eos") {
|
|
309
|
+
cases.push({
|
|
310
|
+
test: function(id, matches, state, eos) {
|
|
311
|
+
return eos;
|
|
312
|
+
},
|
|
313
|
+
value: val,
|
|
314
|
+
name: tkey
|
|
315
|
+
});
|
|
316
|
+
} else {
|
|
285
317
|
cases.push(createGuard(lexer, ruleName, tkey, val));
|
|
286
318
|
}
|
|
287
319
|
if (!hasEmbeddedEndInCases) {
|
|
288
|
-
hasEmbeddedEndInCases = !isString(val) && (val.hasEmbeddedEndInCases || [
|
|
320
|
+
hasEmbeddedEndInCases = !isString(val) && (val.hasEmbeddedEndInCases || ["@pop", "@popall"].includes(val.nextEmbedded || ""));
|
|
289
321
|
}
|
|
290
322
|
}
|
|
291
323
|
}
|
|
292
324
|
const def = lexer.defaultToken;
|
|
293
325
|
return {
|
|
294
326
|
hasEmbeddedEndInCases,
|
|
295
|
-
test: function
|
|
327
|
+
test: function(id, matches, state, eos) {
|
|
296
328
|
for (const _case of cases) {
|
|
297
329
|
const didmatch = (!_case.test || _case.test(id, matches, state, eos));
|
|
298
330
|
if (didmatch) {
|
|
@@ -302,33 +334,42 @@ function compileAction(lexer, ruleName, action) {
|
|
|
302
334
|
return def;
|
|
303
335
|
}
|
|
304
336
|
};
|
|
305
|
-
}
|
|
306
|
-
|
|
307
|
-
|
|
337
|
+
} else {
|
|
338
|
+
throw createError(
|
|
339
|
+
lexer,
|
|
340
|
+
"an action must be a string, an object with a 'token' or 'cases' attribute, or an array of actions; in rule: " + ruleName
|
|
341
|
+
);
|
|
308
342
|
}
|
|
309
343
|
}
|
|
310
344
|
class Rule {
|
|
311
345
|
constructor(name) {
|
|
312
|
-
this.regex = ( new RegExp(
|
|
313
|
-
this.action = {
|
|
346
|
+
this.regex = ( new RegExp(""));
|
|
347
|
+
this.action = {
|
|
348
|
+
token: ""
|
|
349
|
+
};
|
|
314
350
|
this.matchOnlyAtLineStart = false;
|
|
315
|
-
this.name =
|
|
351
|
+
this.name = "";
|
|
316
352
|
this.name = name;
|
|
317
353
|
}
|
|
318
354
|
setRegex(lexer, re) {
|
|
319
355
|
let sregex;
|
|
320
|
-
if (typeof
|
|
356
|
+
if (typeof re === "string") {
|
|
321
357
|
sregex = re;
|
|
322
|
-
}
|
|
323
|
-
else if (re instanceof RegExp) {
|
|
358
|
+
} else if (re instanceof RegExp) {
|
|
324
359
|
sregex = re.source;
|
|
360
|
+
} else {
|
|
361
|
+
throw createError(
|
|
362
|
+
lexer,
|
|
363
|
+
"rules must start with a match string or regular expression: " + this.name
|
|
364
|
+
);
|
|
325
365
|
}
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
366
|
+
this.matchOnlyAtLineStart = (sregex.length > 0 && sregex[0] === "^");
|
|
367
|
+
this.name = this.name + ": " + sregex;
|
|
368
|
+
this.regex = compileRegExp(
|
|
369
|
+
lexer,
|
|
370
|
+
"^(?:" + (this.matchOnlyAtLineStart ? sregex.substr(1) : sregex) + ")",
|
|
371
|
+
true
|
|
372
|
+
);
|
|
332
373
|
}
|
|
333
374
|
setAction(lexer, act) {
|
|
334
375
|
this.action = compileAction(lexer, this.name, act);
|
|
@@ -336,26 +377,25 @@ class Rule {
|
|
|
336
377
|
resolveRegex(state) {
|
|
337
378
|
if (this.regex instanceof RegExp) {
|
|
338
379
|
return this.regex;
|
|
339
|
-
}
|
|
340
|
-
else {
|
|
380
|
+
} else {
|
|
341
381
|
return this.regex(state);
|
|
342
382
|
}
|
|
343
383
|
}
|
|
344
384
|
}
|
|
345
385
|
function compile(languageId, json) {
|
|
346
|
-
if (!json || typeof
|
|
347
|
-
throw ( new Error(
|
|
386
|
+
if (!json || typeof json !== "object") {
|
|
387
|
+
throw ( new Error("Monarch: expecting a language definition object"));
|
|
348
388
|
}
|
|
349
389
|
const lexer = {
|
|
350
390
|
languageId: languageId,
|
|
351
391
|
includeLF: bool(json.includeLF, false),
|
|
352
392
|
noThrow: false,
|
|
353
393
|
maxStack: 100,
|
|
354
|
-
start: (typeof json.start ===
|
|
394
|
+
start: (typeof json.start === "string" ? json.start : null),
|
|
355
395
|
ignoreCase: bool(json.ignoreCase, false),
|
|
356
396
|
unicode: bool(json.unicode, false),
|
|
357
|
-
tokenPostfix: string(json.tokenPostfix,
|
|
358
|
-
defaultToken: string(json.defaultToken,
|
|
397
|
+
tokenPostfix: string(json.tokenPostfix, "." + languageId),
|
|
398
|
+
defaultToken: string(json.defaultToken, "source"),
|
|
359
399
|
usesEmbedded: false,
|
|
360
400
|
stateNames: {},
|
|
361
401
|
tokenizer: {},
|
|
@@ -374,44 +414,48 @@ function compile(languageId, json) {
|
|
|
374
414
|
for (const rule of rules) {
|
|
375
415
|
let include = rule.include;
|
|
376
416
|
if (include) {
|
|
377
|
-
if (typeof
|
|
378
|
-
throw createError(lexer,
|
|
417
|
+
if (typeof include !== "string") {
|
|
418
|
+
throw createError(lexer, "an 'include' attribute must be a string at: " + state);
|
|
379
419
|
}
|
|
380
|
-
if (include[0] ===
|
|
420
|
+
if (include[0] === "@") {
|
|
381
421
|
include = include.substr(1);
|
|
382
422
|
}
|
|
383
423
|
if (!json.tokenizer[include]) {
|
|
384
|
-
throw createError(lexer,
|
|
424
|
+
throw createError(lexer, "include target '" + include + "' is not defined at: " + state);
|
|
385
425
|
}
|
|
386
|
-
addRules(state +
|
|
387
|
-
}
|
|
388
|
-
else {
|
|
426
|
+
addRules(state + "." + include, newrules, json.tokenizer[include]);
|
|
427
|
+
} else {
|
|
389
428
|
const newrule = ( new Rule(state));
|
|
390
429
|
if (Array.isArray(rule) && rule.length >= 1 && rule.length <= 3) {
|
|
391
430
|
newrule.setRegex(lexerMin, rule[0]);
|
|
392
431
|
if (rule.length >= 3) {
|
|
393
|
-
if (typeof (rule[1]) ===
|
|
394
|
-
newrule.setAction(lexerMin, {
|
|
395
|
-
|
|
396
|
-
|
|
432
|
+
if (typeof (rule[1]) === "string") {
|
|
433
|
+
newrule.setAction(lexerMin, {
|
|
434
|
+
token: rule[1],
|
|
435
|
+
next: rule[2]
|
|
436
|
+
});
|
|
437
|
+
} else if (typeof (rule[1]) === "object") {
|
|
397
438
|
const rule1 = rule[1];
|
|
398
439
|
rule1.next = rule[2];
|
|
399
440
|
newrule.setAction(lexerMin, rule1);
|
|
441
|
+
} else {
|
|
442
|
+
throw createError(
|
|
443
|
+
lexer,
|
|
444
|
+
"a next state as the last element of a rule can only be given if the action is either an object or a string, at: " + state
|
|
445
|
+
);
|
|
400
446
|
}
|
|
401
|
-
|
|
402
|
-
throw createError(lexer, 'a next state as the last element of a rule can only be given if the action is either an object or a string, at: ' + state);
|
|
403
|
-
}
|
|
404
|
-
}
|
|
405
|
-
else {
|
|
447
|
+
} else {
|
|
406
448
|
newrule.setAction(lexerMin, rule[1]);
|
|
407
449
|
}
|
|
408
|
-
}
|
|
409
|
-
else {
|
|
450
|
+
} else {
|
|
410
451
|
if (!rule.regex) {
|
|
411
|
-
throw createError(
|
|
452
|
+
throw createError(
|
|
453
|
+
lexer,
|
|
454
|
+
"a rule must either be an array, or an object with a 'regex' or 'include' field at: " + state
|
|
455
|
+
);
|
|
412
456
|
}
|
|
413
457
|
if (rule.name) {
|
|
414
|
-
if (typeof rule.name ===
|
|
458
|
+
if (typeof rule.name === "string") {
|
|
415
459
|
newrule.name = rule.name;
|
|
416
460
|
}
|
|
417
461
|
}
|
|
@@ -425,8 +469,11 @@ function compile(languageId, json) {
|
|
|
425
469
|
}
|
|
426
470
|
}
|
|
427
471
|
}
|
|
428
|
-
if (!json.tokenizer || typeof (json.tokenizer) !==
|
|
429
|
-
throw createError(
|
|
472
|
+
if (!json.tokenizer || typeof (json.tokenizer) !== "object") {
|
|
473
|
+
throw createError(
|
|
474
|
+
lexer,
|
|
475
|
+
"a language definition must define the 'tokenizer' attribute as an object"
|
|
476
|
+
);
|
|
430
477
|
}
|
|
431
478
|
lexer.tokenizer = [];
|
|
432
479
|
for (const key in json.tokenizer) {
|
|
@@ -436,42 +483,60 @@ function compile(languageId, json) {
|
|
|
436
483
|
}
|
|
437
484
|
const rules = json.tokenizer[key];
|
|
438
485
|
lexer.tokenizer[key] = ( new Array());
|
|
439
|
-
addRules(
|
|
486
|
+
addRules("tokenizer." + key, lexer.tokenizer[key], rules);
|
|
440
487
|
}
|
|
441
488
|
}
|
|
442
489
|
lexer.usesEmbedded = lexerMin.usesEmbedded;
|
|
443
490
|
if (json.brackets) {
|
|
444
491
|
if (!(Array.isArray(json.brackets))) {
|
|
445
|
-
throw createError(lexer,
|
|
492
|
+
throw createError(lexer, "the 'brackets' attribute must be defined as an array");
|
|
446
493
|
}
|
|
447
|
-
}
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
494
|
+
} else {
|
|
495
|
+
json.brackets = [{
|
|
496
|
+
open: "{",
|
|
497
|
+
close: "}",
|
|
498
|
+
token: "delimiter.curly"
|
|
499
|
+
}, {
|
|
500
|
+
open: "[",
|
|
501
|
+
close: "]",
|
|
502
|
+
token: "delimiter.square"
|
|
503
|
+
}, {
|
|
504
|
+
open: "(",
|
|
505
|
+
close: ")",
|
|
506
|
+
token: "delimiter.parenthesis"
|
|
507
|
+
}, {
|
|
508
|
+
open: "<",
|
|
509
|
+
close: ">",
|
|
510
|
+
token: "delimiter.angle"
|
|
511
|
+
}];
|
|
455
512
|
}
|
|
456
513
|
const brackets = [];
|
|
457
514
|
for (const el of json.brackets) {
|
|
458
515
|
let desc = el;
|
|
459
516
|
if (desc && Array.isArray(desc) && desc.length === 3) {
|
|
460
|
-
desc = {
|
|
517
|
+
desc = {
|
|
518
|
+
token: desc[2],
|
|
519
|
+
open: desc[0],
|
|
520
|
+
close: desc[1]
|
|
521
|
+
};
|
|
461
522
|
}
|
|
462
523
|
if (desc.open === desc.close) {
|
|
463
|
-
throw createError(
|
|
464
|
-
|
|
524
|
+
throw createError(
|
|
525
|
+
lexer,
|
|
526
|
+
"open and close brackets in a 'brackets' attribute must be different: " + desc.open + "\n hint: use the 'bracket' attribute if matching on equal brackets is required."
|
|
527
|
+
);
|
|
465
528
|
}
|
|
466
|
-
if (typeof desc.open ===
|
|
529
|
+
if (typeof desc.open === "string" && typeof desc.token === "string" && typeof desc.close === "string") {
|
|
467
530
|
brackets.push({
|
|
468
531
|
token: desc.token + lexer.tokenPostfix,
|
|
469
532
|
open: fixCase(lexer, desc.open),
|
|
470
533
|
close: fixCase(lexer, desc.close)
|
|
471
534
|
});
|
|
472
|
-
}
|
|
473
|
-
|
|
474
|
-
|
|
535
|
+
} else {
|
|
536
|
+
throw createError(
|
|
537
|
+
lexer,
|
|
538
|
+
"every element in the 'brackets' array must be a '{open,close,token}' object or array"
|
|
539
|
+
);
|
|
475
540
|
}
|
|
476
541
|
}
|
|
477
542
|
lexer.brackets = brackets;
|