@codingame/monaco-vscode-textmate-service-override 1.82.4 → 1.82.5-next.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (21) hide show
  1. package/external/tslib/tslib.es6.js +11 -0
  2. package/external/vscode-textmate/release/main.d.ts +64 -0
  3. package/index.d.ts +2 -2
  4. package/index.js +2 -2
  5. package/package.json +2 -2
  6. package/textmate.d.ts +5 -0
  7. package/textmate.js +43 -0
  8. package/vscode/src/vs/workbench/services/textMate/browser/arrayOperation.js +64 -0
  9. package/vscode/src/vs/workbench/services/textMate/browser/backgroundTokenization/textMateWorkerTokenizerController.js +159 -0
  10. package/vscode/src/vs/workbench/services/textMate/browser/backgroundTokenization/threadedBackgroundTokenizerFactory.js +202 -0
  11. package/vscode/src/vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateTokenizationWorker.worker.js +89 -0
  12. package/vscode/src/vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateWorkerTokenizer.js +142 -0
  13. package/vscode/src/vs/workbench/services/textMate/browser/textMateTokenizationFeature.d.ts +11 -0
  14. package/vscode/src/vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl.js +461 -0
  15. package/vscode/src/vs/workbench/services/textMate/browser/tokenizationSupport/textMateTokenizationSupport.js +74 -0
  16. package/vscode/src/vs/workbench/services/textMate/browser/tokenizationSupport/tokenizationSupportWithLineLimit.js +38 -0
  17. package/vscode/src/vs/workbench/services/textMate/common/TMGrammarFactory.js +120 -0
  18. package/vscode/src/vs/workbench/services/textMate/common/TMGrammars.js +95 -0
  19. package/vscode/src/vs/workbench/services/textMate/common/TMScopeRegistry.js +26 -0
  20. package/worker.js +1 -1
  21. package/workers/textmate.worker.js +4 -0
@@ -0,0 +1,461 @@
1
+ import { __decorate, __param } from '../../../../../../../external/tslib/tslib.es6.js';
2
+ import * as dom from 'monaco-editor/esm/vs/base/browser/dom.js';
3
+ import { equals } from 'monaco-editor/esm/vs/base/common/arrays.js';
4
+ import { Color } from 'monaco-editor/esm/vs/base/common/color.js';
5
+ import { onUnexpectedError } from 'monaco-editor/esm/vs/base/common/errors.js';
6
+ import { Disposable, DisposableStore } from 'monaco-editor/esm/vs/base/common/lifecycle.js';
7
+ import { FileAccess, nodeModulesAsarUnpackedPath, nodeModulesPath } from 'monaco-editor/esm/vs/base/common/network.js';
8
+ import { observableFromEvent } from 'monaco-editor/esm/vs/base/common/observable.js';
9
+ import { isWeb } from 'monaco-editor/esm/vs/base/common/platform.js';
10
+ import * as resources from 'monaco-editor/esm/vs/base/common/resources.js';
11
+ import * as types from 'monaco-editor/esm/vs/base/common/types.js';
12
+ import { LazyTokenizationSupport, TokenizationRegistry } from 'monaco-editor/esm/vs/editor/common/languages.js';
13
+ import { ILanguageService } from 'monaco-editor/esm/vs/editor/common/languages/language.js';
14
+ import { generateTokensCSSForColorMap } from 'monaco-editor/esm/vs/editor/common/languages/supports/tokenization.js';
15
+ import * as nls from 'monaco-editor/esm/vs/nls.js';
16
+ import { IConfigurationService } from 'monaco-editor/esm/vs/platform/configuration/common/configuration.js';
17
+ import { IExtensionResourceLoaderService } from 'vscode/vscode/vs/platform/extensionResourceLoader/common/extensionResourceLoader';
18
+ import { IInstantiationService } from 'monaco-editor/esm/vs/platform/instantiation/common/instantiation.js';
19
+ import { ILogService } from 'monaco-editor/esm/vs/platform/log/common/log.js';
20
+ import { INotificationService } from 'monaco-editor/esm/vs/platform/notification/common/notification.js';
21
+ import { IProgressService } from 'monaco-editor/esm/vs/platform/progress/common/progress.js';
22
+ import { ITelemetryService } from 'monaco-editor/esm/vs/platform/telemetry/common/telemetry.js';
23
+ import { IWorkbenchEnvironmentService } from 'vscode/vscode/vs/workbench/services/environment/common/environmentService';
24
+ import { TextMateTokenizationSupport } from './tokenizationSupport/textMateTokenizationSupport.js';
25
+ import { TokenizationSupportWithLineLimit } from './tokenizationSupport/tokenizationSupportWithLineLimit.js';
26
+ import { ThreadedBackgroundTokenizerFactory } from './backgroundTokenization/threadedBackgroundTokenizerFactory.js';
27
+ import { TMGrammarFactory, missingTMGrammarErrorMessage } from '../common/TMGrammarFactory.js';
28
+ import { grammarsExtPoint } from '../common/TMGrammars.js';
29
+ import { IWorkbenchThemeService } from 'vscode/vscode/vs/workbench/services/themes/common/workbenchThemeService';
30
+ import * as vscodeTextmate from 'vscode-textmate';
31
+ import * as vscodeOniguruma from 'vscode-oniguruma';
32
+
33
+ var TextMateTokenizationFeature_1;
34
+ let TextMateTokenizationFeature = class TextMateTokenizationFeature extends Disposable {
35
+ static { TextMateTokenizationFeature_1 = this; }
36
+ static { this.reportTokenizationTimeCounter = { sync: 0, async: 0 }; }
37
+ constructor(_languageService, _themeService, _extensionResourceLoaderService, _notificationService, _logService, _configurationService, _progressService, _environmentService, _instantiationService, _telemetryService) {
38
+ super();
39
+ this._languageService = _languageService;
40
+ this._themeService = _themeService;
41
+ this._extensionResourceLoaderService = _extensionResourceLoaderService;
42
+ this._notificationService = _notificationService;
43
+ this._logService = _logService;
44
+ this._configurationService = _configurationService;
45
+ this._progressService = _progressService;
46
+ this._environmentService = _environmentService;
47
+ this._instantiationService = _instantiationService;
48
+ this._telemetryService = _telemetryService;
49
+ this._createdModes = [];
50
+ this._encounteredLanguages = [];
51
+ this._debugMode = false;
52
+ this._debugModePrintFunc = () => { };
53
+ this._grammarDefinitions = null;
54
+ this._grammarFactory = null;
55
+ this._tokenizersRegistrations = ( new DisposableStore());
56
+ this._currentTheme = null;
57
+ this._currentTokenColorMap = null;
58
+ this._threadedBackgroundTokenizerFactory = this._instantiationService.createInstance(ThreadedBackgroundTokenizerFactory, (timeMs, languageId, sourceExtensionId, lineLength, isRandomSample) => this._reportTokenizationTime(timeMs, languageId, sourceExtensionId, lineLength, true, isRandomSample), () => this.getAsyncTokenizationEnabled());
59
+ this._vscodeOniguruma = null;
60
+ this._styleElement = dom.createStyleSheet();
61
+ this._styleElement.className = 'vscode-tokens-styles';
62
+ grammarsExtPoint.setHandler((extensions) => this._handleGrammarsExtPoint(extensions));
63
+ this._updateTheme(this._themeService.getColorTheme(), true);
64
+ this._register(this._themeService.onDidColorThemeChange(() => {
65
+ this._updateTheme(this._themeService.getColorTheme(), false);
66
+ }));
67
+ this._languageService.onDidRequestRichLanguageFeatures((languageId) => {
68
+ this._createdModes.push(languageId);
69
+ });
70
+ }
71
+ getAsyncTokenizationEnabled() {
72
+ return !!this._configurationService.getValue('editor.experimental.asyncTokenization');
73
+ }
74
+ getAsyncTokenizationVerification() {
75
+ return !!this._configurationService.getValue('editor.experimental.asyncTokenizationVerification');
76
+ }
77
+ _handleGrammarsExtPoint(extensions) {
78
+ this._grammarDefinitions = null;
79
+ if (this._grammarFactory) {
80
+ this._grammarFactory.dispose();
81
+ this._grammarFactory = null;
82
+ }
83
+ this._tokenizersRegistrations.clear();
84
+ this._grammarDefinitions = [];
85
+ for (const extension of extensions) {
86
+ const grammars = extension.value;
87
+ for (const grammar of grammars) {
88
+ const validatedGrammar = this._validateGrammarDefinition(extension, grammar);
89
+ if (validatedGrammar) {
90
+ this._grammarDefinitions.push(validatedGrammar);
91
+ if (validatedGrammar.language) {
92
+ const lazyTokenizationSupport = ( new LazyTokenizationSupport(() => this._createTokenizationSupport(validatedGrammar.language)));
93
+ this._tokenizersRegistrations.add(lazyTokenizationSupport);
94
+ this._tokenizersRegistrations.add(TokenizationRegistry.registerFactory(validatedGrammar.language, lazyTokenizationSupport));
95
+ }
96
+ }
97
+ }
98
+ }
99
+ this._threadedBackgroundTokenizerFactory.setGrammarDefinitions(this._grammarDefinitions);
100
+ for (const createdMode of this._createdModes) {
101
+ TokenizationRegistry.getOrCreate(createdMode);
102
+ }
103
+ }
104
+ _validateGrammarDefinition(extension, grammar) {
105
+ if (!validateGrammarExtensionPoint(extension.description.extensionLocation, grammar, extension.collector, this._languageService)) {
106
+ return null;
107
+ }
108
+ const grammarLocation = resources.joinPath(extension.description.extensionLocation, grammar.path);
109
+ const embeddedLanguages = Object.create(null);
110
+ if (grammar.embeddedLanguages) {
111
+ const scopes = ( Object.keys(grammar.embeddedLanguages));
112
+ for (let i = 0, len = scopes.length; i < len; i++) {
113
+ const scope = scopes[i];
114
+ const language = grammar.embeddedLanguages[scope];
115
+ if (typeof language !== 'string') {
116
+ continue;
117
+ }
118
+ if (this._languageService.isRegisteredLanguageId(language)) {
119
+ embeddedLanguages[scope] = this._languageService.languageIdCodec.encodeLanguageId(language);
120
+ }
121
+ }
122
+ }
123
+ const tokenTypes = Object.create(null);
124
+ if (grammar.tokenTypes) {
125
+ const scopes = ( Object.keys(grammar.tokenTypes));
126
+ for (const scope of scopes) {
127
+ const tokenType = grammar.tokenTypes[scope];
128
+ switch (tokenType) {
129
+ case 'string':
130
+ tokenTypes[scope] = 2 ;
131
+ break;
132
+ case 'other':
133
+ tokenTypes[scope] = 0 ;
134
+ break;
135
+ case 'comment':
136
+ tokenTypes[scope] = 1 ;
137
+ break;
138
+ }
139
+ }
140
+ }
141
+ const validLanguageId = grammar.language && this._languageService.isRegisteredLanguageId(grammar.language) ? grammar.language : null;
142
+ function asStringArray(array, defaultValue) {
143
+ if (!Array.isArray(array)) {
144
+ return defaultValue;
145
+ }
146
+ if (!array.every(e => typeof e === 'string')) {
147
+ return defaultValue;
148
+ }
149
+ return array;
150
+ }
151
+ return {
152
+ location: grammarLocation,
153
+ language: validLanguageId || undefined,
154
+ scopeName: grammar.scopeName,
155
+ embeddedLanguages: embeddedLanguages,
156
+ tokenTypes: tokenTypes,
157
+ injectTo: grammar.injectTo,
158
+ balancedBracketSelectors: asStringArray(grammar.balancedBracketScopes, ['*']),
159
+ unbalancedBracketSelectors: asStringArray(grammar.unbalancedBracketScopes, []),
160
+ sourceExtensionId: extension.description.id,
161
+ };
162
+ }
163
+ startDebugMode(printFn, onStop) {
164
+ if (this._debugMode) {
165
+ this._notificationService.error(( nls.localize('alreadyDebugging', "Already Logging.")));
166
+ return;
167
+ }
168
+ this._debugModePrintFunc = printFn;
169
+ this._debugMode = true;
170
+ if (this._debugMode) {
171
+ this._progressService.withProgress({
172
+ location: 15 ,
173
+ buttons: [( nls.localize('stop', "Stop"))]
174
+ }, (progress) => {
175
+ progress.report({
176
+ message: ( nls.localize(
177
+ 'progress1',
178
+ "Preparing to log TM Grammar parsing. Press Stop when finished."
179
+ ))
180
+ });
181
+ return this._getVSCodeOniguruma().then((vscodeOniguruma) => {
182
+ vscodeOniguruma.setDefaultDebugCall(true);
183
+ progress.report({
184
+ message: ( nls.localize('progress2', "Now logging TM Grammar parsing. Press Stop when finished."))
185
+ });
186
+ return ( new Promise((resolve, reject) => { }));
187
+ });
188
+ }, (choice) => {
189
+ this._getVSCodeOniguruma().then((vscodeOniguruma) => {
190
+ this._debugModePrintFunc = () => { };
191
+ this._debugMode = false;
192
+ vscodeOniguruma.setDefaultDebugCall(false);
193
+ onStop();
194
+ });
195
+ });
196
+ }
197
+ }
198
+ _canCreateGrammarFactory() {
199
+ return !!this._grammarDefinitions;
200
+ }
201
+ async _getOrCreateGrammarFactory() {
202
+ if (this._grammarFactory) {
203
+ return this._grammarFactory;
204
+ }
205
+ const vscodeOniguruma = await this._getVSCodeOniguruma();
206
+ const onigLib = Promise.resolve({
207
+ createOnigScanner: (sources) => vscodeOniguruma.createOnigScanner(sources),
208
+ createOnigString: (str) => vscodeOniguruma.createOnigString(str)
209
+ });
210
+ if (this._grammarFactory) {
211
+ return this._grammarFactory;
212
+ }
213
+ this._grammarFactory = ( new TMGrammarFactory({
214
+ logTrace: (msg) => this._logService.trace(msg),
215
+ logError: (msg, err) => this._logService.error(msg, err),
216
+ readFile: (resource) => this._extensionResourceLoaderService.readExtensionResource(resource)
217
+ }, this._grammarDefinitions || [], vscodeTextmate, onigLib));
218
+ this._updateTheme(this._themeService.getColorTheme(), true);
219
+ return this._grammarFactory;
220
+ }
221
+ async _createTokenizationSupport(languageId) {
222
+ if (!this._languageService.isRegisteredLanguageId(languageId)) {
223
+ return null;
224
+ }
225
+ if (!this._canCreateGrammarFactory()) {
226
+ return null;
227
+ }
228
+ try {
229
+ const grammarFactory = await this._getOrCreateGrammarFactory();
230
+ if (!( grammarFactory.has(languageId))) {
231
+ return null;
232
+ }
233
+ const encodedLanguageId = this._languageService.languageIdCodec.encodeLanguageId(languageId);
234
+ const r = await grammarFactory.createGrammar(languageId, encodedLanguageId);
235
+ if (!r.grammar) {
236
+ return null;
237
+ }
238
+ const maxTokenizationLineLength = observableConfigValue('editor.maxTokenizationLineLength', languageId, -1, this._configurationService);
239
+ const tokenization = ( new TextMateTokenizationSupport(
240
+ r.grammar,
241
+ r.initialState,
242
+ r.containsEmbeddedLanguages,
243
+ (textModel, tokenStore) => this._threadedBackgroundTokenizerFactory.createBackgroundTokenizer(textModel, tokenStore, maxTokenizationLineLength),
244
+ () => this.getAsyncTokenizationVerification(),
245
+ (timeMs, lineLength, isRandomSample) => {
246
+ this._reportTokenizationTime(timeMs, languageId, r.sourceExtensionId, lineLength, false, isRandomSample);
247
+ },
248
+ true
249
+ ));
250
+ tokenization.onDidEncounterLanguage((encodedLanguageId) => {
251
+ if (!this._encounteredLanguages[encodedLanguageId]) {
252
+ const languageId = this._languageService.languageIdCodec.decodeLanguageId(encodedLanguageId);
253
+ this._encounteredLanguages[encodedLanguageId] = true;
254
+ this._languageService.requestBasicLanguageFeatures(languageId);
255
+ }
256
+ });
257
+ return ( new TokenizationSupportWithLineLimit(encodedLanguageId, tokenization, maxTokenizationLineLength));
258
+ }
259
+ catch (err) {
260
+ if (err.message && err.message === missingTMGrammarErrorMessage) {
261
+ return null;
262
+ }
263
+ onUnexpectedError(err);
264
+ return null;
265
+ }
266
+ }
267
+ _updateTheme(colorTheme, forceUpdate) {
268
+ if (!forceUpdate && this._currentTheme && this._currentTokenColorMap && equalsTokenRules(this._currentTheme.settings, colorTheme.tokenColors)
269
+ && equals(this._currentTokenColorMap, colorTheme.tokenColorMap)) {
270
+ return;
271
+ }
272
+ this._currentTheme = { name: colorTheme.label, settings: colorTheme.tokenColors };
273
+ this._currentTokenColorMap = colorTheme.tokenColorMap;
274
+ this._grammarFactory?.setTheme(this._currentTheme, this._currentTokenColorMap);
275
+ const colorMap = toColorMap(this._currentTokenColorMap);
276
+ const cssRules = generateTokensCSSForColorMap(colorMap);
277
+ this._styleElement.textContent = cssRules;
278
+ TokenizationRegistry.setColorMap(colorMap);
279
+ if (this._currentTheme && this._currentTokenColorMap) {
280
+ this._threadedBackgroundTokenizerFactory.acceptTheme(this._currentTheme, this._currentTokenColorMap);
281
+ }
282
+ }
283
+ async createTokenizer(languageId) {
284
+ if (!this._languageService.isRegisteredLanguageId(languageId)) {
285
+ return null;
286
+ }
287
+ const grammarFactory = await this._getOrCreateGrammarFactory();
288
+ if (!( grammarFactory.has(languageId))) {
289
+ return null;
290
+ }
291
+ const encodedLanguageId = this._languageService.languageIdCodec.encodeLanguageId(languageId);
292
+ const { grammar } = await grammarFactory.createGrammar(languageId, encodedLanguageId);
293
+ return grammar;
294
+ }
295
+ _getVSCodeOniguruma() {
296
+ if (!this._vscodeOniguruma) {
297
+ this._vscodeOniguruma = (async () => {
298
+ const wasm = await this._loadVSCodeOnigurumaWASM();
299
+ await vscodeOniguruma.loadWASM({
300
+ data: wasm,
301
+ print: (str) => {
302
+ this._debugModePrintFunc(str);
303
+ }
304
+ });
305
+ return vscodeOniguruma;
306
+ })();
307
+ }
308
+ return this._vscodeOniguruma;
309
+ }
310
+ async _loadVSCodeOnigurumaWASM() {
311
+ if (isWeb) {
312
+ const response = await fetch(( ( FileAccess.asBrowserUri('vscode-oniguruma/../onig.wasm')).toString(true)));
313
+ return await response.arrayBuffer();
314
+ }
315
+ else {
316
+ const response = await fetch(this._environmentService.isBuilt
317
+ ? ( ( FileAccess.asBrowserUri(`${nodeModulesAsarUnpackedPath}/vscode-oniguruma/release/onig.wasm`)).toString(true))
318
+ : ( ( FileAccess.asBrowserUri(`${nodeModulesPath}/vscode-oniguruma/release/onig.wasm`)).toString(true)));
319
+ return response;
320
+ }
321
+ }
322
+ _reportTokenizationTime(timeMs, languageId, sourceExtensionId, lineLength, fromWorker, isRandomSample) {
323
+ const key = fromWorker ? 'async' : 'sync';
324
+ if (TextMateTokenizationFeature_1.reportTokenizationTimeCounter[key] > 50) {
325
+ return;
326
+ }
327
+ if (TextMateTokenizationFeature_1.reportTokenizationTimeCounter[key] === 0) {
328
+ setTimeout(() => {
329
+ TextMateTokenizationFeature_1.reportTokenizationTimeCounter[key] = 0;
330
+ }, 1000 * 60 * 60);
331
+ }
332
+ TextMateTokenizationFeature_1.reportTokenizationTimeCounter[key]++;
333
+ this._telemetryService.publicLog2('editor.tokenizedLine', {
334
+ timeMs,
335
+ languageId,
336
+ lineLength,
337
+ fromWorker,
338
+ sourceExtensionId,
339
+ isRandomSample,
340
+ tokenizationSetting: this.getAsyncTokenizationEnabled() ? (this.getAsyncTokenizationVerification() ? 2 : 1) : 0,
341
+ });
342
+ }
343
+ };
344
+ TextMateTokenizationFeature = TextMateTokenizationFeature_1 = ( __decorate([
345
+ ( __param(0, ILanguageService)),
346
+ ( __param(1, IWorkbenchThemeService)),
347
+ ( __param(2, IExtensionResourceLoaderService)),
348
+ ( __param(3, INotificationService)),
349
+ ( __param(4, ILogService)),
350
+ ( __param(5, IConfigurationService)),
351
+ ( __param(6, IProgressService)),
352
+ ( __param(7, IWorkbenchEnvironmentService)),
353
+ ( __param(8, IInstantiationService)),
354
+ ( __param(9, ITelemetryService))
355
+ ], TextMateTokenizationFeature));
356
+ function toColorMap(colorMap) {
357
+ const result = [null];
358
+ for (let i = 1, len = colorMap.length; i < len; i++) {
359
+ result[i] = ( Color.fromHex(colorMap[i]));
360
+ }
361
+ return result;
362
+ }
363
+ function equalsTokenRules(a, b) {
364
+ if (!b || !a || b.length !== a.length) {
365
+ return false;
366
+ }
367
+ for (let i = b.length - 1; i >= 0; i--) {
368
+ const r1 = b[i];
369
+ const r2 = a[i];
370
+ if (r1.scope !== r2.scope) {
371
+ return false;
372
+ }
373
+ const s1 = r1.settings;
374
+ const s2 = r2.settings;
375
+ if (s1 && s2) {
376
+ if (s1.fontStyle !== s2.fontStyle || s1.foreground !== s2.foreground || s1.background !== s2.background) {
377
+ return false;
378
+ }
379
+ }
380
+ else if (!s1 || !s2) {
381
+ return false;
382
+ }
383
+ }
384
+ return true;
385
+ }
386
+ function validateGrammarExtensionPoint(extensionLocation, syntax, collector, _languageService) {
387
+ if (syntax.language && ((typeof syntax.language !== 'string') || !_languageService.isRegisteredLanguageId(syntax.language))) {
388
+ collector.error(( nls.localize(
389
+ 'invalid.language',
390
+ "Unknown language in `contributes.{0}.language`. Provided value: {1}",
391
+ grammarsExtPoint.name,
392
+ String(syntax.language)
393
+ )));
394
+ return false;
395
+ }
396
+ if (!syntax.scopeName || (typeof syntax.scopeName !== 'string')) {
397
+ collector.error(( nls.localize(
398
+ 'invalid.scopeName',
399
+ "Expected string in `contributes.{0}.scopeName`. Provided value: {1}",
400
+ grammarsExtPoint.name,
401
+ String(syntax.scopeName)
402
+ )));
403
+ return false;
404
+ }
405
+ if (!syntax.path || (typeof syntax.path !== 'string')) {
406
+ collector.error(( nls.localize(
407
+ 'invalid.path.0',
408
+ "Expected string in `contributes.{0}.path`. Provided value: {1}",
409
+ grammarsExtPoint.name,
410
+ String(syntax.path)
411
+ )));
412
+ return false;
413
+ }
414
+ if (syntax.injectTo && (!Array.isArray(syntax.injectTo) || ( syntax.injectTo.some(scope => typeof scope !== 'string')))) {
415
+ collector.error(( nls.localize(
416
+ 'invalid.injectTo',
417
+ "Invalid value in `contributes.{0}.injectTo`. Must be an array of language scope names. Provided value: {1}",
418
+ grammarsExtPoint.name,
419
+ JSON.stringify(syntax.injectTo)
420
+ )));
421
+ return false;
422
+ }
423
+ if (syntax.embeddedLanguages && !types.isObject(syntax.embeddedLanguages)) {
424
+ collector.error(( nls.localize(
425
+ 'invalid.embeddedLanguages',
426
+ "Invalid value in `contributes.{0}.embeddedLanguages`. Must be an object map from scope name to language. Provided value: {1}",
427
+ grammarsExtPoint.name,
428
+ JSON.stringify(syntax.embeddedLanguages)
429
+ )));
430
+ return false;
431
+ }
432
+ if (syntax.tokenTypes && !types.isObject(syntax.tokenTypes)) {
433
+ collector.error(( nls.localize(
434
+ 'invalid.tokenTypes',
435
+ "Invalid value in `contributes.{0}.tokenTypes`. Must be an object map from scope name to token type. Provided value: {1}",
436
+ grammarsExtPoint.name,
437
+ JSON.stringify(syntax.tokenTypes)
438
+ )));
439
+ return false;
440
+ }
441
+ const grammarLocation = resources.joinPath(extensionLocation, syntax.path);
442
+ if (!resources.isEqualOrParent(grammarLocation, extensionLocation)) {
443
+ collector.warn(( nls.localize(
444
+ 'invalid.path.1',
445
+ "Expected `contributes.{0}.path` ({1}) to be included inside extension's folder ({2}). This might make the extension non-portable.",
446
+ grammarsExtPoint.name,
447
+ grammarLocation.path,
448
+ extensionLocation.path
449
+ )));
450
+ }
451
+ return true;
452
+ }
453
+ function observableConfigValue(key, languageId, defaultValue, configurationService) {
454
+ return observableFromEvent((handleChange) => configurationService.onDidChangeConfiguration(e => {
455
+ if (e.affectsConfiguration(key, { overrideIdentifier: languageId })) {
456
+ handleChange(e);
457
+ }
458
+ }), () => configurationService.getValue(key, { overrideIdentifier: languageId }) ?? defaultValue);
459
+ }
460
+
461
+ export { TextMateTokenizationFeature };
@@ -0,0 +1,74 @@
1
+ import { Emitter } from 'monaco-editor/esm/vs/base/common/event.js';
2
+ import { Disposable } from 'monaco-editor/esm/vs/base/common/lifecycle.js';
3
+ import { StopWatch } from 'monaco-editor/esm/vs/base/common/stopwatch.js';
4
+ import { TokenMetadata } from 'monaco-editor/esm/vs/editor/common/encodedTokenAttributes.js';
5
+ import { EncodedTokenizationResult } from 'monaco-editor/esm/vs/editor/common/languages.js';
6
+
7
+ class TextMateTokenizationSupport extends Disposable {
8
+ constructor(_grammar, _initialState, _containsEmbeddedLanguages, _createBackgroundTokenizer, _backgroundTokenizerShouldOnlyVerifyTokens, _reportTokenizationTime, _reportSlowTokenization) {
9
+ super();
10
+ this._grammar = _grammar;
11
+ this._initialState = _initialState;
12
+ this._containsEmbeddedLanguages = _containsEmbeddedLanguages;
13
+ this._createBackgroundTokenizer = _createBackgroundTokenizer;
14
+ this._backgroundTokenizerShouldOnlyVerifyTokens = _backgroundTokenizerShouldOnlyVerifyTokens;
15
+ this._reportTokenizationTime = _reportTokenizationTime;
16
+ this._reportSlowTokenization = _reportSlowTokenization;
17
+ this._seenLanguages = [];
18
+ this._onDidEncounterLanguage = this._register(( new Emitter()));
19
+ this.onDidEncounterLanguage = this._onDidEncounterLanguage.event;
20
+ }
21
+ get backgroundTokenizerShouldOnlyVerifyTokens() {
22
+ return this._backgroundTokenizerShouldOnlyVerifyTokens();
23
+ }
24
+ getInitialState() {
25
+ return this._initialState;
26
+ }
27
+ tokenize(line, hasEOL, state) {
28
+ throw new Error('Not supported!');
29
+ }
30
+ createBackgroundTokenizer(textModel, store) {
31
+ if (this._createBackgroundTokenizer) {
32
+ return this._createBackgroundTokenizer(textModel, store);
33
+ }
34
+ return undefined;
35
+ }
36
+ tokenizeEncoded(line, hasEOL, state) {
37
+ const isRandomSample = Math.random() * 10000 < 1;
38
+ const shouldMeasure = this._reportSlowTokenization || isRandomSample;
39
+ const sw = shouldMeasure ? ( new StopWatch(true)) : undefined;
40
+ const textMateResult = this._grammar.tokenizeLine2(line, state, 500);
41
+ if (shouldMeasure) {
42
+ const timeMS = sw.elapsed();
43
+ if (isRandomSample || timeMS > 32) {
44
+ this._reportTokenizationTime(timeMS, line.length, isRandomSample);
45
+ }
46
+ }
47
+ if (textMateResult.stoppedEarly) {
48
+ console.warn(`Time limit reached when tokenizing line: ${line.substring(0, 100)}`);
49
+ return ( new EncodedTokenizationResult(textMateResult.tokens, state));
50
+ }
51
+ if (this._containsEmbeddedLanguages) {
52
+ const seenLanguages = this._seenLanguages;
53
+ const tokens = textMateResult.tokens;
54
+ for (let i = 0, len = (tokens.length >>> 1); i < len; i++) {
55
+ const metadata = tokens[(i << 1) + 1];
56
+ const languageId = TokenMetadata.getLanguageId(metadata);
57
+ if (!seenLanguages[languageId]) {
58
+ seenLanguages[languageId] = true;
59
+ this._onDidEncounterLanguage.fire(languageId);
60
+ }
61
+ }
62
+ }
63
+ let endState;
64
+ if (state.equals(textMateResult.ruleStack)) {
65
+ endState = state;
66
+ }
67
+ else {
68
+ endState = textMateResult.ruleStack;
69
+ }
70
+ return ( new EncodedTokenizationResult(textMateResult.tokens, endState));
71
+ }
72
+ }
73
+
74
+ export { TextMateTokenizationSupport };
@@ -0,0 +1,38 @@
1
+ import { nullTokenizeEncoded } from 'monaco-editor/esm/vs/editor/common/languages/nullTokenize.js';
2
+ import { Disposable } from 'monaco-editor/esm/vs/base/common/lifecycle.js';
3
+ import { keepAlive } from 'monaco-editor/esm/vs/base/common/observable.js';
4
+
5
+ class TokenizationSupportWithLineLimit extends Disposable {
6
+ get backgroundTokenizerShouldOnlyVerifyTokens() {
7
+ return this._actual.backgroundTokenizerShouldOnlyVerifyTokens;
8
+ }
9
+ constructor(_encodedLanguageId, _actual, _maxTokenizationLineLength) {
10
+ super();
11
+ this._encodedLanguageId = _encodedLanguageId;
12
+ this._actual = _actual;
13
+ this._maxTokenizationLineLength = _maxTokenizationLineLength;
14
+ this._register(keepAlive(this._maxTokenizationLineLength));
15
+ }
16
+ getInitialState() {
17
+ return this._actual.getInitialState();
18
+ }
19
+ tokenize(line, hasEOL, state) {
20
+ throw new Error('Not supported!');
21
+ }
22
+ tokenizeEncoded(line, hasEOL, state) {
23
+ if (line.length >= this._maxTokenizationLineLength.get()) {
24
+ return nullTokenizeEncoded(this._encodedLanguageId, state);
25
+ }
26
+ return this._actual.tokenizeEncoded(line, hasEOL, state);
27
+ }
28
+ createBackgroundTokenizer(textModel, store) {
29
+ if (this._actual.createBackgroundTokenizer) {
30
+ return this._actual.createBackgroundTokenizer(textModel, store);
31
+ }
32
+ else {
33
+ return undefined;
34
+ }
35
+ }
36
+ }
37
+
38
+ export { TokenizationSupportWithLineLimit };