@codingame/monaco-vscode-textmate-service-override 24.2.0 → 25.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@codingame/monaco-vscode-textmate-service-override",
3
- "version": "24.2.0",
3
+ "version": "25.0.0",
4
4
  "private": false,
5
5
  "description": "VSCode public API plugged on the monaco editor - textmate service-override",
6
6
  "keywords": [],
@@ -15,10 +15,10 @@
15
15
  },
16
16
  "type": "module",
17
17
  "dependencies": {
18
- "@codingame/monaco-vscode-api": "24.2.0",
19
- "@codingame/monaco-vscode-files-service-override": "24.2.0",
18
+ "@codingame/monaco-vscode-api": "25.0.0",
19
+ "@codingame/monaco-vscode-files-service-override": "25.0.0",
20
20
  "vscode-oniguruma": "1.7.0",
21
- "vscode-textmate": "9.2.1"
21
+ "vscode-textmate": "9.3.1"
22
22
  },
23
23
  "main": "index.js",
24
24
  "module": "index.js",
@@ -3,8 +3,10 @@ import { IObservable } from "@codingame/monaco-vscode-api/vscode/vs/base/common/
3
3
  import { Proxied } from "@codingame/monaco-vscode-api/vscode/vs/base/common/worker/webWorker";
4
4
  import { IBackgroundTokenizationStore, ILanguageIdCodec } from "@codingame/monaco-vscode-api/vscode/vs/editor/common/languages";
5
5
  import { ITextModel } from "@codingame/monaco-vscode-api/vscode/vs/editor/common/model";
6
+ import { IFontTokenOption } from "@codingame/monaco-vscode-api/vscode/vs/editor/common/textModelEvents";
6
7
  import { IConfigurationService } from "@codingame/monaco-vscode-api/vscode/vs/platform/configuration/common/configuration.service";
7
8
  import type { StateDeltas, TextMateTokenizationWorker } from "./worker/textMateTokenizationWorker.worker.js";
9
+ import { ISerializedAnnotation } from "@codingame/monaco-vscode-api/vscode/vs/editor/common/model/tokens/annotations";
8
10
  export declare class TextMateWorkerTokenizerController extends Disposable {
9
11
  private readonly _model;
10
12
  private readonly _worker;
@@ -29,6 +31,7 @@ export declare class TextMateWorkerTokenizerController extends Disposable {
29
31
  /**
30
32
  * This method is called from the worker through the worker host.
31
33
  */
32
- setTokensAndStates(controllerId: number, versionId: number, rawTokens: Uint8Array, stateDeltas: StateDeltas[]): Promise<void>;
34
+ setTokensAndStates(controllerId: number, versionId: number, rawTokens: Uint8Array, fontTokens: ISerializedAnnotation<IFontTokenOption>[], stateDeltas: StateDeltas[]): Promise<void>;
35
+ private _stringEditFromChanges;
33
36
  private get _shouldLog();
34
37
  }
@@ -4,10 +4,14 @@ import '@codingame/monaco-vscode-api/vscode/vs/base/common/observableInternal/in
4
4
  import { LineRange } from '@codingame/monaco-vscode-api/vscode/vs/editor/common/core/ranges/lineRange';
5
5
  import { Range } from '@codingame/monaco-vscode-api/vscode/vs/editor/common/core/range';
6
6
  import { TokenizationStateStore } from '@codingame/monaco-vscode-api/vscode/vs/editor/common/model/textModelTokens';
7
+ import { deserializeFontTokenOptions } from '@codingame/monaco-vscode-api/vscode/vs/editor/common/textModelEvents';
7
8
  import { ContiguousMultilineTokensBuilder } from '@codingame/monaco-vscode-api/vscode/vs/editor/common/tokens/contiguousMultilineTokensBuilder';
8
9
  import { observableConfigValue } from '@codingame/monaco-vscode-api/vscode/vs/platform/observable/common/platformObservableUtils';
9
10
  import { MonotonousIndexTransformer } from '../indexTransformer.js';
10
11
  import { linesLengthEditFromModelContentChange } from '@codingame/monaco-vscode-api/vscode/vs/editor/common/model/textModelStringEdit';
12
+ import { StringEdit } from '@codingame/monaco-vscode-api/vscode/vs/editor/common/core/edits/stringEdit';
13
+ import { OffsetRange } from '@codingame/monaco-vscode-api/vscode/vs/editor/common/core/ranges/offsetRange';
14
+ import { AnnotationsUpdate } from '@codingame/monaco-vscode-api/vscode/vs/editor/common/model/tokens/annotations';
11
15
  import { keepObserved } from '@codingame/monaco-vscode-api/vscode/vs/base/common/observableInternal/utils/utils';
12
16
  import { autorun } from '@codingame/monaco-vscode-api/vscode/vs/base/common/observableInternal/reactions/autorun';
13
17
 
@@ -65,11 +69,12 @@ class TextMateWorkerTokenizerController extends Disposable {
65
69
  requestTokens(startLineNumber, endLineNumberExclusive) {
66
70
  this._worker.$retokenize(this.controllerId, startLineNumber, endLineNumberExclusive);
67
71
  }
68
- async setTokensAndStates(controllerId, versionId, rawTokens, stateDeltas) {
72
+ async setTokensAndStates(controllerId, versionId, rawTokens, fontTokens, stateDeltas) {
69
73
  if (this.controllerId !== controllerId) {
70
74
  return;
71
75
  }
72
76
  let tokens = ContiguousMultilineTokensBuilder.deserialize(( new Uint8Array(rawTokens)));
77
+ const fontTokensUpdate = AnnotationsUpdate.deserialize(fontTokens, deserializeFontTokenOptions());
73
78
  if (this._shouldLog) {
74
79
  console.log('received background tokenization result', {
75
80
  fileName: this._model.uri.fsPath.split('\\').pop(),
@@ -109,6 +114,7 @@ class TextMateWorkerTokenizerController extends Disposable {
109
114
  }
110
115
  }
111
116
  }
117
+ fontTokensUpdate.rebase(this._stringEditFromChanges(this._model, this._pendingChanges));
112
118
  }
113
119
  const curToFutureTransformerStates = MonotonousIndexTransformer.fromMany(( this._pendingChanges.map((c) => linesLengthEditFromModelContentChange(c.changes))));
114
120
  if (!this._applyStateStackDiffFn || !this._initialState) {
@@ -139,6 +145,20 @@ class TextMateWorkerTokenizerController extends Disposable {
139
145
  }
140
146
  }
141
147
  this._backgroundTokenizationStore.setTokens(tokens);
148
+ this._backgroundTokenizationStore.setFontInfo(fontTokensUpdate);
149
+ }
150
+ _stringEditFromChanges(model, pendingChanges) {
151
+ const edits = [];
152
+ for (const change of pendingChanges) {
153
+ for (const innerChanges of change.changes) {
154
+ const range = Range.lift(innerChanges.range);
155
+ const text = innerChanges.text;
156
+ const offsetEditStart = model.getOffsetAt(range.getStartPosition());
157
+ const offsetEditEnd = model.getOffsetAt(range.getEndPosition());
158
+ edits.push(StringEdit.replace(( new OffsetRange(offsetEditStart, offsetEditEnd)), text));
159
+ }
160
+ }
161
+ return StringEdit.compose(edits);
142
162
  }
143
163
  get _shouldLog() { return this._loggingEnabled.get(); }
144
164
  }
@@ -122,10 +122,10 @@ let ThreadedBackgroundTokenizerFactory = class ThreadedBackgroundTokenizerFactor
122
122
  const resource = URI.revive(_resource);
123
123
  return this._extensionResourceLoaderService.readExtensionResource(resource);
124
124
  },
125
- $setTokensAndStates: async (controllerId, versionId, tokens, lineEndStateDeltas) => {
125
+ $setTokensAndStates: async (controllerId, versionId, tokens, fontTokens, lineEndStateDeltas) => {
126
126
  const controller = this._workerTokenizerControllers.get(controllerId);
127
127
  if (controller) {
128
- controller.setTokensAndStates(controllerId, versionId, tokens, lineEndStateDeltas);
128
+ controller.setTokensAndStates(controllerId, versionId, tokens, fontTokens, lineEndStateDeltas);
129
129
  }
130
130
  },
131
131
  $reportTokenizationTime: (timeMs, languageId, sourceExtensionId, lineLength, isRandomSample) => {
@@ -61,8 +61,8 @@ class TextMateTokenizationWorker {
61
61
  }
62
62
  return that._grammarCache[encodedLanguageId];
63
63
  },
64
- setTokensAndStates(versionId, tokens, stateDeltas) {
65
- that._host.$setTokensAndStates(data.controllerId, versionId, tokens, stateDeltas);
64
+ setTokensAndStates(versionId, tokens, fontTokens, stateDeltas) {
65
+ that._host.$setTokensAndStates(data.controllerId, versionId, tokens, fontTokens, stateDeltas);
66
66
  },
67
67
  reportTokenizationTime(timeMs, languageId, sourceExtensionId, lineLength, isRandomSample) {
68
68
  that._host.$reportTokenizationTime(timeMs, languageId, sourceExtensionId, lineLength, isRandomSample);
@@ -1,11 +1,13 @@
1
1
  import { UriComponents } from "@codingame/monaco-vscode-api/vscode/vs/base/common/uri";
2
2
  import { IWebWorkerServer, IWebWorkerClient } from "@codingame/monaco-vscode-api/vscode/vs/base/common/worker/webWorker";
3
+ import { ISerializedAnnotation } from "@codingame/monaco-vscode-api/vscode/vs/editor/common/model/tokens/annotations";
4
+ import { IFontTokenOption } from "@codingame/monaco-vscode-api/vscode/vs/editor/common/textModelEvents";
3
5
  import { StateDeltas } from "./textMateTokenizationWorker.worker.js";
4
6
  export declare abstract class TextMateWorkerHost {
5
7
  static CHANNEL_NAME: string;
6
8
  static getChannel(workerServer: IWebWorkerServer): TextMateWorkerHost;
7
9
  static setChannel(workerClient: IWebWorkerClient<unknown>, obj: TextMateWorkerHost): void;
8
10
  abstract $readFile(_resource: UriComponents): Promise<string>;
9
- abstract $setTokensAndStates(controllerId: number, versionId: number, tokens: Uint8Array, lineEndStateDeltas: StateDeltas[]): Promise<void>;
11
+ abstract $setTokensAndStates(controllerId: number, versionId: number, tokens: Uint8Array, fontTokens: ISerializedAnnotation<IFontTokenOption>[], lineEndStateDeltas: StateDeltas[]): Promise<void>;
10
12
  abstract $reportTokenizationTime(timeMs: number, languageId: string, sourceExtensionId: string | undefined, lineLength: number, isRandomSample: boolean): void;
11
13
  }
@@ -3,9 +3,11 @@ import { LanguageId } from "@codingame/monaco-vscode-api/vscode/vs/editor/common
3
3
  import { IModelChangedEvent, MirrorTextModel } from "@codingame/monaco-vscode-api/vscode/vs/editor/common/model/mirrorTextModel";
4
4
  import { ICreateGrammarResult } from "../../../common/TMGrammarFactory.js";
5
5
  import { StateDeltas } from "./textMateTokenizationWorker.worker.js";
6
+ import { IFontTokenOption } from "@codingame/monaco-vscode-api/vscode/vs/editor/common/textModelEvents";
7
+ import { ISerializedAnnotation } from "@codingame/monaco-vscode-api/vscode/vs/editor/common/model/tokens/annotations";
6
8
  export interface TextMateModelTokenizerHost {
7
9
  getOrCreateGrammar(languageId: string, encodedLanguageId: LanguageId): Promise<ICreateGrammarResult | null>;
8
- setTokensAndStates(versionId: number, tokens: Uint8Array, stateDeltas: StateDeltas[]): void;
10
+ setTokensAndStates(versionId: number, tokens: Uint8Array, fontTokens: ISerializedAnnotation<IFontTokenOption>[], stateDeltas: StateDeltas[]): void;
9
11
  reportTokenizationTime(timeMs: number, languageId: string, sourceExtensionId: string | undefined, lineLength: number, isRandomSample: boolean): void;
10
12
  }
11
13
  export declare class TextMateWorkerTokenizer extends MirrorTextModel {
@@ -25,4 +27,6 @@ export declare class TextMateWorkerTokenizer extends MirrorTextModel {
25
27
  retokenize(startLineNumber: number, endLineNumberExclusive: number): void;
26
28
  private _resetTokenization;
27
29
  private _tokenize;
30
+ private _getFontTokensUpdate;
31
+ private _getOffsetAtLineStart;
28
32
  }
@@ -10,6 +10,9 @@ import { LineTokens } from '@codingame/monaco-vscode-api/vscode/vs/editor/common
10
10
  import { TextMateTokenizationSupport } from '../../tokenizationSupport/textMateTokenizationSupport.js';
11
11
  import { TokenizationSupportWithLineLimit } from '../../tokenizationSupport/tokenizationSupportWithLineLimit.js';
12
12
  import { Disposable } from '@codingame/monaco-vscode-api/vscode/vs/base/common/lifecycle';
13
+ import { serializeFontTokenOptions } from '@codingame/monaco-vscode-api/vscode/vs/editor/common/textModelEvents';
14
+ import { AnnotationsUpdate } from '@codingame/monaco-vscode-api/vscode/vs/editor/common/model/tokens/annotations';
15
+ import { OffsetRange } from '@codingame/monaco-vscode-api/vscode/vs/editor/common/core/ranges/offsetRange';
13
16
  import { observableValue } from '@codingame/monaco-vscode-api/vscode/vs/base/common/observableInternal/observables/observableValue';
14
17
 
15
18
  class TextMateWorkerTokenizer extends MirrorTextModel {
@@ -88,6 +91,7 @@ class TextMateWorkerTokenizer extends MirrorTextModel {
88
91
  let tokenizedLines = 0;
89
92
  const tokenBuilder = ( new ContiguousMultilineTokensBuilder());
90
93
  const stateDeltaBuilder = ( new StateDeltaBuilder());
94
+ const fontTokensUpdate = [];
91
95
  while (true) {
92
96
  const lineToTokenize = this._tokenizerWithStateStore.getFirstInvalidLine();
93
97
  if (lineToTokenize === null || tokenizedLines > 200) {
@@ -105,6 +109,7 @@ class TextMateWorkerTokenizer extends MirrorTextModel {
105
109
  }
106
110
  LineTokens.convertToEndOffset(r.tokens, text.length);
107
111
  tokenBuilder.add(lineToTokenize.lineNumber, r.tokens);
112
+ fontTokensUpdate.push(...this._getFontTokensUpdate(lineToTokenize.lineNumber, r));
108
113
  const deltaMs = ( new Date()).getTime() - startTime;
109
114
  if (deltaMs > 20) {
110
115
  break;
@@ -113,8 +118,10 @@ class TextMateWorkerTokenizer extends MirrorTextModel {
113
118
  if (tokenizedLines === 0) {
114
119
  break;
115
120
  }
121
+ const fontUpdate = AnnotationsUpdate.create(fontTokensUpdate);
122
+ const serializedFontUpdate = fontUpdate.serialize(serializeFontTokenOptions());
116
123
  const stateDeltas = stateDeltaBuilder.getStateDeltas();
117
- this._host.setTokensAndStates(this._versionId, tokenBuilder.serialize(), stateDeltas);
124
+ this._host.setTokensAndStates(this._versionId, tokenBuilder.serialize(), serializedFontUpdate, stateDeltas);
118
125
  const deltaMs = ( new Date()).getTime() - startTime;
119
126
  if (deltaMs > 20) {
120
127
  setTimeout0(() => this._tokenize());
@@ -122,6 +129,37 @@ class TextMateWorkerTokenizer extends MirrorTextModel {
122
129
  }
123
130
  }
124
131
  }
132
+ _getFontTokensUpdate(lineNumber, r) {
133
+ const fontTokens = [];
134
+ const offsetAtLineStart = this._getOffsetAtLineStart(lineNumber);
135
+ const offsetAtNextLineStart = this._getOffsetAtLineStart(lineNumber + 1);
136
+ const offsetAtLineEnd = offsetAtNextLineStart > 0 ? offsetAtNextLineStart - 1 : 0;
137
+ fontTokens.push({
138
+ range: ( new OffsetRange(offsetAtLineStart, offsetAtLineEnd)),
139
+ annotation: undefined
140
+ });
141
+ if (r.fontInfo.length) {
142
+ for (const fontInfo of r.fontInfo) {
143
+ const offsetAtLineStart = this._getOffsetAtLineStart(lineNumber);
144
+ fontTokens.push({
145
+ range: ( new OffsetRange(
146
+ offsetAtLineStart + fontInfo.startIndex,
147
+ offsetAtLineStart + fontInfo.endIndex
148
+ )),
149
+ annotation: {
150
+ fontFamily: fontInfo.fontFamily ?? undefined,
151
+ fontSize: fontInfo.fontSize ?? undefined,
152
+ lineHeight: fontInfo.lineHeight ?? undefined
153
+ }
154
+ });
155
+ }
156
+ }
157
+ return fontTokens;
158
+ }
159
+ _getOffsetAtLineStart(lineNumber) {
160
+ this._ensureLineStarts();
161
+ return lineNumber - 1 > 0 ? this._lineStarts.getPrefixSum(lineNumber - 2) : 0;
162
+ }
125
163
  }
126
164
  class StateDeltaBuilder {
127
165
  constructor() {
@@ -34,6 +34,7 @@ export declare class TextMateTokenizationFeature extends Disposable implements I
34
34
  private readonly _tokenizersRegistrations;
35
35
  private _currentTheme;
36
36
  private _currentTokenColorMap;
37
+ private _currentTokenFontMap;
37
38
  private readonly _threadedBackgroundTokenizerFactory;
38
39
  constructor(_languageService: ILanguageService, _themeService: IWorkbenchThemeService, _extensionResourceLoaderService: IExtensionResourceLoaderService, _notificationService: INotificationService, _logService: ILogService, _configurationService: IConfigurationService, _progressService: IProgressService, _environmentService: IWorkbenchEnvironmentService, _instantiationService: IInstantiationService, _telemetryService: ITelemetryService);
39
40
  private getAsyncTokenizationEnabled;
@@ -13,7 +13,7 @@ import { isObject } from '@codingame/monaco-vscode-api/vscode/vs/base/common/typ
13
13
  import { StandardTokenType } from '@codingame/monaco-vscode-api/vscode/vs/editor/common/encodedTokenAttributes';
14
14
  import { LazyTokenizationSupport, TokenizationRegistry } from '@codingame/monaco-vscode-api/vscode/vs/editor/common/languages';
15
15
  import { ILanguageService } from '@codingame/monaco-vscode-api/vscode/vs/editor/common/languages/language.service';
16
- import { generateTokensCSSForColorMap } from '@codingame/monaco-vscode-api/vscode/vs/editor/common/languages/supports/tokenization';
16
+ import { generateTokensCSSForColorMap, generateTokensCSSForFontMap } from '@codingame/monaco-vscode-api/vscode/vs/editor/common/languages/supports/tokenization';
17
17
  import { localize } from '@codingame/monaco-vscode-api/vscode/vs/nls';
18
18
  import { IConfigurationService } from '@codingame/monaco-vscode-api/vscode/vs/platform/configuration/common/configuration.service';
19
19
  import { IExtensionResourceLoaderService } from '@codingame/monaco-vscode-api/vscode/vs/platform/extensionResourceLoader/common/extensionResourceLoader.service';
@@ -57,6 +57,7 @@ let TextMateTokenizationFeature = class TextMateTokenizationFeature extends Disp
57
57
  this._tokenizersRegistrations = this._register(( new DisposableStore()));
58
58
  this._currentTheme = null;
59
59
  this._currentTokenColorMap = null;
60
+ this._currentTokenFontMap = null;
60
61
  this._threadedBackgroundTokenizerFactory = this._instantiationService.createInstance(ThreadedBackgroundTokenizerFactory, (timeMs, languageId, sourceExtensionId, lineLength, isRandomSample) => this._reportTokenizationTime(timeMs, languageId, sourceExtensionId, lineLength, true, isRandomSample), () => this.getAsyncTokenizationEnabled());
61
62
  this._vscodeOniguruma = null;
62
63
  this._styleElement = createStyleSheet();
@@ -164,7 +165,7 @@ let TextMateTokenizationFeature = class TextMateTokenizationFeature extends Disp
164
165
  }
165
166
  startDebugMode(printFn, onStop) {
166
167
  if (this._debugMode) {
167
- this._notificationService.error(( localize(14221, "Already Logging.")));
168
+ this._notificationService.error(( localize(14276, "Already Logging.")));
168
169
  return;
169
170
  }
170
171
  this._debugModePrintFunc = printFn;
@@ -172,15 +173,15 @@ let TextMateTokenizationFeature = class TextMateTokenizationFeature extends Disp
172
173
  if (this._debugMode) {
173
174
  this._progressService.withProgress({
174
175
  location: ProgressLocation.Notification,
175
- buttons: [( localize(14222, "Stop"))]
176
+ buttons: [( localize(14277, "Stop"))]
176
177
  }, (progress) => {
177
178
  progress.report({
178
- message: ( localize(14223, "Preparing to log TM Grammar parsing. Press Stop when finished."))
179
+ message: ( localize(14278, "Preparing to log TM Grammar parsing. Press Stop when finished."))
179
180
  });
180
181
  return this._getVSCodeOniguruma().then((vscodeOniguruma) => {
181
182
  vscodeOniguruma.setDefaultDebugCall(true);
182
183
  progress.report({
183
- message: ( localize(14224, "Now logging TM Grammar parsing. Press Stop when finished."))
184
+ message: ( localize(14279, "Now logging TM Grammar parsing. Press Stop when finished."))
184
185
  });
185
186
  return ( new Promise((resolve, reject) => { }));
186
187
  });
@@ -266,15 +267,17 @@ let TextMateTokenizationFeature = class TextMateTokenizationFeature extends Disp
266
267
  }
267
268
  _updateTheme(colorTheme, forceUpdate) {
268
269
  if (!forceUpdate && this._currentTheme && this._currentTokenColorMap && equalsTokenRules(this._currentTheme.settings, colorTheme.tokenColors)
269
- && equals(this._currentTokenColorMap, colorTheme.tokenColorMap)) {
270
+ && equals(this._currentTokenColorMap, colorTheme.tokenColorMap) && this._currentTokenFontMap && equals(this._currentTokenFontMap, colorTheme.tokenFontMap)) {
270
271
  return;
271
272
  }
272
273
  this._currentTheme = { name: colorTheme.label, settings: colorTheme.tokenColors };
273
274
  this._currentTokenColorMap = colorTheme.tokenColorMap;
275
+ this._currentTokenFontMap = colorTheme.tokenFontMap;
274
276
  this._grammarFactory?.setTheme(this._currentTheme, this._currentTokenColorMap);
275
277
  const colorMap = toColorMap(this._currentTokenColorMap);
276
- const cssRules = generateTokensCSSForColorMap(colorMap);
277
- this._styleElement.textContent = cssRules;
278
+ const colorCssRules = generateTokensCSSForColorMap(colorMap);
279
+ const fontCssRules = generateTokensCSSForFontMap(this._currentTokenFontMap);
280
+ this._styleElement.textContent = colorCssRules + fontCssRules;
278
281
  TokenizationRegistry.setColorMap(colorMap);
279
282
  if (this._currentTheme && this._currentTokenColorMap) {
280
283
  this._threadedBackgroundTokenizerFactory.acceptTheme(this._currentTheme, this._currentTokenColorMap);
@@ -384,7 +387,7 @@ function equalsTokenRules(a, b) {
384
387
  function validateGrammarExtensionPoint(extensionLocation, syntax, collector, _languageService) {
385
388
  if (syntax.language && ((typeof syntax.language !== 'string') || !_languageService.isRegisteredLanguageId(syntax.language))) {
386
389
  collector.error(( localize(
387
- 14225,
390
+ 14280,
388
391
  "Unknown language in `contributes.{0}.language`. Provided value: {1}",
389
392
  grammarsExtPoint.name,
390
393
  String(syntax.language)
@@ -393,7 +396,7 @@ function validateGrammarExtensionPoint(extensionLocation, syntax, collector, _la
393
396
  }
394
397
  if (!syntax.scopeName || (typeof syntax.scopeName !== 'string')) {
395
398
  collector.error(( localize(
396
- 14226,
399
+ 14281,
397
400
  "Expected string in `contributes.{0}.scopeName`. Provided value: {1}",
398
401
  grammarsExtPoint.name,
399
402
  String(syntax.scopeName)
@@ -402,7 +405,7 @@ function validateGrammarExtensionPoint(extensionLocation, syntax, collector, _la
402
405
  }
403
406
  if (!syntax.path || (typeof syntax.path !== 'string')) {
404
407
  collector.error(( localize(
405
- 14227,
408
+ 14282,
406
409
  "Expected string in `contributes.{0}.path`. Provided value: {1}",
407
410
  grammarsExtPoint.name,
408
411
  String(syntax.path)
@@ -411,7 +414,7 @@ function validateGrammarExtensionPoint(extensionLocation, syntax, collector, _la
411
414
  }
412
415
  if (syntax.injectTo && (!Array.isArray(syntax.injectTo) || ( syntax.injectTo.some(scope => typeof scope !== 'string')))) {
413
416
  collector.error(( localize(
414
- 14228,
417
+ 14283,
415
418
  "Invalid value in `contributes.{0}.injectTo`. Must be an array of language scope names. Provided value: {1}",
416
419
  grammarsExtPoint.name,
417
420
  JSON.stringify(syntax.injectTo)
@@ -420,7 +423,7 @@ function validateGrammarExtensionPoint(extensionLocation, syntax, collector, _la
420
423
  }
421
424
  if (syntax.embeddedLanguages && !isObject(syntax.embeddedLanguages)) {
422
425
  collector.error(( localize(
423
- 14229,
426
+ 14284,
424
427
  "Invalid value in `contributes.{0}.embeddedLanguages`. Must be an object map from scope name to language. Provided value: {1}",
425
428
  grammarsExtPoint.name,
426
429
  JSON.stringify(syntax.embeddedLanguages)
@@ -429,7 +432,7 @@ function validateGrammarExtensionPoint(extensionLocation, syntax, collector, _la
429
432
  }
430
433
  if (syntax.tokenTypes && !isObject(syntax.tokenTypes)) {
431
434
  collector.error(( localize(
432
- 14230,
435
+ 14285,
433
436
  "Invalid value in `contributes.{0}.tokenTypes`. Must be an object map from scope name to token type. Provided value: {1}",
434
437
  grammarsExtPoint.name,
435
438
  JSON.stringify(syntax.tokenTypes)
@@ -439,7 +442,7 @@ function validateGrammarExtensionPoint(extensionLocation, syntax, collector, _la
439
442
  const grammarLocation = joinPath(extensionLocation, syntax.path);
440
443
  if (!isEqualOrParent(grammarLocation, extensionLocation)) {
441
444
  collector.warn(( localize(
442
- 14231,
445
+ 14286,
443
446
  "Expected `contributes.{0}.path` ({1}) to be included inside extension's folder ({2}). This might make the extension non-portable.",
444
447
  grammarsExtPoint.name,
445
448
  grammarLocation.path,
@@ -47,7 +47,7 @@ class TextMateTokenizationSupport extends Disposable {
47
47
  }
48
48
  if (textMateResult.stoppedEarly) {
49
49
  console.warn(`Time limit reached when tokenizing line: ${line.substring(0, 100)}`);
50
- return ( new EncodedTokenizationResult(textMateResult.tokens, state));
50
+ return ( new EncodedTokenizationResult(textMateResult.tokens, textMateResult.fonts, state));
51
51
  }
52
52
  if (this._containsEmbeddedLanguages) {
53
53
  const seenLanguages = this._seenLanguages;
@@ -68,7 +68,7 @@ class TextMateTokenizationSupport extends Disposable {
68
68
  else {
69
69
  endState = textMateResult.ruleStack;
70
70
  }
71
- return ( new EncodedTokenizationResult(textMateResult.tokens, endState));
71
+ return ( new EncodedTokenizationResult(textMateResult.tokens, textMateResult.fonts, endState));
72
72
  }
73
73
  }
74
74
 
@@ -10,26 +10,26 @@ const tokenClassificationRegistry = getTokenClassificationRegistry();
10
10
  const tokenTypeExtPoint = ExtensionsRegistry.registerExtensionPoint({
11
11
  extensionPoint: 'semanticTokenTypes',
12
12
  jsonSchema: {
13
- description: ( localize(14428, 'Contributes semantic token types.')),
13
+ description: ( localize(14486, 'Contributes semantic token types.')),
14
14
  type: 'array',
15
15
  items: {
16
16
  type: 'object',
17
17
  properties: {
18
18
  id: {
19
19
  type: 'string',
20
- description: ( localize(14429, 'The identifier of the semantic token type')),
20
+ description: ( localize(14487, 'The identifier of the semantic token type')),
21
21
  pattern: typeAndModifierIdPattern,
22
- patternErrorMessage: ( localize(14430, 'Identifiers should be in the form letterOrDigit[_-letterOrDigit]*')),
22
+ patternErrorMessage: ( localize(14488, 'Identifiers should be in the form letterOrDigit[_-letterOrDigit]*')),
23
23
  },
24
24
  superType: {
25
25
  type: 'string',
26
- description: ( localize(14431, 'The super type of the semantic token type')),
26
+ description: ( localize(14489, 'The super type of the semantic token type')),
27
27
  pattern: typeAndModifierIdPattern,
28
- patternErrorMessage: ( localize(14432, 'Super types should be in the form letterOrDigit[_-letterOrDigit]*')),
28
+ patternErrorMessage: ( localize(14490, 'Super types should be in the form letterOrDigit[_-letterOrDigit]*')),
29
29
  },
30
30
  description: {
31
31
  type: 'string',
32
- description: ( localize(14433, 'The description of the semantic token type')),
32
+ description: ( localize(14491, 'The description of the semantic token type')),
33
33
  }
34
34
  }
35
35
  }
@@ -38,20 +38,20 @@ const tokenTypeExtPoint = ExtensionsRegistry.registerExtensionPoint({
38
38
  const tokenModifierExtPoint = ExtensionsRegistry.registerExtensionPoint({
39
39
  extensionPoint: 'semanticTokenModifiers',
40
40
  jsonSchema: {
41
- description: ( localize(14434, 'Contributes semantic token modifiers.')),
41
+ description: ( localize(14492, 'Contributes semantic token modifiers.')),
42
42
  type: 'array',
43
43
  items: {
44
44
  type: 'object',
45
45
  properties: {
46
46
  id: {
47
47
  type: 'string',
48
- description: ( localize(14435, 'The identifier of the semantic token modifier')),
48
+ description: ( localize(14493, 'The identifier of the semantic token modifier')),
49
49
  pattern: typeAndModifierIdPattern,
50
- patternErrorMessage: ( localize(14436, 'Identifiers should be in the form letterOrDigit[_-letterOrDigit]*'))
50
+ patternErrorMessage: ( localize(14494, 'Identifiers should be in the form letterOrDigit[_-letterOrDigit]*'))
51
51
  },
52
52
  description: {
53
53
  type: 'string',
54
- description: ( localize(14437, 'The description of the semantic token modifier'))
54
+ description: ( localize(14495, 'The description of the semantic token modifier'))
55
55
  }
56
56
  }
57
57
  }
@@ -60,18 +60,18 @@ const tokenModifierExtPoint = ExtensionsRegistry.registerExtensionPoint({
60
60
  const tokenStyleDefaultsExtPoint = ExtensionsRegistry.registerExtensionPoint({
61
61
  extensionPoint: 'semanticTokenScopes',
62
62
  jsonSchema: {
63
- description: ( localize(14438, 'Contributes semantic token scope maps.')),
63
+ description: ( localize(14496, 'Contributes semantic token scope maps.')),
64
64
  type: 'array',
65
65
  items: {
66
66
  type: 'object',
67
67
  properties: {
68
68
  language: {
69
- description: ( localize(14439, 'Lists the languge for which the defaults are.')),
69
+ description: ( localize(14497, 'Lists the languge for which the defaults are.')),
70
70
  type: 'string'
71
71
  },
72
72
  scopes: {
73
73
  description: ( localize(
74
- 14440,
74
+ 14498,
75
75
  'Maps a semantic token (described by semantic token selector) to one or more textMate scopes used to represent that token.'
76
76
  )),
77
77
  type: 'object',
@@ -91,7 +91,7 @@ class TokenClassificationExtensionPoints {
91
91
  function validateTypeOrModifier(contribution, extensionPoint, collector) {
92
92
  if (typeof contribution.id !== 'string' || contribution.id.length === 0) {
93
93
  collector.error(( localize(
94
- 14441,
94
+ 14499,
95
95
  "'configuration.{0}.id' must be defined and can not be empty",
96
96
  extensionPoint
97
97
  )));
@@ -99,7 +99,7 @@ class TokenClassificationExtensionPoints {
99
99
  }
100
100
  if (!contribution.id.match(typeAndModifierIdPattern)) {
101
101
  collector.error(( localize(
102
- 14442,
102
+ 14500,
103
103
  "'configuration.{0}.id' must follow the pattern letterOrDigit[-_letterOrDigit]*",
104
104
  extensionPoint
105
105
  )));
@@ -108,7 +108,7 @@ class TokenClassificationExtensionPoints {
108
108
  const superType = contribution.superType;
109
109
  if (superType && !superType.match(typeAndModifierIdPattern)) {
110
110
  collector.error(( localize(
111
- 14443,
111
+ 14501,
112
112
  "'configuration.{0}.superType' must follow the pattern letterOrDigit[-_letterOrDigit]*",
113
113
  extensionPoint
114
114
  )));
@@ -116,7 +116,7 @@ class TokenClassificationExtensionPoints {
116
116
  }
117
117
  if (typeof contribution.description !== 'string' || contribution.id.length === 0) {
118
118
  collector.error(( localize(
119
- 14444,
119
+ 14502,
120
120
  "'configuration.{0}.description' must be defined and can not be empty",
121
121
  extensionPoint
122
122
  )));
@@ -129,7 +129,7 @@ class TokenClassificationExtensionPoints {
129
129
  const extensionValue = extension.value;
130
130
  const collector = extension.collector;
131
131
  if (!extensionValue || !Array.isArray(extensionValue)) {
132
- collector.error(( localize(14445, "'configuration.semanticTokenType' must be an array")));
132
+ collector.error(( localize(14503, "'configuration.semanticTokenType' must be an array")));
133
133
  return;
134
134
  }
135
135
  for (const contribution of extensionValue) {
@@ -150,7 +150,7 @@ class TokenClassificationExtensionPoints {
150
150
  const extensionValue = extension.value;
151
151
  const collector = extension.collector;
152
152
  if (!extensionValue || !Array.isArray(extensionValue)) {
153
- collector.error(( localize(14446, "'configuration.semanticTokenModifier' must be an array")));
153
+ collector.error(( localize(14504, "'configuration.semanticTokenModifier' must be an array")));
154
154
  return;
155
155
  }
156
156
  for (const contribution of extensionValue) {
@@ -171,17 +171,17 @@ class TokenClassificationExtensionPoints {
171
171
  const extensionValue = extension.value;
172
172
  const collector = extension.collector;
173
173
  if (!extensionValue || !Array.isArray(extensionValue)) {
174
- collector.error(( localize(14447, "'configuration.semanticTokenScopes' must be an array")));
174
+ collector.error(( localize(14505, "'configuration.semanticTokenScopes' must be an array")));
175
175
  return;
176
176
  }
177
177
  for (const contribution of extensionValue) {
178
178
  if (contribution.language && typeof contribution.language !== 'string') {
179
- collector.error(( localize(14448, "'configuration.semanticTokenScopes.language' must be a string")));
179
+ collector.error(( localize(14506, "'configuration.semanticTokenScopes.language' must be a string")));
180
180
  continue;
181
181
  }
182
182
  if (!contribution.scopes || typeof contribution.scopes !== 'object') {
183
183
  collector.error(( localize(
184
- 14449,
184
+ 14507,
185
185
  "'configuration.semanticTokenScopes.scopes' must be defined as an object"
186
186
  )));
187
187
  continue;
@@ -190,7 +190,7 @@ class TokenClassificationExtensionPoints {
190
190
  const tmScopes = contribution.scopes[selectorString];
191
191
  if (!Array.isArray(tmScopes) || ( tmScopes.some(l => typeof l !== 'string'))) {
192
192
  collector.error(( localize(
193
- 14450,
193
+ 14508,
194
194
  "'configuration.semanticTokenScopes.scopes' values must be an array of strings"
195
195
  )));
196
196
  continue;
@@ -201,7 +201,7 @@ class TokenClassificationExtensionPoints {
201
201
  }
202
202
  catch (e) {
203
203
  collector.error(( localize(
204
- 14451,
204
+ 14509,
205
205
  "configuration.semanticTokenScopes.scopes': Problems parsing selector {0}.",
206
206
  selectorString
207
207
  )));