@codingame/monaco-vscode-textmate-service-override 24.3.0 → 25.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +4 -4
- package/vscode/src/vs/workbench/services/textMate/browser/backgroundTokenization/textMateWorkerTokenizerController.d.ts +4 -1
- package/vscode/src/vs/workbench/services/textMate/browser/backgroundTokenization/textMateWorkerTokenizerController.js +21 -1
- package/vscode/src/vs/workbench/services/textMate/browser/backgroundTokenization/threadedBackgroundTokenizerFactory.js +2 -2
- package/vscode/src/vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateTokenizationWorker.worker.js +2 -2
- package/vscode/src/vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateWorkerHost.d.ts +3 -1
- package/vscode/src/vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateWorkerTokenizer.d.ts +5 -1
- package/vscode/src/vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateWorkerTokenizer.js +39 -1
- package/vscode/src/vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl.d.ts +1 -0
- package/vscode/src/vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl.js +18 -15
- package/vscode/src/vs/workbench/services/textMate/browser/tokenizationSupport/textMateTokenizationSupport.js +2 -2
- package/vscode/src/vs/workbench/services/themes/common/tokenClassificationExtensionPoint.js +24 -24
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@codingame/monaco-vscode-textmate-service-override",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "25.0.0",
|
|
4
4
|
"private": false,
|
|
5
5
|
"description": "VSCode public API plugged on the monaco editor - textmate service-override",
|
|
6
6
|
"keywords": [],
|
|
@@ -15,10 +15,10 @@
|
|
|
15
15
|
},
|
|
16
16
|
"type": "module",
|
|
17
17
|
"dependencies": {
|
|
18
|
-
"@codingame/monaco-vscode-api": "
|
|
19
|
-
"@codingame/monaco-vscode-files-service-override": "
|
|
18
|
+
"@codingame/monaco-vscode-api": "25.0.0",
|
|
19
|
+
"@codingame/monaco-vscode-files-service-override": "25.0.0",
|
|
20
20
|
"vscode-oniguruma": "1.7.0",
|
|
21
|
-
"vscode-textmate": "9.
|
|
21
|
+
"vscode-textmate": "9.3.1"
|
|
22
22
|
},
|
|
23
23
|
"main": "index.js",
|
|
24
24
|
"module": "index.js",
|
|
@@ -3,8 +3,10 @@ import { IObservable } from "@codingame/monaco-vscode-api/vscode/vs/base/common/
|
|
|
3
3
|
import { Proxied } from "@codingame/monaco-vscode-api/vscode/vs/base/common/worker/webWorker";
|
|
4
4
|
import { IBackgroundTokenizationStore, ILanguageIdCodec } from "@codingame/monaco-vscode-api/vscode/vs/editor/common/languages";
|
|
5
5
|
import { ITextModel } from "@codingame/monaco-vscode-api/vscode/vs/editor/common/model";
|
|
6
|
+
import { IFontTokenOption } from "@codingame/monaco-vscode-api/vscode/vs/editor/common/textModelEvents";
|
|
6
7
|
import { IConfigurationService } from "@codingame/monaco-vscode-api/vscode/vs/platform/configuration/common/configuration.service";
|
|
7
8
|
import type { StateDeltas, TextMateTokenizationWorker } from "./worker/textMateTokenizationWorker.worker.js";
|
|
9
|
+
import { ISerializedAnnotation } from "@codingame/monaco-vscode-api/vscode/vs/editor/common/model/tokens/annotations";
|
|
8
10
|
export declare class TextMateWorkerTokenizerController extends Disposable {
|
|
9
11
|
private readonly _model;
|
|
10
12
|
private readonly _worker;
|
|
@@ -29,6 +31,7 @@ export declare class TextMateWorkerTokenizerController extends Disposable {
|
|
|
29
31
|
/**
|
|
30
32
|
* This method is called from the worker through the worker host.
|
|
31
33
|
*/
|
|
32
|
-
setTokensAndStates(controllerId: number, versionId: number, rawTokens: Uint8Array, stateDeltas: StateDeltas[]): Promise<void>;
|
|
34
|
+
setTokensAndStates(controllerId: number, versionId: number, rawTokens: Uint8Array, fontTokens: ISerializedAnnotation<IFontTokenOption>[], stateDeltas: StateDeltas[]): Promise<void>;
|
|
35
|
+
private _stringEditFromChanges;
|
|
33
36
|
private get _shouldLog();
|
|
34
37
|
}
|
|
@@ -4,10 +4,14 @@ import '@codingame/monaco-vscode-api/vscode/vs/base/common/observableInternal/in
|
|
|
4
4
|
import { LineRange } from '@codingame/monaco-vscode-api/vscode/vs/editor/common/core/ranges/lineRange';
|
|
5
5
|
import { Range } from '@codingame/monaco-vscode-api/vscode/vs/editor/common/core/range';
|
|
6
6
|
import { TokenizationStateStore } from '@codingame/monaco-vscode-api/vscode/vs/editor/common/model/textModelTokens';
|
|
7
|
+
import { deserializeFontTokenOptions } from '@codingame/monaco-vscode-api/vscode/vs/editor/common/textModelEvents';
|
|
7
8
|
import { ContiguousMultilineTokensBuilder } from '@codingame/monaco-vscode-api/vscode/vs/editor/common/tokens/contiguousMultilineTokensBuilder';
|
|
8
9
|
import { observableConfigValue } from '@codingame/monaco-vscode-api/vscode/vs/platform/observable/common/platformObservableUtils';
|
|
9
10
|
import { MonotonousIndexTransformer } from '../indexTransformer.js';
|
|
10
11
|
import { linesLengthEditFromModelContentChange } from '@codingame/monaco-vscode-api/vscode/vs/editor/common/model/textModelStringEdit';
|
|
12
|
+
import { StringEdit } from '@codingame/monaco-vscode-api/vscode/vs/editor/common/core/edits/stringEdit';
|
|
13
|
+
import { OffsetRange } from '@codingame/monaco-vscode-api/vscode/vs/editor/common/core/ranges/offsetRange';
|
|
14
|
+
import { AnnotationsUpdate } from '@codingame/monaco-vscode-api/vscode/vs/editor/common/model/tokens/annotations';
|
|
11
15
|
import { keepObserved } from '@codingame/monaco-vscode-api/vscode/vs/base/common/observableInternal/utils/utils';
|
|
12
16
|
import { autorun } from '@codingame/monaco-vscode-api/vscode/vs/base/common/observableInternal/reactions/autorun';
|
|
13
17
|
|
|
@@ -65,11 +69,12 @@ class TextMateWorkerTokenizerController extends Disposable {
|
|
|
65
69
|
requestTokens(startLineNumber, endLineNumberExclusive) {
|
|
66
70
|
this._worker.$retokenize(this.controllerId, startLineNumber, endLineNumberExclusive);
|
|
67
71
|
}
|
|
68
|
-
async setTokensAndStates(controllerId, versionId, rawTokens, stateDeltas) {
|
|
72
|
+
async setTokensAndStates(controllerId, versionId, rawTokens, fontTokens, stateDeltas) {
|
|
69
73
|
if (this.controllerId !== controllerId) {
|
|
70
74
|
return;
|
|
71
75
|
}
|
|
72
76
|
let tokens = ContiguousMultilineTokensBuilder.deserialize(( new Uint8Array(rawTokens)));
|
|
77
|
+
const fontTokensUpdate = AnnotationsUpdate.deserialize(fontTokens, deserializeFontTokenOptions());
|
|
73
78
|
if (this._shouldLog) {
|
|
74
79
|
console.log('received background tokenization result', {
|
|
75
80
|
fileName: this._model.uri.fsPath.split('\\').pop(),
|
|
@@ -109,6 +114,7 @@ class TextMateWorkerTokenizerController extends Disposable {
|
|
|
109
114
|
}
|
|
110
115
|
}
|
|
111
116
|
}
|
|
117
|
+
fontTokensUpdate.rebase(this._stringEditFromChanges(this._model, this._pendingChanges));
|
|
112
118
|
}
|
|
113
119
|
const curToFutureTransformerStates = MonotonousIndexTransformer.fromMany(( this._pendingChanges.map((c) => linesLengthEditFromModelContentChange(c.changes))));
|
|
114
120
|
if (!this._applyStateStackDiffFn || !this._initialState) {
|
|
@@ -139,6 +145,20 @@ class TextMateWorkerTokenizerController extends Disposable {
|
|
|
139
145
|
}
|
|
140
146
|
}
|
|
141
147
|
this._backgroundTokenizationStore.setTokens(tokens);
|
|
148
|
+
this._backgroundTokenizationStore.setFontInfo(fontTokensUpdate);
|
|
149
|
+
}
|
|
150
|
+
_stringEditFromChanges(model, pendingChanges) {
|
|
151
|
+
const edits = [];
|
|
152
|
+
for (const change of pendingChanges) {
|
|
153
|
+
for (const innerChanges of change.changes) {
|
|
154
|
+
const range = Range.lift(innerChanges.range);
|
|
155
|
+
const text = innerChanges.text;
|
|
156
|
+
const offsetEditStart = model.getOffsetAt(range.getStartPosition());
|
|
157
|
+
const offsetEditEnd = model.getOffsetAt(range.getEndPosition());
|
|
158
|
+
edits.push(StringEdit.replace(( new OffsetRange(offsetEditStart, offsetEditEnd)), text));
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
return StringEdit.compose(edits);
|
|
142
162
|
}
|
|
143
163
|
get _shouldLog() { return this._loggingEnabled.get(); }
|
|
144
164
|
}
|
|
@@ -122,10 +122,10 @@ let ThreadedBackgroundTokenizerFactory = class ThreadedBackgroundTokenizerFactor
|
|
|
122
122
|
const resource = URI.revive(_resource);
|
|
123
123
|
return this._extensionResourceLoaderService.readExtensionResource(resource);
|
|
124
124
|
},
|
|
125
|
-
$setTokensAndStates: async (controllerId, versionId, tokens, lineEndStateDeltas) => {
|
|
125
|
+
$setTokensAndStates: async (controllerId, versionId, tokens, fontTokens, lineEndStateDeltas) => {
|
|
126
126
|
const controller = this._workerTokenizerControllers.get(controllerId);
|
|
127
127
|
if (controller) {
|
|
128
|
-
controller.setTokensAndStates(controllerId, versionId, tokens, lineEndStateDeltas);
|
|
128
|
+
controller.setTokensAndStates(controllerId, versionId, tokens, fontTokens, lineEndStateDeltas);
|
|
129
129
|
}
|
|
130
130
|
},
|
|
131
131
|
$reportTokenizationTime: (timeMs, languageId, sourceExtensionId, lineLength, isRandomSample) => {
|
|
@@ -61,8 +61,8 @@ class TextMateTokenizationWorker {
|
|
|
61
61
|
}
|
|
62
62
|
return that._grammarCache[encodedLanguageId];
|
|
63
63
|
},
|
|
64
|
-
setTokensAndStates(versionId, tokens, stateDeltas) {
|
|
65
|
-
that._host.$setTokensAndStates(data.controllerId, versionId, tokens, stateDeltas);
|
|
64
|
+
setTokensAndStates(versionId, tokens, fontTokens, stateDeltas) {
|
|
65
|
+
that._host.$setTokensAndStates(data.controllerId, versionId, tokens, fontTokens, stateDeltas);
|
|
66
66
|
},
|
|
67
67
|
reportTokenizationTime(timeMs, languageId, sourceExtensionId, lineLength, isRandomSample) {
|
|
68
68
|
that._host.$reportTokenizationTime(timeMs, languageId, sourceExtensionId, lineLength, isRandomSample);
|
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
import { UriComponents } from "@codingame/monaco-vscode-api/vscode/vs/base/common/uri";
|
|
2
2
|
import { IWebWorkerServer, IWebWorkerClient } from "@codingame/monaco-vscode-api/vscode/vs/base/common/worker/webWorker";
|
|
3
|
+
import { ISerializedAnnotation } from "@codingame/monaco-vscode-api/vscode/vs/editor/common/model/tokens/annotations";
|
|
4
|
+
import { IFontTokenOption } from "@codingame/monaco-vscode-api/vscode/vs/editor/common/textModelEvents";
|
|
3
5
|
import { StateDeltas } from "./textMateTokenizationWorker.worker.js";
|
|
4
6
|
export declare abstract class TextMateWorkerHost {
|
|
5
7
|
static CHANNEL_NAME: string;
|
|
6
8
|
static getChannel(workerServer: IWebWorkerServer): TextMateWorkerHost;
|
|
7
9
|
static setChannel(workerClient: IWebWorkerClient<unknown>, obj: TextMateWorkerHost): void;
|
|
8
10
|
abstract $readFile(_resource: UriComponents): Promise<string>;
|
|
9
|
-
abstract $setTokensAndStates(controllerId: number, versionId: number, tokens: Uint8Array, lineEndStateDeltas: StateDeltas[]): Promise<void>;
|
|
11
|
+
abstract $setTokensAndStates(controllerId: number, versionId: number, tokens: Uint8Array, fontTokens: ISerializedAnnotation<IFontTokenOption>[], lineEndStateDeltas: StateDeltas[]): Promise<void>;
|
|
10
12
|
abstract $reportTokenizationTime(timeMs: number, languageId: string, sourceExtensionId: string | undefined, lineLength: number, isRandomSample: boolean): void;
|
|
11
13
|
}
|
|
@@ -3,9 +3,11 @@ import { LanguageId } from "@codingame/monaco-vscode-api/vscode/vs/editor/common
|
|
|
3
3
|
import { IModelChangedEvent, MirrorTextModel } from "@codingame/monaco-vscode-api/vscode/vs/editor/common/model/mirrorTextModel";
|
|
4
4
|
import { ICreateGrammarResult } from "../../../common/TMGrammarFactory.js";
|
|
5
5
|
import { StateDeltas } from "./textMateTokenizationWorker.worker.js";
|
|
6
|
+
import { IFontTokenOption } from "@codingame/monaco-vscode-api/vscode/vs/editor/common/textModelEvents";
|
|
7
|
+
import { ISerializedAnnotation } from "@codingame/monaco-vscode-api/vscode/vs/editor/common/model/tokens/annotations";
|
|
6
8
|
export interface TextMateModelTokenizerHost {
|
|
7
9
|
getOrCreateGrammar(languageId: string, encodedLanguageId: LanguageId): Promise<ICreateGrammarResult | null>;
|
|
8
|
-
setTokensAndStates(versionId: number, tokens: Uint8Array, stateDeltas: StateDeltas[]): void;
|
|
10
|
+
setTokensAndStates(versionId: number, tokens: Uint8Array, fontTokens: ISerializedAnnotation<IFontTokenOption>[], stateDeltas: StateDeltas[]): void;
|
|
9
11
|
reportTokenizationTime(timeMs: number, languageId: string, sourceExtensionId: string | undefined, lineLength: number, isRandomSample: boolean): void;
|
|
10
12
|
}
|
|
11
13
|
export declare class TextMateWorkerTokenizer extends MirrorTextModel {
|
|
@@ -25,4 +27,6 @@ export declare class TextMateWorkerTokenizer extends MirrorTextModel {
|
|
|
25
27
|
retokenize(startLineNumber: number, endLineNumberExclusive: number): void;
|
|
26
28
|
private _resetTokenization;
|
|
27
29
|
private _tokenize;
|
|
30
|
+
private _getFontTokensUpdate;
|
|
31
|
+
private _getOffsetAtLineStart;
|
|
28
32
|
}
|
|
@@ -10,6 +10,9 @@ import { LineTokens } from '@codingame/monaco-vscode-api/vscode/vs/editor/common
|
|
|
10
10
|
import { TextMateTokenizationSupport } from '../../tokenizationSupport/textMateTokenizationSupport.js';
|
|
11
11
|
import { TokenizationSupportWithLineLimit } from '../../tokenizationSupport/tokenizationSupportWithLineLimit.js';
|
|
12
12
|
import { Disposable } from '@codingame/monaco-vscode-api/vscode/vs/base/common/lifecycle';
|
|
13
|
+
import { serializeFontTokenOptions } from '@codingame/monaco-vscode-api/vscode/vs/editor/common/textModelEvents';
|
|
14
|
+
import { AnnotationsUpdate } from '@codingame/monaco-vscode-api/vscode/vs/editor/common/model/tokens/annotations';
|
|
15
|
+
import { OffsetRange } from '@codingame/monaco-vscode-api/vscode/vs/editor/common/core/ranges/offsetRange';
|
|
13
16
|
import { observableValue } from '@codingame/monaco-vscode-api/vscode/vs/base/common/observableInternal/observables/observableValue';
|
|
14
17
|
|
|
15
18
|
class TextMateWorkerTokenizer extends MirrorTextModel {
|
|
@@ -88,6 +91,7 @@ class TextMateWorkerTokenizer extends MirrorTextModel {
|
|
|
88
91
|
let tokenizedLines = 0;
|
|
89
92
|
const tokenBuilder = ( new ContiguousMultilineTokensBuilder());
|
|
90
93
|
const stateDeltaBuilder = ( new StateDeltaBuilder());
|
|
94
|
+
const fontTokensUpdate = [];
|
|
91
95
|
while (true) {
|
|
92
96
|
const lineToTokenize = this._tokenizerWithStateStore.getFirstInvalidLine();
|
|
93
97
|
if (lineToTokenize === null || tokenizedLines > 200) {
|
|
@@ -105,6 +109,7 @@ class TextMateWorkerTokenizer extends MirrorTextModel {
|
|
|
105
109
|
}
|
|
106
110
|
LineTokens.convertToEndOffset(r.tokens, text.length);
|
|
107
111
|
tokenBuilder.add(lineToTokenize.lineNumber, r.tokens);
|
|
112
|
+
fontTokensUpdate.push(...this._getFontTokensUpdate(lineToTokenize.lineNumber, r));
|
|
108
113
|
const deltaMs = ( new Date()).getTime() - startTime;
|
|
109
114
|
if (deltaMs > 20) {
|
|
110
115
|
break;
|
|
@@ -113,8 +118,10 @@ class TextMateWorkerTokenizer extends MirrorTextModel {
|
|
|
113
118
|
if (tokenizedLines === 0) {
|
|
114
119
|
break;
|
|
115
120
|
}
|
|
121
|
+
const fontUpdate = AnnotationsUpdate.create(fontTokensUpdate);
|
|
122
|
+
const serializedFontUpdate = fontUpdate.serialize(serializeFontTokenOptions());
|
|
116
123
|
const stateDeltas = stateDeltaBuilder.getStateDeltas();
|
|
117
|
-
this._host.setTokensAndStates(this._versionId, tokenBuilder.serialize(), stateDeltas);
|
|
124
|
+
this._host.setTokensAndStates(this._versionId, tokenBuilder.serialize(), serializedFontUpdate, stateDeltas);
|
|
118
125
|
const deltaMs = ( new Date()).getTime() - startTime;
|
|
119
126
|
if (deltaMs > 20) {
|
|
120
127
|
setTimeout0(() => this._tokenize());
|
|
@@ -122,6 +129,37 @@ class TextMateWorkerTokenizer extends MirrorTextModel {
|
|
|
122
129
|
}
|
|
123
130
|
}
|
|
124
131
|
}
|
|
132
|
+
_getFontTokensUpdate(lineNumber, r) {
|
|
133
|
+
const fontTokens = [];
|
|
134
|
+
const offsetAtLineStart = this._getOffsetAtLineStart(lineNumber);
|
|
135
|
+
const offsetAtNextLineStart = this._getOffsetAtLineStart(lineNumber + 1);
|
|
136
|
+
const offsetAtLineEnd = offsetAtNextLineStart > 0 ? offsetAtNextLineStart - 1 : 0;
|
|
137
|
+
fontTokens.push({
|
|
138
|
+
range: ( new OffsetRange(offsetAtLineStart, offsetAtLineEnd)),
|
|
139
|
+
annotation: undefined
|
|
140
|
+
});
|
|
141
|
+
if (r.fontInfo.length) {
|
|
142
|
+
for (const fontInfo of r.fontInfo) {
|
|
143
|
+
const offsetAtLineStart = this._getOffsetAtLineStart(lineNumber);
|
|
144
|
+
fontTokens.push({
|
|
145
|
+
range: ( new OffsetRange(
|
|
146
|
+
offsetAtLineStart + fontInfo.startIndex,
|
|
147
|
+
offsetAtLineStart + fontInfo.endIndex
|
|
148
|
+
)),
|
|
149
|
+
annotation: {
|
|
150
|
+
fontFamily: fontInfo.fontFamily ?? undefined,
|
|
151
|
+
fontSize: fontInfo.fontSize ?? undefined,
|
|
152
|
+
lineHeight: fontInfo.lineHeight ?? undefined
|
|
153
|
+
}
|
|
154
|
+
});
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
return fontTokens;
|
|
158
|
+
}
|
|
159
|
+
_getOffsetAtLineStart(lineNumber) {
|
|
160
|
+
this._ensureLineStarts();
|
|
161
|
+
return lineNumber - 1 > 0 ? this._lineStarts.getPrefixSum(lineNumber - 2) : 0;
|
|
162
|
+
}
|
|
125
163
|
}
|
|
126
164
|
class StateDeltaBuilder {
|
|
127
165
|
constructor() {
|
package/vscode/src/vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl.d.ts
CHANGED
|
@@ -34,6 +34,7 @@ export declare class TextMateTokenizationFeature extends Disposable implements I
|
|
|
34
34
|
private readonly _tokenizersRegistrations;
|
|
35
35
|
private _currentTheme;
|
|
36
36
|
private _currentTokenColorMap;
|
|
37
|
+
private _currentTokenFontMap;
|
|
37
38
|
private readonly _threadedBackgroundTokenizerFactory;
|
|
38
39
|
constructor(_languageService: ILanguageService, _themeService: IWorkbenchThemeService, _extensionResourceLoaderService: IExtensionResourceLoaderService, _notificationService: INotificationService, _logService: ILogService, _configurationService: IConfigurationService, _progressService: IProgressService, _environmentService: IWorkbenchEnvironmentService, _instantiationService: IInstantiationService, _telemetryService: ITelemetryService);
|
|
39
40
|
private getAsyncTokenizationEnabled;
|
package/vscode/src/vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl.js
CHANGED
|
@@ -13,7 +13,7 @@ import { isObject } from '@codingame/monaco-vscode-api/vscode/vs/base/common/typ
|
|
|
13
13
|
import { StandardTokenType } from '@codingame/monaco-vscode-api/vscode/vs/editor/common/encodedTokenAttributes';
|
|
14
14
|
import { LazyTokenizationSupport, TokenizationRegistry } from '@codingame/monaco-vscode-api/vscode/vs/editor/common/languages';
|
|
15
15
|
import { ILanguageService } from '@codingame/monaco-vscode-api/vscode/vs/editor/common/languages/language.service';
|
|
16
|
-
import { generateTokensCSSForColorMap } from '@codingame/monaco-vscode-api/vscode/vs/editor/common/languages/supports/tokenization';
|
|
16
|
+
import { generateTokensCSSForColorMap, generateTokensCSSForFontMap } from '@codingame/monaco-vscode-api/vscode/vs/editor/common/languages/supports/tokenization';
|
|
17
17
|
import { localize } from '@codingame/monaco-vscode-api/vscode/vs/nls';
|
|
18
18
|
import { IConfigurationService } from '@codingame/monaco-vscode-api/vscode/vs/platform/configuration/common/configuration.service';
|
|
19
19
|
import { IExtensionResourceLoaderService } from '@codingame/monaco-vscode-api/vscode/vs/platform/extensionResourceLoader/common/extensionResourceLoader.service';
|
|
@@ -57,6 +57,7 @@ let TextMateTokenizationFeature = class TextMateTokenizationFeature extends Disp
|
|
|
57
57
|
this._tokenizersRegistrations = this._register(( new DisposableStore()));
|
|
58
58
|
this._currentTheme = null;
|
|
59
59
|
this._currentTokenColorMap = null;
|
|
60
|
+
this._currentTokenFontMap = null;
|
|
60
61
|
this._threadedBackgroundTokenizerFactory = this._instantiationService.createInstance(ThreadedBackgroundTokenizerFactory, (timeMs, languageId, sourceExtensionId, lineLength, isRandomSample) => this._reportTokenizationTime(timeMs, languageId, sourceExtensionId, lineLength, true, isRandomSample), () => this.getAsyncTokenizationEnabled());
|
|
61
62
|
this._vscodeOniguruma = null;
|
|
62
63
|
this._styleElement = createStyleSheet();
|
|
@@ -164,7 +165,7 @@ let TextMateTokenizationFeature = class TextMateTokenizationFeature extends Disp
|
|
|
164
165
|
}
|
|
165
166
|
startDebugMode(printFn, onStop) {
|
|
166
167
|
if (this._debugMode) {
|
|
167
|
-
this._notificationService.error(( localize(
|
|
168
|
+
this._notificationService.error(( localize(14276, "Already Logging.")));
|
|
168
169
|
return;
|
|
169
170
|
}
|
|
170
171
|
this._debugModePrintFunc = printFn;
|
|
@@ -172,15 +173,15 @@ let TextMateTokenizationFeature = class TextMateTokenizationFeature extends Disp
|
|
|
172
173
|
if (this._debugMode) {
|
|
173
174
|
this._progressService.withProgress({
|
|
174
175
|
location: ProgressLocation.Notification,
|
|
175
|
-
buttons: [( localize(
|
|
176
|
+
buttons: [( localize(14277, "Stop"))]
|
|
176
177
|
}, (progress) => {
|
|
177
178
|
progress.report({
|
|
178
|
-
message: ( localize(
|
|
179
|
+
message: ( localize(14278, "Preparing to log TM Grammar parsing. Press Stop when finished."))
|
|
179
180
|
});
|
|
180
181
|
return this._getVSCodeOniguruma().then((vscodeOniguruma) => {
|
|
181
182
|
vscodeOniguruma.setDefaultDebugCall(true);
|
|
182
183
|
progress.report({
|
|
183
|
-
message: ( localize(
|
|
184
|
+
message: ( localize(14279, "Now logging TM Grammar parsing. Press Stop when finished."))
|
|
184
185
|
});
|
|
185
186
|
return ( new Promise((resolve, reject) => { }));
|
|
186
187
|
});
|
|
@@ -266,15 +267,17 @@ let TextMateTokenizationFeature = class TextMateTokenizationFeature extends Disp
|
|
|
266
267
|
}
|
|
267
268
|
_updateTheme(colorTheme, forceUpdate) {
|
|
268
269
|
if (!forceUpdate && this._currentTheme && this._currentTokenColorMap && equalsTokenRules(this._currentTheme.settings, colorTheme.tokenColors)
|
|
269
|
-
&& equals(this._currentTokenColorMap, colorTheme.tokenColorMap)) {
|
|
270
|
+
&& equals(this._currentTokenColorMap, colorTheme.tokenColorMap) && this._currentTokenFontMap && equals(this._currentTokenFontMap, colorTheme.tokenFontMap)) {
|
|
270
271
|
return;
|
|
271
272
|
}
|
|
272
273
|
this._currentTheme = { name: colorTheme.label, settings: colorTheme.tokenColors };
|
|
273
274
|
this._currentTokenColorMap = colorTheme.tokenColorMap;
|
|
275
|
+
this._currentTokenFontMap = colorTheme.tokenFontMap;
|
|
274
276
|
this._grammarFactory?.setTheme(this._currentTheme, this._currentTokenColorMap);
|
|
275
277
|
const colorMap = toColorMap(this._currentTokenColorMap);
|
|
276
|
-
const
|
|
277
|
-
|
|
278
|
+
const colorCssRules = generateTokensCSSForColorMap(colorMap);
|
|
279
|
+
const fontCssRules = generateTokensCSSForFontMap(this._currentTokenFontMap);
|
|
280
|
+
this._styleElement.textContent = colorCssRules + fontCssRules;
|
|
278
281
|
TokenizationRegistry.setColorMap(colorMap);
|
|
279
282
|
if (this._currentTheme && this._currentTokenColorMap) {
|
|
280
283
|
this._threadedBackgroundTokenizerFactory.acceptTheme(this._currentTheme, this._currentTokenColorMap);
|
|
@@ -384,7 +387,7 @@ function equalsTokenRules(a, b) {
|
|
|
384
387
|
function validateGrammarExtensionPoint(extensionLocation, syntax, collector, _languageService) {
|
|
385
388
|
if (syntax.language && ((typeof syntax.language !== 'string') || !_languageService.isRegisteredLanguageId(syntax.language))) {
|
|
386
389
|
collector.error(( localize(
|
|
387
|
-
|
|
390
|
+
14280,
|
|
388
391
|
"Unknown language in `contributes.{0}.language`. Provided value: {1}",
|
|
389
392
|
grammarsExtPoint.name,
|
|
390
393
|
String(syntax.language)
|
|
@@ -393,7 +396,7 @@ function validateGrammarExtensionPoint(extensionLocation, syntax, collector, _la
|
|
|
393
396
|
}
|
|
394
397
|
if (!syntax.scopeName || (typeof syntax.scopeName !== 'string')) {
|
|
395
398
|
collector.error(( localize(
|
|
396
|
-
|
|
399
|
+
14281,
|
|
397
400
|
"Expected string in `contributes.{0}.scopeName`. Provided value: {1}",
|
|
398
401
|
grammarsExtPoint.name,
|
|
399
402
|
String(syntax.scopeName)
|
|
@@ -402,7 +405,7 @@ function validateGrammarExtensionPoint(extensionLocation, syntax, collector, _la
|
|
|
402
405
|
}
|
|
403
406
|
if (!syntax.path || (typeof syntax.path !== 'string')) {
|
|
404
407
|
collector.error(( localize(
|
|
405
|
-
|
|
408
|
+
14282,
|
|
406
409
|
"Expected string in `contributes.{0}.path`. Provided value: {1}",
|
|
407
410
|
grammarsExtPoint.name,
|
|
408
411
|
String(syntax.path)
|
|
@@ -411,7 +414,7 @@ function validateGrammarExtensionPoint(extensionLocation, syntax, collector, _la
|
|
|
411
414
|
}
|
|
412
415
|
if (syntax.injectTo && (!Array.isArray(syntax.injectTo) || ( syntax.injectTo.some(scope => typeof scope !== 'string')))) {
|
|
413
416
|
collector.error(( localize(
|
|
414
|
-
|
|
417
|
+
14283,
|
|
415
418
|
"Invalid value in `contributes.{0}.injectTo`. Must be an array of language scope names. Provided value: {1}",
|
|
416
419
|
grammarsExtPoint.name,
|
|
417
420
|
JSON.stringify(syntax.injectTo)
|
|
@@ -420,7 +423,7 @@ function validateGrammarExtensionPoint(extensionLocation, syntax, collector, _la
|
|
|
420
423
|
}
|
|
421
424
|
if (syntax.embeddedLanguages && !isObject(syntax.embeddedLanguages)) {
|
|
422
425
|
collector.error(( localize(
|
|
423
|
-
|
|
426
|
+
14284,
|
|
424
427
|
"Invalid value in `contributes.{0}.embeddedLanguages`. Must be an object map from scope name to language. Provided value: {1}",
|
|
425
428
|
grammarsExtPoint.name,
|
|
426
429
|
JSON.stringify(syntax.embeddedLanguages)
|
|
@@ -429,7 +432,7 @@ function validateGrammarExtensionPoint(extensionLocation, syntax, collector, _la
|
|
|
429
432
|
}
|
|
430
433
|
if (syntax.tokenTypes && !isObject(syntax.tokenTypes)) {
|
|
431
434
|
collector.error(( localize(
|
|
432
|
-
|
|
435
|
+
14285,
|
|
433
436
|
"Invalid value in `contributes.{0}.tokenTypes`. Must be an object map from scope name to token type. Provided value: {1}",
|
|
434
437
|
grammarsExtPoint.name,
|
|
435
438
|
JSON.stringify(syntax.tokenTypes)
|
|
@@ -439,7 +442,7 @@ function validateGrammarExtensionPoint(extensionLocation, syntax, collector, _la
|
|
|
439
442
|
const grammarLocation = joinPath(extensionLocation, syntax.path);
|
|
440
443
|
if (!isEqualOrParent(grammarLocation, extensionLocation)) {
|
|
441
444
|
collector.warn(( localize(
|
|
442
|
-
|
|
445
|
+
14286,
|
|
443
446
|
"Expected `contributes.{0}.path` ({1}) to be included inside extension's folder ({2}). This might make the extension non-portable.",
|
|
444
447
|
grammarsExtPoint.name,
|
|
445
448
|
grammarLocation.path,
|
|
@@ -47,7 +47,7 @@ class TextMateTokenizationSupport extends Disposable {
|
|
|
47
47
|
}
|
|
48
48
|
if (textMateResult.stoppedEarly) {
|
|
49
49
|
console.warn(`Time limit reached when tokenizing line: ${line.substring(0, 100)}`);
|
|
50
|
-
return ( new EncodedTokenizationResult(textMateResult.tokens, state));
|
|
50
|
+
return ( new EncodedTokenizationResult(textMateResult.tokens, textMateResult.fonts, state));
|
|
51
51
|
}
|
|
52
52
|
if (this._containsEmbeddedLanguages) {
|
|
53
53
|
const seenLanguages = this._seenLanguages;
|
|
@@ -68,7 +68,7 @@ class TextMateTokenizationSupport extends Disposable {
|
|
|
68
68
|
else {
|
|
69
69
|
endState = textMateResult.ruleStack;
|
|
70
70
|
}
|
|
71
|
-
return ( new EncodedTokenizationResult(textMateResult.tokens, endState));
|
|
71
|
+
return ( new EncodedTokenizationResult(textMateResult.tokens, textMateResult.fonts, endState));
|
|
72
72
|
}
|
|
73
73
|
}
|
|
74
74
|
|
|
@@ -10,26 +10,26 @@ const tokenClassificationRegistry = getTokenClassificationRegistry();
|
|
|
10
10
|
const tokenTypeExtPoint = ExtensionsRegistry.registerExtensionPoint({
|
|
11
11
|
extensionPoint: 'semanticTokenTypes',
|
|
12
12
|
jsonSchema: {
|
|
13
|
-
description: ( localize(
|
|
13
|
+
description: ( localize(14486, 'Contributes semantic token types.')),
|
|
14
14
|
type: 'array',
|
|
15
15
|
items: {
|
|
16
16
|
type: 'object',
|
|
17
17
|
properties: {
|
|
18
18
|
id: {
|
|
19
19
|
type: 'string',
|
|
20
|
-
description: ( localize(
|
|
20
|
+
description: ( localize(14487, 'The identifier of the semantic token type')),
|
|
21
21
|
pattern: typeAndModifierIdPattern,
|
|
22
|
-
patternErrorMessage: ( localize(
|
|
22
|
+
patternErrorMessage: ( localize(14488, 'Identifiers should be in the form letterOrDigit[_-letterOrDigit]*')),
|
|
23
23
|
},
|
|
24
24
|
superType: {
|
|
25
25
|
type: 'string',
|
|
26
|
-
description: ( localize(
|
|
26
|
+
description: ( localize(14489, 'The super type of the semantic token type')),
|
|
27
27
|
pattern: typeAndModifierIdPattern,
|
|
28
|
-
patternErrorMessage: ( localize(
|
|
28
|
+
patternErrorMessage: ( localize(14490, 'Super types should be in the form letterOrDigit[_-letterOrDigit]*')),
|
|
29
29
|
},
|
|
30
30
|
description: {
|
|
31
31
|
type: 'string',
|
|
32
|
-
description: ( localize(
|
|
32
|
+
description: ( localize(14491, 'The description of the semantic token type')),
|
|
33
33
|
}
|
|
34
34
|
}
|
|
35
35
|
}
|
|
@@ -38,20 +38,20 @@ const tokenTypeExtPoint = ExtensionsRegistry.registerExtensionPoint({
|
|
|
38
38
|
const tokenModifierExtPoint = ExtensionsRegistry.registerExtensionPoint({
|
|
39
39
|
extensionPoint: 'semanticTokenModifiers',
|
|
40
40
|
jsonSchema: {
|
|
41
|
-
description: ( localize(
|
|
41
|
+
description: ( localize(14492, 'Contributes semantic token modifiers.')),
|
|
42
42
|
type: 'array',
|
|
43
43
|
items: {
|
|
44
44
|
type: 'object',
|
|
45
45
|
properties: {
|
|
46
46
|
id: {
|
|
47
47
|
type: 'string',
|
|
48
|
-
description: ( localize(
|
|
48
|
+
description: ( localize(14493, 'The identifier of the semantic token modifier')),
|
|
49
49
|
pattern: typeAndModifierIdPattern,
|
|
50
|
-
patternErrorMessage: ( localize(
|
|
50
|
+
patternErrorMessage: ( localize(14494, 'Identifiers should be in the form letterOrDigit[_-letterOrDigit]*'))
|
|
51
51
|
},
|
|
52
52
|
description: {
|
|
53
53
|
type: 'string',
|
|
54
|
-
description: ( localize(
|
|
54
|
+
description: ( localize(14495, 'The description of the semantic token modifier'))
|
|
55
55
|
}
|
|
56
56
|
}
|
|
57
57
|
}
|
|
@@ -60,18 +60,18 @@ const tokenModifierExtPoint = ExtensionsRegistry.registerExtensionPoint({
|
|
|
60
60
|
const tokenStyleDefaultsExtPoint = ExtensionsRegistry.registerExtensionPoint({
|
|
61
61
|
extensionPoint: 'semanticTokenScopes',
|
|
62
62
|
jsonSchema: {
|
|
63
|
-
description: ( localize(
|
|
63
|
+
description: ( localize(14496, 'Contributes semantic token scope maps.')),
|
|
64
64
|
type: 'array',
|
|
65
65
|
items: {
|
|
66
66
|
type: 'object',
|
|
67
67
|
properties: {
|
|
68
68
|
language: {
|
|
69
|
-
description: ( localize(
|
|
69
|
+
description: ( localize(14497, 'Lists the languge for which the defaults are.')),
|
|
70
70
|
type: 'string'
|
|
71
71
|
},
|
|
72
72
|
scopes: {
|
|
73
73
|
description: ( localize(
|
|
74
|
-
|
|
74
|
+
14498,
|
|
75
75
|
'Maps a semantic token (described by semantic token selector) to one or more textMate scopes used to represent that token.'
|
|
76
76
|
)),
|
|
77
77
|
type: 'object',
|
|
@@ -91,7 +91,7 @@ class TokenClassificationExtensionPoints {
|
|
|
91
91
|
function validateTypeOrModifier(contribution, extensionPoint, collector) {
|
|
92
92
|
if (typeof contribution.id !== 'string' || contribution.id.length === 0) {
|
|
93
93
|
collector.error(( localize(
|
|
94
|
-
|
|
94
|
+
14499,
|
|
95
95
|
"'configuration.{0}.id' must be defined and can not be empty",
|
|
96
96
|
extensionPoint
|
|
97
97
|
)));
|
|
@@ -99,7 +99,7 @@ class TokenClassificationExtensionPoints {
|
|
|
99
99
|
}
|
|
100
100
|
if (!contribution.id.match(typeAndModifierIdPattern)) {
|
|
101
101
|
collector.error(( localize(
|
|
102
|
-
|
|
102
|
+
14500,
|
|
103
103
|
"'configuration.{0}.id' must follow the pattern letterOrDigit[-_letterOrDigit]*",
|
|
104
104
|
extensionPoint
|
|
105
105
|
)));
|
|
@@ -108,7 +108,7 @@ class TokenClassificationExtensionPoints {
|
|
|
108
108
|
const superType = contribution.superType;
|
|
109
109
|
if (superType && !superType.match(typeAndModifierIdPattern)) {
|
|
110
110
|
collector.error(( localize(
|
|
111
|
-
|
|
111
|
+
14501,
|
|
112
112
|
"'configuration.{0}.superType' must follow the pattern letterOrDigit[-_letterOrDigit]*",
|
|
113
113
|
extensionPoint
|
|
114
114
|
)));
|
|
@@ -116,7 +116,7 @@ class TokenClassificationExtensionPoints {
|
|
|
116
116
|
}
|
|
117
117
|
if (typeof contribution.description !== 'string' || contribution.id.length === 0) {
|
|
118
118
|
collector.error(( localize(
|
|
119
|
-
|
|
119
|
+
14502,
|
|
120
120
|
"'configuration.{0}.description' must be defined and can not be empty",
|
|
121
121
|
extensionPoint
|
|
122
122
|
)));
|
|
@@ -129,7 +129,7 @@ class TokenClassificationExtensionPoints {
|
|
|
129
129
|
const extensionValue = extension.value;
|
|
130
130
|
const collector = extension.collector;
|
|
131
131
|
if (!extensionValue || !Array.isArray(extensionValue)) {
|
|
132
|
-
collector.error(( localize(
|
|
132
|
+
collector.error(( localize(14503, "'configuration.semanticTokenType' must be an array")));
|
|
133
133
|
return;
|
|
134
134
|
}
|
|
135
135
|
for (const contribution of extensionValue) {
|
|
@@ -150,7 +150,7 @@ class TokenClassificationExtensionPoints {
|
|
|
150
150
|
const extensionValue = extension.value;
|
|
151
151
|
const collector = extension.collector;
|
|
152
152
|
if (!extensionValue || !Array.isArray(extensionValue)) {
|
|
153
|
-
collector.error(( localize(
|
|
153
|
+
collector.error(( localize(14504, "'configuration.semanticTokenModifier' must be an array")));
|
|
154
154
|
return;
|
|
155
155
|
}
|
|
156
156
|
for (const contribution of extensionValue) {
|
|
@@ -171,17 +171,17 @@ class TokenClassificationExtensionPoints {
|
|
|
171
171
|
const extensionValue = extension.value;
|
|
172
172
|
const collector = extension.collector;
|
|
173
173
|
if (!extensionValue || !Array.isArray(extensionValue)) {
|
|
174
|
-
collector.error(( localize(
|
|
174
|
+
collector.error(( localize(14505, "'configuration.semanticTokenScopes' must be an array")));
|
|
175
175
|
return;
|
|
176
176
|
}
|
|
177
177
|
for (const contribution of extensionValue) {
|
|
178
178
|
if (contribution.language && typeof contribution.language !== 'string') {
|
|
179
|
-
collector.error(( localize(
|
|
179
|
+
collector.error(( localize(14506, "'configuration.semanticTokenScopes.language' must be a string")));
|
|
180
180
|
continue;
|
|
181
181
|
}
|
|
182
182
|
if (!contribution.scopes || typeof contribution.scopes !== 'object') {
|
|
183
183
|
collector.error(( localize(
|
|
184
|
-
|
|
184
|
+
14507,
|
|
185
185
|
"'configuration.semanticTokenScopes.scopes' must be defined as an object"
|
|
186
186
|
)));
|
|
187
187
|
continue;
|
|
@@ -190,7 +190,7 @@ class TokenClassificationExtensionPoints {
|
|
|
190
190
|
const tmScopes = contribution.scopes[selectorString];
|
|
191
191
|
if (!Array.isArray(tmScopes) || ( tmScopes.some(l => typeof l !== 'string'))) {
|
|
192
192
|
collector.error(( localize(
|
|
193
|
-
|
|
193
|
+
14508,
|
|
194
194
|
"'configuration.semanticTokenScopes.scopes' values must be an array of strings"
|
|
195
195
|
)));
|
|
196
196
|
continue;
|
|
@@ -201,7 +201,7 @@ class TokenClassificationExtensionPoints {
|
|
|
201
201
|
}
|
|
202
202
|
catch (e) {
|
|
203
203
|
collector.error(( localize(
|
|
204
|
-
|
|
204
|
+
14509,
|
|
205
205
|
"configuration.semanticTokenScopes.scopes': Problems parsing selector {0}.",
|
|
206
206
|
selectorString
|
|
207
207
|
)));
|