@codingame/monaco-vscode-textmate-service-override 25.1.2 → 26.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.js +1 -1
- package/package.json +3 -3
- package/vscode/src/vs/workbench/services/textMate/browser/backgroundTokenization/textMateWorkerTokenizerController.js +42 -26
- package/vscode/src/vs/workbench/services/textMate/browser/backgroundTokenization/threadedBackgroundTokenizerFactory.js +32 -24
- package/vscode/src/vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateTokenizationWorker.worker.js +7 -7
- package/vscode/src/vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateWorkerTokenizer.js +28 -12
- package/vscode/src/vs/workbench/services/textMate/browser/textMateTokenizationFeature.contribution.js +14 -8
- package/vscode/src/vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl.js +118 -83
- package/vscode/src/vs/workbench/services/textMate/browser/tokenizationSupport/textMateTokenizationSupport.js +14 -5
- package/vscode/src/vs/workbench/services/textMate/browser/tokenizationSupport/tokenizationSupportWithLineLimit.js +2 -3
- package/vscode/src/vs/workbench/services/textMate/common/TMGrammarFactory.js +15 -14
- package/vscode/src/vs/workbench/services/themes/common/tokenClassificationExtensionPoint.d.ts +0 -3
- package/vscode/src/vs/workbench/services/themes/common/tokenClassificationExtensionPoint.js +0 -241
package/index.js
CHANGED
|
@@ -5,7 +5,7 @@ export { ITextMateTokenizationService } from '@codingame/monaco-vscode-api/vscod
|
|
|
5
5
|
import { SyncDescriptor } from '@codingame/monaco-vscode-api/vscode/vs/platform/instantiation/common/descriptors';
|
|
6
6
|
import { TextMateTokenizationFeature } from './vscode/src/vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl.js';
|
|
7
7
|
import getServiceOverride$1 from '@codingame/monaco-vscode-files-service-override';
|
|
8
|
-
import '
|
|
8
|
+
import '@codingame/monaco-vscode-api/vscode/vs/workbench/services/themes/common/tokenClassificationExtensionPoint';
|
|
9
9
|
import '@codingame/monaco-vscode-api/vscode/vs/workbench/contrib/codeEditor/browser/inspectEditorTokens/inspectEditorTokens';
|
|
10
10
|
import './vscode/src/vs/workbench/services/textMate/browser/textMateTokenizationFeature.contribution.js';
|
|
11
11
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@codingame/monaco-vscode-textmate-service-override",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "26.0.0",
|
|
4
4
|
"private": false,
|
|
5
5
|
"description": "VSCode public API plugged on the monaco editor - textmate service-override",
|
|
6
6
|
"keywords": [],
|
|
@@ -15,8 +15,8 @@
|
|
|
15
15
|
},
|
|
16
16
|
"type": "module",
|
|
17
17
|
"dependencies": {
|
|
18
|
-
"@codingame/monaco-vscode-api": "
|
|
19
|
-
"@codingame/monaco-vscode-files-service-override": "
|
|
18
|
+
"@codingame/monaco-vscode-api": "26.0.0",
|
|
19
|
+
"@codingame/monaco-vscode-files-service-override": "26.0.0"
|
|
20
20
|
},
|
|
21
21
|
"main": "index.js",
|
|
22
22
|
"module": "index.js",
|
|
@@ -16,8 +16,17 @@ import { keepObserved } from '@codingame/monaco-vscode-api/vscode/vs/base/common
|
|
|
16
16
|
import { autorun } from '@codingame/monaco-vscode-api/vscode/vs/base/common/observableInternal/reactions/autorun';
|
|
17
17
|
|
|
18
18
|
class TextMateWorkerTokenizerController extends Disposable {
|
|
19
|
-
static {
|
|
20
|
-
|
|
19
|
+
static {
|
|
20
|
+
this._id = 0;
|
|
21
|
+
}
|
|
22
|
+
constructor(
|
|
23
|
+
_model,
|
|
24
|
+
_worker,
|
|
25
|
+
_languageIdCodec,
|
|
26
|
+
_backgroundTokenizationStore,
|
|
27
|
+
_configurationService,
|
|
28
|
+
_maxTokenizationLineLength
|
|
29
|
+
) {
|
|
21
30
|
super();
|
|
22
31
|
this._model = _model;
|
|
23
32
|
this._worker = _worker;
|
|
@@ -28,19 +37,23 @@ class TextMateWorkerTokenizerController extends Disposable {
|
|
|
28
37
|
this.controllerId = TextMateWorkerTokenizerController._id++;
|
|
29
38
|
this._pendingChanges = [];
|
|
30
39
|
this._states = ( new TokenizationStateStore());
|
|
31
|
-
this._loggingEnabled = observableConfigValue(
|
|
40
|
+
this._loggingEnabled = observableConfigValue(
|
|
41
|
+
"editor.experimental.asyncTokenizationLogging",
|
|
42
|
+
false,
|
|
43
|
+
this._configurationService
|
|
44
|
+
);
|
|
32
45
|
this._register(keepObserved(this._loggingEnabled));
|
|
33
|
-
this._register(this._model.onDidChangeContent(
|
|
46
|
+
this._register(this._model.onDidChangeContent(e => {
|
|
34
47
|
if (this._shouldLog) {
|
|
35
|
-
console.log(
|
|
36
|
-
fileName: this._model.uri.fsPath.split(
|
|
37
|
-
changes: changesToString(e.changes)
|
|
48
|
+
console.log("model change", {
|
|
49
|
+
fileName: this._model.uri.fsPath.split("\\").pop(),
|
|
50
|
+
changes: changesToString(e.changes)
|
|
38
51
|
});
|
|
39
52
|
}
|
|
40
53
|
this._worker.$acceptModelChanged(this.controllerId, e);
|
|
41
54
|
this._pendingChanges.push(e);
|
|
42
55
|
}));
|
|
43
|
-
this._register(this._model.onDidChangeLanguage(
|
|
56
|
+
this._register(this._model.onDidChangeLanguage(e => {
|
|
44
57
|
const languageId = this._model.getLanguageId();
|
|
45
58
|
const encodedLanguageId = this._languageIdCodec.encodeLanguageId(languageId);
|
|
46
59
|
this._worker.$acceptModelLanguageChanged(this.controllerId, languageId, encodedLanguageId);
|
|
@@ -55,7 +68,7 @@ class TextMateWorkerTokenizerController extends Disposable {
|
|
|
55
68
|
languageId,
|
|
56
69
|
encodedLanguageId,
|
|
57
70
|
maxTokenizationLineLength: this._maxTokenizationLineLength.get(),
|
|
58
|
-
controllerId: this.controllerId
|
|
71
|
+
controllerId: this.controllerId
|
|
59
72
|
});
|
|
60
73
|
this._register(autorun(reader => {
|
|
61
74
|
const maxTokenizationLineLength = this._maxTokenizationLineLength.read(reader);
|
|
@@ -76,27 +89,26 @@ class TextMateWorkerTokenizerController extends Disposable {
|
|
|
76
89
|
let tokens = ContiguousMultilineTokensBuilder.deserialize(( new Uint8Array(rawTokens)));
|
|
77
90
|
const fontTokensUpdate = AnnotationsUpdate.deserialize(fontTokens, deserializeFontTokenOptions());
|
|
78
91
|
if (this._shouldLog) {
|
|
79
|
-
console.log(
|
|
80
|
-
fileName: this._model.uri.fsPath.split(
|
|
81
|
-
updatedTokenLines: ( tokens.map(
|
|
82
|
-
updatedStateLines: ( stateDeltas.map(
|
|
92
|
+
console.log("received background tokenization result", {
|
|
93
|
+
fileName: this._model.uri.fsPath.split("\\").pop(),
|
|
94
|
+
updatedTokenLines: ( tokens.map(t => t.getLineRange())).join(" & "),
|
|
95
|
+
updatedStateLines: ( stateDeltas.map(s => ( ( new LineRange(s.startLineNumber, s.startLineNumber + s.stateDeltas.length)).toString()))).join(" & ")
|
|
83
96
|
});
|
|
84
97
|
}
|
|
85
98
|
if (this._shouldLog) {
|
|
86
|
-
const changes = ( ( this._pendingChanges.filter(c => c.versionId <= versionId).map(c => c.changes)).map(c => changesToString(c))).join(
|
|
87
|
-
console.log(
|
|
99
|
+
const changes = ( ( this._pendingChanges.filter(c => c.versionId <= versionId).map(c => c.changes)).map(c => changesToString(c))).join(" then ");
|
|
100
|
+
console.log("Applying changes to local states", changes);
|
|
88
101
|
}
|
|
89
|
-
while (this._pendingChanges.length > 0 &&
|
|
90
|
-
this._pendingChanges[0].versionId <= versionId) {
|
|
102
|
+
while (this._pendingChanges.length > 0 && this._pendingChanges[0].versionId <= versionId) {
|
|
91
103
|
const change = this._pendingChanges.shift();
|
|
92
104
|
this._states.acceptChanges(change.changes);
|
|
93
105
|
}
|
|
94
106
|
if (this._pendingChanges.length > 0) {
|
|
95
107
|
if (this._shouldLog) {
|
|
96
|
-
const changes = ( ( this._pendingChanges.map(c => c.changes)).map(c => changesToString(c))).join(
|
|
97
|
-
console.log(
|
|
108
|
+
const changes = ( ( this._pendingChanges.map(c => c.changes)).map(c => changesToString(c))).join(" then ");
|
|
109
|
+
console.log("Considering non-processed changes", changes);
|
|
98
110
|
}
|
|
99
|
-
const curToFutureTransformerTokens = MonotonousIndexTransformer.fromMany(( this._pendingChanges.map(
|
|
111
|
+
const curToFutureTransformerTokens = MonotonousIndexTransformer.fromMany(( this._pendingChanges.map(c => linesLengthEditFromModelContentChange(c.changes))));
|
|
100
112
|
const b = ( new ContiguousMultilineTokensBuilder());
|
|
101
113
|
for (const t of tokens) {
|
|
102
114
|
for (let i = t.startLineNumber; i <= t.endLineNumber; i++) {
|
|
@@ -116,9 +128,12 @@ class TextMateWorkerTokenizerController extends Disposable {
|
|
|
116
128
|
}
|
|
117
129
|
fontTokensUpdate.rebase(this._stringEditFromChanges(this._model, this._pendingChanges));
|
|
118
130
|
}
|
|
119
|
-
const curToFutureTransformerStates = MonotonousIndexTransformer.fromMany(( this._pendingChanges.map(
|
|
131
|
+
const curToFutureTransformerStates = MonotonousIndexTransformer.fromMany(( this._pendingChanges.map(c => linesLengthEditFromModelContentChange(c.changes))));
|
|
120
132
|
if (!this._applyStateStackDiffFn || !this._initialState) {
|
|
121
|
-
const {
|
|
133
|
+
const {
|
|
134
|
+
applyStateStackDiff,
|
|
135
|
+
INITIAL
|
|
136
|
+
} = await import('@codingame/monaco-vscode-api/_virtual/main').then(function (n) { return n.main; });
|
|
122
137
|
this._applyStateStackDiffFn = applyStateStackDiff;
|
|
123
138
|
this._initialState = INITIAL;
|
|
124
139
|
}
|
|
@@ -130,8 +145,7 @@ class TextMateWorkerTokenizerController extends Disposable {
|
|
|
130
145
|
if (delta) {
|
|
131
146
|
state = this._applyStateStackDiffFn(prevState, delta);
|
|
132
147
|
this._states.setEndState(d.startLineNumber + i, state);
|
|
133
|
-
}
|
|
134
|
-
else {
|
|
148
|
+
} else {
|
|
135
149
|
state = this._states.getEndState(d.startLineNumber + i);
|
|
136
150
|
}
|
|
137
151
|
const offset = curToFutureTransformerStates.transform(d.startLineNumber + i - 1);
|
|
@@ -160,10 +174,12 @@ class TextMateWorkerTokenizerController extends Disposable {
|
|
|
160
174
|
}
|
|
161
175
|
return StringEdit.compose(edits);
|
|
162
176
|
}
|
|
163
|
-
get _shouldLog() {
|
|
177
|
+
get _shouldLog() {
|
|
178
|
+
return this._loggingEnabled.get();
|
|
179
|
+
}
|
|
164
180
|
}
|
|
165
181
|
function changesToString(changes) {
|
|
166
|
-
return ( changes.map(c => ( Range.lift(c.range).toString()) +
|
|
182
|
+
return ( changes.map(c => ( Range.lift(c.range).toString()) + " => " + c.text)).join(" & ");
|
|
167
183
|
}
|
|
168
184
|
|
|
169
185
|
export { TextMateWorkerTokenizerController };
|
|
@@ -15,9 +15,22 @@ import { FileAccess } from '@codingame/monaco-vscode-api/vscode/vs/base/common/n
|
|
|
15
15
|
|
|
16
16
|
var ThreadedBackgroundTokenizerFactory_1;
|
|
17
17
|
let ThreadedBackgroundTokenizerFactory = class ThreadedBackgroundTokenizerFactory {
|
|
18
|
-
static {
|
|
19
|
-
|
|
20
|
-
|
|
18
|
+
static {
|
|
19
|
+
ThreadedBackgroundTokenizerFactory_1 = this;
|
|
20
|
+
}
|
|
21
|
+
static {
|
|
22
|
+
this._reportedMismatchingTokens = false;
|
|
23
|
+
}
|
|
24
|
+
constructor(
|
|
25
|
+
_reportTokenizationTime,
|
|
26
|
+
_shouldTokenizeAsync,
|
|
27
|
+
_extensionResourceLoaderService,
|
|
28
|
+
_configurationService,
|
|
29
|
+
_languageService,
|
|
30
|
+
_notificationService,
|
|
31
|
+
_telemetryService,
|
|
32
|
+
_webWorkerService
|
|
33
|
+
) {
|
|
21
34
|
this._reportTokenizationTime = _reportTokenizationTime;
|
|
22
35
|
this._shouldTokenizeAsync = _shouldTokenizeAsync;
|
|
23
36
|
this._extensionResourceLoaderService = _extensionResourceLoaderService;
|
|
@@ -42,11 +55,14 @@ let ThreadedBackgroundTokenizerFactory = class ThreadedBackgroundTokenizerFactor
|
|
|
42
55
|
return undefined;
|
|
43
56
|
}
|
|
44
57
|
const store = ( new DisposableStore());
|
|
45
|
-
const controllerContainer = this._getWorkerProxy().then(
|
|
58
|
+
const controllerContainer = this._getWorkerProxy().then(workerProxy => {
|
|
46
59
|
if (store.isDisposed || !workerProxy) {
|
|
47
60
|
return undefined;
|
|
48
61
|
}
|
|
49
|
-
const controllerContainer = {
|
|
62
|
+
const controllerContainer = {
|
|
63
|
+
controller: undefined,
|
|
64
|
+
worker: this._worker
|
|
65
|
+
};
|
|
50
66
|
store.add(keepAliveWhenAttached(textModel, () => {
|
|
51
67
|
const controller = ( new TextMateWorkerTokenizerController(
|
|
52
68
|
textModel,
|
|
@@ -76,17 +92,17 @@ let ThreadedBackgroundTokenizerFactory = class ThreadedBackgroundTokenizerFactor
|
|
|
76
92
|
container.controller.requestTokens(startLineNumber, endLineNumberExclusive);
|
|
77
93
|
}
|
|
78
94
|
},
|
|
79
|
-
reportMismatchingTokens:
|
|
95
|
+
reportMismatchingTokens: lineNumber => {
|
|
80
96
|
if (ThreadedBackgroundTokenizerFactory_1._reportedMismatchingTokens) {
|
|
81
97
|
return;
|
|
82
98
|
}
|
|
83
99
|
ThreadedBackgroundTokenizerFactory_1._reportedMismatchingTokens = true;
|
|
84
100
|
this._notificationService.error({
|
|
85
|
-
message:
|
|
86
|
-
name:
|
|
101
|
+
message: "Async Tokenization Token Mismatch in line " + lineNumber,
|
|
102
|
+
name: "Async Tokenization Token Mismatch"
|
|
87
103
|
});
|
|
88
|
-
this._telemetryService.publicLog2(
|
|
89
|
-
}
|
|
104
|
+
this._telemetryService.publicLog2("asyncTokenizationMismatchingTokens", {});
|
|
105
|
+
}
|
|
90
106
|
};
|
|
91
107
|
}
|
|
92
108
|
setGrammarDefinitions(grammarDefinitions) {
|
|
@@ -109,16 +125,16 @@ let ThreadedBackgroundTokenizerFactory = class ThreadedBackgroundTokenizerFactor
|
|
|
109
125
|
async _createWorkerProxy() {
|
|
110
126
|
const createData = {
|
|
111
127
|
grammarDefinitions: this._grammarDefinitions,
|
|
112
|
-
onigurumaWASMUri: ( new URL('../../../../../../../../external/vscode-oniguruma/release/onig.wasm', import.meta.url)).href
|
|
128
|
+
onigurumaWASMUri: ( new URL('../../../../../../../../external/vscode-oniguruma/release/onig.wasm', import.meta.url)).href
|
|
113
129
|
};
|
|
114
130
|
const worker = this._worker = this._webWorkerService.createWorkerClient(( new WebWorkerDescriptor({
|
|
115
131
|
esmModuleLocation: ( FileAccess.asBrowserUri(
|
|
116
|
-
|
|
132
|
+
"vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateTokenizationWorker.workerMain.js"
|
|
117
133
|
)),
|
|
118
|
-
label:
|
|
134
|
+
label: "TextMateWorker"
|
|
119
135
|
})));
|
|
120
136
|
TextMateWorkerHost.setChannel(worker, {
|
|
121
|
-
$readFile: async
|
|
137
|
+
$readFile: async _resource => {
|
|
122
138
|
const resource = URI.revive(_resource);
|
|
123
139
|
return this._extensionResourceLoaderService.readExtensionResource(resource);
|
|
124
140
|
},
|
|
@@ -155,22 +171,14 @@ let ThreadedBackgroundTokenizerFactory = class ThreadedBackgroundTokenizerFactor
|
|
|
155
171
|
this._workerProxyPromise = null;
|
|
156
172
|
}
|
|
157
173
|
};
|
|
158
|
-
ThreadedBackgroundTokenizerFactory = ThreadedBackgroundTokenizerFactory_1 = ( __decorate([
|
|
159
|
-
( __param(2, IExtensionResourceLoaderService)),
|
|
160
|
-
( __param(3, IConfigurationService)),
|
|
161
|
-
( __param(4, ILanguageService)),
|
|
162
|
-
( __param(5, INotificationService)),
|
|
163
|
-
( __param(6, ITelemetryService)),
|
|
164
|
-
( __param(7, IWebWorkerService))
|
|
165
|
-
], ThreadedBackgroundTokenizerFactory));
|
|
174
|
+
ThreadedBackgroundTokenizerFactory = ThreadedBackgroundTokenizerFactory_1 = ( __decorate([( __param(2, IExtensionResourceLoaderService)), ( __param(3, IConfigurationService)), ( __param(4, ILanguageService)), ( __param(5, INotificationService)), ( __param(6, ITelemetryService)), ( __param(7, IWebWorkerService))], ThreadedBackgroundTokenizerFactory));
|
|
166
175
|
function keepAliveWhenAttached(textModel, factory) {
|
|
167
176
|
const disposableStore = ( new DisposableStore());
|
|
168
177
|
const subStore = disposableStore.add(( new DisposableStore()));
|
|
169
178
|
function checkAttached() {
|
|
170
179
|
if (textModel.isAttachedToEditor()) {
|
|
171
180
|
subStore.add(factory());
|
|
172
|
-
}
|
|
173
|
-
else {
|
|
181
|
+
} else {
|
|
174
182
|
subStore.clear();
|
|
175
183
|
}
|
|
176
184
|
}
|
|
@@ -16,7 +16,7 @@ class TextMateTokenizationWorker {
|
|
|
16
16
|
this._host = TextMateWorkerHost.getChannel(workerServer);
|
|
17
17
|
}
|
|
18
18
|
async $init(_createData) {
|
|
19
|
-
const grammarDefinitions = ( _createData.grammarDefinitions.map(
|
|
19
|
+
const grammarDefinitions = ( _createData.grammarDefinitions.map(def => {
|
|
20
20
|
return {
|
|
21
21
|
location: URI.revive(def.location),
|
|
22
22
|
language: def.language,
|
|
@@ -26,7 +26,7 @@ class TextMateTokenizationWorker {
|
|
|
26
26
|
injectTo: def.injectTo,
|
|
27
27
|
balancedBracketSelectors: def.balancedBracketSelectors,
|
|
28
28
|
unbalancedBracketSelectors: def.unbalancedBracketSelectors,
|
|
29
|
-
sourceExtensionId: def.sourceExtensionId
|
|
29
|
+
sourceExtensionId: def.sourceExtensionId
|
|
30
30
|
};
|
|
31
31
|
}));
|
|
32
32
|
this._grammarFactory = this._loadTMGrammarFactory(grammarDefinitions, _createData.onigurumaWASMUri);
|
|
@@ -38,13 +38,13 @@ class TextMateTokenizationWorker {
|
|
|
38
38
|
const bytes = await response.arrayBuffer();
|
|
39
39
|
await vscodeOniguruma.loadWASM(bytes);
|
|
40
40
|
const onigLib = Promise.resolve({
|
|
41
|
-
createOnigScanner:
|
|
42
|
-
createOnigString:
|
|
41
|
+
createOnigScanner: sources => vscodeOniguruma.createOnigScanner(sources),
|
|
42
|
+
createOnigString: str => vscodeOniguruma.createOnigString(str)
|
|
43
43
|
});
|
|
44
44
|
return ( new TMGrammarFactory({
|
|
45
|
-
logTrace:
|
|
45
|
+
logTrace: msg => {},
|
|
46
46
|
logError: (msg, err) => console.error(msg, err),
|
|
47
|
-
readFile:
|
|
47
|
+
readFile: resource => this._host.$readFile(resource)
|
|
48
48
|
}, grammarDefinitions, vscodeTextmate, onigLib));
|
|
49
49
|
}
|
|
50
50
|
$acceptNewModel(data) {
|
|
@@ -66,7 +66,7 @@ class TextMateTokenizationWorker {
|
|
|
66
66
|
},
|
|
67
67
|
reportTokenizationTime(timeMs, languageId, sourceExtensionId, lineLength, isRandomSample) {
|
|
68
68
|
that._host.$reportTokenizationTime(timeMs, languageId, sourceExtensionId, lineLength, isRandomSample);
|
|
69
|
-
}
|
|
69
|
+
}
|
|
70
70
|
}, data.languageId, data.encodedLanguageId, data.maxTokenizationLineLength)));
|
|
71
71
|
}
|
|
72
72
|
$acceptModelChanged(controllerId, e) {
|
|
@@ -16,7 +16,16 @@ import { OffsetRange } from '@codingame/monaco-vscode-api/vscode/vs/editor/commo
|
|
|
16
16
|
import { observableValue } from '@codingame/monaco-vscode-api/vscode/vs/base/common/observableInternal/observables/observableValue';
|
|
17
17
|
|
|
18
18
|
class TextMateWorkerTokenizer extends MirrorTextModel {
|
|
19
|
-
constructor(
|
|
19
|
+
constructor(
|
|
20
|
+
uri,
|
|
21
|
+
lines,
|
|
22
|
+
eol,
|
|
23
|
+
versionId,
|
|
24
|
+
_host,
|
|
25
|
+
_languageId,
|
|
26
|
+
_encodedLanguageId,
|
|
27
|
+
maxTokenizationLineLength
|
|
28
|
+
) {
|
|
20
29
|
super(uri, lines, eol, versionId);
|
|
21
30
|
this._host = _host;
|
|
22
31
|
this._languageId = _languageId;
|
|
@@ -72,8 +81,7 @@ class TextMateWorkerTokenizer extends MirrorTextModel {
|
|
|
72
81
|
false
|
|
73
82
|
)), Disposable.None, this._maxTokenizationLineLength));
|
|
74
83
|
this._tokenizerWithStateStore = ( new TokenizerWithStateStore(this._lines.length, tokenizationSupport));
|
|
75
|
-
}
|
|
76
|
-
else {
|
|
84
|
+
} else {
|
|
77
85
|
this._tokenizerWithStateStore = null;
|
|
78
86
|
}
|
|
79
87
|
this._tokenize();
|
|
@@ -83,7 +91,9 @@ class TextMateWorkerTokenizer extends MirrorTextModel {
|
|
|
83
91
|
return;
|
|
84
92
|
}
|
|
85
93
|
if (!this._diffStateStacksRefEqFn) {
|
|
86
|
-
const {
|
|
94
|
+
const {
|
|
95
|
+
diffStateStacksRefEq
|
|
96
|
+
} = await import('@codingame/monaco-vscode-api/_virtual/main').then(function (n) { return n.main; });
|
|
87
97
|
this._diffStateStacksRefEqFn = diffStateStacksRefEq;
|
|
88
98
|
}
|
|
89
99
|
const startTime = ( new Date()).getTime();
|
|
@@ -103,8 +113,7 @@ class TextMateWorkerTokenizer extends MirrorTextModel {
|
|
|
103
113
|
if (this._tokenizerWithStateStore.store.setEndState(lineToTokenize.lineNumber, r.endState)) {
|
|
104
114
|
const delta = this._diffStateStacksRefEqFn(lineToTokenize.startState, r.endState);
|
|
105
115
|
stateDeltaBuilder.setState(lineToTokenize.lineNumber, delta);
|
|
106
|
-
}
|
|
107
|
-
else {
|
|
116
|
+
} else {
|
|
108
117
|
stateDeltaBuilder.setState(lineToTokenize.lineNumber, null);
|
|
109
118
|
}
|
|
110
119
|
LineTokens.convertToEndOffset(r.tokens, text.length);
|
|
@@ -121,7 +130,12 @@ class TextMateWorkerTokenizer extends MirrorTextModel {
|
|
|
121
130
|
const fontUpdate = AnnotationsUpdate.create(fontTokensUpdate);
|
|
122
131
|
const serializedFontUpdate = fontUpdate.serialize(serializeFontTokenOptions());
|
|
123
132
|
const stateDeltas = stateDeltaBuilder.getStateDeltas();
|
|
124
|
-
this._host.setTokensAndStates(
|
|
133
|
+
this._host.setTokensAndStates(
|
|
134
|
+
this._versionId,
|
|
135
|
+
tokenBuilder.serialize(),
|
|
136
|
+
serializedFontUpdate,
|
|
137
|
+
stateDeltas
|
|
138
|
+
);
|
|
125
139
|
const deltaMs = ( new Date()).getTime() - startTime;
|
|
126
140
|
if (deltaMs > 20) {
|
|
127
141
|
setTimeout0(() => this._tokenize());
|
|
@@ -148,8 +162,8 @@ class TextMateWorkerTokenizer extends MirrorTextModel {
|
|
|
148
162
|
)),
|
|
149
163
|
annotation: {
|
|
150
164
|
fontFamily: fontInfo.fontFamily ?? undefined,
|
|
151
|
-
|
|
152
|
-
|
|
165
|
+
fontSizeMultiplier: fontInfo.fontSizeMultiplier ?? undefined,
|
|
166
|
+
lineHeightMultiplier: fontInfo.lineHeightMultiplier ?? undefined
|
|
153
167
|
}
|
|
154
168
|
});
|
|
155
169
|
}
|
|
@@ -169,9 +183,11 @@ class StateDeltaBuilder {
|
|
|
169
183
|
setState(lineNumber, stackDiff) {
|
|
170
184
|
if (lineNumber === this._lastStartLineNumber + 1) {
|
|
171
185
|
this._stateDeltas[this._stateDeltas.length - 1].stateDeltas.push(stackDiff);
|
|
172
|
-
}
|
|
173
|
-
|
|
174
|
-
|
|
186
|
+
} else {
|
|
187
|
+
this._stateDeltas.push({
|
|
188
|
+
startLineNumber: lineNumber,
|
|
189
|
+
stateDeltas: [stackDiff]
|
|
190
|
+
});
|
|
175
191
|
}
|
|
176
192
|
this._lastStartLineNumber = lineNumber;
|
|
177
193
|
}
|
|
@@ -10,14 +10,18 @@ import { ITextFileService } from '@codingame/monaco-vscode-api/vscode/vs/workben
|
|
|
10
10
|
import { StopWatch } from '@codingame/monaco-vscode-api/vscode/vs/base/common/stopwatch';
|
|
11
11
|
|
|
12
12
|
let TextMateTokenizationInstantiator = class TextMateTokenizationInstantiator {
|
|
13
|
-
static {
|
|
14
|
-
|
|
13
|
+
static {
|
|
14
|
+
this.ID = "workbench.contrib.textMateTokenizationInstantiator";
|
|
15
|
+
}
|
|
16
|
+
constructor(_textMateTokenizationService) {}
|
|
15
17
|
};
|
|
16
|
-
TextMateTokenizationInstantiator = ( __decorate([
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
18
|
+
TextMateTokenizationInstantiator = ( __decorate([( __param(0, ITextMateTokenizationService))], TextMateTokenizationInstantiator));
|
|
19
|
+
registerWorkbenchContribution2(
|
|
20
|
+
TextMateTokenizationInstantiator.ID,
|
|
21
|
+
TextMateTokenizationInstantiator,
|
|
22
|
+
WorkbenchPhase.BlockRestore
|
|
23
|
+
);
|
|
24
|
+
CommandsRegistry.registerCommand("_workbench.colorizeTextMateTokens", async (accessor, resource) => {
|
|
21
25
|
const textModelService = accessor.get(ITextFileService);
|
|
22
26
|
const textModel = resource ? (await textModelService.files.resolve(resource)).textEditorModel : undefined;
|
|
23
27
|
if (!textModel) {
|
|
@@ -33,5 +37,7 @@ CommandsRegistry.registerCommand('_workbench.colorizeTextMateTokens', async (acc
|
|
|
33
37
|
state = tokenizer.tokenizeEncoded(textModel.getLineContent(i), true, state).endState;
|
|
34
38
|
}
|
|
35
39
|
stopwatch.stop();
|
|
36
|
-
return {
|
|
40
|
+
return {
|
|
41
|
+
tokenizeTime: stopwatch.elapsed()
|
|
42
|
+
};
|
|
37
43
|
});
|
package/vscode/src/vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl.js
CHANGED
|
@@ -34,9 +34,27 @@ import { observableFromEvent } from '@codingame/monaco-vscode-api/vscode/vs/base
|
|
|
34
34
|
|
|
35
35
|
var TextMateTokenizationFeature_1;
|
|
36
36
|
let TextMateTokenizationFeature = class TextMateTokenizationFeature extends Disposable {
|
|
37
|
-
static {
|
|
38
|
-
|
|
39
|
-
|
|
37
|
+
static {
|
|
38
|
+
TextMateTokenizationFeature_1 = this;
|
|
39
|
+
}
|
|
40
|
+
static {
|
|
41
|
+
this.reportTokenizationTimeCounter = {
|
|
42
|
+
sync: 0,
|
|
43
|
+
async: 0
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
constructor(
|
|
47
|
+
_languageService,
|
|
48
|
+
_themeService,
|
|
49
|
+
_extensionResourceLoaderService,
|
|
50
|
+
_notificationService,
|
|
51
|
+
_logService,
|
|
52
|
+
_configurationService,
|
|
53
|
+
_progressService,
|
|
54
|
+
_environmentService,
|
|
55
|
+
_instantiationService,
|
|
56
|
+
_telemetryService
|
|
57
|
+
) {
|
|
40
58
|
super();
|
|
41
59
|
this._languageService = _languageService;
|
|
42
60
|
this._themeService = _themeService;
|
|
@@ -51,31 +69,35 @@ let TextMateTokenizationFeature = class TextMateTokenizationFeature extends Disp
|
|
|
51
69
|
this._createdModes = [];
|
|
52
70
|
this._encounteredLanguages = [];
|
|
53
71
|
this._debugMode = false;
|
|
54
|
-
this._debugModePrintFunc = () => {
|
|
72
|
+
this._debugModePrintFunc = () => {};
|
|
55
73
|
this._grammarDefinitions = null;
|
|
56
74
|
this._grammarFactory = null;
|
|
57
75
|
this._tokenizersRegistrations = this._register(( new DisposableStore()));
|
|
58
76
|
this._currentTheme = null;
|
|
59
77
|
this._currentTokenColorMap = null;
|
|
60
78
|
this._currentTokenFontMap = null;
|
|
61
|
-
this._threadedBackgroundTokenizerFactory = this._instantiationService.createInstance(
|
|
79
|
+
this._threadedBackgroundTokenizerFactory = this._instantiationService.createInstance(
|
|
80
|
+
ThreadedBackgroundTokenizerFactory,
|
|
81
|
+
(timeMs, languageId, sourceExtensionId, lineLength, isRandomSample) => this._reportTokenizationTime(timeMs, languageId, sourceExtensionId, lineLength, true, isRandomSample),
|
|
82
|
+
() => this.getAsyncTokenizationEnabled()
|
|
83
|
+
);
|
|
62
84
|
this._vscodeOniguruma = null;
|
|
63
85
|
this._styleElement = createStyleSheet();
|
|
64
|
-
this._styleElement.className =
|
|
65
|
-
grammarsExtPoint.setHandler(
|
|
86
|
+
this._styleElement.className = "vscode-tokens-styles";
|
|
87
|
+
grammarsExtPoint.setHandler(extensions => this._handleGrammarsExtPoint(extensions));
|
|
66
88
|
this._updateTheme(this._themeService.getColorTheme(), true);
|
|
67
89
|
this._register(this._themeService.onDidColorThemeChange(() => {
|
|
68
90
|
this._updateTheme(this._themeService.getColorTheme(), false);
|
|
69
91
|
}));
|
|
70
|
-
this._register(this._languageService.onDidRequestRichLanguageFeatures(
|
|
92
|
+
this._register(this._languageService.onDidRequestRichLanguageFeatures(languageId => {
|
|
71
93
|
this._createdModes.push(languageId);
|
|
72
94
|
}));
|
|
73
95
|
}
|
|
74
96
|
getAsyncTokenizationEnabled() {
|
|
75
|
-
return !!this._configurationService.getValue(
|
|
97
|
+
return !!this._configurationService.getValue("editor.experimental.asyncTokenization");
|
|
76
98
|
}
|
|
77
99
|
getAsyncTokenizationVerification() {
|
|
78
|
-
return !!this._configurationService.getValue(
|
|
100
|
+
return !!this._configurationService.getValue("editor.experimental.asyncTokenizationVerification");
|
|
79
101
|
}
|
|
80
102
|
_handleGrammarsExtPoint(extensions) {
|
|
81
103
|
this._grammarDefinitions = null;
|
|
@@ -94,7 +116,9 @@ let TextMateTokenizationFeature = class TextMateTokenizationFeature extends Disp
|
|
|
94
116
|
if (validatedGrammar.language) {
|
|
95
117
|
const lazyTokenizationSupport = ( new LazyTokenizationSupport(() => this._createTokenizationSupport(validatedGrammar.language)));
|
|
96
118
|
this._tokenizersRegistrations.add(lazyTokenizationSupport);
|
|
97
|
-
this._tokenizersRegistrations.add(
|
|
119
|
+
this._tokenizersRegistrations.add(
|
|
120
|
+
TokenizationRegistry.registerFactory(validatedGrammar.language, lazyTokenizationSupport)
|
|
121
|
+
);
|
|
98
122
|
}
|
|
99
123
|
}
|
|
100
124
|
}
|
|
@@ -105,7 +129,12 @@ let TextMateTokenizationFeature = class TextMateTokenizationFeature extends Disp
|
|
|
105
129
|
}
|
|
106
130
|
}
|
|
107
131
|
_validateGrammarDefinition(extension, grammar) {
|
|
108
|
-
if (!validateGrammarExtensionPoint(
|
|
132
|
+
if (!validateGrammarExtensionPoint(
|
|
133
|
+
extension.description.extensionLocation,
|
|
134
|
+
grammar,
|
|
135
|
+
extension.collector,
|
|
136
|
+
this._languageService
|
|
137
|
+
)) {
|
|
109
138
|
return null;
|
|
110
139
|
}
|
|
111
140
|
const grammarLocation = joinPath(extension.description.extensionLocation, grammar.path);
|
|
@@ -115,7 +144,7 @@ let TextMateTokenizationFeature = class TextMateTokenizationFeature extends Disp
|
|
|
115
144
|
for (let i = 0, len = scopes.length; i < len; i++) {
|
|
116
145
|
const scope = scopes[i];
|
|
117
146
|
const language = grammar.embeddedLanguages[scope];
|
|
118
|
-
if (typeof language !==
|
|
147
|
+
if (typeof language !== "string") {
|
|
119
148
|
continue;
|
|
120
149
|
}
|
|
121
150
|
if (this._languageService.isRegisteredLanguageId(language)) {
|
|
@@ -129,15 +158,15 @@ let TextMateTokenizationFeature = class TextMateTokenizationFeature extends Disp
|
|
|
129
158
|
for (const scope of scopes) {
|
|
130
159
|
const tokenType = grammar.tokenTypes[scope];
|
|
131
160
|
switch (tokenType) {
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
161
|
+
case "string":
|
|
162
|
+
tokenTypes[scope] = StandardTokenType.String;
|
|
163
|
+
break;
|
|
164
|
+
case "other":
|
|
165
|
+
tokenTypes[scope] = StandardTokenType.Other;
|
|
166
|
+
break;
|
|
167
|
+
case "comment":
|
|
168
|
+
tokenTypes[scope] = StandardTokenType.Comment;
|
|
169
|
+
break;
|
|
141
170
|
}
|
|
142
171
|
}
|
|
143
172
|
}
|
|
@@ -146,7 +175,7 @@ let TextMateTokenizationFeature = class TextMateTokenizationFeature extends Disp
|
|
|
146
175
|
if (!Array.isArray(array)) {
|
|
147
176
|
return defaultValue;
|
|
148
177
|
}
|
|
149
|
-
if (!array.every(e => typeof e ===
|
|
178
|
+
if (!array.every(e => typeof e === "string")) {
|
|
150
179
|
return defaultValue;
|
|
151
180
|
}
|
|
152
181
|
return array;
|
|
@@ -158,14 +187,14 @@ let TextMateTokenizationFeature = class TextMateTokenizationFeature extends Disp
|
|
|
158
187
|
embeddedLanguages: embeddedLanguages,
|
|
159
188
|
tokenTypes: tokenTypes,
|
|
160
189
|
injectTo: grammar.injectTo,
|
|
161
|
-
balancedBracketSelectors: asStringArray(grammar.balancedBracketScopes, [
|
|
190
|
+
balancedBracketSelectors: asStringArray(grammar.balancedBracketScopes, ["*"]),
|
|
162
191
|
unbalancedBracketSelectors: asStringArray(grammar.unbalancedBracketScopes, []),
|
|
163
|
-
sourceExtensionId: extension.description.id
|
|
192
|
+
sourceExtensionId: extension.description.id
|
|
164
193
|
};
|
|
165
194
|
}
|
|
166
195
|
startDebugMode(printFn, onStop) {
|
|
167
196
|
if (this._debugMode) {
|
|
168
|
-
this._notificationService.error(( localize(
|
|
197
|
+
this._notificationService.error(( localize(14669, "Already Logging.")));
|
|
169
198
|
return;
|
|
170
199
|
}
|
|
171
200
|
this._debugModePrintFunc = printFn;
|
|
@@ -173,21 +202,21 @@ let TextMateTokenizationFeature = class TextMateTokenizationFeature extends Disp
|
|
|
173
202
|
if (this._debugMode) {
|
|
174
203
|
this._progressService.withProgress({
|
|
175
204
|
location: ProgressLocation.Notification,
|
|
176
|
-
buttons: [( localize(
|
|
177
|
-
},
|
|
205
|
+
buttons: [( localize(14670, "Stop"))]
|
|
206
|
+
}, progress => {
|
|
178
207
|
progress.report({
|
|
179
|
-
message: ( localize(
|
|
208
|
+
message: ( localize(14671, "Preparing to log TM Grammar parsing. Press Stop when finished."))
|
|
180
209
|
});
|
|
181
|
-
return this._getVSCodeOniguruma().then(
|
|
210
|
+
return this._getVSCodeOniguruma().then(vscodeOniguruma => {
|
|
182
211
|
vscodeOniguruma.setDefaultDebugCall(true);
|
|
183
212
|
progress.report({
|
|
184
|
-
message: ( localize(
|
|
213
|
+
message: ( localize(14672, "Now logging TM Grammar parsing. Press Stop when finished."))
|
|
185
214
|
});
|
|
186
|
-
return ( new Promise((resolve, reject) => {
|
|
215
|
+
return ( new Promise((resolve, reject) => {}));
|
|
187
216
|
});
|
|
188
|
-
},
|
|
189
|
-
this._getVSCodeOniguruma().then(
|
|
190
|
-
this._debugModePrintFunc = () => {
|
|
217
|
+
}, choice => {
|
|
218
|
+
this._getVSCodeOniguruma().then(vscodeOniguruma => {
|
|
219
|
+
this._debugModePrintFunc = () => {};
|
|
191
220
|
this._debugMode = false;
|
|
192
221
|
vscodeOniguruma.setDefaultDebugCall(false);
|
|
193
222
|
onStop();
|
|
@@ -204,16 +233,16 @@ let TextMateTokenizationFeature = class TextMateTokenizationFeature extends Disp
|
|
|
204
233
|
}
|
|
205
234
|
const [vscodeTextmate, vscodeOniguruma] = await Promise.all([import('@codingame/monaco-vscode-api/_virtual/main').then(function (n) { return n.main; }), this._getVSCodeOniguruma()]);
|
|
206
235
|
const onigLib = Promise.resolve({
|
|
207
|
-
createOnigScanner:
|
|
208
|
-
createOnigString:
|
|
236
|
+
createOnigScanner: sources => vscodeOniguruma.createOnigScanner(sources),
|
|
237
|
+
createOnigString: str => vscodeOniguruma.createOnigString(str)
|
|
209
238
|
});
|
|
210
239
|
if (this._grammarFactory) {
|
|
211
240
|
return this._grammarFactory;
|
|
212
241
|
}
|
|
213
242
|
this._grammarFactory = ( new TMGrammarFactory({
|
|
214
|
-
logTrace:
|
|
243
|
+
logTrace: msg => this._logService.trace(msg),
|
|
215
244
|
logError: (msg, err) => this._logService.error(msg, err),
|
|
216
|
-
readFile:
|
|
245
|
+
readFile: resource => this._extensionResourceLoaderService.readExtensionResource(resource)
|
|
217
246
|
}, this._grammarDefinitions || [], vscodeTextmate, onigLib));
|
|
218
247
|
this._updateTheme(this._themeService.getColorTheme(), true);
|
|
219
248
|
return this._grammarFactory;
|
|
@@ -235,7 +264,12 @@ let TextMateTokenizationFeature = class TextMateTokenizationFeature extends Disp
|
|
|
235
264
|
if (!r.grammar) {
|
|
236
265
|
return null;
|
|
237
266
|
}
|
|
238
|
-
const maxTokenizationLineLength = observableConfigValue(
|
|
267
|
+
const maxTokenizationLineLength = observableConfigValue(
|
|
268
|
+
"editor.maxTokenizationLineLength",
|
|
269
|
+
languageId,
|
|
270
|
+
-1,
|
|
271
|
+
this._configurationService
|
|
272
|
+
);
|
|
239
273
|
const store = ( new DisposableStore());
|
|
240
274
|
const tokenization = store.add(( new TextMateTokenizationSupport(
|
|
241
275
|
r.grammar,
|
|
@@ -248,7 +282,7 @@ let TextMateTokenizationFeature = class TextMateTokenizationFeature extends Disp
|
|
|
248
282
|
},
|
|
249
283
|
true
|
|
250
284
|
)));
|
|
251
|
-
store.add(tokenization.onDidEncounterLanguage(
|
|
285
|
+
store.add(tokenization.onDidEncounterLanguage(encodedLanguageId => {
|
|
252
286
|
if (!this._encounteredLanguages[encodedLanguageId]) {
|
|
253
287
|
const languageId = this._languageService.languageIdCodec.decodeLanguageId(encodedLanguageId);
|
|
254
288
|
this._encounteredLanguages[encodedLanguageId] = true;
|
|
@@ -256,8 +290,7 @@ let TextMateTokenizationFeature = class TextMateTokenizationFeature extends Disp
|
|
|
256
290
|
}
|
|
257
291
|
}));
|
|
258
292
|
return ( new TokenizationSupportWithLineLimit(encodedLanguageId, tokenization, store, maxTokenizationLineLength));
|
|
259
|
-
}
|
|
260
|
-
catch (err) {
|
|
293
|
+
} catch (err) {
|
|
261
294
|
if (err.message && err.message === missingTMGrammarErrorMessage) {
|
|
262
295
|
return null;
|
|
263
296
|
}
|
|
@@ -266,11 +299,13 @@ let TextMateTokenizationFeature = class TextMateTokenizationFeature extends Disp
|
|
|
266
299
|
}
|
|
267
300
|
}
|
|
268
301
|
_updateTheme(colorTheme, forceUpdate) {
|
|
269
|
-
if (!forceUpdate && this._currentTheme && this._currentTokenColorMap && equalsTokenRules(this._currentTheme.settings, colorTheme.tokenColors)
|
|
270
|
-
&& equals(this._currentTokenColorMap, colorTheme.tokenColorMap) && this._currentTokenFontMap && equals(this._currentTokenFontMap, colorTheme.tokenFontMap)) {
|
|
302
|
+
if (!forceUpdate && this._currentTheme && this._currentTokenColorMap && equalsTokenRules(this._currentTheme.settings, colorTheme.tokenColors) && equals(this._currentTokenColorMap, colorTheme.tokenColorMap) && this._currentTokenFontMap && equals(this._currentTokenFontMap, colorTheme.tokenFontMap)) {
|
|
271
303
|
return;
|
|
272
304
|
}
|
|
273
|
-
this._currentTheme = {
|
|
305
|
+
this._currentTheme = {
|
|
306
|
+
name: colorTheme.label,
|
|
307
|
+
settings: colorTheme.tokenColors
|
|
308
|
+
};
|
|
274
309
|
this._currentTokenColorMap = colorTheme.tokenColorMap;
|
|
275
310
|
this._currentTokenFontMap = colorTheme.tokenFontMap;
|
|
276
311
|
this._grammarFactory?.setTheme(this._currentTheme, this._currentTokenColorMap);
|
|
@@ -292,7 +327,9 @@ let TextMateTokenizationFeature = class TextMateTokenizationFeature extends Disp
|
|
|
292
327
|
return null;
|
|
293
328
|
}
|
|
294
329
|
const encodedLanguageId = this._languageService.languageIdCodec.encodeLanguageId(languageId);
|
|
295
|
-
const {
|
|
330
|
+
const {
|
|
331
|
+
grammar
|
|
332
|
+
} = await grammarFactory.createGrammar(languageId, encodedLanguageId);
|
|
296
333
|
return grammar;
|
|
297
334
|
}
|
|
298
335
|
_getVSCodeOniguruma() {
|
|
@@ -301,7 +338,7 @@ let TextMateTokenizationFeature = class TextMateTokenizationFeature extends Disp
|
|
|
301
338
|
const [vscodeOniguruma, wasm] = await Promise.all([import('../../../../../../../_virtual/main2.js').then(function (n) { return n.main; }), this._loadVSCodeOnigurumaWASM()]);
|
|
302
339
|
await vscodeOniguruma.loadWASM({
|
|
303
340
|
data: wasm,
|
|
304
|
-
print:
|
|
341
|
+
print: str => {
|
|
305
342
|
this._debugModePrintFunc(str);
|
|
306
343
|
}
|
|
307
344
|
});
|
|
@@ -314,14 +351,20 @@ let TextMateTokenizationFeature = class TextMateTokenizationFeature extends Disp
|
|
|
314
351
|
if (isWeb) {
|
|
315
352
|
const response = await fetch(( new URL('../../../../../../../external/vscode-oniguruma/release/onig.wasm', import.meta.url)).href);
|
|
316
353
|
return await response.arrayBuffer();
|
|
317
|
-
}
|
|
318
|
-
else {
|
|
354
|
+
} else {
|
|
319
355
|
const response = await fetch(( ( FileAccess.asBrowserUri(`${nodeModulesPath}/vscode-oniguruma/release/onig.wasm`)).toString(true)));
|
|
320
356
|
return response;
|
|
321
357
|
}
|
|
322
358
|
}
|
|
323
|
-
_reportTokenizationTime(
|
|
324
|
-
|
|
359
|
+
_reportTokenizationTime(
|
|
360
|
+
timeMs,
|
|
361
|
+
languageId,
|
|
362
|
+
sourceExtensionId,
|
|
363
|
+
lineLength,
|
|
364
|
+
fromWorker,
|
|
365
|
+
isRandomSample
|
|
366
|
+
) {
|
|
367
|
+
const key = fromWorker ? "async" : "sync";
|
|
325
368
|
if (TextMateTokenizationFeature_1.reportTokenizationTimeCounter[key] > 50) {
|
|
326
369
|
return;
|
|
327
370
|
}
|
|
@@ -331,29 +374,18 @@ let TextMateTokenizationFeature = class TextMateTokenizationFeature extends Disp
|
|
|
331
374
|
}, 1000 * 60 * 60);
|
|
332
375
|
}
|
|
333
376
|
TextMateTokenizationFeature_1.reportTokenizationTimeCounter[key]++;
|
|
334
|
-
this._telemetryService.publicLog2(
|
|
377
|
+
this._telemetryService.publicLog2("editor.tokenizedLine", {
|
|
335
378
|
timeMs,
|
|
336
379
|
languageId,
|
|
337
380
|
lineLength,
|
|
338
381
|
fromWorker,
|
|
339
382
|
sourceExtensionId,
|
|
340
383
|
isRandomSample,
|
|
341
|
-
tokenizationSetting: this.getAsyncTokenizationEnabled() ? (this.getAsyncTokenizationVerification() ? 2 : 1) : 0
|
|
384
|
+
tokenizationSetting: this.getAsyncTokenizationEnabled() ? (this.getAsyncTokenizationVerification() ? 2 : 1) : 0
|
|
342
385
|
});
|
|
343
386
|
}
|
|
344
387
|
};
|
|
345
|
-
TextMateTokenizationFeature = TextMateTokenizationFeature_1 = ( __decorate([
|
|
346
|
-
( __param(0, ILanguageService)),
|
|
347
|
-
( __param(1, IWorkbenchThemeService)),
|
|
348
|
-
( __param(2, IExtensionResourceLoaderService)),
|
|
349
|
-
( __param(3, INotificationService)),
|
|
350
|
-
( __param(4, ILogService)),
|
|
351
|
-
( __param(5, IConfigurationService)),
|
|
352
|
-
( __param(6, IProgressService)),
|
|
353
|
-
( __param(7, IWorkbenchEnvironmentService)),
|
|
354
|
-
( __param(8, IInstantiationService)),
|
|
355
|
-
( __param(9, ITelemetryService))
|
|
356
|
-
], TextMateTokenizationFeature));
|
|
388
|
+
TextMateTokenizationFeature = TextMateTokenizationFeature_1 = ( __decorate([( __param(0, ILanguageService)), ( __param(1, IWorkbenchThemeService)), ( __param(2, IExtensionResourceLoaderService)), ( __param(3, INotificationService)), ( __param(4, ILogService)), ( __param(5, IConfigurationService)), ( __param(6, IProgressService)), ( __param(7, IWorkbenchEnvironmentService)), ( __param(8, IInstantiationService)), ( __param(9, ITelemetryService))], TextMateTokenizationFeature));
|
|
357
389
|
function toColorMap(colorMap) {
|
|
358
390
|
const result = [null];
|
|
359
391
|
for (let i = 1, len = colorMap.length; i < len; i++) {
|
|
@@ -374,47 +406,46 @@ function equalsTokenRules(a, b) {
|
|
|
374
406
|
const s1 = r1.settings;
|
|
375
407
|
const s2 = r2.settings;
|
|
376
408
|
if (s1 && s2) {
|
|
377
|
-
if (s1.fontStyle !== s2.fontStyle || s1.foreground !== s2.foreground || s1.background !== s2.background) {
|
|
409
|
+
if (s1.fontStyle !== s2.fontStyle || s1.foreground !== s2.foreground || s1.background !== s2.background || s1.lineHeight !== s2.lineHeight || s1.fontSize !== s2.fontSize || s1.fontFamily !== s2.fontFamily) {
|
|
378
410
|
return false;
|
|
379
411
|
}
|
|
380
|
-
}
|
|
381
|
-
else if (!s1 || !s2) {
|
|
412
|
+
} else if (!s1 || !s2) {
|
|
382
413
|
return false;
|
|
383
414
|
}
|
|
384
415
|
}
|
|
385
416
|
return true;
|
|
386
417
|
}
|
|
387
418
|
function validateGrammarExtensionPoint(extensionLocation, syntax, collector, _languageService) {
|
|
388
|
-
if (syntax.language && ((typeof syntax.language !==
|
|
419
|
+
if (syntax.language && ((typeof syntax.language !== "string") || !_languageService.isRegisteredLanguageId(syntax.language))) {
|
|
389
420
|
collector.error(( localize(
|
|
390
|
-
|
|
421
|
+
14673,
|
|
391
422
|
"Unknown language in `contributes.{0}.language`. Provided value: {1}",
|
|
392
423
|
grammarsExtPoint.name,
|
|
393
424
|
String(syntax.language)
|
|
394
425
|
)));
|
|
395
426
|
return false;
|
|
396
427
|
}
|
|
397
|
-
if (!syntax.scopeName || (typeof syntax.scopeName !==
|
|
428
|
+
if (!syntax.scopeName || (typeof syntax.scopeName !== "string")) {
|
|
398
429
|
collector.error(( localize(
|
|
399
|
-
|
|
430
|
+
14674,
|
|
400
431
|
"Expected string in `contributes.{0}.scopeName`. Provided value: {1}",
|
|
401
432
|
grammarsExtPoint.name,
|
|
402
433
|
String(syntax.scopeName)
|
|
403
434
|
)));
|
|
404
435
|
return false;
|
|
405
436
|
}
|
|
406
|
-
if (!syntax.path || (typeof syntax.path !==
|
|
437
|
+
if (!syntax.path || (typeof syntax.path !== "string")) {
|
|
407
438
|
collector.error(( localize(
|
|
408
|
-
|
|
439
|
+
14675,
|
|
409
440
|
"Expected string in `contributes.{0}.path`. Provided value: {1}",
|
|
410
441
|
grammarsExtPoint.name,
|
|
411
442
|
String(syntax.path)
|
|
412
443
|
)));
|
|
413
444
|
return false;
|
|
414
445
|
}
|
|
415
|
-
if (syntax.injectTo && (!Array.isArray(syntax.injectTo) || ( syntax.injectTo.some(scope => typeof scope !==
|
|
446
|
+
if (syntax.injectTo && (!Array.isArray(syntax.injectTo) || ( syntax.injectTo.some(scope => typeof scope !== "string")))) {
|
|
416
447
|
collector.error(( localize(
|
|
417
|
-
|
|
448
|
+
14676,
|
|
418
449
|
"Invalid value in `contributes.{0}.injectTo`. Must be an array of language scope names. Provided value: {1}",
|
|
419
450
|
grammarsExtPoint.name,
|
|
420
451
|
JSON.stringify(syntax.injectTo)
|
|
@@ -423,7 +454,7 @@ function validateGrammarExtensionPoint(extensionLocation, syntax, collector, _la
|
|
|
423
454
|
}
|
|
424
455
|
if (syntax.embeddedLanguages && !isObject(syntax.embeddedLanguages)) {
|
|
425
456
|
collector.error(( localize(
|
|
426
|
-
|
|
457
|
+
14677,
|
|
427
458
|
"Invalid value in `contributes.{0}.embeddedLanguages`. Must be an object map from scope name to language. Provided value: {1}",
|
|
428
459
|
grammarsExtPoint.name,
|
|
429
460
|
JSON.stringify(syntax.embeddedLanguages)
|
|
@@ -432,7 +463,7 @@ function validateGrammarExtensionPoint(extensionLocation, syntax, collector, _la
|
|
|
432
463
|
}
|
|
433
464
|
if (syntax.tokenTypes && !isObject(syntax.tokenTypes)) {
|
|
434
465
|
collector.error(( localize(
|
|
435
|
-
|
|
466
|
+
14678,
|
|
436
467
|
"Invalid value in `contributes.{0}.tokenTypes`. Must be an object map from scope name to token type. Provided value: {1}",
|
|
437
468
|
grammarsExtPoint.name,
|
|
438
469
|
JSON.stringify(syntax.tokenTypes)
|
|
@@ -442,7 +473,7 @@ function validateGrammarExtensionPoint(extensionLocation, syntax, collector, _la
|
|
|
442
473
|
const grammarLocation = joinPath(extensionLocation, syntax.path);
|
|
443
474
|
if (!isEqualOrParent(grammarLocation, extensionLocation)) {
|
|
444
475
|
collector.warn(( localize(
|
|
445
|
-
|
|
476
|
+
14679,
|
|
446
477
|
"Expected `contributes.{0}.path` ({1}) to be included inside extension's folder ({2}). This might make the extension non-portable.",
|
|
447
478
|
grammarsExtPoint.name,
|
|
448
479
|
grammarLocation.path,
|
|
@@ -452,11 +483,15 @@ function validateGrammarExtensionPoint(extensionLocation, syntax, collector, _la
|
|
|
452
483
|
return true;
|
|
453
484
|
}
|
|
454
485
|
function observableConfigValue(key, languageId, defaultValue, configurationService) {
|
|
455
|
-
return observableFromEvent(
|
|
456
|
-
if (e.affectsConfiguration(key, {
|
|
486
|
+
return observableFromEvent(handleChange => configurationService.onDidChangeConfiguration(e => {
|
|
487
|
+
if (e.affectsConfiguration(key, {
|
|
488
|
+
overrideIdentifier: languageId
|
|
489
|
+
})) {
|
|
457
490
|
handleChange(e);
|
|
458
491
|
}
|
|
459
|
-
}), () => configurationService.getValue(key, {
|
|
492
|
+
}), () => configurationService.getValue(key, {
|
|
493
|
+
overrideIdentifier: languageId
|
|
494
|
+
}) ?? defaultValue);
|
|
460
495
|
}
|
|
461
496
|
|
|
462
497
|
export { TextMateTokenizationFeature };
|
|
@@ -6,8 +6,18 @@ import { TokenMetadata } from '@codingame/monaco-vscode-api/vscode/vs/editor/com
|
|
|
6
6
|
import { EncodedTokenizationResult } from '@codingame/monaco-vscode-api/vscode/vs/editor/common/languages';
|
|
7
7
|
|
|
8
8
|
class TextMateTokenizationSupport extends Disposable {
|
|
9
|
-
get onDidEncounterLanguage() {
|
|
10
|
-
|
|
9
|
+
get onDidEncounterLanguage() {
|
|
10
|
+
return this._onDidEncounterLanguage.event;
|
|
11
|
+
}
|
|
12
|
+
constructor(
|
|
13
|
+
_grammar,
|
|
14
|
+
_initialState,
|
|
15
|
+
_containsEmbeddedLanguages,
|
|
16
|
+
_createBackgroundTokenizer,
|
|
17
|
+
_backgroundTokenizerShouldOnlyVerifyTokens,
|
|
18
|
+
_reportTokenizationTime,
|
|
19
|
+
_reportSlowTokenization
|
|
20
|
+
) {
|
|
11
21
|
super();
|
|
12
22
|
this._grammar = _grammar;
|
|
13
23
|
this._initialState = _initialState;
|
|
@@ -26,7 +36,7 @@ class TextMateTokenizationSupport extends Disposable {
|
|
|
26
36
|
return this._initialState;
|
|
27
37
|
}
|
|
28
38
|
tokenize(line, hasEOL, state) {
|
|
29
|
-
throw ( new Error(
|
|
39
|
+
throw ( new Error("Not supported!"));
|
|
30
40
|
}
|
|
31
41
|
createBackgroundTokenizer(textModel, store) {
|
|
32
42
|
if (this._createBackgroundTokenizer) {
|
|
@@ -64,8 +74,7 @@ class TextMateTokenizationSupport extends Disposable {
|
|
|
64
74
|
let endState;
|
|
65
75
|
if (state.equals(textMateResult.ruleStack)) {
|
|
66
76
|
endState = state;
|
|
67
|
-
}
|
|
68
|
-
else {
|
|
77
|
+
} else {
|
|
69
78
|
endState = textMateResult.ruleStack;
|
|
70
79
|
}
|
|
71
80
|
return ( new EncodedTokenizationResult(textMateResult.tokens, textMateResult.fonts, endState));
|
|
@@ -20,7 +20,7 @@ class TokenizationSupportWithLineLimit extends Disposable {
|
|
|
20
20
|
return this._actual.getInitialState();
|
|
21
21
|
}
|
|
22
22
|
tokenize(line, hasEOL, state) {
|
|
23
|
-
throw ( new Error(
|
|
23
|
+
throw ( new Error("Not supported!"));
|
|
24
24
|
}
|
|
25
25
|
tokenizeEncoded(line, hasEOL, state) {
|
|
26
26
|
if (line.length >= this._maxTokenizationLineLength.get()) {
|
|
@@ -31,8 +31,7 @@ class TokenizationSupportWithLineLimit extends Disposable {
|
|
|
31
31
|
createBackgroundTokenizer(textModel, store) {
|
|
32
32
|
if (this._actual.createBackgroundTokenizer) {
|
|
33
33
|
return this._actual.createBackgroundTokenizer(textModel, store);
|
|
34
|
-
}
|
|
35
|
-
else {
|
|
34
|
+
} else {
|
|
36
35
|
return undefined;
|
|
37
36
|
}
|
|
38
37
|
}
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
import { Disposable } from '@codingame/monaco-vscode-api/vscode/vs/base/common/lifecycle';
|
|
3
3
|
import { TMScopeRegistry } from './TMScopeRegistry.js';
|
|
4
4
|
|
|
5
|
-
const missingTMGrammarErrorMessage =
|
|
5
|
+
const missingTMGrammarErrorMessage = "No TM Grammar registered for this language.";
|
|
6
6
|
class TMGrammarFactory extends Disposable {
|
|
7
7
|
constructor(host, grammarDefinitions, vscodeTextmate, onigLib) {
|
|
8
8
|
super();
|
|
@@ -14,7 +14,7 @@ class TMGrammarFactory extends Disposable {
|
|
|
14
14
|
this._languageToScope = ( new Map());
|
|
15
15
|
this._grammarRegistry = this._register(new vscodeTextmate.Registry({
|
|
16
16
|
onigLib: onigLib,
|
|
17
|
-
loadGrammar: async
|
|
17
|
+
loadGrammar: async scopeName => {
|
|
18
18
|
const grammarDefinition = this._scopeRegistry.getGrammarDefinition(scopeName);
|
|
19
19
|
if (!grammarDefinition) {
|
|
20
20
|
this._host.logTrace(`No grammar found for scope ${scopeName}`);
|
|
@@ -24,17 +24,19 @@ class TMGrammarFactory extends Disposable {
|
|
|
24
24
|
try {
|
|
25
25
|
const content = await this._host.readFile(location);
|
|
26
26
|
return vscodeTextmate.parseRawGrammar(content, location.path);
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
|
|
27
|
+
} catch (e) {
|
|
28
|
+
this._host.logError(
|
|
29
|
+
`Unable to load and parse grammar for scope ${scopeName} from ${location}`,
|
|
30
|
+
e
|
|
31
|
+
);
|
|
30
32
|
return null;
|
|
31
33
|
}
|
|
32
34
|
},
|
|
33
|
-
getInjections:
|
|
34
|
-
const scopeParts = scopeName.split(
|
|
35
|
+
getInjections: scopeName => {
|
|
36
|
+
const scopeParts = scopeName.split(".");
|
|
35
37
|
let injections = [];
|
|
36
38
|
for (let i = 1; i <= scopeParts.length; i++) {
|
|
37
|
-
const subScopeName = scopeParts.slice(0, i).join(
|
|
39
|
+
const subScopeName = scopeParts.slice(0, i).join(".");
|
|
38
40
|
injections = [...injections, ...(this._injections[subScopeName] || [])];
|
|
39
41
|
}
|
|
40
42
|
return injections;
|
|
@@ -76,7 +78,7 @@ class TMGrammarFactory extends Disposable {
|
|
|
76
78
|
}
|
|
77
79
|
async createGrammar(languageId, encodedLanguageId) {
|
|
78
80
|
const scopeName = this._languageToScope.get(languageId);
|
|
79
|
-
if (typeof scopeName !==
|
|
81
|
+
if (typeof scopeName !== "string") {
|
|
80
82
|
throw ( new Error(missingTMGrammarErrorMessage));
|
|
81
83
|
}
|
|
82
84
|
const grammarDefinition = this._scopeRegistry.getGrammarDefinition(scopeName);
|
|
@@ -99,11 +101,10 @@ class TMGrammarFactory extends Disposable {
|
|
|
99
101
|
embeddedLanguages,
|
|
100
102
|
tokenTypes: grammarDefinition.tokenTypes,
|
|
101
103
|
balancedBracketSelectors: grammarDefinition.balancedBracketSelectors,
|
|
102
|
-
unbalancedBracketSelectors: grammarDefinition.unbalancedBracketSelectors
|
|
104
|
+
unbalancedBracketSelectors: grammarDefinition.unbalancedBracketSelectors
|
|
103
105
|
});
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
if (err.message && err.message.startsWith('No grammar provided for')) {
|
|
106
|
+
} catch (err) {
|
|
107
|
+
if (err.message && err.message.startsWith("No grammar provided for")) {
|
|
107
108
|
throw ( new Error(missingTMGrammarErrorMessage));
|
|
108
109
|
}
|
|
109
110
|
throw err;
|
|
@@ -113,7 +114,7 @@ class TMGrammarFactory extends Disposable {
|
|
|
113
114
|
grammar: grammar,
|
|
114
115
|
initialState: this._initialState,
|
|
115
116
|
containsEmbeddedLanguages: containsEmbeddedLanguages,
|
|
116
|
-
sourceExtensionId: grammarDefinition.sourceExtensionId
|
|
117
|
+
sourceExtensionId: grammarDefinition.sourceExtensionId
|
|
117
118
|
};
|
|
118
119
|
}
|
|
119
120
|
}
|
|
@@ -1,241 +0,0 @@
|
|
|
1
|
-
|
|
2
|
-
import { __decorate, __param } from '@codingame/monaco-vscode-api/external/tslib/tslib.es6';
|
|
3
|
-
import { localize } from '@codingame/monaco-vscode-api/vscode/vs/nls';
|
|
4
|
-
import { ExtensionsRegistry } from '@codingame/monaco-vscode-api/vscode/vs/workbench/services/extensions/common/extensionsRegistry';
|
|
5
|
-
import { getTokenClassificationRegistry, typeAndModifierIdPattern } from '@codingame/monaco-vscode-api/vscode/vs/platform/theme/common/tokenClassificationRegistry';
|
|
6
|
-
import { registerWorkbenchContribution2, WorkbenchPhase } from '@codingame/monaco-vscode-api/vscode/vs/workbench/common/contributions';
|
|
7
|
-
import { IInstantiationService } from '@codingame/monaco-vscode-api/vscode/vs/platform/instantiation/common/instantiation';
|
|
8
|
-
|
|
9
|
-
const tokenClassificationRegistry = getTokenClassificationRegistry();
|
|
10
|
-
const tokenTypeExtPoint = ExtensionsRegistry.registerExtensionPoint({
|
|
11
|
-
extensionPoint: 'semanticTokenTypes',
|
|
12
|
-
jsonSchema: {
|
|
13
|
-
description: ( localize(14487, 'Contributes semantic token types.')),
|
|
14
|
-
type: 'array',
|
|
15
|
-
items: {
|
|
16
|
-
type: 'object',
|
|
17
|
-
properties: {
|
|
18
|
-
id: {
|
|
19
|
-
type: 'string',
|
|
20
|
-
description: ( localize(14488, 'The identifier of the semantic token type')),
|
|
21
|
-
pattern: typeAndModifierIdPattern,
|
|
22
|
-
patternErrorMessage: ( localize(14489, 'Identifiers should be in the form letterOrDigit[_-letterOrDigit]*')),
|
|
23
|
-
},
|
|
24
|
-
superType: {
|
|
25
|
-
type: 'string',
|
|
26
|
-
description: ( localize(14490, 'The super type of the semantic token type')),
|
|
27
|
-
pattern: typeAndModifierIdPattern,
|
|
28
|
-
patternErrorMessage: ( localize(14491, 'Super types should be in the form letterOrDigit[_-letterOrDigit]*')),
|
|
29
|
-
},
|
|
30
|
-
description: {
|
|
31
|
-
type: 'string',
|
|
32
|
-
description: ( localize(14492, 'The description of the semantic token type')),
|
|
33
|
-
}
|
|
34
|
-
}
|
|
35
|
-
}
|
|
36
|
-
}
|
|
37
|
-
});
|
|
38
|
-
const tokenModifierExtPoint = ExtensionsRegistry.registerExtensionPoint({
|
|
39
|
-
extensionPoint: 'semanticTokenModifiers',
|
|
40
|
-
jsonSchema: {
|
|
41
|
-
description: ( localize(14493, 'Contributes semantic token modifiers.')),
|
|
42
|
-
type: 'array',
|
|
43
|
-
items: {
|
|
44
|
-
type: 'object',
|
|
45
|
-
properties: {
|
|
46
|
-
id: {
|
|
47
|
-
type: 'string',
|
|
48
|
-
description: ( localize(14494, 'The identifier of the semantic token modifier')),
|
|
49
|
-
pattern: typeAndModifierIdPattern,
|
|
50
|
-
patternErrorMessage: ( localize(14495, 'Identifiers should be in the form letterOrDigit[_-letterOrDigit]*'))
|
|
51
|
-
},
|
|
52
|
-
description: {
|
|
53
|
-
type: 'string',
|
|
54
|
-
description: ( localize(14496, 'The description of the semantic token modifier'))
|
|
55
|
-
}
|
|
56
|
-
}
|
|
57
|
-
}
|
|
58
|
-
}
|
|
59
|
-
});
|
|
60
|
-
const tokenStyleDefaultsExtPoint = ExtensionsRegistry.registerExtensionPoint({
|
|
61
|
-
extensionPoint: 'semanticTokenScopes',
|
|
62
|
-
jsonSchema: {
|
|
63
|
-
description: ( localize(14497, 'Contributes semantic token scope maps.')),
|
|
64
|
-
type: 'array',
|
|
65
|
-
items: {
|
|
66
|
-
type: 'object',
|
|
67
|
-
properties: {
|
|
68
|
-
language: {
|
|
69
|
-
description: ( localize(14498, 'Lists the languge for which the defaults are.')),
|
|
70
|
-
type: 'string'
|
|
71
|
-
},
|
|
72
|
-
scopes: {
|
|
73
|
-
description: ( localize(
|
|
74
|
-
14499,
|
|
75
|
-
'Maps a semantic token (described by semantic token selector) to one or more textMate scopes used to represent that token.'
|
|
76
|
-
)),
|
|
77
|
-
type: 'object',
|
|
78
|
-
additionalProperties: {
|
|
79
|
-
type: 'array',
|
|
80
|
-
items: {
|
|
81
|
-
type: 'string'
|
|
82
|
-
}
|
|
83
|
-
}
|
|
84
|
-
}
|
|
85
|
-
}
|
|
86
|
-
}
|
|
87
|
-
}
|
|
88
|
-
});
|
|
89
|
-
class TokenClassificationExtensionPoints {
|
|
90
|
-
constructor() {
|
|
91
|
-
function validateTypeOrModifier(contribution, extensionPoint, collector) {
|
|
92
|
-
if (typeof contribution.id !== 'string' || contribution.id.length === 0) {
|
|
93
|
-
collector.error(( localize(
|
|
94
|
-
14500,
|
|
95
|
-
"'configuration.{0}.id' must be defined and can not be empty",
|
|
96
|
-
extensionPoint
|
|
97
|
-
)));
|
|
98
|
-
return false;
|
|
99
|
-
}
|
|
100
|
-
if (!contribution.id.match(typeAndModifierIdPattern)) {
|
|
101
|
-
collector.error(( localize(
|
|
102
|
-
14501,
|
|
103
|
-
"'configuration.{0}.id' must follow the pattern letterOrDigit[-_letterOrDigit]*",
|
|
104
|
-
extensionPoint
|
|
105
|
-
)));
|
|
106
|
-
return false;
|
|
107
|
-
}
|
|
108
|
-
const superType = contribution.superType;
|
|
109
|
-
if (superType && !superType.match(typeAndModifierIdPattern)) {
|
|
110
|
-
collector.error(( localize(
|
|
111
|
-
14502,
|
|
112
|
-
"'configuration.{0}.superType' must follow the pattern letterOrDigit[-_letterOrDigit]*",
|
|
113
|
-
extensionPoint
|
|
114
|
-
)));
|
|
115
|
-
return false;
|
|
116
|
-
}
|
|
117
|
-
if (typeof contribution.description !== 'string' || contribution.id.length === 0) {
|
|
118
|
-
collector.error(( localize(
|
|
119
|
-
14503,
|
|
120
|
-
"'configuration.{0}.description' must be defined and can not be empty",
|
|
121
|
-
extensionPoint
|
|
122
|
-
)));
|
|
123
|
-
return false;
|
|
124
|
-
}
|
|
125
|
-
return true;
|
|
126
|
-
}
|
|
127
|
-
tokenTypeExtPoint.setHandler((extensions, delta) => {
|
|
128
|
-
for (const extension of delta.added) {
|
|
129
|
-
const extensionValue = extension.value;
|
|
130
|
-
const collector = extension.collector;
|
|
131
|
-
if (!extensionValue || !Array.isArray(extensionValue)) {
|
|
132
|
-
collector.error(( localize(14504, "'configuration.semanticTokenType' must be an array")));
|
|
133
|
-
return;
|
|
134
|
-
}
|
|
135
|
-
for (const contribution of extensionValue) {
|
|
136
|
-
if (validateTypeOrModifier(contribution, 'semanticTokenType', collector)) {
|
|
137
|
-
tokenClassificationRegistry.registerTokenType(contribution.id, contribution.description, contribution.superType);
|
|
138
|
-
}
|
|
139
|
-
}
|
|
140
|
-
}
|
|
141
|
-
for (const extension of delta.removed) {
|
|
142
|
-
const extensionValue = extension.value;
|
|
143
|
-
for (const contribution of extensionValue) {
|
|
144
|
-
tokenClassificationRegistry.deregisterTokenType(contribution.id);
|
|
145
|
-
}
|
|
146
|
-
}
|
|
147
|
-
});
|
|
148
|
-
tokenModifierExtPoint.setHandler((extensions, delta) => {
|
|
149
|
-
for (const extension of delta.added) {
|
|
150
|
-
const extensionValue = extension.value;
|
|
151
|
-
const collector = extension.collector;
|
|
152
|
-
if (!extensionValue || !Array.isArray(extensionValue)) {
|
|
153
|
-
collector.error(( localize(14505, "'configuration.semanticTokenModifier' must be an array")));
|
|
154
|
-
return;
|
|
155
|
-
}
|
|
156
|
-
for (const contribution of extensionValue) {
|
|
157
|
-
if (validateTypeOrModifier(contribution, 'semanticTokenModifier', collector)) {
|
|
158
|
-
tokenClassificationRegistry.registerTokenModifier(contribution.id, contribution.description);
|
|
159
|
-
}
|
|
160
|
-
}
|
|
161
|
-
}
|
|
162
|
-
for (const extension of delta.removed) {
|
|
163
|
-
const extensionValue = extension.value;
|
|
164
|
-
for (const contribution of extensionValue) {
|
|
165
|
-
tokenClassificationRegistry.deregisterTokenModifier(contribution.id);
|
|
166
|
-
}
|
|
167
|
-
}
|
|
168
|
-
});
|
|
169
|
-
tokenStyleDefaultsExtPoint.setHandler((extensions, delta) => {
|
|
170
|
-
for (const extension of delta.added) {
|
|
171
|
-
const extensionValue = extension.value;
|
|
172
|
-
const collector = extension.collector;
|
|
173
|
-
if (!extensionValue || !Array.isArray(extensionValue)) {
|
|
174
|
-
collector.error(( localize(14506, "'configuration.semanticTokenScopes' must be an array")));
|
|
175
|
-
return;
|
|
176
|
-
}
|
|
177
|
-
for (const contribution of extensionValue) {
|
|
178
|
-
if (contribution.language && typeof contribution.language !== 'string') {
|
|
179
|
-
collector.error(( localize(14507, "'configuration.semanticTokenScopes.language' must be a string")));
|
|
180
|
-
continue;
|
|
181
|
-
}
|
|
182
|
-
if (!contribution.scopes || typeof contribution.scopes !== 'object') {
|
|
183
|
-
collector.error(( localize(
|
|
184
|
-
14508,
|
|
185
|
-
"'configuration.semanticTokenScopes.scopes' must be defined as an object"
|
|
186
|
-
)));
|
|
187
|
-
continue;
|
|
188
|
-
}
|
|
189
|
-
for (const selectorString in contribution.scopes) {
|
|
190
|
-
const tmScopes = contribution.scopes[selectorString];
|
|
191
|
-
if (!Array.isArray(tmScopes) || ( tmScopes.some(l => typeof l !== 'string'))) {
|
|
192
|
-
collector.error(( localize(
|
|
193
|
-
14509,
|
|
194
|
-
"'configuration.semanticTokenScopes.scopes' values must be an array of strings"
|
|
195
|
-
)));
|
|
196
|
-
continue;
|
|
197
|
-
}
|
|
198
|
-
try {
|
|
199
|
-
const selector = tokenClassificationRegistry.parseTokenSelector(selectorString, contribution.language);
|
|
200
|
-
tokenClassificationRegistry.registerTokenStyleDefault(selector, { scopesToProbe: ( tmScopes.map(s => s.split(' '))) });
|
|
201
|
-
}
|
|
202
|
-
catch (e) {
|
|
203
|
-
collector.error(( localize(
|
|
204
|
-
14510,
|
|
205
|
-
"configuration.semanticTokenScopes.scopes': Problems parsing selector {0}.",
|
|
206
|
-
selectorString
|
|
207
|
-
)));
|
|
208
|
-
}
|
|
209
|
-
}
|
|
210
|
-
}
|
|
211
|
-
}
|
|
212
|
-
for (const extension of delta.removed) {
|
|
213
|
-
const extensionValue = extension.value;
|
|
214
|
-
for (const contribution of extensionValue) {
|
|
215
|
-
for (const selectorString in contribution.scopes) {
|
|
216
|
-
const tmScopes = contribution.scopes[selectorString];
|
|
217
|
-
try {
|
|
218
|
-
const selector = tokenClassificationRegistry.parseTokenSelector(selectorString, contribution.language);
|
|
219
|
-
tokenClassificationRegistry.registerTokenStyleDefault(selector, { scopesToProbe: ( tmScopes.map(s => s.split(' '))) });
|
|
220
|
-
}
|
|
221
|
-
catch (e) {
|
|
222
|
-
}
|
|
223
|
-
}
|
|
224
|
-
}
|
|
225
|
-
}
|
|
226
|
-
});
|
|
227
|
-
}
|
|
228
|
-
}
|
|
229
|
-
let TokenClassificationExtensionPointWorkbenchContribution = class TokenClassificationExtensionPointWorkbenchContribution {
|
|
230
|
-
static { this.ID = 'workbench.contrib.tokenClassificationExtensionPoint'; }
|
|
231
|
-
constructor(instantiationService) {
|
|
232
|
-
this.instantiationService = instantiationService;
|
|
233
|
-
this.instantiationService.createInstance(TokenClassificationExtensionPoints);
|
|
234
|
-
}
|
|
235
|
-
};
|
|
236
|
-
TokenClassificationExtensionPointWorkbenchContribution = ( __decorate([
|
|
237
|
-
( __param(0, IInstantiationService))
|
|
238
|
-
], TokenClassificationExtensionPointWorkbenchContribution));
|
|
239
|
-
registerWorkbenchContribution2(TokenClassificationExtensionPointWorkbenchContribution.ID, TokenClassificationExtensionPointWorkbenchContribution, WorkbenchPhase.BlockStartup);
|
|
240
|
-
|
|
241
|
-
export { TokenClassificationExtensionPoints };
|