@codingame/monaco-vscode-textmate-service-override 4.1.0 → 4.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@codingame/monaco-vscode-textmate-service-override",
3
- "version": "4.1.0",
3
+ "version": "4.1.2",
4
4
  "keywords": [],
5
5
  "author": {
6
6
  "name": "CodinGame",
@@ -18,10 +18,10 @@
18
18
  "module": "index.js",
19
19
  "types": "index.d.ts",
20
20
  "dependencies": {
21
- "vscode": "npm:@codingame/monaco-vscode-api@4.1.0",
21
+ "vscode": "npm:@codingame/monaco-vscode-api@4.1.2",
22
22
  "vscode-oniguruma": "1.7.0",
23
23
  "vscode-textmate": "9.0.0",
24
- "@codingame/monaco-vscode-files-service-override": "4.1.0"
24
+ "@codingame/monaco-vscode-files-service-override": "4.1.2"
25
25
  },
26
26
  "exports": {
27
27
  ".": {
package/textmate.js CHANGED
@@ -3,7 +3,7 @@ import { StandaloneServices } from 'vscode/vscode/vs/editor/standalone/browser/s
3
3
  import { ITextMateTokenizationService } from 'vscode/vscode/vs/workbench/services/textMate/browser/textMateTokenizationFeature';
4
4
  export { ITextMateTokenizationService } from 'vscode/vscode/vs/workbench/services/textMate/browser/textMateTokenizationFeature';
5
5
  import { SyncDescriptor } from 'vscode/vscode/vs/platform/instantiation/common/descriptors';
6
- import { TextMateTokenizationFeature } from 'vscode/vscode/vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl';
6
+ import { TextMateTokenizationFeature } from './vscode/src/vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl.js';
7
7
  import { TokenClassificationExtensionPoints } from 'vscode/vscode/vs/workbench/services/themes/common/tokenClassificationExtensionPoint';
8
8
  import { registerWorkbenchContribution2 } from 'vscode/vscode/vs/workbench/common/contributions';
9
9
  import { ILifecycleService } from 'vscode/vscode/vs/workbench/services/lifecycle/common/lifecycle';
@@ -0,0 +1,64 @@
1
+ import { compareBy, numberComparator } from 'vscode/vscode/vs/base/common/arrays';
2
+
3
+ class ArrayEdit {
4
+ constructor(
5
+ edits) {
6
+ this.edits = edits.slice().sort(compareBy(c => c.offset, numberComparator));
7
+ }
8
+ applyToArray(array) {
9
+ for (let i = this.edits.length - 1; i >= 0; i--) {
10
+ const c = this.edits[i];
11
+ array.splice(c.offset, c.length, ...( new Array(c.newLength)));
12
+ }
13
+ }
14
+ }
15
+ class SingleArrayEdit {
16
+ constructor(offset, length, newLength) {
17
+ this.offset = offset;
18
+ this.length = length;
19
+ this.newLength = newLength;
20
+ }
21
+ toString() {
22
+ return `[${this.offset}, +${this.length}) -> +${this.newLength}}`;
23
+ }
24
+ }
25
+ class MonotonousIndexTransformer {
26
+ static fromMany(transformations) {
27
+ const transformers = ( transformations.map(t => ( new MonotonousIndexTransformer(t))));
28
+ return ( new CombinedIndexTransformer(transformers));
29
+ }
30
+ constructor(transformation) {
31
+ this.transformation = transformation;
32
+ this.idx = 0;
33
+ this.offset = 0;
34
+ }
35
+ transform(index) {
36
+ let nextChange = this.transformation.edits[this.idx];
37
+ while (nextChange && nextChange.offset + nextChange.length <= index) {
38
+ this.offset += nextChange.newLength - nextChange.length;
39
+ this.idx++;
40
+ nextChange = this.transformation.edits[this.idx];
41
+ }
42
+ if (nextChange && nextChange.offset <= index) {
43
+ return undefined;
44
+ }
45
+ return index + this.offset;
46
+ }
47
+ }
48
+ class CombinedIndexTransformer {
49
+ constructor(transformers) {
50
+ this.transformers = transformers;
51
+ }
52
+ transform(index) {
53
+ for (const transformer of this.transformers) {
54
+ const result = transformer.transform(index);
55
+ if (result === undefined) {
56
+ return undefined;
57
+ }
58
+ index = result;
59
+ }
60
+ return index;
61
+ }
62
+ }
63
+
64
+ export { ArrayEdit, CombinedIndexTransformer, MonotonousIndexTransformer, SingleArrayEdit };
@@ -0,0 +1,162 @@
1
+ import { Disposable } from 'vscode/vscode/vs/base/common/lifecycle';
2
+ import 'vscode/vscode/vs/base/common/observableInternal/derived';
3
+ import { autorun } from 'vscode/vscode/vs/base/common/observableInternal/autorun';
4
+ import { keepObserved, observableFromEvent } from 'vscode/vscode/vs/base/common/observableInternal/utils';
5
+ import 'vscode/vscode/vs/base/common/cancellation';
6
+ import { countEOL } from 'vscode/vscode/vs/editor/common/core/eolCounter';
7
+ import { LineRange } from 'vscode/vscode/vs/editor/common/core/lineRange';
8
+ import { Range } from 'vscode/vscode/vs/editor/common/core/range';
9
+ import { TokenizationStateStore } from 'vscode/vscode/vs/editor/common/model/textModelTokens';
10
+ import { ContiguousMultilineTokensBuilder } from 'vscode/vscode/vs/editor/common/tokens/contiguousMultilineTokensBuilder';
11
+ import { MonotonousIndexTransformer, ArrayEdit, SingleArrayEdit } from '../arrayOperation.js';
12
+ import { applyStateStackDiff, INITIAL } from 'vscode-textmate';
13
+
14
+ class TextMateWorkerTokenizerController extends Disposable {
15
+ static { this._id = 0; }
16
+ constructor(_model, _worker, _languageIdCodec, _backgroundTokenizationStore, _configurationService, _maxTokenizationLineLength) {
17
+ super();
18
+ this._model = _model;
19
+ this._worker = _worker;
20
+ this._languageIdCodec = _languageIdCodec;
21
+ this._backgroundTokenizationStore = _backgroundTokenizationStore;
22
+ this._configurationService = _configurationService;
23
+ this._maxTokenizationLineLength = _maxTokenizationLineLength;
24
+ this.controllerId = TextMateWorkerTokenizerController._id++;
25
+ this._pendingChanges = [];
26
+ this._states = ( new TokenizationStateStore());
27
+ this._loggingEnabled = observableConfigValue('editor.experimental.asyncTokenizationLogging', false, this._configurationService);
28
+ this._register(keepObserved(this._loggingEnabled));
29
+ this._register(this._model.onDidChangeContent((e) => {
30
+ if (this._shouldLog) {
31
+ console.log('model change', {
32
+ fileName: this._model.uri.fsPath.split('\\').pop(),
33
+ changes: changesToString(e.changes),
34
+ });
35
+ }
36
+ this._worker.acceptModelChanged(this.controllerId, e);
37
+ this._pendingChanges.push(e);
38
+ }));
39
+ this._register(this._model.onDidChangeLanguage((e) => {
40
+ const languageId = this._model.getLanguageId();
41
+ const encodedLanguageId = this._languageIdCodec.encodeLanguageId(languageId);
42
+ this._worker.acceptModelLanguageChanged(this.controllerId, languageId, encodedLanguageId);
43
+ }));
44
+ const languageId = this._model.getLanguageId();
45
+ const encodedLanguageId = this._languageIdCodec.encodeLanguageId(languageId);
46
+ this._worker.acceptNewModel({
47
+ uri: this._model.uri,
48
+ versionId: this._model.getVersionId(),
49
+ lines: this._model.getLinesContent(),
50
+ EOL: this._model.getEOL(),
51
+ languageId,
52
+ encodedLanguageId,
53
+ maxTokenizationLineLength: this._maxTokenizationLineLength.get(),
54
+ controllerId: this.controllerId,
55
+ });
56
+ this._register(autorun(reader => {
57
+ const maxTokenizationLineLength = this._maxTokenizationLineLength.read(reader);
58
+ this._worker.acceptMaxTokenizationLineLength(this.controllerId, maxTokenizationLineLength);
59
+ }));
60
+ }
61
+ dispose() {
62
+ super.dispose();
63
+ this._worker.acceptRemovedModel(this.controllerId);
64
+ }
65
+ requestTokens(startLineNumber, endLineNumberExclusive) {
66
+ this._worker.retokenize(this.controllerId, startLineNumber, endLineNumberExclusive);
67
+ }
68
+ async setTokensAndStates(controllerId, versionId, rawTokens, stateDeltas) {
69
+ if (this.controllerId !== controllerId) {
70
+ return;
71
+ }
72
+ let tokens = ContiguousMultilineTokensBuilder.deserialize(( new Uint8Array(rawTokens)));
73
+ if (this._shouldLog) {
74
+ console.log('received background tokenization result', {
75
+ fileName: this._model.uri.fsPath.split('\\').pop(),
76
+ updatedTokenLines: ( tokens.map((t) => t.getLineRange())).join(' & '),
77
+ updatedStateLines: ( stateDeltas.map((s) => ( ( new LineRange(s.startLineNumber, s.startLineNumber + s.stateDeltas.length)).toString()))).join(' & '),
78
+ });
79
+ }
80
+ if (this._shouldLog) {
81
+ const changes = ( ( this._pendingChanges.filter(c => c.versionId <= versionId).map(c => c.changes)).map(c => changesToString(c))).join(' then ');
82
+ console.log('Applying changes to local states', changes);
83
+ }
84
+ while (this._pendingChanges.length > 0 &&
85
+ this._pendingChanges[0].versionId <= versionId) {
86
+ const change = this._pendingChanges.shift();
87
+ this._states.acceptChanges(change.changes);
88
+ }
89
+ if (this._pendingChanges.length > 0) {
90
+ if (this._shouldLog) {
91
+ const changes = ( ( this._pendingChanges.map(c => c.changes)).map(c => changesToString(c))).join(' then ');
92
+ console.log('Considering non-processed changes', changes);
93
+ }
94
+ const curToFutureTransformerTokens = MonotonousIndexTransformer.fromMany(( this._pendingChanges.map((c) => fullLineArrayEditFromModelContentChange(c.changes))));
95
+ const b = ( new ContiguousMultilineTokensBuilder());
96
+ for (const t of tokens) {
97
+ for (let i = t.startLineNumber; i <= t.endLineNumber; i++) {
98
+ const result = curToFutureTransformerTokens.transform(i - 1);
99
+ if (result !== undefined) {
100
+ b.add(i, t.getLineTokens(i));
101
+ }
102
+ }
103
+ }
104
+ tokens = b.finalize();
105
+ for (const change of this._pendingChanges) {
106
+ for (const innerChanges of change.changes) {
107
+ for (let j = 0; j < tokens.length; j++) {
108
+ tokens[j].applyEdit(innerChanges.range, innerChanges.text);
109
+ }
110
+ }
111
+ }
112
+ }
113
+ const curToFutureTransformerStates = MonotonousIndexTransformer.fromMany(( this._pendingChanges.map((c) => fullLineArrayEditFromModelContentChange(c.changes))));
114
+ if (!this._applyStateStackDiffFn || !this._initialState) {
115
+ this._applyStateStackDiffFn = applyStateStackDiff;
116
+ this._initialState = INITIAL;
117
+ }
118
+ for (const d of stateDeltas) {
119
+ let prevState = d.startLineNumber <= 1 ? this._initialState : this._states.getEndState(d.startLineNumber - 1);
120
+ for (let i = 0; i < d.stateDeltas.length; i++) {
121
+ const delta = d.stateDeltas[i];
122
+ let state;
123
+ if (delta) {
124
+ state = this._applyStateStackDiffFn(prevState, delta);
125
+ this._states.setEndState(d.startLineNumber + i, state);
126
+ }
127
+ else {
128
+ state = this._states.getEndState(d.startLineNumber + i);
129
+ }
130
+ const offset = curToFutureTransformerStates.transform(d.startLineNumber + i - 1);
131
+ if (offset !== undefined) {
132
+ this._backgroundTokenizationStore.setEndState(offset + 1, state);
133
+ }
134
+ if (d.startLineNumber + i >= this._model.getLineCount() - 1) {
135
+ this._backgroundTokenizationStore.backgroundTokenizationFinished();
136
+ }
137
+ prevState = state;
138
+ }
139
+ }
140
+ this._backgroundTokenizationStore.setTokens(tokens);
141
+ }
142
+ get _shouldLog() { return this._loggingEnabled.get(); }
143
+ }
144
+ function fullLineArrayEditFromModelContentChange(c) {
145
+ return ( new ArrayEdit(( c.map((c) => ( new SingleArrayEdit(
146
+ c.range.startLineNumber - 1,
147
+ c.range.endLineNumber - c.range.startLineNumber + 1,
148
+ countEOL(c.text)[0] + 1
149
+ ))))));
150
+ }
151
+ function changesToString(changes) {
152
+ return ( changes.map(c => ( Range.lift(c.range).toString()) + ' => ' + c.text)).join(' & ');
153
+ }
154
+ function observableConfigValue(key, defaultValue, configurationService) {
155
+ return observableFromEvent((handleChange) => configurationService.onDidChangeConfiguration(e => {
156
+ if (e.affectsConfiguration(key)) {
157
+ handleChange(e);
158
+ }
159
+ }), () => configurationService.getValue(key) ?? defaultValue);
160
+ }
161
+
162
+ export { TextMateWorkerTokenizerController };
@@ -0,0 +1,202 @@
1
+ import { __decorate, __param } from '../../../../../../../../external/tslib/tslib.es6.js';
2
+ import { DisposableStore, toDisposable } from 'vscode/vscode/vs/base/common/lifecycle';
3
+ import { FileAccess, nodeModulesPath, nodeModulesAsarPath } from 'vscode/vscode/vs/base/common/network';
4
+ import { isWeb } from 'vscode/vscode/vs/base/common/platform';
5
+ import { URI } from 'vscode/vscode/vs/base/common/uri';
6
+ import { createWebWorker } from 'vscode/vscode/vs/editor/browser/services/webWorker';
7
+ import { ILanguageService } from 'vscode/vscode/vs/editor/common/languages/language';
8
+ import { ILanguageConfigurationService } from 'vscode/vscode/vs/editor/common/languages/languageConfigurationRegistry';
9
+ import { IModelService } from 'vscode/vscode/vs/editor/common/services/model';
10
+ import { IConfigurationService } from 'vscode/vscode/vs/platform/configuration/common/configuration';
11
+ import { IEnvironmentService } from 'vscode/vscode/vs/platform/environment/common/environment';
12
+ import { IExtensionResourceLoaderService } from 'vscode/vscode/vs/platform/extensionResourceLoader/common/extensionResourceLoader';
13
+ import { INotificationService } from 'vscode/vscode/vs/platform/notification/common/notification';
14
+ import { ITelemetryService } from 'vscode/vscode/vs/platform/telemetry/common/telemetry';
15
+ import { TextMateWorkerTokenizerController } from './textMateWorkerTokenizerController.js';
16
+
17
+ var ThreadedBackgroundTokenizerFactory_1;
18
+ let ThreadedBackgroundTokenizerFactory = class ThreadedBackgroundTokenizerFactory {
19
+ static { ThreadedBackgroundTokenizerFactory_1 = this; }
20
+ static { this._reportedMismatchingTokens = false; }
21
+ constructor(_reportTokenizationTime, _shouldTokenizeAsync, _extensionResourceLoaderService, _modelService, _languageConfigurationService, _configurationService, _languageService, _environmentService, _notificationService, _telemetryService) {
22
+ this._reportTokenizationTime = _reportTokenizationTime;
23
+ this._shouldTokenizeAsync = _shouldTokenizeAsync;
24
+ this._extensionResourceLoaderService = _extensionResourceLoaderService;
25
+ this._modelService = _modelService;
26
+ this._languageConfigurationService = _languageConfigurationService;
27
+ this._configurationService = _configurationService;
28
+ this._languageService = _languageService;
29
+ this._environmentService = _environmentService;
30
+ this._notificationService = _notificationService;
31
+ this._telemetryService = _telemetryService;
32
+ this._workerProxyPromise = null;
33
+ this._worker = null;
34
+ this._workerProxy = null;
35
+ this._workerTokenizerControllers = ( new Map());
36
+ this._currentTheme = null;
37
+ this._currentTokenColorMap = null;
38
+ this._grammarDefinitions = [];
39
+ }
40
+ dispose() {
41
+ this._disposeWorker();
42
+ }
43
+ createBackgroundTokenizer(textModel, tokenStore, maxTokenizationLineLength) {
44
+ if (!this._shouldTokenizeAsync() || textModel.isTooLargeForSyncing()) {
45
+ return undefined;
46
+ }
47
+ const store = ( new DisposableStore());
48
+ const controllerContainer = this._getWorkerProxy().then((workerProxy) => {
49
+ if (store.isDisposed || !workerProxy) {
50
+ return undefined;
51
+ }
52
+ const controllerContainer = { controller: undefined, worker: this._worker };
53
+ store.add(keepAliveWhenAttached(textModel, () => {
54
+ const controller = ( new TextMateWorkerTokenizerController(
55
+ textModel,
56
+ workerProxy,
57
+ this._languageService.languageIdCodec,
58
+ tokenStore,
59
+ this._configurationService,
60
+ maxTokenizationLineLength
61
+ ));
62
+ controllerContainer.controller = controller;
63
+ this._workerTokenizerControllers.set(controller.controllerId, controller);
64
+ return toDisposable(() => {
65
+ controllerContainer.controller = undefined;
66
+ this._workerTokenizerControllers.delete(controller.controllerId);
67
+ controller.dispose();
68
+ });
69
+ }));
70
+ return controllerContainer;
71
+ });
72
+ return {
73
+ dispose() {
74
+ store.dispose();
75
+ },
76
+ requestTokens: async (startLineNumber, endLineNumberExclusive) => {
77
+ const container = await controllerContainer;
78
+ if (container?.controller && container.worker === this._worker) {
79
+ container.controller.requestTokens(startLineNumber, endLineNumberExclusive);
80
+ }
81
+ },
82
+ reportMismatchingTokens: (lineNumber) => {
83
+ if (ThreadedBackgroundTokenizerFactory_1._reportedMismatchingTokens) {
84
+ return;
85
+ }
86
+ ThreadedBackgroundTokenizerFactory_1._reportedMismatchingTokens = true;
87
+ this._notificationService.error({
88
+ message: 'Async Tokenization Token Mismatch in line ' + lineNumber,
89
+ name: 'Async Tokenization Token Mismatch',
90
+ });
91
+ this._telemetryService.publicLog2('asyncTokenizationMismatchingTokens', {});
92
+ },
93
+ };
94
+ }
95
+ setGrammarDefinitions(grammarDefinitions) {
96
+ this._grammarDefinitions = grammarDefinitions;
97
+ this._disposeWorker();
98
+ }
99
+ acceptTheme(theme, colorMap) {
100
+ this._currentTheme = theme;
101
+ this._currentTokenColorMap = colorMap;
102
+ if (this._currentTheme && this._currentTokenColorMap && this._workerProxy) {
103
+ this._workerProxy.acceptTheme(this._currentTheme, this._currentTokenColorMap);
104
+ }
105
+ }
106
+ _getWorkerProxy() {
107
+ if (!this._workerProxyPromise) {
108
+ this._workerProxyPromise = this._createWorkerProxy();
109
+ }
110
+ return this._workerProxyPromise;
111
+ }
112
+ async _createWorkerProxy() {
113
+ const textmateModuleLocation = `${nodeModulesPath}/vscode-textmate`;
114
+ const textmateModuleLocationAsar = `${nodeModulesAsarPath}/vscode-textmate`;
115
+ const onigurumaModuleLocation = `${nodeModulesPath}/vscode-oniguruma`;
116
+ const onigurumaModuleLocationAsar = `${nodeModulesAsarPath}/vscode-oniguruma`;
117
+ const useAsar = this._environmentService.isBuilt && !isWeb;
118
+ const textmateLocation = useAsar ? textmateModuleLocationAsar : textmateModuleLocation;
119
+ const onigurumaLocation = useAsar ? onigurumaModuleLocationAsar : onigurumaModuleLocation;
120
+ const textmateMain = `${textmateLocation}/release/main.js`;
121
+ const onigurumaMain = `${onigurumaLocation}/release/main.js`;
122
+ const onigurumaWASM = `${onigurumaLocation}/release/onig.wasm`;
123
+ const uri = ( ( FileAccess.asBrowserUri(textmateMain)).toString(true));
124
+ const createData = {
125
+ grammarDefinitions: this._grammarDefinitions,
126
+ textmateMainUri: uri,
127
+ onigurumaMainUri: ( ( FileAccess.asBrowserUri(onigurumaMain)).toString(true)),
128
+ onigurumaWASMUri: ( ( FileAccess.asBrowserUri(onigurumaWASM)).toString(true)),
129
+ };
130
+ const host = {
131
+ readFile: async (_resource) => {
132
+ const resource = URI.revive(_resource);
133
+ return this._extensionResourceLoaderService.readExtensionResource(resource);
134
+ },
135
+ setTokensAndStates: async (controllerId, versionId, tokens, lineEndStateDeltas) => {
136
+ const controller = this._workerTokenizerControllers.get(controllerId);
137
+ if (controller) {
138
+ controller.setTokensAndStates(controllerId, versionId, tokens, lineEndStateDeltas);
139
+ }
140
+ },
141
+ reportTokenizationTime: (timeMs, languageId, sourceExtensionId, lineLength, isRandomSample) => {
142
+ this._reportTokenizationTime(timeMs, languageId, sourceExtensionId, lineLength, isRandomSample);
143
+ }
144
+ };
145
+ const worker = this._worker = createWebWorker(this._modelService, this._languageConfigurationService, {
146
+ createData,
147
+ label: 'textMateWorker',
148
+ moduleId: 'vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateTokenizationWorker.worker',
149
+ host,
150
+ });
151
+ const proxy = await ( worker.getProxy());
152
+ if (this._worker !== worker) {
153
+ return null;
154
+ }
155
+ this._workerProxy = proxy;
156
+ if (this._currentTheme && this._currentTokenColorMap) {
157
+ this._workerProxy.acceptTheme(this._currentTheme, this._currentTokenColorMap);
158
+ }
159
+ return proxy;
160
+ }
161
+ _disposeWorker() {
162
+ for (const controller of ( this._workerTokenizerControllers.values())) {
163
+ controller.dispose();
164
+ }
165
+ this._workerTokenizerControllers.clear();
166
+ if (this._worker) {
167
+ this._worker.dispose();
168
+ this._worker = null;
169
+ }
170
+ this._workerProxy = null;
171
+ this._workerProxyPromise = null;
172
+ }
173
+ };
174
+ ThreadedBackgroundTokenizerFactory = ThreadedBackgroundTokenizerFactory_1 = ( __decorate([
175
+ ( __param(2, IExtensionResourceLoaderService)),
176
+ ( __param(3, IModelService)),
177
+ ( __param(4, ILanguageConfigurationService)),
178
+ ( __param(5, IConfigurationService)),
179
+ ( __param(6, ILanguageService)),
180
+ ( __param(7, IEnvironmentService)),
181
+ ( __param(8, INotificationService)),
182
+ ( __param(9, ITelemetryService))
183
+ ], ThreadedBackgroundTokenizerFactory));
184
+ function keepAliveWhenAttached(textModel, factory) {
185
+ const disposableStore = ( new DisposableStore());
186
+ const subStore = disposableStore.add(( new DisposableStore()));
187
+ function checkAttached() {
188
+ if (textModel.isAttachedToEditor()) {
189
+ subStore.add(factory());
190
+ }
191
+ else {
192
+ subStore.clear();
193
+ }
194
+ }
195
+ checkAttached();
196
+ disposableStore.add(textModel.onDidChangeAttached(() => {
197
+ checkAttached();
198
+ }));
199
+ return disposableStore;
200
+ }
201
+
202
+ export { ThreadedBackgroundTokenizerFactory };
@@ -1,5 +1,5 @@
1
1
  import { URI } from 'vscode/vscode/vs/base/common/uri';
2
- import { TMGrammarFactory } from 'vscode/vscode/vs/workbench/services/textMate/common/TMGrammarFactory';
2
+ import { TMGrammarFactory } from '../../../common/TMGrammarFactory.js';
3
3
  import { TextMateWorkerTokenizer } from './textMateWorkerTokenizer.js';
4
4
  import * as vscodeTextmate from 'vscode-textmate';
5
5
  import * as vscodeOniguruma from 'vscode-oniguruma';
@@ -10,8 +10,8 @@ import { MirrorTextModel } from 'vscode/vscode/vs/editor/common/model/mirrorText
10
10
  import { TokenizerWithStateStore } from 'vscode/vscode/vs/editor/common/model/textModelTokens';
11
11
  import { ContiguousMultilineTokensBuilder } from 'vscode/vscode/vs/editor/common/tokens/contiguousMultilineTokensBuilder';
12
12
  import { LineTokens } from 'vscode/vscode/vs/editor/common/tokens/lineTokens';
13
- import { TextMateTokenizationSupport } from 'vscode/vscode/vs/workbench/services/textMate/browser/tokenizationSupport/textMateTokenizationSupport';
14
- import { TokenizationSupportWithLineLimit } from 'vscode/vscode/vs/workbench/services/textMate/browser/tokenizationSupport/tokenizationSupportWithLineLimit';
13
+ import { TextMateTokenizationSupport } from '../../tokenizationSupport/textMateTokenizationSupport.js';
14
+ import { TokenizationSupportWithLineLimit } from '../../tokenizationSupport/tokenizationSupportWithLineLimit.js';
15
15
  import { diffStateStacksRefEq } from 'vscode-textmate';
16
16
 
17
17
  class TextMateWorkerTokenizer extends MirrorTextModel {
@@ -0,0 +1,484 @@
1
+ import { __decorate, __param } from '../../../../../../../external/tslib/tslib.es6.js';
2
+ import { createStyleSheet } from 'vscode/vscode/vs/base/browser/dom';
3
+ import { equals } from 'vscode/vscode/vs/base/common/arrays';
4
+ import { Color } from 'vscode/vscode/vs/base/common/color';
5
+ import { onUnexpectedError } from 'vscode/vscode/vs/base/common/errors';
6
+ import { Disposable, DisposableStore } from 'vscode/vscode/vs/base/common/lifecycle';
7
+ import { FileAccess, nodeModulesAsarUnpackedPath, nodeModulesPath } from 'vscode/vscode/vs/base/common/network';
8
+ import 'vscode/vscode/vs/base/common/observableInternal/derived';
9
+ import 'vscode/vscode/vs/base/common/observableInternal/autorun';
10
+ import { observableFromEvent } from 'vscode/vscode/vs/base/common/observableInternal/utils';
11
+ import 'vscode/vscode/vs/base/common/cancellation';
12
+ import { isWeb } from 'vscode/vscode/vs/base/common/platform';
13
+ import { joinPath, isEqualOrParent } from 'vscode/vscode/vs/base/common/resources';
14
+ import { isObject } from 'vscode/vscode/vs/base/common/types';
15
+ import { LazyTokenizationSupport, TokenizationRegistry } from 'vscode/vscode/vs/editor/common/languages';
16
+ import { ILanguageService } from 'vscode/vscode/vs/editor/common/languages/language';
17
+ import { generateTokensCSSForColorMap } from 'vscode/vscode/vs/editor/common/languages/supports/tokenization';
18
+ import { localizeWithPath } from 'vscode/vscode/vs/nls';
19
+ import { IConfigurationService } from 'vscode/vscode/vs/platform/configuration/common/configuration';
20
+ import { IExtensionResourceLoaderService } from 'vscode/vscode/vs/platform/extensionResourceLoader/common/extensionResourceLoader';
21
+ import { IInstantiationService } from 'vscode/vscode/vs/platform/instantiation/common/instantiation';
22
+ import { ILogService } from 'vscode/vscode/vs/platform/log/common/log';
23
+ import { INotificationService } from 'vscode/vscode/vs/platform/notification/common/notification';
24
+ import { IProgressService } from 'vscode/vscode/vs/platform/progress/common/progress';
25
+ import { ITelemetryService } from 'vscode/vscode/vs/platform/telemetry/common/telemetry';
26
+ import { IWorkbenchEnvironmentService } from 'vscode/vscode/vs/workbench/services/environment/common/environmentService';
27
+ import { TextMateTokenizationSupport } from './tokenizationSupport/textMateTokenizationSupport.js';
28
+ import { TokenizationSupportWithLineLimit } from './tokenizationSupport/tokenizationSupportWithLineLimit.js';
29
+ import { ThreadedBackgroundTokenizerFactory } from './backgroundTokenization/threadedBackgroundTokenizerFactory.js';
30
+ import { TMGrammarFactory, missingTMGrammarErrorMessage } from '../common/TMGrammarFactory.js';
31
+ import { grammarsExtPoint } from 'vscode/vscode/vs/workbench/services/textMate/common/TMGrammars';
32
+ import { IWorkbenchThemeService } from 'vscode/vscode/vs/workbench/services/themes/common/workbenchThemeService';
33
+ import * as vscodeTextmate from 'vscode-textmate';
34
+ import * as vscodeOniguruma from 'vscode-oniguruma';
35
+
36
+ var TextMateTokenizationFeature_1;
37
+ let TextMateTokenizationFeature = class TextMateTokenizationFeature extends Disposable {
38
+ static { TextMateTokenizationFeature_1 = this; }
39
+ static { this.reportTokenizationTimeCounter = { sync: 0, async: 0 }; }
40
+ constructor(_languageService, _themeService, _extensionResourceLoaderService, _notificationService, _logService, _configurationService, _progressService, _environmentService, _instantiationService, _telemetryService) {
41
+ super();
42
+ this._languageService = _languageService;
43
+ this._themeService = _themeService;
44
+ this._extensionResourceLoaderService = _extensionResourceLoaderService;
45
+ this._notificationService = _notificationService;
46
+ this._logService = _logService;
47
+ this._configurationService = _configurationService;
48
+ this._progressService = _progressService;
49
+ this._environmentService = _environmentService;
50
+ this._instantiationService = _instantiationService;
51
+ this._telemetryService = _telemetryService;
52
+ this._createdModes = [];
53
+ this._encounteredLanguages = [];
54
+ this._debugMode = false;
55
+ this._debugModePrintFunc = () => { };
56
+ this._grammarDefinitions = null;
57
+ this._grammarFactory = null;
58
+ this._tokenizersRegistrations = ( new DisposableStore());
59
+ this._currentTheme = null;
60
+ this._currentTokenColorMap = null;
61
+ this._threadedBackgroundTokenizerFactory = this._instantiationService.createInstance(ThreadedBackgroundTokenizerFactory, (timeMs, languageId, sourceExtensionId, lineLength, isRandomSample) => this._reportTokenizationTime(timeMs, languageId, sourceExtensionId, lineLength, true, isRandomSample), () => this.getAsyncTokenizationEnabled());
62
+ this._vscodeOniguruma = null;
63
+ this._styleElement = createStyleSheet();
64
+ this._styleElement.className = 'vscode-tokens-styles';
65
+ grammarsExtPoint.setHandler((extensions) => this._handleGrammarsExtPoint(extensions));
66
+ this._updateTheme(this._themeService.getColorTheme(), true);
67
+ this._register(this._themeService.onDidColorThemeChange(() => {
68
+ this._updateTheme(this._themeService.getColorTheme(), false);
69
+ }));
70
+ this._languageService.onDidRequestRichLanguageFeatures((languageId) => {
71
+ this._createdModes.push(languageId);
72
+ });
73
+ }
74
+ getAsyncTokenizationEnabled() {
75
+ return !!this._configurationService.getValue('editor.experimental.asyncTokenization');
76
+ }
77
+ getAsyncTokenizationVerification() {
78
+ return !!this._configurationService.getValue('editor.experimental.asyncTokenizationVerification');
79
+ }
80
+ _handleGrammarsExtPoint(extensions) {
81
+ this._grammarDefinitions = null;
82
+ if (this._grammarFactory) {
83
+ this._grammarFactory.dispose();
84
+ this._grammarFactory = null;
85
+ }
86
+ this._tokenizersRegistrations.clear();
87
+ this._grammarDefinitions = [];
88
+ for (const extension of extensions) {
89
+ const grammars = extension.value;
90
+ for (const grammar of grammars) {
91
+ const validatedGrammar = this._validateGrammarDefinition(extension, grammar);
92
+ if (validatedGrammar) {
93
+ this._grammarDefinitions.push(validatedGrammar);
94
+ if (validatedGrammar.language) {
95
+ const lazyTokenizationSupport = ( new LazyTokenizationSupport(() => this._createTokenizationSupport(validatedGrammar.language)));
96
+ this._tokenizersRegistrations.add(lazyTokenizationSupport);
97
+ this._tokenizersRegistrations.add(TokenizationRegistry.registerFactory(validatedGrammar.language, lazyTokenizationSupport));
98
+ }
99
+ }
100
+ }
101
+ }
102
+ this._threadedBackgroundTokenizerFactory.setGrammarDefinitions(this._grammarDefinitions);
103
+ for (const createdMode of this._createdModes) {
104
+ TokenizationRegistry.getOrCreate(createdMode);
105
+ }
106
+ }
107
+ _validateGrammarDefinition(extension, grammar) {
108
+ if (!validateGrammarExtensionPoint(extension.description.extensionLocation, grammar, extension.collector, this._languageService)) {
109
+ return null;
110
+ }
111
+ const grammarLocation = joinPath(extension.description.extensionLocation, grammar.path);
112
+ const embeddedLanguages = Object.create(null);
113
+ if (grammar.embeddedLanguages) {
114
+ const scopes = ( Object.keys(grammar.embeddedLanguages));
115
+ for (let i = 0, len = scopes.length; i < len; i++) {
116
+ const scope = scopes[i];
117
+ const language = grammar.embeddedLanguages[scope];
118
+ if (typeof language !== 'string') {
119
+ continue;
120
+ }
121
+ if (this._languageService.isRegisteredLanguageId(language)) {
122
+ embeddedLanguages[scope] = this._languageService.languageIdCodec.encodeLanguageId(language);
123
+ }
124
+ }
125
+ }
126
+ const tokenTypes = Object.create(null);
127
+ if (grammar.tokenTypes) {
128
+ const scopes = ( Object.keys(grammar.tokenTypes));
129
+ for (const scope of scopes) {
130
+ const tokenType = grammar.tokenTypes[scope];
131
+ switch (tokenType) {
132
+ case 'string':
133
+ tokenTypes[scope] = 2 ;
134
+ break;
135
+ case 'other':
136
+ tokenTypes[scope] = 0 ;
137
+ break;
138
+ case 'comment':
139
+ tokenTypes[scope] = 1 ;
140
+ break;
141
+ }
142
+ }
143
+ }
144
+ const validLanguageId = grammar.language && this._languageService.isRegisteredLanguageId(grammar.language) ? grammar.language : null;
145
+ function asStringArray(array, defaultValue) {
146
+ if (!Array.isArray(array)) {
147
+ return defaultValue;
148
+ }
149
+ if (!array.every(e => typeof e === 'string')) {
150
+ return defaultValue;
151
+ }
152
+ return array;
153
+ }
154
+ return {
155
+ location: grammarLocation,
156
+ language: validLanguageId || undefined,
157
+ scopeName: grammar.scopeName,
158
+ embeddedLanguages: embeddedLanguages,
159
+ tokenTypes: tokenTypes,
160
+ injectTo: grammar.injectTo,
161
+ balancedBracketSelectors: asStringArray(grammar.balancedBracketScopes, ['*']),
162
+ unbalancedBracketSelectors: asStringArray(grammar.unbalancedBracketScopes, []),
163
+ sourceExtensionId: extension.description.id,
164
+ };
165
+ }
166
+ startDebugMode(printFn, onStop) {
167
+ if (this._debugMode) {
168
+ this._notificationService.error(( localizeWithPath(
169
+ 'vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl',
170
+ 'alreadyDebugging',
171
+ "Already Logging."
172
+ )));
173
+ return;
174
+ }
175
+ this._debugModePrintFunc = printFn;
176
+ this._debugMode = true;
177
+ if (this._debugMode) {
178
+ this._progressService.withProgress({
179
+ location: 15 ,
180
+ buttons: [( localizeWithPath(
181
+ 'vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl',
182
+ 'stop',
183
+ "Stop"
184
+ ))]
185
+ }, (progress) => {
186
+ progress.report({
187
+ message: ( localizeWithPath(
188
+ 'vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl',
189
+ 'progress1',
190
+ "Preparing to log TM Grammar parsing. Press Stop when finished."
191
+ ))
192
+ });
193
+ return this._getVSCodeOniguruma().then((vscodeOniguruma) => {
194
+ vscodeOniguruma.setDefaultDebugCall(true);
195
+ progress.report({
196
+ message: ( localizeWithPath(
197
+ 'vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl',
198
+ 'progress2',
199
+ "Now logging TM Grammar parsing. Press Stop when finished."
200
+ ))
201
+ });
202
+ return ( new Promise((resolve, reject) => { }));
203
+ });
204
+ }, (choice) => {
205
+ this._getVSCodeOniguruma().then((vscodeOniguruma) => {
206
+ this._debugModePrintFunc = () => { };
207
+ this._debugMode = false;
208
+ vscodeOniguruma.setDefaultDebugCall(false);
209
+ onStop();
210
+ });
211
+ });
212
+ }
213
+ }
214
+ _canCreateGrammarFactory() {
215
+ return !!this._grammarDefinitions;
216
+ }
217
+ async _getOrCreateGrammarFactory() {
218
+ if (this._grammarFactory) {
219
+ return this._grammarFactory;
220
+ }
221
+ const vscodeOniguruma = await this._getVSCodeOniguruma();
222
+ const onigLib = Promise.resolve({
223
+ createOnigScanner: (sources) => vscodeOniguruma.createOnigScanner(sources),
224
+ createOnigString: (str) => vscodeOniguruma.createOnigString(str)
225
+ });
226
+ if (this._grammarFactory) {
227
+ return this._grammarFactory;
228
+ }
229
+ this._grammarFactory = ( new TMGrammarFactory({
230
+ logTrace: (msg) => this._logService.trace(msg),
231
+ logError: (msg, err) => this._logService.error(msg, err),
232
+ readFile: (resource) => this._extensionResourceLoaderService.readExtensionResource(resource)
233
+ }, this._grammarDefinitions || [], vscodeTextmate, onigLib));
234
+ this._updateTheme(this._themeService.getColorTheme(), true);
235
+ return this._grammarFactory;
236
+ }
237
+ async _createTokenizationSupport(languageId) {
238
+ if (!this._languageService.isRegisteredLanguageId(languageId)) {
239
+ return null;
240
+ }
241
+ if (!this._canCreateGrammarFactory()) {
242
+ return null;
243
+ }
244
+ try {
245
+ const grammarFactory = await this._getOrCreateGrammarFactory();
246
+ if (!( grammarFactory.has(languageId))) {
247
+ return null;
248
+ }
249
+ const encodedLanguageId = this._languageService.languageIdCodec.encodeLanguageId(languageId);
250
+ const r = await grammarFactory.createGrammar(languageId, encodedLanguageId);
251
+ if (!r.grammar) {
252
+ return null;
253
+ }
254
+ const maxTokenizationLineLength = observableConfigValue('editor.maxTokenizationLineLength', languageId, -1, this._configurationService);
255
+ const tokenization = ( new TextMateTokenizationSupport(
256
+ r.grammar,
257
+ r.initialState,
258
+ r.containsEmbeddedLanguages,
259
+ (textModel, tokenStore) => this._threadedBackgroundTokenizerFactory.createBackgroundTokenizer(textModel, tokenStore, maxTokenizationLineLength),
260
+ () => this.getAsyncTokenizationVerification(),
261
+ (timeMs, lineLength, isRandomSample) => {
262
+ this._reportTokenizationTime(timeMs, languageId, r.sourceExtensionId, lineLength, false, isRandomSample);
263
+ },
264
+ true
265
+ ));
266
+ tokenization.onDidEncounterLanguage((encodedLanguageId) => {
267
+ if (!this._encounteredLanguages[encodedLanguageId]) {
268
+ const languageId = this._languageService.languageIdCodec.decodeLanguageId(encodedLanguageId);
269
+ this._encounteredLanguages[encodedLanguageId] = true;
270
+ this._languageService.requestBasicLanguageFeatures(languageId);
271
+ }
272
+ });
273
+ return ( new TokenizationSupportWithLineLimit(encodedLanguageId, tokenization, maxTokenizationLineLength));
274
+ }
275
+ catch (err) {
276
+ if (err.message && err.message === missingTMGrammarErrorMessage) {
277
+ return null;
278
+ }
279
+ onUnexpectedError(err);
280
+ return null;
281
+ }
282
+ }
283
+ _updateTheme(colorTheme, forceUpdate) {
284
+ if (!forceUpdate && this._currentTheme && this._currentTokenColorMap && equalsTokenRules(this._currentTheme.settings, colorTheme.tokenColors)
285
+ && equals(this._currentTokenColorMap, colorTheme.tokenColorMap)) {
286
+ return;
287
+ }
288
+ this._currentTheme = { name: colorTheme.label, settings: colorTheme.tokenColors };
289
+ this._currentTokenColorMap = colorTheme.tokenColorMap;
290
+ this._grammarFactory?.setTheme(this._currentTheme, this._currentTokenColorMap);
291
+ const colorMap = toColorMap(this._currentTokenColorMap);
292
+ const cssRules = generateTokensCSSForColorMap(colorMap);
293
+ this._styleElement.textContent = cssRules;
294
+ TokenizationRegistry.setColorMap(colorMap);
295
+ if (this._currentTheme && this._currentTokenColorMap) {
296
+ this._threadedBackgroundTokenizerFactory.acceptTheme(this._currentTheme, this._currentTokenColorMap);
297
+ }
298
+ }
299
+ async createTokenizer(languageId) {
300
+ if (!this._languageService.isRegisteredLanguageId(languageId)) {
301
+ return null;
302
+ }
303
+ const grammarFactory = await this._getOrCreateGrammarFactory();
304
+ if (!( grammarFactory.has(languageId))) {
305
+ return null;
306
+ }
307
+ const encodedLanguageId = this._languageService.languageIdCodec.encodeLanguageId(languageId);
308
+ const { grammar } = await grammarFactory.createGrammar(languageId, encodedLanguageId);
309
+ return grammar;
310
+ }
311
+ _getVSCodeOniguruma() {
312
+ if (!this._vscodeOniguruma) {
313
+ this._vscodeOniguruma = (async () => {
314
+ const wasm = await this._loadVSCodeOnigurumaWASM();
315
+ await vscodeOniguruma.loadWASM({
316
+ data: wasm,
317
+ print: (str) => {
318
+ this._debugModePrintFunc(str);
319
+ }
320
+ });
321
+ return vscodeOniguruma;
322
+ })();
323
+ }
324
+ return this._vscodeOniguruma;
325
+ }
326
+ async _loadVSCodeOnigurumaWASM() {
327
+ if (isWeb) {
328
+ const response = await fetch(( ( FileAccess.asBrowserUri('vscode-oniguruma/../onig.wasm')).toString(true)));
329
+ return await response.arrayBuffer();
330
+ }
331
+ else {
332
+ const response = await fetch(this._environmentService.isBuilt
333
+ ? ( ( FileAccess.asBrowserUri(`${nodeModulesAsarUnpackedPath}/vscode-oniguruma/release/onig.wasm`)).toString(true))
334
+ : ( ( FileAccess.asBrowserUri(`${nodeModulesPath}/vscode-oniguruma/release/onig.wasm`)).toString(true)));
335
+ return response;
336
+ }
337
+ }
338
+ _reportTokenizationTime(timeMs, languageId, sourceExtensionId, lineLength, fromWorker, isRandomSample) {
339
+ const key = fromWorker ? 'async' : 'sync';
340
+ if (TextMateTokenizationFeature_1.reportTokenizationTimeCounter[key] > 50) {
341
+ return;
342
+ }
343
+ if (TextMateTokenizationFeature_1.reportTokenizationTimeCounter[key] === 0) {
344
+ setTimeout(() => {
345
+ TextMateTokenizationFeature_1.reportTokenizationTimeCounter[key] = 0;
346
+ }, 1000 * 60 * 60);
347
+ }
348
+ TextMateTokenizationFeature_1.reportTokenizationTimeCounter[key]++;
349
+ this._telemetryService.publicLog2('editor.tokenizedLine', {
350
+ timeMs,
351
+ languageId,
352
+ lineLength,
353
+ fromWorker,
354
+ sourceExtensionId,
355
+ isRandomSample,
356
+ tokenizationSetting: this.getAsyncTokenizationEnabled() ? (this.getAsyncTokenizationVerification() ? 2 : 1) : 0,
357
+ });
358
+ }
359
+ };
360
+ TextMateTokenizationFeature = TextMateTokenizationFeature_1 = ( __decorate([
361
+ ( __param(0, ILanguageService)),
362
+ ( __param(1, IWorkbenchThemeService)),
363
+ ( __param(2, IExtensionResourceLoaderService)),
364
+ ( __param(3, INotificationService)),
365
+ ( __param(4, ILogService)),
366
+ ( __param(5, IConfigurationService)),
367
+ ( __param(6, IProgressService)),
368
+ ( __param(7, IWorkbenchEnvironmentService)),
369
+ ( __param(8, IInstantiationService)),
370
+ ( __param(9, ITelemetryService))
371
+ ], TextMateTokenizationFeature));
372
+ function toColorMap(colorMap) {
373
+ const result = [null];
374
+ for (let i = 1, len = colorMap.length; i < len; i++) {
375
+ result[i] = ( Color.fromHex(colorMap[i]));
376
+ }
377
+ return result;
378
+ }
379
+ function equalsTokenRules(a, b) {
380
+ if (!b || !a || b.length !== a.length) {
381
+ return false;
382
+ }
383
+ for (let i = b.length - 1; i >= 0; i--) {
384
+ const r1 = b[i];
385
+ const r2 = a[i];
386
+ if (r1.scope !== r2.scope) {
387
+ return false;
388
+ }
389
+ const s1 = r1.settings;
390
+ const s2 = r2.settings;
391
+ if (s1 && s2) {
392
+ if (s1.fontStyle !== s2.fontStyle || s1.foreground !== s2.foreground || s1.background !== s2.background) {
393
+ return false;
394
+ }
395
+ }
396
+ else if (!s1 || !s2) {
397
+ return false;
398
+ }
399
+ }
400
+ return true;
401
+ }
402
+ function validateGrammarExtensionPoint(extensionLocation, syntax, collector, _languageService) {
403
+ if (syntax.language && ((typeof syntax.language !== 'string') || !_languageService.isRegisteredLanguageId(syntax.language))) {
404
+ collector.error(( localizeWithPath(
405
+ 'vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl',
406
+ 'invalid.language',
407
+ "Unknown language in `contributes.{0}.language`. Provided value: {1}",
408
+ grammarsExtPoint.name,
409
+ String(syntax.language)
410
+ )));
411
+ return false;
412
+ }
413
+ if (!syntax.scopeName || (typeof syntax.scopeName !== 'string')) {
414
+ collector.error(( localizeWithPath(
415
+ 'vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl',
416
+ 'invalid.scopeName',
417
+ "Expected string in `contributes.{0}.scopeName`. Provided value: {1}",
418
+ grammarsExtPoint.name,
419
+ String(syntax.scopeName)
420
+ )));
421
+ return false;
422
+ }
423
+ if (!syntax.path || (typeof syntax.path !== 'string')) {
424
+ collector.error(( localizeWithPath(
425
+ 'vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl',
426
+ 'invalid.path.0',
427
+ "Expected string in `contributes.{0}.path`. Provided value: {1}",
428
+ grammarsExtPoint.name,
429
+ String(syntax.path)
430
+ )));
431
+ return false;
432
+ }
433
+ if (syntax.injectTo && (!Array.isArray(syntax.injectTo) || ( syntax.injectTo.some(scope => typeof scope !== 'string')))) {
434
+ collector.error(( localizeWithPath(
435
+ 'vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl',
436
+ 'invalid.injectTo',
437
+ "Invalid value in `contributes.{0}.injectTo`. Must be an array of language scope names. Provided value: {1}",
438
+ grammarsExtPoint.name,
439
+ JSON.stringify(syntax.injectTo)
440
+ )));
441
+ return false;
442
+ }
443
+ if (syntax.embeddedLanguages && !isObject(syntax.embeddedLanguages)) {
444
+ collector.error(( localizeWithPath(
445
+ 'vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl',
446
+ 'invalid.embeddedLanguages',
447
+ "Invalid value in `contributes.{0}.embeddedLanguages`. Must be an object map from scope name to language. Provided value: {1}",
448
+ grammarsExtPoint.name,
449
+ JSON.stringify(syntax.embeddedLanguages)
450
+ )));
451
+ return false;
452
+ }
453
+ if (syntax.tokenTypes && !isObject(syntax.tokenTypes)) {
454
+ collector.error(( localizeWithPath(
455
+ 'vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl',
456
+ 'invalid.tokenTypes',
457
+ "Invalid value in `contributes.{0}.tokenTypes`. Must be an object map from scope name to token type. Provided value: {1}",
458
+ grammarsExtPoint.name,
459
+ JSON.stringify(syntax.tokenTypes)
460
+ )));
461
+ return false;
462
+ }
463
+ const grammarLocation = joinPath(extensionLocation, syntax.path);
464
+ if (!isEqualOrParent(grammarLocation, extensionLocation)) {
465
+ collector.warn(( localizeWithPath(
466
+ 'vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl',
467
+ 'invalid.path.1',
468
+ "Expected `contributes.{0}.path` ({1}) to be included inside extension's folder ({2}). This might make the extension non-portable.",
469
+ grammarsExtPoint.name,
470
+ grammarLocation.path,
471
+ extensionLocation.path
472
+ )));
473
+ }
474
+ return true;
475
+ }
476
+ function observableConfigValue(key, languageId, defaultValue, configurationService) {
477
+ return observableFromEvent((handleChange) => configurationService.onDidChangeConfiguration(e => {
478
+ if (e.affectsConfiguration(key, { overrideIdentifier: languageId })) {
479
+ handleChange(e);
480
+ }
481
+ }), () => configurationService.getValue(key, { overrideIdentifier: languageId }) ?? defaultValue);
482
+ }
483
+
484
+ export { TextMateTokenizationFeature };
@@ -0,0 +1,74 @@
1
+ import { Emitter } from 'vscode/vscode/vs/base/common/event';
2
+ import { Disposable } from 'vscode/vscode/vs/base/common/lifecycle';
3
+ import { StopWatch } from 'vscode/vscode/vs/base/common/stopwatch';
4
+ import { TokenMetadata } from 'vscode/vscode/vs/editor/common/encodedTokenAttributes';
5
+ import { EncodedTokenizationResult } from 'vscode/vscode/vs/editor/common/languages';
6
+
7
+ class TextMateTokenizationSupport extends Disposable {
8
+ constructor(_grammar, _initialState, _containsEmbeddedLanguages, _createBackgroundTokenizer, _backgroundTokenizerShouldOnlyVerifyTokens, _reportTokenizationTime, _reportSlowTokenization) {
9
+ super();
10
+ this._grammar = _grammar;
11
+ this._initialState = _initialState;
12
+ this._containsEmbeddedLanguages = _containsEmbeddedLanguages;
13
+ this._createBackgroundTokenizer = _createBackgroundTokenizer;
14
+ this._backgroundTokenizerShouldOnlyVerifyTokens = _backgroundTokenizerShouldOnlyVerifyTokens;
15
+ this._reportTokenizationTime = _reportTokenizationTime;
16
+ this._reportSlowTokenization = _reportSlowTokenization;
17
+ this._seenLanguages = [];
18
+ this._onDidEncounterLanguage = this._register(( new Emitter()));
19
+ this.onDidEncounterLanguage = this._onDidEncounterLanguage.event;
20
+ }
21
+ get backgroundTokenizerShouldOnlyVerifyTokens() {
22
+ return this._backgroundTokenizerShouldOnlyVerifyTokens();
23
+ }
24
+ getInitialState() {
25
+ return this._initialState;
26
+ }
27
+ tokenize(line, hasEOL, state) {
28
+ throw new Error('Not supported!');
29
+ }
30
+ createBackgroundTokenizer(textModel, store) {
31
+ if (this._createBackgroundTokenizer) {
32
+ return this._createBackgroundTokenizer(textModel, store);
33
+ }
34
+ return undefined;
35
+ }
36
+ tokenizeEncoded(line, hasEOL, state) {
37
+ const isRandomSample = Math.random() * 10_000 < 1;
38
+ const shouldMeasure = this._reportSlowTokenization || isRandomSample;
39
+ const sw = shouldMeasure ? ( new StopWatch(true)) : undefined;
40
+ const textMateResult = this._grammar.tokenizeLine2(line, state, 500);
41
+ if (shouldMeasure) {
42
+ const timeMS = sw.elapsed();
43
+ if (isRandomSample || timeMS > 32) {
44
+ this._reportTokenizationTime(timeMS, line.length, isRandomSample);
45
+ }
46
+ }
47
+ if (textMateResult.stoppedEarly) {
48
+ console.warn(`Time limit reached when tokenizing line: ${line.substring(0, 100)}`);
49
+ return ( new EncodedTokenizationResult(textMateResult.tokens, state));
50
+ }
51
+ if (this._containsEmbeddedLanguages) {
52
+ const seenLanguages = this._seenLanguages;
53
+ const tokens = textMateResult.tokens;
54
+ for (let i = 0, len = (tokens.length >>> 1); i < len; i++) {
55
+ const metadata = tokens[(i << 1) + 1];
56
+ const languageId = TokenMetadata.getLanguageId(metadata);
57
+ if (!seenLanguages[languageId]) {
58
+ seenLanguages[languageId] = true;
59
+ this._onDidEncounterLanguage.fire(languageId);
60
+ }
61
+ }
62
+ }
63
+ let endState;
64
+ if (state.equals(textMateResult.ruleStack)) {
65
+ endState = state;
66
+ }
67
+ else {
68
+ endState = textMateResult.ruleStack;
69
+ }
70
+ return ( new EncodedTokenizationResult(textMateResult.tokens, endState));
71
+ }
72
+ }
73
+
74
+ export { TextMateTokenizationSupport };
@@ -0,0 +1,41 @@
1
+ import { nullTokenizeEncoded } from 'vscode/vscode/vs/editor/common/languages/nullTokenize';
2
+ import { Disposable } from 'vscode/vscode/vs/base/common/lifecycle';
3
+ import 'vscode/vscode/vs/base/common/observableInternal/derived';
4
+ import 'vscode/vscode/vs/base/common/observableInternal/autorun';
5
+ import { keepObserved } from 'vscode/vscode/vs/base/common/observableInternal/utils';
6
+ import 'vscode/vscode/vs/base/common/cancellation';
7
+
8
+ class TokenizationSupportWithLineLimit extends Disposable {
9
+ get backgroundTokenizerShouldOnlyVerifyTokens() {
10
+ return this._actual.backgroundTokenizerShouldOnlyVerifyTokens;
11
+ }
12
+ constructor(_encodedLanguageId, _actual, _maxTokenizationLineLength) {
13
+ super();
14
+ this._encodedLanguageId = _encodedLanguageId;
15
+ this._actual = _actual;
16
+ this._maxTokenizationLineLength = _maxTokenizationLineLength;
17
+ this._register(keepObserved(this._maxTokenizationLineLength));
18
+ }
19
+ getInitialState() {
20
+ return this._actual.getInitialState();
21
+ }
22
+ tokenize(line, hasEOL, state) {
23
+ throw new Error('Not supported!');
24
+ }
25
+ tokenizeEncoded(line, hasEOL, state) {
26
+ if (line.length >= this._maxTokenizationLineLength.get()) {
27
+ return nullTokenizeEncoded(this._encodedLanguageId, state);
28
+ }
29
+ return this._actual.tokenizeEncoded(line, hasEOL, state);
30
+ }
31
+ createBackgroundTokenizer(textModel, store) {
32
+ if (this._actual.createBackgroundTokenizer) {
33
+ return this._actual.createBackgroundTokenizer(textModel, store);
34
+ }
35
+ else {
36
+ return undefined;
37
+ }
38
+ }
39
+ }
40
+
41
+ export { TokenizationSupportWithLineLimit };
@@ -0,0 +1,120 @@
1
+ import { Disposable } from 'vscode/vscode/vs/base/common/lifecycle';
2
+ import { TMScopeRegistry } from './TMScopeRegistry.js';
3
+
4
+ const missingTMGrammarErrorMessage = 'No TM Grammar registered for this language.';
5
+ class TMGrammarFactory extends Disposable {
6
+ constructor(host, grammarDefinitions, vscodeTextmate, onigLib) {
7
+ super();
8
+ this._host = host;
9
+ this._initialState = vscodeTextmate.INITIAL;
10
+ this._scopeRegistry = ( new TMScopeRegistry());
11
+ this._injections = {};
12
+ this._injectedEmbeddedLanguages = {};
13
+ this._languageToScope = ( new Map());
14
+ this._grammarRegistry = this._register(new vscodeTextmate.Registry({
15
+ onigLib: onigLib,
16
+ loadGrammar: async (scopeName) => {
17
+ const grammarDefinition = this._scopeRegistry.getGrammarDefinition(scopeName);
18
+ if (!grammarDefinition) {
19
+ this._host.logTrace(`No grammar found for scope ${scopeName}`);
20
+ return null;
21
+ }
22
+ const location = grammarDefinition.location;
23
+ try {
24
+ const content = await this._host.readFile(location);
25
+ return vscodeTextmate.parseRawGrammar(content, location.path);
26
+ }
27
+ catch (e) {
28
+ this._host.logError(`Unable to load and parse grammar for scope ${scopeName} from ${location}`, e);
29
+ return null;
30
+ }
31
+ },
32
+ getInjections: (scopeName) => {
33
+ const scopeParts = scopeName.split('.');
34
+ let injections = [];
35
+ for (let i = 1; i <= scopeParts.length; i++) {
36
+ const subScopeName = scopeParts.slice(0, i).join('.');
37
+ injections = [...injections, ...(this._injections[subScopeName] || [])];
38
+ }
39
+ return injections;
40
+ }
41
+ }));
42
+ for (const validGrammar of grammarDefinitions) {
43
+ this._scopeRegistry.register(validGrammar);
44
+ if (validGrammar.injectTo) {
45
+ for (const injectScope of validGrammar.injectTo) {
46
+ let injections = this._injections[injectScope];
47
+ if (!injections) {
48
+ this._injections[injectScope] = injections = [];
49
+ }
50
+ injections.push(validGrammar.scopeName);
51
+ }
52
+ if (validGrammar.embeddedLanguages) {
53
+ for (const injectScope of validGrammar.injectTo) {
54
+ let injectedEmbeddedLanguages = this._injectedEmbeddedLanguages[injectScope];
55
+ if (!injectedEmbeddedLanguages) {
56
+ this._injectedEmbeddedLanguages[injectScope] = injectedEmbeddedLanguages = [];
57
+ }
58
+ injectedEmbeddedLanguages.push(validGrammar.embeddedLanguages);
59
+ }
60
+ }
61
+ }
62
+ if (validGrammar.language) {
63
+ this._languageToScope.set(validGrammar.language, validGrammar.scopeName);
64
+ }
65
+ }
66
+ }
67
+ has(languageId) {
68
+ return ( this._languageToScope.has(languageId));
69
+ }
70
+ setTheme(theme, colorMap) {
71
+ this._grammarRegistry.setTheme(theme, colorMap);
72
+ }
73
+ getColorMap() {
74
+ return this._grammarRegistry.getColorMap();
75
+ }
76
+ async createGrammar(languageId, encodedLanguageId) {
77
+ const scopeName = this._languageToScope.get(languageId);
78
+ if (typeof scopeName !== 'string') {
79
+ throw new Error(missingTMGrammarErrorMessage);
80
+ }
81
+ const grammarDefinition = this._scopeRegistry.getGrammarDefinition(scopeName);
82
+ if (!grammarDefinition) {
83
+ throw new Error(missingTMGrammarErrorMessage);
84
+ }
85
+ const embeddedLanguages = grammarDefinition.embeddedLanguages;
86
+ if (this._injectedEmbeddedLanguages[scopeName]) {
87
+ const injectedEmbeddedLanguages = this._injectedEmbeddedLanguages[scopeName];
88
+ for (const injected of injectedEmbeddedLanguages) {
89
+ for (const scope of ( Object.keys(injected))) {
90
+ embeddedLanguages[scope] = injected[scope];
91
+ }
92
+ }
93
+ }
94
+ const containsEmbeddedLanguages = (( Object.keys(embeddedLanguages)).length > 0);
95
+ let grammar;
96
+ try {
97
+ grammar = await this._grammarRegistry.loadGrammarWithConfiguration(scopeName, encodedLanguageId, {
98
+ embeddedLanguages,
99
+ tokenTypes: grammarDefinition.tokenTypes,
100
+ balancedBracketSelectors: grammarDefinition.balancedBracketSelectors,
101
+ unbalancedBracketSelectors: grammarDefinition.unbalancedBracketSelectors,
102
+ });
103
+ }
104
+ catch (err) {
105
+ if (err.message && err.message.startsWith('No grammar provided for')) {
106
+ throw new Error(missingTMGrammarErrorMessage);
107
+ }
108
+ throw err;
109
+ }
110
+ return {
111
+ languageId: languageId,
112
+ grammar: grammar,
113
+ initialState: this._initialState,
114
+ containsEmbeddedLanguages: containsEmbeddedLanguages,
115
+ sourceExtensionId: grammarDefinition.sourceExtensionId,
116
+ };
117
+ }
118
+ }
119
+
120
+ export { TMGrammarFactory, missingTMGrammarErrorMessage };
@@ -0,0 +1,16 @@
1
+ class TMScopeRegistry {
2
+ constructor() {
3
+ this._scopeNameToLanguageRegistration = Object.create(null);
4
+ }
5
+ reset() {
6
+ this._scopeNameToLanguageRegistration = Object.create(null);
7
+ }
8
+ register(def) {
9
+ this._scopeNameToLanguageRegistration[def.scopeName] = def;
10
+ }
11
+ getGrammarDefinition(scopeName) {
12
+ return this._scopeNameToLanguageRegistration[scopeName] || null;
13
+ }
14
+ }
15
+
16
+ export { TMScopeRegistry };