@codingame/monaco-vscode-textmate-service-override 1.82.4 → 1.82.5-next.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (21) hide show
  1. package/external/tslib/tslib.es6.js +11 -0
  2. package/external/vscode-textmate/release/main.d.ts +64 -0
  3. package/index.d.ts +2 -2
  4. package/index.js +2 -2
  5. package/package.json +2 -2
  6. package/textmate.d.ts +5 -0
  7. package/textmate.js +43 -0
  8. package/vscode/src/vs/workbench/services/textMate/browser/arrayOperation.js +64 -0
  9. package/vscode/src/vs/workbench/services/textMate/browser/backgroundTokenization/textMateWorkerTokenizerController.js +159 -0
  10. package/vscode/src/vs/workbench/services/textMate/browser/backgroundTokenization/threadedBackgroundTokenizerFactory.js +202 -0
  11. package/vscode/src/vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateTokenizationWorker.worker.js +89 -0
  12. package/vscode/src/vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateWorkerTokenizer.js +142 -0
  13. package/vscode/src/vs/workbench/services/textMate/browser/textMateTokenizationFeature.d.ts +11 -0
  14. package/vscode/src/vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl.js +461 -0
  15. package/vscode/src/vs/workbench/services/textMate/browser/tokenizationSupport/textMateTokenizationSupport.js +74 -0
  16. package/vscode/src/vs/workbench/services/textMate/browser/tokenizationSupport/tokenizationSupportWithLineLimit.js +38 -0
  17. package/vscode/src/vs/workbench/services/textMate/common/TMGrammarFactory.js +120 -0
  18. package/vscode/src/vs/workbench/services/textMate/common/TMGrammars.js +95 -0
  19. package/vscode/src/vs/workbench/services/textMate/common/TMScopeRegistry.js +26 -0
  20. package/worker.js +1 -1
  21. package/workers/textmate.worker.js +4 -0
@@ -0,0 +1,11 @@
1
+ function __decorate(decorators, target, key, desc) {
2
+ var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
3
+ if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
4
+ else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
5
+ return c > 3 && r && Object.defineProperty(target, key, r), r;
6
+ }
7
+ function __param(paramIndex, decorator) {
8
+ return function (target, key) { decorator(target, key, paramIndex); }
9
+ }
10
+
11
+ export { __decorate, __param };
@@ -0,0 +1,64 @@
1
+ /**
2
+ * A grammar
3
+ */
4
+ interface IGrammar {
5
+ /**
6
+ * Tokenize `lineText` using previous line state `prevState`.
7
+ */
8
+ tokenizeLine(lineText: string, prevState: StateStack | null, timeLimit?: number): ITokenizeLineResult;
9
+ /**
10
+ * Tokenize `lineText` using previous line state `prevState`.
11
+ * The result contains the tokens in binary format, resolved with the following information:
12
+ * - language
13
+ * - token type (regex, string, comment, other)
14
+ * - font style
15
+ * - foreground color
16
+ * - background color
17
+ * e.g. for getting the languageId: `(metadata & MetadataConsts.LANGUAGEID_MASK) >>> MetadataConsts.LANGUAGEID_OFFSET`
18
+ */
19
+ tokenizeLine2(lineText: string, prevState: StateStack | null, timeLimit?: number): ITokenizeLineResult2;
20
+ }
21
+ interface ITokenizeLineResult {
22
+ readonly tokens: IToken[];
23
+ /**
24
+ * The `prevState` to be passed on to the next line tokenization.
25
+ */
26
+ readonly ruleStack: StateStack;
27
+ /**
28
+ * Did tokenization stop early due to reaching the time limit.
29
+ */
30
+ readonly stoppedEarly: boolean;
31
+ }
32
+ interface ITokenizeLineResult2 {
33
+ /**
34
+ * The tokens in binary format. Each token occupies two array indices. For token i:
35
+ * - at offset 2*i => startIndex
36
+ * - at offset 2*i + 1 => metadata
37
+ *
38
+ */
39
+ readonly tokens: Uint32Array;
40
+ /**
41
+ * The `prevState` to be passed on to the next line tokenization.
42
+ */
43
+ readonly ruleStack: StateStack;
44
+ /**
45
+ * Did tokenization stop early due to reaching the time limit.
46
+ */
47
+ readonly stoppedEarly: boolean;
48
+ }
49
+ interface IToken {
50
+ startIndex: number;
51
+ readonly endIndex: number;
52
+ readonly scopes: string[];
53
+ }
54
+ /**
55
+ * **IMPORTANT** - Immutable!
56
+ */
57
+ interface StateStack {
58
+ _stackElementBrand: void;
59
+ readonly depth: number;
60
+ clone(): StateStack;
61
+ equals(other: StateStack): boolean;
62
+ }
63
+
64
+ export type { IGrammar, IToken, ITokenizeLineResult, ITokenizeLineResult2, StateStack };
package/index.d.ts CHANGED
@@ -1,2 +1,2 @@
1
- export { default } from 'vscode/service-override/textmate'
2
- export * from 'vscode/service-override/textmate'
1
+ export { default } from './textmate.js';
2
+ export { ITextMateTokenizationService } from './vscode/src/vs/workbench/services/textMate/browser/textMateTokenizationFeature.js';
package/index.js CHANGED
@@ -1,2 +1,2 @@
1
- export { default } from 'vscode/service-override/textmate'
2
- export * from 'vscode/service-override/textmate'
1
+ export { default } from './textmate.js';
2
+ export { ITextMateTokenizationService } from 'vscode/vscode/vs/workbench/services/textMate/browser/textMateTokenizationFeature';
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@codingame/monaco-vscode-textmate-service-override",
3
- "version": "1.82.4",
3
+ "version": "1.82.5-next.1",
4
4
  "keywords": [],
5
5
  "author": {
6
6
  "name": "CodinGame",
@@ -18,7 +18,7 @@
18
18
  "module": "index.js",
19
19
  "types": "index.d.ts",
20
20
  "dependencies": {
21
- "vscode": "npm:@codingame/monaco-vscode-api@1.82.4",
21
+ "vscode": "npm:@codingame/monaco-vscode-api@1.82.5-next.1",
22
22
  "monaco-editor": "0.43.0",
23
23
  "vscode-oniguruma": "^2.0.0",
24
24
  "vscode-textmate": "^9.0.0"
package/textmate.d.ts ADDED
@@ -0,0 +1,5 @@
1
+ import { IEditorOverrideServices } from 'vscode/vscode/vs/editor/standalone/browser/standaloneServices';
2
+
3
+ declare function getServiceOverride(): IEditorOverrideServices;
4
+
5
+ export { getServiceOverride as default };
package/textmate.js ADDED
@@ -0,0 +1,43 @@
1
+ import { __decorate, __param } from './external/tslib/tslib.es6.js';
2
+ import { StandaloneServices } from 'monaco-editor/esm/vs/editor/standalone/browser/standaloneServices.js';
3
+ import { ITextMateTokenizationService } from 'vscode/vscode/vs/workbench/services/textMate/browser/textMateTokenizationFeature';
4
+ export { ITextMateTokenizationService } from 'vscode/vscode/vs/workbench/services/textMate/browser/textMateTokenizationFeature';
5
+ import { SyncDescriptor } from 'monaco-editor/esm/vs/platform/instantiation/common/descriptors.js';
6
+ import { TextMateTokenizationFeature } from './vscode/src/vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl.js';
7
+ import { TokenClassificationExtensionPoints } from 'vscode/vscode/vs/workbench/services/themes/common/tokenClassificationExtensionPoint';
8
+ import { Extensions } from 'vscode/vscode/vs/workbench/common/contributions';
9
+ import { Registry } from 'monaco-editor/esm/vs/platform/registry/common/platform.js';
10
+ import { ILifecycleService } from 'vscode/vscode/vs/workbench/services/lifecycle/common/lifecycle';
11
+ import { IInstantiationService } from 'monaco-editor/esm/vs/platform/instantiation/common/instantiation.js';
12
+ import getServiceOverride$1 from 'vscode/service-override/files';
13
+ import { registerServiceInitializeParticipant } from 'vscode/lifecycle';
14
+ import { registerAssets } from 'vscode/assets';
15
+
16
+ const _onigWasm = new URL('vscode-oniguruma/release/onig.wasm', import.meta.url).href;
17
+ registerAssets({
18
+ 'vscode-oniguruma/../onig.wasm': _onigWasm,
19
+ 'vs/../../node_modules/vscode-oniguruma/release/onig.wasm': _onigWasm
20
+ });
21
+ registerServiceInitializeParticipant(async (accessor) => {
22
+ void accessor.get(ILifecycleService).when(2 ).then(() => {
23
+ StandaloneServices.get(ITextMateTokenizationService);
24
+ });
25
+ });
26
+ let ExtensionPoints = class ExtensionPoints {
27
+ constructor(instantiationService) {
28
+ this.instantiationService = instantiationService;
29
+ this.instantiationService.createInstance(TokenClassificationExtensionPoints);
30
+ }
31
+ };
32
+ ExtensionPoints = __decorate([
33
+ ( __param(0, IInstantiationService))
34
+ ], ExtensionPoints);
35
+ ( Registry.as(Extensions.Workbench)).registerWorkbenchContribution(ExtensionPoints, 1 );
36
+ function getServiceOverride() {
37
+ return {
38
+ ...getServiceOverride$1(),
39
+ [( ITextMateTokenizationService.toString())]: new SyncDescriptor(TextMateTokenizationFeature, [], false)
40
+ };
41
+ }
42
+
43
+ export { getServiceOverride as default };
@@ -0,0 +1,64 @@
1
+ import { compareBy, numberComparator } from 'monaco-editor/esm/vs/base/common/arrays.js';
2
+
3
+ class ArrayEdit {
4
+ constructor(
5
+ edits) {
6
+ this.edits = edits.slice().sort(compareBy(c => c.offset, numberComparator));
7
+ }
8
+ applyToArray(array) {
9
+ for (let i = this.edits.length - 1; i >= 0; i--) {
10
+ const c = this.edits[i];
11
+ array.splice(c.offset, c.length, ...( new Array(c.newLength)));
12
+ }
13
+ }
14
+ }
15
+ class SingleArrayEdit {
16
+ constructor(offset, length, newLength) {
17
+ this.offset = offset;
18
+ this.length = length;
19
+ this.newLength = newLength;
20
+ }
21
+ toString() {
22
+ return `[${this.offset}, +${this.length}) -> +${this.newLength}}`;
23
+ }
24
+ }
25
+ class MonotonousIndexTransformer {
26
+ static fromMany(transformations) {
27
+ const transformers = ( transformations.map(t => ( new MonotonousIndexTransformer(t))));
28
+ return ( new CombinedIndexTransformer(transformers));
29
+ }
30
+ constructor(transformation) {
31
+ this.transformation = transformation;
32
+ this.idx = 0;
33
+ this.offset = 0;
34
+ }
35
+ transform(index) {
36
+ let nextChange = this.transformation.edits[this.idx];
37
+ while (nextChange && nextChange.offset + nextChange.length <= index) {
38
+ this.offset += nextChange.newLength - nextChange.length;
39
+ this.idx++;
40
+ nextChange = this.transformation.edits[this.idx];
41
+ }
42
+ if (nextChange && nextChange.offset <= index) {
43
+ return undefined;
44
+ }
45
+ return index + this.offset;
46
+ }
47
+ }
48
+ class CombinedIndexTransformer {
49
+ constructor(transformers) {
50
+ this.transformers = transformers;
51
+ }
52
+ transform(index) {
53
+ for (const transformer of this.transformers) {
54
+ const result = transformer.transform(index);
55
+ if (result === undefined) {
56
+ return undefined;
57
+ }
58
+ index = result;
59
+ }
60
+ return index;
61
+ }
62
+ }
63
+
64
+ export { ArrayEdit, CombinedIndexTransformer, MonotonousIndexTransformer, SingleArrayEdit };
@@ -0,0 +1,159 @@
1
+ import { Disposable } from 'monaco-editor/esm/vs/base/common/lifecycle.js';
2
+ import { keepAlive, autorun, observableFromEvent } from 'monaco-editor/esm/vs/base/common/observable.js';
3
+ import { countEOL } from 'monaco-editor/esm/vs/editor/common/core/eolCounter.js';
4
+ import { LineRange } from 'monaco-editor/esm/vs/editor/common/core/lineRange.js';
5
+ import { Range } from 'monaco-editor/esm/vs/editor/common/core/range.js';
6
+ import { TokenizationStateStore } from 'monaco-editor/esm/vs/editor/common/model/textModelTokens.js';
7
+ import { ContiguousMultilineTokensBuilder } from 'monaco-editor/esm/vs/editor/common/tokens/contiguousMultilineTokensBuilder.js';
8
+ import { MonotonousIndexTransformer, ArrayEdit, SingleArrayEdit } from '../arrayOperation.js';
9
+ import { applyStateStackDiff, INITIAL } from 'vscode-textmate';
10
+
11
+ class TextMateWorkerTokenizerController extends Disposable {
12
+ static { this._id = 0; }
13
+ constructor(_model, _worker, _languageIdCodec, _backgroundTokenizationStore, _configurationService, _maxTokenizationLineLength) {
14
+ super();
15
+ this._model = _model;
16
+ this._worker = _worker;
17
+ this._languageIdCodec = _languageIdCodec;
18
+ this._backgroundTokenizationStore = _backgroundTokenizationStore;
19
+ this._configurationService = _configurationService;
20
+ this._maxTokenizationLineLength = _maxTokenizationLineLength;
21
+ this.controllerId = TextMateWorkerTokenizerController._id++;
22
+ this._pendingChanges = [];
23
+ this._states = ( new TokenizationStateStore());
24
+ this._loggingEnabled = observableConfigValue('editor.experimental.asyncTokenizationLogging', false, this._configurationService);
25
+ this._register(keepAlive(this._loggingEnabled));
26
+ this._register(this._model.onDidChangeContent((e) => {
27
+ if (this._shouldLog) {
28
+ console.log('model change', {
29
+ fileName: this._model.uri.fsPath.split('\\').pop(),
30
+ changes: changesToString(e.changes),
31
+ });
32
+ }
33
+ this._worker.acceptModelChanged(this.controllerId, e);
34
+ this._pendingChanges.push(e);
35
+ }));
36
+ this._register(this._model.onDidChangeLanguage((e) => {
37
+ const languageId = this._model.getLanguageId();
38
+ const encodedLanguageId = this._languageIdCodec.encodeLanguageId(languageId);
39
+ this._worker.acceptModelLanguageChanged(this.controllerId, languageId, encodedLanguageId);
40
+ }));
41
+ const languageId = this._model.getLanguageId();
42
+ const encodedLanguageId = this._languageIdCodec.encodeLanguageId(languageId);
43
+ this._worker.acceptNewModel({
44
+ uri: this._model.uri,
45
+ versionId: this._model.getVersionId(),
46
+ lines: this._model.getLinesContent(),
47
+ EOL: this._model.getEOL(),
48
+ languageId,
49
+ encodedLanguageId,
50
+ maxTokenizationLineLength: this._maxTokenizationLineLength.get(),
51
+ controllerId: this.controllerId,
52
+ });
53
+ this._register(autorun(reader => {
54
+ const maxTokenizationLineLength = this._maxTokenizationLineLength.read(reader);
55
+ this._worker.acceptMaxTokenizationLineLength(this.controllerId, maxTokenizationLineLength);
56
+ }));
57
+ }
58
+ dispose() {
59
+ super.dispose();
60
+ this._worker.acceptRemovedModel(this.controllerId);
61
+ }
62
+ requestTokens(startLineNumber, endLineNumberExclusive) {
63
+ this._worker.retokenize(this.controllerId, startLineNumber, endLineNumberExclusive);
64
+ }
65
+ async setTokensAndStates(controllerId, versionId, rawTokens, stateDeltas) {
66
+ if (this.controllerId !== controllerId) {
67
+ return;
68
+ }
69
+ let tokens = ContiguousMultilineTokensBuilder.deserialize(( new Uint8Array(rawTokens)));
70
+ if (this._shouldLog) {
71
+ console.log('received background tokenization result', {
72
+ fileName: this._model.uri.fsPath.split('\\').pop(),
73
+ updatedTokenLines: ( tokens.map((t) => t.getLineRange())).join(' & '),
74
+ updatedStateLines: ( stateDeltas.map((s) => ( ( new LineRange(s.startLineNumber, s.startLineNumber + s.stateDeltas.length)).toString()))).join(' & '),
75
+ });
76
+ }
77
+ if (this._shouldLog) {
78
+ const changes = ( ( this._pendingChanges.filter(c => c.versionId <= versionId).map(c => c.changes)).map(c => changesToString(c))).join(' then ');
79
+ console.log('Applying changes to local states', changes);
80
+ }
81
+ while (this._pendingChanges.length > 0 &&
82
+ this._pendingChanges[0].versionId <= versionId) {
83
+ const change = this._pendingChanges.shift();
84
+ this._states.acceptChanges(change.changes);
85
+ }
86
+ if (this._pendingChanges.length > 0) {
87
+ if (this._shouldLog) {
88
+ const changes = ( ( this._pendingChanges.map(c => c.changes)).map(c => changesToString(c))).join(' then ');
89
+ console.log('Considering non-processed changes', changes);
90
+ }
91
+ const curToFutureTransformerTokens = MonotonousIndexTransformer.fromMany(( this._pendingChanges.map((c) => fullLineArrayEditFromModelContentChange(c.changes))));
92
+ const b = ( new ContiguousMultilineTokensBuilder());
93
+ for (const t of tokens) {
94
+ for (let i = t.startLineNumber; i <= t.endLineNumber; i++) {
95
+ const result = curToFutureTransformerTokens.transform(i - 1);
96
+ if (result !== undefined) {
97
+ b.add(i, t.getLineTokens(i));
98
+ }
99
+ }
100
+ }
101
+ tokens = b.finalize();
102
+ for (const change of this._pendingChanges) {
103
+ for (const innerChanges of change.changes) {
104
+ for (let j = 0; j < tokens.length; j++) {
105
+ tokens[j].applyEdit(innerChanges.range, innerChanges.text);
106
+ }
107
+ }
108
+ }
109
+ }
110
+ const curToFutureTransformerStates = MonotonousIndexTransformer.fromMany(( this._pendingChanges.map((c) => fullLineArrayEditFromModelContentChange(c.changes))));
111
+ if (!this._applyStateStackDiffFn || !this._initialState) {
112
+ this._applyStateStackDiffFn = applyStateStackDiff;
113
+ this._initialState = INITIAL;
114
+ }
115
+ for (const d of stateDeltas) {
116
+ let prevState = d.startLineNumber <= 1 ? this._initialState : this._states.getEndState(d.startLineNumber - 1);
117
+ for (let i = 0; i < d.stateDeltas.length; i++) {
118
+ const delta = d.stateDeltas[i];
119
+ let state;
120
+ if (delta) {
121
+ state = this._applyStateStackDiffFn(prevState, delta);
122
+ this._states.setEndState(d.startLineNumber + i, state);
123
+ }
124
+ else {
125
+ state = this._states.getEndState(d.startLineNumber + i);
126
+ }
127
+ const offset = curToFutureTransformerStates.transform(d.startLineNumber + i - 1);
128
+ if (offset !== undefined) {
129
+ this._backgroundTokenizationStore.setEndState(offset + 1, state);
130
+ }
131
+ if (d.startLineNumber + i >= this._model.getLineCount() - 1) {
132
+ this._backgroundTokenizationStore.backgroundTokenizationFinished();
133
+ }
134
+ prevState = state;
135
+ }
136
+ }
137
+ this._backgroundTokenizationStore.setTokens(tokens);
138
+ }
139
+ get _shouldLog() { return this._loggingEnabled.get(); }
140
+ }
141
+ function fullLineArrayEditFromModelContentChange(c) {
142
+ return ( new ArrayEdit(( c.map((c) => ( new SingleArrayEdit(
143
+ c.range.startLineNumber - 1,
144
+ c.range.endLineNumber - c.range.startLineNumber + 1,
145
+ countEOL(c.text)[0] + 1
146
+ ))))));
147
+ }
148
+ function changesToString(changes) {
149
+ return ( changes.map(c => ( Range.lift(c.range).toString()) + ' => ' + c.text)).join(' & ');
150
+ }
151
+ function observableConfigValue(key, defaultValue, configurationService) {
152
+ return observableFromEvent((handleChange) => configurationService.onDidChangeConfiguration(e => {
153
+ if (e.affectsConfiguration(key)) {
154
+ handleChange(e);
155
+ }
156
+ }), () => configurationService.getValue(key) ?? defaultValue);
157
+ }
158
+
159
+ export { TextMateWorkerTokenizerController };
@@ -0,0 +1,202 @@
1
+ import { __decorate, __param } from '../../../../../../../../external/tslib/tslib.es6.js';
2
+ import { DisposableStore, toDisposable } from 'monaco-editor/esm/vs/base/common/lifecycle.js';
3
+ import { FileAccess, nodeModulesPath, nodeModulesAsarPath } from 'monaco-editor/esm/vs/base/common/network.js';
4
+ import { isWeb } from 'monaco-editor/esm/vs/base/common/platform.js';
5
+ import { URI } from 'monaco-editor/esm/vs/base/common/uri.js';
6
+ import { createWebWorker } from 'monaco-editor/esm/vs/editor/browser/services/webWorker.js';
7
+ import { ILanguageService } from 'monaco-editor/esm/vs/editor/common/languages/language.js';
8
+ import { ILanguageConfigurationService } from 'monaco-editor/esm/vs/editor/common/languages/languageConfigurationRegistry.js';
9
+ import { IModelService } from 'monaco-editor/esm/vs/editor/common/services/model.js';
10
+ import { IConfigurationService } from 'monaco-editor/esm/vs/platform/configuration/common/configuration.js';
11
+ import { IEnvironmentService } from 'monaco-editor/esm/vs/platform/environment/common/environment.js';
12
+ import { IExtensionResourceLoaderService } from 'vscode/vscode/vs/platform/extensionResourceLoader/common/extensionResourceLoader';
13
+ import { INotificationService } from 'monaco-editor/esm/vs/platform/notification/common/notification.js';
14
+ import { ITelemetryService } from 'monaco-editor/esm/vs/platform/telemetry/common/telemetry.js';
15
+ import { TextMateWorkerTokenizerController } from './textMateWorkerTokenizerController.js';
16
+
17
+ var ThreadedBackgroundTokenizerFactory_1;
18
+ let ThreadedBackgroundTokenizerFactory = class ThreadedBackgroundTokenizerFactory {
19
+ static { ThreadedBackgroundTokenizerFactory_1 = this; }
20
+ static { this._reportedMismatchingTokens = false; }
21
+ constructor(_reportTokenizationTime, _shouldTokenizeAsync, _extensionResourceLoaderService, _modelService, _languageConfigurationService, _configurationService, _languageService, _environmentService, _notificationService, _telemetryService) {
22
+ this._reportTokenizationTime = _reportTokenizationTime;
23
+ this._shouldTokenizeAsync = _shouldTokenizeAsync;
24
+ this._extensionResourceLoaderService = _extensionResourceLoaderService;
25
+ this._modelService = _modelService;
26
+ this._languageConfigurationService = _languageConfigurationService;
27
+ this._configurationService = _configurationService;
28
+ this._languageService = _languageService;
29
+ this._environmentService = _environmentService;
30
+ this._notificationService = _notificationService;
31
+ this._telemetryService = _telemetryService;
32
+ this._workerProxyPromise = null;
33
+ this._worker = null;
34
+ this._workerProxy = null;
35
+ this._workerTokenizerControllers = ( new Map());
36
+ this._currentTheme = null;
37
+ this._currentTokenColorMap = null;
38
+ this._grammarDefinitions = [];
39
+ }
40
+ dispose() {
41
+ this._disposeWorker();
42
+ }
43
+ createBackgroundTokenizer(textModel, tokenStore, maxTokenizationLineLength) {
44
+ if (!this._shouldTokenizeAsync() || textModel.isTooLargeForSyncing()) {
45
+ return undefined;
46
+ }
47
+ const store = ( new DisposableStore());
48
+ const controllerContainer = this._getWorkerProxy().then((workerProxy) => {
49
+ if (store.isDisposed || !workerProxy) {
50
+ return undefined;
51
+ }
52
+ const controllerContainer = { controller: undefined, worker: this._worker };
53
+ store.add(keepAliveWhenAttached(textModel, () => {
54
+ const controller = ( new TextMateWorkerTokenizerController(
55
+ textModel,
56
+ workerProxy,
57
+ this._languageService.languageIdCodec,
58
+ tokenStore,
59
+ this._configurationService,
60
+ maxTokenizationLineLength
61
+ ));
62
+ controllerContainer.controller = controller;
63
+ this._workerTokenizerControllers.set(controller.controllerId, controller);
64
+ return toDisposable(() => {
65
+ controllerContainer.controller = undefined;
66
+ this._workerTokenizerControllers.delete(controller.controllerId);
67
+ controller.dispose();
68
+ });
69
+ }));
70
+ return controllerContainer;
71
+ });
72
+ return {
73
+ dispose() {
74
+ store.dispose();
75
+ },
76
+ requestTokens: async (startLineNumber, endLineNumberExclusive) => {
77
+ const container = await controllerContainer;
78
+ if (container?.controller && container.worker === this._worker) {
79
+ container.controller.requestTokens(startLineNumber, endLineNumberExclusive);
80
+ }
81
+ },
82
+ reportMismatchingTokens: (lineNumber) => {
83
+ if (ThreadedBackgroundTokenizerFactory_1._reportedMismatchingTokens) {
84
+ return;
85
+ }
86
+ ThreadedBackgroundTokenizerFactory_1._reportedMismatchingTokens = true;
87
+ this._notificationService.error({
88
+ message: 'Async Tokenization Token Mismatch in line ' + lineNumber,
89
+ name: 'Async Tokenization Token Mismatch',
90
+ });
91
+ this._telemetryService.publicLog2('asyncTokenizationMismatchingTokens', {});
92
+ },
93
+ };
94
+ }
95
+ setGrammarDefinitions(grammarDefinitions) {
96
+ this._grammarDefinitions = grammarDefinitions;
97
+ this._disposeWorker();
98
+ }
99
+ acceptTheme(theme, colorMap) {
100
+ this._currentTheme = theme;
101
+ this._currentTokenColorMap = colorMap;
102
+ if (this._currentTheme && this._currentTokenColorMap && this._workerProxy) {
103
+ this._workerProxy.acceptTheme(this._currentTheme, this._currentTokenColorMap);
104
+ }
105
+ }
106
+ _getWorkerProxy() {
107
+ if (!this._workerProxyPromise) {
108
+ this._workerProxyPromise = this._createWorkerProxy();
109
+ }
110
+ return this._workerProxyPromise;
111
+ }
112
+ async _createWorkerProxy() {
113
+ const textmateModuleLocation = `${nodeModulesPath}/vscode-textmate`;
114
+ const textmateModuleLocationAsar = `${nodeModulesAsarPath}/vscode-textmate`;
115
+ const onigurumaModuleLocation = `${nodeModulesPath}/vscode-oniguruma`;
116
+ const onigurumaModuleLocationAsar = `${nodeModulesAsarPath}/vscode-oniguruma`;
117
+ const useAsar = this._environmentService.isBuilt && !isWeb;
118
+ const textmateLocation = useAsar ? textmateModuleLocationAsar : textmateModuleLocation;
119
+ const onigurumaLocation = useAsar ? onigurumaModuleLocationAsar : onigurumaModuleLocation;
120
+ const textmateMain = `${textmateLocation}/release/main.js`;
121
+ const onigurumaMain = `${onigurumaLocation}/release/main.js`;
122
+ const onigurumaWASM = `${onigurumaLocation}/release/onig.wasm`;
123
+ const uri = ( ( FileAccess.asBrowserUri(textmateMain)).toString(true));
124
+ const createData = {
125
+ grammarDefinitions: this._grammarDefinitions,
126
+ textmateMainUri: uri,
127
+ onigurumaMainUri: ( ( FileAccess.asBrowserUri(onigurumaMain)).toString(true)),
128
+ onigurumaWASMUri: ( ( FileAccess.asBrowserUri(onigurumaWASM)).toString(true)),
129
+ };
130
+ const host = {
131
+ readFile: async (_resource) => {
132
+ const resource = URI.revive(_resource);
133
+ return this._extensionResourceLoaderService.readExtensionResource(resource);
134
+ },
135
+ setTokensAndStates: async (controllerId, versionId, tokens, lineEndStateDeltas) => {
136
+ const controller = this._workerTokenizerControllers.get(controllerId);
137
+ if (controller) {
138
+ controller.setTokensAndStates(controllerId, versionId, tokens, lineEndStateDeltas);
139
+ }
140
+ },
141
+ reportTokenizationTime: (timeMs, languageId, sourceExtensionId, lineLength, isRandomSample) => {
142
+ this._reportTokenizationTime(timeMs, languageId, sourceExtensionId, lineLength, isRandomSample);
143
+ }
144
+ };
145
+ const worker = this._worker = createWebWorker(this._modelService, this._languageConfigurationService, {
146
+ createData,
147
+ label: 'textMateWorker',
148
+ moduleId: 'vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateTokenizationWorker.worker',
149
+ host,
150
+ });
151
+ const proxy = await ( worker.getProxy());
152
+ if (this._worker !== worker) {
153
+ return null;
154
+ }
155
+ this._workerProxy = proxy;
156
+ if (this._currentTheme && this._currentTokenColorMap) {
157
+ this._workerProxy.acceptTheme(this._currentTheme, this._currentTokenColorMap);
158
+ }
159
+ return proxy;
160
+ }
161
+ _disposeWorker() {
162
+ for (const controller of ( this._workerTokenizerControllers.values())) {
163
+ controller.dispose();
164
+ }
165
+ this._workerTokenizerControllers.clear();
166
+ if (this._worker) {
167
+ this._worker.dispose();
168
+ this._worker = null;
169
+ }
170
+ this._workerProxy = null;
171
+ this._workerProxyPromise = null;
172
+ }
173
+ };
174
+ ThreadedBackgroundTokenizerFactory = ThreadedBackgroundTokenizerFactory_1 = ( __decorate([
175
+ ( __param(2, IExtensionResourceLoaderService)),
176
+ ( __param(3, IModelService)),
177
+ ( __param(4, ILanguageConfigurationService)),
178
+ ( __param(5, IConfigurationService)),
179
+ ( __param(6, ILanguageService)),
180
+ ( __param(7, IEnvironmentService)),
181
+ ( __param(8, INotificationService)),
182
+ ( __param(9, ITelemetryService))
183
+ ], ThreadedBackgroundTokenizerFactory));
184
+ function keepAliveWhenAttached(textModel, factory) {
185
+ const disposableStore = ( new DisposableStore());
186
+ const subStore = disposableStore.add(( new DisposableStore()));
187
+ function checkAttached() {
188
+ if (textModel.isAttachedToEditor()) {
189
+ subStore.add(factory());
190
+ }
191
+ else {
192
+ subStore.clear();
193
+ }
194
+ }
195
+ checkAttached();
196
+ disposableStore.add(textModel.onDidChangeAttached(() => {
197
+ checkAttached();
198
+ }));
199
+ return disposableStore;
200
+ }
201
+
202
+ export { ThreadedBackgroundTokenizerFactory };