@codingame/monaco-vscode-textmate-service-override 8.0.4 → 9.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@codingame/monaco-vscode-textmate-service-override",
3
- "version": "8.0.4",
3
+ "version": "9.0.2",
4
4
  "keywords": [],
5
5
  "author": {
6
6
  "name": "CodinGame",
@@ -29,9 +29,9 @@
29
29
  }
30
30
  },
31
31
  "dependencies": {
32
- "vscode": "npm:@codingame/monaco-vscode-api@8.0.4",
32
+ "vscode": "npm:@codingame/monaco-vscode-api@9.0.2",
33
33
  "vscode-oniguruma": "1.7.0",
34
- "vscode-textmate": "9.0.0",
35
- "@codingame/monaco-vscode-files-service-override": "8.0.4"
34
+ "vscode-textmate": "9.1.0",
35
+ "@codingame/monaco-vscode-files-service-override": "9.0.2"
36
36
  }
37
37
  }
@@ -0,0 +1,19 @@
1
+ import { isESM } from 'vscode/vscode/vs/base/common/amd';
2
+ import { nodeModulesPath, FileAccess } from 'vscode/vscode/vs/base/common/network';
3
+ import 'vscode/vscode/vs/base/common/platform';
4
+ import { assertType } from 'vscode/vscode/vs/base/common/types';
5
+ import 'vscode/vscode/vs/base/common/charCode';
6
+ import 'vscode/vscode/vs/base/common/marshallingIds';
7
+ import 'vscode/vscode/vs/base/common/path';
8
+
9
+ function resolveAmdNodeModulePath(nodeModuleName, pathInsideNodeModule) {
10
+ assertType(isESM);
11
+ const product = globalThis._VSCODE_PRODUCT_JSON;
12
+ Boolean((product ?? globalThis.vscode?.context?.configuration()?.product)?.commit);
13
+ const nodeModulePath = `${nodeModuleName}/${pathInsideNodeModule}`;
14
+ const actualNodeModulesPath = (nodeModulesPath);
15
+ const resourcePath = `${actualNodeModulesPath}/${nodeModulePath}`;
16
+ return ( ( FileAccess.asBrowserUri(resourcePath)).toString(true));
17
+ }
18
+
19
+ export { resolveAmdNodeModulePath };
@@ -9,6 +9,7 @@ import { Disposable } from 'vscode/vscode/vs/base/common/lifecycle';
9
9
  import { ContentWidgetPositionPreference } from 'vscode/vscode/vs/editor/browser/editorBrowser';
10
10
  import { EditorAction, registerEditorContribution, EditorContributionInstantiation, registerEditorAction } from 'vscode/vscode/vs/editor/browser/editorExtensions';
11
11
  import { Range } from 'vscode/vscode/vs/editor/common/core/range';
12
+ import { TreeSitterTokenizationRegistry } from 'vscode/vscode/vs/editor/common/languages';
12
13
  import { StandardTokenType, TokenMetadata, FontStyle, ColorId } from 'vscode/vscode/vs/editor/common/encodedTokenAttributes';
13
14
  import { ILanguageService } from 'vscode/vscode/vs/editor/common/languages/language';
14
15
  import { INotificationService } from 'vscode/vscode/vs/platform/notification/common/notification.service';
@@ -21,6 +22,7 @@ import { IConfigurationService } from 'vscode/vscode/vs/platform/configuration/c
21
22
  import { SEMANTIC_HIGHLIGHTING_SETTING_ID } from 'vscode/vscode/vs/editor/contrib/semanticTokens/common/semanticTokensConfig';
22
23
  import { Schemas } from 'vscode/vscode/vs/base/common/network';
23
24
  import { ILanguageFeaturesService } from 'vscode/vscode/vs/editor/common/services/languageFeatures';
25
+ import { ITreeSitterParserService } from 'vscode/vscode/vs/editor/common/services/treeSitterParserService';
24
26
 
25
27
  var InspectEditorTokensController_1;
26
28
  const $ = $$1;
@@ -30,10 +32,11 @@ let InspectEditorTokensController = class InspectEditorTokensController extends
30
32
  static get(editor) {
31
33
  return editor.getContribution(InspectEditorTokensController_1.ID);
32
34
  }
33
- constructor(editor, textMateService, languageService, themeService, notificationService, configurationService, languageFeaturesService) {
35
+ constructor(editor, textMateService, treeSitterService, languageService, themeService, notificationService, configurationService, languageFeaturesService) {
34
36
  super();
35
37
  this._editor = editor;
36
38
  this._textMateService = textMateService;
39
+ this._treeSitterService = treeSitterService;
37
40
  this._themeService = themeService;
38
41
  this._languageService = languageService;
39
42
  this._notificationService = notificationService;
@@ -61,6 +64,7 @@ let InspectEditorTokensController = class InspectEditorTokensController extends
61
64
  this._widget = ( (new InspectEditorTokensWidget(
62
65
  this._editor,
63
66
  this._textMateService,
67
+ this._treeSitterService,
64
68
  this._languageService,
65
69
  this._themeService,
66
70
  this._notificationService,
@@ -85,17 +89,18 @@ let InspectEditorTokensController = class InspectEditorTokensController extends
85
89
  };
86
90
  InspectEditorTokensController = InspectEditorTokensController_1 = ( (__decorate([
87
91
  ( (__param(1, ITextMateTokenizationService))),
88
- ( (__param(2, ILanguageService))),
89
- ( (__param(3, IWorkbenchThemeService))),
90
- ( (__param(4, INotificationService))),
91
- ( (__param(5, IConfigurationService))),
92
- ( (__param(6, ILanguageFeaturesService)))
92
+ ( (__param(2, ITreeSitterParserService))),
93
+ ( (__param(3, ILanguageService))),
94
+ ( (__param(4, IWorkbenchThemeService))),
95
+ ( (__param(5, INotificationService))),
96
+ ( (__param(6, IConfigurationService))),
97
+ ( (__param(7, ILanguageFeaturesService)))
93
98
  ], InspectEditorTokensController)));
94
99
  class InspectEditorTokens extends EditorAction {
95
100
  constructor() {
96
101
  super({
97
102
  id: 'editor.action.inspectTMScopes',
98
- label: ( localize(2805, "Developer: Inspect Editor Tokens and Scopes")),
103
+ label: ( localize(2806, "Developer: Inspect Editor Tokens and Scopes")),
99
104
  alias: 'Developer: Inspect Editor Tokens and Scopes',
100
105
  precondition: undefined
101
106
  });
@@ -127,7 +132,7 @@ function renderTokenText(tokenText) {
127
132
  }
128
133
  class InspectEditorTokensWidget extends Disposable {
129
134
  static { this._ID = 'editor.contrib.inspectEditorTokensWidget'; }
130
- constructor(editor, textMateService, languageService, themeService, notificationService, configurationService, languageFeaturesService) {
135
+ constructor(editor, textMateService, treeSitterService, languageService, themeService, notificationService, configurationService, languageFeaturesService) {
131
136
  super();
132
137
  this.allowEditorOverflow = true;
133
138
  this._isDisposed = false;
@@ -135,6 +140,7 @@ class InspectEditorTokensWidget extends Disposable {
135
140
  this._languageService = languageService;
136
141
  this._themeService = themeService;
137
142
  this._textMateService = textMateService;
143
+ this._treeSitterService = treeSitterService;
138
144
  this._notificationService = notificationService;
139
145
  this._configurationService = configurationService;
140
146
  this._languageFeaturesService = languageFeaturesService;
@@ -160,13 +166,14 @@ class InspectEditorTokensWidget extends Disposable {
160
166
  _beginCompute(position) {
161
167
  const grammar = this._textMateService.createTokenizer(this._model.getLanguageId());
162
168
  const semanticTokens = this._computeSemanticTokens(position);
169
+ const tree = this._treeSitterService.getParseResult(this._model);
163
170
  clearNode(this._domNode);
164
- this._domNode.appendChild(document.createTextNode(( localize(2806, "Loading..."))));
171
+ this._domNode.appendChild(document.createTextNode(( localize(2807, "Loading..."))));
165
172
  Promise.all([grammar, semanticTokens]).then(([grammar, semanticTokens]) => {
166
173
  if (this._isDisposed) {
167
174
  return;
168
175
  }
169
- this._compute(grammar, semanticTokens, position);
176
+ this._compute(grammar, semanticTokens, tree?.tree, position);
170
177
  this._domNode.style.maxWidth = `${Math.max(this._editor.getLayoutInfo().width * 0.66, 500)}px`;
171
178
  this._editor.layoutContentWidget(this);
172
179
  }, (err) => {
@@ -183,10 +190,11 @@ class InspectEditorTokensWidget extends Disposable {
183
190
  }
184
191
  return this._themeService.getColorTheme().semanticHighlighting;
185
192
  }
186
- _compute(grammar, semanticTokens, position) {
193
+ _compute(grammar, semanticTokens, tree, position) {
187
194
  const textMateTokenInfo = grammar && this._getTokensAtPosition(grammar, position);
188
195
  const semanticTokenInfo = semanticTokens && this._getSemanticTokenAtPosition(semanticTokens, position);
189
- if (!textMateTokenInfo && !semanticTokenInfo) {
196
+ const treeSitterTokenInfo = tree && this._getTreeSitterTokenAtPosition(tree, position);
197
+ if (!textMateTokenInfo && !semanticTokenInfo && !treeSitterTokenInfo) {
190
198
  reset(this._domNode, 'No grammar or semantic tokens available.');
191
199
  return;
192
200
  }
@@ -258,6 +266,27 @@ class InspectEditorTokensWidget extends Disposable {
258
266
  append(tbody, $('tr', undefined, $('td.tiw-metadata-key', undefined, 'foreground'), $('td.tiw-metadata-value', undefined, 'No theme selector')));
259
267
  }
260
268
  }
269
+ if (treeSitterTokenInfo) {
270
+ append(this._domNode, $('hr.tiw-metadata-separator'));
271
+ const table = append(this._domNode, $('table.tiw-metadata-table'));
272
+ const tbody = append(table, $('tbody'));
273
+ append(tbody, $('tr', undefined, $('td.tiw-metadata-key', undefined, 'tree-sitter token'), $('td.tiw-metadata-value', undefined, `${treeSitterTokenInfo.text}`)));
274
+ const scopes = ( (new Array()));
275
+ let node = treeSitterTokenInfo;
276
+ while (node.parent) {
277
+ scopes.push(node.type);
278
+ node = node.parent;
279
+ if (node) {
280
+ scopes.push($('br'));
281
+ }
282
+ }
283
+ append(tbody, $('tr', undefined, $('td.tiw-metadata-key', undefined, 'tree-sitter scopes'), $('td.tiw-metadata-value.tiw-metadata-scopes', undefined, ...scopes)));
284
+ const tokenizationSupport = TreeSitterTokenizationRegistry.get(this._model.getLanguageId());
285
+ const captures = tokenizationSupport?.captureAtPosition(position.lineNumber, position.column, this._model);
286
+ if (captures && captures.length > 0) {
287
+ append(tbody, $('tr', undefined, $('td.tiw-metadata-key', undefined, 'foreground'), $('td.tiw-metadata-value', undefined, captures[0].name)));
288
+ }
289
+ }
261
290
  }
262
291
  _formatMetadata(semantic, tm) {
263
292
  const elements = ( (new Array()));
@@ -440,6 +469,26 @@ class InspectEditorTokensWidget extends Disposable {
440
469
  }
441
470
  return null;
442
471
  }
472
+ _walkTreeforPosition(cursor, pos) {
473
+ const offset = this._model.getOffsetAt(pos);
474
+ cursor.gotoFirstChild();
475
+ let goChild = false;
476
+ let lastGoodNode = null;
477
+ do {
478
+ if (cursor.currentNode.startIndex <= offset && offset < cursor.currentNode.endIndex) {
479
+ goChild = true;
480
+ lastGoodNode = cursor.currentNode;
481
+ }
482
+ else {
483
+ goChild = false;
484
+ }
485
+ } while (goChild ? cursor.gotoFirstChild() : cursor.gotoNextSibling());
486
+ return lastGoodNode;
487
+ }
488
+ _getTreeSitterTokenAtPosition(tree, pos) {
489
+ const cursor = tree.walk();
490
+ return this._walkTreeforPosition(cursor, pos);
491
+ }
443
492
  _renderTokenStyleDefinition(definition, property) {
444
493
  const elements = ( (new Array()));
445
494
  if (definition === undefined) {
@@ -35,17 +35,17 @@ class TextMateWorkerTokenizerController extends Disposable {
35
35
  changes: changesToString(e.changes),
36
36
  });
37
37
  }
38
- this._worker.acceptModelChanged(this.controllerId, e);
38
+ this._worker.$acceptModelChanged(this.controllerId, e);
39
39
  this._pendingChanges.push(e);
40
40
  }));
41
41
  this._register(this._model.onDidChangeLanguage((e) => {
42
42
  const languageId = this._model.getLanguageId();
43
43
  const encodedLanguageId = this._languageIdCodec.encodeLanguageId(languageId);
44
- this._worker.acceptModelLanguageChanged(this.controllerId, languageId, encodedLanguageId);
44
+ this._worker.$acceptModelLanguageChanged(this.controllerId, languageId, encodedLanguageId);
45
45
  }));
46
46
  const languageId = this._model.getLanguageId();
47
47
  const encodedLanguageId = this._languageIdCodec.encodeLanguageId(languageId);
48
- this._worker.acceptNewModel({
48
+ this._worker.$acceptNewModel({
49
49
  uri: this._model.uri,
50
50
  versionId: this._model.getVersionId(),
51
51
  lines: this._model.getLinesContent(),
@@ -57,15 +57,15 @@ class TextMateWorkerTokenizerController extends Disposable {
57
57
  });
58
58
  this._register(autorun(reader => {
59
59
  const maxTokenizationLineLength = this._maxTokenizationLineLength.read(reader);
60
- this._worker.acceptMaxTokenizationLineLength(this.controllerId, maxTokenizationLineLength);
60
+ this._worker.$acceptMaxTokenizationLineLength(this.controllerId, maxTokenizationLineLength);
61
61
  }));
62
62
  }
63
63
  dispose() {
64
64
  super.dispose();
65
- this._worker.acceptRemovedModel(this.controllerId);
65
+ this._worker.$acceptRemovedModel(this.controllerId);
66
66
  }
67
67
  requestTokens(startLineNumber, endLineNumberExclusive) {
68
- this._worker.retokenize(this.controllerId, startLineNumber, endLineNumberExclusive);
68
+ this._worker.$retokenize(this.controllerId, startLineNumber, endLineNumberExclusive);
69
69
  }
70
70
  async setTokensAndStates(controllerId, versionId, rawTokens, stateDeltas) {
71
71
  if (this.controllerId !== controllerId) {
@@ -1,29 +1,27 @@
1
1
  import { __decorate, __param } from 'vscode/external/tslib/tslib.es6.js';
2
+ import 'vscode/vscode/vs/base/common/amd';
2
3
  import { DisposableStore, toDisposable } from 'vscode/vscode/vs/base/common/lifecycle';
3
- import { nodeModulesPath, nodeModulesAsarPath, FileAccess } from 'vscode/vscode/vs/base/common/network';
4
- import { isWeb } from 'vscode/vscode/vs/base/common/platform';
4
+ import { nodeModulesPath, FileAccess } from 'vscode/vscode/vs/base/common/network';
5
+ import 'vscode/vscode/vs/base/common/platform';
5
6
  import { URI } from 'vscode/vscode/vs/base/common/uri';
6
- import { createWebWorker } from 'vscode/vscode/vs/editor/browser/services/webWorker';
7
7
  import { ILanguageService } from 'vscode/vscode/vs/editor/common/languages/language';
8
- import { ILanguageConfigurationService } from 'vscode/vscode/vs/editor/common/languages/languageConfigurationRegistry';
9
- import { IModelService } from 'vscode/vscode/vs/editor/common/services/model';
10
8
  import { IConfigurationService } from 'vscode/vscode/vs/platform/configuration/common/configuration.service';
11
9
  import { IEnvironmentService } from 'vscode/vscode/vs/platform/environment/common/environment.service';
12
10
  import { IExtensionResourceLoaderService } from 'vscode/vscode/vs/platform/extensionResourceLoader/common/extensionResourceLoader.service';
13
11
  import { INotificationService } from 'vscode/vscode/vs/platform/notification/common/notification.service';
14
12
  import { ITelemetryService } from 'vscode/vscode/vs/platform/telemetry/common/telemetry.service';
13
+ import { TextMateWorkerHost } from './worker/textMateWorkerHost.js';
15
14
  import { TextMateWorkerTokenizerController } from './textMateWorkerTokenizerController.js';
15
+ import { createWebWorker } from 'vscode/vscode/vs/base/browser/defaultWorkerFactory';
16
16
 
17
17
  var ThreadedBackgroundTokenizerFactory_1;
18
18
  let ThreadedBackgroundTokenizerFactory = class ThreadedBackgroundTokenizerFactory {
19
19
  static { ThreadedBackgroundTokenizerFactory_1 = this; }
20
20
  static { this._reportedMismatchingTokens = false; }
21
- constructor(_reportTokenizationTime, _shouldTokenizeAsync, _extensionResourceLoaderService, _modelService, _languageConfigurationService, _configurationService, _languageService, _environmentService, _notificationService, _telemetryService) {
21
+ constructor(_reportTokenizationTime, _shouldTokenizeAsync, _extensionResourceLoaderService, _configurationService, _languageService, _environmentService, _notificationService, _telemetryService) {
22
22
  this._reportTokenizationTime = _reportTokenizationTime;
23
23
  this._shouldTokenizeAsync = _shouldTokenizeAsync;
24
24
  this._extensionResourceLoaderService = _extensionResourceLoaderService;
25
- this._modelService = _modelService;
26
- this._languageConfigurationService = _languageConfigurationService;
27
25
  this._configurationService = _configurationService;
28
26
  this._languageService = _languageService;
29
27
  this._environmentService = _environmentService;
@@ -100,7 +98,7 @@ let ThreadedBackgroundTokenizerFactory = class ThreadedBackgroundTokenizerFactor
100
98
  this._currentTheme = theme;
101
99
  this._currentTokenColorMap = colorMap;
102
100
  if (this._currentTheme && this._currentTokenColorMap && this._workerProxy) {
103
- this._workerProxy.acceptTheme(this._currentTheme, this._currentTokenColorMap);
101
+ this._workerProxy.$acceptTheme(this._currentTheme, this._currentTokenColorMap);
104
102
  }
105
103
  }
106
104
  _getWorkerProxy() {
@@ -110,53 +108,38 @@ let ThreadedBackgroundTokenizerFactory = class ThreadedBackgroundTokenizerFactor
110
108
  return this._workerProxyPromise;
111
109
  }
112
110
  async _createWorkerProxy() {
113
- const textmateModuleLocation = `${nodeModulesPath}/vscode-textmate`;
114
- const textmateModuleLocationAsar = `${nodeModulesAsarPath}/vscode-textmate`;
115
111
  const onigurumaModuleLocation = `${nodeModulesPath}/vscode-oniguruma`;
116
- const onigurumaModuleLocationAsar = `${nodeModulesAsarPath}/vscode-oniguruma`;
117
- const useAsar = this._environmentService.isBuilt && !isWeb;
118
- const textmateLocation = useAsar ? textmateModuleLocationAsar : textmateModuleLocation;
119
- const onigurumaLocation = useAsar ? onigurumaModuleLocationAsar : onigurumaModuleLocation;
120
- const textmateMain = `${textmateLocation}/release/main.js`;
121
- const onigurumaMain = `${onigurumaLocation}/release/main.js`;
112
+ const onigurumaLocation = onigurumaModuleLocation;
122
113
  const onigurumaWASM = `${onigurumaLocation}/release/onig.wasm`;
123
- const uri = ( ( FileAccess.asBrowserUri(textmateMain)).toString(true));
124
114
  const createData = {
125
115
  grammarDefinitions: this._grammarDefinitions,
126
- textmateMainUri: uri,
127
- onigurumaMainUri: ( ( FileAccess.asBrowserUri(onigurumaMain)).toString(true)),
128
116
  onigurumaWASMUri: ( ( FileAccess.asBrowserUri(onigurumaWASM)).toString(true)),
129
117
  };
130
- const host = {
131
- readFile: async (_resource) => {
118
+ const worker = this._worker = createWebWorker('vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateTokenizationWorker.worker', 'TextMateWorker');
119
+ TextMateWorkerHost.setChannel(worker, {
120
+ $readFile: async (_resource) => {
132
121
  const resource = URI.revive(_resource);
133
122
  return this._extensionResourceLoaderService.readExtensionResource(resource);
134
123
  },
135
- setTokensAndStates: async (controllerId, versionId, tokens, lineEndStateDeltas) => {
124
+ $setTokensAndStates: async (controllerId, versionId, tokens, lineEndStateDeltas) => {
136
125
  const controller = this._workerTokenizerControllers.get(controllerId);
137
126
  if (controller) {
138
127
  controller.setTokensAndStates(controllerId, versionId, tokens, lineEndStateDeltas);
139
128
  }
140
129
  },
141
- reportTokenizationTime: (timeMs, languageId, sourceExtensionId, lineLength, isRandomSample) => {
130
+ $reportTokenizationTime: (timeMs, languageId, sourceExtensionId, lineLength, isRandomSample) => {
142
131
  this._reportTokenizationTime(timeMs, languageId, sourceExtensionId, lineLength, isRandomSample);
143
132
  }
144
- };
145
- const worker = this._worker = createWebWorker(this._modelService, this._languageConfigurationService, {
146
- createData,
147
- label: 'textMateWorker',
148
- moduleId: 'vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateTokenizationWorker.worker',
149
- host,
150
133
  });
151
- const proxy = await ( worker.getProxy());
134
+ await worker.proxy.$init(createData);
152
135
  if (this._worker !== worker) {
153
136
  return null;
154
137
  }
155
- this._workerProxy = proxy;
138
+ this._workerProxy = worker.proxy;
156
139
  if (this._currentTheme && this._currentTokenColorMap) {
157
- this._workerProxy.acceptTheme(this._currentTheme, this._currentTokenColorMap);
140
+ this._workerProxy.$acceptTheme(this._currentTheme, this._currentTokenColorMap);
158
141
  }
159
- return proxy;
142
+ return worker.proxy;
160
143
  }
161
144
  _disposeWorker() {
162
145
  for (const controller of ( this._workerTokenizerControllers.values())) {
@@ -173,13 +156,11 @@ let ThreadedBackgroundTokenizerFactory = class ThreadedBackgroundTokenizerFactor
173
156
  };
174
157
  ThreadedBackgroundTokenizerFactory = ThreadedBackgroundTokenizerFactory_1 = ( __decorate([
175
158
  ( __param(2, IExtensionResourceLoaderService)),
176
- ( __param(3, IModelService)),
177
- ( __param(4, ILanguageConfigurationService)),
178
- ( __param(5, IConfigurationService)),
179
- ( __param(6, ILanguageService)),
180
- ( __param(7, IEnvironmentService)),
181
- ( __param(8, INotificationService)),
182
- ( __param(9, ITelemetryService))
159
+ ( __param(3, IConfigurationService)),
160
+ ( __param(4, ILanguageService)),
161
+ ( __param(5, IEnvironmentService)),
162
+ ( __param(6, INotificationService)),
163
+ ( __param(7, ITelemetryService))
183
164
  ], ThreadedBackgroundTokenizerFactory));
184
165
  function keepAliveWhenAttached(textModel, factory) {
185
166
  const disposableStore = ( new DisposableStore());
@@ -1,15 +1,21 @@
1
1
  import { URI } from 'vscode/vscode/vs/base/common/uri';
2
2
  import { TMGrammarFactory } from '../../../common/TMGrammarFactory.js';
3
3
  import { TextMateWorkerTokenizer } from './textMateWorkerTokenizer.js';
4
+ import { TextMateWorkerHost } from './textMateWorkerHost.js';
4
5
  import * as vscodeTextmate from 'vscode-textmate';
5
6
  import * as vscodeOniguruma from 'vscode-oniguruma';
6
7
 
8
+ function create(workerServer) {
9
+ return ( new TextMateTokenizationWorker(workerServer));
10
+ }
7
11
  class TextMateTokenizationWorker {
8
- constructor(ctx, _createData) {
9
- this._createData = _createData;
12
+ constructor(workerServer) {
10
13
  this._models = ( new Map());
11
14
  this._grammarCache = [];
12
- this._host = ctx.host;
15
+ this._grammarFactory = Promise.resolve(null);
16
+ this._host = TextMateWorkerHost.getChannel(workerServer);
17
+ }
18
+ async $init(_createData) {
13
19
  const grammarDefinitions = ( _createData.grammarDefinitions.map((def) => {
14
20
  return {
15
21
  location: URI.revive(def.location),
@@ -23,10 +29,10 @@ class TextMateTokenizationWorker {
23
29
  sourceExtensionId: def.sourceExtensionId,
24
30
  };
25
31
  }));
26
- this._grammarFactory = this._loadTMGrammarFactory(grammarDefinitions);
32
+ this._grammarFactory = this._loadTMGrammarFactory(grammarDefinitions, _createData.onigurumaWASMUri);
27
33
  }
28
- async _loadTMGrammarFactory(grammarDefinitions) {
29
- const response = await fetch(this._createData.onigurumaWASMUri);
34
+ async _loadTMGrammarFactory(grammarDefinitions, onigurumaWASMUri) {
35
+ const response = await fetch(onigurumaWASMUri);
30
36
  const bytes = await response.arrayBuffer();
31
37
  await vscodeOniguruma.loadWASM(bytes);
32
38
  const onigLib = Promise.resolve({
@@ -36,10 +42,10 @@ class TextMateTokenizationWorker {
36
42
  return ( new TMGrammarFactory({
37
43
  logTrace: (msg) => { },
38
44
  logError: (msg, err) => console.error(msg, err),
39
- readFile: (resource) => this._host.readFile(resource)
45
+ readFile: (resource) => this._host.$readFile(resource)
40
46
  }, grammarDefinitions, vscodeTextmate, onigLib));
41
47
  }
42
- acceptNewModel(data) {
48
+ $acceptNewModel(data) {
43
49
  const uri = URI.revive(data.uri);
44
50
  const that = this;
45
51
  this._models.set(data.controllerId, ( new TextMateWorkerTokenizer(uri, data.lines, data.EOL, data.versionId, {
@@ -54,36 +60,36 @@ class TextMateTokenizationWorker {
54
60
  return that._grammarCache[encodedLanguageId];
55
61
  },
56
62
  setTokensAndStates(versionId, tokens, stateDeltas) {
57
- that._host.setTokensAndStates(data.controllerId, versionId, tokens, stateDeltas);
63
+ that._host.$setTokensAndStates(data.controllerId, versionId, tokens, stateDeltas);
58
64
  },
59
65
  reportTokenizationTime(timeMs, languageId, sourceExtensionId, lineLength, isRandomSample) {
60
- that._host.reportTokenizationTime(timeMs, languageId, sourceExtensionId, lineLength, isRandomSample);
66
+ that._host.$reportTokenizationTime(timeMs, languageId, sourceExtensionId, lineLength, isRandomSample);
61
67
  },
62
68
  }, data.languageId, data.encodedLanguageId, data.maxTokenizationLineLength)));
63
69
  }
64
- acceptModelChanged(controllerId, e) {
70
+ $acceptModelChanged(controllerId, e) {
65
71
  this._models.get(controllerId).onEvents(e);
66
72
  }
67
- retokenize(controllerId, startLineNumber, endLineNumberExclusive) {
73
+ $retokenize(controllerId, startLineNumber, endLineNumberExclusive) {
68
74
  this._models.get(controllerId).retokenize(startLineNumber, endLineNumberExclusive);
69
75
  }
70
- acceptModelLanguageChanged(controllerId, newLanguageId, newEncodedLanguageId) {
76
+ $acceptModelLanguageChanged(controllerId, newLanguageId, newEncodedLanguageId) {
71
77
  this._models.get(controllerId).onLanguageId(newLanguageId, newEncodedLanguageId);
72
78
  }
73
- acceptRemovedModel(controllerId) {
79
+ $acceptRemovedModel(controllerId) {
74
80
  const model = this._models.get(controllerId);
75
81
  if (model) {
76
82
  model.dispose();
77
83
  this._models.delete(controllerId);
78
84
  }
79
85
  }
80
- async acceptTheme(theme, colorMap) {
86
+ async $acceptTheme(theme, colorMap) {
81
87
  const grammarFactory = await this._grammarFactory;
82
88
  grammarFactory?.setTheme(theme, colorMap);
83
89
  }
84
- acceptMaxTokenizationLineLength(controllerId, value) {
90
+ $acceptMaxTokenizationLineLength(controllerId, value) {
85
91
  this._models.get(controllerId).acceptMaxTokenizationLineLength(value);
86
92
  }
87
93
  }
88
94
 
89
- export { TextMateTokenizationWorker };
95
+ export { TextMateTokenizationWorker, create };
@@ -0,0 +1,11 @@
1
+ class TextMateWorkerHost {
2
+ static { this.CHANNEL_NAME = 'textMateWorkerHost'; }
3
+ static getChannel(workerServer) {
4
+ return workerServer.getChannel(TextMateWorkerHost.CHANNEL_NAME);
5
+ }
6
+ static setChannel(workerClient, obj) {
7
+ workerClient.setChannel(TextMateWorkerHost.CHANNEL_NAME, obj);
8
+ }
9
+ }
10
+
11
+ export { TextMateWorkerHost };
@@ -1,10 +1,12 @@
1
1
  import { __decorate, __param } from 'vscode/external/tslib/tslib.es6.js';
2
+ import { resolveAmdNodeModulePath } from '../../../../amdX.js';
3
+ import 'vscode/vscode/vs/base/common/amd';
2
4
  import { createStyleSheet } from 'vscode/vscode/vs/base/browser/dom';
3
5
  import { equals } from 'vscode/vscode/vs/base/common/arrays';
4
6
  import { Color } from 'vscode/vscode/vs/base/common/color';
5
7
  import { onUnexpectedError } from 'vscode/vscode/vs/base/common/errors';
6
8
  import { Disposable, DisposableStore } from 'vscode/vscode/vs/base/common/lifecycle';
7
- import { FileAccess, nodeModulesAsarUnpackedPath, nodeModulesPath } from 'vscode/vscode/vs/base/common/network';
9
+ import { FileAccess, nodeModulesPath } from 'vscode/vscode/vs/base/common/network';
8
10
  import 'vscode/vscode/vs/base/common/observableInternal/derived';
9
11
  import 'vscode/vscode/vs/base/common/observableInternal/autorun';
10
12
  import { observableFromEvent } from 'vscode/vscode/vs/base/common/observableInternal/utils';
@@ -167,7 +169,7 @@ let TextMateTokenizationFeature = class TextMateTokenizationFeature extends Disp
167
169
  }
168
170
  startDebugMode(printFn, onStop) {
169
171
  if (this._debugMode) {
170
- this._notificationService.error(( localize(2770, "Already Logging.")));
172
+ this._notificationService.error(( localize(2771, "Already Logging.")));
171
173
  return;
172
174
  }
173
175
  this._debugModePrintFunc = printFn;
@@ -175,15 +177,15 @@ let TextMateTokenizationFeature = class TextMateTokenizationFeature extends Disp
175
177
  if (this._debugMode) {
176
178
  this._progressService.withProgress({
177
179
  location: ProgressLocation.Notification,
178
- buttons: [( localize(2771, "Stop"))]
180
+ buttons: [( localize(2772, "Stop"))]
179
181
  }, (progress) => {
180
182
  progress.report({
181
- message: ( localize(2772, "Preparing to log TM Grammar parsing. Press Stop when finished."))
183
+ message: ( localize(2773, "Preparing to log TM Grammar parsing. Press Stop when finished."))
182
184
  });
183
185
  return this._getVSCodeOniguruma().then((vscodeOniguruma) => {
184
186
  vscodeOniguruma.setDefaultDebugCall(true);
185
187
  progress.report({
186
- message: ( localize(2773, "Now logging TM Grammar parsing. Press Stop when finished."))
188
+ message: ( localize(2774, "Now logging TM Grammar parsing. Press Stop when finished."))
187
189
  });
188
190
  return (
189
191
  (new Promise((resolve, reject) => { }))
@@ -315,13 +317,12 @@ let TextMateTokenizationFeature = class TextMateTokenizationFeature extends Disp
315
317
  }
316
318
  async _loadVSCodeOnigurumaWASM() {
317
319
  if (isWeb) {
318
- const response = await fetch(( (( (FileAccess.asBrowserUri('vscode-oniguruma/../onig.wasm'))).toString(true))));
320
+ const response = await fetch(resolveAmdNodeModulePath('vscode-oniguruma', 'release/onig.wasm')
321
+ );
319
322
  return await response.arrayBuffer();
320
323
  }
321
324
  else {
322
- const response = await fetch(this._environmentService.isBuilt
323
- ? ( (( (FileAccess.asBrowserUri(`${nodeModulesAsarUnpackedPath}/vscode-oniguruma/release/onig.wasm`))).toString(true)))
324
- : ( (( (FileAccess.asBrowserUri(`${nodeModulesPath}/vscode-oniguruma/release/onig.wasm`))).toString(true))));
325
+ const response = await fetch(( (( (FileAccess.asBrowserUri(`${nodeModulesPath}/vscode-oniguruma/release/onig.wasm`))).toString(true))));
325
326
  return response;
326
327
  }
327
328
  }
@@ -392,7 +393,7 @@ function equalsTokenRules(a, b) {
392
393
  function validateGrammarExtensionPoint(extensionLocation, syntax, collector, _languageService) {
393
394
  if (syntax.language && ((typeof syntax.language !== 'string') || !_languageService.isRegisteredLanguageId(syntax.language))) {
394
395
  collector.error(( localize(
395
- 2774,
396
+ 2775,
396
397
  "Unknown language in `contributes.{0}.language`. Provided value: {1}",
397
398
  grammarsExtPoint.name,
398
399
  String(syntax.language)
@@ -401,7 +402,7 @@ function validateGrammarExtensionPoint(extensionLocation, syntax, collector, _la
401
402
  }
402
403
  if (!syntax.scopeName || (typeof syntax.scopeName !== 'string')) {
403
404
  collector.error(( localize(
404
- 2775,
405
+ 2776,
405
406
  "Expected string in `contributes.{0}.scopeName`. Provided value: {1}",
406
407
  grammarsExtPoint.name,
407
408
  String(syntax.scopeName)
@@ -410,7 +411,7 @@ function validateGrammarExtensionPoint(extensionLocation, syntax, collector, _la
410
411
  }
411
412
  if (!syntax.path || (typeof syntax.path !== 'string')) {
412
413
  collector.error(( localize(
413
- 2776,
414
+ 2777,
414
415
  "Expected string in `contributes.{0}.path`. Provided value: {1}",
415
416
  grammarsExtPoint.name,
416
417
  String(syntax.path)
@@ -419,7 +420,7 @@ function validateGrammarExtensionPoint(extensionLocation, syntax, collector, _la
419
420
  }
420
421
  if (syntax.injectTo && (!Array.isArray(syntax.injectTo) || ( (syntax.injectTo.some(scope => typeof scope !== 'string'))))) {
421
422
  collector.error(( localize(
422
- 2777,
423
+ 2778,
423
424
  "Invalid value in `contributes.{0}.injectTo`. Must be an array of language scope names. Provided value: {1}",
424
425
  grammarsExtPoint.name,
425
426
  JSON.stringify(syntax.injectTo)
@@ -428,7 +429,7 @@ function validateGrammarExtensionPoint(extensionLocation, syntax, collector, _la
428
429
  }
429
430
  if (syntax.embeddedLanguages && !isObject(syntax.embeddedLanguages)) {
430
431
  collector.error(( localize(
431
- 2778,
432
+ 2779,
432
433
  "Invalid value in `contributes.{0}.embeddedLanguages`. Must be an object map from scope name to language. Provided value: {1}",
433
434
  grammarsExtPoint.name,
434
435
  JSON.stringify(syntax.embeddedLanguages)
@@ -437,7 +438,7 @@ function validateGrammarExtensionPoint(extensionLocation, syntax, collector, _la
437
438
  }
438
439
  if (syntax.tokenTypes && !isObject(syntax.tokenTypes)) {
439
440
  collector.error(( localize(
440
- 2779,
441
+ 2780,
441
442
  "Invalid value in `contributes.{0}.tokenTypes`. Must be an object map from scope name to token type. Provided value: {1}",
442
443
  grammarsExtPoint.name,
443
444
  JSON.stringify(syntax.tokenTypes)
@@ -447,7 +448,7 @@ function validateGrammarExtensionPoint(extensionLocation, syntax, collector, _la
447
448
  const grammarLocation = joinPath(extensionLocation, syntax.path);
448
449
  if (!isEqualOrParent(grammarLocation, extensionLocation)) {
449
450
  collector.warn(( localize(
450
- 2780,
451
+ 2781,
451
452
  "Expected `contributes.{0}.path` ({1}) to be included inside extension's folder ({2}). This might make the extension non-portable.",
452
453
  grammarsExtPoint.name,
453
454
  grammarLocation.path,
@@ -9,26 +9,26 @@ const tokenClassificationRegistry = getTokenClassificationRegistry();
9
9
  const tokenTypeExtPoint = ExtensionsRegistry.registerExtensionPoint({
10
10
  extensionPoint: 'semanticTokenTypes',
11
11
  jsonSchema: {
12
- description: ( localize(2781, 'Contributes semantic token types.')),
12
+ description: ( localize(2782, 'Contributes semantic token types.')),
13
13
  type: 'array',
14
14
  items: {
15
15
  type: 'object',
16
16
  properties: {
17
17
  id: {
18
18
  type: 'string',
19
- description: ( localize(2782, 'The identifier of the semantic token type')),
19
+ description: ( localize(2783, 'The identifier of the semantic token type')),
20
20
  pattern: typeAndModifierIdPattern,
21
- patternErrorMessage: ( localize(2783, 'Identifiers should be in the form letterOrDigit[_-letterOrDigit]*')),
21
+ patternErrorMessage: ( localize(2784, 'Identifiers should be in the form letterOrDigit[_-letterOrDigit]*')),
22
22
  },
23
23
  superType: {
24
24
  type: 'string',
25
- description: ( localize(2784, 'The super type of the semantic token type')),
25
+ description: ( localize(2785, 'The super type of the semantic token type')),
26
26
  pattern: typeAndModifierIdPattern,
27
- patternErrorMessage: ( localize(2785, 'Super types should be in the form letterOrDigit[_-letterOrDigit]*')),
27
+ patternErrorMessage: ( localize(2786, 'Super types should be in the form letterOrDigit[_-letterOrDigit]*')),
28
28
  },
29
29
  description: {
30
30
  type: 'string',
31
- description: ( localize(2786, 'The description of the semantic token type')),
31
+ description: ( localize(2787, 'The description of the semantic token type')),
32
32
  }
33
33
  }
34
34
  }
@@ -37,20 +37,20 @@ const tokenTypeExtPoint = ExtensionsRegistry.registerExtensionPoint({
37
37
  const tokenModifierExtPoint = ExtensionsRegistry.registerExtensionPoint({
38
38
  extensionPoint: 'semanticTokenModifiers',
39
39
  jsonSchema: {
40
- description: ( localize(2787, 'Contributes semantic token modifiers.')),
40
+ description: ( localize(2788, 'Contributes semantic token modifiers.')),
41
41
  type: 'array',
42
42
  items: {
43
43
  type: 'object',
44
44
  properties: {
45
45
  id: {
46
46
  type: 'string',
47
- description: ( localize(2788, 'The identifier of the semantic token modifier')),
47
+ description: ( localize(2789, 'The identifier of the semantic token modifier')),
48
48
  pattern: typeAndModifierIdPattern,
49
- patternErrorMessage: ( localize(2789, 'Identifiers should be in the form letterOrDigit[_-letterOrDigit]*'))
49
+ patternErrorMessage: ( localize(2790, 'Identifiers should be in the form letterOrDigit[_-letterOrDigit]*'))
50
50
  },
51
51
  description: {
52
52
  type: 'string',
53
- description: ( localize(2790, 'The description of the semantic token modifier'))
53
+ description: ( localize(2791, 'The description of the semantic token modifier'))
54
54
  }
55
55
  }
56
56
  }
@@ -59,18 +59,18 @@ const tokenModifierExtPoint = ExtensionsRegistry.registerExtensionPoint({
59
59
  const tokenStyleDefaultsExtPoint = ExtensionsRegistry.registerExtensionPoint({
60
60
  extensionPoint: 'semanticTokenScopes',
61
61
  jsonSchema: {
62
- description: ( localize(2791, 'Contributes semantic token scope maps.')),
62
+ description: ( localize(2792, 'Contributes semantic token scope maps.')),
63
63
  type: 'array',
64
64
  items: {
65
65
  type: 'object',
66
66
  properties: {
67
67
  language: {
68
- description: ( localize(2792, 'Lists the languge for which the defaults are.')),
68
+ description: ( localize(2793, 'Lists the languge for which the defaults are.')),
69
69
  type: 'string'
70
70
  },
71
71
  scopes: {
72
72
  description: ( localize(
73
- 2793,
73
+ 2794,
74
74
  'Maps a semantic token (described by semantic token selector) to one or more textMate scopes used to represent that token.'
75
75
  )),
76
76
  type: 'object',
@@ -90,7 +90,7 @@ class TokenClassificationExtensionPoints {
90
90
  function validateTypeOrModifier(contribution, extensionPoint, collector) {
91
91
  if (typeof contribution.id !== 'string' || contribution.id.length === 0) {
92
92
  collector.error(( localize(
93
- 2794,
93
+ 2795,
94
94
  "'configuration.{0}.id' must be defined and can not be empty",
95
95
  extensionPoint
96
96
  )));
@@ -98,7 +98,7 @@ class TokenClassificationExtensionPoints {
98
98
  }
99
99
  if (!contribution.id.match(typeAndModifierIdPattern)) {
100
100
  collector.error(( localize(
101
- 2795,
101
+ 2796,
102
102
  "'configuration.{0}.id' must follow the pattern letterOrDigit[-_letterOrDigit]*",
103
103
  extensionPoint
104
104
  )));
@@ -107,7 +107,7 @@ class TokenClassificationExtensionPoints {
107
107
  const superType = contribution.superType;
108
108
  if (superType && !superType.match(typeAndModifierIdPattern)) {
109
109
  collector.error(( localize(
110
- 2796,
110
+ 2797,
111
111
  "'configuration.{0}.superType' must follow the pattern letterOrDigit[-_letterOrDigit]*",
112
112
  extensionPoint
113
113
  )));
@@ -115,7 +115,7 @@ class TokenClassificationExtensionPoints {
115
115
  }
116
116
  if (typeof contribution.description !== 'string' || contribution.id.length === 0) {
117
117
  collector.error(( localize(
118
- 2797,
118
+ 2798,
119
119
  "'configuration.{0}.description' must be defined and can not be empty",
120
120
  extensionPoint
121
121
  )));
@@ -128,7 +128,7 @@ class TokenClassificationExtensionPoints {
128
128
  const extensionValue = extension.value;
129
129
  const collector = extension.collector;
130
130
  if (!extensionValue || !Array.isArray(extensionValue)) {
131
- collector.error(( localize(2798, "'configuration.semanticTokenType' must be an array")));
131
+ collector.error(( localize(2799, "'configuration.semanticTokenType' must be an array")));
132
132
  return;
133
133
  }
134
134
  for (const contribution of extensionValue) {
@@ -149,7 +149,7 @@ class TokenClassificationExtensionPoints {
149
149
  const extensionValue = extension.value;
150
150
  const collector = extension.collector;
151
151
  if (!extensionValue || !Array.isArray(extensionValue)) {
152
- collector.error(( localize(2799, "'configuration.semanticTokenModifier' must be an array")));
152
+ collector.error(( localize(2800, "'configuration.semanticTokenModifier' must be an array")));
153
153
  return;
154
154
  }
155
155
  for (const contribution of extensionValue) {
@@ -170,17 +170,17 @@ class TokenClassificationExtensionPoints {
170
170
  const extensionValue = extension.value;
171
171
  const collector = extension.collector;
172
172
  if (!extensionValue || !Array.isArray(extensionValue)) {
173
- collector.error(( localize(2800, "'configuration.semanticTokenScopes' must be an array")));
173
+ collector.error(( localize(2801, "'configuration.semanticTokenScopes' must be an array")));
174
174
  return;
175
175
  }
176
176
  for (const contribution of extensionValue) {
177
177
  if (contribution.language && typeof contribution.language !== 'string') {
178
- collector.error(( localize(2801, "'configuration.semanticTokenScopes.language' must be a string")));
178
+ collector.error(( localize(2802, "'configuration.semanticTokenScopes.language' must be a string")));
179
179
  continue;
180
180
  }
181
181
  if (!contribution.scopes || typeof contribution.scopes !== 'object') {
182
182
  collector.error(( localize(
183
- 2802,
183
+ 2803,
184
184
  "'configuration.semanticTokenScopes.scopes' must be defined as an object"
185
185
  )));
186
186
  continue;
@@ -189,7 +189,7 @@ class TokenClassificationExtensionPoints {
189
189
  const tmScopes = contribution.scopes[selectorString];
190
190
  if (!Array.isArray(tmScopes) || ( (tmScopes.some(l => typeof l !== 'string')))) {
191
191
  collector.error(( localize(
192
- 2803,
192
+ 2804,
193
193
  "'configuration.semanticTokenScopes.scopes' values must be an array of strings"
194
194
  )));
195
195
  continue;
@@ -200,7 +200,7 @@ class TokenClassificationExtensionPoints {
200
200
  }
201
201
  catch (e) {
202
202
  collector.error(( localize(
203
- 2804,
203
+ 2805,
204
204
  "configuration.semanticTokenScopes.scopes': Problems parsing selector {0}.",
205
205
  selectorString
206
206
  )));
@@ -1,4 +1,9 @@
1
- import { TextMateTokenizationWorker } from '../vscode/src/vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateTokenizationWorker.worker.js';
2
- import { initialize } from 'vscode/vscode/vs/editor/editor.worker';
1
+ import { create } from '../vscode/src/vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateTokenizationWorker.worker.js';
2
+ import { SimpleWorkerServer } from 'vscode/vscode/vs/base/common/worker/simpleWorker';
3
3
 
4
- initialize((ctx, createData) => new TextMateTokenizationWorker(ctx, createData));
4
+ const simpleWorker = new SimpleWorkerServer((msg) => {
5
+ globalThis.postMessage(msg);
6
+ }, create);
7
+ globalThis.onmessage = (e) => {
8
+ simpleWorker.onmessage(e.data);
9
+ };