@codingame/monaco-vscode-textmate-service-override 1.82.4 → 1.82.5-next.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (21) hide show
  1. package/external/tslib/tslib.es6.js +11 -0
  2. package/external/vscode-textmate/release/main.d.ts +64 -0
  3. package/index.d.ts +2 -2
  4. package/index.js +2 -2
  5. package/package.json +2 -2
  6. package/textmate.d.ts +5 -0
  7. package/textmate.js +43 -0
  8. package/vscode/src/vs/workbench/services/textMate/browser/arrayOperation.js +64 -0
  9. package/vscode/src/vs/workbench/services/textMate/browser/backgroundTokenization/textMateWorkerTokenizerController.js +159 -0
  10. package/vscode/src/vs/workbench/services/textMate/browser/backgroundTokenization/threadedBackgroundTokenizerFactory.js +202 -0
  11. package/vscode/src/vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateTokenizationWorker.worker.js +89 -0
  12. package/vscode/src/vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateWorkerTokenizer.js +142 -0
  13. package/vscode/src/vs/workbench/services/textMate/browser/textMateTokenizationFeature.d.ts +11 -0
  14. package/vscode/src/vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl.js +461 -0
  15. package/vscode/src/vs/workbench/services/textMate/browser/tokenizationSupport/textMateTokenizationSupport.js +74 -0
  16. package/vscode/src/vs/workbench/services/textMate/browser/tokenizationSupport/tokenizationSupportWithLineLimit.js +38 -0
  17. package/vscode/src/vs/workbench/services/textMate/common/TMGrammarFactory.js +120 -0
  18. package/vscode/src/vs/workbench/services/textMate/common/TMGrammars.js +95 -0
  19. package/vscode/src/vs/workbench/services/textMate/common/TMScopeRegistry.js +26 -0
  20. package/worker.js +1 -1
  21. package/workers/textmate.worker.js +4 -0
@@ -0,0 +1,89 @@
1
+ import { URI } from 'monaco-editor/esm/vs/base/common/uri.js';
2
+ import { TMGrammarFactory } from '../../../common/TMGrammarFactory.js';
3
+ import { TextMateWorkerTokenizer } from './textMateWorkerTokenizer.js';
4
+ import * as vscodeTextmate from 'vscode-textmate';
5
+ import * as vscodeOniguruma from 'vscode-oniguruma';
6
+
7
+ class TextMateTokenizationWorker {
8
+ constructor(ctx, _createData) {
9
+ this._createData = _createData;
10
+ this._models = ( new Map());
11
+ this._grammarCache = [];
12
+ this._host = ctx.host;
13
+ const grammarDefinitions = ( _createData.grammarDefinitions.map((def) => {
14
+ return {
15
+ location: URI.revive(def.location),
16
+ language: def.language,
17
+ scopeName: def.scopeName,
18
+ embeddedLanguages: def.embeddedLanguages,
19
+ tokenTypes: def.tokenTypes,
20
+ injectTo: def.injectTo,
21
+ balancedBracketSelectors: def.balancedBracketSelectors,
22
+ unbalancedBracketSelectors: def.unbalancedBracketSelectors,
23
+ sourceExtensionId: def.sourceExtensionId,
24
+ };
25
+ }));
26
+ this._grammarFactory = this._loadTMGrammarFactory(grammarDefinitions);
27
+ }
28
+ async _loadTMGrammarFactory(grammarDefinitions) {
29
+ const response = await fetch(this._createData.onigurumaWASMUri);
30
+ const bytes = await response.arrayBuffer();
31
+ await vscodeOniguruma.loadWASM(bytes);
32
+ const onigLib = Promise.resolve({
33
+ createOnigScanner: (sources) => vscodeOniguruma.createOnigScanner(sources),
34
+ createOnigString: (str) => vscodeOniguruma.createOnigString(str)
35
+ });
36
+ return ( new TMGrammarFactory({
37
+ logTrace: (msg) => { },
38
+ logError: (msg, err) => console.error(msg, err),
39
+ readFile: (resource) => this._host.readFile(resource)
40
+ }, grammarDefinitions, vscodeTextmate, onigLib));
41
+ }
42
+ acceptNewModel(data) {
43
+ const uri = URI.revive(data.uri);
44
+ const that = this;
45
+ this._models.set(data.controllerId, ( new TextMateWorkerTokenizer(uri, data.lines, data.EOL, data.versionId, {
46
+ async getOrCreateGrammar(languageId, encodedLanguageId) {
47
+ const grammarFactory = await that._grammarFactory;
48
+ if (!grammarFactory) {
49
+ return Promise.resolve(null);
50
+ }
51
+ if (!that._grammarCache[encodedLanguageId]) {
52
+ that._grammarCache[encodedLanguageId] = grammarFactory.createGrammar(languageId, encodedLanguageId);
53
+ }
54
+ return that._grammarCache[encodedLanguageId];
55
+ },
56
+ setTokensAndStates(versionId, tokens, stateDeltas) {
57
+ that._host.setTokensAndStates(data.controllerId, versionId, tokens, stateDeltas);
58
+ },
59
+ reportTokenizationTime(timeMs, languageId, sourceExtensionId, lineLength, isRandomSample) {
60
+ that._host.reportTokenizationTime(timeMs, languageId, sourceExtensionId, lineLength, isRandomSample);
61
+ },
62
+ }, data.languageId, data.encodedLanguageId, data.maxTokenizationLineLength)));
63
+ }
64
+ acceptModelChanged(controllerId, e) {
65
+ this._models.get(controllerId).onEvents(e);
66
+ }
67
+ retokenize(controllerId, startLineNumber, endLineNumberExclusive) {
68
+ this._models.get(controllerId).retokenize(startLineNumber, endLineNumberExclusive);
69
+ }
70
+ acceptModelLanguageChanged(controllerId, newLanguageId, newEncodedLanguageId) {
71
+ this._models.get(controllerId).onLanguageId(newLanguageId, newEncodedLanguageId);
72
+ }
73
+ acceptRemovedModel(controllerId) {
74
+ const model = this._models.get(controllerId);
75
+ if (model) {
76
+ model.dispose();
77
+ this._models.delete(controllerId);
78
+ }
79
+ }
80
+ async acceptTheme(theme, colorMap) {
81
+ const grammarFactory = await this._grammarFactory;
82
+ grammarFactory?.setTheme(theme, colorMap);
83
+ }
84
+ acceptMaxTokenizationLineLength(controllerId, value) {
85
+ this._models.get(controllerId).acceptMaxTokenizationLineLength(value);
86
+ }
87
+ }
88
+
89
+ export { TextMateTokenizationWorker };
@@ -0,0 +1,142 @@
1
+ import { RunOnceScheduler } from 'monaco-editor/esm/vs/base/common/async.js';
2
+ import { observableValue } from 'monaco-editor/esm/vs/base/common/observable.js';
3
+ import { setTimeout0 } from 'monaco-editor/esm/vs/base/common/platform.js';
4
+ import { LineRange } from 'monaco-editor/esm/vs/editor/common/core/lineRange.js';
5
+ import { MirrorTextModel } from 'monaco-editor/esm/vs/editor/common/model/mirrorTextModel.js';
6
+ import { TokenizerWithStateStore } from 'monaco-editor/esm/vs/editor/common/model/textModelTokens.js';
7
+ import { ContiguousMultilineTokensBuilder } from 'monaco-editor/esm/vs/editor/common/tokens/contiguousMultilineTokensBuilder.js';
8
+ import { LineTokens } from 'monaco-editor/esm/vs/editor/common/tokens/lineTokens.js';
9
+ import { TextMateTokenizationSupport } from '../../tokenizationSupport/textMateTokenizationSupport.js';
10
+ import { TokenizationSupportWithLineLimit } from '../../tokenizationSupport/tokenizationSupportWithLineLimit.js';
11
+ import { diffStateStacksRefEq } from 'vscode-textmate';
12
+
13
+ class TextMateWorkerTokenizer extends MirrorTextModel {
14
+ constructor(uri, lines, eol, versionId, _host, _languageId, _encodedLanguageId, maxTokenizationLineLength) {
15
+ super(uri, lines, eol, versionId);
16
+ this._host = _host;
17
+ this._languageId = _languageId;
18
+ this._encodedLanguageId = _encodedLanguageId;
19
+ this._tokenizerWithStateStore = null;
20
+ this._isDisposed = false;
21
+ this._maxTokenizationLineLength = observableValue('_maxTokenizationLineLength', -1);
22
+ this._tokenizeDebouncer = ( new RunOnceScheduler(() => this._tokenize(), 10));
23
+ this._maxTokenizationLineLength.set(maxTokenizationLineLength, undefined);
24
+ this._resetTokenization();
25
+ }
26
+ dispose() {
27
+ this._isDisposed = true;
28
+ super.dispose();
29
+ }
30
+ onLanguageId(languageId, encodedLanguageId) {
31
+ this._languageId = languageId;
32
+ this._encodedLanguageId = encodedLanguageId;
33
+ this._resetTokenization();
34
+ }
35
+ onEvents(e) {
36
+ super.onEvents(e);
37
+ this._tokenizerWithStateStore?.store.acceptChanges(e.changes);
38
+ this._tokenizeDebouncer.schedule();
39
+ }
40
+ acceptMaxTokenizationLineLength(maxTokenizationLineLength) {
41
+ this._maxTokenizationLineLength.set(maxTokenizationLineLength, undefined);
42
+ }
43
+ retokenize(startLineNumber, endLineNumberExclusive) {
44
+ if (this._tokenizerWithStateStore) {
45
+ this._tokenizerWithStateStore.store.invalidateEndStateRange(( new LineRange(startLineNumber, endLineNumberExclusive)));
46
+ this._tokenizeDebouncer.schedule();
47
+ }
48
+ }
49
+ async _resetTokenization() {
50
+ this._tokenizerWithStateStore = null;
51
+ const languageId = this._languageId;
52
+ const encodedLanguageId = this._encodedLanguageId;
53
+ const r = await this._host.getOrCreateGrammar(languageId, encodedLanguageId);
54
+ if (this._isDisposed || languageId !== this._languageId || encodedLanguageId !== this._encodedLanguageId || !r) {
55
+ return;
56
+ }
57
+ if (r.grammar) {
58
+ const tokenizationSupport = ( new TokenizationSupportWithLineLimit(this._encodedLanguageId, ( new TextMateTokenizationSupport(
59
+ r.grammar,
60
+ r.initialState,
61
+ false,
62
+ undefined,
63
+ () => false,
64
+ (timeMs, lineLength, isRandomSample) => {
65
+ this._host.reportTokenizationTime(timeMs, languageId, r.sourceExtensionId, lineLength, isRandomSample);
66
+ },
67
+ false
68
+ )), this._maxTokenizationLineLength));
69
+ this._tokenizerWithStateStore = ( new TokenizerWithStateStore(this._lines.length, tokenizationSupport));
70
+ }
71
+ else {
72
+ this._tokenizerWithStateStore = null;
73
+ }
74
+ this._tokenize();
75
+ }
76
+ async _tokenize() {
77
+ if (this._isDisposed || !this._tokenizerWithStateStore) {
78
+ return;
79
+ }
80
+ if (!this._diffStateStacksRefEqFn) {
81
+ this._diffStateStacksRefEqFn = diffStateStacksRefEq;
82
+ }
83
+ const startTime = ( new Date()).getTime();
84
+ while (true) {
85
+ let tokenizedLines = 0;
86
+ const tokenBuilder = ( new ContiguousMultilineTokensBuilder());
87
+ const stateDeltaBuilder = ( new StateDeltaBuilder());
88
+ while (true) {
89
+ const lineToTokenize = this._tokenizerWithStateStore.getFirstInvalidLine();
90
+ if (lineToTokenize === null || tokenizedLines > 200) {
91
+ break;
92
+ }
93
+ tokenizedLines++;
94
+ const text = this._lines[lineToTokenize.lineNumber - 1];
95
+ const r = this._tokenizerWithStateStore.tokenizationSupport.tokenizeEncoded(text, true, lineToTokenize.startState);
96
+ if (this._tokenizerWithStateStore.store.setEndState(lineToTokenize.lineNumber, r.endState)) {
97
+ const delta = this._diffStateStacksRefEqFn(lineToTokenize.startState, r.endState);
98
+ stateDeltaBuilder.setState(lineToTokenize.lineNumber, delta);
99
+ }
100
+ else {
101
+ stateDeltaBuilder.setState(lineToTokenize.lineNumber, null);
102
+ }
103
+ LineTokens.convertToEndOffset(r.tokens, text.length);
104
+ tokenBuilder.add(lineToTokenize.lineNumber, r.tokens);
105
+ const deltaMs = ( new Date()).getTime() - startTime;
106
+ if (deltaMs > 20) {
107
+ break;
108
+ }
109
+ }
110
+ if (tokenizedLines === 0) {
111
+ break;
112
+ }
113
+ const stateDeltas = stateDeltaBuilder.getStateDeltas();
114
+ this._host.setTokensAndStates(this._versionId, tokenBuilder.serialize(), stateDeltas);
115
+ const deltaMs = ( new Date()).getTime() - startTime;
116
+ if (deltaMs > 20) {
117
+ setTimeout0(() => this._tokenize());
118
+ return;
119
+ }
120
+ }
121
+ }
122
+ }
123
+ class StateDeltaBuilder {
124
+ constructor() {
125
+ this._lastStartLineNumber = -1;
126
+ this._stateDeltas = [];
127
+ }
128
+ setState(lineNumber, stackDiff) {
129
+ if (lineNumber === this._lastStartLineNumber + 1) {
130
+ this._stateDeltas[this._stateDeltas.length - 1].stateDeltas.push(stackDiff);
131
+ }
132
+ else {
133
+ this._stateDeltas.push({ startLineNumber: lineNumber, stateDeltas: [stackDiff] });
134
+ }
135
+ this._lastStartLineNumber = lineNumber;
136
+ }
137
+ getStateDeltas() {
138
+ return this._stateDeltas;
139
+ }
140
+ }
141
+
142
+ export { TextMateWorkerTokenizer };
@@ -0,0 +1,11 @@
1
+ import { ServiceIdentifier } from 'vscode/vscode/vs/platform/instantiation/common/instantiation';
2
+ import { IGrammar } from '../../../../../../../external/vscode-textmate/release/main.js';
3
+
4
+ declare const ITextMateTokenizationService: ServiceIdentifier<ITextMateTokenizationService>;
5
+ interface ITextMateTokenizationService {
6
+ readonly _serviceBrand: undefined;
7
+ createTokenizer(languageId: string): Promise<IGrammar | null>;
8
+ startDebugMode(printFn: (str: string) => void, onStop: () => void): void;
9
+ }
10
+
11
+ export { ITextMateTokenizationService };