@codingame/monaco-vscode-textmate-service-override 1.82.4 → 1.82.5-next.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/external/tslib/tslib.es6.js +11 -0
- package/external/vscode-textmate/release/main.d.ts +64 -0
- package/index.d.ts +2 -2
- package/index.js +2 -2
- package/package.json +2 -2
- package/textmate.d.ts +5 -0
- package/textmate.js +43 -0
- package/vscode/src/vs/workbench/services/textMate/browser/arrayOperation.js +64 -0
- package/vscode/src/vs/workbench/services/textMate/browser/backgroundTokenization/textMateWorkerTokenizerController.js +159 -0
- package/vscode/src/vs/workbench/services/textMate/browser/backgroundTokenization/threadedBackgroundTokenizerFactory.js +202 -0
- package/vscode/src/vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateTokenizationWorker.worker.js +89 -0
- package/vscode/src/vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateWorkerTokenizer.js +142 -0
- package/vscode/src/vs/workbench/services/textMate/browser/textMateTokenizationFeature.d.ts +11 -0
- package/vscode/src/vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl.js +461 -0
- package/vscode/src/vs/workbench/services/textMate/browser/tokenizationSupport/textMateTokenizationSupport.js +74 -0
- package/vscode/src/vs/workbench/services/textMate/browser/tokenizationSupport/tokenizationSupportWithLineLimit.js +38 -0
- package/vscode/src/vs/workbench/services/textMate/common/TMGrammarFactory.js +120 -0
- package/vscode/src/vs/workbench/services/textMate/common/TMGrammars.js +95 -0
- package/vscode/src/vs/workbench/services/textMate/common/TMScopeRegistry.js +26 -0
- package/worker.js +1 -1
- package/workers/textmate.worker.js +4 -0
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
function __decorate(decorators, target, key, desc) {
|
|
2
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
3
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
4
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
5
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
6
|
+
}
|
|
7
|
+
function __param(paramIndex, decorator) {
|
|
8
|
+
return function (target, key) { decorator(target, key, paramIndex); }
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export { __decorate, __param };
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A grammar
|
|
3
|
+
*/
|
|
4
|
+
interface IGrammar {
|
|
5
|
+
/**
|
|
6
|
+
* Tokenize `lineText` using previous line state `prevState`.
|
|
7
|
+
*/
|
|
8
|
+
tokenizeLine(lineText: string, prevState: StateStack | null, timeLimit?: number): ITokenizeLineResult;
|
|
9
|
+
/**
|
|
10
|
+
* Tokenize `lineText` using previous line state `prevState`.
|
|
11
|
+
* The result contains the tokens in binary format, resolved with the following information:
|
|
12
|
+
* - language
|
|
13
|
+
* - token type (regex, string, comment, other)
|
|
14
|
+
* - font style
|
|
15
|
+
* - foreground color
|
|
16
|
+
* - background color
|
|
17
|
+
* e.g. for getting the languageId: `(metadata & MetadataConsts.LANGUAGEID_MASK) >>> MetadataConsts.LANGUAGEID_OFFSET`
|
|
18
|
+
*/
|
|
19
|
+
tokenizeLine2(lineText: string, prevState: StateStack | null, timeLimit?: number): ITokenizeLineResult2;
|
|
20
|
+
}
|
|
21
|
+
interface ITokenizeLineResult {
|
|
22
|
+
readonly tokens: IToken[];
|
|
23
|
+
/**
|
|
24
|
+
* The `prevState` to be passed on to the next line tokenization.
|
|
25
|
+
*/
|
|
26
|
+
readonly ruleStack: StateStack;
|
|
27
|
+
/**
|
|
28
|
+
* Did tokenization stop early due to reaching the time limit.
|
|
29
|
+
*/
|
|
30
|
+
readonly stoppedEarly: boolean;
|
|
31
|
+
}
|
|
32
|
+
interface ITokenizeLineResult2 {
|
|
33
|
+
/**
|
|
34
|
+
* The tokens in binary format. Each token occupies two array indices. For token i:
|
|
35
|
+
* - at offset 2*i => startIndex
|
|
36
|
+
* - at offset 2*i + 1 => metadata
|
|
37
|
+
*
|
|
38
|
+
*/
|
|
39
|
+
readonly tokens: Uint32Array;
|
|
40
|
+
/**
|
|
41
|
+
* The `prevState` to be passed on to the next line tokenization.
|
|
42
|
+
*/
|
|
43
|
+
readonly ruleStack: StateStack;
|
|
44
|
+
/**
|
|
45
|
+
* Did tokenization stop early due to reaching the time limit.
|
|
46
|
+
*/
|
|
47
|
+
readonly stoppedEarly: boolean;
|
|
48
|
+
}
|
|
49
|
+
interface IToken {
|
|
50
|
+
startIndex: number;
|
|
51
|
+
readonly endIndex: number;
|
|
52
|
+
readonly scopes: string[];
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* **IMPORTANT** - Immutable!
|
|
56
|
+
*/
|
|
57
|
+
interface StateStack {
|
|
58
|
+
_stackElementBrand: void;
|
|
59
|
+
readonly depth: number;
|
|
60
|
+
clone(): StateStack;
|
|
61
|
+
equals(other: StateStack): boolean;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export type { IGrammar, IToken, ITokenizeLineResult, ITokenizeLineResult2, StateStack };
|
package/index.d.ts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export { default } from '
|
|
2
|
-
export
|
|
1
|
+
export { default } from './textmate.js';
|
|
2
|
+
export { ITextMateTokenizationService } from './vscode/src/vs/workbench/services/textMate/browser/textMateTokenizationFeature.js';
|
package/index.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export { default } from '
|
|
2
|
-
export
|
|
1
|
+
export { default } from './textmate.js';
|
|
2
|
+
export { ITextMateTokenizationService } from 'vscode/vscode/vs/workbench/services/textMate/browser/textMateTokenizationFeature';
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@codingame/monaco-vscode-textmate-service-override",
|
|
3
|
-
"version": "1.82.
|
|
3
|
+
"version": "1.82.5-next.1",
|
|
4
4
|
"keywords": [],
|
|
5
5
|
"author": {
|
|
6
6
|
"name": "CodinGame",
|
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
"module": "index.js",
|
|
19
19
|
"types": "index.d.ts",
|
|
20
20
|
"dependencies": {
|
|
21
|
-
"vscode": "npm:@codingame/monaco-vscode-api@1.82.
|
|
21
|
+
"vscode": "npm:@codingame/monaco-vscode-api@1.82.5-next.1",
|
|
22
22
|
"monaco-editor": "0.43.0",
|
|
23
23
|
"vscode-oniguruma": "^2.0.0",
|
|
24
24
|
"vscode-textmate": "^9.0.0"
|
package/textmate.d.ts
ADDED
package/textmate.js
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { __decorate, __param } from './external/tslib/tslib.es6.js';
|
|
2
|
+
import { StandaloneServices } from 'monaco-editor/esm/vs/editor/standalone/browser/standaloneServices.js';
|
|
3
|
+
import { ITextMateTokenizationService } from 'vscode/vscode/vs/workbench/services/textMate/browser/textMateTokenizationFeature';
|
|
4
|
+
export { ITextMateTokenizationService } from 'vscode/vscode/vs/workbench/services/textMate/browser/textMateTokenizationFeature';
|
|
5
|
+
import { SyncDescriptor } from 'monaco-editor/esm/vs/platform/instantiation/common/descriptors.js';
|
|
6
|
+
import { TextMateTokenizationFeature } from './vscode/src/vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl.js';
|
|
7
|
+
import { TokenClassificationExtensionPoints } from 'vscode/vscode/vs/workbench/services/themes/common/tokenClassificationExtensionPoint';
|
|
8
|
+
import { Extensions } from 'vscode/vscode/vs/workbench/common/contributions';
|
|
9
|
+
import { Registry } from 'monaco-editor/esm/vs/platform/registry/common/platform.js';
|
|
10
|
+
import { ILifecycleService } from 'vscode/vscode/vs/workbench/services/lifecycle/common/lifecycle';
|
|
11
|
+
import { IInstantiationService } from 'monaco-editor/esm/vs/platform/instantiation/common/instantiation.js';
|
|
12
|
+
import getServiceOverride$1 from 'vscode/service-override/files';
|
|
13
|
+
import { registerServiceInitializeParticipant } from 'vscode/lifecycle';
|
|
14
|
+
import { registerAssets } from 'vscode/assets';
|
|
15
|
+
|
|
16
|
+
const _onigWasm = new URL('vscode-oniguruma/release/onig.wasm', import.meta.url).href;
|
|
17
|
+
registerAssets({
|
|
18
|
+
'vscode-oniguruma/../onig.wasm': _onigWasm,
|
|
19
|
+
'vs/../../node_modules/vscode-oniguruma/release/onig.wasm': _onigWasm
|
|
20
|
+
});
|
|
21
|
+
registerServiceInitializeParticipant(async (accessor) => {
|
|
22
|
+
void accessor.get(ILifecycleService).when(2 ).then(() => {
|
|
23
|
+
StandaloneServices.get(ITextMateTokenizationService);
|
|
24
|
+
});
|
|
25
|
+
});
|
|
26
|
+
let ExtensionPoints = class ExtensionPoints {
|
|
27
|
+
constructor(instantiationService) {
|
|
28
|
+
this.instantiationService = instantiationService;
|
|
29
|
+
this.instantiationService.createInstance(TokenClassificationExtensionPoints);
|
|
30
|
+
}
|
|
31
|
+
};
|
|
32
|
+
ExtensionPoints = __decorate([
|
|
33
|
+
( __param(0, IInstantiationService))
|
|
34
|
+
], ExtensionPoints);
|
|
35
|
+
( Registry.as(Extensions.Workbench)).registerWorkbenchContribution(ExtensionPoints, 1 );
|
|
36
|
+
function getServiceOverride() {
|
|
37
|
+
return {
|
|
38
|
+
...getServiceOverride$1(),
|
|
39
|
+
[( ITextMateTokenizationService.toString())]: new SyncDescriptor(TextMateTokenizationFeature, [], false)
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export { getServiceOverride as default };
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import { compareBy, numberComparator } from 'monaco-editor/esm/vs/base/common/arrays.js';
|
|
2
|
+
|
|
3
|
+
class ArrayEdit {
|
|
4
|
+
constructor(
|
|
5
|
+
edits) {
|
|
6
|
+
this.edits = edits.slice().sort(compareBy(c => c.offset, numberComparator));
|
|
7
|
+
}
|
|
8
|
+
applyToArray(array) {
|
|
9
|
+
for (let i = this.edits.length - 1; i >= 0; i--) {
|
|
10
|
+
const c = this.edits[i];
|
|
11
|
+
array.splice(c.offset, c.length, ...( new Array(c.newLength)));
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
class SingleArrayEdit {
|
|
16
|
+
constructor(offset, length, newLength) {
|
|
17
|
+
this.offset = offset;
|
|
18
|
+
this.length = length;
|
|
19
|
+
this.newLength = newLength;
|
|
20
|
+
}
|
|
21
|
+
toString() {
|
|
22
|
+
return `[${this.offset}, +${this.length}) -> +${this.newLength}}`;
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
class MonotonousIndexTransformer {
|
|
26
|
+
static fromMany(transformations) {
|
|
27
|
+
const transformers = ( transformations.map(t => ( new MonotonousIndexTransformer(t))));
|
|
28
|
+
return ( new CombinedIndexTransformer(transformers));
|
|
29
|
+
}
|
|
30
|
+
constructor(transformation) {
|
|
31
|
+
this.transformation = transformation;
|
|
32
|
+
this.idx = 0;
|
|
33
|
+
this.offset = 0;
|
|
34
|
+
}
|
|
35
|
+
transform(index) {
|
|
36
|
+
let nextChange = this.transformation.edits[this.idx];
|
|
37
|
+
while (nextChange && nextChange.offset + nextChange.length <= index) {
|
|
38
|
+
this.offset += nextChange.newLength - nextChange.length;
|
|
39
|
+
this.idx++;
|
|
40
|
+
nextChange = this.transformation.edits[this.idx];
|
|
41
|
+
}
|
|
42
|
+
if (nextChange && nextChange.offset <= index) {
|
|
43
|
+
return undefined;
|
|
44
|
+
}
|
|
45
|
+
return index + this.offset;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
class CombinedIndexTransformer {
|
|
49
|
+
constructor(transformers) {
|
|
50
|
+
this.transformers = transformers;
|
|
51
|
+
}
|
|
52
|
+
transform(index) {
|
|
53
|
+
for (const transformer of this.transformers) {
|
|
54
|
+
const result = transformer.transform(index);
|
|
55
|
+
if (result === undefined) {
|
|
56
|
+
return undefined;
|
|
57
|
+
}
|
|
58
|
+
index = result;
|
|
59
|
+
}
|
|
60
|
+
return index;
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export { ArrayEdit, CombinedIndexTransformer, MonotonousIndexTransformer, SingleArrayEdit };
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
import { Disposable } from 'monaco-editor/esm/vs/base/common/lifecycle.js';
|
|
2
|
+
import { keepAlive, autorun, observableFromEvent } from 'monaco-editor/esm/vs/base/common/observable.js';
|
|
3
|
+
import { countEOL } from 'monaco-editor/esm/vs/editor/common/core/eolCounter.js';
|
|
4
|
+
import { LineRange } from 'monaco-editor/esm/vs/editor/common/core/lineRange.js';
|
|
5
|
+
import { Range } from 'monaco-editor/esm/vs/editor/common/core/range.js';
|
|
6
|
+
import { TokenizationStateStore } from 'monaco-editor/esm/vs/editor/common/model/textModelTokens.js';
|
|
7
|
+
import { ContiguousMultilineTokensBuilder } from 'monaco-editor/esm/vs/editor/common/tokens/contiguousMultilineTokensBuilder.js';
|
|
8
|
+
import { MonotonousIndexTransformer, ArrayEdit, SingleArrayEdit } from '../arrayOperation.js';
|
|
9
|
+
import { applyStateStackDiff, INITIAL } from 'vscode-textmate';
|
|
10
|
+
|
|
11
|
+
class TextMateWorkerTokenizerController extends Disposable {
|
|
12
|
+
static { this._id = 0; }
|
|
13
|
+
constructor(_model, _worker, _languageIdCodec, _backgroundTokenizationStore, _configurationService, _maxTokenizationLineLength) {
|
|
14
|
+
super();
|
|
15
|
+
this._model = _model;
|
|
16
|
+
this._worker = _worker;
|
|
17
|
+
this._languageIdCodec = _languageIdCodec;
|
|
18
|
+
this._backgroundTokenizationStore = _backgroundTokenizationStore;
|
|
19
|
+
this._configurationService = _configurationService;
|
|
20
|
+
this._maxTokenizationLineLength = _maxTokenizationLineLength;
|
|
21
|
+
this.controllerId = TextMateWorkerTokenizerController._id++;
|
|
22
|
+
this._pendingChanges = [];
|
|
23
|
+
this._states = ( new TokenizationStateStore());
|
|
24
|
+
this._loggingEnabled = observableConfigValue('editor.experimental.asyncTokenizationLogging', false, this._configurationService);
|
|
25
|
+
this._register(keepAlive(this._loggingEnabled));
|
|
26
|
+
this._register(this._model.onDidChangeContent((e) => {
|
|
27
|
+
if (this._shouldLog) {
|
|
28
|
+
console.log('model change', {
|
|
29
|
+
fileName: this._model.uri.fsPath.split('\\').pop(),
|
|
30
|
+
changes: changesToString(e.changes),
|
|
31
|
+
});
|
|
32
|
+
}
|
|
33
|
+
this._worker.acceptModelChanged(this.controllerId, e);
|
|
34
|
+
this._pendingChanges.push(e);
|
|
35
|
+
}));
|
|
36
|
+
this._register(this._model.onDidChangeLanguage((e) => {
|
|
37
|
+
const languageId = this._model.getLanguageId();
|
|
38
|
+
const encodedLanguageId = this._languageIdCodec.encodeLanguageId(languageId);
|
|
39
|
+
this._worker.acceptModelLanguageChanged(this.controllerId, languageId, encodedLanguageId);
|
|
40
|
+
}));
|
|
41
|
+
const languageId = this._model.getLanguageId();
|
|
42
|
+
const encodedLanguageId = this._languageIdCodec.encodeLanguageId(languageId);
|
|
43
|
+
this._worker.acceptNewModel({
|
|
44
|
+
uri: this._model.uri,
|
|
45
|
+
versionId: this._model.getVersionId(),
|
|
46
|
+
lines: this._model.getLinesContent(),
|
|
47
|
+
EOL: this._model.getEOL(),
|
|
48
|
+
languageId,
|
|
49
|
+
encodedLanguageId,
|
|
50
|
+
maxTokenizationLineLength: this._maxTokenizationLineLength.get(),
|
|
51
|
+
controllerId: this.controllerId,
|
|
52
|
+
});
|
|
53
|
+
this._register(autorun(reader => {
|
|
54
|
+
const maxTokenizationLineLength = this._maxTokenizationLineLength.read(reader);
|
|
55
|
+
this._worker.acceptMaxTokenizationLineLength(this.controllerId, maxTokenizationLineLength);
|
|
56
|
+
}));
|
|
57
|
+
}
|
|
58
|
+
dispose() {
|
|
59
|
+
super.dispose();
|
|
60
|
+
this._worker.acceptRemovedModel(this.controllerId);
|
|
61
|
+
}
|
|
62
|
+
requestTokens(startLineNumber, endLineNumberExclusive) {
|
|
63
|
+
this._worker.retokenize(this.controllerId, startLineNumber, endLineNumberExclusive);
|
|
64
|
+
}
|
|
65
|
+
async setTokensAndStates(controllerId, versionId, rawTokens, stateDeltas) {
|
|
66
|
+
if (this.controllerId !== controllerId) {
|
|
67
|
+
return;
|
|
68
|
+
}
|
|
69
|
+
let tokens = ContiguousMultilineTokensBuilder.deserialize(( new Uint8Array(rawTokens)));
|
|
70
|
+
if (this._shouldLog) {
|
|
71
|
+
console.log('received background tokenization result', {
|
|
72
|
+
fileName: this._model.uri.fsPath.split('\\').pop(),
|
|
73
|
+
updatedTokenLines: ( tokens.map((t) => t.getLineRange())).join(' & '),
|
|
74
|
+
updatedStateLines: ( stateDeltas.map((s) => ( ( new LineRange(s.startLineNumber, s.startLineNumber + s.stateDeltas.length)).toString()))).join(' & '),
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
if (this._shouldLog) {
|
|
78
|
+
const changes = ( ( this._pendingChanges.filter(c => c.versionId <= versionId).map(c => c.changes)).map(c => changesToString(c))).join(' then ');
|
|
79
|
+
console.log('Applying changes to local states', changes);
|
|
80
|
+
}
|
|
81
|
+
while (this._pendingChanges.length > 0 &&
|
|
82
|
+
this._pendingChanges[0].versionId <= versionId) {
|
|
83
|
+
const change = this._pendingChanges.shift();
|
|
84
|
+
this._states.acceptChanges(change.changes);
|
|
85
|
+
}
|
|
86
|
+
if (this._pendingChanges.length > 0) {
|
|
87
|
+
if (this._shouldLog) {
|
|
88
|
+
const changes = ( ( this._pendingChanges.map(c => c.changes)).map(c => changesToString(c))).join(' then ');
|
|
89
|
+
console.log('Considering non-processed changes', changes);
|
|
90
|
+
}
|
|
91
|
+
const curToFutureTransformerTokens = MonotonousIndexTransformer.fromMany(( this._pendingChanges.map((c) => fullLineArrayEditFromModelContentChange(c.changes))));
|
|
92
|
+
const b = ( new ContiguousMultilineTokensBuilder());
|
|
93
|
+
for (const t of tokens) {
|
|
94
|
+
for (let i = t.startLineNumber; i <= t.endLineNumber; i++) {
|
|
95
|
+
const result = curToFutureTransformerTokens.transform(i - 1);
|
|
96
|
+
if (result !== undefined) {
|
|
97
|
+
b.add(i, t.getLineTokens(i));
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
tokens = b.finalize();
|
|
102
|
+
for (const change of this._pendingChanges) {
|
|
103
|
+
for (const innerChanges of change.changes) {
|
|
104
|
+
for (let j = 0; j < tokens.length; j++) {
|
|
105
|
+
tokens[j].applyEdit(innerChanges.range, innerChanges.text);
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
const curToFutureTransformerStates = MonotonousIndexTransformer.fromMany(( this._pendingChanges.map((c) => fullLineArrayEditFromModelContentChange(c.changes))));
|
|
111
|
+
if (!this._applyStateStackDiffFn || !this._initialState) {
|
|
112
|
+
this._applyStateStackDiffFn = applyStateStackDiff;
|
|
113
|
+
this._initialState = INITIAL;
|
|
114
|
+
}
|
|
115
|
+
for (const d of stateDeltas) {
|
|
116
|
+
let prevState = d.startLineNumber <= 1 ? this._initialState : this._states.getEndState(d.startLineNumber - 1);
|
|
117
|
+
for (let i = 0; i < d.stateDeltas.length; i++) {
|
|
118
|
+
const delta = d.stateDeltas[i];
|
|
119
|
+
let state;
|
|
120
|
+
if (delta) {
|
|
121
|
+
state = this._applyStateStackDiffFn(prevState, delta);
|
|
122
|
+
this._states.setEndState(d.startLineNumber + i, state);
|
|
123
|
+
}
|
|
124
|
+
else {
|
|
125
|
+
state = this._states.getEndState(d.startLineNumber + i);
|
|
126
|
+
}
|
|
127
|
+
const offset = curToFutureTransformerStates.transform(d.startLineNumber + i - 1);
|
|
128
|
+
if (offset !== undefined) {
|
|
129
|
+
this._backgroundTokenizationStore.setEndState(offset + 1, state);
|
|
130
|
+
}
|
|
131
|
+
if (d.startLineNumber + i >= this._model.getLineCount() - 1) {
|
|
132
|
+
this._backgroundTokenizationStore.backgroundTokenizationFinished();
|
|
133
|
+
}
|
|
134
|
+
prevState = state;
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
this._backgroundTokenizationStore.setTokens(tokens);
|
|
138
|
+
}
|
|
139
|
+
get _shouldLog() { return this._loggingEnabled.get(); }
|
|
140
|
+
}
|
|
141
|
+
function fullLineArrayEditFromModelContentChange(c) {
|
|
142
|
+
return ( new ArrayEdit(( c.map((c) => ( new SingleArrayEdit(
|
|
143
|
+
c.range.startLineNumber - 1,
|
|
144
|
+
c.range.endLineNumber - c.range.startLineNumber + 1,
|
|
145
|
+
countEOL(c.text)[0] + 1
|
|
146
|
+
))))));
|
|
147
|
+
}
|
|
148
|
+
function changesToString(changes) {
|
|
149
|
+
return ( changes.map(c => ( Range.lift(c.range).toString()) + ' => ' + c.text)).join(' & ');
|
|
150
|
+
}
|
|
151
|
+
function observableConfigValue(key, defaultValue, configurationService) {
|
|
152
|
+
return observableFromEvent((handleChange) => configurationService.onDidChangeConfiguration(e => {
|
|
153
|
+
if (e.affectsConfiguration(key)) {
|
|
154
|
+
handleChange(e);
|
|
155
|
+
}
|
|
156
|
+
}), () => configurationService.getValue(key) ?? defaultValue);
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
export { TextMateWorkerTokenizerController };
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
import { __decorate, __param } from '../../../../../../../../external/tslib/tslib.es6.js';
|
|
2
|
+
import { DisposableStore, toDisposable } from 'monaco-editor/esm/vs/base/common/lifecycle.js';
|
|
3
|
+
import { FileAccess, nodeModulesPath, nodeModulesAsarPath } from 'monaco-editor/esm/vs/base/common/network.js';
|
|
4
|
+
import { isWeb } from 'monaco-editor/esm/vs/base/common/platform.js';
|
|
5
|
+
import { URI } from 'monaco-editor/esm/vs/base/common/uri.js';
|
|
6
|
+
import { createWebWorker } from 'monaco-editor/esm/vs/editor/browser/services/webWorker.js';
|
|
7
|
+
import { ILanguageService } from 'monaco-editor/esm/vs/editor/common/languages/language.js';
|
|
8
|
+
import { ILanguageConfigurationService } from 'monaco-editor/esm/vs/editor/common/languages/languageConfigurationRegistry.js';
|
|
9
|
+
import { IModelService } from 'monaco-editor/esm/vs/editor/common/services/model.js';
|
|
10
|
+
import { IConfigurationService } from 'monaco-editor/esm/vs/platform/configuration/common/configuration.js';
|
|
11
|
+
import { IEnvironmentService } from 'monaco-editor/esm/vs/platform/environment/common/environment.js';
|
|
12
|
+
import { IExtensionResourceLoaderService } from 'vscode/vscode/vs/platform/extensionResourceLoader/common/extensionResourceLoader';
|
|
13
|
+
import { INotificationService } from 'monaco-editor/esm/vs/platform/notification/common/notification.js';
|
|
14
|
+
import { ITelemetryService } from 'monaco-editor/esm/vs/platform/telemetry/common/telemetry.js';
|
|
15
|
+
import { TextMateWorkerTokenizerController } from './textMateWorkerTokenizerController.js';
|
|
16
|
+
|
|
17
|
+
var ThreadedBackgroundTokenizerFactory_1;
|
|
18
|
+
let ThreadedBackgroundTokenizerFactory = class ThreadedBackgroundTokenizerFactory {
|
|
19
|
+
static { ThreadedBackgroundTokenizerFactory_1 = this; }
|
|
20
|
+
static { this._reportedMismatchingTokens = false; }
|
|
21
|
+
constructor(_reportTokenizationTime, _shouldTokenizeAsync, _extensionResourceLoaderService, _modelService, _languageConfigurationService, _configurationService, _languageService, _environmentService, _notificationService, _telemetryService) {
|
|
22
|
+
this._reportTokenizationTime = _reportTokenizationTime;
|
|
23
|
+
this._shouldTokenizeAsync = _shouldTokenizeAsync;
|
|
24
|
+
this._extensionResourceLoaderService = _extensionResourceLoaderService;
|
|
25
|
+
this._modelService = _modelService;
|
|
26
|
+
this._languageConfigurationService = _languageConfigurationService;
|
|
27
|
+
this._configurationService = _configurationService;
|
|
28
|
+
this._languageService = _languageService;
|
|
29
|
+
this._environmentService = _environmentService;
|
|
30
|
+
this._notificationService = _notificationService;
|
|
31
|
+
this._telemetryService = _telemetryService;
|
|
32
|
+
this._workerProxyPromise = null;
|
|
33
|
+
this._worker = null;
|
|
34
|
+
this._workerProxy = null;
|
|
35
|
+
this._workerTokenizerControllers = ( new Map());
|
|
36
|
+
this._currentTheme = null;
|
|
37
|
+
this._currentTokenColorMap = null;
|
|
38
|
+
this._grammarDefinitions = [];
|
|
39
|
+
}
|
|
40
|
+
dispose() {
|
|
41
|
+
this._disposeWorker();
|
|
42
|
+
}
|
|
43
|
+
createBackgroundTokenizer(textModel, tokenStore, maxTokenizationLineLength) {
|
|
44
|
+
if (!this._shouldTokenizeAsync() || textModel.isTooLargeForSyncing()) {
|
|
45
|
+
return undefined;
|
|
46
|
+
}
|
|
47
|
+
const store = ( new DisposableStore());
|
|
48
|
+
const controllerContainer = this._getWorkerProxy().then((workerProxy) => {
|
|
49
|
+
if (store.isDisposed || !workerProxy) {
|
|
50
|
+
return undefined;
|
|
51
|
+
}
|
|
52
|
+
const controllerContainer = { controller: undefined, worker: this._worker };
|
|
53
|
+
store.add(keepAliveWhenAttached(textModel, () => {
|
|
54
|
+
const controller = ( new TextMateWorkerTokenizerController(
|
|
55
|
+
textModel,
|
|
56
|
+
workerProxy,
|
|
57
|
+
this._languageService.languageIdCodec,
|
|
58
|
+
tokenStore,
|
|
59
|
+
this._configurationService,
|
|
60
|
+
maxTokenizationLineLength
|
|
61
|
+
));
|
|
62
|
+
controllerContainer.controller = controller;
|
|
63
|
+
this._workerTokenizerControllers.set(controller.controllerId, controller);
|
|
64
|
+
return toDisposable(() => {
|
|
65
|
+
controllerContainer.controller = undefined;
|
|
66
|
+
this._workerTokenizerControllers.delete(controller.controllerId);
|
|
67
|
+
controller.dispose();
|
|
68
|
+
});
|
|
69
|
+
}));
|
|
70
|
+
return controllerContainer;
|
|
71
|
+
});
|
|
72
|
+
return {
|
|
73
|
+
dispose() {
|
|
74
|
+
store.dispose();
|
|
75
|
+
},
|
|
76
|
+
requestTokens: async (startLineNumber, endLineNumberExclusive) => {
|
|
77
|
+
const container = await controllerContainer;
|
|
78
|
+
if (container?.controller && container.worker === this._worker) {
|
|
79
|
+
container.controller.requestTokens(startLineNumber, endLineNumberExclusive);
|
|
80
|
+
}
|
|
81
|
+
},
|
|
82
|
+
reportMismatchingTokens: (lineNumber) => {
|
|
83
|
+
if (ThreadedBackgroundTokenizerFactory_1._reportedMismatchingTokens) {
|
|
84
|
+
return;
|
|
85
|
+
}
|
|
86
|
+
ThreadedBackgroundTokenizerFactory_1._reportedMismatchingTokens = true;
|
|
87
|
+
this._notificationService.error({
|
|
88
|
+
message: 'Async Tokenization Token Mismatch in line ' + lineNumber,
|
|
89
|
+
name: 'Async Tokenization Token Mismatch',
|
|
90
|
+
});
|
|
91
|
+
this._telemetryService.publicLog2('asyncTokenizationMismatchingTokens', {});
|
|
92
|
+
},
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
setGrammarDefinitions(grammarDefinitions) {
|
|
96
|
+
this._grammarDefinitions = grammarDefinitions;
|
|
97
|
+
this._disposeWorker();
|
|
98
|
+
}
|
|
99
|
+
acceptTheme(theme, colorMap) {
|
|
100
|
+
this._currentTheme = theme;
|
|
101
|
+
this._currentTokenColorMap = colorMap;
|
|
102
|
+
if (this._currentTheme && this._currentTokenColorMap && this._workerProxy) {
|
|
103
|
+
this._workerProxy.acceptTheme(this._currentTheme, this._currentTokenColorMap);
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
_getWorkerProxy() {
|
|
107
|
+
if (!this._workerProxyPromise) {
|
|
108
|
+
this._workerProxyPromise = this._createWorkerProxy();
|
|
109
|
+
}
|
|
110
|
+
return this._workerProxyPromise;
|
|
111
|
+
}
|
|
112
|
+
async _createWorkerProxy() {
|
|
113
|
+
const textmateModuleLocation = `${nodeModulesPath}/vscode-textmate`;
|
|
114
|
+
const textmateModuleLocationAsar = `${nodeModulesAsarPath}/vscode-textmate`;
|
|
115
|
+
const onigurumaModuleLocation = `${nodeModulesPath}/vscode-oniguruma`;
|
|
116
|
+
const onigurumaModuleLocationAsar = `${nodeModulesAsarPath}/vscode-oniguruma`;
|
|
117
|
+
const useAsar = this._environmentService.isBuilt && !isWeb;
|
|
118
|
+
const textmateLocation = useAsar ? textmateModuleLocationAsar : textmateModuleLocation;
|
|
119
|
+
const onigurumaLocation = useAsar ? onigurumaModuleLocationAsar : onigurumaModuleLocation;
|
|
120
|
+
const textmateMain = `${textmateLocation}/release/main.js`;
|
|
121
|
+
const onigurumaMain = `${onigurumaLocation}/release/main.js`;
|
|
122
|
+
const onigurumaWASM = `${onigurumaLocation}/release/onig.wasm`;
|
|
123
|
+
const uri = ( ( FileAccess.asBrowserUri(textmateMain)).toString(true));
|
|
124
|
+
const createData = {
|
|
125
|
+
grammarDefinitions: this._grammarDefinitions,
|
|
126
|
+
textmateMainUri: uri,
|
|
127
|
+
onigurumaMainUri: ( ( FileAccess.asBrowserUri(onigurumaMain)).toString(true)),
|
|
128
|
+
onigurumaWASMUri: ( ( FileAccess.asBrowserUri(onigurumaWASM)).toString(true)),
|
|
129
|
+
};
|
|
130
|
+
const host = {
|
|
131
|
+
readFile: async (_resource) => {
|
|
132
|
+
const resource = URI.revive(_resource);
|
|
133
|
+
return this._extensionResourceLoaderService.readExtensionResource(resource);
|
|
134
|
+
},
|
|
135
|
+
setTokensAndStates: async (controllerId, versionId, tokens, lineEndStateDeltas) => {
|
|
136
|
+
const controller = this._workerTokenizerControllers.get(controllerId);
|
|
137
|
+
if (controller) {
|
|
138
|
+
controller.setTokensAndStates(controllerId, versionId, tokens, lineEndStateDeltas);
|
|
139
|
+
}
|
|
140
|
+
},
|
|
141
|
+
reportTokenizationTime: (timeMs, languageId, sourceExtensionId, lineLength, isRandomSample) => {
|
|
142
|
+
this._reportTokenizationTime(timeMs, languageId, sourceExtensionId, lineLength, isRandomSample);
|
|
143
|
+
}
|
|
144
|
+
};
|
|
145
|
+
const worker = this._worker = createWebWorker(this._modelService, this._languageConfigurationService, {
|
|
146
|
+
createData,
|
|
147
|
+
label: 'textMateWorker',
|
|
148
|
+
moduleId: 'vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateTokenizationWorker.worker',
|
|
149
|
+
host,
|
|
150
|
+
});
|
|
151
|
+
const proxy = await ( worker.getProxy());
|
|
152
|
+
if (this._worker !== worker) {
|
|
153
|
+
return null;
|
|
154
|
+
}
|
|
155
|
+
this._workerProxy = proxy;
|
|
156
|
+
if (this._currentTheme && this._currentTokenColorMap) {
|
|
157
|
+
this._workerProxy.acceptTheme(this._currentTheme, this._currentTokenColorMap);
|
|
158
|
+
}
|
|
159
|
+
return proxy;
|
|
160
|
+
}
|
|
161
|
+
_disposeWorker() {
|
|
162
|
+
for (const controller of ( this._workerTokenizerControllers.values())) {
|
|
163
|
+
controller.dispose();
|
|
164
|
+
}
|
|
165
|
+
this._workerTokenizerControllers.clear();
|
|
166
|
+
if (this._worker) {
|
|
167
|
+
this._worker.dispose();
|
|
168
|
+
this._worker = null;
|
|
169
|
+
}
|
|
170
|
+
this._workerProxy = null;
|
|
171
|
+
this._workerProxyPromise = null;
|
|
172
|
+
}
|
|
173
|
+
};
|
|
174
|
+
ThreadedBackgroundTokenizerFactory = ThreadedBackgroundTokenizerFactory_1 = ( __decorate([
|
|
175
|
+
( __param(2, IExtensionResourceLoaderService)),
|
|
176
|
+
( __param(3, IModelService)),
|
|
177
|
+
( __param(4, ILanguageConfigurationService)),
|
|
178
|
+
( __param(5, IConfigurationService)),
|
|
179
|
+
( __param(6, ILanguageService)),
|
|
180
|
+
( __param(7, IEnvironmentService)),
|
|
181
|
+
( __param(8, INotificationService)),
|
|
182
|
+
( __param(9, ITelemetryService))
|
|
183
|
+
], ThreadedBackgroundTokenizerFactory));
|
|
184
|
+
function keepAliveWhenAttached(textModel, factory) {
|
|
185
|
+
const disposableStore = ( new DisposableStore());
|
|
186
|
+
const subStore = disposableStore.add(( new DisposableStore()));
|
|
187
|
+
function checkAttached() {
|
|
188
|
+
if (textModel.isAttachedToEditor()) {
|
|
189
|
+
subStore.add(factory());
|
|
190
|
+
}
|
|
191
|
+
else {
|
|
192
|
+
subStore.clear();
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
checkAttached();
|
|
196
|
+
disposableStore.add(textModel.onDidChangeAttached(() => {
|
|
197
|
+
checkAttached();
|
|
198
|
+
}));
|
|
199
|
+
return disposableStore;
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
export { ThreadedBackgroundTokenizerFactory };
|