@codingame/monaco-vscode-treesitter-service-override 14.0.6 → 15.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/typescript.scm +88 -21
- package/index.js +5 -2
- package/package.json +6 -5
- package/vscode/src/vs/editor/common/model/tokenStore.d.ts +66 -0
- package/vscode/src/vs/editor/common/model/tokenStore.js +406 -0
- package/vscode/src/vs/editor/common/model/treeSitterTokenStoreService.d.ts +29 -0
- package/vscode/src/vs/editor/common/model/treeSitterTokenStoreService.js +129 -0
- package/vscode/src/vs/editor/common/services/treeSitter/treeSitterParserService.d.ts +14 -15
- package/vscode/src/vs/editor/common/services/treeSitter/treeSitterParserService.js +72 -60
- package/vscode/src/vs/editor/common/services/treeSitterParserService.d.ts +48 -0
- package/vscode/src/vs/editor/common/services/treeSitterParserService.js +36 -0
- package/vscode/src/vs/workbench/services/treeSitter/browser/treeSitterCodeEditors.d.ts +27 -0
- package/vscode/src/vs/workbench/services/treeSitter/browser/treeSitterCodeEditors.js +98 -0
- package/vscode/src/vs/workbench/services/treeSitter/browser/treeSitterTokenizationFeature.contribution.js +1 -1
- package/vscode/src/vs/workbench/services/treeSitter/browser/treeSitterTokenizationFeature.d.ts +27 -11
- package/vscode/src/vs/workbench/services/treeSitter/browser/treeSitterTokenizationFeature.js +313 -130
package/assets/typescript.scm
CHANGED
|
@@ -25,20 +25,6 @@
|
|
|
25
25
|
(regex) @string.regexp
|
|
26
26
|
(number) @constant.numeric
|
|
27
27
|
|
|
28
|
-
; Template TODO: These don't seem to be working
|
|
29
|
-
|
|
30
|
-
(template_substitution
|
|
31
|
-
"${" @punctuation.definition.template-expression.begin
|
|
32
|
-
"}" @punctuation.definition.template-expression.end)
|
|
33
|
-
|
|
34
|
-
(template_type
|
|
35
|
-
"${" @punctuation.definition.template-expression.begin
|
|
36
|
-
"}" @punctuation.definition.template-expression.end)
|
|
37
|
-
|
|
38
|
-
(type_arguments
|
|
39
|
-
"<" @punctuation.bracket
|
|
40
|
-
">" @punctuation.bracket)
|
|
41
|
-
|
|
42
28
|
; Properties
|
|
43
29
|
|
|
44
30
|
(member_expression
|
|
@@ -86,6 +72,19 @@
|
|
|
86
72
|
left: (identifier) @entity.name.function
|
|
87
73
|
right: [(function_expression) (arrow_function)])
|
|
88
74
|
|
|
75
|
+
(required_parameter
|
|
76
|
+
(identifier) @variable.parameter)
|
|
77
|
+
|
|
78
|
+
(required_parameter
|
|
79
|
+
(rest_pattern
|
|
80
|
+
(identifier) @variable.parameter))
|
|
81
|
+
|
|
82
|
+
(optional_parameter
|
|
83
|
+
(identifier) @variable.parameter)
|
|
84
|
+
|
|
85
|
+
(catch_clause
|
|
86
|
+
parameter: (identifier) @variable.parameter)
|
|
87
|
+
|
|
89
88
|
; Function and method calls
|
|
90
89
|
|
|
91
90
|
(call_expression
|
|
@@ -100,6 +99,8 @@
|
|
|
100
99
|
function: (member_expression
|
|
101
100
|
property: (property_identifier) @entity.name.function))
|
|
102
101
|
|
|
102
|
+
(new_expression) @new.expr
|
|
103
|
+
|
|
103
104
|
(new_expression
|
|
104
105
|
constructor: (identifier) @entity.name.function)
|
|
105
106
|
|
|
@@ -107,8 +108,10 @@
|
|
|
107
108
|
; Special identifiers
|
|
108
109
|
|
|
109
110
|
(predefined_type) @support.type
|
|
110
|
-
(predefined_type (["string" "boolean" "number" "any"])) @support.type.primitive
|
|
111
|
+
(predefined_type (["string" "boolean" "number" "any" "unknown"])) @support.type.primitive
|
|
111
112
|
(type_identifier) @entity.name.type
|
|
113
|
+
(internal_module
|
|
114
|
+
name: (identifier) @entity.name.type.ts)
|
|
112
115
|
|
|
113
116
|
([
|
|
114
117
|
(identifier)
|
|
@@ -119,6 +122,9 @@
|
|
|
119
122
|
(extends_clause
|
|
120
123
|
value: (identifier) @entity.other.inherited-class)
|
|
121
124
|
|
|
125
|
+
(implements_clause
|
|
126
|
+
(type_identifier) @entity.other.inherited-class)
|
|
127
|
+
|
|
122
128
|
; Tokens
|
|
123
129
|
|
|
124
130
|
[
|
|
@@ -198,6 +204,36 @@
|
|
|
198
204
|
"|"
|
|
199
205
|
] @keyword.operator
|
|
200
206
|
|
|
207
|
+
(union_type
|
|
208
|
+
("|") @keyword.operator.type)
|
|
209
|
+
|
|
210
|
+
(intersection_type
|
|
211
|
+
("&") @keyword.operator.type)
|
|
212
|
+
|
|
213
|
+
(type_annotation
|
|
214
|
+
(":") @keyword.operator.type.annotation)
|
|
215
|
+
|
|
216
|
+
[
|
|
217
|
+
"{"
|
|
218
|
+
"}"
|
|
219
|
+
"("
|
|
220
|
+
")"
|
|
221
|
+
"["
|
|
222
|
+
"]"
|
|
223
|
+
] @punctuation
|
|
224
|
+
|
|
225
|
+
(template_substitution
|
|
226
|
+
"${" @punctuation.definition.template-expression.begin
|
|
227
|
+
"}" @punctuation.definition.template-expression.end)
|
|
228
|
+
|
|
229
|
+
(template_type
|
|
230
|
+
"${" @punctuation.definition.template-expression.begin
|
|
231
|
+
"}" @punctuation.definition.template-expression.end)
|
|
232
|
+
|
|
233
|
+
(type_arguments
|
|
234
|
+
"<" @punctuation.definition.typeparameters
|
|
235
|
+
">" @punctuation.definition.typeparameters)
|
|
236
|
+
|
|
201
237
|
; Keywords
|
|
202
238
|
|
|
203
239
|
("typeof") @keyword.operator.expression.typeof
|
|
@@ -270,6 +306,10 @@
|
|
|
270
306
|
"var"
|
|
271
307
|
] @storage.type
|
|
272
308
|
|
|
309
|
+
[
|
|
310
|
+
"module"
|
|
311
|
+
] @storage.type.namespace.ts
|
|
312
|
+
|
|
273
313
|
[
|
|
274
314
|
"debugger"
|
|
275
315
|
"target"
|
|
@@ -289,11 +329,14 @@
|
|
|
289
329
|
(public_field_definition
|
|
290
330
|
("?") @keyword.operator.optional)
|
|
291
331
|
|
|
292
|
-
(
|
|
332
|
+
(property_signature
|
|
333
|
+
("?") @keyword.operator.optional)
|
|
334
|
+
|
|
335
|
+
(optional_parameter
|
|
293
336
|
([
|
|
294
337
|
"?"
|
|
295
338
|
":"
|
|
296
|
-
]) @keyword.operator.optional
|
|
339
|
+
]) @keyword.operator.optional)
|
|
297
340
|
|
|
298
341
|
(ternary_expression
|
|
299
342
|
([
|
|
@@ -306,16 +349,40 @@
|
|
|
306
349
|
|
|
307
350
|
(rest_pattern) @keyword.operator.rest
|
|
308
351
|
|
|
309
|
-
(spread_element
|
|
352
|
+
(spread_element
|
|
353
|
+
("...") @keyword.operator.spread)
|
|
310
354
|
|
|
311
355
|
; Language constants
|
|
312
356
|
|
|
313
357
|
[
|
|
314
|
-
(true)
|
|
315
|
-
(false)
|
|
316
358
|
(null)
|
|
359
|
+
] @constant.language.null
|
|
360
|
+
|
|
361
|
+
[
|
|
317
362
|
(undefined)
|
|
318
|
-
] @constant.language
|
|
363
|
+
] @constant.language.undefined
|
|
364
|
+
|
|
365
|
+
((identifier) @constant.language.nan
|
|
366
|
+
(#eq? @constant.language.nan "NaN"))
|
|
367
|
+
|
|
368
|
+
((identifier) @constant.language.infinity
|
|
369
|
+
(#eq? @constant.language.infinity "Infinity"))
|
|
370
|
+
|
|
371
|
+
[
|
|
372
|
+
(true)
|
|
373
|
+
] @constant.language.boolean.true
|
|
374
|
+
|
|
375
|
+
[
|
|
376
|
+
(false)
|
|
377
|
+
] @constant.language.boolean.false
|
|
378
|
+
|
|
379
|
+
(literal_type
|
|
380
|
+
[
|
|
381
|
+
(null)
|
|
382
|
+
(undefined)
|
|
383
|
+
(true)
|
|
384
|
+
(false)
|
|
385
|
+
] @support.type.builtin)
|
|
319
386
|
|
|
320
387
|
(namespace_import
|
|
321
388
|
"*" @constant.language)
|
package/index.js
CHANGED
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
|
|
2
2
|
import { SyncDescriptor } from '@codingame/monaco-vscode-api/vscode/vs/platform/instantiation/common/descriptors';
|
|
3
|
-
import {
|
|
3
|
+
import { TreeSitterImporter } from './vscode/src/vs/editor/common/services/treeSitterParserService.js';
|
|
4
|
+
import { ITreeSitterParserService, ITreeSitterImporter } from '@codingame/monaco-vscode-api/vscode/vs/editor/common/services/treeSitterParserService.service';
|
|
4
5
|
import { TreeSitterTextModelService } from './vscode/src/vs/editor/common/services/treeSitter/treeSitterParserService.js';
|
|
5
6
|
import { ITreeSitterTokenizationFeature } from '@codingame/monaco-vscode-api/vscode/vs/workbench/services/treeSitter/browser/treeSitterTokenizationFeature.service';
|
|
6
7
|
import { TreeSitterTokenizationFeature } from './vscode/src/vs/workbench/services/treeSitter/browser/treeSitterTokenizationFeature.js';
|
|
7
8
|
import { registerAssets } from '@codingame/monaco-vscode-api/assets';
|
|
8
|
-
import { ITreeSitterTokenizationStoreService
|
|
9
|
+
import { ITreeSitterTokenizationStoreService } from '@codingame/monaco-vscode-api/vscode/vs/editor/common/model/treeSitterTokenStoreService.service';
|
|
10
|
+
import { TreeSitterTokenizationStoreService } from './vscode/src/vs/editor/common/model/treeSitterTokenStoreService.js';
|
|
9
11
|
import './vscode/src/vs/workbench/services/treeSitter/browser/treeSitterTokenizationFeature.contribution.js';
|
|
10
12
|
|
|
11
13
|
registerAssets({
|
|
@@ -15,6 +17,7 @@ registerAssets({
|
|
|
15
17
|
});
|
|
16
18
|
function getServiceOverride() {
|
|
17
19
|
return {
|
|
20
|
+
[ITreeSitterImporter.toString()]: new SyncDescriptor(TreeSitterImporter, [], false),
|
|
18
21
|
[ITreeSitterParserService.toString()]: new SyncDescriptor(TreeSitterTextModelService, [], false),
|
|
19
22
|
[ITreeSitterTokenizationFeature.toString()]: new SyncDescriptor(TreeSitterTokenizationFeature, [], false),
|
|
20
23
|
[ITreeSitterTokenizationStoreService.toString()]: new SyncDescriptor(TreeSitterTokenizationStoreService, [], false)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@codingame/monaco-vscode-treesitter-service-override",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "15.0.1",
|
|
4
4
|
"private": false,
|
|
5
5
|
"description": "VSCode public API plugged on the monaco editor - treesitter service-override",
|
|
6
6
|
"keywords": [],
|
|
@@ -15,10 +15,11 @@
|
|
|
15
15
|
},
|
|
16
16
|
"type": "module",
|
|
17
17
|
"dependencies": {
|
|
18
|
-
"@codingame/monaco-vscode-168b98e5-dc20-5807-b1f9-798f1f92b37f-common": "
|
|
19
|
-
"@codingame/monaco-vscode-
|
|
20
|
-
"@codingame/monaco-vscode-
|
|
21
|
-
"@
|
|
18
|
+
"@codingame/monaco-vscode-168b98e5-dc20-5807-b1f9-798f1f92b37f-common": "15.0.1",
|
|
19
|
+
"@codingame/monaco-vscode-7ba0af96-90c2-5e11-ad7f-befdbbf246c8-common": "15.0.1",
|
|
20
|
+
"@codingame/monaco-vscode-9d0168a3-519b-57f3-9bcc-89efc41f951a-common": "15.0.1",
|
|
21
|
+
"@codingame/monaco-vscode-api": "15.0.1",
|
|
22
|
+
"@vscode/tree-sitter-wasm": "0.1.3"
|
|
22
23
|
},
|
|
23
24
|
"main": "index.js",
|
|
24
25
|
"module": "index.js",
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import { IDisposable } from "@codingame/monaco-vscode-api/vscode/vs/base/common/lifecycle";
|
|
2
|
+
import { ITextModel } from "@codingame/monaco-vscode-api/vscode/vs/editor/common/model";
|
|
3
|
+
declare class ListNode implements IDisposable {
|
|
4
|
+
readonly height: number;
|
|
5
|
+
parent?: ListNode;
|
|
6
|
+
private readonly _children;
|
|
7
|
+
get children(): ReadonlyArray<Node>;
|
|
8
|
+
private _length;
|
|
9
|
+
get length(): number;
|
|
10
|
+
constructor(height: number);
|
|
11
|
+
static create(node1: Node, node2: Node): ListNode;
|
|
12
|
+
canAppendChild(): boolean;
|
|
13
|
+
appendChild(node: Node): void;
|
|
14
|
+
private _updateParentLength;
|
|
15
|
+
unappendChild(): Node;
|
|
16
|
+
prependChild(node: Node): void;
|
|
17
|
+
unprependChild(): Node;
|
|
18
|
+
lastChild(): Node;
|
|
19
|
+
dispose(): void;
|
|
20
|
+
}
|
|
21
|
+
export declare enum TokenQuality {
|
|
22
|
+
None = 0,
|
|
23
|
+
ViewportGuess = 1,
|
|
24
|
+
EditGuess = 2,
|
|
25
|
+
Accurate = 3
|
|
26
|
+
}
|
|
27
|
+
type Node = ListNode | LeafNode;
|
|
28
|
+
interface LeafNode {
|
|
29
|
+
readonly length: number;
|
|
30
|
+
parent?: ListNode;
|
|
31
|
+
token: number;
|
|
32
|
+
tokenQuality: TokenQuality;
|
|
33
|
+
height: 0;
|
|
34
|
+
}
|
|
35
|
+
export interface TokenUpdate {
|
|
36
|
+
readonly startOffsetInclusive: number;
|
|
37
|
+
readonly length: number;
|
|
38
|
+
readonly token: number;
|
|
39
|
+
}
|
|
40
|
+
export declare class TokenStore implements IDisposable {
|
|
41
|
+
private readonly _textModel;
|
|
42
|
+
private _root;
|
|
43
|
+
get root(): Node;
|
|
44
|
+
constructor(_textModel: ITextModel);
|
|
45
|
+
private createEmptyRoot;
|
|
46
|
+
buildStore(tokens: TokenUpdate[], tokenQuality: TokenQuality): void;
|
|
47
|
+
private createFromUpdates;
|
|
48
|
+
update(length: number, tokens: TokenUpdate[], tokenQuality: TokenQuality): void;
|
|
49
|
+
delete(length: number, startOffset: number): void;
|
|
50
|
+
private replace;
|
|
51
|
+
private traverseInOrderInRange;
|
|
52
|
+
getTokenAt(offset: number): TokenUpdate | undefined;
|
|
53
|
+
getTokensInRange(startOffsetInclusive: number, endOffsetExclusive: number): TokenUpdate[];
|
|
54
|
+
markForRefresh(startOffsetInclusive: number, endOffsetExclusive: number): void;
|
|
55
|
+
rangeHasTokens(startOffsetInclusive: number, endOffsetExclusive: number, minimumTokenQuality: TokenQuality): boolean;
|
|
56
|
+
rangeNeedsRefresh(startOffsetInclusive: number, endOffsetExclusive: number): boolean;
|
|
57
|
+
getNeedsRefresh(): {
|
|
58
|
+
startOffset: number;
|
|
59
|
+
endOffset: number;
|
|
60
|
+
}[];
|
|
61
|
+
deepCopy(): TokenStore;
|
|
62
|
+
private _copyNodeIterative;
|
|
63
|
+
printTree(root?: Node): string;
|
|
64
|
+
dispose(): void;
|
|
65
|
+
}
|
|
66
|
+
export {};
|
|
@@ -0,0 +1,406 @@
|
|
|
1
|
+
|
|
2
|
+
|
|
3
|
+
class ListNode {
|
|
4
|
+
get children() { return this._children; }
|
|
5
|
+
get length() { return this._length; }
|
|
6
|
+
constructor(height) {
|
|
7
|
+
this.height = height;
|
|
8
|
+
this._children = [];
|
|
9
|
+
this._length = 0;
|
|
10
|
+
}
|
|
11
|
+
static create(node1, node2) {
|
|
12
|
+
const list = ( new ListNode(node1.height + 1));
|
|
13
|
+
list.appendChild(node1);
|
|
14
|
+
list.appendChild(node2);
|
|
15
|
+
return list;
|
|
16
|
+
}
|
|
17
|
+
canAppendChild() {
|
|
18
|
+
return this._children.length < 3;
|
|
19
|
+
}
|
|
20
|
+
appendChild(node) {
|
|
21
|
+
if (!this.canAppendChild()) {
|
|
22
|
+
throw ( new Error('Cannot insert more than 3 children in a ListNode'));
|
|
23
|
+
}
|
|
24
|
+
this._children.push(node);
|
|
25
|
+
this._length += node.length;
|
|
26
|
+
this._updateParentLength(node.length);
|
|
27
|
+
node.parent = this;
|
|
28
|
+
}
|
|
29
|
+
_updateParentLength(delta) {
|
|
30
|
+
let updateParent = this.parent;
|
|
31
|
+
while (updateParent) {
|
|
32
|
+
updateParent._length += delta;
|
|
33
|
+
updateParent = updateParent.parent;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
unappendChild() {
|
|
37
|
+
const child = this._children.pop();
|
|
38
|
+
this._length -= child.length;
|
|
39
|
+
this._updateParentLength(-child.length);
|
|
40
|
+
return child;
|
|
41
|
+
}
|
|
42
|
+
prependChild(node) {
|
|
43
|
+
if (this._children.length >= 3) {
|
|
44
|
+
throw ( new Error('Cannot prepend more than 3 children in a ListNode'));
|
|
45
|
+
}
|
|
46
|
+
this._children.unshift(node);
|
|
47
|
+
this._length += node.length;
|
|
48
|
+
this._updateParentLength(node.length);
|
|
49
|
+
node.parent = this;
|
|
50
|
+
}
|
|
51
|
+
unprependChild() {
|
|
52
|
+
const child = this._children.shift();
|
|
53
|
+
this._length -= child.length;
|
|
54
|
+
this._updateParentLength(-child.length);
|
|
55
|
+
return child;
|
|
56
|
+
}
|
|
57
|
+
lastChild() {
|
|
58
|
+
return this._children[this._children.length - 1];
|
|
59
|
+
}
|
|
60
|
+
dispose() {
|
|
61
|
+
this._children.splice(0, this._children.length);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
var TokenQuality;
|
|
65
|
+
(function (TokenQuality) {
|
|
66
|
+
TokenQuality[TokenQuality["None"] = 0] = "None";
|
|
67
|
+
TokenQuality[TokenQuality["ViewportGuess"] = 1] = "ViewportGuess";
|
|
68
|
+
TokenQuality[TokenQuality["EditGuess"] = 2] = "EditGuess";
|
|
69
|
+
TokenQuality[TokenQuality["Accurate"] = 3] = "Accurate";
|
|
70
|
+
})(TokenQuality || (TokenQuality = {}));
|
|
71
|
+
function isLeaf(node) {
|
|
72
|
+
return node.token !== undefined;
|
|
73
|
+
}
|
|
74
|
+
function append(node, nodeToAppend) {
|
|
75
|
+
let curNode = node;
|
|
76
|
+
const parents = [];
|
|
77
|
+
let nodeToAppendOfCorrectHeight;
|
|
78
|
+
while (true) {
|
|
79
|
+
if (nodeToAppend.height === curNode.height) {
|
|
80
|
+
nodeToAppendOfCorrectHeight = nodeToAppend;
|
|
81
|
+
break;
|
|
82
|
+
}
|
|
83
|
+
if (isLeaf(curNode)) {
|
|
84
|
+
throw ( new Error('unexpected'));
|
|
85
|
+
}
|
|
86
|
+
parents.push(curNode);
|
|
87
|
+
curNode = curNode.lastChild();
|
|
88
|
+
}
|
|
89
|
+
for (let i = parents.length - 1; i >= 0; i--) {
|
|
90
|
+
const parent = parents[i];
|
|
91
|
+
if (nodeToAppendOfCorrectHeight) {
|
|
92
|
+
if (parent.children.length >= 3) {
|
|
93
|
+
const newList = ListNode.create(parent.unappendChild(), nodeToAppendOfCorrectHeight);
|
|
94
|
+
nodeToAppendOfCorrectHeight = newList;
|
|
95
|
+
}
|
|
96
|
+
else {
|
|
97
|
+
parent.appendChild(nodeToAppendOfCorrectHeight);
|
|
98
|
+
nodeToAppendOfCorrectHeight = undefined;
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
if (nodeToAppendOfCorrectHeight) {
|
|
103
|
+
const newList = ( new ListNode(nodeToAppendOfCorrectHeight.height + 1));
|
|
104
|
+
newList.appendChild(node);
|
|
105
|
+
newList.appendChild(nodeToAppendOfCorrectHeight);
|
|
106
|
+
return newList;
|
|
107
|
+
}
|
|
108
|
+
else {
|
|
109
|
+
return node;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
function prepend(list, nodeToAppend) {
|
|
113
|
+
let curNode = list;
|
|
114
|
+
const parents = [];
|
|
115
|
+
while (nodeToAppend.height !== curNode.height) {
|
|
116
|
+
if (isLeaf(curNode)) {
|
|
117
|
+
throw ( new Error('unexpected'));
|
|
118
|
+
}
|
|
119
|
+
parents.push(curNode);
|
|
120
|
+
curNode = curNode.children[0];
|
|
121
|
+
}
|
|
122
|
+
let nodeToPrependOfCorrectHeight = nodeToAppend;
|
|
123
|
+
for (let i = parents.length - 1; i >= 0; i--) {
|
|
124
|
+
const parent = parents[i];
|
|
125
|
+
if (nodeToPrependOfCorrectHeight) {
|
|
126
|
+
if (parent.children.length >= 3) {
|
|
127
|
+
nodeToPrependOfCorrectHeight = ListNode.create(nodeToPrependOfCorrectHeight, parent.unprependChild());
|
|
128
|
+
}
|
|
129
|
+
else {
|
|
130
|
+
parent.prependChild(nodeToPrependOfCorrectHeight);
|
|
131
|
+
nodeToPrependOfCorrectHeight = undefined;
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
if (nodeToPrependOfCorrectHeight) {
|
|
136
|
+
return ListNode.create(nodeToPrependOfCorrectHeight, list);
|
|
137
|
+
}
|
|
138
|
+
else {
|
|
139
|
+
return list;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
function concat(node1, node2) {
|
|
143
|
+
if (node1.height === node2.height) {
|
|
144
|
+
return ListNode.create(node1, node2);
|
|
145
|
+
}
|
|
146
|
+
else if (node1.height > node2.height) {
|
|
147
|
+
return append(node1, node2);
|
|
148
|
+
}
|
|
149
|
+
else {
|
|
150
|
+
return prepend(node2, node1);
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
class TokenStore {
|
|
154
|
+
get root() {
|
|
155
|
+
return this._root;
|
|
156
|
+
}
|
|
157
|
+
constructor(_textModel) {
|
|
158
|
+
this._textModel = _textModel;
|
|
159
|
+
this._root = this.createEmptyRoot();
|
|
160
|
+
}
|
|
161
|
+
createEmptyRoot() {
|
|
162
|
+
return {
|
|
163
|
+
length: this._textModel.getValueLength(),
|
|
164
|
+
token: 0,
|
|
165
|
+
height: 0,
|
|
166
|
+
tokenQuality: TokenQuality.None
|
|
167
|
+
};
|
|
168
|
+
}
|
|
169
|
+
buildStore(tokens, tokenQuality) {
|
|
170
|
+
this._root = this.createFromUpdates(tokens, tokenQuality);
|
|
171
|
+
}
|
|
172
|
+
createFromUpdates(tokens, tokenQuality) {
|
|
173
|
+
if (tokens.length === 0) {
|
|
174
|
+
return this.createEmptyRoot();
|
|
175
|
+
}
|
|
176
|
+
let newRoot = {
|
|
177
|
+
length: tokens[0].length,
|
|
178
|
+
token: tokens[0].token,
|
|
179
|
+
height: 0,
|
|
180
|
+
tokenQuality
|
|
181
|
+
};
|
|
182
|
+
for (let j = 1; j < tokens.length; j++) {
|
|
183
|
+
newRoot = append(newRoot, { length: tokens[j].length, token: tokens[j].token, height: 0, tokenQuality });
|
|
184
|
+
}
|
|
185
|
+
return newRoot;
|
|
186
|
+
}
|
|
187
|
+
update(length, tokens, tokenQuality) {
|
|
188
|
+
if (tokens.length === 0) {
|
|
189
|
+
return;
|
|
190
|
+
}
|
|
191
|
+
this.replace(length, tokens[0].startOffsetInclusive, tokens, tokenQuality);
|
|
192
|
+
}
|
|
193
|
+
delete(length, startOffset) {
|
|
194
|
+
this.replace(length, startOffset, [], TokenQuality.EditGuess);
|
|
195
|
+
}
|
|
196
|
+
replace(length, updateOffsetStart, tokens, tokenQuality) {
|
|
197
|
+
const firstUnchangedOffsetAfterUpdate = updateOffsetStart + length;
|
|
198
|
+
const precedingNodes = [];
|
|
199
|
+
const postcedingNodes = [];
|
|
200
|
+
const stack = [{ node: this._root, offset: 0 }];
|
|
201
|
+
while (stack.length > 0) {
|
|
202
|
+
const node = stack.pop();
|
|
203
|
+
const currentOffset = node.offset;
|
|
204
|
+
if (currentOffset < updateOffsetStart && currentOffset + node.node.length <= updateOffsetStart) {
|
|
205
|
+
node.node.parent = undefined;
|
|
206
|
+
precedingNodes.push(node.node);
|
|
207
|
+
continue;
|
|
208
|
+
}
|
|
209
|
+
else if (isLeaf(node.node) && (currentOffset < updateOffsetStart)) {
|
|
210
|
+
precedingNodes.push({ length: updateOffsetStart - currentOffset, token: node.node.token, height: 0, tokenQuality: node.node.tokenQuality });
|
|
211
|
+
}
|
|
212
|
+
if ((updateOffsetStart <= currentOffset) && (currentOffset + node.node.length <= firstUnchangedOffsetAfterUpdate)) {
|
|
213
|
+
continue;
|
|
214
|
+
}
|
|
215
|
+
if (currentOffset >= firstUnchangedOffsetAfterUpdate) {
|
|
216
|
+
node.node.parent = undefined;
|
|
217
|
+
postcedingNodes.push(node.node);
|
|
218
|
+
continue;
|
|
219
|
+
}
|
|
220
|
+
else if (isLeaf(node.node) && (currentOffset + node.node.length >= firstUnchangedOffsetAfterUpdate)) {
|
|
221
|
+
postcedingNodes.push({ length: currentOffset + node.node.length - firstUnchangedOffsetAfterUpdate, token: node.node.token, height: 0, tokenQuality: node.node.tokenQuality });
|
|
222
|
+
continue;
|
|
223
|
+
}
|
|
224
|
+
if (!isLeaf(node.node)) {
|
|
225
|
+
let childOffset = currentOffset + node.node.length;
|
|
226
|
+
for (let i = node.node.children.length - 1; i >= 0; i--) {
|
|
227
|
+
childOffset -= node.node.children[i].length;
|
|
228
|
+
stack.push({ node: node.node.children[i], offset: childOffset });
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
let allNodes;
|
|
233
|
+
if (tokens.length > 0) {
|
|
234
|
+
allNodes = precedingNodes.concat(this.createFromUpdates(tokens, tokenQuality), postcedingNodes);
|
|
235
|
+
}
|
|
236
|
+
else {
|
|
237
|
+
allNodes = precedingNodes.concat(postcedingNodes);
|
|
238
|
+
}
|
|
239
|
+
let newRoot = allNodes[0];
|
|
240
|
+
for (let i = 1; i < allNodes.length; i++) {
|
|
241
|
+
newRoot = concat(newRoot, allNodes[i]);
|
|
242
|
+
}
|
|
243
|
+
this._root = newRoot ?? this.createEmptyRoot();
|
|
244
|
+
}
|
|
245
|
+
traverseInOrderInRange(startOffsetInclusive, endOffsetExclusive, visitor) {
|
|
246
|
+
const stack = [{ node: this._root, offset: 0 }];
|
|
247
|
+
while (stack.length > 0) {
|
|
248
|
+
const { node, offset } = stack.pop();
|
|
249
|
+
const nodeEnd = offset + node.length;
|
|
250
|
+
if (nodeEnd <= startOffsetInclusive || offset >= endOffsetExclusive) {
|
|
251
|
+
continue;
|
|
252
|
+
}
|
|
253
|
+
if (visitor(node, offset)) {
|
|
254
|
+
return;
|
|
255
|
+
}
|
|
256
|
+
if (!isLeaf(node)) {
|
|
257
|
+
let childOffset = offset + node.length;
|
|
258
|
+
for (let i = node.children.length - 1; i >= 0; i--) {
|
|
259
|
+
childOffset -= node.children[i].length;
|
|
260
|
+
stack.push({ node: node.children[i], offset: childOffset });
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
getTokenAt(offset) {
|
|
266
|
+
let result;
|
|
267
|
+
this.traverseInOrderInRange(offset, this._root.length, (node, offset) => {
|
|
268
|
+
if (isLeaf(node)) {
|
|
269
|
+
result = { token: node.token, startOffsetInclusive: offset, length: node.length };
|
|
270
|
+
return true;
|
|
271
|
+
}
|
|
272
|
+
return false;
|
|
273
|
+
});
|
|
274
|
+
return result;
|
|
275
|
+
}
|
|
276
|
+
getTokensInRange(startOffsetInclusive, endOffsetExclusive) {
|
|
277
|
+
const result = [];
|
|
278
|
+
this.traverseInOrderInRange(startOffsetInclusive, endOffsetExclusive, (node, offset) => {
|
|
279
|
+
if (isLeaf(node)) {
|
|
280
|
+
let clippedLength = node.length;
|
|
281
|
+
let clippedOffset = offset;
|
|
282
|
+
if ((offset < startOffsetInclusive) && (offset + node.length > endOffsetExclusive)) {
|
|
283
|
+
clippedOffset = startOffsetInclusive;
|
|
284
|
+
clippedLength = endOffsetExclusive - startOffsetInclusive;
|
|
285
|
+
}
|
|
286
|
+
else if (offset < startOffsetInclusive) {
|
|
287
|
+
clippedLength -= (startOffsetInclusive - offset);
|
|
288
|
+
clippedOffset = startOffsetInclusive;
|
|
289
|
+
}
|
|
290
|
+
else if (offset + node.length > endOffsetExclusive) {
|
|
291
|
+
clippedLength -= (offset + node.length - endOffsetExclusive);
|
|
292
|
+
}
|
|
293
|
+
result.push({ token: node.token, startOffsetInclusive: clippedOffset, length: clippedLength });
|
|
294
|
+
}
|
|
295
|
+
return false;
|
|
296
|
+
});
|
|
297
|
+
return result;
|
|
298
|
+
}
|
|
299
|
+
markForRefresh(startOffsetInclusive, endOffsetExclusive) {
|
|
300
|
+
this.traverseInOrderInRange(startOffsetInclusive, endOffsetExclusive, (node) => {
|
|
301
|
+
if (isLeaf(node)) {
|
|
302
|
+
node.tokenQuality = TokenQuality.None;
|
|
303
|
+
}
|
|
304
|
+
return false;
|
|
305
|
+
});
|
|
306
|
+
}
|
|
307
|
+
rangeHasTokens(startOffsetInclusive, endOffsetExclusive, minimumTokenQuality) {
|
|
308
|
+
let hasAny = true;
|
|
309
|
+
this.traverseInOrderInRange(startOffsetInclusive, endOffsetExclusive, (node) => {
|
|
310
|
+
if (isLeaf(node) && (node.tokenQuality < minimumTokenQuality)) {
|
|
311
|
+
hasAny = false;
|
|
312
|
+
}
|
|
313
|
+
return false;
|
|
314
|
+
});
|
|
315
|
+
return hasAny;
|
|
316
|
+
}
|
|
317
|
+
rangeNeedsRefresh(startOffsetInclusive, endOffsetExclusive) {
|
|
318
|
+
let needsRefresh = false;
|
|
319
|
+
this.traverseInOrderInRange(startOffsetInclusive, endOffsetExclusive, (node) => {
|
|
320
|
+
if (isLeaf(node) && (node.tokenQuality !== TokenQuality.Accurate)) {
|
|
321
|
+
needsRefresh = true;
|
|
322
|
+
}
|
|
323
|
+
return false;
|
|
324
|
+
});
|
|
325
|
+
return needsRefresh;
|
|
326
|
+
}
|
|
327
|
+
getNeedsRefresh() {
|
|
328
|
+
const result = [];
|
|
329
|
+
this.traverseInOrderInRange(0, this._textModel.getValueLength(), (node, offset) => {
|
|
330
|
+
if (isLeaf(node) && (node.tokenQuality !== TokenQuality.Accurate)) {
|
|
331
|
+
if ((result.length > 0) && (result[result.length - 1].endOffset === offset)) {
|
|
332
|
+
result[result.length - 1].endOffset += node.length;
|
|
333
|
+
}
|
|
334
|
+
else {
|
|
335
|
+
result.push({ startOffset: offset, endOffset: offset + node.length });
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
return false;
|
|
339
|
+
});
|
|
340
|
+
return result;
|
|
341
|
+
}
|
|
342
|
+
deepCopy() {
|
|
343
|
+
const newStore = ( new TokenStore(this._textModel));
|
|
344
|
+
newStore._root = this._copyNodeIterative(this._root);
|
|
345
|
+
return newStore;
|
|
346
|
+
}
|
|
347
|
+
_copyNodeIterative(root) {
|
|
348
|
+
const newRoot = isLeaf(root)
|
|
349
|
+
? { length: root.length, token: root.token, tokenQuality: root.tokenQuality, height: root.height }
|
|
350
|
+
: ( new ListNode(root.height));
|
|
351
|
+
const stack = [[root, newRoot]];
|
|
352
|
+
while (stack.length > 0) {
|
|
353
|
+
const [oldNode, clonedNode] = stack.pop();
|
|
354
|
+
if (!isLeaf(oldNode)) {
|
|
355
|
+
for (const child of oldNode.children) {
|
|
356
|
+
const childCopy = isLeaf(child)
|
|
357
|
+
? { length: child.length, token: child.token, tokenQuality: child.tokenQuality, height: child.height }
|
|
358
|
+
: ( new ListNode(child.height));
|
|
359
|
+
clonedNode.appendChild(childCopy);
|
|
360
|
+
stack.push([child, childCopy]);
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
return newRoot;
|
|
365
|
+
}
|
|
366
|
+
printTree(root = this._root) {
|
|
367
|
+
const result = [];
|
|
368
|
+
const stack = [[root, 0]];
|
|
369
|
+
while (stack.length > 0) {
|
|
370
|
+
const [node, depth] = stack.pop();
|
|
371
|
+
const indent = ' '.repeat(depth);
|
|
372
|
+
if (isLeaf(node)) {
|
|
373
|
+
result.push(`${indent}Leaf(length: ${node.length}, token: ${node.token}, refresh: ${node.tokenQuality})\n`);
|
|
374
|
+
}
|
|
375
|
+
else {
|
|
376
|
+
result.push(`${indent}List(length: ${node.length})\n`);
|
|
377
|
+
for (let i = node.children.length - 1; i >= 0; i--) {
|
|
378
|
+
stack.push([node.children[i], depth + 1]);
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
return result.join('');
|
|
383
|
+
}
|
|
384
|
+
dispose() {
|
|
385
|
+
const stack = [[this._root, false]];
|
|
386
|
+
while (stack.length > 0) {
|
|
387
|
+
const [node, visited] = stack.pop();
|
|
388
|
+
if (isLeaf(node)) {
|
|
389
|
+
node.parent = undefined;
|
|
390
|
+
}
|
|
391
|
+
else if (!visited) {
|
|
392
|
+
stack.push([node, true]);
|
|
393
|
+
for (let i = node.children.length - 1; i >= 0; i--) {
|
|
394
|
+
stack.push([node.children[i], false]);
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
else {
|
|
398
|
+
node.dispose();
|
|
399
|
+
node.parent = undefined;
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
this._root = undefined;
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
export { TokenQuality, TokenStore };
|