@codingame/monaco-vscode-treesitter-service-override 15.0.2 → 16.0.0

package/index.js

@@ -13,7 +13,7 @@ import './vscode/src/vs/workbench/services/treeSitter/browser/treeSitterTokeniza
 registerAssets({
     'vs/../../node_modules/@vscode/tree-sitter-wasm/wasm/tree-sitter.wasm': new URL('@vscode/tree-sitter-wasm/wasm/tree-sitter.wasm', import.meta.url).href,
     'vs/../../node_modules/@vscode/tree-sitter-wasm/wasm/tree-sitter-typescript.wasm': new URL('@vscode/tree-sitter-wasm/wasm/tree-sitter-typescript.wasm', import.meta.url).href,
-    'vs/editor/common/languages/highlights/typescript.scm': new URL('./assets/typescript.scm', import.meta.url).href
+    'vs/editor/common/languages/highlights/typescript.scm': new URL('./vscode/src/vs/editor/common/languages/highlights/typescript.scm', import.meta.url).href
 });
 function getServiceOverride() {
     return {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@codingame/monaco-vscode-treesitter-service-override",
-  "version": "15.0.2",
+  "version": "16.0.0",
   "private": false,
   "description": "VSCode public API plugged on the monaco editor - treesitter service-override",
   "keywords": [],
@@ -15,11 +15,11 @@
   },
   "type": "module",
   "dependencies": {
-    "@codingame/monaco-vscode-168b98e5-dc20-5807-b1f9-798f1f92b37f-common": "15.0.2",
-    "@codingame/monaco-vscode-7ba0af96-90c2-5e11-ad7f-befdbbf246c8-common": "15.0.2",
-    "@codingame/monaco-vscode-9d0168a3-519b-57f3-9bcc-89efc41f951a-common": "15.0.2",
-    "@codingame/monaco-vscode-api": "15.0.2",
-    "@vscode/tree-sitter-wasm": "0.1.3"
+    "@codingame/monaco-vscode-7ba0af96-90c2-5e11-ad7f-befdbbf246c8-common": "16.0.0",
+    "@codingame/monaco-vscode-9d0168a3-519b-57f3-9bcc-89efc41f951a-common": "16.0.0",
+    "@codingame/monaco-vscode-ae8a8ca1-f243-508b-9c37-c88ebbd295fa-common": "16.0.0",
+    "@codingame/monaco-vscode-api": "16.0.0",
+    "@vscode/tree-sitter-wasm": "0.1.4"
   },
   "main": "index.js",
   "module": "index.js",
@@ -28,6 +28,9 @@
     ".": {
       "default": "./index.js"
     },
+    "./vscode/*.css": {
+      "default": "./vscode/src/*.css"
+    },
     "./vscode/*": {
       "types": "./vscode/src/*.d.ts",
       "default": "./vscode/src/*.js"

package/vscode/src/vs/editor/common/languages/highlights/typescript.scm

@@ -0,0 +1,488 @@
+; Order matters! Place lower precedence first.
+
+; Variables
+
+(identifier) @variable.ts
+
+(_
+  object: (identifier) @variable.other.object.ts)
+
+; Literals
+
+(this) @variable.language.this.ts
+(super) @variable.language.super.ts
+
+(comment) @comment.ts
+
+; TODO: This doesn't seem to be working
+(escape_sequence) @constant.character.escape.ts
+
+((string) @string.quoted.single.ts
+  (#match? @string.quoted.single.ts "^'.*'$"))
+
+((string) @string.quoted.double.ts
+  (#match? @string.quoted.double.ts "^\".*\"$"))
+
+([
+  (template_string)
+  (template_literal_type)
+] @string.template.ts)
+
+(string .
+  ([
+    "\""
+    "'"
+  ]) @punctuation.definition.string.begin.ts)
+
+(string
+  ([
+    "\""
+    "'"
+  ]) @punctuation.definition.string.end.ts .)
+
+(template_string . ("`") @punctuation.definition.string.template.begin.ts)
+
+(template_string ("`") @punctuation.definition.string.template.end.ts .)
+
+; NOTE: the typescript grammar doesn't break regex into nice parts so as to capture parts of it separately
+(regex) @string.regexp.ts
+(number) @constant.numeric.ts
+
+; Properties
+
+(member_expression
+  object: (this)
+  property: (property_identifier) @variable.ts)
+
+(member_expression
+  property: (property_identifier) @variable.other.constant.ts
+  (#match? @variable.other.constant.ts "^[A-Z][A-Z_]+$"))
+
+[
+  (property_identifier)
+  (shorthand_property_identifier)
+  (shorthand_property_identifier_pattern)] @variable.ts
+
+; Function and method definitions
+
+(function_expression
+  name: (identifier) @entity.name.function.ts)
+(function_signature
+  name: (identifier) @entity.name.function.ts)
+(function_declaration
+  name: (identifier) @entity.name.function.ts)
+(method_definition
+  name: (property_identifier) @meta.definition.method.ts @entity.name.function.ts
+  (#not-eq? @entity.name.function.ts "constructor"))
+(method_definition
+  name: (property_identifier) @meta.definition.method.ts @storage.type.ts
+  (#eq? @storage.type.ts "constructor"))
+(method_signature
+  name: (property_identifier) @meta.definition.method.ts @entity.name.function.ts)
+(generator_function_declaration
+  "*" @keyword.generator.asterisk.ts)
+(generator_function_declaration
+  name: (identifier) @meta.definition.function.ts @entity.name.function.ts)
+
+(pair
+  key: (property_identifier) @entity.name.function.ts
+  value: [(function_expression) (arrow_function)])
+
+(assignment_expression
+  left: (member_expression
+    property: (property_identifier) @entity.name.function.ts)
+  right: [(function_expression) (arrow_function)])
+
+(variable_declarator
+  name: (identifier) @entity.name.function.ts
+  value: [(function_expression) (arrow_function)])
+
+(assignment_expression
+  left: (identifier) @entity.name.function.ts
+  right: [(function_expression) (arrow_function)])
+
+(required_parameter
+  (identifier) @variable.parameter.ts)
+
+(required_parameter
+  (_
+    ([
+      (identifier)
+      (shorthand_property_identifier_pattern)
+    ]) @variable.parameter.ts))
+
+(optional_parameter
+  (identifier) @variable.parameter.ts)
+
+(optional_parameter
+  (_
+    ([
+      (identifier)
+      (shorthand_property_identifier_pattern)
+    ]) @variable.parameter.ts))
+
+(catch_clause
+  parameter: (identifier) @variable.parameter.ts)
+
+(index_signature
+  name: (identifier) @variable.parameter.ts)
+
+(arrow_function
+  parameter: (identifier) @variable.parameter.ts)
+
+; Function and method calls
+
+(call_expression
+  function: (identifier) @entity.name.function.ts)
+
+(call_expression
+  function: (member_expression
+  	object: (identifier) @support.class.promise.ts)
+    (#eq? @support.class.promise.ts "Promise"))
+
+(call_expression
+  function: (member_expression
+    property: (property_identifier) @entity.name.function.ts))
+
+(new_expression) @new.expr.ts
+
+(new_expression
+  constructor: (identifier) @entity.name.function.ts)
+
+
+; Special identifiers
+
+(predefined_type) @support.type.ts
+(predefined_type (["string" "boolean" "number" "any" "unknown" "never" "void"])) @support.type.primitive.ts
+
+(_
+  (type_identifier) @entity.name.type.ts)
+
+(type_annotation
+  ([
+    (type_identifier)
+    (nested_type_identifier)
+   ]) @meta.type.annotation.ts @entity.name.type.ts)
+
+(class_declaration
+  (type_identifier) @entity.name.type.class.ts)
+
+(type_alias_declaration
+  (type_identifier) @entity.name.type.alias.ts)
+(type_alias_declaration
+  value: (_
+    (type_identifier) @entity.name.type.ts))
+
+(interface_declaration
+  (type_identifier) @entity.name.type.interface.ts)
+
+(internal_module
+  name: (identifier) @entity.name.type.ts)
+
+(enum_declaration
+  name: (identifier) @entity.name.type.enum.ts)
+
+(
+  [
+    (_ name: (identifier))
+    (shorthand_property_identifier)
+    (shorthand_property_identifier_pattern)
+  ] @variable.other.constant.ts
+  (#match? @variable.other.constant.ts "^[A-Z][A-Z_]+$"))
+
+(extends_clause
+  value: (identifier) @entity.other.inherited-class.ts)
+
+(extends_type_clause
+  type: (type_identifier) @entity.other.inherited-class.ts)
+
+(implements_clause
+  (type_identifier) @entity.other.inherited-class.ts)
+
+; Tokens
+
+[
+  ";"
+  "?."
+  "."
+  ","
+  ":"
+  "?"
+] @punctuation.delimiter.ts
+
+[
+  "!"
+  "~"
+  "==="
+  "!=="
+  "&&"
+  "||"
+  "??"
+] @keyword.operator.logical.ts
+
+(binary_expression ([
+  "-"
+  "+"
+  "*"
+  "/"
+  "%"
+  "^"
+]) @keyword.operator.arithmetic.ts)
+
+(binary_expression ([
+  "<"
+  "<="
+  ">"
+  ">="
+]) @keyword.operator.relational.ts)
+
+(unary_expression ([
+  "-"
+]) @keyword.operator.arithmetic.ts)
+
+[
+  "="
+] @keyword.operator.assignment.ts
+
+(augmented_assignment_expression ([
+  "-="
+  "+="
+  "*="
+  "/="
+  "%="
+  "^="
+  "&="
+  "|="
+  "&&="
+  "||="
+  "??="
+]) @keyword.operator.assignment.compound.ts)
+
+[
+  "++"
+] @keyword.operator.increment.ts
+
+[
+  "--"
+] @keyword.operator.decrement.ts
+
+[
+  "**"
+  "**="
+  "<<"
+  "<<="
+  "=="
+  "!="
+  ">>"
+  ">>="
+  ">>>"
+  ">>>="
+  "~"
+  "&"
+  "|"
+] @keyword.operator.ts
+
+(union_type
+  ("|") @keyword.operator.type.ts)
+
+(intersection_type
+  ("&") @keyword.operator.type.ts)
+
+(type_annotation
+  (":") @keyword.operator.type.annotation.ts)
+
+(index_signature
+  (":") @keyword.operator.type.annotation.ts)
+
+(type_predicate_annotation
+  (":") @keyword.operator.type.annotation.ts)
+
+(conditional_type
+  ([
+    "?"
+    ":"
+  ]) @keyword.operator.ternary.ts)
+
+[
+  "{"
+  "}"
+  "("
+  ")"
+  "["
+  "]"
+] @punctuation.ts
+
+(template_substitution
+  "${" @punctuation.definition.template-expression.begin.ts
+  "}" @punctuation.definition.template-expression.end.ts)
+
+(template_type
+  "${" @punctuation.definition.template-expression.begin.ts
+  "}" @punctuation.definition.template-expression.end.ts)
+
+(type_arguments
+  "<" @punctuation.definition.typeparameters.begin.ts
+  ">" @punctuation.definition.typeparameters.end.ts)
+
+(type_parameters
+  "<" @punctuation.definition.typeparameters.begin.ts
+  ">" @punctuation.definition.typeparameters.end.ts)
+
+; Keywords
+
+("typeof") @keyword.operator.expression.typeof.ts
+
+(binary_expression "instanceof" @keyword.operator.expression.instanceof.ts)
+
+("of") @keyword.operator.expression.of.ts
+
+("is") @keyword.operator.expression.is.ts
+
+[
+  "delete"
+  "in"
+  "infer"
+  "keyof"
+] @keyword.operator.expression.ts
+
+[
+  "as"
+  "await"
+  "break"
+  "case"
+  "catch"
+  "continue"
+  "default"
+  "do"
+  "else"
+  "export"
+  "finally"
+  "for"
+  "from"
+  "if"
+  "import"
+  "require"
+  "return"
+  "satisfies"
+  "switch"
+  "throw"
+  "try"
+  "while"
+  "yield"
+] @keyword.control.ts
+
+[
+  "abstract"
+  "async"
+  "declare"
+  "extends"
+  "implements"
+  "override"
+  "private"
+  "protected"
+  "public"
+  "readonly"
+  "static"
+] @storage.modifier.ts
+
+[
+  "=>"
+  "class"
+  "const"
+  "enum"
+  "function"
+  "get"
+  "interface"
+  "let"
+  "namespace"
+  "set"
+  "var"
+] @storage.type.ts
+
+("type") @storage.type.type.ts
+
+[
+  "module"
+] @storage.type.namespace.ts
+
+[
+  "debugger"
+  "target"
+  "with"
+] @keyword.ts
+
+(regex_flags) @keyword.ts
+
+(unary_expression
+  "void" @keyword.operator.expression.void.ts)
+
+[
+  "new"
+] @keyword.operator.new.ts
+
+(public_field_definition
+  ("?") @keyword.operator.optional.ts)
+
+(property_signature
+  ("?") @keyword.operator.optional.ts)
+
+(method_signature
+  ("?") @keyword.operator.optional.ts)
+
+(optional_parameter
+  ([
+    "?"
+    ":"
+  ]) @keyword.operator.optional.ts)
+
+(ternary_expression
+  ([
+    "?"
+    ":"
+  ]) @keyword.operator.ternary.ts)
+
+(optional_chain
+  ("?.") @punctuation.accessor.optional.ts)
+
+(rest_pattern
+  ("...") @keyword.operator.rest.ts)
+(rest_type
+  ("...") @keyword.operator.rest.ts)
+
+(spread_element
+  ("...") @keyword.operator.spread.ts)
+
+; Language constants
+
+[
+  (null)
+] @constant.language.null.ts
+
+[
+  (undefined)
+] @constant.language.undefined.ts
+
+((identifier) @constant.language.nan.ts
+  (#eq? @constant.language.nan.ts "NaN"))
+
+((identifier) @constant.language.infinity.ts
+  (#eq? @constant.language.infinity.ts "Infinity"))
+
+[
+  (true)
+] @constant.language.boolean.true.ts
+
+[
+  (false)
+] @constant.language.boolean.false.ts
+
+(literal_type
+  [
+    (null)
+    (undefined)
+    (true)
+    (false)
+  ] @support.type.builtin.ts)
+
+(namespace_import
+  "*" @constant.language.ts)

package/vscode/src/vs/editor/common/model/tokenStore.d.ts

@@ -27,7 +27,6 @@ export declare enum TokenQuality {
 type Node = ListNode | LeafNode;
 interface LeafNode {
     readonly length: number;
-    parent?: ListNode;
     token: number;
     tokenQuality: TokenQuality;
     height: 0;

package/vscode/src/vs/editor/common/model/tokenStore.js

@@ -24,7 +24,9 @@ class ListNode {
         this._children.push(node);
         this._length += node.length;
         this._updateParentLength(node.length);
-        node.parent = this;
+        if (!isLeaf(node)) {
+            node.parent = this;
+        }
     }
     _updateParentLength(delta) {
         let updateParent = this.parent;
@@ -46,7 +48,9 @@ class ListNode {
         this._children.unshift(node);
         this._length += node.length;
         this._updateParentLength(node.length);
-        node.parent = this;
+        if (!isLeaf(node)) {
+            node.parent = this;
+        }
     }
     unprependChild() {
         const child = this._children.shift();
@@ -202,7 +206,9 @@ class TokenStore {
             const node = stack.pop();
             const currentOffset = node.offset;
             if (currentOffset < updateOffsetStart && currentOffset + node.node.length <= updateOffsetStart) {
-                node.node.parent = undefined;
+                if (!isLeaf(node.node)) {
+                    node.node.parent = undefined;
+                }
                 precedingNodes.push(node.node);
                 continue;
             }
@@ -213,11 +219,13 @@ class TokenStore {
                 continue;
             }
             if (currentOffset >= firstUnchangedOffsetAfterUpdate) {
-                node.node.parent = undefined;
+                if (!isLeaf(node.node)) {
+                    node.node.parent = undefined;
+                }
                 postcedingNodes.push(node.node);
                 continue;
             }
-            else if (isLeaf(node.node) && (currentOffset + node.node.length >= firstUnchangedOffsetAfterUpdate)) {
+            else if (isLeaf(node.node) && (currentOffset + node.node.length > firstUnchangedOffsetAfterUpdate)) {
                 postcedingNodes.push({ length: currentOffset + node.node.length - firstUnchangedOffsetAfterUpdate, token: node.node.token, height: 0, tokenQuality: node.node.tokenQuality });
                 continue;
             }
@@ -385,9 +393,7 @@ class TokenStore {
         const stack = [[this._root, false]];
         while (stack.length > 0) {
             const [node, visited] = stack.pop();
-            if (isLeaf(node)) {
-                node.parent = undefined;
-            }
+            if (isLeaf(node)) ;
             else if (!visited) {
                 stack.push([node, true]);
                 for (let i = node.children.length - 1; i >= 0; i--) {

package/vscode/src/vs/editor/common/model/treeSitterTokenStoreService.d.ts

@@ -2,6 +2,7 @@ import { Range } from "@codingame/monaco-vscode-api/vscode/vs/editor/common/core
 import { ITextModel } from "@codingame/monaco-vscode-api/vscode/vs/editor/common/model";
 import { TokenQuality, TokenUpdate } from "./tokenStore.js";
 import { IDisposable } from "@codingame/monaco-vscode-api/vscode/vs/base/common/lifecycle";
+import { IModelContentChangedEvent } from "@codingame/monaco-vscode-api/vscode/vs/editor/common/textModelEvents";
 import { ITreeSitterTokenizationStoreService } from "@codingame/monaco-vscode-api/vscode/vs/editor/common/model/treeSitterTokenStoreService.service";
 export interface TokenInformation {
     tokens: Uint32Array;
@@ -12,6 +13,7 @@ export declare class TreeSitterTokenizationStoreService implements ITreeSitterTo
     private readonly tokens;
     constructor();
     setTokens(model: ITextModel, tokens: TokenUpdate[], tokenQuality: TokenQuality): void;
+    handleContentChanged(model: ITextModel, e: IModelContentChangedEvent): void;
     rangeHasTokens(model: ITextModel, range: Range, minimumTokenQuality: TokenQuality): boolean;
     hasTokens(model: ITextModel, accurateForRange?: Range): boolean;
     getTokens(model: ITextModel, line: number): Uint32Array | undefined;
@@ -25,5 +27,6 @@ export declare class TreeSitterTokenizationStoreService implements ITreeSitterTo
         startOffset: number;
         endOffset: number;
     }[];
+    delete(model: ITextModel): void;
     dispose(): void;
 }

package/vscode/src/vs/editor/common/model/treeSitterTokenStoreService.js

@@ -14,33 +14,6 @@ class TreeSitterTokenizationStoreService {
         const store = disposables.add(( new TokenStore(model)));
         this.tokens.set(model, { store: store, accurateVersion: model.getVersionId(), disposables, guessVersion: model.getVersionId() });
         store.buildStore(tokens, tokenQuality);
-        disposables.add(model.onDidChangeContent(e => {
-            const storeInfo = this.tokens.get(model);
-            if (!storeInfo) {
-                return;
-            }
-            storeInfo.guessVersion = e.versionId;
-            for (const change of e.changes) {
-                if (change.text.length > change.rangeLength) {
-                    const offset = change.rangeOffset > 0 ? change.rangeOffset - 1 : change.rangeOffset;
-                    const oldToken = storeInfo.store.getTokenAt(offset);
-                    let newToken;
-                    if (oldToken) {
-                        newToken = { startOffsetInclusive: oldToken.startOffsetInclusive, length: oldToken.length + change.text.length - change.rangeLength, token: oldToken.token };
-                    }
-                    else {
-                        newToken = { startOffsetInclusive: offset, length: change.text.length + 1, token: 0 };
-                    }
-                    storeInfo.store.update(oldToken?.length ?? 0, [newToken], TokenQuality.EditGuess);
-                }
-                else if (change.text.length < change.rangeLength) {
-                    const deletedCharCount = change.rangeLength - change.text.length;
-                    storeInfo.store.delete(deletedCharCount, change.rangeOffset);
-                }
-                const refreshLength = change.rangeLength > change.text.length ? change.rangeLength : change.text.length;
-                storeInfo.store.markForRefresh(change.rangeOffset, change.rangeOffset + refreshLength);
-            }
-        }));
         disposables.add(model.onWillDispose(() => {
             const storeInfo = this.tokens.get(model);
             if (storeInfo) {
@@ -49,6 +22,32 @@ class TreeSitterTokenizationStoreService {
             }
         }));
     }
+    handleContentChanged(model, e) {
+        const storeInfo = this.tokens.get(model);
+        if (!storeInfo) {
+            return;
+        }
+        storeInfo.guessVersion = e.versionId;
+        for (const change of e.changes) {
+            if (change.text.length > change.rangeLength) {
+                const offset = change.rangeOffset > 0 ? change.rangeOffset - 1 : change.rangeOffset;
+                const oldToken = storeInfo.store.getTokenAt(offset);
+                let newToken;
+                if (oldToken) {
+                    newToken = { startOffsetInclusive: oldToken.startOffsetInclusive, length: oldToken.length + change.text.length - change.rangeLength, token: oldToken.token };
+                    storeInfo.store.markForRefresh(offset, change.rangeOffset + (change.text.length > change.rangeLength ? change.text.length : change.rangeLength));
+                }
+                else {
+                    newToken = { startOffsetInclusive: offset, length: change.text.length, token: 0 };
+                }
+                storeInfo.store.update(oldToken?.length ?? 0, [newToken], TokenQuality.EditGuess);
+            }
+            else if (change.text.length < change.rangeLength) {
+                const deletedCharCount = change.rangeLength - change.text.length;
+                storeInfo.store.delete(deletedCharCount, change.rangeOffset);
+            }
+        }
+    }
     rangeHasTokens(model, range, minimumTokenQuality) {
         const tokens = this.tokens.get(model);
         if (!tokens) {
@@ -119,6 +118,13 @@ class TreeSitterTokenizationStoreService {
             endOffset: range.endOffset
         })));
     }
+    delete(model) {
+        const storeInfo = this.tokens.get(model);
+        if (storeInfo) {
+            storeInfo.disposables.dispose();
+            this.tokens.delete(model);
+        }
+    }
     dispose() {
         for (const [, value] of this.tokens) {
             value.disposables.dispose();