@codingame/monaco-vscode-treesitter-service-override 10.0.0

package/assets/typescript.scm

@@ -0,0 +1,325 @@
+; Order matters! Place lower precedence first.
+; Adapted from https://github.com/zed-industries/zed/blob/main/crates/languages/src/typescript/highlights.scm
+
+; Variables
+
+(identifier) @variable
+
+; Literals
+
+(this) @variable.language.this
+(super) @variable.language.super
+
+(comment) @comment
+
+; TODO: This doesn't seem to be working
+(escape_sequence) @constant.character.escape
+
+[
+  (string)
+  (template_string)
+  (template_literal_type)
+] @string
+
+; NOTE: the typescript grammar doesn't break regex into nice parts so as to capture parts of it separately
+(regex) @string.regexp
+(number) @constant.numeric
+
+; Template TODO: These don't seem to be working
+
+(template_substitution
+  "${" @punctuation.definition.template-expression.begin
+  "}" @punctuation.definition.template-expression.end)
+
+(template_type
+  "${" @punctuation.definition.template-expression.begin
+  "}" @punctuation.definition.template-expression.end)
+
+(type_arguments
+  "<" @punctuation.bracket
+  ">" @punctuation.bracket)
+
+; Properties
+
+(member_expression
+  object: (this)
+  property: (property_identifier) @variable)
+
+(member_expression
+  property: (property_identifier) @variable.other.constant
+  (#match? @variable.other.constant "^[A-Z][A-Z_]+$"))
+
+[
+  (property_identifier)
+  (shorthand_property_identifier)
+  (shorthand_property_identifier_pattern)] @variable
+
+; Function and method definitions
+
+(function_expression
+  name: (identifier) @entity.name.function)
+(function_declaration
+  name: (identifier) @entity.name.function)
+(method_definition
+  name: (property_identifier) @meta.definition.method @entity.name.function)
+(method_definition
+  name: (property_identifier) @storage.type
+  (#eq? @storage.type "constructor"))
+(method_signature
+  name: (property_identifier) @meta.definition.method @entity.name.function)
+
+(pair
+  key: (property_identifier) @entity.name.function
+  value: [(function_expression) (arrow_function)])
+
+(assignment_expression
+  left: (member_expression
+    property: (property_identifier) @entity.name.function)
+  right: [(function_expression) (arrow_function)])
+
+(variable_declarator
+  name: (identifier) @entity.name.function
+  value: [(function_expression) (arrow_function)])
+
+(assignment_expression
+  left: (identifier) @entity.name.function
+  right: [(function_expression) (arrow_function)])
+
+; Function and method calls
+
+(call_expression
+  function: (identifier) @entity.name.function)
+
+(call_expression
+  function: (member_expression
+  	object: (identifier) @support.class.promise)
+    (#eq? @support.class.promise "Promise"))
+
+(call_expression
+  function: (member_expression
+    property: (property_identifier) @entity.name.function))
+
+(new_expression
+  constructor: (identifier) @entity.name.function)
+
+
+; Special identifiers
+
+(predefined_type) @support.type
+(predefined_type (["string" "boolean" "number" "any"])) @support.type.primitive
+(type_identifier) @entity.name.type
+
+(("const")
+  (variable_declarator
+  	name: (identifier) @variable.other.constant))
+
+([
+  (identifier)
+  (shorthand_property_identifier)
+  (shorthand_property_identifier_pattern)] @variable.other.constant
+  (#match? @variable.other.constant "^[A-Z][A-Z_]+$"))
+
+(extends_clause
+  value: (identifier) @entity.other.inherited-class)
+
+; Tokens
+
+[
+  ";"
+  "?."
+  "."
+  ","
+  ":"
+  "?"
+] @punctuation.delimiter
+
+[
+  "!"
+  "~"
+  "==="
+  "!=="
+  "&&"
+  "||"
+  "??"
+] @keyword.operator.logical
+
+(binary_expression ([
+  "-"
+  "+"
+  "*"
+  "/"
+  "%"
+  "^"
+]) @keyword.operator.arithmetic)
+
+(binary_expression ([
+  "<"
+  "<="
+  ">"
+  ">="
+]) @keyword.operator.relational)
+
+[
+  "="
+] @keyword.operator.assignment
+
+(augmented_assignment_expression ([
+  "-="
+  "+="
+  "*="
+  "/="
+  "%="
+  "^="
+  "&="
+  "|="
+  "&&="
+  "||="
+  "??="
+]) @keyword.operator.assignment.compound)
+
+[
+  "++"
+] @keyword.operator.increment
+
+[
+  "--"
+] @keyword.operator.decrement
+
+[
+  "**"
+  "**="
+  "<<"
+  "<<="
+  "=="
+  "!="
+  "=>"
+  ">>"
+  ">>="
+  ">>>"
+  ">>>="
+  "~"
+  "&"
+  "|"
+] @keyword.operator
+
+; Keywords
+
+("typeof") @keyword.operator.expression.typeof
+
+(binary_expression "instanceof" @keyword.operator.expression.instanceof)
+
+("of") @keyword.operator.expression.of
+
+("is") @keyword.operator.expression.is
+
+[
+  "delete"
+  "in"
+  "infer"
+  "keyof"
+] @keyword.operator.expression
+
+[
+  "as"
+  "await"
+  "break"
+  "case"
+  "catch"
+  "continue"
+  "default"
+  "do"
+  "else"
+  "export"
+  "finally"
+  "for"
+  "from"
+  "if"
+  "import"
+  "require"
+  "return"
+  "satisfies"
+  "switch"
+  "throw"
+  "try"
+  "type"
+  "while"
+  "yield"
+] @keyword.control
+
+[
+  "abstract"
+  "async"
+  "declare"
+  "extends"
+  "implements"
+  "override"
+  "private"
+  "protected"
+  "public"
+  "readonly"
+  "static"
+] @storage.modifier
+
+[
+  "=>"
+  "class"
+  "const"
+  "enum"
+  "function"
+  "get"
+  "interface"
+  "let"
+  "namespace"
+  "set"
+  "var"
+] @storage.type
+
+[
+  "debugger"
+  "target"
+  "with"
+] @keyword
+
+(regex_flags) @keyword
+
+[
+  "void"
+] @support.type.primitive
+
+[
+  "new"
+] @keyword.operator.new
+
+(public_field_definition
+  ("?") @keyword.operator.optional)
+
+(optional_parameter)
+  ([
+    "?"
+    ":"
+  ]) @keyword.operator.optional
+
+(ternary_expression
+  ([
+    "?"
+    ":"
+  ]) @keyword.operator.ternary)
+
+(optional_chain
+  ("?.") @punctuation.accessor.optional)
+
+(rest_pattern) @keyword.operator.rest
+
+(spread_element) @keyword.operator.spread
+
+; Language constants
+
+[
+  (true)
+  (false)
+  (null)
+  (undefined)
+] @constant.language
+
+(namespace_import
+  "*" @constant.language)

package/index.d.ts

@@ -0,0 +1 @@
+export { default } from 'vscode/service-override/treesitter';

package/index.js

@@ -0,0 +1 @@
+export { default } from './treesitter.js';

package/package.json

@@ -0,0 +1,32 @@
+{
+  "name": "@codingame/monaco-vscode-treesitter-service-override",
+  "version": "10.0.0",
+  "keywords": [],
+  "author": {
+    "name": "CodinGame",
+    "url": "http://www.codingame.com"
+  },
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git@github.com:CodinGame/monaco-vscode-api.git"
+  },
+  "type": "module",
+  "private": false,
+  "description": "VSCode public API plugged on the monaco editor - treesitter service-override",
+  "main": "index.js",
+  "module": "index.js",
+  "types": "index.d.ts",
+  "exports": {
+    ".": {
+      "default": "./index.js"
+    },
+    "./vscode/*": {
+      "default": "./vscode/src/*.js"
+    }
+  },
+  "dependencies": {
+    "vscode": "npm:@codingame/monaco-vscode-api@10.0.0",
+    "@vscode/tree-sitter-wasm": "^0.0.4"
+  }
+}

package/treesitter.js

@@ -0,0 +1,21 @@
+import { SyncDescriptor } from 'vscode/vscode/vs/platform/instantiation/common/descriptors';
+import { ITreeSitterParserService } from 'vscode/vscode/vs/editor/common/services/treeSitterParserService';
+import { TreeSitterTextModelService } from './vscode/src/vs/editor/browser/services/treeSitter/treeSitterParserService.js';
+import { ITreeSitterTokenizationFeature } from 'vscode/vscode/vs/workbench/services/treeSitter/browser/treeSitterTokenizationFeature.service';
+import { TreeSitterTokenizationFeature } from './vscode/src/vs/workbench/services/treeSitter/browser/treeSitterTokenizationFeature.js';
+import { registerAssets } from 'vscode/assets';
+import './vscode/src/vs/workbench/services/treeSitter/browser/treeSitterTokenizationFeature.contribution.js';
+
+registerAssets({
+    'vs/../../node_modules/@vscode/tree-sitter-wasm/wasm/tree-sitter.wasm': new URL('@vscode/tree-sitter-wasm/wasm/tree-sitter.wasm', import.meta.url).href,
+    'vs/../../node_modules/@vscode/tree-sitter-wasm/wasm/tree-sitter-typescript.wasm': new URL('@vscode/tree-sitter-wasm/wasm/tree-sitter-typescript.wasm', import.meta.url).href,
+    'vs/editor/common/languages/highlights/typescript.scm': new URL('./assets/typescript.scm', import.meta.url).href
+});
+function getServiceOverride() {
+    return {
+        [( ITreeSitterParserService.toString())]: new SyncDescriptor(TreeSitterTextModelService, [], false),
+        [( ITreeSitterTokenizationFeature.toString())]: new SyncDescriptor(TreeSitterTokenizationFeature, [], false)
+    };
+}
+
+export { getServiceOverride as default };

package/vscode/src/vs/editor/browser/services/treeSitter/treeSitterParserService.js

@@ -0,0 +1,455 @@
+import { __decorate, __param } from 'vscode/external/tslib/tslib.es6.js';
+import { nodeModulesPath, FileAccess } from 'vscode/vscode/vs/base/common/network';
+import { EDITOR_EXPERIMENTAL_PREFER_TREESITTER } from 'vscode/vscode/vs/editor/common/services/treeSitterParserService';
+import { IModelService } from 'vscode/vscode/vs/editor/common/services/model';
+import { Disposable, DisposableStore, dispose, DisposableMap } from 'vscode/vscode/vs/base/common/lifecycle';
+import { IFileService } from 'vscode/vscode/vs/platform/files/common/files.service';
+import { ITelemetryService } from 'vscode/vscode/vs/platform/telemetry/common/telemetry.service';
+import { ILogService } from 'vscode/vscode/vs/platform/log/common/log.service';
+import { IConfigurationService } from 'vscode/vscode/vs/platform/configuration/common/configuration.service';
+import { setTimeout0 } from 'vscode/vscode/vs/base/common/platform';
+import { Emitter, Event } from 'vscode/vscode/vs/base/common/event';
+import { cancelOnDispose } from 'vscode/vscode/vs/base/common/cancellation';
+import { IEnvironmentService } from 'vscode/vscode/vs/platform/environment/common/environment.service';
+import { isCancellationError, CancellationError } from 'vscode/vscode/vs/base/common/errors';
+import 'vscode/vscode/vs/base/common/arrays';
+import 'vscode/vscode/vs/base/common/observableInternal/autorun';
+import 'vscode/vscode/vs/base/common/observableInternal/derived';
+import { PromiseResult } from 'vscode/vscode/vs/base/common/observableInternal/promise';
+import 'vscode/vscode/vs/base/common/observableInternal/utils';
+import { Range } from 'vscode/vscode/vs/editor/common/core/range';
+
+const EDITOR_TREESITTER_TELEMETRY = 'editor.experimental.treeSitterTelemetry';
+const MODULE_LOCATION_SUBPATH = `@vscode/tree-sitter-wasm/wasm`;
+const FILENAME_TREESITTER_WASM = `tree-sitter.wasm`;
+function getModuleLocation(environmentService) {
+    return `${nodeModulesPath}/${MODULE_LOCATION_SUBPATH}`;
+}
+class TextModelTreeSitter extends Disposable {
+    get parseResult() { return this._parseResult; }
+    constructor(model, _treeSitterLanguages, _treeSitterImporter, _logService, _telemetryService) {
+        super();
+        this.model = model;
+        this._treeSitterLanguages = _treeSitterLanguages;
+        this._treeSitterImporter = _treeSitterImporter;
+        this._logService = _logService;
+        this._telemetryService = _telemetryService;
+        this._onDidChangeParseResult = this._register(( new Emitter()));
+        this.onDidChangeParseResult = this._onDidChangeParseResult.event;
+        this._languageSessionDisposables = this._register(( new DisposableStore()));
+        this._register(Event.runAndSubscribe(this.model.onDidChangeLanguage, (e => this._onDidChangeLanguage(e ? e.newLanguage : this.model.getLanguageId()))));
+    }
+    async _onDidChangeLanguage(languageId) {
+        this._languageSessionDisposables.clear();
+        this._parseResult = undefined;
+        const token = cancelOnDispose(this._languageSessionDisposables);
+        let language;
+        try {
+            language = await this._getLanguage(languageId, token);
+        }
+        catch (e) {
+            if (isCancellationError(e)) {
+                return;
+            }
+            throw e;
+        }
+        const Parser = await this._treeSitterImporter.getParserClass();
+        if (token.isCancellationRequested) {
+            return;
+        }
+        const treeSitterTree = this._languageSessionDisposables.add(( new TreeSitterParseResult(( new Parser()), language, this._logService, this._telemetryService)));
+        this._languageSessionDisposables.add(this.model.onDidChangeContent(e => this._onDidChangeContent(treeSitterTree, e.changes)));
+        await this._onDidChangeContent(treeSitterTree, []);
+        if (token.isCancellationRequested) {
+            return;
+        }
+        this._parseResult = treeSitterTree;
+    }
+    _getLanguage(languageId, token) {
+        const language = this._treeSitterLanguages.getOrInitLanguage(languageId);
+        if (language) {
+            return Promise.resolve(language);
+        }
+        const disposables = [];
+        return ( new Promise((resolve, reject) => {
+            disposables.push(this._treeSitterLanguages.onDidAddLanguage(e => {
+                if (e.id === languageId) {
+                    dispose(disposables);
+                    resolve(e.language);
+                }
+            }));
+            token.onCancellationRequested(() => {
+                dispose(disposables);
+                reject(( new CancellationError()));
+            }, undefined, disposables);
+        }));
+    }
+    async _onDidChangeContent(treeSitterTree, changes) {
+        const diff = await treeSitterTree.onDidChangeContent(this.model, changes);
+        if (!diff || diff.length > 0) {
+            const ranges = diff ? ( diff.map(r => ( new Range(
+                r.startPosition.row + 1,
+                r.startPosition.column + 1,
+                r.endPosition.row + 1,
+                r.endPosition.column + 1
+            )))) : [this.model.getFullModelRange()];
+            this._onDidChangeParseResult.fire(ranges);
+        }
+    }
+}
+class TreeSitterParseResult {
+    constructor(parser, language, _logService, _telemetryService) {
+        this.parser = parser;
+        this.language = language;
+        this._logService = _logService;
+        this._telemetryService = _telemetryService;
+        this._isDisposed = false;
+        this._onDidChangeContentQueue = Promise.resolve();
+        this._newEdits = true;
+        this.parser.setTimeoutMicros(50 * 1000);
+        this.parser.setLanguage(language);
+    }
+    dispose() {
+        this._isDisposed = true;
+        this._tree?.delete();
+        this.parser?.delete();
+    }
+    get tree() { return this._tree; }
+    set tree(newTree) {
+        this._tree?.delete();
+        this._tree = newTree;
+    }
+    get isDisposed() { return this._isDisposed; }
+    async onDidChangeContent(model, changes) {
+        const oldTree = this.tree?.copy();
+        this._applyEdits(model, changes);
+        return ( new Promise(resolve => {
+            this._onDidChangeContentQueue = this._onDidChangeContentQueue.then(async () => {
+                if (this.isDisposed) {
+                    return;
+                }
+                await this._parseAndUpdateTree(model);
+                resolve((this.tree && oldTree) ? oldTree.getChangedRanges(this.tree) : undefined);
+            }).catch((e) => {
+                this._logService.error('Error parsing tree-sitter tree', e);
+            });
+        }));
+    }
+    _applyEdits(model, changes) {
+        for (const change of changes) {
+            const newEndOffset = change.rangeOffset + change.text.length;
+            const newEndPosition = model.getPositionAt(newEndOffset);
+            this.tree?.edit({
+                startIndex: change.rangeOffset,
+                oldEndIndex: change.rangeOffset + change.rangeLength,
+                newEndIndex: change.rangeOffset + change.text.length,
+                startPosition: { row: change.range.startLineNumber - 1, column: change.range.startColumn - 1 },
+                oldEndPosition: { row: change.range.endLineNumber - 1, column: change.range.endColumn - 1 },
+                newEndPosition: { row: newEndPosition.lineNumber - 1, column: newEndPosition.column - 1 }
+            });
+            this._newEdits = true;
+        }
+    }
+    async _parseAndUpdateTree(model) {
+        const tree = await this._parse(model);
+        if (!this._newEdits) {
+            this.tree = tree;
+        }
+    }
+    _parse(model) {
+        let parseType = "fullParse" ;
+        if (this.tree) {
+            parseType = "incrementalParse" ;
+        }
+        return this._parseAndYield(model, parseType);
+    }
+    async _parseAndYield(model, parseType) {
+        const language = model.getLanguageId();
+        let tree;
+        let time = 0;
+        let passes = 0;
+        this._newEdits = false;
+        do {
+            const timer = performance.now();
+            try {
+                tree = this.parser.parse((index, position) => this._parseCallback(model, index), this.tree);
+            }
+            catch (e) {
+            }
+            finally {
+                time += performance.now() - timer;
+                passes++;
+            }
+            await ( new Promise(resolve => setTimeout0(resolve)));
+            if (model.isDisposed() || this.isDisposed) {
+                return;
+            }
+        } while (!tree && !this._newEdits);
+        this.sendParseTimeTelemetry(parseType, language, time, passes);
+        return tree;
+    }
+    _parseCallback(textModel, index) {
+        try {
+            return textModel.getTextBuffer().getNearestChunk(index);
+        }
+        catch (e) {
+            this._logService.debug('Error getting chunk for tree-sitter parsing', e);
+        }
+        return null;
+    }
+    sendParseTimeTelemetry(parseType, languageId, time, passes) {
+        this._logService.debug(`Tree parsing (${parseType}) took ${time} ms and ${passes} passes.`);
+        if (parseType === "fullParse" ) {
+            this._telemetryService.publicLog2(`treeSitter.fullParse`, { languageId, time, passes });
+        }
+        else {
+            this._telemetryService.publicLog2(`treeSitter.incrementalParse`, { languageId, time, passes });
+        }
+    }
+}
+class TreeSitterLanguages extends Disposable {
+    constructor(_treeSitterImporter, _fileService, _environmentService, _registeredLanguages) {
+        super();
+        this._treeSitterImporter = _treeSitterImporter;
+        this._fileService = _fileService;
+        this._environmentService = _environmentService;
+        this._registeredLanguages = _registeredLanguages;
+        this._languages = ( new AsyncCache());
+        this._onDidAddLanguage = this._register(( new Emitter()));
+        this.onDidAddLanguage = this._onDidAddLanguage.event;
+    }
+    getOrInitLanguage(languageId) {
+        if (this._languages.isCached(languageId)) {
+            return this._languages.getSyncIfCached(languageId);
+        }
+        else {
+            this._addLanguage(languageId);
+            return undefined;
+        }
+    }
+    async getLanguage(languageId) {
+        if (this._languages.isCached(languageId)) {
+            return this._languages.getSyncIfCached(languageId);
+        }
+        else {
+            await this._addLanguage(languageId);
+            return this._languages.get(languageId);
+        }
+    }
+    async _addLanguage(languageId) {
+        const languagePromise = this._languages.get(languageId);
+        if (!languagePromise) {
+            this._languages.set(languageId, this._fetchLanguage(languageId));
+            const language = await this._languages.get(languageId);
+            if (!language) {
+                return undefined;
+            }
+            this._onDidAddLanguage.fire({ id: languageId, language });
+        }
+    }
+    async _fetchLanguage(languageId) {
+        const grammarName = this._registeredLanguages.get(languageId);
+        const languageLocation = this._getLanguageLocation(languageId);
+        if (!grammarName || !languageLocation) {
+            return undefined;
+        }
+        const wasmPath = `${languageLocation}/${grammarName}.wasm`;
+        const languageFile = await (this._fileService.readFile(( FileAccess.asFileUri(wasmPath))));
+        const Parser = await this._treeSitterImporter.getParserClass();
+        return Parser.Language.load(languageFile.value.buffer);
+    }
+    _getLanguageLocation(languageId) {
+        const grammarName = this._registeredLanguages.get(languageId);
+        if (!grammarName) {
+            return undefined;
+        }
+        return getModuleLocation();
+    }
+}
+class TreeSitterImporter {
+    async _getTreeSitterImport() {
+        if (!this._treeSitterImport) {
+            this._treeSitterImport = await import('@vscode/tree-sitter-wasm').then(module => ({ Parser: module.default ?? module }));
+        }
+        return this._treeSitterImport;
+    }
+    async getParserClass() {
+        if (!this._parserClass) {
+            this._parserClass = (await this._getTreeSitterImport()).Parser;
+        }
+        return this._parserClass;
+    }
+}
+let TreeSitterTextModelService = class TreeSitterTextModelService extends Disposable {
+    constructor(_modelService, fileService, _telemetryService, _logService, _configurationService, _environmentService) {
+        super();
+        this._modelService = _modelService;
+        this._telemetryService = _telemetryService;
+        this._logService = _logService;
+        this._configurationService = _configurationService;
+        this._environmentService = _environmentService;
+        this._textModelTreeSitters = this._register(( new DisposableMap()));
+        this._registeredLanguages = ( new Map());
+        this._treeSitterImporter = ( new TreeSitterImporter());
+        this._onDidUpdateTree = this._register(( new Emitter()));
+        this.onDidUpdateTree = this._onDidUpdateTree.event;
+        this._hasInit = false;
+        this._treeSitterLanguages = this._register(( new TreeSitterLanguages(
+            this._treeSitterImporter,
+            fileService,
+            this._environmentService,
+            this._registeredLanguages
+        )));
+        this.onDidAddLanguage = this._treeSitterLanguages.onDidAddLanguage;
+        this._register(this._configurationService.onDidChangeConfiguration(e => {
+            if (e.affectsConfiguration(EDITOR_EXPERIMENTAL_PREFER_TREESITTER)) {
+                this._supportedLanguagesChanged();
+            }
+        }));
+        this._supportedLanguagesChanged();
+    }
+    getOrInitLanguage(languageId) {
+        return this._treeSitterLanguages.getOrInitLanguage(languageId);
+    }
+    getParseResult(textModel) {
+        const textModelTreeSitter = this._textModelTreeSitters.get(textModel);
+        return textModelTreeSitter?.textModelTreeSitter.parseResult;
+    }
+    async getTree(content, languageId) {
+        await this._init;
+        const language = await this._treeSitterLanguages.getLanguage(languageId);
+        const Parser = await this._treeSitterImporter.getParserClass();
+        if (language) {
+            const parser = ( new Parser());
+            parser.setLanguage(language);
+            return parser.parse(content);
+        }
+        return undefined;
+    }
+    async _doInitParser() {
+        const Parser = await this._treeSitterImporter.getParserClass();
+        await Parser.init({
+            locateFile(_file, _folder) {
+                return ( ( FileAccess.asBrowserUri(`${getModuleLocation()}/${FILENAME_TREESITTER_WASM}`)).toString(true));
+            }
+        });
+        return true;
+    }
+    async _initParser(hasLanguages) {
+        if (this._hasInit) {
+            return this._init;
+        }
+        if (hasLanguages) {
+            this._hasInit = true;
+            this._init = this._doInitParser();
+            this._init.then(() => this._registerModelServiceListeners());
+        }
+        else {
+            this._init = Promise.resolve(false);
+        }
+        return this._init;
+    }
+    async _supportedLanguagesChanged() {
+        const setting = this._getSetting();
+        let hasLanguages = true;
+        if (setting.length === 0) {
+            hasLanguages = false;
+        }
+        if (setting.includes('typescript')) {
+            this._addGrammar('typescript', 'tree-sitter-typescript');
+        }
+        else {
+            this._removeGrammar('typescript');
+        }
+        return this._initParser(hasLanguages);
+    }
+    _getSetting() {
+        const setting = this._configurationService.getValue(EDITOR_EXPERIMENTAL_PREFER_TREESITTER);
+        if (setting && setting.length > 0) {
+            return setting;
+        }
+        else {
+            const expSetting = this._configurationService.getValue(EDITOR_TREESITTER_TELEMETRY);
+            if (expSetting) {
+                return ['typescript'];
+            }
+        }
+        return [];
+    }
+    async _registerModelServiceListeners() {
+        this._register(this._modelService.onModelAdded(model => {
+            this._createTextModelTreeSitter(model);
+        }));
+        this._register(this._modelService.onModelRemoved(model => {
+            this._textModelTreeSitters.deleteAndDispose(model);
+        }));
+        this._modelService.getModels().forEach(model => this._createTextModelTreeSitter(model));
+    }
+    _createTextModelTreeSitter(model) {
+        const textModelTreeSitter = ( new TextModelTreeSitter(
+            model,
+            this._treeSitterLanguages,
+            this._treeSitterImporter,
+            this._logService,
+            this._telemetryService
+        ));
+        const disposables = ( new DisposableStore());
+        disposables.add(textModelTreeSitter);
+        disposables.add(textModelTreeSitter.onDidChangeParseResult((ranges) => this._onDidUpdateTree.fire({ textModel: model, ranges })));
+        this._textModelTreeSitters.set(model, {
+            textModelTreeSitter,
+            disposables,
+            dispose: disposables.dispose.bind(disposables)
+        });
+    }
+    _addGrammar(languageId, grammarName) {
+        if (!( this._registeredLanguages.has(languageId))) {
+            this._registeredLanguages.set(languageId, grammarName);
+        }
+    }
+    _removeGrammar(languageId) {
+        if (( this._registeredLanguages.has(languageId))) {
+            this._registeredLanguages.delete('typescript');
+        }
+    }
+};
+TreeSitterTextModelService = ( __decorate([
+    ( __param(0, IModelService)),
+    ( __param(1, IFileService)),
+    ( __param(2, ITelemetryService)),
+    ( __param(3, ILogService)),
+    ( __param(4, IConfigurationService)),
+    ( __param(5, IEnvironmentService))
+], TreeSitterTextModelService));
+class PromiseWithSyncAccess {
+    get result() {
+        return this._result;
+    }
+    constructor(promise) {
+        this.promise = promise;
+        promise.then(result => {
+            this._result = ( new PromiseResult(result, undefined));
+        }).catch(e => {
+            this._result = ( new PromiseResult(undefined, e));
+        });
+    }
+}
+class AsyncCache {
+    constructor() {
+        this._values = ( new Map());
+    }
+    set(key, promise) {
+        this._values.set(key, ( new PromiseWithSyncAccess(promise)));
+    }
+    get(key) {
+        return this._values.get(key)?.promise;
+    }
+    getSyncIfCached(key) {
+        return this._values.get(key)?.result?.data;
+    }
+    isCached(key) {
+        return this._values.get(key)?.result !== undefined;
+    }
+}
+
+export { TextModelTreeSitter, TreeSitterImporter, TreeSitterLanguages, TreeSitterParseResult, TreeSitterTextModelService };

package/vscode/src/vs/workbench/services/treeSitter/browser/treeSitterTokenizationFeature.contribution.js

@@ -0,0 +1,15 @@
+import { __decorate, __param } from 'vscode/external/tslib/tslib.es6.js';
+import { registerWorkbenchContribution2 } from 'vscode/vscode/vs/workbench/common/contributions';
+import '../../../../editor/browser/services/treeSitter/treeSitterParserService.js';
+import { ITreeSitterParserService } from 'vscode/vscode/vs/editor/common/services/treeSitterParserService';
+import { ITreeSitterTokenizationFeature } from 'vscode/vscode/vs/workbench/services/treeSitter/browser/treeSitterTokenizationFeature.service';
+
+let TreeSitterTokenizationInstantiator = class TreeSitterTokenizationInstantiator {
+    static { this.ID = 'workbench.contrib.treeSitterTokenizationInstantiator'; }
+    constructor(_treeSitterTokenizationService, _treeSitterTokenizationFeature) { }
+};
+TreeSitterTokenizationInstantiator = ( __decorate([
+    ( __param(0, ITreeSitterParserService)),
+    ( __param(1, ITreeSitterTokenizationFeature))
+], TreeSitterTokenizationInstantiator));
+registerWorkbenchContribution2(TreeSitterTokenizationInstantiator.ID, TreeSitterTokenizationInstantiator, 2 );

package/vscode/src/vs/workbench/services/treeSitter/browser/treeSitterTokenizationFeature.js

@@ -0,0 +1,212 @@
+import { __decorate, __param } from 'vscode/external/tslib/tslib.es6.js';
+import { Emitter, Event } from 'vscode/vscode/vs/base/common/event';
+import { Disposable, DisposableMap, DisposableStore } from 'vscode/vscode/vs/base/common/lifecycle';
+import { FileAccess } from 'vscode/vscode/vs/base/common/network';
+import { LazyTokenizationSupport, TreeSitterTokenizationRegistry } from 'vscode/vscode/vs/editor/common/languages';
+import { EDITOR_EXPERIMENTAL_PREFER_TREESITTER, ITreeSitterParserService } from 'vscode/vscode/vs/editor/common/services/treeSitterParserService';
+import { ColumnRange } from 'vscode/vscode/vs/editor/contrib/inlineCompletions/browser/utils';
+import { IConfigurationService } from 'vscode/vscode/vs/platform/configuration/common/configuration.service';
+import { IFileService } from 'vscode/vscode/vs/platform/files/common/files.service';
+import { IInstantiationService } from 'vscode/vscode/vs/platform/instantiation/common/instantiation';
+import { IThemeService } from 'vscode/vscode/vs/platform/theme/common/themeService.service';
+import { findMetadata } from 'vscode/vscode/vs/workbench/services/themes/common/colorThemeData';
+import { ILanguageService } from 'vscode/vscode/vs/editor/common/languages/language';
+
+const ALLOWED_SUPPORT = ['typescript'];
+let TreeSitterTokenizationFeature = class TreeSitterTokenizationFeature extends Disposable {
+    constructor(_languageService, _configurationService, _instantiationService, _fileService) {
+        super();
+        this._languageService = _languageService;
+        this._configurationService = _configurationService;
+        this._instantiationService = _instantiationService;
+        this._fileService = _fileService;
+        this._tokenizersRegistrations = ( new DisposableMap());
+        this._handleGrammarsExtPoint();
+        this._register(this._configurationService.onDidChangeConfiguration(e => {
+            if (e.affectsConfiguration(EDITOR_EXPERIMENTAL_PREFER_TREESITTER)) {
+                this._handleGrammarsExtPoint();
+            }
+        }));
+    }
+    _getSetting() {
+        return this._configurationService.getValue(EDITOR_EXPERIMENTAL_PREFER_TREESITTER) || [];
+    }
+    _handleGrammarsExtPoint() {
+        const setting = this._getSetting();
+        for (const languageId of setting) {
+            if (ALLOWED_SUPPORT.includes(languageId) && !( this._tokenizersRegistrations.has(languageId))) {
+                const lazyTokenizationSupport = ( new LazyTokenizationSupport(() => this._createTokenizationSupport(languageId)));
+                const disposableStore = ( new DisposableStore());
+                disposableStore.add(lazyTokenizationSupport);
+                disposableStore.add(TreeSitterTokenizationRegistry.registerFactory(languageId, lazyTokenizationSupport));
+                this._tokenizersRegistrations.set(languageId, disposableStore);
+                TreeSitterTokenizationRegistry.getOrCreate(languageId);
+            }
+        }
+        const languagesToUnregister = [...( this._tokenizersRegistrations.keys())].filter(languageId => !setting.includes(languageId));
+        for (const languageId of languagesToUnregister) {
+            this._tokenizersRegistrations.deleteAndDispose(languageId);
+        }
+    }
+    async _fetchQueries(newLanguage) {
+        const languageLocation = `vs/editor/common/languages/highlights/${newLanguage}.scm`;
+        const query = await this._fileService.readFile(( FileAccess.asFileUri(languageLocation)));
+        return ( query.value.toString());
+    }
+    async _createTokenizationSupport(languageId) {
+        const queries = await this._fetchQueries(languageId);
+        return this._instantiationService.createInstance(TreeSitterTokenizationSupport, queries, languageId, this._languageService.languageIdCodec);
+    }
+};
+TreeSitterTokenizationFeature = ( __decorate([
+    ( __param(0, ILanguageService)),
+    ( __param(1, IConfigurationService)),
+    ( __param(2, IInstantiationService)),
+    ( __param(3, IFileService))
+], TreeSitterTokenizationFeature));
+let TreeSitterTokenizationSupport = class TreeSitterTokenizationSupport extends Disposable {
+    constructor(_queries, _languageId, _languageIdCodec, _treeSitterService, _themeService) {
+        super();
+        this._queries = _queries;
+        this._languageId = _languageId;
+        this._languageIdCodec = _languageIdCodec;
+        this._treeSitterService = _treeSitterService;
+        this._themeService = _themeService;
+        this._onDidChangeTokens = ( new Emitter());
+        this.onDidChangeTokens = this._onDidChangeTokens.event;
+        this._register(Event.runAndSubscribe(this._themeService.onDidColorThemeChange, () => this.reset()));
+        this._register(this._treeSitterService.onDidUpdateTree((e) => {
+            const maxLine = e.textModel.getLineCount();
+            this._onDidChangeTokens.fire({
+                textModel: e.textModel,
+                changes: {
+                    semanticTokensApplied: false,
+                    ranges: ( e.ranges.map(
+                        range => ({ fromLineNumber: range.startLineNumber, toLineNumber: range.endLineNumber < maxLine ? range.endLineNumber : maxLine })
+                    )),
+                }
+            });
+        }));
+    }
+    _getTree(textModel) {
+        return this._treeSitterService.getParseResult(textModel);
+    }
+    _ensureQuery() {
+        if (!this._query) {
+            const language = this._treeSitterService.getOrInitLanguage(this._languageId);
+            if (!language) {
+                if (!this._languageAddedListener) {
+                    this._languageAddedListener = this._register(Event.onceIf(this._treeSitterService.onDidAddLanguage, e => e.id === this._languageId)((e) => {
+                        this._query = e.language.query(this._queries);
+                    }));
+                }
+                return;
+            }
+            this._query = language.query(this._queries);
+        }
+        return this._query;
+    }
+    reset() {
+        this._colorThemeData = this._themeService.getColorTheme();
+    }
+    captureAtPosition(lineNumber, column, textModel) {
+        const tree = this._getTree(textModel);
+        const captures = this._captureAtRange(lineNumber, ( new ColumnRange(column, column)), tree?.tree);
+        return captures;
+    }
+    captureAtPositionTree(lineNumber, column, tree) {
+        const captures = this._captureAtRange(lineNumber, ( new ColumnRange(column, column)), tree);
+        return captures;
+    }
+    _captureAtRange(lineNumber, columnRange, tree) {
+        const query = this._ensureQuery();
+        if (!tree || !query) {
+            return [];
+        }
+        return query.captures(tree.rootNode, { startPosition: { row: lineNumber - 1, column: columnRange.startColumn - 1 }, endPosition: { row: lineNumber - 1, column: columnRange.endColumnExclusive } });
+    }
+    tokenizeEncoded(lineNumber, textModel) {
+        const lineLength = textModel.getLineMaxColumn(lineNumber);
+        const tree = this._getTree(textModel);
+        const captures = this._captureAtRange(lineNumber, ( new ColumnRange(1, lineLength)), tree?.tree);
+        if (captures.length === 0) {
+            return undefined;
+        }
+        const endOffsetsAndScopes = Array(captures.length);
+        endOffsetsAndScopes.fill({ endOffset: 0, scopes: [] });
+        let tokenIndex = 0;
+        const lineStartOffset = textModel.getOffsetAt({ lineNumber: lineNumber, column: 1 });
+        const increaseSizeOfTokensByOneToken = () => {
+            endOffsetsAndScopes.push({ endOffset: 0, scopes: [] });
+        };
+        const encodedLanguageId = this._languageIdCodec.encodeLanguageId(this._languageId);
+        for (let captureIndex = 0; captureIndex < captures.length; captureIndex++) {
+            const capture = captures[captureIndex];
+            const tokenEndIndex = capture.node.endIndex < lineStartOffset + lineLength ? capture.node.endIndex : lineStartOffset + lineLength;
+            const tokenStartIndex = capture.node.startIndex < lineStartOffset ? lineStartOffset : capture.node.startIndex;
+            const lineRelativeOffset = tokenEndIndex - lineStartOffset;
+            let previousTokenEnd;
+            const currentTokenLength = tokenEndIndex - tokenStartIndex;
+            if (captureIndex > 0) {
+                previousTokenEnd = endOffsetsAndScopes[(tokenIndex - 1)].endOffset;
+            }
+            else {
+                previousTokenEnd = tokenStartIndex - lineStartOffset - 1;
+            }
+            const intermediateTokenOffset = lineRelativeOffset - currentTokenLength;
+            if ((previousTokenEnd >= 0) && (previousTokenEnd < intermediateTokenOffset)) {
+                endOffsetsAndScopes[tokenIndex] = { endOffset: intermediateTokenOffset, scopes: [] };
+                tokenIndex++;
+                increaseSizeOfTokensByOneToken();
+            }
+            const addCurrentTokenToArray = () => {
+                endOffsetsAndScopes[tokenIndex] = { endOffset: lineRelativeOffset, scopes: [capture.name] };
+                tokenIndex++;
+            };
+            if (previousTokenEnd >= lineRelativeOffset) {
+                const previousTokenStartOffset = ((tokenIndex >= 2) ? endOffsetsAndScopes[tokenIndex - 2].endOffset : 0);
+                const originalPreviousTokenEndOffset = endOffsetsAndScopes[tokenIndex - 1].endOffset;
+                if ((previousTokenStartOffset + currentTokenLength) === originalPreviousTokenEndOffset) {
+                    endOffsetsAndScopes[tokenIndex - 1].scopes.push(capture.name);
+                }
+                else {
+                    endOffsetsAndScopes[tokenIndex - 1].endOffset = intermediateTokenOffset;
+                    addCurrentTokenToArray();
+                    increaseSizeOfTokensByOneToken();
+                    endOffsetsAndScopes[tokenIndex].endOffset = originalPreviousTokenEndOffset;
+                    endOffsetsAndScopes[tokenIndex].scopes = endOffsetsAndScopes[tokenIndex - 2].scopes;
+                    tokenIndex++;
+                }
+            }
+            else {
+                addCurrentTokenToArray();
+            }
+        }
+        if (captures[captures.length - 1].node.endPosition.column + 1 < lineLength) {
+            increaseSizeOfTokensByOneToken();
+            endOffsetsAndScopes[tokenIndex].endOffset = lineLength - 1;
+            tokenIndex++;
+        }
+        const tokens = ( new Uint32Array((tokenIndex) * 2));
+        for (let i = 0; i < tokenIndex; i++) {
+            const token = endOffsetsAndScopes[i];
+            if (token.endOffset === 0 && token.scopes.length === 0) {
+                break;
+            }
+            tokens[i * 2] = token.endOffset;
+            tokens[i * 2 + 1] = findMetadata(this._colorThemeData, token.scopes, encodedLanguageId);
+        }
+        return tokens;
+    }
+    dispose() {
+        super.dispose();
+        this._query?.delete();
+        this._query = undefined;
+    }
+};
+TreeSitterTokenizationSupport = ( __decorate([
+    ( __param(3, ITreeSitterParserService)),
+    ( __param(4, IThemeService))
+], TreeSitterTokenizationSupport));
+
+export { TreeSitterTokenizationFeature };