monaco-editor-core 0.55.0-dev-20251022 → 0.55.0-dev-20251024

package/esm/vs/editor/common/model/tokens/treeSitter/treeSitterSyntaxTokenBackend.js

@@ -40,7 +40,7 @@ let TreeSitterSyntaxTokenBackend = class TreeSitterSyntaxTokenBackend extends Ab
                 return undefined;
             }
             const currentLanguage = this._languageIdObs.read(reader);
-            const treeSitterLang = this._treeSitterLibraryService.getLanguage(currentLanguage, reader);
+            const treeSitterLang = this._treeSitterLibraryService.getLanguage(currentLanguage, false, reader);
             if (!treeSitterLang) {
                 return undefined;
             }

package/esm/vs/editor/common/model/tokens/treeSitter/treeSitterSyntaxTokenBackend.js.map

@@ -1 +1 @@
-{"version":3,"sources":["file:///mnt/vss/_work/1/s/dependencies/vscode/out-editor-src/vs/editor/common/model/tokens/treeSitter/treeSitterSyntaxTokenBackend.ts","vs/editor/common/model/tokens/treeSitter/treeSitterSyntaxTokenBackend.ts"],"names":[],"mappings":"AAAA;;;gGAGgG;;;;;;;;;;AAEhG,OAAO,EAAE,OAAO,EAAS,MAAM,qCAAqC,CAAC;AACrE,OAAO,EAAE,YAAY,EAAE,MAAM,yCAAyC,CAAC;AAKvE,OAAO,EAAE,UAAU,EAAE,MAAM,+BAA+B,CAAC;AAE3D,OAAO,EAAE,0BAA0B,EAAE,MAAM,kCAAkC,CAAC;AAC9E,OAAO,EAAE,OAAO,EAAE,OAAO,EAAe,iBAAiB,EAAE,MAAM,0CAA0C,CAAC;AAC5G,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EAAE,qBAAqB,EAAE,MAAM,+DAA+D,CAAC;AACtG,OAAO,EAAE,0BAA0B,EAAE,MAAM,iCAAiC,CAAC;AAC7E,OAAO,EAAE,yBAAyB,EAAE,MAAM,0DAA0D,CAAC;AAG9F,IAAM,4BAA4B,GAAlC,MAAM,4BAA6B,SAAQ,0BAA0B;IAQ3E,YACkB,cAAmC,EACpD,eAAiC,EACjC,SAAoB,EACpB,iBAAoD,EACzB,yBAAqE,EACzE,qBAA6D;QAEpF,KAAK,CAAC,eAAe,EAAE,SAAS,CAAC,CAAC;QAPjB,mBAAc,GAAd,cAAc,CAAqB;QAIR,8BAAyB,GAAzB,yBAAyB,CAA2B;QACxD,0BAAqB,GAArB,qBAAqB,CAAuB;QAb3E,iCAA4B,kDAAuE;QAC1F,4CAAuC,GAAkB,IAAI,CAAC,SAAS,CAAC,IAAI,OAAO,EAAQ,CAAC,CAAC;QAChG,2CAAsC,GAAgB,IAAI,CAAC,uCAAuC,CAAC,KAAK,CAAC;QAgBxH,MAAM,kBAAkB,GAAG,IAAI,iBAAiB,CAAC,IAAI,CAAC,yBAAyB,CAAC,cAAc,EAAE,CAAC,CAAC;QAGlG,MAAM,cAAc,GAAG,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE;YAC7C,MAAM,MAAM,GAAG,kBAAkB,CAAC,aAAa,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,cAAc,EAAE,CAAC;YAChF,OAAO,MAAM,CAAC;QACf,CAAC,CAAC,CAAC;QAGH,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE;YACnC,MAAM,WAAW,GAAG,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAChD,IAAI,CAAC,WAAW,EAAE,CAAC;gBAClB,OAAO,SAAS,CAAC;YAClB,CAAC;YAED,MAAM,eAAe,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACzD,MAAM,cAAc,GAAG,IAAI,CAAC,yBAAyB,CAAC,WAAW,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;YAC3F,IAAI,CAAC,cAAc,EAAE,CAAC;gBACrB,OAAO,SAAS,CAAC;YAClB,CAAC;YAED,MAAM,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;YACjC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,YAAY,CAAC,GAAG,EAAE;gBAClC,MAAM,CAAC,MAAM,EAAE,CAAC;YACjB,CAAC,CAAC,CAAC,CAAC;YACJ,MAAM,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC;YAEnC,MAAM,OAAO,GAAG,IAAI,CAAC,yBAAyB,CAAC,mBAAmB,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;YAC5F,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;gBAC3B,OAAO,SAAS,CAAC;YAClB,CAAC;YAED,OAAO,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,qBAAqB,CAAC,cAAc,CAAC,cAAc,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,EAAE,WAAW,EAAE,aAAa,CAAA,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;QACnK,CAAC,CAAC,CAAC;QAGH,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE;YAC/C,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC1C,IAAI,CAAC,SAAS,EAAE,CAAC;gBAChB,OAAO,SAAS,CAAC;YAClB,CAAC;YAED,MAAM,OAAO,GAAG,IAAI,CAAC,yBAAyB,CAAC,sBAAsB,CAAC,SAAS,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;YACpG,IAAI,CAAC,OAAO,EAAE,CAAC;gBACd,OAAO,SAAS,CAAC;YAClB,CAAC;YAED,OAAO,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,qBAAqB,CAAC,cAAc,CAAC,0BAA0B,EAAE,SAAS,EAAE,OAAO,EAAE,IAAI,CAAC,gBAAgB,EAAE,iBAAiB,CAAC,CAAC,CAAC;QAC9J,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE;YAC/B,MAAM,QAAQ,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACrD,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACf,OAAO;YACR,CAAC;YACD,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,EAAE;gBACjD,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YACzC,CAAC,CAAC,CAAC,CAAC;YACJ,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,iCAAiC,CAAC,CAAC,CAAC,EAAE;gBAC/D,IAAI,CAAC,4BAA4B,gDAAwC,CAAC;gBAC1E,IAAI,CAAC,uCAAuC,CAAC,IAAI,EAAE,CAAC;YACrD,CAAC,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC,CAAC;IACL,CAAC;IAEM,aAAa,CAAC,UAAkB;QACtC,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,EAAE,CAAC;QAC3C,IAAI,CAAC,KAAK,EAAE,CAAC;YACZ,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC;YAC3D,OAAO,UAAU,CAAC,WAAW,CAAC,OAAO,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAC/D,CAAC;QACD,OAAO,KAAK,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;IACxC,CAAC;IAEM,sBAAsB,CAAC,uBAAgC,IAAI;QACjE,IAAI,oBAAoB,EAAE,CAAC;YAC1B,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC;gBAC5B,qBAAqB,EAAE,KAAK;gBAC5B,MAAM,EAAE;oBACP;wBACC,cAAc,EAAE,CAAC;wBACjB,YAAY,EAAE,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE;qBAC5C;iBACD;aACD,CAAC,CAAC;QACJ,CAAC;IACF,CAAC;IAEe,uBAAuB;QACtC,sDAAsD;IACvD,CAAC;IAEe,sBAAsB,CAAC,CAA4B;QAClE,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;YACf,iFAAiF;YACjF,IAAI,CAAC,sBAAsB,CAAC,KAAK,CAAC,CAAC;QACpC,CAAC;aAAM,CAAC;YACP,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,EAAE,CAAC;YAC3C,KAAK,EAAE,oBAAoB,CAAC,CAAC,CAAC,CAAC;QAChC,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;QACnC,SAAS,EAAE,mBAAmB,CAAC,CAAC,CAAC,CAAC;IACnC,CAAC;IAEe,iBAAiB,CAAC,UAAkB;QACnD,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,EAAE,CAAC;QAC3C,IAAI,CAAC,KAAK,EAAE,CAAC;YACZ,OAAO;QACR,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,wBAAwB,CAAC,UAAU,CAAC,EAAE,CAAC;YACjD,KAAK,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC;QACnC,CAAC;IACF,CAAC;IAEe,wBAAwB,CAAC,UAAkB;QAC1D,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,EAAE,CAAC;QAC3C,IAAI,CAAC,KAAK,EAAE,CAAC;YACZ,OAAO,KAAK,CAAC;QACd,CAAC;QACD,OAAO,KAAK,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC;IACnD,CAAC;IAEe,iBAAiB,CAAC,UAAkB;QACnD,2DAA2D;QAC3D,OAAO,IAAI,CAAC;IACb,CAAC;IAEe,gCAAgC,CAAC,UAAkB,EAAE,MAAc,EAAE,SAAiB;QACrG,wGAAwG;QACxG,uCAA+B;IAChC,CAAC;IAEe,eAAe,CAAC,UAAkB,EAAE,KAAe;QAClE,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,EAAE,CAAC;QAC3C,IAAI,CAAC,KAAK,EAAE,CAAC;YACZ,OAAO,IAAI,CAAC;QACb,CAAC;QACD,OAAO,KAAK,CAAC,eAAe,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;IACjD,CAAC;IAED,IAAoB,SAAS;QAC5B,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,EAAE,CAAC;QAC3C,IAAI,CAAC,KAAK,EAAE,CAAC;YACZ,OAAO,KAAK,CAAC;QACd,CAAC;QACD,OAAO,KAAK,CAAC,SAAS,EAAE,CAAC;IAC1B,CAAC;CACD,CAAA;AAvKY,4BAA4B;IAatC,WAAA,yBAAyB,CAAA;IACzB,WAAA,qBAAqB,CAAA;GAdX,4BAA4B,CAuKxC","file":"treeSitterSyntaxTokenBackend.js","sourceRoot":"file:///mnt/vss/_work/1/s/dependencies/vscode/out-editor-src","sourcesContent":["/*---------------------------------------------------------------------------------------------\n *  Copyright (c) Microsoft Corporation. All rights reserved.\n *  Licensed under the MIT License. See License.txt in the project root for license information.\n *--------------------------------------------------------------------------------------------*/\n\nimport { Emitter, Event } from '../../../../../base/common/event.js';\nimport { toDisposable } from '../../../../../base/common/lifecycle.js';\nimport { StandardTokenType } from '../../../encodedTokenAttributes.js';\nimport { ILanguageIdCodec } from '../../../languages.js';\nimport { IModelContentChangedEvent } from '../../../textModelEvents.js';\nimport { BackgroundTokenizationState } from '../../../tokenizationTextModelPart.js';\nimport { LineTokens } from '../../../tokens/lineTokens.js';\nimport { TextModel } from '../../textModel.js';\nimport { AbstractSyntaxTokenBackend } from '../abstractSyntaxTokenBackend.js';\nimport { autorun, derived, IObservable, ObservablePromise } from '../../../../../base/common/observable.js';\nimport { TreeSitterTree } from './treeSitterTree.js';\nimport { IInstantiationService } from '../../../../../platform/instantiation/common/instantiation.js';\nimport { TreeSitterTokenizationImpl } from './treeSitterTokenizationImpl.js';\nimport { ITreeSitterLibraryService } from '../../../services/treeSitter/treeSitterLibraryService.js';\nimport { LineRange } from '../../../core/ranges/lineRange.js';\n\nexport class TreeSitterSyntaxTokenBackend extends AbstractSyntaxTokenBackend {\n\tprotected _backgroundTokenizationState: BackgroundTokenizationState = BackgroundTokenizationState.InProgress;\n\tprotected readonly _onDidChangeBackgroundTokenizationState: Emitter<void> = this._register(new Emitter<void>());\n\tpublic readonly onDidChangeBackgroundTokenizationState: Event<void> = this._onDidChangeBackgroundTokenizationState.event;\n\n\tprivate readonly _tree: IObservable<TreeSitterTree | undefined>;\n\tprivate readonly _tokenizationImpl: IObservable<TreeSitterTokenizationImpl | undefined>;\n\n\tconstructor(\n\t\tprivate readonly _languageIdObs: IObservable<string>,\n\t\tlanguageIdCodec: ILanguageIdCodec,\n\t\ttextModel: TextModel,\n\t\tvisibleLineRanges: IObservable<readonly LineRange[]>,\n\t\t@ITreeSitterLibraryService private readonly _treeSitterLibraryService: ITreeSitterLibraryService,\n\t\t@IInstantiationService private readonly _instantiationService: IInstantiationService\n\t) {\n\t\tsuper(languageIdCodec, textModel);\n\n\n\t\tconst parserClassPromise = new ObservablePromise(this._treeSitterLibraryService.getParserClass());\n\n\n\t\tconst parserClassObs = derived(this, reader => {\n\t\t\tconst parser = parserClassPromise.promiseResult?.read(reader)?.getDataOrThrow();\n\t\t\treturn parser;\n\t\t});\n\n\n\t\tthis._tree = derived(this, reader => {\n\t\t\tconst parserClass = parserClassObs.read(reader);\n\t\t\tif (!parserClass) {\n\t\t\t\treturn undefined;\n\t\t\t}\n\n\t\t\tconst currentLanguage = this._languageIdObs.read(reader);\n\t\t\tconst treeSitterLang = this._treeSitterLibraryService.getLanguage(currentLanguage, reader);\n\t\t\tif (!treeSitterLang) {\n\t\t\t\treturn undefined;\n\t\t\t}\n\n\t\t\tconst parser = new parserClass();\n\t\t\treader.store.add(toDisposable(() => {\n\t\t\t\tparser.delete();\n\t\t\t}));\n\t\t\tparser.setLanguage(treeSitterLang);\n\n\t\t\tconst queries = this._treeSitterLibraryService.getInjectionQueries(currentLanguage, reader);\n\t\t\tif (queries === undefined) {\n\t\t\t\treturn undefined;\n\t\t\t}\n\n\t\t\treturn reader.store.add(this._instantiationService.createInstance(TreeSitterTree, currentLanguage, undefined, parser, parserClass, /*queries, */this._textModel));\n\t\t});\n\n\n\t\tthis._tokenizationImpl = derived(this, reader => {\n\t\t\tconst treeModel = this._tree.read(reader);\n\t\t\tif (!treeModel) {\n\t\t\t\treturn undefined;\n\t\t\t}\n\n\t\t\tconst queries = this._treeSitterLibraryService.getHighlightingQueries(treeModel.languageId, reader);\n\t\t\tif (!queries) {\n\t\t\t\treturn undefined;\n\t\t\t}\n\n\t\t\treturn reader.store.add(this._instantiationService.createInstance(TreeSitterTokenizationImpl, treeModel, queries, this._languageIdCodec, visibleLineRanges));\n\t\t});\n\n\t\tthis._register(autorun(reader => {\n\t\t\tconst tokModel = this._tokenizationImpl.read(reader);\n\t\t\tif (!tokModel) {\n\t\t\t\treturn;\n\t\t\t}\n\t\t\treader.store.add(tokModel.onDidChangeTokens((e) => {\n\t\t\t\tthis._onDidChangeTokens.fire(e.changes);\n\t\t\t}));\n\t\t\treader.store.add(tokModel.onDidChangeBackgroundTokenization(e => {\n\t\t\t\tthis._backgroundTokenizationState = BackgroundTokenizationState.Completed;\n\t\t\t\tthis._onDidChangeBackgroundTokenizationState.fire();\n\t\t\t}));\n\t\t}));\n\t}\n\n\tpublic getLineTokens(lineNumber: number): LineTokens {\n\t\tconst model = this._tokenizationImpl.get();\n\t\tif (!model) {\n\t\t\tconst content = this._textModel.getLineContent(lineNumber);\n\t\t\treturn LineTokens.createEmpty(content, this._languageIdCodec);\n\t\t}\n\t\treturn model.getLineTokens(lineNumber);\n\t}\n\n\tpublic todo_resetTokenization(fireTokenChangeEvent: boolean = true): void {\n\t\tif (fireTokenChangeEvent) {\n\t\t\tthis._onDidChangeTokens.fire({\n\t\t\t\tsemanticTokensApplied: false,\n\t\t\t\tranges: [\n\t\t\t\t\t{\n\t\t\t\t\t\tfromLineNumber: 1,\n\t\t\t\t\t\ttoLineNumber: this._textModel.getLineCount(),\n\t\t\t\t\t},\n\t\t\t\t],\n\t\t\t});\n\t\t}\n\t}\n\n\tpublic override handleDidChangeAttached(): void {\n\t\t// TODO @alexr00 implement for background tokenization\n\t}\n\n\tpublic override handleDidChangeContent(e: IModelContentChangedEvent): void {\n\t\tif (e.isFlush) {\n\t\t\t// Don't fire the event, as the view might not have got the text change event yet\n\t\t\tthis.todo_resetTokenization(false);\n\t\t} else {\n\t\t\tconst model = this._tokenizationImpl.get();\n\t\t\tmodel?.handleContentChanged(e);\n\t\t}\n\n\t\tconst treeModel = this._tree.get();\n\t\ttreeModel?.handleContentChange(e);\n\t}\n\n\tpublic override forceTokenization(lineNumber: number): void {\n\t\tconst model = this._tokenizationImpl.get();\n\t\tif (!model) {\n\t\t\treturn;\n\t\t}\n\t\tif (!model.hasAccurateTokensForLine(lineNumber)) {\n\t\t\tmodel.tokenizeEncoded(lineNumber);\n\t\t}\n\t}\n\n\tpublic override hasAccurateTokensForLine(lineNumber: number): boolean {\n\t\tconst model = this._tokenizationImpl.get();\n\t\tif (!model) {\n\t\t\treturn false;\n\t\t}\n\t\treturn model.hasAccurateTokensForLine(lineNumber);\n\t}\n\n\tpublic override isCheapToTokenize(lineNumber: number): boolean {\n\t\t// TODO @alexr00 determine what makes it cheap to tokenize?\n\t\treturn true;\n\t}\n\n\tpublic override getTokenTypeIfInsertingCharacter(lineNumber: number, column: number, character: string): StandardTokenType {\n\t\t// TODO @alexr00 implement once we have custom parsing and don't just feed in the whole text model value\n\t\treturn StandardTokenType.Other;\n\t}\n\n\tpublic override tokenizeLinesAt(lineNumber: number, lines: string[]): LineTokens[] | null {\n\t\tconst model = this._tokenizationImpl.get();\n\t\tif (!model) {\n\t\t\treturn null;\n\t\t}\n\t\treturn model.tokenizeLinesAt(lineNumber, lines);\n\t}\n\n\tpublic override get hasTokens(): boolean {\n\t\tconst model = this._tokenizationImpl.get();\n\t\tif (!model) {\n\t\t\treturn false;\n\t\t}\n\t\treturn model.hasTokens();\n\t}\n}\n","/*---------------------------------------------------------------------------------------------\n *  Copyright (c) Microsoft Corporation. All rights reserved.\n *  Licensed under the MIT License. See License.txt in the project root for license information.\n *--------------------------------------------------------------------------------------------*/\n\nimport { Emitter, Event } from '../../../../../base/common/event.js';\nimport { toDisposable } from '../../../../../base/common/lifecycle.js';\nimport { StandardTokenType } from '../../../encodedTokenAttributes.js';\nimport { ILanguageIdCodec } from '../../../languages.js';\nimport { IModelContentChangedEvent } from '../../../textModelEvents.js';\nimport { BackgroundTokenizationState } from '../../../tokenizationTextModelPart.js';\nimport { LineTokens } from '../../../tokens/lineTokens.js';\nimport { TextModel } from '../../textModel.js';\nimport { AbstractSyntaxTokenBackend } from '../abstractSyntaxTokenBackend.js';\nimport { autorun, derived, IObservable, ObservablePromise } from '../../../../../base/common/observable.js';\nimport { TreeSitterTree } from './treeSitterTree.js';\nimport { IInstantiationService } from '../../../../../platform/instantiation/common/instantiation.js';\nimport { TreeSitterTokenizationImpl } from './treeSitterTokenizationImpl.js';\nimport { ITreeSitterLibraryService } from '../../../services/treeSitter/treeSitterLibraryService.js';\nimport { LineRange } from '../../../core/ranges/lineRange.js';\n\nexport class TreeSitterSyntaxTokenBackend extends AbstractSyntaxTokenBackend {\n\tprotected _backgroundTokenizationState: BackgroundTokenizationState = BackgroundTokenizationState.InProgress;\n\tprotected readonly _onDidChangeBackgroundTokenizationState: Emitter<void> = this._register(new Emitter<void>());\n\tpublic readonly onDidChangeBackgroundTokenizationState: Event<void> = this._onDidChangeBackgroundTokenizationState.event;\n\n\tprivate readonly _tree: IObservable<TreeSitterTree | undefined>;\n\tprivate readonly _tokenizationImpl: IObservable<TreeSitterTokenizationImpl | undefined>;\n\n\tconstructor(\n\t\tprivate readonly _languageIdObs: IObservable<string>,\n\t\tlanguageIdCodec: ILanguageIdCodec,\n\t\ttextModel: TextModel,\n\t\tvisibleLineRanges: IObservable<readonly LineRange[]>,\n\t\t@ITreeSitterLibraryService private readonly _treeSitterLibraryService: ITreeSitterLibraryService,\n\t\t@IInstantiationService private readonly _instantiationService: IInstantiationService\n\t) {\n\t\tsuper(languageIdCodec, textModel);\n\n\n\t\tconst parserClassPromise = new ObservablePromise(this._treeSitterLibraryService.getParserClass());\n\n\n\t\tconst parserClassObs = derived(this, reader => {\n\t\t\tconst parser = parserClassPromise.promiseResult?.read(reader)?.getDataOrThrow();\n\t\t\treturn parser;\n\t\t});\n\n\n\t\tthis._tree = derived(this, reader => {\n\t\t\tconst parserClass = parserClassObs.read(reader);\n\t\t\tif (!parserClass) {\n\t\t\t\treturn undefined;\n\t\t\t}\n\n\t\t\tconst currentLanguage = this._languageIdObs.read(reader);\n\t\t\tconst treeSitterLang = this._treeSitterLibraryService.getLanguage(currentLanguage, reader);\n\t\t\tif (!treeSitterLang) {\n\t\t\t\treturn undefined;\n\t\t\t}\n\n\t\t\tconst parser = new parserClass();\n\t\t\treader.store.add(toDisposable(() => {\n\t\t\t\tparser.delete();\n\t\t\t}));\n\t\t\tparser.setLanguage(treeSitterLang);\n\n\t\t\tconst queries = this._treeSitterLibraryService.getInjectionQueries(currentLanguage, reader);\n\t\t\tif (queries === undefined) {\n\t\t\t\treturn undefined;\n\t\t\t}\n\n\t\t\treturn reader.store.add(this._instantiationService.createInstance(TreeSitterTree, currentLanguage, undefined, parser, parserClass, /*queries, */this._textModel));\n\t\t});\n\n\n\t\tthis._tokenizationImpl = derived(this, reader => {\n\t\t\tconst treeModel = this._tree.read(reader);\n\t\t\tif (!treeModel) {\n\t\t\t\treturn undefined;\n\t\t\t}\n\n\t\t\tconst queries = this._treeSitterLibraryService.getHighlightingQueries(treeModel.languageId, reader);\n\t\t\tif (!queries) {\n\t\t\t\treturn undefined;\n\t\t\t}\n\n\t\t\treturn reader.store.add(this._instantiationService.createInstance(TreeSitterTokenizationImpl, treeModel, queries, this._languageIdCodec, visibleLineRanges));\n\t\t});\n\n\t\tthis._register(autorun(reader => {\n\t\t\tconst tokModel = this._tokenizationImpl.read(reader);\n\t\t\tif (!tokModel) {\n\t\t\t\treturn;\n\t\t\t}\n\t\t\treader.store.add(tokModel.onDidChangeTokens((e) => {\n\t\t\t\tthis._onDidChangeTokens.fire(e.changes);\n\t\t\t}));\n\t\t\treader.store.add(tokModel.onDidChangeBackgroundTokenization(e => {\n\t\t\t\tthis._backgroundTokenizationState = BackgroundTokenizationState.Completed;\n\t\t\t\tthis._onDidChangeBackgroundTokenizationState.fire();\n\t\t\t}));\n\t\t}));\n\t}\n\n\tpublic getLineTokens(lineNumber: number): LineTokens {\n\t\tconst model = this._tokenizationImpl.get();\n\t\tif (!model) {\n\t\t\tconst content = this._textModel.getLineContent(lineNumber);\n\t\t\treturn LineTokens.createEmpty(content, this._languageIdCodec);\n\t\t}\n\t\treturn model.getLineTokens(lineNumber);\n\t}\n\n\tpublic todo_resetTokenization(fireTokenChangeEvent: boolean = true): void {\n\t\tif (fireTokenChangeEvent) {\n\t\t\tthis._onDidChangeTokens.fire({\n\t\t\t\tsemanticTokensApplied: false,\n\t\t\t\tranges: [\n\t\t\t\t\t{\n\t\t\t\t\t\tfromLineNumber: 1,\n\t\t\t\t\t\ttoLineNumber: this._textModel.getLineCount(),\n\t\t\t\t\t},\n\t\t\t\t],\n\t\t\t});\n\t\t}\n\t}\n\n\tpublic override handleDidChangeAttached(): void {\n\t\t// TODO @alexr00 implement for background tokenization\n\t}\n\n\tpublic override handleDidChangeContent(e: IModelContentChangedEvent): void {\n\t\tif (e.isFlush) {\n\t\t\t// Don't fire the event, as the view might not have got the text change event yet\n\t\t\tthis.todo_resetTokenization(false);\n\t\t} else {\n\t\t\tconst model = this._tokenizationImpl.get();\n\t\t\tmodel?.handleContentChanged(e);\n\t\t}\n\n\t\tconst treeModel = this._tree.get();\n\t\ttreeModel?.handleContentChange(e);\n\t}\n\n\tpublic override forceTokenization(lineNumber: number): void {\n\t\tconst model = this._tokenizationImpl.get();\n\t\tif (!model) {\n\t\t\treturn;\n\t\t}\n\t\tif (!model.hasAccurateTokensForLine(lineNumber)) {\n\t\t\tmodel.tokenizeEncoded(lineNumber);\n\t\t}\n\t}\n\n\tpublic override hasAccurateTokensForLine(lineNumber: number): boolean {\n\t\tconst model = this._tokenizationImpl.get();\n\t\tif (!model) {\n\t\t\treturn false;\n\t\t}\n\t\treturn model.hasAccurateTokensForLine(lineNumber);\n\t}\n\n\tpublic override isCheapToTokenize(lineNumber: number): boolean {\n\t\t// TODO @alexr00 determine what makes it cheap to tokenize?\n\t\treturn true;\n\t}\n\n\tpublic override getTokenTypeIfInsertingCharacter(lineNumber: number, column: number, character: string): StandardTokenType {\n\t\t// TODO @alexr00 implement once we have custom parsing and don't just feed in the whole text model value\n\t\treturn StandardTokenType.Other;\n\t}\n\n\tpublic override tokenizeLinesAt(lineNumber: number, lines: string[]): LineTokens[] | null {\n\t\tconst model = this._tokenizationImpl.get();\n\t\tif (!model) {\n\t\t\treturn null;\n\t\t}\n\t\treturn model.tokenizeLinesAt(lineNumber, lines);\n\t}\n\n\tpublic override get hasTokens(): boolean {\n\t\tconst model = this._tokenizationImpl.get();\n\t\tif (!model) {\n\t\t\treturn false;\n\t\t}\n\t\treturn model.hasTokens();\n\t}\n}\n"]}
+{"version":3,"sources":["file:///mnt/vss/_work/1/s/dependencies/vscode/out-editor-src/vs/editor/common/model/tokens/treeSitter/treeSitterSyntaxTokenBackend.ts","vs/editor/common/model/tokens/treeSitter/treeSitterSyntaxTokenBackend.ts"],"names":[],"mappings":"AAAA;;;gGAGgG;;;;;;;;;;AAEhG,OAAO,EAAE,OAAO,EAAS,MAAM,qCAAqC,CAAC;AACrE,OAAO,EAAE,YAAY,EAAE,MAAM,yCAAyC,CAAC;AAKvE,OAAO,EAAE,UAAU,EAAE,MAAM,+BAA+B,CAAC;AAE3D,OAAO,EAAE,0BAA0B,EAAE,MAAM,kCAAkC,CAAC;AAC9E,OAAO,EAAE,OAAO,EAAE,OAAO,EAAe,iBAAiB,EAAE,MAAM,0CAA0C,CAAC;AAC5G,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EAAE,qBAAqB,EAAE,MAAM,+DAA+D,CAAC;AACtG,OAAO,EAAE,0BAA0B,EAAE,MAAM,iCAAiC,CAAC;AAC7E,OAAO,EAAE,yBAAyB,EAAE,MAAM,0DAA0D,CAAC;AAG9F,IAAM,4BAA4B,GAAlC,MAAM,4BAA6B,SAAQ,0BAA0B;IAQ3E,YACkB,cAAmC,EACpD,eAAiC,EACjC,SAAoB,EACpB,iBAAoD,EACzB,yBAAqE,EACzE,qBAA6D;QAEpF,KAAK,CAAC,eAAe,EAAE,SAAS,CAAC,CAAC;QAPjB,mBAAc,GAAd,cAAc,CAAqB;QAIR,8BAAyB,GAAzB,yBAAyB,CAA2B;QACxD,0BAAqB,GAArB,qBAAqB,CAAuB;QAb3E,iCAA4B,kDAAuE;QAC1F,4CAAuC,GAAkB,IAAI,CAAC,SAAS,CAAC,IAAI,OAAO,EAAQ,CAAC,CAAC;QAChG,2CAAsC,GAAgB,IAAI,CAAC,uCAAuC,CAAC,KAAK,CAAC;QAgBxH,MAAM,kBAAkB,GAAG,IAAI,iBAAiB,CAAC,IAAI,CAAC,yBAAyB,CAAC,cAAc,EAAE,CAAC,CAAC;QAGlG,MAAM,cAAc,GAAG,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE;YAC7C,MAAM,MAAM,GAAG,kBAAkB,CAAC,aAAa,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,cAAc,EAAE,CAAC;YAChF,OAAO,MAAM,CAAC;QACf,CAAC,CAAC,CAAC;QAGH,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE;YACnC,MAAM,WAAW,GAAG,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAChD,IAAI,CAAC,WAAW,EAAE,CAAC;gBAClB,OAAO,SAAS,CAAC;YAClB,CAAC;YAED,MAAM,eAAe,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACzD,MAAM,cAAc,GAAG,IAAI,CAAC,yBAAyB,CAAC,WAAW,CAAC,eAAe,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;YAClG,IAAI,CAAC,cAAc,EAAE,CAAC;gBACrB,OAAO,SAAS,CAAC;YAClB,CAAC;YAED,MAAM,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;YACjC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,YAAY,CAAC,GAAG,EAAE;gBAClC,MAAM,CAAC,MAAM,EAAE,CAAC;YACjB,CAAC,CAAC,CAAC,CAAC;YACJ,MAAM,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC;YAEnC,MAAM,OAAO,GAAG,IAAI,CAAC,yBAAyB,CAAC,mBAAmB,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;YAC5F,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;gBAC3B,OAAO,SAAS,CAAC;YAClB,CAAC;YAED,OAAO,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,qBAAqB,CAAC,cAAc,CAAC,cAAc,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,EAAE,WAAW,EAAE,aAAa,CAAA,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;QACnK,CAAC,CAAC,CAAC;QAGH,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE;YAC/C,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC1C,IAAI,CAAC,SAAS,EAAE,CAAC;gBAChB,OAAO,SAAS,CAAC;YAClB,CAAC;YAED,MAAM,OAAO,GAAG,IAAI,CAAC,yBAAyB,CAAC,sBAAsB,CAAC,SAAS,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;YACpG,IAAI,CAAC,OAAO,EAAE,CAAC;gBACd,OAAO,SAAS,CAAC;YAClB,CAAC;YAED,OAAO,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,qBAAqB,CAAC,cAAc,CAAC,0BAA0B,EAAE,SAAS,EAAE,OAAO,EAAE,IAAI,CAAC,gBAAgB,EAAE,iBAAiB,CAAC,CAAC,CAAC;QAC9J,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE;YAC/B,MAAM,QAAQ,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACrD,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACf,OAAO;YACR,CAAC;YACD,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,EAAE;gBACjD,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YACzC,CAAC,CAAC,CAAC,CAAC;YACJ,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,iCAAiC,CAAC,CAAC,CAAC,EAAE;gBAC/D,IAAI,CAAC,4BAA4B,gDAAwC,CAAC;gBAC1E,IAAI,CAAC,uCAAuC,CAAC,IAAI,EAAE,CAAC;YACrD,CAAC,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC,CAAC;IACL,CAAC;IAEM,aAAa,CAAC,UAAkB;QACtC,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,EAAE,CAAC;QAC3C,IAAI,CAAC,KAAK,EAAE,CAAC;YACZ,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC;YAC3D,OAAO,UAAU,CAAC,WAAW,CAAC,OAAO,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAC/D,CAAC;QACD,OAAO,KAAK,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;IACxC,CAAC;IAEM,sBAAsB,CAAC,uBAAgC,IAAI;QACjE,IAAI,oBAAoB,EAAE,CAAC;YAC1B,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC;gBAC5B,qBAAqB,EAAE,KAAK;gBAC5B,MAAM,EAAE;oBACP;wBACC,cAAc,EAAE,CAAC;wBACjB,YAAY,EAAE,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE;qBAC5C;iBACD;aACD,CAAC,CAAC;QACJ,CAAC;IACF,CAAC;IAEe,uBAAuB;QACtC,sDAAsD;IACvD,CAAC;IAEe,sBAAsB,CAAC,CAA4B;QAClE,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;YACf,iFAAiF;YACjF,IAAI,CAAC,sBAAsB,CAAC,KAAK,CAAC,CAAC;QACpC,CAAC;aAAM,CAAC;YACP,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,EAAE,CAAC;YAC3C,KAAK,EAAE,oBAAoB,CAAC,CAAC,CAAC,CAAC;QAChC,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;QACnC,SAAS,EAAE,mBAAmB,CAAC,CAAC,CAAC,CAAC;IACnC,CAAC;IAEe,iBAAiB,CAAC,UAAkB;QACnD,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,EAAE,CAAC;QAC3C,IAAI,CAAC,KAAK,EAAE,CAAC;YACZ,OAAO;QACR,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,wBAAwB,CAAC,UAAU,CAAC,EAAE,CAAC;YACjD,KAAK,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC;QACnC,CAAC;IACF,CAAC;IAEe,wBAAwB,CAAC,UAAkB;QAC1D,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,EAAE,CAAC;QAC3C,IAAI,CAAC,KAAK,EAAE,CAAC;YACZ,OAAO,KAAK,CAAC;QACd,CAAC;QACD,OAAO,KAAK,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC;IACnD,CAAC;IAEe,iBAAiB,CAAC,UAAkB;QACnD,2DAA2D;QAC3D,OAAO,IAAI,CAAC;IACb,CAAC;IAEe,gCAAgC,CAAC,UAAkB,EAAE,MAAc,EAAE,SAAiB;QACrG,wGAAwG;QACxG,uCAA+B;IAChC,CAAC;IAEe,eAAe,CAAC,UAAkB,EAAE,KAAe;QAClE,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,EAAE,CAAC;QAC3C,IAAI,CAAC,KAAK,EAAE,CAAC;YACZ,OAAO,IAAI,CAAC;QACb,CAAC;QACD,OAAO,KAAK,CAAC,eAAe,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;IACjD,CAAC;IAED,IAAoB,SAAS;QAC5B,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,EAAE,CAAC;QAC3C,IAAI,CAAC,KAAK,EAAE,CAAC;YACZ,OAAO,KAAK,CAAC;QACd,CAAC;QACD,OAAO,KAAK,CAAC,SAAS,EAAE,CAAC;IAC1B,CAAC;CACD,CAAA;AAvKY,4BAA4B;IAatC,WAAA,yBAAyB,CAAA;IACzB,WAAA,qBAAqB,CAAA;GAdX,4BAA4B,CAuKxC","file":"treeSitterSyntaxTokenBackend.js","sourceRoot":"file:///mnt/vss/_work/1/s/dependencies/vscode/out-editor-src","sourcesContent":["/*---------------------------------------------------------------------------------------------\n *  Copyright (c) Microsoft Corporation. All rights reserved.\n *  Licensed under the MIT License. See License.txt in the project root for license information.\n *--------------------------------------------------------------------------------------------*/\n\nimport { Emitter, Event } from '../../../../../base/common/event.js';\nimport { toDisposable } from '../../../../../base/common/lifecycle.js';\nimport { StandardTokenType } from '../../../encodedTokenAttributes.js';\nimport { ILanguageIdCodec } from '../../../languages.js';\nimport { IModelContentChangedEvent } from '../../../textModelEvents.js';\nimport { BackgroundTokenizationState } from '../../../tokenizationTextModelPart.js';\nimport { LineTokens } from '../../../tokens/lineTokens.js';\nimport { TextModel } from '../../textModel.js';\nimport { AbstractSyntaxTokenBackend } from '../abstractSyntaxTokenBackend.js';\nimport { autorun, derived, IObservable, ObservablePromise } from '../../../../../base/common/observable.js';\nimport { TreeSitterTree } from './treeSitterTree.js';\nimport { IInstantiationService } from '../../../../../platform/instantiation/common/instantiation.js';\nimport { TreeSitterTokenizationImpl } from './treeSitterTokenizationImpl.js';\nimport { ITreeSitterLibraryService } from '../../../services/treeSitter/treeSitterLibraryService.js';\nimport { LineRange } from '../../../core/ranges/lineRange.js';\n\nexport class TreeSitterSyntaxTokenBackend extends AbstractSyntaxTokenBackend {\n\tprotected _backgroundTokenizationState: BackgroundTokenizationState = BackgroundTokenizationState.InProgress;\n\tprotected readonly _onDidChangeBackgroundTokenizationState: Emitter<void> = this._register(new Emitter<void>());\n\tpublic readonly onDidChangeBackgroundTokenizationState: Event<void> = this._onDidChangeBackgroundTokenizationState.event;\n\n\tprivate readonly _tree: IObservable<TreeSitterTree | undefined>;\n\tprivate readonly _tokenizationImpl: IObservable<TreeSitterTokenizationImpl | undefined>;\n\n\tconstructor(\n\t\tprivate readonly _languageIdObs: IObservable<string>,\n\t\tlanguageIdCodec: ILanguageIdCodec,\n\t\ttextModel: TextModel,\n\t\tvisibleLineRanges: IObservable<readonly LineRange[]>,\n\t\t@ITreeSitterLibraryService private readonly _treeSitterLibraryService: ITreeSitterLibraryService,\n\t\t@IInstantiationService private readonly _instantiationService: IInstantiationService\n\t) {\n\t\tsuper(languageIdCodec, textModel);\n\n\n\t\tconst parserClassPromise = new ObservablePromise(this._treeSitterLibraryService.getParserClass());\n\n\n\t\tconst parserClassObs = derived(this, reader => {\n\t\t\tconst parser = parserClassPromise.promiseResult?.read(reader)?.getDataOrThrow();\n\t\t\treturn parser;\n\t\t});\n\n\n\t\tthis._tree = derived(this, reader => {\n\t\t\tconst parserClass = parserClassObs.read(reader);\n\t\t\tif (!parserClass) {\n\t\t\t\treturn undefined;\n\t\t\t}\n\n\t\t\tconst currentLanguage = this._languageIdObs.read(reader);\n\t\t\tconst treeSitterLang = this._treeSitterLibraryService.getLanguage(currentLanguage, false, reader);\n\t\t\tif (!treeSitterLang) {\n\t\t\t\treturn undefined;\n\t\t\t}\n\n\t\t\tconst parser = new parserClass();\n\t\t\treader.store.add(toDisposable(() => {\n\t\t\t\tparser.delete();\n\t\t\t}));\n\t\t\tparser.setLanguage(treeSitterLang);\n\n\t\t\tconst queries = this._treeSitterLibraryService.getInjectionQueries(currentLanguage, reader);\n\t\t\tif (queries === undefined) {\n\t\t\t\treturn undefined;\n\t\t\t}\n\n\t\t\treturn reader.store.add(this._instantiationService.createInstance(TreeSitterTree, currentLanguage, undefined, parser, parserClass, /*queries, */this._textModel));\n\t\t});\n\n\n\t\tthis._tokenizationImpl = derived(this, reader => {\n\t\t\tconst treeModel = this._tree.read(reader);\n\t\t\tif (!treeModel) {\n\t\t\t\treturn undefined;\n\t\t\t}\n\n\t\t\tconst queries = this._treeSitterLibraryService.getHighlightingQueries(treeModel.languageId, reader);\n\t\t\tif (!queries) {\n\t\t\t\treturn undefined;\n\t\t\t}\n\n\t\t\treturn reader.store.add(this._instantiationService.createInstance(TreeSitterTokenizationImpl, treeModel, queries, this._languageIdCodec, visibleLineRanges));\n\t\t});\n\n\t\tthis._register(autorun(reader => {\n\t\t\tconst tokModel = this._tokenizationImpl.read(reader);\n\t\t\tif (!tokModel) {\n\t\t\t\treturn;\n\t\t\t}\n\t\t\treader.store.add(tokModel.onDidChangeTokens((e) => {\n\t\t\t\tthis._onDidChangeTokens.fire(e.changes);\n\t\t\t}));\n\t\t\treader.store.add(tokModel.onDidChangeBackgroundTokenization(e => {\n\t\t\t\tthis._backgroundTokenizationState = BackgroundTokenizationState.Completed;\n\t\t\t\tthis._onDidChangeBackgroundTokenizationState.fire();\n\t\t\t}));\n\t\t}));\n\t}\n\n\tpublic getLineTokens(lineNumber: number): LineTokens {\n\t\tconst model = this._tokenizationImpl.get();\n\t\tif (!model) {\n\t\t\tconst content = this._textModel.getLineContent(lineNumber);\n\t\t\treturn LineTokens.createEmpty(content, this._languageIdCodec);\n\t\t}\n\t\treturn model.getLineTokens(lineNumber);\n\t}\n\n\tpublic todo_resetTokenization(fireTokenChangeEvent: boolean = true): void {\n\t\tif (fireTokenChangeEvent) {\n\t\t\tthis._onDidChangeTokens.fire({\n\t\t\t\tsemanticTokensApplied: false,\n\t\t\t\tranges: [\n\t\t\t\t\t{\n\t\t\t\t\t\tfromLineNumber: 1,\n\t\t\t\t\t\ttoLineNumber: this._textModel.getLineCount(),\n\t\t\t\t\t},\n\t\t\t\t],\n\t\t\t});\n\t\t}\n\t}\n\n\tpublic override handleDidChangeAttached(): void {\n\t\t// TODO @alexr00 implement for background tokenization\n\t}\n\n\tpublic override handleDidChangeContent(e: IModelContentChangedEvent): void {\n\t\tif (e.isFlush) {\n\t\t\t// Don't fire the event, as the view might not have got the text change event yet\n\t\t\tthis.todo_resetTokenization(false);\n\t\t} else {\n\t\t\tconst model = this._tokenizationImpl.get();\n\t\t\tmodel?.handleContentChanged(e);\n\t\t}\n\n\t\tconst treeModel = this._tree.get();\n\t\ttreeModel?.handleContentChange(e);\n\t}\n\n\tpublic override forceTokenization(lineNumber: number): void {\n\t\tconst model = this._tokenizationImpl.get();\n\t\tif (!model) {\n\t\t\treturn;\n\t\t}\n\t\tif (!model.hasAccurateTokensForLine(lineNumber)) {\n\t\t\tmodel.tokenizeEncoded(lineNumber);\n\t\t}\n\t}\n\n\tpublic override hasAccurateTokensForLine(lineNumber: number): boolean {\n\t\tconst model = this._tokenizationImpl.get();\n\t\tif (!model) {\n\t\t\treturn false;\n\t\t}\n\t\treturn model.hasAccurateTokensForLine(lineNumber);\n\t}\n\n\tpublic override isCheapToTokenize(lineNumber: number): boolean {\n\t\t// TODO @alexr00 determine what makes it cheap to tokenize?\n\t\treturn true;\n\t}\n\n\tpublic override getTokenTypeIfInsertingCharacter(lineNumber: number, column: number, character: string): StandardTokenType {\n\t\t// TODO @alexr00 implement once we have custom parsing and don't just feed in the whole text model value\n\t\treturn StandardTokenType.Other;\n\t}\n\n\tpublic override tokenizeLinesAt(lineNumber: number, lines: string[]): LineTokens[] | null {\n\t\tconst model = this._tokenizationImpl.get();\n\t\tif (!model) {\n\t\t\treturn null;\n\t\t}\n\t\treturn model.tokenizeLinesAt(lineNumber, lines);\n\t}\n\n\tpublic override get hasTokens(): boolean {\n\t\tconst model = this._tokenizationImpl.get();\n\t\tif (!model) {\n\t\t\treturn false;\n\t\t}\n\t\treturn model.hasTokens();\n\t}\n}\n","/*---------------------------------------------------------------------------------------------\n *  Copyright (c) Microsoft Corporation. All rights reserved.\n *  Licensed under the MIT License. See License.txt in the project root for license information.\n *--------------------------------------------------------------------------------------------*/\n\nimport { Emitter, Event } from '../../../../../base/common/event.js';\nimport { toDisposable } from '../../../../../base/common/lifecycle.js';\nimport { StandardTokenType } from '../../../encodedTokenAttributes.js';\nimport { ILanguageIdCodec } from '../../../languages.js';\nimport { IModelContentChangedEvent } from '../../../textModelEvents.js';\nimport { BackgroundTokenizationState } from '../../../tokenizationTextModelPart.js';\nimport { LineTokens } from '../../../tokens/lineTokens.js';\nimport { TextModel } from '../../textModel.js';\nimport { AbstractSyntaxTokenBackend } from '../abstractSyntaxTokenBackend.js';\nimport { autorun, derived, IObservable, ObservablePromise } from '../../../../../base/common/observable.js';\nimport { TreeSitterTree } from './treeSitterTree.js';\nimport { IInstantiationService } from '../../../../../platform/instantiation/common/instantiation.js';\nimport { TreeSitterTokenizationImpl } from './treeSitterTokenizationImpl.js';\nimport { ITreeSitterLibraryService } from '../../../services/treeSitter/treeSitterLibraryService.js';\nimport { LineRange } from '../../../core/ranges/lineRange.js';\n\nexport class TreeSitterSyntaxTokenBackend extends AbstractSyntaxTokenBackend {\n\tprotected _backgroundTokenizationState: BackgroundTokenizationState = BackgroundTokenizationState.InProgress;\n\tprotected readonly _onDidChangeBackgroundTokenizationState: Emitter<void> = this._register(new Emitter<void>());\n\tpublic readonly onDidChangeBackgroundTokenizationState: Event<void> = this._onDidChangeBackgroundTokenizationState.event;\n\n\tprivate readonly _tree: IObservable<TreeSitterTree | undefined>;\n\tprivate readonly _tokenizationImpl: IObservable<TreeSitterTokenizationImpl | undefined>;\n\n\tconstructor(\n\t\tprivate readonly _languageIdObs: IObservable<string>,\n\t\tlanguageIdCodec: ILanguageIdCodec,\n\t\ttextModel: TextModel,\n\t\tvisibleLineRanges: IObservable<readonly LineRange[]>,\n\t\t@ITreeSitterLibraryService private readonly _treeSitterLibraryService: ITreeSitterLibraryService,\n\t\t@IInstantiationService private readonly _instantiationService: IInstantiationService\n\t) {\n\t\tsuper(languageIdCodec, textModel);\n\n\n\t\tconst parserClassPromise = new ObservablePromise(this._treeSitterLibraryService.getParserClass());\n\n\n\t\tconst parserClassObs = derived(this, reader => {\n\t\t\tconst parser = parserClassPromise.promiseResult?.read(reader)?.getDataOrThrow();\n\t\t\treturn parser;\n\t\t});\n\n\n\t\tthis._tree = derived(this, reader => {\n\t\t\tconst parserClass = parserClassObs.read(reader);\n\t\t\tif (!parserClass) {\n\t\t\t\treturn undefined;\n\t\t\t}\n\n\t\t\tconst currentLanguage = this._languageIdObs.read(reader);\n\t\t\tconst treeSitterLang = this._treeSitterLibraryService.getLanguage(currentLanguage, false, reader);\n\t\t\tif (!treeSitterLang) {\n\t\t\t\treturn undefined;\n\t\t\t}\n\n\t\t\tconst parser = new parserClass();\n\t\t\treader.store.add(toDisposable(() => {\n\t\t\t\tparser.delete();\n\t\t\t}));\n\t\t\tparser.setLanguage(treeSitterLang);\n\n\t\t\tconst queries = this._treeSitterLibraryService.getInjectionQueries(currentLanguage, reader);\n\t\t\tif (queries === undefined) {\n\t\t\t\treturn undefined;\n\t\t\t}\n\n\t\t\treturn reader.store.add(this._instantiationService.createInstance(TreeSitterTree, currentLanguage, undefined, parser, parserClass, /*queries, */this._textModel));\n\t\t});\n\n\n\t\tthis._tokenizationImpl = derived(this, reader => {\n\t\t\tconst treeModel = this._tree.read(reader);\n\t\t\tif (!treeModel) {\n\t\t\t\treturn undefined;\n\t\t\t}\n\n\t\t\tconst queries = this._treeSitterLibraryService.getHighlightingQueries(treeModel.languageId, reader);\n\t\t\tif (!queries) {\n\t\t\t\treturn undefined;\n\t\t\t}\n\n\t\t\treturn reader.store.add(this._instantiationService.createInstance(TreeSitterTokenizationImpl, treeModel, queries, this._languageIdCodec, visibleLineRanges));\n\t\t});\n\n\t\tthis._register(autorun(reader => {\n\t\t\tconst tokModel = this._tokenizationImpl.read(reader);\n\t\t\tif (!tokModel) {\n\t\t\t\treturn;\n\t\t\t}\n\t\t\treader.store.add(tokModel.onDidChangeTokens((e) => {\n\t\t\t\tthis._onDidChangeTokens.fire(e.changes);\n\t\t\t}));\n\t\t\treader.store.add(tokModel.onDidChangeBackgroundTokenization(e => {\n\t\t\t\tthis._backgroundTokenizationState = BackgroundTokenizationState.Completed;\n\t\t\t\tthis._onDidChangeBackgroundTokenizationState.fire();\n\t\t\t}));\n\t\t}));\n\t}\n\n\tpublic getLineTokens(lineNumber: number): LineTokens {\n\t\tconst model = this._tokenizationImpl.get();\n\t\tif (!model) {\n\t\t\tconst content = this._textModel.getLineContent(lineNumber);\n\t\t\treturn LineTokens.createEmpty(content, this._languageIdCodec);\n\t\t}\n\t\treturn model.getLineTokens(lineNumber);\n\t}\n\n\tpublic todo_resetTokenization(fireTokenChangeEvent: boolean = true): void {\n\t\tif (fireTokenChangeEvent) {\n\t\t\tthis._onDidChangeTokens.fire({\n\t\t\t\tsemanticTokensApplied: false,\n\t\t\t\tranges: [\n\t\t\t\t\t{\n\t\t\t\t\t\tfromLineNumber: 1,\n\t\t\t\t\t\ttoLineNumber: this._textModel.getLineCount(),\n\t\t\t\t\t},\n\t\t\t\t],\n\t\t\t});\n\t\t}\n\t}\n\n\tpublic override handleDidChangeAttached(): void {\n\t\t// TODO @alexr00 implement for background tokenization\n\t}\n\n\tpublic override handleDidChangeContent(e: IModelContentChangedEvent): void {\n\t\tif (e.isFlush) {\n\t\t\t// Don't fire the event, as the view might not have got the text change event yet\n\t\t\tthis.todo_resetTokenization(false);\n\t\t} else {\n\t\t\tconst model = this._tokenizationImpl.get();\n\t\t\tmodel?.handleContentChanged(e);\n\t\t}\n\n\t\tconst treeModel = this._tree.get();\n\t\ttreeModel?.handleContentChange(e);\n\t}\n\n\tpublic override forceTokenization(lineNumber: number): void {\n\t\tconst model = this._tokenizationImpl.get();\n\t\tif (!model) {\n\t\t\treturn;\n\t\t}\n\t\tif (!model.hasAccurateTokensForLine(lineNumber)) {\n\t\t\tmodel.tokenizeEncoded(lineNumber);\n\t\t}\n\t}\n\n\tpublic override hasAccurateTokensForLine(lineNumber: number): boolean {\n\t\tconst model = this._tokenizationImpl.get();\n\t\tif (!model) {\n\t\t\treturn false;\n\t\t}\n\t\treturn model.hasAccurateTokensForLine(lineNumber);\n\t}\n\n\tpublic override isCheapToTokenize(lineNumber: number): boolean {\n\t\t// TODO @alexr00 determine what makes it cheap to tokenize?\n\t\treturn true;\n\t}\n\n\tpublic override getTokenTypeIfInsertingCharacter(lineNumber: number, column: number, character: string): StandardTokenType {\n\t\t// TODO @alexr00 implement once we have custom parsing and don't just feed in the whole text model value\n\t\treturn StandardTokenType.Other;\n\t}\n\n\tpublic override tokenizeLinesAt(lineNumber: number, lines: string[]): LineTokens[] | null {\n\t\tconst model = this._tokenizationImpl.get();\n\t\tif (!model) {\n\t\t\treturn null;\n\t\t}\n\t\treturn model.tokenizeLinesAt(lineNumber, lines);\n\t}\n\n\tpublic override get hasTokens(): boolean {\n\t\tconst model = this._tokenizationImpl.get();\n\t\tif (!model) {\n\t\t\treturn false;\n\t\t}\n\t\treturn model.hasTokens();\n\t}\n}\n"]}

package/esm/vs/editor/common/services/treeSitter/treeSitterLibraryService.js.map

@@ -1 +1 @@
-{"version":3,"sources":["file:///mnt/vss/_work/1/s/dependencies/vscode/out-editor-src/vs/editor/common/services/treeSitter/treeSitterLibraryService.ts","vs/editor/common/services/treeSitter/treeSitterLibraryService.ts"],"names":[],"mappings":"AAAA;;;gGAGgG;AAGhG,OAAO,EAAE,eAAe,EAAE,MAAM,4DAA4D,CAAC;AAG7F,MAAM,CAAC,MAAM,yBAAyB,GAAG,eAAe,CAA4B,0BAA0B,CAAC,CAAC","file":"treeSitterLibraryService.js","sourceRoot":"file:///mnt/vss/_work/1/s/dependencies/vscode/out-editor-src","sourcesContent":["/*---------------------------------------------------------------------------------------------\n *  Copyright (c) Microsoft Corporation. All rights reserved.\n *  Licensed under the MIT License. See License.txt in the project root for license information.\n *--------------------------------------------------------------------------------------------*/\n\nimport { Language, Parser, Query } from '@vscode/tree-sitter-wasm';\nimport { createDecorator } from '../../../../platform/instantiation/common/instantiation.js';\nimport { IReader } from '../../../../base/common/observable.js';\n\nexport const ITreeSitterLibraryService = createDecorator<ITreeSitterLibraryService>('treeSitterLibraryService');\n\nexport interface ITreeSitterLibraryService {\n\treadonly _serviceBrand: undefined;\n\n\tgetParserClass(): Promise<typeof Parser>;\n\n\tsupportsLanguage(languageId: string, reader: IReader | undefined): boolean;\n\tgetLanguage(languageId: string, reader: IReader | undefined): Language | undefined;\n\t/**\n\t * Return value of null indicates that there are no injection queries for this language.\n\t * @param languageId\n\t * @param reader\n\t */\n\tgetInjectionQueries(languageId: string, reader: IReader | undefined): Query | null | undefined;\n\t/**\n\t * Return value of null indicates that there are no highlights queries for this language.\n\t * @param languageId\n\t * @param reader\n\t */\n\tgetHighlightingQueries(languageId: string, reader: IReader | undefined): Query | null | undefined;\n}\n","/*---------------------------------------------------------------------------------------------\n *  Copyright (c) Microsoft Corporation. All rights reserved.\n *  Licensed under the MIT License. See License.txt in the project root for license information.\n *--------------------------------------------------------------------------------------------*/\n\nimport { Language, Parser, Query } from '@vscode/tree-sitter-wasm';\nimport { createDecorator } from '../../../../platform/instantiation/common/instantiation.js';\nimport { IReader } from '../../../../base/common/observable.js';\n\nexport const ITreeSitterLibraryService = createDecorator<ITreeSitterLibraryService>('treeSitterLibraryService');\n\nexport interface ITreeSitterLibraryService {\n\treadonly _serviceBrand: undefined;\n\n\tgetParserClass(): Promise<typeof Parser>;\n\n\tsupportsLanguage(languageId: string, reader: IReader | undefined): boolean;\n\tgetLanguage(languageId: string, reader: IReader | undefined): Language | undefined;\n\t/**\n\t * Return value of null indicates that there are no injection queries for this language.\n\t * @param languageId\n\t * @param reader\n\t */\n\tgetInjectionQueries(languageId: string, reader: IReader | undefined): Query | null | undefined;\n\t/**\n\t * Return value of null indicates that there are no highlights queries for this language.\n\t * @param languageId\n\t * @param reader\n\t */\n\tgetHighlightingQueries(languageId: string, reader: IReader | undefined): Query | null | undefined;\n}\n"]}
+{"version":3,"sources":["file:///mnt/vss/_work/1/s/dependencies/vscode/out-editor-src/vs/editor/common/services/treeSitter/treeSitterLibraryService.ts","vs/editor/common/services/treeSitter/treeSitterLibraryService.ts"],"names":[],"mappings":"AAAA;;;gGAGgG;AAGhG,OAAO,EAAE,eAAe,EAAE,MAAM,4DAA4D,CAAC;AAG7F,MAAM,CAAC,MAAM,yBAAyB,GAAG,eAAe,CAA4B,0BAA0B,CAAC,CAAC","file":"treeSitterLibraryService.js","sourceRoot":"file:///mnt/vss/_work/1/s/dependencies/vscode/out-editor-src","sourcesContent":["/*---------------------------------------------------------------------------------------------\n *  Copyright (c) Microsoft Corporation. All rights reserved.\n *  Licensed under the MIT License. See License.txt in the project root for license information.\n *--------------------------------------------------------------------------------------------*/\n\nimport { Language, Parser, Query } from '@vscode/tree-sitter-wasm';\nimport { createDecorator } from '../../../../platform/instantiation/common/instantiation.js';\nimport { IReader } from '../../../../base/common/observable.js';\n\nexport const ITreeSitterLibraryService = createDecorator<ITreeSitterLibraryService>('treeSitterLibraryService');\n\nexport interface ITreeSitterLibraryService {\n\treadonly _serviceBrand: undefined;\n\n\t/**\n\t * Gets the tree sitter Parser constructor.\n\t */\n\tgetParserClass(): Promise<typeof Parser>;\n\n\t/**\n\t * Checks whether a language is supported and available based setting enablement.\n\t * @param languageId The language identifier to check.\n\t * @param reader Optional observable reader.\n\t */\n\tsupportsLanguage(languageId: string, reader: IReader | undefined): boolean;\n\n\t/**\n\t * Gets the tree sitter Language object synchronously.\n\t * @param languageId The language identifier to retrieve.\n\t * @param ignoreSupportsCheck Whether to ignore the supportsLanguage check.\n\t * @param reader Optional observable reader.\n\t */\n\tgetLanguage(languageId: string, ignoreSupportsCheck: boolean, reader: IReader | undefined): Language | undefined;\n\n\t/**\n\t * Gets the injection queries for a language. A return value of `null`\n\t * indicates that there are no highlights queries for this language.\n\t * @param languageId The language identifier to retrieve queries for.\n\t * @param reader Optional observable reader.\n\t */\n\tgetInjectionQueries(languageId: string, reader: IReader | undefined): Query | null | undefined;\n\n\t/**\n\t * Gets the highlighting queries for a language. A return value of `null`\n\t * indicates that there are no highlights queries for this language.\n\t * @param languageId The language identifier to retrieve queries for.\n\t * @param reader Optional observable reader.\n\t */\n\tgetHighlightingQueries(languageId: string, reader: IReader | undefined): Query | null | undefined;\n}\n","/*---------------------------------------------------------------------------------------------\n *  Copyright (c) Microsoft Corporation. All rights reserved.\n *  Licensed under the MIT License. See License.txt in the project root for license information.\n *--------------------------------------------------------------------------------------------*/\n\nimport { Language, Parser, Query } from '@vscode/tree-sitter-wasm';\nimport { createDecorator } from '../../../../platform/instantiation/common/instantiation.js';\nimport { IReader } from '../../../../base/common/observable.js';\n\nexport const ITreeSitterLibraryService = createDecorator<ITreeSitterLibraryService>('treeSitterLibraryService');\n\nexport interface ITreeSitterLibraryService {\n\treadonly _serviceBrand: undefined;\n\n\t/**\n\t * Gets the tree sitter Parser constructor.\n\t */\n\tgetParserClass(): Promise<typeof Parser>;\n\n\t/**\n\t * Checks whether a language is supported and available based setting enablement.\n\t * @param languageId The language identifier to check.\n\t * @param reader Optional observable reader.\n\t */\n\tsupportsLanguage(languageId: string, reader: IReader | undefined): boolean;\n\n\t/**\n\t * Gets the tree sitter Language object synchronously.\n\t * @param languageId The language identifier to retrieve.\n\t * @param ignoreSupportsCheck Whether to ignore the supportsLanguage check.\n\t * @param reader Optional observable reader.\n\t */\n\tgetLanguage(languageId: string, ignoreSupportsCheck: boolean, reader: IReader | undefined): Language | undefined;\n\n\t/**\n\t * Gets the injection queries for a language. A return value of `null`\n\t * indicates that there are no highlights queries for this language.\n\t * @param languageId The language identifier to retrieve queries for.\n\t * @param reader Optional observable reader.\n\t */\n\tgetInjectionQueries(languageId: string, reader: IReader | undefined): Query | null | undefined;\n\n\t/**\n\t * Gets the highlighting queries for a language. A return value of `null`\n\t * indicates that there are no highlights queries for this language.\n\t * @param languageId The language identifier to retrieve queries for.\n\t * @param reader Optional observable reader.\n\t */\n\tgetHighlightingQueries(languageId: string, reader: IReader | undefined): Query | null | undefined;\n}\n"]}