@theia/monaco 1.28.0-next.3 → 1.28.0-next.33

package/src/browser/monaco-frontend-application-contribution.ts

@@ -74,7 +74,10 @@ export class MonacoFrontendApplicationContribution implements FrontendApplicatio
         // Monaco registers certain quick access providers (e.g. QuickCommandAccess) at import time, but we want to use our own.
         this.quickAccessRegistry.clear();
 
-        // Incomparability of enum types between public and private API's
+        /**
+         * @monaco-uplift.Should be guaranteed to work.
+         * Incomparable enums prevent TypeScript from believing that public ITextModel satisfied private ITextModel
+         */
         setSnippetSuggestSupport(this.snippetSuggestProvider as unknown as CompletionItemProvider);
 
         for (const language of monaco.languages.getLanguages()) {
@@ -167,7 +170,7 @@ export class MonacoFrontendApplicationContribution implements FrontendApplicatio
                 new editorBoolConstructor(id++, 'wordBasedSuggestions', true, editorGeneratedPreferenceProperties['editor.wordBasedSuggestions']),
                 new editorStringEnumConstructor(id++, 'wordBasedSuggestionsMode', 'matchingDocuments', editorGeneratedPreferenceProperties['editor.wordBasedSuggestionsMode'].enum, editorGeneratedPreferenceProperties['editor.wordBasedSuggestionsMode']),
                 new editorBoolConstructor(id++, 'stablePeek', false, editorGeneratedPreferenceProperties['editor.stablePeek']),
-                new editorIntConstructor(id++, 'maxTokenizationLength', 20000, 1, MAX_SAFE_INTEGER, editorGeneratedPreferenceProperties['editor.maxTokenizationLength']),
+                new editorIntConstructor(id++, 'maxTokenizationLineLength', 20000, 1, MAX_SAFE_INTEGER, editorGeneratedPreferenceProperties['editor.maxTokenizationLineLength']),
             );
         }
     }

package/src/browser/monaco-indexed-db.ts

@@ -19,7 +19,7 @@ import { Disposable, DisposableCollection } from '@theia/core/lib/common/disposa
 import { ThemeService } from '@theia/core/lib/browser/theming';
 import * as monaco from '@theia/monaco-editor-core';
 import { injectable } from '@theia/core/shared/inversify';
-import type { ThemeMix } from './textmate/monaco-theme-registry';
+import type { ThemeMix } from './textmate/monaco-theme-types';
 import { Theme } from '@theia/core/lib/common/theme';
 
 let _monacoDB: Promise<idb.IDBPDatabase> | undefined;

package/src/browser/monaco-quick-input-service.ts

@@ -214,7 +214,7 @@ export class MonacoQuickInputService implements QuickInputService {
     }
 
     createInputBox(): InputBox {
-        return this.monacoService.createInputBox() as unknown as InputBox;
+        return this.monacoService.createInputBox();
     }
 
     input(options?: InputOptions, token?: monaco.CancellationToken): Promise<string | undefined> {
@@ -253,6 +253,7 @@ export class MonacoQuickInputService implements QuickInputService {
         return new Promise<T | undefined>((resolve, reject) => {
             const quickPick = this.monacoService.createQuickPick<MonacoQuickPickItem<T>>();
             const wrapped = this.wrapQuickPick(quickPick);
+            wrapped.items = items;
 
             if (options) {
                 wrapped.canSelectMany = !!options.canSelectMany;
@@ -321,8 +322,6 @@ export class MonacoQuickInputService implements QuickInputService {
                     }
                 });
             }
-
-            wrapped.items = items;
             wrapped.show();
         }).then(item => {
             if (item?.execute) {

package/src/browser/monaco-text-model-service.ts

@@ -134,7 +134,9 @@ export class MonacoTextModelService implements ITextModelService {
         switch (editorPreference) {
             case 'editor.tabSize': return 'tabSize';
             case 'editor.insertSpaces': return 'insertSpaces';
-            case 'editor.bracketPairColorization.enabled': return 'bracketColorizationOptions';
+            case 'editor.bracketPairColorization.enabled':
+            case 'editor.bracketPairColorization.independentColorPoolPerBracketType':
+                return 'bracketColorizationOptions';
             case 'editor.trimAutoWhitespace': return 'trimAutoWhitespace';
 
         }
@@ -142,28 +144,21 @@ export class MonacoTextModelService implements ITextModelService {
     }
 
     protected updateModel(model: MonacoEditorModel, change?: EditorPreferenceChange): void {
-        if (change) {
-            if (!change.affects(model.uri, model.languageId)) {
-                return;
-            }
+        if (!change) {
+            model.autoSave = this.editorPreferences.get('files.autoSave', undefined, model.uri);
+            model.autoSaveDelay = this.editorPreferences.get('files.autoSaveDelay', undefined, model.uri);
+            model.textEditorModel.updateOptions(this.getModelOptions(model));
+        } else if (change.affects(model.uri, model.languageId)) {
             if (change.preferenceName === 'files.autoSave') {
                 model.autoSave = this.editorPreferences.get('files.autoSave', undefined, model.uri);
             }
             if (change.preferenceName === 'files.autoSaveDelay') {
                 model.autoSaveDelay = this.editorPreferences.get('files.autoSaveDelay', undefined, model.uri);
             }
-            const modelOption = this.modelOptions[change.preferenceName];
+            const modelOption = this.toModelOption(change.preferenceName);
             if (modelOption) {
-                const options: ITextModelUpdateOptions = {};
-                // eslint-disable-next-line @typescript-eslint/no-explicit-any
-                const newValue = change.newValue as any;
-                options[modelOption] = change.preferenceName === 'editor.bracketPairColorization.enabled' ? { enabled: newValue } : newValue;
-                model.textEditorModel.updateOptions(options);
+                model.textEditorModel.updateOptions(this.getModelOptions(model));
             }
-        } else {
-            model.autoSave = this.editorPreferences.get('files.autoSave', undefined, model.uri);
-            model.autoSaveDelay = this.editorPreferences.get('files.autoSaveDelay', undefined, model.uri);
-            model.textEditorModel.updateOptions(this.getModelOptions(model));
         }
     }
 
@@ -176,7 +171,11 @@ export class MonacoTextModelService implements ITextModelService {
         return {
             tabSize: this.editorPreferences.get({ preferenceName: 'editor.tabSize', overrideIdentifier }, undefined, uri),
             insertSpaces: this.editorPreferences.get({ preferenceName: 'editor.insertSpaces', overrideIdentifier }, undefined, uri),
-            bracketColorizationOptions: { enabled: this.editorPreferences.get({ preferenceName: 'editor.bracketPairColorization.enabled', overrideIdentifier }, undefined, uri) },
+            bracketColorizationOptions: {
+                enabled: this.editorPreferences.get({ preferenceName: 'editor.bracketPairColorization.enabled', overrideIdentifier }, undefined, uri),
+                independentColorPoolPerBracketType: this.editorPreferences.get(
+                    { preferenceName: 'editor.bracketPairColorization.independentColorPoolPerBracketType', overrideIdentifier }, undefined, uri),
+            },
             trimAutoWhitespace: this.editorPreferences.get({ preferenceName: 'editor.trimAutoWhitespace', overrideIdentifier }, undefined, uri),
         };
     }

package/src/browser/textmate/monaco-textmate-frontend-bindings.ts

@@ -19,17 +19,18 @@ import { FrontendApplicationContribution, isBasicWasmSupported } from '@theia/co
 import { bindContributionProvider } from '@theia/core';
 import { TextmateRegistry } from './textmate-registry';
 import { LanguageGrammarDefinitionContribution } from './textmate-contribution';
-import { MonacoTextmateService, OnigasmPromise } from './monaco-textmate-service';
+import { MonacoTextmateService } from './monaco-textmate-service';
 import { MonacoThemeRegistry } from './monaco-theme-registry';
-import { loadWASM, OnigScanner, OnigString } from 'onigasm';
-import { IOnigLib } from 'vscode-textmate';
+import { loadWASM, createOnigScanner, OnigScanner, createOnigString, OnigString } from 'vscode-oniguruma';
+import { IOnigLib, IRawGrammar, parseRawGrammar, Registry } from 'vscode-textmate';
+import { OnigasmPromise, TextmateRegistryFactory, ThemeMix } from './monaco-theme-types';
 
 export class OnigasmLib implements IOnigLib {
     createOnigScanner(sources: string[]): OnigScanner {
-        return new OnigScanner(sources);
+        return createOnigScanner(sources);
     }
     createOnigString(sources: string): OnigString {
-        return new OnigString(sources);
+        return createOnigString(sources);
     }
 }
 
@@ -40,6 +41,35 @@ export default (bind: interfaces.Bind, unbind: interfaces.Unbind, isBound: inter
     bindContributionProvider(bind, LanguageGrammarDefinitionContribution);
     bind(TextmateRegistry).toSelf().inSingletonScope();
     bind(MonacoThemeRegistry).toSelf().inSingletonScope();
+    bind(TextmateRegistryFactory).toFactory(({ container }) => (theme?: ThemeMix) => {
+        const onigLib = container.get<OnigasmPromise>(OnigasmPromise);
+        const textmateRegistry = container.get(TextmateRegistry);
+        return new Registry({
+            onigLib,
+            theme,
+            loadGrammar: async (scopeName: string) => {
+                const provider = textmateRegistry.getProvider(scopeName);
+                if (provider) {
+                    const definition = await provider.getGrammarDefinition();
+                    let rawGrammar: IRawGrammar;
+                    if (typeof definition.content === 'string') {
+                        rawGrammar = parseRawGrammar(definition.content, definition.format === 'json' ? 'grammar.json' : 'grammar.plist');
+                    } else {
+                        rawGrammar = definition.content as IRawGrammar;
+                    }
+                    return rawGrammar;
+                }
+                return undefined;
+            },
+            getInjections: (scopeName: string) => {
+                const provider = textmateRegistry.getProvider(scopeName);
+                if (provider && provider.getInjections) {
+                    return provider.getInjections(scopeName);
+                }
+                return [];
+            }
+        });
+    });
 };
 
 export async function dynamicOnigasmLib(ctx: interfaces.Context): Promise<IOnigLib> {
@@ -57,7 +87,7 @@ export async function createOnigasmLib(): Promise<IOnigLib> {
 
 export async function fetchOnigasm(): Promise<ArrayBuffer> {
     // Using Webpack's wasm loader should give us a URL to fetch the resource from:
-    const onigasmPath: string = require('onigasm/lib/onigasm.wasm');
+    const onigasmPath: string = require('vscode-oniguruma/release/onig.wasm');
     const response = await fetch(onigasmPath, { method: 'GET' });
     return response.arrayBuffer();
 }

package/src/browser/textmate/monaco-textmate-service.ts

@@ -15,7 +15,7 @@
 // *****************************************************************************
 
 import { injectable, inject, named } from '@theia/core/shared/inversify';
-import { Registry, IOnigLib, IRawGrammar, parseRawGrammar } from 'vscode-textmate';
+import { Registry } from 'vscode-textmate';
 import { ILogger, ContributionProvider, DisposableCollection, Disposable } from '@theia/core';
 import { FrontendApplicationContribution, isBasicWasmSupported } from '@theia/core/lib/browser';
 import { ThemeService } from '@theia/core/lib/browser/theming';
@@ -31,9 +31,7 @@ import { StandaloneServices } from '@theia/monaco-editor-core/esm/vs/editor/stan
 import { ILanguageService } from '@theia/monaco-editor-core/esm/vs/editor/common/languages/language';
 import { TokenizationSupportAdapter } from '@theia/monaco-editor-core/esm/vs/editor/standalone/browser/standaloneLanguages';
 import { LanguageService } from '@theia/monaco-editor-core/esm/vs/editor/common/services/languageService';
-
-export const OnigasmPromise = Symbol('OnigasmPromise');
-export type OnigasmPromise = Promise<IOnigLib>;
+import { OnigasmPromise, TextmateRegistryFactory } from './monaco-theme-types';
 
 @injectable()
 export class MonacoTextmateService implements FrontendApplicationContribution {
@@ -67,6 +65,9 @@ export class MonacoTextmateService implements FrontendApplicationContribution {
     @inject(EditorPreferences)
     protected readonly preferences: EditorPreferences;
 
+    @inject(TextmateRegistryFactory)
+    protected readonly registryFactory: TextmateRegistryFactory;
+
     initialize(): void {
         if (!isBasicWasmSupported) {
             console.log('Textmate support deactivated because WebAssembly is not detected.');
@@ -81,31 +82,7 @@ export class MonacoTextmateService implements FrontendApplicationContribution {
             }
         }
 
-        this.grammarRegistry = new Registry({
-            getOnigLib: () => this.onigasmPromise,
-            theme: this.monacoThemeRegistry.getThemeData(this.currentEditorTheme),
-            loadGrammar: async (scopeName: string) => {
-                const provider = this.textmateRegistry.getProvider(scopeName);
-                if (provider) {
-                    const definition = await provider.getGrammarDefinition();
-                    let rawGrammar: IRawGrammar;
-                    if (typeof definition.content === 'string') {
-                        rawGrammar = parseRawGrammar(definition.content, definition.format === 'json' ? 'grammar.json' : 'grammar.plist');
-                    } else {
-                        rawGrammar = definition.content as IRawGrammar;
-                    }
-                    return rawGrammar;
-                }
-                return undefined;
-            },
-            getInjections: (scopeName: string) => {
-                const provider = this.textmateRegistry.getProvider(scopeName);
-                if (provider && provider.getInjections) {
-                    return provider.getInjections(scopeName);
-                }
-                return [];
-            }
-        });
+        this.grammarRegistry = this.registryFactory(this.monacoThemeRegistry.getThemeData(this.currentEditorTheme));
 
         this.tokenizerOption.lineLimit = this.preferences['editor.maxTokenizationLineLength'];
         this.preferences.onPreferenceChanged(e => {

package/src/browser/textmate/monaco-theme-registry.ts

@@ -17,22 +17,20 @@
 
 /* eslint-disable @typescript-eslint/no-explicit-any */
 
-import { injectable } from '@theia/core/shared/inversify';
-import { IRawTheme, Registry, IRawThemeSetting } from 'vscode-textmate';
+import { inject, injectable } from '@theia/core/shared/inversify';
+import { IRawThemeSetting } from 'vscode-textmate';
 import * as monaco from '@theia/monaco-editor-core';
-import { IStandaloneTheme, IStandaloneThemeService } from '@theia/monaco-editor-core/esm/vs/editor/standalone/common/standaloneTheme';
+import { IStandaloneThemeService } from '@theia/monaco-editor-core/esm/vs/editor/standalone/common/standaloneTheme';
 import { StandaloneServices } from '@theia/monaco-editor-core/esm/vs/editor/standalone/browser/standaloneServices';
 import { StandaloneThemeService } from '@theia/monaco-editor-core/esm/vs/editor/standalone/browser/standaloneThemeService';
 import { Color } from '@theia/monaco-editor-core/esm/vs/base/common/color';
-
-export interface ThemeMix extends IRawTheme, monaco.editor.IStandaloneThemeData { }
-export interface MixStandaloneTheme extends IStandaloneTheme {
-    themeData: ThemeMix
-}
+import { MixStandaloneTheme, TextmateRegistryFactory, ThemeMix } from './monaco-theme-types';
 
 @injectable()
 export class MonacoThemeRegistry {
 
+    @inject(TextmateRegistryFactory) protected readonly registryFactory: TextmateRegistryFactory;
+
     initializeDefaultThemes(): void {
         this.register(require('../../../data/monaco-themes/vscode/dark_theia.json'), {
             './dark_vs.json': require('../../../data/monaco-themes/vscode/dark_vs.json'),
@@ -129,8 +127,7 @@ export class MonacoThemeRegistry {
                 }
             });
 
-            const reg = new Registry();
-            reg.setTheme(result);
+            const reg = this.registryFactory(result);
             result.encodedTokensColors = reg.getColorMap();
             // index 0 has to be set to null as it is 'undefined' by default, but monaco code expects it to be null
             // eslint-disable-next-line no-null/no-null

package/src/browser/textmate/monaco-theme-types.ts

@@ -15,6 +15,18 @@
 // *****************************************************************************
 
 import * as monaco from '@theia/monaco-editor-core';
+import { IStandaloneTheme } from '@theia/monaco-editor-core/esm/vs/editor/standalone/common/standaloneTheme';
+import { IOnigLib, IRawTheme, Registry } from 'vscode-textmate';
+
+export interface ThemeMix extends IRawTheme, monaco.editor.IStandaloneThemeData { }
+export interface MixStandaloneTheme extends IStandaloneTheme {
+    themeData: ThemeMix
+}
+
+export const OnigasmPromise = Symbol('OnigasmPromise');
+export type OnigasmPromise = Promise<IOnigLib>;
+export const TextmateRegistryFactory = Symbol('TextmateRegistryFactory');
+export type TextmateRegistryFactory = (currentTheme?: ThemeMix) => Registry;
 
 export type MonacoThemeColor = monaco.editor.IColors;
 export interface MonacoTokenRule extends monaco.editor.ITokenThemeRule { };

package/src/browser/textmate/textmate-tokenizer.ts

@@ -49,17 +49,6 @@ export interface TokenizerOption {
 
 }
 
-export namespace TokenizerOption {
-    /**
-     * The default TextMate tokenizer option.
-     *
-     * @deprecated Use the current value of `editor.maxTokenizationLineLength` preference instead.
-     */
-    export const DEFAULT: TokenizerOption = {
-        lineLimit: 400
-    };
-}
-
 export function createTextmateTokenizer(grammar: IGrammar, options: TokenizerOption): monaco.languages.EncodedTokensProvider & monaco.languages.TokensProvider {
     if (options.lineLimit !== undefined && (options.lineLimit <= 0 || !Number.isInteger(options.lineLimit))) {
         throw new Error(`The 'lineLimit' must be a positive integer. It was ${options.lineLimit}.`);
@@ -67,24 +56,22 @@ export function createTextmateTokenizer(grammar: IGrammar, options: TokenizerOpt
     return {
         getInitialState: () => new TokenizerState(INITIAL),
         tokenizeEncoded(line: string, state: TokenizerState): monaco.languages.IEncodedLineTokens {
-            let processedLine = line;
             if (options.lineLimit !== undefined && line.length > options.lineLimit) {
-                // Line is too long to be tokenized
-                processedLine = line.substring(0, options.lineLimit);
+                // Skip tokenizing the line if it exceeds the line limit.
+                return { endState: state.ruleStack, tokens: new Uint32Array() };
             }
-            const result = grammar.tokenizeLine2(processedLine, state.ruleStack);
+            const result = grammar.tokenizeLine2(line, state.ruleStack, 500);
             return {
                 endState: new TokenizerState(result.ruleStack),
                 tokens: result.tokens
             };
         },
         tokenize(line: string, state: TokenizerState): monaco.languages.ILineTokens {
-            let processedLine = line;
             if (options.lineLimit !== undefined && line.length > options.lineLimit) {
-                // Line is too long to be tokenized
-                processedLine = line.substring(0, options.lineLimit);
+                // Skip tokenizing the line if it exceeds the line limit.
+                return { endState: state.ruleStack, tokens: [] };
             }
-            const result = grammar.tokenizeLine(processedLine, state.ruleStack);
+            const result = grammar.tokenizeLine(line, state.ruleStack, 500);
             return {
                 endState: new TokenizerState(result.ruleStack),
                 tokens: result.tokens.map(t => ({