@difizen/libro-cofine-textmate 0.1.2

package/src/monaco-textmate-service.ts

@@ -0,0 +1,220 @@
+import { EditorHandlerContribution } from '@difizen/libro-cofine-editor-core';
+import type { Contribution } from '@difizen/mana-app';
+import {
+  Disposable,
+  DisposableCollection,
+  contrib,
+  inject,
+  singleton,
+} from '@difizen/mana-app';
+import * as monaco from '@difizen/monaco-editor-core';
+import { TokenizationRegistry } from '@difizen/monaco-editor-core/esm/vs/editor/common/languages';
+import { ILanguageService } from '@difizen/monaco-editor-core/esm/vs/editor/common/languages/language';
+import { TokenizationSupportAdapter } from '@difizen/monaco-editor-core/esm/vs/editor/standalone/browser/standaloneLanguages';
+import { StandaloneServices } from '@difizen/monaco-editor-core/esm/vs/editor/standalone/browser/standaloneServices';
+import { IStandaloneThemeService } from '@difizen/monaco-editor-core/esm/vs/editor/standalone/common/standaloneTheme';
+
+import {
+  isBasicWasmSupported,
+  MonacoGrammarRegistry,
+  OnigurumaPromise,
+} from './monaco-grammar-registry.js';
+import { MonacoThemeRegistry } from './monaco-theme-registry.js';
+import {
+  getEncodedLanguageId,
+  LanguageGrammarDefinitionContribution,
+} from './textmate-contribution.js';
+import { TextmateRegistry } from './textmate-registry.js';
+import type { TokenizerOption } from './textmate-tokenizer.js';
+import { createTextmateTokenizer } from './textmate-tokenizer.js';
+
+@singleton({ contrib: EditorHandlerContribution })
+export class MonacoTextmateService implements EditorHandlerContribution {
+  protected readonly tokenizerOption: TokenizerOption = {
+    lineLimit: 400,
+  };
+
+  protected readonly _activatedLanguages = new Set<string>();
+
+  @contrib(LanguageGrammarDefinitionContribution)
+  protected readonly grammarProviders: Contribution.Provider<LanguageGrammarDefinitionContribution>;
+
+  @inject(TextmateRegistry)
+  protected readonly textmateRegistry: TextmateRegistry;
+  @inject(MonacoGrammarRegistry)
+  protected readonly grammarRegistry: MonacoGrammarRegistry;
+
+  @inject(OnigurumaPromise)
+  protected readonly onigasmPromise: OnigurumaPromise;
+
+  @inject(MonacoThemeRegistry)
+  protected readonly monacoThemeRegistry: MonacoThemeRegistry;
+
+  beforeCreate() {
+    this.initialize();
+  }
+  afterCreate() {
+    //
+  }
+  canHandle() {
+    return true;
+  }
+  dispose() {
+    //
+  }
+
+  initialize(): void {
+    if (!isBasicWasmSupported) {
+      console.warn('Textmate support deactivated because WebAssembly is not detected.');
+      return;
+    }
+
+    for (const grammarProvider of this.grammarProviders.getContributions()) {
+      try {
+        grammarProvider.registerTextmateLanguage(this.textmateRegistry);
+      } catch (err) {
+        console.error(err);
+      }
+    }
+    for (const id of this.textmateRegistry.languages) {
+      this.activateLanguage(id);
+    }
+    // const theme = this.monacoThemeRegistry.getThemeData(this.currentEditorTheme);
+    // if (!theme) {
+    //   return;
+    // }
+    // this.grammarRegistry.setupRegistry(theme);
+    this.monacoThemeRegistry.onThemeChanged(() => {
+      this.updateTheme();
+    });
+  }
+
+  protected readonly toDisposeOnUpdateTheme = new DisposableCollection();
+
+  protected updateTheme = (): void => {
+    this.toDisposeOnUpdateTheme.dispose();
+
+    const { currentEditorTheme } = this;
+    document.body.classList.add(currentEditorTheme);
+    this.toDisposeOnUpdateTheme.push(
+      Disposable.create(() => document.body.classList.remove(currentEditorTheme)),
+    );
+
+    // first update registry to run tokenization with the proper theme
+    const theme = this.monacoThemeRegistry.getThemeData(currentEditorTheme);
+    if (theme) {
+      this.grammarRegistry.setupRegistry(theme);
+    }
+
+    // then trigger tokenization by setting monaco theme
+    monaco.editor.setTheme(currentEditorTheme);
+  };
+
+  protected get currentEditorTheme(): string {
+    return this.monacoThemeRegistry.getMonacoThemeName();
+  }
+
+  activateLanguage(language: string): Disposable {
+    const toDispose = new DisposableCollection(
+      Disposable.create(() => {
+        /* mark as not disposed */
+      }),
+    );
+    toDispose.push(
+      this.waitForLanguage(language, () =>
+        this.doActivateLanguage(language, toDispose),
+      ),
+    );
+    return toDispose;
+  }
+
+  protected async doActivateLanguage(
+    languageId: string,
+    toDispose: DisposableCollection,
+  ): Promise<void> {
+    if (this._activatedLanguages.has(languageId)) {
+      return;
+    }
+    this._activatedLanguages.add(languageId);
+    toDispose.push(
+      Disposable.create(() => this._activatedLanguages.delete(languageId)),
+    );
+
+    const scopeName = this.textmateRegistry.getScope(languageId);
+    if (!scopeName) {
+      return;
+    }
+    const provider = this.textmateRegistry.getProvider(scopeName);
+    if (!provider) {
+      return;
+    }
+
+    const configuration = this.textmateRegistry.getGrammarConfiguration(languageId);
+    const initialLanguage = getEncodedLanguageId(languageId);
+
+    await this.onigasmPromise;
+    if (toDispose.disposed) {
+      return;
+    }
+    if (!this.grammarRegistry.registry) {
+      return;
+    }
+    try {
+      const grammar = await this.grammarRegistry.registry.loadGrammarWithConfiguration(
+        scopeName,
+        initialLanguage,
+        configuration,
+      );
+      if (toDispose.disposed) {
+        return;
+      }
+      if (!grammar) {
+        throw new Error(
+          `no grammar for ${scopeName}, ${initialLanguage}, ${JSON.stringify(
+            configuration,
+          )}`,
+        );
+      }
+      const options = configuration.tokenizerOption
+        ? configuration.tokenizerOption
+        : this.tokenizerOption;
+      const tokenizer = createTextmateTokenizer(grammar, options);
+      toDispose.push(monaco.languages.setTokensProvider(languageId, tokenizer));
+      const support = TokenizationRegistry.get(languageId);
+      const themeService = StandaloneServices.get(IStandaloneThemeService);
+      const adapter = new TokenizationSupportAdapter(
+        themeService,
+        {
+          language: languageId,
+          id: StandaloneServices.get(
+            ILanguageService,
+          )._registry.languageIdCodec._languageToLanguageId.get(languageId),
+        },
+        tokenizer,
+      );
+      support!.tokenize = adapter.tokenize.bind(adapter);
+      this.updateTheme();
+    } catch (error) {
+      console.warn('No grammar for this language id', languageId, error);
+    }
+  }
+
+  protected waitForLanguage(
+    language: string,
+    cb: () => {
+      //
+    },
+  ): Disposable {
+    const modeService = StandaloneServices.get(ILanguageService);
+    for (const modeId of Object.keys(modeService._instantiatedModes || {})) {
+      const mode = modeService._instantiatedModes[modeId];
+      if (mode.getId() === language) {
+        cb();
+        return Disposable.create(() => {
+          //
+        });
+      }
+    }
+    return monaco.languages.onLanguage(language, cb);
+  }
+}

package/src/monaco-theme-registry.ts

@@ -0,0 +1,201 @@
+import type {
+  MixedTheme,
+  ITextmateThemeSetting,
+} from '@difizen/libro-cofine-editor-core';
+import {
+  MixedThemeRegistry,
+  InitializeContribution,
+} from '@difizen/libro-cofine-editor-core';
+import type { Color } from '@difizen/mana-app';
+import { Emitter, inject, singleton } from '@difizen/mana-app';
+import * as monaco from '@difizen/monaco-editor-core';
+import { StandaloneServices } from '@difizen/monaco-editor-core/esm/vs/editor/standalone/browser/standaloneServices';
+import { IStandaloneThemeService } from '@difizen/monaco-editor-core/esm/vs/editor/standalone/common/standaloneTheme';
+
+import { MonacoGrammarRegistry } from './monaco-grammar-registry.js';
+
+export interface MixStandaloneTheme {
+  themeData: MixedTheme;
+}
+
+@singleton({ contrib: [MixedThemeRegistry, InitializeContribution] })
+export class MonacoThemeRegistry implements MixedThemeRegistry, InitializeContribution {
+  @inject(MonacoGrammarRegistry)
+  protected readonly grammarRegistry: MonacoGrammarRegistry;
+
+  protected registedTheme: string[] = [];
+
+  onInitialize() {
+    const standaloneThemeService = StandaloneServices.get(IStandaloneThemeService);
+    standaloneThemeService.onDidColorThemeChange(() => {
+      this.themeChangedEmitter.fire();
+    });
+  }
+
+  protected themeChangedEmitter = new Emitter<void>();
+
+  onThemeChanged = this.themeChangedEmitter.event;
+
+  getThemeData(): MixedTheme;
+  getThemeData(name: string): MixedTheme | undefined;
+  getThemeData(name?: string): MixedTheme | undefined {
+    const theme = this.doGetTheme(name);
+    return theme && theme.themeData;
+  }
+
+  getTheme(): MixStandaloneTheme;
+  getTheme(name: string): MixStandaloneTheme | undefined;
+  getTheme(name?: string): MixStandaloneTheme | undefined {
+    return this.doGetTheme(name);
+  }
+
+  getMonacoThemeName() {
+    const standaloneThemeService = StandaloneServices.get(IStandaloneThemeService);
+    return standaloneThemeService.getColorTheme().themeName as string;
+  }
+
+  protected doGetTheme(name: string | undefined): MixStandaloneTheme | undefined {
+    const standaloneThemeService = StandaloneServices.get(IStandaloneThemeService);
+    const theme = !name
+      ? standaloneThemeService.getTheme()
+      : standaloneThemeService._knownThemes.get(name);
+    return theme as MixStandaloneTheme | undefined;
+  }
+
+  setTheme(name: string, data: MixedTheme): void {
+    // monaco auto refreshes a theme with new data
+    monaco.editor.defineTheme(name, data);
+  }
+
+  registerThemes(
+    themeOptions: Record<string, ITextmateThemeSetting>,
+    setTheme: (name: string, data: monaco.editor.IStandaloneThemeData) => void,
+  ): void {
+    Object.keys(themeOptions).forEach((key) => {
+      this.doRegisterThemes(themeOptions[key], themeOptions, setTheme);
+    });
+  }
+  doRegisterThemes(
+    option: ITextmateThemeSetting,
+    themeOptions: Record<string, ITextmateThemeSetting>,
+    setTheme: (name: string, data: monaco.editor.IStandaloneThemeData) => void,
+  ): MixedTheme {
+    const result: MixedTheme = {
+      name: option.name,
+      base: option.base || 'vs',
+      inherit: true,
+      colors: {},
+      rules: [],
+      settings: [],
+    };
+    if (typeof option.include !== 'undefined') {
+      const themeName = option.include
+        .replaceAll('/', '')
+        .replaceAll('_', '-')
+        .replaceAll('.json', '');
+
+      const parentOption = themeOptions[themeName];
+      if (!parentOption || this.registedTheme.includes(parentOption.name)) {
+        console.error(`Couldn't resolve includes theme ${option.include}.`);
+      } else {
+        const parentTheme = this.doRegisterThemes(parentOption, themeOptions, setTheme);
+        Object.assign(result.colors, parentTheme.colors);
+        result.rules.push(...parentTheme.rules);
+        result.settings.push(...parentTheme.settings);
+      }
+    }
+    const { tokenColors } = option;
+    if (Array.isArray(tokenColors)) {
+      for (const tokenColor of tokenColors) {
+        if (tokenColor.scope && tokenColor.settings) {
+          result.settings.push({
+            scope: tokenColor.scope,
+            settings: {
+              foreground: this.normalizeColor(tokenColor.settings.foreground),
+              background: this.normalizeColor(tokenColor.settings.background),
+              fontStyle: tokenColor.settings.fontStyle,
+            },
+          });
+        }
+      }
+    }
+    if (option.colors) {
+      Object.assign(result.colors, option.colors);
+      result.encodedTokensColors = Object.keys(result.colors).map(
+        (key) => result.colors[key],
+      );
+    }
+    if (option.name && option.base) {
+      for (const setting of result.settings) {
+        this.transform(setting, (rule) => result.rules.push(rule));
+      }
+      const standaloneThemeService = StandaloneServices.get(IStandaloneThemeService);
+
+      // the default rule (scope empty) is always the first rule. Ignore all other default rules.
+      const defaultTheme = standaloneThemeService._knownThemes.get(result.base)!;
+      const foreground =
+        result.colors['editor.foreground'] ||
+        defaultTheme.getColor('editor.foreground');
+      const background =
+        result.colors['editor.background'] ||
+        defaultTheme.getColor('editor.background');
+      result.settings.unshift({
+        settings: {
+          foreground: this.normalizeColor(foreground),
+          background: this.normalizeColor(background),
+        },
+      });
+
+      const reg = this.grammarRegistry.getRegistry(result);
+      reg.setTheme(result);
+      result.encodedTokensColors = reg.getColorMap();
+      // index 0 has to be set to null as it is 'undefined' by default, but monaco code expects it to be null
+      if (result.encodedTokensColors) {
+        result.encodedTokensColors[0] = null!;
+      }
+      setTheme(option.name, result);
+    }
+    return result;
+  }
+
+  protected transform(
+    tokenColor: any,
+    acceptor: (rule: monaco.editor.ITokenThemeRule) => void,
+  ): void {
+    if (typeof tokenColor.scope === 'undefined') {
+      tokenColor.scope = [''];
+    } else if (typeof tokenColor.scope === 'string') {
+      tokenColor.scope = tokenColor.scope
+        .split(',')
+        .map((scope: string) => scope.trim());
+    }
+
+    for (const scope of tokenColor.scope) {
+      acceptor({
+        ...tokenColor.settings,
+        token: scope,
+      });
+    }
+  }
+
+  protected normalizeColor(color: string | Color | undefined): string | undefined {
+    if (!color) {
+      return undefined;
+    }
+    const normalized = String(color).replace(/^#/, '').slice(0, 6);
+    if (normalized.length < 6 || !normalized.match(/^[0-9A-Fa-f]{6}$/)) {
+      // ignoring not normalized colors to avoid breaking token color indexes between monaco and vscode-textmate
+      console.error(
+        `Color '${normalized}' is NOT normalized, it must have 6 positions.`,
+      );
+      return undefined;
+    }
+    return `#${normalized}`;
+  }
+}
+
+export namespace MonacoThemeRegistry {
+  export const DARK_DEFAULT_THEME = 'e2-dark';
+  export const LIGHT_DEFAULT_THEME = 'e2-light';
+  export const HC_DEFAULT_THEME = 'e2-hc';
+}

package/src/monaco-theme-types.ts

@@ -0,0 +1,5 @@
+import type * as monaco from '@difizen/monaco-editor-core';
+
+export interface MonacoTheme extends monaco.editor.IStandaloneThemeData {
+  name: string;
+}

package/src/textmate-contribution.ts

@@ -0,0 +1,15 @@
+import { Syringe } from '@difizen/mana-app';
+import * as monaco from '@difizen/monaco-editor-core';
+
+import type { TextmateRegistry } from './textmate-registry.js';
+
+export const LanguageGrammarDefinitionContribution = Syringe.defineToken(
+  'LanguageGrammarDefinitionContribution',
+);
+export interface LanguageGrammarDefinitionContribution {
+  registerTextmateLanguage: (registry: TextmateRegistry) => void;
+  _finishRegisterTextmateLanguage?: boolean;
+}
+export function getEncodedLanguageId(languageId: string): number {
+  return monaco.languages.getEncodedLanguageId(languageId);
+}

package/src/textmate-registry.ts

@@ -0,0 +1,134 @@
+/* eslint-disable no-console */
+/* eslint-disable no-restricted-syntax */
+
+import { Disposable } from '@difizen/mana-app';
+import { singleton } from '@difizen/mana-app';
+import type { IGrammarConfiguration } from 'vscode-textmate';
+
+import type { TokenizerOption } from './textmate-tokenizer.js';
+
+export interface TextmateGrammarConfiguration extends IGrammarConfiguration {
+  /**
+   * Optional options to further refine the tokenization of the grammar.
+   */
+  readonly tokenizerOption?: TokenizerOption;
+}
+
+export interface GrammarDefinitionProvider {
+  getGrammarDefinition: () => Promise<GrammarDefinition>;
+  getInjections?: (scopeName: string) => string[];
+}
+
+export interface GrammarDefinition {
+  format: 'json' | 'plist';
+  content: Record<string, unknown> | string;
+  location?: string;
+}
+
+@singleton()
+export class TextmateRegistry {
+  protected readonly scopeToProvider = new Map<string, GrammarDefinitionProvider[]>();
+  protected readonly languageToConfig = new Map<
+    string,
+    TextmateGrammarConfiguration[]
+  >();
+  protected readonly languageIdToScope = new Map<string, string[]>();
+
+  get languages(): IterableIterator<string> {
+    return this.languageIdToScope.keys();
+  }
+
+  registerTextmateGrammarScope(
+    scope: string,
+    provider: GrammarDefinitionProvider,
+  ): Disposable {
+    const providers = this.scopeToProvider.get(scope) || [];
+    const existingProvider = providers[0];
+    if (existingProvider) {
+      Promise.all([
+        existingProvider.getGrammarDefinition(),
+        provider.getGrammarDefinition(),
+      ])
+        .then(([a, b]) => {
+          if (a.location !== b.location || (!a.location && !b.location)) {
+            console.warn(
+              `a registered grammar provider for '${scope}' scope is overridden`,
+            );
+          }
+          return;
+        })
+        .catch(console.error);
+    }
+    providers.unshift(provider);
+    this.scopeToProvider.set(scope, providers);
+    return Disposable.create(() => {
+      const index = providers.indexOf(provider);
+      if (index !== -1) {
+        providers.splice(index, 1);
+      }
+    });
+  }
+
+  getProvider(scope: string): GrammarDefinitionProvider | undefined {
+    const providers = this.scopeToProvider.get(scope);
+    return providers && providers[0];
+  }
+
+  mapLanguageIdToTextmateGrammar(languageId: string, scope: string): Disposable {
+    const scopes = this.languageIdToScope.get(languageId) || [];
+    const existingScope = scopes[0];
+    if (typeof existingScope === 'string') {
+      console.warn(
+        `'${languageId}' language is remapped from '${existingScope}' to '${scope}' scope`,
+      );
+    }
+    scopes.unshift(scope);
+    this.languageIdToScope.set(languageId, scopes);
+    return Disposable.create(() => {
+      const index = scopes.indexOf(scope);
+      if (index !== -1) {
+        scopes.splice(index, 1);
+      }
+    });
+  }
+
+  getScope(languageId: string): string | undefined {
+    const scopes = this.languageIdToScope.get(languageId);
+    return scopes && scopes[0];
+  }
+
+  getLanguageId(scope: string): string | undefined {
+    for (const languageId of this.languageIdToScope.keys()) {
+      if (this.getScope(languageId) === scope) {
+        return languageId;
+      }
+    }
+    return undefined;
+  }
+
+  registerGrammarConfiguration(
+    languageId: string,
+    config: TextmateGrammarConfiguration,
+  ): Disposable {
+    const configs = this.languageToConfig.get(languageId) || [];
+    const existingConfig = configs[0];
+    if (existingConfig) {
+      console.warn(
+        `a registered grammar configuration for '${languageId}' language is overridden`,
+      );
+    }
+    configs.unshift(config);
+    this.languageToConfig.set(languageId, configs);
+    return Disposable.create(() => {
+      const index = configs.indexOf(config);
+      if (index !== -1) {
+        configs.splice(index, 1);
+      }
+    });
+  }
+
+  getGrammarConfiguration(languageId: string): TextmateGrammarConfiguration {
+    const configs = this.languageToConfig.get(languageId);
+    return (configs && configs[0]) || {};
+  }
+}

package/src/textmate-theme-contribution.ts

@@ -0,0 +1,39 @@
+/* eslint-disable global-require */
+import type { ThemeRegistry } from '@difizen/libro-cofine-editor-core';
+import { ThemeContribution } from '@difizen/libro-cofine-editor-core';
+import { singleton } from '@difizen/mana-app';
+
+import darkDefaults from './data/monaco-themes/vscode/dark_defaults.json';
+import darkEditor from './data/monaco-themes/vscode/dark_editor.json';
+import darkPlus from './data/monaco-themes/vscode/dark_plus.json';
+import darkVS from './data/monaco-themes/vscode/dark_vs.json';
+import HCBlack from './data/monaco-themes/vscode/hc_black.json';
+import HCBlackDefault from './data/monaco-themes/vscode/hc_black_defaults.json';
+import HCEditor from './data/monaco-themes/vscode/hc_editor.json';
+import lightDefaults from './data/monaco-themes/vscode/light_defaults.json';
+import lightEditor from './data/monaco-themes/vscode/light_editor.json';
+import lightPlus from './data/monaco-themes/vscode/light_plus.json';
+import lightVS from './data/monaco-themes/vscode/light_vs.json';
+
+@singleton({ contrib: ThemeContribution })
+export class TextmateThemeContribution implements ThemeContribution {
+  registerItem(registry: ThemeRegistry): void {
+    if (!registry.mixedThemeEnable) {
+      console.warn('cannot register textmate themes');
+      return;
+    }
+    registry.registerMixedTheme(darkDefaults, 'dark-defaults', 'vs-dark');
+    registry.registerMixedTheme(darkVS, 'dark-vs', 'vs-dark');
+    registry.registerMixedTheme(darkPlus, 'dark-plus', 'vs-dark');
+    registry.registerMixedTheme(darkEditor, 'e2-dark', 'vs-dark');
+
+    registry.registerMixedTheme(lightDefaults, 'light-defaults', 'vs');
+    registry.registerMixedTheme(lightVS, 'light-vs', 'vs');
+    registry.registerMixedTheme(lightPlus, 'light-plus', 'vs');
+    registry.registerMixedTheme(lightEditor, 'e2-light', 'vs');
+
+    registry.registerMixedTheme(HCBlackDefault, 'hc-black-defaults', 'hc-black');
+    registry.registerMixedTheme(HCBlack, 'hc-black', 'hc-black');
+    registry.registerMixedTheme(HCEditor, 'e2-hc', 'hc-black');
+  }
+}

package/src/textmate-tokenizer.ts

@@ -0,0 +1,55 @@
+/* eslint-disable @typescript-eslint/no-namespace */
+
+import type monaco from '@difizen/monaco-editor-core';
+import type { IGrammar, StateStack } from 'vscode-textmate';
+import { INITIAL } from 'vscode-textmate';
+
+/**
+ * Options for the TextMate tokenizer.
+ */
+export interface TokenizerOption {
+  /**
+   * Maximum line length that will be handled by the TextMate tokenizer. If the length of the actual line exceeds this
+   * limit, the tokenizer terminates and the tokenization of any subsequent lines might be broken.
+   *
+   * If the `lineLimit` is not defined, it means, there are no line length limits. Otherwise, it must be a positive
+   * integer or an error will be thrown.
+   */
+  lineLimit?: number;
+}
+
+export namespace TokenizerOption {
+  /**
+   * The default TextMate tokenizer option.
+   *
+   * @deprecated Use the current value of `editor.maxTokenizationLineLength` preference instead.
+   */
+  export const DEFAULT: TokenizerOption = {
+    lineLimit: 400,
+  };
+}
+
+export function createTextmateTokenizer(
+  grammar: IGrammar,
+  options: TokenizerOption,
+): monaco.languages.EncodedTokensProvider {
+  if (
+    options.lineLimit !== undefined &&
+    (options.lineLimit <= 0 || !Number.isInteger(options.lineLimit))
+  ) {
+    throw new Error(
+      `The 'lineLimit' must be a positive integer. It was ${options.lineLimit}.`,
+    );
+  }
+  return {
+    getInitialState: () => INITIAL,
+    tokenizeEncoded(
+      line: string,
+      state: monaco.languages.IState,
+    ): monaco.languages.IEncodedLineTokens {
+      const tokenizeLineResult2 = grammar.tokenizeLine2(line, state as StateStack);
+      const { tokens, ruleStack: endState } = tokenizeLineResult2;
+      return { tokens, endState };
+    },
+  };
+}