@opensumi/ide-editor 2.21.13 → 2.22.0

package/src/browser/monaco-contrib/tokenizer/textmate-registry.ts

@@ -0,0 +1,107 @@
+/** ******************************************************************************
+ * Copyright (C) 2018 Ericsson and others.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v. 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ *
+ * This Source Code may also be made available under the following Secondary
+ * Licenses when the conditions for such availability set forth in the Eclipse
+ * Public License v. 2.0 are satisfied: GNU General Public License, version 2
+ * with the GNU Classpath Exception which is available at
+ * https://www.gnu.org/software/classpath/license.html.
+ *
+ * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
+ ********************************************************************************/
+
+ // Some code copied and modified from https://github.com/eclipse-theia/theia/tree/v1.14.0/packages/monaco/src/browser/textmate/textmate-registry.ts
+
+import { IGrammarConfiguration } from 'vscode-textmate';
+
+import { Injectable } from '@opensumi/di';
+import { getDebugLogger, URI } from '@opensumi/ide-core-common';
+
+import { TokenizerOption } from './textmate-tokenizer';
+
+export interface TextmateGrammarConfiguration extends IGrammarConfiguration {
+  /**
+   * Optional options to further refine the tokenization of the grammar.
+   */
+  readonly tokenizerOption?: TokenizerOption;
+}
+
+export interface GrammarDefinitionProvider {
+  getGrammarDefinition(): Promise<GrammarDefinition>;
+  getInjections?(scopeName: string): string[];
+}
+
+export interface GrammarDefinition {
+  format: 'json' | 'plist';
+  content?: object | string;
+  location: URI;
+}
+
+@Injectable()
+export class TextmateRegistry {
+  readonly scopeToProvider = new Map<string, GrammarDefinitionProvider>();
+  readonly languageToConfig = new Map<string, () => TextmateGrammarConfiguration>();
+  readonly languageIdToScope = new Map<string, string>();
+
+  // 将语法定义描述绑定到某一个Scope：source.json.comments
+  registerTextmateGrammarScope(scope: string, description: GrammarDefinitionProvider) {
+    if (this.scopeToProvider.has(scope)) {
+      getDebugLogger().warn(new Error(`a registered grammar provider for '${scope}' scope is overridden`));
+    }
+    this.scopeToProvider.set(scope, description);
+
+    return () => {
+      this.scopeToProvider.delete(scope);
+    };
+  }
+
+  getProvider(scope: string): GrammarDefinitionProvider | undefined {
+    return this.scopeToProvider.get(scope);
+  }
+
+  mapLanguageIdToTextmateGrammar(languageId: string, scope: string) {
+    const existingScope = this.getScope(languageId);
+    if (typeof existingScope === 'string') {
+      getDebugLogger().warn(
+        new Error(`'${languageId}' language is remapped from '${existingScope}' to '${scope}' scope`),
+      );
+    }
+    this.languageIdToScope.set(languageId, scope);
+
+    return () => {
+      this.languageIdToScope.delete(languageId);
+    };
+  }
+
+  getScope(languageId: string): string | undefined {
+    return this.languageIdToScope.get(languageId);
+  }
+
+  getLanguageId(scope: string): string | undefined {
+    for (const key of this.languageIdToScope.keys()) {
+      if (this.languageIdToScope.get(key) === scope) {
+        return key;
+      }
+    }
+    return undefined;
+  }
+
+  registerGrammarConfiguration(languageId: string, getConfig: () => TextmateGrammarConfiguration) {
+    if (this.languageToConfig.has(languageId)) {
+      // console.warn(new Error(`a registered grammar configuration for '${languageId}' language is overridden`));
+    }
+    this.languageToConfig.set(languageId, getConfig);
+
+    return () => {
+      this.languageToConfig.delete(languageId);
+    };
+  }
+
+  getGrammarConfiguration(languageId: string): () => TextmateGrammarConfiguration {
+    return this.languageToConfig.get(languageId) || (() => ({}));
+  }
+}

package/src/browser/monaco-contrib/tokenizer/textmate-tokenizer.ts

@@ -0,0 +1,104 @@
+import { INITIAL, StackElement, IGrammar } from 'vscode-textmate';
+
+import { Disposable, Emitter, Event } from '@opensumi/ide-core-common/lib/utils';
+import {
+  MetadataConsts,
+  TokenMetadata,
+} from '@opensumi/monaco-editor-core/esm/vs/editor/common/encodedTokenAttributes';
+import * as monaco from '@opensumi/monaco-editor-core/esm/vs/editor/editor.api';
+
+export class TokenizerState implements monaco.languages.IState {
+  constructor(public readonly ruleStack: StackElement) {}
+
+  clone(): monaco.languages.IState {
+    return new TokenizerState(this.ruleStack);
+  }
+
+  equals(other: monaco.languages.IState): boolean {
+    return other instanceof TokenizerState && (other === this || other.ruleStack === this.ruleStack);
+  }
+}
+
+/**
+ * Options for the TextMate tokenizer.
+ */
+export interface TokenizerOption {
+  /**
+   * Maximum line length that will be handled by the TextMate tokenizer. If the length of the actual line exceeds this
+   * limit, the tokenizer terminates and the tokenization of any subsequent lines might be broken.
+   *
+   * If the `lineLimit` is not defined, it means, there are no line length limits. Otherwise, it must be a positive
+   * integer or an error will be thrown.
+   */
+  readonly lineLimit?: number;
+}
+
+export class TextMateTokenizer extends Disposable implements monaco.languages.EncodedTokensProvider {
+  private readonly seenLanguages: boolean[];
+
+  private readonly onDidEncounterLanguageEmitter: Emitter<number> = new Emitter<number>();
+  public readonly onDidEncounterLanguage: Event<number> = this.onDidEncounterLanguageEmitter.event;
+
+  constructor(
+    private readonly grammar: IGrammar,
+    private readonly options: TokenizerOption,
+    private readonly conatinsEmbeddedLanguages?: boolean,
+  ) {
+    super();
+    this.seenLanguages = [];
+  }
+
+  getInitialState(): monaco.languages.IState {
+    return new TokenizerState(INITIAL);
+  }
+
+  tokenizeEncoded(line: string, state: TokenizerState): monaco.languages.IEncodedLineTokens {
+    // copied from vscode/src/vs/editor/common/modes/nullMode.ts
+    if (this.options.lineLimit !== undefined && line.length > this.options.lineLimit) {
+      const tokens = new Uint32Array(2);
+      tokens[0] = 0;
+      tokens[1] =
+        (1 << MetadataConsts.LANGUAGEID_OFFSET) |
+        (0 << MetadataConsts.TOKEN_TYPE_OFFSET) |
+        (0 << MetadataConsts.FONT_STYLE_OFFSET) |
+        (1 << MetadataConsts.FOREGROUND_OFFSET) |
+        (2 << MetadataConsts.BACKGROUND_OFFSET) |
+        (MetadataConsts.BALANCED_BRACKETS_MASK >>> 0);
+      // Line is too long to be tokenized
+      return {
+        endState: new TokenizerState(INITIAL),
+        tokens,
+      };
+    }
+
+    const result = this.grammar.tokenizeLine2(line, state.ruleStack, 500);
+
+    if (this.conatinsEmbeddedLanguages) {
+      const seenLanguages = this.seenLanguages;
+      const tokens = result.tokens;
+
+      // Must check if any of the embedded languages was hit
+      for (let i = 0, len = tokens.length >>> 1; i < len; i++) {
+        const metadata = tokens[(i << 1) + 1];
+        const languageId = TokenMetadata.getLanguageId(metadata);
+        if (!seenLanguages[languageId]) {
+          seenLanguages[languageId] = true;
+          this.onDidEncounterLanguageEmitter.fire(languageId);
+        }
+      }
+    }
+
+    return {
+      endState: new TokenizerState(result.ruleStack),
+      tokens: result.tokens,
+    };
+  }
+
+  tokenize?(line: string, state: TokenizerState): monaco.languages.ILineTokens {
+    const result = this.grammar.tokenizeLine(line, state.ruleStack);
+    return {
+      endState: new TokenizerState(result.ruleStack),
+      tokens: result.tokens.map((t) => ({ ...t, scopes: t.scopes.join('\r\n') })),
+    };
+  }
+}