@opensumi/ide-editor 2.12.1-next-079c1930

package/lib/browser/monaco-contrib/tokenizer/textmate-tokenizer.js

@@ -0,0 +1,75 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createTextmateTokenizer = exports.TokenizerState = void 0;
+/********************************************************************************
+ * Copyright (C) 2018 Ericsson and others.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v. 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ *
+ * This Source Code may also be made available under the following Secondary
+ * Licenses when the conditions for such availability set forth in the Eclipse
+ * Public License v. 2.0 are satisfied: GNU General Public License, version 2
+ * with the GNU Classpath Exception which is available at
+ * https://www.gnu.org/software/classpath/license.html.
+ *
+ * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
+ ********************************************************************************/
+const vscode_textmate_1 = require("vscode-textmate");
+class TokenizerState {
+    constructor(ruleStack) {
+        this.ruleStack = ruleStack;
+    }
+    clone() {
+        return new TokenizerState(this.ruleStack);
+    }
+    equals(other) {
+        return (other instanceof TokenizerState &&
+            (other === this || other.ruleStack === this.ruleStack));
+    }
+}
+exports.TokenizerState = TokenizerState;
+function createTextmateTokenizer(grammar, options) {
+    if (options.lineLimit !== undefined &&
+        (options.lineLimit <= 0 || !Number.isInteger(options.lineLimit))) {
+        throw new Error(`The 'lineLimit' must be a positive integer. It was ${options.lineLimit}.`);
+    }
+    return {
+        getInitialState: () => new TokenizerState(vscode_textmate_1.INITIAL),
+        tokenizeEncoded(line, state) {
+            // copied from vscode/src/vs/editor/common/modes/nullMode.ts
+            if (options.lineLimit !== undefined && line.length > options.lineLimit) {
+                const tokens = new Uint32Array(2);
+                tokens[0] = 0;
+                tokens[1] =
+                    ((1 << 0 /* LANGUAGEID_OFFSET */) |
+                        (0 << 8 /* TOKEN_TYPE_OFFSET */) |
+                        (0 << 11 /* FONT_STYLE_OFFSET */) |
+                        (1 << 14 /* FOREGROUND_OFFSET */) |
+                        (2 << 23 /* BACKGROUND_OFFSET */)) >>>
+                        0;
+                // Line is too long to be tokenized
+                return {
+                    endState: new TokenizerState(vscode_textmate_1.INITIAL),
+                    tokens,
+                };
+            }
+            const result = grammar.tokenizeLine2(line, state.ruleStack);
+            return {
+                endState: new TokenizerState(result.ruleStack),
+                tokens: result.tokens,
+            };
+        },
+        tokenize(line, state) {
+            const result = grammar.tokenizeLine(line, state.ruleStack);
+            return {
+                endState: new TokenizerState(result.ruleStack),
+                tokens: result.tokens
+                    .map((t) => (Object.assign(Object.assign({}, t), { scopes: t.scopes.join('\r\n') }))),
+            };
+        },
+    };
+}
+exports.createTextmateTokenizer = createTextmateTokenizer;
+//# sourceMappingURL=textmate-tokenizer.js.map

package/lib/browser/monaco-contrib/tokenizer/textmate-tokenizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"textmate-tokenizer.js","sourceRoot":"","sources":["../../../../src/browser/monaco-contrib/tokenizer/textmate-tokenizer.ts"],"names":[],"mappings":";;;AACA;;;;;;;;;;;;;;kFAckF;AAElF,qDAAkE;AAElE,MAAa,cAAc;IACzB,YAA4B,SAAuB;QAAvB,cAAS,GAAT,SAAS,CAAc;IAAG,CAAC;IAEvD,KAAK;QACH,OAAO,IAAI,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC5C,CAAC;IAED,MAAM,CAAC,KAA8B;QACnC,OAAO,CACL,KAAK,YAAY,cAAc;YAC/B,CAAC,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,SAAS,KAAK,IAAI,CAAC,SAAS,CAAC,CACvD,CAAC;IACJ,CAAC;CACF;AAbD,wCAaC;AAwCD,SAAgB,uBAAuB,CACrC,OAAiB,EACjB,OAAwB;IAExB,IACE,OAAO,CAAC,SAAS,KAAK,SAAS;QAC/B,CAAC,OAAO,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAChE;QACA,MAAM,IAAI,KAAK,CACb,sDAAsD,OAAO,CAAC,SAAS,GAAG,CAC3E,CAAC;KACH;IACD,OAAO;QACL,eAAe,EAAE,GAAG,EAAE,CAAC,IAAI,cAAc,CAAC,yBAAO,CAAC;QAClD,eAAe,CAAC,IAAY,EAAE,KAAqB;YACjD,4DAA4D;YAC5D,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,IAAI,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE;gBACtE,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,CAAC,CAAC,CAAC;gBAClC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;gBACd,MAAM,CAAC,CAAC,CAAC;oBACP,CAAC,CAAC,CAAC,6BAAoC,CAAC;wBACtC,CAAC,CAAC,6BAAoC,CAAC;wBACvC,CAAC,CAAC,8BAAoC,CAAC;wBACvC,CAAC,CAAC,8BAAoC,CAAC;wBACvC,CAAC,CAAC,8BAAoC,CAAC,CAAC;wBAC1C,CAAC,CAAC;gBACJ,mCAAmC;gBACnC,OAAO;oBACL,QAAQ,EAAE,IAAI,cAAc,CAAC,yBAAO,CAAC;oBACrC,MAAM;iBACP,CAAC;aACH;YACD,MAAM,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,IAAI,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;YAC5D,OAAO;gBACL,QAAQ,EAAE,IAAI,cAAc,CAAC,MAAM,CAAC,SAAS,CAAC;gBAC9C,MAAM,EAAE,MAAM,CAAC,MAAM;aACtB,CAAC;QACJ,CAAC;QACD,QAAQ,CAAC,IAAY,EAAE,KAAqB;YAC1C,MAAM,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,IAAI,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;YAC3D,OAAO;gBACL,QAAQ,EAAE,IAAI,cAAc,CAAC,MAAM,CAAC,SAAS,CAAC;gBAC9C,MAAM,EAAE,MAAM,CAAC,MAAM;qBAClB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,iCAAM,CAAC,KAAE,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,IAAG,CAAC;aACzD,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC;AA/CD,0DA+CC"}

package/lib/browser/monaco-contrib/tokenizer/textmate.service.d.ts

@@ -0,0 +1,54 @@
+import { IThemeData } from '@opensumi/ide-theme';
+import { URI } from '@opensumi/ide-core-common';
+import { Registry } from 'vscode-textmate';
+import { GrammarsContribution, ITextmateTokenizerService, LanguagesContribution } from '@opensumi/ide-monaco/lib/browser/contrib/tokenizer';
+import { WithEventBus, PreferenceService } from '@opensumi/ide-core-browser';
+import { IEditorDocumentModelService } from '../../doc-model/types';
+export declare function getEncodedLanguageId(languageId: string): number;
+export declare function getLegalThemeName(name: string): string;
+export declare type CharacterPair = [string, string];
+export declare class TextmateService extends WithEventBus implements ITextmateTokenizerService {
+    private textmateRegistry;
+    private fileServiceClient;
+    preferenceService: PreferenceService;
+    private logger;
+    private readonly monacoService;
+    editorDocumentModelService: IEditorDocumentModelService;
+    private readonly appConfig;
+    grammarRegistry: Registry;
+    private registeredGrammarDisposableCollection;
+    private injections;
+    private activatedLanguage;
+    initialized: boolean;
+    private dynamicLanguages;
+    /**
+     * start contribution 做初始化
+     */
+    init(): void;
+    listenThemeChange(): void;
+    registerLanguage(language: LanguagesContribution, extPath: URI): Promise<void>;
+    registerLanguages(languages: LanguagesContribution[], extPath: URI): Promise<void>;
+    registerGrammar(grammar: GrammarsContribution, extPath: URI): Promise<void>;
+    unregisterGrammar(grammar: GrammarsContribution): void;
+    doRegisterGrammar(grammar: GrammarsContribution): void;
+    activateLanguage(languageId: string): Promise<void>;
+    private setTokensProviderByLanguageId;
+    setTheme(theme: IThemeData): void;
+    private generateEncodedTokenColors;
+    private createRegex;
+    private safeParseJSON;
+    private convertFolding;
+    private convertIndentationRules;
+    private convertEmbeddedLanguages;
+    private convertTokenTypes;
+    private extractValidSurroundingPairs;
+    private extractValidBrackets;
+    private extractValidAutoClosingPairs;
+    private extractValidCommentRule;
+    private initGrammarRegistry;
+    private activateLanguages;
+    private getOnigLib;
+    private listenPreferenceChange;
+    testTokenize(line: string, languageId: string): Promise<void>;
+}
+//# sourceMappingURL=textmate.service.d.ts.map

package/lib/browser/monaco-contrib/tokenizer/textmate.service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"textmate.service.d.ts","sourceRoot":"","sources":["../../../../src/browser/monaco-contrib/tokenizer/textmate.service.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAGjD,OAAO,EAAE,GAAG,EAAc,MAAM,2BAA2B,CAAC;AAM5D,OAAO,EAAE,QAAQ,EAAyF,MAAM,iBAAiB,CAAC;AAClI,OAAO,EAAe,oBAAoB,EAAE,yBAAyB,EAAE,qBAAqB,EAAY,MAAM,oDAAoD,CAAC;AAEnK,OAAO,EAAE,YAAY,EAAoC,iBAAiB,EAA0F,MAAM,4BAA4B,CAAC;AAGvM,OAAO,EAAE,2BAA2B,EAAE,MAAM,uBAAuB,CAAC;AAKpE,wBAAgB,oBAAoB,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CAE/D;AAED,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,UAE7C;AAuBD,oBAAY,aAAa,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAQ7C,qBACa,eAAgB,SAAQ,YAAa,YAAW,yBAAyB;IAEpF,OAAO,CAAC,gBAAgB,CAAmB;IAG3C,OAAO,CAAC,iBAAiB,CAAqB;IAG9C,iBAAiB,EAAE,iBAAiB,CAAC;IAGrC,OAAO,CAAC,MAAM,CAAU;IAGxB,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAgB;IAG9C,0BAA0B,EAAE,2BAA2B,CAAC;IAGxD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAY;IAE/B,eAAe,EAAE,QAAQ,CAAC;IAEjC,OAAO,CAAC,qCAAqC,CAAiC;IAE9E,OAAO,CAAC,UAAU,CAA+B;IAEjD,OAAO,CAAC,iBAAiB,CAAqB;IAEvC,WAAW,UAAS;IAE3B,OAAO,CAAC,gBAAgB,CAAiC;IAEzD;;OAEG;IACH,IAAI;IAOJ,iBAAiB;IAOX,gBAAgB,CAAC,QAAQ,EAAE,qBAAqB,EAAE,OAAO,EAAE,GAAG;IAI9D,iBAAiB,CAAC,SAAS,EAAE,qBAAqB,EAAE,EAAE,OAAO,EAAE,GAAG;IAwElE,eAAe,CAAC,OAAO,EAAE,oBAAoB,EAAE,OAAO,EAAE,GAAG;IAUjE,iBAAiB,CAAC,OAAO,EAAE,oBAAoB;IAQ/C,iBAAiB,CAAC,OAAO,EAAE,oBAAoB;IAwDzC,gBAAgB,CAAC,UAAU,EAAE,MAAM;YAU3B,6BAA6B;IA4BpC,QAAQ,CAAC,KAAK,EAAE,UAAU;IAMjC,OAAO,CAAC,0BAA0B;IAUlC,OAAO,CAAC,WAAW;IAOnB,OAAO,CAAC,aAAa;IAYrB,OAAO,CAAC,cAAc;IAoBtB,OAAO,CAAC,uBAAuB;IAiB/B,OAAO,CAAC,wBAAwB;IAqBhC,OAAO,CAAC,iBAAiB;IA0BzB,OAAO,CAAC,4BAA4B;IAyCpC,OAAO,CAAC,oBAAoB;IAwB5B,OAAO,CAAC,4BAA4B;IA8CpC,OAAO,CAAC,uBAAuB;YA8BjB,mBAAmB;IAsCjC,OAAO,CAAC,iBAAiB;YAQX,UAAU;IAmBxB,OAAO,CAAC,sBAAsB;IAUxB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM;CAuBpD"}