@codingame/monaco-vscode-editor-api 11.1.2 → 12.0.0

package/vscode/src/vs/editor/standalone/common/monarch/monarchCommon.d.ts

@@ -0,0 +1,72 @@
+export declare enum MonarchBracket {
+    None = 0,
+    Open = 1,
+    Close = -1
+}
+export interface ILexerMin {
+    languageId: string;
+    includeLF: boolean;
+    noThrow: boolean;
+    ignoreCase: boolean;
+    unicode: boolean;
+    usesEmbedded: boolean;
+    defaultToken: string;
+    stateNames: {
+        [stateName: string]: any;
+    };
+    [attr: string]: any;
+}
+export interface ILexer extends ILexerMin {
+    maxStack: number;
+    start: string | null;
+    ignoreCase: boolean;
+    unicode: boolean;
+    tokenPostfix: string;
+    tokenizer: {
+        [stateName: string]: IRule[];
+    };
+    brackets: IBracket[];
+}
+export interface IBracket {
+    token: string;
+    open: string;
+    close: string;
+}
+export type FuzzyAction = IAction | string;
+export declare function isFuzzyActionArr(what: FuzzyAction | FuzzyAction[]): what is FuzzyAction[];
+export declare function isFuzzyAction(what: FuzzyAction | FuzzyAction[]): what is FuzzyAction;
+export declare function isString(what: FuzzyAction): what is string;
+export declare function isIAction(what: FuzzyAction): what is IAction;
+export interface IRule {
+    action: FuzzyAction;
+    matchOnlyAtLineStart: boolean;
+    name: string;
+    resolveRegex(state: string): RegExp;
+}
+export interface IAction {
+    group?: FuzzyAction[];
+    test?: (id: string, matches: string[], state: string, eos: boolean) => FuzzyAction;
+    token?: string;
+    tokenSubst?: boolean;
+    next?: string;
+    nextEmbedded?: string;
+    bracket?: MonarchBracket;
+    log?: string;
+    switchTo?: string;
+    goBack?: number;
+    transform?: (states: string[]) => string[];
+}
+export interface IBranch {
+    name: string;
+    value: FuzzyAction;
+    test?: (id: string, matches: string[], state: string, eos: boolean) => boolean;
+}
+export declare function empty(s: string): boolean;
+export declare function fixCase(lexer: ILexerMin, str: string): string;
+export declare function sanitize(s: string): string;
+export declare function log(lexer: ILexerMin, msg: string): void;
+export declare function createError(lexer: ILexerMin, msg: string): Error;
+export declare function substituteMatches(lexer: ILexerMin, str: string, id: string, matches: string[], state: string): string;
+export declare function substituteMatchesRe(lexer: ILexerMin, str: string, state: string): string;
+export declare function findRules(lexer: ILexer, inState: string): IRule[] | null;
+export declare function stateExists(lexer: ILexerMin, inState: string): boolean;

package/vscode/src/vs/editor/standalone/common/monarch/monarchCommon.js

@@ -0,0 +1,111 @@
+
+
+var MonarchBracket;
+(function (MonarchBracket) {
+    MonarchBracket[MonarchBracket["None"] = 0] = "None";
+    MonarchBracket[MonarchBracket["Open"] = 1] = "Open";
+    MonarchBracket[MonarchBracket["Close"] = -1] = "Close";
+})(MonarchBracket || (MonarchBracket = {}));
+function isFuzzyActionArr(what) {
+    return (Array.isArray(what));
+}
+function isFuzzyAction(what) {
+    return !isFuzzyActionArr(what);
+}
+function isString(what) {
+    return (typeof what === 'string');
+}
+function isIAction(what) {
+    return !isString(what);
+}
+function empty(s) {
+    return (s ? false : true);
+}
+function fixCase(lexer, str) {
+    return (lexer.ignoreCase && str ? str.toLowerCase() : str);
+}
+function sanitize(s) {
+    return s.replace(/[&<>'"_]/g, '-');
+}
+function log(lexer, msg) {
+    console.log(`${lexer.languageId}: ${msg}`);
+}
+function createError(lexer, msg) {
+    return ( new Error(`${lexer.languageId}: ${msg}`));
+}
+function substituteMatches(lexer, str, id, matches, state) {
+    const re = /\$((\$)|(#)|(\d\d?)|[sS](\d\d?)|@(\w+))/g;
+    let stateMatches = null;
+    return str.replace(re, function (full, sub, dollar, hash, n, s, attr, ofs, total) {
+        if (!empty(dollar)) {
+            return '$';
+        }
+        if (!empty(hash)) {
+            return fixCase(lexer, id);
+        }
+        if (!empty(n) && n < matches.length) {
+            return fixCase(lexer, matches[n]);
+        }
+        if (!empty(attr) && lexer && typeof (lexer[attr]) === 'string') {
+            return lexer[attr];
+        }
+        if (stateMatches === null) {
+            stateMatches = state.split('.');
+            stateMatches.unshift(state);
+        }
+        if (!empty(s) && s < stateMatches.length) {
+            return fixCase(lexer, stateMatches[s]);
+        }
+        return '';
+    });
+}
+function substituteMatchesRe(lexer, str, state) {
+    const re = /\$[sS](\d\d?)/g;
+    let stateMatches = null;
+    return str.replace(re, function (full, s) {
+        if (stateMatches === null) {
+            stateMatches = state.split('.');
+            stateMatches.unshift(state);
+        }
+        if (!empty(s) && s < stateMatches.length) {
+            return fixCase(lexer, stateMatches[s]);
+        }
+        return '';
+    });
+}
+function findRules(lexer, inState) {
+    let state = inState;
+    while (state && state.length > 0) {
+        const rules = lexer.tokenizer[state];
+        if (rules) {
+            return rules;
+        }
+        const idx = state.lastIndexOf('.');
+        if (idx < 0) {
+            state = null;
+        }
+        else {
+            state = state.substr(0, idx);
+        }
+    }
+    return null;
+}
+function stateExists(lexer, inState) {
+    let state = inState;
+    while (state && state.length > 0) {
+        const exist = lexer.stateNames[state];
+        if (exist) {
+            return true;
+        }
+        const idx = state.lastIndexOf('.');
+        if (idx < 0) {
+            state = null;
+        }
+        else {
+            state = state.substr(0, idx);
+        }
+    }
+    return false;
+}
+
+export { MonarchBracket, createError, empty, findRules, fixCase, isFuzzyAction, isFuzzyActionArr, isIAction, isString, log, sanitize, stateExists, substituteMatches, substituteMatchesRe };

package/vscode/src/vs/editor/standalone/common/monarch/monarchCompile.d.ts

@@ -0,0 +1,3 @@
+import * as monarchCommon from "./monarchCommon.js";
+import { IMonarchLanguage } from "./monarchTypes.js";
+export declare function compile(languageId: string, json: IMonarchLanguage): monarchCommon.ILexer;

package/vscode/src/vs/editor/standalone/common/monarch/monarchCompile.js

@@ -0,0 +1,476 @@
+
+import { createError, empty, substituteMatchesRe, substituteMatches, fixCase, MonarchBracket, stateExists } from './monarchCommon.js';
+
+function isArrayOf(elemType, obj) {
+    if (!obj) {
+        return false;
+    }
+    if (!(Array.isArray(obj))) {
+        return false;
+    }
+    for (const el of obj) {
+        if (!(elemType(el))) {
+            return false;
+        }
+    }
+    return true;
+}
+function bool(prop, defValue) {
+    if (typeof prop === 'boolean') {
+        return prop;
+    }
+    return defValue;
+}
+function string(prop, defValue) {
+    if (typeof (prop) === 'string') {
+        return prop;
+    }
+    return defValue;
+}
+function arrayToHash(array) {
+    const result = {};
+    for (const e of array) {
+        result[e] = true;
+    }
+    return result;
+}
+function createKeywordMatcher(arr, caseInsensitive = false) {
+    if (caseInsensitive) {
+        arr = ( arr.map(function (x) { return x.toLowerCase(); }));
+    }
+    const hash = arrayToHash(arr);
+    if (caseInsensitive) {
+        return function (word) {
+            return hash[word.toLowerCase()] !== undefined && hash.hasOwnProperty(word.toLowerCase());
+        };
+    }
+    else {
+        return function (word) {
+            return hash[word] !== undefined && hash.hasOwnProperty(word);
+        };
+    }
+}
+function compileRegExp(lexer, str, handleSn) {
+    str = str.replace(/@@/g, `\x01`);
+    let n = 0;
+    let hadExpansion;
+    do {
+        hadExpansion = false;
+        str = str.replace(/@(\w+)/g, function (s, attr) {
+            hadExpansion = true;
+            let sub = '';
+            if (typeof (lexer[attr]) === 'string') {
+                sub = lexer[attr];
+            }
+            else if (lexer[attr] && lexer[attr] instanceof RegExp) {
+                sub = lexer[attr].source;
+            }
+            else {
+                if (lexer[attr] === undefined) {
+                    throw createError(lexer, 'language definition does not contain attribute \'' + attr + '\', used at: ' + str);
+                }
+                else {
+                    throw createError(lexer, 'attribute reference \'' + attr + '\' must be a string, used at: ' + str);
+                }
+            }
+            return (empty(sub) ? '' : '(?:' + sub + ')');
+        });
+        n++;
+    } while (hadExpansion && n < 5);
+    str = str.replace(/\x01/g, '@');
+    const flags = (lexer.ignoreCase ? 'i' : '') + (lexer.unicode ? 'u' : '');
+    if (handleSn) {
+        const match = str.match(/\$[sS](\d\d?)/g);
+        if (match) {
+            let lastState = null;
+            let lastRegEx = null;
+            return (state) => {
+                if (lastRegEx && lastState === state) {
+                    return lastRegEx;
+                }
+                lastState = state;
+                lastRegEx = ( new RegExp(substituteMatchesRe(lexer, str, state), flags));
+                return lastRegEx;
+            };
+        }
+    }
+    return ( new RegExp(str, flags));
+}
+function selectScrutinee(id, matches, state, num) {
+    if (num < 0) {
+        return id;
+    }
+    if (num < matches.length) {
+        return matches[num];
+    }
+    if (num >= 100) {
+        num = num - 100;
+        const parts = state.split('.');
+        parts.unshift(state);
+        if (num < parts.length) {
+            return parts[num];
+        }
+    }
+    return null;
+}
+function createGuard(lexer, ruleName, tkey, val) {
+    let scrut = -1;
+    let oppat = tkey;
+    let matches = tkey.match(/^\$(([sS]?)(\d\d?)|#)(.*)$/);
+    if (matches) {
+        if (matches[3]) {
+            scrut = parseInt(matches[3]);
+            if (matches[2]) {
+                scrut = scrut + 100;
+            }
+        }
+        oppat = matches[4];
+    }
+    let op = '~';
+    let pat = oppat;
+    if (!oppat || oppat.length === 0) {
+        op = '!=';
+        pat = '';
+    }
+    else if (/^\w*$/.test(pat)) {
+        op = '==';
+    }
+    else {
+        matches = oppat.match(/^(@|!@|~|!~|==|!=)(.*)$/);
+        if (matches) {
+            op = matches[1];
+            pat = matches[2];
+        }
+    }
+    let tester;
+    if ((op === '~' || op === '!~') && /^(\w|\|)*$/.test(pat)) {
+        const inWords = createKeywordMatcher(pat.split('|'), lexer.ignoreCase);
+        tester = function (s) { return (op === '~' ? inWords(s) : !inWords(s)); };
+    }
+    else if (op === '@' || op === '!@') {
+        const words = lexer[pat];
+        if (!words) {
+            throw createError(lexer, 'the @ match target \'' + pat + '\' is not defined, in rule: ' + ruleName);
+        }
+        if (!(isArrayOf(function (elem) { return (typeof (elem) === 'string'); }, words))) {
+            throw createError(lexer, 'the @ match target \'' + pat + '\' must be an array of strings, in rule: ' + ruleName);
+        }
+        const inWords = createKeywordMatcher(words, lexer.ignoreCase);
+        tester = function (s) { return (op === '@' ? inWords(s) : !inWords(s)); };
+    }
+    else if (op === '~' || op === '!~') {
+        if (pat.indexOf('$') < 0) {
+            const re = compileRegExp(lexer, '^' + pat + '$', false);
+            tester = function (s) { return (op === '~' ? re.test(s) : !re.test(s)); };
+        }
+        else {
+            tester = function (s, id, matches, state) {
+                const re = compileRegExp(lexer, '^' + substituteMatches(lexer, pat, id, matches, state) + '$', false);
+                return re.test(s);
+            };
+        }
+    }
+    else {
+        if (pat.indexOf('$') < 0) {
+            const patx = fixCase(lexer, pat);
+            tester = function (s) { return (op === '==' ? s === patx : s !== patx); };
+        }
+        else {
+            const patx = fixCase(lexer, pat);
+            tester = function (s, id, matches, state, eos) {
+                const patexp = substituteMatches(lexer, patx, id, matches, state);
+                return (op === '==' ? s === patexp : s !== patexp);
+            };
+        }
+    }
+    if (scrut === -1) {
+        return {
+            name: tkey, value: val, test: function (id, matches, state, eos) {
+                return tester(id, id, matches, state, eos);
+            }
+        };
+    }
+    else {
+        return {
+            name: tkey, value: val, test: function (id, matches, state, eos) {
+                const scrutinee = selectScrutinee(id, matches, state, scrut);
+                return tester(!scrutinee ? '' : scrutinee, id, matches, state, eos);
+            }
+        };
+    }
+}
+function compileAction(lexer, ruleName, action) {
+    if (!action) {
+        return { token: '' };
+    }
+    else if (typeof (action) === 'string') {
+        return action;
+    }
+    else if (action.token || action.token === '') {
+        if (typeof (action.token) !== 'string') {
+            throw createError(lexer, 'a \'token\' attribute must be of type string, in rule: ' + ruleName);
+        }
+        else {
+            const newAction = { token: action.token };
+            if (action.token.indexOf('$') >= 0) {
+                newAction.tokenSubst = true;
+            }
+            if (typeof (action.bracket) === 'string') {
+                if (action.bracket === '@open') {
+                    newAction.bracket = MonarchBracket.Open;
+                }
+                else if (action.bracket === '@close') {
+                    newAction.bracket = MonarchBracket.Close;
+                }
+                else {
+                    throw createError(lexer, 'a \'bracket\' attribute must be either \'@open\' or \'@close\', in rule: ' + ruleName);
+                }
+            }
+            if (action.next) {
+                if (typeof (action.next) !== 'string') {
+                    throw createError(lexer, 'the next state must be a string value in rule: ' + ruleName);
+                }
+                else {
+                    let next = action.next;
+                    if (!/^(@pop|@push|@popall)$/.test(next)) {
+                        if (next[0] === '@') {
+                            next = next.substr(1);
+                        }
+                        if (next.indexOf('$') < 0) {
+                            if (!stateExists(lexer, substituteMatches(lexer, next, '', [], ''))) {
+                                throw createError(lexer, 'the next state \'' + action.next + '\' is not defined in rule: ' + ruleName);
+                            }
+                        }
+                    }
+                    newAction.next = next;
+                }
+            }
+            if (typeof (action.goBack) === 'number') {
+                newAction.goBack = action.goBack;
+            }
+            if (typeof (action.switchTo) === 'string') {
+                newAction.switchTo = action.switchTo;
+            }
+            if (typeof (action.log) === 'string') {
+                newAction.log = action.log;
+            }
+            if (typeof (action.nextEmbedded) === 'string') {
+                newAction.nextEmbedded = action.nextEmbedded;
+                lexer.usesEmbedded = true;
+            }
+            return newAction;
+        }
+    }
+    else if (Array.isArray(action)) {
+        const results = [];
+        for (let i = 0, len = action.length; i < len; i++) {
+            results[i] = compileAction(lexer, ruleName, action[i]);
+        }
+        return { group: results };
+    }
+    else if (action.cases) {
+        const cases = [];
+        for (const tkey in action.cases) {
+            if (action.cases.hasOwnProperty(tkey)) {
+                const val = compileAction(lexer, ruleName, action.cases[tkey]);
+                if (tkey === '@default' || tkey === '@' || tkey === '') {
+                    cases.push({ test: undefined, value: val, name: tkey });
+                }
+                else if (tkey === '@eos') {
+                    cases.push({ test: function (id, matches, state, eos) { return eos; }, value: val, name: tkey });
+                }
+                else {
+                    cases.push(createGuard(lexer, ruleName, tkey, val));
+                }
+            }
+        }
+        const def = lexer.defaultToken;
+        return {
+            test: function (id, matches, state, eos) {
+                for (const _case of cases) {
+                    const didmatch = (!_case.test || _case.test(id, matches, state, eos));
+                    if (didmatch) {
+                        return _case.value;
+                    }
+                }
+                return def;
+            }
+        };
+    }
+    else {
+        throw createError(lexer, 'an action must be a string, an object with a \'token\' or \'cases\' attribute, or an array of actions; in rule: ' + ruleName);
+    }
+}
+class Rule {
+    constructor(name) {
+        this.regex = ( new RegExp(''));
+        this.action = { token: '' };
+        this.matchOnlyAtLineStart = false;
+        this.name = '';
+        this.name = name;
+    }
+    setRegex(lexer, re) {
+        let sregex;
+        if (typeof (re) === 'string') {
+            sregex = re;
+        }
+        else if (re instanceof RegExp) {
+            sregex = re.source;
+        }
+        else {
+            throw createError(lexer, 'rules must start with a match string or regular expression: ' + this.name);
+        }
+        this.matchOnlyAtLineStart = (sregex.length > 0 && sregex[0] === '^');
+        this.name = this.name + ': ' + sregex;
+        this.regex = compileRegExp(lexer, '^(?:' + (this.matchOnlyAtLineStart ? sregex.substr(1) : sregex) + ')', true);
+    }
+    setAction(lexer, act) {
+        this.action = compileAction(lexer, this.name, act);
+    }
+    resolveRegex(state) {
+        if (this.regex instanceof RegExp) {
+            return this.regex;
+        }
+        else {
+            return this.regex(state);
+        }
+    }
+}
+function compile(languageId, json) {
+    if (!json || typeof (json) !== 'object') {
+        throw ( new Error('Monarch: expecting a language definition object'));
+    }
+    const lexer = {
+        languageId: languageId,
+        includeLF: bool(json.includeLF, false),
+        noThrow: false,
+        maxStack: 100,
+        start: (typeof json.start === 'string' ? json.start : null),
+        ignoreCase: bool(json.ignoreCase, false),
+        unicode: bool(json.unicode, false),
+        tokenPostfix: string(json.tokenPostfix, '.' + languageId),
+        defaultToken: string(json.defaultToken, 'source'),
+        usesEmbedded: false,
+        stateNames: {},
+        tokenizer: {},
+        brackets: []
+    };
+    const lexerMin = json;
+    lexerMin.languageId = languageId;
+    lexerMin.includeLF = lexer.includeLF;
+    lexerMin.ignoreCase = lexer.ignoreCase;
+    lexerMin.unicode = lexer.unicode;
+    lexerMin.noThrow = lexer.noThrow;
+    lexerMin.usesEmbedded = lexer.usesEmbedded;
+    lexerMin.stateNames = json.tokenizer;
+    lexerMin.defaultToken = lexer.defaultToken;
+    function addRules(state, newrules, rules) {
+        for (const rule of rules) {
+            let include = rule.include;
+            if (include) {
+                if (typeof (include) !== 'string') {
+                    throw createError(lexer, 'an \'include\' attribute must be a string at: ' + state);
+                }
+                if (include[0] === '@') {
+                    include = include.substr(1);
+                }
+                if (!json.tokenizer[include]) {
+                    throw createError(lexer, 'include target \'' + include + '\' is not defined at: ' + state);
+                }
+                addRules(state + '.' + include, newrules, json.tokenizer[include]);
+            }
+            else {
+                const newrule = ( new Rule(state));
+                if (Array.isArray(rule) && rule.length >= 1 && rule.length <= 3) {
+                    newrule.setRegex(lexerMin, rule[0]);
+                    if (rule.length >= 3) {
+                        if (typeof (rule[1]) === 'string') {
+                            newrule.setAction(lexerMin, { token: rule[1], next: rule[2] });
+                        }
+                        else if (typeof (rule[1]) === 'object') {
+                            const rule1 = rule[1];
+                            rule1.next = rule[2];
+                            newrule.setAction(lexerMin, rule1);
+                        }
+                        else {
+                            throw createError(lexer, 'a next state as the last element of a rule can only be given if the action is either an object or a string, at: ' + state);
+                        }
+                    }
+                    else {
+                        newrule.setAction(lexerMin, rule[1]);
+                    }
+                }
+                else {
+                    if (!rule.regex) {
+                        throw createError(lexer, 'a rule must either be an array, or an object with a \'regex\' or \'include\' field at: ' + state);
+                    }
+                    if (rule.name) {
+                        if (typeof rule.name === 'string') {
+                            newrule.name = rule.name;
+                        }
+                    }
+                    if (rule.matchOnlyAtStart) {
+                        newrule.matchOnlyAtLineStart = bool(rule.matchOnlyAtLineStart, false);
+                    }
+                    newrule.setRegex(lexerMin, rule.regex);
+                    newrule.setAction(lexerMin, rule.action);
+                }
+                newrules.push(newrule);
+            }
+        }
+    }
+    if (!json.tokenizer || typeof (json.tokenizer) !== 'object') {
+        throw createError(lexer, 'a language definition must define the \'tokenizer\' attribute as an object');
+    }
+    lexer.tokenizer = [];
+    for (const key in json.tokenizer) {
+        if (json.tokenizer.hasOwnProperty(key)) {
+            if (!lexer.start) {
+                lexer.start = key;
+            }
+            const rules = json.tokenizer[key];
+            lexer.tokenizer[key] = ( new Array());
+            addRules('tokenizer.' + key, lexer.tokenizer[key], rules);
+        }
+    }
+    lexer.usesEmbedded = lexerMin.usesEmbedded;
+    if (json.brackets) {
+        if (!(Array.isArray(json.brackets))) {
+            throw createError(lexer, 'the \'brackets\' attribute must be defined as an array');
+        }
+    }
+    else {
+        json.brackets = [
+            { open: '{', close: '}', token: 'delimiter.curly' },
+            { open: '[', close: ']', token: 'delimiter.square' },
+            { open: '(', close: ')', token: 'delimiter.parenthesis' },
+            { open: '<', close: '>', token: 'delimiter.angle' }
+        ];
+    }
+    const brackets = [];
+    for (const el of json.brackets) {
+        let desc = el;
+        if (desc && Array.isArray(desc) && desc.length === 3) {
+            desc = { token: desc[2], open: desc[0], close: desc[1] };
+        }
+        if (desc.open === desc.close) {
+            throw createError(lexer, 'open and close brackets in a \'brackets\' attribute must be different: ' + desc.open +
+                '\n hint: use the \'bracket\' attribute if matching on equal brackets is required.');
+        }
+        if (typeof desc.open === 'string' && typeof desc.token === 'string' && typeof desc.close === 'string') {
+            brackets.push({
+                token: desc.token + lexer.tokenPostfix,
+                open: fixCase(lexer, desc.open),
+                close: fixCase(lexer, desc.close)
+            });
+        }
+        else {
+            throw createError(lexer, 'every element in the \'brackets\' array must be a \'{open,close,token}\' object or array');
+        }
+    }
+    lexer.brackets = brackets;
+    lexer.noThrow = true;
+    return lexer;
+}
+
+export { compile };

package/vscode/src/vs/editor/standalone/common/monarch/monarchLexer.d.ts

@@ -0,0 +1,33 @@
+import { Disposable, IDisposable } from "vscode/vscode/vs/base/common/lifecycle";
+import * as languages from "vscode/vscode/vs/editor/common/languages";
+import { ILanguageService } from "vscode/vscode/vs/editor/common/languages/language";
+import * as monarchCommon from "./monarchCommon.js";
+import { IStandaloneThemeService } from "vscode/vscode/vs/editor/standalone/common/standaloneTheme";
+import { IConfigurationService } from "vscode/vscode/vs/platform/configuration/common/configuration.service";
+export type ILoadStatus = {
+    loaded: true;
+} | {
+    loaded: false;
+    promise: Promise<void>;
+};
+export declare class MonarchTokenizer extends Disposable implements languages.ITokenizationSupport, IDisposable {
+    private readonly _configurationService;
+    private readonly _languageService;
+    private readonly _standaloneThemeService;
+    private readonly _languageId;
+    private readonly _lexer;
+    private readonly _embeddedLanguages;
+    embeddedLoaded: Promise<void>;
+    private _maxTokenizationLineLength;
+    constructor(languageService: ILanguageService, standaloneThemeService: IStandaloneThemeService, languageId: string, lexer: monarchCommon.ILexer, _configurationService: IConfigurationService);
+    getLoadStatus(): ILoadStatus;
+    getInitialState(): languages.IState;
+    tokenize(line: string, hasEOL: boolean, lineState: languages.IState): languages.TokenizationResult;
+    tokenizeEncoded(line: string, hasEOL: boolean, lineState: languages.IState): languages.EncodedTokenizationResult;
+    private _tokenize;
+    private _findLeavingNestedLanguageOffset;
+    private _nestedTokenize;
+    private _safeRuleName;
+    private _myTokenize;
+    private _getNestedEmbeddedLanguageData;
+}