@termaxjs/editor 0.1.0

package/dist/highlight/microTokenizer.js

@@ -0,0 +1,951 @@
+const KEYWORDS_BY_LANG = {
+    typescript: new Set([
+        "break",
+        "case",
+        "catch",
+        "class",
+        "const",
+        "continue",
+        "debugger",
+        "default",
+        "delete",
+        "do",
+        "else",
+        "enum",
+        "export",
+        "extends",
+        "false",
+        "finally",
+        "for",
+        "function",
+        "if",
+        "import",
+        "in",
+        "instanceof",
+        "new",
+        "null",
+        "return",
+        "super",
+        "switch",
+        "this",
+        "throw",
+        "true",
+        "try",
+        "typeof",
+        "var",
+        "void",
+        "while",
+        "with",
+        "as",
+        "async",
+        "await",
+        "from",
+        "get",
+        "interface",
+        "is",
+        "let",
+        "of",
+        "namespace",
+        "set",
+        "type",
+        "yield",
+        "public",
+        "private",
+        "protected",
+        "readonly",
+        "static",
+        "declare",
+        "abstract",
+    ]),
+    javascript: new Set([
+        "break",
+        "case",
+        "catch",
+        "class",
+        "const",
+        "continue",
+        "debugger",
+        "default",
+        "delete",
+        "do",
+        "else",
+        "export",
+        "extends",
+        "false",
+        "finally",
+        "for",
+        "function",
+        "if",
+        "import",
+        "in",
+        "instanceof",
+        "new",
+        "null",
+        "return",
+        "super",
+        "switch",
+        "this",
+        "throw",
+        "true",
+        "try",
+        "typeof",
+        "var",
+        "void",
+        "while",
+        "with",
+        "as",
+        "async",
+        "await",
+        "from",
+        "get",
+        "let",
+        "of",
+        "set",
+        "yield",
+        "static",
+    ]),
+    rust: new Set([
+        "as",
+        "break",
+        "const",
+        "continue",
+        "crate",
+        "else",
+        "enum",
+        "extern",
+        "false",
+        "fn",
+        "for",
+        "if",
+        "impl",
+        "in",
+        "let",
+        "loop",
+        "match",
+        "mod",
+        "move",
+        "mut",
+        "pub",
+        "ref",
+        "return",
+        "self",
+        "Self",
+        "static",
+        "struct",
+        "super",
+        "trait",
+        "true",
+        "type",
+        "unsafe",
+        "use",
+        "where",
+        "while",
+        "async",
+        "await",
+        "dyn",
+    ]),
+    python: new Set([
+        "and",
+        "as",
+        "assert",
+        "async",
+        "await",
+        "break",
+        "class",
+        "continue",
+        "def",
+        "del",
+        "elif",
+        "else",
+        "except",
+        "False",
+        "finally",
+        "for",
+        "from",
+        "global",
+        "if",
+        "import",
+        "in",
+        "is",
+        "lambda",
+        "None",
+        "nonlocal",
+        "not",
+        "or",
+        "pass",
+        "raise",
+        "return",
+        "True",
+        "try",
+        "while",
+        "with",
+        "yield",
+    ]),
+    go: new Set([
+        "break",
+        "default",
+        "func",
+        "interface",
+        "select",
+        "case",
+        "defer",
+        "go",
+        "map",
+        "struct",
+        "chan",
+        "else",
+        "goto",
+        "package",
+        "switch",
+        "const",
+        "fallthrough",
+        "if",
+        "range",
+        "type",
+        "continue",
+        "for",
+        "import",
+        "return",
+        "var",
+        "true",
+        "false",
+        "nil",
+        "iota",
+    ]),
+    json: new Set(["true", "false", "null"]),
+    html: new Set(["DOCTYPE", "html", "head", "body", "div", "span", "script", "style"]),
+    css: new Set(["important", "root", "media", "keyframes", "charset", "import", "supports"]),
+    markdown: new Set([]),
+    sql: new Set([
+        "SELECT",
+        "FROM",
+        "WHERE",
+        "INSERT",
+        "UPDATE",
+        "DELETE",
+        "CREATE",
+        "TABLE",
+        "DROP",
+        "ALTER",
+        "ADD",
+        "JOIN",
+        "INNER",
+        "LEFT",
+        "RIGHT",
+        "ON",
+        "GROUP",
+        "BY",
+        "ORDER",
+        "select",
+        "from",
+        "where",
+        "insert",
+        "update",
+        "delete",
+        "create",
+        "table",
+        "drop",
+        "alter",
+        "add",
+        "join",
+        "inner",
+        "left",
+        "right",
+        "on",
+        "group",
+        "by",
+        "order",
+    ]),
+    shell: new Set([
+        "if",
+        "then",
+        "else",
+        "elif",
+        "fi",
+        "case",
+        "esac",
+        "for",
+        "while",
+        "until",
+        "do",
+        "done",
+        "in",
+        "function",
+        "select",
+        "time",
+        "return",
+        "exit",
+        "export",
+        "local",
+        "readonly",
+        "unset",
+        "shift",
+        "source",
+        "alias",
+        "echo",
+        "printf",
+    ]),
+    cpp: new Set([
+        "auto",
+        "break",
+        "case",
+        "catch",
+        "class",
+        "const",
+        "constexpr",
+        "continue",
+        "default",
+        "delete",
+        "do",
+        "else",
+        "enum",
+        "explicit",
+        "export",
+        "extern",
+        "false",
+        "for",
+        "friend",
+        "goto",
+        "if",
+        "inline",
+        "mutable",
+        "namespace",
+        "new",
+        "noexcept",
+        "nullptr",
+        "operator",
+        "private",
+        "protected",
+        "public",
+        "register",
+        "reinterpret_cast",
+        "return",
+        "sizeof",
+        "static",
+        "static_assert",
+        "static_cast",
+        "struct",
+        "switch",
+        "template",
+        "this",
+        "throw",
+        "true",
+        "try",
+        "typedef",
+        "typeid",
+        "typename",
+        "union",
+        "using",
+        "virtual",
+        "void",
+        "volatile",
+        "while",
+    ]),
+    csharp: new Set([
+        "abstract",
+        "as",
+        "async",
+        "await",
+        "base",
+        "bool",
+        "break",
+        "byte",
+        "case",
+        "catch",
+        "char",
+        "checked",
+        "class",
+        "const",
+        "continue",
+        "decimal",
+        "default",
+        "delegate",
+        "do",
+        "double",
+        "else",
+        "enum",
+        "event",
+        "explicit",
+        "extern",
+        "false",
+        "finally",
+        "fixed",
+        "float",
+        "for",
+        "foreach",
+        "goto",
+        "if",
+        "implicit",
+        "in",
+        "int",
+        "interface",
+        "internal",
+        "is",
+        "lock",
+        "long",
+        "namespace",
+        "new",
+        "null",
+        "object",
+        "operator",
+        "out",
+        "override",
+        "params",
+        "private",
+        "protected",
+        "public",
+        "readonly",
+        "ref",
+        "return",
+        "sbyte",
+        "sealed",
+        "short",
+        "sizeof",
+        "stackalloc",
+        "static",
+        "string",
+        "struct",
+        "switch",
+        "this",
+        "throw",
+        "true",
+        "try",
+        "typeof",
+        "uint",
+        "ulong",
+        "unchecked",
+        "unsafe",
+        "ushort",
+        "using",
+        "virtual",
+        "void",
+        "volatile",
+        "while",
+        "yield",
+        "record",
+        "var",
+    ]),
+    java: new Set([
+        "abstract",
+        "assert",
+        "boolean",
+        "break",
+        "byte",
+        "case",
+        "catch",
+        "char",
+        "class",
+        "const",
+        "continue",
+        "default",
+        "do",
+        "double",
+        "else",
+        "enum",
+        "extends",
+        "final",
+        "finally",
+        "float",
+        "for",
+        "goto",
+        "if",
+        "implements",
+        "import",
+        "instanceof",
+        "int",
+        "interface",
+        "long",
+        "native",
+        "new",
+        "package",
+        "private",
+        "protected",
+        "public",
+        "return",
+        "short",
+        "static",
+        "strictfp",
+        "super",
+        "switch",
+        "synchronized",
+        "this",
+        "throw",
+        "throws",
+        "transient",
+        "try",
+        "void",
+        "volatile",
+        "while",
+        "true",
+        "false",
+        "null",
+        "record",
+        "sealed",
+    ]),
+    php: new Set([
+        "abstract",
+        "and",
+        "array",
+        "as",
+        "break",
+        "callable",
+        "case",
+        "catch",
+        "class",
+        "clone",
+        "const",
+        "continue",
+        "declare",
+        "default",
+        "die",
+        "do",
+        "echo",
+        "else",
+        "elseif",
+        "empty",
+        "enddeclare",
+        "endfor",
+        "endforeach",
+        "endif",
+        "endswitch",
+        "endwhile",
+        "eval",
+        "exit",
+        "extends",
+        "final",
+        "finally",
+        "fn",
+        "for",
+        "foreach",
+        "function",
+        "global",
+        "goto",
+        "if",
+        "implements",
+        "include",
+        "include_once",
+        "instanceof",
+        "insteadof",
+        "interface",
+        "isset",
+        "list",
+        "match",
+        "namespace",
+        "new",
+        "or",
+        "print",
+        "private",
+        "protected",
+        "public",
+        "readonly",
+        "require",
+        "require_once",
+        "return",
+        "static",
+        "switch",
+        "throw",
+        "trait",
+        "try",
+        "unset",
+        "use",
+        "var",
+        "while",
+        "xor",
+        "yield",
+        "true",
+        "false",
+        "null",
+    ]),
+    ruby: new Set([
+        "alias",
+        "and",
+        "BEGIN",
+        "begin",
+        "break",
+        "case",
+        "class",
+        "def",
+        "defined?",
+        "do",
+        "else",
+        "elsif",
+        "END",
+        "end",
+        "ensure",
+        "false",
+        "for",
+        "if",
+        "in",
+        "module",
+        "next",
+        "nil",
+        "not",
+        "or",
+        "redo",
+        "rescue",
+        "retry",
+        "return",
+        "self",
+        "super",
+        "then",
+        "true",
+        "undef",
+        "unless",
+        "until",
+        "when",
+        "while",
+        "yield",
+        "require",
+        "include",
+        "attr_accessor",
+        "attr_reader",
+        "attr_writer",
+    ]),
+    yaml: new Set(["true", "false", "yes", "no", "on", "off", "null", "~"]),
+    toml: new Set(["true", "false"]),
+    dockerfile: new Set([
+        "FROM",
+        "RUN",
+        "CMD",
+        "LABEL",
+        "MAINTAINER",
+        "EXPOSE",
+        "ENV",
+        "ADD",
+        "COPY",
+        "ENTRYPOINT",
+        "VOLUME",
+        "USER",
+        "WORKDIR",
+        "ARG",
+        "ONBUILD",
+        "STOPSIGNAL",
+        "HEALTHCHECK",
+        "SHELL",
+    ]),
+    graphql: new Set([
+        "query",
+        "mutation",
+        "subscription",
+        "fragment",
+        "on",
+        "type",
+        "interface",
+        "union",
+        "enum",
+        "scalar",
+        "input",
+        "implements",
+        "directive",
+        "schema",
+    ]),
+};
+const TYPES = new Set([
+    "string",
+    "number",
+    "boolean",
+    "any",
+    "void",
+    "never",
+    "unknown",
+    "object",
+    "bigint",
+    "symbol",
+    "i8",
+    "i16",
+    "i32",
+    "i64",
+    "i128",
+    "u8",
+    "u16",
+    "u32",
+    "u64",
+    "u128",
+    "f32",
+    "f64",
+    "usize",
+    "isize",
+    "str",
+    "bool",
+    "char",
+    "Vec",
+    "String",
+    "Option",
+    "Result",
+    "Box",
+    "Rc",
+    "Arc",
+    "HashMap",
+    "HashSet",
+    "int",
+    "int8",
+    "int16",
+    "int32",
+    "int64",
+    "uint",
+    "uint8",
+    "uint16",
+    "uint32",
+    "uint64",
+    "uintptr",
+    "float32",
+    "float64",
+    "complex64",
+    "complex128",
+    "byte",
+    "rune",
+    "error",
+    "Integer",
+    "Float",
+    "Double",
+    "Boolean",
+    "Character",
+    "List",
+    "Map",
+    "Set",
+]);
+export function tokenizeLineWithState(line, language = "typescript", state = {}) {
+    const langKey = language.toLowerCase();
+    const keywords = KEYWORDS_BY_LANG[langKey] || KEYWORDS_BY_LANG.typescript;
+    const tokens = [];
+    let i = 0;
+    const len = line.length;
+    let inBlockComment = !!state.inBlockComment;
+    let inXmlComment = !!state.inXmlComment;
+    let inMultiLineString = state.inMultiLineString || null;
+    // Handle carried multi-line block comment
+    if (inBlockComment) {
+        const endIdx = line.indexOf("*/");
+        if (endIdx === -1) {
+            tokens.push({ text: line, kind: "comment" });
+            return { tokens, nextState: { inBlockComment: true } };
+        }
+        tokens.push({ text: line.slice(0, endIdx + 2), kind: "comment" });
+        inBlockComment = false;
+        i = endIdx + 2;
+    }
+    // Handle carried multi-line XML comment
+    if (inXmlComment) {
+        const endIdx = line.indexOf("-->");
+        if (endIdx === -1) {
+            tokens.push({ text: line, kind: "comment" });
+            return { tokens, nextState: { inXmlComment: true } };
+        }
+        tokens.push({ text: line.slice(0, endIdx + 3), kind: "comment" });
+        inXmlComment = false;
+        i = endIdx + 3;
+    }
+    // Handle carried multi-line string (template literal, Python triple quotes)
+    if (inMultiLineString) {
+        const delim = inMultiLineString;
+        let endIdx = -1;
+        let searchPos = 0;
+        while (searchPos < len) {
+            const idx = line.indexOf(delim, searchPos);
+            if (idx === -1)
+                break;
+            // Check for escape character before delimiter if 1-char delimiter
+            if (delim.length === 1 && idx > 0 && line[idx - 1] === "\\") {
+                searchPos = idx + 1;
+                continue;
+            }
+            endIdx = idx;
+            break;
+        }
+        if (endIdx === -1) {
+            tokens.push({ text: line, kind: "string" });
+            return { tokens, nextState: { inMultiLineString: delim } };
+        }
+        tokens.push({ text: line.slice(0, endIdx + delim.length), kind: "string" });
+        inMultiLineString = null;
+        i = endIdx + delim.length;
+    }
+    while (i < len) {
+        const char = line[i];
+        // Single-line comments
+        if (char === "/" && line[i + 1] === "/") {
+            tokens.push({ text: line.slice(i), kind: "comment" });
+            break;
+        }
+        // Multi-line block comment start
+        if (char === "/" && line[i + 1] === "*") {
+            const endIdx = line.indexOf("*/", i + 2);
+            if (endIdx === -1) {
+                tokens.push({ text: line.slice(i), kind: "comment" });
+                return { tokens, nextState: { inBlockComment: true } };
+            }
+            tokens.push({ text: line.slice(i, endIdx + 2), kind: "comment" });
+            i = endIdx + 2;
+            continue;
+        }
+        // Single-line hash comments for Python/Shell/Ruby/Yaml/Toml/Dockerfile
+        if (char === "#" &&
+            (langKey === "python" ||
+                langKey === "shell" ||
+                langKey === "ruby" ||
+                langKey === "yaml" ||
+                langKey === "toml" ||
+                langKey === "dockerfile")) {
+            tokens.push({ text: line.slice(i), kind: "comment" });
+            break;
+        }
+        // SQL comments
+        if (char === "-" && line[i + 1] === "-" && langKey === "sql") {
+            tokens.push({ text: line.slice(i), kind: "comment" });
+            break;
+        }
+        // HTML / XML comment start
+        if (char === "<" && line.slice(i, i + 4) === "<!--") {
+            const endIdx = line.indexOf("-->", i + 4);
+            if (endIdx === -1) {
+                tokens.push({ text: line.slice(i), kind: "comment" });
+                return { tokens, nextState: { inXmlComment: true } };
+            }
+            tokens.push({ text: line.slice(i, endIdx + 3), kind: "comment" });
+            i = endIdx + 3;
+            continue;
+        }
+        // Whitespace fast-path
+        if (/\s/.test(char)) {
+            const start = i;
+            while (i < len && /\s/.test(line[i])) {
+                i++;
+            }
+            tokens.push({ text: line.slice(start, i), kind: "plain" });
+            continue;
+        }
+        // HTML / XML Tags
+        if ((langKey === "html" || langKey === "xml") && char === "<") {
+            let endTag = i + 1;
+            while (endTag < len && !/[ >/]/.test(line[endTag])) {
+                endTag++;
+            }
+            const tagText = line.slice(i, endTag);
+            tokens.push({ text: tagText, kind: "tag" });
+            i = endTag;
+            continue;
+        }
+        // Python / TOML Triple-quoted strings (""" or ''')
+        if ((langKey === "python" || langKey === "toml") &&
+            (line.slice(i, i + 3) === '"""' || line.slice(i, i + 3) === "'''")) {
+            const delim = line.slice(i, i + 3);
+            const endIdx = line.indexOf(delim, i + 3);
+            if (endIdx === -1) {
+                tokens.push({ text: line.slice(i), kind: "string" });
+                return { tokens, nextState: { inMultiLineString: delim } };
+            }
+            tokens.push({ text: line.slice(i, endIdx + 3), kind: "string" });
+            i = endIdx + 3;
+            continue;
+        }
+        // Template literals (`)
+        if (char === "`") {
+            const start = i;
+            i++;
+            let closed = false;
+            while (i < len) {
+                if (line[i] === "\\") {
+                    i += 2;
+                    continue;
+                }
+                if (line[i] === "`") {
+                    i++;
+                    closed = true;
+                    break;
+                }
+                i++;
+            }
+            if (!closed) {
+                tokens.push({ text: line.slice(start), kind: "string" });
+                return { tokens, nextState: { inMultiLineString: "`" } };
+            }
+            tokens.push({ text: line.slice(start, i), kind: "string" });
+            continue;
+        }
+        // Standard Strings (double quotes, single quotes)
+        if (char === '"' || char === "'") {
+            const quote = char;
+            const start = i;
+            i++;
+            while (i < len && line[i] !== quote) {
+                if (line[i] === "\\")
+                    i++;
+                i++;
+            }
+            if (i < len)
+                i++;
+            tokens.push({ text: line.slice(start, i), kind: "string" });
+            continue;
+        }
+        // Numbers
+        if (/[0-9]/.test(char)) {
+            const start = i;
+            while (i < len && /[0-9a-fA-FxX._]/.test(line[i])) {
+                i++;
+            }
+            tokens.push({ text: line.slice(start, i), kind: "number" });
+            continue;
+        }
+        // Identifiers, keywords, types
+        if (/[a-zA-Z_$]/.test(char)) {
+            const start = i;
+            while (i < len && /[a-zA-Z0-9_$-]/.test(line[i])) {
+                i++;
+            }
+            const word = line.slice(start, i);
+            if (keywords.has(word)) {
+                tokens.push({ text: word, kind: "keyword" });
+            }
+            else if (TYPES.has(word)) {
+                tokens.push({ text: word, kind: "type" });
+            }
+            else if (i < len && line[i] === "(") {
+                tokens.push({ text: word, kind: "function" });
+            }
+            else if ((langKey === "html" || langKey === "css" || langKey === "yaml") &&
+                i < len &&
+                line[i] === ":") {
+                tokens.push({ text: word, kind: "attribute" });
+            }
+            else {
+                tokens.push({ text: word, kind: "plain" });
+            }
+            continue;
+        }
+        // Punctuation
+        if (/[{}[\],.;:]/.test(char) || char === "(" || char === ")") {
+            tokens.push({ text: char, kind: "punctuation" });
+            i++;
+            continue;
+        }
+        // Operators
+        if (/[+\-*%=<>!&|^~?]/.test(char) || char === "/") {
+            const start = i;
+            while (i < len && (/[+\-*%=<>!&|^~?]/.test(line[i]) || line[i] === "/")) {
+                i++;
+            }
+            tokens.push({ text: line.slice(start, i), kind: "operator" });
+            continue;
+        }
+        // Catch-all: guaranteed consumption for decorators (@), preprocessors (#),
+        // escapes (\), markdown, emojis, etc. Consume a full code point so
+        // surrogate pairs (emoji) stay in one token.
+        const code = line.codePointAt(i) ?? 0;
+        const charLen = code > 0xffff ? 2 : 1;
+        tokens.push({ text: line.slice(i, i + charLen), kind: "plain" });
+        i += charLen;
+    }
+    return { tokens, nextState: {} };
+}
+export function tokenizeLine(line, language = "typescript", state = {}) {
+    const result = tokenizeLineWithState(line, language, state);
+    const arr = result.tokens;
+    Object.defineProperty(arr, "nextState", {
+        value: result.nextState,
+        enumerable: false,
+        writable: true,
+        configurable: true,
+    });
+    return arr;
+}
+//# sourceMappingURL=microTokenizer.js.map