@nocturnium/svelte-ide 1.0.0-rc.1

package/dist/components/editor/tokenizer/index.js

@@ -0,0 +1,215 @@
+/**
+ * Tokenizer module for syntax highlighting
+ *
+ * This module provides zero-dependency syntax highlighting through custom tokenizers.
+ * Each language has its own tokenizer that produces tokens for styling.
+ */
+export * from './types';
+export * from './base';
+// Language tokenizers
+export { JavaScriptTokenizer, createJavaScriptTokenizer, createTypeScriptTokenizer, createJSXTokenizer, createTSXTokenizer } from './languages/javascript';
+export { HTMLTokenizer, createHTMLTokenizer, createXMLTokenizer } from './languages/html';
+export { CSSTokenizer, createCSSTokenizer } from './languages/css';
+export { JSONTokenizer, createJSONTokenizer } from './languages/json';
+export { PythonTokenizer, createPythonTokenizer } from './languages/python';
+export { GoTokenizer, createGoTokenizer } from './languages/go';
+export { MarkdownTokenizer, createMarkdownTokenizer } from './languages/markdown';
+export { SvelteTokenizer, createSvelteTokenizer } from './languages/svelte';
+import { PlaintextTokenizer } from './base';
+import { createJavaScriptTokenizer, createTypeScriptTokenizer, createJSXTokenizer, createTSXTokenizer } from './languages/javascript';
+import { createHTMLTokenizer, createXMLTokenizer } from './languages/html';
+import { createCSSTokenizer } from './languages/css';
+import { createJSONTokenizer } from './languages/json';
+import { createPythonTokenizer } from './languages/python';
+import { createGoTokenizer } from './languages/go';
+import { createMarkdownTokenizer } from './languages/markdown';
+import { createSvelteTokenizer } from './languages/svelte';
+/**
+ * Language aliases mapping file extensions and common names to canonical language names
+ */
+const languageAliases = {
+    // JavaScript
+    js: 'javascript',
+    mjs: 'javascript',
+    cjs: 'javascript',
+    // TypeScript
+    ts: 'typescript',
+    mts: 'typescript',
+    cts: 'typescript',
+    // JSX/TSX
+    jsx: 'jsx',
+    tsx: 'tsx',
+    // HTML
+    htm: 'html',
+    xhtml: 'html',
+    vue: 'html',
+    // Svelte
+    svelte: 'svelte',
+    // XML
+    svg: 'xml',
+    xsl: 'xml',
+    xslt: 'xml',
+    rss: 'xml',
+    atom: 'xml',
+    plist: 'xml',
+    // CSS
+    scss: 'css',
+    sass: 'css',
+    less: 'css',
+    // JSON
+    jsonc: 'json',
+    json5: 'json',
+    // Python
+    py: 'python',
+    pyw: 'python',
+    pyi: 'python',
+    // Go
+    golang: 'go',
+    // Markdown
+    md: 'markdown',
+    mkd: 'markdown',
+    mkdn: 'markdown',
+    mdown: 'markdown',
+    markdown: 'markdown',
+    // Plain text
+    txt: 'plaintext',
+    text: 'plaintext'
+};
+/**
+ * Tokenizer factory functions by language
+ */
+const tokenizerFactories = {
+    javascript: createJavaScriptTokenizer,
+    typescript: createTypeScriptTokenizer,
+    jsx: createJSXTokenizer,
+    tsx: createTSXTokenizer,
+    html: createHTMLTokenizer,
+    xml: createXMLTokenizer,
+    css: createCSSTokenizer,
+    json: createJSONTokenizer,
+    python: createPythonTokenizer,
+    go: createGoTokenizer,
+    markdown: createMarkdownTokenizer,
+    svelte: createSvelteTokenizer,
+    plaintext: () => new PlaintextTokenizer()
+};
+/**
+ * Cached tokenizer instances with LRU eviction
+ */
+const tokenizerCache = new Map();
+/** Maximum number of tokenizers to keep in cache */
+const MAX_TOKENIZER_CACHE_SIZE = 20;
+/**
+ * Get the canonical language name from an alias or extension
+ */
+export function resolveLanguage(language) {
+    const normalized = language.toLowerCase().replace(/^\./, '');
+    return languageAliases[normalized] ?? normalized;
+}
+/**
+ * Get a tokenizer for a language (with LRU cache eviction)
+ */
+export function getTokenizer(language) {
+    const resolved = resolveLanguage(language);
+    // Check cache - if found, move to end for LRU tracking
+    const cached = tokenizerCache.get(resolved);
+    if (cached) {
+        // Move to end (most recently used) by re-inserting
+        tokenizerCache.delete(resolved);
+        tokenizerCache.set(resolved, cached);
+        return cached;
+    }
+    // Evict least recently used if cache is full
+    if (tokenizerCache.size >= MAX_TOKENIZER_CACHE_SIZE) {
+        // Map iterates in insertion order, so first key is LRU
+        const lruKey = tokenizerCache.keys().next().value;
+        if (lruKey) {
+            tokenizerCache.delete(lruKey);
+        }
+    }
+    // Create new tokenizer
+    const factory = tokenizerFactories[resolved] ?? tokenizerFactories['plaintext'];
+    const tokenizer = factory();
+    // Cache it
+    tokenizerCache.set(resolved, tokenizer);
+    return tokenizer;
+}
+/**
+ * Clear the tokenizer cache (useful for testing or memory management)
+ */
+export function clearTokenizerCache() {
+    tokenizerCache.clear();
+}
+/**
+ * Check if a language is supported
+ */
+export function isLanguageSupported(language) {
+    const resolved = resolveLanguage(language);
+    return resolved in tokenizerFactories;
+}
+/**
+ * Get list of supported languages
+ */
+export function getSupportedLanguages() {
+    return Object.keys(tokenizerFactories);
+}
+/**
+ * Tokenize content for a given language
+ */
+export function tokenize(content, language) {
+    const tokenizer = getTokenizer(language);
+    const lines = content.split('\n');
+    const result = [];
+    let state = tokenizer.getInitialState();
+    for (let i = 0; i < lines.length; i++) {
+        const tokenized = tokenizer.tokenizeLine(lines[i], i + 1, state);
+        result.push(tokenized);
+        state = tokenized.state ?? state;
+    }
+    return result;
+}
+/**
+ * Valid CSS class name pattern (alphanumeric, hyphens, underscores only)
+ */
+const SAFE_CLASS_PATTERN = /^[a-zA-Z0-9_-]+$/;
+/**
+ * Get CSS class for a token type
+ * Sanitizes input to prevent XSS through class name injection
+ */
+export function getTokenClass(type) {
+    // Convert dots to hyphens for CSS class naming
+    const className = type.replace(/\./g, '-');
+    // Validate the resulting class name contains only safe characters
+    // This prevents any potential XSS if an invalid type somehow gets through
+    if (!SAFE_CLASS_PATTERN.test(className)) {
+        return 'token-text'; // Fallback to safe default
+    }
+    return `token-${className}`;
+}
+/**
+ * Convert tokens to HTML spans for rendering
+ */
+export function tokensToHTML(tokens) {
+    return tokens
+        .map((token) => {
+        const className = getTokenClass(token.type);
+        const escapedText = escapeHTML(token.text);
+        if (token.type === 'text') {
+            return escapedText;
+        }
+        return `<span class="${className}">${escapedText}</span>`;
+    })
+        .join('');
+}
+/**
+ * Escape HTML special characters
+ * Unified HTML escaping function for all editor components
+ */
+export function escapeHTML(text) {
+    return text
+        .replace(/&/g, '&amp;')
+        .replace(/</g, '&lt;')
+        .replace(/>/g, '&gt;')
+        .replace(/"/g, '&quot;')
+        .replace(/'/g, '&#039;');
+}

package/dist/components/editor/tokenizer/languages/css.d.ts

@@ -0,0 +1,17 @@
+/**
+ * CSS tokenizer
+ */
+import type { TokenizedLine, TokenizerState } from '../types';
+interface CSSTokenizerState extends TokenizerState {
+    inRule?: boolean;
+    inValue?: boolean;
+}
+export declare class CSSTokenizer {
+    language: string;
+    getInitialState(): CSSTokenizerState;
+    tokenizeLine(line: string, lineNumber: number, prevState?: CSSTokenizerState): TokenizedLine;
+    private getNextToken;
+    private tokenizeString;
+}
+export declare function createCSSTokenizer(): CSSTokenizer;
+export {};

package/dist/components/editor/tokenizer/languages/css.js

@@ -0,0 +1,194 @@
+/**
+ * CSS tokenizer
+ */
+import { createToken } from '../base';
+// CSS properties (common ones)
+const properties = new Set([
+    'align-content', 'align-items', 'align-self', 'animation', 'animation-delay',
+    'animation-direction', 'animation-duration', 'animation-fill-mode', 'animation-iteration-count',
+    'animation-name', 'animation-play-state', 'animation-timing-function', 'background',
+    'background-attachment', 'background-blend-mode', 'background-clip', 'background-color',
+    'background-image', 'background-origin', 'background-position', 'background-repeat',
+    'background-size', 'border', 'border-bottom', 'border-color', 'border-left', 'border-radius',
+    'border-right', 'border-style', 'border-top', 'border-width', 'bottom', 'box-shadow',
+    'box-sizing', 'color', 'content', 'cursor', 'display', 'filter', 'flex', 'flex-basis',
+    'flex-direction', 'flex-flow', 'flex-grow', 'flex-shrink', 'flex-wrap', 'float', 'font',
+    'font-family', 'font-size', 'font-style', 'font-weight', 'gap', 'grid', 'grid-area',
+    'grid-column', 'grid-gap', 'grid-row', 'grid-template', 'grid-template-areas',
+    'grid-template-columns', 'grid-template-rows', 'height', 'justify-content', 'justify-items',
+    'justify-self', 'left', 'letter-spacing', 'line-height', 'list-style', 'margin',
+    'margin-bottom', 'margin-left', 'margin-right', 'margin-top', 'max-height', 'max-width',
+    'min-height', 'min-width', 'object-fit', 'opacity', 'order', 'outline', 'overflow',
+    'overflow-x', 'overflow-y', 'padding', 'padding-bottom', 'padding-left', 'padding-right',
+    'padding-top', 'place-content', 'place-items', 'place-self', 'pointer-events', 'position',
+    'resize', 'right', 'row-gap', 'text-align', 'text-decoration', 'text-indent', 'text-overflow',
+    'text-shadow', 'text-transform', 'top', 'transform', 'transform-origin', 'transition',
+    'transition-delay', 'transition-duration', 'transition-property', 'transition-timing-function',
+    'user-select', 'vertical-align', 'visibility', 'white-space', 'width', 'word-break',
+    'word-spacing', 'word-wrap', 'z-index'
+]);
+// CSS at-rules
+const atRules = new Set([
+    '@media', '@keyframes', '@import', '@font-face', '@supports', '@page', '@charset',
+    '@namespace', '@viewport', '@counter-style', '@font-feature-values', '@property', '@layer'
+]);
+export class CSSTokenizer {
+    language = 'css';
+    getInitialState() {
+        return {};
+    }
+    tokenizeLine(line, lineNumber, prevState) {
+        const tokens = [];
+        let pos = 0;
+        const state = { ...prevState };
+        // Handle multi-line comment continuation
+        if (state.inBlockComment) {
+            const endIdx = line.indexOf('*/');
+            if (endIdx !== -1) {
+                tokens.push(createToken('comment.block', line.slice(0, endIdx + 2), 0));
+                pos = endIdx + 2;
+                state.inBlockComment = false;
+            }
+            else {
+                tokens.push(createToken('comment.block', line, 0));
+                return { lineNumber, tokens, text: line, state };
+            }
+        }
+        while (pos < line.length) {
+            const remaining = line.slice(pos);
+            const token = this.getNextToken(remaining, pos, state);
+            if (token) {
+                tokens.push(token);
+                pos = token.end;
+            }
+            else {
+                tokens.push(createToken('text', remaining[0], pos));
+                pos += 1;
+            }
+        }
+        if (tokens.length === 0) {
+            tokens.push(createToken('text', '', 0));
+        }
+        return { lineNumber, tokens, text: line, state };
+    }
+    getNextToken(text, pos, state) {
+        // Whitespace
+        const wsMatch = text.match(/^[ \t]+/);
+        if (wsMatch) {
+            return createToken('text', wsMatch[0], pos);
+        }
+        // Comments
+        if (text.startsWith('/*')) {
+            const endIdx = text.indexOf('*/', 2);
+            if (endIdx !== -1) {
+                return createToken('comment.block', text.slice(0, endIdx + 2), pos);
+            }
+            else {
+                state.inBlockComment = true;
+                return createToken('comment.block', text, pos);
+            }
+        }
+        // At-rules
+        const atRuleMatch = text.match(/^@[a-zA-Z-]+/);
+        if (atRuleMatch) {
+            return createToken('keyword', atRuleMatch[0], pos);
+        }
+        // Selectors - class
+        const classMatch = text.match(/^\.[a-zA-Z_-][a-zA-Z0-9_-]*/);
+        if (classMatch) {
+            return createToken('type.class', classMatch[0], pos);
+        }
+        // Selectors - id
+        const idMatch = text.match(/^#[a-zA-Z_-][a-zA-Z0-9_-]*/);
+        if (idMatch) {
+            return createToken('constant', idMatch[0], pos);
+        }
+        // Pseudo-classes and pseudo-elements
+        const pseudoMatch = text.match(/^:{1,2}[a-zA-Z-]+(?:\([^)]*\))?/);
+        if (pseudoMatch) {
+            return createToken('variable', pseudoMatch[0], pos);
+        }
+        // Property names
+        const propMatch = text.match(/^[a-zA-Z-]+(?=\s*:)/);
+        if (propMatch) {
+            const prop = propMatch[0];
+            if (properties.has(prop) || prop.startsWith('--')) {
+                return createToken('property', prop, pos);
+            }
+            return createToken('property', prop, pos);
+        }
+        // CSS variables
+        const varMatch = text.match(/^var\s*\(\s*--[a-zA-Z0-9_-]+\s*(?:,[^)]*)?\)/);
+        if (varMatch) {
+            return createToken('function.call', varMatch[0], pos);
+        }
+        // Functions
+        const funcMatch = text.match(/^[a-zA-Z-]+(?=\s*\()/);
+        if (funcMatch) {
+            return createToken('function', funcMatch[0], pos);
+        }
+        // Colors - hex
+        const hexMatch = text.match(/^#(?:[0-9a-fA-F]{8}|[0-9a-fA-F]{6}|[0-9a-fA-F]{3,4})\b/);
+        if (hexMatch) {
+            return createToken('constant', hexMatch[0], pos);
+        }
+        // Numbers with units
+        const numMatch = text.match(/^-?(?:\d+\.?\d*|\.\d+)(?:px|em|rem|%|vh|vw|vmin|vmax|ch|ex|cm|mm|in|pt|pc|deg|rad|grad|turn|s|ms|Hz|kHz|dpi|dpcm|dppx)?/);
+        if (numMatch) {
+            return createToken('number', numMatch[0], pos);
+        }
+        // Strings
+        if (text.startsWith('"') || text.startsWith("'")) {
+            return this.tokenizeString(text, pos, text[0]);
+        }
+        // Keywords (color names, etc.)
+        const keywordMatch = text.match(/^[a-zA-Z-]+/);
+        if (keywordMatch) {
+            const word = keywordMatch[0];
+            // Common CSS keywords
+            if (['inherit', 'initial', 'unset', 'revert', 'none', 'auto', 'transparent', 'currentColor', 'important'].includes(word)) {
+                return createToken('keyword', word, pos);
+            }
+            // Color names
+            if (['red', 'blue', 'green', 'white', 'black', 'yellow', 'orange', 'purple', 'pink', 'gray', 'grey'].includes(word)) {
+                return createToken('constant', word, pos);
+            }
+            return createToken('text', word, pos);
+        }
+        // Punctuation
+        if (text[0] === '{' || text[0] === '}') {
+            if (text[0] === '{')
+                state.inRule = true;
+            if (text[0] === '}')
+                state.inRule = false;
+            return createToken('punctuation.brace', text[0], pos);
+        }
+        if (text[0] === '(' || text[0] === ')') {
+            return createToken('punctuation.paren', text[0], pos);
+        }
+        if (text[0] === '[' || text[0] === ']') {
+            return createToken('punctuation.bracket', text[0], pos);
+        }
+        if (text[0] === ':' || text[0] === ';' || text[0] === ',') {
+            return createToken('punctuation.separator', text[0], pos);
+        }
+        return createToken('text', text[0], pos);
+    }
+    tokenizeString(text, pos, delimiter) {
+        let i = 1;
+        while (i < text.length) {
+            if (text[i] === '\\' && i + 1 < text.length) {
+                i += 2;
+                continue;
+            }
+            if (text[i] === delimiter) {
+                return createToken('string', text.slice(0, i + 1), pos);
+            }
+            i++;
+        }
+        return createToken('string', text, pos);
+    }
+}
+export function createCSSTokenizer() {
+    return new CSSTokenizer();
+}

package/dist/components/editor/tokenizer/languages/go.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Go tokenizer
+ */
+import type { TokenizedLine, TokenizerState } from '../types';
+interface GoTokenizerState extends TokenizerState {
+    inRawString?: boolean;
+}
+export declare class GoTokenizer {
+    language: string;
+    getInitialState(): GoTokenizerState;
+    tokenizeLine(line: string, lineNumber: number, prevState?: GoTokenizerState): TokenizedLine;
+    private getNextToken;
+    private classifyIdentifier;
+    private tokenizeString;
+}
+export declare function createGoTokenizer(): GoTokenizer;
+export {};

package/dist/components/editor/tokenizer/languages/go.js

@@ -0,0 +1,220 @@
+/**
+ * Go tokenizer
+ */
+import { createToken } from '../base';
+// Go keywords
+const keywords = new Set([
+    'break', 'case', 'chan', 'const', 'continue', 'default', 'defer', 'else',
+    'fallthrough', 'for', 'func', 'go', 'goto', 'if', 'import', 'interface',
+    'map', 'package', 'range', 'return', 'select', 'struct', 'switch', 'type', 'var'
+]);
+const controlKeywords = new Set([
+    'break', 'case', 'continue', 'default', 'defer', 'else', 'fallthrough',
+    'for', 'go', 'goto', 'if', 'range', 'return', 'select', 'switch'
+]);
+const builtins = new Set([
+    'true', 'false', 'nil', 'iota'
+]);
+const builtinFunctions = new Set([
+    'append', 'cap', 'close', 'complex', 'copy', 'delete', 'imag', 'len',
+    'make', 'new', 'panic', 'print', 'println', 'real', 'recover'
+]);
+const builtinTypes = new Set([
+    'bool', 'byte', 'complex64', 'complex128', 'error', 'float32', 'float64',
+    'int', 'int8', 'int16', 'int32', 'int64', 'rune', 'string',
+    'uint', 'uint8', 'uint16', 'uint32', 'uint64', 'uintptr'
+]);
+export class GoTokenizer {
+    language = 'go';
+    getInitialState() {
+        return {};
+    }
+    tokenizeLine(line, lineNumber, prevState) {
+        const tokens = [];
+        let pos = 0;
+        const state = { ...prevState };
+        // Handle block comment continuation
+        if (state.inBlockComment) {
+            const endIdx = line.indexOf('*/');
+            if (endIdx !== -1) {
+                tokens.push(createToken('comment.block', line.slice(0, endIdx + 2), 0));
+                pos = endIdx + 2;
+                state.inBlockComment = false;
+            }
+            else {
+                tokens.push(createToken('comment.block', line, 0));
+                return { lineNumber, tokens, text: line, state };
+            }
+        }
+        // Handle raw string continuation
+        if (state.inRawString) {
+            const endIdx = line.indexOf('`');
+            if (endIdx !== -1) {
+                tokens.push(createToken('string', line.slice(0, endIdx + 1), 0));
+                pos = endIdx + 1;
+                state.inRawString = false;
+            }
+            else {
+                tokens.push(createToken('string', line, 0));
+                return { lineNumber, tokens, text: line, state };
+            }
+        }
+        while (pos < line.length) {
+            const remaining = line.slice(pos);
+            const token = this.getNextToken(remaining, pos, state);
+            if (token) {
+                tokens.push(token);
+                pos = token.end;
+            }
+            else {
+                tokens.push(createToken('text', remaining[0], pos));
+                pos += 1;
+            }
+        }
+        if (tokens.length === 0) {
+            tokens.push(createToken('text', '', 0));
+        }
+        return { lineNumber, tokens, text: line, state };
+    }
+    getNextToken(text, pos, state) {
+        // Whitespace
+        const wsMatch = text.match(/^[ \t]+/);
+        if (wsMatch) {
+            return createToken('text', wsMatch[0], pos);
+        }
+        // Line comments
+        if (text.startsWith('//')) {
+            return createToken('comment.line', text, pos);
+        }
+        // Block comments
+        if (text.startsWith('/*')) {
+            const endIdx = text.indexOf('*/', 2);
+            if (endIdx !== -1) {
+                return createToken('comment.block', text.slice(0, endIdx + 2), pos);
+            }
+            else {
+                state.inBlockComment = true;
+                return createToken('comment.block', text, pos);
+            }
+        }
+        // Raw strings
+        if (text.startsWith('`')) {
+            const endIdx = text.indexOf('`', 1);
+            if (endIdx !== -1) {
+                return createToken('string', text.slice(0, endIdx + 1), pos);
+            }
+            else {
+                state.inRawString = true;
+                return createToken('string', text, pos);
+            }
+        }
+        // Interpreted strings
+        if (text.startsWith('"')) {
+            return this.tokenizeString(text, pos);
+        }
+        // Rune literals
+        if (text.startsWith("'")) {
+            const runeMatch = text.match(/^'(?:[^'\\]|\\(?:[abfnrtv\\'"]|[0-7]{3}|x[0-9a-fA-F]{2}|u[0-9a-fA-F]{4}|U[0-9a-fA-F]{8}))'/);
+            if (runeMatch) {
+                return createToken('string', runeMatch[0], pos);
+            }
+        }
+        // Numbers
+        const numMatch = text.match(/^(?:0[xX][0-9a-fA-F_]+|0[bB][01_]+|0[oO][0-7_]+|(?:\d[\d_]*\.?[\d_]*|\.\d[\d_]*)(?:[eE][+-]?\d[\d_]*)?i?)/);
+        if (numMatch) {
+            return createToken('number', numMatch[0], pos);
+        }
+        // Identifiers and keywords
+        const identMatch = text.match(/^[a-zA-Z_][a-zA-Z0-9_]*/);
+        if (identMatch) {
+            const word = identMatch[0];
+            return createToken(this.classifyIdentifier(word, text, identMatch[0].length), word, pos);
+        }
+        // Operators
+        const opMatch = text.match(/^(?:<-|:=|&&|\|\||<<|>>|&\^|[+\-*/%&|^!<>=]=?)/);
+        if (opMatch) {
+            return createToken('operator', opMatch[0], pos);
+        }
+        // Punctuation
+        const punctMatch = text.match(/^[{}[\](),.:;]/);
+        if (punctMatch) {
+            const char = punctMatch[0];
+            let type = 'punctuation';
+            if (char === '{' || char === '}')
+                type = 'punctuation.brace';
+            else if (char === '[' || char === ']')
+                type = 'punctuation.bracket';
+            else if (char === '(' || char === ')')
+                type = 'punctuation.paren';
+            else if (char === ',' || char === ';' || char === ':')
+                type = 'punctuation.separator';
+            else if (char === '.')
+                type = 'punctuation.accessor';
+            return createToken(type, char, pos);
+        }
+        return createToken('text', text[0], pos);
+    }
+    classifyIdentifier(word, context, wordLength) {
+        // Built-in constants
+        if (builtins.has(word)) {
+            if (word === 'true' || word === 'false')
+                return 'constant.boolean';
+            if (word === 'nil')
+                return 'constant.null';
+            return 'constant.builtin';
+        }
+        // Keywords
+        if (keywords.has(word)) {
+            if (controlKeywords.has(word))
+                return 'keyword.control';
+            if (word === 'func' || word === 'type' || word === 'struct' || word === 'interface') {
+                return 'keyword.definition';
+            }
+            if (word === 'import' || word === 'package')
+                return 'keyword.module';
+            return 'keyword';
+        }
+        // Built-in types
+        if (builtinTypes.has(word)) {
+            return 'type.builtin';
+        }
+        // Built-in functions
+        if (builtinFunctions.has(word)) {
+            const afterWord = context.slice(wordLength).trim();
+            if (afterWord.startsWith('(')) {
+                return 'function.call';
+            }
+            return 'function';
+        }
+        // Check if followed by ( - likely a function call
+        const afterWord = context.slice(wordLength).trim();
+        if (afterWord.startsWith('(')) {
+            return 'function.call';
+        }
+        // Check if it's a type (PascalCase)
+        if (/^[A-Z][a-zA-Z0-9]*$/.test(word)) {
+            return 'type.class';
+        }
+        return 'variable';
+    }
+    tokenizeString(text, pos) {
+        let i = 1;
+        while (i < text.length) {
+            if (text[i] === '\\' && i + 1 < text.length) {
+                i += 2;
+                continue;
+            }
+            if (text[i] === '"') {
+                return createToken('string', text.slice(0, i + 1), pos);
+            }
+            if (text[i] === '\n') {
+                break;
+            }
+            i++;
+        }
+        return createToken('string', text.slice(0, i), pos);
+    }
+}
+export function createGoTokenizer() {
+    return new GoTokenizer();
+}

package/dist/components/editor/tokenizer/languages/html.d.ts

@@ -0,0 +1,24 @@
+/**
+ * HTML/XML tokenizer
+ */
+import type { TokenizedLine, TokenizerState } from '../types';
+interface HTMLTokenizerState extends TokenizerState {
+    inTag?: boolean;
+    inScript?: boolean;
+    inStyle?: boolean;
+    tagName?: string;
+}
+export declare class HTMLTokenizer {
+    language: string;
+    private isXML;
+    constructor(options?: {
+        xml?: boolean;
+    });
+    getInitialState(): HTMLTokenizerState;
+    tokenizeLine(line: string, lineNumber: number, prevState?: HTMLTokenizerState): TokenizedLine;
+    private getNextToken;
+    private tokenizeTag;
+}
+export declare function createHTMLTokenizer(): HTMLTokenizer;
+export declare function createXMLTokenizer(): HTMLTokenizer;
+export {};