@nocturnium/svelte-ide 1.0.0-rc.1

package/dist/components/editor/tokenizer/languages/json.js

@@ -0,0 +1,91 @@
+/**
+ * JSON tokenizer
+ */
+import { createToken } from '../base';
+export class JSONTokenizer {
+    language = 'json';
+    getInitialState() {
+        return {};
+    }
+    tokenizeLine(line, lineNumber, prevState) {
+        const tokens = [];
+        let pos = 0;
+        const state = { ...prevState };
+        while (pos < line.length) {
+            const remaining = line.slice(pos);
+            const token = this.getNextToken(remaining, pos);
+            if (token) {
+                tokens.push(token);
+                pos = token.end;
+            }
+            else {
+                tokens.push(createToken('text', remaining[0], pos));
+                pos += 1;
+            }
+        }
+        if (tokens.length === 0) {
+            tokens.push(createToken('text', '', 0));
+        }
+        return { lineNumber, tokens, text: line, state };
+    }
+    getNextToken(text, pos) {
+        // Whitespace
+        const wsMatch = text.match(/^[ \t]+/);
+        if (wsMatch) {
+            return createToken('text', wsMatch[0], pos);
+        }
+        // Property keys (strings before colon)
+        if (text.startsWith('"')) {
+            const stringEnd = this.findStringEnd(text);
+            const str = text.slice(0, stringEnd);
+            // Check if followed by colon (property key)
+            const afterString = text.slice(stringEnd).trim();
+            if (afterString.startsWith(':')) {
+                return createToken('property', str, pos);
+            }
+            return createToken('string', str, pos);
+        }
+        // Numbers
+        const numMatch = text.match(/^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?/);
+        if (numMatch) {
+            return createToken('number', numMatch[0], pos);
+        }
+        // Boolean and null
+        const boolMatch = text.match(/^(?:true|false)\b/);
+        if (boolMatch) {
+            return createToken('constant.boolean', boolMatch[0], pos);
+        }
+        const nullMatch = text.match(/^null\b/);
+        if (nullMatch) {
+            return createToken('constant.null', nullMatch[0], pos);
+        }
+        // Punctuation
+        if (text[0] === '{' || text[0] === '}') {
+            return createToken('punctuation.brace', text[0], pos);
+        }
+        if (text[0] === '[' || text[0] === ']') {
+            return createToken('punctuation.bracket', text[0], pos);
+        }
+        if (text[0] === ':' || text[0] === ',') {
+            return createToken('punctuation.separator', text[0], pos);
+        }
+        return createToken('text', text[0], pos);
+    }
+    findStringEnd(text) {
+        let i = 1;
+        while (i < text.length) {
+            if (text[i] === '\\' && i + 1 < text.length) {
+                i += 2;
+                continue;
+            }
+            if (text[i] === '"') {
+                return i + 1;
+            }
+            i++;
+        }
+        return text.length;
+    }
+}
+export function createJSONTokenizer() {
+    return new JSONTokenizer();
+}

package/dist/components/editor/tokenizer/languages/markdown.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Markdown tokenizer
+ */
+import type { TokenizedLine, TokenizerState } from '../types';
+interface MarkdownTokenizerState extends TokenizerState {
+    inFencedCode?: boolean;
+    codeLanguage?: string;
+}
+export declare class MarkdownTokenizer {
+    language: string;
+    getInitialState(): MarkdownTokenizerState;
+    tokenizeLine(line: string, lineNumber: number, prevState?: MarkdownTokenizerState): TokenizedLine;
+    private tokenizeInline;
+}
+export declare function createMarkdownTokenizer(): MarkdownTokenizer;
+export {};

package/dist/components/editor/tokenizer/languages/markdown.js

@@ -0,0 +1,156 @@
+/**
+ * Markdown tokenizer
+ */
+import { createToken } from '../base';
+export class MarkdownTokenizer {
+    language = 'markdown';
+    getInitialState() {
+        return {};
+    }
+    tokenizeLine(line, lineNumber, prevState) {
+        const tokens = [];
+        const state = { ...prevState };
+        // Handle fenced code block
+        if (state.inFencedCode) {
+            if (line.match(/^```\s*$/)) {
+                tokens.push(createToken('markup.code', line, 0));
+                state.inFencedCode = false;
+                state.codeLanguage = undefined;
+            }
+            else {
+                tokens.push(createToken('markup.code', line, 0));
+            }
+            return { lineNumber, tokens, text: line, state };
+        }
+        // Check for fenced code block start
+        const fenceMatch = line.match(/^```(\w*)/);
+        if (fenceMatch) {
+            tokens.push(createToken('markup.code', line, 0));
+            state.inFencedCode = true;
+            state.codeLanguage = fenceMatch[1] || undefined;
+            return { lineNumber, tokens, text: line, state };
+        }
+        // Headings
+        const headingMatch = line.match(/^(#{1,6})\s+(.*)$/);
+        if (headingMatch) {
+            tokens.push(createToken('markup.heading', headingMatch[1], 0));
+            tokens.push(createToken('text', ' ', headingMatch[1].length));
+            this.tokenizeInline(headingMatch[2], headingMatch[1].length + 1, tokens);
+            return { lineNumber, tokens, text: line, state };
+        }
+        // Horizontal rule
+        if (line.match(/^(?:[-*_]\s*){3,}$/)) {
+            tokens.push(createToken('punctuation', line, 0));
+            return { lineNumber, tokens, text: line, state };
+        }
+        // Block quotes
+        const quoteMatch = line.match(/^(>\s*)/);
+        if (quoteMatch) {
+            tokens.push(createToken('markup.quote', quoteMatch[1], 0));
+            this.tokenizeInline(line.slice(quoteMatch[1].length), quoteMatch[1].length, tokens);
+            return { lineNumber, tokens, text: line, state };
+        }
+        // Unordered lists
+        const ulMatch = line.match(/^(\s*[-*+])\s/);
+        if (ulMatch) {
+            tokens.push(createToken('markup.list', ulMatch[1], 0));
+            tokens.push(createToken('text', ' ', ulMatch[1].length));
+            this.tokenizeInline(line.slice(ulMatch[0].length), ulMatch[0].length, tokens);
+            return { lineNumber, tokens, text: line, state };
+        }
+        // Ordered lists
+        const olMatch = line.match(/^(\s*\d+\.)\s/);
+        if (olMatch) {
+            tokens.push(createToken('markup.list', olMatch[1], 0));
+            tokens.push(createToken('text', ' ', olMatch[1].length));
+            this.tokenizeInline(line.slice(olMatch[0].length), olMatch[0].length, tokens);
+            return { lineNumber, tokens, text: line, state };
+        }
+        // Indented code block
+        if (line.match(/^(?:    |\t)/)) {
+            tokens.push(createToken('markup.code', line, 0));
+            return { lineNumber, tokens, text: line, state };
+        }
+        // Regular paragraph
+        this.tokenizeInline(line, 0, tokens);
+        if (tokens.length === 0) {
+            tokens.push(createToken('text', '', 0));
+        }
+        return { lineNumber, tokens, text: line, state };
+    }
+    tokenizeInline(text, offset, tokens) {
+        let pos = 0;
+        while (pos < text.length) {
+            const remaining = text.slice(pos);
+            // Inline code
+            const codeMatch = remaining.match(/^`([^`]+)`/);
+            if (codeMatch) {
+                tokens.push(createToken('markup.code', codeMatch[0], offset + pos));
+                pos += codeMatch[0].length;
+                continue;
+            }
+            // Images ![alt](url)
+            const imgMatch = remaining.match(/^!\[([^\]]*)\]\(([^)]+)\)/);
+            if (imgMatch) {
+                tokens.push(createToken('markup.link', imgMatch[0], offset + pos));
+                pos += imgMatch[0].length;
+                continue;
+            }
+            // Links [text](url)
+            const linkMatch = remaining.match(/^\[([^\]]+)\]\(([^)]+)\)/);
+            if (linkMatch) {
+                tokens.push(createToken('markup.link', linkMatch[0], offset + pos));
+                pos += linkMatch[0].length;
+                continue;
+            }
+            // Reference-style links [text][ref]
+            const refLinkMatch = remaining.match(/^\[([^\]]+)\]\[([^\]]*)\]/);
+            if (refLinkMatch) {
+                tokens.push(createToken('markup.link', refLinkMatch[0], offset + pos));
+                pos += refLinkMatch[0].length;
+                continue;
+            }
+            // Autolinks <url>
+            const autoLinkMatch = remaining.match(/^<(https?:\/\/[^>]+)>/);
+            if (autoLinkMatch) {
+                tokens.push(createToken('markup.link', autoLinkMatch[0], offset + pos));
+                pos += autoLinkMatch[0].length;
+                continue;
+            }
+            // Bold **text** or __text__
+            const boldMatch = remaining.match(/^(?:\*\*([^*]+)\*\*|__([^_]+)__)/);
+            if (boldMatch) {
+                tokens.push(createToken('markup.bold', boldMatch[0], offset + pos));
+                pos += boldMatch[0].length;
+                continue;
+            }
+            // Italic *text* or _text_
+            const italicMatch = remaining.match(/^(?:\*([^*]+)\*|_([^_]+)_)/);
+            if (italicMatch) {
+                tokens.push(createToken('markup.italic', italicMatch[0], offset + pos));
+                pos += italicMatch[0].length;
+                continue;
+            }
+            // Strikethrough ~~text~~
+            const strikeMatch = remaining.match(/^~~([^~]+)~~/);
+            if (strikeMatch) {
+                tokens.push(createToken('punctuation', strikeMatch[0], offset + pos));
+                pos += strikeMatch[0].length;
+                continue;
+            }
+            // Regular text
+            const textMatch = remaining.match(/^[^`*_\[!<~]+/);
+            if (textMatch) {
+                tokens.push(createToken('text', textMatch[0], offset + pos));
+                pos += textMatch[0].length;
+                continue;
+            }
+            // Single character fallback
+            tokens.push(createToken('text', remaining[0], offset + pos));
+            pos += 1;
+        }
+    }
+}
+export function createMarkdownTokenizer() {
+    return new MarkdownTokenizer();
+}

package/dist/components/editor/tokenizer/languages/python.d.ts

@@ -0,0 +1,20 @@
+/**
+ * Python tokenizer
+ */
+import type { TokenizedLine, TokenizerState } from '../types';
+interface PythonTokenizerState extends TokenizerState {
+    stringDelimiter?: string;
+    stringTriple?: boolean;
+}
+export declare class PythonTokenizer {
+    language: string;
+    getInitialState(): PythonTokenizerState;
+    tokenizeLine(line: string, lineNumber: number, prevState?: PythonTokenizerState): TokenizedLine;
+    private getNextToken;
+    private classifyIdentifier;
+    private tokenizeString;
+    private tokenizeTripleString;
+    private tokenizeMultilineStringContinuation;
+}
+export declare function createPythonTokenizer(): PythonTokenizer;
+export {};

package/dist/components/editor/tokenizer/languages/python.js

@@ -0,0 +1,227 @@
+/**
+ * Python tokenizer
+ */
+import { createToken } from '../base';
+// Python keywords
+const keywords = new Set([
+    'and', 'as', 'assert', 'async', 'await', 'break', 'class', 'continue',
+    'def', 'del', 'elif', 'else', 'except', 'finally', 'for', 'from',
+    'global', 'if', 'import', 'in', 'is', 'lambda', 'nonlocal', 'not',
+    'or', 'pass', 'raise', 'return', 'try', 'while', 'with', 'yield'
+]);
+const controlKeywords = new Set([
+    'if', 'elif', 'else', 'for', 'while', 'try', 'except', 'finally',
+    'with', 'break', 'continue', 'return', 'raise', 'yield', 'await'
+]);
+const builtins = new Set([
+    'True', 'False', 'None', 'self', 'cls'
+]);
+const builtinFunctions = new Set([
+    'abs', 'all', 'any', 'ascii', 'bin', 'bool', 'bytearray', 'bytes',
+    'callable', 'chr', 'classmethod', 'compile', 'complex', 'delattr',
+    'dict', 'dir', 'divmod', 'enumerate', 'eval', 'exec', 'filter',
+    'float', 'format', 'frozenset', 'getattr', 'globals', 'hasattr',
+    'hash', 'help', 'hex', 'id', 'input', 'int', 'isinstance', 'issubclass',
+    'iter', 'len', 'list', 'locals', 'map', 'max', 'memoryview', 'min',
+    'next', 'object', 'oct', 'open', 'ord', 'pow', 'print', 'property',
+    'range', 'repr', 'reversed', 'round', 'set', 'setattr', 'slice',
+    'sorted', 'staticmethod', 'str', 'sum', 'super', 'tuple', 'type',
+    'vars', 'zip', '__import__'
+]);
+export class PythonTokenizer {
+    language = 'python';
+    getInitialState() {
+        return {};
+    }
+    tokenizeLine(line, lineNumber, prevState) {
+        const tokens = [];
+        let pos = 0;
+        const state = { ...prevState };
+        // Handle multi-line string continuation
+        if (state.inMultilineString && state.stringDelimiter) {
+            const result = this.tokenizeMultilineStringContinuation(line, pos, state);
+            tokens.push(...result.tokens);
+            pos = result.pos;
+            if (pos >= line.length) {
+                return { lineNumber, tokens, text: line, state };
+            }
+        }
+        while (pos < line.length) {
+            const remaining = line.slice(pos);
+            const token = this.getNextToken(remaining, pos, state);
+            if (token) {
+                tokens.push(token);
+                pos = token.end;
+            }
+            else {
+                tokens.push(createToken('text', remaining[0], pos));
+                pos += 1;
+            }
+        }
+        if (tokens.length === 0) {
+            tokens.push(createToken('text', '', 0));
+        }
+        return { lineNumber, tokens, text: line, state };
+    }
+    getNextToken(text, pos, state) {
+        // Whitespace
+        const wsMatch = text.match(/^[ \t]+/);
+        if (wsMatch) {
+            return createToken('text', wsMatch[0], pos);
+        }
+        // Comments
+        if (text.startsWith('#')) {
+            return createToken('comment.line', text, pos);
+        }
+        // Triple-quoted strings
+        const tripleMatch = text.match(/^(?:f|r|b|fr|rf|br|rb)?(?:'''|""")/i);
+        if (tripleMatch) {
+            const prefix = tripleMatch[0];
+            const delimiter = prefix.slice(-3);
+            return this.tokenizeTripleString(text, pos, delimiter, state);
+        }
+        // Regular strings
+        const stringMatch = text.match(/^(?:f|r|b|fr|rf|br|rb)?["']/i);
+        if (stringMatch) {
+            const prefix = stringMatch[0];
+            const delimiter = prefix.slice(-1);
+            return this.tokenizeString(text, pos, delimiter);
+        }
+        // Numbers
+        const numMatch = text.match(/^(?:0[xX][0-9a-fA-F]+|0[bB][01]+|0[oO][0-7]+|(?:\d+\.?\d*|\.\d+)(?:[eE][+-]?\d+)?j?)/);
+        if (numMatch) {
+            return createToken('number', numMatch[0], pos);
+        }
+        // Decorators
+        if (text.startsWith('@')) {
+            const decoratorMatch = text.match(/^@[a-zA-Z_][a-zA-Z0-9_.]*/);
+            if (decoratorMatch) {
+                return createToken('function', decoratorMatch[0], pos);
+            }
+        }
+        // Identifiers and keywords
+        const identMatch = text.match(/^[a-zA-Z_][a-zA-Z0-9_]*/);
+        if (identMatch) {
+            const word = identMatch[0];
+            return createToken(this.classifyIdentifier(word, text, identMatch[0].length), word, pos);
+        }
+        // Operators
+        const opMatch = text.match(/^(?:->|:=|==|!=|<=|>=|<<|>>|\*\*|\/\/|[+\-*/%&|^~<>=!@])/);
+        if (opMatch) {
+            return createToken('operator', opMatch[0], pos);
+        }
+        // Punctuation
+        const punctMatch = text.match(/^[{}[\](),.:;]/);
+        if (punctMatch) {
+            const char = punctMatch[0];
+            let type = 'punctuation';
+            if (char === '{' || char === '}')
+                type = 'punctuation.brace';
+            else if (char === '[' || char === ']')
+                type = 'punctuation.bracket';
+            else if (char === '(' || char === ')')
+                type = 'punctuation.paren';
+            else if (char === ',' || char === ';' || char === ':')
+                type = 'punctuation.separator';
+            else if (char === '.')
+                type = 'punctuation.accessor';
+            return createToken(type, char, pos);
+        }
+        return createToken('text', text[0], pos);
+    }
+    classifyIdentifier(word, context, wordLength) {
+        // Built-in constants
+        if (builtins.has(word)) {
+            if (word === 'True' || word === 'False')
+                return 'constant.boolean';
+            if (word === 'None')
+                return 'constant.null';
+            return 'constant.builtin';
+        }
+        // Keywords
+        if (keywords.has(word)) {
+            if (controlKeywords.has(word))
+                return 'keyword.control';
+            if (word === 'def' || word === 'class' || word === 'lambda')
+                return 'keyword.definition';
+            if (word === 'import' || word === 'from' || word === 'as')
+                return 'keyword.module';
+            return 'keyword';
+        }
+        // Built-in functions
+        if (builtinFunctions.has(word)) {
+            const afterWord = context.slice(wordLength).trim();
+            if (afterWord.startsWith('(')) {
+                return 'function.call';
+            }
+            return 'type.builtin';
+        }
+        // Check if followed by ( - likely a function call
+        const afterWord = context.slice(wordLength).trim();
+        if (afterWord.startsWith('(')) {
+            return 'function.call';
+        }
+        // Check if it's a type (PascalCase)
+        if (/^[A-Z][a-zA-Z0-9]*$/.test(word)) {
+            return 'type.class';
+        }
+        // Magic methods/attributes
+        if (word.startsWith('__') && word.endsWith('__')) {
+            return 'constant.builtin';
+        }
+        return 'variable';
+    }
+    tokenizeString(text, pos, delimiter) {
+        // Find prefix length
+        const prefixMatch = text.match(/^(?:f|r|b|fr|rf|br|rb)?/i);
+        const prefixLen = prefixMatch ? prefixMatch[0].length : 0;
+        let i = prefixLen + 1;
+        while (i < text.length) {
+            if (text[i] === '\\' && i + 1 < text.length) {
+                i += 2;
+                continue;
+            }
+            if (text[i] === delimiter) {
+                return createToken('string', text.slice(0, i + 1), pos);
+            }
+            if (text[i] === '\n') {
+                break;
+            }
+            i++;
+        }
+        return createToken('string', text.slice(0, i), pos);
+    }
+    tokenizeTripleString(text, pos, delimiter, state) {
+        // Find prefix length
+        const prefixMatch = text.match(/^(?:f|r|b|fr|rf|br|rb)?/i);
+        const prefixLen = prefixMatch ? prefixMatch[0].length : 0;
+        const searchStart = prefixLen + 3;
+        const endIdx = text.indexOf(delimiter, searchStart);
+        if (endIdx !== -1) {
+            return createToken('string', text.slice(0, endIdx + 3), pos);
+        }
+        else {
+            state.inMultilineString = true;
+            state.stringDelimiter = delimiter;
+            return createToken('string', text, pos);
+        }
+    }
+    tokenizeMultilineStringContinuation(line, startPos, state) {
+        const tokens = [];
+        const delimiter = state.stringDelimiter || '"""';
+        const endIdx = line.indexOf(delimiter);
+        if (endIdx !== -1) {
+            tokens.push(createToken('string', line.slice(0, endIdx + 3), startPos));
+            state.inMultilineString = false;
+            state.stringDelimiter = undefined;
+            return { tokens, pos: endIdx + 3 };
+        }
+        else {
+            tokens.push(createToken('string', line, startPos));
+            return { tokens, pos: line.length };
+        }
+    }
+}
+export function createPythonTokenizer() {
+    return new PythonTokenizer();
+}

package/dist/components/editor/tokenizer/languages/svelte.d.ts

@@ -0,0 +1,40 @@
+/**
+ * Svelte tokenizer for syntax highlighting
+ *
+ * Supports:
+ * - Template directives: {#if}, {#each}, {@html}, etc.
+ * - JavaScript expressions in curly braces
+ * - Svelte-specific attributes: bind:, on:, use:, transition:, etc.
+ * - Script and style sections
+ * - HTML template parts
+ */
+import type { LanguageTokenizer, TokenizedLine, TokenizerState } from '../types';
+/**
+ * Svelte-specific tokenizer state
+ */
+export interface SvelteTokenizerState extends TokenizerState {
+    /** Current context: 'template', 'script', or 'style' */
+    context: 'template' | 'script' | 'style';
+    /** Depth of nested script/style tags */
+    tagDepth: number;
+    /** Script/style tokenizer state */
+    innerState?: TokenizerState;
+}
+/**
+ * Svelte tokenizer
+ */
+export declare class SvelteTokenizer implements LanguageTokenizer {
+    language: string;
+    private tsTokenizer;
+    private cssTokenizer;
+    constructor();
+    getInitialState(): SvelteTokenizerState;
+    tokenizeLine(line: string, lineNumber: number, state: SvelteTokenizerState): TokenizedLine;
+    private tokenizeTemplate;
+    private tokenizeTag;
+    private tokenizeJSExpression;
+}
+/**
+ * Create a Svelte tokenizer
+ */
+export declare function createSvelteTokenizer(): SvelteTokenizer;