@nocturnium/svelte-ide 1.0.1 → 1.0.3

package/dist/components/editor/index.js

@@ -4,13 +4,27 @@
 // Components
 export { default as Editor } from './Editor.svelte';
 export { default as CustomEditor } from './CustomEditor.svelte';
-export { default as CollaborativeEditor } from './CollaborativeEditor.svelte';
 export { default as EditorTabs } from './EditorTabs.svelte';
 export { default as EditorPane } from './EditorPane.svelte';
 export { default as FileIcon } from './FileIcon.svelte';
 export { default as FileExplorer } from './FileExplorer.svelte';
-// Core utilities
-export * from './core';
+// Core utilities (explicitly excluding CRDT binding; use @nocturnium/svelte-ide/crdt)
+export * from './core/state';
+export * from './core/navigation';
+export * from './core/keybindings';
+export * from './core/search';
+export * from './core/folding';
+export * from './core/multi-cursor';
+export * from './core/complexity-analyzer';
+export * from './core/ai-awareness';
+export * from './core/semantic-analyzer';
+export * from './core/commands';
+export * from './core/bracket-healer';
+export * from './core/git-blame';
+export * from './core/snippet-manager';
+export * from './core/quick-actions';
+export * from './core/diagnostics';
+export * from './core/breakpoints';
 // Theme
 export * from './theme';
 // Languages (explicit exports to avoid conflicts with tokenizer)
@@ -18,4 +32,4 @@ export { getLanguageExtension, getLanguageConfig, getLanguageFromExtension, getL
 // Re-export from languages (these exist in both but we prefer languages versions)
 export { getSupportedLanguages, isLanguageSupported } from './languages';
 // Tokenizer (explicit exports to avoid conflicts)
-export { getTokenizer, tokenize, getTokenClass, tokensToHTML, PlaintextTokenizer, SimpleTokenizer, GrammarTokenizer, createToken } from './tokenizer';
+export { getTokenizer, tokenize, getTokenClass, tokensToHTML, PlaintextTokenizer, createToken } from './tokenizer';

package/dist/components/editor/tokenizer/base.d.ts

@@ -1,35 +1,11 @@
 /**
  * Base tokenizer with common functionality
  */
-import type { Token, TokenizedLine, TokenizerState, TokenType, LanguageGrammar, LanguageTokenizer } from './types';
+import type { Token, TokenizedLine, TokenizerState, TokenType, LanguageTokenizer } from './types';
 /**
  * Create a token
  */
 export declare function createToken(type: TokenType, text: string, start: number): Token;
-/**
- * Base tokenizer class using grammar rules
- */
-export declare class GrammarTokenizer implements LanguageTokenizer {
-    language: string;
-    private grammar;
-    constructor(grammar: LanguageGrammar);
-    getInitialState(): TokenizerState;
-    tokenizeLine(line: string, lineNumber: number, prevState?: TokenizerState): TokenizedLine;
-    private updateState;
-}
-/**
- * Simple regex-based tokenizer for basic languages
- */
-export declare class SimpleTokenizer implements LanguageTokenizer {
-    language: string;
-    private patterns;
-    constructor(language: string, patterns: Array<{
-        type: TokenType;
-        regex: RegExp;
-    }>);
-    getInitialState(): TokenizerState;
-    tokenizeLine(line: string, lineNumber: number, prevState?: TokenizerState): TokenizedLine;
-}
 /**
  * Plaintext tokenizer - no highlighting
  */

package/dist/components/editor/tokenizer/base.js

@@ -12,178 +12,6 @@ export function createToken(type, text, start) {
         end: start + text.length
     };
 }
-/**
- * Base tokenizer class using grammar rules
- */
-export class GrammarTokenizer {
-    language;
-    grammar;
-    constructor(grammar) {
-        this.language = grammar.language;
-        this.grammar = grammar;
-    }
-    getInitialState() {
-        return {};
-    }
-    tokenizeLine(line, lineNumber, prevState) {
-        const tokens = [];
-        let pos = 0;
-        const state = { ...prevState };
-        let currentRuleSet = 'root';
-        // Determine starting rule set based on state
-        if (state.inBlockComment) {
-            currentRuleSet = 'blockComment';
-        }
-        else if (state.inTemplateLiteral) {
-            currentRuleSet = 'templateLiteral';
-        }
-        else if (state.inMultilineString) {
-            currentRuleSet = 'multilineString';
-        }
-        while (pos < line.length) {
-            const remaining = line.slice(pos);
-            let matched = false;
-            // Try to match rules in current rule set
-            const rules = this.grammar.rules[currentRuleSet] || this.grammar.rules['root'] || [];
-            for (const rule of rules) {
-                const match = remaining.match(rule.pattern);
-                if (match && match.index === 0) {
-                    const text = match[0];
-                    tokens.push(createToken(rule.type, text, pos));
-                    pos += text.length;
-                    matched = true;
-                    // Handle state transitions
-                    if (rule.nextState) {
-                        currentRuleSet = rule.nextState;
-                        this.updateState(state, rule.nextState, true);
-                    }
-                    if (rule.popState) {
-                        currentRuleSet = 'root';
-                        this.updateState(state, currentRuleSet, false);
-                    }
-                    break;
-                }
-            }
-            // No match - consume single character as text
-            if (!matched) {
-                const char = line[pos];
-                // Try to merge with previous text token
-                const lastToken = tokens[tokens.length - 1];
-                if (lastToken && lastToken.type === 'text' && lastToken.end === pos) {
-                    lastToken.text += char;
-                    lastToken.end += 1;
-                }
-                else {
-                    tokens.push(createToken('text', char, pos));
-                }
-                pos += 1;
-            }
-        }
-        // Handle empty lines
-        if (tokens.length === 0) {
-            tokens.push(createToken('text', '', 0));
-        }
-        return {
-            lineNumber,
-            tokens,
-            text: line,
-            state: { ...state }
-        };
-    }
-    updateState(state, ruleSet, entering) {
-        switch (ruleSet) {
-            case 'blockComment':
-                state.inBlockComment = entering;
-                break;
-            case 'templateLiteral':
-                state.inTemplateLiteral = entering;
-                break;
-            case 'multilineString':
-                state.inMultilineString = entering;
-                break;
-        }
-    }
-}
-/**
- * Simple regex-based tokenizer for basic languages
- */
-export class SimpleTokenizer {
-    language;
-    patterns;
-    constructor(language, patterns) {
-        this.language = language;
-        this.patterns = patterns;
-    }
-    getInitialState() {
-        return {};
-    }
-    tokenizeLine(line, lineNumber, prevState) {
-        const tokens = [];
-        let pos = 0;
-        const state = { ...prevState };
-        // Handle block comment continuation
-        if (state.inBlockComment) {
-            const endMatch = line.indexOf('*/');
-            if (endMatch !== -1) {
-                tokens.push(createToken('comment.block', line.slice(0, endMatch + 2), 0));
-                pos = endMatch + 2;
-                state.inBlockComment = false;
-            }
-            else {
-                tokens.push(createToken('comment.block', line, 0));
-                return { lineNumber, tokens, text: line, state };
-            }
-        }
-        while (pos < line.length) {
-            const remaining = line.slice(pos);
-            let matched = false;
-            // Check for block comment start
-            if (remaining.startsWith('/*')) {
-                const endMatch = remaining.indexOf('*/', 2);
-                if (endMatch !== -1) {
-                    const text = remaining.slice(0, endMatch + 2);
-                    tokens.push(createToken('comment.block', text, pos));
-                    pos += text.length;
-                }
-                else {
-                    tokens.push(createToken('comment.block', remaining, pos));
-                    state.inBlockComment = true;
-                    pos = line.length;
-                }
-                matched = true;
-                continue;
-            }
-            // Try each pattern
-            for (const { type, regex } of this.patterns) {
-                const match = remaining.match(regex);
-                if (match && match.index === 0) {
-                    const text = match[0];
-                    tokens.push(createToken(type, text, pos));
-                    pos += text.length;
-                    matched = true;
-                    break;
-                }
-            }
-            // No match - consume character
-            if (!matched) {
-                const char = remaining[0];
-                const lastToken = tokens[tokens.length - 1];
-                if (lastToken && lastToken.type === 'text' && lastToken.end === pos) {
-                    lastToken.text += char;
-                    lastToken.end += 1;
-                }
-                else {
-                    tokens.push(createToken('text', char, pos));
-                }
-                pos += 1;
-            }
-        }
-        if (tokens.length === 0) {
-            tokens.push(createToken('text', '', 0));
-        }
-        return { lineNumber, tokens, text: line, state };
-    }
-}
 /**
  * Plaintext tokenizer - no highlighting
  */

package/dist/components/editor/tokenizer/index.d.ts

@@ -15,6 +15,10 @@ export { GoTokenizer, createGoTokenizer } from './languages/go';
 export { MarkdownTokenizer, createMarkdownTokenizer } from './languages/markdown';
 export { SvelteTokenizer, createSvelteTokenizer } from './languages/svelte';
 import type { LanguageTokenizer, Token, TokenizedLine, TokenizerState, TokenType } from './types';
+/**
+ * Tokenizer factory functions by language
+ */
+export declare const tokenizerFactories: Record<string, () => LanguageTokenizer>;
 /**
  * Get the canonical language name from an alias or extension
  */

package/dist/components/editor/tokenizer/index.js

@@ -78,7 +78,7 @@ const languageAliases = {
 /**
  * Tokenizer factory functions by language
  */
-const tokenizerFactories = {
+export const tokenizerFactories = {
     javascript: createJavaScriptTokenizer,
     typescript: createTypeScriptTokenizer,
     jsx: createJSXTokenizer,

package/dist/components/editor/tokenizer/languages/html.d.ts

@@ -3,14 +3,15 @@
  */
 import type { TokenizedLine, TokenizerState } from '../types';
 interface HTMLTokenizerState extends TokenizerState {
-    inTag?: boolean;
     inScript?: boolean;
     inStyle?: boolean;
-    tagName?: string;
+    innerState?: TokenizerState;
 }
 export declare class HTMLTokenizer {
     language: string;
     private isXML;
+    private jsTokenizer;
+    private cssTokenizer;
     constructor(options?: {
         xml?: boolean;
     });

package/dist/components/editor/tokenizer/languages/html.js

@@ -2,9 +2,13 @@
  * HTML/XML tokenizer
  */
 import { createToken } from '../base';
+import { createCSSTokenizer } from './css';
+import { createJavaScriptTokenizer } from './javascript';
 export class HTMLTokenizer {
     language;
     isXML;
+    jsTokenizer = createJavaScriptTokenizer();
+    cssTokenizer = createCSSTokenizer();
     constructor(options = {}) {
         this.isXML = options.xml ?? false;
         this.language = this.isXML ? 'xml' : 'html';
@@ -29,6 +33,54 @@ export class HTMLTokenizer {
                 return { lineNumber, tokens, text: line, state };
             }
         }
+        if (!this.isXML && state.inScript) {
+            const closeScriptMatch = line.match(/<\/script>/i);
+            if (closeScriptMatch) {
+                const scriptPart = line.slice(0, closeScriptMatch.index);
+                if (scriptPart) {
+                    const result = this.jsTokenizer.tokenizeLine(scriptPart, lineNumber, state.innerState ?? this.jsTokenizer.getInitialState());
+                    tokens.push(...result.tokens);
+                    state.innerState = result.state;
+                }
+                state.inScript = false;
+                state.innerState = undefined;
+                pos = closeScriptMatch.index;
+            }
+            else {
+                const result = this.jsTokenizer.tokenizeLine(line, lineNumber, state.innerState ?? this.jsTokenizer.getInitialState());
+                state.innerState = result.state;
+                return {
+                    lineNumber,
+                    tokens: result.tokens,
+                    text: line,
+                    state
+                };
+            }
+        }
+        if (!this.isXML && state.inStyle) {
+            const closeStyleMatch = line.match(/<\/style>/i);
+            if (closeStyleMatch) {
+                const stylePart = line.slice(0, closeStyleMatch.index);
+                if (stylePart) {
+                    const result = this.cssTokenizer.tokenizeLine(stylePart, lineNumber, state.innerState ?? this.cssTokenizer.getInitialState());
+                    tokens.push(...result.tokens);
+                    state.innerState = result.state;
+                }
+                state.inStyle = false;
+                state.innerState = undefined;
+                pos = closeStyleMatch.index;
+            }
+            else {
+                const result = this.cssTokenizer.tokenizeLine(line, lineNumber, state.innerState ?? this.cssTokenizer.getInitialState());
+                state.innerState = result.state;
+                return {
+                    lineNumber,
+                    tokens: result.tokens,
+                    text: line,
+                    state
+                };
+            }
+        }
         while (pos < line.length) {
             const remaining = line.slice(pos);
             const token = this.getNextToken(remaining, pos, state, line);
@@ -89,10 +141,14 @@ export class HTMLTokenizer {
         const closingTagMatch = text.match(/^<\/([a-zA-Z][a-zA-Z0-9:-]*)>/);
         if (closingTagMatch) {
             const tagName = closingTagMatch[1].toLowerCase();
-            if (tagName === 'script')
+            if (tagName === 'script') {
                 state.inScript = false;
-            if (tagName === 'style')
+                state.innerState = undefined;
+            }
+            if (tagName === 'style') {
                 state.inStyle = false;
+                state.innerState = undefined;
+            }
             return createToken('tag', closingTagMatch[0], pos);
         }
         // Opening tag
@@ -121,17 +177,19 @@ export class HTMLTokenizer {
         }
         if (openingMatch) {
             const tagName = openingMatch[1].toLowerCase();
-            if (tagName === 'script')
+            if (!this.isXML && tagName === 'script') {
                 state.inScript = true;
-            if (tagName === 'style')
+                state.innerState = this.jsTokenizer.getInitialState();
+            }
+            if (!this.isXML && tagName === 'style') {
                 state.inStyle = true;
+                state.innerState = this.cssTokenizer.getInitialState();
+            }
             return createToken('tag', openingMatch[0], pos);
         }
         // Partial tag - just match the tag name part
         const partialMatch = text.match(/^<([a-zA-Z][a-zA-Z0-9:-]*)/);
         if (partialMatch) {
-            state.inTag = true;
-            state.tagName = partialMatch[1];
             return createToken('tag.name', partialMatch[0], pos);
         }
         return createToken('tag.punctuation', '<', pos);

package/dist/components/editor/tokenizer/languages/javascript.d.ts

@@ -4,9 +4,6 @@
 import type { TokenizedLine, TokenizerState } from '../types';
 interface JSTokenizerState extends TokenizerState {
     templateDepth?: number;
-    jsxDepth?: number;
-    /** Track if regex is valid in current context (for division vs regex ambiguity) */
-    expectExpression?: boolean;
     /**
      * Last significant token text, used for regex/division disambiguation.
      * Stored per-line in the threaded state (not on the tokenizer instance) so a
@@ -45,8 +42,19 @@ export declare class JavaScriptTokenizer {
      */
     private tryMatchRegex;
     private tokenizeString;
-    private tokenizeTemplateLiteral;
-    private tokenizeTemplateLiteralContinuation;
+    /**
+     * Scan the string portion of a template literal, starting at `startPos`. Emits
+     * one `string.template` token for the run of literal characters and stops at one
+     * of three boundaries:
+     *  - a closing backtick → ends the literal (clears template state);
+     *  - a `${` → emits the `${` delimiter and enters interpolation
+     *    (templateDepth = 1) so the expression is tokenized as code by the caller;
+     *  - end of line → the literal spans lines and continues next line.
+     *
+     * `isStart` is true when this is the opening backtick (vs. a continuation of a
+     * multi-line literal or the resumption after a `${…}` interpolation).
+     */
+    private scanTemplateString;
     private tokenizeJSXTag;
 }
 export declare function createJavaScriptTokenizer(): JavaScriptTokenizer;

package/dist/components/editor/tokenizer/languages/javascript.js

@@ -234,8 +234,7 @@ export class JavaScriptTokenizer {
     }
     getInitialState() {
         return {
-            templateDepth: 0,
-            jsxDepth: 0
+            templateDepth: 0
         };
     }
     tokenizeLine(line, lineNumber, prevState) {
@@ -263,22 +262,39 @@ export class JavaScriptTokenizer {
                 return { lineNumber, tokens, text: line, state };
             }
         }
-        // Handle template literal continuation
-        if (state.inTemplateLiteral && (state.templateDepth ?? 0) > 0) {
-            const result = this.tokenizeTemplateLiteralContinuation(line, pos, state);
-            tokens.push(...result.tokens);
-            pos = result.pos;
-            if (pos >= line.length) {
-                return { lineNumber, tokens, text: line, state };
-            }
-        }
         while (pos < line.length) {
+            // Template literals: scan the string portion (between the backticks and
+            // `${` / `}`) as a single string.template token. The interpolation
+            // expression is tokenized as ordinary code below, tracking brace depth in
+            // state.templateDepth so the closing backtick is recognised. This fixes the
+            // leak where an unclosed template state bled onto every following line.
+            const inStringPortion = state.inTemplateLiteral && (state.templateDepth ?? 0) === 0;
+            const startsTemplate = !state.inTemplateLiteral && line[pos] === '`';
+            if (inStringPortion || startsTemplate) {
+                const result = this.scanTemplateString(line, pos, state, startsTemplate);
+                tokens.push(...result.tokens);
+                pos = result.pos;
+                continue;
+            }
             const remaining = line.slice(pos);
             const token = this.getNextToken(remaining, pos, state);
             if (token) {
                 tokens.push(token);
                 this.updateLastToken(token, state);
                 pos = token.end;
+                // Track brace nesting inside a ${...} interpolation so we know when it
+                // closes and we return to the template's string portion. Braces inside
+                // strings/comments are separate token types, so they aren't miscounted.
+                if (state.inTemplateLiteral &&
+                    (state.templateDepth ?? 0) > 0 &&
+                    token.type === 'punctuation.brace') {
+                    if (token.text === '{') {
+                        state.templateDepth = (state.templateDepth ?? 0) + 1;
+                    }
+                    else if (token.text === '}') {
+                        state.templateDepth = Math.max(0, (state.templateDepth ?? 1) - 1);
+                    }
+                }
             }
             else {
                 // No match - shouldn't happen but handle gracefully
@@ -314,10 +330,10 @@ export class JavaScriptTokenizer {
                 return createToken('comment.block', text, pos);
             }
         }
-        // Template literals
-        if (text.startsWith('`')) {
-            return this.tokenizeTemplateLiteral(text, pos, state);
-        }
+        // Template literals are handled in tokenizeLine's main loop
+        // (scanTemplateString) so the string portions and the ${...} interpolation
+        // expression are tokenized separately. A backtick only reaches here while
+        // inside an interpolation (a nested template); fall through to consume it.
         // Regular strings
         if (text.startsWith('"') || text.startsWith("'")) {
             return this.tokenizeString(text, pos, text[0]);
@@ -552,59 +568,55 @@ export class JavaScriptTokenizer {
         // Unterminated string at end of line
         return createToken('string', text, pos);
     }
-    tokenizeTemplateLiteral(text, pos, state) {
-        let i = 1;
-        let result = '`';
-        while (i < text.length) {
-            if (text[i] === '\\' && i + 1 < text.length) {
-                result += text.slice(i, i + 2);
-                i += 2;
-                continue;
-            }
-            if (text[i] === '`') {
-                result += '`';
-                return createToken('string.template', result, pos);
-            }
-            if (text[i] === '$' && text[i + 1] === '{') {
-                // Template expression - for simplicity, tokenize up to this point
-                if (result.length > 1) {
-                    // Return string part first
-                    state.inTemplateLiteral = true;
-                    state.templateDepth = (state.templateDepth ?? 0) + 1;
-                    return createToken('string.template', result, pos);
-                }
-            }
-            result += text[i];
-            i++;
-        }
-        // Multi-line template literal
-        state.inTemplateLiteral = true;
-        return createToken('string.template', result, pos);
-    }
-    tokenizeTemplateLiteralContinuation(line, startPos, state) {
-        const tokens = [];
+    /**
+     * Scan the string portion of a template literal, starting at `startPos`. Emits
+     * one `string.template` token for the run of literal characters and stops at one
+     * of three boundaries:
+     *  - a closing backtick → ends the literal (clears template state);
+     *  - a `${` → emits the `${` delimiter and enters interpolation
+     *    (templateDepth = 1) so the expression is tokenized as code by the caller;
+     *  - end of line → the literal spans lines and continues next line.
+     *
+     * `isStart` is true when this is the opening backtick (vs. a continuation of a
+     * multi-line literal or the resumption after a `${…}` interpolation).
+     */
+    scanTemplateString(line, startPos, state, isStart) {
         let pos = startPos;
         let result = '';
+        if (isStart) {
+            state.inTemplateLiteral = true;
+            state.templateDepth = 0;
+            result = '`';
+            pos += 1;
+        }
         while (pos < line.length) {
-            if (line[pos] === '\\' && pos + 1 < line.length) {
+            const ch = line[pos];
+            if (ch === '\\' && pos + 1 < line.length) {
                 result += line.slice(pos, pos + 2);
                 pos += 2;
                 continue;
             }
-            if (line[pos] === '`') {
+            if (ch === '`') {
                 result += '`';
-                tokens.push(createToken('string.template', result, startPos));
                 state.inTemplateLiteral = false;
-                state.templateDepth = Math.max(0, (state.templateDepth ?? 1) - 1);
-                return { tokens, pos: pos + 1 };
+                state.templateDepth = 0;
+                return { tokens: [createToken('string.template', result, startPos)], pos: pos + 1 };
             }
-            result += line[pos];
-            pos++;
-        }
-        if (result) {
-            tokens.push(createToken('string.template', result, startPos));
+            if (ch === '$' && line[pos + 1] === '{') {
+                const tokens = [];
+                if (result) {
+                    tokens.push(createToken('string.template', result, startPos));
+                }
+                tokens.push(createToken('string.template', '${', startPos + result.length));
+                state.templateDepth = 1;
+                return { tokens, pos: pos + 2 };
+            }
+            result += ch;
+            pos += 1;
         }
-        return { tokens, pos };
+        // End of line inside the string portion → multi-line template literal.
+        state.inTemplateLiteral = true;
+        return { tokens: result ? [createToken('string.template', result, startPos)] : [], pos };
     }
     tokenizeJSXTag(text, pos, _state) {
         // Simple JSX tag detection

package/dist/components/editor/tokenizer/languages/svelte.d.ts

@@ -29,7 +29,7 @@ export declare class SvelteTokenizer implements LanguageTokenizer {
     private cssTokenizer;
     constructor();
     getInitialState(): SvelteTokenizerState;
-    tokenizeLine(line: string, lineNumber: number, state: SvelteTokenizerState): TokenizedLine;
+    tokenizeLine(line: string, lineNumber: number, prevState?: SvelteTokenizerState): TokenizedLine;
     private tokenizeTemplate;
     private tokenizeTag;
     private tokenizeJSExpression;

package/dist/components/editor/tokenizer/languages/svelte.js

@@ -65,9 +65,14 @@ export class SvelteTokenizer {
             tagDepth: 0
         };
     }
-    tokenizeLine(line, lineNumber, state) {
+    tokenizeLine(line, lineNumber, prevState) {
         const tokens = [];
         let pos = 0;
+        const state = {
+            ...this.getInitialState(),
+            ...prevState,
+            innerState: prevState?.innerState ? { ...prevState.innerState } : undefined
+        };
         // Handle script context
         if (state.context === 'script') {
             const closeScriptMatch = line.match(/<\/script>/i);

package/dist/components/editor/tokenizer/types.d.ts

@@ -56,31 +56,3 @@ export interface LanguageTokenizer {
     /** Get initial state */
     getInitialState(): TokenizerState;
 }
-/**
- * Token rule for regex-based tokenization
- */
-export interface TokenRule {
-    /** Token type to assign */
-    type: TokenType;
-    /** Regex pattern to match */
-    pattern: RegExp;
-    /** Optional: next state after matching */
-    nextState?: string;
-    /** Optional: pop state after matching */
-    popState?: boolean;
-}
-/**
- * Grammar definition for a language
- */
-export interface LanguageGrammar {
-    /** Language identifier */
-    language: string;
-    /** File extensions */
-    extensions: string[];
-    /** Line comment prefix */
-    lineComment?: string;
-    /** Block comment start/end */
-    blockComment?: [string, string];
-    /** Token rules by state */
-    rules: Record<string, TokenRule[]>;
-}