@nocturnium/svelte-ide 1.0.2 → 1.0.4

package/dist/components/editor/core/state.js

@@ -7,7 +7,7 @@
  */
 import { getTokenizer } from '../tokenizer';
 import { HISTORY_GROUP_TIMEOUT_MS, MAX_HISTORY_SIZE } from '../constants';
-import { CursorManager, createCursorManager, getSelectionStart, getSelectionEnd, isSelectionEmpty } from './multi-cursor';
+import { CursorManager, createCursorManager, comparePositions, getSelectionStart, getSelectionEnd, isSelectionEmpty } from './multi-cursor';
 /**
  * Core editor state class
  */
@@ -361,10 +361,12 @@ export class EditorState {
                 const start = getSelectionStart(cursor.selection);
                 const end = getSelectionEnd(cursor.selection);
                 this.deleteRangeInternal(start, end);
+                this.transformCursorUpdatesForDelete(updates, start, end);
                 currentPos = start;
             }
             // Insert text
             const newPos = this.insertAtInternal(currentPos, text);
+            this.transformCursorUpdatesForInsert(updates, currentPos, text);
             updates.push({ id: cursor.id, position: newPos });
         }
         // Apply all cursor updates using batch update to avoid multiple merge/emit calls
@@ -453,6 +455,7 @@ export class EditorState {
                 const start = getSelectionStart(cursor.selection);
                 const end = getSelectionEnd(cursor.selection);
                 this.deleteRangeInternal(start, end);
+                this.transformCursorUpdatesForDelete(updates, start, end);
                 updates.push({ id: cursor.id, position: start });
             }
         }
@@ -519,25 +522,32 @@ export class EditorState {
         for (const cursor of cursors) {
             const { line, column } = cursor.selection.head;
             if (column > 0) {
+                const from = { line, column: column - 1 };
+                const to = { line, column };
                 // Delete within line
                 const currentLine = this._lines[line];
-                currentLine.text = currentLine.text.slice(0, column - 1) + currentLine.text.slice(column);
+                currentLine.text =
+                    currentLine.text.slice(0, from.column) + currentLine.text.slice(to.column);
                 // Re-tokenize to end of document so deleting a multi-line construct
                 // delimiter re-propagates state to lines below.
                 this.retokenize(line, this._lines.length);
-                updates.push({ id: cursor.id, position: { line, column: column - 1 } });
+                this.transformCursorUpdatesForDelete(updates, from, to);
+                updates.push({ id: cursor.id, position: from });
             }
             else if (line > 0) {
                 // Join with previous line
                 const prevLine = this._lines[line - 1];
                 const currentLine = this._lines[line];
                 const newColumn = prevLine.text.length;
+                const from = { line: line - 1, column: newColumn };
+                const to = { line, column: 0 };
                 prevLine.text += currentLine.text;
                 this._lines.splice(line, 1);
                 this._tokenizerStates.splice(line, 1);
                 this.renumberLines(line);
                 this.retokenize(line - 1, this._lines.length);
-                updates.push({ id: cursor.id, position: { line: line - 1, column: newColumn } });
+                this.transformCursorUpdatesForDelete(updates, from, to);
+                updates.push({ id: cursor.id, position: from });
             }
         }
         // Apply all cursor updates using batch update
@@ -564,28 +574,41 @@ export class EditorState {
             return;
         }
         this.saveHistory('delete');
+        const updates = [];
         for (const cursor of cursors) {
             const { line, column } = cursor.selection.head;
             const currentLine = this._lines[line];
             if (column < currentLine.text.length) {
+                const from = { line, column };
+                const to = { line, column: column + 1 };
                 // Delete within line
-                currentLine.text = currentLine.text.slice(0, column) + currentLine.text.slice(column + 1);
+                currentLine.text =
+                    currentLine.text.slice(0, from.column) + currentLine.text.slice(to.column);
                 // Re-tokenize to end of document so deleting a multi-line construct
                 // delimiter re-propagates state to lines below.
                 this.retokenize(line, this._lines.length);
+                this.transformCursorUpdatesForDelete(updates, from, to);
+                updates.push({ id: cursor.id, position: from });
             }
             else if (line < this._lines.length - 1) {
                 // Join with next line
                 const nextLine = this._lines[line + 1];
+                const from = { line, column };
+                const to = { line: line + 1, column: 0 };
                 currentLine.text += nextLine.text;
                 this._lines.splice(line + 1, 1);
                 this._tokenizerStates.splice(line + 1, 1);
                 this.renumberLines(line + 1);
                 this.retokenize(line, this._lines.length);
+                this.transformCursorUpdatesForDelete(updates, from, to);
+                updates.push({ id: cursor.id, position: from });
             }
         }
+        // Apply all cursor updates using batch update
+        this._cursorManager.batchUpdateCursors(updates);
         this.emitChange({ type: 'delete', from: this.cursor, to: this.cursor });
         this.emitSelectionChange();
+        this.emitCursorChange();
     }
     /**
      * Insert a new line (enter key)
@@ -624,8 +647,8 @@ export class EditorState {
             timestamp: now
         };
         if (shouldMerge) {
-            // Update the last entry instead of creating a new one
-            this._undoStack[this._undoStack.length - 1] = entry;
+            // Keep the original restore snapshot; only extend the group's recency.
+            this._undoStack[this._undoStack.length - 1].timestamp = now;
         }
         else {
             this._undoStack.push(entry);
@@ -715,6 +738,57 @@ export class EditorState {
             head: { line: sel.head.line, column: sel.head.column }
         };
     }
+    transformCursorUpdatesForInsert(updates, at, text) {
+        for (const update of updates) {
+            update.position = this.transformPositionForInsert(update.position, at, text);
+        }
+    }
+    transformPositionForInsert(position, at, text) {
+        if (comparePositions(position, at) < 0) {
+            return position;
+        }
+        const textLines = text.split('\n');
+        const insertedLineCount = textLines.length - 1;
+        if (insertedLineCount === 0) {
+            if (position.line !== at.line) {
+                return position;
+            }
+            return { line: position.line, column: position.column + text.length };
+        }
+        if (position.line === at.line) {
+            return {
+                line: position.line + insertedLineCount,
+                column: textLines[textLines.length - 1].length + position.column - at.column
+            };
+        }
+        return { line: position.line + insertedLineCount, column: position.column };
+    }
+    transformCursorUpdatesForDelete(updates, from, to) {
+        for (const update of updates) {
+            update.position = this.transformPositionForDelete(update.position, from, to);
+        }
+    }
+    transformPositionForDelete(position, from, to) {
+        if (comparePositions(position, from) <= 0) {
+            return position;
+        }
+        if (comparePositions(position, to) <= 0) {
+            return from;
+        }
+        if (from.line === to.line) {
+            if (position.line !== from.line) {
+                return position;
+            }
+            return { line: position.line, column: position.column - (to.column - from.column) };
+        }
+        if (position.line === to.line) {
+            return {
+                line: from.line,
+                column: from.column + position.column - to.column
+            };
+        }
+        return { line: position.line - (to.line - from.line), column: position.column };
+    }
     /**
      * Check if undo is available
      */
@@ -738,11 +812,56 @@ export class EditorState {
             return true;
         if (!a || !b)
             return a === b;
-        return (a.inBlockComment === b.inBlockComment &&
-            a.inTemplateLiteral === b.inTemplateLiteral &&
-            a.inMultilineString === b.inMultilineString &&
-            a.stringDelimiter === b.stringDelimiter &&
-            a.templateDepth === b.templateDepth);
+        const aKeys = Object.keys(a)
+            .filter((key) => a[key] !== undefined)
+            .sort();
+        const bKeys = Object.keys(b)
+            .filter((key) => b[key] !== undefined)
+            .sort();
+        if (aKeys.length !== bKeys.length)
+            return false;
+        for (let i = 0; i < aKeys.length; i++) {
+            const key = aKeys[i];
+            if (key !== bKeys[i])
+                return false;
+            if (!this.tokenizerStateValuesEqual(a[key], b[key])) {
+                return false;
+            }
+        }
+        return true;
+    }
+    tokenizerStateValuesEqual(a, b) {
+        if (a === b)
+            return true;
+        if (typeof a !== typeof b)
+            return false;
+        if (a === null || b === null)
+            return a === b;
+        if (typeof a !== 'object' || typeof b !== 'object')
+            return false;
+        if (Array.isArray(a) || Array.isArray(b)) {
+            if (!Array.isArray(a) || !Array.isArray(b) || a.length !== b.length)
+                return false;
+            return a.every((value, index) => this.tokenizerStateValuesEqual(value, b[index]));
+        }
+        const aRecord = a;
+        const bRecord = b;
+        const aKeys = Object.keys(aRecord)
+            .filter((key) => aRecord[key] !== undefined)
+            .sort();
+        const bKeys = Object.keys(bRecord)
+            .filter((key) => bRecord[key] !== undefined)
+            .sort();
+        if (aKeys.length !== bKeys.length)
+            return false;
+        for (let i = 0; i < aKeys.length; i++) {
+            const key = aKeys[i];
+            if (key !== bKeys[i])
+                return false;
+            if (!this.tokenizerStateValuesEqual(aRecord[key], bRecord[key]))
+                return false;
+        }
+        return true;
     }
     /**
      * Re-tokenize a range of lines with early-exit optimization

package/dist/components/editor/editor-find.d.ts

@@ -17,6 +17,7 @@ export interface EditorFindDeps {
     scrollCursorIntoView: () => void;
     focusEditor: () => void;
     isReadonly: () => boolean;
+    lineToVisualRow?: (line: number) => number;
 }
 export interface MatchRect {
     top: number;

package/dist/components/editor/editor-find.js

@@ -92,19 +92,20 @@ export function createEditorFind(deps) {
             return [];
         const { scrollTop, viewportHeight } = viewport;
         const { lineHeight, charWidth, gutterWidth, contentPadding } = measurements;
-        const firstVisibleLine = Math.max(0, Math.floor(scrollTop / lineHeight) - 1);
-        const lineCount = deps.getLineCount();
-        const lastVisibleLine = Math.min(lineCount - 1, Math.ceil((scrollTop + viewportHeight) / lineHeight) + 1);
+        const lineToVisualRow = deps.lineToVisualRow ?? ((line) => line);
+        const firstVisibleRow = Math.max(0, Math.floor(scrollTop / lineHeight) - 1);
+        const lastVisibleRow = Math.max(0, Math.ceil((scrollTop + viewportHeight) / lineHeight) + 1);
         const rects = [];
         for (let i = 0; i < matches.length; i++) {
             const match = matches[i];
-            if (match.line < firstVisibleLine || match.line > lastVisibleLine)
+            const visualRow = lineToVisualRow(match.line);
+            if (visualRow < firstVisibleRow || visualRow > lastVisibleRow)
                 continue;
             const width = (match.endColumn - match.startColumn) * charWidth;
             if (width <= 0)
                 continue;
             rects.push({
-                top: match.line * lineHeight,
+                top: visualRow * lineHeight,
                 left: gutterWidth + contentPadding + match.startColumn * charWidth,
                 width,
                 height: lineHeight,

package/dist/components/editor/editor-input.d.ts

@@ -30,6 +30,7 @@ export interface EditorInputDeps {
         lineHeight: number;
         gutterWidth: number;
     };
+    visualRowToLine?: (visualRow: number) => number;
     /**
      * Editor tab size (columns per tab stop). Optional for backwards
      * compatibility; when omitted, DEFAULT_TAB_SIZE is used. Needed so that

package/dist/components/editor/editor-input.js

@@ -82,7 +82,10 @@ export function createEditorInput(deps) {
         const x = e.clientX - rect.left - gutterWidth - CONTENT_PADDING;
         const y = e.clientY - rect.top;
         const editorState = deps.getEditorState();
-        const line = Math.max(0, Math.min(Math.floor((y + scrollTop) / lineHeight), editorState.lineCount - 1));
+        const visualRow = Math.max(0, Math.min(Math.floor((y + scrollTop) / lineHeight), editorState.lineCount - 1));
+        const line = deps.visualRowToLine
+            ? deps.visualRowToLine(visualRow)
+            : Math.max(0, Math.min(visualRow, editorState.lineCount - 1));
         const lineContent = editorState.getLine(line);
         const lineText = lineContent?.text ?? '';
         const tabSize = deps.getTabSize?.() ?? DEFAULT_TAB_SIZE;

package/dist/components/editor/editor-scroll.d.ts

@@ -2,6 +2,7 @@ import type { Selection } from './core';
 export interface ScrollConfig {
     getEditorContent: () => HTMLDivElement | null;
     getSelection: () => Selection;
+    lineToVisualRow?: (line: number) => number;
     getMeasurements: () => {
         lineHeight: number;
         charWidth: number;

package/dist/components/editor/editor-scroll.js

@@ -8,7 +8,8 @@ export function createEditorScroll(config) {
             return;
         const selection = config.getSelection();
         const { lineHeight, charWidth, gutterWidth } = config.getMeasurements();
-        const cursorTop = selection.head.line * lineHeight;
+        const lineToVisualRow = config.lineToVisualRow ?? ((line) => line);
+        const cursorTop = lineToVisualRow(selection.head.line) * lineHeight;
         const cursorLeft = gutterWidth + CONTENT_PADDING + selection.head.column * charWidth;
         const viewportHeight = editorContent.clientHeight;
         const viewportWidth = editorContent.clientWidth;

package/dist/components/editor/index.d.ts

@@ -3,13 +3,29 @@
  */
 export { default as Editor } from './Editor.svelte';
 export { default as CustomEditor } from './CustomEditor.svelte';
-export { default as CollaborativeEditor } from './CollaborativeEditor.svelte';
 export { default as EditorTabs } from './EditorTabs.svelte';
 export { default as EditorPane } from './EditorPane.svelte';
 export { default as FileIcon } from './FileIcon.svelte';
 export { default as FileExplorer } from './FileExplorer.svelte';
-export * from './core';
+export * from './core/state';
+export * from './core/navigation';
+export * from './core/keybindings';
+export * from './core/search';
+export * from './core/folding';
+export * from './core/multi-cursor';
+export * from './core/complexity-analyzer';
+export * from './core/ai-awareness';
+export * from './core/semantic-analyzer';
+export * from './core/commands';
+export * from './core/bracket-healer';
+export * from './core/git-blame';
+export * from './core/snippet-manager';
+export * from './core/quick-actions';
+export * from './core/diagnostics';
+export * from './core/breakpoints';
+export type { Position } from './core/state';
+export type { Diagnostic, Range } from './core/quick-actions';
 export * from './theme';
 export { getLanguageExtension, getLanguageConfig, getLanguageFromExtension, getLanguageFromFilename, getLanguageFromMimeType, getAllLanguageConfigs, resolveLanguage, type LanguageConfig } from './languages';
 export { getSupportedLanguages, isLanguageSupported } from './languages';
-export { getTokenizer, tokenize, getTokenClass, tokensToHTML, PlaintextTokenizer, SimpleTokenizer, GrammarTokenizer, createToken, type Token, type TokenizedLine, type TokenizerState, type TokenType, type LanguageTokenizer, type TokenRule, type LanguageGrammar } from './tokenizer';
+export { getTokenizer, tokenize, getTokenClass, tokensToHTML, PlaintextTokenizer, createToken, type Token, type TokenizedLine, type TokenizerState, type TokenType, type LanguageTokenizer } from './tokenizer';

package/dist/components/editor/index.js

@@ -4,13 +4,27 @@
 // Components
 export { default as Editor } from './Editor.svelte';
 export { default as CustomEditor } from './CustomEditor.svelte';
-export { default as CollaborativeEditor } from './CollaborativeEditor.svelte';
 export { default as EditorTabs } from './EditorTabs.svelte';
 export { default as EditorPane } from './EditorPane.svelte';
 export { default as FileIcon } from './FileIcon.svelte';
 export { default as FileExplorer } from './FileExplorer.svelte';
-// Core utilities
-export * from './core';
+// Core utilities (explicitly excluding CRDT binding; use @nocturnium/svelte-ide/crdt)
+export * from './core/state';
+export * from './core/navigation';
+export * from './core/keybindings';
+export * from './core/search';
+export * from './core/folding';
+export * from './core/multi-cursor';
+export * from './core/complexity-analyzer';
+export * from './core/ai-awareness';
+export * from './core/semantic-analyzer';
+export * from './core/commands';
+export * from './core/bracket-healer';
+export * from './core/git-blame';
+export * from './core/snippet-manager';
+export * from './core/quick-actions';
+export * from './core/diagnostics';
+export * from './core/breakpoints';
 // Theme
 export * from './theme';
 // Languages (explicit exports to avoid conflicts with tokenizer)
@@ -18,4 +32,4 @@ export { getLanguageExtension, getLanguageConfig, getLanguageFromExtension, getL
 // Re-export from languages (these exist in both but we prefer languages versions)
 export { getSupportedLanguages, isLanguageSupported } from './languages';
 // Tokenizer (explicit exports to avoid conflicts)
-export { getTokenizer, tokenize, getTokenClass, tokensToHTML, PlaintextTokenizer, SimpleTokenizer, GrammarTokenizer, createToken } from './tokenizer';
+export { getTokenizer, tokenize, getTokenClass, tokensToHTML, PlaintextTokenizer, createToken } from './tokenizer';

package/dist/components/editor/tokenizer/base.d.ts

@@ -1,35 +1,11 @@
 /**
  * Base tokenizer with common functionality
  */
-import type { Token, TokenizedLine, TokenizerState, TokenType, LanguageGrammar, LanguageTokenizer } from './types';
+import type { Token, TokenizedLine, TokenizerState, TokenType, LanguageTokenizer } from './types';
 /**
  * Create a token
  */
 export declare function createToken(type: TokenType, text: string, start: number): Token;
-/**
- * Base tokenizer class using grammar rules
- */
-export declare class GrammarTokenizer implements LanguageTokenizer {
-    language: string;
-    private grammar;
-    constructor(grammar: LanguageGrammar);
-    getInitialState(): TokenizerState;
-    tokenizeLine(line: string, lineNumber: number, prevState?: TokenizerState): TokenizedLine;
-    private updateState;
-}
-/**
- * Simple regex-based tokenizer for basic languages
- */
-export declare class SimpleTokenizer implements LanguageTokenizer {
-    language: string;
-    private patterns;
-    constructor(language: string, patterns: Array<{
-        type: TokenType;
-        regex: RegExp;
-    }>);
-    getInitialState(): TokenizerState;
-    tokenizeLine(line: string, lineNumber: number, prevState?: TokenizerState): TokenizedLine;
-}
 /**
  * Plaintext tokenizer - no highlighting
  */

package/dist/components/editor/tokenizer/base.js

@@ -12,178 +12,6 @@ export function createToken(type, text, start) {
         end: start + text.length
     };
 }
-/**
- * Base tokenizer class using grammar rules
- */
-export class GrammarTokenizer {
-    language;
-    grammar;
-    constructor(grammar) {
-        this.language = grammar.language;
-        this.grammar = grammar;
-    }
-    getInitialState() {
-        return {};
-    }
-    tokenizeLine(line, lineNumber, prevState) {
-        const tokens = [];
-        let pos = 0;
-        const state = { ...prevState };
-        let currentRuleSet = 'root';
-        // Determine starting rule set based on state
-        if (state.inBlockComment) {
-            currentRuleSet = 'blockComment';
-        }
-        else if (state.inTemplateLiteral) {
-            currentRuleSet = 'templateLiteral';
-        }
-        else if (state.inMultilineString) {
-            currentRuleSet = 'multilineString';
-        }
-        while (pos < line.length) {
-            const remaining = line.slice(pos);
-            let matched = false;
-            // Try to match rules in current rule set
-            const rules = this.grammar.rules[currentRuleSet] || this.grammar.rules['root'] || [];
-            for (const rule of rules) {
-                const match = remaining.match(rule.pattern);
-                if (match && match.index === 0) {
-                    const text = match[0];
-                    tokens.push(createToken(rule.type, text, pos));
-                    pos += text.length;
-                    matched = true;
-                    // Handle state transitions
-                    if (rule.nextState) {
-                        currentRuleSet = rule.nextState;
-                        this.updateState(state, rule.nextState, true);
-                    }
-                    if (rule.popState) {
-                        currentRuleSet = 'root';
-                        this.updateState(state, currentRuleSet, false);
-                    }
-                    break;
-                }
-            }
-            // No match - consume single character as text
-            if (!matched) {
-                const char = line[pos];
-                // Try to merge with previous text token
-                const lastToken = tokens[tokens.length - 1];
-                if (lastToken && lastToken.type === 'text' && lastToken.end === pos) {
-                    lastToken.text += char;
-                    lastToken.end += 1;
-                }
-                else {
-                    tokens.push(createToken('text', char, pos));
-                }
-                pos += 1;
-            }
-        }
-        // Handle empty lines
-        if (tokens.length === 0) {
-            tokens.push(createToken('text', '', 0));
-        }
-        return {
-            lineNumber,
-            tokens,
-            text: line,
-            state: { ...state }
-        };
-    }
-    updateState(state, ruleSet, entering) {
-        switch (ruleSet) {
-            case 'blockComment':
-                state.inBlockComment = entering;
-                break;
-            case 'templateLiteral':
-                state.inTemplateLiteral = entering;
-                break;
-            case 'multilineString':
-                state.inMultilineString = entering;
-                break;
-        }
-    }
-}
-/**
- * Simple regex-based tokenizer for basic languages
- */
-export class SimpleTokenizer {
-    language;
-    patterns;
-    constructor(language, patterns) {
-        this.language = language;
-        this.patterns = patterns;
-    }
-    getInitialState() {
-        return {};
-    }
-    tokenizeLine(line, lineNumber, prevState) {
-        const tokens = [];
-        let pos = 0;
-        const state = { ...prevState };
-        // Handle block comment continuation
-        if (state.inBlockComment) {
-            const endMatch = line.indexOf('*/');
-            if (endMatch !== -1) {
-                tokens.push(createToken('comment.block', line.slice(0, endMatch + 2), 0));
-                pos = endMatch + 2;
-                state.inBlockComment = false;
-            }
-            else {
-                tokens.push(createToken('comment.block', line, 0));
-                return { lineNumber, tokens, text: line, state };
-            }
-        }
-        while (pos < line.length) {
-            const remaining = line.slice(pos);
-            let matched = false;
-            // Check for block comment start
-            if (remaining.startsWith('/*')) {
-                const endMatch = remaining.indexOf('*/', 2);
-                if (endMatch !== -1) {
-                    const text = remaining.slice(0, endMatch + 2);
-                    tokens.push(createToken('comment.block', text, pos));
-                    pos += text.length;
-                }
-                else {
-                    tokens.push(createToken('comment.block', remaining, pos));
-                    state.inBlockComment = true;
-                    pos = line.length;
-                }
-                matched = true;
-                continue;
-            }
-            // Try each pattern
-            for (const { type, regex } of this.patterns) {
-                const match = remaining.match(regex);
-                if (match && match.index === 0) {
-                    const text = match[0];
-                    tokens.push(createToken(type, text, pos));
-                    pos += text.length;
-                    matched = true;
-                    break;
-                }
-            }
-            // No match - consume character
-            if (!matched) {
-                const char = remaining[0];
-                const lastToken = tokens[tokens.length - 1];
-                if (lastToken && lastToken.type === 'text' && lastToken.end === pos) {
-                    lastToken.text += char;
-                    lastToken.end += 1;
-                }
-                else {
-                    tokens.push(createToken('text', char, pos));
-                }
-                pos += 1;
-            }
-        }
-        if (tokens.length === 0) {
-            tokens.push(createToken('text', '', 0));
-        }
-        return { lineNumber, tokens, text: line, state };
-    }
-}
 /**
  * Plaintext tokenizer - no highlighting
  */

package/dist/components/editor/tokenizer/index.d.ts

@@ -15,6 +15,10 @@ export { GoTokenizer, createGoTokenizer } from './languages/go';
 export { MarkdownTokenizer, createMarkdownTokenizer } from './languages/markdown';
 export { SvelteTokenizer, createSvelteTokenizer } from './languages/svelte';
 import type { LanguageTokenizer, Token, TokenizedLine, TokenizerState, TokenType } from './types';
+/**
+ * Tokenizer factory functions by language
+ */
+export declare const tokenizerFactories: Record<string, () => LanguageTokenizer>;
 /**
  * Get the canonical language name from an alias or extension
  */

package/dist/components/editor/tokenizer/index.js

@@ -78,7 +78,7 @@ const languageAliases = {
 /**
  * Tokenizer factory functions by language
  */
-const tokenizerFactories = {
+export const tokenizerFactories = {
     javascript: createJavaScriptTokenizer,
     typescript: createTypeScriptTokenizer,
     jsx: createJSXTokenizer,

package/dist/components/editor/tokenizer/languages/html.d.ts

@@ -3,14 +3,15 @@
  */
 import type { TokenizedLine, TokenizerState } from '../types';
 interface HTMLTokenizerState extends TokenizerState {
-    inTag?: boolean;
     inScript?: boolean;
     inStyle?: boolean;
-    tagName?: string;
+    innerState?: TokenizerState;
 }
 export declare class HTMLTokenizer {
     language: string;
     private isXML;
+    private jsTokenizer;
+    private cssTokenizer;
     constructor(options?: {
         xml?: boolean;
     });