@kroy665/code-editor-engine 1.0.0

package/dist/language/Extensions.js

@@ -0,0 +1,368 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ExtensibleEditor = exports.BuiltInExtensions = exports.ExtensionLoader = exports.ExtensionHost = exports.ActivationEvent = void 0;
+const EventSystem_1 = require("../core/EventSystem");
+const CodeEditor_1 = require("../core/CodeEditor");
+/**
+ * Extension activation events
+ */
+var ActivationEvent;
+(function (ActivationEvent) {
+    ActivationEvent["OnStartup"] = "onStartup";
+    ActivationEvent["OnLanguage"] = "onLanguage";
+    ActivationEvent["OnCommand"] = "onCommand";
+    ActivationEvent["OnFileSystem"] = "onFileSystem";
+    ActivationEvent["OnDebug"] = "onDebug";
+    ActivationEvent["OnView"] = "onView";
+})(ActivationEvent || (exports.ActivationEvent = ActivationEvent = {}));
+/**
+ * Extension state storage
+ */
+class ExtensionMemento {
+    constructor() {
+        this.storage = new Map();
+    }
+    get(key, defaultValue) {
+        const value = this.storage.get(key);
+        return value !== undefined ? value : defaultValue;
+    }
+    async update(key, value) {
+        if (value === undefined) {
+            this.storage.delete(key);
+        }
+        else {
+            this.storage.set(key, value);
+        }
+    }
+    keys() {
+        return Array.from(this.storage.keys());
+    }
+    clear() {
+        this.storage.clear();
+    }
+}
+/**
+ * Extension context implementation
+ */
+class ExtensionContextImpl {
+    constructor(extensionId, editor) {
+        this.extensionId = extensionId;
+        this.editor = editor;
+        this.disposables = new EventSystem_1.CompositeDisposable();
+        this.globalState = new ExtensionMemento();
+        this.workspaceState = new ExtensionMemento();
+    }
+    registerCommand(_commandId, command) {
+        const disposable = this.editor.registerCommand(command);
+        this.disposables.add(disposable);
+        return disposable;
+    }
+    registerLanguageService(service) {
+        const disposable = this.editor.registerLanguageService(service);
+        this.disposables.add(disposable);
+        return disposable;
+    }
+    registerTextDocumentProvider(_scheme, _provider) {
+        // In a full implementation, this would register with a document provider registry
+        const disposable = new EventSystem_1.DisposableImpl(() => {
+            // Cleanup provider registration
+        });
+        this.disposables.add(disposable);
+        return disposable;
+    }
+    dispose() {
+        this.disposables.dispose();
+    }
+}
+/**
+ * Extension host for managing extension lifecycle
+ */
+class ExtensionHost extends EventSystem_1.TypedEventEmitter {
+    constructor(editor) {
+        super();
+        this.editor = editor;
+        this.extensions = new Map();
+        this.contexts = new Map();
+        this.activatedExtensions = new Set();
+        this.disposables = new EventSystem_1.CompositeDisposable();
+    }
+    /**
+     * Register an extension
+     */
+    async registerExtension(extension) {
+        if (this.extensions.has(extension.id)) {
+            throw new Error(`Extension with id '${extension.id}' is already registered`);
+        }
+        this.extensions.set(extension.id, extension);
+        // Create extension context
+        const context = new ExtensionContextImpl(extension.id, this.editor);
+        this.contexts.set(extension.id, context);
+        // Auto-activate if no activation events specified
+        if (!extension.activationEvents || extension.activationEvents.length === 0) {
+            await this.activateExtension(extension.id);
+        }
+    }
+    /**
+     * Activate an extension
+     */
+    async activateExtension(extensionId) {
+        if (this.activatedExtensions.has(extensionId)) {
+            return; // Already activated
+        }
+        const extension = this.extensions.get(extensionId);
+        if (!extension) {
+            throw new Error(`Extension '${extensionId}' not found`);
+        }
+        const context = this.contexts.get(extensionId);
+        if (!context) {
+            throw new Error(`Context for extension '${extensionId}' not found`);
+        }
+        try {
+            await extension.activate(context);
+            this.activatedExtensions.add(extensionId);
+            this.emit('extension-activated', { extension });
+        }
+        catch (error) {
+            this.emit('extension-error', { extension, error: error });
+            throw error;
+        }
+    }
+    /**
+     * Deactivate an extension
+     */
+    async deactivateExtension(extensionId) {
+        if (!this.activatedExtensions.has(extensionId)) {
+            return; // Not activated
+        }
+        const extension = this.extensions.get(extensionId);
+        if (!extension) {
+            return;
+        }
+        const context = this.contexts.get(extensionId);
+        if (context) {
+            context.dispose();
+        }
+        try {
+            if (extension.deactivate) {
+                await extension.deactivate();
+            }
+            this.activatedExtensions.delete(extensionId);
+            this.emit('extension-deactivated', { extension });
+        }
+        catch (error) {
+            this.emit('extension-error', { extension, error: error });
+            throw error;
+        }
+    }
+    /**
+     * Get an extension by ID
+     */
+    getExtension(extensionId) {
+        return this.extensions.get(extensionId);
+    }
+    /**
+     * Get all registered extensions
+     */
+    getAllExtensions() {
+        return Array.from(this.extensions.values());
+    }
+    /**
+     * Get all activated extensions
+     */
+    getActivatedExtensions() {
+        return Array.from(this.activatedExtensions)
+            .map((id) => this.extensions.get(id))
+            .filter((ext) => ext !== undefined);
+    }
+    /**
+     * Check if an extension is activated
+     */
+    isActivated(extensionId) {
+        return this.activatedExtensions.has(extensionId);
+    }
+    /**
+     * Activate extensions based on event
+     */
+    async activateByEvent(event) {
+        const toActivate = [];
+        for (const [id, extension] of this.extensions) {
+            if (this.activatedExtensions.has(id)) {
+                continue; // Already activated
+            }
+            if (extension.activationEvents?.includes(event)) {
+                toActivate.push(id);
+            }
+        }
+        // Activate extensions in parallel
+        await Promise.allSettled(toActivate.map((id) => this.activateExtension(id)));
+    }
+    /**
+     * Dispose the extension host
+     */
+    dispose() {
+        // Deactivate all extensions
+        const activatedIds = Array.from(this.activatedExtensions);
+        for (const id of activatedIds) {
+            try {
+                this.deactivateExtension(id);
+            }
+            catch (error) {
+                console.error(`Error deactivating extension ${id}:`, error);
+            }
+        }
+        this.disposables.dispose();
+        super.dispose();
+    }
+}
+exports.ExtensionHost = ExtensionHost;
+/**
+ * Extension loader for loading extensions from manifests
+ */
+class ExtensionLoader {
+    constructor(extensionHost) {
+        this.extensionHost = extensionHost;
+    } // eslint-disable-line @typescript-eslint/no-unused-vars
+    /**
+     * Load extension from manifest
+     */
+    async loadFromManifest(manifest, extensionCode) {
+        // Validate manifest
+        this.validateManifest(manifest);
+        // Enhance extension with manifest data
+        const enhancedExtension = {
+            ...extensionCode,
+            id: manifest.id,
+            name: manifest.name,
+            version: manifest.version,
+            ...(manifest.description && { description: manifest.description }),
+            ...(manifest.activationEvents && { activationEvents: manifest.activationEvents }),
+        };
+        // Register with extension host
+        await this.extensionHost.registerExtension(enhancedExtension);
+    }
+    /**
+     * Load multiple extensions
+     */
+    async loadExtensions(extensions) {
+        // Sort by dependencies (simplified - in production you'd need proper dependency resolution)
+        const sorted = this.sortByDependencies(extensions);
+        for (const { manifest, code } of sorted) {
+            try {
+                await this.loadFromManifest(manifest, code);
+            }
+            catch (error) {
+                console.error(`Failed to load extension ${manifest.id}:`, error);
+            }
+        }
+    }
+    validateManifest(manifest) {
+        if (!manifest.id) {
+            throw new Error('Extension manifest must have an id');
+        }
+        if (!manifest.name) {
+            throw new Error('Extension manifest must have a name');
+        }
+        if (!manifest.version) {
+            throw new Error('Extension manifest must have a version');
+        }
+        // Validate semver
+        if (!/^\d+\.\d+\.\d+/.test(manifest.version)) {
+            throw new Error('Extension version must be a valid semver');
+        }
+    }
+    sortByDependencies(extensions) {
+        // Simplified dependency sorting - in production you'd implement proper topological sort
+        return extensions.sort((a, b) => {
+            const aDeps = Object.keys(a.manifest.dependencies || {}).length;
+            const bDeps = Object.keys(b.manifest.dependencies || {}).length;
+            return aDeps - bDeps;
+        });
+    }
+}
+exports.ExtensionLoader = ExtensionLoader;
+/**
+ * Built-in extensions
+ */
+class BuiltInExtensions {
+    /**
+     * Create a basic language support extension
+     */
+    static createLanguageExtension(languageId, languageName, languageService) {
+        return {
+            id: `builtin.${languageId}`,
+            name: `${languageName} Language Support`,
+            version: '1.0.0',
+            description: `Built-in ${languageName} language support`,
+            activationEvents: [`onLanguage:${languageId}`],
+            async activate(context) {
+                context.registerLanguageService(languageService);
+            },
+        };
+    }
+    /**
+     * Create a basic commands extension
+     */
+    static createCommandsExtension(commands) {
+        return {
+            id: 'builtin.commands',
+            name: 'Built-in Commands',
+            version: '1.0.0',
+            description: 'Essential editor commands',
+            async activate(context) {
+                for (const command of commands) {
+                    context.registerCommand(command.id, command);
+                }
+            },
+        };
+    }
+    /**
+     * Create a basic themes extension
+     */
+    static createThemesExtension() {
+        return {
+            id: 'builtin.themes',
+            name: 'Built-in Themes',
+            version: '1.0.0',
+            description: 'Default editor themes',
+            async activate(_context) {
+                // In a full implementation, this would register theme providers
+                console.log('Themes extension activated');
+            },
+        };
+    }
+}
+exports.BuiltInExtensions = BuiltInExtensions;
+/**
+ * Factory for creating extension-enabled editors
+ */
+class ExtensibleEditor {
+    static create(options = {}) {
+        const editor = new CodeEditor_1.CodeEditorEngine(options.editorOptions);
+        const extensionHost = new ExtensionHost(editor);
+        // Load built-in extensions if requested
+        if (options.builtInExtensions !== false) {
+            ExtensibleEditor.loadBuiltInExtensions(extensionHost);
+        }
+        // Load provided extensions
+        if (options.extensions) {
+            for (const extension of options.extensions) {
+                extensionHost.registerExtension(extension);
+            }
+        }
+        return { editor, extensionHost };
+    }
+    static async loadBuiltInExtensions(extensionHost) {
+        // Load built-in language extensions
+        const { TokenizerLanguageService } = require('./Tokenizer');
+        const jsExtension = BuiltInExtensions.createLanguageExtension('javascript', 'JavaScript', new TokenizerLanguageService('javascript'));
+        const tsExtension = BuiltInExtensions.createLanguageExtension('typescript', 'TypeScript', new TokenizerLanguageService('typescript'));
+        const pyExtension = BuiltInExtensions.createLanguageExtension('python', 'Python', new TokenizerLanguageService('python'));
+        await extensionHost.registerExtension(jsExtension);
+        await extensionHost.registerExtension(tsExtension);
+        await extensionHost.registerExtension(pyExtension);
+        // Activate language extensions
+        await extensionHost.activateByEvent('onStartup');
+    }
+}
+exports.ExtensibleEditor = ExtensibleEditor;
+//# sourceMappingURL=Extensions.js.map

package/dist/language/Extensions.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"Extensions.js","sourceRoot":"","sources":["../../src/language/Extensions.ts"],"names":[],"mappings":";;;AAUA,qDAA6F;AAC7F,mDAAsD;AAEtD;;GAEG;AACH,IAAY,eAOX;AAPD,WAAY,eAAe;IACvB,0CAAuB,CAAA;IACvB,4CAAyB,CAAA;IACzB,0CAAuB,CAAA;IACvB,gDAA6B,CAAA;IAC7B,sCAAmB,CAAA;IACnB,oCAAiB,CAAA;AACrB,CAAC,EAPW,eAAe,+BAAf,eAAe,QAO1B;AAuJD;;GAEG;AACH,MAAM,gBAAgB;IAAtB;QACY,YAAO,GAAG,IAAI,GAAG,EAAe,CAAC;IAwB7C,CAAC;IApBG,GAAG,CAAI,GAAW,EAAE,YAAgB;QAChC,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACpC,OAAO,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC;IACtD,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,GAAW,EAAE,KAAU;QAChC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACtB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC7B,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QACjC,CAAC;IACL,CAAC;IAED,IAAI;QACA,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IAC3C,CAAC;IAED,KAAK;QACD,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;IACzB,CAAC;CACJ;AAED;;GAEG;AACH,MAAM,oBAAoB;IAKtB,YACoB,WAAmB,EAClB,MAAkB;QADnB,gBAAW,GAAX,WAAW,CAAQ;QAClB,WAAM,GAAN,MAAM,CAAY;QAJtB,gBAAW,GAAG,IAAI,iCAAmB,EAAE,CAAC;QAMrD,IAAI,CAAC,WAAW,GAAG,IAAI,gBAAgB,EAAE,CAAC;QAC1C,IAAI,CAAC,cAAc,GAAG,IAAI,gBAAgB,EAAE,CAAC;IACjD,CAAC;IAED,eAAe,CAAC,UAAkB,EAAE,OAAgB;QAChD,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;QACxD,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QACjC,OAAO,UAAU,CAAC;IACtB,CAAC;IAED,uBAAuB,CAAC,OAAwB;QAC5C,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,uBAAuB,CAAC,OAAO,CAAC,CAAC;QAChE,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QACjC,OAAO,UAAU,CAAC;IACtB,CAAC;IAED,4BAA4B,CAAC,OAAe,EAAE,SAA+B;QACzE,kFAAkF;QAClF,MAAM,UAAU,GAAG,IAAI,4BAAc,CAAC,GAAG,EAAE;YACvC,gCAAgC;QACpC,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QACjC,OAAO,UAAU,CAAC;IACtB,CAAC;IAED,OAAO;QACH,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;IAC/B,CAAC;CACJ;AAED;;GAEG;AACH,MAAa,aAAc,SAAQ,+BAIjC;IAME,YAA6B,MAAkB;QAC3C,KAAK,EAAE,CAAC;QADiB,WAAM,GAAN,MAAM,CAAY;QAL9B,eAAU,GAAG,IAAI,GAAG,EAAqB,CAAC;QAC1C,aAAQ,GAAG,IAAI,GAAG,EAAgC,CAAC;QACnD,wBAAmB,GAAG,IAAI,GAAG,EAAU,CAAC;QACxC,gBAAW,GAAG,IAAI,iCAAmB,EAAE,CAAC;IAIzD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,iBAAiB,CAAC,SAAoB;QACxC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,CAAC;YACpC,MAAM,IAAI,KAAK,CAAC,sBAAsB,SAAS,CAAC,EAAE,yBAAyB,CAAC,CAAC;QACjF,CAAC;QAED,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;QAE7C,2BAA2B;QAC3B,MAAM,OAAO,GAAG,IAAI,oBAAoB,CAAC,SAAS,CAAC,EAAE,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QACpE,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;QAEzC,kDAAkD;QAClD,IAAI,CAAC,SAAS,CAAC,gBAAgB,IAAI,SAAS,CAAC,gBAAgB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzE,MAAM,IAAI,CAAC,iBAAiB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAC/C,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,iBAAiB,CAAC,WAAmB;QACvC,IAAI,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC;YAC5C,OAAO,CAAC,oBAAoB;QAChC,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QACnD,IAAI,CAAC,SAAS,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,cAAc,WAAW,aAAa,CAAC,CAAC;QAC5D,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QAC/C,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,MAAM,IAAI,KAAK,CAAC,0BAA0B,WAAW,aAAa,CAAC,CAAC;QACxE,CAAC;QAED,IAAI,CAAC;YACD,MAAM,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YAClC,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;YAC1C,IAAI,CAAC,IAAI,CAAC,qBAAqB,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;QACpD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,KAAc,EAAE,CAAC,CAAC;YACnE,MAAM,KAAK,CAAC;QAChB,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,mBAAmB,CAAC,WAAmB;QACzC,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC;YAC7C,OAAO,CAAC,gBAAgB;QAC5B,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QACnD,IAAI,CAAC,SAAS,EAAE,CAAC;YACb,OAAO;QACX,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QAC/C,IAAI,OAAO,EAAE,CAAC;YACV,OAAO,CAAC,OAAO,EAAE,CAAC;QACtB,CAAC;QAED,IAAI,CAAC;YACD,IAAI,SAAS,CAAC,UAAU,EAAE,CAAC;gBACvB,MAAM,SAAS,CAAC,UAAU,EAAE,CAAC;YACjC,CAAC;YACD,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;YAC7C,IAAI,CAAC,IAAI,CAAC,uBAAuB,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;QACtD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,KAAc,EAAE,CAAC,CAAC;YACnE,MAAM,KAAK,CAAC;QAChB,CAAC;IACL,CAAC;IAED;;OAEG;IACH,YAAY,CAAC,WAAmB;QAC5B,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;IAC5C,CAAC;IAED;;OAEG;IACH,gBAAgB;QACZ,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;IAChD,CAAC;IAED;;OAEG;IACH,sBAAsB;QAClB,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC;aACtC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;aACpC,MAAM,CAAC,CAAC,GAAG,EAAoB,EAAE,CAAC,GAAG,KAAK,SAAS,CAAC,CAAC;IAC9D,CAAC;IAED;;OAEG;IACH,WAAW,CAAC,WAAmB;QAC3B,OAAO,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;IACrD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,eAAe,CAAC,KAAa;QAC/B,MAAM,UAAU,GAAa,EAAE,CAAC;QAEhC,KAAK,MAAM,CAAC,EAAE,EAAE,SAAS,CAAC,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YAC5C,IAAI,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;gBACnC,SAAS,CAAC,oBAAoB;YAClC,CAAC;YAED,IAAI,SAAS,CAAC,gBAAgB,EAAE,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC9C,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACxB,CAAC;QACL,CAAC;QAED,kCAAkC;QAClC,MAAM,OAAO,CAAC,UAAU,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACjF,CAAC;IAED;;OAEG;IACM,OAAO;QACZ,4BAA4B;QAC5B,MAAM,YAAY,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;QAC1D,KAAK,MAAM,EAAE,IAAI,YAAY,EAAE,CAAC;YAC5B,IAAI,CAAC;gBACD,IAAI,CAAC,mBAAmB,CAAC,EAAE,CAAC,CAAC;YACjC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACb,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC;YAChE,CAAC;QACL,CAAC;QAED,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;QAC3B,KAAK,CAAC,OAAO,EAAE,CAAC;IACpB,CAAC;CACJ;AA/JD,sCA+JC;AAED;;GAEG;AACH,MAAa,eAAe;IACxB,YAA6B,aAA4B;QAA5B,kBAAa,GAAb,aAAa,CAAe;IAAG,CAAC,CAAC,wDAAwD;IAEtH;;OAEG;IACH,KAAK,CAAC,gBAAgB,CAAC,QAA2B,EAAE,aAAwB;QACxE,oBAAoB;QACpB,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QAEhC,uCAAuC;QACvC,MAAM,iBAAiB,GAAc;YACjC,GAAG,aAAa;YAChB,EAAE,EAAE,QAAQ,CAAC,EAAE;YACf,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,OAAO,EAAE,QAAQ,CAAC,OAAO;YACzB,GAAG,CAAC,QAAQ,CAAC,WAAW,IAAI,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,CAAC;YAClE,GAAG,CAAC,QAAQ,CAAC,gBAAgB,IAAI,EAAE,gBAAgB,EAAE,QAAQ,CAAC,gBAAgB,EAAE,CAAC;SACpF,CAAC;QAEF,+BAA+B;QAC/B,MAAM,IAAI,CAAC,aAAa,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,CAAC;IAClE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,cAAc,CAChB,UAAmE;QAEnE,4FAA4F;QAC5F,MAAM,MAAM,GAAG,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,CAAC;QAEnD,KAAK,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,MAAM,EAAE,CAAC;YACtC,IAAI,CAAC;gBACD,MAAM,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;YAChD,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACb,OAAO,CAAC,KAAK,CAAC,4BAA4B,QAAQ,CAAC,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC;YACrE,CAAC;QACL,CAAC;IACL,CAAC;IAEO,gBAAgB,CAAC,QAA2B;QAChD,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;QAC1D,CAAC;QACD,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;QAC3D,CAAC;QACD,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;QAC9D,CAAC;QAED,kBAAkB;QAClB,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;YAC3C,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAChE,CAAC;IACL,CAAC;IAEO,kBAAkB,CACtB,UAAmE;QAEnE,wFAAwF;QACxF,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YAC5B,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;YAChE,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;YAChE,OAAO,KAAK,GAAG,KAAK,CAAC;QACzB,CAAC,CAAC,CAAC;IACP,CAAC;CACJ;AArED,0CAqEC;AAED;;GAEG;AACH,MAAa,iBAAiB;IAC1B;;OAEG;IACH,MAAM,CAAC,uBAAuB,CAC1B,UAAkB,EAClB,YAAoB,EACpB,eAAgC;QAEhC,OAAO;YACH,EAAE,EAAE,WAAW,UAAU,EAAE;YAC3B,IAAI,EAAE,GAAG,YAAY,mBAAmB;YACxC,OAAO,EAAE,OAAO;YAChB,WAAW,EAAE,YAAY,YAAY,mBAAmB;YACxD,gBAAgB,EAAE,CAAC,cAAc,UAAU,EAAE,CAAC;YAE9C,KAAK,CAAC,QAAQ,CAAC,OAAyB;gBACpC,OAAO,CAAC,uBAAuB,CAAC,eAAe,CAAC,CAAC;YACrD,CAAC;SACJ,CAAC;IACN,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,uBAAuB,CAAC,QAAmB;QAC9C,OAAO;YACH,EAAE,EAAE,kBAAkB;YACtB,IAAI,EAAE,mBAAmB;YACzB,OAAO,EAAE,OAAO;YAChB,WAAW,EAAE,2BAA2B;YAExC,KAAK,CAAC,QAAQ,CAAC,OAAyB;gBACpC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;oBAC7B,OAAO,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;gBACjD,CAAC;YACL,CAAC;SACJ,CAAC;IACN,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,qBAAqB;QACxB,OAAO;YACH,EAAE,EAAE,gBAAgB;YACpB,IAAI,EAAE,iBAAiB;YACvB,OAAO,EAAE,OAAO;YAChB,WAAW,EAAE,uBAAuB;YAEpC,KAAK,CAAC,QAAQ,CAAC,QAA0B;gBACrC,gEAAgE;gBAChE,OAAO,CAAC,GAAG,CAAC,4BAA4B,CAAC,CAAC;YAC9C,CAAC;SACJ,CAAC;IACN,CAAC;CACJ;AAxDD,8CAwDC;AAiCD;;GAEG;AACH,MAAa,gBAAgB;IACzB,MAAM,CAAC,MAAM,CACT,UAII,EAAE;QAEN,MAAM,MAAM,GAAG,IAAI,6BAAgB,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;QAC3D,MAAM,aAAa,GAAG,IAAI,aAAa,CAAC,MAAM,CAAC,CAAC;QAEhD,wCAAwC;QACxC,IAAI,OAAO,CAAC,iBAAiB,KAAK,KAAK,EAAE,CAAC;YACtC,gBAAgB,CAAC,qBAAqB,CAAC,aAAa,CAAC,CAAC;QAC1D,CAAC;QAED,2BAA2B;QAC3B,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;YACrB,KAAK,MAAM,SAAS,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;gBACzC,aAAa,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;YAC/C,CAAC;QACL,CAAC;QAED,OAAO,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC;IACrC,CAAC;IAEO,MAAM,CAAC,KAAK,CAAC,qBAAqB,CAAC,aAA4B;QACnE,oCAAoC;QACpC,MAAM,EAAE,wBAAwB,EAAE,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;QAE5D,MAAM,WAAW,GAAG,iBAAiB,CAAC,uBAAuB,CACzD,YAAY,EACZ,YAAY,EACZ,IAAI,wBAAwB,CAAC,YAAY,CAAC,CAC7C,CAAC;QAEF,MAAM,WAAW,GAAG,iBAAiB,CAAC,uBAAuB,CACzD,YAAY,EACZ,YAAY,EACZ,IAAI,wBAAwB,CAAC,YAAY,CAAC,CAC7C,CAAC;QAEF,MAAM,WAAW,GAAG,iBAAiB,CAAC,uBAAuB,CACzD,QAAQ,EACR,QAAQ,EACR,IAAI,wBAAwB,CAAC,QAAQ,CAAC,CACzC,CAAC;QAEF,MAAM,aAAa,CAAC,iBAAiB,CAAC,WAAW,CAAC,CAAC;QACnD,MAAM,aAAa,CAAC,iBAAiB,CAAC,WAAW,CAAC,CAAC;QACnD,MAAM,aAAa,CAAC,iBAAiB,CAAC,WAAW,CAAC,CAAC;QAEnD,+BAA+B;QAC/B,MAAM,aAAa,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC;IACrD,CAAC;CACJ;AAvDD,4CAuDC"}

package/dist/language/Tokenizer.d.ts

@@ -0,0 +1,108 @@
+import { Token, TokenType, Range, TextDocument, LanguageService } from '../types/core';
+/**
+ * Token with additional metadata for highlighting
+ */
+export interface EnhancedToken extends Token {
+    readonly priority: number;
+    readonly scopes: string[];
+}
+/**
+ * Tokenization rule
+ */
+export interface TokenRule {
+    readonly pattern: RegExp;
+    readonly type: TokenType;
+    readonly priority?: number;
+    readonly scopes?: string[];
+    readonly push?: string;
+    readonly pop?: boolean;
+    readonly set?: string;
+}
+/**
+ * Language tokenization definition
+ */
+export interface LanguageDefinition {
+    readonly languageId: string;
+    readonly name: string;
+    readonly extensions: string[];
+    readonly states: Record<string, TokenRule[]>;
+    readonly defaultState: string;
+    readonly brackets?: BracketDefinition[];
+    readonly comments?: CommentDefinition;
+    readonly strings?: StringDefinition[];
+    readonly keywords?: string[];
+    readonly builtins?: string[];
+    readonly operators?: string[];
+}
+export interface BracketDefinition {
+    readonly open: string;
+    readonly close: string;
+    readonly type?: 'round' | 'square' | 'curly' | 'angle';
+}
+export interface CommentDefinition {
+    readonly line?: string;
+    readonly block?: {
+        start: string;
+        end: string;
+    };
+}
+export interface StringDefinition {
+    readonly delimiter: string;
+    readonly escape?: string;
+    readonly multiline?: boolean;
+    readonly raw?: boolean;
+}
+/**
+ * Fast, incremental tokenizer
+ */
+export declare class Tokenizer {
+    private readonly languageDefinitions;
+    private readonly tokenCache;
+    private readonly stateStack;
+    /**
+     * Register a language definition
+     */
+    registerLanguage(definition: LanguageDefinition): void;
+    /**
+     * Get language definition
+     */
+    getLanguage(languageId: string): LanguageDefinition | undefined;
+    /**
+     * Tokenize a document
+     */
+    tokenize(document: TextDocument, range?: Range): EnhancedToken[];
+    /**
+     * Tokenize incrementally after a change
+     */
+    tokenizeIncremental(document: TextDocument, _previousTokens: EnhancedToken[], _change: {
+        range: Range;
+        text: string;
+    }): EnhancedToken[];
+    private tokenizeWithLanguage;
+    private tokenizeLine;
+    private resolveTokenType;
+    private getTokenModifiers;
+    private tokenizeAsPlainText;
+    private getCacheKey;
+    private clearCacheForLanguage;
+    private clearCacheForDocument;
+}
+/**
+ * Built-in language definitions
+ */
+export declare class BuiltInLanguages {
+    static javascript(): LanguageDefinition;
+    static typescript(): LanguageDefinition;
+    static python(): LanguageDefinition;
+}
+/**
+ * Language service that provides tokenization
+ */
+export declare class TokenizerLanguageService implements LanguageService {
+    readonly languageId: string;
+    private tokenizer;
+    constructor(languageId: string);
+    tokenize(document: TextDocument, range?: Range): Promise<Token[]>;
+    registerLanguage(definition: LanguageDefinition): void;
+}
+//# sourceMappingURL=Tokenizer.d.ts.map

package/dist/language/Tokenizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Tokenizer.d.ts","sourceRoot":"","sources":["../../src/language/Tokenizer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAEvF;;GAEG;AACH,MAAM,WAAW,aAAc,SAAQ,KAAK;IACxC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IACzB,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,GAAG,CAAC,EAAE,OAAO,CAAC;IACvB,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAC/B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,UAAU,EAAE,MAAM,EAAE,CAAC;IAC9B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;IAC7C,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,QAAQ,CAAC,EAAE,iBAAiB,EAAE,CAAC;IACxC,QAAQ,CAAC,QAAQ,CAAC,EAAE,iBAAiB,CAAC;IACtC,QAAQ,CAAC,OAAO,CAAC,EAAE,gBAAgB,EAAE,CAAC;IACtC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IAC7B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IAC7B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;CACjC;AAED,MAAM,WAAW,iBAAiB;IAC9B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,GAAG,QAAQ,GAAG,OAAO,GAAG,OAAO,CAAC;CAC1D;AAED,MAAM,WAAW,iBAAiB;IAC9B,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC;CACnD;AAED,MAAM,WAAW,gBAAgB;IAC7B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,SAAS,CAAC,EAAE,OAAO,CAAC;IAC7B,QAAQ,CAAC,GAAG,CAAC,EAAE,OAAO,CAAC;CAC1B;AAYD;;GAEG;AACH,qBAAa,SAAS;IAClB,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAyC;IAC7E,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAsC;IACjE,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAwB;IAEnD;;OAEG;IACH,gBAAgB,CAAC,UAAU,EAAE,kBAAkB,GAAG,IAAI;IAOtD;;OAEG;IACH,WAAW,CAAC,UAAU,EAAE,MAAM,GAAG,kBAAkB,GAAG,SAAS;IAI/D;;OAEG;IACH,QAAQ,CAAC,QAAQ,EAAE,YAAY,EAAE,KAAK,CAAC,EAAE,KAAK,GAAG,aAAa,EAAE;IAkBhE;;OAEG;IACH,mBAAmB,CACf,QAAQ,EAAE,YAAY,EACtB,eAAe,EAAE,aAAa,EAAE,EAChC,OAAO,EAAE;QAAE,KAAK,EAAE,KAAK,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,GACxC,aAAa,EAAE;IAOlB,OAAO,CAAC,oBAAoB;IA6B5B,OAAO,CAAC,YAAY;IA0FpB,OAAO,CAAC,gBAAgB;IAuBxB,OAAO,CAAC,iBAAiB;IAuBzB,OAAO,CAAC,mBAAmB;IAoB3B,OAAO,CAAC,WAAW;IAOnB,OAAO,CAAC,qBAAqB;IAM7B,OAAO,CAAC,qBAAqB;CAOhC;AAED;;GAEG;AACH,qBAAa,gBAAgB;IACzB,MAAM,CAAC,UAAU,IAAI,kBAAkB;IAoKvC,MAAM,CAAC,UAAU,IAAI,kBAAkB;IAiCvC,MAAM,CAAC,MAAM,IAAI,kBAAkB;CA6MtC;AAED;;GAEG;AACH,qBAAa,wBAAyB,YAAW,eAAe;aAGhC,UAAU,EAAE,MAAM;IAF9C,OAAO,CAAC,SAAS,CAAmB;gBAER,UAAU,EAAE,MAAM;IAOxC,QAAQ,CAAC,QAAQ,EAAE,YAAY,EAAE,KAAK,CAAC,EAAE,KAAK,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;IAIvE,gBAAgB,CAAC,UAAU,EAAE,kBAAkB,GAAG,IAAI;CAGzD"}