ctxman 3.0.0

package/lib/plugins/PluginManager.js

@@ -0,0 +1,302 @@
+/**
+ * PluginManager - Plugin System Core
+ * v3.0.0 - Plugin architecture
+ *
+ * Responsibilities:
+ * - Load and manage plugins
+ * - Plugin lifecycle management
+ * - Plugin dependency resolution
+ * - Event-driven plugin communication
+ */
+
+import fs from 'fs';
+import path from 'path';
+import { EventEmitter } from 'events';
+import { getLogger } from '../utils/logger.js';
+
+const logger = getLogger('PluginManager');
+
+export class PluginManager extends EventEmitter {
+  constructor(options = {}) {
+    super();
+
+    this.options = {
+      pluginPaths: [
+        './lib/languages',
+        './lib/exporters',
+        './plugins'
+      ],
+      autoLoad: true,
+      lazy: true, // Lazy load plugins
+      ...options
+    };
+
+    this.plugins = new Map();
+    this.loaded = new Map();
+    this.stats = {
+      registered: 0,
+      loaded: 0,
+      errors: 0
+    };
+  }
+
+  /**
+   * Initialize plugin manager
+   */
+  async initialize() {
+    logger.info('Initializing plugin manager...');
+
+    if (this.options.autoLoad) {
+      await this.discoverPlugins();
+    }
+
+    this.emit('initialized');
+  }
+
+  /**
+   * Discover plugins from plugin paths
+   */
+  async discoverPlugins() {
+    for (const pluginPath of this.options.pluginPaths) {
+      const fullPath = path.resolve(pluginPath);
+
+      if (!fs.existsSync(fullPath)) {
+        logger.debug(`Plugin path not found: ${pluginPath}`);
+        continue;
+      }
+
+      await this.discoverFromDirectory(fullPath);
+    }
+
+    logger.info(`Discovered ${this.plugins.size} plugins`);
+  }
+
+  /**
+   * Discover plugins from a directory
+   * @param {string} dirPath
+   */
+  async discoverFromDirectory(dirPath) {
+    try {
+      const entries = fs.readdirSync(dirPath, { withFileTypes: true });
+
+      for (const entry of entries) {
+        if (entry.isFile() && entry.name.endsWith('.js')) {
+          const pluginPath = path.join(dirPath, entry.name);
+          await this.registerPluginFromFile(pluginPath);
+        }
+      }
+    } catch (error) {
+      logger.error(`Error discovering plugins in ${dirPath}: ${error.message}`);
+    }
+  }
+
+  /**
+   * Register plugin from file
+   * @param {string} pluginPath
+   */
+  async registerPluginFromFile(pluginPath) {
+    try {
+      // Register without loading (lazy)
+      const pluginName = path.basename(pluginPath, '.js');
+
+      this.plugins.set(pluginName, {
+        name: pluginName,
+        path: pluginPath,
+        loaded: false,
+        instance: null
+      });
+
+      this.stats.registered++;
+      logger.debug(`Registered plugin: ${pluginName}`);
+
+    } catch (error) {
+      logger.error(`Failed to register plugin ${pluginPath}: ${error.message}`);
+      this.stats.errors++;
+    }
+  }
+
+  /**
+   * Load a plugin
+   * @param {string} pluginName
+   * @returns {Promise<Plugin>}
+   */
+  async load(pluginName) {
+    const plugin = this.plugins.get(pluginName);
+
+    if (!plugin) {
+      throw new Error(`Plugin not found: ${pluginName}`);
+    }
+
+    // Return cached instance if already loaded
+    if (plugin.loaded && plugin.instance) {
+      logger.debug(`Plugin already loaded: ${pluginName}`);
+      return plugin.instance;
+    }
+
+    // Load plugin
+    logger.info(`Loading plugin: ${pluginName}`);
+    const startTime = Date.now();
+
+    try {
+      const module = await import(plugin.path);
+      const PluginClass = module.default || module[Object.keys(module)[0]];
+
+      let instance;
+      if (typeof PluginClass === 'function') {
+        instance = new PluginClass();
+      } else {
+        instance = PluginClass;
+      }
+
+      plugin.instance = instance;
+      plugin.loaded = true;
+
+      const elapsed = Date.now() - startTime;
+      this.stats.loaded++;
+
+      logger.info(`Plugin loaded: ${pluginName} (${elapsed}ms)`);
+      this.emit('plugin:loaded', { name: pluginName, elapsed });
+
+      return instance;
+
+    } catch (error) {
+      logger.error(`Failed to load plugin ${pluginName}: ${error.message}`);
+      this.stats.errors++;
+      this.emit('plugin:error', { name: pluginName, error });
+      throw error;
+    }
+  }
+
+  /**
+   * Load multiple plugins
+   * @param {Array<string>} pluginNames
+   * @returns {Promise<Array<Plugin>>}
+   */
+  async loadMultiple(pluginNames) {
+    const promises = pluginNames.map(name => this.load(name));
+    return Promise.all(promises);
+  }
+
+  /**
+   * Get loaded plugin
+   * @param {string} pluginName
+   * @returns {Plugin|null}
+   */
+  get(pluginName) {
+    const plugin = this.plugins.get(pluginName);
+    return plugin?.instance || null;
+  }
+
+  /**
+   * Check if plugin is loaded
+   * @param {string} pluginName
+   * @returns {boolean}
+   */
+  isLoaded(pluginName) {
+    const plugin = this.plugins.get(pluginName);
+    return plugin?.loaded || false;
+  }
+
+  /**
+   * Unload a plugin
+   * @param {string} pluginName
+   */
+  unload(pluginName) {
+    const plugin = this.plugins.get(pluginName);
+
+    if (!plugin || !plugin.loaded) {
+      return;
+    }
+
+    // Call cleanup if available
+    if (plugin.instance && typeof plugin.instance.cleanup === 'function') {
+      plugin.instance.cleanup();
+    }
+
+    plugin.loaded = false;
+    plugin.instance = null;
+    this.stats.loaded--;
+
+    logger.info(`Plugin unloaded: ${pluginName}`);
+    this.emit('plugin:unloaded', { name: pluginName });
+  }
+
+  /**
+   * Unload all plugins
+   */
+  unloadAll() {
+    for (const [name, plugin] of this.plugins) {
+      if (plugin.loaded) {
+        this.unload(name);
+      }
+    }
+  }
+
+  /**
+   * List all registered plugins
+   * @returns {Array<string>}
+   */
+  list() {
+    return Array.from(this.plugins.keys());
+  }
+
+  /**
+   * List loaded plugins
+   * @returns {Array<string>}
+   */
+  listLoaded() {
+    return Array.from(this.plugins.entries())
+      .filter(([, plugin]) => plugin.loaded)
+      .map(([name]) => name);
+  }
+
+  /**
+   * Get plugin info
+   * @param {string} pluginName
+   * @returns {object|null}
+   */
+  getInfo(pluginName) {
+    const plugin = this.plugins.get(pluginName);
+
+    if (!plugin) {
+      return null;
+    }
+
+    return {
+      name: plugin.name,
+      loaded: plugin.loaded,
+      path: plugin.path,
+      hasInstance: plugin.instance !== null
+    };
+  }
+
+  /**
+   * Get statistics
+   * @returns {object}
+   */
+  getStats() {
+    return { ...this.stats };
+  }
+
+  /**
+   * Register plugin programmatically
+   * @param {string} name
+   * @param {Plugin} instance
+   */
+  register(name, instance) {
+    this.plugins.set(name, {
+      name,
+      path: null,
+      loaded: true,
+      instance
+    });
+
+    this.stats.registered++;
+    this.stats.loaded++;
+
+    logger.info(`Plugin registered: ${name}`);
+    this.emit('plugin:registered', { name });
+  }
+}
+
+export default PluginManager;

package/lib/rag/DataSourcePlugin.js

@@ -0,0 +1,31 @@
+/**
+ * Base class for all data source plugins (e.g., GitHub, Slack, Linear)
+ */
+export class DataSourcePlugin {
+  /**
+   * @param {string} name - Unique name of the data source
+   */
+  constructor(name) {
+    this.name = name;
+  }
+
+  /**
+   * Validates if the necessary configuration (tokens, urls) is present.
+   * @param {Object} config 
+   * @returns {Promise<boolean>}
+   */
+  async validateConfig(config) {
+    return true;
+  }
+
+  /**
+   * Collects data from the source.
+   * Generator function that yields documents to be indexed.
+   * 
+   * @param {Object} options 
+   * @yields {import('./Indexer').IndexableDocument}
+   */
+  async *collect(options = {}) {
+    throw new Error('Not implemented');
+  }
+}

package/lib/rag/EmbeddingProvider.js

@@ -0,0 +1,75 @@
+import { pipeline } from '@xenova/transformers';
+
+/**
+ * Abstract base class for embedding providers
+ */
+export class EmbeddingProvider {
+  /**
+   * Generates an embedding for the given text.
+   * @param {string} text 
+   * @returns {Promise<number[]>}
+   */
+  async embed(text) {
+    throw new Error('Not implemented');
+  }
+}
+
+/**
+ * Uses @xenova/transformers to generate embeddings locally using ONNX models.
+ * Default model: Xenova/all-MiniLM-L6-v2 (384 dimensions)
+ */
+export class TransformersEmbeddingProvider extends EmbeddingProvider {
+  constructor(modelName = 'Xenova/all-MiniLM-L6-v2') {
+    super();
+    this.modelName = modelName;
+    this.pipe = null;
+  }
+
+  async init() {
+    if (!this.pipe) {
+      // Load the pipeline. This will download the model if not present.
+      this.pipe = await pipeline('feature-extraction', this.modelName);
+    }
+  }
+
+  async embed(text) {
+    if (!this.pipe) {
+      await this.init();
+    }
+    // pooling: 'mean' and normalize: true are standard for sentence embeddings
+    const result = await this.pipe(text, { pooling: 'mean', normalize: true });
+    return Array.from(result.data);
+  }
+}
+
+/**
+ * Mock provider for testing without loading heavy models.
+ */
+export class MockEmbeddingProvider extends EmbeddingProvider {
+  constructor(dimension = 384) {
+    super();
+    this.dimension = dimension;
+  }
+
+  async embed(text) {
+    // Generate a deterministic-ish vector based on text length for consistent testing
+    // or just random if text is ignored.
+    // Let's make it pseudo-random based on text hash to be stable.
+    let hash = 0;
+    for (let i = 0; i < text.length; i++) {
+      hash = ((hash << 5) - hash) + text.charCodeAt(i);
+      hash |= 0;
+    }
+    
+    const vector = [];
+    for (let i = 0; i < this.dimension; i++) {
+      // Simple pseudo-random generator
+      const val = Math.sin(hash + i);
+      vector.push(val);
+    }
+
+    // Normalize
+    const norm = Math.sqrt(vector.reduce((sum, val) => sum + val * val, 0));
+    return vector.map(val => val / norm);
+  }
+}

package/lib/rag/Indexer.js

@@ -0,0 +1,83 @@
+import { getLogger } from '../utils/logger.js';
+
+const logger = getLogger('Indexer');
+
+/**
+ * @typedef {Object} IndexableDocument
+ * @property {string} text - Content to embed
+ * @property {string} id - Unique ID from source
+ * @property {Object} metadata - Additional context (source, author, url, etc.)
+ */
+
+export class Indexer {
+  /**
+   * @param {import('./stores/BaseVectorStore').BaseVectorStore} vectorStore 
+   */
+  constructor(vectorStore) {
+    this.vectorStore = vectorStore;
+    this.sources = new Map();
+  }
+
+  /**
+   * Register a data source plugin
+   * @param {import('./DataSourcePlugin').DataSourcePlugin} sourcePlugin 
+   */
+  registerSource(sourcePlugin) {
+    this.sources.set(sourcePlugin.name, sourcePlugin);
+    logger.info(`Registered data source: ${sourcePlugin.name}`);
+  }
+
+  /**
+   * Run indexing for a specific source or all sources
+   * @param {string} [sourceName] - Optional source name to run specific indexer
+   * @param {Object} options - Options passed to collect method
+   */
+  async index(sourceName, options = {}) {
+    const sourcesToRun = sourceName 
+      ? [this.sources.get(sourceName)].filter(Boolean)
+      : Array.from(this.sources.values());
+
+    if (sourcesToRun.length === 0) {
+      logger.warn(`No sources found to index (requested: ${sourceName || 'all'})`);
+      return;
+    }
+
+    for (const source of sourcesToRun) {
+      logger.info(`Starting indexing for source: ${source.name}`);
+      let count = 0;
+      const batch = [];
+      const BATCH_SIZE = 10; // Process in small batches
+
+      try {
+        for await (const doc of source.collect(options)) {
+          batch.push({
+            text: doc.text,
+            metadata: {
+              ...doc.metadata,
+              sourceId: doc.id,
+              source: source.name,
+              indexedAt: new Date().toISOString()
+            }
+          });
+
+          if (batch.length >= BATCH_SIZE) {
+            await this.vectorStore.addDocuments([...batch]);
+            count += batch.length;
+            batch.length = 0; // Clear batch
+            logger.debug(`Indexed ${count} documents from ${source.name}`);
+          }
+        }
+
+        // Flush remaining
+        if (batch.length > 0) {
+          await this.vectorStore.addDocuments(batch);
+          count += batch.length;
+        }
+
+        logger.info(`Completed indexing ${source.name}. Total documents: ${count}`);
+      } catch (error) {
+        logger.error(`Error indexing ${source.name}: ${error.message}`);
+      }
+    }
+  }
+}

package/lib/rag/VectorStoreFactory.js

@@ -0,0 +1,24 @@
+import { LanceDBStore } from './stores/LanceDBStore.js';
+import { ChromaDBStore } from './stores/ChromaDBStore.js';
+import { PineconeStore } from './stores/PineconeStore.js';
+
+export class VectorStoreFactory {
+  /**
+   * @param {string} type 
+   * @param {import('./EmbeddingProvider').EmbeddingProvider} embeddingProvider 
+   * @param {Object} config 
+   */
+  static create(type, embeddingProvider, config = {}) {
+    switch (type.toLowerCase()) {
+      case 'lancedb':
+        return new LanceDBStore(embeddingProvider, config.path);
+      case 'chroma':
+      case 'chromadb':
+        return new ChromaDBStore(embeddingProvider, config);
+      case 'pinecone':
+        return new PineconeStore(embeddingProvider, config);
+      default:
+        throw new Error(`Unknown VectorStore type: ${type}`);
+    }
+  }
+}

package/lib/rag/sources/FileDataSource.js

@@ -0,0 +1,70 @@
+import fs from 'fs/promises';
+import path from 'path';
+import { DataSourcePlugin } from '../DataSourcePlugin.js';
+import { TokenizerManager } from '../../utils/tokenizer-adapter.js';
+
+export class FileDataSource extends DataSourcePlugin {
+  constructor() {
+    super('local-files');
+    this.tokenizer = new TokenizerManager();
+  }
+
+  async validateConfig(config) {
+    // Requires a target directory
+    return !!config.targetDir;
+  }
+
+  async *collect(options = {}) {
+    const targetDir = options.targetDir || process.cwd();
+    const maxDepth = options.maxDepth || 5;
+    const include = options.include || ['.js', '.md', '.ts', '.json'];
+    const exclude = options.exclude || ['node_modules', '.git', 'dist', 'coverage'];
+
+    yield* this.walkDirectory(targetDir, maxDepth, include, exclude);
+  }
+
+  async *walkDirectory(dir, depth, include, exclude) {
+    if (depth < 0) return;
+
+    let entries;
+    try {
+      entries = await fs.readdir(dir, { withFileTypes: true });
+    } catch (error) {
+      console.warn(`[FileDataSource] Skipping ${dir}: ${error.message}`);
+      return;
+    }
+
+    for (const entry of entries) {
+      const fullPath = path.join(dir, entry.name);
+      
+      if (exclude.some(ex => fullPath.includes(ex))) continue;
+
+      if (entry.isDirectory()) {
+        yield* this.walkDirectory(fullPath, depth - 1, include, exclude);
+      } else if (entry.isFile()) {
+        const ext = path.extname(entry.name);
+        if (include.length > 0 && !include.includes(ext)) continue;
+
+        try {
+          const content = await fs.readFile(fullPath, 'utf-8');
+          // Basic text check (skip binaries)
+          if (content.includes('\0')) continue; 
+
+          yield {
+            id: fullPath,
+            text: content,
+            metadata: {
+              source: 'file-system',
+              path: fullPath,
+              extension: ext,
+              filename: entry.name,
+              size: content.length
+            }
+          };
+        } catch (error) {
+           console.warn(`[FileDataSource] Failed to read ${fullPath}: ${error.message}`);
+        }
+      }
+    }
+  }
+}

package/lib/rag/stores/BaseVectorStore.js

@@ -0,0 +1,43 @@
+/**
+ * @typedef {Object} VectorDocument
+ * @property {string} text
+ * @property {Array<number>} vector
+ * @property {string} id
+ * @property {Object} metadata
+ */
+
+/**
+ * @typedef {Object} SearchResult
+ * @property {string} text
+ * @property {number} score
+ * @property {Object} metadata
+ */
+
+export class BaseVectorStore {
+  /**
+   * @param {import('../EmbeddingProvider').EmbeddingProvider} embeddingProvider
+   */
+  constructor(embeddingProvider) {
+    this.embeddingProvider = embeddingProvider;
+  }
+
+  async init() {
+    throw new Error('Method not implemented');
+  }
+
+  /**
+   * @param {Array<{text: string, metadata: Object}>} docs 
+   */
+  async addDocuments(docs) {
+    throw new Error('Method not implemented');
+  }
+
+  /**
+   * @param {string} queryText 
+   * @param {number} limit 
+   * @returns {Promise<Array<SearchResult>>}
+   */
+  async search(queryText, limit = 5) {
+    throw new Error('Method not implemented');
+  }
+}

package/lib/rag/stores/ChromaDBStore.js

@@ -0,0 +1,85 @@
+import { BaseVectorStore } from './BaseVectorStore.js';
+
+export class ChromaDBStore extends BaseVectorStore {
+  /**
+   * @param {import('../EmbeddingProvider').EmbeddingProvider} embeddingProvider
+   * @param {Object} config
+   * @param {string} [config.collectionName]
+   * @param {string} [config.url]
+   */
+  constructor(embeddingProvider, config = {}) {
+    super(embeddingProvider);
+    this.collectionName = config.collectionName || 'ctxman-docs';
+    this.url = config.url || 'http://localhost:8000';
+    this.client = null;
+    this.collection = null;
+  }
+
+  async init() {
+    if (this.client) return;
+
+    try {
+      const { ChromaClient } = await import('chromadb');
+      this.client = new ChromaClient({ path: this.url });
+      
+      this.collection = await this.client.getOrCreateCollection({
+        name: this.collectionName,
+      });
+    } catch (error) {
+      if (error.code === 'ERR_MODULE_NOT_FOUND') {
+        throw new Error('Please install chromadb to use ChromaDBStore: npm install chromadb');
+      }
+      throw error;
+    }
+  }
+
+  async addDocuments(docs) {
+    if (!this.collection) await this.init();
+    if (docs.length === 0) return;
+
+    const ids = docs.map(d => d.id || crypto.randomUUID());
+    const metadatas = docs.map(d => d.metadata || {});
+    const documents = docs.map(d => d.text);
+    
+    // Chroma can handle embeddings, but we have a provider.
+    // If we want to use our provider:
+    const embeddings = [];
+    for (const doc of docs) {
+        embeddings.push(await this.embeddingProvider.embed(doc.text));
+    }
+
+    await this.collection.add({
+      ids,
+      embeddings,
+      metadatas,
+      documents
+    });
+  }
+
+  async search(queryText, limit = 5) {
+    if (!this.collection) await this.init();
+
+    const queryEmbedding = await this.embeddingProvider.embed(queryText);
+
+    const results = await this.collection.query({
+      queryEmbeddings: [queryEmbedding],
+      nResults: limit
+    });
+
+    // Chroma returns arrays of arrays
+    if (!results.ids || results.ids.length === 0) return [];
+
+    const hits = [];
+    const count = results.ids[0].length;
+    
+    for (let i = 0; i < count; i++) {
+      hits.push({
+        text: results.documents[0][i],
+        score: results.distances ? results.distances[0][i] : 0,
+        metadata: results.metadatas[0][i]
+      });
+    }
+
+    return hits;
+  }
+}