@sylphx/flow 1.1.0 → 1.2.0

package/src/services/search/knowledge-indexer.ts

@@ -1,422 +0,0 @@
-/**
- * Knowledge indexer implementation
- * Extends BaseIndexer for knowledge base indexing
- *
- * Features:
- * - Auto-indexing on startup (mandatory)
- * - File watching for automatic re-indexing on changes
- * - Debounced re-indexing (2 seconds after last change)
- */
-
-import fs from 'node:fs';
-import path from 'node:path';
-import chokidar from 'chokidar';
-import { logger } from '../../utils/logger.js';
-import { getKnowledgeDir } from '../../utils/paths.js';
-import { VectorStorage } from '../storage/lancedb-vector-storage.js';
-import { BaseIndexer } from './base-indexer.js';
-import type { EmbeddingProvider } from './embeddings.js';
-import { getDefaultEmbeddingProvider } from './embeddings.js';
-import { createIndexer, type Indexer } from './functional-indexer.js';
-import { buildSearchIndex, type SearchIndex } from './tfidf.js';
-
-/**
- * Knowledge indexer singleton
- */
-class KnowledgeIndexer extends BaseIndexer {
-  private embeddingProvider?: EmbeddingProvider;
-  private vectorStorage?: VectorStorage;
-  private watcher?: chokidar.FSWatcher;
-  private reindexTimer?: NodeJS.Timeout;
-
-  constructor(embeddingProvider?: EmbeddingProvider, options?: { autoWatch?: boolean }) {
-    super({ name: 'knowledge' });
-    this.embeddingProvider = embeddingProvider;
-
-    // Start file watching only if explicitly enabled or in MCP server context
-    // This prevents file watchers from starting during init command or other non-server contexts
-    const shouldAutoWatch = options?.autoWatch ?? process.env.MCP_SERVER_MODE === 'true';
-    if (shouldAutoWatch) {
-      this.startWatching();
-    }
-  }
-
-  /**
-   * Scan knowledge directory for markdown files
-   */
-  private scanKnowledgeFiles(dir: string): Array<{ uri: string; content: string }> {
-    const results: Array<{ uri: string; content: string }> = [];
-
-    const scan = (currentDir: string, baseDir: string) => {
-      const entries = fs.readdirSync(currentDir, { withFileTypes: true });
-
-      for (const entry of entries) {
-        const fullPath = path.join(currentDir, entry.name);
-
-        if (entry.isDirectory()) {
-          scan(fullPath, baseDir);
-        } else if (entry.isFile() && entry.name.endsWith('.md')) {
-          const relativePath = path.relative(baseDir, fullPath);
-          const uriPath = relativePath.replace(/\.md$/, '').replace(/\\/g, '/');
-          const content = fs.readFileSync(fullPath, 'utf8');
-
-          results.push({
-            uri: `knowledge://${uriPath}`,
-            content,
-          });
-        }
-      }
-    };
-
-    scan(dir, dir);
-    return results;
-  }
-
-  /**
-   * Build knowledge index
-   */
-  protected async buildIndex(): Promise<SearchIndex> {
-    const knowledgeDir = getKnowledgeDir();
-
-    if (!fs.existsSync(knowledgeDir)) {
-      throw new Error(`Knowledge directory not found: ${knowledgeDir}`);
-    }
-
-    const files = this.scanKnowledgeFiles(knowledgeDir);
-    logger.info('Found knowledge files', { count: files.length });
-
-    // Build TF-IDF index
-    const index = await buildSearchIndex(files);
-
-    // Build vector index if embedding provider is available
-    if (this.embeddingProvider && files.length > 0) {
-      logger.info('Building vector index for knowledge');
-
-      try {
-        const vectorPath = path.join(
-          getKnowledgeDir(),
-          '..',
-          '.sylphx-flow',
-          'knowledge-vectors.hnsw'
-        );
-        this.vectorStorage = new VectorStorage(
-          vectorPath,
-          this.embeddingProvider.dimensions || 1536
-        );
-        await this.vectorStorage.initialize();
-
-        // Process files in batches
-        const batchSize = 10;
-        for (let i = 0; i < files.length; i += batchSize) {
-          const batch = files.slice(i, i + batchSize);
-          const embeddings = await this.embeddingProvider.generateEmbeddings(
-            batch.map((file) => file.content)
-          );
-
-          for (let j = 0; j < batch.length; j++) {
-            const file = batch[j];
-            const embedding = embeddings[j];
-
-            await this.vectorStorage.addDocument({
-              id: file.uri,
-              embedding,
-              metadata: {
-                type: 'knowledge',
-                content: file.content.slice(0, 500),
-                category: 'knowledge',
-                language: 'markdown',
-              },
-            });
-          }
-
-          logger.info('Processed knowledge files', {
-            processed: Math.min(i + batchSize, files.length),
-            total: files.length,
-          });
-        }
-
-        await this.vectorStorage.save();
-        logger.info('Vector index built successfully');
-      } catch (error) {
-        logger.error(' Failed to build vector index:', error);
-        this.vectorStorage = undefined;
-      }
-    }
-
-    return index;
-  }
-
-  /**
-   * Start watching knowledge directory for changes
-   * MANDATORY: Auto-enabled to prevent stale data from misleading users
-   */
-  private startWatching(): void {
-    const knowledgeDir = getKnowledgeDir();
-
-    if (!fs.existsSync(knowledgeDir)) {
-      logger.warn('Knowledge directory not found', { knowledgeDir });
-      return;
-    }
-
-    try {
-      this.watcher = chokidar.watch(`${knowledgeDir}/**/*.md`, {
-        ignored: /(^|[/\\])\../, // Ignore dotfiles
-        persistent: true,
-        ignoreInitial: true, // Don't trigger on initial scan
-        awaitWriteFinish: {
-          stabilityThreshold: 300,
-          pollInterval: 100,
-        },
-      });
-
-      this.watcher.on('all', (event, filePath) => {
-        logger.debug('Knowledge file changed', { event, file: path.basename(filePath) });
-
-        // Debounce: Wait 2 seconds after last change before re-indexing
-        if (this.reindexTimer) {
-          clearTimeout(this.reindexTimer);
-        }
-
-        this.reindexTimer = setTimeout(() => {
-          logger.info('Re-indexing knowledge base due to file changes');
-          this.clearCache();
-          this.startBackgroundIndexing();
-        }, 2000);
-      });
-
-      logger.info('Watching knowledge directory for changes', { knowledgeDir });
-    } catch (error) {
-      logger.error(' Failed to start file watching:', error);
-      // Don't throw - indexing can still work without watching
-    }
-  }
-
-  /**
-   * Stop watching (for cleanup)
-   */
-  stopWatching(): void {
-    if (this.reindexTimer) {
-      clearTimeout(this.reindexTimer);
-      this.reindexTimer = undefined;
-    }
-
-    if (this.watcher) {
-      this.watcher.close();
-      this.watcher = undefined;
-      logger.info('Stopped watching knowledge directory');
-    }
-  }
-
-  /**
-   * Override clearCache to also stop any pending reindex
-   */
-  clearCache(): void {
-    if (this.reindexTimer) {
-      clearTimeout(this.reindexTimer);
-      this.reindexTimer = undefined;
-    }
-    super.clearCache();
-  }
-
-  /**
-   * Get vector storage for hybrid search
-   * Returns undefined if no embedding provider or vector index not built
-   */
-  getVectorStorage(): VectorStorage | undefined {
-    return this.vectorStorage;
-  }
-
-  /**
-   * Check if vector search is available
-   */
-  hasVectorSearch(): boolean {
-    return this.vectorStorage !== undefined;
-  }
-}
-
-// Singleton instance
-let knowledgeIndexer: KnowledgeIndexer | null = null;
-
-export function getKnowledgeIndexer(
-  embeddingProvider?: EmbeddingProvider,
-  options?: { autoWatch?: boolean }
-): KnowledgeIndexer {
-  if (!knowledgeIndexer) {
-    knowledgeIndexer = new KnowledgeIndexer(embeddingProvider, options);
-  }
-  return knowledgeIndexer;
-}
-
-export async function getKnowledgeIndexerWithEmbeddings(): Promise<KnowledgeIndexer> {
-  const embeddingProvider = await getDefaultEmbeddingProvider();
-  return getKnowledgeIndexer(embeddingProvider);
-}
-
-// ============================================================================
-// FUNCTIONAL ALTERNATIVE (Composition over Inheritance)
-// ============================================================================
-
-/**
- * Create functional knowledge indexer using composition
- * Modern alternative to class-based KnowledgeIndexer
- *
- * @example
- * const indexer = createKnowledgeIndexerFunctional();
- * const status = indexer.getStatus();
- * const index = await indexer.loadIndex();
- */
-export function createKnowledgeIndexerFunctional(
-  embeddingProvider?: EmbeddingProvider
-): Indexer & { stopWatching: () => void } {
-  const knowledgeDir = getKnowledgeDir();
-  let watcher: chokidar.FSWatcher | undefined;
-  let reindexTimer: NodeJS.Timeout | undefined;
-
-  // Core indexer with pure functions
-  const coreIndexer = createIndexer({
-    name: 'knowledge',
-    buildIndex: async (): Promise<SearchIndex> => {
-      if (!fs.existsSync(knowledgeDir)) {
-        throw new Error(`Knowledge directory not found: ${knowledgeDir}`);
-      }
-
-      // Scan knowledge files
-      const scanKnowledgeFiles = (dir: string): Array<{ uri: string; content: string }> => {
-        const results: Array<{ uri: string; content: string }> = [];
-
-        const scan = (currentDir: string, baseDir: string) => {
-          const entries = fs.readdirSync(currentDir, { withFileTypes: true });
-
-          for (const entry of entries) {
-            const fullPath = path.join(currentDir, entry.name);
-
-            if (entry.isDirectory()) {
-              scan(fullPath, baseDir);
-            } else if (entry.isFile() && entry.name.endsWith('.md')) {
-              const relativePath = path.relative(baseDir, fullPath);
-              const uriPath = relativePath.replace(/\.md$/, '').replace(/\\/g, '/');
-              const content = fs.readFileSync(fullPath, 'utf8');
-
-              results.push({
-                uri: `knowledge://${uriPath}`,
-                content,
-              });
-            }
-          }
-        };
-
-        scan(dir, dir);
-        return results;
-      };
-
-      const files = scanKnowledgeFiles(knowledgeDir);
-      logger.info('Found knowledge files', { count: files.length });
-
-      // Build TF-IDF index
-      const index = await buildSearchIndex(files);
-
-      // Build vector index if embedding provider available
-      if (embeddingProvider && files.length > 0) {
-        logger.info('Building vector index for knowledge');
-
-        try {
-          const vectorPath = path.join(
-            knowledgeDir,
-            '..',
-            '.sylphx-flow',
-            'knowledge-vectors.hnsw'
-          );
-          const vectorStorage = new VectorStorage(vectorPath, embeddingProvider.dimensions || 1536);
-          await vectorStorage.initialize();
-
-          // Process in batches
-          const batchSize = 10;
-          for (let i = 0; i < files.length; i += batchSize) {
-            const batch = files.slice(i, i + batchSize);
-            const embeddings = await embeddingProvider.generateEmbeddings(
-              batch.map((file) => file.content)
-            );
-
-            for (let j = 0; j < batch.length; j++) {
-              const file = batch[j];
-              const embedding = embeddings[j];
-
-              await vectorStorage.addDocument({
-                id: file.uri,
-                embedding,
-                metadata: {
-                  type: 'knowledge',
-                  content: file.content.slice(0, 500),
-                  category: 'knowledge',
-                  language: 'markdown',
-                },
-              });
-            }
-
-            logger.info('Processed knowledge files', {
-              processed: Math.min(i + batchSize, files.length),
-              total: files.length,
-            });
-          }
-
-          await vectorStorage.save();
-          logger.info('Vector index built successfully');
-        } catch (error) {
-          logger.error(' Failed to build vector index:', error);
-        }
-      }
-
-      return index;
-    },
-  });
-
-  // Add file watching
-  if (fs.existsSync(knowledgeDir)) {
-    try {
-      watcher = chokidar.watch(`${knowledgeDir}/**/*.md`, {
-        ignored: /(^|[/\\])\../,
-        persistent: true,
-        ignoreInitial: true,
-        awaitWriteFinish: {
-          stabilityThreshold: 300,
-          pollInterval: 100,
-        },
-      });
-
-      watcher.on('all', (event, filePath) => {
-        logger.debug('Knowledge file changed', { event, file: path.basename(filePath) });
-
-        // Debounce: Wait 2 seconds after last change
-        if (reindexTimer) {
-          clearTimeout(reindexTimer);
-        }
-
-        reindexTimer = setTimeout(() => {
-          logger.info('Re-indexing knowledge base due to file changes');
-          coreIndexer.clearCache();
-          coreIndexer.startBackgroundIndexing();
-        }, 2000);
-      });
-
-      logger.info('Watching knowledge directory for changes', { knowledgeDir });
-    } catch (error) {
-      logger.error(' Failed to start file watching:', error);
-    }
-  }
-
-  // Return enhanced indexer with watching capabilities
-  return {
-    ...coreIndexer,
-    stopWatching: () => {
-      if (reindexTimer) {
-        clearTimeout(reindexTimer);
-        reindexTimer = undefined;
-      }
-      if (watcher) {
-        watcher.close();
-        watcher = undefined;
-        logger.info('Stopped watching knowledge directory');
-      }
-    },
-  };
-}

package/src/services/search/semantic-search.ts

@@ -1,244 +0,0 @@
-/**
- * Semantic search implementation using TF-IDF
- * Runtime indexing with intelligent caching
- */
-
-import fs from 'node:fs';
-import path from 'node:path';
-import { getKnowledgeDir } from './paths.js';
-import { buildSearchIndex, processQuery, type SearchIndex } from './tfidf.js';
-import { createLogger } from '../../utils/debug-logger.js';
-
-const log = createLogger('search:query');
-
-let cachedIndex: SearchIndex | null = null;
-let indexingPromise: Promise<SearchIndex> | null = null;
-const indexingStatus = {
-  isIndexing: false,
-  progress: 0,
-  error: undefined as string | undefined,
-};
-
-/**
- * Scan knowledge directory for markdown files
- */
-function scanKnowledgeFiles(dir: string): Array<{ uri: string; content: string }> {
-  const results: Array<{ uri: string; content: string }> = [];
-
-  function scan(currentDir: string, baseDir: string) {
-    const entries = fs.readdirSync(currentDir, { withFileTypes: true });
-
-    for (const entry of entries) {
-      const fullPath = path.join(currentDir, entry.name);
-
-      if (entry.isDirectory()) {
-        scan(fullPath, baseDir);
-      } else if (entry.isFile() && entry.name.endsWith('.md')) {
-        const relativePath = path.relative(baseDir, fullPath);
-        const uriPath = relativePath.replace(/\.md$/, '').replace(/\\/g, '/');
-        const content = fs.readFileSync(fullPath, 'utf8');
-
-        results.push({
-          uri: `knowledge://${uriPath}`,
-          content,
-        });
-      }
-    }
-  }
-
-  scan(dir, dir);
-  return results;
-}
-
-/**
- * Build knowledge index at runtime
- */
-async function buildKnowledgeIndex(): Promise<SearchIndex> {
-  const knowledgeDir = getKnowledgeDir();
-
-  if (!fs.existsSync(knowledgeDir)) {
-    throw new Error(`Knowledge directory not found: ${knowledgeDir}`);
-  }
-
-  log('Building knowledge search index');
-  const files = scanKnowledgeFiles(knowledgeDir);
-  log(`Found ${files.length} knowledge files`);
-
-  const index = buildSearchIndex(files);
-  log(`Knowledge index built: ${index.totalDocuments} documents, ${index.idf.size} terms`);
-
-  return index;
-}
-
-/**
- * Load or build search index (with caching)
- */
-export async function loadSearchIndex(): Promise<SearchIndex | null> {
-  // Return cached index if available
-  if (cachedIndex) {
-    return cachedIndex;
-  }
-
-  // If already indexing, wait for it
-  if (indexingPromise) {
-    return indexingPromise;
-  }
-
-  // Start indexing
-  indexingStatus.isIndexing = true;
-  indexingStatus.progress = 0;
-  indexingStatus.error = undefined;
-
-  indexingPromise = buildKnowledgeIndex()
-    .then((index) => {
-      cachedIndex = index;
-      indexingStatus.isIndexing = false;
-      indexingStatus.progress = 100;
-      return index;
-    })
-    .catch((error) => {
-      indexingStatus.isIndexing = false;
-      indexingStatus.error = error instanceof Error ? error.message : String(error);
-      log('Failed to build knowledge index:', error instanceof Error ? error.message : String(error));
-      throw error;
-    });
-
-  return indexingPromise;
-}
-
-/**
- * Start background indexing (non-blocking)
- */
-export function startKnowledgeIndexing() {
-  if (indexingStatus.isIndexing || cachedIndex) {
-    return;
-  }
-
-  log('Starting background knowledge indexing');
-  loadSearchIndex().catch((error) => {
-    log('Background knowledge indexing failed:', error instanceof Error ? error.message : String(error));
-  });
-}
-
-/**
- * Get indexing status
- */
-export function getKnowledgeIndexingStatus() {
-  return {
-    isIndexing: indexingStatus.isIndexing,
-    progress: indexingStatus.progress,
-    isReady: cachedIndex !== null,
-    error: indexingStatus.error,
-  };
-}
-
-/**
- * Search knowledge base using semantic search
- */
-export async function semanticSearch(
-  query: string,
-  options: {
-    limit?: number;
-    minScore?: number;
-    categories?: string[];
-  } = {}
-): Promise<
-  Array<{
-    uri: string;
-    score: number;
-    matchedTerms: string[];
-    relevance: number; // 0-100 percentage
-  }>
-> {
-  const { limit = 5, minScore = 0.01, categories } = options;
-
-  const index = await loadSearchIndex();
-  if (!index) {
-    return [];
-  }
-
-  // Process query into TF-IDF vector
-  const queryVector = await processQuery(query, index.idf);
-
-  // Calculate cosine similarity for each document
-  const results = index.documents.map((doc) => {
-    let dotProduct = 0;
-    const matchedTerms: string[] = [];
-
-    // Calculate dot product and track matched terms
-    for (const [term, queryScore] of queryVector.entries()) {
-      const docScore = doc.terms.get(term);
-      if (docScore) {
-        dotProduct += queryScore * docScore;
-        matchedTerms.push(term);
-      }
-    }
-
-    // Calculate query magnitude
-    let queryMagnitude = 0;
-    for (const score of queryVector.values()) {
-      queryMagnitude += score * score;
-    }
-    queryMagnitude = Math.sqrt(queryMagnitude);
-
-    // Cosine similarity
-    const score =
-      queryMagnitude === 0 || doc.magnitude === 0
-        ? 0
-        : dotProduct / (queryMagnitude * doc.magnitude);
-
-    return {
-      uri: doc.uri,
-      score,
-      matchedTerms,
-      relevance: Math.round(score * 100),
-    };
-  });
-
-  // Filter by categories if specified
-  let filtered = results;
-  if (categories && categories.length > 0) {
-    filtered = results.filter((result) => {
-      const category = result.uri.split('/')[1]; // knowledge://stacks/react-app → stacks
-      return categories.includes(category);
-    });
-  }
-
-  // Filter by minimum score and sort
-  return filtered
-    .filter((result) => result.score >= minScore)
-    .sort((a, b) => b.score - a.score)
-    .slice(0, limit);
-}
-
-/**
- * Get search index statistics
- */
-export async function getSearchIndexStats(): Promise<{
-  loaded: boolean;
-  totalDocuments: number;
-  uniqueTerms: number;
-  generatedAt: string;
-  version: string;
-} | null> {
-  const index = await loadSearchIndex();
-  if (!index) {
-    return null;
-  }
-
-  return {
-    loaded: true,
-    totalDocuments: index.totalDocuments,
-    uniqueTerms: index.idf.size,
-    generatedAt: index.metadata.generatedAt,
-    version: index.metadata.version,
-  };
-}
-
-/**
- * Clear cached index (useful for testing)
- */
-export function clearSearchIndexCache(): void {
-  cachedIndex = null;
-  indexingPromise = null;
-}