@sylphx/flow 1.1.0 → 1.2.0

package/src/services/search/embeddings.ts

@@ -1,316 +0,0 @@
-/**
- * Embedding generation utilities
- * Supports OpenAI embeddings (with fallback to mock embeddings)
- */
-
-import { secretUtils } from '../../utils/secret-utils.js';
-import { envSecurity, securitySchemas } from '../../utils/security.js';
-import { generateMockEmbedding } from '../storage/vector-storage.js';
-import { createLogger } from '../../utils/debug-logger.js';
-
-const log = createLogger('search:embeddings');
-
-export interface ModelInfo {
-  id: string;
-  object: string;
-  created?: number;
-  owned_by?: string;
-}
-
-export interface EmbeddingModelOption {
-  id: string;
-  description: string;
-}
-
-export interface EmbeddingProvider {
-  name: string;
-  dimensions: number;
-  generateEmbedding(text: string): Promise<number[]>;
-  generateEmbeddings(texts: string[]): Promise<number[][]>;
-}
-
-/**
- * OpenAI Embedding Provider
- * Requires OPENAI_API_KEY environment variable
- */
-export class OpenAIEmbeddingProvider implements EmbeddingProvider {
-  name = 'openai';
-  dimensions = 1536; // text-embedding-3-small
-  private apiKey: string;
-  private model: string;
-  private baseURL: string;
-
-  constructor(
-    options: {
-      apiKey?: string;
-      model?: 'text-embedding-3-small' | 'text-embedding-3-large' | 'text-embedding-ada-002';
-      baseURL?: string;
-    } = {}
-  ) {
-    // Validate and get API key with security checks
-    this.apiKey = options.apiKey || envSecurity.getEnvVar('OPENAI_API_KEY') || '';
-    this.model = options.model || 'text-embedding-3-small';
-
-    // Validate base URL
-    const providedBaseURL =
-      options.baseURL || envSecurity.getEnvVar('OPENAI_BASE_URL', 'https://api.openai.com/v1');
-    if (providedBaseURL) {
-      try {
-        this.baseURL = securitySchemas.url.parse(providedBaseURL);
-      } catch (_error) {
-        log('Invalid OPENAI_BASE_URL format, using default');
-        this.baseURL = 'https://api.openai.com/v1';
-      }
-    } else {
-      this.baseURL = 'https://api.openai.com/v1';
-    }
-
-    // Set dimensions based on model
-    if (this.model === 'text-embedding-3-large') {
-      this.dimensions = 3072;
-    } else if (this.model === 'text-embedding-ada-002') {
-      this.dimensions = 1536;
-    }
-
-    if (this.apiKey) {
-      // Validate API key format
-      try {
-        securitySchemas.apiKey.parse(this.apiKey);
-      } catch (_error) {
-        log('Invalid OPENAI_API_KEY format, using mock implementation');
-        this.apiKey = '';
-      }
-    } else {
-      log('OPENAI_API_KEY not set, using mock implementation');
-    }
-  }
-
-  async generateEmbedding(text: string): Promise<number[]> {
-    if (!this.apiKey) {
-      log('Using mock embedding (no API key)');
-      return generateMockEmbedding(text, this.dimensions);
-    }
-
-    try {
-      const response = await fetch(`${this.baseURL}/embeddings`, {
-        method: 'POST',
-        headers: {
-          'Content-Type': 'application/json',
-          Authorization: `Bearer ${this.apiKey}`,
-        },
-        body: JSON.stringify({
-          model: this.model,
-          input: text,
-        }),
-      });
-
-      if (!response.ok) {
-        const error = await response.text();
-        throw new Error(`OpenAI API error: ${response.status} ${error}`);
-      }
-
-      const data = await response.json();
-      return data.data[0].embedding;
-    } catch (error) {
-      log('Failed to generate OpenAI embedding:', error instanceof Error ? error.message : String(error));
-      log('Falling back to mock embedding');
-      return generateMockEmbedding(text, this.dimensions);
-    }
-  }
-
-  async generateEmbeddings(texts: string[]): Promise<number[][]> {
-    if (!this.apiKey) {
-      log('Using mock embeddings (no API key)');
-      return texts.map((text) => generateMockEmbedding(text, this.dimensions));
-    }
-
-    try {
-      const response = await fetch(`${this.baseURL}/embeddings`, {
-        method: 'POST',
-        headers: {
-          'Content-Type': 'application/json',
-          Authorization: `Bearer ${this.apiKey}`,
-        },
-        body: JSON.stringify({
-          model: this.model,
-          input: texts,
-        }),
-      });
-
-      if (!response.ok) {
-        const error = await response.text();
-        throw new Error(`OpenAI API error: ${response.status} ${error}`);
-      }
-
-      const data = await response.json();
-      return data.data.map((item: { embedding: number[] }) => item.embedding);
-    } catch (error) {
-      log('Failed to generate OpenAI embeddings:', error instanceof Error ? error.message : String(error));
-      log('Falling back to mock embeddings');
-      return texts.map((text) => generateMockEmbedding(text, this.dimensions));
-    }
-  }
-
-  /**
-   * List all available models from the OpenAI-compatible API
-   */
-  async listModels(): Promise<ModelInfo[]> {
-    if (!this.apiKey) {
-      throw new Error('API key required to list models');
-    }
-
-    try {
-      const response = await fetch(`${this.baseURL}/models`, {
-        method: 'GET',
-        headers: {
-          'Content-Type': 'application/json',
-          Authorization: `Bearer ${this.apiKey}`,
-        },
-      });
-
-      if (!response.ok) {
-        const error = await response.text();
-        throw new Error(`OpenAI API error: ${response.status} ${error}`);
-      }
-
-      const data = await response.json();
-      return data.data.map((model: any) => ({
-        id: model.id,
-        object: model.object,
-        created: model.created,
-        owned_by: model.owned_by,
-      }));
-    } catch (error) {
-      if (error instanceof Error) {
-        throw new Error(`Failed to list models: ${error.message}`);
-      }
-      throw new Error('Failed to list models: Unknown error');
-    }
-  }
-
-  /**
-   * Filter models to only include embedding models
-   */
-  async listEmbeddingModels(): Promise<ModelInfo[]> {
-    const allModels = await this.listModels();
-    return allModels.filter(
-      (model) => model.id.includes('embedding') || model.id.includes('text-embedding')
-    );
-  }
-
-  /**
-   * Test if the API connection is working
-   */
-  async testConnection(): Promise<boolean> {
-    try {
-      await this.listModels();
-      return true;
-    } catch (_error) {
-      return false;
-    }
-  }
-
-  /**
-   * Get available embedding models with their details
-   */
-  async getEmbeddingModelOptions(): Promise<EmbeddingModelOption[]> {
-    const embeddingModels = await this.listEmbeddingModels();
-
-    return embeddingModels.map((model) => ({
-      id: model.id,
-      description: this.getModelDescription(model.id),
-    }));
-  }
-
-  private getModelDescription(modelId: string): string {
-    const descriptions: Record<string, string> = {
-      'text-embedding-3-small': 'Latest small embedding model (1536 dimensions)',
-      'text-embedding-3-large': 'Latest large embedding model (3072 dimensions)',
-      'text-embedding-ada-002': 'Legacy embedding model (1536 dimensions)',
-    };
-
-    return descriptions[modelId] || `Embedding model: ${modelId}`;
-  }
-}
-
-/**
- * Mock Embedding Provider (for testing without API key)
- */
-export class MockEmbeddingProvider implements EmbeddingProvider {
-  name = 'mock';
-  dimensions: number;
-
-  constructor(dimensions = 1536) {
-    this.dimensions = dimensions;
-  }
-
-  async generateEmbedding(text: string): Promise<number[]> {
-    return generateMockEmbedding(text, this.dimensions);
-  }
-
-  async generateEmbeddings(texts: string[]): Promise<number[][]> {
-    return texts.map((text) => generateMockEmbedding(text, this.dimensions));
-  }
-}
-
-/**
- * Get default embedding provider
- * Uses OpenAI if API key is available, otherwise mock
- */
-export async function getDefaultEmbeddingProvider(): Promise<EmbeddingProvider> {
-  // Try to load from secrets first
-  let secrets: Record<string, string> = {};
-  try {
-    secrets = await secretUtils.loadSecrets(process.cwd()).catch(() => ({}));
-  } catch (_error) {
-    // Ignore if secretUtils is not available
-  }
-
-  const apiKey = secrets.OPENAI_API_KEY || envSecurity.getEnvVar('OPENAI_API_KEY');
-  const baseURL =
-    secrets.OPENAI_BASE_URL ||
-    envSecurity.getEnvVar('OPENAI_BASE_URL', 'https://api.openai.com/v1');
-  const model = secrets.EMBEDDING_MODEL || envSecurity.getEnvVar('EMBEDDING_MODEL');
-
-  if (apiKey) {
-    return new OpenAIEmbeddingProvider({
-      apiKey,
-      baseURL,
-      model: model as any,
-    });
-  }
-
-  // Return mock embeddings silently
-  return new MockEmbeddingProvider();
-}
-
-/**
- * Chunk text into smaller pieces for embedding
- * Useful for long documents
- */
-export function chunkText(
-  text: string,
-  options: {
-    maxChunkSize?: number; // Max characters per chunk
-    overlap?: number; // Overlap between chunks
-  } = {}
-): string[] {
-  const { maxChunkSize = 1000, overlap = 100 } = options;
-
-  const chunks: string[] = [];
-  let start = 0;
-
-  while (start < text.length) {
-    const end = Math.min(start + maxChunkSize, text.length);
-    const chunk = text.slice(start, end);
-    chunks.push(chunk);
-
-    // Move start position with overlap
-    start = end - overlap;
-    if (start >= text.length) {
-      break;
-    }
-  }
-
-  return chunks;
-}

package/src/services/search/functional-indexer.ts

@@ -1,323 +0,0 @@
-/**
- * Functional Indexer
- * Pure functional implementation using composition instead of inheritance
- *
- * Replaces BaseIndexer class with factory function and closures
- */
-
-import type { SearchIndex } from './tfidf.js';
-import { createLogger } from '../../utils/debug-logger.js';
-
-const log = createLogger('search:indexing');
-
-// ============================================================================
-// TYPES
-// ============================================================================
-
-export interface IndexingStatus {
-  isIndexing: boolean;
-  progress: number; // 0-100
-  totalItems: number;
-  indexedItems: number;
-  startTime: number;
-  error?: string;
-}
-
-export interface IndexerConfig {
-  name: string; // 'knowledge' or 'codebase'
-  buildIndex: () => Promise<SearchIndex>; // Injected build function
-  autoStart?: boolean; // Auto-start indexing (default: true)
-}
-
-export interface IndexerState {
-  readonly cachedIndex: SearchIndex | null;
-  readonly indexingPromise: Promise<SearchIndex> | null;
-  readonly status: IndexingStatus;
-}
-
-export interface Indexer {
-  readonly getStatus: () => IndexingStatus;
-  readonly isReady: () => boolean;
-  readonly loadIndex: () => Promise<SearchIndex>;
-  readonly clearCache: () => void;
-  readonly getStats: () => Promise<{
-    totalDocuments: number;
-    uniqueTerms: number;
-    generatedAt: string;
-    version: string;
-  } | null>;
-  readonly startBackgroundIndexing: () => void;
-}
-
-// ============================================================================
-// STATE MANAGEMENT (Pure Functions)
-// ============================================================================
-
-/**
- * Create initial indexer state
- */
-const createInitialState = (): IndexerState => ({
-  cachedIndex: null,
-  indexingPromise: null,
-  status: {
-    isIndexing: false,
-    progress: 0,
-    totalItems: 0,
-    indexedItems: 0,
-    startTime: 0,
-  },
-});
-
-/**
- * Update status to indexing
- */
-const setIndexing = (state: IndexerState): IndexerState => ({
-  ...state,
-  status: {
-    ...state.status,
-    isIndexing: true,
-    progress: 0,
-    startTime: Date.now(),
-    error: undefined,
-  },
-});
-
-/**
- * Update with completed index
- */
-const setCompleted = (state: IndexerState, index: SearchIndex): IndexerState => ({
-  ...state,
-  cachedIndex: index,
-  indexingPromise: null,
-  status: {
-    ...state.status,
-    isIndexing: false,
-    progress: 100,
-    totalItems: index.totalDocuments,
-    indexedItems: index.totalDocuments,
-  },
-});
-
-/**
- * Update with error
- */
-const setError = (state: IndexerState, error: Error): IndexerState => ({
-  ...state,
-  indexingPromise: null,
-  status: {
-    ...state.status,
-    isIndexing: false,
-    error: error.message,
-  },
-});
-
-/**
- * Set indexing promise
- */
-const setIndexingPromise = (state: IndexerState, promise: Promise<SearchIndex>): IndexerState => ({
-  ...state,
-  indexingPromise: promise,
-});
-
-// ============================================================================
-// INDEXER FACTORY (Composition over Inheritance)
-// ============================================================================
-
-/**
- * Create indexer with closure-based state management
- * Replaces BaseIndexer class with pure functional approach
- *
- * @example
- * const knowledgeIndexer = createIndexer({
- *   name: 'knowledge',
- *   buildIndex: async () => {
- *     // Knowledge-specific indexing logic
- *     return searchIndex;
- *   }
- * });
- *
- * const status = knowledgeIndexer.getStatus();
- * const index = await knowledgeIndexer.loadIndex();
- */
-export const createIndexer = (config: IndexerConfig): Indexer => {
-  // State managed in closure (not mutable class fields)
-  let state = createInitialState();
-
-  // Auto-start indexing (default behavior)
-  const shouldAutoStart = config.autoStart !== false;
-  if (shouldAutoStart) {
-    setTimeout(() => indexer.startBackgroundIndexing(), 0);
-  }
-
-  // ========================================================================
-  // PUBLIC API
-  // ========================================================================
-
-  const indexer: Indexer = {
-    /**
-     * Get current indexing status (immutable copy)
-     */
-    getStatus: () => ({ ...state.status }),
-
-    /**
-     * Check if index is ready
-     */
-    isReady: () => state.cachedIndex !== null && !state.status.isIndexing,
-
-    /**
-     * Load or build index (with caching and deduplication)
-     */
-    loadIndex: async () => {
-      // Return cached index if available
-      if (state.cachedIndex) {
-        return state.cachedIndex;
-      }
-
-      // If already indexing, wait for existing promise
-      if (state.indexingPromise) {
-        return state.indexingPromise;
-      }
-
-      // Start new indexing
-      state = setIndexing(state);
-
-      const indexingPromise = config
-        .buildIndex()
-        .then((index) => {
-          state = setCompleted(state, index);
-          log(`${config.name} indexing complete:`, index.totalDocuments, 'documents');
-          return index;
-        })
-        .catch((error) => {
-          state = setError(state, error as Error);
-          log(`${config.name} indexing failed:`, error instanceof Error ? error.message : String(error));
-          throw error;
-        });
-
-      state = setIndexingPromise(state, indexingPromise);
-      return indexingPromise;
-    },
-
-    /**
-     * Clear cache and reset state
-     */
-    clearCache: () => {
-      state = createInitialState();
-    },
-
-    /**
-     * Get index statistics
-     */
-    getStats: async () => {
-      const index = await indexer.loadIndex();
-      if (!index) {
-        return null;
-      }
-
-      return {
-        totalDocuments: index.totalDocuments,
-        uniqueTerms: index.idf.size,
-        generatedAt: index.metadata.generatedAt,
-        version: index.metadata.version,
-      };
-    },
-
-    /**
-     * Start background indexing (non-blocking)
-     */
-    startBackgroundIndexing: () => {
-      if (state.status.isIndexing || state.cachedIndex) {
-        return;
-      }
-
-      log(`Starting background ${config.name} indexing`);
-      indexer.loadIndex().catch((error) => {
-        log(`Background ${config.name} indexing failed:`, error instanceof Error ? error.message : String(error));
-      });
-    },
-  };
-
-  return Object.freeze(indexer); // Make API immutable
-};
-
-// ============================================================================
-// UTILITIES
-// ============================================================================
-
-/**
- * Compose indexer with additional behavior
- * Example of function composition instead of inheritance
- */
-export const withLogging =
-  (baseIndexer: Indexer) =>
-  (config: { verbose?: boolean } = {}): Indexer => {
-    const { verbose = false } = config;
-
-    return {
-      ...baseIndexer,
-      loadIndex: async () => {
-        if (verbose) {
-          log('Loading index...');
-        }
-        const index = await baseIndexer.loadIndex();
-        if (verbose) {
-          log('Index loaded:', index.totalDocuments, 'documents');
-        }
-        return index;
-      },
-    };
-  };
-
-/**
- * Compose indexer with custom cache strategy
- */
-export const withCacheTTL =
-  (baseIndexer: Indexer) =>
-  (ttlMs: number): Indexer => {
-    let lastIndexTime = 0;
-
-    return {
-      ...baseIndexer,
-      loadIndex: async () => {
-        const now = Date.now();
-        const isExpired = now - lastIndexTime > ttlMs;
-
-        if (isExpired) {
-          baseIndexer.clearCache();
-        }
-
-        const index = await baseIndexer.loadIndex();
-        lastIndexTime = now;
-        return index;
-      },
-    };
-  };
-
-/**
- * Compose indexer with retry logic
- */
-export const withRetry =
-  (baseIndexer: Indexer) =>
-  (maxRetries = 3, delayMs = 1000): Indexer => {
-    return {
-      ...baseIndexer,
-      loadIndex: async () => {
-        let lastError: Error | null = null;
-
-        for (let i = 0; i < maxRetries; i++) {
-          try {
-            return await baseIndexer.loadIndex();
-          } catch (error) {
-            lastError = error as Error;
-            log(`Indexing attempt ${i + 1}/${maxRetries} failed:`, error instanceof Error ? error.message : String(error));
-
-            if (i < maxRetries - 1) {
-              await new Promise((resolve) => setTimeout(resolve, delayMs));
-            }
-          }
-        }
-
-        throw lastError || new Error('Indexing failed after retries');
-      },
-    };
-  };

package/src/services/search/index.ts

@@ -1,27 +0,0 @@
-/**
- * Search Services Index
- * Provides unified interface for all search-related services
- */
-
-export { CodebaseIndexer } from './codebase-indexer.js';
-// Types
-export type {
-  CodebaseFile,
-  CodebaseIndexerOptions,
-  IndexCache,
-  IndexingStatus,
-} from './codebase-indexer-types.js';
-export type { EmbeddingProvider } from './embeddings.js';
-
-// Embeddings
-export { createEmbeddingProvider, getDefaultEmbeddingProvider } from './embeddings.js';
-export { SemanticSearchService } from './semantic-search.js';
-export type { SearchIndex, SearchResult as TFIDFSearchResult } from './tfidf.js';
-// TF-IDF
-export { buildSearchIndex, searchDocuments } from './tfidf.js';
-export type {
-  SearchOptions,
-  SearchResult,
-} from './unified-search-service.js';
-// Main services
-export { getSearchService } from './unified-search-service.js';