cognitive-core 0.2.4 → 0.3.1

package/src/embeddings/manager.ts

@@ -1,293 +0,0 @@
-/**
- * Embedding Manager
- *
- * Orchestrates embedding generation, storage, and search.
- * Provides automatic fallback to BM25 when no embedding provider is configured.
- */
-
-import type { EmbeddingProvider } from './provider.js';
-import { SqliteVectorStore } from './vector-store.js';
-import { BM25Index, type MemoryItemType } from './bm25.js';
-
-export { type MemoryItemType } from './bm25.js';
-
-export interface SearchResult {
-  id: string;
-  score: number;
-  content: string;
-}
-
-export interface EmbeddingManagerOptions {
-  /** Embedding provider (null for BM25-only mode) */
-  provider?: EmbeddingProvider | null;
-  /** Shared DB connection from SqlitePersistence (preferred) */
-  db?: import('better-sqlite3').Database;
-  /** Path to sqlite-vec database file (legacy: separate DB file) */
-  vectorStorePath?: string;
-  /** Embedding dimension (required if provider is set) */
-  dimension?: number;
-  /** BM25 parameters */
-  bm25?: {
-    k1?: number;
-    b?: number;
-  };
-}
-
-/**
- * Manages embeddings with automatic fallback to BM25
- */
-export class EmbeddingManager {
-  private provider: EmbeddingProvider | null;
-  private vectorStore: SqliteVectorStore | null;
-  private bm25Index: BM25Index;
-  private useVectorSearch: boolean;
-
-  constructor(options: EmbeddingManagerOptions) {
-    this.provider = options.provider ?? null;
-    this.useVectorSearch = this.provider !== null && !!(options.vectorStorePath || options.db);
-
-    // Initialize vector store if we have a provider and a DB source
-    if (this.useVectorSearch) {
-      const dimension = options.dimension ?? this.provider!.dimension;
-      if (options.db) {
-        this.vectorStore = new SqliteVectorStore({ db: options.db, dimension });
-      } else if (options.vectorStorePath) {
-        this.vectorStore = new SqliteVectorStore({ dbPath: options.vectorStorePath, dimension });
-      } else {
-        this.vectorStore = null;
-      }
-    } else {
-      this.vectorStore = null;
-    }
-
-    // Always maintain BM25 as fallback
-    this.bm25Index = new BM25Index(options.bm25);
-  }
-
-  /**
-   * Check if vector search is available
-   */
-  get hasVectorSearch(): boolean {
-    return this.useVectorSearch && this.vectorStore !== null;
-  }
-
-  /**
-   * Get the embedding dimension (if available)
-   */
-  get dimension(): number | null {
-    return this.provider?.dimension ?? null;
-  }
-
-  /**
-   * Store an item with its embedding
-   * Returns the embedding if generated, undefined otherwise
-   */
-  async store(
-    id: string,
-    content: string,
-    type: MemoryItemType
-  ): Promise<number[] | undefined> {
-    // Always add to BM25 index for fallback
-    this.bm25Index.add(id, content, type);
-
-    // Generate and store embedding if provider available
-    if (this.useVectorSearch && this.provider && this.vectorStore) {
-      try {
-        const embedding = await this.provider.embed(content);
-        this.vectorStore.insert(id, embedding, type, content);
-        return embedding;
-      } catch (error) {
-        console.error(`Failed to generate embedding for ${id}:`, error);
-        // Fall through to return undefined - BM25 is still available
-      }
-    }
-
-    return undefined;
-  }
-
-  /**
-   * Store multiple items with embeddings (batch)
-   * More efficient than calling store() multiple times
-   */
-  async storeBatch(
-    items: Array<{ id: string; content: string; type: MemoryItemType }>
-  ): Promise<Map<string, number[] | undefined>> {
-    const results = new Map<string, number[] | undefined>();
-
-    // Add all to BM25 index
-    for (const item of items) {
-      this.bm25Index.add(item.id, item.content, item.type);
-    }
-
-    // Batch embed if provider available
-    if (this.useVectorSearch && this.provider && this.vectorStore) {
-      try {
-        const contents = items.map((i) => i.content);
-        const embeddings = await this.provider.embedBatch(contents);
-
-        const batchItems = items.map((item, i) => ({
-          id: item.id,
-          embedding: embeddings[i],
-          type: item.type,
-          content: item.content,
-        }));
-
-        this.vectorStore.insertBatch(batchItems);
-
-        for (let i = 0; i < items.length; i++) {
-          results.set(items[i].id, embeddings[i]);
-        }
-      } catch (error) {
-        console.error('Failed to generate batch embeddings:', error);
-        // Fall through - items are still in BM25 index
-        for (const item of items) {
-          results.set(item.id, undefined);
-        }
-      }
-    } else {
-      for (const item of items) {
-        results.set(item.id, undefined);
-      }
-    }
-
-    return results;
-  }
-
-  /**
-   * Search for similar items
-   * Uses vector search if available, falls back to BM25
-   */
-  async search(
-    query: string,
-    options: {
-      k?: number;
-      type?: MemoryItemType;
-      threshold?: number;
-    } = {}
-  ): Promise<SearchResult[]> {
-    // Use vector search if available
-    if (this.useVectorSearch && this.provider && this.vectorStore) {
-      try {
-        const queryEmbedding = await this.provider.embed(query);
-        const results = this.vectorStore.search(queryEmbedding, options);
-        return results.map((r) => ({
-          id: r.id,
-          score: r.score,
-          content: r.content,
-        }));
-      } catch (error) {
-        console.error('Vector search failed, falling back to BM25:', error);
-        // Fall through to BM25
-      }
-    }
-
-    // Fallback to BM25
-    return this.bm25Index.search(query, options);
-  }
-
-  /**
-   * Search using a pre-computed embedding
-   * Only works if vector store is available
-   */
-  searchByEmbedding(
-    embedding: number[],
-    options: {
-      k?: number;
-      type?: MemoryItemType;
-      threshold?: number;
-    } = {}
-  ): SearchResult[] | null {
-    if (!this.vectorStore) {
-      return null;
-    }
-
-    const results = this.vectorStore.search(embedding, options);
-    return results.map((r) => ({
-      id: r.id,
-      score: r.score,
-      content: r.content,
-    }));
-  }
-
-  /**
-   * Generate an embedding without storing it
-   */
-  async embed(text: string): Promise<number[] | null> {
-    if (!this.provider) {
-      return null;
-    }
-
-    try {
-      return await this.provider.embed(text);
-    } catch (error) {
-      console.error('Failed to generate embedding:', error);
-      return null;
-    }
-  }
-
-  /**
-   * Generate embeddings for multiple texts without storing
-   */
-  async embedBatch(texts: string[]): Promise<number[][] | null> {
-    if (!this.provider) {
-      return null;
-    }
-
-    try {
-      return await this.provider.embedBatch(texts);
-    } catch (error) {
-      console.error('Failed to generate batch embeddings:', error);
-      return null;
-    }
-  }
-
-  /**
-   * Remove an item from all indexes
-   */
-  remove(id: string): void {
-    this.bm25Index.remove(id);
-    this.vectorStore?.delete(id);
-  }
-
-  /**
-   * Check if an item exists
-   */
-  has(id: string): boolean {
-    return this.bm25Index.has(id);
-  }
-
-  /**
-   * Get statistics about the indexes
-   */
-  getStats(): {
-    bm25Count: number;
-    vectorCount: number | null;
-    hasVectorSearch: boolean;
-    dimension: number | null;
-    providerName: string | null;
-  } {
-    return {
-      bm25Count: this.bm25Index.size,
-      vectorCount: this.vectorStore?.count ?? null,
-      hasVectorSearch: this.hasVectorSearch,
-      dimension: this.dimension,
-      providerName: this.provider?.name ?? null,
-    };
-  }
-
-  /**
-   * Close all resources
-   */
-  close(): void {
-    this.vectorStore?.close();
-  }
-}
-
-/**
- * Create an embedding manager
- */
-export function createEmbeddingManager(
-  options: EmbeddingManagerOptions
-): EmbeddingManager {
-  return new EmbeddingManager(options);
-}

package/src/embeddings/provider.ts

@@ -1,318 +0,0 @@
-/**
- * Embedding Provider Interface and Implementations
- *
- * Provides a unified interface for generating embeddings from text.
- * Supports hosted providers (OpenAI, etc.) and local models.
- */
-
-/**
- * Interface for embedding providers (hosted or local)
- */
-export interface EmbeddingProvider {
-  /** Provider name for logging/debugging */
-  readonly name: string;
-
-  /** Embedding dimension */
-  readonly dimension: number;
-
-  /** Embed a single text */
-  embed(text: string): Promise<number[]>;
-
-  /** Embed multiple texts (batch) */
-  embedBatch(texts: string[]): Promise<number[][]>;
-}
-
-/**
- * OpenAI embedding provider
- */
-export class OpenAIEmbeddingProvider implements EmbeddingProvider {
-  readonly name = "openai";
-  readonly dimension: number;
-  private apiKey: string;
-  private model: string;
-  private baseUrl: string;
-
-  constructor(options: {
-    apiKey: string;
-    model?: string;
-    dimension?: number;
-    baseUrl?: string;
-  }) {
-    this.apiKey = options.apiKey;
-    this.model = options.model ?? "text-embedding-3-small";
-    this.dimension = options.dimension ?? 1536;
-    this.baseUrl = options.baseUrl ?? "https://api.openai.com/v1";
-  }
-
-  async embed(text: string): Promise<number[]> {
-    const [embedding] = await this.embedBatch([text]);
-    return embedding;
-  }
-
-  async embedBatch(texts: string[]): Promise<number[][]> {
-    const response = await fetch(`${this.baseUrl}/embeddings`, {
-      method: "POST",
-      headers: {
-        Authorization: `Bearer ${this.apiKey}`,
-        "Content-Type": "application/json",
-      },
-      body: JSON.stringify({
-        model: this.model,
-        input: texts,
-        dimensions: this.dimension,
-      }),
-    });
-
-    if (!response.ok) {
-      const error = await response.text();
-      throw new Error(`OpenAI embedding error: ${response.status} - ${error}`);
-    }
-
-    const data = (await response.json()) as {
-      data: Array<{ embedding: number[]; index: number }>;
-    };
-
-    // Sort by index to maintain order
-    const sorted = data.data.sort((a, b) => a.index - b.index);
-    return sorted.map((item) => item.embedding);
-  }
-}
-
-/**
- * Anthropic Voyage embedding provider
- * Uses Voyage AI embeddings via Anthropic partnership
- */
-export class VoyageEmbeddingProvider implements EmbeddingProvider {
-  readonly name = "voyage";
-  readonly dimension: number;
-  private apiKey: string;
-  private model: string;
-
-  constructor(options: { apiKey: string; model?: string; dimension?: number }) {
-    this.apiKey = options.apiKey;
-    this.model = options.model ?? "voyage-2";
-    this.dimension = options.dimension ?? 1024;
-  }
-
-  async embed(text: string): Promise<number[]> {
-    const [embedding] = await this.embedBatch([text]);
-    return embedding;
-  }
-
-  async embedBatch(texts: string[]): Promise<number[][]> {
-    const response = await fetch("https://api.voyageai.com/v1/embeddings", {
-      method: "POST",
-      headers: {
-        Authorization: `Bearer ${this.apiKey}`,
-        "Content-Type": "application/json",
-      },
-      body: JSON.stringify({
-        model: this.model,
-        input: texts,
-      }),
-    });
-
-    if (!response.ok) {
-      const error = await response.text();
-      throw new Error(`Voyage embedding error: ${response.status} - ${error}`);
-    }
-
-    const data = (await response.json()) as {
-      data: Array<{ embedding: number[]; index: number }>;
-    };
-
-    const sorted = data.data.sort((a, b) => a.index - b.index);
-    return sorted.map((item) => item.embedding);
-  }
-}
-
-/**
- * Local embedding provider using @huggingface/transformers
- * Runs transformer models locally in Node.js using ONNX runtime
- */
-export class LocalEmbeddingProvider implements EmbeddingProvider {
-  readonly name = "local";
-  readonly dimension: number;
-  private modelName: string;
-  private extractor: any = null;
-  private initPromise: Promise<void> | null = null;
-
-  /** Default model - all-MiniLM-L6-v2 is small and fast with good quality */
-  static readonly DEFAULT_MODEL = "sentence-transformers/all-MiniLM-L6-v2";
-  /** Dimensions for common models */
-  static readonly MODEL_DIMENSIONS: Record<string, number> = {
-    "sentence-transformers/all-MiniLM-L6-v2": 384,
-    "sentence-transformers/all-MiniLM-L12-v2": 384,
-    "BAAI/bge-small-en-v1.5": 384,
-    "BAAI/bge-base-en-v1.5": 768,
-    "intfloat/e5-small-v2": 384,
-    "intfloat/e5-base-v2": 768,
-    // Keep Xenova aliases for backwards compatibility
-    "Xenova/all-MiniLM-L6-v2": 384,
-    "Xenova/all-MiniLM-L12-v2": 384,
-    "Xenova/bge-small-en-v1.5": 384,
-    "Xenova/bge-base-en-v1.5": 768,
-    "Xenova/e5-small-v2": 384,
-    "Xenova/e5-base-v2": 768,
-  };
-
-  constructor(options?: {
-    modelName?: string;
-    modelPath?: string; // Alias for modelName for backwards compatibility
-    dimension?: number;
-  }) {
-    this.modelName =
-      options?.modelName ??
-      options?.modelPath ??
-      LocalEmbeddingProvider.DEFAULT_MODEL;
-
-    // Auto-detect dimension from known models or use provided/default
-    this.dimension =
-      options?.dimension ??
-      LocalEmbeddingProvider.MODEL_DIMENSIONS[this.modelName] ??
-      384;
-  }
-
-  /** Get the model name/path */
-  get modelPath(): string {
-    return this.modelName;
-  }
-
-  private async initialize(): Promise<void> {
-    if (this.extractor) return;
-
-    // Prevent multiple concurrent initializations
-    if (this.initPromise) {
-      await this.initPromise;
-      return;
-    }
-
-    this.initPromise = this.doInitialize();
-    await this.initPromise;
-  }
-
-  private async doInitialize(): Promise<void> {
-    try {
-      // Dynamic import — @huggingface/transformers is an optional dependency
-      const { pipeline } = await import("@huggingface/transformers");
-
-      // Create feature-extraction pipeline for embeddings
-      this.extractor = await pipeline("feature-extraction", this.modelName, {
-        dtype: "fp32", // Use fp32 for better compatibility
-      });
-    } catch (error) {
-      const message = error instanceof Error ? error.message : String(error);
-      if (
-        message.includes("Cannot find module") ||
-        message.includes("ERR_MODULE_NOT_FOUND")
-      ) {
-        throw new Error(
-          `Local embeddings require @huggingface/transformers. Install it with: npm install @huggingface/transformers`,
-        );
-      }
-      throw new Error(
-        `Failed to initialize local embedding model "${this.modelName}": ${message}`,
-      );
-    }
-  }
-
-  async embed(text: string): Promise<number[]> {
-    await this.initialize();
-
-    // Run the extractor
-    const output = await this.extractor(text, {
-      pooling: "mean", // Mean pooling over tokens
-      normalize: true, // L2 normalize the output
-    });
-
-    // Convert tensor to array
-    return Array.from(output.data as Float32Array);
-  }
-
-  async embedBatch(texts: string[]): Promise<number[][]> {
-    await this.initialize();
-
-    // Process in batch for better efficiency
-    const results: number[][] = [];
-
-    for (const text of texts) {
-      const output = await this.extractor(text, {
-        pooling: "mean",
-        normalize: true,
-      });
-      results.push(Array.from(output.data as Float32Array));
-    }
-
-    return results;
-  }
-
-  /**
-   * Check if a model is available locally (cached)
-   * This can be used to pre-warm the cache
-   */
-  async isModelCached(): Promise<boolean> {
-    try {
-      // Returns false if @huggingface/transformers is not installed
-      const { env } = await import("@huggingface/transformers");
-      return env.cacheDir !== undefined;
-    } catch {
-      return false;
-    }
-  }
-}
-
-/**
- * Configuration for embedding providers
- */
-export interface EmbeddingProviderConfig {
-  type: "openai" | "voyage" | "local" | "none";
-  apiKey?: string;
-  model?: string;
-  modelPath?: string;
-  dimension?: number;
-  baseUrl?: string;
-}
-
-/**
- * Create an embedding provider from configuration
- */
-export function createEmbeddingProvider(
-  config: EmbeddingProviderConfig,
-): EmbeddingProvider | null {
-  switch (config.type) {
-    case "openai":
-      if (!config.apiKey) {
-        throw new Error("OpenAI embedding provider requires apiKey");
-      }
-      return new OpenAIEmbeddingProvider({
-        apiKey: config.apiKey,
-        model: config.model,
-        dimension: config.dimension,
-        baseUrl: config.baseUrl,
-      });
-
-    case "voyage":
-      if (!config.apiKey) {
-        throw new Error("Voyage embedding provider requires apiKey");
-      }
-      return new VoyageEmbeddingProvider({
-        apiKey: config.apiKey,
-        model: config.model,
-        dimension: config.dimension,
-      });
-
-    case "local":
-      // Local provider now has sensible defaults
-      return new LocalEmbeddingProvider({
-        modelName: config.model,
-        modelPath: config.modelPath,
-        dimension: config.dimension,
-      });
-
-    case "none":
-      return null;
-
-    default:
-      return null;
-  }
-}