agentdb 3.0.0-alpha.2 → 3.0.0-alpha.20

package/dist/src/services/enhanced-embeddings.d.ts

@@ -0,0 +1,271 @@
+/**
+ * Enhanced Embedding Service with RuVector Integration
+ *
+ * A comprehensive embedding service supporting:
+ * - Multiple providers (@huggingface/transformers, OpenAI, Cohere, custom)
+ * - LRU cache with O(1) operations using doubly-linked list
+ * - Batch processing with semaphore-controlled parallelism
+ * - Text pre-processing pipeline (normalization, chunking, deduplication)
+ * - Direct RuVector storage and hybrid search
+ * - Lazy model loading with runtime switching
+ */
+import type { VectorBackend } from '../backends/VectorBackend.js';
+/** @inline Maximum allowed vector dimension for bounds checking */
+export declare const MAX_VECTOR_DIMENSION = 4096;
+/** @inline Maximum allowed cache size to prevent memory exhaustion */
+export declare const MAX_CACHE_SIZE = 100000;
+/** @inline Default cache size for embedding storage */
+export declare const DEFAULT_CACHE_SIZE = 10000;
+/** @inline Maximum batch size to prevent memory exhaustion */
+export declare const MAX_BATCH_SIZE = 10000;
+/**
+ * Supported embedding models
+ */
+export type SupportedModel = 'all-MiniLM-L6-v2' | 'all-mpnet-base-v2' | 'bge-small-en-v1.5' | string;
+/**
+ * Embedding provider types
+ */
+export type EmbeddingProvider = 'transformers' | 'openai' | 'cohere' | 'custom';
+/**
+ * Configuration for the enhanced embedding service (user input)
+ */
+export interface EnhancedEmbeddingConfig {
+    /** Embedding provider (default: 'transformers') */
+    provider?: EmbeddingProvider;
+    /** Model name (default: 'all-MiniLM-L6-v2') */
+    model?: SupportedModel;
+    /** Vector dimension (auto-detected from model if not specified) */
+    dimension?: number;
+    /** API key for OpenAI/Cohere providers */
+    apiKey?: string;
+    /** Custom embedding function for 'custom' provider */
+    customEmbedder?: CustomEmbedder;
+    /** LRU cache configuration */
+    cache?: Partial<CacheConfig>;
+    /** Batch processing configuration */
+    batch?: Partial<BatchConfig>;
+    /** Pre-processing pipeline configuration */
+    preprocessing?: Partial<PreprocessingConfig>;
+    /** RuVector backend for direct storage */
+    vectorBackend?: VectorBackend;
+}
+/**
+ * Cache configuration
+ */
+export interface CacheConfig {
+    /** Maximum number of cached embeddings (default: 10000) */
+    maxSize: number;
+    /** Enable cache statistics tracking (default: true) */
+    trackStats: boolean;
+}
+/**
+ * Batch processing configuration
+ */
+export interface BatchConfig {
+    /** Batch size for processing (default: 32) */
+    batchSize: number;
+    /** Maximum concurrent batches (default: 4) */
+    maxConcurrency: number;
+    /** Enable progress callbacks (default: true) */
+    enableProgress: boolean;
+}
+/**
+ * Pre-processing pipeline configuration
+ */
+export interface PreprocessingConfig {
+    /** Enable text normalization (default: true) */
+    normalize: boolean;
+    /** Maximum text length before chunking (default: 512 tokens approx) */
+    maxLength: number;
+    /** Chunk overlap for long texts (default: 50 chars) */
+    chunkOverlap: number;
+    /** Enable deduplication (default: true) */
+    deduplicate: boolean;
+}
+/**
+ * Custom embedding function signature
+ */
+export type CustomEmbedder = (text: string) => Promise<Float32Array>;
+/**
+ * Search result from the embedding service
+ */
+export interface SearchResult {
+    /** Unique identifier */
+    id: string;
+    /** Original text (if stored) */
+    text?: string;
+    /** Similarity score (0-1, higher is better) */
+    similarity: number;
+    /** Raw distance from vector search */
+    distance: number;
+    /** Associated metadata */
+    metadata?: Record<string, unknown>;
+}
+/**
+ * Statistics for the embedding service
+ */
+export interface EmbeddingStats {
+    /** Total embeddings generated */
+    totalEmbeddings: number;
+    /** Cache statistics */
+    cache: {
+        hits: number;
+        misses: number;
+        hitRate: number;
+        size: number;
+        maxSize: number;
+    };
+    /** Batch processing statistics */
+    batch: {
+        totalBatches: number;
+        averageBatchSize: number;
+    };
+    /** Model information */
+    model: {
+        name: string;
+        dimension: number;
+        provider: EmbeddingProvider;
+        loaded: boolean;
+    };
+    /** Vector backend statistics (if connected) */
+    vectorBackend?: {
+        count: number;
+        backend: string;
+    };
+}
+/**
+ * Progress callback for batch operations
+ */
+export type ProgressCallback = (progress: {
+    current: number;
+    total: number;
+    percentage: number;
+}) => void;
+/**
+ * Enhanced Embedding Service with RuVector Integration
+ */
+export declare class EnhancedEmbeddingService {
+    private config;
+    private cache;
+    private semaphore;
+    private pipeline;
+    private modelLoaded;
+    private totalEmbeddings;
+    private totalBatches;
+    private totalBatchItems;
+    constructor(config?: EnhancedEmbeddingConfig);
+    /**
+     * Generate embedding for a single text
+     */
+    embed(text: string): Promise<Float32Array>;
+    /**
+     * Generate embeddings for multiple texts with batch processing
+     */
+    embedBatch(texts: string[], onProgress?: ProgressCallback): Promise<Float32Array[]>;
+    /**
+     * Search for similar texts using RuVector backend
+     */
+    search(query: string, k?: number): Promise<SearchResult[]>;
+    /**
+     * Store text with embedding in RuVector
+     */
+    store(id: string, text: string, metadata?: Record<string, unknown>): Promise<void>;
+    /**
+     * Store multiple texts with embeddings in RuVector (batch operation)
+     */
+    storeBatch(items: Array<{
+        id: string;
+        text: string;
+        metadata?: Record<string, unknown>;
+    }>, onProgress?: ProgressCallback): Promise<void>;
+    /**
+     * Hybrid search combining embedding similarity with metadata filters
+     */
+    hybridSearch(query: string, k?: number, filter?: Record<string, unknown>): Promise<SearchResult[]>;
+    /**
+     * Lazy load the embedding model
+     */
+    loadModel(): Promise<void>;
+    /**
+     * Switch to a different model at runtime
+     */
+    switchModel(model: SupportedModel, provider?: EmbeddingProvider): Promise<void>;
+    /**
+     * Get service statistics
+     */
+    getStats(): EmbeddingStats;
+    /**
+     * Clear the embedding cache
+     */
+    clearCache(): void;
+    /**
+     * Get current model dimension
+     */
+    getDimension(): number;
+    /**
+     * Pre-process text according to config
+     */
+    private preprocessText;
+    /**
+     * Chunk long text into smaller pieces with overlap
+     */
+    chunkText(text: string): string[];
+    /**
+     * Generate cache key for text
+     */
+    private getCacheKey;
+    /**
+     * Deduplicate texts and create mapping
+     */
+    private deduplicateTexts;
+    /**
+     * Generate embedding using configured provider
+     */
+    private generateEmbedding;
+    /**
+     * Process a batch of texts
+     */
+    private processBatch;
+    /**
+     * Load transformers.js model
+     */
+    private loadTransformersModel;
+    /**
+     * Embed with @huggingface/transformers (single)
+     */
+    private embedWithTransformers;
+    /**
+     * Batch embed with transformers
+     */
+    private batchEmbedWithTransformers;
+    /**
+     * Embed with OpenAI API
+     */
+    private embedWithOpenAI;
+    /**
+     * Batch embed with OpenAI API
+     */
+    private batchEmbedWithOpenAI;
+    /**
+     * Embed with Cohere API
+     */
+    private embedWithCohere;
+    /**
+     * Batch embed with Cohere API
+     */
+    private batchEmbedWithCohere;
+    /**
+     * Embed with custom function
+     */
+    private embedWithCustom;
+    /**
+     * Generate mock embedding for testing/fallback
+     */
+    private mockEmbedding;
+}
+/**
+ * Create an enhanced embedding service with default configuration
+ */
+export declare function createEmbeddingService(config?: EnhancedEmbeddingConfig): EnhancedEmbeddingService;
+export default EnhancedEmbeddingService;
+//# sourceMappingURL=enhanced-embeddings.d.ts.map

package/dist/src/services/enhanced-embeddings.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"enhanced-embeddings.d.ts","sourceRoot":"","sources":["../../../src/services/enhanced-embeddings.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAsC,MAAM,8BAA8B,CAAC;AAMtG,mEAAmE;AACnE,eAAO,MAAM,oBAAoB,OAAO,CAAC;AAEzC,sEAAsE;AACtE,eAAO,MAAM,cAAc,SAAU,CAAC;AAEtC,uDAAuD;AACvD,eAAO,MAAM,kBAAkB,QAAQ,CAAC;AAKxC,8DAA8D;AAC9D,eAAO,MAAM,cAAc,QAAQ,CAAC;AA4EpC;;GAEG;AACH,MAAM,MAAM,cAAc,GACtB,kBAAkB,GAClB,mBAAmB,GACnB,mBAAmB,GACnB,MAAM,CAAC;AAEX;;GAEG;AACH,MAAM,MAAM,iBAAiB,GAAG,cAAc,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,CAAC;AAEhF;;GAEG;AACH,MAAM,WAAW,uBAAuB;IACtC,mDAAmD;IACnD,QAAQ,CAAC,EAAE,iBAAiB,CAAC;IAE7B,+CAA+C;IAC/C,KAAK,CAAC,EAAE,cAAc,CAAC;IAEvB,mEAAmE;IACnE,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,0CAA0C;IAC1C,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB,sDAAsD;IACtD,cAAc,CAAC,EAAE,cAAc,CAAC;IAEhC,8BAA8B;IAC9B,KAAK,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;IAE7B,qCAAqC;IACrC,KAAK,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;IAE7B,4CAA4C;IAC5C,aAAa,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC,CAAC;IAE7C,0CAA0C;IAC1C,aAAa,CAAC,EAAE,aAAa,CAAC;CAC/B;AAiBD;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,2DAA2D;IAC3D,OAAO,EAAE,MAAM,CAAC;IAEhB,uDAAuD;IACvD,UAAU,EAAE,OAAO,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,8CAA8C;IAC9C,SAAS,EAAE,MAAM,CAAC;IAElB,8CAA8C;IAC9C,cAAc,EAAE,MAAM,CAAC;IAEvB,gDAAgD;IAChD,cAAc,EAAE,OAAO,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,gDAAgD;IAChD,SAAS,EAAE,OAAO,CAAC;IAEnB,uEAAuE;IACvE,SAAS,EAAE,MAAM,CAAC;IAElB,uDAAuD;IACvD,YAAY,EAAE,MAAM,CAAC;IAErB,2CAA2C;IAC3C,WAAW,EAAE,OAAO,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,YAAY,CAAC,CAAC;AAErE;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,wBAAwB;IACxB,EAAE,EAAE,MAAM,CAAC;IAEX,gCAAgC;IAChC,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd,+CAA+C;IAC/C,UAAU,EAAE,MAAM,CAAC;IAEnB,sCAAsC;IACtC,QAAQ,EAAE,MAAM,CAAC;IAEjB,0BAA0B;IAC1B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,iCAAiC;IACjC,eAAe,EAAE,MAAM,CAAC;IAExB,uBAAuB;IACvB,KAAK,EAAE;QACL,IAAI,EAAE,MAAM,CAAC;QACb,MAAM,EAAE,MAAM,CAAC;QACf,OAAO,EAAE,MAAM,CAAC;QAChB,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC;IAEF,kCAAkC;IAClC,KAAK,EAAE;QACL,YAAY,EAAE,MAAM,CAAC;QACrB,gBAAgB,EAAE,MAAM,CAAC;KAC1B,CAAC;IAEF,wBAAwB;IACxB,KAAK,EAAE;QACL,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,EAAE,MAAM,CAAC;QAClB,QAAQ,EAAE,iBAAiB,CAAC;QAC5B,MAAM,EAAE,OAAO,CAAC;KACjB,CAAC;IAEF,+CAA+C;IAC/C,aAAa,CAAC,EAAE;QACd,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,MAAM,gBAAgB,GAAG,CAAC,QAAQ,EAAE;IACxC,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;CACpB,KAAK,IAAI,CAAC;AAoRX;;GAEG;AACH,qBAAa,wBAAwB;IACnC,OAAO,CAAC,MAAM,CAA0B;IACxC,OAAO,CAAC,KAAK,CAAW;IACxB,OAAO,CAAC,SAAS,CAAY;IAC7B,OAAO,CAAC,QAAQ,CAAa;IAC7B,OAAO,CAAC,WAAW,CAAS;IAG5B,OAAO,CAAC,eAAe,CAAK;IAC5B,OAAO,CAAC,YAAY,CAAK;IACzB,OAAO,CAAC,eAAe,CAAK;gBAEhB,MAAM,GAAE,uBAA4B;IAwChD;;OAEG;IACG,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IAqBhD;;OAEG;IACG,UAAU,CACd,KAAK,EAAE,MAAM,EAAE,EACf,UAAU,CAAC,EAAE,gBAAgB,GAC5B,OAAO,CAAC,YAAY,EAAE,CAAC;IAwH1B;;OAEG;IACG,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,GAAE,MAAW,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAqBpE;;OAEG;IACG,KAAK,CACT,EAAE,EAAE,MAAM,EACV,IAAI,EAAE,MAAM,EACZ,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GACjC,OAAO,CAAC,IAAI,CAAC;IAahB;;OAEG;IACG,UAAU,CACd,KAAK,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;KAAE,CAAC,EAC9E,UAAU,CAAC,EAAE,gBAAgB,GAC5B,OAAO,CAAC,IAAI,CAAC;IAsBhB;;OAEG;IACG,YAAY,CAChB,KAAK,EAAE,MAAM,EACb,CAAC,GAAE,MAAW,EACd,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAC/B,OAAO,CAAC,YAAY,EAAE,CAAC;IAwB1B;;OAEG;IACG,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IAWhC;;OAEG;IACG,WAAW,CAAC,KAAK,EAAE,cAAc,EAAE,QAAQ,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC;IA2BrF;;OAEG;IACH,QAAQ,IAAI,cAAc;IA4B1B;;OAEG;IACH,UAAU,IAAI,IAAI;IAIlB;;OAEG;IACH,YAAY,IAAI,MAAM;IAQtB;;OAEG;IACH,OAAO,CAAC,cAAc;IA6BtB;;OAEG;IACH,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE;IAuBjC;;OAEG;IACH,OAAO,CAAC,WAAW;IAInB;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAqBxB;;OAEG;YACW,iBAAiB;IAe/B;;OAEG;YACW,YAAY;IAe1B;;OAEG;YACW,qBAAqB;IA4BnC;;OAEG;YACW,qBAAqB;IAiBnC;;OAEG;YACW,0BAA0B;IAuBxC;;OAEG;YACW,eAAe;IA+C7B;;OAEG;YACW,oBAAoB;IAsDlC;;OAEG;YACW,eAAe;IA6C7B;;OAEG;YACW,oBAAoB;IAkDlC;;OAEG;YACW,eAAe;IAQ7B;;OAEG;IACH,OAAO,CAAC,aAAa;CAmCtB;AAMD;;GAEG;AACH,wBAAgB,sBAAsB,CACpC,MAAM,CAAC,EAAE,uBAAuB,GAC/B,wBAAwB,CAE1B;AAMD,eAAe,wBAAwB,CAAC"}