agentic-flow 2.0.1-alpha.14 → 2.0.1-alpha.16

package/dist/intelligence/EmbeddingService.d.ts

@@ -1,17 +1,19 @@
 /**
  * EmbeddingService - Unified embedding interface for agentic-flow
  *
- * Supports two backends:
- * - Simple: Hash-based, ~0.01ms, not semantic
- * - ONNX: Real neural embeddings via ruvector-onnx-embeddings-wasm, ~10-50ms, semantic
+ * Uses ruvector@0.1.61+ for ONNX embeddings with:
+ * - SIMD128 acceleration (6x faster)
+ * - Parallel worker threads (7 workers)
+ * - all-MiniLM-L6-v2 model (384 dimensions)
  *
  * Configure via:
- * - AGENTIC_FLOW_EMBEDDINGS=simple|onnx (default: simple)
+ * - AGENTIC_FLOW_EMBEDDINGS=simple|onnx|auto (default: auto)
  * - AGENTIC_FLOW_EMBEDDING_MODEL=all-MiniLM-L6-v2 (default)
  */
-export type EmbeddingBackend = 'simple' | 'onnx' | 'onnx-native';
+export type EmbeddingBackend = 'simple' | 'onnx' | 'auto';
 export interface EmbeddingStats {
     backend: EmbeddingBackend;
+    effectiveBackend: EmbeddingBackend;
     dimension: number;
     totalEmbeddings: number;
     totalLatencyMs: number;
@@ -20,27 +22,50 @@ export interface EmbeddingStats {
     modelLoaded: boolean;
     modelName?: string;
     simdAvailable?: boolean;
+    parallelWorkers?: number;
+}
+export interface SimilarityResult {
+    similarity: number;
+    timeMs: number;
+}
+export interface SearchResult {
+    text: string;
+    index: number;
+    similarity: number;
+}
+export interface DuplicateGroup {
+    indices: number[];
+    texts: string[];
+    similarity: number;
 }
 export declare class EmbeddingService {
     private static instance;
     private backend;
+    private effectiveBackend;
     private dimension;
     private modelName;
-    private embedder;
-    private wasmModule;
-    private loadingPromise;
     private modelLoaded;
+    private loadingPromise;
     private totalEmbeddings;
     private totalLatencyMs;
     private cacheHits;
     private cache;
     private cacheEnabled;
+    private corpus;
     private constructor();
     static getInstance(): EmbeddingService;
     /**
-     * Get current backend
+     * Resolve the effective backend based on ONNX detection
+     */
+    private resolveBackend;
+    /**
+     * Get configured backend (may be 'auto')
      */
     getBackend(): EmbeddingBackend;
+    /**
+     * Get effective backend after detection
+     */
+    getEffectiveBackend(): EmbeddingBackend;
     /**
      * Get embedding dimension
      */
@@ -49,23 +74,56 @@ export declare class EmbeddingService {
      * Check if ONNX model is loaded
      */
     isModelLoaded(): boolean;
-    /**
-     * Initialize ONNX embedder (lazy load on first use)
-     */
-    initOnnx(): Promise<void>;
-    private _loadOnnxModel;
     /**
      * Generate embedding for text
+     * Auto-detects ONNX and uses it if available (default behavior)
      */
     embed(text: string): Promise<Float32Array>;
     /**
-     * Generate embeddings for multiple texts
+     * Generate embeddings for multiple texts (batch processing with parallel workers)
+     * Batch processing provides significant speedup with parallel ONNX workers
      */
     embedBatch(texts: string[]): Promise<Float32Array[]>;
     /**
      * Compute similarity between two texts
      */
     similarity(text1: string, text2: string): Promise<number>;
+    /**
+     * Compute NxN similarity matrix for a list of texts
+     * Uses parallel workers for ONNX backend
+     */
+    similarityMatrix(texts: string[]): Promise<number[][]>;
+    /**
+     * Build a corpus for semantic search
+     */
+    buildCorpus(texts: string[]): Promise<void>;
+    /**
+     * Semantic search against the corpus
+     * Returns top-k most similar texts
+     */
+    semanticSearch(query: string, topK?: number): Promise<SearchResult[]>;
+    /**
+     * Find near-duplicate texts in a list
+     * Groups texts with similarity above threshold
+     */
+    findDuplicates(texts: string[], threshold?: number): Promise<DuplicateGroup[]>;
+    /**
+     * K-means clustering of texts
+     * Returns cluster assignments and centroids
+     */
+    clusterTexts(texts: string[], k?: number, maxIterations?: number): Promise<{
+        clusters: number[];
+        centroids: Float32Array[];
+    }>;
+    /**
+     * Stream embeddings for large batches (memory efficient)
+     * Yields embeddings one at a time
+     */
+    streamEmbed(texts: string[], batchSize?: number): AsyncGenerator<{
+        index: number;
+        text: string;
+        embedding: Float32Array;
+    }>;
     /**
      * Simple hash-based embedding (fast, not semantic)
      */
@@ -82,14 +140,29 @@ export declare class EmbeddingService {
      * Clear cache
      */
     clearCache(): void;
+    /**
+     * Clear corpus
+     */
+    clearCorpus(): void;
+    /**
+     * Shutdown (cleanup workers)
+     */
+    shutdown(): Promise<void>;
     /**
      * Reset instance (for testing)
      */
-    static reset(): void;
+    static reset(): Promise<void>;
 }
 export declare function getEmbeddingService(): EmbeddingService;
 export declare function embed(text: string): Promise<Float32Array>;
 export declare function embedBatch(texts: string[]): Promise<Float32Array[]>;
 export declare function textSimilarity(text1: string, text2: string): Promise<number>;
 export declare function simpleEmbed(text: string, dim?: number): Float32Array;
+export declare function similarityMatrix(texts: string[]): Promise<number[][]>;
+export declare function semanticSearch(query: string, topK?: number): Promise<SearchResult[]>;
+export declare function findDuplicates(texts: string[], threshold?: number): Promise<DuplicateGroup[]>;
+export declare function clusterTexts(texts: string[], k?: number): Promise<{
+    clusters: number[];
+    centroids: Float32Array[];
+}>;
 //# sourceMappingURL=EmbeddingService.d.ts.map

package/dist/intelligence/EmbeddingService.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"EmbeddingService.d.ts","sourceRoot":"","sources":["../../src/intelligence/EmbeddingService.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAiDH,MAAM,MAAM,gBAAgB,GAAG,QAAQ,GAAG,MAAM,GAAG,aAAa,CAAC;AAEjE,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,gBAAgB,CAAC;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,OAAO,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAyCD,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAiC;IAExD,OAAO,CAAC,OAAO,CAAmB;IAClC,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,SAAS,CAAS;IAG1B,OAAO,CAAC,QAAQ,CAA6B;IAC7C,OAAO,CAAC,UAAU,CAA2B;IAC7C,OAAO,CAAC,cAAc,CAA8B;IACpD,OAAO,CAAC,WAAW,CAAkB;IAGrC,OAAO,CAAC,eAAe,CAAa;IACpC,OAAO,CAAC,cAAc,CAAa;IACnC,OAAO,CAAC,SAAS,CAAa;IAG9B,OAAO,CAAC,KAAK,CAAiB;IAC9B,OAAO,CAAC,YAAY,CAAU;IAE9B,OAAO;IAQP,MAAM,CAAC,WAAW,IAAI,gBAAgB;IAOtC;;OAEG;IACH,UAAU,IAAI,gBAAgB;IAI9B;;OAEG;IACH,YAAY,IAAI,MAAM;IAItB;;OAEG;IACH,aAAa,IAAI,OAAO;IAIxB;;OAEG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;YAQjB,cAAc;IA+C5B;;OAEG;IACG,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IAqChD;;OAEG;IACG,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAkB1D;;OAEG;IACG,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAa/D;;OAEG;IACH,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,GAAE,MAAY,GAAG,YAAY;IAwB1D;;OAEG;IACH,gBAAgB,CAAC,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,YAAY,GAAG,MAAM;IAiB1D;;OAEG;IACH,QAAQ,IAAI,cAAc;IAc1B;;OAEG;IACH,UAAU,IAAI,IAAI;IAIlB;;OAEG;IACH,MAAM,CAAC,KAAK,IAAI,IAAI;CAMrB;AAGD,wBAAgB,mBAAmB,IAAI,gBAAgB,CAEtD;AAGD,wBAAsB,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAE/D;AAED,wBAAsB,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAEzE;AAED,wBAAsB,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAElF;AAED,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,GAAE,MAAY,GAAG,YAAY,CAEzE"}
+{"version":3,"file":"EmbeddingService.d.ts","sourceRoot":"","sources":["../../src/intelligence/EmbeddingService.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,MAAM,MAAM,gBAAgB,GAAG,QAAQ,GAAG,MAAM,GAAG,MAAM,CAAC;AAE1D,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,gBAAgB,CAAC;IAC1B,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,SAAS,EAAE,MAAM,CAAC;IAClB,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,OAAO,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;CACpB;AAyFD,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAiC;IAExD,OAAO,CAAC,OAAO,CAAmB;IAClC,OAAO,CAAC,gBAAgB,CAAiC;IACzD,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,SAAS,CAAS;IAG1B,OAAO,CAAC,WAAW,CAAkB;IACrC,OAAO,CAAC,cAAc,CAA8B;IAGpD,OAAO,CAAC,eAAe,CAAa;IACpC,OAAO,CAAC,cAAc,CAAa;IACnC,OAAO,CAAC,SAAS,CAAa;IAG9B,OAAO,CAAC,KAAK,CAAiB;IAC9B,OAAO,CAAC,YAAY,CAAU;IAG9B,OAAO,CAAC,MAAM,CAAkF;IAEhG,OAAO;IASP,MAAM,CAAC,WAAW,IAAI,gBAAgB;IAOtC;;OAEG;YACW,cAAc;IAsB5B;;OAEG;IACH,UAAU,IAAI,gBAAgB;IAI9B;;OAEG;IACH,mBAAmB,IAAI,gBAAgB;IAIvC;;OAEG;IACH,YAAY,IAAI,MAAM;IAItB;;OAEG;IACH,aAAa,IAAI,OAAO;IAIxB;;;OAGG;IACG,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IAwChD;;;OAGG;IACG,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAyC1D;;OAEG;IACG,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAa/D;;;OAGG;IACG,gBAAgB,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;IAiB5D;;OAEG;IACG,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAKjD;;;OAGG;IACG,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,GAAE,MAAU,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAsB9E;;;OAGG;IACG,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,GAAE,MAAY,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAoCzF;;;OAGG;IACG,YAAY,CAChB,KAAK,EAAE,MAAM,EAAE,EACf,CAAC,GAAE,MAAU,EACb,aAAa,GAAE,MAAY,GAC1B,OAAO,CAAC;QAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;QAAC,SAAS,EAAE,YAAY,EAAE,CAAA;KAAE,CAAC;IAyE7D;;;OAGG;IACI,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,GAAE,MAAW,GAAG,cAAc,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,YAAY,CAAA;KAAE,CAAC;IAerI;;OAEG;IACH,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,GAAE,MAAY,GAAG,YAAY;IAwB1D;;OAEG;IACH,gBAAgB,CAAC,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,YAAY,GAAG,MAAM;IAiB1D;;OAEG;IACH,QAAQ,IAAI,cAAc;IAmB1B;;OAEG;IACH,UAAU,IAAI,IAAI;IAIlB;;OAEG;IACH,WAAW,IAAI,IAAI;IAInB;;OAEG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAM/B;;OAEG;WACU,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAQpC;AAGD,wBAAgB,mBAAmB,IAAI,gBAAgB,CAEtD;AAGD,wBAAsB,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAE/D;AAED,wBAAsB,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAEzE;AAED,wBAAsB,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAElF;AAED,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,GAAE,MAAY,GAAG,YAAY,CAEzE;AAED,wBAAsB,gBAAgB,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAE3E;AAED,wBAAsB,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,GAAE,MAAU,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAE7F;AAED,wBAAsB,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,GAAE,MAAY,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAExG;AAED,wBAAsB,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,GAAE,MAAU,GAAG,OAAO,CAAC;IAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;IAAC,SAAS,EAAE,YAAY,EAAE,CAAA;CAAE,CAAC,CAE7H"}