kiro-memory 1.8.1 → 2.1.0

package/plugin/dist/services/search/VectorSearch.js

@@ -226,8 +226,8 @@ var EmbeddingService = class {
   initialized = false;
   initializing = null;
   /**
-   * Inizializza il servizio di embedding.
-   * Tenta fastembed, poi @huggingface/transformers, poi fallback a null.
+   * Initialize the embedding service.
+   * Tries fastembed, then @huggingface/transformers, then fallback to null.
    */
   async initialize() {
     if (this.initialized) return this.provider !== null;
@@ -248,11 +248,11 @@ var EmbeddingService = class {
         });
         this.provider = "fastembed";
         this.initialized = true;
-        logger.info("EMBEDDING", "Inizializzato con fastembed (BGE-small-en-v1.5)");
+        logger.info("EMBEDDING", "Initialized with fastembed (BGE-small-en-v1.5)");
         return true;
       }
     } catch (error) {
-      logger.debug("EMBEDDING", `fastembed non disponibile: ${error}`);
+      logger.debug("EMBEDDING", `fastembed not available: ${error}`);
     }
     try {
       const transformers = await import("@huggingface/transformers");
@@ -263,20 +263,20 @@ var EmbeddingService = class {
         });
         this.provider = "transformers";
         this.initialized = true;
-        logger.info("EMBEDDING", "Inizializzato con @huggingface/transformers (all-MiniLM-L6-v2)");
+        logger.info("EMBEDDING", "Initialized with @huggingface/transformers (all-MiniLM-L6-v2)");
         return true;
       }
     } catch (error) {
-      logger.debug("EMBEDDING", `@huggingface/transformers non disponibile: ${error}`);
+      logger.debug("EMBEDDING", `@huggingface/transformers not available: ${error}`);
     }
     this.provider = null;
     this.initialized = true;
-    logger.warn("EMBEDDING", "Nessun provider embedding disponibile, ricerca semantica disabilitata");
+    logger.warn("EMBEDDING", "No embedding provider available, semantic search disabled");
     return false;
   }
   /**
-   * Genera embedding per un singolo testo.
-   * Ritorna Float32Array con 384 dimensioni, o null se non disponibile.
+   * Generate embedding for a single text.
+   * Returns Float32Array with 384 dimensions, or null if not available.
    */
   async embed(text) {
     if (!this.initialized) await this.initialize();
@@ -289,46 +289,111 @@ var EmbeddingService = class {
         return await this._embedTransformers(truncated);
       }
     } catch (error) {
-      logger.error("EMBEDDING", `Errore generazione embedding: ${error}`);
+      logger.error("EMBEDDING", `Error generating embedding: ${error}`);
     }
     return null;
   }
   /**
-   * Genera embeddings in batch.
+   * Generate embeddings in batch.
+   * Uses native batch support when available (fastembed, transformers),
+   * falls back to serial processing on batch failure.
    */
   async embedBatch(texts) {
     if (!this.initialized) await this.initialize();
     if (!this.provider || !this.model) return texts.map(() => null);
-    const results = [];
-    for (const text of texts) {
-      try {
-        const embedding = await this.embed(text);
-        results.push(embedding);
-      } catch {
-        results.push(null);
+    if (texts.length === 0) return [];
+    const truncated = texts.map((t) => t.substring(0, 2e3));
+    try {
+      if (this.provider === "fastembed") {
+        return await this._embedBatchFastembed(truncated);
+      } else if (this.provider === "transformers") {
+        return await this._embedBatchTransformers(truncated);
       }
+    } catch (error) {
+      logger.warn("EMBEDDING", `Batch embedding failed, falling back to serial: ${error}`);
     }
-    return results;
+    return this._embedBatchSerial(truncated);
   }
   /**
-   * Verifica se il servizio è disponibile.
+   * Check if the service is available.
    */
   isAvailable() {
     return this.initialized && this.provider !== null;
   }
   /**
-   * Nome del provider attivo.
+   * Name of the active provider.
    */
   getProvider() {
     return this.provider;
   }
   /**
-   * Dimensioni del vettore embedding.
+   * Embedding vector dimensions.
    */
   getDimensions() {
     return 384;
   }
-  // --- Provider specifici ---
+  // --- Batch implementations ---
+  /**
+   * Native batch embedding with fastembed.
+   * FlagEmbedding.embed() accepts string[] and returns an async iterable of batches.
+   */
+  async _embedBatchFastembed(texts) {
+    const results = [];
+    const embeddings = this.model.embed(texts, texts.length);
+    for await (const batch of embeddings) {
+      if (batch) {
+        for (const vec of batch) {
+          results.push(vec instanceof Float32Array ? vec : new Float32Array(vec));
+        }
+      }
+    }
+    while (results.length < texts.length) {
+      results.push(null);
+    }
+    return results;
+  }
+  /**
+   * Batch embedding with @huggingface/transformers pipeline.
+   * The pipeline accepts string[] and returns a Tensor with shape [N, dims].
+   */
+  async _embedBatchTransformers(texts) {
+    const output = await this.model(texts, {
+      pooling: "mean",
+      normalize: true
+    });
+    if (!output?.data) {
+      return texts.map(() => null);
+    }
+    const dims = this.getDimensions();
+    const data = output.data instanceof Float32Array ? output.data : new Float32Array(output.data);
+    const results = [];
+    for (let i = 0; i < texts.length; i++) {
+      const offset = i * dims;
+      if (offset + dims <= data.length) {
+        results.push(data.slice(offset, offset + dims));
+      } else {
+        results.push(null);
+      }
+    }
+    return results;
+  }
+  /**
+   * Serial fallback: embed texts one at a time.
+   * Used when native batch fails.
+   */
+  async _embedBatchSerial(texts) {
+    const results = [];
+    for (const text of texts) {
+      try {
+        const embedding = await this.embed(text);
+        results.push(embedding);
+      } catch {
+        results.push(null);
+      }
+    }
+    return results;
+  }
+  // --- Single-text provider implementations ---
   async _embedFastembed(text) {
     const embeddings = this.model.embed([text], 1);
     for await (const batch of embeddings) {
@@ -359,17 +424,21 @@ function getEmbeddingService() {
 }
 
 // src/services/search/VectorSearch.ts
+var DEFAULT_MAX_CANDIDATES = 2e3;
 function cosineSimilarity(a, b) {
-  if (a.length !== b.length) return 0;
+  const len = a.length;
+  if (len !== b.length) return 0;
   let dotProduct = 0;
   let normA = 0;
   let normB = 0;
-  for (let i = 0; i < a.length; i++) {
-    dotProduct += a[i] * b[i];
-    normA += a[i] * a[i];
-    normB += b[i] * b[i];
-  }
-  const denominator = Math.sqrt(normA) * Math.sqrt(normB);
+  for (let i = 0; i < len; i++) {
+    const ai = a[i];
+    const bi = b[i];
+    dotProduct += ai * bi;
+    normA += ai * ai;
+    normB += bi * bi;
+  }
+  const denominator = Math.sqrt(normA * normB);
   if (denominator === 0) return 0;
   return dotProduct / denominator;
 }
@@ -382,23 +451,36 @@ function bufferToFloat32(buf) {
 }
 var VectorSearch = class {
   /**
-   * Ricerca semantica: calcola cosine similarity tra query e tutti gli embeddings.
+   * Semantic search with SQL pre-filtering for scalability.
+   *
+   * 2-phase strategy:
+   * 1. SQL pre-filters by project + sorts by recency (loads max N candidates)
+   * 2. JS computes cosine similarity only on filtered candidates
+   *
+   * With 50k observations and maxCandidates=2000, loads only ~4% of data.
    */
   async search(db, queryEmbedding, options = {}) {
     const limit = options.limit || 10;
     const threshold = options.threshold || 0.3;
+    const maxCandidates = options.maxCandidates || DEFAULT_MAX_CANDIDATES;
     try {
-      let sql = `
+      const conditions = [];
+      const params = [];
+      if (options.project) {
+        conditions.push("o.project = ?");
+        params.push(options.project);
+      }
+      const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
+      const sql = `
         SELECT e.observation_id, e.embedding,
                o.title, o.text, o.type, o.project, o.created_at, o.created_at_epoch
         FROM observation_embeddings e
         JOIN observations o ON o.id = e.observation_id
+        ${whereClause}
+        ORDER BY o.created_at_epoch DESC
+        LIMIT ?
       `;
-      const params = [];
-      if (options.project) {
-        sql += " WHERE o.project = ?";
-        params.push(options.project);
-      }
+      params.push(maxCandidates);
       const rows = db.query(sql).all(...params);
       const scored = [];
       for (const row of rows) {
@@ -419,14 +501,15 @@ var VectorSearch = class {
         }
       }
       scored.sort((a, b) => b.similarity - a.similarity);
+      logger.debug("VECTOR", `Search: ${rows.length} candidates \u2192 ${scored.length} above threshold \u2192 ${Math.min(scored.length, limit)} results`);
       return scored.slice(0, limit);
     } catch (error) {
-      logger.error("VECTOR", `Errore ricerca vettoriale: ${error}`);
+      logger.error("VECTOR", `Vector search error: ${error}`);
       return [];
     }
   }
   /**
-   * Salva embedding per un'osservazione.
+   * Store embedding for an observation.
    */
   async storeEmbedding(db, observationId, embedding, model) {
     try {
@@ -442,18 +525,18 @@ var VectorSearch = class {
         embedding.length,
         (/* @__PURE__ */ new Date()).toISOString()
       );
-      logger.debug("VECTOR", `Embedding salvato per osservazione ${observationId}`);
+      logger.debug("VECTOR", `Embedding saved for observation ${observationId}`);
     } catch (error) {
-      logger.error("VECTOR", `Errore salvataggio embedding: ${error}`);
+      logger.error("VECTOR", `Error saving embedding: ${error}`);
     }
   }
   /**
-   * Genera embeddings per osservazioni che non li hanno ancora.
+   * Generate embeddings for observations that don't have them yet.
    */
   async backfillEmbeddings(db, batchSize = 50) {
     const embeddingService2 = getEmbeddingService();
     if (!await embeddingService2.initialize()) {
-      logger.warn("VECTOR", "Embedding service non disponibile, backfill saltato");
+      logger.warn("VECTOR", "Embedding service not available, backfill skipped");
       return 0;
     }
     const rows = db.query(`
@@ -479,11 +562,11 @@ var VectorSearch = class {
         count++;
       }
     }
-    logger.info("VECTOR", `Backfill completato: ${count}/${rows.length} embeddings generati`);
+    logger.info("VECTOR", `Backfill completed: ${count}/${rows.length} embeddings generated`);
     return count;
   }
   /**
-   * Statistiche sugli embeddings.
+   * Embedding statistics.
    */
   getStats(db) {
     try {