@goshenkata/dryscan-core 1.2.4 → 1.2.6

package/src/services/DuplicationCache.ts

@@ -1,4 +1,8 @@
-import { DuplicateGroup } from "../types";
+import debug from "debug";
+import { cosineSimilarity } from "@langchain/core/utils/math";
+import { DuplicateGroup, IndexUnit } from "../types";
+
+const log = debug("DryScan:DuplicationCache");
 
 /**
  * In-memory cache for duplicate comparison scores.
@@ -11,6 +15,13 @@ export class DuplicationCache {
   private readonly fileIndex = new Map<string, Set<string>>();
   private initialized = false;
 
+  /** Per-run similarity matrix from a single batched library call (reset each run). */
+  private embSimMatrix: number[][] = [];
+  /** Maps unit ID to its row/column index in embSimMatrix. */
+  private embSimIndex = new Map<string, number>();
+  /** Per-run memoization of parent unit similarity scores (reset each run). */
+  private parentSimCache = new Map<string, number>();
+
   static getInstance(): DuplicationCache {
     if (!DuplicationCache.instance) {
       DuplicationCache.instance = new DuplicationCache();
@@ -85,6 +96,101 @@ export class DuplicationCache {
     this.comparisons.clear();
     this.fileIndex.clear();
     this.initialized = false;
+    this.embSimMatrix = [];
+    this.embSimIndex.clear();
+    this.clearRunCaches();
+  }
+
+  /**
+   * Resets per-run memoization (parent similarities).
+   * The embedding matrix is intentionally preserved so incremental runs can
+   * reuse clean×clean values across calls.
+   */
+  clearRunCaches(): void {
+    this.parentSimCache.clear();
+  }
+
+  /**
+   * Builds or incrementally updates the embedding similarity matrix.
+   *
+   * Full rebuild (default): replaces the entire matrix — O(n²).
+   * Incremental (dirtyPaths provided + prior matrix exists): copies clean×clean
+   * cells from the old matrix and recomputes only dirty rows via one batched
+   * cosineSimilarity call — O(d·n) where d = number of dirty units.
+   */
+  buildEmbSimCache(units: IndexUnit[], dirtyPaths?: string[]): void {
+    const embedded = units.filter(u => Array.isArray(u.embedding) && u.embedding.length > 0);
+    if (embedded.length < 2) {
+      this.embSimMatrix = [];
+      this.embSimIndex.clear();
+      return;
+    }
+
+    const embeddings = embedded.map(u => u.embedding as number[]);
+    const newIndex = new Map(embedded.map((u, i) => [u.id, i] as [string, number]));
+    const dirtySet = dirtyPaths ? new Set(dirtyPaths) : null;
+    const hasPriorMatrix = this.embSimMatrix.length > 0;
+
+    if (!dirtySet || !hasPriorMatrix) {
+      // Full rebuild
+      this.embSimIndex = newIndex;
+      this.embSimMatrix = cosineSimilarity(embeddings, embeddings);
+      log("Built full embedding similarity matrix: %d units", embedded.length);
+      return;
+    }
+
+    // Incremental: identify dirty unit IDs
+    const dirtyIds = new Set(embedded.filter(u => dirtySet.has(u.filePath)).map(u => u.id));
+
+    if (dirtyIds.size === 0) {
+      log("Matrix reused: no dirty units detected");
+      return;
+    }
+
+    const n = embedded.length;
+
+    // Start with zeroes; copy clean×clean values from prior matrix
+    const newMatrix: number[][] = Array.from({ length: n }, () => new Array(n).fill(0));
+    for (let i = 0; i < n; i++) {
+      for (let j = 0; j < n; j++) {
+        if (dirtyIds.has(embedded[i].id) || dirtyIds.has(embedded[j].id)) continue;
+        const oi = this.embSimIndex.get(embedded[i].id);
+        const oj = this.embSimIndex.get(embedded[j].id);
+        if (oi !== undefined && oj !== undefined) newMatrix[i][j] = this.embSimMatrix[oi][oj];
+      }
+    }
+
+    // Recompute dirty rows in one batched call
+    const dirtyIndices = embedded.reduce<number[]>((acc, u, i) => (dirtyIds.has(u.id) ? [...acc, i] : acc), []);
+    const dirtyRows = cosineSimilarity(dirtyIndices.map(i => embeddings[i]), embeddings);
+    dirtyIndices.forEach((rowIdx, di) => {
+      for (let j = 0; j < n; j++) {
+        newMatrix[rowIdx][j] = dirtyRows[di][j];
+        newMatrix[j][rowIdx] = dirtyRows[di][j];
+      }
+    });
+
+    this.embSimIndex = newIndex;
+    this.embSimMatrix = newMatrix;
+    log("Incremental matrix update: %d dirty unit(s) out of %d total", dirtyIds.size, n);
+  }
+
+  /** Returns the pre-computed cosine similarity for a pair of unit IDs, if available. */
+  getEmbSim(id1: string, id2: string): number | undefined {
+    const i = this.embSimIndex.get(id1);
+    const j = this.embSimIndex.get(id2);
+    if (i === undefined || j === undefined) return undefined;
+    return this.embSimMatrix[i][j];
+  }
+
+  /** Returns the memoized parent similarity for the given stable key, if available. */
+  getParentSim(key: string): number | undefined {
+    return this.parentSimCache.get(key);
+  }
+
+  /** Stores a memoized parent similarity for the given stable key. */
+  setParentSim(key: string, sim: number): void {
+    this.parentSimCache.set(key, sim);
   }
 
   private addKeyForFile(filePath: string, key: string): void {

package/src/services/UpdateService.ts

@@ -12,14 +12,17 @@ export class UpdateService {
     private readonly exclusionService: ExclusionService
   ) {}
 
-  async updateIndex(): Promise<void> {
+  /** Returns the list of file paths that were modified or deleted (dirty). */
+  async updateIndex(): Promise<string[]> {
     const extractor = this.deps.extractor;
     const cache = DuplicationCache.getInstance();
 
     try {
       const changeSet = await performIncrementalUpdate(this.deps.repoPath, extractor, this.deps.db);
       await this.exclusionService.cleanupExcludedFiles();
-      await cache.invalidate([...changeSet.changed, ...changeSet.deleted]);
+      const dirtyPaths = [...changeSet.changed, ...changeSet.deleted, ...changeSet.added];
+      await cache.invalidate(dirtyPaths);
+      return dirtyPaths;
     } catch (err) {
       log("Error during index update:", err);
       throw err;