@goshenkata/dryscan-core 1.4.7 → 1.4.8

package/dist/services/ParallelSimilarity.d.ts

@@ -1,6 +1,7 @@
 /**
- * Computes cosine similarity using a deterministic backend chain.
- * Default chain: GPU -> worker threads -> synchronous fallback.
+ * Computes cosineSimilarity(A, B) using worker threads for large inputs,
+ * falling back to the synchronous library call for small ones.
+ * B is packed into a SharedArrayBuffer shared across all workers — no copies.
  */
 declare function parallelCosineSimilarity(A: number[][], B: number[][]): Promise<number[][]>;
 

package/dist/services/ParallelSimilarity.js

@@ -3,192 +3,20 @@ import "../chunk-EUXUH3YW.js";
 // src/services/ParallelSimilarity.ts
 import os from "os";
 import { Worker } from "worker_threads";
-import debug from "debug";
 import { cosineSimilarity } from "@langchain/core/utils/math";
-var log = debug("DryScan:ParallelSimilarity");
-var GPU_TARGET_CELLS = 2e6;
-var ZERO_NORM_EPSILON = 1e-12;
-var gpuKernel = null;
-var gpuRuntime = null;
-var gpuKernelDims = null;
-var gpuCtor = null;
-var gpuCapabilitiesLogged = false;
-var normalizedMatrixCache = /* @__PURE__ */ new WeakMap();
-async function getGpuCtor() {
-  if (gpuCtor) return gpuCtor;
-  const moduleName = "gpu.js";
-  try {
-    const mod = await import(moduleName);
-    gpuCtor = mod.GPU;
-    return gpuCtor;
-  } catch (_err) {
-    return null;
-  }
-}
-function logGpuCapabilities(GPU) {
-  if (gpuCapabilitiesLogged) return;
-  const flags = GPU;
-  log(
-    "SIM_GPU capabilities supported=%s headlessgl=%s webgl=%s webgl2=%s singlePrecision=%s",
-    String(flags.isGPUSupported),
-    String(flags.isHeadlessGLSupported),
-    String(flags.isWebGLSupported),
-    String(flags.isWebGL2Supported),
-    String(flags.isSinglePrecisionSupported)
-  );
-  gpuCapabilitiesLogged = true;
-}
-function backendPreference() {
-  const value = (process.env.DRYSCAN_SIM_BACKEND ?? "auto").toLowerCase();
-  if (value === "gpu") return "gpu";
-  if (value === "worker") return "worker";
-  if (value === "sync") return "sync";
-  return "auto";
-}
-function matrixFromGpuResult(result, rows, cols) {
-  if (!Array.isArray(result)) {
-    throw new Error("GPU kernel returned a non-array result");
-  }
-  return Array.from({ length: rows }, (_, rowIdx) => {
-    const row = result[rowIdx];
-    if (!row || typeof row !== "object") {
-      throw new Error("GPU kernel returned malformed rows");
-    }
-    const typed = row;
-    return Array.from({ length: cols }, (_2, colIdx) => Number(typed[colIdx] ?? 0));
-  });
-}
-function normalizeRows(rows, dims) {
-  return rows.map((row) => {
-    const normalized = new Float32Array(dims);
-    let normSq = 0;
-    for (let i = 0; i < dims; i++) {
-      const value = Number(row[i] ?? 0);
-      normalized[i] = value;
-      normSq += value * value;
-    }
-    const norm = Math.sqrt(normSq);
-    if (norm > ZERO_NORM_EPSILON) {
-      for (let i = 0; i < dims; i++) {
-        normalized[i] /= norm;
-      }
-    } else {
-      normalized.fill(0);
-    }
-    return normalized;
-  });
-}
-function getNormalizedRows(rows, dims) {
-  const cached = normalizedMatrixCache.get(rows);
-  if (cached && cached.dims === dims) {
-    return cached.rows;
-  }
-  const normalized = normalizeRows(rows, dims);
-  normalizedMatrixCache.set(rows, { dims, rows: normalized });
-  return normalized;
-}
-async function runGpuCosineSimilarity(A, B) {
-  const GPU = await getGpuCtor();
-  if (!GPU) {
-    throw new Error("gpu.js module not available");
-  }
-  logGpuCapabilities(GPU);
-  if (GPU.isGPUSupported === false) {
-    throw new Error("GPU.js reported GPU support unavailable");
-  }
-  const dims = A[0]?.length ?? 0;
-  if (!dims || !B[0]?.length || B[0].length !== dims) {
-    throw new Error("Matrix dimensions are invalid for GPU similarity");
-  }
-  const normalizeStartMs = performance.now();
-  const normalizedA = getNormalizedRows(A, dims);
-  const normalizedB = getNormalizedRows(B, dims);
-  const normalizeMs = Math.round(performance.now() - normalizeStartMs);
-  if (!gpuRuntime) {
-    gpuRuntime = new GPU({ mode: "gpu" });
-    const runtimeMode = String(gpuRuntime.mode ?? "unknown");
-    log("SIM_GPU runtime mode=%s", runtimeMode);
-    if (runtimeMode === "cpu") {
-      throw new Error("GPU runtime initialized in CPU mode");
-    }
-  }
-  const shouldRecreateKernel = !gpuKernel || gpuKernelDims !== dims;
-  if (shouldRecreateKernel) {
-    const kernel = gpuRuntime.createKernel(function(a, b) {
-      let dot = 0;
-      for (let i = 0; i < this.constants.dims; i++) {
-        const av = a[this.thread.y][i];
-        const bv = b[this.thread.x][i];
-        dot += av * bv;
-      }
-      return dot;
-    }).setOutput([1, 1]).setConstants({ dims });
-    if (typeof kernel.setPrecision === "function") {
-      kernel.setPrecision("single");
-    }
-    if (typeof kernel.setTactic === "function") {
-      kernel.setTactic("speed");
-    }
-    if (typeof kernel.setDynamicArguments === "function") {
-      kernel.setDynamicArguments(true);
-    }
-    if (typeof kernel.setDynamicOutput === "function") {
-      kernel.setDynamicOutput(true);
-    }
-    gpuKernel = kernel;
-    gpuKernelDims = dims;
-  }
-  if (!gpuKernel) {
-    throw new Error("GPU kernel initialization failed");
-  }
-  let targetCells = GPU_TARGET_CELLS;
-  if (!Number.isFinite(targetCells) || targetCells <= 0) {
-    targetCells = 2e6;
-  }
-  let rowsPerBatch = Math.floor(targetCells / normalizedB.length);
-  if (rowsPerBatch < 1) {
-    rowsPerBatch = 1;
-  }
-  const result = [];
-  const batches = Math.ceil(normalizedA.length / rowsPerBatch);
-  const kernelStartMs = performance.now();
-  for (let start = 0; start < normalizedA.length; start += rowsPerBatch) {
-    const batch = normalizedA.slice(start, start + rowsPerBatch);
-    gpuKernel.setOutput([normalizedB.length, batch.length]);
-    const raw = gpuKernel(batch, normalizedB);
-    result.push(...matrixFromGpuResult(raw, batch.length, normalizedB.length));
-  }
-  const kernelMs = Math.round(performance.now() - kernelStartMs);
-  log(
-    "SIM_GPU details dims=%d aRows=%d bRows=%d rowsPerBatch=%d batches=%d targetCells=%d normalizeMs=%d kernelMs=%d",
-    dims,
-    A.length,
-    B.length,
-    rowsPerBatch,
-    batches,
-    targetCells,
-    normalizeMs,
-    kernelMs
-  );
-  return result;
-}
-async function computeWithWorkers(A, B) {
+var MIN_PARALLEL_ROWS = 50;
+async function parallelCosineSimilarity(A, B) {
+  if (A.length === 0 || B.length === 0) return [];
+  if (A.length < MIN_PARALLEL_ROWS) return cosineSimilarity(A, B);
   const dims = A[0].length;
-  const cpuCount = Math.max(1, os.cpus().length);
-  const chunkSize = Math.max(1, Math.ceil(A.length / cpuCount));
+  const chunkSize = Math.ceil(A.length / os.cpus().length);
   const sharedB = new SharedArrayBuffer(B.length * dims * 8);
   const bView = new Float64Array(sharedB);
   B.forEach((row, i) => bView.set(row, i * dims));
   const runningTsSource = new URL(import.meta.url).pathname.endsWith(".ts");
-  let workerFile = "./cosineSimilarityWorker.js";
-  if (runningTsSource) {
-    workerFile = "./cosineSimilarityWorker.ts";
-  }
+  const workerFile = runningTsSource ? "./cosineSimilarityWorker.ts" : "./cosineSimilarityWorker.js";
   const workerUrl = new URL(workerFile, import.meta.url);
-  let execArgv = [];
-  if (runningTsSource) {
-    execArgv = ["--import", "tsx/esm"];
-  }
+  const execArgv = runningTsSource ? ["--import", "tsx/esm"] : [];
   const chunks = Array.from(
     { length: Math.ceil(A.length / chunkSize) },
     (_, i) => A.slice(i * chunkSize, (i + 1) * chunkSize)
@@ -196,50 +24,6 @@ async function computeWithWorkers(A, B) {
   const results = await Promise.all(chunks.map((chunk) => runWorker(chunk, sharedB, B.length, dims, workerUrl, execArgv)));
   return results.flat();
 }
-function computeSequential(A, B) {
-  return cosineSimilarity(A, B);
-}
-async function runWithTiming(name, fn) {
-  const startMs = performance.now();
-  log("SIM_TRY backend=%s", name);
-  try {
-    const result = await Promise.resolve(fn());
-    const durationMs = Math.round(performance.now() - startMs);
-    log("SIM_DONE backend=%s durationMs=%d", name, durationMs);
-    return result;
-  } catch (err) {
-    const durationMs = Math.round(performance.now() - startMs);
-    log("SIM_FAIL backend=%s durationMs=%d reason=%s", name, durationMs, err?.message ?? "unknown");
-    throw err;
-  }
-}
-async function parallelCosineSimilarity(A, B) {
-  if (A.length === 0 || B.length === 0) return [];
-  const preference = backendPreference();
-  const dims = A[0]?.length ?? 0;
-  log("SIM_START rows=%d cols=%d dims=%d preference=%s", A.length, B.length, dims, preference);
-  if (preference === "sync") {
-    return runWithTiming("sync", () => computeSequential(A, B));
-  }
-  if (preference === "worker") {
-    try {
-      return await runWithTiming("worker", () => computeWithWorkers(A, B));
-    } catch (_workerErr) {
-      return runWithTiming("sync", () => computeSequential(A, B));
-    }
-  }
-  try {
-    return await runWithTiming("gpu", () => runGpuCosineSimilarity(A, B));
-  } catch (_gpuErr) {
-    log("SIM_CHAIN continue_after=gpu-fail next=worker");
-  }
-  try {
-    return await runWithTiming("worker", () => computeWithWorkers(A, B));
-  } catch (_workerErr) {
-    log("SIM_CHAIN continue_after=worker-fail next=sync");
-    return runWithTiming("sync", () => computeSequential(A, B));
-  }
-}
 function runWorker(chunk, sharedB, bCount, dims, workerUrl, execArgv) {
   return new Promise((resolve, reject) => {
     const rowsFlat = new Float64Array(chunk.length * dims);

package/dist/services/ParallelSimilarity.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/services/ParallelSimilarity.ts"],"sourcesContent":["import os from \"node:os\";\nimport { Worker } from \"node:worker_threads\";\nimport debug from \"debug\";\nimport { cosineSimilarity } from \"@langchain/core/utils/math\";\n\nconst log = debug(\"DryScan:ParallelSimilarity\");\n\n/**\n * Target number of output cells per GPU launch.\n * Large enough to keep GPU occupancy healthy, bounded enough to avoid oversized output buffers.\n */\nconst GPU_TARGET_CELLS = 2_000_000;\nconst ZERO_NORM_EPSILON = 1e-12;\n\ntype BackendPreference = \"auto\" | \"gpu\" | \"worker\" | \"sync\";\ntype GPUCtor = new (settings?: { mode?: string }) => {\n  createKernel: (fn: (...args: any[]) => number) => GpuKernel;\n};\ntype GPUModule = {\n  GPU: GPUCtor & { isGPUSupported?: boolean };\n};\n\ntype GpuInputMatrix = ArrayLike<ArrayLike<number>>;\ntype GpuKernel = {\n  (a: GpuInputMatrix, b: GpuInputMatrix): number[][] | Float32Array[] | number[];\n  setOutput: (output: [number, number]) => GpuKernel;\n  setConstants: (constants: { dims: number }) => GpuKernel;\n  setDynamicOutput?: (enabled: boolean) => GpuKernel;\n};\n\nlet gpuKernel:\n  | GpuKernel\n  | null = null;\nlet gpuRuntime: InstanceType<GPUCtor> | null = null;\nlet gpuKernelDims: number | null = null;\nlet gpuCtor: (GPUCtor & { isGPUSupported?: boolean }) | null = null;\nlet gpuCapabilitiesLogged = false;\n\nconst normalizedMatrixCache = new WeakMap<number[][], { dims: number; rows: Float32Array[] }>();\n\ntype BackendName = \"gpu\" | \"worker\" | \"sync\";\n\nasync function getGpuCtor(): Promise<(GPUCtor & { isGPUSupported?: boolean }) | null> {\n  if (gpuCtor) return gpuCtor;\n\n  const moduleName = \"gpu.js\";\n  try {\n    const mod = (await import(moduleName)) as unknown as GPUModule;\n    gpuCtor = mod.GPU;\n    return gpuCtor;\n  } catch (_err) {\n    return null;\n  }\n}\n\nfunction logGpuCapabilities(GPU: GPUCtor & { isGPUSupported?: boolean }): void {\n  if (gpuCapabilitiesLogged) return;\n\n  const flags = GPU as unknown as {\n    isGPUSupported?: boolean;\n    isHeadlessGLSupported?: boolean;\n    isWebGLSupported?: boolean;\n    isWebGL2Supported?: boolean;\n    isSinglePrecisionSupported?: boolean;\n  };\n\n  log(\n    \"SIM_GPU capabilities supported=%s headlessgl=%s webgl=%s webgl2=%s singlePrecision=%s\",\n    String(flags.isGPUSupported),\n    String(flags.isHeadlessGLSupported),\n    String(flags.isWebGLSupported),\n    String(flags.isWebGL2Supported),\n    String(flags.isSinglePrecisionSupported),\n  );\n\n  gpuCapabilitiesLogged = true;\n}\n\nfunction backendPreference(): BackendPreference {\n  const value = (process.env.DRYSCAN_SIM_BACKEND ?? \"auto\").toLowerCase();\n  if (value === \"gpu\") return \"gpu\";\n  if (value === \"worker\") return \"worker\";\n  if (value === \"sync\") return \"sync\";\n  return \"auto\";\n}\n\nfunction matrixFromGpuResult(result: unknown, rows: number, cols: number): number[][] {\n  if (!Array.isArray(result)) {\n    throw new Error(\"GPU kernel returned a non-array result\");\n  }\n\n  return Array.from({ length: rows }, (_, rowIdx) => {\n    const row = result[rowIdx] as unknown;\n    if (!row || typeof row !== \"object\") {\n      throw new Error(\"GPU kernel returned malformed rows\");\n    }\n    const typed = row as ArrayLike<number>;\n    return Array.from({ length: cols }, (_, colIdx) => Number(typed[colIdx] ?? 0));\n  });\n}\n\nfunction normalizeRows(rows: number[][], dims: number): Float32Array[] {\n  return rows.map((row) => {\n    const normalized = new Float32Array(dims);\n    let normSq = 0;\n    for (let i = 0; i < dims; i++) {\n      const value = Number(row[i] ?? 0);\n      normalized[i] = value;\n      normSq += value * value;\n    }\n\n    const norm = Math.sqrt(normSq);\n    if (norm > ZERO_NORM_EPSILON) {\n      for (let i = 0; i < dims; i++) {\n        normalized[i] /= norm;\n      }\n    } else {\n      normalized.fill(0);\n    }\n\n    return normalized;\n  });\n}\n\nfunction getNormalizedRows(rows: number[][], dims: number): Float32Array[] {\n  const cached = normalizedMatrixCache.get(rows);\n  if (cached && cached.dims === dims) {\n    return cached.rows;\n  }\n\n  const normalized = normalizeRows(rows, dims);\n  normalizedMatrixCache.set(rows, { dims, rows: normalized });\n  return normalized;\n}\n\nasync function runGpuCosineSimilarity(A: number[][], B: number[][]): Promise<number[][]> {\n  const GPU = await getGpuCtor();\n  if (!GPU) {\n    throw new Error(\"gpu.js module not available\");\n  }\n\n  logGpuCapabilities(GPU);\n\n  if (GPU.isGPUSupported === false) {\n    throw new Error(\"GPU.js reported GPU support unavailable\");\n  }\n\n  const dims = A[0]?.length ?? 0;\n  if (!dims || !B[0]?.length || B[0].length !== dims) {\n    throw new Error(\"Matrix dimensions are invalid for GPU similarity\");\n  }\n\n  const normalizeStartMs = performance.now();\n  const normalizedA = getNormalizedRows(A, dims);\n  const normalizedB = getNormalizedRows(B, dims);\n  const normalizeMs = Math.round(performance.now() - normalizeStartMs);\n\n  if (!gpuRuntime) {\n    gpuRuntime = new GPU({ mode: \"gpu\" });\n    const runtimeMode = String((gpuRuntime as unknown as { mode?: string }).mode ?? \"unknown\");\n    log(\"SIM_GPU runtime mode=%s\", runtimeMode);\n    if (runtimeMode === \"cpu\") {\n      throw new Error(\"GPU runtime initialized in CPU mode\");\n    }\n  }\n\n  const shouldRecreateKernel =\n    !gpuKernel\n    || gpuKernelDims !== dims;\n\n  if (shouldRecreateKernel) {\n    const kernel = gpuRuntime\n      .createKernel(function (this: any, a: number[][], b: number[][]) {\n        let dot = 0;\n        for (let i = 0; i < this.constants.dims; i++) {\n          const av = a[this.thread.y][i];\n          const bv = b[this.thread.x][i];\n          dot += av * bv;\n        }\n        return dot;\n      })\n      .setOutput([1, 1])\n      .setConstants({ dims });\n\n    if (typeof (kernel as any).setPrecision === \"function\") {\n      (kernel as any).setPrecision(\"single\");\n    }\n    if (typeof (kernel as any).setTactic === \"function\") {\n      (kernel as any).setTactic(\"speed\");\n    }\n    if (typeof (kernel as any).setDynamicArguments === \"function\") {\n      (kernel as any).setDynamicArguments(true);\n    }\n\n    if (typeof kernel.setDynamicOutput === \"function\") {\n      kernel.setDynamicOutput(true);\n    }\n\n    gpuKernel = kernel;\n\n    gpuKernelDims = dims;\n  }\n\n  if (!gpuKernel) {\n    throw new Error(\"GPU kernel initialization failed\");\n  }\n\n  let targetCells = GPU_TARGET_CELLS;\n  if (!Number.isFinite(targetCells) || targetCells <= 0) {\n    targetCells = 2_000_000;\n  }\n\n  let rowsPerBatch = Math.floor(targetCells / normalizedB.length);\n  if (rowsPerBatch < 1) {\n    rowsPerBatch = 1;\n  }\n\n  const result: number[][] = [];\n  const batches = Math.ceil(normalizedA.length / rowsPerBatch);\n  const kernelStartMs = performance.now();\n\n  for (let start = 0; start < normalizedA.length; start += rowsPerBatch) {\n    const batch = normalizedA.slice(start, start + rowsPerBatch);\n    gpuKernel.setOutput([normalizedB.length, batch.length]);\n    const raw = gpuKernel(batch, normalizedB);\n    result.push(...matrixFromGpuResult(raw, batch.length, normalizedB.length));\n  }\n  const kernelMs = Math.round(performance.now() - kernelStartMs);\n\n  log(\n    \"SIM_GPU details dims=%d aRows=%d bRows=%d rowsPerBatch=%d batches=%d targetCells=%d normalizeMs=%d kernelMs=%d\",\n    dims,\n    A.length,\n    B.length,\n    rowsPerBatch,\n    batches,\n    targetCells,\n    normalizeMs,\n    kernelMs,\n  );\n\n  return result;\n}\n\nasync function computeWithWorkers(A: number[][], B: number[][]): Promise<number[][]> {\n  const dims = A[0].length;\n  const cpuCount = Math.max(1, os.cpus().length);\n  const chunkSize = Math.max(1, Math.ceil(A.length / cpuCount));\n\n  const sharedB = new SharedArrayBuffer(B.length * dims * 8);\n  const bView = new Float64Array(sharedB);\n  B.forEach((row, i) => bView.set(row, i * dims));\n\n  const runningTsSource = new URL(import.meta.url).pathname.endsWith(\".ts\");\n  let workerFile = \"./cosineSimilarityWorker.js\";\n  if (runningTsSource) {\n    workerFile = \"./cosineSimilarityWorker.ts\";\n  }\n  const workerUrl = new URL(workerFile, import.meta.url);\n  let execArgv: string[] = [];\n  if (runningTsSource) {\n    execArgv = [\"--import\", \"tsx/esm\"];\n  }\n\n  const chunks = Array.from(\n    { length: Math.ceil(A.length / chunkSize) },\n    (_, i) => A.slice(i * chunkSize, (i + 1) * chunkSize),\n  );\n\n  const results = await Promise.all(chunks.map(chunk => runWorker(chunk, sharedB, B.length, dims, workerUrl, execArgv)));\n  return results.flat();\n}\n\nfunction computeSequential(A: number[][], B: number[][]): number[][] {\n  return cosineSimilarity(A, B);\n}\n\nasync function runWithTiming(name: BackendName, fn: () => Promise<number[][]> | number[][]): Promise<number[][]> {\n  const startMs = performance.now();\n  log(\"SIM_TRY backend=%s\", name);\n\n  try {\n    const result = await Promise.resolve(fn());\n    const durationMs = Math.round(performance.now() - startMs);\n    log(\"SIM_DONE backend=%s durationMs=%d\", name, durationMs);\n    return result;\n  } catch (err: any) {\n    const durationMs = Math.round(performance.now() - startMs);\n    log(\"SIM_FAIL backend=%s durationMs=%d reason=%s\", name, durationMs, err?.message ?? \"unknown\");\n    throw err;\n  }\n}\n\n/**\n * Computes cosine similarity using a deterministic backend chain.\n * Default chain: GPU -> worker threads -> synchronous fallback.\n */\nexport async function parallelCosineSimilarity(A: number[][], B: number[][]): Promise<number[][]> {\n  if (A.length === 0 || B.length === 0) return [];\n\n  const preference = backendPreference();\n  const dims = A[0]?.length ?? 0;\n  log(\"SIM_START rows=%d cols=%d dims=%d preference=%s\", A.length, B.length, dims, preference);\n\n  if (preference === \"sync\") {\n    return runWithTiming(\"sync\", () => computeSequential(A, B));\n  }\n\n  if (preference === \"worker\") {\n    try {\n      return await runWithTiming(\"worker\", () => computeWithWorkers(A, B));\n    } catch (_workerErr) {\n      return runWithTiming(\"sync\", () => computeSequential(A, B));\n    }\n  }\n\n  try {\n    return await runWithTiming(\"gpu\", () => runGpuCosineSimilarity(A, B));\n  } catch (_gpuErr) {\n    log(\"SIM_CHAIN continue_after=gpu-fail next=worker\");\n  }\n\n  try {\n    return await runWithTiming(\"worker\", () => computeWithWorkers(A, B));\n  } catch (_workerErr) {\n    log(\"SIM_CHAIN continue_after=worker-fail next=sync\");\n    return runWithTiming(\"sync\", () => computeSequential(A, B));\n  }\n}\n\nfunction runWorker(\n  chunk: number[][],\n  sharedB: SharedArrayBuffer,\n  bCount: number,\n  dims: number,\n  workerUrl: URL,\n  execArgv: string[],\n): Promise<number[][]> {\n  return new Promise((resolve, reject) => {\n    const rowsFlat = new Float64Array(chunk.length * dims);\n    chunk.forEach((row, i) => rowsFlat.set(row, i * dims));\n\n    const worker = new Worker(workerUrl, {\n      workerData: { rowsBuffer: rowsFlat.buffer, rowCount: chunk.length, allBuffer: sharedB, allCount: bCount, dims },\n      transferList: [rowsFlat.buffer],\n      execArgv,\n    });\n\n    worker.once(\"message\", ({ result }) => resolve(result));\n    worker.once(\"error\", reject);\n  });\n}\n"],"mappings":";;;AAAA,OAAO,QAAQ;AACf,SAAS,cAAc;AACvB,OAAO,WAAW;AAClB,SAAS,wBAAwB;AAEjC,IAAM,MAAM,MAAM,4BAA4B;AAM9C,IAAM,mBAAmB;AACzB,IAAM,oBAAoB;AAkB1B,IAAI,YAEO;AACX,IAAI,aAA2C;AAC/C,IAAI,gBAA+B;AACnC,IAAI,UAA2D;AAC/D,IAAI,wBAAwB;AAE5B,IAAM,wBAAwB,oBAAI,QAA4D;AAI9F,eAAe,aAAuE;AACpF,MAAI,QAAS,QAAO;AAEpB,QAAM,aAAa;AACnB,MAAI;AACF,UAAM,MAAO,MAAM,OAAO;AAC1B,cAAU,IAAI;AACd,WAAO;AAAA,EACT,SAAS,MAAM;AACb,WAAO;AAAA,EACT;AACF;AAEA,SAAS,mBAAmB,KAAmD;AAC7E,MAAI,sBAAuB;AAE3B,QAAM,QAAQ;AAQd;AAAA,IACE;AAAA,IACA,OAAO,MAAM,cAAc;AAAA,IAC3B,OAAO,MAAM,qBAAqB;AAAA,IAClC,OAAO,MAAM,gBAAgB;AAAA,IAC7B,OAAO,MAAM,iBAAiB;AAAA,IAC9B,OAAO,MAAM,0BAA0B;AAAA,EACzC;AAEA,0BAAwB;AAC1B;AAEA,SAAS,oBAAuC;AAC9C,QAAM,SAAS,QAAQ,IAAI,uBAAuB,QAAQ,YAAY;AACtE,MAAI,UAAU,MAAO,QAAO;AAC5B,MAAI,UAAU,SAAU,QAAO;AAC/B,MAAI,UAAU,OAAQ,QAAO;AAC7B,SAAO;AACT;AAEA,SAAS,oBAAoB,QAAiB,MAAc,MAA0B;AACpF,MAAI,CAAC,MAAM,QAAQ,MAAM,GAAG;AAC1B,UAAM,IAAI,MAAM,wCAAwC;AAAA,EAC1D;AAEA,SAAO,MAAM,KAAK,EAAE,QAAQ,KAAK,GAAG,CAAC,GAAG,WAAW;AACjD,UAAM,MAAM,OAAO,MAAM;AACzB,QAAI,CAAC,OAAO,OAAO,QAAQ,UAAU;AACnC,YAAM,IAAI,MAAM,oCAAoC;AAAA,IACtD;AACA,UAAM,QAAQ;AACd,WAAO,MAAM,KAAK,EAAE,QAAQ,KAAK,GAAG,CAACA,IAAG,WAAW,OAAO,MAAM,MAAM,KAAK,CAAC,CAAC;AAAA,EAC/E,CAAC;AACH;AAEA,SAAS,cAAc,MAAkB,MAA8B;AACrE,SAAO,KAAK,IAAI,CAAC,QAAQ;AACvB,UAAM,aAAa,IAAI,aAAa,IAAI;AACxC,QAAI,SAAS;AACb,aAAS,IAAI,GAAG,IAAI,MAAM,KAAK;AAC7B,YAAM,QAAQ,OAAO,IAAI,CAAC,KAAK,CAAC;AAChC,iBAAW,CAAC,IAAI;AAChB,gBAAU,QAAQ;AAAA,IACpB;AAEA,UAAM,OAAO,KAAK,KAAK,MAAM;AAC7B,QAAI,OAAO,mBAAmB;AAC5B,eAAS,IAAI,GAAG,IAAI,MAAM,KAAK;AAC7B,mBAAW,CAAC,KAAK;AAAA,MACnB;AAAA,IACF,OAAO;AACL,iBAAW,KAAK,CAAC;AAAA,IACnB;AAEA,WAAO;AAAA,EACT,CAAC;AACH;AAEA,SAAS,kBAAkB,MAAkB,MAA8B;AACzE,QAAM,SAAS,sBAAsB,IAAI,IAAI;AAC7C,MAAI,UAAU,OAAO,SAAS,MAAM;AAClC,WAAO,OAAO;AAAA,EAChB;AAEA,QAAM,aAAa,cAAc,MAAM,IAAI;AAC3C,wBAAsB,IAAI,MAAM,EAAE,MAAM,MAAM,WAAW,CAAC;AAC1D,SAAO;AACT;AAEA,eAAe,uBAAuB,GAAe,GAAoC;AACvF,QAAM,MAAM,MAAM,WAAW;AAC7B,MAAI,CAAC,KAAK;AACR,UAAM,IAAI,MAAM,6BAA6B;AAAA,EAC/C;AAEA,qBAAmB,GAAG;AAEtB,MAAI,IAAI,mBAAmB,OAAO;AAChC,UAAM,IAAI,MAAM,yCAAyC;AAAA,EAC3D;AAEA,QAAM,OAAO,EAAE,CAAC,GAAG,UAAU;AAC7B,MAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,EAAE,WAAW,MAAM;AAClD,UAAM,IAAI,MAAM,kDAAkD;AAAA,EACpE;AAEA,QAAM,mBAAmB,YAAY,IAAI;AACzC,QAAM,cAAc,kBAAkB,GAAG,IAAI;AAC7C,QAAM,cAAc,kBAAkB,GAAG,IAAI;AAC7C,QAAM,cAAc,KAAK,MAAM,YAAY,IAAI,IAAI,gBAAgB;AAEnE,MAAI,CAAC,YAAY;AACf,iBAAa,IAAI,IAAI,EAAE,MAAM,MAAM,CAAC;AACpC,UAAM,cAAc,OAAQ,WAA4C,QAAQ,SAAS;AACzF,QAAI,2BAA2B,WAAW;AAC1C,QAAI,gBAAgB,OAAO;AACzB,YAAM,IAAI,MAAM,qCAAqC;AAAA,IACvD;AAAA,EACF;AAEA,QAAM,uBACJ,CAAC,aACE,kBAAkB;AAEvB,MAAI,sBAAsB;AACxB,UAAM,SAAS,WACZ,aAAa,SAAqB,GAAe,GAAe;AAC/D,UAAI,MAAM;AACV,eAAS,IAAI,GAAG,IAAI,KAAK,UAAU,MAAM,KAAK;AAC5C,cAAM,KAAK,EAAE,KAAK,OAAO,CAAC,EAAE,CAAC;AAC7B,cAAM,KAAK,EAAE,KAAK,OAAO,CAAC,EAAE,CAAC;AAC7B,eAAO,KAAK;AAAA,MACd;AACA,aAAO;AAAA,IACT,CAAC,EACA,UAAU,CAAC,GAAG,CAAC,CAAC,EAChB,aAAa,EAAE,KAAK,CAAC;AAExB,QAAI,OAAQ,OAAe,iBAAiB,YAAY;AACtD,MAAC,OAAe,aAAa,QAAQ;AAAA,IACvC;AACA,QAAI,OAAQ,OAAe,cAAc,YAAY;AACnD,MAAC,OAAe,UAAU,OAAO;AAAA,IACnC;AACA,QAAI,OAAQ,OAAe,wBAAwB,YAAY;AAC7D,MAAC,OAAe,oBAAoB,IAAI;AAAA,IAC1C;AAEA,QAAI,OAAO,OAAO,qBAAqB,YAAY;AACjD,aAAO,iBAAiB,IAAI;AAAA,IAC9B;AAEA,gBAAY;AAEZ,oBAAgB;AAAA,EAClB;AAEA,MAAI,CAAC,WAAW;AACd,UAAM,IAAI,MAAM,kCAAkC;AAAA,EACpD;AAEA,MAAI,cAAc;AAClB,MAAI,CAAC,OAAO,SAAS,WAAW,KAAK,eAAe,GAAG;AACrD,kBAAc;AAAA,EAChB;AAEA,MAAI,eAAe,KAAK,MAAM,cAAc,YAAY,MAAM;AAC9D,MAAI,eAAe,GAAG;AACpB,mBAAe;AAAA,EACjB;AAEA,QAAM,SAAqB,CAAC;AAC5B,QAAM,UAAU,KAAK,KAAK,YAAY,SAAS,YAAY;AAC3D,QAAM,gBAAgB,YAAY,IAAI;AAEtC,WAAS,QAAQ,GAAG,QAAQ,YAAY,QAAQ,SAAS,cAAc;AACrE,UAAM,QAAQ,YAAY,MAAM,OAAO,QAAQ,YAAY;AAC3D,cAAU,UAAU,CAAC,YAAY,QAAQ,MAAM,MAAM,CAAC;AACtD,UAAM,MAAM,UAAU,OAAO,WAAW;AACxC,WAAO,KAAK,GAAG,oBAAoB,KAAK,MAAM,QAAQ,YAAY,MAAM,CAAC;AAAA,EAC3E;AACA,QAAM,WAAW,KAAK,MAAM,YAAY,IAAI,IAAI,aAAa;AAE7D;AAAA,IACE;AAAA,IACA;AAAA,IACA,EAAE;AAAA,IACF,EAAE;AAAA,IACF;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AAEA,SAAO;AACT;AAEA,eAAe,mBAAmB,GAAe,GAAoC;AACnF,QAAM,OAAO,EAAE,CAAC,EAAE;AAClB,QAAM,WAAW,KAAK,IAAI,GAAG,GAAG,KAAK,EAAE,MAAM;AAC7C,QAAM,YAAY,KAAK,IAAI,GAAG,KAAK,KAAK,EAAE,SAAS,QAAQ,CAAC;AAE5D,QAAM,UAAU,IAAI,kBAAkB,EAAE,SAAS,OAAO,CAAC;AACzD,QAAM,QAAQ,IAAI,aAAa,OAAO;AACtC,IAAE,QAAQ,CAAC,KAAK,MAAM,MAAM,IAAI,KAAK,IAAI,IAAI,CAAC;AAE9C,QAAM,kBAAkB,IAAI,IAAI,YAAY,GAAG,EAAE,SAAS,SAAS,KAAK;AACxE,MAAI,aAAa;AACjB,MAAI,iBAAiB;AACnB,iBAAa;AAAA,EACf;AACA,QAAM,YAAY,IAAI,IAAI,YAAY,YAAY,GAAG;AACrD,MAAI,WAAqB,CAAC;AAC1B,MAAI,iBAAiB;AACnB,eAAW,CAAC,YAAY,SAAS;AAAA,EACnC;AAEA,QAAM,SAAS,MAAM;AAAA,IACnB,EAAE,QAAQ,KAAK,KAAK,EAAE,SAAS,SAAS,EAAE;AAAA,IAC1C,CAAC,GAAG,MAAM,EAAE,MAAM,IAAI,YAAY,IAAI,KAAK,SAAS;AAAA,EACtD;AAEA,QAAM,UAAU,MAAM,QAAQ,IAAI,OAAO,IAAI,WAAS,UAAU,OAAO,SAAS,EAAE,QAAQ,MAAM,WAAW,QAAQ,CAAC,CAAC;AACrH,SAAO,QAAQ,KAAK;AACtB;AAEA,SAAS,kBAAkB,GAAe,GAA2B;AACnE,SAAO,iBAAiB,GAAG,CAAC;AAC9B;AAEA,eAAe,cAAc,MAAmB,IAAiE;AAC/G,QAAM,UAAU,YAAY,IAAI;AAChC,MAAI,sBAAsB,IAAI;AAE9B,MAAI;AACF,UAAM,SAAS,MAAM,QAAQ,QAAQ,GAAG,CAAC;AACzC,UAAM,aAAa,KAAK,MAAM,YAAY,IAAI,IAAI,OAAO;AACzD,QAAI,qCAAqC,MAAM,UAAU;AACzD,WAAO;AAAA,EACT,SAAS,KAAU;AACjB,UAAM,aAAa,KAAK,MAAM,YAAY,IAAI,IAAI,OAAO;AACzD,QAAI,+CAA+C,MAAM,YAAY,KAAK,WAAW,SAAS;AAC9F,UAAM;AAAA,EACR;AACF;AAMA,eAAsB,yBAAyB,GAAe,GAAoC;AAChG,MAAI,EAAE,WAAW,KAAK,EAAE,WAAW,EAAG,QAAO,CAAC;AAE9C,QAAM,aAAa,kBAAkB;AACrC,QAAM,OAAO,EAAE,CAAC,GAAG,UAAU;AAC7B,MAAI,mDAAmD,EAAE,QAAQ,EAAE,QAAQ,MAAM,UAAU;AAE3F,MAAI,eAAe,QAAQ;AACzB,WAAO,cAAc,QAAQ,MAAM,kBAAkB,GAAG,CAAC,CAAC;AAAA,EAC5D;AAEA,MAAI,eAAe,UAAU;AAC3B,QAAI;AACF,aAAO,MAAM,cAAc,UAAU,MAAM,mBAAmB,GAAG,CAAC,CAAC;AAAA,IACrE,SAAS,YAAY;AACnB,aAAO,cAAc,QAAQ,MAAM,kBAAkB,GAAG,CAAC,CAAC;AAAA,IAC5D;AAAA,EACF;AAEA,MAAI;AACF,WAAO,MAAM,cAAc,OAAO,MAAM,uBAAuB,GAAG,CAAC,CAAC;AAAA,EACtE,SAAS,SAAS;AAChB,QAAI,+CAA+C;AAAA,EACrD;AAEA,MAAI;AACF,WAAO,MAAM,cAAc,UAAU,MAAM,mBAAmB,GAAG,CAAC,CAAC;AAAA,EACrE,SAAS,YAAY;AACnB,QAAI,gDAAgD;AACpD,WAAO,cAAc,QAAQ,MAAM,kBAAkB,GAAG,CAAC,CAAC;AAAA,EAC5D;AACF;AAEA,SAAS,UACP,OACA,SACA,QACA,MACA,WACA,UACqB;AACrB,SAAO,IAAI,QAAQ,CAAC,SAAS,WAAW;AACtC,UAAM,WAAW,IAAI,aAAa,MAAM,SAAS,IAAI;AACrD,UAAM,QAAQ,CAAC,KAAK,MAAM,SAAS,IAAI,KAAK,IAAI,IAAI,CAAC;AAErD,UAAM,SAAS,IAAI,OAAO,WAAW;AAAA,MACnC,YAAY,EAAE,YAAY,SAAS,QAAQ,UAAU,MAAM,QAAQ,WAAW,SAAS,UAAU,QAAQ,KAAK;AAAA,MAC9G,cAAc,CAAC,SAAS,MAAM;AAAA,MAC9B;AAAA,IACF,CAAC;AAED,WAAO,KAAK,WAAW,CAAC,EAAE,OAAO,MAAM,QAAQ,MAAM,CAAC;AACtD,WAAO,KAAK,SAAS,MAAM;AAAA,EAC7B,CAAC;AACH;","names":["_"]}
+{"version":3,"sources":["../../src/services/ParallelSimilarity.ts"],"sourcesContent":["import os from \"node:os\";\nimport { Worker } from \"node:worker_threads\";\nimport { cosineSimilarity } from \"@langchain/core/utils/math\";\n\n/** Minimum row count below which synchronous is faster than worker overhead. */\nconst MIN_PARALLEL_ROWS = 50;\n\n/**\n * Computes cosineSimilarity(A, B) using worker threads for large inputs,\n * falling back to the synchronous library call for small ones.\n * B is packed into a SharedArrayBuffer shared across all workers — no copies.\n */\nexport async function parallelCosineSimilarity(A: number[][], B: number[][]): Promise<number[][]> {\n  if (A.length === 0 || B.length === 0) return [];\n  if (A.length < MIN_PARALLEL_ROWS) return cosineSimilarity(A, B);\n\n  const dims = A[0].length;\n  const chunkSize = Math.ceil(A.length / os.cpus().length);\n\n  const sharedB = new SharedArrayBuffer(B.length * dims * 8);\n  const bView = new Float64Array(sharedB);\n  B.forEach((row, i) => bView.set(row, i * dims));\n\n  const runningTsSource = new URL(import.meta.url).pathname.endsWith('.ts');\n  const workerFile = runningTsSource ? './cosineSimilarityWorker.ts' : './cosineSimilarityWorker.js';\n  const workerUrl = new URL(workerFile, import.meta.url);\n  const execArgv = runningTsSource ? ['--import', 'tsx/esm'] : [];\n\n  const chunks = Array.from(\n    { length: Math.ceil(A.length / chunkSize) },\n    (_, i) => A.slice(i * chunkSize, (i + 1) * chunkSize),\n  );\n\n  const results = await Promise.all(chunks.map(chunk => runWorker(chunk, sharedB, B.length, dims, workerUrl, execArgv)));\n  return results.flat();\n}\n\nfunction runWorker(\n  chunk: number[][],\n  sharedB: SharedArrayBuffer,\n  bCount: number,\n  dims: number,\n  workerUrl: URL,\n  execArgv: string[],\n): Promise<number[][]> {\n  return new Promise((resolve, reject) => {\n    const rowsFlat = new Float64Array(chunk.length * dims);\n    chunk.forEach((row, i) => rowsFlat.set(row, i * dims));\n\n    const worker = new Worker(workerUrl, {\n      workerData: { rowsBuffer: rowsFlat.buffer, rowCount: chunk.length, allBuffer: sharedB, allCount: bCount, dims },\n      transferList: [rowsFlat.buffer],\n      execArgv,\n    });\n\n    worker.once(\"message\", ({ result }) => resolve(result));\n    worker.once(\"error\", reject);\n  });\n}\n"],"mappings":";;;AAAA,OAAO,QAAQ;AACf,SAAS,cAAc;AACvB,SAAS,wBAAwB;AAGjC,IAAM,oBAAoB;AAO1B,eAAsB,yBAAyB,GAAe,GAAoC;AAChG,MAAI,EAAE,WAAW,KAAK,EAAE,WAAW,EAAG,QAAO,CAAC;AAC9C,MAAI,EAAE,SAAS,kBAAmB,QAAO,iBAAiB,GAAG,CAAC;AAE9D,QAAM,OAAO,EAAE,CAAC,EAAE;AAClB,QAAM,YAAY,KAAK,KAAK,EAAE,SAAS,GAAG,KAAK,EAAE,MAAM;AAEvD,QAAM,UAAU,IAAI,kBAAkB,EAAE,SAAS,OAAO,CAAC;AACzD,QAAM,QAAQ,IAAI,aAAa,OAAO;AACtC,IAAE,QAAQ,CAAC,KAAK,MAAM,MAAM,IAAI,KAAK,IAAI,IAAI,CAAC;AAE9C,QAAM,kBAAkB,IAAI,IAAI,YAAY,GAAG,EAAE,SAAS,SAAS,KAAK;AACxE,QAAM,aAAa,kBAAkB,gCAAgC;AACrE,QAAM,YAAY,IAAI,IAAI,YAAY,YAAY,GAAG;AACrD,QAAM,WAAW,kBAAkB,CAAC,YAAY,SAAS,IAAI,CAAC;AAE9D,QAAM,SAAS,MAAM;AAAA,IACnB,EAAE,QAAQ,KAAK,KAAK,EAAE,SAAS,SAAS,EAAE;AAAA,IAC1C,CAAC,GAAG,MAAM,EAAE,MAAM,IAAI,YAAY,IAAI,KAAK,SAAS;AAAA,EACtD;AAEA,QAAM,UAAU,MAAM,QAAQ,IAAI,OAAO,IAAI,WAAS,UAAU,OAAO,SAAS,EAAE,QAAQ,MAAM,WAAW,QAAQ,CAAC,CAAC;AACrH,SAAO,QAAQ,KAAK;AACtB;AAEA,SAAS,UACP,OACA,SACA,QACA,MACA,WACA,UACqB;AACrB,SAAO,IAAI,QAAQ,CAAC,SAAS,WAAW;AACtC,UAAM,WAAW,IAAI,aAAa,MAAM,SAAS,IAAI;AACrD,UAAM,QAAQ,CAAC,KAAK,MAAM,SAAS,IAAI,KAAK,IAAI,IAAI,CAAC;AAErD,UAAM,SAAS,IAAI,OAAO,WAAW;AAAA,MACnC,YAAY,EAAE,YAAY,SAAS,QAAQ,UAAU,MAAM,QAAQ,WAAW,SAAS,UAAU,QAAQ,KAAK;AAAA,MAC9G,cAAc,CAAC,SAAS,MAAM;AAAA,MAC9B;AAAA,IACF,CAAC;AAED,WAAO,KAAK,WAAW,CAAC,EAAE,OAAO,MAAM,QAAQ,MAAM,CAAC;AACtD,WAAO,KAAK,SAAS,MAAM;AAAA,EAC7B,CAAC;AACH;","names":[]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@goshenkata/dryscan-core",
-  "version": "1.4.7",
+  "version": "1.4.8",
   "description": "Core library for DryScan - semantic code duplication analyzer",
   "type": "module",
   "main": "./dist/index.js",
@@ -54,7 +54,6 @@
     "@langchain/ollama": "^1.1.0",
     "debug": "^4.4.3",
     "glob-gitignore": "^1.0.15",
-    "gpu.js": "^2.16.0",
     "ignore": "^7.0.5",
     "jsonschema": "^1.5.0",
     "minimatch": "^10.1.1",