@velvetmonkey/flywheel-memory 2.5.10 → 2.5.12

package/dist/embedding-worker.js

@@ -0,0 +1,76 @@
+// src/core/read/embedding-worker.ts
+import { parentPort } from "node:worker_threads";
+if (!parentPort) {
+  throw new Error("embedding-worker.ts must be run as a worker thread");
+}
+var port = parentPort;
+var pipeline = null;
+var modelDims = 0;
+async function loadModel(modelId) {
+  const MAX_RETRIES = 3;
+  const RETRY_DELAYS = [2e3, 5e3, 1e4];
+  for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
+    try {
+      const transformers = await Function("specifier", "return import(specifier)")("@huggingface/transformers");
+      pipeline = await transformers.pipeline("feature-extraction", modelId, {
+        dtype: "fp32"
+      });
+      const probe = await pipeline("test", { pooling: "mean", normalize: true });
+      modelDims = probe.data.length;
+      port.postMessage({ type: "ready", dims: modelDims });
+      return;
+    } catch (err) {
+      const errMsg = err instanceof Error ? err.message : String(err);
+      if (err instanceof Error && (errMsg.includes("Cannot find package") || errMsg.includes("MODULE_NOT_FOUND") || errMsg.includes("Cannot find module") || errMsg.includes("ERR_MODULE_NOT_FOUND"))) {
+        port.postMessage({
+          type: "error",
+          message: "Semantic search requires @huggingface/transformers. Install it with: npm install @huggingface/transformers",
+          fatal: true
+        });
+        return;
+      }
+      if (attempt < MAX_RETRIES) {
+        const delay = RETRY_DELAYS[attempt - 1];
+        await new Promise((resolve) => setTimeout(resolve, delay));
+      } else {
+        port.postMessage({
+          type: "error",
+          message: `Model load failed after ${MAX_RETRIES} attempts: ${errMsg}`,
+          fatal: true
+        });
+      }
+    }
+  }
+}
+async function embed(id, text) {
+  if (!pipeline) {
+    port.postMessage({ type: "error", id, message: "Model not loaded", fatal: false });
+    return;
+  }
+  try {
+    const truncated = text.slice(0, 2e3);
+    const result = await pipeline(truncated, { pooling: "mean", normalize: true });
+    const embedding = new Float32Array(result.data);
+    port.postMessage({ type: "result", id, embedding }, [embedding.buffer]);
+  } catch (err) {
+    port.postMessage({
+      type: "error",
+      id,
+      message: err instanceof Error ? err.message : String(err),
+      fatal: false
+    });
+  }
+}
+port.on("message", (msg) => {
+  switch (msg.type) {
+    case "init":
+      loadModel(msg.modelId);
+      break;
+    case "embed":
+      embed(msg.id, msg.text);
+      break;
+    case "shutdown":
+      process.exit(0);
+      break;
+  }
+});