@delali/narsil 0.2.4-next.51 → 0.2.4-next.54

package/dist/vector/search-worker.mjs

@@ -1,32 +1,29 @@
-import { openSharedWorkerCopy, createScalarQuantizer } from '../chunk-PMBHH44R.mjs';
-import { searchOrdinals, MAX_WORKER_COPIES, createVectorStore, createHNSWIndex } from '../chunk-X6YZO4MQ.mjs';
-import '../chunk-RP23PWMV.mjs';
+import { openSharedVectorField, MAX_WORKER_COPIES, createVectorStore, osqBitsOf, createHNSWIndex, createOsqQuantizer, osqRecordBytes } from '../chunk-U2AL3SON.mjs';
+import '../chunk-ZJ4CSAB2.mjs';
+import '../chunk-UTOFOJOR.mjs';
 import '../chunk-2GWVQWGA.mjs';
+import '../chunk-FRQVMFY7.mjs';
 
 // src/vector/worker-copy.ts
+function restoreQuantizer(snapshot, store) {
+  const bits = osqBitsOf(snapshot.quantization);
+  if (bits === null || snapshot.codes === null) return null;
+  const quantizer = createOsqQuantizer(snapshot.dimension, bits, snapshot.metric, store);
+  quantizer.restoreCentroid(snapshot.codes.centroid);
+  const recordBytes = osqRecordBytes(snapshot.dimension, bits);
+  for (let ordinal = 0; ordinal < snapshot.codes.present.length; ordinal++) {
+    if (snapshot.codes.present[ordinal] !== 1) continue;
+    const docId = store.docIdForOrdinal(ordinal);
+    if (docId === void 0) continue;
+    quantizer.restoreRecord(docId, snapshot.codes.records.subarray(ordinal * recordBytes, (ordinal + 1) * recordBytes));
+  }
+  return quantizer;
+}
 function restoreWorkerCopy(snapshot) {
-  const store = createVectorStore();
+  const store = createVectorStore({ dimension: snapshot.dimension, codeBits: osqBitsOf(snapshot.quantization) });
   store.restoreSnapshot(snapshot.store);
   const tombstones = new Set(snapshot.tombstones);
-  let quantizer = null;
-  if (snapshot.quantization === "sq8") {
-    const sq8 = createScalarQuantizer(snapshot.dimension, store);
-    if (snapshot.calibration !== null) {
-      sq8.restoreCalibration(snapshot.calibration.alpha, snapshot.calibration.offset);
-    } else {
-      const live = [];
-      for (const [docId, entry] of store.entries()) {
-        if (tombstones.has(docId)) continue;
-        live.push(entry.vector);
-      }
-      sq8.calibrate(live);
-    }
-    for (const [docId, entry] of store.entries()) {
-      if (tombstones.has(docId)) continue;
-      sq8.quantize(docId, entry.vector);
-    }
-    quantizer = sq8;
-  }
+  const quantizer = restoreQuantizer(snapshot, store);
   const graph = createHNSWIndex(
     snapshot.dimension,
     store,
@@ -34,6 +31,7 @@ function restoreWorkerCopy(snapshot) {
     quantizer ?? void 0
   );
   graph.restoreSnapshot(snapshot.graph);
+  for (const docId of tombstones) graph.markTombstone(docId);
   return { store, graph, tombstones };
 }
 
@@ -51,29 +49,53 @@ function handleLoad(request) {
 }
 function handleLoadShared(request) {
   ensureRoom(request.handle);
-  copies.set(request.handle, { kind: "shared", copy: openSharedWorkerCopy(request.snapshot, request.scratchSlot) });
+  const existing = copies.get(request.handle);
+  if (existing?.kind === "shared") {
+    existing.view.adopt(request.handles);
+  } else {
+    copies.set(request.handle, { kind: "shared", view: openSharedVectorField(request.handles, request.scratchSlot) });
+  }
   return { type: "ack", requestId: request.requestId, handle: request.handle };
 }
 function handleDrop(request) {
+  const held = copies.get(request.handle);
+  if (held?.kind === "shared") held.view.close();
   copies.delete(request.handle);
   return { type: "ack", requestId: request.requestId, handle: request.handle };
 }
+function yieldToEventLoop() {
+  return new Promise((resolve) => {
+    if (typeof setImmediate === "function") setImmediate(resolve);
+    else setTimeout(resolve, 0);
+  });
+}
+async function handleInsert(request) {
+  const entry = copies.get(request.handle);
+  if (!entry) {
+    throw new Error(`Search worker holds no copy for handle ${request.handle}`);
+  }
+  if (entry.kind !== "shared") {
+    throw new Error(`Handle ${request.handle} holds a cloned copy, which takes no insertions`);
+  }
+  for (const ordinal of request.ordinals) {
+    entry.view.insertOrdinal(ordinal);
+    await yieldToEventLoop();
+  }
+  return { type: "inserted", requestId: request.requestId, outcome: entry.view.takeOutcome() };
+}
 function handleSearch(request) {
   const entry = copies.get(request.handle);
   if (!entry) {
     throw new Error(`Search worker holds no copy for handle ${request.handle}`);
   }
   if (entry.kind !== "clone") {
-    throw new Error(`Handle ${request.handle} holds a shared copy, which answers ordinal searches alone`);
+    throw new Error(`Handle ${request.handle} holds a shared field, which answers ordinal searches alone`);
   }
-  const hits = entry.copy.graph.search(
-    request.query,
-    request.k,
-    request.metric,
-    request.minSimilarity,
-    request.filter,
-    request.efSearch
-  );
+  const hits = entry.copy.graph.search(request.query, request.k, request.metric, request.minSimilarity, {
+    filter: request.filter,
+    efSearch: request.efSearch,
+    oversample: request.oversample
+  });
   const docIds = [];
   const scores = new Float64Array(hits.length);
   for (let i = 0; i < hits.length; i++) {
@@ -90,16 +112,11 @@ function handleSearchOrdinals(request) {
   if (entry.kind !== "shared") {
     throw new Error(`Handle ${request.handle} holds a cloned copy, which answers document id searches alone`);
   }
-  const hits = searchOrdinals(
-    entry.copy.searchState,
-    request.query,
-    request.k,
-    request.metric,
-    request.minSimilarity,
-    entry.copy.rankByOrdinal,
-    request.filter,
-    request.efSearch
-  );
+  const hits = entry.view.searchOrdinals(request.query, request.k, request.metric, request.minSimilarity, {
+    filter: request.filter,
+    efSearch: request.efSearch,
+    oversample: request.oversample
+  });
   const ordinals = new Uint32Array(hits.length);
   const scores = new Float64Array(hits.length);
   for (let i = 0; i < hits.length; i++) {
@@ -108,11 +125,12 @@ function handleSearchOrdinals(request) {
   }
   return { type: "ordinalResult", requestId: request.requestId, ordinals, scores };
 }
-function handleAnyRequest(raw) {
+async function handleAnyRequest(raw) {
   const request = raw;
   if (request.type === "load") return handleLoad(request);
   if (request.type === "loadShared") return handleLoadShared(request);
   if (request.type === "drop") return handleDrop(request);
+  if (request.type === "insertOrdinals") return handleInsert(request);
   if (request.type === "search") return handleSearch(request);
   if (request.type === "searchOrdinals") return handleSearchOrdinals(request);
   throw new Error(`Unknown request type: ${request.type}`);
@@ -138,11 +156,10 @@ async function setupAsync() {
   if (parentPort) {
     const port = parentPort;
     port.on("message", (raw) => {
-      try {
-        port.postMessage(handleAnyRequest(raw));
-      } catch (err) {
-        port.postMessage(errorFor(raw, err));
-      }
+      handleAnyRequest(raw).then(
+        (reply) => port.postMessage(reply),
+        (err) => port.postMessage(errorFor(raw, err))
+      );
     });
     return;
   }
@@ -150,11 +167,10 @@ async function setupAsync() {
   if (globalSelf && typeof globalSelf.postMessage === "function") {
     const webSelf = globalSelf;
     webSelf.onmessage = (event) => {
-      try {
-        webSelf.postMessage(handleAnyRequest(event.data));
-      } catch (err) {
-        webSelf.postMessage(errorFor(event.data, err));
-      }
+      handleAnyRequest(event.data).then(
+        (reply) => webSelf.postMessage(reply),
+        (err) => webSelf.postMessage(errorFor(event.data, err))
+      );
     };
   }
 }

package/dist/workers/entry.mjs

@@ -1,9 +1,10 @@
-import { isValidWorkerAction } from '../chunk-WYJXSLER.mjs';
-import { NarsilError, ErrorCodes } from '../chunk-P4ELXVJQ.mjs';
+import { startIdleHeapCleanup } from '../chunk-Y3L5MDRV.mjs';
+import { threadSlotOfWorker, isValidWorkerAction } from '../chunk-POWOTUKJ.mjs';
+import { ErrorCodes, NarsilError, isNarsilError } from '../chunk-FRQVMFY7.mjs';
 
 // src/workers/action-handler.ts
-function buildErrorResponse(requestId, code, message) {
-  return { type: "error", requestId, code, message };
+function buildErrorResponse(requestId, code, message, details) {
+  return details === void 0 || Object.keys(details).length === 0 ? { type: "error", requestId, code, message } : { type: "error", requestId, code, message, details };
 }
 function buildSuccessResponse(requestId, data) {
   return { type: "success", requestId, data };
@@ -17,7 +18,7 @@ async function importBootstrapModule(moduleUrl) {
   await import(moduleUrl);
 }
 async function stopServingRequests() {
-  const { closeRequestThread } = await import('../thread-PZLEO6JL.mjs');
+  const { closeRequestThread } = await import('../thread-5UP6MGED.mjs');
   closeRequestThread();
 }
 function holdsCopies(executor) {
@@ -48,7 +49,7 @@ function createActionHandler(executor) {
         if (!holdsCopies(executor)) {
           throw new NarsilError(ErrorCodes.CONFIG_INVALID, "This executor holds no copies to answer requests from");
         }
-        const starting = import('../thread-PZLEO6JL.mjs').then(
+        const starting = import('../thread-5UP6MGED.mjs').then(
           ({ serveRequests }) => serveRequests(executor, action.settings)
         );
         serving.start = starting.then(
@@ -69,9 +70,10 @@ function createActionHandler(executor) {
       const result = await executor.execute(action);
       post(buildSuccessResponse(action.requestId, result));
     } catch (err) {
-      const code = err?.code ?? "UNKNOWN_ERROR";
+      const failure = isNarsilError(err) ? err : null;
+      const code = failure?.code ?? ErrorCodes.WORKER_ACTION_FAILED;
       const message = err instanceof Error ? err.message : String(err);
-      post(buildErrorResponse(action.requestId, code, message));
+      post(buildErrorResponse(action.requestId, code, message, failure?.details));
     }
     return false;
   };
@@ -91,22 +93,30 @@ function workerIdOf(workerData) {
 }
 async function setup() {
   let parentPort = null;
-  let scratchSlot;
+  let threadSlot;
   try {
     const workerThreads = await import('worker_threads');
     parentPort = workerThreads.parentPort ?? null;
-    scratchSlot = workerIdOf(workerThreads.workerData);
+    const workerId = workerIdOf(workerThreads.workerData);
+    threadSlot = workerId === void 0 ? void 0 : threadSlotOfWorker(workerId);
   } catch {
     parentPort = null;
   }
-  const { createDirectExecutor } = await import('../direct-executor-OREXU72C.mjs');
-  const handleAction = createActionHandler(createDirectExecutor({ scratchSlot }));
+  const { createDirectExecutor } = await import('../direct-executor-RBA5O7EN.mjs');
+  const handleAction = createActionHandler(createDirectExecutor({ threadSlot }));
   if (parentPort) {
     const port = parentPort;
+    startIdleHeapCleanup();
     port.on("message", (raw) => {
       if (!isValidWorkerAction(raw)) {
         const requestId = raw?.requestId ?? "unknown";
-        port.postMessage(buildErrorResponse(requestId, "INVALID_ACTION", "Received an invalid worker action"));
+        port.postMessage(
+          buildErrorResponse(
+            requestId,
+            ErrorCodes.WORKER_ACTION_FAILED,
+            "This worker received a message that is no valid action"
+          )
+        );
         return;
       }
       handleAction(raw, (msg) => port.postMessage(msg)).then((shouldClose) => {
@@ -124,7 +134,13 @@ async function setup() {
       const raw = event.data;
       if (!isValidWorkerAction(raw)) {
         const requestId = raw?.requestId ?? "unknown";
-        webSelf.postMessage(buildErrorResponse(requestId, "INVALID_ACTION", "Received an invalid worker action"));
+        webSelf.postMessage(
+          buildErrorResponse(
+            requestId,
+            ErrorCodes.WORKER_ACTION_FAILED,
+            "This worker received a message that is no valid action"
+          )
+        );
         return;
       }
       handleAction(raw, (msg) => webSelf.postMessage(msg)).then((shouldClose) => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@delali/narsil",
-  "version": "0.2.4-next.51",
+  "version": "0.2.4-next.54",
   "type": "module",
   "description": "A distributed full-text search engine with cross-language portable binary format.",
   "author": "Delali (https://sondelali.com)",
@@ -34,6 +34,22 @@
     "#platform/heap-statistics": {
       "browser": "./src/runtime/heap-statistics.browser.ts",
       "default": "./src/runtime/heap-statistics.ts"
+    },
+    "#platform/idle-heap-cleanup": {
+      "browser": "./src/runtime/idle-heap-cleanup.browser.ts",
+      "default": "./src/runtime/idle-heap-cleanup.ts"
+    },
+    "#platform/vector-file": {
+      "browser": "./src/vector/vector-store/vector-file.browser.ts",
+      "default": "./src/vector/vector-store/vector-file.ts"
+    },
+    "#platform/native-core": {
+      "browser": "./src/vector/native/loader.browser.ts",
+      "default": "./src/vector/native/loader.ts"
+    },
+    "#platform/native-crc32": {
+      "browser": "./src/serialization/native-crc32.browser.ts",
+      "default": "./src/serialization/native-crc32.ts"
     }
   },
   "sideEffects": false,
@@ -49,6 +65,31 @@
   "engines": {
     "node": ">=22"
   },
+  "scripts": {
+    "build": "rm -rf dist && tsup && node scripts/check-browser-bundle.mjs && tsx scripts/api-surface.ts",
+    "test": "vitest run --exclude='**/vector/recall.test.ts'",
+    "test:etcd": "vitest run src/__tests__/distribution/coordinator/etcd-integration/*.test.ts",
+    "test:watch": "vitest",
+    "test:coverage": "vitest run --coverage",
+    "test:recall": "vitest run src/__tests__/vector/recall.test.ts",
+    "test:embeddings": "NARSIL_TEST_EMBEDDINGS=1 vitest run embedding-e2e",
+    "bench": "vitest bench",
+    "stemmers": "tsx snowball/build.ts",
+    "revisions": "tsx scripts/analysis-revisions.ts",
+    "revisions:write": "tsx scripts/analysis-revisions.ts --write",
+    "revisions:record": "tsx scripts/analysis-revisions.ts --record",
+    "api": "tsx scripts/api-surface.ts",
+    "api:write": "tsx scripts/api-surface.ts --write",
+    "lookalikes": "tsx scripts/lookalike-counts.ts",
+    "lookalikes:collect": "tsx scripts/lookalike-counts.ts --collect",
+    "fold-table": "tsx scripts/fold-table.ts",
+    "fold-table:write": "tsx scripts/fold-table.ts --write",
+    "lint": "biome check src/",
+    "typecheck:src": "tsc --noEmit",
+    "typecheck:tests": "tsc --noEmit -p tsconfig.test.json",
+    "typecheck:tooling": "tsc --noEmit -p scripts/tsconfig.json && tsc --noEmit -p benchmarks/tsconfig.json && tsc --noEmit -p wasm/tsconfig.json && tsc --noEmit -p snowball/tsconfig.json",
+    "typecheck": "pnpm run typecheck:src && pnpm run typecheck:tests && pnpm run typecheck:tooling"
+  },
   "exports": {
     ".": {
       "browser": {
@@ -82,10 +123,6 @@
       "import": "./dist/invalidation/broadcast-channel.mjs",
       "types": "./dist/invalidation/broadcast-channel.d.ts"
     },
-    "./vector/hnsw-build-worker": {
-      "import": "./dist/vector/hnsw-build-worker.mjs",
-      "types": "./dist/vector/hnsw-build-worker.d.ts"
-    },
     "./vector/search-worker": {
       "import": "./dist/vector/search-worker.mjs",
       "types": "./dist/vector/search-worker.d.ts"
@@ -145,6 +182,7 @@
   },
   "devDependencies": {
     "@biomejs/biome": "2.4.4",
+    "@delali/narsil-embeddings-transformers": "workspace:*",
     "@grpc/grpc-js": "1.14.4",
     "@microsoft/api-extractor": "7.58.12",
     "@types/node": "22.15.32",
@@ -162,8 +200,7 @@
     "typescript": "5.9.3",
     "uWebSockets.js": "github:uNetworking/uWebSockets.js#v20.58.0",
     "vitest": "4.0.18",
-    "wabt": "1.0.39",
-    "@delali/narsil-embeddings-transformers": "0.1.7"
+    "wabt": "1.0.39"
   },
   "peerDependencies": {
     "@grpc/grpc-js": "1.14.4",
@@ -185,28 +222,12 @@
       "optional": true
     }
   },
-  "scripts": {
-    "build": "rm -rf dist && tsup && node scripts/check-browser-bundle.mjs && tsx scripts/api-surface.ts",
-    "test": "vitest run --exclude='**/vector/recall.test.ts'",
-    "test:etcd": "vitest run src/__tests__/distribution/coordinator/etcd-integration/*.test.ts",
-    "test:watch": "vitest",
-    "test:coverage": "vitest run --coverage",
-    "test:recall": "vitest run src/__tests__/vector/recall.test.ts",
-    "test:embeddings": "NARSIL_TEST_EMBEDDINGS=1 vitest run embedding-e2e",
-    "bench": "vitest bench",
-    "stemmers": "tsx snowball/build.ts",
-    "revisions": "tsx scripts/analysis-revisions.ts",
-    "revisions:write": "tsx scripts/analysis-revisions.ts --write",
-    "api": "tsx scripts/api-surface.ts",
-    "api:write": "tsx scripts/api-surface.ts --write",
-    "lookalikes": "tsx scripts/lookalike-counts.ts",
-    "lookalikes:collect": "tsx scripts/lookalike-counts.ts --collect",
-    "fold-table": "tsx scripts/fold-table.ts",
-    "fold-table:write": "tsx scripts/fold-table.ts --write",
-    "lint": "biome check src/",
-    "typecheck:src": "tsc --noEmit",
-    "typecheck:tests": "tsc --noEmit -p tsconfig.test.json",
-    "typecheck:tooling": "tsc --noEmit -p scripts/tsconfig.json && tsc --noEmit -p benchmarks/tsconfig.json && tsc --noEmit -p wasm/tsconfig.json && tsc --noEmit -p snowball/tsconfig.json",
-    "typecheck": "pnpm run typecheck:src && pnpm run typecheck:tests && pnpm run typecheck:tooling"
+  "optionalDependencies": {
+    "@delali/narsil-native-darwin-arm64": "0.2.4-next.54",
+    "@delali/narsil-native-darwin-x64": "0.2.4-next.54",
+    "@delali/narsil-native-linux-arm64": "0.2.4-next.54",
+    "@delali/narsil-native-linux-x64": "0.2.4-next.54",
+    "@delali/narsil-native-win32-arm64": "0.2.4-next.54",
+    "@delali/narsil-native-win32-x64": "0.2.4-next.54"
   }
-}
+}

package/dist/brute-force-DbHYCTc3.d.ts

@@ -1,8 +0,0 @@
-/**
- * Similarity metric a vector search ranks by.
- *
- * @internal
- */
-type VectorMetric = 'cosine' | 'dotProduct' | 'euclidean';
-
-export type { VectorMetric as V };

package/dist/chunk-CBLS46IE.mjs

@@ -1,10 +0,0 @@
-// src/search/constants.ts
-var RESULT_WINDOW = 1e4;
-var DEFAULT_PAGE_SIZE = 10;
-var MAX_CURSOR_LENGTH = 40960;
-var MAX_SORT_FIELDS = 8;
-var MAX_SORT_FIELD_NAME_LENGTH = 255;
-var MAX_CURSOR_ANCHOR_CODE_POINTS = 512;
-var PREFIX_MAX_EXPANSIONS = 50;
-
-export { DEFAULT_PAGE_SIZE, MAX_CURSOR_ANCHOR_CODE_POINTS, MAX_CURSOR_LENGTH, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, PREFIX_MAX_EXPANSIONS, RESULT_WINDOW };

package/dist/chunk-PIWKZPP7.mjs

@@ -1,64 +0,0 @@
-import { NarsilError, ErrorCodes } from './chunk-P4ELXVJQ.mjs';
-
-// src/narsil/internals.ts
-var engineCores = /* @__PURE__ */ new WeakMap();
-function bindEngineCore(engine, core) {
-  engineCores.set(engine, core);
-}
-function engineCoreOf(engine) {
-  return engineCores.get(engine);
-}
-
-// src/distribution/constants.ts
-var WIRE_BATCH_BUDGET = {
-  maxCount: 1e3,
-  maxBytes: 8388608,
-  overheadBytes: 256
-};
-var MAX_PARTITION_COUNT = 65536;
-var MAX_REPLICATION_FACTOR = 255;
-
-// src/engine/resolve-vector-text.ts
-async function resolveVectorText(params, embeddingAdapter, signal, embeddingAdapterName) {
-  if (!params.vector) return params;
-  if (params.vector.text === void 0 && params.vector.value === void 0) return params;
-  if (params.vector.value !== void 0 && params.vector.text === void 0) return params;
-  if (params.vector.text !== void 0 && params.vector.value !== void 0) {
-    throw new NarsilError(ErrorCodes.EMBEDDING_CONFIG_INVALID, "Vector query cannot have both 'text' and 'value'");
-  }
-  if (!embeddingAdapter) {
-    throw new NarsilError(
-      ErrorCodes.EMBEDDING_CONFIG_INVALID,
-      embeddingAdapterName ? `Vector query with 'text' needs embedding adapter "${embeddingAdapterName}", which is not registered on this engine. Register it with registerEmbeddingAdapter or supply a raw vector.` : "Vector query with 'text' requires an embedding adapter on the index or instance",
-      embeddingAdapterName ? { adapter: embeddingAdapterName } : void 0
-    );
-  }
-  if (typeof params.vector.text !== "string") {
-    throw new NarsilError(
-      ErrorCodes.DOC_VALIDATION_FAILED,
-      `Vector query 'text' must be a string, got ${typeof params.vector.text}`
-    );
-  }
-  if (params.vector.text.length === 0) {
-    throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Vector query 'text' must not be empty");
-  }
-  const raw = await embeddingAdapter.embed(params.vector.text, "query", signal);
-  if (!(raw instanceof Float32Array)) {
-    throw new NarsilError(
-      ErrorCodes.EMBEDDING_FAILED,
-      `Adapter returned ${typeof raw} for query embedding, expected Float32Array`
-    );
-  }
-  if (raw.length !== embeddingAdapter.dimensions) {
-    throw new NarsilError(
-      ErrorCodes.EMBEDDING_DIMENSION_MISMATCH,
-      `Adapter returned ${raw.length}-dimensional vector for query, expected ${embeddingAdapter.dimensions}`,
-      { expected: embeddingAdapter.dimensions, actual: raw.length }
-    );
-  }
-  const resolved = { ...params.vector, value: Array.from(raw) };
-  delete resolved.text;
-  return { ...params, vector: resolved };
-}
-
-export { MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR, WIRE_BATCH_BUDGET, bindEngineCore, engineCoreOf, resolveVectorText };