@delali/narsil 0.2.4-next.49 → 0.2.4-next.51

package/dist/chunk-PIWKZPP7.mjs

@@ -0,0 +1,64 @@
+import { NarsilError, ErrorCodes } from './chunk-P4ELXVJQ.mjs';
+
+// src/narsil/internals.ts
+var engineCores = /* @__PURE__ */ new WeakMap();
+function bindEngineCore(engine, core) {
+  engineCores.set(engine, core);
+}
+function engineCoreOf(engine) {
+  return engineCores.get(engine);
+}
+
+// src/distribution/constants.ts
+var WIRE_BATCH_BUDGET = {
+  maxCount: 1e3,
+  maxBytes: 8388608,
+  overheadBytes: 256
+};
+var MAX_PARTITION_COUNT = 65536;
+var MAX_REPLICATION_FACTOR = 255;
+
+// src/engine/resolve-vector-text.ts
+async function resolveVectorText(params, embeddingAdapter, signal, embeddingAdapterName) {
+  if (!params.vector) return params;
+  if (params.vector.text === void 0 && params.vector.value === void 0) return params;
+  if (params.vector.value !== void 0 && params.vector.text === void 0) return params;
+  if (params.vector.text !== void 0 && params.vector.value !== void 0) {
+    throw new NarsilError(ErrorCodes.EMBEDDING_CONFIG_INVALID, "Vector query cannot have both 'text' and 'value'");
+  }
+  if (!embeddingAdapter) {
+    throw new NarsilError(
+      ErrorCodes.EMBEDDING_CONFIG_INVALID,
+      embeddingAdapterName ? `Vector query with 'text' needs embedding adapter "${embeddingAdapterName}", which is not registered on this engine. Register it with registerEmbeddingAdapter or supply a raw vector.` : "Vector query with 'text' requires an embedding adapter on the index or instance",
+      embeddingAdapterName ? { adapter: embeddingAdapterName } : void 0
+    );
+  }
+  if (typeof params.vector.text !== "string") {
+    throw new NarsilError(
+      ErrorCodes.DOC_VALIDATION_FAILED,
+      `Vector query 'text' must be a string, got ${typeof params.vector.text}`
+    );
+  }
+  if (params.vector.text.length === 0) {
+    throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Vector query 'text' must not be empty");
+  }
+  const raw = await embeddingAdapter.embed(params.vector.text, "query", signal);
+  if (!(raw instanceof Float32Array)) {
+    throw new NarsilError(
+      ErrorCodes.EMBEDDING_FAILED,
+      `Adapter returned ${typeof raw} for query embedding, expected Float32Array`
+    );
+  }
+  if (raw.length !== embeddingAdapter.dimensions) {
+    throw new NarsilError(
+      ErrorCodes.EMBEDDING_DIMENSION_MISMATCH,
+      `Adapter returned ${raw.length}-dimensional vector for query, expected ${embeddingAdapter.dimensions}`,
+      { expected: embeddingAdapter.dimensions, actual: raw.length }
+    );
+  }
+  const resolved = { ...params.vector, value: Array.from(raw) };
+  delete resolved.text;
+  return { ...params, vector: resolved };
+}
+
+export { MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR, WIRE_BATCH_BUDGET, bindEngineCore, engineCoreOf, resolveVectorText };

package/dist/{chunk-HGJC5L65.mjs → chunk-PMBHH44R.mjs}

@@ -1,4 +1,4 @@
-import { createArenaSimd, WASM_PAGE_BYTES, VECTOR_STORE_INITIAL_CAPACITY, QUANTIZATION_PADDING_FACTOR } from './chunk-JXHUHFJ5.mjs';
+import { createSharedArenaSimd, importAdjacency, createHNSWWorkspace, createArenaSimd, WASM_PAGE_BYTES, VECTOR_STORE_INITIAL_CAPACITY, arenaFloat32Distance, ABSENT_DOCUMENT_RANK, QUANTIZATION_PADDING_FACTOR } from './chunk-X6YZO4MQ.mjs';
 
 // src/vector/scalar-quantization-math.ts
 function deriveSq8Constants(alpha, offset, dimensions) {
@@ -68,6 +68,125 @@ function arenaQuantizedDistance(simd, queryByteOffset, codeByteOffset, dimension
   return 1 - realDot / (queryMagnitude * documentMagnitude);
 }
 
+// src/vector/shared-generation/worker-view.ts
+function createVectorReader(snapshot, simd, scratchSlot) {
+  const { layout, magnitudes, rankByOrdinal } = snapshot;
+  const { dimension, slots } = layout;
+  const vectors = new Float32Array(snapshot.memory.buffer, layout.vectorsOffset, slots * dimension);
+  const scratchByteOffset = scratchSlot * layout.float32ScratchStride;
+  const scratch = simd ? new Float32Array(snapshot.memory.buffer, scratchByteOffset, dimension) : null;
+  const holdsDocument = (ordinal) => ordinal >= 0 && ordinal < slots && rankByOrdinal[ordinal] !== ABSENT_DOCUMENT_RANK;
+  return {
+    entryForOrdinal(ordinal) {
+      if (!holdsDocument(ordinal)) return void 0;
+      const base = ordinal * dimension;
+      return { vector: vectors.subarray(base, base + dimension), magnitude: magnitudes[ordinal] };
+    },
+    prepareQueryArena(query) {
+      if (!simd || !scratch || query.length !== dimension) return null;
+      scratch.set(query, 0);
+      return { magnitude: simd.magnitude(scratchByteOffset, dimension) };
+    },
+    distanceFromArena(prepared, ordinal, metric) {
+      if (!simd || !holdsDocument(ordinal)) return Number.POSITIVE_INFINITY;
+      const byteOffset = layout.vectorsOffset + ordinal * dimension * 4;
+      return arenaFloat32Distance(
+        simd,
+        scratchByteOffset,
+        byteOffset,
+        dimension,
+        metric,
+        prepared.magnitude,
+        magnitudes[ordinal]
+      );
+    }
+  };
+}
+function createQuantizerReader(snapshot, simd, scratchSlot) {
+  if (snapshot.quantization !== "sq8" || snapshot.calibration === null) return void 0;
+  const { layout, codeSums, codeMagnitudes, codePresent } = snapshot;
+  const { dimension, slots } = layout;
+  const constants = deriveSq8Constants(snapshot.calibration.alpha, snapshot.calibration.offset, dimension);
+  const codes = new Uint8Array(snapshot.memory.buffer, layout.codesOffset, slots * dimension);
+  const scratchByteOffset = layout.codeScratchOffset + scratchSlot * layout.codeScratchStride;
+  const scratch = simd ? new Uint8Array(snapshot.memory.buffer, scratchByteOffset, dimension) : null;
+  let liveCodes = 0;
+  for (let ordinal = 0; ordinal < codePresent.length; ordinal++) {
+    if (codePresent[ordinal] === 1) liveCodes++;
+  }
+  const holdsCodes = (ordinal) => ordinal >= 0 && ordinal < slots && codePresent[ordinal] === 1;
+  return {
+    get size() {
+      return liveCodes;
+    },
+    isCalibrated: () => true,
+    prepareQuery(query) {
+      const quantized = new Uint8Array(dimension);
+      const { sum, sumSq } = quantizeVectorInto(quantized, query, dimension, constants);
+      return { quantized, sum, sumSq, magnitude: quantizedMagnitude(constants, sumSq, sum) };
+    },
+    distanceFromPreparedByOrdinal(prepared, ordinal, metric) {
+      if (!holdsCodes(ordinal)) return Number.POSITIVE_INFINITY;
+      return scalarQuantizedDistance(
+        codes,
+        codeSums,
+        codeMagnitudes,
+        dimension,
+        constants,
+        prepared.quantized,
+        prepared.sum,
+        prepared.magnitude,
+        ordinal,
+        metric
+      );
+    },
+    prepareQueryArena(query) {
+      if (!simd || !scratch) return null;
+      const { sum, sumSq } = quantizeVectorInto(scratch, query, dimension, constants);
+      return { sum, sumSq, magnitude: quantizedMagnitude(constants, sumSq, sum) };
+    },
+    distanceFromArena(prepared, ordinal, metric) {
+      if (!simd || !holdsCodes(ordinal)) return Number.POSITIVE_INFINITY;
+      return arenaQuantizedDistance(
+        simd,
+        scratchByteOffset,
+        layout.codesOffset + ordinal * dimension,
+        dimension,
+        constants,
+        prepared.sum,
+        prepared.magnitude,
+        codeSums[ordinal],
+        codeMagnitudes[ordinal],
+        metric
+      );
+    }
+  };
+}
+function openSharedWorkerCopy(snapshot, scratchSlot) {
+  if (scratchSlot < 0 || scratchSlot >= snapshot.layout.workerSlots) {
+    throw new Error(`Scratch slot ${scratchSlot} is outside the ${snapshot.layout.workerSlots} reserved slots`);
+  }
+  const simd = createSharedArenaSimd(snapshot.memory);
+  const adjacency = importAdjacency(snapshot.graph.adjacency);
+  const capacity = Math.max(snapshot.graph.capacity, adjacency.capacity);
+  const searchState = {
+    dimension: snapshot.dimension,
+    store: createVectorReader(snapshot, simd, scratchSlot),
+    quantizer: createQuantizerReader(snapshot, simd, scratchSlot),
+    adjacency,
+    tombstones: snapshot.graph.tombstones,
+    tombstoneCount: snapshot.graph.tombstoneCount,
+    nodeCount: snapshot.graph.nodeCount,
+    capacity,
+    visited: new Uint32Array(capacity),
+    visitStamp: 0,
+    entryPointOrd: snapshot.graph.entryPointOrd,
+    topLayer: snapshot.graph.topLayer,
+    workspace: createHNSWWorkspace()
+  };
+  return { searchState, rankByOrdinal: snapshot.rankByOrdinal };
+}
+
 // src/vector/scalar-quantization-calibration.ts
 function computeCalibrationBounds(vectors, dimensions) {
   let globalMin = Number.POSITIVE_INFINITY;
@@ -370,4 +489,4 @@ function createScalarQuantizer(dimensions, ordinalSource) {
   };
 }
 
-export { arenaQuantizedDistance, createScalarQuantizer, deriveSq8Constants, quantizeVectorInto, quantizedMagnitude, scalarQuantizedDistance };
+export { createScalarQuantizer, openSharedWorkerCopy };

package/dist/chunk-RP23PWMV.mjs

@@ -0,0 +1,11 @@
+// src/vector/hnsw/constants.ts
+var MAX_LAYER_CAP = 32;
+var MAX_M = 512;
+var INITIAL_UPPER_SLOTS = 64;
+var INITIAL_LIST_CAPACITY = 64;
+var COMPACTION_TOMBSTONE_RATIO = 0.1;
+var COMPACTION_ABSOLUTE_THRESHOLD = 1e3;
+var SQ8_OVERSELECTION_FACTOR = 2;
+var SQ8_RERANK_FLOOR = 10;
+
+export { COMPACTION_ABSOLUTE_THRESHOLD, COMPACTION_TOMBSTONE_RATIO, INITIAL_LIST_CAPACITY, INITIAL_UPPER_SLOTS, MAX_LAYER_CAP, MAX_M, SQ8_OVERSELECTION_FACTOR, SQ8_RERANK_FLOOR };

package/dist/{chunk-RDGXYRF4.mjs → chunk-WYJXSLER.mjs}

@@ -1,20 +1,3 @@
-// src/core/id-generator.ts
-function generateId() {
-  const bytes = new Uint8Array(16);
-  crypto.getRandomValues(bytes);
-  const now = Date.now();
-  bytes[0] = Math.floor(now / 2 ** 40);
-  bytes[1] = Math.floor(now / 2 ** 32) % 256;
-  bytes[2] = Math.floor(now / 2 ** 24) % 256;
-  bytes[3] = Math.floor(now / 2 ** 16) % 256;
-  bytes[4] = Math.floor(now / 2 ** 8) % 256;
-  bytes[5] = now % 256;
-  bytes[6] = bytes[6] & 15 | 112;
-  bytes[8] = bytes[8] & 63 | 128;
-  const hex = Array.from(bytes, (b) => b.toString(16).padStart(2, "0")).join("");
-  return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
-}
-
 // src/workers/protocol.ts
 var KNOWN_ACTION_TYPES = [
   "insert",
@@ -26,6 +9,10 @@ var KNOWN_ACTION_TYPES = [
   "has",
   "count",
   "createIndex",
+  "loadVectorCopy",
+  "dropVectorCopy",
+  "serveRequests",
+  "stopServing",
   "dropIndex",
   "getStats",
   "clear",
@@ -41,8 +28,10 @@ var KNOWN_ACTION_TYPES = [
   "bootstrap",
   "shutdown"
 ];
+var nextRequestSequence = 0;
 function createRequestId() {
-  return generateId();
+  nextRequestSequence += 1;
+  return String(nextRequestSequence);
 }
 function isValidWorkerAction(msg) {
   if (msg === null || msg === void 0 || typeof msg !== "object") {
@@ -68,4 +57,4 @@ function isValidWorkerResponse(msg) {
   return true;
 }
 
-export { createRequestId, generateId, isValidWorkerAction, isValidWorkerResponse };
+export { createRequestId, isValidWorkerAction, isValidWorkerResponse };

package/dist/{chunk-JXHUHFJ5.mjs → chunk-X6YZO4MQ.mjs}

@@ -1,3 +1,4 @@
+import { INITIAL_LIST_CAPACITY, COMPACTION_TOMBSTONE_RATIO, COMPACTION_ABSOLUTE_THRESHOLD, MAX_LAYER_CAP, SQ8_OVERSELECTION_FACTOR, SQ8_RERANK_FLOOR, INITIAL_UPPER_SLOTS } from './chunk-RP23PWMV.mjs';
 import { compareCodePoints } from './chunk-2GWVQWGA.mjs';
 
 // src/vector/constants.ts
@@ -546,16 +547,6 @@ function createVectorStore() {
   };
 }
 
-// src/vector/hnsw/constants.ts
-var MAX_LAYER_CAP = 32;
-var MAX_M = 512;
-var INITIAL_UPPER_SLOTS = 64;
-var INITIAL_LIST_CAPACITY = 64;
-var COMPACTION_TOMBSTONE_RATIO = 0.1;
-var COMPACTION_ABSOLUTE_THRESHOLD = 1e3;
-var SQ8_OVERSELECTION_FACTOR = 2;
-var SQ8_RERANK_FLOOR = 10;
-
 // src/vector/hnsw/adjacency.ts
 var ABSENT = -1;
 function createAdjacency(m, mMax0) {
@@ -1578,4 +1569,4 @@ function createHNSWIndex(dimension, store, config, quantizer) {
   };
 }
 
-export { ABSENT_DOCUMENT_RANK, HNSW_BUILD_TIMEOUT_MS, MAX_M, MAX_WASM_PAGES, MAX_WORKER_COPIES, MAX_WORKER_DIMENSION, MAX_WORKER_VECTORS, QUANTIZATION_PADDING_FACTOR, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_STORE_INITIAL_CAPACITY, WASM_PAGE_BYTES, addToOrdinalFilter, arenaFloat32Distance, cosineSimilarityWithMagnitudes, createArenaSimd, createHNSWIndex, createHNSWWorkspace, createOrdinalFilter, createSharedArenaSimd, createVectorStore, dotProduct, euclideanDistance, importAdjacency, isSimdAvailable, magnitude, ordinalFilterHas, ordinalFilterValues, removeFromOrdinalFilter, searchOrdinals, toScore };
+export { ABSENT_DOCUMENT_RANK, HNSW_BUILD_TIMEOUT_MS, MAX_WASM_PAGES, MAX_WORKER_COPIES, MAX_WORKER_DIMENSION, MAX_WORKER_VECTORS, QUANTIZATION_PADDING_FACTOR, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_STORE_INITIAL_CAPACITY, WASM_PAGE_BYTES, addToOrdinalFilter, arenaFloat32Distance, cosineSimilarityWithMagnitudes, createArenaSimd, createHNSWIndex, createHNSWWorkspace, createOrdinalFilter, createSharedArenaSimd, createVectorStore, dotProduct, entryForOrd, euclideanDistance, importAdjacency, isSimdAvailable, magnitude, ordinalFilterHas, ordinalFilterValues, removeFromOrdinalFilter, searchOrdinals, toDistance, toScore };

package/dist/{chunk-OWCFK42Q.mjs → chunk-XE6XR2OE.mjs}

@@ -1,14 +1,5 @@
 import { DEFAULT_PAGE_SIZE } from './chunk-CBLS46IE.mjs';
 
-// src/distribution/constants.ts
-var WIRE_BATCH_BUDGET = {
-  maxCount: 1e3,
-  maxBytes: 8388608,
-  overheadBytes: 256
-};
-var MAX_PARTITION_COUNT = 65536;
-var MAX_REPLICATION_FACTOR = 255;
-
 // src/engine/constants.ts
 var MIN_DOCUMENTS_FOR_SEGMENTS = 64;
 var BATCH_CHUNK_SIZE = 1e3;
@@ -19,6 +10,8 @@ var MAX_INDEX_NAME_LENGTH = 255;
 var MAX_DOC_ID_LENGTH = 512;
 var MIN_SWEEP_INTERVAL_MS = 100;
 var MAX_SWEEP_INTERVAL_MS = 6e4;
+var HEAP_PRESSURE_WARNING_FRACTION = 0.9;
+var HEAP_PRESSURE_REARM_FRACTION = 0.8;
 var STATISTICS_BROADCAST_INTERVAL_MS = 5e3;
 var FOREIGN_STATS_TTL_MS = 6e4;
 var DEFAULT_SUGGEST_LIMIT = 10;
@@ -27,4 +20,4 @@ var SUGGEST_OVERSAMPLE_FACTOR = 1.5;
 var SUGGEST_OVERSAMPLE_PADDING = 10;
 var MAX_SUGGEST_SCATTER_LIMIT = Math.ceil(MAX_SUGGEST_LIMIT * SUGGEST_OVERSAMPLE_FACTOR) + SUGGEST_OVERSAMPLE_PADDING;
 
-export { BATCH_CHUNK_SIZE, DEFAULT_LIMIT, DEFAULT_OFFSET, DEFAULT_SUGGEST_LIMIT, FOREIGN_STATS_TTL_MS, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, MAX_INDEX_NAME_LENGTH, MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, MAX_SWEEP_INTERVAL_MS, MIN_DOCUMENTS_FOR_SEGMENTS, MIN_SWEEP_INTERVAL_MS, STATISTICS_BROADCAST_INTERVAL_MS, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING, WIRE_BATCH_BUDGET };
+export { BATCH_CHUNK_SIZE, DEFAULT_LIMIT, DEFAULT_OFFSET, DEFAULT_SUGGEST_LIMIT, FOREIGN_STATS_TTL_MS, HEAP_PRESSURE_REARM_FRACTION, HEAP_PRESSURE_WARNING_FRACTION, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, MAX_INDEX_NAME_LENGTH, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, MAX_SWEEP_INTERVAL_MS, MIN_DOCUMENTS_FOR_SEGMENTS, MIN_SWEEP_INTERVAL_MS, STATISTICS_BROADCAST_INTERVAL_MS, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING };