@delali/narsil 0.2.4-next.51 → 0.2.4-next.54

package/dist/chunk-PMBHH44R.mjs

@@ -1,492 +0,0 @@
-import { createSharedArenaSimd, importAdjacency, createHNSWWorkspace, createArenaSimd, WASM_PAGE_BYTES, VECTOR_STORE_INITIAL_CAPACITY, arenaFloat32Distance, ABSENT_DOCUMENT_RANK, QUANTIZATION_PADDING_FACTOR } from './chunk-X6YZO4MQ.mjs';
-
-// src/vector/scalar-quantization-math.ts
-function deriveSq8Constants(alpha, offset, dimensions) {
-  return {
-    alpha,
-    offset,
-    alphaSquared: alpha * alpha,
-    alphaTimesOffset: alpha * offset,
-    dimensionsTimesOffsetSquared: dimensions * offset * offset,
-    invAlpha: alpha > 0 ? 1 / alpha : 0
-  };
-}
-function quantizeVectorInto(target, vector, dimensions, constants) {
-  const { offset, invAlpha } = constants;
-  let sum = 0;
-  let sumSq = 0;
-  for (let d = 0; d < dimensions; d++) {
-    const normalized = (vector[d] - offset) * invAlpha;
-    const scaled = normalized + 0.5;
-    const q = scaled < 0 ? 0 : scaled > 255 ? 255 : scaled | 0;
-    target[d] = q;
-    sum += q;
-    sumSq += q * q;
-  }
-  return { sum, sumSq };
-}
-function quantizedMagnitude(constants, sumSq, sum) {
-  const val = constants.alphaSquared * sumSq + 2 * constants.alphaTimesOffset * sum + constants.dimensionsTimesOffsetSquared;
-  return val > 0 ? Math.sqrt(val) : 0;
-}
-function realDotFromInt(constants, intDot, querySum, documentSum) {
-  return constants.alphaSquared * intDot + constants.alphaTimesOffset * (querySum + documentSum) + constants.dimensionsTimesOffsetSquared;
-}
-function scalarQuantizedDistance(codes, documentSums, documentMagnitudes, dimensions, constants, queryQuantized, querySum, queryMagnitude, ordinal, metric) {
-  const base = ordinal * dimensions;
-  if (metric === "euclidean") {
-    let intSqDist = 0;
-    for (let d = 0; d < dimensions; d++) {
-      const diff = queryQuantized[d] - codes[base + d];
-      intSqDist += diff * diff;
-    }
-    return constants.alpha * Math.sqrt(intSqDist);
-  }
-  let intDot = 0;
-  for (let d = 0; d < dimensions; d++) {
-    intDot += queryQuantized[d] * codes[base + d];
-  }
-  const realDot = realDotFromInt(constants, intDot, querySum, documentSums[ordinal]);
-  if (metric === "dotProduct") {
-    return -realDot;
-  }
-  const vecMag = documentMagnitudes[ordinal];
-  if (!vecMag || vecMag === 0 || queryMagnitude === 0) return 1;
-  return 1 - realDot / (queryMagnitude * vecMag);
-}
-function arenaQuantizedDistance(simd, queryByteOffset, codeByteOffset, dimensions, constants, querySum, queryMagnitude, documentSum, documentMagnitude, metric) {
-  if (metric === "euclidean") {
-    const intSqDist = simd.sqdist_u8(queryByteOffset, codeByteOffset, dimensions);
-    return constants.alpha * Math.sqrt(intSqDist);
-  }
-  const intDot = simd.dot_u8(queryByteOffset, codeByteOffset, dimensions);
-  const realDot = realDotFromInt(constants, intDot, querySum, documentSum);
-  if (metric === "dotProduct") {
-    return -realDot;
-  }
-  if (!documentMagnitude || documentMagnitude === 0 || queryMagnitude === 0) return 1;
-  return 1 - realDot / (queryMagnitude * documentMagnitude);
-}
-
-// src/vector/shared-generation/worker-view.ts
-function createVectorReader(snapshot, simd, scratchSlot) {
-  const { layout, magnitudes, rankByOrdinal } = snapshot;
-  const { dimension, slots } = layout;
-  const vectors = new Float32Array(snapshot.memory.buffer, layout.vectorsOffset, slots * dimension);
-  const scratchByteOffset = scratchSlot * layout.float32ScratchStride;
-  const scratch = simd ? new Float32Array(snapshot.memory.buffer, scratchByteOffset, dimension) : null;
-  const holdsDocument = (ordinal) => ordinal >= 0 && ordinal < slots && rankByOrdinal[ordinal] !== ABSENT_DOCUMENT_RANK;
-  return {
-    entryForOrdinal(ordinal) {
-      if (!holdsDocument(ordinal)) return void 0;
-      const base = ordinal * dimension;
-      return { vector: vectors.subarray(base, base + dimension), magnitude: magnitudes[ordinal] };
-    },
-    prepareQueryArena(query) {
-      if (!simd || !scratch || query.length !== dimension) return null;
-      scratch.set(query, 0);
-      return { magnitude: simd.magnitude(scratchByteOffset, dimension) };
-    },
-    distanceFromArena(prepared, ordinal, metric) {
-      if (!simd || !holdsDocument(ordinal)) return Number.POSITIVE_INFINITY;
-      const byteOffset = layout.vectorsOffset + ordinal * dimension * 4;
-      return arenaFloat32Distance(
-        simd,
-        scratchByteOffset,
-        byteOffset,
-        dimension,
-        metric,
-        prepared.magnitude,
-        magnitudes[ordinal]
-      );
-    }
-  };
-}
-function createQuantizerReader(snapshot, simd, scratchSlot) {
-  if (snapshot.quantization !== "sq8" || snapshot.calibration === null) return void 0;
-  const { layout, codeSums, codeMagnitudes, codePresent } = snapshot;
-  const { dimension, slots } = layout;
-  const constants = deriveSq8Constants(snapshot.calibration.alpha, snapshot.calibration.offset, dimension);
-  const codes = new Uint8Array(snapshot.memory.buffer, layout.codesOffset, slots * dimension);
-  const scratchByteOffset = layout.codeScratchOffset + scratchSlot * layout.codeScratchStride;
-  const scratch = simd ? new Uint8Array(snapshot.memory.buffer, scratchByteOffset, dimension) : null;
-  let liveCodes = 0;
-  for (let ordinal = 0; ordinal < codePresent.length; ordinal++) {
-    if (codePresent[ordinal] === 1) liveCodes++;
-  }
-  const holdsCodes = (ordinal) => ordinal >= 0 && ordinal < slots && codePresent[ordinal] === 1;
-  return {
-    get size() {
-      return liveCodes;
-    },
-    isCalibrated: () => true,
-    prepareQuery(query) {
-      const quantized = new Uint8Array(dimension);
-      const { sum, sumSq } = quantizeVectorInto(quantized, query, dimension, constants);
-      return { quantized, sum, sumSq, magnitude: quantizedMagnitude(constants, sumSq, sum) };
-    },
-    distanceFromPreparedByOrdinal(prepared, ordinal, metric) {
-      if (!holdsCodes(ordinal)) return Number.POSITIVE_INFINITY;
-      return scalarQuantizedDistance(
-        codes,
-        codeSums,
-        codeMagnitudes,
-        dimension,
-        constants,
-        prepared.quantized,
-        prepared.sum,
-        prepared.magnitude,
-        ordinal,
-        metric
-      );
-    },
-    prepareQueryArena(query) {
-      if (!simd || !scratch) return null;
-      const { sum, sumSq } = quantizeVectorInto(scratch, query, dimension, constants);
-      return { sum, sumSq, magnitude: quantizedMagnitude(constants, sumSq, sum) };
-    },
-    distanceFromArena(prepared, ordinal, metric) {
-      if (!simd || !holdsCodes(ordinal)) return Number.POSITIVE_INFINITY;
-      return arenaQuantizedDistance(
-        simd,
-        scratchByteOffset,
-        layout.codesOffset + ordinal * dimension,
-        dimension,
-        constants,
-        prepared.sum,
-        prepared.magnitude,
-        codeSums[ordinal],
-        codeMagnitudes[ordinal],
-        metric
-      );
-    }
-  };
-}
-function openSharedWorkerCopy(snapshot, scratchSlot) {
-  if (scratchSlot < 0 || scratchSlot >= snapshot.layout.workerSlots) {
-    throw new Error(`Scratch slot ${scratchSlot} is outside the ${snapshot.layout.workerSlots} reserved slots`);
-  }
-  const simd = createSharedArenaSimd(snapshot.memory);
-  const adjacency = importAdjacency(snapshot.graph.adjacency);
-  const capacity = Math.max(snapshot.graph.capacity, adjacency.capacity);
-  const searchState = {
-    dimension: snapshot.dimension,
-    store: createVectorReader(snapshot, simd, scratchSlot),
-    quantizer: createQuantizerReader(snapshot, simd, scratchSlot),
-    adjacency,
-    tombstones: snapshot.graph.tombstones,
-    tombstoneCount: snapshot.graph.tombstoneCount,
-    nodeCount: snapshot.graph.nodeCount,
-    capacity,
-    visited: new Uint32Array(capacity),
-    visitStamp: 0,
-    entryPointOrd: snapshot.graph.entryPointOrd,
-    topLayer: snapshot.graph.topLayer,
-    workspace: createHNSWWorkspace()
-  };
-  return { searchState, rankByOrdinal: snapshot.rankByOrdinal };
-}
-
-// src/vector/scalar-quantization-calibration.ts
-function computeCalibrationBounds(vectors, dimensions) {
-  let globalMin = Number.POSITIVE_INFINITY;
-  let globalMax = Number.NEGATIVE_INFINITY;
-  let count = 0;
-  for (const vec of vectors) {
-    for (let d = 0; d < dimensions; d++) {
-      const val = vec[d];
-      if (val < globalMin) globalMin = val;
-      if (val > globalMax) globalMax = val;
-    }
-    count++;
-  }
-  if (count === 0) return null;
-  const range = globalMax - globalMin;
-  const pad = range * QUANTIZATION_PADDING_FACTOR;
-  globalMin -= pad;
-  globalMax += pad;
-  if (globalMin === globalMax) {
-    globalMin -= 1e-3;
-    globalMax += 1e-3;
-  }
-  return { alpha: (globalMax - globalMin) / 255, offset: globalMin };
-}
-
-// src/vector/scalar-quantization.ts
-function createScalarQuantizer(dimensions, ordinalSource) {
-  const docToOrd = /* @__PURE__ */ new Map();
-  let simd = createArenaSimd();
-  const arenaByteOffset = simd ? Math.max(16, Math.ceil(dimensions / 16) * 16) : 0;
-  let capacity = 0;
-  let quantizedArena = simd ? new Uint8Array(simd.memory.buffer, arenaByteOffset) : new Uint8Array(0);
-  let scratch = simd ? new Uint8Array(simd.memory.buffer, 0, arenaByteOffset) : new Uint8Array(0);
-  let sums = new Float64Array(0);
-  let sumSqs = new Float64Array(0);
-  let mags = new Float64Array(0);
-  let present = new Uint8Array(0);
-  let selfNextOrd = 0;
-  let liveCount = 0;
-  let constants = deriveSq8Constants(0, 0, dimensions);
-  let calibrated = false;
-  function ensureCapacity(needed) {
-    if (needed <= capacity) return;
-    let newCap = capacity === 0 ? VECTOR_STORE_INITIAL_CAPACITY : capacity;
-    while (newCap < needed) newCap *= 2;
-    const nextSums = new Float64Array(newCap);
-    nextSums.set(sums);
-    sums = nextSums;
-    const nextSumSqs = new Float64Array(newCap);
-    nextSumSqs.set(sumSqs);
-    sumSqs = nextSumSqs;
-    const nextMags = new Float64Array(newCap);
-    nextMags.set(mags);
-    mags = nextMags;
-    const nextPresent = new Uint8Array(newCap);
-    nextPresent.set(present);
-    present = nextPresent;
-    if (simd) {
-      const requiredBytes = arenaByteOffset + newCap * dimensions;
-      const have = simd.memory.buffer.byteLength;
-      if (requiredBytes > have) {
-        try {
-          simd.memory.grow(Math.ceil((requiredBytes - have) / WASM_PAGE_BYTES));
-        } catch {
-          const migrated = new Uint8Array(newCap * dimensions);
-          migrated.set(quantizedArena.subarray(0, capacity * dimensions));
-          quantizedArena = migrated;
-          simd = null;
-          capacity = newCap;
-          return;
-        }
-      }
-      quantizedArena = new Uint8Array(simd.memory.buffer, arenaByteOffset);
-      scratch = new Uint8Array(simd.memory.buffer, 0, arenaByteOffset);
-    } else {
-      const nextArena = new Uint8Array(newCap * dimensions);
-      nextArena.set(quantizedArena);
-      quantizedArena = nextArena;
-    }
-    capacity = newCap;
-  }
-  function resolveOrdinal(docId) {
-    const fromSource = ordinalSource?.getOrdinal(docId);
-    if (fromSource !== void 0) return fromSource;
-    const known = docToOrd.get(docId);
-    if (known !== void 0) return known;
-    return selfNextOrd++;
-  }
-  function quantizeInto(ord, vector) {
-    const base = ord * dimensions;
-    const { sum, sumSq } = quantizeVectorInto(
-      quantizedArena.subarray(base, base + dimensions),
-      vector,
-      dimensions,
-      constants
-    );
-    sums[ord] = sum;
-    sumSqs[ord] = sumSq;
-    mags[ord] = quantizedMagnitude(constants, sumSq, sum);
-  }
-  function storeEntry(docId, ord) {
-    if (present[ord] === 0) {
-      present[ord] = 1;
-      liveCount++;
-    }
-    docToOrd.set(docId, ord);
-  }
-  function calibrateFromVectors(vectors) {
-    const bounds = computeCalibrationBounds(vectors, dimensions);
-    if (bounds === null) return;
-    constants = deriveSq8Constants(bounds.alpha, bounds.offset, dimensions);
-    calibrated = true;
-  }
-  function isOutsideBounds(vector) {
-    const currentMin = constants.offset;
-    const currentMax = constants.offset + constants.alpha * 255;
-    for (let d = 0; d < dimensions; d++) {
-      if (vector[d] < currentMin || vector[d] > currentMax) {
-        return true;
-      }
-    }
-    return false;
-  }
-  function distanceScalar(queryQuantized, querySum, queryMagnitude, ord, metric) {
-    if (ord < 0 || ord >= capacity || present[ord] === 0) return Number.POSITIVE_INFINITY;
-    return scalarQuantizedDistance(
-      quantizedArena,
-      sums,
-      mags,
-      dimensions,
-      constants,
-      queryQuantized,
-      querySum,
-      queryMagnitude,
-      ord,
-      metric
-    );
-  }
-  return {
-    get dimensions() {
-      return dimensions;
-    },
-    get size() {
-      return liveCount;
-    },
-    get calibration() {
-      return calibrated ? { alpha: constants.alpha, offset: constants.offset } : null;
-    },
-    isCalibrated() {
-      return calibrated;
-    },
-    calibrate(vectors) {
-      calibrateFromVectors(vectors);
-    },
-    needsRecalibration(vector) {
-      if (!calibrated) return false;
-      return isOutsideBounds(vector);
-    },
-    recalibrateAll(vectors) {
-      const collected = [];
-      const rawVectors = [];
-      for (const pair of vectors) {
-        collected.push(pair);
-        rawVectors.push(pair[1]);
-      }
-      calibrateFromVectors(rawVectors);
-      docToOrd.clear();
-      present.fill(0);
-      liveCount = 0;
-      for (const [docId, vec] of collected) {
-        const ord = resolveOrdinal(docId);
-        ensureCapacity(ord + 1);
-        quantizeInto(ord, vec);
-        storeEntry(docId, ord);
-      }
-    },
-    quantize(docId, vector) {
-      if (!calibrated) {
-        calibrateFromVectors([vector]);
-      }
-      const ord = resolveOrdinal(docId);
-      ensureCapacity(ord + 1);
-      quantizeInto(ord, vector);
-      storeEntry(docId, ord);
-    },
-    remove(docId) {
-      const ord = docToOrd.get(docId);
-      if (ord === void 0) return;
-      docToOrd.delete(docId);
-      if (present[ord] === 1) {
-        present[ord] = 0;
-        liveCount--;
-      }
-    },
-    getQuantized(docId) {
-      const ord = docToOrd.get(docId);
-      if (ord === void 0 || present[ord] === 0) return void 0;
-      const base = ord * dimensions;
-      return quantizedArena.subarray(base, base + dimensions);
-    },
-    prepareQuery(query) {
-      if (!calibrated) return null;
-      const quantized = new Uint8Array(dimensions);
-      const { sum, sumSq } = quantizeVectorInto(quantized, query, dimensions, constants);
-      const mag = quantizedMagnitude(constants, sumSq, sum);
-      return { quantized, sum, sumSq, magnitude: mag };
-    },
-    distanceFromPrepared(prepared, docId, metric) {
-      const ord = docToOrd.get(docId);
-      if (ord === void 0) return Number.POSITIVE_INFINITY;
-      return distanceScalar(prepared.quantized, prepared.sum, prepared.magnitude, ord, metric);
-    },
-    distanceFromPreparedByOrdinal(prepared, ordinal, metric) {
-      return distanceScalar(prepared.quantized, prepared.sum, prepared.magnitude, ordinal, metric);
-    },
-    prepareQueryArena(query) {
-      if (!calibrated || !simd) return null;
-      const { sum, sumSq } = quantizeVectorInto(scratch, query, dimensions, constants);
-      const magnitude = quantizedMagnitude(constants, sumSq, sum);
-      return { sum, sumSq, magnitude };
-    },
-    distanceFromArena(prepared, ordinal, metric) {
-      if (!simd || ordinal < 0 || ordinal >= capacity || present[ordinal] === 0) {
-        return Number.POSITIVE_INFINITY;
-      }
-      const arenaOffset = arenaByteOffset + ordinal * dimensions;
-      return arenaQuantizedDistance(
-        simd,
-        0,
-        arenaOffset,
-        dimensions,
-        constants,
-        prepared.sum,
-        prepared.magnitude,
-        sums[ordinal],
-        mags[ordinal],
-        metric
-      );
-    },
-    hasOrdinal(ordinal) {
-      return ordinal >= 0 && ordinal < capacity && present[ordinal] === 1;
-    },
-    restoreCalibration(restoredAlpha, restoredOffset) {
-      constants = deriveSq8Constants(restoredAlpha, restoredOffset, dimensions);
-      calibrated = true;
-    },
-    restoreEntry(docId, quantized, sum, sumSq) {
-      const ord = resolveOrdinal(docId);
-      ensureCapacity(ord + 1);
-      const base = ord * dimensions;
-      quantizedArena.set(quantized.subarray(0, dimensions), base);
-      sums[ord] = sum;
-      sumSqs[ord] = sumSq;
-      mags[ord] = quantizedMagnitude(constants, sumSq, sum);
-      storeEntry(docId, ord);
-    },
-    copyStateInto(codes, targetSums, targetSumSqs, targetMagnitudes, targetPresent) {
-      const copySlots = Math.min(capacity, Math.floor(codes.length / dimensions));
-      if (copySlots <= 0) return;
-      codes.set(quantizedArena.subarray(0, copySlots * dimensions));
-      targetSums.set(sums.subarray(0, copySlots));
-      targetSumSqs.set(sumSqs.subarray(0, copySlots));
-      targetMagnitudes.set(mags.subarray(0, copySlots));
-      targetPresent.set(present.subarray(0, copySlots));
-    },
-    serialize() {
-      const serializedVectors = {};
-      const serializedSums = {};
-      const serializedSumSqs = {};
-      for (const [docId, ord] of docToOrd) {
-        if (present[ord] === 0) continue;
-        const base = ord * dimensions;
-        serializedVectors[docId] = Array.from(quantizedArena.subarray(base, base + dimensions));
-        serializedSums[docId] = sums[ord];
-        serializedSumSqs[docId] = sumSqs[ord];
-      }
-      return {
-        alpha: constants.alpha,
-        offset: constants.offset,
-        quantizedVectors: serializedVectors,
-        vectorSums: serializedSums,
-        vectorSumSqs: serializedSumSqs
-      };
-    },
-    clear() {
-      docToOrd.clear();
-      if (!simd) {
-        quantizedArena = new Uint8Array(0);
-      }
-      sums = new Float64Array(0);
-      sumSqs = new Float64Array(0);
-      mags = new Float64Array(0);
-      present = new Uint8Array(0);
-      capacity = 0;
-      selfNextOrd = 0;
-      liveCount = 0;
-      constants = deriveSq8Constants(0, 0, dimensions);
-      calibrated = false;
-    }
-  };
-}
-
-export { createScalarQuantizer, openSharedWorkerCopy };

package/dist/chunk-RP23PWMV.mjs

@@ -1,11 +0,0 @@
-// src/vector/hnsw/constants.ts
-var MAX_LAYER_CAP = 32;
-var MAX_M = 512;
-var INITIAL_UPPER_SLOTS = 64;
-var INITIAL_LIST_CAPACITY = 64;
-var COMPACTION_TOMBSTONE_RATIO = 0.1;
-var COMPACTION_ABSOLUTE_THRESHOLD = 1e3;
-var SQ8_OVERSELECTION_FACTOR = 2;
-var SQ8_RERANK_FLOOR = 10;
-
-export { COMPACTION_ABSOLUTE_THRESHOLD, COMPACTION_TOMBSTONE_RATIO, INITIAL_LIST_CAPACITY, INITIAL_UPPER_SLOTS, MAX_LAYER_CAP, MAX_M, SQ8_OVERSELECTION_FACTOR, SQ8_RERANK_FLOOR };