@delali/narsil 0.2.2-next.29 → 0.2.3-next.32

package/dist/chunk-OBXS7OZM.mjs

@@ -0,0 +1,376 @@
+import { createArenaSimd } from './chunk-WDL632CN.mjs';
+
+// src/vector/scalar-quantization-math.ts
+function deriveSq8Constants(alpha, offset, dimensions) {
+  return {
+    alpha,
+    offset,
+    alphaSquared: alpha * alpha,
+    alphaTimesOffset: alpha * offset,
+    dimensionsTimesOffsetSquared: dimensions * offset * offset,
+    invAlpha: alpha > 0 ? 1 / alpha : 0
+  };
+}
+function quantizeVectorInto(target, vector, dimensions, constants) {
+  const { offset, invAlpha } = constants;
+  let sum = 0;
+  let sumSq = 0;
+  for (let d = 0; d < dimensions; d++) {
+    const normalized = (vector[d] - offset) * invAlpha;
+    const scaled = normalized + 0.5;
+    const q = scaled < 0 ? 0 : scaled > 255 ? 255 : scaled | 0;
+    target[d] = q;
+    sum += q;
+    sumSq += q * q;
+  }
+  return { sum, sumSq };
+}
+function quantizedMagnitude(constants, sumSq, sum) {
+  const val = constants.alphaSquared * sumSq + 2 * constants.alphaTimesOffset * sum + constants.dimensionsTimesOffsetSquared;
+  return val > 0 ? Math.sqrt(val) : 0;
+}
+function realDotFromInt(constants, intDot, querySum, documentSum) {
+  return constants.alphaSquared * intDot + constants.alphaTimesOffset * (querySum + documentSum) + constants.dimensionsTimesOffsetSquared;
+}
+function scalarQuantizedDistance(codes, documentSums, documentMagnitudes, dimensions, constants, queryQuantized, querySum, queryMagnitude, ordinal, metric) {
+  const base = ordinal * dimensions;
+  if (metric === "euclidean") {
+    let intSqDist = 0;
+    for (let d = 0; d < dimensions; d++) {
+      const diff = queryQuantized[d] - codes[base + d];
+      intSqDist += diff * diff;
+    }
+    return constants.alpha * Math.sqrt(intSqDist);
+  }
+  let intDot = 0;
+  for (let d = 0; d < dimensions; d++) {
+    intDot += queryQuantized[d] * codes[base + d];
+  }
+  const realDot = realDotFromInt(constants, intDot, querySum, documentSums[ordinal]);
+  if (metric === "dotProduct") {
+    return -realDot;
+  }
+  const vecMag = documentMagnitudes[ordinal];
+  if (!vecMag || vecMag === 0 || queryMagnitude === 0) return 1;
+  return 1 - realDot / (queryMagnitude * vecMag);
+}
+function arenaQuantizedDistance(simd, queryByteOffset, codeByteOffset, dimensions, constants, querySum, queryMagnitude, documentSum, documentMagnitude, metric) {
+  if (metric === "euclidean") {
+    const intSqDist = simd.sqdist_u8(queryByteOffset, codeByteOffset, dimensions);
+    return constants.alpha * Math.sqrt(intSqDist);
+  }
+  const intDot = simd.dot_u8(queryByteOffset, codeByteOffset, dimensions);
+  const realDot = realDotFromInt(constants, intDot, querySum, documentSum);
+  if (metric === "dotProduct") {
+    return -realDot;
+  }
+  if (!documentMagnitude || documentMagnitude === 0 || queryMagnitude === 0) return 1;
+  return 1 - realDot / (queryMagnitude * documentMagnitude);
+}
+
+// src/vector/scalar-quantization-calibration.ts
+var PADDING_FACTOR = 0.01;
+function computeCalibrationBounds(vectors, dimensions) {
+  let globalMin = Number.POSITIVE_INFINITY;
+  let globalMax = Number.NEGATIVE_INFINITY;
+  let count = 0;
+  for (const vec of vectors) {
+    for (let d = 0; d < dimensions; d++) {
+      const val = vec[d];
+      if (val < globalMin) globalMin = val;
+      if (val > globalMax) globalMax = val;
+    }
+    count++;
+  }
+  if (count === 0) return null;
+  const range = globalMax - globalMin;
+  const pad = range * PADDING_FACTOR;
+  globalMin -= pad;
+  globalMax += pad;
+  if (globalMin === globalMax) {
+    globalMin -= 1e-3;
+    globalMax += 1e-3;
+  }
+  return { alpha: (globalMax - globalMin) / 255, offset: globalMin };
+}
+
+// src/vector/scalar-quantization.ts
+var INITIAL_CAPACITY = 16;
+var PAGE_BYTES = 65536;
+function createScalarQuantizer(dimensions, ordinalSource) {
+  const docToOrd = /* @__PURE__ */ new Map();
+  let simd = createArenaSimd();
+  const arenaByteOffset = simd ? Math.max(16, Math.ceil(dimensions / 16) * 16) : 0;
+  let capacity = 0;
+  let quantizedArena = simd ? new Uint8Array(simd.memory.buffer, arenaByteOffset) : new Uint8Array(0);
+  let scratch = simd ? new Uint8Array(simd.memory.buffer, 0, arenaByteOffset) : new Uint8Array(0);
+  let sums = new Float64Array(0);
+  let sumSqs = new Float64Array(0);
+  let mags = new Float64Array(0);
+  let present = new Uint8Array(0);
+  let selfNextOrd = 0;
+  let liveCount = 0;
+  let constants = deriveSq8Constants(0, 0, dimensions);
+  let calibrated = false;
+  function ensureCapacity(needed) {
+    if (needed <= capacity) return;
+    let newCap = capacity === 0 ? INITIAL_CAPACITY : capacity;
+    while (newCap < needed) newCap *= 2;
+    const nextSums = new Float64Array(newCap);
+    nextSums.set(sums);
+    sums = nextSums;
+    const nextSumSqs = new Float64Array(newCap);
+    nextSumSqs.set(sumSqs);
+    sumSqs = nextSumSqs;
+    const nextMags = new Float64Array(newCap);
+    nextMags.set(mags);
+    mags = nextMags;
+    const nextPresent = new Uint8Array(newCap);
+    nextPresent.set(present);
+    present = nextPresent;
+    if (simd) {
+      const requiredBytes = arenaByteOffset + newCap * dimensions;
+      const have = simd.memory.buffer.byteLength;
+      if (requiredBytes > have) {
+        try {
+          simd.memory.grow(Math.ceil((requiredBytes - have) / PAGE_BYTES));
+        } catch {
+          const migrated = new Uint8Array(newCap * dimensions);
+          migrated.set(quantizedArena.subarray(0, capacity * dimensions));
+          quantizedArena = migrated;
+          simd = null;
+          capacity = newCap;
+          return;
+        }
+      }
+      quantizedArena = new Uint8Array(simd.memory.buffer, arenaByteOffset);
+      scratch = new Uint8Array(simd.memory.buffer, 0, arenaByteOffset);
+    } else {
+      const nextArena = new Uint8Array(newCap * dimensions);
+      nextArena.set(quantizedArena);
+      quantizedArena = nextArena;
+    }
+    capacity = newCap;
+  }
+  function resolveOrdinal(docId) {
+    const fromSource = ordinalSource?.getOrdinal(docId);
+    if (fromSource !== void 0) return fromSource;
+    const known = docToOrd.get(docId);
+    if (known !== void 0) return known;
+    return selfNextOrd++;
+  }
+  function quantizeInto(ord, vector) {
+    const base = ord * dimensions;
+    const { sum, sumSq } = quantizeVectorInto(
+      quantizedArena.subarray(base, base + dimensions),
+      vector,
+      dimensions,
+      constants
+    );
+    sums[ord] = sum;
+    sumSqs[ord] = sumSq;
+    mags[ord] = quantizedMagnitude(constants, sumSq, sum);
+  }
+  function storeEntry(docId, ord) {
+    if (present[ord] === 0) {
+      present[ord] = 1;
+      liveCount++;
+    }
+    docToOrd.set(docId, ord);
+  }
+  function calibrateFromVectors(vectors) {
+    const bounds = computeCalibrationBounds(vectors, dimensions);
+    if (bounds === null) return;
+    constants = deriveSq8Constants(bounds.alpha, bounds.offset, dimensions);
+    calibrated = true;
+  }
+  function isOutsideBounds(vector) {
+    const currentMin = constants.offset;
+    const currentMax = constants.offset + constants.alpha * 255;
+    for (let d = 0; d < dimensions; d++) {
+      if (vector[d] < currentMin || vector[d] > currentMax) {
+        return true;
+      }
+    }
+    return false;
+  }
+  function distanceScalar(queryQuantized, querySum, queryMagnitude, ord, metric) {
+    if (ord < 0 || ord >= capacity || present[ord] === 0) return Number.POSITIVE_INFINITY;
+    return scalarQuantizedDistance(
+      quantizedArena,
+      sums,
+      mags,
+      dimensions,
+      constants,
+      queryQuantized,
+      querySum,
+      queryMagnitude,
+      ord,
+      metric
+    );
+  }
+  return {
+    get dimensions() {
+      return dimensions;
+    },
+    get size() {
+      return liveCount;
+    },
+    get calibration() {
+      return calibrated ? { alpha: constants.alpha, offset: constants.offset } : null;
+    },
+    isCalibrated() {
+      return calibrated;
+    },
+    calibrate(vectors) {
+      calibrateFromVectors(vectors);
+    },
+    needsRecalibration(vector) {
+      if (!calibrated) return false;
+      return isOutsideBounds(vector);
+    },
+    recalibrateAll(vectors) {
+      const collected = [];
+      const rawVectors = [];
+      for (const pair of vectors) {
+        collected.push(pair);
+        rawVectors.push(pair[1]);
+      }
+      calibrateFromVectors(rawVectors);
+      docToOrd.clear();
+      present.fill(0);
+      liveCount = 0;
+      for (const [docId, vec] of collected) {
+        const ord = resolveOrdinal(docId);
+        ensureCapacity(ord + 1);
+        quantizeInto(ord, vec);
+        storeEntry(docId, ord);
+      }
+    },
+    quantize(docId, vector) {
+      if (!calibrated) {
+        calibrateFromVectors([vector]);
+      }
+      const ord = resolveOrdinal(docId);
+      ensureCapacity(ord + 1);
+      quantizeInto(ord, vector);
+      storeEntry(docId, ord);
+    },
+    remove(docId) {
+      const ord = docToOrd.get(docId);
+      if (ord === void 0) return;
+      docToOrd.delete(docId);
+      if (present[ord] === 1) {
+        present[ord] = 0;
+        liveCount--;
+      }
+    },
+    getQuantized(docId) {
+      const ord = docToOrd.get(docId);
+      if (ord === void 0 || present[ord] === 0) return void 0;
+      const base = ord * dimensions;
+      return quantizedArena.subarray(base, base + dimensions);
+    },
+    prepareQuery(query) {
+      if (!calibrated) return null;
+      const quantized = new Uint8Array(dimensions);
+      const { sum, sumSq } = quantizeVectorInto(quantized, query, dimensions, constants);
+      const mag = quantizedMagnitude(constants, sumSq, sum);
+      return { quantized, sum, sumSq, magnitude: mag };
+    },
+    distanceFromPrepared(prepared, docId, metric) {
+      const ord = docToOrd.get(docId);
+      if (ord === void 0) return Number.POSITIVE_INFINITY;
+      return distanceScalar(prepared.quantized, prepared.sum, prepared.magnitude, ord, metric);
+    },
+    distanceFromPreparedByOrdinal(prepared, ordinal, metric) {
+      return distanceScalar(prepared.quantized, prepared.sum, prepared.magnitude, ordinal, metric);
+    },
+    prepareQueryArena(query) {
+      if (!calibrated || !simd) return null;
+      const { sum, sumSq } = quantizeVectorInto(scratch, query, dimensions, constants);
+      const magnitude = quantizedMagnitude(constants, sumSq, sum);
+      return { sum, sumSq, magnitude };
+    },
+    distanceFromArena(prepared, ordinal, metric) {
+      if (!simd || ordinal < 0 || ordinal >= capacity || present[ordinal] === 0) {
+        return Number.POSITIVE_INFINITY;
+      }
+      const arenaOffset = arenaByteOffset + ordinal * dimensions;
+      return arenaQuantizedDistance(
+        simd,
+        0,
+        arenaOffset,
+        dimensions,
+        constants,
+        prepared.sum,
+        prepared.magnitude,
+        sums[ordinal],
+        mags[ordinal],
+        metric
+      );
+    },
+    hasOrdinal(ordinal) {
+      return ordinal >= 0 && ordinal < capacity && present[ordinal] === 1;
+    },
+    restoreCalibration(restoredAlpha, restoredOffset) {
+      constants = deriveSq8Constants(restoredAlpha, restoredOffset, dimensions);
+      calibrated = true;
+    },
+    restoreEntry(docId, quantized, sum, sumSq) {
+      const ord = resolveOrdinal(docId);
+      ensureCapacity(ord + 1);
+      const base = ord * dimensions;
+      quantizedArena.set(quantized.subarray(0, dimensions), base);
+      sums[ord] = sum;
+      sumSqs[ord] = sumSq;
+      mags[ord] = quantizedMagnitude(constants, sumSq, sum);
+      storeEntry(docId, ord);
+    },
+    copyStateInto(codes, targetSums, targetSumSqs, targetMagnitudes, targetPresent) {
+      const copySlots = Math.min(capacity, Math.floor(codes.length / dimensions));
+      if (copySlots <= 0) return;
+      codes.set(quantizedArena.subarray(0, copySlots * dimensions));
+      targetSums.set(sums.subarray(0, copySlots));
+      targetSumSqs.set(sumSqs.subarray(0, copySlots));
+      targetMagnitudes.set(mags.subarray(0, copySlots));
+      targetPresent.set(present.subarray(0, copySlots));
+    },
+    serialize() {
+      const serializedVectors = {};
+      const serializedSums = {};
+      const serializedSumSqs = {};
+      for (const [docId, ord] of docToOrd) {
+        if (present[ord] === 0) continue;
+        const base = ord * dimensions;
+        serializedVectors[docId] = Array.from(quantizedArena.subarray(base, base + dimensions));
+        serializedSums[docId] = sums[ord];
+        serializedSumSqs[docId] = sumSqs[ord];
+      }
+      return {
+        alpha: constants.alpha,
+        offset: constants.offset,
+        quantizedVectors: serializedVectors,
+        vectorSums: serializedSums,
+        vectorSumSqs: serializedSumSqs
+      };
+    },
+    clear() {
+      docToOrd.clear();
+      if (!simd) {
+        quantizedArena = new Uint8Array(0);
+      }
+      sums = new Float64Array(0);
+      sumSqs = new Float64Array(0);
+      mags = new Float64Array(0);
+      present = new Uint8Array(0);
+      capacity = 0;
+      selfNextOrd = 0;
+      liveCount = 0;
+      constants = deriveSq8Constants(0, 0, dimensions);
+      calibrated = false;
+    }
+  };
+}
+
+export { arenaQuantizedDistance, createScalarQuantizer, deriveSq8Constants, quantizeVectorInto, quantizedMagnitude, scalarQuantizedDistance };