@delali/narsil 0.2.4-next.50 → 0.2.4-next.52

package/dist/vector/search-worker.d.ts

@@ -1,4 +1,20 @@
-import { V as VectorMetric } from '../brute-force-DbHYCTc3.js';
+import { d as VectorQuantizationMode } from '../schema-OSWaaZzx.js';
+import { A as ArenaSimd } from '../simd-DW5AtJ6B.js';
+import '../adapters-79OIa4xm.js';
+
+/**
+ * Similarity metric a vector search ranks by.
+ *
+ * @internal
+ */
+type VectorMetric = 'cosine' | 'dotProduct' | 'euclidean';
+
+/**
+ * Each dimension of a document code holds this many bits.
+ *
+ * @internal
+ */
+type OsqBits = 1 | 2 | 4 | 8;
 
 /**
  * A filter over store ordinals, one bit per ordinal, with the number of set
@@ -18,74 +34,199 @@ interface OrdinalFilter {
 }
 
 /**
- * Every stored vector in the form the engine hands to another thread.
+ * This buffer grows in place, so a typed array built over it without an
+ * explicit length reports the new length after every growth, on every thread
+ * that holds it.
  *
  * @internal
  */
-interface VectorStoreSnapshot {
-    /** Each vector carries this many components. */
+type GrowableBuffer = SharedArrayBuffer | ArrayBuffer;
+
+/**
+ * This geometry gives the byte offset of every region inside one block. A
+ * block begins with one query scratch for each thread slot, and the slots
+ * follow them, each holding one record of a fixed stride.
+ *
+ * @internal
+ */
+interface BlockGeometry {
+    /** Each thread's query scratch spans this many bytes. */
+    scratchStride: number;
+    /** The slots start at this byte offset. */
+    slotsOffset: number;
+    /** Each slot spans this many bytes. */
+    slotStride: number;
+    /** The block holds this many slots at most. */
+    capacity: number;
+}
+/**
+ * This is the geometry of a block holding float32 vectors, one per slot.
+ *
+ * @internal
+ */
+interface VectorBlockLayout extends BlockGeometry {
+    /** Every vector of the field has this many components. */
     dimension: number;
-    /** The store spans this many ordinals, deleted ones included. */
-    slots: number;
-    /** This holds every vector end to end, `slots * dimension` components long. */
-    vectors: Float32Array;
-    /** Each ordinal's vector length, so a reader need not recompute it. */
-    magnitudes: Float64Array;
-    /** The document at each ordinal, `null` where the ordinal holds none. */
-    docIds: Array<string | null>;
+}
+/**
+ * One block keeps its bytes in a shared WebAssembly memory that every thread
+ * binds the distance kernels to, in a private WebAssembly memory where the
+ * runtime shares no memory, or in plain bytes where the runtime offers no
+ * WebAssembly.
+ *
+ * @internal
+ */
+type VectorBlockStorage = {
+    kind: 'shared-memory';
+    memory: WebAssembly.Memory;
+} | {
+    kind: 'private-memory';
+    simd: ArenaSimd;
+} | {
+    kind: 'bytes';
+    buffer: GrowableBuffer;
+};
+/**
+ * This handle names one block of a field, which every thread holding the
+ * field opens on its own side.
+ *
+ * @internal
+ */
+interface VectorBlockHandle<Layout extends BlockGeometry = BlockGeometry> {
+    /** The block follows this layout. */
+    layout: Layout;
+    /** The block keeps its bytes here. */
+    storage: VectorBlockStorage;
+}
+
+/**
+ * This is the geometry of a block holding one code record per slot: the packed code
+ * with its lower, upper, correction, and sum inline, so a comparison reads
+ * one contiguous record. Each thread's scratch holds one staged query.
+ *
+ * @internal
+ */
+interface VectorCodeBlockLayout extends BlockGeometry {
+    /** Every vector of the field has this many components. */
+    dimension: number;
+    /** Each level of a document code holds this many bits. */
+    bits: OsqBits;
+    /** Each level of a query code holds this many bits. */
+    queryBits: OsqBits;
+    /** A packed document code spans this many bytes. */
+    codeBytes: number;
+    /** A staged query spans this many bytes. */
+    queryBytes: number;
 }
 
 /**
- * The two constants a scalar quantizer turns a vector component into a byte
- * with, and turns that byte back into a distance with.
+ * A thread opens these handles so that it can read one field's vectors,
+ * codes, and document ids in place. They name the blocks holding the vectors,
+ * the blocks holding the code records, the side tables that hold one entry
+ * per ordinal, and the checkpoint files that hold the vectors of the
+ * ordinals a field on disk has released from memory.
  *
- * A worker holding a copy of a vector field receives these rather than deriving
- * its own, because a copy taken after a delete would otherwise measure a
- * narrower range of values and answer the same query differently from the
- * thread that built the index.
+ * The main thread appends ordinals, and because the tables grow in place, a
+ * thread that opened the handles once goes on reading the current state.
+ * Adding a block, releasing one, or adding a file is a structural change, so
+ * the main thread bumps the layout revision and sends the handles again.
  *
  * @internal
  */
-interface ScalarQuantizerCalibration {
-    /** Each step of the byte scale spans this much of the component range. */
-    alpha: number;
-    /** The byte scale starts at this component value. */
-    offset: number;
+interface SharedVectorStoreHandles {
+    /** Every vector of the field has this many components. */
+    dimension: number;
+    /** Each level of a document code holds this many bits, and null where the field keeps no codes. */
+    codeBits: OsqBits | null;
+    /** Every float block of the field follows this layout. */
+    layout: VectorBlockLayout;
+    /** Every code block of the field follows this layout, and null where the field keeps no codes. */
+    codeLayout: VectorCodeBlockLayout | null;
+    /** The main thread raises this on every structural change, so a thread compares it against what it opened. */
+    layoutRevision: number;
+    /** Every thread reads these counters through atomics. */
+    header: Int32Array;
+    /** These blocks hold the vectors, in ordinal order, with null where the store released a block to disk or never filled it. */
+    blocks: Array<VectorBlockHandle<VectorBlockLayout> | null>;
+    /** These blocks hold the code records, in ordinal order. */
+    codeBlocks: VectorBlockHandle<VectorCodeBlockLayout>[];
+    /** These are the paths of the checkpoint files holding vectors, with an empty string where no ordinal reads a file any more. */
+    vectorFiles: string[];
+    /** This holds each ordinal's vector magnitude. */
+    magnitudes: GrowableBuffer;
+    /** This holds one byte per ordinal, which reads 1 where the ordinal holds a live vector. */
+    present: GrowableBuffer;
+    /** This holds each ordinal's partition, or a negative marker where the store knows none. */
+    partitions: GrowableBuffer;
+    /** This holds every document id, UTF-8 encoded end to end in ordinal order. */
+    docIdBytes: GrowableBuffer;
+    /** This holds the byte each ordinal's id starts at, with one closing offset after the last. */
+    docIdOffsets: GrowableBuffer;
+    /** This holds one byte per ordinal, which reads 1 where the ordinal holds a code record. */
+    codePresent: GrowableBuffer;
+    /** This holds the index into `vectorFiles` of the file holding each ordinal's vector, or -1 where a block holds it. */
+    diskFile: GrowableBuffer;
+    /** This holds the byte offset of each ordinal's vector inside its file. */
+    diskOffset: GrowableBuffer;
+    /** This holds the centroid the quantizer takes every code against. */
+    centroid: Float32Array;
 }
 
 /**
- * Every node's neighbours in the flat form the engine hands to another thread.
+ * The engine clones a field's vectors to another thread in this form, which
+ * it uses where the runtime shares no memory between threads.
  *
  * @internal
  */
-interface AdjacencySnapshot {
-    /** Each node keeps this many neighbours per layer above the base. */
-    m: number;
-    /** Each node keeps this many neighbours on the base layer. */
-    mMax0: number;
-    /** The arrays span this many ordinals. */
-    capacity: number;
-    /** No node sits at an ordinal at or above this. */
+interface VectorStoreSnapshot {
+    /** Every vector of the field has this many components. */
+    dimension: number;
+    /** The store spans this many ordinals, deleted ones included. */
     slots: number;
-    /** The upper-layer arena holds this many entries in use. */
-    upperUsed: number;
-    /** Each ordinal's top layer, or -1 where no node sits there. */
-    nodeLevels: Int8Array;
-    /** The base layer's neighbours, a count followed by neighbours per ordinal. */
-    level0: Int32Array;
-    /** Where each ordinal's upper layers start, or -1 where it has none. */
-    upperBase: Int32Array;
-    /** Every upper layer's neighbours, a count followed by neighbours per layer. */
-    upper: Int32Array;
+    /** This holds every vector end to end, `slots * dimension` components long. */
+    vectors: Float32Array;
+    /** This holds each ordinal's vector length, so a reader takes it from here. */
+    magnitudes: Float64Array;
+    /** This names the document at each ordinal, and it reads `null` where the ordinal holds none. */
+    docIds: Array<string | null>;
+}
+
+/**
+ * A thread opens these handles so that it can search or extend one graph in
+ * place. They name the counters every thread reads through atomics, the
+ * adjacency arrays that grow without moving, the lock word of every node, and
+ * the record of the locks each thread holds, which the main thread reads to
+ * release what a dead thread left behind.
+ *
+ * @internal
+ */
+interface SharedGraphHandles {
+    /** This holds the entry point, the layer count, the node counts, the arena cursor, and the graph's shape. */
+    header: Int32Array;
+    /** This holds each ordinal's top layer plus one, and it reads zero where the graph holds no node there. */
+    nodeLevels: GrowableBuffer;
+    /** This holds the base layer's neighbours, a count followed by that many neighbours for each ordinal. */
+    level0: GrowableBuffer;
+    /** This holds where each ordinal's upper layers start plus one, and it reads zero where the ordinal reaches none. */
+    upperBase: GrowableBuffer;
+    /** This holds every upper layer's neighbours, a count followed by that many neighbours for each layer. */
+    upper: GrowableBuffer;
+    /** This holds one lock word per ordinal. */
+    locks: GrowableBuffer;
+    /** This holds one byte per ordinal, which reads 1 once a caller removes the document. */
+    tombstones: GrowableBuffer;
+    /** This records the node lock and the graph lock each thread slot holds right now. */
+    heldLocks: Int32Array;
 }
 
 /**
- * A built graph in the form the engine hands to another thread.
+ * This holds a built graph copied out flat, which the engine sends to another
+ * thread where the runtime shares no memory.
  *
  * @internal
  */
 interface HNSWSnapshot {
-    /** Each vector carries this many components. */
+    /** Every vector of the field has this many components. */
     dimension: number;
     /** Each node keeps this many neighbours per layer. */
     m: number;
@@ -93,16 +234,24 @@ interface HNSWSnapshot {
     efConstruction: number;
     /** The graph ranks by this metric. */
     metric: VectorMetric;
-    /** This holds every node's neighbours, laid out flat. */
-    adjacency: AdjacencySnapshot;
-    /** A byte per ordinal, set to 1 where the document has been deleted. */
+    /** Every node of the graph lies below this ordinal. */
+    slots: number;
+    /** The upper-layer arena holds this many entries in use. */
+    upperUsed: number;
+    /** This holds each ordinal's top layer plus one, and it reads zero where the graph holds no node there. */
+    nodeLevels: Uint8Array;
+    /** This holds the base layer's neighbours, a count followed by that many neighbours for each ordinal. */
+    level0: Int32Array;
+    /** This holds where each ordinal's upper layers start plus one, and it reads zero where the ordinal reaches none. */
+    upperBase: Int32Array;
+    /** This holds every upper layer's neighbours, a count followed by that many neighbours for each layer. */
+    upper: Int32Array;
+    /** This holds one byte per ordinal, which reads 1 once a caller removes the document. */
     tombstones: Uint8Array;
-    /** This many ordinals are tombstoned. */
+    /** The graph holds this many tombstoned ordinals. */
     tombstoneCount: number;
     /** The graph holds this many nodes, tombstoned ones included. */
     nodeCount: number;
-    /** The graph's own arrays span this many ordinals. */
-    capacity: number;
     /** Every search starts at this ordinal, and it is -1 while the graph is empty. */
     entryPointOrd: number;
     /** The graph reaches this many layers. */
@@ -110,93 +259,76 @@ interface HNSWSnapshot {
 }
 
 /**
- * Where each region of a frozen generation sits inside its shared memory.
+ * A thread opens these handles so that it can search one vector field in
+ * place and place vectors in its graph, and they name the store's shared
+ * structures alongside the graph's.
  *
- * The memory opens with one float32 query scratch slot and one code query
- * scratch slot per worker, so concurrent searches never write over each
- * other, followed by the float32 vector arena and the quantised code arena.
+ * The main thread sends them once, and again only when a block joins the
+ * store or a fresh graph replaces the old one, because every other change
+ * grows the shared buffers in place.
  *
  * @internal
  */
-interface SharedGenerationLayout {
-    /** Each vector carries this many components. */
+interface SharedVectorFieldHandles {
+    /** Every vector of the field has this many components. */
     dimension: number;
-    /** The arenas span this many ordinals. */
-    slots: number;
-    /** The memory holds this many per-worker scratch slot pairs. */
-    workerSlots: number;
-    /** Each worker's float32 query scratch spans this many bytes. */
-    float32ScratchStride: number;
-    /** Each worker's code query scratch spans this many bytes. */
-    codeScratchStride: number;
-    /** The code query scratch slots start at this byte offset. */
-    codeScratchOffset: number;
-    /** The float32 vector arena starts at this byte offset. */
-    vectorsOffset: number;
-    /** The quantised code arena starts at this byte offset. */
-    codesOffset: number;
-    /** The memory spans this many bytes, a whole number of pages. */
-    totalBytes: number;
+    /** The threads estimate distances from code records under any mode but `none`. */
+    quantization: VectorQuantizationMode;
+    /** The quantizer takes the codes under this metric, which the graph also ranks by. */
+    metric: VectorMetric;
+    /** These are the store's shared structures. */
+    store: SharedVectorStoreHandles;
+    /** These are the graph's shared structures, and they read null while the field answers by exact comparison. */
+    graph: SharedGraphHandles | null;
+    /** A thread answers by exact comparison where a filter admits a smaller share of the live vectors than this. */
+    filterThreshold: number;
+    /** This reads true where searches on the thread answer from the graph, and false while the threads build it. */
+    searchable: boolean;
 }
 /**
- * One frozen copy of a vector field, shared across every search thread.
- *
- * The typed arrays are views over shared buffers, so posting this to a worker
- * hands over the same bytes rather than a copy, and nothing in it is written
- * after the freeze apart from the per-worker scratch slots inside `memory`.
+ * A thread reports this after placing a batch of vectors in a graph.
  *
  * @internal
  */
-interface SharedGenerationSnapshot {
-    /** Each vector carries this many components. */
-    dimension: number;
-    /** The workers compute quantised distances when this reads `sq8`. */
-    quantization: 'sq8' | 'none';
-    /** The constants every code was derived with, present when quantised. */
-    calibration: ScalarQuantizerCalibration | null;
-    /** The shared memory holding the scratch slots and both arenas. */
-    memory: WebAssembly.Memory;
-    /** Where each region sits inside the shared memory. */
-    layout: SharedGenerationLayout;
-    /** Each ordinal's vector magnitude. */
-    magnitudes: Float64Array;
-    /** Each ordinal's code sum, present when quantised. */
-    codeSums: Float64Array;
-    /** Each ordinal's reconstructed code magnitude, present when quantised. */
-    codeMagnitudes: Float64Array;
-    /** A byte per ordinal, 1 where codes exist, present when quantised. */
-    codePresent: Uint8Array;
-    /** The built graph, its arrays backed by shared buffers. */
-    graph: HNSWSnapshot;
-    /**
-     * Each ordinal's document id rank in code point order, or
-     * the absent-document rank where the ordinal holds no document.
-     */
-    rankByOrdinal: Uint32Array;
+interface GraphInsertOutcome {
+    /** The thread placed this many vectors the graph did not hold before. */
+    placed: number;
 }
 
 /**
- * One vector field's searchable state in the form the engine clones to a
- * worker, used where the runtime cannot share memory.
+ * The code records of one field copied out flat, which the engine sends to
+ * a worker that shares no memory with it.
+ *
+ * @internal
+ */
+interface WorkerCopyCodes {
+    /** The centroid the quantizer takes every record against. */
+    centroid: Float32Array;
+    /** This holds one record per ordinal, end to end. */
+    records: Uint8Array;
+    /** This holds one byte per ordinal, which reads 1 where the ordinal holds a record. */
+    present: Uint8Array;
+}
+/**
+ * The engine clones one vector field's searchable state to a worker in this
+ * form, which it uses where the runtime shares no memory.
  *
  * @internal
  */
 interface WorkerCopySnapshot {
-    /** Each vector carries this many components. */
+    /** Every vector of the field has this many components. */
     dimension: number;
-    /** The worker rebuilds a quantizer when this reads `sq8`. */
-    quantization: 'sq8' | 'none';
-    /**
-     * The constants the calling thread quantizes with, carried so that the worker
-     * derives the same codes rather than recalibrating over a set that a delete
-     * has already narrowed.
-     */
-    calibration: ScalarQuantizerCalibration | null;
+    /** The worker estimates from code records under any mode but `none`. */
+    quantization: VectorQuantizationMode;
+    /** The quantizer takes the codes under this metric. */
+    metric: VectorMetric;
+    /** The records the calling thread wrote, so the worker rewrites none, or null where the field holds no codes. */
+    codes: WorkerCopyCodes | null;
     /** This holds every vector and the document id at each ordinal. */
     store: VectorStoreSnapshot;
     /** This holds the built graph. */
     graph: HNSWSnapshot;
-    /** These documents have been deleted and must not be returned. */
+    /** The worker leaves these deleted documents out of every result. */
     tombstones: string[];
 }
 
@@ -220,21 +352,21 @@ interface VectorLoadRequest {
     snapshot: WorkerCopySnapshot;
 }
 /**
- * Asks the search worker to open a frozen shared copy under a handle.
+ * Asks the search worker to open a field in place under a handle.
  *
  * @internal
  */
-interface SharedCopyLoadRequest {
-    /** This marks the message as a shared copy load. */
+interface SharedFieldLoadRequest {
+    /** This marks the message as a shared field load. */
     type: 'loadShared';
     /** The reply carries this back so the engine can match it to its caller. */
     requestId: string;
-    /** Later messages name the copy by this handle. */
+    /** Later messages name the field by this handle. */
     handle: string;
     /** The worker writes query scratch into this reserved slot alone. */
     scratchSlot: number;
-    /** The frozen copy, its buffers shared rather than cloned. */
-    snapshot: SharedGenerationSnapshot;
+    /** These are the field's shared structures. */
+    handles: SharedVectorFieldHandles;
 }
 /**
  * Asks the search worker to release a copy it holds.
@@ -249,6 +381,21 @@ interface VectorDropRequest {
     /** The worker releases the copy held under this handle. */
     handle: string;
 }
+/**
+ * Asks the search worker to place vectors in the graph of a field it holds.
+ *
+ * @internal
+ */
+interface VectorInsertRequest {
+    /** This marks the message as a graph insertion. */
+    type: 'insertOrdinals';
+    /** The reply carries this back so the engine can match it to its caller. */
+    requestId: string;
+    /** The worker places the vectors in the field held under this handle. */
+    handle: string;
+    /** The ordinals to place, whose vectors the shared store already holds. */
+    ordinals: Int32Array;
+}
 /**
  * Asks the search worker to answer one nearest-neighbour query from a cloned
  * copy.
@@ -274,19 +421,21 @@ interface VectorSearchRequest {
     filter?: OrdinalFilter;
     /** The worker explores this many candidates, or its own default when absent. */
     efSearch?: number;
+    /** The worker re-scores this many times the requested count against full precision, or its own default when absent. */
+    oversample?: number;
 }
 /**
  * Asks the search worker to answer one nearest-neighbour query from a shared
- * copy, returning ordinals for the engine to map back to document ids.
+ * field, returning ordinals for the engine to map back to document ids.
  *
  * @internal
  */
 interface VectorOrdinalSearchRequest {
-    /** This marks the message as a search over a shared copy. */
+    /** This marks the message as a search over a shared field. */
     type: 'searchOrdinals';
     /** The reply carries this back so the engine can match it to its caller. */
     requestId: string;
-    /** The worker searches the copy held under this handle. */
+    /** The worker searches the field held under this handle. */
     handle: string;
     /** The worker ranks against this query vector. */
     query: Float32Array;
@@ -300,13 +449,15 @@ interface VectorOrdinalSearchRequest {
     filter?: OrdinalFilter;
     /** The worker explores this many candidates, or its own default when absent. */
     efSearch?: number;
+    /** The worker re-scores this many times the requested count against full precision, or its own default when absent. */
+    oversample?: number;
 }
 /**
  * Every message the engine posts to the search worker.
  *
  * @internal
  */
-type VectorWorkerRequest = VectorLoadRequest | SharedCopyLoadRequest | VectorDropRequest | VectorSearchRequest | VectorOrdinalSearchRequest;
+type VectorWorkerRequest = VectorLoadRequest | SharedFieldLoadRequest | VectorDropRequest | VectorInsertRequest | VectorSearchRequest | VectorOrdinalSearchRequest;
 /**
  * What the search worker returns once a copy is loaded or released.
  *
@@ -320,6 +471,19 @@ interface VectorAckResponse {
     /** This names the copy the acknowledgement belongs to. */
     handle: string;
 }
+/**
+ * The search worker returns this once it has placed a batch of vectors.
+ *
+ * @internal
+ */
+interface VectorInsertResponse {
+    /** This marks the message as a completed insertion. */
+    type: 'inserted';
+    /** This matches the request the engine sent. */
+    requestId: string;
+    /** This is what the worker observed while placing the batch. */
+    outcome: GraphInsertOutcome;
+}
 /**
  * What the search worker returns for a completed search over a cloned copy.
  *
@@ -336,7 +500,7 @@ interface VectorSearchResponse {
     scores: Float64Array;
 }
 /**
- * What the search worker returns for a completed search over a shared copy.
+ * The search worker returns this for a completed search over a shared field.
  *
  * @internal
  */
@@ -368,6 +532,6 @@ interface VectorWorkerError {
  *
  * @internal
  */
-type VectorWorkerMessage = VectorAckResponse | VectorSearchResponse | VectorOrdinalSearchResponse | VectorWorkerError;
+type VectorWorkerMessage = VectorAckResponse | VectorInsertResponse | VectorSearchResponse | VectorOrdinalSearchResponse | VectorWorkerError;
 
-export { type AdjacencySnapshot, type HNSWSnapshot, type OrdinalFilter, type ScalarQuantizerCalibration, type SharedCopyLoadRequest, type SharedGenerationLayout, type SharedGenerationSnapshot, type VectorAckResponse, type VectorDropRequest, type VectorLoadRequest, VectorMetric, type VectorOrdinalSearchRequest, type VectorOrdinalSearchResponse, type VectorSearchRequest, type VectorSearchResponse, type VectorStoreSnapshot, type VectorWorkerError, type VectorWorkerMessage, type VectorWorkerRequest, type WorkerCopySnapshot };
+export { ArenaSimd, type BlockGeometry, type GraphInsertOutcome, type GrowableBuffer, type HNSWSnapshot, type OrdinalFilter, type OsqBits, type SharedFieldLoadRequest, type SharedGraphHandles, type SharedVectorFieldHandles, type SharedVectorStoreHandles, type VectorAckResponse, type VectorBlockHandle, type VectorBlockLayout, type VectorBlockStorage, type VectorCodeBlockLayout, type VectorDropRequest, type VectorInsertRequest, type VectorInsertResponse, type VectorLoadRequest, type VectorMetric, type VectorOrdinalSearchRequest, type VectorOrdinalSearchResponse, type VectorSearchRequest, type VectorSearchResponse, type VectorStoreSnapshot, type VectorWorkerError, type VectorWorkerMessage, type VectorWorkerRequest, type WorkerCopyCodes, type WorkerCopySnapshot };