@delali/narsil 0.2.4-next.51 → 0.2.4-next.52

package/dist/index.d.ts

@@ -1,13 +1,14 @@
-import { c as StopWordOverride, C as CustomTokenizer } from './schema-BqRx9t4E.js';
-export { A as AnyDocument, B as BM25Params, E as EmbeddingFieldConfig, F as FieldType, I as IndexConfig, b as InsertOptions, P as PartitionConfig, S as SchemaDefinition, a as ScoringMode, V as VectorIndexConfig, d as VectorQuantizationMode, W as WriteOptions } from './schema-BqRx9t4E.js';
+import { b as StopWordOverride, C as CustomTokenizer } from './schema-OSWaaZzx.js';
+export { A as AnyDocument, B as BM25Params, E as EmbeddingFieldConfig, F as FieldType, I as IndexConfig, a as InsertOptions, P as PartitionConfig, S as SchemaDefinition, c as ScoringMode, V as VectorIndexConfig, d as VectorQuantizationMode, e as VectorStorageMode, W as WriteOptions } from './schema-OSWaaZzx.js';
 import { L as LanguageModule } from './language-DB8KG1z5.js';
 export { T as TokenizerConfig } from './language-DB8KG1z5.js';
-export { A as ArrayFilter, B as BatchResult, C as ClientErrorCode, a as ClientErrorCodes, k as ComparisonFilter, D as DocumentProjection, E as ErrorCode, b as ErrorCodes, F as FacetConfig, l as FacetResult, m as FieldFilter, n as FilterExpression, G as GeoFilter, o as GeoPolygonFilter, p as GeoRadiusFilter, q as GroupConfig, r as GroupReducer, s as GroupResult, H as HighlightConfig, t as HighlightMatch, u as Hit, v as HybridConfig, I as IndexInfo, f as IndexStats, L as ListParams, g as ListResult, w as ListedDocument, M as MemoryStats, N as NarsilError, c as NarsilErrorCode, P as PartitionStatsResult, h as PreflightResult, x as PresenceFilter, y as ProcessMemoryReport, z as QueryCoverage, Q as QueryParams, e as QueryResult, J as ScoreComponents, K as SearchMode, S as ServerErrorCode, d as ServerErrorCodes, O as SortField, R as SortSpec, T as StringFilter, i as SuggestParams, j as SuggestResult, U as TermMatchPolicy, V as VectorMaintenanceResult, W as VectorQueryConfig, X as WorkerCopyReport } from './memory-Bzd5xcEW.js';
-import { N as NarsilConfig } from './config-OON7IJ_B.js';
-export { A as AnalysisConfig, D as DurabilityConfig, I as IndexContext, a as IndexLifecycleConfig, b as InsertContext, M as MainCopyQueries, c as NarsilPlugin, P as PartitionContext, R as RemoveContext, S as SearchContext, d as StaleAnalysis, U as UpdateContext, W as WorkerConfig, e as WorkerContext } from './config-OON7IJ_B.js';
-import { a as Narsil } from './engine-9PtFY50q.js';
-export { D as DocumentWriteOperations, I as IndexLifecycleOperations, N as NarsilEventMap } from './engine-9PtFY50q.js';
+export { A as ArrayFilter, B as BatchResult, C as ClientErrorCode, a as ClientErrorCodes, k as ComparisonFilter, D as DocumentProjection, E as ErrorCode, b as ErrorCodes, F as FacetConfig, l as FacetResult, m as FieldFilter, n as FilterExpression, G as GeoFilter, o as GeoPolygonFilter, p as GeoRadiusFilter, q as GroupConfig, r as GroupReducer, s as GroupResult, H as HighlightConfig, t as HighlightMatch, u as Hit, v as HybridConfig, I as IndexInfo, f as IndexStats, L as ListParams, g as ListResult, w as ListedDocument, M as MemoryStats, N as NarsilError, c as NarsilErrorCode, P as PartitionStatsResult, h as PreflightResult, x as PresenceFilter, y as ProcessMemoryReport, z as QueryCoverage, Q as QueryParams, e as QueryResult, J as ScoreComponents, K as SearchMode, S as ServerErrorCode, d as ServerErrorCodes, O as SortField, R as SortSpec, T as StringFilter, i as SuggestParams, j as SuggestResult, U as TermMatchPolicy, V as VectorMaintenanceResult, W as VectorQueryConfig, X as WorkerCopyReport } from './memory-OmgkX_Z2.js';
+import { N as NarsilConfig } from './config-BcYZkdMw.js';
+export { A as AnalysisConfig, D as DurabilityConfig, I as IndexContext, a as IndexLifecycleConfig, b as InsertContext, M as MainCopyQueries, c as NarsilPlugin, P as PartitionContext, R as RemoveContext, S as SearchContext, d as StaleAnalysis, U as UpdateContext, W as WorkerConfig, e as WorkerContext } from './config-BcYZkdMw.js';
+import { a as Narsil } from './engine-zhms1qId.js';
+export { D as DocumentWriteOperations, I as IndexLifecycleOperations, N as NarsilEventMap } from './engine-zhms1qId.js';
 export { E as EmbeddingAdapter, I as InvalidationAdapter, a as InvalidationEvent, b as PartitionStatistics, P as PersistenceAdapter } from './adapters-79OIa4xm.js';
+export { i as isSimdAvailable } from './simd-DW5AtJ6B.js';
 
 /**
  * Registers a tokeniser under a name that an index config can then reference.
@@ -165,19 +166,6 @@ declare function getLanguage(name: string): LanguageModule;
  */
 declare function createNarsil(config?: NarsilConfig): Promise<Narsil>;
 
-/**
- * Reports whether the runtime loaded the SIMD module that speeds up vector
- * distance work.
- *
- * Vector search works either way, so read this to explain a difference in
- * search latency between two hosts rather than to decide whether to search.
- *
- * @returns True once the SIMD module has loaded.
- *
- * @public
- */
-declare function isSimdAvailable(): boolean;
-
 /**
  * Engine version written into the engine-version bytes of every `.nrsl` file,
  * for diagnostics.
@@ -190,4 +178,4 @@ declare function isSimdAvailable(): boolean;
  */
 declare const VERSION = "0.1.0";
 
-export { CustomTokenizer, LanguageModule, Narsil, NarsilConfig, StopWordOverride, VERSION, clearNormalizationCache, configureNormalizationCache, createNarsil, getLanguage, getNormalizationCacheSize, getStopWords, getTokenizer, hasStopWords, hasTokenizer, isSimdAvailable, registerLanguage, registerStopWords, registerTokenizer, resetNormalizationCache };
+export { CustomTokenizer, LanguageModule, Narsil, NarsilConfig, StopWordOverride, VERSION, clearNormalizationCache, configureNormalizationCache, createNarsil, getLanguage, getNormalizationCacheSize, getStopWords, getTokenizer, hasStopWords, hasTokenizer, registerLanguage, registerStopWords, registerTokenizer, resetNormalizationCache };

package/dist/index.mjs

@@ -1,18 +1,19 @@
-export { VERSION, createNarsil } from './chunk-ND34I3SV.mjs';
-import './chunk-ZHUQKZBE.mjs';
+export { VERSION, createNarsil } from './chunk-ULJ36FK4.mjs';
 import './chunk-GCM3JFVA.mjs';
-import './chunk-PIWKZPP7.mjs';
-import './chunk-XE6XR2OE.mjs';
-export { getLanguage, getStopWords, getTokenizer, hasStopWords, hasTokenizer, registerLanguage, registerStopWords, registerTokenizer } from './chunk-BEIK3EOZ.mjs';
-export { clearNormalizationCache, configureNormalizationCache, getNormalizationCacheSize, resetNormalizationCache } from './chunk-CMVG2UEF.mjs';
+import './chunk-U36IDJN7.mjs';
+export { getLanguage, getStopWords, getTokenizer, hasStopWords, hasTokenizer, registerLanguage, registerStopWords, registerTokenizer } from './chunk-2FO7KZ6K.mjs';
 import './chunk-AV4IJKD2.mjs';
-import './chunk-CBLS46IE.mjs';
+import './chunk-YPYULM5E.mjs';
+import './chunk-3GWP2TO6.mjs';
+export { clearNormalizationCache, configureNormalizationCache, getNormalizationCacheSize, resetNormalizationCache } from './chunk-WM7IS45M.mjs';
+import './chunk-ZT5BRDFW.mjs';
 import './chunk-TGP5M5R7.mjs';
-import './chunk-PMBHH44R.mjs';
-export { isSimdAvailable } from './chunk-X6YZO4MQ.mjs';
-import './chunk-RP23PWMV.mjs';
-import './chunk-KMMJMDUA.mjs';
+export { isSimdAvailable } from './chunk-T62YOSGS.mjs';
+import './chunk-ZJ4CSAB2.mjs';
+import './chunk-WFKIGEBY.mjs';
+import './chunk-3L4U6EJQ.mjs';
+import './chunk-IFBKLBGB.mjs';
 import './chunk-U2BKYEC7.mjs';
 import './chunk-2GWVQWGA.mjs';
-import './chunk-WYJXSLER.mjs';
+import './chunk-POWOTUKJ.mjs';
 export { ClientErrorCodes, ErrorCodes, NarsilError, ServerErrorCodes } from './chunk-P4ELXVJQ.mjs';

package/dist/{memory-Bzd5xcEW.d.ts → memory-OmgkX_Z2.d.ts}

@@ -1,4 +1,4 @@
-import { A as AnyDocument, S as SchemaDefinition, a as ScoringMode } from './schema-BqRx9t4E.js';
+import { A as AnyDocument, S as SchemaDefinition, c as ScoringMode } from './schema-OSWaaZzx.js';
 
 /**
  * Every code a {@link NarsilError} carries, grouped by the part of the engine
@@ -845,6 +845,16 @@ interface VectorQueryConfig {
      * value when omitted.
      */
     efSearch?: number;
+    /**
+     * How many times `limit` a quantized field re-scores against its
+     * full-precision vectors before it returns the best `limit`. A value below
+     * 1 or one that is not a finite number fails with `CONFIG_INVALID`. The
+     * engine ignores this on a field whose quantization is `none`. When a query
+     * omits it, the engine takes 3 for `osq1` and `osq2`, 2 for `osq8`, and 2
+     * for `osq4` at 1,024 dimensions and above, but 5 for `osq4` below 1,024,
+     * where a four-bit code ranks the true neighbours less reliably.
+     */
+    oversample?: number;
 }
 /**
  * How a hybrid query merges its keyword ranking with its vector ranking.

package/dist/persistence/durability/checkpoint-worker.d.ts

@@ -1,4 +1,4 @@
-import { P as PartitionConfig, a as ScoringMode, V as VectorIndexConfig } from '../../schema-BqRx9t4E.js';
+import { P as PartitionConfig, c as ScoringMode, V as VectorIndexConfig } from '../../schema-OSWaaZzx.js';
 import '../../adapters-79OIa4xm.js';
 
 interface IndexMetadata {
@@ -43,6 +43,119 @@ interface IndexEmbeddingMetadata {
     adapter?: string;
     fields: Record<string, string | string[]>;
 }
+/**
+ * Persisted surface-form dictionary of one partition. Keys are the
+ * unstemmed display tokens; the value is the occurrence count when the
+ * surface equals its index token, or `[count, indexToken]` when stemming
+ * changed it.
+ */
+type SerializedSurfaceForms = Record<string, number | [number, string]>;
+
+interface SegmentPayload {
+    documentCount: number;
+    docIds: string[];
+    fieldNames: string[];
+    tokens: string[];
+    postingOffsets: Uint32Array;
+    postingDocIds: Uint32Array;
+    postingFrequencies: Uint16Array;
+    postingFieldIndices: Uint8Array;
+    positionOffsets: Uint32Array | null;
+    positionValues: Uint32Array | null;
+    fieldLengthNames: string[];
+    fieldLengthColumns: Uint32Array[];
+    totalFieldLengths: Record<string, number>;
+    docFrequencies: Record<string, number>;
+    surfaceForms: SerializedSurfaceForms | null;
+    numeric: Array<{
+        fieldPath: string;
+        docIds: Uint32Array;
+        values: Float64Array;
+    }>;
+    boolean: Array<{
+        fieldPath: string;
+        trueDocs: Uint32Array;
+        falseDocs: Uint32Array;
+    }>;
+    enums: Array<{
+        fieldPath: string;
+        values: string[];
+        offsets: Uint32Array;
+        docIds: Uint32Array;
+    }>;
+    geo: Array<{
+        fieldPath: string;
+        docIds: Uint32Array;
+        latitudes: Float64Array;
+        longitudes: Float64Array;
+    }>;
+}
+
+interface SegmentPostingColumns {
+    postingOffsets: Uint32Array;
+    postingDocIds: Uint32Array;
+    postingFrequencies: Uint16Array;
+    postingFieldIndices: Uint8Array;
+    positionOffsets: Uint32Array | null;
+    positionValues: Uint32Array | null;
+}
+interface SegmentColumns extends SegmentPostingColumns {
+    documentCount: number;
+    fieldNames: readonly string[];
+    fieldLengthNames: readonly string[];
+    fieldLengthColumns: readonly Uint32Array[];
+    totalFieldLengths: Readonly<Record<string, number>>;
+    numeric: SegmentPayload['numeric'];
+    boolean: SegmentPayload['boolean'];
+    enums: SegmentPayload['enums'];
+    geo: SegmentPayload['geo'];
+    surfaceForms: SerializedSurfaceForms | null;
+}
+
+interface EncodedDocumentTableData {
+    blob: Uint8Array;
+    offsets: Uint32Array;
+}
+
+interface ExternalIdTableData {
+    blob: Uint8Array;
+    offsets: Uint32Array;
+    sortedOrdinals: Uint32Array;
+}
+
+interface SurfaceTableData {
+    blob: Uint8Array;
+    offsets: Uint32Array;
+    counts: Uint32Array;
+}
+
+interface FrozenTokenTableData {
+    blob: Uint8Array;
+    offsets: Uint32Array;
+    payloadSlots: Uint32Array;
+    documentFrequencies: Uint32Array;
+}
+
+interface SharedSegmentSnapshot extends Omit<SegmentColumns, 'surfaceForms'> {
+    segmentId: string;
+    tokenTable: FrozenTokenTableData;
+    idTable: ExternalIdTableData;
+    documentTable: EncodedDocumentTableData;
+    /** The surface forms packed flat, or null where the segment holds none. */
+    surfaceTable: SurfaceTableData | null;
+}
+
+interface CapturedFrozenSegment {
+    snapshot: SharedSegmentSnapshot;
+    tombstonedDocIds: string[];
+}
+interface CapturedWholePartition {
+    partitionId: number;
+    totalPartitions: number;
+    docCount: number;
+    liveTail: Uint8Array | null;
+    frozen: CapturedFrozenSegment[];
+}
 
 interface PartitionCheckpoint {
     partitionId: number;
@@ -50,20 +163,42 @@ interface PartitionCheckpoint {
     primaryTerm: number;
 }
 
+interface SegmentRef {
+    id: number;
+    key: string;
+    docCount: number;
+    tombstoneCount: number;
+}
+interface PartitionManifestEntry {
+    partitionId: number;
+    nextSegmentId: number;
+    segments: SegmentRef[];
+}
+
+interface CheckpointSegmentsWritten {
+    checkpoint: PartitionCheckpoint[];
+    partitions: PartitionManifestEntry[];
+}
+
 interface CheckpointWorkerRequest {
     root: string;
     metadata: IndexMetadata;
     targets: PartitionCheckpoint[];
     compactionThreshold: number;
+    capturedPartitions?: CapturedWholePartition[];
 }
 interface CheckpointWorkerSuccess {
     type: 'success';
+    segments: CheckpointSegmentsWritten;
 }
 interface CheckpointWorkerError {
     type: 'error';
     message: string;
 }
-type CheckpointWorkerMessage = CheckpointWorkerSuccess | CheckpointWorkerError;
+interface CheckpointWorkerHeartbeat {
+    type: 'heartbeat';
+}
+type CheckpointWorkerMessage = CheckpointWorkerSuccess | CheckpointWorkerError | CheckpointWorkerHeartbeat;
 declare function handleRequest(raw: unknown): Promise<CheckpointWorkerSuccess>;
 
-export { type CheckpointWorkerError, type CheckpointWorkerMessage, type CheckpointWorkerRequest, type CheckpointWorkerSuccess, handleRequest };
+export { type CheckpointWorkerError, type CheckpointWorkerHeartbeat, type CheckpointWorkerMessage, type CheckpointWorkerRequest, type CheckpointWorkerSuccess, handleRequest };

package/dist/persistence/durability/checkpoint-worker.mjs

@@ -1,26 +1,27 @@
-import { createDurableDirectory, writeSegmentedCheckpoint } from '../../chunk-ND34I3SV.mjs';
-import '../../chunk-ZHUQKZBE.mjs';
+import { CHECKPOINT_WORKER_HEARTBEAT_MS, createDurableDirectory, writeCheckpointSegments } from '../../chunk-ULJ36FK4.mjs';
 import '../../chunk-GCM3JFVA.mjs';
-import '../../chunk-PIWKZPP7.mjs';
-import '../../chunk-XE6XR2OE.mjs';
-import '../../chunk-BEIK3EOZ.mjs';
-import '../../chunk-CMVG2UEF.mjs';
+import '../../chunk-U36IDJN7.mjs';
+import '../../chunk-2FO7KZ6K.mjs';
 import '../../chunk-AV4IJKD2.mjs';
-import '../../chunk-CBLS46IE.mjs';
+import '../../chunk-YPYULM5E.mjs';
+import '../../chunk-3GWP2TO6.mjs';
+import '../../chunk-WM7IS45M.mjs';
+import '../../chunk-ZT5BRDFW.mjs';
 import '../../chunk-TGP5M5R7.mjs';
-import '../../chunk-PMBHH44R.mjs';
-import '../../chunk-X6YZO4MQ.mjs';
-import '../../chunk-RP23PWMV.mjs';
-import '../../chunk-KMMJMDUA.mjs';
+import '../../chunk-T62YOSGS.mjs';
+import '../../chunk-ZJ4CSAB2.mjs';
+import '../../chunk-WFKIGEBY.mjs';
+import '../../chunk-3L4U6EJQ.mjs';
+import '../../chunk-IFBKLBGB.mjs';
 import '../../chunk-U2BKYEC7.mjs';
 import '../../chunk-2GWVQWGA.mjs';
-import '../../chunk-WYJXSLER.mjs';
+import '../../chunk-POWOTUKJ.mjs';
 import '../../chunk-P4ELXVJQ.mjs';
 
 // src/persistence/durability/rebuild.ts
-async function rebuildSnapshotFromDurable(root, metadata, targets, compactionThreshold) {
+async function rebuildSegmentsFromDurable(root, metadata, targets, compactionThreshold, capturedPartitions = []) {
   const directory = createDurableDirectory(root);
-  await writeSegmentedCheckpoint({ directory, metadata, targets, compactionThreshold });
+  return writeCheckpointSegments({ directory, metadata, targets, compactionThreshold, capturedPartitions });
 }
 
 // src/persistence/durability/checkpoint-worker.ts
@@ -38,8 +39,17 @@ async function handleRequest(raw) {
   if (!Number.isInteger(request.compactionThreshold) || request.compactionThreshold <= 0) {
     throw new Error("Checkpoint request has an invalid compaction threshold");
   }
-  await rebuildSnapshotFromDurable(request.root, request.metadata, request.targets, request.compactionThreshold);
-  return { type: "success" };
+  if (request.capturedPartitions !== void 0 && !Array.isArray(request.capturedPartitions)) {
+    throw new Error("Checkpoint request names captured partitions in a shape the worker cannot read");
+  }
+  const segments = await rebuildSegmentsFromDurable(
+    request.root,
+    request.metadata,
+    request.targets,
+    request.compactionThreshold,
+    request.capturedPartitions
+  );
+  return { type: "success", segments };
 }
 async function setupAsync() {
   let parentPort = null;
@@ -54,10 +64,14 @@ async function setupAsync() {
   }
   const port = parentPort;
   port.on("message", (raw) => {
+    const heartbeat = setInterval(
+      () => port.postMessage({ type: "heartbeat" }),
+      CHECKPOINT_WORKER_HEARTBEAT_MS
+    );
     handleRequest(raw).then((result) => port.postMessage(result)).catch((err) => {
       const message = err instanceof Error ? err.message : String(err);
       port.postMessage({ type: "error", message });
-    });
+    }).finally(() => clearInterval(heartbeat));
   });
 }
 function setupWorker() {

package/dist/react.d.ts

@@ -1,10 +1,10 @@
-import { N as NarsilError, L as ListParams, g as ListResult, I as IndexInfo, f as IndexStats, Q as QueryParams, h as PreflightResult, e as QueryResult, i as SuggestParams, j as SuggestResult } from './memory-Bzd5xcEW.js';
-export { C as ClientErrorCode, a as ClientErrorCodes, E as ErrorCode, b as ErrorCodes, c as NarsilErrorCode, S as ServerErrorCode, d as ServerErrorCodes } from './memory-Bzd5xcEW.js';
-import { e as TaskRecord, d as TaskProgress, b as ImportResult, c as TaskListQuery, T as TaskListPage } from './tasks-DT7L1p6w.js';
-export { I as ImportError, f as TaskStatus, g as TaskType } from './tasks-DT7L1p6w.js';
+import { N as NarsilError, L as ListParams, g as ListResult, I as IndexInfo, f as IndexStats, Q as QueryParams, h as PreflightResult, e as QueryResult, i as SuggestParams, j as SuggestResult } from './memory-OmgkX_Z2.js';
+export { C as ClientErrorCode, a as ClientErrorCodes, E as ErrorCode, b as ErrorCodes, c as NarsilErrorCode, S as ServerErrorCode, d as ServerErrorCodes } from './memory-OmgkX_Z2.js';
+import { e as TaskRecord, d as TaskProgress, b as ImportResult, c as TaskListQuery, T as TaskListPage } from './tasks-DiF1c9v2.js';
+export { I as ImportError, f as TaskStatus, g as TaskType } from './tasks-DiF1c9v2.js';
 import { ReactNode, ReactElement } from 'react';
-import { N as NarsilClient, I as ImportSource } from './client-B_X6_AbT.js';
-import { A as AnyDocument } from './schema-BqRx9t4E.js';
+import { N as NarsilClient, I as ImportSource } from './client-BMJWVxOP.js';
+import { A as AnyDocument } from './schema-OSWaaZzx.js';
 import './adapters-79OIa4xm.js';
 
 /**

package/dist/{schema-BqRx9t4E.d.ts → schema-OSWaaZzx.d.ts}

@@ -40,14 +40,36 @@ type SchemaDefinition = {
     [field: string]: FieldType | SchemaDefinition;
 };
 /**
- * How the engine stores vectors once a field is promoted to an HNSW graph.
+ * How the engine codes vectors once a field is promoted to an HNSW graph.
  *
- * `sq8` quantises each component to a signed byte, which cuts memory to a
- * quarter and costs a small amount of recall. `none` keeps full precision.
+ * Every `osq` mode applies optimised scalar quantization, which centres each
+ * vector on the field's centroid and then fits an interval to that vector
+ * alone. The digit in the mode name gives the bits that each dimension's code
+ * holds, so `osq8` stores a byte per dimension, while `osq4`, `osq2`, and
+ * `osq1` store four, two, and one bit. A search ranks its candidates by their
+ * codes before it re-scores the nearest of them against the full-precision
+ * vectors. `none` stores no code, so the engine ranks by the vectors alone.
+ *
+ * When the configuration leaves this unset, the engine takes `osq4` at 384
+ * dimensions and above and `osq8` below 384.
+ *
+ * @public
+ */
+type VectorQuantizationMode = 'osq8' | 'osq4' | 'osq2' | 'osq1' | 'none';
+/**
+ * Where a vector field keeps its full-precision vectors.
+ *
+ * A `memory` field holds them in memory. A `disk` field keeps its codes and
+ * its graph in memory, while it reads a full-precision vector from the
+ * field's checkpoint file whenever a search re-scores that candidate or a
+ * document read returns it. A `disk` field therefore needs filesystem
+ * durability. Asking for one on an engine without it fails with
+ * `CONFIG_INVALID`. When the configuration leaves this unset, an engine with
+ * filesystem durability takes `disk` and every other engine takes `memory`.
  *
  * @public
  */
-type VectorQuantizationMode = 'sq8' | 'none';
+type VectorStorageMode = 'memory' | 'disk';
 /**
  * Controls when a vector field moves from a brute-force scan to an HNSW graph,
  * and how that graph is built.
@@ -74,8 +96,10 @@ interface VectorIndexConfig {
         efConstruction?: number;
         metric?: 'cosine' | 'dotProduct' | 'euclidean';
     };
-    /** The engine stores the promoted vectors at this precision, and it quantises to `sq8` by default. Set `none` to keep full precision. */
+    /** The engine codes the promoted vectors in this mode. When this is unset, it picks the mode by the field's dimension. Set `none` to rank by the vectors alone. */
     quantization?: VectorQuantizationMode;
+    /** The engine keeps the field's full-precision vectors here. When this is unset, it picks `disk` under filesystem durability and `memory` on every other engine. */
+    storage?: VectorStorageMode;
 }
 /**
  * Connects an index to an embedding adapter and says which fields feed it.
@@ -250,4 +274,4 @@ interface InsertOptions extends WriteOptions {
     skipClone?: boolean;
 }
 
-export type { AnyDocument as A, BM25Params as B, CustomTokenizer as C, EmbeddingFieldConfig as E, FieldType as F, IndexConfig as I, PartitionConfig as P, SchemaDefinition as S, VectorIndexConfig as V, WriteOptions as W, ScoringMode as a, InsertOptions as b, StopWordOverride as c, VectorQuantizationMode as d };
+export type { AnyDocument as A, BM25Params as B, CustomTokenizer as C, EmbeddingFieldConfig as E, FieldType as F, IndexConfig as I, PartitionConfig as P, SchemaDefinition as S, VectorIndexConfig as V, WriteOptions as W, InsertOptions as a, StopWordOverride as b, ScoringMode as c, VectorQuantizationMode as d, VectorStorageMode as e };

package/dist/serialization/crc32-worker.mjs

@@ -1,4 +1,5 @@
-import { crc32 } from '../chunk-KMMJMDUA.mjs';
+import { crc32 } from '../chunk-3L4U6EJQ.mjs';
+import '../chunk-IFBKLBGB.mjs';
 
 // src/serialization/crc32-worker.ts
 function handleRequest(raw) {

package/dist/server.d.ts

@@ -1,14 +1,14 @@
-import { b as ServerOptions, N as NarsilServer } from './capabilities-BWm4IITU.js';
-export { A as ASYNC_IMPORT_CAPABILITY, C as CorsOptions, E as ErrorEnvelope, I as INDEX_LIFECYCLE_CAPABILITY, O as OnRequestHook, R as REBUILD_ANALYSIS_CAPABILITY, c as RequestContext, d as RequestDenial, S as SERVER_CAPABILITIES, e as ServerLimits, T as TASK_CANCEL_CAPABILITY, a as TASK_FILTER_CAPABILITY } from './capabilities-BWm4IITU.js';
-import { a as Narsil } from './engine-9PtFY50q.js';
-import { d as TaskProgress, b as ImportResult, h as TaskStore, e as TaskRecord } from './tasks-DT7L1p6w.js';
-export { C as CapabilitiesResponse, a as CreateIndexEmbedding, i as CreateIndexRequest, H as HttpIndexConfig, I as ImportError, M as MemoryStatsResponse, T as TaskListPage, c as TaskListQuery, f as TaskStatus, g as TaskType } from './tasks-DT7L1p6w.js';
-export { d as ServerErrorCodes } from './memory-Bzd5xcEW.js';
+import { b as ServerOptions, N as NarsilServer } from './capabilities-CuTSXvbN.js';
+export { A as ASYNC_IMPORT_CAPABILITY, C as CorsOptions, E as ErrorEnvelope, I as INDEX_LIFECYCLE_CAPABILITY, O as OnRequestHook, R as REBUILD_ANALYSIS_CAPABILITY, c as RequestContext, d as RequestDenial, S as SERVER_CAPABILITIES, e as ServerLimits, T as TASK_CANCEL_CAPABILITY, a as TASK_FILTER_CAPABILITY } from './capabilities-CuTSXvbN.js';
+import { a as Narsil } from './engine-zhms1qId.js';
+import { d as TaskProgress, b as ImportResult, h as TaskStore, e as TaskRecord } from './tasks-DiF1c9v2.js';
+export { C as CapabilitiesResponse, a as CreateIndexEmbedding, i as CreateIndexRequest, H as HttpIndexConfig, I as ImportError, M as MemoryStatsResponse, T as TaskListPage, c as TaskListQuery, f as TaskStatus, g as TaskType } from './tasks-DiF1c9v2.js';
+export { d as ServerErrorCodes } from './memory-OmgkX_Z2.js';
 import './adapters-79OIa4xm.js';
-import './config-Czh2YNDi.js';
-import './config-OON7IJ_B.js';
-import './schema-BqRx9t4E.js';
-import './types-DXPxaXPO.js';
+import './config-qMLpi5T4.js';
+import './config-BcYZkdMw.js';
+import './schema-OSWaaZzx.js';
+import './types-LEzMyjVX.js';
 import './types-D5OwhtsQ.js';
 
 /**

package/dist/server.mjs

@@ -1,10 +1,12 @@
-import { DEFAULT_MAX_RETAINED_TASKS, loadUWebSockets, resolveCors, authorizerFor, createSharedGateBuffer, gateSlotOfWorker, createRouteRunner, corsWriter, createSharedRequestGate, MAIN_THREAD_GATE_SLOT, registerServerRoutes, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_BODY_BYTES, RUNNING_TASK_TTL_MS, DEFAULT_TASK_PAGE_SIZE, TERMINAL_TASK_TTL_MS, TASK_PROGRESS_WRITE_INTERVAL_MS, serializeNarsilError, acceptorOf, createDocumentReadHandlers, sendError, respondError, respondJson, MAX_INDEX_TOUCH_INTERVAL_MS, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, clearGateSlot, parseJson, validateCreateIndex, rejectInvalid, validateBatch, serializeBatchResult, validateDocumentBody, validateSuggest, validateQuery, parseTaskListQuery, parseJsonOptional, validateRebalance, sendBinary, badRequest, SERVER_CAPABILITIES } from './chunk-CV5H7XL7.mjs';
-export { ASYNC_IMPORT_CAPABILITY, INDEX_LIFECYCLE_CAPABILITY, REBUILD_ANALYSIS_CAPABILITY, SERVER_CAPABILITIES, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, httpStatusForNarsilError } from './chunk-CV5H7XL7.mjs';
-import { validateIndexName } from './chunk-MHE7YWAU.mjs';
-import { engineCoreOf, resolveVectorText } from './chunk-PIWKZPP7.mjs';
-import './chunk-XE6XR2OE.mjs';
+import { DEFAULT_MAX_RETAINED_TASKS, loadUWebSockets, resolveCors, authorizerFor, createSharedGateBuffer, gateSlotOfWorker, createRouteRunner, corsWriter, createSharedRequestGate, MAIN_THREAD_GATE_SLOT, registerServerRoutes, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_BODY_BYTES, RUNNING_TASK_TTL_MS, DEFAULT_TASK_PAGE_SIZE, TERMINAL_TASK_TTL_MS, TASK_PROGRESS_WRITE_INTERVAL_MS, serializeNarsilError, acceptorOf, createDocumentReadHandlers, sendError, respondError, respondJson, MAX_INDEX_TOUCH_INTERVAL_MS, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, clearGateSlot, parseJson, validateCreateIndex, rejectInvalid, validateBatch, serializeBatchResult, validateDocumentBody, validateSuggest, validateQuery, parseTaskListQuery, parseJsonOptional, validateRebalance, sendBinary, IMPORT_FLUSHES_IN_FLIGHT, badRequest, SERVER_CAPABILITIES } from './chunk-EY2ULUMT.mjs';
+export { ASYNC_IMPORT_CAPABILITY, INDEX_LIFECYCLE_CAPABILITY, REBUILD_ANALYSIS_CAPABILITY, SERVER_CAPABILITIES, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, httpStatusForNarsilError } from './chunk-EY2ULUMT.mjs';
+import { validateIndexName } from './chunk-72AXTIBW.mjs';
+import { engineCoreOf, providedDocId, resolveVectorText } from './chunk-U36IDJN7.mjs';
 import { detectRuntime } from './chunk-AV4IJKD2.mjs';
-import './chunk-CBLS46IE.mjs';
+import './chunk-3GWP2TO6.mjs';
+import './chunk-ZT5BRDFW.mjs';
+import './chunk-WFKIGEBY.mjs';
+import { startIdleHeapCleanup } from './chunk-Y3L5MDRV.mjs';
 import { NarsilError, ServerErrorCodes, ErrorCodes } from './chunk-P4ELXVJQ.mjs';
 export { ServerErrorCodes } from './chunk-P4ELXVJQ.mjs';
 import { randomUUID } from 'crypto';
@@ -444,10 +446,7 @@ async function runImport(engine, options) {
     failed += 1;
     if (errors.length < maxErrors) errors.push(error);
   };
-  const flush = async () => {
-    if (pending.length === 0) return;
-    const documents = pending.map((entry) => entry.document);
-    pending = [];
+  const flushDocuments = async (documents) => {
     const result = await engine.insertBatch(indexName, documents, { skipClone: true });
     indexed += result.succeeded.length;
     for (const failure of result.failed) {
@@ -455,36 +454,74 @@ async function runImport(engine, options) {
       recordFailure({ docId: failure.docId, code: serialized.code, message: serialized.message });
     }
   };
+  const inFlight = /* @__PURE__ */ new Set();
+  let flushError;
+  const startFlush = () => {
+    if (pending.length === 0) return;
+    const documents = pending.map((entry) => entry.document);
+    pending = [];
+    const run = flushDocuments(documents).catch((err) => {
+      flushError ??= err;
+    }).finally(() => inFlight.delete(run));
+    inFlight.add(run);
+  };
+  const settleFlushes = async () => {
+    await Promise.all(inFlight);
+    if (flushError !== void 0) throw flushError;
+  };
   const report = () => {
     onProgress?.({ indexed, failed, bytesProcessed, bytesTotal });
   };
-  for (const line of iterateNdjson(body, maxLineBytes)) {
-    bytesProcessed = line.bytesConsumed;
-    let parsed;
-    try {
-      parsed = JSON.parse(line.text);
-    } catch {
-      recordFailure({ line: line.lineNumber, code: ServerErrorCodes.INVALID_JSON, message: "Line is not valid JSON" });
-      continue;
-    }
-    if (!isPlainObject(parsed)) {
-      recordFailure({
-        line: line.lineNumber,
-        code: ServerErrorCodes.INVALID_REQUEST,
-        message: "Line is not a JSON object"
-      });
-      continue;
+  const seenIds = /* @__PURE__ */ new Set();
+  const repeated = [];
+  try {
+    for (const line of iterateNdjson(body, maxLineBytes)) {
+      bytesProcessed = line.bytesConsumed;
+      let parsed;
+      try {
+        parsed = JSON.parse(line.text);
+      } catch {
+        recordFailure({ line: line.lineNumber, code: ServerErrorCodes.INVALID_JSON, message: "Line is not valid JSON" });
+        continue;
+      }
+      if (!isPlainObject(parsed)) {
+        recordFailure({
+          line: line.lineNumber,
+          code: ServerErrorCodes.INVALID_REQUEST,
+          message: "Line is not a JSON object"
+        });
+        continue;
+      }
+      const docId = providedDocId(parsed);
+      if (docId !== void 0) {
+        if (seenIds.has(docId)) {
+          repeated.push({ document: parsed, line: line.lineNumber });
+          continue;
+        }
+        seenIds.add(docId);
+      }
+      pending.push({ document: parsed, line: line.lineNumber });
+      if (pending.length >= batchSize) {
+        signal?.throwIfAborted();
+        startFlush();
+        if (inFlight.size >= IMPORT_FLUSHES_IN_FLIGHT) await Promise.race(inFlight);
+        if (flushError !== void 0) await settleFlushes();
+        report();
+        await yieldToEventLoop();
+        signal?.throwIfAborted();
+      }
     }
-    pending.push({ document: parsed, line: line.lineNumber });
-    if (pending.length >= batchSize) {
-      signal?.throwIfAborted();
-      await flush();
-      report();
-      await yieldToEventLoop();
+    startFlush();
+    await settleFlushes();
+    for (let start = 0; start < repeated.length; start += batchSize) {
       signal?.throwIfAborted();
+      await flushDocuments(repeated.slice(start, start + batchSize).map((entry) => entry.document));
     }
+  } catch (err) {
+    await Promise.allSettled(inFlight);
+    report();
+    throw err;
   }
-  await flush();
   bytesProcessed = bytesTotal;
   report();
   return { indexed, failed, errors, errorsTruncated: failed > errors.length };
@@ -804,7 +841,7 @@ async function answerRequest(handlers, request) {
     params: request.params,
     query: new URLSearchParams(request.query),
     contentType: request.contentType,
-    rawBody: request.rawBody === null ? null : Buffer.from(request.rawBody),
+    rawBody: request.rawBody === null ? null : Buffer.from(request.rawBody.buffer, request.rawBody.byteOffset, request.rawBody.byteLength),
     abort: NO_ABORT,
     hookContext: request.hookContext
   };
@@ -1262,6 +1299,7 @@ var NarsilHttpServer = class {
   listenSocket = null;
   ready = false;
   threads = null;
+  stopIdleHeapCleanup = () => void 0;
   constructor(engine, options = {}) {
     this.host = options.host ?? "127.0.0.1";
     this.port = options.port ?? 9876;
@@ -1301,10 +1339,13 @@ var NarsilHttpServer = class {
       });
     });
     await this.deps.tasks.reconcile();
+    this.stopIdleHeapCleanup = startIdleHeapCleanup();
     this.ready = true;
   }
   async close() {
     this.ready = false;
+    this.stopIdleHeapCleanup();
+    this.stopIdleHeapCleanup = () => void 0;
     if (this.listenSocket && this.uws) {
       this.uws.us_listen_socket_close(this.listenSocket);
       this.listenSocket = null;