@delali/narsil 0.2.4-next.51 → 0.2.4-next.54

package/dist/distribution.mjs

@@ -1,23 +1,24 @@
 import { MAX_MESSAGE_SIZE_BYTES } from './chunk-UNEEFTEU.mjs';
-import { validateIndexName, indexConfigKey, putIndexMetadata, getIndexMetadata } from './chunk-MHE7YWAU.mjs';
-import { createEngineCore, checkHeapAfterRecovery, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, createEngineIndex, applyIndexEntry, applyDeleteEntry, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, createForwardBatchResultMessage, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage, validateReplicationEntry, computeEntryChecksum, createEntryMessage, createEntryBatchMessage, createInsyncConfirmMessage, validateAckPayload, createInsyncRemoveMessage, validateInsyncConfirmPayload, createInsyncAddMessage } from './chunk-ND34I3SV.mjs';
-import { now, listBindingOf, queryBindingOf, foldGroupReducer, clampLimit, dedupePinnedEntries } from './chunk-ZHUQKZBE.mjs';
+import { validateIndexName, indexConfigKey, putIndexMetadata, getIndexMetadata } from './chunk-NTZYA3SJ.mjs';
+import { createEngineCore, checkHeapAfterRecovery, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, createEngineIndex, applyIndexEntry, applyDeleteEntry, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, createForwardBatchResultMessage, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage, validateReplicationEntry, computeEntryChecksum, createEntryMessage, createEntryBatchMessage, createInsyncConfirmMessage, validateAckPayload, createInsyncRemoveMessage, validateInsyncConfirmPayload, createInsyncAddMessage } from './chunk-BH5SYB5W.mjs';
 import { ReplicationMessageTypes, TransportErrorCodes, ClusterMessageTypes, QueryMessageTypes, TransportError } from './chunk-GCM3JFVA.mjs';
-import { MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR, WIRE_BATCH_BUDGET } from './chunk-PIWKZPP7.mjs';
-import { DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, MAX_INDEX_NAME_LENGTH, MAX_DOC_ID_LENGTH, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING, INDEX_NAME_PATTERN } from './chunk-XE6XR2OE.mjs';
-import { generateId, deserializePayloadV2, createPartitionIndex, getLanguage, FILTER_EXPRESSION_KEYS, FIELD_FILTER_OPERATORS } from './chunk-BEIK3EOZ.mjs';
-import { resolveProjection, clampRowCount, sortSignatureOf, requireMatchingCursor, decodePageCursor, normalizeSort, encodePageCursor, applyProjection, readSortValues, requireWithinResultWindow, fnv1a, mergePartitionStats, MAX_NESTING_DEPTH } from './chunk-CMVG2UEF.mjs';
+import { providedDocId, MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR, WIRE_BATCH_BUDGET } from './chunk-4NVPQ6S5.mjs';
+import { generateId, deserializePayloadV2, createPartitionIndex, getLanguage, FILTER_EXPRESSION_KEYS, FIELD_FILTER_OPERATORS } from './chunk-XNQAIO74.mjs';
 import './chunk-AV4IJKD2.mjs';
-import { DEFAULT_PAGE_SIZE, MAX_SORT_FIELDS, MAX_CURSOR_LENGTH, RESULT_WINDOW } from './chunk-CBLS46IE.mjs';
+import { now, listBindingOf, queryBindingOf, foldGroupReducer, clampLimit, dedupePinnedEntries } from './chunk-TVT5V4E7.mjs';
+import { DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, MAX_INDEX_NAME_LENGTH, MAX_DOC_ID_LENGTH, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING, INDEX_NAME_PATTERN } from './chunk-D2S642IV.mjs';
+import { resolveProjection, clampRowCount, sortSignatureOf, requireMatchingCursor, decodePageCursor, normalizeSort, encodePageCursor, applyProjection, readSortValues, requireWithinResultWindow, fnv1a, mergePartitionStats } from './chunk-5T6AOR37.mjs';
+import { DEFAULT_PAGE_SIZE, MAX_SORT_FIELDS, MAX_CURSOR_LENGTH, RESULT_WINDOW, MAX_NESTING_DEPTH } from './chunk-L225SHNB.mjs';
 import './chunk-TGP5M5R7.mjs';
-import './chunk-PMBHH44R.mjs';
-import './chunk-X6YZO4MQ.mjs';
-import './chunk-RP23PWMV.mjs';
-import { crc32 } from './chunk-KMMJMDUA.mjs';
+import './chunk-U2AL3SON.mjs';
+import { MIN_OVERSAMPLE } from './chunk-ZJ4CSAB2.mjs';
+import './chunk-UTOFOJOR.mjs';
+import { crc32 } from './chunk-3L4U6EJQ.mjs';
+import './chunk-IFBKLBGB.mjs';
 import './chunk-U2BKYEC7.mjs';
 import { compareCodePoints, toComparableSortValue, compareComparableKeys, compareSortValues } from './chunk-2GWVQWGA.mjs';
-import './chunk-WYJXSLER.mjs';
-import { ErrorCodes, NarsilError, describeError } from './chunk-P4ELXVJQ.mjs';
+import './chunk-POWOTUKJ.mjs';
+import { ErrorCodes, NarsilError, describeError } from './chunk-FRQVMFY7.mjs';
 import { encode, decode } from '@msgpack/msgpack';
 
 // src/distribution/cluster/constants.ts
@@ -5159,7 +5160,7 @@ async function applyLocalReplicationEntry(core, entry) {
     const vecIndexes = manager.getVectorIndexes();
     if (entry.operation === "INDEX") {
       applyIndexEntry(entry, manager, indexEntry.vectorFieldPaths, vecIndexes);
-      const appliedDocument = manager.get(entry.documentId);
+      const appliedDocument = manager.getRef(entry.documentId);
       if (appliedDocument !== void 0) {
         await core.orchestrator.replicateToWorkers({
           type: "insert",
@@ -6088,6 +6089,7 @@ var MAX_PREFIX_LENGTH = 1024;
 var MAX_PINNED_ENTRIES = 1e3;
 var MAX_PINNED_POSITION = RESULT_WINDOW;
 var MAX_EF_SEARCH = 1e4;
+var MAX_OVERSAMPLE = 100;
 var MAX_GROUP_FIELDS = 64;
 var MAX_HIGHLIGHT_TAG_LENGTH = 256;
 var MAX_HIGHLIGHT_SNIPPET_LENGTH = 65536;
@@ -6722,6 +6724,14 @@ function validateEfSearchParam(value) {
     );
   }
 }
+function validateOversampleParam(value) {
+  if (typeof value !== "number" || !Number.isFinite(value) || value < MIN_OVERSAMPLE || value > MAX_OVERSAMPLE) {
+    throwInvalid(
+      CONFIG_INVALID,
+      `Invalid SearchPayload: "params.vector.oversample" must be a number between ${MIN_OVERSAMPLE} and ${MAX_OVERSAMPLE}`
+    );
+  }
+}
 
 // src/distribution/query/validators/search.ts
 var MAX_FACET_SHARD_SIZE = oversampledShardSize(MAX_FACET_SIZE);
@@ -6784,6 +6794,9 @@ function validateVectorParams(value) {
   if (value.efSearch !== null) {
     validateEfSearchParam(value.efSearch);
   }
+  if (value.oversample !== null) {
+    validateOversampleParam(value.oversample);
+  }
 }
 function validateSortParams(value) {
   if (!Array.isArray(value)) {
@@ -7560,6 +7573,7 @@ function convertWireVectorToLocal(vector) {
   if (vector.similarity !== null) local.similarity = vector.similarity;
   if (vector.metric !== null) local.metric = vector.metric;
   if (vector.efSearch !== null) local.efSearch = vector.efSearch;
+  if (vector.oversample !== null) local.oversample = vector.oversample;
   return local;
 }
 function convertWireHybridToLocal(hybrid) {
@@ -7659,7 +7673,8 @@ function convertLocalVectorToWire(vector) {
     text: vector.text ?? null,
     similarity: vector.similarity ?? null,
     metric: vector.metric ?? null,
-    efSearch: vector.efSearch ?? null
+    efSearch: vector.efSearch ?? null,
+    oversample: vector.oversample ?? null
   };
 }
 function convertLocalHybridToWire(hybrid) {
@@ -9189,7 +9204,7 @@ async function routeDropIndex(name, coordinator, engine) {
   await coordinator.dropSchema(name);
 }
 async function routeInsert(indexName, document, docId, deps, options) {
-  const resolvedDocId = docId ?? generateId();
+  const resolvedDocId = docId ?? providedDocId(document) ?? generateId();
   const resolution = await resolvePrimaryAssignment(indexName, resolvedDocId, deps, false);
   if (resolution === null) {
     return deps.engine.insert(indexName, document, resolvedDocId, options);

package/dist/embeddings/openai.mjs

@@ -1,4 +1,4 @@
-import { NarsilError, ErrorCodes } from '../chunk-P4ELXVJQ.mjs';
+import { NarsilError, ErrorCodes } from '../chunk-FRQVMFY7.mjs';
 
 // src/embeddings/constants.ts
 var RETRYABLE_STATUS_CODES = /* @__PURE__ */ new Set([429, 500, 502, 503]);

package/dist/{engine-9PtFY50q.d.ts → engine-Dj4I91Jv.d.ts}

@@ -1,6 +1,6 @@
 import { E as EmbeddingAdapter } from './adapters-79OIa4xm.js';
-import { B as BatchResult, I as IndexInfo, f as IndexStats, P as PartitionStatsResult, L as ListParams, g as ListResult, Q as QueryParams, e as QueryResult, h as PreflightResult, i as SuggestParams, j as SuggestResult, M as MemoryStats, V as VectorMaintenanceResult } from './memory-Bzd5xcEW.js';
-import { A as AnyDocument, b as InsertOptions, W as WriteOptions, I as IndexConfig, P as PartitionConfig } from './schema-BqRx9t4E.js';
+import { B as BatchResult, I as IndexInfo, f as IndexStats, P as PartitionStatsResult, L as ListParams, g as ListResult, Q as QueryParams, e as QueryResult, h as PreflightResult, j as SuggestParams, k as SuggestResult, M as MemoryStats, V as VectorMaintenanceResult } from './memory-xrXOZMbw.js';
+import { A as AnyDocument, a as InsertOptions, W as WriteOptions, I as IndexConfig, P as PartitionConfig } from './schema-mLpXZBEh.js';
 
 /**
  * Every event the engine emits, with the payload each one carries.
@@ -68,17 +68,23 @@ type NarsilEventMap = {
         partitionCount: number;
     };
     /**
-     * The process heap crossed nine tenths of its limit during a write or a
+     * The process uses nine tenths of its heap, measured during a write or a
      * load, so the next large index may end the process with an out-of-memory
-     * error. The engine emits the event once per crossing and arms it again
-     * once the heap falls below eight tenths of the limit. Raise the limit with
+     * error. Where `--max-old-space-size` or `--max-old-space-size-percentage`
+     * sets a limit, on the command line or in `NODE_OPTIONS`, the engine measures
+     * the used bytes against that figure and `heapLimit` reports it. Where
+     * neither flag sets one, the engine compares the used bytes with the headroom
+     * that V8 reports, which runs about 192 MB above the true ceiling, so a
+     * default heap below about 2 GB may end before this event fires. The engine
+     * emits the event once per crossing and arms it again once the headroom
+     * recovers to two tenths. Raise the limit with
      * `--max-old-space-size-percentage` or `--max-old-space-size`, or close an
      * idle index; see {@link ProcessMemoryReport.heapLimit}.
      */
     heapPressure: {
-        /** The heap crossed nine tenths of its limit during a write to this index or a load of it. */
+        /** The heap reaches that point during a write to this index or a load of it. */
         indexName: string;
-        /** The process is using this much heap, in bytes. */
+        /** The process uses this much heap, in bytes. */
         heapUsed: number;
         /** The heap may grow to this many bytes. */
         heapLimit: number;
@@ -155,7 +161,9 @@ interface DocumentWriteOperations {
      * @param indexName - The index that receives the document.
      * @param document - Its fields must match the types the schema declares.
      * @param docId - Pass an id to control it yourself, or omit it and read the
-     * returned value.
+     * returned value. An id that differs from the document's own `id` field
+     * raises `DOC_VALIDATION_FAILED`, because the stored document would
+     * otherwise contradict the key that it is stored under.
      * @param options - Per-write settings, such as skipping the defensive copy
      * or waiting for every worker copy to apply the write.
      * @returns The id the document is stored under.
@@ -294,6 +302,11 @@ interface Narsil extends IndexLifecycleOperations, DocumentWriteOperations {
     /**
      * Fetches one stored document by id, without searching.
      *
+     * A vector field comes back as a `Float32Array`, whatever form you give it.
+     * `JSON.stringify` writes one as an object keyed by index, so convert it
+     * with `Array.from`, or project the field away, before you serialise the
+     * document yourself. The HTTP routes convert it for you.
+     *
      * @param indexName - The index holding the document.
      * @param docId - The document to fetch.
      * @returns The document, or `undefined` when the index holds no such id.