@delali/narsil 0.1.11 → 0.1.12

package/dist/{chunk-CU6BVZEU.mjs → chunk-MEW7HD75.mjs}

@@ -1,4 +1,4 @@
-import { getNestedValue, getLanguage, extractVectorFieldsFromSchema, createDirectExecutor, createPartitionRouter, createPartitionManager, createVectorIndex, validateSchema, deserializePayloadV2, tokenize, fanOutQuery, validateRequiredFields, validateVectorPromotion, createPartitionIndex, flattenSchema, fnv1a, detectRuntime, isNodeMainThread, spawnNodeWorker, kWayMerge } from './chunk-Q4OVNESD.mjs';
+import { getNestedValue, getLanguage, extractVectorFieldsFromSchema, createDirectExecutor, createPartitionRouter, createPartitionManager, createVectorIndex, validateSchema, deserializePayloadV2, deserializePayloadV1, tokenize, fanOutQuery, validateRequiredFields, validateVectorPromotion, createPartitionIndex, flattenSchema, fnv1a, detectRuntime, serializeMetadata, isNodeMainThread, spawnNodeWorker, deserializeMetadata, kWayMerge } from './chunk-HMO6OMZ3.mjs';
 import { crc32, crc32Init, crc32Update, crc32Final } from './chunk-KMMJMDUA.mjs';
 import { ErrorCodes, NarsilError } from './chunk-ALB35PRQ.mjs';
 import { generateId, createRequestId, isValidWorkerResponse } from './chunk-HRABJLPB.mjs';
@@ -49,9 +49,15 @@ function createRebalancer() {
       });
       const newPartitions = [];
       for (let i = 0; i < newPartitionCount; i++) {
-        newPartitions.push(createPartitionIndex(i));
+        newPartitions.push(createPartitionIndex(i, manager.config.trackPositions ?? true));
       }
       let documentsProcessed = 0;
+      const insertOptions = {
+        validate: false,
+        stopWordOverride: manager.config.stopWords,
+        customTokenizer: manager.config.tokenizer,
+        collectSurfaces: manager.config.surfaceForms === true
+      };
       for (let chunkStart = 0; chunkStart < collectedDocs.length; chunkStart += CHUNK_SIZE) {
         const chunkEnd = Math.min(chunkStart + CHUNK_SIZE, collectedDocs.length);
         for (let i = chunkStart; i < chunkEnd; i++) {
@@ -61,7 +67,7 @@ function createRebalancer() {
           if (targetPartition.has(docId)) {
             continue;
           }
-          targetPartition.insert(docId, document, manager.schema, manager.language, { validate: false });
+          targetPartition.insert(docId, document, manager.schema, manager.language, insertOptions);
           documentsProcessed++;
         }
         onProgress?.({
@@ -748,141 +754,6 @@ function readHeader(data) {
     reserved
   };
 }
-var VALID_HNSW_METRICS = /* @__PURE__ */ new Set(["cosine", "dotProduct", "euclidean"]);
-function validateHnswMetric(value) {
-  if (value === void 0 || value === null) return void 0;
-  if (typeof value === "string" && VALID_HNSW_METRICS.has(value)) {
-    return value;
-  }
-  return void 0;
-}
-function wireToPartition(raw) {
-  const documents = {};
-  for (const [docId, doc] of Object.entries(raw.documents ?? {})) {
-    documents[docId] = {
-      fields: doc.fields,
-      fieldLengths: doc.field_lengths ?? {}
-    };
-  }
-  const invertedIndex = {};
-  for (const [token, list] of Object.entries(raw.inverted_index ?? {})) {
-    invertedIndex[token] = {
-      docFrequency: list.doc_freq,
-      postings: (list.postings ?? []).map((p) => ({
-        docId: p.doc_id,
-        termFrequency: p.term_freq,
-        field: p.field,
-        positions: p.positions ?? []
-      }))
-    };
-  }
-  const numeric = {};
-  for (const [field, entries] of Object.entries(raw.field_indexes?.numeric ?? {})) {
-    numeric[field] = entries.map((e) => ({ value: e.value, docId: e.doc_id }));
-  }
-  const boolean = {};
-  for (const [field, idx] of Object.entries(raw.field_indexes?.boolean ?? {})) {
-    boolean[field] = { trueDocs: idx.true_docs ?? [], falseDocs: idx.false_docs ?? [] };
-  }
-  const geopoint = {};
-  for (const [field, entries] of Object.entries(raw.field_indexes?.geopoint ?? {})) {
-    geopoint[field] = entries.map((e) => ({ lat: e.lat, lon: e.lon, docId: e.doc_id }));
-  }
-  const vectorData = {};
-  for (const [field, data] of Object.entries(raw.vector_data ?? {})) {
-    vectorData[field] = {
-      dimension: data.dimension,
-      vectors: (data.vectors ?? []).map((v) => ({ docId: v.doc_id, vector: v.vector })),
-      hnswGraph: data.hnsw_graph ? {
-        entryPoint: data.hnsw_graph.entry_point,
-        maxLayer: data.hnsw_graph.max_layer,
-        m: data.hnsw_graph.m,
-        efConstruction: data.hnsw_graph.ef_construction,
-        metric: validateHnswMetric(data.hnsw_graph.metric),
-        nodes: data.hnsw_graph.nodes
-      } : null,
-      sq8: data.sq8 ? {
-        alpha: data.sq8.alpha,
-        offset: data.sq8.offset,
-        quantizedVectors: data.sq8.quantized_vectors,
-        vectorSums: data.sq8.vector_sums,
-        vectorSumSqs: data.sq8.vector_sum_sqs
-      } : null
-    };
-  }
-  return {
-    indexName: raw.index_name,
-    partitionId: raw.partition_id,
-    totalPartitions: raw.total_partitions,
-    language: raw.language ?? "english",
-    schema: raw.schema ?? {},
-    docCount: raw.doc_count ?? 0,
-    avgDocLength: raw.avg_doc_length ?? 0,
-    documents,
-    invertedIndex,
-    fieldIndexes: {
-      numeric,
-      boolean,
-      enum: raw.field_indexes?.enum ?? {},
-      geopoint
-    },
-    vectorData,
-    statistics: {
-      totalDocuments: raw.statistics?.total_documents ?? 0,
-      totalFieldLengths: raw.statistics?.total_field_lengths ?? {},
-      averageFieldLengths: raw.statistics?.average_field_lengths ?? {},
-      docFrequencies: raw.statistics?.doc_frequencies ?? {}
-    }
-  };
-}
-function deserializePayloadV1(data) {
-  const raw = decode(data);
-  return wireToPartition(raw);
-}
-function metadataToWire(meta) {
-  const wire = {
-    index_name: meta.indexName,
-    schema: meta.schema,
-    language: meta.language,
-    partition_count: meta.partitionCount,
-    bm25_params: meta.bm25Params,
-    created_at: meta.createdAt,
-    engine_version: meta.engineVersion
-  };
-  if (meta.vectorFields) {
-    wire.vector_fields = meta.vectorFields;
-  }
-  if (meta.embedding) {
-    wire.embedding = meta.embedding.adapter !== void 0 ? { adapter: meta.embedding.adapter, fields: meta.embedding.fields } : { fields: meta.embedding.fields };
-  }
-  return wire;
-}
-function wireToMetadata(raw) {
-  const meta = {
-    indexName: raw.index_name,
-    schema: raw.schema ?? {},
-    language: raw.language ?? "english",
-    partitionCount: raw.partition_count ?? 1,
-    bm25Params: raw.bm25_params ?? { k1: 1.2, b: 0.75 },
-    createdAt: raw.created_at ?? 0,
-    engineVersion: raw.engine_version ?? "0.0.0"
-  };
-  if (raw.vector_fields) {
-    meta.vectorFields = raw.vector_fields;
-  }
-  if (raw.embedding && typeof raw.embedding === "object" && typeof raw.embedding.fields === "object") {
-    meta.embedding = typeof raw.embedding.adapter === "string" ? { adapter: raw.embedding.adapter, fields: raw.embedding.fields } : { fields: raw.embedding.fields };
-  }
-  return meta;
-}
-function serializeMetadata(meta) {
-  const wire = metadataToWire(meta);
-  return encode(wire);
-}
-function deserializeMetadata(data) {
-  const raw = decode(data);
-  return wireToMetadata(raw);
-}
 
 // src/serialization/envelope.ts
 var CURRENT_ENVELOPE_VERSION = 1;
@@ -1907,6 +1778,7 @@ function mergeTimeOrderedSegments(ordered, fallback) {
     collectDocuments(merged, part, winner, segIndex);
     collectInvertedIndex(merged, part, winner, segIndex);
     collectFieldIndexes(merged, part, winner, segIndex);
+    collectSurfaceForms(merged, part);
   }
   recomputeStatistics(merged);
   validateMergedPartition(merged);
@@ -2034,6 +1906,26 @@ function collectFieldIndexes(merged, part, winner, segIndex) {
     }
   }
 }
+function collectSurfaceForms(merged, part) {
+  if (!part.surfaceForms) return;
+  let target = merged.surfaceForms;
+  if (!target) {
+    target = /* @__PURE__ */ Object.create(null);
+    merged.surfaceForms = target;
+  }
+  for (const [surface, value] of Object.entries(part.surfaceForms)) {
+    const count = typeof value === "number" ? value : value[0];
+    const token = typeof value === "number" ? surface : value[1];
+    const existing = target[surface];
+    if (existing === void 0) {
+      target[surface] = value;
+      continue;
+    }
+    const existingCount = typeof existing === "number" ? existing : existing[0];
+    const total = existingCount + count;
+    target[surface] = token === surface ? total : [total, token];
+  }
+}
 function recomputeStatistics(merged) {
   const totalFieldLengths = /* @__PURE__ */ Object.create(null);
   let totalDocuments = 0;
@@ -2244,6 +2136,9 @@ function reconstructSchemaFromMetadata(metadata) {
   if (metadata.embedding) {
     config.embedding = metadata.embedding.adapter !== void 0 ? { adapter: metadata.embedding.adapter, fields: metadata.embedding.fields } : { fields: metadata.embedding.fields };
   }
+  if (metadata.surfaceForms === true) {
+    config.surfaceForms = true;
+  }
   return config;
 }
 
@@ -3830,6 +3725,9 @@ function buildMetadata(indexName, hooks) {
   if (config.embedding) {
     metadata.embedding = config.embedding;
   }
+  if (config.surfaceForms) {
+    metadata.surfaceForms = true;
+  }
   return metadata;
 }
 function createDurabilityIntegration(tier, hooks) {
@@ -4349,7 +4247,8 @@ function createDurabilityFromConfig(config, wiring) {
         language: entry.language.name,
         k1: entry.config.bm25?.k1 ?? 1.2,
         b: entry.config.bm25?.b ?? 0.75,
-        ...embedding !== void 0 ? { embedding } : {}
+        ...embedding !== void 0 ? { embedding } : {},
+        ...entry.config.surfaceForms === true ? { surfaceForms: true } : {}
       };
     },
     createIndexFromMetadata: wiring.createIndexFromMetadata,
@@ -5949,10 +5848,15 @@ function highlightField(text, queryTokens, language, options) {
     return qt.token.toLowerCase();
   });
   const matchedRanges = [];
+  const prefixToken = options?.prefixToken;
   for (let i = 0; i < fieldResult.tokens.length; i++) {
     if (i >= charOffsets.length) break;
     const fieldToken = fieldResult.tokens[i].token;
     const stemmedField = language.stemmer ? language.stemmer(fieldToken) : fieldToken;
+    if (prefixToken !== void 0 && fieldToken.startsWith(prefixToken)) {
+      matchedRanges.push({ start: charOffsets[i].start, end: charOffsets[i].end });
+      continue;
+    }
     for (const stemmedQuery of stemmedQueryTokens) {
       if (stemmedField === stemmedQuery) {
         matchedRanges.push({ start: charOffsets[i].start, end: charOffsets[i].end });
@@ -5996,6 +5900,22 @@ function applyHighlights(hits, params, language, config) {
     customTokenizer: config.tokenizer,
     stopWordOverride: config.stopWords
   });
+  let prefixToken;
+  if (params.prefix === true && params.exact !== true && queryTokenResult.tokens.length > 0) {
+    const lastPosition = queryTokenResult.tokens[queryTokenResult.tokens.length - 1].position;
+    const unstemmed = tokenize(params.term ?? "", language, {
+      stem: false,
+      removeStopWords: true,
+      customTokenizer: config.tokenizer,
+      stopWordOverride: config.stopWords
+    });
+    for (const t of unstemmed.tokens) {
+      if (t.position === lastPosition) {
+        prefixToken = t.token;
+        break;
+      }
+    }
+  }
   for (const hit of hits) {
     const highlights = {};
     for (const field of params.highlight.fields) {
@@ -6005,7 +5925,8 @@ function applyHighlights(hits, params, language, config) {
         highlights[field] = highlightField(fieldValue, queryTokenResult.tokens, language, {
           preTag: params.highlight.preTag,
           postTag: params.highlight.postTag,
-          maxSnippetLength: params.highlight.maxSnippetLength
+          maxSnippetLength: params.highlight.maxSnippetLength,
+          prefixToken
         });
       }
     }
@@ -6564,8 +6485,8 @@ async function createSnapshot(manager, entry) {
   for (const [fieldPath, vecIndex] of snapshotVecIndexes) {
     vectorPayloads[fieldPath] = vecIndex.serialize();
   }
-  const { encode: encode11 } = await import('@msgpack/msgpack');
-  return encode11({
+  const { encode: encode10 } = await import('@msgpack/msgpack');
+  return encode10({
     version: 2,
     schema: entry.config.schema,
     language: entry.language.name,
@@ -6577,8 +6498,8 @@ async function restoreFromSnapshot(indexName, data, executor, indexRegistry, get
   if (!(data instanceof Uint8Array)) {
     throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Snapshot data must be a Uint8Array");
   }
-  const { decode: decode11 } = await import('@msgpack/msgpack');
-  const envelope = decode11(data);
+  const { decode: decode10 } = await import('@msgpack/msgpack');
+  const envelope = decode10(data);
   if (envelope.version !== 1 && envelope.version !== 2) {
     throw new NarsilError(
       ErrorCodes.DOC_VALIDATION_FAILED,
@@ -6654,25 +6575,36 @@ function executeSuggest(manager, language, params) {
     return { terms: [], elapsed: performance.now() - t0 };
   }
   const stemmed = language.stemmer ? language.stemmer(lastToken) : lastToken;
-  const prefixes = stemmed !== lastToken ? [lastToken, stemmed] : [lastToken];
-  const partitions = manager.getAllPartitions();
   const merged = /* @__PURE__ */ new Map();
   const perPartitionLimit = limit * 2;
-  for (const partition of partitions) {
-    const seen = /* @__PURE__ */ new Set();
-    for (const prefix of prefixes) {
-      const suggestions = partition.suggestTerms(prefix, perPartitionLimit);
-      for (const s of suggestions) {
-        if (seen.has(s.term)) continue;
-        seen.add(s.term);
-        merged.set(s.term, (merged.get(s.term) ?? 0) + s.documentFrequency);
+  for (const partition of manager.getAllPartitions()) {
+    for (const suggestion of partition.suggestTerms(lastToken, stemmed, perPartitionLimit)) {
+      let entry = merged.get(suggestion.token);
+      if (!entry) {
+        entry = { documentFrequency: 0, surfaceOccurrences: /* @__PURE__ */ new Map() };
+        merged.set(suggestion.token, entry);
+      }
+      entry.documentFrequency += suggestion.documentFrequency;
+      for (const s of suggestion.surfaces) {
+        entry.surfaceOccurrences.set(s.surface, (entry.surfaceOccurrences.get(s.surface) ?? 0) + s.occurrences);
       }
     }
   }
-  const terms = Array.from(merged.entries()).map(([term, documentFrequency]) => ({ term, documentFrequency })).sort((a, b) => b.documentFrequency - a.documentFrequency);
+  const terms = Array.from(merged.values()).map((entry) => ({ term: pickDisplaySurface(entry.surfaceOccurrences), documentFrequency: entry.documentFrequency })).sort((a, b) => b.documentFrequency - a.documentFrequency || (a.term < b.term ? -1 : 1));
   if (terms.length > limit) terms.length = limit;
   return { terms, elapsed: performance.now() - t0 };
 }
+function pickDisplaySurface(surfaceOccurrences) {
+  let best = "";
+  let bestCount = -1;
+  for (const [surface, occurrences] of surfaceOccurrences) {
+    if (occurrences > bestCount || occurrences === bestCount && surface < best) {
+      best = surface;
+      bestCount = occurrences;
+    }
+  }
+  return best;
+}
 
 // src/engine/vector-maintenance.ts
 function resolveVectorIndexes(manager, indexName, fieldName) {

package/dist/{config-DX--2Owg.d.ts → config-CojEPOgS.d.ts}

@@ -1,5 +1,5 @@
 import { b as PersistenceAdapter, I as InvalidationAdapter, E as EmbeddingAdapter } from './adapters-Bo2JQg8b.js';
-import { A as AnyDocument, b as ScoringMode, S as SchemaDefinition, I as IndexConfig } from './schema-C3VhNvfw.js';
+import { A as AnyDocument, b as ScoringMode, S as SchemaDefinition, I as IndexConfig } from './schema-DCgkO1j4.js';
 
 declare const ErrorCodes: {
     readonly SCHEMA_INVALID_TYPE: "SCHEMA_INVALID_TYPE";
@@ -156,6 +156,15 @@ interface QueryParams {
     termMatch?: TermMatchPolicy;
     tolerance?: number;
     prefixLength?: number;
+    /**
+     * Treat the last query token as an unfinished word so it also matches
+     * indexed terms that complete it ("secur" matches "security"). Earlier
+     * tokens must match fully; `tolerance` keeps applying to them but not to
+     * the prefix token. Completions score against a shared document frequency
+     * and are demoted below full-word matches. Ignored when `exact` is true.
+     * Off by default.
+     */
+    prefix?: boolean;
     exact?: boolean;
     facets?: FacetConfig;
     sort?: Record<string, 'asc' | 'desc'>;

package/dist/{direct-executor-6FOKJKLD.mjs → direct-executor-E52FFGDE.mjs}

@@ -1,4 +1,4 @@
-export { createDirectExecutor } from './chunk-Q4OVNESD.mjs';
+export { createDirectExecutor } from './chunk-HMO6OMZ3.mjs';
 import './chunk-S7RJJMHI.mjs';
 import './chunk-TW6BXTOL.mjs';
 import './chunk-ALB35PRQ.mjs';

package/dist/distribution/coordinator/etcd.d.ts

@@ -1,5 +1,5 @@
-import { C as ClusterCoordinator } from '../../types-CTVVm4GG.js';
-import '../../schema-C3VhNvfw.js';
+import { C as ClusterCoordinator } from '../../types-DHX6zGeD.js';
+import '../../schema-DCgkO1j4.js';
 import '../../adapters-Bo2JQg8b.js';
 
 interface EtcdCoordinatorConfig {

package/dist/distribution/coordinator/in-memory.d.ts

@@ -1,5 +1,5 @@
-import { C as ClusterCoordinator } from '../../types-CTVVm4GG.js';
-import '../../schema-C3VhNvfw.js';
+import { C as ClusterCoordinator } from '../../types-DHX6zGeD.js';
+import '../../schema-DCgkO1j4.js';
 import '../../adapters-Bo2JQg8b.js';
 
 declare function createInMemoryCoordinator(): ClusterCoordinator;

package/dist/distribution.d.ts

@@ -1,7 +1,7 @@
-import { C as ClusterCoordinator, N as NodeRegistration, A as AllocationTable, a as NodeRole, b as NodeCapacity } from './types-CTVVm4GG.js';
-export { c as AllocationConstraints, d as AllocationEvent, e as NodeEvent, P as PartitionAssignment, f as PartitionState, S as SchemaEvent } from './types-CTVVm4GG.js';
-import { B as BatchResult, Q as QueryParams, u as QueryResult, N as NarsilConfig } from './config-DX--2Owg.js';
-import { I as IndexConfig, A as AnyDocument } from './schema-C3VhNvfw.js';
+import { C as ClusterCoordinator, N as NodeRegistration, A as AllocationTable, a as NodeRole, b as NodeCapacity } from './types-DHX6zGeD.js';
+export { c as AllocationConstraints, d as AllocationEvent, e as NodeEvent, P as PartitionAssignment, f as PartitionState, S as SchemaEvent } from './types-DHX6zGeD.js';
+import { B as BatchResult, Q as QueryParams, u as QueryResult, N as NarsilConfig } from './config-CojEPOgS.js';
+import { I as IndexConfig, A as AnyDocument } from './schema-DCgkO1j4.js';
 export { createInMemoryCoordinator } from './distribution/coordinator/in-memory.js';
 import './adapters-Bo2JQg8b.js';
 

package/dist/distribution.mjs

@@ -1,7 +1,7 @@
 export { createInMemoryCoordinator } from './chunk-RQLXC2YS.mjs';
 import './chunk-WISGW24C.mjs';
-import { ReplicationMessageTypes, ClusterMessageTypes, DEFAULT_TRANSPORT_CONFIG, TransportError, TransportErrorCodes, MAX_MESSAGE_SIZE_BYTES, createEngineCore, createNarsilFromCore, QueryMessageTypes, applyIndexEntry, applyDeleteEntry, validateEntryPayload, createAckMessage, validateReplicationEntry, MAX_PARTITION_COUNT, MAX_INDEX_NAME_LENGTH, validateIndexName, computeEntryChecksum, createInsyncConfirmMessage, createEntryMessage, getIndexMetadata, validateAckPayload, createInsyncRemoveMessage, validateInsyncConfirmPayload, putIndexMetadata, createForwardMessage } from './chunk-CU6BVZEU.mjs';
-import { deserializePayloadV2, createPartitionIndex, getLanguage, fnv1a, mergePartitionStats, MAX_NESTING_DEPTH } from './chunk-Q4OVNESD.mjs';
+import { ReplicationMessageTypes, ClusterMessageTypes, DEFAULT_TRANSPORT_CONFIG, TransportError, TransportErrorCodes, MAX_MESSAGE_SIZE_BYTES, createEngineCore, createNarsilFromCore, QueryMessageTypes, applyIndexEntry, applyDeleteEntry, validateEntryPayload, createAckMessage, validateReplicationEntry, MAX_PARTITION_COUNT, MAX_INDEX_NAME_LENGTH, validateIndexName, computeEntryChecksum, createInsyncConfirmMessage, createEntryMessage, getIndexMetadata, validateAckPayload, createInsyncRemoveMessage, validateInsyncConfirmPayload, putIndexMetadata, createForwardMessage } from './chunk-MEW7HD75.mjs';
+import { deserializePayloadV2, createPartitionIndex, getLanguage, fnv1a, mergePartitionStats, MAX_NESTING_DEPTH } from './chunk-HMO6OMZ3.mjs';
 import './chunk-S7RJJMHI.mjs';
 import { crc32 } from './chunk-KMMJMDUA.mjs';
 import './chunk-TW6BXTOL.mjs';
@@ -5057,7 +5057,8 @@ async function restoreReplicationPartition(core, engine, indexName, partitionId,
     });
   }
   validatePartitionSnapshotPayload(indexName, partitionId, partitionCount, schema, partition);
-  createPartitionIndex(partitionId).deserialize(partition, schema);
+  const trackPositions = core.indexRegistry.get(indexName)?.config.trackPositions ?? true;
+  createPartitionIndex(partitionId, trackPositions).deserialize(partition, schema);
   const restoreIndex = await ensureReplicationIndex(core, engine, indexName, schema, partition.language, partitionCount);
   if (restoreIndex.created) {
     await core.orchestrator.replicateToWorkers({