@delali/narsil 0.3.1-next.77 → 0.3.1-next.78

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,15 +1,15 @@
1
- import { validateWorkerConfig, validateIndexName, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateBM25Params, validatePartitionConfig, executeQuery, queryBindingOf, executePreflight, executeSuggest, executeListDocuments, validateDocId } from './chunk-G7IP3UDZ.mjs';
1
+ import { validateWorkerConfig, validateIndexName, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateBM25Params, validatePartitionConfig, executeQuery, queryBindingOf, executePreflight, executeSuggest, executeListDocuments, validateDocId } from './chunk-R6WBLBTC.mjs';
2
2
  import { ReplicationMessageTypes } from './chunk-GCM3JFVA.mjs';
3
- import { WIRE_BATCH_BUDGET, resolveVectorText, bindEngineCore, restoredConfigFields, restoredEmbedding, collectRequiredFieldFailures, embedChunkDocuments, providedDocId, requireDocumentObject, admitInsert, asBatchInsertError, requireMatchingDocId, embedDocumentFields, assertDocumentCarriesMappedVectors, admitBufferedUpsert, createPendingAdmissions, recordPendingAdmission } from './chunk-BT62MUM5.mjs';
4
- import { NATIVE_CRC32_MIN_BYTES, generateId, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, splitWorkerBudget, resolveWorkerCount, createDirectExecutor, createPartitionRouter, requirePartialWordSearchFor, MAX_FILE_BYTES, resolveWorkerEntry, missingWorkerEntry, readHeapStatistics, nodeThreadId, CHECKSUM_YIELD_CHUNK_BYTES, createPartitionManager, buildSegmentPayload, resolvePartitionInsertOptions, createFrozenSegment, resolveRequestThreadCount, vectorsPerFile, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, deadBitsWhere, NO_LIVE_VECTOR_AT_POSITION, createPartitionIndex, selectionKeeps, decodeMapWithoutFields, isCompositePartition, segmentArraysOf, mergeSegmentContents, survivorDocumentsOf, partitionStateOf, VECTOR_FILE_MAX_VECTORS, deadBytesFor, resolveIndexAnalysis, mergeFrozenSegments, workerResourceLimits, decodeVectorIndexParts, bytesToVectors, freezeSegmentShared, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, SHUTDOWN_TIMEOUT_MS, decodeVectorFilePayload, decodeVectorGraphPartPayload, assembleGraphParts, countDead, encodeSegmentState, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, VALID_HNSW_METRICS, spawnNodeWorker, decodeVectorGraphPayload, graphPartsOfEarlierPayload, decodeStoredDocument, isDead } from './chunk-DFFQOA7S.mjs';
5
- import { requirePatternSearchFor, collectQueryTermStats, mergePartitionStats, oversampledFacetConfig, fnv1a, availablePatternSearch, kWayMerge, mergeFacets, PATTERN_HOST, setNestedValue } from './chunk-D4EF7J3I.mjs';
3
+ import { WIRE_BATCH_BUDGET, resolveVectorText, bindEngineCore, restoredConfigFields, restoredEmbedding, collectRequiredFieldFailures, embedChunkDocuments, providedDocId, requireDocumentObject, admitInsert, asBatchInsertError, requireMatchingDocId, embedDocumentFields, assertDocumentCarriesMappedVectors, admitBufferedUpsert, createPendingAdmissions, recordPendingAdmission } from './chunk-XSZTF4ZQ.mjs';
4
+ import { NATIVE_CRC32_MIN_BYTES, generateId, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, splitWorkerBudget, resolveWorkerCount, createDirectExecutor, createPartitionRouter, requirePartialWordSearchFor, MAX_FILE_BYTES, resolveWorkerEntry, missingWorkerEntry, readHeapStatistics, nodeThreadId, CHECKSUM_YIELD_CHUNK_BYTES, resolvePartitionInsertOptions, resolveIndexAnalysis, createPartitionManager, buildSegmentPayload, createFrozenSegment, resolveRequestThreadCount, vectorsPerFile, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, deadBitsWhere, NO_LIVE_VECTOR_AT_POSITION, createPartitionIndex, selectionKeeps, decodeMapWithoutFields, isCompositePartition, segmentArraysOf, mergeSegmentContents, survivorDocumentsOf, partitionStateOf, VECTOR_FILE_MAX_VECTORS, deadBytesFor, mergeFrozenSegments, workerResourceLimits, decodeVectorIndexParts, bytesToVectors, freezeSegmentShared, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, SHUTDOWN_TIMEOUT_MS, decodeVectorFilePayload, decodeVectorGraphPartPayload, assembleGraphParts, countDead, encodeSegmentState, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, VALID_HNSW_METRICS, spawnNodeWorker, decodeVectorGraphPayload, graphPartsOfEarlierPayload, decodeStoredDocument, isDead } from './chunk-DFW3U5UW.mjs';
5
+ import { requirePatternSearchFor, collectQueryTermStats, mergePartitionStats, oversampledFacetConfig, fnv1a, availablePatternSearch, kWayMerge, mergeFacets, PATTERN_HOST, setNestedValue } from './chunk-P2BEMMOD.mjs';
6
6
  import { detectRuntime, getLanguage } from './chunk-4MQ5F4IL.mjs';
7
- import { MAX_INDEX_CONFIGURATION_BYTES, STATISTICS_BROADCAST_INTERVAL_MS, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, HEAP_PRESSURE_WARNING_FRACTION, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, FOREIGN_STATS_TTL_MS, MAX_IMPORT_REFUSALS_REPORTED, BATCH_CHUNK_SIZE, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION, EXPORT_CHUNK_BYTES } from './chunk-V3B24XR7.mjs';
7
+ import { MAX_INDEX_CONFIGURATION_BYTES, STATISTICS_BROADCAST_INTERVAL_MS, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, HEAP_PRESSURE_WARNING_FRACTION, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, FOREIGN_STATS_TTL_MS, MAX_IMPORT_REFUSALS_REPORTED, BATCH_CHUNK_SIZE, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION, EXPORT_CHUNK_BYTES } from './chunk-IOZPBNSG.mjs';
8
8
  import { releaseLocksHeldBy } from './chunk-AMGDG4ZJ.mjs';
9
9
  import { requireSearchBackendSettings } from './chunk-2E35UM3G.mjs';
10
10
  import { DEFAULT_PATTERN_WORK_CAP, MATCHER_STATE_BUDGET_BYTES, PATTERN_TEST_OPERATORS, PATTERN_EXPRESSION_OPERATORS, MATCHER_CODE_POINT_BYTES, MATCHER_SIMULATION_BYTES_PER_POSITION, MATCHER_STATE_OVERHEAD_BYTES } from './chunk-4344I4MR.mjs';
11
- import { getNestedValue, parseVectorDimension } from './chunk-NO3ABFLE.mjs';
12
- import { extractVectorFieldsFromSchema, messagePackUpperBoundBytes, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, raiseEarlierReleaseGraphSetting, flattenSchema, isSingleStringFieldType, requireValidFieldTypes, encodeDocument, VECTOR_PATTERN, schemaFieldsOf, MAX_DOCUMENT_NESTING_DEPTH, isQuantizationMode, isStorageMode, validateRequiredFields, requireDocumentWithinSizeLimit, validateDocument, validateDocumentStrict } from './chunk-356UOTK2.mjs';
11
+ import { getNestedValue, parseVectorDimension } from './chunk-WFVOBZS4.mjs';
12
+ import { extractVectorFieldsFromSchema, messagePackUpperBoundBytes, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, raiseEarlierReleaseGraphSetting, partialFieldsOf, flattenSchema, isSingleStringFieldType, requireValidFieldTypes, encodeDocument, VECTOR_PATTERN, schemaFieldsOf, MAX_DOCUMENT_NESTING_DEPTH, isQuantizationMode, isStorageMode, validateRequiredFields, requireDocumentWithinSizeLimit, validateDocument, validateDocumentStrict } from './chunk-PXDQFYD5.mjs';
13
13
  import { MIN_M } from './chunk-KEHK2O6P.mjs';
14
14
  import { compareCodePoints, foldedCodePointCount, codePointCount } from './chunk-IBR6634K.mjs';
15
15
  import { createRequestId, isValidWorkerResponse, threadSlotOfWorker } from './chunk-POWOTUKJ.mjs';
@@ -3505,7 +3505,7 @@ function selectionOfMask(segment, keep) {
3505
3505
  }
3506
3506
  function frozenLiveTail(live, shared) {
3507
3507
  if (live.count() === 0) return null;
3508
- const payload = { ...live.encodeSegment(), partialSpellings: null };
3508
+ const payload = live.encodeSegment();
3509
3509
  const documents = survivorDocumentsOf(partitionStateOf(live), payload);
3510
3510
  if (shared) {
3511
3511
  const snapshot = freezeSegmentShared(payload, documents);
@@ -4325,6 +4325,23 @@ function requireEntriesAscend(blob, offsets, order, name) {
4325
4325
  }
4326
4326
  }
4327
4327
  }
4328
+ function requireSpellingEntriesAscend(blob, offsets, tokens, name) {
4329
+ for (let at = 1; at < tokens.length; at++) {
4330
+ const order = compareEntries(blob, offsets, at - 1, at) || tokens[at - 1] - tokens[at];
4331
+ if (order >= 0) corrupt2(`${name} fails to ascend strictly at entry ${at}`, { field: name, at });
4332
+ }
4333
+ }
4334
+ function requireBelow(values, limit, name) {
4335
+ for (let at = 0; at < values.length; at++) {
4336
+ if (values[at] >= limit)
4337
+ corrupt2(`${name} holds ${values[at]} at entry ${at}, at or above ${limit}`, { field: name, at });
4338
+ }
4339
+ }
4340
+ function requireNonZero(values, name) {
4341
+ for (let at = 0; at < values.length; at++) {
4342
+ if (values[at] === 0) corrupt2(`${name} holds 0 at entry ${at}`, { field: name, at });
4343
+ }
4344
+ }
4328
4345
  function requirePermutation(order, name) {
4329
4346
  const seen = new Uint8Array(order.length);
4330
4347
  for (let at = 0; at < order.length; at++) {
@@ -4464,6 +4481,34 @@ function decodeSurfaceForms(raw, context) {
4464
4481
  requireUtf8Entries(blob, offsets, "surfaceForms.text");
4465
4482
  return counts.length === 0 ? null : { blob, offsets, counts };
4466
4483
  }
4484
+ function decodePartialWords(raw, tokenCount, context) {
4485
+ if (raw === null || raw === void 0) return null;
4486
+ const column = requireRecord(raw, "partialWords");
4487
+ const fields = requireStrings(column.fields, "partialWords.fields");
4488
+ const spellings = bytesFrom(requireBytes(column.text, "partialWords.text", 1), context.shared);
4489
+ const spellingOffsets = uint32Column(column, "offsets", "partialWords", context);
4490
+ const spellingTokens = uint32Column(column, "tokens", "partialWords", context);
4491
+ const spellingCounts = uint32Column(column, "counts", "partialWords", context);
4492
+ const entries = spellingTokens.length;
4493
+ requireCount(spellingOffsets.length, entries + 1, "partialWords.offsets");
4494
+ requireCount(spellingCounts.length, entries, "partialWords.counts");
4495
+ requireOffsets(spellingOffsets, spellings.length, "partialWords.offsets");
4496
+ requireUtf8Entries(spellings, spellingOffsets, "partialWords.text");
4497
+ requireBelow(spellingTokens, tokenCount, "partialWords.tokens");
4498
+ requireNonZero(spellingCounts, "partialWords.counts");
4499
+ requireSpellingEntriesAscend(spellings, spellingOffsets, spellingTokens, "partialWords");
4500
+ const runs = uint32sFrom(requireBytes(column.runs, "partialWords.runs", 4 * CODE_POINTS_PER_RUN), context.shared);
4501
+ requireRunsAscend(runs, "partialWords.runs");
4502
+ const runOffsets = uint32Column(column, "runOffsets", "partialWords", context);
4503
+ const runEntries = uint32Column(column, "runEntries", "partialWords", context);
4504
+ requireCount(runOffsets.length, runs.length / CODE_POINTS_PER_RUN + 1, "partialWords.runOffsets");
4505
+ requireOffsets(runOffsets, runEntries.length, "partialWords.runOffsets");
4506
+ requireBelow(runEntries, entries, "partialWords.runEntries");
4507
+ for (let run = 0; run + 1 < runOffsets.length; run++) {
4508
+ requireStrictlyAscending(runEntries, runOffsets[run], runOffsets[run + 1], "partialWords.runEntries");
4509
+ }
4510
+ return { fields, spellings, spellingOffsets, spellingTokens, spellingCounts, runs, runOffsets, runEntries };
4511
+ }
4467
4512
 
4468
4513
  // src/serialization/keyword-segment/types.ts
4469
4514
  var KEYWORD_SEGMENT_PAYLOAD_VERSION = 1;
@@ -4584,12 +4629,13 @@ function decodeKeywordSegment(payload, shared) {
4584
4629
  corrupt2(`fieldNames holds ${fieldNames.length} names, above the limit of ${MAX_SEGMENT_FIELD_NAMES}`);
4585
4630
  }
4586
4631
  const context = { documentCount, shared };
4632
+ const postings = decodePostings(raw, fieldNames, context);
4587
4633
  const arrays = {
4588
4634
  documentCount,
4589
4635
  fieldNames,
4590
4636
  idTable: decodeIds(raw, context),
4591
4637
  documentTable: decodeDocuments(raw, schema, context),
4592
- ...decodePostings(raw, fieldNames, context),
4638
+ ...postings,
4593
4639
  ...decodeFieldLengths(raw.fieldLengths, context),
4594
4640
  numeric: decodeValueColumns(raw.numeric, "numeric", context),
4595
4641
  sizes: decodeValueColumns(raw.sizes, "sizes", context),
@@ -4597,7 +4643,8 @@ function decodeKeywordSegment(payload, shared) {
4597
4643
  enums: decodeEnumColumns(raw.enum, context),
4598
4644
  geo: decodeGeopointColumns(raw.geopoint, context),
4599
4645
  pattern: decodePatternColumns(raw.pattern, context),
4600
- surfaceTable: decodeSurfaceForms(raw.surfaceForms, context)
4646
+ surfaceTable: decodeSurfaceForms(raw.surfaceForms, context),
4647
+ partialWords: decodePartialWords(raw.partialWords, postings.tokenTable.offsets.length - 1, context)
4601
4648
  };
4602
4649
  return { schema, tombstones, arrays };
4603
4650
  }
@@ -4721,7 +4768,7 @@ function postingsInTokenOrder(arrays) {
4721
4768
  }
4722
4769
 
4723
4770
  // src/serialization/keyword-segment/encode.ts
4724
- var TOP_LEVEL_ENTRIES = 27;
4771
+ var TOP_LEVEL_ENTRIES = 28;
4725
4772
  function writeFieldLengths(out, arrays) {
4726
4773
  out.array(arrays.fieldLengthNames.length);
4727
4774
  for (let f = 0; f < arrays.fieldLengthNames.length; f++) {
@@ -4783,6 +4830,21 @@ function writeSurfaceForms(out, arrays) {
4783
4830
  out.key("offsets").binary(littleEndianBytes(table.offsets));
4784
4831
  out.key("counts").binary(littleEndianBytes(table.counts));
4785
4832
  }
4833
+ function writePartialWords(out, arrays) {
4834
+ const table = arrays.partialWords;
4835
+ if (table === null) {
4836
+ out.value(null);
4837
+ return;
4838
+ }
4839
+ out.map(8).key("fields").value(table.fields);
4840
+ out.key("text").binary(table.spellings);
4841
+ out.key("offsets").binary(littleEndianBytes(table.spellingOffsets));
4842
+ out.key("tokens").binary(littleEndianBytes(table.spellingTokens));
4843
+ out.key("counts").binary(littleEndianBytes(table.spellingCounts));
4844
+ out.key("runs").binary(littleEndianBytes(table.runs));
4845
+ out.key("runOffsets").binary(littleEndianBytes(table.runOffsets));
4846
+ out.key("runEntries").binary(littleEndianBytes(table.runEntries));
4847
+ }
4786
4848
  function encodeKeywordSegment(contents) {
4787
4849
  const { arrays } = contents;
4788
4850
  const postings = postingsInTokenOrder(arrays);
@@ -4823,6 +4885,8 @@ function encodeKeywordSegment(contents) {
4823
4885
  writePatternColumns(out, arrays.pattern);
4824
4886
  out.key("surfaceForms");
4825
4887
  writeSurfaceForms(out, arrays);
4888
+ out.key("partialWords");
4889
+ writePartialWords(out, arrays);
4826
4890
  return out;
4827
4891
  }
4828
4892
 
@@ -4901,7 +4965,7 @@ function fixedBytesOf(segment, chargesTokens) {
4901
4965
  const surfaces = segment.sharedSnapshot?.surfaceTable ?? null;
4902
4966
  if (surfaces !== null) bytes += surfaces.blob.byteLength + surfaces.offsets.byteLength + surfaces.counts.byteLength;
4903
4967
  else bytes += messagePackUpperBoundBytes(arrays.readSurfaceForms());
4904
- return bytes;
4968
+ return bytes + segment.partialWords.bytes();
4905
4969
  }
4906
4970
  function keptDocumentsOf(selection, chargesTokens) {
4907
4971
  const { segment } = selection;
@@ -5030,16 +5094,26 @@ function emptyPayload() {
5030
5094
  sizes: []
5031
5095
  };
5032
5096
  }
5033
- function contentsOfGroup(group, schema, tombstones) {
5097
+ function noSpellingsOf(analysis) {
5098
+ if (analysis === null || analysis.fields.length === 0) return null;
5099
+ return { fields: [...analysis.fields], tokens: [], spellings: [], counts: [] };
5100
+ }
5101
+ function contentsOfGroup(group, request, tombstones) {
5102
+ const { schema, partialWordAnalysis } = request;
5034
5103
  if (group === null) {
5104
+ const payload = { ...emptyPayload(), partialSpellings: noSpellingsOf(partialWordAnalysis) };
5035
5105
  return {
5036
5106
  schema,
5037
5107
  tombstones,
5038
- arrays: segmentArraysOf(emptyPayload(), { blob: new Uint8Array(0), offsets: new Uint32Array(1) })
5108
+ arrays: segmentArraysOf(payload, { blob: new Uint8Array(0), offsets: new Uint32Array(1) })
5039
5109
  };
5040
5110
  }
5041
- const merged = mergeSegmentContents(group.selections);
5042
- return { schema, tombstones, arrays: segmentArraysOf(merged.payload, merged.documents) };
5111
+ const merged = mergeSegmentContents(group.selections, partialWordAnalysis);
5112
+ return {
5113
+ schema,
5114
+ tombstones,
5115
+ arrays: segmentArraysOf(merged.payload, merged.documents, merged.reusedPartialWords)
5116
+ };
5043
5117
  }
5044
5118
  function requireRoomForAnotherSegment(state) {
5045
5119
  if (state.segments.length < MAX_SEGMENTS_PER_PARTITION) return;
@@ -5052,7 +5126,7 @@ function requireRoomForAnotherSegment(state) {
5052
5126
  async function writeGroup(state, group) {
5053
5127
  const { request } = state;
5054
5128
  const tombstones = state.tombstonesWritten ? [] : [...request.tombstones];
5055
- const contents = contentsOfGroup(group, request.schema, tombstones);
5129
+ const contents = contentsOfGroup(group, request, tombstones);
5056
5130
  const encoded = encodeKeywordSegment(contents);
5057
5131
  if (HEADER_SIZE + encoded.byteLength > MAX_FILE_BYTES && group !== null && group.documentCount > 1) {
5058
5132
  const [first, second] = halvesOf(group);
@@ -5168,6 +5242,7 @@ async function compactPartitionSegments(input) {
5168
5242
  partitionId: input.partitionId,
5169
5243
  nextSegmentId: input.nextSegmentId,
5170
5244
  schema: input.schema,
5245
+ partialWordAnalysis: input.partialWordAnalysis,
5171
5246
  selections: frozen.map((segment) => ({ segment })),
5172
5247
  tombstones: retainedTombstones(frozen, contents),
5173
5248
  mergesIntoOneFile: true
@@ -5183,6 +5258,27 @@ async function compactPartitionSegments(input) {
5183
5258
  }
5184
5259
 
5185
5260
  // src/persistence/durability/segment/write.ts
5261
+ function partialWordAnalysisOf(config) {
5262
+ const fields = partialFieldsOf(config.schema);
5263
+ if (fields.length === 0) return null;
5264
+ let resolved = null;
5265
+ const resolve = () => {
5266
+ resolved ??= {
5267
+ language: getLanguage(config.language ?? "english"),
5268
+ options: resolvePartitionInsertOptions(config, resolveIndexAnalysis(config))
5269
+ };
5270
+ return resolved;
5271
+ };
5272
+ return {
5273
+ fields,
5274
+ get language() {
5275
+ return resolve().language;
5276
+ },
5277
+ get options() {
5278
+ return resolve().options;
5279
+ }
5280
+ };
5281
+ }
5186
5282
  function resolveCompactionThreshold(configured) {
5187
5283
  if (!Number.isInteger(configured) || configured <= 0 || configured > MAX_SEGMENTS_PER_PARTITION) {
5188
5284
  return DEFAULT_COMPACTION_THRESHOLD;
@@ -5202,6 +5298,7 @@ async function writeWholePartition(context, partitionId, priorPartition, whole)
5202
5298
  partitionId,
5203
5299
  nextSegmentId: priorPartition?.nextSegmentId ?? 0,
5204
5300
  schema: context.input.metadata.schema,
5301
+ partialWordAnalysis: context.partialWordAnalysis,
5205
5302
  selections: whole.selections,
5206
5303
  tombstones: []
5207
5304
  });
@@ -5224,6 +5321,7 @@ async function writeLoggedChanges(context, partitionId, priorPartition, entries)
5224
5321
  partitionId,
5225
5322
  nextSegmentId,
5226
5323
  schema: context.input.metadata.schema,
5324
+ partialWordAnalysis: context.partialWordAnalysis,
5227
5325
  selections: built.selections,
5228
5326
  tombstones: built.tombstones
5229
5327
  });
@@ -5242,6 +5340,7 @@ async function writeLoggedChanges(context, partitionId, priorPartition, entries)
5242
5340
  indexName: context.indexName,
5243
5341
  partitionId,
5244
5342
  schema: context.input.metadata.schema,
5343
+ partialWordAnalysis: context.partialWordAnalysis,
5245
5344
  segments,
5246
5345
  nextSegmentId,
5247
5346
  compactionThreshold: context.compactionThreshold
@@ -5268,6 +5367,7 @@ async function writeCheckpointSegments(input) {
5268
5367
  input,
5269
5368
  indexName: input.metadata.indexName,
5270
5369
  config,
5370
+ partialWordAnalysis: partialWordAnalysisOf(config),
5271
5371
  vectorFieldPaths: new Set(extractVectorFieldsFromSchema(config.schema).keys()),
5272
5372
  compactionThreshold: resolveCompactionThreshold(input.compactionThreshold)
5273
5373
  };