@delali/narsil 0.3.1-next.87 → 0.3.1-next.88

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (29) hide show
  1. package/dist/browser/{chunk-BBDAYUCO.mjs → chunk-3GZQKGVQ.mjs} +25 -9
  2. package/dist/browser/{chunk-3BXQPIXK.mjs → chunk-JACCR3BJ.mjs} +3 -3
  3. package/dist/browser/{chunk-TBMNPGFO.mjs → chunk-RAHUQOJL.mjs} +2074 -881
  4. package/dist/browser/{durability-wiring-7RUN7LNS.mjs → durability-wiring-VSUBLUDK.mjs} +3 -3
  5. package/dist/browser/{snapshot-A42DRERF.mjs → snapshot-5MXBEDXF.mjs} +3 -3
  6. package/dist/browser/{write-4U6Q26TJ.mjs → write-6SBYTS62.mjs} +2 -2
  7. package/dist/{chunk-PSUJCOSE.mjs → chunk-A3UB2MXL.mjs} +172 -122
  8. package/dist/{chunk-JFJBA3PD.mjs → chunk-CL4OM7GB.mjs} +3 -3
  9. package/dist/{chunk-AXDAS2VA.mjs → chunk-DCK4P5BI.mjs} +28 -30
  10. package/dist/{chunk-XBXKPWMR.mjs → chunk-EMXAYPTQ.mjs} +1735 -651
  11. package/dist/{chunk-IKST36MA.mjs → chunk-MWJIAREY.mjs} +4 -4
  12. package/dist/{chunk-T7DMUVTL.mjs → chunk-QAMUC2BJ.mjs} +27 -11
  13. package/dist/{chunk-AQU2QHSY.mjs → chunk-R6BMMQXH.mjs} +114 -45
  14. package/dist/{chunk-K4OWQ2M3.mjs → chunk-S4LQDSUL.mjs} +3 -3
  15. package/dist/{chunk-MKK4OZ5W.mjs → chunk-V7YUSKKZ.mjs} +3 -3
  16. package/dist/{chunk-WQSHDZ7Z.mjs → chunk-VY4DXZ5P.mjs} +1 -1
  17. package/dist/{chunk-SYRFSF6V.mjs → chunk-X4SEXL4U.mjs} +49 -11
  18. package/dist/{direct-executor-LD6JW57R.mjs → direct-executor-EGESD26S.mjs} +3 -3
  19. package/dist/distribution.mjs +14 -27
  20. package/dist/{durability-wiring-5S32KDDC.mjs → durability-wiring-ZLRQU6FG.mjs} +10 -10
  21. package/dist/index.browser.mjs +1 -1
  22. package/dist/index.mjs +5 -5
  23. package/dist/partial-words.mjs +1 -1
  24. package/dist/persistence/durability/checkpoint-worker.mjs +8 -8
  25. package/dist/{snapshot-6BYSUPWL.mjs → snapshot-22CUEYYW.mjs} +10 -10
  26. package/dist/{thread-GS65E6RM.mjs → thread-ZP3UMIZT.mjs} +3 -3
  27. package/dist/workers/entry.mjs +3 -3
  28. package/dist/{write-HLEZTL2L.mjs → write-PXDKPG2R.mjs} +6 -6
  29. package/package.json +13 -8
@@ -1,5 +1,5 @@
1
- import { KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, restoreVectorFields, KEYWORD_SEGMENT_PAYLOAD_VERSION, littleEndianBytes } from './chunk-3BXQPIXK.mjs';
2
- import { VERSION, flattenSchema, packChecksummedEnvelope, openChecksummedEnvelope, requireValidFieldTypes, isCompositePartition, crc32, reconstructSchemaFromMetadata, extractVectorFieldsFromSchema, readHeader, HEADER_SIZE, engineVersionParts, writeHeader, partialFieldsOf, getLanguage, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, VECTOR_PATTERN, isQuantizationMode, isStorageMode, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createFrozenSegment, resolvePartitionInsertOptions, resolveIndexAnalysis, createPartitionManager, createPartitionRouter, buildSegmentPayload, fnv1a, selectionKeeps, decodeMapWithoutFields, prepareDocumentVectors, messagePackUpperBoundBytes, segmentArraysOf, mergeSegmentContents } from './chunk-TBMNPGFO.mjs';
1
+ import { KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, restoreVectorFields, KEYWORD_SEGMENT_PAYLOAD_VERSION, littleEndianBytes } from './chunk-JACCR3BJ.mjs';
2
+ import { VERSION, flattenSchema, packChecksummedEnvelope, openChecksummedEnvelope, requireValidFieldTypes, isCompositePartition, crc32, reconstructSchemaFromMetadata, extractVectorFieldsFromSchema, readHeader, HEADER_SIZE, engineVersionParts, writeHeader, partialFieldsOf, getLanguage, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, VECTOR_PATTERN, isQuantizationMode, isStorageMode, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, fnv1a, selectionKeeps, decodeMapWithoutFields, prepareDocumentVectors, messagePackUpperBoundBytes, segmentArraysOf, mergeSegmentContents, resolveIndexAnalysis } from './chunk-RAHUQOJL.mjs';
3
3
  import { decodeVectorGraphPartPayload, assembleGraphParts, decodeVectorFilePayload, countDead, VECTOR_FILE_MAX_VECTORS, deadBytesFor } from './chunk-NQW2YXEX.mjs';
4
4
  import { compareCodePoints, NarsilError, ErrorCodes, MAX_FILE_BYTES, VALID_HNSW_METRICS } from './chunk-FHFFWLEK.mjs';
5
5
  import { decode, encode } from '@msgpack/msgpack';
@@ -2176,7 +2176,7 @@ function noSpellingsOf(analysis) {
2176
2176
  return { fields: [...analysis.fields], tokens: [], spellings: [], counts: [] };
2177
2177
  }
2178
2178
  function contentsOfGroup(group, request, tombstones) {
2179
- const { schema, partialWordAnalysis } = request;
2179
+ const { schema, partialWordAnalysis, textAnalysis } = request;
2180
2180
  if (group === null) {
2181
2181
  const payload = { ...emptyPayload(), partialSpellings: noSpellingsOf(partialWordAnalysis) };
2182
2182
  return {
@@ -2185,7 +2185,7 @@ function contentsOfGroup(group, request, tombstones) {
2185
2185
  arrays: segmentArraysOf(payload, { blob: new Uint8Array(0), offsets: new Uint32Array(1) })
2186
2186
  };
2187
2187
  }
2188
- const merged = mergeSegmentContents(group.selections, partialWordAnalysis);
2188
+ const merged = mergeSegmentContents(group.selections, partialWordAnalysis, textAnalysis);
2189
2189
  return {
2190
2190
  schema,
2191
2191
  tombstones,
@@ -2320,6 +2320,7 @@ async function compactPartitionSegments(input) {
2320
2320
  nextSegmentId: input.nextSegmentId,
2321
2321
  schema: input.schema,
2322
2322
  partialWordAnalysis: input.partialWordAnalysis,
2323
+ textAnalysis: input.textAnalysis,
2323
2324
  selections: frozen.map((segment) => ({ segment })),
2324
2325
  tombstones: retainedTombstones(frozen, contents),
2325
2326
  mergesIntoOneFile: true
@@ -2335,9 +2336,7 @@ async function compactPartitionSegments(input) {
2335
2336
  }
2336
2337
 
2337
2338
  // src/persistence/durability/segment/write.ts
2338
- function partialWordAnalysisOf(config) {
2339
- const fields = partialFieldsOf(config.schema);
2340
- if (fields.length === 0) return null;
2339
+ function textAnalysisOf(config) {
2341
2340
  let resolved = null;
2342
2341
  const resolve = () => {
2343
2342
  resolved ??= {
@@ -2347,7 +2346,6 @@ function partialWordAnalysisOf(config) {
2347
2346
  return resolved;
2348
2347
  };
2349
2348
  return {
2350
- fields,
2351
2349
  get language() {
2352
2350
  return resolve().language;
2353
2351
  },
@@ -2356,6 +2354,19 @@ function partialWordAnalysisOf(config) {
2356
2354
  }
2357
2355
  };
2358
2356
  }
2357
+ function partialWordAnalysisOf(config, text) {
2358
+ const fields = partialFieldsOf(config.schema);
2359
+ if (fields.length === 0) return null;
2360
+ return {
2361
+ fields,
2362
+ get language() {
2363
+ return text.language;
2364
+ },
2365
+ get options() {
2366
+ return text.options;
2367
+ }
2368
+ };
2369
+ }
2359
2370
  function resolveCompactionThreshold(configured) {
2360
2371
  if (!Number.isInteger(configured) || configured <= 0 || configured > MAX_SEGMENTS_PER_PARTITION) {
2361
2372
  return DEFAULT_COMPACTION_THRESHOLD;
@@ -2376,6 +2387,7 @@ async function writeWholePartition(context, partitionId, priorPartition, whole)
2376
2387
  nextSegmentId: priorPartition?.nextSegmentId ?? 0,
2377
2388
  schema: context.input.metadata.schema,
2378
2389
  partialWordAnalysis: context.partialWordAnalysis,
2390
+ textAnalysis: context.textAnalysis,
2379
2391
  selections: whole.selections,
2380
2392
  tombstones: []
2381
2393
  });
@@ -2399,6 +2411,7 @@ async function writeLoggedChanges(context, partitionId, priorPartition, entries)
2399
2411
  nextSegmentId,
2400
2412
  schema: context.input.metadata.schema,
2401
2413
  partialWordAnalysis: context.partialWordAnalysis,
2414
+ textAnalysis: context.textAnalysis,
2402
2415
  selections: built.selections,
2403
2416
  tombstones: built.tombstones
2404
2417
  });
@@ -2418,6 +2431,7 @@ async function writeLoggedChanges(context, partitionId, priorPartition, entries)
2418
2431
  partitionId,
2419
2432
  schema: context.input.metadata.schema,
2420
2433
  partialWordAnalysis: context.partialWordAnalysis,
2434
+ textAnalysis: context.textAnalysis,
2421
2435
  segments,
2422
2436
  nextSegmentId,
2423
2437
  compactionThreshold: context.compactionThreshold
@@ -2440,11 +2454,13 @@ function carryForwardUncheckpointedPartitions(prior, targets, partitions, checkp
2440
2454
  }
2441
2455
  async function writeCheckpointSegments(input) {
2442
2456
  const config = reconstructSchemaFromMetadata(input.metadata);
2457
+ const textAnalysis = textAnalysisOf(config);
2443
2458
  const context = {
2444
2459
  input,
2445
2460
  indexName: input.metadata.indexName,
2446
2461
  config,
2447
- partialWordAnalysis: partialWordAnalysisOf(config),
2462
+ partialWordAnalysis: partialWordAnalysisOf(config, textAnalysis),
2463
+ textAnalysis,
2448
2464
  vectorFieldPaths: new Set(extractVectorFieldsFromSchema(config.schema).keys()),
2449
2465
  compactionThreshold: resolveCompactionThreshold(input.compactionThreshold)
2450
2466
  };
@@ -1,4 +1,4 @@
1
- import { removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors, getNestedValue, requireValidFieldTypes, VECTOR_PATTERN } from './chunk-TBMNPGFO.mjs';
1
+ import { removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors, getNestedValue, requireValidFieldTypes, VECTOR_PATTERN } from './chunk-RAHUQOJL.mjs';
2
2
  import { NarsilError, ErrorCodes } from './chunk-FHFFWLEK.mjs';
3
3
  import { decode } from '@msgpack/msgpack';
4
4
 
@@ -28,12 +28,12 @@ function applyIndexEntry(entry, manager, vectorFieldPaths, vecIndexes) {
28
28
  }
29
29
  }
30
30
  }
31
- function applyDeleteEntry(entry, manager, vecIndexes) {
31
+ function applyDeleteEntry(entry, manager, vecIndexes, removedSpellings) {
32
32
  removeDocumentVectors(entry.documentId, vecIndexes);
33
33
  if (!manager.has(entry.documentId)) {
34
34
  return;
35
35
  }
36
- manager.remove(entry.documentId);
36
+ manager.remove(entry.documentId, removedSpellings);
37
37
  }
38
38
  function restoreVectorFields(document, vectorFieldPaths) {
39
39
  for (const fieldPath of vectorFieldPaths) {