@delali/narsil 0.3.1-next.140 → 0.3.1-next.141

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. package/dist/{analysis-change-JN23AZQP.mjs → analysis-change-HQUT2Z3I.mjs} +13 -13
  2. package/dist/{analysis-change-entry-5JSIMEKG.mjs → analysis-change-entry-TZTG4AZB.mjs} +1 -1
  3. package/dist/browser/{analysis-change-57MIRJWR.mjs → analysis-change-JPV775LR.mjs} +407 -6
  4. package/dist/browser/{analysis-change-entry-ICGRR3IN.mjs → analysis-change-entry-7HR4LWL3.mjs} +1 -1
  5. package/dist/browser/chunk-3ZBJYAS4.mjs +19 -0
  6. package/dist/browser/{chunk-ZAIZQOTM.mjs → chunk-5DB2VDGI.mjs} +951 -1134
  7. package/dist/browser/chunk-6RHTAVZX.mjs +656 -0
  8. package/dist/browser/chunk-CCOAOO73.mjs +86 -0
  9. package/dist/browser/{chunk-3EYDURFD.mjs → chunk-HKX2G6WH.mjs} +1615 -2266
  10. package/dist/browser/{chunk-CGVBGHBX.mjs → chunk-KTOK275N.mjs} +531 -830
  11. package/dist/browser/{chunk-5LYWZJE2.mjs → chunk-LVY2LI35.mjs} +1 -1
  12. package/dist/browser/chunk-MVM33ZUL.mjs +993 -0
  13. package/dist/browser/chunk-MWUK3DA5.mjs +2634 -0
  14. package/dist/browser/{chunk-67KSBUYR.mjs → chunk-R4WUDZQV.mjs} +8 -5
  15. package/dist/browser/{chunk-6BSB4N6B.mjs → chunk-TCLEOZ4P.mjs} +2 -2
  16. package/dist/browser/chunk-VFYMPCOV.mjs +212 -0
  17. package/dist/browser/chunk-WCFCHIBB.mjs +177 -0
  18. package/dist/browser/{chunk-E5SFTOQR.mjs → chunk-XNWPP5NS.mjs} +2 -2
  19. package/dist/browser/{chunk-QHNNULPW.mjs → chunk-ZEV26AQ6.mjs} +5 -0
  20. package/dist/browser/cold-module-MJMKNUYJ.mjs +745 -0
  21. package/dist/browser/{durability-wiring-MTIAN6VT.mjs → durability-wiring-SNOWIT6Z.mjs} +231 -51
  22. package/dist/browser/{import-Q7MXD6BS.mjs → import-TLB2A5LJ.mjs} +6 -5
  23. package/dist/browser/module-BUFNZMEU.mjs +12 -0
  24. package/dist/browser/{module-5J3PEUEB.mjs → module-BVP4TKZU.mjs} +395 -108
  25. package/dist/{restore-queue-RYSRRFS4.mjs → browser/restore-queue-ONYNYUDE.mjs} +2 -2
  26. package/dist/browser/{snapshot-YBJ5MF2M.mjs → snapshot-QXBKHJAG.mjs} +108 -25
  27. package/dist/browser/{write-RTWI5MXL.mjs → write-WHFBDWQS.mjs} +63 -91
  28. package/dist/{capabilities-DXAmysqJ.d.ts → capabilities-BCdUwK_L.d.ts} +2 -2
  29. package/dist/{chunk-HMDT3I63.mjs → chunk-46NZOPHS.mjs} +2 -2
  30. package/dist/{chunk-XXKWEG53.mjs → chunk-5WYVUPGD.mjs} +1 -1
  31. package/dist/chunk-6SNBQESJ.mjs +214 -0
  32. package/dist/chunk-7AUASRE7.mjs +653 -0
  33. package/dist/{chunk-4GCN34JV.mjs → chunk-AST3SHFA.mjs} +26 -24
  34. package/dist/chunk-BOJZGUS6.mjs +177 -0
  35. package/dist/{chunk-IP4HJOGC.mjs → chunk-BRUUBXCB.mjs} +4 -3
  36. package/dist/{chunk-J3EHATEZ.mjs → chunk-DQNKVOJ2.mjs} +63 -63
  37. package/dist/chunk-FOCV4CBZ.mjs +87 -0
  38. package/dist/{chunk-FFVLPCLH.mjs → chunk-GLZHNSGB.mjs} +1 -1
  39. package/dist/chunk-GWOS5DYC.mjs +2637 -0
  40. package/dist/{chunk-W2XDK66Z.mjs → chunk-IO5F7ZBE.mjs} +112 -438
  41. package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
  42. package/dist/{chunk-LLDFIKGP.mjs → chunk-KEIYVZUX.mjs} +4 -4
  43. package/dist/{chunk-UBXJQJII.mjs → chunk-KGOXHLLZ.mjs} +5 -1
  44. package/dist/{chunk-FKARMUWH.mjs → chunk-KZL72VTC.mjs} +68 -16
  45. package/dist/{chunk-TCWQJL7O.mjs → chunk-L76W3IZJ.mjs} +5 -0
  46. package/dist/{chunk-CPCGEIOM.mjs → chunk-LA4WNDHL.mjs} +83 -799
  47. package/dist/{chunk-PNNVTKMT.mjs → chunk-NJJCPFXN.mjs} +2 -2
  48. package/dist/chunk-NN33CZRE.mjs +86 -0
  49. package/dist/chunk-NRUXBYPH.mjs +121 -0
  50. package/dist/{chunk-UJWGQIXU.mjs → chunk-QHPYS427.mjs} +1 -0
  51. package/dist/{chunk-VSUYTPME.mjs → chunk-RHKRTYGO.mjs} +13 -4
  52. package/dist/{chunk-UEV53BBO.mjs → chunk-SRJBFG2T.mjs} +406 -5
  53. package/dist/{chunk-K3AQOZVM.mjs → chunk-TNBIK2LQ.mjs} +1638 -1644
  54. package/dist/chunk-UUZCQQG2.mjs +209 -0
  55. package/dist/{chunk-XMQNOD5V.mjs → chunk-VEFN6X6Q.mjs} +2 -2
  56. package/dist/{chunk-LK3MOTXF.mjs → chunk-VJEGOELZ.mjs} +8 -5
  57. package/dist/chunk-W2COJ5AH.mjs +148 -0
  58. package/dist/{chunk-IBATGS3J.mjs → chunk-WJGO5PGV.mjs} +241 -124
  59. package/dist/{chunk-CCCGEPJM.mjs → chunk-XZ3SRNDJ.mjs} +24 -63
  60. package/dist/{chunk-MTQTX4BP.mjs → chunk-YHUTF6EA.mjs} +400 -92
  61. package/dist/{chunk-B5I3IL4H.mjs → chunk-YLNSJQLX.mjs} +4 -4
  62. package/dist/{chunk-6VHWXAOI.mjs → chunk-Z4HFMYNX.mjs} +29 -24
  63. package/dist/{chunk-U4X4AWKL.mjs → chunk-ZPWS2SKJ.mjs} +293 -270
  64. package/dist/{client-DApBeLbk.d.ts → client-B12YFALP.d.ts} +3 -3
  65. package/dist/client.d.ts +7 -7
  66. package/dist/cold-module-4X7WJCFM.mjs +783 -0
  67. package/dist/{config-B0H-vp4U.d.ts → config-BhZxIeb1.d.ts} +6 -1
  68. package/dist/{config-Brysl9NO.d.ts → config-CIG5yDyg.d.ts} +2 -2
  69. package/dist/{direct-executor-J5UDLPL3.mjs → direct-executor-7XOQ7VQP.mjs} +5 -5
  70. package/dist/distribution/coordinator/etcd.d.ts +2 -2
  71. package/dist/distribution/coordinator/in-memory.d.ts +2 -2
  72. package/dist/distribution.d.ts +7 -7
  73. package/dist/distribution.mjs +123 -42
  74. package/dist/{durability-wiring-AYOTAPNQ.mjs → durability-wiring-324D2DDD.mjs} +213 -40
  75. package/dist/{engine-CuLMbH1-.d.ts → engine-CtzbHOmG.d.ts} +2 -2
  76. package/dist/{import-QRZAQ5RH.mjs → import-OOBUW3RR.mjs} +14 -13
  77. package/dist/{index-HYnqS1SE.d.ts → index-DsEurhSl.d.ts} +1 -1
  78. package/dist/index.browser.mjs +3 -3
  79. package/dist/index.d.ts +6 -6
  80. package/dist/index.mjs +11 -11
  81. package/dist/{long-query-thread-U7K5A72G.mjs → long-query-thread-4SW7RWDE.mjs} +8 -8
  82. package/dist/{module-L2OZO5PQ.mjs → module-DXSJ5LT2.mjs} +374 -83
  83. package/dist/module-IYPLWHGC.mjs +48 -0
  84. package/dist/paged-partition-archive-HFP45PLU.mjs +269 -0
  85. package/dist/partial-words.mjs +2 -2
  86. package/dist/persistence/durability/checkpoint-worker.d.ts +5 -1
  87. package/dist/persistence/durability/checkpoint-worker.mjs +17 -15
  88. package/dist/react.d.ts +9 -9
  89. package/dist/{browser/restore-queue-DISFECE4.mjs → restore-queue-FEYAC4SB.mjs} +2 -2
  90. package/dist/{schema-BTXbHCqA.d.ts → schema-B-Runv0n.d.ts} +7 -0
  91. package/dist/server.d.ts +10 -10
  92. package/dist/server.mjs +8 -7
  93. package/dist/{snapshot-BTRZ3MZK.mjs → snapshot-CGM65VCD.mjs} +60 -25
  94. package/dist/{tasks-DjGjK70Y.d.ts → tasks-CjRJRGTb.d.ts} +6 -4
  95. package/dist/{thread-USB3GOPY.mjs → thread-KGNAIKPD.mjs} +8 -8
  96. package/dist/vector/search-worker.d.ts +1 -1
  97. package/dist/vector/search-worker.mjs +3 -2
  98. package/dist/workers/entry.mjs +7 -7
  99. package/dist/{write-GHTA56OW.mjs → write-2DZPEQL2.mjs} +70 -98
  100. package/openapi.json +13 -0
  101. package/package.json +12 -8
  102. package/dist/browser/chunk-XVMOQQ34.mjs +0 -384
  103. package/dist/chunk-BDIDUW2B.mjs +0 -387
  104. package/dist/browser/{chunk-4MFH226Q.mjs → chunk-D33A5VMU.mjs} +15 -15
@@ -1,7 +1,8 @@
1
- import { KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, restoreVectorFields, KEYWORD_SEGMENT_PAYLOAD_VERSION, littleEndianBytes, uint64Bytes } from './chunk-6BSB4N6B.mjs';
2
- import { decodeVectorGraphPartPayload, assembleGraphParts, decodeVectorFilePayload, countDead, VECTOR_FILE_MAX_VECTORS, deadBytesFor } from './chunk-5LYWZJE2.mjs';
3
- import { VERSION, DEFAULT_BM25_B, DEFAULT_BM25_K1, rulesOfStoredSynonyms, flattenSchema, packChecksummedEnvelope, openChecksummedEnvelope, requireValidFieldTypes, isCompositePartition, crc32, reconstructSchemaFromMetadata, extractVectorFieldsFromSchema, readHeader, HEADER_SIZE, engineVersionParts, writeHeader, partialFieldsOf, getLanguage, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, VECTOR_PATTERN, isQuantizationMode, isStorageMode, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, fnv1a, selectionKeeps, decodeMapWithoutFields, prepareDocumentVectors, messagePackUpperBoundBytes, segmentArraysOf, mergeSegmentContents, resolveIndexAnalysis } from './chunk-ZAIZQOTM.mjs';
4
- import { compareCodePoints, MAX_FILE_BYTES, VALID_HNSW_METRICS } from './chunk-4MFH226Q.mjs';
1
+ import { MAX_SEGMENT_ID, MANIFEST_ENVELOPE_VERSION, EARLIER_SEGMENT_MANIFEST_VERSION, SEGMENT_MANIFEST_VERSION, MAX_SEGMENTS_PER_PARTITION, segmentKey, encodeSegmentManifest, manifestKey, manifestReferencedKeys, segmentsPrefix, snapshotBundleKey, readListedFile, requireListedChecksum, vectorFileKey, DEFAULT_COMPACTION_THRESHOLD, writeSegmentsOf, readSegmentFile, graphPartKey, requireListedSize, SEGMENT_PAGE_BYTES, STABLE_HEAD_ATTEMPTS, RECORD_PAGE_BYTES, MERGED_SEGMENT_GROWTH_ALLOWANCE, MERGED_SEGMENT_TARGET_BYTES, COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT, SEGMENT_FILE_FRAMING_SLACK_BYTES } from './chunk-MVM33ZUL.mjs';
2
+ import { restoreVectorFields } from './chunk-TCLEOZ4P.mjs';
3
+ import { decodeVectorGraphPartPayload, assembleGraphParts, decodeVectorFilePayload, countDead, VECTOR_FILE_MAX_VECTORS, deadBytesFor } from './chunk-LVY2LI35.mjs';
4
+ import { VERSION, DEFAULT_BM25_B, DEFAULT_BM25_K1, rulesOfStoredSynonyms, flattenSchema, openChecksummedEnvelope, requireValidFieldTypes, isCompositePartition, crc32, reconstructSchemaFromMetadata, extractVectorFieldsFromSchema, readHeader, HEADER_SIZE, engineVersionParts, writeHeader, packChecksummedEnvelope, packEnvelopeWithChecksum, checksumOfChunks, partialFieldsOf, getLanguage, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, VECTOR_PATTERN, isQuantizationMode, isStorageMode, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, decodeMapWithoutFields, prepareDocumentVectors, resolveIndexAnalysis } from './chunk-5DB2VDGI.mjs';
5
+ import { compareCodePoints, MAX_FILE_BYTES, VALID_HNSW_METRICS } from './chunk-D33A5VMU.mjs';
5
6
  import { NarsilError, ErrorCodes } from './chunk-35B42UFM.mjs';
6
7
  import { decode, encode } from '@msgpack/msgpack';
7
8
 
@@ -57,6 +58,9 @@ function metadataFromConfig(indexName, config, partitionCount, documentCount) {
57
58
  if (config.trackPositions !== void 0) {
58
59
  metadata.trackPositions = config.trackPositions;
59
60
  }
61
+ if (config.textStorage !== void 0) {
62
+ metadata.textStorage = config.textStorage;
63
+ }
60
64
  if (config.strict !== void 0) {
61
65
  metadata.strict = config.strict;
62
66
  }
@@ -107,6 +111,7 @@ function metadataConfigOf(entry) {
107
111
  ...entry.config.partitions !== void 0 ? { partitionLimits: entry.config.partitions } : {},
108
112
  ...entry.config.defaultScoring !== void 0 ? { defaultScoring: entry.config.defaultScoring } : {},
109
113
  ...entry.config.trackPositions !== void 0 ? { trackPositions: entry.config.trackPositions } : {},
114
+ ...entry.config.textStorage !== void 0 ? { textStorage: entry.config.textStorage } : {},
110
115
  ...entry.config.strict !== void 0 ? { strict: entry.config.strict } : {},
111
116
  ...entry.config.required !== void 0 ? { required: entry.config.required } : {},
112
117
  ...entry.config.vectorPromotion !== void 0 ? { vectorPromotion: entry.config.vectorPromotion } : {},
@@ -150,26 +155,6 @@ function indexMetadataOf(indexName, entry, partitionCount) {
150
155
  return metadataFromConfig(indexName, metadataConfigOf(entry), partitionCount);
151
156
  }
152
157
 
153
- // src/persistence/durability/constants.ts
154
- var DEFAULT_CHECKPOINT_INTERVAL_MS = 3e5;
155
- var DEFAULT_CHECKPOINT_MUTATION_THRESHOLD = 1e5;
156
- var DEFAULT_ASYNC_FLUSH_INTERVAL_MS = 1e3;
157
- var DIRECTORY_IDENTITY_CHECK_INTERVAL_MS = 1e3;
158
- var DIRECTORY_LOCK_ATTEMPTS = 3;
159
- var DIRECTORY_LOCK_SETTLE_MS = 50;
160
- var DEFAULT_SEGMENT_MAX_BYTES = 67108864;
161
- var DEFAULT_COMPACTION_THRESHOLD = 12;
162
- var COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT = 2e5;
163
- var SEGMENT_FILE_FRAMING_SLACK_BYTES = 65536;
164
- var SEGMENT_TOKEN_TABLE_SHARE_OF_FILE = 0.25;
165
- var WHOLE_PARTITION_MIN_CHANGED_SHARE = 0.5;
166
- var WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT = 1e5;
167
- var LOG_RECORDS_PER_CHECKPOINT = 1e5;
168
- var WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES = 2147483648;
169
- var CHECKPOINT_WORKER_TIMEOUT_MS = 6e5;
170
- var CHECKPOINT_WORKER_IDLE_MS = 5e3;
171
- var CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS = 50;
172
-
173
158
  // src/serialization/stored-vector-values.ts
174
159
  function dropStoredVectorValues(documents, schema) {
175
160
  const vectorFieldPaths = [];
@@ -403,6 +388,9 @@ function metadataToWire(meta) {
403
388
  if (meta.trackPositions !== void 0) {
404
389
  wire.track_positions = meta.trackPositions;
405
390
  }
391
+ if (meta.textStorage !== void 0) {
392
+ wire.text_storage = meta.textStorage;
393
+ }
406
394
  if (meta.strict !== void 0) {
407
395
  wire.strict = meta.strict;
408
396
  }
@@ -514,6 +502,14 @@ function wireToMetadata(raw) {
514
502
  if (typeof raw.track_positions === "boolean") {
515
503
  meta.trackPositions = raw.track_positions;
516
504
  }
505
+ if (raw.text_storage !== void 0) {
506
+ if (raw.text_storage !== "memory" && raw.text_storage !== "disk") {
507
+ throw new NarsilError(ErrorCodes.CONFIG_INVALID, "text_storage must be 'memory' or 'disk'", {
508
+ textStorage: raw.text_storage
509
+ });
510
+ }
511
+ meta.textStorage = raw.text_storage;
512
+ }
517
513
  if (typeof raw.strict === "boolean") {
518
514
  meta.strict = raw.strict;
519
515
  }
@@ -664,7 +660,7 @@ async function packEnvelope(payloadBytes, options) {
664
660
  const checksum = flags.checksumPresent ? crc32(finalPayload) : 0;
665
661
  const [major, minor, patch] = engineVersionParts();
666
662
  const envelopeFormatVersion = options.envelopeFormatVersion ?? CURRENT_ENVELOPE_VERSION;
667
- const header2 = {
663
+ const header = {
668
664
  magic: "NRSL",
669
665
  envelopeFormatVersion,
670
666
  engineVersionMajor: major,
@@ -675,64 +671,64 @@ async function packEnvelope(payloadBytes, options) {
675
671
  checksum,
676
672
  reserved: new Uint8Array(14)
677
673
  };
678
- const headerBytes = writeHeader(header2);
674
+ const headerBytes = writeHeader(header);
679
675
  const envelope = new Uint8Array(HEADER_SIZE + finalPayload.length);
680
676
  envelope.set(headerBytes, 0);
681
677
  envelope.set(finalPayload, HEADER_SIZE);
682
678
  return envelope;
683
679
  }
684
680
  async function unpackEnvelope(data, maxAcceptedVersion = CURRENT_ENVELOPE_VERSION) {
685
- const header2 = readHeader(data);
686
- if (header2.envelopeFormatVersion > maxAcceptedVersion) {
681
+ const header = readHeader(data);
682
+ if (header.envelopeFormatVersion > maxAcceptedVersion) {
687
683
  throw new NarsilError(
688
684
  ErrorCodes.ENVELOPE_VERSION_MISMATCH,
689
- `This data holds Narsil envelope format v${header2.envelopeFormatVersion}, while this build of @delali/narsil reads up to v${maxAcceptedVersion}, so upgrade the package to read it.`
685
+ `This data holds Narsil envelope format v${header.envelopeFormatVersion}, while this build of @delali/narsil reads up to v${maxAcceptedVersion}, so upgrade the package to read it.`
690
686
  );
691
687
  }
692
- if (header2.flags.encryptionEnabled) {
688
+ if (header.flags.encryptionEnabled) {
693
689
  throw new NarsilError(
694
690
  ErrorCodes.ENVELOPE_VERSION_MISMATCH,
695
691
  "Encrypted envelopes are not supported in this version of Narsil"
696
692
  );
697
693
  }
698
- const compAlgo = header2.flags.compressionAlgorithm;
699
- if (header2.flags.compressionEnabled && compAlgo !== "gzip" && compAlgo !== "none") {
694
+ const compAlgo = header.flags.compressionAlgorithm;
695
+ if (header.flags.compressionEnabled && compAlgo !== "gzip" && compAlgo !== "none") {
700
696
  throw new NarsilError(
701
697
  ErrorCodes.ENVELOPE_VERSION_MISMATCH,
702
698
  `Compression algorithm "${compAlgo}" is not supported in this version of Narsil`
703
699
  );
704
700
  }
705
701
  const availablePayload = data.length - HEADER_SIZE;
706
- if (availablePayload < header2.payloadLength) {
702
+ if (availablePayload < header.payloadLength) {
707
703
  throw new NarsilError(
708
704
  ErrorCodes.PERSISTENCE_LOAD_FAILED,
709
- `Envelope data is truncated: header declares ${header2.payloadLength} bytes of payload but only ${availablePayload} bytes are available`
705
+ `Envelope data is truncated: header declares ${header.payloadLength} bytes of payload but only ${availablePayload} bytes are available`
710
706
  );
711
707
  }
712
- let payloadBytes = new Uint8Array(data.slice(HEADER_SIZE, HEADER_SIZE + header2.payloadLength));
713
- if (header2.flags.checksumPresent) {
708
+ let payloadBytes = new Uint8Array(data.slice(HEADER_SIZE, HEADER_SIZE + header.payloadLength));
709
+ if (header.flags.checksumPresent) {
714
710
  const computed = crc32(payloadBytes);
715
- if (computed !== header2.checksum) {
711
+ if (computed !== header.checksum) {
716
712
  throw new NarsilError(ErrorCodes.PERSISTENCE_CRC_MISMATCH, "CRC32 checksum mismatch: data may be corrupted", {
717
- expected: header2.checksum,
713
+ expected: header.checksum,
718
714
  computed
719
715
  });
720
716
  }
721
717
  }
722
- if (header2.flags.compressionEnabled) {
718
+ if (header.flags.compressionEnabled) {
723
719
  payloadBytes = await decompressGzip(payloadBytes);
724
720
  }
725
- return { header: header2, payloadBytes };
721
+ return { header, payloadBytes };
726
722
  }
727
723
  async function unpackEnvelopeBytes(data) {
728
724
  return unpackEnvelope(data, SNAPSHOT_ENVELOPE_VERSION);
729
725
  }
730
726
  async function unpackIndexSnapshotEnvelope(data) {
731
- const { header: header2, payloadBytes } = await unpackEnvelope(data, INDEX_SNAPSHOT_ENVELOPE_VERSION);
732
- if (header2.envelopeFormatVersion !== INDEX_SNAPSHOT_ENVELOPE_VERSION) {
727
+ const { header, payloadBytes } = await unpackEnvelope(data, INDEX_SNAPSHOT_ENVELOPE_VERSION);
728
+ if (header.envelopeFormatVersion !== INDEX_SNAPSHOT_ENVELOPE_VERSION) {
733
729
  throw new NarsilError(
734
730
  ErrorCodes.ENVELOPE_VERSION_MISMATCH,
735
- `Expected an index snapshot envelope of format v${INDEX_SNAPSHOT_ENVELOPE_VERSION}, received format v${header2.envelopeFormatVersion}`
731
+ `Expected an index snapshot envelope of format v${INDEX_SNAPSHOT_ENVELOPE_VERSION}, received format v${header.envelopeFormatVersion}`
736
732
  );
737
733
  }
738
734
  return payloadBytes;
@@ -742,127 +738,9 @@ async function writeMetadataEnvelope(meta, options = {}) {
742
738
  return packEnvelope(payloadBytes, options);
743
739
  }
744
740
  async function readMetadataEnvelope(data) {
745
- const { header: header2, payloadBytes } = await unpackEnvelope(data);
741
+ const { header, payloadBytes } = await unpackEnvelope(data);
746
742
  const metadata = deserializeMetadata(payloadBytes);
747
- return { header: header2, metadata };
748
- }
749
-
750
- // src/persistence/durability/segment/layout.ts
751
- var SEGMENT_ID_WIDTH = 16;
752
- function manifestKey(indexName) {
753
- return `${indexName}/manifest`;
754
- }
755
- function snapshotBundleKey(indexName) {
756
- return `${indexName}/snapshot`;
757
- }
758
- function segmentPrefix(indexName, partitionId) {
759
- return `${indexName}/segments/${partitionId}/`;
760
- }
761
- function segmentKey(indexName, partitionId, segmentId) {
762
- return `${segmentPrefix(indexName, partitionId)}s${formatSegmentId(segmentId)}`;
763
- }
764
- function segmentsPrefix(indexName) {
765
- return `${indexName}/segments/`;
766
- }
767
- function vectorFileKey(indexName, fieldPath, fileId) {
768
- return `${segmentsPrefix(indexName)}vec-${encodeFieldPath(fieldPath)}-f${formatSegmentId(fileId)}`;
769
- }
770
- function requireGeneration(generation) {
771
- if (Number.isSafeInteger(generation) && generation > 0) return;
772
- throw new NarsilError(
773
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
774
- `Vector graph generation ${generation} must be a positive integer`,
775
- { generation }
776
- );
777
- }
778
- function graphPartKey(indexName, fieldPath, generation, part) {
779
- requireGeneration(generation);
780
- return `${segmentsPrefix(indexName)}vec-${encodeFieldPath(fieldPath)}-graph-g${generation}-p${formatSegmentId(part)}`;
781
- }
782
- function formatSegmentId(segmentId) {
783
- if (!Number.isInteger(segmentId) || segmentId < 0) {
784
- throw new NarsilError(
785
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
786
- `Segment id ${segmentId} must be a non-negative integer`,
787
- { segmentId }
788
- );
789
- }
790
- return segmentId.toString().padStart(SEGMENT_ID_WIDTH, "0");
791
- }
792
- var FIELD_PATH_PATTERN = /^[A-Za-z0-9_.]+$/;
793
- function encodeFieldPath(fieldPath) {
794
- if (!FIELD_PATH_PATTERN.test(fieldPath)) {
795
- throw new NarsilError(
796
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
797
- `Vector field path "${fieldPath}" contains characters that cannot be encoded in a segment key`,
798
- { fieldPath }
799
- );
800
- }
801
- const fingerprint = fnv1a(fieldPath).toString(36);
802
- return `${fieldPath.replace(/\./g, "_")}-${fingerprint}`;
803
- }
804
- var SEGMENT_MANIFEST_VERSION = 7;
805
- var EARLIER_SEGMENT_MANIFEST_VERSION = 6;
806
- var MANIFEST_ENVELOPE_VERSION = 2;
807
- var MAX_SEGMENTS_PER_PARTITION = 65536;
808
- var MAX_SEGMENT_ID = 281474976710655;
809
- function fileRefOnTheWire(ref) {
810
- return { key: ref.key, bytes: ref.bytes, checksum: ref.checksum };
811
- }
812
- function encodeSegmentManifest(manifest) {
813
- const payload = encode({
814
- version: SEGMENT_MANIFEST_VERSION,
815
- schema: manifest.schema,
816
- language: manifest.language,
817
- ...manifest.analysisGeneration ? { analysisGeneration: manifest.analysisGeneration } : {},
818
- checkpoint: manifest.checkpoint.map((c) => ({
819
- partitionId: c.partitionId,
820
- lastSeqNo: c.lastSeqNo,
821
- primaryTerm: c.primaryTerm,
822
- ...c.termHistory === void 0 ? {} : {
823
- termHistory: c.termHistory.map((start) => ({ primaryTerm: start.primaryTerm, firstSeqNo: start.firstSeqNo }))
824
- }
825
- })),
826
- partitions: manifest.partitions.map((p) => ({
827
- partitionId: p.partitionId,
828
- nextSegmentId: p.nextSegmentId,
829
- segments: p.segments.map((s) => ({
830
- id: s.id,
831
- key: s.key,
832
- docCount: s.docCount,
833
- tombstoneCount: s.tombstoneCount,
834
- bytes: s.bytes,
835
- checksum: s.checksum
836
- }))
837
- })),
838
- vectors: manifest.vectors.map((v) => ({
839
- fieldPath: v.fieldPath,
840
- nextFileId: v.nextFileId,
841
- files: v.files.map((f) => ({ id: f.id, ...fileRefOnTheWire(f), count: f.count, dead: f.dead })),
842
- graphGeneration: v.graphGeneration,
843
- graphParts: v.graphParts.map(fileRefOnTheWire)
844
- }))
845
- });
846
- return packChecksummedEnvelope([payload], MANIFEST_ENVELOPE_VERSION, "segment manifest");
847
- }
848
- function manifestReferencedKeys(manifest) {
849
- const keys = /* @__PURE__ */ new Set();
850
- for (const partition of manifest.partitions) {
851
- for (const segment of partition.segments) {
852
- keys.add(segment.key);
853
- }
854
- }
855
- for (const vector of manifest.vectors) {
856
- for (const file of vector.files) keys.add(file.key);
857
- for (const part of vector.graphParts) keys.add(part.key);
858
- }
859
- return keys;
860
- }
861
- function manifestFileRefs(manifest) {
862
- const refs = [];
863
- for (const partition of manifest.partitions) refs.push(...partition.segments);
864
- for (const vector of manifest.vectors) refs.push(...vector.files, ...vector.graphParts);
865
- return refs;
743
+ return { header, metadata };
866
744
  }
867
745
 
868
746
  // src/persistence/durability/segment/manifest-fields.ts
@@ -893,7 +771,20 @@ function readFileRef(raw, expectedKey, what) {
893
771
  invalidManifest(`gives the ${what} "${expectedKey}" an invalid length`, { key: expectedKey, bytes });
894
772
  }
895
773
  if (!isUint32(raw.checksum)) invalidManifest(`gives the ${what} "${expectedKey}" an invalid checksum`);
896
- return { key: expectedKey, bytes, checksum: raw.checksum };
774
+ if (raw.headOffset === void 0) return { key: expectedKey, bytes, checksum: raw.checksum };
775
+ const { headOffset, headLength, headChecksum } = raw;
776
+ if (!isSequenceNumber(headOffset) || !isUint32(headLength) || !isUint32(headChecksum)) {
777
+ invalidManifest(`gives the ${what} "${expectedKey}" an invalid head`, { key: expectedKey });
778
+ }
779
+ if (headOffset + headLength > bytes - HEADER_SIZE) {
780
+ invalidManifest(`ends the head of the ${what} "${expectedKey}" outside its payload`, {
781
+ key: expectedKey,
782
+ headOffset,
783
+ headLength,
784
+ bytes
785
+ });
786
+ }
787
+ return { key: expectedKey, bytes, checksum: raw.checksum, headOffset, headLength, headChecksum };
897
788
  }
898
789
 
899
790
  // src/persistence/durability/segment/manifest-vectors-decode.ts
@@ -1151,83 +1042,292 @@ function contentsOfCaptured(captured) {
1151
1042
  return contents;
1152
1043
  }
1153
1044
 
1154
- // src/persistence/durability/segment/store.ts
1155
- function directoryCheckpointStore(directory) {
1156
- const store = {
1157
- read: (key) => directory.read(key),
1158
- write: (key, chunks) => directory.atomicWrite(key, chunks),
1159
- remove: (key) => directory.remove(key),
1160
- list: (prefix) => directory.list(prefix),
1161
- pathOf: (key) => directory.pathOf(key)
1162
- };
1163
- const byteLength = directory.byteLength;
1164
- if (byteLength !== void 0) store.byteLength = (key) => byteLength.call(directory, key);
1165
- return store;
1166
- }
1167
- function joined(chunks) {
1168
- if (chunks.length === 1) return chunks[0];
1169
- let total = 0;
1170
- for (const chunk of chunks) total += chunk.byteLength;
1171
- const bytes = new Uint8Array(total);
1045
+ // src/serialization/crc32-combine.ts
1046
+ var IEEE_POLYNOMIAL = 3988292384;
1047
+ var CRC_BITS = 32;
1048
+ var zeroByteOperators = [];
1049
+ function timesVector(operator, vector) {
1050
+ let sum = 0;
1051
+ let rest = vector >>> 0;
1052
+ for (let row = 0; rest !== 0; row++, rest >>>= 1) {
1053
+ if (rest & 1) sum ^= operator[row];
1054
+ }
1055
+ return sum >>> 0;
1056
+ }
1057
+ function squared(operator) {
1058
+ const result = new Uint32Array(CRC_BITS);
1059
+ for (let row = 0; row < CRC_BITS; row++) result[row] = timesVector(operator, operator[row]);
1060
+ return result;
1061
+ }
1062
+ function firstOperator() {
1063
+ let oneZeroBit = new Uint32Array(CRC_BITS);
1064
+ oneZeroBit[0] = IEEE_POLYNOMIAL;
1065
+ for (let row = 1; row < CRC_BITS; row++) oneZeroBit[row] = 1 << row - 1;
1066
+ for (let doubling = 0; doubling < 3; doubling++) oneZeroBit = squared(oneZeroBit);
1067
+ return oneZeroBit;
1068
+ }
1069
+ function operatorForZeroBytesPower(power) {
1070
+ if (zeroByteOperators.length === 0) zeroByteOperators.push(firstOperator());
1071
+ while (zeroByteOperators.length <= power) {
1072
+ zeroByteOperators.push(squared(zeroByteOperators[zeroByteOperators.length - 1]));
1073
+ }
1074
+ return zeroByteOperators[power];
1075
+ }
1076
+ function crc32Combine(first, second, secondByteLength) {
1077
+ let crc = first >>> 0;
1078
+ let rest = secondByteLength;
1079
+ for (let power = 0; rest > 0; power++) {
1080
+ if (rest % 2 === 1) crc = timesVector(operatorForZeroBytesPower(power), crc);
1081
+ rest = Math.floor(rest / 2);
1082
+ }
1083
+ return (crc ^ second) >>> 0;
1084
+ }
1085
+ var BIN8 = 196;
1086
+ var BIN16 = 197;
1087
+ var BIN32 = 198;
1088
+ var ARRAY16 = 220;
1089
+ var ARRAY32 = 221;
1090
+ var FIXMAP = 128;
1091
+ var FIXARRAY = 144;
1092
+ var FIXED_LIST_MAX = 15;
1093
+ var FIXSTR = 160;
1094
+ var FIXSTR_MAX_LENGTH = 31;
1095
+ var ASCII_MAX = 127;
1096
+ var UTF8_MAX_BYTES_PER_UNIT = 3;
1097
+ var STRING_HEADER_MAX_BYTES = 5;
1098
+ var BIN8_MAX_BYTES = 255;
1099
+ var UINT16_MAX = 65535;
1100
+ function binHeader(byteLength) {
1101
+ if (byteLength <= BIN8_MAX_BYTES) return Uint8Array.of(BIN8, byteLength);
1102
+ if (byteLength <= UINT16_MAX) return Uint8Array.of(BIN16, byteLength >> 8, byteLength & 255);
1103
+ const header = new Uint8Array(5);
1104
+ header[0] = BIN32;
1105
+ new DataView(header.buffer).setUint32(1, byteLength, false);
1106
+ return header;
1107
+ }
1108
+ function arrayHeader(count) {
1109
+ if (count <= FIXED_LIST_MAX) return Uint8Array.of(FIXARRAY | count);
1110
+ if (count <= UINT16_MAX) return Uint8Array.of(ARRAY16, count >> 8, count & 255);
1111
+ const header = new Uint8Array(5);
1112
+ header[0] = ARRAY32;
1113
+ new DataView(header.buffer).setUint32(1, count, false);
1114
+ return header;
1115
+ }
1116
+ function createPayloadBuilder() {
1117
+ const chunks = [];
1172
1118
  let offset = 0;
1173
- for (const chunk of chunks) {
1174
- bytes.set(chunk, offset);
1119
+ function push(chunk) {
1120
+ chunks.push(chunk);
1175
1121
  offset += chunk.byteLength;
1176
1122
  }
1123
+ return {
1124
+ chunks,
1125
+ get offset() {
1126
+ return offset;
1127
+ },
1128
+ push,
1129
+ entry(key, value) {
1130
+ push(encode(key));
1131
+ push(encode(value));
1132
+ },
1133
+ rawEntry(key, encoded) {
1134
+ push(encode(key));
1135
+ const at = offset;
1136
+ push(encoded);
1137
+ return at;
1138
+ },
1139
+ bytesEntry(key, bytes) {
1140
+ push(encode(key));
1141
+ push(binHeader(bytes.byteLength));
1142
+ const at = offset;
1143
+ push(bytes);
1144
+ return at;
1145
+ }
1146
+ };
1147
+ }
1148
+ function pageChecksums(bytes, pageBytes) {
1149
+ const pages = Math.ceil(bytes.byteLength / pageBytes);
1150
+ const checksums = new Uint8Array(pages * 4);
1151
+ const view = new DataView(checksums.buffer);
1152
+ for (let page = 0; page < pages; page++) {
1153
+ const from = page * pageBytes;
1154
+ view.setUint32(page * 4, crc32(bytes.subarray(from, Math.min(from + pageBytes, bytes.byteLength))), true);
1155
+ }
1156
+ return checksums;
1157
+ }
1158
+ function sectionOf(name, offset, bytes, pageBytes) {
1159
+ return { name, offset, length: bytes.byteLength, pageBytes, checksums: pageChecksums(bytes, pageBytes) };
1160
+ }
1161
+ function uint32Bytes(values) {
1162
+ const bytes = new Uint8Array(values.length * 4);
1163
+ const view = new DataView(bytes.buffer);
1164
+ for (let i = 0; i < values.length; i++) view.setUint32(i * 4, values[i], true);
1177
1165
  return bytes;
1178
1166
  }
1179
- function adapterFailure(code, verb, key, error) {
1180
- if (error instanceof NarsilError) return error;
1181
- const cause = error instanceof Error ? error.message : String(error);
1182
- return new NarsilError(code, `The persistence adapter failed to ${verb} "${key}": ${cause}`, { key, cause });
1167
+ function writtenAsShortAscii(docId, list, at) {
1168
+ if (docId.length > FIXSTR_MAX_LENGTH) return false;
1169
+ for (let unit = 0; unit < docId.length; unit++) {
1170
+ const code = docId.charCodeAt(unit);
1171
+ if (code > ASCII_MAX) return false;
1172
+ list[at + 1 + unit] = code;
1173
+ }
1174
+ list[at] = FIXSTR | docId.length;
1175
+ return true;
1183
1176
  }
1184
- async function guarded(run, code, verb, key) {
1185
- try {
1186
- return await run();
1187
- } catch (error) {
1188
- throw adapterFailure(code, verb, key, error);
1177
+ function encodedIdList(docIds) {
1178
+ const header = arrayHeader(docIds.length);
1179
+ let capacity = header.byteLength;
1180
+ for (const docId of docIds) capacity += docId.length * UTF8_MAX_BYTES_PER_UNIT + STRING_HEADER_MAX_BYTES;
1181
+ const list = new Uint8Array(capacity);
1182
+ list.set(header);
1183
+ const offsets = new Uint32Array(docIds.length + 1);
1184
+ let cursor = header.byteLength;
1185
+ for (let i = 0; i < docIds.length; i++) {
1186
+ offsets[i] = cursor;
1187
+ if (writtenAsShortAscii(docIds[i], list, cursor)) {
1188
+ cursor += docIds[i].length + 1;
1189
+ continue;
1190
+ }
1191
+ const encoded = encode(docIds[i]);
1192
+ list.set(encoded, cursor);
1193
+ cursor += encoded.byteLength;
1189
1194
  }
1195
+ offsets[docIds.length] = cursor;
1196
+ return { list: list.subarray(0, cursor), offsets };
1190
1197
  }
1191
- function adapterCheckpointStore(adapter) {
1192
- return {
1193
- read: (key) => guarded(() => adapter.load(key), ErrorCodes.PERSISTENCE_LOAD_FAILED, "load", key),
1194
- write: (key, chunks) => guarded(() => adapter.save(key, joined(chunks)), ErrorCodes.PERSISTENCE_SAVE_FAILED, "save", key),
1195
- remove: (key) => guarded(() => adapter.delete(key), ErrorCodes.PERSISTENCE_DELETE_FAILED, "delete", key),
1196
- list: (prefix) => guarded(() => adapter.list(prefix), ErrorCodes.PERSISTENCE_LOAD_FAILED, "list", prefix)
1197
- };
1198
+ function positionsInIdOrder(docIds) {
1199
+ const positions = Array.from(docIds.keys());
1200
+ positions.sort((left, right) => compareCodePoints(docIds[left], docIds[right]));
1201
+ return Uint32Array.from(positions);
1198
1202
  }
1199
- async function readListedFile(store, key, expected) {
1200
- const length = store.byteLength === void 0 ? void 0 : await store.byteLength(key);
1201
- if (length === null) missingFile(expected.what, key);
1202
- if (length !== void 0) requireListedLength(expected.what, key, length, expected.bytes);
1203
- const bytes = await store.read(key);
1204
- if (bytes === null) missingFile(expected.what, key);
1205
- requireListedLength(expected.what, key, bytes.byteLength, expected.bytes);
1203
+ function magnitudeBytes(magnitudes) {
1204
+ const bytes = new Uint8Array(magnitudes.length * 8);
1205
+ const view = new DataView(bytes.buffer);
1206
+ for (let i = 0; i < magnitudes.length; i++) view.setFloat64(i * 8, magnitudes[i], true);
1206
1207
  return bytes;
1207
1208
  }
1208
- function missingFile(what, key) {
1209
- throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The ${what} "${key}" that the manifest lists is missing`, {
1210
- key
1211
- });
1212
- }
1213
- function requireListedLength(what, key, found, listed) {
1214
- if (found === listed) return;
1215
- throw new NarsilError(
1216
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
1217
- `The ${what} "${key}" holds ${found} bytes where the manifest lists ${listed}`,
1218
- { key, bytes: found, listed }
1209
+ function checksumAcrossPages(checksums, byteLength, pageBytes) {
1210
+ const view = new DataView(checksums.buffer, checksums.byteOffset, checksums.byteLength);
1211
+ let whole = 0;
1212
+ for (let page = 0; page * 4 < checksums.byteLength; page++) {
1213
+ const pageLength = Math.min(pageBytes, byteLength - page * pageBytes);
1214
+ whole = crc32Combine(whole, view.getUint32(page * 4, true), pageLength);
1215
+ }
1216
+ return whole;
1217
+ }
1218
+ function encodedHeadAndVectorsPlace(sections, headOffsetAfterHeader, vectors, checksums, vectorsKeyAndHeaderBytes) {
1219
+ let headLength = 0;
1220
+ let head = new Uint8Array(0);
1221
+ let vectorsSection = {
1222
+ name: "vectors",
1223
+ offset: 0,
1224
+ length: vectors.byteLength,
1225
+ pageBytes: SEGMENT_PAGE_BYTES,
1226
+ checksums
1227
+ };
1228
+ for (let attempt = 0; attempt < STABLE_HEAD_ATTEMPTS; attempt++) {
1229
+ vectorsSection = {
1230
+ ...vectorsSection,
1231
+ offset: headOffsetAfterHeader(headLength) + headLength + vectorsKeyAndHeaderBytes
1232
+ };
1233
+ head = encode({ sections: [...sections, vectorsSection] });
1234
+ if (head.byteLength === headLength) break;
1235
+ headLength = head.byteLength;
1236
+ }
1237
+ return { head, vectorsSection };
1238
+ }
1239
+ function pagedVectorFileChunks(payload, heldMagnitudes) {
1240
+ const builder = createPayloadBuilder();
1241
+ const { codes } = payload;
1242
+ builder.push(Uint8Array.of(FIXMAP | 10));
1243
+ builder.entry("v", payload.v);
1244
+ builder.entry("fieldName", payload.fieldName);
1245
+ builder.entry("dimension", payload.dimension);
1246
+ const { list, offsets } = encodedIdList(payload.docIds);
1247
+ const docIdsAt = builder.rawEntry("docIds", list);
1248
+ const sections = [sectionOf("docIds", docIdsAt, list, SEGMENT_PAGE_BYTES)];
1249
+ builder.push(encode("codes"));
1250
+ if (codes === null) {
1251
+ builder.push(encode(null));
1252
+ } else {
1253
+ builder.push(Uint8Array.of(FIXMAP | 3));
1254
+ builder.entry("bits", codes.bits);
1255
+ builder.entry("centroid", codes.centroid);
1256
+ const recordsAt = builder.bytesEntry("records", codes.records);
1257
+ sections.push(sectionOf("records", recordsAt, codes.records, RECORD_PAGE_BYTES));
1258
+ }
1259
+ const magnitudes = magnitudeBytes(heldMagnitudes);
1260
+ sections.push(sectionOf("magnitudes", builder.bytesEntry("magnitudes", magnitudes), magnitudes, SEGMENT_PAGE_BYTES));
1261
+ const idOffsets = uint32Bytes(offsets);
1262
+ sections.push(sectionOf("idOffsets", builder.bytesEntry("idOffsets", idOffsets), idOffsets, SEGMENT_PAGE_BYTES));
1263
+ const idOrder = uint32Bytes(positionsInIdOrder(payload.docIds));
1264
+ sections.push(sectionOf("idOrder", builder.bytesEntry("idOrder", idOrder), idOrder, SEGMENT_PAGE_BYTES));
1265
+ const headKey = encode("head");
1266
+ const vectorsKey = encode("vectors");
1267
+ const vectorsHeader = binHeader(payload.vectors.byteLength);
1268
+ const beforeHead = builder.offset + headKey.byteLength;
1269
+ const vectorChecksums = pageChecksums(payload.vectors, SEGMENT_PAGE_BYTES);
1270
+ const { head } = encodedHeadAndVectorsPlace(
1271
+ sections,
1272
+ (headLength) => beforeHead + binHeader(headLength).byteLength,
1273
+ payload.vectors,
1274
+ vectorChecksums,
1275
+ vectorsKey.byteLength + vectorsHeader.byteLength
1219
1276
  );
1277
+ const headOffset = builder.bytesEntry("head", head);
1278
+ builder.push(vectorsKey);
1279
+ builder.push(vectorsHeader);
1280
+ const beforeVectors = [...builder.chunks];
1281
+ builder.push(payload.vectors);
1282
+ return {
1283
+ chunks: builder.chunks,
1284
+ headOffset,
1285
+ headLength: head.byteLength,
1286
+ headChecksum: crc32(head),
1287
+ beforeVectors,
1288
+ vectors: {
1289
+ byteLength: payload.vectors.byteLength,
1290
+ checksum: checksumAcrossPages(vectorChecksums, payload.vectors.byteLength, SEGMENT_PAGE_BYTES)
1291
+ }
1292
+ };
1220
1293
  }
1221
- function requireListedChecksum(what, key, found, listed) {
1222
- if (found === listed) return;
1223
- throw new NarsilError(
1224
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
1225
- `The ${what} "${key}" carries checksum ${found} where the manifest lists ${listed}`,
1226
- { key, checksum: found, listed }
1227
- );
1294
+ function nodeOffsetsOf(graph) {
1295
+ const offsets = new Uint32Array(graph.levels.length + 1);
1296
+ const view = new DataView(graph.neighbours.buffer, graph.neighbours.byteOffset, graph.neighbours.byteLength);
1297
+ let cursor = 0;
1298
+ for (let at = 0; at < graph.levels.length; at++) {
1299
+ offsets[at] = cursor;
1300
+ for (let layer = 0; layer < graph.levels[at]; layer++) cursor += (view.getUint32(cursor, true) + 1) * 4;
1301
+ }
1302
+ offsets[graph.levels.length] = cursor;
1303
+ return uint32Bytes(offsets);
1304
+ }
1305
+ function pagedGraphPartChunks(part) {
1306
+ const builder = createPayloadBuilder();
1307
+ builder.push(Uint8Array.of(FIXMAP | 4));
1308
+ builder.entry("v", part.v);
1309
+ builder.entry("fieldName", part.fieldName);
1310
+ builder.push(encode("graphs"));
1311
+ builder.push(arrayHeader(part.graphs.length));
1312
+ const layouts = [];
1313
+ for (const graph of part.graphs) {
1314
+ const { levels, neighbours, ...header } = graph;
1315
+ const headerEntries = Object.entries(header).filter(([, value]) => value !== void 0);
1316
+ builder.push(Uint8Array.of(FIXMAP | headerEntries.length + 2));
1317
+ for (const [key, value] of headerEntries) builder.entry(key, value);
1318
+ const levelsAt = builder.bytesEntry("levels", levels);
1319
+ const neighboursAt = builder.bytesEntry("neighbours", neighbours);
1320
+ layouts.push({
1321
+ ...header,
1322
+ levels: sectionOf("levels", levelsAt, levels, SEGMENT_PAGE_BYTES),
1323
+ nodeOffsets: nodeOffsetsOf(graph),
1324
+ neighbours: sectionOf("neighbours", neighboursAt, neighbours, RECORD_PAGE_BYTES)
1325
+ });
1326
+ }
1327
+ const head = encode({ graphs: layouts });
1328
+ const headOffset = builder.bytesEntry("head", head);
1329
+ return { chunks: builder.chunks, headOffset, headLength: head.byteLength, headChecksum: crc32(head) };
1228
1330
  }
1229
-
1230
- // src/persistence/durability/segment/vector-load.ts
1231
1331
  var VECTOR_FILE_ENVELOPE_VERSION = 2;
1232
1332
  var GRAPH_PART_ENVELOPE_VERSION = 2;
1233
1333
  function liveVectorsOf(ref) {
@@ -1237,13 +1337,13 @@ function liveVectorsOf(ref) {
1237
1337
  }
1238
1338
  async function openListedFile(store, ref, what, allowsCompression) {
1239
1339
  const bytes = await readListedFile(store, ref.key, { bytes: ref.bytes, what });
1240
- const { header: header2, payload } = await openChecksummedEnvelope(bytes, {
1340
+ const { header, payload } = await openChecksummedEnvelope(bytes, {
1241
1341
  envelopeFormatVersion: VECTOR_FILE_ENVELOPE_VERSION,
1242
1342
  what: `${what} "${ref.key}"`,
1243
1343
  allowsCompression
1244
1344
  });
1245
- requireListedChecksum(what, ref.key, header2.checksum, ref.checksum);
1246
- return { payload, payloadLength: header2.payloadLength };
1345
+ requireListedChecksum(what, ref.key, header.checksum, ref.checksum);
1346
+ return { payload, payloadLength: header.payloadLength };
1247
1347
  }
1248
1348
  async function readGraphParts(store, ref) {
1249
1349
  if (ref.graphParts.length === 0) return null;
@@ -1281,7 +1381,46 @@ async function readVectorFile(store, file, fieldPath) {
1281
1381
  }
1282
1382
  return { payload, payloadLength };
1283
1383
  }
1284
- async function loadVectorField(store, ref, vectorIndex) {
1384
+ function holdsEveryHead(ref) {
1385
+ if (ref.graphParts.length === 0 || ref.files.length === 0) return false;
1386
+ return [...ref.files, ...ref.graphParts].every((file) => file.headOffset !== void 0);
1387
+ }
1388
+ async function loadFieldOnDemand(store, ref, vectorIndex, resolvePartition) {
1389
+ const { pathOf } = store;
1390
+ if (pathOf === void 0 || vectorIndex.storage !== "disk" || !holdsEveryHead(ref)) return false;
1391
+ const { loadFilesByParts, openColdVectorField, osqBitsOf, waitBeforeColdLoad } = await import('./cold-module-MJMKNUYJ.mjs');
1392
+ await Promise.all([
1393
+ ...ref.files.map((file) => requireListedSize(store, file.key, { bytes: file.bytes, what: "vector file" })),
1394
+ ...ref.graphParts.map((part) => requireListedSize(store, part.key, { bytes: part.bytes, what: "graph part" }))
1395
+ ]);
1396
+ const files = await Promise.all(
1397
+ ref.files.map(async (file) => ({ ref: file, path: await pathOf.call(store, file.key) }))
1398
+ );
1399
+ const graphParts = await Promise.all(
1400
+ ref.graphParts.map(async (part) => ({ ref: part, path: await pathOf.call(store, part.key) }))
1401
+ );
1402
+ const field = openColdVectorField({
1403
+ fieldPath: ref.fieldPath,
1404
+ dimension: vectorIndex.dimension,
1405
+ metric: vectorIndex.metric,
1406
+ codeBits: osqBitsOf(vectorIndex.quantization),
1407
+ files,
1408
+ graphParts,
1409
+ resolvePartition
1410
+ });
1411
+ if (field === null) return false;
1412
+ vectorIndex.openColdField(field, async (beginRestore, stillWanted, firstSearchAnswered) => {
1413
+ await waitBeforeColdLoad(firstSearchAnswered);
1414
+ if (!stillWanted()) return;
1415
+ const graph = await readGraphParts(store, ref);
1416
+ if (!stillWanted()) return;
1417
+ const restore = beginRestore({ liveVectors: liveVectorsOf(ref), holdsGraph: true, graph });
1418
+ await loadFilesByParts(files, restore, ref.fieldPath, vectorIndex.dimension, stillWanted);
1419
+ });
1420
+ return true;
1421
+ }
1422
+ async function loadVectorField(store, ref, vectorIndex, resolvePartition = null) {
1423
+ if (resolvePartition !== null && await loadFieldOnDemand(store, ref, vectorIndex, resolvePartition)) return;
1285
1424
  const restore = vectorIndex.restoreCheckpoint({
1286
1425
  liveVectors: liveVectorsOf(ref),
1287
1426
  holdsGraph: ref.graphParts.length > 0,
@@ -1304,24 +1443,24 @@ async function loadVectorField(store, ref, vectorIndex) {
1304
1443
  }
1305
1444
  var EMPTY_BIN = new Uint8Array(0);
1306
1445
  var EMPTY_BIN_BYTES = 2;
1307
- var BIN8 = 196;
1308
- var BIN16 = 197;
1309
- var BIN32 = 198;
1310
- var BIN8_MAX_BYTES = 255;
1446
+ var BIN82 = 196;
1447
+ var BIN162 = 197;
1448
+ var BIN322 = 198;
1449
+ var BIN8_MAX_BYTES2 = 255;
1311
1450
  var BIN16_MAX_BYTES = 65535;
1312
- function binHeader(byteLength) {
1313
- if (byteLength <= BIN8_MAX_BYTES) return Uint8Array.of(BIN8, byteLength);
1314
- if (byteLength <= BIN16_MAX_BYTES) return Uint8Array.of(BIN16, byteLength >> 8, byteLength & 255);
1315
- const header2 = new Uint8Array(5);
1316
- header2[0] = BIN32;
1317
- new DataView(header2.buffer).setUint32(1, byteLength, false);
1318
- return header2;
1451
+ function binHeader2(byteLength) {
1452
+ if (byteLength <= BIN8_MAX_BYTES2) return Uint8Array.of(BIN82, byteLength);
1453
+ if (byteLength <= BIN16_MAX_BYTES) return Uint8Array.of(BIN162, byteLength >> 8, byteLength & 255);
1454
+ const header = new Uint8Array(5);
1455
+ header[0] = BIN322;
1456
+ new DataView(header.buffer).setUint32(1, byteLength, false);
1457
+ return header;
1319
1458
  }
1320
1459
  function chunksEndingInBytes(withTheBytes, withEmptyBytesInTheirPlace, bytes) {
1321
1460
  const head = encode(withEmptyBytesInTheirPlace);
1322
- const endsWithTheEmptyBin = head[head.length - EMPTY_BIN_BYTES] === BIN8 && head[head.length - 1] === 0;
1461
+ const endsWithTheEmptyBin = head[head.length - EMPTY_BIN_BYTES] === BIN82 && head[head.length - 1] === 0;
1323
1462
  if (!endsWithTheEmptyBin) return [encode(withTheBytes)];
1324
- return [head.subarray(0, head.length - EMPTY_BIN_BYTES), binHeader(bytes.byteLength), bytes];
1463
+ return [head.subarray(0, head.length - EMPTY_BIN_BYTES), binHeader2(bytes.byteLength), bytes];
1325
1464
  }
1326
1465
  function vectorPartChunks(part) {
1327
1466
  return chunksEndingInBytes(part, { ...part, vectors: EMPTY_BIN }, part.vectors);
@@ -1337,20 +1476,27 @@ function keptFileRef(input, prior, kept) {
1337
1476
  { key: kept.key }
1338
1477
  );
1339
1478
  }
1340
- return { ...kept, bytes: listed.bytes, checksum: listed.checksum };
1479
+ return { ...kept, bytes: listed.bytes, checksum: listed.checksum, ...headFieldsOf(listed) };
1480
+ }
1481
+ function headFieldsOf(ref) {
1482
+ return ref.headOffset === void 0 ? {} : { headOffset: ref.headOffset, headLength: ref.headLength, headChecksum: ref.headChecksum };
1483
+ }
1484
+ function headRefOf(paged) {
1485
+ return paged === null ? {} : { headOffset: paged.headOffset, headLength: paged.headLength, headChecksum: paged.headChecksum };
1341
1486
  }
1342
1487
  async function writeGraphParts(input, fieldPath, plan, generation) {
1343
1488
  const parts = plan.graphParts ?? [];
1344
1489
  const refs = [];
1345
1490
  for (let part = 0; part < parts.length; part++) {
1346
1491
  const key = graphPartKey(input.indexName, fieldPath, generation, part);
1492
+ const paged = plan.storesHeads ? pagedGraphPartChunks(parts[part]) : null;
1347
1493
  const packed = await packChecksummedEnvelope(
1348
- [encode(parts[part])],
1494
+ paged === null ? [encode(parts[part])] : paged.chunks,
1349
1495
  GRAPH_PART_ENVELOPE_VERSION,
1350
1496
  `graph part "${key}"`
1351
1497
  );
1352
1498
  await input.store.write(key, packed.chunks);
1353
- refs.push({ key, bytes: packed.byteLength, checksum: packed.checksum });
1499
+ refs.push({ key, bytes: packed.byteLength, checksum: packed.checksum, ...headRefOf(paged) });
1354
1500
  }
1355
1501
  return refs;
1356
1502
  }
@@ -1364,17 +1510,28 @@ async function writeField(input, fieldPath, plan, outcome) {
1364
1510
  const files = plan.kept.map((kept) => keptFileRef(input, prior, kept));
1365
1511
  const written = [];
1366
1512
  for (let index = 0; index < plan.newFiles; index++) {
1367
- const { payload, ordinals, dead } = plan.readNewFile(index);
1513
+ const { payload, ordinals, dead, magnitudes } = plan.readNewFile(index);
1368
1514
  const id = nextFileId;
1369
1515
  nextFileId += 1;
1370
1516
  const key = vectorFileKey(input.indexName, fieldPath, id);
1371
- const packed = await packChecksummedEnvelope(
1372
- vectorPartChunks(payload),
1517
+ const paged = plan.storesHeads && magnitudes !== null ? pagedVectorFileChunks(payload, magnitudes) : null;
1518
+ const what = `vector file "${key}"`;
1519
+ const packed = paged === null ? await packChecksummedEnvelope(vectorPartChunks(payload), VECTOR_FILE_ENVELOPE_VERSION, what) : packEnvelopeWithChecksum(
1520
+ paged.chunks,
1373
1521
  VECTOR_FILE_ENVELOPE_VERSION,
1374
- `vector file "${key}"`
1522
+ what,
1523
+ crc32Combine(await checksumOfChunks(paged.beforeVectors), paged.vectors.checksum, paged.vectors.byteLength)
1375
1524
  );
1376
1525
  await input.store.write(key, packed.chunks);
1377
- files.push({ id, key, count: payload.docIds.length, dead, bytes: packed.byteLength, checksum: packed.checksum });
1526
+ files.push({
1527
+ id,
1528
+ key,
1529
+ count: payload.docIds.length,
1530
+ dead,
1531
+ bytes: packed.byteLength,
1532
+ checksum: packed.checksum,
1533
+ ...headRefOf(paged)
1534
+ });
1378
1535
  written.push({ id, key, ordinals });
1379
1536
  outcome.layouts.push({
1380
1537
  fieldPath,
@@ -1577,10 +1734,10 @@ var RECORD_LENGTH_SIZE = 4;
1577
1734
  var FRAME_CRC_SIZE = 4;
1578
1735
  var MAX_WAL_RECORD_BYTES = 268435456;
1579
1736
  function writeSegmentHeader() {
1580
- const header2 = new Uint8Array(SEGMENT_HEADER_SIZE);
1581
- header2.set(WAL_MAGIC, 0);
1582
- header2[4] = WAL_FORMAT_VERSION & 255;
1583
- return header2;
1737
+ const header = new Uint8Array(SEGMENT_HEADER_SIZE);
1738
+ header.set(WAL_MAGIC, 0);
1739
+ header[4] = WAL_FORMAT_VERSION & 255;
1740
+ return header;
1584
1741
  }
1585
1742
  function frameRecord(entry) {
1586
1743
  const payload = encode(entry);
@@ -1621,9 +1778,9 @@ function corrupt(message, details) {
1621
1778
  throw new NarsilError(ErrorCodes.PERSISTENCE_WAL_CORRUPT, message, details);
1622
1779
  }
1623
1780
  function readDurableRegion(data, durableByteLength, maxRecordBytes = MAX_WAL_RECORD_BYTES) {
1624
- const header2 = checkSegmentHeader(data);
1625
- if (!header2.ok) {
1626
- corrupt(`WAL segment header invalid in durable region: ${header2.reason}`, { reason: header2.reason });
1781
+ const header = checkSegmentHeader(data);
1782
+ if (!header.ok) {
1783
+ corrupt(`WAL segment header invalid in durable region: ${header.reason}`, { reason: header.reason });
1627
1784
  }
1628
1785
  if (durableByteLength < SEGMENT_HEADER_SIZE || durableByteLength > data.length) {
1629
1786
  corrupt("Commit marker durable byte length is outside the segment", {
@@ -1722,6 +1879,9 @@ function readTailBeyondFrontier(data, durableByteLength, lastDurableSeqNo, maxRe
1722
1879
  function walPrefix(indexName, partitionId) {
1723
1880
  return `${indexName}/wal/${partitionId}/`;
1724
1881
  }
1882
+ function nextSegmentKey(indexName, partitionId) {
1883
+ return `${walPrefix(indexName, partitionId)}next`;
1884
+ }
1725
1885
  function segmentStartSeqNo(key, prefix) {
1726
1886
  if (!key.startsWith(prefix)) {
1727
1887
  return null;
@@ -1766,14 +1926,14 @@ async function* readSegmentsInOrder(directory, wal) {
1766
1926
  continue;
1767
1927
  }
1768
1928
  if (startSeqNo < marker.state.activeSegmentSeqNo) {
1769
- const header2 = checkSegmentHeader(bytes);
1770
- if (!header2.ok) {
1929
+ const header = checkSegmentHeader(bytes);
1930
+ if (!header.ok) {
1771
1931
  throw new NarsilError(
1772
1932
  ErrorCodes.PERSISTENCE_WAL_CORRUPT,
1773
- `Sealed WAL segment header invalid: ${header2.reason}`,
1933
+ `Sealed WAL segment header invalid: ${header.reason}`,
1774
1934
  {
1775
1935
  key,
1776
- reason: header2.reason
1936
+ reason: header.reason
1777
1937
  }
1778
1938
  );
1779
1939
  }
@@ -1903,569 +2063,23 @@ async function buildSegmentFromEntries(input) {
1903
2063
  if (input.keepsWriteRecords) built.writeRecords = /* @__PURE__ */ new Map([[segment, writeRecordsOf(documents, written)]]);
1904
2064
  return built;
1905
2065
  }
1906
- var FIXMAP = 128;
1907
- var MAP16 = 222;
1908
- var MAP32 = 223;
1909
- var FIXARRAY = 144;
1910
- var ARRAY16 = 220;
1911
- var ARRAY32 = 221;
1912
- var BIN82 = 196;
1913
- var BIN162 = 197;
1914
- var BIN322 = 198;
1915
- var FIX_COUNT_LIMIT = 16;
1916
- var UINT8_LIMIT = 256;
1917
- var UINT16_LIMIT = 65536;
1918
- function header(fixed, sixteen, thirtyTwo, count) {
1919
- if (count < FIX_COUNT_LIMIT) return Uint8Array.of(fixed | count);
1920
- if (count < UINT16_LIMIT) return Uint8Array.of(sixteen, count >> 8, count & 255);
1921
- const bytes = new Uint8Array(5);
1922
- bytes[0] = thirtyTwo;
1923
- new DataView(bytes.buffer).setUint32(1, count, false);
1924
- return bytes;
1925
- }
1926
- function binHeader2(byteLength) {
1927
- if (byteLength < UINT8_LIMIT) return Uint8Array.of(BIN82, byteLength);
1928
- if (byteLength < UINT16_LIMIT) return Uint8Array.of(BIN162, byteLength >> 8, byteLength & 255);
1929
- const bytes = new Uint8Array(5);
1930
- bytes[0] = BIN322;
1931
- new DataView(bytes.buffer).setUint32(1, byteLength, false);
1932
- return bytes;
1933
- }
1934
- var MessagePackChunks = class {
1935
- chunks = [];
1936
- byteLength = 0;
1937
- push(chunk) {
1938
- if (chunk.byteLength === 0) return;
1939
- this.chunks.push(chunk);
1940
- this.byteLength += chunk.byteLength;
1941
- }
1942
- map(entries) {
1943
- this.push(header(FIXMAP, MAP16, MAP32, entries));
1944
- return this;
1945
- }
1946
- array(items) {
1947
- this.push(header(FIXARRAY, ARRAY16, ARRAY32, items));
1948
- return this;
1949
- }
1950
- value(value) {
1951
- this.push(encode(value));
1952
- return this;
1953
- }
1954
- key(name) {
1955
- return this.value(name);
1956
- }
1957
- binary(bytes) {
1958
- this.push(binHeader2(bytes.byteLength));
1959
- this.push(bytes);
1960
- return this;
1961
- }
1962
- binaryOrNil(bytes) {
1963
- return bytes === null ? this.value(null) : this.binary(bytes);
1964
- }
1965
- };
1966
-
1967
- // src/serialization/keyword-segment/token-order.ts
1968
- function slotsAreInTokenOrder(payloadSlots) {
1969
- for (let at = 0; at < payloadSlots.length; at++) {
1970
- if (payloadSlots[at] !== at) return false;
1971
- }
1972
- return true;
1973
- }
1974
- function reorderedPositions(arrays, postingOffsets) {
1975
- const { positionOffsets, positionValues, tokenTable } = arrays;
1976
- if (positionOffsets === null || positionValues === null) return { positionOffsets: null, positionValues: null };
1977
- const postings = postingOffsets[postingOffsets.length - 1];
1978
- const offsets = new Uint32Array(postings + 1);
1979
- const values = new Uint32Array(positionValues.length);
1980
- let cursor = 0;
1981
- for (let token = 0; token < tokenTable.payloadSlots.length; token++) {
1982
- const slot = tokenTable.payloadSlots[token];
1983
- for (let from = arrays.postingOffsets[slot]; from < arrays.postingOffsets[slot + 1]; from++) {
1984
- const to = postingOffsets[token] + (from - arrays.postingOffsets[slot]);
1985
- offsets[to] = cursor;
1986
- const start = positionOffsets[from];
1987
- const end = positionOffsets[from + 1];
1988
- values.set(positionValues.subarray(start, end), cursor);
1989
- cursor += end - start;
1990
- }
1991
- }
1992
- offsets[postings] = cursor;
1993
- return { positionOffsets: offsets, positionValues: values };
1994
- }
1995
- function postingsInTokenOrder(arrays) {
1996
- const { tokenTable } = arrays;
1997
- if (slotsAreInTokenOrder(tokenTable.payloadSlots)) return arrays;
1998
- const tokens = tokenTable.payloadSlots.length;
1999
- const postingOffsets = new Uint32Array(tokens + 1);
2000
- for (let token = 0; token < tokens; token++) {
2001
- const slot = tokenTable.payloadSlots[token];
2002
- postingOffsets[token + 1] = postingOffsets[token] + arrays.postingOffsets[slot + 1] - arrays.postingOffsets[slot];
2003
- }
2004
- const postings = postingOffsets[tokens];
2005
- const postingDocIds = new Uint32Array(postings);
2006
- const postingFrequencies = new Uint16Array(postings);
2007
- const postingFieldIndices = new Uint16Array(postings);
2008
- for (let token = 0; token < tokens; token++) {
2009
- const slot = tokenTable.payloadSlots[token];
2010
- const start = arrays.postingOffsets[slot];
2011
- const end = arrays.postingOffsets[slot + 1];
2012
- postingDocIds.set(arrays.postingDocIds.subarray(start, end), postingOffsets[token]);
2013
- postingFrequencies.set(arrays.postingFrequencies.subarray(start, end), postingOffsets[token]);
2014
- postingFieldIndices.set(arrays.postingFieldIndices.subarray(start, end), postingOffsets[token]);
2015
- }
2016
- return {
2017
- postingOffsets,
2018
- postingDocIds,
2019
- postingFrequencies,
2020
- postingFieldIndices,
2021
- ...reorderedPositions(arrays, postingOffsets)
2022
- };
2023
- }
2024
-
2025
- // src/serialization/keyword-segment/encode.ts
2026
- var TOP_LEVEL_ENTRIES = 28;
2027
- function writeFieldLengths(out, arrays) {
2028
- out.array(arrays.fieldLengthNames.length);
2029
- for (let f = 0; f < arrays.fieldLengthNames.length; f++) {
2030
- out.map(2).key("field").value(arrays.fieldLengthNames[f]);
2031
- out.key("lengths").binary(littleEndianBytes(arrays.fieldLengthColumns[f]));
2032
- }
2033
- }
2034
- function writeValueColumns(out, columns) {
2035
- out.array(columns.length);
2036
- for (const column of columns) {
2037
- out.map(3).key("field").value(column.fieldPath);
2038
- out.key("documents").binary(littleEndianBytes(column.docIds));
2039
- out.key("values").binary(littleEndianBytes(column.values));
2040
- }
2041
- }
2042
- function writeBooleanColumns(out, columns) {
2043
- out.array(columns.length);
2044
- for (const column of columns) {
2045
- out.map(3).key("field").value(column.fieldPath);
2046
- out.key("trueDocuments").binary(littleEndianBytes(column.trueDocs));
2047
- out.key("falseDocuments").binary(littleEndianBytes(column.falseDocs));
2048
- }
2049
- }
2050
- function writeEnumColumns(out, columns) {
2051
- out.array(columns.length);
2052
- for (const column of columns) {
2053
- out.map(4).key("field").value(column.fieldPath);
2054
- out.key("values").value(column.values);
2055
- out.key("offsets").binary(littleEndianBytes(column.offsets));
2056
- out.key("documents").binary(littleEndianBytes(column.docIds));
2057
- }
2058
- }
2059
- function writeGeopointColumns(out, columns) {
2060
- out.array(columns.length);
2061
- for (const column of columns) {
2062
- out.map(4).key("field").value(column.fieldPath);
2063
- out.key("documents").binary(littleEndianBytes(column.docIds));
2064
- out.key("latitudes").binary(littleEndianBytes(column.latitudes));
2065
- out.key("longitudes").binary(littleEndianBytes(column.longitudes));
2066
- }
2067
- }
2068
- function writePatternColumns(out, columns) {
2069
- out.array(columns.length);
2070
- for (const column of columns) {
2071
- out.map(5).key("field").value(column.fieldPath);
2072
- out.key("documents").binary(littleEndianBytes(column.docIds));
2073
- out.key("runs").binary(littleEndianBytes(column.runs));
2074
- out.key("offsets").binary(littleEndianBytes(column.offsets));
2075
- out.key("postings").binary(littleEndianBytes(column.postings));
2076
- }
2077
- }
2078
- function writeSurfaceForms(out, arrays) {
2079
- const table = arrays.surfaceTable;
2080
- if (table === null || table.counts.length === 0) {
2081
- out.value(null);
2082
- return;
2083
- }
2084
- out.map(3).key("text").binary(table.blob);
2085
- out.key("offsets").binary(littleEndianBytes(table.offsets));
2086
- out.key("counts").binary(littleEndianBytes(table.counts));
2087
- }
2088
- function writePartialWords(out, arrays) {
2089
- const table = arrays.partialWords;
2090
- if (table === null) {
2091
- out.value(null);
2092
- return;
2093
- }
2094
- out.map(8).key("fields").value(table.fields);
2095
- out.key("text").binary(table.spellings);
2096
- out.key("offsets").binary(littleEndianBytes(table.spellingOffsets));
2097
- out.key("tokens").binary(littleEndianBytes(table.spellingTokens));
2098
- out.key("counts").binary(littleEndianBytes(table.spellingCounts));
2099
- out.key("runs").binary(littleEndianBytes(table.runs));
2100
- out.key("runOffsets").binary(littleEndianBytes(table.runOffsets));
2101
- out.key("runEntries").binary(littleEndianBytes(table.runEntries));
2102
- }
2103
- function encodeKeywordSegment(contents) {
2104
- const { arrays } = contents;
2105
- const postings = postingsInTokenOrder(arrays);
2106
- const out = new MessagePackChunks();
2107
- const { writeRecords } = contents;
2108
- out.map(writeRecords === void 0 ? TOP_LEVEL_ENTRIES : TOP_LEVEL_ENTRIES + 2);
2109
- out.key("v").value(KEYWORD_SEGMENT_PAYLOAD_VERSION);
2110
- out.key("schema").value(contents.schema);
2111
- out.key("tombstones").value(contents.tombstones);
2112
- out.key("documentCount").value(arrays.documentCount);
2113
- out.key("ids").binary(arrays.idTable.blob);
2114
- out.key("idOffsets").binary(littleEndianBytes(arrays.idTable.offsets));
2115
- out.key("idOrder").binary(littleEndianBytes(arrays.idTable.sortedOrdinals));
2116
- out.key("documents").binary(arrays.documentTable.blob);
2117
- out.key("documentOffsets").binary(littleEndianBytes(arrays.documentTable.offsets));
2118
- out.key("fieldNames").value(arrays.fieldNames);
2119
- out.key("tokens").binary(arrays.tokenTable.blob);
2120
- out.key("tokenOffsets").binary(littleEndianBytes(arrays.tokenTable.offsets));
2121
- out.key("documentFrequencies").binary(littleEndianBytes(arrays.tokenTable.documentFrequencies));
2122
- out.key("postingOffsets").binary(littleEndianBytes(postings.postingOffsets));
2123
- out.key("postingDocuments").binary(littleEndianBytes(postings.postingDocIds));
2124
- out.key("postingFrequencies").binary(littleEndianBytes(postings.postingFrequencies));
2125
- out.key("postingFields").binary(littleEndianBytes(postings.postingFieldIndices));
2126
- out.key("positionOffsets").binaryOrNil(postings.positionOffsets === null ? null : littleEndianBytes(postings.positionOffsets));
2127
- out.key("positions").binaryOrNil(postings.positionValues === null ? null : littleEndianBytes(postings.positionValues));
2128
- out.key("fieldLengths");
2129
- writeFieldLengths(out, arrays);
2130
- out.key("numeric");
2131
- writeValueColumns(out, arrays.numeric);
2132
- out.key("sizes");
2133
- writeValueColumns(out, arrays.sizes);
2134
- out.key("boolean");
2135
- writeBooleanColumns(out, arrays.boolean);
2136
- out.key("enum");
2137
- writeEnumColumns(out, arrays.enums);
2138
- out.key("geopoint");
2139
- writeGeopointColumns(out, arrays.geo);
2140
- out.key("pattern");
2141
- writePatternColumns(out, arrays.pattern);
2142
- out.key("surfaceForms");
2143
- writeSurfaceForms(out, arrays);
2144
- out.key("partialWords");
2145
- writePartialWords(out, arrays);
2146
- if (writeRecords !== void 0) {
2147
- out.key("seqNos").binary(uint64Bytes(writeRecords.seqNos));
2148
- out.key("primaryTerms").binary(uint64Bytes(writeRecords.primaryTerms));
2149
- }
2150
- return out;
2151
- }
2152
-
2153
- // src/persistence/durability/segment/segment-file.ts
2154
- function sharesMemoryWithWorkers() {
2155
- return typeof SharedArrayBuffer === "function";
2156
- }
2157
- async function readSegmentFile(store, ref) {
2158
- const bytes = await readListedFile(store, ref.key, { bytes: ref.bytes, what: "segment" });
2159
- const { header: header2, payload } = await openChecksummedEnvelope(bytes, {
2160
- envelopeFormatVersion: KEYWORD_SEGMENT_ENVELOPE_VERSION,
2161
- what: `segment "${ref.key}"`,
2162
- allowsCompression: true
2163
- });
2164
- requireListedChecksum("segment", ref.key, header2.checksum, ref.checksum);
2165
- return decodeKeywordSegment(payload, sharesMemoryWithWorkers());
2166
- }
2167
- var ORDINAL_BYTES = 4;
2168
- var POSTING_BYTES = 8;
2169
- var VALUE_ENTRY_BYTES = 12;
2170
- var GEOPOINT_ENTRY_BYTES = 20;
2171
- var TOKEN_ENTRY_BYTES = 12;
2172
- var STRING_HEADER_BYTES = 5;
2173
- var encoder = new TextEncoder();
2174
- function documentBytes(segment, ordinal) {
2175
- const table = segment.arrays.documentTable;
2176
- if (table !== null) return table.offsets[ordinal + 1] - table.offsets[ordinal];
2177
- return messagePackUpperBoundBytes(segment.documentSource.docAt(ordinal));
2178
- }
2179
- function addPerEntry(costs, docIds, bytes) {
2180
- for (let at = 0; at < docIds.length; at++) costs[docIds[at]] += bytes;
2181
- }
2182
- function chargePostings(segment, costs, chargesTokens) {
2183
- const arrays = segment.arrays;
2184
- const positions = arrays.positionOffsets;
2185
- for (let p = 0; p < arrays.postingDocIds.length; p++) {
2186
- let bytes = POSTING_BYTES;
2187
- if (positions !== null) bytes += ORDINAL_BYTES * (1 + positions[p + 1] - positions[p]);
2188
- costs[arrays.postingDocIds[p]] += bytes;
2189
- }
2190
- if (!chargesTokens) return;
2191
- const table = arrays.tokenTable;
2192
- for (let at = 0; at < table.size; at++) {
2193
- const slot = table.payloadSlot(at);
2194
- const bytes = table.byteLengthAt(at) + TOKEN_ENTRY_BYTES;
2195
- for (let p = arrays.postingOffsets[slot]; p < arrays.postingOffsets[slot + 1]; p++) {
2196
- costs[arrays.postingDocIds[p]] += bytes;
2197
- }
2198
- }
2199
- }
2200
- function chargeColumns(segment, costs) {
2201
- const arrays = segment.arrays;
2202
- for (const column of [...arrays.numeric, ...arrays.sizes]) addPerEntry(costs, column.docIds, VALUE_ENTRY_BYTES);
2203
- for (const column of arrays.boolean) {
2204
- addPerEntry(costs, column.trueDocs, ORDINAL_BYTES);
2205
- addPerEntry(costs, column.falseDocs, ORDINAL_BYTES);
2206
- }
2207
- for (const column of arrays.enums) addPerEntry(costs, column.docIds, ORDINAL_BYTES);
2208
- for (const column of arrays.geo) addPerEntry(costs, column.docIds, GEOPOINT_ENTRY_BYTES);
2209
- for (const column of arrays.pattern) {
2210
- addPerEntry(costs, column.docIds, ORDINAL_BYTES);
2211
- addPerEntry(costs, column.postings, ORDINAL_BYTES);
2212
- }
2213
- }
2214
- function stringListBytes(values) {
2215
- let bytes = STRING_HEADER_BYTES;
2216
- for (const value of values) bytes += STRING_HEADER_BYTES + encoder.encode(value).byteLength;
2217
- return bytes;
2218
- }
2219
- function fixedBytesOf(segment, chargesTokens) {
2220
- const arrays = segment.arrays;
2221
- let bytes = stringListBytes(arrays.fieldNames);
2222
- if (!chargesTokens) bytes += arrays.tokenTable.bytes;
2223
- for (const column of arrays.enums) bytes += stringListBytes(column.values) + column.offsets.byteLength;
2224
- for (const column of arrays.pattern) bytes += column.runs.byteLength + column.offsets.byteLength;
2225
- const surfaces = segment.sharedSnapshot?.surfaceTable ?? null;
2226
- if (surfaces !== null) bytes += surfaces.blob.byteLength + surfaces.offsets.byteLength + surfaces.counts.byteLength;
2227
- else bytes += messagePackUpperBoundBytes(arrays.readSurfaceForms());
2228
- return bytes + segment.partialWords.bytes();
2229
- }
2230
- function keptDocumentsOf(selection, chargesTokens) {
2231
- const { segment } = selection;
2232
- const count = segment.arrays.documentCount;
2233
- const costs = new Float64Array(count);
2234
- const perDocument = ORDINAL_BYTES * (3 + segment.arrays.fieldLengthColumns.length);
2235
- for (let ordinal = 0; ordinal < count; ordinal++) {
2236
- costs[ordinal] = perDocument + encoder.encode(segment.arrays.idTable.idAt(ordinal)).byteLength;
2237
- }
2238
- chargePostings(segment, costs, chargesTokens);
2239
- chargeColumns(segment, costs);
2240
- const ordinals = [];
2241
- for (let ordinal = 0; ordinal < count; ordinal++) {
2242
- if (!selectionKeeps(selection, ordinal)) continue;
2243
- costs[ordinal] += documentBytes(segment, ordinal);
2244
- ordinals.push(ordinal);
2245
- }
2246
- return { segment, ordinals, costs };
2247
- }
2248
- function selectionOf(segment, ordinals) {
2249
- const marked = new Uint8Array(segment.arrays.documentCount);
2250
- for (const ordinal of ordinals) marked[ordinal] = 1;
2251
- return { segment, keep: (ordinal) => marked[ordinal] === 1 };
2252
- }
2253
- function tooLargeForOneFile(cost, budget) {
2254
- throw new NarsilError(
2255
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
2256
- `A document needs up to ${Math.ceil(cost)} bytes of a segment, more than the ${Math.max(0, Math.floor(budget))} that one file leaves it`,
2257
- { bytes: Math.ceil(cost), budget: Math.floor(budget), maximum: MAX_FILE_BYTES }
2258
- );
2259
- }
2260
- function splitIntoSegmentFiles(selections, schema, tombstones) {
2261
- let tokenBytes = 0;
2262
- for (const { segment } of selections) tokenBytes += segment.arrays.tokenTable.bytes;
2263
- const chargesTokens = tokenBytes > MAX_FILE_BYTES * SEGMENT_TOKEN_TABLE_SHARE_OF_FILE;
2264
- let fixed = HEADER_SIZE + SEGMENT_FILE_FRAMING_SLACK_BYTES + encode(schema).byteLength + encode(tombstones).byteLength;
2265
- for (const { segment } of selections) fixed += fixedBytesOf(segment, chargesTokens);
2266
- const budget = MAX_FILE_BYTES - fixed;
2267
- const groups = [];
2268
- let current = [];
2269
- let currentBytes = 0;
2270
- let currentCount = 0;
2271
- const flush = () => {
2272
- if (currentCount === 0) return;
2273
- groups.push({
2274
- selections: current.map((entry) => selectionOf(entry.segment, entry.ordinals)),
2275
- documentCount: currentCount
2276
- });
2277
- current = [];
2278
- currentBytes = 0;
2279
- currentCount = 0;
2280
- };
2281
- for (const selection of selections) {
2282
- const kept = keptDocumentsOf(selection, chargesTokens);
2283
- let entry = null;
2284
- for (const ordinal of kept.ordinals) {
2285
- const cost = kept.costs[ordinal];
2286
- if (cost > budget) tooLargeForOneFile(cost, budget);
2287
- if (currentBytes + cost > budget) {
2288
- flush();
2289
- entry = null;
2290
- }
2291
- if (entry === null) {
2292
- entry = { segment: kept.segment, ordinals: [] };
2293
- current.push(entry);
2294
- }
2295
- entry.ordinals.push(ordinal);
2296
- currentBytes += cost;
2297
- currentCount += 1;
2298
- }
2299
- }
2300
- flush();
2301
- return groups;
2302
- }
2303
- function halvesOf(group) {
2304
- const flat = [];
2305
- for (const selection of group.selections) {
2306
- for (let ordinal = 0; ordinal < selection.segment.arrays.documentCount; ordinal++) {
2307
- if (!selectionKeeps(selection, ordinal)) continue;
2308
- flat.push({ segment: selection.segment, ordinal });
2309
- }
2310
- }
2311
- const middle = Math.ceil(flat.length / 2);
2312
- const groupOf = (entries) => {
2313
- const bySegment = /* @__PURE__ */ new Map();
2314
- for (const { segment, ordinal } of entries) {
2315
- let ordinals = bySegment.get(segment);
2316
- if (ordinals === void 0) {
2317
- ordinals = [];
2318
- bySegment.set(segment, ordinals);
2319
- }
2320
- ordinals.push(ordinal);
2321
- }
2322
- return {
2323
- selections: [...bySegment].map(([segment, ordinals]) => selectionOf(segment, ordinals)),
2324
- documentCount: entries.length
2325
- };
2326
- };
2327
- return [groupOf(flat.slice(0, middle)), groupOf(flat.slice(middle))];
2328
- }
2329
-
2330
- // src/persistence/durability/segment/segment-write.ts
2331
- function emptyPayload() {
2332
- return {
2333
- documentCount: 0,
2334
- docIds: [],
2335
- fieldNames: [],
2336
- tokens: [],
2337
- postingOffsets: new Uint32Array(1),
2338
- postingDocIds: new Uint32Array(0),
2339
- postingFrequencies: new Uint16Array(0),
2340
- postingFieldIndices: new Uint16Array(0),
2341
- positionOffsets: null,
2342
- positionValues: null,
2343
- fieldLengthNames: [],
2344
- fieldLengthColumns: [],
2345
- totalFieldLengths: {},
2346
- docFrequencies: {},
2347
- surfaceForms: null,
2348
- partialSpellings: null,
2349
- numeric: [],
2350
- boolean: [],
2351
- enums: [],
2352
- geo: [],
2353
- pattern: [],
2354
- sizes: []
2355
- };
2356
- }
2357
- function noSpellingsOf(analysis) {
2358
- if (analysis === null || analysis.fields.length === 0) return null;
2359
- return { fields: [...analysis.fields], tokens: [], spellings: [], counts: [] };
2360
- }
2361
- function writeRecordsOfGroup(group, records, count) {
2362
- const seqNos = new Float64Array(count);
2363
- const primaryTerms = new Float64Array(count);
2364
- let at = 0;
2365
- for (const selection of group?.selections ?? []) {
2366
- const source = records.get(selection.segment);
2367
- for (let ordinal = 0; ordinal < selection.segment.arrays.documentCount; ordinal++) {
2368
- if (!selectionKeeps(selection, ordinal)) continue;
2369
- if (source !== void 0) {
2370
- seqNos[at] = source.seqNos[ordinal];
2371
- primaryTerms[at] = source.primaryTerms[ordinal];
2372
- }
2373
- at += 1;
2374
- }
2375
- }
2376
- return { seqNos, primaryTerms };
2377
- }
2378
- function withWriteRecords(contents, group, request) {
2379
- if (request.writeRecords === void 0) return contents;
2380
- const writeRecords = writeRecordsOfGroup(group, request.writeRecords, contents.arrays.documentCount);
2381
- return { ...contents, writeRecords };
2382
- }
2383
- function contentsOfGroup(group, request, tombstones) {
2384
- const { schema, partialWordAnalysis, textAnalysis } = request;
2385
- if (group === null) {
2386
- const payload = { ...emptyPayload(), partialSpellings: noSpellingsOf(partialWordAnalysis) };
2387
- return {
2388
- schema,
2389
- tombstones,
2390
- arrays: segmentArraysOf(payload, { blob: new Uint8Array(0), offsets: new Uint32Array(1) })
2391
- };
2392
- }
2393
- const merged = mergeSegmentContents(group.selections, partialWordAnalysis, textAnalysis);
2394
- return {
2395
- schema,
2396
- tombstones,
2397
- arrays: segmentArraysOf(merged.payload, merged.documents, merged.reusedPartialWords)
2398
- };
2399
- }
2400
- function requireRoomForAnotherSegment(state) {
2401
- if (state.segments.length < MAX_SEGMENTS_PER_PARTITION) return;
2402
- throw new NarsilError(
2403
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
2404
- `Partition ${state.request.partitionId} would hold more than ${MAX_SEGMENTS_PER_PARTITION} segments`,
2405
- { partitionId: state.request.partitionId, maximum: MAX_SEGMENTS_PER_PARTITION }
2406
- );
2407
- }
2408
- async function writeGroup(state, group) {
2409
- const { request } = state;
2410
- const tombstones = state.tombstonesWritten ? [] : [...request.tombstones];
2411
- const contents = withWriteRecords(contentsOfGroup(group, request, tombstones), group, request);
2412
- const encoded = encodeKeywordSegment(contents);
2413
- if (HEADER_SIZE + encoded.byteLength > MAX_FILE_BYTES && group !== null && group.documentCount > 1) {
2414
- const [first, second] = halvesOf(group);
2415
- await writeGroup(state, first);
2416
- await writeGroup(state, second);
2417
- return;
2418
- }
2419
- requireRoomForAnotherSegment(state);
2420
- const id = state.nextSegmentId;
2421
- const key = segmentKey(request.indexName, request.partitionId, id);
2422
- const packed = await packChecksummedEnvelope(encoded.chunks, KEYWORD_SEGMENT_ENVELOPE_VERSION, `segment "${key}"`);
2423
- await request.store.write(key, packed.chunks);
2424
- state.segments.push({
2425
- id,
2426
- key,
2427
- bytes: packed.byteLength,
2428
- checksum: packed.checksum,
2429
- docCount: contents.arrays.documentCount,
2430
- tombstoneCount: tombstones.length
2431
- });
2432
- state.nextSegmentId = id + 1;
2433
- state.tombstonesWritten = true;
2434
- }
2435
- function keptDocumentCount(selections) {
2436
- let count = 0;
2437
- for (const selection of selections) {
2438
- for (let ordinal = 0; ordinal < selection.segment.arrays.documentCount; ordinal++) {
2439
- if (selectionKeeps(selection, ordinal)) count += 1;
2440
- }
2441
- }
2442
- return count;
2443
- }
2444
- async function writeSegmentsOf(request) {
2445
- const state = { request, segments: [], nextSegmentId: request.nextSegmentId, tombstonesWritten: false };
2446
- const groups = request.mergesIntoOneFile === true ? [{ selections: [...request.selections], documentCount: keptDocumentCount(request.selections) }] : splitIntoSegmentFiles(request.selections, request.schema, request.tombstones);
2447
- for (const group of groups) {
2448
- if (group.documentCount > 0) await writeGroup(state, group);
2449
- }
2450
- if (!state.tombstonesWritten && request.tombstones.length > 0) await writeGroup(state, null);
2451
- return { segments: state.segments, nextSegmentId: state.nextSegmentId };
2452
- }
2453
2066
 
2454
2067
  // src/persistence/durability/segment/compaction.ts
2455
- function windowFits(segments, start, windowLength) {
2068
+ function windowFits(segments, start, windowLength, paged) {
2456
2069
  let documents = 0;
2457
2070
  let bytes = SEGMENT_FILE_FRAMING_SLACK_BYTES;
2458
2071
  for (let at = start; at < start + windowLength; at++) {
2459
2072
  documents += segments[at].docCount;
2460
2073
  bytes += segments[at].bytes;
2461
2074
  }
2075
+ if (paged) return bytes * MERGED_SEGMENT_GROWTH_ALLOWANCE <= MERGED_SEGMENT_TARGET_BYTES;
2462
2076
  return documents <= COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT && bytes <= MAX_FILE_BYTES;
2463
2077
  }
2464
- function smallestFittingWindow(segments, windowLength) {
2078
+ function smallestFittingWindow(segments, windowLength, paged) {
2465
2079
  let best = null;
2466
2080
  let bestBytes = Number.POSITIVE_INFINITY;
2467
2081
  for (let start = 0; start + windowLength <= segments.length; start++) {
2468
- if (!windowFits(segments, start, windowLength)) continue;
2082
+ if (!windowFits(segments, start, windowLength, paged)) continue;
2469
2083
  let bytes = 0;
2470
2084
  for (let at = start; at < start + windowLength; at++) bytes += segments[at].bytes;
2471
2085
  if (bytes < bestBytes) {
@@ -2475,9 +2089,9 @@ function smallestFittingWindow(segments, windowLength) {
2475
2089
  }
2476
2090
  return best;
2477
2091
  }
2478
- function chooseWindow(segments, wantedLength) {
2092
+ function chooseWindow(segments, wantedLength, paged) {
2479
2093
  for (let windowLength = Math.min(wantedLength, segments.length); windowLength >= 2; windowLength--) {
2480
- const window = smallestFittingWindow(segments, windowLength);
2094
+ const window = smallestFittingWindow(segments, windowLength, paged);
2481
2095
  if (window !== null) return window;
2482
2096
  }
2483
2097
  return null;
@@ -2518,11 +2132,39 @@ async function compactPartitionSegments(input) {
2518
2132
  if (segments.length <= compactionThreshold) {
2519
2133
  return { segments, nextSegmentId: input.nextSegmentId };
2520
2134
  }
2521
- const window = chooseWindow(segments, segments.length - compactionThreshold + 1);
2135
+ const paged = input.paged === true;
2136
+ const window = chooseWindow(segments, segments.length - compactionThreshold + 1, paged);
2522
2137
  if (window === null) {
2523
2138
  return { segments, nextSegmentId: input.nextSegmentId };
2524
2139
  }
2525
2140
  const windowRefs = segments.slice(window.start, window.start + window.windowLength);
2141
+ const replacing = (written2) => ({
2142
+ segments: [
2143
+ ...segments.slice(0, window.start),
2144
+ ...written2.segments,
2145
+ ...segments.slice(window.start + window.windowLength)
2146
+ ],
2147
+ nextSegmentId: written2.nextSegmentId
2148
+ });
2149
+ if (paged) {
2150
+ const { directory } = input.store;
2151
+ if (directory === void 0) return { segments, nextSegmentId: input.nextSegmentId };
2152
+ const { mergePagedWindow } = await import('./module-BUFNZMEU.mjs');
2153
+ return replacing(
2154
+ await mergePagedWindow({
2155
+ store: input.store,
2156
+ directory,
2157
+ indexName: input.indexName,
2158
+ partitionId: input.partitionId,
2159
+ schema: input.schema,
2160
+ partialWordAnalysis: input.partialWordAnalysis,
2161
+ textAnalysis: input.textAnalysis,
2162
+ nextSegmentId: input.nextSegmentId,
2163
+ refs: windowRefs,
2164
+ dropsEveryTombstone: window.start === 0
2165
+ })
2166
+ );
2167
+ }
2526
2168
  const contents = [];
2527
2169
  for (const ref of windowRefs) contents.push(await readSegmentFile(input.store, ref));
2528
2170
  const frozen = contents.map(frozenOf);
@@ -2541,14 +2183,7 @@ async function compactPartitionSegments(input) {
2541
2183
  mergesIntoOneFile: true,
2542
2184
  ...writeRecords === void 0 ? {} : { writeRecords }
2543
2185
  });
2544
- return {
2545
- segments: [
2546
- ...segments.slice(0, window.start),
2547
- ...written.segments,
2548
- ...segments.slice(window.start + window.windowLength)
2549
- ],
2550
- nextSegmentId: written.nextSegmentId
2551
- };
2186
+ return replacing(written);
2552
2187
  }
2553
2188
 
2554
2189
  // src/persistence/durability/segment/write.ts
@@ -2595,6 +2230,21 @@ function wholeContentsOf(input, partitionId) {
2595
2230
  const captured = input.capturedPartitions?.find((candidate) => candidate.partitionId === partitionId);
2596
2231
  return captured === void 0 ? void 0 : contentsOfCaptured(captured);
2597
2232
  }
2233
+ function pagedSettingsOf(context) {
2234
+ if (context.input.metadata.textStorage !== "disk") return {};
2235
+ return { paged: true, keepsWriteRecords: context.input.keepsWriteRecords === true };
2236
+ }
2237
+ async function tombstonesOfDelta(context, older, selections, deleted) {
2238
+ const { directory } = context.input.store;
2239
+ if (context.input.metadata.textStorage !== "disk" || directory === void 0) return deleted;
2240
+ const { pagedTombstonesFor } = await import('./module-BUFNZMEU.mjs');
2241
+ return pagedTombstonesFor(
2242
+ directory,
2243
+ older,
2244
+ selections.map((selection) => selection.segment),
2245
+ deleted
2246
+ );
2247
+ }
2598
2248
  async function writeWholePartition(context, partitionId, priorPartition, whole) {
2599
2249
  const written = await writeSegmentsOf({
2600
2250
  store: context.input.store,
@@ -2606,7 +2256,8 @@ async function writeWholePartition(context, partitionId, priorPartition, whole)
2606
2256
  textAnalysis: context.textAnalysis,
2607
2257
  selections: whole.selections,
2608
2258
  tombstones: [],
2609
- ...whole.writeRecords === void 0 ? {} : { writeRecords: whole.writeRecords }
2259
+ ...whole.writeRecords === void 0 ? {} : { writeRecords: whole.writeRecords },
2260
+ ...pagedSettingsOf(context)
2610
2261
  });
2611
2262
  return { partitionId, nextSegmentId: written.nextSegmentId, segments: written.segments };
2612
2263
  }
@@ -2622,6 +2273,7 @@ async function writeLoggedChanges(context, partitionId, priorPartition, entries)
2622
2273
  let segments = priorPartition ? [...priorPartition.segments] : [];
2623
2274
  let nextSegmentId = priorPartition?.nextSegmentId ?? 0;
2624
2275
  if (built !== null) {
2276
+ const tombstones = await tombstonesOfDelta(context, segments, built.selections, built.tombstones);
2625
2277
  const written = await writeSegmentsOf({
2626
2278
  store: context.input.store,
2627
2279
  indexName: context.indexName,
@@ -2631,8 +2283,9 @@ async function writeLoggedChanges(context, partitionId, priorPartition, entries)
2631
2283
  partialWordAnalysis: context.partialWordAnalysis,
2632
2284
  textAnalysis: context.textAnalysis,
2633
2285
  selections: built.selections,
2634
- tombstones: built.tombstones,
2635
- ...built.writeRecords === void 0 ? {} : { writeRecords: built.writeRecords }
2286
+ tombstones,
2287
+ ...built.writeRecords === void 0 ? {} : { writeRecords: built.writeRecords },
2288
+ ...pagedSettingsOf(context)
2636
2289
  });
2637
2290
  segments.push(...written.segments);
2638
2291
  nextSegmentId = written.nextSegmentId;
@@ -2653,7 +2306,8 @@ async function writeLoggedChanges(context, partitionId, priorPartition, entries)
2653
2306
  textAnalysis: context.textAnalysis,
2654
2307
  segments,
2655
2308
  nextSegmentId,
2656
- compactionThreshold: context.compactionThreshold
2309
+ compactionThreshold: context.compactionThreshold,
2310
+ paged: context.input.metadata.textStorage === "disk"
2657
2311
  });
2658
2312
  segments = compacted.segments;
2659
2313
  nextSegmentId = compacted.nextSegmentId;
@@ -2716,6 +2370,7 @@ async function writeCheckpointSegments(input) {
2716
2370
  async function commitCheckpointManifest(store, metadata, written, vectors) {
2717
2371
  const indexName = metadata.indexName;
2718
2372
  const manifest = {
2373
+ version: SEGMENT_MANIFEST_VERSION,
2719
2374
  schema: metadata.schema,
2720
2375
  language: metadata.language,
2721
2376
  ...metadata.analysisGeneration ? { analysisGeneration: metadata.analysisGeneration } : {},
@@ -2769,8 +2424,41 @@ function removeWhatTheSegmentReplaces(manager, segment, contents) {
2769
2424
  function yieldToEventLoop() {
2770
2425
  return new Promise((resolve) => setTimeout(resolve, 0));
2771
2426
  }
2427
+ function removeFromPartition(manager, partitionId, docIds) {
2428
+ const { composite, docPartitionMap, insertOptions } = manager.diskSegmentAccess(partitionId);
2429
+ composite.beginBatch();
2430
+ try {
2431
+ for (const docId of docIds) {
2432
+ if (!composite.has(docId)) continue;
2433
+ composite.remove(docId, manager.config.schema, manager.language, insertOptions);
2434
+ docPartitionMap.delete(docId);
2435
+ }
2436
+ } finally {
2437
+ composite.endBatch();
2438
+ }
2439
+ }
2440
+ async function attachPagedSegment(directory, ref, partitionId, manager, options) {
2441
+ const { checkPagedSegmentFully, createPagedFrozenSegment, openPagedSnapshot, pagedListingOf } = await import('./module-BUFNZMEU.mjs');
2442
+ if (options.decodesEveryDocument) await checkPagedSegmentFully(directory, ref);
2443
+ const snapshot = openPagedSnapshot(await pagedListingOf(directory, ref));
2444
+ requireAgreedSchema(options.schema, snapshot.schema, ref.key);
2445
+ const segment = createPagedFrozenSegment(snapshot);
2446
+ removeFromPartition(manager, partitionId, snapshot.tombstones);
2447
+ options.writeRecords?.loadPagedSegment(manager.indexName, partitionId, segment);
2448
+ if (snapshot.documentCount > 0) manager.attachFrozenSegment(partitionId, segment);
2449
+ else segment.paged?.close();
2450
+ }
2772
2451
  async function loadPartitionSegments(store, partition, manager, options) {
2452
+ const { directory } = store;
2773
2453
  for (const ref of partition.segments) {
2454
+ if (manager.config.textStorage === "disk" && directory !== void 0) {
2455
+ const { isPagedSegmentFile } = await import('./module-BUFNZMEU.mjs');
2456
+ if (await isPagedSegmentFile(directory, ref.key)) {
2457
+ await attachPagedSegment(directory, ref, partition.partitionId, manager, options);
2458
+ await yieldToEventLoop();
2459
+ continue;
2460
+ }
2461
+ }
2774
2462
  const contents = await readSegmentFile(store, ref);
2775
2463
  requireAgreedSchema(options.schema, contents.schema, ref.key);
2776
2464
  if (options.decodesEveryDocument) decodeEveryStoredDocument(contents.arrays.documentTable);
@@ -2815,6 +2503,14 @@ async function loadSegmentedCheckpoint(store, indexName, manifest, deps, decodes
2815
2503
  if (partition.partitionId > highestPartitionId) highestPartitionId = partition.partitionId;
2816
2504
  }
2817
2505
  growPartitionsTo(deps.manager, highestPartitionId);
2506
+ const { directory } = store;
2507
+ const opensVectorsOnDemand = manifest.vectors.some(
2508
+ (field) => deps.vectorIndexes.get(field.fieldPath)?.storage === "disk" && field.files.some((file) => file.headOffset !== void 0)
2509
+ );
2510
+ if (!decodesEveryDocument && directory !== void 0 && opensVectorsOnDemand) {
2511
+ const { startReadingOpeningSectionsAhead } = await import('./module-BUFNZMEU.mjs');
2512
+ await startReadingOpeningSectionsAhead(directory, manifest);
2513
+ }
2818
2514
  for (const partition of manifest.partitions) {
2819
2515
  if (partition.partitionId >= deps.manager.partitionCount) {
2820
2516
  throw new NarsilError(
@@ -2832,7 +2528,12 @@ async function loadSegmentedCheckpoint(store, indexName, manifest, deps, decodes
2832
2528
  for (const vector of manifest.vectors) {
2833
2529
  const vectorIndex = deps.vectorIndexes.get(vector.fieldPath);
2834
2530
  if (vectorIndex === void 0) continue;
2835
- await loadVectorField(store, vector, vectorIndex);
2531
+ await loadVectorField(
2532
+ store,
2533
+ vector,
2534
+ vectorIndex,
2535
+ decodesEveryDocument ? null : (docId) => deps.manager.partitionIdOf(docId)
2536
+ );
2836
2537
  }
2837
2538
  return manifest.checkpoint;
2838
2539
  }
@@ -2941,4 +2642,4 @@ function deserializePayloadV2(data) {
2941
2642
  };
2942
2643
  }
2943
2644
 
2944
- export { CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_IDLE_MS, CHECKPOINT_WORKER_TIMEOUT_MS, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_SEGMENT_MAX_BYTES, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, DIRECTORY_LOCK_ATTEMPTS, DIRECTORY_LOCK_SETTLE_MS, LOG_RECORDS_PER_CHECKPOINT, MAX_SEGMENTS_PER_PARTITION, SEGMENT_HEADER_SIZE, SINGLE_NODE_PRIMARY_TERM, VECTOR_FILE_ENVELOPE_VERSION, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, adapterCheckpointStore, buildEntry, captureWholePartition, checkpointLastSeqNo, checkpointPrimaryTerm, commitCheckpointManifest, createSeqOwner, decodeManifestPayload, decodeSegmentManifest, deserializePayloadV1, deserializePayloadV2, directoryCheckpointStore, documentsOfPayloadV2, durableMetadataConfigOf, durableRecordMissing, encodeMarkerSlot, encodeSegmentManifest, frameRecord, growPartitionsTo, indexMetadataOf, invalidManifest, isId, isRecord, isSequenceNumber, isUint32, listedKeysOf, loadSegmentedCheckpoint, manifestFileRefs, manifestKey, manifestReferencedKeys, markerSlotOffset, metadataConfigOf, metadataFromConfig, nextMarkerSlot, openManifestPayload, partitionContentsOf, readCommitMarker, readCurrentManifest, readGraphParts, readListedFile, readMetadataEnvelope, readSegmentFile, readSegmentManifest, readVectorFile, readWalSegments, reclaimUnlistedKeys, removeCheckpointGarbage, snapshotBundleKey, unpackEnvelopeBytes, unpackIndexSnapshotEnvelope, writeCheckpointSegments, writeLiveVectors, writeMetadataEnvelope, writeSegmentHeader };
2645
+ export { SEGMENT_HEADER_SIZE, SINGLE_NODE_PRIMARY_TERM, VECTOR_FILE_ENVELOPE_VERSION, buildEntry, captureWholePartition, checkpointLastSeqNo, checkpointPrimaryTerm, commitCheckpointManifest, createSeqOwner, decodeManifestPayload, decodeSegmentManifest, deserializePayloadV1, deserializePayloadV2, documentsOfPayloadV2, durableMetadataConfigOf, durableRecordMissing, encodeMarkerSlot, frameRecord, growPartitionsTo, indexMetadataOf, invalidManifest, isId, isRecord, isSequenceNumber, isUint32, listedKeysOf, loadSegmentedCheckpoint, markerSlotOffset, metadataConfigOf, metadataFromConfig, nextMarkerSlot, nextSegmentKey, openManifestPayload, partitionContentsOf, readCommitMarker, readCurrentManifest, readGraphParts, readMetadataEnvelope, readSegmentManifest, readVectorFile, readWalSegments, reclaimUnlistedKeys, removeCheckpointGarbage, unpackEnvelopeBytes, unpackIndexSnapshotEnvelope, writeCheckpointSegments, writeLiveVectors, writeMetadataEnvelope, writeSegmentHeader };