@delali/narsil 0.3.1-next.140 → 0.3.1-next.143

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. package/dist/{analysis-change-JN23AZQP.mjs → analysis-change-HQUT2Z3I.mjs} +13 -13
  2. package/dist/{analysis-change-entry-5JSIMEKG.mjs → analysis-change-entry-TZTG4AZB.mjs} +1 -1
  3. package/dist/browser/{analysis-change-57MIRJWR.mjs → analysis-change-JPV775LR.mjs} +407 -6
  4. package/dist/browser/{analysis-change-entry-ICGRR3IN.mjs → analysis-change-entry-7HR4LWL3.mjs} +1 -1
  5. package/dist/browser/chunk-3ZBJYAS4.mjs +19 -0
  6. package/dist/browser/{chunk-ZAIZQOTM.mjs → chunk-5DB2VDGI.mjs} +951 -1134
  7. package/dist/browser/chunk-6RHTAVZX.mjs +656 -0
  8. package/dist/browser/chunk-CCOAOO73.mjs +86 -0
  9. package/dist/browser/{chunk-3EYDURFD.mjs → chunk-HKX2G6WH.mjs} +1615 -2266
  10. package/dist/browser/{chunk-CGVBGHBX.mjs → chunk-KTOK275N.mjs} +531 -830
  11. package/dist/browser/{chunk-5LYWZJE2.mjs → chunk-LVY2LI35.mjs} +1 -1
  12. package/dist/browser/chunk-MVM33ZUL.mjs +993 -0
  13. package/dist/browser/chunk-MWUK3DA5.mjs +2634 -0
  14. package/dist/browser/{chunk-67KSBUYR.mjs → chunk-R4WUDZQV.mjs} +8 -5
  15. package/dist/browser/{chunk-6BSB4N6B.mjs → chunk-TCLEOZ4P.mjs} +2 -2
  16. package/dist/browser/chunk-VFYMPCOV.mjs +212 -0
  17. package/dist/browser/chunk-WCFCHIBB.mjs +177 -0
  18. package/dist/browser/{chunk-E5SFTOQR.mjs → chunk-XNWPP5NS.mjs} +2 -2
  19. package/dist/browser/{chunk-QHNNULPW.mjs → chunk-ZEV26AQ6.mjs} +5 -0
  20. package/dist/browser/cold-module-MJMKNUYJ.mjs +745 -0
  21. package/dist/browser/{durability-wiring-MTIAN6VT.mjs → durability-wiring-SNOWIT6Z.mjs} +231 -51
  22. package/dist/browser/{import-Q7MXD6BS.mjs → import-TLB2A5LJ.mjs} +6 -5
  23. package/dist/browser/module-BUFNZMEU.mjs +12 -0
  24. package/dist/browser/{module-5J3PEUEB.mjs → module-BVP4TKZU.mjs} +395 -108
  25. package/dist/{restore-queue-RYSRRFS4.mjs → browser/restore-queue-ONYNYUDE.mjs} +2 -2
  26. package/dist/browser/{snapshot-YBJ5MF2M.mjs → snapshot-QXBKHJAG.mjs} +108 -25
  27. package/dist/browser/{write-RTWI5MXL.mjs → write-WHFBDWQS.mjs} +63 -91
  28. package/dist/{capabilities-DXAmysqJ.d.ts → capabilities-BCdUwK_L.d.ts} +2 -2
  29. package/dist/{chunk-HMDT3I63.mjs → chunk-46NZOPHS.mjs} +2 -2
  30. package/dist/{chunk-XXKWEG53.mjs → chunk-5WYVUPGD.mjs} +1 -1
  31. package/dist/chunk-6SNBQESJ.mjs +214 -0
  32. package/dist/chunk-7AUASRE7.mjs +653 -0
  33. package/dist/{chunk-4GCN34JV.mjs → chunk-AST3SHFA.mjs} +26 -24
  34. package/dist/chunk-BOJZGUS6.mjs +177 -0
  35. package/dist/{chunk-IP4HJOGC.mjs → chunk-BRUUBXCB.mjs} +4 -3
  36. package/dist/{chunk-J3EHATEZ.mjs → chunk-DQNKVOJ2.mjs} +63 -63
  37. package/dist/chunk-FOCV4CBZ.mjs +87 -0
  38. package/dist/{chunk-FFVLPCLH.mjs → chunk-GLZHNSGB.mjs} +1 -1
  39. package/dist/chunk-GWOS5DYC.mjs +2637 -0
  40. package/dist/{chunk-W2XDK66Z.mjs → chunk-IO5F7ZBE.mjs} +112 -438
  41. package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
  42. package/dist/{chunk-LLDFIKGP.mjs → chunk-KEIYVZUX.mjs} +4 -4
  43. package/dist/{chunk-UBXJQJII.mjs → chunk-KGOXHLLZ.mjs} +5 -1
  44. package/dist/{chunk-FKARMUWH.mjs → chunk-KZL72VTC.mjs} +68 -16
  45. package/dist/{chunk-TCWQJL7O.mjs → chunk-L76W3IZJ.mjs} +5 -0
  46. package/dist/{chunk-CPCGEIOM.mjs → chunk-LA4WNDHL.mjs} +83 -799
  47. package/dist/{chunk-PNNVTKMT.mjs → chunk-NJJCPFXN.mjs} +2 -2
  48. package/dist/chunk-NN33CZRE.mjs +86 -0
  49. package/dist/chunk-NRUXBYPH.mjs +121 -0
  50. package/dist/{chunk-UJWGQIXU.mjs → chunk-QHPYS427.mjs} +1 -0
  51. package/dist/{chunk-VSUYTPME.mjs → chunk-RHKRTYGO.mjs} +13 -4
  52. package/dist/{chunk-UEV53BBO.mjs → chunk-SRJBFG2T.mjs} +406 -5
  53. package/dist/{chunk-K3AQOZVM.mjs → chunk-TNBIK2LQ.mjs} +1638 -1644
  54. package/dist/chunk-UUZCQQG2.mjs +209 -0
  55. package/dist/{chunk-XMQNOD5V.mjs → chunk-VEFN6X6Q.mjs} +2 -2
  56. package/dist/{chunk-LK3MOTXF.mjs → chunk-VJEGOELZ.mjs} +8 -5
  57. package/dist/chunk-W2COJ5AH.mjs +148 -0
  58. package/dist/{chunk-IBATGS3J.mjs → chunk-WJGO5PGV.mjs} +241 -124
  59. package/dist/{chunk-CCCGEPJM.mjs → chunk-XZ3SRNDJ.mjs} +24 -63
  60. package/dist/{chunk-MTQTX4BP.mjs → chunk-YHUTF6EA.mjs} +400 -92
  61. package/dist/{chunk-B5I3IL4H.mjs → chunk-YLNSJQLX.mjs} +4 -4
  62. package/dist/{chunk-6VHWXAOI.mjs → chunk-Z4HFMYNX.mjs} +29 -24
  63. package/dist/{chunk-U4X4AWKL.mjs → chunk-ZPWS2SKJ.mjs} +293 -270
  64. package/dist/{client-DApBeLbk.d.ts → client-B12YFALP.d.ts} +3 -3
  65. package/dist/client.d.ts +7 -7
  66. package/dist/cold-module-4X7WJCFM.mjs +783 -0
  67. package/dist/{config-B0H-vp4U.d.ts → config-BhZxIeb1.d.ts} +6 -1
  68. package/dist/{config-Brysl9NO.d.ts → config-CIG5yDyg.d.ts} +2 -2
  69. package/dist/{direct-executor-J5UDLPL3.mjs → direct-executor-7XOQ7VQP.mjs} +5 -5
  70. package/dist/distribution/coordinator/etcd.d.ts +2 -2
  71. package/dist/distribution/coordinator/in-memory.d.ts +2 -2
  72. package/dist/distribution.d.ts +7 -7
  73. package/dist/distribution.mjs +123 -42
  74. package/dist/{durability-wiring-AYOTAPNQ.mjs → durability-wiring-324D2DDD.mjs} +213 -40
  75. package/dist/{engine-CuLMbH1-.d.ts → engine-CtzbHOmG.d.ts} +2 -2
  76. package/dist/{import-QRZAQ5RH.mjs → import-OOBUW3RR.mjs} +14 -13
  77. package/dist/{index-HYnqS1SE.d.ts → index-DsEurhSl.d.ts} +1 -1
  78. package/dist/index.browser.mjs +3 -3
  79. package/dist/index.d.ts +6 -6
  80. package/dist/index.mjs +11 -11
  81. package/dist/{long-query-thread-U7K5A72G.mjs → long-query-thread-4SW7RWDE.mjs} +8 -8
  82. package/dist/{module-L2OZO5PQ.mjs → module-DXSJ5LT2.mjs} +374 -83
  83. package/dist/module-IYPLWHGC.mjs +48 -0
  84. package/dist/paged-partition-archive-HFP45PLU.mjs +269 -0
  85. package/dist/partial-words.mjs +2 -2
  86. package/dist/persistence/durability/checkpoint-worker.d.ts +5 -1
  87. package/dist/persistence/durability/checkpoint-worker.mjs +17 -15
  88. package/dist/react.d.ts +9 -9
  89. package/dist/{browser/restore-queue-DISFECE4.mjs → restore-queue-FEYAC4SB.mjs} +2 -2
  90. package/dist/{schema-BTXbHCqA.d.ts → schema-B-Runv0n.d.ts} +7 -0
  91. package/dist/server.d.ts +10 -10
  92. package/dist/server.mjs +8 -7
  93. package/dist/{snapshot-BTRZ3MZK.mjs → snapshot-CGM65VCD.mjs} +60 -25
  94. package/dist/{tasks-DjGjK70Y.d.ts → tasks-CjRJRGTb.d.ts} +6 -4
  95. package/dist/{thread-USB3GOPY.mjs → thread-KGNAIKPD.mjs} +8 -8
  96. package/dist/vector/search-worker.d.ts +1 -1
  97. package/dist/vector/search-worker.mjs +3 -2
  98. package/dist/workers/entry.mjs +7 -7
  99. package/dist/{write-GHTA56OW.mjs → write-2DZPEQL2.mjs} +70 -98
  100. package/openapi.json +13 -0
  101. package/package.json +12 -8
  102. package/dist/browser/chunk-XVMOQQ34.mjs +0 -384
  103. package/dist/chunk-BDIDUW2B.mjs +0 -387
  104. package/dist/browser/{chunk-4MFH226Q.mjs → chunk-D33A5VMU.mjs} +15 -15
@@ -1,233 +1,19 @@
1
- import { KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, restoreVectorFields, KEYWORD_SEGMENT_PAYLOAD_VERSION, littleEndianBytes, uint64Bytes } from './chunk-LLDFIKGP.mjs';
2
- import { WIRE_BATCH_BUDGET } from './chunk-P36MUFEJ.mjs';
1
+ import { WIRE_BATCH_BUDGET } from './chunk-JDDXV6UF.mjs';
2
+ import { DEFAULT_COMPACTION_THRESHOLD, writeSegmentsOf, readSegmentFile, MERGED_SEGMENT_GROWTH_ALLOWANCE, MERGED_SEGMENT_TARGET_BYTES, COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT, SEGMENT_FILE_FRAMING_SLACK_BYTES } from './chunk-7AUASRE7.mjs';
3
+ import { encodeSegmentManifest, SEGMENT_MANIFEST_VERSION, manifestKey, manifestReferencedKeys, segmentsPrefix, snapshotBundleKey, MAX_SEGMENTS_PER_PARTITION } from './chunk-UUZCQQG2.mjs';
4
+ import { restoreVectorFields } from './chunk-KEIYVZUX.mjs';
3
5
  import { ReplicationMessageTypes } from './chunk-YH5OIL7B.mjs';
4
- import { packChecksummedEnvelope, crc32, openChecksummedEnvelope, reconstructSchemaFromMetadata, HEADER_SIZE, prepareDocumentVectors } from './chunk-W2XDK66Z.mjs';
5
- import { freezeSegmentShared, createSharedFrozenSegment, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, selectionKeeps, decodeMapWithoutFields, shareSegmentArrays, segmentArraysOf, mergeSegmentContents } from './chunk-IBATGS3J.mjs';
6
- import { fnv1a, isCompositePartition, survivorDocumentsOf, partitionStateOf } from './chunk-U4X4AWKL.mjs';
6
+ import { crc32, reconstructSchemaFromMetadata, prepareDocumentVectors } from './chunk-IO5F7ZBE.mjs';
7
+ import { freezeSegmentShared, createSharedFrozenSegment, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, decodeMapWithoutFields, shareSegmentArrays } from './chunk-WJGO5PGV.mjs';
8
+ import { isCompositePartition, survivorDocumentsOf, partitionStateOf } from './chunk-ZPWS2SKJ.mjs';
7
9
  import { generateId } from './chunk-N37VMC62.mjs';
8
- import { extractVectorFieldsFromSchema, partialFieldsOf, messagePackUpperBoundBytes } from './chunk-UBXJQJII.mjs';
10
+ import { extractVectorFieldsFromSchema, partialFieldsOf } from './chunk-KGOXHLLZ.mjs';
9
11
  import { MAX_FILE_BYTES } from './chunk-U476IXKQ.mjs';
10
12
  import { resolveIndexAnalysis } from './chunk-O4LJQI4G.mjs';
11
13
  import { getLanguage } from './chunk-WNBRW4Q2.mjs';
12
14
  import { NarsilError, ErrorCodes } from './chunk-HNHCNTVG.mjs';
13
15
  import { encode, decode } from '@msgpack/msgpack';
14
16
 
15
- // src/persistence/durability/constants.ts
16
- var DEFAULT_CHECKPOINT_INTERVAL_MS = 3e5;
17
- var DEFAULT_CHECKPOINT_MUTATION_THRESHOLD = 1e5;
18
- var DEFAULT_ASYNC_FLUSH_INTERVAL_MS = 1e3;
19
- var DIRECTORY_IDENTITY_CHECK_INTERVAL_MS = 1e3;
20
- var DIRECTORY_LOCK_ATTEMPTS = 3;
21
- var DIRECTORY_LOCK_SETTLE_MS = 50;
22
- var DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS = 1e3;
23
- var DEFAULT_SEGMENT_MAX_BYTES = 67108864;
24
- var DEFAULT_COMPACTION_THRESHOLD = 12;
25
- var COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT = 2e5;
26
- var SEGMENT_FILE_FRAMING_SLACK_BYTES = 65536;
27
- var SEGMENT_TOKEN_TABLE_SHARE_OF_FILE = 0.25;
28
- var WHOLE_PARTITION_MIN_CHANGED_SHARE = 0.5;
29
- var WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT = 1e5;
30
- var LOG_RECORDS_PER_CHECKPOINT = 1e5;
31
- var WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES = 2147483648;
32
- var CHECKPOINT_WORKER_TIMEOUT_MS = 6e5;
33
- var CHECKPOINT_WORKER_HEARTBEAT_MS = 5e3;
34
- var CHECKPOINT_WORKER_IDLE_MS = 5e3;
35
- var CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS = 50;
36
-
37
- // src/persistence/durability/segment/layout.ts
38
- var SEGMENT_ID_WIDTH = 16;
39
- function manifestKey(indexName) {
40
- return `${indexName}/manifest`;
41
- }
42
- function snapshotBundleKey(indexName) {
43
- return `${indexName}/snapshot`;
44
- }
45
- function segmentPrefix(indexName, partitionId) {
46
- return `${indexName}/segments/${partitionId}/`;
47
- }
48
- function segmentKey(indexName, partitionId, segmentId) {
49
- return `${segmentPrefix(indexName, partitionId)}s${formatSegmentId(segmentId)}`;
50
- }
51
- function segmentsPrefix(indexName) {
52
- return `${indexName}/segments/`;
53
- }
54
- function vectorFileKey(indexName, fieldPath, fileId) {
55
- return `${segmentsPrefix(indexName)}vec-${encodeFieldPath(fieldPath)}-f${formatSegmentId(fileId)}`;
56
- }
57
- function requireGeneration(generation) {
58
- if (Number.isSafeInteger(generation) && generation > 0) return;
59
- throw new NarsilError(
60
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
61
- `Vector graph generation ${generation} must be a positive integer`,
62
- { generation }
63
- );
64
- }
65
- function graphPartKey(indexName, fieldPath, generation, part) {
66
- requireGeneration(generation);
67
- return `${segmentsPrefix(indexName)}vec-${encodeFieldPath(fieldPath)}-graph-g${generation}-p${formatSegmentId(part)}`;
68
- }
69
- function formatSegmentId(segmentId) {
70
- if (!Number.isInteger(segmentId) || segmentId < 0) {
71
- throw new NarsilError(
72
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
73
- `Segment id ${segmentId} must be a non-negative integer`,
74
- { segmentId }
75
- );
76
- }
77
- return segmentId.toString().padStart(SEGMENT_ID_WIDTH, "0");
78
- }
79
- var FIELD_PATH_PATTERN = /^[A-Za-z0-9_.]+$/;
80
- function encodeFieldPath(fieldPath) {
81
- if (!FIELD_PATH_PATTERN.test(fieldPath)) {
82
- throw new NarsilError(
83
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
84
- `Vector field path "${fieldPath}" contains characters that cannot be encoded in a segment key`,
85
- { fieldPath }
86
- );
87
- }
88
- const fingerprint = fnv1a(fieldPath).toString(36);
89
- return `${fieldPath.replace(/\./g, "_")}-${fingerprint}`;
90
- }
91
- var SEGMENT_MANIFEST_VERSION = 7;
92
- var EARLIER_SEGMENT_MANIFEST_VERSION = 6;
93
- var MANIFEST_ENVELOPE_VERSION = 2;
94
- var MAX_SEGMENTS_PER_PARTITION = 65536;
95
- var MAX_SEGMENT_ID = 281474976710655;
96
- function fileRefOnTheWire(ref) {
97
- return { key: ref.key, bytes: ref.bytes, checksum: ref.checksum };
98
- }
99
- function encodeSegmentManifest(manifest) {
100
- const payload = encode({
101
- version: SEGMENT_MANIFEST_VERSION,
102
- schema: manifest.schema,
103
- language: manifest.language,
104
- ...manifest.analysisGeneration ? { analysisGeneration: manifest.analysisGeneration } : {},
105
- checkpoint: manifest.checkpoint.map((c) => ({
106
- partitionId: c.partitionId,
107
- lastSeqNo: c.lastSeqNo,
108
- primaryTerm: c.primaryTerm,
109
- ...c.termHistory === void 0 ? {} : {
110
- termHistory: c.termHistory.map((start) => ({ primaryTerm: start.primaryTerm, firstSeqNo: start.firstSeqNo }))
111
- }
112
- })),
113
- partitions: manifest.partitions.map((p) => ({
114
- partitionId: p.partitionId,
115
- nextSegmentId: p.nextSegmentId,
116
- segments: p.segments.map((s) => ({
117
- id: s.id,
118
- key: s.key,
119
- docCount: s.docCount,
120
- tombstoneCount: s.tombstoneCount,
121
- bytes: s.bytes,
122
- checksum: s.checksum
123
- }))
124
- })),
125
- vectors: manifest.vectors.map((v) => ({
126
- fieldPath: v.fieldPath,
127
- nextFileId: v.nextFileId,
128
- files: v.files.map((f) => ({ id: f.id, ...fileRefOnTheWire(f), count: f.count, dead: f.dead })),
129
- graphGeneration: v.graphGeneration,
130
- graphParts: v.graphParts.map(fileRefOnTheWire)
131
- }))
132
- });
133
- return packChecksummedEnvelope([payload], MANIFEST_ENVELOPE_VERSION, "segment manifest");
134
- }
135
- function manifestReferencedKeys(manifest) {
136
- const keys = /* @__PURE__ */ new Set();
137
- for (const partition of manifest.partitions) {
138
- for (const segment of partition.segments) {
139
- keys.add(segment.key);
140
- }
141
- }
142
- for (const vector of manifest.vectors) {
143
- for (const file of vector.files) keys.add(file.key);
144
- for (const part of vector.graphParts) keys.add(part.key);
145
- }
146
- return keys;
147
- }
148
- function manifestFileRefs(manifest) {
149
- const refs = [];
150
- for (const partition of manifest.partitions) refs.push(...partition.segments);
151
- for (const vector of manifest.vectors) refs.push(...vector.files, ...vector.graphParts);
152
- return refs;
153
- }
154
-
155
- // src/persistence/durability/segment/store.ts
156
- function directoryCheckpointStore(directory) {
157
- const store = {
158
- read: (key) => directory.read(key),
159
- write: (key, chunks) => directory.atomicWrite(key, chunks),
160
- remove: (key) => directory.remove(key),
161
- list: (prefix) => directory.list(prefix),
162
- pathOf: (key) => directory.pathOf(key)
163
- };
164
- const byteLength = directory.byteLength;
165
- if (byteLength !== void 0) store.byteLength = (key) => byteLength.call(directory, key);
166
- return store;
167
- }
168
- function joined(chunks) {
169
- if (chunks.length === 1) return chunks[0];
170
- let total = 0;
171
- for (const chunk of chunks) total += chunk.byteLength;
172
- const bytes = new Uint8Array(total);
173
- let offset = 0;
174
- for (const chunk of chunks) {
175
- bytes.set(chunk, offset);
176
- offset += chunk.byteLength;
177
- }
178
- return bytes;
179
- }
180
- function adapterFailure(code, verb, key, error) {
181
- if (error instanceof NarsilError) return error;
182
- const cause = error instanceof Error ? error.message : String(error);
183
- return new NarsilError(code, `The persistence adapter failed to ${verb} "${key}": ${cause}`, { key, cause });
184
- }
185
- async function guarded(run, code, verb, key) {
186
- try {
187
- return await run();
188
- } catch (error) {
189
- throw adapterFailure(code, verb, key, error);
190
- }
191
- }
192
- function adapterCheckpointStore(adapter) {
193
- return {
194
- read: (key) => guarded(() => adapter.load(key), ErrorCodes.PERSISTENCE_LOAD_FAILED, "load", key),
195
- write: (key, chunks) => guarded(() => adapter.save(key, joined(chunks)), ErrorCodes.PERSISTENCE_SAVE_FAILED, "save", key),
196
- remove: (key) => guarded(() => adapter.delete(key), ErrorCodes.PERSISTENCE_DELETE_FAILED, "delete", key),
197
- list: (prefix) => guarded(() => adapter.list(prefix), ErrorCodes.PERSISTENCE_LOAD_FAILED, "list", prefix)
198
- };
199
- }
200
- async function readListedFile(store, key, expected) {
201
- const length = store.byteLength === void 0 ? void 0 : await store.byteLength(key);
202
- if (length === null) missingFile(expected.what, key);
203
- if (length !== void 0) requireListedLength(expected.what, key, length, expected.bytes);
204
- const bytes = await store.read(key);
205
- if (bytes === null) missingFile(expected.what, key);
206
- requireListedLength(expected.what, key, bytes.byteLength, expected.bytes);
207
- return bytes;
208
- }
209
- function missingFile(what, key) {
210
- throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The ${what} "${key}" that the manifest lists is missing`, {
211
- key
212
- });
213
- }
214
- function requireListedLength(what, key, found, listed) {
215
- if (found === listed) return;
216
- throw new NarsilError(
217
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
218
- `The ${what} "${key}" holds ${found} bytes where the manifest lists ${listed}`,
219
- { key, bytes: found, listed }
220
- );
221
- }
222
- function requireListedChecksum(what, key, found, listed) {
223
- if (found === listed) return;
224
- throw new NarsilError(
225
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
226
- `The ${what} "${key}" carries checksum ${found} where the manifest lists ${listed}`,
227
- { key, checksum: found, listed }
228
- );
229
- }
230
-
231
17
  // src/persistence/durability/segment/captured-partition.ts
232
18
  function keepMaskOf(segment) {
233
19
  const keep = new Uint8Array(segment.arrays.documentCount);
@@ -678,10 +464,10 @@ var RECORD_LENGTH_SIZE = 4;
678
464
  var FRAME_CRC_SIZE = 4;
679
465
  var MAX_WAL_RECORD_BYTES = 268435456;
680
466
  function writeSegmentHeader() {
681
- const header2 = new Uint8Array(SEGMENT_HEADER_SIZE);
682
- header2.set(WAL_MAGIC, 0);
683
- header2[4] = WAL_FORMAT_VERSION & 255;
684
- return header2;
467
+ const header = new Uint8Array(SEGMENT_HEADER_SIZE);
468
+ header.set(WAL_MAGIC, 0);
469
+ header[4] = WAL_FORMAT_VERSION & 255;
470
+ return header;
685
471
  }
686
472
  function frameRecord(entry) {
687
473
  const payload = encode(entry);
@@ -722,9 +508,9 @@ function corrupt(message, details) {
722
508
  throw new NarsilError(ErrorCodes.PERSISTENCE_WAL_CORRUPT, message, details);
723
509
  }
724
510
  function readDurableRegion(data, durableByteLength, maxRecordBytes = MAX_WAL_RECORD_BYTES) {
725
- const header2 = checkSegmentHeader(data);
726
- if (!header2.ok) {
727
- corrupt(`WAL segment header invalid in durable region: ${header2.reason}`, { reason: header2.reason });
511
+ const header = checkSegmentHeader(data);
512
+ if (!header.ok) {
513
+ corrupt(`WAL segment header invalid in durable region: ${header.reason}`, { reason: header.reason });
728
514
  }
729
515
  if (durableByteLength < SEGMENT_HEADER_SIZE || durableByteLength > data.length) {
730
516
  corrupt("Commit marker durable byte length is outside the segment", {
@@ -823,6 +609,9 @@ function readTailBeyondFrontier(data, durableByteLength, lastDurableSeqNo, maxRe
823
609
  function walPrefix(indexName, partitionId) {
824
610
  return `${indexName}/wal/${partitionId}/`;
825
611
  }
612
+ function nextSegmentKey(indexName, partitionId) {
613
+ return `${walPrefix(indexName, partitionId)}next`;
614
+ }
826
615
  function segmentStartSeqNo(key, prefix) {
827
616
  if (!key.startsWith(prefix)) {
828
617
  return null;
@@ -867,14 +656,14 @@ async function* readSegmentsInOrder(directory, wal) {
867
656
  continue;
868
657
  }
869
658
  if (startSeqNo < marker.state.activeSegmentSeqNo) {
870
- const header2 = checkSegmentHeader(bytes);
871
- if (!header2.ok) {
659
+ const header = checkSegmentHeader(bytes);
660
+ if (!header.ok) {
872
661
  throw new NarsilError(
873
662
  ErrorCodes.PERSISTENCE_WAL_CORRUPT,
874
- `Sealed WAL segment header invalid: ${header2.reason}`,
663
+ `Sealed WAL segment header invalid: ${header.reason}`,
875
664
  {
876
665
  key,
877
- reason: header2.reason
666
+ reason: header.reason
878
667
  }
879
668
  );
880
669
  }
@@ -1004,569 +793,23 @@ async function buildSegmentFromEntries(input) {
1004
793
  if (input.keepsWriteRecords) built.writeRecords = /* @__PURE__ */ new Map([[segment, writeRecordsOf(documents, written)]]);
1005
794
  return built;
1006
795
  }
1007
- var FIXMAP = 128;
1008
- var MAP16 = 222;
1009
- var MAP32 = 223;
1010
- var FIXARRAY = 144;
1011
- var ARRAY16 = 220;
1012
- var ARRAY32 = 221;
1013
- var BIN8 = 196;
1014
- var BIN16 = 197;
1015
- var BIN32 = 198;
1016
- var FIX_COUNT_LIMIT = 16;
1017
- var UINT8_LIMIT = 256;
1018
- var UINT16_LIMIT = 65536;
1019
- function header(fixed, sixteen, thirtyTwo, count) {
1020
- if (count < FIX_COUNT_LIMIT) return Uint8Array.of(fixed | count);
1021
- if (count < UINT16_LIMIT) return Uint8Array.of(sixteen, count >> 8, count & 255);
1022
- const bytes = new Uint8Array(5);
1023
- bytes[0] = thirtyTwo;
1024
- new DataView(bytes.buffer).setUint32(1, count, false);
1025
- return bytes;
1026
- }
1027
- function binHeader(byteLength) {
1028
- if (byteLength < UINT8_LIMIT) return Uint8Array.of(BIN8, byteLength);
1029
- if (byteLength < UINT16_LIMIT) return Uint8Array.of(BIN16, byteLength >> 8, byteLength & 255);
1030
- const bytes = new Uint8Array(5);
1031
- bytes[0] = BIN32;
1032
- new DataView(bytes.buffer).setUint32(1, byteLength, false);
1033
- return bytes;
1034
- }
1035
- var MessagePackChunks = class {
1036
- chunks = [];
1037
- byteLength = 0;
1038
- push(chunk) {
1039
- if (chunk.byteLength === 0) return;
1040
- this.chunks.push(chunk);
1041
- this.byteLength += chunk.byteLength;
1042
- }
1043
- map(entries) {
1044
- this.push(header(FIXMAP, MAP16, MAP32, entries));
1045
- return this;
1046
- }
1047
- array(items) {
1048
- this.push(header(FIXARRAY, ARRAY16, ARRAY32, items));
1049
- return this;
1050
- }
1051
- value(value) {
1052
- this.push(encode(value));
1053
- return this;
1054
- }
1055
- key(name) {
1056
- return this.value(name);
1057
- }
1058
- binary(bytes) {
1059
- this.push(binHeader(bytes.byteLength));
1060
- this.push(bytes);
1061
- return this;
1062
- }
1063
- binaryOrNil(bytes) {
1064
- return bytes === null ? this.value(null) : this.binary(bytes);
1065
- }
1066
- };
1067
-
1068
- // src/serialization/keyword-segment/token-order.ts
1069
- function slotsAreInTokenOrder(payloadSlots) {
1070
- for (let at = 0; at < payloadSlots.length; at++) {
1071
- if (payloadSlots[at] !== at) return false;
1072
- }
1073
- return true;
1074
- }
1075
- function reorderedPositions(arrays, postingOffsets) {
1076
- const { positionOffsets, positionValues, tokenTable } = arrays;
1077
- if (positionOffsets === null || positionValues === null) return { positionOffsets: null, positionValues: null };
1078
- const postings = postingOffsets[postingOffsets.length - 1];
1079
- const offsets = new Uint32Array(postings + 1);
1080
- const values = new Uint32Array(positionValues.length);
1081
- let cursor = 0;
1082
- for (let token = 0; token < tokenTable.payloadSlots.length; token++) {
1083
- const slot = tokenTable.payloadSlots[token];
1084
- for (let from = arrays.postingOffsets[slot]; from < arrays.postingOffsets[slot + 1]; from++) {
1085
- const to = postingOffsets[token] + (from - arrays.postingOffsets[slot]);
1086
- offsets[to] = cursor;
1087
- const start = positionOffsets[from];
1088
- const end = positionOffsets[from + 1];
1089
- values.set(positionValues.subarray(start, end), cursor);
1090
- cursor += end - start;
1091
- }
1092
- }
1093
- offsets[postings] = cursor;
1094
- return { positionOffsets: offsets, positionValues: values };
1095
- }
1096
- function postingsInTokenOrder(arrays) {
1097
- const { tokenTable } = arrays;
1098
- if (slotsAreInTokenOrder(tokenTable.payloadSlots)) return arrays;
1099
- const tokens = tokenTable.payloadSlots.length;
1100
- const postingOffsets = new Uint32Array(tokens + 1);
1101
- for (let token = 0; token < tokens; token++) {
1102
- const slot = tokenTable.payloadSlots[token];
1103
- postingOffsets[token + 1] = postingOffsets[token] + arrays.postingOffsets[slot + 1] - arrays.postingOffsets[slot];
1104
- }
1105
- const postings = postingOffsets[tokens];
1106
- const postingDocIds = new Uint32Array(postings);
1107
- const postingFrequencies = new Uint16Array(postings);
1108
- const postingFieldIndices = new Uint16Array(postings);
1109
- for (let token = 0; token < tokens; token++) {
1110
- const slot = tokenTable.payloadSlots[token];
1111
- const start = arrays.postingOffsets[slot];
1112
- const end = arrays.postingOffsets[slot + 1];
1113
- postingDocIds.set(arrays.postingDocIds.subarray(start, end), postingOffsets[token]);
1114
- postingFrequencies.set(arrays.postingFrequencies.subarray(start, end), postingOffsets[token]);
1115
- postingFieldIndices.set(arrays.postingFieldIndices.subarray(start, end), postingOffsets[token]);
1116
- }
1117
- return {
1118
- postingOffsets,
1119
- postingDocIds,
1120
- postingFrequencies,
1121
- postingFieldIndices,
1122
- ...reorderedPositions(arrays, postingOffsets)
1123
- };
1124
- }
1125
-
1126
- // src/serialization/keyword-segment/encode.ts
1127
- var TOP_LEVEL_ENTRIES = 28;
1128
- function writeFieldLengths(out, arrays) {
1129
- out.array(arrays.fieldLengthNames.length);
1130
- for (let f = 0; f < arrays.fieldLengthNames.length; f++) {
1131
- out.map(2).key("field").value(arrays.fieldLengthNames[f]);
1132
- out.key("lengths").binary(littleEndianBytes(arrays.fieldLengthColumns[f]));
1133
- }
1134
- }
1135
- function writeValueColumns(out, columns) {
1136
- out.array(columns.length);
1137
- for (const column of columns) {
1138
- out.map(3).key("field").value(column.fieldPath);
1139
- out.key("documents").binary(littleEndianBytes(column.docIds));
1140
- out.key("values").binary(littleEndianBytes(column.values));
1141
- }
1142
- }
1143
- function writeBooleanColumns(out, columns) {
1144
- out.array(columns.length);
1145
- for (const column of columns) {
1146
- out.map(3).key("field").value(column.fieldPath);
1147
- out.key("trueDocuments").binary(littleEndianBytes(column.trueDocs));
1148
- out.key("falseDocuments").binary(littleEndianBytes(column.falseDocs));
1149
- }
1150
- }
1151
- function writeEnumColumns(out, columns) {
1152
- out.array(columns.length);
1153
- for (const column of columns) {
1154
- out.map(4).key("field").value(column.fieldPath);
1155
- out.key("values").value(column.values);
1156
- out.key("offsets").binary(littleEndianBytes(column.offsets));
1157
- out.key("documents").binary(littleEndianBytes(column.docIds));
1158
- }
1159
- }
1160
- function writeGeopointColumns(out, columns) {
1161
- out.array(columns.length);
1162
- for (const column of columns) {
1163
- out.map(4).key("field").value(column.fieldPath);
1164
- out.key("documents").binary(littleEndianBytes(column.docIds));
1165
- out.key("latitudes").binary(littleEndianBytes(column.latitudes));
1166
- out.key("longitudes").binary(littleEndianBytes(column.longitudes));
1167
- }
1168
- }
1169
- function writePatternColumns(out, columns) {
1170
- out.array(columns.length);
1171
- for (const column of columns) {
1172
- out.map(5).key("field").value(column.fieldPath);
1173
- out.key("documents").binary(littleEndianBytes(column.docIds));
1174
- out.key("runs").binary(littleEndianBytes(column.runs));
1175
- out.key("offsets").binary(littleEndianBytes(column.offsets));
1176
- out.key("postings").binary(littleEndianBytes(column.postings));
1177
- }
1178
- }
1179
- function writeSurfaceForms(out, arrays) {
1180
- const table = arrays.surfaceTable;
1181
- if (table === null || table.counts.length === 0) {
1182
- out.value(null);
1183
- return;
1184
- }
1185
- out.map(3).key("text").binary(table.blob);
1186
- out.key("offsets").binary(littleEndianBytes(table.offsets));
1187
- out.key("counts").binary(littleEndianBytes(table.counts));
1188
- }
1189
- function writePartialWords(out, arrays) {
1190
- const table = arrays.partialWords;
1191
- if (table === null) {
1192
- out.value(null);
1193
- return;
1194
- }
1195
- out.map(8).key("fields").value(table.fields);
1196
- out.key("text").binary(table.spellings);
1197
- out.key("offsets").binary(littleEndianBytes(table.spellingOffsets));
1198
- out.key("tokens").binary(littleEndianBytes(table.spellingTokens));
1199
- out.key("counts").binary(littleEndianBytes(table.spellingCounts));
1200
- out.key("runs").binary(littleEndianBytes(table.runs));
1201
- out.key("runOffsets").binary(littleEndianBytes(table.runOffsets));
1202
- out.key("runEntries").binary(littleEndianBytes(table.runEntries));
1203
- }
1204
- function encodeKeywordSegment(contents) {
1205
- const { arrays } = contents;
1206
- const postings = postingsInTokenOrder(arrays);
1207
- const out = new MessagePackChunks();
1208
- const { writeRecords } = contents;
1209
- out.map(writeRecords === void 0 ? TOP_LEVEL_ENTRIES : TOP_LEVEL_ENTRIES + 2);
1210
- out.key("v").value(KEYWORD_SEGMENT_PAYLOAD_VERSION);
1211
- out.key("schema").value(contents.schema);
1212
- out.key("tombstones").value(contents.tombstones);
1213
- out.key("documentCount").value(arrays.documentCount);
1214
- out.key("ids").binary(arrays.idTable.blob);
1215
- out.key("idOffsets").binary(littleEndianBytes(arrays.idTable.offsets));
1216
- out.key("idOrder").binary(littleEndianBytes(arrays.idTable.sortedOrdinals));
1217
- out.key("documents").binary(arrays.documentTable.blob);
1218
- out.key("documentOffsets").binary(littleEndianBytes(arrays.documentTable.offsets));
1219
- out.key("fieldNames").value(arrays.fieldNames);
1220
- out.key("tokens").binary(arrays.tokenTable.blob);
1221
- out.key("tokenOffsets").binary(littleEndianBytes(arrays.tokenTable.offsets));
1222
- out.key("documentFrequencies").binary(littleEndianBytes(arrays.tokenTable.documentFrequencies));
1223
- out.key("postingOffsets").binary(littleEndianBytes(postings.postingOffsets));
1224
- out.key("postingDocuments").binary(littleEndianBytes(postings.postingDocIds));
1225
- out.key("postingFrequencies").binary(littleEndianBytes(postings.postingFrequencies));
1226
- out.key("postingFields").binary(littleEndianBytes(postings.postingFieldIndices));
1227
- out.key("positionOffsets").binaryOrNil(postings.positionOffsets === null ? null : littleEndianBytes(postings.positionOffsets));
1228
- out.key("positions").binaryOrNil(postings.positionValues === null ? null : littleEndianBytes(postings.positionValues));
1229
- out.key("fieldLengths");
1230
- writeFieldLengths(out, arrays);
1231
- out.key("numeric");
1232
- writeValueColumns(out, arrays.numeric);
1233
- out.key("sizes");
1234
- writeValueColumns(out, arrays.sizes);
1235
- out.key("boolean");
1236
- writeBooleanColumns(out, arrays.boolean);
1237
- out.key("enum");
1238
- writeEnumColumns(out, arrays.enums);
1239
- out.key("geopoint");
1240
- writeGeopointColumns(out, arrays.geo);
1241
- out.key("pattern");
1242
- writePatternColumns(out, arrays.pattern);
1243
- out.key("surfaceForms");
1244
- writeSurfaceForms(out, arrays);
1245
- out.key("partialWords");
1246
- writePartialWords(out, arrays);
1247
- if (writeRecords !== void 0) {
1248
- out.key("seqNos").binary(uint64Bytes(writeRecords.seqNos));
1249
- out.key("primaryTerms").binary(uint64Bytes(writeRecords.primaryTerms));
1250
- }
1251
- return out;
1252
- }
1253
-
1254
- // src/persistence/durability/segment/segment-file.ts
1255
- function sharesMemoryWithWorkers() {
1256
- return typeof SharedArrayBuffer === "function";
1257
- }
1258
- async function readSegmentFile(store, ref) {
1259
- const bytes = await readListedFile(store, ref.key, { bytes: ref.bytes, what: "segment" });
1260
- const { header: header2, payload } = await openChecksummedEnvelope(bytes, {
1261
- envelopeFormatVersion: KEYWORD_SEGMENT_ENVELOPE_VERSION,
1262
- what: `segment "${ref.key}"`,
1263
- allowsCompression: true
1264
- });
1265
- requireListedChecksum("segment", ref.key, header2.checksum, ref.checksum);
1266
- return decodeKeywordSegment(payload, sharesMemoryWithWorkers());
1267
- }
1268
- var ORDINAL_BYTES = 4;
1269
- var POSTING_BYTES = 8;
1270
- var VALUE_ENTRY_BYTES = 12;
1271
- var GEOPOINT_ENTRY_BYTES = 20;
1272
- var TOKEN_ENTRY_BYTES = 12;
1273
- var STRING_HEADER_BYTES = 5;
1274
- var encoder = new TextEncoder();
1275
- function documentBytes(segment, ordinal) {
1276
- const table = segment.arrays.documentTable;
1277
- if (table !== null) return table.offsets[ordinal + 1] - table.offsets[ordinal];
1278
- return messagePackUpperBoundBytes(segment.documentSource.docAt(ordinal));
1279
- }
1280
- function addPerEntry(costs, docIds, bytes) {
1281
- for (let at = 0; at < docIds.length; at++) costs[docIds[at]] += bytes;
1282
- }
1283
- function chargePostings(segment, costs, chargesTokens) {
1284
- const arrays = segment.arrays;
1285
- const positions = arrays.positionOffsets;
1286
- for (let p = 0; p < arrays.postingDocIds.length; p++) {
1287
- let bytes = POSTING_BYTES;
1288
- if (positions !== null) bytes += ORDINAL_BYTES * (1 + positions[p + 1] - positions[p]);
1289
- costs[arrays.postingDocIds[p]] += bytes;
1290
- }
1291
- if (!chargesTokens) return;
1292
- const table = arrays.tokenTable;
1293
- for (let at = 0; at < table.size; at++) {
1294
- const slot = table.payloadSlot(at);
1295
- const bytes = table.byteLengthAt(at) + TOKEN_ENTRY_BYTES;
1296
- for (let p = arrays.postingOffsets[slot]; p < arrays.postingOffsets[slot + 1]; p++) {
1297
- costs[arrays.postingDocIds[p]] += bytes;
1298
- }
1299
- }
1300
- }
1301
- function chargeColumns(segment, costs) {
1302
- const arrays = segment.arrays;
1303
- for (const column of [...arrays.numeric, ...arrays.sizes]) addPerEntry(costs, column.docIds, VALUE_ENTRY_BYTES);
1304
- for (const column of arrays.boolean) {
1305
- addPerEntry(costs, column.trueDocs, ORDINAL_BYTES);
1306
- addPerEntry(costs, column.falseDocs, ORDINAL_BYTES);
1307
- }
1308
- for (const column of arrays.enums) addPerEntry(costs, column.docIds, ORDINAL_BYTES);
1309
- for (const column of arrays.geo) addPerEntry(costs, column.docIds, GEOPOINT_ENTRY_BYTES);
1310
- for (const column of arrays.pattern) {
1311
- addPerEntry(costs, column.docIds, ORDINAL_BYTES);
1312
- addPerEntry(costs, column.postings, ORDINAL_BYTES);
1313
- }
1314
- }
1315
- function stringListBytes(values) {
1316
- let bytes = STRING_HEADER_BYTES;
1317
- for (const value of values) bytes += STRING_HEADER_BYTES + encoder.encode(value).byteLength;
1318
- return bytes;
1319
- }
1320
- function fixedBytesOf(segment, chargesTokens) {
1321
- const arrays = segment.arrays;
1322
- let bytes = stringListBytes(arrays.fieldNames);
1323
- if (!chargesTokens) bytes += arrays.tokenTable.bytes;
1324
- for (const column of arrays.enums) bytes += stringListBytes(column.values) + column.offsets.byteLength;
1325
- for (const column of arrays.pattern) bytes += column.runs.byteLength + column.offsets.byteLength;
1326
- const surfaces = segment.sharedSnapshot?.surfaceTable ?? null;
1327
- if (surfaces !== null) bytes += surfaces.blob.byteLength + surfaces.offsets.byteLength + surfaces.counts.byteLength;
1328
- else bytes += messagePackUpperBoundBytes(arrays.readSurfaceForms());
1329
- return bytes + segment.partialWords.bytes();
1330
- }
1331
- function keptDocumentsOf(selection, chargesTokens) {
1332
- const { segment } = selection;
1333
- const count = segment.arrays.documentCount;
1334
- const costs = new Float64Array(count);
1335
- const perDocument = ORDINAL_BYTES * (3 + segment.arrays.fieldLengthColumns.length);
1336
- for (let ordinal = 0; ordinal < count; ordinal++) {
1337
- costs[ordinal] = perDocument + encoder.encode(segment.arrays.idTable.idAt(ordinal)).byteLength;
1338
- }
1339
- chargePostings(segment, costs, chargesTokens);
1340
- chargeColumns(segment, costs);
1341
- const ordinals = [];
1342
- for (let ordinal = 0; ordinal < count; ordinal++) {
1343
- if (!selectionKeeps(selection, ordinal)) continue;
1344
- costs[ordinal] += documentBytes(segment, ordinal);
1345
- ordinals.push(ordinal);
1346
- }
1347
- return { segment, ordinals, costs };
1348
- }
1349
- function selectionOf(segment, ordinals) {
1350
- const marked = new Uint8Array(segment.arrays.documentCount);
1351
- for (const ordinal of ordinals) marked[ordinal] = 1;
1352
- return { segment, keep: (ordinal) => marked[ordinal] === 1 };
1353
- }
1354
- function tooLargeForOneFile(cost, budget) {
1355
- throw new NarsilError(
1356
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
1357
- `A document needs up to ${Math.ceil(cost)} bytes of a segment, more than the ${Math.max(0, Math.floor(budget))} that one file leaves it`,
1358
- { bytes: Math.ceil(cost), budget: Math.floor(budget), maximum: MAX_FILE_BYTES }
1359
- );
1360
- }
1361
- function splitIntoSegmentFiles(selections, schema, tombstones) {
1362
- let tokenBytes = 0;
1363
- for (const { segment } of selections) tokenBytes += segment.arrays.tokenTable.bytes;
1364
- const chargesTokens = tokenBytes > MAX_FILE_BYTES * SEGMENT_TOKEN_TABLE_SHARE_OF_FILE;
1365
- let fixed = HEADER_SIZE + SEGMENT_FILE_FRAMING_SLACK_BYTES + encode(schema).byteLength + encode(tombstones).byteLength;
1366
- for (const { segment } of selections) fixed += fixedBytesOf(segment, chargesTokens);
1367
- const budget = MAX_FILE_BYTES - fixed;
1368
- const groups = [];
1369
- let current = [];
1370
- let currentBytes = 0;
1371
- let currentCount = 0;
1372
- const flush = () => {
1373
- if (currentCount === 0) return;
1374
- groups.push({
1375
- selections: current.map((entry) => selectionOf(entry.segment, entry.ordinals)),
1376
- documentCount: currentCount
1377
- });
1378
- current = [];
1379
- currentBytes = 0;
1380
- currentCount = 0;
1381
- };
1382
- for (const selection of selections) {
1383
- const kept = keptDocumentsOf(selection, chargesTokens);
1384
- let entry = null;
1385
- for (const ordinal of kept.ordinals) {
1386
- const cost = kept.costs[ordinal];
1387
- if (cost > budget) tooLargeForOneFile(cost, budget);
1388
- if (currentBytes + cost > budget) {
1389
- flush();
1390
- entry = null;
1391
- }
1392
- if (entry === null) {
1393
- entry = { segment: kept.segment, ordinals: [] };
1394
- current.push(entry);
1395
- }
1396
- entry.ordinals.push(ordinal);
1397
- currentBytes += cost;
1398
- currentCount += 1;
1399
- }
1400
- }
1401
- flush();
1402
- return groups;
1403
- }
1404
- function halvesOf(group) {
1405
- const flat = [];
1406
- for (const selection of group.selections) {
1407
- for (let ordinal = 0; ordinal < selection.segment.arrays.documentCount; ordinal++) {
1408
- if (!selectionKeeps(selection, ordinal)) continue;
1409
- flat.push({ segment: selection.segment, ordinal });
1410
- }
1411
- }
1412
- const middle = Math.ceil(flat.length / 2);
1413
- const groupOf = (entries) => {
1414
- const bySegment = /* @__PURE__ */ new Map();
1415
- for (const { segment, ordinal } of entries) {
1416
- let ordinals = bySegment.get(segment);
1417
- if (ordinals === void 0) {
1418
- ordinals = [];
1419
- bySegment.set(segment, ordinals);
1420
- }
1421
- ordinals.push(ordinal);
1422
- }
1423
- return {
1424
- selections: [...bySegment].map(([segment, ordinals]) => selectionOf(segment, ordinals)),
1425
- documentCount: entries.length
1426
- };
1427
- };
1428
- return [groupOf(flat.slice(0, middle)), groupOf(flat.slice(middle))];
1429
- }
1430
-
1431
- // src/persistence/durability/segment/segment-write.ts
1432
- function emptyPayload() {
1433
- return {
1434
- documentCount: 0,
1435
- docIds: [],
1436
- fieldNames: [],
1437
- tokens: [],
1438
- postingOffsets: new Uint32Array(1),
1439
- postingDocIds: new Uint32Array(0),
1440
- postingFrequencies: new Uint16Array(0),
1441
- postingFieldIndices: new Uint16Array(0),
1442
- positionOffsets: null,
1443
- positionValues: null,
1444
- fieldLengthNames: [],
1445
- fieldLengthColumns: [],
1446
- totalFieldLengths: {},
1447
- docFrequencies: {},
1448
- surfaceForms: null,
1449
- partialSpellings: null,
1450
- numeric: [],
1451
- boolean: [],
1452
- enums: [],
1453
- geo: [],
1454
- pattern: [],
1455
- sizes: []
1456
- };
1457
- }
1458
- function noSpellingsOf(analysis) {
1459
- if (analysis === null || analysis.fields.length === 0) return null;
1460
- return { fields: [...analysis.fields], tokens: [], spellings: [], counts: [] };
1461
- }
1462
- function writeRecordsOfGroup(group, records, count) {
1463
- const seqNos = new Float64Array(count);
1464
- const primaryTerms = new Float64Array(count);
1465
- let at = 0;
1466
- for (const selection of group?.selections ?? []) {
1467
- const source = records.get(selection.segment);
1468
- for (let ordinal = 0; ordinal < selection.segment.arrays.documentCount; ordinal++) {
1469
- if (!selectionKeeps(selection, ordinal)) continue;
1470
- if (source !== void 0) {
1471
- seqNos[at] = source.seqNos[ordinal];
1472
- primaryTerms[at] = source.primaryTerms[ordinal];
1473
- }
1474
- at += 1;
1475
- }
1476
- }
1477
- return { seqNos, primaryTerms };
1478
- }
1479
- function withWriteRecords(contents, group, request) {
1480
- if (request.writeRecords === void 0) return contents;
1481
- const writeRecords = writeRecordsOfGroup(group, request.writeRecords, contents.arrays.documentCount);
1482
- return { ...contents, writeRecords };
1483
- }
1484
- function contentsOfGroup(group, request, tombstones) {
1485
- const { schema, partialWordAnalysis, textAnalysis } = request;
1486
- if (group === null) {
1487
- const payload = { ...emptyPayload(), partialSpellings: noSpellingsOf(partialWordAnalysis) };
1488
- return {
1489
- schema,
1490
- tombstones,
1491
- arrays: segmentArraysOf(payload, { blob: new Uint8Array(0), offsets: new Uint32Array(1) })
1492
- };
1493
- }
1494
- const merged = mergeSegmentContents(group.selections, partialWordAnalysis, textAnalysis);
1495
- return {
1496
- schema,
1497
- tombstones,
1498
- arrays: segmentArraysOf(merged.payload, merged.documents, merged.reusedPartialWords)
1499
- };
1500
- }
1501
- function requireRoomForAnotherSegment(state) {
1502
- if (state.segments.length < MAX_SEGMENTS_PER_PARTITION) return;
1503
- throw new NarsilError(
1504
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
1505
- `Partition ${state.request.partitionId} would hold more than ${MAX_SEGMENTS_PER_PARTITION} segments`,
1506
- { partitionId: state.request.partitionId, maximum: MAX_SEGMENTS_PER_PARTITION }
1507
- );
1508
- }
1509
- async function writeGroup(state, group) {
1510
- const { request } = state;
1511
- const tombstones = state.tombstonesWritten ? [] : [...request.tombstones];
1512
- const contents = withWriteRecords(contentsOfGroup(group, request, tombstones), group, request);
1513
- const encoded = encodeKeywordSegment(contents);
1514
- if (HEADER_SIZE + encoded.byteLength > MAX_FILE_BYTES && group !== null && group.documentCount > 1) {
1515
- const [first, second] = halvesOf(group);
1516
- await writeGroup(state, first);
1517
- await writeGroup(state, second);
1518
- return;
1519
- }
1520
- requireRoomForAnotherSegment(state);
1521
- const id = state.nextSegmentId;
1522
- const key = segmentKey(request.indexName, request.partitionId, id);
1523
- const packed = await packChecksummedEnvelope(encoded.chunks, KEYWORD_SEGMENT_ENVELOPE_VERSION, `segment "${key}"`);
1524
- await request.store.write(key, packed.chunks);
1525
- state.segments.push({
1526
- id,
1527
- key,
1528
- bytes: packed.byteLength,
1529
- checksum: packed.checksum,
1530
- docCount: contents.arrays.documentCount,
1531
- tombstoneCount: tombstones.length
1532
- });
1533
- state.nextSegmentId = id + 1;
1534
- state.tombstonesWritten = true;
1535
- }
1536
- function keptDocumentCount(selections) {
1537
- let count = 0;
1538
- for (const selection of selections) {
1539
- for (let ordinal = 0; ordinal < selection.segment.arrays.documentCount; ordinal++) {
1540
- if (selectionKeeps(selection, ordinal)) count += 1;
1541
- }
1542
- }
1543
- return count;
1544
- }
1545
- async function writeSegmentsOf(request) {
1546
- const state = { request, segments: [], nextSegmentId: request.nextSegmentId, tombstonesWritten: false };
1547
- const groups = request.mergesIntoOneFile === true ? [{ selections: [...request.selections], documentCount: keptDocumentCount(request.selections) }] : splitIntoSegmentFiles(request.selections, request.schema, request.tombstones);
1548
- for (const group of groups) {
1549
- if (group.documentCount > 0) await writeGroup(state, group);
1550
- }
1551
- if (!state.tombstonesWritten && request.tombstones.length > 0) await writeGroup(state, null);
1552
- return { segments: state.segments, nextSegmentId: state.nextSegmentId };
1553
- }
1554
796
 
1555
797
  // src/persistence/durability/segment/compaction.ts
1556
- function windowFits(segments, start, windowLength) {
798
+ function windowFits(segments, start, windowLength, paged) {
1557
799
  let documents = 0;
1558
800
  let bytes = SEGMENT_FILE_FRAMING_SLACK_BYTES;
1559
801
  for (let at = start; at < start + windowLength; at++) {
1560
802
  documents += segments[at].docCount;
1561
803
  bytes += segments[at].bytes;
1562
804
  }
805
+ if (paged) return bytes * MERGED_SEGMENT_GROWTH_ALLOWANCE <= MERGED_SEGMENT_TARGET_BYTES;
1563
806
  return documents <= COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT && bytes <= MAX_FILE_BYTES;
1564
807
  }
1565
- function smallestFittingWindow(segments, windowLength) {
808
+ function smallestFittingWindow(segments, windowLength, paged) {
1566
809
  let best = null;
1567
810
  let bestBytes = Number.POSITIVE_INFINITY;
1568
811
  for (let start = 0; start + windowLength <= segments.length; start++) {
1569
- if (!windowFits(segments, start, windowLength)) continue;
812
+ if (!windowFits(segments, start, windowLength, paged)) continue;
1570
813
  let bytes = 0;
1571
814
  for (let at = start; at < start + windowLength; at++) bytes += segments[at].bytes;
1572
815
  if (bytes < bestBytes) {
@@ -1576,9 +819,9 @@ function smallestFittingWindow(segments, windowLength) {
1576
819
  }
1577
820
  return best;
1578
821
  }
1579
- function chooseWindow(segments, wantedLength) {
822
+ function chooseWindow(segments, wantedLength, paged) {
1580
823
  for (let windowLength = Math.min(wantedLength, segments.length); windowLength >= 2; windowLength--) {
1581
- const window = smallestFittingWindow(segments, windowLength);
824
+ const window = smallestFittingWindow(segments, windowLength, paged);
1582
825
  if (window !== null) return window;
1583
826
  }
1584
827
  return null;
@@ -1619,11 +862,39 @@ async function compactPartitionSegments(input) {
1619
862
  if (segments.length <= compactionThreshold) {
1620
863
  return { segments, nextSegmentId: input.nextSegmentId };
1621
864
  }
1622
- const window = chooseWindow(segments, segments.length - compactionThreshold + 1);
865
+ const paged = input.paged === true;
866
+ const window = chooseWindow(segments, segments.length - compactionThreshold + 1, paged);
1623
867
  if (window === null) {
1624
868
  return { segments, nextSegmentId: input.nextSegmentId };
1625
869
  }
1626
870
  const windowRefs = segments.slice(window.start, window.start + window.windowLength);
871
+ const replacing = (written2) => ({
872
+ segments: [
873
+ ...segments.slice(0, window.start),
874
+ ...written2.segments,
875
+ ...segments.slice(window.start + window.windowLength)
876
+ ],
877
+ nextSegmentId: written2.nextSegmentId
878
+ });
879
+ if (paged) {
880
+ const { directory } = input.store;
881
+ if (directory === void 0) return { segments, nextSegmentId: input.nextSegmentId };
882
+ const { mergePagedWindow } = await import('./module-IYPLWHGC.mjs');
883
+ return replacing(
884
+ await mergePagedWindow({
885
+ store: input.store,
886
+ directory,
887
+ indexName: input.indexName,
888
+ partitionId: input.partitionId,
889
+ schema: input.schema,
890
+ partialWordAnalysis: input.partialWordAnalysis,
891
+ textAnalysis: input.textAnalysis,
892
+ nextSegmentId: input.nextSegmentId,
893
+ refs: windowRefs,
894
+ dropsEveryTombstone: window.start === 0
895
+ })
896
+ );
897
+ }
1627
898
  const contents = [];
1628
899
  for (const ref of windowRefs) contents.push(await readSegmentFile(input.store, ref));
1629
900
  const frozen = contents.map(frozenOf);
@@ -1642,14 +913,7 @@ async function compactPartitionSegments(input) {
1642
913
  mergesIntoOneFile: true,
1643
914
  ...writeRecords === void 0 ? {} : { writeRecords }
1644
915
  });
1645
- return {
1646
- segments: [
1647
- ...segments.slice(0, window.start),
1648
- ...written.segments,
1649
- ...segments.slice(window.start + window.windowLength)
1650
- ],
1651
- nextSegmentId: written.nextSegmentId
1652
- };
916
+ return replacing(written);
1653
917
  }
1654
918
 
1655
919
  // src/persistence/durability/segment/write.ts
@@ -1696,6 +960,21 @@ function wholeContentsOf(input, partitionId) {
1696
960
  const captured = input.capturedPartitions?.find((candidate) => candidate.partitionId === partitionId);
1697
961
  return captured === void 0 ? void 0 : contentsOfCaptured(captured);
1698
962
  }
963
+ function pagedSettingsOf(context) {
964
+ if (context.input.metadata.textStorage !== "disk") return {};
965
+ return { paged: true, keepsWriteRecords: context.input.keepsWriteRecords === true };
966
+ }
967
+ async function tombstonesOfDelta(context, older, selections, deleted) {
968
+ const { directory } = context.input.store;
969
+ if (context.input.metadata.textStorage !== "disk" || directory === void 0) return deleted;
970
+ const { pagedTombstonesFor } = await import('./module-IYPLWHGC.mjs');
971
+ return pagedTombstonesFor(
972
+ directory,
973
+ older,
974
+ selections.map((selection) => selection.segment),
975
+ deleted
976
+ );
977
+ }
1699
978
  async function writeWholePartition(context, partitionId, priorPartition, whole) {
1700
979
  const written = await writeSegmentsOf({
1701
980
  store: context.input.store,
@@ -1707,7 +986,8 @@ async function writeWholePartition(context, partitionId, priorPartition, whole)
1707
986
  textAnalysis: context.textAnalysis,
1708
987
  selections: whole.selections,
1709
988
  tombstones: [],
1710
- ...whole.writeRecords === void 0 ? {} : { writeRecords: whole.writeRecords }
989
+ ...whole.writeRecords === void 0 ? {} : { writeRecords: whole.writeRecords },
990
+ ...pagedSettingsOf(context)
1711
991
  });
1712
992
  return { partitionId, nextSegmentId: written.nextSegmentId, segments: written.segments };
1713
993
  }
@@ -1723,6 +1003,7 @@ async function writeLoggedChanges(context, partitionId, priorPartition, entries)
1723
1003
  let segments = priorPartition ? [...priorPartition.segments] : [];
1724
1004
  let nextSegmentId = priorPartition?.nextSegmentId ?? 0;
1725
1005
  if (built !== null) {
1006
+ const tombstones = await tombstonesOfDelta(context, segments, built.selections, built.tombstones);
1726
1007
  const written = await writeSegmentsOf({
1727
1008
  store: context.input.store,
1728
1009
  indexName: context.indexName,
@@ -1732,8 +1013,9 @@ async function writeLoggedChanges(context, partitionId, priorPartition, entries)
1732
1013
  partialWordAnalysis: context.partialWordAnalysis,
1733
1014
  textAnalysis: context.textAnalysis,
1734
1015
  selections: built.selections,
1735
- tombstones: built.tombstones,
1736
- ...built.writeRecords === void 0 ? {} : { writeRecords: built.writeRecords }
1016
+ tombstones,
1017
+ ...built.writeRecords === void 0 ? {} : { writeRecords: built.writeRecords },
1018
+ ...pagedSettingsOf(context)
1737
1019
  });
1738
1020
  segments.push(...written.segments);
1739
1021
  nextSegmentId = written.nextSegmentId;
@@ -1754,7 +1036,8 @@ async function writeLoggedChanges(context, partitionId, priorPartition, entries)
1754
1036
  textAnalysis: context.textAnalysis,
1755
1037
  segments,
1756
1038
  nextSegmentId,
1757
- compactionThreshold: context.compactionThreshold
1039
+ compactionThreshold: context.compactionThreshold,
1040
+ paged: context.input.metadata.textStorage === "disk"
1758
1041
  });
1759
1042
  segments = compacted.segments;
1760
1043
  nextSegmentId = compacted.nextSegmentId;
@@ -1817,6 +1100,7 @@ async function writeCheckpointSegments(input) {
1817
1100
  async function commitCheckpointManifest(store, metadata, written, vectors) {
1818
1101
  const indexName = metadata.indexName;
1819
1102
  const manifest = {
1103
+ version: SEGMENT_MANIFEST_VERSION,
1820
1104
  schema: metadata.schema,
1821
1105
  language: metadata.language,
1822
1106
  ...metadata.analysisGeneration ? { analysisGeneration: metadata.analysisGeneration } : {},
@@ -1839,4 +1123,4 @@ async function removeCheckpointGarbage(store, garbage) {
1839
1123
  }
1840
1124
  }
1841
1125
 
1842
- export { CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_HEARTBEAT_MS, CHECKPOINT_WORKER_IDLE_MS, CHECKPOINT_WORKER_TIMEOUT_MS, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_SEGMENT_MAX_BYTES, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, DIRECTORY_LOCK_ATTEMPTS, DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS, DIRECTORY_LOCK_SETTLE_MS, EARLIER_SEGMENT_MANIFEST_VERSION, LOG_RECORDS_PER_CHECKPOINT, MANIFEST_ENVELOPE_VERSION, MAX_SEGMENTS_PER_PARTITION, MAX_SEGMENT_ID, SEGMENT_HEADER_SIZE, SEGMENT_MANIFEST_VERSION, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, adapterCheckpointStore, buildEntry, captureWholePartition, checkpointLastSeqNo, checkpointPrimaryTerm, commitCheckpointManifest, computeEntryChecksum, createAckMessage, createEntryBatchMessage, createEntryMessage, createForwardBatchMessage, createForwardBatchResultMessage, createForwardMessage, createInsyncAddMessage, createInsyncConfirmMessage, createInsyncRemoveMessage, createPrimaryResignMessage, directoryCheckpointStore, durableRecordMissing, encodeMarkerSlot, encodeSegmentManifest, frameRecord, graphPartKey, manifestFileRefs, manifestKey, manifestReferencedKeys, markerSlotOffset, nextMarkerSlot, optionalIndexUuid, partitionContentsOf, readCommitMarker, readListedFile, readSegmentFile, readWalSegments, removeCheckpointGarbage, requireListedChecksum, segmentKey, segmentsPrefix, snapshotBundleKey, validateAckPayload, validateEntryBatchPayload, validateEntryPayload, validateForwardBatchPayload, validateForwardBatchResultPayload, validateInsyncConfirmPayload, vectorFileKey, writeCheckpointSegments, writeSegmentHeader };
1126
+ export { SEGMENT_HEADER_SIZE, buildEntry, captureWholePartition, checkpointLastSeqNo, checkpointPrimaryTerm, commitCheckpointManifest, computeEntryChecksum, createAckMessage, createEntryBatchMessage, createEntryMessage, createForwardBatchMessage, createForwardBatchResultMessage, createForwardMessage, createInsyncAddMessage, createInsyncConfirmMessage, createInsyncRemoveMessage, createPrimaryResignMessage, durableRecordMissing, encodeMarkerSlot, frameRecord, markerSlotOffset, nextMarkerSlot, nextSegmentKey, optionalIndexUuid, partitionContentsOf, readCommitMarker, readWalSegments, removeCheckpointGarbage, validateAckPayload, validateEntryBatchPayload, validateEntryPayload, validateForwardBatchPayload, validateForwardBatchResultPayload, validateInsyncConfirmPayload, writeCheckpointSegments, writeSegmentHeader };