@delali/narsil 0.3.1-next.131 → 0.3.1-next.141

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. package/dist/adapters/filesystem.mjs +1 -1
  2. package/dist/adapters/indexeddb.mjs +1 -1
  3. package/dist/analysis-change-HQUT2Z3I.mjs +43 -0
  4. package/dist/analysis-change-entry-TZTG4AZB.mjs +1 -0
  5. package/dist/browser/analysis-change-JPV775LR.mjs +900 -0
  6. package/dist/browser/analysis-change-entry-7HR4LWL3.mjs +1 -0
  7. package/dist/browser/{chunk-UDGB2MWT.mjs → chunk-35B42UFM.mjs} +4 -0
  8. package/dist/browser/chunk-3ZBJYAS4.mjs +19 -0
  9. package/dist/browser/{chunk-2Y3CO4EQ.mjs → chunk-5DB2VDGI.mjs} +1149 -1853
  10. package/dist/browser/chunk-6RHTAVZX.mjs +656 -0
  11. package/dist/browser/chunk-CCOAOO73.mjs +86 -0
  12. package/dist/browser/{chunk-AIIJDWMG.mjs → chunk-D33A5VMU.mjs} +16 -16
  13. package/dist/browser/{chunk-VBBLTD7O.mjs → chunk-HKX2G6WH.mjs} +1520 -2159
  14. package/dist/browser/{chunk-NSCSR6MA.mjs → chunk-KTOK275N.mjs} +606 -830
  15. package/dist/browser/{chunk-D7WQF6DV.mjs → chunk-LVY2LI35.mjs} +2 -2
  16. package/dist/browser/chunk-MVM33ZUL.mjs +993 -0
  17. package/dist/browser/chunk-MWUK3DA5.mjs +2634 -0
  18. package/dist/browser/{chunk-VEGHFP2P.mjs → chunk-MZI3TEOW.mjs} +4 -2
  19. package/dist/browser/chunk-OIZS5KIX.mjs +6 -0
  20. package/dist/browser/chunk-R4WUDZQV.mjs +176 -0
  21. package/dist/browser/{chunk-M5UAAXGU.mjs → chunk-TCLEOZ4P.mjs} +3 -3
  22. package/dist/browser/chunk-VFYMPCOV.mjs +212 -0
  23. package/dist/browser/chunk-WCFCHIBB.mjs +177 -0
  24. package/dist/browser/chunk-XNWPP5NS.mjs +21 -0
  25. package/dist/browser/chunk-ZEV26AQ6.mjs +270 -0
  26. package/dist/browser/cold-module-MJMKNUYJ.mjs +745 -0
  27. package/dist/browser/{durability-wiring-FNZSRH4N.mjs → durability-wiring-SNOWIT6Z.mjs} +370 -95
  28. package/dist/browser/import-TLB2A5LJ.mjs +9 -0
  29. package/dist/browser/module-BUFNZMEU.mjs +12 -0
  30. package/dist/browser/{module-4HIMOQM5.mjs → module-BVP4TKZU.mjs} +397 -109
  31. package/dist/browser/{phrase-module-WESYVU7K.mjs → phrase-module-HGVBO24W.mjs} +1 -1
  32. package/dist/{restore-queue-JAMNRC6T.mjs → browser/restore-queue-ONYNYUDE.mjs} +8 -2
  33. package/dist/browser/{runtime-3X7ZJ5T3.mjs → runtime-VEUH674B.mjs} +2 -2
  34. package/dist/browser/{snapshot-V7VF6BN6.mjs → snapshot-QXBKHJAG.mjs} +135 -183
  35. package/dist/browser/{write-YIC2MKRY.mjs → write-WHFBDWQS.mjs} +64 -90
  36. package/dist/{capabilities-B6WxrWxt.d.ts → capabilities-BCdUwK_L.d.ts} +10 -3
  37. package/dist/chunk-46NZOPHS.mjs +21 -0
  38. package/dist/{chunk-JT3YZGCE.mjs → chunk-5WYVUPGD.mjs} +3 -188
  39. package/dist/chunk-6SNBQESJ.mjs +214 -0
  40. package/dist/{chunk-2C3WXBQS.mjs → chunk-74XB2SXI.mjs} +1 -1
  41. package/dist/chunk-7AUASRE7.mjs +653 -0
  42. package/dist/{chunk-NMXRXFGZ.mjs → chunk-AST3SHFA.mjs} +27 -25
  43. package/dist/chunk-BOJZGUS6.mjs +177 -0
  44. package/dist/{chunk-T4OFG4IQ.mjs → chunk-BRUUBXCB.mjs} +92 -17
  45. package/dist/{chunk-5LX6MOIB.mjs → chunk-C3YDYX6C.mjs} +1 -1
  46. package/dist/chunk-CZQ33DLW.mjs +78 -0
  47. package/dist/{chunk-4GDLFDPZ.mjs → chunk-DD2673TX.mjs} +3 -2
  48. package/dist/{chunk-EUCX5K74.mjs → chunk-DQNKVOJ2.mjs} +65 -65
  49. package/dist/{chunk-RNSC4NFI.mjs → chunk-EEBCDP36.mjs} +3 -1
  50. package/dist/chunk-FOCV4CBZ.mjs +87 -0
  51. package/dist/{chunk-HCGJKDZV.mjs → chunk-GLZHNSGB.mjs} +1 -1
  52. package/dist/chunk-GWOS5DYC.mjs +2637 -0
  53. package/dist/{chunk-PATMDIMN.mjs → chunk-HJJWY6PT.mjs} +1 -1
  54. package/dist/{chunk-CZ7ISLXA.mjs → chunk-HNHCNTVG.mjs} +4 -0
  55. package/dist/{chunk-Y2U5VVWX.mjs → chunk-ICW45TTU.mjs} +1 -1
  56. package/dist/{chunk-TGHNXN7W.mjs → chunk-IO5F7ZBE.mjs} +261 -948
  57. package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
  58. package/dist/{chunk-7OIQ2SIO.mjs → chunk-KEIYVZUX.mjs} +5 -5
  59. package/dist/{chunk-PU6SKH7O.mjs → chunk-KGOXHLLZ.mjs} +7 -3
  60. package/dist/{chunk-WVRDRLIC.mjs → chunk-KZL72VTC.mjs} +70 -18
  61. package/dist/chunk-L76W3IZJ.mjs +193 -0
  62. package/dist/{chunk-2JWNJSNE.mjs → chunk-LA4WNDHL.mjs} +88 -801
  63. package/dist/{chunk-ZR4VM4A6.mjs → chunk-N37VMC62.mjs} +1 -18
  64. package/dist/{chunk-JJ6EUMXB.mjs → chunk-NJJCPFXN.mjs} +2 -2
  65. package/dist/chunk-NN33CZRE.mjs +86 -0
  66. package/dist/{chunk-2WBY4Y4M.mjs → chunk-NOT5QOHU.mjs} +1 -1
  67. package/dist/{chunk-D5TFRB5G.mjs → chunk-NPNK57CA.mjs} +1 -1
  68. package/dist/chunk-NRUXBYPH.mjs +121 -0
  69. package/dist/{chunk-6SSVIW7Y.mjs → chunk-O4LJQI4G.mjs} +31 -5
  70. package/dist/{chunk-UJWGQIXU.mjs → chunk-QHPYS427.mjs} +1 -0
  71. package/dist/{chunk-F6AAXI7T.mjs → chunk-RHKRTYGO.mjs} +13 -4
  72. package/dist/{chunk-2SEL6SBS.mjs → chunk-RKAVHNRW.mjs} +2 -2
  73. package/dist/{chunk-RCMNL7RI.mjs → chunk-SBMAWTRG.mjs} +41 -4
  74. package/dist/chunk-SRJBFG2T.mjs +881 -0
  75. package/dist/{chunk-X5CNUKD5.mjs → chunk-TNBIK2LQ.mjs} +1663 -1652
  76. package/dist/{chunk-NNO7CO3A.mjs → chunk-UIKN2OTQ.mjs} +1 -1
  77. package/dist/chunk-UUZCQQG2.mjs +209 -0
  78. package/dist/{chunk-6M6SDH3S.mjs → chunk-VEFN6X6Q.mjs} +8 -8
  79. package/dist/chunk-VJEGOELZ.mjs +177 -0
  80. package/dist/chunk-W2COJ5AH.mjs +148 -0
  81. package/dist/{chunk-RWESFSJI.mjs → chunk-WJGO5PGV.mjs} +317 -174
  82. package/dist/{chunk-7X5JST5L.mjs → chunk-WNBRW4Q2.mjs} +16 -2
  83. package/dist/{chunk-SZGRYEIC.mjs → chunk-XS2VM7UE.mjs} +3 -3
  84. package/dist/{chunk-Y4BVTQ53.mjs → chunk-XZ3SRNDJ.mjs} +25 -64
  85. package/dist/{chunk-K77FJXKS.mjs → chunk-YHUTF6EA.mjs} +474 -93
  86. package/dist/{chunk-OGLV4I3L.mjs → chunk-YLNSJQLX.mjs} +12 -7
  87. package/dist/{chunk-SOHMMNCH.mjs → chunk-Z4HFMYNX.mjs} +32 -27
  88. package/dist/{chunk-5W7AEVTH.mjs → chunk-ZMBFWQXN.mjs} +2 -2
  89. package/dist/{chunk-2MKX2SCO.mjs → chunk-ZPWS2SKJ.mjs} +297 -274
  90. package/dist/{chunk-7XY6VD7B.mjs → chunk-ZSWHPKGM.mjs} +1 -1
  91. package/dist/{client-gNAfaHcE.d.ts → client-B12YFALP.d.ts} +18 -3
  92. package/dist/client.d.ts +7 -7
  93. package/dist/client.mjs +11 -1
  94. package/dist/cold-module-4X7WJCFM.mjs +783 -0
  95. package/dist/{config-D1cDv3sE.d.ts → config-BhZxIeb1.d.ts} +10 -1
  96. package/dist/{config-CmYJO34X.d.ts → config-CIG5yDyg.d.ts} +2 -2
  97. package/dist/{direct-executor-HQ4QVDX7.mjs → direct-executor-7XOQ7VQP.mjs} +14 -14
  98. package/dist/distribution/coordinator/etcd.d.ts +2 -2
  99. package/dist/distribution/coordinator/etcd.mjs +1 -1
  100. package/dist/distribution/coordinator/in-memory.d.ts +2 -2
  101. package/dist/distribution/transport/grpc.mjs +1 -1
  102. package/dist/distribution.d.ts +24 -8
  103. package/dist/distribution.mjs +982 -211
  104. package/dist/{durability-wiring-S7OX2MAI.mjs → durability-wiring-324D2DDD.mjs} +362 -94
  105. package/dist/embeddings/openai.mjs +2 -2
  106. package/dist/{engine-m_Ddr1bz.d.ts → engine-CtzbHOmG.d.ts} +64 -7
  107. package/dist/import-OOBUW3RR.mjs +44 -0
  108. package/dist/{index-HYnqS1SE.d.ts → index-DsEurhSl.d.ts} +1 -1
  109. package/dist/index.browser.mjs +5 -5
  110. package/dist/index.d.ts +6 -6
  111. package/dist/index.mjs +22 -22
  112. package/dist/{long-query-thread-BSL25PD6.mjs → long-query-thread-4SW7RWDE.mjs} +15 -15
  113. package/dist/{module-2E4XYSGT.mjs → module-DXSJ5LT2.mjs} +378 -87
  114. package/dist/module-IYPLWHGC.mjs +48 -0
  115. package/dist/paged-partition-archive-HFP45PLU.mjs +269 -0
  116. package/dist/partial-words.mjs +5 -5
  117. package/dist/pattern.mjs +3 -3
  118. package/dist/persistence/durability/checkpoint-worker.d.ts +16 -1
  119. package/dist/persistence/durability/checkpoint-worker.mjs +28 -26
  120. package/dist/{phrase-module-JERF2IDG.mjs → phrase-module-5QHBD2VJ.mjs} +1 -1
  121. package/dist/react.d.ts +9 -9
  122. package/dist/react.mjs +4 -0
  123. package/dist/{rerank-DMMT4FZA.mjs → rerank-KTLC3E6N.mjs} +5 -5
  124. package/dist/rerankers/http.mjs +2 -2
  125. package/dist/{browser/restore-queue-GEZ54CA5.mjs → restore-queue-FEYAC4SB.mjs} +8 -2
  126. package/dist/{runtime-7MLQ7J64.mjs → runtime-DYV75DCB.mjs} +6 -6
  127. package/dist/{schema-BTXbHCqA.d.ts → schema-B-Runv0n.d.ts} +7 -0
  128. package/dist/server.d.ts +12 -10
  129. package/dist/server.mjs +54 -21
  130. package/dist/{snapshot-DOQLGGEX.mjs → snapshot-CGM65VCD.mjs} +101 -197
  131. package/dist/synonyms.mjs +6 -4
  132. package/dist/{tasks-Co8HfRD7.d.ts → tasks-CjRJRGTb.d.ts} +26 -8
  133. package/dist/{thread-SKO7CN3R.mjs → thread-KGNAIKPD.mjs} +14 -14
  134. package/dist/vector/search-worker.d.ts +1 -1
  135. package/dist/vector/search-worker.mjs +6 -5
  136. package/dist/workers/entry.mjs +8 -8
  137. package/dist/{write-ZKR7SNRT.mjs → write-2DZPEQL2.mjs} +82 -107
  138. package/openapi.json +114 -6
  139. package/package.json +12 -8
@@ -1,231 +1,19 @@
1
- import { KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, restoreVectorFields, KEYWORD_SEGMENT_PAYLOAD_VERSION, littleEndianBytes, uint64Bytes } from './chunk-7OIQ2SIO.mjs';
2
- import { WIRE_BATCH_BUDGET } from './chunk-P36MUFEJ.mjs';
1
+ import { WIRE_BATCH_BUDGET } from './chunk-JDDXV6UF.mjs';
2
+ import { DEFAULT_COMPACTION_THRESHOLD, writeSegmentsOf, readSegmentFile, MERGED_SEGMENT_GROWTH_ALLOWANCE, MERGED_SEGMENT_TARGET_BYTES, COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT, SEGMENT_FILE_FRAMING_SLACK_BYTES } from './chunk-7AUASRE7.mjs';
3
+ import { encodeSegmentManifest, SEGMENT_MANIFEST_VERSION, manifestKey, manifestReferencedKeys, segmentsPrefix, snapshotBundleKey, MAX_SEGMENTS_PER_PARTITION } from './chunk-UUZCQQG2.mjs';
4
+ import { restoreVectorFields } from './chunk-KEIYVZUX.mjs';
3
5
  import { ReplicationMessageTypes } from './chunk-YH5OIL7B.mjs';
4
- import { packChecksummedEnvelope, crc32, openChecksummedEnvelope, reconstructSchemaFromMetadata, HEADER_SIZE, prepareDocumentVectors } from './chunk-TGHNXN7W.mjs';
5
- import { freezeSegmentShared, createSharedFrozenSegment, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, selectionKeeps, decodeMapWithoutFields, shareSegmentArrays, segmentArraysOf, mergeSegmentContents } from './chunk-RWESFSJI.mjs';
6
- import { fnv1a, isCompositePartition, survivorDocumentsOf, partitionStateOf } from './chunk-2MKX2SCO.mjs';
7
- import { generateId, getLanguage } from './chunk-ZR4VM4A6.mjs';
8
- import { extractVectorFieldsFromSchema, partialFieldsOf, messagePackUpperBoundBytes } from './chunk-PU6SKH7O.mjs';
6
+ import { crc32, reconstructSchemaFromMetadata, prepareDocumentVectors } from './chunk-IO5F7ZBE.mjs';
7
+ import { freezeSegmentShared, createSharedFrozenSegment, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, decodeMapWithoutFields, shareSegmentArrays } from './chunk-WJGO5PGV.mjs';
8
+ import { isCompositePartition, survivorDocumentsOf, partitionStateOf } from './chunk-ZPWS2SKJ.mjs';
9
+ import { generateId } from './chunk-N37VMC62.mjs';
10
+ import { extractVectorFieldsFromSchema, partialFieldsOf } from './chunk-KGOXHLLZ.mjs';
9
11
  import { MAX_FILE_BYTES } from './chunk-U476IXKQ.mjs';
10
- import { resolveIndexAnalysis } from './chunk-6SSVIW7Y.mjs';
11
- import { NarsilError, ErrorCodes } from './chunk-CZ7ISLXA.mjs';
12
+ import { resolveIndexAnalysis } from './chunk-O4LJQI4G.mjs';
13
+ import { getLanguage } from './chunk-WNBRW4Q2.mjs';
14
+ import { NarsilError, ErrorCodes } from './chunk-HNHCNTVG.mjs';
12
15
  import { encode, decode } from '@msgpack/msgpack';
13
16
 
14
- // src/persistence/durability/constants.ts
15
- var DEFAULT_CHECKPOINT_INTERVAL_MS = 3e5;
16
- var DEFAULT_CHECKPOINT_MUTATION_THRESHOLD = 1e5;
17
- var DEFAULT_ASYNC_FLUSH_INTERVAL_MS = 1e3;
18
- var DIRECTORY_IDENTITY_CHECK_INTERVAL_MS = 1e3;
19
- var DIRECTORY_LOCK_ATTEMPTS = 3;
20
- var DIRECTORY_LOCK_SETTLE_MS = 50;
21
- var DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS = 1e3;
22
- var DEFAULT_SEGMENT_MAX_BYTES = 67108864;
23
- var DEFAULT_COMPACTION_THRESHOLD = 12;
24
- var COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT = 2e5;
25
- var SEGMENT_FILE_FRAMING_SLACK_BYTES = 65536;
26
- var SEGMENT_TOKEN_TABLE_SHARE_OF_FILE = 0.25;
27
- var WHOLE_PARTITION_MIN_CHANGED_SHARE = 0.5;
28
- var WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT = 1e5;
29
- var LOG_RECORDS_PER_CHECKPOINT = 1e5;
30
- var WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES = 2147483648;
31
- var CHECKPOINT_WORKER_TIMEOUT_MS = 6e5;
32
- var CHECKPOINT_WORKER_HEARTBEAT_MS = 5e3;
33
- var CHECKPOINT_WORKER_IDLE_MS = 5e3;
34
- var CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS = 50;
35
-
36
- // src/persistence/durability/segment/layout.ts
37
- var SEGMENT_ID_WIDTH = 16;
38
- function manifestKey(indexName) {
39
- return `${indexName}/manifest`;
40
- }
41
- function snapshotBundleKey(indexName) {
42
- return `${indexName}/snapshot`;
43
- }
44
- function segmentPrefix(indexName, partitionId) {
45
- return `${indexName}/segments/${partitionId}/`;
46
- }
47
- function segmentKey(indexName, partitionId, segmentId) {
48
- return `${segmentPrefix(indexName, partitionId)}s${formatSegmentId(segmentId)}`;
49
- }
50
- function segmentsPrefix(indexName) {
51
- return `${indexName}/segments/`;
52
- }
53
- function vectorFileKey(indexName, fieldPath, fileId) {
54
- return `${segmentsPrefix(indexName)}vec-${encodeFieldPath(fieldPath)}-f${formatSegmentId(fileId)}`;
55
- }
56
- function requireGeneration(generation) {
57
- if (Number.isSafeInteger(generation) && generation > 0) return;
58
- throw new NarsilError(
59
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
60
- `Vector graph generation ${generation} must be a positive integer`,
61
- { generation }
62
- );
63
- }
64
- function graphPartKey(indexName, fieldPath, generation, part) {
65
- requireGeneration(generation);
66
- return `${segmentsPrefix(indexName)}vec-${encodeFieldPath(fieldPath)}-graph-g${generation}-p${formatSegmentId(part)}`;
67
- }
68
- function formatSegmentId(segmentId) {
69
- if (!Number.isInteger(segmentId) || segmentId < 0) {
70
- throw new NarsilError(
71
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
72
- `Segment id ${segmentId} must be a non-negative integer`,
73
- { segmentId }
74
- );
75
- }
76
- return segmentId.toString().padStart(SEGMENT_ID_WIDTH, "0");
77
- }
78
- var FIELD_PATH_PATTERN = /^[A-Za-z0-9_.]+$/;
79
- function encodeFieldPath(fieldPath) {
80
- if (!FIELD_PATH_PATTERN.test(fieldPath)) {
81
- throw new NarsilError(
82
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
83
- `Vector field path "${fieldPath}" contains characters that cannot be encoded in a segment key`,
84
- { fieldPath }
85
- );
86
- }
87
- const fingerprint = fnv1a(fieldPath).toString(36);
88
- return `${fieldPath.replace(/\./g, "_")}-${fingerprint}`;
89
- }
90
- var SEGMENT_MANIFEST_VERSION = 7;
91
- var EARLIER_SEGMENT_MANIFEST_VERSION = 6;
92
- var MANIFEST_ENVELOPE_VERSION = 2;
93
- var MAX_SEGMENTS_PER_PARTITION = 65536;
94
- var MAX_SEGMENT_ID = 281474976710655;
95
- function fileRefOnTheWire(ref) {
96
- return { key: ref.key, bytes: ref.bytes, checksum: ref.checksum };
97
- }
98
- function encodeSegmentManifest(manifest) {
99
- const payload = encode({
100
- version: SEGMENT_MANIFEST_VERSION,
101
- schema: manifest.schema,
102
- language: manifest.language,
103
- checkpoint: manifest.checkpoint.map((c) => ({
104
- partitionId: c.partitionId,
105
- lastSeqNo: c.lastSeqNo,
106
- primaryTerm: c.primaryTerm,
107
- ...c.termHistory === void 0 ? {} : {
108
- termHistory: c.termHistory.map((start) => ({ primaryTerm: start.primaryTerm, firstSeqNo: start.firstSeqNo }))
109
- }
110
- })),
111
- partitions: manifest.partitions.map((p) => ({
112
- partitionId: p.partitionId,
113
- nextSegmentId: p.nextSegmentId,
114
- segments: p.segments.map((s) => ({
115
- id: s.id,
116
- key: s.key,
117
- docCount: s.docCount,
118
- tombstoneCount: s.tombstoneCount,
119
- bytes: s.bytes,
120
- checksum: s.checksum
121
- }))
122
- })),
123
- vectors: manifest.vectors.map((v) => ({
124
- fieldPath: v.fieldPath,
125
- nextFileId: v.nextFileId,
126
- files: v.files.map((f) => ({ id: f.id, ...fileRefOnTheWire(f), count: f.count, dead: f.dead })),
127
- graphGeneration: v.graphGeneration,
128
- graphParts: v.graphParts.map(fileRefOnTheWire)
129
- }))
130
- });
131
- return packChecksummedEnvelope([payload], MANIFEST_ENVELOPE_VERSION, "segment manifest");
132
- }
133
- function manifestReferencedKeys(manifest) {
134
- const keys = /* @__PURE__ */ new Set();
135
- for (const partition of manifest.partitions) {
136
- for (const segment of partition.segments) {
137
- keys.add(segment.key);
138
- }
139
- }
140
- for (const vector of manifest.vectors) {
141
- for (const file of vector.files) keys.add(file.key);
142
- for (const part of vector.graphParts) keys.add(part.key);
143
- }
144
- return keys;
145
- }
146
- function manifestFileRefs(manifest) {
147
- const refs = [];
148
- for (const partition of manifest.partitions) refs.push(...partition.segments);
149
- for (const vector of manifest.vectors) refs.push(...vector.files, ...vector.graphParts);
150
- return refs;
151
- }
152
-
153
- // src/persistence/durability/segment/store.ts
154
- function directoryCheckpointStore(directory) {
155
- const store = {
156
- read: (key) => directory.read(key),
157
- write: (key, chunks) => directory.atomicWrite(key, chunks),
158
- remove: (key) => directory.remove(key),
159
- list: (prefix) => directory.list(prefix),
160
- pathOf: (key) => directory.pathOf(key)
161
- };
162
- const byteLength = directory.byteLength;
163
- if (byteLength !== void 0) store.byteLength = (key) => byteLength.call(directory, key);
164
- return store;
165
- }
166
- function joined(chunks) {
167
- if (chunks.length === 1) return chunks[0];
168
- let total = 0;
169
- for (const chunk of chunks) total += chunk.byteLength;
170
- const bytes = new Uint8Array(total);
171
- let offset = 0;
172
- for (const chunk of chunks) {
173
- bytes.set(chunk, offset);
174
- offset += chunk.byteLength;
175
- }
176
- return bytes;
177
- }
178
- function adapterFailure(code, verb, key, error) {
179
- if (error instanceof NarsilError) return error;
180
- const cause = error instanceof Error ? error.message : String(error);
181
- return new NarsilError(code, `The persistence adapter failed to ${verb} "${key}": ${cause}`, { key, cause });
182
- }
183
- async function guarded(run, code, verb, key) {
184
- try {
185
- return await run();
186
- } catch (error) {
187
- throw adapterFailure(code, verb, key, error);
188
- }
189
- }
190
- function adapterCheckpointStore(adapter) {
191
- return {
192
- read: (key) => guarded(() => adapter.load(key), ErrorCodes.PERSISTENCE_LOAD_FAILED, "load", key),
193
- write: (key, chunks) => guarded(() => adapter.save(key, joined(chunks)), ErrorCodes.PERSISTENCE_SAVE_FAILED, "save", key),
194
- remove: (key) => guarded(() => adapter.delete(key), ErrorCodes.PERSISTENCE_DELETE_FAILED, "delete", key),
195
- list: (prefix) => guarded(() => adapter.list(prefix), ErrorCodes.PERSISTENCE_LOAD_FAILED, "list", prefix)
196
- };
197
- }
198
- async function readListedFile(store, key, expected) {
199
- const length = store.byteLength === void 0 ? void 0 : await store.byteLength(key);
200
- if (length === null) missingFile(expected.what, key);
201
- if (length !== void 0) requireListedLength(expected.what, key, length, expected.bytes);
202
- const bytes = await store.read(key);
203
- if (bytes === null) missingFile(expected.what, key);
204
- requireListedLength(expected.what, key, bytes.byteLength, expected.bytes);
205
- return bytes;
206
- }
207
- function missingFile(what, key) {
208
- throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The ${what} "${key}" that the manifest lists is missing`, {
209
- key
210
- });
211
- }
212
- function requireListedLength(what, key, found, listed) {
213
- if (found === listed) return;
214
- throw new NarsilError(
215
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
216
- `The ${what} "${key}" holds ${found} bytes where the manifest lists ${listed}`,
217
- { key, bytes: found, listed }
218
- );
219
- }
220
- function requireListedChecksum(what, key, found, listed) {
221
- if (found === listed) return;
222
- throw new NarsilError(
223
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
224
- `The ${what} "${key}" carries checksum ${found} where the manifest lists ${listed}`,
225
- { key, checksum: found, listed }
226
- );
227
- }
228
-
229
17
  // src/persistence/durability/segment/captured-partition.ts
230
18
  function keepMaskOf(segment) {
231
19
  const keep = new Uint8Array(segment.arrays.documentCount);
@@ -676,10 +464,10 @@ var RECORD_LENGTH_SIZE = 4;
676
464
  var FRAME_CRC_SIZE = 4;
677
465
  var MAX_WAL_RECORD_BYTES = 268435456;
678
466
  function writeSegmentHeader() {
679
- const header2 = new Uint8Array(SEGMENT_HEADER_SIZE);
680
- header2.set(WAL_MAGIC, 0);
681
- header2[4] = WAL_FORMAT_VERSION & 255;
682
- return header2;
467
+ const header = new Uint8Array(SEGMENT_HEADER_SIZE);
468
+ header.set(WAL_MAGIC, 0);
469
+ header[4] = WAL_FORMAT_VERSION & 255;
470
+ return header;
683
471
  }
684
472
  function frameRecord(entry) {
685
473
  const payload = encode(entry);
@@ -720,9 +508,9 @@ function corrupt(message, details) {
720
508
  throw new NarsilError(ErrorCodes.PERSISTENCE_WAL_CORRUPT, message, details);
721
509
  }
722
510
  function readDurableRegion(data, durableByteLength, maxRecordBytes = MAX_WAL_RECORD_BYTES) {
723
- const header2 = checkSegmentHeader(data);
724
- if (!header2.ok) {
725
- corrupt(`WAL segment header invalid in durable region: ${header2.reason}`, { reason: header2.reason });
511
+ const header = checkSegmentHeader(data);
512
+ if (!header.ok) {
513
+ corrupt(`WAL segment header invalid in durable region: ${header.reason}`, { reason: header.reason });
726
514
  }
727
515
  if (durableByteLength < SEGMENT_HEADER_SIZE || durableByteLength > data.length) {
728
516
  corrupt("Commit marker durable byte length is outside the segment", {
@@ -821,6 +609,9 @@ function readTailBeyondFrontier(data, durableByteLength, lastDurableSeqNo, maxRe
821
609
  function walPrefix(indexName, partitionId) {
822
610
  return `${indexName}/wal/${partitionId}/`;
823
611
  }
612
+ function nextSegmentKey(indexName, partitionId) {
613
+ return `${walPrefix(indexName, partitionId)}next`;
614
+ }
824
615
  function segmentStartSeqNo(key, prefix) {
825
616
  if (!key.startsWith(prefix)) {
826
617
  return null;
@@ -865,14 +656,14 @@ async function* readSegmentsInOrder(directory, wal) {
865
656
  continue;
866
657
  }
867
658
  if (startSeqNo < marker.state.activeSegmentSeqNo) {
868
- const header2 = checkSegmentHeader(bytes);
869
- if (!header2.ok) {
659
+ const header = checkSegmentHeader(bytes);
660
+ if (!header.ok) {
870
661
  throw new NarsilError(
871
662
  ErrorCodes.PERSISTENCE_WAL_CORRUPT,
872
- `Sealed WAL segment header invalid: ${header2.reason}`,
663
+ `Sealed WAL segment header invalid: ${header.reason}`,
873
664
  {
874
665
  key,
875
- reason: header2.reason
666
+ reason: header.reason
876
667
  }
877
668
  );
878
669
  }
@@ -1002,569 +793,23 @@ async function buildSegmentFromEntries(input) {
1002
793
  if (input.keepsWriteRecords) built.writeRecords = /* @__PURE__ */ new Map([[segment, writeRecordsOf(documents, written)]]);
1003
794
  return built;
1004
795
  }
1005
- var FIXMAP = 128;
1006
- var MAP16 = 222;
1007
- var MAP32 = 223;
1008
- var FIXARRAY = 144;
1009
- var ARRAY16 = 220;
1010
- var ARRAY32 = 221;
1011
- var BIN8 = 196;
1012
- var BIN16 = 197;
1013
- var BIN32 = 198;
1014
- var FIX_COUNT_LIMIT = 16;
1015
- var UINT8_LIMIT = 256;
1016
- var UINT16_LIMIT = 65536;
1017
- function header(fixed, sixteen, thirtyTwo, count) {
1018
- if (count < FIX_COUNT_LIMIT) return Uint8Array.of(fixed | count);
1019
- if (count < UINT16_LIMIT) return Uint8Array.of(sixteen, count >> 8, count & 255);
1020
- const bytes = new Uint8Array(5);
1021
- bytes[0] = thirtyTwo;
1022
- new DataView(bytes.buffer).setUint32(1, count, false);
1023
- return bytes;
1024
- }
1025
- function binHeader(byteLength) {
1026
- if (byteLength < UINT8_LIMIT) return Uint8Array.of(BIN8, byteLength);
1027
- if (byteLength < UINT16_LIMIT) return Uint8Array.of(BIN16, byteLength >> 8, byteLength & 255);
1028
- const bytes = new Uint8Array(5);
1029
- bytes[0] = BIN32;
1030
- new DataView(bytes.buffer).setUint32(1, byteLength, false);
1031
- return bytes;
1032
- }
1033
- var MessagePackChunks = class {
1034
- chunks = [];
1035
- byteLength = 0;
1036
- push(chunk) {
1037
- if (chunk.byteLength === 0) return;
1038
- this.chunks.push(chunk);
1039
- this.byteLength += chunk.byteLength;
1040
- }
1041
- map(entries) {
1042
- this.push(header(FIXMAP, MAP16, MAP32, entries));
1043
- return this;
1044
- }
1045
- array(items) {
1046
- this.push(header(FIXARRAY, ARRAY16, ARRAY32, items));
1047
- return this;
1048
- }
1049
- value(value) {
1050
- this.push(encode(value));
1051
- return this;
1052
- }
1053
- key(name) {
1054
- return this.value(name);
1055
- }
1056
- binary(bytes) {
1057
- this.push(binHeader(bytes.byteLength));
1058
- this.push(bytes);
1059
- return this;
1060
- }
1061
- binaryOrNil(bytes) {
1062
- return bytes === null ? this.value(null) : this.binary(bytes);
1063
- }
1064
- };
1065
-
1066
- // src/serialization/keyword-segment/token-order.ts
1067
- function slotsAreInTokenOrder(payloadSlots) {
1068
- for (let at = 0; at < payloadSlots.length; at++) {
1069
- if (payloadSlots[at] !== at) return false;
1070
- }
1071
- return true;
1072
- }
1073
- function reorderedPositions(arrays, postingOffsets) {
1074
- const { positionOffsets, positionValues, tokenTable } = arrays;
1075
- if (positionOffsets === null || positionValues === null) return { positionOffsets: null, positionValues: null };
1076
- const postings = postingOffsets[postingOffsets.length - 1];
1077
- const offsets = new Uint32Array(postings + 1);
1078
- const values = new Uint32Array(positionValues.length);
1079
- let cursor = 0;
1080
- for (let token = 0; token < tokenTable.payloadSlots.length; token++) {
1081
- const slot = tokenTable.payloadSlots[token];
1082
- for (let from = arrays.postingOffsets[slot]; from < arrays.postingOffsets[slot + 1]; from++) {
1083
- const to = postingOffsets[token] + (from - arrays.postingOffsets[slot]);
1084
- offsets[to] = cursor;
1085
- const start = positionOffsets[from];
1086
- const end = positionOffsets[from + 1];
1087
- values.set(positionValues.subarray(start, end), cursor);
1088
- cursor += end - start;
1089
- }
1090
- }
1091
- offsets[postings] = cursor;
1092
- return { positionOffsets: offsets, positionValues: values };
1093
- }
1094
- function postingsInTokenOrder(arrays) {
1095
- const { tokenTable } = arrays;
1096
- if (slotsAreInTokenOrder(tokenTable.payloadSlots)) return arrays;
1097
- const tokens = tokenTable.payloadSlots.length;
1098
- const postingOffsets = new Uint32Array(tokens + 1);
1099
- for (let token = 0; token < tokens; token++) {
1100
- const slot = tokenTable.payloadSlots[token];
1101
- postingOffsets[token + 1] = postingOffsets[token] + arrays.postingOffsets[slot + 1] - arrays.postingOffsets[slot];
1102
- }
1103
- const postings = postingOffsets[tokens];
1104
- const postingDocIds = new Uint32Array(postings);
1105
- const postingFrequencies = new Uint16Array(postings);
1106
- const postingFieldIndices = new Uint16Array(postings);
1107
- for (let token = 0; token < tokens; token++) {
1108
- const slot = tokenTable.payloadSlots[token];
1109
- const start = arrays.postingOffsets[slot];
1110
- const end = arrays.postingOffsets[slot + 1];
1111
- postingDocIds.set(arrays.postingDocIds.subarray(start, end), postingOffsets[token]);
1112
- postingFrequencies.set(arrays.postingFrequencies.subarray(start, end), postingOffsets[token]);
1113
- postingFieldIndices.set(arrays.postingFieldIndices.subarray(start, end), postingOffsets[token]);
1114
- }
1115
- return {
1116
- postingOffsets,
1117
- postingDocIds,
1118
- postingFrequencies,
1119
- postingFieldIndices,
1120
- ...reorderedPositions(arrays, postingOffsets)
1121
- };
1122
- }
1123
-
1124
- // src/serialization/keyword-segment/encode.ts
1125
- var TOP_LEVEL_ENTRIES = 28;
1126
- function writeFieldLengths(out, arrays) {
1127
- out.array(arrays.fieldLengthNames.length);
1128
- for (let f = 0; f < arrays.fieldLengthNames.length; f++) {
1129
- out.map(2).key("field").value(arrays.fieldLengthNames[f]);
1130
- out.key("lengths").binary(littleEndianBytes(arrays.fieldLengthColumns[f]));
1131
- }
1132
- }
1133
- function writeValueColumns(out, columns) {
1134
- out.array(columns.length);
1135
- for (const column of columns) {
1136
- out.map(3).key("field").value(column.fieldPath);
1137
- out.key("documents").binary(littleEndianBytes(column.docIds));
1138
- out.key("values").binary(littleEndianBytes(column.values));
1139
- }
1140
- }
1141
- function writeBooleanColumns(out, columns) {
1142
- out.array(columns.length);
1143
- for (const column of columns) {
1144
- out.map(3).key("field").value(column.fieldPath);
1145
- out.key("trueDocuments").binary(littleEndianBytes(column.trueDocs));
1146
- out.key("falseDocuments").binary(littleEndianBytes(column.falseDocs));
1147
- }
1148
- }
1149
- function writeEnumColumns(out, columns) {
1150
- out.array(columns.length);
1151
- for (const column of columns) {
1152
- out.map(4).key("field").value(column.fieldPath);
1153
- out.key("values").value(column.values);
1154
- out.key("offsets").binary(littleEndianBytes(column.offsets));
1155
- out.key("documents").binary(littleEndianBytes(column.docIds));
1156
- }
1157
- }
1158
- function writeGeopointColumns(out, columns) {
1159
- out.array(columns.length);
1160
- for (const column of columns) {
1161
- out.map(4).key("field").value(column.fieldPath);
1162
- out.key("documents").binary(littleEndianBytes(column.docIds));
1163
- out.key("latitudes").binary(littleEndianBytes(column.latitudes));
1164
- out.key("longitudes").binary(littleEndianBytes(column.longitudes));
1165
- }
1166
- }
1167
- function writePatternColumns(out, columns) {
1168
- out.array(columns.length);
1169
- for (const column of columns) {
1170
- out.map(5).key("field").value(column.fieldPath);
1171
- out.key("documents").binary(littleEndianBytes(column.docIds));
1172
- out.key("runs").binary(littleEndianBytes(column.runs));
1173
- out.key("offsets").binary(littleEndianBytes(column.offsets));
1174
- out.key("postings").binary(littleEndianBytes(column.postings));
1175
- }
1176
- }
1177
- function writeSurfaceForms(out, arrays) {
1178
- const table = arrays.surfaceTable;
1179
- if (table === null || table.counts.length === 0) {
1180
- out.value(null);
1181
- return;
1182
- }
1183
- out.map(3).key("text").binary(table.blob);
1184
- out.key("offsets").binary(littleEndianBytes(table.offsets));
1185
- out.key("counts").binary(littleEndianBytes(table.counts));
1186
- }
1187
- function writePartialWords(out, arrays) {
1188
- const table = arrays.partialWords;
1189
- if (table === null) {
1190
- out.value(null);
1191
- return;
1192
- }
1193
- out.map(8).key("fields").value(table.fields);
1194
- out.key("text").binary(table.spellings);
1195
- out.key("offsets").binary(littleEndianBytes(table.spellingOffsets));
1196
- out.key("tokens").binary(littleEndianBytes(table.spellingTokens));
1197
- out.key("counts").binary(littleEndianBytes(table.spellingCounts));
1198
- out.key("runs").binary(littleEndianBytes(table.runs));
1199
- out.key("runOffsets").binary(littleEndianBytes(table.runOffsets));
1200
- out.key("runEntries").binary(littleEndianBytes(table.runEntries));
1201
- }
1202
- function encodeKeywordSegment(contents) {
1203
- const { arrays } = contents;
1204
- const postings = postingsInTokenOrder(arrays);
1205
- const out = new MessagePackChunks();
1206
- const { writeRecords } = contents;
1207
- out.map(writeRecords === void 0 ? TOP_LEVEL_ENTRIES : TOP_LEVEL_ENTRIES + 2);
1208
- out.key("v").value(KEYWORD_SEGMENT_PAYLOAD_VERSION);
1209
- out.key("schema").value(contents.schema);
1210
- out.key("tombstones").value(contents.tombstones);
1211
- out.key("documentCount").value(arrays.documentCount);
1212
- out.key("ids").binary(arrays.idTable.blob);
1213
- out.key("idOffsets").binary(littleEndianBytes(arrays.idTable.offsets));
1214
- out.key("idOrder").binary(littleEndianBytes(arrays.idTable.sortedOrdinals));
1215
- out.key("documents").binary(arrays.documentTable.blob);
1216
- out.key("documentOffsets").binary(littleEndianBytes(arrays.documentTable.offsets));
1217
- out.key("fieldNames").value(arrays.fieldNames);
1218
- out.key("tokens").binary(arrays.tokenTable.blob);
1219
- out.key("tokenOffsets").binary(littleEndianBytes(arrays.tokenTable.offsets));
1220
- out.key("documentFrequencies").binary(littleEndianBytes(arrays.tokenTable.documentFrequencies));
1221
- out.key("postingOffsets").binary(littleEndianBytes(postings.postingOffsets));
1222
- out.key("postingDocuments").binary(littleEndianBytes(postings.postingDocIds));
1223
- out.key("postingFrequencies").binary(littleEndianBytes(postings.postingFrequencies));
1224
- out.key("postingFields").binary(littleEndianBytes(postings.postingFieldIndices));
1225
- out.key("positionOffsets").binaryOrNil(postings.positionOffsets === null ? null : littleEndianBytes(postings.positionOffsets));
1226
- out.key("positions").binaryOrNil(postings.positionValues === null ? null : littleEndianBytes(postings.positionValues));
1227
- out.key("fieldLengths");
1228
- writeFieldLengths(out, arrays);
1229
- out.key("numeric");
1230
- writeValueColumns(out, arrays.numeric);
1231
- out.key("sizes");
1232
- writeValueColumns(out, arrays.sizes);
1233
- out.key("boolean");
1234
- writeBooleanColumns(out, arrays.boolean);
1235
- out.key("enum");
1236
- writeEnumColumns(out, arrays.enums);
1237
- out.key("geopoint");
1238
- writeGeopointColumns(out, arrays.geo);
1239
- out.key("pattern");
1240
- writePatternColumns(out, arrays.pattern);
1241
- out.key("surfaceForms");
1242
- writeSurfaceForms(out, arrays);
1243
- out.key("partialWords");
1244
- writePartialWords(out, arrays);
1245
- if (writeRecords !== void 0) {
1246
- out.key("seqNos").binary(uint64Bytes(writeRecords.seqNos));
1247
- out.key("primaryTerms").binary(uint64Bytes(writeRecords.primaryTerms));
1248
- }
1249
- return out;
1250
- }
1251
-
1252
- // src/persistence/durability/segment/segment-file.ts
1253
- function sharesMemoryWithWorkers() {
1254
- return typeof SharedArrayBuffer === "function";
1255
- }
1256
- async function readSegmentFile(store, ref) {
1257
- const bytes = await readListedFile(store, ref.key, { bytes: ref.bytes, what: "segment" });
1258
- const { header: header2, payload } = await openChecksummedEnvelope(bytes, {
1259
- envelopeFormatVersion: KEYWORD_SEGMENT_ENVELOPE_VERSION,
1260
- what: `segment "${ref.key}"`,
1261
- allowsCompression: true
1262
- });
1263
- requireListedChecksum("segment", ref.key, header2.checksum, ref.checksum);
1264
- return decodeKeywordSegment(payload, sharesMemoryWithWorkers());
1265
- }
1266
- var ORDINAL_BYTES = 4;
1267
- var POSTING_BYTES = 8;
1268
- var VALUE_ENTRY_BYTES = 12;
1269
- var GEOPOINT_ENTRY_BYTES = 20;
1270
- var TOKEN_ENTRY_BYTES = 12;
1271
- var STRING_HEADER_BYTES = 5;
1272
- var encoder = new TextEncoder();
1273
- function documentBytes(segment, ordinal) {
1274
- const table = segment.arrays.documentTable;
1275
- if (table !== null) return table.offsets[ordinal + 1] - table.offsets[ordinal];
1276
- return messagePackUpperBoundBytes(segment.documentSource.docAt(ordinal));
1277
- }
1278
- function addPerEntry(costs, docIds, bytes) {
1279
- for (let at = 0; at < docIds.length; at++) costs[docIds[at]] += bytes;
1280
- }
1281
- function chargePostings(segment, costs, chargesTokens) {
1282
- const arrays = segment.arrays;
1283
- const positions = arrays.positionOffsets;
1284
- for (let p = 0; p < arrays.postingDocIds.length; p++) {
1285
- let bytes = POSTING_BYTES;
1286
- if (positions !== null) bytes += ORDINAL_BYTES * (1 + positions[p + 1] - positions[p]);
1287
- costs[arrays.postingDocIds[p]] += bytes;
1288
- }
1289
- if (!chargesTokens) return;
1290
- const table = arrays.tokenTable;
1291
- for (let at = 0; at < table.size; at++) {
1292
- const slot = table.payloadSlot(at);
1293
- const bytes = table.byteLengthAt(at) + TOKEN_ENTRY_BYTES;
1294
- for (let p = arrays.postingOffsets[slot]; p < arrays.postingOffsets[slot + 1]; p++) {
1295
- costs[arrays.postingDocIds[p]] += bytes;
1296
- }
1297
- }
1298
- }
1299
- function chargeColumns(segment, costs) {
1300
- const arrays = segment.arrays;
1301
- for (const column of [...arrays.numeric, ...arrays.sizes]) addPerEntry(costs, column.docIds, VALUE_ENTRY_BYTES);
1302
- for (const column of arrays.boolean) {
1303
- addPerEntry(costs, column.trueDocs, ORDINAL_BYTES);
1304
- addPerEntry(costs, column.falseDocs, ORDINAL_BYTES);
1305
- }
1306
- for (const column of arrays.enums) addPerEntry(costs, column.docIds, ORDINAL_BYTES);
1307
- for (const column of arrays.geo) addPerEntry(costs, column.docIds, GEOPOINT_ENTRY_BYTES);
1308
- for (const column of arrays.pattern) {
1309
- addPerEntry(costs, column.docIds, ORDINAL_BYTES);
1310
- addPerEntry(costs, column.postings, ORDINAL_BYTES);
1311
- }
1312
- }
1313
- function stringListBytes(values) {
1314
- let bytes = STRING_HEADER_BYTES;
1315
- for (const value of values) bytes += STRING_HEADER_BYTES + encoder.encode(value).byteLength;
1316
- return bytes;
1317
- }
1318
- function fixedBytesOf(segment, chargesTokens) {
1319
- const arrays = segment.arrays;
1320
- let bytes = stringListBytes(arrays.fieldNames);
1321
- if (!chargesTokens) bytes += arrays.tokenTable.bytes;
1322
- for (const column of arrays.enums) bytes += stringListBytes(column.values) + column.offsets.byteLength;
1323
- for (const column of arrays.pattern) bytes += column.runs.byteLength + column.offsets.byteLength;
1324
- const surfaces = segment.sharedSnapshot?.surfaceTable ?? null;
1325
- if (surfaces !== null) bytes += surfaces.blob.byteLength + surfaces.offsets.byteLength + surfaces.counts.byteLength;
1326
- else bytes += messagePackUpperBoundBytes(arrays.readSurfaceForms());
1327
- return bytes + segment.partialWords.bytes();
1328
- }
1329
- function keptDocumentsOf(selection, chargesTokens) {
1330
- const { segment } = selection;
1331
- const count = segment.arrays.documentCount;
1332
- const costs = new Float64Array(count);
1333
- const perDocument = ORDINAL_BYTES * (3 + segment.arrays.fieldLengthColumns.length);
1334
- for (let ordinal = 0; ordinal < count; ordinal++) {
1335
- costs[ordinal] = perDocument + encoder.encode(segment.arrays.idTable.idAt(ordinal)).byteLength;
1336
- }
1337
- chargePostings(segment, costs, chargesTokens);
1338
- chargeColumns(segment, costs);
1339
- const ordinals = [];
1340
- for (let ordinal = 0; ordinal < count; ordinal++) {
1341
- if (!selectionKeeps(selection, ordinal)) continue;
1342
- costs[ordinal] += documentBytes(segment, ordinal);
1343
- ordinals.push(ordinal);
1344
- }
1345
- return { segment, ordinals, costs };
1346
- }
1347
- function selectionOf(segment, ordinals) {
1348
- const marked = new Uint8Array(segment.arrays.documentCount);
1349
- for (const ordinal of ordinals) marked[ordinal] = 1;
1350
- return { segment, keep: (ordinal) => marked[ordinal] === 1 };
1351
- }
1352
- function tooLargeForOneFile(cost, budget) {
1353
- throw new NarsilError(
1354
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
1355
- `A document needs up to ${Math.ceil(cost)} bytes of a segment, more than the ${Math.max(0, Math.floor(budget))} that one file leaves it`,
1356
- { bytes: Math.ceil(cost), budget: Math.floor(budget), maximum: MAX_FILE_BYTES }
1357
- );
1358
- }
1359
- function splitIntoSegmentFiles(selections, schema, tombstones) {
1360
- let tokenBytes = 0;
1361
- for (const { segment } of selections) tokenBytes += segment.arrays.tokenTable.bytes;
1362
- const chargesTokens = tokenBytes > MAX_FILE_BYTES * SEGMENT_TOKEN_TABLE_SHARE_OF_FILE;
1363
- let fixed = HEADER_SIZE + SEGMENT_FILE_FRAMING_SLACK_BYTES + encode(schema).byteLength + encode(tombstones).byteLength;
1364
- for (const { segment } of selections) fixed += fixedBytesOf(segment, chargesTokens);
1365
- const budget = MAX_FILE_BYTES - fixed;
1366
- const groups = [];
1367
- let current = [];
1368
- let currentBytes = 0;
1369
- let currentCount = 0;
1370
- const flush = () => {
1371
- if (currentCount === 0) return;
1372
- groups.push({
1373
- selections: current.map((entry) => selectionOf(entry.segment, entry.ordinals)),
1374
- documentCount: currentCount
1375
- });
1376
- current = [];
1377
- currentBytes = 0;
1378
- currentCount = 0;
1379
- };
1380
- for (const selection of selections) {
1381
- const kept = keptDocumentsOf(selection, chargesTokens);
1382
- let entry = null;
1383
- for (const ordinal of kept.ordinals) {
1384
- const cost = kept.costs[ordinal];
1385
- if (cost > budget) tooLargeForOneFile(cost, budget);
1386
- if (currentBytes + cost > budget) {
1387
- flush();
1388
- entry = null;
1389
- }
1390
- if (entry === null) {
1391
- entry = { segment: kept.segment, ordinals: [] };
1392
- current.push(entry);
1393
- }
1394
- entry.ordinals.push(ordinal);
1395
- currentBytes += cost;
1396
- currentCount += 1;
1397
- }
1398
- }
1399
- flush();
1400
- return groups;
1401
- }
1402
- function halvesOf(group) {
1403
- const flat = [];
1404
- for (const selection of group.selections) {
1405
- for (let ordinal = 0; ordinal < selection.segment.arrays.documentCount; ordinal++) {
1406
- if (!selectionKeeps(selection, ordinal)) continue;
1407
- flat.push({ segment: selection.segment, ordinal });
1408
- }
1409
- }
1410
- const middle = Math.ceil(flat.length / 2);
1411
- const groupOf = (entries) => {
1412
- const bySegment = /* @__PURE__ */ new Map();
1413
- for (const { segment, ordinal } of entries) {
1414
- let ordinals = bySegment.get(segment);
1415
- if (ordinals === void 0) {
1416
- ordinals = [];
1417
- bySegment.set(segment, ordinals);
1418
- }
1419
- ordinals.push(ordinal);
1420
- }
1421
- return {
1422
- selections: [...bySegment].map(([segment, ordinals]) => selectionOf(segment, ordinals)),
1423
- documentCount: entries.length
1424
- };
1425
- };
1426
- return [groupOf(flat.slice(0, middle)), groupOf(flat.slice(middle))];
1427
- }
1428
-
1429
- // src/persistence/durability/segment/segment-write.ts
1430
- function emptyPayload() {
1431
- return {
1432
- documentCount: 0,
1433
- docIds: [],
1434
- fieldNames: [],
1435
- tokens: [],
1436
- postingOffsets: new Uint32Array(1),
1437
- postingDocIds: new Uint32Array(0),
1438
- postingFrequencies: new Uint16Array(0),
1439
- postingFieldIndices: new Uint16Array(0),
1440
- positionOffsets: null,
1441
- positionValues: null,
1442
- fieldLengthNames: [],
1443
- fieldLengthColumns: [],
1444
- totalFieldLengths: {},
1445
- docFrequencies: {},
1446
- surfaceForms: null,
1447
- partialSpellings: null,
1448
- numeric: [],
1449
- boolean: [],
1450
- enums: [],
1451
- geo: [],
1452
- pattern: [],
1453
- sizes: []
1454
- };
1455
- }
1456
- function noSpellingsOf(analysis) {
1457
- if (analysis === null || analysis.fields.length === 0) return null;
1458
- return { fields: [...analysis.fields], tokens: [], spellings: [], counts: [] };
1459
- }
1460
- function writeRecordsOfGroup(group, records, count) {
1461
- const seqNos = new Float64Array(count);
1462
- const primaryTerms = new Float64Array(count);
1463
- let at = 0;
1464
- for (const selection of group?.selections ?? []) {
1465
- const source = records.get(selection.segment);
1466
- for (let ordinal = 0; ordinal < selection.segment.arrays.documentCount; ordinal++) {
1467
- if (!selectionKeeps(selection, ordinal)) continue;
1468
- if (source !== void 0) {
1469
- seqNos[at] = source.seqNos[ordinal];
1470
- primaryTerms[at] = source.primaryTerms[ordinal];
1471
- }
1472
- at += 1;
1473
- }
1474
- }
1475
- return { seqNos, primaryTerms };
1476
- }
1477
- function withWriteRecords(contents, group, request) {
1478
- if (request.writeRecords === void 0) return contents;
1479
- const writeRecords = writeRecordsOfGroup(group, request.writeRecords, contents.arrays.documentCount);
1480
- return { ...contents, writeRecords };
1481
- }
1482
- function contentsOfGroup(group, request, tombstones) {
1483
- const { schema, partialWordAnalysis, textAnalysis } = request;
1484
- if (group === null) {
1485
- const payload = { ...emptyPayload(), partialSpellings: noSpellingsOf(partialWordAnalysis) };
1486
- return {
1487
- schema,
1488
- tombstones,
1489
- arrays: segmentArraysOf(payload, { blob: new Uint8Array(0), offsets: new Uint32Array(1) })
1490
- };
1491
- }
1492
- const merged = mergeSegmentContents(group.selections, partialWordAnalysis, textAnalysis);
1493
- return {
1494
- schema,
1495
- tombstones,
1496
- arrays: segmentArraysOf(merged.payload, merged.documents, merged.reusedPartialWords)
1497
- };
1498
- }
1499
- function requireRoomForAnotherSegment(state) {
1500
- if (state.segments.length < MAX_SEGMENTS_PER_PARTITION) return;
1501
- throw new NarsilError(
1502
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
1503
- `Partition ${state.request.partitionId} would hold more than ${MAX_SEGMENTS_PER_PARTITION} segments`,
1504
- { partitionId: state.request.partitionId, maximum: MAX_SEGMENTS_PER_PARTITION }
1505
- );
1506
- }
1507
- async function writeGroup(state, group) {
1508
- const { request } = state;
1509
- const tombstones = state.tombstonesWritten ? [] : [...request.tombstones];
1510
- const contents = withWriteRecords(contentsOfGroup(group, request, tombstones), group, request);
1511
- const encoded = encodeKeywordSegment(contents);
1512
- if (HEADER_SIZE + encoded.byteLength > MAX_FILE_BYTES && group !== null && group.documentCount > 1) {
1513
- const [first, second] = halvesOf(group);
1514
- await writeGroup(state, first);
1515
- await writeGroup(state, second);
1516
- return;
1517
- }
1518
- requireRoomForAnotherSegment(state);
1519
- const id = state.nextSegmentId;
1520
- const key = segmentKey(request.indexName, request.partitionId, id);
1521
- const packed = await packChecksummedEnvelope(encoded.chunks, KEYWORD_SEGMENT_ENVELOPE_VERSION, `segment "${key}"`);
1522
- await request.store.write(key, packed.chunks);
1523
- state.segments.push({
1524
- id,
1525
- key,
1526
- bytes: packed.byteLength,
1527
- checksum: packed.checksum,
1528
- docCount: contents.arrays.documentCount,
1529
- tombstoneCount: tombstones.length
1530
- });
1531
- state.nextSegmentId = id + 1;
1532
- state.tombstonesWritten = true;
1533
- }
1534
- function keptDocumentCount(selections) {
1535
- let count = 0;
1536
- for (const selection of selections) {
1537
- for (let ordinal = 0; ordinal < selection.segment.arrays.documentCount; ordinal++) {
1538
- if (selectionKeeps(selection, ordinal)) count += 1;
1539
- }
1540
- }
1541
- return count;
1542
- }
1543
- async function writeSegmentsOf(request) {
1544
- const state = { request, segments: [], nextSegmentId: request.nextSegmentId, tombstonesWritten: false };
1545
- const groups = request.mergesIntoOneFile === true ? [{ selections: [...request.selections], documentCount: keptDocumentCount(request.selections) }] : splitIntoSegmentFiles(request.selections, request.schema, request.tombstones);
1546
- for (const group of groups) {
1547
- if (group.documentCount > 0) await writeGroup(state, group);
1548
- }
1549
- if (!state.tombstonesWritten && request.tombstones.length > 0) await writeGroup(state, null);
1550
- return { segments: state.segments, nextSegmentId: state.nextSegmentId };
1551
- }
1552
796
 
1553
797
  // src/persistence/durability/segment/compaction.ts
1554
- function windowFits(segments, start, windowLength) {
798
+ function windowFits(segments, start, windowLength, paged) {
1555
799
  let documents = 0;
1556
800
  let bytes = SEGMENT_FILE_FRAMING_SLACK_BYTES;
1557
801
  for (let at = start; at < start + windowLength; at++) {
1558
802
  documents += segments[at].docCount;
1559
803
  bytes += segments[at].bytes;
1560
804
  }
805
+ if (paged) return bytes * MERGED_SEGMENT_GROWTH_ALLOWANCE <= MERGED_SEGMENT_TARGET_BYTES;
1561
806
  return documents <= COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT && bytes <= MAX_FILE_BYTES;
1562
807
  }
1563
- function smallestFittingWindow(segments, windowLength) {
808
+ function smallestFittingWindow(segments, windowLength, paged) {
1564
809
  let best = null;
1565
810
  let bestBytes = Number.POSITIVE_INFINITY;
1566
811
  for (let start = 0; start + windowLength <= segments.length; start++) {
1567
- if (!windowFits(segments, start, windowLength)) continue;
812
+ if (!windowFits(segments, start, windowLength, paged)) continue;
1568
813
  let bytes = 0;
1569
814
  for (let at = start; at < start + windowLength; at++) bytes += segments[at].bytes;
1570
815
  if (bytes < bestBytes) {
@@ -1574,9 +819,9 @@ function smallestFittingWindow(segments, windowLength) {
1574
819
  }
1575
820
  return best;
1576
821
  }
1577
- function chooseWindow(segments, wantedLength) {
822
+ function chooseWindow(segments, wantedLength, paged) {
1578
823
  for (let windowLength = Math.min(wantedLength, segments.length); windowLength >= 2; windowLength--) {
1579
- const window = smallestFittingWindow(segments, windowLength);
824
+ const window = smallestFittingWindow(segments, windowLength, paged);
1580
825
  if (window !== null) return window;
1581
826
  }
1582
827
  return null;
@@ -1617,11 +862,39 @@ async function compactPartitionSegments(input) {
1617
862
  if (segments.length <= compactionThreshold) {
1618
863
  return { segments, nextSegmentId: input.nextSegmentId };
1619
864
  }
1620
- const window = chooseWindow(segments, segments.length - compactionThreshold + 1);
865
+ const paged = input.paged === true;
866
+ const window = chooseWindow(segments, segments.length - compactionThreshold + 1, paged);
1621
867
  if (window === null) {
1622
868
  return { segments, nextSegmentId: input.nextSegmentId };
1623
869
  }
1624
870
  const windowRefs = segments.slice(window.start, window.start + window.windowLength);
871
+ const replacing = (written2) => ({
872
+ segments: [
873
+ ...segments.slice(0, window.start),
874
+ ...written2.segments,
875
+ ...segments.slice(window.start + window.windowLength)
876
+ ],
877
+ nextSegmentId: written2.nextSegmentId
878
+ });
879
+ if (paged) {
880
+ const { directory } = input.store;
881
+ if (directory === void 0) return { segments, nextSegmentId: input.nextSegmentId };
882
+ const { mergePagedWindow } = await import('./module-IYPLWHGC.mjs');
883
+ return replacing(
884
+ await mergePagedWindow({
885
+ store: input.store,
886
+ directory,
887
+ indexName: input.indexName,
888
+ partitionId: input.partitionId,
889
+ schema: input.schema,
890
+ partialWordAnalysis: input.partialWordAnalysis,
891
+ textAnalysis: input.textAnalysis,
892
+ nextSegmentId: input.nextSegmentId,
893
+ refs: windowRefs,
894
+ dropsEveryTombstone: window.start === 0
895
+ })
896
+ );
897
+ }
1625
898
  const contents = [];
1626
899
  for (const ref of windowRefs) contents.push(await readSegmentFile(input.store, ref));
1627
900
  const frozen = contents.map(frozenOf);
@@ -1640,14 +913,7 @@ async function compactPartitionSegments(input) {
1640
913
  mergesIntoOneFile: true,
1641
914
  ...writeRecords === void 0 ? {} : { writeRecords }
1642
915
  });
1643
- return {
1644
- segments: [
1645
- ...segments.slice(0, window.start),
1646
- ...written.segments,
1647
- ...segments.slice(window.start + window.windowLength)
1648
- ],
1649
- nextSegmentId: written.nextSegmentId
1650
- };
916
+ return replacing(written);
1651
917
  }
1652
918
 
1653
919
  // src/persistence/durability/segment/write.ts
@@ -1694,6 +960,21 @@ function wholeContentsOf(input, partitionId) {
1694
960
  const captured = input.capturedPartitions?.find((candidate) => candidate.partitionId === partitionId);
1695
961
  return captured === void 0 ? void 0 : contentsOfCaptured(captured);
1696
962
  }
963
+ function pagedSettingsOf(context) {
964
+ if (context.input.metadata.textStorage !== "disk") return {};
965
+ return { paged: true, keepsWriteRecords: context.input.keepsWriteRecords === true };
966
+ }
967
+ async function tombstonesOfDelta(context, older, selections, deleted) {
968
+ const { directory } = context.input.store;
969
+ if (context.input.metadata.textStorage !== "disk" || directory === void 0) return deleted;
970
+ const { pagedTombstonesFor } = await import('./module-IYPLWHGC.mjs');
971
+ return pagedTombstonesFor(
972
+ directory,
973
+ older,
974
+ selections.map((selection) => selection.segment),
975
+ deleted
976
+ );
977
+ }
1697
978
  async function writeWholePartition(context, partitionId, priorPartition, whole) {
1698
979
  const written = await writeSegmentsOf({
1699
980
  store: context.input.store,
@@ -1705,7 +986,8 @@ async function writeWholePartition(context, partitionId, priorPartition, whole)
1705
986
  textAnalysis: context.textAnalysis,
1706
987
  selections: whole.selections,
1707
988
  tombstones: [],
1708
- ...whole.writeRecords === void 0 ? {} : { writeRecords: whole.writeRecords }
989
+ ...whole.writeRecords === void 0 ? {} : { writeRecords: whole.writeRecords },
990
+ ...pagedSettingsOf(context)
1709
991
  });
1710
992
  return { partitionId, nextSegmentId: written.nextSegmentId, segments: written.segments };
1711
993
  }
@@ -1721,6 +1003,7 @@ async function writeLoggedChanges(context, partitionId, priorPartition, entries)
1721
1003
  let segments = priorPartition ? [...priorPartition.segments] : [];
1722
1004
  let nextSegmentId = priorPartition?.nextSegmentId ?? 0;
1723
1005
  if (built !== null) {
1006
+ const tombstones = await tombstonesOfDelta(context, segments, built.selections, built.tombstones);
1724
1007
  const written = await writeSegmentsOf({
1725
1008
  store: context.input.store,
1726
1009
  indexName: context.indexName,
@@ -1730,8 +1013,9 @@ async function writeLoggedChanges(context, partitionId, priorPartition, entries)
1730
1013
  partialWordAnalysis: context.partialWordAnalysis,
1731
1014
  textAnalysis: context.textAnalysis,
1732
1015
  selections: built.selections,
1733
- tombstones: built.tombstones,
1734
- ...built.writeRecords === void 0 ? {} : { writeRecords: built.writeRecords }
1016
+ tombstones,
1017
+ ...built.writeRecords === void 0 ? {} : { writeRecords: built.writeRecords },
1018
+ ...pagedSettingsOf(context)
1735
1019
  });
1736
1020
  segments.push(...written.segments);
1737
1021
  nextSegmentId = written.nextSegmentId;
@@ -1752,7 +1036,8 @@ async function writeLoggedChanges(context, partitionId, priorPartition, entries)
1752
1036
  textAnalysis: context.textAnalysis,
1753
1037
  segments,
1754
1038
  nextSegmentId,
1755
- compactionThreshold: context.compactionThreshold
1039
+ compactionThreshold: context.compactionThreshold,
1040
+ paged: context.input.metadata.textStorage === "disk"
1756
1041
  });
1757
1042
  segments = compacted.segments;
1758
1043
  nextSegmentId = compacted.nextSegmentId;
@@ -1815,8 +1100,10 @@ async function writeCheckpointSegments(input) {
1815
1100
  async function commitCheckpointManifest(store, metadata, written, vectors) {
1816
1101
  const indexName = metadata.indexName;
1817
1102
  const manifest = {
1103
+ version: SEGMENT_MANIFEST_VERSION,
1818
1104
  schema: metadata.schema,
1819
1105
  language: metadata.language,
1106
+ ...metadata.analysisGeneration ? { analysisGeneration: metadata.analysisGeneration } : {},
1820
1107
  checkpoint: written.checkpoint,
1821
1108
  partitions: written.partitions,
1822
1109
  vectors
@@ -1836,4 +1123,4 @@ async function removeCheckpointGarbage(store, garbage) {
1836
1123
  }
1837
1124
  }
1838
1125
 
1839
- export { CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_HEARTBEAT_MS, CHECKPOINT_WORKER_IDLE_MS, CHECKPOINT_WORKER_TIMEOUT_MS, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_SEGMENT_MAX_BYTES, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, DIRECTORY_LOCK_ATTEMPTS, DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS, DIRECTORY_LOCK_SETTLE_MS, EARLIER_SEGMENT_MANIFEST_VERSION, LOG_RECORDS_PER_CHECKPOINT, MANIFEST_ENVELOPE_VERSION, MAX_SEGMENTS_PER_PARTITION, MAX_SEGMENT_ID, SEGMENT_HEADER_SIZE, SEGMENT_MANIFEST_VERSION, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, adapterCheckpointStore, buildEntry, captureWholePartition, checkpointLastSeqNo, checkpointPrimaryTerm, commitCheckpointManifest, computeEntryChecksum, createAckMessage, createEntryBatchMessage, createEntryMessage, createForwardBatchMessage, createForwardBatchResultMessage, createForwardMessage, createInsyncAddMessage, createInsyncConfirmMessage, createInsyncRemoveMessage, createPrimaryResignMessage, directoryCheckpointStore, durableRecordMissing, encodeMarkerSlot, encodeSegmentManifest, frameRecord, graphPartKey, manifestFileRefs, manifestKey, manifestReferencedKeys, markerSlotOffset, nextMarkerSlot, optionalIndexUuid, partitionContentsOf, readCommitMarker, readListedFile, readSegmentFile, readWalSegments, removeCheckpointGarbage, requireListedChecksum, segmentKey, segmentsPrefix, snapshotBundleKey, validateAckPayload, validateEntryBatchPayload, validateEntryPayload, validateForwardBatchPayload, validateForwardBatchResultPayload, validateInsyncConfirmPayload, vectorFileKey, writeCheckpointSegments, writeSegmentHeader };
1126
+ export { SEGMENT_HEADER_SIZE, buildEntry, captureWholePartition, checkpointLastSeqNo, checkpointPrimaryTerm, commitCheckpointManifest, computeEntryChecksum, createAckMessage, createEntryBatchMessage, createEntryMessage, createForwardBatchMessage, createForwardBatchResultMessage, createForwardMessage, createInsyncAddMessage, createInsyncConfirmMessage, createInsyncRemoveMessage, createPrimaryResignMessage, durableRecordMissing, encodeMarkerSlot, frameRecord, markerSlotOffset, nextMarkerSlot, nextSegmentKey, optionalIndexUuid, partitionContentsOf, readCommitMarker, readWalSegments, removeCheckpointGarbage, validateAckPayload, validateEntryBatchPayload, validateEntryPayload, validateForwardBatchPayload, validateForwardBatchResultPayload, validateInsyncConfirmPayload, writeCheckpointSegments, writeSegmentHeader };