@delali/narsil 0.3.1-next.140 → 0.3.1-next.141

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. package/dist/{analysis-change-JN23AZQP.mjs → analysis-change-HQUT2Z3I.mjs} +13 -13
  2. package/dist/{analysis-change-entry-5JSIMEKG.mjs → analysis-change-entry-TZTG4AZB.mjs} +1 -1
  3. package/dist/browser/{analysis-change-57MIRJWR.mjs → analysis-change-JPV775LR.mjs} +407 -6
  4. package/dist/browser/{analysis-change-entry-ICGRR3IN.mjs → analysis-change-entry-7HR4LWL3.mjs} +1 -1
  5. package/dist/browser/chunk-3ZBJYAS4.mjs +19 -0
  6. package/dist/browser/{chunk-ZAIZQOTM.mjs → chunk-5DB2VDGI.mjs} +951 -1134
  7. package/dist/browser/chunk-6RHTAVZX.mjs +656 -0
  8. package/dist/browser/chunk-CCOAOO73.mjs +86 -0
  9. package/dist/browser/{chunk-3EYDURFD.mjs → chunk-HKX2G6WH.mjs} +1615 -2266
  10. package/dist/browser/{chunk-CGVBGHBX.mjs → chunk-KTOK275N.mjs} +531 -830
  11. package/dist/browser/{chunk-5LYWZJE2.mjs → chunk-LVY2LI35.mjs} +1 -1
  12. package/dist/browser/chunk-MVM33ZUL.mjs +993 -0
  13. package/dist/browser/chunk-MWUK3DA5.mjs +2634 -0
  14. package/dist/browser/{chunk-67KSBUYR.mjs → chunk-R4WUDZQV.mjs} +8 -5
  15. package/dist/browser/{chunk-6BSB4N6B.mjs → chunk-TCLEOZ4P.mjs} +2 -2
  16. package/dist/browser/chunk-VFYMPCOV.mjs +212 -0
  17. package/dist/browser/chunk-WCFCHIBB.mjs +177 -0
  18. package/dist/browser/{chunk-E5SFTOQR.mjs → chunk-XNWPP5NS.mjs} +2 -2
  19. package/dist/browser/{chunk-QHNNULPW.mjs → chunk-ZEV26AQ6.mjs} +5 -0
  20. package/dist/browser/cold-module-MJMKNUYJ.mjs +745 -0
  21. package/dist/browser/{durability-wiring-MTIAN6VT.mjs → durability-wiring-SNOWIT6Z.mjs} +231 -51
  22. package/dist/browser/{import-Q7MXD6BS.mjs → import-TLB2A5LJ.mjs} +6 -5
  23. package/dist/browser/module-BUFNZMEU.mjs +12 -0
  24. package/dist/browser/{module-5J3PEUEB.mjs → module-BVP4TKZU.mjs} +395 -108
  25. package/dist/{restore-queue-RYSRRFS4.mjs → browser/restore-queue-ONYNYUDE.mjs} +2 -2
  26. package/dist/browser/{snapshot-YBJ5MF2M.mjs → snapshot-QXBKHJAG.mjs} +108 -25
  27. package/dist/browser/{write-RTWI5MXL.mjs → write-WHFBDWQS.mjs} +63 -91
  28. package/dist/{capabilities-DXAmysqJ.d.ts → capabilities-BCdUwK_L.d.ts} +2 -2
  29. package/dist/{chunk-HMDT3I63.mjs → chunk-46NZOPHS.mjs} +2 -2
  30. package/dist/{chunk-XXKWEG53.mjs → chunk-5WYVUPGD.mjs} +1 -1
  31. package/dist/chunk-6SNBQESJ.mjs +214 -0
  32. package/dist/chunk-7AUASRE7.mjs +653 -0
  33. package/dist/{chunk-4GCN34JV.mjs → chunk-AST3SHFA.mjs} +26 -24
  34. package/dist/chunk-BOJZGUS6.mjs +177 -0
  35. package/dist/{chunk-IP4HJOGC.mjs → chunk-BRUUBXCB.mjs} +4 -3
  36. package/dist/{chunk-J3EHATEZ.mjs → chunk-DQNKVOJ2.mjs} +63 -63
  37. package/dist/chunk-FOCV4CBZ.mjs +87 -0
  38. package/dist/{chunk-FFVLPCLH.mjs → chunk-GLZHNSGB.mjs} +1 -1
  39. package/dist/chunk-GWOS5DYC.mjs +2637 -0
  40. package/dist/{chunk-W2XDK66Z.mjs → chunk-IO5F7ZBE.mjs} +112 -438
  41. package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
  42. package/dist/{chunk-LLDFIKGP.mjs → chunk-KEIYVZUX.mjs} +4 -4
  43. package/dist/{chunk-UBXJQJII.mjs → chunk-KGOXHLLZ.mjs} +5 -1
  44. package/dist/{chunk-FKARMUWH.mjs → chunk-KZL72VTC.mjs} +68 -16
  45. package/dist/{chunk-TCWQJL7O.mjs → chunk-L76W3IZJ.mjs} +5 -0
  46. package/dist/{chunk-CPCGEIOM.mjs → chunk-LA4WNDHL.mjs} +83 -799
  47. package/dist/{chunk-PNNVTKMT.mjs → chunk-NJJCPFXN.mjs} +2 -2
  48. package/dist/chunk-NN33CZRE.mjs +86 -0
  49. package/dist/chunk-NRUXBYPH.mjs +121 -0
  50. package/dist/{chunk-UJWGQIXU.mjs → chunk-QHPYS427.mjs} +1 -0
  51. package/dist/{chunk-VSUYTPME.mjs → chunk-RHKRTYGO.mjs} +13 -4
  52. package/dist/{chunk-UEV53BBO.mjs → chunk-SRJBFG2T.mjs} +406 -5
  53. package/dist/{chunk-K3AQOZVM.mjs → chunk-TNBIK2LQ.mjs} +1638 -1644
  54. package/dist/chunk-UUZCQQG2.mjs +209 -0
  55. package/dist/{chunk-XMQNOD5V.mjs → chunk-VEFN6X6Q.mjs} +2 -2
  56. package/dist/{chunk-LK3MOTXF.mjs → chunk-VJEGOELZ.mjs} +8 -5
  57. package/dist/chunk-W2COJ5AH.mjs +148 -0
  58. package/dist/{chunk-IBATGS3J.mjs → chunk-WJGO5PGV.mjs} +241 -124
  59. package/dist/{chunk-CCCGEPJM.mjs → chunk-XZ3SRNDJ.mjs} +24 -63
  60. package/dist/{chunk-MTQTX4BP.mjs → chunk-YHUTF6EA.mjs} +400 -92
  61. package/dist/{chunk-B5I3IL4H.mjs → chunk-YLNSJQLX.mjs} +4 -4
  62. package/dist/{chunk-6VHWXAOI.mjs → chunk-Z4HFMYNX.mjs} +29 -24
  63. package/dist/{chunk-U4X4AWKL.mjs → chunk-ZPWS2SKJ.mjs} +293 -270
  64. package/dist/{client-DApBeLbk.d.ts → client-B12YFALP.d.ts} +3 -3
  65. package/dist/client.d.ts +7 -7
  66. package/dist/cold-module-4X7WJCFM.mjs +783 -0
  67. package/dist/{config-B0H-vp4U.d.ts → config-BhZxIeb1.d.ts} +6 -1
  68. package/dist/{config-Brysl9NO.d.ts → config-CIG5yDyg.d.ts} +2 -2
  69. package/dist/{direct-executor-J5UDLPL3.mjs → direct-executor-7XOQ7VQP.mjs} +5 -5
  70. package/dist/distribution/coordinator/etcd.d.ts +2 -2
  71. package/dist/distribution/coordinator/in-memory.d.ts +2 -2
  72. package/dist/distribution.d.ts +7 -7
  73. package/dist/distribution.mjs +123 -42
  74. package/dist/{durability-wiring-AYOTAPNQ.mjs → durability-wiring-324D2DDD.mjs} +213 -40
  75. package/dist/{engine-CuLMbH1-.d.ts → engine-CtzbHOmG.d.ts} +2 -2
  76. package/dist/{import-QRZAQ5RH.mjs → import-OOBUW3RR.mjs} +14 -13
  77. package/dist/{index-HYnqS1SE.d.ts → index-DsEurhSl.d.ts} +1 -1
  78. package/dist/index.browser.mjs +3 -3
  79. package/dist/index.d.ts +6 -6
  80. package/dist/index.mjs +11 -11
  81. package/dist/{long-query-thread-U7K5A72G.mjs → long-query-thread-4SW7RWDE.mjs} +8 -8
  82. package/dist/{module-L2OZO5PQ.mjs → module-DXSJ5LT2.mjs} +374 -83
  83. package/dist/module-IYPLWHGC.mjs +48 -0
  84. package/dist/paged-partition-archive-HFP45PLU.mjs +269 -0
  85. package/dist/partial-words.mjs +2 -2
  86. package/dist/persistence/durability/checkpoint-worker.d.ts +5 -1
  87. package/dist/persistence/durability/checkpoint-worker.mjs +17 -15
  88. package/dist/react.d.ts +9 -9
  89. package/dist/{browser/restore-queue-DISFECE4.mjs → restore-queue-FEYAC4SB.mjs} +2 -2
  90. package/dist/{schema-BTXbHCqA.d.ts → schema-B-Runv0n.d.ts} +7 -0
  91. package/dist/server.d.ts +10 -10
  92. package/dist/server.mjs +8 -7
  93. package/dist/{snapshot-BTRZ3MZK.mjs → snapshot-CGM65VCD.mjs} +60 -25
  94. package/dist/{tasks-DjGjK70Y.d.ts → tasks-CjRJRGTb.d.ts} +6 -4
  95. package/dist/{thread-USB3GOPY.mjs → thread-KGNAIKPD.mjs} +8 -8
  96. package/dist/vector/search-worker.d.ts +1 -1
  97. package/dist/vector/search-worker.mjs +3 -2
  98. package/dist/workers/entry.mjs +7 -7
  99. package/dist/{write-GHTA56OW.mjs → write-2DZPEQL2.mjs} +70 -98
  100. package/openapi.json +13 -0
  101. package/package.json +12 -8
  102. package/dist/browser/chunk-XVMOQQ34.mjs +0 -384
  103. package/dist/chunk-BDIDUW2B.mjs +0 -387
  104. package/dist/browser/{chunk-4MFH226Q.mjs → chunk-D33A5VMU.mjs} +15 -15
@@ -0,0 +1,269 @@
1
+ import { readVectorFile, VECTOR_FILE_ENVELOPE_VERSION } from './chunk-NRUXBYPH.mjs';
2
+ import { loadFailed, decodeSections, SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES } from './chunk-W2COJ5AH.mjs';
3
+ import { vectorsPerFile } from './chunk-Z4HFMYNX.mjs';
4
+ import { isDead, VECTOR_FILE_PAYLOAD_VERSION } from './chunk-ICW45TTU.mjs';
5
+ import './chunk-TNBIK2LQ.mjs';
6
+ import './chunk-37CD3OKS.mjs';
7
+ import './chunk-DJHHF2L5.mjs';
8
+ import { PARTITION_COPY_VECTOR_RUN_BYTES } from './chunk-JDDXV6UF.mjs';
9
+ import { segmentsPrefix, encodeSegmentManifest, directoryCheckpointStore, vectorFileKey } from './chunk-UUZCQQG2.mjs';
10
+ import { HEADER_SIZE, crc32Init, crc32Final, engineVersionParts, writeHeader, RESERVED_SIZE, crc32Update } from './chunk-IO5F7ZBE.mjs';
11
+ import './chunk-VEFN6X6Q.mjs';
12
+ import './chunk-5WYVUPGD.mjs';
13
+ import './chunk-HJSN7UYG.mjs';
14
+ import './chunk-UIKN2OTQ.mjs';
15
+ import './chunk-AV4IJKD2.mjs';
16
+ import './chunk-EEBCDP36.mjs';
17
+ import './chunk-WJGO5PGV.mjs';
18
+ import { isCompositePartition } from './chunk-ZPWS2SKJ.mjs';
19
+ import { generateId } from './chunk-N37VMC62.mjs';
20
+ import './chunk-ZATRDAYU.mjs';
21
+ import './chunk-EBIHRL7B.mjs';
22
+ import './chunk-KOKM4PQN.mjs';
23
+ import './chunk-QSK35UC2.mjs';
24
+ import './chunk-74XB2SXI.mjs';
25
+ import './chunk-RKAVHNRW.mjs';
26
+ import './chunk-WQBCBKY3.mjs';
27
+ import './chunk-JUKYYXSP.mjs';
28
+ import { parseVectorDimension } from './chunk-DQNKVOJ2.mjs';
29
+ import './chunk-KGOXHLLZ.mjs';
30
+ import './chunk-U476IXKQ.mjs';
31
+ import './chunk-FQBFZAWG.mjs';
32
+ import './chunk-VJS4JFBQ.mjs';
33
+ import './chunk-GES7U2PM.mjs';
34
+ import './chunk-ZSWHPKGM.mjs';
35
+ import './chunk-O4LJQI4G.mjs';
36
+ import './chunk-X5CV2E2B.mjs';
37
+ import './chunk-WNBRW4Q2.mjs';
38
+ import './chunk-TGP5M5R7.mjs';
39
+ import './chunk-U2BKYEC7.mjs';
40
+ import './chunk-NPNK57CA.mjs';
41
+ import './chunk-4UMZ7KQL.mjs';
42
+ import './chunk-R6A773WK.mjs';
43
+ import './chunk-YTJCGKM4.mjs';
44
+ import './chunk-QHPYS427.mjs';
45
+ import './chunk-GLZHNSGB.mjs';
46
+ import './chunk-22YPZVMG.mjs';
47
+ import './chunk-RFBLQLLZ.mjs';
48
+ import './chunk-L5ORED7V.mjs';
49
+ import './chunk-HNHCNTVG.mjs';
50
+ import { decode, encode } from '@msgpack/msgpack';
51
+
52
+ var FLOAT32_BYTES = 4;
53
+ var BIN8 = 196;
54
+ var BIN16 = 197;
55
+ var BIN32 = 198;
56
+ var UINT8_MAX = 255;
57
+ var UINT16_MAX = 65535;
58
+ var FIXMAP_OF_SIX = 134;
59
+ var VECTOR_FILE_PAGE_SIZES = /* @__PURE__ */ new Set([SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES]);
60
+ function binHeader(length) {
61
+ if (length <= UINT8_MAX) return Uint8Array.of(BIN8, length);
62
+ if (length <= UINT16_MAX) return Uint8Array.of(BIN16, length >> 8, length & 255);
63
+ const header = new Uint8Array(5);
64
+ header[0] = BIN32;
65
+ new DataView(header.buffer).setUint32(1, length, false);
66
+ return header;
67
+ }
68
+ async function docIdsOf(checkpoint, directory, ref, fieldPath) {
69
+ const { headOffset, headLength } = ref;
70
+ if (headOffset === void 0 || headLength === void 0) {
71
+ const { payload } = await readVectorFile(directoryCheckpointStore(directory), ref, fieldPath);
72
+ return payload.docIds;
73
+ }
74
+ const head = await checkpoint.readRange(ref, HEADER_SIZE + headOffset, headLength);
75
+ const raw = decode(head);
76
+ const sections = decodeSections(raw.sections, ref.bytes - HEADER_SIZE, `head of "${ref.key}"`, VECTOR_FILE_PAGE_SIZES);
77
+ const section = sections.find((candidate) => candidate.name === "docIds");
78
+ if (section === void 0) loadFailed(`The head of "${ref.key}" lists no section "docIds"`);
79
+ const docIds = decode(await checkpoint.readRange(ref, HEADER_SIZE + section.offset, section.length));
80
+ if (!Array.isArray(docIds) || docIds.length !== ref.count) {
81
+ loadFailed(`The vector file "${ref.key}" lists the wrong number of IDs`);
82
+ }
83
+ return docIds;
84
+ }
85
+ async function writePartitionVectorFile(checkpoint, directory, target, slots) {
86
+ const { dimension, fieldPath, temporaryKey } = target;
87
+ const width = dimension * FLOAT32_BYTES;
88
+ const handle = await directory.appendHandle(temporaryKey);
89
+ let state = crc32Init();
90
+ let payloadLength = 0;
91
+ async function appendPayload(bytes) {
92
+ state = crc32Update(state, bytes);
93
+ payloadLength += bytes.byteLength;
94
+ await handle.append(bytes);
95
+ }
96
+ try {
97
+ await handle.append(new Uint8Array(HEADER_SIZE));
98
+ await appendPayload(Uint8Array.of(FIXMAP_OF_SIX));
99
+ for (const [key, value] of [
100
+ ["v", VECTOR_FILE_PAYLOAD_VERSION],
101
+ ["fieldName", fieldPath],
102
+ ["dimension", dimension],
103
+ ["docIds", slots.map((slot) => slot.docId)],
104
+ ["codes", null]
105
+ ]) {
106
+ await appendPayload(encode(key));
107
+ await appendPayload(encode(value));
108
+ }
109
+ await appendPayload(encode("vectors"));
110
+ await appendPayload(binHeader(slots.length * width));
111
+ let at = 0;
112
+ while (at < slots.length) {
113
+ const first = slots[at];
114
+ let end = at + 1;
115
+ const limit = Math.max(1, Math.floor(PARTITION_COPY_VECTOR_RUN_BYTES / width));
116
+ while (end < slots.length && end - at < limit && slots[end].ref === first.ref && slots[end].position === first.position + (end - at)) {
117
+ end += 1;
118
+ }
119
+ const offset = first.ref.bytes - (first.ref.count - first.position) * width;
120
+ await appendPayload(await checkpoint.readRange(first.ref, offset, (end - at) * width));
121
+ at = end;
122
+ }
123
+ await handle.sync();
124
+ } finally {
125
+ await handle.close();
126
+ }
127
+ const checksum = crc32Final(state);
128
+ const [major, minor, patch] = engineVersionParts();
129
+ const header = writeHeader({
130
+ magic: "NRSL",
131
+ envelopeFormatVersion: VECTOR_FILE_ENVELOPE_VERSION,
132
+ engineVersionMajor: major,
133
+ engineVersionMinor: minor,
134
+ engineVersionPatch: patch,
135
+ payloadLength,
136
+ flags: {
137
+ compressionEnabled: false,
138
+ compressionAlgorithm: "none",
139
+ checksumPresent: true,
140
+ encryptionEnabled: false
141
+ },
142
+ checksum,
143
+ reserved: new Uint8Array(RESERVED_SIZE)
144
+ });
145
+ const patched = await directory.markerHandle(temporaryKey);
146
+ try {
147
+ await patched.writeSlot(0, header);
148
+ } finally {
149
+ await patched.close();
150
+ }
151
+ return {
152
+ temporaryKey,
153
+ ref: {
154
+ id: target.id,
155
+ key: target.key,
156
+ count: slots.length,
157
+ dead: null,
158
+ bytes: HEADER_SIZE + payloadLength,
159
+ checksum
160
+ }
161
+ };
162
+ }
163
+ async function partitionField(checkpoint, directory, scope, field, written) {
164
+ const fieldType = scope.manifest.schema[field.fieldPath];
165
+ const dimension = fieldType === void 0 ? null : parseVectorDimension(fieldType);
166
+ if (dimension === null) loadFailed(`The manifest gives vector field "${field.fieldPath}" no dimension`);
167
+ const capacity = vectorsPerFile(dimension, null);
168
+ const files = [];
169
+ let pending = [];
170
+ async function flush() {
171
+ if (pending.length === 0) return;
172
+ const id = files.length;
173
+ const key = vectorFileKey(scope.indexName, field.fieldPath, id);
174
+ const temporaryKey = `${scope.temporaryPrefix}${written.size}`;
175
+ const file = await writePartitionVectorFile(
176
+ checkpoint,
177
+ directory,
178
+ { key, id, temporaryKey, fieldPath: field.fieldPath, dimension },
179
+ pending
180
+ );
181
+ files.push(file.ref);
182
+ written.set(key, temporaryKey);
183
+ pending = [];
184
+ }
185
+ for (const ref of field.files) {
186
+ const docIds = await docIdsOf(checkpoint, directory, ref, field.fieldPath);
187
+ for (let position = 0; position < docIds.length; position++) {
188
+ if (isDead(ref.dead, position) || !scope.holds(docIds[position])) continue;
189
+ pending.push({ docId: docIds[position], ref, position });
190
+ if (pending.length === capacity) await flush();
191
+ }
192
+ }
193
+ await flush();
194
+ return { fieldPath: field.fieldPath, nextFileId: files.length, files, graphGeneration: 0, graphParts: [] };
195
+ }
196
+ async function openPagedPartitionArchive(checkpoint, directory, indexName, partitionId, manager) {
197
+ const { manifest } = checkpoint;
198
+ const temporaryPrefix = `${segmentsPrefix(indexName)}partition-copy-${partitionId}-${generateId()}-`;
199
+ const written = /* @__PURE__ */ new Map();
200
+ const partition = manager.getPartition(partitionId);
201
+ async function removeWritten() {
202
+ for (const temporaryKey of written.values()) await directory.remove(temporaryKey);
203
+ }
204
+ try {
205
+ const vectors = [];
206
+ for (const field of manifest.vectors) {
207
+ vectors.push(
208
+ await partitionField(
209
+ checkpoint,
210
+ directory,
211
+ { indexName, manifest, temporaryPrefix, holds: (docId) => partition.has(docId) },
212
+ field,
213
+ written
214
+ )
215
+ );
216
+ }
217
+ const narrowed = {
218
+ ...manifest,
219
+ partitions: manifest.partitions.filter((entry) => entry.partitionId === partitionId),
220
+ checkpoint: manifest.checkpoint.filter((entry) => entry.partitionId === partitionId),
221
+ vectors
222
+ };
223
+ const encoded = await encodeSegmentManifest(narrowed);
224
+ let manifestLength = 0;
225
+ for (const chunk of encoded.chunks) manifestLength += chunk.byteLength;
226
+ const manifestBytes = new Uint8Array(manifestLength);
227
+ let cursor = 0;
228
+ for (const chunk of encoded.chunks) {
229
+ manifestBytes.set(chunk, cursor);
230
+ cursor += chunk.byteLength;
231
+ }
232
+ const { readRange } = directory;
233
+ return {
234
+ manifestBytes,
235
+ manifest: narrowed,
236
+ lastSeqNo: narrowed.checkpoint[0]?.lastSeqNo ?? 0,
237
+ directory,
238
+ readFile: (ref) => checkpoint.readFile(ref),
239
+ readRange: (ref, offset, length) => {
240
+ const temporaryKey = written.get(ref.key);
241
+ if (temporaryKey === void 0 || readRange === void 0) return checkpoint.readRange(ref, offset, length);
242
+ return readRange.call(directory, temporaryKey, offset, length);
243
+ },
244
+ release: () => {
245
+ void removeWritten().catch(() => void 0).finally(() => checkpoint.release());
246
+ }
247
+ };
248
+ } catch (error) {
249
+ await removeWritten().catch(() => void 0);
250
+ throw error;
251
+ }
252
+ }
253
+ async function removeReceivedFilesNothingReads(directory, manager, partitionId, currentPrefix) {
254
+ const partition = manager.getPartition(partitionId);
255
+ const read = /* @__PURE__ */ new Set();
256
+ if (isCompositePartition(partition)) {
257
+ for (const segment of partition.frozenSegments()) {
258
+ if (segment.paged !== null) read.add(segment.paged.key);
259
+ }
260
+ }
261
+ const everyReceipt = `${manager.indexName}/received/p${partitionId}/`;
262
+ for (const key of await directory.list(everyReceipt)) {
263
+ if (read.has(key)) continue;
264
+ if (key.startsWith(currentPrefix) && [...read].some((held) => held.startsWith(currentPrefix))) continue;
265
+ await directory.remove(key);
266
+ }
267
+ }
268
+
269
+ export { openPagedPartitionArchive, removeReceivedFilesNothingReads };
@@ -1,5 +1,5 @@
1
- import { partialWordSearchModule } from './chunk-J3EHATEZ.mjs';
2
- import './chunk-UBXJQJII.mjs';
1
+ import { partialWordSearchModule } from './chunk-DQNKVOJ2.mjs';
2
+ import './chunk-KGOXHLLZ.mjs';
3
3
  import './chunk-U476IXKQ.mjs';
4
4
  import './chunk-FQBFZAWG.mjs';
5
5
  import './chunk-VJS4JFBQ.mjs';
@@ -1,4 +1,4 @@
1
- import { c as BM25FieldParams, P as PartitionConfig, e as ScoringMode, V as VectorIndexConfig } from '../../schema-BTXbHCqA.js';
1
+ import { c as BM25FieldParams, P as PartitionConfig, e as ScoringMode, V as VectorIndexConfig } from '../../schema-B-Runv0n.js';
2
2
  import '../../adapters-BQFQ4PRz.js';
3
3
  import '../../synonyms-CpOAHPa-.js';
4
4
 
@@ -153,6 +153,9 @@ interface FileRef {
153
153
  key: string;
154
154
  bytes: number;
155
155
  checksum: number;
156
+ headOffset?: number;
157
+ headLength?: number;
158
+ headChecksum?: number;
156
159
  }
157
160
  interface SegmentRef extends FileRef {
158
161
  id: number;
@@ -210,6 +213,7 @@ interface IndexMetadata {
210
213
  partitionLimits?: PartitionConfig;
211
214
  defaultScoring?: ScoringMode;
212
215
  trackPositions?: boolean;
216
+ textStorage?: 'memory' | 'disk';
213
217
  strict?: boolean;
214
218
  required?: string[];
215
219
  vectorPromotion?: VectorIndexConfig;
@@ -1,30 +1,32 @@
1
- import { createDurableDirectory } from '../../chunk-4GCN34JV.mjs';
1
+ import { createDurableDirectory } from '../../chunk-AST3SHFA.mjs';
2
2
  import '../../chunk-W55X2BYN.mjs';
3
3
  import '../../chunk-HLPRHOSO.mjs';
4
- import { CHECKPOINT_WORKER_HEARTBEAT_MS, writeCheckpointSegments, directoryCheckpointStore } from '../../chunk-CPCGEIOM.mjs';
5
- import '../../chunk-LLDFIKGP.mjs';
6
- import '../../chunk-P36MUFEJ.mjs';
4
+ import { writeCheckpointSegments } from '../../chunk-LA4WNDHL.mjs';
5
+ import '../../chunk-JDDXV6UF.mjs';
6
+ import { CHECKPOINT_WORKER_HEARTBEAT_MS } from '../../chunk-7AUASRE7.mjs';
7
+ import { directoryCheckpointStore } from '../../chunk-UUZCQQG2.mjs';
8
+ import '../../chunk-KEIYVZUX.mjs';
7
9
  import '../../chunk-YH5OIL7B.mjs';
8
- import '../../chunk-W2XDK66Z.mjs';
9
- import '../../chunk-XMQNOD5V.mjs';
10
- import '../../chunk-XXKWEG53.mjs';
10
+ import '../../chunk-IO5F7ZBE.mjs';
11
+ import '../../chunk-VEFN6X6Q.mjs';
12
+ import '../../chunk-5WYVUPGD.mjs';
13
+ import '../../chunk-HJSN7UYG.mjs';
11
14
  import '../../chunk-UIKN2OTQ.mjs';
12
15
  import '../../chunk-AV4IJKD2.mjs';
13
- import '../../chunk-IBATGS3J.mjs';
14
- import '../../chunk-U4X4AWKL.mjs';
16
+ import '../../chunk-EEBCDP36.mjs';
17
+ import '../../chunk-WJGO5PGV.mjs';
18
+ import '../../chunk-ZPWS2SKJ.mjs';
15
19
  import '../../chunk-N37VMC62.mjs';
16
20
  import '../../chunk-ZATRDAYU.mjs';
17
21
  import '../../chunk-EBIHRL7B.mjs';
18
- import '../../chunk-HJSN7UYG.mjs';
19
- import '../../chunk-EEBCDP36.mjs';
20
22
  import '../../chunk-KOKM4PQN.mjs';
21
23
  import '../../chunk-QSK35UC2.mjs';
22
24
  import '../../chunk-74XB2SXI.mjs';
23
25
  import '../../chunk-RKAVHNRW.mjs';
24
26
  import '../../chunk-WQBCBKY3.mjs';
25
27
  import '../../chunk-JUKYYXSP.mjs';
26
- import '../../chunk-J3EHATEZ.mjs';
27
- import '../../chunk-UBXJQJII.mjs';
28
+ import '../../chunk-DQNKVOJ2.mjs';
29
+ import '../../chunk-KGOXHLLZ.mjs';
28
30
  import '../../chunk-U476IXKQ.mjs';
29
31
  import '../../chunk-FQBFZAWG.mjs';
30
32
  import '../../chunk-VJS4JFBQ.mjs';
@@ -39,8 +41,8 @@ import '../../chunk-NPNK57CA.mjs';
39
41
  import '../../chunk-4UMZ7KQL.mjs';
40
42
  import '../../chunk-R6A773WK.mjs';
41
43
  import '../../chunk-YTJCGKM4.mjs';
42
- import '../../chunk-UJWGQIXU.mjs';
43
- import '../../chunk-FFVLPCLH.mjs';
44
+ import '../../chunk-QHPYS427.mjs';
45
+ import '../../chunk-GLZHNSGB.mjs';
44
46
  import '../../chunk-22YPZVMG.mjs';
45
47
  import '../../chunk-RFBLQLLZ.mjs';
46
48
  import '../../chunk-L5ORED7V.mjs';
package/dist/react.d.ts CHANGED
@@ -1,15 +1,15 @@
1
- import { N as NarsilError, L as ListParams, f as ListResult, I as IndexInfo, k as IndexStats, Q as QueryParams, j as PreflightResult, m as QueryResult, g as SuggestParams, h as SuggestResult } from './config-B0H-vp4U.js';
2
- export { C as ClientErrorCode, a as ClientErrorCodes, E as ErrorCode, b as ErrorCodes, c as NarsilErrorCode, S as ServerErrorCode, d as ServerErrorCodes, i as isNarsilError } from './config-B0H-vp4U.js';
3
- import { f as TaskRecord, e as TaskProgress, c as ImportResult, d as TaskListQuery, T as TaskListPage } from './tasks-DjGjK70Y.js';
4
- export { E as ErrorDetail, I as ImportError, g as TaskStatus, h as TaskType } from './tasks-DjGjK70Y.js';
1
+ import { N as NarsilError, L as ListParams, f as ListResult, I as IndexInfo, k as IndexStats, Q as QueryParams, j as PreflightResult, m as QueryResult, g as SuggestParams, h as SuggestResult } from './config-BhZxIeb1.js';
2
+ export { C as ClientErrorCode, a as ClientErrorCodes, E as ErrorCode, b as ErrorCodes, c as NarsilErrorCode, S as ServerErrorCode, d as ServerErrorCodes, i as isNarsilError } from './config-BhZxIeb1.js';
3
+ import { f as TaskRecord, e as TaskProgress, c as ImportResult, d as TaskListQuery, T as TaskListPage } from './tasks-CjRJRGTb.js';
4
+ export { E as ErrorDetail, I as ImportError, g as TaskStatus, h as TaskType } from './tasks-CjRJRGTb.js';
5
5
  import { ReactNode, ReactElement } from 'react';
6
- import { N as Narsil } from './engine-CuLMbH1-.js';
7
- import { N as NarsilClient, I as ImportSource } from './client-DApBeLbk.js';
8
- import { A as AnyDocument } from './schema-BTXbHCqA.js';
6
+ import { N as Narsil } from './engine-CtzbHOmG.js';
7
+ import { N as NarsilClient, I as ImportSource } from './client-B12YFALP.js';
8
+ import { A as AnyDocument } from './schema-B-Runv0n.js';
9
9
  import './adapters-BQFQ4PRz.js';
10
10
  import './synonyms-CpOAHPa-.js';
11
- import './index-HYnqS1SE.js';
12
- import './config-Brysl9NO.js';
11
+ import './index-DsEurhSl.js';
12
+ import './config-CIG5yDyg.js';
13
13
  import './types-DvncmvuQ.js';
14
14
 
15
15
  /**
@@ -1,4 +1,4 @@
1
- import { abandonChangeOf } from './chunk-E5SFTOQR.mjs';
1
+ import { abandonChangeOf } from './chunk-46NZOPHS.mjs';
2
2
 
3
3
  // src/narsil/restore-queue.ts
4
4
  function restoreDepsOf(core, host) {
@@ -24,7 +24,7 @@ function restoreDepsOf(core, host) {
24
24
  function queueRestore(core, host, restoresInFlight, indexName, data) {
25
25
  const previous = restoresInFlight.get(indexName) ?? Promise.resolve();
26
26
  const restored = previous.then(async () => {
27
- const { restoreFromSnapshot } = await import('./snapshot-YBJ5MF2M.mjs');
27
+ const { restoreFromSnapshot } = await import('./snapshot-CGM65VCD.mjs');
28
28
  await restoreFromSnapshot(indexName, data, restoreDepsOf(core, host));
29
29
  });
30
30
  const settled = restored.then(
@@ -254,6 +254,13 @@ interface IndexConfig {
254
254
  * because the engine's highlighter analyses the stored text again.
255
255
  */
256
256
  trackPositions?: boolean;
257
+ /**
258
+ * Setting this to `'disk'` lets the index answer from segment files that the
259
+ * engine reads by position, so its keyword index and documents can outgrow
260
+ * the node's memory once a checkpoint covers them. The default is `'memory'`,
261
+ * and `'disk'` raises `CONFIG_INVALID` without filesystem durability.
262
+ */
263
+ textStorage?: 'memory' | 'disk';
257
264
  /** These settings control when vector fields move to an HNSW graph, and how that graph is built. */
258
265
  vectorPromotion?: VectorIndexConfig;
259
266
  /**
package/dist/server.d.ts CHANGED
@@ -1,15 +1,15 @@
1
- import { G as GroupResult, Q as QueryParams, n as GroupConfig, m as QueryResult, L as ListParams, g as SuggestParams, k as IndexStats, f as ListResult, j as PreflightResult, h as SuggestResult } from './config-B0H-vp4U.js';
2
- export { C as ClientErrorCode, a as ClientErrorCodes, E as ErrorCode, b as ErrorCodes, N as NarsilError, c as NarsilErrorCode, S as ServerErrorCode, d as ServerErrorCodes, i as isNarsilError } from './config-B0H-vp4U.js';
3
- import { E as ErrorEnvelope, c as ServerOptions, N as NarsilServer } from './capabilities-DXAmysqJ.js';
4
- export { A as ASYNC_IMPORT_CAPABILITY, B as BM25_TUNING_CAPABILITY, C as CHANGE_ANALYSIS_CAPABILITY, d as CorsOptions, D as DOCUMENT_EXPORT_CAPABILITY, I as INDEX_LIFECYCLE_CAPABILITY, O as OnRequestHook, R as REBUILD_ANALYSIS_CAPABILITY, e as RequestContext, f as RequestDenial, S as SERVER_CAPABILITIES, a as SYNONYMS_CAPABILITY, g as ServerLimits, T as TASK_CANCEL_CAPABILITY, b as TASK_FILTER_CAPABILITY } from './capabilities-DXAmysqJ.js';
5
- import { N as Narsil } from './engine-CuLMbH1-.js';
6
- import { j as CreateIndexRequest, H as HttpIndexConfig, B as BatchBody, a as ChangeAnalysisBody, D as DocumentBody, k as InsertBody, l as MultiGetBody, R as RebalanceBody, m as ReplicationFactorBody, d as TaskListQuery, V as VectorFieldBody, S as SynonymSetBody, n as BatchResponse, C as CapabilitiesResponse, o as ClusterAllocationResponse, p as ClusterTopologyResponse, q as DocumentCountResponse, r as DocumentExistsResponse, s as DocumentIdResponse, t as DocumentPutResponse, u as DocumentRemovedResponse, v as DocumentResponse, c as ImportResult, w as IndexClearedResponse, x as IndexCreatedResponse, y as IndexDroppedResponse, z as IndexListResponse, A as IndexPartitionsResponse, F as IndexStateResponse, L as LivenessResponse, M as MemoryStatsResponse, G as MultiGetResponse, O as OkResponse, J as ReadinessResponse, K as ReplicationFactorResponse, T as TaskListPage, f as TaskRecord, N as VectorMaintenanceResponse, P as VersionResponse, e as TaskProgress, i as TaskStore } from './tasks-DjGjK70Y.js';
7
- export { Q as BatchFailure, U as ClusterMemberReport, W as ClusterNodeReport, b as CreateIndexEmbedding, E as ErrorDetail, I as ImportError, X as PartitionAllocationReport, g as TaskStatus, h as TaskType } from './tasks-DjGjK70Y.js';
8
- import { e as ClusterHealth } from './config-Brysl9NO.js';
9
- import { S as SchemaDefinition, P as PartitionConfig, V as VectorIndexConfig, B as BM25Params, F as FieldType } from './schema-BTXbHCqA.js';
1
+ import { G as GroupResult, Q as QueryParams, n as GroupConfig, m as QueryResult, L as ListParams, g as SuggestParams, k as IndexStats, f as ListResult, j as PreflightResult, h as SuggestResult } from './config-BhZxIeb1.js';
2
+ export { C as ClientErrorCode, a as ClientErrorCodes, E as ErrorCode, b as ErrorCodes, N as NarsilError, c as NarsilErrorCode, S as ServerErrorCode, d as ServerErrorCodes, i as isNarsilError } from './config-BhZxIeb1.js';
3
+ import { E as ErrorEnvelope, c as ServerOptions, N as NarsilServer } from './capabilities-BCdUwK_L.js';
4
+ export { A as ASYNC_IMPORT_CAPABILITY, B as BM25_TUNING_CAPABILITY, C as CHANGE_ANALYSIS_CAPABILITY, d as CorsOptions, D as DOCUMENT_EXPORT_CAPABILITY, I as INDEX_LIFECYCLE_CAPABILITY, O as OnRequestHook, R as REBUILD_ANALYSIS_CAPABILITY, e as RequestContext, f as RequestDenial, S as SERVER_CAPABILITIES, a as SYNONYMS_CAPABILITY, g as ServerLimits, T as TASK_CANCEL_CAPABILITY, b as TASK_FILTER_CAPABILITY } from './capabilities-BCdUwK_L.js';
5
+ import { N as Narsil } from './engine-CtzbHOmG.js';
6
+ import { j as CreateIndexRequest, H as HttpIndexConfig, B as BatchBody, a as ChangeAnalysisBody, D as DocumentBody, k as InsertBody, l as MultiGetBody, R as RebalanceBody, m as ReplicationFactorBody, d as TaskListQuery, V as VectorFieldBody, S as SynonymSetBody, n as BatchResponse, C as CapabilitiesResponse, o as ClusterAllocationResponse, p as ClusterTopologyResponse, q as DocumentCountResponse, r as DocumentExistsResponse, s as DocumentIdResponse, t as DocumentPutResponse, u as DocumentRemovedResponse, v as DocumentResponse, c as ImportResult, w as IndexClearedResponse, x as IndexCreatedResponse, y as IndexDroppedResponse, z as IndexListResponse, A as IndexPartitionsResponse, F as IndexStateResponse, L as LivenessResponse, M as MemoryStatsResponse, G as MultiGetResponse, O as OkResponse, J as ReadinessResponse, K as ReplicationFactorResponse, T as TaskListPage, f as TaskRecord, N as VectorMaintenanceResponse, P as VersionResponse, e as TaskProgress, i as TaskStore } from './tasks-CjRJRGTb.js';
7
+ export { Q as BatchFailure, U as ClusterMemberReport, W as ClusterNodeReport, b as CreateIndexEmbedding, E as ErrorDetail, I as ImportError, X as PartitionAllocationReport, g as TaskStatus, h as TaskType } from './tasks-CjRJRGTb.js';
8
+ import { e as ClusterHealth } from './config-CIG5yDyg.js';
9
+ import { S as SchemaDefinition, P as PartitionConfig, V as VectorIndexConfig, B as BM25Params, F as FieldType } from './schema-B-Runv0n.js';
10
10
  import './adapters-BQFQ4PRz.js';
11
11
  import './synonyms-CpOAHPa-.js';
12
- import './index-HYnqS1SE.js';
12
+ import './index-DsEurhSl.js';
13
13
  import './types-DvncmvuQ.js';
14
14
 
15
15
  /**
package/dist/server.mjs CHANGED
@@ -2,22 +2,22 @@ import { answerRequest } from './chunk-ZMBFWQXN.mjs';
2
2
  import { DEFAULT_MAX_RETAINED_TASKS, loadUWebSockets, resolveCors, authorizerFor, createSharedGateBuffer, gateSlotOfWorker, createBodyBudgetBuffer, createRouteRunner, corsHeaderReader, createSharedBodyBudget, MAIN_THREAD_GATE_SLOT, createSharedRequestGate, registerServerRoutes, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RESTORE_BYTES, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_BODY_BYTES, DEFAULT_MAX_RERANK_WINDOW, DEFAULT_MAX_BODY_BYTES_IN_FLIGHT_FRACTION, RUNNING_TASK_TTL_MS, DEFAULT_TASK_PAGE_SIZE, TERMINAL_TASK_TTL_MS, TASK_PROGRESS_WRITE_INTERVAL_MS, serializeNarsilError, createVersionHandler, acceptorOf, MAX_BODY_BYTES_IN_FLIGHT_FRACTION, createDocumentReadHandlers, sendError, respondError, respondJson, sendText, PROMETHEUS_TEXT_CONTENT_TYPE, MAX_INDEX_TOUCH_INTERVAL_MS, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, closeEveryThreadAtExit, REQUEST_THREAD_CLOSE_AT_EXIT_MS, clearGateSlot, clearBodyBudgetSlot, createCloseAtExitWords, createDepartureWord, departureConfirmed, parseJson, validateCreateIndex, rejectInvalid, isRemoveOnlyBatch, validateBatch, serializeBatchResult, parseWriteJson, validateDocumentBody, validateSuggest, validateQuery, refuseRegex, withServerRerankWindow, respondAnswer, parseTaskListQuery, parseJsonOptional, validateChangeAnalysis, validateRebalance, badRequest, sendStream, SNAPSHOT_ARCHIVE_CONTENT_TYPE, DOCUMENT_EXPORT_CONTENT_TYPE, parseRefusalOf, IMPORT_FLUSHES_IN_FLIGHT, confirmDeparture, SERVER_CAPABILITIES, toHttpError, MIN_UNBOUNDED_SERVER_LIMIT, MIN_COUNTED_SERVER_LIMIT } from './chunk-SBMAWTRG.mjs';
3
3
  export { ASYNC_IMPORT_CAPABILITY, BM25_TUNING_CAPABILITY, CHANGE_ANALYSIS_CAPABILITY, DOCUMENT_EXPORT_CAPABILITY, INDEX_LIFECYCLE_CAPABILITY, REBUILD_ANALYSIS_CAPABILITY, SERVER_CAPABILITIES, SYNONYMS_CAPABILITY, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, httpStatusForNarsilError } from './chunk-SBMAWTRG.mjs';
4
4
  import { rerankOnHost } from './chunk-HJJWY6PT.mjs';
5
- import { writeBlockLookupOf, indexReadsOf, isRecord, validateIndexName } from './chunk-IP4HJOGC.mjs';
6
- import './chunk-TCWQJL7O.mjs';
5
+ import { writeBlockLookupOf, indexReadsOf, isRecord, validateIndexName } from './chunk-BRUUBXCB.mjs';
7
6
  import { readMemoryFigures } from './chunk-DD2673TX.mjs';
7
+ import './chunk-L76W3IZJ.mjs';
8
8
  import './chunk-IWVEG5CX.mjs';
9
9
  import './chunk-DJHHF2L5.mjs';
10
- import './chunk-P36MUFEJ.mjs';
11
- import { rerankWindowCapOf, engineCoreOf, providedDocId, resolveVectorText } from './chunk-XXKWEG53.mjs';
10
+ import './chunk-JDDXV6UF.mjs';
11
+ import { rerankWindowCapOf, engineCoreOf, providedDocId, resolveVectorText } from './chunk-5WYVUPGD.mjs';
12
+ import { MAX_RERANK_WINDOW } from './chunk-HJSN7UYG.mjs';
12
13
  import { detectRuntime } from './chunk-AV4IJKD2.mjs';
14
+ import './chunk-EEBCDP36.mjs';
13
15
  import './chunk-N37VMC62.mjs';
14
16
  import './chunk-ZATRDAYU.mjs';
15
17
  import './chunk-EBIHRL7B.mjs';
16
- import { MAX_RERANK_WINDOW } from './chunk-HJSN7UYG.mjs';
17
- import './chunk-EEBCDP36.mjs';
18
18
  import './chunk-74XB2SXI.mjs';
19
19
  import './chunk-WQBCBKY3.mjs';
20
- import './chunk-UBXJQJII.mjs';
20
+ import './chunk-KGOXHLLZ.mjs';
21
21
  import './chunk-U476IXKQ.mjs';
22
22
  import './chunk-FQBFZAWG.mjs';
23
23
  import './chunk-VJS4JFBQ.mjs';
@@ -787,6 +787,7 @@ function mapHttpIndexConfig(http) {
787
787
  if (http.defaultScoring !== void 0) config.defaultScoring = http.defaultScoring;
788
788
  if (http.bm25 !== void 0) config.bm25 = http.bm25;
789
789
  if (http.trackPositions !== void 0) config.trackPositions = http.trackPositions;
790
+ if (http.textStorage !== void 0) config.textStorage = http.textStorage;
790
791
  if (http.surfaceForms !== void 0) config.surfaceForms = http.surfaceForms === true;
791
792
  if (http.vectorPromotion !== void 0) config.vectorPromotion = http.vectorPromotion;
792
793
  if (http.strict !== void 0) config.strict = http.strict;
@@ -1,38 +1,42 @@
1
- import { documentsOfPayloadV2 } from './chunk-PNNVTKMT.mjs';
2
- import { buildArchiveFromMemory, archiveStream, memoryCheckpointStore, readSnapshotArchive } from './chunk-FKARMUWH.mjs';
3
- import { indexMetadataOf, unpackIndexSnapshotEnvelope, loadSegmentedCheckpoint, writeMetadataEnvelope, readVectorFile, readGraphParts, deserializePayloadV1, SINGLE_NODE_PRIMARY_TERM } from './chunk-MTQTX4BP.mjs';
1
+ import { documentsOfPayloadV2 } from './chunk-NJJCPFXN.mjs';
2
+ import { buildArchiveFromMemory, archiveStream, memoryCheckpointStore, readSnapshotArchive, writeArchiveFile } from './chunk-KZL72VTC.mjs';
3
+ import { indexMetadataOf, unpackIndexSnapshotEnvelope, loadSegmentedCheckpoint, writeMetadataEnvelope, deserializePayloadV1, SINGLE_NODE_PRIMARY_TERM } from './chunk-YHUTF6EA.mjs';
4
+ import { readVectorFile, readGraphParts } from './chunk-NRUXBYPH.mjs';
4
5
  import { createByteReader } from './chunk-CZQ33DLW.mjs';
5
- import { restoredConfigFields, restoredEmbedding, rejectSnapshot } from './chunk-TCWQJL7O.mjs';
6
- import './chunk-6VHWXAOI.mjs';
6
+ import { restoredConfigFields, restoredEmbedding, rejectSnapshot } from './chunk-L76W3IZJ.mjs';
7
+ import './chunk-W2COJ5AH.mjs';
8
+ import './chunk-Z4HFMYNX.mjs';
7
9
  import { decodeVectorIndexParts, bytesToVectors } from './chunk-ICW45TTU.mjs';
8
- import { MAX_SNAPSHOT_ANALYSIS_ATTEMPTS, requireRestoreMemory, earlierSnapshotFootprint, archiveFootprint } from './chunk-LK3MOTXF.mjs';
9
- import './chunk-K3AQOZVM.mjs';
10
+ import { MAX_SNAPSHOT_ANALYSIS_ATTEMPTS, requireRestoreMemory, earlierSnapshotFootprint, storedBytesOfSegments, archiveFootprint } from './chunk-VJEGOELZ.mjs';
11
+ import './chunk-TNBIK2LQ.mjs';
10
12
  import './chunk-37CD3OKS.mjs';
11
13
  import './chunk-DJHHF2L5.mjs';
12
- import { manifestKey, encodeSegmentManifest, manifestFileRefs, readSegmentFile } from './chunk-CPCGEIOM.mjs';
13
- import './chunk-LLDFIKGP.mjs';
14
- import './chunk-P36MUFEJ.mjs';
14
+ import './chunk-LA4WNDHL.mjs';
15
+ import './chunk-JDDXV6UF.mjs';
16
+ import { readSegmentFile } from './chunk-7AUASRE7.mjs';
17
+ import { manifestKey, encodeSegmentManifest, manifestFileRefs } from './chunk-UUZCQQG2.mjs';
18
+ import './chunk-KEIYVZUX.mjs';
15
19
  import './chunk-YH5OIL7B.mjs';
16
- import { encodeIndexConfiguration, hasNrslMagic, HEADER_SIZE, readHeader, validateRequiredFieldsInSchema, validateEmbeddingConfig, storedArrayPositionGap, decodeIndexConfiguration, prepareStoredSynonyms, arrayPositionsAreStale, insertDocumentVectors } from './chunk-W2XDK66Z.mjs';
17
- import { validatePatternValueLimit, requireSurfaceFormsForPartialFields } from './chunk-XMQNOD5V.mjs';
18
- import './chunk-XXKWEG53.mjs';
20
+ import { encodeIndexConfiguration, hasNrslMagic, HEADER_SIZE, readHeader, validateRequiredFieldsInSchema, validateEmbeddingConfig, storedArrayPositionGap, decodeIndexConfiguration, prepareStoredSynonyms, arrayPositionsAreStale, insertDocumentVectors } from './chunk-IO5F7ZBE.mjs';
21
+ import { validatePatternValueLimit, requireSurfaceFormsForPartialFields } from './chunk-VEFN6X6Q.mjs';
22
+ import './chunk-5WYVUPGD.mjs';
23
+ import './chunk-HJSN7UYG.mjs';
19
24
  import './chunk-UIKN2OTQ.mjs';
20
25
  import './chunk-AV4IJKD2.mjs';
21
- import { decodeEveryStoredDocument } from './chunk-IBATGS3J.mjs';
22
- import { ARRAY_POSITION_GAP } from './chunk-U4X4AWKL.mjs';
26
+ import './chunk-EEBCDP36.mjs';
27
+ import { decodeEveryStoredDocument } from './chunk-WJGO5PGV.mjs';
28
+ import { ARRAY_POSITION_GAP, isCompositePartition } from './chunk-ZPWS2SKJ.mjs';
23
29
  import './chunk-N37VMC62.mjs';
24
30
  import './chunk-ZATRDAYU.mjs';
25
31
  import './chunk-EBIHRL7B.mjs';
26
- import './chunk-HJSN7UYG.mjs';
27
- import './chunk-EEBCDP36.mjs';
28
32
  import './chunk-KOKM4PQN.mjs';
29
33
  import './chunk-QSK35UC2.mjs';
30
34
  import './chunk-74XB2SXI.mjs';
31
35
  import './chunk-RKAVHNRW.mjs';
32
36
  import './chunk-WQBCBKY3.mjs';
33
37
  import './chunk-JUKYYXSP.mjs';
34
- import { parseVectorDimension } from './chunk-J3EHATEZ.mjs';
35
- import { raiseEarlierReleaseGraphSetting, validateSchema, validateVectorPromotion, validateVectorStorage, flattenSchema } from './chunk-UBXJQJII.mjs';
38
+ import { parseVectorDimension } from './chunk-DQNKVOJ2.mjs';
39
+ import { raiseEarlierReleaseGraphSetting, validateSchema, validateVectorPromotion, validateVectorStorage, validateTextStorage, flattenSchema } from './chunk-KGOXHLLZ.mjs';
36
40
  import { MAX_FILE_BYTES } from './chunk-U476IXKQ.mjs';
37
41
  import './chunk-FQBFZAWG.mjs';
38
42
  import './chunk-VJS4JFBQ.mjs';
@@ -48,8 +52,8 @@ import './chunk-4UMZ7KQL.mjs';
48
52
  import './chunk-R6A773WK.mjs';
49
53
  import './chunk-YTJCGKM4.mjs';
50
54
  import './chunk-RIIUZJQ3.mjs';
51
- import './chunk-UJWGQIXU.mjs';
52
- import { loadVectorCodeFor } from './chunk-FFVLPCLH.mjs';
55
+ import './chunk-QHPYS427.mjs';
56
+ import { loadVectorCodeFor } from './chunk-GLZHNSGB.mjs';
53
57
  import './chunk-22YPZVMG.mjs';
54
58
  import './chunk-RFBLQLLZ.mjs';
55
59
  import './chunk-L5ORED7V.mjs';
@@ -79,6 +83,14 @@ function requireSchemaOfManifest(key, schema, manifest) {
79
83
  async function validateStagedCheckpoint(store, manifest) {
80
84
  for (const partition of manifest.partitions) {
81
85
  for (const ref of partition.segments) {
86
+ const { directory } = store;
87
+ if (directory !== void 0) {
88
+ const { checkPagedSegmentFully, isPagedSegmentFile } = await import('./module-IYPLWHGC.mjs');
89
+ if (await isPagedSegmentFile(directory, ref.key)) {
90
+ requireSchemaOfManifest(ref.key, await checkPagedSegmentFully(directory, ref), manifest);
91
+ continue;
92
+ }
93
+ }
82
94
  const contents = await readSegmentFile(store, ref);
83
95
  requireSchemaOfManifest(ref.key, contents.schema, manifest);
84
96
  decodeEveryStoredDocument(contents.arrays.documentTable);
@@ -134,6 +146,7 @@ function prepareRestoredIndex(configuration, deps, reject) {
134
146
  requireSurfaceFormsForPartialFields(schema, configFields.surfaceForms);
135
147
  validateVectorPromotion(configFields.vectorPromotion);
136
148
  validateVectorStorage(configFields.vectorPromotion, deps.filesystemDurability);
149
+ validateTextStorage(configFields.textStorage, deps.filesystemDurability);
137
150
  if (configFields.required !== void 0 && configFields.required.length > 0) {
138
151
  validateRequiredFieldsInSchema(configFields.required, schema);
139
152
  }
@@ -279,7 +292,14 @@ async function commitToStorage(indexName, restored, previousPartitions, deps) {
279
292
  }
280
293
  await staging.finish();
281
294
  const relocate = await staging.relocation().catch(() => null);
282
- if (relocate !== null) for (const vectorIndex of vectorIndexes) vectorIndex.relocateVectorFiles(relocate);
295
+ if (relocate === null) return;
296
+ for (const vectorIndex of vectorIndexes) vectorIndex.relocateVectorFiles(relocate);
297
+ const { manager } = restored.detached;
298
+ for (let partitionId = 0; partitionId < manager.partitionCount; partitionId++) {
299
+ const partition = manager.getPartition(partitionId);
300
+ if (!isCompositePartition(partition)) continue;
301
+ for (const segment of partition.frozenSegments()) segment.paged?.relocate(relocate);
302
+ }
283
303
  }
284
304
  function activate(indexName, restored, entry, deps) {
285
305
  deps.executor.installDetachedIndex(restored.detached);
@@ -402,6 +422,9 @@ function partitionCountOf(manifest) {
402
422
  function missingPart(what) {
403
423
  throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The archive holds no ${what}`);
404
424
  }
425
+ function keepsTextOnDisk(prepared, staging) {
426
+ return prepared.config.textStorage === "disk" && staging.directory !== void 0;
427
+ }
405
428
  async function stageArchive(indexName, reader, staging, deps) {
406
429
  const staged = { prepared: null, manifest: null };
407
430
  await readSnapshotArchive(reader, {
@@ -412,17 +435,29 @@ async function stageArchive(indexName, reader, staging, deps) {
412
435
  if (staged.prepared === null) missingPart("configuration");
413
436
  requireSameSchema(staged.prepared, manifest2);
414
437
  const replacesIndex = deps.indexRegistry.has(indexName);
415
- requireRestoreMemory(indexName, archiveFootprint(staged.prepared, manifest2, replacesIndex), replacesIndex);
438
+ const segmentBytes = keepsTextOnDisk(staged.prepared, staging) ? 0 : storedBytesOfSegments(manifest2);
439
+ requireRestoreMemory(
440
+ indexName,
441
+ archiveFootprint(staged.prepared, manifest2, replacesIndex, segmentBytes),
442
+ replacesIndex
443
+ );
416
444
  staged.manifest = renamed(manifest2, sourceName, indexName);
417
445
  },
418
- file: async (entryName, _ref, bytes) => {
446
+ file: async (entryName, _ref, chunks) => {
419
447
  if (staged.manifest === null) missingPart("manifest");
420
- await staging.write(`${indexName}/${entryName}`, [bytes]);
448
+ await writeArchiveFile(staging, `${indexName}/${entryName}`, chunks);
421
449
  }
422
450
  });
423
451
  const { prepared, manifest } = staged;
424
452
  if (prepared === null) missingPart("configuration");
425
453
  if (manifest === null) missingPart("manifest");
454
+ const { directory } = staging;
455
+ if (keepsTextOnDisk(prepared, staging) && directory !== void 0) {
456
+ const { bytesThatSegmentsHoldInMemory } = await import('./module-IYPLWHGC.mjs');
457
+ const replacesIndex = deps.indexRegistry.has(indexName);
458
+ const segmentBytes = await bytesThatSegmentsHoldInMemory(directory, manifest);
459
+ requireRestoreMemory(indexName, archiveFootprint(prepared, manifest, replacesIndex, segmentBytes), replacesIndex);
460
+ }
426
461
  await validateStagedCheckpoint(staging, manifest);
427
462
  return { prepared, manifest };
428
463
  }