@delali/narsil 0.3.1-next.140 → 0.3.1-next.141

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. package/dist/{analysis-change-JN23AZQP.mjs → analysis-change-HQUT2Z3I.mjs} +13 -13
  2. package/dist/{analysis-change-entry-5JSIMEKG.mjs → analysis-change-entry-TZTG4AZB.mjs} +1 -1
  3. package/dist/browser/{analysis-change-57MIRJWR.mjs → analysis-change-JPV775LR.mjs} +407 -6
  4. package/dist/browser/{analysis-change-entry-ICGRR3IN.mjs → analysis-change-entry-7HR4LWL3.mjs} +1 -1
  5. package/dist/browser/chunk-3ZBJYAS4.mjs +19 -0
  6. package/dist/browser/{chunk-ZAIZQOTM.mjs → chunk-5DB2VDGI.mjs} +951 -1134
  7. package/dist/browser/chunk-6RHTAVZX.mjs +656 -0
  8. package/dist/browser/chunk-CCOAOO73.mjs +86 -0
  9. package/dist/browser/{chunk-3EYDURFD.mjs → chunk-HKX2G6WH.mjs} +1615 -2266
  10. package/dist/browser/{chunk-CGVBGHBX.mjs → chunk-KTOK275N.mjs} +531 -830
  11. package/dist/browser/{chunk-5LYWZJE2.mjs → chunk-LVY2LI35.mjs} +1 -1
  12. package/dist/browser/chunk-MVM33ZUL.mjs +993 -0
  13. package/dist/browser/chunk-MWUK3DA5.mjs +2634 -0
  14. package/dist/browser/{chunk-67KSBUYR.mjs → chunk-R4WUDZQV.mjs} +8 -5
  15. package/dist/browser/{chunk-6BSB4N6B.mjs → chunk-TCLEOZ4P.mjs} +2 -2
  16. package/dist/browser/chunk-VFYMPCOV.mjs +212 -0
  17. package/dist/browser/chunk-WCFCHIBB.mjs +177 -0
  18. package/dist/browser/{chunk-E5SFTOQR.mjs → chunk-XNWPP5NS.mjs} +2 -2
  19. package/dist/browser/{chunk-QHNNULPW.mjs → chunk-ZEV26AQ6.mjs} +5 -0
  20. package/dist/browser/cold-module-MJMKNUYJ.mjs +745 -0
  21. package/dist/browser/{durability-wiring-MTIAN6VT.mjs → durability-wiring-SNOWIT6Z.mjs} +231 -51
  22. package/dist/browser/{import-Q7MXD6BS.mjs → import-TLB2A5LJ.mjs} +6 -5
  23. package/dist/browser/module-BUFNZMEU.mjs +12 -0
  24. package/dist/browser/{module-5J3PEUEB.mjs → module-BVP4TKZU.mjs} +395 -108
  25. package/dist/{restore-queue-RYSRRFS4.mjs → browser/restore-queue-ONYNYUDE.mjs} +2 -2
  26. package/dist/browser/{snapshot-YBJ5MF2M.mjs → snapshot-QXBKHJAG.mjs} +108 -25
  27. package/dist/browser/{write-RTWI5MXL.mjs → write-WHFBDWQS.mjs} +63 -91
  28. package/dist/{capabilities-DXAmysqJ.d.ts → capabilities-BCdUwK_L.d.ts} +2 -2
  29. package/dist/{chunk-HMDT3I63.mjs → chunk-46NZOPHS.mjs} +2 -2
  30. package/dist/{chunk-XXKWEG53.mjs → chunk-5WYVUPGD.mjs} +1 -1
  31. package/dist/chunk-6SNBQESJ.mjs +214 -0
  32. package/dist/chunk-7AUASRE7.mjs +653 -0
  33. package/dist/{chunk-4GCN34JV.mjs → chunk-AST3SHFA.mjs} +26 -24
  34. package/dist/chunk-BOJZGUS6.mjs +177 -0
  35. package/dist/{chunk-IP4HJOGC.mjs → chunk-BRUUBXCB.mjs} +4 -3
  36. package/dist/{chunk-J3EHATEZ.mjs → chunk-DQNKVOJ2.mjs} +63 -63
  37. package/dist/chunk-FOCV4CBZ.mjs +87 -0
  38. package/dist/{chunk-FFVLPCLH.mjs → chunk-GLZHNSGB.mjs} +1 -1
  39. package/dist/chunk-GWOS5DYC.mjs +2637 -0
  40. package/dist/{chunk-W2XDK66Z.mjs → chunk-IO5F7ZBE.mjs} +112 -438
  41. package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
  42. package/dist/{chunk-LLDFIKGP.mjs → chunk-KEIYVZUX.mjs} +4 -4
  43. package/dist/{chunk-UBXJQJII.mjs → chunk-KGOXHLLZ.mjs} +5 -1
  44. package/dist/{chunk-FKARMUWH.mjs → chunk-KZL72VTC.mjs} +68 -16
  45. package/dist/{chunk-TCWQJL7O.mjs → chunk-L76W3IZJ.mjs} +5 -0
  46. package/dist/{chunk-CPCGEIOM.mjs → chunk-LA4WNDHL.mjs} +83 -799
  47. package/dist/{chunk-PNNVTKMT.mjs → chunk-NJJCPFXN.mjs} +2 -2
  48. package/dist/chunk-NN33CZRE.mjs +86 -0
  49. package/dist/chunk-NRUXBYPH.mjs +121 -0
  50. package/dist/{chunk-UJWGQIXU.mjs → chunk-QHPYS427.mjs} +1 -0
  51. package/dist/{chunk-VSUYTPME.mjs → chunk-RHKRTYGO.mjs} +13 -4
  52. package/dist/{chunk-UEV53BBO.mjs → chunk-SRJBFG2T.mjs} +406 -5
  53. package/dist/{chunk-K3AQOZVM.mjs → chunk-TNBIK2LQ.mjs} +1638 -1644
  54. package/dist/chunk-UUZCQQG2.mjs +209 -0
  55. package/dist/{chunk-XMQNOD5V.mjs → chunk-VEFN6X6Q.mjs} +2 -2
  56. package/dist/{chunk-LK3MOTXF.mjs → chunk-VJEGOELZ.mjs} +8 -5
  57. package/dist/chunk-W2COJ5AH.mjs +148 -0
  58. package/dist/{chunk-IBATGS3J.mjs → chunk-WJGO5PGV.mjs} +241 -124
  59. package/dist/{chunk-CCCGEPJM.mjs → chunk-XZ3SRNDJ.mjs} +24 -63
  60. package/dist/{chunk-MTQTX4BP.mjs → chunk-YHUTF6EA.mjs} +400 -92
  61. package/dist/{chunk-B5I3IL4H.mjs → chunk-YLNSJQLX.mjs} +4 -4
  62. package/dist/{chunk-6VHWXAOI.mjs → chunk-Z4HFMYNX.mjs} +29 -24
  63. package/dist/{chunk-U4X4AWKL.mjs → chunk-ZPWS2SKJ.mjs} +293 -270
  64. package/dist/{client-DApBeLbk.d.ts → client-B12YFALP.d.ts} +3 -3
  65. package/dist/client.d.ts +7 -7
  66. package/dist/cold-module-4X7WJCFM.mjs +783 -0
  67. package/dist/{config-B0H-vp4U.d.ts → config-BhZxIeb1.d.ts} +6 -1
  68. package/dist/{config-Brysl9NO.d.ts → config-CIG5yDyg.d.ts} +2 -2
  69. package/dist/{direct-executor-J5UDLPL3.mjs → direct-executor-7XOQ7VQP.mjs} +5 -5
  70. package/dist/distribution/coordinator/etcd.d.ts +2 -2
  71. package/dist/distribution/coordinator/in-memory.d.ts +2 -2
  72. package/dist/distribution.d.ts +7 -7
  73. package/dist/distribution.mjs +123 -42
  74. package/dist/{durability-wiring-AYOTAPNQ.mjs → durability-wiring-324D2DDD.mjs} +213 -40
  75. package/dist/{engine-CuLMbH1-.d.ts → engine-CtzbHOmG.d.ts} +2 -2
  76. package/dist/{import-QRZAQ5RH.mjs → import-OOBUW3RR.mjs} +14 -13
  77. package/dist/{index-HYnqS1SE.d.ts → index-DsEurhSl.d.ts} +1 -1
  78. package/dist/index.browser.mjs +3 -3
  79. package/dist/index.d.ts +6 -6
  80. package/dist/index.mjs +11 -11
  81. package/dist/{long-query-thread-U7K5A72G.mjs → long-query-thread-4SW7RWDE.mjs} +8 -8
  82. package/dist/{module-L2OZO5PQ.mjs → module-DXSJ5LT2.mjs} +374 -83
  83. package/dist/module-IYPLWHGC.mjs +48 -0
  84. package/dist/paged-partition-archive-HFP45PLU.mjs +269 -0
  85. package/dist/partial-words.mjs +2 -2
  86. package/dist/persistence/durability/checkpoint-worker.d.ts +5 -1
  87. package/dist/persistence/durability/checkpoint-worker.mjs +17 -15
  88. package/dist/react.d.ts +9 -9
  89. package/dist/{browser/restore-queue-DISFECE4.mjs → restore-queue-FEYAC4SB.mjs} +2 -2
  90. package/dist/{schema-BTXbHCqA.d.ts → schema-B-Runv0n.d.ts} +7 -0
  91. package/dist/server.d.ts +10 -10
  92. package/dist/server.mjs +8 -7
  93. package/dist/{snapshot-BTRZ3MZK.mjs → snapshot-CGM65VCD.mjs} +60 -25
  94. package/dist/{tasks-DjGjK70Y.d.ts → tasks-CjRJRGTb.d.ts} +6 -4
  95. package/dist/{thread-USB3GOPY.mjs → thread-KGNAIKPD.mjs} +8 -8
  96. package/dist/vector/search-worker.d.ts +1 -1
  97. package/dist/vector/search-worker.mjs +3 -2
  98. package/dist/workers/entry.mjs +7 -7
  99. package/dist/{write-GHTA56OW.mjs → write-2DZPEQL2.mjs} +70 -98
  100. package/openapi.json +13 -0
  101. package/package.json +12 -8
  102. package/dist/browser/chunk-XVMOQQ34.mjs +0 -384
  103. package/dist/chunk-BDIDUW2B.mjs +0 -387
  104. package/dist/browser/{chunk-4MFH226Q.mjs → chunk-D33A5VMU.mjs} +15 -15
@@ -1,17 +1,17 @@
1
- import { requirePatternSearchFor, requirePartialWordSearchFor, createPartitionIndex, buildPartialWordTableFrom, recordValueSize, patternIndexOf, encodeFieldIndexes, encodePartialSpellingsOf, countPartialSpelling, ARRAY_POSITION_GAP, copyVectorsInto, isCompositePartition, fnv1a, partialSpellingsOfDocuments, createFrozenPartialWords, requirePatternSearch, documentValueBytes, cloneProjected, projectIncludedPaths, valueWithOwnBuffers, createSurfaceRegistry, MAX_TERM_FREQUENCY, loadPhraseCodeFor, joinPatternWorkMeter, fanOutQuery, sanitizeGlobalStats, countsWithoutScores, fanOutMatchCount, createCompositePartition, createGeoIndex, documentWithOwnBuffers, readProjectedPath, createNumericIndex, createBooleanIndex, createEnumIndex, ValueAffixes } from './chunk-U4X4AWKL.mjs';
1
+ import { requirePatternSearchFor, requirePartialWordSearchFor, createPartitionIndex, cloneProjected, projectIncludedPaths, valueWithOwnBuffers, buildPartialWordTableFrom, createSurfaceRegistry, encodePartialSpellingsOf, recordValueSize, patternIndexOf, encodeFieldIndexes, countPartialSpelling, ARRAY_POSITION_GAP, documentWithOwnBuffers, readProjectedPath, copyVectorsInto, isCompositePartition, fnv1a, partialSpellingsOfDocuments, createFrozenPartialWords, requirePatternSearch, documentValueBytes, MAX_TERM_FREQUENCY, loadPhraseCodeFor, joinPatternWorkMeter, fanOutQuery, sanitizeGlobalStats, countsWithoutScores, fanOutMatchCount, createGeoIndex, createNumericIndex, createBooleanIndex, createEnumIndex, ValueAffixes, createCompositePartition } from './chunk-ZPWS2SKJ.mjs';
2
2
  import { generateId } from './chunk-N37VMC62.mjs';
3
3
  import { readHeapStatistics, readResidentMemory } from './chunk-ZATRDAYU.mjs';
4
4
  import { GrowableUint32, bitsetHas, bitsetSet, createBitSet } from './chunk-JUKYYXSP.mjs';
5
- import { tokenizeOptions, getNestedValue, sameFields, visitDocumentSurfaceSpellings, visitDocumentSpellings } from './chunk-J3EHATEZ.mjs';
6
- import { partialFieldsOf, schemaFieldsOf, encodeDocument, validateDocument, validateDocumentStrict, extractVectorFieldsFromSchema } from './chunk-UBXJQJII.mjs';
5
+ import { getNestedValue, tokenizeOptions, sameFields, visitDocumentSurfaceSpellings, visitDocumentSpellings } from './chunk-DQNKVOJ2.mjs';
6
+ import { partialFieldsOf, schemaFieldsOf, encodeDocument, validateDocument, validateDocumentStrict, extractVectorFieldsFromSchema } from './chunk-KGOXHLLZ.mjs';
7
7
  import { fuzzyPrefixOf, boundedLevenshtein } from './chunk-GES7U2PM.mjs';
8
8
  import { isWordIndexedBase } from './chunk-ZSWHPKGM.mjs';
9
9
  import { resolveIndexAnalysis, synonymSearchModule } from './chunk-O4LJQI4G.mjs';
10
10
  import { getLanguage } from './chunk-WNBRW4Q2.mjs';
11
- import { visitTokens, codePointOrder, encodeStringBlob, encodeLookupKey } from './chunk-NPNK57CA.mjs';
11
+ import { encodeStringBlob, encodeLookupKey, visitTokens, codePointOrder } from './chunk-NPNK57CA.mjs';
12
12
  import { SEGMENT_TOKEN_TABLE_START, SEGMENT_POSTING_LOG_START, SEGMENT_POSITION_POOL_START, SEGMENT_FIELD_SCRATCH_START, FROZEN_REMOVAL_INDEX_MAX_PROCESS_FRACTION } from './chunk-4UMZ7KQL.mjs';
13
13
  import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
14
- import { loadedVectorCode, loadVectorCode, loadVectorCodeFor, requireLoadedVectorCode } from './chunk-FFVLPCLH.mjs';
14
+ import { loadedVectorCode, loadVectorCode, loadVectorCodeFor, requireLoadedVectorCode } from './chunk-GLZHNSGB.mjs';
15
15
  import { PROTOTYPE_POLLUTION_KEYS } from './chunk-L5ORED7V.mjs';
16
16
  import { NarsilError, ErrorCodes } from './chunk-HNHCNTVG.mjs';
17
17
  import { Decoder, decode } from '@msgpack/msgpack';
@@ -114,15 +114,19 @@ function createCapacityChecks(source) {
114
114
  }
115
115
 
116
116
  // src/partitioning/manager/doc-locations.ts
117
+ function documentIdsHeldInMemory(partition) {
118
+ return isCompositePartition(partition) ? partition.documentIdsHeldInMemory() : partition.docIds();
119
+ }
117
120
  function rebuildDocLocations(locations, partitions) {
118
121
  locations.clear();
119
122
  for (let i = 0; i < partitions.length; i++) {
120
- for (const docId of partitions[i].docIds()) {
123
+ for (const docId of documentIdsHeldInMemory(partitions[i])) {
121
124
  locations.set(docId, i);
122
125
  }
123
126
  }
124
127
  }
125
128
  function recordSegmentDocLocations(locations, partitionId, segment) {
129
+ if (segment.paged !== null) return;
126
130
  for (const internalId of segment.docStore.allInternalIds()) {
127
131
  const docId = segment.docStore.getExternalId(internalId);
128
132
  if (docId !== void 0) locations.set(docId, partitionId);
@@ -134,7 +138,7 @@ function rebuildPartitionDocLocations(locations, partitionId, partition) {
134
138
  locations.delete(docId);
135
139
  }
136
140
  }
137
- for (const docId of partition.docIds()) {
141
+ for (const docId of documentIdsHeldInMemory(partition)) {
138
142
  locations.set(docId, partitionId);
139
143
  }
140
144
  }
@@ -151,6 +155,109 @@ function requireNewSegmentDocuments(segment, docPartitionMap) {
151
155
  }
152
156
  }
153
157
 
158
+ // src/partitioning/manager/write-log.ts
159
+ var REANALYSIS_WRITE_LOG = 0;
160
+ var CHECKPOINT_WRITE_LOG = 1;
161
+ function createWriteLogRecorder() {
162
+ const current = [null, null];
163
+ return {
164
+ open(owner) {
165
+ const log = { docIds: /* @__PURE__ */ new Set(), replaced: false };
166
+ current[owner] = log;
167
+ return {
168
+ get partitionsReplaced() {
169
+ return log.replaced;
170
+ },
171
+ get pendingDocuments() {
172
+ return log.docIds.size;
173
+ },
174
+ takeWrittenDocIds() {
175
+ const taken = log.docIds;
176
+ log.docIds = /* @__PURE__ */ new Set();
177
+ return taken;
178
+ },
179
+ close() {
180
+ if (current[owner] === log) current[owner] = null;
181
+ }
182
+ };
183
+ },
184
+ documentWritten(docId) {
185
+ current[0]?.docIds.add(docId);
186
+ current[1]?.docIds.add(docId);
187
+ },
188
+ segmentWritten(segment) {
189
+ for (const log of current) {
190
+ if (log === null) continue;
191
+ for (const internalId of segment.docStore.allInternalIds()) {
192
+ const docId = segment.docStore.getExternalId(internalId);
193
+ if (docId !== void 0) log.docIds.add(docId);
194
+ }
195
+ }
196
+ },
197
+ partitionsReplaced() {
198
+ for (const log of current) {
199
+ if (log !== null) log.replaced = true;
200
+ }
201
+ }
202
+ };
203
+ }
204
+
205
+ // src/partitioning/manager/segment-operations.ts
206
+ function createSegmentOperations(source) {
207
+ const { docPartitionMap, writeLogs, validatePartitionId, asCompositePartition, resolveInsertOptions } = source;
208
+ return {
209
+ mergeSegment(partitionId, payload, documents) {
210
+ validatePartitionId(partitionId);
211
+ const partition = source.partitionAt(partitionId);
212
+ partition.beginBatch();
213
+ partition.mergeSegmentPayload(payload, documents);
214
+ partition.endBatch();
215
+ for (const docId of payload.docIds) {
216
+ docPartitionMap.set(docId, partitionId);
217
+ writeLogs.documentWritten(docId);
218
+ }
219
+ },
220
+ attachFrozenSegment(partitionId, segment) {
221
+ validatePartitionId(partitionId);
222
+ if (segment.paged === null) requireNewSegmentDocuments(segment, docPartitionMap);
223
+ asCompositePartition(partitionId).attachFrozenSegment(segment);
224
+ if (segment.paged !== null) return;
225
+ recordSegmentDocLocations(docPartitionMap, partitionId, segment);
226
+ writeLogs.segmentWritten(segment);
227
+ },
228
+ openCheckpointChangeLog() {
229
+ return writeLogs.open(CHECKPOINT_WRITE_LOG);
230
+ },
231
+ diskSegmentAccess(partitionId) {
232
+ validatePartitionId(partitionId);
233
+ return {
234
+ composite: asCompositePartition(partitionId),
235
+ docPartitionMap,
236
+ insertOptions: resolveInsertOptions()
237
+ };
238
+ },
239
+ swapFrozenSegments(partitionId, dropSegmentIds, replacement, removedSpellings) {
240
+ validatePartitionId(partitionId);
241
+ const applied = asCompositePartition(partitionId).swapFrozenSegments(
242
+ dropSegmentIds,
243
+ replacement,
244
+ removedSpellings
245
+ );
246
+ recordSegmentDocLocations(docPartitionMap, partitionId, replacement);
247
+ return applied;
248
+ },
249
+ freezeLiveTail(partitionId, freeze) {
250
+ validatePartitionId(partitionId);
251
+ return asCompositePartition(partitionId).freezeLiveTail(freeze);
252
+ },
253
+ replaceLiveTail(partitionId, segment) {
254
+ validatePartitionId(partitionId);
255
+ asCompositePartition(partitionId).replaceLiveTail(segment);
256
+ recordSegmentDocLocations(docPartitionMap, partitionId, segment);
257
+ }
258
+ };
259
+ }
260
+
154
261
  // src/partitioning/manager/stats.ts
155
262
  function aggregateStatsOf(partitions) {
156
263
  let totalDocuments = 0;
@@ -186,47 +293,13 @@ function partitionStatsOf(partitions) {
186
293
  }));
187
294
  }
188
295
 
189
- // src/partitioning/manager/write-log.ts
190
- function createWriteLogRecorder() {
191
- let current = null;
192
- return {
193
- open() {
194
- const log = { docIds: /* @__PURE__ */ new Set(), replaced: false };
195
- current = log;
196
- return {
197
- get partitionsReplaced() {
198
- return log.replaced;
199
- },
200
- get pendingDocuments() {
201
- return log.docIds.size;
202
- },
203
- takeWrittenDocIds() {
204
- const taken = log.docIds;
205
- log.docIds = /* @__PURE__ */ new Set();
206
- return taken;
207
- },
208
- close() {
209
- if (current === log) current = null;
210
- }
211
- };
212
- },
213
- documentWritten(docId) {
214
- if (current !== null) current.docIds.add(docId);
215
- },
216
- segmentWritten(segment) {
217
- if (current === null) return;
218
- for (const internalId of segment.docStore.allInternalIds()) {
219
- const docId = segment.docStore.getExternalId(internalId);
220
- if (docId !== void 0) current.docIds.add(docId);
221
- }
222
- },
223
- partitionsReplaced() {
224
- if (current !== null) current.replaced = true;
225
- }
226
- };
227
- }
228
-
229
296
  // src/partitioning/manager/index.ts
297
+ function closePagedSegmentsLeftBehind(earlier, kept) {
298
+ for (const partition of earlier) {
299
+ if (!isCompositePartition(partition) || kept.includes(partition)) continue;
300
+ for (const segment of partition.frozenSegments()) segment.paged?.close();
301
+ }
302
+ }
230
303
  function createPartitionManager(indexName, initialConfig, initialLanguage, router, initialPartitionCount, vectorIndexes) {
231
304
  let config = initialConfig;
232
305
  let language = initialLanguage;
@@ -240,7 +313,7 @@ function createPartitionManager(indexName, initialConfig, initialLanguage, route
240
313
  const vecIndexes = vectorIndexes ?? /* @__PURE__ */ new Map();
241
314
  let partitions = [];
242
315
  const docPartitionMap = /* @__PURE__ */ new Map();
243
- const writeLog = createWriteLogRecorder();
316
+ const writeLogs = createWriteLogRecorder();
244
317
  for (let i = 0; i < count; i++) {
245
318
  partitions.push(createPartitionIndex(i, trackPositions));
246
319
  }
@@ -260,7 +333,14 @@ function createPartitionManager(indexName, initialConfig, initialLanguage, route
260
333
  if (partialWordAnalysis !== null) partitions[partitionId].preparePartialWords(partialWordAnalysis);
261
334
  }
262
335
  function locateDocument(docId) {
263
- return docPartitionMap.get(docId);
336
+ const known = docPartitionMap.get(docId);
337
+ if (known !== void 0 || config.textStorage !== "disk") return known;
338
+ const routed = router.route(docId, partitions.length);
339
+ if (partitions[routed]?.has(docId)) return routed;
340
+ for (let i = 0; i < partitions.length; i++) {
341
+ if (i !== routed && partitions[i].has(docId)) return i;
342
+ }
343
+ return void 0;
264
344
  }
265
345
  function asCompositePartition(partitionId) {
266
346
  const existing = partitions[partitionId];
@@ -306,12 +386,14 @@ function createPartitionManager(indexName, initialConfig, initialLanguage, route
306
386
  return [...partitions];
307
387
  },
308
388
  setPartitions(newPartitions) {
309
- writeLog.partitionsReplaced();
389
+ writeLogs.partitionsReplaced();
390
+ const earlier = partitions;
310
391
  partitions = [...newPartitions];
392
+ closePagedSegmentsLeftBehind(earlier, partitions);
311
393
  rebuildDocLocations(docPartitionMap, partitions);
312
394
  },
313
395
  openWriteLog() {
314
- return writeLog.open();
396
+ return writeLogs.open(REANALYSIS_WRITE_LOG);
315
397
  },
316
398
  adoptRebuiltPartitions(rebuilt) {
317
399
  if (rebuilt.length !== partitions.length) {
@@ -321,7 +403,9 @@ function createPartitionManager(indexName, initialConfig, initialLanguage, route
321
403
  { indexName, partitionCount: partitions.length, rebuiltCount: rebuilt.length }
322
404
  );
323
405
  }
406
+ const earlier = partitions;
324
407
  partitions = [...rebuilt];
408
+ closePagedSegmentsLeftBehind(earlier, partitions);
325
409
  },
326
410
  adoptAnalysis(change) {
327
411
  config = change.config;
@@ -341,13 +425,13 @@ function createPartitionManager(indexName, initialConfig, initialLanguage, route
341
425
  );
342
426
  }
343
427
  const partition = createPartitionIndex(partitions.length, trackPositions);
344
- writeLog.partitionsReplaced();
428
+ writeLogs.partitionsReplaced();
345
429
  partitions.push(partition);
346
430
  return partition;
347
431
  },
348
432
  removePartition(partitionId) {
349
433
  validatePartitionId(partitionId);
350
- writeLog.partitionsReplaced();
434
+ writeLogs.partitionsReplaced();
351
435
  partitions.splice(partitionId, 1);
352
436
  rebuildDocLocations(docPartitionMap, partitions);
353
437
  },
@@ -361,7 +445,7 @@ function createPartitionManager(indexName, initialConfig, initialLanguage, route
361
445
  if (newCount >= partitions.length) {
362
446
  return;
363
447
  }
364
- writeLog.partitionsReplaced();
448
+ writeLogs.partitionsReplaced();
365
449
  for (const [docId, pid] of docPartitionMap) {
366
450
  if (pid >= newCount) {
367
451
  docPartitionMap.delete(docId);
@@ -384,7 +468,7 @@ function createPartitionManager(indexName, initialConfig, initialLanguage, route
384
468
  const insertOpts = resolveInsertOptions(options);
385
469
  partitions[pid].insert(docId, document, config.schema, language, insertOpts);
386
470
  docPartitionMap.set(docId, pid);
387
- writeLog.documentWritten(docId);
471
+ writeLogs.documentWritten(docId);
388
472
  },
389
473
  removedSpellingsOf(docId) {
390
474
  const pid = locateDocument(docId);
@@ -398,7 +482,7 @@ function createPartitionManager(indexName, initialConfig, initialLanguage, route
398
482
  }
399
483
  partitions[pid].remove(docId, config.schema, language, resolveInsertOptions(), removedSpellings);
400
484
  docPartitionMap.delete(docId);
401
- writeLog.documentWritten(docId);
485
+ writeLogs.documentWritten(docId);
402
486
  },
403
487
  beginBatchRemove() {
404
488
  for (let i = 0; i < partitions.length; i++) {
@@ -418,7 +502,7 @@ function createPartitionManager(indexName, initialConfig, initialLanguage, route
418
502
  const updateOpts = resolveInsertOptions(options);
419
503
  partitions[pid].update(docId, document, config.schema, language, updateOpts, removedSpellings);
420
504
  docPartitionMap.set(docId, pid);
421
- writeLog.documentWritten(docId);
505
+ writeLogs.documentWritten(docId);
422
506
  },
423
507
  get(docId, projection) {
424
508
  const pid = locateDocument(docId);
@@ -458,46 +542,17 @@ function createPartitionManager(indexName, initialConfig, initialLanguage, route
458
542
  validatePartitionId(partitionId);
459
543
  return partitions[partitionId].serialize(indexName, partitions.length, language.name, config.schema);
460
544
  },
461
- mergeSegment(partitionId, payload, documents) {
462
- validatePartitionId(partitionId);
463
- const partition = partitions[partitionId];
464
- partition.beginBatch();
465
- partition.mergeSegmentPayload(payload, documents);
466
- partition.endBatch();
467
- for (const docId of payload.docIds) {
468
- docPartitionMap.set(docId, partitionId);
469
- writeLog.documentWritten(docId);
470
- }
471
- },
472
- attachFrozenSegment(partitionId, segment) {
473
- validatePartitionId(partitionId);
474
- requireNewSegmentDocuments(segment, docPartitionMap);
475
- asCompositePartition(partitionId).attachFrozenSegment(segment);
476
- recordSegmentDocLocations(docPartitionMap, partitionId, segment);
477
- writeLog.segmentWritten(segment);
478
- },
479
- swapFrozenSegments(partitionId, dropSegmentIds, replacement, removedSpellings) {
480
- validatePartitionId(partitionId);
481
- const applied = asCompositePartition(partitionId).swapFrozenSegments(
482
- dropSegmentIds,
483
- replacement,
484
- removedSpellings
485
- );
486
- recordSegmentDocLocations(docPartitionMap, partitionId, replacement);
487
- return applied;
488
- },
489
- freezeLiveTail(partitionId, freeze) {
490
- validatePartitionId(partitionId);
491
- return asCompositePartition(partitionId).freezeLiveTail(freeze);
492
- },
493
- replaceLiveTail(partitionId, segment) {
494
- validatePartitionId(partitionId);
495
- asCompositePartition(partitionId).replaceLiveTail(segment);
496
- recordSegmentDocLocations(docPartitionMap, partitionId, segment);
497
- },
545
+ ...createSegmentOperations({
546
+ docPartitionMap,
547
+ writeLogs,
548
+ partitionAt: (partitionId) => partitions[partitionId],
549
+ validatePartitionId,
550
+ asCompositePartition,
551
+ resolveInsertOptions: () => resolveInsertOptions()
552
+ }),
498
553
  deserializePartition(partitionId, data) {
499
554
  validatePartitionId(partitionId);
500
- writeLog.partitionsReplaced();
555
+ writeLogs.partitionsReplaced();
501
556
  partitions[partitionId].deserialize(data, config.schema);
502
557
  preparePartialWords(partitionId);
503
558
  rebuildPartitionDocLocations(docPartitionMap, partitionId, partitions[partitionId]);
@@ -974,11 +1029,17 @@ function encodeDocumentTableData(documents) {
974
1029
  }
975
1030
  function wrapEncodedDocumentTable(data) {
976
1031
  const { blob, offsets } = data;
977
- const count = offsets.length - 1;
978
- const discarded = (data.discardedFields ?? []).map((fieldPath) => fieldPath.split("."));
1032
+ return createEncodedDocumentSource(
1033
+ offsets.length - 1,
1034
+ blob.byteLength + offsets.byteLength,
1035
+ (ordinal) => blob.subarray(offsets[ordinal], offsets[ordinal + 1]),
1036
+ data.discardedFields
1037
+ );
1038
+ }
1039
+ function createEncodedDocumentSource(count, byteLength, bytesAt, discardedFields) {
1040
+ const discarded = (discardedFields ?? []).map((fieldPath) => fieldPath.split("."));
979
1041
  const touchesDiscarded = (path) => discarded.some((discardedPath) => sharesPrefix(discardedPath, path));
980
- const bytesAt = (ordinal) => blob.subarray(offsets[ordinal], offsets[ordinal + 1]);
981
- const docAt = (ordinal) => documentWithOwnBuffers(decodeStoredDocument(bytesAt(ordinal), data.discardedFields));
1042
+ const docAt = (ordinal) => documentWithOwnBuffers(decodeStoredDocument(bytesAt(ordinal), discardedFields));
982
1043
  function projectedPathAt(ordinal, path) {
983
1044
  const segments = encodedProjectionPathOf(path);
984
1045
  if (segments === null) return { found: false, value: void 0 };
@@ -988,10 +1049,10 @@ function wrapEncodedDocumentTable(data) {
988
1049
  }
989
1050
  return {
990
1051
  count,
991
- byteLength: blob.byteLength + offsets.byteLength,
1052
+ byteLength,
992
1053
  docAt,
993
1054
  readAt(ordinal) {
994
- return decodeStoredDocument(bytesAt(ordinal), data.discardedFields);
1055
+ return decodeStoredDocument(bytesAt(ordinal), discardedFields);
995
1056
  },
996
1057
  fieldReader(fieldPath) {
997
1058
  const path = fieldPath.split(".");
@@ -1084,6 +1145,9 @@ function wrapExternalIdTable(data) {
1084
1145
  const ordinal = sortedOrdinals[lo];
1085
1146
  return compareBytes(blob, offsets[ordinal], offsets[ordinal + 1], query) === 0 ? ordinal : -1;
1086
1147
  },
1148
+ sortedOrdinalAt(position) {
1149
+ return sortedOrdinals[position];
1150
+ },
1087
1151
  collectSortedIds(excluded) {
1088
1152
  const ids = [];
1089
1153
  for (let position = 0; position < count; position++) {
@@ -1103,6 +1167,7 @@ function buildExternalIdTable(docIds) {
1103
1167
  characters += docIds[ordinal].length;
1104
1168
  }
1105
1169
  let sortedIds = null;
1170
+ let sortedOrdinals = null;
1106
1171
  return {
1107
1172
  count: docIds.length,
1108
1173
  bytes: characters,
@@ -1112,6 +1177,10 @@ function buildExternalIdTable(docIds) {
1112
1177
  ordinalOf(docId) {
1113
1178
  return ordinals.get(docId) ?? -1;
1114
1179
  },
1180
+ sortedOrdinalAt(position) {
1181
+ if (sortedOrdinals === null) sortedOrdinals = codePointOrder(docIds);
1182
+ return sortedOrdinals[position];
1183
+ },
1115
1184
  collectSortedIds(excluded) {
1116
1185
  if (sortedIds === null) {
1117
1186
  const ids = [...docIds];
@@ -1650,6 +1719,10 @@ function createFrozenRemovalPass(documents, surfaces) {
1650
1719
  });
1651
1720
  return spellings;
1652
1721
  }
1722
+ function subtract(ordinal, analysis) {
1723
+ if (surfaces.tableStaysOnDisk()) surfaces.subtractOnceLoaded(ordinal, analysis);
1724
+ else surfaces.subtractSpellings(ordinal, heldSpellingsOf(ordinal, analysis));
1725
+ }
1653
1726
  return {
1654
1727
  get settled() {
1655
1728
  return pending.length === 0;
@@ -1658,17 +1731,17 @@ function createFrozenRemovalPass(documents, surfaces) {
1658
1731
  if (bound !== null) return;
1659
1732
  const scoped = { ...analysis, fields: documents.textFields };
1660
1733
  bound = scoped;
1661
- for (const ordinal of pending) surfaces.subtractSpellings(ordinal, heldSpellingsOf(ordinal, scoped));
1734
+ for (const ordinal of pending) subtract(ordinal, scoped);
1662
1735
  pending = [];
1663
1736
  },
1664
1737
  spellingsOf(ordinal) {
1665
- return bound === null ? void 0 : heldSpellingsOf(ordinal, bound);
1738
+ return bound === null || surfaces.tableStaysOnDisk() ? void 0 : heldSpellingsOf(ordinal, bound);
1666
1739
  },
1667
1740
  afterRemoval(ordinal, spellings) {
1668
1741
  if (spellings !== void 0) {
1669
1742
  surfaces.subtractSpellings(ordinal, spellings);
1670
1743
  } else if (bound !== null) {
1671
- surfaces.subtractSpellings(ordinal, heldSpellingsOf(ordinal, bound));
1744
+ subtract(ordinal, bound);
1672
1745
  } else {
1673
1746
  pending.push(ordinal);
1674
1747
  }
@@ -1692,15 +1765,32 @@ function addForms(target, forms) {
1692
1765
  if (Array.isArray(value)) target.add(surface, value[1], value[0]);
1693
1766
  }
1694
1767
  }
1695
- function createFrozenSurfaceSlot(load, documents) {
1768
+ function createFrozenSurfaceSlot(load, documents, tableOnDisk) {
1696
1769
  let registry = null;
1697
1770
  let uncounted = null;
1771
+ let table = tableOnDisk;
1772
+ let waitingRemovals = [];
1773
+ let waitingAnalysis = null;
1774
+ function subtractWaitingRemovals(target) {
1775
+ if (waitingAnalysis === null || target.size() === 0) return;
1776
+ for (const ordinal of waitingRemovals) {
1777
+ const spellings = [];
1778
+ visitDocumentSurfaceSpellings(documents.documentAt(ordinal), waitingAnalysis, (spelling) => {
1779
+ if (target.holds(spelling)) spellings.push(spelling);
1780
+ });
1781
+ for (const spelling of spellings) target.subtract(spelling, 1);
1782
+ }
1783
+ }
1698
1784
  function loaded() {
1699
1785
  if (registry === null) {
1700
1786
  const built = createSurfaceRegistry();
1701
1787
  const forms = load();
1702
1788
  if (forms !== null) built.deserialize(forms);
1789
+ subtractWaitingRemovals(built);
1703
1790
  registry = built;
1791
+ table = void 0;
1792
+ waitingRemovals = [];
1793
+ waitingAnalysis = null;
1704
1794
  }
1705
1795
  return registry;
1706
1796
  }
@@ -1708,12 +1798,22 @@ function createFrozenSurfaceSlot(load, documents) {
1708
1798
  return uncounted === null || uncounted[ordinal] === 0;
1709
1799
  }
1710
1800
  return {
1801
+ heldBytes: () => table?.heldBytes() ?? 0,
1711
1802
  reader: {
1712
- candidatesForPrefix: (prefix) => loaded().candidatesForPrefix(prefix),
1803
+ candidatesForPrefix: (prefix) => table !== void 0 && waitingRemovals.length === 0 ? table.candidatesForPrefix(prefix) : loaded().candidatesForPrefix(prefix),
1713
1804
  stemChangedTotalFor: (token) => loaded().stemChangedTotalFor(token),
1714
1805
  size: () => loaded().size(),
1715
1806
  serialize: () => loaded().serialize()
1716
1807
  },
1808
+ tableStaysOnDisk() {
1809
+ return registry === null && tableOnDisk !== void 0;
1810
+ },
1811
+ subtractOnceLoaded(ordinal, analysis) {
1812
+ uncounted ??= new Uint8Array(documents.count);
1813
+ uncounted[ordinal] = 1;
1814
+ waitingAnalysis = analysis;
1815
+ waitingRemovals.push(ordinal);
1816
+ },
1717
1817
  holdsAny() {
1718
1818
  return loaded().size() > 0;
1719
1819
  },
@@ -1861,6 +1961,9 @@ function wrapFrozenTokenTable(data) {
1861
1961
  }
1862
1962
  return token;
1863
1963
  },
1964
+ tokenBytesAt(sortedIndex) {
1965
+ return blob.subarray(offsets[sortedIndex], offsets[sortedIndex + 1]);
1966
+ },
1864
1967
  byteLengthAt(sortedIndex) {
1865
1968
  return offsets[sortedIndex + 1] - offsets[sortedIndex];
1866
1969
  },
@@ -2135,10 +2238,10 @@ function buildGeoReaders(entries) {
2135
2238
  }
2136
2239
  return readers;
2137
2240
  }
2138
- function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot, partialWordTable, survivingSurfaceForms) {
2241
+ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot, partialWordTable, survivingSurfaceForms, pagedParts) {
2139
2242
  const tombstones = createFrozenTombstones();
2140
- const counts = createFrozenLiveCounts(source, source.tokenTable, tombstones, storedFrequenciesOf(source));
2141
- const postingViews = createFrozenPostingViews(source, tombstones, counts);
2243
+ const counts = pagedParts?.createCounts(tombstones) ?? createFrozenLiveCounts(source, source.tokenTable, tombstones, storedFrequenciesOf(source));
2244
+ const postingViews = pagedParts?.createViews(tombstones, counts) ?? createFrozenPostingViews(source, tombstones, counts);
2142
2245
  const lengthColumns = /* @__PURE__ */ new Map();
2143
2246
  for (let i = 0; i < source.fieldLengthNames.length; i++) {
2144
2247
  lengthColumns.set(source.fieldLengthNames[i], source.fieldLengthColumns[i]);
@@ -2155,7 +2258,11 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
2155
2258
  documentAt: (ordinal) => documentSource.readAt(ordinal)
2156
2259
  };
2157
2260
  const loadSurfaceForms = survivingSurfaceForms === void 0 ? source.readSurfaceForms : () => survivingSurfaceForms;
2158
- const surfaces = createFrozenSurfaceSlot(loadSurfaceForms, textDocuments);
2261
+ const surfaces = createFrozenSurfaceSlot(
2262
+ loadSurfaceForms,
2263
+ textDocuments,
2264
+ survivingSurfaceForms === void 0 ? source.surfaceTableOnDisk : void 0
2265
+ );
2159
2266
  const removals = createFrozenRemovalPass(textDocuments, surfaces);
2160
2267
  const numericIndexes = /* @__PURE__ */ new Map();
2161
2268
  for (const entry of source.numeric) numericIndexes.set(entry.fieldPath, createFrozenNumericReader(entry));
@@ -2177,6 +2284,7 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
2177
2284
  segmentId,
2178
2285
  documentSource,
2179
2286
  sharedSnapshot,
2287
+ paged: pagedParts?.access ?? null,
2180
2288
  arrays: source,
2181
2289
  invertedIdx: createFrozenInvertedReader(source.tokenTable, postingViews, counts),
2182
2290
  docStore,
@@ -2192,7 +2300,7 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
2192
2300
  sizeIndexes,
2193
2301
  fieldNameTable,
2194
2302
  partialWords,
2195
- trackPositions: source.positionOffsets !== null,
2303
+ trackPositions: pagedParts?.access.snapshot.tracksPositions ?? source.positionOffsets !== null,
2196
2304
  flatSchemaCache: null,
2197
2305
  lastSchemaRef: null,
2198
2306
  sortColumns: null,
@@ -2236,6 +2344,9 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
2236
2344
  };
2237
2345
  return segment;
2238
2346
  }
2347
+ function assemblePagedFrozenSegment(segmentId, source, documentSource, partialWordTable, parts) {
2348
+ return assembleFrozenSegment(segmentId, source, documentSource, null, partialWordTable, void 0, parts);
2349
+ }
2239
2350
  function createFrozenSegment(payload, documents, segmentId) {
2240
2351
  const tokenTable = buildFrozenTokenTable(payload.tokens, payload.docFrequencies);
2241
2352
  return assembleFrozenSegment(
@@ -3099,7 +3210,20 @@ function buildSegmentPayload(documents, schema, language, options, trackPosition
3099
3210
 
3100
3211
  // src/workers/segment-actions.ts
3101
3212
  function isSegmentAction(action) {
3102
- return action.type === "buildSegment" || action.type === "mergeSegments" || action.type === "attachSegments" || action.type === "compactSegments" || action.type === "swapSegments" || action.type === "freezeLiveTail";
3213
+ return action.type === "buildSegment" || action.type === "mergeSegments" || action.type === "attachSegments" || action.type === "compactSegments" || action.type === "swapSegments" || action.type === "freezeLiveTail" || action.type === "adoptDiskSegments";
3214
+ }
3215
+ function isPagedSnapshot(snapshot) {
3216
+ return "path" in snapshot;
3217
+ }
3218
+ async function attachSegments(entry, action) {
3219
+ for (const segment of action.segments) {
3220
+ growPartitionsTo(entry.manager, segment.partitionId);
3221
+ const surfaceForms = segment.survivingSurfaceForms;
3222
+ const spellings = surfaceForms === void 0 ? void 0 : NO_REMOVED_SPELLINGS;
3223
+ const frozen = isPagedSnapshot(segment.snapshot) ? (await import('./module-IYPLWHGC.mjs')).createPagedFrozenSegment(segment.snapshot) : createSharedFrozenSegment(segment.snapshot, surfaceForms);
3224
+ for (const docId of segment.tombstonedDocIds ?? []) frozen.tombstoneDocument(docId, spellings);
3225
+ entry.manager.attachFrozenSegment(segment.partitionId, frozen);
3226
+ }
3103
3227
  }
3104
3228
  function growPartitionsTo(manager, partitionId) {
3105
3229
  while (manager.partitionCount <= partitionId) {
@@ -3148,17 +3272,10 @@ function runSegmentAction(entry, action) {
3148
3272
  }
3149
3273
  return void 0;
3150
3274
  }
3151
- case "attachSegments": {
3152
- for (const segment of action.segments) {
3153
- growPartitionsTo(entry.manager, segment.partitionId);
3154
- const surfaceForms = segment.survivingSurfaceForms;
3155
- const frozen = createSharedFrozenSegment(segment.snapshot, surfaceForms);
3156
- const spellings = surfaceForms === void 0 ? void 0 : NO_REMOVED_SPELLINGS;
3157
- for (const docId of segment.tombstonedDocIds ?? []) frozen.tombstoneDocument(docId, spellings);
3158
- entry.manager.attachFrozenSegment(segment.partitionId, frozen);
3159
- }
3160
- return void 0;
3161
- }
3275
+ case "attachSegments":
3276
+ return attachSegments(entry, action);
3277
+ case "adoptDiskSegments":
3278
+ return import('./module-IYPLWHGC.mjs').then((code) => code.adoptSegmentsOnWorker(entry.manager, action));
3162
3279
  case "compactSegments": {
3163
3280
  const partition = requireComposite(entry, action.indexName, action.partitionId, "compact");
3164
3281
  return mergeFrozenSegments(
@@ -3560,4 +3677,4 @@ function createDirectExecutor(options) {
3560
3677
  };
3561
3678
  }
3562
3679
 
3563
- export { availableSynonymModule, buildSegmentPayload, createDirectExecutor, createFrozenSegment, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, decodeEveryStoredDocument, decodeMapWithoutFields, decodeStoredDocument, freezeSegmentShared, isEmptySynonymSet, mergeFrozenSegments, mergeSegmentContents, registerSynonyms, resolvePartialWordAnalysis, resolvePartitionInsertOptions, segmentArraysOf, selectionKeeps, shareSegmentArrays, synonymModule };
3680
+ export { assemblePagedFrozenSegment, availableSynonymModule, buildSegmentPayload, createDirectExecutor, createEncodedDocumentSource, createFrozenSegment, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, decodeEveryStoredDocument, decodeMapWithoutFields, decodeStoredDocument, decodeSurfaceTable, encodeSurfaceTable, endOfValue, fieldIndexRemapOf, freezeSegmentShared, holdsADiscardedField, isEmptySynonymSet, mergeFieldIndexes, mergeFieldLengths, mergeFrozenSegments, mergePartialWords, mergeSegmentContents, mergeSurfaceForms, registerSynonyms, resolvePartialWordAnalysis, resolvePartitionInsertOptions, segmentArraysOf, selectionKeeps, shareSegmentArrays, synonymModule, wrapFrozenTokenTable };