@delali/narsil 0.3.1-next.131 → 0.3.1-next.141

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. package/dist/adapters/filesystem.mjs +1 -1
  2. package/dist/adapters/indexeddb.mjs +1 -1
  3. package/dist/analysis-change-HQUT2Z3I.mjs +43 -0
  4. package/dist/analysis-change-entry-TZTG4AZB.mjs +1 -0
  5. package/dist/browser/analysis-change-JPV775LR.mjs +900 -0
  6. package/dist/browser/analysis-change-entry-7HR4LWL3.mjs +1 -0
  7. package/dist/browser/{chunk-UDGB2MWT.mjs → chunk-35B42UFM.mjs} +4 -0
  8. package/dist/browser/chunk-3ZBJYAS4.mjs +19 -0
  9. package/dist/browser/{chunk-2Y3CO4EQ.mjs → chunk-5DB2VDGI.mjs} +1149 -1853
  10. package/dist/browser/chunk-6RHTAVZX.mjs +656 -0
  11. package/dist/browser/chunk-CCOAOO73.mjs +86 -0
  12. package/dist/browser/{chunk-AIIJDWMG.mjs → chunk-D33A5VMU.mjs} +16 -16
  13. package/dist/browser/{chunk-VBBLTD7O.mjs → chunk-HKX2G6WH.mjs} +1520 -2159
  14. package/dist/browser/{chunk-NSCSR6MA.mjs → chunk-KTOK275N.mjs} +606 -830
  15. package/dist/browser/{chunk-D7WQF6DV.mjs → chunk-LVY2LI35.mjs} +2 -2
  16. package/dist/browser/chunk-MVM33ZUL.mjs +993 -0
  17. package/dist/browser/chunk-MWUK3DA5.mjs +2634 -0
  18. package/dist/browser/{chunk-VEGHFP2P.mjs → chunk-MZI3TEOW.mjs} +4 -2
  19. package/dist/browser/chunk-OIZS5KIX.mjs +6 -0
  20. package/dist/browser/chunk-R4WUDZQV.mjs +176 -0
  21. package/dist/browser/{chunk-M5UAAXGU.mjs → chunk-TCLEOZ4P.mjs} +3 -3
  22. package/dist/browser/chunk-VFYMPCOV.mjs +212 -0
  23. package/dist/browser/chunk-WCFCHIBB.mjs +177 -0
  24. package/dist/browser/chunk-XNWPP5NS.mjs +21 -0
  25. package/dist/browser/chunk-ZEV26AQ6.mjs +270 -0
  26. package/dist/browser/cold-module-MJMKNUYJ.mjs +745 -0
  27. package/dist/browser/{durability-wiring-FNZSRH4N.mjs → durability-wiring-SNOWIT6Z.mjs} +370 -95
  28. package/dist/browser/import-TLB2A5LJ.mjs +9 -0
  29. package/dist/browser/module-BUFNZMEU.mjs +12 -0
  30. package/dist/browser/{module-4HIMOQM5.mjs → module-BVP4TKZU.mjs} +397 -109
  31. package/dist/browser/{phrase-module-WESYVU7K.mjs → phrase-module-HGVBO24W.mjs} +1 -1
  32. package/dist/{restore-queue-JAMNRC6T.mjs → browser/restore-queue-ONYNYUDE.mjs} +8 -2
  33. package/dist/browser/{runtime-3X7ZJ5T3.mjs → runtime-VEUH674B.mjs} +2 -2
  34. package/dist/browser/{snapshot-V7VF6BN6.mjs → snapshot-QXBKHJAG.mjs} +135 -183
  35. package/dist/browser/{write-YIC2MKRY.mjs → write-WHFBDWQS.mjs} +64 -90
  36. package/dist/{capabilities-B6WxrWxt.d.ts → capabilities-BCdUwK_L.d.ts} +10 -3
  37. package/dist/chunk-46NZOPHS.mjs +21 -0
  38. package/dist/{chunk-JT3YZGCE.mjs → chunk-5WYVUPGD.mjs} +3 -188
  39. package/dist/chunk-6SNBQESJ.mjs +214 -0
  40. package/dist/{chunk-2C3WXBQS.mjs → chunk-74XB2SXI.mjs} +1 -1
  41. package/dist/chunk-7AUASRE7.mjs +653 -0
  42. package/dist/{chunk-NMXRXFGZ.mjs → chunk-AST3SHFA.mjs} +27 -25
  43. package/dist/chunk-BOJZGUS6.mjs +177 -0
  44. package/dist/{chunk-T4OFG4IQ.mjs → chunk-BRUUBXCB.mjs} +92 -17
  45. package/dist/{chunk-5LX6MOIB.mjs → chunk-C3YDYX6C.mjs} +1 -1
  46. package/dist/chunk-CZQ33DLW.mjs +78 -0
  47. package/dist/{chunk-4GDLFDPZ.mjs → chunk-DD2673TX.mjs} +3 -2
  48. package/dist/{chunk-EUCX5K74.mjs → chunk-DQNKVOJ2.mjs} +65 -65
  49. package/dist/{chunk-RNSC4NFI.mjs → chunk-EEBCDP36.mjs} +3 -1
  50. package/dist/chunk-FOCV4CBZ.mjs +87 -0
  51. package/dist/{chunk-HCGJKDZV.mjs → chunk-GLZHNSGB.mjs} +1 -1
  52. package/dist/chunk-GWOS5DYC.mjs +2637 -0
  53. package/dist/{chunk-PATMDIMN.mjs → chunk-HJJWY6PT.mjs} +1 -1
  54. package/dist/{chunk-CZ7ISLXA.mjs → chunk-HNHCNTVG.mjs} +4 -0
  55. package/dist/{chunk-Y2U5VVWX.mjs → chunk-ICW45TTU.mjs} +1 -1
  56. package/dist/{chunk-TGHNXN7W.mjs → chunk-IO5F7ZBE.mjs} +261 -948
  57. package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
  58. package/dist/{chunk-7OIQ2SIO.mjs → chunk-KEIYVZUX.mjs} +5 -5
  59. package/dist/{chunk-PU6SKH7O.mjs → chunk-KGOXHLLZ.mjs} +7 -3
  60. package/dist/{chunk-WVRDRLIC.mjs → chunk-KZL72VTC.mjs} +70 -18
  61. package/dist/chunk-L76W3IZJ.mjs +193 -0
  62. package/dist/{chunk-2JWNJSNE.mjs → chunk-LA4WNDHL.mjs} +88 -801
  63. package/dist/{chunk-ZR4VM4A6.mjs → chunk-N37VMC62.mjs} +1 -18
  64. package/dist/{chunk-JJ6EUMXB.mjs → chunk-NJJCPFXN.mjs} +2 -2
  65. package/dist/chunk-NN33CZRE.mjs +86 -0
  66. package/dist/{chunk-2WBY4Y4M.mjs → chunk-NOT5QOHU.mjs} +1 -1
  67. package/dist/{chunk-D5TFRB5G.mjs → chunk-NPNK57CA.mjs} +1 -1
  68. package/dist/chunk-NRUXBYPH.mjs +121 -0
  69. package/dist/{chunk-6SSVIW7Y.mjs → chunk-O4LJQI4G.mjs} +31 -5
  70. package/dist/{chunk-UJWGQIXU.mjs → chunk-QHPYS427.mjs} +1 -0
  71. package/dist/{chunk-F6AAXI7T.mjs → chunk-RHKRTYGO.mjs} +13 -4
  72. package/dist/{chunk-2SEL6SBS.mjs → chunk-RKAVHNRW.mjs} +2 -2
  73. package/dist/{chunk-RCMNL7RI.mjs → chunk-SBMAWTRG.mjs} +41 -4
  74. package/dist/chunk-SRJBFG2T.mjs +881 -0
  75. package/dist/{chunk-X5CNUKD5.mjs → chunk-TNBIK2LQ.mjs} +1663 -1652
  76. package/dist/{chunk-NNO7CO3A.mjs → chunk-UIKN2OTQ.mjs} +1 -1
  77. package/dist/chunk-UUZCQQG2.mjs +209 -0
  78. package/dist/{chunk-6M6SDH3S.mjs → chunk-VEFN6X6Q.mjs} +8 -8
  79. package/dist/chunk-VJEGOELZ.mjs +177 -0
  80. package/dist/chunk-W2COJ5AH.mjs +148 -0
  81. package/dist/{chunk-RWESFSJI.mjs → chunk-WJGO5PGV.mjs} +317 -174
  82. package/dist/{chunk-7X5JST5L.mjs → chunk-WNBRW4Q2.mjs} +16 -2
  83. package/dist/{chunk-SZGRYEIC.mjs → chunk-XS2VM7UE.mjs} +3 -3
  84. package/dist/{chunk-Y4BVTQ53.mjs → chunk-XZ3SRNDJ.mjs} +25 -64
  85. package/dist/{chunk-K77FJXKS.mjs → chunk-YHUTF6EA.mjs} +474 -93
  86. package/dist/{chunk-OGLV4I3L.mjs → chunk-YLNSJQLX.mjs} +12 -7
  87. package/dist/{chunk-SOHMMNCH.mjs → chunk-Z4HFMYNX.mjs} +32 -27
  88. package/dist/{chunk-5W7AEVTH.mjs → chunk-ZMBFWQXN.mjs} +2 -2
  89. package/dist/{chunk-2MKX2SCO.mjs → chunk-ZPWS2SKJ.mjs} +297 -274
  90. package/dist/{chunk-7XY6VD7B.mjs → chunk-ZSWHPKGM.mjs} +1 -1
  91. package/dist/{client-gNAfaHcE.d.ts → client-B12YFALP.d.ts} +18 -3
  92. package/dist/client.d.ts +7 -7
  93. package/dist/client.mjs +11 -1
  94. package/dist/cold-module-4X7WJCFM.mjs +783 -0
  95. package/dist/{config-D1cDv3sE.d.ts → config-BhZxIeb1.d.ts} +10 -1
  96. package/dist/{config-CmYJO34X.d.ts → config-CIG5yDyg.d.ts} +2 -2
  97. package/dist/{direct-executor-HQ4QVDX7.mjs → direct-executor-7XOQ7VQP.mjs} +14 -14
  98. package/dist/distribution/coordinator/etcd.d.ts +2 -2
  99. package/dist/distribution/coordinator/etcd.mjs +1 -1
  100. package/dist/distribution/coordinator/in-memory.d.ts +2 -2
  101. package/dist/distribution/transport/grpc.mjs +1 -1
  102. package/dist/distribution.d.ts +24 -8
  103. package/dist/distribution.mjs +982 -211
  104. package/dist/{durability-wiring-S7OX2MAI.mjs → durability-wiring-324D2DDD.mjs} +362 -94
  105. package/dist/embeddings/openai.mjs +2 -2
  106. package/dist/{engine-m_Ddr1bz.d.ts → engine-CtzbHOmG.d.ts} +64 -7
  107. package/dist/import-OOBUW3RR.mjs +44 -0
  108. package/dist/{index-HYnqS1SE.d.ts → index-DsEurhSl.d.ts} +1 -1
  109. package/dist/index.browser.mjs +5 -5
  110. package/dist/index.d.ts +6 -6
  111. package/dist/index.mjs +22 -22
  112. package/dist/{long-query-thread-BSL25PD6.mjs → long-query-thread-4SW7RWDE.mjs} +15 -15
  113. package/dist/{module-2E4XYSGT.mjs → module-DXSJ5LT2.mjs} +378 -87
  114. package/dist/module-IYPLWHGC.mjs +48 -0
  115. package/dist/paged-partition-archive-HFP45PLU.mjs +269 -0
  116. package/dist/partial-words.mjs +5 -5
  117. package/dist/pattern.mjs +3 -3
  118. package/dist/persistence/durability/checkpoint-worker.d.ts +16 -1
  119. package/dist/persistence/durability/checkpoint-worker.mjs +28 -26
  120. package/dist/{phrase-module-JERF2IDG.mjs → phrase-module-5QHBD2VJ.mjs} +1 -1
  121. package/dist/react.d.ts +9 -9
  122. package/dist/react.mjs +4 -0
  123. package/dist/{rerank-DMMT4FZA.mjs → rerank-KTLC3E6N.mjs} +5 -5
  124. package/dist/rerankers/http.mjs +2 -2
  125. package/dist/{browser/restore-queue-GEZ54CA5.mjs → restore-queue-FEYAC4SB.mjs} +8 -2
  126. package/dist/{runtime-7MLQ7J64.mjs → runtime-DYV75DCB.mjs} +6 -6
  127. package/dist/{schema-BTXbHCqA.d.ts → schema-B-Runv0n.d.ts} +7 -0
  128. package/dist/server.d.ts +12 -10
  129. package/dist/server.mjs +54 -21
  130. package/dist/{snapshot-DOQLGGEX.mjs → snapshot-CGM65VCD.mjs} +101 -197
  131. package/dist/synonyms.mjs +6 -4
  132. package/dist/{tasks-Co8HfRD7.d.ts → tasks-CjRJRGTb.d.ts} +26 -8
  133. package/dist/{thread-SKO7CN3R.mjs → thread-KGNAIKPD.mjs} +14 -14
  134. package/dist/vector/search-worker.d.ts +1 -1
  135. package/dist/vector/search-worker.mjs +6 -5
  136. package/dist/workers/entry.mjs +8 -8
  137. package/dist/{write-ZKR7SNRT.mjs → write-2DZPEQL2.mjs} +82 -107
  138. package/openapi.json +114 -6
  139. package/package.json +12 -8
@@ -1,18 +1,19 @@
1
- import { requirePatternSearchFor, requirePartialWordSearchFor, createPartitionIndex, buildPartialWordTableFrom, recordValueSize, patternIndexOf, encodeFieldIndexes, encodePartialSpellingsOf, countPartialSpelling, ARRAY_POSITION_GAP, copyVectorsInto, isCompositePartition, fnv1a, partialSpellingsOfDocuments, createFrozenPartialWords, requirePatternSearch, documentValueBytes, cloneProjected, projectIncludedPaths, valueWithOwnBuffers, createSurfaceRegistry, MAX_TERM_FREQUENCY, loadPhraseCodeFor, joinPatternWorkMeter, fanOutQuery, sanitizeGlobalStats, countsWithoutScores, fanOutMatchCount, createCompositePartition, createGeoIndex, documentWithOwnBuffers, readProjectedPath, createNumericIndex, createBooleanIndex, createEnumIndex, ValueAffixes } from './chunk-2MKX2SCO.mjs';
2
- import { generateId, getLanguage } from './chunk-ZR4VM4A6.mjs';
1
+ import { requirePatternSearchFor, requirePartialWordSearchFor, createPartitionIndex, cloneProjected, projectIncludedPaths, valueWithOwnBuffers, buildPartialWordTableFrom, createSurfaceRegistry, encodePartialSpellingsOf, recordValueSize, patternIndexOf, encodeFieldIndexes, countPartialSpelling, ARRAY_POSITION_GAP, documentWithOwnBuffers, readProjectedPath, copyVectorsInto, isCompositePartition, fnv1a, partialSpellingsOfDocuments, createFrozenPartialWords, requirePatternSearch, documentValueBytes, MAX_TERM_FREQUENCY, loadPhraseCodeFor, joinPatternWorkMeter, fanOutQuery, sanitizeGlobalStats, countsWithoutScores, fanOutMatchCount, createGeoIndex, createNumericIndex, createBooleanIndex, createEnumIndex, ValueAffixes, createCompositePartition } from './chunk-ZPWS2SKJ.mjs';
2
+ import { generateId } from './chunk-N37VMC62.mjs';
3
3
  import { readHeapStatistics, readResidentMemory } from './chunk-ZATRDAYU.mjs';
4
4
  import { GrowableUint32, bitsetHas, bitsetSet, createBitSet } from './chunk-JUKYYXSP.mjs';
5
- import { tokenizeOptions, getNestedValue, sameFields, visitDocumentSurfaceSpellings, visitDocumentSpellings } from './chunk-EUCX5K74.mjs';
6
- import { schemaFieldsOf, partialFieldsOf, encodeDocument, validateDocument, validateDocumentStrict, extractVectorFieldsFromSchema } from './chunk-PU6SKH7O.mjs';
5
+ import { getNestedValue, tokenizeOptions, sameFields, visitDocumentSurfaceSpellings, visitDocumentSpellings } from './chunk-DQNKVOJ2.mjs';
6
+ import { partialFieldsOf, schemaFieldsOf, encodeDocument, validateDocument, validateDocumentStrict, extractVectorFieldsFromSchema } from './chunk-KGOXHLLZ.mjs';
7
7
  import { fuzzyPrefixOf, boundedLevenshtein } from './chunk-GES7U2PM.mjs';
8
- import { isWordIndexedBase } from './chunk-7XY6VD7B.mjs';
9
- import { resolveIndexAnalysis, synonymSearchModule } from './chunk-6SSVIW7Y.mjs';
10
- import { visitTokens, codePointOrder, encodeStringBlob, encodeLookupKey } from './chunk-D5TFRB5G.mjs';
8
+ import { isWordIndexedBase } from './chunk-ZSWHPKGM.mjs';
9
+ import { resolveIndexAnalysis, synonymSearchModule } from './chunk-O4LJQI4G.mjs';
10
+ import { getLanguage } from './chunk-WNBRW4Q2.mjs';
11
+ import { encodeStringBlob, encodeLookupKey, visitTokens, codePointOrder } from './chunk-NPNK57CA.mjs';
11
12
  import { SEGMENT_TOKEN_TABLE_START, SEGMENT_POSTING_LOG_START, SEGMENT_POSITION_POOL_START, SEGMENT_FIELD_SCRATCH_START, FROZEN_REMOVAL_INDEX_MAX_PROCESS_FRACTION } from './chunk-4UMZ7KQL.mjs';
12
13
  import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
13
- import { loadedVectorCode, loadVectorCode, loadVectorCodeFor, requireLoadedVectorCode } from './chunk-HCGJKDZV.mjs';
14
+ import { loadedVectorCode, loadVectorCode, loadVectorCodeFor, requireLoadedVectorCode } from './chunk-GLZHNSGB.mjs';
14
15
  import { PROTOTYPE_POLLUTION_KEYS } from './chunk-L5ORED7V.mjs';
15
- import { NarsilError, ErrorCodes } from './chunk-CZ7ISLXA.mjs';
16
+ import { NarsilError, ErrorCodes } from './chunk-HNHCNTVG.mjs';
16
17
  import { Decoder, decode } from '@msgpack/msgpack';
17
18
 
18
19
  // src/core/synonyms/platform.ts
@@ -112,6 +113,36 @@ function createCapacityChecks(source) {
112
113
  };
113
114
  }
114
115
 
116
+ // src/partitioning/manager/doc-locations.ts
117
+ function documentIdsHeldInMemory(partition) {
118
+ return isCompositePartition(partition) ? partition.documentIdsHeldInMemory() : partition.docIds();
119
+ }
120
+ function rebuildDocLocations(locations, partitions) {
121
+ locations.clear();
122
+ for (let i = 0; i < partitions.length; i++) {
123
+ for (const docId of documentIdsHeldInMemory(partitions[i])) {
124
+ locations.set(docId, i);
125
+ }
126
+ }
127
+ }
128
+ function recordSegmentDocLocations(locations, partitionId, segment) {
129
+ if (segment.paged !== null) return;
130
+ for (const internalId of segment.docStore.allInternalIds()) {
131
+ const docId = segment.docStore.getExternalId(internalId);
132
+ if (docId !== void 0) locations.set(docId, partitionId);
133
+ }
134
+ }
135
+ function rebuildPartitionDocLocations(locations, partitionId, partition) {
136
+ for (const [docId, pid] of locations) {
137
+ if (pid === partitionId) {
138
+ locations.delete(docId);
139
+ }
140
+ }
141
+ for (const docId of documentIdsHeldInMemory(partition)) {
142
+ locations.set(docId, partitionId);
143
+ }
144
+ }
145
+
115
146
  // src/partitioning/manager/segments.ts
116
147
  function requireNewSegmentDocuments(segment, docPartitionMap) {
117
148
  for (const internalId of segment.docStore.allInternalIds()) {
@@ -124,6 +155,109 @@ function requireNewSegmentDocuments(segment, docPartitionMap) {
124
155
  }
125
156
  }
126
157
 
158
+ // src/partitioning/manager/write-log.ts
159
+ var REANALYSIS_WRITE_LOG = 0;
160
+ var CHECKPOINT_WRITE_LOG = 1;
161
+ function createWriteLogRecorder() {
162
+ const current = [null, null];
163
+ return {
164
+ open(owner) {
165
+ const log = { docIds: /* @__PURE__ */ new Set(), replaced: false };
166
+ current[owner] = log;
167
+ return {
168
+ get partitionsReplaced() {
169
+ return log.replaced;
170
+ },
171
+ get pendingDocuments() {
172
+ return log.docIds.size;
173
+ },
174
+ takeWrittenDocIds() {
175
+ const taken = log.docIds;
176
+ log.docIds = /* @__PURE__ */ new Set();
177
+ return taken;
178
+ },
179
+ close() {
180
+ if (current[owner] === log) current[owner] = null;
181
+ }
182
+ };
183
+ },
184
+ documentWritten(docId) {
185
+ current[0]?.docIds.add(docId);
186
+ current[1]?.docIds.add(docId);
187
+ },
188
+ segmentWritten(segment) {
189
+ for (const log of current) {
190
+ if (log === null) continue;
191
+ for (const internalId of segment.docStore.allInternalIds()) {
192
+ const docId = segment.docStore.getExternalId(internalId);
193
+ if (docId !== void 0) log.docIds.add(docId);
194
+ }
195
+ }
196
+ },
197
+ partitionsReplaced() {
198
+ for (const log of current) {
199
+ if (log !== null) log.replaced = true;
200
+ }
201
+ }
202
+ };
203
+ }
204
+
205
+ // src/partitioning/manager/segment-operations.ts
206
+ function createSegmentOperations(source) {
207
+ const { docPartitionMap, writeLogs, validatePartitionId, asCompositePartition, resolveInsertOptions } = source;
208
+ return {
209
+ mergeSegment(partitionId, payload, documents) {
210
+ validatePartitionId(partitionId);
211
+ const partition = source.partitionAt(partitionId);
212
+ partition.beginBatch();
213
+ partition.mergeSegmentPayload(payload, documents);
214
+ partition.endBatch();
215
+ for (const docId of payload.docIds) {
216
+ docPartitionMap.set(docId, partitionId);
217
+ writeLogs.documentWritten(docId);
218
+ }
219
+ },
220
+ attachFrozenSegment(partitionId, segment) {
221
+ validatePartitionId(partitionId);
222
+ if (segment.paged === null) requireNewSegmentDocuments(segment, docPartitionMap);
223
+ asCompositePartition(partitionId).attachFrozenSegment(segment);
224
+ if (segment.paged !== null) return;
225
+ recordSegmentDocLocations(docPartitionMap, partitionId, segment);
226
+ writeLogs.segmentWritten(segment);
227
+ },
228
+ openCheckpointChangeLog() {
229
+ return writeLogs.open(CHECKPOINT_WRITE_LOG);
230
+ },
231
+ diskSegmentAccess(partitionId) {
232
+ validatePartitionId(partitionId);
233
+ return {
234
+ composite: asCompositePartition(partitionId),
235
+ docPartitionMap,
236
+ insertOptions: resolveInsertOptions()
237
+ };
238
+ },
239
+ swapFrozenSegments(partitionId, dropSegmentIds, replacement, removedSpellings) {
240
+ validatePartitionId(partitionId);
241
+ const applied = asCompositePartition(partitionId).swapFrozenSegments(
242
+ dropSegmentIds,
243
+ replacement,
244
+ removedSpellings
245
+ );
246
+ recordSegmentDocLocations(docPartitionMap, partitionId, replacement);
247
+ return applied;
248
+ },
249
+ freezeLiveTail(partitionId, freeze) {
250
+ validatePartitionId(partitionId);
251
+ return asCompositePartition(partitionId).freezeLiveTail(freeze);
252
+ },
253
+ replaceLiveTail(partitionId, segment) {
254
+ validatePartitionId(partitionId);
255
+ asCompositePartition(partitionId).replaceLiveTail(segment);
256
+ recordSegmentDocLocations(docPartitionMap, partitionId, segment);
257
+ }
258
+ };
259
+ }
260
+
127
261
  // src/partitioning/manager/stats.ts
128
262
  function aggregateStatsOf(partitions) {
129
263
  let totalDocuments = 0;
@@ -159,59 +293,27 @@ function partitionStatsOf(partitions) {
159
293
  }));
160
294
  }
161
295
 
162
- // src/partitioning/manager/write-log.ts
163
- function createWriteLogRecorder() {
164
- let current = null;
165
- return {
166
- open() {
167
- const log = { docIds: /* @__PURE__ */ new Set(), replaced: false };
168
- current = log;
169
- return {
170
- get partitionsReplaced() {
171
- return log.replaced;
172
- },
173
- get pendingDocuments() {
174
- return log.docIds.size;
175
- },
176
- takeWrittenDocIds() {
177
- const taken = log.docIds;
178
- log.docIds = /* @__PURE__ */ new Set();
179
- return taken;
180
- },
181
- close() {
182
- if (current === log) current = null;
183
- }
184
- };
185
- },
186
- documentWritten(docId) {
187
- if (current !== null) current.docIds.add(docId);
188
- },
189
- segmentWritten(segment) {
190
- if (current === null) return;
191
- for (const internalId of segment.docStore.allInternalIds()) {
192
- const docId = segment.docStore.getExternalId(internalId);
193
- if (docId !== void 0) current.docIds.add(docId);
194
- }
195
- },
196
- partitionsReplaced() {
197
- if (current !== null) current.replaced = true;
198
- }
199
- };
200
- }
201
-
202
296
  // src/partitioning/manager/index.ts
203
- function createPartitionManager(indexName, config, language, router, initialPartitionCount, vectorIndexes) {
297
+ function closePagedSegmentsLeftBehind(earlier, kept) {
298
+ for (const partition of earlier) {
299
+ if (!isCompositePartition(partition) || kept.includes(partition)) continue;
300
+ for (const segment of partition.frozenSegments()) segment.paged?.close();
301
+ }
302
+ }
303
+ function createPartitionManager(indexName, initialConfig, initialLanguage, router, initialPartitionCount, vectorIndexes) {
304
+ let config = initialConfig;
305
+ let language = initialLanguage;
204
306
  requirePatternSearchFor(config.schema);
205
307
  requirePartialWordSearchFor(config.schema);
206
308
  const count = initialPartitionCount ?? 1;
207
- const analysis = resolveIndexAnalysis(config);
208
- const partialWordAnalysis = resolvePartialWordAnalysis(config, language, analysis);
209
- const textAnalysis = { language, options: resolvePartitionInsertOptions(config, analysis) };
309
+ let analysis = resolveIndexAnalysis(config);
310
+ let partialWordAnalysis = resolvePartialWordAnalysis(config, language, analysis);
311
+ let textAnalysis = { language, options: resolvePartitionInsertOptions(config, analysis) };
210
312
  const trackPositions = config.trackPositions ?? true;
211
313
  const vecIndexes = vectorIndexes ?? /* @__PURE__ */ new Map();
212
314
  let partitions = [];
213
315
  const docPartitionMap = /* @__PURE__ */ new Map();
214
- const writeLog = createWriteLogRecorder();
316
+ const writeLogs = createWriteLogRecorder();
215
317
  for (let i = 0; i < count; i++) {
216
318
  partitions.push(createPartitionIndex(i, trackPositions));
217
319
  }
@@ -231,7 +333,14 @@ function createPartitionManager(indexName, config, language, router, initialPart
231
333
  if (partialWordAnalysis !== null) partitions[partitionId].preparePartialWords(partialWordAnalysis);
232
334
  }
233
335
  function locateDocument(docId) {
234
- return docPartitionMap.get(docId);
336
+ const known = docPartitionMap.get(docId);
337
+ if (known !== void 0 || config.textStorage !== "disk") return known;
338
+ const routed = router.route(docId, partitions.length);
339
+ if (partitions[routed]?.has(docId)) return routed;
340
+ for (let i = 0; i < partitions.length; i++) {
341
+ if (i !== routed && partitions[i].has(docId)) return i;
342
+ }
343
+ return void 0;
235
344
  }
236
345
  function asCompositePartition(partitionId) {
237
346
  const existing = partitions[partitionId];
@@ -240,30 +349,6 @@ function createPartitionManager(indexName, config, language, router, initialPart
240
349
  partitions[partitionId] = composite;
241
350
  return composite;
242
351
  }
243
- function rebuildDocPartitionMap() {
244
- docPartitionMap.clear();
245
- for (let i = 0; i < partitions.length; i++) {
246
- for (const docId of partitions[i].docIds()) {
247
- docPartitionMap.set(docId, i);
248
- }
249
- }
250
- }
251
- function registerSegmentDocuments(partitionId, segment) {
252
- for (const internalId of segment.docStore.allInternalIds()) {
253
- const docId = segment.docStore.getExternalId(internalId);
254
- if (docId !== void 0) docPartitionMap.set(docId, partitionId);
255
- }
256
- }
257
- function rebuildDocPartitionMapForPartition(partitionId) {
258
- for (const [docId, pid] of docPartitionMap) {
259
- if (pid === partitionId) {
260
- docPartitionMap.delete(docId);
261
- }
262
- }
263
- for (const docId of partitions[partitionId].docIds()) {
264
- docPartitionMap.set(docId, partitionId);
265
- }
266
- }
267
352
  const manager = {
268
353
  get partitionCount() {
269
354
  return partitions.length;
@@ -301,12 +386,14 @@ function createPartitionManager(indexName, config, language, router, initialPart
301
386
  return [...partitions];
302
387
  },
303
388
  setPartitions(newPartitions) {
304
- writeLog.partitionsReplaced();
389
+ writeLogs.partitionsReplaced();
390
+ const earlier = partitions;
305
391
  partitions = [...newPartitions];
306
- rebuildDocPartitionMap();
392
+ closePagedSegmentsLeftBehind(earlier, partitions);
393
+ rebuildDocLocations(docPartitionMap, partitions);
307
394
  },
308
395
  openWriteLog() {
309
- return writeLog.open();
396
+ return writeLogs.open(REANALYSIS_WRITE_LOG);
310
397
  },
311
398
  adoptRebuiltPartitions(rebuilt) {
312
399
  if (rebuilt.length !== partitions.length) {
@@ -316,7 +403,17 @@ function createPartitionManager(indexName, config, language, router, initialPart
316
403
  { indexName, partitionCount: partitions.length, rebuiltCount: rebuilt.length }
317
404
  );
318
405
  }
406
+ const earlier = partitions;
319
407
  partitions = [...rebuilt];
408
+ closePagedSegmentsLeftBehind(earlier, partitions);
409
+ },
410
+ adoptAnalysis(change) {
411
+ config = change.config;
412
+ language = change.language;
413
+ analysis = change.analysis;
414
+ partialWordAnalysis = change.partialWordAnalysis;
415
+ textAnalysis = { language, options: resolvePartitionInsertOptions(config, analysis) };
416
+ manager.synonyms = change.synonyms;
320
417
  },
321
418
  addPartition() {
322
419
  const currentMaxPartitions = config.partitions?.maxPartitions;
@@ -328,15 +425,15 @@ function createPartitionManager(indexName, config, language, router, initialPart
328
425
  );
329
426
  }
330
427
  const partition = createPartitionIndex(partitions.length, trackPositions);
331
- writeLog.partitionsReplaced();
428
+ writeLogs.partitionsReplaced();
332
429
  partitions.push(partition);
333
430
  return partition;
334
431
  },
335
432
  removePartition(partitionId) {
336
433
  validatePartitionId(partitionId);
337
- writeLog.partitionsReplaced();
434
+ writeLogs.partitionsReplaced();
338
435
  partitions.splice(partitionId, 1);
339
- rebuildDocPartitionMap();
436
+ rebuildDocLocations(docPartitionMap, partitions);
340
437
  },
341
438
  trimPartitions(newCount) {
342
439
  if (newCount < 1) {
@@ -348,7 +445,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
348
445
  if (newCount >= partitions.length) {
349
446
  return;
350
447
  }
351
- writeLog.partitionsReplaced();
448
+ writeLogs.partitionsReplaced();
352
449
  for (const [docId, pid] of docPartitionMap) {
353
450
  if (pid >= newCount) {
354
451
  docPartitionMap.delete(docId);
@@ -358,7 +455,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
358
455
  },
359
456
  ...createCapacityChecks({
360
457
  indexName,
361
- config,
458
+ config: initialConfig,
362
459
  partitionCount: () => partitions.length,
363
460
  documentCount: () => manager.countDocuments(),
364
461
  getPartition: (partitionId) => manager.getPartition(partitionId)
@@ -371,7 +468,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
371
468
  const insertOpts = resolveInsertOptions(options);
372
469
  partitions[pid].insert(docId, document, config.schema, language, insertOpts);
373
470
  docPartitionMap.set(docId, pid);
374
- writeLog.documentWritten(docId);
471
+ writeLogs.documentWritten(docId);
375
472
  },
376
473
  removedSpellingsOf(docId) {
377
474
  const pid = locateDocument(docId);
@@ -385,7 +482,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
385
482
  }
386
483
  partitions[pid].remove(docId, config.schema, language, resolveInsertOptions(), removedSpellings);
387
484
  docPartitionMap.delete(docId);
388
- writeLog.documentWritten(docId);
485
+ writeLogs.documentWritten(docId);
389
486
  },
390
487
  beginBatchRemove() {
391
488
  for (let i = 0; i < partitions.length; i++) {
@@ -405,7 +502,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
405
502
  const updateOpts = resolveInsertOptions(options);
406
503
  partitions[pid].update(docId, document, config.schema, language, updateOpts, removedSpellings);
407
504
  docPartitionMap.set(docId, pid);
408
- writeLog.documentWritten(docId);
505
+ writeLogs.documentWritten(docId);
409
506
  },
410
507
  get(docId, projection) {
411
508
  const pid = locateDocument(docId);
@@ -445,49 +542,20 @@ function createPartitionManager(indexName, config, language, router, initialPart
445
542
  validatePartitionId(partitionId);
446
543
  return partitions[partitionId].serialize(indexName, partitions.length, language.name, config.schema);
447
544
  },
448
- mergeSegment(partitionId, payload, documents) {
449
- validatePartitionId(partitionId);
450
- const partition = partitions[partitionId];
451
- partition.beginBatch();
452
- partition.mergeSegmentPayload(payload, documents);
453
- partition.endBatch();
454
- for (const docId of payload.docIds) {
455
- docPartitionMap.set(docId, partitionId);
456
- writeLog.documentWritten(docId);
457
- }
458
- },
459
- attachFrozenSegment(partitionId, segment) {
460
- validatePartitionId(partitionId);
461
- requireNewSegmentDocuments(segment, docPartitionMap);
462
- asCompositePartition(partitionId).attachFrozenSegment(segment);
463
- registerSegmentDocuments(partitionId, segment);
464
- writeLog.segmentWritten(segment);
465
- },
466
- swapFrozenSegments(partitionId, dropSegmentIds, replacement, removedSpellings) {
467
- validatePartitionId(partitionId);
468
- const applied = asCompositePartition(partitionId).swapFrozenSegments(
469
- dropSegmentIds,
470
- replacement,
471
- removedSpellings
472
- );
473
- registerSegmentDocuments(partitionId, replacement);
474
- return applied;
475
- },
476
- freezeLiveTail(partitionId, freeze) {
477
- validatePartitionId(partitionId);
478
- return asCompositePartition(partitionId).freezeLiveTail(freeze);
479
- },
480
- replaceLiveTail(partitionId, segment) {
481
- validatePartitionId(partitionId);
482
- asCompositePartition(partitionId).replaceLiveTail(segment);
483
- registerSegmentDocuments(partitionId, segment);
484
- },
545
+ ...createSegmentOperations({
546
+ docPartitionMap,
547
+ writeLogs,
548
+ partitionAt: (partitionId) => partitions[partitionId],
549
+ validatePartitionId,
550
+ asCompositePartition,
551
+ resolveInsertOptions: () => resolveInsertOptions()
552
+ }),
485
553
  deserializePartition(partitionId, data) {
486
554
  validatePartitionId(partitionId);
487
- writeLog.partitionsReplaced();
555
+ writeLogs.partitionsReplaced();
488
556
  partitions[partitionId].deserialize(data, config.schema);
489
557
  preparePartialWords(partitionId);
490
- rebuildDocPartitionMapForPartition(partitionId);
558
+ rebuildPartitionDocLocations(docPartitionMap, partitionId, partitions[partitionId]);
491
559
  },
492
560
  getAggregateStats() {
493
561
  return aggregateStatsOf(partitions);
@@ -961,11 +1029,17 @@ function encodeDocumentTableData(documents) {
961
1029
  }
962
1030
  function wrapEncodedDocumentTable(data) {
963
1031
  const { blob, offsets } = data;
964
- const count = offsets.length - 1;
965
- const discarded = (data.discardedFields ?? []).map((fieldPath) => fieldPath.split("."));
1032
+ return createEncodedDocumentSource(
1033
+ offsets.length - 1,
1034
+ blob.byteLength + offsets.byteLength,
1035
+ (ordinal) => blob.subarray(offsets[ordinal], offsets[ordinal + 1]),
1036
+ data.discardedFields
1037
+ );
1038
+ }
1039
+ function createEncodedDocumentSource(count, byteLength, bytesAt, discardedFields) {
1040
+ const discarded = (discardedFields ?? []).map((fieldPath) => fieldPath.split("."));
966
1041
  const touchesDiscarded = (path) => discarded.some((discardedPath) => sharesPrefix(discardedPath, path));
967
- const bytesAt = (ordinal) => blob.subarray(offsets[ordinal], offsets[ordinal + 1]);
968
- const docAt = (ordinal) => documentWithOwnBuffers(decodeStoredDocument(bytesAt(ordinal), data.discardedFields));
1042
+ const docAt = (ordinal) => documentWithOwnBuffers(decodeStoredDocument(bytesAt(ordinal), discardedFields));
969
1043
  function projectedPathAt(ordinal, path) {
970
1044
  const segments = encodedProjectionPathOf(path);
971
1045
  if (segments === null) return { found: false, value: void 0 };
@@ -975,10 +1049,10 @@ function wrapEncodedDocumentTable(data) {
975
1049
  }
976
1050
  return {
977
1051
  count,
978
- byteLength: blob.byteLength + offsets.byteLength,
1052
+ byteLength,
979
1053
  docAt,
980
1054
  readAt(ordinal) {
981
- return decodeStoredDocument(bytesAt(ordinal), data.discardedFields);
1055
+ return decodeStoredDocument(bytesAt(ordinal), discardedFields);
982
1056
  },
983
1057
  fieldReader(fieldPath) {
984
1058
  const path = fieldPath.split(".");
@@ -1071,6 +1145,9 @@ function wrapExternalIdTable(data) {
1071
1145
  const ordinal = sortedOrdinals[lo];
1072
1146
  return compareBytes(blob, offsets[ordinal], offsets[ordinal + 1], query) === 0 ? ordinal : -1;
1073
1147
  },
1148
+ sortedOrdinalAt(position) {
1149
+ return sortedOrdinals[position];
1150
+ },
1074
1151
  collectSortedIds(excluded) {
1075
1152
  const ids = [];
1076
1153
  for (let position = 0; position < count; position++) {
@@ -1090,6 +1167,7 @@ function buildExternalIdTable(docIds) {
1090
1167
  characters += docIds[ordinal].length;
1091
1168
  }
1092
1169
  let sortedIds = null;
1170
+ let sortedOrdinals = null;
1093
1171
  return {
1094
1172
  count: docIds.length,
1095
1173
  bytes: characters,
@@ -1099,6 +1177,10 @@ function buildExternalIdTable(docIds) {
1099
1177
  ordinalOf(docId) {
1100
1178
  return ordinals.get(docId) ?? -1;
1101
1179
  },
1180
+ sortedOrdinalAt(position) {
1181
+ if (sortedOrdinals === null) sortedOrdinals = codePointOrder(docIds);
1182
+ return sortedOrdinals[position];
1183
+ },
1102
1184
  collectSortedIds(excluded) {
1103
1185
  if (sortedIds === null) {
1104
1186
  const ids = [...docIds];
@@ -1637,6 +1719,10 @@ function createFrozenRemovalPass(documents, surfaces) {
1637
1719
  });
1638
1720
  return spellings;
1639
1721
  }
1722
+ function subtract(ordinal, analysis) {
1723
+ if (surfaces.tableStaysOnDisk()) surfaces.subtractOnceLoaded(ordinal, analysis);
1724
+ else surfaces.subtractSpellings(ordinal, heldSpellingsOf(ordinal, analysis));
1725
+ }
1640
1726
  return {
1641
1727
  get settled() {
1642
1728
  return pending.length === 0;
@@ -1645,17 +1731,17 @@ function createFrozenRemovalPass(documents, surfaces) {
1645
1731
  if (bound !== null) return;
1646
1732
  const scoped = { ...analysis, fields: documents.textFields };
1647
1733
  bound = scoped;
1648
- for (const ordinal of pending) surfaces.subtractSpellings(ordinal, heldSpellingsOf(ordinal, scoped));
1734
+ for (const ordinal of pending) subtract(ordinal, scoped);
1649
1735
  pending = [];
1650
1736
  },
1651
1737
  spellingsOf(ordinal) {
1652
- return bound === null ? void 0 : heldSpellingsOf(ordinal, bound);
1738
+ return bound === null || surfaces.tableStaysOnDisk() ? void 0 : heldSpellingsOf(ordinal, bound);
1653
1739
  },
1654
1740
  afterRemoval(ordinal, spellings) {
1655
1741
  if (spellings !== void 0) {
1656
1742
  surfaces.subtractSpellings(ordinal, spellings);
1657
1743
  } else if (bound !== null) {
1658
- surfaces.subtractSpellings(ordinal, heldSpellingsOf(ordinal, bound));
1744
+ subtract(ordinal, bound);
1659
1745
  } else {
1660
1746
  pending.push(ordinal);
1661
1747
  }
@@ -1679,15 +1765,32 @@ function addForms(target, forms) {
1679
1765
  if (Array.isArray(value)) target.add(surface, value[1], value[0]);
1680
1766
  }
1681
1767
  }
1682
- function createFrozenSurfaceSlot(load, documents) {
1768
+ function createFrozenSurfaceSlot(load, documents, tableOnDisk) {
1683
1769
  let registry = null;
1684
1770
  let uncounted = null;
1771
+ let table = tableOnDisk;
1772
+ let waitingRemovals = [];
1773
+ let waitingAnalysis = null;
1774
+ function subtractWaitingRemovals(target) {
1775
+ if (waitingAnalysis === null || target.size() === 0) return;
1776
+ for (const ordinal of waitingRemovals) {
1777
+ const spellings = [];
1778
+ visitDocumentSurfaceSpellings(documents.documentAt(ordinal), waitingAnalysis, (spelling) => {
1779
+ if (target.holds(spelling)) spellings.push(spelling);
1780
+ });
1781
+ for (const spelling of spellings) target.subtract(spelling, 1);
1782
+ }
1783
+ }
1685
1784
  function loaded() {
1686
1785
  if (registry === null) {
1687
1786
  const built = createSurfaceRegistry();
1688
1787
  const forms = load();
1689
1788
  if (forms !== null) built.deserialize(forms);
1789
+ subtractWaitingRemovals(built);
1690
1790
  registry = built;
1791
+ table = void 0;
1792
+ waitingRemovals = [];
1793
+ waitingAnalysis = null;
1691
1794
  }
1692
1795
  return registry;
1693
1796
  }
@@ -1695,12 +1798,22 @@ function createFrozenSurfaceSlot(load, documents) {
1695
1798
  return uncounted === null || uncounted[ordinal] === 0;
1696
1799
  }
1697
1800
  return {
1801
+ heldBytes: () => table?.heldBytes() ?? 0,
1698
1802
  reader: {
1699
- candidatesForPrefix: (prefix) => loaded().candidatesForPrefix(prefix),
1803
+ candidatesForPrefix: (prefix) => table !== void 0 && waitingRemovals.length === 0 ? table.candidatesForPrefix(prefix) : loaded().candidatesForPrefix(prefix),
1700
1804
  stemChangedTotalFor: (token) => loaded().stemChangedTotalFor(token),
1701
1805
  size: () => loaded().size(),
1702
1806
  serialize: () => loaded().serialize()
1703
1807
  },
1808
+ tableStaysOnDisk() {
1809
+ return registry === null && tableOnDisk !== void 0;
1810
+ },
1811
+ subtractOnceLoaded(ordinal, analysis) {
1812
+ uncounted ??= new Uint8Array(documents.count);
1813
+ uncounted[ordinal] = 1;
1814
+ waitingAnalysis = analysis;
1815
+ waitingRemovals.push(ordinal);
1816
+ },
1704
1817
  holdsAny() {
1705
1818
  return loaded().size() > 0;
1706
1819
  },
@@ -1848,6 +1961,9 @@ function wrapFrozenTokenTable(data) {
1848
1961
  }
1849
1962
  return token;
1850
1963
  },
1964
+ tokenBytesAt(sortedIndex) {
1965
+ return blob.subarray(offsets[sortedIndex], offsets[sortedIndex + 1]);
1966
+ },
1851
1967
  byteLengthAt(sortedIndex) {
1852
1968
  return offsets[sortedIndex + 1] - offsets[sortedIndex];
1853
1969
  },
@@ -2122,10 +2238,10 @@ function buildGeoReaders(entries) {
2122
2238
  }
2123
2239
  return readers;
2124
2240
  }
2125
- function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot, partialWordTable, survivingSurfaceForms) {
2241
+ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot, partialWordTable, survivingSurfaceForms, pagedParts) {
2126
2242
  const tombstones = createFrozenTombstones();
2127
- const counts = createFrozenLiveCounts(source, source.tokenTable, tombstones, storedFrequenciesOf(source));
2128
- const postingViews = createFrozenPostingViews(source, tombstones, counts);
2243
+ const counts = pagedParts?.createCounts(tombstones) ?? createFrozenLiveCounts(source, source.tokenTable, tombstones, storedFrequenciesOf(source));
2244
+ const postingViews = pagedParts?.createViews(tombstones, counts) ?? createFrozenPostingViews(source, tombstones, counts);
2129
2245
  const lengthColumns = /* @__PURE__ */ new Map();
2130
2246
  for (let i = 0; i < source.fieldLengthNames.length; i++) {
2131
2247
  lengthColumns.set(source.fieldLengthNames[i], source.fieldLengthColumns[i]);
@@ -2142,7 +2258,11 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
2142
2258
  documentAt: (ordinal) => documentSource.readAt(ordinal)
2143
2259
  };
2144
2260
  const loadSurfaceForms = survivingSurfaceForms === void 0 ? source.readSurfaceForms : () => survivingSurfaceForms;
2145
- const surfaces = createFrozenSurfaceSlot(loadSurfaceForms, textDocuments);
2261
+ const surfaces = createFrozenSurfaceSlot(
2262
+ loadSurfaceForms,
2263
+ textDocuments,
2264
+ survivingSurfaceForms === void 0 ? source.surfaceTableOnDisk : void 0
2265
+ );
2146
2266
  const removals = createFrozenRemovalPass(textDocuments, surfaces);
2147
2267
  const numericIndexes = /* @__PURE__ */ new Map();
2148
2268
  for (const entry of source.numeric) numericIndexes.set(entry.fieldPath, createFrozenNumericReader(entry));
@@ -2164,6 +2284,7 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
2164
2284
  segmentId,
2165
2285
  documentSource,
2166
2286
  sharedSnapshot,
2287
+ paged: pagedParts?.access ?? null,
2167
2288
  arrays: source,
2168
2289
  invertedIdx: createFrozenInvertedReader(source.tokenTable, postingViews, counts),
2169
2290
  docStore,
@@ -2179,7 +2300,7 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
2179
2300
  sizeIndexes,
2180
2301
  fieldNameTable,
2181
2302
  partialWords,
2182
- trackPositions: source.positionOffsets !== null,
2303
+ trackPositions: pagedParts?.access.snapshot.tracksPositions ?? source.positionOffsets !== null,
2183
2304
  flatSchemaCache: null,
2184
2305
  lastSchemaRef: null,
2185
2306
  sortColumns: null,
@@ -2223,6 +2344,9 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
2223
2344
  };
2224
2345
  return segment;
2225
2346
  }
2347
+ function assemblePagedFrozenSegment(segmentId, source, documentSource, partialWordTable, parts) {
2348
+ return assembleFrozenSegment(segmentId, source, documentSource, null, partialWordTable, void 0, parts);
2349
+ }
2226
2350
  function createFrozenSegment(payload, documents, segmentId) {
2227
2351
  const tokenTable = buildFrozenTokenTable(payload.tokens, payload.docFrequencies);
2228
2352
  return assembleFrozenSegment(
@@ -3086,7 +3210,20 @@ function buildSegmentPayload(documents, schema, language, options, trackPosition
3086
3210
 
3087
3211
  // src/workers/segment-actions.ts
3088
3212
  function isSegmentAction(action) {
3089
- return action.type === "buildSegment" || action.type === "mergeSegments" || action.type === "attachSegments" || action.type === "compactSegments" || action.type === "swapSegments" || action.type === "freezeLiveTail";
3213
+ return action.type === "buildSegment" || action.type === "mergeSegments" || action.type === "attachSegments" || action.type === "compactSegments" || action.type === "swapSegments" || action.type === "freezeLiveTail" || action.type === "adoptDiskSegments";
3214
+ }
3215
+ function isPagedSnapshot(snapshot) {
3216
+ return "path" in snapshot;
3217
+ }
3218
+ async function attachSegments(entry, action) {
3219
+ for (const segment of action.segments) {
3220
+ growPartitionsTo(entry.manager, segment.partitionId);
3221
+ const surfaceForms = segment.survivingSurfaceForms;
3222
+ const spellings = surfaceForms === void 0 ? void 0 : NO_REMOVED_SPELLINGS;
3223
+ const frozen = isPagedSnapshot(segment.snapshot) ? (await import('./module-IYPLWHGC.mjs')).createPagedFrozenSegment(segment.snapshot) : createSharedFrozenSegment(segment.snapshot, surfaceForms);
3224
+ for (const docId of segment.tombstonedDocIds ?? []) frozen.tombstoneDocument(docId, spellings);
3225
+ entry.manager.attachFrozenSegment(segment.partitionId, frozen);
3226
+ }
3090
3227
  }
3091
3228
  function growPartitionsTo(manager, partitionId) {
3092
3229
  while (manager.partitionCount <= partitionId) {
@@ -3135,17 +3272,10 @@ function runSegmentAction(entry, action) {
3135
3272
  }
3136
3273
  return void 0;
3137
3274
  }
3138
- case "attachSegments": {
3139
- for (const segment of action.segments) {
3140
- growPartitionsTo(entry.manager, segment.partitionId);
3141
- const surfaceForms = segment.survivingSurfaceForms;
3142
- const frozen = createSharedFrozenSegment(segment.snapshot, surfaceForms);
3143
- const spellings = surfaceForms === void 0 ? void 0 : NO_REMOVED_SPELLINGS;
3144
- for (const docId of segment.tombstonedDocIds ?? []) frozen.tombstoneDocument(docId, spellings);
3145
- entry.manager.attachFrozenSegment(segment.partitionId, frozen);
3146
- }
3147
- return void 0;
3148
- }
3275
+ case "attachSegments":
3276
+ return attachSegments(entry, action);
3277
+ case "adoptDiskSegments":
3278
+ return import('./module-IYPLWHGC.mjs').then((code) => code.adoptSegmentsOnWorker(entry.manager, action));
3149
3279
  case "compactSegments": {
3150
3280
  const partition = requireComposite(entry, action.indexName, action.partitionId, "compact");
3151
3281
  return mergeFrozenSegments(
@@ -3247,6 +3377,34 @@ function heldVectorOf(held, fieldName, docId) {
3247
3377
  return held.fields.get(fieldName)?.view.vectorOf(docId);
3248
3378
  }
3249
3379
 
3380
+ // src/workers/direct-executor/entry.ts
3381
+ function indexEntryOf(manager, vectorIndexes, analysisStale, awaitedCopyLoad) {
3382
+ return {
3383
+ manager,
3384
+ get config() {
3385
+ return manager.config;
3386
+ },
3387
+ get language() {
3388
+ return manager.language;
3389
+ },
3390
+ searchOptions: {
3391
+ get stopWords() {
3392
+ return manager.analysis.stopWords;
3393
+ },
3394
+ get customTokenizer() {
3395
+ return manager.analysis.customTokenizer;
3396
+ },
3397
+ get synonyms() {
3398
+ return manager.synonyms;
3399
+ }
3400
+ },
3401
+ vectorIndexes,
3402
+ vectorCopies: createHeldVectorCopies(),
3403
+ analysisStale,
3404
+ awaitedCopyLoad
3405
+ };
3406
+ }
3407
+
3250
3408
  // src/workers/direct-executor/index.ts
3251
3409
  var NO_VECTOR_WORKER_COPIES = { enabled: false };
3252
3410
  function createDirectExecutor(options) {
@@ -3298,22 +3456,7 @@ function createDirectExecutor(options) {
3298
3456
  const partitionCount = config.partitions?.maxPartitions ?? 1;
3299
3457
  const vectorIndexes = vectorIndexesFor(indexName, config, copies);
3300
3458
  const manager = createPartitionManager(indexName, config, language, router, partitionCount, vectorIndexes);
3301
- return {
3302
- manager,
3303
- config,
3304
- language,
3305
- searchOptions: {
3306
- stopWords: manager.analysis.stopWords,
3307
- customTokenizer: manager.analysis.customTokenizer,
3308
- get synonyms() {
3309
- return manager.synonyms;
3310
- }
3311
- },
3312
- vectorIndexes,
3313
- vectorCopies: createHeldVectorCopies(),
3314
- analysisStale,
3315
- awaitedCopyLoad
3316
- };
3459
+ return indexEntryOf(manager, vectorIndexes, analysisStale, awaitedCopyLoad);
3317
3460
  }
3318
3461
  function disposeEntry(entry) {
3319
3462
  for (const vectorIndex of entry.vectorIndexes.values()) {
@@ -3534,4 +3677,4 @@ function createDirectExecutor(options) {
3534
3677
  };
3535
3678
  }
3536
3679
 
3537
- export { availableSynonymModule, buildSegmentPayload, createDirectExecutor, createFrozenSegment, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, decodeEveryStoredDocument, decodeMapWithoutFields, decodeStoredDocument, freezeSegmentShared, isEmptySynonymSet, mergeFrozenSegments, mergeSegmentContents, registerSynonyms, resolvePartitionInsertOptions, segmentArraysOf, selectionKeeps, shareSegmentArrays, synonymModule };
3680
+ export { assemblePagedFrozenSegment, availableSynonymModule, buildSegmentPayload, createDirectExecutor, createEncodedDocumentSource, createFrozenSegment, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, decodeEveryStoredDocument, decodeMapWithoutFields, decodeStoredDocument, decodeSurfaceTable, encodeSurfaceTable, endOfValue, fieldIndexRemapOf, freezeSegmentShared, holdsADiscardedField, isEmptySynonymSet, mergeFieldIndexes, mergeFieldLengths, mergeFrozenSegments, mergePartialWords, mergeSegmentContents, mergeSurfaceForms, registerSynonyms, resolvePartialWordAnalysis, resolvePartitionInsertOptions, segmentArraysOf, selectionKeeps, shareSegmentArrays, synonymModule, wrapFrozenTokenTable };