@delali/narsil 0.3.1-next.131 → 0.3.1-next.141

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. package/dist/adapters/filesystem.mjs +1 -1
  2. package/dist/adapters/indexeddb.mjs +1 -1
  3. package/dist/analysis-change-HQUT2Z3I.mjs +43 -0
  4. package/dist/analysis-change-entry-TZTG4AZB.mjs +1 -0
  5. package/dist/browser/analysis-change-JPV775LR.mjs +900 -0
  6. package/dist/browser/analysis-change-entry-7HR4LWL3.mjs +1 -0
  7. package/dist/browser/{chunk-UDGB2MWT.mjs → chunk-35B42UFM.mjs} +4 -0
  8. package/dist/browser/chunk-3ZBJYAS4.mjs +19 -0
  9. package/dist/browser/{chunk-2Y3CO4EQ.mjs → chunk-5DB2VDGI.mjs} +1149 -1853
  10. package/dist/browser/chunk-6RHTAVZX.mjs +656 -0
  11. package/dist/browser/chunk-CCOAOO73.mjs +86 -0
  12. package/dist/browser/{chunk-AIIJDWMG.mjs → chunk-D33A5VMU.mjs} +16 -16
  13. package/dist/browser/{chunk-VBBLTD7O.mjs → chunk-HKX2G6WH.mjs} +1520 -2159
  14. package/dist/browser/{chunk-NSCSR6MA.mjs → chunk-KTOK275N.mjs} +606 -830
  15. package/dist/browser/{chunk-D7WQF6DV.mjs → chunk-LVY2LI35.mjs} +2 -2
  16. package/dist/browser/chunk-MVM33ZUL.mjs +993 -0
  17. package/dist/browser/chunk-MWUK3DA5.mjs +2634 -0
  18. package/dist/browser/{chunk-VEGHFP2P.mjs → chunk-MZI3TEOW.mjs} +4 -2
  19. package/dist/browser/chunk-OIZS5KIX.mjs +6 -0
  20. package/dist/browser/chunk-R4WUDZQV.mjs +176 -0
  21. package/dist/browser/{chunk-M5UAAXGU.mjs → chunk-TCLEOZ4P.mjs} +3 -3
  22. package/dist/browser/chunk-VFYMPCOV.mjs +212 -0
  23. package/dist/browser/chunk-WCFCHIBB.mjs +177 -0
  24. package/dist/browser/chunk-XNWPP5NS.mjs +21 -0
  25. package/dist/browser/chunk-ZEV26AQ6.mjs +270 -0
  26. package/dist/browser/cold-module-MJMKNUYJ.mjs +745 -0
  27. package/dist/browser/{durability-wiring-FNZSRH4N.mjs → durability-wiring-SNOWIT6Z.mjs} +370 -95
  28. package/dist/browser/import-TLB2A5LJ.mjs +9 -0
  29. package/dist/browser/module-BUFNZMEU.mjs +12 -0
  30. package/dist/browser/{module-4HIMOQM5.mjs → module-BVP4TKZU.mjs} +397 -109
  31. package/dist/browser/{phrase-module-WESYVU7K.mjs → phrase-module-HGVBO24W.mjs} +1 -1
  32. package/dist/{restore-queue-JAMNRC6T.mjs → browser/restore-queue-ONYNYUDE.mjs} +8 -2
  33. package/dist/browser/{runtime-3X7ZJ5T3.mjs → runtime-VEUH674B.mjs} +2 -2
  34. package/dist/browser/{snapshot-V7VF6BN6.mjs → snapshot-QXBKHJAG.mjs} +135 -183
  35. package/dist/browser/{write-YIC2MKRY.mjs → write-WHFBDWQS.mjs} +64 -90
  36. package/dist/{capabilities-B6WxrWxt.d.ts → capabilities-BCdUwK_L.d.ts} +10 -3
  37. package/dist/chunk-46NZOPHS.mjs +21 -0
  38. package/dist/{chunk-JT3YZGCE.mjs → chunk-5WYVUPGD.mjs} +3 -188
  39. package/dist/chunk-6SNBQESJ.mjs +214 -0
  40. package/dist/{chunk-2C3WXBQS.mjs → chunk-74XB2SXI.mjs} +1 -1
  41. package/dist/chunk-7AUASRE7.mjs +653 -0
  42. package/dist/{chunk-NMXRXFGZ.mjs → chunk-AST3SHFA.mjs} +27 -25
  43. package/dist/chunk-BOJZGUS6.mjs +177 -0
  44. package/dist/{chunk-T4OFG4IQ.mjs → chunk-BRUUBXCB.mjs} +92 -17
  45. package/dist/{chunk-5LX6MOIB.mjs → chunk-C3YDYX6C.mjs} +1 -1
  46. package/dist/chunk-CZQ33DLW.mjs +78 -0
  47. package/dist/{chunk-4GDLFDPZ.mjs → chunk-DD2673TX.mjs} +3 -2
  48. package/dist/{chunk-EUCX5K74.mjs → chunk-DQNKVOJ2.mjs} +65 -65
  49. package/dist/{chunk-RNSC4NFI.mjs → chunk-EEBCDP36.mjs} +3 -1
  50. package/dist/chunk-FOCV4CBZ.mjs +87 -0
  51. package/dist/{chunk-HCGJKDZV.mjs → chunk-GLZHNSGB.mjs} +1 -1
  52. package/dist/chunk-GWOS5DYC.mjs +2637 -0
  53. package/dist/{chunk-PATMDIMN.mjs → chunk-HJJWY6PT.mjs} +1 -1
  54. package/dist/{chunk-CZ7ISLXA.mjs → chunk-HNHCNTVG.mjs} +4 -0
  55. package/dist/{chunk-Y2U5VVWX.mjs → chunk-ICW45TTU.mjs} +1 -1
  56. package/dist/{chunk-TGHNXN7W.mjs → chunk-IO5F7ZBE.mjs} +261 -948
  57. package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
  58. package/dist/{chunk-7OIQ2SIO.mjs → chunk-KEIYVZUX.mjs} +5 -5
  59. package/dist/{chunk-PU6SKH7O.mjs → chunk-KGOXHLLZ.mjs} +7 -3
  60. package/dist/{chunk-WVRDRLIC.mjs → chunk-KZL72VTC.mjs} +70 -18
  61. package/dist/chunk-L76W3IZJ.mjs +193 -0
  62. package/dist/{chunk-2JWNJSNE.mjs → chunk-LA4WNDHL.mjs} +88 -801
  63. package/dist/{chunk-ZR4VM4A6.mjs → chunk-N37VMC62.mjs} +1 -18
  64. package/dist/{chunk-JJ6EUMXB.mjs → chunk-NJJCPFXN.mjs} +2 -2
  65. package/dist/chunk-NN33CZRE.mjs +86 -0
  66. package/dist/{chunk-2WBY4Y4M.mjs → chunk-NOT5QOHU.mjs} +1 -1
  67. package/dist/{chunk-D5TFRB5G.mjs → chunk-NPNK57CA.mjs} +1 -1
  68. package/dist/chunk-NRUXBYPH.mjs +121 -0
  69. package/dist/{chunk-6SSVIW7Y.mjs → chunk-O4LJQI4G.mjs} +31 -5
  70. package/dist/{chunk-UJWGQIXU.mjs → chunk-QHPYS427.mjs} +1 -0
  71. package/dist/{chunk-F6AAXI7T.mjs → chunk-RHKRTYGO.mjs} +13 -4
  72. package/dist/{chunk-2SEL6SBS.mjs → chunk-RKAVHNRW.mjs} +2 -2
  73. package/dist/{chunk-RCMNL7RI.mjs → chunk-SBMAWTRG.mjs} +41 -4
  74. package/dist/chunk-SRJBFG2T.mjs +881 -0
  75. package/dist/{chunk-X5CNUKD5.mjs → chunk-TNBIK2LQ.mjs} +1663 -1652
  76. package/dist/{chunk-NNO7CO3A.mjs → chunk-UIKN2OTQ.mjs} +1 -1
  77. package/dist/chunk-UUZCQQG2.mjs +209 -0
  78. package/dist/{chunk-6M6SDH3S.mjs → chunk-VEFN6X6Q.mjs} +8 -8
  79. package/dist/chunk-VJEGOELZ.mjs +177 -0
  80. package/dist/chunk-W2COJ5AH.mjs +148 -0
  81. package/dist/{chunk-RWESFSJI.mjs → chunk-WJGO5PGV.mjs} +317 -174
  82. package/dist/{chunk-7X5JST5L.mjs → chunk-WNBRW4Q2.mjs} +16 -2
  83. package/dist/{chunk-SZGRYEIC.mjs → chunk-XS2VM7UE.mjs} +3 -3
  84. package/dist/{chunk-Y4BVTQ53.mjs → chunk-XZ3SRNDJ.mjs} +25 -64
  85. package/dist/{chunk-K77FJXKS.mjs → chunk-YHUTF6EA.mjs} +474 -93
  86. package/dist/{chunk-OGLV4I3L.mjs → chunk-YLNSJQLX.mjs} +12 -7
  87. package/dist/{chunk-SOHMMNCH.mjs → chunk-Z4HFMYNX.mjs} +32 -27
  88. package/dist/{chunk-5W7AEVTH.mjs → chunk-ZMBFWQXN.mjs} +2 -2
  89. package/dist/{chunk-2MKX2SCO.mjs → chunk-ZPWS2SKJ.mjs} +297 -274
  90. package/dist/{chunk-7XY6VD7B.mjs → chunk-ZSWHPKGM.mjs} +1 -1
  91. package/dist/{client-gNAfaHcE.d.ts → client-B12YFALP.d.ts} +18 -3
  92. package/dist/client.d.ts +7 -7
  93. package/dist/client.mjs +11 -1
  94. package/dist/cold-module-4X7WJCFM.mjs +783 -0
  95. package/dist/{config-D1cDv3sE.d.ts → config-BhZxIeb1.d.ts} +10 -1
  96. package/dist/{config-CmYJO34X.d.ts → config-CIG5yDyg.d.ts} +2 -2
  97. package/dist/{direct-executor-HQ4QVDX7.mjs → direct-executor-7XOQ7VQP.mjs} +14 -14
  98. package/dist/distribution/coordinator/etcd.d.ts +2 -2
  99. package/dist/distribution/coordinator/etcd.mjs +1 -1
  100. package/dist/distribution/coordinator/in-memory.d.ts +2 -2
  101. package/dist/distribution/transport/grpc.mjs +1 -1
  102. package/dist/distribution.d.ts +24 -8
  103. package/dist/distribution.mjs +982 -211
  104. package/dist/{durability-wiring-S7OX2MAI.mjs → durability-wiring-324D2DDD.mjs} +362 -94
  105. package/dist/embeddings/openai.mjs +2 -2
  106. package/dist/{engine-m_Ddr1bz.d.ts → engine-CtzbHOmG.d.ts} +64 -7
  107. package/dist/import-OOBUW3RR.mjs +44 -0
  108. package/dist/{index-HYnqS1SE.d.ts → index-DsEurhSl.d.ts} +1 -1
  109. package/dist/index.browser.mjs +5 -5
  110. package/dist/index.d.ts +6 -6
  111. package/dist/index.mjs +22 -22
  112. package/dist/{long-query-thread-BSL25PD6.mjs → long-query-thread-4SW7RWDE.mjs} +15 -15
  113. package/dist/{module-2E4XYSGT.mjs → module-DXSJ5LT2.mjs} +378 -87
  114. package/dist/module-IYPLWHGC.mjs +48 -0
  115. package/dist/paged-partition-archive-HFP45PLU.mjs +269 -0
  116. package/dist/partial-words.mjs +5 -5
  117. package/dist/pattern.mjs +3 -3
  118. package/dist/persistence/durability/checkpoint-worker.d.ts +16 -1
  119. package/dist/persistence/durability/checkpoint-worker.mjs +28 -26
  120. package/dist/{phrase-module-JERF2IDG.mjs → phrase-module-5QHBD2VJ.mjs} +1 -1
  121. package/dist/react.d.ts +9 -9
  122. package/dist/react.mjs +4 -0
  123. package/dist/{rerank-DMMT4FZA.mjs → rerank-KTLC3E6N.mjs} +5 -5
  124. package/dist/rerankers/http.mjs +2 -2
  125. package/dist/{browser/restore-queue-GEZ54CA5.mjs → restore-queue-FEYAC4SB.mjs} +8 -2
  126. package/dist/{runtime-7MLQ7J64.mjs → runtime-DYV75DCB.mjs} +6 -6
  127. package/dist/{schema-BTXbHCqA.d.ts → schema-B-Runv0n.d.ts} +7 -0
  128. package/dist/server.d.ts +12 -10
  129. package/dist/server.mjs +54 -21
  130. package/dist/{snapshot-DOQLGGEX.mjs → snapshot-CGM65VCD.mjs} +101 -197
  131. package/dist/synonyms.mjs +6 -4
  132. package/dist/{tasks-Co8HfRD7.d.ts → tasks-CjRJRGTb.d.ts} +26 -8
  133. package/dist/{thread-SKO7CN3R.mjs → thread-KGNAIKPD.mjs} +14 -14
  134. package/dist/vector/search-worker.d.ts +1 -1
  135. package/dist/vector/search-worker.mjs +6 -5
  136. package/dist/workers/entry.mjs +8 -8
  137. package/dist/{write-ZKR7SNRT.mjs → write-2DZPEQL2.mjs} +82 -107
  138. package/openapi.json +114 -6
  139. package/package.json +12 -8
@@ -1,8 +1,8 @@
1
- import { GrowableUint32, SEGMENT_TOKEN_TABLE_START, SEGMENT_POSTING_LOG_START, SEGMENT_POSITION_POOL_START, SEGMENT_FIELD_SCRATCH_START, LOOKUP_KEY_SCRATCH_BYTES, bitsetHas, bitsetSet, createBitSet, bitsetIsEmpty, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY, PRUNING_REJECTION_SLACK, bitsetOrInto, bitsetAndNotInto, bitsetAndInto, SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD, TARGET_BLOCK_ENTRIES, SORT_COLUMN_REBUILD_FRACTION_SHIFT, bitsetFromSet } from './chunk-26A5JFT2.mjs';
2
- import { MAX_DOCUMENT_ENCODING_DEPTH, singleFoldTable, appendFoldedCodePoint, compareCodePoints, EARLIER_RELEASE_MIN_M, MIN_M, MAX_M, DOCUMENT_ENCODER_RETAINED_BUFFER_BYTES, MAX_FILE_BYTES, detectRuntime, splitWorkerBudget, resolveWorkerCount, CHECKSUM_YIELD_CHUNK_BYTES, resolveWorkerEntry, missingWorkerEntry, DEFAULT_REQUEST_TIMEOUT_MS, toReducedSortValue, resolveRequestThreadCount, defaultSortMode, holdsUnpairedSurrogate, compareComparableValues, readSortField, exceedsCodePoints, toComparableSortValue, SORT_VALUE_MAX_CODE_POINTS, compareComparableKeys, truncateSortString, sortModeOf, isSortMode, SORT_MODES, SHUTDOWN_TIMEOUT_MS, MIN_OVERSAMPLE, MAX_RENUMBERED_SEARCH_ATTEMPTS, compareSortValues, DEFAULT_BACKPRESSURE_LIMIT, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, codePointCount, noteThreadSlotGone, threadSlotOfWorker } from './chunk-AIIJDWMG.mjs';
3
1
  import { DEFAULT_MAX_RERANK_WINDOW, DEFAULT_RERANK_TIMEOUT_MS } from './chunk-JPTQZWJQ.mjs';
4
- import { ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, STATISTICS_BROADCAST_INTERVAL_MS, MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, MAX_SUGGEST_LIMIT, MAX_INDEX_CONFIGURATION_BYTES, FOREIGN_STATS_TTL_MS, HEAP_PRESSURE_WARNING_FRACTION, MAX_IMPORT_REFUSALS_REPORTED, DEFAULT_SUGGEST_LIMIT, DEFAULT_LIMIT, DEFAULT_OFFSET, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION, BATCH_CHUNK_SIZE, MAX_DOC_ID_LENGTH } from './chunk-VEGHFP2P.mjs';
5
- import { ErrorCodes, NarsilError, createNarsilError, DEFAULT_FUZZY_PREFIX_LENGTH, MAX_CURSOR_LENGTH, MAX_CURSOR_ANCHOR_CODE_POINTS, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, BM25_DISPLAYED_NAME_LENGTH, RESULT_WINDOW, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, DEFAULT_PAGE_SIZE, DEFAULT_MAX_PER_GROUP, PREFIX_MAX_EXPANSIONS, MAX_TOLERANCE, MAX_PREFIX_LENGTH } from './chunk-UDGB2MWT.mjs';
2
+ import { TARGET_BLOCK_ENTRIES, bitsetHas, createBitSet, LOOKUP_KEY_SCRATCH_BYTES, GrowableUint32, SEGMENT_TOKEN_TABLE_START, SEGMENT_POSTING_LOG_START, SEGMENT_POSITION_POOL_START, SEGMENT_FIELD_SCRATCH_START, bitsetIsEmpty, bitsetSet, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY, PRUNING_REJECTION_SLACK, bitsetOrInto, bitsetAndNotInto, bitsetAndInto, SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD, SORT_COLUMN_REBUILD_FRACTION_SHIFT, bitsetFromSet } from './chunk-26A5JFT2.mjs';
3
+ import { MAX_DOCUMENT_ENCODING_DEPTH, singleFoldTable, appendFoldedCodePoint, compareCodePoints, EARLIER_RELEASE_MIN_M, MIN_M, MAX_M, MAX_FILE_BYTES, DOCUMENT_ENCODER_RETAINED_BUFFER_BYTES, CHECKSUM_YIELD_CHUNK_BYTES, detectRuntime, splitWorkerBudget, resolveWorkerCount, toReducedSortValue, compareComparableValues, resolveWorkerEntry, missingWorkerEntry, DEFAULT_REQUEST_TIMEOUT_MS, defaultSortMode, resolveRequestThreadCount, readSortField, holdsUnpairedSurrogate, truncateSortString, exceedsCodePoints, toComparableSortValue, SORT_VALUE_MAX_CODE_POINTS, compareComparableKeys, sortModeOf, isSortMode, SORT_MODES, SHUTDOWN_TIMEOUT_MS, MIN_OVERSAMPLE, MAX_RENUMBERED_SEARCH_ATTEMPTS, compareSortValues, DEFAULT_BACKPRESSURE_LIMIT, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, codePointCount, noteThreadSlotGone, threadSlotOfWorker } from './chunk-D33A5VMU.mjs';
4
+ import { MAX_INDEX_CONFIGURATION_BYTES, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, STATISTICS_BROADCAST_INTERVAL_MS, MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_SUGGEST_LIMIT, FOREIGN_STATS_TTL_MS, HEAP_PRESSURE_WARNING_FRACTION, DEFAULT_SUGGEST_LIMIT, DEFAULT_LIMIT, DEFAULT_OFFSET, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION, BATCH_CHUNK_SIZE, MAX_DOC_ID_LENGTH } from './chunk-MZI3TEOW.mjs';
5
+ import { ErrorCodes, NarsilError, createNarsilError, DEFAULT_FUZZY_PREFIX_LENGTH, MAX_CURSOR_LENGTH, MAX_CURSOR_ANCHOR_CODE_POINTS, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, BM25_DISPLAYED_NAME_LENGTH, RESULT_WINDOW, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, DEFAULT_PAGE_SIZE, DEFAULT_MAX_PER_GROUP, PREFIX_MAX_EXPANSIONS, MAX_TOLERANCE, MAX_PREFIX_LENGTH } from './chunk-35B42UFM.mjs';
6
6
  import { Encoder, Decoder, decode, encode } from '@msgpack/msgpack';
7
7
 
8
8
  // src/analysis/registry.ts
@@ -2287,6 +2287,162 @@ var PATTERN_TEST_OPERATORS = ["eq", "ne", "in", "nin", "startsWith", "endsWith",
2287
2287
  var PATTERN_EXPRESSION_OPERATORS = ["wildcard", "regex"];
2288
2288
  var PATTERN_ELEMENT_OPERATORS = ["matchesAny", "containsAll"];
2289
2289
 
2290
+ // src/core/hash.ts
2291
+ var FNV_OFFSET_BASIS = 2166136261;
2292
+ var FNV_PRIME = 16777619;
2293
+ var encoder = new TextEncoder();
2294
+ function fnv1aBytes(bytes) {
2295
+ return fnv1aContinue(FNV_OFFSET_BASIS, bytes, bytes.length) >>> 0;
2296
+ }
2297
+ function fnv1aContinue(start, bytes, length) {
2298
+ let hash = start;
2299
+ for (let i = 0; i < length; i++) {
2300
+ hash = Math.imul(hash ^ bytes[i], FNV_PRIME);
2301
+ }
2302
+ return hash;
2303
+ }
2304
+ var scratchNumber = new Float64Array(1);
2305
+ var scratchBytes = new Uint8Array(scratchNumber.buffer);
2306
+ var HOST_IS_LITTLE_ENDIAN = new Uint8Array(new Uint16Array([1]).buffer)[0] === 1;
2307
+ function fnv1aContinueTaggedNumbers(start, tag, numbers) {
2308
+ let hash = start;
2309
+ const b = scratchBytes;
2310
+ for (let i = 0; i < numbers.length; i++) {
2311
+ scratchNumber[0] = Math.fround(numbers[i]);
2312
+ hash = Math.imul(hash ^ tag, FNV_PRIME);
2313
+ if (HOST_IS_LITTLE_ENDIAN) {
2314
+ hash = Math.imul(hash ^ b[7], FNV_PRIME);
2315
+ hash = Math.imul(hash ^ b[6], FNV_PRIME);
2316
+ hash = Math.imul(hash ^ b[5], FNV_PRIME);
2317
+ hash = Math.imul(hash ^ b[4], FNV_PRIME);
2318
+ hash = Math.imul(hash ^ b[3], FNV_PRIME);
2319
+ hash = Math.imul(hash ^ b[2], FNV_PRIME);
2320
+ hash = Math.imul(hash ^ b[1], FNV_PRIME);
2321
+ hash = Math.imul(hash ^ b[0], FNV_PRIME);
2322
+ } else {
2323
+ for (let at = 0; at < 8; at++) hash = Math.imul(hash ^ b[at], FNV_PRIME);
2324
+ }
2325
+ }
2326
+ return hash;
2327
+ }
2328
+ function fnv1a(input) {
2329
+ return fnv1aBytes(encoder.encode(input));
2330
+ }
2331
+
2332
+ // src/partitioning/router.ts
2333
+ function createPartitionRouter() {
2334
+ function route(docId, partitionCount) {
2335
+ if (partitionCount <= 0) {
2336
+ throw new NarsilError(
2337
+ ErrorCodes.INDEX_NOT_FOUND,
2338
+ `Partition count must be greater than 0, received ${partitionCount}`,
2339
+ { partitionCount }
2340
+ );
2341
+ }
2342
+ return fnv1a(docId) % partitionCount;
2343
+ }
2344
+ function routeBatch(docIds, partitionCount) {
2345
+ const groups = /* @__PURE__ */ new Map();
2346
+ for (let i = 0; i < docIds.length; i++) {
2347
+ const pid = route(docIds[i], partitionCount);
2348
+ let group = groups.get(pid);
2349
+ if (!group) {
2350
+ group = [];
2351
+ groups.set(pid, group);
2352
+ }
2353
+ group.push(docIds[i]);
2354
+ }
2355
+ return groups;
2356
+ }
2357
+ return { route, routeBatch };
2358
+ }
2359
+
2360
+ // src/plugins/registry.ts
2361
+ var WRITE_REJECTING_HOOKS = /* @__PURE__ */ new Set(["beforeInsert", "beforeUpdate", "beforeRemove"]);
2362
+ function isThenable(value) {
2363
+ return value !== null && typeof value === "object" && typeof value.then === "function";
2364
+ }
2365
+ function thrownText(err, plugin, hookName) {
2366
+ if (err instanceof Error) return err.message;
2367
+ try {
2368
+ return String(err);
2369
+ } catch {
2370
+ return `The ${hookName} hook of plugin "${plugin}" rejects the write`;
2371
+ }
2372
+ }
2373
+ function asWriteRejection(err, plugin, hookName) {
2374
+ if (err instanceof NarsilError) return err;
2375
+ return new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, thrownText(err, plugin, hookName), {
2376
+ plugin,
2377
+ hook: hookName
2378
+ });
2379
+ }
2380
+ function invoke(hook, hookName, context) {
2381
+ const rejectsWrites = WRITE_REJECTING_HOOKS.has(hookName);
2382
+ let result;
2383
+ try {
2384
+ result = hook.run(context);
2385
+ } catch (err) {
2386
+ throw rejectsWrites ? asWriteRejection(err, hook.plugin, hookName) : err;
2387
+ }
2388
+ if (!isThenable(result)) return null;
2389
+ if (!rejectsWrites) return result;
2390
+ return Promise.resolve(result).catch((err) => {
2391
+ throw asWriteRejection(err, hook.plugin, hookName);
2392
+ });
2393
+ }
2394
+ async function continueAsync(pending, hooks, startIndex, hookName, context) {
2395
+ await pending;
2396
+ for (let i = startIndex; i < hooks.length; i++) {
2397
+ await invoke(hooks[i], hookName, context);
2398
+ }
2399
+ }
2400
+ function createPluginRegistry() {
2401
+ const plugins = [];
2402
+ return {
2403
+ register(plugin) {
2404
+ plugins.push(plugin);
2405
+ },
2406
+ hasHooks(hookName) {
2407
+ return plugins.some((p) => typeof p[hookName] === "function");
2408
+ },
2409
+ runHook(hookName, context) {
2410
+ const hooks = [];
2411
+ for (const plugin of plugins) {
2412
+ const hook = plugin[hookName];
2413
+ if (typeof hook === "function") {
2414
+ hooks.push({ plugin: plugin.name, run: hook.bind(plugin) });
2415
+ }
2416
+ }
2417
+ if (hooks.length === 0) return;
2418
+ for (let i = 0; i < hooks.length; i++) {
2419
+ const pending = invoke(hooks[i], hookName, context);
2420
+ if (pending !== null) {
2421
+ return continueAsync(pending, hooks, i + 1, hookName, context);
2422
+ }
2423
+ }
2424
+ }
2425
+ };
2426
+ }
2427
+
2428
+ // src/rerank/registry.ts
2429
+ function validateRerankAdapter(name, adapter) {
2430
+ if (typeof name !== "string" || name.trim().length === 0) {
2431
+ throw new NarsilError(ErrorCodes.RERANK_CONFIG_INVALID, "A rerank adapter needs a non-empty string name");
2432
+ }
2433
+ const rerank = typeof adapter === "object" && adapter !== null ? adapter.rerank : void 0;
2434
+ if (typeof rerank !== "function") {
2435
+ throw new NarsilError(ErrorCodes.RERANK_CONFIG_INVALID, `Rerank adapter "${name}" must provide a rerank function`, {
2436
+ adapter: name
2437
+ });
2438
+ }
2439
+ }
2440
+ function registerEngineRerankAdapter(core, name, adapter) {
2441
+ core.guardShutdown();
2442
+ validateRerankAdapter(name, adapter);
2443
+ core.rerankAdapters.set(name, adapter);
2444
+ }
2445
+
2290
2446
  // src/geo/constants.ts
2291
2447
  var DEG_TO_RAD = Math.PI / 180;
2292
2448
  var MAX_LATITUDE_DEGREES = 90;
@@ -2776,6 +2932,10 @@ function validateVectorStorage(config, filesystemDurability) {
2776
2932
  });
2777
2933
  }
2778
2934
  }
2935
+ function validateTextStorage(textStorage, filesystemDurability) {
2936
+ if (textStorage === void 0 || textStorage === "memory" || textStorage === "disk" && filesystemDurability) return;
2937
+ fail("textStorage must be 'memory', or 'disk' with filesystem durability", { textStorage });
2938
+ }
2779
2939
  function raiseEarlierReleaseGraphSetting(config) {
2780
2940
  if (typeof config !== "object" || config === null) return config;
2781
2941
  const hnsw = config.hnswConfig;
@@ -2812,91 +2972,213 @@ function validateVectorPromotion(config) {
2812
2972
  }
2813
2973
  }
2814
2974
 
2815
- // src/core/partition/utils.ts
2816
- function getOrCreateFieldNameIndex(table, fieldName) {
2817
- const existing = table.indexMap.get(fieldName);
2818
- if (existing !== void 0) return existing;
2819
- const idx = table.names.length;
2820
- table.names.push(fieldName);
2821
- table.indexMap.set(fieldName, idx);
2822
- return idx;
2823
- }
2824
- var VECTOR_PATTERN2 = /^vector\[(\d+)]$/;
2825
- function getNestedValue(obj, path) {
2826
- const parts = path.split(".");
2827
- let current = obj;
2828
- for (let i = 0; i < parts.length; i++) {
2829
- if (current === null || current === void 0 || typeof current !== "object") return void 0;
2830
- if (!Object.hasOwn(current, parts[i])) return void 0;
2831
- current = current[parts[i]];
2832
- }
2833
- return current;
2834
- }
2835
- function stringArraysEqual(a, b) {
2836
- if (a === b) return true;
2837
- if (!a || !b) return false;
2838
- if (a.length !== b.length) return false;
2839
- for (let i = 0; i < a.length; i++) {
2840
- if (a[i] !== b[i]) return false;
2975
+ // src/schema/embedding-validator.ts
2976
+ var VECTOR_DIMENSION_PATTERN = /^vector\[(\d+)]$/;
2977
+ function validateEmbeddingConfig(embeddingConfig, schema, instanceAdapter) {
2978
+ const fieldEntries = Object.keys(embeddingConfig.fields);
2979
+ if (fieldEntries.length === 0) {
2980
+ throw new NarsilError(
2981
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
2982
+ "Embedding config must define at least one field mapping"
2983
+ );
2841
2984
  }
2842
- return true;
2843
- }
2844
- function textFieldsChanged(oldDoc, newDoc, fields) {
2845
- for (const { path, base } of fields) {
2846
- if (!isWordIndexedBase(base)) continue;
2847
- const oldVal = getNestedValue(oldDoc, path);
2848
- const newVal = getNestedValue(newDoc, path);
2849
- if (base === "string") {
2850
- if (oldVal !== newVal) return true;
2851
- } else {
2852
- if (!stringArraysEqual(oldVal, newVal)) return true;
2853
- }
2985
+ if (typeof embeddingConfig.adapter === "string") {
2986
+ throw new NarsilError(
2987
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
2988
+ `Embedding adapter "${embeddingConfig.adapter}" must be resolved against the engine registry before validation`,
2989
+ { adapter: embeddingConfig.adapter }
2990
+ );
2854
2991
  }
2855
- return false;
2856
- }
2857
- function parseVectorDimension(fieldType) {
2858
- const match = VECTOR_PATTERN2.exec(fieldType);
2859
- return match ? Number.parseInt(match[1], 10) : null;
2860
- }
2861
- function tokenizeOptions(options) {
2862
- return {
2863
- stem: true,
2864
- removeStopWords: true,
2865
- collectSurfaces: options?.collectSurfaces === true,
2866
- stopWordOverride: options?.stopWordOverride,
2867
- customTokenizer: options?.customTokenizer
2868
- };
2869
- }
2870
- function getFlatSchema(state, schema) {
2871
- if (state.lastSchemaRef === schema && state.flatSchemaCache) return state.flatSchemaCache;
2872
- state.flatSchemaCache = flattenSchema(schema);
2873
- state.lastSchemaRef = schema;
2874
- return state.flatSchemaCache;
2875
- }
2876
-
2877
- // src/core/projection.ts
2878
- function requireFieldPathList(paths, option) {
2879
- if (paths === void 0) return;
2880
- if (!Array.isArray(paths)) {
2992
+ const resolvedAdapter = embeddingConfig.adapter ?? instanceAdapter;
2993
+ if (!resolvedAdapter) {
2881
2994
  throw new NarsilError(
2882
- ErrorCodes.SEARCH_INVALID_FIELD,
2883
- `The "document.${option}" option takes an array of field paths`,
2884
- { option: `document.${option}`, received: paths === null ? "null" : typeof paths }
2995
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
2996
+ "Embedding config requires an adapter (set on the index or the instance)"
2885
2997
  );
2886
2998
  }
2887
- for (let position = 0; position < paths.length; position++) {
2888
- const entry = paths[position];
2889
- if (typeof entry === "string") continue;
2999
+ if (typeof resolvedAdapter.dimensions !== "number" || !Number.isInteger(resolvedAdapter.dimensions) || resolvedAdapter.dimensions <= 0) {
2890
3000
  throw new NarsilError(
2891
- ErrorCodes.SEARCH_INVALID_FIELD,
2892
- `The "document.${option}" option takes field paths as strings, and entry ${position} is not a string`,
2893
- { option: `document.${option}`, position, received: entry === null ? "null" : typeof entry }
3001
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
3002
+ `Adapter dimensions must be a positive integer, got ${resolvedAdapter.dimensions}`
2894
3003
  );
2895
3004
  }
2896
- }
2897
- function splitPaths(paths) {
2898
- if (!paths) return [];
2899
- const split = [];
3005
+ const flatFields = flattenSchema(schema);
3006
+ for (const [targetField, sourceFields] of Object.entries(embeddingConfig.fields)) {
3007
+ const targetType = flatFields[targetField];
3008
+ if (targetType === void 0) {
3009
+ throw new NarsilError(
3010
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
3011
+ `Embedding target field "${targetField}" does not exist in the schema`,
3012
+ { field: targetField }
3013
+ );
3014
+ }
3015
+ const vectorMatch = VECTOR_DIMENSION_PATTERN.exec(targetType);
3016
+ if (!vectorMatch) {
3017
+ throw new NarsilError(
3018
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
3019
+ `Embedding target field "${targetField}" must be a vector type, got "${targetType}"`,
3020
+ { field: targetField, type: targetType }
3021
+ );
3022
+ }
3023
+ const schemaDimension = Number.parseInt(vectorMatch[1], 10);
3024
+ if (schemaDimension !== resolvedAdapter.dimensions) {
3025
+ throw new NarsilError(
3026
+ ErrorCodes.EMBEDDING_DIMENSION_MISMATCH,
3027
+ `Embedding target field "${targetField}" has dimension ${schemaDimension} but the adapter produces ${resolvedAdapter.dimensions}-dimensional vectors`,
3028
+ { field: targetField, expected: schemaDimension, actual: resolvedAdapter.dimensions }
3029
+ );
3030
+ }
3031
+ const sources = Array.isArray(sourceFields) ? sourceFields : [sourceFields];
3032
+ if (sources.length === 0) {
3033
+ throw new NarsilError(
3034
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
3035
+ `Embedding target field "${targetField}" must have at least one source field`,
3036
+ { field: targetField }
3037
+ );
3038
+ }
3039
+ for (const source of sources) {
3040
+ const sourceType = flatFields[source];
3041
+ if (sourceType === void 0) {
3042
+ throw new NarsilError(
3043
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
3044
+ `Embedding source field "${source}" does not exist in the schema`,
3045
+ { field: source, targetField }
3046
+ );
3047
+ }
3048
+ if (!isSingleStringFieldType(sourceType)) {
3049
+ throw new NarsilError(
3050
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
3051
+ `Embedding source field "${source}" must be a string type, got "${sourceType}"`,
3052
+ { field: source, type: sourceType, targetField }
3053
+ );
3054
+ }
3055
+ }
3056
+ }
3057
+ return resolvedAdapter;
3058
+ }
3059
+ function validateRegisteredAdapter(name, adapter) {
3060
+ if (typeof name !== "string" || name.trim().length === 0) {
3061
+ throw new NarsilError(ErrorCodes.EMBEDDING_CONFIG_INVALID, "Embedding adapter names must be non-empty strings");
3062
+ }
3063
+ if (typeof adapter?.embed !== "function") {
3064
+ throw new NarsilError(
3065
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
3066
+ `Embedding adapter "${name}" must provide an embed function`,
3067
+ { adapter: name }
3068
+ );
3069
+ }
3070
+ if (typeof adapter.dimensions !== "number" || !Number.isInteger(adapter.dimensions) || adapter.dimensions <= 0) {
3071
+ throw new NarsilError(
3072
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
3073
+ `Embedding adapter "${name}" dimensions must be a positive integer, got ${adapter.dimensions}`,
3074
+ { adapter: name, dimensions: adapter.dimensions }
3075
+ );
3076
+ }
3077
+ }
3078
+ function validateRequiredFieldsInSchema(required, schema) {
3079
+ if (required.length === 0) return;
3080
+ const flatFields = flattenSchema(schema);
3081
+ for (const field of required) {
3082
+ if (flatFields[field] === void 0) {
3083
+ throw new NarsilError(
3084
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
3085
+ `Required field "${field}" does not exist in the schema`,
3086
+ { field }
3087
+ );
3088
+ }
3089
+ }
3090
+ }
3091
+
3092
+ // src/runtime/heap-statistics.browser.ts
3093
+ function readHeapStatistics() {
3094
+ return null;
3095
+ }
3096
+
3097
+ // src/core/partition/utils.ts
3098
+ function getOrCreateFieldNameIndex(table, fieldName) {
3099
+ const existing = table.indexMap.get(fieldName);
3100
+ if (existing !== void 0) return existing;
3101
+ const idx = table.names.length;
3102
+ table.names.push(fieldName);
3103
+ table.indexMap.set(fieldName, idx);
3104
+ return idx;
3105
+ }
3106
+ var VECTOR_PATTERN2 = /^vector\[(\d+)]$/;
3107
+ function getNestedValue(obj, path) {
3108
+ const parts = path.split(".");
3109
+ let current = obj;
3110
+ for (let i = 0; i < parts.length; i++) {
3111
+ if (current === null || current === void 0 || typeof current !== "object") return void 0;
3112
+ if (!Object.hasOwn(current, parts[i])) return void 0;
3113
+ current = current[parts[i]];
3114
+ }
3115
+ return current;
3116
+ }
3117
+ function stringArraysEqual(a, b) {
3118
+ if (a === b) return true;
3119
+ if (!a || !b) return false;
3120
+ if (a.length !== b.length) return false;
3121
+ for (let i = 0; i < a.length; i++) {
3122
+ if (a[i] !== b[i]) return false;
3123
+ }
3124
+ return true;
3125
+ }
3126
+ function textFieldsChanged(oldDoc, newDoc, fields) {
3127
+ for (const { path, base } of fields) {
3128
+ if (!isWordIndexedBase(base)) continue;
3129
+ const oldVal = getNestedValue(oldDoc, path);
3130
+ const newVal = getNestedValue(newDoc, path);
3131
+ if (base === "string") {
3132
+ if (oldVal !== newVal) return true;
3133
+ } else {
3134
+ if (!stringArraysEqual(oldVal, newVal)) return true;
3135
+ }
3136
+ }
3137
+ return false;
3138
+ }
3139
+ function parseVectorDimension(fieldType) {
3140
+ const match = VECTOR_PATTERN2.exec(fieldType);
3141
+ return match ? Number.parseInt(match[1], 10) : null;
3142
+ }
3143
+ function tokenizeOptions(options) {
3144
+ return {
3145
+ stem: true,
3146
+ removeStopWords: true,
3147
+ collectSurfaces: options?.collectSurfaces === true,
3148
+ stopWordOverride: options?.stopWordOverride,
3149
+ customTokenizer: options?.customTokenizer
3150
+ };
3151
+ }
3152
+ function getFlatSchema(state, schema) {
3153
+ if (state.lastSchemaRef === schema && state.flatSchemaCache) return state.flatSchemaCache;
3154
+ state.flatSchemaCache = flattenSchema(schema);
3155
+ state.lastSchemaRef = schema;
3156
+ return state.flatSchemaCache;
3157
+ }
3158
+
3159
+ // src/core/projection.ts
3160
+ function requireFieldPathList(paths, option) {
3161
+ if (paths === void 0) return;
3162
+ if (!Array.isArray(paths)) {
3163
+ throw new NarsilError(
3164
+ ErrorCodes.SEARCH_INVALID_FIELD,
3165
+ `The "document.${option}" option takes an array of field paths`,
3166
+ { option: `document.${option}`, received: paths === null ? "null" : typeof paths }
3167
+ );
3168
+ }
3169
+ for (let position = 0; position < paths.length; position++) {
3170
+ const entry = paths[position];
3171
+ if (typeof entry === "string") continue;
3172
+ throw new NarsilError(
3173
+ ErrorCodes.SEARCH_INVALID_FIELD,
3174
+ `The "document.${option}" option takes field paths as strings, and entry ${position} is not a string`,
3175
+ { option: `document.${option}`, position, received: entry === null ? "null" : typeof entry }
3176
+ );
3177
+ }
3178
+ }
3179
+ function splitPaths(paths) {
3180
+ if (!paths) return [];
3181
+ const split = [];
2900
3182
  for (const path of paths) {
2901
3183
  const segments = path.split(".").filter((segment) => segment.length > 0);
2902
3184
  if (segments.length === 0) continue;
@@ -7538,7 +7820,7 @@ function requireLoadedPhraseCode() {
7538
7820
  }
7539
7821
  function loadPhraseCode() {
7540
7822
  if (loading === null) {
7541
- loading = import('./phrase-module-WESYVU7K.mjs').then(
7823
+ loading = import('./phrase-module-HGVBO24W.mjs').then(
7542
7824
  (code) => {
7543
7825
  loaded = code;
7544
7826
  return code;
@@ -8128,7 +8410,7 @@ function fieldLengthBound(columns, fieldIndex, internalId) {
8128
8410
  function entriesCovered(bounds) {
8129
8411
  return bounds.blockCount === 0 ? 0 : bounds.entryEnd[bounds.blockCount - 1];
8130
8412
  }
8131
- function scan(list, columns, previous, keepBlocks) {
8413
+ function scanPostingBlocks(list, columns, previous, keepBlocks) {
8132
8414
  const { docIds, termFrequencies, fieldNameIndices, length } = list;
8133
8415
  const startEntry = previous !== null && keepBlocks > 0 ? previous.entryEnd[keepBlocks - 1] : 0;
8134
8416
  const capacity = keepBlocks + Math.ceil((length - startEntry) / TARGET_BLOCK_ENTRIES) + 1;
@@ -8202,12 +8484,13 @@ function blockBoundsFor(list, columns) {
8202
8484
  const covered = entriesCovered(existing);
8203
8485
  if (covered === list.length) return existing;
8204
8486
  if (list.ordered && covered < list.length) {
8205
- const extended = scan(list, columns, existing, Math.max(0, existing.blockCount - 1));
8487
+ const extended = scanPostingBlocks(list, columns, existing, Math.max(0, existing.blockCount - 1));
8206
8488
  boundsByList.set(list, extended);
8207
8489
  return extended;
8208
8490
  }
8209
8491
  }
8210
- const built = scan(list, columns, null, 0);
8492
+ const stored = list.storedBlockBounds?.() ?? null;
8493
+ const built = stored === null ? scanPostingBlocks(list, columns, null, 0) : { ...stored, structureRevision: list.structureRevision };
8211
8494
  boundsByList.set(list, built);
8212
8495
  return built;
8213
8496
  }
@@ -8400,33 +8683,33 @@ function boostsKeepScoresNonNegative(fields) {
8400
8683
  }
8401
8684
  return true;
8402
8685
  }
8403
- function samePairUpperBound(scan2, list, maxBoost, maxAverageLength) {
8404
- if (scan2.scoresAreZero) return 0;
8405
- const bounds = blockBoundsFor(list, scan2.fields.lengthColumns);
8686
+ function samePairUpperBound(scan, list, maxBoost, maxAverageLength) {
8687
+ if (scan.scoresAreZero) return 0;
8688
+ const bounds = blockBoundsFor(list, scan.fields.lengthColumns);
8406
8689
  let best = 0;
8407
8690
  for (let block = 0; block < bounds.blockCount; block++) {
8408
8691
  const bestEntry = computeBM25WithIDF(
8409
8692
  bounds.maxTermFrequency[block],
8410
- scan2.idf,
8693
+ scan.idf,
8411
8694
  bounds.minFieldLength[block],
8412
8695
  maxAverageLength,
8413
- scan2.k1,
8414
- scan2.b
8696
+ scan.k1,
8697
+ scan.b
8415
8698
  );
8416
8699
  const blockBound = bestEntry * maxBoost * bounds.maxEntriesPerDocument[block];
8417
8700
  if (blockBound > best) best = blockBound;
8418
8701
  }
8419
8702
  return best;
8420
8703
  }
8421
- function fieldPairUpperBound(scan2, list, maxBoost, maxAverageLength) {
8422
- if (scan2.scoresAreZero) return 0;
8423
- const bounds = blockBoundsFor(list, scan2.fields.lengthColumns);
8704
+ function fieldPairUpperBound(scan, list, maxBoost, maxAverageLength) {
8705
+ if (scan.scoresAreZero) return 0;
8706
+ const bounds = blockBoundsFor(list, scan.fields.lengthColumns);
8424
8707
  let best = 0;
8425
8708
  for (let block = 0; block < bounds.blockCount; block++) {
8426
8709
  const bestEntry = blockEntryBound(
8427
- scan2.fields,
8710
+ scan.fields,
8428
8711
  bounds.maxTermFrequency[block],
8429
- scan2.idf,
8712
+ scan.idf,
8430
8713
  bounds.minFieldLength[block],
8431
8714
  maxAverageLength
8432
8715
  );
@@ -8435,22 +8718,22 @@ function fieldPairUpperBound(scan2, list, maxBoost, maxAverageLength) {
8435
8718
  }
8436
8719
  return best;
8437
8720
  }
8438
- function samePairEntryScore(scan2, entry, internalId) {
8439
- const { fields, columns } = scan2;
8721
+ function samePairEntryScore(scan, entry, internalId) {
8722
+ const { fields, columns } = scan;
8440
8723
  const fieldIndex = columns.fieldNameIndices[entry];
8441
8724
  const averageLength = fields.averageLengths[fieldIndex];
8442
8725
  const fieldLength = fieldLengthOf(fields.lengthColumns, fieldIndex, internalId, averageLength);
8443
- const bm25 = scan2.scoresAreZero ? 0 : computeBM25WithIDF(columns.termFrequencies[entry], scan2.idf, fieldLength, averageLength, scan2.k1, scan2.b);
8726
+ const bm25 = scan.scoresAreZero ? 0 : computeBM25WithIDF(columns.termFrequencies[entry], scan.idf, fieldLength, averageLength, scan.k1, scan.b);
8444
8727
  return bm25 * fields.boosts[fieldIndex];
8445
8728
  }
8446
- function fieldPairEntryScore(scan2, entry, internalId) {
8447
- const { fields, columns } = scan2;
8729
+ function fieldPairEntryScore(scan, entry, internalId) {
8730
+ const { fields, columns } = scan;
8448
8731
  const fieldIndex = columns.fieldNameIndices[entry];
8449
8732
  const averageLength = fields.averageLengths[fieldIndex];
8450
8733
  const fieldLength = fieldLengthOf(fields.lengthColumns, fieldIndex, internalId, averageLength);
8451
- const bm25 = scan2.scoresAreZero ? 0 : computeBM25WithIDF(
8734
+ const bm25 = scan.scoresAreZero ? 0 : computeBM25WithIDF(
8452
8735
  columns.termFrequencies[entry],
8453
- scan2.idf,
8736
+ scan.idf,
8454
8737
  fieldLength,
8455
8738
  averageLength,
8456
8739
  fields.k1s[fieldIndex],
@@ -8458,48 +8741,48 @@ function fieldPairEntryScore(scan2, entry, internalId) {
8458
8741
  );
8459
8742
  return bm25 * fields.boosts[fieldIndex];
8460
8743
  }
8461
- function samePairScoreEveryDocument(scan2, buffer) {
8462
- const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = scan2.columns;
8463
- const searchable = scan2.fields.searchable;
8744
+ function samePairScoreEveryDocument(scan, buffer) {
8745
+ const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = scan.columns;
8746
+ const searchable = scan.fields.searchable;
8464
8747
  for (let entry = 0; entry < count; entry++) {
8465
8748
  const internalId = docIds[entry];
8466
8749
  if (hasDeleted && deletedDocs.has(internalId)) continue;
8467
8750
  if (searchable[fieldNameIndices[entry]] !== 1) continue;
8468
- addScore(buffer, internalId, samePairEntryScore(scan2, entry, internalId));
8751
+ addScore(buffer, internalId, samePairEntryScore(scan, entry, internalId));
8469
8752
  }
8470
8753
  }
8471
- function fieldPairScoreEveryDocument(scan2, buffer) {
8472
- const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = scan2.columns;
8473
- const searchable = scan2.fields.searchable;
8754
+ function fieldPairScoreEveryDocument(scan, buffer) {
8755
+ const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = scan.columns;
8756
+ const searchable = scan.fields.searchable;
8474
8757
  for (let entry = 0; entry < count; entry++) {
8475
8758
  const internalId = docIds[entry];
8476
8759
  if (hasDeleted && deletedDocs.has(internalId)) continue;
8477
8760
  if (searchable[fieldNameIndices[entry]] !== 1) continue;
8478
- addScore(buffer, internalId, fieldPairEntryScore(scan2, entry, internalId));
8761
+ addScore(buffer, internalId, fieldPairEntryScore(scan, entry, internalId));
8479
8762
  }
8480
8763
  }
8481
- function samePairScoreKnownDocuments(scan2, buffer) {
8482
- const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = scan2.columns;
8483
- const searchable = scan2.fields.searchable;
8764
+ function samePairScoreKnownDocuments(scan, buffer) {
8765
+ const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = scan.columns;
8766
+ const searchable = scan.fields.searchable;
8484
8767
  let counted = 0;
8485
8768
  for (let entry = 0; entry < count; entry++) {
8486
8769
  const internalId = docIds[entry];
8487
8770
  if (hasDeleted && deletedDocs.has(internalId)) continue;
8488
8771
  if (searchable[fieldNameIndices[entry]] !== 1) continue;
8489
- if (hasScore(buffer, internalId)) addScore(buffer, internalId, samePairEntryScore(scan2, entry, internalId));
8772
+ if (hasScore(buffer, internalId)) addScore(buffer, internalId, samePairEntryScore(scan, entry, internalId));
8490
8773
  else if (markCounted(buffer, internalId)) counted++;
8491
8774
  }
8492
8775
  return counted;
8493
8776
  }
8494
- function fieldPairScoreKnownDocuments(scan2, buffer) {
8495
- const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = scan2.columns;
8496
- const searchable = scan2.fields.searchable;
8777
+ function fieldPairScoreKnownDocuments(scan, buffer) {
8778
+ const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = scan.columns;
8779
+ const searchable = scan.fields.searchable;
8497
8780
  let counted = 0;
8498
8781
  for (let entry = 0; entry < count; entry++) {
8499
8782
  const internalId = docIds[entry];
8500
8783
  if (hasDeleted && deletedDocs.has(internalId)) continue;
8501
8784
  if (searchable[fieldNameIndices[entry]] !== 1) continue;
8502
- if (hasScore(buffer, internalId)) addScore(buffer, internalId, fieldPairEntryScore(scan2, entry, internalId));
8785
+ if (hasScore(buffer, internalId)) addScore(buffer, internalId, fieldPairEntryScore(scan, entry, internalId));
8503
8786
  else if (markCounted(buffer, internalId)) counted++;
8504
8787
  }
8505
8788
  return counted;
@@ -8516,24 +8799,24 @@ function firstEntryOf(docIds, count, internalId) {
8516
8799
  }
8517
8800
  function samePairQueryOrderScore(scans, internalId) {
8518
8801
  let score = 0;
8519
- for (const scan2 of scans) {
8520
- const { docIds, deletedDocs, hasDeleted, count } = scan2.columns;
8802
+ for (const scan of scans) {
8803
+ const { docIds, deletedDocs, hasDeleted, count } = scan.columns;
8521
8804
  if (hasDeleted && deletedDocs.has(internalId)) continue;
8522
8805
  for (let entry = firstEntryOf(docIds, count, internalId); entry < count && docIds[entry] === internalId; entry++) {
8523
- if (scan2.fields.searchable[scan2.columns.fieldNameIndices[entry]] !== 1) continue;
8524
- score += samePairEntryScore(scan2, entry, internalId);
8806
+ if (scan.fields.searchable[scan.columns.fieldNameIndices[entry]] !== 1) continue;
8807
+ score += samePairEntryScore(scan, entry, internalId);
8525
8808
  }
8526
8809
  }
8527
8810
  return score;
8528
8811
  }
8529
8812
  function fieldPairQueryOrderScore(scans, internalId) {
8530
8813
  let score = 0;
8531
- for (const scan2 of scans) {
8532
- const { docIds, deletedDocs, hasDeleted, count } = scan2.columns;
8814
+ for (const scan of scans) {
8815
+ const { docIds, deletedDocs, hasDeleted, count } = scan.columns;
8533
8816
  if (hasDeleted && deletedDocs.has(internalId)) continue;
8534
8817
  for (let entry = firstEntryOf(docIds, count, internalId); entry < count && docIds[entry] === internalId; entry++) {
8535
- if (scan2.fields.searchable[scan2.columns.fieldNameIndices[entry]] !== 1) continue;
8536
- score += fieldPairEntryScore(scan2, entry, internalId);
8818
+ if (scan.fields.searchable[scan.columns.fieldNameIndices[entry]] !== 1) continue;
8819
+ score += fieldPairEntryScore(scan, entry, internalId);
8537
8820
  }
8538
8821
  }
8539
8822
  return score;
@@ -8613,7 +8896,7 @@ function multiTermTopK(request) {
8613
8896
  }
8614
8897
  const maxBoost = bestSearchable(fields.searchable, fields.boosts);
8615
8898
  const maxAverageLength = bestSearchable(fields.searchable, fields.averageLengths);
8616
- const upperBounds = scans.map((scan2, term) => scorer.upperBound(scan2, terms[term].list, maxBoost, maxAverageLength));
8899
+ const upperBounds = scans.map((scan, term) => scorer.upperBound(scan, terms[term].list, maxBoost, maxAverageLength));
8617
8900
  const order = scans.map((_, term) => term).sort((left, right) => upperBounds[right] - upperBounds[left]);
8618
8901
  const remainingBound = new Float64Array(order.length + 1);
8619
8902
  for (let position2 = order.length - 1; position2 >= 0; position2--) {
@@ -9822,512 +10105,46 @@ function createPartitionIndex(partitionId, trackPositions = true) {
9822
10105
  return sortedPageOf(state, request);
9823
10106
  },
9824
10107
  sortValues(docId, fields, fieldTypes, modes) {
9825
- return sortValuesOf(state, docId, fields, fieldTypes, modes);
9826
- },
9827
- applyFilters(filters, schema, meter) {
9828
- return applyPartitionFilters(state, filters, schema, meter);
9829
- },
9830
- applyFiltersBitset(filters, schema, meter) {
9831
- return applyPartitionFiltersBitset(state, filters, schema, meter);
9832
- },
9833
- filterMatches(filters, schema, meter) {
9834
- return partitionFilterMatches(state, filters, schema, meter);
9835
- },
9836
- computeFacets(matched, config, schema) {
9837
- return computeFacets(state, matched, config, schema);
9838
- },
9839
- suggestTerms(surfacePrefix, stemmedPrefix, limit) {
9840
- return suggestDisplayTerms(state, surfacePrefix, stemmedPrefix, limit);
9841
- },
9842
- expandTermPrefix(surfacePrefix, stemmedToken, maxExpansions) {
9843
- return expandTermPrefix([state], surfacePrefix, stemmedToken, maxExpansions);
9844
- },
9845
- expandPartialWords(analysis, requests) {
9846
- return expandPartialWords([state], analysis, requests);
9847
- },
9848
- preparePartialWords(analysis) {
9849
- expandPartialWords([state], analysis, []);
9850
- },
9851
- serialize(indexName, totalPartitions, language, schema) {
9852
- return serializePartition(state, partitionId, indexName, totalPartitions, language, schema);
9853
- },
9854
- deserialize(data, schema) {
9855
- deserializePartition(state, data, clearAll, schema);
9856
- }
9857
- };
9858
- statesByPartition.set(partition, state);
9859
- return partition;
9860
- }
9861
-
9862
- // src/engine/orchestration/turn.ts
9863
- function afterCurrentTurn(callback) {
9864
- if (typeof setImmediate === "function") {
9865
- setImmediate(callback);
9866
- return;
9867
- }
9868
- setTimeout(callback, 0);
9869
- }
9870
- function yieldToEventLoop() {
9871
- return new Promise((resolve) => afterCurrentTurn(resolve));
9872
- }
9873
-
9874
- // src/partitioning/constants.ts
9875
- var REBALANCE_CHUNK_SIZE = 1e3;
9876
- var DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE = 1e4;
9877
- var TEXT_REBUILD_CHUNK_DOCUMENTS = 1e3;
9878
- var TEXT_REBUILD_SWITCH_BACKLOG_DOCUMENTS = 1e3;
9879
-
9880
- // src/partitioning/rebalancer.ts
9881
- function createRebalancer() {
9882
- const activeManagers = /* @__PURE__ */ new WeakSet();
9883
- let activeCount = 0;
9884
- async function rebalance(manager, newPartitionCount, router, onProgress) {
9885
- if (activeManagers.has(manager)) {
9886
- throw new NarsilError(
9887
- ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
9888
- "A rebalance operation is already in progress"
9889
- );
9890
- }
9891
- if (newPartitionCount <= 0) {
9892
- throw new NarsilError(
9893
- ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
9894
- `New partition count must be greater than 0, received ${newPartitionCount}`,
9895
- { newPartitionCount }
9896
- );
9897
- }
9898
- if (newPartitionCount === manager.partitionCount) {
9899
- throw new NarsilError(
9900
- ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
9901
- `New partition count (${newPartitionCount}) is the same as the current count`,
9902
- { newPartitionCount, currentPartitionCount: manager.partitionCount }
9903
- );
9904
- }
9905
- activeManagers.add(manager);
9906
- activeCount++;
9907
- try {
9908
- const currentPartitions = manager.getAllPartitions();
9909
- let documentsTotal = 0;
9910
- for (const partition of currentPartitions) {
9911
- documentsTotal += partition.count();
9912
- }
9913
- onProgress?.({
9914
- phase: "scanning",
9915
- documentsProcessed: 0,
9916
- documentsTotal
9917
- });
9918
- const newPartitions = [];
9919
- for (let i = 0; i < newPartitionCount; i++) {
9920
- newPartitions.push(createPartitionIndex(i, manager.config.trackPositions ?? true));
9921
- }
9922
- let documentsProcessed = 0;
9923
- let chunkFill = 0;
9924
- const insertOptions = {
9925
- validate: false,
9926
- skipClone: true,
9927
- stopWordOverride: manager.analysis.stopWords,
9928
- customTokenizer: manager.analysis.customTokenizer,
9929
- collectSurfaces: manager.config.surfaceForms !== false
9930
- };
9931
- for (const partition of currentPartitions) {
9932
- const docIds = [...partition.docIds()];
9933
- for (const docId of docIds) {
9934
- const document = partition.getRef(docId);
9935
- if (!document) continue;
9936
- const targetPartitionId = router.route(docId, newPartitionCount);
9937
- const targetPartition = newPartitions[targetPartitionId];
9938
- if (!targetPartition.has(docId)) {
9939
- targetPartition.insert(docId, document, manager.schema, manager.language, insertOptions);
9940
- documentsProcessed++;
9941
- }
9942
- chunkFill++;
9943
- if (chunkFill >= REBALANCE_CHUNK_SIZE) {
9944
- chunkFill = 0;
9945
- onProgress?.({
9946
- phase: "moving",
9947
- documentsProcessed,
9948
- documentsTotal
9949
- });
9950
- await yieldToEventLoop();
9951
- }
9952
- }
9953
- }
9954
- if (chunkFill > 0) {
9955
- onProgress?.({
9956
- phase: "moving",
9957
- documentsProcessed,
9958
- documentsTotal
9959
- });
9960
- await yieldToEventLoop();
9961
- }
9962
- onProgress?.({
9963
- phase: "swapping",
9964
- documentsProcessed,
9965
- documentsTotal
9966
- });
9967
- manager.setPartitions(newPartitions);
9968
- onProgress?.({
9969
- phase: "complete",
9970
- documentsProcessed,
9971
- documentsTotal
9972
- });
9973
- } finally {
9974
- activeManagers.delete(manager);
9975
- activeCount--;
9976
- }
9977
- }
9978
- return {
9979
- rebalance,
9980
- isRebalancing() {
9981
- return activeCount > 0;
9982
- }
9983
- };
9984
- }
9985
-
9986
- // src/core/hash.ts
9987
- var FNV_OFFSET_BASIS = 2166136261;
9988
- var FNV_PRIME = 16777619;
9989
- var encoder = new TextEncoder();
9990
- function fnv1aBytes(bytes) {
9991
- return fnv1aContinue(FNV_OFFSET_BASIS, bytes, bytes.length) >>> 0;
9992
- }
9993
- function fnv1aContinue(start, bytes, length) {
9994
- let hash = start;
9995
- for (let i = 0; i < length; i++) {
9996
- hash = Math.imul(hash ^ bytes[i], FNV_PRIME);
9997
- }
9998
- return hash;
9999
- }
10000
- var scratchNumber = new Float64Array(1);
10001
- var scratchBytes = new Uint8Array(scratchNumber.buffer);
10002
- var HOST_IS_LITTLE_ENDIAN = new Uint8Array(new Uint16Array([1]).buffer)[0] === 1;
10003
- function fnv1aContinueTaggedNumbers(start, tag, numbers) {
10004
- let hash = start;
10005
- const b = scratchBytes;
10006
- for (let i = 0; i < numbers.length; i++) {
10007
- scratchNumber[0] = Math.fround(numbers[i]);
10008
- hash = Math.imul(hash ^ tag, FNV_PRIME);
10009
- if (HOST_IS_LITTLE_ENDIAN) {
10010
- hash = Math.imul(hash ^ b[7], FNV_PRIME);
10011
- hash = Math.imul(hash ^ b[6], FNV_PRIME);
10012
- hash = Math.imul(hash ^ b[5], FNV_PRIME);
10013
- hash = Math.imul(hash ^ b[4], FNV_PRIME);
10014
- hash = Math.imul(hash ^ b[3], FNV_PRIME);
10015
- hash = Math.imul(hash ^ b[2], FNV_PRIME);
10016
- hash = Math.imul(hash ^ b[1], FNV_PRIME);
10017
- hash = Math.imul(hash ^ b[0], FNV_PRIME);
10018
- } else {
10019
- for (let at = 0; at < 8; at++) hash = Math.imul(hash ^ b[at], FNV_PRIME);
10020
- }
10021
- }
10022
- return hash;
10023
- }
10024
- function fnv1a(input) {
10025
- return fnv1aBytes(encoder.encode(input));
10026
- }
10027
-
10028
- // src/partitioning/router.ts
10029
- function createPartitionRouter() {
10030
- function route(docId, partitionCount) {
10031
- if (partitionCount <= 0) {
10032
- throw new NarsilError(
10033
- ErrorCodes.INDEX_NOT_FOUND,
10034
- `Partition count must be greater than 0, received ${partitionCount}`,
10035
- { partitionCount }
10036
- );
10037
- }
10038
- return fnv1a(docId) % partitionCount;
10039
- }
10040
- function routeBatch(docIds, partitionCount) {
10041
- const groups = /* @__PURE__ */ new Map();
10042
- for (let i = 0; i < docIds.length; i++) {
10043
- const pid = route(docIds[i], partitionCount);
10044
- let group = groups.get(pid);
10045
- if (!group) {
10046
- group = [];
10047
- groups.set(pid, group);
10048
- }
10049
- group.push(docIds[i]);
10050
- }
10051
- return groups;
10052
- }
10053
- return { route, routeBatch };
10054
- }
10055
-
10056
- // src/plugins/registry.ts
10057
- var WRITE_REJECTING_HOOKS = /* @__PURE__ */ new Set(["beforeInsert", "beforeUpdate", "beforeRemove"]);
10058
- function isThenable(value) {
10059
- return value !== null && typeof value === "object" && typeof value.then === "function";
10060
- }
10061
- function thrownText(err, plugin, hookName) {
10062
- if (err instanceof Error) return err.message;
10063
- try {
10064
- return String(err);
10065
- } catch {
10066
- return `The ${hookName} hook of plugin "${plugin}" rejects the write`;
10067
- }
10068
- }
10069
- function asWriteRejection(err, plugin, hookName) {
10070
- if (err instanceof NarsilError) return err;
10071
- return new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, thrownText(err, plugin, hookName), {
10072
- plugin,
10073
- hook: hookName
10074
- });
10075
- }
10076
- function invoke(hook, hookName, context) {
10077
- const rejectsWrites = WRITE_REJECTING_HOOKS.has(hookName);
10078
- let result;
10079
- try {
10080
- result = hook.run(context);
10081
- } catch (err) {
10082
- throw rejectsWrites ? asWriteRejection(err, hook.plugin, hookName) : err;
10083
- }
10084
- if (!isThenable(result)) return null;
10085
- if (!rejectsWrites) return result;
10086
- return Promise.resolve(result).catch((err) => {
10087
- throw asWriteRejection(err, hook.plugin, hookName);
10088
- });
10089
- }
10090
- async function continueAsync(pending, hooks, startIndex, hookName, context) {
10091
- await pending;
10092
- for (let i = startIndex; i < hooks.length; i++) {
10093
- await invoke(hooks[i], hookName, context);
10094
- }
10095
- }
10096
- function createPluginRegistry() {
10097
- const plugins = [];
10098
- return {
10099
- register(plugin) {
10100
- plugins.push(plugin);
10101
- },
10102
- hasHooks(hookName) {
10103
- return plugins.some((p) => typeof p[hookName] === "function");
10104
- },
10105
- runHook(hookName, context) {
10106
- const hooks = [];
10107
- for (const plugin of plugins) {
10108
- const hook = plugin[hookName];
10109
- if (typeof hook === "function") {
10110
- hooks.push({ plugin: plugin.name, run: hook.bind(plugin) });
10111
- }
10112
- }
10113
- if (hooks.length === 0) return;
10114
- for (let i = 0; i < hooks.length; i++) {
10115
- const pending = invoke(hooks[i], hookName, context);
10116
- if (pending !== null) {
10117
- return continueAsync(pending, hooks, i + 1, hookName, context);
10118
- }
10119
- }
10120
- }
10121
- };
10122
- }
10123
-
10124
- // src/rerank/registry.ts
10125
- function validateRerankAdapter(name, adapter) {
10126
- if (typeof name !== "string" || name.trim().length === 0) {
10127
- throw new NarsilError(ErrorCodes.RERANK_CONFIG_INVALID, "A rerank adapter needs a non-empty string name");
10128
- }
10129
- const rerank = typeof adapter === "object" && adapter !== null ? adapter.rerank : void 0;
10130
- if (typeof rerank !== "function") {
10131
- throw new NarsilError(ErrorCodes.RERANK_CONFIG_INVALID, `Rerank adapter "${name}" must provide a rerank function`, {
10132
- adapter: name
10133
- });
10134
- }
10135
- }
10136
- function registerEngineRerankAdapter(core, name, adapter) {
10137
- core.guardShutdown();
10138
- validateRerankAdapter(name, adapter);
10139
- core.rerankAdapters.set(name, adapter);
10140
- }
10141
-
10142
- // src/schema/embedding-validator.ts
10143
- var VECTOR_DIMENSION_PATTERN = /^vector\[(\d+)]$/;
10144
- function validateEmbeddingConfig(embeddingConfig, schema, instanceAdapter) {
10145
- const fieldEntries = Object.keys(embeddingConfig.fields);
10146
- if (fieldEntries.length === 0) {
10147
- throw new NarsilError(
10148
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
10149
- "Embedding config must define at least one field mapping"
10150
- );
10151
- }
10152
- if (typeof embeddingConfig.adapter === "string") {
10153
- throw new NarsilError(
10154
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
10155
- `Embedding adapter "${embeddingConfig.adapter}" must be resolved against the engine registry before validation`,
10156
- { adapter: embeddingConfig.adapter }
10157
- );
10158
- }
10159
- const resolvedAdapter = embeddingConfig.adapter ?? instanceAdapter;
10160
- if (!resolvedAdapter) {
10161
- throw new NarsilError(
10162
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
10163
- "Embedding config requires an adapter (set on the index or the instance)"
10164
- );
10165
- }
10166
- if (typeof resolvedAdapter.dimensions !== "number" || !Number.isInteger(resolvedAdapter.dimensions) || resolvedAdapter.dimensions <= 0) {
10167
- throw new NarsilError(
10168
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
10169
- `Adapter dimensions must be a positive integer, got ${resolvedAdapter.dimensions}`
10170
- );
10171
- }
10172
- const flatFields = flattenSchema(schema);
10173
- for (const [targetField, sourceFields] of Object.entries(embeddingConfig.fields)) {
10174
- const targetType = flatFields[targetField];
10175
- if (targetType === void 0) {
10176
- throw new NarsilError(
10177
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
10178
- `Embedding target field "${targetField}" does not exist in the schema`,
10179
- { field: targetField }
10180
- );
10181
- }
10182
- const vectorMatch = VECTOR_DIMENSION_PATTERN.exec(targetType);
10183
- if (!vectorMatch) {
10184
- throw new NarsilError(
10185
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
10186
- `Embedding target field "${targetField}" must be a vector type, got "${targetType}"`,
10187
- { field: targetField, type: targetType }
10188
- );
10189
- }
10190
- const schemaDimension = Number.parseInt(vectorMatch[1], 10);
10191
- if (schemaDimension !== resolvedAdapter.dimensions) {
10192
- throw new NarsilError(
10193
- ErrorCodes.EMBEDDING_DIMENSION_MISMATCH,
10194
- `Embedding target field "${targetField}" has dimension ${schemaDimension} but the adapter produces ${resolvedAdapter.dimensions}-dimensional vectors`,
10195
- { field: targetField, expected: schemaDimension, actual: resolvedAdapter.dimensions }
10196
- );
10197
- }
10198
- const sources = Array.isArray(sourceFields) ? sourceFields : [sourceFields];
10199
- if (sources.length === 0) {
10200
- throw new NarsilError(
10201
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
10202
- `Embedding target field "${targetField}" must have at least one source field`,
10203
- { field: targetField }
10204
- );
10205
- }
10206
- for (const source of sources) {
10207
- const sourceType = flatFields[source];
10208
- if (sourceType === void 0) {
10209
- throw new NarsilError(
10210
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
10211
- `Embedding source field "${source}" does not exist in the schema`,
10212
- { field: source, targetField }
10213
- );
10214
- }
10215
- if (!isSingleStringFieldType(sourceType)) {
10216
- throw new NarsilError(
10217
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
10218
- `Embedding source field "${source}" must be a string type, got "${sourceType}"`,
10219
- { field: source, type: sourceType, targetField }
10220
- );
10221
- }
10222
- }
10223
- }
10224
- return resolvedAdapter;
10225
- }
10226
- function validateRegisteredAdapter(name, adapter) {
10227
- if (typeof name !== "string" || name.trim().length === 0) {
10228
- throw new NarsilError(ErrorCodes.EMBEDDING_CONFIG_INVALID, "Embedding adapter names must be non-empty strings");
10229
- }
10230
- if (typeof adapter?.embed !== "function") {
10231
- throw new NarsilError(
10232
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
10233
- `Embedding adapter "${name}" must provide an embed function`,
10234
- { adapter: name }
10235
- );
10236
- }
10237
- if (typeof adapter.dimensions !== "number" || !Number.isInteger(adapter.dimensions) || adapter.dimensions <= 0) {
10238
- throw new NarsilError(
10239
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
10240
- `Embedding adapter "${name}" dimensions must be a positive integer, got ${adapter.dimensions}`,
10241
- { adapter: name, dimensions: adapter.dimensions }
10242
- );
10243
- }
10244
- }
10245
- function validateRequiredFieldsInSchema(required, schema) {
10246
- if (required.length === 0) return;
10247
- const flatFields = flattenSchema(schema);
10248
- for (const field of required) {
10249
- if (flatFields[field] === void 0) {
10250
- throw new NarsilError(
10251
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
10252
- `Required field "${field}" does not exist in the schema`,
10253
- { field }
10254
- );
10255
- }
10256
- }
10257
- }
10258
-
10259
- // src/runtime/heap-statistics.browser.ts
10260
- function readHeapStatistics() {
10261
- return null;
10262
- }
10263
-
10264
- // src/search/oversample.ts
10265
- function oversampledShardSize(size) {
10266
- return Math.ceil(size * 1.5) + 10;
10267
- }
10268
-
10269
- // src/search/facets.ts
10270
- function keptValueCount(fieldConfig) {
10271
- return keptFacetValues(fieldConfig?.limit);
10272
- }
10273
- function everyValueFacetConfig(config) {
10274
- const widened = {};
10275
- for (const [field, fieldConfig] of Object.entries(config)) {
10276
- widened[field] = { ...fieldConfig, limit: void 0 };
10277
- }
10278
- return widened;
10279
- }
10280
- function oversampledFacetConfig(config) {
10281
- const widened = {};
10282
- for (const [field, fieldConfig] of Object.entries(config)) {
10283
- if (fieldConfig?.ranges !== void 0 || fieldConfig?.sort === "asc") {
10284
- widened[field] = { ...fieldConfig, limit: void 0 };
10285
- continue;
10286
- }
10287
- const kept = keptValueCount(fieldConfig);
10288
- widened[field] = kept === void 0 ? fieldConfig : { ...fieldConfig, limit: oversampledShardSize(kept) };
10289
- }
10290
- return widened;
10291
- }
10292
- function mergeFacets(partitionFacets, config) {
10293
- const merged = /* @__PURE__ */ new Map();
10294
- const bounds = /* @__PURE__ */ new Map();
10295
- for (const partition of partitionFacets) {
10296
- for (const [field, facetResult] of Object.entries(partition)) {
10297
- let fieldMap = merged.get(field);
10298
- if (!fieldMap) {
10299
- fieldMap = /* @__PURE__ */ new Map();
10300
- merged.set(field, fieldMap);
10301
- }
10302
- bounds.set(field, (bounds.get(field) ?? 0) + facetResult.errorBound);
10303
- for (const [value, count] of Object.entries(facetResult.values)) {
10304
- fieldMap.set(value, (fieldMap.get(value) ?? 0) + count);
10305
- }
10306
- }
10307
- }
10308
- const result = {};
10309
- for (const [field, valueMap] of merged) {
10310
- const fieldConfig = config[field];
10311
- const ascending = fieldConfig?.sort === "asc";
10312
- const ordered = Array.from(valueMap.entries());
10313
- ordered.sort((a, b) => (ascending ? a[1] - b[1] : b[1] - a[1]) || compareCodePoints(a[0], b[0]));
10314
- const kept = Math.min(keptValueCount(fieldConfig) ?? ordered.length, ordered.length);
10315
- let largestLeftOut = 0;
10316
- for (let index = kept; index < ordered.length; index++) {
10317
- if (ordered[index][1] > largestLeftOut) largestLeftOut = ordered[index][1];
10318
- }
10319
- const errorBound = (bounds.get(field) ?? 0) + largestLeftOut;
10320
- const values = {};
10321
- for (let index = 0; index < kept; index++) {
10322
- values[ordered[index][0]] = ordered[index][1];
10108
+ return sortValuesOf(state, docId, fields, fieldTypes, modes);
10109
+ },
10110
+ applyFilters(filters, schema, meter) {
10111
+ return applyPartitionFilters(state, filters, schema, meter);
10112
+ },
10113
+ applyFiltersBitset(filters, schema, meter) {
10114
+ return applyPartitionFiltersBitset(state, filters, schema, meter);
10115
+ },
10116
+ filterMatches(filters, schema, meter) {
10117
+ return partitionFilterMatches(state, filters, schema, meter);
10118
+ },
10119
+ computeFacets(matched, config, schema) {
10120
+ return computeFacets(state, matched, config, schema);
10121
+ },
10122
+ suggestTerms(surfacePrefix, stemmedPrefix, limit) {
10123
+ return suggestDisplayTerms(state, surfacePrefix, stemmedPrefix, limit);
10124
+ },
10125
+ expandTermPrefix(surfacePrefix, stemmedToken, maxExpansions) {
10126
+ return expandTermPrefix([state], surfacePrefix, stemmedToken, maxExpansions);
10127
+ },
10128
+ expandPartialWords(analysis, requests) {
10129
+ return expandPartialWords([state], analysis, requests);
10130
+ },
10131
+ preparePartialWords(analysis) {
10132
+ expandPartialWords([state], analysis, []);
10133
+ },
10134
+ serialize(indexName, totalPartitions, language, schema) {
10135
+ return serializePartition(state, partitionId, indexName, totalPartitions, language, schema);
10136
+ },
10137
+ deserialize(data, schema) {
10138
+ deserializePartition(state, data, clearAll, schema);
10323
10139
  }
10324
- result[field] = { values, count: kept, errorBound };
10325
- }
10326
- return result;
10140
+ };
10141
+ statesByPartition.set(partition, state);
10142
+ return partition;
10327
10143
  }
10328
10144
 
10329
10145
  // src/core/partition/frozen/memory.ts
10330
10146
  function frozenSegmentBytes(segment) {
10147
+ if (segment.paged !== null) return segment.paged.heldBytes(segment);
10331
10148
  const arrays = segment.arrays;
10332
10149
  let bytes = arrays.tokenTable.bytes + arrays.idTable.bytes;
10333
10150
  bytes += arrays.postingOffsets.byteLength;
@@ -10439,6 +10256,71 @@ function compositeSortedDocIds(frozen, live) {
10439
10256
  return merged;
10440
10257
  }
10441
10258
 
10259
+ // src/search/oversample.ts
10260
+ function oversampledShardSize(size) {
10261
+ return Math.ceil(size * 1.5) + 10;
10262
+ }
10263
+
10264
+ // src/search/facets.ts
10265
+ function keptValueCount(fieldConfig) {
10266
+ return keptFacetValues(fieldConfig?.limit);
10267
+ }
10268
+ function everyValueFacetConfig(config) {
10269
+ const widened = {};
10270
+ for (const [field, fieldConfig] of Object.entries(config)) {
10271
+ widened[field] = { ...fieldConfig, limit: void 0 };
10272
+ }
10273
+ return widened;
10274
+ }
10275
+ function oversampledFacetConfig(config) {
10276
+ const widened = {};
10277
+ for (const [field, fieldConfig] of Object.entries(config)) {
10278
+ if (fieldConfig?.ranges !== void 0 || fieldConfig?.sort === "asc") {
10279
+ widened[field] = { ...fieldConfig, limit: void 0 };
10280
+ continue;
10281
+ }
10282
+ const kept = keptValueCount(fieldConfig);
10283
+ widened[field] = kept === void 0 ? fieldConfig : { ...fieldConfig, limit: oversampledShardSize(kept) };
10284
+ }
10285
+ return widened;
10286
+ }
10287
+ function mergeFacets(partitionFacets, config) {
10288
+ const merged = /* @__PURE__ */ new Map();
10289
+ const bounds = /* @__PURE__ */ new Map();
10290
+ for (const partition of partitionFacets) {
10291
+ for (const [field, facetResult] of Object.entries(partition)) {
10292
+ let fieldMap = merged.get(field);
10293
+ if (!fieldMap) {
10294
+ fieldMap = /* @__PURE__ */ new Map();
10295
+ merged.set(field, fieldMap);
10296
+ }
10297
+ bounds.set(field, (bounds.get(field) ?? 0) + facetResult.errorBound);
10298
+ for (const [value, count] of Object.entries(facetResult.values)) {
10299
+ fieldMap.set(value, (fieldMap.get(value) ?? 0) + count);
10300
+ }
10301
+ }
10302
+ }
10303
+ const result = {};
10304
+ for (const [field, valueMap] of merged) {
10305
+ const fieldConfig = config[field];
10306
+ const ascending = fieldConfig?.sort === "asc";
10307
+ const ordered = Array.from(valueMap.entries());
10308
+ ordered.sort((a, b) => (ascending ? a[1] - b[1] : b[1] - a[1]) || compareCodePoints(a[0], b[0]));
10309
+ const kept = Math.min(keptValueCount(fieldConfig) ?? ordered.length, ordered.length);
10310
+ let largestLeftOut = 0;
10311
+ for (let index = kept; index < ordered.length; index++) {
10312
+ if (ordered[index][1] > largestLeftOut) largestLeftOut = ordered[index][1];
10313
+ }
10314
+ const errorBound = (bounds.get(field) ?? 0) + largestLeftOut;
10315
+ const values = {};
10316
+ for (let index = 0; index < kept; index++) {
10317
+ values[ordered[index][0]] = ordered[index][1];
10318
+ }
10319
+ result[field] = { values, count: kept, errorBound };
10320
+ }
10321
+ return result;
10322
+ }
10323
+
10442
10324
  // src/core/partition/composite/filters.ts
10443
10325
  var WORD_BITS = 32;
10444
10326
  function alignToWord(value) {
@@ -10519,6 +10401,29 @@ function subMatchesView(matches, layout, subIndex) {
10519
10401
  };
10520
10402
  }
10521
10403
 
10404
+ // src/core/partition/composite/facets.ts
10405
+ function compositeFacets(subs, layout, matched, config, schema) {
10406
+ const everyValue = everyValueFacetConfig(config);
10407
+ if ("ordinalBitset" in matched) {
10408
+ const ordinalLayout = layout();
10409
+ return mergeFacets(
10410
+ subs.map(
10411
+ (sub, index) => computeFacets(
10412
+ sub,
10413
+ { ordinalBitset: subBitsetView(matched.ordinalBitset, ordinalLayout, index) },
10414
+ everyValue,
10415
+ schema
10416
+ )
10417
+ ),
10418
+ config
10419
+ );
10420
+ }
10421
+ return mergeFacets(
10422
+ subs.map((sub) => computeFacets(sub, matched, everyValue, schema)),
10423
+ config
10424
+ );
10425
+ }
10426
+
10522
10427
  // src/core/partition/scored-merge.ts
10523
10428
  function kWayMerge(arrays) {
10524
10429
  const nonEmpty = arrays.filter((a) => a.length > 0);
@@ -10898,7 +10803,8 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
10898
10803
  }
10899
10804
  function cachedDocFrequencies() {
10900
10805
  if (docFrequenciesCache === null) {
10901
- docFrequenciesCache = mergeDocFrequencies(subs());
10806
+ const paged = frozen.find((segment) => segment.paged !== null)?.paged;
10807
+ docFrequenciesCache = paged ? paged.termFrequenciesAcross(subs) : mergeDocFrequencies(subs());
10902
10808
  }
10903
10809
  return docFrequenciesCache;
10904
10810
  }
@@ -10969,6 +10875,20 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
10969
10875
  frozen.push(segment);
10970
10876
  bindSegments();
10971
10877
  },
10878
+ frozenSegments() {
10879
+ return frozen;
10880
+ },
10881
+ replaceFrozenSegments(segments) {
10882
+ frozen.length = 0;
10883
+ frozen.push(...segments);
10884
+ bindSegments();
10885
+ },
10886
+ documentIdsHeldInMemory() {
10887
+ return compositeDocIds(
10888
+ frozen.filter((segment) => segment.paged === null),
10889
+ live
10890
+ );
10891
+ },
10972
10892
  mergeSegment(segment) {
10973
10893
  invalidateDocFrequencies();
10974
10894
  live.mergeSegment(segment);
@@ -10991,6 +10911,7 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
10991
10911
  live.deserialize(data, schema);
10992
10912
  },
10993
10913
  clear() {
10914
+ for (const segment of frozen) segment.paged?.close();
10994
10915
  frozen.length = 0;
10995
10916
  invalidateDocFrequencies();
10996
10917
  live.clear();
@@ -11057,25 +10978,7 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
11057
10978
  return compositeFilterMatches(subs(), layout(), filters, schema, meter);
11058
10979
  },
11059
10980
  computeFacets(matched, config, schema) {
11060
- const everyValue = everyValueFacetConfig(config);
11061
- if ("ordinalBitset" in matched) {
11062
- const ordinalLayout = layout();
11063
- return mergeFacets(
11064
- subs().map(
11065
- (sub, index) => computeFacets(
11066
- sub,
11067
- { ordinalBitset: subBitsetView(matched.ordinalBitset, ordinalLayout, index) },
11068
- everyValue,
11069
- schema
11070
- )
11071
- ),
11072
- config
11073
- );
11074
- }
11075
- return mergeFacets(
11076
- subs().map((sub) => computeFacets(sub, matched, everyValue, schema)),
11077
- config
11078
- );
10981
+ return compositeFacets(subs(), layout, matched, config, schema);
11079
10982
  },
11080
10983
  sortedPage(request) {
11081
10984
  return compositeSortedPage(subs(), layout(), request);
@@ -11202,6 +11105,36 @@ function createCapacityChecks(source) {
11202
11105
  };
11203
11106
  }
11204
11107
 
11108
+ // src/partitioning/manager/doc-locations.ts
11109
+ function documentIdsHeldInMemory(partition) {
11110
+ return isCompositePartition(partition) ? partition.documentIdsHeldInMemory() : partition.docIds();
11111
+ }
11112
+ function rebuildDocLocations(locations, partitions) {
11113
+ locations.clear();
11114
+ for (let i = 0; i < partitions.length; i++) {
11115
+ for (const docId of documentIdsHeldInMemory(partitions[i])) {
11116
+ locations.set(docId, i);
11117
+ }
11118
+ }
11119
+ }
11120
+ function recordSegmentDocLocations(locations, partitionId, segment) {
11121
+ if (segment.paged !== null) return;
11122
+ for (const internalId of segment.docStore.allInternalIds()) {
11123
+ const docId = segment.docStore.getExternalId(internalId);
11124
+ if (docId !== void 0) locations.set(docId, partitionId);
11125
+ }
11126
+ }
11127
+ function rebuildPartitionDocLocations(locations, partitionId, partition) {
11128
+ for (const [docId, pid] of locations) {
11129
+ if (pid === partitionId) {
11130
+ locations.delete(docId);
11131
+ }
11132
+ }
11133
+ for (const docId of documentIdsHeldInMemory(partition)) {
11134
+ locations.set(docId, partitionId);
11135
+ }
11136
+ }
11137
+
11205
11138
  // src/partitioning/manager/nested-values.ts
11206
11139
  function setNestedValue(obj, path, value) {
11207
11140
  if (!path.includes(".")) {
@@ -11240,6 +11173,109 @@ function requireNewSegmentDocuments(segment, docPartitionMap) {
11240
11173
  }
11241
11174
  }
11242
11175
 
11176
+ // src/partitioning/manager/write-log.ts
11177
+ var REANALYSIS_WRITE_LOG = 0;
11178
+ var CHECKPOINT_WRITE_LOG = 1;
11179
+ function createWriteLogRecorder() {
11180
+ const current = [null, null];
11181
+ return {
11182
+ open(owner) {
11183
+ const log = { docIds: /* @__PURE__ */ new Set(), replaced: false };
11184
+ current[owner] = log;
11185
+ return {
11186
+ get partitionsReplaced() {
11187
+ return log.replaced;
11188
+ },
11189
+ get pendingDocuments() {
11190
+ return log.docIds.size;
11191
+ },
11192
+ takeWrittenDocIds() {
11193
+ const taken = log.docIds;
11194
+ log.docIds = /* @__PURE__ */ new Set();
11195
+ return taken;
11196
+ },
11197
+ close() {
11198
+ if (current[owner] === log) current[owner] = null;
11199
+ }
11200
+ };
11201
+ },
11202
+ documentWritten(docId) {
11203
+ current[0]?.docIds.add(docId);
11204
+ current[1]?.docIds.add(docId);
11205
+ },
11206
+ segmentWritten(segment) {
11207
+ for (const log of current) {
11208
+ if (log === null) continue;
11209
+ for (const internalId of segment.docStore.allInternalIds()) {
11210
+ const docId = segment.docStore.getExternalId(internalId);
11211
+ if (docId !== void 0) log.docIds.add(docId);
11212
+ }
11213
+ }
11214
+ },
11215
+ partitionsReplaced() {
11216
+ for (const log of current) {
11217
+ if (log !== null) log.replaced = true;
11218
+ }
11219
+ }
11220
+ };
11221
+ }
11222
+
11223
+ // src/partitioning/manager/segment-operations.ts
11224
+ function createSegmentOperations(source) {
11225
+ const { docPartitionMap, writeLogs, validatePartitionId, asCompositePartition, resolveInsertOptions } = source;
11226
+ return {
11227
+ mergeSegment(partitionId, payload, documents) {
11228
+ validatePartitionId(partitionId);
11229
+ const partition = source.partitionAt(partitionId);
11230
+ partition.beginBatch();
11231
+ partition.mergeSegmentPayload(payload, documents);
11232
+ partition.endBatch();
11233
+ for (const docId of payload.docIds) {
11234
+ docPartitionMap.set(docId, partitionId);
11235
+ writeLogs.documentWritten(docId);
11236
+ }
11237
+ },
11238
+ attachFrozenSegment(partitionId, segment) {
11239
+ validatePartitionId(partitionId);
11240
+ if (segment.paged === null) requireNewSegmentDocuments(segment, docPartitionMap);
11241
+ asCompositePartition(partitionId).attachFrozenSegment(segment);
11242
+ if (segment.paged !== null) return;
11243
+ recordSegmentDocLocations(docPartitionMap, partitionId, segment);
11244
+ writeLogs.segmentWritten(segment);
11245
+ },
11246
+ openCheckpointChangeLog() {
11247
+ return writeLogs.open(CHECKPOINT_WRITE_LOG);
11248
+ },
11249
+ diskSegmentAccess(partitionId) {
11250
+ validatePartitionId(partitionId);
11251
+ return {
11252
+ composite: asCompositePartition(partitionId),
11253
+ docPartitionMap,
11254
+ insertOptions: resolveInsertOptions()
11255
+ };
11256
+ },
11257
+ swapFrozenSegments(partitionId, dropSegmentIds, replacement, removedSpellings) {
11258
+ validatePartitionId(partitionId);
11259
+ const applied = asCompositePartition(partitionId).swapFrozenSegments(
11260
+ dropSegmentIds,
11261
+ replacement,
11262
+ removedSpellings
11263
+ );
11264
+ recordSegmentDocLocations(docPartitionMap, partitionId, replacement);
11265
+ return applied;
11266
+ },
11267
+ freezeLiveTail(partitionId, freeze) {
11268
+ validatePartitionId(partitionId);
11269
+ return asCompositePartition(partitionId).freezeLiveTail(freeze);
11270
+ },
11271
+ replaceLiveTail(partitionId, segment) {
11272
+ validatePartitionId(partitionId);
11273
+ asCompositePartition(partitionId).replaceLiveTail(segment);
11274
+ recordSegmentDocLocations(docPartitionMap, partitionId, segment);
11275
+ }
11276
+ };
11277
+ }
11278
+
11243
11279
  // src/partitioning/manager/stats.ts
11244
11280
  function aggregateStatsOf(partitions) {
11245
11281
  let totalDocuments = 0;
@@ -11275,59 +11311,27 @@ function partitionStatsOf(partitions) {
11275
11311
  }));
11276
11312
  }
11277
11313
 
11278
- // src/partitioning/manager/write-log.ts
11279
- function createWriteLogRecorder() {
11280
- let current = null;
11281
- return {
11282
- open() {
11283
- const log = { docIds: /* @__PURE__ */ new Set(), replaced: false };
11284
- current = log;
11285
- return {
11286
- get partitionsReplaced() {
11287
- return log.replaced;
11288
- },
11289
- get pendingDocuments() {
11290
- return log.docIds.size;
11291
- },
11292
- takeWrittenDocIds() {
11293
- const taken = log.docIds;
11294
- log.docIds = /* @__PURE__ */ new Set();
11295
- return taken;
11296
- },
11297
- close() {
11298
- if (current === log) current = null;
11299
- }
11300
- };
11301
- },
11302
- documentWritten(docId) {
11303
- if (current !== null) current.docIds.add(docId);
11304
- },
11305
- segmentWritten(segment) {
11306
- if (current === null) return;
11307
- for (const internalId of segment.docStore.allInternalIds()) {
11308
- const docId = segment.docStore.getExternalId(internalId);
11309
- if (docId !== void 0) current.docIds.add(docId);
11310
- }
11311
- },
11312
- partitionsReplaced() {
11313
- if (current !== null) current.replaced = true;
11314
- }
11315
- };
11316
- }
11317
-
11318
11314
  // src/partitioning/manager/index.ts
11319
- function createPartitionManager(indexName, config, language, router, initialPartitionCount, vectorIndexes) {
11315
+ function closePagedSegmentsLeftBehind(earlier, kept) {
11316
+ for (const partition of earlier) {
11317
+ if (!isCompositePartition(partition) || kept.includes(partition)) continue;
11318
+ for (const segment of partition.frozenSegments()) segment.paged?.close();
11319
+ }
11320
+ }
11321
+ function createPartitionManager(indexName, initialConfig, initialLanguage, router, initialPartitionCount, vectorIndexes) {
11322
+ let config = initialConfig;
11323
+ let language = initialLanguage;
11320
11324
  requirePatternSearchFor(config.schema);
11321
11325
  requirePartialWordSearchFor(config.schema);
11322
11326
  const count = initialPartitionCount ?? 1;
11323
- const analysis = resolveIndexAnalysis(config);
11324
- const partialWordAnalysis = resolvePartialWordAnalysis(config, language, analysis);
11325
- const textAnalysis = { language, options: resolvePartitionInsertOptions(config, analysis) };
11327
+ let analysis = resolveIndexAnalysis(config);
11328
+ let partialWordAnalysis = resolvePartialWordAnalysis(config, language, analysis);
11329
+ let textAnalysis = { language, options: resolvePartitionInsertOptions(config, analysis) };
11326
11330
  const trackPositions = config.trackPositions ?? true;
11327
11331
  const vecIndexes = vectorIndexes ?? /* @__PURE__ */ new Map();
11328
11332
  let partitions = [];
11329
11333
  const docPartitionMap = /* @__PURE__ */ new Map();
11330
- const writeLog = createWriteLogRecorder();
11334
+ const writeLogs = createWriteLogRecorder();
11331
11335
  for (let i = 0; i < count; i++) {
11332
11336
  partitions.push(createPartitionIndex(i, trackPositions));
11333
11337
  }
@@ -11347,38 +11351,21 @@ function createPartitionManager(indexName, config, language, router, initialPart
11347
11351
  if (partialWordAnalysis !== null) partitions[partitionId].preparePartialWords(partialWordAnalysis);
11348
11352
  }
11349
11353
  function locateDocument(docId) {
11350
- return docPartitionMap.get(docId);
11351
- }
11352
- function asCompositePartition(partitionId) {
11353
- const existing = partitions[partitionId];
11354
- if (isCompositePartition(existing)) return existing;
11355
- const composite = createCompositePartition(partitionId, trackPositions, existing, textAnalysis);
11356
- partitions[partitionId] = composite;
11357
- return composite;
11358
- }
11359
- function rebuildDocPartitionMap() {
11360
- docPartitionMap.clear();
11354
+ const known = docPartitionMap.get(docId);
11355
+ if (known !== void 0 || config.textStorage !== "disk") return known;
11356
+ const routed = router.route(docId, partitions.length);
11357
+ if (partitions[routed]?.has(docId)) return routed;
11361
11358
  for (let i = 0; i < partitions.length; i++) {
11362
- for (const docId of partitions[i].docIds()) {
11363
- docPartitionMap.set(docId, i);
11364
- }
11365
- }
11366
- }
11367
- function registerSegmentDocuments(partitionId, segment) {
11368
- for (const internalId of segment.docStore.allInternalIds()) {
11369
- const docId = segment.docStore.getExternalId(internalId);
11370
- if (docId !== void 0) docPartitionMap.set(docId, partitionId);
11359
+ if (i !== routed && partitions[i].has(docId)) return i;
11371
11360
  }
11361
+ return void 0;
11372
11362
  }
11373
- function rebuildDocPartitionMapForPartition(partitionId) {
11374
- for (const [docId, pid] of docPartitionMap) {
11375
- if (pid === partitionId) {
11376
- docPartitionMap.delete(docId);
11377
- }
11378
- }
11379
- for (const docId of partitions[partitionId].docIds()) {
11380
- docPartitionMap.set(docId, partitionId);
11381
- }
11363
+ function asCompositePartition(partitionId) {
11364
+ const existing = partitions[partitionId];
11365
+ if (isCompositePartition(existing)) return existing;
11366
+ const composite = createCompositePartition(partitionId, trackPositions, existing, textAnalysis);
11367
+ partitions[partitionId] = composite;
11368
+ return composite;
11382
11369
  }
11383
11370
  const manager = {
11384
11371
  get partitionCount() {
@@ -11417,12 +11404,14 @@ function createPartitionManager(indexName, config, language, router, initialPart
11417
11404
  return [...partitions];
11418
11405
  },
11419
11406
  setPartitions(newPartitions) {
11420
- writeLog.partitionsReplaced();
11407
+ writeLogs.partitionsReplaced();
11408
+ const earlier = partitions;
11421
11409
  partitions = [...newPartitions];
11422
- rebuildDocPartitionMap();
11410
+ closePagedSegmentsLeftBehind(earlier, partitions);
11411
+ rebuildDocLocations(docPartitionMap, partitions);
11423
11412
  },
11424
11413
  openWriteLog() {
11425
- return writeLog.open();
11414
+ return writeLogs.open(REANALYSIS_WRITE_LOG);
11426
11415
  },
11427
11416
  adoptRebuiltPartitions(rebuilt) {
11428
11417
  if (rebuilt.length !== partitions.length) {
@@ -11432,7 +11421,17 @@ function createPartitionManager(indexName, config, language, router, initialPart
11432
11421
  { indexName, partitionCount: partitions.length, rebuiltCount: rebuilt.length }
11433
11422
  );
11434
11423
  }
11424
+ const earlier = partitions;
11435
11425
  partitions = [...rebuilt];
11426
+ closePagedSegmentsLeftBehind(earlier, partitions);
11427
+ },
11428
+ adoptAnalysis(change) {
11429
+ config = change.config;
11430
+ language = change.language;
11431
+ analysis = change.analysis;
11432
+ partialWordAnalysis = change.partialWordAnalysis;
11433
+ textAnalysis = { language, options: resolvePartitionInsertOptions(config, analysis) };
11434
+ manager.synonyms = change.synonyms;
11436
11435
  },
11437
11436
  addPartition() {
11438
11437
  const currentMaxPartitions = config.partitions?.maxPartitions;
@@ -11444,15 +11443,15 @@ function createPartitionManager(indexName, config, language, router, initialPart
11444
11443
  );
11445
11444
  }
11446
11445
  const partition = createPartitionIndex(partitions.length, trackPositions);
11447
- writeLog.partitionsReplaced();
11446
+ writeLogs.partitionsReplaced();
11448
11447
  partitions.push(partition);
11449
11448
  return partition;
11450
11449
  },
11451
11450
  removePartition(partitionId) {
11452
11451
  validatePartitionId(partitionId);
11453
- writeLog.partitionsReplaced();
11452
+ writeLogs.partitionsReplaced();
11454
11453
  partitions.splice(partitionId, 1);
11455
- rebuildDocPartitionMap();
11454
+ rebuildDocLocations(docPartitionMap, partitions);
11456
11455
  },
11457
11456
  trimPartitions(newCount) {
11458
11457
  if (newCount < 1) {
@@ -11464,7 +11463,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
11464
11463
  if (newCount >= partitions.length) {
11465
11464
  return;
11466
11465
  }
11467
- writeLog.partitionsReplaced();
11466
+ writeLogs.partitionsReplaced();
11468
11467
  for (const [docId, pid] of docPartitionMap) {
11469
11468
  if (pid >= newCount) {
11470
11469
  docPartitionMap.delete(docId);
@@ -11474,7 +11473,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
11474
11473
  },
11475
11474
  ...createCapacityChecks({
11476
11475
  indexName,
11477
- config,
11476
+ config: initialConfig,
11478
11477
  partitionCount: () => partitions.length,
11479
11478
  documentCount: () => manager.countDocuments(),
11480
11479
  getPartition: (partitionId) => manager.getPartition(partitionId)
@@ -11487,7 +11486,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
11487
11486
  const insertOpts = resolveInsertOptions(options);
11488
11487
  partitions[pid].insert(docId, document, config.schema, language, insertOpts);
11489
11488
  docPartitionMap.set(docId, pid);
11490
- writeLog.documentWritten(docId);
11489
+ writeLogs.documentWritten(docId);
11491
11490
  },
11492
11491
  removedSpellingsOf(docId) {
11493
11492
  const pid = locateDocument(docId);
@@ -11501,7 +11500,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
11501
11500
  }
11502
11501
  partitions[pid].remove(docId, config.schema, language, resolveInsertOptions(), removedSpellings);
11503
11502
  docPartitionMap.delete(docId);
11504
- writeLog.documentWritten(docId);
11503
+ writeLogs.documentWritten(docId);
11505
11504
  },
11506
11505
  beginBatchRemove() {
11507
11506
  for (let i = 0; i < partitions.length; i++) {
@@ -11521,7 +11520,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
11521
11520
  const updateOpts = resolveInsertOptions(options);
11522
11521
  partitions[pid].update(docId, document, config.schema, language, updateOpts, removedSpellings);
11523
11522
  docPartitionMap.set(docId, pid);
11524
- writeLog.documentWritten(docId);
11523
+ writeLogs.documentWritten(docId);
11525
11524
  },
11526
11525
  get(docId, projection) {
11527
11526
  const pid = locateDocument(docId);
@@ -11561,49 +11560,20 @@ function createPartitionManager(indexName, config, language, router, initialPart
11561
11560
  validatePartitionId(partitionId);
11562
11561
  return partitions[partitionId].serialize(indexName, partitions.length, language.name, config.schema);
11563
11562
  },
11564
- mergeSegment(partitionId, payload, documents) {
11565
- validatePartitionId(partitionId);
11566
- const partition = partitions[partitionId];
11567
- partition.beginBatch();
11568
- partition.mergeSegmentPayload(payload, documents);
11569
- partition.endBatch();
11570
- for (const docId of payload.docIds) {
11571
- docPartitionMap.set(docId, partitionId);
11572
- writeLog.documentWritten(docId);
11573
- }
11574
- },
11575
- attachFrozenSegment(partitionId, segment) {
11576
- validatePartitionId(partitionId);
11577
- requireNewSegmentDocuments(segment, docPartitionMap);
11578
- asCompositePartition(partitionId).attachFrozenSegment(segment);
11579
- registerSegmentDocuments(partitionId, segment);
11580
- writeLog.segmentWritten(segment);
11581
- },
11582
- swapFrozenSegments(partitionId, dropSegmentIds, replacement, removedSpellings) {
11583
- validatePartitionId(partitionId);
11584
- const applied = asCompositePartition(partitionId).swapFrozenSegments(
11585
- dropSegmentIds,
11586
- replacement,
11587
- removedSpellings
11588
- );
11589
- registerSegmentDocuments(partitionId, replacement);
11590
- return applied;
11591
- },
11592
- freezeLiveTail(partitionId, freeze) {
11593
- validatePartitionId(partitionId);
11594
- return asCompositePartition(partitionId).freezeLiveTail(freeze);
11595
- },
11596
- replaceLiveTail(partitionId, segment) {
11597
- validatePartitionId(partitionId);
11598
- asCompositePartition(partitionId).replaceLiveTail(segment);
11599
- registerSegmentDocuments(partitionId, segment);
11600
- },
11563
+ ...createSegmentOperations({
11564
+ docPartitionMap,
11565
+ writeLogs,
11566
+ partitionAt: (partitionId) => partitions[partitionId],
11567
+ validatePartitionId,
11568
+ asCompositePartition,
11569
+ resolveInsertOptions: () => resolveInsertOptions()
11570
+ }),
11601
11571
  deserializePartition(partitionId, data) {
11602
11572
  validatePartitionId(partitionId);
11603
- writeLog.partitionsReplaced();
11573
+ writeLogs.partitionsReplaced();
11604
11574
  partitions[partitionId].deserialize(data, config.schema);
11605
11575
  preparePartialWords(partitionId);
11606
- rebuildDocPartitionMapForPartition(partitionId);
11576
+ rebuildPartitionDocLocations(docPartitionMap, partitionId, partitions[partitionId]);
11607
11577
  },
11608
11578
  getAggregateStats() {
11609
11579
  return aggregateStatsOf(partitions);
@@ -13062,11 +13032,17 @@ function encodeDocumentTableData(documents) {
13062
13032
  }
13063
13033
  function wrapEncodedDocumentTable(data) {
13064
13034
  const { blob, offsets } = data;
13065
- const count = offsets.length - 1;
13066
- const discarded = (data.discardedFields ?? []).map((fieldPath) => fieldPath.split("."));
13035
+ return createEncodedDocumentSource(
13036
+ offsets.length - 1,
13037
+ blob.byteLength + offsets.byteLength,
13038
+ (ordinal) => blob.subarray(offsets[ordinal], offsets[ordinal + 1]),
13039
+ data.discardedFields
13040
+ );
13041
+ }
13042
+ function createEncodedDocumentSource(count, byteLength, bytesAt, discardedFields) {
13043
+ const discarded = (discardedFields ?? []).map((fieldPath) => fieldPath.split("."));
13067
13044
  const touchesDiscarded = (path) => discarded.some((discardedPath) => sharesPrefix(discardedPath, path));
13068
- const bytesAt = (ordinal) => blob.subarray(offsets[ordinal], offsets[ordinal + 1]);
13069
- const docAt = (ordinal) => documentWithOwnBuffers(decodeStoredDocument(bytesAt(ordinal), data.discardedFields));
13045
+ const docAt = (ordinal) => documentWithOwnBuffers(decodeStoredDocument(bytesAt(ordinal), discardedFields));
13070
13046
  function projectedPathAt(ordinal, path) {
13071
13047
  const segments = encodedProjectionPathOf(path);
13072
13048
  if (segments === null) return { found: false, value: void 0 };
@@ -13076,10 +13052,10 @@ function wrapEncodedDocumentTable(data) {
13076
13052
  }
13077
13053
  return {
13078
13054
  count,
13079
- byteLength: blob.byteLength + offsets.byteLength,
13055
+ byteLength,
13080
13056
  docAt,
13081
13057
  readAt(ordinal) {
13082
- return decodeStoredDocument(bytesAt(ordinal), data.discardedFields);
13058
+ return decodeStoredDocument(bytesAt(ordinal), discardedFields);
13083
13059
  },
13084
13060
  fieldReader(fieldPath) {
13085
13061
  const path = fieldPath.split(".");
@@ -13159,9 +13135,9 @@ function basicPlaneUtf8Bytes(text) {
13159
13135
  return bytes;
13160
13136
  }
13161
13137
  function encodeStringBlob(strings) {
13162
- const joined2 = strings.join("");
13163
- if (SURROGATE.test(joined2)) return encodeOneByOne(strings);
13164
- const blob = encoder3.encode(joined2);
13138
+ const joined = strings.join("");
13139
+ if (SURROGATE.test(joined)) return encodeOneByOne(strings);
13140
+ const blob = encoder3.encode(joined);
13165
13141
  const offsets = new Uint32Array(strings.length + 1);
13166
13142
  let cursor = 0;
13167
13143
  for (let i = 0; i < strings.length; i++) {
@@ -13242,6 +13218,9 @@ function wrapExternalIdTable(data) {
13242
13218
  const ordinal = sortedOrdinals[lo];
13243
13219
  return compareBytes(blob, offsets[ordinal], offsets[ordinal + 1], query) === 0 ? ordinal : -1;
13244
13220
  },
13221
+ sortedOrdinalAt(position) {
13222
+ return sortedOrdinals[position];
13223
+ },
13245
13224
  collectSortedIds(excluded) {
13246
13225
  const ids = [];
13247
13226
  for (let position = 0; position < count; position++) {
@@ -13261,6 +13240,7 @@ function buildExternalIdTable(docIds) {
13261
13240
  characters += docIds[ordinal].length;
13262
13241
  }
13263
13242
  let sortedIds = null;
13243
+ let sortedOrdinals = null;
13264
13244
  return {
13265
13245
  count: docIds.length,
13266
13246
  bytes: characters,
@@ -13270,6 +13250,10 @@ function buildExternalIdTable(docIds) {
13270
13250
  ordinalOf(docId) {
13271
13251
  return ordinals.get(docId) ?? -1;
13272
13252
  },
13253
+ sortedOrdinalAt(position) {
13254
+ if (sortedOrdinals === null) sortedOrdinals = codePointOrder(docIds);
13255
+ return sortedOrdinals[position];
13256
+ },
13273
13257
  collectSortedIds(excluded) {
13274
13258
  if (sortedIds === null) {
13275
13259
  const ids = [...docIds];
@@ -13841,6 +13825,10 @@ function createFrozenRemovalPass(documents, surfaces) {
13841
13825
  });
13842
13826
  return spellings;
13843
13827
  }
13828
+ function subtract(ordinal, analysis) {
13829
+ if (surfaces.tableStaysOnDisk()) surfaces.subtractOnceLoaded(ordinal, analysis);
13830
+ else surfaces.subtractSpellings(ordinal, heldSpellingsOf(ordinal, analysis));
13831
+ }
13844
13832
  return {
13845
13833
  get settled() {
13846
13834
  return pending.length === 0;
@@ -13849,17 +13837,17 @@ function createFrozenRemovalPass(documents, surfaces) {
13849
13837
  if (bound !== null) return;
13850
13838
  const scoped = { ...analysis, fields: documents.textFields };
13851
13839
  bound = scoped;
13852
- for (const ordinal of pending) surfaces.subtractSpellings(ordinal, heldSpellingsOf(ordinal, scoped));
13840
+ for (const ordinal of pending) subtract(ordinal, scoped);
13853
13841
  pending = [];
13854
13842
  },
13855
13843
  spellingsOf(ordinal) {
13856
- return bound === null ? void 0 : heldSpellingsOf(ordinal, bound);
13844
+ return bound === null || surfaces.tableStaysOnDisk() ? void 0 : heldSpellingsOf(ordinal, bound);
13857
13845
  },
13858
13846
  afterRemoval(ordinal, spellings) {
13859
13847
  if (spellings !== void 0) {
13860
13848
  surfaces.subtractSpellings(ordinal, spellings);
13861
13849
  } else if (bound !== null) {
13862
- surfaces.subtractSpellings(ordinal, heldSpellingsOf(ordinal, bound));
13850
+ subtract(ordinal, bound);
13863
13851
  } else {
13864
13852
  pending.push(ordinal);
13865
13853
  }
@@ -13883,15 +13871,32 @@ function addForms(target, forms) {
13883
13871
  if (Array.isArray(value)) target.add(surface, value[1], value[0]);
13884
13872
  }
13885
13873
  }
13886
- function createFrozenSurfaceSlot(load, documents) {
13874
+ function createFrozenSurfaceSlot(load, documents, tableOnDisk) {
13887
13875
  let registry = null;
13888
13876
  let uncounted = null;
13877
+ let table = tableOnDisk;
13878
+ let waitingRemovals = [];
13879
+ let waitingAnalysis = null;
13880
+ function subtractWaitingRemovals(target) {
13881
+ if (waitingAnalysis === null || target.size() === 0) return;
13882
+ for (const ordinal of waitingRemovals) {
13883
+ const spellings = [];
13884
+ visitDocumentSurfaceSpellings(documents.documentAt(ordinal), waitingAnalysis, (spelling) => {
13885
+ if (target.holds(spelling)) spellings.push(spelling);
13886
+ });
13887
+ for (const spelling of spellings) target.subtract(spelling, 1);
13888
+ }
13889
+ }
13889
13890
  function loaded3() {
13890
13891
  if (registry === null) {
13891
13892
  const built = createSurfaceRegistry();
13892
13893
  const forms = load();
13893
13894
  if (forms !== null) built.deserialize(forms);
13895
+ subtractWaitingRemovals(built);
13894
13896
  registry = built;
13897
+ table = void 0;
13898
+ waitingRemovals = [];
13899
+ waitingAnalysis = null;
13895
13900
  }
13896
13901
  return registry;
13897
13902
  }
@@ -13899,12 +13904,22 @@ function createFrozenSurfaceSlot(load, documents) {
13899
13904
  return uncounted === null || uncounted[ordinal] === 0;
13900
13905
  }
13901
13906
  return {
13907
+ heldBytes: () => table?.heldBytes() ?? 0,
13902
13908
  reader: {
13903
- candidatesForPrefix: (prefix) => loaded3().candidatesForPrefix(prefix),
13909
+ candidatesForPrefix: (prefix) => table !== void 0 && waitingRemovals.length === 0 ? table.candidatesForPrefix(prefix) : loaded3().candidatesForPrefix(prefix),
13904
13910
  stemChangedTotalFor: (token) => loaded3().stemChangedTotalFor(token),
13905
13911
  size: () => loaded3().size(),
13906
13912
  serialize: () => loaded3().serialize()
13907
13913
  },
13914
+ tableStaysOnDisk() {
13915
+ return registry === null && tableOnDisk !== void 0;
13916
+ },
13917
+ subtractOnceLoaded(ordinal, analysis) {
13918
+ uncounted ??= new Uint8Array(documents.count);
13919
+ uncounted[ordinal] = 1;
13920
+ waitingAnalysis = analysis;
13921
+ waitingRemovals.push(ordinal);
13922
+ },
13908
13923
  holdsAny() {
13909
13924
  return loaded3().size() > 0;
13910
13925
  },
@@ -14052,6 +14067,9 @@ function wrapFrozenTokenTable(data) {
14052
14067
  }
14053
14068
  return token;
14054
14069
  },
14070
+ tokenBytesAt(sortedIndex) {
14071
+ return blob.subarray(offsets[sortedIndex], offsets[sortedIndex + 1]);
14072
+ },
14055
14073
  byteLengthAt(sortedIndex) {
14056
14074
  return offsets[sortedIndex + 1] - offsets[sortedIndex];
14057
14075
  },
@@ -14326,10 +14344,10 @@ function buildGeoReaders(entries) {
14326
14344
  }
14327
14345
  return readers;
14328
14346
  }
14329
- function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot, partialWordTable, survivingSurfaceForms) {
14347
+ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot, partialWordTable, survivingSurfaceForms, pagedParts) {
14330
14348
  const tombstones = createFrozenTombstones();
14331
- const counts = createFrozenLiveCounts(source, source.tokenTable, tombstones, storedFrequenciesOf(source));
14332
- const postingViews = createFrozenPostingViews(source, tombstones, counts);
14349
+ const counts = pagedParts?.createCounts(tombstones) ?? createFrozenLiveCounts(source, source.tokenTable, tombstones, storedFrequenciesOf(source));
14350
+ const postingViews = pagedParts?.createViews(tombstones, counts) ?? createFrozenPostingViews(source, tombstones, counts);
14333
14351
  const lengthColumns = /* @__PURE__ */ new Map();
14334
14352
  for (let i = 0; i < source.fieldLengthNames.length; i++) {
14335
14353
  lengthColumns.set(source.fieldLengthNames[i], source.fieldLengthColumns[i]);
@@ -14346,7 +14364,11 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
14346
14364
  documentAt: (ordinal) => documentSource.readAt(ordinal)
14347
14365
  };
14348
14366
  const loadSurfaceForms = survivingSurfaceForms === void 0 ? source.readSurfaceForms : () => survivingSurfaceForms;
14349
- const surfaces = createFrozenSurfaceSlot(loadSurfaceForms, textDocuments);
14367
+ const surfaces = createFrozenSurfaceSlot(
14368
+ loadSurfaceForms,
14369
+ textDocuments,
14370
+ survivingSurfaceForms === void 0 ? source.surfaceTableOnDisk : void 0
14371
+ );
14350
14372
  const removals = createFrozenRemovalPass(textDocuments, surfaces);
14351
14373
  const numericIndexes = /* @__PURE__ */ new Map();
14352
14374
  for (const entry of source.numeric) numericIndexes.set(entry.fieldPath, createFrozenNumericReader(entry));
@@ -14368,6 +14390,7 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
14368
14390
  segmentId,
14369
14391
  documentSource,
14370
14392
  sharedSnapshot,
14393
+ paged: pagedParts?.access ?? null,
14371
14394
  arrays: source,
14372
14395
  invertedIdx: createFrozenInvertedReader(source.tokenTable, postingViews, counts),
14373
14396
  docStore,
@@ -14383,7 +14406,7 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
14383
14406
  sizeIndexes,
14384
14407
  fieldNameTable,
14385
14408
  partialWords,
14386
- trackPositions: source.positionOffsets !== null,
14409
+ trackPositions: pagedParts?.access.snapshot.tracksPositions ?? source.positionOffsets !== null,
14387
14410
  flatSchemaCache: null,
14388
14411
  lastSchemaRef: null,
14389
14412
  sortColumns: null,
@@ -14427,6 +14450,9 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
14427
14450
  };
14428
14451
  return segment;
14429
14452
  }
14453
+ function assemblePagedFrozenSegment(segmentId, source, documentSource, partialWordTable, parts) {
14454
+ return assembleFrozenSegment(segmentId, source, documentSource, null, partialWordTable, void 0, parts);
14455
+ }
14430
14456
  function createFrozenSegment(payload, documents, segmentId) {
14431
14457
  const tokenTable = buildFrozenTokenTable(payload.tokens, payload.docFrequencies);
14432
14458
  return assembleFrozenSegment(
@@ -15290,7 +15316,20 @@ function buildSegmentPayload(documents, schema, language, options, trackPosition
15290
15316
 
15291
15317
  // src/workers/segment-actions.ts
15292
15318
  function isSegmentAction(action) {
15293
- return action.type === "buildSegment" || action.type === "mergeSegments" || action.type === "attachSegments" || action.type === "compactSegments" || action.type === "swapSegments" || action.type === "freezeLiveTail";
15319
+ return action.type === "buildSegment" || action.type === "mergeSegments" || action.type === "attachSegments" || action.type === "compactSegments" || action.type === "swapSegments" || action.type === "freezeLiveTail" || action.type === "adoptDiskSegments";
15320
+ }
15321
+ function isPagedSnapshot(snapshot) {
15322
+ return "path" in snapshot;
15323
+ }
15324
+ async function attachSegments(entry, action) {
15325
+ for (const segment of action.segments) {
15326
+ growPartitionsTo(entry.manager, segment.partitionId);
15327
+ const surfaceForms = segment.survivingSurfaceForms;
15328
+ const spellings = surfaceForms === void 0 ? void 0 : NO_REMOVED_SPELLINGS;
15329
+ const frozen = isPagedSnapshot(segment.snapshot) ? (await import('./module-BUFNZMEU.mjs')).createPagedFrozenSegment(segment.snapshot) : createSharedFrozenSegment(segment.snapshot, surfaceForms);
15330
+ for (const docId of segment.tombstonedDocIds ?? []) frozen.tombstoneDocument(docId, spellings);
15331
+ entry.manager.attachFrozenSegment(segment.partitionId, frozen);
15332
+ }
15294
15333
  }
15295
15334
  function growPartitionsTo(manager, partitionId) {
15296
15335
  while (manager.partitionCount <= partitionId) {
@@ -15339,17 +15378,10 @@ function runSegmentAction(entry, action) {
15339
15378
  }
15340
15379
  return void 0;
15341
15380
  }
15342
- case "attachSegments": {
15343
- for (const segment of action.segments) {
15344
- growPartitionsTo(entry.manager, segment.partitionId);
15345
- const surfaceForms = segment.survivingSurfaceForms;
15346
- const frozen = createSharedFrozenSegment(segment.snapshot, surfaceForms);
15347
- const spellings = surfaceForms === void 0 ? void 0 : NO_REMOVED_SPELLINGS;
15348
- for (const docId of segment.tombstonedDocIds ?? []) frozen.tombstoneDocument(docId, spellings);
15349
- entry.manager.attachFrozenSegment(segment.partitionId, frozen);
15350
- }
15351
- return void 0;
15352
- }
15381
+ case "attachSegments":
15382
+ return attachSegments(entry, action);
15383
+ case "adoptDiskSegments":
15384
+ return import('./module-BUFNZMEU.mjs').then((code) => code.adoptSegmentsOnWorker(entry.manager, action));
15353
15385
  case "compactSegments": {
15354
15386
  const partition = requireComposite(entry, action.indexName, action.partitionId, "compact");
15355
15387
  return mergeFrozenSegments(
@@ -15397,7 +15429,7 @@ function requireLoadedVectorCode() {
15397
15429
  function loadVectorCode() {
15398
15430
  if (loaded2 !== null) return Promise.resolve(loaded2);
15399
15431
  if (loading2 === null) {
15400
- loading2 = import('./module-4HIMOQM5.mjs').then(
15432
+ loading2 = import('./module-BVP4TKZU.mjs').then(
15401
15433
  (code) => {
15402
15434
  loaded2 = code;
15403
15435
  return code;
@@ -15473,7 +15505,7 @@ function closeHeldVectorCopies(held) {
15473
15505
  held.fields.clear();
15474
15506
  held.searchers.clear();
15475
15507
  }
15476
- function yieldToEventLoop2() {
15508
+ function yieldToEventLoop() {
15477
15509
  return new Promise((resolve) => {
15478
15510
  if (typeof setImmediate === "function") setImmediate(resolve);
15479
15511
  else setTimeout(resolve, 0);
@@ -15484,7 +15516,7 @@ async function insertIntoHeldGraph(held, fieldName, handle, ordinals) {
15484
15516
  if (field === void 0) return null;
15485
15517
  for (const ordinal of ordinals) {
15486
15518
  field.view.insertOrdinal(ordinal);
15487
- await yieldToEventLoop2();
15519
+ await yieldToEventLoop();
15488
15520
  }
15489
15521
  return field.view.takeOutcome();
15490
15522
  }
@@ -15492,6 +15524,34 @@ function heldVectorOf(held, fieldName, docId) {
15492
15524
  return held.fields.get(fieldName)?.view.vectorOf(docId);
15493
15525
  }
15494
15526
 
15527
+ // src/workers/direct-executor/entry.ts
15528
+ function indexEntryOf(manager, vectorIndexes, analysisStale, awaitedCopyLoad) {
15529
+ return {
15530
+ manager,
15531
+ get config() {
15532
+ return manager.config;
15533
+ },
15534
+ get language() {
15535
+ return manager.language;
15536
+ },
15537
+ searchOptions: {
15538
+ get stopWords() {
15539
+ return manager.analysis.stopWords;
15540
+ },
15541
+ get customTokenizer() {
15542
+ return manager.analysis.customTokenizer;
15543
+ },
15544
+ get synonyms() {
15545
+ return manager.synonyms;
15546
+ }
15547
+ },
15548
+ vectorIndexes,
15549
+ vectorCopies: createHeldVectorCopies(),
15550
+ analysisStale,
15551
+ awaitedCopyLoad
15552
+ };
15553
+ }
15554
+
15495
15555
  // src/workers/direct-executor/index.ts
15496
15556
  var NO_VECTOR_WORKER_COPIES = { enabled: false };
15497
15557
  function createDirectExecutor(options) {
@@ -15543,22 +15603,7 @@ function createDirectExecutor(options) {
15543
15603
  const partitionCount = config.partitions?.maxPartitions ?? 1;
15544
15604
  const vectorIndexes = vectorIndexesFor(indexName, config, copies);
15545
15605
  const manager = createPartitionManager(indexName, config, language, router, partitionCount, vectorIndexes);
15546
- return {
15547
- manager,
15548
- config,
15549
- language,
15550
- searchOptions: {
15551
- stopWords: manager.analysis.stopWords,
15552
- customTokenizer: manager.analysis.customTokenizer,
15553
- get synonyms() {
15554
- return manager.synonyms;
15555
- }
15556
- },
15557
- vectorIndexes,
15558
- vectorCopies: createHeldVectorCopies(),
15559
- analysisStale,
15560
- awaitedCopyLoad
15561
- };
15606
+ return indexEntryOf(manager, vectorIndexes, analysisStale, awaitedCopyLoad);
15562
15607
  }
15563
15608
  function disposeEntry(entry) {
15564
15609
  for (const vectorIndex of entry.vectorIndexes.values()) {
@@ -15779,40 +15824,73 @@ function createDirectExecutor(options) {
15779
15824
  };
15780
15825
  }
15781
15826
 
15827
+ // src/engine/orchestration/turn.ts
15828
+ function afterCurrentTurn(callback) {
15829
+ if (typeof setImmediate === "function") {
15830
+ setImmediate(callback);
15831
+ return;
15832
+ }
15833
+ setTimeout(callback, 0);
15834
+ }
15835
+ function yieldToEventLoop2() {
15836
+ return new Promise((resolve) => afterCurrentTurn(resolve));
15837
+ }
15838
+
15839
+ // src/partitioning/constants.ts
15840
+ var REBALANCE_CHUNK_SIZE = 1e3;
15841
+ var DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE = 1e4;
15842
+ var TEXT_REBUILD_CHUNK_DOCUMENTS = 1e3;
15843
+ var TEXT_REBUILD_SWITCH_BACKLOG_DOCUMENTS = 1e3;
15844
+
15845
+ // src/partitioning/rewrite-spill-of.ts
15846
+ async function rewriteSpillOf(storage, manager, target) {
15847
+ if (storage === void 0 || (target?.config ?? manager.config).textStorage !== "disk") return null;
15848
+ const { openRewriteSpill } = await import('./module-BUFNZMEU.mjs');
15849
+ return openRewriteSpill(storage, manager, target);
15850
+ }
15851
+
15782
15852
  // src/partitioning/text-rebuild.ts
15853
+ function textRebuildTargetOf(config, language, analysis) {
15854
+ return { config, language, analysis, partialWordAnalysis: resolvePartialWordAnalysis(config, language, analysis) };
15855
+ }
15783
15856
  function interrupted(pass) {
15784
15857
  return pass.log.partitionsReplaced || !pass.hooks.isCurrent();
15785
15858
  }
15786
15859
  async function buildPartition(pass, source, partitionId) {
15787
- const { manager, options } = pass;
15788
- const target = createPartitionIndex(partitionId, manager.config.trackPositions ?? true);
15860
+ const { manager, language, options } = pass;
15861
+ const target = pass.spill?.newPartition(partitionId) ?? createPartitionIndex(partitionId, manager.config.trackPositions ?? true);
15789
15862
  const docIds = [...source.docIds()];
15790
15863
  target.beginBatch();
15791
15864
  for (let start = 0; start < docIds.length; start += TEXT_REBUILD_CHUNK_DOCUMENTS) {
15792
15865
  const end = Math.min(start + TEXT_REBUILD_CHUNK_DOCUMENTS, docIds.length);
15793
15866
  for (let i = start; i < end; i++) {
15794
15867
  const document = manager.getRef(docIds[i]);
15795
- if (document !== void 0) target.insert(docIds[i], document, manager.schema, manager.language, options);
15868
+ if (document !== void 0) target.insert(docIds[i], document, manager.schema, language, options);
15796
15869
  }
15797
- await yieldToEventLoop();
15870
+ if (pass.spill !== null) {
15871
+ target.endBatch();
15872
+ await pass.spill.spillIfFull(target, partitionId);
15873
+ target.beginBatch();
15874
+ }
15875
+ await yieldToEventLoop2();
15798
15876
  if (interrupted(pass)) return null;
15799
15877
  }
15800
15878
  target.endBatch();
15801
- if (manager.partialWordAnalysis !== null) target.preparePartialWords(manager.partialWordAnalysis);
15879
+ if (pass.partialWordAnalysis !== null) target.preparePartialWords(pass.partialWordAnalysis);
15802
15880
  return target;
15803
15881
  }
15804
15882
  function applyWrittenDocuments(pass, built) {
15805
- const { manager, options } = pass;
15883
+ const { manager, language, options } = pass;
15806
15884
  for (const docId of pass.log.takeWrittenDocIds()) {
15807
15885
  for (const partition of built) {
15808
15886
  if (!partition.has(docId)) continue;
15809
- partition.remove(docId, manager.schema, manager.language, options);
15887
+ partition.remove(docId, manager.schema, language, options);
15810
15888
  break;
15811
15889
  }
15812
15890
  const partitionId = manager.partitionIdOf(docId);
15813
15891
  const document = manager.getRef(docId);
15814
15892
  if (partitionId === void 0 || document === void 0) continue;
15815
- built[partitionId].insert(docId, document, manager.schema, manager.language, options);
15893
+ built[partitionId].insert(docId, document, manager.schema, language, options);
15816
15894
  }
15817
15895
  }
15818
15896
  async function rebuildOnce(pass) {
@@ -15826,22 +15904,48 @@ async function rebuildOnce(pass) {
15826
15904
  }
15827
15905
  while (pass.log.pendingDocuments > TEXT_REBUILD_SWITCH_BACKLOG_DOCUMENTS) {
15828
15906
  applyWrittenDocuments(pass, built);
15829
- await yieldToEventLoop();
15907
+ await yieldToEventLoop2();
15830
15908
  if (interrupted(pass)) return false;
15831
15909
  }
15832
- applyWrittenDocuments(pass, built);
15833
- pass.manager.adoptRebuiltPartitions(built);
15834
- pass.hooks.onSwitch();
15835
- return true;
15910
+ const keepCurrent = () => {
15911
+ if (interrupted(pass)) return false;
15912
+ applyWrittenDocuments(pass, built);
15913
+ return true;
15914
+ };
15915
+ if (pass.hooks.holdBeforeSwitch !== void 0 && !await pass.hooks.holdBeforeSwitch(keepCurrent)) return false;
15916
+ const swap = () => {
15917
+ if (interrupted(pass)) return false;
15918
+ applyWrittenDocuments(pass, built);
15919
+ pass.manager.adoptRebuiltPartitions(built);
15920
+ pass.hooks.onSwitch();
15921
+ return true;
15922
+ };
15923
+ return pass.hooks.runSwitch === void 0 ? swap() : pass.hooks.runSwitch(swap);
15836
15924
  }
15837
- async function rebuildTextIndexes(manager, hooks) {
15838
- const options = resolvePartitionInsertOptions(manager.config, manager.analysis, { validate: false, skipClone: true });
15925
+ async function rebuildTextIndexes(manager, hooks, target) {
15926
+ const options = resolvePartitionInsertOptions(
15927
+ target?.config ?? manager.config,
15928
+ target?.analysis ?? manager.analysis,
15929
+ {
15930
+ validate: false,
15931
+ skipClone: true
15932
+ }
15933
+ );
15934
+ const language = target?.language ?? manager.language;
15935
+ const partialWordAnalysis = target === void 0 ? manager.partialWordAnalysis : target.partialWordAnalysis;
15936
+ const spill = await rewriteSpillOf(hooks.rewriteStorage?.(), manager, target);
15839
15937
  while (hooks.isCurrent()) {
15840
15938
  const log = manager.openWriteLog();
15939
+ let swapped = false;
15841
15940
  try {
15842
- if (await rebuildOnce({ manager, log, hooks, options })) return true;
15941
+ swapped = await rebuildOnce({ manager, log, hooks, language, partialWordAnalysis, options, spill });
15942
+ if (swapped) {
15943
+ hooks.onRewriteSpill?.(spill);
15944
+ return true;
15945
+ }
15843
15946
  } finally {
15844
15947
  log.close();
15948
+ if (!swapped) await spill?.abandon();
15845
15949
  }
15846
15950
  }
15847
15951
  return false;
@@ -15875,8 +15979,10 @@ function toError(value) {
15875
15979
  function wireAnalysisRebuild(wiring) {
15876
15980
  return createAnalysisRebuildCoordinator(wiring.config, {
15877
15981
  getManager: wiring.getManager,
15982
+ rewriteStorage: () => wiring.durabilityManager?.rewriteStorage?.(),
15878
15983
  desyncIndex: wiring.desyncIndex,
15879
15984
  resyncIndex: wiring.resyncIndex,
15985
+ hasPendingChange: wiring.hasPendingChange,
15880
15986
  async persistAnalysisRevision(indexName) {
15881
15987
  const durability = wiring.durabilityManager;
15882
15988
  if (durability === null) {
@@ -15894,6 +16000,7 @@ function wireAnalysisRebuild(wiring) {
15894
16000
  }
15895
16001
  function createAnalysisRebuildCoordinator(config, deps) {
15896
16002
  const stale = /* @__PURE__ */ new Map();
16003
+ const exclusiveRuns = /* @__PURE__ */ new Map();
15897
16004
  const rebuildMode = config?.rebuild ?? "auto";
15898
16005
  let queue = Promise.resolve();
15899
16006
  let active = null;
@@ -15906,8 +16013,13 @@ function createAnalysisRebuildCoordinator(config, deps) {
15906
16013
  const partitionCount = manager.partitionCount;
15907
16014
  deps.emit({ indexName, status: "started", partitionsRebuilt: 0, partitionCount });
15908
16015
  const wasPromoted = deps.desyncIndex(indexName);
16016
+ let spill = null;
15909
16017
  try {
15910
16018
  const rebuilt = await rebuildTextIndexes(manager, {
16019
+ rewriteStorage: () => deps.rewriteStorage?.(),
16020
+ onRewriteSpill(kept) {
16021
+ spill = kept;
16022
+ },
15911
16023
  isCurrent: () => deps.getManager(indexName) === manager && stale.get(indexName) === entry && !entry.abandoned,
15912
16024
  onPartitionBuilt(partitionsBuilt) {
15913
16025
  entry.partitionsRebuilt = partitionsBuilt;
@@ -15922,6 +16034,7 @@ function createAnalysisRebuildCoordinator(config, deps) {
15922
16034
  return;
15923
16035
  }
15924
16036
  await deps.persistAnalysisRevision(indexName);
16037
+ await spill?.removeUnread(manager);
15925
16038
  await deps.resyncIndex(indexName, wasPromoted);
15926
16039
  deps.emit({
15927
16040
  indexName,
@@ -15969,7 +16082,7 @@ function createAnalysisRebuildCoordinator(config, deps) {
15969
16082
  await config?.onStaleAnalysis?.({ ...entry.index, documentCount }, () => enqueue(indexName, entry));
15970
16083
  }
15971
16084
  async function review(indexName, entry) {
15972
- if (entry.run !== null || deps.getManager(indexName) === void 0) return;
16085
+ if (entry.run !== null || deps.getManager(indexName) === void 0 || deps.hasPendingChange(indexName)) return;
15973
16086
  try {
15974
16087
  await announce(indexName, entry);
15975
16088
  } catch (err) {
@@ -16001,8 +16114,22 @@ function createAnalysisRebuildCoordinator(config, deps) {
16001
16114
  return stale.has(indexName);
16002
16115
  },
16003
16116
  isRunning(indexName) {
16117
+ if ((exclusiveRuns.get(indexName) ?? 0) > 0) return true;
16004
16118
  return stale.get(indexName)?.run !== null && stale.get(indexName)?.run !== void 0;
16005
16119
  },
16120
+ runExclusive(indexName, work, turnEnds) {
16121
+ exclusiveRuns.set(indexName, (exclusiveRuns.get(indexName) ?? 0) + 1);
16122
+ const run = queue.then(work).finally(() => {
16123
+ const remaining = (exclusiveRuns.get(indexName) ?? 1) - 1;
16124
+ if (remaining === 0) exclusiveRuns.delete(indexName);
16125
+ else exclusiveRuns.set(indexName, remaining);
16126
+ });
16127
+ queue = Promise.race([run, turnEnds ?? run]).then(
16128
+ () => void 0,
16129
+ () => void 0
16130
+ );
16131
+ return run;
16132
+ },
16006
16133
  async abandon(indexName) {
16007
16134
  const entry = stale.get(indexName);
16008
16135
  if (entry !== void 0 && entry.run !== null) entry.abandoned = true;
@@ -16181,6 +16308,9 @@ function reconstructSchemaFromMetadata(metadata) {
16181
16308
  if (metadata.trackPositions !== void 0) {
16182
16309
  config.trackPositions = metadata.trackPositions;
16183
16310
  }
16311
+ if (metadata.textStorage !== void 0) {
16312
+ config.textStorage = metadata.textStorage;
16313
+ }
16184
16314
  if (metadata.strict !== void 0) {
16185
16315
  config.strict = metadata.strict;
16186
16316
  }
@@ -16401,6 +16531,10 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
16401
16531
  if (loadingVectorCode !== null) await loadingVectorCode;
16402
16532
  const existing = deps.indexRegistry.get(metadata.indexName);
16403
16533
  if (existing !== void 0) {
16534
+ if ((metadata.analysisGeneration ?? 0) > existing.analysisGeneration) {
16535
+ const { adoptSavedAnalysis } = await import('./analysis-change-JPV775LR.mjs');
16536
+ adoptSavedAnalysis(deps.executor.getManager(metadata.indexName), existing, metadata);
16537
+ }
16404
16538
  if (loadData && deps.executor.getManager(metadata.indexName) === void 0) {
16405
16539
  deps.executor.createIndex(
16406
16540
  metadata.indexName,
@@ -16431,6 +16565,7 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
16431
16565
  const synonyms = prepared?.encoded ?? null;
16432
16566
  try {
16433
16567
  validateVectorStorage(indexConfig.vectorPromotion, deps.filesystemDurability);
16568
+ validateTextStorage(indexConfig.textStorage, deps.filesystemDurability);
16434
16569
  if (loadData) {
16435
16570
  deps.executor.createIndex(metadata.indexName, indexConfig, language, false, prepared?.lookup ?? null);
16436
16571
  }
@@ -16449,7 +16584,10 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
16449
16584
  partitionCount: metadata.partitionCount,
16450
16585
  analysisRevision: metadata.analysisRevision ?? null,
16451
16586
  arrayPositionGap: metadata.arrayPositionGap ?? null,
16452
- synonyms
16587
+ synonyms,
16588
+ analysisGeneration: metadata.analysisGeneration ?? 0,
16589
+ pendingAnalysis: metadata.pendingAnalysis ?? null,
16590
+ durableAnalysis: null
16453
16591
  });
16454
16592
  if (loadData) await deps.indexState().registerOpen(metadata.indexName);
16455
16593
  else deps.indexState().registerClosed(metadata.indexName);
@@ -16849,6 +16987,7 @@ function wireIndexState(wiring) {
16849
16987
  }
16850
16988
  try {
16851
16989
  await durability.manager.recoverIndex(indexName);
16990
+ await wiring.resumeChange(indexName);
16852
16991
  await analysisRebuild.review(indexName);
16853
16992
  await orchestrator.openIndex(indexName);
16854
16993
  await wiring.onOpen?.(indexName);
@@ -17178,6 +17317,9 @@ function checkHeapAfterRecovery(core) {
17178
17317
 
17179
17318
  // src/core/partition/frozen/share.ts
17180
17319
  function shareFrozenSegment(segment, analysis) {
17320
+ if (segment.paged !== null) {
17321
+ return { snapshot: segment.paged.snapshot, tombstonedDocIds: segment.tombstonedDocIds() };
17322
+ }
17181
17323
  if (segment.sharedSnapshot !== null) {
17182
17324
  const tombstonedDocIds = segment.tombstonedDocIds();
17183
17325
  if (tombstonedDocIds.length === 0 || !segment.removals.settled) {
@@ -17569,14 +17711,18 @@ function assertConfigReachesWorker(indexName, config, bootstrapModule) {
17569
17711
  { indexName }
17570
17712
  );
17571
17713
  }
17714
+ if (bootstrapModule !== void 0) return;
17572
17715
  const language = config.language ?? "english";
17573
- if (language !== "english" && bootstrapModule === void 0) {
17716
+ const requireRegistered = (setting, name) => {
17574
17717
  throw new NarsilError(
17575
17718
  ErrorCodes.CONFIG_INVALID,
17576
- `Index "${indexName}" uses language "${language}", which a worker thread registers only from a bootstrap module. Set workers.bootstrapModule to a module that registers it`,
17577
- { indexName, language }
17719
+ `Index "${indexName}" uses ${setting} "${name}", which a worker thread registers only from a bootstrap module. Set workers.bootstrapModule to a module that registers it`,
17720
+ { indexName, [setting]: name }
17578
17721
  );
17579
- }
17722
+ };
17723
+ if (language !== "english") requireRegistered("language", language);
17724
+ if (typeof config.tokenizer === "string") requireRegistered("tokenizer", config.tokenizer);
17725
+ if (typeof config.stopWords === "string") requireRegistered("stopWords", config.stopWords);
17580
17726
  }
17581
17727
  function workerIneligibility(indexName, config, bootstrapModule) {
17582
17728
  try {
@@ -17772,7 +17918,7 @@ async function replicateToWorkers(state, action) {
17772
17918
  const loading3 = state.copyLoadBuffers.get(action.indexName);
17773
17919
  if (loading3 !== void 0) {
17774
17920
  queueForCopies(state, detached);
17775
- if (loading3.length % LOAD_BUFFER_YIELD_INTERVAL === 0) await yieldToEventLoop();
17921
+ if (loading3.length % LOAD_BUFFER_YIELD_INTERVAL === 0) await yieldToEventLoop2();
17776
17922
  return;
17777
17923
  }
17778
17924
  const queue = state.replicationQueues.get(action.indexName);
@@ -17980,7 +18126,7 @@ async function compactPartitionSegments(state, indexName, manager, partitionId,
17980
18126
  }
17981
18127
  async function compactIndexSegments(state, indexName, policy) {
17982
18128
  const manager = state.executor.getManager(indexName);
17983
- if (!manager) return;
18129
+ if (!manager || manager.config.textStorage === "disk") return;
17984
18130
  const partitionIds = /* @__PURE__ */ new Set();
17985
18131
  for (let partitionId = 0; partitionId < manager.partitionCount; partitionId++) partitionIds.add(partitionId);
17986
18132
  for (const partitionId of state.segmentLedger.get(indexName)?.keys() ?? []) partitionIds.add(partitionId);
@@ -19431,6 +19577,7 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
19431
19577
  scaleOutBeforeBatch: (indexName, incomingCount) => scaleOutBeforeBatch(state, indexName, incomingCount),
19432
19578
  replicateToWorkers: replicate,
19433
19579
  awaitReplication: (indexName) => awaitReplicationIdle(state, indexName),
19580
+ queueDiskAdoption: (action) => queueForCopies(state, action),
19434
19581
  awaitWrites: (indexName) => awaitWritesApplied(state, indexName),
19435
19582
  awaitCompactions: () => awaitCompactions(state),
19436
19583
  openIndex,
@@ -19504,7 +19651,7 @@ async function emitWriteEventAfter(source, indexName, write) {
19504
19651
  // src/engine/core.ts
19505
19652
  async function createEngineCore(config, hooks, capacityScope = "index") {
19506
19653
  validateWorkerConfig(config?.workers, config?.lifecycle);
19507
- const rerankSettings = config?.rerankTimeout === void 0 && config?.maxRerankWindow === void 0 ? { timeoutMs: DEFAULT_RERANK_TIMEOUT_MS, maxWindow: DEFAULT_MAX_RERANK_WINDOW } : (await import('./runtime-3X7ZJ5T3.mjs')).resolveRerankSettings(config);
19654
+ const rerankSettings = config?.rerankTimeout === void 0 && config?.maxRerankWindow === void 0 ? { timeoutMs: DEFAULT_RERANK_TIMEOUT_MS, maxWindow: DEFAULT_MAX_RERANK_WINDOW } : (await import('./runtime-VEUH674B.mjs')).resolveRerankSettings(config);
19508
19655
  const eventHandlers = eventHandlersOf(config?.events);
19509
19656
  const vectorWorkerCount = splitWorkerBudget(resolveWorkerCount(config?.workers?.count)).vector;
19510
19657
  const vectorCopyPolicy = {
@@ -19515,7 +19662,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
19515
19662
  if (config?.lifecycle !== void 0 && durabilityTier === null) {
19516
19663
  throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index lifecycle settings require durability");
19517
19664
  }
19518
- const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-FNZSRH4N.mjs');
19665
+ const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-SNOWIT6Z.mjs');
19519
19666
  const filesystemDurability = durabilityTier?.kind === "wal";
19520
19667
  const executor = createDirectExecutor({
19521
19668
  vectorWorkerCopies: vectorCopyPolicy,
@@ -19575,7 +19722,6 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
19575
19722
  },
19576
19723
  vectorCopyPolicy
19577
19724
  );
19578
- const rebalancer = createRebalancer();
19579
19725
  const rebalanceRouter = createPartitionRouter();
19580
19726
  const waqMap = /* @__PURE__ */ new Map();
19581
19727
  const rebalanceTargets = /* @__PURE__ */ new Map();
@@ -19635,6 +19781,8 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
19635
19781
  emitCheckpointHeld(report) {
19636
19782
  emitEngineEvent(eventHandlers, "checkpointHeld", report);
19637
19783
  },
19784
+ queueDiskAdoption: orchestrator.queueDiskAdoption,
19785
+ awaitCopiesOfIndex: (indexName) => orchestrator.awaitReplication(indexName),
19638
19786
  publishCheckpointedPartitions: (indexName, partitions) => invalidation?.publishPartitions(indexName, partitions) ?? Promise.resolve(),
19639
19787
  recordCheckpoint(indexName, documentCount, partitionCount) {
19640
19788
  const entry = indexRegistry.get(indexName);
@@ -19685,6 +19833,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
19685
19833
  entry.analysisRevision = entry.language.revision;
19686
19834
  entry.arrayPositionGap = ARRAY_POSITION_GAP;
19687
19835
  },
19836
+ hasPendingChange: (indexName) => (indexRegistry.get(indexName)?.pendingAnalysis ?? null) !== null,
19688
19837
  durabilityManager: durability?.manager ?? null
19689
19838
  });
19690
19839
  const indexState = wireIndexState({
@@ -19696,6 +19845,10 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
19696
19845
  indexRegistry,
19697
19846
  rebalancingIndexes,
19698
19847
  requireManager,
19848
+ resumeChange: async (indexName) => {
19849
+ if ((indexRegistry.get(indexName)?.pendingAnalysis ?? null) === null) return;
19850
+ await (await import('./analysis-change-entry-7HR4LWL3.mjs')).resumeChangeOf(engineCore, indexName);
19851
+ },
19699
19852
  onClose: hooks?.onIndexClose,
19700
19853
  onOpen: (indexName) => {
19701
19854
  heapPressureNotifier.check(indexName);
@@ -19724,7 +19877,6 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
19724
19877
  checkHeapPressure: heapPressureNotifier.check
19725
19878
  };
19726
19879
  const rebalanceCtx = {
19727
- rebalancer,
19728
19880
  router: rebalanceRouter,
19729
19881
  waqMap,
19730
19882
  rebalancingIndexes,
@@ -19736,7 +19888,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
19736
19888
  checkWatermark: watermarkNotifier.check,
19737
19889
  requireIndex
19738
19890
  };
19739
- return {
19891
+ const engineCore = {
19740
19892
  executor,
19741
19893
  pluginRegistry,
19742
19894
  durability,
@@ -19751,7 +19903,6 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
19751
19903
  shutdownState,
19752
19904
  abortController,
19753
19905
  orchestrator,
19754
- rebalancer,
19755
19906
  rebalanceRouter,
19756
19907
  rebalancingIndexes,
19757
19908
  waqMap,
@@ -19767,6 +19918,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
19767
19918
  rebalanceCtx,
19768
19919
  patternWorkCap: config?.patternWorkCap ?? DEFAULT_PATTERN_WORK_CAP
19769
19920
  };
19921
+ return engineCore;
19770
19922
  }
19771
19923
 
19772
19924
  // src/engine/lifecycle.ts
@@ -19777,6 +19929,7 @@ async function startEngineCore(core, config) {
19777
19929
  checkHeapAfterRecovery(core);
19778
19930
  }
19779
19931
  if (core.invalidation) await core.invalidation.start();
19932
+ if (core.durability) await (await import('./analysis-change-entry-7HR4LWL3.mjs')).resumeEveryChange(core);
19780
19933
  if (config?.lifecycle === void 0) await core.analysisRebuild.reviewStaleIndexes();
19781
19934
  } catch (err) {
19782
19935
  await core.durability?.manager.shutdown().catch(() => void 0);
@@ -19814,7 +19967,7 @@ async function shutdownEngine(core) {
19814
19967
  }
19815
19968
  }
19816
19969
  if (core.rerankAdapters.size > 0) {
19817
- await (await import('./runtime-3X7ZJ5T3.mjs')).shutdownRerankAdapters(core.rerankAdapters);
19970
+ await (await import('./runtime-VEUH674B.mjs')).shutdownRerankAdapters(core.rerankAdapters);
19818
19971
  }
19819
19972
  await orchestrator.shutdown();
19820
19973
  await executor.shutdown();
@@ -20124,9 +20277,12 @@ function requireWithinFileLimit(byteLength, what) {
20124
20277
  );
20125
20278
  }
20126
20279
  async function packChecksummedEnvelope(payloadChunks, envelopeFormatVersion, what) {
20280
+ requireWithinFileLimit(HEADER_SIZE + payloadByteLength(payloadChunks), what);
20281
+ return packEnvelopeWithChecksum(payloadChunks, envelopeFormatVersion, what, await checksumOfChunks(payloadChunks));
20282
+ }
20283
+ function packEnvelopeWithChecksum(payloadChunks, envelopeFormatVersion, what, checksum) {
20127
20284
  const payloadLength = payloadByteLength(payloadChunks);
20128
20285
  requireWithinFileLimit(HEADER_SIZE + payloadLength, what);
20129
- const checksum = await checksumOfChunks(payloadChunks);
20130
20286
  const [major, minor, patch] = engineVersionParts();
20131
20287
  const header = writeHeader({
20132
20288
  envelopeFormatVersion,
@@ -20244,7 +20400,8 @@ function indexConfigurationOf(entry) {
20244
20400
  entry.embeddingAdapterName,
20245
20401
  entry.analysisRevision,
20246
20402
  entry.arrayPositionGap,
20247
- synonyms
20403
+ synonyms,
20404
+ entry.analysisGeneration
20248
20405
  );
20249
20406
  }
20250
20407
  function requireConfigurationWithinSizeLimit(config, language) {
@@ -20265,7 +20422,7 @@ function requireConfigurationWithinSizeLimit(config, language) {
20265
20422
  { bytes, maximum: MAX_INDEX_CONFIGURATION_BYTES }
20266
20423
  );
20267
20424
  }
20268
- function configurationPayloadOf(config, language, adapterName, analysisRevision, arrayPositionGap, synonyms) {
20425
+ function configurationPayloadOf(config, language, adapterName, analysisRevision, arrayPositionGap, synonyms, analysisGeneration = 0) {
20269
20426
  const bm25 = config.bm25;
20270
20427
  return {
20271
20428
  schema: config.schema,
@@ -20275,6 +20432,7 @@ function configurationPayloadOf(config, language, adapterName, analysisRevision,
20275
20432
  ...typeof config.tokenizer === "string" ? { tokenizer: config.tokenizer } : {},
20276
20433
  ...typeof config.stopWords === "string" ? { stopWords: config.stopWords } : {},
20277
20434
  ...config.stopWords instanceof Set ? { stopWordList: [...config.stopWords].sort(compareCodePoints) } : {},
20435
+ ...analysisGeneration > 0 ? { analysisGeneration } : {},
20278
20436
  ...bm25 !== void 0 ? {
20279
20437
  bm25: {
20280
20438
  ...bm25.k1 !== void 0 ? { k1: bm25.k1 } : {},
@@ -20286,6 +20444,7 @@ function configurationPayloadOf(config, language, adapterName, analysisRevision,
20286
20444
  ...config.partitions !== void 0 ? { partitionConfig: config.partitions } : {},
20287
20445
  ...config.defaultScoring !== void 0 ? { defaultScoring: config.defaultScoring } : {},
20288
20446
  ...config.trackPositions !== void 0 ? { trackPositions: config.trackPositions } : {},
20447
+ ...config.textStorage !== void 0 ? { textStorage: config.textStorage } : {},
20289
20448
  ...config.strict !== void 0 ? { strict: config.strict } : {},
20290
20449
  ...config.required !== void 0 ? { required: config.required } : {},
20291
20450
  ...config.vectorPromotion !== void 0 ? { vectorPromotion: config.vectorPromotion } : {},
@@ -20374,6 +20533,7 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
20374
20533
  requireSurfaceFormsForPartialFields(schema, indexConfig.surfaceForms);
20375
20534
  validateVectorPromotion(indexConfig.vectorPromotion);
20376
20535
  validateVectorStorage(indexConfig.vectorPromotion, core.filesystemDurability);
20536
+ validateTextStorage(indexConfig.textStorage, core.filesystemDurability);
20377
20537
  if (indexConfig.bm25 !== void 0) {
20378
20538
  validateBM25Params(
20379
20539
  indexConfig.bm25,
@@ -20449,7 +20609,10 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
20449
20609
  partitionCount: indexConfig.partitions?.maxPartitions ?? 1,
20450
20610
  analysisRevision: language.revision,
20451
20611
  arrayPositionGap: ARRAY_POSITION_GAP,
20452
- synonyms: synonyms?.encoded ?? null
20612
+ synonyms: synonyms?.encoded ?? null,
20613
+ analysisGeneration: 0,
20614
+ pendingAnalysis: null,
20615
+ durableAnalysis: null
20453
20616
  });
20454
20617
  await core.indexState.registerOpen(name);
20455
20618
  if (core.durability) {
@@ -20460,8 +20623,14 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
20460
20623
  async function dropEngineIndex(core, name) {
20461
20624
  core.guardShutdown();
20462
20625
  const entry = core.requireIndex(name);
20626
+ const changeRecorded = () => entry.pendingAnalysis !== null || entry.durableAnalysis !== null;
20627
+ const abandonChange = async () => {
20628
+ await (await import('./analysis-change-entry-7HR4LWL3.mjs')).abandonChangeOf(core, name);
20629
+ };
20630
+ if (changeRecorded()) await abandonChange();
20463
20631
  await core.indexState.drop(name, async () => {
20464
20632
  await core.analysisRebuild.abandon(name);
20633
+ if (changeRecorded()) await abandonChange();
20465
20634
  await core.orchestrator.closeIndex(name);
20466
20635
  if (core.durability) {
20467
20636
  try {
@@ -21293,7 +21462,7 @@ function snapshotSegments(segments) {
21293
21462
  }
21294
21463
  return shared2;
21295
21464
  }
21296
- function attachSegments(orchestrator, indexName, segments) {
21465
+ function attachSegments2(orchestrator, indexName, segments) {
21297
21466
  return orchestrator.replicateToWorkers({
21298
21467
  type: "attachSegments",
21299
21468
  indexName,
@@ -21304,11 +21473,11 @@ function attachSegments(orchestrator, indexName, segments) {
21304
21473
  async function broadcastBuiltSegments(orchestrator, indexName, segments, skipClone) {
21305
21474
  const frozen = tryFreezeSegmentsForAttach(segments);
21306
21475
  if (frozen !== null) {
21307
- await attachSegments(orchestrator, indexName, frozen);
21476
+ await attachSegments2(orchestrator, indexName, frozen);
21308
21477
  return;
21309
21478
  }
21310
21479
  const shared2 = snapshotSegments(segments);
21311
- if (shared2.length > 0) await attachSegments(orchestrator, indexName, shared2);
21480
+ if (shared2.length > 0) await attachSegments2(orchestrator, indexName, shared2);
21312
21481
  const plain = payloadSegments(segments);
21313
21482
  if (plain.length === 0) return;
21314
21483
  await orchestrator.replicateToWorkers({
@@ -22035,303 +22204,33 @@ async function updateDocumentBatch(ctx, indexName, updates, options) {
22035
22204
  for (let i = 0; i < prepared.length; i++) {
22036
22205
  const update = prepared[i];
22037
22206
  const failure = failures[i];
22038
- if (failure !== null) {
22039
- let error = failure.error;
22040
- try {
22041
- await rollbackUpdatedDocument(ctx, indexName, update.docId, update.rollbackDoc, error);
22042
- } catch (rollbackError) {
22043
- error = rollbackError;
22044
- }
22045
- failed.push({ docId: update.docId, error: asUpdateError(error) });
22046
- continue;
22047
- }
22048
- if (!update.buffered) await settleUpdate(ctx, indexName, update);
22049
- for (const [fieldPath, vec] of update.extractedVectors) {
22050
- if (vec !== null) touchedVectorFields.add(fieldPath);
22051
- }
22052
- succeeded.push(update.docId);
22053
- }
22054
- if (chunkEnd < updates.length) {
22055
- ctx.checkHeapPressure(indexName);
22056
- await new Promise((r) => setTimeout(r, 0));
22057
- }
22058
- }
22059
- for (const fieldPath of touchedVectorFields) {
22060
- updateBatchVecIndexes.get(fieldPath)?.scheduleBuild();
22061
- }
22062
- ctx.checkHeapPressure(indexName);
22063
- if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
22064
- return { succeeded, failed };
22065
- }
22066
-
22067
- // src/partitioning/write-ahead-queue.ts
22068
- function createWriteAheadQueue(maxSize = DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE) {
22069
- const entries = [];
22070
- const docStates = /* @__PURE__ */ new Map();
22071
- let nextSequence = 1;
22072
- let resolveReplayed = () => void 0;
22073
- const replayed = new Promise((resolve) => {
22074
- resolveReplayed = resolve;
22075
- });
22076
- return {
22077
- push(entry) {
22078
- if (entries.length >= maxSize) {
22079
- throw new NarsilError(
22080
- ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
22081
- `Write-ahead queue is full (${maxSize} entries). Retry after rebalancing completes.`,
22082
- { maxSize, currentSize: entries.length }
22083
- );
22084
- }
22085
- const seq = nextSequence++;
22086
- entries.push({ ...entry, sequenceNumber: seq });
22087
- docStates.set(entry.docId, entry.action === "remove" ? "absent" : "present");
22088
- return seq;
22089
- },
22090
- drain() {
22091
- const result = entries.splice(0);
22092
- result.sort((a, b) => a.sequenceNumber - b.sequenceNumber);
22093
- return result;
22094
- },
22095
- clear() {
22096
- entries.length = 0;
22097
- docStates.clear();
22098
- },
22099
- bufferedDocState(docId) {
22100
- return docStates.get(docId);
22101
- },
22102
- whenReplayed() {
22103
- return replayed;
22104
- },
22105
- markReplayed() {
22106
- resolveReplayed();
22107
- },
22108
- get size() {
22109
- return entries.length;
22110
- },
22111
- get isFull() {
22112
- return entries.length >= maxSize;
22113
- }
22114
- };
22115
- }
22116
-
22117
- // src/engine/rebalance-executor.ts
22118
- function warnHookError(hook, err) {
22119
- console.warn(`${hook} plugin hook error:`, err instanceof Error ? err.message : String(err));
22120
- }
22121
- function applyReplayUpdateVectors(entry, vectorFieldPaths2) {
22122
- const updateVectors = /* @__PURE__ */ new Map();
22123
- for (const fieldPath of vectorFieldPaths2) {
22124
- updateVectors.set(fieldPath, extractVectorFromDoc(entry.document, fieldPath));
22125
- }
22126
- let updateDoc = entry.document;
22127
- if (updateVectors.size > 0) {
22128
- updateDoc = structuredClone(entry.document);
22129
- for (const fieldPath of updateVectors.keys()) {
22130
- deleteNestedValue(updateDoc, fieldPath);
22131
- }
22132
- }
22133
- return { updateDoc, updateVectors };
22134
- }
22135
- async function replayEntry(manager, indexName, entry, ctx) {
22136
- const vecIndexes = manager.getVectorIndexes();
22137
- const vectorFieldPaths2 = ctx.requireIndex(indexName).vectorFieldPaths;
22138
- if (entry.action === "insert" && entry.document) {
22139
- const { partitionDoc, extractedVectors } = prepareDocumentVectors(
22140
- entry.document,
22141
- vectorFieldPaths2
22142
- );
22143
- const overwrote = manager.has(entry.docId);
22144
- const removedSpellings = overwrote ? manager.removedSpellingsOf(entry.docId) : void 0;
22145
- if (overwrote) {
22146
- removeDocumentVectors(entry.docId, vecIndexes);
22147
- manager.remove(entry.docId, removedSpellings);
22148
- }
22149
- manager.insert(entry.docId, partitionDoc);
22150
- if (extractedVectors.size > 0) {
22151
- insertDocumentVectors(entry.docId, extractedVectors, vecIndexes, manager.partitionIdOf(entry.docId));
22152
- for (const fieldPath of extractedVectors.keys()) {
22153
- vecIndexes.get(fieldPath)?.scheduleBuild();
22154
- }
22155
- }
22156
- try {
22157
- await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: entry.docId, document: entry.document });
22158
- } catch (err) {
22159
- warnHookError("afterInsert", err);
22160
- }
22161
- const replicated = {
22162
- indexName,
22163
- docId: entry.docId,
22164
- document: entry.document,
22165
- requestId: `replicate-insert-${entry.docId}`
22166
- };
22167
- await ctx.orchestrator.replicateToWorkers(
22168
- overwrote ? { type: "update", ...replicated, removedSpellings } : { type: "insert", ...replicated }
22169
- );
22170
- return;
22171
- }
22172
- if (entry.action === "remove") {
22173
- const removedSpellings = manager.removedSpellingsOf(entry.docId);
22174
- manager.remove(entry.docId, removedSpellings);
22175
- removeDocumentVectors(entry.docId, vecIndexes);
22176
- try {
22177
- await ctx.pluginRegistry.runHook("afterRemove", { indexName, docId: entry.docId });
22178
- } catch (err) {
22179
- warnHookError("afterRemove", err);
22180
- }
22181
- await ctx.orchestrator.replicateToWorkers({
22182
- type: "remove",
22183
- indexName,
22184
- docId: entry.docId,
22185
- requestId: `replicate-remove-${entry.docId}`,
22186
- removedSpellings
22187
- });
22188
- return;
22189
- }
22190
- if (entry.action === "update" && entry.document) {
22191
- const oldDocument = manager.get(entry.docId);
22192
- const { updateDoc, updateVectors } = applyReplayUpdateVectors(entry, vectorFieldPaths2);
22193
- const removedSpellings = manager.removedSpellingsOf(entry.docId);
22194
- if (manager.has(entry.docId)) {
22195
- manager.update(entry.docId, updateDoc, void 0, removedSpellings);
22196
- } else {
22197
- manager.insert(entry.docId, updateDoc);
22198
- }
22199
- for (const [fieldPath, newVec] of updateVectors) {
22200
- const vecIndex = vecIndexes.get(fieldPath);
22201
- if (!vecIndex) continue;
22202
- if (newVec === null) {
22203
- if (vecIndex.has(entry.docId)) {
22204
- vecIndex.remove(entry.docId);
22205
- }
22206
- } else {
22207
- const oldVec = vecIndex.getVector(entry.docId);
22208
- if (!vectorsEqual(oldVec, newVec)) {
22209
- if (vecIndex.has(entry.docId)) {
22210
- vecIndex.remove(entry.docId);
22211
- }
22212
- vecIndex.insert(entry.docId, newVec);
22213
- vecIndex.scheduleBuild();
22214
- }
22215
- }
22216
- }
22217
- try {
22218
- await ctx.pluginRegistry.runHook("afterUpdate", {
22219
- indexName,
22220
- docId: entry.docId,
22221
- oldDocument: oldDocument ?? {},
22222
- newDocument: entry.document
22223
- });
22224
- } catch (err) {
22225
- warnHookError("afterUpdate", err);
22226
- }
22227
- await ctx.orchestrator.replicateToWorkers({
22228
- type: "update",
22229
- indexName,
22230
- docId: entry.docId,
22231
- document: entry.document,
22232
- requestId: `replicate-update-${entry.docId}`,
22233
- removedSpellings
22234
- });
22235
- }
22236
- }
22237
- async function replayQueued(manager, indexName, waq, ctx) {
22238
- while (true) {
22239
- const entries = waq.drain();
22240
- if (entries.length === 0) break;
22241
- for (const entry of entries) {
22242
- try {
22243
- await replayEntry(manager, indexName, entry, ctx);
22244
- } catch (replayErr) {
22245
- const isMissing = replayErr instanceof NarsilError && replayErr.code === ErrorCodes.DOC_NOT_FOUND;
22246
- if (!isMissing) {
22247
- console.warn(`WAQ replay failed for ${entry.action} on doc "${entry.docId}":`, replayErr);
22248
- }
22249
- }
22250
- }
22251
- }
22252
- }
22253
- async function executeRebalance(manager, indexName, targetPartitionCount, ctx) {
22254
- if (targetPartitionCount <= 0 || !Number.isInteger(targetPartitionCount)) {
22255
- throw new NarsilError(
22256
- ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
22257
- `Target partition count must be a positive integer, got ${targetPartitionCount}`,
22258
- { targetPartitionCount }
22259
- );
22260
- }
22261
- const entry = ctx.requireIndex(indexName);
22262
- const maxPartitions = entry.config.partitions?.maxPartitions;
22263
- if (maxPartitions !== void 0 && targetPartitionCount > maxPartitions) {
22264
- throw new NarsilError(
22265
- ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
22266
- `Target partition count (${targetPartitionCount}) is above the maximum of ${maxPartitions} partitions`,
22267
- { targetPartitionCount, maxPartitions }
22268
- );
22269
- }
22270
- if (ctx.rebalancingIndexes.has(indexName)) {
22271
- throw new NarsilError(
22272
- ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
22273
- `Index "${indexName}" is already being rebalanced`
22274
- );
22275
- }
22276
- if (targetPartitionCount === manager.partitionCount) {
22277
- return;
22278
- }
22279
- const waq = createWriteAheadQueue();
22280
- ctx.waqMap.set(indexName, waq);
22281
- ctx.rebalancingIndexes.add(indexName);
22282
- ctx.rebalanceTargets.set(indexName, targetPartitionCount);
22283
- const wasPromoted = ctx.orchestrator.desyncIndex(indexName);
22284
- try {
22285
- const oldCount = manager.partitionCount;
22286
- await ctx.rebalancer.rebalance(manager, targetPartitionCount, ctx.router, (progress) => {
22287
- if (progress.phase === "complete") {
22288
- emitEngineEvent(ctx.eventHandlers, "partitionRebalance", {
22289
- indexName,
22290
- oldCount,
22291
- newCount: targetPartitionCount
22292
- });
22207
+ if (failure !== null) {
22208
+ let error = failure.error;
22209
+ try {
22210
+ await rollbackUpdatedDocument(ctx, indexName, update.docId, update.rollbackDoc, error);
22211
+ } catch (rollbackError) {
22212
+ error = rollbackError;
22213
+ }
22214
+ failed.push({ docId: update.docId, error: asUpdateError(error) });
22215
+ continue;
22293
22216
  }
22294
- });
22295
- try {
22296
- await ctx.pluginRegistry.runHook("onPartitionSplit", {
22297
- indexName,
22298
- oldPartitionCount: oldCount,
22299
- newPartitionCount: targetPartitionCount
22300
- });
22301
- } catch (hookErr) {
22302
- console.warn("onPartitionSplit plugin hook failed:", hookErr instanceof Error ? hookErr.message : String(hookErr));
22303
- }
22304
- await replayQueued(manager, indexName, waq, ctx);
22305
- if (ctx.durabilityManager) {
22306
- await ctx.durabilityManager.persistMetadata(indexName);
22307
- await (ctx.durabilityManager.checkpointFromMemory?.(indexName) ?? ctx.durabilityManager.checkpoint(indexName));
22308
- await replayQueued(manager, indexName, waq, ctx);
22309
- }
22310
- await ctx.orchestrator.resyncIndex(indexName, wasPromoted);
22311
- do {
22312
- await replayQueued(manager, indexName, waq, ctx);
22313
- } while (waq.size > 0);
22314
- } catch (err) {
22315
- try {
22316
- await replayQueued(manager, indexName, waq, ctx);
22317
- await ctx.orchestrator.resyncIndex(indexName, wasPromoted);
22318
- do {
22319
- await replayQueued(manager, indexName, waq, ctx);
22320
- } while (waq.size > 0);
22321
- } catch (recoveryErr) {
22322
- console.warn(
22323
- `Rebalance failure recovery for index "${indexName}" did not complete:`,
22324
- recoveryErr instanceof Error ? recoveryErr.message : String(recoveryErr)
22325
- );
22217
+ if (!update.buffered) await settleUpdate(ctx, indexName, update);
22218
+ for (const [fieldPath, vec] of update.extractedVectors) {
22219
+ if (vec !== null) touchedVectorFields.add(fieldPath);
22220
+ }
22221
+ succeeded.push(update.docId);
22326
22222
  }
22327
- throw err;
22328
- } finally {
22329
- ctx.rebalancingIndexes.delete(indexName);
22330
- ctx.rebalanceTargets.delete(indexName);
22331
- ctx.waqMap.delete(indexName);
22332
- waq.markReplayed();
22223
+ if (chunkEnd < updates.length) {
22224
+ ctx.checkHeapPressure(indexName);
22225
+ await new Promise((r) => setTimeout(r, 0));
22226
+ }
22227
+ }
22228
+ for (const fieldPath of touchedVectorFields) {
22229
+ updateBatchVecIndexes.get(fieldPath)?.scheduleBuild();
22333
22230
  }
22334
- ctx.checkWatermark(indexName);
22231
+ ctx.checkHeapPressure(indexName);
22232
+ if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
22233
+ return { succeeded, failed };
22335
22234
  }
22336
22235
 
22337
22236
  // src/engine/vector-maintenance.ts
@@ -22373,6 +22272,7 @@ function getVectorMaintenanceStatus(manager) {
22373
22272
  bufferSize: status.bufferSize,
22374
22273
  building: status.building,
22375
22274
  waitingForMemory: status.waitingForMemory,
22275
+ loadingFromFiles: status.loadingFromFiles === true,
22376
22276
  estimatedCompactMs: status.estimatedCompactMs,
22377
22277
  estimatedOptimizeMs: status.estimatedOptimizeMs
22378
22278
  });
@@ -22479,646 +22379,6 @@ function bindEngineCore(engine, core) {
22479
22379
  engineCores.set(engine, core);
22480
22380
  }
22481
22381
 
22482
- // src/engine/snapshot/restore-config.ts
22483
- function rejectSnapshot(message) {
22484
- throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Invalid snapshot: ${message}`);
22485
- }
22486
- function isPlainObject3(value) {
22487
- return value !== null && typeof value === "object" && !Array.isArray(value);
22488
- }
22489
- function restoredBm25(raw, reject2) {
22490
- if (raw === void 0) {
22491
- return void 0;
22492
- }
22493
- if (!isPlainObject3(raw)) {
22494
- reject2("bm25 must be an object");
22495
- }
22496
- const fields = restoredBm25Fields(raw.fields, reject2);
22497
- return {
22498
- ...restoredBm25Pair(raw, "bm25", reject2),
22499
- ...fields !== void 0 ? { fields } : {}
22500
- };
22501
- }
22502
- function restoredBm25Pair(raw, label, reject2) {
22503
- const { k1, b } = raw;
22504
- if (k1 !== void 0 && (typeof k1 !== "number" || !Number.isFinite(k1))) {
22505
- reject2(`${label} k1 must be a finite number`);
22506
- }
22507
- if (b !== void 0 && (typeof b !== "number" || !Number.isFinite(b))) {
22508
- reject2(`${label} b must be a finite number`);
22509
- }
22510
- return {
22511
- ...k1 !== void 0 ? { k1 } : {},
22512
- ...b !== void 0 ? { b } : {}
22513
- };
22514
- }
22515
- function restoredBm25Fields(raw, reject2) {
22516
- if (raw === void 0) return void 0;
22517
- if (!isPlainObject3(raw)) reject2("bm25 fields must be an object");
22518
- const source = raw;
22519
- const fields = /* @__PURE__ */ Object.create(null);
22520
- for (const name of Object.keys(source)) {
22521
- const pair = source[name];
22522
- if (!isPlainObject3(pair)) reject2("bm25 fields must map each field to an object");
22523
- fields[name] = restoredBm25Pair(pair, "bm25 field", reject2);
22524
- }
22525
- return fields;
22526
- }
22527
- function restoredStopWords(name, list, reject2) {
22528
- if (name !== void 0 && typeof name !== "string") {
22529
- reject2("stopWords must be a name");
22530
- }
22531
- if (list !== void 0 && (!Array.isArray(list) || !list.every((word) => typeof word === "string"))) {
22532
- reject2("stopWordList must be a list of words");
22533
- }
22534
- if (typeof name === "string" && Array.isArray(list)) {
22535
- reject2("stopWords and stopWordList cannot both be present");
22536
- }
22537
- if (typeof name === "string") {
22538
- return name;
22539
- }
22540
- if (Array.isArray(list)) {
22541
- return new Set(list);
22542
- }
22543
- return void 0;
22544
- }
22545
- function restoredPartitionConfig(raw, reject2 = rejectSnapshot) {
22546
- if (raw === void 0) {
22547
- return void 0;
22548
- }
22549
- if (!isPlainObject3(raw)) {
22550
- reject2("partitionConfig must be an object");
22551
- }
22552
- const { maxDocsPerPartition, maxPartitions, watermark } = raw;
22553
- if (maxDocsPerPartition !== void 0 && (typeof maxDocsPerPartition !== "number" || !Number.isInteger(maxDocsPerPartition) || maxDocsPerPartition < 1)) {
22554
- reject2("partitionConfig maxDocsPerPartition must be a positive integer");
22555
- }
22556
- if (maxPartitions !== void 0 && (typeof maxPartitions !== "number" || !Number.isInteger(maxPartitions) || maxPartitions < 1)) {
22557
- reject2("partitionConfig maxPartitions must be a positive integer");
22558
- }
22559
- if (watermark !== void 0 && (typeof watermark !== "number" || !(watermark > 0) || watermark > 1)) {
22560
- reject2("partitionConfig watermark must be above 0 and at most 1");
22561
- }
22562
- return {
22563
- ...typeof maxDocsPerPartition === "number" ? { maxDocsPerPartition } : {},
22564
- ...typeof maxPartitions === "number" ? { maxPartitions } : {},
22565
- ...typeof watermark === "number" ? { watermark } : {}
22566
- };
22567
- }
22568
- function restoredScoring(raw, reject2) {
22569
- if (raw === void 0) {
22570
- return void 0;
22571
- }
22572
- if (raw !== "local" && raw !== "dfs" && raw !== "broadcast") {
22573
- reject2("defaultScoring must be 'local', 'dfs', or 'broadcast'");
22574
- }
22575
- return raw;
22576
- }
22577
- function restoredBoolean(raw, field, reject2) {
22578
- if (raw === void 0) {
22579
- return void 0;
22580
- }
22581
- if (typeof raw !== "boolean") {
22582
- reject2(`${field} must be a boolean`);
22583
- }
22584
- return raw;
22585
- }
22586
- function restoredRequired(raw, reject2) {
22587
- if (raw === void 0) {
22588
- return void 0;
22589
- }
22590
- if (!Array.isArray(raw) || !raw.every((field) => typeof field === "string")) {
22591
- reject2("required must be a list of field paths");
22592
- }
22593
- return raw;
22594
- }
22595
- function restoredVectorPromotion(raw, reject2) {
22596
- if (raw === void 0) {
22597
- return void 0;
22598
- }
22599
- if (!isPlainObject3(raw) || raw.hnswConfig !== void 0 && !isPlainObject3(raw.hnswConfig)) {
22600
- reject2("vectorPromotion must be an object");
22601
- }
22602
- return raw;
22603
- }
22604
- function restoredEmbedding(raw, reject2 = rejectSnapshot) {
22605
- if (raw === void 0) {
22606
- return void 0;
22607
- }
22608
- if (!isPlainObject3(raw)) {
22609
- reject2("embedding must be an object");
22610
- }
22611
- const { fields, adapter } = raw;
22612
- if (adapter !== void 0 && typeof adapter !== "string") {
22613
- reject2("embedding adapter must be a name");
22614
- }
22615
- if (!isPlainObject3(fields)) {
22616
- reject2("embedding fields must be an object");
22617
- }
22618
- for (const value of Object.values(fields)) {
22619
- const isPath = typeof value === "string";
22620
- const isPathList = Array.isArray(value) && value.every((path) => typeof path === "string");
22621
- if (!isPath && !isPathList) {
22622
- reject2("embedding fields must map vector fields to source paths");
22623
- }
22624
- }
22625
- return {
22626
- fields,
22627
- ...adapter !== void 0 ? { adapter } : {}
22628
- };
22629
- }
22630
- function readSharedConfigFields(source, reject2) {
22631
- if (source.tokenizer !== void 0 && typeof source.tokenizer !== "string") {
22632
- reject2("tokenizer must be a name");
22633
- }
22634
- const stopWords2 = restoredStopWords(source.stopWords, source.stopWordList, reject2);
22635
- const bm25 = restoredBm25(source.bm25, reject2);
22636
- const surfaceForms = restoredBoolean(source.surfaceForms, "surfaceForms", reject2);
22637
- const defaultScoring = restoredScoring(source.defaultScoring, reject2);
22638
- const trackPositions = restoredBoolean(source.trackPositions, "trackPositions", reject2);
22639
- const strict = restoredBoolean(source.strict, "strict", reject2);
22640
- const required = restoredRequired(source.required, reject2);
22641
- const vectorPromotion = restoredVectorPromotion(source.vectorPromotion, reject2);
22642
- if (source.patternValueLimit !== void 0 && typeof source.patternValueLimit !== "number") {
22643
- reject2("patternValueLimit must be a number");
22644
- }
22645
- return {
22646
- ...typeof source.tokenizer === "string" ? { tokenizer: source.tokenizer } : {},
22647
- ...stopWords2 !== void 0 ? { stopWords: stopWords2 } : {},
22648
- ...bm25 !== void 0 ? { bm25 } : {},
22649
- ...surfaceForms !== void 0 ? { surfaceForms } : {},
22650
- ...defaultScoring !== void 0 ? { defaultScoring } : {},
22651
- ...trackPositions !== void 0 ? { trackPositions } : {},
22652
- ...strict !== void 0 ? { strict } : {},
22653
- ...required !== void 0 ? { required } : {},
22654
- ...vectorPromotion !== void 0 ? { vectorPromotion } : {},
22655
- ...typeof source.patternValueLimit === "number" ? { patternValueLimit: source.patternValueLimit } : {}
22656
- };
22657
- }
22658
- function restoredConfigFields(envelope, reject2 = rejectSnapshot) {
22659
- const shared2 = readSharedConfigFields(envelope, reject2);
22660
- const partitions = restoredPartitionConfig(envelope.partitionConfig, reject2);
22661
- return {
22662
- ...shared2,
22663
- ...partitions !== void 0 ? { partitions } : {}
22664
- };
22665
- }
22666
-
22667
- // src/persistence/archive/byte-reader.ts
22668
- async function* chunksOf(source) {
22669
- if (source instanceof Uint8Array) {
22670
- yield source;
22671
- return;
22672
- }
22673
- if (typeof source.getReader === "function") {
22674
- const reader = source.getReader();
22675
- try {
22676
- for (; ; ) {
22677
- const step = await reader.read();
22678
- if (step.done) return;
22679
- yield step.value;
22680
- }
22681
- } finally {
22682
- reader.releaseLock();
22683
- }
22684
- }
22685
- yield* source;
22686
- }
22687
- function createByteReader(source) {
22688
- const iterator = chunksOf(source);
22689
- const pending = [];
22690
- let pendingBytes = 0;
22691
- let ended = false;
22692
- async function fill(length) {
22693
- while (pendingBytes < length && !ended) {
22694
- const step = await iterator.next();
22695
- if (step.done) {
22696
- ended = true;
22697
- break;
22698
- }
22699
- if (step.value.byteLength === 0) continue;
22700
- pending.push(step.value);
22701
- pendingBytes += step.value.byteLength;
22702
- }
22703
- }
22704
- function take(length) {
22705
- const out = new Uint8Array(length);
22706
- let filled = 0;
22707
- while (filled < length) {
22708
- const head = pending[0];
22709
- const needed = length - filled;
22710
- if (head.byteLength <= needed) {
22711
- out.set(head, filled);
22712
- filled += head.byteLength;
22713
- pending.shift();
22714
- } else {
22715
- out.set(head.subarray(0, needed), filled);
22716
- pending[0] = head.subarray(needed);
22717
- filled += needed;
22718
- }
22719
- }
22720
- pendingBytes -= length;
22721
- return out;
22722
- }
22723
- return {
22724
- async readExactly(length) {
22725
- await fill(length);
22726
- if (pendingBytes === 0 && length > 0) return null;
22727
- if (pendingBytes < length) return take(pendingBytes);
22728
- return take(length);
22729
- },
22730
- async peek(length) {
22731
- await fill(length);
22732
- const available = Math.min(length, pendingBytes);
22733
- const copy = take(available);
22734
- pending.unshift(copy);
22735
- pendingBytes += copy.byteLength;
22736
- return copy;
22737
- },
22738
- async close() {
22739
- await iterator.return(void 0);
22740
- }
22741
- };
22742
- }
22743
-
22744
- // src/engine/export/lines.ts
22745
- var NEWLINE = 10;
22746
- function joined(parts, total) {
22747
- if (parts.length === 1) return parts[0];
22748
- const bytes = new Uint8Array(total);
22749
- let offset = 0;
22750
- for (const part of parts) {
22751
- bytes.set(part, offset);
22752
- offset += part.byteLength;
22753
- }
22754
- return bytes;
22755
- }
22756
- async function* exportLines(source, maxLineBytes) {
22757
- const decoder4 = new TextDecoder("utf-8", { fatal: true });
22758
- let parts = [];
22759
- let pending = 0;
22760
- let overflowing = false;
22761
- let line = 0;
22762
- const finishLine = () => {
22763
- line += 1;
22764
- if (overflowing) {
22765
- overflowing = false;
22766
- parts = [];
22767
- pending = 0;
22768
- return { line, refused: "too-long" };
22769
- }
22770
- const bytes = joined(parts, pending);
22771
- parts = [];
22772
- pending = 0;
22773
- try {
22774
- return { line, text: decoder4.decode(bytes) };
22775
- } catch {
22776
- return { line, refused: "not-utf8" };
22777
- }
22778
- };
22779
- for await (const chunk of chunksOf(source)) {
22780
- let start = 0;
22781
- for (; ; ) {
22782
- const newline = chunk.indexOf(NEWLINE, start);
22783
- const end = newline < 0 ? chunk.byteLength : newline;
22784
- const piece = chunk.subarray(start, end);
22785
- if (!overflowing && pending + piece.byteLength <= maxLineBytes) {
22786
- if (piece.byteLength > 0) parts.push(piece.slice());
22787
- pending += piece.byteLength;
22788
- } else {
22789
- overflowing = true;
22790
- parts = [];
22791
- }
22792
- if (newline < 0) break;
22793
- yield finishLine();
22794
- start = newline + 1;
22795
- }
22796
- }
22797
- if (pending > 0 || overflowing) yield finishLine();
22798
- }
22799
-
22800
- // src/serialization/base64.ts
22801
- var ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
22802
- var PAD = 61;
22803
- var BYTES_PER_GROUP = 3;
22804
- var CHARACTERS_PER_GROUP = 4;
22805
- var ASCII_LIMIT = 128;
22806
- var ENCODE = new Uint8Array(ALPHABET.length);
22807
- var DECODE = new Int8Array(ASCII_LIMIT).fill(-1);
22808
- for (let at = 0; at < ALPHABET.length; at++) {
22809
- ENCODE[at] = ALPHABET.charCodeAt(at);
22810
- DECODE[ALPHABET.charCodeAt(at)] = at;
22811
- }
22812
- var asciiDecoder = new TextDecoder();
22813
- function encodeBase64(bytes) {
22814
- const out = new Uint8Array(Math.ceil(bytes.byteLength / BYTES_PER_GROUP) * CHARACTERS_PER_GROUP);
22815
- let write = 0;
22816
- let read = 0;
22817
- for (; read + BYTES_PER_GROUP <= bytes.byteLength; read += BYTES_PER_GROUP) {
22818
- const group = bytes[read] << 16 | bytes[read + 1] << 8 | bytes[read + 2];
22819
- out[write++] = ENCODE[group >>> 18];
22820
- out[write++] = ENCODE[group >>> 12 & 63];
22821
- out[write++] = ENCODE[group >>> 6 & 63];
22822
- out[write++] = ENCODE[group & 63];
22823
- }
22824
- const left = bytes.byteLength - read;
22825
- if (left > 0) {
22826
- const group = bytes[read] << 16 | (left === 2 ? bytes[read + 1] << 8 : 0);
22827
- out[write++] = ENCODE[group >>> 18];
22828
- out[write++] = ENCODE[group >>> 12 & 63];
22829
- out[write++] = left === 2 ? ENCODE[group >>> 6 & 63] : PAD;
22830
- out[write++] = PAD;
22831
- }
22832
- return asciiDecoder.decode(out);
22833
- }
22834
- function sextetAt(text, at) {
22835
- const code = text.charCodeAt(at);
22836
- return code < ASCII_LIMIT ? DECODE[code] : -1;
22837
- }
22838
- function decodeExactBase64(text) {
22839
- if (text.length % CHARACTERS_PER_GROUP !== 0) return null;
22840
- const padding = text.endsWith("==") ? 2 : text.endsWith("=") ? 1 : 0;
22841
- const out = new Uint8Array(text.length / CHARACTERS_PER_GROUP * BYTES_PER_GROUP - padding);
22842
- const fullEnd = padding === 0 ? text.length : text.length - CHARACTERS_PER_GROUP;
22843
- let write = 0;
22844
- for (let at = 0; at < fullEnd; at += CHARACTERS_PER_GROUP) {
22845
- const a2 = sextetAt(text, at);
22846
- const b2 = sextetAt(text, at + 1);
22847
- const c2 = sextetAt(text, at + 2);
22848
- const d = sextetAt(text, at + 3);
22849
- if ((a2 | b2 | c2 | d) < 0) return null;
22850
- const group = a2 << 18 | b2 << 12 | c2 << 6 | d;
22851
- out[write++] = group >>> 16;
22852
- out[write++] = group >>> 8 & 255;
22853
- out[write++] = group & 255;
22854
- }
22855
- if (padding === 0) return out;
22856
- const a = sextetAt(text, fullEnd);
22857
- const b = sextetAt(text, fullEnd + 1);
22858
- const c = padding === 1 ? sextetAt(text, fullEnd + 2) : 0;
22859
- if ((a | b | c) < 0) return null;
22860
- if (padding === 2 && (b & 15) !== 0) return null;
22861
- if (padding === 1 && (c & 3) !== 0) return null;
22862
- out[write++] = a << 2 | b >>> 4;
22863
- if (padding === 1) out[write] = (b & 15) << 4 | c >>> 2;
22864
- return out;
22865
- }
22866
-
22867
- // src/engine/export/tags.ts
22868
- var TAG_PREFIX = "$";
22869
- var DATE_TAG = "$date";
22870
- var BYTES_TAG = "$bytes";
22871
- var NUMBER_TAG = "$number";
22872
- var OBJECT_TAG = "$object";
22873
- var MILLISECONDS_PER_SECOND = 1e3;
22874
- var NANOSECONDS_PER_MILLISECOND = 1e6;
22875
- var MAX_NANOSECONDS = 999999999;
22876
- var MAX_DATE_MILLISECONDS = 864e13;
22877
- var ESCAPED_TAG_PREFIX = "\\u0024";
22878
- var NON_FINITE_NUMBERS = /* @__PURE__ */ new Map([
22879
- ["NaN", Number.NaN],
22880
- ["Infinity", Number.POSITIVE_INFINITY],
22881
- ["-Infinity", Number.NEGATIVE_INFINITY]
22882
- ]);
22883
- function refuse(path, reason) {
22884
- throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path === "" ? "(document)" : path}" ${reason}`, {
22885
- field: path
22886
- });
22887
- }
22888
- function isRecord(value) {
22889
- return typeof value === "object" && value !== null && !Array.isArray(value);
22890
- }
22891
- function startsWithTagPrefix(key) {
22892
- return key.startsWith(TAG_PREFIX);
22893
- }
22894
- function soleTagKey(record) {
22895
- let sole = null;
22896
- for (const key of Object.keys(record)) {
22897
- if (sole !== null) return null;
22898
- sole = key;
22899
- }
22900
- return sole !== null && startsWithTagPrefix(sole) ? sole : null;
22901
- }
22902
- function dateOf(value, path) {
22903
- if (!Array.isArray(value) || value.length !== 2)
22904
- refuse(path, `holds a "${DATE_TAG}" tag other than [seconds, nanoseconds]`);
22905
- const seconds = value[0];
22906
- const nanoseconds = value[1];
22907
- if (typeof seconds !== "number" || !Number.isSafeInteger(seconds)) {
22908
- refuse(
22909
- path,
22910
- `holds a "${DATE_TAG}" tag whose seconds are not an integer from -${Number.MAX_SAFE_INTEGER} to ${Number.MAX_SAFE_INTEGER}`
22911
- );
22912
- }
22913
- if (typeof nanoseconds !== "number" || !Number.isInteger(nanoseconds) || nanoseconds < 0 || nanoseconds > MAX_NANOSECONDS) {
22914
- refuse(path, `holds a "${DATE_TAG}" tag whose nanoseconds are not an integer from 0 to ${MAX_NANOSECONDS}`);
22915
- }
22916
- const milliseconds = seconds * MILLISECONDS_PER_SECOND + nanoseconds / NANOSECONDS_PER_MILLISECOND;
22917
- if (Math.abs(milliseconds) > MAX_DATE_MILLISECONDS)
22918
- refuse(path, `holds a "${DATE_TAG}" tag outside the range of a date`);
22919
- return new Date(milliseconds);
22920
- }
22921
- function bytesOf(value, path) {
22922
- const bytes = typeof value === "string" ? decodeExactBase64(value) : null;
22923
- if (bytes === null) refuse(path, `holds a "${BYTES_TAG}" tag whose text is not padded base64`);
22924
- return bytes;
22925
- }
22926
- function numberOf(value, path) {
22927
- const number = typeof value === "string" ? NON_FINITE_NUMBERS.get(value) : void 0;
22928
- if (number === void 0) refuse(path, `holds a "${NUMBER_TAG}" tag other than "NaN", "Infinity", or "-Infinity"`);
22929
- return number;
22930
- }
22931
- function requireDepth(path, depth) {
22932
- if (depth >= MAX_DOCUMENT_NESTING_DEPTH) refuse(path, `nests deeper than ${MAX_DOCUMENT_NESTING_DEPTH} levels`);
22933
- }
22934
- function untagMap(record, path, depth) {
22935
- requireDepth(path, depth);
22936
- for (const key of Object.keys(record)) {
22937
- record[key] = untag(record[key], path === "" ? key : `${path}.${key}`, depth + 1);
22938
- }
22939
- return record;
22940
- }
22941
- function untagList(list, path, depth) {
22942
- requireDepth(path, depth);
22943
- for (let at = 0; at < list.length; at++) list[at] = untag(list[at], `${path}[${at}]`, depth + 1);
22944
- return list;
22945
- }
22946
- function untagTag(tag, value, path, depth) {
22947
- switch (tag) {
22948
- case DATE_TAG:
22949
- return dateOf(value, path);
22950
- case BYTES_TAG:
22951
- return bytesOf(value, path);
22952
- case NUMBER_TAG:
22953
- return numberOf(value, path);
22954
- case OBJECT_TAG:
22955
- if (!isRecord(value)) refuse(path, `holds an "${OBJECT_TAG}" tag whose value is not an object`);
22956
- return untagMap(value, path, depth);
22957
- default:
22958
- return refuse(path, `holds an object whose only key, "${tag}", names no tag`);
22959
- }
22960
- }
22961
- function untag(value, path, depth) {
22962
- if (typeof value !== "object" || value === null) return value;
22963
- if (Array.isArray(value)) return untagList(value, path, depth);
22964
- const record = value;
22965
- const tag = soleTagKey(record);
22966
- return tag === null ? untagMap(record, path, depth) : untagTag(tag, record[tag], path, depth);
22967
- }
22968
- function untagDocumentLine(text, parsed) {
22969
- if (!text.includes(`"${TAG_PREFIX}`) && !text.includes(ESCAPED_TAG_PREFIX)) return parsed;
22970
- return untagMap(parsed, "", 0);
22971
- }
22972
-
22973
- // src/engine/export/import.ts
22974
- var DOCUMENT_EXPORT_FORMAT = "narsil-export";
22975
- var DOCUMENT_EXPORT_VERSION = 1;
22976
- function invalidHeader(message) {
22977
- throw new NarsilError(ErrorCodes.CONFIG_INVALID, `The document export header ${message}`);
22978
- }
22979
- function isRecord2(value) {
22980
- return typeof value === "object" && value !== null && !Array.isArray(value);
22981
- }
22982
- function configOfHeader(line) {
22983
- if (line === void 0) invalidHeader("is missing");
22984
- if (!("text" in line)) {
22985
- invalidHeader(
22986
- line.refused === "too-long" ? `passes the line limit of ${MAX_EXPORT_LINE_BYTES} bytes` : "is not UTF-8"
22987
- );
22988
- }
22989
- let raw;
22990
- try {
22991
- raw = JSON.parse(line.text);
22992
- } catch {
22993
- invalidHeader("is not JSON");
22994
- }
22995
- if (!isRecord2(raw) || raw.format !== DOCUMENT_EXPORT_FORMAT || raw.version !== DOCUMENT_EXPORT_VERSION) {
22996
- invalidHeader(`must name format "${DOCUMENT_EXPORT_FORMAT}" and version ${DOCUMENT_EXPORT_VERSION}`);
22997
- }
22998
- const configuration = requireConfigurationShape(raw.index, "document export header");
22999
- const fields = restoredConfigFields(configuration, (message) => invalidHeader(message));
23000
- const embedding = restoredEmbedding(configuration.embedding, (message) => invalidHeader(message));
23001
- return {
23002
- schema: configuration.schema,
23003
- language: configuration.language,
23004
- ...fields,
23005
- ...embedding === void 0 ? {} : {
23006
- embedding: {
23007
- fields: embedding.fields,
23008
- ...embedding.adapter === void 0 ? {} : { adapter: embedding.adapter }
23009
- }
23010
- }
23011
- };
23012
- }
23013
- async function prepareIndex(engine, indexName, config) {
23014
- if (!engine.listIndexes().some((index) => index.name === indexName)) {
23015
- await engine.createIndex(indexName, config);
23016
- return;
23017
- }
23018
- const existing = flattenSchema(engine.getStats(indexName).schema);
23019
- const exported = flattenSchema(config.schema);
23020
- const fields = /* @__PURE__ */ new Set([...Object.keys(existing), ...Object.keys(exported)]);
23021
- for (const field of fields) {
23022
- if (existing[field] === exported[field]) continue;
23023
- throw new NarsilError(
23024
- ErrorCodes.CONFIG_INVALID,
23025
- `Index "${indexName}" gives field "${field}" another type than the export does, so the import cannot write into it`,
23026
- { indexName, field }
23027
- );
23028
- }
23029
- }
23030
- function highestReportedAt(refused) {
23031
- let highest = 0;
23032
- for (let at = 1; at < refused.length; at++) {
23033
- if (refused[at].line > refused[highest].line) highest = at;
23034
- }
23035
- return highest;
23036
- }
23037
- function refuse2(state, line, code, message) {
23038
- state.result.refusedCount += 1;
23039
- const refused = state.result.refused;
23040
- if (refused.length < MAX_IMPORT_REFUSALS_REPORTED) {
23041
- refused.push({ line, code, message });
23042
- if (refused.length === MAX_IMPORT_REFUSALS_REPORTED) state.highestReported = highestReportedAt(refused);
23043
- return;
23044
- }
23045
- if (line >= refused[state.highestReported].line) return;
23046
- refused[state.highestReported] = { line, code, message };
23047
- state.highestReported = highestReportedAt(refused);
23048
- }
23049
- async function flush(state) {
23050
- if (state.batch.length === 0) return;
23051
- const batch = state.batch;
23052
- state.batch = [];
23053
- state.batchIds = /* @__PURE__ */ new Set();
23054
- const lineOf = new Map(batch.map((entry) => [entry.docId, entry.line]));
23055
- const inserts = [];
23056
- const updates = [];
23057
- for (const entry of batch) {
23058
- if (await state.engine.has(state.indexName, entry.docId))
23059
- updates.push({ docId: entry.docId, document: entry.document });
23060
- else inserts.push(entry.document);
23061
- }
23062
- for (const outcome of [
23063
- inserts.length === 0 ? null : await state.engine.insertBatch(state.indexName, inserts, { skipClone: true }),
23064
- updates.length === 0 ? null : await state.engine.updateBatch(state.indexName, updates)
23065
- ]) {
23066
- if (outcome === null) continue;
23067
- state.result.imported += outcome.succeeded.length;
23068
- for (const failure of outcome.failed) {
23069
- const code = failure.error instanceof NarsilError ? failure.error.code : ErrorCodes.DOC_VALIDATION_FAILED;
23070
- refuse2(state, lineOf.get(failure.docId) ?? 0, code, failure.error.message);
23071
- }
23072
- }
23073
- }
23074
- async function takeLine(state, entry) {
23075
- if (!("text" in entry)) {
23076
- const message = entry.refused === "too-long" ? `The line passes the limit of ${MAX_EXPORT_LINE_BYTES} bytes` : "The line is not UTF-8";
23077
- refuse2(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, message);
23078
- return;
23079
- }
23080
- if (entry.text.length === 0) return;
23081
- let document;
23082
- try {
23083
- document = JSON.parse(entry.text);
23084
- } catch {
23085
- refuse2(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, "The line is not JSON");
23086
- return;
23087
- }
23088
- if (!isRecord2(document) || typeof document.id !== "string" || document.id.length === 0) {
23089
- refuse2(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, "The line holds no document with a string id");
23090
- return;
23091
- }
23092
- try {
23093
- untagDocumentLine(entry.text, document);
23094
- } catch (error) {
23095
- if (!(error instanceof NarsilError)) throw error;
23096
- refuse2(state, entry.line, error.code, error.message);
23097
- return;
23098
- }
23099
- if (state.batchIds.has(document.id)) await flush(state);
23100
- state.batch.push({ line: entry.line, docId: document.id, document });
23101
- state.batchIds.add(document.id);
23102
- if (state.batch.length >= IMPORT_BATCH_DOCUMENTS) await flush(state);
23103
- }
23104
- async function readDocumentExport(engine, indexName, source) {
23105
- const lines = exportLines(source, MAX_EXPORT_LINE_BYTES);
23106
- const header = await lines.next();
23107
- await prepareIndex(engine, indexName, configOfHeader(header.done === true ? void 0 : header.value));
23108
- const state = {
23109
- engine,
23110
- indexName,
23111
- batch: [],
23112
- batchIds: /* @__PURE__ */ new Set(),
23113
- result: { imported: 0, refusedCount: 0, refused: [] },
23114
- highestReported: 0
23115
- };
23116
- for await (const entry of lines) await takeLine(state, entry);
23117
- await flush(state);
23118
- state.result.refused.sort((a, b) => a.line - b.line);
23119
- return state.result;
23120
- }
23121
-
23122
22382
  // src/narsil/portability.ts
23123
22383
  function portabilityOperations(core, config, engine) {
23124
22384
  async function withOpenIndex2(indexName, action) {
@@ -23150,14 +22410,14 @@ function portabilityOperations(core, config, engine) {
23150
22410
  return {
23151
22411
  snapshot(indexName) {
23152
22412
  return withOpenIndex2(indexName, async () => {
23153
- const { createSnapshot } = await import('./snapshot-V7VF6BN6.mjs');
22413
+ const { createSnapshot } = await import('./snapshot-QXBKHJAG.mjs');
23154
22414
  return createSnapshot(snapshotDepsOf(indexName));
23155
22415
  });
23156
22416
  },
23157
22417
  async restore(indexName, data) {
23158
22418
  core.guardShutdown();
23159
22419
  validateIndexName(indexName);
23160
- const restored = import('./restore-queue-GEZ54CA5.mjs').then(
22420
+ const restored = import('./restore-queue-ONYNYUDE.mjs').then(
23161
22421
  ({ queueRestore }) => queueRestore(core, restoreHost, restoresInFlight, indexName, data)
23162
22422
  );
23163
22423
  await emitWriteEventAfter(core, indexName, restored);
@@ -23166,8 +22426,8 @@ function portabilityOperations(core, config, engine) {
23166
22426
  exportDocuments(indexName) {
23167
22427
  return withOpenIndex2(indexName, async () => {
23168
22428
  const [{ checkpointForArchive }, { documentExportStream }] = await Promise.all([
23169
- import('./snapshot-V7VF6BN6.mjs'),
23170
- import('./write-YIC2MKRY.mjs')
22429
+ import('./snapshot-QXBKHJAG.mjs'),
22430
+ import('./write-WHFBDWQS.mjs')
23171
22431
  ]);
23172
22432
  const header = indexConfigurationOf(core.requireIndex(indexName));
23173
22433
  return documentExportStream(header, await checkpointForArchive(snapshotDepsOf(indexName)));
@@ -23175,7 +22435,10 @@ function portabilityOperations(core, config, engine) {
23175
22435
  },
23176
22436
  importDocumentExport(indexName, data) {
23177
22437
  core.guardShutdown();
23178
- return emitWriteEventAfter(core, indexName, readDocumentExport(engine(), indexName, data));
22438
+ const imported = import('./import-TLB2A5LJ.mjs').then(
22439
+ ({ readDocumentExport }) => readDocumentExport(engine(), indexName, data)
22440
+ );
22441
+ return emitWriteEventAfter(core, indexName, imported);
23179
22442
  }
23180
22443
  };
23181
22444
  }
@@ -23185,14 +22448,14 @@ var ABSENT_TAG = 0;
23185
22448
  var NULL_TAG = 1;
23186
22449
  var FALSE_TAG = 2;
23187
22450
  var TRUE_TAG = 3;
23188
- var NUMBER_TAG2 = 4;
22451
+ var NUMBER_TAG = 4;
23189
22452
  var STRING_TAG = 5;
23190
22453
  var LIST_TAG = 6;
23191
22454
  var MAP_TAG = 7;
23192
22455
  var LENGTH_BYTES = 4;
23193
22456
  var LAST_ASCII_UNIT = 127;
23194
22457
  var encoder5 = new TextEncoder();
23195
- function flush2(stream) {
22458
+ function flush(stream) {
23196
22459
  stream.hash = fnv1aContinue(stream.hash, stream.bytes, stream.length);
23197
22460
  stream.length = 0;
23198
22461
  }
@@ -23256,8 +22519,8 @@ function writeNumberList(stream, list) {
23256
22519
  if (stream.clock !== void 0) stream.clock.tick(list.members.length);
23257
22520
  writeByte(stream, LIST_TAG);
23258
22521
  writeUint32(stream, list.members.length);
23259
- flush2(stream);
23260
- stream.hash = fnv1aContinueTaggedNumbers(stream.hash, NUMBER_TAG2, list.members);
22522
+ flush(stream);
22523
+ stream.hash = fnv1aContinueTaggedNumbers(stream.hash, NUMBER_TAG, list.members);
23261
22524
  }
23262
22525
  function writeValue(stream, value) {
23263
22526
  if (value instanceof NumberList) {
@@ -23277,7 +22540,7 @@ function writeValue(stream, value) {
23277
22540
  return;
23278
22541
  }
23279
22542
  if (typeof value === "number") {
23280
- writeByte(stream, NUMBER_TAG2);
22543
+ writeByte(stream, NUMBER_TAG);
23281
22544
  writeNumber(stream, value);
23282
22545
  return;
23283
22546
  }
@@ -23351,7 +22614,7 @@ function bindingOf(values, clock) {
23351
22614
  for (const value of values) {
23352
22615
  writeValue(stream, value);
23353
22616
  }
23354
- flush2(stream);
22617
+ flush(stream);
23355
22618
  return (stream.hash >>> 0).toString(16).padStart(8, "0");
23356
22619
  }
23357
22620
  function queryBindingOf(params, clock) {
@@ -24472,7 +23735,7 @@ function requireCursorDepth(cursor, signature, binding) {
24472
23735
  }
24473
23736
  function hybridSearchOf(params, context, limit, offset, cursorDepth) {
24474
23737
  if ((params.hybrid?.depth ?? null) === null) return executeHybridSearch(params, context, limit, offset, cursorDepth);
24475
- return import('./runtime-3X7ZJ5T3.mjs').then(
23738
+ return import('./runtime-VEUH674B.mjs').then(
24476
23739
  (runtime) => runtime.searchFixedDepthHybrid(params, context, { cursorDepth: cursorDepth ?? 0, offset, limit }, runtimeHost)
24477
23740
  );
24478
23741
  }
@@ -24482,7 +23745,7 @@ function executeQuery(request, callerContext) {
24482
23745
  if (loadingPhraseCode === null) return runQuery(request, callerContext);
24483
23746
  return loadingPhraseCode.then(() => runQuery(request, callerContext));
24484
23747
  }
24485
- return import('./runtime-3X7ZJ5T3.mjs').then(
23748
+ return import('./runtime-VEUH674B.mjs').then(
24486
23749
  (runtime) => runtime.executeRerankedQuery(request, callerContext, executeQuery, runtimeHost)
24487
23750
  );
24488
23751
  }
@@ -24838,16 +24101,22 @@ function pickDisplaySurface(surfaceOccurrences) {
24838
24101
  // src/narsil/reads.ts
24839
24102
  async function enterRead(core, indexName) {
24840
24103
  core.guardShutdown();
24841
- if (core.rebalancingIndexes.has(indexName)) await yieldToEventLoop();
24104
+ if (core.rebalancingIndexes.has(indexName)) await yieldToEventLoop2();
24842
24105
  }
24843
24106
  async function readUnderOneAnalysis(core, indexName, read) {
24844
- let analysisStale = core.analysisRebuild.isStale(indexName);
24845
- let result = await read();
24846
- while (core.analysisRebuild.isStale(indexName) !== analysisStale) {
24847
- analysisStale = core.analysisRebuild.isStale(indexName);
24848
- result = await read();
24107
+ const manager = core.requireManager(indexName);
24108
+ for (; ; ) {
24109
+ const analysis = manager.analysis;
24110
+ const analysisStale = core.analysisRebuild.isStale(indexName);
24111
+ try {
24112
+ const result = await read();
24113
+ if (manager.analysis === analysis && core.analysisRebuild.isStale(indexName) === analysisStale) {
24114
+ return { result, analysisStale };
24115
+ }
24116
+ } catch (error) {
24117
+ if (manager.analysis === analysis) throw error;
24118
+ }
24849
24119
  }
24850
- return { result, analysisStale };
24851
24120
  }
24852
24121
  function phraseReadinessOf(core, indexName, config) {
24853
24122
  return () => {
@@ -24875,7 +24144,7 @@ async function runEngineQuery(core, indexName, params, options) {
24875
24144
  const manager = core.requireManager(indexName);
24876
24145
  const scoped = scopedParams(params, options);
24877
24146
  await core.pluginRegistry.runHook("beforeSearch", { indexName, params: scoped });
24878
- const rerank = scoped.rerank === void 0 ? void 0 : await import('./runtime-3X7ZJ5T3.mjs');
24147
+ const rerank = scoped.rerank === void 0 ? void 0 : await import('./runtime-VEUH674B.mjs');
24879
24148
  const resolvedParams = await resolveVectorText(
24880
24149
  rerank === void 0 ? scoped : rerank.withRerankQueryText(scoped),
24881
24150
  entry.embeddingAdapter,
@@ -24883,25 +24152,24 @@ async function runEngineQuery(core, indexName, params, options) {
24883
24152
  entry.embeddingAdapterName
24884
24153
  );
24885
24154
  const workerSearch = core.orchestrator.hasWorkerPool() ? core.orchestrator.searchViaWorker.bind(core.orchestrator) : void 0;
24886
- const context = {
24887
- manager,
24888
- language: entry.language,
24889
- config: entry.config,
24890
- workerSearch,
24891
- indexName,
24892
- broadcastStats: broadcastStatsFor(core, options),
24893
- partitionIds: options?.partitionIds,
24894
- rescoreWindow: options?.rescoreWindow,
24895
- indexBm25: options?.indexBm25,
24896
- cursorBinding: lazyQueryBinding(params),
24897
- patternWorkCap: core.patternWorkCap,
24898
- rerank: rerank?.rerankRunnerFor(core),
24899
- requirePhraseReady: phraseReadinessOf(core, indexName, entry.config)
24900
- };
24901
24155
  const { result, analysisStale } = await readUnderOneAnalysis(
24902
24156
  core,
24903
24157
  indexName,
24904
- () => executeQuery(resolvedParams, context)
24158
+ () => executeQuery(resolvedParams, {
24159
+ manager,
24160
+ language: manager.language,
24161
+ config: manager.config,
24162
+ workerSearch,
24163
+ indexName,
24164
+ broadcastStats: broadcastStatsFor(core, options),
24165
+ partitionIds: options?.partitionIds,
24166
+ rescoreWindow: options?.rescoreWindow,
24167
+ indexBm25: options?.indexBm25,
24168
+ cursorBinding: lazyQueryBinding(params),
24169
+ patternWorkCap: core.patternWorkCap,
24170
+ rerank: rerank?.rerankRunnerFor(core),
24171
+ requirePhraseReady: phraseReadinessOf(core, indexName, entry.config)
24172
+ })
24905
24173
  );
24906
24174
  if (analysisStale) {
24907
24175
  result.analysisStale = true;
@@ -24935,23 +24203,22 @@ async function runEnginePreflight(core, indexName, params, options) {
24935
24203
  entry.embeddingAdapterName
24936
24204
  );
24937
24205
  const workerSearch = core.orchestrator.hasWorkerPool() ? core.orchestrator.searchViaWorker.bind(core.orchestrator) : void 0;
24938
- const context = {
24939
- manager,
24940
- language: entry.language,
24941
- config: entry.config,
24942
- workerSearch,
24943
- indexName,
24944
- broadcastStats: broadcastStatsFor(core, options),
24945
- partitionIds: options?.partitionIds,
24946
- indexBm25: options?.indexBm25,
24947
- cursorBinding: lazyQueryBinding(params),
24948
- patternWorkCap: core.patternWorkCap,
24949
- requirePhraseReady: phraseReadinessOf(core, indexName, entry.config)
24950
- };
24951
24206
  const { result, analysisStale } = await readUnderOneAnalysis(
24952
24207
  core,
24953
24208
  indexName,
24954
- () => executePreflight(resolvedParams, context)
24209
+ () => executePreflight(resolvedParams, {
24210
+ manager,
24211
+ language: manager.language,
24212
+ config: manager.config,
24213
+ workerSearch,
24214
+ indexName,
24215
+ broadcastStats: broadcastStatsFor(core, options),
24216
+ partitionIds: options?.partitionIds,
24217
+ indexBm25: options?.indexBm25,
24218
+ cursorBinding: lazyQueryBinding(params),
24219
+ patternWorkCap: core.patternWorkCap,
24220
+ requirePhraseReady: phraseReadinessOf(core, indexName, entry.config)
24221
+ })
24955
24222
  );
24956
24223
  if (analysisStale) {
24957
24224
  result.analysisStale = true;
@@ -25039,8 +24306,11 @@ function createNarsilFromCore(core, config) {
25039
24306
  release();
25040
24307
  }
25041
24308
  }
24309
+ const rebalancesStarting = /* @__PURE__ */ new Map();
25042
24310
  function withOpenIndexWrite(indexName, action) {
25043
- return emitWriteEventAfter(core, indexName, withOpenIndex2(indexName, action));
24311
+ const starting = rebalancesStarting.get(indexName);
24312
+ const written = starting === void 0 ? withOpenIndex2(indexName, action) : starting.then(() => withOpenIndex2(indexName, action));
24313
+ return emitWriteEventAfter(core, indexName, written);
25044
24314
  }
25045
24315
  const narsil = {
25046
24316
  createIndex(name, indexConfig) {
@@ -25164,9 +24434,21 @@ function createNarsilFromCore(core, config) {
25164
24434
  return runEngineSuggest(core, indexName, params);
25165
24435
  },
25166
24436
  async rebuildAnalysis(indexName) {
24437
+ guardShutdown();
24438
+ const entry = requireIndex(indexName);
24439
+ await withOpenIndexWrite(indexName, async () => {
24440
+ if (entry.pendingAnalysis === null) return core.analysisRebuild.rebuild(indexName);
24441
+ return (await import('./analysis-change-JPV775LR.mjs')).resumeAnalysisChange(core, indexName);
24442
+ });
24443
+ },
24444
+ async changeAnalysis(indexName, change) {
25167
24445
  guardShutdown();
25168
24446
  requireIndex(indexName);
25169
- await withOpenIndexWrite(indexName, () => core.analysisRebuild.rebuild(indexName));
24447
+ const changed = withOpenIndex2(
24448
+ indexName,
24449
+ async () => (await import('./analysis-change-JPV775LR.mjs')).changeIndexAnalysis(core, indexName, change)
24450
+ );
24451
+ await emitWriteEventAfter(core, indexName, changed);
25170
24452
  },
25171
24453
  ...portabilityOperations(core, config, () => narsil),
25172
24454
  ...synonymOperations(core),
@@ -25185,15 +24467,29 @@ function createNarsilFromCore(core, config) {
25185
24467
  await executor.execute({ type: "clear", indexName, requestId: indexName });
25186
24468
  await orchestrator.replicateToWorkers({ type: "clear", indexName, requestId: `replicate-clear-${indexName}` });
25187
24469
  core.watermarkNotifier.forget(indexName);
24470
+ const durable = core.durability?.manager;
24471
+ await (durable?.checkpointFromMemory?.(indexName) ?? durable?.checkpoint(indexName));
25188
24472
  });
25189
24473
  },
25190
24474
  async rebalance(indexName, targetPartitionCount) {
25191
24475
  guardShutdown();
25192
24476
  requireIndex(indexName);
25193
- return withOpenIndexWrite(
25194
- indexName,
25195
- () => executeRebalance(requireManager(indexName), indexName, targetPartitionCount, rebalanceCtx)
25196
- );
24477
+ let writesMayFollow = () => void 0;
24478
+ const started = new Promise((resolve) => {
24479
+ writesMayFollow = resolve;
24480
+ });
24481
+ function releaseWrites() {
24482
+ writesMayFollow();
24483
+ if (rebalancesStarting.get(indexName) === started) rebalancesStarting.delete(indexName);
24484
+ }
24485
+ rebalancesStarting.set(indexName, started);
24486
+ const rebalanced = withOpenIndex2(indexName, async () => {
24487
+ const { executeRebalance } = await import('./analysis-change-JPV775LR.mjs');
24488
+ const running = executeRebalance(requireManager(indexName), indexName, targetPartitionCount, rebalanceCtx);
24489
+ releaseWrites();
24490
+ return running;
24491
+ });
24492
+ return emitWriteEventAfter(core, indexName, rebalanced).finally(releaseWrites);
25197
24493
  },
25198
24494
  async updatePartitionConfig(indexName, partitionConfig) {
25199
24495
  guardShutdown();
@@ -25312,4 +24608,4 @@ async function createNarsil(config) {
25312
24608
  // src/index.ts
25313
24609
  var VERSION = "0.1.0";
25314
24610
 
25315
- export { ARRAY_POSITION_GAP, BYTES_TAG, DATE_TAG, DEFAULT_BM25_B, DEFAULT_BM25_K1, DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, HEADER_SIZE, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, NUMBER_TAG, OBJECT_TAG, VECTOR_PATTERN, VERSION, arrayPositionsAreStale, buildSegmentPayload, clearNormalizationCache, configureNormalizationCache, crc32, createByteReader, createFrozenSegment, createNarsil, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, decodeEveryStoredDocument, decodeIndexConfiguration, decodeMapWithoutFields, decodeStoredDocument, encodeBase64, encodeDocument, encodeIndexConfiguration, engineVersionParts, extractVectorFieldsFromSchema, flattenSchema, fnv1a, freezeSegmentShared, getLanguage, getNestedValue, getNormalizationCacheSize, getStopWords, getTokenizer, hasNrslMagic, hasStopWords, hasTokenizer, insertDocumentVectors, isCompositePartition, isQuantizationMode, isStorageMode, loadVectorCodeFor, mergeSegmentContents, messagePackUpperBoundBytes, openChecksummedEnvelope, packChecksummedEnvelope, parseVectorDimension, partialFieldsOf, partitionStateOf, prepareDocumentVectors, prepareStoredSynonyms, raiseEarlierReleaseGraphSetting, readHeader, readHeapStatistics, readResidentMemory, reconstructSchemaFromMetadata, registerLanguage, registerPartialWordSearch, registerPatternSearch, registerStopWords, registerSynonyms, registerTokenizer, rejectSnapshot, removeDocumentVectors, requireSurfaceFormsForPartialFields, requireValidFieldTypes, resetNormalizationCache, resolveIndexAnalysis, resolvePartitionInsertOptions, restoredConfigFields, restoredEmbedding, rulesOfStoredSynonyms, segmentArraysOf, selectionKeeps, shareSegmentArrays, startsWithTagPrefix, storedArrayPositionGap, survivorDocumentsOf, validateEmbeddingConfig, validatePatternValueLimit, validateRequiredFieldsInSchema, validateSchema, validateVectorPromotion, validateVectorStorage, writeHeader };
24611
+ export { ARRAY_POSITION_GAP, DEFAULT_BM25_B, DEFAULT_BM25_K1, DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE, HEADER_SIZE, MAX_DOCUMENT_NESTING_DEPTH, REBALANCE_CHUNK_SIZE, VECTOR_PATTERN, VERSION, arrayPositionsAreStale, assemblePagedFrozenSegment, buildPartialWordTableFrom, buildSegmentPayload, checksumOfChunks, clearNormalizationCache, configureNormalizationCache, crc32, crc32Final, crc32Init, crc32Update, createCompositePartition, createEncodedDocumentSource, createFrozenSegment, createNarsil, createPartitionIndex, createPartitionManager, createPartitionRouter, createRequestId, createSharedFrozenSegment, decodeEveryStoredDocument, decodeIndexConfiguration, decodeMapWithoutFields, decodeStoredDocument, decodeSurfaceTable, deleteNestedValue, emitEngineEvent, encodeDocument, encodeIndexConfiguration, encodeSurfaceTable, endOfValue, engineVersionParts, extractVectorFieldsFromSchema, extractVectorFromDoc, fieldIndexRemapOf, flattenSchema, fnv1a, freezeSegmentShared, generateId, getLanguage, getNestedValue, getNormalizationCacheSize, getStopWords, getTokenizer, hasNrslMagic, hasStopWords, hasTokenizer, holdsADiscardedField, insertDocumentVectors, isCompositePartition, isQuantizationMode, isStorageMode, loadVectorCodeFor, mergeDocFrequencies, mergeFieldIndexes2 as mergeFieldIndexes, mergeFieldLengths, mergePartialWords, mergeSegmentContents, mergeSurfaceForms2 as mergeSurfaceForms, messagePackUpperBoundBytes, openChecksummedEnvelope, packChecksummedEnvelope, packEnvelopeWithChecksum, parseVectorDimension, partialFieldsOf, partitionStateOf, prepareConfiguredSynonyms, prepareDocumentVectors, prepareStoredSynonyms, raiseEarlierReleaseGraphSetting, readHeader, readHeapStatistics, readResidentMemory, rebuildTextIndexes, reconstructSchemaFromMetadata, registerLanguage, registerPartialWordSearch, registerPatternSearch, registerStopWords, registerSynonyms, registerTokenizer, removeDocumentVectors, requireConfigurationShape, requireConfigurationWithinSizeLimit, requireSurfaceFormsForPartialFields, requireValidFieldTypes, requireWithinFileLimit, resetNormalizationCache, resolveIndexAnalysis, resolvePartialWordAnalysis, resolvePartitionInsertOptions, rewriteSpillOf, rulesOfStoredSynonyms, scanPostingBlocks, segmentArraysOf, selectionKeeps, shareSegmentArrays, storedArrayPositionGap, survivorDocumentsOf, textRebuildTargetOf, validateEmbeddingConfig, validatePatternValueLimit, validateRequiredFieldsInSchema, validateSchema, validateTextStorage, validateVectorPromotion, validateVectorStorage, vectorsEqual, wrapFrozenTokenTable, writeHeader, yieldToEventLoop2 as yieldToEventLoop };