@delali/narsil 0.3.1-next.140 → 0.3.1-next.141

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. package/dist/{analysis-change-JN23AZQP.mjs → analysis-change-HQUT2Z3I.mjs} +13 -13
  2. package/dist/{analysis-change-entry-5JSIMEKG.mjs → analysis-change-entry-TZTG4AZB.mjs} +1 -1
  3. package/dist/browser/{analysis-change-57MIRJWR.mjs → analysis-change-JPV775LR.mjs} +407 -6
  4. package/dist/browser/{analysis-change-entry-ICGRR3IN.mjs → analysis-change-entry-7HR4LWL3.mjs} +1 -1
  5. package/dist/browser/chunk-3ZBJYAS4.mjs +19 -0
  6. package/dist/browser/{chunk-ZAIZQOTM.mjs → chunk-5DB2VDGI.mjs} +951 -1134
  7. package/dist/browser/chunk-6RHTAVZX.mjs +656 -0
  8. package/dist/browser/chunk-CCOAOO73.mjs +86 -0
  9. package/dist/browser/{chunk-3EYDURFD.mjs → chunk-HKX2G6WH.mjs} +1615 -2266
  10. package/dist/browser/{chunk-CGVBGHBX.mjs → chunk-KTOK275N.mjs} +531 -830
  11. package/dist/browser/{chunk-5LYWZJE2.mjs → chunk-LVY2LI35.mjs} +1 -1
  12. package/dist/browser/chunk-MVM33ZUL.mjs +993 -0
  13. package/dist/browser/chunk-MWUK3DA5.mjs +2634 -0
  14. package/dist/browser/{chunk-67KSBUYR.mjs → chunk-R4WUDZQV.mjs} +8 -5
  15. package/dist/browser/{chunk-6BSB4N6B.mjs → chunk-TCLEOZ4P.mjs} +2 -2
  16. package/dist/browser/chunk-VFYMPCOV.mjs +212 -0
  17. package/dist/browser/chunk-WCFCHIBB.mjs +177 -0
  18. package/dist/browser/{chunk-E5SFTOQR.mjs → chunk-XNWPP5NS.mjs} +2 -2
  19. package/dist/browser/{chunk-QHNNULPW.mjs → chunk-ZEV26AQ6.mjs} +5 -0
  20. package/dist/browser/cold-module-MJMKNUYJ.mjs +745 -0
  21. package/dist/browser/{durability-wiring-MTIAN6VT.mjs → durability-wiring-SNOWIT6Z.mjs} +231 -51
  22. package/dist/browser/{import-Q7MXD6BS.mjs → import-TLB2A5LJ.mjs} +6 -5
  23. package/dist/browser/module-BUFNZMEU.mjs +12 -0
  24. package/dist/browser/{module-5J3PEUEB.mjs → module-BVP4TKZU.mjs} +395 -108
  25. package/dist/{restore-queue-RYSRRFS4.mjs → browser/restore-queue-ONYNYUDE.mjs} +2 -2
  26. package/dist/browser/{snapshot-YBJ5MF2M.mjs → snapshot-QXBKHJAG.mjs} +108 -25
  27. package/dist/browser/{write-RTWI5MXL.mjs → write-WHFBDWQS.mjs} +63 -91
  28. package/dist/{capabilities-DXAmysqJ.d.ts → capabilities-BCdUwK_L.d.ts} +2 -2
  29. package/dist/{chunk-HMDT3I63.mjs → chunk-46NZOPHS.mjs} +2 -2
  30. package/dist/{chunk-XXKWEG53.mjs → chunk-5WYVUPGD.mjs} +1 -1
  31. package/dist/chunk-6SNBQESJ.mjs +214 -0
  32. package/dist/chunk-7AUASRE7.mjs +653 -0
  33. package/dist/{chunk-4GCN34JV.mjs → chunk-AST3SHFA.mjs} +26 -24
  34. package/dist/chunk-BOJZGUS6.mjs +177 -0
  35. package/dist/{chunk-IP4HJOGC.mjs → chunk-BRUUBXCB.mjs} +4 -3
  36. package/dist/{chunk-J3EHATEZ.mjs → chunk-DQNKVOJ2.mjs} +63 -63
  37. package/dist/chunk-FOCV4CBZ.mjs +87 -0
  38. package/dist/{chunk-FFVLPCLH.mjs → chunk-GLZHNSGB.mjs} +1 -1
  39. package/dist/chunk-GWOS5DYC.mjs +2637 -0
  40. package/dist/{chunk-W2XDK66Z.mjs → chunk-IO5F7ZBE.mjs} +112 -438
  41. package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
  42. package/dist/{chunk-LLDFIKGP.mjs → chunk-KEIYVZUX.mjs} +4 -4
  43. package/dist/{chunk-UBXJQJII.mjs → chunk-KGOXHLLZ.mjs} +5 -1
  44. package/dist/{chunk-FKARMUWH.mjs → chunk-KZL72VTC.mjs} +68 -16
  45. package/dist/{chunk-TCWQJL7O.mjs → chunk-L76W3IZJ.mjs} +5 -0
  46. package/dist/{chunk-CPCGEIOM.mjs → chunk-LA4WNDHL.mjs} +83 -799
  47. package/dist/{chunk-PNNVTKMT.mjs → chunk-NJJCPFXN.mjs} +2 -2
  48. package/dist/chunk-NN33CZRE.mjs +86 -0
  49. package/dist/chunk-NRUXBYPH.mjs +121 -0
  50. package/dist/{chunk-UJWGQIXU.mjs → chunk-QHPYS427.mjs} +1 -0
  51. package/dist/{chunk-VSUYTPME.mjs → chunk-RHKRTYGO.mjs} +13 -4
  52. package/dist/{chunk-UEV53BBO.mjs → chunk-SRJBFG2T.mjs} +406 -5
  53. package/dist/{chunk-K3AQOZVM.mjs → chunk-TNBIK2LQ.mjs} +1638 -1644
  54. package/dist/chunk-UUZCQQG2.mjs +209 -0
  55. package/dist/{chunk-XMQNOD5V.mjs → chunk-VEFN6X6Q.mjs} +2 -2
  56. package/dist/{chunk-LK3MOTXF.mjs → chunk-VJEGOELZ.mjs} +8 -5
  57. package/dist/chunk-W2COJ5AH.mjs +148 -0
  58. package/dist/{chunk-IBATGS3J.mjs → chunk-WJGO5PGV.mjs} +241 -124
  59. package/dist/{chunk-CCCGEPJM.mjs → chunk-XZ3SRNDJ.mjs} +24 -63
  60. package/dist/{chunk-MTQTX4BP.mjs → chunk-YHUTF6EA.mjs} +400 -92
  61. package/dist/{chunk-B5I3IL4H.mjs → chunk-YLNSJQLX.mjs} +4 -4
  62. package/dist/{chunk-6VHWXAOI.mjs → chunk-Z4HFMYNX.mjs} +29 -24
  63. package/dist/{chunk-U4X4AWKL.mjs → chunk-ZPWS2SKJ.mjs} +293 -270
  64. package/dist/{client-DApBeLbk.d.ts → client-B12YFALP.d.ts} +3 -3
  65. package/dist/client.d.ts +7 -7
  66. package/dist/cold-module-4X7WJCFM.mjs +783 -0
  67. package/dist/{config-B0H-vp4U.d.ts → config-BhZxIeb1.d.ts} +6 -1
  68. package/dist/{config-Brysl9NO.d.ts → config-CIG5yDyg.d.ts} +2 -2
  69. package/dist/{direct-executor-J5UDLPL3.mjs → direct-executor-7XOQ7VQP.mjs} +5 -5
  70. package/dist/distribution/coordinator/etcd.d.ts +2 -2
  71. package/dist/distribution/coordinator/in-memory.d.ts +2 -2
  72. package/dist/distribution.d.ts +7 -7
  73. package/dist/distribution.mjs +123 -42
  74. package/dist/{durability-wiring-AYOTAPNQ.mjs → durability-wiring-324D2DDD.mjs} +213 -40
  75. package/dist/{engine-CuLMbH1-.d.ts → engine-CtzbHOmG.d.ts} +2 -2
  76. package/dist/{import-QRZAQ5RH.mjs → import-OOBUW3RR.mjs} +14 -13
  77. package/dist/{index-HYnqS1SE.d.ts → index-DsEurhSl.d.ts} +1 -1
  78. package/dist/index.browser.mjs +3 -3
  79. package/dist/index.d.ts +6 -6
  80. package/dist/index.mjs +11 -11
  81. package/dist/{long-query-thread-U7K5A72G.mjs → long-query-thread-4SW7RWDE.mjs} +8 -8
  82. package/dist/{module-L2OZO5PQ.mjs → module-DXSJ5LT2.mjs} +374 -83
  83. package/dist/module-IYPLWHGC.mjs +48 -0
  84. package/dist/paged-partition-archive-HFP45PLU.mjs +269 -0
  85. package/dist/partial-words.mjs +2 -2
  86. package/dist/persistence/durability/checkpoint-worker.d.ts +5 -1
  87. package/dist/persistence/durability/checkpoint-worker.mjs +17 -15
  88. package/dist/react.d.ts +9 -9
  89. package/dist/{browser/restore-queue-DISFECE4.mjs → restore-queue-FEYAC4SB.mjs} +2 -2
  90. package/dist/{schema-BTXbHCqA.d.ts → schema-B-Runv0n.d.ts} +7 -0
  91. package/dist/server.d.ts +10 -10
  92. package/dist/server.mjs +8 -7
  93. package/dist/{snapshot-BTRZ3MZK.mjs → snapshot-CGM65VCD.mjs} +60 -25
  94. package/dist/{tasks-DjGjK70Y.d.ts → tasks-CjRJRGTb.d.ts} +6 -4
  95. package/dist/{thread-USB3GOPY.mjs → thread-KGNAIKPD.mjs} +8 -8
  96. package/dist/vector/search-worker.d.ts +1 -1
  97. package/dist/vector/search-worker.mjs +3 -2
  98. package/dist/workers/entry.mjs +7 -7
  99. package/dist/{write-GHTA56OW.mjs → write-2DZPEQL2.mjs} +70 -98
  100. package/openapi.json +13 -0
  101. package/package.json +12 -8
  102. package/dist/browser/chunk-XVMOQQ34.mjs +0 -384
  103. package/dist/chunk-BDIDUW2B.mjs +0 -387
  104. package/dist/browser/{chunk-4MFH226Q.mjs → chunk-D33A5VMU.mjs} +15 -15
@@ -1,6 +1,6 @@
1
- import { GrowableUint32, SEGMENT_TOKEN_TABLE_START, SEGMENT_POSTING_LOG_START, SEGMENT_POSITION_POOL_START, SEGMENT_FIELD_SCRATCH_START, LOOKUP_KEY_SCRATCH_BYTES, bitsetHas, bitsetSet, createBitSet, bitsetIsEmpty, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY, PRUNING_REJECTION_SLACK, bitsetOrInto, bitsetAndNotInto, bitsetAndInto, SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD, TARGET_BLOCK_ENTRIES, SORT_COLUMN_REBUILD_FRACTION_SHIFT, bitsetFromSet } from './chunk-26A5JFT2.mjs';
2
- import { MAX_DOCUMENT_ENCODING_DEPTH, singleFoldTable, appendFoldedCodePoint, compareCodePoints, EARLIER_RELEASE_MIN_M, MIN_M, MAX_M, DOCUMENT_ENCODER_RETAINED_BUFFER_BYTES, MAX_FILE_BYTES, detectRuntime, splitWorkerBudget, resolveWorkerCount, CHECKSUM_YIELD_CHUNK_BYTES, resolveWorkerEntry, missingWorkerEntry, DEFAULT_REQUEST_TIMEOUT_MS, toReducedSortValue, resolveRequestThreadCount, defaultSortMode, holdsUnpairedSurrogate, compareComparableValues, readSortField, exceedsCodePoints, toComparableSortValue, SORT_VALUE_MAX_CODE_POINTS, compareComparableKeys, truncateSortString, sortModeOf, isSortMode, SORT_MODES, SHUTDOWN_TIMEOUT_MS, MIN_OVERSAMPLE, MAX_RENUMBERED_SEARCH_ATTEMPTS, compareSortValues, DEFAULT_BACKPRESSURE_LIMIT, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, codePointCount, noteThreadSlotGone, threadSlotOfWorker } from './chunk-4MFH226Q.mjs';
3
1
  import { DEFAULT_MAX_RERANK_WINDOW, DEFAULT_RERANK_TIMEOUT_MS } from './chunk-JPTQZWJQ.mjs';
2
+ import { TARGET_BLOCK_ENTRIES, bitsetHas, createBitSet, LOOKUP_KEY_SCRATCH_BYTES, GrowableUint32, SEGMENT_TOKEN_TABLE_START, SEGMENT_POSTING_LOG_START, SEGMENT_POSITION_POOL_START, SEGMENT_FIELD_SCRATCH_START, bitsetIsEmpty, bitsetSet, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY, PRUNING_REJECTION_SLACK, bitsetOrInto, bitsetAndNotInto, bitsetAndInto, SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD, SORT_COLUMN_REBUILD_FRACTION_SHIFT, bitsetFromSet } from './chunk-26A5JFT2.mjs';
3
+ import { MAX_DOCUMENT_ENCODING_DEPTH, singleFoldTable, appendFoldedCodePoint, compareCodePoints, EARLIER_RELEASE_MIN_M, MIN_M, MAX_M, MAX_FILE_BYTES, DOCUMENT_ENCODER_RETAINED_BUFFER_BYTES, CHECKSUM_YIELD_CHUNK_BYTES, detectRuntime, splitWorkerBudget, resolveWorkerCount, toReducedSortValue, compareComparableValues, resolveWorkerEntry, missingWorkerEntry, DEFAULT_REQUEST_TIMEOUT_MS, defaultSortMode, resolveRequestThreadCount, readSortField, holdsUnpairedSurrogate, truncateSortString, exceedsCodePoints, toComparableSortValue, SORT_VALUE_MAX_CODE_POINTS, compareComparableKeys, sortModeOf, isSortMode, SORT_MODES, SHUTDOWN_TIMEOUT_MS, MIN_OVERSAMPLE, MAX_RENUMBERED_SEARCH_ATTEMPTS, compareSortValues, DEFAULT_BACKPRESSURE_LIMIT, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, codePointCount, noteThreadSlotGone, threadSlotOfWorker } from './chunk-D33A5VMU.mjs';
4
4
  import { MAX_INDEX_CONFIGURATION_BYTES, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, STATISTICS_BROADCAST_INTERVAL_MS, MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_SUGGEST_LIMIT, FOREIGN_STATS_TTL_MS, HEAP_PRESSURE_WARNING_FRACTION, DEFAULT_SUGGEST_LIMIT, DEFAULT_LIMIT, DEFAULT_OFFSET, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION, BATCH_CHUNK_SIZE, MAX_DOC_ID_LENGTH } from './chunk-MZI3TEOW.mjs';
5
5
  import { ErrorCodes, NarsilError, createNarsilError, DEFAULT_FUZZY_PREFIX_LENGTH, MAX_CURSOR_LENGTH, MAX_CURSOR_ANCHOR_CODE_POINTS, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, BM25_DISPLAYED_NAME_LENGTH, RESULT_WINDOW, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, DEFAULT_PAGE_SIZE, DEFAULT_MAX_PER_GROUP, PREFIX_MAX_EXPANSIONS, MAX_TOLERANCE, MAX_PREFIX_LENGTH } from './chunk-35B42UFM.mjs';
6
6
  import { Encoder, Decoder, decode, encode } from '@msgpack/msgpack';
@@ -2287,6 +2287,162 @@ var PATTERN_TEST_OPERATORS = ["eq", "ne", "in", "nin", "startsWith", "endsWith",
2287
2287
  var PATTERN_EXPRESSION_OPERATORS = ["wildcard", "regex"];
2288
2288
  var PATTERN_ELEMENT_OPERATORS = ["matchesAny", "containsAll"];
2289
2289
 
2290
+ // src/core/hash.ts
2291
+ var FNV_OFFSET_BASIS = 2166136261;
2292
+ var FNV_PRIME = 16777619;
2293
+ var encoder = new TextEncoder();
2294
+ function fnv1aBytes(bytes) {
2295
+ return fnv1aContinue(FNV_OFFSET_BASIS, bytes, bytes.length) >>> 0;
2296
+ }
2297
+ function fnv1aContinue(start, bytes, length) {
2298
+ let hash = start;
2299
+ for (let i = 0; i < length; i++) {
2300
+ hash = Math.imul(hash ^ bytes[i], FNV_PRIME);
2301
+ }
2302
+ return hash;
2303
+ }
2304
+ var scratchNumber = new Float64Array(1);
2305
+ var scratchBytes = new Uint8Array(scratchNumber.buffer);
2306
+ var HOST_IS_LITTLE_ENDIAN = new Uint8Array(new Uint16Array([1]).buffer)[0] === 1;
2307
+ function fnv1aContinueTaggedNumbers(start, tag, numbers) {
2308
+ let hash = start;
2309
+ const b = scratchBytes;
2310
+ for (let i = 0; i < numbers.length; i++) {
2311
+ scratchNumber[0] = Math.fround(numbers[i]);
2312
+ hash = Math.imul(hash ^ tag, FNV_PRIME);
2313
+ if (HOST_IS_LITTLE_ENDIAN) {
2314
+ hash = Math.imul(hash ^ b[7], FNV_PRIME);
2315
+ hash = Math.imul(hash ^ b[6], FNV_PRIME);
2316
+ hash = Math.imul(hash ^ b[5], FNV_PRIME);
2317
+ hash = Math.imul(hash ^ b[4], FNV_PRIME);
2318
+ hash = Math.imul(hash ^ b[3], FNV_PRIME);
2319
+ hash = Math.imul(hash ^ b[2], FNV_PRIME);
2320
+ hash = Math.imul(hash ^ b[1], FNV_PRIME);
2321
+ hash = Math.imul(hash ^ b[0], FNV_PRIME);
2322
+ } else {
2323
+ for (let at = 0; at < 8; at++) hash = Math.imul(hash ^ b[at], FNV_PRIME);
2324
+ }
2325
+ }
2326
+ return hash;
2327
+ }
2328
+ function fnv1a(input) {
2329
+ return fnv1aBytes(encoder.encode(input));
2330
+ }
2331
+
2332
+ // src/partitioning/router.ts
2333
+ function createPartitionRouter() {
2334
+ function route(docId, partitionCount) {
2335
+ if (partitionCount <= 0) {
2336
+ throw new NarsilError(
2337
+ ErrorCodes.INDEX_NOT_FOUND,
2338
+ `Partition count must be greater than 0, received ${partitionCount}`,
2339
+ { partitionCount }
2340
+ );
2341
+ }
2342
+ return fnv1a(docId) % partitionCount;
2343
+ }
2344
+ function routeBatch(docIds, partitionCount) {
2345
+ const groups = /* @__PURE__ */ new Map();
2346
+ for (let i = 0; i < docIds.length; i++) {
2347
+ const pid = route(docIds[i], partitionCount);
2348
+ let group = groups.get(pid);
2349
+ if (!group) {
2350
+ group = [];
2351
+ groups.set(pid, group);
2352
+ }
2353
+ group.push(docIds[i]);
2354
+ }
2355
+ return groups;
2356
+ }
2357
+ return { route, routeBatch };
2358
+ }
2359
+
2360
+ // src/plugins/registry.ts
2361
+ var WRITE_REJECTING_HOOKS = /* @__PURE__ */ new Set(["beforeInsert", "beforeUpdate", "beforeRemove"]);
2362
+ function isThenable(value) {
2363
+ return value !== null && typeof value === "object" && typeof value.then === "function";
2364
+ }
2365
+ function thrownText(err, plugin, hookName) {
2366
+ if (err instanceof Error) return err.message;
2367
+ try {
2368
+ return String(err);
2369
+ } catch {
2370
+ return `The ${hookName} hook of plugin "${plugin}" rejects the write`;
2371
+ }
2372
+ }
2373
+ function asWriteRejection(err, plugin, hookName) {
2374
+ if (err instanceof NarsilError) return err;
2375
+ return new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, thrownText(err, plugin, hookName), {
2376
+ plugin,
2377
+ hook: hookName
2378
+ });
2379
+ }
2380
+ function invoke(hook, hookName, context) {
2381
+ const rejectsWrites = WRITE_REJECTING_HOOKS.has(hookName);
2382
+ let result;
2383
+ try {
2384
+ result = hook.run(context);
2385
+ } catch (err) {
2386
+ throw rejectsWrites ? asWriteRejection(err, hook.plugin, hookName) : err;
2387
+ }
2388
+ if (!isThenable(result)) return null;
2389
+ if (!rejectsWrites) return result;
2390
+ return Promise.resolve(result).catch((err) => {
2391
+ throw asWriteRejection(err, hook.plugin, hookName);
2392
+ });
2393
+ }
2394
+ async function continueAsync(pending, hooks, startIndex, hookName, context) {
2395
+ await pending;
2396
+ for (let i = startIndex; i < hooks.length; i++) {
2397
+ await invoke(hooks[i], hookName, context);
2398
+ }
2399
+ }
2400
+ function createPluginRegistry() {
2401
+ const plugins = [];
2402
+ return {
2403
+ register(plugin) {
2404
+ plugins.push(plugin);
2405
+ },
2406
+ hasHooks(hookName) {
2407
+ return plugins.some((p) => typeof p[hookName] === "function");
2408
+ },
2409
+ runHook(hookName, context) {
2410
+ const hooks = [];
2411
+ for (const plugin of plugins) {
2412
+ const hook = plugin[hookName];
2413
+ if (typeof hook === "function") {
2414
+ hooks.push({ plugin: plugin.name, run: hook.bind(plugin) });
2415
+ }
2416
+ }
2417
+ if (hooks.length === 0) return;
2418
+ for (let i = 0; i < hooks.length; i++) {
2419
+ const pending = invoke(hooks[i], hookName, context);
2420
+ if (pending !== null) {
2421
+ return continueAsync(pending, hooks, i + 1, hookName, context);
2422
+ }
2423
+ }
2424
+ }
2425
+ };
2426
+ }
2427
+
2428
+ // src/rerank/registry.ts
2429
+ function validateRerankAdapter(name, adapter) {
2430
+ if (typeof name !== "string" || name.trim().length === 0) {
2431
+ throw new NarsilError(ErrorCodes.RERANK_CONFIG_INVALID, "A rerank adapter needs a non-empty string name");
2432
+ }
2433
+ const rerank = typeof adapter === "object" && adapter !== null ? adapter.rerank : void 0;
2434
+ if (typeof rerank !== "function") {
2435
+ throw new NarsilError(ErrorCodes.RERANK_CONFIG_INVALID, `Rerank adapter "${name}" must provide a rerank function`, {
2436
+ adapter: name
2437
+ });
2438
+ }
2439
+ }
2440
+ function registerEngineRerankAdapter(core, name, adapter) {
2441
+ core.guardShutdown();
2442
+ validateRerankAdapter(name, adapter);
2443
+ core.rerankAdapters.set(name, adapter);
2444
+ }
2445
+
2290
2446
  // src/geo/constants.ts
2291
2447
  var DEG_TO_RAD = Math.PI / 180;
2292
2448
  var MAX_LATITUDE_DEGREES = 90;
@@ -2776,6 +2932,10 @@ function validateVectorStorage(config, filesystemDurability) {
2776
2932
  });
2777
2933
  }
2778
2934
  }
2935
+ function validateTextStorage(textStorage, filesystemDurability) {
2936
+ if (textStorage === void 0 || textStorage === "memory" || textStorage === "disk" && filesystemDurability) return;
2937
+ fail("textStorage must be 'memory', or 'disk' with filesystem durability", { textStorage });
2938
+ }
2779
2939
  function raiseEarlierReleaseGraphSetting(config) {
2780
2940
  if (typeof config !== "object" || config === null) return config;
2781
2941
  const hnsw = config.hnswConfig;
@@ -2812,91 +2972,213 @@ function validateVectorPromotion(config) {
2812
2972
  }
2813
2973
  }
2814
2974
 
2815
- // src/core/partition/utils.ts
2816
- function getOrCreateFieldNameIndex(table, fieldName) {
2817
- const existing = table.indexMap.get(fieldName);
2818
- if (existing !== void 0) return existing;
2819
- const idx = table.names.length;
2820
- table.names.push(fieldName);
2821
- table.indexMap.set(fieldName, idx);
2822
- return idx;
2823
- }
2824
- var VECTOR_PATTERN2 = /^vector\[(\d+)]$/;
2825
- function getNestedValue(obj, path) {
2826
- const parts = path.split(".");
2827
- let current = obj;
2828
- for (let i = 0; i < parts.length; i++) {
2829
- if (current === null || current === void 0 || typeof current !== "object") return void 0;
2830
- if (!Object.hasOwn(current, parts[i])) return void 0;
2831
- current = current[parts[i]];
2832
- }
2833
- return current;
2834
- }
2835
- function stringArraysEqual(a, b) {
2836
- if (a === b) return true;
2837
- if (!a || !b) return false;
2838
- if (a.length !== b.length) return false;
2839
- for (let i = 0; i < a.length; i++) {
2840
- if (a[i] !== b[i]) return false;
2975
+ // src/schema/embedding-validator.ts
2976
+ var VECTOR_DIMENSION_PATTERN = /^vector\[(\d+)]$/;
2977
+ function validateEmbeddingConfig(embeddingConfig, schema, instanceAdapter) {
2978
+ const fieldEntries = Object.keys(embeddingConfig.fields);
2979
+ if (fieldEntries.length === 0) {
2980
+ throw new NarsilError(
2981
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
2982
+ "Embedding config must define at least one field mapping"
2983
+ );
2841
2984
  }
2842
- return true;
2843
- }
2844
- function textFieldsChanged(oldDoc, newDoc, fields) {
2845
- for (const { path, base } of fields) {
2846
- if (!isWordIndexedBase(base)) continue;
2847
- const oldVal = getNestedValue(oldDoc, path);
2848
- const newVal = getNestedValue(newDoc, path);
2849
- if (base === "string") {
2850
- if (oldVal !== newVal) return true;
2851
- } else {
2852
- if (!stringArraysEqual(oldVal, newVal)) return true;
2853
- }
2985
+ if (typeof embeddingConfig.adapter === "string") {
2986
+ throw new NarsilError(
2987
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
2988
+ `Embedding adapter "${embeddingConfig.adapter}" must be resolved against the engine registry before validation`,
2989
+ { adapter: embeddingConfig.adapter }
2990
+ );
2854
2991
  }
2855
- return false;
2856
- }
2857
- function parseVectorDimension(fieldType) {
2858
- const match = VECTOR_PATTERN2.exec(fieldType);
2859
- return match ? Number.parseInt(match[1], 10) : null;
2860
- }
2861
- function tokenizeOptions(options) {
2862
- return {
2863
- stem: true,
2864
- removeStopWords: true,
2865
- collectSurfaces: options?.collectSurfaces === true,
2866
- stopWordOverride: options?.stopWordOverride,
2867
- customTokenizer: options?.customTokenizer
2868
- };
2869
- }
2870
- function getFlatSchema(state, schema) {
2871
- if (state.lastSchemaRef === schema && state.flatSchemaCache) return state.flatSchemaCache;
2872
- state.flatSchemaCache = flattenSchema(schema);
2873
- state.lastSchemaRef = schema;
2874
- return state.flatSchemaCache;
2875
- }
2876
-
2877
- // src/core/projection.ts
2878
- function requireFieldPathList(paths, option) {
2879
- if (paths === void 0) return;
2880
- if (!Array.isArray(paths)) {
2992
+ const resolvedAdapter = embeddingConfig.adapter ?? instanceAdapter;
2993
+ if (!resolvedAdapter) {
2881
2994
  throw new NarsilError(
2882
- ErrorCodes.SEARCH_INVALID_FIELD,
2883
- `The "document.${option}" option takes an array of field paths`,
2884
- { option: `document.${option}`, received: paths === null ? "null" : typeof paths }
2995
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
2996
+ "Embedding config requires an adapter (set on the index or the instance)"
2885
2997
  );
2886
2998
  }
2887
- for (let position = 0; position < paths.length; position++) {
2888
- const entry = paths[position];
2889
- if (typeof entry === "string") continue;
2999
+ if (typeof resolvedAdapter.dimensions !== "number" || !Number.isInteger(resolvedAdapter.dimensions) || resolvedAdapter.dimensions <= 0) {
2890
3000
  throw new NarsilError(
2891
- ErrorCodes.SEARCH_INVALID_FIELD,
2892
- `The "document.${option}" option takes field paths as strings, and entry ${position} is not a string`,
2893
- { option: `document.${option}`, position, received: entry === null ? "null" : typeof entry }
3001
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
3002
+ `Adapter dimensions must be a positive integer, got ${resolvedAdapter.dimensions}`
2894
3003
  );
2895
3004
  }
2896
- }
2897
- function splitPaths(paths) {
2898
- if (!paths) return [];
2899
- const split = [];
3005
+ const flatFields = flattenSchema(schema);
3006
+ for (const [targetField, sourceFields] of Object.entries(embeddingConfig.fields)) {
3007
+ const targetType = flatFields[targetField];
3008
+ if (targetType === void 0) {
3009
+ throw new NarsilError(
3010
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
3011
+ `Embedding target field "${targetField}" does not exist in the schema`,
3012
+ { field: targetField }
3013
+ );
3014
+ }
3015
+ const vectorMatch = VECTOR_DIMENSION_PATTERN.exec(targetType);
3016
+ if (!vectorMatch) {
3017
+ throw new NarsilError(
3018
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
3019
+ `Embedding target field "${targetField}" must be a vector type, got "${targetType}"`,
3020
+ { field: targetField, type: targetType }
3021
+ );
3022
+ }
3023
+ const schemaDimension = Number.parseInt(vectorMatch[1], 10);
3024
+ if (schemaDimension !== resolvedAdapter.dimensions) {
3025
+ throw new NarsilError(
3026
+ ErrorCodes.EMBEDDING_DIMENSION_MISMATCH,
3027
+ `Embedding target field "${targetField}" has dimension ${schemaDimension} but the adapter produces ${resolvedAdapter.dimensions}-dimensional vectors`,
3028
+ { field: targetField, expected: schemaDimension, actual: resolvedAdapter.dimensions }
3029
+ );
3030
+ }
3031
+ const sources = Array.isArray(sourceFields) ? sourceFields : [sourceFields];
3032
+ if (sources.length === 0) {
3033
+ throw new NarsilError(
3034
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
3035
+ `Embedding target field "${targetField}" must have at least one source field`,
3036
+ { field: targetField }
3037
+ );
3038
+ }
3039
+ for (const source of sources) {
3040
+ const sourceType = flatFields[source];
3041
+ if (sourceType === void 0) {
3042
+ throw new NarsilError(
3043
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
3044
+ `Embedding source field "${source}" does not exist in the schema`,
3045
+ { field: source, targetField }
3046
+ );
3047
+ }
3048
+ if (!isSingleStringFieldType(sourceType)) {
3049
+ throw new NarsilError(
3050
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
3051
+ `Embedding source field "${source}" must be a string type, got "${sourceType}"`,
3052
+ { field: source, type: sourceType, targetField }
3053
+ );
3054
+ }
3055
+ }
3056
+ }
3057
+ return resolvedAdapter;
3058
+ }
3059
+ function validateRegisteredAdapter(name, adapter) {
3060
+ if (typeof name !== "string" || name.trim().length === 0) {
3061
+ throw new NarsilError(ErrorCodes.EMBEDDING_CONFIG_INVALID, "Embedding adapter names must be non-empty strings");
3062
+ }
3063
+ if (typeof adapter?.embed !== "function") {
3064
+ throw new NarsilError(
3065
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
3066
+ `Embedding adapter "${name}" must provide an embed function`,
3067
+ { adapter: name }
3068
+ );
3069
+ }
3070
+ if (typeof adapter.dimensions !== "number" || !Number.isInteger(adapter.dimensions) || adapter.dimensions <= 0) {
3071
+ throw new NarsilError(
3072
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
3073
+ `Embedding adapter "${name}" dimensions must be a positive integer, got ${adapter.dimensions}`,
3074
+ { adapter: name, dimensions: adapter.dimensions }
3075
+ );
3076
+ }
3077
+ }
3078
+ function validateRequiredFieldsInSchema(required, schema) {
3079
+ if (required.length === 0) return;
3080
+ const flatFields = flattenSchema(schema);
3081
+ for (const field of required) {
3082
+ if (flatFields[field] === void 0) {
3083
+ throw new NarsilError(
3084
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
3085
+ `Required field "${field}" does not exist in the schema`,
3086
+ { field }
3087
+ );
3088
+ }
3089
+ }
3090
+ }
3091
+
3092
+ // src/runtime/heap-statistics.browser.ts
3093
+ function readHeapStatistics() {
3094
+ return null;
3095
+ }
3096
+
3097
+ // src/core/partition/utils.ts
3098
+ function getOrCreateFieldNameIndex(table, fieldName) {
3099
+ const existing = table.indexMap.get(fieldName);
3100
+ if (existing !== void 0) return existing;
3101
+ const idx = table.names.length;
3102
+ table.names.push(fieldName);
3103
+ table.indexMap.set(fieldName, idx);
3104
+ return idx;
3105
+ }
3106
+ var VECTOR_PATTERN2 = /^vector\[(\d+)]$/;
3107
+ function getNestedValue(obj, path) {
3108
+ const parts = path.split(".");
3109
+ let current = obj;
3110
+ for (let i = 0; i < parts.length; i++) {
3111
+ if (current === null || current === void 0 || typeof current !== "object") return void 0;
3112
+ if (!Object.hasOwn(current, parts[i])) return void 0;
3113
+ current = current[parts[i]];
3114
+ }
3115
+ return current;
3116
+ }
3117
+ function stringArraysEqual(a, b) {
3118
+ if (a === b) return true;
3119
+ if (!a || !b) return false;
3120
+ if (a.length !== b.length) return false;
3121
+ for (let i = 0; i < a.length; i++) {
3122
+ if (a[i] !== b[i]) return false;
3123
+ }
3124
+ return true;
3125
+ }
3126
+ function textFieldsChanged(oldDoc, newDoc, fields) {
3127
+ for (const { path, base } of fields) {
3128
+ if (!isWordIndexedBase(base)) continue;
3129
+ const oldVal = getNestedValue(oldDoc, path);
3130
+ const newVal = getNestedValue(newDoc, path);
3131
+ if (base === "string") {
3132
+ if (oldVal !== newVal) return true;
3133
+ } else {
3134
+ if (!stringArraysEqual(oldVal, newVal)) return true;
3135
+ }
3136
+ }
3137
+ return false;
3138
+ }
3139
+ function parseVectorDimension(fieldType) {
3140
+ const match = VECTOR_PATTERN2.exec(fieldType);
3141
+ return match ? Number.parseInt(match[1], 10) : null;
3142
+ }
3143
+ function tokenizeOptions(options) {
3144
+ return {
3145
+ stem: true,
3146
+ removeStopWords: true,
3147
+ collectSurfaces: options?.collectSurfaces === true,
3148
+ stopWordOverride: options?.stopWordOverride,
3149
+ customTokenizer: options?.customTokenizer
3150
+ };
3151
+ }
3152
+ function getFlatSchema(state, schema) {
3153
+ if (state.lastSchemaRef === schema && state.flatSchemaCache) return state.flatSchemaCache;
3154
+ state.flatSchemaCache = flattenSchema(schema);
3155
+ state.lastSchemaRef = schema;
3156
+ return state.flatSchemaCache;
3157
+ }
3158
+
3159
+ // src/core/projection.ts
3160
+ function requireFieldPathList(paths, option) {
3161
+ if (paths === void 0) return;
3162
+ if (!Array.isArray(paths)) {
3163
+ throw new NarsilError(
3164
+ ErrorCodes.SEARCH_INVALID_FIELD,
3165
+ `The "document.${option}" option takes an array of field paths`,
3166
+ { option: `document.${option}`, received: paths === null ? "null" : typeof paths }
3167
+ );
3168
+ }
3169
+ for (let position = 0; position < paths.length; position++) {
3170
+ const entry = paths[position];
3171
+ if (typeof entry === "string") continue;
3172
+ throw new NarsilError(
3173
+ ErrorCodes.SEARCH_INVALID_FIELD,
3174
+ `The "document.${option}" option takes field paths as strings, and entry ${position} is not a string`,
3175
+ { option: `document.${option}`, position, received: entry === null ? "null" : typeof entry }
3176
+ );
3177
+ }
3178
+ }
3179
+ function splitPaths(paths) {
3180
+ if (!paths) return [];
3181
+ const split = [];
2900
3182
  for (const path of paths) {
2901
3183
  const segments = path.split(".").filter((segment) => segment.length > 0);
2902
3184
  if (segments.length === 0) continue;
@@ -8128,7 +8410,7 @@ function fieldLengthBound(columns, fieldIndex, internalId) {
8128
8410
  function entriesCovered(bounds) {
8129
8411
  return bounds.blockCount === 0 ? 0 : bounds.entryEnd[bounds.blockCount - 1];
8130
8412
  }
8131
- function scan(list, columns, previous, keepBlocks) {
8413
+ function scanPostingBlocks(list, columns, previous, keepBlocks) {
8132
8414
  const { docIds, termFrequencies, fieldNameIndices, length } = list;
8133
8415
  const startEntry = previous !== null && keepBlocks > 0 ? previous.entryEnd[keepBlocks - 1] : 0;
8134
8416
  const capacity = keepBlocks + Math.ceil((length - startEntry) / TARGET_BLOCK_ENTRIES) + 1;
@@ -8202,12 +8484,13 @@ function blockBoundsFor(list, columns) {
8202
8484
  const covered = entriesCovered(existing);
8203
8485
  if (covered === list.length) return existing;
8204
8486
  if (list.ordered && covered < list.length) {
8205
- const extended = scan(list, columns, existing, Math.max(0, existing.blockCount - 1));
8487
+ const extended = scanPostingBlocks(list, columns, existing, Math.max(0, existing.blockCount - 1));
8206
8488
  boundsByList.set(list, extended);
8207
8489
  return extended;
8208
8490
  }
8209
8491
  }
8210
- const built = scan(list, columns, null, 0);
8492
+ const stored = list.storedBlockBounds?.() ?? null;
8493
+ const built = stored === null ? scanPostingBlocks(list, columns, null, 0) : { ...stored, structureRevision: list.structureRevision };
8211
8494
  boundsByList.set(list, built);
8212
8495
  return built;
8213
8496
  }
@@ -8400,33 +8683,33 @@ function boostsKeepScoresNonNegative(fields) {
8400
8683
  }
8401
8684
  return true;
8402
8685
  }
8403
- function samePairUpperBound(scan2, list, maxBoost, maxAverageLength) {
8404
- if (scan2.scoresAreZero) return 0;
8405
- const bounds = blockBoundsFor(list, scan2.fields.lengthColumns);
8686
+ function samePairUpperBound(scan, list, maxBoost, maxAverageLength) {
8687
+ if (scan.scoresAreZero) return 0;
8688
+ const bounds = blockBoundsFor(list, scan.fields.lengthColumns);
8406
8689
  let best = 0;
8407
8690
  for (let block = 0; block < bounds.blockCount; block++) {
8408
8691
  const bestEntry = computeBM25WithIDF(
8409
8692
  bounds.maxTermFrequency[block],
8410
- scan2.idf,
8693
+ scan.idf,
8411
8694
  bounds.minFieldLength[block],
8412
8695
  maxAverageLength,
8413
- scan2.k1,
8414
- scan2.b
8696
+ scan.k1,
8697
+ scan.b
8415
8698
  );
8416
8699
  const blockBound = bestEntry * maxBoost * bounds.maxEntriesPerDocument[block];
8417
8700
  if (blockBound > best) best = blockBound;
8418
8701
  }
8419
8702
  return best;
8420
8703
  }
8421
- function fieldPairUpperBound(scan2, list, maxBoost, maxAverageLength) {
8422
- if (scan2.scoresAreZero) return 0;
8423
- const bounds = blockBoundsFor(list, scan2.fields.lengthColumns);
8704
+ function fieldPairUpperBound(scan, list, maxBoost, maxAverageLength) {
8705
+ if (scan.scoresAreZero) return 0;
8706
+ const bounds = blockBoundsFor(list, scan.fields.lengthColumns);
8424
8707
  let best = 0;
8425
8708
  for (let block = 0; block < bounds.blockCount; block++) {
8426
8709
  const bestEntry = blockEntryBound(
8427
- scan2.fields,
8710
+ scan.fields,
8428
8711
  bounds.maxTermFrequency[block],
8429
- scan2.idf,
8712
+ scan.idf,
8430
8713
  bounds.minFieldLength[block],
8431
8714
  maxAverageLength
8432
8715
  );
@@ -8435,22 +8718,22 @@ function fieldPairUpperBound(scan2, list, maxBoost, maxAverageLength) {
8435
8718
  }
8436
8719
  return best;
8437
8720
  }
8438
- function samePairEntryScore(scan2, entry, internalId) {
8439
- const { fields, columns } = scan2;
8721
+ function samePairEntryScore(scan, entry, internalId) {
8722
+ const { fields, columns } = scan;
8440
8723
  const fieldIndex = columns.fieldNameIndices[entry];
8441
8724
  const averageLength = fields.averageLengths[fieldIndex];
8442
8725
  const fieldLength = fieldLengthOf(fields.lengthColumns, fieldIndex, internalId, averageLength);
8443
- const bm25 = scan2.scoresAreZero ? 0 : computeBM25WithIDF(columns.termFrequencies[entry], scan2.idf, fieldLength, averageLength, scan2.k1, scan2.b);
8726
+ const bm25 = scan.scoresAreZero ? 0 : computeBM25WithIDF(columns.termFrequencies[entry], scan.idf, fieldLength, averageLength, scan.k1, scan.b);
8444
8727
  return bm25 * fields.boosts[fieldIndex];
8445
8728
  }
8446
- function fieldPairEntryScore(scan2, entry, internalId) {
8447
- const { fields, columns } = scan2;
8729
+ function fieldPairEntryScore(scan, entry, internalId) {
8730
+ const { fields, columns } = scan;
8448
8731
  const fieldIndex = columns.fieldNameIndices[entry];
8449
8732
  const averageLength = fields.averageLengths[fieldIndex];
8450
8733
  const fieldLength = fieldLengthOf(fields.lengthColumns, fieldIndex, internalId, averageLength);
8451
- const bm25 = scan2.scoresAreZero ? 0 : computeBM25WithIDF(
8734
+ const bm25 = scan.scoresAreZero ? 0 : computeBM25WithIDF(
8452
8735
  columns.termFrequencies[entry],
8453
- scan2.idf,
8736
+ scan.idf,
8454
8737
  fieldLength,
8455
8738
  averageLength,
8456
8739
  fields.k1s[fieldIndex],
@@ -8458,48 +8741,48 @@ function fieldPairEntryScore(scan2, entry, internalId) {
8458
8741
  );
8459
8742
  return bm25 * fields.boosts[fieldIndex];
8460
8743
  }
8461
- function samePairScoreEveryDocument(scan2, buffer) {
8462
- const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = scan2.columns;
8463
- const searchable = scan2.fields.searchable;
8744
+ function samePairScoreEveryDocument(scan, buffer) {
8745
+ const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = scan.columns;
8746
+ const searchable = scan.fields.searchable;
8464
8747
  for (let entry = 0; entry < count; entry++) {
8465
8748
  const internalId = docIds[entry];
8466
8749
  if (hasDeleted && deletedDocs.has(internalId)) continue;
8467
8750
  if (searchable[fieldNameIndices[entry]] !== 1) continue;
8468
- addScore(buffer, internalId, samePairEntryScore(scan2, entry, internalId));
8751
+ addScore(buffer, internalId, samePairEntryScore(scan, entry, internalId));
8469
8752
  }
8470
8753
  }
8471
- function fieldPairScoreEveryDocument(scan2, buffer) {
8472
- const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = scan2.columns;
8473
- const searchable = scan2.fields.searchable;
8754
+ function fieldPairScoreEveryDocument(scan, buffer) {
8755
+ const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = scan.columns;
8756
+ const searchable = scan.fields.searchable;
8474
8757
  for (let entry = 0; entry < count; entry++) {
8475
8758
  const internalId = docIds[entry];
8476
8759
  if (hasDeleted && deletedDocs.has(internalId)) continue;
8477
8760
  if (searchable[fieldNameIndices[entry]] !== 1) continue;
8478
- addScore(buffer, internalId, fieldPairEntryScore(scan2, entry, internalId));
8761
+ addScore(buffer, internalId, fieldPairEntryScore(scan, entry, internalId));
8479
8762
  }
8480
8763
  }
8481
- function samePairScoreKnownDocuments(scan2, buffer) {
8482
- const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = scan2.columns;
8483
- const searchable = scan2.fields.searchable;
8764
+ function samePairScoreKnownDocuments(scan, buffer) {
8765
+ const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = scan.columns;
8766
+ const searchable = scan.fields.searchable;
8484
8767
  let counted = 0;
8485
8768
  for (let entry = 0; entry < count; entry++) {
8486
8769
  const internalId = docIds[entry];
8487
8770
  if (hasDeleted && deletedDocs.has(internalId)) continue;
8488
8771
  if (searchable[fieldNameIndices[entry]] !== 1) continue;
8489
- if (hasScore(buffer, internalId)) addScore(buffer, internalId, samePairEntryScore(scan2, entry, internalId));
8772
+ if (hasScore(buffer, internalId)) addScore(buffer, internalId, samePairEntryScore(scan, entry, internalId));
8490
8773
  else if (markCounted(buffer, internalId)) counted++;
8491
8774
  }
8492
8775
  return counted;
8493
8776
  }
8494
- function fieldPairScoreKnownDocuments(scan2, buffer) {
8495
- const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = scan2.columns;
8496
- const searchable = scan2.fields.searchable;
8777
+ function fieldPairScoreKnownDocuments(scan, buffer) {
8778
+ const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = scan.columns;
8779
+ const searchable = scan.fields.searchable;
8497
8780
  let counted = 0;
8498
8781
  for (let entry = 0; entry < count; entry++) {
8499
8782
  const internalId = docIds[entry];
8500
8783
  if (hasDeleted && deletedDocs.has(internalId)) continue;
8501
8784
  if (searchable[fieldNameIndices[entry]] !== 1) continue;
8502
- if (hasScore(buffer, internalId)) addScore(buffer, internalId, fieldPairEntryScore(scan2, entry, internalId));
8785
+ if (hasScore(buffer, internalId)) addScore(buffer, internalId, fieldPairEntryScore(scan, entry, internalId));
8503
8786
  else if (markCounted(buffer, internalId)) counted++;
8504
8787
  }
8505
8788
  return counted;
@@ -8516,24 +8799,24 @@ function firstEntryOf(docIds, count, internalId) {
8516
8799
  }
8517
8800
  function samePairQueryOrderScore(scans, internalId) {
8518
8801
  let score = 0;
8519
- for (const scan2 of scans) {
8520
- const { docIds, deletedDocs, hasDeleted, count } = scan2.columns;
8802
+ for (const scan of scans) {
8803
+ const { docIds, deletedDocs, hasDeleted, count } = scan.columns;
8521
8804
  if (hasDeleted && deletedDocs.has(internalId)) continue;
8522
8805
  for (let entry = firstEntryOf(docIds, count, internalId); entry < count && docIds[entry] === internalId; entry++) {
8523
- if (scan2.fields.searchable[scan2.columns.fieldNameIndices[entry]] !== 1) continue;
8524
- score += samePairEntryScore(scan2, entry, internalId);
8806
+ if (scan.fields.searchable[scan.columns.fieldNameIndices[entry]] !== 1) continue;
8807
+ score += samePairEntryScore(scan, entry, internalId);
8525
8808
  }
8526
8809
  }
8527
8810
  return score;
8528
8811
  }
8529
8812
  function fieldPairQueryOrderScore(scans, internalId) {
8530
8813
  let score = 0;
8531
- for (const scan2 of scans) {
8532
- const { docIds, deletedDocs, hasDeleted, count } = scan2.columns;
8814
+ for (const scan of scans) {
8815
+ const { docIds, deletedDocs, hasDeleted, count } = scan.columns;
8533
8816
  if (hasDeleted && deletedDocs.has(internalId)) continue;
8534
8817
  for (let entry = firstEntryOf(docIds, count, internalId); entry < count && docIds[entry] === internalId; entry++) {
8535
- if (scan2.fields.searchable[scan2.columns.fieldNameIndices[entry]] !== 1) continue;
8536
- score += fieldPairEntryScore(scan2, entry, internalId);
8818
+ if (scan.fields.searchable[scan.columns.fieldNameIndices[entry]] !== 1) continue;
8819
+ score += fieldPairEntryScore(scan, entry, internalId);
8537
8820
  }
8538
8821
  }
8539
8822
  return score;
@@ -8613,7 +8896,7 @@ function multiTermTopK(request) {
8613
8896
  }
8614
8897
  const maxBoost = bestSearchable(fields.searchable, fields.boosts);
8615
8898
  const maxAverageLength = bestSearchable(fields.searchable, fields.averageLengths);
8616
- const upperBounds = scans.map((scan2, term) => scorer.upperBound(scan2, terms[term].list, maxBoost, maxAverageLength));
8899
+ const upperBounds = scans.map((scan, term) => scorer.upperBound(scan, terms[term].list, maxBoost, maxAverageLength));
8617
8900
  const order = scans.map((_, term) => term).sort((left, right) => upperBounds[right] - upperBounds[left]);
8618
8901
  const remainingBound = new Float64Array(order.length + 1);
8619
8902
  for (let position2 = order.length - 1; position2 >= 0; position2--) {
@@ -9764,570 +10047,104 @@ function createPartitionIndex(partitionId, trackPositions = true) {
9764
10047
  const next = indexDocument(state, docId, document, fields, language, options);
9765
10048
  state.docStore.store(docId, document, next.fieldLengths);
9766
10049
  state.stats.addDocument(next.fieldLengths, next.tokensByField);
9767
- state.partialWords.afterDocument(partialFieldsOf(schema), internalId, next.partialSpellings, 1);
9768
- recordSortValues(state, internalId, document);
9769
- } else {
9770
- updateFieldIndexOnly(state, docId, stored.fields, document, fields);
9771
- state.docStore.store(docId, document, stored.fieldLengths);
9772
- const internalId = state.docStore.getInternalId(docId);
9773
- if (internalId !== void 0) recordSortValues(state, internalId, document);
9774
- }
9775
- },
9776
- get(docId, projection) {
9777
- const stored = state.docStore.get(docId);
9778
- if (!stored) return void 0;
9779
- return cloneProjected(stored.fields, projection);
9780
- },
9781
- getRef(docId) {
9782
- const stored = state.docStore.get(docId);
9783
- if (!stored) return void 0;
9784
- return stored.fields;
9785
- },
9786
- fieldValues(docId, fieldPaths) {
9787
- const stored = state.docStore.get(docId);
9788
- if (!stored) return void 0;
9789
- return fieldPaths.map((fieldPath) => getNestedValue(stored.fields, fieldPath));
9790
- },
9791
- has(docId) {
9792
- return state.docStore.has(docId);
9793
- },
9794
- count() {
9795
- return state.docStore.count();
9796
- },
9797
- *docIds() {
9798
- for (const [id] of state.docStore.all()) {
9799
- yield id;
9800
- }
9801
- },
9802
- sortedDocIds() {
9803
- return state.docStore.sortedDocIds();
9804
- },
9805
- releaseSortedDocIds() {
9806
- state.docStore.releaseSortedDocIds();
9807
- },
9808
- clear: clearAll,
9809
- estimateMemoryBytes() {
9810
- return estimatePartitionBytes(state);
9811
- },
9812
- readStates() {
9813
- return [state];
9814
- },
9815
- searchFulltext(params) {
9816
- return searchFulltext(state, params);
9817
- },
9818
- searchFulltextMatches(params) {
9819
- return searchFulltextMatches(state, params);
9820
- },
9821
- sortedPage(request) {
9822
- return sortedPageOf(state, request);
9823
- },
9824
- sortValues(docId, fields, fieldTypes, modes) {
9825
- return sortValuesOf(state, docId, fields, fieldTypes, modes);
9826
- },
9827
- applyFilters(filters, schema, meter) {
9828
- return applyPartitionFilters(state, filters, schema, meter);
9829
- },
9830
- applyFiltersBitset(filters, schema, meter) {
9831
- return applyPartitionFiltersBitset(state, filters, schema, meter);
9832
- },
9833
- filterMatches(filters, schema, meter) {
9834
- return partitionFilterMatches(state, filters, schema, meter);
9835
- },
9836
- computeFacets(matched, config, schema) {
9837
- return computeFacets(state, matched, config, schema);
9838
- },
9839
- suggestTerms(surfacePrefix, stemmedPrefix, limit) {
9840
- return suggestDisplayTerms(state, surfacePrefix, stemmedPrefix, limit);
9841
- },
9842
- expandTermPrefix(surfacePrefix, stemmedToken, maxExpansions) {
9843
- return expandTermPrefix([state], surfacePrefix, stemmedToken, maxExpansions);
9844
- },
9845
- expandPartialWords(analysis, requests) {
9846
- return expandPartialWords([state], analysis, requests);
9847
- },
9848
- preparePartialWords(analysis) {
9849
- expandPartialWords([state], analysis, []);
9850
- },
9851
- serialize(indexName, totalPartitions, language, schema) {
9852
- return serializePartition(state, partitionId, indexName, totalPartitions, language, schema);
9853
- },
9854
- deserialize(data, schema) {
9855
- deserializePartition(state, data, clearAll, schema);
9856
- }
9857
- };
9858
- statesByPartition.set(partition, state);
9859
- return partition;
9860
- }
9861
-
9862
- // src/engine/orchestration/turn.ts
9863
- function afterCurrentTurn(callback) {
9864
- if (typeof setImmediate === "function") {
9865
- setImmediate(callback);
9866
- return;
9867
- }
9868
- setTimeout(callback, 0);
9869
- }
9870
- function yieldToEventLoop() {
9871
- return new Promise((resolve) => afterCurrentTurn(resolve));
9872
- }
9873
-
9874
- // src/partitioning/constants.ts
9875
- var REBALANCE_CHUNK_SIZE = 1e3;
9876
- var DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE = 1e4;
9877
- var TEXT_REBUILD_CHUNK_DOCUMENTS = 1e3;
9878
- var TEXT_REBUILD_SWITCH_BACKLOG_DOCUMENTS = 1e3;
9879
-
9880
- // src/partitioning/rebalancer.ts
9881
- function createRebalancer() {
9882
- const activeManagers = /* @__PURE__ */ new WeakSet();
9883
- let activeCount = 0;
9884
- async function rebalance(manager, newPartitionCount, router, onProgress) {
9885
- if (activeManagers.has(manager)) {
9886
- throw new NarsilError(
9887
- ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
9888
- "A rebalance operation is already in progress"
9889
- );
9890
- }
9891
- if (newPartitionCount <= 0) {
9892
- throw new NarsilError(
9893
- ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
9894
- `New partition count must be greater than 0, received ${newPartitionCount}`,
9895
- { newPartitionCount }
9896
- );
9897
- }
9898
- if (newPartitionCount === manager.partitionCount) {
9899
- throw new NarsilError(
9900
- ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
9901
- `New partition count (${newPartitionCount}) is the same as the current count`,
9902
- { newPartitionCount, currentPartitionCount: manager.partitionCount }
9903
- );
9904
- }
9905
- activeManagers.add(manager);
9906
- activeCount++;
9907
- try {
9908
- const currentPartitions = manager.getAllPartitions();
9909
- let documentsTotal = 0;
9910
- for (const partition of currentPartitions) {
9911
- documentsTotal += partition.count();
9912
- }
9913
- onProgress?.({
9914
- phase: "scanning",
9915
- documentsProcessed: 0,
9916
- documentsTotal
9917
- });
9918
- const newPartitions = [];
9919
- for (let i = 0; i < newPartitionCount; i++) {
9920
- newPartitions.push(createPartitionIndex(i, manager.config.trackPositions ?? true));
9921
- }
9922
- let documentsProcessed = 0;
9923
- let chunkFill = 0;
9924
- const insertOptions = {
9925
- validate: false,
9926
- skipClone: true,
9927
- stopWordOverride: manager.analysis.stopWords,
9928
- customTokenizer: manager.analysis.customTokenizer,
9929
- collectSurfaces: manager.config.surfaceForms !== false
9930
- };
9931
- for (const partition of currentPartitions) {
9932
- const docIds = [...partition.docIds()];
9933
- for (const docId of docIds) {
9934
- const document = partition.getRef(docId);
9935
- if (!document) continue;
9936
- const targetPartitionId = router.route(docId, newPartitionCount);
9937
- const targetPartition = newPartitions[targetPartitionId];
9938
- if (!targetPartition.has(docId)) {
9939
- targetPartition.insert(docId, document, manager.schema, manager.language, insertOptions);
9940
- documentsProcessed++;
9941
- }
9942
- chunkFill++;
9943
- if (chunkFill >= REBALANCE_CHUNK_SIZE) {
9944
- chunkFill = 0;
9945
- onProgress?.({
9946
- phase: "moving",
9947
- documentsProcessed,
9948
- documentsTotal
9949
- });
9950
- await yieldToEventLoop();
9951
- }
9952
- }
9953
- }
9954
- if (chunkFill > 0) {
9955
- onProgress?.({
9956
- phase: "moving",
9957
- documentsProcessed,
9958
- documentsTotal
9959
- });
9960
- await yieldToEventLoop();
9961
- }
9962
- onProgress?.({
9963
- phase: "swapping",
9964
- documentsProcessed,
9965
- documentsTotal
9966
- });
9967
- manager.setPartitions(newPartitions);
9968
- onProgress?.({
9969
- phase: "complete",
9970
- documentsProcessed,
9971
- documentsTotal
9972
- });
9973
- } finally {
9974
- activeManagers.delete(manager);
9975
- activeCount--;
9976
- }
9977
- }
9978
- return {
9979
- rebalance,
9980
- isRebalancing() {
9981
- return activeCount > 0;
9982
- }
9983
- };
9984
- }
9985
-
9986
- // src/core/hash.ts
9987
- var FNV_OFFSET_BASIS = 2166136261;
9988
- var FNV_PRIME = 16777619;
9989
- var encoder = new TextEncoder();
9990
- function fnv1aBytes(bytes) {
9991
- return fnv1aContinue(FNV_OFFSET_BASIS, bytes, bytes.length) >>> 0;
9992
- }
9993
- function fnv1aContinue(start, bytes, length) {
9994
- let hash = start;
9995
- for (let i = 0; i < length; i++) {
9996
- hash = Math.imul(hash ^ bytes[i], FNV_PRIME);
9997
- }
9998
- return hash;
9999
- }
10000
- var scratchNumber = new Float64Array(1);
10001
- var scratchBytes = new Uint8Array(scratchNumber.buffer);
10002
- var HOST_IS_LITTLE_ENDIAN = new Uint8Array(new Uint16Array([1]).buffer)[0] === 1;
10003
- function fnv1aContinueTaggedNumbers(start, tag, numbers) {
10004
- let hash = start;
10005
- const b = scratchBytes;
10006
- for (let i = 0; i < numbers.length; i++) {
10007
- scratchNumber[0] = Math.fround(numbers[i]);
10008
- hash = Math.imul(hash ^ tag, FNV_PRIME);
10009
- if (HOST_IS_LITTLE_ENDIAN) {
10010
- hash = Math.imul(hash ^ b[7], FNV_PRIME);
10011
- hash = Math.imul(hash ^ b[6], FNV_PRIME);
10012
- hash = Math.imul(hash ^ b[5], FNV_PRIME);
10013
- hash = Math.imul(hash ^ b[4], FNV_PRIME);
10014
- hash = Math.imul(hash ^ b[3], FNV_PRIME);
10015
- hash = Math.imul(hash ^ b[2], FNV_PRIME);
10016
- hash = Math.imul(hash ^ b[1], FNV_PRIME);
10017
- hash = Math.imul(hash ^ b[0], FNV_PRIME);
10018
- } else {
10019
- for (let at = 0; at < 8; at++) hash = Math.imul(hash ^ b[at], FNV_PRIME);
10020
- }
10021
- }
10022
- return hash;
10023
- }
10024
- function fnv1a(input) {
10025
- return fnv1aBytes(encoder.encode(input));
10026
- }
10027
-
10028
- // src/partitioning/router.ts
10029
- function createPartitionRouter() {
10030
- function route(docId, partitionCount) {
10031
- if (partitionCount <= 0) {
10032
- throw new NarsilError(
10033
- ErrorCodes.INDEX_NOT_FOUND,
10034
- `Partition count must be greater than 0, received ${partitionCount}`,
10035
- { partitionCount }
10036
- );
10037
- }
10038
- return fnv1a(docId) % partitionCount;
10039
- }
10040
- function routeBatch(docIds, partitionCount) {
10041
- const groups = /* @__PURE__ */ new Map();
10042
- for (let i = 0; i < docIds.length; i++) {
10043
- const pid = route(docIds[i], partitionCount);
10044
- let group = groups.get(pid);
10045
- if (!group) {
10046
- group = [];
10047
- groups.set(pid, group);
10048
- }
10049
- group.push(docIds[i]);
10050
- }
10051
- return groups;
10052
- }
10053
- return { route, routeBatch };
10054
- }
10055
-
10056
- // src/plugins/registry.ts
10057
- var WRITE_REJECTING_HOOKS = /* @__PURE__ */ new Set(["beforeInsert", "beforeUpdate", "beforeRemove"]);
10058
- function isThenable(value) {
10059
- return value !== null && typeof value === "object" && typeof value.then === "function";
10060
- }
10061
- function thrownText(err, plugin, hookName) {
10062
- if (err instanceof Error) return err.message;
10063
- try {
10064
- return String(err);
10065
- } catch {
10066
- return `The ${hookName} hook of plugin "${plugin}" rejects the write`;
10067
- }
10068
- }
10069
- function asWriteRejection(err, plugin, hookName) {
10070
- if (err instanceof NarsilError) return err;
10071
- return new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, thrownText(err, plugin, hookName), {
10072
- plugin,
10073
- hook: hookName
10074
- });
10075
- }
10076
- function invoke(hook, hookName, context) {
10077
- const rejectsWrites = WRITE_REJECTING_HOOKS.has(hookName);
10078
- let result;
10079
- try {
10080
- result = hook.run(context);
10081
- } catch (err) {
10082
- throw rejectsWrites ? asWriteRejection(err, hook.plugin, hookName) : err;
10083
- }
10084
- if (!isThenable(result)) return null;
10085
- if (!rejectsWrites) return result;
10086
- return Promise.resolve(result).catch((err) => {
10087
- throw asWriteRejection(err, hook.plugin, hookName);
10088
- });
10089
- }
10090
- async function continueAsync(pending, hooks, startIndex, hookName, context) {
10091
- await pending;
10092
- for (let i = startIndex; i < hooks.length; i++) {
10093
- await invoke(hooks[i], hookName, context);
10094
- }
10095
- }
10096
- function createPluginRegistry() {
10097
- const plugins = [];
10098
- return {
10099
- register(plugin) {
10100
- plugins.push(plugin);
10101
- },
10102
- hasHooks(hookName) {
10103
- return plugins.some((p) => typeof p[hookName] === "function");
10104
- },
10105
- runHook(hookName, context) {
10106
- const hooks = [];
10107
- for (const plugin of plugins) {
10108
- const hook = plugin[hookName];
10109
- if (typeof hook === "function") {
10110
- hooks.push({ plugin: plugin.name, run: hook.bind(plugin) });
10111
- }
10112
- }
10113
- if (hooks.length === 0) return;
10114
- for (let i = 0; i < hooks.length; i++) {
10115
- const pending = invoke(hooks[i], hookName, context);
10116
- if (pending !== null) {
10117
- return continueAsync(pending, hooks, i + 1, hookName, context);
10118
- }
10119
- }
10120
- }
10121
- };
10122
- }
10123
-
10124
- // src/rerank/registry.ts
10125
- function validateRerankAdapter(name, adapter) {
10126
- if (typeof name !== "string" || name.trim().length === 0) {
10127
- throw new NarsilError(ErrorCodes.RERANK_CONFIG_INVALID, "A rerank adapter needs a non-empty string name");
10128
- }
10129
- const rerank = typeof adapter === "object" && adapter !== null ? adapter.rerank : void 0;
10130
- if (typeof rerank !== "function") {
10131
- throw new NarsilError(ErrorCodes.RERANK_CONFIG_INVALID, `Rerank adapter "${name}" must provide a rerank function`, {
10132
- adapter: name
10133
- });
10134
- }
10135
- }
10136
- function registerEngineRerankAdapter(core, name, adapter) {
10137
- core.guardShutdown();
10138
- validateRerankAdapter(name, adapter);
10139
- core.rerankAdapters.set(name, adapter);
10140
- }
10141
-
10142
- // src/schema/embedding-validator.ts
10143
- var VECTOR_DIMENSION_PATTERN = /^vector\[(\d+)]$/;
10144
- function validateEmbeddingConfig(embeddingConfig, schema, instanceAdapter) {
10145
- const fieldEntries = Object.keys(embeddingConfig.fields);
10146
- if (fieldEntries.length === 0) {
10147
- throw new NarsilError(
10148
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
10149
- "Embedding config must define at least one field mapping"
10150
- );
10151
- }
10152
- if (typeof embeddingConfig.adapter === "string") {
10153
- throw new NarsilError(
10154
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
10155
- `Embedding adapter "${embeddingConfig.adapter}" must be resolved against the engine registry before validation`,
10156
- { adapter: embeddingConfig.adapter }
10157
- );
10158
- }
10159
- const resolvedAdapter = embeddingConfig.adapter ?? instanceAdapter;
10160
- if (!resolvedAdapter) {
10161
- throw new NarsilError(
10162
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
10163
- "Embedding config requires an adapter (set on the index or the instance)"
10164
- );
10165
- }
10166
- if (typeof resolvedAdapter.dimensions !== "number" || !Number.isInteger(resolvedAdapter.dimensions) || resolvedAdapter.dimensions <= 0) {
10167
- throw new NarsilError(
10168
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
10169
- `Adapter dimensions must be a positive integer, got ${resolvedAdapter.dimensions}`
10170
- );
10171
- }
10172
- const flatFields = flattenSchema(schema);
10173
- for (const [targetField, sourceFields] of Object.entries(embeddingConfig.fields)) {
10174
- const targetType = flatFields[targetField];
10175
- if (targetType === void 0) {
10176
- throw new NarsilError(
10177
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
10178
- `Embedding target field "${targetField}" does not exist in the schema`,
10179
- { field: targetField }
10180
- );
10181
- }
10182
- const vectorMatch = VECTOR_DIMENSION_PATTERN.exec(targetType);
10183
- if (!vectorMatch) {
10184
- throw new NarsilError(
10185
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
10186
- `Embedding target field "${targetField}" must be a vector type, got "${targetType}"`,
10187
- { field: targetField, type: targetType }
10188
- );
10189
- }
10190
- const schemaDimension = Number.parseInt(vectorMatch[1], 10);
10191
- if (schemaDimension !== resolvedAdapter.dimensions) {
10192
- throw new NarsilError(
10193
- ErrorCodes.EMBEDDING_DIMENSION_MISMATCH,
10194
- `Embedding target field "${targetField}" has dimension ${schemaDimension} but the adapter produces ${resolvedAdapter.dimensions}-dimensional vectors`,
10195
- { field: targetField, expected: schemaDimension, actual: resolvedAdapter.dimensions }
10196
- );
10197
- }
10198
- const sources = Array.isArray(sourceFields) ? sourceFields : [sourceFields];
10199
- if (sources.length === 0) {
10200
- throw new NarsilError(
10201
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
10202
- `Embedding target field "${targetField}" must have at least one source field`,
10203
- { field: targetField }
10204
- );
10205
- }
10206
- for (const source of sources) {
10207
- const sourceType = flatFields[source];
10208
- if (sourceType === void 0) {
10209
- throw new NarsilError(
10210
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
10211
- `Embedding source field "${source}" does not exist in the schema`,
10212
- { field: source, targetField }
10213
- );
10214
- }
10215
- if (!isSingleStringFieldType(sourceType)) {
10216
- throw new NarsilError(
10217
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
10218
- `Embedding source field "${source}" must be a string type, got "${sourceType}"`,
10219
- { field: source, type: sourceType, targetField }
10220
- );
10221
- }
10222
- }
10223
- }
10224
- return resolvedAdapter;
10225
- }
10226
- function validateRegisteredAdapter(name, adapter) {
10227
- if (typeof name !== "string" || name.trim().length === 0) {
10228
- throw new NarsilError(ErrorCodes.EMBEDDING_CONFIG_INVALID, "Embedding adapter names must be non-empty strings");
10229
- }
10230
- if (typeof adapter?.embed !== "function") {
10231
- throw new NarsilError(
10232
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
10233
- `Embedding adapter "${name}" must provide an embed function`,
10234
- { adapter: name }
10235
- );
10236
- }
10237
- if (typeof adapter.dimensions !== "number" || !Number.isInteger(adapter.dimensions) || adapter.dimensions <= 0) {
10238
- throw new NarsilError(
10239
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
10240
- `Embedding adapter "${name}" dimensions must be a positive integer, got ${adapter.dimensions}`,
10241
- { adapter: name, dimensions: adapter.dimensions }
10242
- );
10243
- }
10244
- }
10245
- function validateRequiredFieldsInSchema(required, schema) {
10246
- if (required.length === 0) return;
10247
- const flatFields = flattenSchema(schema);
10248
- for (const field of required) {
10249
- if (flatFields[field] === void 0) {
10250
- throw new NarsilError(
10251
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
10252
- `Required field "${field}" does not exist in the schema`,
10253
- { field }
10254
- );
10255
- }
10256
- }
10257
- }
10258
-
10259
- // src/runtime/heap-statistics.browser.ts
10260
- function readHeapStatistics() {
10261
- return null;
10262
- }
10263
-
10264
- // src/search/oversample.ts
10265
- function oversampledShardSize(size) {
10266
- return Math.ceil(size * 1.5) + 10;
10267
- }
10268
-
10269
- // src/search/facets.ts
10270
- function keptValueCount(fieldConfig) {
10271
- return keptFacetValues(fieldConfig?.limit);
10272
- }
10273
- function everyValueFacetConfig(config) {
10274
- const widened = {};
10275
- for (const [field, fieldConfig] of Object.entries(config)) {
10276
- widened[field] = { ...fieldConfig, limit: void 0 };
10277
- }
10278
- return widened;
10279
- }
10280
- function oversampledFacetConfig(config) {
10281
- const widened = {};
10282
- for (const [field, fieldConfig] of Object.entries(config)) {
10283
- if (fieldConfig?.ranges !== void 0 || fieldConfig?.sort === "asc") {
10284
- widened[field] = { ...fieldConfig, limit: void 0 };
10285
- continue;
10286
- }
10287
- const kept = keptValueCount(fieldConfig);
10288
- widened[field] = kept === void 0 ? fieldConfig : { ...fieldConfig, limit: oversampledShardSize(kept) };
10289
- }
10290
- return widened;
10291
- }
10292
- function mergeFacets(partitionFacets, config) {
10293
- const merged = /* @__PURE__ */ new Map();
10294
- const bounds = /* @__PURE__ */ new Map();
10295
- for (const partition of partitionFacets) {
10296
- for (const [field, facetResult] of Object.entries(partition)) {
10297
- let fieldMap = merged.get(field);
10298
- if (!fieldMap) {
10299
- fieldMap = /* @__PURE__ */ new Map();
10300
- merged.set(field, fieldMap);
10050
+ state.partialWords.afterDocument(partialFieldsOf(schema), internalId, next.partialSpellings, 1);
10051
+ recordSortValues(state, internalId, document);
10052
+ } else {
10053
+ updateFieldIndexOnly(state, docId, stored.fields, document, fields);
10054
+ state.docStore.store(docId, document, stored.fieldLengths);
10055
+ const internalId = state.docStore.getInternalId(docId);
10056
+ if (internalId !== void 0) recordSortValues(state, internalId, document);
10301
10057
  }
10302
- bounds.set(field, (bounds.get(field) ?? 0) + facetResult.errorBound);
10303
- for (const [value, count] of Object.entries(facetResult.values)) {
10304
- fieldMap.set(value, (fieldMap.get(value) ?? 0) + count);
10058
+ },
10059
+ get(docId, projection) {
10060
+ const stored = state.docStore.get(docId);
10061
+ if (!stored) return void 0;
10062
+ return cloneProjected(stored.fields, projection);
10063
+ },
10064
+ getRef(docId) {
10065
+ const stored = state.docStore.get(docId);
10066
+ if (!stored) return void 0;
10067
+ return stored.fields;
10068
+ },
10069
+ fieldValues(docId, fieldPaths) {
10070
+ const stored = state.docStore.get(docId);
10071
+ if (!stored) return void 0;
10072
+ return fieldPaths.map((fieldPath) => getNestedValue(stored.fields, fieldPath));
10073
+ },
10074
+ has(docId) {
10075
+ return state.docStore.has(docId);
10076
+ },
10077
+ count() {
10078
+ return state.docStore.count();
10079
+ },
10080
+ *docIds() {
10081
+ for (const [id] of state.docStore.all()) {
10082
+ yield id;
10305
10083
  }
10084
+ },
10085
+ sortedDocIds() {
10086
+ return state.docStore.sortedDocIds();
10087
+ },
10088
+ releaseSortedDocIds() {
10089
+ state.docStore.releaseSortedDocIds();
10090
+ },
10091
+ clear: clearAll,
10092
+ estimateMemoryBytes() {
10093
+ return estimatePartitionBytes(state);
10094
+ },
10095
+ readStates() {
10096
+ return [state];
10097
+ },
10098
+ searchFulltext(params) {
10099
+ return searchFulltext(state, params);
10100
+ },
10101
+ searchFulltextMatches(params) {
10102
+ return searchFulltextMatches(state, params);
10103
+ },
10104
+ sortedPage(request) {
10105
+ return sortedPageOf(state, request);
10106
+ },
10107
+ sortValues(docId, fields, fieldTypes, modes) {
10108
+ return sortValuesOf(state, docId, fields, fieldTypes, modes);
10109
+ },
10110
+ applyFilters(filters, schema, meter) {
10111
+ return applyPartitionFilters(state, filters, schema, meter);
10112
+ },
10113
+ applyFiltersBitset(filters, schema, meter) {
10114
+ return applyPartitionFiltersBitset(state, filters, schema, meter);
10115
+ },
10116
+ filterMatches(filters, schema, meter) {
10117
+ return partitionFilterMatches(state, filters, schema, meter);
10118
+ },
10119
+ computeFacets(matched, config, schema) {
10120
+ return computeFacets(state, matched, config, schema);
10121
+ },
10122
+ suggestTerms(surfacePrefix, stemmedPrefix, limit) {
10123
+ return suggestDisplayTerms(state, surfacePrefix, stemmedPrefix, limit);
10124
+ },
10125
+ expandTermPrefix(surfacePrefix, stemmedToken, maxExpansions) {
10126
+ return expandTermPrefix([state], surfacePrefix, stemmedToken, maxExpansions);
10127
+ },
10128
+ expandPartialWords(analysis, requests) {
10129
+ return expandPartialWords([state], analysis, requests);
10130
+ },
10131
+ preparePartialWords(analysis) {
10132
+ expandPartialWords([state], analysis, []);
10133
+ },
10134
+ serialize(indexName, totalPartitions, language, schema) {
10135
+ return serializePartition(state, partitionId, indexName, totalPartitions, language, schema);
10136
+ },
10137
+ deserialize(data, schema) {
10138
+ deserializePartition(state, data, clearAll, schema);
10306
10139
  }
10307
- }
10308
- const result = {};
10309
- for (const [field, valueMap] of merged) {
10310
- const fieldConfig = config[field];
10311
- const ascending = fieldConfig?.sort === "asc";
10312
- const ordered = Array.from(valueMap.entries());
10313
- ordered.sort((a, b) => (ascending ? a[1] - b[1] : b[1] - a[1]) || compareCodePoints(a[0], b[0]));
10314
- const kept = Math.min(keptValueCount(fieldConfig) ?? ordered.length, ordered.length);
10315
- let largestLeftOut = 0;
10316
- for (let index = kept; index < ordered.length; index++) {
10317
- if (ordered[index][1] > largestLeftOut) largestLeftOut = ordered[index][1];
10318
- }
10319
- const errorBound = (bounds.get(field) ?? 0) + largestLeftOut;
10320
- const values = {};
10321
- for (let index = 0; index < kept; index++) {
10322
- values[ordered[index][0]] = ordered[index][1];
10323
- }
10324
- result[field] = { values, count: kept, errorBound };
10325
- }
10326
- return result;
10140
+ };
10141
+ statesByPartition.set(partition, state);
10142
+ return partition;
10327
10143
  }
10328
10144
 
10329
10145
  // src/core/partition/frozen/memory.ts
10330
10146
  function frozenSegmentBytes(segment) {
10147
+ if (segment.paged !== null) return segment.paged.heldBytes(segment);
10331
10148
  const arrays = segment.arrays;
10332
10149
  let bytes = arrays.tokenTable.bytes + arrays.idTable.bytes;
10333
10150
  bytes += arrays.postingOffsets.byteLength;
@@ -10439,6 +10256,71 @@ function compositeSortedDocIds(frozen, live) {
10439
10256
  return merged;
10440
10257
  }
10441
10258
 
10259
+ // src/search/oversample.ts
10260
+ function oversampledShardSize(size) {
10261
+ return Math.ceil(size * 1.5) + 10;
10262
+ }
10263
+
10264
+ // src/search/facets.ts
10265
+ function keptValueCount(fieldConfig) {
10266
+ return keptFacetValues(fieldConfig?.limit);
10267
+ }
10268
+ function everyValueFacetConfig(config) {
10269
+ const widened = {};
10270
+ for (const [field, fieldConfig] of Object.entries(config)) {
10271
+ widened[field] = { ...fieldConfig, limit: void 0 };
10272
+ }
10273
+ return widened;
10274
+ }
10275
+ function oversampledFacetConfig(config) {
10276
+ const widened = {};
10277
+ for (const [field, fieldConfig] of Object.entries(config)) {
10278
+ if (fieldConfig?.ranges !== void 0 || fieldConfig?.sort === "asc") {
10279
+ widened[field] = { ...fieldConfig, limit: void 0 };
10280
+ continue;
10281
+ }
10282
+ const kept = keptValueCount(fieldConfig);
10283
+ widened[field] = kept === void 0 ? fieldConfig : { ...fieldConfig, limit: oversampledShardSize(kept) };
10284
+ }
10285
+ return widened;
10286
+ }
10287
+ function mergeFacets(partitionFacets, config) {
10288
+ const merged = /* @__PURE__ */ new Map();
10289
+ const bounds = /* @__PURE__ */ new Map();
10290
+ for (const partition of partitionFacets) {
10291
+ for (const [field, facetResult] of Object.entries(partition)) {
10292
+ let fieldMap = merged.get(field);
10293
+ if (!fieldMap) {
10294
+ fieldMap = /* @__PURE__ */ new Map();
10295
+ merged.set(field, fieldMap);
10296
+ }
10297
+ bounds.set(field, (bounds.get(field) ?? 0) + facetResult.errorBound);
10298
+ for (const [value, count] of Object.entries(facetResult.values)) {
10299
+ fieldMap.set(value, (fieldMap.get(value) ?? 0) + count);
10300
+ }
10301
+ }
10302
+ }
10303
+ const result = {};
10304
+ for (const [field, valueMap] of merged) {
10305
+ const fieldConfig = config[field];
10306
+ const ascending = fieldConfig?.sort === "asc";
10307
+ const ordered = Array.from(valueMap.entries());
10308
+ ordered.sort((a, b) => (ascending ? a[1] - b[1] : b[1] - a[1]) || compareCodePoints(a[0], b[0]));
10309
+ const kept = Math.min(keptValueCount(fieldConfig) ?? ordered.length, ordered.length);
10310
+ let largestLeftOut = 0;
10311
+ for (let index = kept; index < ordered.length; index++) {
10312
+ if (ordered[index][1] > largestLeftOut) largestLeftOut = ordered[index][1];
10313
+ }
10314
+ const errorBound = (bounds.get(field) ?? 0) + largestLeftOut;
10315
+ const values = {};
10316
+ for (let index = 0; index < kept; index++) {
10317
+ values[ordered[index][0]] = ordered[index][1];
10318
+ }
10319
+ result[field] = { values, count: kept, errorBound };
10320
+ }
10321
+ return result;
10322
+ }
10323
+
10442
10324
  // src/core/partition/composite/filters.ts
10443
10325
  var WORD_BITS = 32;
10444
10326
  function alignToWord(value) {
@@ -10519,6 +10401,29 @@ function subMatchesView(matches, layout, subIndex) {
10519
10401
  };
10520
10402
  }
10521
10403
 
10404
+ // src/core/partition/composite/facets.ts
10405
+ function compositeFacets(subs, layout, matched, config, schema) {
10406
+ const everyValue = everyValueFacetConfig(config);
10407
+ if ("ordinalBitset" in matched) {
10408
+ const ordinalLayout = layout();
10409
+ return mergeFacets(
10410
+ subs.map(
10411
+ (sub, index) => computeFacets(
10412
+ sub,
10413
+ { ordinalBitset: subBitsetView(matched.ordinalBitset, ordinalLayout, index) },
10414
+ everyValue,
10415
+ schema
10416
+ )
10417
+ ),
10418
+ config
10419
+ );
10420
+ }
10421
+ return mergeFacets(
10422
+ subs.map((sub) => computeFacets(sub, matched, everyValue, schema)),
10423
+ config
10424
+ );
10425
+ }
10426
+
10522
10427
  // src/core/partition/scored-merge.ts
10523
10428
  function kWayMerge(arrays) {
10524
10429
  const nonEmpty = arrays.filter((a) => a.length > 0);
@@ -10898,7 +10803,8 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
10898
10803
  }
10899
10804
  function cachedDocFrequencies() {
10900
10805
  if (docFrequenciesCache === null) {
10901
- docFrequenciesCache = mergeDocFrequencies(subs());
10806
+ const paged = frozen.find((segment) => segment.paged !== null)?.paged;
10807
+ docFrequenciesCache = paged ? paged.termFrequenciesAcross(subs) : mergeDocFrequencies(subs());
10902
10808
  }
10903
10809
  return docFrequenciesCache;
10904
10810
  }
@@ -10969,6 +10875,20 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
10969
10875
  frozen.push(segment);
10970
10876
  bindSegments();
10971
10877
  },
10878
+ frozenSegments() {
10879
+ return frozen;
10880
+ },
10881
+ replaceFrozenSegments(segments) {
10882
+ frozen.length = 0;
10883
+ frozen.push(...segments);
10884
+ bindSegments();
10885
+ },
10886
+ documentIdsHeldInMemory() {
10887
+ return compositeDocIds(
10888
+ frozen.filter((segment) => segment.paged === null),
10889
+ live
10890
+ );
10891
+ },
10972
10892
  mergeSegment(segment) {
10973
10893
  invalidateDocFrequencies();
10974
10894
  live.mergeSegment(segment);
@@ -10991,6 +10911,7 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
10991
10911
  live.deserialize(data, schema);
10992
10912
  },
10993
10913
  clear() {
10914
+ for (const segment of frozen) segment.paged?.close();
10994
10915
  frozen.length = 0;
10995
10916
  invalidateDocFrequencies();
10996
10917
  live.clear();
@@ -11057,25 +10978,7 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
11057
10978
  return compositeFilterMatches(subs(), layout(), filters, schema, meter);
11058
10979
  },
11059
10980
  computeFacets(matched, config, schema) {
11060
- const everyValue = everyValueFacetConfig(config);
11061
- if ("ordinalBitset" in matched) {
11062
- const ordinalLayout = layout();
11063
- return mergeFacets(
11064
- subs().map(
11065
- (sub, index) => computeFacets(
11066
- sub,
11067
- { ordinalBitset: subBitsetView(matched.ordinalBitset, ordinalLayout, index) },
11068
- everyValue,
11069
- schema
11070
- )
11071
- ),
11072
- config
11073
- );
11074
- }
11075
- return mergeFacets(
11076
- subs().map((sub) => computeFacets(sub, matched, everyValue, schema)),
11077
- config
11078
- );
10981
+ return compositeFacets(subs(), layout, matched, config, schema);
11079
10982
  },
11080
10983
  sortedPage(request) {
11081
10984
  return compositeSortedPage(subs(), layout(), request);
@@ -11203,15 +11106,19 @@ function createCapacityChecks(source) {
11203
11106
  }
11204
11107
 
11205
11108
  // src/partitioning/manager/doc-locations.ts
11109
+ function documentIdsHeldInMemory(partition) {
11110
+ return isCompositePartition(partition) ? partition.documentIdsHeldInMemory() : partition.docIds();
11111
+ }
11206
11112
  function rebuildDocLocations(locations, partitions) {
11207
11113
  locations.clear();
11208
11114
  for (let i = 0; i < partitions.length; i++) {
11209
- for (const docId of partitions[i].docIds()) {
11115
+ for (const docId of documentIdsHeldInMemory(partitions[i])) {
11210
11116
  locations.set(docId, i);
11211
11117
  }
11212
11118
  }
11213
11119
  }
11214
11120
  function recordSegmentDocLocations(locations, partitionId, segment) {
11121
+ if (segment.paged !== null) return;
11215
11122
  for (const internalId of segment.docStore.allInternalIds()) {
11216
11123
  const docId = segment.docStore.getExternalId(internalId);
11217
11124
  if (docId !== void 0) locations.set(docId, partitionId);
@@ -11223,7 +11130,7 @@ function rebuildPartitionDocLocations(locations, partitionId, partition) {
11223
11130
  locations.delete(docId);
11224
11131
  }
11225
11132
  }
11226
- for (const docId of partition.docIds()) {
11133
+ for (const docId of documentIdsHeldInMemory(partition)) {
11227
11134
  locations.set(docId, partitionId);
11228
11135
  }
11229
11136
  }
@@ -11266,6 +11173,109 @@ function requireNewSegmentDocuments(segment, docPartitionMap) {
11266
11173
  }
11267
11174
  }
11268
11175
 
11176
+ // src/partitioning/manager/write-log.ts
11177
+ var REANALYSIS_WRITE_LOG = 0;
11178
+ var CHECKPOINT_WRITE_LOG = 1;
11179
+ function createWriteLogRecorder() {
11180
+ const current = [null, null];
11181
+ return {
11182
+ open(owner) {
11183
+ const log = { docIds: /* @__PURE__ */ new Set(), replaced: false };
11184
+ current[owner] = log;
11185
+ return {
11186
+ get partitionsReplaced() {
11187
+ return log.replaced;
11188
+ },
11189
+ get pendingDocuments() {
11190
+ return log.docIds.size;
11191
+ },
11192
+ takeWrittenDocIds() {
11193
+ const taken = log.docIds;
11194
+ log.docIds = /* @__PURE__ */ new Set();
11195
+ return taken;
11196
+ },
11197
+ close() {
11198
+ if (current[owner] === log) current[owner] = null;
11199
+ }
11200
+ };
11201
+ },
11202
+ documentWritten(docId) {
11203
+ current[0]?.docIds.add(docId);
11204
+ current[1]?.docIds.add(docId);
11205
+ },
11206
+ segmentWritten(segment) {
11207
+ for (const log of current) {
11208
+ if (log === null) continue;
11209
+ for (const internalId of segment.docStore.allInternalIds()) {
11210
+ const docId = segment.docStore.getExternalId(internalId);
11211
+ if (docId !== void 0) log.docIds.add(docId);
11212
+ }
11213
+ }
11214
+ },
11215
+ partitionsReplaced() {
11216
+ for (const log of current) {
11217
+ if (log !== null) log.replaced = true;
11218
+ }
11219
+ }
11220
+ };
11221
+ }
11222
+
11223
+ // src/partitioning/manager/segment-operations.ts
11224
+ function createSegmentOperations(source) {
11225
+ const { docPartitionMap, writeLogs, validatePartitionId, asCompositePartition, resolveInsertOptions } = source;
11226
+ return {
11227
+ mergeSegment(partitionId, payload, documents) {
11228
+ validatePartitionId(partitionId);
11229
+ const partition = source.partitionAt(partitionId);
11230
+ partition.beginBatch();
11231
+ partition.mergeSegmentPayload(payload, documents);
11232
+ partition.endBatch();
11233
+ for (const docId of payload.docIds) {
11234
+ docPartitionMap.set(docId, partitionId);
11235
+ writeLogs.documentWritten(docId);
11236
+ }
11237
+ },
11238
+ attachFrozenSegment(partitionId, segment) {
11239
+ validatePartitionId(partitionId);
11240
+ if (segment.paged === null) requireNewSegmentDocuments(segment, docPartitionMap);
11241
+ asCompositePartition(partitionId).attachFrozenSegment(segment);
11242
+ if (segment.paged !== null) return;
11243
+ recordSegmentDocLocations(docPartitionMap, partitionId, segment);
11244
+ writeLogs.segmentWritten(segment);
11245
+ },
11246
+ openCheckpointChangeLog() {
11247
+ return writeLogs.open(CHECKPOINT_WRITE_LOG);
11248
+ },
11249
+ diskSegmentAccess(partitionId) {
11250
+ validatePartitionId(partitionId);
11251
+ return {
11252
+ composite: asCompositePartition(partitionId),
11253
+ docPartitionMap,
11254
+ insertOptions: resolveInsertOptions()
11255
+ };
11256
+ },
11257
+ swapFrozenSegments(partitionId, dropSegmentIds, replacement, removedSpellings) {
11258
+ validatePartitionId(partitionId);
11259
+ const applied = asCompositePartition(partitionId).swapFrozenSegments(
11260
+ dropSegmentIds,
11261
+ replacement,
11262
+ removedSpellings
11263
+ );
11264
+ recordSegmentDocLocations(docPartitionMap, partitionId, replacement);
11265
+ return applied;
11266
+ },
11267
+ freezeLiveTail(partitionId, freeze) {
11268
+ validatePartitionId(partitionId);
11269
+ return asCompositePartition(partitionId).freezeLiveTail(freeze);
11270
+ },
11271
+ replaceLiveTail(partitionId, segment) {
11272
+ validatePartitionId(partitionId);
11273
+ asCompositePartition(partitionId).replaceLiveTail(segment);
11274
+ recordSegmentDocLocations(docPartitionMap, partitionId, segment);
11275
+ }
11276
+ };
11277
+ }
11278
+
11269
11279
  // src/partitioning/manager/stats.ts
11270
11280
  function aggregateStatsOf(partitions) {
11271
11281
  let totalDocuments = 0;
@@ -11301,47 +11311,13 @@ function partitionStatsOf(partitions) {
11301
11311
  }));
11302
11312
  }
11303
11313
 
11304
- // src/partitioning/manager/write-log.ts
11305
- function createWriteLogRecorder() {
11306
- let current = null;
11307
- return {
11308
- open() {
11309
- const log = { docIds: /* @__PURE__ */ new Set(), replaced: false };
11310
- current = log;
11311
- return {
11312
- get partitionsReplaced() {
11313
- return log.replaced;
11314
- },
11315
- get pendingDocuments() {
11316
- return log.docIds.size;
11317
- },
11318
- takeWrittenDocIds() {
11319
- const taken = log.docIds;
11320
- log.docIds = /* @__PURE__ */ new Set();
11321
- return taken;
11322
- },
11323
- close() {
11324
- if (current === log) current = null;
11325
- }
11326
- };
11327
- },
11328
- documentWritten(docId) {
11329
- if (current !== null) current.docIds.add(docId);
11330
- },
11331
- segmentWritten(segment) {
11332
- if (current === null) return;
11333
- for (const internalId of segment.docStore.allInternalIds()) {
11334
- const docId = segment.docStore.getExternalId(internalId);
11335
- if (docId !== void 0) current.docIds.add(docId);
11336
- }
11337
- },
11338
- partitionsReplaced() {
11339
- if (current !== null) current.replaced = true;
11340
- }
11341
- };
11342
- }
11343
-
11344
11314
  // src/partitioning/manager/index.ts
11315
+ function closePagedSegmentsLeftBehind(earlier, kept) {
11316
+ for (const partition of earlier) {
11317
+ if (!isCompositePartition(partition) || kept.includes(partition)) continue;
11318
+ for (const segment of partition.frozenSegments()) segment.paged?.close();
11319
+ }
11320
+ }
11345
11321
  function createPartitionManager(indexName, initialConfig, initialLanguage, router, initialPartitionCount, vectorIndexes) {
11346
11322
  let config = initialConfig;
11347
11323
  let language = initialLanguage;
@@ -11355,7 +11331,7 @@ function createPartitionManager(indexName, initialConfig, initialLanguage, route
11355
11331
  const vecIndexes = vectorIndexes ?? /* @__PURE__ */ new Map();
11356
11332
  let partitions = [];
11357
11333
  const docPartitionMap = /* @__PURE__ */ new Map();
11358
- const writeLog = createWriteLogRecorder();
11334
+ const writeLogs = createWriteLogRecorder();
11359
11335
  for (let i = 0; i < count; i++) {
11360
11336
  partitions.push(createPartitionIndex(i, trackPositions));
11361
11337
  }
@@ -11375,7 +11351,14 @@ function createPartitionManager(indexName, initialConfig, initialLanguage, route
11375
11351
  if (partialWordAnalysis !== null) partitions[partitionId].preparePartialWords(partialWordAnalysis);
11376
11352
  }
11377
11353
  function locateDocument(docId) {
11378
- return docPartitionMap.get(docId);
11354
+ const known = docPartitionMap.get(docId);
11355
+ if (known !== void 0 || config.textStorage !== "disk") return known;
11356
+ const routed = router.route(docId, partitions.length);
11357
+ if (partitions[routed]?.has(docId)) return routed;
11358
+ for (let i = 0; i < partitions.length; i++) {
11359
+ if (i !== routed && partitions[i].has(docId)) return i;
11360
+ }
11361
+ return void 0;
11379
11362
  }
11380
11363
  function asCompositePartition(partitionId) {
11381
11364
  const existing = partitions[partitionId];
@@ -11421,12 +11404,14 @@ function createPartitionManager(indexName, initialConfig, initialLanguage, route
11421
11404
  return [...partitions];
11422
11405
  },
11423
11406
  setPartitions(newPartitions) {
11424
- writeLog.partitionsReplaced();
11407
+ writeLogs.partitionsReplaced();
11408
+ const earlier = partitions;
11425
11409
  partitions = [...newPartitions];
11410
+ closePagedSegmentsLeftBehind(earlier, partitions);
11426
11411
  rebuildDocLocations(docPartitionMap, partitions);
11427
11412
  },
11428
11413
  openWriteLog() {
11429
- return writeLog.open();
11414
+ return writeLogs.open(REANALYSIS_WRITE_LOG);
11430
11415
  },
11431
11416
  adoptRebuiltPartitions(rebuilt) {
11432
11417
  if (rebuilt.length !== partitions.length) {
@@ -11436,7 +11421,9 @@ function createPartitionManager(indexName, initialConfig, initialLanguage, route
11436
11421
  { indexName, partitionCount: partitions.length, rebuiltCount: rebuilt.length }
11437
11422
  );
11438
11423
  }
11424
+ const earlier = partitions;
11439
11425
  partitions = [...rebuilt];
11426
+ closePagedSegmentsLeftBehind(earlier, partitions);
11440
11427
  },
11441
11428
  adoptAnalysis(change) {
11442
11429
  config = change.config;
@@ -11456,13 +11443,13 @@ function createPartitionManager(indexName, initialConfig, initialLanguage, route
11456
11443
  );
11457
11444
  }
11458
11445
  const partition = createPartitionIndex(partitions.length, trackPositions);
11459
- writeLog.partitionsReplaced();
11446
+ writeLogs.partitionsReplaced();
11460
11447
  partitions.push(partition);
11461
11448
  return partition;
11462
11449
  },
11463
11450
  removePartition(partitionId) {
11464
11451
  validatePartitionId(partitionId);
11465
- writeLog.partitionsReplaced();
11452
+ writeLogs.partitionsReplaced();
11466
11453
  partitions.splice(partitionId, 1);
11467
11454
  rebuildDocLocations(docPartitionMap, partitions);
11468
11455
  },
@@ -11476,7 +11463,7 @@ function createPartitionManager(indexName, initialConfig, initialLanguage, route
11476
11463
  if (newCount >= partitions.length) {
11477
11464
  return;
11478
11465
  }
11479
- writeLog.partitionsReplaced();
11466
+ writeLogs.partitionsReplaced();
11480
11467
  for (const [docId, pid] of docPartitionMap) {
11481
11468
  if (pid >= newCount) {
11482
11469
  docPartitionMap.delete(docId);
@@ -11499,7 +11486,7 @@ function createPartitionManager(indexName, initialConfig, initialLanguage, route
11499
11486
  const insertOpts = resolveInsertOptions(options);
11500
11487
  partitions[pid].insert(docId, document, config.schema, language, insertOpts);
11501
11488
  docPartitionMap.set(docId, pid);
11502
- writeLog.documentWritten(docId);
11489
+ writeLogs.documentWritten(docId);
11503
11490
  },
11504
11491
  removedSpellingsOf(docId) {
11505
11492
  const pid = locateDocument(docId);
@@ -11513,7 +11500,7 @@ function createPartitionManager(indexName, initialConfig, initialLanguage, route
11513
11500
  }
11514
11501
  partitions[pid].remove(docId, config.schema, language, resolveInsertOptions(), removedSpellings);
11515
11502
  docPartitionMap.delete(docId);
11516
- writeLog.documentWritten(docId);
11503
+ writeLogs.documentWritten(docId);
11517
11504
  },
11518
11505
  beginBatchRemove() {
11519
11506
  for (let i = 0; i < partitions.length; i++) {
@@ -11533,7 +11520,7 @@ function createPartitionManager(indexName, initialConfig, initialLanguage, route
11533
11520
  const updateOpts = resolveInsertOptions(options);
11534
11521
  partitions[pid].update(docId, document, config.schema, language, updateOpts, removedSpellings);
11535
11522
  docPartitionMap.set(docId, pid);
11536
- writeLog.documentWritten(docId);
11523
+ writeLogs.documentWritten(docId);
11537
11524
  },
11538
11525
  get(docId, projection) {
11539
11526
  const pid = locateDocument(docId);
@@ -11573,46 +11560,17 @@ function createPartitionManager(indexName, initialConfig, initialLanguage, route
11573
11560
  validatePartitionId(partitionId);
11574
11561
  return partitions[partitionId].serialize(indexName, partitions.length, language.name, config.schema);
11575
11562
  },
11576
- mergeSegment(partitionId, payload, documents) {
11577
- validatePartitionId(partitionId);
11578
- const partition = partitions[partitionId];
11579
- partition.beginBatch();
11580
- partition.mergeSegmentPayload(payload, documents);
11581
- partition.endBatch();
11582
- for (const docId of payload.docIds) {
11583
- docPartitionMap.set(docId, partitionId);
11584
- writeLog.documentWritten(docId);
11585
- }
11586
- },
11587
- attachFrozenSegment(partitionId, segment) {
11588
- validatePartitionId(partitionId);
11589
- requireNewSegmentDocuments(segment, docPartitionMap);
11590
- asCompositePartition(partitionId).attachFrozenSegment(segment);
11591
- recordSegmentDocLocations(docPartitionMap, partitionId, segment);
11592
- writeLog.segmentWritten(segment);
11593
- },
11594
- swapFrozenSegments(partitionId, dropSegmentIds, replacement, removedSpellings) {
11595
- validatePartitionId(partitionId);
11596
- const applied = asCompositePartition(partitionId).swapFrozenSegments(
11597
- dropSegmentIds,
11598
- replacement,
11599
- removedSpellings
11600
- );
11601
- recordSegmentDocLocations(docPartitionMap, partitionId, replacement);
11602
- return applied;
11603
- },
11604
- freezeLiveTail(partitionId, freeze) {
11605
- validatePartitionId(partitionId);
11606
- return asCompositePartition(partitionId).freezeLiveTail(freeze);
11607
- },
11608
- replaceLiveTail(partitionId, segment) {
11609
- validatePartitionId(partitionId);
11610
- asCompositePartition(partitionId).replaceLiveTail(segment);
11611
- recordSegmentDocLocations(docPartitionMap, partitionId, segment);
11612
- },
11563
+ ...createSegmentOperations({
11564
+ docPartitionMap,
11565
+ writeLogs,
11566
+ partitionAt: (partitionId) => partitions[partitionId],
11567
+ validatePartitionId,
11568
+ asCompositePartition,
11569
+ resolveInsertOptions: () => resolveInsertOptions()
11570
+ }),
11613
11571
  deserializePartition(partitionId, data) {
11614
11572
  validatePartitionId(partitionId);
11615
- writeLog.partitionsReplaced();
11573
+ writeLogs.partitionsReplaced();
11616
11574
  partitions[partitionId].deserialize(data, config.schema);
11617
11575
  preparePartialWords(partitionId);
11618
11576
  rebuildPartitionDocLocations(docPartitionMap, partitionId, partitions[partitionId]);
@@ -13074,11 +13032,17 @@ function encodeDocumentTableData(documents) {
13074
13032
  }
13075
13033
  function wrapEncodedDocumentTable(data) {
13076
13034
  const { blob, offsets } = data;
13077
- const count = offsets.length - 1;
13078
- const discarded = (data.discardedFields ?? []).map((fieldPath) => fieldPath.split("."));
13035
+ return createEncodedDocumentSource(
13036
+ offsets.length - 1,
13037
+ blob.byteLength + offsets.byteLength,
13038
+ (ordinal) => blob.subarray(offsets[ordinal], offsets[ordinal + 1]),
13039
+ data.discardedFields
13040
+ );
13041
+ }
13042
+ function createEncodedDocumentSource(count, byteLength, bytesAt, discardedFields) {
13043
+ const discarded = (discardedFields ?? []).map((fieldPath) => fieldPath.split("."));
13079
13044
  const touchesDiscarded = (path) => discarded.some((discardedPath) => sharesPrefix(discardedPath, path));
13080
- const bytesAt = (ordinal) => blob.subarray(offsets[ordinal], offsets[ordinal + 1]);
13081
- const docAt = (ordinal) => documentWithOwnBuffers(decodeStoredDocument(bytesAt(ordinal), data.discardedFields));
13045
+ const docAt = (ordinal) => documentWithOwnBuffers(decodeStoredDocument(bytesAt(ordinal), discardedFields));
13082
13046
  function projectedPathAt(ordinal, path) {
13083
13047
  const segments = encodedProjectionPathOf(path);
13084
13048
  if (segments === null) return { found: false, value: void 0 };
@@ -13088,10 +13052,10 @@ function wrapEncodedDocumentTable(data) {
13088
13052
  }
13089
13053
  return {
13090
13054
  count,
13091
- byteLength: blob.byteLength + offsets.byteLength,
13055
+ byteLength,
13092
13056
  docAt,
13093
13057
  readAt(ordinal) {
13094
- return decodeStoredDocument(bytesAt(ordinal), data.discardedFields);
13058
+ return decodeStoredDocument(bytesAt(ordinal), discardedFields);
13095
13059
  },
13096
13060
  fieldReader(fieldPath) {
13097
13061
  const path = fieldPath.split(".");
@@ -13254,6 +13218,9 @@ function wrapExternalIdTable(data) {
13254
13218
  const ordinal = sortedOrdinals[lo];
13255
13219
  return compareBytes(blob, offsets[ordinal], offsets[ordinal + 1], query) === 0 ? ordinal : -1;
13256
13220
  },
13221
+ sortedOrdinalAt(position) {
13222
+ return sortedOrdinals[position];
13223
+ },
13257
13224
  collectSortedIds(excluded) {
13258
13225
  const ids = [];
13259
13226
  for (let position = 0; position < count; position++) {
@@ -13273,6 +13240,7 @@ function buildExternalIdTable(docIds) {
13273
13240
  characters += docIds[ordinal].length;
13274
13241
  }
13275
13242
  let sortedIds = null;
13243
+ let sortedOrdinals = null;
13276
13244
  return {
13277
13245
  count: docIds.length,
13278
13246
  bytes: characters,
@@ -13282,6 +13250,10 @@ function buildExternalIdTable(docIds) {
13282
13250
  ordinalOf(docId) {
13283
13251
  return ordinals.get(docId) ?? -1;
13284
13252
  },
13253
+ sortedOrdinalAt(position) {
13254
+ if (sortedOrdinals === null) sortedOrdinals = codePointOrder(docIds);
13255
+ return sortedOrdinals[position];
13256
+ },
13285
13257
  collectSortedIds(excluded) {
13286
13258
  if (sortedIds === null) {
13287
13259
  const ids = [...docIds];
@@ -13853,6 +13825,10 @@ function createFrozenRemovalPass(documents, surfaces) {
13853
13825
  });
13854
13826
  return spellings;
13855
13827
  }
13828
+ function subtract(ordinal, analysis) {
13829
+ if (surfaces.tableStaysOnDisk()) surfaces.subtractOnceLoaded(ordinal, analysis);
13830
+ else surfaces.subtractSpellings(ordinal, heldSpellingsOf(ordinal, analysis));
13831
+ }
13856
13832
  return {
13857
13833
  get settled() {
13858
13834
  return pending.length === 0;
@@ -13861,17 +13837,17 @@ function createFrozenRemovalPass(documents, surfaces) {
13861
13837
  if (bound !== null) return;
13862
13838
  const scoped = { ...analysis, fields: documents.textFields };
13863
13839
  bound = scoped;
13864
- for (const ordinal of pending) surfaces.subtractSpellings(ordinal, heldSpellingsOf(ordinal, scoped));
13840
+ for (const ordinal of pending) subtract(ordinal, scoped);
13865
13841
  pending = [];
13866
13842
  },
13867
13843
  spellingsOf(ordinal) {
13868
- return bound === null ? void 0 : heldSpellingsOf(ordinal, bound);
13844
+ return bound === null || surfaces.tableStaysOnDisk() ? void 0 : heldSpellingsOf(ordinal, bound);
13869
13845
  },
13870
13846
  afterRemoval(ordinal, spellings) {
13871
13847
  if (spellings !== void 0) {
13872
13848
  surfaces.subtractSpellings(ordinal, spellings);
13873
13849
  } else if (bound !== null) {
13874
- surfaces.subtractSpellings(ordinal, heldSpellingsOf(ordinal, bound));
13850
+ subtract(ordinal, bound);
13875
13851
  } else {
13876
13852
  pending.push(ordinal);
13877
13853
  }
@@ -13895,15 +13871,32 @@ function addForms(target, forms) {
13895
13871
  if (Array.isArray(value)) target.add(surface, value[1], value[0]);
13896
13872
  }
13897
13873
  }
13898
- function createFrozenSurfaceSlot(load, documents) {
13874
+ function createFrozenSurfaceSlot(load, documents, tableOnDisk) {
13899
13875
  let registry = null;
13900
13876
  let uncounted = null;
13877
+ let table = tableOnDisk;
13878
+ let waitingRemovals = [];
13879
+ let waitingAnalysis = null;
13880
+ function subtractWaitingRemovals(target) {
13881
+ if (waitingAnalysis === null || target.size() === 0) return;
13882
+ for (const ordinal of waitingRemovals) {
13883
+ const spellings = [];
13884
+ visitDocumentSurfaceSpellings(documents.documentAt(ordinal), waitingAnalysis, (spelling) => {
13885
+ if (target.holds(spelling)) spellings.push(spelling);
13886
+ });
13887
+ for (const spelling of spellings) target.subtract(spelling, 1);
13888
+ }
13889
+ }
13901
13890
  function loaded3() {
13902
13891
  if (registry === null) {
13903
13892
  const built = createSurfaceRegistry();
13904
13893
  const forms = load();
13905
13894
  if (forms !== null) built.deserialize(forms);
13895
+ subtractWaitingRemovals(built);
13906
13896
  registry = built;
13897
+ table = void 0;
13898
+ waitingRemovals = [];
13899
+ waitingAnalysis = null;
13907
13900
  }
13908
13901
  return registry;
13909
13902
  }
@@ -13911,12 +13904,22 @@ function createFrozenSurfaceSlot(load, documents) {
13911
13904
  return uncounted === null || uncounted[ordinal] === 0;
13912
13905
  }
13913
13906
  return {
13907
+ heldBytes: () => table?.heldBytes() ?? 0,
13914
13908
  reader: {
13915
- candidatesForPrefix: (prefix) => loaded3().candidatesForPrefix(prefix),
13909
+ candidatesForPrefix: (prefix) => table !== void 0 && waitingRemovals.length === 0 ? table.candidatesForPrefix(prefix) : loaded3().candidatesForPrefix(prefix),
13916
13910
  stemChangedTotalFor: (token) => loaded3().stemChangedTotalFor(token),
13917
13911
  size: () => loaded3().size(),
13918
13912
  serialize: () => loaded3().serialize()
13919
13913
  },
13914
+ tableStaysOnDisk() {
13915
+ return registry === null && tableOnDisk !== void 0;
13916
+ },
13917
+ subtractOnceLoaded(ordinal, analysis) {
13918
+ uncounted ??= new Uint8Array(documents.count);
13919
+ uncounted[ordinal] = 1;
13920
+ waitingAnalysis = analysis;
13921
+ waitingRemovals.push(ordinal);
13922
+ },
13920
13923
  holdsAny() {
13921
13924
  return loaded3().size() > 0;
13922
13925
  },
@@ -14064,6 +14067,9 @@ function wrapFrozenTokenTable(data) {
14064
14067
  }
14065
14068
  return token;
14066
14069
  },
14070
+ tokenBytesAt(sortedIndex) {
14071
+ return blob.subarray(offsets[sortedIndex], offsets[sortedIndex + 1]);
14072
+ },
14067
14073
  byteLengthAt(sortedIndex) {
14068
14074
  return offsets[sortedIndex + 1] - offsets[sortedIndex];
14069
14075
  },
@@ -14338,10 +14344,10 @@ function buildGeoReaders(entries) {
14338
14344
  }
14339
14345
  return readers;
14340
14346
  }
14341
- function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot, partialWordTable, survivingSurfaceForms) {
14347
+ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot, partialWordTable, survivingSurfaceForms, pagedParts) {
14342
14348
  const tombstones = createFrozenTombstones();
14343
- const counts = createFrozenLiveCounts(source, source.tokenTable, tombstones, storedFrequenciesOf(source));
14344
- const postingViews = createFrozenPostingViews(source, tombstones, counts);
14349
+ const counts = pagedParts?.createCounts(tombstones) ?? createFrozenLiveCounts(source, source.tokenTable, tombstones, storedFrequenciesOf(source));
14350
+ const postingViews = pagedParts?.createViews(tombstones, counts) ?? createFrozenPostingViews(source, tombstones, counts);
14345
14351
  const lengthColumns = /* @__PURE__ */ new Map();
14346
14352
  for (let i = 0; i < source.fieldLengthNames.length; i++) {
14347
14353
  lengthColumns.set(source.fieldLengthNames[i], source.fieldLengthColumns[i]);
@@ -14358,7 +14364,11 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
14358
14364
  documentAt: (ordinal) => documentSource.readAt(ordinal)
14359
14365
  };
14360
14366
  const loadSurfaceForms = survivingSurfaceForms === void 0 ? source.readSurfaceForms : () => survivingSurfaceForms;
14361
- const surfaces = createFrozenSurfaceSlot(loadSurfaceForms, textDocuments);
14367
+ const surfaces = createFrozenSurfaceSlot(
14368
+ loadSurfaceForms,
14369
+ textDocuments,
14370
+ survivingSurfaceForms === void 0 ? source.surfaceTableOnDisk : void 0
14371
+ );
14362
14372
  const removals = createFrozenRemovalPass(textDocuments, surfaces);
14363
14373
  const numericIndexes = /* @__PURE__ */ new Map();
14364
14374
  for (const entry of source.numeric) numericIndexes.set(entry.fieldPath, createFrozenNumericReader(entry));
@@ -14380,6 +14390,7 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
14380
14390
  segmentId,
14381
14391
  documentSource,
14382
14392
  sharedSnapshot,
14393
+ paged: pagedParts?.access ?? null,
14383
14394
  arrays: source,
14384
14395
  invertedIdx: createFrozenInvertedReader(source.tokenTable, postingViews, counts),
14385
14396
  docStore,
@@ -14395,7 +14406,7 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
14395
14406
  sizeIndexes,
14396
14407
  fieldNameTable,
14397
14408
  partialWords,
14398
- trackPositions: source.positionOffsets !== null,
14409
+ trackPositions: pagedParts?.access.snapshot.tracksPositions ?? source.positionOffsets !== null,
14399
14410
  flatSchemaCache: null,
14400
14411
  lastSchemaRef: null,
14401
14412
  sortColumns: null,
@@ -14439,6 +14450,9 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
14439
14450
  };
14440
14451
  return segment;
14441
14452
  }
14453
+ function assemblePagedFrozenSegment(segmentId, source, documentSource, partialWordTable, parts) {
14454
+ return assembleFrozenSegment(segmentId, source, documentSource, null, partialWordTable, void 0, parts);
14455
+ }
14442
14456
  function createFrozenSegment(payload, documents, segmentId) {
14443
14457
  const tokenTable = buildFrozenTokenTable(payload.tokens, payload.docFrequencies);
14444
14458
  return assembleFrozenSegment(
@@ -15302,7 +15316,20 @@ function buildSegmentPayload(documents, schema, language, options, trackPosition
15302
15316
 
15303
15317
  // src/workers/segment-actions.ts
15304
15318
  function isSegmentAction(action) {
15305
- return action.type === "buildSegment" || action.type === "mergeSegments" || action.type === "attachSegments" || action.type === "compactSegments" || action.type === "swapSegments" || action.type === "freezeLiveTail";
15319
+ return action.type === "buildSegment" || action.type === "mergeSegments" || action.type === "attachSegments" || action.type === "compactSegments" || action.type === "swapSegments" || action.type === "freezeLiveTail" || action.type === "adoptDiskSegments";
15320
+ }
15321
+ function isPagedSnapshot(snapshot) {
15322
+ return "path" in snapshot;
15323
+ }
15324
+ async function attachSegments(entry, action) {
15325
+ for (const segment of action.segments) {
15326
+ growPartitionsTo(entry.manager, segment.partitionId);
15327
+ const surfaceForms = segment.survivingSurfaceForms;
15328
+ const spellings = surfaceForms === void 0 ? void 0 : NO_REMOVED_SPELLINGS;
15329
+ const frozen = isPagedSnapshot(segment.snapshot) ? (await import('./module-BUFNZMEU.mjs')).createPagedFrozenSegment(segment.snapshot) : createSharedFrozenSegment(segment.snapshot, surfaceForms);
15330
+ for (const docId of segment.tombstonedDocIds ?? []) frozen.tombstoneDocument(docId, spellings);
15331
+ entry.manager.attachFrozenSegment(segment.partitionId, frozen);
15332
+ }
15306
15333
  }
15307
15334
  function growPartitionsTo(manager, partitionId) {
15308
15335
  while (manager.partitionCount <= partitionId) {
@@ -15351,17 +15378,10 @@ function runSegmentAction(entry, action) {
15351
15378
  }
15352
15379
  return void 0;
15353
15380
  }
15354
- case "attachSegments": {
15355
- for (const segment of action.segments) {
15356
- growPartitionsTo(entry.manager, segment.partitionId);
15357
- const surfaceForms = segment.survivingSurfaceForms;
15358
- const frozen = createSharedFrozenSegment(segment.snapshot, surfaceForms);
15359
- const spellings = surfaceForms === void 0 ? void 0 : NO_REMOVED_SPELLINGS;
15360
- for (const docId of segment.tombstonedDocIds ?? []) frozen.tombstoneDocument(docId, spellings);
15361
- entry.manager.attachFrozenSegment(segment.partitionId, frozen);
15362
- }
15363
- return void 0;
15364
- }
15381
+ case "attachSegments":
15382
+ return attachSegments(entry, action);
15383
+ case "adoptDiskSegments":
15384
+ return import('./module-BUFNZMEU.mjs').then((code) => code.adoptSegmentsOnWorker(entry.manager, action));
15365
15385
  case "compactSegments": {
15366
15386
  const partition = requireComposite(entry, action.indexName, action.partitionId, "compact");
15367
15387
  return mergeFrozenSegments(
@@ -15409,7 +15429,7 @@ function requireLoadedVectorCode() {
15409
15429
  function loadVectorCode() {
15410
15430
  if (loaded2 !== null) return Promise.resolve(loaded2);
15411
15431
  if (loading2 === null) {
15412
- loading2 = import('./module-5J3PEUEB.mjs').then(
15432
+ loading2 = import('./module-BVP4TKZU.mjs').then(
15413
15433
  (code) => {
15414
15434
  loaded2 = code;
15415
15435
  return code;
@@ -15485,7 +15505,7 @@ function closeHeldVectorCopies(held) {
15485
15505
  held.fields.clear();
15486
15506
  held.searchers.clear();
15487
15507
  }
15488
- function yieldToEventLoop2() {
15508
+ function yieldToEventLoop() {
15489
15509
  return new Promise((resolve) => {
15490
15510
  if (typeof setImmediate === "function") setImmediate(resolve);
15491
15511
  else setTimeout(resolve, 0);
@@ -15496,7 +15516,7 @@ async function insertIntoHeldGraph(held, fieldName, handle, ordinals) {
15496
15516
  if (field === void 0) return null;
15497
15517
  for (const ordinal of ordinals) {
15498
15518
  field.view.insertOrdinal(ordinal);
15499
- await yieldToEventLoop2();
15519
+ await yieldToEventLoop();
15500
15520
  }
15501
15521
  return field.view.takeOutcome();
15502
15522
  }
@@ -15804,6 +15824,31 @@ function createDirectExecutor(options) {
15804
15824
  };
15805
15825
  }
15806
15826
 
15827
+ // src/engine/orchestration/turn.ts
15828
+ function afterCurrentTurn(callback) {
15829
+ if (typeof setImmediate === "function") {
15830
+ setImmediate(callback);
15831
+ return;
15832
+ }
15833
+ setTimeout(callback, 0);
15834
+ }
15835
+ function yieldToEventLoop2() {
15836
+ return new Promise((resolve) => afterCurrentTurn(resolve));
15837
+ }
15838
+
15839
+ // src/partitioning/constants.ts
15840
+ var REBALANCE_CHUNK_SIZE = 1e3;
15841
+ var DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE = 1e4;
15842
+ var TEXT_REBUILD_CHUNK_DOCUMENTS = 1e3;
15843
+ var TEXT_REBUILD_SWITCH_BACKLOG_DOCUMENTS = 1e3;
15844
+
15845
+ // src/partitioning/rewrite-spill-of.ts
15846
+ async function rewriteSpillOf(storage, manager, target) {
15847
+ if (storage === void 0 || (target?.config ?? manager.config).textStorage !== "disk") return null;
15848
+ const { openRewriteSpill } = await import('./module-BUFNZMEU.mjs');
15849
+ return openRewriteSpill(storage, manager, target);
15850
+ }
15851
+
15807
15852
  // src/partitioning/text-rebuild.ts
15808
15853
  function textRebuildTargetOf(config, language, analysis) {
15809
15854
  return { config, language, analysis, partialWordAnalysis: resolvePartialWordAnalysis(config, language, analysis) };
@@ -15813,7 +15858,7 @@ function interrupted(pass) {
15813
15858
  }
15814
15859
  async function buildPartition(pass, source, partitionId) {
15815
15860
  const { manager, language, options } = pass;
15816
- const target = createPartitionIndex(partitionId, manager.config.trackPositions ?? true);
15861
+ const target = pass.spill?.newPartition(partitionId) ?? createPartitionIndex(partitionId, manager.config.trackPositions ?? true);
15817
15862
  const docIds = [...source.docIds()];
15818
15863
  target.beginBatch();
15819
15864
  for (let start = 0; start < docIds.length; start += TEXT_REBUILD_CHUNK_DOCUMENTS) {
@@ -15822,7 +15867,12 @@ async function buildPartition(pass, source, partitionId) {
15822
15867
  const document = manager.getRef(docIds[i]);
15823
15868
  if (document !== void 0) target.insert(docIds[i], document, manager.schema, language, options);
15824
15869
  }
15825
- await yieldToEventLoop();
15870
+ if (pass.spill !== null) {
15871
+ target.endBatch();
15872
+ await pass.spill.spillIfFull(target, partitionId);
15873
+ target.beginBatch();
15874
+ }
15875
+ await yieldToEventLoop2();
15826
15876
  if (interrupted(pass)) return null;
15827
15877
  }
15828
15878
  target.endBatch();
@@ -15854,7 +15904,7 @@ async function rebuildOnce(pass) {
15854
15904
  }
15855
15905
  while (pass.log.pendingDocuments > TEXT_REBUILD_SWITCH_BACKLOG_DOCUMENTS) {
15856
15906
  applyWrittenDocuments(pass, built);
15857
- await yieldToEventLoop();
15907
+ await yieldToEventLoop2();
15858
15908
  if (interrupted(pass)) return false;
15859
15909
  }
15860
15910
  const keepCurrent = () => {
@@ -15883,12 +15933,19 @@ async function rebuildTextIndexes(manager, hooks, target) {
15883
15933
  );
15884
15934
  const language = target?.language ?? manager.language;
15885
15935
  const partialWordAnalysis = target === void 0 ? manager.partialWordAnalysis : target.partialWordAnalysis;
15936
+ const spill = await rewriteSpillOf(hooks.rewriteStorage?.(), manager, target);
15886
15937
  while (hooks.isCurrent()) {
15887
15938
  const log = manager.openWriteLog();
15939
+ let swapped = false;
15888
15940
  try {
15889
- if (await rebuildOnce({ manager, log, hooks, language, partialWordAnalysis, options })) return true;
15941
+ swapped = await rebuildOnce({ manager, log, hooks, language, partialWordAnalysis, options, spill });
15942
+ if (swapped) {
15943
+ hooks.onRewriteSpill?.(spill);
15944
+ return true;
15945
+ }
15890
15946
  } finally {
15891
15947
  log.close();
15948
+ if (!swapped) await spill?.abandon();
15892
15949
  }
15893
15950
  }
15894
15951
  return false;
@@ -15922,6 +15979,7 @@ function toError(value) {
15922
15979
  function wireAnalysisRebuild(wiring) {
15923
15980
  return createAnalysisRebuildCoordinator(wiring.config, {
15924
15981
  getManager: wiring.getManager,
15982
+ rewriteStorage: () => wiring.durabilityManager?.rewriteStorage?.(),
15925
15983
  desyncIndex: wiring.desyncIndex,
15926
15984
  resyncIndex: wiring.resyncIndex,
15927
15985
  hasPendingChange: wiring.hasPendingChange,
@@ -15955,8 +16013,13 @@ function createAnalysisRebuildCoordinator(config, deps) {
15955
16013
  const partitionCount = manager.partitionCount;
15956
16014
  deps.emit({ indexName, status: "started", partitionsRebuilt: 0, partitionCount });
15957
16015
  const wasPromoted = deps.desyncIndex(indexName);
16016
+ let spill = null;
15958
16017
  try {
15959
16018
  const rebuilt = await rebuildTextIndexes(manager, {
16019
+ rewriteStorage: () => deps.rewriteStorage?.(),
16020
+ onRewriteSpill(kept) {
16021
+ spill = kept;
16022
+ },
15960
16023
  isCurrent: () => deps.getManager(indexName) === manager && stale.get(indexName) === entry && !entry.abandoned,
15961
16024
  onPartitionBuilt(partitionsBuilt) {
15962
16025
  entry.partitionsRebuilt = partitionsBuilt;
@@ -15971,6 +16034,7 @@ function createAnalysisRebuildCoordinator(config, deps) {
15971
16034
  return;
15972
16035
  }
15973
16036
  await deps.persistAnalysisRevision(indexName);
16037
+ await spill?.removeUnread(manager);
15974
16038
  await deps.resyncIndex(indexName, wasPromoted);
15975
16039
  deps.emit({
15976
16040
  indexName,
@@ -16244,6 +16308,9 @@ function reconstructSchemaFromMetadata(metadata) {
16244
16308
  if (metadata.trackPositions !== void 0) {
16245
16309
  config.trackPositions = metadata.trackPositions;
16246
16310
  }
16311
+ if (metadata.textStorage !== void 0) {
16312
+ config.textStorage = metadata.textStorage;
16313
+ }
16247
16314
  if (metadata.strict !== void 0) {
16248
16315
  config.strict = metadata.strict;
16249
16316
  }
@@ -16465,7 +16532,7 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
16465
16532
  const existing = deps.indexRegistry.get(metadata.indexName);
16466
16533
  if (existing !== void 0) {
16467
16534
  if ((metadata.analysisGeneration ?? 0) > existing.analysisGeneration) {
16468
- const { adoptSavedAnalysis } = await import('./analysis-change-57MIRJWR.mjs');
16535
+ const { adoptSavedAnalysis } = await import('./analysis-change-JPV775LR.mjs');
16469
16536
  adoptSavedAnalysis(deps.executor.getManager(metadata.indexName), existing, metadata);
16470
16537
  }
16471
16538
  if (loadData && deps.executor.getManager(metadata.indexName) === void 0) {
@@ -16498,6 +16565,7 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
16498
16565
  const synonyms = prepared?.encoded ?? null;
16499
16566
  try {
16500
16567
  validateVectorStorage(indexConfig.vectorPromotion, deps.filesystemDurability);
16568
+ validateTextStorage(indexConfig.textStorage, deps.filesystemDurability);
16501
16569
  if (loadData) {
16502
16570
  deps.executor.createIndex(metadata.indexName, indexConfig, language, false, prepared?.lookup ?? null);
16503
16571
  }
@@ -17249,6 +17317,9 @@ function checkHeapAfterRecovery(core) {
17249
17317
 
17250
17318
  // src/core/partition/frozen/share.ts
17251
17319
  function shareFrozenSegment(segment, analysis) {
17320
+ if (segment.paged !== null) {
17321
+ return { snapshot: segment.paged.snapshot, tombstonedDocIds: segment.tombstonedDocIds() };
17322
+ }
17252
17323
  if (segment.sharedSnapshot !== null) {
17253
17324
  const tombstonedDocIds = segment.tombstonedDocIds();
17254
17325
  if (tombstonedDocIds.length === 0 || !segment.removals.settled) {
@@ -17847,7 +17918,7 @@ async function replicateToWorkers(state, action) {
17847
17918
  const loading3 = state.copyLoadBuffers.get(action.indexName);
17848
17919
  if (loading3 !== void 0) {
17849
17920
  queueForCopies(state, detached);
17850
- if (loading3.length % LOAD_BUFFER_YIELD_INTERVAL === 0) await yieldToEventLoop();
17921
+ if (loading3.length % LOAD_BUFFER_YIELD_INTERVAL === 0) await yieldToEventLoop2();
17851
17922
  return;
17852
17923
  }
17853
17924
  const queue = state.replicationQueues.get(action.indexName);
@@ -18055,7 +18126,7 @@ async function compactPartitionSegments(state, indexName, manager, partitionId,
18055
18126
  }
18056
18127
  async function compactIndexSegments(state, indexName, policy) {
18057
18128
  const manager = state.executor.getManager(indexName);
18058
- if (!manager) return;
18129
+ if (!manager || manager.config.textStorage === "disk") return;
18059
18130
  const partitionIds = /* @__PURE__ */ new Set();
18060
18131
  for (let partitionId = 0; partitionId < manager.partitionCount; partitionId++) partitionIds.add(partitionId);
18061
18132
  for (const partitionId of state.segmentLedger.get(indexName)?.keys() ?? []) partitionIds.add(partitionId);
@@ -19506,6 +19577,7 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
19506
19577
  scaleOutBeforeBatch: (indexName, incomingCount) => scaleOutBeforeBatch(state, indexName, incomingCount),
19507
19578
  replicateToWorkers: replicate,
19508
19579
  awaitReplication: (indexName) => awaitReplicationIdle(state, indexName),
19580
+ queueDiskAdoption: (action) => queueForCopies(state, action),
19509
19581
  awaitWrites: (indexName) => awaitWritesApplied(state, indexName),
19510
19582
  awaitCompactions: () => awaitCompactions(state),
19511
19583
  openIndex,
@@ -19590,7 +19662,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
19590
19662
  if (config?.lifecycle !== void 0 && durabilityTier === null) {
19591
19663
  throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index lifecycle settings require durability");
19592
19664
  }
19593
- const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-MTIAN6VT.mjs');
19665
+ const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-SNOWIT6Z.mjs');
19594
19666
  const filesystemDurability = durabilityTier?.kind === "wal";
19595
19667
  const executor = createDirectExecutor({
19596
19668
  vectorWorkerCopies: vectorCopyPolicy,
@@ -19650,7 +19722,6 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
19650
19722
  },
19651
19723
  vectorCopyPolicy
19652
19724
  );
19653
- const rebalancer = createRebalancer();
19654
19725
  const rebalanceRouter = createPartitionRouter();
19655
19726
  const waqMap = /* @__PURE__ */ new Map();
19656
19727
  const rebalanceTargets = /* @__PURE__ */ new Map();
@@ -19710,6 +19781,8 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
19710
19781
  emitCheckpointHeld(report) {
19711
19782
  emitEngineEvent(eventHandlers, "checkpointHeld", report);
19712
19783
  },
19784
+ queueDiskAdoption: orchestrator.queueDiskAdoption,
19785
+ awaitCopiesOfIndex: (indexName) => orchestrator.awaitReplication(indexName),
19713
19786
  publishCheckpointedPartitions: (indexName, partitions) => invalidation?.publishPartitions(indexName, partitions) ?? Promise.resolve(),
19714
19787
  recordCheckpoint(indexName, documentCount, partitionCount) {
19715
19788
  const entry = indexRegistry.get(indexName);
@@ -19774,7 +19847,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
19774
19847
  requireManager,
19775
19848
  resumeChange: async (indexName) => {
19776
19849
  if ((indexRegistry.get(indexName)?.pendingAnalysis ?? null) === null) return;
19777
- await (await import('./analysis-change-entry-ICGRR3IN.mjs')).resumeChangeOf(engineCore, indexName);
19850
+ await (await import('./analysis-change-entry-7HR4LWL3.mjs')).resumeChangeOf(engineCore, indexName);
19778
19851
  },
19779
19852
  onClose: hooks?.onIndexClose,
19780
19853
  onOpen: (indexName) => {
@@ -19804,7 +19877,6 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
19804
19877
  checkHeapPressure: heapPressureNotifier.check
19805
19878
  };
19806
19879
  const rebalanceCtx = {
19807
- rebalancer,
19808
19880
  router: rebalanceRouter,
19809
19881
  waqMap,
19810
19882
  rebalancingIndexes,
@@ -19831,7 +19903,6 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
19831
19903
  shutdownState,
19832
19904
  abortController,
19833
19905
  orchestrator,
19834
- rebalancer,
19835
19906
  rebalanceRouter,
19836
19907
  rebalancingIndexes,
19837
19908
  waqMap,
@@ -19858,7 +19929,7 @@ async function startEngineCore(core, config) {
19858
19929
  checkHeapAfterRecovery(core);
19859
19930
  }
19860
19931
  if (core.invalidation) await core.invalidation.start();
19861
- if (core.durability) await (await import('./analysis-change-entry-ICGRR3IN.mjs')).resumeEveryChange(core);
19932
+ if (core.durability) await (await import('./analysis-change-entry-7HR4LWL3.mjs')).resumeEveryChange(core);
19862
19933
  if (config?.lifecycle === void 0) await core.analysisRebuild.reviewStaleIndexes();
19863
19934
  } catch (err) {
19864
19935
  await core.durability?.manager.shutdown().catch(() => void 0);
@@ -20206,9 +20277,12 @@ function requireWithinFileLimit(byteLength, what) {
20206
20277
  );
20207
20278
  }
20208
20279
  async function packChecksummedEnvelope(payloadChunks, envelopeFormatVersion, what) {
20280
+ requireWithinFileLimit(HEADER_SIZE + payloadByteLength(payloadChunks), what);
20281
+ return packEnvelopeWithChecksum(payloadChunks, envelopeFormatVersion, what, await checksumOfChunks(payloadChunks));
20282
+ }
20283
+ function packEnvelopeWithChecksum(payloadChunks, envelopeFormatVersion, what, checksum) {
20209
20284
  const payloadLength = payloadByteLength(payloadChunks);
20210
20285
  requireWithinFileLimit(HEADER_SIZE + payloadLength, what);
20211
- const checksum = await checksumOfChunks(payloadChunks);
20212
20286
  const [major, minor, patch] = engineVersionParts();
20213
20287
  const header = writeHeader({
20214
20288
  envelopeFormatVersion,
@@ -20370,6 +20444,7 @@ function configurationPayloadOf(config, language, adapterName, analysisRevision,
20370
20444
  ...config.partitions !== void 0 ? { partitionConfig: config.partitions } : {},
20371
20445
  ...config.defaultScoring !== void 0 ? { defaultScoring: config.defaultScoring } : {},
20372
20446
  ...config.trackPositions !== void 0 ? { trackPositions: config.trackPositions } : {},
20447
+ ...config.textStorage !== void 0 ? { textStorage: config.textStorage } : {},
20373
20448
  ...config.strict !== void 0 ? { strict: config.strict } : {},
20374
20449
  ...config.required !== void 0 ? { required: config.required } : {},
20375
20450
  ...config.vectorPromotion !== void 0 ? { vectorPromotion: config.vectorPromotion } : {},
@@ -20458,6 +20533,7 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
20458
20533
  requireSurfaceFormsForPartialFields(schema, indexConfig.surfaceForms);
20459
20534
  validateVectorPromotion(indexConfig.vectorPromotion);
20460
20535
  validateVectorStorage(indexConfig.vectorPromotion, core.filesystemDurability);
20536
+ validateTextStorage(indexConfig.textStorage, core.filesystemDurability);
20461
20537
  if (indexConfig.bm25 !== void 0) {
20462
20538
  validateBM25Params(
20463
20539
  indexConfig.bm25,
@@ -20549,7 +20625,7 @@ async function dropEngineIndex(core, name) {
20549
20625
  const entry = core.requireIndex(name);
20550
20626
  const changeRecorded = () => entry.pendingAnalysis !== null || entry.durableAnalysis !== null;
20551
20627
  const abandonChange = async () => {
20552
- await (await import('./analysis-change-entry-ICGRR3IN.mjs')).abandonChangeOf(core, name);
20628
+ await (await import('./analysis-change-entry-7HR4LWL3.mjs')).abandonChangeOf(core, name);
20553
20629
  };
20554
20630
  if (changeRecorded()) await abandonChange();
20555
20631
  await core.indexState.drop(name, async () => {
@@ -21386,7 +21462,7 @@ function snapshotSegments(segments) {
21386
21462
  }
21387
21463
  return shared2;
21388
21464
  }
21389
- function attachSegments(orchestrator, indexName, segments) {
21465
+ function attachSegments2(orchestrator, indexName, segments) {
21390
21466
  return orchestrator.replicateToWorkers({
21391
21467
  type: "attachSegments",
21392
21468
  indexName,
@@ -21397,11 +21473,11 @@ function attachSegments(orchestrator, indexName, segments) {
21397
21473
  async function broadcastBuiltSegments(orchestrator, indexName, segments, skipClone) {
21398
21474
  const frozen = tryFreezeSegmentsForAttach(segments);
21399
21475
  if (frozen !== null) {
21400
- await attachSegments(orchestrator, indexName, frozen);
21476
+ await attachSegments2(orchestrator, indexName, frozen);
21401
21477
  return;
21402
21478
  }
21403
21479
  const shared2 = snapshotSegments(segments);
21404
- if (shared2.length > 0) await attachSegments(orchestrator, indexName, shared2);
21480
+ if (shared2.length > 0) await attachSegments2(orchestrator, indexName, shared2);
21405
21481
  const plain = payloadSegments(segments);
21406
21482
  if (plain.length === 0) return;
21407
21483
  await orchestrator.replicateToWorkers({
@@ -22157,284 +22233,6 @@ async function updateDocumentBatch(ctx, indexName, updates, options) {
22157
22233
  return { succeeded, failed };
22158
22234
  }
22159
22235
 
22160
- // src/partitioning/write-ahead-queue.ts
22161
- function createWriteAheadQueue(maxSize = DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE) {
22162
- const entries = [];
22163
- const docStates = /* @__PURE__ */ new Map();
22164
- let nextSequence = 1;
22165
- let resolveReplayed = () => void 0;
22166
- const replayed = new Promise((resolve) => {
22167
- resolveReplayed = resolve;
22168
- });
22169
- return {
22170
- push(entry) {
22171
- if (entries.length >= maxSize) {
22172
- throw new NarsilError(
22173
- ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
22174
- `Write-ahead queue is full (${maxSize} entries). Retry after rebalancing completes.`,
22175
- { maxSize, currentSize: entries.length }
22176
- );
22177
- }
22178
- const seq = nextSequence++;
22179
- entries.push({ ...entry, sequenceNumber: seq });
22180
- docStates.set(entry.docId, entry.action === "remove" ? "absent" : "present");
22181
- return seq;
22182
- },
22183
- drain() {
22184
- const result = entries.splice(0);
22185
- result.sort((a, b) => a.sequenceNumber - b.sequenceNumber);
22186
- return result;
22187
- },
22188
- clear() {
22189
- entries.length = 0;
22190
- docStates.clear();
22191
- },
22192
- bufferedDocState(docId) {
22193
- return docStates.get(docId);
22194
- },
22195
- whenReplayed() {
22196
- return replayed;
22197
- },
22198
- markReplayed() {
22199
- resolveReplayed();
22200
- },
22201
- get size() {
22202
- return entries.length;
22203
- },
22204
- get isFull() {
22205
- return entries.length >= maxSize;
22206
- }
22207
- };
22208
- }
22209
-
22210
- // src/engine/rebalance-executor.ts
22211
- function warnHookError(hook, err) {
22212
- console.warn(`${hook} plugin hook error:`, err instanceof Error ? err.message : String(err));
22213
- }
22214
- function applyReplayUpdateVectors(entry, vectorFieldPaths2) {
22215
- const updateVectors = /* @__PURE__ */ new Map();
22216
- for (const fieldPath of vectorFieldPaths2) {
22217
- updateVectors.set(fieldPath, extractVectorFromDoc(entry.document, fieldPath));
22218
- }
22219
- let updateDoc = entry.document;
22220
- if (updateVectors.size > 0) {
22221
- updateDoc = structuredClone(entry.document);
22222
- for (const fieldPath of updateVectors.keys()) {
22223
- deleteNestedValue(updateDoc, fieldPath);
22224
- }
22225
- }
22226
- return { updateDoc, updateVectors };
22227
- }
22228
- async function replayEntry(manager, indexName, entry, ctx) {
22229
- const vecIndexes = manager.getVectorIndexes();
22230
- const vectorFieldPaths2 = ctx.requireIndex(indexName).vectorFieldPaths;
22231
- if (entry.action === "insert" && entry.document) {
22232
- const { partitionDoc, extractedVectors } = prepareDocumentVectors(
22233
- entry.document,
22234
- vectorFieldPaths2
22235
- );
22236
- const overwrote = manager.has(entry.docId);
22237
- const removedSpellings = overwrote ? manager.removedSpellingsOf(entry.docId) : void 0;
22238
- if (overwrote) {
22239
- removeDocumentVectors(entry.docId, vecIndexes);
22240
- manager.remove(entry.docId, removedSpellings);
22241
- }
22242
- manager.insert(entry.docId, partitionDoc);
22243
- if (extractedVectors.size > 0) {
22244
- insertDocumentVectors(entry.docId, extractedVectors, vecIndexes, manager.partitionIdOf(entry.docId));
22245
- for (const fieldPath of extractedVectors.keys()) {
22246
- vecIndexes.get(fieldPath)?.scheduleBuild();
22247
- }
22248
- }
22249
- try {
22250
- await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: entry.docId, document: entry.document });
22251
- } catch (err) {
22252
- warnHookError("afterInsert", err);
22253
- }
22254
- const replicated = {
22255
- indexName,
22256
- docId: entry.docId,
22257
- document: entry.document,
22258
- requestId: `replicate-insert-${entry.docId}`
22259
- };
22260
- await ctx.orchestrator.replicateToWorkers(
22261
- overwrote ? { type: "update", ...replicated, removedSpellings } : { type: "insert", ...replicated }
22262
- );
22263
- return;
22264
- }
22265
- if (entry.action === "remove") {
22266
- const removedSpellings = manager.removedSpellingsOf(entry.docId);
22267
- manager.remove(entry.docId, removedSpellings);
22268
- removeDocumentVectors(entry.docId, vecIndexes);
22269
- try {
22270
- await ctx.pluginRegistry.runHook("afterRemove", { indexName, docId: entry.docId });
22271
- } catch (err) {
22272
- warnHookError("afterRemove", err);
22273
- }
22274
- await ctx.orchestrator.replicateToWorkers({
22275
- type: "remove",
22276
- indexName,
22277
- docId: entry.docId,
22278
- requestId: `replicate-remove-${entry.docId}`,
22279
- removedSpellings
22280
- });
22281
- return;
22282
- }
22283
- if (entry.action === "update" && entry.document) {
22284
- const oldDocument = manager.get(entry.docId);
22285
- const { updateDoc, updateVectors } = applyReplayUpdateVectors(entry, vectorFieldPaths2);
22286
- const removedSpellings = manager.removedSpellingsOf(entry.docId);
22287
- if (manager.has(entry.docId)) {
22288
- manager.update(entry.docId, updateDoc, void 0, removedSpellings);
22289
- } else {
22290
- manager.insert(entry.docId, updateDoc);
22291
- }
22292
- for (const [fieldPath, newVec] of updateVectors) {
22293
- const vecIndex = vecIndexes.get(fieldPath);
22294
- if (!vecIndex) continue;
22295
- if (newVec === null) {
22296
- if (vecIndex.has(entry.docId)) {
22297
- vecIndex.remove(entry.docId);
22298
- }
22299
- } else {
22300
- const oldVec = vecIndex.getVector(entry.docId);
22301
- if (!vectorsEqual(oldVec, newVec)) {
22302
- if (vecIndex.has(entry.docId)) {
22303
- vecIndex.remove(entry.docId);
22304
- }
22305
- vecIndex.insert(entry.docId, newVec);
22306
- vecIndex.scheduleBuild();
22307
- }
22308
- }
22309
- }
22310
- try {
22311
- await ctx.pluginRegistry.runHook("afterUpdate", {
22312
- indexName,
22313
- docId: entry.docId,
22314
- oldDocument: oldDocument ?? {},
22315
- newDocument: entry.document
22316
- });
22317
- } catch (err) {
22318
- warnHookError("afterUpdate", err);
22319
- }
22320
- await ctx.orchestrator.replicateToWorkers({
22321
- type: "update",
22322
- indexName,
22323
- docId: entry.docId,
22324
- document: entry.document,
22325
- requestId: `replicate-update-${entry.docId}`,
22326
- removedSpellings
22327
- });
22328
- }
22329
- }
22330
- async function replayQueued(manager, indexName, waq, ctx) {
22331
- while (true) {
22332
- const entries = waq.drain();
22333
- if (entries.length === 0) break;
22334
- for (const entry of entries) {
22335
- try {
22336
- await replayEntry(manager, indexName, entry, ctx);
22337
- } catch (replayErr) {
22338
- const isMissing = replayErr instanceof NarsilError && replayErr.code === ErrorCodes.DOC_NOT_FOUND;
22339
- if (!isMissing) {
22340
- console.warn(`WAQ replay failed for ${entry.action} on doc "${entry.docId}":`, replayErr);
22341
- }
22342
- }
22343
- }
22344
- }
22345
- }
22346
- async function executeRebalance(manager, indexName, targetPartitionCount, ctx) {
22347
- if (targetPartitionCount <= 0 || !Number.isInteger(targetPartitionCount)) {
22348
- throw new NarsilError(
22349
- ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
22350
- `Target partition count must be a positive integer, got ${targetPartitionCount}`,
22351
- { targetPartitionCount }
22352
- );
22353
- }
22354
- const entry = ctx.requireIndex(indexName);
22355
- const maxPartitions = entry.config.partitions?.maxPartitions;
22356
- if (maxPartitions !== void 0 && targetPartitionCount > maxPartitions) {
22357
- throw new NarsilError(
22358
- ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
22359
- `Target partition count (${targetPartitionCount}) is above the maximum of ${maxPartitions} partitions`,
22360
- { targetPartitionCount, maxPartitions }
22361
- );
22362
- }
22363
- if (ctx.rebalancingIndexes.has(indexName)) {
22364
- throw new NarsilError(
22365
- ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
22366
- `Index "${indexName}" is already being rebalanced`
22367
- );
22368
- }
22369
- const registered4 = ctx.requireIndex(indexName);
22370
- if (registered4.pendingAnalysis !== null || registered4.durableAnalysis !== null) {
22371
- throw new NarsilError(
22372
- ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
22373
- `Index "${indexName}" is changing its analysis settings, so try the rebalance again once the change ends`,
22374
- { indexName }
22375
- );
22376
- }
22377
- if (targetPartitionCount === manager.partitionCount) {
22378
- return;
22379
- }
22380
- const waq = createWriteAheadQueue();
22381
- ctx.waqMap.set(indexName, waq);
22382
- ctx.rebalancingIndexes.add(indexName);
22383
- ctx.rebalanceTargets.set(indexName, targetPartitionCount);
22384
- const wasPromoted = ctx.orchestrator.desyncIndex(indexName);
22385
- try {
22386
- const oldCount = manager.partitionCount;
22387
- await ctx.rebalancer.rebalance(manager, targetPartitionCount, ctx.router, (progress) => {
22388
- if (progress.phase === "complete") {
22389
- emitEngineEvent(ctx.eventHandlers, "partitionRebalance", {
22390
- indexName,
22391
- oldCount,
22392
- newCount: targetPartitionCount
22393
- });
22394
- }
22395
- });
22396
- try {
22397
- await ctx.pluginRegistry.runHook("onPartitionSplit", {
22398
- indexName,
22399
- oldPartitionCount: oldCount,
22400
- newPartitionCount: targetPartitionCount
22401
- });
22402
- } catch (hookErr) {
22403
- console.warn("onPartitionSplit plugin hook failed:", hookErr instanceof Error ? hookErr.message : String(hookErr));
22404
- }
22405
- await replayQueued(manager, indexName, waq, ctx);
22406
- if (ctx.durabilityManager) {
22407
- await ctx.durabilityManager.persistMetadata(indexName);
22408
- await (ctx.durabilityManager.checkpointFromMemory?.(indexName) ?? ctx.durabilityManager.checkpoint(indexName));
22409
- await replayQueued(manager, indexName, waq, ctx);
22410
- }
22411
- await ctx.orchestrator.resyncIndex(indexName, wasPromoted);
22412
- do {
22413
- await replayQueued(manager, indexName, waq, ctx);
22414
- } while (waq.size > 0);
22415
- } catch (err) {
22416
- try {
22417
- await replayQueued(manager, indexName, waq, ctx);
22418
- await ctx.orchestrator.resyncIndex(indexName, wasPromoted);
22419
- do {
22420
- await replayQueued(manager, indexName, waq, ctx);
22421
- } while (waq.size > 0);
22422
- } catch (recoveryErr) {
22423
- console.warn(
22424
- `Rebalance failure recovery for index "${indexName}" did not complete:`,
22425
- recoveryErr instanceof Error ? recoveryErr.message : String(recoveryErr)
22426
- );
22427
- }
22428
- throw err;
22429
- } finally {
22430
- ctx.rebalancingIndexes.delete(indexName);
22431
- ctx.rebalanceTargets.delete(indexName);
22432
- ctx.waqMap.delete(indexName);
22433
- waq.markReplayed();
22434
- }
22435
- ctx.checkWatermark(indexName);
22436
- }
22437
-
22438
22236
  // src/engine/vector-maintenance.ts
22439
22237
  function resolveVectorIndexes(manager, indexName, fieldName) {
22440
22238
  const all = manager.getVectorIndexes();
@@ -22474,6 +22272,7 @@ function getVectorMaintenanceStatus(manager) {
22474
22272
  bufferSize: status.bufferSize,
22475
22273
  building: status.building,
22476
22274
  waitingForMemory: status.waitingForMemory,
22275
+ loadingFromFiles: status.loadingFromFiles === true,
22477
22276
  estimatedCompactMs: status.estimatedCompactMs,
22478
22277
  estimatedOptimizeMs: status.estimatedOptimizeMs
22479
22278
  });
@@ -22611,14 +22410,14 @@ function portabilityOperations(core, config, engine) {
22611
22410
  return {
22612
22411
  snapshot(indexName) {
22613
22412
  return withOpenIndex2(indexName, async () => {
22614
- const { createSnapshot } = await import('./snapshot-YBJ5MF2M.mjs');
22413
+ const { createSnapshot } = await import('./snapshot-QXBKHJAG.mjs');
22615
22414
  return createSnapshot(snapshotDepsOf(indexName));
22616
22415
  });
22617
22416
  },
22618
22417
  async restore(indexName, data) {
22619
22418
  core.guardShutdown();
22620
22419
  validateIndexName(indexName);
22621
- const restored = import('./restore-queue-DISFECE4.mjs').then(
22420
+ const restored = import('./restore-queue-ONYNYUDE.mjs').then(
22622
22421
  ({ queueRestore }) => queueRestore(core, restoreHost, restoresInFlight, indexName, data)
22623
22422
  );
22624
22423
  await emitWriteEventAfter(core, indexName, restored);
@@ -22627,8 +22426,8 @@ function portabilityOperations(core, config, engine) {
22627
22426
  exportDocuments(indexName) {
22628
22427
  return withOpenIndex2(indexName, async () => {
22629
22428
  const [{ checkpointForArchive }, { documentExportStream }] = await Promise.all([
22630
- import('./snapshot-YBJ5MF2M.mjs'),
22631
- import('./write-RTWI5MXL.mjs')
22429
+ import('./snapshot-QXBKHJAG.mjs'),
22430
+ import('./write-WHFBDWQS.mjs')
22632
22431
  ]);
22633
22432
  const header = indexConfigurationOf(core.requireIndex(indexName));
22634
22433
  return documentExportStream(header, await checkpointForArchive(snapshotDepsOf(indexName)));
@@ -22636,7 +22435,7 @@ function portabilityOperations(core, config, engine) {
22636
22435
  },
22637
22436
  importDocumentExport(indexName, data) {
22638
22437
  core.guardShutdown();
22639
- const imported = import('./import-Q7MXD6BS.mjs').then(
22438
+ const imported = import('./import-TLB2A5LJ.mjs').then(
22640
22439
  ({ readDocumentExport }) => readDocumentExport(engine(), indexName, data)
22641
22440
  );
22642
22441
  return emitWriteEventAfter(core, indexName, imported);
@@ -24302,7 +24101,7 @@ function pickDisplaySurface(surfaceOccurrences) {
24302
24101
  // src/narsil/reads.ts
24303
24102
  async function enterRead(core, indexName) {
24304
24103
  core.guardShutdown();
24305
- if (core.rebalancingIndexes.has(indexName)) await yieldToEventLoop();
24104
+ if (core.rebalancingIndexes.has(indexName)) await yieldToEventLoop2();
24306
24105
  }
24307
24106
  async function readUnderOneAnalysis(core, indexName, read) {
24308
24107
  const manager = core.requireManager(indexName);
@@ -24507,8 +24306,11 @@ function createNarsilFromCore(core, config) {
24507
24306
  release();
24508
24307
  }
24509
24308
  }
24309
+ const rebalancesStarting = /* @__PURE__ */ new Map();
24510
24310
  function withOpenIndexWrite(indexName, action) {
24511
- return emitWriteEventAfter(core, indexName, withOpenIndex2(indexName, action));
24311
+ const starting = rebalancesStarting.get(indexName);
24312
+ const written = starting === void 0 ? withOpenIndex2(indexName, action) : starting.then(() => withOpenIndex2(indexName, action));
24313
+ return emitWriteEventAfter(core, indexName, written);
24512
24314
  }
24513
24315
  const narsil = {
24514
24316
  createIndex(name, indexConfig) {
@@ -24636,16 +24438,17 @@ function createNarsilFromCore(core, config) {
24636
24438
  const entry = requireIndex(indexName);
24637
24439
  await withOpenIndexWrite(indexName, async () => {
24638
24440
  if (entry.pendingAnalysis === null) return core.analysisRebuild.rebuild(indexName);
24639
- return (await import('./analysis-change-57MIRJWR.mjs')).resumeAnalysisChange(core, indexName);
24441
+ return (await import('./analysis-change-JPV775LR.mjs')).resumeAnalysisChange(core, indexName);
24640
24442
  });
24641
24443
  },
24642
24444
  async changeAnalysis(indexName, change) {
24643
24445
  guardShutdown();
24644
24446
  requireIndex(indexName);
24645
- await withOpenIndexWrite(
24447
+ const changed = withOpenIndex2(
24646
24448
  indexName,
24647
- async () => (await import('./analysis-change-57MIRJWR.mjs')).changeIndexAnalysis(core, indexName, change)
24449
+ async () => (await import('./analysis-change-JPV775LR.mjs')).changeIndexAnalysis(core, indexName, change)
24648
24450
  );
24451
+ await emitWriteEventAfter(core, indexName, changed);
24649
24452
  },
24650
24453
  ...portabilityOperations(core, config, () => narsil),
24651
24454
  ...synonymOperations(core),
@@ -24664,15 +24467,29 @@ function createNarsilFromCore(core, config) {
24664
24467
  await executor.execute({ type: "clear", indexName, requestId: indexName });
24665
24468
  await orchestrator.replicateToWorkers({ type: "clear", indexName, requestId: `replicate-clear-${indexName}` });
24666
24469
  core.watermarkNotifier.forget(indexName);
24470
+ const durable = core.durability?.manager;
24471
+ await (durable?.checkpointFromMemory?.(indexName) ?? durable?.checkpoint(indexName));
24667
24472
  });
24668
24473
  },
24669
24474
  async rebalance(indexName, targetPartitionCount) {
24670
24475
  guardShutdown();
24671
24476
  requireIndex(indexName);
24672
- return withOpenIndexWrite(
24673
- indexName,
24674
- () => executeRebalance(requireManager(indexName), indexName, targetPartitionCount, rebalanceCtx)
24675
- );
24477
+ let writesMayFollow = () => void 0;
24478
+ const started = new Promise((resolve) => {
24479
+ writesMayFollow = resolve;
24480
+ });
24481
+ function releaseWrites() {
24482
+ writesMayFollow();
24483
+ if (rebalancesStarting.get(indexName) === started) rebalancesStarting.delete(indexName);
24484
+ }
24485
+ rebalancesStarting.set(indexName, started);
24486
+ const rebalanced = withOpenIndex2(indexName, async () => {
24487
+ const { executeRebalance } = await import('./analysis-change-JPV775LR.mjs');
24488
+ const running = executeRebalance(requireManager(indexName), indexName, targetPartitionCount, rebalanceCtx);
24489
+ releaseWrites();
24490
+ return running;
24491
+ });
24492
+ return emitWriteEventAfter(core, indexName, rebalanced).finally(releaseWrites);
24676
24493
  },
24677
24494
  async updatePartitionConfig(indexName, partitionConfig) {
24678
24495
  guardShutdown();
@@ -24791,4 +24608,4 @@ async function createNarsil(config) {
24791
24608
  // src/index.ts
24792
24609
  var VERSION = "0.1.0";
24793
24610
 
24794
- export { ARRAY_POSITION_GAP, DEFAULT_BM25_B, DEFAULT_BM25_K1, HEADER_SIZE, MAX_DOCUMENT_NESTING_DEPTH, VECTOR_PATTERN, VERSION, arrayPositionsAreStale, buildSegmentPayload, clearNormalizationCache, configureNormalizationCache, crc32, createFrozenSegment, createNarsil, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, decodeEveryStoredDocument, decodeIndexConfiguration, decodeMapWithoutFields, decodeStoredDocument, emitEngineEvent, encodeDocument, encodeIndexConfiguration, engineVersionParts, extractVectorFieldsFromSchema, flattenSchema, fnv1a, freezeSegmentShared, getLanguage, getNestedValue, getNormalizationCacheSize, getStopWords, getTokenizer, hasNrslMagic, hasStopWords, hasTokenizer, insertDocumentVectors, isCompositePartition, isQuantizationMode, isStorageMode, loadVectorCodeFor, mergeSegmentContents, messagePackUpperBoundBytes, openChecksummedEnvelope, packChecksummedEnvelope, parseVectorDimension, partialFieldsOf, partitionStateOf, prepareConfiguredSynonyms, prepareDocumentVectors, prepareStoredSynonyms, raiseEarlierReleaseGraphSetting, readHeader, readHeapStatistics, readResidentMemory, rebuildTextIndexes, reconstructSchemaFromMetadata, registerLanguage, registerPartialWordSearch, registerPatternSearch, registerStopWords, registerSynonyms, registerTokenizer, removeDocumentVectors, requireConfigurationShape, requireConfigurationWithinSizeLimit, requireSurfaceFormsForPartialFields, requireValidFieldTypes, resetNormalizationCache, resolveIndexAnalysis, resolvePartialWordAnalysis, resolvePartitionInsertOptions, rulesOfStoredSynonyms, segmentArraysOf, selectionKeeps, shareSegmentArrays, storedArrayPositionGap, survivorDocumentsOf, textRebuildTargetOf, validateEmbeddingConfig, validatePatternValueLimit, validateRequiredFieldsInSchema, validateSchema, validateVectorPromotion, validateVectorStorage, writeHeader };
24611
+ export { ARRAY_POSITION_GAP, DEFAULT_BM25_B, DEFAULT_BM25_K1, DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE, HEADER_SIZE, MAX_DOCUMENT_NESTING_DEPTH, REBALANCE_CHUNK_SIZE, VECTOR_PATTERN, VERSION, arrayPositionsAreStale, assemblePagedFrozenSegment, buildPartialWordTableFrom, buildSegmentPayload, checksumOfChunks, clearNormalizationCache, configureNormalizationCache, crc32, crc32Final, crc32Init, crc32Update, createCompositePartition, createEncodedDocumentSource, createFrozenSegment, createNarsil, createPartitionIndex, createPartitionManager, createPartitionRouter, createRequestId, createSharedFrozenSegment, decodeEveryStoredDocument, decodeIndexConfiguration, decodeMapWithoutFields, decodeStoredDocument, decodeSurfaceTable, deleteNestedValue, emitEngineEvent, encodeDocument, encodeIndexConfiguration, encodeSurfaceTable, endOfValue, engineVersionParts, extractVectorFieldsFromSchema, extractVectorFromDoc, fieldIndexRemapOf, flattenSchema, fnv1a, freezeSegmentShared, generateId, getLanguage, getNestedValue, getNormalizationCacheSize, getStopWords, getTokenizer, hasNrslMagic, hasStopWords, hasTokenizer, holdsADiscardedField, insertDocumentVectors, isCompositePartition, isQuantizationMode, isStorageMode, loadVectorCodeFor, mergeDocFrequencies, mergeFieldIndexes2 as mergeFieldIndexes, mergeFieldLengths, mergePartialWords, mergeSegmentContents, mergeSurfaceForms2 as mergeSurfaceForms, messagePackUpperBoundBytes, openChecksummedEnvelope, packChecksummedEnvelope, packEnvelopeWithChecksum, parseVectorDimension, partialFieldsOf, partitionStateOf, prepareConfiguredSynonyms, prepareDocumentVectors, prepareStoredSynonyms, raiseEarlierReleaseGraphSetting, readHeader, readHeapStatistics, readResidentMemory, rebuildTextIndexes, reconstructSchemaFromMetadata, registerLanguage, registerPartialWordSearch, registerPatternSearch, registerStopWords, registerSynonyms, registerTokenizer, removeDocumentVectors, requireConfigurationShape, requireConfigurationWithinSizeLimit, requireSurfaceFormsForPartialFields, requireValidFieldTypes, requireWithinFileLimit, resetNormalizationCache, resolveIndexAnalysis, resolvePartialWordAnalysis, resolvePartitionInsertOptions, rewriteSpillOf, rulesOfStoredSynonyms, scanPostingBlocks, segmentArraysOf, selectionKeeps, shareSegmentArrays, storedArrayPositionGap, survivorDocumentsOf, textRebuildTargetOf, validateEmbeddingConfig, validatePatternValueLimit, validateRequiredFieldsInSchema, validateSchema, validateTextStorage, validateVectorPromotion, validateVectorStorage, vectorsEqual, wrapFrozenTokenTable, writeHeader, yieldToEventLoop2 as yieldToEventLoop };