@delali/narsil 0.3.1-next.131 → 0.3.1-next.140

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. package/dist/adapters/filesystem.mjs +1 -1
  2. package/dist/adapters/indexeddb.mjs +1 -1
  3. package/dist/analysis-change-JN23AZQP.mjs +43 -0
  4. package/dist/analysis-change-entry-5JSIMEKG.mjs +1 -0
  5. package/dist/browser/analysis-change-57MIRJWR.mjs +499 -0
  6. package/dist/browser/analysis-change-entry-ICGRR3IN.mjs +1 -0
  7. package/dist/browser/{chunk-UDGB2MWT.mjs → chunk-35B42UFM.mjs} +4 -0
  8. package/dist/browser/{chunk-VBBLTD7O.mjs → chunk-3EYDURFD.mjs} +39 -27
  9. package/dist/browser/{chunk-AIIJDWMG.mjs → chunk-4MFH226Q.mjs} +1 -1
  10. package/dist/browser/{chunk-D7WQF6DV.mjs → chunk-5LYWZJE2.mjs} +2 -2
  11. package/dist/browser/chunk-67KSBUYR.mjs +173 -0
  12. package/dist/browser/{chunk-M5UAAXGU.mjs → chunk-6BSB4N6B.mjs} +2 -2
  13. package/dist/browser/{chunk-NSCSR6MA.mjs → chunk-CGVBGHBX.mjs} +81 -6
  14. package/dist/browser/chunk-E5SFTOQR.mjs +21 -0
  15. package/dist/browser/{chunk-VEGHFP2P.mjs → chunk-MZI3TEOW.mjs} +4 -2
  16. package/dist/browser/chunk-OIZS5KIX.mjs +6 -0
  17. package/dist/browser/chunk-QHNNULPW.mjs +265 -0
  18. package/dist/browser/chunk-XVMOQQ34.mjs +384 -0
  19. package/dist/browser/{chunk-2Y3CO4EQ.mjs → chunk-ZAIZQOTM.mjs} +256 -777
  20. package/dist/browser/{durability-wiring-FNZSRH4N.mjs → durability-wiring-MTIAN6VT.mjs} +145 -50
  21. package/dist/browser/import-Q7MXD6BS.mjs +8 -0
  22. package/dist/browser/{module-4HIMOQM5.mjs → module-5J3PEUEB.mjs} +7 -6
  23. package/dist/browser/{phrase-module-WESYVU7K.mjs → phrase-module-HGVBO24W.mjs} +1 -1
  24. package/dist/{restore-queue-JAMNRC6T.mjs → browser/restore-queue-DISFECE4.mjs} +8 -2
  25. package/dist/browser/{runtime-3X7ZJ5T3.mjs → runtime-VEUH674B.mjs} +2 -2
  26. package/dist/browser/{snapshot-V7VF6BN6.mjs → snapshot-YBJ5MF2M.mjs} +36 -167
  27. package/dist/browser/{write-YIC2MKRY.mjs → write-RTWI5MXL.mjs} +8 -6
  28. package/dist/{capabilities-B6WxrWxt.d.ts → capabilities-DXAmysqJ.d.ts} +10 -3
  29. package/dist/{chunk-NMXRXFGZ.mjs → chunk-4GCN34JV.mjs} +2 -2
  30. package/dist/{chunk-SOHMMNCH.mjs → chunk-6VHWXAOI.mjs} +4 -4
  31. package/dist/{chunk-2C3WXBQS.mjs → chunk-74XB2SXI.mjs} +1 -1
  32. package/dist/{chunk-OGLV4I3L.mjs → chunk-B5I3IL4H.mjs} +12 -7
  33. package/dist/chunk-BDIDUW2B.mjs +387 -0
  34. package/dist/{chunk-5LX6MOIB.mjs → chunk-C3YDYX6C.mjs} +1 -1
  35. package/dist/{chunk-Y4BVTQ53.mjs → chunk-CCCGEPJM.mjs} +2 -2
  36. package/dist/{chunk-2JWNJSNE.mjs → chunk-CPCGEIOM.mjs} +11 -8
  37. package/dist/chunk-CZQ33DLW.mjs +78 -0
  38. package/dist/{chunk-4GDLFDPZ.mjs → chunk-DD2673TX.mjs} +3 -2
  39. package/dist/{chunk-RNSC4NFI.mjs → chunk-EEBCDP36.mjs} +3 -1
  40. package/dist/{chunk-HCGJKDZV.mjs → chunk-FFVLPCLH.mjs} +1 -1
  41. package/dist/{chunk-WVRDRLIC.mjs → chunk-FKARMUWH.mjs} +6 -6
  42. package/dist/{chunk-PATMDIMN.mjs → chunk-HJJWY6PT.mjs} +1 -1
  43. package/dist/chunk-HMDT3I63.mjs +21 -0
  44. package/dist/{chunk-CZ7ISLXA.mjs → chunk-HNHCNTVG.mjs} +4 -0
  45. package/dist/{chunk-RWESFSJI.mjs → chunk-IBATGS3J.mjs} +87 -61
  46. package/dist/{chunk-Y2U5VVWX.mjs → chunk-ICW45TTU.mjs} +1 -1
  47. package/dist/{chunk-T4OFG4IQ.mjs → chunk-IP4HJOGC.mjs} +90 -16
  48. package/dist/{chunk-EUCX5K74.mjs → chunk-J3EHATEZ.mjs} +3 -3
  49. package/dist/{chunk-X5CNUKD5.mjs → chunk-K3AQOZVM.mjs} +37 -20
  50. package/dist/chunk-LK3MOTXF.mjs +174 -0
  51. package/dist/{chunk-7OIQ2SIO.mjs → chunk-LLDFIKGP.mjs} +4 -4
  52. package/dist/{chunk-K77FJXKS.mjs → chunk-MTQTX4BP.mjs} +81 -8
  53. package/dist/{chunk-ZR4VM4A6.mjs → chunk-N37VMC62.mjs} +1 -18
  54. package/dist/{chunk-2WBY4Y4M.mjs → chunk-NOT5QOHU.mjs} +1 -1
  55. package/dist/{chunk-D5TFRB5G.mjs → chunk-NPNK57CA.mjs} +1 -1
  56. package/dist/{chunk-6SSVIW7Y.mjs → chunk-O4LJQI4G.mjs} +31 -5
  57. package/dist/{chunk-JJ6EUMXB.mjs → chunk-PNNVTKMT.mjs} +2 -2
  58. package/dist/{chunk-2SEL6SBS.mjs → chunk-RKAVHNRW.mjs} +2 -2
  59. package/dist/{chunk-RCMNL7RI.mjs → chunk-SBMAWTRG.mjs} +41 -4
  60. package/dist/chunk-TCWQJL7O.mjs +188 -0
  61. package/dist/{chunk-2MKX2SCO.mjs → chunk-U4X4AWKL.mjs} +7 -7
  62. package/dist/{chunk-PU6SKH7O.mjs → chunk-UBXJQJII.mjs} +2 -2
  63. package/dist/chunk-UEV53BBO.mjs +480 -0
  64. package/dist/{chunk-NNO7CO3A.mjs → chunk-UIKN2OTQ.mjs} +1 -1
  65. package/dist/{chunk-F6AAXI7T.mjs → chunk-VSUYTPME.mjs} +3 -3
  66. package/dist/{chunk-TGHNXN7W.mjs → chunk-W2XDK66Z.mjs} +180 -541
  67. package/dist/{chunk-7X5JST5L.mjs → chunk-WNBRW4Q2.mjs} +16 -2
  68. package/dist/{chunk-6M6SDH3S.mjs → chunk-XMQNOD5V.mjs} +8 -8
  69. package/dist/{chunk-SZGRYEIC.mjs → chunk-XS2VM7UE.mjs} +3 -3
  70. package/dist/{chunk-JT3YZGCE.mjs → chunk-XXKWEG53.mjs} +3 -188
  71. package/dist/{chunk-5W7AEVTH.mjs → chunk-ZMBFWQXN.mjs} +2 -2
  72. package/dist/{chunk-7XY6VD7B.mjs → chunk-ZSWHPKGM.mjs} +1 -1
  73. package/dist/{client-gNAfaHcE.d.ts → client-DApBeLbk.d.ts} +17 -2
  74. package/dist/client.d.ts +5 -5
  75. package/dist/client.mjs +11 -1
  76. package/dist/{config-D1cDv3sE.d.ts → config-B0H-vp4U.d.ts} +4 -0
  77. package/dist/{config-CmYJO34X.d.ts → config-Brysl9NO.d.ts} +1 -1
  78. package/dist/{direct-executor-HQ4QVDX7.mjs → direct-executor-J5UDLPL3.mjs} +14 -14
  79. package/dist/distribution/coordinator/etcd.mjs +1 -1
  80. package/dist/distribution/transport/grpc.mjs +1 -1
  81. package/dist/distribution.d.ts +21 -5
  82. package/dist/distribution.mjs +882 -192
  83. package/dist/{durability-wiring-S7OX2MAI.mjs → durability-wiring-AYOTAPNQ.mjs} +165 -70
  84. package/dist/embeddings/openai.mjs +2 -2
  85. package/dist/{engine-m_Ddr1bz.d.ts → engine-CuLMbH1-.d.ts} +63 -6
  86. package/dist/import-QRZAQ5RH.mjs +43 -0
  87. package/dist/index.browser.mjs +4 -4
  88. package/dist/index.d.ts +4 -4
  89. package/dist/index.mjs +20 -20
  90. package/dist/{long-query-thread-BSL25PD6.mjs → long-query-thread-U7K5A72G.mjs} +14 -14
  91. package/dist/{module-2E4XYSGT.mjs → module-L2OZO5PQ.mjs} +11 -11
  92. package/dist/partial-words.mjs +5 -5
  93. package/dist/pattern.mjs +3 -3
  94. package/dist/persistence/durability/checkpoint-worker.d.ts +11 -0
  95. package/dist/persistence/durability/checkpoint-worker.mjs +23 -23
  96. package/dist/{phrase-module-JERF2IDG.mjs → phrase-module-5QHBD2VJ.mjs} +1 -1
  97. package/dist/react.d.ts +7 -7
  98. package/dist/react.mjs +4 -0
  99. package/dist/{rerank-DMMT4FZA.mjs → rerank-KTLC3E6N.mjs} +5 -5
  100. package/dist/rerankers/http.mjs +2 -2
  101. package/dist/{browser/restore-queue-GEZ54CA5.mjs → restore-queue-RYSRRFS4.mjs} +8 -2
  102. package/dist/{runtime-7MLQ7J64.mjs → runtime-DYV75DCB.mjs} +6 -6
  103. package/dist/server.d.ts +10 -8
  104. package/dist/server.mjs +51 -19
  105. package/dist/{snapshot-DOQLGGEX.mjs → snapshot-BTRZ3MZK.mjs} +59 -190
  106. package/dist/synonyms.mjs +6 -4
  107. package/dist/{tasks-Co8HfRD7.d.ts → tasks-DjGjK70Y.d.ts} +22 -6
  108. package/dist/{thread-SKO7CN3R.mjs → thread-USB3GOPY.mjs} +13 -13
  109. package/dist/vector/search-worker.mjs +5 -5
  110. package/dist/workers/entry.mjs +7 -7
  111. package/dist/{write-ZKR7SNRT.mjs → write-GHTA56OW.mjs} +25 -22
  112. package/openapi.json +101 -6
  113. package/package.json +7 -7
@@ -1,8 +1,8 @@
1
1
  import { GrowableUint32, SEGMENT_TOKEN_TABLE_START, SEGMENT_POSTING_LOG_START, SEGMENT_POSITION_POOL_START, SEGMENT_FIELD_SCRATCH_START, LOOKUP_KEY_SCRATCH_BYTES, bitsetHas, bitsetSet, createBitSet, bitsetIsEmpty, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY, PRUNING_REJECTION_SLACK, bitsetOrInto, bitsetAndNotInto, bitsetAndInto, SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD, TARGET_BLOCK_ENTRIES, SORT_COLUMN_REBUILD_FRACTION_SHIFT, bitsetFromSet } from './chunk-26A5JFT2.mjs';
2
- import { MAX_DOCUMENT_ENCODING_DEPTH, singleFoldTable, appendFoldedCodePoint, compareCodePoints, EARLIER_RELEASE_MIN_M, MIN_M, MAX_M, DOCUMENT_ENCODER_RETAINED_BUFFER_BYTES, MAX_FILE_BYTES, detectRuntime, splitWorkerBudget, resolveWorkerCount, CHECKSUM_YIELD_CHUNK_BYTES, resolveWorkerEntry, missingWorkerEntry, DEFAULT_REQUEST_TIMEOUT_MS, toReducedSortValue, resolveRequestThreadCount, defaultSortMode, holdsUnpairedSurrogate, compareComparableValues, readSortField, exceedsCodePoints, toComparableSortValue, SORT_VALUE_MAX_CODE_POINTS, compareComparableKeys, truncateSortString, sortModeOf, isSortMode, SORT_MODES, SHUTDOWN_TIMEOUT_MS, MIN_OVERSAMPLE, MAX_RENUMBERED_SEARCH_ATTEMPTS, compareSortValues, DEFAULT_BACKPRESSURE_LIMIT, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, codePointCount, noteThreadSlotGone, threadSlotOfWorker } from './chunk-AIIJDWMG.mjs';
2
+ import { MAX_DOCUMENT_ENCODING_DEPTH, singleFoldTable, appendFoldedCodePoint, compareCodePoints, EARLIER_RELEASE_MIN_M, MIN_M, MAX_M, DOCUMENT_ENCODER_RETAINED_BUFFER_BYTES, MAX_FILE_BYTES, detectRuntime, splitWorkerBudget, resolveWorkerCount, CHECKSUM_YIELD_CHUNK_BYTES, resolveWorkerEntry, missingWorkerEntry, DEFAULT_REQUEST_TIMEOUT_MS, toReducedSortValue, resolveRequestThreadCount, defaultSortMode, holdsUnpairedSurrogate, compareComparableValues, readSortField, exceedsCodePoints, toComparableSortValue, SORT_VALUE_MAX_CODE_POINTS, compareComparableKeys, truncateSortString, sortModeOf, isSortMode, SORT_MODES, SHUTDOWN_TIMEOUT_MS, MIN_OVERSAMPLE, MAX_RENUMBERED_SEARCH_ATTEMPTS, compareSortValues, DEFAULT_BACKPRESSURE_LIMIT, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, codePointCount, noteThreadSlotGone, threadSlotOfWorker } from './chunk-4MFH226Q.mjs';
3
3
  import { DEFAULT_MAX_RERANK_WINDOW, DEFAULT_RERANK_TIMEOUT_MS } from './chunk-JPTQZWJQ.mjs';
4
- import { ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, STATISTICS_BROADCAST_INTERVAL_MS, MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, MAX_SUGGEST_LIMIT, MAX_INDEX_CONFIGURATION_BYTES, FOREIGN_STATS_TTL_MS, HEAP_PRESSURE_WARNING_FRACTION, MAX_IMPORT_REFUSALS_REPORTED, DEFAULT_SUGGEST_LIMIT, DEFAULT_LIMIT, DEFAULT_OFFSET, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION, BATCH_CHUNK_SIZE, MAX_DOC_ID_LENGTH } from './chunk-VEGHFP2P.mjs';
5
- import { ErrorCodes, NarsilError, createNarsilError, DEFAULT_FUZZY_PREFIX_LENGTH, MAX_CURSOR_LENGTH, MAX_CURSOR_ANCHOR_CODE_POINTS, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, BM25_DISPLAYED_NAME_LENGTH, RESULT_WINDOW, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, DEFAULT_PAGE_SIZE, DEFAULT_MAX_PER_GROUP, PREFIX_MAX_EXPANSIONS, MAX_TOLERANCE, MAX_PREFIX_LENGTH } from './chunk-UDGB2MWT.mjs';
4
+ import { MAX_INDEX_CONFIGURATION_BYTES, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, STATISTICS_BROADCAST_INTERVAL_MS, MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_SUGGEST_LIMIT, FOREIGN_STATS_TTL_MS, HEAP_PRESSURE_WARNING_FRACTION, DEFAULT_SUGGEST_LIMIT, DEFAULT_LIMIT, DEFAULT_OFFSET, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION, BATCH_CHUNK_SIZE, MAX_DOC_ID_LENGTH } from './chunk-MZI3TEOW.mjs';
5
+ import { ErrorCodes, NarsilError, createNarsilError, DEFAULT_FUZZY_PREFIX_LENGTH, MAX_CURSOR_LENGTH, MAX_CURSOR_ANCHOR_CODE_POINTS, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, BM25_DISPLAYED_NAME_LENGTH, RESULT_WINDOW, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, DEFAULT_PAGE_SIZE, DEFAULT_MAX_PER_GROUP, PREFIX_MAX_EXPANSIONS, MAX_TOLERANCE, MAX_PREFIX_LENGTH } from './chunk-35B42UFM.mjs';
6
6
  import { Encoder, Decoder, decode, encode } from '@msgpack/msgpack';
7
7
 
8
8
  // src/analysis/registry.ts
@@ -7538,7 +7538,7 @@ function requireLoadedPhraseCode() {
7538
7538
  }
7539
7539
  function loadPhraseCode() {
7540
7540
  if (loading === null) {
7541
- loading = import('./phrase-module-WESYVU7K.mjs').then(
7541
+ loading = import('./phrase-module-HGVBO24W.mjs').then(
7542
7542
  (code) => {
7543
7543
  loaded = code;
7544
7544
  return code;
@@ -11202,6 +11202,32 @@ function createCapacityChecks(source) {
11202
11202
  };
11203
11203
  }
11204
11204
 
11205
+ // src/partitioning/manager/doc-locations.ts
11206
+ function rebuildDocLocations(locations, partitions) {
11207
+ locations.clear();
11208
+ for (let i = 0; i < partitions.length; i++) {
11209
+ for (const docId of partitions[i].docIds()) {
11210
+ locations.set(docId, i);
11211
+ }
11212
+ }
11213
+ }
11214
+ function recordSegmentDocLocations(locations, partitionId, segment) {
11215
+ for (const internalId of segment.docStore.allInternalIds()) {
11216
+ const docId = segment.docStore.getExternalId(internalId);
11217
+ if (docId !== void 0) locations.set(docId, partitionId);
11218
+ }
11219
+ }
11220
+ function rebuildPartitionDocLocations(locations, partitionId, partition) {
11221
+ for (const [docId, pid] of locations) {
11222
+ if (pid === partitionId) {
11223
+ locations.delete(docId);
11224
+ }
11225
+ }
11226
+ for (const docId of partition.docIds()) {
11227
+ locations.set(docId, partitionId);
11228
+ }
11229
+ }
11230
+
11205
11231
  // src/partitioning/manager/nested-values.ts
11206
11232
  function setNestedValue(obj, path, value) {
11207
11233
  if (!path.includes(".")) {
@@ -11316,13 +11342,15 @@ function createWriteLogRecorder() {
11316
11342
  }
11317
11343
 
11318
11344
  // src/partitioning/manager/index.ts
11319
- function createPartitionManager(indexName, config, language, router, initialPartitionCount, vectorIndexes) {
11345
+ function createPartitionManager(indexName, initialConfig, initialLanguage, router, initialPartitionCount, vectorIndexes) {
11346
+ let config = initialConfig;
11347
+ let language = initialLanguage;
11320
11348
  requirePatternSearchFor(config.schema);
11321
11349
  requirePartialWordSearchFor(config.schema);
11322
11350
  const count = initialPartitionCount ?? 1;
11323
- const analysis = resolveIndexAnalysis(config);
11324
- const partialWordAnalysis = resolvePartialWordAnalysis(config, language, analysis);
11325
- const textAnalysis = { language, options: resolvePartitionInsertOptions(config, analysis) };
11351
+ let analysis = resolveIndexAnalysis(config);
11352
+ let partialWordAnalysis = resolvePartialWordAnalysis(config, language, analysis);
11353
+ let textAnalysis = { language, options: resolvePartitionInsertOptions(config, analysis) };
11326
11354
  const trackPositions = config.trackPositions ?? true;
11327
11355
  const vecIndexes = vectorIndexes ?? /* @__PURE__ */ new Map();
11328
11356
  let partitions = [];
@@ -11356,30 +11384,6 @@ function createPartitionManager(indexName, config, language, router, initialPart
11356
11384
  partitions[partitionId] = composite;
11357
11385
  return composite;
11358
11386
  }
11359
- function rebuildDocPartitionMap() {
11360
- docPartitionMap.clear();
11361
- for (let i = 0; i < partitions.length; i++) {
11362
- for (const docId of partitions[i].docIds()) {
11363
- docPartitionMap.set(docId, i);
11364
- }
11365
- }
11366
- }
11367
- function registerSegmentDocuments(partitionId, segment) {
11368
- for (const internalId of segment.docStore.allInternalIds()) {
11369
- const docId = segment.docStore.getExternalId(internalId);
11370
- if (docId !== void 0) docPartitionMap.set(docId, partitionId);
11371
- }
11372
- }
11373
- function rebuildDocPartitionMapForPartition(partitionId) {
11374
- for (const [docId, pid] of docPartitionMap) {
11375
- if (pid === partitionId) {
11376
- docPartitionMap.delete(docId);
11377
- }
11378
- }
11379
- for (const docId of partitions[partitionId].docIds()) {
11380
- docPartitionMap.set(docId, partitionId);
11381
- }
11382
- }
11383
11387
  const manager = {
11384
11388
  get partitionCount() {
11385
11389
  return partitions.length;
@@ -11419,7 +11423,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
11419
11423
  setPartitions(newPartitions) {
11420
11424
  writeLog.partitionsReplaced();
11421
11425
  partitions = [...newPartitions];
11422
- rebuildDocPartitionMap();
11426
+ rebuildDocLocations(docPartitionMap, partitions);
11423
11427
  },
11424
11428
  openWriteLog() {
11425
11429
  return writeLog.open();
@@ -11434,6 +11438,14 @@ function createPartitionManager(indexName, config, language, router, initialPart
11434
11438
  }
11435
11439
  partitions = [...rebuilt];
11436
11440
  },
11441
+ adoptAnalysis(change) {
11442
+ config = change.config;
11443
+ language = change.language;
11444
+ analysis = change.analysis;
11445
+ partialWordAnalysis = change.partialWordAnalysis;
11446
+ textAnalysis = { language, options: resolvePartitionInsertOptions(config, analysis) };
11447
+ manager.synonyms = change.synonyms;
11448
+ },
11437
11449
  addPartition() {
11438
11450
  const currentMaxPartitions = config.partitions?.maxPartitions;
11439
11451
  if (currentMaxPartitions !== void 0 && partitions.length >= currentMaxPartitions) {
@@ -11452,7 +11464,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
11452
11464
  validatePartitionId(partitionId);
11453
11465
  writeLog.partitionsReplaced();
11454
11466
  partitions.splice(partitionId, 1);
11455
- rebuildDocPartitionMap();
11467
+ rebuildDocLocations(docPartitionMap, partitions);
11456
11468
  },
11457
11469
  trimPartitions(newCount) {
11458
11470
  if (newCount < 1) {
@@ -11474,7 +11486,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
11474
11486
  },
11475
11487
  ...createCapacityChecks({
11476
11488
  indexName,
11477
- config,
11489
+ config: initialConfig,
11478
11490
  partitionCount: () => partitions.length,
11479
11491
  documentCount: () => manager.countDocuments(),
11480
11492
  getPartition: (partitionId) => manager.getPartition(partitionId)
@@ -11576,7 +11588,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
11576
11588
  validatePartitionId(partitionId);
11577
11589
  requireNewSegmentDocuments(segment, docPartitionMap);
11578
11590
  asCompositePartition(partitionId).attachFrozenSegment(segment);
11579
- registerSegmentDocuments(partitionId, segment);
11591
+ recordSegmentDocLocations(docPartitionMap, partitionId, segment);
11580
11592
  writeLog.segmentWritten(segment);
11581
11593
  },
11582
11594
  swapFrozenSegments(partitionId, dropSegmentIds, replacement, removedSpellings) {
@@ -11586,7 +11598,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
11586
11598
  replacement,
11587
11599
  removedSpellings
11588
11600
  );
11589
- registerSegmentDocuments(partitionId, replacement);
11601
+ recordSegmentDocLocations(docPartitionMap, partitionId, replacement);
11590
11602
  return applied;
11591
11603
  },
11592
11604
  freezeLiveTail(partitionId, freeze) {
@@ -11596,14 +11608,14 @@ function createPartitionManager(indexName, config, language, router, initialPart
11596
11608
  replaceLiveTail(partitionId, segment) {
11597
11609
  validatePartitionId(partitionId);
11598
11610
  asCompositePartition(partitionId).replaceLiveTail(segment);
11599
- registerSegmentDocuments(partitionId, segment);
11611
+ recordSegmentDocLocations(docPartitionMap, partitionId, segment);
11600
11612
  },
11601
11613
  deserializePartition(partitionId, data) {
11602
11614
  validatePartitionId(partitionId);
11603
11615
  writeLog.partitionsReplaced();
11604
11616
  partitions[partitionId].deserialize(data, config.schema);
11605
11617
  preparePartialWords(partitionId);
11606
- rebuildDocPartitionMapForPartition(partitionId);
11618
+ rebuildPartitionDocLocations(docPartitionMap, partitionId, partitions[partitionId]);
11607
11619
  },
11608
11620
  getAggregateStats() {
11609
11621
  return aggregateStatsOf(partitions);
@@ -13159,9 +13171,9 @@ function basicPlaneUtf8Bytes(text) {
13159
13171
  return bytes;
13160
13172
  }
13161
13173
  function encodeStringBlob(strings) {
13162
- const joined2 = strings.join("");
13163
- if (SURROGATE.test(joined2)) return encodeOneByOne(strings);
13164
- const blob = encoder3.encode(joined2);
13174
+ const joined = strings.join("");
13175
+ if (SURROGATE.test(joined)) return encodeOneByOne(strings);
13176
+ const blob = encoder3.encode(joined);
13165
13177
  const offsets = new Uint32Array(strings.length + 1);
13166
13178
  let cursor = 0;
13167
13179
  for (let i = 0; i < strings.length; i++) {
@@ -15397,7 +15409,7 @@ function requireLoadedVectorCode() {
15397
15409
  function loadVectorCode() {
15398
15410
  if (loaded2 !== null) return Promise.resolve(loaded2);
15399
15411
  if (loading2 === null) {
15400
- loading2 = import('./module-4HIMOQM5.mjs').then(
15412
+ loading2 = import('./module-5J3PEUEB.mjs').then(
15401
15413
  (code) => {
15402
15414
  loaded2 = code;
15403
15415
  return code;
@@ -15492,6 +15504,34 @@ function heldVectorOf(held, fieldName, docId) {
15492
15504
  return held.fields.get(fieldName)?.view.vectorOf(docId);
15493
15505
  }
15494
15506
 
15507
+ // src/workers/direct-executor/entry.ts
15508
+ function indexEntryOf(manager, vectorIndexes, analysisStale, awaitedCopyLoad) {
15509
+ return {
15510
+ manager,
15511
+ get config() {
15512
+ return manager.config;
15513
+ },
15514
+ get language() {
15515
+ return manager.language;
15516
+ },
15517
+ searchOptions: {
15518
+ get stopWords() {
15519
+ return manager.analysis.stopWords;
15520
+ },
15521
+ get customTokenizer() {
15522
+ return manager.analysis.customTokenizer;
15523
+ },
15524
+ get synonyms() {
15525
+ return manager.synonyms;
15526
+ }
15527
+ },
15528
+ vectorIndexes,
15529
+ vectorCopies: createHeldVectorCopies(),
15530
+ analysisStale,
15531
+ awaitedCopyLoad
15532
+ };
15533
+ }
15534
+
15495
15535
  // src/workers/direct-executor/index.ts
15496
15536
  var NO_VECTOR_WORKER_COPIES = { enabled: false };
15497
15537
  function createDirectExecutor(options) {
@@ -15543,22 +15583,7 @@ function createDirectExecutor(options) {
15543
15583
  const partitionCount = config.partitions?.maxPartitions ?? 1;
15544
15584
  const vectorIndexes = vectorIndexesFor(indexName, config, copies);
15545
15585
  const manager = createPartitionManager(indexName, config, language, router, partitionCount, vectorIndexes);
15546
- return {
15547
- manager,
15548
- config,
15549
- language,
15550
- searchOptions: {
15551
- stopWords: manager.analysis.stopWords,
15552
- customTokenizer: manager.analysis.customTokenizer,
15553
- get synonyms() {
15554
- return manager.synonyms;
15555
- }
15556
- },
15557
- vectorIndexes,
15558
- vectorCopies: createHeldVectorCopies(),
15559
- analysisStale,
15560
- awaitedCopyLoad
15561
- };
15586
+ return indexEntryOf(manager, vectorIndexes, analysisStale, awaitedCopyLoad);
15562
15587
  }
15563
15588
  function disposeEntry(entry) {
15564
15589
  for (const vectorIndex of entry.vectorIndexes.values()) {
@@ -15780,11 +15805,14 @@ function createDirectExecutor(options) {
15780
15805
  }
15781
15806
 
15782
15807
  // src/partitioning/text-rebuild.ts
15808
+ function textRebuildTargetOf(config, language, analysis) {
15809
+ return { config, language, analysis, partialWordAnalysis: resolvePartialWordAnalysis(config, language, analysis) };
15810
+ }
15783
15811
  function interrupted(pass) {
15784
15812
  return pass.log.partitionsReplaced || !pass.hooks.isCurrent();
15785
15813
  }
15786
15814
  async function buildPartition(pass, source, partitionId) {
15787
- const { manager, options } = pass;
15815
+ const { manager, language, options } = pass;
15788
15816
  const target = createPartitionIndex(partitionId, manager.config.trackPositions ?? true);
15789
15817
  const docIds = [...source.docIds()];
15790
15818
  target.beginBatch();
@@ -15792,27 +15820,27 @@ async function buildPartition(pass, source, partitionId) {
15792
15820
  const end = Math.min(start + TEXT_REBUILD_CHUNK_DOCUMENTS, docIds.length);
15793
15821
  for (let i = start; i < end; i++) {
15794
15822
  const document = manager.getRef(docIds[i]);
15795
- if (document !== void 0) target.insert(docIds[i], document, manager.schema, manager.language, options);
15823
+ if (document !== void 0) target.insert(docIds[i], document, manager.schema, language, options);
15796
15824
  }
15797
15825
  await yieldToEventLoop();
15798
15826
  if (interrupted(pass)) return null;
15799
15827
  }
15800
15828
  target.endBatch();
15801
- if (manager.partialWordAnalysis !== null) target.preparePartialWords(manager.partialWordAnalysis);
15829
+ if (pass.partialWordAnalysis !== null) target.preparePartialWords(pass.partialWordAnalysis);
15802
15830
  return target;
15803
15831
  }
15804
15832
  function applyWrittenDocuments(pass, built) {
15805
- const { manager, options } = pass;
15833
+ const { manager, language, options } = pass;
15806
15834
  for (const docId of pass.log.takeWrittenDocIds()) {
15807
15835
  for (const partition of built) {
15808
15836
  if (!partition.has(docId)) continue;
15809
- partition.remove(docId, manager.schema, manager.language, options);
15837
+ partition.remove(docId, manager.schema, language, options);
15810
15838
  break;
15811
15839
  }
15812
15840
  const partitionId = manager.partitionIdOf(docId);
15813
15841
  const document = manager.getRef(docId);
15814
15842
  if (partitionId === void 0 || document === void 0) continue;
15815
- built[partitionId].insert(docId, document, manager.schema, manager.language, options);
15843
+ built[partitionId].insert(docId, document, manager.schema, language, options);
15816
15844
  }
15817
15845
  }
15818
15846
  async function rebuildOnce(pass) {
@@ -15829,17 +15857,36 @@ async function rebuildOnce(pass) {
15829
15857
  await yieldToEventLoop();
15830
15858
  if (interrupted(pass)) return false;
15831
15859
  }
15832
- applyWrittenDocuments(pass, built);
15833
- pass.manager.adoptRebuiltPartitions(built);
15834
- pass.hooks.onSwitch();
15835
- return true;
15860
+ const keepCurrent = () => {
15861
+ if (interrupted(pass)) return false;
15862
+ applyWrittenDocuments(pass, built);
15863
+ return true;
15864
+ };
15865
+ if (pass.hooks.holdBeforeSwitch !== void 0 && !await pass.hooks.holdBeforeSwitch(keepCurrent)) return false;
15866
+ const swap = () => {
15867
+ if (interrupted(pass)) return false;
15868
+ applyWrittenDocuments(pass, built);
15869
+ pass.manager.adoptRebuiltPartitions(built);
15870
+ pass.hooks.onSwitch();
15871
+ return true;
15872
+ };
15873
+ return pass.hooks.runSwitch === void 0 ? swap() : pass.hooks.runSwitch(swap);
15836
15874
  }
15837
- async function rebuildTextIndexes(manager, hooks) {
15838
- const options = resolvePartitionInsertOptions(manager.config, manager.analysis, { validate: false, skipClone: true });
15875
+ async function rebuildTextIndexes(manager, hooks, target) {
15876
+ const options = resolvePartitionInsertOptions(
15877
+ target?.config ?? manager.config,
15878
+ target?.analysis ?? manager.analysis,
15879
+ {
15880
+ validate: false,
15881
+ skipClone: true
15882
+ }
15883
+ );
15884
+ const language = target?.language ?? manager.language;
15885
+ const partialWordAnalysis = target === void 0 ? manager.partialWordAnalysis : target.partialWordAnalysis;
15839
15886
  while (hooks.isCurrent()) {
15840
15887
  const log = manager.openWriteLog();
15841
15888
  try {
15842
- if (await rebuildOnce({ manager, log, hooks, options })) return true;
15889
+ if (await rebuildOnce({ manager, log, hooks, language, partialWordAnalysis, options })) return true;
15843
15890
  } finally {
15844
15891
  log.close();
15845
15892
  }
@@ -15877,6 +15924,7 @@ function wireAnalysisRebuild(wiring) {
15877
15924
  getManager: wiring.getManager,
15878
15925
  desyncIndex: wiring.desyncIndex,
15879
15926
  resyncIndex: wiring.resyncIndex,
15927
+ hasPendingChange: wiring.hasPendingChange,
15880
15928
  async persistAnalysisRevision(indexName) {
15881
15929
  const durability = wiring.durabilityManager;
15882
15930
  if (durability === null) {
@@ -15894,6 +15942,7 @@ function wireAnalysisRebuild(wiring) {
15894
15942
  }
15895
15943
  function createAnalysisRebuildCoordinator(config, deps) {
15896
15944
  const stale = /* @__PURE__ */ new Map();
15945
+ const exclusiveRuns = /* @__PURE__ */ new Map();
15897
15946
  const rebuildMode = config?.rebuild ?? "auto";
15898
15947
  let queue = Promise.resolve();
15899
15948
  let active = null;
@@ -15969,7 +16018,7 @@ function createAnalysisRebuildCoordinator(config, deps) {
15969
16018
  await config?.onStaleAnalysis?.({ ...entry.index, documentCount }, () => enqueue(indexName, entry));
15970
16019
  }
15971
16020
  async function review(indexName, entry) {
15972
- if (entry.run !== null || deps.getManager(indexName) === void 0) return;
16021
+ if (entry.run !== null || deps.getManager(indexName) === void 0 || deps.hasPendingChange(indexName)) return;
15973
16022
  try {
15974
16023
  await announce(indexName, entry);
15975
16024
  } catch (err) {
@@ -16001,8 +16050,22 @@ function createAnalysisRebuildCoordinator(config, deps) {
16001
16050
  return stale.has(indexName);
16002
16051
  },
16003
16052
  isRunning(indexName) {
16053
+ if ((exclusiveRuns.get(indexName) ?? 0) > 0) return true;
16004
16054
  return stale.get(indexName)?.run !== null && stale.get(indexName)?.run !== void 0;
16005
16055
  },
16056
+ runExclusive(indexName, work, turnEnds) {
16057
+ exclusiveRuns.set(indexName, (exclusiveRuns.get(indexName) ?? 0) + 1);
16058
+ const run = queue.then(work).finally(() => {
16059
+ const remaining = (exclusiveRuns.get(indexName) ?? 1) - 1;
16060
+ if (remaining === 0) exclusiveRuns.delete(indexName);
16061
+ else exclusiveRuns.set(indexName, remaining);
16062
+ });
16063
+ queue = Promise.race([run, turnEnds ?? run]).then(
16064
+ () => void 0,
16065
+ () => void 0
16066
+ );
16067
+ return run;
16068
+ },
16006
16069
  async abandon(indexName) {
16007
16070
  const entry = stale.get(indexName);
16008
16071
  if (entry !== void 0 && entry.run !== null) entry.abandoned = true;
@@ -16401,6 +16464,10 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
16401
16464
  if (loadingVectorCode !== null) await loadingVectorCode;
16402
16465
  const existing = deps.indexRegistry.get(metadata.indexName);
16403
16466
  if (existing !== void 0) {
16467
+ if ((metadata.analysisGeneration ?? 0) > existing.analysisGeneration) {
16468
+ const { adoptSavedAnalysis } = await import('./analysis-change-57MIRJWR.mjs');
16469
+ adoptSavedAnalysis(deps.executor.getManager(metadata.indexName), existing, metadata);
16470
+ }
16404
16471
  if (loadData && deps.executor.getManager(metadata.indexName) === void 0) {
16405
16472
  deps.executor.createIndex(
16406
16473
  metadata.indexName,
@@ -16449,7 +16516,10 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
16449
16516
  partitionCount: metadata.partitionCount,
16450
16517
  analysisRevision: metadata.analysisRevision ?? null,
16451
16518
  arrayPositionGap: metadata.arrayPositionGap ?? null,
16452
- synonyms
16519
+ synonyms,
16520
+ analysisGeneration: metadata.analysisGeneration ?? 0,
16521
+ pendingAnalysis: metadata.pendingAnalysis ?? null,
16522
+ durableAnalysis: null
16453
16523
  });
16454
16524
  if (loadData) await deps.indexState().registerOpen(metadata.indexName);
16455
16525
  else deps.indexState().registerClosed(metadata.indexName);
@@ -16849,6 +16919,7 @@ function wireIndexState(wiring) {
16849
16919
  }
16850
16920
  try {
16851
16921
  await durability.manager.recoverIndex(indexName);
16922
+ await wiring.resumeChange(indexName);
16852
16923
  await analysisRebuild.review(indexName);
16853
16924
  await orchestrator.openIndex(indexName);
16854
16925
  await wiring.onOpen?.(indexName);
@@ -17569,14 +17640,18 @@ function assertConfigReachesWorker(indexName, config, bootstrapModule) {
17569
17640
  { indexName }
17570
17641
  );
17571
17642
  }
17643
+ if (bootstrapModule !== void 0) return;
17572
17644
  const language = config.language ?? "english";
17573
- if (language !== "english" && bootstrapModule === void 0) {
17645
+ const requireRegistered = (setting, name) => {
17574
17646
  throw new NarsilError(
17575
17647
  ErrorCodes.CONFIG_INVALID,
17576
- `Index "${indexName}" uses language "${language}", which a worker thread registers only from a bootstrap module. Set workers.bootstrapModule to a module that registers it`,
17577
- { indexName, language }
17648
+ `Index "${indexName}" uses ${setting} "${name}", which a worker thread registers only from a bootstrap module. Set workers.bootstrapModule to a module that registers it`,
17649
+ { indexName, [setting]: name }
17578
17650
  );
17579
- }
17651
+ };
17652
+ if (language !== "english") requireRegistered("language", language);
17653
+ if (typeof config.tokenizer === "string") requireRegistered("tokenizer", config.tokenizer);
17654
+ if (typeof config.stopWords === "string") requireRegistered("stopWords", config.stopWords);
17580
17655
  }
17581
17656
  function workerIneligibility(indexName, config, bootstrapModule) {
17582
17657
  try {
@@ -19504,7 +19579,7 @@ async function emitWriteEventAfter(source, indexName, write) {
19504
19579
  // src/engine/core.ts
19505
19580
  async function createEngineCore(config, hooks, capacityScope = "index") {
19506
19581
  validateWorkerConfig(config?.workers, config?.lifecycle);
19507
- const rerankSettings = config?.rerankTimeout === void 0 && config?.maxRerankWindow === void 0 ? { timeoutMs: DEFAULT_RERANK_TIMEOUT_MS, maxWindow: DEFAULT_MAX_RERANK_WINDOW } : (await import('./runtime-3X7ZJ5T3.mjs')).resolveRerankSettings(config);
19582
+ const rerankSettings = config?.rerankTimeout === void 0 && config?.maxRerankWindow === void 0 ? { timeoutMs: DEFAULT_RERANK_TIMEOUT_MS, maxWindow: DEFAULT_MAX_RERANK_WINDOW } : (await import('./runtime-VEUH674B.mjs')).resolveRerankSettings(config);
19508
19583
  const eventHandlers = eventHandlersOf(config?.events);
19509
19584
  const vectorWorkerCount = splitWorkerBudget(resolveWorkerCount(config?.workers?.count)).vector;
19510
19585
  const vectorCopyPolicy = {
@@ -19515,7 +19590,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
19515
19590
  if (config?.lifecycle !== void 0 && durabilityTier === null) {
19516
19591
  throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index lifecycle settings require durability");
19517
19592
  }
19518
- const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-FNZSRH4N.mjs');
19593
+ const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-MTIAN6VT.mjs');
19519
19594
  const filesystemDurability = durabilityTier?.kind === "wal";
19520
19595
  const executor = createDirectExecutor({
19521
19596
  vectorWorkerCopies: vectorCopyPolicy,
@@ -19685,6 +19760,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
19685
19760
  entry.analysisRevision = entry.language.revision;
19686
19761
  entry.arrayPositionGap = ARRAY_POSITION_GAP;
19687
19762
  },
19763
+ hasPendingChange: (indexName) => (indexRegistry.get(indexName)?.pendingAnalysis ?? null) !== null,
19688
19764
  durabilityManager: durability?.manager ?? null
19689
19765
  });
19690
19766
  const indexState = wireIndexState({
@@ -19696,6 +19772,10 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
19696
19772
  indexRegistry,
19697
19773
  rebalancingIndexes,
19698
19774
  requireManager,
19775
+ resumeChange: async (indexName) => {
19776
+ if ((indexRegistry.get(indexName)?.pendingAnalysis ?? null) === null) return;
19777
+ await (await import('./analysis-change-entry-ICGRR3IN.mjs')).resumeChangeOf(engineCore, indexName);
19778
+ },
19699
19779
  onClose: hooks?.onIndexClose,
19700
19780
  onOpen: (indexName) => {
19701
19781
  heapPressureNotifier.check(indexName);
@@ -19736,7 +19816,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
19736
19816
  checkWatermark: watermarkNotifier.check,
19737
19817
  requireIndex
19738
19818
  };
19739
- return {
19819
+ const engineCore = {
19740
19820
  executor,
19741
19821
  pluginRegistry,
19742
19822
  durability,
@@ -19767,6 +19847,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
19767
19847
  rebalanceCtx,
19768
19848
  patternWorkCap: config?.patternWorkCap ?? DEFAULT_PATTERN_WORK_CAP
19769
19849
  };
19850
+ return engineCore;
19770
19851
  }
19771
19852
 
19772
19853
  // src/engine/lifecycle.ts
@@ -19777,6 +19858,7 @@ async function startEngineCore(core, config) {
19777
19858
  checkHeapAfterRecovery(core);
19778
19859
  }
19779
19860
  if (core.invalidation) await core.invalidation.start();
19861
+ if (core.durability) await (await import('./analysis-change-entry-ICGRR3IN.mjs')).resumeEveryChange(core);
19780
19862
  if (config?.lifecycle === void 0) await core.analysisRebuild.reviewStaleIndexes();
19781
19863
  } catch (err) {
19782
19864
  await core.durability?.manager.shutdown().catch(() => void 0);
@@ -19814,7 +19896,7 @@ async function shutdownEngine(core) {
19814
19896
  }
19815
19897
  }
19816
19898
  if (core.rerankAdapters.size > 0) {
19817
- await (await import('./runtime-3X7ZJ5T3.mjs')).shutdownRerankAdapters(core.rerankAdapters);
19899
+ await (await import('./runtime-VEUH674B.mjs')).shutdownRerankAdapters(core.rerankAdapters);
19818
19900
  }
19819
19901
  await orchestrator.shutdown();
19820
19902
  await executor.shutdown();
@@ -20244,7 +20326,8 @@ function indexConfigurationOf(entry) {
20244
20326
  entry.embeddingAdapterName,
20245
20327
  entry.analysisRevision,
20246
20328
  entry.arrayPositionGap,
20247
- synonyms
20329
+ synonyms,
20330
+ entry.analysisGeneration
20248
20331
  );
20249
20332
  }
20250
20333
  function requireConfigurationWithinSizeLimit(config, language) {
@@ -20265,7 +20348,7 @@ function requireConfigurationWithinSizeLimit(config, language) {
20265
20348
  { bytes, maximum: MAX_INDEX_CONFIGURATION_BYTES }
20266
20349
  );
20267
20350
  }
20268
- function configurationPayloadOf(config, language, adapterName, analysisRevision, arrayPositionGap, synonyms) {
20351
+ function configurationPayloadOf(config, language, adapterName, analysisRevision, arrayPositionGap, synonyms, analysisGeneration = 0) {
20269
20352
  const bm25 = config.bm25;
20270
20353
  return {
20271
20354
  schema: config.schema,
@@ -20275,6 +20358,7 @@ function configurationPayloadOf(config, language, adapterName, analysisRevision,
20275
20358
  ...typeof config.tokenizer === "string" ? { tokenizer: config.tokenizer } : {},
20276
20359
  ...typeof config.stopWords === "string" ? { stopWords: config.stopWords } : {},
20277
20360
  ...config.stopWords instanceof Set ? { stopWordList: [...config.stopWords].sort(compareCodePoints) } : {},
20361
+ ...analysisGeneration > 0 ? { analysisGeneration } : {},
20278
20362
  ...bm25 !== void 0 ? {
20279
20363
  bm25: {
20280
20364
  ...bm25.k1 !== void 0 ? { k1: bm25.k1 } : {},
@@ -20449,7 +20533,10 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
20449
20533
  partitionCount: indexConfig.partitions?.maxPartitions ?? 1,
20450
20534
  analysisRevision: language.revision,
20451
20535
  arrayPositionGap: ARRAY_POSITION_GAP,
20452
- synonyms: synonyms?.encoded ?? null
20536
+ synonyms: synonyms?.encoded ?? null,
20537
+ analysisGeneration: 0,
20538
+ pendingAnalysis: null,
20539
+ durableAnalysis: null
20453
20540
  });
20454
20541
  await core.indexState.registerOpen(name);
20455
20542
  if (core.durability) {
@@ -20460,8 +20547,14 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
20460
20547
  async function dropEngineIndex(core, name) {
20461
20548
  core.guardShutdown();
20462
20549
  const entry = core.requireIndex(name);
20550
+ const changeRecorded = () => entry.pendingAnalysis !== null || entry.durableAnalysis !== null;
20551
+ const abandonChange = async () => {
20552
+ await (await import('./analysis-change-entry-ICGRR3IN.mjs')).abandonChangeOf(core, name);
20553
+ };
20554
+ if (changeRecorded()) await abandonChange();
20463
20555
  await core.indexState.drop(name, async () => {
20464
20556
  await core.analysisRebuild.abandon(name);
20557
+ if (changeRecorded()) await abandonChange();
20465
20558
  await core.orchestrator.closeIndex(name);
20466
20559
  if (core.durability) {
20467
20560
  try {
@@ -22273,6 +22366,14 @@ async function executeRebalance(manager, indexName, targetPartitionCount, ctx) {
22273
22366
  `Index "${indexName}" is already being rebalanced`
22274
22367
  );
22275
22368
  }
22369
+ const registered4 = ctx.requireIndex(indexName);
22370
+ if (registered4.pendingAnalysis !== null || registered4.durableAnalysis !== null) {
22371
+ throw new NarsilError(
22372
+ ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
22373
+ `Index "${indexName}" is changing its analysis settings, so try the rebalance again once the change ends`,
22374
+ { indexName }
22375
+ );
22376
+ }
22276
22377
  if (targetPartitionCount === manager.partitionCount) {
22277
22378
  return;
22278
22379
  }
@@ -22479,646 +22580,6 @@ function bindEngineCore(engine, core) {
22479
22580
  engineCores.set(engine, core);
22480
22581
  }
22481
22582
 
22482
- // src/engine/snapshot/restore-config.ts
22483
- function rejectSnapshot(message) {
22484
- throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Invalid snapshot: ${message}`);
22485
- }
22486
- function isPlainObject3(value) {
22487
- return value !== null && typeof value === "object" && !Array.isArray(value);
22488
- }
22489
- function restoredBm25(raw, reject2) {
22490
- if (raw === void 0) {
22491
- return void 0;
22492
- }
22493
- if (!isPlainObject3(raw)) {
22494
- reject2("bm25 must be an object");
22495
- }
22496
- const fields = restoredBm25Fields(raw.fields, reject2);
22497
- return {
22498
- ...restoredBm25Pair(raw, "bm25", reject2),
22499
- ...fields !== void 0 ? { fields } : {}
22500
- };
22501
- }
22502
- function restoredBm25Pair(raw, label, reject2) {
22503
- const { k1, b } = raw;
22504
- if (k1 !== void 0 && (typeof k1 !== "number" || !Number.isFinite(k1))) {
22505
- reject2(`${label} k1 must be a finite number`);
22506
- }
22507
- if (b !== void 0 && (typeof b !== "number" || !Number.isFinite(b))) {
22508
- reject2(`${label} b must be a finite number`);
22509
- }
22510
- return {
22511
- ...k1 !== void 0 ? { k1 } : {},
22512
- ...b !== void 0 ? { b } : {}
22513
- };
22514
- }
22515
- function restoredBm25Fields(raw, reject2) {
22516
- if (raw === void 0) return void 0;
22517
- if (!isPlainObject3(raw)) reject2("bm25 fields must be an object");
22518
- const source = raw;
22519
- const fields = /* @__PURE__ */ Object.create(null);
22520
- for (const name of Object.keys(source)) {
22521
- const pair = source[name];
22522
- if (!isPlainObject3(pair)) reject2("bm25 fields must map each field to an object");
22523
- fields[name] = restoredBm25Pair(pair, "bm25 field", reject2);
22524
- }
22525
- return fields;
22526
- }
22527
- function restoredStopWords(name, list, reject2) {
22528
- if (name !== void 0 && typeof name !== "string") {
22529
- reject2("stopWords must be a name");
22530
- }
22531
- if (list !== void 0 && (!Array.isArray(list) || !list.every((word) => typeof word === "string"))) {
22532
- reject2("stopWordList must be a list of words");
22533
- }
22534
- if (typeof name === "string" && Array.isArray(list)) {
22535
- reject2("stopWords and stopWordList cannot both be present");
22536
- }
22537
- if (typeof name === "string") {
22538
- return name;
22539
- }
22540
- if (Array.isArray(list)) {
22541
- return new Set(list);
22542
- }
22543
- return void 0;
22544
- }
22545
- function restoredPartitionConfig(raw, reject2 = rejectSnapshot) {
22546
- if (raw === void 0) {
22547
- return void 0;
22548
- }
22549
- if (!isPlainObject3(raw)) {
22550
- reject2("partitionConfig must be an object");
22551
- }
22552
- const { maxDocsPerPartition, maxPartitions, watermark } = raw;
22553
- if (maxDocsPerPartition !== void 0 && (typeof maxDocsPerPartition !== "number" || !Number.isInteger(maxDocsPerPartition) || maxDocsPerPartition < 1)) {
22554
- reject2("partitionConfig maxDocsPerPartition must be a positive integer");
22555
- }
22556
- if (maxPartitions !== void 0 && (typeof maxPartitions !== "number" || !Number.isInteger(maxPartitions) || maxPartitions < 1)) {
22557
- reject2("partitionConfig maxPartitions must be a positive integer");
22558
- }
22559
- if (watermark !== void 0 && (typeof watermark !== "number" || !(watermark > 0) || watermark > 1)) {
22560
- reject2("partitionConfig watermark must be above 0 and at most 1");
22561
- }
22562
- return {
22563
- ...typeof maxDocsPerPartition === "number" ? { maxDocsPerPartition } : {},
22564
- ...typeof maxPartitions === "number" ? { maxPartitions } : {},
22565
- ...typeof watermark === "number" ? { watermark } : {}
22566
- };
22567
- }
22568
- function restoredScoring(raw, reject2) {
22569
- if (raw === void 0) {
22570
- return void 0;
22571
- }
22572
- if (raw !== "local" && raw !== "dfs" && raw !== "broadcast") {
22573
- reject2("defaultScoring must be 'local', 'dfs', or 'broadcast'");
22574
- }
22575
- return raw;
22576
- }
22577
- function restoredBoolean(raw, field, reject2) {
22578
- if (raw === void 0) {
22579
- return void 0;
22580
- }
22581
- if (typeof raw !== "boolean") {
22582
- reject2(`${field} must be a boolean`);
22583
- }
22584
- return raw;
22585
- }
22586
- function restoredRequired(raw, reject2) {
22587
- if (raw === void 0) {
22588
- return void 0;
22589
- }
22590
- if (!Array.isArray(raw) || !raw.every((field) => typeof field === "string")) {
22591
- reject2("required must be a list of field paths");
22592
- }
22593
- return raw;
22594
- }
22595
- function restoredVectorPromotion(raw, reject2) {
22596
- if (raw === void 0) {
22597
- return void 0;
22598
- }
22599
- if (!isPlainObject3(raw) || raw.hnswConfig !== void 0 && !isPlainObject3(raw.hnswConfig)) {
22600
- reject2("vectorPromotion must be an object");
22601
- }
22602
- return raw;
22603
- }
22604
- function restoredEmbedding(raw, reject2 = rejectSnapshot) {
22605
- if (raw === void 0) {
22606
- return void 0;
22607
- }
22608
- if (!isPlainObject3(raw)) {
22609
- reject2("embedding must be an object");
22610
- }
22611
- const { fields, adapter } = raw;
22612
- if (adapter !== void 0 && typeof adapter !== "string") {
22613
- reject2("embedding adapter must be a name");
22614
- }
22615
- if (!isPlainObject3(fields)) {
22616
- reject2("embedding fields must be an object");
22617
- }
22618
- for (const value of Object.values(fields)) {
22619
- const isPath = typeof value === "string";
22620
- const isPathList = Array.isArray(value) && value.every((path) => typeof path === "string");
22621
- if (!isPath && !isPathList) {
22622
- reject2("embedding fields must map vector fields to source paths");
22623
- }
22624
- }
22625
- return {
22626
- fields,
22627
- ...adapter !== void 0 ? { adapter } : {}
22628
- };
22629
- }
22630
- function readSharedConfigFields(source, reject2) {
22631
- if (source.tokenizer !== void 0 && typeof source.tokenizer !== "string") {
22632
- reject2("tokenizer must be a name");
22633
- }
22634
- const stopWords2 = restoredStopWords(source.stopWords, source.stopWordList, reject2);
22635
- const bm25 = restoredBm25(source.bm25, reject2);
22636
- const surfaceForms = restoredBoolean(source.surfaceForms, "surfaceForms", reject2);
22637
- const defaultScoring = restoredScoring(source.defaultScoring, reject2);
22638
- const trackPositions = restoredBoolean(source.trackPositions, "trackPositions", reject2);
22639
- const strict = restoredBoolean(source.strict, "strict", reject2);
22640
- const required = restoredRequired(source.required, reject2);
22641
- const vectorPromotion = restoredVectorPromotion(source.vectorPromotion, reject2);
22642
- if (source.patternValueLimit !== void 0 && typeof source.patternValueLimit !== "number") {
22643
- reject2("patternValueLimit must be a number");
22644
- }
22645
- return {
22646
- ...typeof source.tokenizer === "string" ? { tokenizer: source.tokenizer } : {},
22647
- ...stopWords2 !== void 0 ? { stopWords: stopWords2 } : {},
22648
- ...bm25 !== void 0 ? { bm25 } : {},
22649
- ...surfaceForms !== void 0 ? { surfaceForms } : {},
22650
- ...defaultScoring !== void 0 ? { defaultScoring } : {},
22651
- ...trackPositions !== void 0 ? { trackPositions } : {},
22652
- ...strict !== void 0 ? { strict } : {},
22653
- ...required !== void 0 ? { required } : {},
22654
- ...vectorPromotion !== void 0 ? { vectorPromotion } : {},
22655
- ...typeof source.patternValueLimit === "number" ? { patternValueLimit: source.patternValueLimit } : {}
22656
- };
22657
- }
22658
- function restoredConfigFields(envelope, reject2 = rejectSnapshot) {
22659
- const shared2 = readSharedConfigFields(envelope, reject2);
22660
- const partitions = restoredPartitionConfig(envelope.partitionConfig, reject2);
22661
- return {
22662
- ...shared2,
22663
- ...partitions !== void 0 ? { partitions } : {}
22664
- };
22665
- }
22666
-
22667
- // src/persistence/archive/byte-reader.ts
22668
- async function* chunksOf(source) {
22669
- if (source instanceof Uint8Array) {
22670
- yield source;
22671
- return;
22672
- }
22673
- if (typeof source.getReader === "function") {
22674
- const reader = source.getReader();
22675
- try {
22676
- for (; ; ) {
22677
- const step = await reader.read();
22678
- if (step.done) return;
22679
- yield step.value;
22680
- }
22681
- } finally {
22682
- reader.releaseLock();
22683
- }
22684
- }
22685
- yield* source;
22686
- }
22687
- function createByteReader(source) {
22688
- const iterator = chunksOf(source);
22689
- const pending = [];
22690
- let pendingBytes = 0;
22691
- let ended = false;
22692
- async function fill(length) {
22693
- while (pendingBytes < length && !ended) {
22694
- const step = await iterator.next();
22695
- if (step.done) {
22696
- ended = true;
22697
- break;
22698
- }
22699
- if (step.value.byteLength === 0) continue;
22700
- pending.push(step.value);
22701
- pendingBytes += step.value.byteLength;
22702
- }
22703
- }
22704
- function take(length) {
22705
- const out = new Uint8Array(length);
22706
- let filled = 0;
22707
- while (filled < length) {
22708
- const head = pending[0];
22709
- const needed = length - filled;
22710
- if (head.byteLength <= needed) {
22711
- out.set(head, filled);
22712
- filled += head.byteLength;
22713
- pending.shift();
22714
- } else {
22715
- out.set(head.subarray(0, needed), filled);
22716
- pending[0] = head.subarray(needed);
22717
- filled += needed;
22718
- }
22719
- }
22720
- pendingBytes -= length;
22721
- return out;
22722
- }
22723
- return {
22724
- async readExactly(length) {
22725
- await fill(length);
22726
- if (pendingBytes === 0 && length > 0) return null;
22727
- if (pendingBytes < length) return take(pendingBytes);
22728
- return take(length);
22729
- },
22730
- async peek(length) {
22731
- await fill(length);
22732
- const available = Math.min(length, pendingBytes);
22733
- const copy = take(available);
22734
- pending.unshift(copy);
22735
- pendingBytes += copy.byteLength;
22736
- return copy;
22737
- },
22738
- async close() {
22739
- await iterator.return(void 0);
22740
- }
22741
- };
22742
- }
22743
-
22744
- // src/engine/export/lines.ts
22745
- var NEWLINE = 10;
22746
- function joined(parts, total) {
22747
- if (parts.length === 1) return parts[0];
22748
- const bytes = new Uint8Array(total);
22749
- let offset = 0;
22750
- for (const part of parts) {
22751
- bytes.set(part, offset);
22752
- offset += part.byteLength;
22753
- }
22754
- return bytes;
22755
- }
22756
- async function* exportLines(source, maxLineBytes) {
22757
- const decoder4 = new TextDecoder("utf-8", { fatal: true });
22758
- let parts = [];
22759
- let pending = 0;
22760
- let overflowing = false;
22761
- let line = 0;
22762
- const finishLine = () => {
22763
- line += 1;
22764
- if (overflowing) {
22765
- overflowing = false;
22766
- parts = [];
22767
- pending = 0;
22768
- return { line, refused: "too-long" };
22769
- }
22770
- const bytes = joined(parts, pending);
22771
- parts = [];
22772
- pending = 0;
22773
- try {
22774
- return { line, text: decoder4.decode(bytes) };
22775
- } catch {
22776
- return { line, refused: "not-utf8" };
22777
- }
22778
- };
22779
- for await (const chunk of chunksOf(source)) {
22780
- let start = 0;
22781
- for (; ; ) {
22782
- const newline = chunk.indexOf(NEWLINE, start);
22783
- const end = newline < 0 ? chunk.byteLength : newline;
22784
- const piece = chunk.subarray(start, end);
22785
- if (!overflowing && pending + piece.byteLength <= maxLineBytes) {
22786
- if (piece.byteLength > 0) parts.push(piece.slice());
22787
- pending += piece.byteLength;
22788
- } else {
22789
- overflowing = true;
22790
- parts = [];
22791
- }
22792
- if (newline < 0) break;
22793
- yield finishLine();
22794
- start = newline + 1;
22795
- }
22796
- }
22797
- if (pending > 0 || overflowing) yield finishLine();
22798
- }
22799
-
22800
- // src/serialization/base64.ts
22801
- var ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
22802
- var PAD = 61;
22803
- var BYTES_PER_GROUP = 3;
22804
- var CHARACTERS_PER_GROUP = 4;
22805
- var ASCII_LIMIT = 128;
22806
- var ENCODE = new Uint8Array(ALPHABET.length);
22807
- var DECODE = new Int8Array(ASCII_LIMIT).fill(-1);
22808
- for (let at = 0; at < ALPHABET.length; at++) {
22809
- ENCODE[at] = ALPHABET.charCodeAt(at);
22810
- DECODE[ALPHABET.charCodeAt(at)] = at;
22811
- }
22812
- var asciiDecoder = new TextDecoder();
22813
- function encodeBase64(bytes) {
22814
- const out = new Uint8Array(Math.ceil(bytes.byteLength / BYTES_PER_GROUP) * CHARACTERS_PER_GROUP);
22815
- let write = 0;
22816
- let read = 0;
22817
- for (; read + BYTES_PER_GROUP <= bytes.byteLength; read += BYTES_PER_GROUP) {
22818
- const group = bytes[read] << 16 | bytes[read + 1] << 8 | bytes[read + 2];
22819
- out[write++] = ENCODE[group >>> 18];
22820
- out[write++] = ENCODE[group >>> 12 & 63];
22821
- out[write++] = ENCODE[group >>> 6 & 63];
22822
- out[write++] = ENCODE[group & 63];
22823
- }
22824
- const left = bytes.byteLength - read;
22825
- if (left > 0) {
22826
- const group = bytes[read] << 16 | (left === 2 ? bytes[read + 1] << 8 : 0);
22827
- out[write++] = ENCODE[group >>> 18];
22828
- out[write++] = ENCODE[group >>> 12 & 63];
22829
- out[write++] = left === 2 ? ENCODE[group >>> 6 & 63] : PAD;
22830
- out[write++] = PAD;
22831
- }
22832
- return asciiDecoder.decode(out);
22833
- }
22834
- function sextetAt(text, at) {
22835
- const code = text.charCodeAt(at);
22836
- return code < ASCII_LIMIT ? DECODE[code] : -1;
22837
- }
22838
- function decodeExactBase64(text) {
22839
- if (text.length % CHARACTERS_PER_GROUP !== 0) return null;
22840
- const padding = text.endsWith("==") ? 2 : text.endsWith("=") ? 1 : 0;
22841
- const out = new Uint8Array(text.length / CHARACTERS_PER_GROUP * BYTES_PER_GROUP - padding);
22842
- const fullEnd = padding === 0 ? text.length : text.length - CHARACTERS_PER_GROUP;
22843
- let write = 0;
22844
- for (let at = 0; at < fullEnd; at += CHARACTERS_PER_GROUP) {
22845
- const a2 = sextetAt(text, at);
22846
- const b2 = sextetAt(text, at + 1);
22847
- const c2 = sextetAt(text, at + 2);
22848
- const d = sextetAt(text, at + 3);
22849
- if ((a2 | b2 | c2 | d) < 0) return null;
22850
- const group = a2 << 18 | b2 << 12 | c2 << 6 | d;
22851
- out[write++] = group >>> 16;
22852
- out[write++] = group >>> 8 & 255;
22853
- out[write++] = group & 255;
22854
- }
22855
- if (padding === 0) return out;
22856
- const a = sextetAt(text, fullEnd);
22857
- const b = sextetAt(text, fullEnd + 1);
22858
- const c = padding === 1 ? sextetAt(text, fullEnd + 2) : 0;
22859
- if ((a | b | c) < 0) return null;
22860
- if (padding === 2 && (b & 15) !== 0) return null;
22861
- if (padding === 1 && (c & 3) !== 0) return null;
22862
- out[write++] = a << 2 | b >>> 4;
22863
- if (padding === 1) out[write] = (b & 15) << 4 | c >>> 2;
22864
- return out;
22865
- }
22866
-
22867
- // src/engine/export/tags.ts
22868
- var TAG_PREFIX = "$";
22869
- var DATE_TAG = "$date";
22870
- var BYTES_TAG = "$bytes";
22871
- var NUMBER_TAG = "$number";
22872
- var OBJECT_TAG = "$object";
22873
- var MILLISECONDS_PER_SECOND = 1e3;
22874
- var NANOSECONDS_PER_MILLISECOND = 1e6;
22875
- var MAX_NANOSECONDS = 999999999;
22876
- var MAX_DATE_MILLISECONDS = 864e13;
22877
- var ESCAPED_TAG_PREFIX = "\\u0024";
22878
- var NON_FINITE_NUMBERS = /* @__PURE__ */ new Map([
22879
- ["NaN", Number.NaN],
22880
- ["Infinity", Number.POSITIVE_INFINITY],
22881
- ["-Infinity", Number.NEGATIVE_INFINITY]
22882
- ]);
22883
- function refuse(path, reason) {
22884
- throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path === "" ? "(document)" : path}" ${reason}`, {
22885
- field: path
22886
- });
22887
- }
22888
- function isRecord(value) {
22889
- return typeof value === "object" && value !== null && !Array.isArray(value);
22890
- }
22891
- function startsWithTagPrefix(key) {
22892
- return key.startsWith(TAG_PREFIX);
22893
- }
22894
- function soleTagKey(record) {
22895
- let sole = null;
22896
- for (const key of Object.keys(record)) {
22897
- if (sole !== null) return null;
22898
- sole = key;
22899
- }
22900
- return sole !== null && startsWithTagPrefix(sole) ? sole : null;
22901
- }
22902
- function dateOf(value, path) {
22903
- if (!Array.isArray(value) || value.length !== 2)
22904
- refuse(path, `holds a "${DATE_TAG}" tag other than [seconds, nanoseconds]`);
22905
- const seconds = value[0];
22906
- const nanoseconds = value[1];
22907
- if (typeof seconds !== "number" || !Number.isSafeInteger(seconds)) {
22908
- refuse(
22909
- path,
22910
- `holds a "${DATE_TAG}" tag whose seconds are not an integer from -${Number.MAX_SAFE_INTEGER} to ${Number.MAX_SAFE_INTEGER}`
22911
- );
22912
- }
22913
- if (typeof nanoseconds !== "number" || !Number.isInteger(nanoseconds) || nanoseconds < 0 || nanoseconds > MAX_NANOSECONDS) {
22914
- refuse(path, `holds a "${DATE_TAG}" tag whose nanoseconds are not an integer from 0 to ${MAX_NANOSECONDS}`);
22915
- }
22916
- const milliseconds = seconds * MILLISECONDS_PER_SECOND + nanoseconds / NANOSECONDS_PER_MILLISECOND;
22917
- if (Math.abs(milliseconds) > MAX_DATE_MILLISECONDS)
22918
- refuse(path, `holds a "${DATE_TAG}" tag outside the range of a date`);
22919
- return new Date(milliseconds);
22920
- }
22921
- function bytesOf(value, path) {
22922
- const bytes = typeof value === "string" ? decodeExactBase64(value) : null;
22923
- if (bytes === null) refuse(path, `holds a "${BYTES_TAG}" tag whose text is not padded base64`);
22924
- return bytes;
22925
- }
22926
- function numberOf(value, path) {
22927
- const number = typeof value === "string" ? NON_FINITE_NUMBERS.get(value) : void 0;
22928
- if (number === void 0) refuse(path, `holds a "${NUMBER_TAG}" tag other than "NaN", "Infinity", or "-Infinity"`);
22929
- return number;
22930
- }
22931
- function requireDepth(path, depth) {
22932
- if (depth >= MAX_DOCUMENT_NESTING_DEPTH) refuse(path, `nests deeper than ${MAX_DOCUMENT_NESTING_DEPTH} levels`);
22933
- }
22934
- function untagMap(record, path, depth) {
22935
- requireDepth(path, depth);
22936
- for (const key of Object.keys(record)) {
22937
- record[key] = untag(record[key], path === "" ? key : `${path}.${key}`, depth + 1);
22938
- }
22939
- return record;
22940
- }
22941
- function untagList(list, path, depth) {
22942
- requireDepth(path, depth);
22943
- for (let at = 0; at < list.length; at++) list[at] = untag(list[at], `${path}[${at}]`, depth + 1);
22944
- return list;
22945
- }
22946
- function untagTag(tag, value, path, depth) {
22947
- switch (tag) {
22948
- case DATE_TAG:
22949
- return dateOf(value, path);
22950
- case BYTES_TAG:
22951
- return bytesOf(value, path);
22952
- case NUMBER_TAG:
22953
- return numberOf(value, path);
22954
- case OBJECT_TAG:
22955
- if (!isRecord(value)) refuse(path, `holds an "${OBJECT_TAG}" tag whose value is not an object`);
22956
- return untagMap(value, path, depth);
22957
- default:
22958
- return refuse(path, `holds an object whose only key, "${tag}", names no tag`);
22959
- }
22960
- }
22961
- function untag(value, path, depth) {
22962
- if (typeof value !== "object" || value === null) return value;
22963
- if (Array.isArray(value)) return untagList(value, path, depth);
22964
- const record = value;
22965
- const tag = soleTagKey(record);
22966
- return tag === null ? untagMap(record, path, depth) : untagTag(tag, record[tag], path, depth);
22967
- }
22968
- function untagDocumentLine(text, parsed) {
22969
- if (!text.includes(`"${TAG_PREFIX}`) && !text.includes(ESCAPED_TAG_PREFIX)) return parsed;
22970
- return untagMap(parsed, "", 0);
22971
- }
22972
-
22973
- // src/engine/export/import.ts
22974
- var DOCUMENT_EXPORT_FORMAT = "narsil-export";
22975
- var DOCUMENT_EXPORT_VERSION = 1;
22976
- function invalidHeader(message) {
22977
- throw new NarsilError(ErrorCodes.CONFIG_INVALID, `The document export header ${message}`);
22978
- }
22979
- function isRecord2(value) {
22980
- return typeof value === "object" && value !== null && !Array.isArray(value);
22981
- }
22982
- function configOfHeader(line) {
22983
- if (line === void 0) invalidHeader("is missing");
22984
- if (!("text" in line)) {
22985
- invalidHeader(
22986
- line.refused === "too-long" ? `passes the line limit of ${MAX_EXPORT_LINE_BYTES} bytes` : "is not UTF-8"
22987
- );
22988
- }
22989
- let raw;
22990
- try {
22991
- raw = JSON.parse(line.text);
22992
- } catch {
22993
- invalidHeader("is not JSON");
22994
- }
22995
- if (!isRecord2(raw) || raw.format !== DOCUMENT_EXPORT_FORMAT || raw.version !== DOCUMENT_EXPORT_VERSION) {
22996
- invalidHeader(`must name format "${DOCUMENT_EXPORT_FORMAT}" and version ${DOCUMENT_EXPORT_VERSION}`);
22997
- }
22998
- const configuration = requireConfigurationShape(raw.index, "document export header");
22999
- const fields = restoredConfigFields(configuration, (message) => invalidHeader(message));
23000
- const embedding = restoredEmbedding(configuration.embedding, (message) => invalidHeader(message));
23001
- return {
23002
- schema: configuration.schema,
23003
- language: configuration.language,
23004
- ...fields,
23005
- ...embedding === void 0 ? {} : {
23006
- embedding: {
23007
- fields: embedding.fields,
23008
- ...embedding.adapter === void 0 ? {} : { adapter: embedding.adapter }
23009
- }
23010
- }
23011
- };
23012
- }
23013
- async function prepareIndex(engine, indexName, config) {
23014
- if (!engine.listIndexes().some((index) => index.name === indexName)) {
23015
- await engine.createIndex(indexName, config);
23016
- return;
23017
- }
23018
- const existing = flattenSchema(engine.getStats(indexName).schema);
23019
- const exported = flattenSchema(config.schema);
23020
- const fields = /* @__PURE__ */ new Set([...Object.keys(existing), ...Object.keys(exported)]);
23021
- for (const field of fields) {
23022
- if (existing[field] === exported[field]) continue;
23023
- throw new NarsilError(
23024
- ErrorCodes.CONFIG_INVALID,
23025
- `Index "${indexName}" gives field "${field}" another type than the export does, so the import cannot write into it`,
23026
- { indexName, field }
23027
- );
23028
- }
23029
- }
23030
- function highestReportedAt(refused) {
23031
- let highest = 0;
23032
- for (let at = 1; at < refused.length; at++) {
23033
- if (refused[at].line > refused[highest].line) highest = at;
23034
- }
23035
- return highest;
23036
- }
23037
- function refuse2(state, line, code, message) {
23038
- state.result.refusedCount += 1;
23039
- const refused = state.result.refused;
23040
- if (refused.length < MAX_IMPORT_REFUSALS_REPORTED) {
23041
- refused.push({ line, code, message });
23042
- if (refused.length === MAX_IMPORT_REFUSALS_REPORTED) state.highestReported = highestReportedAt(refused);
23043
- return;
23044
- }
23045
- if (line >= refused[state.highestReported].line) return;
23046
- refused[state.highestReported] = { line, code, message };
23047
- state.highestReported = highestReportedAt(refused);
23048
- }
23049
- async function flush(state) {
23050
- if (state.batch.length === 0) return;
23051
- const batch = state.batch;
23052
- state.batch = [];
23053
- state.batchIds = /* @__PURE__ */ new Set();
23054
- const lineOf = new Map(batch.map((entry) => [entry.docId, entry.line]));
23055
- const inserts = [];
23056
- const updates = [];
23057
- for (const entry of batch) {
23058
- if (await state.engine.has(state.indexName, entry.docId))
23059
- updates.push({ docId: entry.docId, document: entry.document });
23060
- else inserts.push(entry.document);
23061
- }
23062
- for (const outcome of [
23063
- inserts.length === 0 ? null : await state.engine.insertBatch(state.indexName, inserts, { skipClone: true }),
23064
- updates.length === 0 ? null : await state.engine.updateBatch(state.indexName, updates)
23065
- ]) {
23066
- if (outcome === null) continue;
23067
- state.result.imported += outcome.succeeded.length;
23068
- for (const failure of outcome.failed) {
23069
- const code = failure.error instanceof NarsilError ? failure.error.code : ErrorCodes.DOC_VALIDATION_FAILED;
23070
- refuse2(state, lineOf.get(failure.docId) ?? 0, code, failure.error.message);
23071
- }
23072
- }
23073
- }
23074
- async function takeLine(state, entry) {
23075
- if (!("text" in entry)) {
23076
- const message = entry.refused === "too-long" ? `The line passes the limit of ${MAX_EXPORT_LINE_BYTES} bytes` : "The line is not UTF-8";
23077
- refuse2(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, message);
23078
- return;
23079
- }
23080
- if (entry.text.length === 0) return;
23081
- let document;
23082
- try {
23083
- document = JSON.parse(entry.text);
23084
- } catch {
23085
- refuse2(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, "The line is not JSON");
23086
- return;
23087
- }
23088
- if (!isRecord2(document) || typeof document.id !== "string" || document.id.length === 0) {
23089
- refuse2(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, "The line holds no document with a string id");
23090
- return;
23091
- }
23092
- try {
23093
- untagDocumentLine(entry.text, document);
23094
- } catch (error) {
23095
- if (!(error instanceof NarsilError)) throw error;
23096
- refuse2(state, entry.line, error.code, error.message);
23097
- return;
23098
- }
23099
- if (state.batchIds.has(document.id)) await flush(state);
23100
- state.batch.push({ line: entry.line, docId: document.id, document });
23101
- state.batchIds.add(document.id);
23102
- if (state.batch.length >= IMPORT_BATCH_DOCUMENTS) await flush(state);
23103
- }
23104
- async function readDocumentExport(engine, indexName, source) {
23105
- const lines = exportLines(source, MAX_EXPORT_LINE_BYTES);
23106
- const header = await lines.next();
23107
- await prepareIndex(engine, indexName, configOfHeader(header.done === true ? void 0 : header.value));
23108
- const state = {
23109
- engine,
23110
- indexName,
23111
- batch: [],
23112
- batchIds: /* @__PURE__ */ new Set(),
23113
- result: { imported: 0, refusedCount: 0, refused: [] },
23114
- highestReported: 0
23115
- };
23116
- for await (const entry of lines) await takeLine(state, entry);
23117
- await flush(state);
23118
- state.result.refused.sort((a, b) => a.line - b.line);
23119
- return state.result;
23120
- }
23121
-
23122
22583
  // src/narsil/portability.ts
23123
22584
  function portabilityOperations(core, config, engine) {
23124
22585
  async function withOpenIndex2(indexName, action) {
@@ -23150,14 +22611,14 @@ function portabilityOperations(core, config, engine) {
23150
22611
  return {
23151
22612
  snapshot(indexName) {
23152
22613
  return withOpenIndex2(indexName, async () => {
23153
- const { createSnapshot } = await import('./snapshot-V7VF6BN6.mjs');
22614
+ const { createSnapshot } = await import('./snapshot-YBJ5MF2M.mjs');
23154
22615
  return createSnapshot(snapshotDepsOf(indexName));
23155
22616
  });
23156
22617
  },
23157
22618
  async restore(indexName, data) {
23158
22619
  core.guardShutdown();
23159
22620
  validateIndexName(indexName);
23160
- const restored = import('./restore-queue-GEZ54CA5.mjs').then(
22621
+ const restored = import('./restore-queue-DISFECE4.mjs').then(
23161
22622
  ({ queueRestore }) => queueRestore(core, restoreHost, restoresInFlight, indexName, data)
23162
22623
  );
23163
22624
  await emitWriteEventAfter(core, indexName, restored);
@@ -23166,8 +22627,8 @@ function portabilityOperations(core, config, engine) {
23166
22627
  exportDocuments(indexName) {
23167
22628
  return withOpenIndex2(indexName, async () => {
23168
22629
  const [{ checkpointForArchive }, { documentExportStream }] = await Promise.all([
23169
- import('./snapshot-V7VF6BN6.mjs'),
23170
- import('./write-YIC2MKRY.mjs')
22630
+ import('./snapshot-YBJ5MF2M.mjs'),
22631
+ import('./write-RTWI5MXL.mjs')
23171
22632
  ]);
23172
22633
  const header = indexConfigurationOf(core.requireIndex(indexName));
23173
22634
  return documentExportStream(header, await checkpointForArchive(snapshotDepsOf(indexName)));
@@ -23175,7 +22636,10 @@ function portabilityOperations(core, config, engine) {
23175
22636
  },
23176
22637
  importDocumentExport(indexName, data) {
23177
22638
  core.guardShutdown();
23178
- return emitWriteEventAfter(core, indexName, readDocumentExport(engine(), indexName, data));
22639
+ const imported = import('./import-Q7MXD6BS.mjs').then(
22640
+ ({ readDocumentExport }) => readDocumentExport(engine(), indexName, data)
22641
+ );
22642
+ return emitWriteEventAfter(core, indexName, imported);
23179
22643
  }
23180
22644
  };
23181
22645
  }
@@ -23185,14 +22649,14 @@ var ABSENT_TAG = 0;
23185
22649
  var NULL_TAG = 1;
23186
22650
  var FALSE_TAG = 2;
23187
22651
  var TRUE_TAG = 3;
23188
- var NUMBER_TAG2 = 4;
22652
+ var NUMBER_TAG = 4;
23189
22653
  var STRING_TAG = 5;
23190
22654
  var LIST_TAG = 6;
23191
22655
  var MAP_TAG = 7;
23192
22656
  var LENGTH_BYTES = 4;
23193
22657
  var LAST_ASCII_UNIT = 127;
23194
22658
  var encoder5 = new TextEncoder();
23195
- function flush2(stream) {
22659
+ function flush(stream) {
23196
22660
  stream.hash = fnv1aContinue(stream.hash, stream.bytes, stream.length);
23197
22661
  stream.length = 0;
23198
22662
  }
@@ -23256,8 +22720,8 @@ function writeNumberList(stream, list) {
23256
22720
  if (stream.clock !== void 0) stream.clock.tick(list.members.length);
23257
22721
  writeByte(stream, LIST_TAG);
23258
22722
  writeUint32(stream, list.members.length);
23259
- flush2(stream);
23260
- stream.hash = fnv1aContinueTaggedNumbers(stream.hash, NUMBER_TAG2, list.members);
22723
+ flush(stream);
22724
+ stream.hash = fnv1aContinueTaggedNumbers(stream.hash, NUMBER_TAG, list.members);
23261
22725
  }
23262
22726
  function writeValue(stream, value) {
23263
22727
  if (value instanceof NumberList) {
@@ -23277,7 +22741,7 @@ function writeValue(stream, value) {
23277
22741
  return;
23278
22742
  }
23279
22743
  if (typeof value === "number") {
23280
- writeByte(stream, NUMBER_TAG2);
22744
+ writeByte(stream, NUMBER_TAG);
23281
22745
  writeNumber(stream, value);
23282
22746
  return;
23283
22747
  }
@@ -23351,7 +22815,7 @@ function bindingOf(values, clock) {
23351
22815
  for (const value of values) {
23352
22816
  writeValue(stream, value);
23353
22817
  }
23354
- flush2(stream);
22818
+ flush(stream);
23355
22819
  return (stream.hash >>> 0).toString(16).padStart(8, "0");
23356
22820
  }
23357
22821
  function queryBindingOf(params, clock) {
@@ -24472,7 +23936,7 @@ function requireCursorDepth(cursor, signature, binding) {
24472
23936
  }
24473
23937
  function hybridSearchOf(params, context, limit, offset, cursorDepth) {
24474
23938
  if ((params.hybrid?.depth ?? null) === null) return executeHybridSearch(params, context, limit, offset, cursorDepth);
24475
- return import('./runtime-3X7ZJ5T3.mjs').then(
23939
+ return import('./runtime-VEUH674B.mjs').then(
24476
23940
  (runtime) => runtime.searchFixedDepthHybrid(params, context, { cursorDepth: cursorDepth ?? 0, offset, limit }, runtimeHost)
24477
23941
  );
24478
23942
  }
@@ -24482,7 +23946,7 @@ function executeQuery(request, callerContext) {
24482
23946
  if (loadingPhraseCode === null) return runQuery(request, callerContext);
24483
23947
  return loadingPhraseCode.then(() => runQuery(request, callerContext));
24484
23948
  }
24485
- return import('./runtime-3X7ZJ5T3.mjs').then(
23949
+ return import('./runtime-VEUH674B.mjs').then(
24486
23950
  (runtime) => runtime.executeRerankedQuery(request, callerContext, executeQuery, runtimeHost)
24487
23951
  );
24488
23952
  }
@@ -24841,13 +24305,19 @@ async function enterRead(core, indexName) {
24841
24305
  if (core.rebalancingIndexes.has(indexName)) await yieldToEventLoop();
24842
24306
  }
24843
24307
  async function readUnderOneAnalysis(core, indexName, read) {
24844
- let analysisStale = core.analysisRebuild.isStale(indexName);
24845
- let result = await read();
24846
- while (core.analysisRebuild.isStale(indexName) !== analysisStale) {
24847
- analysisStale = core.analysisRebuild.isStale(indexName);
24848
- result = await read();
24308
+ const manager = core.requireManager(indexName);
24309
+ for (; ; ) {
24310
+ const analysis = manager.analysis;
24311
+ const analysisStale = core.analysisRebuild.isStale(indexName);
24312
+ try {
24313
+ const result = await read();
24314
+ if (manager.analysis === analysis && core.analysisRebuild.isStale(indexName) === analysisStale) {
24315
+ return { result, analysisStale };
24316
+ }
24317
+ } catch (error) {
24318
+ if (manager.analysis === analysis) throw error;
24319
+ }
24849
24320
  }
24850
- return { result, analysisStale };
24851
24321
  }
24852
24322
  function phraseReadinessOf(core, indexName, config) {
24853
24323
  return () => {
@@ -24875,7 +24345,7 @@ async function runEngineQuery(core, indexName, params, options) {
24875
24345
  const manager = core.requireManager(indexName);
24876
24346
  const scoped = scopedParams(params, options);
24877
24347
  await core.pluginRegistry.runHook("beforeSearch", { indexName, params: scoped });
24878
- const rerank = scoped.rerank === void 0 ? void 0 : await import('./runtime-3X7ZJ5T3.mjs');
24348
+ const rerank = scoped.rerank === void 0 ? void 0 : await import('./runtime-VEUH674B.mjs');
24879
24349
  const resolvedParams = await resolveVectorText(
24880
24350
  rerank === void 0 ? scoped : rerank.withRerankQueryText(scoped),
24881
24351
  entry.embeddingAdapter,
@@ -24883,25 +24353,24 @@ async function runEngineQuery(core, indexName, params, options) {
24883
24353
  entry.embeddingAdapterName
24884
24354
  );
24885
24355
  const workerSearch = core.orchestrator.hasWorkerPool() ? core.orchestrator.searchViaWorker.bind(core.orchestrator) : void 0;
24886
- const context = {
24887
- manager,
24888
- language: entry.language,
24889
- config: entry.config,
24890
- workerSearch,
24891
- indexName,
24892
- broadcastStats: broadcastStatsFor(core, options),
24893
- partitionIds: options?.partitionIds,
24894
- rescoreWindow: options?.rescoreWindow,
24895
- indexBm25: options?.indexBm25,
24896
- cursorBinding: lazyQueryBinding(params),
24897
- patternWorkCap: core.patternWorkCap,
24898
- rerank: rerank?.rerankRunnerFor(core),
24899
- requirePhraseReady: phraseReadinessOf(core, indexName, entry.config)
24900
- };
24901
24356
  const { result, analysisStale } = await readUnderOneAnalysis(
24902
24357
  core,
24903
24358
  indexName,
24904
- () => executeQuery(resolvedParams, context)
24359
+ () => executeQuery(resolvedParams, {
24360
+ manager,
24361
+ language: manager.language,
24362
+ config: manager.config,
24363
+ workerSearch,
24364
+ indexName,
24365
+ broadcastStats: broadcastStatsFor(core, options),
24366
+ partitionIds: options?.partitionIds,
24367
+ rescoreWindow: options?.rescoreWindow,
24368
+ indexBm25: options?.indexBm25,
24369
+ cursorBinding: lazyQueryBinding(params),
24370
+ patternWorkCap: core.patternWorkCap,
24371
+ rerank: rerank?.rerankRunnerFor(core),
24372
+ requirePhraseReady: phraseReadinessOf(core, indexName, entry.config)
24373
+ })
24905
24374
  );
24906
24375
  if (analysisStale) {
24907
24376
  result.analysisStale = true;
@@ -24935,23 +24404,22 @@ async function runEnginePreflight(core, indexName, params, options) {
24935
24404
  entry.embeddingAdapterName
24936
24405
  );
24937
24406
  const workerSearch = core.orchestrator.hasWorkerPool() ? core.orchestrator.searchViaWorker.bind(core.orchestrator) : void 0;
24938
- const context = {
24939
- manager,
24940
- language: entry.language,
24941
- config: entry.config,
24942
- workerSearch,
24943
- indexName,
24944
- broadcastStats: broadcastStatsFor(core, options),
24945
- partitionIds: options?.partitionIds,
24946
- indexBm25: options?.indexBm25,
24947
- cursorBinding: lazyQueryBinding(params),
24948
- patternWorkCap: core.patternWorkCap,
24949
- requirePhraseReady: phraseReadinessOf(core, indexName, entry.config)
24950
- };
24951
24407
  const { result, analysisStale } = await readUnderOneAnalysis(
24952
24408
  core,
24953
24409
  indexName,
24954
- () => executePreflight(resolvedParams, context)
24410
+ () => executePreflight(resolvedParams, {
24411
+ manager,
24412
+ language: manager.language,
24413
+ config: manager.config,
24414
+ workerSearch,
24415
+ indexName,
24416
+ broadcastStats: broadcastStatsFor(core, options),
24417
+ partitionIds: options?.partitionIds,
24418
+ indexBm25: options?.indexBm25,
24419
+ cursorBinding: lazyQueryBinding(params),
24420
+ patternWorkCap: core.patternWorkCap,
24421
+ requirePhraseReady: phraseReadinessOf(core, indexName, entry.config)
24422
+ })
24955
24423
  );
24956
24424
  if (analysisStale) {
24957
24425
  result.analysisStale = true;
@@ -25164,9 +24632,20 @@ function createNarsilFromCore(core, config) {
25164
24632
  return runEngineSuggest(core, indexName, params);
25165
24633
  },
25166
24634
  async rebuildAnalysis(indexName) {
24635
+ guardShutdown();
24636
+ const entry = requireIndex(indexName);
24637
+ await withOpenIndexWrite(indexName, async () => {
24638
+ if (entry.pendingAnalysis === null) return core.analysisRebuild.rebuild(indexName);
24639
+ return (await import('./analysis-change-57MIRJWR.mjs')).resumeAnalysisChange(core, indexName);
24640
+ });
24641
+ },
24642
+ async changeAnalysis(indexName, change) {
25167
24643
  guardShutdown();
25168
24644
  requireIndex(indexName);
25169
- await withOpenIndexWrite(indexName, () => core.analysisRebuild.rebuild(indexName));
24645
+ await withOpenIndexWrite(
24646
+ indexName,
24647
+ async () => (await import('./analysis-change-57MIRJWR.mjs')).changeIndexAnalysis(core, indexName, change)
24648
+ );
25170
24649
  },
25171
24650
  ...portabilityOperations(core, config, () => narsil),
25172
24651
  ...synonymOperations(core),
@@ -25312,4 +24791,4 @@ async function createNarsil(config) {
25312
24791
  // src/index.ts
25313
24792
  var VERSION = "0.1.0";
25314
24793
 
25315
- export { ARRAY_POSITION_GAP, BYTES_TAG, DATE_TAG, DEFAULT_BM25_B, DEFAULT_BM25_K1, DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, HEADER_SIZE, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, NUMBER_TAG, OBJECT_TAG, VECTOR_PATTERN, VERSION, arrayPositionsAreStale, buildSegmentPayload, clearNormalizationCache, configureNormalizationCache, crc32, createByteReader, createFrozenSegment, createNarsil, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, decodeEveryStoredDocument, decodeIndexConfiguration, decodeMapWithoutFields, decodeStoredDocument, encodeBase64, encodeDocument, encodeIndexConfiguration, engineVersionParts, extractVectorFieldsFromSchema, flattenSchema, fnv1a, freezeSegmentShared, getLanguage, getNestedValue, getNormalizationCacheSize, getStopWords, getTokenizer, hasNrslMagic, hasStopWords, hasTokenizer, insertDocumentVectors, isCompositePartition, isQuantizationMode, isStorageMode, loadVectorCodeFor, mergeSegmentContents, messagePackUpperBoundBytes, openChecksummedEnvelope, packChecksummedEnvelope, parseVectorDimension, partialFieldsOf, partitionStateOf, prepareDocumentVectors, prepareStoredSynonyms, raiseEarlierReleaseGraphSetting, readHeader, readHeapStatistics, readResidentMemory, reconstructSchemaFromMetadata, registerLanguage, registerPartialWordSearch, registerPatternSearch, registerStopWords, registerSynonyms, registerTokenizer, rejectSnapshot, removeDocumentVectors, requireSurfaceFormsForPartialFields, requireValidFieldTypes, resetNormalizationCache, resolveIndexAnalysis, resolvePartitionInsertOptions, restoredConfigFields, restoredEmbedding, rulesOfStoredSynonyms, segmentArraysOf, selectionKeeps, shareSegmentArrays, startsWithTagPrefix, storedArrayPositionGap, survivorDocumentsOf, validateEmbeddingConfig, validatePatternValueLimit, validateRequiredFieldsInSchema, validateSchema, validateVectorPromotion, validateVectorStorage, writeHeader };
24794
+ export { ARRAY_POSITION_GAP, DEFAULT_BM25_B, DEFAULT_BM25_K1, HEADER_SIZE, MAX_DOCUMENT_NESTING_DEPTH, VECTOR_PATTERN, VERSION, arrayPositionsAreStale, buildSegmentPayload, clearNormalizationCache, configureNormalizationCache, crc32, createFrozenSegment, createNarsil, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, decodeEveryStoredDocument, decodeIndexConfiguration, decodeMapWithoutFields, decodeStoredDocument, emitEngineEvent, encodeDocument, encodeIndexConfiguration, engineVersionParts, extractVectorFieldsFromSchema, flattenSchema, fnv1a, freezeSegmentShared, getLanguage, getNestedValue, getNormalizationCacheSize, getStopWords, getTokenizer, hasNrslMagic, hasStopWords, hasTokenizer, insertDocumentVectors, isCompositePartition, isQuantizationMode, isStorageMode, loadVectorCodeFor, mergeSegmentContents, messagePackUpperBoundBytes, openChecksummedEnvelope, packChecksummedEnvelope, parseVectorDimension, partialFieldsOf, partitionStateOf, prepareConfiguredSynonyms, prepareDocumentVectors, prepareStoredSynonyms, raiseEarlierReleaseGraphSetting, readHeader, readHeapStatistics, readResidentMemory, rebuildTextIndexes, reconstructSchemaFromMetadata, registerLanguage, registerPartialWordSearch, registerPatternSearch, registerStopWords, registerSynonyms, registerTokenizer, removeDocumentVectors, requireConfigurationShape, requireConfigurationWithinSizeLimit, requireSurfaceFormsForPartialFields, requireValidFieldTypes, resetNormalizationCache, resolveIndexAnalysis, resolvePartialWordAnalysis, resolvePartitionInsertOptions, rulesOfStoredSynonyms, segmentArraysOf, selectionKeeps, shareSegmentArrays, storedArrayPositionGap, survivorDocumentsOf, textRebuildTargetOf, validateEmbeddingConfig, validatePatternValueLimit, validateRequiredFieldsInSchema, validateSchema, validateVectorPromotion, validateVectorStorage, writeHeader };