@delali/narsil 0.3.1-next.130 → 0.3.1-next.140

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. package/dist/adapters/filesystem.mjs +1 -1
  2. package/dist/adapters/indexeddb.mjs +1 -1
  3. package/dist/analysis-change-JN23AZQP.mjs +43 -0
  4. package/dist/analysis-change-entry-5JSIMEKG.mjs +1 -0
  5. package/dist/browser/analysis-change-57MIRJWR.mjs +499 -0
  6. package/dist/browser/analysis-change-entry-ICGRR3IN.mjs +1 -0
  7. package/dist/browser/{chunk-UDGB2MWT.mjs → chunk-35B42UFM.mjs} +4 -0
  8. package/dist/browser/{chunk-VBBLTD7O.mjs → chunk-3EYDURFD.mjs} +39 -27
  9. package/dist/browser/{chunk-AIIJDWMG.mjs → chunk-4MFH226Q.mjs} +1 -1
  10. package/dist/browser/{chunk-D7WQF6DV.mjs → chunk-5LYWZJE2.mjs} +2 -2
  11. package/dist/browser/chunk-67KSBUYR.mjs +173 -0
  12. package/dist/browser/{chunk-M5UAAXGU.mjs → chunk-6BSB4N6B.mjs} +2 -2
  13. package/dist/browser/{chunk-NSCSR6MA.mjs → chunk-CGVBGHBX.mjs} +81 -6
  14. package/dist/browser/chunk-E5SFTOQR.mjs +21 -0
  15. package/dist/browser/{chunk-VEGHFP2P.mjs → chunk-MZI3TEOW.mjs} +4 -2
  16. package/dist/browser/chunk-OIZS5KIX.mjs +6 -0
  17. package/dist/browser/chunk-QHNNULPW.mjs +265 -0
  18. package/dist/browser/chunk-XVMOQQ34.mjs +384 -0
  19. package/dist/browser/{chunk-2Y3CO4EQ.mjs → chunk-ZAIZQOTM.mjs} +256 -777
  20. package/dist/browser/{durability-wiring-FNZSRH4N.mjs → durability-wiring-MTIAN6VT.mjs} +145 -50
  21. package/dist/browser/import-Q7MXD6BS.mjs +8 -0
  22. package/dist/browser/{module-4HIMOQM5.mjs → module-5J3PEUEB.mjs} +7 -6
  23. package/dist/browser/{phrase-module-WESYVU7K.mjs → phrase-module-HGVBO24W.mjs} +1 -1
  24. package/dist/{restore-queue-JAMNRC6T.mjs → browser/restore-queue-DISFECE4.mjs} +8 -2
  25. package/dist/browser/{runtime-3X7ZJ5T3.mjs → runtime-VEUH674B.mjs} +2 -2
  26. package/dist/browser/{snapshot-V7VF6BN6.mjs → snapshot-YBJ5MF2M.mjs} +36 -167
  27. package/dist/browser/{write-YIC2MKRY.mjs → write-RTWI5MXL.mjs} +8 -6
  28. package/dist/{capabilities-Bl8G9taj.d.ts → capabilities-DXAmysqJ.d.ts} +11 -11
  29. package/dist/{chunk-NMXRXFGZ.mjs → chunk-4GCN34JV.mjs} +2 -2
  30. package/dist/{chunk-SOHMMNCH.mjs → chunk-6VHWXAOI.mjs} +4 -4
  31. package/dist/{chunk-2C3WXBQS.mjs → chunk-74XB2SXI.mjs} +1 -1
  32. package/dist/{chunk-QQTXT65X.mjs → chunk-B5I3IL4H.mjs} +12 -7
  33. package/dist/chunk-BDIDUW2B.mjs +387 -0
  34. package/dist/{chunk-5LX6MOIB.mjs → chunk-C3YDYX6C.mjs} +1 -1
  35. package/dist/{chunk-Y4BVTQ53.mjs → chunk-CCCGEPJM.mjs} +2 -2
  36. package/dist/{chunk-2JWNJSNE.mjs → chunk-CPCGEIOM.mjs} +11 -8
  37. package/dist/chunk-CZQ33DLW.mjs +78 -0
  38. package/dist/{chunk-4GDLFDPZ.mjs → chunk-DD2673TX.mjs} +3 -2
  39. package/dist/{chunk-RNSC4NFI.mjs → chunk-EEBCDP36.mjs} +3 -1
  40. package/dist/{chunk-HCGJKDZV.mjs → chunk-FFVLPCLH.mjs} +1 -1
  41. package/dist/{chunk-WVRDRLIC.mjs → chunk-FKARMUWH.mjs} +6 -6
  42. package/dist/{chunk-PATMDIMN.mjs → chunk-HJJWY6PT.mjs} +1 -1
  43. package/dist/chunk-HMDT3I63.mjs +21 -0
  44. package/dist/{chunk-CZ7ISLXA.mjs → chunk-HNHCNTVG.mjs} +4 -0
  45. package/dist/{chunk-RWESFSJI.mjs → chunk-IBATGS3J.mjs} +87 -61
  46. package/dist/{chunk-Y2U5VVWX.mjs → chunk-ICW45TTU.mjs} +1 -1
  47. package/dist/{chunk-T4OFG4IQ.mjs → chunk-IP4HJOGC.mjs} +90 -16
  48. package/dist/{chunk-EUCX5K74.mjs → chunk-J3EHATEZ.mjs} +3 -3
  49. package/dist/{chunk-X5CNUKD5.mjs → chunk-K3AQOZVM.mjs} +37 -20
  50. package/dist/chunk-LK3MOTXF.mjs +174 -0
  51. package/dist/{chunk-7OIQ2SIO.mjs → chunk-LLDFIKGP.mjs} +4 -4
  52. package/dist/{chunk-K77FJXKS.mjs → chunk-MTQTX4BP.mjs} +81 -8
  53. package/dist/{chunk-ZR4VM4A6.mjs → chunk-N37VMC62.mjs} +1 -18
  54. package/dist/{chunk-2WBY4Y4M.mjs → chunk-NOT5QOHU.mjs} +1 -1
  55. package/dist/{chunk-D5TFRB5G.mjs → chunk-NPNK57CA.mjs} +1 -1
  56. package/dist/{chunk-6SSVIW7Y.mjs → chunk-O4LJQI4G.mjs} +31 -5
  57. package/dist/{chunk-JJ6EUMXB.mjs → chunk-PNNVTKMT.mjs} +2 -2
  58. package/dist/{chunk-2SEL6SBS.mjs → chunk-RKAVHNRW.mjs} +2 -2
  59. package/dist/{chunk-XABMCBHA.mjs → chunk-SBMAWTRG.mjs} +684 -99
  60. package/dist/chunk-TCWQJL7O.mjs +188 -0
  61. package/dist/{chunk-2MKX2SCO.mjs → chunk-U4X4AWKL.mjs} +7 -7
  62. package/dist/{chunk-PU6SKH7O.mjs → chunk-UBXJQJII.mjs} +2 -2
  63. package/dist/chunk-UEV53BBO.mjs +480 -0
  64. package/dist/{chunk-NNO7CO3A.mjs → chunk-UIKN2OTQ.mjs} +1 -1
  65. package/dist/{chunk-F6AAXI7T.mjs → chunk-VSUYTPME.mjs} +3 -3
  66. package/dist/{chunk-TGHNXN7W.mjs → chunk-W2XDK66Z.mjs} +180 -541
  67. package/dist/{chunk-7X5JST5L.mjs → chunk-WNBRW4Q2.mjs} +16 -2
  68. package/dist/{chunk-6M6SDH3S.mjs → chunk-XMQNOD5V.mjs} +8 -8
  69. package/dist/{chunk-SZGRYEIC.mjs → chunk-XS2VM7UE.mjs} +3 -3
  70. package/dist/{chunk-JT3YZGCE.mjs → chunk-XXKWEG53.mjs} +3 -188
  71. package/dist/{chunk-YBWSV5DV.mjs → chunk-ZMBFWQXN.mjs} +2 -2
  72. package/dist/{chunk-7XY6VD7B.mjs → chunk-ZSWHPKGM.mjs} +1 -1
  73. package/dist/{client-Cx-TiqPE.d.ts → client-DApBeLbk.d.ts} +18 -3
  74. package/dist/client.d.ts +7 -8
  75. package/dist/client.mjs +11 -1
  76. package/dist/{events-DErop1LM.d.ts → config-B0H-vp4U.d.ts} +424 -2
  77. package/dist/{config-CzB9YvF_.d.ts → config-Brysl9NO.d.ts} +3 -3
  78. package/dist/{direct-executor-HQ4QVDX7.mjs → direct-executor-J5UDLPL3.mjs} +14 -14
  79. package/dist/distribution/coordinator/etcd.d.ts +2 -2
  80. package/dist/distribution/coordinator/etcd.mjs +1 -1
  81. package/dist/distribution/coordinator/in-memory.d.ts +2 -2
  82. package/dist/distribution/transport/grpc.mjs +1 -1
  83. package/dist/distribution.d.ts +24 -9
  84. package/dist/distribution.mjs +882 -192
  85. package/dist/{durability-wiring-S7OX2MAI.mjs → durability-wiring-AYOTAPNQ.mjs} +165 -70
  86. package/dist/embeddings/openai.mjs +2 -2
  87. package/dist/{engine-B_W9sdZK.d.ts → engine-CuLMbH1-.d.ts} +64 -7
  88. package/dist/import-QRZAQ5RH.mjs +43 -0
  89. package/dist/{index-CKI1fDEl.d.ts → index-HYnqS1SE.d.ts} +2 -2
  90. package/dist/index.browser.mjs +4 -4
  91. package/dist/index.d.ts +6 -7
  92. package/dist/index.mjs +20 -20
  93. package/dist/{long-query-thread-WVSMIBYP.mjs → long-query-thread-U7K5A72G.mjs} +14 -14
  94. package/dist/{module-2E4XYSGT.mjs → module-L2OZO5PQ.mjs} +11 -11
  95. package/dist/partial-words.mjs +5 -5
  96. package/dist/pattern.mjs +3 -3
  97. package/dist/persistence/durability/checkpoint-worker.d.ts +12 -1
  98. package/dist/persistence/durability/checkpoint-worker.mjs +23 -23
  99. package/dist/{phrase-module-JERF2IDG.mjs → phrase-module-5QHBD2VJ.mjs} +1 -1
  100. package/dist/react.d.ts +11 -8
  101. package/dist/react.mjs +4 -0
  102. package/dist/{rerank-DMMT4FZA.mjs → rerank-KTLC3E6N.mjs} +5 -5
  103. package/dist/rerankers/http.mjs +2 -2
  104. package/dist/{browser/restore-queue-GEZ54CA5.mjs → restore-queue-RYSRRFS4.mjs} +8 -2
  105. package/dist/{runtime-7MLQ7J64.mjs → runtime-DYV75DCB.mjs} +6 -6
  106. package/dist/{schema-CS-clLun.d.ts → schema-BTXbHCqA.d.ts} +4 -1
  107. package/dist/server.d.ts +200 -11
  108. package/dist/server.mjs +66 -23
  109. package/dist/{snapshot-DOQLGGEX.mjs → snapshot-BTRZ3MZK.mjs} +59 -190
  110. package/dist/synonyms.mjs +6 -4
  111. package/dist/tasks-DjGjK70Y.d.ts +720 -0
  112. package/dist/{thread-DNK5XWXG.mjs → thread-USB3GOPY.mjs} +13 -13
  113. package/dist/vector/search-worker.d.ts +1 -1
  114. package/dist/vector/search-worker.mjs +5 -5
  115. package/dist/workers/entry.mjs +7 -7
  116. package/dist/{write-ZKR7SNRT.mjs → write-GHTA56OW.mjs} +25 -22
  117. package/openapi.json +5310 -0
  118. package/package.json +16 -8
  119. package/dist/config-hOWGBeC6.d.ts +0 -414
  120. package/dist/tasks-BfF-VRhm.d.ts +0 -302
@@ -1,27 +1,27 @@
1
- import { validateWorkerConfig, validateIndexName, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateBM25Params, validatePartitionConfig, lazyQueryBinding, executeQuery, executePreflight, executeSuggest, executeListDocuments, storedBM25Params, validateDocId } from './chunk-6M6SDH3S.mjs';
2
- import { resolveVectorText, bindEngineCore, restoredConfigFields, restoredEmbedding, collectRequiredFieldFailures, embedChunkDocuments, providedDocId, requireDocumentObject, admitInsert, asBatchInsertError, requireMatchingDocId, embedDocumentFields, assertDocumentCarriesMappedVectors, admitBufferedUpsert, createPendingAdmissions, recordPendingAdmission } from './chunk-JT3YZGCE.mjs';
3
- import { splitWorkerBudget, resolveWorkerCount, resolveWorkerEntry, missingWorkerEntry, DEFAULT_REQUEST_TIMEOUT_MS, resolveRequestThreadCount, workerResourceLimits, DEFAULT_BACKPRESSURE_LIMIT, SHUTDOWN_TIMEOUT_MS, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH } from './chunk-NNO7CO3A.mjs';
1
+ import { validateWorkerConfig, validateIndexName, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateBM25Params, validatePartitionConfig, executeQuery, lazyQueryBinding, executePreflight, executeSuggest, executeListDocuments, storedBM25Params, validateDocId } from './chunk-XMQNOD5V.mjs';
2
+ import { resolveVectorText, bindEngineCore, collectRequiredFieldFailures, embedChunkDocuments, providedDocId, requireDocumentObject, admitInsert, asBatchInsertError, requireMatchingDocId, embedDocumentFields, assertDocumentCarriesMappedVectors, admitBufferedUpsert, createPendingAdmissions, recordPendingAdmission } from './chunk-XXKWEG53.mjs';
3
+ import { splitWorkerBudget, resolveWorkerCount, resolveWorkerEntry, missingWorkerEntry, DEFAULT_REQUEST_TIMEOUT_MS, resolveRequestThreadCount, workerResourceLimits, DEFAULT_BACKPRESSURE_LIMIT, SHUTDOWN_TIMEOUT_MS, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH } from './chunk-UIKN2OTQ.mjs';
4
4
  import { detectRuntime } from './chunk-AV4IJKD2.mjs';
5
- import { isEmptySynonymSet, synonymModule, createDirectExecutor, createPartitionRouter, availableSynonymModule, mergeFrozenSegments, createSharedFrozenSegment, freezeSegmentShared, resolvePartitionInsertOptions, createFrozenSegment } from './chunk-RWESFSJI.mjs';
6
- import { ARRAY_POSITION_GAP, requirePatternSearchFor, requirePartialWordSearchFor, loadPhraseCodeFor, collectQueryTermStats, requireLoadedPhraseCode, mergePartitionStats, createPartitionIndex, oversampledFacetConfig, availablePatternSearch, PATTERN_HOST, kWayMerge, mergeFacets, isCompositePartition, fnv1a, encodeSegmentState } from './chunk-2MKX2SCO.mjs';
7
- import { generateId, getLanguage } from './chunk-ZR4VM4A6.mjs';
5
+ import { resolvePartialWordAnalysis, resolvePartitionInsertOptions, isEmptySynonymSet, synonymModule, createDirectExecutor, createPartitionRouter, availableSynonymModule, mergeFrozenSegments, createSharedFrozenSegment, freezeSegmentShared, createFrozenSegment } from './chunk-IBATGS3J.mjs';
6
+ import { ARRAY_POSITION_GAP, requirePatternSearchFor, requirePartialWordSearchFor, loadPhraseCodeFor, collectQueryTermStats, requireLoadedPhraseCode, createPartitionIndex, mergePartitionStats, oversampledFacetConfig, availablePatternSearch, PATTERN_HOST, kWayMerge, mergeFacets, isCompositePartition, fnv1a, encodeSegmentState } from './chunk-U4X4AWKL.mjs';
7
+ import { generateId } from './chunk-N37VMC62.mjs';
8
8
  import { readHeapStatistics } from './chunk-ZATRDAYU.mjs';
9
9
  import { countEngineThread } from './chunk-EBIHRL7B.mjs';
10
10
  import { DEFAULT_MAX_RERANK_WINDOW, DEFAULT_RERANK_TIMEOUT_MS } from './chunk-HJSN7UYG.mjs';
11
- import { MAX_INDEX_CONFIGURATION_BYTES, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, STATISTICS_BROADCAST_INTERVAL_MS, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, HEAP_PRESSURE_WARNING_FRACTION, FOREIGN_STATS_TTL_MS, MAX_IMPORT_REFUSALS_REPORTED, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION, BATCH_CHUNK_SIZE } from './chunk-RNSC4NFI.mjs';
12
- import { requireSearchBackendSettings } from './chunk-2C3WXBQS.mjs';
13
- import { DEFAULT_PATTERN_WORK_CAP, MATCHER_STATE_BUDGET_BYTES } from './chunk-2SEL6SBS.mjs';
14
- import { getNestedValue } from './chunk-EUCX5K74.mjs';
15
- import { flattenSchema, raiseEarlierReleaseGraphSetting, messagePackUpperBoundBytes, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, schemaFieldsOf, extractVectorFieldsFromSchema, validateRequiredFields, requireDocumentWithinSizeLimit, validateDocument, validateDocumentStrict } from './chunk-PU6SKH7O.mjs';
11
+ import { MAX_INDEX_CONFIGURATION_BYTES, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, STATISTICS_BROADCAST_INTERVAL_MS, HEAP_PRESSURE_WARNING_FRACTION, FOREIGN_STATS_TTL_MS, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION, BATCH_CHUNK_SIZE } from './chunk-EEBCDP36.mjs';
12
+ import { requireSearchBackendSettings } from './chunk-74XB2SXI.mjs';
13
+ import { DEFAULT_PATTERN_WORK_CAP, MATCHER_STATE_BUDGET_BYTES } from './chunk-RKAVHNRW.mjs';
14
+ import { getNestedValue } from './chunk-J3EHATEZ.mjs';
15
+ import { flattenSchema, raiseEarlierReleaseGraphSetting, messagePackUpperBoundBytes, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, schemaFieldsOf, extractVectorFieldsFromSchema, validateRequiredFields, requireDocumentWithinSizeLimit, validateDocument, validateDocumentStrict } from './chunk-UBXJQJII.mjs';
16
16
  import { NATIVE_CRC32_MIN_BYTES, MAX_FILE_BYTES, CHECKSUM_YIELD_CHUNK_BYTES } from './chunk-U476IXKQ.mjs';
17
- import { isSingleStringFieldType } from './chunk-7XY6VD7B.mjs';
18
- import { withOpenIndex, oneAtATime } from './chunk-6SSVIW7Y.mjs';
17
+ import { isSingleStringFieldType } from './chunk-ZSWHPKGM.mjs';
18
+ import { withOpenIndex, oneAtATime } from './chunk-O4LJQI4G.mjs';
19
+ import { getLanguage } from './chunk-WNBRW4Q2.mjs';
19
20
  import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
20
21
  import { createRequestId, isValidWorkerResponse } from './chunk-UJWGQIXU.mjs';
21
- import { loadVectorCodeFor, requireLoadedVectorCode } from './chunk-HCGJKDZV.mjs';
22
+ import { loadVectorCodeFor, requireLoadedVectorCode } from './chunk-FFVLPCLH.mjs';
22
23
  import { noteThreadSlotLive, threadSlotOfWorker, noteThreadSlotGone } from './chunk-RFBLQLLZ.mjs';
23
- import { MAX_DOCUMENT_NESTING_DEPTH } from './chunk-L5ORED7V.mjs';
24
- import { ErrorCodes, NarsilError } from './chunk-CZ7ISLXA.mjs';
24
+ import { ErrorCodes, NarsilError } from './chunk-HNHCNTVG.mjs';
25
25
  import { encode, decode } from '@msgpack/msgpack';
26
26
  import * as zlib from 'zlib';
27
27
 
@@ -480,11 +480,14 @@ function validateRequiredFieldsInSchema(required, schema) {
480
480
  }
481
481
 
482
482
  // src/partitioning/text-rebuild.ts
483
+ function textRebuildTargetOf(config, language, analysis) {
484
+ return { config, language, analysis, partialWordAnalysis: resolvePartialWordAnalysis(config, language, analysis) };
485
+ }
483
486
  function interrupted(pass) {
484
487
  return pass.log.partitionsReplaced || !pass.hooks.isCurrent();
485
488
  }
486
489
  async function buildPartition(pass, source, partitionId) {
487
- const { manager, options } = pass;
490
+ const { manager, language, options } = pass;
488
491
  const target = createPartitionIndex(partitionId, manager.config.trackPositions ?? true);
489
492
  const docIds = [...source.docIds()];
490
493
  target.beginBatch();
@@ -492,27 +495,27 @@ async function buildPartition(pass, source, partitionId) {
492
495
  const end = Math.min(start + TEXT_REBUILD_CHUNK_DOCUMENTS, docIds.length);
493
496
  for (let i = start; i < end; i++) {
494
497
  const document = manager.getRef(docIds[i]);
495
- if (document !== void 0) target.insert(docIds[i], document, manager.schema, manager.language, options);
498
+ if (document !== void 0) target.insert(docIds[i], document, manager.schema, language, options);
496
499
  }
497
500
  await yieldToEventLoop();
498
501
  if (interrupted(pass)) return null;
499
502
  }
500
503
  target.endBatch();
501
- if (manager.partialWordAnalysis !== null) target.preparePartialWords(manager.partialWordAnalysis);
504
+ if (pass.partialWordAnalysis !== null) target.preparePartialWords(pass.partialWordAnalysis);
502
505
  return target;
503
506
  }
504
507
  function applyWrittenDocuments(pass, built) {
505
- const { manager, options } = pass;
508
+ const { manager, language, options } = pass;
506
509
  for (const docId of pass.log.takeWrittenDocIds()) {
507
510
  for (const partition of built) {
508
511
  if (!partition.has(docId)) continue;
509
- partition.remove(docId, manager.schema, manager.language, options);
512
+ partition.remove(docId, manager.schema, language, options);
510
513
  break;
511
514
  }
512
515
  const partitionId = manager.partitionIdOf(docId);
513
516
  const document = manager.getRef(docId);
514
517
  if (partitionId === void 0 || document === void 0) continue;
515
- built[partitionId].insert(docId, document, manager.schema, manager.language, options);
518
+ built[partitionId].insert(docId, document, manager.schema, language, options);
516
519
  }
517
520
  }
518
521
  async function rebuildOnce(pass) {
@@ -529,17 +532,36 @@ async function rebuildOnce(pass) {
529
532
  await yieldToEventLoop();
530
533
  if (interrupted(pass)) return false;
531
534
  }
532
- applyWrittenDocuments(pass, built);
533
- pass.manager.adoptRebuiltPartitions(built);
534
- pass.hooks.onSwitch();
535
- return true;
535
+ const keepCurrent = () => {
536
+ if (interrupted(pass)) return false;
537
+ applyWrittenDocuments(pass, built);
538
+ return true;
539
+ };
540
+ if (pass.hooks.holdBeforeSwitch !== void 0 && !await pass.hooks.holdBeforeSwitch(keepCurrent)) return false;
541
+ const swap = () => {
542
+ if (interrupted(pass)) return false;
543
+ applyWrittenDocuments(pass, built);
544
+ pass.manager.adoptRebuiltPartitions(built);
545
+ pass.hooks.onSwitch();
546
+ return true;
547
+ };
548
+ return pass.hooks.runSwitch === void 0 ? swap() : pass.hooks.runSwitch(swap);
536
549
  }
537
- async function rebuildTextIndexes(manager, hooks) {
538
- const options = resolvePartitionInsertOptions(manager.config, manager.analysis, { validate: false, skipClone: true });
550
+ async function rebuildTextIndexes(manager, hooks, target) {
551
+ const options = resolvePartitionInsertOptions(
552
+ target?.config ?? manager.config,
553
+ target?.analysis ?? manager.analysis,
554
+ {
555
+ validate: false,
556
+ skipClone: true
557
+ }
558
+ );
559
+ const language = target?.language ?? manager.language;
560
+ const partialWordAnalysis = target === void 0 ? manager.partialWordAnalysis : target.partialWordAnalysis;
539
561
  while (hooks.isCurrent()) {
540
562
  const log = manager.openWriteLog();
541
563
  try {
542
- if (await rebuildOnce({ manager, log, hooks, options })) return true;
564
+ if (await rebuildOnce({ manager, log, hooks, language, partialWordAnalysis, options })) return true;
543
565
  } finally {
544
566
  log.close();
545
567
  }
@@ -577,6 +599,7 @@ function wireAnalysisRebuild(wiring) {
577
599
  getManager: wiring.getManager,
578
600
  desyncIndex: wiring.desyncIndex,
579
601
  resyncIndex: wiring.resyncIndex,
602
+ hasPendingChange: wiring.hasPendingChange,
580
603
  async persistAnalysisRevision(indexName) {
581
604
  const durability = wiring.durabilityManager;
582
605
  if (durability === null) {
@@ -594,6 +617,7 @@ function wireAnalysisRebuild(wiring) {
594
617
  }
595
618
  function createAnalysisRebuildCoordinator(config, deps) {
596
619
  const stale = /* @__PURE__ */ new Map();
620
+ const exclusiveRuns = /* @__PURE__ */ new Map();
597
621
  const rebuildMode = config?.rebuild ?? "auto";
598
622
  let queue = Promise.resolve();
599
623
  let active = null;
@@ -669,7 +693,7 @@ function createAnalysisRebuildCoordinator(config, deps) {
669
693
  await config?.onStaleAnalysis?.({ ...entry.index, documentCount }, () => enqueue(indexName, entry));
670
694
  }
671
695
  async function review(indexName, entry) {
672
- if (entry.run !== null || deps.getManager(indexName) === void 0) return;
696
+ if (entry.run !== null || deps.getManager(indexName) === void 0 || deps.hasPendingChange(indexName)) return;
673
697
  try {
674
698
  await announce(indexName, entry);
675
699
  } catch (err) {
@@ -701,8 +725,22 @@ function createAnalysisRebuildCoordinator(config, deps) {
701
725
  return stale.has(indexName);
702
726
  },
703
727
  isRunning(indexName) {
728
+ if ((exclusiveRuns.get(indexName) ?? 0) > 0) return true;
704
729
  return stale.get(indexName)?.run !== null && stale.get(indexName)?.run !== void 0;
705
730
  },
731
+ runExclusive(indexName, work, turnEnds) {
732
+ exclusiveRuns.set(indexName, (exclusiveRuns.get(indexName) ?? 0) + 1);
733
+ const run = queue.then(work).finally(() => {
734
+ const remaining = (exclusiveRuns.get(indexName) ?? 1) - 1;
735
+ if (remaining === 0) exclusiveRuns.delete(indexName);
736
+ else exclusiveRuns.set(indexName, remaining);
737
+ });
738
+ queue = Promise.race([run, turnEnds ?? run]).then(
739
+ () => void 0,
740
+ () => void 0
741
+ );
742
+ return run;
743
+ },
706
744
  async abandon(indexName) {
707
745
  const entry = stale.get(indexName);
708
746
  if (entry !== void 0 && entry.run !== null) entry.abandoned = true;
@@ -958,6 +996,10 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
958
996
  if (loadingVectorCode !== null) await loadingVectorCode;
959
997
  const existing = deps.indexRegistry.get(metadata.indexName);
960
998
  if (existing !== void 0) {
999
+ if ((metadata.analysisGeneration ?? 0) > existing.analysisGeneration) {
1000
+ const { adoptSavedAnalysis } = await import('./analysis-change-JN23AZQP.mjs');
1001
+ adoptSavedAnalysis(deps.executor.getManager(metadata.indexName), existing, metadata);
1002
+ }
961
1003
  if (loadData && deps.executor.getManager(metadata.indexName) === void 0) {
962
1004
  deps.executor.createIndex(
963
1005
  metadata.indexName,
@@ -1006,7 +1048,10 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
1006
1048
  partitionCount: metadata.partitionCount,
1007
1049
  analysisRevision: metadata.analysisRevision ?? null,
1008
1050
  arrayPositionGap: metadata.arrayPositionGap ?? null,
1009
- synonyms
1051
+ synonyms,
1052
+ analysisGeneration: metadata.analysisGeneration ?? 0,
1053
+ pendingAnalysis: metadata.pendingAnalysis ?? null,
1054
+ durableAnalysis: null
1010
1055
  });
1011
1056
  if (loadData) await deps.indexState().registerOpen(metadata.indexName);
1012
1057
  else deps.indexState().registerClosed(metadata.indexName);
@@ -1406,6 +1451,7 @@ function wireIndexState(wiring) {
1406
1451
  }
1407
1452
  try {
1408
1453
  await durability.manager.recoverIndex(indexName);
1454
+ await wiring.resumeChange(indexName);
1409
1455
  await analysisRebuild.review(indexName);
1410
1456
  await orchestrator.openIndex(indexName);
1411
1457
  await wiring.onOpen?.(indexName);
@@ -2106,14 +2152,18 @@ function assertConfigReachesWorker(indexName, config, bootstrapModule) {
2106
2152
  { indexName }
2107
2153
  );
2108
2154
  }
2155
+ if (bootstrapModule !== void 0) return;
2109
2156
  const language = config.language ?? "english";
2110
- if (language !== "english" && bootstrapModule === void 0) {
2157
+ const requireRegistered = (setting, name) => {
2111
2158
  throw new NarsilError(
2112
2159
  ErrorCodes.CONFIG_INVALID,
2113
- `Index "${indexName}" uses language "${language}", which a worker thread registers only from a bootstrap module. Set workers.bootstrapModule to a module that registers it`,
2114
- { indexName, language }
2160
+ `Index "${indexName}" uses ${setting} "${name}", which a worker thread registers only from a bootstrap module. Set workers.bootstrapModule to a module that registers it`,
2161
+ { indexName, [setting]: name }
2115
2162
  );
2116
- }
2163
+ };
2164
+ if (language !== "english") requireRegistered("language", language);
2165
+ if (typeof config.tokenizer === "string") requireRegistered("tokenizer", config.tokenizer);
2166
+ if (typeof config.stopWords === "string") requireRegistered("stopWords", config.stopWords);
2117
2167
  }
2118
2168
  function workerIneligibility(indexName, config, bootstrapModule) {
2119
2169
  try {
@@ -4064,7 +4114,7 @@ async function emitWriteEventAfter(source, indexName, write) {
4064
4114
  // src/engine/core.ts
4065
4115
  async function createEngineCore(config, hooks, capacityScope = "index") {
4066
4116
  validateWorkerConfig(config?.workers, config?.lifecycle);
4067
- const rerankSettings = config?.rerankTimeout === void 0 && config?.maxRerankWindow === void 0 ? { timeoutMs: DEFAULT_RERANK_TIMEOUT_MS, maxWindow: DEFAULT_MAX_RERANK_WINDOW } : (await import('./runtime-7MLQ7J64.mjs')).resolveRerankSettings(config);
4117
+ const rerankSettings = config?.rerankTimeout === void 0 && config?.maxRerankWindow === void 0 ? { timeoutMs: DEFAULT_RERANK_TIMEOUT_MS, maxWindow: DEFAULT_MAX_RERANK_WINDOW } : (await import('./runtime-DYV75DCB.mjs')).resolveRerankSettings(config);
4068
4118
  const eventHandlers = eventHandlersOf(config?.events);
4069
4119
  const vectorWorkerCount = splitWorkerBudget(resolveWorkerCount(config?.workers?.count)).vector;
4070
4120
  const vectorCopyPolicy = {
@@ -4075,7 +4125,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
4075
4125
  if (config?.lifecycle !== void 0 && durabilityTier === null) {
4076
4126
  throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index lifecycle settings require durability");
4077
4127
  }
4078
- const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-S7OX2MAI.mjs');
4128
+ const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-AYOTAPNQ.mjs');
4079
4129
  const filesystemDurability = durabilityTier?.kind === "wal";
4080
4130
  const executor = createDirectExecutor({
4081
4131
  vectorWorkerCopies: vectorCopyPolicy,
@@ -4245,6 +4295,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
4245
4295
  entry.analysisRevision = entry.language.revision;
4246
4296
  entry.arrayPositionGap = ARRAY_POSITION_GAP;
4247
4297
  },
4298
+ hasPendingChange: (indexName) => (indexRegistry.get(indexName)?.pendingAnalysis ?? null) !== null,
4248
4299
  durabilityManager: durability?.manager ?? null
4249
4300
  });
4250
4301
  const indexState = wireIndexState({
@@ -4256,6 +4307,10 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
4256
4307
  indexRegistry,
4257
4308
  rebalancingIndexes,
4258
4309
  requireManager,
4310
+ resumeChange: async (indexName) => {
4311
+ if ((indexRegistry.get(indexName)?.pendingAnalysis ?? null) === null) return;
4312
+ await (await import('./analysis-change-entry-5JSIMEKG.mjs')).resumeChangeOf(engineCore, indexName);
4313
+ },
4259
4314
  onClose: hooks?.onIndexClose,
4260
4315
  onOpen: (indexName) => {
4261
4316
  heapPressureNotifier.check(indexName);
@@ -4296,7 +4351,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
4296
4351
  checkWatermark: watermarkNotifier.check,
4297
4352
  requireIndex
4298
4353
  };
4299
- return {
4354
+ const engineCore = {
4300
4355
  executor,
4301
4356
  pluginRegistry,
4302
4357
  durability,
@@ -4327,6 +4382,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
4327
4382
  rebalanceCtx,
4328
4383
  patternWorkCap: config?.patternWorkCap ?? DEFAULT_PATTERN_WORK_CAP
4329
4384
  };
4385
+ return engineCore;
4330
4386
  }
4331
4387
 
4332
4388
  // src/engine/lifecycle.ts
@@ -4337,6 +4393,7 @@ async function startEngineCore(core, config) {
4337
4393
  checkHeapAfterRecovery(core);
4338
4394
  }
4339
4395
  if (core.invalidation) await core.invalidation.start();
4396
+ if (core.durability) await (await import('./analysis-change-entry-5JSIMEKG.mjs')).resumeEveryChange(core);
4340
4397
  if (config?.lifecycle === void 0) await core.analysisRebuild.reviewStaleIndexes();
4341
4398
  } catch (err) {
4342
4399
  await core.durability?.manager.shutdown().catch(() => void 0);
@@ -4374,7 +4431,7 @@ async function shutdownEngine(core) {
4374
4431
  }
4375
4432
  }
4376
4433
  if (core.rerankAdapters.size > 0) {
4377
- await (await import('./runtime-7MLQ7J64.mjs')).shutdownRerankAdapters(core.rerankAdapters);
4434
+ await (await import('./runtime-DYV75DCB.mjs')).shutdownRerankAdapters(core.rerankAdapters);
4378
4435
  }
4379
4436
  await orchestrator.shutdown();
4380
4437
  await executor.shutdown();
@@ -4711,7 +4768,8 @@ function indexConfigurationOf(entry) {
4711
4768
  entry.embeddingAdapterName,
4712
4769
  entry.analysisRevision,
4713
4770
  entry.arrayPositionGap,
4714
- synonyms
4771
+ synonyms,
4772
+ entry.analysisGeneration
4715
4773
  );
4716
4774
  }
4717
4775
  function requireConfigurationWithinSizeLimit(config, language) {
@@ -4732,7 +4790,7 @@ function requireConfigurationWithinSizeLimit(config, language) {
4732
4790
  { bytes, maximum: MAX_INDEX_CONFIGURATION_BYTES }
4733
4791
  );
4734
4792
  }
4735
- function configurationPayloadOf(config, language, adapterName, analysisRevision, arrayPositionGap, synonyms) {
4793
+ function configurationPayloadOf(config, language, adapterName, analysisRevision, arrayPositionGap, synonyms, analysisGeneration = 0) {
4736
4794
  const bm25 = config.bm25;
4737
4795
  return {
4738
4796
  schema: config.schema,
@@ -4742,6 +4800,7 @@ function configurationPayloadOf(config, language, adapterName, analysisRevision,
4742
4800
  ...typeof config.tokenizer === "string" ? { tokenizer: config.tokenizer } : {},
4743
4801
  ...typeof config.stopWords === "string" ? { stopWords: config.stopWords } : {},
4744
4802
  ...config.stopWords instanceof Set ? { stopWordList: [...config.stopWords].sort(compareCodePoints) } : {},
4803
+ ...analysisGeneration > 0 ? { analysisGeneration } : {},
4745
4804
  ...bm25 !== void 0 ? {
4746
4805
  bm25: {
4747
4806
  ...bm25.k1 !== void 0 ? { k1: bm25.k1 } : {},
@@ -4916,7 +4975,10 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
4916
4975
  partitionCount: indexConfig.partitions?.maxPartitions ?? 1,
4917
4976
  analysisRevision: language.revision,
4918
4977
  arrayPositionGap: ARRAY_POSITION_GAP,
4919
- synonyms: synonyms?.encoded ?? null
4978
+ synonyms: synonyms?.encoded ?? null,
4979
+ analysisGeneration: 0,
4980
+ pendingAnalysis: null,
4981
+ durableAnalysis: null
4920
4982
  });
4921
4983
  await core.indexState.registerOpen(name);
4922
4984
  if (core.durability) {
@@ -4927,8 +4989,14 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
4927
4989
  async function dropEngineIndex(core, name) {
4928
4990
  core.guardShutdown();
4929
4991
  const entry = core.requireIndex(name);
4992
+ const changeRecorded = () => entry.pendingAnalysis !== null || entry.durableAnalysis !== null;
4993
+ const abandonChange = async () => {
4994
+ await (await import('./analysis-change-entry-5JSIMEKG.mjs')).abandonChangeOf(core, name);
4995
+ };
4996
+ if (changeRecorded()) await abandonChange();
4930
4997
  await core.indexState.drop(name, async () => {
4931
4998
  await core.analysisRebuild.abandon(name);
4999
+ if (changeRecorded()) await abandonChange();
4932
5000
  await core.orchestrator.closeIndex(name);
4933
5001
  if (core.durability) {
4934
5002
  try {
@@ -6387,6 +6455,14 @@ async function executeRebalance(manager, indexName, targetPartitionCount, ctx) {
6387
6455
  `Index "${indexName}" is already being rebalanced`
6388
6456
  );
6389
6457
  }
6458
+ const registered = ctx.requireIndex(indexName);
6459
+ if (registered.pendingAnalysis !== null || registered.durableAnalysis !== null) {
6460
+ throw new NarsilError(
6461
+ ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
6462
+ `Index "${indexName}" is changing its analysis settings, so try the rebalance again once the change ends`,
6463
+ { indexName }
6464
+ );
6465
+ }
6390
6466
  if (targetPartitionCount === manager.partitionCount) {
6391
6467
  return;
6392
6468
  }
@@ -6570,461 +6646,6 @@ function bm25Operations(core) {
6570
6646
  };
6571
6647
  }
6572
6648
 
6573
- // src/persistence/archive/byte-reader.ts
6574
- async function* chunksOf(source) {
6575
- if (source instanceof Uint8Array) {
6576
- yield source;
6577
- return;
6578
- }
6579
- if (typeof source.getReader === "function") {
6580
- const reader = source.getReader();
6581
- try {
6582
- for (; ; ) {
6583
- const step = await reader.read();
6584
- if (step.done) return;
6585
- yield step.value;
6586
- }
6587
- } finally {
6588
- reader.releaseLock();
6589
- }
6590
- }
6591
- yield* source;
6592
- }
6593
- function createByteReader(source) {
6594
- const iterator = chunksOf(source);
6595
- const pending = [];
6596
- let pendingBytes = 0;
6597
- let ended = false;
6598
- async function fill(length) {
6599
- while (pendingBytes < length && !ended) {
6600
- const step = await iterator.next();
6601
- if (step.done) {
6602
- ended = true;
6603
- break;
6604
- }
6605
- if (step.value.byteLength === 0) continue;
6606
- pending.push(step.value);
6607
- pendingBytes += step.value.byteLength;
6608
- }
6609
- }
6610
- function take(length) {
6611
- const out = new Uint8Array(length);
6612
- let filled = 0;
6613
- while (filled < length) {
6614
- const head = pending[0];
6615
- const needed = length - filled;
6616
- if (head.byteLength <= needed) {
6617
- out.set(head, filled);
6618
- filled += head.byteLength;
6619
- pending.shift();
6620
- } else {
6621
- out.set(head.subarray(0, needed), filled);
6622
- pending[0] = head.subarray(needed);
6623
- filled += needed;
6624
- }
6625
- }
6626
- pendingBytes -= length;
6627
- return out;
6628
- }
6629
- return {
6630
- async readExactly(length) {
6631
- await fill(length);
6632
- if (pendingBytes === 0 && length > 0) return null;
6633
- if (pendingBytes < length) return take(pendingBytes);
6634
- return take(length);
6635
- },
6636
- async peek(length) {
6637
- await fill(length);
6638
- const available = Math.min(length, pendingBytes);
6639
- const copy = take(available);
6640
- pending.unshift(copy);
6641
- pendingBytes += copy.byteLength;
6642
- return copy;
6643
- },
6644
- async close() {
6645
- await iterator.return(void 0);
6646
- }
6647
- };
6648
- }
6649
-
6650
- // src/engine/export/lines.ts
6651
- var NEWLINE = 10;
6652
- function joined(parts, total) {
6653
- if (parts.length === 1) return parts[0];
6654
- const bytes = new Uint8Array(total);
6655
- let offset = 0;
6656
- for (const part of parts) {
6657
- bytes.set(part, offset);
6658
- offset += part.byteLength;
6659
- }
6660
- return bytes;
6661
- }
6662
- async function* exportLines(source, maxLineBytes) {
6663
- const decoder = new TextDecoder("utf-8", { fatal: true });
6664
- let parts = [];
6665
- let pending = 0;
6666
- let overflowing = false;
6667
- let line = 0;
6668
- const finishLine = () => {
6669
- line += 1;
6670
- if (overflowing) {
6671
- overflowing = false;
6672
- parts = [];
6673
- pending = 0;
6674
- return { line, refused: "too-long" };
6675
- }
6676
- const bytes = joined(parts, pending);
6677
- parts = [];
6678
- pending = 0;
6679
- try {
6680
- return { line, text: decoder.decode(bytes) };
6681
- } catch {
6682
- return { line, refused: "not-utf8" };
6683
- }
6684
- };
6685
- for await (const chunk of chunksOf(source)) {
6686
- let start = 0;
6687
- for (; ; ) {
6688
- const newline = chunk.indexOf(NEWLINE, start);
6689
- const end = newline < 0 ? chunk.byteLength : newline;
6690
- const piece = chunk.subarray(start, end);
6691
- if (!overflowing && pending + piece.byteLength <= maxLineBytes) {
6692
- if (piece.byteLength > 0) parts.push(piece.slice());
6693
- pending += piece.byteLength;
6694
- } else {
6695
- overflowing = true;
6696
- parts = [];
6697
- }
6698
- if (newline < 0) break;
6699
- yield finishLine();
6700
- start = newline + 1;
6701
- }
6702
- }
6703
- if (pending > 0 || overflowing) yield finishLine();
6704
- }
6705
-
6706
- // src/serialization/base64.ts
6707
- var ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
6708
- var PAD = 61;
6709
- var BYTES_PER_GROUP = 3;
6710
- var CHARACTERS_PER_GROUP = 4;
6711
- var ASCII_LIMIT = 128;
6712
- var ENCODE = new Uint8Array(ALPHABET.length);
6713
- var DECODE = new Int8Array(ASCII_LIMIT).fill(-1);
6714
- for (let at = 0; at < ALPHABET.length; at++) {
6715
- ENCODE[at] = ALPHABET.charCodeAt(at);
6716
- DECODE[ALPHABET.charCodeAt(at)] = at;
6717
- }
6718
- var asciiDecoder = new TextDecoder();
6719
- function encodeBase64(bytes) {
6720
- const out = new Uint8Array(Math.ceil(bytes.byteLength / BYTES_PER_GROUP) * CHARACTERS_PER_GROUP);
6721
- let write = 0;
6722
- let read = 0;
6723
- for (; read + BYTES_PER_GROUP <= bytes.byteLength; read += BYTES_PER_GROUP) {
6724
- const group = bytes[read] << 16 | bytes[read + 1] << 8 | bytes[read + 2];
6725
- out[write++] = ENCODE[group >>> 18];
6726
- out[write++] = ENCODE[group >>> 12 & 63];
6727
- out[write++] = ENCODE[group >>> 6 & 63];
6728
- out[write++] = ENCODE[group & 63];
6729
- }
6730
- const left = bytes.byteLength - read;
6731
- if (left > 0) {
6732
- const group = bytes[read] << 16 | (left === 2 ? bytes[read + 1] << 8 : 0);
6733
- out[write++] = ENCODE[group >>> 18];
6734
- out[write++] = ENCODE[group >>> 12 & 63];
6735
- out[write++] = left === 2 ? ENCODE[group >>> 6 & 63] : PAD;
6736
- out[write++] = PAD;
6737
- }
6738
- return asciiDecoder.decode(out);
6739
- }
6740
- function sextetAt(text, at) {
6741
- const code = text.charCodeAt(at);
6742
- return code < ASCII_LIMIT ? DECODE[code] : -1;
6743
- }
6744
- function decodeExactBase64(text) {
6745
- if (text.length % CHARACTERS_PER_GROUP !== 0) return null;
6746
- const padding = text.endsWith("==") ? 2 : text.endsWith("=") ? 1 : 0;
6747
- const out = new Uint8Array(text.length / CHARACTERS_PER_GROUP * BYTES_PER_GROUP - padding);
6748
- const fullEnd = padding === 0 ? text.length : text.length - CHARACTERS_PER_GROUP;
6749
- let write = 0;
6750
- for (let at = 0; at < fullEnd; at += CHARACTERS_PER_GROUP) {
6751
- const a2 = sextetAt(text, at);
6752
- const b2 = sextetAt(text, at + 1);
6753
- const c2 = sextetAt(text, at + 2);
6754
- const d = sextetAt(text, at + 3);
6755
- if ((a2 | b2 | c2 | d) < 0) return null;
6756
- const group = a2 << 18 | b2 << 12 | c2 << 6 | d;
6757
- out[write++] = group >>> 16;
6758
- out[write++] = group >>> 8 & 255;
6759
- out[write++] = group & 255;
6760
- }
6761
- if (padding === 0) return out;
6762
- const a = sextetAt(text, fullEnd);
6763
- const b = sextetAt(text, fullEnd + 1);
6764
- const c = padding === 1 ? sextetAt(text, fullEnd + 2) : 0;
6765
- if ((a | b | c) < 0) return null;
6766
- if (padding === 2 && (b & 15) !== 0) return null;
6767
- if (padding === 1 && (c & 3) !== 0) return null;
6768
- out[write++] = a << 2 | b >>> 4;
6769
- if (padding === 1) out[write] = (b & 15) << 4 | c >>> 2;
6770
- return out;
6771
- }
6772
-
6773
- // src/engine/export/tags.ts
6774
- var TAG_PREFIX = "$";
6775
- var DATE_TAG = "$date";
6776
- var BYTES_TAG = "$bytes";
6777
- var NUMBER_TAG = "$number";
6778
- var OBJECT_TAG = "$object";
6779
- var MILLISECONDS_PER_SECOND = 1e3;
6780
- var NANOSECONDS_PER_MILLISECOND = 1e6;
6781
- var MAX_NANOSECONDS = 999999999;
6782
- var MAX_DATE_MILLISECONDS = 864e13;
6783
- var ESCAPED_TAG_PREFIX = "\\u0024";
6784
- var NON_FINITE_NUMBERS = /* @__PURE__ */ new Map([
6785
- ["NaN", Number.NaN],
6786
- ["Infinity", Number.POSITIVE_INFINITY],
6787
- ["-Infinity", Number.NEGATIVE_INFINITY]
6788
- ]);
6789
- function refuse(path, reason) {
6790
- throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path === "" ? "(document)" : path}" ${reason}`, {
6791
- field: path
6792
- });
6793
- }
6794
- function isRecord(value) {
6795
- return typeof value === "object" && value !== null && !Array.isArray(value);
6796
- }
6797
- function startsWithTagPrefix(key) {
6798
- return key.startsWith(TAG_PREFIX);
6799
- }
6800
- function soleTagKey(record) {
6801
- let sole = null;
6802
- for (const key of Object.keys(record)) {
6803
- if (sole !== null) return null;
6804
- sole = key;
6805
- }
6806
- return sole !== null && startsWithTagPrefix(sole) ? sole : null;
6807
- }
6808
- function dateOf(value, path) {
6809
- if (!Array.isArray(value) || value.length !== 2)
6810
- refuse(path, `holds a "${DATE_TAG}" tag other than [seconds, nanoseconds]`);
6811
- const seconds = value[0];
6812
- const nanoseconds = value[1];
6813
- if (typeof seconds !== "number" || !Number.isSafeInteger(seconds)) {
6814
- refuse(
6815
- path,
6816
- `holds a "${DATE_TAG}" tag whose seconds are not an integer from -${Number.MAX_SAFE_INTEGER} to ${Number.MAX_SAFE_INTEGER}`
6817
- );
6818
- }
6819
- if (typeof nanoseconds !== "number" || !Number.isInteger(nanoseconds) || nanoseconds < 0 || nanoseconds > MAX_NANOSECONDS) {
6820
- refuse(path, `holds a "${DATE_TAG}" tag whose nanoseconds are not an integer from 0 to ${MAX_NANOSECONDS}`);
6821
- }
6822
- const milliseconds = seconds * MILLISECONDS_PER_SECOND + nanoseconds / NANOSECONDS_PER_MILLISECOND;
6823
- if (Math.abs(milliseconds) > MAX_DATE_MILLISECONDS)
6824
- refuse(path, `holds a "${DATE_TAG}" tag outside the range of a date`);
6825
- return new Date(milliseconds);
6826
- }
6827
- function bytesOf(value, path) {
6828
- const bytes = typeof value === "string" ? decodeExactBase64(value) : null;
6829
- if (bytes === null) refuse(path, `holds a "${BYTES_TAG}" tag whose text is not padded base64`);
6830
- return bytes;
6831
- }
6832
- function numberOf(value, path) {
6833
- const number = typeof value === "string" ? NON_FINITE_NUMBERS.get(value) : void 0;
6834
- if (number === void 0) refuse(path, `holds a "${NUMBER_TAG}" tag other than "NaN", "Infinity", or "-Infinity"`);
6835
- return number;
6836
- }
6837
- function requireDepth(path, depth) {
6838
- if (depth >= MAX_DOCUMENT_NESTING_DEPTH) refuse(path, `nests deeper than ${MAX_DOCUMENT_NESTING_DEPTH} levels`);
6839
- }
6840
- function untagMap(record, path, depth) {
6841
- requireDepth(path, depth);
6842
- for (const key of Object.keys(record)) {
6843
- record[key] = untag(record[key], path === "" ? key : `${path}.${key}`, depth + 1);
6844
- }
6845
- return record;
6846
- }
6847
- function untagList(list, path, depth) {
6848
- requireDepth(path, depth);
6849
- for (let at = 0; at < list.length; at++) list[at] = untag(list[at], `${path}[${at}]`, depth + 1);
6850
- return list;
6851
- }
6852
- function untagTag(tag, value, path, depth) {
6853
- switch (tag) {
6854
- case DATE_TAG:
6855
- return dateOf(value, path);
6856
- case BYTES_TAG:
6857
- return bytesOf(value, path);
6858
- case NUMBER_TAG:
6859
- return numberOf(value, path);
6860
- case OBJECT_TAG:
6861
- if (!isRecord(value)) refuse(path, `holds an "${OBJECT_TAG}" tag whose value is not an object`);
6862
- return untagMap(value, path, depth);
6863
- default:
6864
- return refuse(path, `holds an object whose only key, "${tag}", names no tag`);
6865
- }
6866
- }
6867
- function untag(value, path, depth) {
6868
- if (typeof value !== "object" || value === null) return value;
6869
- if (Array.isArray(value)) return untagList(value, path, depth);
6870
- const record = value;
6871
- const tag = soleTagKey(record);
6872
- return tag === null ? untagMap(record, path, depth) : untagTag(tag, record[tag], path, depth);
6873
- }
6874
- function untagDocumentLine(text, parsed) {
6875
- if (!text.includes(`"${TAG_PREFIX}`) && !text.includes(ESCAPED_TAG_PREFIX)) return parsed;
6876
- return untagMap(parsed, "", 0);
6877
- }
6878
-
6879
- // src/engine/export/import.ts
6880
- var DOCUMENT_EXPORT_FORMAT = "narsil-export";
6881
- var DOCUMENT_EXPORT_VERSION = 1;
6882
- function invalidHeader(message) {
6883
- throw new NarsilError(ErrorCodes.CONFIG_INVALID, `The document export header ${message}`);
6884
- }
6885
- function isRecord2(value) {
6886
- return typeof value === "object" && value !== null && !Array.isArray(value);
6887
- }
6888
- function configOfHeader(line) {
6889
- if (line === void 0) invalidHeader("is missing");
6890
- if (!("text" in line)) {
6891
- invalidHeader(
6892
- line.refused === "too-long" ? `passes the line limit of ${MAX_EXPORT_LINE_BYTES} bytes` : "is not UTF-8"
6893
- );
6894
- }
6895
- let raw;
6896
- try {
6897
- raw = JSON.parse(line.text);
6898
- } catch {
6899
- invalidHeader("is not JSON");
6900
- }
6901
- if (!isRecord2(raw) || raw.format !== DOCUMENT_EXPORT_FORMAT || raw.version !== DOCUMENT_EXPORT_VERSION) {
6902
- invalidHeader(`must name format "${DOCUMENT_EXPORT_FORMAT}" and version ${DOCUMENT_EXPORT_VERSION}`);
6903
- }
6904
- const configuration = requireConfigurationShape(raw.index, "document export header");
6905
- const fields = restoredConfigFields(configuration, (message) => invalidHeader(message));
6906
- const embedding = restoredEmbedding(configuration.embedding, (message) => invalidHeader(message));
6907
- return {
6908
- schema: configuration.schema,
6909
- language: configuration.language,
6910
- ...fields,
6911
- ...embedding === void 0 ? {} : {
6912
- embedding: {
6913
- fields: embedding.fields,
6914
- ...embedding.adapter === void 0 ? {} : { adapter: embedding.adapter }
6915
- }
6916
- }
6917
- };
6918
- }
6919
- async function prepareIndex(engine, indexName, config) {
6920
- if (!engine.listIndexes().some((index) => index.name === indexName)) {
6921
- await engine.createIndex(indexName, config);
6922
- return;
6923
- }
6924
- const existing = flattenSchema(engine.getStats(indexName).schema);
6925
- const exported = flattenSchema(config.schema);
6926
- const fields = /* @__PURE__ */ new Set([...Object.keys(existing), ...Object.keys(exported)]);
6927
- for (const field of fields) {
6928
- if (existing[field] === exported[field]) continue;
6929
- throw new NarsilError(
6930
- ErrorCodes.CONFIG_INVALID,
6931
- `Index "${indexName}" gives field "${field}" another type than the export does, so the import cannot write into it`,
6932
- { indexName, field }
6933
- );
6934
- }
6935
- }
6936
- function highestReportedAt(refused) {
6937
- let highest = 0;
6938
- for (let at = 1; at < refused.length; at++) {
6939
- if (refused[at].line > refused[highest].line) highest = at;
6940
- }
6941
- return highest;
6942
- }
6943
- function refuse2(state, line, code, message) {
6944
- state.result.refusedCount += 1;
6945
- const refused = state.result.refused;
6946
- if (refused.length < MAX_IMPORT_REFUSALS_REPORTED) {
6947
- refused.push({ line, code, message });
6948
- if (refused.length === MAX_IMPORT_REFUSALS_REPORTED) state.highestReported = highestReportedAt(refused);
6949
- return;
6950
- }
6951
- if (line >= refused[state.highestReported].line) return;
6952
- refused[state.highestReported] = { line, code, message };
6953
- state.highestReported = highestReportedAt(refused);
6954
- }
6955
- async function flush(state) {
6956
- if (state.batch.length === 0) return;
6957
- const batch = state.batch;
6958
- state.batch = [];
6959
- state.batchIds = /* @__PURE__ */ new Set();
6960
- const lineOf = new Map(batch.map((entry) => [entry.docId, entry.line]));
6961
- const inserts = [];
6962
- const updates = [];
6963
- for (const entry of batch) {
6964
- if (await state.engine.has(state.indexName, entry.docId))
6965
- updates.push({ docId: entry.docId, document: entry.document });
6966
- else inserts.push(entry.document);
6967
- }
6968
- for (const outcome of [
6969
- inserts.length === 0 ? null : await state.engine.insertBatch(state.indexName, inserts, { skipClone: true }),
6970
- updates.length === 0 ? null : await state.engine.updateBatch(state.indexName, updates)
6971
- ]) {
6972
- if (outcome === null) continue;
6973
- state.result.imported += outcome.succeeded.length;
6974
- for (const failure of outcome.failed) {
6975
- const code = failure.error instanceof NarsilError ? failure.error.code : ErrorCodes.DOC_VALIDATION_FAILED;
6976
- refuse2(state, lineOf.get(failure.docId) ?? 0, code, failure.error.message);
6977
- }
6978
- }
6979
- }
6980
- async function takeLine(state, entry) {
6981
- if (!("text" in entry)) {
6982
- const message = entry.refused === "too-long" ? `The line passes the limit of ${MAX_EXPORT_LINE_BYTES} bytes` : "The line is not UTF-8";
6983
- refuse2(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, message);
6984
- return;
6985
- }
6986
- if (entry.text.length === 0) return;
6987
- let document;
6988
- try {
6989
- document = JSON.parse(entry.text);
6990
- } catch {
6991
- refuse2(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, "The line is not JSON");
6992
- return;
6993
- }
6994
- if (!isRecord2(document) || typeof document.id !== "string" || document.id.length === 0) {
6995
- refuse2(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, "The line holds no document with a string id");
6996
- return;
6997
- }
6998
- try {
6999
- untagDocumentLine(entry.text, document);
7000
- } catch (error) {
7001
- if (!(error instanceof NarsilError)) throw error;
7002
- refuse2(state, entry.line, error.code, error.message);
7003
- return;
7004
- }
7005
- if (state.batchIds.has(document.id)) await flush(state);
7006
- state.batch.push({ line: entry.line, docId: document.id, document });
7007
- state.batchIds.add(document.id);
7008
- if (state.batch.length >= IMPORT_BATCH_DOCUMENTS) await flush(state);
7009
- }
7010
- async function readDocumentExport(engine, indexName, source) {
7011
- const lines = exportLines(source, MAX_EXPORT_LINE_BYTES);
7012
- const header = await lines.next();
7013
- await prepareIndex(engine, indexName, configOfHeader(header.done === true ? void 0 : header.value));
7014
- const state = {
7015
- engine,
7016
- indexName,
7017
- batch: [],
7018
- batchIds: /* @__PURE__ */ new Set(),
7019
- result: { imported: 0, refusedCount: 0, refused: [] },
7020
- highestReported: 0
7021
- };
7022
- for await (const entry of lines) await takeLine(state, entry);
7023
- await flush(state);
7024
- state.result.refused.sort((a, b) => a.line - b.line);
7025
- return state.result;
7026
- }
7027
-
7028
6649
  // src/narsil/portability.ts
7029
6650
  function portabilityOperations(core, config, engine) {
7030
6651
  async function withOpenIndex2(indexName, action) {
@@ -7056,14 +6677,14 @@ function portabilityOperations(core, config, engine) {
7056
6677
  return {
7057
6678
  snapshot(indexName) {
7058
6679
  return withOpenIndex2(indexName, async () => {
7059
- const { createSnapshot } = await import('./snapshot-DOQLGGEX.mjs');
6680
+ const { createSnapshot } = await import('./snapshot-BTRZ3MZK.mjs');
7060
6681
  return createSnapshot(snapshotDepsOf(indexName));
7061
6682
  });
7062
6683
  },
7063
6684
  async restore(indexName, data) {
7064
6685
  core.guardShutdown();
7065
6686
  validateIndexName(indexName);
7066
- const restored = import('./restore-queue-JAMNRC6T.mjs').then(
6687
+ const restored = import('./restore-queue-RYSRRFS4.mjs').then(
7067
6688
  ({ queueRestore }) => queueRestore(core, restoreHost, restoresInFlight, indexName, data)
7068
6689
  );
7069
6690
  await emitWriteEventAfter(core, indexName, restored);
@@ -7072,8 +6693,8 @@ function portabilityOperations(core, config, engine) {
7072
6693
  exportDocuments(indexName) {
7073
6694
  return withOpenIndex2(indexName, async () => {
7074
6695
  const [{ checkpointForArchive }, { documentExportStream }] = await Promise.all([
7075
- import('./snapshot-DOQLGGEX.mjs'),
7076
- import('./write-ZKR7SNRT.mjs')
6696
+ import('./snapshot-BTRZ3MZK.mjs'),
6697
+ import('./write-GHTA56OW.mjs')
7077
6698
  ]);
7078
6699
  const header = indexConfigurationOf(core.requireIndex(indexName));
7079
6700
  return documentExportStream(header, await checkpointForArchive(snapshotDepsOf(indexName)));
@@ -7081,7 +6702,10 @@ function portabilityOperations(core, config, engine) {
7081
6702
  },
7082
6703
  importDocumentExport(indexName, data) {
7083
6704
  core.guardShutdown();
7084
- return emitWriteEventAfter(core, indexName, readDocumentExport(engine(), indexName, data));
6705
+ const imported = import('./import-QRZAQ5RH.mjs').then(
6706
+ ({ readDocumentExport }) => readDocumentExport(engine(), indexName, data)
6707
+ );
6708
+ return emitWriteEventAfter(core, indexName, imported);
7085
6709
  }
7086
6710
  };
7087
6711
  }
@@ -7092,13 +6716,19 @@ async function enterRead(core, indexName) {
7092
6716
  if (core.rebalancingIndexes.has(indexName)) await yieldToEventLoop();
7093
6717
  }
7094
6718
  async function readUnderOneAnalysis(core, indexName, read) {
7095
- let analysisStale = core.analysisRebuild.isStale(indexName);
7096
- let result = await read();
7097
- while (core.analysisRebuild.isStale(indexName) !== analysisStale) {
7098
- analysisStale = core.analysisRebuild.isStale(indexName);
7099
- result = await read();
6719
+ const manager = core.requireManager(indexName);
6720
+ for (; ; ) {
6721
+ const analysis = manager.analysis;
6722
+ const analysisStale = core.analysisRebuild.isStale(indexName);
6723
+ try {
6724
+ const result = await read();
6725
+ if (manager.analysis === analysis && core.analysisRebuild.isStale(indexName) === analysisStale) {
6726
+ return { result, analysisStale };
6727
+ }
6728
+ } catch (error) {
6729
+ if (manager.analysis === analysis) throw error;
6730
+ }
7100
6731
  }
7101
- return { result, analysisStale };
7102
6732
  }
7103
6733
  function phraseReadinessOf(core, indexName, config) {
7104
6734
  return () => {
@@ -7131,7 +6761,7 @@ async function runEngineQuery(core, indexName, params, options) {
7131
6761
  const manager = core.requireManager(indexName);
7132
6762
  const scoped = scopedParams(params, options);
7133
6763
  await core.pluginRegistry.runHook("beforeSearch", { indexName, params: scoped });
7134
- const rerank = scoped.rerank === void 0 ? void 0 : await import('./runtime-7MLQ7J64.mjs');
6764
+ const rerank = scoped.rerank === void 0 ? void 0 : await import('./runtime-DYV75DCB.mjs');
7135
6765
  const resolvedParams = await resolveVectorText(
7136
6766
  rerank === void 0 ? scoped : rerank.withRerankQueryText(scoped),
7137
6767
  entry.embeddingAdapter,
@@ -7139,25 +6769,24 @@ async function runEngineQuery(core, indexName, params, options) {
7139
6769
  entry.embeddingAdapterName
7140
6770
  );
7141
6771
  const workerSearch = core.orchestrator.hasWorkerPool() ? core.orchestrator.searchViaWorker.bind(core.orchestrator) : void 0;
7142
- const context = {
7143
- manager,
7144
- language: entry.language,
7145
- config: entry.config,
7146
- workerSearch,
7147
- indexName,
7148
- broadcastStats: broadcastStatsFor(core, options),
7149
- partitionIds: options?.partitionIds,
7150
- rescoreWindow: options?.rescoreWindow,
7151
- indexBm25: options?.indexBm25,
7152
- cursorBinding: lazyQueryBinding(params),
7153
- patternWorkCap: core.patternWorkCap,
7154
- rerank: rerank?.rerankRunnerFor(core),
7155
- requirePhraseReady: phraseReadinessOf(core, indexName, entry.config)
7156
- };
7157
6772
  const { result, analysisStale } = await readUnderOneAnalysis(
7158
6773
  core,
7159
6774
  indexName,
7160
- () => executeQuery(resolvedParams, context)
6775
+ () => executeQuery(resolvedParams, {
6776
+ manager,
6777
+ language: manager.language,
6778
+ config: manager.config,
6779
+ workerSearch,
6780
+ indexName,
6781
+ broadcastStats: broadcastStatsFor(core, options),
6782
+ partitionIds: options?.partitionIds,
6783
+ rescoreWindow: options?.rescoreWindow,
6784
+ indexBm25: options?.indexBm25,
6785
+ cursorBinding: lazyQueryBinding(params),
6786
+ patternWorkCap: core.patternWorkCap,
6787
+ rerank: rerank?.rerankRunnerFor(core),
6788
+ requirePhraseReady: phraseReadinessOf(core, indexName, entry.config)
6789
+ })
7161
6790
  );
7162
6791
  if (analysisStale) {
7163
6792
  result.analysisStale = true;
@@ -7191,23 +6820,22 @@ async function runEnginePreflight(core, indexName, params, options) {
7191
6820
  entry.embeddingAdapterName
7192
6821
  );
7193
6822
  const workerSearch = core.orchestrator.hasWorkerPool() ? core.orchestrator.searchViaWorker.bind(core.orchestrator) : void 0;
7194
- const context = {
7195
- manager,
7196
- language: entry.language,
7197
- config: entry.config,
7198
- workerSearch,
7199
- indexName,
7200
- broadcastStats: broadcastStatsFor(core, options),
7201
- partitionIds: options?.partitionIds,
7202
- indexBm25: options?.indexBm25,
7203
- cursorBinding: lazyQueryBinding(params),
7204
- patternWorkCap: core.patternWorkCap,
7205
- requirePhraseReady: phraseReadinessOf(core, indexName, entry.config)
7206
- };
7207
6823
  const { result, analysisStale } = await readUnderOneAnalysis(
7208
6824
  core,
7209
6825
  indexName,
7210
- () => executePreflight(resolvedParams, context)
6826
+ () => executePreflight(resolvedParams, {
6827
+ manager,
6828
+ language: manager.language,
6829
+ config: manager.config,
6830
+ workerSearch,
6831
+ indexName,
6832
+ broadcastStats: broadcastStatsFor(core, options),
6833
+ partitionIds: options?.partitionIds,
6834
+ indexBm25: options?.indexBm25,
6835
+ cursorBinding: lazyQueryBinding(params),
6836
+ patternWorkCap: core.patternWorkCap,
6837
+ requirePhraseReady: phraseReadinessOf(core, indexName, entry.config)
6838
+ })
7211
6839
  );
7212
6840
  if (analysisStale) {
7213
6841
  result.analysisStale = true;
@@ -7448,9 +7076,20 @@ function createNarsilFromCore(core, config) {
7448
7076
  return runEngineSuggest(core, indexName, params);
7449
7077
  },
7450
7078
  async rebuildAnalysis(indexName) {
7079
+ guardShutdown();
7080
+ const entry = requireIndex(indexName);
7081
+ await withOpenIndexWrite(indexName, async () => {
7082
+ if (entry.pendingAnalysis === null) return core.analysisRebuild.rebuild(indexName);
7083
+ return (await import('./analysis-change-JN23AZQP.mjs')).resumeAnalysisChange(core, indexName);
7084
+ });
7085
+ },
7086
+ async changeAnalysis(indexName, change) {
7451
7087
  guardShutdown();
7452
7088
  requireIndex(indexName);
7453
- await withOpenIndexWrite(indexName, () => core.analysisRebuild.rebuild(indexName));
7089
+ await withOpenIndexWrite(
7090
+ indexName,
7091
+ async () => (await import('./analysis-change-JN23AZQP.mjs')).changeIndexAnalysis(core, indexName, change)
7092
+ );
7454
7093
  },
7455
7094
  ...portabilityOperations(core, config, () => narsil),
7456
7095
  ...synonymOperations(core),
@@ -7596,4 +7235,4 @@ async function createNarsil(config) {
7596
7235
  // src/index.ts
7597
7236
  var VERSION = "0.1.0";
7598
7237
 
7599
- export { BYTES_TAG, DATE_TAG, DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, HEADER_SIZE, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, NUMBER_TAG, OBJECT_TAG, VERSION, applyStoredIndexBm25, arrayPositionsAreStale, crc322 as crc32, crc32Final, crc32Init, crc32Update, createByteReader, createEngineCore, createEngineIndex, createNarsil, createNarsilFromCore, decodeIndexConfiguration, emptySerializablePartition, encodeBase64, encodeIndexConfiguration, engineVersionParts, hasNrslMagic, insertDocumentVectors, openChecksummedEnvelope, packChecksummedEnvelope, prepareConfiguredSynonyms, prepareDocumentVectors, prepareStoredSynonyms, readHeader, reconstructSchemaFromMetadata, removeDocumentVectors, requireConfigurationWithinSizeLimit, requireKnownConfig, resolveRunnableConfig, rulesOfStoredSynonyms, runEngineListDocuments, runEnginePreflight, runEngineQuery, runEngineQueryStats, runEngineSuggest, startEngineCore, startsWithTagPrefix, storedArrayPositionGap, validateEmbeddingConfig, validateRequiredFieldsInSchema, writeHeader, yieldToEventLoop };
7238
+ export { HEADER_SIZE, VERSION, applyStoredIndexBm25, arrayPositionsAreStale, crc322 as crc32, crc32Final, crc32Init, crc32Update, createEngineCore, createEngineIndex, createNarsil, createNarsilFromCore, decodeIndexConfiguration, emitEngineEvent, emptySerializablePartition, encodeIndexConfiguration, engineVersionParts, hasNrslMagic, insertDocumentVectors, openChecksummedEnvelope, packChecksummedEnvelope, prepareConfiguredSynonyms, prepareDocumentVectors, prepareStoredSynonyms, readHeader, rebuildTextIndexes, reconstructSchemaFromMetadata, removeDocumentVectors, requireConfigurationShape, requireConfigurationWithinSizeLimit, requireKnownConfig, resolveRunnableConfig, rulesOfStoredSynonyms, runEngineListDocuments, runEnginePreflight, runEngineQuery, runEngineQueryStats, runEngineSuggest, startEngineCore, storedArrayPositionGap, textRebuildTargetOf, validateEmbeddingConfig, validateRequiredFieldsInSchema, writeHeader, yieldToEventLoop };