@delali/narsil 0.3.1-next.82 → 0.3.1-next.84

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,11 +1,11 @@
1
- import { validateWorkerConfig, validateIndexName, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateBM25Params, validatePartitionConfig, executeQuery, queryBindingOf, executePreflight, executeSuggest, executeListDocuments, validateDocId } from './chunk-4D6OH4EF.mjs';
1
+ import { validateWorkerConfig, validateIndexName, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateBM25Params, validatePartitionConfig, queryBindingOf, executeQuery, executePreflight, executeSuggest, executeListDocuments, validateDocId } from './chunk-4D6OH4EF.mjs';
2
2
  import { ReplicationMessageTypes } from './chunk-GCM3JFVA.mjs';
3
3
  import { WIRE_BATCH_BUDGET, resolveVectorText, bindEngineCore, restoredConfigFields, restoredEmbedding, collectRequiredFieldFailures, embedChunkDocuments, providedDocId, requireDocumentObject, admitInsert, asBatchInsertError, requireMatchingDocId, embedDocumentFields, assertDocumentCarriesMappedVectors, admitBufferedUpsert, createPendingAdmissions, recordPendingAdmission } from './chunk-2453RJYK.mjs';
4
- import { NATIVE_CRC32_MIN_BYTES, generateId, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, splitWorkerBudget, resolveWorkerCount, createDirectExecutor, createPartitionRouter, MAX_FILE_BYTES, resolveWorkerEntry, missingWorkerEntry, readHeapStatistics, nodeThreadId, CHECKSUM_YIELD_CHUNK_BYTES, resolvePartitionInsertOptions, resolveIndexAnalysis, createPartitionManager, buildSegmentPayload, createFrozenSegment, resolveRequestThreadCount, vectorsPerFile, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, deadBitsWhere, NO_LIVE_VECTOR_AT_POSITION, createPartitionIndex, selectionKeeps, decodeMapWithoutFields, isCompositePartition, segmentArraysOf, mergeSegmentContents, survivorDocumentsOf, partitionStateOf, VECTOR_FILE_MAX_VECTORS, deadBytesFor, mergeFrozenSegments, workerResourceLimits, decodeVectorIndexParts, bytesToVectors, freezeSegmentShared, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, SHUTDOWN_TIMEOUT_MS, decodeVectorFilePayload, decodeVectorGraphPartPayload, assembleGraphParts, countDead, encodeSegmentState, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, VALID_HNSW_METRICS, spawnNodeWorker, decodeVectorGraphPayload, graphPartsOfEarlierPayload, decodeStoredDocument, isDead } from './chunk-4CWHWOBI.mjs';
4
+ import { NATIVE_CRC32_MIN_BYTES, generateId, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, splitWorkerBudget, resolveWorkerCount, createDirectExecutor, createPartitionRouter, MAX_FILE_BYTES, resolveWorkerEntry, missingWorkerEntry, readHeapStatistics, nodeThreadId, CHECKSUM_YIELD_CHUNK_BYTES, resolvePartitionInsertOptions, resolveIndexAnalysis, createPartitionManager, buildSegmentPayload, createFrozenSegment, resolveRequestThreadCount, vectorsPerFile, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, createPartitionIndex, selectionKeeps, decodeMapWithoutFields, isCompositePartition, segmentArraysOf, mergeSegmentContents, survivorDocumentsOf, partitionStateOf, VECTOR_FILE_MAX_VECTORS, deadBytesFor, mergeFrozenSegments, workerResourceLimits, decodeVectorIndexParts, bytesToVectors, freezeSegmentShared, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, SHUTDOWN_TIMEOUT_MS, decodeVectorFilePayload, decodeVectorGraphPartPayload, assembleGraphParts, countDead, encodeSegmentState, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, VALID_HNSW_METRICS, spawnNodeWorker, decodeVectorGraphPayload, graphPartsOfEarlierPayload, decodeStoredDocument, isDead } from './chunk-IA67SOJO.mjs';
5
5
  import { requirePatternSearchFor, requirePartialWordSearchFor, collectQueryTermStats, mergePartitionStats, oversampledFacetConfig, fnv1a, availablePatternSearch, PATTERN_HOST, kWayMerge, mergeFacets, setNestedValue } from './chunk-XYDP7CCJ.mjs';
6
6
  import { detectRuntime, getLanguage } from './chunk-OLXM65YH.mjs';
7
7
  import { MAX_INDEX_CONFIGURATION_BYTES, STATISTICS_BROADCAST_INTERVAL_MS, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, HEAP_PRESSURE_WARNING_FRACTION, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, FOREIGN_STATS_TTL_MS, MAX_IMPORT_REFUSALS_REPORTED, BATCH_CHUNK_SIZE, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION, EXPORT_CHUNK_BYTES } from './chunk-ZCKPRCYI.mjs';
8
- import { releaseLocksHeldBy } from './chunk-IN5CH4P2.mjs';
8
+ import { releaseLocksHeldBy } from './chunk-RZXNWQSM.mjs';
9
9
  import { requireSearchBackendSettings } from './chunk-B3UOQJD3.mjs';
10
10
  import { DEFAULT_PATTERN_WORK_CAP, MATCHER_STATE_BUDGET_BYTES } from './chunk-SREPGC6O.mjs';
11
11
  import { getNestedValue, parseVectorDimension } from './chunk-4VBG6PEH.mjs';
@@ -160,6 +160,8 @@ function yieldToEventLoop() {
160
160
  // src/partitioning/constants.ts
161
161
  var REBALANCE_CHUNK_SIZE = 1e3;
162
162
  var DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE = 1e4;
163
+ var TEXT_REBUILD_CHUNK_DOCUMENTS = 1e3;
164
+ var TEXT_REBUILD_SWITCH_BACKLOG_DOCUMENTS = 1e3;
163
165
 
164
166
  // src/partitioning/rebalancer.ts
165
167
  function createRebalancer() {
@@ -452,6 +454,74 @@ function validateRequiredFieldsInSchema(required, schema) {
452
454
  }
453
455
  }
454
456
 
457
+ // src/partitioning/text-rebuild.ts
458
+ function interrupted(pass) {
459
+ return pass.log.partitionsReplaced || !pass.hooks.isCurrent();
460
+ }
461
+ async function buildPartition(pass, source, partitionId) {
462
+ const { manager, options } = pass;
463
+ const target = createPartitionIndex(partitionId, manager.config.trackPositions ?? true);
464
+ const docIds = [...source.docIds()];
465
+ target.beginBatch();
466
+ for (let start = 0; start < docIds.length; start += TEXT_REBUILD_CHUNK_DOCUMENTS) {
467
+ const end = Math.min(start + TEXT_REBUILD_CHUNK_DOCUMENTS, docIds.length);
468
+ for (let i = start; i < end; i++) {
469
+ const document = manager.getRef(docIds[i]);
470
+ if (document !== void 0) target.insert(docIds[i], document, manager.schema, manager.language, options);
471
+ }
472
+ await yieldToEventLoop();
473
+ if (interrupted(pass)) return null;
474
+ }
475
+ target.endBatch();
476
+ if (manager.partialWordAnalysis !== null) target.preparePartialWords(manager.partialWordAnalysis);
477
+ return target;
478
+ }
479
+ function applyWrittenDocuments(pass, built) {
480
+ const { manager, options } = pass;
481
+ for (const docId of pass.log.takeWrittenDocIds()) {
482
+ for (const partition of built) {
483
+ if (!partition.has(docId)) continue;
484
+ partition.remove(docId, manager.schema, manager.language, options);
485
+ break;
486
+ }
487
+ const partitionId = manager.partitionIdOf(docId);
488
+ const document = manager.getRef(docId);
489
+ if (partitionId === void 0 || document === void 0) continue;
490
+ built[partitionId].insert(docId, document, manager.schema, manager.language, options);
491
+ }
492
+ }
493
+ async function rebuildOnce(pass) {
494
+ const partitionCount = pass.manager.partitionCount;
495
+ const built = [];
496
+ for (let partitionId = 0; partitionId < partitionCount; partitionId++) {
497
+ const partition = await buildPartition(pass, pass.manager.getPartition(partitionId), partitionId);
498
+ if (partition === null) return false;
499
+ built.push(partition);
500
+ pass.hooks.onPartitionBuilt(built.length);
501
+ }
502
+ while (pass.log.pendingDocuments > TEXT_REBUILD_SWITCH_BACKLOG_DOCUMENTS) {
503
+ applyWrittenDocuments(pass, built);
504
+ await yieldToEventLoop();
505
+ if (interrupted(pass)) return false;
506
+ }
507
+ applyWrittenDocuments(pass, built);
508
+ pass.manager.adoptRebuiltPartitions(built);
509
+ pass.hooks.onSwitch();
510
+ return true;
511
+ }
512
+ async function rebuildTextIndexes(manager, hooks) {
513
+ const options = resolvePartitionInsertOptions(manager.config, manager.analysis, { validate: false, skipClone: true });
514
+ while (hooks.isCurrent()) {
515
+ const log = manager.openWriteLog();
516
+ try {
517
+ if (await rebuildOnce({ manager, log, hooks, options })) return true;
518
+ } finally {
519
+ log.close();
520
+ }
521
+ }
522
+ return false;
523
+ }
524
+
455
525
  // src/engine/events.ts
456
526
  function eventHandlersOf(events) {
457
527
  const handlers = /* @__PURE__ */ new Map();
@@ -474,11 +544,6 @@ function emitEngineEvent(eventHandlers, event, payload) {
474
544
  }
475
545
 
476
546
  // src/engine/analysis-rebuild.ts
477
- function yieldToEventLoop2() {
478
- return new Promise((resolve) => {
479
- setTimeout(resolve, 0);
480
- });
481
- }
482
547
  function toError(value) {
483
548
  return value instanceof Error ? value : new Error(String(value));
484
549
  }
@@ -489,9 +554,13 @@ function wireAnalysisRebuild(wiring) {
489
554
  resyncIndex: wiring.resyncIndex,
490
555
  async persistAnalysisRevision(indexName) {
491
556
  const durability = wiring.durabilityManager;
492
- if (durability === null) return;
493
- await durability.persistMetadata(indexName);
557
+ if (durability === null) {
558
+ wiring.recordCurrentAnalysis(indexName);
559
+ return;
560
+ }
494
561
  await (durability.checkpointFromMemory?.(indexName) ?? durability.checkpoint(indexName));
562
+ wiring.recordCurrentAnalysis(indexName);
563
+ await durability.persistMetadata(indexName);
495
564
  },
496
565
  emit(payload) {
497
566
  emitEngineEvent(wiring.eventHandlers, "analysisRebuild", payload);
@@ -502,9 +571,10 @@ function createAnalysisRebuildCoordinator(config, deps) {
502
571
  const stale = /* @__PURE__ */ new Map();
503
572
  const rebuildMode = config?.rebuild ?? "auto";
504
573
  let queue = Promise.resolve();
574
+ let active = null;
505
575
  async function rebuildNow(indexName, entry) {
506
576
  const manager = deps.getManager(indexName);
507
- if (manager === void 0) {
577
+ if (manager === void 0 || entry.abandoned) {
508
578
  entry.run = null;
509
579
  return;
510
580
  }
@@ -512,12 +582,20 @@ function createAnalysisRebuildCoordinator(config, deps) {
512
582
  deps.emit({ indexName, status: "started", partitionsRebuilt: 0, partitionCount });
513
583
  const wasPromoted = deps.desyncIndex(indexName);
514
584
  try {
515
- for (let partitionId = 0; partitionId < manager.partitionCount; partitionId += 1) {
516
- manager.rebuildTextIndex(partitionId);
517
- entry.partitionsRebuilt = partitionId + 1;
518
- await yieldToEventLoop2();
585
+ const rebuilt = await rebuildTextIndexes(manager, {
586
+ isCurrent: () => deps.getManager(indexName) === manager && stale.get(indexName) === entry && !entry.abandoned,
587
+ onPartitionBuilt(partitionsBuilt) {
588
+ entry.partitionsRebuilt = partitionsBuilt;
589
+ },
590
+ onSwitch() {
591
+ stale.delete(indexName);
592
+ }
593
+ });
594
+ if (!rebuilt) {
595
+ entry.run = null;
596
+ if (deps.getManager(indexName) === manager) await deps.resyncIndex(indexName, wasPromoted);
597
+ return;
519
598
  }
520
- stale.delete(indexName);
521
599
  await deps.persistAnalysisRevision(indexName);
522
600
  await deps.resyncIndex(indexName, wasPromoted);
523
601
  deps.emit({
@@ -528,7 +606,6 @@ function createAnalysisRebuildCoordinator(config, deps) {
528
606
  });
529
607
  } catch (err) {
530
608
  const error = toError(err);
531
- stale.set(indexName, entry);
532
609
  entry.run = null;
533
610
  await deps.resyncIndex(indexName, wasPromoted).catch(() => void 0);
534
611
  deps.emit({
@@ -541,11 +618,20 @@ function createAnalysisRebuildCoordinator(config, deps) {
541
618
  throw error;
542
619
  }
543
620
  }
621
+ function runTracked(indexName, entry) {
622
+ const done = rebuildNow(indexName, entry);
623
+ const tracked = { indexName, entry, done };
624
+ active = tracked;
625
+ return done.finally(() => {
626
+ if (active === tracked) active = null;
627
+ });
628
+ }
544
629
  function enqueue(indexName, entry) {
545
630
  if (entry.run !== null) {
546
631
  return entry.run;
547
632
  }
548
- const run = queue.then(() => rebuildNow(indexName, entry));
633
+ entry.abandoned = false;
634
+ const run = queue.then(() => runTracked(indexName, entry));
549
635
  entry.run = run;
550
636
  queue = run.then(
551
637
  () => void 0,
@@ -580,7 +666,7 @@ function createAnalysisRebuildCoordinator(config, deps) {
580
666
  if (existing !== void 0 && existing.run !== null) {
581
667
  return;
582
668
  }
583
- stale.set(index.indexName, { index, partitionsRebuilt: 0, run: null });
669
+ stale.set(index.indexName, { index, partitionsRebuilt: 0, run: null, abandoned: false });
584
670
  },
585
671
  clearStale(indexName) {
586
672
  stale.delete(indexName);
@@ -591,6 +677,14 @@ function createAnalysisRebuildCoordinator(config, deps) {
591
677
  isRunning(indexName) {
592
678
  return stale.get(indexName)?.run !== null && stale.get(indexName)?.run !== void 0;
593
679
  },
680
+ async abandon(indexName) {
681
+ const entry = stale.get(indexName);
682
+ if (entry !== void 0 && entry.run !== null) entry.abandoned = true;
683
+ const running = active;
684
+ if (running === null || running.indexName !== indexName) return;
685
+ running.entry.abandoned = true;
686
+ await running.done.catch(() => void 0);
687
+ },
594
688
  review(indexName) {
595
689
  const entry = stale.get(indexName);
596
690
  return entry === void 0 ? Promise.resolve() : review(indexName, entry);
@@ -1400,7 +1494,7 @@ function buildEntry(fields) {
1400
1494
  }
1401
1495
 
1402
1496
  // src/serialization/chunk-checksum.ts
1403
- function yieldToEventLoop3() {
1497
+ function yieldToEventLoop2() {
1404
1498
  if (typeof setImmediate === "function") {
1405
1499
  return new Promise((resolve) => setImmediate(resolve));
1406
1500
  }
@@ -1412,7 +1506,7 @@ async function advanceChecksum(start, payload) {
1412
1506
  const end = Math.min(offset + CHECKSUM_YIELD_CHUNK_BYTES, payload.length);
1413
1507
  state = crc32Update(state, payload.subarray(offset, end));
1414
1508
  if (end < payload.length) {
1415
- await yieldToEventLoop3();
1509
+ await yieldToEventLoop2();
1416
1510
  }
1417
1511
  }
1418
1512
  return state;
@@ -3756,7 +3850,8 @@ async function readVectorFile(store, file, fieldPath) {
3756
3850
  async function loadVectorField(store, ref, vectorIndex) {
3757
3851
  const restore = vectorIndex.restoreCheckpoint({
3758
3852
  liveVectors: liveVectorsOf(ref),
3759
- holdsGraph: ref.graphParts.length > 0
3853
+ holdsGraph: ref.graphParts.length > 0,
3854
+ graph: await readGraphParts(store, ref)
3760
3855
  });
3761
3856
  for (const file of ref.files) {
3762
3857
  const { payload, payloadLength } = await readVectorFile(store, file, ref.fieldPath);
@@ -3771,7 +3866,7 @@ async function loadVectorField(store, ref, vectorIndex) {
3771
3866
  }
3772
3867
  });
3773
3868
  }
3774
- restore.finish(await readGraphParts(store, ref));
3869
+ restore.finish();
3775
3870
  }
3776
3871
  var EMPTY_BIN = new Uint8Array(0);
3777
3872
  var EMPTY_BIN_BYTES = 2;
@@ -3835,7 +3930,7 @@ async function writeField(input, fieldPath, plan, outcome) {
3835
3930
  const files = plan.kept.map((kept) => keptFileRef(input, prior, kept));
3836
3931
  const written = [];
3837
3932
  for (let index = 0; index < plan.newFiles; index++) {
3838
- const { payload, ordinals } = plan.readNewFile(index);
3933
+ const { payload, ordinals, dead } = plan.readNewFile(index);
3839
3934
  const id = nextFileId;
3840
3935
  nextFileId += 1;
3841
3936
  const key = vectorFileKey(input.indexName, fieldPath, id);
@@ -3845,13 +3940,13 @@ async function writeField(input, fieldPath, plan, outcome) {
3845
3940
  `vector file "${key}"`
3846
3941
  );
3847
3942
  await input.store.write(key, packed.chunks);
3848
- const dead = deadBitsWhere(ordinals.length, (position) => ordinals[position] === NO_LIVE_VECTOR_AT_POSITION);
3849
3943
  files.push({ id, key, count: payload.docIds.length, dead, bytes: packed.byteLength, checksum: packed.checksum });
3850
3944
  written.push({ id, key, ordinals });
3851
3945
  outcome.layouts.push({
3852
3946
  fieldPath,
3853
3947
  key,
3854
3948
  docIds: payload.docIds,
3949
+ dead,
3855
3950
  vectorsOffset: packed.byteLength - payload.vectors.byteLength
3856
3951
  });
3857
3952
  }
@@ -5638,7 +5733,7 @@ function removeWhatTheSegmentReplaces(manager, segment, contents) {
5638
5733
  manager.endBatchRemove();
5639
5734
  }
5640
5735
  }
5641
- function yieldToEventLoop4() {
5736
+ function yieldToEventLoop3() {
5642
5737
  return new Promise((resolve) => setTimeout(resolve, 0));
5643
5738
  }
5644
5739
  async function loadPartitionSegments(store, partition, manager, options) {
@@ -5649,7 +5744,7 @@ async function loadPartitionSegments(store, partition, manager, options) {
5649
5744
  const segment = createSharedFrozenSegment(shareSegmentArrays(contents.arrays, manager.partialWordAnalysis));
5650
5745
  removeWhatTheSegmentReplaces(manager, segment, contents);
5651
5746
  if (contents.arrays.documentCount > 0) manager.attachFrozenSegment(partition.partitionId, segment);
5652
- await yieldToEventLoop4();
5747
+ await yieldToEventLoop3();
5653
5748
  }
5654
5749
  }
5655
5750
 
@@ -5722,7 +5817,12 @@ async function adoptVectorLayouts(store, vectorIndexes, layouts) {
5722
5817
  const vecIndex = vectorIndexes.get(layout.fieldPath);
5723
5818
  if (vecIndex === void 0 || vecIndex.storage !== "disk") continue;
5724
5819
  const path = await pathOf.call(store, layout.key);
5725
- await vecIndex.adoptDiskLayout({ path, docIds: layout.docIds, vectorsOffset: layout.vectorsOffset });
5820
+ await vecIndex.adoptDiskLayout({
5821
+ path,
5822
+ docIds: layout.docIds,
5823
+ dead: layout.dead,
5824
+ vectorsOffset: layout.vectorsOffset
5825
+ });
5726
5826
  }
5727
5827
  }
5728
5828
  async function garbageThatNoFieldStillReads(store, vectorIndexes, keys) {
@@ -6677,7 +6777,11 @@ async function readGraph(store, ref) {
6677
6777
  async function loadVectorField2(store, ref, vectorIndex, fileRefs) {
6678
6778
  let liveVectors = 0;
6679
6779
  for (const file of ref.files) liveVectors += file.count - countDead(file.dead, file.count);
6680
- const restore = vectorIndex.restoreCheckpoint({ liveVectors, holdsGraph: ref.graphKey !== null });
6780
+ const restore = vectorIndex.restoreCheckpoint({
6781
+ liveVectors,
6782
+ holdsGraph: ref.graphKey !== null,
6783
+ graph: await readGraph(store, ref)
6784
+ });
6681
6785
  for (const file of ref.files) {
6682
6786
  const bytes = await store.read(file.key);
6683
6787
  if (bytes === null) {
@@ -6710,7 +6814,7 @@ async function loadVectorField2(store, ref, vectorIndex, fileRefs) {
6710
6814
  }
6711
6815
  });
6712
6816
  }
6713
- restore.finish(await readGraph(store, ref));
6817
+ restore.finish();
6714
6818
  }
6715
6819
  async function loadEarlierLayout(store, indexName, manifest, deps) {
6716
6820
  let highestPartitionId = manifest.partitions.length - 1;
@@ -8068,7 +8172,7 @@ function metadataConfigOf(entry) {
8068
8172
  b: entry.config.bm25?.b ?? 0.75,
8069
8173
  ...embedding !== void 0 ? { embedding } : {},
8070
8174
  surfaceForms: entry.config.surfaceForms !== false,
8071
- analysisRevision: entry.language.revision,
8175
+ ...entry.analysisRevision !== null ? { analysisRevision: entry.analysisRevision } : {},
8072
8176
  ...typeof entry.config.tokenizer === "string" ? { tokenizer: entry.config.tokenizer } : {},
8073
8177
  ...typeof entry.config.stopWords === "string" ? { stopWords: entry.config.stopWords } : {},
8074
8178
  ...entry.config.stopWords instanceof Set ? { stopWordList: [...entry.config.stopWords].sort(compareCodePoints) } : {},
@@ -8156,7 +8260,8 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
8156
8260
  indexUuid: metadata.indexUuid ?? null,
8157
8261
  heldPartitions: metadata.heldPartitions ?? null,
8158
8262
  documentCount: metadata.documentCount ?? 0,
8159
- partitionCount: metadata.partitionCount
8263
+ partitionCount: metadata.partitionCount,
8264
+ analysisRevision: metadata.analysisRevision ?? null
8160
8265
  });
8161
8266
  if (loadData) await deps.indexState().registerOpen(metadata.indexName);
8162
8267
  else deps.indexState().registerClosed(metadata.indexName);
@@ -8279,8 +8384,14 @@ function createIndexStateCoordinator(config, callbacks) {
8279
8384
  return entry;
8280
8385
  }
8281
8386
  }
8387
+ function transitionPending(indexName, entry) {
8388
+ if (entries.get(indexName) !== entry) return true;
8389
+ if (entry.state === "dropping" && entry.dropPromise !== null) return true;
8390
+ return entry.state === "closing" && entry.closePromise !== null;
8391
+ }
8282
8392
  async function reopen(indexName, explicit) {
8283
- const entry = await awaitPendingTransitions(indexName);
8393
+ let entry = await awaitPendingTransitions(indexName);
8394
+ while (transitionPending(indexName, entry)) entry = await awaitPendingTransitions(indexName);
8284
8395
  if (entry.state === "open") return;
8285
8396
  if (explicit) {
8286
8397
  entry.reopenFailures = 0;
@@ -10695,7 +10806,13 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
10695
10806
  idleSweep: null
10696
10807
  };
10697
10808
  startIdleSweep(state);
10809
+ function settlingCopyWork(indexName) {
10810
+ return state.copyTransitions.get(indexName)?.done ?? state.compactionsInFlight.get(indexName);
10811
+ }
10698
10812
  async function resyncIndex(indexName, wasScaledOut) {
10813
+ for (let pending = settlingCopyWork(indexName); pending !== void 0; pending = settlingCopyWork(indexName)) {
10814
+ await pending;
10815
+ }
10699
10816
  state.desyncedIndexes.delete(indexName);
10700
10817
  if (!wasScaledOut) return;
10701
10818
  await scaleOutIndex(state, indexName, copyThresholdReason(state, indexName));
@@ -11047,6 +11164,10 @@ function createEngineCore(config, hooks, capacityScope = "index") {
11047
11164
  getManager: (indexName) => executor.getManager(indexName),
11048
11165
  desyncIndex: (indexName) => orchestrator.desyncIndex(indexName),
11049
11166
  resyncIndex: (indexName, wasPromoted) => orchestrator.resyncIndex(indexName, wasPromoted),
11167
+ recordCurrentAnalysis: (indexName) => {
11168
+ const entry = indexRegistry.get(indexName);
11169
+ if (entry !== void 0 && !analysisRebuild.isStale(indexName)) entry.analysisRevision = entry.language.revision;
11170
+ },
11050
11171
  durabilityManager: durability?.manager ?? null
11051
11172
  });
11052
11173
  const indexState = wireIndexState({
@@ -11221,11 +11342,16 @@ function refuseCode(entry) {
11221
11342
  }
11222
11343
  function indexConfigurationOf(entry) {
11223
11344
  refuseCode(entry);
11224
- return configurationPayloadOf(entry.config, entry.language, entry.embeddingAdapterName);
11345
+ return configurationPayloadOf(entry.config, entry.language, entry.embeddingAdapterName, entry.analysisRevision);
11225
11346
  }
11226
11347
  function requireConfigurationWithinSizeLimit(config, language) {
11227
11348
  const adapter = config.embedding?.adapter;
11228
- const payload = configurationPayloadOf(config, language, typeof adapter === "string" ? adapter : null);
11349
+ const payload = configurationPayloadOf(
11350
+ config,
11351
+ language,
11352
+ typeof adapter === "string" ? adapter : null,
11353
+ language.revision
11354
+ );
11229
11355
  if (messagePackUpperBoundBytes(payload) <= MAX_INDEX_CONFIGURATION_BYTES) return;
11230
11356
  const bytes = encode(payload).byteLength;
11231
11357
  if (bytes <= MAX_INDEX_CONFIGURATION_BYTES) return;
@@ -11235,12 +11361,12 @@ function requireConfigurationWithinSizeLimit(config, language) {
11235
11361
  { bytes, maximum: MAX_INDEX_CONFIGURATION_BYTES }
11236
11362
  );
11237
11363
  }
11238
- function configurationPayloadOf(config, language, adapterName) {
11364
+ function configurationPayloadOf(config, language, adapterName, analysisRevision) {
11239
11365
  const bm25 = config.bm25;
11240
11366
  return {
11241
11367
  schema: config.schema,
11242
11368
  language: language.name,
11243
- analysisRevision: language.revision,
11369
+ ...analysisRevision !== null ? { analysisRevision } : {},
11244
11370
  ...typeof config.tokenizer === "string" ? { tokenizer: config.tokenizer } : {},
11245
11371
  ...typeof config.stopWords === "string" ? { stopWords: config.stopWords } : {},
11246
11372
  ...config.stopWords instanceof Set ? { stopWordList: [...config.stopWords].sort(compareCodePoints) } : {},
@@ -11399,7 +11525,8 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
11399
11525
  indexUuid: indexUuid ?? null,
11400
11526
  heldPartitions: null,
11401
11527
  documentCount: 0,
11402
- partitionCount: indexConfig.partitions?.maxPartitions ?? 1
11528
+ partitionCount: indexConfig.partitions?.maxPartitions ?? 1,
11529
+ analysisRevision: language.revision
11403
11530
  });
11404
11531
  await core.indexState.registerOpen(name);
11405
11532
  if (core.durability) {
@@ -11411,6 +11538,7 @@ async function dropEngineIndex(core, name) {
11411
11538
  core.guardShutdown();
11412
11539
  const entry = core.requireIndex(name);
11413
11540
  await core.indexState.drop(name, async () => {
11541
+ await core.analysisRebuild.abandon(name);
11414
11542
  await core.orchestrator.closeIndex(name);
11415
11543
  if (core.durability) {
11416
11544
  try {
@@ -13690,7 +13818,7 @@ function partitionVectorPlan(vectorIndex, docIds) {
13690
13818
  codes: null,
13691
13819
  vectors: vectorsToBytes(packed)
13692
13820
  };
13693
- files.push({ payload, ordinals: new Int32Array(slice.length) });
13821
+ files.push({ payload, ordinals: new Int32Array(slice.length), dead: null });
13694
13822
  }
13695
13823
  return {
13696
13824
  unchanged: false,
@@ -14186,7 +14314,8 @@ async function createRestoredIndex(indexName, prepared, partitionCount, deps) {
14186
14314
  indexUuid: null,
14187
14315
  heldPartitions: null,
14188
14316
  documentCount: 0,
14189
- partitionCount
14317
+ partitionCount,
14318
+ analysisRevision: prepared.storedRevision
14190
14319
  });
14191
14320
  await deps.indexState.registerOpen(indexName);
14192
14321
  const release = await deps.indexState.acquire(indexName, false);
@@ -14571,6 +14700,15 @@ async function enterRead(core, indexName) {
14571
14700
  core.guardShutdown();
14572
14701
  if (core.rebalancingIndexes.has(indexName)) await yieldToEventLoop();
14573
14702
  }
14703
+ async function readUnderOneAnalysis(core, indexName, read) {
14704
+ let analysisStale = core.analysisRebuild.isStale(indexName);
14705
+ let result = await read();
14706
+ while (core.analysisRebuild.isStale(indexName) !== analysisStale) {
14707
+ analysisStale = core.analysisRebuild.isStale(indexName);
14708
+ result = await read();
14709
+ }
14710
+ return { result, analysisStale };
14711
+ }
14574
14712
  function scopedParams(params, options) {
14575
14713
  if (options?.globalStats === void 0) {
14576
14714
  return params;
@@ -14603,7 +14741,7 @@ async function runEngineQuery(core, indexName, params, options) {
14603
14741
  entry.embeddingAdapterName
14604
14742
  );
14605
14743
  const workerSearch = core.orchestrator.hasWorkerPool() ? core.orchestrator.searchViaWorker.bind(core.orchestrator) : void 0;
14606
- const result = await executeQuery(resolvedParams, {
14744
+ const context = {
14607
14745
  manager,
14608
14746
  language: entry.language,
14609
14747
  config: entry.config,
@@ -14613,7 +14751,15 @@ async function runEngineQuery(core, indexName, params, options) {
14613
14751
  partitionIds: options?.partitionIds,
14614
14752
  cursorBinding: queryBindingOf(params),
14615
14753
  patternWorkCap: core.patternWorkCap
14616
- });
14754
+ };
14755
+ const { result, analysisStale } = await readUnderOneAnalysis(
14756
+ core,
14757
+ indexName,
14758
+ () => executeQuery(resolvedParams, context)
14759
+ );
14760
+ if (analysisStale) {
14761
+ result.analysisStale = true;
14762
+ }
14617
14763
  if (core.pluginRegistry.hasHooks("afterSearch")) {
14618
14764
  const observed = { ...result, hits: [...result.hits] };
14619
14765
  try {
@@ -14622,9 +14768,6 @@ async function runEngineQuery(core, indexName, params, options) {
14622
14768
  console.warn("afterSearch plugin hook error:", err);
14623
14769
  }
14624
14770
  }
14625
- if (core.analysisRebuild.isStale(indexName)) {
14626
- result.analysisStale = true;
14627
- }
14628
14771
  return result;
14629
14772
  } finally {
14630
14773
  release();
@@ -14645,7 +14788,7 @@ async function runEnginePreflight(core, indexName, params, options) {
14645
14788
  entry.embeddingAdapterName
14646
14789
  );
14647
14790
  const workerSearch = core.orchestrator.hasWorkerPool() ? core.orchestrator.searchViaWorker.bind(core.orchestrator) : void 0;
14648
- const result = await executePreflight(resolvedParams, {
14791
+ const context = {
14649
14792
  manager,
14650
14793
  language: entry.language,
14651
14794
  config: entry.config,
@@ -14655,8 +14798,13 @@ async function runEnginePreflight(core, indexName, params, options) {
14655
14798
  partitionIds: options?.partitionIds,
14656
14799
  cursorBinding: queryBindingOf(params),
14657
14800
  patternWorkCap: core.patternWorkCap
14658
- });
14659
- if (core.analysisRebuild.isStale(indexName)) {
14801
+ };
14802
+ const { result, analysisStale } = await readUnderOneAnalysis(
14803
+ core,
14804
+ indexName,
14805
+ () => executePreflight(resolvedParams, context)
14806
+ );
14807
+ if (analysisStale) {
14660
14808
  result.analysisStale = true;
14661
14809
  }
14662
14810
  return result;
@@ -1,9 +1,9 @@
1
- export { createDirectExecutor } from './chunk-4CWHWOBI.mjs';
1
+ export { createDirectExecutor } from './chunk-IA67SOJO.mjs';
2
2
  import './chunk-XYDP7CCJ.mjs';
3
3
  import './chunk-OLXM65YH.mjs';
4
4
  import './chunk-ZCKPRCYI.mjs';
5
5
  import './chunk-TGP5M5R7.mjs';
6
- import './chunk-IN5CH4P2.mjs';
6
+ import './chunk-RZXNWQSM.mjs';
7
7
  import './chunk-ZJ4CSAB2.mjs';
8
8
  import './chunk-B3UOQJD3.mjs';
9
9
  import './chunk-SREPGC6O.mjs';
@@ -1,15 +1,15 @@
1
1
  import { MAX_MESSAGE_SIZE_BYTES } from './chunk-UNEEFTEU.mjs';
2
2
  import { validateIndexName, indexConfigKey, withPartitionCount, indexSettingsOf, putIndexMetadata, getIndexMetadata, getClusterIndexConfig } from './chunk-GNYXCLJS.mjs';
3
- import { requireKnownConfig, resolveRunnableConfig, createEngineCore, startEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, createEngineIndex, buildArchiveFromMemory, indexMetadataOf, encodeIndexConfiguration, archiveStream, applyIndexEntry, applyDeleteEntry, requireConfigurationWithinSizeLimit, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, createForwardBatchResultMessage, memoryCheckpointStore, readSnapshotArchive, createByteReader, openChecksummedEnvelope, loadPartitionSegments, readListedFile, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage, validateReplicationEntry, crc32Init, crc32Final, crc32Update, computeEntryChecksum, createEntryMessage, createEntryBatchMessage, createInsyncConfirmMessage, validateAckPayload, createInsyncRemoveMessage, validateInsyncConfirmPayload, createInsyncAddMessage } from './chunk-IH3GKNDQ.mjs';
3
+ import { requireKnownConfig, resolveRunnableConfig, createEngineCore, startEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, createEngineIndex, encodeIndexConfiguration, buildArchiveFromMemory, indexMetadataOf, archiveStream, applyIndexEntry, applyDeleteEntry, requireConfigurationWithinSizeLimit, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, createForwardBatchResultMessage, memoryCheckpointStore, readSnapshotArchive, createByteReader, openChecksummedEnvelope, loadPartitionSegments, readListedFile, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage, validateReplicationEntry, crc32Init, crc32Final, crc32Update, computeEntryChecksum, createEntryMessage, createEntryBatchMessage, createInsyncConfirmMessage, validateAckPayload, createInsyncRemoveMessage, validateInsyncConfirmPayload, createInsyncAddMessage } from './chunk-XIQZKKTK.mjs';
4
4
  import { now, listBindingOf, requireSurfaceFormsForPartialFields, queryBindingOf, hitsKeptPerGroup, clampLimit, resolveHybridFusion, dedupePinnedEntries, foldGroupReducer } from './chunk-4D6OH4EF.mjs';
5
5
  import { ReplicationMessageTypes, TransportErrorCodes, ClusterMessageTypes, QueryMessageTypes, TransportError } from './chunk-GCM3JFVA.mjs';
6
6
  import { MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR, WIRE_BATCH_BUDGET, providedDocId } from './chunk-2453RJYK.mjs';
7
- import { generateId, readHeapStatistics, decodeVectorFilePayload, bytesToVectors, isDead } from './chunk-4CWHWOBI.mjs';
7
+ import { generateId, readHeapStatistics, decodeVectorFilePayload, bytesToVectors, isDead } from './chunk-IA67SOJO.mjs';
8
8
  import { oversampledShardSize, resolveProjection, clampRowCount, sortSignatureOf, requireMatchingCursor, decodePageCursor, normalizeSort, sortModesOf, encodePageCursor, applyProjection, readSortValues, requireWithinResultWindow, fnv1a, sortSignatureEntry, FILTER_EXPRESSION_KEYS, filterTestCount, fnv1aUint32s, mergePartitionStats, FIELD_FILTER_OPERATORS } from './chunk-XYDP7CCJ.mjs';
9
9
  import { getLanguage } from './chunk-OLXM65YH.mjs';
10
10
  import { DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, MAX_INDEX_NAME_LENGTH, MAX_DOC_ID_LENGTH, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING, INDEX_NAME_PATTERN } from './chunk-ZCKPRCYI.mjs';
11
11
  import './chunk-TGP5M5R7.mjs';
12
- import './chunk-IN5CH4P2.mjs';
12
+ import './chunk-RZXNWQSM.mjs';
13
13
  import { MIN_OVERSAMPLE } from './chunk-ZJ4CSAB2.mjs';
14
14
  import './chunk-B3UOQJD3.mjs';
15
15
  import { isPatternTolerance } from './chunk-SREPGC6O.mjs';
@@ -5623,6 +5623,7 @@ async function archiveReplicationPartition(core, indexName, partitionId, lastSeq
5623
5623
  const entry = core.requireIndex(indexName);
5624
5624
  const manager = core.requireManager(indexName);
5625
5625
  let captured = 0;
5626
+ const config = await encodeIndexConfiguration(entry);
5626
5627
  const checkpoint = await buildArchiveFromMemory({
5627
5628
  indexName,
5628
5629
  metadata: indexMetadataOf(indexName, entry, manager.partitionCount),
@@ -5634,7 +5635,6 @@ async function archiveReplicationPartition(core, indexName, partitionId, lastSeq
5634
5635
  },
5635
5636
  partitionId
5636
5637
  });
5637
- const config = await encodeIndexConfiguration(entry);
5638
5638
  return { stream: archiveStream({ ...checkpoint, config, sourceIndexName: indexName }), lastSeqNo: captured };
5639
5639
  } finally {
5640
5640
  release();
@@ -5660,6 +5660,7 @@ function requireRestoreTarget(indexName, partitionId, partitionCount) {
5660
5660
  async function receiveArchive(indexName, partitionId, source, schema) {
5661
5661
  const store = memoryCheckpointStore();
5662
5662
  let language = "";
5663
+ let analysisRevision = null;
5663
5664
  let received = null;
5664
5665
  await readSnapshotArchive(createByteReader(source), {
5665
5666
  config: async (bytes) => {
@@ -5671,6 +5672,7 @@ async function receiveArchive(indexName, partitionId, source, schema) {
5671
5672
  const raw = decode(payload);
5672
5673
  if (typeof raw.language !== "string") restoreFailed("partition archive names no language", { indexName });
5673
5674
  language = raw.language;
5675
+ analysisRevision = typeof raw.analysisRevision === "string" ? raw.analysisRevision : null;
5674
5676
  },
5675
5677
  manifest: async (manifest, sourceName) => {
5676
5678
  const listed = manifest.partitions.map((partition) => partition.partitionId);
@@ -5697,7 +5699,7 @@ async function receiveArchive(indexName, partitionId, source, schema) {
5697
5699
  file: async (name, _ref, bytes) => store.write(`${indexName}/${name}`, [bytes])
5698
5700
  });
5699
5701
  if (received === null) restoreFailed("partition archive holds no manifest", { indexName, partitionId });
5700
- return { language, manifest: received, store };
5702
+ return { language, analysisRevision, manifest: received, store };
5701
5703
  }
5702
5704
  function emptyPartition(manager, partitionId) {
5703
5705
  return {
@@ -5749,6 +5751,18 @@ async function replacePartition(manager, partitionId, archive) {
5749
5751
  vectorIndex.scheduleBuild();
5750
5752
  }
5751
5753
  }
5754
+ function recordReceivedAnalysis(core, indexName, receivedRevision) {
5755
+ const entry = core.requireIndex(indexName);
5756
+ const current = entry.language.revision;
5757
+ if (receivedRevision === current) return;
5758
+ entry.analysisRevision = entry.analysisRevision === current || entry.analysisRevision === receivedRevision ? receivedRevision : null;
5759
+ core.analysisRebuild.markStale({
5760
+ indexName,
5761
+ language: entry.language.name,
5762
+ storedRevision: entry.analysisRevision,
5763
+ currentRevision: current
5764
+ });
5765
+ }
5752
5766
  async function restoreReplicationPartition(core, engine, indexName, partitionId, source, indexConfig, partitionCount) {
5753
5767
  core.guardShutdown();
5754
5768
  requireRestoreTarget(indexName, partitionId, partitionCount);
@@ -5775,6 +5789,7 @@ async function restoreReplicationPartition(core, engine, indexName, partitionId,
5775
5789
  const wasScaledOut = core.orchestrator.desyncIndex(indexName);
5776
5790
  try {
5777
5791
  await replacePartition(core.requireManager(indexName), partitionId, archive);
5792
+ recordReceivedAnalysis(core, indexName, archive.analysisRevision);
5778
5793
  } finally {
5779
5794
  await core.orchestrator.resyncIndex(indexName, wasScaledOut);
5780
5795
  }