@delali/narsil 0.3.1-next.83 → 0.3.1-next.84

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,7 +1,7 @@
1
- import { validateWorkerConfig, validateIndexName, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateBM25Params, validatePartitionConfig, executeQuery, queryBindingOf, executePreflight, executeSuggest, executeListDocuments, validateDocId } from './chunk-4D6OH4EF.mjs';
1
+ import { validateWorkerConfig, validateIndexName, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateBM25Params, validatePartitionConfig, queryBindingOf, executeQuery, executePreflight, executeSuggest, executeListDocuments, validateDocId } from './chunk-4D6OH4EF.mjs';
2
2
  import { ReplicationMessageTypes } from './chunk-GCM3JFVA.mjs';
3
3
  import { WIRE_BATCH_BUDGET, resolveVectorText, bindEngineCore, restoredConfigFields, restoredEmbedding, collectRequiredFieldFailures, embedChunkDocuments, providedDocId, requireDocumentObject, admitInsert, asBatchInsertError, requireMatchingDocId, embedDocumentFields, assertDocumentCarriesMappedVectors, admitBufferedUpsert, createPendingAdmissions, recordPendingAdmission } from './chunk-2453RJYK.mjs';
4
- import { NATIVE_CRC32_MIN_BYTES, generateId, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, splitWorkerBudget, resolveWorkerCount, createDirectExecutor, createPartitionRouter, MAX_FILE_BYTES, resolveWorkerEntry, missingWorkerEntry, readHeapStatistics, nodeThreadId, CHECKSUM_YIELD_CHUNK_BYTES, resolvePartitionInsertOptions, resolveIndexAnalysis, createPartitionManager, buildSegmentPayload, createFrozenSegment, resolveRequestThreadCount, vectorsPerFile, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, createPartitionIndex, selectionKeeps, decodeMapWithoutFields, isCompositePartition, segmentArraysOf, mergeSegmentContents, survivorDocumentsOf, partitionStateOf, VECTOR_FILE_MAX_VECTORS, deadBytesFor, mergeFrozenSegments, workerResourceLimits, decodeVectorIndexParts, bytesToVectors, freezeSegmentShared, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, SHUTDOWN_TIMEOUT_MS, decodeVectorFilePayload, decodeVectorGraphPartPayload, assembleGraphParts, countDead, encodeSegmentState, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, VALID_HNSW_METRICS, spawnNodeWorker, decodeVectorGraphPayload, graphPartsOfEarlierPayload, decodeStoredDocument, isDead } from './chunk-CUP2E4OH.mjs';
4
+ import { NATIVE_CRC32_MIN_BYTES, generateId, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, splitWorkerBudget, resolveWorkerCount, createDirectExecutor, createPartitionRouter, MAX_FILE_BYTES, resolveWorkerEntry, missingWorkerEntry, readHeapStatistics, nodeThreadId, CHECKSUM_YIELD_CHUNK_BYTES, resolvePartitionInsertOptions, resolveIndexAnalysis, createPartitionManager, buildSegmentPayload, createFrozenSegment, resolveRequestThreadCount, vectorsPerFile, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, createPartitionIndex, selectionKeeps, decodeMapWithoutFields, isCompositePartition, segmentArraysOf, mergeSegmentContents, survivorDocumentsOf, partitionStateOf, VECTOR_FILE_MAX_VECTORS, deadBytesFor, mergeFrozenSegments, workerResourceLimits, decodeVectorIndexParts, bytesToVectors, freezeSegmentShared, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, SHUTDOWN_TIMEOUT_MS, decodeVectorFilePayload, decodeVectorGraphPartPayload, assembleGraphParts, countDead, encodeSegmentState, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, VALID_HNSW_METRICS, spawnNodeWorker, decodeVectorGraphPayload, graphPartsOfEarlierPayload, decodeStoredDocument, isDead } from './chunk-IA67SOJO.mjs';
5
5
  import { requirePatternSearchFor, requirePartialWordSearchFor, collectQueryTermStats, mergePartitionStats, oversampledFacetConfig, fnv1a, availablePatternSearch, PATTERN_HOST, kWayMerge, mergeFacets, setNestedValue } from './chunk-XYDP7CCJ.mjs';
6
6
  import { detectRuntime, getLanguage } from './chunk-OLXM65YH.mjs';
7
7
  import { MAX_INDEX_CONFIGURATION_BYTES, STATISTICS_BROADCAST_INTERVAL_MS, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, HEAP_PRESSURE_WARNING_FRACTION, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, FOREIGN_STATS_TTL_MS, MAX_IMPORT_REFUSALS_REPORTED, BATCH_CHUNK_SIZE, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION, EXPORT_CHUNK_BYTES } from './chunk-ZCKPRCYI.mjs';
@@ -160,6 +160,8 @@ function yieldToEventLoop() {
160
160
  // src/partitioning/constants.ts
161
161
  var REBALANCE_CHUNK_SIZE = 1e3;
162
162
  var DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE = 1e4;
163
+ var TEXT_REBUILD_CHUNK_DOCUMENTS = 1e3;
164
+ var TEXT_REBUILD_SWITCH_BACKLOG_DOCUMENTS = 1e3;
163
165
 
164
166
  // src/partitioning/rebalancer.ts
165
167
  function createRebalancer() {
@@ -452,6 +454,74 @@ function validateRequiredFieldsInSchema(required, schema) {
452
454
  }
453
455
  }
454
456
 
457
+ // src/partitioning/text-rebuild.ts
458
+ function interrupted(pass) {
459
+ return pass.log.partitionsReplaced || !pass.hooks.isCurrent();
460
+ }
461
+ async function buildPartition(pass, source, partitionId) {
462
+ const { manager, options } = pass;
463
+ const target = createPartitionIndex(partitionId, manager.config.trackPositions ?? true);
464
+ const docIds = [...source.docIds()];
465
+ target.beginBatch();
466
+ for (let start = 0; start < docIds.length; start += TEXT_REBUILD_CHUNK_DOCUMENTS) {
467
+ const end = Math.min(start + TEXT_REBUILD_CHUNK_DOCUMENTS, docIds.length);
468
+ for (let i = start; i < end; i++) {
469
+ const document = manager.getRef(docIds[i]);
470
+ if (document !== void 0) target.insert(docIds[i], document, manager.schema, manager.language, options);
471
+ }
472
+ await yieldToEventLoop();
473
+ if (interrupted(pass)) return null;
474
+ }
475
+ target.endBatch();
476
+ if (manager.partialWordAnalysis !== null) target.preparePartialWords(manager.partialWordAnalysis);
477
+ return target;
478
+ }
479
+ function applyWrittenDocuments(pass, built) {
480
+ const { manager, options } = pass;
481
+ for (const docId of pass.log.takeWrittenDocIds()) {
482
+ for (const partition of built) {
483
+ if (!partition.has(docId)) continue;
484
+ partition.remove(docId, manager.schema, manager.language, options);
485
+ break;
486
+ }
487
+ const partitionId = manager.partitionIdOf(docId);
488
+ const document = manager.getRef(docId);
489
+ if (partitionId === void 0 || document === void 0) continue;
490
+ built[partitionId].insert(docId, document, manager.schema, manager.language, options);
491
+ }
492
+ }
493
+ async function rebuildOnce(pass) {
494
+ const partitionCount = pass.manager.partitionCount;
495
+ const built = [];
496
+ for (let partitionId = 0; partitionId < partitionCount; partitionId++) {
497
+ const partition = await buildPartition(pass, pass.manager.getPartition(partitionId), partitionId);
498
+ if (partition === null) return false;
499
+ built.push(partition);
500
+ pass.hooks.onPartitionBuilt(built.length);
501
+ }
502
+ while (pass.log.pendingDocuments > TEXT_REBUILD_SWITCH_BACKLOG_DOCUMENTS) {
503
+ applyWrittenDocuments(pass, built);
504
+ await yieldToEventLoop();
505
+ if (interrupted(pass)) return false;
506
+ }
507
+ applyWrittenDocuments(pass, built);
508
+ pass.manager.adoptRebuiltPartitions(built);
509
+ pass.hooks.onSwitch();
510
+ return true;
511
+ }
512
+ async function rebuildTextIndexes(manager, hooks) {
513
+ const options = resolvePartitionInsertOptions(manager.config, manager.analysis, { validate: false, skipClone: true });
514
+ while (hooks.isCurrent()) {
515
+ const log = manager.openWriteLog();
516
+ try {
517
+ if (await rebuildOnce({ manager, log, hooks, options })) return true;
518
+ } finally {
519
+ log.close();
520
+ }
521
+ }
522
+ return false;
523
+ }
524
+
455
525
  // src/engine/events.ts
456
526
  function eventHandlersOf(events) {
457
527
  const handlers = /* @__PURE__ */ new Map();
@@ -474,11 +544,6 @@ function emitEngineEvent(eventHandlers, event, payload) {
474
544
  }
475
545
 
476
546
  // src/engine/analysis-rebuild.ts
477
- function yieldToEventLoop2() {
478
- return new Promise((resolve) => {
479
- setTimeout(resolve, 0);
480
- });
481
- }
482
547
  function toError(value) {
483
548
  return value instanceof Error ? value : new Error(String(value));
484
549
  }
@@ -489,9 +554,13 @@ function wireAnalysisRebuild(wiring) {
489
554
  resyncIndex: wiring.resyncIndex,
490
555
  async persistAnalysisRevision(indexName) {
491
556
  const durability = wiring.durabilityManager;
492
- if (durability === null) return;
493
- await durability.persistMetadata(indexName);
557
+ if (durability === null) {
558
+ wiring.recordCurrentAnalysis(indexName);
559
+ return;
560
+ }
494
561
  await (durability.checkpointFromMemory?.(indexName) ?? durability.checkpoint(indexName));
562
+ wiring.recordCurrentAnalysis(indexName);
563
+ await durability.persistMetadata(indexName);
495
564
  },
496
565
  emit(payload) {
497
566
  emitEngineEvent(wiring.eventHandlers, "analysisRebuild", payload);
@@ -502,9 +571,10 @@ function createAnalysisRebuildCoordinator(config, deps) {
502
571
  const stale = /* @__PURE__ */ new Map();
503
572
  const rebuildMode = config?.rebuild ?? "auto";
504
573
  let queue = Promise.resolve();
574
+ let active = null;
505
575
  async function rebuildNow(indexName, entry) {
506
576
  const manager = deps.getManager(indexName);
507
- if (manager === void 0) {
577
+ if (manager === void 0 || entry.abandoned) {
508
578
  entry.run = null;
509
579
  return;
510
580
  }
@@ -512,12 +582,20 @@ function createAnalysisRebuildCoordinator(config, deps) {
512
582
  deps.emit({ indexName, status: "started", partitionsRebuilt: 0, partitionCount });
513
583
  const wasPromoted = deps.desyncIndex(indexName);
514
584
  try {
515
- for (let partitionId = 0; partitionId < manager.partitionCount; partitionId += 1) {
516
- manager.rebuildTextIndex(partitionId);
517
- entry.partitionsRebuilt = partitionId + 1;
518
- await yieldToEventLoop2();
585
+ const rebuilt = await rebuildTextIndexes(manager, {
586
+ isCurrent: () => deps.getManager(indexName) === manager && stale.get(indexName) === entry && !entry.abandoned,
587
+ onPartitionBuilt(partitionsBuilt) {
588
+ entry.partitionsRebuilt = partitionsBuilt;
589
+ },
590
+ onSwitch() {
591
+ stale.delete(indexName);
592
+ }
593
+ });
594
+ if (!rebuilt) {
595
+ entry.run = null;
596
+ if (deps.getManager(indexName) === manager) await deps.resyncIndex(indexName, wasPromoted);
597
+ return;
519
598
  }
520
- stale.delete(indexName);
521
599
  await deps.persistAnalysisRevision(indexName);
522
600
  await deps.resyncIndex(indexName, wasPromoted);
523
601
  deps.emit({
@@ -528,7 +606,6 @@ function createAnalysisRebuildCoordinator(config, deps) {
528
606
  });
529
607
  } catch (err) {
530
608
  const error = toError(err);
531
- stale.set(indexName, entry);
532
609
  entry.run = null;
533
610
  await deps.resyncIndex(indexName, wasPromoted).catch(() => void 0);
534
611
  deps.emit({
@@ -541,11 +618,20 @@ function createAnalysisRebuildCoordinator(config, deps) {
541
618
  throw error;
542
619
  }
543
620
  }
621
+ function runTracked(indexName, entry) {
622
+ const done = rebuildNow(indexName, entry);
623
+ const tracked = { indexName, entry, done };
624
+ active = tracked;
625
+ return done.finally(() => {
626
+ if (active === tracked) active = null;
627
+ });
628
+ }
544
629
  function enqueue(indexName, entry) {
545
630
  if (entry.run !== null) {
546
631
  return entry.run;
547
632
  }
548
- const run = queue.then(() => rebuildNow(indexName, entry));
633
+ entry.abandoned = false;
634
+ const run = queue.then(() => runTracked(indexName, entry));
549
635
  entry.run = run;
550
636
  queue = run.then(
551
637
  () => void 0,
@@ -580,7 +666,7 @@ function createAnalysisRebuildCoordinator(config, deps) {
580
666
  if (existing !== void 0 && existing.run !== null) {
581
667
  return;
582
668
  }
583
- stale.set(index.indexName, { index, partitionsRebuilt: 0, run: null });
669
+ stale.set(index.indexName, { index, partitionsRebuilt: 0, run: null, abandoned: false });
584
670
  },
585
671
  clearStale(indexName) {
586
672
  stale.delete(indexName);
@@ -591,6 +677,14 @@ function createAnalysisRebuildCoordinator(config, deps) {
591
677
  isRunning(indexName) {
592
678
  return stale.get(indexName)?.run !== null && stale.get(indexName)?.run !== void 0;
593
679
  },
680
+ async abandon(indexName) {
681
+ const entry = stale.get(indexName);
682
+ if (entry !== void 0 && entry.run !== null) entry.abandoned = true;
683
+ const running = active;
684
+ if (running === null || running.indexName !== indexName) return;
685
+ running.entry.abandoned = true;
686
+ await running.done.catch(() => void 0);
687
+ },
594
688
  review(indexName) {
595
689
  const entry = stale.get(indexName);
596
690
  return entry === void 0 ? Promise.resolve() : review(indexName, entry);
@@ -1400,7 +1494,7 @@ function buildEntry(fields) {
1400
1494
  }
1401
1495
 
1402
1496
  // src/serialization/chunk-checksum.ts
1403
- function yieldToEventLoop3() {
1497
+ function yieldToEventLoop2() {
1404
1498
  if (typeof setImmediate === "function") {
1405
1499
  return new Promise((resolve) => setImmediate(resolve));
1406
1500
  }
@@ -1412,7 +1506,7 @@ async function advanceChecksum(start, payload) {
1412
1506
  const end = Math.min(offset + CHECKSUM_YIELD_CHUNK_BYTES, payload.length);
1413
1507
  state = crc32Update(state, payload.subarray(offset, end));
1414
1508
  if (end < payload.length) {
1415
- await yieldToEventLoop3();
1509
+ await yieldToEventLoop2();
1416
1510
  }
1417
1511
  }
1418
1512
  return state;
@@ -5639,7 +5733,7 @@ function removeWhatTheSegmentReplaces(manager, segment, contents) {
5639
5733
  manager.endBatchRemove();
5640
5734
  }
5641
5735
  }
5642
- function yieldToEventLoop4() {
5736
+ function yieldToEventLoop3() {
5643
5737
  return new Promise((resolve) => setTimeout(resolve, 0));
5644
5738
  }
5645
5739
  async function loadPartitionSegments(store, partition, manager, options) {
@@ -5650,7 +5744,7 @@ async function loadPartitionSegments(store, partition, manager, options) {
5650
5744
  const segment = createSharedFrozenSegment(shareSegmentArrays(contents.arrays, manager.partialWordAnalysis));
5651
5745
  removeWhatTheSegmentReplaces(manager, segment, contents);
5652
5746
  if (contents.arrays.documentCount > 0) manager.attachFrozenSegment(partition.partitionId, segment);
5653
- await yieldToEventLoop4();
5747
+ await yieldToEventLoop3();
5654
5748
  }
5655
5749
  }
5656
5750
 
@@ -8078,7 +8172,7 @@ function metadataConfigOf(entry) {
8078
8172
  b: entry.config.bm25?.b ?? 0.75,
8079
8173
  ...embedding !== void 0 ? { embedding } : {},
8080
8174
  surfaceForms: entry.config.surfaceForms !== false,
8081
- analysisRevision: entry.language.revision,
8175
+ ...entry.analysisRevision !== null ? { analysisRevision: entry.analysisRevision } : {},
8082
8176
  ...typeof entry.config.tokenizer === "string" ? { tokenizer: entry.config.tokenizer } : {},
8083
8177
  ...typeof entry.config.stopWords === "string" ? { stopWords: entry.config.stopWords } : {},
8084
8178
  ...entry.config.stopWords instanceof Set ? { stopWordList: [...entry.config.stopWords].sort(compareCodePoints) } : {},
@@ -8166,7 +8260,8 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
8166
8260
  indexUuid: metadata.indexUuid ?? null,
8167
8261
  heldPartitions: metadata.heldPartitions ?? null,
8168
8262
  documentCount: metadata.documentCount ?? 0,
8169
- partitionCount: metadata.partitionCount
8263
+ partitionCount: metadata.partitionCount,
8264
+ analysisRevision: metadata.analysisRevision ?? null
8170
8265
  });
8171
8266
  if (loadData) await deps.indexState().registerOpen(metadata.indexName);
8172
8267
  else deps.indexState().registerClosed(metadata.indexName);
@@ -8289,8 +8384,14 @@ function createIndexStateCoordinator(config, callbacks) {
8289
8384
  return entry;
8290
8385
  }
8291
8386
  }
8387
+ function transitionPending(indexName, entry) {
8388
+ if (entries.get(indexName) !== entry) return true;
8389
+ if (entry.state === "dropping" && entry.dropPromise !== null) return true;
8390
+ return entry.state === "closing" && entry.closePromise !== null;
8391
+ }
8292
8392
  async function reopen(indexName, explicit) {
8293
- const entry = await awaitPendingTransitions(indexName);
8393
+ let entry = await awaitPendingTransitions(indexName);
8394
+ while (transitionPending(indexName, entry)) entry = await awaitPendingTransitions(indexName);
8294
8395
  if (entry.state === "open") return;
8295
8396
  if (explicit) {
8296
8397
  entry.reopenFailures = 0;
@@ -10705,7 +10806,13 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
10705
10806
  idleSweep: null
10706
10807
  };
10707
10808
  startIdleSweep(state);
10809
+ function settlingCopyWork(indexName) {
10810
+ return state.copyTransitions.get(indexName)?.done ?? state.compactionsInFlight.get(indexName);
10811
+ }
10708
10812
  async function resyncIndex(indexName, wasScaledOut) {
10813
+ for (let pending = settlingCopyWork(indexName); pending !== void 0; pending = settlingCopyWork(indexName)) {
10814
+ await pending;
10815
+ }
10709
10816
  state.desyncedIndexes.delete(indexName);
10710
10817
  if (!wasScaledOut) return;
10711
10818
  await scaleOutIndex(state, indexName, copyThresholdReason(state, indexName));
@@ -11057,6 +11164,10 @@ function createEngineCore(config, hooks, capacityScope = "index") {
11057
11164
  getManager: (indexName) => executor.getManager(indexName),
11058
11165
  desyncIndex: (indexName) => orchestrator.desyncIndex(indexName),
11059
11166
  resyncIndex: (indexName, wasPromoted) => orchestrator.resyncIndex(indexName, wasPromoted),
11167
+ recordCurrentAnalysis: (indexName) => {
11168
+ const entry = indexRegistry.get(indexName);
11169
+ if (entry !== void 0 && !analysisRebuild.isStale(indexName)) entry.analysisRevision = entry.language.revision;
11170
+ },
11060
11171
  durabilityManager: durability?.manager ?? null
11061
11172
  });
11062
11173
  const indexState = wireIndexState({
@@ -11231,11 +11342,16 @@ function refuseCode(entry) {
11231
11342
  }
11232
11343
  function indexConfigurationOf(entry) {
11233
11344
  refuseCode(entry);
11234
- return configurationPayloadOf(entry.config, entry.language, entry.embeddingAdapterName);
11345
+ return configurationPayloadOf(entry.config, entry.language, entry.embeddingAdapterName, entry.analysisRevision);
11235
11346
  }
11236
11347
  function requireConfigurationWithinSizeLimit(config, language) {
11237
11348
  const adapter = config.embedding?.adapter;
11238
- const payload = configurationPayloadOf(config, language, typeof adapter === "string" ? adapter : null);
11349
+ const payload = configurationPayloadOf(
11350
+ config,
11351
+ language,
11352
+ typeof adapter === "string" ? adapter : null,
11353
+ language.revision
11354
+ );
11239
11355
  if (messagePackUpperBoundBytes(payload) <= MAX_INDEX_CONFIGURATION_BYTES) return;
11240
11356
  const bytes = encode(payload).byteLength;
11241
11357
  if (bytes <= MAX_INDEX_CONFIGURATION_BYTES) return;
@@ -11245,12 +11361,12 @@ function requireConfigurationWithinSizeLimit(config, language) {
11245
11361
  { bytes, maximum: MAX_INDEX_CONFIGURATION_BYTES }
11246
11362
  );
11247
11363
  }
11248
- function configurationPayloadOf(config, language, adapterName) {
11364
+ function configurationPayloadOf(config, language, adapterName, analysisRevision) {
11249
11365
  const bm25 = config.bm25;
11250
11366
  return {
11251
11367
  schema: config.schema,
11252
11368
  language: language.name,
11253
- analysisRevision: language.revision,
11369
+ ...analysisRevision !== null ? { analysisRevision } : {},
11254
11370
  ...typeof config.tokenizer === "string" ? { tokenizer: config.tokenizer } : {},
11255
11371
  ...typeof config.stopWords === "string" ? { stopWords: config.stopWords } : {},
11256
11372
  ...config.stopWords instanceof Set ? { stopWordList: [...config.stopWords].sort(compareCodePoints) } : {},
@@ -11409,7 +11525,8 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
11409
11525
  indexUuid: indexUuid ?? null,
11410
11526
  heldPartitions: null,
11411
11527
  documentCount: 0,
11412
- partitionCount: indexConfig.partitions?.maxPartitions ?? 1
11528
+ partitionCount: indexConfig.partitions?.maxPartitions ?? 1,
11529
+ analysisRevision: language.revision
11413
11530
  });
11414
11531
  await core.indexState.registerOpen(name);
11415
11532
  if (core.durability) {
@@ -11421,6 +11538,7 @@ async function dropEngineIndex(core, name) {
11421
11538
  core.guardShutdown();
11422
11539
  const entry = core.requireIndex(name);
11423
11540
  await core.indexState.drop(name, async () => {
11541
+ await core.analysisRebuild.abandon(name);
11424
11542
  await core.orchestrator.closeIndex(name);
11425
11543
  if (core.durability) {
11426
11544
  try {
@@ -14196,7 +14314,8 @@ async function createRestoredIndex(indexName, prepared, partitionCount, deps) {
14196
14314
  indexUuid: null,
14197
14315
  heldPartitions: null,
14198
14316
  documentCount: 0,
14199
- partitionCount
14317
+ partitionCount,
14318
+ analysisRevision: prepared.storedRevision
14200
14319
  });
14201
14320
  await deps.indexState.registerOpen(indexName);
14202
14321
  const release = await deps.indexState.acquire(indexName, false);
@@ -14581,6 +14700,15 @@ async function enterRead(core, indexName) {
14581
14700
  core.guardShutdown();
14582
14701
  if (core.rebalancingIndexes.has(indexName)) await yieldToEventLoop();
14583
14702
  }
14703
+ async function readUnderOneAnalysis(core, indexName, read) {
14704
+ let analysisStale = core.analysisRebuild.isStale(indexName);
14705
+ let result = await read();
14706
+ while (core.analysisRebuild.isStale(indexName) !== analysisStale) {
14707
+ analysisStale = core.analysisRebuild.isStale(indexName);
14708
+ result = await read();
14709
+ }
14710
+ return { result, analysisStale };
14711
+ }
14584
14712
  function scopedParams(params, options) {
14585
14713
  if (options?.globalStats === void 0) {
14586
14714
  return params;
@@ -14613,7 +14741,7 @@ async function runEngineQuery(core, indexName, params, options) {
14613
14741
  entry.embeddingAdapterName
14614
14742
  );
14615
14743
  const workerSearch = core.orchestrator.hasWorkerPool() ? core.orchestrator.searchViaWorker.bind(core.orchestrator) : void 0;
14616
- const result = await executeQuery(resolvedParams, {
14744
+ const context = {
14617
14745
  manager,
14618
14746
  language: entry.language,
14619
14747
  config: entry.config,
@@ -14623,7 +14751,15 @@ async function runEngineQuery(core, indexName, params, options) {
14623
14751
  partitionIds: options?.partitionIds,
14624
14752
  cursorBinding: queryBindingOf(params),
14625
14753
  patternWorkCap: core.patternWorkCap
14626
- });
14754
+ };
14755
+ const { result, analysisStale } = await readUnderOneAnalysis(
14756
+ core,
14757
+ indexName,
14758
+ () => executeQuery(resolvedParams, context)
14759
+ );
14760
+ if (analysisStale) {
14761
+ result.analysisStale = true;
14762
+ }
14627
14763
  if (core.pluginRegistry.hasHooks("afterSearch")) {
14628
14764
  const observed = { ...result, hits: [...result.hits] };
14629
14765
  try {
@@ -14632,9 +14768,6 @@ async function runEngineQuery(core, indexName, params, options) {
14632
14768
  console.warn("afterSearch plugin hook error:", err);
14633
14769
  }
14634
14770
  }
14635
- if (core.analysisRebuild.isStale(indexName)) {
14636
- result.analysisStale = true;
14637
- }
14638
14771
  return result;
14639
14772
  } finally {
14640
14773
  release();
@@ -14655,7 +14788,7 @@ async function runEnginePreflight(core, indexName, params, options) {
14655
14788
  entry.embeddingAdapterName
14656
14789
  );
14657
14790
  const workerSearch = core.orchestrator.hasWorkerPool() ? core.orchestrator.searchViaWorker.bind(core.orchestrator) : void 0;
14658
- const result = await executePreflight(resolvedParams, {
14791
+ const context = {
14659
14792
  manager,
14660
14793
  language: entry.language,
14661
14794
  config: entry.config,
@@ -14665,8 +14798,13 @@ async function runEnginePreflight(core, indexName, params, options) {
14665
14798
  partitionIds: options?.partitionIds,
14666
14799
  cursorBinding: queryBindingOf(params),
14667
14800
  patternWorkCap: core.patternWorkCap
14668
- });
14669
- if (core.analysisRebuild.isStale(indexName)) {
14801
+ };
14802
+ const { result, analysisStale } = await readUnderOneAnalysis(
14803
+ core,
14804
+ indexName,
14805
+ () => executePreflight(resolvedParams, context)
14806
+ );
14807
+ if (analysisStale) {
14670
14808
  result.analysisStale = true;
14671
14809
  }
14672
14810
  return result;
@@ -1,4 +1,4 @@
1
- export { createDirectExecutor } from './chunk-CUP2E4OH.mjs';
1
+ export { createDirectExecutor } from './chunk-IA67SOJO.mjs';
2
2
  import './chunk-XYDP7CCJ.mjs';
3
3
  import './chunk-OLXM65YH.mjs';
4
4
  import './chunk-ZCKPRCYI.mjs';
@@ -1,10 +1,10 @@
1
1
  import { MAX_MESSAGE_SIZE_BYTES } from './chunk-UNEEFTEU.mjs';
2
2
  import { validateIndexName, indexConfigKey, withPartitionCount, indexSettingsOf, putIndexMetadata, getIndexMetadata, getClusterIndexConfig } from './chunk-GNYXCLJS.mjs';
3
- import { requireKnownConfig, resolveRunnableConfig, createEngineCore, startEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, createEngineIndex, buildArchiveFromMemory, indexMetadataOf, encodeIndexConfiguration, archiveStream, applyIndexEntry, applyDeleteEntry, requireConfigurationWithinSizeLimit, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, createForwardBatchResultMessage, memoryCheckpointStore, readSnapshotArchive, createByteReader, openChecksummedEnvelope, loadPartitionSegments, readListedFile, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage, validateReplicationEntry, crc32Init, crc32Final, crc32Update, computeEntryChecksum, createEntryMessage, createEntryBatchMessage, createInsyncConfirmMessage, validateAckPayload, createInsyncRemoveMessage, validateInsyncConfirmPayload, createInsyncAddMessage } from './chunk-7JZZ3HNV.mjs';
3
+ import { requireKnownConfig, resolveRunnableConfig, createEngineCore, startEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, createEngineIndex, encodeIndexConfiguration, buildArchiveFromMemory, indexMetadataOf, archiveStream, applyIndexEntry, applyDeleteEntry, requireConfigurationWithinSizeLimit, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, createForwardBatchResultMessage, memoryCheckpointStore, readSnapshotArchive, createByteReader, openChecksummedEnvelope, loadPartitionSegments, readListedFile, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage, validateReplicationEntry, crc32Init, crc32Final, crc32Update, computeEntryChecksum, createEntryMessage, createEntryBatchMessage, createInsyncConfirmMessage, validateAckPayload, createInsyncRemoveMessage, validateInsyncConfirmPayload, createInsyncAddMessage } from './chunk-XIQZKKTK.mjs';
4
4
  import { now, listBindingOf, requireSurfaceFormsForPartialFields, queryBindingOf, hitsKeptPerGroup, clampLimit, resolveHybridFusion, dedupePinnedEntries, foldGroupReducer } from './chunk-4D6OH4EF.mjs';
5
5
  import { ReplicationMessageTypes, TransportErrorCodes, ClusterMessageTypes, QueryMessageTypes, TransportError } from './chunk-GCM3JFVA.mjs';
6
6
  import { MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR, WIRE_BATCH_BUDGET, providedDocId } from './chunk-2453RJYK.mjs';
7
- import { generateId, readHeapStatistics, decodeVectorFilePayload, bytesToVectors, isDead } from './chunk-CUP2E4OH.mjs';
7
+ import { generateId, readHeapStatistics, decodeVectorFilePayload, bytesToVectors, isDead } from './chunk-IA67SOJO.mjs';
8
8
  import { oversampledShardSize, resolveProjection, clampRowCount, sortSignatureOf, requireMatchingCursor, decodePageCursor, normalizeSort, sortModesOf, encodePageCursor, applyProjection, readSortValues, requireWithinResultWindow, fnv1a, sortSignatureEntry, FILTER_EXPRESSION_KEYS, filterTestCount, fnv1aUint32s, mergePartitionStats, FIELD_FILTER_OPERATORS } from './chunk-XYDP7CCJ.mjs';
9
9
  import { getLanguage } from './chunk-OLXM65YH.mjs';
10
10
  import { DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, MAX_INDEX_NAME_LENGTH, MAX_DOC_ID_LENGTH, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING, INDEX_NAME_PATTERN } from './chunk-ZCKPRCYI.mjs';
@@ -5623,6 +5623,7 @@ async function archiveReplicationPartition(core, indexName, partitionId, lastSeq
5623
5623
  const entry = core.requireIndex(indexName);
5624
5624
  const manager = core.requireManager(indexName);
5625
5625
  let captured = 0;
5626
+ const config = await encodeIndexConfiguration(entry);
5626
5627
  const checkpoint = await buildArchiveFromMemory({
5627
5628
  indexName,
5628
5629
  metadata: indexMetadataOf(indexName, entry, manager.partitionCount),
@@ -5634,7 +5635,6 @@ async function archiveReplicationPartition(core, indexName, partitionId, lastSeq
5634
5635
  },
5635
5636
  partitionId
5636
5637
  });
5637
- const config = await encodeIndexConfiguration(entry);
5638
5638
  return { stream: archiveStream({ ...checkpoint, config, sourceIndexName: indexName }), lastSeqNo: captured };
5639
5639
  } finally {
5640
5640
  release();
@@ -5660,6 +5660,7 @@ function requireRestoreTarget(indexName, partitionId, partitionCount) {
5660
5660
  async function receiveArchive(indexName, partitionId, source, schema) {
5661
5661
  const store = memoryCheckpointStore();
5662
5662
  let language = "";
5663
+ let analysisRevision = null;
5663
5664
  let received = null;
5664
5665
  await readSnapshotArchive(createByteReader(source), {
5665
5666
  config: async (bytes) => {
@@ -5671,6 +5672,7 @@ async function receiveArchive(indexName, partitionId, source, schema) {
5671
5672
  const raw = decode(payload);
5672
5673
  if (typeof raw.language !== "string") restoreFailed("partition archive names no language", { indexName });
5673
5674
  language = raw.language;
5675
+ analysisRevision = typeof raw.analysisRevision === "string" ? raw.analysisRevision : null;
5674
5676
  },
5675
5677
  manifest: async (manifest, sourceName) => {
5676
5678
  const listed = manifest.partitions.map((partition) => partition.partitionId);
@@ -5697,7 +5699,7 @@ async function receiveArchive(indexName, partitionId, source, schema) {
5697
5699
  file: async (name, _ref, bytes) => store.write(`${indexName}/${name}`, [bytes])
5698
5700
  });
5699
5701
  if (received === null) restoreFailed("partition archive holds no manifest", { indexName, partitionId });
5700
- return { language, manifest: received, store };
5702
+ return { language, analysisRevision, manifest: received, store };
5701
5703
  }
5702
5704
  function emptyPartition(manager, partitionId) {
5703
5705
  return {
@@ -5749,6 +5751,18 @@ async function replacePartition(manager, partitionId, archive) {
5749
5751
  vectorIndex.scheduleBuild();
5750
5752
  }
5751
5753
  }
5754
+ function recordReceivedAnalysis(core, indexName, receivedRevision) {
5755
+ const entry = core.requireIndex(indexName);
5756
+ const current = entry.language.revision;
5757
+ if (receivedRevision === current) return;
5758
+ entry.analysisRevision = entry.analysisRevision === current || entry.analysisRevision === receivedRevision ? receivedRevision : null;
5759
+ core.analysisRebuild.markStale({
5760
+ indexName,
5761
+ language: entry.language.name,
5762
+ storedRevision: entry.analysisRevision,
5763
+ currentRevision: current
5764
+ });
5765
+ }
5752
5766
  async function restoreReplicationPartition(core, engine, indexName, partitionId, source, indexConfig, partitionCount) {
5753
5767
  core.guardShutdown();
5754
5768
  requireRestoreTarget(indexName, partitionId, partitionCount);
@@ -5775,6 +5789,7 @@ async function restoreReplicationPartition(core, engine, indexName, partitionId,
5775
5789
  const wasScaledOut = core.orchestrator.desyncIndex(indexName);
5776
5790
  try {
5777
5791
  await replacePartition(core.requireManager(indexName), partitionId, archive);
5792
+ recordReceivedAnalysis(core, indexName, archive.analysisRevision);
5778
5793
  } finally {
5779
5794
  await core.orchestrator.resyncIndex(indexName, wasScaledOut);
5780
5795
  }