@delali/narsil 0.3.1-next.83 → 0.3.1-next.84

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -7612,65 +7612,6 @@ function mergeSegmentState(target, source) {
7612
7612
  target.scoreBuffer = null;
7613
7613
  }
7614
7614
 
7615
- // src/core/partition/rebuild.ts
7616
- var EMPTY_STATS = {
7617
- totalDocuments: 0,
7618
- totalFieldLengths: {},
7619
- averageFieldLengths: {},
7620
- docFrequencies: {}
7621
- };
7622
- function textFieldsOf(fields) {
7623
- return fields.filter((field) => isWordIndexedBase(field.base));
7624
- }
7625
- function rebuildTextIndex(state, schema, language, options) {
7626
- const textFields = textFieldsOf(schemaFieldsOf(schema));
7627
- state.invertedIdx.clear();
7628
- state.surfaceRegistry.clear();
7629
- state.stats.deserialize(EMPTY_STATS);
7630
- state.invertedIdx.beginBatch();
7631
- try {
7632
- for (const [docId, stored] of state.docStore.all()) {
7633
- const internalId = state.docStore.getInternalId(docId);
7634
- if (internalId === void 0) continue;
7635
- const fields = stored.fields;
7636
- const fieldLengths = {};
7637
- const tokensByField = {};
7638
- for (const { path: fieldPath, base } of textFields) {
7639
- const value = getNestedValue(fields, fieldPath);
7640
- if (value === void 0 || value === null) continue;
7641
- if (base === "string") {
7642
- indexStringField(
7643
- state,
7644
- internalId,
7645
- fieldPath,
7646
- value,
7647
- language,
7648
- options,
7649
- fieldLengths,
7650
- tokensByField
7651
- );
7652
- } else {
7653
- indexStringArrayField(
7654
- state,
7655
- internalId,
7656
- fieldPath,
7657
- value,
7658
- language,
7659
- options,
7660
- fieldLengths,
7661
- tokensByField
7662
- );
7663
- }
7664
- }
7665
- state.docStore.storeRef(docId, stored.fields, fieldLengths);
7666
- state.stats.addDocument(fieldLengths, tokensByField);
7667
- }
7668
- } finally {
7669
- state.invertedIdx.endBatch();
7670
- state.partialWords.forgetSpellings();
7671
- }
7672
- }
7673
-
7674
7615
  // src/core/partition/block-bounds.ts
7675
7616
  var ABSENT_FIELD_LENGTH_BOUND = 0;
7676
7617
  var boundsByList = /* @__PURE__ */ new WeakMap();
@@ -9163,9 +9104,6 @@ function createPartitionIndex(partitionId, trackPositions = true) {
9163
9104
  if (internalId !== void 0) recordSortValues(state, internalId, document2);
9164
9105
  }
9165
9106
  },
9166
- rebuildTextIndex(schema, language, options) {
9167
- rebuildTextIndex(state, schema, language, options);
9168
- },
9169
9107
  get(docId, projection) {
9170
9108
  const stored = state.docStore.get(docId);
9171
9109
  if (!stored) return void 0;
@@ -9259,6 +9197,8 @@ function yieldToEventLoop() {
9259
9197
  // src/partitioning/constants.ts
9260
9198
  var REBALANCE_CHUNK_SIZE = 1e3;
9261
9199
  var DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE = 1e4;
9200
+ var TEXT_REBUILD_CHUNK_DOCUMENTS = 1e3;
9201
+ var TEXT_REBUILD_SWITCH_BACKLOG_DOCUMENTS = 1e3;
9262
9202
 
9263
9203
  // src/partitioning/rebalancer.ts
9264
9204
  function createRebalancer() {
@@ -11162,18 +11102,6 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
11162
11102
  if (frozen.length === 0) return live.encodeSegment();
11163
11103
  return composedScratch().encodeSegment();
11164
11104
  },
11165
- rebuildTextIndex(schema, language, options) {
11166
- invalidateDocFrequencies();
11167
- if (frozen.length > 0) {
11168
- for (const segment of frozen) {
11169
- const payload = encodeSegmentState(segment);
11170
- if (payload.documentCount === 0) continue;
11171
- live.mergeSegmentPayload(payload, survivorDocumentsOf(segment, payload));
11172
- }
11173
- frozen.length = 0;
11174
- }
11175
- live.rebuildTextIndex(schema, language, options);
11176
- },
11177
11105
  serialize(indexName, totalPartitions, language, schema) {
11178
11106
  if (frozen.length === 0) return live.serialize(indexName, totalPartitions, language, schema);
11179
11107
  return composedScratch().serialize(indexName, totalPartitions, language, schema);
@@ -11417,6 +11345,81 @@ function setNestedValue(obj, path, value) {
11417
11345
  current[segments[segments.length - 1]] = value;
11418
11346
  }
11419
11347
 
11348
+ // src/partitioning/manager/stats.ts
11349
+ function aggregateStatsOf(partitions) {
11350
+ let totalDocuments = 0;
11351
+ const docFrequencies = /* @__PURE__ */ Object.create(null);
11352
+ const totalFieldLengths = /* @__PURE__ */ Object.create(null);
11353
+ for (let i = 0; i < partitions.length; i++) {
11354
+ const stats = partitions[i].stats;
11355
+ totalDocuments += stats.totalDocuments;
11356
+ for (const [term, freq] of Object.entries(stats.docFrequencies)) {
11357
+ docFrequencies[term] = (docFrequencies[term] ?? 0) + freq;
11358
+ }
11359
+ for (const [field, length] of Object.entries(stats.totalFieldLengths)) {
11360
+ totalFieldLengths[field] = (totalFieldLengths[field] ?? 0) + length;
11361
+ }
11362
+ }
11363
+ return { totalDocuments, docFrequencies, totalFieldLengths };
11364
+ }
11365
+ function estimateBytesOf(partitions, vecIndexes) {
11366
+ let total = 0;
11367
+ for (let i = 0; i < partitions.length; i++) {
11368
+ total += partitions[i].estimateMemoryBytes();
11369
+ }
11370
+ for (const [, vecIndex] of vecIndexes) {
11371
+ total += vecIndex.estimateMemoryBytes();
11372
+ }
11373
+ return total;
11374
+ }
11375
+ function partitionStatsOf(partitions) {
11376
+ return partitions.map((p, i) => ({
11377
+ partitionId: i,
11378
+ documentCount: p.count(),
11379
+ estimatedMemoryBytes: p.estimateMemoryBytes()
11380
+ }));
11381
+ }
11382
+
11383
+ // src/partitioning/manager/write-log.ts
11384
+ function createWriteLogRecorder() {
11385
+ let current = null;
11386
+ return {
11387
+ open() {
11388
+ const log = { docIds: /* @__PURE__ */ new Set(), replaced: false };
11389
+ current = log;
11390
+ return {
11391
+ get partitionsReplaced() {
11392
+ return log.replaced;
11393
+ },
11394
+ get pendingDocuments() {
11395
+ return log.docIds.size;
11396
+ },
11397
+ takeWrittenDocIds() {
11398
+ const taken = log.docIds;
11399
+ log.docIds = /* @__PURE__ */ new Set();
11400
+ return taken;
11401
+ },
11402
+ close() {
11403
+ if (current === log) current = null;
11404
+ }
11405
+ };
11406
+ },
11407
+ documentWritten(docId) {
11408
+ if (current !== null) current.docIds.add(docId);
11409
+ },
11410
+ segmentWritten(segment) {
11411
+ if (current === null) return;
11412
+ for (const internalId of segment.docStore.allInternalIds()) {
11413
+ const docId = segment.docStore.getExternalId(internalId);
11414
+ if (docId !== void 0) current.docIds.add(docId);
11415
+ }
11416
+ },
11417
+ partitionsReplaced() {
11418
+ if (current !== null) current.replaced = true;
11419
+ }
11420
+ };
11421
+ }
11422
+
11420
11423
  // src/partitioning/manager/index.ts
11421
11424
  function createPartitionManager(indexName, config, language, router, initialPartitionCount, vectorIndexes) {
11422
11425
  requirePatternSearchFor(config.schema);
@@ -11428,6 +11431,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
11428
11431
  const vecIndexes = vectorIndexes ?? /* @__PURE__ */ new Map();
11429
11432
  let partitions = [];
11430
11433
  const docPartitionMap = /* @__PURE__ */ new Map();
11434
+ const writeLog = createWriteLogRecorder();
11431
11435
  for (let i = 0; i < count; i++) {
11432
11436
  partitions.push(createPartitionIndex(i, trackPositions));
11433
11437
  }
@@ -11515,9 +11519,23 @@ function createPartitionManager(indexName, config, language, router, initialPart
11515
11519
  return [...partitions];
11516
11520
  },
11517
11521
  setPartitions(newPartitions) {
11522
+ writeLog.partitionsReplaced();
11518
11523
  partitions = [...newPartitions];
11519
11524
  rebuildDocPartitionMap();
11520
11525
  },
11526
+ openWriteLog() {
11527
+ return writeLog.open();
11528
+ },
11529
+ adoptRebuiltPartitions(rebuilt) {
11530
+ if (rebuilt.length !== partitions.length) {
11531
+ throw new NarsilError(
11532
+ ErrorCodes.PARTITION_CORRUPTED,
11533
+ `Index "${indexName}" holds ${partitions.length} partitions, but the rebuild produced ${rebuilt.length}`,
11534
+ { indexName, partitionCount: partitions.length, rebuiltCount: rebuilt.length }
11535
+ );
11536
+ }
11537
+ partitions = [...rebuilt];
11538
+ },
11521
11539
  addPartition() {
11522
11540
  const currentMaxPartitions = config.partitions?.maxPartitions;
11523
11541
  if (currentMaxPartitions !== void 0 && partitions.length >= currentMaxPartitions) {
@@ -11528,11 +11546,13 @@ function createPartitionManager(indexName, config, language, router, initialPart
11528
11546
  );
11529
11547
  }
11530
11548
  const partition = createPartitionIndex(partitions.length, trackPositions);
11549
+ writeLog.partitionsReplaced();
11531
11550
  partitions.push(partition);
11532
11551
  return partition;
11533
11552
  },
11534
11553
  removePartition(partitionId) {
11535
11554
  validatePartitionId(partitionId);
11555
+ writeLog.partitionsReplaced();
11536
11556
  partitions.splice(partitionId, 1);
11537
11557
  rebuildDocPartitionMap();
11538
11558
  },
@@ -11546,6 +11566,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
11546
11566
  if (newCount >= partitions.length) {
11547
11567
  return;
11548
11568
  }
11569
+ writeLog.partitionsReplaced();
11549
11570
  for (const [docId, pid] of docPartitionMap) {
11550
11571
  if (pid >= newCount) {
11551
11572
  docPartitionMap.delete(docId);
@@ -11568,6 +11589,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
11568
11589
  const insertOpts = resolveInsertOptions(options);
11569
11590
  partitions[pid].insert(docId, document2, config.schema, language, insertOpts);
11570
11591
  docPartitionMap.set(docId, pid);
11592
+ writeLog.documentWritten(docId);
11571
11593
  },
11572
11594
  remove(docId) {
11573
11595
  const pid = locateDocument(docId);
@@ -11576,11 +11598,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
11576
11598
  }
11577
11599
  partitions[pid].remove(docId, config.schema, language, resolveInsertOptions());
11578
11600
  docPartitionMap.delete(docId);
11579
- },
11580
- rebuildTextIndex(partitionId) {
11581
- validatePartitionId(partitionId);
11582
- partitions[partitionId].rebuildTextIndex(config.schema, language, resolveInsertOptions());
11583
- preparePartialWords(partitionId);
11601
+ writeLog.documentWritten(docId);
11584
11602
  },
11585
11603
  beginBatchRemove() {
11586
11604
  for (let i = 0; i < partitions.length; i++) {
@@ -11600,6 +11618,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
11600
11618
  const updateOpts = resolveInsertOptions(options);
11601
11619
  partitions[pid].update(docId, document2, config.schema, language, updateOpts);
11602
11620
  docPartitionMap.set(docId, pid);
11621
+ writeLog.documentWritten(docId);
11603
11622
  },
11604
11623
  get(docId, projection) {
11605
11624
  const pid = locateDocument(docId);
@@ -11650,6 +11669,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
11650
11669
  partition.endBatch();
11651
11670
  for (const docId of payload.docIds) {
11652
11671
  docPartitionMap.set(docId, partitionId);
11672
+ writeLog.documentWritten(docId);
11653
11673
  }
11654
11674
  },
11655
11675
  attachFrozenSegment(partitionId, segment) {
@@ -11664,6 +11684,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
11664
11684
  }
11665
11685
  asCompositePartition(partitionId).attachFrozenSegment(segment);
11666
11686
  registerSegmentDocuments(partitionId, segment);
11687
+ writeLog.segmentWritten(segment);
11667
11688
  },
11668
11689
  swapFrozenSegments(partitionId, dropSegmentIds, replacement) {
11669
11690
  validatePartitionId(partitionId);
@@ -11681,42 +11702,19 @@ function createPartitionManager(indexName, config, language, router, initialPart
11681
11702
  },
11682
11703
  deserializePartition(partitionId, data) {
11683
11704
  validatePartitionId(partitionId);
11705
+ writeLog.partitionsReplaced();
11684
11706
  partitions[partitionId].deserialize(data, config.schema);
11685
11707
  preparePartialWords(partitionId);
11686
11708
  rebuildDocPartitionMapForPartition(partitionId);
11687
11709
  },
11688
11710
  getAggregateStats() {
11689
- let totalDocuments = 0;
11690
- const docFrequencies = /* @__PURE__ */ Object.create(null);
11691
- const totalFieldLengths = /* @__PURE__ */ Object.create(null);
11692
- for (let i = 0; i < partitions.length; i++) {
11693
- const stats = partitions[i].stats;
11694
- totalDocuments += stats.totalDocuments;
11695
- for (const [term, freq] of Object.entries(stats.docFrequencies)) {
11696
- docFrequencies[term] = (docFrequencies[term] ?? 0) + freq;
11697
- }
11698
- for (const [field, length] of Object.entries(stats.totalFieldLengths)) {
11699
- totalFieldLengths[field] = (totalFieldLengths[field] ?? 0) + length;
11700
- }
11701
- }
11702
- return { totalDocuments, docFrequencies, totalFieldLengths };
11711
+ return aggregateStatsOf(partitions);
11703
11712
  },
11704
11713
  estimateMemoryBytes() {
11705
- let total = 0;
11706
- for (let i = 0; i < partitions.length; i++) {
11707
- total += partitions[i].estimateMemoryBytes();
11708
- }
11709
- for (const [, vecIndex] of vecIndexes) {
11710
- total += vecIndex.estimateMemoryBytes();
11711
- }
11712
- return total;
11714
+ return estimateBytesOf(partitions, vecIndexes);
11713
11715
  },
11714
11716
  getPartitionStats() {
11715
- return partitions.map((p, i) => ({
11716
- partitionId: i,
11717
- documentCount: p.count(),
11718
- estimatedMemoryBytes: p.estimateMemoryBytes()
11719
- }));
11717
+ return partitionStatsOf(partitions);
11720
11718
  },
11721
11719
  getVectorIndexes() {
11722
11720
  return vecIndexes;
@@ -21004,6 +21002,74 @@ function createDirectExecutor(options) {
21004
21002
  };
21005
21003
  }
21006
21004
 
21005
+ // src/partitioning/text-rebuild.ts
21006
+ function interrupted(pass) {
21007
+ return pass.log.partitionsReplaced || !pass.hooks.isCurrent();
21008
+ }
21009
+ async function buildPartition(pass, source, partitionId) {
21010
+ const { manager, options } = pass;
21011
+ const target = createPartitionIndex(partitionId, manager.config.trackPositions ?? true);
21012
+ const docIds = [...source.docIds()];
21013
+ target.beginBatch();
21014
+ for (let start = 0; start < docIds.length; start += TEXT_REBUILD_CHUNK_DOCUMENTS) {
21015
+ const end = Math.min(start + TEXT_REBUILD_CHUNK_DOCUMENTS, docIds.length);
21016
+ for (let i = start; i < end; i++) {
21017
+ const document2 = manager.getRef(docIds[i]);
21018
+ if (document2 !== void 0) target.insert(docIds[i], document2, manager.schema, manager.language, options);
21019
+ }
21020
+ await yieldToEventLoop();
21021
+ if (interrupted(pass)) return null;
21022
+ }
21023
+ target.endBatch();
21024
+ if (manager.partialWordAnalysis !== null) target.preparePartialWords(manager.partialWordAnalysis);
21025
+ return target;
21026
+ }
21027
+ function applyWrittenDocuments(pass, built) {
21028
+ const { manager, options } = pass;
21029
+ for (const docId of pass.log.takeWrittenDocIds()) {
21030
+ for (const partition of built) {
21031
+ if (!partition.has(docId)) continue;
21032
+ partition.remove(docId, manager.schema, manager.language, options);
21033
+ break;
21034
+ }
21035
+ const partitionId = manager.partitionIdOf(docId);
21036
+ const document2 = manager.getRef(docId);
21037
+ if (partitionId === void 0 || document2 === void 0) continue;
21038
+ built[partitionId].insert(docId, document2, manager.schema, manager.language, options);
21039
+ }
21040
+ }
21041
+ async function rebuildOnce(pass) {
21042
+ const partitionCount = pass.manager.partitionCount;
21043
+ const built = [];
21044
+ for (let partitionId = 0; partitionId < partitionCount; partitionId++) {
21045
+ const partition = await buildPartition(pass, pass.manager.getPartition(partitionId), partitionId);
21046
+ if (partition === null) return false;
21047
+ built.push(partition);
21048
+ pass.hooks.onPartitionBuilt(built.length);
21049
+ }
21050
+ while (pass.log.pendingDocuments > TEXT_REBUILD_SWITCH_BACKLOG_DOCUMENTS) {
21051
+ applyWrittenDocuments(pass, built);
21052
+ await yieldToEventLoop();
21053
+ if (interrupted(pass)) return false;
21054
+ }
21055
+ applyWrittenDocuments(pass, built);
21056
+ pass.manager.adoptRebuiltPartitions(built);
21057
+ pass.hooks.onSwitch();
21058
+ return true;
21059
+ }
21060
+ async function rebuildTextIndexes(manager, hooks) {
21061
+ const options = resolvePartitionInsertOptions(manager.config, manager.analysis, { validate: false, skipClone: true });
21062
+ while (hooks.isCurrent()) {
21063
+ const log = manager.openWriteLog();
21064
+ try {
21065
+ if (await rebuildOnce({ manager, log, hooks, options })) return true;
21066
+ } finally {
21067
+ log.close();
21068
+ }
21069
+ }
21070
+ return false;
21071
+ }
21072
+
21007
21073
  // src/engine/events.ts
21008
21074
  function eventHandlersOf(events) {
21009
21075
  const handlers = /* @__PURE__ */ new Map();
@@ -21026,11 +21092,6 @@ function emitEngineEvent(eventHandlers, event, payload) {
21026
21092
  }
21027
21093
 
21028
21094
  // src/engine/analysis-rebuild.ts
21029
- function yieldToEventLoop4() {
21030
- return new Promise((resolve) => {
21031
- setTimeout(resolve, 0);
21032
- });
21033
- }
21034
21095
  function toError(value) {
21035
21096
  return value instanceof Error ? value : new Error(String(value));
21036
21097
  }
@@ -21041,9 +21102,13 @@ function wireAnalysisRebuild(wiring) {
21041
21102
  resyncIndex: wiring.resyncIndex,
21042
21103
  async persistAnalysisRevision(indexName) {
21043
21104
  const durability = wiring.durabilityManager;
21044
- if (durability === null) return;
21045
- await durability.persistMetadata(indexName);
21105
+ if (durability === null) {
21106
+ wiring.recordCurrentAnalysis(indexName);
21107
+ return;
21108
+ }
21046
21109
  await (durability.checkpointFromMemory?.(indexName) ?? durability.checkpoint(indexName));
21110
+ wiring.recordCurrentAnalysis(indexName);
21111
+ await durability.persistMetadata(indexName);
21047
21112
  },
21048
21113
  emit(payload) {
21049
21114
  emitEngineEvent(wiring.eventHandlers, "analysisRebuild", payload);
@@ -21054,9 +21119,10 @@ function createAnalysisRebuildCoordinator(config, deps) {
21054
21119
  const stale = /* @__PURE__ */ new Map();
21055
21120
  const rebuildMode = config?.rebuild ?? "auto";
21056
21121
  let queue = Promise.resolve();
21122
+ let active = null;
21057
21123
  async function rebuildNow(indexName, entry) {
21058
21124
  const manager = deps.getManager(indexName);
21059
- if (manager === void 0) {
21125
+ if (manager === void 0 || entry.abandoned) {
21060
21126
  entry.run = null;
21061
21127
  return;
21062
21128
  }
@@ -21064,12 +21130,20 @@ function createAnalysisRebuildCoordinator(config, deps) {
21064
21130
  deps.emit({ indexName, status: "started", partitionsRebuilt: 0, partitionCount });
21065
21131
  const wasPromoted = deps.desyncIndex(indexName);
21066
21132
  try {
21067
- for (let partitionId = 0; partitionId < manager.partitionCount; partitionId += 1) {
21068
- manager.rebuildTextIndex(partitionId);
21069
- entry.partitionsRebuilt = partitionId + 1;
21070
- await yieldToEventLoop4();
21133
+ const rebuilt = await rebuildTextIndexes(manager, {
21134
+ isCurrent: () => deps.getManager(indexName) === manager && stale.get(indexName) === entry && !entry.abandoned,
21135
+ onPartitionBuilt(partitionsBuilt) {
21136
+ entry.partitionsRebuilt = partitionsBuilt;
21137
+ },
21138
+ onSwitch() {
21139
+ stale.delete(indexName);
21140
+ }
21141
+ });
21142
+ if (!rebuilt) {
21143
+ entry.run = null;
21144
+ if (deps.getManager(indexName) === manager) await deps.resyncIndex(indexName, wasPromoted);
21145
+ return;
21071
21146
  }
21072
- stale.delete(indexName);
21073
21147
  await deps.persistAnalysisRevision(indexName);
21074
21148
  await deps.resyncIndex(indexName, wasPromoted);
21075
21149
  deps.emit({
@@ -21080,7 +21154,6 @@ function createAnalysisRebuildCoordinator(config, deps) {
21080
21154
  });
21081
21155
  } catch (err) {
21082
21156
  const error = toError(err);
21083
- stale.set(indexName, entry);
21084
21157
  entry.run = null;
21085
21158
  await deps.resyncIndex(indexName, wasPromoted).catch(() => void 0);
21086
21159
  deps.emit({
@@ -21093,11 +21166,20 @@ function createAnalysisRebuildCoordinator(config, deps) {
21093
21166
  throw error;
21094
21167
  }
21095
21168
  }
21169
+ function runTracked(indexName, entry) {
21170
+ const done = rebuildNow(indexName, entry);
21171
+ const tracked = { indexName, entry, done };
21172
+ active = tracked;
21173
+ return done.finally(() => {
21174
+ if (active === tracked) active = null;
21175
+ });
21176
+ }
21096
21177
  function enqueue(indexName, entry) {
21097
21178
  if (entry.run !== null) {
21098
21179
  return entry.run;
21099
21180
  }
21100
- const run = queue.then(() => rebuildNow(indexName, entry));
21181
+ entry.abandoned = false;
21182
+ const run = queue.then(() => runTracked(indexName, entry));
21101
21183
  entry.run = run;
21102
21184
  queue = run.then(
21103
21185
  () => void 0,
@@ -21132,7 +21214,7 @@ function createAnalysisRebuildCoordinator(config, deps) {
21132
21214
  if (existing !== void 0 && existing.run !== null) {
21133
21215
  return;
21134
21216
  }
21135
- stale.set(index.indexName, { index, partitionsRebuilt: 0, run: null });
21217
+ stale.set(index.indexName, { index, partitionsRebuilt: 0, run: null, abandoned: false });
21136
21218
  },
21137
21219
  clearStale(indexName) {
21138
21220
  stale.delete(indexName);
@@ -21143,6 +21225,14 @@ function createAnalysisRebuildCoordinator(config, deps) {
21143
21225
  isRunning(indexName) {
21144
21226
  return stale.get(indexName)?.run !== null && stale.get(indexName)?.run !== void 0;
21145
21227
  },
21228
+ async abandon(indexName) {
21229
+ const entry = stale.get(indexName);
21230
+ if (entry !== void 0 && entry.run !== null) entry.abandoned = true;
21231
+ const running = active;
21232
+ if (running === null || running.indexName !== indexName) return;
21233
+ running.entry.abandoned = true;
21234
+ await running.done.catch(() => void 0);
21235
+ },
21146
21236
  review(indexName) {
21147
21237
  const entry = stale.get(indexName);
21148
21238
  return entry === void 0 ? Promise.resolve() : review(indexName, entry);
@@ -21809,7 +21899,7 @@ function buildEntry(fields) {
21809
21899
  }
21810
21900
 
21811
21901
  // src/serialization/chunk-checksum.ts
21812
- function yieldToEventLoop5() {
21902
+ function yieldToEventLoop4() {
21813
21903
  if (typeof setImmediate === "function") {
21814
21904
  return new Promise((resolve) => setImmediate(resolve));
21815
21905
  }
@@ -21821,7 +21911,7 @@ async function advanceChecksum(start, payload) {
21821
21911
  const end = Math.min(offset + CHECKSUM_YIELD_CHUNK_BYTES, payload.length);
21822
21912
  state = crc32Update(state, payload.subarray(offset, end));
21823
21913
  if (end < payload.length) {
21824
- await yieldToEventLoop5();
21914
+ await yieldToEventLoop4();
21825
21915
  }
21826
21916
  }
21827
21917
  return state;
@@ -25830,7 +25920,7 @@ function removeWhatTheSegmentReplaces(manager, segment, contents) {
25830
25920
  manager.endBatchRemove();
25831
25921
  }
25832
25922
  }
25833
- function yieldToEventLoop6() {
25923
+ function yieldToEventLoop5() {
25834
25924
  return new Promise((resolve) => setTimeout(resolve, 0));
25835
25925
  }
25836
25926
  async function loadPartitionSegments(store, partition, manager, options) {
@@ -25841,7 +25931,7 @@ async function loadPartitionSegments(store, partition, manager, options) {
25841
25931
  const segment = createSharedFrozenSegment(shareSegmentArrays(contents.arrays, manager.partialWordAnalysis));
25842
25932
  removeWhatTheSegmentReplaces(manager, segment, contents);
25843
25933
  if (contents.arrays.documentCount > 0) manager.attachFrozenSegment(partition.partitionId, segment);
25844
- await yieldToEventLoop6();
25934
+ await yieldToEventLoop5();
25845
25935
  }
25846
25936
  }
25847
25937
 
@@ -28269,7 +28359,7 @@ function metadataConfigOf(entry) {
28269
28359
  b: entry.config.bm25?.b ?? 0.75,
28270
28360
  ...embedding !== void 0 ? { embedding } : {},
28271
28361
  surfaceForms: entry.config.surfaceForms !== false,
28272
- analysisRevision: entry.language.revision,
28362
+ ...entry.analysisRevision !== null ? { analysisRevision: entry.analysisRevision } : {},
28273
28363
  ...typeof entry.config.tokenizer === "string" ? { tokenizer: entry.config.tokenizer } : {},
28274
28364
  ...typeof entry.config.stopWords === "string" ? { stopWords: entry.config.stopWords } : {},
28275
28365
  ...entry.config.stopWords instanceof Set ? { stopWordList: [...entry.config.stopWords].sort(compareCodePoints) } : {},
@@ -28510,7 +28600,8 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
28510
28600
  indexUuid: metadata.indexUuid ?? null,
28511
28601
  heldPartitions: metadata.heldPartitions ?? null,
28512
28602
  documentCount: metadata.documentCount ?? 0,
28513
- partitionCount: metadata.partitionCount
28603
+ partitionCount: metadata.partitionCount,
28604
+ analysisRevision: metadata.analysisRevision ?? null
28514
28605
  });
28515
28606
  if (loadData) await deps.indexState().registerOpen(metadata.indexName);
28516
28607
  else deps.indexState().registerClosed(metadata.indexName);
@@ -28633,8 +28724,14 @@ function createIndexStateCoordinator(config, callbacks) {
28633
28724
  return entry;
28634
28725
  }
28635
28726
  }
28727
+ function transitionPending(indexName, entry) {
28728
+ if (entries.get(indexName) !== entry) return true;
28729
+ if (entry.state === "dropping" && entry.dropPromise !== null) return true;
28730
+ return entry.state === "closing" && entry.closePromise !== null;
28731
+ }
28636
28732
  async function reopen(indexName, explicit) {
28637
- const entry = await awaitPendingTransitions(indexName);
28733
+ let entry = await awaitPendingTransitions(indexName);
28734
+ while (transitionPending(indexName, entry)) entry = await awaitPendingTransitions(indexName);
28638
28735
  if (entry.state === "open") return;
28639
28736
  if (explicit) {
28640
28737
  entry.reopenFailures = 0;
@@ -31053,7 +31150,13 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
31053
31150
  idleSweep: null
31054
31151
  };
31055
31152
  startIdleSweep(state);
31153
+ function settlingCopyWork(indexName) {
31154
+ return state.copyTransitions.get(indexName)?.done ?? state.compactionsInFlight.get(indexName);
31155
+ }
31056
31156
  async function resyncIndex(indexName, wasScaledOut) {
31157
+ for (let pending = settlingCopyWork(indexName); pending !== void 0; pending = settlingCopyWork(indexName)) {
31158
+ await pending;
31159
+ }
31057
31160
  state.desyncedIndexes.delete(indexName);
31058
31161
  if (!wasScaledOut) return;
31059
31162
  await scaleOutIndex(state, indexName, copyThresholdReason(state, indexName));
@@ -31405,6 +31508,10 @@ function createEngineCore(config, hooks, capacityScope = "index") {
31405
31508
  getManager: (indexName) => executor.getManager(indexName),
31406
31509
  desyncIndex: (indexName) => orchestrator.desyncIndex(indexName),
31407
31510
  resyncIndex: (indexName, wasPromoted) => orchestrator.resyncIndex(indexName, wasPromoted),
31511
+ recordCurrentAnalysis: (indexName) => {
31512
+ const entry = indexRegistry.get(indexName);
31513
+ if (entry !== void 0 && !analysisRebuild.isStale(indexName)) entry.analysisRevision = entry.language.revision;
31514
+ },
31408
31515
  durabilityManager: durability?.manager ?? null
31409
31516
  });
31410
31517
  const indexState = wireIndexState({
@@ -31578,11 +31685,16 @@ function refuseCode(entry) {
31578
31685
  }
31579
31686
  function indexConfigurationOf(entry) {
31580
31687
  refuseCode(entry);
31581
- return configurationPayloadOf(entry.config, entry.language, entry.embeddingAdapterName);
31688
+ return configurationPayloadOf(entry.config, entry.language, entry.embeddingAdapterName, entry.analysisRevision);
31582
31689
  }
31583
31690
  function requireConfigurationWithinSizeLimit(config, language) {
31584
31691
  const adapter = config.embedding?.adapter;
31585
- const payload = configurationPayloadOf(config, language, typeof adapter === "string" ? adapter : null);
31692
+ const payload = configurationPayloadOf(
31693
+ config,
31694
+ language,
31695
+ typeof adapter === "string" ? adapter : null,
31696
+ language.revision
31697
+ );
31586
31698
  if (messagePackUpperBoundBytes(payload) <= MAX_INDEX_CONFIGURATION_BYTES) return;
31587
31699
  const bytes = encode(payload).byteLength;
31588
31700
  if (bytes <= MAX_INDEX_CONFIGURATION_BYTES) return;
@@ -31592,12 +31704,12 @@ function requireConfigurationWithinSizeLimit(config, language) {
31592
31704
  { bytes, maximum: MAX_INDEX_CONFIGURATION_BYTES }
31593
31705
  );
31594
31706
  }
31595
- function configurationPayloadOf(config, language, adapterName) {
31707
+ function configurationPayloadOf(config, language, adapterName, analysisRevision) {
31596
31708
  const bm25 = config.bm25;
31597
31709
  return {
31598
31710
  schema: config.schema,
31599
31711
  language: language.name,
31600
- analysisRevision: language.revision,
31712
+ ...analysisRevision !== null ? { analysisRevision } : {},
31601
31713
  ...typeof config.tokenizer === "string" ? { tokenizer: config.tokenizer } : {},
31602
31714
  ...typeof config.stopWords === "string" ? { stopWords: config.stopWords } : {},
31603
31715
  ...config.stopWords instanceof Set ? { stopWordList: [...config.stopWords].sort(compareCodePoints) } : {},
@@ -31756,7 +31868,8 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
31756
31868
  indexUuid: indexUuid ?? null,
31757
31869
  heldPartitions: null,
31758
31870
  documentCount: 0,
31759
- partitionCount: indexConfig.partitions?.maxPartitions ?? 1
31871
+ partitionCount: indexConfig.partitions?.maxPartitions ?? 1,
31872
+ analysisRevision: language.revision
31760
31873
  });
31761
31874
  await core.indexState.registerOpen(name);
31762
31875
  if (core.durability) {
@@ -31768,6 +31881,7 @@ async function dropEngineIndex(core, name) {
31768
31881
  core.guardShutdown();
31769
31882
  const entry = core.requireIndex(name);
31770
31883
  await core.indexState.drop(name, async () => {
31884
+ await core.analysisRebuild.abandon(name);
31771
31885
  await core.orchestrator.closeIndex(name);
31772
31886
  if (core.durability) {
31773
31887
  try {
@@ -35068,7 +35182,8 @@ async function createRestoredIndex(indexName, prepared, partitionCount, deps) {
35068
35182
  indexUuid: null,
35069
35183
  heldPartitions: null,
35070
35184
  documentCount: 0,
35071
- partitionCount
35185
+ partitionCount,
35186
+ analysisRevision: prepared.storedRevision
35072
35187
  });
35073
35188
  await deps.indexState.registerOpen(indexName);
35074
35189
  const release = await deps.indexState.acquire(indexName, false);
@@ -36965,6 +37080,15 @@ async function enterRead(core, indexName) {
36965
37080
  core.guardShutdown();
36966
37081
  if (core.rebalancingIndexes.has(indexName)) await yieldToEventLoop();
36967
37082
  }
37083
+ async function readUnderOneAnalysis(core, indexName, read) {
37084
+ let analysisStale = core.analysisRebuild.isStale(indexName);
37085
+ let result = await read();
37086
+ while (core.analysisRebuild.isStale(indexName) !== analysisStale) {
37087
+ analysisStale = core.analysisRebuild.isStale(indexName);
37088
+ result = await read();
37089
+ }
37090
+ return { result, analysisStale };
37091
+ }
36968
37092
  function scopedParams(params, options) {
36969
37093
  {
36970
37094
  return params;
@@ -36992,7 +37116,7 @@ async function runEngineQuery(core, indexName, params, options) {
36992
37116
  entry.embeddingAdapterName
36993
37117
  );
36994
37118
  const workerSearch = core.orchestrator.hasWorkerPool() ? core.orchestrator.searchViaWorker.bind(core.orchestrator) : void 0;
36995
- const result = await executeQuery(resolvedParams, {
37119
+ const context = {
36996
37120
  manager,
36997
37121
  language: entry.language,
36998
37122
  config: entry.config,
@@ -37002,7 +37126,15 @@ async function runEngineQuery(core, indexName, params, options) {
37002
37126
  partitionIds: options?.partitionIds,
37003
37127
  cursorBinding: queryBindingOf(params),
37004
37128
  patternWorkCap: core.patternWorkCap
37005
- });
37129
+ };
37130
+ const { result, analysisStale } = await readUnderOneAnalysis(
37131
+ core,
37132
+ indexName,
37133
+ () => executeQuery(resolvedParams, context)
37134
+ );
37135
+ if (analysisStale) {
37136
+ result.analysisStale = true;
37137
+ }
37006
37138
  if (core.pluginRegistry.hasHooks("afterSearch")) {
37007
37139
  const observed = { ...result, hits: [...result.hits] };
37008
37140
  try {
@@ -37011,9 +37143,6 @@ async function runEngineQuery(core, indexName, params, options) {
37011
37143
  console.warn("afterSearch plugin hook error:", err);
37012
37144
  }
37013
37145
  }
37014
- if (core.analysisRebuild.isStale(indexName)) {
37015
- result.analysisStale = true;
37016
- }
37017
37146
  return result;
37018
37147
  } finally {
37019
37148
  release();
@@ -37034,7 +37163,7 @@ async function runEnginePreflight(core, indexName, params, options) {
37034
37163
  entry.embeddingAdapterName
37035
37164
  );
37036
37165
  const workerSearch = core.orchestrator.hasWorkerPool() ? core.orchestrator.searchViaWorker.bind(core.orchestrator) : void 0;
37037
- const result = await executePreflight(resolvedParams, {
37166
+ const context = {
37038
37167
  manager,
37039
37168
  language: entry.language,
37040
37169
  config: entry.config,
@@ -37044,8 +37173,13 @@ async function runEnginePreflight(core, indexName, params, options) {
37044
37173
  partitionIds: options?.partitionIds,
37045
37174
  cursorBinding: queryBindingOf(params),
37046
37175
  patternWorkCap: core.patternWorkCap
37047
- });
37048
- if (core.analysisRebuild.isStale(indexName)) {
37176
+ };
37177
+ const { result, analysisStale } = await readUnderOneAnalysis(
37178
+ core,
37179
+ indexName,
37180
+ () => executePreflight(resolvedParams, context)
37181
+ );
37182
+ if (analysisStale) {
37049
37183
  result.analysisStale = true;
37050
37184
  }
37051
37185
  return result;