@delali/narsil 0.3.1-next.83 → 0.3.1-next.85

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,19 +1,19 @@
1
- import { validateWorkerConfig, validateIndexName, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateBM25Params, validatePartitionConfig, executeQuery, queryBindingOf, executePreflight, executeSuggest, executeListDocuments, validateDocId } from './chunk-4D6OH4EF.mjs';
1
+ import { validateWorkerConfig, validateIndexName, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateBM25Params, validatePartitionConfig, queryBindingOf, executeQuery, executePreflight, executeSuggest, executeListDocuments, validateDocId } from './chunk-4D6OH4EF.mjs';
2
2
  import { ReplicationMessageTypes } from './chunk-GCM3JFVA.mjs';
3
3
  import { WIRE_BATCH_BUDGET, resolveVectorText, bindEngineCore, restoredConfigFields, restoredEmbedding, collectRequiredFieldFailures, embedChunkDocuments, providedDocId, requireDocumentObject, admitInsert, asBatchInsertError, requireMatchingDocId, embedDocumentFields, assertDocumentCarriesMappedVectors, admitBufferedUpsert, createPendingAdmissions, recordPendingAdmission } from './chunk-2453RJYK.mjs';
4
- import { NATIVE_CRC32_MIN_BYTES, generateId, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, splitWorkerBudget, resolveWorkerCount, createDirectExecutor, createPartitionRouter, MAX_FILE_BYTES, resolveWorkerEntry, missingWorkerEntry, readHeapStatistics, nodeThreadId, CHECKSUM_YIELD_CHUNK_BYTES, resolvePartitionInsertOptions, resolveIndexAnalysis, createPartitionManager, buildSegmentPayload, createFrozenSegment, resolveRequestThreadCount, vectorsPerFile, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, createPartitionIndex, selectionKeeps, decodeMapWithoutFields, isCompositePartition, segmentArraysOf, mergeSegmentContents, survivorDocumentsOf, partitionStateOf, VECTOR_FILE_MAX_VECTORS, deadBytesFor, mergeFrozenSegments, workerResourceLimits, decodeVectorIndexParts, bytesToVectors, freezeSegmentShared, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, SHUTDOWN_TIMEOUT_MS, decodeVectorFilePayload, decodeVectorGraphPartPayload, assembleGraphParts, countDead, encodeSegmentState, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, VALID_HNSW_METRICS, spawnNodeWorker, decodeVectorGraphPayload, graphPartsOfEarlierPayload, decodeStoredDocument, isDead } from './chunk-CUP2E4OH.mjs';
4
+ import { NATIVE_CRC32_MIN_BYTES, generateId, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, splitWorkerBudget, resolveWorkerCount, createDirectExecutor, createPartitionRouter, MAX_FILE_BYTES, resolveWorkerEntry, missingWorkerEntry, readHeapStatistics, nodeThreadId, CHECKSUM_YIELD_CHUNK_BYTES, resolvePartitionInsertOptions, resolveIndexAnalysis, createPartitionManager, buildSegmentPayload, createFrozenSegment, resolveRequestThreadCount, vectorsPerFile, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, createPartitionIndex, selectionKeeps, decodeMapWithoutFields, isCompositePartition, segmentArraysOf, mergeSegmentContents, survivorDocumentsOf, partitionStateOf, VECTOR_FILE_MAX_VECTORS, deadBytesFor, mergeFrozenSegments, workerResourceLimits, decodeVectorIndexParts, bytesToVectors, freezeSegmentShared, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, SHUTDOWN_TIMEOUT_MS, decodeVectorFilePayload, decodeVectorGraphPartPayload, assembleGraphParts, countDead, encodeSegmentState, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, VALID_HNSW_METRICS, spawnNodeWorker, decodeVectorGraphPayload, graphPartsOfEarlierPayload, decodeStoredDocument, isDead } from './chunk-RCSXJCI5.mjs';
5
5
  import { requirePatternSearchFor, requirePartialWordSearchFor, collectQueryTermStats, mergePartitionStats, oversampledFacetConfig, fnv1a, availablePatternSearch, PATTERN_HOST, kWayMerge, mergeFacets, setNestedValue } from './chunk-XYDP7CCJ.mjs';
6
6
  import { detectRuntime, getLanguage } from './chunk-OLXM65YH.mjs';
7
7
  import { MAX_INDEX_CONFIGURATION_BYTES, STATISTICS_BROADCAST_INTERVAL_MS, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, HEAP_PRESSURE_WARNING_FRACTION, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, FOREIGN_STATS_TTL_MS, MAX_IMPORT_REFUSALS_REPORTED, BATCH_CHUNK_SIZE, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION, EXPORT_CHUNK_BYTES } from './chunk-ZCKPRCYI.mjs';
8
- import { releaseLocksHeldBy } from './chunk-RZXNWQSM.mjs';
9
- import { requireSearchBackendSettings } from './chunk-B3UOQJD3.mjs';
8
+ import { releaseLocksHeldBy } from './chunk-NIWDQTZB.mjs';
9
+ import { requireSearchBackendSettings } from './chunk-NFF5TDRG.mjs';
10
10
  import { DEFAULT_PATTERN_WORK_CAP, MATCHER_STATE_BUDGET_BYTES } from './chunk-SREPGC6O.mjs';
11
11
  import { getNestedValue, parseVectorDimension } from './chunk-4VBG6PEH.mjs';
12
12
  import { extractVectorFieldsFromSchema, messagePackUpperBoundBytes, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, raiseEarlierReleaseGraphSetting, partialFieldsOf, flattenSchema, requireValidFieldTypes, encodeDocument, schemaFieldsOf, isQuantizationMode, isStorageMode, validateRequiredFields, requireDocumentWithinSizeLimit, validateDocument, validateDocumentStrict } from './chunk-KJ7LB3TR.mjs';
13
13
  import { MIN_M } from './chunk-KEHK2O6P.mjs';
14
14
  import { isSingleStringFieldType, VECTOR_PATTERN, MAX_DOCUMENT_NESTING_DEPTH } from './chunk-LS5IRM23.mjs';
15
15
  import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
16
- import { createRequestId, isValidWorkerResponse, threadSlotOfWorker } from './chunk-POWOTUKJ.mjs';
16
+ import { createRequestId, isValidWorkerResponse, threadSlotOfWorker } from './chunk-YRX6XMOQ.mjs';
17
17
  import { ErrorCodes, NarsilError } from './chunk-TPJU7NDJ.mjs';
18
18
  import { openSync, close, readFileSync } from 'fs';
19
19
  import { rename, readFile, writeFile, unlink, realpath, readdir, readlink } from 'fs/promises';
@@ -160,6 +160,8 @@ function yieldToEventLoop() {
160
160
  // src/partitioning/constants.ts
161
161
  var REBALANCE_CHUNK_SIZE = 1e3;
162
162
  var DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE = 1e4;
163
+ var TEXT_REBUILD_CHUNK_DOCUMENTS = 1e3;
164
+ var TEXT_REBUILD_SWITCH_BACKLOG_DOCUMENTS = 1e3;
163
165
 
164
166
  // src/partitioning/rebalancer.ts
165
167
  function createRebalancer() {
@@ -452,6 +454,74 @@ function validateRequiredFieldsInSchema(required, schema) {
452
454
  }
453
455
  }
454
456
 
457
+ // src/partitioning/text-rebuild.ts
458
+ function interrupted(pass) {
459
+ return pass.log.partitionsReplaced || !pass.hooks.isCurrent();
460
+ }
461
+ async function buildPartition(pass, source, partitionId) {
462
+ const { manager, options } = pass;
463
+ const target = createPartitionIndex(partitionId, manager.config.trackPositions ?? true);
464
+ const docIds = [...source.docIds()];
465
+ target.beginBatch();
466
+ for (let start = 0; start < docIds.length; start += TEXT_REBUILD_CHUNK_DOCUMENTS) {
467
+ const end = Math.min(start + TEXT_REBUILD_CHUNK_DOCUMENTS, docIds.length);
468
+ for (let i = start; i < end; i++) {
469
+ const document = manager.getRef(docIds[i]);
470
+ if (document !== void 0) target.insert(docIds[i], document, manager.schema, manager.language, options);
471
+ }
472
+ await yieldToEventLoop();
473
+ if (interrupted(pass)) return null;
474
+ }
475
+ target.endBatch();
476
+ if (manager.partialWordAnalysis !== null) target.preparePartialWords(manager.partialWordAnalysis);
477
+ return target;
478
+ }
479
+ function applyWrittenDocuments(pass, built) {
480
+ const { manager, options } = pass;
481
+ for (const docId of pass.log.takeWrittenDocIds()) {
482
+ for (const partition of built) {
483
+ if (!partition.has(docId)) continue;
484
+ partition.remove(docId, manager.schema, manager.language, options);
485
+ break;
486
+ }
487
+ const partitionId = manager.partitionIdOf(docId);
488
+ const document = manager.getRef(docId);
489
+ if (partitionId === void 0 || document === void 0) continue;
490
+ built[partitionId].insert(docId, document, manager.schema, manager.language, options);
491
+ }
492
+ }
493
+ async function rebuildOnce(pass) {
494
+ const partitionCount = pass.manager.partitionCount;
495
+ const built = [];
496
+ for (let partitionId = 0; partitionId < partitionCount; partitionId++) {
497
+ const partition = await buildPartition(pass, pass.manager.getPartition(partitionId), partitionId);
498
+ if (partition === null) return false;
499
+ built.push(partition);
500
+ pass.hooks.onPartitionBuilt(built.length);
501
+ }
502
+ while (pass.log.pendingDocuments > TEXT_REBUILD_SWITCH_BACKLOG_DOCUMENTS) {
503
+ applyWrittenDocuments(pass, built);
504
+ await yieldToEventLoop();
505
+ if (interrupted(pass)) return false;
506
+ }
507
+ applyWrittenDocuments(pass, built);
508
+ pass.manager.adoptRebuiltPartitions(built);
509
+ pass.hooks.onSwitch();
510
+ return true;
511
+ }
512
+ async function rebuildTextIndexes(manager, hooks) {
513
+ const options = resolvePartitionInsertOptions(manager.config, manager.analysis, { validate: false, skipClone: true });
514
+ while (hooks.isCurrent()) {
515
+ const log = manager.openWriteLog();
516
+ try {
517
+ if (await rebuildOnce({ manager, log, hooks, options })) return true;
518
+ } finally {
519
+ log.close();
520
+ }
521
+ }
522
+ return false;
523
+ }
524
+
455
525
  // src/engine/events.ts
456
526
  function eventHandlersOf(events) {
457
527
  const handlers = /* @__PURE__ */ new Map();
@@ -474,11 +544,6 @@ function emitEngineEvent(eventHandlers, event, payload) {
474
544
  }
475
545
 
476
546
  // src/engine/analysis-rebuild.ts
477
- function yieldToEventLoop2() {
478
- return new Promise((resolve) => {
479
- setTimeout(resolve, 0);
480
- });
481
- }
482
547
  function toError(value) {
483
548
  return value instanceof Error ? value : new Error(String(value));
484
549
  }
@@ -489,9 +554,13 @@ function wireAnalysisRebuild(wiring) {
489
554
  resyncIndex: wiring.resyncIndex,
490
555
  async persistAnalysisRevision(indexName) {
491
556
  const durability = wiring.durabilityManager;
492
- if (durability === null) return;
493
- await durability.persistMetadata(indexName);
557
+ if (durability === null) {
558
+ wiring.recordCurrentAnalysis(indexName);
559
+ return;
560
+ }
494
561
  await (durability.checkpointFromMemory?.(indexName) ?? durability.checkpoint(indexName));
562
+ wiring.recordCurrentAnalysis(indexName);
563
+ await durability.persistMetadata(indexName);
495
564
  },
496
565
  emit(payload) {
497
566
  emitEngineEvent(wiring.eventHandlers, "analysisRebuild", payload);
@@ -502,9 +571,10 @@ function createAnalysisRebuildCoordinator(config, deps) {
502
571
  const stale = /* @__PURE__ */ new Map();
503
572
  const rebuildMode = config?.rebuild ?? "auto";
504
573
  let queue = Promise.resolve();
574
+ let active = null;
505
575
  async function rebuildNow(indexName, entry) {
506
576
  const manager = deps.getManager(indexName);
507
- if (manager === void 0) {
577
+ if (manager === void 0 || entry.abandoned) {
508
578
  entry.run = null;
509
579
  return;
510
580
  }
@@ -512,12 +582,20 @@ function createAnalysisRebuildCoordinator(config, deps) {
512
582
  deps.emit({ indexName, status: "started", partitionsRebuilt: 0, partitionCount });
513
583
  const wasPromoted = deps.desyncIndex(indexName);
514
584
  try {
515
- for (let partitionId = 0; partitionId < manager.partitionCount; partitionId += 1) {
516
- manager.rebuildTextIndex(partitionId);
517
- entry.partitionsRebuilt = partitionId + 1;
518
- await yieldToEventLoop2();
585
+ const rebuilt = await rebuildTextIndexes(manager, {
586
+ isCurrent: () => deps.getManager(indexName) === manager && stale.get(indexName) === entry && !entry.abandoned,
587
+ onPartitionBuilt(partitionsBuilt) {
588
+ entry.partitionsRebuilt = partitionsBuilt;
589
+ },
590
+ onSwitch() {
591
+ stale.delete(indexName);
592
+ }
593
+ });
594
+ if (!rebuilt) {
595
+ entry.run = null;
596
+ if (deps.getManager(indexName) === manager) await deps.resyncIndex(indexName, wasPromoted);
597
+ return;
519
598
  }
520
- stale.delete(indexName);
521
599
  await deps.persistAnalysisRevision(indexName);
522
600
  await deps.resyncIndex(indexName, wasPromoted);
523
601
  deps.emit({
@@ -528,7 +606,6 @@ function createAnalysisRebuildCoordinator(config, deps) {
528
606
  });
529
607
  } catch (err) {
530
608
  const error = toError(err);
531
- stale.set(indexName, entry);
532
609
  entry.run = null;
533
610
  await deps.resyncIndex(indexName, wasPromoted).catch(() => void 0);
534
611
  deps.emit({
@@ -541,11 +618,20 @@ function createAnalysisRebuildCoordinator(config, deps) {
541
618
  throw error;
542
619
  }
543
620
  }
621
+ function runTracked(indexName, entry) {
622
+ const done = rebuildNow(indexName, entry);
623
+ const tracked = { indexName, entry, done };
624
+ active = tracked;
625
+ return done.finally(() => {
626
+ if (active === tracked) active = null;
627
+ });
628
+ }
544
629
  function enqueue(indexName, entry) {
545
630
  if (entry.run !== null) {
546
631
  return entry.run;
547
632
  }
548
- const run = queue.then(() => rebuildNow(indexName, entry));
633
+ entry.abandoned = false;
634
+ const run = queue.then(() => runTracked(indexName, entry));
549
635
  entry.run = run;
550
636
  queue = run.then(
551
637
  () => void 0,
@@ -580,7 +666,7 @@ function createAnalysisRebuildCoordinator(config, deps) {
580
666
  if (existing !== void 0 && existing.run !== null) {
581
667
  return;
582
668
  }
583
- stale.set(index.indexName, { index, partitionsRebuilt: 0, run: null });
669
+ stale.set(index.indexName, { index, partitionsRebuilt: 0, run: null, abandoned: false });
584
670
  },
585
671
  clearStale(indexName) {
586
672
  stale.delete(indexName);
@@ -591,6 +677,14 @@ function createAnalysisRebuildCoordinator(config, deps) {
591
677
  isRunning(indexName) {
592
678
  return stale.get(indexName)?.run !== null && stale.get(indexName)?.run !== void 0;
593
679
  },
680
+ async abandon(indexName) {
681
+ const entry = stale.get(indexName);
682
+ if (entry !== void 0 && entry.run !== null) entry.abandoned = true;
683
+ const running = active;
684
+ if (running === null || running.indexName !== indexName) return;
685
+ running.entry.abandoned = true;
686
+ await running.done.catch(() => void 0);
687
+ },
594
688
  review(indexName) {
595
689
  const entry = stale.get(indexName);
596
690
  return entry === void 0 ? Promise.resolve() : review(indexName, entry);
@@ -1400,7 +1494,7 @@ function buildEntry(fields) {
1400
1494
  }
1401
1495
 
1402
1496
  // src/serialization/chunk-checksum.ts
1403
- function yieldToEventLoop3() {
1497
+ function yieldToEventLoop2() {
1404
1498
  if (typeof setImmediate === "function") {
1405
1499
  return new Promise((resolve) => setImmediate(resolve));
1406
1500
  }
@@ -1412,7 +1506,7 @@ async function advanceChecksum(start, payload) {
1412
1506
  const end = Math.min(offset + CHECKSUM_YIELD_CHUNK_BYTES, payload.length);
1413
1507
  state = crc32Update(state, payload.subarray(offset, end));
1414
1508
  if (end < payload.length) {
1415
- await yieldToEventLoop3();
1509
+ await yieldToEventLoop2();
1416
1510
  }
1417
1511
  }
1418
1512
  return state;
@@ -5639,7 +5733,7 @@ function removeWhatTheSegmentReplaces(manager, segment, contents) {
5639
5733
  manager.endBatchRemove();
5640
5734
  }
5641
5735
  }
5642
- function yieldToEventLoop4() {
5736
+ function yieldToEventLoop3() {
5643
5737
  return new Promise((resolve) => setTimeout(resolve, 0));
5644
5738
  }
5645
5739
  async function loadPartitionSegments(store, partition, manager, options) {
@@ -5650,7 +5744,7 @@ async function loadPartitionSegments(store, partition, manager, options) {
5650
5744
  const segment = createSharedFrozenSegment(shareSegmentArrays(contents.arrays, manager.partialWordAnalysis));
5651
5745
  removeWhatTheSegmentReplaces(manager, segment, contents);
5652
5746
  if (contents.arrays.documentCount > 0) manager.attachFrozenSegment(partition.partitionId, segment);
5653
- await yieldToEventLoop4();
5747
+ await yieldToEventLoop3();
5654
5748
  }
5655
5749
  }
5656
5750
 
@@ -8078,7 +8172,7 @@ function metadataConfigOf(entry) {
8078
8172
  b: entry.config.bm25?.b ?? 0.75,
8079
8173
  ...embedding !== void 0 ? { embedding } : {},
8080
8174
  surfaceForms: entry.config.surfaceForms !== false,
8081
- analysisRevision: entry.language.revision,
8175
+ ...entry.analysisRevision !== null ? { analysisRevision: entry.analysisRevision } : {},
8082
8176
  ...typeof entry.config.tokenizer === "string" ? { tokenizer: entry.config.tokenizer } : {},
8083
8177
  ...typeof entry.config.stopWords === "string" ? { stopWords: entry.config.stopWords } : {},
8084
8178
  ...entry.config.stopWords instanceof Set ? { stopWordList: [...entry.config.stopWords].sort(compareCodePoints) } : {},
@@ -8166,7 +8260,8 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
8166
8260
  indexUuid: metadata.indexUuid ?? null,
8167
8261
  heldPartitions: metadata.heldPartitions ?? null,
8168
8262
  documentCount: metadata.documentCount ?? 0,
8169
- partitionCount: metadata.partitionCount
8263
+ partitionCount: metadata.partitionCount,
8264
+ analysisRevision: metadata.analysisRevision ?? null
8170
8265
  });
8171
8266
  if (loadData) await deps.indexState().registerOpen(metadata.indexName);
8172
8267
  else deps.indexState().registerClosed(metadata.indexName);
@@ -8289,8 +8384,14 @@ function createIndexStateCoordinator(config, callbacks) {
8289
8384
  return entry;
8290
8385
  }
8291
8386
  }
8387
+ function transitionPending(indexName, entry) {
8388
+ if (entries.get(indexName) !== entry) return true;
8389
+ if (entry.state === "dropping" && entry.dropPromise !== null) return true;
8390
+ return entry.state === "closing" && entry.closePromise !== null;
8391
+ }
8292
8392
  async function reopen(indexName, explicit) {
8293
- const entry = await awaitPendingTransitions(indexName);
8393
+ let entry = await awaitPendingTransitions(indexName);
8394
+ while (transitionPending(indexName, entry)) entry = await awaitPendingTransitions(indexName);
8294
8395
  if (entry.state === "open") return;
8295
8396
  if (explicit) {
8296
8397
  entry.reopenFailures = 0;
@@ -8879,6 +8980,7 @@ function shareFrozenSegment(segment, analysis) {
8879
8980
  }
8880
8981
 
8881
8982
  // src/engine/worker-resync.ts
8983
+ var copyLoadSequence = 0;
8882
8984
  function captureComposite(manager, partitionId) {
8883
8985
  const partition = manager.getPartition(partitionId);
8884
8986
  if (!isCompositePartition(partition)) return null;
@@ -8939,8 +9041,8 @@ async function sendPartition(indexName, executors, captured) {
8939
9041
  )
8940
9042
  );
8941
9043
  }
8942
- async function transferIndexToPool(indexName, pool, config, manager, analysisStale = false) {
8943
- await loadIndexOntoWorkers(
9044
+ function transferIndexToPool(indexName, pool, config, manager, analysisStale = false) {
9045
+ return loadIndexOntoWorkers(
8944
9046
  indexName,
8945
9047
  pool.getAllExecutors(),
8946
9048
  config,
@@ -8950,6 +9052,8 @@ async function transferIndexToPool(indexName, pool, config, manager, analysisSta
8950
9052
  );
8951
9053
  }
8952
9054
  async function loadIndexOntoWorkers(indexName, executors, config, manager, onDropped, analysisStale = false) {
9055
+ copyLoadSequence += 1;
9056
+ const copyLoad = `${indexName}#${copyLoadSequence}`;
8953
9057
  const captured = capturePartitions(manager);
8954
9058
  await Promise.allSettled(
8955
9059
  executors.map(
@@ -8964,13 +9068,15 @@ async function loadIndexOntoWorkers(indexName, executors, config, manager, onDro
8964
9068
  indexName,
8965
9069
  config,
8966
9070
  requestId: `resync-create-${indexName}`,
8967
- analysisStale
9071
+ analysisStale,
9072
+ copyLoad
8968
9073
  })
8969
9074
  )
8970
9075
  );
8971
9076
  for (const partition of captured) {
8972
9077
  await sendPartition(indexName, executors, partition);
8973
9078
  }
9079
+ return copyLoad;
8974
9080
  }
8975
9081
 
8976
9082
  // src/engine/orchestration/constants.ts
@@ -9198,6 +9304,14 @@ function enqueueReplication(state, indexName, action) {
9198
9304
  queue.inFlight.push(run);
9199
9305
  queue.tail = run;
9200
9306
  }
9307
+ function markCopiesLoaded(state, indexName, copyLoad) {
9308
+ enqueueReplication(state, indexName, {
9309
+ type: "copyLoaded",
9310
+ indexName,
9311
+ copyLoad,
9312
+ requestId: `copy-loaded-${copyLoad}`
9313
+ });
9314
+ }
9201
9315
  async function replicateToWorkers(state, action) {
9202
9316
  if (!("indexName" in action)) {
9203
9317
  if (state.workerPool === null) return;
@@ -10163,8 +10277,9 @@ async function loadOntoReplacement(state, replacement, indexName) {
10163
10277
  if (entry === void 0 || manager === void 0 || !state.scaledOutIndexes.has(indexName)) return;
10164
10278
  const buffered = [];
10165
10279
  state.copyLoadBuffers.set(indexName, buffered);
10280
+ let copyLoad;
10166
10281
  try {
10167
- await loadIndexOntoWorkers(
10282
+ copyLoad = await loadIndexOntoWorkers(
10168
10283
  indexName,
10169
10284
  [replacement.executor],
10170
10285
  entry.config,
@@ -10177,6 +10292,7 @@ async function loadOntoReplacement(state, replacement, indexName) {
10177
10292
  state.copyLoadBuffers.delete(indexName);
10178
10293
  for (const action of buffered) enqueueReplication(state, indexName, action);
10179
10294
  }
10295
+ markCopiesLoaded(state, indexName, copyLoad);
10180
10296
  refreshVectorCopies(state, indexName);
10181
10297
  }
10182
10298
  async function replaceWorker(state, pool, workerId) {
@@ -10353,7 +10469,13 @@ async function loadCopies(state, indexName, reason) {
10353
10469
  state.copyLoadBuffers.set(indexName, buffered);
10354
10470
  try {
10355
10471
  freezeLiveTailsBeforeCopiesLoad(manager);
10356
- await transferIndexToPool(indexName, pool, entry.config, manager, state.callbacks?.isAnalysisStale?.(indexName));
10472
+ const copyLoad = await transferIndexToPool(
10473
+ indexName,
10474
+ pool,
10475
+ entry.config,
10476
+ manager,
10477
+ state.callbacks?.isAnalysisStale?.(indexName)
10478
+ );
10357
10479
  if (state.desyncedIndexes.has(indexName)) {
10358
10480
  discardLoadedCopies(state, pool, indexName);
10359
10481
  return;
@@ -10362,6 +10484,7 @@ async function loadCopies(state, indexName, reason) {
10362
10484
  state.lastAccessAt.set(indexName, Date.now());
10363
10485
  state.poolRetryDelayMs = POOL_RESTART_DELAY_MS;
10364
10486
  for (const action of buffered) enqueueReplication(state, indexName, action);
10487
+ markCopiesLoaded(state, indexName, copyLoad);
10365
10488
  } catch (err) {
10366
10489
  if (dropReason !== void 0) state.droppedCopies.set(indexName, dropReason);
10367
10490
  throw err;
@@ -10705,7 +10828,13 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
10705
10828
  idleSweep: null
10706
10829
  };
10707
10830
  startIdleSweep(state);
10831
+ function settlingCopyWork(indexName) {
10832
+ return state.copyTransitions.get(indexName)?.done ?? state.compactionsInFlight.get(indexName);
10833
+ }
10708
10834
  async function resyncIndex(indexName, wasScaledOut) {
10835
+ for (let pending = settlingCopyWork(indexName); pending !== void 0; pending = settlingCopyWork(indexName)) {
10836
+ await pending;
10837
+ }
10709
10838
  state.desyncedIndexes.delete(indexName);
10710
10839
  if (!wasScaledOut) return;
10711
10840
  await scaleOutIndex(state, indexName, copyThresholdReason(state, indexName));
@@ -10763,8 +10892,15 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
10763
10892
  const manager = executor.getManager(indexName);
10764
10893
  if (pool !== null && entry !== void 0 && manager !== void 0) {
10765
10894
  try {
10766
- await transferIndexToPool(indexName, pool, entry.config, manager, callbacks?.isAnalysisStale?.(indexName));
10895
+ const copyLoad = await transferIndexToPool(
10896
+ indexName,
10897
+ pool,
10898
+ entry.config,
10899
+ manager,
10900
+ callbacks?.isAnalysisStale?.(indexName)
10901
+ );
10767
10902
  state.scaledOutIndexes.add(indexName);
10903
+ markCopiesLoaded(state, indexName, copyLoad);
10768
10904
  refreshVectorCopies(state, indexName);
10769
10905
  } catch (repairError) {
10770
10906
  throw new NarsilError(
@@ -11057,6 +11193,10 @@ function createEngineCore(config, hooks, capacityScope = "index") {
11057
11193
  getManager: (indexName) => executor.getManager(indexName),
11058
11194
  desyncIndex: (indexName) => orchestrator.desyncIndex(indexName),
11059
11195
  resyncIndex: (indexName, wasPromoted) => orchestrator.resyncIndex(indexName, wasPromoted),
11196
+ recordCurrentAnalysis: (indexName) => {
11197
+ const entry = indexRegistry.get(indexName);
11198
+ if (entry !== void 0 && !analysisRebuild.isStale(indexName)) entry.analysisRevision = entry.language.revision;
11199
+ },
11060
11200
  durabilityManager: durability?.manager ?? null
11061
11201
  });
11062
11202
  const indexState = wireIndexState({
@@ -11231,11 +11371,16 @@ function refuseCode(entry) {
11231
11371
  }
11232
11372
  function indexConfigurationOf(entry) {
11233
11373
  refuseCode(entry);
11234
- return configurationPayloadOf(entry.config, entry.language, entry.embeddingAdapterName);
11374
+ return configurationPayloadOf(entry.config, entry.language, entry.embeddingAdapterName, entry.analysisRevision);
11235
11375
  }
11236
11376
  function requireConfigurationWithinSizeLimit(config, language) {
11237
11377
  const adapter = config.embedding?.adapter;
11238
- const payload = configurationPayloadOf(config, language, typeof adapter === "string" ? adapter : null);
11378
+ const payload = configurationPayloadOf(
11379
+ config,
11380
+ language,
11381
+ typeof adapter === "string" ? adapter : null,
11382
+ language.revision
11383
+ );
11239
11384
  if (messagePackUpperBoundBytes(payload) <= MAX_INDEX_CONFIGURATION_BYTES) return;
11240
11385
  const bytes = encode(payload).byteLength;
11241
11386
  if (bytes <= MAX_INDEX_CONFIGURATION_BYTES) return;
@@ -11245,12 +11390,12 @@ function requireConfigurationWithinSizeLimit(config, language) {
11245
11390
  { bytes, maximum: MAX_INDEX_CONFIGURATION_BYTES }
11246
11391
  );
11247
11392
  }
11248
- function configurationPayloadOf(config, language, adapterName) {
11393
+ function configurationPayloadOf(config, language, adapterName, analysisRevision) {
11249
11394
  const bm25 = config.bm25;
11250
11395
  return {
11251
11396
  schema: config.schema,
11252
11397
  language: language.name,
11253
- analysisRevision: language.revision,
11398
+ ...analysisRevision !== null ? { analysisRevision } : {},
11254
11399
  ...typeof config.tokenizer === "string" ? { tokenizer: config.tokenizer } : {},
11255
11400
  ...typeof config.stopWords === "string" ? { stopWords: config.stopWords } : {},
11256
11401
  ...config.stopWords instanceof Set ? { stopWordList: [...config.stopWords].sort(compareCodePoints) } : {},
@@ -11409,7 +11554,8 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
11409
11554
  indexUuid: indexUuid ?? null,
11410
11555
  heldPartitions: null,
11411
11556
  documentCount: 0,
11412
- partitionCount: indexConfig.partitions?.maxPartitions ?? 1
11557
+ partitionCount: indexConfig.partitions?.maxPartitions ?? 1,
11558
+ analysisRevision: language.revision
11413
11559
  });
11414
11560
  await core.indexState.registerOpen(name);
11415
11561
  if (core.durability) {
@@ -11421,6 +11567,7 @@ async function dropEngineIndex(core, name) {
11421
11567
  core.guardShutdown();
11422
11568
  const entry = core.requireIndex(name);
11423
11569
  await core.indexState.drop(name, async () => {
11570
+ await core.analysisRebuild.abandon(name);
11424
11571
  await core.orchestrator.closeIndex(name);
11425
11572
  if (core.durability) {
11426
11573
  try {
@@ -14196,7 +14343,8 @@ async function createRestoredIndex(indexName, prepared, partitionCount, deps) {
14196
14343
  indexUuid: null,
14197
14344
  heldPartitions: null,
14198
14345
  documentCount: 0,
14199
- partitionCount
14346
+ partitionCount,
14347
+ analysisRevision: prepared.storedRevision
14200
14348
  });
14201
14349
  await deps.indexState.registerOpen(indexName);
14202
14350
  const release = await deps.indexState.acquire(indexName, false);
@@ -14581,6 +14729,15 @@ async function enterRead(core, indexName) {
14581
14729
  core.guardShutdown();
14582
14730
  if (core.rebalancingIndexes.has(indexName)) await yieldToEventLoop();
14583
14731
  }
14732
+ async function readUnderOneAnalysis(core, indexName, read) {
14733
+ let analysisStale = core.analysisRebuild.isStale(indexName);
14734
+ let result = await read();
14735
+ while (core.analysisRebuild.isStale(indexName) !== analysisStale) {
14736
+ analysisStale = core.analysisRebuild.isStale(indexName);
14737
+ result = await read();
14738
+ }
14739
+ return { result, analysisStale };
14740
+ }
14584
14741
  function scopedParams(params, options) {
14585
14742
  if (options?.globalStats === void 0) {
14586
14743
  return params;
@@ -14613,7 +14770,7 @@ async function runEngineQuery(core, indexName, params, options) {
14613
14770
  entry.embeddingAdapterName
14614
14771
  );
14615
14772
  const workerSearch = core.orchestrator.hasWorkerPool() ? core.orchestrator.searchViaWorker.bind(core.orchestrator) : void 0;
14616
- const result = await executeQuery(resolvedParams, {
14773
+ const context = {
14617
14774
  manager,
14618
14775
  language: entry.language,
14619
14776
  config: entry.config,
@@ -14623,7 +14780,15 @@ async function runEngineQuery(core, indexName, params, options) {
14623
14780
  partitionIds: options?.partitionIds,
14624
14781
  cursorBinding: queryBindingOf(params),
14625
14782
  patternWorkCap: core.patternWorkCap
14626
- });
14783
+ };
14784
+ const { result, analysisStale } = await readUnderOneAnalysis(
14785
+ core,
14786
+ indexName,
14787
+ () => executeQuery(resolvedParams, context)
14788
+ );
14789
+ if (analysisStale) {
14790
+ result.analysisStale = true;
14791
+ }
14627
14792
  if (core.pluginRegistry.hasHooks("afterSearch")) {
14628
14793
  const observed = { ...result, hits: [...result.hits] };
14629
14794
  try {
@@ -14632,9 +14797,6 @@ async function runEngineQuery(core, indexName, params, options) {
14632
14797
  console.warn("afterSearch plugin hook error:", err);
14633
14798
  }
14634
14799
  }
14635
- if (core.analysisRebuild.isStale(indexName)) {
14636
- result.analysisStale = true;
14637
- }
14638
14800
  return result;
14639
14801
  } finally {
14640
14802
  release();
@@ -14655,7 +14817,7 @@ async function runEnginePreflight(core, indexName, params, options) {
14655
14817
  entry.embeddingAdapterName
14656
14818
  );
14657
14819
  const workerSearch = core.orchestrator.hasWorkerPool() ? core.orchestrator.searchViaWorker.bind(core.orchestrator) : void 0;
14658
- const result = await executePreflight(resolvedParams, {
14820
+ const context = {
14659
14821
  manager,
14660
14822
  language: entry.language,
14661
14823
  config: entry.config,
@@ -14665,8 +14827,13 @@ async function runEnginePreflight(core, indexName, params, options) {
14665
14827
  partitionIds: options?.partitionIds,
14666
14828
  cursorBinding: queryBindingOf(params),
14667
14829
  patternWorkCap: core.patternWorkCap
14668
- });
14669
- if (core.analysisRebuild.isStale(indexName)) {
14830
+ };
14831
+ const { result, analysisStale } = await readUnderOneAnalysis(
14832
+ core,
14833
+ indexName,
14834
+ () => executePreflight(resolvedParams, context)
14835
+ );
14836
+ if (analysisStale) {
14670
14837
  result.analysisStale = true;
14671
14838
  }
14672
14839
  return result;
@@ -18,7 +18,7 @@ function stopUsingTheNativeSearchCore(error) {
18
18
  }
19
19
  function noteFallback(operation) {
20
20
  }
21
- var NATIVE_CORE_ABI_VERSION = 2;
21
+ var NATIVE_CORE_ABI_VERSION = 3;
22
22
  var BINARY_FILE_NAME = "narsil-core.node";
23
23
  var MUSL_DIRECTORY = "musl";
24
24
  var PLATFORMS_WITH_A_BINARY = /* @__PURE__ */ new Set([