@delali/narsil 0.3.1-next.84 → 0.3.1-next.85

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,19 +1,19 @@
1
1
  import { validateWorkerConfig, validateIndexName, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateBM25Params, validatePartitionConfig, queryBindingOf, executeQuery, executePreflight, executeSuggest, executeListDocuments, validateDocId } from './chunk-4D6OH4EF.mjs';
2
2
  import { ReplicationMessageTypes } from './chunk-GCM3JFVA.mjs';
3
3
  import { WIRE_BATCH_BUDGET, resolveVectorText, bindEngineCore, restoredConfigFields, restoredEmbedding, collectRequiredFieldFailures, embedChunkDocuments, providedDocId, requireDocumentObject, admitInsert, asBatchInsertError, requireMatchingDocId, embedDocumentFields, assertDocumentCarriesMappedVectors, admitBufferedUpsert, createPendingAdmissions, recordPendingAdmission } from './chunk-2453RJYK.mjs';
4
- import { NATIVE_CRC32_MIN_BYTES, generateId, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, splitWorkerBudget, resolveWorkerCount, createDirectExecutor, createPartitionRouter, MAX_FILE_BYTES, resolveWorkerEntry, missingWorkerEntry, readHeapStatistics, nodeThreadId, CHECKSUM_YIELD_CHUNK_BYTES, resolvePartitionInsertOptions, resolveIndexAnalysis, createPartitionManager, buildSegmentPayload, createFrozenSegment, resolveRequestThreadCount, vectorsPerFile, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, createPartitionIndex, selectionKeeps, decodeMapWithoutFields, isCompositePartition, segmentArraysOf, mergeSegmentContents, survivorDocumentsOf, partitionStateOf, VECTOR_FILE_MAX_VECTORS, deadBytesFor, mergeFrozenSegments, workerResourceLimits, decodeVectorIndexParts, bytesToVectors, freezeSegmentShared, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, SHUTDOWN_TIMEOUT_MS, decodeVectorFilePayload, decodeVectorGraphPartPayload, assembleGraphParts, countDead, encodeSegmentState, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, VALID_HNSW_METRICS, spawnNodeWorker, decodeVectorGraphPayload, graphPartsOfEarlierPayload, decodeStoredDocument, isDead } from './chunk-IA67SOJO.mjs';
4
+ import { NATIVE_CRC32_MIN_BYTES, generateId, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, splitWorkerBudget, resolveWorkerCount, createDirectExecutor, createPartitionRouter, MAX_FILE_BYTES, resolveWorkerEntry, missingWorkerEntry, readHeapStatistics, nodeThreadId, CHECKSUM_YIELD_CHUNK_BYTES, resolvePartitionInsertOptions, resolveIndexAnalysis, createPartitionManager, buildSegmentPayload, createFrozenSegment, resolveRequestThreadCount, vectorsPerFile, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, createPartitionIndex, selectionKeeps, decodeMapWithoutFields, isCompositePartition, segmentArraysOf, mergeSegmentContents, survivorDocumentsOf, partitionStateOf, VECTOR_FILE_MAX_VECTORS, deadBytesFor, mergeFrozenSegments, workerResourceLimits, decodeVectorIndexParts, bytesToVectors, freezeSegmentShared, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, SHUTDOWN_TIMEOUT_MS, decodeVectorFilePayload, decodeVectorGraphPartPayload, assembleGraphParts, countDead, encodeSegmentState, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, VALID_HNSW_METRICS, spawnNodeWorker, decodeVectorGraphPayload, graphPartsOfEarlierPayload, decodeStoredDocument, isDead } from './chunk-RCSXJCI5.mjs';
5
5
  import { requirePatternSearchFor, requirePartialWordSearchFor, collectQueryTermStats, mergePartitionStats, oversampledFacetConfig, fnv1a, availablePatternSearch, PATTERN_HOST, kWayMerge, mergeFacets, setNestedValue } from './chunk-XYDP7CCJ.mjs';
6
6
  import { detectRuntime, getLanguage } from './chunk-OLXM65YH.mjs';
7
7
  import { MAX_INDEX_CONFIGURATION_BYTES, STATISTICS_BROADCAST_INTERVAL_MS, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, HEAP_PRESSURE_WARNING_FRACTION, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, FOREIGN_STATS_TTL_MS, MAX_IMPORT_REFUSALS_REPORTED, BATCH_CHUNK_SIZE, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION, EXPORT_CHUNK_BYTES } from './chunk-ZCKPRCYI.mjs';
8
- import { releaseLocksHeldBy } from './chunk-RZXNWQSM.mjs';
9
- import { requireSearchBackendSettings } from './chunk-B3UOQJD3.mjs';
8
+ import { releaseLocksHeldBy } from './chunk-NIWDQTZB.mjs';
9
+ import { requireSearchBackendSettings } from './chunk-NFF5TDRG.mjs';
10
10
  import { DEFAULT_PATTERN_WORK_CAP, MATCHER_STATE_BUDGET_BYTES } from './chunk-SREPGC6O.mjs';
11
11
  import { getNestedValue, parseVectorDimension } from './chunk-4VBG6PEH.mjs';
12
12
  import { extractVectorFieldsFromSchema, messagePackUpperBoundBytes, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, raiseEarlierReleaseGraphSetting, partialFieldsOf, flattenSchema, requireValidFieldTypes, encodeDocument, schemaFieldsOf, isQuantizationMode, isStorageMode, validateRequiredFields, requireDocumentWithinSizeLimit, validateDocument, validateDocumentStrict } from './chunk-KJ7LB3TR.mjs';
13
13
  import { MIN_M } from './chunk-KEHK2O6P.mjs';
14
14
  import { isSingleStringFieldType, VECTOR_PATTERN, MAX_DOCUMENT_NESTING_DEPTH } from './chunk-LS5IRM23.mjs';
15
15
  import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
16
- import { createRequestId, isValidWorkerResponse, threadSlotOfWorker } from './chunk-POWOTUKJ.mjs';
16
+ import { createRequestId, isValidWorkerResponse, threadSlotOfWorker } from './chunk-YRX6XMOQ.mjs';
17
17
  import { ErrorCodes, NarsilError } from './chunk-TPJU7NDJ.mjs';
18
18
  import { openSync, close, readFileSync } from 'fs';
19
19
  import { rename, readFile, writeFile, unlink, realpath, readdir, readlink } from 'fs/promises';
@@ -8980,6 +8980,7 @@ function shareFrozenSegment(segment, analysis) {
8980
8980
  }
8981
8981
 
8982
8982
  // src/engine/worker-resync.ts
8983
+ var copyLoadSequence = 0;
8983
8984
  function captureComposite(manager, partitionId) {
8984
8985
  const partition = manager.getPartition(partitionId);
8985
8986
  if (!isCompositePartition(partition)) return null;
@@ -9040,8 +9041,8 @@ async function sendPartition(indexName, executors, captured) {
9040
9041
  )
9041
9042
  );
9042
9043
  }
9043
- async function transferIndexToPool(indexName, pool, config, manager, analysisStale = false) {
9044
- await loadIndexOntoWorkers(
9044
+ function transferIndexToPool(indexName, pool, config, manager, analysisStale = false) {
9045
+ return loadIndexOntoWorkers(
9045
9046
  indexName,
9046
9047
  pool.getAllExecutors(),
9047
9048
  config,
@@ -9051,6 +9052,8 @@ async function transferIndexToPool(indexName, pool, config, manager, analysisSta
9051
9052
  );
9052
9053
  }
9053
9054
  async function loadIndexOntoWorkers(indexName, executors, config, manager, onDropped, analysisStale = false) {
9055
+ copyLoadSequence += 1;
9056
+ const copyLoad = `${indexName}#${copyLoadSequence}`;
9054
9057
  const captured = capturePartitions(manager);
9055
9058
  await Promise.allSettled(
9056
9059
  executors.map(
@@ -9065,13 +9068,15 @@ async function loadIndexOntoWorkers(indexName, executors, config, manager, onDro
9065
9068
  indexName,
9066
9069
  config,
9067
9070
  requestId: `resync-create-${indexName}`,
9068
- analysisStale
9071
+ analysisStale,
9072
+ copyLoad
9069
9073
  })
9070
9074
  )
9071
9075
  );
9072
9076
  for (const partition of captured) {
9073
9077
  await sendPartition(indexName, executors, partition);
9074
9078
  }
9079
+ return copyLoad;
9075
9080
  }
9076
9081
 
9077
9082
  // src/engine/orchestration/constants.ts
@@ -9299,6 +9304,14 @@ function enqueueReplication(state, indexName, action) {
9299
9304
  queue.inFlight.push(run);
9300
9305
  queue.tail = run;
9301
9306
  }
9307
+ function markCopiesLoaded(state, indexName, copyLoad) {
9308
+ enqueueReplication(state, indexName, {
9309
+ type: "copyLoaded",
9310
+ indexName,
9311
+ copyLoad,
9312
+ requestId: `copy-loaded-${copyLoad}`
9313
+ });
9314
+ }
9302
9315
  async function replicateToWorkers(state, action) {
9303
9316
  if (!("indexName" in action)) {
9304
9317
  if (state.workerPool === null) return;
@@ -10264,8 +10277,9 @@ async function loadOntoReplacement(state, replacement, indexName) {
10264
10277
  if (entry === void 0 || manager === void 0 || !state.scaledOutIndexes.has(indexName)) return;
10265
10278
  const buffered = [];
10266
10279
  state.copyLoadBuffers.set(indexName, buffered);
10280
+ let copyLoad;
10267
10281
  try {
10268
- await loadIndexOntoWorkers(
10282
+ copyLoad = await loadIndexOntoWorkers(
10269
10283
  indexName,
10270
10284
  [replacement.executor],
10271
10285
  entry.config,
@@ -10278,6 +10292,7 @@ async function loadOntoReplacement(state, replacement, indexName) {
10278
10292
  state.copyLoadBuffers.delete(indexName);
10279
10293
  for (const action of buffered) enqueueReplication(state, indexName, action);
10280
10294
  }
10295
+ markCopiesLoaded(state, indexName, copyLoad);
10281
10296
  refreshVectorCopies(state, indexName);
10282
10297
  }
10283
10298
  async function replaceWorker(state, pool, workerId) {
@@ -10454,7 +10469,13 @@ async function loadCopies(state, indexName, reason) {
10454
10469
  state.copyLoadBuffers.set(indexName, buffered);
10455
10470
  try {
10456
10471
  freezeLiveTailsBeforeCopiesLoad(manager);
10457
- await transferIndexToPool(indexName, pool, entry.config, manager, state.callbacks?.isAnalysisStale?.(indexName));
10472
+ const copyLoad = await transferIndexToPool(
10473
+ indexName,
10474
+ pool,
10475
+ entry.config,
10476
+ manager,
10477
+ state.callbacks?.isAnalysisStale?.(indexName)
10478
+ );
10458
10479
  if (state.desyncedIndexes.has(indexName)) {
10459
10480
  discardLoadedCopies(state, pool, indexName);
10460
10481
  return;
@@ -10463,6 +10484,7 @@ async function loadCopies(state, indexName, reason) {
10463
10484
  state.lastAccessAt.set(indexName, Date.now());
10464
10485
  state.poolRetryDelayMs = POOL_RESTART_DELAY_MS;
10465
10486
  for (const action of buffered) enqueueReplication(state, indexName, action);
10487
+ markCopiesLoaded(state, indexName, copyLoad);
10466
10488
  } catch (err) {
10467
10489
  if (dropReason !== void 0) state.droppedCopies.set(indexName, dropReason);
10468
10490
  throw err;
@@ -10870,8 +10892,15 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
10870
10892
  const manager = executor.getManager(indexName);
10871
10893
  if (pool !== null && entry !== void 0 && manager !== void 0) {
10872
10894
  try {
10873
- await transferIndexToPool(indexName, pool, entry.config, manager, callbacks?.isAnalysisStale?.(indexName));
10895
+ const copyLoad = await transferIndexToPool(
10896
+ indexName,
10897
+ pool,
10898
+ entry.config,
10899
+ manager,
10900
+ callbacks?.isAnalysisStale?.(indexName)
10901
+ );
10874
10902
  state.scaledOutIndexes.add(indexName);
10903
+ markCopiesLoaded(state, indexName, copyLoad);
10875
10904
  refreshVectorCopies(state, indexName);
10876
10905
  } catch (repairError) {
10877
10906
  throw new NarsilError(
@@ -18,7 +18,7 @@ function stopUsingTheNativeSearchCore(error) {
18
18
  }
19
19
  function noteFallback(operation) {
20
20
  }
21
- var NATIVE_CORE_ABI_VERSION = 2;
21
+ var NATIVE_CORE_ABI_VERSION = 3;
22
22
  var BINARY_FILE_NAME = "narsil-core.node";
23
23
  var MUSL_DIRECTORY = "musl";
24
24
  var PLATFORMS_WITH_A_BINARY = /* @__PURE__ */ new Set([