@delali/narsil 0.3.1-next.95 → 0.3.1-next.96

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (28) hide show
  1. package/dist/browser/{chunk-LULHQXCK.mjs → chunk-3MYKLHRO.mjs} +2 -2
  2. package/dist/browser/{chunk-D3CC2NMZ.mjs → chunk-KPMIHLOV.mjs} +5 -5
  3. package/dist/browser/{chunk-3U636ADD.mjs → chunk-ZGMR46O3.mjs} +1 -1
  4. package/dist/browser/{durability-wiring-3ZRKSPJD.mjs → durability-wiring-R7P64REL.mjs} +3 -3
  5. package/dist/browser/{restore-queue-RBULSOVX.mjs → restore-queue-WMYC5ZK6.mjs} +1 -1
  6. package/dist/browser/{snapshot-Y3NGUJHX.mjs → snapshot-3A3WDNHO.mjs} +110 -58
  7. package/dist/browser/{write-ZUBDNLCH.mjs → write-JODZHKIF.mjs} +2 -2
  8. package/dist/{chunk-54HYC5FT.mjs → chunk-2BY6K52M.mjs} +3 -3
  9. package/dist/{chunk-J7SDAOTP.mjs → chunk-GMY7F2VA.mjs} +1 -1
  10. package/dist/{chunk-7QONURHW.mjs → chunk-H75XXPMY.mjs} +1 -1
  11. package/dist/{chunk-L26LEDHO.mjs → chunk-LJBJQPB5.mjs} +3 -3
  12. package/dist/{chunk-JFUE6F2Y.mjs → chunk-O5WPRAVZ.mjs} +5 -5
  13. package/dist/{chunk-MASE2LTJ.mjs → chunk-TCYUAUHF.mjs} +2 -2
  14. package/dist/{chunk-Q4UTU5ZE.mjs → chunk-VVFLEGZI.mjs} +2 -2
  15. package/dist/distribution.d.ts +1 -1
  16. package/dist/distribution.mjs +5 -5
  17. package/dist/{durability-wiring-FRPV6JLZ.mjs → durability-wiring-2VBNYYDY.mjs} +6 -6
  18. package/dist/{engine-BGt9lI1V.d.ts → engine-CbZmXUp6.d.ts} +2 -2
  19. package/dist/index.browser.mjs +1 -1
  20. package/dist/index.d.ts +2 -2
  21. package/dist/index.mjs +1 -1
  22. package/dist/persistence/durability/checkpoint-worker.mjs +4 -4
  23. package/dist/react.d.ts +1 -1
  24. package/dist/{restore-queue-I5SI6AUQ.mjs → restore-queue-UFIQJTSG.mjs} +1 -1
  25. package/dist/server.d.ts +1 -1
  26. package/dist/{snapshot-YC6TS4OP.mjs → snapshot-XII3ANGW.mjs} +115 -61
  27. package/dist/{write-M2OALSA4.mjs → write-6M3VG4TU.mjs} +2 -2
  28. package/package.json +7 -7
@@ -1,5 +1,5 @@
1
- import { KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, restoreVectorFields, KEYWORD_SEGMENT_PAYLOAD_VERSION, littleEndianBytes } from './chunk-3U636ADD.mjs';
2
- import { VERSION, rulesOfStoredSynonyms, flattenSchema, packChecksummedEnvelope, openChecksummedEnvelope, requireValidFieldTypes, isCompositePartition, crc32, reconstructSchemaFromMetadata, extractVectorFieldsFromSchema, readHeader, HEADER_SIZE, engineVersionParts, writeHeader, partialFieldsOf, getLanguage, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, VECTOR_PATTERN, isQuantizationMode, isStorageMode, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, fnv1a, selectionKeeps, decodeMapWithoutFields, prepareDocumentVectors, messagePackUpperBoundBytes, segmentArraysOf, mergeSegmentContents, resolveIndexAnalysis } from './chunk-D3CC2NMZ.mjs';
1
+ import { KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, restoreVectorFields, KEYWORD_SEGMENT_PAYLOAD_VERSION, littleEndianBytes } from './chunk-ZGMR46O3.mjs';
2
+ import { VERSION, rulesOfStoredSynonyms, flattenSchema, packChecksummedEnvelope, openChecksummedEnvelope, requireValidFieldTypes, isCompositePartition, crc32, reconstructSchemaFromMetadata, extractVectorFieldsFromSchema, readHeader, HEADER_SIZE, engineVersionParts, writeHeader, partialFieldsOf, getLanguage, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, VECTOR_PATTERN, isQuantizationMode, isStorageMode, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, fnv1a, selectionKeeps, decodeMapWithoutFields, prepareDocumentVectors, messagePackUpperBoundBytes, segmentArraysOf, mergeSegmentContents, resolveIndexAnalysis } from './chunk-KPMIHLOV.mjs';
3
3
  import { decodeVectorGraphPartPayload, assembleGraphParts, decodeVectorFilePayload, countDead, VECTOR_FILE_MAX_VECTORS, deadBytesFor } from './chunk-WKPJ4E73.mjs';
4
4
  import { compareCodePoints, NarsilError, ErrorCodes, MAX_FILE_BYTES, VALID_HNSW_METRICS } from './chunk-HARBSUWP.mjs';
5
5
  import { decode, encode } from '@msgpack/msgpack';
@@ -18980,7 +18980,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
18980
18980
  if (config?.lifecycle !== void 0 && durabilityTier === null) {
18981
18981
  throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index lifecycle settings require durability");
18982
18982
  }
18983
- const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-3ZRKSPJD.mjs');
18983
+ const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-R7P64REL.mjs');
18984
18984
  const filesystemDurability = durabilityTier?.kind === "wal";
18985
18985
  const executor = createDirectExecutor({
18986
18986
  vectorWorkerCopies: vectorCopyPolicy,
@@ -22399,14 +22399,14 @@ function portabilityOperations(core, config, engine) {
22399
22399
  return {
22400
22400
  snapshot(indexName) {
22401
22401
  return withOpenIndex(indexName, async () => {
22402
- const { createSnapshot } = await import('./snapshot-Y3NGUJHX.mjs');
22402
+ const { createSnapshot } = await import('./snapshot-3A3WDNHO.mjs');
22403
22403
  return createSnapshot(snapshotDepsOf(indexName));
22404
22404
  });
22405
22405
  },
22406
22406
  async restore(indexName, data) {
22407
22407
  core.guardShutdown();
22408
22408
  validateIndexName(indexName);
22409
- const restored = import('./restore-queue-RBULSOVX.mjs').then(
22409
+ const restored = import('./restore-queue-WMYC5ZK6.mjs').then(
22410
22410
  ({ queueRestore }) => queueRestore(core, restoreHost, restoresInFlight, indexName, data)
22411
22411
  );
22412
22412
  await emitWriteEventAfter(core, indexName, restored);
@@ -22415,8 +22415,8 @@ function portabilityOperations(core, config, engine) {
22415
22415
  exportDocuments(indexName) {
22416
22416
  return withOpenIndex(indexName, async () => {
22417
22417
  const [{ checkpointForArchive }, { documentExportStream }] = await Promise.all([
22418
- import('./snapshot-Y3NGUJHX.mjs'),
22419
- import('./write-ZUBDNLCH.mjs')
22418
+ import('./snapshot-3A3WDNHO.mjs'),
22419
+ import('./write-JODZHKIF.mjs')
22420
22420
  ]);
22421
22421
  const header = indexConfigurationOf(core.requireIndex(indexName));
22422
22422
  return documentExportStream(header, await checkpointForArchive(snapshotDepsOf(indexName)));
@@ -1,4 +1,4 @@
1
- import { removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors, getNestedValue, requireValidFieldTypes, VECTOR_PATTERN } from './chunk-D3CC2NMZ.mjs';
1
+ import { removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors, getNestedValue, requireValidFieldTypes, VECTOR_PATTERN } from './chunk-KPMIHLOV.mjs';
2
2
  import { NarsilError, ErrorCodes } from './chunk-HARBSUWP.mjs';
3
3
  import { decode } from '@msgpack/msgpack';
4
4
 
@@ -1,7 +1,7 @@
1
1
  import { nodeThreadId, spawnNodeWorker } from './chunk-HOZW3PKP.mjs';
2
- import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, decodeSegmentManifest, readListedFile, adapterCheckpointStore, readMetadataEnvelope, readSegmentManifest, reclaimUnlistedKeys, snapshotBundleKey, manifestReferencedKeys, loadSegmentedCheckpoint, checkpointLastSeqNo, writeMetadataEnvelope, metadataFromConfig, DIRECTORY_LOCK_ATTEMPTS, createSeqOwner, readCurrentManifest, manifestFileRefs, DIRECTORY_LOCK_SETTLE_MS, commitCheckpointManifest, removeCheckpointGarbage, readSegmentFile, isRecord, invalidManifest, deserializePayloadV2, unpackEnvelopeBytes, checkpointPrimaryTerm, SINGLE_NODE_PRIMARY_TERM, writeCheckpointSegments, partitionContentsOf, writeLiveVectors, frameRecord, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, captureWholePartition, readCommitMarker, isId, isUint32, isSequenceNumber, readWalSegments, durableRecordMissing, growPartitionsTo, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, MAX_SEGMENTS_PER_PARTITION, VECTOR_FILE_ENVELOPE_VERSION, writeSegmentHeader, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, SEGMENT_HEADER_SIZE, documentsOfPayloadV2, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, LOG_RECORDS_PER_CHECKPOINT, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, metadataConfigOf, buildEntry } from './chunk-LULHQXCK.mjs';
3
- import { applyDeleteEntry, applyIndexEntry } from './chunk-3U636ADD.mjs';
4
- import { encodeDocument, fnv1a, requireValidFieldTypes, openChecksummedEnvelope, HEADER_SIZE, buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-D3CC2NMZ.mjs';
2
+ import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, decodeSegmentManifest, readListedFile, adapterCheckpointStore, readMetadataEnvelope, readSegmentManifest, reclaimUnlistedKeys, snapshotBundleKey, manifestReferencedKeys, loadSegmentedCheckpoint, checkpointLastSeqNo, writeMetadataEnvelope, metadataFromConfig, DIRECTORY_LOCK_ATTEMPTS, createSeqOwner, readCurrentManifest, manifestFileRefs, DIRECTORY_LOCK_SETTLE_MS, commitCheckpointManifest, removeCheckpointGarbage, readSegmentFile, isRecord, invalidManifest, deserializePayloadV2, unpackEnvelopeBytes, checkpointPrimaryTerm, SINGLE_NODE_PRIMARY_TERM, writeCheckpointSegments, partitionContentsOf, writeLiveVectors, frameRecord, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, captureWholePartition, readCommitMarker, isId, isUint32, isSequenceNumber, readWalSegments, durableRecordMissing, growPartitionsTo, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, MAX_SEGMENTS_PER_PARTITION, VECTOR_FILE_ENVELOPE_VERSION, writeSegmentHeader, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, SEGMENT_HEADER_SIZE, documentsOfPayloadV2, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, LOG_RECORDS_PER_CHECKPOINT, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, metadataConfigOf, buildEntry } from './chunk-3MYKLHRO.mjs';
3
+ import { applyDeleteEntry, applyIndexEntry } from './chunk-ZGMR46O3.mjs';
4
+ import { encodeDocument, fnv1a, requireValidFieldTypes, openChecksummedEnvelope, HEADER_SIZE, buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-KPMIHLOV.mjs';
5
5
  import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-WKPJ4E73.mjs';
6
6
  import { NarsilError, ErrorCodes, compareCodePoints, MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, detectRuntime, MIN_M, resolveWorkerEntry, MAX_FILE_BYTES } from './chunk-HARBSUWP.mjs';
7
7
  import { decode } from '@msgpack/msgpack';
@@ -18,7 +18,7 @@ function restoreDepsOf(core, host) {
18
18
  function queueRestore(core, host, restoresInFlight, indexName, data) {
19
19
  const previous = restoresInFlight.get(indexName) ?? Promise.resolve();
20
20
  const restored = previous.then(async () => {
21
- const { restoreFromSnapshot } = await import('./snapshot-Y3NGUJHX.mjs');
21
+ const { restoreFromSnapshot } = await import('./snapshot-3A3WDNHO.mjs');
22
22
  await restoreFromSnapshot(indexName, data, restoreDepsOf(core, host));
23
23
  });
24
24
  const settled = restored.then(
@@ -1,7 +1,7 @@
1
1
  import { vectorsPerFile } from './chunk-FYFQJKFW.mjs';
2
- import { indexMetadataOf, writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, writeLiveVectors, commitCheckpointManifest, manifestKey, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, manifestFileRefs, unpackIndexSnapshotEnvelope, encodeSegmentManifest, loadSegmentedCheckpoint, writeMetadataEnvelope, openManifestPayload, listedKeysOf, decodeManifestPayload, readSegmentFile, readVectorFile, readGraphParts, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-LULHQXCK.mjs';
3
- import './chunk-3U636ADD.mjs';
4
- import { encodeIndexConfiguration, createByteReader, hasNrslMagic, HEADER_SIZE, readHeader, raiseEarlierReleaseGraphSetting, restoredConfigFields, restoredEmbedding, getLanguage, validateSchema, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateVectorPromotion, validateVectorStorage, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, readHeapStatistics, readResidentMemory, prepareStoredSynonyms, rejectSnapshot, extractVectorFieldsFromSchema, loadVectorCodeFor, decodeEveryStoredDocument, parseVectorDimension, insertDocumentVectors, decodeIndexConfiguration, flattenSchema } from './chunk-D3CC2NMZ.mjs';
2
+ import { indexMetadataOf, writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, writeLiveVectors, commitCheckpointManifest, manifestKey, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, manifestFileRefs, unpackIndexSnapshotEnvelope, encodeSegmentManifest, loadSegmentedCheckpoint, writeMetadataEnvelope, openManifestPayload, listedKeysOf, decodeManifestPayload, readSegmentFile, readVectorFile, readGraphParts, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-3MYKLHRO.mjs';
3
+ import './chunk-ZGMR46O3.mjs';
4
+ import { encodeIndexConfiguration, createByteReader, hasNrslMagic, HEADER_SIZE, readHeader, raiseEarlierReleaseGraphSetting, restoredConfigFields, restoredEmbedding, getLanguage, validateSchema, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateVectorPromotion, validateVectorStorage, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, prepareStoredSynonyms, rejectSnapshot, readHeapStatistics, readResidentMemory, extractVectorFieldsFromSchema, loadVectorCodeFor, decodeEveryStoredDocument, parseVectorDimension, insertDocumentVectors, decodeIndexConfiguration, flattenSchema } from './chunk-KPMIHLOV.mjs';
5
5
  import { decodeVectorIndexParts, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, bytesToVectors } from './chunk-WKPJ4E73.mjs';
6
6
  import { NarsilError, ErrorCodes, MAX_FILE_BYTES } from './chunk-HARBSUWP.mjs';
7
7
  import { decode } from '@msgpack/msgpack';
@@ -499,43 +499,84 @@ async function validateStagedCheckpoint(store, manifest) {
499
499
  }
500
500
 
501
501
  // src/engine/snapshot/constants.ts
502
- var RESTORE_HEAP_BYTES_PER_STORED_BYTE = 0.25;
503
- var RESTORE_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 9.82;
504
- var RESTORE_HEAP_BYTES_PER_VECTOR = 330;
505
- var RESTORE_HEAP_BYTES_PER_VECTOR_DIMENSION = 0.3;
506
- var RESTORE_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 923;
507
- var RESTORE_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 0.25;
508
- var RESTORE_EXTERNAL_BYTES_PER_STORED_BYTE = 5.07;
509
- var RESTORE_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 18.12;
510
- var RESTORE_EXTERNAL_BYTES_PER_VECTOR = 1853;
511
- var RESTORE_EXTERNAL_BYTES_PER_VECTOR_DIMENSION = 11.62;
512
- var RESTORE_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 2676;
513
- var RESTORE_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 30.19;
502
+ var RESTORE_WORST_HEAP_BYTES_PER_STORED_BYTE = 0.2;
503
+ var RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 7.86;
504
+ var RESTORE_WORST_HEAP_BYTES_PER_VECTOR = 264;
505
+ var RESTORE_WORST_HEAP_BYTES_PER_VECTOR_DIMENSION = 0.24;
506
+ var RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 738;
507
+ var RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 0.2;
508
+ var RESTORE_WORST_EXTERNAL_BYTES_PER_STORED_BYTE = 4.06;
509
+ var RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 14.5;
510
+ var RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR = 1482;
511
+ var RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR_DIMENSION = 9.29;
512
+ var RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 2141;
513
+ var RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 24.15;
514
+ var RESTORE_FLOOR_HEAP_BYTES_PER_STORED_BYTE = 0.16;
515
+ var RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 6.16;
516
+ var RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR = 204;
517
+ var RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR_DIMENSION = 0.17;
518
+ var RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 695;
519
+ var RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 0.04;
520
+ var RESTORE_FLOOR_EXTERNAL_BYTES_PER_STORED_BYTE = 2.14;
521
+ var RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 5.05;
522
+ var RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR = 1080;
523
+ var RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR_DIMENSION = 0.72;
524
+ var RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 1744;
525
+ var RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 20.05;
526
+ var RESTORE_ESTIMATE_MARGIN = 1.25;
514
527
 
515
528
  // src/engine/snapshot/memory-budget.ts
516
529
  var MEGABYTE_BYTES = 1048576;
517
530
  var ARCHIVE_RATES = {
518
- heap: {
519
- perStoredByte: RESTORE_HEAP_BYTES_PER_STORED_BYTE,
520
- perVector: RESTORE_HEAP_BYTES_PER_VECTOR,
521
- perVectorDimension: RESTORE_HEAP_BYTES_PER_VECTOR_DIMENSION
531
+ worst: {
532
+ heap: {
533
+ perStoredByte: RESTORE_WORST_HEAP_BYTES_PER_STORED_BYTE,
534
+ perVector: RESTORE_WORST_HEAP_BYTES_PER_VECTOR,
535
+ perVectorDimension: RESTORE_WORST_HEAP_BYTES_PER_VECTOR_DIMENSION
536
+ },
537
+ external: {
538
+ perStoredByte: RESTORE_WORST_EXTERNAL_BYTES_PER_STORED_BYTE,
539
+ perVector: RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR,
540
+ perVectorDimension: RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR_DIMENSION
541
+ }
522
542
  },
523
- external: {
524
- perStoredByte: RESTORE_EXTERNAL_BYTES_PER_STORED_BYTE,
525
- perVector: RESTORE_EXTERNAL_BYTES_PER_VECTOR,
526
- perVectorDimension: RESTORE_EXTERNAL_BYTES_PER_VECTOR_DIMENSION
543
+ floor: {
544
+ heap: {
545
+ perStoredByte: RESTORE_FLOOR_HEAP_BYTES_PER_STORED_BYTE,
546
+ perVector: RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR,
547
+ perVectorDimension: RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR_DIMENSION
548
+ },
549
+ external: {
550
+ perStoredByte: RESTORE_FLOOR_EXTERNAL_BYTES_PER_STORED_BYTE,
551
+ perVector: RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR,
552
+ perVectorDimension: RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR_DIMENSION
553
+ }
527
554
  }
528
555
  };
529
556
  var EARLIER_SNAPSHOT_RATES = {
530
- heap: {
531
- perStoredByte: RESTORE_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
532
- perVector: RESTORE_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
533
- perVectorDimension: RESTORE_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
557
+ worst: {
558
+ heap: {
559
+ perStoredByte: RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
560
+ perVector: RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
561
+ perVectorDimension: RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
562
+ },
563
+ external: {
564
+ perStoredByte: RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
565
+ perVector: RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
566
+ perVectorDimension: RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
567
+ }
534
568
  },
535
- external: {
536
- perStoredByte: RESTORE_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
537
- perVector: RESTORE_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
538
- perVectorDimension: RESTORE_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
569
+ floor: {
570
+ heap: {
571
+ perStoredByte: RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
572
+ perVector: RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
573
+ perVectorDimension: RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
574
+ },
575
+ external: {
576
+ perStoredByte: RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
577
+ perVector: RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
578
+ perVectorDimension: RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
579
+ }
539
580
  }
540
581
  };
541
582
  function bytesAt(rate, storedBytes, vectors) {
@@ -543,10 +584,12 @@ function bytesAt(rate, storedBytes, vectors) {
543
584
  for (const { dimension, count } of vectors) bytes += count * (rate.perVector + dimension * rate.perVectorDimension);
544
585
  return bytes;
545
586
  }
546
- function footprintAt(rates, storedBytes, vectors) {
587
+ function footprintAt(rates, replacesIndex, storedBytes, vectors) {
588
+ const chosen = replacesIndex ? rates.worst : rates.floor;
589
+ const scale = replacesIndex ? RESTORE_ESTIMATE_MARGIN : 1 / RESTORE_ESTIMATE_MARGIN;
547
590
  return {
548
- heapBytes: bytesAt(rates.heap, storedBytes, vectors),
549
- externalBytes: bytesAt(rates.external, storedBytes, vectors)
591
+ heapBytes: bytesAt(chosen.heap, storedBytes, vectors) * scale,
592
+ externalBytes: bytesAt(chosen.external, storedBytes, vectors) * scale
550
593
  };
551
594
  }
552
595
  function deadCountOf(dead) {
@@ -557,7 +600,7 @@ function deadCountOf(dead) {
557
600
  }
558
601
  return count;
559
602
  }
560
- function archiveFootprint(prepared, manifest) {
603
+ function archiveFootprint(prepared, manifest, replacesIndex) {
561
604
  let segmentBytes = 0;
562
605
  for (const partition of manifest.partitions) {
563
606
  for (const segment of partition.segments) segmentBytes += segment.bytes;
@@ -568,36 +611,45 @@ function archiveFootprint(prepared, manifest) {
568
611
  const dimension = dimensions.get(field.fieldPath) ?? 0;
569
612
  for (const file of field.files) vectors.push({ dimension, count: Math.max(0, file.count - deadCountOf(file.dead)) });
570
613
  }
571
- return footprintAt(ARCHIVE_RATES, segmentBytes, vectors);
614
+ return footprintAt(ARCHIVE_RATES, replacesIndex, segmentBytes, vectors);
572
615
  }
573
- function earlierSnapshotFootprint(prepared, partitionBytes, vectorCounts) {
616
+ function earlierSnapshotFootprint(prepared, partitionBytes, vectorCounts, replacesIndex) {
574
617
  const dimensions = extractVectorFieldsFromSchema(prepared.config.schema);
575
618
  const vectors = [...vectorCounts].map(([fieldPath, count]) => ({ dimension: dimensions.get(fieldPath) ?? 0, count }));
576
- return footprintAt(EARLIER_SNAPSHOT_RATES, partitionBytes, vectors);
619
+ return footprintAt(EARLIER_SNAPSHOT_RATES, replacesIndex, partitionBytes, vectors);
577
620
  }
578
621
  function inMegabytes(bytes) {
579
622
  return Math.ceil(bytes / MEGABYTE_BYTES);
580
623
  }
581
- function refuse(indexName, budget, neededBytes, availableBytes) {
582
- const where = budget === "heap" ? "of JavaScript heap" : "of process memory";
583
- throw new NarsilError(
584
- ErrorCodes.RESTORE_MEMORY_EXCEEDED,
585
- `The restore of index "${indexName}" needs about ${inMegabytes(neededBytes)} MB ${where}, and this process has ${inMegabytes(Math.max(0, availableBytes))} MB free, so the index keeps its earlier contents. Drop the index first to free the memory that it holds, or give the process more memory, then restore again`,
586
- { indexName, budget, neededBytes, availableBytes }
587
- );
588
- }
589
- function requireRestoreMemory(indexName, footprint) {
624
+ function shortfallOf(heapBytes, processBytes, countInUse) {
590
625
  const heap = readHeapStatistics();
591
626
  if (heap !== null) {
592
- const availableBytes = (heap.configuredLimitBytes ?? heap.limitBytes) - heap.usedBytes;
593
- if (footprint.heapBytes > availableBytes) refuse(indexName, "heap", footprint.heapBytes, availableBytes);
627
+ const limitBytes = heap.configuredLimitBytes ?? heap.limitBytes;
628
+ const availableBytes = countInUse ? limitBytes - heap.usedBytes : limitBytes;
629
+ if (heapBytes > availableBytes) return { budget: "heap", neededBytes: heapBytes, availableBytes };
594
630
  }
595
631
  const resident = readResidentMemory();
596
632
  if (resident !== null) {
597
- const neededBytes = footprint.heapBytes + footprint.externalBytes;
598
- const availableBytes = resident.limitBytes - resident.residentBytes;
599
- if (neededBytes > availableBytes) refuse(indexName, "process", neededBytes, availableBytes);
633
+ const availableBytes = countInUse ? resident.limitBytes - resident.residentBytes : resident.limitBytes;
634
+ if (processBytes > availableBytes) return { budget: "process", neededBytes: processBytes, availableBytes };
600
635
  }
636
+ return null;
637
+ }
638
+ function refuse(indexName, replacesIndex, shortfall) {
639
+ const { budget, neededBytes, availableBytes } = shortfall;
640
+ const where = budget === "heap" ? "of JavaScript heap" : "of process memory";
641
+ const outcome = replacesIndex ? "so the index keeps its earlier contents. Drop the index first to free the memory that it holds, or give the process more memory, then restore again" : "so the engine creates no index. Give the process more memory, then restore again";
642
+ throw new NarsilError(
643
+ ErrorCodes.RESTORE_MEMORY_EXCEEDED,
644
+ `The restore of index "${indexName}" needs about ${inMegabytes(neededBytes)} MB ${where}, and this process has ${inMegabytes(Math.max(0, availableBytes))} MB free, ${outcome}`,
645
+ { indexName, budget, neededBytes, availableBytes }
646
+ );
647
+ }
648
+ function requireRestoreMemory(indexName, footprint, replacesIndex) {
649
+ const processBytes = footprint.heapBytes + footprint.externalBytes;
650
+ if (shortfallOf(footprint.heapBytes, processBytes, true) === null) return;
651
+ const shortfall = shortfallOf(footprint.heapBytes, processBytes, replacesIndex);
652
+ if (shortfall !== null) refuse(indexName, replacesIndex, shortfall);
601
653
  }
602
654
 
603
655
  // src/workers/detached-index.ts
@@ -898,9 +950,8 @@ async function stageArchive(indexName, reader, staging, deps) {
898
950
  manifest: async (manifest2, sourceName) => {
899
951
  if (staged.prepared === null) missingPart("configuration");
900
952
  requireSameSchema(staged.prepared, manifest2);
901
- if (deps.indexRegistry.has(indexName)) {
902
- requireRestoreMemory(indexName, archiveFootprint(staged.prepared, manifest2));
903
- }
953
+ const replacesIndex = deps.indexRegistry.has(indexName);
954
+ requireRestoreMemory(indexName, archiveFootprint(staged.prepared, manifest2, replacesIndex), replacesIndex);
904
955
  staged.manifest = renamed(manifest2, sourceName, indexName);
905
956
  },
906
957
  file: async (entryName, _ref, bytes) => {
@@ -990,7 +1041,7 @@ function vectorsOfSnapshot(envelope) {
990
1041
  }
991
1042
  return byDocument;
992
1043
  }
993
- function footprintOf(envelope, prepared) {
1044
+ function footprintOf(envelope, prepared, replacesIndex) {
994
1045
  let partitionBytes = 0;
995
1046
  for (const payload of envelope.partitions ?? []) partitionBytes += payload.byteLength;
996
1047
  const vectorCounts = /* @__PURE__ */ new Map();
@@ -999,7 +1050,7 @@ function footprintOf(envelope, prepared) {
999
1050
  for (const part of decodeVectorIndexParts(rawParts)) count += part.docIds.length;
1000
1051
  vectorCounts.set(fieldPath, count);
1001
1052
  }
1002
- return earlierSnapshotFootprint(prepared, partitionBytes, vectorCounts);
1053
+ return earlierSnapshotFootprint(prepared, partitionBytes, vectorCounts, replacesIndex);
1003
1054
  }
1004
1055
  function documentsOfPartition(envelope, payload) {
1005
1056
  return envelope.version === 2 ? documentsOfPayloadV2(payload).documents : deserializePayloadV1(payload).documents;
@@ -1059,7 +1110,8 @@ async function buildRestoredIndex(indexName, envelope, prepared, staging, deps)
1059
1110
  async function restoreEarlierSnapshot(indexName, data, deps) {
1060
1111
  const envelope = await decodeEarlierSnapshot(data);
1061
1112
  const prepared = prepareRestoredIndex(envelope, deps);
1062
- if (deps.indexRegistry.has(indexName)) requireRestoreMemory(indexName, footprintOf(envelope, prepared));
1113
+ const replacesIndex = deps.indexRegistry.has(indexName);
1114
+ requireRestoreMemory(indexName, footprintOf(envelope, prepared, replacesIndex), replacesIndex);
1063
1115
  const analysedNow = { ...prepared, storedRevision: prepared.language.revision };
1064
1116
  const staging = deps.durability?.manager.stageRestore?.(indexName) ?? null;
1065
1117
  try {
@@ -1,5 +1,5 @@
1
- import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-3U636ADD.mjs';
2
- import { decodeStoredDocument, openChecksummedEnvelope, NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-D3CC2NMZ.mjs';
1
+ import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-ZGMR46O3.mjs';
2
+ import { decodeStoredDocument, openChecksummedEnvelope, NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-KPMIHLOV.mjs';
3
3
  import { decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-WKPJ4E73.mjs';
4
4
  import { EXPORT_CHUNK_BYTES, NarsilError, ErrorCodes } from './chunk-HARBSUWP.mjs';
5
5
  import { decode } from '@msgpack/msgpack';
@@ -1,6 +1,6 @@
1
- import { decodeSegmentManifest, loadPartitionSegments, loadVectorField } from './chunk-MASE2LTJ.mjs';
2
- import { manifestKey, segmentsPrefix } from './chunk-Q4UTU5ZE.mjs';
3
- import { readHeader, HEADER_SIZE, crc32, engineVersionParts, writeHeader } from './chunk-JFUE6F2Y.mjs';
1
+ import { decodeSegmentManifest, loadPartitionSegments, loadVectorField } from './chunk-TCYUAUHF.mjs';
2
+ import { manifestKey, segmentsPrefix } from './chunk-VVFLEGZI.mjs';
3
+ import { readHeader, HEADER_SIZE, crc32, engineVersionParts, writeHeader } from './chunk-O5WPRAVZ.mjs';
4
4
  import { VALID_HNSW_METRICS } from './chunk-RDJKYKMQ.mjs';
5
5
  import { requireValidFieldTypes, isQuantizationMode, isStorageMode } from './chunk-4ZJWRETC.mjs';
6
6
  import { VECTOR_PATTERN } from './chunk-L5ORED7V.mjs';
@@ -1,4 +1,4 @@
1
- import { removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors } from './chunk-JFUE6F2Y.mjs';
1
+ import { removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors } from './chunk-O5WPRAVZ.mjs';
2
2
  import { getNestedValue } from './chunk-SBCEHIPL.mjs';
3
3
  import { requireValidFieldTypes } from './chunk-4ZJWRETC.mjs';
4
4
  import { VECTOR_PATTERN } from './chunk-L5ORED7V.mjs';
@@ -1,5 +1,5 @@
1
1
  import { nodeThreadId } from './chunk-C7SLA6CM.mjs';
2
- import { DIRECTORY_LOCK_ATTEMPTS, DIRECTORY_LOCK_SETTLE_MS, DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS } from './chunk-Q4UTU5ZE.mjs';
2
+ import { DIRECTORY_LOCK_ATTEMPTS, DIRECTORY_LOCK_SETTLE_MS, DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS } from './chunk-VVFLEGZI.mjs';
3
3
  import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
4
4
  import { NarsilError, ErrorCodes } from './chunk-SYWGORQB.mjs';
5
5
  import { openSync, close, readFileSync } from 'fs';
@@ -1,8 +1,8 @@
1
1
  import { vectorsPerFile } from './chunk-MWT5WCJ5.mjs';
2
- import { writeLiveVectors, decodeSegmentManifest, openManifestPayload, listedKeysOf, decodeManifestPayload, SINGLE_NODE_PRIMARY_TERM } from './chunk-MASE2LTJ.mjs';
2
+ import { writeLiveVectors, decodeSegmentManifest, openManifestPayload, listedKeysOf, decodeManifestPayload, SINGLE_NODE_PRIMARY_TERM } from './chunk-TCYUAUHF.mjs';
3
3
  import { vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION } from './chunk-F52QGNZ2.mjs';
4
- import { writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, commitCheckpointManifest, manifestKey, manifestFileRefs, partitionContentsOf } from './chunk-Q4UTU5ZE.mjs';
5
- import { HEADER_SIZE } from './chunk-JFUE6F2Y.mjs';
4
+ import { writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, commitCheckpointManifest, manifestKey, manifestFileRefs, partitionContentsOf } from './chunk-VVFLEGZI.mjs';
5
+ import { HEADER_SIZE } from './chunk-O5WPRAVZ.mjs';
6
6
  import { MAX_FILE_BYTES } from './chunk-RDJKYKMQ.mjs';
7
7
  import { NarsilError, ErrorCodes } from './chunk-SYWGORQB.mjs';
8
8
 
@@ -4008,7 +4008,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
4008
4008
  if (config?.lifecycle !== void 0 && durabilityTier === null) {
4009
4009
  throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index lifecycle settings require durability");
4010
4010
  }
4011
- const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-FRPV6JLZ.mjs');
4011
+ const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-2VBNYYDY.mjs');
4012
4012
  const filesystemDurability = durabilityTier?.kind === "wal";
4013
4013
  const executor = createDirectExecutor({
4014
4014
  vectorWorkerCopies: vectorCopyPolicy,
@@ -6908,14 +6908,14 @@ function portabilityOperations(core, config, engine) {
6908
6908
  return {
6909
6909
  snapshot(indexName) {
6910
6910
  return withOpenIndex(indexName, async () => {
6911
- const { createSnapshot } = await import('./snapshot-YC6TS4OP.mjs');
6911
+ const { createSnapshot } = await import('./snapshot-XII3ANGW.mjs');
6912
6912
  return createSnapshot(snapshotDepsOf(indexName));
6913
6913
  });
6914
6914
  },
6915
6915
  async restore(indexName, data) {
6916
6916
  core.guardShutdown();
6917
6917
  validateIndexName(indexName);
6918
- const restored = import('./restore-queue-I5SI6AUQ.mjs').then(
6918
+ const restored = import('./restore-queue-UFIQJTSG.mjs').then(
6919
6919
  ({ queueRestore }) => queueRestore(core, restoreHost, restoresInFlight, indexName, data)
6920
6920
  );
6921
6921
  await emitWriteEventAfter(core, indexName, restored);
@@ -6924,8 +6924,8 @@ function portabilityOperations(core, config, engine) {
6924
6924
  exportDocuments(indexName) {
6925
6925
  return withOpenIndex(indexName, async () => {
6926
6926
  const [{ checkpointForArchive }, { documentExportStream }] = await Promise.all([
6927
- import('./snapshot-YC6TS4OP.mjs'),
6928
- import('./write-M2OALSA4.mjs')
6927
+ import('./snapshot-XII3ANGW.mjs'),
6928
+ import('./write-6M3VG4TU.mjs')
6929
6929
  ]);
6930
6930
  const header = indexConfigurationOf(core.requireIndex(indexName));
6931
6931
  return documentExportStream(header, await checkpointForArchive(snapshotDepsOf(indexName)));
@@ -1,6 +1,6 @@
1
1
  import { decodeVectorGraphPartPayload, assembleGraphParts, decodeVectorFilePayload, countDead, VECTOR_FILE_MAX_VECTORS, deadBytesFor } from './chunk-F52QGNZ2.mjs';
2
- import { MAX_SEGMENT_ID, MANIFEST_ENVELOPE_VERSION, EARLIER_SEGMENT_MANIFEST_VERSION, SEGMENT_MANIFEST_VERSION, MAX_SEGMENTS_PER_PARTITION, segmentKey, readSegmentFile, readListedFile, requireListedChecksum, vectorFileKey, graphPartKey } from './chunk-Q4UTU5ZE.mjs';
3
- import { VERSION, rulesOfStoredSynonyms, openChecksummedEnvelope, HEADER_SIZE, packChecksummedEnvelope } from './chunk-JFUE6F2Y.mjs';
2
+ import { MAX_SEGMENT_ID, MANIFEST_ENVELOPE_VERSION, EARLIER_SEGMENT_MANIFEST_VERSION, SEGMENT_MANIFEST_VERSION, MAX_SEGMENTS_PER_PARTITION, segmentKey, readSegmentFile, readListedFile, requireListedChecksum, vectorFileKey, graphPartKey } from './chunk-VVFLEGZI.mjs';
3
+ import { VERSION, rulesOfStoredSynonyms, openChecksummedEnvelope, HEADER_SIZE, packChecksummedEnvelope } from './chunk-O5WPRAVZ.mjs';
4
4
  import { MAX_FILE_BYTES } from './chunk-RDJKYKMQ.mjs';
5
5
  import { decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays } from './chunk-3LYY62M6.mjs';
6
6
  import { flattenSchema, requireValidFieldTypes } from './chunk-4ZJWRETC.mjs';
@@ -1,7 +1,7 @@
1
- import { KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, restoreVectorFields, KEYWORD_SEGMENT_PAYLOAD_VERSION, littleEndianBytes } from './chunk-J7SDAOTP.mjs';
1
+ import { KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, restoreVectorFields, KEYWORD_SEGMENT_PAYLOAD_VERSION, littleEndianBytes } from './chunk-GMY7F2VA.mjs';
2
2
  import { ReplicationMessageTypes } from './chunk-OHP7QR33.mjs';
3
3
  import { WIRE_BATCH_BUDGET } from './chunk-P36MUFEJ.mjs';
4
- import { packChecksummedEnvelope, crc32, openChecksummedEnvelope, reconstructSchemaFromMetadata, HEADER_SIZE, prepareDocumentVectors } from './chunk-JFUE6F2Y.mjs';
4
+ import { packChecksummedEnvelope, crc32, openChecksummedEnvelope, reconstructSchemaFromMetadata, HEADER_SIZE, prepareDocumentVectors } from './chunk-O5WPRAVZ.mjs';
5
5
  import { MAX_FILE_BYTES } from './chunk-RDJKYKMQ.mjs';
6
6
  import { freezeSegmentShared, createSharedFrozenSegment, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, selectionKeeps, decodeMapWithoutFields, shareSegmentArrays, segmentArraysOf, mergeSegmentContents } from './chunk-3LYY62M6.mjs';
7
7
  import { fnv1a, isCompositePartition, survivorDocumentsOf, partitionStateOf } from './chunk-CHOGQSHK.mjs';
@@ -5,7 +5,7 @@ import { I as IndexConfig, A as AnyDocument, a as InsertOptions } from './schema
5
5
  import { a as SynonymRule } from './synonyms-CpOAHPa-.js';
6
6
  import { b as NodeCapacity, N as NodeRole } from './index-ONAEaqtx.js';
7
7
  export { c as AllocationConstraints, d as AllocationEvent, A as AllocationTable, C as ClusterCoordinator, I as IndexPartitionBytes, e as NodeEvent, f as NodeMemoryReport, a as NodeRegistration, P as PartitionAssignment, g as PartitionState, S as SchemaEvent, U as UnassignedReason } from './index-ONAEaqtx.js';
8
- import { N as Narsil } from './engine-BGt9lI1V.js';
8
+ import { N as Narsil } from './engine-CbZmXUp6.js';
9
9
  export { L as ListenHandler, N as NodeTransport, R as RespondFn, T as TransportConfig, a as TransportMessage } from './types-Cb9kiY5P.js';
10
10
  import './config-DPphZfuM.js';
11
11
  import './adapters-79OIa4xm.js';
@@ -1,17 +1,17 @@
1
- import { buildArchiveFromMemory, archiveStream, memoryCheckpointStore, readSnapshotArchive } from './chunk-L26LEDHO.mjs';
1
+ import { buildArchiveFromMemory, archiveStream, memoryCheckpointStore, readSnapshotArchive } from './chunk-LJBJQPB5.mjs';
2
2
  import './chunk-MWT5WCJ5.mjs';
3
- import { indexMetadataOf, loadPartitionSegments } from './chunk-MASE2LTJ.mjs';
3
+ import { indexMetadataOf, loadPartitionSegments } from './chunk-TCYUAUHF.mjs';
4
4
  import { decodeVectorFilePayload, bytesToVectors, isDead } from './chunk-F52QGNZ2.mjs';
5
5
  import { MAX_MESSAGE_SIZE_BYTES } from './chunk-UNEEFTEU.mjs';
6
6
  import { DEFAULT_NODE_LIFECYCLE_CONFIG, DEFAULT_CONTROLLER_CONFIG, validateIndexName, readIndexState, readIndexMetadataSnapshot, storeSynonymSet, deleteSynonymSet, indexConfigKey, indexUuidOfStoredMetadata, getIndexMetadata, withPartitionCount, indexSettingsOf, putIndexMetadata, indexConfigFromSettings, metadataWithSynonymSet, isRecord, storedSynonymSetOf, ALLOCATION_RETRY_DELAY_MS, TEARDOWN_CAS_ATTEMPTS, TEARDOWN_RETRY_ROUNDS, isValidInteger, ALLOCATION_CAS_ATTEMPTS, TEARDOWN_RETRY_DELAY_MS, RECOVERY_CAS_ATTEMPTS, ESTIMATED_PARTITION_BYTES, REBALANCE_THRESHOLD, PARTITION_STORES_TIMEOUT_MS, LEADERSHIP_IMBALANCE_THRESHOLD, BOOTSTRAP_CAS_ATTEMPTS, getClusterIndexConfig } from './chunk-GJZHU4FC.mjs';
7
7
  import './chunk-ICL2AELD.mjs';
8
8
  import './chunk-37CD3OKS.mjs';
9
9
  import './chunk-DJHHF2L5.mjs';
10
- import { validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, createForwardBatchResultMessage, readListedFile, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage, computeEntryChecksum, createEntryMessage, createEntryBatchMessage, createInsyncConfirmMessage, validateAckPayload, createInsyncRemoveMessage, validateInsyncConfirmPayload, createInsyncAddMessage } from './chunk-Q4UTU5ZE.mjs';
11
- import { applyIndexEntry, applyDeleteEntry, validateReplicationEntry } from './chunk-J7SDAOTP.mjs';
10
+ import { validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, createForwardBatchResultMessage, readListedFile, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage, computeEntryChecksum, createEntryMessage, createEntryBatchMessage, createInsyncConfirmMessage, validateAckPayload, createInsyncRemoveMessage, validateInsyncConfirmPayload, createInsyncAddMessage } from './chunk-VVFLEGZI.mjs';
11
+ import { applyIndexEntry, applyDeleteEntry, validateReplicationEntry } from './chunk-GMY7F2VA.mjs';
12
12
  import { TransportErrorCodes, ReplicationMessageTypes, ClusterMessageTypes, QueryMessageTypes, TransportError } from './chunk-OHP7QR33.mjs';
13
13
  import { MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR, WIRE_BATCH_BUDGET } from './chunk-P36MUFEJ.mjs';
14
- import { requireKnownConfig, resolveRunnableConfig, createEngineCore, startEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, createEngineIndex, encodeIndexConfiguration, requireConfigurationWithinSizeLimit, prepareConfiguredSynonyms, createByteReader, openChecksummedEnvelope, emptySerializablePartition, crc32Init, crc32Final, crc32Update } from './chunk-JFUE6F2Y.mjs';
14
+ import { requireKnownConfig, resolveRunnableConfig, createEngineCore, startEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, createEngineIndex, encodeIndexConfiguration, requireConfigurationWithinSizeLimit, prepareConfiguredSynonyms, createByteReader, openChecksummedEnvelope, emptySerializablePartition, crc32Init, crc32Final, crc32Update } from './chunk-O5WPRAVZ.mjs';
15
15
  import { now, listBindingOf, requireSurfaceFormsForPartialFields, queryBindingOf, hitsKeptPerGroup, clampLimit, resolveHybridFusion, dedupePinnedEntries, foldGroupReducer } from './chunk-R36MIGBU.mjs';
16
16
  import './chunk-RDJKYKMQ.mjs';
17
17
  import { synonymModule, readHeapStatistics } from './chunk-3LYY62M6.mjs';
@@ -1,14 +1,14 @@
1
- import { readMetadataEnvelope, readSegmentManifest, reclaimUnlistedKeys, loadSegmentedCheckpoint, writeMetadataEnvelope, readCurrentManifest, deserializePayloadV2, unpackEnvelopeBytes, growPartitionsTo, documentsOfPayloadV2 } from './chunk-54HYC5FT.mjs';
2
- import { decodeSegmentManifest, metadataFromConfig, createSeqOwner, isRecord, invalidManifest, SINGLE_NODE_PRIMARY_TERM, writeLiveVectors, isId, isUint32, isSequenceNumber, VECTOR_FILE_ENVELOPE_VERSION, metadataConfigOf } from './chunk-MASE2LTJ.mjs';
1
+ import { readMetadataEnvelope, readSegmentManifest, reclaimUnlistedKeys, loadSegmentedCheckpoint, writeMetadataEnvelope, readCurrentManifest, deserializePayloadV2, unpackEnvelopeBytes, growPartitionsTo, documentsOfPayloadV2 } from './chunk-2BY6K52M.mjs';
2
+ import { decodeSegmentManifest, metadataFromConfig, createSeqOwner, isRecord, invalidManifest, SINGLE_NODE_PRIMARY_TERM, writeLiveVectors, isId, isUint32, isSequenceNumber, VECTOR_FILE_ENVELOPE_VERSION, metadataConfigOf } from './chunk-TCYUAUHF.mjs';
3
3
  import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-F52QGNZ2.mjs';
4
4
  import './chunk-37CD3OKS.mjs';
5
- import { createDurableDirectory, getFs } from './chunk-7QONURHW.mjs';
5
+ import { createDurableDirectory, getFs } from './chunk-H75XXPMY.mjs';
6
6
  import { spawnNodeWorker } from './chunk-C7SLA6CM.mjs';
7
- import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, readListedFile, adapterCheckpointStore, snapshotBundleKey, manifestReferencedKeys, checkpointLastSeqNo, manifestFileRefs, commitCheckpointManifest, removeCheckpointGarbage, readSegmentFile, checkpointPrimaryTerm, writeCheckpointSegments, partitionContentsOf, frameRecord, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, captureWholePartition, readCommitMarker, readWalSegments, durableRecordMissing, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, MAX_SEGMENTS_PER_PARTITION, writeSegmentHeader, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, SEGMENT_HEADER_SIZE, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, LOG_RECORDS_PER_CHECKPOINT, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, buildEntry } from './chunk-Q4UTU5ZE.mjs';
8
- import { applyDeleteEntry, applyIndexEntry } from './chunk-J7SDAOTP.mjs';
7
+ import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, readListedFile, adapterCheckpointStore, snapshotBundleKey, manifestReferencedKeys, checkpointLastSeqNo, manifestFileRefs, commitCheckpointManifest, removeCheckpointGarbage, readSegmentFile, checkpointPrimaryTerm, writeCheckpointSegments, partitionContentsOf, frameRecord, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, captureWholePartition, readCommitMarker, readWalSegments, durableRecordMissing, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, MAX_SEGMENTS_PER_PARTITION, writeSegmentHeader, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, SEGMENT_HEADER_SIZE, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, LOG_RECORDS_PER_CHECKPOINT, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, buildEntry } from './chunk-VVFLEGZI.mjs';
8
+ import { applyDeleteEntry, applyIndexEntry } from './chunk-GMY7F2VA.mjs';
9
9
  import './chunk-OHP7QR33.mjs';
10
10
  import './chunk-P36MUFEJ.mjs';
11
- import { openChecksummedEnvelope, HEADER_SIZE } from './chunk-JFUE6F2Y.mjs';
11
+ import { openChecksummedEnvelope, HEADER_SIZE } from './chunk-O5WPRAVZ.mjs';
12
12
  import './chunk-R36MIGBU.mjs';
13
13
  import { resolveWorkerEntry, MAX_FILE_BYTES } from './chunk-RDJKYKMQ.mjs';
14
14
  import { buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-3LYY62M6.mjs';
@@ -386,8 +386,8 @@ interface Narsil extends IndexLifecycleOperations, DocumentWriteOperations, Syno
386
386
  * other than the ones that this release implements. It raises
387
387
  * `CONFIG_INVALID` when the archive's configuration is invalid,
388
388
  * `DOC_VALIDATION_FAILED` when the data is empty or the release 0.3 file is
389
- * invalid, `RESTORE_MEMORY_EXCEEDED` when the restored index and the one that
390
- * it replaces need more memory than is free, and `WORKER_TIMEOUT` when a
389
+ * invalid, `RESTORE_MEMORY_EXCEEDED` when the restored index needs more
390
+ * memory than the process has free, and `WORKER_TIMEOUT` when a
391
391
  * worker still holds the old index 30 seconds after the engine asks for it.
392
392
  */
393
393
  restore(indexName: string, data: ByteSource): Promise<void>;
@@ -1,2 +1,2 @@
1
- export { VERSION, clearNormalizationCache, configureNormalizationCache, createNarsil, getLanguage, getNormalizationCacheSize, getStopWords, getTokenizer, hasStopWords, hasTokenizer, registerLanguage, registerPartialWordSearch, registerPatternSearch, registerStopWords, registerSynonyms, registerTokenizer, resetNormalizationCache } from './browser/chunk-D3CC2NMZ.mjs';
1
+ export { VERSION, clearNormalizationCache, configureNormalizationCache, createNarsil, getLanguage, getNormalizationCacheSize, getStopWords, getTokenizer, hasStopWords, hasTokenizer, registerLanguage, registerPartialWordSearch, registerPatternSearch, registerStopWords, registerSynonyms, registerTokenizer, resetNormalizationCache } from './browser/chunk-KPMIHLOV.mjs';
2
2
  export { ClientErrorCodes, ErrorCodes, NarsilError, ServerErrorCodes, isNarsilError, isSimdAvailable } from './browser/chunk-HARBSUWP.mjs';
package/dist/index.d.ts CHANGED
@@ -9,8 +9,8 @@ export { T as TokenizerConfig } from './language-DB8KG1z5.js';
9
9
  export { A as ArrayFilter, B as BatchResult, C as ClientErrorCode, a as ClientErrorCodes, n as ComparisonFilter, D as DocumentImportResult, o as DocumentProjection, E as ErrorCode, b as ErrorCodes, F as FacetConfig, p as FacetResult, q as FieldFilter, r as FilterExpression, G as GeoFilter, s as GeoPolygonFilter, t as GeoRadiusFilter, u as GroupConfig, v as GroupReducer, w as GroupResult, H as HighlightConfig, x as HighlightMatch, y as Hit, z as HybridConfig, I as IndexInfo, g as IndexStats, L as ListParams, h as ListResult, J as ListedDocument, M as MemoryStats, N as NarsilError, c as NarsilErrorCode, f as NarsilEventHandlers, m as NarsilEventMap, P as PartitionStatsResult, j as PreflightResult, K as PresenceFilter, O as ProcessMemoryReport, R as QueryCoverage, Q as QueryParams, e as QueryResult, T as ScoreComponents, U as SearchMode, S as ServerErrorCode, d as ServerErrorCodes, W as SortField, X as SortSpec, Y as StringFilter, k as SuggestParams, l as SuggestResult, Z as TermMatchPolicy, V as VectorMaintenanceResult, _ as VectorQueryConfig, $ as WorkerCopyReport, i as isNarsilError } from './memory-COdoGthn.js';
10
10
  import { N as NarsilConfig } from './config-DPphZfuM.js';
11
11
  export { A as AnalysisConfig, D as DurabilityConfig, I as IndexContext, a as IndexLifecycleConfig, b as InsertContext, M as MainCopyQueries, c as NarsilPlugin, P as PartitionContext, R as RemoveContext, S as SearchContext, d as StaleAnalysis, U as UpdateContext, W as WorkerConfig, e as WorkerContext } from './config-DPphZfuM.js';
12
- import { N as Narsil } from './engine-BGt9lI1V.js';
13
- export { B as ByteSource, D as DocumentWriteOperations, I as IndexLifecycleOperations, S as SynonymOperations } from './engine-BGt9lI1V.js';
12
+ import { N as Narsil } from './engine-CbZmXUp6.js';
13
+ export { B as ByteSource, D as DocumentWriteOperations, I as IndexLifecycleOperations, S as SynonymOperations } from './engine-CbZmXUp6.js';
14
14
  export { E as EmbeddingAdapter, I as InvalidationAdapter, a as InvalidationEvent, b as PartitionStatistics, P as PersistenceAdapter } from './adapters-79OIa4xm.js';
15
15
 
16
16
  /**
package/dist/index.mjs CHANGED
@@ -1,4 +1,4 @@
1
- export { VERSION, createNarsil } from './chunk-JFUE6F2Y.mjs';
1
+ export { VERSION, createNarsil } from './chunk-O5WPRAVZ.mjs';
2
2
  import './chunk-R36MIGBU.mjs';
3
3
  import './chunk-RDJKYKMQ.mjs';
4
4
  export { registerSynonyms } from './chunk-3LYY62M6.mjs';
@@ -1,10 +1,10 @@
1
- import { createDurableDirectory } from '../../chunk-7QONURHW.mjs';
1
+ import { createDurableDirectory } from '../../chunk-H75XXPMY.mjs';
2
2
  import '../../chunk-C7SLA6CM.mjs';
3
- import { CHECKPOINT_WORKER_HEARTBEAT_MS, writeCheckpointSegments, directoryCheckpointStore } from '../../chunk-Q4UTU5ZE.mjs';
4
- import '../../chunk-J7SDAOTP.mjs';
3
+ import { CHECKPOINT_WORKER_HEARTBEAT_MS, writeCheckpointSegments, directoryCheckpointStore } from '../../chunk-VVFLEGZI.mjs';
4
+ import '../../chunk-GMY7F2VA.mjs';
5
5
  import '../../chunk-OHP7QR33.mjs';
6
6
  import '../../chunk-P36MUFEJ.mjs';
7
- import '../../chunk-JFUE6F2Y.mjs';
7
+ import '../../chunk-O5WPRAVZ.mjs';
8
8
  import '../../chunk-R36MIGBU.mjs';
9
9
  import '../../chunk-RDJKYKMQ.mjs';
10
10
  import '../../chunk-3LYY62M6.mjs';
package/dist/react.d.ts CHANGED
@@ -3,7 +3,7 @@ export { C as ClientErrorCode, a as ClientErrorCodes, E as ErrorCode, b as Error
3
3
  import { e as TaskRecord, d as TaskProgress, b as ImportResult, c as TaskListQuery, T as TaskListPage } from './tasks-BJmamiC4.js';
4
4
  export { I as ImportError, f as TaskStatus, g as TaskType } from './tasks-BJmamiC4.js';
5
5
  import { ReactNode, ReactElement } from 'react';
6
- import { N as Narsil } from './engine-BGt9lI1V.js';
6
+ import { N as Narsil } from './engine-CbZmXUp6.js';
7
7
  import { N as NarsilClient, I as ImportSource } from './client-DKwterMb.js';
8
8
  import { A as AnyDocument } from './schema-DttzM-ln.js';
9
9
  import './synonyms-CpOAHPa-.js';
@@ -18,7 +18,7 @@ function restoreDepsOf(core, host) {
18
18
  function queueRestore(core, host, restoresInFlight, indexName, data) {
19
19
  const previous = restoresInFlight.get(indexName) ?? Promise.resolve();
20
20
  const restored = previous.then(async () => {
21
- const { restoreFromSnapshot } = await import('./snapshot-YC6TS4OP.mjs');
21
+ const { restoreFromSnapshot } = await import('./snapshot-XII3ANGW.mjs');
22
22
  await restoreFromSnapshot(indexName, data, restoreDepsOf(core, host));
23
23
  });
24
24
  const settled = restored.then(
package/dist/server.d.ts CHANGED
@@ -1,7 +1,7 @@
1
1
  export { C as ClientErrorCode, a as ClientErrorCodes, E as ErrorCode, b as ErrorCodes, N as NarsilError, c as NarsilErrorCode, S as ServerErrorCode, d as ServerErrorCodes, i as isNarsilError } from './memory-COdoGthn.js';
2
2
  import { c as ServerOptions, N as NarsilServer } from './capabilities-BvxvFzVA.js';
3
3
  export { A as ASYNC_IMPORT_CAPABILITY, C as CorsOptions, D as DOCUMENT_EXPORT_CAPABILITY, E as ErrorEnvelope, I as INDEX_LIFECYCLE_CAPABILITY, O as OnRequestHook, R as REBUILD_ANALYSIS_CAPABILITY, d as RequestContext, e as RequestDenial, S as SERVER_CAPABILITIES, a as SYNONYMS_CAPABILITY, f as ServerLimits, T as TASK_CANCEL_CAPABILITY, b as TASK_FILTER_CAPABILITY } from './capabilities-BvxvFzVA.js';
4
- import { N as Narsil } from './engine-BGt9lI1V.js';
4
+ import { N as Narsil } from './engine-CbZmXUp6.js';
5
5
  import { d as TaskProgress, b as ImportResult, h as TaskStore, e as TaskRecord } from './tasks-BJmamiC4.js';
6
6
  export { C as CapabilitiesResponse, a as CreateIndexEmbedding, i as CreateIndexRequest, H as HttpIndexConfig, I as ImportError, M as MemoryStatsResponse, T as TaskListPage, c as TaskListQuery, f as TaskStatus, g as TaskType } from './tasks-BJmamiC4.js';
7
7
  import './schema-DttzM-ln.js';
@@ -1,16 +1,16 @@
1
- import { unpackIndexSnapshotEnvelope, loadSegmentedCheckpoint, writeMetadataEnvelope, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-54HYC5FT.mjs';
2
- import { buildArchiveFromMemory, archiveStream, memoryCheckpointStore, readSnapshotArchive } from './chunk-L26LEDHO.mjs';
1
+ import { unpackIndexSnapshotEnvelope, loadSegmentedCheckpoint, writeMetadataEnvelope, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-2BY6K52M.mjs';
2
+ import { buildArchiveFromMemory, archiveStream, memoryCheckpointStore, readSnapshotArchive } from './chunk-LJBJQPB5.mjs';
3
3
  import './chunk-MWT5WCJ5.mjs';
4
- import { indexMetadataOf, readVectorFile, readGraphParts, SINGLE_NODE_PRIMARY_TERM } from './chunk-MASE2LTJ.mjs';
4
+ import { indexMetadataOf, readVectorFile, readGraphParts, SINGLE_NODE_PRIMARY_TERM } from './chunk-TCYUAUHF.mjs';
5
5
  import { decodeVectorIndexParts, bytesToVectors } from './chunk-F52QGNZ2.mjs';
6
6
  import './chunk-ICL2AELD.mjs';
7
7
  import './chunk-37CD3OKS.mjs';
8
8
  import './chunk-DJHHF2L5.mjs';
9
- import { manifestKey, encodeSegmentManifest, manifestFileRefs, readSegmentFile } from './chunk-Q4UTU5ZE.mjs';
10
- import './chunk-J7SDAOTP.mjs';
9
+ import { manifestKey, encodeSegmentManifest, manifestFileRefs, readSegmentFile } from './chunk-VVFLEGZI.mjs';
10
+ import './chunk-GMY7F2VA.mjs';
11
11
  import './chunk-OHP7QR33.mjs';
12
12
  import './chunk-P36MUFEJ.mjs';
13
- import { encodeIndexConfiguration, createByteReader, hasNrslMagic, HEADER_SIZE, readHeader, validateRequiredFieldsInSchema, validateEmbeddingConfig, decodeIndexConfiguration, prepareStoredSynonyms, insertDocumentVectors } from './chunk-JFUE6F2Y.mjs';
13
+ import { encodeIndexConfiguration, createByteReader, hasNrslMagic, HEADER_SIZE, readHeader, validateRequiredFieldsInSchema, validateEmbeddingConfig, decodeIndexConfiguration, prepareStoredSynonyms, insertDocumentVectors } from './chunk-O5WPRAVZ.mjs';
14
14
  import { validatePatternValueLimit, requireSurfaceFormsForPartialFields } from './chunk-R36MIGBU.mjs';
15
15
  import { MAX_FILE_BYTES } from './chunk-RDJKYKMQ.mjs';
16
16
  import { readHeapStatistics, readResidentMemory, decodeEveryStoredDocument } from './chunk-3LYY62M6.mjs';
@@ -38,6 +38,7 @@ import './chunk-7X5JST5L.mjs';
38
38
  import './chunk-7KC56574.mjs';
39
39
  import './chunk-R6A773WK.mjs';
40
40
  import './chunk-YTJCGKM4.mjs';
41
+ import { collectAndReturnPages } from './chunk-5XNIRLFC.mjs';
41
42
  import './chunk-R6PR4WYZ.mjs';
42
43
  import { loadVectorCodeFor } from './chunk-HBVLF3HA.mjs';
43
44
  import './chunk-L5ORED7V.mjs';
@@ -84,43 +85,84 @@ async function validateStagedCheckpoint(store, manifest) {
84
85
  }
85
86
 
86
87
  // src/engine/snapshot/constants.ts
87
- var RESTORE_HEAP_BYTES_PER_STORED_BYTE = 0.25;
88
- var RESTORE_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 9.82;
89
- var RESTORE_HEAP_BYTES_PER_VECTOR = 330;
90
- var RESTORE_HEAP_BYTES_PER_VECTOR_DIMENSION = 0.3;
91
- var RESTORE_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 923;
92
- var RESTORE_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 0.25;
93
- var RESTORE_EXTERNAL_BYTES_PER_STORED_BYTE = 5.07;
94
- var RESTORE_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 18.12;
95
- var RESTORE_EXTERNAL_BYTES_PER_VECTOR = 1853;
96
- var RESTORE_EXTERNAL_BYTES_PER_VECTOR_DIMENSION = 11.62;
97
- var RESTORE_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 2676;
98
- var RESTORE_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 30.19;
88
+ var RESTORE_WORST_HEAP_BYTES_PER_STORED_BYTE = 0.2;
89
+ var RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 7.86;
90
+ var RESTORE_WORST_HEAP_BYTES_PER_VECTOR = 264;
91
+ var RESTORE_WORST_HEAP_BYTES_PER_VECTOR_DIMENSION = 0.24;
92
+ var RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 738;
93
+ var RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 0.2;
94
+ var RESTORE_WORST_EXTERNAL_BYTES_PER_STORED_BYTE = 4.06;
95
+ var RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 14.5;
96
+ var RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR = 1482;
97
+ var RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR_DIMENSION = 9.29;
98
+ var RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 2141;
99
+ var RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 24.15;
100
+ var RESTORE_FLOOR_HEAP_BYTES_PER_STORED_BYTE = 0.16;
101
+ var RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 6.16;
102
+ var RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR = 204;
103
+ var RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR_DIMENSION = 0.17;
104
+ var RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 695;
105
+ var RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 0.04;
106
+ var RESTORE_FLOOR_EXTERNAL_BYTES_PER_STORED_BYTE = 2.14;
107
+ var RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 5.05;
108
+ var RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR = 1080;
109
+ var RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR_DIMENSION = 0.72;
110
+ var RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 1744;
111
+ var RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 20.05;
112
+ var RESTORE_ESTIMATE_MARGIN = 1.25;
99
113
 
100
114
  // src/engine/snapshot/memory-budget.ts
101
115
  var MEGABYTE_BYTES = 1048576;
102
116
  var ARCHIVE_RATES = {
103
- heap: {
104
- perStoredByte: RESTORE_HEAP_BYTES_PER_STORED_BYTE,
105
- perVector: RESTORE_HEAP_BYTES_PER_VECTOR,
106
- perVectorDimension: RESTORE_HEAP_BYTES_PER_VECTOR_DIMENSION
117
+ worst: {
118
+ heap: {
119
+ perStoredByte: RESTORE_WORST_HEAP_BYTES_PER_STORED_BYTE,
120
+ perVector: RESTORE_WORST_HEAP_BYTES_PER_VECTOR,
121
+ perVectorDimension: RESTORE_WORST_HEAP_BYTES_PER_VECTOR_DIMENSION
122
+ },
123
+ external: {
124
+ perStoredByte: RESTORE_WORST_EXTERNAL_BYTES_PER_STORED_BYTE,
125
+ perVector: RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR,
126
+ perVectorDimension: RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR_DIMENSION
127
+ }
107
128
  },
108
- external: {
109
- perStoredByte: RESTORE_EXTERNAL_BYTES_PER_STORED_BYTE,
110
- perVector: RESTORE_EXTERNAL_BYTES_PER_VECTOR,
111
- perVectorDimension: RESTORE_EXTERNAL_BYTES_PER_VECTOR_DIMENSION
129
+ floor: {
130
+ heap: {
131
+ perStoredByte: RESTORE_FLOOR_HEAP_BYTES_PER_STORED_BYTE,
132
+ perVector: RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR,
133
+ perVectorDimension: RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR_DIMENSION
134
+ },
135
+ external: {
136
+ perStoredByte: RESTORE_FLOOR_EXTERNAL_BYTES_PER_STORED_BYTE,
137
+ perVector: RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR,
138
+ perVectorDimension: RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR_DIMENSION
139
+ }
112
140
  }
113
141
  };
114
142
  var EARLIER_SNAPSHOT_RATES = {
115
- heap: {
116
- perStoredByte: RESTORE_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
117
- perVector: RESTORE_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
118
- perVectorDimension: RESTORE_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
143
+ worst: {
144
+ heap: {
145
+ perStoredByte: RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
146
+ perVector: RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
147
+ perVectorDimension: RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
148
+ },
149
+ external: {
150
+ perStoredByte: RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
151
+ perVector: RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
152
+ perVectorDimension: RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
153
+ }
119
154
  },
120
- external: {
121
- perStoredByte: RESTORE_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
122
- perVector: RESTORE_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
123
- perVectorDimension: RESTORE_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
155
+ floor: {
156
+ heap: {
157
+ perStoredByte: RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
158
+ perVector: RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
159
+ perVectorDimension: RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
160
+ },
161
+ external: {
162
+ perStoredByte: RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
163
+ perVector: RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
164
+ perVectorDimension: RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
165
+ }
124
166
  }
125
167
  };
126
168
  function bytesAt(rate, storedBytes, vectors) {
@@ -128,10 +170,12 @@ function bytesAt(rate, storedBytes, vectors) {
128
170
  for (const { dimension, count } of vectors) bytes += count * (rate.perVector + dimension * rate.perVectorDimension);
129
171
  return bytes;
130
172
  }
131
- function footprintAt(rates, storedBytes, vectors) {
173
+ function footprintAt(rates, replacesIndex, storedBytes, vectors) {
174
+ const chosen = replacesIndex ? rates.worst : rates.floor;
175
+ const scale = replacesIndex ? RESTORE_ESTIMATE_MARGIN : 1 / RESTORE_ESTIMATE_MARGIN;
132
176
  return {
133
- heapBytes: bytesAt(rates.heap, storedBytes, vectors),
134
- externalBytes: bytesAt(rates.external, storedBytes, vectors)
177
+ heapBytes: bytesAt(chosen.heap, storedBytes, vectors) * scale,
178
+ externalBytes: bytesAt(chosen.external, storedBytes, vectors) * scale
135
179
  };
136
180
  }
137
181
  function deadCountOf(dead) {
@@ -142,7 +186,7 @@ function deadCountOf(dead) {
142
186
  }
143
187
  return count;
144
188
  }
145
- function archiveFootprint(prepared, manifest) {
189
+ function archiveFootprint(prepared, manifest, replacesIndex) {
146
190
  let segmentBytes = 0;
147
191
  for (const partition of manifest.partitions) {
148
192
  for (const segment of partition.segments) segmentBytes += segment.bytes;
@@ -153,36 +197,46 @@ function archiveFootprint(prepared, manifest) {
153
197
  const dimension = dimensions.get(field.fieldPath) ?? 0;
154
198
  for (const file of field.files) vectors.push({ dimension, count: Math.max(0, file.count - deadCountOf(file.dead)) });
155
199
  }
156
- return footprintAt(ARCHIVE_RATES, segmentBytes, vectors);
200
+ return footprintAt(ARCHIVE_RATES, replacesIndex, segmentBytes, vectors);
157
201
  }
158
- function earlierSnapshotFootprint(prepared, partitionBytes, vectorCounts) {
202
+ function earlierSnapshotFootprint(prepared, partitionBytes, vectorCounts, replacesIndex) {
159
203
  const dimensions = extractVectorFieldsFromSchema(prepared.config.schema);
160
204
  const vectors = [...vectorCounts].map(([fieldPath, count]) => ({ dimension: dimensions.get(fieldPath) ?? 0, count }));
161
- return footprintAt(EARLIER_SNAPSHOT_RATES, partitionBytes, vectors);
205
+ return footprintAt(EARLIER_SNAPSHOT_RATES, replacesIndex, partitionBytes, vectors);
162
206
  }
163
207
  function inMegabytes(bytes) {
164
208
  return Math.ceil(bytes / MEGABYTE_BYTES);
165
209
  }
166
- function refuse(indexName, budget, neededBytes, availableBytes) {
167
- const where = budget === "heap" ? "of JavaScript heap" : "of process memory";
168
- throw new NarsilError(
169
- ErrorCodes.RESTORE_MEMORY_EXCEEDED,
170
- `The restore of index "${indexName}" needs about ${inMegabytes(neededBytes)} MB ${where}, and this process has ${inMegabytes(Math.max(0, availableBytes))} MB free, so the index keeps its earlier contents. Drop the index first to free the memory that it holds, or give the process more memory, then restore again`,
171
- { indexName, budget, neededBytes, availableBytes }
172
- );
173
- }
174
- function requireRestoreMemory(indexName, footprint) {
210
+ function shortfallOf(heapBytes, processBytes, countInUse) {
175
211
  const heap = readHeapStatistics();
176
212
  if (heap !== null) {
177
- const availableBytes = (heap.configuredLimitBytes ?? heap.limitBytes) - heap.usedBytes;
178
- if (footprint.heapBytes > availableBytes) refuse(indexName, "heap", footprint.heapBytes, availableBytes);
213
+ const limitBytes = heap.configuredLimitBytes ?? heap.limitBytes;
214
+ const availableBytes = countInUse ? limitBytes - heap.usedBytes : limitBytes;
215
+ if (heapBytes > availableBytes) return { budget: "heap", neededBytes: heapBytes, availableBytes };
179
216
  }
180
217
  const resident = readResidentMemory();
181
218
  if (resident !== null) {
182
- const neededBytes = footprint.heapBytes + footprint.externalBytes;
183
- const availableBytes = resident.limitBytes - resident.residentBytes;
184
- if (neededBytes > availableBytes) refuse(indexName, "process", neededBytes, availableBytes);
219
+ const availableBytes = countInUse ? resident.limitBytes - resident.residentBytes : resident.limitBytes;
220
+ if (processBytes > availableBytes) return { budget: "process", neededBytes: processBytes, availableBytes };
185
221
  }
222
+ return null;
223
+ }
224
+ function refuse(indexName, replacesIndex, shortfall) {
225
+ const { budget, neededBytes, availableBytes } = shortfall;
226
+ const where = budget === "heap" ? "of JavaScript heap" : "of process memory";
227
+ const outcome = replacesIndex ? "so the index keeps its earlier contents. Drop the index first to free the memory that it holds, or give the process more memory, then restore again" : "so the engine creates no index. Give the process more memory, then restore again";
228
+ throw new NarsilError(
229
+ ErrorCodes.RESTORE_MEMORY_EXCEEDED,
230
+ `The restore of index "${indexName}" needs about ${inMegabytes(neededBytes)} MB ${where}, and this process has ${inMegabytes(Math.max(0, availableBytes))} MB free, ${outcome}`,
231
+ { indexName, budget, neededBytes, availableBytes }
232
+ );
233
+ }
234
+ function requireRestoreMemory(indexName, footprint, replacesIndex) {
235
+ const processBytes = footprint.heapBytes + footprint.externalBytes;
236
+ if (shortfallOf(footprint.heapBytes, processBytes, true) === null) return;
237
+ const collected = collectAndReturnPages();
238
+ const shortfall = shortfallOf(footprint.heapBytes, processBytes, collected || replacesIndex);
239
+ if (shortfall !== null) refuse(indexName, replacesIndex, shortfall);
186
240
  }
187
241
 
188
242
  // src/workers/detached-index.ts
@@ -483,9 +537,8 @@ async function stageArchive(indexName, reader, staging, deps) {
483
537
  manifest: async (manifest2, sourceName) => {
484
538
  if (staged.prepared === null) missingPart("configuration");
485
539
  requireSameSchema(staged.prepared, manifest2);
486
- if (deps.indexRegistry.has(indexName)) {
487
- requireRestoreMemory(indexName, archiveFootprint(staged.prepared, manifest2));
488
- }
540
+ const replacesIndex = deps.indexRegistry.has(indexName);
541
+ requireRestoreMemory(indexName, archiveFootprint(staged.prepared, manifest2, replacesIndex), replacesIndex);
489
542
  staged.manifest = renamed(manifest2, sourceName, indexName);
490
543
  },
491
544
  file: async (entryName, _ref, bytes) => {
@@ -575,7 +628,7 @@ function vectorsOfSnapshot(envelope) {
575
628
  }
576
629
  return byDocument;
577
630
  }
578
- function footprintOf(envelope, prepared) {
631
+ function footprintOf(envelope, prepared, replacesIndex) {
579
632
  let partitionBytes = 0;
580
633
  for (const payload of envelope.partitions ?? []) partitionBytes += payload.byteLength;
581
634
  const vectorCounts = /* @__PURE__ */ new Map();
@@ -584,7 +637,7 @@ function footprintOf(envelope, prepared) {
584
637
  for (const part of decodeVectorIndexParts(rawParts)) count += part.docIds.length;
585
638
  vectorCounts.set(fieldPath, count);
586
639
  }
587
- return earlierSnapshotFootprint(prepared, partitionBytes, vectorCounts);
640
+ return earlierSnapshotFootprint(prepared, partitionBytes, vectorCounts, replacesIndex);
588
641
  }
589
642
  function documentsOfPartition(envelope, payload) {
590
643
  return envelope.version === 2 ? documentsOfPayloadV2(payload).documents : deserializePayloadV1(payload).documents;
@@ -644,7 +697,8 @@ async function buildRestoredIndex(indexName, envelope, prepared, staging, deps)
644
697
  async function restoreEarlierSnapshot(indexName, data, deps) {
645
698
  const envelope = await decodeEarlierSnapshot(data);
646
699
  const prepared = prepareRestoredIndex(envelope, deps);
647
- if (deps.indexRegistry.has(indexName)) requireRestoreMemory(indexName, footprintOf(envelope, prepared));
700
+ const replacesIndex = deps.indexRegistry.has(indexName);
701
+ requireRestoreMemory(indexName, footprintOf(envelope, prepared, replacesIndex), replacesIndex);
648
702
  const analysedNow = { ...prepared, storedRevision: prepared.language.revision };
649
703
  const staging = deps.durability?.manager.stageRestore?.(indexName) ?? null;
650
704
  try {
@@ -1,7 +1,7 @@
1
1
  import { decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-F52QGNZ2.mjs';
2
2
  import './chunk-37CD3OKS.mjs';
3
- import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-J7SDAOTP.mjs';
4
- import { openChecksummedEnvelope, NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-JFUE6F2Y.mjs';
3
+ import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-GMY7F2VA.mjs';
4
+ import { openChecksummedEnvelope, NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-O5WPRAVZ.mjs';
5
5
  import './chunk-R36MIGBU.mjs';
6
6
  import './chunk-RDJKYKMQ.mjs';
7
7
  import { decodeStoredDocument } from './chunk-3LYY62M6.mjs';
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@delali/narsil",
3
- "version": "0.3.1-next.95",
3
+ "version": "0.3.1-next.96",
4
4
  "type": "module",
5
5
  "description": "Distributed search engine with full-text, vector, hybrid, and geosearch. One engine, embedded or on a server, with a cross-language .nrsl format.",
6
6
  "author": "Delali (https://sondelali.com)",
@@ -263,11 +263,11 @@
263
263
  }
264
264
  },
265
265
  "optionalDependencies": {
266
- "@delali/narsil-native-darwin-arm64": "0.3.1-next.95",
267
- "@delali/narsil-native-darwin-x64": "0.3.1-next.95",
268
- "@delali/narsil-native-linux-arm64": "0.3.1-next.95",
269
- "@delali/narsil-native-linux-x64": "0.3.1-next.95",
270
- "@delali/narsil-native-win32-arm64": "0.3.1-next.95",
271
- "@delali/narsil-native-win32-x64": "0.3.1-next.95"
266
+ "@delali/narsil-native-darwin-arm64": "0.3.1-next.96",
267
+ "@delali/narsil-native-darwin-x64": "0.3.1-next.96",
268
+ "@delali/narsil-native-linux-arm64": "0.3.1-next.96",
269
+ "@delali/narsil-native-linux-x64": "0.3.1-next.96",
270
+ "@delali/narsil-native-win32-arm64": "0.3.1-next.96",
271
+ "@delali/narsil-native-win32-x64": "0.3.1-next.96"
272
272
  }
273
273
  }