@delali/narsil 0.3.1-next.94 → 0.3.1-next.96
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/{chunk-4JTBL63P.mjs → chunk-3MYKLHRO.mjs} +2 -2
- package/dist/browser/{chunk-5S74HQRT.mjs → chunk-FYFQJKFW.mjs} +1 -1
- package/dist/browser/{chunk-HC6DMR4T.mjs → chunk-KPMIHLOV.mjs} +6 -6
- package/dist/browser/{chunk-PP6N5CCS.mjs → chunk-ZGMR46O3.mjs} +1 -1
- package/dist/browser/{durability-wiring-7DKYHKT5.mjs → durability-wiring-R7P64REL.mjs} +3 -3
- package/dist/browser/{module-O5WH32EL.mjs → module-IEOQI5GI.mjs} +2 -2
- package/dist/browser/{restore-queue-Z7FZZNKH.mjs → restore-queue-WMYC5ZK6.mjs} +1 -1
- package/dist/browser/{snapshot-PPTZ565G.mjs → snapshot-3A3WDNHO.mjs} +111 -59
- package/dist/browser/{write-2ZAA7UDH.mjs → write-JODZHKIF.mjs} +2 -2
- package/dist/{capabilities-xsJfdFoQ.d.ts → capabilities-BvxvFzVA.d.ts} +2 -2
- package/dist/{chunk-QWWCLRXX.mjs → chunk-2BY6K52M.mjs} +3 -3
- package/dist/{chunk-UAUZGXJP.mjs → chunk-3LYY62M6.mjs} +1 -1
- package/dist/{chunk-EZD4MWOU.mjs → chunk-GMY7F2VA.mjs} +1 -1
- package/dist/{chunk-EWBXO45L.mjs → chunk-H75XXPMY.mjs} +1 -1
- package/dist/{chunk-K6A4FCVR.mjs → chunk-HBVLF3HA.mjs} +1 -1
- package/dist/{chunk-Z3MOCHG5.mjs → chunk-ICL2AELD.mjs} +1 -1
- package/dist/{chunk-J53IHLG7.mjs → chunk-LJBJQPB5.mjs} +4 -4
- package/dist/{chunk-7VALMHZF.mjs → chunk-MWT5WCJ5.mjs} +1 -1
- package/dist/{chunk-C45FUYPI.mjs → chunk-O5WPRAVZ.mjs} +7 -7
- package/dist/{chunk-AF2CIKYC.mjs → chunk-T2QN2FA4.mjs} +1 -1
- package/dist/{chunk-WBS3EYC4.mjs → chunk-TCYUAUHF.mjs} +3 -3
- package/dist/{chunk-EX62BQT6.mjs → chunk-VVFLEGZI.mjs} +3 -3
- package/dist/{client-CgayPw2R.d.ts → client-DKwterMb.d.ts} +2 -2
- package/dist/client.d.ts +6 -6
- package/dist/{config-3kBsQ0Pi.d.ts → config-6YBtxekG.d.ts} +1 -1
- package/dist/{config-B-RnO6dr.d.ts → config-DPphZfuM.d.ts} +1 -1
- package/dist/{direct-executor-IJTRU3YN.mjs → direct-executor-5KJGVQHM.mjs} +2 -2
- package/dist/distribution.d.ts +5 -5
- package/dist/distribution.mjs +9 -9
- package/dist/{durability-wiring-Y55SCPFC.mjs → durability-wiring-2VBNYYDY.mjs} +8 -8
- package/dist/{engine-DSUfY2F0.d.ts → engine-CbZmXUp6.d.ts} +3 -3
- package/dist/index.browser.mjs +1 -1
- package/dist/index.d.ts +5 -5
- package/dist/index.mjs +3 -3
- package/dist/{memory-wuXcYOD1.d.ts → memory-COdoGthn.d.ts} +1 -1
- package/dist/{module-VITS4ZR5.mjs → module-IUTNXWMV.mjs} +5 -5
- package/dist/persistence/durability/checkpoint-worker.mjs +6 -6
- package/dist/react.d.ts +6 -6
- package/dist/{restore-queue-4SG45BLF.mjs → restore-queue-UFIQJTSG.mjs} +1 -1
- package/dist/server.d.ts +8 -8
- package/dist/{snapshot-XBKE7PUG.mjs → snapshot-XII3ANGW.mjs} +119 -65
- package/dist/{tasks-Bj9WBlFG.d.ts → tasks-BJmamiC4.d.ts} +1 -1
- package/dist/vector/search-worker.mjs +2 -2
- package/dist/workers/entry.mjs +2 -2
- package/dist/{write-BMNV7P2H.mjs → write-6M3VG4TU.mjs} +4 -4
- package/package.json +7 -7
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { N as NarsilConfig } from './config-
|
|
1
|
+
import { N as NarsilConfig } from './config-DPphZfuM.js';
|
|
2
2
|
import { A as AllocationTable, N as NodeRole, a as NodeRegistration, C as ClusterCoordinator, b as NodeCapacity } from './index-ONAEaqtx.js';
|
|
3
3
|
import { N as NodeTransport } from './types-Cb9kiY5P.js';
|
|
4
4
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { P as PersistenceAdapter, I as InvalidationAdapter, E as EmbeddingAdapter } from './adapters-79OIa4xm.js';
|
|
2
|
-
import { Q as QueryParams, e as QueryResult, f as NarsilEventHandlers } from './memory-
|
|
2
|
+
import { Q as QueryParams, e as QueryResult, f as NarsilEventHandlers } from './memory-COdoGthn.js';
|
|
3
3
|
import { I as IndexConfig, A as AnyDocument } from './schema-DttzM-ln.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export { createDirectExecutor } from './chunk-
|
|
1
|
+
export { createDirectExecutor } from './chunk-3LYY62M6.mjs';
|
|
2
2
|
import './chunk-CHOGQSHK.mjs';
|
|
3
3
|
import './chunk-ZHS44YKF.mjs';
|
|
4
4
|
import './chunk-TGP5M5R7.mjs';
|
|
@@ -17,6 +17,6 @@ import './chunk-7X5JST5L.mjs';
|
|
|
17
17
|
import './chunk-7KC56574.mjs';
|
|
18
18
|
import './chunk-R6A773WK.mjs';
|
|
19
19
|
import './chunk-YTJCGKM4.mjs';
|
|
20
|
-
import './chunk-
|
|
20
|
+
import './chunk-HBVLF3HA.mjs';
|
|
21
21
|
import './chunk-L5ORED7V.mjs';
|
|
22
22
|
import './chunk-SYWGORQB.mjs';
|
package/dist/distribution.d.ts
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
import { a as CreateIndexOptions, C as ClusterNamespace, b as ClusterNodeConfig } from './config-
|
|
2
|
-
export { c as ClusterControllerConfig, d as ClusterMemoryConfig, e as ClusterNodeInfo, f as ClusterQueryConfig, N as NodeReadiness, R as ReplicationConfig } from './config-
|
|
3
|
-
import { B as BatchResult, L as ListParams, h as ListResult, k as SuggestParams, l as SuggestResult, Q as QueryParams, j as PreflightResult, g as IndexStats, P as PartitionStatsResult, M as MemoryStats, m as NarsilEventMap, e as QueryResult } from './memory-
|
|
1
|
+
import { a as CreateIndexOptions, C as ClusterNamespace, b as ClusterNodeConfig } from './config-6YBtxekG.js';
|
|
2
|
+
export { c as ClusterControllerConfig, d as ClusterMemoryConfig, e as ClusterNodeInfo, f as ClusterQueryConfig, N as NodeReadiness, R as ReplicationConfig } from './config-6YBtxekG.js';
|
|
3
|
+
import { B as BatchResult, L as ListParams, h as ListResult, k as SuggestParams, l as SuggestResult, Q as QueryParams, j as PreflightResult, g as IndexStats, P as PartitionStatsResult, M as MemoryStats, m as NarsilEventMap, e as QueryResult } from './memory-COdoGthn.js';
|
|
4
4
|
import { I as IndexConfig, A as AnyDocument, a as InsertOptions } from './schema-DttzM-ln.js';
|
|
5
5
|
import { a as SynonymRule } from './synonyms-CpOAHPa-.js';
|
|
6
6
|
import { b as NodeCapacity, N as NodeRole } from './index-ONAEaqtx.js';
|
|
7
7
|
export { c as AllocationConstraints, d as AllocationEvent, A as AllocationTable, C as ClusterCoordinator, I as IndexPartitionBytes, e as NodeEvent, f as NodeMemoryReport, a as NodeRegistration, P as PartitionAssignment, g as PartitionState, S as SchemaEvent, U as UnassignedReason } from './index-ONAEaqtx.js';
|
|
8
|
-
import { N as Narsil } from './engine-
|
|
8
|
+
import { N as Narsil } from './engine-CbZmXUp6.js';
|
|
9
9
|
export { L as ListenHandler, N as NodeTransport, R as RespondFn, T as TransportConfig, a as TransportMessage } from './types-Cb9kiY5P.js';
|
|
10
|
-
import './config-
|
|
10
|
+
import './config-DPphZfuM.js';
|
|
11
11
|
import './adapters-79OIa4xm.js';
|
|
12
12
|
|
|
13
13
|
/**
|
package/dist/distribution.mjs
CHANGED
|
@@ -1,20 +1,20 @@
|
|
|
1
|
-
import { buildArchiveFromMemory, archiveStream, memoryCheckpointStore, readSnapshotArchive } from './chunk-
|
|
2
|
-
import './chunk-
|
|
3
|
-
import { indexMetadataOf, loadPartitionSegments } from './chunk-
|
|
1
|
+
import { buildArchiveFromMemory, archiveStream, memoryCheckpointStore, readSnapshotArchive } from './chunk-LJBJQPB5.mjs';
|
|
2
|
+
import './chunk-MWT5WCJ5.mjs';
|
|
3
|
+
import { indexMetadataOf, loadPartitionSegments } from './chunk-TCYUAUHF.mjs';
|
|
4
4
|
import { decodeVectorFilePayload, bytesToVectors, isDead } from './chunk-F52QGNZ2.mjs';
|
|
5
5
|
import { MAX_MESSAGE_SIZE_BYTES } from './chunk-UNEEFTEU.mjs';
|
|
6
6
|
import { DEFAULT_NODE_LIFECYCLE_CONFIG, DEFAULT_CONTROLLER_CONFIG, validateIndexName, readIndexState, readIndexMetadataSnapshot, storeSynonymSet, deleteSynonymSet, indexConfigKey, indexUuidOfStoredMetadata, getIndexMetadata, withPartitionCount, indexSettingsOf, putIndexMetadata, indexConfigFromSettings, metadataWithSynonymSet, isRecord, storedSynonymSetOf, ALLOCATION_RETRY_DELAY_MS, TEARDOWN_CAS_ATTEMPTS, TEARDOWN_RETRY_ROUNDS, isValidInteger, ALLOCATION_CAS_ATTEMPTS, TEARDOWN_RETRY_DELAY_MS, RECOVERY_CAS_ATTEMPTS, ESTIMATED_PARTITION_BYTES, REBALANCE_THRESHOLD, PARTITION_STORES_TIMEOUT_MS, LEADERSHIP_IMBALANCE_THRESHOLD, BOOTSTRAP_CAS_ATTEMPTS, getClusterIndexConfig } from './chunk-GJZHU4FC.mjs';
|
|
7
|
-
import './chunk-
|
|
7
|
+
import './chunk-ICL2AELD.mjs';
|
|
8
8
|
import './chunk-37CD3OKS.mjs';
|
|
9
9
|
import './chunk-DJHHF2L5.mjs';
|
|
10
|
-
import { validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, createForwardBatchResultMessage, readListedFile, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage, computeEntryChecksum, createEntryMessage, createEntryBatchMessage, createInsyncConfirmMessage, validateAckPayload, createInsyncRemoveMessage, validateInsyncConfirmPayload, createInsyncAddMessage } from './chunk-
|
|
11
|
-
import { applyIndexEntry, applyDeleteEntry, validateReplicationEntry } from './chunk-
|
|
10
|
+
import { validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, createForwardBatchResultMessage, readListedFile, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage, computeEntryChecksum, createEntryMessage, createEntryBatchMessage, createInsyncConfirmMessage, validateAckPayload, createInsyncRemoveMessage, validateInsyncConfirmPayload, createInsyncAddMessage } from './chunk-VVFLEGZI.mjs';
|
|
11
|
+
import { applyIndexEntry, applyDeleteEntry, validateReplicationEntry } from './chunk-GMY7F2VA.mjs';
|
|
12
12
|
import { TransportErrorCodes, ReplicationMessageTypes, ClusterMessageTypes, QueryMessageTypes, TransportError } from './chunk-OHP7QR33.mjs';
|
|
13
13
|
import { MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR, WIRE_BATCH_BUDGET } from './chunk-P36MUFEJ.mjs';
|
|
14
|
-
import { requireKnownConfig, resolveRunnableConfig, createEngineCore, startEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, createEngineIndex, encodeIndexConfiguration, requireConfigurationWithinSizeLimit, prepareConfiguredSynonyms, createByteReader, openChecksummedEnvelope, emptySerializablePartition, crc32Init, crc32Final, crc32Update } from './chunk-
|
|
14
|
+
import { requireKnownConfig, resolveRunnableConfig, createEngineCore, startEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, createEngineIndex, encodeIndexConfiguration, requireConfigurationWithinSizeLimit, prepareConfiguredSynonyms, createByteReader, openChecksummedEnvelope, emptySerializablePartition, crc32Init, crc32Final, crc32Update } from './chunk-O5WPRAVZ.mjs';
|
|
15
15
|
import { now, listBindingOf, requireSurfaceFormsForPartialFields, queryBindingOf, hitsKeptPerGroup, clampLimit, resolveHybridFusion, dedupePinnedEntries, foldGroupReducer } from './chunk-R36MIGBU.mjs';
|
|
16
16
|
import './chunk-RDJKYKMQ.mjs';
|
|
17
|
-
import { synonymModule, readHeapStatistics } from './chunk-
|
|
17
|
+
import { synonymModule, readHeapStatistics } from './chunk-3LYY62M6.mjs';
|
|
18
18
|
import { oversampledShardSize, resolveProjection, clampRowCount, sortSignatureOf, requireMatchingCursor, decodePageCursor, normalizeSort, sortModesOf, encodePageCursor, applyProjection, readSortValues, requireWithinResultWindow, fnv1a, sortSignatureEntry, FILTER_EXPRESSION_KEYS, filterTestCount, fnv1aUint32s, mergePartitionStats, FIELD_FILTER_OPERATORS } from './chunk-CHOGQSHK.mjs';
|
|
19
19
|
import { providedDocId } from './chunk-WLOF6HSP.mjs';
|
|
20
20
|
import './chunk-AV4IJKD2.mjs';
|
|
@@ -41,7 +41,7 @@ import { toComparableSortValue, compareComparableKeys, isSortMode, SORT_MODES, c
|
|
|
41
41
|
import { compareCodePoints, exceedsCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
42
42
|
import { collectAndReturnPages } from './chunk-5XNIRLFC.mjs';
|
|
43
43
|
import './chunk-R6PR4WYZ.mjs';
|
|
44
|
-
import './chunk-
|
|
44
|
+
import './chunk-HBVLF3HA.mjs';
|
|
45
45
|
import { MAX_STORED_VECTOR_DIMENSION, MAX_NESTING_DEPTH } from './chunk-L5ORED7V.mjs';
|
|
46
46
|
import { ErrorCodes, NarsilError, describeError } from './chunk-SYWGORQB.mjs';
|
|
47
47
|
import { decode, encode } from '@msgpack/msgpack';
|
|
@@ -1,17 +1,17 @@
|
|
|
1
|
-
import { readMetadataEnvelope, readSegmentManifest, reclaimUnlistedKeys, loadSegmentedCheckpoint, writeMetadataEnvelope, readCurrentManifest, deserializePayloadV2, unpackEnvelopeBytes, growPartitionsTo, documentsOfPayloadV2 } from './chunk-
|
|
2
|
-
import { decodeSegmentManifest, metadataFromConfig, createSeqOwner, isRecord, invalidManifest, SINGLE_NODE_PRIMARY_TERM, writeLiveVectors, isId, isUint32, isSequenceNumber, VECTOR_FILE_ENVELOPE_VERSION, metadataConfigOf } from './chunk-
|
|
1
|
+
import { readMetadataEnvelope, readSegmentManifest, reclaimUnlistedKeys, loadSegmentedCheckpoint, writeMetadataEnvelope, readCurrentManifest, deserializePayloadV2, unpackEnvelopeBytes, growPartitionsTo, documentsOfPayloadV2 } from './chunk-2BY6K52M.mjs';
|
|
2
|
+
import { decodeSegmentManifest, metadataFromConfig, createSeqOwner, isRecord, invalidManifest, SINGLE_NODE_PRIMARY_TERM, writeLiveVectors, isId, isUint32, isSequenceNumber, VECTOR_FILE_ENVELOPE_VERSION, metadataConfigOf } from './chunk-TCYUAUHF.mjs';
|
|
3
3
|
import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-F52QGNZ2.mjs';
|
|
4
4
|
import './chunk-37CD3OKS.mjs';
|
|
5
|
-
import { createDurableDirectory, getFs } from './chunk-
|
|
5
|
+
import { createDurableDirectory, getFs } from './chunk-H75XXPMY.mjs';
|
|
6
6
|
import { spawnNodeWorker } from './chunk-C7SLA6CM.mjs';
|
|
7
|
-
import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, readListedFile, adapterCheckpointStore, snapshotBundleKey, manifestReferencedKeys, checkpointLastSeqNo, manifestFileRefs, commitCheckpointManifest, removeCheckpointGarbage, readSegmentFile, checkpointPrimaryTerm, writeCheckpointSegments, partitionContentsOf, frameRecord, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, captureWholePartition, readCommitMarker, readWalSegments, durableRecordMissing, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, MAX_SEGMENTS_PER_PARTITION, writeSegmentHeader, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, SEGMENT_HEADER_SIZE, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, LOG_RECORDS_PER_CHECKPOINT, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, buildEntry } from './chunk-
|
|
8
|
-
import { applyDeleteEntry, applyIndexEntry } from './chunk-
|
|
7
|
+
import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, readListedFile, adapterCheckpointStore, snapshotBundleKey, manifestReferencedKeys, checkpointLastSeqNo, manifestFileRefs, commitCheckpointManifest, removeCheckpointGarbage, readSegmentFile, checkpointPrimaryTerm, writeCheckpointSegments, partitionContentsOf, frameRecord, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, captureWholePartition, readCommitMarker, readWalSegments, durableRecordMissing, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, MAX_SEGMENTS_PER_PARTITION, writeSegmentHeader, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, SEGMENT_HEADER_SIZE, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, LOG_RECORDS_PER_CHECKPOINT, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, buildEntry } from './chunk-VVFLEGZI.mjs';
|
|
8
|
+
import { applyDeleteEntry, applyIndexEntry } from './chunk-GMY7F2VA.mjs';
|
|
9
9
|
import './chunk-OHP7QR33.mjs';
|
|
10
10
|
import './chunk-P36MUFEJ.mjs';
|
|
11
|
-
import { openChecksummedEnvelope, HEADER_SIZE } from './chunk-
|
|
11
|
+
import { openChecksummedEnvelope, HEADER_SIZE } from './chunk-O5WPRAVZ.mjs';
|
|
12
12
|
import './chunk-R36MIGBU.mjs';
|
|
13
13
|
import { resolveWorkerEntry, MAX_FILE_BYTES } from './chunk-RDJKYKMQ.mjs';
|
|
14
|
-
import { buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-
|
|
14
|
+
import { buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-3LYY62M6.mjs';
|
|
15
15
|
import { fnv1a } from './chunk-CHOGQSHK.mjs';
|
|
16
16
|
import './chunk-WLOF6HSP.mjs';
|
|
17
17
|
import { detectRuntime } from './chunk-AV4IJKD2.mjs';
|
|
@@ -37,7 +37,7 @@ import './chunk-7KC56574.mjs';
|
|
|
37
37
|
import './chunk-R6A773WK.mjs';
|
|
38
38
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
39
39
|
import './chunk-R6PR4WYZ.mjs';
|
|
40
|
-
import './chunk-
|
|
40
|
+
import './chunk-HBVLF3HA.mjs';
|
|
41
41
|
import './chunk-L5ORED7V.mjs';
|
|
42
42
|
import { NarsilError, ErrorCodes } from './chunk-SYWGORQB.mjs';
|
|
43
43
|
import { decode } from '@msgpack/msgpack';
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { E as EmbeddingAdapter } from './adapters-79OIa4xm.js';
|
|
2
|
-
import { B as BatchResult, I as IndexInfo, g as IndexStats, P as PartitionStatsResult, L as ListParams, h as ListResult, Q as QueryParams, e as QueryResult, j as PreflightResult, k as SuggestParams, l as SuggestResult, D as DocumentImportResult, M as MemoryStats, V as VectorMaintenanceResult, m as NarsilEventMap } from './memory-
|
|
2
|
+
import { B as BatchResult, I as IndexInfo, g as IndexStats, P as PartitionStatsResult, L as ListParams, h as ListResult, Q as QueryParams, e as QueryResult, j as PreflightResult, k as SuggestParams, l as SuggestResult, D as DocumentImportResult, M as MemoryStats, V as VectorMaintenanceResult, m as NarsilEventMap } from './memory-COdoGthn.js';
|
|
3
3
|
import { A as AnyDocument, a as InsertOptions, W as WriteOptions, I as IndexConfig, P as PartitionConfig } from './schema-DttzM-ln.js';
|
|
4
4
|
import { a as SynonymRule } from './synonyms-CpOAHPa-.js';
|
|
5
5
|
|
|
@@ -386,8 +386,8 @@ interface Narsil extends IndexLifecycleOperations, DocumentWriteOperations, Syno
|
|
|
386
386
|
* other than the ones that this release implements. It raises
|
|
387
387
|
* `CONFIG_INVALID` when the archive's configuration is invalid,
|
|
388
388
|
* `DOC_VALIDATION_FAILED` when the data is empty or the release 0.3 file is
|
|
389
|
-
* invalid, `RESTORE_MEMORY_EXCEEDED` when the restored index
|
|
390
|
-
*
|
|
389
|
+
* invalid, `RESTORE_MEMORY_EXCEEDED` when the restored index needs more
|
|
390
|
+
* memory than the process has free, and `WORKER_TIMEOUT` when a
|
|
391
391
|
* worker still holds the old index 30 seconds after the engine asks for it.
|
|
392
392
|
*/
|
|
393
393
|
restore(indexName: string, data: ByteSource): Promise<void>;
|
package/dist/index.browser.mjs
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export { VERSION, clearNormalizationCache, configureNormalizationCache, createNarsil, getLanguage, getNormalizationCacheSize, getStopWords, getTokenizer, hasStopWords, hasTokenizer, registerLanguage, registerPartialWordSearch, registerPatternSearch, registerStopWords, registerSynonyms, registerTokenizer, resetNormalizationCache } from './browser/chunk-
|
|
1
|
+
export { VERSION, clearNormalizationCache, configureNormalizationCache, createNarsil, getLanguage, getNormalizationCacheSize, getStopWords, getTokenizer, hasStopWords, hasTokenizer, registerLanguage, registerPartialWordSearch, registerPatternSearch, registerStopWords, registerSynonyms, registerTokenizer, resetNormalizationCache } from './browser/chunk-KPMIHLOV.mjs';
|
|
2
2
|
export { ClientErrorCodes, ErrorCodes, NarsilError, ServerErrorCodes, isNarsilError, isSimdAvailable } from './browser/chunk-HARBSUWP.mjs';
|
package/dist/index.d.ts
CHANGED
|
@@ -6,11 +6,11 @@ import { S as SynonymSearch } from './synonyms-CpOAHPa-.js';
|
|
|
6
6
|
export { O as OneWaySynonymRule, a as SynonymRule, T as TwoWaySynonymRule } from './synonyms-CpOAHPa-.js';
|
|
7
7
|
import { L as LanguageModule } from './language-DB8KG1z5.js';
|
|
8
8
|
export { T as TokenizerConfig } from './language-DB8KG1z5.js';
|
|
9
|
-
export { A as ArrayFilter, B as BatchResult, C as ClientErrorCode, a as ClientErrorCodes, n as ComparisonFilter, D as DocumentImportResult, o as DocumentProjection, E as ErrorCode, b as ErrorCodes, F as FacetConfig, p as FacetResult, q as FieldFilter, r as FilterExpression, G as GeoFilter, s as GeoPolygonFilter, t as GeoRadiusFilter, u as GroupConfig, v as GroupReducer, w as GroupResult, H as HighlightConfig, x as HighlightMatch, y as Hit, z as HybridConfig, I as IndexInfo, g as IndexStats, L as ListParams, h as ListResult, J as ListedDocument, M as MemoryStats, N as NarsilError, c as NarsilErrorCode, f as NarsilEventHandlers, m as NarsilEventMap, P as PartitionStatsResult, j as PreflightResult, K as PresenceFilter, O as ProcessMemoryReport, R as QueryCoverage, Q as QueryParams, e as QueryResult, T as ScoreComponents, U as SearchMode, S as ServerErrorCode, d as ServerErrorCodes, W as SortField, X as SortSpec, Y as StringFilter, k as SuggestParams, l as SuggestResult, Z as TermMatchPolicy, V as VectorMaintenanceResult, _ as VectorQueryConfig, $ as WorkerCopyReport, i as isNarsilError } from './memory-
|
|
10
|
-
import { N as NarsilConfig } from './config-
|
|
11
|
-
export { A as AnalysisConfig, D as DurabilityConfig, I as IndexContext, a as IndexLifecycleConfig, b as InsertContext, M as MainCopyQueries, c as NarsilPlugin, P as PartitionContext, R as RemoveContext, S as SearchContext, d as StaleAnalysis, U as UpdateContext, W as WorkerConfig, e as WorkerContext } from './config-
|
|
12
|
-
import { N as Narsil } from './engine-
|
|
13
|
-
export { B as ByteSource, D as DocumentWriteOperations, I as IndexLifecycleOperations, S as SynonymOperations } from './engine-
|
|
9
|
+
export { A as ArrayFilter, B as BatchResult, C as ClientErrorCode, a as ClientErrorCodes, n as ComparisonFilter, D as DocumentImportResult, o as DocumentProjection, E as ErrorCode, b as ErrorCodes, F as FacetConfig, p as FacetResult, q as FieldFilter, r as FilterExpression, G as GeoFilter, s as GeoPolygonFilter, t as GeoRadiusFilter, u as GroupConfig, v as GroupReducer, w as GroupResult, H as HighlightConfig, x as HighlightMatch, y as Hit, z as HybridConfig, I as IndexInfo, g as IndexStats, L as ListParams, h as ListResult, J as ListedDocument, M as MemoryStats, N as NarsilError, c as NarsilErrorCode, f as NarsilEventHandlers, m as NarsilEventMap, P as PartitionStatsResult, j as PreflightResult, K as PresenceFilter, O as ProcessMemoryReport, R as QueryCoverage, Q as QueryParams, e as QueryResult, T as ScoreComponents, U as SearchMode, S as ServerErrorCode, d as ServerErrorCodes, W as SortField, X as SortSpec, Y as StringFilter, k as SuggestParams, l as SuggestResult, Z as TermMatchPolicy, V as VectorMaintenanceResult, _ as VectorQueryConfig, $ as WorkerCopyReport, i as isNarsilError } from './memory-COdoGthn.js';
|
|
10
|
+
import { N as NarsilConfig } from './config-DPphZfuM.js';
|
|
11
|
+
export { A as AnalysisConfig, D as DurabilityConfig, I as IndexContext, a as IndexLifecycleConfig, b as InsertContext, M as MainCopyQueries, c as NarsilPlugin, P as PartitionContext, R as RemoveContext, S as SearchContext, d as StaleAnalysis, U as UpdateContext, W as WorkerConfig, e as WorkerContext } from './config-DPphZfuM.js';
|
|
12
|
+
import { N as Narsil } from './engine-CbZmXUp6.js';
|
|
13
|
+
export { B as ByteSource, D as DocumentWriteOperations, I as IndexLifecycleOperations, S as SynonymOperations } from './engine-CbZmXUp6.js';
|
|
14
14
|
export { E as EmbeddingAdapter, I as InvalidationAdapter, a as InvalidationEvent, b as PartitionStatistics, P as PersistenceAdapter } from './adapters-79OIa4xm.js';
|
|
15
15
|
|
|
16
16
|
/**
|
package/dist/index.mjs
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
export { VERSION, createNarsil } from './chunk-
|
|
1
|
+
export { VERSION, createNarsil } from './chunk-O5WPRAVZ.mjs';
|
|
2
2
|
import './chunk-R36MIGBU.mjs';
|
|
3
3
|
import './chunk-RDJKYKMQ.mjs';
|
|
4
|
-
export { registerSynonyms } from './chunk-
|
|
4
|
+
export { registerSynonyms } from './chunk-3LYY62M6.mjs';
|
|
5
5
|
export { registerPartialWordSearch, registerPatternSearch } from './chunk-CHOGQSHK.mjs';
|
|
6
6
|
import './chunk-WLOF6HSP.mjs';
|
|
7
7
|
import './chunk-AV4IJKD2.mjs';
|
|
@@ -27,6 +27,6 @@ export { clearNormalizationCache, configureNormalizationCache, getNormalizationC
|
|
|
27
27
|
import './chunk-R6A773WK.mjs';
|
|
28
28
|
import './chunk-YTJCGKM4.mjs';
|
|
29
29
|
import './chunk-R6PR4WYZ.mjs';
|
|
30
|
-
import './chunk-
|
|
30
|
+
import './chunk-HBVLF3HA.mjs';
|
|
31
31
|
import './chunk-L5ORED7V.mjs';
|
|
32
32
|
export { ClientErrorCodes, ErrorCodes, NarsilError, ServerErrorCodes, isNarsilError } from './chunk-SYWGORQB.mjs';
|
|
@@ -954,7 +954,7 @@ interface VectorQueryConfig {
|
|
|
954
954
|
* candidates that it re-scores from k and from the field's dimension. It
|
|
955
955
|
* rounds that number up, and it re-scores at least 3k candidates under
|
|
956
956
|
* `osq1` and `osq2`, and at least k under `osq4` and `osq8`. Below 768
|
|
957
|
-
* dimensions, it re-scores 3.23k + 55.9√k under `osq1` and 2.6k +
|
|
957
|
+
* dimensions, it re-scores 3.23k + 55.9√k under `osq1` and 2.6k + 22.5√k under
|
|
958
958
|
* `osq2`. From 768 to 1,023 dimensions, it re-scores 2.2k + 27√k under
|
|
959
959
|
* `osq1` and k + 16√k under `osq2`. At 1,024 dimensions and above, it
|
|
960
960
|
* re-scores 3k under both. Under `osq4`, it re-scores 5k below 1,024
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { VECTOR_WORKER_COPIES_ALLOWED, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_FILTER_THRESHOLD, recordCheckpoint, planCheckpoint, deserialize, assignStorePartitions, liveSize, ordinalFilterForSubsets, threadsHoldCurrentLayout, filterForOptions, OSQ4_MIN_DIMENSION, emptyFieldBeforeRestore, ESTIMATED_MS_PER_TOMBSTONE, graphNeedsRebuild, ESTIMATED_MS_PER_VECTOR_REBUILD, releaseReplacedVectors, WORKER_COPY_MIN_VECTORS, allLiveDocIds, recalibrateFromStore, signatureOf, adoptGraph, calibrateQuantizer, fieldHandlesOf, BUILD_CHUNK_SIZE, yieldToEventLoop, DISK_ADOPTION_YIELD_INTERVAL } from './chunk-
|
|
1
|
+
import { VECTOR_WORKER_COPIES_ALLOWED, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_FILTER_THRESHOLD, recordCheckpoint, planCheckpoint, deserialize, assignStorePartitions, liveSize, ordinalFilterForSubsets, threadsHoldCurrentLayout, filterForOptions, OSQ4_MIN_DIMENSION, emptyFieldBeforeRestore, ESTIMATED_MS_PER_TOMBSTONE, graphNeedsRebuild, ESTIMATED_MS_PER_VECTOR_REBUILD, releaseReplacedVectors, WORKER_COPY_MIN_VECTORS, allLiveDocIds, recalibrateFromStore, signatureOf, adoptGraph, calibrateQuantizer, fieldHandlesOf, BUILD_CHUNK_SIZE, yieldToEventLoop, DISK_ADOPTION_YIELD_INTERVAL } from './chunk-MWT5WCJ5.mjs';
|
|
2
2
|
import { isDead, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors } from './chunk-F52QGNZ2.mjs';
|
|
3
|
-
import { createVectorStore } from './chunk-
|
|
4
|
-
export { openSharedVectorField } from './chunk-
|
|
5
|
-
import { osqBitsOf, createOsqQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, ordinalFilterValues, STORE_UNPLACED_COUNT, STORE_LIVE_COUNT, STORE_REMOVED_COUNT, createOrdinalFilter, addToOrdinalFilter, nativeScoresOf, toScore, ordinalFilterHas, magnitude, nativeStoreScratchBytes, toDistance, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, createHNSWIndex, topLayerInParts, sharedMemoryAvailable, requireWholeNeighbourLists, releaseLocksHeldBy } from './chunk-
|
|
6
|
-
export { releaseLocksHeldBy } from './chunk-
|
|
3
|
+
import { createVectorStore } from './chunk-T2QN2FA4.mjs';
|
|
4
|
+
export { openSharedVectorField } from './chunk-T2QN2FA4.mjs';
|
|
5
|
+
import { osqBitsOf, createOsqQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, ordinalFilterValues, STORE_UNPLACED_COUNT, STORE_LIVE_COUNT, STORE_REMOVED_COUNT, createOrdinalFilter, addToOrdinalFilter, nativeScoresOf, toScore, ordinalFilterHas, magnitude, nativeStoreScratchBytes, toDistance, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, createHNSWIndex, topLayerInParts, sharedMemoryAvailable, requireWholeNeighbourLists, releaseLocksHeldBy } from './chunk-ICL2AELD.mjs';
|
|
6
|
+
export { releaseLocksHeldBy } from './chunk-ICL2AELD.mjs';
|
|
7
7
|
import './chunk-37CD3OKS.mjs';
|
|
8
8
|
import { noteFallback } from './chunk-DJHHF2L5.mjs';
|
|
9
9
|
import { spawnNodeWorker } from './chunk-C7SLA6CM.mjs';
|
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
import { createDurableDirectory } from '../../chunk-
|
|
1
|
+
import { createDurableDirectory } from '../../chunk-H75XXPMY.mjs';
|
|
2
2
|
import '../../chunk-C7SLA6CM.mjs';
|
|
3
|
-
import { CHECKPOINT_WORKER_HEARTBEAT_MS, writeCheckpointSegments, directoryCheckpointStore } from '../../chunk-
|
|
4
|
-
import '../../chunk-
|
|
3
|
+
import { CHECKPOINT_WORKER_HEARTBEAT_MS, writeCheckpointSegments, directoryCheckpointStore } from '../../chunk-VVFLEGZI.mjs';
|
|
4
|
+
import '../../chunk-GMY7F2VA.mjs';
|
|
5
5
|
import '../../chunk-OHP7QR33.mjs';
|
|
6
6
|
import '../../chunk-P36MUFEJ.mjs';
|
|
7
|
-
import '../../chunk-
|
|
7
|
+
import '../../chunk-O5WPRAVZ.mjs';
|
|
8
8
|
import '../../chunk-R36MIGBU.mjs';
|
|
9
9
|
import '../../chunk-RDJKYKMQ.mjs';
|
|
10
|
-
import '../../chunk-
|
|
10
|
+
import '../../chunk-3LYY62M6.mjs';
|
|
11
11
|
import '../../chunk-CHOGQSHK.mjs';
|
|
12
12
|
import '../../chunk-WLOF6HSP.mjs';
|
|
13
13
|
import '../../chunk-AV4IJKD2.mjs';
|
|
@@ -33,7 +33,7 @@ import '../../chunk-7KC56574.mjs';
|
|
|
33
33
|
import '../../chunk-R6A773WK.mjs';
|
|
34
34
|
import '../../chunk-YTJCGKM4.mjs';
|
|
35
35
|
import '../../chunk-R6PR4WYZ.mjs';
|
|
36
|
-
import '../../chunk-
|
|
36
|
+
import '../../chunk-HBVLF3HA.mjs';
|
|
37
37
|
import '../../chunk-L5ORED7V.mjs';
|
|
38
38
|
import '../../chunk-SYWGORQB.mjs';
|
|
39
39
|
|
package/dist/react.d.ts
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { N as NarsilError, L as ListParams, h as ListResult, I as IndexInfo, g as IndexStats, Q as QueryParams, j as PreflightResult, e as QueryResult, k as SuggestParams, l as SuggestResult } from './memory-
|
|
2
|
-
export { C as ClientErrorCode, a as ClientErrorCodes, E as ErrorCode, b as ErrorCodes, c as NarsilErrorCode, S as ServerErrorCode, d as ServerErrorCodes, i as isNarsilError } from './memory-
|
|
3
|
-
import { e as TaskRecord, d as TaskProgress, b as ImportResult, c as TaskListQuery, T as TaskListPage } from './tasks-
|
|
4
|
-
export { I as ImportError, f as TaskStatus, g as TaskType } from './tasks-
|
|
1
|
+
import { N as NarsilError, L as ListParams, h as ListResult, I as IndexInfo, g as IndexStats, Q as QueryParams, j as PreflightResult, e as QueryResult, k as SuggestParams, l as SuggestResult } from './memory-COdoGthn.js';
|
|
2
|
+
export { C as ClientErrorCode, a as ClientErrorCodes, E as ErrorCode, b as ErrorCodes, c as NarsilErrorCode, S as ServerErrorCode, d as ServerErrorCodes, i as isNarsilError } from './memory-COdoGthn.js';
|
|
3
|
+
import { e as TaskRecord, d as TaskProgress, b as ImportResult, c as TaskListQuery, T as TaskListPage } from './tasks-BJmamiC4.js';
|
|
4
|
+
export { I as ImportError, f as TaskStatus, g as TaskType } from './tasks-BJmamiC4.js';
|
|
5
5
|
import { ReactNode, ReactElement } from 'react';
|
|
6
|
-
import { N as Narsil } from './engine-
|
|
7
|
-
import { N as NarsilClient, I as ImportSource } from './client-
|
|
6
|
+
import { N as Narsil } from './engine-CbZmXUp6.js';
|
|
7
|
+
import { N as NarsilClient, I as ImportSource } from './client-DKwterMb.js';
|
|
8
8
|
import { A as AnyDocument } from './schema-DttzM-ln.js';
|
|
9
9
|
import './synonyms-CpOAHPa-.js';
|
|
10
10
|
import './adapters-79OIa4xm.js';
|
|
@@ -18,7 +18,7 @@ function restoreDepsOf(core, host) {
|
|
|
18
18
|
function queueRestore(core, host, restoresInFlight, indexName, data) {
|
|
19
19
|
const previous = restoresInFlight.get(indexName) ?? Promise.resolve();
|
|
20
20
|
const restored = previous.then(async () => {
|
|
21
|
-
const { restoreFromSnapshot } = await import('./snapshot-
|
|
21
|
+
const { restoreFromSnapshot } = await import('./snapshot-XII3ANGW.mjs');
|
|
22
22
|
await restoreFromSnapshot(indexName, data, restoreDepsOf(core, host));
|
|
23
23
|
});
|
|
24
24
|
const settled = restored.then(
|
package/dist/server.d.ts
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
|
-
export { C as ClientErrorCode, a as ClientErrorCodes, E as ErrorCode, b as ErrorCodes, N as NarsilError, c as NarsilErrorCode, S as ServerErrorCode, d as ServerErrorCodes, i as isNarsilError } from './memory-
|
|
2
|
-
import { c as ServerOptions, N as NarsilServer } from './capabilities-
|
|
3
|
-
export { A as ASYNC_IMPORT_CAPABILITY, C as CorsOptions, D as DOCUMENT_EXPORT_CAPABILITY, E as ErrorEnvelope, I as INDEX_LIFECYCLE_CAPABILITY, O as OnRequestHook, R as REBUILD_ANALYSIS_CAPABILITY, d as RequestContext, e as RequestDenial, S as SERVER_CAPABILITIES, a as SYNONYMS_CAPABILITY, f as ServerLimits, T as TASK_CANCEL_CAPABILITY, b as TASK_FILTER_CAPABILITY } from './capabilities-
|
|
4
|
-
import { N as Narsil } from './engine-
|
|
5
|
-
import { d as TaskProgress, b as ImportResult, h as TaskStore, e as TaskRecord } from './tasks-
|
|
6
|
-
export { C as CapabilitiesResponse, a as CreateIndexEmbedding, i as CreateIndexRequest, H as HttpIndexConfig, I as ImportError, M as MemoryStatsResponse, T as TaskListPage, c as TaskListQuery, f as TaskStatus, g as TaskType } from './tasks-
|
|
1
|
+
export { C as ClientErrorCode, a as ClientErrorCodes, E as ErrorCode, b as ErrorCodes, N as NarsilError, c as NarsilErrorCode, S as ServerErrorCode, d as ServerErrorCodes, i as isNarsilError } from './memory-COdoGthn.js';
|
|
2
|
+
import { c as ServerOptions, N as NarsilServer } from './capabilities-BvxvFzVA.js';
|
|
3
|
+
export { A as ASYNC_IMPORT_CAPABILITY, C as CorsOptions, D as DOCUMENT_EXPORT_CAPABILITY, E as ErrorEnvelope, I as INDEX_LIFECYCLE_CAPABILITY, O as OnRequestHook, R as REBUILD_ANALYSIS_CAPABILITY, d as RequestContext, e as RequestDenial, S as SERVER_CAPABILITIES, a as SYNONYMS_CAPABILITY, f as ServerLimits, T as TASK_CANCEL_CAPABILITY, b as TASK_FILTER_CAPABILITY } from './capabilities-BvxvFzVA.js';
|
|
4
|
+
import { N as Narsil } from './engine-CbZmXUp6.js';
|
|
5
|
+
import { d as TaskProgress, b as ImportResult, h as TaskStore, e as TaskRecord } from './tasks-BJmamiC4.js';
|
|
6
|
+
export { C as CapabilitiesResponse, a as CreateIndexEmbedding, i as CreateIndexRequest, H as HttpIndexConfig, I as ImportError, M as MemoryStatsResponse, T as TaskListPage, c as TaskListQuery, f as TaskStatus, g as TaskType } from './tasks-BJmamiC4.js';
|
|
7
7
|
import './schema-DttzM-ln.js';
|
|
8
8
|
import './adapters-79OIa4xm.js';
|
|
9
9
|
import './synonyms-CpOAHPa-.js';
|
|
10
|
-
import './config-
|
|
11
|
-
import './config-
|
|
10
|
+
import './config-6YBtxekG.js';
|
|
11
|
+
import './config-DPphZfuM.js';
|
|
12
12
|
import './index-ONAEaqtx.js';
|
|
13
13
|
import './types-Cb9kiY5P.js';
|
|
14
14
|
|
|
@@ -1,19 +1,19 @@
|
|
|
1
|
-
import { unpackIndexSnapshotEnvelope, loadSegmentedCheckpoint, writeMetadataEnvelope, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-
|
|
2
|
-
import { buildArchiveFromMemory, archiveStream, memoryCheckpointStore, readSnapshotArchive } from './chunk-
|
|
3
|
-
import './chunk-
|
|
4
|
-
import { indexMetadataOf, readVectorFile, readGraphParts, SINGLE_NODE_PRIMARY_TERM } from './chunk-
|
|
1
|
+
import { unpackIndexSnapshotEnvelope, loadSegmentedCheckpoint, writeMetadataEnvelope, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-2BY6K52M.mjs';
|
|
2
|
+
import { buildArchiveFromMemory, archiveStream, memoryCheckpointStore, readSnapshotArchive } from './chunk-LJBJQPB5.mjs';
|
|
3
|
+
import './chunk-MWT5WCJ5.mjs';
|
|
4
|
+
import { indexMetadataOf, readVectorFile, readGraphParts, SINGLE_NODE_PRIMARY_TERM } from './chunk-TCYUAUHF.mjs';
|
|
5
5
|
import { decodeVectorIndexParts, bytesToVectors } from './chunk-F52QGNZ2.mjs';
|
|
6
|
-
import './chunk-
|
|
6
|
+
import './chunk-ICL2AELD.mjs';
|
|
7
7
|
import './chunk-37CD3OKS.mjs';
|
|
8
8
|
import './chunk-DJHHF2L5.mjs';
|
|
9
|
-
import { manifestKey, encodeSegmentManifest, manifestFileRefs, readSegmentFile } from './chunk-
|
|
10
|
-
import './chunk-
|
|
9
|
+
import { manifestKey, encodeSegmentManifest, manifestFileRefs, readSegmentFile } from './chunk-VVFLEGZI.mjs';
|
|
10
|
+
import './chunk-GMY7F2VA.mjs';
|
|
11
11
|
import './chunk-OHP7QR33.mjs';
|
|
12
12
|
import './chunk-P36MUFEJ.mjs';
|
|
13
|
-
import { encodeIndexConfiguration, createByteReader, hasNrslMagic, HEADER_SIZE, readHeader, validateRequiredFieldsInSchema, validateEmbeddingConfig, decodeIndexConfiguration, prepareStoredSynonyms, insertDocumentVectors } from './chunk-
|
|
13
|
+
import { encodeIndexConfiguration, createByteReader, hasNrslMagic, HEADER_SIZE, readHeader, validateRequiredFieldsInSchema, validateEmbeddingConfig, decodeIndexConfiguration, prepareStoredSynonyms, insertDocumentVectors } from './chunk-O5WPRAVZ.mjs';
|
|
14
14
|
import { validatePatternValueLimit, requireSurfaceFormsForPartialFields } from './chunk-R36MIGBU.mjs';
|
|
15
15
|
import { MAX_FILE_BYTES } from './chunk-RDJKYKMQ.mjs';
|
|
16
|
-
import { readHeapStatistics, readResidentMemory, decodeEveryStoredDocument } from './chunk-
|
|
16
|
+
import { readHeapStatistics, readResidentMemory, decodeEveryStoredDocument } from './chunk-3LYY62M6.mjs';
|
|
17
17
|
import './chunk-CHOGQSHK.mjs';
|
|
18
18
|
import { restoredConfigFields, restoredEmbedding, rejectSnapshot } from './chunk-WLOF6HSP.mjs';
|
|
19
19
|
import './chunk-AV4IJKD2.mjs';
|
|
@@ -38,8 +38,9 @@ import './chunk-7X5JST5L.mjs';
|
|
|
38
38
|
import './chunk-7KC56574.mjs';
|
|
39
39
|
import './chunk-R6A773WK.mjs';
|
|
40
40
|
import './chunk-YTJCGKM4.mjs';
|
|
41
|
+
import { collectAndReturnPages } from './chunk-5XNIRLFC.mjs';
|
|
41
42
|
import './chunk-R6PR4WYZ.mjs';
|
|
42
|
-
import { loadVectorCodeFor } from './chunk-
|
|
43
|
+
import { loadVectorCodeFor } from './chunk-HBVLF3HA.mjs';
|
|
43
44
|
import './chunk-L5ORED7V.mjs';
|
|
44
45
|
import { NarsilError, ErrorCodes } from './chunk-SYWGORQB.mjs';
|
|
45
46
|
import { decode } from '@msgpack/msgpack';
|
|
@@ -84,43 +85,84 @@ async function validateStagedCheckpoint(store, manifest) {
|
|
|
84
85
|
}
|
|
85
86
|
|
|
86
87
|
// src/engine/snapshot/constants.ts
|
|
87
|
-
var
|
|
88
|
-
var
|
|
89
|
-
var
|
|
90
|
-
var
|
|
91
|
-
var
|
|
92
|
-
var
|
|
93
|
-
var
|
|
94
|
-
var
|
|
95
|
-
var
|
|
96
|
-
var
|
|
97
|
-
var
|
|
98
|
-
var
|
|
88
|
+
var RESTORE_WORST_HEAP_BYTES_PER_STORED_BYTE = 0.2;
|
|
89
|
+
var RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 7.86;
|
|
90
|
+
var RESTORE_WORST_HEAP_BYTES_PER_VECTOR = 264;
|
|
91
|
+
var RESTORE_WORST_HEAP_BYTES_PER_VECTOR_DIMENSION = 0.24;
|
|
92
|
+
var RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 738;
|
|
93
|
+
var RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 0.2;
|
|
94
|
+
var RESTORE_WORST_EXTERNAL_BYTES_PER_STORED_BYTE = 4.06;
|
|
95
|
+
var RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 14.5;
|
|
96
|
+
var RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR = 1482;
|
|
97
|
+
var RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR_DIMENSION = 9.29;
|
|
98
|
+
var RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 2141;
|
|
99
|
+
var RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 24.15;
|
|
100
|
+
var RESTORE_FLOOR_HEAP_BYTES_PER_STORED_BYTE = 0.16;
|
|
101
|
+
var RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 6.16;
|
|
102
|
+
var RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR = 204;
|
|
103
|
+
var RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR_DIMENSION = 0.17;
|
|
104
|
+
var RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 695;
|
|
105
|
+
var RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 0.04;
|
|
106
|
+
var RESTORE_FLOOR_EXTERNAL_BYTES_PER_STORED_BYTE = 2.14;
|
|
107
|
+
var RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 5.05;
|
|
108
|
+
var RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR = 1080;
|
|
109
|
+
var RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR_DIMENSION = 0.72;
|
|
110
|
+
var RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 1744;
|
|
111
|
+
var RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 20.05;
|
|
112
|
+
var RESTORE_ESTIMATE_MARGIN = 1.25;
|
|
99
113
|
|
|
100
114
|
// src/engine/snapshot/memory-budget.ts
|
|
101
115
|
var MEGABYTE_BYTES = 1048576;
|
|
102
116
|
var ARCHIVE_RATES = {
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
117
|
+
worst: {
|
|
118
|
+
heap: {
|
|
119
|
+
perStoredByte: RESTORE_WORST_HEAP_BYTES_PER_STORED_BYTE,
|
|
120
|
+
perVector: RESTORE_WORST_HEAP_BYTES_PER_VECTOR,
|
|
121
|
+
perVectorDimension: RESTORE_WORST_HEAP_BYTES_PER_VECTOR_DIMENSION
|
|
122
|
+
},
|
|
123
|
+
external: {
|
|
124
|
+
perStoredByte: RESTORE_WORST_EXTERNAL_BYTES_PER_STORED_BYTE,
|
|
125
|
+
perVector: RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR,
|
|
126
|
+
perVectorDimension: RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR_DIMENSION
|
|
127
|
+
}
|
|
107
128
|
},
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
129
|
+
floor: {
|
|
130
|
+
heap: {
|
|
131
|
+
perStoredByte: RESTORE_FLOOR_HEAP_BYTES_PER_STORED_BYTE,
|
|
132
|
+
perVector: RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR,
|
|
133
|
+
perVectorDimension: RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR_DIMENSION
|
|
134
|
+
},
|
|
135
|
+
external: {
|
|
136
|
+
perStoredByte: RESTORE_FLOOR_EXTERNAL_BYTES_PER_STORED_BYTE,
|
|
137
|
+
perVector: RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR,
|
|
138
|
+
perVectorDimension: RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR_DIMENSION
|
|
139
|
+
}
|
|
112
140
|
}
|
|
113
141
|
};
|
|
114
142
|
var EARLIER_SNAPSHOT_RATES = {
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
143
|
+
worst: {
|
|
144
|
+
heap: {
|
|
145
|
+
perStoredByte: RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
|
|
146
|
+
perVector: RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
|
|
147
|
+
perVectorDimension: RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
|
|
148
|
+
},
|
|
149
|
+
external: {
|
|
150
|
+
perStoredByte: RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
|
|
151
|
+
perVector: RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
|
|
152
|
+
perVectorDimension: RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
|
|
153
|
+
}
|
|
119
154
|
},
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
155
|
+
floor: {
|
|
156
|
+
heap: {
|
|
157
|
+
perStoredByte: RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
|
|
158
|
+
perVector: RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
|
|
159
|
+
perVectorDimension: RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
|
|
160
|
+
},
|
|
161
|
+
external: {
|
|
162
|
+
perStoredByte: RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
|
|
163
|
+
perVector: RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
|
|
164
|
+
perVectorDimension: RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
|
|
165
|
+
}
|
|
124
166
|
}
|
|
125
167
|
};
|
|
126
168
|
function bytesAt(rate, storedBytes, vectors) {
|
|
@@ -128,10 +170,12 @@ function bytesAt(rate, storedBytes, vectors) {
|
|
|
128
170
|
for (const { dimension, count } of vectors) bytes += count * (rate.perVector + dimension * rate.perVectorDimension);
|
|
129
171
|
return bytes;
|
|
130
172
|
}
|
|
131
|
-
function footprintAt(rates, storedBytes, vectors) {
|
|
173
|
+
function footprintAt(rates, replacesIndex, storedBytes, vectors) {
|
|
174
|
+
const chosen = replacesIndex ? rates.worst : rates.floor;
|
|
175
|
+
const scale = replacesIndex ? RESTORE_ESTIMATE_MARGIN : 1 / RESTORE_ESTIMATE_MARGIN;
|
|
132
176
|
return {
|
|
133
|
-
heapBytes: bytesAt(
|
|
134
|
-
externalBytes: bytesAt(
|
|
177
|
+
heapBytes: bytesAt(chosen.heap, storedBytes, vectors) * scale,
|
|
178
|
+
externalBytes: bytesAt(chosen.external, storedBytes, vectors) * scale
|
|
135
179
|
};
|
|
136
180
|
}
|
|
137
181
|
function deadCountOf(dead) {
|
|
@@ -142,7 +186,7 @@ function deadCountOf(dead) {
|
|
|
142
186
|
}
|
|
143
187
|
return count;
|
|
144
188
|
}
|
|
145
|
-
function archiveFootprint(prepared, manifest) {
|
|
189
|
+
function archiveFootprint(prepared, manifest, replacesIndex) {
|
|
146
190
|
let segmentBytes = 0;
|
|
147
191
|
for (const partition of manifest.partitions) {
|
|
148
192
|
for (const segment of partition.segments) segmentBytes += segment.bytes;
|
|
@@ -153,36 +197,46 @@ function archiveFootprint(prepared, manifest) {
|
|
|
153
197
|
const dimension = dimensions.get(field.fieldPath) ?? 0;
|
|
154
198
|
for (const file of field.files) vectors.push({ dimension, count: Math.max(0, file.count - deadCountOf(file.dead)) });
|
|
155
199
|
}
|
|
156
|
-
return footprintAt(ARCHIVE_RATES, segmentBytes, vectors);
|
|
200
|
+
return footprintAt(ARCHIVE_RATES, replacesIndex, segmentBytes, vectors);
|
|
157
201
|
}
|
|
158
|
-
function earlierSnapshotFootprint(prepared, partitionBytes, vectorCounts) {
|
|
202
|
+
function earlierSnapshotFootprint(prepared, partitionBytes, vectorCounts, replacesIndex) {
|
|
159
203
|
const dimensions = extractVectorFieldsFromSchema(prepared.config.schema);
|
|
160
204
|
const vectors = [...vectorCounts].map(([fieldPath, count]) => ({ dimension: dimensions.get(fieldPath) ?? 0, count }));
|
|
161
|
-
return footprintAt(EARLIER_SNAPSHOT_RATES, partitionBytes, vectors);
|
|
205
|
+
return footprintAt(EARLIER_SNAPSHOT_RATES, replacesIndex, partitionBytes, vectors);
|
|
162
206
|
}
|
|
163
207
|
function inMegabytes(bytes) {
|
|
164
208
|
return Math.ceil(bytes / MEGABYTE_BYTES);
|
|
165
209
|
}
|
|
166
|
-
function
|
|
167
|
-
const where = budget === "heap" ? "of JavaScript heap" : "of process memory";
|
|
168
|
-
throw new NarsilError(
|
|
169
|
-
ErrorCodes.RESTORE_MEMORY_EXCEEDED,
|
|
170
|
-
`The restore of index "${indexName}" needs about ${inMegabytes(neededBytes)} MB ${where}, and this process has ${inMegabytes(Math.max(0, availableBytes))} MB free, so the index keeps its earlier contents. Drop the index first to free the memory that it holds, or give the process more memory, then restore again`,
|
|
171
|
-
{ indexName, budget, neededBytes, availableBytes }
|
|
172
|
-
);
|
|
173
|
-
}
|
|
174
|
-
function requireRestoreMemory(indexName, footprint) {
|
|
210
|
+
function shortfallOf(heapBytes, processBytes, countInUse) {
|
|
175
211
|
const heap = readHeapStatistics();
|
|
176
212
|
if (heap !== null) {
|
|
177
|
-
const
|
|
178
|
-
|
|
213
|
+
const limitBytes = heap.configuredLimitBytes ?? heap.limitBytes;
|
|
214
|
+
const availableBytes = countInUse ? limitBytes - heap.usedBytes : limitBytes;
|
|
215
|
+
if (heapBytes > availableBytes) return { budget: "heap", neededBytes: heapBytes, availableBytes };
|
|
179
216
|
}
|
|
180
217
|
const resident = readResidentMemory();
|
|
181
218
|
if (resident !== null) {
|
|
182
|
-
const
|
|
183
|
-
|
|
184
|
-
if (neededBytes > availableBytes) refuse(indexName, "process", neededBytes, availableBytes);
|
|
219
|
+
const availableBytes = countInUse ? resident.limitBytes - resident.residentBytes : resident.limitBytes;
|
|
220
|
+
if (processBytes > availableBytes) return { budget: "process", neededBytes: processBytes, availableBytes };
|
|
185
221
|
}
|
|
222
|
+
return null;
|
|
223
|
+
}
|
|
224
|
+
function refuse(indexName, replacesIndex, shortfall) {
|
|
225
|
+
const { budget, neededBytes, availableBytes } = shortfall;
|
|
226
|
+
const where = budget === "heap" ? "of JavaScript heap" : "of process memory";
|
|
227
|
+
const outcome = replacesIndex ? "so the index keeps its earlier contents. Drop the index first to free the memory that it holds, or give the process more memory, then restore again" : "so the engine creates no index. Give the process more memory, then restore again";
|
|
228
|
+
throw new NarsilError(
|
|
229
|
+
ErrorCodes.RESTORE_MEMORY_EXCEEDED,
|
|
230
|
+
`The restore of index "${indexName}" needs about ${inMegabytes(neededBytes)} MB ${where}, and this process has ${inMegabytes(Math.max(0, availableBytes))} MB free, ${outcome}`,
|
|
231
|
+
{ indexName, budget, neededBytes, availableBytes }
|
|
232
|
+
);
|
|
233
|
+
}
|
|
234
|
+
function requireRestoreMemory(indexName, footprint, replacesIndex) {
|
|
235
|
+
const processBytes = footprint.heapBytes + footprint.externalBytes;
|
|
236
|
+
if (shortfallOf(footprint.heapBytes, processBytes, true) === null) return;
|
|
237
|
+
const collected = collectAndReturnPages();
|
|
238
|
+
const shortfall = shortfallOf(footprint.heapBytes, processBytes, collected || replacesIndex);
|
|
239
|
+
if (shortfall !== null) refuse(indexName, replacesIndex, shortfall);
|
|
186
240
|
}
|
|
187
241
|
|
|
188
242
|
// src/workers/detached-index.ts
|
|
@@ -483,9 +537,8 @@ async function stageArchive(indexName, reader, staging, deps) {
|
|
|
483
537
|
manifest: async (manifest2, sourceName) => {
|
|
484
538
|
if (staged.prepared === null) missingPart("configuration");
|
|
485
539
|
requireSameSchema(staged.prepared, manifest2);
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
}
|
|
540
|
+
const replacesIndex = deps.indexRegistry.has(indexName);
|
|
541
|
+
requireRestoreMemory(indexName, archiveFootprint(staged.prepared, manifest2, replacesIndex), replacesIndex);
|
|
489
542
|
staged.manifest = renamed(manifest2, sourceName, indexName);
|
|
490
543
|
},
|
|
491
544
|
file: async (entryName, _ref, bytes) => {
|
|
@@ -575,7 +628,7 @@ function vectorsOfSnapshot(envelope) {
|
|
|
575
628
|
}
|
|
576
629
|
return byDocument;
|
|
577
630
|
}
|
|
578
|
-
function footprintOf(envelope, prepared) {
|
|
631
|
+
function footprintOf(envelope, prepared, replacesIndex) {
|
|
579
632
|
let partitionBytes = 0;
|
|
580
633
|
for (const payload of envelope.partitions ?? []) partitionBytes += payload.byteLength;
|
|
581
634
|
const vectorCounts = /* @__PURE__ */ new Map();
|
|
@@ -584,7 +637,7 @@ function footprintOf(envelope, prepared) {
|
|
|
584
637
|
for (const part of decodeVectorIndexParts(rawParts)) count += part.docIds.length;
|
|
585
638
|
vectorCounts.set(fieldPath, count);
|
|
586
639
|
}
|
|
587
|
-
return earlierSnapshotFootprint(prepared, partitionBytes, vectorCounts);
|
|
640
|
+
return earlierSnapshotFootprint(prepared, partitionBytes, vectorCounts, replacesIndex);
|
|
588
641
|
}
|
|
589
642
|
function documentsOfPartition(envelope, payload) {
|
|
590
643
|
return envelope.version === 2 ? documentsOfPayloadV2(payload).documents : deserializePayloadV1(payload).documents;
|
|
@@ -644,7 +697,8 @@ async function buildRestoredIndex(indexName, envelope, prepared, staging, deps)
|
|
|
644
697
|
async function restoreEarlierSnapshot(indexName, data, deps) {
|
|
645
698
|
const envelope = await decodeEarlierSnapshot(data);
|
|
646
699
|
const prepared = prepareRestoredIndex(envelope, deps);
|
|
647
|
-
|
|
700
|
+
const replacesIndex = deps.indexRegistry.has(indexName);
|
|
701
|
+
requireRestoreMemory(indexName, footprintOf(envelope, prepared, replacesIndex), replacesIndex);
|
|
648
702
|
const analysedNow = { ...prepared, storedRevision: prepared.language.revision };
|
|
649
703
|
const staging = deps.durability?.manager.stageRestore?.(indexName) ?? null;
|
|
650
704
|
try {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { openSharedVectorField, createVectorStore } from '../chunk-
|
|
2
|
-
import { osqBitsOf, createHNSWIndex, createOsqQuantizer } from '../chunk-
|
|
1
|
+
import { openSharedVectorField, createVectorStore } from '../chunk-T2QN2FA4.mjs';
|
|
2
|
+
import { osqBitsOf, createHNSWIndex, createOsqQuantizer } from '../chunk-ICL2AELD.mjs';
|
|
3
3
|
import { osqRecordBytes } from '../chunk-37CD3OKS.mjs';
|
|
4
4
|
import '../chunk-DJHHF2L5.mjs';
|
|
5
5
|
import '../chunk-KOKM4PQN.mjs';
|