@delali/narsil 0.3.1-next.180 → 0.3.1-next.183
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/{chunk-JHFBQKUF.mjs → chunk-ZZUXMJIK.mjs} +12 -14
- package/dist/browser/{durability-config-SGB5ZP5K.mjs → durability-config-75NSFCQN.mjs} +6 -3
- package/dist/browser/{optional-writes-TB34FYUW.mjs → optional-writes-IQFTXANH.mjs} +2 -0
- package/dist/browser/{storage-leaf-QFXIWDFK.mjs → storage-leaf-2CO6SQ7R.mjs} +859 -250
- package/dist/{capabilities-BQIdsaWy.d.ts → capabilities-DAHXphRn.d.ts} +2 -2
- package/dist/{chunk-QFJMRB7D.mjs → chunk-2DK74OQL.mjs} +1 -1
- package/dist/{chunk-F3FDHXCJ.mjs → chunk-3OEXOSWI.mjs} +251 -50
- package/dist/{chunk-GO6TRSJ6.mjs → chunk-632ABEIR.mjs} +1 -1
- package/dist/{chunk-34IOOYXA.mjs → chunk-6ICOWB6I.mjs} +1 -1
- package/dist/{chunk-XNFBYAXI.mjs → chunk-C3EINJNJ.mjs} +1 -1
- package/dist/{chunk-JCWC75NH.mjs → chunk-C77OBNPP.mjs} +1 -1
- package/dist/{chunk-YPV2L7M5.mjs → chunk-CKS4QQEC.mjs} +3 -3
- package/dist/{chunk-2C3BXZGA.mjs → chunk-CWXR7OP4.mjs} +18 -20
- package/dist/{chunk-BIIROQ5Q.mjs → chunk-EGWBZSHK.mjs} +2 -2
- package/dist/{chunk-XQQKETAF.mjs → chunk-F2UOTRY7.mjs} +2 -2
- package/dist/{chunk-XTSFRWAS.mjs → chunk-FIXUVFUC.mjs} +3 -3
- package/dist/{chunk-Y27E2MGC.mjs → chunk-GGG5R4CU.mjs} +1 -1
- package/dist/{chunk-2KWTQ2P7.mjs → chunk-JVR5KMO4.mjs} +2 -2
- package/dist/{chunk-XITVDRRH.mjs → chunk-M3H7CTIC.mjs} +6 -6
- package/dist/{chunk-YXZDTKQS.mjs → chunk-MCAJOLVK.mjs} +2 -2
- package/dist/{chunk-QDZJM7NI.mjs → chunk-MSIN4NM3.mjs} +3 -1
- package/dist/{chunk-UISV46WX.mjs → chunk-NPXORGMK.mjs} +2 -2
- package/dist/{chunk-R7PLSVK3.mjs → chunk-OQANB7SJ.mjs} +1 -1
- package/dist/{chunk-KHJPMZE2.mjs → chunk-PK3BGGHO.mjs} +1 -1
- package/dist/{chunk-ISJ35523.mjs → chunk-PXEGKQHS.mjs} +2 -2
- package/dist/{chunk-LOQHA6QK.mjs → chunk-SA7U67XF.mjs} +2 -2
- package/dist/{chunk-JDIATDPY.mjs → chunk-T3YI4KN6.mjs} +1 -1
- package/dist/{chunk-ZB5X5WAK.mjs → chunk-TTTBE4TB.mjs} +10 -5
- package/dist/{chunk-A4XYDUVN.mjs → chunk-V7F5DOSC.mjs} +2 -2
- package/dist/{chunk-26E5MO3Y.mjs → chunk-WUWSUJ2X.mjs} +4 -4
- package/dist/{chunk-XFOL77WH.mjs → chunk-YWSIRKKQ.mjs} +1 -1
- package/dist/{chunk-7FC2ZILY.mjs → chunk-ZVZ2V2PZ.mjs} +37 -67
- package/dist/{client-CeUxvIvd.d.ts → client-DpqtT2XG.d.ts} +3 -3
- package/dist/client.d.ts +6 -6
- package/dist/{config-DehU-VGL.d.ts → config-C687-reW.d.ts} +11 -1
- package/dist/{config-CIpLXXxm.d.ts → config-D9WR3MhO.d.ts} +1 -1
- package/dist/{direct-executor-2WQSGBN3.mjs → direct-executor-JXYIL4OQ.mjs} +1 -1
- package/dist/distribution.d.ts +4 -4
- package/dist/distribution.mjs +391 -138
- package/dist/{durability-config-XV4XR7Y4.mjs → durability-config-NWU2G2SJ.mjs} +6 -3
- package/dist/{engine-P-4xhUe4.d.ts → engine-De6JzTW8.d.ts} +123 -72
- package/dist/index.browser.mjs +1 -1
- package/dist/index.d.ts +4 -4
- package/dist/index.mjs +1 -1
- package/dist/index.react-native.mjs +1 -1
- package/dist/{long-query-thread-AAYTLYZK.mjs → long-query-thread-HPKGRYWM.mjs} +10 -10
- package/dist/{optional-writes-N2DFWWZ3.mjs → optional-writes-QE7QUI45.mjs} +5 -3
- package/dist/{paged-partition-archive-X7L27E5Z.mjs → paged-partition-archive-IK2UOY2G.mjs} +9 -9
- package/dist/partial-words.browser.mjs +1 -1
- package/dist/partial-words.mjs +1 -1
- package/dist/persistence/durability/checkpoint-worker.d.ts +9 -0
- package/dist/persistence/durability/checkpoint-worker.mjs +23 -12
- package/dist/{query-actions-2JOLPELC.mjs → query-actions-COEFVUH3.mjs} +1 -1
- package/dist/query-suggestions/merge-worker.mjs +4 -4
- package/dist/query-suggestions.browser.mjs +1 -1
- package/dist/query-suggestions.d.ts +1 -1
- package/dist/query-suggestions.mjs +2 -2
- package/dist/react-native/chunks/{chunk-F4MIGBBP.mjs → chunk-6A5XE2MA.mjs} +18 -20
- package/dist/react-native/chunks/{chunk-H344ZXRH.mjs → chunk-DN2GVVR4.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-GVWLACAT.mjs → chunk-DZ4FQW5K.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-57R74JWC.mjs → chunk-G4YYO5TC.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-F6BJYCF2.mjs → chunk-UPN4NUXU.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-GGBATERA.mjs → chunk-XNIH3WIB.mjs} +1 -1
- package/dist/react-native/chunks/{durability-config-3IOXIPYZ.mjs → durability-config-RVTWSXPH.mjs} +6 -3
- package/dist/react-native/chunks/{optional-writes-TXTJ6HUZ.mjs → optional-writes-YZ7WTR7V.mjs} +4 -2
- package/dist/react-native/chunks/{restore-queue-SZLWIL3B.mjs → restore-queue-IGKD6Q2Y.mjs} +4 -4
- package/dist/react-native/chunks/{segment-codec-leaf-WMUZ5P4F.mjs → segment-codec-leaf-JTTGDT3G.mjs} +2 -2
- package/dist/react-native/chunks/{segment-merge-leaf-Q2OEJIQ3.mjs → segment-merge-leaf-D5ZFJRUB.mjs} +3 -3
- package/dist/react-native/chunks/{storage-leaf-OXYV6O6P.mjs → storage-leaf-ZKIM6IF6.mjs} +864 -255
- package/dist/react-native/chunks/{store-leaf-ZMDXYK4E.mjs → store-leaf-PHBOTT5J.mjs} +3 -3
- package/dist/react-native/chunks/{suggestions-leaf-FIMGQZ2T.mjs → suggestions-leaf-R7MIH6Q3.mjs} +3 -3
- package/dist/react-native/chunks/{worker-copies-72TGFYQB.mjs → worker-copies-U35KPVJ7.mjs} +2 -2
- package/dist/react-native/partial-words.mjs +1 -1
- package/dist/react-native/query-suggestions.mjs +1 -1
- package/dist/react-native/sync.mjs +1 -1
- package/dist/react-native/synonyms.mjs +1 -1
- package/dist/react.d.ts +7 -7
- package/dist/{rerank-E2HFKKGJ.mjs → rerank-LYL3D7LU.mjs} +1 -1
- package/dist/{restore-queue-PUVBN6L3.mjs → restore-queue-LF4DTVJC.mjs} +4 -4
- package/dist/{segment-codec-leaf-24Z3MMQI.mjs → segment-codec-leaf-ACALKS2C.mjs} +2 -2
- package/dist/{segment-merge-leaf-FQQDU45E.mjs → segment-merge-leaf-SKMPASFK.mjs} +2 -2
- package/dist/server.d.ts +8 -8
- package/dist/server.mjs +16 -16
- package/dist/{storage-leaf-KIA7XO6Y.mjs → storage-leaf-OJ55R4C5.mjs} +588 -158
- package/dist/{store-leaf-ESVLL4JO.mjs → store-leaf-CWHHEDRK.mjs} +3 -3
- package/dist/{suggestions-leaf-FXF4AHAY.mjs → suggestions-leaf-ZTU3ZTWJ.mjs} +5 -5
- package/dist/sync/filesystem.d.ts +2 -2
- package/dist/sync/filesystem.mjs +6 -6
- package/dist/sync/indexeddb.d.ts +2 -2
- package/dist/sync.browser.mjs +1 -1
- package/dist/sync.d.ts +3 -3
- package/dist/sync.mjs +1 -1
- package/dist/synonyms.browser.mjs +1 -1
- package/dist/synonyms.mjs +1 -1
- package/dist/{tasks-CXtQ3Zpp.d.ts → tasks-X2IJQc8J.d.ts} +2 -2
- package/dist/{thread-OWXVP2ZA.mjs → thread-BEGFRULV.mjs} +9 -9
- package/dist/{types-CDN70Y_C.d.ts → types-D0YasjoX.d.ts} +1 -1
- package/dist/{worker-copies-XXX7PF4J.mjs → worker-copies-373P6IYZ.mjs} +4 -4
- package/dist/workers/entry.mjs +7 -7
- package/openapi.json +32 -0
- package/package.json +7 -7
package/dist/distribution.mjs
CHANGED
|
@@ -1,39 +1,39 @@
|
|
|
1
|
-
import { routableAllocation, resolvePartitionId, resolveNodeTargets, resolvePrimaryAssignment, requireAssignedPrimary, assertSufficientActiveReplicas, getInSyncReplicaTargets } from './chunk-
|
|
2
|
-
import { emptySerializablePartition } from './chunk-
|
|
3
|
-
import { applyStoredIndexBm25 } from './chunk-
|
|
4
|
-
import { keyBytesOf, querySuggestionsOf, createQuerySuggestionsRuntime, publicBuildOf, buildRecordToWire, buildRecordFromWire, SUGGESTIONS_SCHEMA } from './chunk-
|
|
5
|
-
import { stampAnalysisGeneration, holdsNamedSettings, dropHeldAnalysisChange, releaseHeldAnalysisChange, heldAnalysisIsBuilt, buildHeldAnalysisChange, archiveStream, buildArchiveFromMemory, indexMetadataOf, applyIndexEntry, applyDeleteEntry, resolveAnalysisTarget, requireConfigurationWithinSizeLimit, memoryCheckpointStore, createByteReader, writeArchiveFile, namedSettingsOf, createDetachedIndex, loadPartitionSegments, holdsSettings, configOfNamedSettings, closePartitionWriter, requireRestoreMemory, archiveFootprint, storedBytesOfSegments, validateReplicationEntry, readSegmentManifest, replaySyncedWal, nestedSchemaOf } from './chunk-
|
|
6
|
-
import { readVectorFile } from './chunk-
|
|
1
|
+
import { routableAllocation, resolvePartitionId, resolveNodeTargets, resolvePrimaryAssignment, requireAssignedPrimary, assertSufficientActiveReplicas, getInSyncReplicaTargets } from './chunk-EGWBZSHK.mjs';
|
|
2
|
+
import { emptySerializablePartition } from './chunk-2DK74OQL.mjs';
|
|
3
|
+
import { applyStoredIndexBm25 } from './chunk-632ABEIR.mjs';
|
|
4
|
+
import { keyBytesOf, querySuggestionsOf, createQuerySuggestionsRuntime, publicBuildOf, buildRecordToWire, buildRecordFromWire, SUGGESTIONS_SCHEMA } from './chunk-WUWSUJ2X.mjs';
|
|
5
|
+
import { stampAnalysisGeneration, holdsNamedSettings, dropHeldAnalysisChange, releaseHeldAnalysisChange, heldAnalysisIsBuilt, buildHeldAnalysisChange, archiveStream, buildArchiveFromMemory, indexMetadataOf, applyIndexEntry, applyDeleteEntry, resolveAnalysisTarget, requireConfigurationWithinSizeLimit, memoryCheckpointStore, createByteReader, writeArchiveFile, namedSettingsOf, createDetachedIndex, loadPartitionSegments, holdsSettings, configOfNamedSettings, closePartitionWriter, requireRestoreMemory, archiveFootprint, storedBytesOfSegments, validateReplicationEntry, readSegmentManifest, replaySyncedWal, nestedSchemaOf } from './chunk-ZVZ2V2PZ.mjs';
|
|
6
|
+
import { readVectorFile } from './chunk-CKS4QQEC.mjs';
|
|
7
7
|
import { addBackgroundAdmission, workWaitingForMemory, resolveHybridFusion, fuseRankings } from './chunk-TBOZHYCF.mjs';
|
|
8
8
|
import { MAX_MESSAGE_SIZE_BYTES } from './chunk-UNEEFTEU.mjs';
|
|
9
|
-
import { bindEngineWriteBlock, bindIndexReads, syncServingSourceOf, bindSyncServing, validateBackupConfig, BACKUP_SWEEP_INTERVAL_MS, bindWriteBlockLookup, backupKeys, validateBackupName, storedFilesOf, BACKUP_MANIFEST_READS_AT_ONCE, decodeBackupManifest, BACKUP_JOBS_KEY, BACKUP_DELETES_AT_ONCE, BACKUP_PROGRESS_REPORT_INTERVAL_MS, BACKUP_PARTITION_RESULT_RETENTION_MS, BACKUP_JOB_POLL_INTERVAL_MS, SYNC_BACKUP_TAKER, BACKUP_FILES_AT_ONCE, isCount, storeWithoutSaveIfAbsent, CLUSTER_UUID_KEY, isSha256Hex, BACKUP_PUBLISH_RETRY_MS, readRestorePlacement, BACKUP_JOB_CAS_ATTEMPTS, BACKUP_JOB_CAS_RETRY_MS, MAX_CLUSTER_UUID_BYTES, MAX_FINISHED_BACKUP_JOBS, BACKUP_FILE_CHUNK_BYTES, encodeBackupManifest, SYNC_BACKUPS_KEPT, BACKUP_PARTITIONS_AT_ONCE, BACKUP_PARTITION_POLL_INTERVAL_MS, BACKUP_PRIMARY_WAIT_MS, readBackupPartition, readStoredFile } from './chunk-
|
|
10
|
-
import { DEFAULT_NODE_LIFECYCLE_CONFIG, DEFAULT_CONTROLLER_CONFIG, readIndexMetadataSnapshot, indexMetadataSnapshotOf, getIndexMetadata, getClusterIndexConfig, withPartitionCount, validateIndexName, suggestionStateOfSettings, indexConfigKey, metadataWithSync, indexConfigFromSettings, metadataWithBm25, readIndexState, storeSynonymSet, deleteSynonymSet, indexSettingsOf, putIndexMetadata, isValidInteger, isRecord, HIDDEN_GENERATION_PREFIX, isRecord2, decodeIndexSettings, metadataWithSynonymSet, RESHARD_RECORD_CAS_ATTEMPTS, indexUuidOfStoredMetadata, storedSynonymSetOf, RESHARD_FORWARD_BATCH_DOCUMENTS, RESHARD_RETRY_DELAY_MS, TEARDOWN_CAS_ATTEMPTS, RESHARD_RECORD_CHECK_INTERVAL_MS, RESHARD_POLL_INTERVAL_MS, ALLOCATION_RETRY_DELAY_MS, TEARDOWN_RETRY_ROUNDS, ANALYSIS_CHANGE_RETRY_MS, putIndexReplicationFactor, ALLOCATION_CAS_ATTEMPTS, TEARDOWN_RETRY_DELAY_MS, RESHARD_CONTROLLER_POLL_INTERVAL_MS, MAX_RESHARD_CUTOVER_ATTEMPTS, ANALYSIS_CHANGE_POLL_INTERVAL_MS, RECOVERY_CAS_ATTEMPTS, ESTIMATED_PARTITION_BYTES, REBALANCE_THRESHOLD, putGenerationMetadata, RESHARD_CUTOVER_LAG_ENTRIES, ANALYSIS_REBUILD_RESEND_MS, RESHARD_CUTOVER_TIMEOUT_MS, PARTITION_STORES_TIMEOUT_MS, SYNC_EPOCH_CAS_ATTEMPTS, LEADERSHIP_IMBALANCE_THRESHOLD, BOOTSTRAP_CAS_ATTEMPTS, metadataWithSuggestionState } from './chunk-
|
|
11
|
-
import { MAX_WAIT_FOR_ACTIVE_REPLICAS, MIN_CONTROLLER_LEASE_TTL_MS, MAX_NODE_LEFT_DELAY_MS, DEFAULT_MAX_PER_SOURCE_SNAPSHOTS, DEFAULT_MAX_STREAMS_PER_INDEX, DEFAULT_CAPACITY, CATCH_UP_TICK_MS, MEMORY_SAMPLE_INTERVAL_MS, createWritePressure, WRITE_MEMORY_SAMPLE_MAX_AGE_MS, DEFAULT_IN_FLIGHT_WRITE_FRACTION, MAX_SETTINGS_REPLACEMENT_ATTEMPTS, SETTINGS_CONTENTION_REASON, MAX_DROP_ACKNOWLEDGE_WAIT_MS, DEFAULT_PARTITION_COUNT, DEFAULT_REPLICATION_FACTOR, DEFAULT_CREATE_INDEX_WAIT_MS, DEFAULT_BOOTSTRAP_SYNC_DEADLINE_MS, DROP_ACKNOWLEDGE_POLL_INTERVAL_MS, COORDINATOR_REQUEST_BUCKET_BOUNDS_SECONDS, COPY_FAILURE_ATTEMPTS, SEARCH_CONTEXT_REAPER_INTERVAL_MS, DECOMMISSION_RECHECK_INTERVAL_MS, refuseEncodeWithoutRoom, answerRefusalOf, withAdmittedWrite, estimateForwardedBytes, PARTITION_BYTES_INDEX_OVERHEAD_BYTES, MAX_SOURCE_ID_LENGTH, PARTITION_BYTES_ENTRY_BYTES, PARTITION_BYTES_REPORT_BUDGET_BYTES, CATCH_UP_FEED_WINDOW_MS, MAX_CATCH_UP_IN_FLIGHT_BYTES, SYNCED_POSITION_REPORT_TIMEOUT_MS, SNAPSHOT_CHUNK_YIELD_INTERVAL_MS,
|
|
12
|
-
export { DEFAULT_CAPACITY, DEFAULT_PARTITION_COUNT, DEFAULT_REPLICATION_FACTOR } from './chunk-
|
|
13
|
-
import { encodeIndexConfiguration, MAX_SNAPSHOT_ANALYSIS_ATTEMPTS, requireSyncSetting, requireNoQuerySuggestions, decodeIndexConfiguration, INDEX_CONFIGURATION_ENVELOPE_VERSION } from './chunk-
|
|
9
|
+
import { bindEngineWriteBlock, bindIndexReads, syncServingSourceOf, bindSyncServing, validateBackupConfig, BACKUP_SWEEP_INTERVAL_MS, bindWriteBlockLookup, backupKeys, validateBackupName, storedFilesOf, BACKUP_MANIFEST_READS_AT_ONCE, decodeBackupManifest, BACKUP_JOBS_KEY, BACKUP_DELETES_AT_ONCE, BACKUP_PROGRESS_REPORT_INTERVAL_MS, BACKUP_PARTITION_RESULT_RETENTION_MS, BACKUP_JOB_POLL_INTERVAL_MS, SYNC_BACKUP_TAKER, BACKUP_FILES_AT_ONCE, isCount, storeWithoutSaveIfAbsent, CLUSTER_UUID_KEY, isSha256Hex, BACKUP_PUBLISH_RETRY_MS, readRestorePlacement, BACKUP_JOB_CAS_ATTEMPTS, BACKUP_JOB_CAS_RETRY_MS, MAX_CLUSTER_UUID_BYTES, MAX_FINISHED_BACKUP_JOBS, BACKUP_FILE_CHUNK_BYTES, encodeBackupManifest, SYNC_BACKUPS_KEPT, BACKUP_PARTITIONS_AT_ONCE, BACKUP_PARTITION_POLL_INTERVAL_MS, BACKUP_PRIMARY_WAIT_MS, readBackupPartition, readStoredFile } from './chunk-PXEGKQHS.mjs';
|
|
10
|
+
import { DEFAULT_NODE_LIFECYCLE_CONFIG, DEFAULT_CONTROLLER_CONFIG, readIndexMetadataSnapshot, indexMetadataSnapshotOf, getIndexMetadata, getClusterIndexConfig, withPartitionCount, validateIndexName, suggestionStateOfSettings, indexConfigKey, metadataWithSync, indexConfigFromSettings, metadataWithBm25, readIndexState, storeSynonymSet, deleteSynonymSet, indexSettingsOf, putIndexMetadata, isValidInteger, isRecord, HIDDEN_GENERATION_PREFIX, isRecord2, decodeIndexSettings, metadataWithSynonymSet, RESHARD_RECORD_CAS_ATTEMPTS, indexUuidOfStoredMetadata, storedSynonymSetOf, RESHARD_FORWARD_BATCH_DOCUMENTS, RESHARD_RETRY_DELAY_MS, TEARDOWN_CAS_ATTEMPTS, RESHARD_RECORD_CHECK_INTERVAL_MS, RESHARD_POLL_INTERVAL_MS, ALLOCATION_RETRY_DELAY_MS, TEARDOWN_RETRY_ROUNDS, ANALYSIS_CHANGE_RETRY_MS, putIndexReplicationFactor, ALLOCATION_CAS_ATTEMPTS, TEARDOWN_RETRY_DELAY_MS, RESHARD_CONTROLLER_POLL_INTERVAL_MS, MAX_RESHARD_CUTOVER_ATTEMPTS, ANALYSIS_CHANGE_POLL_INTERVAL_MS, RECOVERY_CAS_ATTEMPTS, ESTIMATED_PARTITION_BYTES, REBALANCE_THRESHOLD, putGenerationMetadata, RESHARD_CUTOVER_LAG_ENTRIES, ANALYSIS_REBUILD_RESEND_MS, RESHARD_CUTOVER_TIMEOUT_MS, PARTITION_STORES_TIMEOUT_MS, SYNC_EPOCH_CAS_ATTEMPTS, LEADERSHIP_IMBALANCE_THRESHOLD, BOOTSTRAP_CAS_ATTEMPTS, metadataWithSuggestionState } from './chunk-F2UOTRY7.mjs';
|
|
11
|
+
import { MAX_WAIT_FOR_ACTIVE_REPLICAS, MIN_CONTROLLER_LEASE_TTL_MS, MAX_NODE_LEFT_DELAY_MS, DEFAULT_MAX_PER_SOURCE_SNAPSHOTS, DEFAULT_MAX_STREAMS_PER_INDEX, DEFAULT_CAPACITY, CATCH_UP_TICK_MS, MEMORY_SAMPLE_INTERVAL_MS, createWritePressure, WRITE_MEMORY_SAMPLE_MAX_AGE_MS, DEFAULT_IN_FLIGHT_WRITE_FRACTION, MAX_SETTINGS_REPLACEMENT_ATTEMPTS, SETTINGS_CONTENTION_REASON, MAX_DROP_ACKNOWLEDGE_WAIT_MS, DEFAULT_PARTITION_COUNT, DEFAULT_REPLICATION_FACTOR, DEFAULT_CREATE_INDEX_WAIT_MS, DEFAULT_BOOTSTRAP_SYNC_DEADLINE_MS, DROP_ACKNOWLEDGE_POLL_INTERVAL_MS, COORDINATOR_REQUEST_BUCKET_BOUNDS_SECONDS, COPY_FAILURE_ATTEMPTS, SEARCH_CONTEXT_REAPER_INTERVAL_MS, DECOMMISSION_RECHECK_INTERVAL_MS, refuseEncodeWithoutRoom, answerRefusalOf, withAdmittedWrite, estimateForwardedBytes, PARTITION_BYTES_INDEX_OVERHEAD_BYTES, MAX_SOURCE_ID_LENGTH, ADMISSION_TIMEOUT_MS, PARTITION_BYTES_ENTRY_BYTES, PARTITION_BYTES_REPORT_BUDGET_BYTES, CATCH_UP_FEED_WINDOW_MS, MAX_CATCH_UP_IN_FLIGHT_BYTES, SYNCED_POSITION_REPORT_TIMEOUT_MS, SNAPSHOT_CHUNK_YIELD_INTERVAL_MS, ANSWER_DECODE_HEAP_BYTES_PER_PAYLOAD_BYTE, REGISTRATION_LEASE_DRIFT_SHARE, REGISTRATION_LEASE_CONFIRM_TIMEOUT_MS, WRITE_APPLY_YIELD_INTERVAL_MS, ROLLBACK_RETRY_INITIAL_DELAY_MS, ROLLBACK_RETRY_DELAY_CAP_MS, FETCH_GROUP_DOCUMENT_IDS, COPY_FAILURE_RETRY_DELAY_MS, SEARCH_CONTEXT_ENTRY_HEAP_BYTES, CLEAR_PAGE_SIZE, estimateDocumentsBytes, estimateDocumentBytes, PRIMARY_CHANGE_RETRY_WINDOW_MS, PRIMARY_CHANGE_POLL_INTERVAL_MS, PARTITION_WRITE_CONCURRENCY, FORWARD_RETRY_INITIAL_DELAY_MS, FORWARD_RETRY_DELAY_CAP_MS } from './chunk-GGG5R4CU.mjs';
|
|
12
|
+
export { DEFAULT_CAPACITY, DEFAULT_PARTITION_COUNT, DEFAULT_REPLICATION_FACTOR } from './chunk-GGG5R4CU.mjs';
|
|
13
|
+
import { encodeIndexConfiguration, MAX_SNAPSHOT_ANALYSIS_ATTEMPTS, requireSyncSetting, requireNoQuerySuggestions, decodeIndexConfiguration, INDEX_CONFIGURATION_ENVELOPE_VERSION } from './chunk-SA7U67XF.mjs';
|
|
14
14
|
import { restoredConfigFields, restoredEmbedding } from './chunk-X4UMDSB4.mjs';
|
|
15
|
-
import { estimatedArchiveBytes, readSnapshotArchive, archiveLayout, openManifestPayload, sourceIndexNameOf, listedKeysOf, decodeManifestPayload } from './chunk-
|
|
15
|
+
import { estimatedArchiveBytes, readSnapshotArchive, archiveLayout, openManifestPayload, sourceIndexNameOf, listedKeysOf, decodeManifestPayload } from './chunk-NPXORGMK.mjs';
|
|
16
16
|
import { CONTROLLER_LEASE_KEY } from './chunk-44W3CBNG.mjs';
|
|
17
17
|
import './chunk-T4HMSZCX.mjs';
|
|
18
|
-
import { createForwardBatchResultMessage, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, optionalIndexUuid, computeEntryChecksum, createPrimaryResignMessage, validateInsyncConfirmPayload, buildEntry, createInsyncRemoveMessage, createEntryBatchMessage, createInsyncConfirmMessage, createEntryMessage, createInsyncAddMessage, validateAckPayload, checkpointLastSeqNo, checkpointPrimaryTerm, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage } from './chunk-
|
|
19
|
-
import { directoryCheckpointStore, readListedFile, encodeSegmentManifest } from './chunk-
|
|
18
|
+
import { createForwardBatchResultMessage, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, optionalIndexUuid, computeEntryChecksum, createPrimaryResignMessage, validateInsyncConfirmPayload, buildEntry, createInsyncRemoveMessage, createEntryBatchMessage, createInsyncConfirmMessage, createEntryMessage, createInsyncAddMessage, validateAckPayload, checkpointLastSeqNo, checkpointPrimaryTerm, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage } from './chunk-3OEXOSWI.mjs';
|
|
19
|
+
import { directoryCheckpointStore, readListedFile, encodeSegmentManifest } from './chunk-TTTBE4TB.mjs';
|
|
20
20
|
import { TransportErrorCodes, ClusterMessageTypes, ReplicationMessageTypes, SuggestionMessageTypes, SyncMessageTypes, QueryMessageTypes, TransportError, DEFAULT_TRANSPORT_CONFIG } from './chunk-5FLRXF73.mjs';
|
|
21
21
|
import { MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR, WIRE_BATCH_BUDGET } from './chunk-JDDXV6UF.mjs';
|
|
22
|
-
import { manifestFileRefs, segmentKey, SEGMENT_MANIFEST_VERSION, vectorFileKey } from './chunk-
|
|
22
|
+
import { manifestFileRefs, segmentKey, SEGMENT_MANIFEST_VERSION, vectorFileKey } from './chunk-PK3BGGHO.mjs';
|
|
23
23
|
import { getPath, getFs } from './chunk-HLPRHOSO.mjs';
|
|
24
|
-
import './chunk-
|
|
24
|
+
import './chunk-MSIN4NM3.mjs';
|
|
25
25
|
import './chunk-CV3Q2EKO.mjs';
|
|
26
26
|
import './chunk-LNGE7SLY.mjs';
|
|
27
27
|
import './chunk-I2WJCRIH.mjs';
|
|
28
|
-
import './chunk-
|
|
29
|
-
import { deserializePartition } from './chunk-
|
|
28
|
+
import './chunk-YWSIRKKQ.mjs';
|
|
29
|
+
import { deserializePartition } from './chunk-OQANB7SJ.mjs';
|
|
30
30
|
import { decodeVectorFilePayload, bytesToVectors, isDead, MIN_EF_SEARCH, MAX_EF_SEARCH, MIN_OVERSAMPLE } from './chunk-TNEU32EN.mjs';
|
|
31
|
-
import './chunk-
|
|
32
|
-
import { RestoredHistory } from './chunk-
|
|
31
|
+
import './chunk-JVR5KMO4.mjs';
|
|
32
|
+
import { RestoredHistory } from './chunk-MCAJOLVK.mjs';
|
|
33
33
|
import { oneSyncChangeAtATime } from './chunk-V6EUBAIT.mjs';
|
|
34
|
-
import { historyFilesOf, removeHistoryFolder, QUERY_HISTORY_FOLDER, RESTORE_STAGING_INFIX, openChecksummedEnvelope, historyFolderOf, listHistoryFolder, historyFileOf, COUNTS_STREAM_CHUNK_BYTES, BUILD_WRITE_BATCH, restoreStagingPrefix, crc32Init, crc32Final, crc32Update, MAX_COUNTS_STREAM_KEY_BYTES, MAX_HISTORY_FILE_BYTES, packChecksummedEnvelope, storedStateOf } from './chunk-
|
|
34
|
+
import { historyFilesOf, removeHistoryFolder, QUERY_HISTORY_FOLDER, RESTORE_STAGING_INFIX, openChecksummedEnvelope, historyFolderOf, listHistoryFolder, historyFileOf, COUNTS_STREAM_CHUNK_BYTES, BUILD_WRITE_BATCH, restoreStagingPrefix, crc32Init, crc32Final, crc32Update, MAX_COUNTS_STREAM_KEY_BYTES, MAX_HISTORY_FILE_BYTES, packChecksummedEnvelope, storedStateOf } from './chunk-C77OBNPP.mjs';
|
|
35
35
|
import { boundQuerySuggestionsOf, bindQuerySuggestions } from './chunk-S2POFTLN.mjs';
|
|
36
|
-
import { oversampledShardSize, runtimeHost, bindRerankWindowSource, rerankWindowCapOf, validateMemoryFractions, requireKnownConfig, importOptionalReads, resolveRunnableConfig, createEngineCore, startEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, emitEngineEvent, synonymModule, createEngineIndex, generateId, getLanguage, resolveIndexAnalysis, memoryFractionsOf, createMemoryMonitor, readMemoryFigures, FULL_MEMORY_COLLECTION_INTERVAL_MS, validateBM25Params, storedBM25Params, requireSurfaceFormsForPartialFields, bm25SetsValue, prepareConfiguredSynonyms, storedArrayPositionGap, updateDocumentVectors, arrayPositionsAreStale, fnv1a, readRecentMemoryFigures, normalizeSort, sortModesOf, resolveProjection, requireLoadedOptionalReads, applyProjection, FILTER_EXPRESSION_KEYS, filterTestCount, setNestedValue, validateBM25Shape, CASE_FOLD_EXPANSION_FACTOR, PATTERN_RUN_HEAP_BYTES, ENUM_VALUE_HEAP_BYTES, DOCUMENT_VALUE_HEAP_BYTES, DOCUMENT_HEAP_BYTES_PER_ESTIMATED_BYTE, DOCUMENT_PROPERTY_HEAP_BYTES, VECTOR_DOCUMENT_HEAP_BYTES, VECTOR_COMPONENT_HEAP_BYTES, hasLanguage, FIELD_FILTER_OPERATORS, yieldToEventLoop, NORMALIZATION_EXPANSION_FACTOR, TERM_UNIT_HEAP_BYTES, customTokensOf, visitTokens, clampRowCount, now, sortSignatureOf, listBindingOf, requireMatchingCursor, decodePageCursor, encodePageCursor, DISTINCT_TERM_HEAP_BYTES, PARTIAL_SPELLING_HEAP_BYTES, clampLimit, requireWithinResultWindow, providedDocId, holdsQuotedText, queryBindingOf, engineCoreOf, sortSignatureEntry, fnv1aUint32s, loadRerankRuntime, mergePartitionStats, mergeEntries, regionsOverlap, compareScores } from './chunk-
|
|
36
|
+
import { oversampledShardSize, runtimeHost, bindRerankWindowSource, rerankWindowCapOf, validateMemoryFractions, requireKnownConfig, importOptionalReads, resolveRunnableConfig, createEngineCore, startEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, emitEngineEvent, synonymModule, createEngineIndex, generateId, getLanguage, resolveIndexAnalysis, memoryFractionsOf, createMemoryMonitor, readMemoryFigures, FULL_MEMORY_COLLECTION_INTERVAL_MS, validateBM25Params, storedBM25Params, requireSurfaceFormsForPartialFields, bm25SetsValue, prepareConfiguredSynonyms, storedArrayPositionGap, updateDocumentVectors, arrayPositionsAreStale, fnv1a, readRecentMemoryFigures, normalizeSort, sortModesOf, resolveProjection, requireLoadedOptionalReads, applyProjection, FILTER_EXPRESSION_KEYS, filterTestCount, setNestedValue, validateBM25Shape, CASE_FOLD_EXPANSION_FACTOR, PATTERN_RUN_HEAP_BYTES, ENUM_VALUE_HEAP_BYTES, DOCUMENT_VALUE_HEAP_BYTES, DOCUMENT_HEAP_BYTES_PER_ESTIMATED_BYTE, DOCUMENT_PROPERTY_HEAP_BYTES, VECTOR_DOCUMENT_HEAP_BYTES, VECTOR_COMPONENT_HEAP_BYTES, hasLanguage, FIELD_FILTER_OPERATORS, yieldToEventLoop, NORMALIZATION_EXPANSION_FACTOR, TERM_UNIT_HEAP_BYTES, customTokensOf, visitTokens, clampRowCount, now, sortSignatureOf, listBindingOf, requireMatchingCursor, decodePageCursor, encodePageCursor, DISTINCT_TERM_HEAP_BYTES, PARTIAL_SPELLING_HEAP_BYTES, clampLimit, requireWithinResultWindow, providedDocId, holdsQuotedText, queryBindingOf, engineCoreOf, sortSignatureEntry, fnv1aUint32s, loadRerankRuntime, mergePartitionStats, mergeEntries, regionsOverlap, compareScores } from './chunk-CWXR7OP4.mjs';
|
|
37
37
|
import { encodeDocument, encodeDocumentReply } from './chunk-KLVMK3R5.mjs';
|
|
38
38
|
import { collectYoungGenerationBeforeRefusal, collectGarbageBeforeRefusal, onRefusalAfterCollection, reservedHeapBytes, reserveHeapBytes } from './chunk-LPRRFCVU.mjs';
|
|
39
39
|
import { decode } from './chunk-GXCDWLQI.mjs';
|
|
@@ -2016,6 +2016,7 @@ function validateCountPayload(decoded) {
|
|
|
2016
2016
|
validatePartitionIdsArray(decoded.partitionIds, "partitionIds", CONFIG_INVALID);
|
|
2017
2017
|
return decoded;
|
|
2018
2018
|
}
|
|
2019
|
+
var STORED_SIZE_FIELDS = ["savedBytes", "vectorSavedBytes", "logBytes"];
|
|
2019
2020
|
function validateCountResultPayload(decoded) {
|
|
2020
2021
|
if (!isRecord3(decoded)) {
|
|
2021
2022
|
throwInvalid(CONFIG_INVALID, "Invalid CountResultPayload: expected an object");
|
|
@@ -2036,6 +2037,10 @@ function validateCountResultPayload(decoded) {
|
|
|
2036
2037
|
MAX_COUNT_VALUE,
|
|
2037
2038
|
CONFIG_INVALID
|
|
2038
2039
|
);
|
|
2040
|
+
for (const figure2 of STORED_SIZE_FIELDS) {
|
|
2041
|
+
if (entry[figure2] === void 0) continue;
|
|
2042
|
+
validateNonNegativeInteger(entry[figure2], `partitions[${i}].${figure2}`, MAX_COUNT_VALUE, CONFIG_INVALID);
|
|
2043
|
+
}
|
|
2039
2044
|
}
|
|
2040
2045
|
validateStringField(decoded.language, "language", MAX_LANGUAGE_NAME_LENGTH, CONFIG_INVALID);
|
|
2041
2046
|
return decoded;
|
|
@@ -3421,8 +3426,11 @@ function createCatchUpState() {
|
|
|
3421
3426
|
cursors: /* @__PURE__ */ new Map(),
|
|
3422
3427
|
pendingAdmissions: /* @__PURE__ */ new Map(),
|
|
3423
3428
|
admittedReplicas: /* @__PURE__ */ new Map(),
|
|
3429
|
+
admissionsToWithdraw: /* @__PURE__ */ new Map(),
|
|
3424
3430
|
acknowledgedSeqNos: /* @__PURE__ */ new Map(),
|
|
3425
3431
|
removedReplicas: /* @__PURE__ */ new Map(),
|
|
3432
|
+
admissionGenerations: /* @__PURE__ */ new Map(),
|
|
3433
|
+
lastAdmissionGeneration: 0,
|
|
3426
3434
|
startedTerms: /* @__PURE__ */ new Map(),
|
|
3427
3435
|
inFlightBytes: 0,
|
|
3428
3436
|
timer: null,
|
|
@@ -3431,26 +3439,40 @@ function createCatchUpState() {
|
|
|
3431
3439
|
stopped: false
|
|
3432
3440
|
};
|
|
3433
3441
|
}
|
|
3434
|
-
function
|
|
3442
|
+
function openFreshSyncSession(cursor, appliedSeqNo) {
|
|
3443
|
+
cursor.feed?.cancel();
|
|
3444
|
+
cursor.appliedSeqNo = appliedSeqNo;
|
|
3445
|
+
cursor.syncEpoch += 1;
|
|
3446
|
+
}
|
|
3447
|
+
function noteAcknowledgedPosition(state, key, replicaNodeId, appliedSeqNo) {
|
|
3448
|
+
const acknowledged = state.acknowledgedSeqNos.get(key) ?? /* @__PURE__ */ new Map();
|
|
3449
|
+
acknowledged.set(replicaNodeId, appliedSeqNo);
|
|
3450
|
+
state.acknowledgedSeqNos.set(key, acknowledged);
|
|
3451
|
+
}
|
|
3452
|
+
function recordReplicaPosition(state, indexName, partitionId, replicaNodeId, appliedSeqNo, logEnd) {
|
|
3435
3453
|
const key = partitionKey(indexName, partitionId);
|
|
3454
|
+
const reported = Math.min(appliedSeqNo, logEnd);
|
|
3436
3455
|
const partition = state.cursors.get(key) ?? /* @__PURE__ */ new Map();
|
|
3437
3456
|
const existing = partition.get(replicaNodeId);
|
|
3438
3457
|
if (existing === void 0) {
|
|
3439
|
-
partition.set(replicaNodeId, { appliedSeqNo, busy: false, syncEpoch: 0, feed: null });
|
|
3440
|
-
} else if (existing.
|
|
3441
|
-
existing
|
|
3442
|
-
|
|
3443
|
-
existing.
|
|
3444
|
-
} else if (appliedSeqNo > existing.appliedSeqNo) {
|
|
3445
|
-
existing.appliedSeqNo = appliedSeqNo;
|
|
3446
|
-
} else if (appliedSeqNo < existing.appliedSeqNo) {
|
|
3447
|
-
existing.appliedSeqNo = appliedSeqNo;
|
|
3448
|
-
existing.syncEpoch += 1;
|
|
3458
|
+
partition.set(replicaNodeId, { appliedSeqNo: reported, busy: false, syncEpoch: 0, feed: null });
|
|
3459
|
+
} else if (reported < existing.appliedSeqNo) {
|
|
3460
|
+
openFreshSyncSession(existing, reported);
|
|
3461
|
+
} else {
|
|
3462
|
+
existing.appliedSeqNo = reported;
|
|
3449
3463
|
}
|
|
3450
3464
|
state.cursors.set(key, partition);
|
|
3451
|
-
|
|
3452
|
-
|
|
3453
|
-
|
|
3465
|
+
noteAcknowledgedPosition(state, key, replicaNodeId, reported);
|
|
3466
|
+
}
|
|
3467
|
+
function recordSnapshotSyncRequest(state, indexName, partitionId, replicaNodeId) {
|
|
3468
|
+
const key = partitionKey(indexName, partitionId);
|
|
3469
|
+
const partition = state.cursors.get(key);
|
|
3470
|
+
const existing = partition?.get(replicaNodeId);
|
|
3471
|
+
if (partition === void 0 || existing === void 0) return;
|
|
3472
|
+
openFreshSyncSession(existing, 0);
|
|
3473
|
+
partition.delete(replicaNodeId);
|
|
3474
|
+
if (partition.size === 0) state.cursors.delete(key);
|
|
3475
|
+
state.acknowledgedSeqNos.get(key)?.delete(replicaNodeId);
|
|
3454
3476
|
}
|
|
3455
3477
|
function forgetReplica(state, indexName, partitionId, replicaNodeId) {
|
|
3456
3478
|
const key = partitionKey(indexName, partitionId);
|
|
@@ -3477,41 +3499,28 @@ function forgetCatchUpIndex(state, indexName) {
|
|
|
3477
3499
|
}
|
|
3478
3500
|
forgetKeysOfIndex(state.pendingAdmissions, indexName);
|
|
3479
3501
|
forgetKeysOfIndex(state.admittedReplicas, indexName);
|
|
3502
|
+
forgetKeysOfIndex(state.admissionsToWithdraw, indexName);
|
|
3480
3503
|
forgetKeysOfIndex(state.acknowledgedSeqNos, indexName);
|
|
3481
3504
|
forgetKeysOfIndex(state.removedReplicas, indexName);
|
|
3505
|
+
forgetKeysOfIndex(state.admissionGenerations, indexName);
|
|
3482
3506
|
forgetKeysOfIndex(state.startedTerms, indexName);
|
|
3483
3507
|
}
|
|
3484
3508
|
function getPendingAdmissions(state, indexName, partitionId) {
|
|
3485
3509
|
const pending = state.pendingAdmissions.get(partitionKey(indexName, partitionId));
|
|
3486
3510
|
return pending === void 0 ? [] : [...pending];
|
|
3487
3511
|
}
|
|
3488
|
-
function
|
|
3512
|
+
function recordAdmissionToWithdraw(state, indexName, partitionId, replicaNodeId, primaryTerm) {
|
|
3489
3513
|
const key = partitionKey(indexName, partitionId);
|
|
3490
|
-
const
|
|
3491
|
-
|
|
3492
|
-
state.
|
|
3514
|
+
const owed = state.admissionsToWithdraw.get(key) ?? /* @__PURE__ */ new Map();
|
|
3515
|
+
owed.set(replicaNodeId, { primaryTerm, withdrawing: false });
|
|
3516
|
+
state.admissionsToWithdraw.set(key, owed);
|
|
3493
3517
|
}
|
|
3494
|
-
function
|
|
3518
|
+
function forgetAdmissionToWithdraw(state, indexName, partitionId, replicaNodeId) {
|
|
3495
3519
|
const key = partitionKey(indexName, partitionId);
|
|
3496
|
-
const
|
|
3497
|
-
if (
|
|
3498
|
-
|
|
3499
|
-
if (
|
|
3500
|
-
}
|
|
3501
|
-
function admittedBeyondTable(state, indexName, partitionId, assignment, tableVersion) {
|
|
3502
|
-
const admitted = state.admittedReplicas.get(partitionKey(indexName, partitionId));
|
|
3503
|
-
if (admitted === void 0) return [];
|
|
3504
|
-
const beyond = [];
|
|
3505
|
-
for (const [replicaNodeId, record] of [...admitted]) {
|
|
3506
|
-
const settled = assignment.inSyncSet.includes(replicaNodeId) || !assignment.replicas.includes(replicaNodeId);
|
|
3507
|
-
const superseded = record.tableVersion !== null && tableVersion !== void 0 && tableVersion >= record.tableVersion;
|
|
3508
|
-
if (settled || superseded || record.primaryTerm !== assignment.primaryTerm) {
|
|
3509
|
-
forgetAdmittedReplica(state, indexName, partitionId, replicaNodeId);
|
|
3510
|
-
continue;
|
|
3511
|
-
}
|
|
3512
|
-
beyond.push(replicaNodeId);
|
|
3513
|
-
}
|
|
3514
|
-
return beyond;
|
|
3520
|
+
const owed = state.admissionsToWithdraw.get(key);
|
|
3521
|
+
if (owed === void 0) return;
|
|
3522
|
+
owed.delete(replicaNodeId);
|
|
3523
|
+
if (owed.size === 0) state.admissionsToWithdraw.delete(key);
|
|
3515
3524
|
}
|
|
3516
3525
|
function markPendingAdmission(state, indexName, partitionId, replicaNodeId) {
|
|
3517
3526
|
const key = partitionKey(indexName, partitionId);
|
|
@@ -3521,9 +3530,32 @@ function markPendingAdmission(state, indexName, partitionId, replicaNodeId) {
|
|
|
3521
3530
|
}
|
|
3522
3531
|
pending.add(replicaNodeId);
|
|
3523
3532
|
state.pendingAdmissions.set(key, pending);
|
|
3524
|
-
state
|
|
3533
|
+
forgetRemovedReplica(state, indexName, partitionId, replicaNodeId);
|
|
3534
|
+
advanceAdmissionGeneration(state, indexName, partitionId, replicaNodeId);
|
|
3525
3535
|
return true;
|
|
3526
3536
|
}
|
|
3537
|
+
function advanceAdmissionGeneration(state, indexName, partitionId, replicaNodeId) {
|
|
3538
|
+
const key = partitionKey(indexName, partitionId);
|
|
3539
|
+
const generations = state.admissionGenerations.get(key) ?? /* @__PURE__ */ new Map();
|
|
3540
|
+
state.lastAdmissionGeneration += 1;
|
|
3541
|
+
generations.set(replicaNodeId, state.lastAdmissionGeneration);
|
|
3542
|
+
state.admissionGenerations.set(key, generations);
|
|
3543
|
+
return state.lastAdmissionGeneration;
|
|
3544
|
+
}
|
|
3545
|
+
function admissionGenerationOf(state, indexName, partitionId, replicaNodeId) {
|
|
3546
|
+
const current = state.admissionGenerations.get(partitionKey(indexName, partitionId))?.get(replicaNodeId);
|
|
3547
|
+
return current ?? advanceAdmissionGeneration(state, indexName, partitionId, replicaNodeId);
|
|
3548
|
+
}
|
|
3549
|
+
function admissionGenerationIs(state, indexName, partitionId, replicaNodeId, generation) {
|
|
3550
|
+
return state.admissionGenerations.get(partitionKey(indexName, partitionId))?.get(replicaNodeId) === generation;
|
|
3551
|
+
}
|
|
3552
|
+
function forgetRemovedReplica(state, indexName, partitionId, replicaNodeId) {
|
|
3553
|
+
const key = partitionKey(indexName, partitionId);
|
|
3554
|
+
const removed = state.removedReplicas.get(key);
|
|
3555
|
+
if (removed === void 0) return;
|
|
3556
|
+
removed.delete(replicaNodeId);
|
|
3557
|
+
if (removed.size === 0) state.removedReplicas.delete(key);
|
|
3558
|
+
}
|
|
3527
3559
|
function recordRemovedReplica(state, indexName, partitionId, replicaNodeId, primaryTerm) {
|
|
3528
3560
|
const key = partitionKey(indexName, partitionId);
|
|
3529
3561
|
const removed = state.removedReplicas.get(key) ?? /* @__PURE__ */ new Map();
|
|
@@ -3571,6 +3603,41 @@ function keepAcknowledgementsOf(state, ledPartitionKeys) {
|
|
|
3571
3603
|
}
|
|
3572
3604
|
}
|
|
3573
3605
|
|
|
3606
|
+
// src/distribution/cluster-node/catch-up/admitted.ts
|
|
3607
|
+
function recordAdmittedReplica(state, indexName, partitionId, replicaNodeId, primaryTerm, generation) {
|
|
3608
|
+
const key = partitionKey(indexName, partitionId);
|
|
3609
|
+
const admitted = state.admittedReplicas.get(key) ?? /* @__PURE__ */ new Map();
|
|
3610
|
+
admitted.set(replicaNodeId, { primaryTerm, generation });
|
|
3611
|
+
state.admittedReplicas.set(key, admitted);
|
|
3612
|
+
}
|
|
3613
|
+
function forgetAdmittedReplica(state, indexName, partitionId, replicaNodeId) {
|
|
3614
|
+
const key = partitionKey(indexName, partitionId);
|
|
3615
|
+
const admitted = state.admittedReplicas.get(key);
|
|
3616
|
+
if (admitted === void 0) return;
|
|
3617
|
+
admitted.delete(replicaNodeId);
|
|
3618
|
+
if (admitted.size === 0) state.admittedReplicas.delete(key);
|
|
3619
|
+
}
|
|
3620
|
+
function recordRemovalUnlessReadmitted(state, indexName, partitionId, replicaNodeId, primaryTerm, generationAtRemoval) {
|
|
3621
|
+
if (!admissionGenerationIs(state, indexName, partitionId, replicaNodeId, generationAtRemoval)) return;
|
|
3622
|
+
forgetAdmittedReplica(state, indexName, partitionId, replicaNodeId);
|
|
3623
|
+
recordRemovedReplica(state, indexName, partitionId, replicaNodeId, primaryTerm);
|
|
3624
|
+
}
|
|
3625
|
+
function admittedBeyondTable(state, indexName, partitionId, assignment, generationAtLiveRead) {
|
|
3626
|
+
const admitted = state.admittedReplicas.get(partitionKey(indexName, partitionId));
|
|
3627
|
+
if (admitted === void 0) return [];
|
|
3628
|
+
const beyond = [];
|
|
3629
|
+
for (const [replicaNodeId, record] of [...admitted]) {
|
|
3630
|
+
const settled = assignment.inSyncSet.includes(replicaNodeId) || !assignment.replicas.includes(replicaNodeId);
|
|
3631
|
+
const superseded = generationAtLiveRead !== void 0 && record.generation <= generationAtLiveRead;
|
|
3632
|
+
if (settled || superseded || record.primaryTerm !== assignment.primaryTerm) {
|
|
3633
|
+
forgetAdmittedReplica(state, indexName, partitionId, replicaNodeId);
|
|
3634
|
+
continue;
|
|
3635
|
+
}
|
|
3636
|
+
beyond.push(replicaNodeId);
|
|
3637
|
+
}
|
|
3638
|
+
return beyond;
|
|
3639
|
+
}
|
|
3640
|
+
|
|
3574
3641
|
// src/distribution/cluster-node/write-routing/replication.ts
|
|
3575
3642
|
async function assertPrimaryWriteAuthority(entry, deps) {
|
|
3576
3643
|
const table = await deps.coordinator.getAllocation(entry.indexName);
|
|
@@ -3642,6 +3709,7 @@ async function removeEachFailedReplica(entry, failedReplicas, deps) {
|
|
|
3642
3709
|
);
|
|
3643
3710
|
}
|
|
3644
3711
|
for (const replicaNodeId of failedReplicas) {
|
|
3712
|
+
const generationAtRemoval = admissionGenerationOf(deps.catchUp, entry.indexName, entry.partitionId, replicaNodeId);
|
|
3645
3713
|
const accepted = await requestInsyncRemovalWithTargets(
|
|
3646
3714
|
entry.indexName,
|
|
3647
3715
|
entry.partitionId,
|
|
@@ -3659,8 +3727,14 @@ async function removeEachFailedReplica(entry, failedReplicas, deps) {
|
|
|
3659
3727
|
{ indexName: entry.indexName, partitionId: entry.partitionId, replicaNodeId }
|
|
3660
3728
|
);
|
|
3661
3729
|
}
|
|
3662
|
-
|
|
3663
|
-
|
|
3730
|
+
recordRemovalUnlessReadmitted(
|
|
3731
|
+
deps.catchUp,
|
|
3732
|
+
entry.indexName,
|
|
3733
|
+
entry.partitionId,
|
|
3734
|
+
replicaNodeId,
|
|
3735
|
+
entry.primaryTerm,
|
|
3736
|
+
generationAtRemoval
|
|
3737
|
+
);
|
|
3664
3738
|
}
|
|
3665
3739
|
}
|
|
3666
3740
|
function replicaTargetsOf(entry, assignment, deps) {
|
|
@@ -7977,7 +8051,7 @@ function createAnalysisView(deps) {
|
|
|
7977
8051
|
answerAt(indexName, requested, read) {
|
|
7978
8052
|
const generation = engine.analysisGenerationOf(indexName);
|
|
7979
8053
|
if (generation === void 0) return read();
|
|
7980
|
-
const readAtRequested = (
|
|
8054
|
+
const readAtRequested = (answerOf2) => answerOf2().then((answer2) => {
|
|
7981
8055
|
const after = engine.analysisGenerationOf(indexName) ?? requested;
|
|
7982
8056
|
if (after !== requested) throw generationMissing(indexName, requested, after);
|
|
7983
8057
|
return answer2;
|
|
@@ -9621,22 +9695,17 @@ async function executeBootstrapSync(state, entry, indexName, partitionId, primar
|
|
|
9621
9695
|
// src/distribution/cluster-node/catch-up/admission.ts
|
|
9622
9696
|
async function sendToController(payload, controllerNodeId, deps) {
|
|
9623
9697
|
const targets = await resolveNodeTargets(controllerNodeId, deps);
|
|
9698
|
+
let sentWithoutAnswer = false;
|
|
9624
9699
|
for (const target of targets) {
|
|
9625
9700
|
try {
|
|
9626
9701
|
const result = await requestInsyncAdmission(payload, target, deps.transport, deps.nodeId);
|
|
9627
|
-
|
|
9702
|
+
if (result.accepted) return "accepted";
|
|
9703
|
+
return sentWithoutAnswer ? "unanswered" : "refused";
|
|
9628
9704
|
} catch (_) {
|
|
9705
|
+
sentWithoutAnswer = true;
|
|
9629
9706
|
}
|
|
9630
9707
|
}
|
|
9631
|
-
return
|
|
9632
|
-
}
|
|
9633
|
-
async function versionAfterAdmission(deps, indexName) {
|
|
9634
|
-
if (deps.readLiveAllocation === void 0) return null;
|
|
9635
|
-
try {
|
|
9636
|
-
return (await deps.readLiveAllocation(indexName))?.version ?? null;
|
|
9637
|
-
} catch (_) {
|
|
9638
|
-
return null;
|
|
9639
|
-
}
|
|
9708
|
+
return "unanswered";
|
|
9640
9709
|
}
|
|
9641
9710
|
function withTimeout(work, timeoutMs, fallback) {
|
|
9642
9711
|
return new Promise((resolve) => {
|
|
@@ -9651,25 +9720,26 @@ function withTimeout(work, timeoutMs, fallback) {
|
|
|
9651
9720
|
});
|
|
9652
9721
|
});
|
|
9653
9722
|
}
|
|
9654
|
-
async function withdrawAdmission(
|
|
9723
|
+
async function withdrawAdmission(removal, controllerNodeId, deps) {
|
|
9655
9724
|
const targets = await resolveNodeTargets(controllerNodeId, deps);
|
|
9656
9725
|
for (const target of targets) {
|
|
9657
9726
|
try {
|
|
9658
9727
|
const result = await requestInsyncRemoval(
|
|
9659
|
-
|
|
9660
|
-
|
|
9661
|
-
|
|
9662
|
-
|
|
9728
|
+
removal.indexName,
|
|
9729
|
+
removal.partitionId,
|
|
9730
|
+
removal.replicaNodeId,
|
|
9731
|
+
removal.primaryTerm,
|
|
9663
9732
|
target,
|
|
9664
9733
|
deps.transport,
|
|
9665
9734
|
deps.nodeId
|
|
9666
9735
|
);
|
|
9667
9736
|
if (result.accepted) {
|
|
9668
|
-
return;
|
|
9737
|
+
return true;
|
|
9669
9738
|
}
|
|
9670
9739
|
} catch (_) {
|
|
9671
9740
|
}
|
|
9672
9741
|
}
|
|
9742
|
+
return false;
|
|
9673
9743
|
}
|
|
9674
9744
|
async function proposeAdmission(state, deps, indexName, partitionId, assignment, replicaNodeId, cursor) {
|
|
9675
9745
|
if (claimAdmission(state, deps, indexName, partitionId, replicaNodeId, cursor)) {
|
|
@@ -9699,17 +9769,25 @@ async function requestClaimedAdmission(state, deps, indexName, partitionId, assi
|
|
|
9699
9769
|
appliedSeqNo: cursor.appliedSeqNo,
|
|
9700
9770
|
commitPoint: log.commitPoint
|
|
9701
9771
|
};
|
|
9702
|
-
const
|
|
9703
|
-
|
|
9704
|
-
|
|
9705
|
-
|
|
9772
|
+
const answer2 = await withTimeout(
|
|
9773
|
+
sendToController(payload, controllerNodeId, deps),
|
|
9774
|
+
ADMISSION_TIMEOUT_MS,
|
|
9775
|
+
"unanswered"
|
|
9776
|
+
);
|
|
9777
|
+
const outlived = answer2 === "accepted" && cursor.syncEpoch !== proposedEpoch;
|
|
9778
|
+
const withdrawn = outlived && await withTimeout(withdrawAdmission(payload, controllerNodeId, deps), ADMISSION_TIMEOUT_MS, false);
|
|
9779
|
+
if (answer2 === "unanswered" || outlived && !withdrawn) {
|
|
9780
|
+
recordAdmissionToWithdraw(state, indexName, partitionId, replicaNodeId, assignment.primaryTerm);
|
|
9781
|
+
} else if (answer2 === "accepted" && !outlived) {
|
|
9782
|
+
forgetAdmissionToWithdraw(state, indexName, partitionId, replicaNodeId);
|
|
9783
|
+
forgetRemovedReplica(state, indexName, partitionId, replicaNodeId);
|
|
9706
9784
|
recordAdmittedReplica(
|
|
9707
9785
|
state,
|
|
9708
9786
|
indexName,
|
|
9709
9787
|
partitionId,
|
|
9710
9788
|
replicaNodeId,
|
|
9711
9789
|
assignment.primaryTerm,
|
|
9712
|
-
|
|
9790
|
+
advanceAdmissionGeneration(state, indexName, partitionId, replicaNodeId)
|
|
9713
9791
|
);
|
|
9714
9792
|
}
|
|
9715
9793
|
} finally {
|
|
@@ -9753,6 +9831,43 @@ function untilCancelled(feed, work, fallback) {
|
|
|
9753
9831
|
return Promise.race([work, feed.whenCancelled.then(() => fallback)]);
|
|
9754
9832
|
}
|
|
9755
9833
|
|
|
9834
|
+
// src/distribution/cluster-node/catch-up/withdrawals.ts
|
|
9835
|
+
async function withdrawAppliedAdmission(state, deps, indexName, partitionId, replicaNodeId, owed) {
|
|
9836
|
+
owed.withdrawing = true;
|
|
9837
|
+
try {
|
|
9838
|
+
const controllerNodeId = await deps.coordinator.getLeaseHolder(CONTROLLER_LEASE_KEY);
|
|
9839
|
+
if (controllerNodeId === null) return;
|
|
9840
|
+
const removal = { indexName, partitionId, replicaNodeId, primaryTerm: owed.primaryTerm };
|
|
9841
|
+
const withdrawn = await withTimeout(withdrawAdmission(removal, controllerNodeId, deps), ADMISSION_TIMEOUT_MS, false);
|
|
9842
|
+
if (!withdrawn || state.admissionsToWithdraw.get(partitionKey(indexName, partitionId))?.get(replicaNodeId) !== owed) {
|
|
9843
|
+
return;
|
|
9844
|
+
}
|
|
9845
|
+
forgetAdmissionToWithdraw(state, indexName, partitionId, replicaNodeId);
|
|
9846
|
+
forgetAdmittedReplica(state, indexName, partitionId, replicaNodeId);
|
|
9847
|
+
if (!getPendingAdmissions(state, indexName, partitionId).includes(replicaNodeId)) {
|
|
9848
|
+
recordRemovedReplica(state, indexName, partitionId, replicaNodeId, owed.primaryTerm);
|
|
9849
|
+
}
|
|
9850
|
+
} finally {
|
|
9851
|
+
owed.withdrawing = false;
|
|
9852
|
+
}
|
|
9853
|
+
}
|
|
9854
|
+
function withdrawAppliedAdmissions(state, deps, indexName, partitionId, assignment) {
|
|
9855
|
+
const owedByReplica = state.admissionsToWithdraw.get(partitionKey(indexName, partitionId));
|
|
9856
|
+
if (owedByReplica === void 0) return [];
|
|
9857
|
+
const pending = getPendingAdmissions(state, indexName, partitionId);
|
|
9858
|
+
const started = [];
|
|
9859
|
+
for (const [replicaNodeId, owed] of [...owedByReplica]) {
|
|
9860
|
+
if (owed.primaryTerm !== assignment.primaryTerm || !assignment.replicas.includes(replicaNodeId)) {
|
|
9861
|
+
forgetAdmissionToWithdraw(state, indexName, partitionId, replicaNodeId);
|
|
9862
|
+
continue;
|
|
9863
|
+
}
|
|
9864
|
+
if (owed.withdrawing || pending.includes(replicaNodeId) || !assignment.inSyncSet.includes(replicaNodeId)) continue;
|
|
9865
|
+
started.push(withdrawAppliedAdmission(state, deps, indexName, partitionId, replicaNodeId, owed).catch(() => {
|
|
9866
|
+
}));
|
|
9867
|
+
}
|
|
9868
|
+
return started;
|
|
9869
|
+
}
|
|
9870
|
+
|
|
9756
9871
|
// src/distribution/cluster-node/catch-up/pump.ts
|
|
9757
9872
|
function entryBytes(entry) {
|
|
9758
9873
|
return REPLICATION_ENTRY_FIXED_OVERHEAD_BYTES + entry.indexName.length + entry.documentId.length + (entry.document?.byteLength ?? 0);
|
|
@@ -9888,7 +10003,7 @@ async function sendBatch(state, deps, batch, replicaNodeId, cursor, feed) {
|
|
|
9888
10003
|
if (!acknowledged || feed.cancelled || cursor.syncEpoch !== sendingEpoch) {
|
|
9889
10004
|
return false;
|
|
9890
10005
|
}
|
|
9891
|
-
cursor.appliedSeqNo = batch[batch.length - 1].seqNo;
|
|
10006
|
+
cursor.appliedSeqNo = Math.max(cursor.appliedSeqNo, batch[batch.length - 1].seqNo);
|
|
9892
10007
|
return true;
|
|
9893
10008
|
}
|
|
9894
10009
|
function cancelFeeds(replicas) {
|
|
@@ -9908,25 +10023,41 @@ async function pumpPartition(state, deps, key, replicas, started, registered) {
|
|
|
9908
10023
|
const separator = key.lastIndexOf(":");
|
|
9909
10024
|
const indexName = key.slice(0, separator);
|
|
9910
10025
|
const partitionId = Number(key.slice(separator + 1));
|
|
10026
|
+
const generationAtLiveRead = state.lastAdmissionGeneration;
|
|
9911
10027
|
const table = await (deps.readLiveAllocation?.(indexName) ?? deps.coordinator.getAllocation(indexName));
|
|
9912
10028
|
const assignment = table?.assignments.get(partitionId);
|
|
9913
10029
|
if (assignment === void 0 || assignment.primary !== deps.nodeId) {
|
|
9914
10030
|
cancelFeeds(replicas);
|
|
9915
10031
|
state.cursors.delete(key);
|
|
9916
10032
|
state.pendingAdmissions.delete(key);
|
|
10033
|
+
state.admissionsToWithdraw.delete(key);
|
|
9917
10034
|
return;
|
|
9918
10035
|
}
|
|
10036
|
+
for (const withdrawal of withdrawAppliedAdmissions(state, deps, indexName, partitionId, assignment)) {
|
|
10037
|
+
const work = withdrawal.finally(() => state.replicaWork.delete(work));
|
|
10038
|
+
state.replicaWork.add(work);
|
|
10039
|
+
started.push(work);
|
|
10040
|
+
}
|
|
9919
10041
|
for (const [replicaNodeId, cursor] of [...replicas]) {
|
|
9920
10042
|
if (state.stopped) {
|
|
9921
10043
|
return;
|
|
9922
10044
|
}
|
|
9923
|
-
const admitted = admittedBeyondTable(state, indexName, partitionId, assignment,
|
|
10045
|
+
const admitted = admittedBeyondTable(state, indexName, partitionId, assignment, generationAtLiveRead).includes(
|
|
9924
10046
|
replicaNodeId
|
|
9925
10047
|
);
|
|
9926
|
-
if (!assignment.replicas.includes(replicaNodeId)
|
|
10048
|
+
if (!assignment.replicas.includes(replicaNodeId)) {
|
|
9927
10049
|
forgetReplica(state, indexName, partitionId, replicaNodeId);
|
|
9928
10050
|
continue;
|
|
9929
10051
|
}
|
|
10052
|
+
if (admitted) {
|
|
10053
|
+
continue;
|
|
10054
|
+
}
|
|
10055
|
+
if (assignment.inSyncSet.includes(replicaNodeId)) {
|
|
10056
|
+
if (!getPendingAdmissions(state, indexName, partitionId).includes(replicaNodeId)) {
|
|
10057
|
+
forgetReplica(state, indexName, partitionId, replicaNodeId);
|
|
10058
|
+
}
|
|
10059
|
+
continue;
|
|
10060
|
+
}
|
|
9930
10061
|
if (cursor.busy) {
|
|
9931
10062
|
if (cursor.feed !== null && feedIsStale(cursor.feed, assignment, registered, replicaNodeId)) {
|
|
9932
10063
|
cursor.feed.cancel();
|
|
@@ -9952,17 +10083,19 @@ async function pumpPartition(state, deps, key, replicas, started, registered) {
|
|
|
9952
10083
|
}
|
|
9953
10084
|
}
|
|
9954
10085
|
function runCatchUpTick(state, deps) {
|
|
9955
|
-
|
|
10086
|
+
const idle = state.cursors.size === 0 && state.admissionsToWithdraw.size === 0;
|
|
10087
|
+
if (state.stopped || state.activeTick !== null || idle) {
|
|
9956
10088
|
return state.activeTick ?? Promise.resolve();
|
|
9957
10089
|
}
|
|
9958
10090
|
const started = [];
|
|
9959
10091
|
const tick = (async () => {
|
|
9960
10092
|
const registered = await registeredNodes(deps);
|
|
9961
|
-
for (const
|
|
10093
|
+
for (const key of /* @__PURE__ */ new Set([...state.cursors.keys(), ...state.admissionsToWithdraw.keys()])) {
|
|
9962
10094
|
if (state.stopped) {
|
|
9963
10095
|
return;
|
|
9964
10096
|
}
|
|
9965
10097
|
try {
|
|
10098
|
+
const replicas = state.cursors.get(key) ?? /* @__PURE__ */ new Map();
|
|
9966
10099
|
await pumpPartition(state, deps, key, replicas, started, registered);
|
|
9967
10100
|
} catch (_) {
|
|
9968
10101
|
}
|
|
@@ -14816,7 +14949,7 @@ async function openOnce(core, indexName, partitionId) {
|
|
|
14816
14949
|
if (directory === void 0) throw durabilityRequired(indexName, partitionId);
|
|
14817
14950
|
if ((checkpoint.manifest.analysisGeneration ?? 0) !== entry.analysisGeneration) return null;
|
|
14818
14951
|
const config = await encodeIndexConfiguration(entry);
|
|
14819
|
-
const { openPagedPartitionArchive } = await import('./paged-partition-archive-
|
|
14952
|
+
const { openPagedPartitionArchive } = await import('./paged-partition-archive-IK2UOY2G.mjs');
|
|
14820
14953
|
const archive = await openPagedPartitionArchive(checkpoint, directory, indexName, partitionId, manager);
|
|
14821
14954
|
handedOver = true;
|
|
14822
14955
|
try {
|
|
@@ -15524,7 +15657,7 @@ async function archiveReplicationPartition(core, indexName, partitionId, lastSeq
|
|
|
15524
15657
|
);
|
|
15525
15658
|
}
|
|
15526
15659
|
try {
|
|
15527
|
-
const { openPagedPartitionArchive } = await import('./paged-partition-archive-
|
|
15660
|
+
const { openPagedPartitionArchive } = await import('./paged-partition-archive-IK2UOY2G.mjs');
|
|
15528
15661
|
const archive = await openPagedPartitionArchive(checkpoint2, directory, indexName, partitionId, manager);
|
|
15529
15662
|
return {
|
|
15530
15663
|
stream: archiveStream({ ...archive, config, sourceIndexName: indexName }),
|
|
@@ -15739,7 +15872,7 @@ async function restoreReplicationPartition(core, engine, indexName, partitionId,
|
|
|
15739
15872
|
if (directory !== null) {
|
|
15740
15873
|
const durable = core.durability?.manager;
|
|
15741
15874
|
await (durable?.checkpointFromMemory?.(indexName) ?? durable?.checkpoint(indexName));
|
|
15742
|
-
const { removeReceivedFilesNothingReads } = await import('./paged-partition-archive-
|
|
15875
|
+
const { removeReceivedFilesNothingReads } = await import('./paged-partition-archive-IK2UOY2G.mjs');
|
|
15743
15876
|
await removeReceivedFilesNothingReads(
|
|
15744
15877
|
directory,
|
|
15745
15878
|
core.requireManager(indexName),
|
|
@@ -15940,7 +16073,13 @@ async function createClusterLocalEngine(config, hooks) {
|
|
|
15940
16073
|
partitionStatsForRead: async (indexName) => {
|
|
15941
16074
|
const release = await core.indexState.acquire(indexName);
|
|
15942
16075
|
try {
|
|
15943
|
-
|
|
16076
|
+
const stats = core.requireManager(indexName).getPartitionStats();
|
|
16077
|
+
const durable = core.durability?.manager;
|
|
16078
|
+
durable?.addStoredSizes(indexName, stats);
|
|
16079
|
+
return {
|
|
16080
|
+
partitions: stats,
|
|
16081
|
+
vectorSavedBytes: durable?.vectorSavedBytes?.(indexName)
|
|
16082
|
+
};
|
|
15944
16083
|
} finally {
|
|
15945
16084
|
release();
|
|
15946
16085
|
}
|
|
@@ -18978,6 +19117,48 @@ function createRankedDocumentReader(deps, indexName, allocation) {
|
|
|
18978
19117
|
};
|
|
18979
19118
|
}
|
|
18980
19119
|
|
|
19120
|
+
// src/distribution/cluster-node/reads/count-answer.ts
|
|
19121
|
+
function vectorSharesOf(held, vectorBytes) {
|
|
19122
|
+
const shares = /* @__PURE__ */ new Map();
|
|
19123
|
+
if (held.length === 0) return shares;
|
|
19124
|
+
const total = BigInt(vectorBytes);
|
|
19125
|
+
let documents = 0n;
|
|
19126
|
+
let lowestPartitionId = held[0].partitionId;
|
|
19127
|
+
for (const partition of held) {
|
|
19128
|
+
documents += BigInt(partition.documentCount);
|
|
19129
|
+
lowestPartitionId = Math.min(lowestPartitionId, partition.partitionId);
|
|
19130
|
+
}
|
|
19131
|
+
let given = 0n;
|
|
19132
|
+
for (const partition of held) {
|
|
19133
|
+
const share = documents > 0n ? total * BigInt(partition.documentCount) / documents : total / BigInt(held.length);
|
|
19134
|
+
shares.set(partition.partitionId, Number(share));
|
|
19135
|
+
given += share;
|
|
19136
|
+
}
|
|
19137
|
+
shares.set(lowestPartitionId, (shares.get(lowestPartitionId) ?? 0) + Number(total - given));
|
|
19138
|
+
return shares;
|
|
19139
|
+
}
|
|
19140
|
+
function countEntryOf(partition, vectorShare) {
|
|
19141
|
+
return {
|
|
19142
|
+
partitionId: partition.partitionId,
|
|
19143
|
+
documentCount: partition.documentCount,
|
|
19144
|
+
estimatedMemoryBytes: partition.estimatedMemoryBytes,
|
|
19145
|
+
...partition.savedBytes === void 0 ? {} : { savedBytes: partition.savedBytes },
|
|
19146
|
+
...vectorShare === void 0 ? {} : { vectorSavedBytes: vectorShare },
|
|
19147
|
+
...partition.logBytes === void 0 ? {} : { logBytes: partition.logBytes }
|
|
19148
|
+
};
|
|
19149
|
+
}
|
|
19150
|
+
async function localCountAnswer(engine, indexName, partitionIds) {
|
|
19151
|
+
const { partitions, vectorSavedBytes } = await engine.partitionStatsForRead(indexName);
|
|
19152
|
+
const heldIds = engine.heldPartitionsOf(indexName);
|
|
19153
|
+
const held = heldIds === void 0 ? partitions : partitions.filter((partition) => heldIds.includes(partition.partitionId));
|
|
19154
|
+
const shares = vectorSavedBytes === void 0 ? /* @__PURE__ */ new Map() : vectorSharesOf(held, vectorSavedBytes);
|
|
19155
|
+
const requested = new Set(partitionIds);
|
|
19156
|
+
return {
|
|
19157
|
+
partitions: partitions.filter((partition) => requested.has(partition.partitionId)).map((partition) => countEntryOf(partition, shares.get(partition.partitionId))),
|
|
19158
|
+
language: engine.getStats(indexName).language
|
|
19159
|
+
};
|
|
19160
|
+
}
|
|
19161
|
+
|
|
18981
19162
|
// src/distribution/cluster-node/reads/scatter.ts
|
|
18982
19163
|
async function gatherFromEveryPartition(allocation, indexName, read) {
|
|
18983
19164
|
const routing = selectReplicasForQuery(allocation, randomSelector);
|
|
@@ -19004,32 +19185,87 @@ async function sendReadRequest(deps, targetNodeId, message, indexName, validate,
|
|
|
19004
19185
|
return validate(decoded);
|
|
19005
19186
|
}
|
|
19006
19187
|
|
|
19188
|
+
// src/distribution/cluster-node/reads/every-copy.ts
|
|
19189
|
+
function otherCopiesByNode(allocation, readBy) {
|
|
19190
|
+
const byNode = /* @__PURE__ */ new Map();
|
|
19191
|
+
for (const [partitionId, assignment] of allocation.assignments) {
|
|
19192
|
+
const copies = new Set(assignment.replicas);
|
|
19193
|
+
if (assignment.primary !== null) copies.add(assignment.primary);
|
|
19194
|
+
for (const nodeId of copies) {
|
|
19195
|
+
if (nodeId === readBy.get(partitionId)) continue;
|
|
19196
|
+
const partitionIds = byNode.get(nodeId) ?? [];
|
|
19197
|
+
partitionIds.push(partitionId);
|
|
19198
|
+
byNode.set(nodeId, partitionIds);
|
|
19199
|
+
}
|
|
19200
|
+
}
|
|
19201
|
+
return byNode;
|
|
19202
|
+
}
|
|
19203
|
+
function answerOf(deps, indexName, nodeId, partitionIds) {
|
|
19204
|
+
if (nodeId === deps.nodeId) return localCountAnswer(deps.engine, indexName, partitionIds);
|
|
19205
|
+
const message = createCountMessage({ indexName, partitionIds }, deps.nodeId);
|
|
19206
|
+
return sendReadRequest(deps, nodeId, message, indexName, validateCountResultPayload);
|
|
19207
|
+
}
|
|
19208
|
+
function addAnswer(copies, partitionIds, answer2) {
|
|
19209
|
+
const answered = new Map(answer2.partitions.map((entry) => [entry.partitionId, entry]));
|
|
19210
|
+
for (const partitionId of partitionIds) {
|
|
19211
|
+
const entry = answered.get(partitionId);
|
|
19212
|
+
if (entry === void 0) return false;
|
|
19213
|
+
copies.get(partitionId)?.push(entry);
|
|
19214
|
+
}
|
|
19215
|
+
return true;
|
|
19216
|
+
}
|
|
19217
|
+
async function countsOfEveryCopy(deps, indexName, allocation, readCopies, readBy) {
|
|
19218
|
+
const copies = /* @__PURE__ */ new Map();
|
|
19219
|
+
for (const [partitionId, entry] of readCopies) copies.set(partitionId, [entry]);
|
|
19220
|
+
const requests = [...otherCopiesByNode(allocation, readBy)].map(async ([nodeId, partitionIds]) => {
|
|
19221
|
+
const answer2 = await answerOf(deps, indexName, nodeId, partitionIds);
|
|
19222
|
+
return { partitionIds, answer: answer2 };
|
|
19223
|
+
});
|
|
19224
|
+
let answers;
|
|
19225
|
+
try {
|
|
19226
|
+
answers = await Promise.all(requests);
|
|
19227
|
+
} catch {
|
|
19228
|
+
return null;
|
|
19229
|
+
}
|
|
19230
|
+
for (const { partitionIds, answer: answer2 } of answers) {
|
|
19231
|
+
if (!addAnswer(copies, partitionIds, answer2)) return null;
|
|
19232
|
+
}
|
|
19233
|
+
return copies;
|
|
19234
|
+
}
|
|
19235
|
+
function sumOf(entries, figure2) {
|
|
19236
|
+
let total = 0;
|
|
19237
|
+
for (const entry of entries) {
|
|
19238
|
+
const value = figure2(entry);
|
|
19239
|
+
if (value === void 0) return void 0;
|
|
19240
|
+
total += value;
|
|
19241
|
+
}
|
|
19242
|
+
return total;
|
|
19243
|
+
}
|
|
19244
|
+
function savedBytesWithVectors(entry) {
|
|
19245
|
+
if (entry.savedBytes === void 0 || entry.vectorSavedBytes === void 0) return void 0;
|
|
19246
|
+
return entry.savedBytes + entry.vectorSavedBytes;
|
|
19247
|
+
}
|
|
19248
|
+
|
|
19007
19249
|
// src/distribution/cluster-node/reads/counts.ts
|
|
19008
19250
|
async function gatherPartitionCounts(deps, indexName, allocation) {
|
|
19009
19251
|
const countsByPartition = /* @__PURE__ */ new Map();
|
|
19252
|
+
const readBy = /* @__PURE__ */ new Map();
|
|
19010
19253
|
let language = "";
|
|
19011
19254
|
const gathered = await gatherFromEveryPartition(allocation, indexName, async (group) => {
|
|
19012
19255
|
if (group.nodeId === deps.nodeId) {
|
|
19013
|
-
|
|
19014
|
-
const partitions = (await deps.engine.partitionStatsForRead(indexName)).filter((partition) => requested.has(partition.partitionId)).map((partition) => ({
|
|
19015
|
-
partitionId: partition.partitionId,
|
|
19016
|
-
documentCount: partition.documentCount,
|
|
19017
|
-
estimatedMemoryBytes: partition.estimatedMemoryBytes
|
|
19018
|
-
}));
|
|
19019
|
-
return { partitions, language: deps.engine.getStats(indexName).language };
|
|
19256
|
+
return { nodeId: group.nodeId, answer: await localCountAnswer(deps.engine, indexName, group.partitionIds) };
|
|
19020
19257
|
}
|
|
19021
19258
|
const message = createCountMessage({ indexName, partitionIds: group.partitionIds }, deps.nodeId);
|
|
19022
|
-
|
|
19259
|
+
const answer2 = await sendReadRequest(deps, group.nodeId, message, indexName, validateCountResultPayload);
|
|
19260
|
+
return { nodeId: group.nodeId, answer: answer2 };
|
|
19023
19261
|
});
|
|
19024
|
-
for (const
|
|
19262
|
+
for (const { nodeId, answer: answer2 } of gathered) {
|
|
19025
19263
|
if (language.length === 0) {
|
|
19026
|
-
language =
|
|
19264
|
+
language = answer2.language;
|
|
19027
19265
|
}
|
|
19028
|
-
for (const partition of
|
|
19029
|
-
countsByPartition.set(partition.partitionId,
|
|
19030
|
-
|
|
19031
|
-
estimatedMemoryBytes: partition.estimatedMemoryBytes
|
|
19032
|
-
});
|
|
19266
|
+
for (const partition of answer2.partitions) {
|
|
19267
|
+
countsByPartition.set(partition.partitionId, partition);
|
|
19268
|
+
readBy.set(partition.partitionId, nodeId);
|
|
19033
19269
|
}
|
|
19034
19270
|
}
|
|
19035
19271
|
const missing = [];
|
|
@@ -19045,7 +19281,10 @@ async function gatherPartitionCounts(deps, indexName, allocation) {
|
|
|
19045
19281
|
{ indexName, partitionIds: missing.sort((a, b) => a - b) }
|
|
19046
19282
|
);
|
|
19047
19283
|
}
|
|
19048
|
-
return { countsByPartition, language };
|
|
19284
|
+
return { countsByPartition, readBy, language };
|
|
19285
|
+
}
|
|
19286
|
+
function figure(name, value) {
|
|
19287
|
+
return value === void 0 ? {} : { [name]: value };
|
|
19049
19288
|
}
|
|
19050
19289
|
async function countCluster(deps, indexName) {
|
|
19051
19290
|
const allocation = await routableAllocation(deps.config.coordinator, indexName);
|
|
@@ -19070,17 +19309,26 @@ async function statsCluster(deps, indexName) {
|
|
|
19070
19309
|
indexName
|
|
19071
19310
|
});
|
|
19072
19311
|
}
|
|
19073
|
-
const { countsByPartition, language } = await gatherPartitionCounts(deps, indexName, allocation);
|
|
19312
|
+
const { countsByPartition, readBy, language } = await gatherPartitionCounts(deps, indexName, allocation);
|
|
19313
|
+
const everyCopy = await countsOfEveryCopy(deps, indexName, allocation, countsByPartition, readBy);
|
|
19074
19314
|
let documentCount = 0;
|
|
19075
19315
|
let estimatedMemoryBytes = 0;
|
|
19076
19316
|
for (const entry of countsByPartition.values()) {
|
|
19077
19317
|
documentCount += entry.documentCount;
|
|
19078
19318
|
estimatedMemoryBytes += entry.estimatedMemoryBytes;
|
|
19079
19319
|
}
|
|
19320
|
+
const allCopies = everyCopy === null ? null : [...everyCopy.values()].flat();
|
|
19080
19321
|
return {
|
|
19081
19322
|
documentCount,
|
|
19082
19323
|
partitionCount: allocation.assignments.size,
|
|
19083
19324
|
estimatedMemoryBytes,
|
|
19325
|
+
...figure("savedBytes", sumOf(countsByPartition.values(), savedBytesWithVectors)),
|
|
19326
|
+
...figure(
|
|
19327
|
+
"logBytes",
|
|
19328
|
+
sumOf(countsByPartition.values(), (entry) => entry.logBytes)
|
|
19329
|
+
),
|
|
19330
|
+
...figure("savedBytesOfEveryCopy", allCopies === null ? void 0 : sumOf(allCopies, savedBytesWithVectors)),
|
|
19331
|
+
...figure("logBytesOfEveryCopy", allCopies === null ? void 0 : sumOf(allCopies, (entry) => entry.logBytes)),
|
|
19084
19332
|
language,
|
|
19085
19333
|
schema
|
|
19086
19334
|
};
|
|
@@ -19088,14 +19336,22 @@ async function statsCluster(deps, indexName) {
|
|
|
19088
19336
|
async function partitionStatsCluster(deps, indexName) {
|
|
19089
19337
|
const allocation = await routableAllocation(deps.config.coordinator, indexName);
|
|
19090
19338
|
if (allocation === null) {
|
|
19091
|
-
return deps.engine.partitionStatsForRead(indexName);
|
|
19339
|
+
return (await deps.engine.partitionStatsForRead(indexName)).partitions;
|
|
19092
19340
|
}
|
|
19093
|
-
const { countsByPartition } = await gatherPartitionCounts(deps, indexName, allocation);
|
|
19094
|
-
|
|
19095
|
-
|
|
19096
|
-
|
|
19097
|
-
|
|
19098
|
-
|
|
19341
|
+
const { countsByPartition, readBy } = await gatherPartitionCounts(deps, indexName, allocation);
|
|
19342
|
+
const everyCopy = await countsOfEveryCopy(deps, indexName, allocation, countsByPartition, readBy);
|
|
19343
|
+
return Array.from(countsByPartition.entries()).map(([partitionId, entry]) => {
|
|
19344
|
+
const copies = everyCopy?.get(partitionId);
|
|
19345
|
+
return {
|
|
19346
|
+
partitionId,
|
|
19347
|
+
documentCount: entry.documentCount,
|
|
19348
|
+
estimatedMemoryBytes: entry.estimatedMemoryBytes,
|
|
19349
|
+
...figure("savedBytes", entry.savedBytes),
|
|
19350
|
+
...figure("logBytes", entry.logBytes),
|
|
19351
|
+
...figure("savedBytesOfEveryCopy", copies === void 0 ? void 0 : sumOf(copies, (copy) => copy.savedBytes)),
|
|
19352
|
+
...figure("logBytesOfEveryCopy", copies === void 0 ? void 0 : sumOf(copies, (copy) => copy.logBytes))
|
|
19353
|
+
};
|
|
19354
|
+
}).sort((a, b) => a.partitionId - b.partitionId);
|
|
19099
19355
|
}
|
|
19100
19356
|
|
|
19101
19357
|
// src/distribution/cluster-node/reads/index-list.ts
|
|
@@ -19421,7 +19677,7 @@ async function queryCluster(deps, indexName, params, settingName) {
|
|
|
19421
19677
|
(reader) => searchCluster(deps, indexName, settingName, params, allocation, reader, false)
|
|
19422
19678
|
);
|
|
19423
19679
|
}
|
|
19424
|
-
const { rerankClusterQuery } = await import('./rerank-
|
|
19680
|
+
const { rerankClusterQuery } = await import('./rerank-LYL3D7LU.mjs');
|
|
19425
19681
|
return withRankedReader(
|
|
19426
19682
|
deps,
|
|
19427
19683
|
indexName,
|
|
@@ -21311,14 +21567,8 @@ function convertLocalFacetsToWire2(facets) {
|
|
|
21311
21567
|
// src/distribution/cluster-node/message-handler/read-handlers.ts
|
|
21312
21568
|
async function handleCount(message, respond, deps) {
|
|
21313
21569
|
const payload = validateCountPayload(decode(message.payload));
|
|
21314
|
-
const
|
|
21315
|
-
|
|
21316
|
-
partitionId: partition.partitionId,
|
|
21317
|
-
documentCount: partition.documentCount,
|
|
21318
|
-
estimatedMemoryBytes: partition.estimatedMemoryBytes
|
|
21319
|
-
}));
|
|
21320
|
-
const language = deps.engine.getStats(payload.indexName).language;
|
|
21321
|
-
await respond(createCountResultMessage({ partitions, language }, deps.nodeId, message.requestId));
|
|
21570
|
+
const answer2 = await localCountAnswer(deps.engine, payload.indexName, payload.partitionIds);
|
|
21571
|
+
await respond(createCountResultMessage(answer2, deps.nodeId, message.requestId));
|
|
21322
21572
|
}
|
|
21323
21573
|
async function handleList(message, respond, deps) {
|
|
21324
21574
|
const payload = validateListPayload(decode(message.payload));
|
|
@@ -21826,24 +22076,27 @@ async function handleSyncRequestMessage(message, respond, deps) {
|
|
|
21826
22076
|
);
|
|
21827
22077
|
}
|
|
21828
22078
|
const release = await deps.engine.acquireIndexForReplication(request.indexName);
|
|
21829
|
-
const
|
|
21830
|
-
if (message.sourceId === deps.nodeId) return;
|
|
21831
|
-
recordReplicaPosition(
|
|
21832
|
-
deps.writeDeps.catchUp,
|
|
21833
|
-
request.indexName,
|
|
21834
|
-
request.partitionId,
|
|
21835
|
-
message.sourceId,
|
|
21836
|
-
appliedSeqNo
|
|
21837
|
-
);
|
|
21838
|
-
};
|
|
22079
|
+
const fromReplica = message.sourceId !== deps.nodeId;
|
|
21839
22080
|
try {
|
|
21840
22081
|
const log = deps.writeDeps.getReplicationLog(request.indexName, request.partitionId);
|
|
21841
22082
|
const tier = decideSyncTier(log, request.lastSeqNo, request.lastPrimaryTerm);
|
|
21842
22083
|
if (tier === "incremental") {
|
|
21843
|
-
|
|
22084
|
+
if (fromReplica) {
|
|
22085
|
+
recordReplicaPosition(
|
|
22086
|
+
deps.writeDeps.catchUp,
|
|
22087
|
+
request.indexName,
|
|
22088
|
+
request.partitionId,
|
|
22089
|
+
message.sourceId,
|
|
22090
|
+
request.lastSeqNo,
|
|
22091
|
+
log.localLogEnd
|
|
22092
|
+
);
|
|
22093
|
+
}
|
|
21844
22094
|
await sendSyncEntriesResponse(message, respond, deps, log.getEntriesFrom(request.lastSeqNo + 1));
|
|
21845
22095
|
return;
|
|
21846
22096
|
}
|
|
22097
|
+
if (fromReplica) {
|
|
22098
|
+
recordSnapshotSyncRequest(deps.writeDeps.catchUp, request.indexName, request.partitionId, message.sourceId);
|
|
22099
|
+
}
|
|
21847
22100
|
const snapshotSeqNo = log.localLogEnd;
|
|
21848
22101
|
let snapshotPrimaryTerm = log.termAt(snapshotSeqNo) ?? 0;
|
|
21849
22102
|
const sink = createSingleResponseSink(respond);
|