@delali/narsil 0.4.5 → 0.4.6-next.333
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/browser/{chunk-VAPCFSMX.mjs → chunk-3PGIOK7Z.mjs} +16 -1
- package/dist/browser/{chunk-263ERYKA.mjs → chunk-75MBWQFW.mjs} +71 -27
- package/dist/browser/{chunk-VIVG3OXM.mjs → chunk-FBDTHGDR.mjs} +7 -3
- package/dist/browser/{chunk-56UNHLAF.mjs → chunk-HIC42RBU.mjs} +1 -1
- package/dist/browser/{optional-reads-NQO5DKM4.mjs → optional-reads-4O4CJ7TR.mjs} +1 -1
- package/dist/browser/{runtime-OMKE7DF4.mjs → runtime-AMEB4J2D.mjs} +2 -1
- package/dist/browser/{segment-codec-leaf-GXSRX5VT.mjs → segment-codec-leaf-NDVBIBJJ.mjs} +3 -0
- package/dist/browser/{storage-leaf-42IZEOJJ.mjs → storage-leaf-NCOXQTXQ.mjs} +32 -12
- package/dist/{capabilities-DaJKuKlG.d.ts → capabilities-YazdElfW.d.ts} +2 -2
- package/dist/{chunk-KGLBLKYF.mjs → chunk-26AKUBDX.mjs} +66 -51
- package/dist/{react-native/chunks/chunk-3BEUTRYM.mjs → chunk-2M55WPWK.mjs} +1 -1
- package/dist/{chunk-AAH5CD3M.mjs → chunk-34QTB6O7.mjs} +1 -1
- package/dist/{chunk-GLPBA2YZ.mjs → chunk-3EFLY5LQ.mjs} +2 -2
- package/dist/{chunk-GDCTK5LU.mjs → chunk-3TENTQKL.mjs} +2 -2
- package/dist/{chunk-3PO2MOFY.mjs → chunk-3UAYJW6R.mjs} +2 -2
- package/dist/{chunk-VV6P6LKJ.mjs → chunk-3WYTRM24.mjs} +80 -78
- package/dist/chunk-43UYNPPC.mjs +110 -0
- package/dist/{chunk-UDDZVZTV.mjs → chunk-47NXBGQG.mjs} +2 -2
- package/dist/{chunk-RXQZ53NV.mjs → chunk-5CYLXJYQ.mjs} +2 -2
- package/dist/{chunk-6RNOECOF.mjs → chunk-5E46Y42F.mjs} +2 -2
- package/dist/{chunk-FAK7Y3OX.mjs → chunk-6U4C7VAD.mjs} +37 -9
- package/dist/{chunk-EY5U4P5V.mjs → chunk-7NMLFP2T.mjs} +17 -2
- package/dist/{chunk-BVLR6S5J.mjs → chunk-E6CHNSMY.mjs} +5 -5
- package/dist/{chunk-CP4ZQR2X.mjs → chunk-FBGUKBW2.mjs} +15 -9
- package/dist/{chunk-ZGGZXRGM.mjs → chunk-GYQKMYPG.mjs} +85 -38
- package/dist/{chunk-ICOCO7R4.mjs → chunk-GZ4YU36L.mjs} +1 -1
- package/dist/{chunk-3Q6CDT32.mjs → chunk-H2R4BTSO.mjs} +3 -3
- package/dist/{chunk-7SMVY4AX.mjs → chunk-HFF2R73P.mjs} +11 -11
- package/dist/{chunk-CPAE2QJP.mjs → chunk-HJCYLTVA.mjs} +1 -1
- package/dist/{chunk-XX4SKT54.mjs → chunk-KFUHQC6X.mjs} +22 -12
- package/dist/{chunk-5SHWA6PE.mjs → chunk-L2GYY6KY.mjs} +1 -1
- package/dist/{chunk-DEYEVTFV.mjs → chunk-L2W4FGTD.mjs} +2 -2
- package/dist/{chunk-4JLHHEBN.mjs → chunk-M5RYB6FE.mjs} +1 -1
- package/dist/{chunk-PBUME456.mjs → chunk-NOEMVUBS.mjs} +1 -1
- package/dist/{chunk-UQFWTROO.mjs → chunk-NSMF4OHE.mjs} +3 -3
- package/dist/{chunk-C3OQQ3M6.mjs → chunk-OPWZ7PZZ.mjs} +1 -1
- package/dist/{chunk-H7IIA2KL.mjs → chunk-PBVLCGCW.mjs} +1 -1
- package/dist/{chunk-REM3KDXN.mjs → chunk-U6B3HHTP.mjs} +4 -4
- package/dist/{chunk-M2Z5WVWW.mjs → chunk-UGMOBSMD.mjs} +2 -0
- package/dist/{chunk-43T3P3BE.mjs → chunk-WW3RDS57.mjs} +1 -1
- package/dist/{client-DetSCqP2.d.ts → client-B9YNYYs_.d.ts} +38 -12
- package/dist/client.d.ts +8 -8
- package/dist/client.mjs +2 -2
- package/dist/{config-BB4MRAsB.d.ts → config-DJFXyHMZ.d.ts} +2 -2
- package/dist/{config-XEHd4MFx.d.ts → config-DlvP2PSi.d.ts} +2 -2
- package/dist/{direct-executor-TWLEPVQO.mjs → direct-executor-2Y3JPVXT.mjs} +3 -3
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +6 -2
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution/transport/tcp.mjs +24 -19
- package/dist/distribution.d.ts +14 -9
- package/dist/distribution.mjs +808 -646
- package/dist/{engine-DGG_yX0p.d.ts → engine-DP4PrdKM.d.ts} +51 -13
- package/dist/{index-fX_FbuRT.d.ts → index-CcivYH_R.d.ts} +1 -1
- package/dist/index.browser.mjs +3 -3
- package/dist/index.d.ts +6 -6
- package/dist/index.mjs +3 -3
- package/dist/index.react-native.mjs +3 -3
- package/dist/{long-query-thread-HK5FP64H.mjs → long-query-thread-6B2HXSBI.mjs} +13 -13
- package/dist/{msgpack-leaf-LIT7TKZW.mjs → msgpack-leaf-LTAJ5QGB.mjs} +2 -2
- package/dist/{optional-reads-ESDVS225.mjs → optional-reads-5QWIOFOU.mjs} +1 -1
- package/dist/{optional-writes-DPWRGFKK.mjs → optional-writes-27NHTIAL.mjs} +5 -5
- package/dist/{paged-partition-archive-NITVFYOC.mjs → paged-partition-archive-EFCEC4TD.mjs} +13 -13
- package/dist/partial-words.browser.mjs +3 -3
- package/dist/partial-words.mjs +3 -3
- package/dist/pattern.browser.mjs +2 -2
- package/dist/pattern.mjs +2 -2
- package/dist/persistence/durability/checkpoint-worker.d.ts +2 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +10 -10
- package/dist/{query-actions-AA5LE2WV.mjs → query-actions-H6H54EJE.mjs} +3 -3
- package/dist/query-suggestions/merge-worker.mjs +6 -6
- package/dist/query-suggestions.browser.mjs +3 -3
- package/dist/query-suggestions.d.ts +2 -2
- package/dist/query-suggestions.mjs +4 -4
- package/dist/react-native/chunks/{chunk-6KVWTGEJ.mjs → chunk-7TURQEZZ.mjs} +84 -37
- package/dist/react-native/chunks/{chunk-ESVBC3AT.mjs → chunk-BIORJHQZ.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-AOLLV37A.mjs → chunk-CBG4EUYD.mjs} +17 -2
- package/dist/{chunk-7HVZVD2N.mjs → react-native/chunks/chunk-DYAAQB5R.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-HGZNA5A6.mjs → chunk-GCD64RZ2.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-OVYLUGEW.mjs → chunk-UTYWL2W6.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-ZZQ2RIYZ.mjs → chunk-XCJVLU6J.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-574N6ERF.mjs → chunk-YMIPG3VC.mjs} +3 -3
- package/dist/react-native/chunks/{chunk-ZF3VSGEI.mjs → chunk-ZSJABTIE.mjs} +1 -1
- package/dist/react-native/chunks/{msgpack-leaf-FOUPTT7J.mjs → msgpack-leaf-Q4BPA4G7.mjs} +2 -2
- package/dist/react-native/chunks/{optional-reads-TN3PT6M4.mjs → optional-reads-M6F26KFO.mjs} +1 -1
- package/dist/react-native/chunks/{optional-writes-UYVKLJU6.mjs → optional-writes-NYFF5NTC.mjs} +4 -4
- package/dist/react-native/chunks/{restore-queue-AZIZ5ABX.mjs → restore-queue-4OXFRQ6N.mjs} +6 -6
- package/dist/react-native/chunks/{runtime-UOHD2XUJ.mjs → runtime-7UKADH3G.mjs} +2 -1
- package/dist/react-native/chunks/{segment-codec-leaf-TYO4LKQ6.mjs → segment-codec-leaf-YKUO5DU6.mjs} +4 -4
- package/dist/react-native/chunks/{segment-merge-leaf-KM52IXXQ.mjs → segment-merge-leaf-HIOWV5UN.mjs} +5 -5
- package/dist/react-native/chunks/{storage-leaf-HKJBZXIP.mjs → storage-leaf-H64VNQKE.mjs} +40 -20
- package/dist/react-native/chunks/{store-leaf-LS5LT25L.mjs → store-leaf-ZG5DR7QA.mjs} +5 -5
- package/dist/react-native/chunks/{suggestions-leaf-IJ3Z4EH3.mjs → suggestions-leaf-AJZ3SHZO.mjs} +5 -5
- package/dist/react-native/chunks/{worker-copies-VNXZRBVW.mjs → worker-copies-IY4ZIA6W.mjs} +4 -4
- package/dist/react-native/partial-words.mjs +3 -3
- package/dist/react-native/pattern.mjs +2 -2
- package/dist/react-native/query-suggestions.mjs +3 -3
- package/dist/react-native/sync.mjs +12 -7
- package/dist/react-native/synonyms.mjs +3 -3
- package/dist/react.d.ts +9 -9
- package/dist/{rerank-X4HOST2N.mjs → rerank-JTIQNROU.mjs} +25 -12
- package/dist/{restore-queue-CI72WRAF.mjs → restore-queue-JIJW3XKK.mjs} +6 -6
- package/dist/{runtime-USQYKN3I.mjs → runtime-WWFVQOBJ.mjs} +2 -1
- package/dist/{schema-CFnQ-WI3.d.ts → schema-CPQUdJTs.d.ts} +6 -0
- package/dist/{segment-codec-leaf-7R5H2IYU.mjs → segment-codec-leaf-LTIODLBH.mjs} +4 -4
- package/dist/{segment-merge-leaf-IWL4VH7C.mjs → segment-merge-leaf-CXOMXPAP.mjs} +4 -4
- package/dist/server.d.ts +22 -11
- package/dist/server.mjs +24 -20
- package/dist/{storage-leaf-6YCHFH2O.mjs → storage-leaf-VB5OG66O.mjs} +22 -22
- package/dist/{store-leaf-V5N2REHF.mjs → store-leaf-RBK3Y2H4.mjs} +5 -5
- package/dist/{suggestions-leaf-XFJ4L37A.mjs → suggestions-leaf-27TNMJ37.mjs} +7 -7
- package/dist/sync/filesystem.d.ts +3 -3
- package/dist/sync/filesystem.mjs +8 -8
- package/dist/sync/indexeddb.browser.mjs +1 -1
- package/dist/sync/indexeddb.d.ts +3 -3
- package/dist/sync/indexeddb.mjs +2 -2
- package/dist/sync.browser.mjs +7 -6
- package/dist/sync.d.ts +4 -4
- package/dist/sync.mjs +12 -7
- package/dist/synonyms.browser.mjs +3 -3
- package/dist/synonyms.mjs +3 -3
- package/dist/{tasks-Bpi5YNZF.d.ts → tasks-4rvZJuoN.d.ts} +8 -4
- package/dist/{thread-UQAIJ5CS.mjs → thread-J32AOE3H.mjs} +12 -12
- package/dist/{types-DGjk684l.d.ts → types-C3hkFf_3.d.ts} +2 -2
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/{worker-copies-44MSY3IU.mjs → worker-copies-F2WVSQQO.mjs} +6 -6
- package/dist/workers/entry.mjs +9 -9
- package/openapi.json +58 -2
- package/package.json +5 -7
package/dist/distribution.mjs
CHANGED
|
@@ -1,45 +1,46 @@
|
|
|
1
|
-
import { routableAllocation, resolvePartitionId, resolveNodeTargets, resolvePrimaryAssignment, requireAssignedPrimary, assertSufficientActiveReplicas, getInSyncReplicaTargets } from './chunk-
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
1
|
+
import { routableAllocation, resolvePartitionId, resolveNodeTargets, resolvePrimaryAssignment, requireAssignedPrimary, assertSufficientActiveReplicas, getInSyncReplicaTargets } from './chunk-L2W4FGTD.mjs';
|
|
2
|
+
import { createVectorReadView, keepsAnyVectorField } from './chunk-43UYNPPC.mjs';
|
|
3
|
+
import { emptySerializablePartition } from './chunk-PBVLCGCW.mjs';
|
|
4
|
+
import { applyStoredIndexBm25 } from './chunk-OPWZ7PZZ.mjs';
|
|
5
|
+
import { keyBytesOf, querySuggestionsOf, createQuerySuggestionsRuntime, publicBuildOf, buildRecordToWire, buildRecordFromWire, SUGGESTIONS_SCHEMA } from './chunk-U6B3HHTP.mjs';
|
|
6
|
+
import { stampAnalysisGeneration, holdsNamedSettings, dropHeldAnalysisChange, releaseHeldAnalysisChange, heldAnalysisIsBuilt, buildHeldAnalysisChange, archiveStream, buildArchiveFromMemory, indexMetadataOf, applyIndexEntry, applyDeleteEntry, documentExportStream, followSourceChanges, requireChangesSetting, resolveAnalysisTarget, requireConfigurationWithinSizeLimit, memoryCheckpointStore, createByteReader, writeArchiveFile, namedSettingsOf, createDetachedIndex, loadPartitionSegments, changesStatusOf, holdsSettings, configOfNamedSettings, closePartitionWriter, requireRestoreMemory, archiveFootprint, storedBytesOfSegments, validateReplicationEntry, readSegmentManifest, replaySyncedWal, nestedSchemaOf } from './chunk-KFUHQC6X.mjs';
|
|
7
|
+
import { readVectorFile } from './chunk-NSMF4OHE.mjs';
|
|
7
8
|
import { addBackgroundAdmission, workWaitingForMemory, resolveHybridFusion, fuseRankings } from './chunk-GXED6R3Q.mjs';
|
|
8
9
|
import { UNADDRESSABLE_TARGET_DETAIL, MAX_MESSAGE_SIZE_BYTES } from './chunk-HZ3OAUTL.mjs';
|
|
9
|
-
import { bindEngineWriteBlock, bindIndexReads, syncServingSourceOf, bindSyncServing, changesServingSourceOf, bindChangesServing, validateBackupConfig, BACKUP_SWEEP_INTERVAL_MS, bindWriteBlockLookup, backupKeys, validateBackupName, CHANGES_BACKUP_TAKER, BACKUP_MANIFEST_READS_AT_ONCE, storedFilesOf, decodeBackupManifest, BACKUP_JOBS_KEY, BACKUP_DELETES_AT_ONCE, BACKUP_PROGRESS_REPORT_INTERVAL_MS, BACKUP_PARTITION_RESULT_RETENTION_MS, BACKUP_JOB_POLL_INTERVAL_MS, BACKUP_FILES_AT_ONCE, isCount, storeWithoutSaveIfAbsent, CLUSTER_UUID_KEY, isSha256Hex, BACKUP_PUBLISH_RETRY_MS, isBackupTaker, readRestorePlacement, SYNC_BACKUP_TAKER, BACKUP_JOB_CAS_ATTEMPTS, BACKUP_JOB_CAS_RETRY_MS, MAX_CLUSTER_UUID_BYTES, MAX_FINISHED_BACKUP_JOBS, BACKUP_FILE_CHUNK_BYTES, encodeBackupManifest, SYNC_BACKUPS_KEPT, BACKUP_PARTITIONS_AT_ONCE, BACKUP_PARTITION_POLL_INTERVAL_MS, BACKUP_PRIMARY_WAIT_MS, readBackupPartition, readStoredFile } from './chunk-
|
|
10
|
-
import { DEFAULT_NODE_LIFECYCLE_CONFIG, DEFAULT_CONTROLLER_CONFIG,
|
|
10
|
+
import { bindEngineWriteBlock, bindIndexReads, syncServingSourceOf, bindSyncServing, changesServingSourceOf, bindChangesServing, validateBackupConfig, BACKUP_SWEEP_INTERVAL_MS, bindWriteBlockLookup, backupKeys, validateBackupName, CHANGES_BACKUP_TAKER, BACKUP_MANIFEST_READS_AT_ONCE, storedFilesOf, decodeBackupManifest, BACKUP_JOBS_KEY, BACKUP_DELETES_AT_ONCE, BACKUP_PROGRESS_REPORT_INTERVAL_MS, BACKUP_PARTITION_RESULT_RETENTION_MS, BACKUP_JOB_POLL_INTERVAL_MS, BACKUP_FILES_AT_ONCE, isCount, storeWithoutSaveIfAbsent, CLUSTER_UUID_KEY, isSha256Hex, BACKUP_PUBLISH_RETRY_MS, isBackupTaker, readRestorePlacement, SYNC_BACKUP_TAKER, BACKUP_JOB_CAS_ATTEMPTS, BACKUP_JOB_CAS_RETRY_MS, MAX_CLUSTER_UUID_BYTES, MAX_FINISHED_BACKUP_JOBS, BACKUP_FILE_CHUNK_BYTES, encodeBackupManifest, SYNC_BACKUPS_KEPT, BACKUP_PARTITIONS_AT_ONCE, BACKUP_PARTITION_POLL_INTERVAL_MS, BACKUP_PRIMARY_WAIT_MS, readBackupPartition, readStoredFile } from './chunk-HFF2R73P.mjs';
|
|
11
|
+
import { DEFAULT_NODE_LIFECYCLE_CONFIG, DEFAULT_CONTROLLER_CONFIG, getIndexMetadata, getClusterIndexConfig, withPartitionCount, validateIndexName, readIndexMetadataSnapshot, indexConfigKey, metadataWithSync, suggestionStateOfSettings, indexConfigFromSettings, metadataWithBm25, readIndexState, storeSynonymSet, deleteSynonymSet, indexSettingsOf, putIndexMetadata, isValidInteger, isRecord, HIDDEN_GENERATION_PREFIX, indexMetadataSnapshotOf, metadataWithSynonymSet, RESHARD_RECORD_CAS_ATTEMPTS, indexUuidOfStoredMetadata, storedSynonymSetOf, RESHARD_FORWARD_BATCH_DOCUMENTS, isRecord2, decodeIndexSettings, RESHARD_RETRY_DELAY_MS, TEARDOWN_CAS_ATTEMPTS, RESHARD_RECORD_CHECK_INTERVAL_MS, RESHARD_POLL_INTERVAL_MS, ALLOCATION_RETRY_DELAY_MS, TEARDOWN_RETRY_ROUNDS, ANALYSIS_CHANGE_RETRY_MS, putIndexReplicationFactor, ALLOCATION_CAS_ATTEMPTS, TEARDOWN_RETRY_DELAY_MS, RESHARD_CONTROLLER_POLL_INTERVAL_MS, MAX_RESHARD_CUTOVER_ATTEMPTS, ANALYSIS_CHANGE_POLL_INTERVAL_MS, RECOVERY_CAS_ATTEMPTS, ESTIMATED_PARTITION_BYTES, REBALANCE_THRESHOLD, putGenerationMetadata, RESHARD_CUTOVER_LAG_ENTRIES, ANALYSIS_REBUILD_RESEND_MS, RESHARD_CUTOVER_TIMEOUT_MS, PARTITION_STORES_TIMEOUT_MS, SYNC_EPOCH_CAS_ATTEMPTS, syncSettingsOfSnapshot, epochFeaturesOn, LEADERSHIP_IMBALANCE_THRESHOLD, BOOTSTRAP_CAS_ATTEMPTS, metadataWithSuggestionState, UNFED_REPLICA_CHECK_INTERVAL_MS, UNFED_REPLICA_RESYNC_MS } from './chunk-3WYTRM24.mjs';
|
|
11
12
|
import './chunk-4H4RG2FR.mjs';
|
|
12
|
-
import { MAX_WAIT_FOR_ACTIVE_REPLICAS, MIN_CONTROLLER_LEASE_TTL_MS, MAX_NODE_LEFT_DELAY_MS, DEFAULT_MAX_PER_SOURCE_SNAPSHOTS, DEFAULT_MAX_STREAMS_PER_INDEX, DEFAULT_CAPACITY, CATCH_UP_TICK_MS, MEMORY_SAMPLE_INTERVAL_MS, createWritePressure, WRITE_MEMORY_SAMPLE_MAX_AGE_MS, DEFAULT_IN_FLIGHT_WRITE_FRACTION, MAX_SETTINGS_REPLACEMENT_ATTEMPTS, SETTINGS_CONTENTION_REASON, MAX_DROP_ACKNOWLEDGE_WAIT_MS, DEFAULT_PARTITION_COUNT, DEFAULT_REPLICATION_FACTOR, DEFAULT_CREATE_INDEX_WAIT_MS, DEFAULT_BOOTSTRAP_SYNC_DEADLINE_MS, DROP_ACKNOWLEDGE_POLL_INTERVAL_MS, COORDINATOR_REQUEST_BUCKET_BOUNDS_SECONDS, COPY_FAILURE_ATTEMPTS, SEARCH_CONTEXT_REAPER_INTERVAL_MS, DECOMMISSION_RECHECK_INTERVAL_MS, refuseEncodeWithoutRoom, answerRefusalOf, withAdmittedWrite, estimateForwardedBytes, PARTITION_BYTES_INDEX_OVERHEAD_BYTES, MAX_SOURCE_ID_LENGTH, ADMISSION_TIMEOUT_MS, PARTITION_BYTES_ENTRY_BYTES, PARTITION_BYTES_REPORT_BUDGET_BYTES, CATCH_UP_FEED_WINDOW_MS, MAX_CATCH_UP_IN_FLIGHT_BYTES, SYNCED_POSITION_REPORT_TIMEOUT_MS, SNAPSHOT_CHUNK_YIELD_INTERVAL_MS, ANSWER_DECODE_HEAP_BYTES_PER_PAYLOAD_BYTE, ADMISSION_RETRIES_AFTER_IN_SYNC_CHANGE, REGISTRATION_LEASE_DRIFT_SHARE, REGISTRATION_LEASE_CONFIRM_TIMEOUT_MS, WRITE_APPLY_YIELD_INTERVAL_MS, ROLLBACK_RETRY_INITIAL_DELAY_MS, ROLLBACK_RETRY_DELAY_CAP_MS, OUT_OF_STEP_REMOVAL_RETRY_INITIAL_DELAY_MS, OUT_OF_STEP_REMOVAL_RETRY_DELAY_CAP_MS, FETCH_GROUP_DOCUMENT_IDS, COPY_FAILURE_RETRY_DELAY_MS, SEARCH_CONTEXT_ENTRY_HEAP_BYTES, CLEAR_PAGE_SIZE, estimateDocumentsBytes, estimateDocumentBytes, PRIMARY_CHANGE_RETRY_WINDOW_MS, PRIMARY_CHANGE_POLL_INTERVAL_MS, PARTITION_WRITE_CONCURRENCY, FORWARD_RETRY_INITIAL_DELAY_MS, FORWARD_RETRY_DELAY_CAP_MS } from './chunk-
|
|
13
|
-
export { DEFAULT_CAPACITY, DEFAULT_PARTITION_COUNT, DEFAULT_REPLICATION_FACTOR } from './chunk-
|
|
14
|
-
import { encodeIndexConfiguration, MAX_SNAPSHOT_ANALYSIS_ATTEMPTS, decodeIndexConfiguration, requireSyncSetting, requireNoQuerySuggestions, validChangeFilter, createDocumentTest, INDEX_CONFIGURATION_ENVELOPE_VERSION, changesNotEnabled } from './chunk-
|
|
15
|
-
import { restoredConfigFields, restoredEmbedding } from './chunk-
|
|
16
|
-
import { readSnapshotArchive, archiveLayout, openManifestPayload, sourceIndexNameOf, listedKeysOf, decodeManifestPayload, estimatedArchiveBytes } from './chunk-
|
|
13
|
+
import { MAX_WAIT_FOR_ACTIVE_REPLICAS, MIN_CONTROLLER_LEASE_TTL_MS, MAX_NODE_LEFT_DELAY_MS, DEFAULT_MAX_PER_SOURCE_SNAPSHOTS, DEFAULT_MAX_STREAMS_PER_INDEX, DEFAULT_CAPACITY, CATCH_UP_TICK_MS, MEMORY_SAMPLE_INTERVAL_MS, createWritePressure, WRITE_MEMORY_SAMPLE_MAX_AGE_MS, DEFAULT_IN_FLIGHT_WRITE_FRACTION, MAX_SETTINGS_REPLACEMENT_ATTEMPTS, SETTINGS_CONTENTION_REASON, MAX_DROP_ACKNOWLEDGE_WAIT_MS, DEFAULT_PARTITION_COUNT, DEFAULT_REPLICATION_FACTOR, DEFAULT_CREATE_INDEX_WAIT_MS, DEFAULT_BOOTSTRAP_SYNC_DEADLINE_MS, DROP_ACKNOWLEDGE_POLL_INTERVAL_MS, COORDINATOR_REQUEST_BUCKET_BOUNDS_SECONDS, COPY_FAILURE_ATTEMPTS, SEARCH_CONTEXT_REAPER_INTERVAL_MS, DECOMMISSION_RECHECK_INTERVAL_MS, refuseEncodeWithoutRoom, answerRefusalOf, withAdmittedWrite, estimateForwardedBytes, PARTITION_BYTES_INDEX_OVERHEAD_BYTES, MAX_SOURCE_ID_LENGTH, ADMISSION_TIMEOUT_MS, PARTITION_BYTES_ENTRY_BYTES, PARTITION_BYTES_REPORT_BUDGET_BYTES, CATCH_UP_FEED_WINDOW_MS, MAX_CATCH_UP_IN_FLIGHT_BYTES, SYNCED_POSITION_REPORT_TIMEOUT_MS, SNAPSHOT_CHUNK_YIELD_INTERVAL_MS, ANSWER_DECODE_HEAP_BYTES_PER_PAYLOAD_BYTE, ADMISSION_RETRIES_AFTER_IN_SYNC_CHANGE, REGISTRATION_LEASE_DRIFT_SHARE, REGISTRATION_LEASE_CONFIRM_TIMEOUT_MS, WRITE_APPLY_YIELD_INTERVAL_MS, ROLLBACK_RETRY_INITIAL_DELAY_MS, ROLLBACK_RETRY_DELAY_CAP_MS, OUT_OF_STEP_REMOVAL_RETRY_INITIAL_DELAY_MS, OUT_OF_STEP_REMOVAL_RETRY_DELAY_CAP_MS, FETCH_GROUP_DOCUMENT_IDS, COPY_FAILURE_RETRY_DELAY_MS, SEARCH_CONTEXT_ENTRY_HEAP_BYTES, CLEAR_PAGE_SIZE, estimateDocumentsBytes, estimateDocumentBytes, PRIMARY_CHANGE_RETRY_WINDOW_MS, PRIMARY_CHANGE_POLL_INTERVAL_MS, PARTITION_WRITE_CONCURRENCY, FORWARD_RETRY_INITIAL_DELAY_MS, FORWARD_RETRY_DELAY_CAP_MS } from './chunk-34QTB6O7.mjs';
|
|
14
|
+
export { DEFAULT_CAPACITY, DEFAULT_PARTITION_COUNT, DEFAULT_REPLICATION_FACTOR } from './chunk-34QTB6O7.mjs';
|
|
15
|
+
import { encodeIndexConfiguration, MAX_SNAPSHOT_ANALYSIS_ATTEMPTS, decodeIndexConfiguration, requireSyncSetting, requireNoQuerySuggestions, validChangeFilter, createDocumentTest, INDEX_CONFIGURATION_ENVELOPE_VERSION, changesNotEnabled } from './chunk-3TENTQKL.mjs';
|
|
16
|
+
import { restoredConfigFields, restoredEmbedding } from './chunk-UGMOBSMD.mjs';
|
|
17
|
+
import { readSnapshotArchive, archiveLayout, openManifestPayload, sourceIndexNameOf, listedKeysOf, decodeManifestPayload, estimatedArchiveBytes } from './chunk-H2R4BTSO.mjs';
|
|
17
18
|
import { CONTROLLER_LEASE_KEY } from './chunk-44W3CBNG.mjs';
|
|
18
19
|
import './chunk-T4HMSZCX.mjs';
|
|
19
|
-
import { createForwardBatchResultMessage, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, DEFAULT_LOG_RETENTION_BYTES, buildEntry, optionalIndexUuid, entryByteEstimate, computeEntryChecksum, createPrimaryResignMessage, validateInsyncConfirmPayload, createInsyncRemoveMessage, SNAPSHOT_CHUNK_SIZE, createEntryBatchMessage,
|
|
20
|
-
import { directoryCheckpointStore, readListedFile, encodeSegmentManifest } from './chunk-
|
|
20
|
+
import { createForwardBatchResultMessage, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, DEFAULT_LOG_RETENTION_BYTES, buildEntry, invalidReplicationPayload, optionalIndexUuid, validateForwardOperation, entryByteEstimate, computeEntryChecksum, createPrimaryResignMessage, validateInsyncConfirmPayload, createInsyncRemoveMessage, SNAPSHOT_CHUNK_SIZE, createEntryBatchMessage, createEntryMessage, validateAckPayload, createInsyncAddMessage, createInsyncConfirmMessage, checkpointLastSeqNo, checkpointPrimaryTerm, INSYNC_CAS_ATTEMPTS, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage } from './chunk-26AKUBDX.mjs';
|
|
21
|
+
import { directoryCheckpointStore, readListedFile, encodeSegmentManifest } from './chunk-47NXBGQG.mjs';
|
|
21
22
|
import { TransportErrorCodes, ClusterMessageTypes, ReplicationMessageTypes, SuggestionMessageTypes, SyncMessageTypes, QueryMessageTypes, TransportError, DEFAULT_TRANSPORT_CONFIG } from './chunk-5FLRXF73.mjs';
|
|
22
23
|
import { MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR, WIRE_BATCH_BUDGET } from './chunk-JDDXV6UF.mjs';
|
|
23
|
-
import { manifestFileRefs, segmentKey, SEGMENT_MANIFEST_VERSION, vectorFileKey } from './chunk-
|
|
24
|
+
import { manifestFileRefs, segmentKey, SEGMENT_MANIFEST_VERSION, vectorFileKey } from './chunk-HJCYLTVA.mjs';
|
|
24
25
|
import './chunk-S5QMWECL.mjs';
|
|
25
26
|
import './chunk-LNGE7SLY.mjs';
|
|
26
|
-
import './chunk-
|
|
27
|
-
import './chunk-
|
|
28
|
-
import { deserializePartition } from './chunk-
|
|
27
|
+
import './chunk-L2GYY6KY.mjs';
|
|
28
|
+
import './chunk-2M55WPWK.mjs';
|
|
29
|
+
import { deserializePartition } from './chunk-3UAYJW6R.mjs';
|
|
29
30
|
import { decodeVectorFilePayload, bytesToVectors, isDead, MIN_EF_SEARCH, MAX_EF_SEARCH, MIN_OVERSAMPLE } from './chunk-R724DJSG.mjs';
|
|
30
|
-
import './chunk-
|
|
31
|
-
import { RestoredHistory } from './chunk-
|
|
31
|
+
import './chunk-5E46Y42F.mjs';
|
|
32
|
+
import { RestoredHistory } from './chunk-3EFLY5LQ.mjs';
|
|
32
33
|
import { oneSyncChangeAtATime } from './chunk-V6EUBAIT.mjs';
|
|
33
|
-
import { historyFilesOf, removeHistoryFolder, QUERY_HISTORY_FOLDER, RESTORE_STAGING_INFIX, openChecksummedEnvelope, historyFolderOf, listHistoryFolder, historyFileOf, COUNTS_STREAM_CHUNK_BYTES, BUILD_WRITE_BATCH, restoreStagingPrefix, crc32Init, crc32Final, crc32Update, MAX_COUNTS_STREAM_KEY_BYTES, MAX_HISTORY_FILE_BYTES, packChecksummedEnvelope, storedStateOf } from './chunk-
|
|
34
|
+
import { historyFilesOf, removeHistoryFolder, QUERY_HISTORY_FOLDER, RESTORE_STAGING_INFIX, openChecksummedEnvelope, historyFolderOf, listHistoryFolder, historyFileOf, COUNTS_STREAM_CHUNK_BYTES, BUILD_WRITE_BATCH, restoreStagingPrefix, crc32Init, crc32Final, crc32Update, MAX_COUNTS_STREAM_KEY_BYTES, MAX_HISTORY_FILE_BYTES, packChecksummedEnvelope, storedStateOf } from './chunk-5CYLXJYQ.mjs';
|
|
34
35
|
import { boundQuerySuggestionsOf, bindQuerySuggestions } from './chunk-S2POFTLN.mjs';
|
|
35
|
-
import { oversampledShardSize, runtimeHost, bindRerankWindowSource, rerankWindowCapOf, validateMemoryFractions, requireKnownConfig, importOptionalReads, resolveRunnableConfig, createEngineCore, startEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, emitEngineEvent, synonymModule, createEngineIndex, generateId, documentChurnOf, isCompositePartition, getLanguage, resolveIndexAnalysis, memoryFractionsOf, createMemoryMonitor, readMemoryFigures, FULL_MEMORY_COLLECTION_INTERVAL_MS, validateBM25Params, storedBM25Params, requireSurfaceFormsForPartialFields, bm25SetsValue, prepareConfiguredSynonyms, storedArrayPositionGap, updateDocumentVectors, arrayPositionsAreStale, fnv1a, readRecentMemoryFigures, normalizeSort, sortModesOf,
|
|
36
|
+
import { oversampledShardSize, runtimeHost, bindRerankWindowSource, rerankWindowCapOf, validateMemoryFractions, requireKnownConfig, importOptionalReads, resolveRunnableConfig, createEngineCore, startEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, emitEngineEvent, synonymModule, createEngineIndex, generateId, documentChurnOf, isCompositePartition, getLanguage, resolveIndexAnalysis, memoryFractionsOf, createMemoryMonitor, readMemoryFigures, FULL_MEMORY_COLLECTION_INTERVAL_MS, validateBM25Params, storedBM25Params, requireSurfaceFormsForPartialFields, validateReturnVectors, bm25SetsValue, prepareConfiguredSynonyms, storedArrayPositionGap, updateDocumentVectors, arrayPositionsAreStale, fnv1a, readRecentMemoryFigures, normalizeSort, sortModesOf, noteDocumentChurn, MAP_TABLE_MIN_CAPACITY, MAP_TABLE_SHRINK_DIVISOR, resolveProjection, applyProjection, requireLoadedOptionalReads, FILTER_EXPRESSION_KEYS, filterTestCount, setNestedValue, CASE_FOLD_EXPANSION_FACTOR, PATTERN_RUN_HEAP_BYTES, ENUM_VALUE_HEAP_BYTES, validateBM25Shape, DOCUMENT_VALUE_HEAP_BYTES, DOCUMENT_HEAP_BYTES_PER_ESTIMATED_BYTE, DOCUMENT_PROPERTY_HEAP_BYTES, VECTOR_DOCUMENT_HEAP_BYTES, VECTOR_COMPONENT_HEAP_BYTES, hasLanguage, FIELD_FILTER_OPERATORS, yieldToEventLoop, NORMALIZATION_EXPANSION_FACTOR, TERM_UNIT_HEAP_BYTES, customTokensOf, visitTokens, getVectorFieldPaths, resolvePartitionInsertOptions, clampRowCount, now, sortSignatureOf, listBindingOf, requireMatchingCursor, decodePageCursor, encodePageCursor, DISTINCT_TERM_HEAP_BYTES, PARTIAL_SPELLING_HEAP_BYTES, clampLimit, requireWithinResultWindow, providedDocId, MAP_TABLE_HEADER_BYTES, MAP_TABLE_SLOT_BYTES, holdsQuotedText, queryBindingOf, engineCoreOf, sortSignatureEntry, fnv1aUint32s, loadRerankRuntime, mergePartitionStats, mergeEntries, regionsOverlap, compareScores } from './chunk-GYQKMYPG.mjs';
|
|
36
37
|
import { encodeDocument, encodeDocumentReply } from './chunk-2CKDCC65.mjs';
|
|
37
38
|
import { collectYoungGenerationBeforeRefusal, collectGarbageBeforeRefusal, onRefusalAfterCollection, reservedHeapBytes, reserveHeapBytes } from './chunk-BRTABNC7.mjs';
|
|
38
39
|
import { decode } from './chunk-Y7GXGJFZ.mjs';
|
|
39
40
|
import { newSyncEpoch } from './chunk-BUMREPLU.mjs';
|
|
40
41
|
import './chunk-7T4LAYT4.mjs';
|
|
41
|
-
import { MAX_FILTER_DEPTH, MAX_FILTER_TESTS, isPatternTolerance, MAX_FILTER_ARRAY_SIZE, buildStoredValues } from './chunk-
|
|
42
|
-
import { loadMessagePack, validateSchema, requireCreatableSchema, schemaFieldsOf, getNestedValue, MAX_NESTING_DEPTH, MAX_STORED_VECTOR_DIMENSION, MIN_POLYGON_POINTS, isLatitude, MAX_LATITUDE_DEGREES, isLongitude, MAX_LONGITUDE_DEGREES } from './chunk-
|
|
42
|
+
import { MAX_FILTER_DEPTH, MAX_FILTER_TESTS, isPatternTolerance, MAX_FILTER_ARRAY_SIZE, buildStoredValues } from './chunk-M5RYB6FE.mjs';
|
|
43
|
+
import { loadMessagePack, validateSchema, requireCreatableSchema, schemaFieldsOf, getNestedValue, MAX_NESTING_DEPTH, MAX_STORED_VECTOR_DIMENSION, MIN_POLYGON_POINTS, isLatitude, MAX_LATITUDE_DEGREES, isLongitude, MAX_LONGITUDE_DEGREES } from './chunk-7NMLFP2T.mjs';
|
|
43
44
|
import './chunk-U2BKYEC7.mjs';
|
|
44
45
|
import { shardEntryRangeOf } from './chunk-PSZEGK6W.mjs';
|
|
45
46
|
import { collectAndReturnPages } from './chunk-5EI5CXO6.mjs';
|
|
@@ -116,6 +117,19 @@ function encodeRebalanceRefusal(error) {
|
|
|
116
117
|
return encode({ error: true, code, message: text2 });
|
|
117
118
|
}
|
|
118
119
|
}
|
|
120
|
+
|
|
121
|
+
// src/distribution/request-decode.ts
|
|
122
|
+
function decodeRequestPayload(payload, messageType) {
|
|
123
|
+
try {
|
|
124
|
+
return decode(payload);
|
|
125
|
+
} catch (error) {
|
|
126
|
+
throw new NarsilError(
|
|
127
|
+
TransportErrorCodes.DECODE_FAILED,
|
|
128
|
+
`The payload of a ${messageType} request is not valid MessagePack`,
|
|
129
|
+
{ messageType, cause: error instanceof Error ? error.message : String(error) }
|
|
130
|
+
);
|
|
131
|
+
}
|
|
132
|
+
}
|
|
119
133
|
var PHASES = /* @__PURE__ */ new Set(["copying", "cutover", "aborting"]);
|
|
120
134
|
var ABSENT = new Uint8Array(0);
|
|
121
135
|
function rebalanceRecordKey(indexName) {
|
|
@@ -322,6 +336,9 @@ function malformed2() {
|
|
|
322
336
|
{ key: BACKUP_JOBS_KEY }
|
|
323
337
|
);
|
|
324
338
|
}
|
|
339
|
+
function nilAsAbsent(value) {
|
|
340
|
+
return value === null ? void 0 : value;
|
|
341
|
+
}
|
|
325
342
|
function isOptionalString(value) {
|
|
326
343
|
return value === void 0 || typeof value === "string";
|
|
327
344
|
}
|
|
@@ -331,21 +348,26 @@ function decodeJobError(value) {
|
|
|
331
348
|
return { code: value.code, message: value.message };
|
|
332
349
|
}
|
|
333
350
|
function decodeJob(value) {
|
|
334
|
-
if (!isRecord(value)
|
|
351
|
+
if (!isRecord(value)) malformed2();
|
|
352
|
+
const indexName = nilAsAbsent(value.indexName);
|
|
353
|
+
const generation = nilAsAbsent(value.generation);
|
|
354
|
+
const nameIndexUuid = nilAsAbsent(value.nameIndexUuid);
|
|
355
|
+
const finishedAt = nilAsAbsent(value.finishedAt);
|
|
356
|
+
if (typeof value.kind !== "string" || !KINDS.has(value.kind) || typeof value.backupName !== "string" || !isOptionalString(indexName) || !isOptionalString(generation) || !isOptionalString(nameIndexUuid) || typeof value.status !== "string" || !STATUSES.has(value.status) || !isCount(value.partitionsDone) || !isCount(value.partitionsTotal) || !isCount(value.startedAt) || !(finishedAt === void 0 || isCount(finishedAt))) {
|
|
335
357
|
malformed2();
|
|
336
358
|
}
|
|
337
|
-
const error = decodeJobError(value.error);
|
|
359
|
+
const error = decodeJobError(nilAsAbsent(value.error));
|
|
338
360
|
return {
|
|
339
361
|
kind: value.kind,
|
|
340
362
|
backupName: value.backupName,
|
|
341
|
-
...
|
|
342
|
-
...
|
|
343
|
-
...
|
|
363
|
+
...indexName === void 0 ? {} : { indexName },
|
|
364
|
+
...generation === void 0 ? {} : { generation },
|
|
365
|
+
...nameIndexUuid === void 0 ? {} : { nameIndexUuid },
|
|
344
366
|
status: value.status,
|
|
345
367
|
partitionsDone: value.partitionsDone,
|
|
346
368
|
partitionsTotal: value.partitionsTotal,
|
|
347
369
|
startedAt: value.startedAt,
|
|
348
|
-
...
|
|
370
|
+
...finishedAt === void 0 ? {} : { finishedAt },
|
|
349
371
|
...error === void 0 ? {} : { error }
|
|
350
372
|
};
|
|
351
373
|
}
|
|
@@ -927,6 +949,66 @@ async function admitDeletion(coordinator, store, backupName) {
|
|
|
927
949
|
await claimBackupStore(store, await clusterUuidOf(coordinator));
|
|
928
950
|
}
|
|
929
951
|
|
|
952
|
+
// src/distribution/backups/config-agreement.ts
|
|
953
|
+
function isNil(value) {
|
|
954
|
+
return value === null || value === void 0;
|
|
955
|
+
}
|
|
956
|
+
function sameBytes(left, right) {
|
|
957
|
+
if (left.byteLength !== right.byteLength) return false;
|
|
958
|
+
for (let at = 0; at < left.byteLength; at++) {
|
|
959
|
+
if (left[at] !== right[at]) return false;
|
|
960
|
+
}
|
|
961
|
+
return true;
|
|
962
|
+
}
|
|
963
|
+
function pushMembers(pending, left, right) {
|
|
964
|
+
for (const key of Object.keys(left)) pending.push([left[key], right[key]]);
|
|
965
|
+
for (const key of Object.keys(right)) {
|
|
966
|
+
if (!Object.hasOwn(left, key)) pending.push([void 0, right[key]]);
|
|
967
|
+
}
|
|
968
|
+
}
|
|
969
|
+
function sameDecodedValue(left, right) {
|
|
970
|
+
const pending = [[left, right]];
|
|
971
|
+
for (let pair = pending.pop(); pair !== void 0; pair = pending.pop()) {
|
|
972
|
+
const [a, b] = pair;
|
|
973
|
+
if (a === b || isNil(a) && isNil(b)) continue;
|
|
974
|
+
if (typeof a === "number" && typeof b === "number" && Number.isNaN(a) && Number.isNaN(b)) continue;
|
|
975
|
+
if (a instanceof Uint8Array || b instanceof Uint8Array) {
|
|
976
|
+
if (a instanceof Uint8Array && b instanceof Uint8Array && sameBytes(a, b)) continue;
|
|
977
|
+
return false;
|
|
978
|
+
}
|
|
979
|
+
if (Array.isArray(a) || Array.isArray(b)) {
|
|
980
|
+
if (!Array.isArray(a) || !Array.isArray(b) || a.length !== b.length) return false;
|
|
981
|
+
for (let at = 0; at < a.length; at++) pending.push([a[at], b[at]]);
|
|
982
|
+
continue;
|
|
983
|
+
}
|
|
984
|
+
if (!isRecord(a) || !isRecord(b)) return false;
|
|
985
|
+
pushMembers(pending, a, b);
|
|
986
|
+
}
|
|
987
|
+
return true;
|
|
988
|
+
}
|
|
989
|
+
function settingsChanged(indexName) {
|
|
990
|
+
return new NarsilError(
|
|
991
|
+
ErrorCodes.ANALYSIS_CHANGE_NOT_READY,
|
|
992
|
+
`The settings of index '${indexName}' changed while its partitions were backed up, so the backup stores no manifest`,
|
|
993
|
+
{ indexName }
|
|
994
|
+
);
|
|
995
|
+
}
|
|
996
|
+
async function agreedConfigOf(store, results, indexName, backupName) {
|
|
997
|
+
const [first] = results;
|
|
998
|
+
if (first === void 0) throw settingsChanged(indexName);
|
|
999
|
+
const distinct = /* @__PURE__ */ new Map();
|
|
1000
|
+
for (const result of results) {
|
|
1001
|
+
if (result.config.sha256 !== first.config.sha256) distinct.set(result.config.sha256, result.config);
|
|
1002
|
+
}
|
|
1003
|
+
if (distinct.size === 0) return first.config;
|
|
1004
|
+
const decodedFirst = await decodeIndexConfiguration(await loadCheckedBytes(store, first.config, backupName));
|
|
1005
|
+
for (const config of distinct.values()) {
|
|
1006
|
+
const decoded = await decodeIndexConfiguration(await loadCheckedBytes(store, config, backupName));
|
|
1007
|
+
if (!sameDecodedValue(decodedFirst, decoded)) throw settingsChanged(indexName);
|
|
1008
|
+
}
|
|
1009
|
+
return first.config;
|
|
1010
|
+
}
|
|
1011
|
+
|
|
930
1012
|
// src/distribution/cluster/node-targets.ts
|
|
931
1013
|
function targetsOfRegisteredNode(nodeId, address) {
|
|
932
1014
|
return address.length === 0 || address === nodeId ? [nodeId] : [address, nodeId];
|
|
@@ -953,9 +1035,7 @@ var MAX_PINNED_ENTRIES = 1e3;
|
|
|
953
1035
|
var MAX_PINNED_POSITION = RESULT_WINDOW;
|
|
954
1036
|
var MAX_OVERSAMPLE = 100;
|
|
955
1037
|
var MAX_GROUP_FIELDS = 64;
|
|
956
|
-
var
|
|
957
|
-
var MAX_HIGHLIGHT_SNIPPET_LENGTH = 65536;
|
|
958
|
-
var MAX_LIST_CURSOR_LENGTH = 4096;
|
|
1038
|
+
var MAX_LIST_CURSOR_LENGTH = 40960;
|
|
959
1039
|
var MAX_SUGGEST_WIRE_LIMIT = 1e3;
|
|
960
1040
|
var MAX_COUNT_VALUE = Number.MAX_SAFE_INTEGER;
|
|
961
1041
|
var MAX_LANGUAGE_NAME_LENGTH = 64;
|
|
@@ -987,6 +1067,14 @@ function isInteger(value) {
|
|
|
987
1067
|
function isFiniteNumber(value) {
|
|
988
1068
|
return typeof value === "number" && Number.isFinite(value);
|
|
989
1069
|
}
|
|
1070
|
+
function restoreMissingAsNil(record, members) {
|
|
1071
|
+
for (const member of members) {
|
|
1072
|
+
if (!Object.hasOwn(record, member)) record[member] = null;
|
|
1073
|
+
}
|
|
1074
|
+
}
|
|
1075
|
+
function dropNilMember(record, member) {
|
|
1076
|
+
if (record[member] === null) delete record[member];
|
|
1077
|
+
}
|
|
990
1078
|
function validateStringField(value, fieldLabel, maxLength, errorCode) {
|
|
991
1079
|
if (typeof value !== "string") {
|
|
992
1080
|
throwInvalid(errorCode, `Invalid payload: "${fieldLabel}" must be a string`);
|
|
@@ -1109,32 +1197,6 @@ function validateDocumentId(value, fieldLabel) {
|
|
|
1109
1197
|
validateStringField(value.docId, `${fieldLabel}.docId`, MAX_DOC_ID_LENGTH, CONFIG_INVALID);
|
|
1110
1198
|
validatePartitionId(value.partitionId, `${fieldLabel}.partitionId`, CONFIG_INVALID);
|
|
1111
1199
|
}
|
|
1112
|
-
function validateHighlightConfig(value) {
|
|
1113
|
-
if (!isRecord3(value)) {
|
|
1114
|
-
throwInvalid(CONFIG_INVALID, 'Invalid FetchPayload: "highlight" must be an object or null');
|
|
1115
|
-
}
|
|
1116
|
-
if (value.fields !== null) {
|
|
1117
|
-
validateStringArray(value.fields, "highlight.fields", MAX_FIELDS_LIST, 255, SEARCH_INVALID_FIELD);
|
|
1118
|
-
}
|
|
1119
|
-
if (typeof value.before !== "string" || value.before.length > MAX_HIGHLIGHT_TAG_LENGTH) {
|
|
1120
|
-
throwInvalid(
|
|
1121
|
-
CONFIG_INVALID,
|
|
1122
|
-
`Invalid FetchPayload: "highlight.before" must be a string of at most ${MAX_HIGHLIGHT_TAG_LENGTH} characters`
|
|
1123
|
-
);
|
|
1124
|
-
}
|
|
1125
|
-
if (typeof value.after !== "string" || value.after.length > MAX_HIGHLIGHT_TAG_LENGTH) {
|
|
1126
|
-
throwInvalid(
|
|
1127
|
-
CONFIG_INVALID,
|
|
1128
|
-
`Invalid FetchPayload: "highlight.after" must be a string of at most ${MAX_HIGHLIGHT_TAG_LENGTH} characters`
|
|
1129
|
-
);
|
|
1130
|
-
}
|
|
1131
|
-
if (typeof value.maxSnippetLength !== "number" || !Number.isFinite(value.maxSnippetLength) || !Number.isInteger(value.maxSnippetLength) || value.maxSnippetLength <= 0 || value.maxSnippetLength > MAX_HIGHLIGHT_SNIPPET_LENGTH) {
|
|
1132
|
-
throwInvalid(
|
|
1133
|
-
CONFIG_INVALID,
|
|
1134
|
-
`Invalid FetchPayload: "highlight.maxSnippetLength" must be a positive integer at most ${MAX_HIGHLIGHT_SNIPPET_LENGTH}`
|
|
1135
|
-
);
|
|
1136
|
-
}
|
|
1137
|
-
}
|
|
1138
1200
|
function validateFetchPayload(decoded) {
|
|
1139
1201
|
if (!isRecord3(decoded)) {
|
|
1140
1202
|
throwInvalid(CONFIG_INVALID, "Invalid FetchPayload: expected an object");
|
|
@@ -1153,11 +1215,13 @@ function validateFetchPayload(decoded) {
|
|
|
1153
1215
|
for (let i = 0; i < decoded.documentIds.length; i++) {
|
|
1154
1216
|
validateDocumentId(decoded.documentIds[i], `documentIds[${i}]`);
|
|
1155
1217
|
}
|
|
1218
|
+
decoded.fields ??= null;
|
|
1219
|
+
decoded.highlight = null;
|
|
1156
1220
|
if (decoded.fields !== null) {
|
|
1157
1221
|
validateStringArray(decoded.fields, "fields", MAX_FIELDS_LIST, 255, SEARCH_INVALID_FIELD);
|
|
1158
1222
|
}
|
|
1159
|
-
if (decoded.
|
|
1160
|
-
|
|
1223
|
+
if (decoded.vectors !== null && decoded.vectors !== void 0 && typeof decoded.vectors !== "boolean") {
|
|
1224
|
+
throwInvalid(CONFIG_INVALID, 'Invalid FetchPayload: "vectors" must be a boolean, or null');
|
|
1161
1225
|
}
|
|
1162
1226
|
if (decoded.contextId !== null && decoded.contextId !== void 0) {
|
|
1163
1227
|
validateStringField(decoded.contextId, "contextId", MAX_SEARCH_CONTEXT_ID_LENGTH, CONFIG_INVALID);
|
|
@@ -1183,9 +1247,7 @@ function validateFetchResultPayload(decoded) {
|
|
|
1183
1247
|
if (!isRecord3(doc.document)) {
|
|
1184
1248
|
throwInvalid(CONFIG_INVALID, `Invalid FetchResultPayload: "documents[${i}].document" must be an object`);
|
|
1185
1249
|
}
|
|
1186
|
-
|
|
1187
|
-
throwInvalid(CONFIG_INVALID, `Invalid FetchResultPayload: "documents[${i}].highlights" must be an object or null`);
|
|
1188
|
-
}
|
|
1250
|
+
doc.highlights = null;
|
|
1189
1251
|
}
|
|
1190
1252
|
return decoded;
|
|
1191
1253
|
}
|
|
@@ -1515,11 +1577,16 @@ var ALLOWED_MODES = ["fulltext", "vector", "hybrid"];
|
|
|
1515
1577
|
var ALLOWED_FACET_ORDERS = ["asc", "desc"];
|
|
1516
1578
|
var ALLOWED_METRICS = ["cosine", "dotProduct", "euclidean"];
|
|
1517
1579
|
var ALLOWED_HYBRID_STRATEGIES = ["rrf", "linear"];
|
|
1580
|
+
var HYBRID_OPTIONAL_MEMBERS = ["strategy", "k", "alpha", "depth"];
|
|
1581
|
+
var GROUP_OPTIONAL_MEMBERS = ["limit"];
|
|
1582
|
+
var FACET_OPTIONAL_MEMBERS = ["sort", "ranges"];
|
|
1583
|
+
var BM25_MEMBERS = ["k1", "b", "fields"];
|
|
1584
|
+
var BM25_FIELD_MEMBERS = ["k1", "b"];
|
|
1518
1585
|
function validateHybridParams(value) {
|
|
1519
1586
|
if (!isRecord3(value)) {
|
|
1520
1587
|
throwInvalid(CONFIG_INVALID, 'Invalid SearchPayload: "params.hybrid" must be an object');
|
|
1521
1588
|
}
|
|
1522
|
-
|
|
1589
|
+
restoreMissingAsNil(value, HYBRID_OPTIONAL_MEMBERS);
|
|
1523
1590
|
if (value.strategy !== null && !ALLOWED_HYBRID_STRATEGIES.includes(value.strategy)) {
|
|
1524
1591
|
throwInvalid(
|
|
1525
1592
|
SEARCH_INVALID_MODE,
|
|
@@ -1549,6 +1616,7 @@ function validateGroupParams(value) {
|
|
|
1549
1616
|
if (!isRecord3(value)) {
|
|
1550
1617
|
throwInvalid(CONFIG_INVALID, 'Invalid SearchPayload: "params.group" must be an object');
|
|
1551
1618
|
}
|
|
1619
|
+
restoreMissingAsNil(value, GROUP_OPTIONAL_MEMBERS);
|
|
1552
1620
|
const fields = validateStringArray(
|
|
1553
1621
|
value.fields,
|
|
1554
1622
|
"params.group.fields",
|
|
@@ -1609,6 +1677,7 @@ function validateFacetParams(value) {
|
|
|
1609
1677
|
if (!isRecord3(entry)) {
|
|
1610
1678
|
throwInvalid(CONFIG_INVALID, `Invalid SearchPayload: "params.facets[${i}]" must be an object`);
|
|
1611
1679
|
}
|
|
1680
|
+
restoreMissingAsNil(entry, FACET_OPTIONAL_MEMBERS);
|
|
1612
1681
|
validateFieldName(entry.field, `params.facets[${i}].field`, SEARCH_INVALID_FIELD);
|
|
1613
1682
|
if (entry.sort !== null && !ALLOWED_FACET_ORDERS.includes(entry.sort)) {
|
|
1614
1683
|
throwInvalid(
|
|
@@ -1621,6 +1690,31 @@ function validateFacetParams(value) {
|
|
|
1621
1690
|
}
|
|
1622
1691
|
}
|
|
1623
1692
|
}
|
|
1693
|
+
function presentMembers(value, members) {
|
|
1694
|
+
const kept = {};
|
|
1695
|
+
for (const member of members) {
|
|
1696
|
+
const memberValue = value[member];
|
|
1697
|
+
if (memberValue !== void 0 && memberValue !== null) kept[member] = memberValue;
|
|
1698
|
+
}
|
|
1699
|
+
return kept;
|
|
1700
|
+
}
|
|
1701
|
+
function validateWireBm25Param(params, member, ranges) {
|
|
1702
|
+
const value = params[member];
|
|
1703
|
+
if (value === void 0 || value === null) return;
|
|
1704
|
+
if (isRecord3(value)) {
|
|
1705
|
+
const kept = presentMembers(value, BM25_MEMBERS);
|
|
1706
|
+
if (isRecord3(kept.fields)) {
|
|
1707
|
+
kept.fields = Object.fromEntries(
|
|
1708
|
+
Object.entries(kept.fields).map(([name, pair]) => [
|
|
1709
|
+
name,
|
|
1710
|
+
isRecord3(pair) ? presentMembers(pair, BM25_FIELD_MEMBERS) : pair
|
|
1711
|
+
])
|
|
1712
|
+
);
|
|
1713
|
+
}
|
|
1714
|
+
params[member] = kept;
|
|
1715
|
+
}
|
|
1716
|
+
validateBM25Shape(params[member], ranges);
|
|
1717
|
+
}
|
|
1624
1718
|
function validateTermMatchParam(value) {
|
|
1625
1719
|
if (value === "all" || value === "any") {
|
|
1626
1720
|
return;
|
|
@@ -1723,6 +1817,16 @@ function validateOversampleParam(value) {
|
|
|
1723
1817
|
var MAX_FACET_SHARD_SIZE = oversampledShardSize(MAX_FACET_SIZE);
|
|
1724
1818
|
var ALLOWED_SCORING = ["local", "dfs", "broadcast"];
|
|
1725
1819
|
var ALLOWED_SORT_DIRECTIONS = ["asc", "desc"];
|
|
1820
|
+
var VECTOR_OPTIONAL_MEMBERS = [
|
|
1821
|
+
"value",
|
|
1822
|
+
"text",
|
|
1823
|
+
"similarity",
|
|
1824
|
+
"metric",
|
|
1825
|
+
"efSearch",
|
|
1826
|
+
"oversample",
|
|
1827
|
+
"rescoreWindow"
|
|
1828
|
+
];
|
|
1829
|
+
var SEARCH_OPTIONAL_MEMBERS = ["globalStats", "facetShardSize"];
|
|
1726
1830
|
function validateIndexNameField2(value, fieldLabel) {
|
|
1727
1831
|
if (typeof value !== "string") {
|
|
1728
1832
|
throwInvalid(CONFIG_INVALID, `Invalid SearchPayload: "${fieldLabel}" must be a string`);
|
|
@@ -1734,7 +1838,7 @@ function validateVectorParams(value) {
|
|
|
1734
1838
|
if (!isRecord3(value)) {
|
|
1735
1839
|
throwInvalid(CONFIG_INVALID, 'Invalid SearchPayload: "params.vector" must be an object');
|
|
1736
1840
|
}
|
|
1737
|
-
|
|
1841
|
+
restoreMissingAsNil(value, VECTOR_OPTIONAL_MEMBERS);
|
|
1738
1842
|
validateFieldName(value.field, "params.vector.field", SEARCH_INVALID_FIELD);
|
|
1739
1843
|
if (value.value !== null) {
|
|
1740
1844
|
if (!Array.isArray(value.value)) {
|
|
@@ -1808,6 +1912,7 @@ function validateSortParams(value) {
|
|
|
1808
1912
|
throwInvalid(CONFIG_INVALID, `Invalid SearchPayload: "params.sort[${i}]" must be an object`);
|
|
1809
1913
|
}
|
|
1810
1914
|
validateFieldName(entry.field, `params.sort[${i}].field`, SEARCH_INVALID_FIELD);
|
|
1915
|
+
dropNilMember(entry, "mode");
|
|
1811
1916
|
if (!ALLOWED_SORT_DIRECTIONS.includes(entry.direction)) {
|
|
1812
1917
|
throwInvalid(
|
|
1813
1918
|
SEARCH_INVALID_MODE,
|
|
@@ -1911,8 +2016,8 @@ function validateParams(params) {
|
|
|
1911
2016
|
if (params.boost !== null) {
|
|
1912
2017
|
validateBoostParams(params.boost);
|
|
1913
2018
|
}
|
|
1914
|
-
|
|
1915
|
-
|
|
2019
|
+
validateWireBm25Param(params, "bm25", "settable");
|
|
2020
|
+
validateWireBm25Param(params, "indexBm25", "stored");
|
|
1916
2021
|
validateAnalysisGeneration(params.analysisGeneration, "SearchPayload");
|
|
1917
2022
|
if (params.tolerance !== null) {
|
|
1918
2023
|
if (!isInteger(params.tolerance) || params.tolerance < 0 || params.tolerance > MAX_TOLERANCE) {
|
|
@@ -1980,6 +2085,7 @@ function validateSearchPayload(decoded) {
|
|
|
1980
2085
|
throwInvalid(CONFIG_INVALID, 'Invalid SearchPayload: "params" must be an object');
|
|
1981
2086
|
}
|
|
1982
2087
|
validateParams(decoded.params);
|
|
2088
|
+
restoreMissingAsNil(decoded, SEARCH_OPTIONAL_MEMBERS);
|
|
1983
2089
|
if (decoded.globalStats !== null) {
|
|
1984
2090
|
validateGlobalStatsShape(decoded.globalStats);
|
|
1985
2091
|
}
|
|
@@ -2020,6 +2126,7 @@ function validateCountPayload(decoded) {
|
|
|
2020
2126
|
return decoded;
|
|
2021
2127
|
}
|
|
2022
2128
|
var STORED_SIZE_FIELDS = ["savedBytes", "vectorSavedBytes", "logBytes"];
|
|
2129
|
+
var LIST_OPTIONAL_MEMBERS = ["cursor", "filters", "sort", "fields"];
|
|
2023
2130
|
function validateCountResultPayload(decoded) {
|
|
2024
2131
|
if (!isRecord3(decoded)) {
|
|
2025
2132
|
throwInvalid(CONFIG_INVALID, "Invalid CountResultPayload: expected an object");
|
|
@@ -2041,6 +2148,7 @@ function validateCountResultPayload(decoded) {
|
|
|
2041
2148
|
CONFIG_INVALID
|
|
2042
2149
|
);
|
|
2043
2150
|
for (const figure2 of STORED_SIZE_FIELDS) {
|
|
2151
|
+
dropNilMember(entry, figure2);
|
|
2044
2152
|
if (entry[figure2] === void 0) continue;
|
|
2045
2153
|
validateNonNegativeInteger(entry[figure2], `partitions[${i}].${figure2}`, MAX_COUNT_VALUE, CONFIG_INVALID);
|
|
2046
2154
|
}
|
|
@@ -2064,6 +2172,7 @@ function validateListSort(value) {
|
|
|
2064
2172
|
throwInvalid(CONFIG_INVALID, `Invalid ListPayload: "sort[${i}]" must be an object`);
|
|
2065
2173
|
}
|
|
2066
2174
|
validateFieldName(entry.field, `sort[${i}].field`, SEARCH_INVALID_FIELD);
|
|
2175
|
+
dropNilMember(entry, "mode");
|
|
2067
2176
|
if (entry.direction !== "asc" && entry.direction !== "desc") {
|
|
2068
2177
|
throwInvalid(SEARCH_INVALID_MODE, `Invalid ListPayload: "sort[${i}].direction" must be "asc" or "desc"`);
|
|
2069
2178
|
}
|
|
@@ -2081,6 +2190,7 @@ function validateListPayload(decoded) {
|
|
|
2081
2190
|
}
|
|
2082
2191
|
validateIndexNameField3(decoded, "ListPayload");
|
|
2083
2192
|
validatePartitionIdsArray(decoded.partitionIds, "partitionIds", CONFIG_INVALID);
|
|
2193
|
+
restoreMissingAsNil(decoded, LIST_OPTIONAL_MEMBERS);
|
|
2084
2194
|
if (decoded.cursor !== null) {
|
|
2085
2195
|
validateStringField(decoded.cursor, "cursor", MAX_LIST_CURSOR_LENGTH, CONFIG_INVALID);
|
|
2086
2196
|
}
|
|
@@ -2094,6 +2204,9 @@ function validateListPayload(decoded) {
|
|
|
2094
2204
|
if (decoded.fields !== null) {
|
|
2095
2205
|
validateStringArray(decoded.fields, "fields", MAX_FIELDS_LIST, 255, SEARCH_INVALID_FIELD);
|
|
2096
2206
|
}
|
|
2207
|
+
if (decoded.vectors !== null && decoded.vectors !== void 0 && typeof decoded.vectors !== "boolean") {
|
|
2208
|
+
throwInvalid(CONFIG_INVALID, 'Invalid ListPayload: "vectors" must be a boolean, or null');
|
|
2209
|
+
}
|
|
2097
2210
|
return decoded;
|
|
2098
2211
|
}
|
|
2099
2212
|
function validateSortValues(value, fieldLabel) {
|
|
@@ -2139,6 +2252,7 @@ function validateListResultPayload(decoded) {
|
|
|
2139
2252
|
if (!isRecord3(entry.document)) {
|
|
2140
2253
|
throwInvalid(CONFIG_INVALID, `Invalid ListResultPayload: "entries[${i}].document" must be an object`);
|
|
2141
2254
|
}
|
|
2255
|
+
entry.sortValues ??= null;
|
|
2142
2256
|
if (entry.sortValues !== null) {
|
|
2143
2257
|
validateSortValues(entry.sortValues, `entries[${i}].sortValues`);
|
|
2144
2258
|
}
|
|
@@ -3890,7 +4004,8 @@ function validatePartitionStoresResultPayload(decoded) {
|
|
|
3890
4004
|
if (typeof decoded.indexName !== "string" || decoded.indexName.length === 0) {
|
|
3891
4005
|
return null;
|
|
3892
4006
|
}
|
|
3893
|
-
|
|
4007
|
+
const indexUuid = decoded.indexUuid ?? null;
|
|
4008
|
+
if (indexUuid !== null && typeof indexUuid !== "string") {
|
|
3894
4009
|
return null;
|
|
3895
4010
|
}
|
|
3896
4011
|
if (!Array.isArray(decoded.partitionIds)) {
|
|
@@ -3903,7 +4018,7 @@ function validatePartitionStoresResultPayload(decoded) {
|
|
|
3903
4018
|
}
|
|
3904
4019
|
partitionIds.push(partitionId);
|
|
3905
4020
|
}
|
|
3906
|
-
return { indexName: decoded.indexName, indexUuid
|
|
4021
|
+
return { indexName: decoded.indexName, indexUuid, partitionIds };
|
|
3907
4022
|
}
|
|
3908
4023
|
function createPartitionStoresMessage(payload, sourceNodeId, requestId) {
|
|
3909
4024
|
return {
|
|
@@ -4673,8 +4788,9 @@ async function writeLocalOnly(deps, indexName, write) {
|
|
|
4673
4788
|
deps.writeRecords?.dropIndex(indexName);
|
|
4674
4789
|
}
|
|
4675
4790
|
}
|
|
4791
|
+
var WHOLE_DOCUMENT = { document: { vectors: true } };
|
|
4676
4792
|
async function storedCopyOf(scope, documentId) {
|
|
4677
|
-
const stored = await scope.deps.engine.get(scope.indexName, documentId);
|
|
4793
|
+
const stored = await scope.deps.engine.get(scope.indexName, documentId, WHOLE_DOCUMENT);
|
|
4678
4794
|
if (stored === void 0) {
|
|
4679
4795
|
throw new NarsilError(
|
|
4680
4796
|
ErrorCodes.REPLICATION_ENTRY_INVALID,
|
|
@@ -5203,6 +5319,24 @@ async function compensateRemove(scope, documentId, previousDocument, originalErr
|
|
|
5203
5319
|
}
|
|
5204
5320
|
}
|
|
5205
5321
|
|
|
5322
|
+
// src/distribution/cluster-node/write-routing/forwarded-document.ts
|
|
5323
|
+
function decodeForwardedDocument(bytes, documentId) {
|
|
5324
|
+
let decoded;
|
|
5325
|
+
try {
|
|
5326
|
+
decoded = decode(bytes);
|
|
5327
|
+
} catch (err) {
|
|
5328
|
+
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Forwarded document could not be decoded: ${String(err)}`, {
|
|
5329
|
+
documentId
|
|
5330
|
+
});
|
|
5331
|
+
}
|
|
5332
|
+
if (!isRecord(decoded)) {
|
|
5333
|
+
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "A forwarded document must be a MessagePack map", {
|
|
5334
|
+
documentId
|
|
5335
|
+
});
|
|
5336
|
+
}
|
|
5337
|
+
return decoded;
|
|
5338
|
+
}
|
|
5339
|
+
|
|
5206
5340
|
// src/distribution/cluster-node/reshard/state.ts
|
|
5207
5341
|
function createReshardState() {
|
|
5208
5342
|
return {
|
|
@@ -5326,13 +5460,13 @@ function isRecord5(value) {
|
|
|
5326
5460
|
function requireCount(record, field, payloadName) {
|
|
5327
5461
|
const value = record[field];
|
|
5328
5462
|
if (typeof value !== "number" || !Number.isInteger(value) || value < 0) {
|
|
5329
|
-
throw
|
|
5463
|
+
throw invalidReplicationPayload(`Invalid ${payloadName}: "${field}" must be a non-negative integer`);
|
|
5330
5464
|
}
|
|
5331
5465
|
return value;
|
|
5332
5466
|
}
|
|
5333
5467
|
function requireIndexName(record, payloadName) {
|
|
5334
5468
|
if (typeof record.indexName !== "string" || record.indexName.length === 0) {
|
|
5335
|
-
throw
|
|
5469
|
+
throw invalidReplicationPayload(`Invalid ${payloadName}: "indexName" must be a non-empty string`);
|
|
5336
5470
|
}
|
|
5337
5471
|
return record.indexName;
|
|
5338
5472
|
}
|
|
@@ -5353,9 +5487,9 @@ function createTermStartAckMessage(payload, sourceId, requestId) {
|
|
|
5353
5487
|
};
|
|
5354
5488
|
}
|
|
5355
5489
|
function validateTermStartPayload(payload) {
|
|
5356
|
-
const decoded =
|
|
5490
|
+
const decoded = decodeRequestPayload(payload, ReplicationMessageTypes.TERM_START);
|
|
5357
5491
|
if (!isRecord5(decoded)) {
|
|
5358
|
-
throw
|
|
5492
|
+
throw invalidReplicationPayload("Invalid TermStartPayload: expected an object");
|
|
5359
5493
|
}
|
|
5360
5494
|
const validated = {
|
|
5361
5495
|
indexName: requireIndexName(decoded, "TermStartPayload"),
|
|
@@ -6169,7 +6303,7 @@ async function applyPrimaryRemove(indexName, docId, partitionId, assignment, dep
|
|
|
6169
6303
|
const queued = { indexName, partitionId, assignment, deps };
|
|
6170
6304
|
await inLeadingPartitionQueue(queued, async (scope) => {
|
|
6171
6305
|
if (condition !== void 0 && decideCondition(scope, docId, condition) === "acknowledge") return;
|
|
6172
|
-
const previousDocument = await deps.engine.get(indexName, docId);
|
|
6306
|
+
const previousDocument = await deps.engine.get(indexName, docId, WHOLE_DOCUMENT);
|
|
6173
6307
|
const previousRecord = previousRecordOf(scope, docId);
|
|
6174
6308
|
const removed = await numberedWrite(
|
|
6175
6309
|
scope,
|
|
@@ -6193,7 +6327,7 @@ async function applyPrimaryUpdate(indexName, docId, document, partitionId, assig
|
|
|
6193
6327
|
if (condition !== void 0 && decideCondition(scope, docId, condition) === "acknowledge") return;
|
|
6194
6328
|
const projected = projectOne(deps, indexName, document);
|
|
6195
6329
|
try {
|
|
6196
|
-
const previousDocument = await deps.engine.get(indexName, docId);
|
|
6330
|
+
const previousDocument = await deps.engine.get(indexName, docId, WHOLE_DOCUMENT);
|
|
6197
6331
|
const previousRecord = previousRecordOf(scope, docId);
|
|
6198
6332
|
const updated = await numberedWrite(
|
|
6199
6333
|
scope,
|
|
@@ -6278,7 +6412,7 @@ async function applyPrimaryUpdateBatch(indexName, updates, partitionId, assignme
|
|
|
6278
6412
|
acknowledged.push(update.docId);
|
|
6279
6413
|
return;
|
|
6280
6414
|
}
|
|
6281
|
-
const previousDocument = await deps.engine.get(indexName, update.docId);
|
|
6415
|
+
const previousDocument = await deps.engine.get(indexName, update.docId, WHOLE_DOCUMENT);
|
|
6282
6416
|
const previousRecord = previousRecordOf(scope, update.docId);
|
|
6283
6417
|
const updated = await numberedWrite(
|
|
6284
6418
|
scope,
|
|
@@ -6325,7 +6459,7 @@ async function applyPrimaryRemoveBatch(indexName, docIds, partitionId, assignmen
|
|
|
6325
6459
|
acknowledged.push(docId);
|
|
6326
6460
|
continue;
|
|
6327
6461
|
}
|
|
6328
|
-
const previousDocument = await deps.engine.get(indexName, docId);
|
|
6462
|
+
const previousDocument = await deps.engine.get(indexName, docId, WHOLE_DOCUMENT);
|
|
6329
6463
|
const previousRecord = previousRecordOf(scope, docId);
|
|
6330
6464
|
const removed = await numberedWrite(
|
|
6331
6465
|
scope,
|
|
@@ -6378,9 +6512,9 @@ async function applyAdmittedForwardedWrite(payload, deps) {
|
|
|
6378
6512
|
}
|
|
6379
6513
|
if (payload.operation === "insert") {
|
|
6380
6514
|
if (payload.document === null) {
|
|
6381
|
-
throw new NarsilError(ErrorCodes.
|
|
6515
|
+
throw new NarsilError(ErrorCodes.REPLICATION_ENTRY_INVALID, "Invalid ForwardPayload: insert requires a document");
|
|
6382
6516
|
}
|
|
6383
|
-
const document =
|
|
6517
|
+
const document = decodeForwardedDocument(payload.document, payload.documentId);
|
|
6384
6518
|
return applyPrimaryInsert(
|
|
6385
6519
|
payload.indexName,
|
|
6386
6520
|
document,
|
|
@@ -6421,15 +6555,15 @@ async function applyAdmittedForwardedWrite(payload, deps) {
|
|
|
6421
6555
|
}
|
|
6422
6556
|
async function resolveForwardedUpdateDocument(indexName, documentId, document, updateFields, deps) {
|
|
6423
6557
|
if (document !== null) {
|
|
6424
|
-
return
|
|
6558
|
+
return decodeForwardedDocument(document, documentId);
|
|
6425
6559
|
}
|
|
6426
6560
|
if (updateFields === null) {
|
|
6427
6561
|
throw new NarsilError(
|
|
6428
|
-
ErrorCodes.
|
|
6562
|
+
ErrorCodes.REPLICATION_ENTRY_INVALID,
|
|
6429
6563
|
"Invalid ForwardPayload: update requires a document or updateFields"
|
|
6430
6564
|
);
|
|
6431
6565
|
}
|
|
6432
|
-
const existing = await deps.engine.get(indexName, documentId);
|
|
6566
|
+
const existing = await deps.engine.get(indexName, documentId, WHOLE_DOCUMENT);
|
|
6433
6567
|
if (existing === void 0) {
|
|
6434
6568
|
throw new NarsilError(ErrorCodes.DOC_NOT_FOUND, `Document '${documentId}' does not exist in index '${indexName}'`, {
|
|
6435
6569
|
indexName,
|
|
@@ -6979,21 +7113,17 @@ function decodeOperationDocument(wire, results, index) {
|
|
|
6979
7113
|
if (wire.document === null) {
|
|
6980
7114
|
results[index] = failureResult(
|
|
6981
7115
|
wire.documentId,
|
|
6982
|
-
new NarsilError(ErrorCodes.
|
|
7116
|
+
new NarsilError(ErrorCodes.REPLICATION_ENTRY_INVALID, "A forwarded insert requires a document", {
|
|
6983
7117
|
documentId: wire.documentId
|
|
6984
7118
|
})
|
|
6985
7119
|
);
|
|
6986
7120
|
return null;
|
|
6987
7121
|
}
|
|
6988
7122
|
try {
|
|
6989
|
-
return
|
|
7123
|
+
return decodeForwardedDocument(wire.document, wire.documentId);
|
|
6990
7124
|
} catch (err) {
|
|
6991
|
-
|
|
6992
|
-
|
|
6993
|
-
new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Forwarded document could not be decoded: ${String(err)}`, {
|
|
6994
|
-
documentId: wire.documentId
|
|
6995
|
-
})
|
|
6996
|
-
);
|
|
7125
|
+
const error = err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, String(err));
|
|
7126
|
+
results[index] = failureResult(wire.documentId, error);
|
|
6997
7127
|
return null;
|
|
6998
7128
|
}
|
|
6999
7129
|
}
|
|
@@ -7032,6 +7162,7 @@ async function routeCreateIndex(name, requestedConfig, options, coordinator, eng
|
|
|
7032
7162
|
const config = { ...requestedConfig, schema: validateSchema(requestedConfig.schema) };
|
|
7033
7163
|
requireCreatableSchema(config.schema);
|
|
7034
7164
|
requireSurfaceFormsForPartialFields(config.schema, config.surfaceForms);
|
|
7165
|
+
validateReturnVectors(config.returnVectors);
|
|
7035
7166
|
if (config.bm25 !== void 0) validateBM25Params(config.bm25, config.schema, ErrorCodes.CONFIG_INVALID);
|
|
7036
7167
|
const setsFieldPairs = config.bm25?.fields !== void 0 && bm25SetsValue({ fields: config.bm25.fields });
|
|
7037
7168
|
if (setsFieldPairs) {
|
|
@@ -7324,7 +7455,7 @@ function refuseDecodeWithoutRoom(payload) {
|
|
|
7324
7455
|
const refusal2 = answerRefusalOf(payload.byteLength * ANSWER_DECODE_HEAP_BYTES_PER_PAYLOAD_BYTE);
|
|
7325
7456
|
if (refusal2 !== null) throw markNodeRejection(refusal2);
|
|
7326
7457
|
}
|
|
7327
|
-
async function readFromCopies(config, nodeId, engine, indexName, documentRouting,
|
|
7458
|
+
async function readFromCopies(config, nodeId, engine, indexName, documentRouting, read) {
|
|
7328
7459
|
const documents = /* @__PURE__ */ new Map();
|
|
7329
7460
|
await sendToEveryPartition(documentRouting.routing, async (targetNodeId, partitionIds) => {
|
|
7330
7461
|
const documentIds = partitionIds.flatMap(
|
|
@@ -7334,13 +7465,23 @@ async function readFromCopies(config, nodeId, engine, indexName, documentRouting
|
|
|
7334
7465
|
if (targetNodeId === nodeId) {
|
|
7335
7466
|
const local = await engine.getMultiple(
|
|
7336
7467
|
indexName,
|
|
7337
|
-
documentIds.map((ref) => ref.docId)
|
|
7468
|
+
documentIds.map((ref) => ref.docId),
|
|
7469
|
+
{ document: { vectors: read.vectors } }
|
|
7338
7470
|
);
|
|
7339
7471
|
for (const [docId, document] of local) found.set(docId, document);
|
|
7340
7472
|
} else {
|
|
7341
7473
|
for (let start = 0; start < documentIds.length; start += FETCH_GROUP_DOCUMENT_IDS) {
|
|
7342
7474
|
const chunk = documentIds.slice(start, start + FETCH_GROUP_DOCUMENT_IDS);
|
|
7343
|
-
const fetchMessage = createFetchMessage(
|
|
7475
|
+
const fetchMessage = createFetchMessage(
|
|
7476
|
+
{
|
|
7477
|
+
indexName,
|
|
7478
|
+
documentIds: chunk,
|
|
7479
|
+
fields: read.fields,
|
|
7480
|
+
...read.vectors ? {} : { vectors: false },
|
|
7481
|
+
highlight: null
|
|
7482
|
+
},
|
|
7483
|
+
nodeId
|
|
7484
|
+
);
|
|
7344
7485
|
const response = await sendToNode2(config, targetNodeId, fetchMessage, {
|
|
7345
7486
|
message: `Node '${targetNodeId}' did not answer a document fetch for index '${indexName}'`,
|
|
7346
7487
|
details: { indexName }
|
|
@@ -7361,7 +7502,7 @@ async function readFromCopies(config, nodeId, engine, indexName, documentRouting
|
|
|
7361
7502
|
});
|
|
7362
7503
|
return documents;
|
|
7363
7504
|
}
|
|
7364
|
-
async function readDocumentsFromCopies(config, nodeId, engine, indexName, docIds, allocation,
|
|
7505
|
+
async function readDocumentsFromCopies(config, nodeId, engine, indexName, docIds, allocation, read, requireEveryPartition) {
|
|
7365
7506
|
const documentRouting = routeDocumentIds(docIds, allocation);
|
|
7366
7507
|
const unreachablePartitions = documentRouting.routing.unavailablePartitions;
|
|
7367
7508
|
if (requireEveryPartition && unreachablePartitions.length > 0) {
|
|
@@ -7371,10 +7512,10 @@ async function readDocumentsFromCopies(config, nodeId, engine, indexName, docIds
|
|
|
7371
7512
|
{ indexName, partitionIds: [...unreachablePartitions].sort((a, b) => a - b) }
|
|
7372
7513
|
);
|
|
7373
7514
|
}
|
|
7374
|
-
return readFromCopies(config, nodeId, engine, indexName, documentRouting,
|
|
7515
|
+
return readFromCopies(config, nodeId, engine, indexName, documentRouting, read);
|
|
7375
7516
|
}
|
|
7376
|
-
async function readDistributedDocuments(config, nodeId, engine, indexName, docIds, allocation) {
|
|
7377
|
-
return readDocumentsFromCopies(config, nodeId, engine, indexName, docIds, allocation, null, true);
|
|
7517
|
+
async function readDistributedDocuments(config, nodeId, engine, indexName, docIds, allocation, vectors) {
|
|
7518
|
+
return readDocumentsFromCopies(config, nodeId, engine, indexName, docIds, allocation, { fields: null, vectors }, true);
|
|
7378
7519
|
}
|
|
7379
7520
|
|
|
7380
7521
|
// src/distribution/backups/partition-exchange.ts
|
|
@@ -7489,17 +7630,6 @@ function readPartitionResult(value, backupName, partitionId, partitionCount) {
|
|
|
7489
7630
|
throw unreadableResult(backupName, partitionId);
|
|
7490
7631
|
}
|
|
7491
7632
|
}
|
|
7492
|
-
function sameConfigOf(results, indexName) {
|
|
7493
|
-
const [first] = results;
|
|
7494
|
-
if (first === void 0 || results.some((result) => result.config.sha256 !== first.config.sha256)) {
|
|
7495
|
-
throw new NarsilError(
|
|
7496
|
-
ErrorCodes.ANALYSIS_CHANGE_NOT_READY,
|
|
7497
|
-
`The settings of index '${indexName}' changed while its partitions were backed up, so the backup stores no manifest`,
|
|
7498
|
-
{ indexName }
|
|
7499
|
-
);
|
|
7500
|
-
}
|
|
7501
|
-
return first.config;
|
|
7502
|
-
}
|
|
7503
7633
|
async function backUpEveryPartition(context, generation, partitionCount) {
|
|
7504
7634
|
const { jobId, job } = context;
|
|
7505
7635
|
const results = new Array(partitionCount);
|
|
@@ -7540,7 +7670,7 @@ async function runBackupCreation(context, request) {
|
|
|
7540
7670
|
const partitions = results.map((result) => result.partition);
|
|
7541
7671
|
const config = await configWithSuggestionState(
|
|
7542
7672
|
context,
|
|
7543
|
-
|
|
7673
|
+
await agreedConfigOf(store, results, indexName, job.backupName),
|
|
7544
7674
|
suggestionStateOfSettings(named)
|
|
7545
7675
|
);
|
|
7546
7676
|
const queryHistory = request.queryHistory ? await backUpQueryHistory(context, indexName) : void 0;
|
|
@@ -7852,12 +7982,7 @@ function malformed3(kind, reason) {
|
|
|
7852
7982
|
});
|
|
7853
7983
|
}
|
|
7854
7984
|
function decodeRequest(kind, message) {
|
|
7855
|
-
|
|
7856
|
-
try {
|
|
7857
|
-
decoded = decode(message.payload);
|
|
7858
|
-
} catch (_) {
|
|
7859
|
-
throw malformed3(kind, "the payload is not MessagePack");
|
|
7860
|
-
}
|
|
7985
|
+
const decoded = decodeRequestPayload(message.payload, message.type);
|
|
7861
7986
|
if (!isRecord(decoded)) throw malformed3(kind, "expected an object");
|
|
7862
7987
|
return decoded;
|
|
7863
7988
|
}
|
|
@@ -7913,14 +8038,14 @@ async function startJob(deps, store, job, run, afterJob) {
|
|
|
7913
8038
|
return { jobId, ended };
|
|
7914
8039
|
}
|
|
7915
8040
|
function takenByOf(request) {
|
|
7916
|
-
if (request.takenBy === void 0) return void 0;
|
|
8041
|
+
if (request.takenBy === void 0 || request.takenBy === null) return void 0;
|
|
7917
8042
|
if (!isBackupTaker(request.takenBy)) {
|
|
7918
8043
|
throw malformed3("backup", `"takenBy" must be "${SYNC_BACKUP_TAKER}" or "${CHANGES_BACKUP_TAKER}"`);
|
|
7919
8044
|
}
|
|
7920
8045
|
return request.takenBy;
|
|
7921
8046
|
}
|
|
7922
8047
|
function queryHistoryOf(request) {
|
|
7923
|
-
if (request.queryHistory === void 0) return false;
|
|
8048
|
+
if (request.queryHistory === void 0 || request.queryHistory === null) return false;
|
|
7924
8049
|
if (typeof request.queryHistory !== "boolean") throw malformed3("backup", '"queryHistory" must be a boolean');
|
|
7925
8050
|
return request.queryHistory;
|
|
7926
8051
|
}
|
|
@@ -8297,12 +8422,7 @@ function invalid(what) {
|
|
|
8297
8422
|
return new NarsilError(ErrorCodes.CONFIG_INVALID, `Invalid backup partition message: ${what}`);
|
|
8298
8423
|
}
|
|
8299
8424
|
function readPayload(message) {
|
|
8300
|
-
|
|
8301
|
-
try {
|
|
8302
|
-
decoded = decode(message.payload);
|
|
8303
|
-
} catch (_) {
|
|
8304
|
-
decoded = null;
|
|
8305
|
-
}
|
|
8425
|
+
const decoded = decodeRequestPayload(message.payload, message.type);
|
|
8306
8426
|
if (!isRecord(decoded)) throw invalid("expected an object");
|
|
8307
8427
|
const { jobId, indexName, partitionId } = decoded;
|
|
8308
8428
|
if (typeof jobId !== "string" || jobId.length === 0) throw invalid('"jobId" must be a non-empty string');
|
|
@@ -8713,280 +8833,59 @@ async function openSnapshotSpoolDirectory(durabilityDirectory) {
|
|
|
8713
8833
|
return { path, dispose: () => rm(path, { recursive: true, force: true }) };
|
|
8714
8834
|
}
|
|
8715
8835
|
|
|
8716
|
-
// src/distribution/cluster-node/
|
|
8717
|
-
|
|
8718
|
-
|
|
8719
|
-
|
|
8720
|
-
|
|
8721
|
-
|
|
8722
|
-
|
|
8723
|
-
};
|
|
8724
|
-
}
|
|
8725
|
-
function generationMissing(indexName, requestedGeneration, heldGeneration) {
|
|
8726
|
-
return new NarsilError(
|
|
8727
|
-
ErrorCodes.ANALYSIS_GENERATION_MISSING,
|
|
8728
|
-
`This node holds analysis generation ${heldGeneration} of index '${indexName}', so it cannot answer for generation ${requestedGeneration}`,
|
|
8729
|
-
{ indexName, requestedGeneration, heldGeneration }
|
|
8730
|
-
);
|
|
8836
|
+
// src/distribution/cluster-node/schema-diff.ts
|
|
8837
|
+
var SCHEMA_DIFF_MAX_DEPTH = MAX_NESTING_DEPTH;
|
|
8838
|
+
var DEPTH_EXCEEDED_SENTINEL = "depth-exceeded";
|
|
8839
|
+
function diffSchemas(expected, actual) {
|
|
8840
|
+
const diffs = [];
|
|
8841
|
+
walk(expected, actual, "", diffs, 1);
|
|
8842
|
+
return diffs;
|
|
8731
8843
|
}
|
|
8732
|
-
function
|
|
8733
|
-
|
|
8734
|
-
|
|
8735
|
-
|
|
8736
|
-
|
|
8737
|
-
|
|
8738
|
-
function recordGeneration(indexName, snapshot) {
|
|
8739
|
-
const generation = snapshot.settings?.analysisGeneration ?? 0;
|
|
8740
|
-
const { indexUuid } = snapshot.record;
|
|
8741
|
-
const known = held.get(indexName);
|
|
8742
|
-
if (known?.indexUuid === indexUuid && known.generation >= generation) return known.generation;
|
|
8743
|
-
held.set(indexName, { indexUuid, generation });
|
|
8744
|
-
return generation;
|
|
8745
|
-
}
|
|
8746
|
-
async function refresh(indexName) {
|
|
8747
|
-
const snapshot = await readIndexMetadataSnapshot(coordinator, indexName);
|
|
8748
|
-
if (snapshot === null) {
|
|
8749
|
-
held.delete(indexName);
|
|
8750
|
-
return 0;
|
|
8751
|
-
}
|
|
8752
|
-
return recordGeneration(indexName, snapshot);
|
|
8753
|
-
}
|
|
8754
|
-
function catchUp(indexName, generation, settings) {
|
|
8755
|
-
engine.buildHeldAnalysis(indexName, { generation, ...namedSettingsOfMetadata(settings) }).then(() => engine.releaseHeldAnalysis(indexName, generation)).catch((error) => {
|
|
8756
|
-
if (!stopped) onError(error);
|
|
8844
|
+
function walk(expected, actual, prefix, diffs, depth) {
|
|
8845
|
+
if (depth > SCHEMA_DIFF_MAX_DEPTH) {
|
|
8846
|
+
diffs.push({
|
|
8847
|
+
path: prefix === "" ? DEPTH_EXCEEDED_SENTINEL : prefix,
|
|
8848
|
+
expected: DEPTH_EXCEEDED_SENTINEL,
|
|
8849
|
+
actual: DEPTH_EXCEEDED_SENTINEL
|
|
8757
8850
|
});
|
|
8851
|
+
return;
|
|
8758
8852
|
}
|
|
8759
|
-
|
|
8760
|
-
|
|
8761
|
-
|
|
8762
|
-
const
|
|
8763
|
-
const
|
|
8764
|
-
|
|
8765
|
-
|
|
8766
|
-
|
|
8767
|
-
|
|
8768
|
-
const localPending = engine.pendingAnalysisGenerationOf(indexName);
|
|
8769
|
-
if (published > generation) {
|
|
8770
|
-
if (localPending !== published && engine.holdsAnalysisSettings(indexName, namedSettingsOfMetadata(settings))) {
|
|
8771
|
-
if (localPending !== null) await engine.dropHeldAnalysis(indexName);
|
|
8772
|
-
await engine.stampAnalysisGeneration(indexName, published);
|
|
8773
|
-
return;
|
|
8774
|
-
}
|
|
8775
|
-
catchUp(indexName, published, settings);
|
|
8776
|
-
return;
|
|
8853
|
+
const expectedKeys = Object.keys(expected).sort(compareCodePoints);
|
|
8854
|
+
const actualKeys = new Set(Object.keys(actual));
|
|
8855
|
+
for (const key of expectedKeys) {
|
|
8856
|
+
const path = prefix === "" ? key : `${prefix}.${key}`;
|
|
8857
|
+
const expectedValue = expected[key];
|
|
8858
|
+
const actualValue = actual[key];
|
|
8859
|
+
if (!actualKeys.has(key)) {
|
|
8860
|
+
diffs.push({ path, expected: describe(expectedValue), actual: "(absent)" });
|
|
8861
|
+
continue;
|
|
8777
8862
|
}
|
|
8778
|
-
|
|
8779
|
-
|
|
8863
|
+
actualKeys.delete(key);
|
|
8864
|
+
const expectedIsObject = typeof expectedValue === "object" && expectedValue !== null;
|
|
8865
|
+
const actualIsObject = typeof actualValue === "object" && actualValue !== null;
|
|
8866
|
+
if (expectedIsObject !== actualIsObject) {
|
|
8867
|
+
diffs.push({ path, expected: describe(expectedValue), actual: describe(actualValue) });
|
|
8868
|
+
continue;
|
|
8869
|
+
}
|
|
8870
|
+
if (expectedIsObject && actualIsObject) {
|
|
8871
|
+
walk(expectedValue, actualValue, path, diffs, depth + 1);
|
|
8872
|
+
continue;
|
|
8873
|
+
}
|
|
8874
|
+
if (expectedValue !== actualValue) {
|
|
8875
|
+
diffs.push({ path, expected: String(expectedValue), actual: String(actualValue) });
|
|
8780
8876
|
}
|
|
8781
8877
|
}
|
|
8782
|
-
|
|
8783
|
-
const
|
|
8784
|
-
|
|
8785
|
-
return next;
|
|
8878
|
+
for (const extraKey of actualKeys) {
|
|
8879
|
+
const path = prefix === "" ? extraKey : `${prefix}.${extraKey}`;
|
|
8880
|
+
diffs.push({ path, expected: "(absent)", actual: describe(actual[extraKey]) });
|
|
8786
8881
|
}
|
|
8787
|
-
|
|
8788
|
-
|
|
8789
|
-
|
|
8790
|
-
|
|
8791
|
-
}
|
|
8792
|
-
await settle(indexName);
|
|
8882
|
+
}
|
|
8883
|
+
function describe(value) {
|
|
8884
|
+
if (value === null) {
|
|
8885
|
+
return "null";
|
|
8793
8886
|
}
|
|
8794
|
-
|
|
8795
|
-
|
|
8796
|
-
const reached = engine.analysisGenerationOf(indexName) ?? generation;
|
|
8797
|
-
if (reached !== requested) throw generationMissing(indexName, requested, reached);
|
|
8798
|
-
}
|
|
8799
|
-
return {
|
|
8800
|
-
held(indexName) {
|
|
8801
|
-
return held.get(indexName)?.generation;
|
|
8802
|
-
},
|
|
8803
|
-
async generationFor(indexName) {
|
|
8804
|
-
return held.get(indexName)?.generation ?? refresh(indexName);
|
|
8805
|
-
},
|
|
8806
|
-
refresh,
|
|
8807
|
-
answerAt(indexName, requested, read) {
|
|
8808
|
-
const generation = engine.analysisGenerationOf(indexName);
|
|
8809
|
-
if (generation === void 0) return read();
|
|
8810
|
-
const readAtRequested = (answerOf3) => answerOf3().then((answer2) => {
|
|
8811
|
-
const after = engine.analysisGenerationOf(indexName) ?? requested;
|
|
8812
|
-
if (after !== requested) throw generationMissing(indexName, requested, after);
|
|
8813
|
-
return answer2;
|
|
8814
|
-
});
|
|
8815
|
-
if (generation === requested) return readAtRequested(read);
|
|
8816
|
-
return reachThenRead(indexName, requested, generation).then(() => readAtRequested(read));
|
|
8817
|
-
},
|
|
8818
|
-
settle,
|
|
8819
|
-
async start() {
|
|
8820
|
-
if (unwatch !== null) return;
|
|
8821
|
-
stopped = false;
|
|
8822
|
-
unwatch = await coordinator.watchIndexMetadata((event) => {
|
|
8823
|
-
if (event.metadata === null) {
|
|
8824
|
-
held.delete(event.indexName);
|
|
8825
|
-
return;
|
|
8826
|
-
}
|
|
8827
|
-
try {
|
|
8828
|
-
recordGeneration(event.indexName, indexMetadataSnapshotOf(event.metadata, event.indexName));
|
|
8829
|
-
} catch (error) {
|
|
8830
|
-
onError(error);
|
|
8831
|
-
}
|
|
8832
|
-
void settle(event.indexName);
|
|
8833
|
-
});
|
|
8834
|
-
for (const indexName of await coordinator.listSchemas()) {
|
|
8835
|
-
if (stopped) return;
|
|
8836
|
-
if (!held.has(indexName)) await refresh(indexName).catch(onError);
|
|
8837
|
-
}
|
|
8838
|
-
},
|
|
8839
|
-
stop() {
|
|
8840
|
-
stopped = true;
|
|
8841
|
-
unwatch?.();
|
|
8842
|
-
unwatch = null;
|
|
8843
|
-
held.clear();
|
|
8844
|
-
}
|
|
8845
|
-
};
|
|
8846
|
-
}
|
|
8847
|
-
|
|
8848
|
-
// src/distribution/cluster-node/bm25-view.ts
|
|
8849
|
-
function settingOfMetadata(bytes, indexName) {
|
|
8850
|
-
const decoded = decode(bytes);
|
|
8851
|
-
if (!isRecord2(decoded) || typeof decoded.indexUuid !== "string") return null;
|
|
8852
|
-
const settings = decodeIndexSettings(decoded.settings, indexName);
|
|
8853
|
-
return { indexUuid: decoded.indexUuid, bm25: settings?.bm25, revision: settings?.bm25Revision };
|
|
8854
|
-
}
|
|
8855
|
-
function isLaterThan(next, held) {
|
|
8856
|
-
return (next.revision ?? 0) > (held.revision ?? 0);
|
|
8857
|
-
}
|
|
8858
|
-
function createBm25View(deps) {
|
|
8859
|
-
const { coordinator, engine, onError } = deps;
|
|
8860
|
-
const held = /* @__PURE__ */ new Map();
|
|
8861
|
-
let unwatch = null;
|
|
8862
|
-
function applyToLocalCopy(indexName, indexUuid) {
|
|
8863
|
-
if (engine.indexUuidOf(indexName) !== indexUuid) return Promise.resolve();
|
|
8864
|
-
return engine.applyStoredBm25(indexName, () => {
|
|
8865
|
-
const latest = held.get(indexName);
|
|
8866
|
-
if (latest === void 0 || latest.indexUuid !== indexUuid) return null;
|
|
8867
|
-
return engine.indexUuidOf(indexName) === indexUuid ? { bm25: latest.bm25 } : null;
|
|
8868
|
-
}).then(
|
|
8869
|
-
() => void 0,
|
|
8870
|
-
(error) => {
|
|
8871
|
-
if (error instanceof NarsilError && error.code === ErrorCodes.INDEX_NOT_FOUND) return;
|
|
8872
|
-
onError(error);
|
|
8873
|
-
}
|
|
8874
|
-
);
|
|
8875
|
-
}
|
|
8876
|
-
function hold(indexName, next) {
|
|
8877
|
-
const current2 = held.get(indexName);
|
|
8878
|
-
if (current2 !== void 0 && current2.indexUuid === next.indexUuid && !isLaterThan(next, current2)) return false;
|
|
8879
|
-
held.set(indexName, next);
|
|
8880
|
-
return true;
|
|
8881
|
-
}
|
|
8882
|
-
function receive(indexName, metadata) {
|
|
8883
|
-
if (metadata === null) {
|
|
8884
|
-
held.delete(indexName);
|
|
8885
|
-
return;
|
|
8886
|
-
}
|
|
8887
|
-
let next;
|
|
8888
|
-
try {
|
|
8889
|
-
next = settingOfMetadata(metadata, indexName);
|
|
8890
|
-
} catch (error) {
|
|
8891
|
-
onError(error);
|
|
8892
|
-
return;
|
|
8893
|
-
}
|
|
8894
|
-
if (next !== null && hold(indexName, next)) void applyToLocalCopy(indexName, next.indexUuid);
|
|
8895
|
-
}
|
|
8896
|
-
async function current(indexName) {
|
|
8897
|
-
const known = held.get(indexName);
|
|
8898
|
-
if (known !== void 0) return known;
|
|
8899
|
-
const snapshot = await readIndexMetadataSnapshot(coordinator, indexName);
|
|
8900
|
-
if (snapshot === null) return null;
|
|
8901
|
-
const read = {
|
|
8902
|
-
indexUuid: snapshot.record.indexUuid,
|
|
8903
|
-
bm25: snapshot.settings?.bm25,
|
|
8904
|
-
revision: snapshot.settings?.bm25Revision
|
|
8905
|
-
};
|
|
8906
|
-
const racing = held.get(indexName);
|
|
8907
|
-
if (racing === void 0 || racing.indexUuid === read.indexUuid && isLaterThan(read, racing)) {
|
|
8908
|
-
held.set(indexName, read);
|
|
8909
|
-
return read;
|
|
8910
|
-
}
|
|
8911
|
-
return racing;
|
|
8912
|
-
}
|
|
8913
|
-
return {
|
|
8914
|
-
held: (indexName) => held.get(indexName),
|
|
8915
|
-
current,
|
|
8916
|
-
async start() {
|
|
8917
|
-
if (unwatch !== null) return;
|
|
8918
|
-
unwatch = await coordinator.watchIndexMetadata((event) => receive(event.indexName, event.metadata));
|
|
8919
|
-
for (const indexName of await coordinator.listSchemas()) {
|
|
8920
|
-
if (unwatch === null) return;
|
|
8921
|
-
await current(indexName).catch(onError);
|
|
8922
|
-
}
|
|
8923
|
-
},
|
|
8924
|
-
stop() {
|
|
8925
|
-
unwatch?.();
|
|
8926
|
-
unwatch = null;
|
|
8927
|
-
held.clear();
|
|
8928
|
-
},
|
|
8929
|
-
async applyWritten(indexName, written) {
|
|
8930
|
-
hold(indexName, written);
|
|
8931
|
-
await applyToLocalCopy(indexName, written.indexUuid);
|
|
8932
|
-
},
|
|
8933
|
-
applyToLocalCopy
|
|
8934
|
-
};
|
|
8935
|
-
}
|
|
8936
|
-
|
|
8937
|
-
// src/distribution/cluster-node/schema-diff.ts
|
|
8938
|
-
var SCHEMA_DIFF_MAX_DEPTH = MAX_NESTING_DEPTH;
|
|
8939
|
-
var DEPTH_EXCEEDED_SENTINEL = "depth-exceeded";
|
|
8940
|
-
function diffSchemas(expected, actual) {
|
|
8941
|
-
const diffs = [];
|
|
8942
|
-
walk(expected, actual, "", diffs, 1);
|
|
8943
|
-
return diffs;
|
|
8944
|
-
}
|
|
8945
|
-
function walk(expected, actual, prefix, diffs, depth) {
|
|
8946
|
-
if (depth > SCHEMA_DIFF_MAX_DEPTH) {
|
|
8947
|
-
diffs.push({
|
|
8948
|
-
path: prefix === "" ? DEPTH_EXCEEDED_SENTINEL : prefix,
|
|
8949
|
-
expected: DEPTH_EXCEEDED_SENTINEL,
|
|
8950
|
-
actual: DEPTH_EXCEEDED_SENTINEL
|
|
8951
|
-
});
|
|
8952
|
-
return;
|
|
8953
|
-
}
|
|
8954
|
-
const expectedKeys = Object.keys(expected).sort(compareCodePoints);
|
|
8955
|
-
const actualKeys = new Set(Object.keys(actual));
|
|
8956
|
-
for (const key of expectedKeys) {
|
|
8957
|
-
const path = prefix === "" ? key : `${prefix}.${key}`;
|
|
8958
|
-
const expectedValue = expected[key];
|
|
8959
|
-
const actualValue = actual[key];
|
|
8960
|
-
if (!actualKeys.has(key)) {
|
|
8961
|
-
diffs.push({ path, expected: describe(expectedValue), actual: "(absent)" });
|
|
8962
|
-
continue;
|
|
8963
|
-
}
|
|
8964
|
-
actualKeys.delete(key);
|
|
8965
|
-
const expectedIsObject = typeof expectedValue === "object" && expectedValue !== null;
|
|
8966
|
-
const actualIsObject = typeof actualValue === "object" && actualValue !== null;
|
|
8967
|
-
if (expectedIsObject !== actualIsObject) {
|
|
8968
|
-
diffs.push({ path, expected: describe(expectedValue), actual: describe(actualValue) });
|
|
8969
|
-
continue;
|
|
8970
|
-
}
|
|
8971
|
-
if (expectedIsObject && actualIsObject) {
|
|
8972
|
-
walk(expectedValue, actualValue, path, diffs, depth + 1);
|
|
8973
|
-
continue;
|
|
8974
|
-
}
|
|
8975
|
-
if (expectedValue !== actualValue) {
|
|
8976
|
-
diffs.push({ path, expected: String(expectedValue), actual: String(actualValue) });
|
|
8977
|
-
}
|
|
8978
|
-
}
|
|
8979
|
-
for (const extraKey of actualKeys) {
|
|
8980
|
-
const path = prefix === "" ? extraKey : `${prefix}.${extraKey}`;
|
|
8981
|
-
diffs.push({ path, expected: "(absent)", actual: describe(actual[extraKey]) });
|
|
8982
|
-
}
|
|
8983
|
-
}
|
|
8984
|
-
function describe(value) {
|
|
8985
|
-
if (value === null) {
|
|
8986
|
-
return "null";
|
|
8987
|
-
}
|
|
8988
|
-
if (typeof value === "object") {
|
|
8989
|
-
return "object";
|
|
8887
|
+
if (typeof value === "object") {
|
|
8888
|
+
return "object";
|
|
8990
8889
|
}
|
|
8991
8890
|
return String(value);
|
|
8992
8891
|
}
|
|
@@ -11033,6 +10932,27 @@ function scheduleStandbyRetry(state, retryMs, onRetry) {
|
|
|
11033
10932
|
onRetry();
|
|
11034
10933
|
}, retryMs);
|
|
11035
10934
|
}
|
|
10935
|
+
function readPingTimestamp(payload) {
|
|
10936
|
+
const decoded = decodeRequestPayload(payload, ClusterMessageTypes.PING);
|
|
10937
|
+
if (!isRecord(decoded) || !isValidInteger(decoded.timestamp) || decoded.timestamp < 0) {
|
|
10938
|
+
throw new NarsilError(
|
|
10939
|
+
ErrorCodes.CONFIG_INVALID,
|
|
10940
|
+
"Invalid PingPayload: it needs a timestamp that is a non-negative integer"
|
|
10941
|
+
);
|
|
10942
|
+
}
|
|
10943
|
+
return decoded.timestamp;
|
|
10944
|
+
}
|
|
10945
|
+
function pongReply(message) {
|
|
10946
|
+
try {
|
|
10947
|
+
const pong = { timestamp: readPingTimestamp(message.payload), respondedAt: Date.now() };
|
|
10948
|
+
return { type: ClusterMessageTypes.PONG, payload: encode(pong) };
|
|
10949
|
+
} catch (error) {
|
|
10950
|
+
return { type: `${message.type}.error`, payload: encodeErrorReplyFrom(error, ErrorCodes.CONFIG_INVALID) };
|
|
10951
|
+
}
|
|
10952
|
+
}
|
|
10953
|
+
function answerPing(message, respond, nodeId) {
|
|
10954
|
+
return respond({ ...pongReply(message), sourceId: nodeId, requestId: message.requestId });
|
|
10955
|
+
}
|
|
11036
10956
|
|
|
11037
10957
|
// src/distribution/cluster/allocator/deciders/balance.ts
|
|
11038
10958
|
var balanceDecider = {
|
|
@@ -12868,34 +12788,32 @@ async function sendRejection(respond, controllerNodeId, requestId, indexName, pa
|
|
|
12868
12788
|
};
|
|
12869
12789
|
await respond(response);
|
|
12870
12790
|
}
|
|
12871
|
-
async function
|
|
12872
|
-
let decoded;
|
|
12791
|
+
async function readBootstrapCompleteOrRefuse(message, respond, controllerNodeId) {
|
|
12873
12792
|
try {
|
|
12874
|
-
|
|
12875
|
-
|
|
12876
|
-
|
|
12877
|
-
|
|
12878
|
-
|
|
12879
|
-
|
|
12880
|
-
|
|
12881
|
-
await
|
|
12882
|
-
|
|
12793
|
+
const payload = validateBootstrapCompletePayload(decodeRequestPayload(message.payload, message.type));
|
|
12794
|
+
if (payload !== null) return payload;
|
|
12795
|
+
throw new NarsilError(
|
|
12796
|
+
ErrorCodes.CONFIG_INVALID,
|
|
12797
|
+
"Invalid BootstrapCompletePayload: it needs an indexName, a partitionId, a nodeId, and a primaryTerm"
|
|
12798
|
+
);
|
|
12799
|
+
} catch (error) {
|
|
12800
|
+
await respond({
|
|
12801
|
+
type: `${message.type}.error`,
|
|
12802
|
+
sourceId: controllerNodeId,
|
|
12803
|
+
requestId: message.requestId,
|
|
12804
|
+
payload: encodeRebalanceRefusal(error)
|
|
12805
|
+
});
|
|
12806
|
+
return null;
|
|
12883
12807
|
}
|
|
12808
|
+
}
|
|
12809
|
+
async function rejectBootstrapComplete(message, respond, controllerNodeId) {
|
|
12810
|
+
const payload = await readBootstrapCompleteOrRefuse(message, respond, controllerNodeId);
|
|
12811
|
+
if (payload === null) return;
|
|
12884
12812
|
await sendRejection(respond, controllerNodeId, message.requestId, payload.indexName, payload.partitionId);
|
|
12885
12813
|
}
|
|
12886
12814
|
async function handleBootstrapCompleteMessage(message, respond, coordinator, controllerNodeId) {
|
|
12887
|
-
|
|
12888
|
-
|
|
12889
|
-
decoded = decode(message.payload);
|
|
12890
|
-
} catch (_) {
|
|
12891
|
-
await sendRejection(respond, controllerNodeId, message.requestId, "", -1);
|
|
12892
|
-
return;
|
|
12893
|
-
}
|
|
12894
|
-
const payload = validateBootstrapCompletePayload(decoded);
|
|
12895
|
-
if (payload === null) {
|
|
12896
|
-
await sendRejection(respond, controllerNodeId, message.requestId, "", -1);
|
|
12897
|
-
return;
|
|
12898
|
-
}
|
|
12815
|
+
const payload = await readBootstrapCompleteOrRefuse(message, respond, controllerNodeId);
|
|
12816
|
+
if (payload === null) return;
|
|
12899
12817
|
if (message.sourceId !== payload.nodeId) {
|
|
12900
12818
|
await sendRejection(respond, controllerNodeId, message.requestId, payload.indexName, payload.partitionId);
|
|
12901
12819
|
return;
|
|
@@ -12959,7 +12877,6 @@ async function processBootstrapComplete(payload, coordinator, controllerNodeId)
|
|
|
12959
12877
|
}
|
|
12960
12878
|
|
|
12961
12879
|
// src/distribution/cluster/controller/event-loop/insync-messages.ts
|
|
12962
|
-
var UNKNOWN_PARTITION_ID = -1;
|
|
12963
12880
|
function validateInsyncRemovePayload(decoded) {
|
|
12964
12881
|
if (!isRecord(decoded)) {
|
|
12965
12882
|
return null;
|
|
@@ -13008,22 +12925,25 @@ function validateInsyncAddPayload(decoded) {
|
|
|
13008
12925
|
replicaStartedAt: decoded.replicaStartedAt
|
|
13009
12926
|
};
|
|
13010
12927
|
}
|
|
13011
|
-
function
|
|
12928
|
+
function readRequestOrRefuse(message, respond, nodeId, validate, payloadName) {
|
|
13012
12929
|
try {
|
|
13013
|
-
|
|
13014
|
-
|
|
12930
|
+
const payload = validate(decodeRequestPayload(message.payload, message.type));
|
|
12931
|
+
if (payload !== null) return payload;
|
|
12932
|
+
throw new NarsilError(
|
|
12933
|
+
ErrorCodes.CONFIG_INVALID,
|
|
12934
|
+
`Invalid ${payloadName}: a member is missing or holds a value of the wrong type`
|
|
12935
|
+
);
|
|
12936
|
+
} catch (error) {
|
|
12937
|
+
respond({
|
|
12938
|
+
type: `${message.type}.error`,
|
|
12939
|
+
sourceId: nodeId,
|
|
12940
|
+
requestId: message.requestId,
|
|
12941
|
+
payload: encodeRebalanceRefusal(error)
|
|
12942
|
+
}).catch(() => {
|
|
12943
|
+
});
|
|
13015
12944
|
return null;
|
|
13016
12945
|
}
|
|
13017
12946
|
}
|
|
13018
|
-
function describeRequest(decoded) {
|
|
13019
|
-
if (!isRecord(decoded)) {
|
|
13020
|
-
return { indexName: "", partitionId: UNKNOWN_PARTITION_ID };
|
|
13021
|
-
}
|
|
13022
|
-
return {
|
|
13023
|
-
indexName: typeof decoded.indexName === "string" ? decoded.indexName : "",
|
|
13024
|
-
partitionId: isValidInteger(decoded.partitionId) ? decoded.partitionId : UNKNOWN_PARTITION_ID
|
|
13025
|
-
};
|
|
13026
|
-
}
|
|
13027
12947
|
async function assertSenderLeadsPartition(coordinator, indexName, partitionId, sourceId) {
|
|
13028
12948
|
const table = await coordinator.getAllocation(indexName);
|
|
13029
12949
|
const assignment = table?.assignments.get(partitionId);
|
|
@@ -13055,12 +12975,8 @@ async function respondRefusal(respond, request, nodeId, requestId) {
|
|
|
13055
12975
|
}
|
|
13056
12976
|
}
|
|
13057
12977
|
function handleInsyncRemoveMessage(state, message, respond, coordinator, nodeId, isActive) {
|
|
13058
|
-
const
|
|
13059
|
-
|
|
13060
|
-
if (payload === null) {
|
|
13061
|
-
void respondRefusal(respond, describeRequest(decoded), nodeId, message.requestId);
|
|
13062
|
-
return;
|
|
13063
|
-
}
|
|
12978
|
+
const payload = readRequestOrRefuse(message, respond, nodeId, validateInsyncRemovePayload, "InsyncRemovePayload");
|
|
12979
|
+
if (payload === null) return;
|
|
13064
12980
|
enqueue(state, async () => {
|
|
13065
12981
|
const mayRemove = await assertSenderMayRemove(coordinator, payload, message.sourceId);
|
|
13066
12982
|
if (!mayRemove || !isActive()) {
|
|
@@ -13072,12 +12988,8 @@ function handleInsyncRemoveMessage(state, message, respond, coordinator, nodeId,
|
|
|
13072
12988
|
});
|
|
13073
12989
|
}
|
|
13074
12990
|
function handleInsyncAddMessage(state, message, respond, coordinator, nodeId, isActive, onError) {
|
|
13075
|
-
const
|
|
13076
|
-
|
|
13077
|
-
if (payload === null) {
|
|
13078
|
-
void respondRefusal(respond, describeRequest(decoded), nodeId, message.requestId);
|
|
13079
|
-
return;
|
|
13080
|
-
}
|
|
12991
|
+
const payload = readRequestOrRefuse(message, respond, nodeId, validateInsyncAddPayload, "InsyncAddPayload");
|
|
12992
|
+
if (payload === null) return;
|
|
13081
12993
|
enqueue(state, async () => {
|
|
13082
12994
|
const leadsPartition = await assertSenderLeadsPartition(
|
|
13083
12995
|
coordinator,
|
|
@@ -13146,12 +13058,8 @@ async function resign(payload, sourceId, coordinator, controllerNodeId) {
|
|
|
13146
13058
|
return false;
|
|
13147
13059
|
}
|
|
13148
13060
|
function handlePrimaryResignMessage(state, message, respond, coordinator, nodeId, isActive, onError) {
|
|
13149
|
-
const
|
|
13150
|
-
|
|
13151
|
-
if (payload === null) {
|
|
13152
|
-
void respondRefusal(respond, describeRequest(decoded), nodeId, message.requestId);
|
|
13153
|
-
return;
|
|
13154
|
-
}
|
|
13061
|
+
const payload = readRequestOrRefuse(message, respond, nodeId, validatePrimaryResignPayload, "PrimaryResignPayload");
|
|
13062
|
+
if (payload === null) return;
|
|
13155
13063
|
enqueue(state, async () => {
|
|
13156
13064
|
const accepted = isActive() && await resign(payload, message.sourceId, coordinator, nodeId);
|
|
13157
13065
|
const confirm = { indexName: payload.indexName, partitionId: payload.partitionId, accepted };
|
|
@@ -13576,12 +13484,7 @@ function startRebalanceDriver(ctx, indexName, jobId) {
|
|
|
13576
13484
|
|
|
13577
13485
|
// src/distribution/cluster/controller/event-loop/rebalance/request.ts
|
|
13578
13486
|
function readPayload2(message) {
|
|
13579
|
-
|
|
13580
|
-
try {
|
|
13581
|
-
decoded = decode(message.payload);
|
|
13582
|
-
} catch (_) {
|
|
13583
|
-
decoded = null;
|
|
13584
|
-
}
|
|
13487
|
+
const decoded = decodeRequestPayload(message.payload, message.type);
|
|
13585
13488
|
if (!isRecord(decoded) || typeof decoded.indexName !== "string" || decoded.indexName.length === 0) {
|
|
13586
13489
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "A partition count change must name an index", {});
|
|
13587
13490
|
}
|
|
@@ -13697,12 +13600,7 @@ function encodeRefusal(error) {
|
|
|
13697
13600
|
}
|
|
13698
13601
|
}
|
|
13699
13602
|
function readPayload3(message) {
|
|
13700
|
-
|
|
13701
|
-
try {
|
|
13702
|
-
decoded = decode(message.payload);
|
|
13703
|
-
} catch (_) {
|
|
13704
|
-
decoded = null;
|
|
13705
|
-
}
|
|
13603
|
+
const decoded = decodeRequestPayload(message.payload, message.type);
|
|
13706
13604
|
if (!isRecord(decoded) || typeof decoded.indexName !== "string") {
|
|
13707
13605
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "A replication factor request must name an index", {});
|
|
13708
13606
|
}
|
|
@@ -13780,6 +13678,11 @@ async function startEventLoop(state, coordinator, transport, nodeId, isActive, o
|
|
|
13780
13678
|
}
|
|
13781
13679
|
});
|
|
13782
13680
|
state.unwatchTransport = await transport.listen((message, respond) => {
|
|
13681
|
+
if (message.type === ClusterMessageTypes.PING) {
|
|
13682
|
+
answerPing(message, respond, nodeId).catch(() => {
|
|
13683
|
+
});
|
|
13684
|
+
return;
|
|
13685
|
+
}
|
|
13783
13686
|
if (message.type === ClusterMessageTypes.REPLICATION_FACTOR) {
|
|
13784
13687
|
handleReplicationFactorMessage(state, message, respond, coordinator, nodeId, isActive, onError);
|
|
13785
13688
|
return;
|
|
@@ -14629,7 +14532,7 @@ async function answerSyncRead(deps, request) {
|
|
|
14629
14532
|
return { frames: read.frames, head, highestSeqNo, oldestSeqNo, bytesBehind };
|
|
14630
14533
|
}
|
|
14631
14534
|
async function handleSyncRead(message, respond, deps) {
|
|
14632
|
-
const request = validateSyncReadRequest(
|
|
14535
|
+
const request = validateSyncReadRequest(decodeRequestPayload(message.payload, message.type));
|
|
14633
14536
|
if (request === null) {
|
|
14634
14537
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "A sync.read request is malformed", {
|
|
14635
14538
|
sourceNodeId: message.sourceId
|
|
@@ -16054,7 +15957,7 @@ async function openOnce(core, indexName, partitionId) {
|
|
|
16054
15957
|
if (directory === void 0) throw durabilityRequired(indexName, partitionId);
|
|
16055
15958
|
if ((checkpoint.manifest.analysisGeneration ?? 0) !== entry.analysisGeneration) return null;
|
|
16056
15959
|
const config = await encodeIndexConfiguration(entry);
|
|
16057
|
-
const { openPagedPartitionArchive } = await import('./paged-partition-archive-
|
|
15960
|
+
const { openPagedPartitionArchive } = await import('./paged-partition-archive-EFCEC4TD.mjs');
|
|
16058
15961
|
const archive = await openPagedPartitionArchive(checkpoint, directory, indexName, partitionId, manager);
|
|
16059
15962
|
handedOver = true;
|
|
16060
15963
|
try {
|
|
@@ -16769,7 +16672,7 @@ async function archiveReplicationPartition(core, indexName, partitionId, lastSeq
|
|
|
16769
16672
|
);
|
|
16770
16673
|
}
|
|
16771
16674
|
try {
|
|
16772
|
-
const { openPagedPartitionArchive } = await import('./paged-partition-archive-
|
|
16675
|
+
const { openPagedPartitionArchive } = await import('./paged-partition-archive-EFCEC4TD.mjs');
|
|
16773
16676
|
const archive = await openPagedPartitionArchive(checkpoint2, directory, indexName, partitionId, manager);
|
|
16774
16677
|
return {
|
|
16775
16678
|
stream: archiveStream({ ...archive, config, sourceIndexName: indexName }),
|
|
@@ -16984,7 +16887,7 @@ async function restoreReplicationPartition(core, engine, indexName, partitionId,
|
|
|
16984
16887
|
if (directory !== null) {
|
|
16985
16888
|
const durable = core.durability?.manager;
|
|
16986
16889
|
await (durable?.checkpointFromMemory?.(indexName) ?? durable?.checkpoint(indexName));
|
|
16987
|
-
const { removeReceivedFilesNothingReads } = await import('./paged-partition-archive-
|
|
16890
|
+
const { removeReceivedFilesNothingReads } = await import('./paged-partition-archive-EFCEC4TD.mjs');
|
|
16988
16891
|
await removeReceivedFilesNothingReads(
|
|
16989
16892
|
directory,
|
|
16990
16893
|
core.requireManager(indexName),
|
|
@@ -20028,7 +19931,7 @@ async function readPinPresence(deps, indexName, params, allocation) {
|
|
|
20028
19931
|
if (params.pinned === void 0 || params.searchAfter !== void 0) return null;
|
|
20029
19932
|
const pinnedIds = new Set(params.pinned.map((entry) => entry.docId));
|
|
20030
19933
|
const { verifiable, unverifiable } = splitPinnedByReachability(pinnedIds, allocation);
|
|
20031
|
-
const documents = verifiable.length === 0 ? /* @__PURE__ */ new Map() : await readDistributedDocuments(deps.config, deps.nodeId, deps.engine, indexName, verifiable, allocation);
|
|
19934
|
+
const documents = verifiable.length === 0 ? /* @__PURE__ */ new Map() : await readDistributedDocuments(deps.config, deps.nodeId, deps.engine, indexName, verifiable, allocation, false);
|
|
20032
19935
|
return { pinnedIds, stored: new Set(documents.keys()), unverifiable };
|
|
20033
19936
|
}
|
|
20034
19937
|
function dropUnstoredPinnedEntries(distributed, presence) {
|
|
@@ -20058,8 +19961,9 @@ function releaseContextsOfDepartedNodes(coordinator, registry) {
|
|
|
20058
19961
|
}
|
|
20059
19962
|
|
|
20060
19963
|
// src/distribution/cluster-node/search-contexts/documents.ts
|
|
20061
|
-
function documentOfSnapshot(snapshot) {
|
|
19964
|
+
function documentOfSnapshot(snapshot, withVectors = true) {
|
|
20062
19965
|
const document = structuredClone(snapshot.fields);
|
|
19966
|
+
if (!withVectors) return document;
|
|
20063
19967
|
for (const [fieldPath2, vector] of snapshot.vectors) setNestedValue(document, fieldPath2, new Float32Array(vector));
|
|
20064
19968
|
return document;
|
|
20065
19969
|
}
|
|
@@ -20176,7 +20080,7 @@ function createRankedDocumentReader(deps, indexName, allocation) {
|
|
|
20176
20080
|
resolveNodeTargets: deps.resolveNodeTargets
|
|
20177
20081
|
});
|
|
20178
20082
|
}
|
|
20179
|
-
function readOwnContext(context, docIds, last) {
|
|
20083
|
+
function readOwnContext(context, docIds, last, vectors) {
|
|
20180
20084
|
const registry = deps.searchContexts;
|
|
20181
20085
|
const found = registry?.read(context.contextId, docIds) ?? null;
|
|
20182
20086
|
if (last) {
|
|
@@ -20185,10 +20089,10 @@ function createRankedDocumentReader(deps, indexName, allocation) {
|
|
|
20185
20089
|
}
|
|
20186
20090
|
if (found === null) throw contextLost(context, indexName);
|
|
20187
20091
|
const documents = /* @__PURE__ */ new Map();
|
|
20188
|
-
for (const [docId, snapshot] of found) documents.set(docId, documentOfSnapshot(snapshot));
|
|
20092
|
+
for (const [docId, snapshot] of found) documents.set(docId, documentOfSnapshot(snapshot, vectors));
|
|
20189
20093
|
return documents;
|
|
20190
20094
|
}
|
|
20191
|
-
async function readRemoteContext(context, docIds, last, fields) {
|
|
20095
|
+
async function readRemoteContext(context, docIds, last, fields, vectors) {
|
|
20192
20096
|
const documents = /* @__PURE__ */ new Map();
|
|
20193
20097
|
for (let start = 0; start < docIds.length; start += FETCH_GROUP_DOCUMENT_IDS) {
|
|
20194
20098
|
const chunk = docIds.slice(start, start + FETCH_GROUP_DOCUMENT_IDS);
|
|
@@ -20198,6 +20102,7 @@ function createRankedDocumentReader(deps, indexName, allocation) {
|
|
|
20198
20102
|
indexName,
|
|
20199
20103
|
documentIds: chunk.map((docId) => ({ docId, partitionId: resolvePartitionId(docId, partitionCount) })),
|
|
20200
20104
|
fields,
|
|
20105
|
+
...vectors ? {} : { vectors: false },
|
|
20201
20106
|
highlight: null,
|
|
20202
20107
|
contextId: context.contextId,
|
|
20203
20108
|
releaseContext
|
|
@@ -20251,7 +20156,7 @@ function createRankedDocumentReader(deps, indexName, allocation) {
|
|
|
20251
20156
|
const reads = [];
|
|
20252
20157
|
for (const { context, docIds: contextIds } of byContext.values()) {
|
|
20253
20158
|
reads.push(
|
|
20254
|
-
context.nodeId === deps.nodeId && deps.searchContexts !== void 0 ? Promise.resolve().then(() => readOwnContext(context, contextIds, options.last)) : readRemoteContext(context, contextIds, options.last, options.fields)
|
|
20159
|
+
context.nodeId === deps.nodeId && deps.searchContexts !== void 0 ? Promise.resolve().then(() => readOwnContext(context, contextIds, options.last, options.vectors)) : readRemoteContext(context, contextIds, options.last, options.fields, options.vectors)
|
|
20255
20160
|
);
|
|
20256
20161
|
}
|
|
20257
20162
|
if (unsourced.length > 0) {
|
|
@@ -20263,7 +20168,7 @@ function createRankedDocumentReader(deps, indexName, allocation) {
|
|
|
20263
20168
|
indexName,
|
|
20264
20169
|
unsourced,
|
|
20265
20170
|
allocation,
|
|
20266
|
-
options.fields,
|
|
20171
|
+
{ fields: options.fields, vectors: options.vectors },
|
|
20267
20172
|
options.requireEveryPartition
|
|
20268
20173
|
)
|
|
20269
20174
|
);
|
|
@@ -20556,47 +20461,64 @@ function readSortValues(document, fields, modes) {
|
|
|
20556
20461
|
return fields.map((field, index) => reduceSortValue(readSortField(document, field), modes[index] ?? "min"));
|
|
20557
20462
|
}
|
|
20558
20463
|
|
|
20559
|
-
// src/distribution/cluster-node/reads/list.ts
|
|
20560
|
-
|
|
20561
|
-
|
|
20562
|
-
|
|
20563
|
-
return
|
|
20464
|
+
// src/distribution/cluster-node/reads/local-list-page.ts
|
|
20465
|
+
var VECTORS_CHOSEN_BY_THE_LIST = { schema: {} };
|
|
20466
|
+
function toWireSortValue(value) {
|
|
20467
|
+
if (typeof value === "string") return value;
|
|
20468
|
+
if (typeof value === "number") return Number.isFinite(value) ? value : null;
|
|
20469
|
+
if (typeof value === "boolean") return value;
|
|
20470
|
+
return null;
|
|
20564
20471
|
}
|
|
20565
|
-
function
|
|
20566
|
-
if (
|
|
20567
|
-
|
|
20568
|
-
|
|
20569
|
-
|
|
20570
|
-
return null;
|
|
20571
|
-
}
|
|
20572
|
-
const fields = new Set(projection.include.map((path) => path.join(".")));
|
|
20573
|
-
for (const sortField of sortFieldNames) {
|
|
20574
|
-
fields.add(sortField);
|
|
20575
|
-
}
|
|
20576
|
-
return Array.from(fields);
|
|
20577
|
-
}
|
|
20578
|
-
function toWireSortValue(value) {
|
|
20579
|
-
if (typeof value === "string") return value;
|
|
20580
|
-
if (typeof value === "number") return Number.isFinite(value) ? value : null;
|
|
20581
|
-
if (typeof value === "boolean") return value;
|
|
20582
|
-
return null;
|
|
20472
|
+
function fieldsToList(fields, sortFieldNames) {
|
|
20473
|
+
if (fields === null || fields.length === 0 || sortFieldNames === null) return fields;
|
|
20474
|
+
const listed = new Set(fields);
|
|
20475
|
+
for (const sortField of sortFieldNames) listed.add(sortField);
|
|
20476
|
+
return listed.size === fields.length ? fields : Array.from(listed);
|
|
20583
20477
|
}
|
|
20584
|
-
async function
|
|
20478
|
+
async function listLocalPage(engine, payload) {
|
|
20479
|
+
const normalizedSort = payload.sort === null ? null : normalizeSort(payload.sort);
|
|
20480
|
+
const sortFieldNames = normalizedSort === null ? null : normalizedSort.map((entry) => entry.field);
|
|
20481
|
+
const sortModes = normalizedSort === null ? [] : sortModesOf(normalizedSort);
|
|
20482
|
+
const listedFields = fieldsToList(payload.fields, sortFieldNames);
|
|
20483
|
+
const answerProjection = listedFields === payload.fields || payload.fields === null ? null : resolveProjection({ include: payload.fields }, VECTORS_CHOSEN_BY_THE_LIST);
|
|
20585
20484
|
const listParams = {
|
|
20586
20485
|
cursor: payload.cursor ?? void 0,
|
|
20587
20486
|
limit: payload.limit,
|
|
20588
20487
|
filters: payload.filters ?? void 0,
|
|
20589
20488
|
sort: payload.sort ?? void 0,
|
|
20590
|
-
document:
|
|
20489
|
+
document: { ...listedFields === null ? {} : { include: listedFields }, vectors: payload.vectors !== false }
|
|
20591
20490
|
};
|
|
20592
|
-
const result = await
|
|
20593
|
-
const entries = result.documents.map((listed) =>
|
|
20594
|
-
|
|
20595
|
-
|
|
20596
|
-
|
|
20597
|
-
|
|
20491
|
+
const result = await engine.listPartitions(payload.indexName, listParams, payload.partitionIds);
|
|
20492
|
+
const entries = result.documents.map((listed) => {
|
|
20493
|
+
const stored = listed.document;
|
|
20494
|
+
return {
|
|
20495
|
+
docId: listed.id,
|
|
20496
|
+
document: answerProjection === null || stored === void 0 ? stored : applyProjection(stored, answerProjection),
|
|
20497
|
+
sortValues: sortFieldNames === null ? null : readSortValues(stored, sortFieldNames, sortModes).map(toWireSortValue)
|
|
20498
|
+
};
|
|
20499
|
+
});
|
|
20598
20500
|
return { entries, total: result.total, hasMore: result.cursor !== null };
|
|
20599
20501
|
}
|
|
20502
|
+
|
|
20503
|
+
// src/distribution/cluster-node/reads/list.ts
|
|
20504
|
+
function clampListLimit(limit) {
|
|
20505
|
+
const clamped = Math.max(1, clampLimit(limit));
|
|
20506
|
+
requireWithinResultWindow(clamped, 0);
|
|
20507
|
+
return clamped;
|
|
20508
|
+
}
|
|
20509
|
+
function wireFieldsFor(projection, sortFieldNames) {
|
|
20510
|
+
if (projection.kind === "none") {
|
|
20511
|
+
return sortFieldNames;
|
|
20512
|
+
}
|
|
20513
|
+
if (projection.kind !== "fields" || projection.include === null) {
|
|
20514
|
+
return null;
|
|
20515
|
+
}
|
|
20516
|
+
const fields = new Set(projection.include.map((path) => path.join(".")));
|
|
20517
|
+
for (const sortField of sortFieldNames) {
|
|
20518
|
+
fields.add(sortField);
|
|
20519
|
+
}
|
|
20520
|
+
return Array.from(fields);
|
|
20521
|
+
}
|
|
20600
20522
|
async function listCluster(deps, indexName, params) {
|
|
20601
20523
|
const allocation = await routableAllocation(deps.config.coordinator, indexName);
|
|
20602
20524
|
if (allocation === null) {
|
|
@@ -20612,7 +20534,8 @@ async function listCluster(deps, indexName, params) {
|
|
|
20612
20534
|
const normalizedSort = normalizeSort(params.sort);
|
|
20613
20535
|
const sortFieldNames = normalizedSort.length === 0 ? null : normalizedSort.map((entry) => entry.field);
|
|
20614
20536
|
const directions = normalizedSort.map((entry) => entry.direction);
|
|
20615
|
-
const
|
|
20537
|
+
const settings = await deps.vectorReads.settingsOf(indexName, allocation);
|
|
20538
|
+
const projection = resolveProjection(params.document, settings);
|
|
20616
20539
|
const wireSort = sortFieldNames === null ? null : normalizedSort;
|
|
20617
20540
|
const payload = {
|
|
20618
20541
|
indexName,
|
|
@@ -20621,18 +20544,12 @@ async function listCluster(deps, indexName, params) {
|
|
|
20621
20544
|
limit,
|
|
20622
20545
|
filters: params.filters ?? null,
|
|
20623
20546
|
sort: wireSort,
|
|
20624
|
-
fields: wireFieldsFor(projection, sortFieldNames ?? [])
|
|
20547
|
+
fields: wireFieldsFor(projection, sortFieldNames ?? []),
|
|
20548
|
+
...keepsAnyVectorField(projection, settings) ? {} : { vectors: false }
|
|
20625
20549
|
};
|
|
20626
20550
|
const gathered = await gatherFromEveryPartition(allocation, indexName, (group) => {
|
|
20627
20551
|
if (group.nodeId === deps.nodeId) {
|
|
20628
|
-
return
|
|
20629
|
-
deps,
|
|
20630
|
-
indexName,
|
|
20631
|
-
{ ...payload, partitionIds: group.partitionIds },
|
|
20632
|
-
group.partitionIds,
|
|
20633
|
-
sortFieldNames,
|
|
20634
|
-
sortModesOf(normalizedSort)
|
|
20635
|
-
);
|
|
20552
|
+
return listLocalPage(deps.engine, { ...payload, partitionIds: group.partitionIds });
|
|
20636
20553
|
}
|
|
20637
20554
|
const message = createListMessage({ ...payload, partitionIds: group.partitionIds }, deps.nodeId);
|
|
20638
20555
|
return sendReadRequest(deps, group.nodeId, message, indexName, validateListResultPayload, true);
|
|
@@ -20786,12 +20703,27 @@ var clusterRerankHost = {
|
|
|
20786
20703
|
getNestedValue,
|
|
20787
20704
|
applyProjection
|
|
20788
20705
|
};
|
|
20789
|
-
async function readClusterDocuments(deps, indexName, docIds) {
|
|
20706
|
+
async function readClusterDocuments(deps, indexName, docIds, options) {
|
|
20790
20707
|
const allocation = await routableAllocation(deps.config.coordinator, indexName);
|
|
20791
20708
|
if (allocation === null) {
|
|
20792
|
-
return deps.engine.getMultiple(indexName, docIds);
|
|
20793
|
-
}
|
|
20794
|
-
|
|
20709
|
+
return deps.engine.getMultiple(indexName, docIds, options);
|
|
20710
|
+
}
|
|
20711
|
+
const settings = await deps.vectorReads.settingsOf(indexName, allocation);
|
|
20712
|
+
const projection = resolveProjection(options?.document, settings);
|
|
20713
|
+
const found = await readDocumentsFromCopies(
|
|
20714
|
+
deps.config,
|
|
20715
|
+
deps.nodeId,
|
|
20716
|
+
deps.engine,
|
|
20717
|
+
indexName,
|
|
20718
|
+
docIds,
|
|
20719
|
+
allocation,
|
|
20720
|
+
{ fields: fetchFieldsFor(projection), vectors: keepsAnyVectorField(projection, settings) },
|
|
20721
|
+
true
|
|
20722
|
+
);
|
|
20723
|
+
if (projection.kind === "full") return found;
|
|
20724
|
+
const documents = /* @__PURE__ */ new Map();
|
|
20725
|
+
for (const [docId, document] of found) documents.set(docId, applyProjection(document, projection));
|
|
20726
|
+
return documents;
|
|
20795
20727
|
}
|
|
20796
20728
|
function routingConfig(query, searchContexts) {
|
|
20797
20729
|
const config = { searchContexts };
|
|
@@ -20832,33 +20764,37 @@ async function queryCluster(deps, indexName, params, settingName) {
|
|
|
20832
20764
|
deps.analysisView,
|
|
20833
20765
|
indexName,
|
|
20834
20766
|
() => rerunOnLostContext(async () => {
|
|
20767
|
+
const settings = await deps.vectorReads.settingsOf(indexName, allocation);
|
|
20768
|
+
const search = { indexName, settingName, allocation, settings };
|
|
20835
20769
|
if (params.rerank === void 0) {
|
|
20836
20770
|
return withRankedReader(
|
|
20837
20771
|
deps,
|
|
20838
20772
|
indexName,
|
|
20839
20773
|
allocation,
|
|
20840
|
-
(reader) => searchCluster(deps,
|
|
20774
|
+
(reader) => searchCluster(deps, search, params, reader, false)
|
|
20841
20775
|
);
|
|
20842
20776
|
}
|
|
20843
|
-
const { rerankClusterQuery } = await import('./rerank-
|
|
20777
|
+
const { rerankClusterQuery } = await import('./rerank-JTIQNROU.mjs');
|
|
20844
20778
|
return withRankedReader(
|
|
20845
20779
|
deps,
|
|
20846
20780
|
indexName,
|
|
20847
20781
|
allocation,
|
|
20848
20782
|
(reader) => rerankClusterQuery(
|
|
20849
20783
|
deps,
|
|
20850
|
-
indexName,
|
|
20851
20784
|
params,
|
|
20852
|
-
|
|
20853
|
-
(
|
|
20785
|
+
settings,
|
|
20786
|
+
(firstStage) => searchCluster(deps, search, firstStage, reader, true),
|
|
20787
|
+
(docIds, read) => reader.read(docIds, { ...read, fields: null, requireEveryPartition: true }),
|
|
20854
20788
|
clusterRerankHost
|
|
20855
20789
|
)
|
|
20856
20790
|
);
|
|
20857
20791
|
})
|
|
20858
20792
|
);
|
|
20859
20793
|
}
|
|
20860
|
-
async function searchCluster(deps,
|
|
20861
|
-
const
|
|
20794
|
+
async function searchCluster(deps, search, params, reader, readsLater) {
|
|
20795
|
+
const { indexName, settingName, allocation, settings } = search;
|
|
20796
|
+
const projection = resolveProjection(params.document, settings);
|
|
20797
|
+
const vectors = keepsAnyVectorField(projection, settings);
|
|
20862
20798
|
const groupsRead = params.group !== void 0 && (params.group.reduce !== void 0 || projection.kind !== "none");
|
|
20863
20799
|
const readsDocuments = readsLater || projection.kind !== "none" || groupsRead;
|
|
20864
20800
|
const { wireParams, carriesBm25 } = heldBm25RoutingFor(deps, settingName, params) ?? await bm25RoutingFor(deps, settingName, params);
|
|
@@ -20884,7 +20820,7 @@ async function searchCluster(deps, indexName, settingName, params, allocation, r
|
|
|
20884
20820
|
if (projection.kind !== "none") {
|
|
20885
20821
|
const found = await reader.read(
|
|
20886
20822
|
distributed.scored.map((entry) => entry.docId),
|
|
20887
|
-
{ last: !groupsRead && !readsLater, fields: fetchFieldsFor(projection), requireEveryPartition: false }
|
|
20823
|
+
{ last: !groupsRead && !readsLater, fields: fetchFieldsFor(projection), vectors, requireEveryPartition: false }
|
|
20888
20824
|
);
|
|
20889
20825
|
for (const [docId, document] of found) documents.set(docId, applyProjection(document, projection));
|
|
20890
20826
|
}
|
|
@@ -20907,7 +20843,12 @@ async function searchCluster(deps, indexName, settingName, params, allocation, r
|
|
|
20907
20843
|
params,
|
|
20908
20844
|
distributed,
|
|
20909
20845
|
projection,
|
|
20910
|
-
(docIds, last) => reader.read(docIds, {
|
|
20846
|
+
(docIds, last) => reader.read(docIds, {
|
|
20847
|
+
last: last && !readsLater,
|
|
20848
|
+
fields: null,
|
|
20849
|
+
vectors: vectors || params.group?.reduce !== void 0,
|
|
20850
|
+
requireEveryPartition: true
|
|
20851
|
+
})
|
|
20911
20852
|
);
|
|
20912
20853
|
if (groups !== void 0) {
|
|
20913
20854
|
result.groups = groups;
|
|
@@ -21044,14 +20985,20 @@ function describeLocalCopy(deps, indexName) {
|
|
|
21044
20985
|
return { indexName, indexUuid, partitionIds };
|
|
21045
20986
|
}
|
|
21046
20987
|
async function handlePartitionStores(message, respond, deps) {
|
|
21047
|
-
const payload = validatePartitionStoresPayload(
|
|
21048
|
-
|
|
20988
|
+
const payload = validatePartitionStoresPayload(decodeRequestPayload(message.payload, message.type));
|
|
20989
|
+
if (payload === null) {
|
|
20990
|
+
throw new NarsilError(
|
|
20991
|
+
ErrorCodes.CONFIG_INVALID,
|
|
20992
|
+
"Invalid PartitionStoresPayload: it needs an indexName that is a non-empty string"
|
|
20993
|
+
);
|
|
20994
|
+
}
|
|
20995
|
+
const result = describeLocalCopy(deps, payload.indexName);
|
|
21049
20996
|
await respond(createPartitionStoresResultMessage(result, deps.nodeId, message.requestId));
|
|
21050
20997
|
}
|
|
21051
20998
|
|
|
21052
20999
|
// src/distribution/cluster-node/rebuild-analysis.ts
|
|
21053
21000
|
function validateRebuildPayload(decoded) {
|
|
21054
|
-
if (!isRecord(decoded) || typeof decoded.indexName !== "string" || decoded.indexName.length === 0 || typeof decoded.indexUuid !== "string" || decoded.indexUuid.length === 0 || decoded.generation !== void 0 && (!Number.isSafeInteger(decoded.generation) || Number(decoded.generation) < 1)) {
|
|
21001
|
+
if (!isRecord(decoded) || typeof decoded.indexName !== "string" || decoded.indexName.length === 0 || typeof decoded.indexUuid !== "string" || decoded.indexUuid.length === 0 || decoded.generation !== void 0 && decoded.generation !== null && (!Number.isSafeInteger(decoded.generation) || Number(decoded.generation) < 1)) {
|
|
21055
21002
|
throw new NarsilError(
|
|
21056
21003
|
ErrorCodes.CONFIG_INVALID,
|
|
21057
21004
|
"Invalid RebuildAnalysisPayload: it needs an indexName, an indexUuid, and a generation of 1 or more where it carries one"
|
|
@@ -21111,7 +21058,7 @@ async function rebuildLocalCopies(engine, payload) {
|
|
|
21111
21058
|
return { indexName, rebuiltPartitions: heldPartitionIdsOf(engine, indexName) };
|
|
21112
21059
|
}
|
|
21113
21060
|
async function handleRebuildAnalysis(message, respond, deps) {
|
|
21114
|
-
const payload = validateRebuildPayload(
|
|
21061
|
+
const payload = validateRebuildPayload(decodeRequestPayload(message.payload, message.type));
|
|
21115
21062
|
const result = payload.generation === void 0 ? await rebuildLocalCopies(deps.engine, payload) : await buildHeldCopies(deps.engine, deps.coordinator, payload, payload.generation);
|
|
21116
21063
|
await respond({
|
|
21117
21064
|
type: ClusterMessageTypes.REBUILD_ANALYSIS,
|
|
@@ -21207,6 +21154,7 @@ async function dropServedIndex(name, coordinator, engine) {
|
|
|
21207
21154
|
}
|
|
21208
21155
|
|
|
21209
21156
|
// src/distribution/cluster-node/node-operations.ts
|
|
21157
|
+
var DOCUMENT_PRESENCE = { document: false };
|
|
21210
21158
|
function createClusterNodeOperations(deps) {
|
|
21211
21159
|
const { trackOp, readDeps, writeDeps, engine, coordinator } = deps;
|
|
21212
21160
|
function served(indexName, fn) {
|
|
@@ -21347,14 +21295,17 @@ function createClusterNodeOperations(deps) {
|
|
|
21347
21295
|
)
|
|
21348
21296
|
);
|
|
21349
21297
|
},
|
|
21350
|
-
async get(indexName, docId) {
|
|
21351
|
-
return read(indexName, async (name) => (await readClusterDocuments(readDeps, name, [docId])).get(docId));
|
|
21298
|
+
async get(indexName, docId, options) {
|
|
21299
|
+
return read(indexName, async (name) => (await readClusterDocuments(readDeps, name, [docId], options)).get(docId));
|
|
21352
21300
|
},
|
|
21353
|
-
async getMultiple(indexName, docIds) {
|
|
21354
|
-
return read(indexName, (name) => readClusterDocuments(readDeps, name, docIds));
|
|
21301
|
+
async getMultiple(indexName, docIds, options) {
|
|
21302
|
+
return read(indexName, (name) => readClusterDocuments(readDeps, name, docIds, options));
|
|
21355
21303
|
},
|
|
21356
21304
|
async has(indexName, docId) {
|
|
21357
|
-
return read(
|
|
21305
|
+
return read(
|
|
21306
|
+
indexName,
|
|
21307
|
+
async (name) => (await readClusterDocuments(readDeps, name, [docId], DOCUMENT_PRESENCE)).has(docId)
|
|
21308
|
+
);
|
|
21358
21309
|
}
|
|
21359
21310
|
};
|
|
21360
21311
|
}
|
|
@@ -21717,7 +21668,7 @@ function createNodeWiring(input) {
|
|
|
21717
21668
|
])
|
|
21718
21669
|
)
|
|
21719
21670
|
};
|
|
21720
|
-
return readDistributedDocuments(routingConfig2, nodeId, engine, indexName, docIds, primariesOnly);
|
|
21671
|
+
return readDistributedDocuments(routingConfig2, nodeId, engine, indexName, docIds, primariesOnly, true);
|
|
21721
21672
|
}
|
|
21722
21673
|
const registration = {
|
|
21723
21674
|
nodeId,
|
|
@@ -22166,7 +22117,7 @@ function readSnapshotPage(deps, job, cursor, onFirstPage) {
|
|
|
22166
22117
|
[job.partitionId]
|
|
22167
22118
|
);
|
|
22168
22119
|
const ids = listed.documents.map((document) => document.id);
|
|
22169
|
-
const stored = await engine.getMultiple(job.sourceGeneration, ids);
|
|
22120
|
+
const stored = await engine.getMultiple(job.sourceGeneration, ids, { document: { vectors: true } });
|
|
22170
22121
|
const operations = [];
|
|
22171
22122
|
for (const id of ids) {
|
|
22172
22123
|
const document = stored.get(id);
|
|
@@ -22471,12 +22422,7 @@ function invalid2(what) {
|
|
|
22471
22422
|
return new NarsilError(ErrorCodes.CONFIG_INVALID, `Invalid partition copy message: ${what}`);
|
|
22472
22423
|
}
|
|
22473
22424
|
function decodedRecord(message) {
|
|
22474
|
-
|
|
22475
|
-
try {
|
|
22476
|
-
decoded = decode(message.payload);
|
|
22477
|
-
} catch (_) {
|
|
22478
|
-
decoded = null;
|
|
22479
|
-
}
|
|
22425
|
+
const decoded = decodeRequestPayload(message.payload, message.type);
|
|
22480
22426
|
if (!isRecord(decoded)) throw invalid2("expected an object");
|
|
22481
22427
|
return decoded;
|
|
22482
22428
|
}
|
|
@@ -22575,14 +22521,20 @@ async function handleReshardPurge(message, respond, deps) {
|
|
|
22575
22521
|
|
|
22576
22522
|
// src/distribution/cluster-node/message-handler/index-settings.ts
|
|
22577
22523
|
async function handleIndexSettings(message, respond, deps) {
|
|
22578
|
-
const payload = validateIndexSettingsPayload(
|
|
22579
|
-
|
|
22580
|
-
|
|
22581
|
-
|
|
22524
|
+
const payload = validateIndexSettingsPayload(decodeRequestPayload(message.payload, message.type));
|
|
22525
|
+
if (payload === null) {
|
|
22526
|
+
throw new NarsilError(
|
|
22527
|
+
ErrorCodes.CONFIG_INVALID,
|
|
22528
|
+
"Invalid IndexSettingsPayload: it needs an indexName and an indexUuid that are non-empty strings"
|
|
22529
|
+
);
|
|
22530
|
+
}
|
|
22531
|
+
const applied = await applyStoredSynonyms(deps.engine, deps.coordinator, payload.indexName, payload.indexUuid);
|
|
22532
|
+
await respond(
|
|
22533
|
+
createIndexSettingsResultMessage({ indexName: payload.indexName, applied }, deps.nodeId, message.requestId)
|
|
22534
|
+
);
|
|
22582
22535
|
}
|
|
22583
22536
|
async function handleSearch(message, respond, deps) {
|
|
22584
|
-
const
|
|
22585
|
-
const payload = validateSearchPayload(decoded);
|
|
22537
|
+
const payload = validateSearchPayload(decodeRequestPayload(message.payload, message.type));
|
|
22586
22538
|
const queryParams = wireParamsToLocal(payload.params, payload.facetShardSize);
|
|
22587
22539
|
if (queryParams.group?.limit !== void 0) {
|
|
22588
22540
|
queryParams.group = { ...queryParams.group, limit: oversampledShardSize(queryParams.group.limit) };
|
|
@@ -22661,12 +22613,12 @@ async function convertLocalGroupsToWire(groups, sortFields, sortModes, deps, ind
|
|
|
22661
22613
|
}
|
|
22662
22614
|
return wireGroups;
|
|
22663
22615
|
}
|
|
22616
|
+
var VECTORS_CHOSEN_BY_THE_FETCH = { schema: {} };
|
|
22664
22617
|
async function handleFetch(message, respond, deps) {
|
|
22665
|
-
const
|
|
22666
|
-
const
|
|
22667
|
-
const projection = resolveProjection(payload.fields === null ? void 0 : { include: payload.fields });
|
|
22618
|
+
const payload = validateFetchPayload(decodeRequestPayload(message.payload, message.type));
|
|
22619
|
+
const read = { fields: payload.fields, vectors: payload.vectors !== false };
|
|
22668
22620
|
const contextId = payload.contextId ?? null;
|
|
22669
|
-
const documents = contextId === null ? await readCurrentDocuments(deps, payload.indexName, payload.documentIds,
|
|
22621
|
+
const documents = contextId === null ? await readCurrentDocuments(deps, payload.indexName, payload.documentIds, read) : readContextDocuments(deps, contextId, payload.documentIds, payload.releaseContext !== false, read);
|
|
22670
22622
|
refuseEncodeWithoutRoom(documents.map((fetched) => fetched.document));
|
|
22671
22623
|
const resultPayload = { documents };
|
|
22672
22624
|
await respond({
|
|
@@ -22676,21 +22628,24 @@ async function handleFetch(message, respond, deps) {
|
|
|
22676
22628
|
payload: encodeDocumentReply(resultPayload)
|
|
22677
22629
|
});
|
|
22678
22630
|
}
|
|
22679
|
-
async function readCurrentDocuments(deps, indexName, documentIds,
|
|
22631
|
+
async function readCurrentDocuments(deps, indexName, documentIds, read) {
|
|
22680
22632
|
const documents = [];
|
|
22633
|
+
const options = {
|
|
22634
|
+
document: { ...read.fields === null ? {} : { include: read.fields }, vectors: read.vectors }
|
|
22635
|
+
};
|
|
22681
22636
|
for (const docRef of documentIds) {
|
|
22682
|
-
const doc = await deps.engine.get(indexName, docRef.docId);
|
|
22637
|
+
const doc = await deps.engine.get(indexName, docRef.docId, options);
|
|
22683
22638
|
if (doc !== void 0) {
|
|
22684
|
-
documents.push({
|
|
22685
|
-
docId: docRef.docId,
|
|
22686
|
-
document: applyProjection(doc, projection),
|
|
22687
|
-
highlights: null
|
|
22688
|
-
});
|
|
22639
|
+
documents.push({ docId: docRef.docId, document: doc, highlights: null });
|
|
22689
22640
|
}
|
|
22690
22641
|
}
|
|
22691
22642
|
return documents;
|
|
22692
22643
|
}
|
|
22693
|
-
function readContextDocuments(deps, contextId, documentIds, releaseContext,
|
|
22644
|
+
function readContextDocuments(deps, contextId, documentIds, releaseContext, read) {
|
|
22645
|
+
const projection = resolveProjection(
|
|
22646
|
+
read.fields === null ? void 0 : { include: read.fields },
|
|
22647
|
+
VECTORS_CHOSEN_BY_THE_FETCH
|
|
22648
|
+
);
|
|
22694
22649
|
const docIds = documentIds.map((docRef) => docRef.docId);
|
|
22695
22650
|
const found = deps.searchContexts?.read(contextId, docIds) ?? null;
|
|
22696
22651
|
if (releaseContext) deps.searchContexts?.release(contextId);
|
|
@@ -22708,7 +22663,7 @@ function readContextDocuments(deps, contextId, documentIds, releaseContext, proj
|
|
|
22708
22663
|
if (snapshot === void 0) continue;
|
|
22709
22664
|
documents.push({
|
|
22710
22665
|
docId,
|
|
22711
|
-
document: applyProjection(documentOfSnapshot(snapshot), projection),
|
|
22666
|
+
document: applyProjection(documentOfSnapshot(snapshot, read.vectors), projection),
|
|
22712
22667
|
highlights: null
|
|
22713
22668
|
});
|
|
22714
22669
|
}
|
|
@@ -22721,8 +22676,7 @@ function openSearchContext(deps, coordinatorNodeId, keepAliveMs, snapshots) {
|
|
|
22721
22676
|
return deps.searchContexts.open(coordinatorNodeId, keepAliveOf(keepAliveMs), snapshots);
|
|
22722
22677
|
}
|
|
22723
22678
|
async function handleStats(message, respond, deps) {
|
|
22724
|
-
const
|
|
22725
|
-
const payload = validateStatsPayload(decoded);
|
|
22679
|
+
const payload = validateStatsPayload(decodeRequestPayload(message.payload, message.type));
|
|
22726
22680
|
const resultPayload = await deps.analysisView.answerAt(
|
|
22727
22681
|
payload.indexName,
|
|
22728
22682
|
payload.analysisGeneration ?? 0,
|
|
@@ -22767,39 +22721,18 @@ function convertLocalFacetsToWire2(facets) {
|
|
|
22767
22721
|
|
|
22768
22722
|
// src/distribution/cluster-node/message-handler/read-handlers.ts
|
|
22769
22723
|
async function handleCount(message, respond, deps) {
|
|
22770
|
-
const payload = validateCountPayload(
|
|
22724
|
+
const payload = validateCountPayload(decodeRequestPayload(message.payload, message.type));
|
|
22771
22725
|
const answer2 = await localCountAnswer(deps.engine, payload.indexName, payload.partitionIds);
|
|
22772
22726
|
await respond(createCountResultMessage(answer2, deps.nodeId, message.requestId));
|
|
22773
22727
|
}
|
|
22774
22728
|
async function handleList(message, respond, deps) {
|
|
22775
|
-
const payload = validateListPayload(
|
|
22776
|
-
const
|
|
22777
|
-
|
|
22778
|
-
|
|
22779
|
-
filters: payload.filters ?? void 0,
|
|
22780
|
-
sort: payload.sort ?? void 0,
|
|
22781
|
-
document: payload.fields === null ? void 0 : { include: payload.fields }
|
|
22782
|
-
};
|
|
22783
|
-
const result = await deps.engine.listPartitions(payload.indexName, listParams, payload.partitionIds);
|
|
22784
|
-
const normalizedSort = payload.sort === null ? null : normalizeSort(payload.sort);
|
|
22785
|
-
const sortFields = normalizedSort === null ? null : normalizedSort.map((entry) => entry.field);
|
|
22786
|
-
const sortModes = normalizedSort === null ? [] : sortModesOf(normalizedSort);
|
|
22787
|
-
const entries = result.documents.map((listed) => ({
|
|
22788
|
-
docId: listed.id,
|
|
22789
|
-
document: listed.document,
|
|
22790
|
-
sortValues: sortFields === null ? null : readSortValues(listed.document, sortFields, sortModes).map(toWireSortValue3)
|
|
22791
|
-
}));
|
|
22792
|
-
refuseEncodeWithoutRoom(entries.map((entry) => entry.document));
|
|
22793
|
-
await respond(
|
|
22794
|
-
createListResultMessage(
|
|
22795
|
-
{ entries, total: result.total, hasMore: result.cursor !== null },
|
|
22796
|
-
deps.nodeId,
|
|
22797
|
-
message.requestId
|
|
22798
|
-
)
|
|
22799
|
-
);
|
|
22729
|
+
const payload = validateListPayload(decodeRequestPayload(message.payload, message.type));
|
|
22730
|
+
const page = await listLocalPage(deps.engine, payload);
|
|
22731
|
+
refuseEncodeWithoutRoom(page.entries.map((entry) => entry.document));
|
|
22732
|
+
await respond(createListResultMessage(page, deps.nodeId, message.requestId));
|
|
22800
22733
|
}
|
|
22801
22734
|
async function handleSuggest(message, respond, deps) {
|
|
22802
|
-
const payload = validateSuggestPayload(
|
|
22735
|
+
const payload = validateSuggestPayload(decodeRequestPayload(message.payload, message.type));
|
|
22803
22736
|
const result = await deps.analysisView.answerAt(
|
|
22804
22737
|
payload.indexName,
|
|
22805
22738
|
payload.analysisGeneration ?? 0,
|
|
@@ -22818,7 +22751,7 @@ async function handleSuggest(message, respond, deps) {
|
|
|
22818
22751
|
);
|
|
22819
22752
|
}
|
|
22820
22753
|
async function handlePreflight(message, respond, deps) {
|
|
22821
|
-
const payload = validatePreflightPayload(
|
|
22754
|
+
const payload = validatePreflightPayload(decodeRequestPayload(message.payload, message.type));
|
|
22822
22755
|
const params = wireParamsToLocal(payload.params);
|
|
22823
22756
|
const result = await deps.analysisView.answerAt(
|
|
22824
22757
|
payload.indexName,
|
|
@@ -22838,12 +22771,6 @@ async function handlePreflight(message, respond, deps) {
|
|
|
22838
22771
|
)
|
|
22839
22772
|
);
|
|
22840
22773
|
}
|
|
22841
|
-
function toWireSortValue3(value) {
|
|
22842
|
-
if (typeof value === "string") return value;
|
|
22843
|
-
if (typeof value === "number") return Number.isFinite(value) ? value : null;
|
|
22844
|
-
if (typeof value === "boolean") return value;
|
|
22845
|
-
return null;
|
|
22846
|
-
}
|
|
22847
22774
|
|
|
22848
22775
|
// src/distribution/replication/sync-primary.ts
|
|
22849
22776
|
function decideSyncTier(log, replicaLastSeqNo, replicaLastPrimaryTerm) {
|
|
@@ -23390,17 +23317,26 @@ function currentSyncSessionOf(request, sourceNodeId, assignment, deps) {
|
|
|
23390
23317
|
);
|
|
23391
23318
|
}
|
|
23392
23319
|
async function sendSyncEntriesResponse(message, respond, deps, entries, syncSession) {
|
|
23393
|
-
const
|
|
23394
|
-
|
|
23395
|
-
|
|
23396
|
-
syncSession
|
|
23397
|
-
};
|
|
23398
|
-
await respond({
|
|
23399
|
-
type: ReplicationMessageTypes.SYNC_ENTRIES,
|
|
23400
|
-
sourceId: deps.nodeId,
|
|
23401
|
-
requestId: message.requestId,
|
|
23402
|
-
payload: encode(payload)
|
|
23320
|
+
const batches = chunkByBudget(entries, {
|
|
23321
|
+
...WIRE_BATCH_BUDGET,
|
|
23322
|
+
payloadBytesOf: (entry) => (entry.document?.byteLength ?? 0) + (entry.previousDocument?.byteLength ?? 0)
|
|
23403
23323
|
});
|
|
23324
|
+
if (batches.length === 0) {
|
|
23325
|
+
batches.push([]);
|
|
23326
|
+
}
|
|
23327
|
+
for (let index = 0; index < batches.length; index++) {
|
|
23328
|
+
const payload = {
|
|
23329
|
+
entries: batches[index],
|
|
23330
|
+
isLast: index === batches.length - 1,
|
|
23331
|
+
syncSession
|
|
23332
|
+
};
|
|
23333
|
+
await respond({
|
|
23334
|
+
type: ReplicationMessageTypes.SYNC_ENTRIES,
|
|
23335
|
+
sourceId: deps.nodeId,
|
|
23336
|
+
requestId: message.requestId,
|
|
23337
|
+
payload: encode(payload)
|
|
23338
|
+
});
|
|
23339
|
+
}
|
|
23404
23340
|
}
|
|
23405
23341
|
|
|
23406
23342
|
// src/distribution/cluster-node/message-handler/entry-authority.ts
|
|
@@ -23467,8 +23403,7 @@ async function resolveValidatedAssignment(entry, sourceNodeId, deps) {
|
|
|
23467
23403
|
return live;
|
|
23468
23404
|
}
|
|
23469
23405
|
async function handleForward(message, respond, deps) {
|
|
23470
|
-
const
|
|
23471
|
-
const payload = validateForwardPayload(decoded);
|
|
23406
|
+
const payload = validateForwardPayload(decodeRequestPayload(message.payload, message.type));
|
|
23472
23407
|
const heldIndexUuid = deps.engine.indexUuidOf(payload.indexName);
|
|
23473
23408
|
if (typeof heldIndexUuid === "string" && holdsAnotherIndex(heldIndexUuid, payload.indexUuid)) {
|
|
23474
23409
|
throw anotherIndexAsPrimary(payload.indexName, deps.nodeId, heldIndexUuid);
|
|
@@ -23482,7 +23417,7 @@ async function handleForward(message, respond, deps) {
|
|
|
23482
23417
|
});
|
|
23483
23418
|
}
|
|
23484
23419
|
async function handleForwardBatch(message, respond, deps) {
|
|
23485
|
-
const payload = validateForwardBatchPayload(
|
|
23420
|
+
const payload = validateForwardBatchPayload(decodeRequestPayload(message.payload, message.type));
|
|
23486
23421
|
const heldIndexUuid = deps.engine.indexUuidOf(payload.indexName);
|
|
23487
23422
|
if (typeof heldIndexUuid === "string" && holdsAnotherIndex(heldIndexUuid, payload.indexUuid)) {
|
|
23488
23423
|
const refusal2 = anotherIndexAsPrimary(payload.indexName, deps.nodeId, heldIndexUuid);
|
|
@@ -23500,14 +23435,14 @@ async function handleForwardBatch(message, respond, deps) {
|
|
|
23500
23435
|
await respond(createForwardBatchResultMessage({ results }, deps.nodeId, message.requestId));
|
|
23501
23436
|
}
|
|
23502
23437
|
async function handleReplicationEntry(message, respond, deps) {
|
|
23503
|
-
const payload = validateEntryPayload(
|
|
23438
|
+
const payload = validateEntryPayload(decodeRequestPayload(message.payload, message.type));
|
|
23504
23439
|
const { entry } = payload;
|
|
23505
23440
|
refuseAnotherIndex(entry.indexName, payload.indexUuid, deps);
|
|
23506
23441
|
await applyEntriesFromPrimary([entry], message.sourceId, deps, payload.previousPrimaryTerm);
|
|
23507
23442
|
await respond(createAckMessage(entry.seqNo, entry.partitionId, entry.indexName, deps.nodeId, message.requestId));
|
|
23508
23443
|
}
|
|
23509
23444
|
async function handleReplicationEntryBatch(message, respond, deps) {
|
|
23510
|
-
const payload = validateEntryBatchPayload(
|
|
23445
|
+
const payload = validateEntryBatchPayload(decodeRequestPayload(message.payload, message.type));
|
|
23511
23446
|
const lastEntry = payload.entries[payload.entries.length - 1];
|
|
23512
23447
|
refuseAnotherIndex(lastEntry.indexName, payload.indexUuid, deps);
|
|
23513
23448
|
await applyEntriesFromPrimary(payload.entries, message.sourceId, deps, payload.previousPrimaryTerm);
|
|
@@ -23648,26 +23583,14 @@ async function applyRun2(run, log, deps) {
|
|
|
23648
23583
|
if (applied < run.length) throw error;
|
|
23649
23584
|
}
|
|
23650
23585
|
function validateForwardPayload(decoded) {
|
|
23651
|
-
if (
|
|
23652
|
-
throw
|
|
23653
|
-
}
|
|
23654
|
-
const record = decoded;
|
|
23655
|
-
if (typeof record.indexName !== "string" || record.indexName.length === 0) {
|
|
23656
|
-
throw new NarsilError(ErrorCodes.CONFIG_INVALID, 'Invalid ForwardPayload: "indexName" must be a non-empty string');
|
|
23657
|
-
}
|
|
23658
|
-
if (typeof record.documentId !== "string" || record.documentId.length === 0) {
|
|
23659
|
-
throw new NarsilError(ErrorCodes.CONFIG_INVALID, 'Invalid ForwardPayload: "documentId" must be a non-empty string');
|
|
23660
|
-
}
|
|
23661
|
-
if (record.operation !== "insert" && record.operation !== "remove" && record.operation !== "update") {
|
|
23662
|
-
throw new NarsilError(
|
|
23663
|
-
ErrorCodes.CONFIG_INVALID,
|
|
23664
|
-
'Invalid ForwardPayload: "operation" must be "insert", "remove", or "update"'
|
|
23665
|
-
);
|
|
23586
|
+
if (!isRecord(decoded)) {
|
|
23587
|
+
throw invalidReplicationPayload("Invalid ForwardPayload: expected an object");
|
|
23666
23588
|
}
|
|
23667
|
-
|
|
23668
|
-
|
|
23669
|
-
throw new NarsilError(ErrorCodes.CONFIG_INVALID, 'Invalid ForwardPayload: "indexUuid" must be a non-empty string');
|
|
23589
|
+
if (typeof decoded.indexName !== "string" || decoded.indexName.length === 0) {
|
|
23590
|
+
throw invalidReplicationPayload('Invalid ForwardPayload: "indexName" must be a non-empty string');
|
|
23670
23591
|
}
|
|
23592
|
+
validateForwardOperation(decoded, "ForwardPayload");
|
|
23593
|
+
optionalIndexUuid(decoded.indexUuid, "ForwardPayload");
|
|
23671
23594
|
return decoded;
|
|
23672
23595
|
}
|
|
23673
23596
|
|
|
@@ -23770,6 +23693,9 @@ function createDataNodeHandler(deps) {
|
|
|
23770
23693
|
case SuggestionMessageTypes.COUNTS:
|
|
23771
23694
|
await deps.answerQueryCounts?.(message, respond);
|
|
23772
23695
|
return;
|
|
23696
|
+
case ClusterMessageTypes.PING:
|
|
23697
|
+
await answerPing(message, respond, deps.nodeId);
|
|
23698
|
+
return;
|
|
23773
23699
|
default:
|
|
23774
23700
|
return;
|
|
23775
23701
|
}
|
|
@@ -23807,7 +23733,7 @@ function refuseMessage(nodeId, message, respond, code, text2) {
|
|
|
23807
23733
|
});
|
|
23808
23734
|
}
|
|
23809
23735
|
function createNotReadyHandler(nodeId) {
|
|
23810
|
-
return (message, respond) => refuseMessage(
|
|
23736
|
+
return (message, respond) => message.type === ClusterMessageTypes.PING ? answerPing(message, respond, nodeId) : refuseMessage(
|
|
23811
23737
|
nodeId,
|
|
23812
23738
|
message,
|
|
23813
23739
|
respond,
|
|
@@ -24239,12 +24165,7 @@ function createClusterHost(deps) {
|
|
|
24239
24165
|
};
|
|
24240
24166
|
}
|
|
24241
24167
|
function indexNameOf2(message) {
|
|
24242
|
-
|
|
24243
|
-
try {
|
|
24244
|
-
decoded = decode(message.payload);
|
|
24245
|
-
} catch {
|
|
24246
|
-
decoded = null;
|
|
24247
|
-
}
|
|
24168
|
+
const decoded = decodeRequestPayload(message.payload, message.type);
|
|
24248
24169
|
if (!isRecord(decoded) || typeof decoded.indexName !== "string") {
|
|
24249
24170
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, `A ${message.type} request needs "indexName" as a string`);
|
|
24250
24171
|
}
|
|
@@ -24615,6 +24536,249 @@ function createClusterQuerySuggestions(deps) {
|
|
|
24615
24536
|
};
|
|
24616
24537
|
}
|
|
24617
24538
|
|
|
24539
|
+
// src/distribution/cluster-node/analysis-change/view.ts
|
|
24540
|
+
function namedSettingsOfMetadata(settings) {
|
|
24541
|
+
return {
|
|
24542
|
+
language: settings?.language ?? "english",
|
|
24543
|
+
...settings?.tokenizer !== void 0 ? { tokenizer: settings.tokenizer } : {},
|
|
24544
|
+
...settings?.stopWords !== void 0 ? { stopWords: settings.stopWords } : {},
|
|
24545
|
+
...settings?.stopWordList !== void 0 ? { stopWordList: settings.stopWordList } : {}
|
|
24546
|
+
};
|
|
24547
|
+
}
|
|
24548
|
+
function generationMissing(indexName, requestedGeneration, heldGeneration) {
|
|
24549
|
+
return new NarsilError(
|
|
24550
|
+
ErrorCodes.ANALYSIS_GENERATION_MISSING,
|
|
24551
|
+
`This node holds analysis generation ${heldGeneration} of index '${indexName}', so it cannot answer for generation ${requestedGeneration}`,
|
|
24552
|
+
{ indexName, requestedGeneration, heldGeneration }
|
|
24553
|
+
);
|
|
24554
|
+
}
|
|
24555
|
+
function createAnalysisView(deps) {
|
|
24556
|
+
const { coordinator, engine, onError } = deps;
|
|
24557
|
+
const held = /* @__PURE__ */ new Map();
|
|
24558
|
+
const settling = /* @__PURE__ */ new Map();
|
|
24559
|
+
let unwatch = null;
|
|
24560
|
+
let stopped = false;
|
|
24561
|
+
function recordGeneration(indexName, snapshot) {
|
|
24562
|
+
const generation = snapshot.settings?.analysisGeneration ?? 0;
|
|
24563
|
+
const { indexUuid } = snapshot.record;
|
|
24564
|
+
const known = held.get(indexName);
|
|
24565
|
+
if (known?.indexUuid === indexUuid && known.generation >= generation) return known.generation;
|
|
24566
|
+
held.set(indexName, { indexUuid, generation });
|
|
24567
|
+
return generation;
|
|
24568
|
+
}
|
|
24569
|
+
async function refresh(indexName) {
|
|
24570
|
+
const snapshot = await readIndexMetadataSnapshot(coordinator, indexName);
|
|
24571
|
+
if (snapshot === null) {
|
|
24572
|
+
held.delete(indexName);
|
|
24573
|
+
return 0;
|
|
24574
|
+
}
|
|
24575
|
+
return recordGeneration(indexName, snapshot);
|
|
24576
|
+
}
|
|
24577
|
+
function catchUp(indexName, generation, settings) {
|
|
24578
|
+
engine.buildHeldAnalysis(indexName, { generation, ...namedSettingsOfMetadata(settings) }).then(() => engine.releaseHeldAnalysis(indexName, generation)).catch((error) => {
|
|
24579
|
+
if (!stopped) onError(error);
|
|
24580
|
+
});
|
|
24581
|
+
}
|
|
24582
|
+
async function settleOnce(indexName) {
|
|
24583
|
+
const generation = engine.analysisGenerationOf(indexName);
|
|
24584
|
+
if (stopped || generation === void 0) return;
|
|
24585
|
+
const snapshot = await readIndexMetadataSnapshot(coordinator, indexName);
|
|
24586
|
+
const localUuid = engine.indexUuidOf(indexName);
|
|
24587
|
+
if (snapshot === null || typeof localUuid === "string" && localUuid !== snapshot.record.indexUuid) return;
|
|
24588
|
+
recordGeneration(indexName, snapshot);
|
|
24589
|
+
const { settings } = snapshot;
|
|
24590
|
+
const published = settings?.analysisGeneration ?? 0;
|
|
24591
|
+
const localPending = engine.pendingAnalysisGenerationOf(indexName);
|
|
24592
|
+
if (published > generation) {
|
|
24593
|
+
if (localPending !== published && engine.holdsAnalysisSettings(indexName, namedSettingsOfMetadata(settings))) {
|
|
24594
|
+
if (localPending !== null) await engine.dropHeldAnalysis(indexName);
|
|
24595
|
+
await engine.stampAnalysisGeneration(indexName, published);
|
|
24596
|
+
return;
|
|
24597
|
+
}
|
|
24598
|
+
catchUp(indexName, published, settings);
|
|
24599
|
+
return;
|
|
24600
|
+
}
|
|
24601
|
+
if (localPending !== null && localPending > published && settings?.pendingAnalysis?.generation !== localPending) {
|
|
24602
|
+
await engine.dropHeldAnalysis(indexName);
|
|
24603
|
+
}
|
|
24604
|
+
}
|
|
24605
|
+
function settle(indexName) {
|
|
24606
|
+
const next = (settling.get(indexName) ?? Promise.resolve()).then(() => settleOnce(indexName)).catch(onError);
|
|
24607
|
+
settling.set(indexName, next);
|
|
24608
|
+
return next;
|
|
24609
|
+
}
|
|
24610
|
+
async function reach(indexName, requested) {
|
|
24611
|
+
if (engine.heldAnalysisIsBuilt(indexName, requested)) {
|
|
24612
|
+
await engine.releaseHeldAnalysis(indexName, requested).catch(onError);
|
|
24613
|
+
return;
|
|
24614
|
+
}
|
|
24615
|
+
await settle(indexName);
|
|
24616
|
+
}
|
|
24617
|
+
async function reachThenRead(indexName, requested, generation) {
|
|
24618
|
+
if (generation < requested) await reach(indexName, requested);
|
|
24619
|
+
const reached = engine.analysisGenerationOf(indexName) ?? generation;
|
|
24620
|
+
if (reached !== requested) throw generationMissing(indexName, requested, reached);
|
|
24621
|
+
}
|
|
24622
|
+
return {
|
|
24623
|
+
held(indexName) {
|
|
24624
|
+
return held.get(indexName)?.generation;
|
|
24625
|
+
},
|
|
24626
|
+
async generationFor(indexName) {
|
|
24627
|
+
return held.get(indexName)?.generation ?? refresh(indexName);
|
|
24628
|
+
},
|
|
24629
|
+
refresh,
|
|
24630
|
+
answerAt(indexName, requested, read) {
|
|
24631
|
+
const generation = engine.analysisGenerationOf(indexName);
|
|
24632
|
+
if (generation === void 0) return read();
|
|
24633
|
+
const readAtRequested = (answerOf3) => answerOf3().then((answer2) => {
|
|
24634
|
+
const after = engine.analysisGenerationOf(indexName) ?? requested;
|
|
24635
|
+
if (after !== requested) throw generationMissing(indexName, requested, after);
|
|
24636
|
+
return answer2;
|
|
24637
|
+
});
|
|
24638
|
+
if (generation === requested) return readAtRequested(read);
|
|
24639
|
+
return reachThenRead(indexName, requested, generation).then(() => readAtRequested(read));
|
|
24640
|
+
},
|
|
24641
|
+
settle,
|
|
24642
|
+
async start() {
|
|
24643
|
+
if (unwatch !== null) return;
|
|
24644
|
+
stopped = false;
|
|
24645
|
+
unwatch = await coordinator.watchIndexMetadata((event) => {
|
|
24646
|
+
if (event.metadata === null) {
|
|
24647
|
+
held.delete(event.indexName);
|
|
24648
|
+
return;
|
|
24649
|
+
}
|
|
24650
|
+
try {
|
|
24651
|
+
recordGeneration(event.indexName, indexMetadataSnapshotOf(event.metadata, event.indexName));
|
|
24652
|
+
} catch (error) {
|
|
24653
|
+
onError(error);
|
|
24654
|
+
}
|
|
24655
|
+
void settle(event.indexName);
|
|
24656
|
+
});
|
|
24657
|
+
for (const indexName of await coordinator.listSchemas()) {
|
|
24658
|
+
if (stopped) return;
|
|
24659
|
+
if (!held.has(indexName)) await refresh(indexName).catch(onError);
|
|
24660
|
+
}
|
|
24661
|
+
},
|
|
24662
|
+
stop() {
|
|
24663
|
+
stopped = true;
|
|
24664
|
+
unwatch?.();
|
|
24665
|
+
unwatch = null;
|
|
24666
|
+
held.clear();
|
|
24667
|
+
}
|
|
24668
|
+
};
|
|
24669
|
+
}
|
|
24670
|
+
|
|
24671
|
+
// src/distribution/cluster-node/bm25-view.ts
|
|
24672
|
+
function settingOfMetadata(bytes, indexName) {
|
|
24673
|
+
const decoded = decode(bytes);
|
|
24674
|
+
if (!isRecord2(decoded) || typeof decoded.indexUuid !== "string") return null;
|
|
24675
|
+
const settings = decodeIndexSettings(decoded.settings, indexName);
|
|
24676
|
+
return { indexUuid: decoded.indexUuid, bm25: settings?.bm25, revision: settings?.bm25Revision };
|
|
24677
|
+
}
|
|
24678
|
+
function isLaterThan(next, held) {
|
|
24679
|
+
return (next.revision ?? 0) > (held.revision ?? 0);
|
|
24680
|
+
}
|
|
24681
|
+
function createBm25View(deps) {
|
|
24682
|
+
const { coordinator, engine, onError } = deps;
|
|
24683
|
+
const held = /* @__PURE__ */ new Map();
|
|
24684
|
+
let unwatch = null;
|
|
24685
|
+
function applyToLocalCopy(indexName, indexUuid) {
|
|
24686
|
+
if (engine.indexUuidOf(indexName) !== indexUuid) return Promise.resolve();
|
|
24687
|
+
return engine.applyStoredBm25(indexName, () => {
|
|
24688
|
+
const latest = held.get(indexName);
|
|
24689
|
+
if (latest === void 0 || latest.indexUuid !== indexUuid) return null;
|
|
24690
|
+
return engine.indexUuidOf(indexName) === indexUuid ? { bm25: latest.bm25 } : null;
|
|
24691
|
+
}).then(
|
|
24692
|
+
() => void 0,
|
|
24693
|
+
(error) => {
|
|
24694
|
+
if (error instanceof NarsilError && error.code === ErrorCodes.INDEX_NOT_FOUND) return;
|
|
24695
|
+
onError(error);
|
|
24696
|
+
}
|
|
24697
|
+
);
|
|
24698
|
+
}
|
|
24699
|
+
function hold(indexName, next) {
|
|
24700
|
+
const current2 = held.get(indexName);
|
|
24701
|
+
if (current2 !== void 0 && current2.indexUuid === next.indexUuid && !isLaterThan(next, current2)) return false;
|
|
24702
|
+
held.set(indexName, next);
|
|
24703
|
+
return true;
|
|
24704
|
+
}
|
|
24705
|
+
function receive(indexName, metadata) {
|
|
24706
|
+
if (metadata === null) {
|
|
24707
|
+
held.delete(indexName);
|
|
24708
|
+
return;
|
|
24709
|
+
}
|
|
24710
|
+
let next;
|
|
24711
|
+
try {
|
|
24712
|
+
next = settingOfMetadata(metadata, indexName);
|
|
24713
|
+
} catch (error) {
|
|
24714
|
+
onError(error);
|
|
24715
|
+
return;
|
|
24716
|
+
}
|
|
24717
|
+
if (next !== null && hold(indexName, next)) void applyToLocalCopy(indexName, next.indexUuid);
|
|
24718
|
+
}
|
|
24719
|
+
async function current(indexName) {
|
|
24720
|
+
const known = held.get(indexName);
|
|
24721
|
+
if (known !== void 0) return known;
|
|
24722
|
+
const snapshot = await readIndexMetadataSnapshot(coordinator, indexName);
|
|
24723
|
+
if (snapshot === null) return null;
|
|
24724
|
+
const read = {
|
|
24725
|
+
indexUuid: snapshot.record.indexUuid,
|
|
24726
|
+
bm25: snapshot.settings?.bm25,
|
|
24727
|
+
revision: snapshot.settings?.bm25Revision
|
|
24728
|
+
};
|
|
24729
|
+
const racing = held.get(indexName);
|
|
24730
|
+
if (racing === void 0 || racing.indexUuid === read.indexUuid && isLaterThan(read, racing)) {
|
|
24731
|
+
held.set(indexName, read);
|
|
24732
|
+
return read;
|
|
24733
|
+
}
|
|
24734
|
+
return racing;
|
|
24735
|
+
}
|
|
24736
|
+
return {
|
|
24737
|
+
held: (indexName) => held.get(indexName),
|
|
24738
|
+
current,
|
|
24739
|
+
async start() {
|
|
24740
|
+
if (unwatch !== null) return;
|
|
24741
|
+
unwatch = await coordinator.watchIndexMetadata((event) => receive(event.indexName, event.metadata));
|
|
24742
|
+
for (const indexName of await coordinator.listSchemas()) {
|
|
24743
|
+
if (unwatch === null) return;
|
|
24744
|
+
await current(indexName).catch(onError);
|
|
24745
|
+
}
|
|
24746
|
+
},
|
|
24747
|
+
stop() {
|
|
24748
|
+
unwatch?.();
|
|
24749
|
+
unwatch = null;
|
|
24750
|
+
held.clear();
|
|
24751
|
+
},
|
|
24752
|
+
async applyWritten(indexName, written) {
|
|
24753
|
+
hold(indexName, written);
|
|
24754
|
+
await applyToLocalCopy(indexName, written.indexUuid);
|
|
24755
|
+
},
|
|
24756
|
+
applyToLocalCopy
|
|
24757
|
+
};
|
|
24758
|
+
}
|
|
24759
|
+
|
|
24760
|
+
// src/distribution/cluster-node/read-views.ts
|
|
24761
|
+
function createReadViews(base, coordinator, engine, onError) {
|
|
24762
|
+
const bm25View = createBm25View({ coordinator, engine, onError });
|
|
24763
|
+
const analysisView = createAnalysisView({ coordinator, engine, onError });
|
|
24764
|
+
const vectorReads = createVectorReadView({ coordinator, onError });
|
|
24765
|
+
return {
|
|
24766
|
+
bm25View,
|
|
24767
|
+
analysisView,
|
|
24768
|
+
readDeps: { ...base, bm25View, analysisView, vectorReads },
|
|
24769
|
+
async start() {
|
|
24770
|
+
await bm25View.start();
|
|
24771
|
+
await analysisView.start();
|
|
24772
|
+
await vectorReads.start();
|
|
24773
|
+
},
|
|
24774
|
+
stop() {
|
|
24775
|
+
bm25View.stop();
|
|
24776
|
+
analysisView.stop();
|
|
24777
|
+
vectorReads.stop();
|
|
24778
|
+
}
|
|
24779
|
+
};
|
|
24780
|
+
}
|
|
24781
|
+
|
|
24618
24782
|
// src/distribution/cluster-node/reshard/request.ts
|
|
24619
24783
|
async function waitForSwap(config, indexName, { jobId, targetGeneration }, guard) {
|
|
24620
24784
|
for (; ; ) {
|
|
@@ -24754,11 +24918,11 @@ function clusterNodeEngine(node, options) {
|
|
|
24754
24918
|
async query(indexName, params) {
|
|
24755
24919
|
return node.query(indexName, params);
|
|
24756
24920
|
},
|
|
24757
|
-
async get(indexName, docId) {
|
|
24758
|
-
return node.get(indexName, docId);
|
|
24921
|
+
async get(indexName, docId, options2) {
|
|
24922
|
+
return node.get(indexName, docId, options2);
|
|
24759
24923
|
},
|
|
24760
|
-
async getMultiple(indexName, docIds) {
|
|
24761
|
-
return node.getMultiple(indexName, docIds);
|
|
24924
|
+
async getMultiple(indexName, docIds, options2) {
|
|
24925
|
+
return node.getMultiple(indexName, docIds, options2);
|
|
24762
24926
|
},
|
|
24763
24927
|
async has(indexName, docId) {
|
|
24764
24928
|
return node.has(indexName, docId);
|
|
@@ -25066,9 +25230,9 @@ async function createClusterNode(callerConfig) {
|
|
|
25066
25230
|
extraMessages: (message, respond) => backups.controllerMessages(message, respond) || suggestions.controllerMessages(message, respond)
|
|
25067
25231
|
});
|
|
25068
25232
|
}
|
|
25069
|
-
const
|
|
25070
|
-
const
|
|
25071
|
-
const
|
|
25233
|
+
const readBase = { config: routingConfig2, nodeId, engine, resolveNodeTargets: resolveNodeTargets3, searchContexts };
|
|
25234
|
+
const readViews = createReadViews(readBase, config.coordinator, engine, forwardOnError);
|
|
25235
|
+
const { analysisView, readDeps } = readViews;
|
|
25072
25236
|
const servingName = (indexName) => servingNameOf(allocationView.routing, indexName);
|
|
25073
25237
|
const deviceSync = createClusterDeviceSync({
|
|
25074
25238
|
...cleanupDeps,
|
|
@@ -25132,8 +25296,7 @@ async function createClusterNode(callerConfig) {
|
|
|
25132
25296
|
async start() {
|
|
25133
25297
|
guardShutdown();
|
|
25134
25298
|
await allocationView.start();
|
|
25135
|
-
await
|
|
25136
|
-
await analysisView.start();
|
|
25299
|
+
await readViews.start();
|
|
25137
25300
|
await deviceSync.start();
|
|
25138
25301
|
const reconcileDeps = { engine, coordinator: config.coordinator, nodeId, onError: forwardOnError };
|
|
25139
25302
|
await settleLocalIndexes(reconcileDeps, orphanedIndexes, analysisView.settle);
|
|
@@ -25188,8 +25351,7 @@ async function createClusterNode(callerConfig) {
|
|
|
25188
25351
|
await operationTracker.drain();
|
|
25189
25352
|
await suggestions.stop();
|
|
25190
25353
|
allocationView.stop();
|
|
25191
|
-
|
|
25192
|
-
analysisView.stop();
|
|
25354
|
+
readViews.stop();
|
|
25193
25355
|
await deviceSync.stop();
|
|
25194
25356
|
unregisterHandler?.();
|
|
25195
25357
|
unregisterHandler = null;
|