@delali/narsil 0.4.6-next.332 → 0.4.6-next.334
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/{chunk-VAPCFSMX.mjs → chunk-3PGIOK7Z.mjs} +16 -1
- package/dist/browser/{chunk-VIVG3OXM.mjs → chunk-FBDTHGDR.mjs} +7 -3
- package/dist/browser/{chunk-56UNHLAF.mjs → chunk-HIC42RBU.mjs} +1 -1
- package/dist/browser/{chunk-DJDBJB2U.mjs → chunk-J343PP7Q.mjs} +85 -28
- package/dist/browser/{optional-reads-NQO5DKM4.mjs → optional-reads-4O4CJ7TR.mjs} +1 -1
- package/dist/browser/{optional-writes-3HKIVZFU.mjs → optional-writes-OFPF77WY.mjs} +2 -1
- package/dist/browser/{runtime-OMKE7DF4.mjs → runtime-AMEB4J2D.mjs} +2 -1
- package/dist/browser/{segment-codec-leaf-GXSRX5VT.mjs → segment-codec-leaf-E4YHXUAI.mjs} +7 -2
- package/dist/browser/{storage-leaf-72PEG6IP.mjs → storage-leaf-WJMD66YX.mjs} +15 -2
- package/dist/{capabilities-DaJKuKlG.d.ts → capabilities-YazdElfW.d.ts} +2 -2
- package/dist/{chunk-UAITBI74.mjs → chunk-2QQCR7KF.mjs} +15 -9
- package/dist/{chunk-NNRDQYBD.mjs → chunk-4QNWK42S.mjs} +104 -41
- package/dist/{chunk-G6P6YTTW.mjs → chunk-5PUNOJC7.mjs} +1 -1
- package/dist/chunk-6S5QPDPP.mjs +110 -0
- package/dist/{chunk-EY5U4P5V.mjs → chunk-7NMLFP2T.mjs} +17 -2
- package/dist/{react-native/chunks/chunk-NUOJ2EAK.mjs → chunk-AKLCYPUY.mjs} +1 -1
- package/dist/{chunk-WRDGPEN6.mjs → chunk-B276ZQIH.mjs} +2 -2
- package/dist/{chunk-HWIIMLLR.mjs → chunk-B6VO3BCX.mjs} +6 -6
- package/dist/{chunk-3GWUUKDJ.mjs → chunk-BVFJ5XAU.mjs} +2 -2
- package/dist/{chunk-WCGFMFWK.mjs → chunk-CHGJGKST.mjs} +2 -2
- package/dist/{chunk-AQI4DIFA.mjs → chunk-CVCBVUH4.mjs} +2 -2
- package/dist/{chunk-CN42TL6V.mjs → chunk-HB3EBTKC.mjs} +5 -5
- package/dist/{react-native/chunks/chunk-ALSPAMEE.mjs → chunk-HNE4HNC2.mjs} +2 -2
- package/dist/{chunk-RBHEMWN6.mjs → chunk-I255IQTR.mjs} +3 -3
- package/dist/{chunk-VIHHCXBO.mjs → chunk-IALJ2X74.mjs} +3 -3
- package/dist/{chunk-EDZJNZKV.mjs → chunk-ICEZRKIS.mjs} +1 -1
- package/dist/{chunk-JXRMHWQU.mjs → chunk-IVVNKTHC.mjs} +1 -1
- package/dist/{chunk-MUBQER6D.mjs → chunk-KHZ6LAFB.mjs} +1 -1
- package/dist/{chunk-5SHWA6PE.mjs → chunk-L2GYY6KY.mjs} +1 -1
- package/dist/{chunk-4JLHHEBN.mjs → chunk-M5RYB6FE.mjs} +1 -1
- package/dist/{chunk-ONDHZ4WW.mjs → chunk-N3DJTTKK.mjs} +2 -2
- package/dist/{chunk-JASE2FE6.mjs → chunk-QWYIMVYT.mjs} +2 -2
- package/dist/{chunk-GTB2S2VX.mjs → chunk-R5OFJZNI.mjs} +2 -2
- package/dist/{chunk-7DW2ITZ3.mjs → chunk-RHFUQ76T.mjs} +1 -1
- package/dist/{chunk-FWZ5OK5G.mjs → chunk-SCUYG24W.mjs} +4 -4
- package/dist/{chunk-DVFCDOXW.mjs → chunk-TNBOFWDS.mjs} +1 -1
- package/dist/{chunk-PM2UOKYB.mjs → chunk-UFDD57KD.mjs} +37 -9
- package/dist/{chunk-M2Z5WVWW.mjs → chunk-UGMOBSMD.mjs} +2 -0
- package/dist/{chunk-CZHAPGVH.mjs → chunk-UJQ2S5QU.mjs} +80 -78
- package/dist/{chunk-43T3P3BE.mjs → chunk-WW3RDS57.mjs} +1 -1
- package/dist/{chunk-BOTZ2GSW.mjs → chunk-Z622PLNH.mjs} +25 -14
- package/dist/{client-DetSCqP2.d.ts → client-B9YNYYs_.d.ts} +38 -12
- package/dist/client.d.ts +8 -8
- package/dist/client.mjs +2 -2
- package/dist/{config-BB4MRAsB.d.ts → config-DJFXyHMZ.d.ts} +2 -2
- package/dist/{config-XEHd4MFx.d.ts → config-DlvP2PSi.d.ts} +2 -2
- package/dist/{direct-executor-3C5WFR33.mjs → direct-executor-P2Z7T57F.mjs} +3 -3
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution/transport/tcp.mjs +1 -1
- package/dist/distribution.d.ts +14 -9
- package/dist/distribution.mjs +391 -315
- package/dist/{engine-DGG_yX0p.d.ts → engine-DP4PrdKM.d.ts} +51 -13
- package/dist/{index-fX_FbuRT.d.ts → index-CcivYH_R.d.ts} +1 -1
- package/dist/index.browser.mjs +3 -3
- package/dist/index.d.ts +6 -6
- package/dist/index.mjs +3 -3
- package/dist/index.react-native.mjs +3 -3
- package/dist/{long-query-thread-C62WK6NI.mjs → long-query-thread-SOJ25FZU.mjs} +13 -13
- package/dist/{msgpack-leaf-LIT7TKZW.mjs → msgpack-leaf-LTAJ5QGB.mjs} +2 -2
- package/dist/{optional-reads-ESDVS225.mjs → optional-reads-5QWIOFOU.mjs} +1 -1
- package/dist/{optional-writes-AGDEVXC4.mjs → optional-writes-GVHTEENI.mjs} +5 -5
- package/dist/{paged-partition-archive-EABR3N3F.mjs → paged-partition-archive-XCETVLZW.mjs} +13 -13
- package/dist/partial-words.browser.mjs +3 -3
- package/dist/partial-words.mjs +3 -3
- package/dist/pattern.browser.mjs +2 -2
- package/dist/pattern.mjs +2 -2
- package/dist/persistence/durability/checkpoint-worker.d.ts +2 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +10 -10
- package/dist/{query-actions-QH5UQRCY.mjs → query-actions-WKC4SQEZ.mjs} +3 -3
- package/dist/query-suggestions/merge-worker.mjs +6 -6
- package/dist/query-suggestions.browser.mjs +3 -3
- package/dist/query-suggestions.d.ts +2 -2
- package/dist/query-suggestions.mjs +4 -4
- package/dist/react-native/chunks/{chunk-PTCHRZCP.mjs → chunk-4JY2CZWV.mjs} +3 -3
- package/dist/react-native/chunks/{chunk-ESVBC3AT.mjs → chunk-BIORJHQZ.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-AOLLV37A.mjs → chunk-CBG4EUYD.mjs} +17 -2
- package/dist/react-native/chunks/{chunk-GPMPHDDO.mjs → chunk-FN3HBVNY.mjs} +2 -2
- package/dist/{chunk-AWMO3NKJ.mjs → react-native/chunks/chunk-LR6OK725.mjs} +1 -1
- package/dist/{chunk-57MN2UAL.mjs → react-native/chunks/chunk-MPNHORPY.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-V2H7EDTM.mjs → chunk-QT2G6OMI.mjs} +103 -40
- package/dist/react-native/chunks/{chunk-BGRX5SPG.mjs → chunk-QZUFYRFX.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-ZF3VSGEI.mjs → chunk-ZSJABTIE.mjs} +1 -1
- package/dist/react-native/chunks/{msgpack-leaf-FOUPTT7J.mjs → msgpack-leaf-Q4BPA4G7.mjs} +2 -2
- package/dist/react-native/chunks/{optional-reads-TN3PT6M4.mjs → optional-reads-M6F26KFO.mjs} +1 -1
- package/dist/react-native/chunks/{optional-writes-F4FJKEZ4.mjs → optional-writes-TCVHZRRM.mjs} +4 -4
- package/dist/react-native/chunks/{restore-queue-MLFBOOZP.mjs → restore-queue-RLQ3L223.mjs} +6 -6
- package/dist/{runtime-USQYKN3I.mjs → react-native/chunks/runtime-7UKADH3G.mjs} +2 -1
- package/dist/react-native/chunks/{segment-codec-leaf-Z3OS3B2K.mjs → segment-codec-leaf-IM6YCP7M.mjs} +4 -4
- package/dist/react-native/chunks/{segment-merge-leaf-TNHU77VW.mjs → segment-merge-leaf-32YR2ZTM.mjs} +5 -5
- package/dist/react-native/chunks/{storage-leaf-NF733K7B.mjs → storage-leaf-RKI6S6T5.mjs} +23 -10
- package/dist/react-native/chunks/{store-leaf-UDKCNQB7.mjs → store-leaf-YQXG6NOD.mjs} +5 -5
- package/dist/react-native/chunks/{suggestions-leaf-M6ITOPNM.mjs → suggestions-leaf-Q3ABULFB.mjs} +5 -5
- package/dist/react-native/chunks/{worker-copies-GG7CKUBI.mjs → worker-copies-W63KWZZN.mjs} +4 -4
- package/dist/react-native/partial-words.mjs +3 -3
- package/dist/react-native/pattern.mjs +2 -2
- package/dist/react-native/query-suggestions.mjs +3 -3
- package/dist/react-native/sync.mjs +12 -7
- package/dist/react-native/synonyms.mjs +3 -3
- package/dist/react.d.ts +9 -9
- package/dist/{rerank-LCMA5PZG.mjs → rerank-OJLSVUYV.mjs} +25 -12
- package/dist/{restore-queue-NSA3RWF6.mjs → restore-queue-BYO4AJLZ.mjs} +6 -6
- package/dist/{react-native/chunks/runtime-UOHD2XUJ.mjs → runtime-WWFVQOBJ.mjs} +2 -1
- package/dist/{schema-CFnQ-WI3.d.ts → schema-CPQUdJTs.d.ts} +6 -0
- package/dist/{segment-codec-leaf-DQKKIN4F.mjs → segment-codec-leaf-V52PQEVL.mjs} +4 -4
- package/dist/{segment-merge-leaf-67Z6OMRF.mjs → segment-merge-leaf-M3ZLJYDD.mjs} +4 -4
- package/dist/server.d.ts +22 -11
- package/dist/server.mjs +23 -21
- package/dist/{storage-leaf-FUH75D3X.mjs → storage-leaf-YYO4FLHQ.mjs} +22 -22
- package/dist/{store-leaf-CKZFFFTX.mjs → store-leaf-KK7VD3P3.mjs} +5 -5
- package/dist/{suggestions-leaf-NOSJIRSD.mjs → suggestions-leaf-EJCNU3NM.mjs} +7 -7
- package/dist/sync/filesystem.d.ts +3 -3
- package/dist/sync/filesystem.mjs +8 -8
- package/dist/sync/indexeddb.browser.mjs +1 -1
- package/dist/sync/indexeddb.d.ts +3 -3
- package/dist/sync/indexeddb.mjs +2 -2
- package/dist/sync.browser.mjs +7 -6
- package/dist/sync.d.ts +4 -4
- package/dist/sync.mjs +12 -7
- package/dist/synonyms.browser.mjs +3 -3
- package/dist/synonyms.mjs +3 -3
- package/dist/{tasks-Bpi5YNZF.d.ts → tasks-4rvZJuoN.d.ts} +8 -4
- package/dist/{thread-T5F3HSMC.mjs → thread-6BVFFMP7.mjs} +12 -12
- package/dist/{types-DGjk684l.d.ts → types-C3hkFf_3.d.ts} +2 -2
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/{worker-copies-TPEHVXPI.mjs → worker-copies-5V5JGCWU.mjs} +6 -6
- package/dist/workers/entry.mjs +9 -9
- package/openapi.json +58 -2
- package/package.json +5 -5
package/dist/distribution.mjs
CHANGED
|
@@ -1,45 +1,46 @@
|
|
|
1
|
-
import { routableAllocation, resolvePartitionId, resolveNodeTargets, resolvePrimaryAssignment, requireAssignedPrimary, assertSufficientActiveReplicas, getInSyncReplicaTargets } from './chunk-
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
1
|
+
import { routableAllocation, resolvePartitionId, resolveNodeTargets, resolvePrimaryAssignment, requireAssignedPrimary, assertSufficientActiveReplicas, getInSyncReplicaTargets } from './chunk-N3DJTTKK.mjs';
|
|
2
|
+
import { createVectorReadView, keepsAnyVectorField } from './chunk-6S5QPDPP.mjs';
|
|
3
|
+
import { emptySerializablePartition } from './chunk-RHFUQ76T.mjs';
|
|
4
|
+
import { applyStoredIndexBm25 } from './chunk-TNBOFWDS.mjs';
|
|
5
|
+
import { keyBytesOf, querySuggestionsOf, createQuerySuggestionsRuntime, publicBuildOf, buildRecordToWire, buildRecordFromWire, SUGGESTIONS_SCHEMA } from './chunk-SCUYG24W.mjs';
|
|
6
|
+
import { stampAnalysisGeneration, holdsNamedSettings, dropHeldAnalysisChange, releaseHeldAnalysisChange, heldAnalysisIsBuilt, buildHeldAnalysisChange, archiveStream, buildArchiveFromMemory, indexMetadataOf, applyIndexEntry, applyDeleteEntry, documentExportStream, followSourceChanges, requireChangesSetting, resolveAnalysisTarget, requireConfigurationWithinSizeLimit, memoryCheckpointStore, createByteReader, writeArchiveFile, namedSettingsOf, createDetachedIndex, loadPartitionSegments, changesStatusOf, holdsSettings, configOfNamedSettings, closePartitionWriter, requireRestoreMemory, archiveFootprint, storedBytesOfSegments, validateReplicationEntry, readSegmentManifest, replaySyncedWal, nestedSchemaOf } from './chunk-Z622PLNH.mjs';
|
|
7
|
+
import { readVectorFile } from './chunk-IALJ2X74.mjs';
|
|
7
8
|
import { addBackgroundAdmission, workWaitingForMemory, resolveHybridFusion, fuseRankings } from './chunk-GXED6R3Q.mjs';
|
|
8
9
|
import { UNADDRESSABLE_TARGET_DETAIL, MAX_MESSAGE_SIZE_BYTES } from './chunk-HZ3OAUTL.mjs';
|
|
9
|
-
import { bindEngineWriteBlock, bindIndexReads, syncServingSourceOf, bindSyncServing, changesServingSourceOf, bindChangesServing, validateBackupConfig, BACKUP_SWEEP_INTERVAL_MS, bindWriteBlockLookup, backupKeys, validateBackupName, CHANGES_BACKUP_TAKER, BACKUP_MANIFEST_READS_AT_ONCE, storedFilesOf, decodeBackupManifest, BACKUP_JOBS_KEY, BACKUP_DELETES_AT_ONCE, BACKUP_PROGRESS_REPORT_INTERVAL_MS, BACKUP_PARTITION_RESULT_RETENTION_MS, BACKUP_JOB_POLL_INTERVAL_MS, BACKUP_FILES_AT_ONCE, isCount, storeWithoutSaveIfAbsent, CLUSTER_UUID_KEY, isSha256Hex, BACKUP_PUBLISH_RETRY_MS, isBackupTaker, readRestorePlacement, SYNC_BACKUP_TAKER, BACKUP_JOB_CAS_ATTEMPTS, BACKUP_JOB_CAS_RETRY_MS, MAX_CLUSTER_UUID_BYTES, MAX_FINISHED_BACKUP_JOBS, BACKUP_FILE_CHUNK_BYTES, encodeBackupManifest, SYNC_BACKUPS_KEPT, BACKUP_PARTITIONS_AT_ONCE, BACKUP_PARTITION_POLL_INTERVAL_MS, BACKUP_PRIMARY_WAIT_MS, readBackupPartition, readStoredFile } from './chunk-
|
|
10
|
-
import { DEFAULT_NODE_LIFECYCLE_CONFIG, DEFAULT_CONTROLLER_CONFIG,
|
|
10
|
+
import { bindEngineWriteBlock, bindIndexReads, syncServingSourceOf, bindSyncServing, changesServingSourceOf, bindChangesServing, validateBackupConfig, BACKUP_SWEEP_INTERVAL_MS, bindWriteBlockLookup, backupKeys, validateBackupName, CHANGES_BACKUP_TAKER, BACKUP_MANIFEST_READS_AT_ONCE, storedFilesOf, decodeBackupManifest, BACKUP_JOBS_KEY, BACKUP_DELETES_AT_ONCE, BACKUP_PROGRESS_REPORT_INTERVAL_MS, BACKUP_PARTITION_RESULT_RETENTION_MS, BACKUP_JOB_POLL_INTERVAL_MS, BACKUP_FILES_AT_ONCE, isCount, storeWithoutSaveIfAbsent, CLUSTER_UUID_KEY, isSha256Hex, BACKUP_PUBLISH_RETRY_MS, isBackupTaker, readRestorePlacement, SYNC_BACKUP_TAKER, BACKUP_JOB_CAS_ATTEMPTS, BACKUP_JOB_CAS_RETRY_MS, MAX_CLUSTER_UUID_BYTES, MAX_FINISHED_BACKUP_JOBS, BACKUP_FILE_CHUNK_BYTES, encodeBackupManifest, SYNC_BACKUPS_KEPT, BACKUP_PARTITIONS_AT_ONCE, BACKUP_PARTITION_POLL_INTERVAL_MS, BACKUP_PRIMARY_WAIT_MS, readBackupPartition, readStoredFile } from './chunk-BVFJ5XAU.mjs';
|
|
11
|
+
import { DEFAULT_NODE_LIFECYCLE_CONFIG, DEFAULT_CONTROLLER_CONFIG, getIndexMetadata, getClusterIndexConfig, withPartitionCount, validateIndexName, readIndexMetadataSnapshot, indexConfigKey, metadataWithSync, suggestionStateOfSettings, indexConfigFromSettings, metadataWithBm25, readIndexState, storeSynonymSet, deleteSynonymSet, indexSettingsOf, putIndexMetadata, isValidInteger, isRecord, HIDDEN_GENERATION_PREFIX, indexMetadataSnapshotOf, metadataWithSynonymSet, RESHARD_RECORD_CAS_ATTEMPTS, indexUuidOfStoredMetadata, storedSynonymSetOf, RESHARD_FORWARD_BATCH_DOCUMENTS, isRecord2, decodeIndexSettings, RESHARD_RETRY_DELAY_MS, TEARDOWN_CAS_ATTEMPTS, RESHARD_RECORD_CHECK_INTERVAL_MS, RESHARD_POLL_INTERVAL_MS, ALLOCATION_RETRY_DELAY_MS, TEARDOWN_RETRY_ROUNDS, ANALYSIS_CHANGE_RETRY_MS, putIndexReplicationFactor, ALLOCATION_CAS_ATTEMPTS, TEARDOWN_RETRY_DELAY_MS, RESHARD_CONTROLLER_POLL_INTERVAL_MS, MAX_RESHARD_CUTOVER_ATTEMPTS, ANALYSIS_CHANGE_POLL_INTERVAL_MS, RECOVERY_CAS_ATTEMPTS, ESTIMATED_PARTITION_BYTES, REBALANCE_THRESHOLD, putGenerationMetadata, RESHARD_CUTOVER_LAG_ENTRIES, ANALYSIS_REBUILD_RESEND_MS, RESHARD_CUTOVER_TIMEOUT_MS, PARTITION_STORES_TIMEOUT_MS, SYNC_EPOCH_CAS_ATTEMPTS, syncSettingsOfSnapshot, epochFeaturesOn, LEADERSHIP_IMBALANCE_THRESHOLD, BOOTSTRAP_CAS_ATTEMPTS, metadataWithSuggestionState, UNFED_REPLICA_CHECK_INTERVAL_MS, UNFED_REPLICA_RESYNC_MS } from './chunk-UJQ2S5QU.mjs';
|
|
11
12
|
import './chunk-4H4RG2FR.mjs';
|
|
12
|
-
import { MAX_WAIT_FOR_ACTIVE_REPLICAS, MIN_CONTROLLER_LEASE_TTL_MS, MAX_NODE_LEFT_DELAY_MS, DEFAULT_MAX_PER_SOURCE_SNAPSHOTS, DEFAULT_MAX_STREAMS_PER_INDEX, DEFAULT_CAPACITY, CATCH_UP_TICK_MS, MEMORY_SAMPLE_INTERVAL_MS, createWritePressure, WRITE_MEMORY_SAMPLE_MAX_AGE_MS, DEFAULT_IN_FLIGHT_WRITE_FRACTION, MAX_SETTINGS_REPLACEMENT_ATTEMPTS, SETTINGS_CONTENTION_REASON, MAX_DROP_ACKNOWLEDGE_WAIT_MS, DEFAULT_PARTITION_COUNT, DEFAULT_REPLICATION_FACTOR, DEFAULT_CREATE_INDEX_WAIT_MS, DEFAULT_BOOTSTRAP_SYNC_DEADLINE_MS, DROP_ACKNOWLEDGE_POLL_INTERVAL_MS, COORDINATOR_REQUEST_BUCKET_BOUNDS_SECONDS, COPY_FAILURE_ATTEMPTS, SEARCH_CONTEXT_REAPER_INTERVAL_MS, DECOMMISSION_RECHECK_INTERVAL_MS, refuseEncodeWithoutRoom, answerRefusalOf, withAdmittedWrite, estimateForwardedBytes, PARTITION_BYTES_INDEX_OVERHEAD_BYTES, MAX_SOURCE_ID_LENGTH, ADMISSION_TIMEOUT_MS, PARTITION_BYTES_ENTRY_BYTES, PARTITION_BYTES_REPORT_BUDGET_BYTES, CATCH_UP_FEED_WINDOW_MS, MAX_CATCH_UP_IN_FLIGHT_BYTES, SYNCED_POSITION_REPORT_TIMEOUT_MS, SNAPSHOT_CHUNK_YIELD_INTERVAL_MS, ANSWER_DECODE_HEAP_BYTES_PER_PAYLOAD_BYTE, ADMISSION_RETRIES_AFTER_IN_SYNC_CHANGE, REGISTRATION_LEASE_DRIFT_SHARE, REGISTRATION_LEASE_CONFIRM_TIMEOUT_MS, WRITE_APPLY_YIELD_INTERVAL_MS, ROLLBACK_RETRY_INITIAL_DELAY_MS, ROLLBACK_RETRY_DELAY_CAP_MS, OUT_OF_STEP_REMOVAL_RETRY_INITIAL_DELAY_MS, OUT_OF_STEP_REMOVAL_RETRY_DELAY_CAP_MS, FETCH_GROUP_DOCUMENT_IDS, COPY_FAILURE_RETRY_DELAY_MS, SEARCH_CONTEXT_ENTRY_HEAP_BYTES, CLEAR_PAGE_SIZE, estimateDocumentsBytes, estimateDocumentBytes, PRIMARY_CHANGE_RETRY_WINDOW_MS, PRIMARY_CHANGE_POLL_INTERVAL_MS, PARTITION_WRITE_CONCURRENCY, FORWARD_RETRY_INITIAL_DELAY_MS, FORWARD_RETRY_DELAY_CAP_MS } from './chunk-
|
|
13
|
-
export { DEFAULT_CAPACITY, DEFAULT_PARTITION_COUNT, DEFAULT_REPLICATION_FACTOR } from './chunk-
|
|
14
|
-
import { encodeIndexConfiguration, MAX_SNAPSHOT_ANALYSIS_ATTEMPTS, decodeIndexConfiguration, requireSyncSetting, requireNoQuerySuggestions, validChangeFilter, createDocumentTest, INDEX_CONFIGURATION_ENVELOPE_VERSION, changesNotEnabled } from './chunk-
|
|
15
|
-
import { restoredConfigFields, restoredEmbedding } from './chunk-
|
|
16
|
-
import { readSnapshotArchive, archiveLayout, openManifestPayload, sourceIndexNameOf, listedKeysOf, decodeManifestPayload, estimatedArchiveBytes } from './chunk-
|
|
13
|
+
import { MAX_WAIT_FOR_ACTIVE_REPLICAS, MIN_CONTROLLER_LEASE_TTL_MS, MAX_NODE_LEFT_DELAY_MS, DEFAULT_MAX_PER_SOURCE_SNAPSHOTS, DEFAULT_MAX_STREAMS_PER_INDEX, DEFAULT_CAPACITY, CATCH_UP_TICK_MS, MEMORY_SAMPLE_INTERVAL_MS, createWritePressure, WRITE_MEMORY_SAMPLE_MAX_AGE_MS, DEFAULT_IN_FLIGHT_WRITE_FRACTION, MAX_SETTINGS_REPLACEMENT_ATTEMPTS, SETTINGS_CONTENTION_REASON, MAX_DROP_ACKNOWLEDGE_WAIT_MS, DEFAULT_PARTITION_COUNT, DEFAULT_REPLICATION_FACTOR, DEFAULT_CREATE_INDEX_WAIT_MS, DEFAULT_BOOTSTRAP_SYNC_DEADLINE_MS, DROP_ACKNOWLEDGE_POLL_INTERVAL_MS, COORDINATOR_REQUEST_BUCKET_BOUNDS_SECONDS, COPY_FAILURE_ATTEMPTS, SEARCH_CONTEXT_REAPER_INTERVAL_MS, DECOMMISSION_RECHECK_INTERVAL_MS, refuseEncodeWithoutRoom, answerRefusalOf, withAdmittedWrite, estimateForwardedBytes, PARTITION_BYTES_INDEX_OVERHEAD_BYTES, MAX_SOURCE_ID_LENGTH, ADMISSION_TIMEOUT_MS, PARTITION_BYTES_ENTRY_BYTES, PARTITION_BYTES_REPORT_BUDGET_BYTES, CATCH_UP_FEED_WINDOW_MS, MAX_CATCH_UP_IN_FLIGHT_BYTES, SYNCED_POSITION_REPORT_TIMEOUT_MS, SNAPSHOT_CHUNK_YIELD_INTERVAL_MS, ANSWER_DECODE_HEAP_BYTES_PER_PAYLOAD_BYTE, ADMISSION_RETRIES_AFTER_IN_SYNC_CHANGE, REGISTRATION_LEASE_DRIFT_SHARE, REGISTRATION_LEASE_CONFIRM_TIMEOUT_MS, WRITE_APPLY_YIELD_INTERVAL_MS, ROLLBACK_RETRY_INITIAL_DELAY_MS, ROLLBACK_RETRY_DELAY_CAP_MS, OUT_OF_STEP_REMOVAL_RETRY_INITIAL_DELAY_MS, OUT_OF_STEP_REMOVAL_RETRY_DELAY_CAP_MS, FETCH_GROUP_DOCUMENT_IDS, COPY_FAILURE_RETRY_DELAY_MS, SEARCH_CONTEXT_ENTRY_HEAP_BYTES, CLEAR_PAGE_SIZE, estimateDocumentsBytes, estimateDocumentBytes, PRIMARY_CHANGE_RETRY_WINDOW_MS, PRIMARY_CHANGE_POLL_INTERVAL_MS, PARTITION_WRITE_CONCURRENCY, FORWARD_RETRY_INITIAL_DELAY_MS, FORWARD_RETRY_DELAY_CAP_MS } from './chunk-KHZ6LAFB.mjs';
|
|
14
|
+
export { DEFAULT_CAPACITY, DEFAULT_PARTITION_COUNT, DEFAULT_REPLICATION_FACTOR } from './chunk-KHZ6LAFB.mjs';
|
|
15
|
+
import { encodeIndexConfiguration, MAX_SNAPSHOT_ANALYSIS_ATTEMPTS, decodeIndexConfiguration, requireSyncSetting, requireNoQuerySuggestions, validChangeFilter, createDocumentTest, INDEX_CONFIGURATION_ENVELOPE_VERSION, changesNotEnabled } from './chunk-CVCBVUH4.mjs';
|
|
16
|
+
import { restoredConfigFields, restoredEmbedding } from './chunk-UGMOBSMD.mjs';
|
|
17
|
+
import { readSnapshotArchive, archiveLayout, openManifestPayload, sourceIndexNameOf, listedKeysOf, decodeManifestPayload, estimatedArchiveBytes } from './chunk-I255IQTR.mjs';
|
|
17
18
|
import { CONTROLLER_LEASE_KEY } from './chunk-44W3CBNG.mjs';
|
|
18
19
|
import './chunk-T4HMSZCX.mjs';
|
|
19
|
-
import { createForwardBatchResultMessage, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, DEFAULT_LOG_RETENTION_BYTES, buildEntry, invalidReplicationPayload, optionalIndexUuid, validateForwardOperation, entryByteEstimate, computeEntryChecksum, createPrimaryResignMessage, validateInsyncConfirmPayload, createInsyncRemoveMessage, SNAPSHOT_CHUNK_SIZE, createEntryBatchMessage, createEntryMessage, validateAckPayload, createInsyncAddMessage, createInsyncConfirmMessage, checkpointLastSeqNo, checkpointPrimaryTerm, INSYNC_CAS_ATTEMPTS, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage } from './chunk-
|
|
20
|
-
import { directoryCheckpointStore, readListedFile, encodeSegmentManifest } from './chunk-
|
|
20
|
+
import { createForwardBatchResultMessage, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, DEFAULT_LOG_RETENTION_BYTES, buildEntry, invalidReplicationPayload, optionalIndexUuid, validateForwardOperation, entryByteEstimate, computeEntryChecksum, createPrimaryResignMessage, validateInsyncConfirmPayload, createInsyncRemoveMessage, SNAPSHOT_CHUNK_SIZE, createEntryBatchMessage, createEntryMessage, validateAckPayload, createInsyncAddMessage, createInsyncConfirmMessage, checkpointLastSeqNo, checkpointPrimaryTerm, INSYNC_CAS_ATTEMPTS, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage } from './chunk-B6VO3BCX.mjs';
|
|
21
|
+
import { directoryCheckpointStore, readListedFile, encodeSegmentManifest } from './chunk-QWYIMVYT.mjs';
|
|
21
22
|
import { TransportErrorCodes, ClusterMessageTypes, ReplicationMessageTypes, SuggestionMessageTypes, SyncMessageTypes, QueryMessageTypes, TransportError, DEFAULT_TRANSPORT_CONFIG } from './chunk-5FLRXF73.mjs';
|
|
22
23
|
import { MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR, WIRE_BATCH_BUDGET } from './chunk-JDDXV6UF.mjs';
|
|
23
|
-
import { manifestFileRefs, segmentKey, SEGMENT_MANIFEST_VERSION, vectorFileKey } from './chunk-
|
|
24
|
+
import { manifestFileRefs, segmentKey, SEGMENT_MANIFEST_VERSION, vectorFileKey } from './chunk-ICEZRKIS.mjs';
|
|
24
25
|
import './chunk-S5QMWECL.mjs';
|
|
25
26
|
import './chunk-LNGE7SLY.mjs';
|
|
26
|
-
import './chunk-
|
|
27
|
-
import './chunk-
|
|
28
|
-
import { deserializePartition } from './chunk-
|
|
27
|
+
import './chunk-L2GYY6KY.mjs';
|
|
28
|
+
import './chunk-AKLCYPUY.mjs';
|
|
29
|
+
import { deserializePartition } from './chunk-HNE4HNC2.mjs';
|
|
29
30
|
import { decodeVectorFilePayload, bytesToVectors, isDead, MIN_EF_SEARCH, MAX_EF_SEARCH, MIN_OVERSAMPLE } from './chunk-R724DJSG.mjs';
|
|
30
|
-
import './chunk-
|
|
31
|
-
import { RestoredHistory } from './chunk-
|
|
31
|
+
import './chunk-B276ZQIH.mjs';
|
|
32
|
+
import { RestoredHistory } from './chunk-CHGJGKST.mjs';
|
|
32
33
|
import { oneSyncChangeAtATime } from './chunk-V6EUBAIT.mjs';
|
|
33
|
-
import { historyFilesOf, removeHistoryFolder, QUERY_HISTORY_FOLDER, RESTORE_STAGING_INFIX, openChecksummedEnvelope, historyFolderOf, listHistoryFolder, historyFileOf, COUNTS_STREAM_CHUNK_BYTES, BUILD_WRITE_BATCH, restoreStagingPrefix, crc32Init, crc32Final, crc32Update, MAX_COUNTS_STREAM_KEY_BYTES, MAX_HISTORY_FILE_BYTES, packChecksummedEnvelope, storedStateOf } from './chunk-
|
|
34
|
+
import { historyFilesOf, removeHistoryFolder, QUERY_HISTORY_FOLDER, RESTORE_STAGING_INFIX, openChecksummedEnvelope, historyFolderOf, listHistoryFolder, historyFileOf, COUNTS_STREAM_CHUNK_BYTES, BUILD_WRITE_BATCH, restoreStagingPrefix, crc32Init, crc32Final, crc32Update, MAX_COUNTS_STREAM_KEY_BYTES, MAX_HISTORY_FILE_BYTES, packChecksummedEnvelope, storedStateOf } from './chunk-R5OFJZNI.mjs';
|
|
34
35
|
import { boundQuerySuggestionsOf, bindQuerySuggestions } from './chunk-S2POFTLN.mjs';
|
|
35
|
-
import { oversampledShardSize, runtimeHost, bindRerankWindowSource, rerankWindowCapOf, validateMemoryFractions, requireKnownConfig, importOptionalReads, resolveRunnableConfig, createEngineCore, startEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, emitEngineEvent, synonymModule, createEngineIndex, generateId, documentChurnOf, isCompositePartition, getLanguage, resolveIndexAnalysis, memoryFractionsOf, createMemoryMonitor, readMemoryFigures, FULL_MEMORY_COLLECTION_INTERVAL_MS, validateBM25Params, storedBM25Params, requireSurfaceFormsForPartialFields, bm25SetsValue, prepareConfiguredSynonyms, storedArrayPositionGap, updateDocumentVectors, arrayPositionsAreStale, fnv1a, readRecentMemoryFigures,
|
|
36
|
+
import { oversampledShardSize, runtimeHost, bindRerankWindowSource, rerankWindowCapOf, validateMemoryFractions, requireKnownConfig, importOptionalReads, resolveRunnableConfig, createEngineCore, startEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, emitEngineEvent, synonymModule, createEngineIndex, generateId, documentChurnOf, isCompositePartition, getLanguage, resolveIndexAnalysis, memoryFractionsOf, createMemoryMonitor, readMemoryFigures, FULL_MEMORY_COLLECTION_INTERVAL_MS, validateBM25Params, storedBM25Params, requireSurfaceFormsForPartialFields, validateBooleanOnlySettings, bm25SetsValue, prepareConfiguredSynonyms, storedArrayPositionGap, updateDocumentVectors, arrayPositionsAreStale, fnv1a, readRecentMemoryFigures, normalizeSort, sortModesOf, noteDocumentChurn, MAP_TABLE_MIN_CAPACITY, MAP_TABLE_SHRINK_DIVISOR, resolveProjection, applyProjection, requireLoadedOptionalReads, FILTER_EXPRESSION_KEYS, filterTestCount, setNestedValue, CASE_FOLD_EXPANSION_FACTOR, PATTERN_RUN_HEAP_BYTES, ENUM_VALUE_HEAP_BYTES, validateBM25Shape, DOCUMENT_VALUE_HEAP_BYTES, DOCUMENT_HEAP_BYTES_PER_ESTIMATED_BYTE, DOCUMENT_PROPERTY_HEAP_BYTES, VECTOR_DOCUMENT_HEAP_BYTES, VECTOR_COMPONENT_HEAP_BYTES, hasLanguage, FIELD_FILTER_OPERATORS, yieldToEventLoop, NORMALIZATION_EXPANSION_FACTOR, TERM_UNIT_HEAP_BYTES, customTokensOf, visitTokens, getVectorFieldPaths, resolvePartitionInsertOptions, clampRowCount, now, sortSignatureOf, listBindingOf, requireMatchingCursor, decodePageCursor, encodePageCursor, DISTINCT_TERM_HEAP_BYTES, PARTIAL_SPELLING_HEAP_BYTES, clampLimit, requireWithinResultWindow, providedDocId, MAP_TABLE_HEADER_BYTES, MAP_TABLE_SLOT_BYTES, holdsQuotedText, queryBindingOf, engineCoreOf, sortSignatureEntry, fnv1aUint32s, loadRerankRuntime, mergePartitionStats, mergeEntries, regionsOverlap, compareScores } from './chunk-4QNWK42S.mjs';
|
|
36
37
|
import { encodeDocument, encodeDocumentReply } from './chunk-2CKDCC65.mjs';
|
|
37
38
|
import { collectYoungGenerationBeforeRefusal, collectGarbageBeforeRefusal, onRefusalAfterCollection, reservedHeapBytes, reserveHeapBytes } from './chunk-BRTABNC7.mjs';
|
|
38
39
|
import { decode } from './chunk-Y7GXGJFZ.mjs';
|
|
39
40
|
import { newSyncEpoch } from './chunk-BUMREPLU.mjs';
|
|
40
41
|
import './chunk-7T4LAYT4.mjs';
|
|
41
|
-
import { MAX_FILTER_DEPTH, MAX_FILTER_TESTS, isPatternTolerance, MAX_FILTER_ARRAY_SIZE, buildStoredValues } from './chunk-
|
|
42
|
-
import { loadMessagePack, validateSchema, requireCreatableSchema, schemaFieldsOf, getNestedValue, MAX_NESTING_DEPTH, MAX_STORED_VECTOR_DIMENSION, MIN_POLYGON_POINTS, isLatitude, MAX_LATITUDE_DEGREES, isLongitude, MAX_LONGITUDE_DEGREES } from './chunk-
|
|
42
|
+
import { MAX_FILTER_DEPTH, MAX_FILTER_TESTS, isPatternTolerance, MAX_FILTER_ARRAY_SIZE, buildStoredValues } from './chunk-M5RYB6FE.mjs';
|
|
43
|
+
import { loadMessagePack, validateSchema, requireCreatableSchema, schemaFieldsOf, getNestedValue, MAX_NESTING_DEPTH, MAX_STORED_VECTOR_DIMENSION, MIN_POLYGON_POINTS, isLatitude, MAX_LATITUDE_DEGREES, isLongitude, MAX_LONGITUDE_DEGREES } from './chunk-7NMLFP2T.mjs';
|
|
43
44
|
import './chunk-U2BKYEC7.mjs';
|
|
44
45
|
import { shardEntryRangeOf } from './chunk-PSZEGK6W.mjs';
|
|
45
46
|
import { collectAndReturnPages } from './chunk-5EI5CXO6.mjs';
|
|
@@ -1219,6 +1220,9 @@ function validateFetchPayload(decoded) {
|
|
|
1219
1220
|
if (decoded.fields !== null) {
|
|
1220
1221
|
validateStringArray(decoded.fields, "fields", MAX_FIELDS_LIST, 255, SEARCH_INVALID_FIELD);
|
|
1221
1222
|
}
|
|
1223
|
+
if (decoded.vectors !== null && decoded.vectors !== void 0 && typeof decoded.vectors !== "boolean") {
|
|
1224
|
+
throwInvalid(CONFIG_INVALID, 'Invalid FetchPayload: "vectors" must be a boolean, or null');
|
|
1225
|
+
}
|
|
1222
1226
|
if (decoded.contextId !== null && decoded.contextId !== void 0) {
|
|
1223
1227
|
validateStringField(decoded.contextId, "contextId", MAX_SEARCH_CONTEXT_ID_LENGTH, CONFIG_INVALID);
|
|
1224
1228
|
}
|
|
@@ -2200,6 +2204,9 @@ function validateListPayload(decoded) {
|
|
|
2200
2204
|
if (decoded.fields !== null) {
|
|
2201
2205
|
validateStringArray(decoded.fields, "fields", MAX_FIELDS_LIST, 255, SEARCH_INVALID_FIELD);
|
|
2202
2206
|
}
|
|
2207
|
+
if (decoded.vectors !== null && decoded.vectors !== void 0 && typeof decoded.vectors !== "boolean") {
|
|
2208
|
+
throwInvalid(CONFIG_INVALID, 'Invalid ListPayload: "vectors" must be a boolean, or null');
|
|
2209
|
+
}
|
|
2203
2210
|
return decoded;
|
|
2204
2211
|
}
|
|
2205
2212
|
function validateSortValues(value, fieldLabel) {
|
|
@@ -4781,8 +4788,9 @@ async function writeLocalOnly(deps, indexName, write) {
|
|
|
4781
4788
|
deps.writeRecords?.dropIndex(indexName);
|
|
4782
4789
|
}
|
|
4783
4790
|
}
|
|
4791
|
+
var WHOLE_DOCUMENT = { document: { vectors: true } };
|
|
4784
4792
|
async function storedCopyOf(scope, documentId) {
|
|
4785
|
-
const stored = await scope.deps.engine.get(scope.indexName, documentId);
|
|
4793
|
+
const stored = await scope.deps.engine.get(scope.indexName, documentId, WHOLE_DOCUMENT);
|
|
4786
4794
|
if (stored === void 0) {
|
|
4787
4795
|
throw new NarsilError(
|
|
4788
4796
|
ErrorCodes.REPLICATION_ENTRY_INVALID,
|
|
@@ -6295,7 +6303,7 @@ async function applyPrimaryRemove(indexName, docId, partitionId, assignment, dep
|
|
|
6295
6303
|
const queued = { indexName, partitionId, assignment, deps };
|
|
6296
6304
|
await inLeadingPartitionQueue(queued, async (scope) => {
|
|
6297
6305
|
if (condition !== void 0 && decideCondition(scope, docId, condition) === "acknowledge") return;
|
|
6298
|
-
const previousDocument = await deps.engine.get(indexName, docId);
|
|
6306
|
+
const previousDocument = await deps.engine.get(indexName, docId, WHOLE_DOCUMENT);
|
|
6299
6307
|
const previousRecord = previousRecordOf(scope, docId);
|
|
6300
6308
|
const removed = await numberedWrite(
|
|
6301
6309
|
scope,
|
|
@@ -6319,7 +6327,7 @@ async function applyPrimaryUpdate(indexName, docId, document, partitionId, assig
|
|
|
6319
6327
|
if (condition !== void 0 && decideCondition(scope, docId, condition) === "acknowledge") return;
|
|
6320
6328
|
const projected = projectOne(deps, indexName, document);
|
|
6321
6329
|
try {
|
|
6322
|
-
const previousDocument = await deps.engine.get(indexName, docId);
|
|
6330
|
+
const previousDocument = await deps.engine.get(indexName, docId, WHOLE_DOCUMENT);
|
|
6323
6331
|
const previousRecord = previousRecordOf(scope, docId);
|
|
6324
6332
|
const updated = await numberedWrite(
|
|
6325
6333
|
scope,
|
|
@@ -6404,7 +6412,7 @@ async function applyPrimaryUpdateBatch(indexName, updates, partitionId, assignme
|
|
|
6404
6412
|
acknowledged.push(update.docId);
|
|
6405
6413
|
return;
|
|
6406
6414
|
}
|
|
6407
|
-
const previousDocument = await deps.engine.get(indexName, update.docId);
|
|
6415
|
+
const previousDocument = await deps.engine.get(indexName, update.docId, WHOLE_DOCUMENT);
|
|
6408
6416
|
const previousRecord = previousRecordOf(scope, update.docId);
|
|
6409
6417
|
const updated = await numberedWrite(
|
|
6410
6418
|
scope,
|
|
@@ -6451,7 +6459,7 @@ async function applyPrimaryRemoveBatch(indexName, docIds, partitionId, assignmen
|
|
|
6451
6459
|
acknowledged.push(docId);
|
|
6452
6460
|
continue;
|
|
6453
6461
|
}
|
|
6454
|
-
const previousDocument = await deps.engine.get(indexName, docId);
|
|
6462
|
+
const previousDocument = await deps.engine.get(indexName, docId, WHOLE_DOCUMENT);
|
|
6455
6463
|
const previousRecord = previousRecordOf(scope, docId);
|
|
6456
6464
|
const removed = await numberedWrite(
|
|
6457
6465
|
scope,
|
|
@@ -6555,7 +6563,7 @@ async function resolveForwardedUpdateDocument(indexName, documentId, document, u
|
|
|
6555
6563
|
"Invalid ForwardPayload: update requires a document or updateFields"
|
|
6556
6564
|
);
|
|
6557
6565
|
}
|
|
6558
|
-
const existing = await deps.engine.get(indexName, documentId);
|
|
6566
|
+
const existing = await deps.engine.get(indexName, documentId, WHOLE_DOCUMENT);
|
|
6559
6567
|
if (existing === void 0) {
|
|
6560
6568
|
throw new NarsilError(ErrorCodes.DOC_NOT_FOUND, `Document '${documentId}' does not exist in index '${indexName}'`, {
|
|
6561
6569
|
indexName,
|
|
@@ -7154,6 +7162,7 @@ async function routeCreateIndex(name, requestedConfig, options, coordinator, eng
|
|
|
7154
7162
|
const config = { ...requestedConfig, schema: validateSchema(requestedConfig.schema) };
|
|
7155
7163
|
requireCreatableSchema(config.schema);
|
|
7156
7164
|
requireSurfaceFormsForPartialFields(config.schema, config.surfaceForms);
|
|
7165
|
+
validateBooleanOnlySettings(config);
|
|
7157
7166
|
if (config.bm25 !== void 0) validateBM25Params(config.bm25, config.schema, ErrorCodes.CONFIG_INVALID);
|
|
7158
7167
|
const setsFieldPairs = config.bm25?.fields !== void 0 && bm25SetsValue({ fields: config.bm25.fields });
|
|
7159
7168
|
if (setsFieldPairs) {
|
|
@@ -7446,7 +7455,7 @@ function refuseDecodeWithoutRoom(payload) {
|
|
|
7446
7455
|
const refusal2 = answerRefusalOf(payload.byteLength * ANSWER_DECODE_HEAP_BYTES_PER_PAYLOAD_BYTE);
|
|
7447
7456
|
if (refusal2 !== null) throw markNodeRejection(refusal2);
|
|
7448
7457
|
}
|
|
7449
|
-
async function readFromCopies(config, nodeId, engine, indexName, documentRouting,
|
|
7458
|
+
async function readFromCopies(config, nodeId, engine, indexName, documentRouting, read) {
|
|
7450
7459
|
const documents = /* @__PURE__ */ new Map();
|
|
7451
7460
|
await sendToEveryPartition(documentRouting.routing, async (targetNodeId, partitionIds) => {
|
|
7452
7461
|
const documentIds = partitionIds.flatMap(
|
|
@@ -7456,13 +7465,23 @@ async function readFromCopies(config, nodeId, engine, indexName, documentRouting
|
|
|
7456
7465
|
if (targetNodeId === nodeId) {
|
|
7457
7466
|
const local = await engine.getMultiple(
|
|
7458
7467
|
indexName,
|
|
7459
|
-
documentIds.map((ref) => ref.docId)
|
|
7468
|
+
documentIds.map((ref) => ref.docId),
|
|
7469
|
+
{ document: { vectors: read.vectors } }
|
|
7460
7470
|
);
|
|
7461
7471
|
for (const [docId, document] of local) found.set(docId, document);
|
|
7462
7472
|
} else {
|
|
7463
7473
|
for (let start = 0; start < documentIds.length; start += FETCH_GROUP_DOCUMENT_IDS) {
|
|
7464
7474
|
const chunk = documentIds.slice(start, start + FETCH_GROUP_DOCUMENT_IDS);
|
|
7465
|
-
const fetchMessage = createFetchMessage(
|
|
7475
|
+
const fetchMessage = createFetchMessage(
|
|
7476
|
+
{
|
|
7477
|
+
indexName,
|
|
7478
|
+
documentIds: chunk,
|
|
7479
|
+
fields: read.fields,
|
|
7480
|
+
...read.vectors ? {} : { vectors: false },
|
|
7481
|
+
highlight: null
|
|
7482
|
+
},
|
|
7483
|
+
nodeId
|
|
7484
|
+
);
|
|
7466
7485
|
const response = await sendToNode2(config, targetNodeId, fetchMessage, {
|
|
7467
7486
|
message: `Node '${targetNodeId}' did not answer a document fetch for index '${indexName}'`,
|
|
7468
7487
|
details: { indexName }
|
|
@@ -7483,7 +7502,7 @@ async function readFromCopies(config, nodeId, engine, indexName, documentRouting
|
|
|
7483
7502
|
});
|
|
7484
7503
|
return documents;
|
|
7485
7504
|
}
|
|
7486
|
-
async function readDocumentsFromCopies(config, nodeId, engine, indexName, docIds, allocation,
|
|
7505
|
+
async function readDocumentsFromCopies(config, nodeId, engine, indexName, docIds, allocation, read, requireEveryPartition) {
|
|
7487
7506
|
const documentRouting = routeDocumentIds(docIds, allocation);
|
|
7488
7507
|
const unreachablePartitions = documentRouting.routing.unavailablePartitions;
|
|
7489
7508
|
if (requireEveryPartition && unreachablePartitions.length > 0) {
|
|
@@ -7493,10 +7512,10 @@ async function readDocumentsFromCopies(config, nodeId, engine, indexName, docIds
|
|
|
7493
7512
|
{ indexName, partitionIds: [...unreachablePartitions].sort((a, b) => a - b) }
|
|
7494
7513
|
);
|
|
7495
7514
|
}
|
|
7496
|
-
return readFromCopies(config, nodeId, engine, indexName, documentRouting,
|
|
7515
|
+
return readFromCopies(config, nodeId, engine, indexName, documentRouting, read);
|
|
7497
7516
|
}
|
|
7498
|
-
async function readDistributedDocuments(config, nodeId, engine, indexName, docIds, allocation) {
|
|
7499
|
-
return readDocumentsFromCopies(config, nodeId, engine, indexName, docIds, allocation, null, true);
|
|
7517
|
+
async function readDistributedDocuments(config, nodeId, engine, indexName, docIds, allocation, vectors) {
|
|
7518
|
+
return readDocumentsFromCopies(config, nodeId, engine, indexName, docIds, allocation, { fields: null, vectors }, true);
|
|
7500
7519
|
}
|
|
7501
7520
|
|
|
7502
7521
|
// src/distribution/backups/partition-exchange.ts
|
|
@@ -8814,227 +8833,6 @@ async function openSnapshotSpoolDirectory(durabilityDirectory) {
|
|
|
8814
8833
|
return { path, dispose: () => rm(path, { recursive: true, force: true }) };
|
|
8815
8834
|
}
|
|
8816
8835
|
|
|
8817
|
-
// src/distribution/cluster-node/analysis-change/view.ts
|
|
8818
|
-
function namedSettingsOfMetadata(settings) {
|
|
8819
|
-
return {
|
|
8820
|
-
language: settings?.language ?? "english",
|
|
8821
|
-
...settings?.tokenizer !== void 0 ? { tokenizer: settings.tokenizer } : {},
|
|
8822
|
-
...settings?.stopWords !== void 0 ? { stopWords: settings.stopWords } : {},
|
|
8823
|
-
...settings?.stopWordList !== void 0 ? { stopWordList: settings.stopWordList } : {}
|
|
8824
|
-
};
|
|
8825
|
-
}
|
|
8826
|
-
function generationMissing(indexName, requestedGeneration, heldGeneration) {
|
|
8827
|
-
return new NarsilError(
|
|
8828
|
-
ErrorCodes.ANALYSIS_GENERATION_MISSING,
|
|
8829
|
-
`This node holds analysis generation ${heldGeneration} of index '${indexName}', so it cannot answer for generation ${requestedGeneration}`,
|
|
8830
|
-
{ indexName, requestedGeneration, heldGeneration }
|
|
8831
|
-
);
|
|
8832
|
-
}
|
|
8833
|
-
function createAnalysisView(deps) {
|
|
8834
|
-
const { coordinator, engine, onError } = deps;
|
|
8835
|
-
const held = /* @__PURE__ */ new Map();
|
|
8836
|
-
const settling = /* @__PURE__ */ new Map();
|
|
8837
|
-
let unwatch = null;
|
|
8838
|
-
let stopped = false;
|
|
8839
|
-
function recordGeneration(indexName, snapshot) {
|
|
8840
|
-
const generation = snapshot.settings?.analysisGeneration ?? 0;
|
|
8841
|
-
const { indexUuid } = snapshot.record;
|
|
8842
|
-
const known = held.get(indexName);
|
|
8843
|
-
if (known?.indexUuid === indexUuid && known.generation >= generation) return known.generation;
|
|
8844
|
-
held.set(indexName, { indexUuid, generation });
|
|
8845
|
-
return generation;
|
|
8846
|
-
}
|
|
8847
|
-
async function refresh(indexName) {
|
|
8848
|
-
const snapshot = await readIndexMetadataSnapshot(coordinator, indexName);
|
|
8849
|
-
if (snapshot === null) {
|
|
8850
|
-
held.delete(indexName);
|
|
8851
|
-
return 0;
|
|
8852
|
-
}
|
|
8853
|
-
return recordGeneration(indexName, snapshot);
|
|
8854
|
-
}
|
|
8855
|
-
function catchUp(indexName, generation, settings) {
|
|
8856
|
-
engine.buildHeldAnalysis(indexName, { generation, ...namedSettingsOfMetadata(settings) }).then(() => engine.releaseHeldAnalysis(indexName, generation)).catch((error) => {
|
|
8857
|
-
if (!stopped) onError(error);
|
|
8858
|
-
});
|
|
8859
|
-
}
|
|
8860
|
-
async function settleOnce(indexName) {
|
|
8861
|
-
const generation = engine.analysisGenerationOf(indexName);
|
|
8862
|
-
if (stopped || generation === void 0) return;
|
|
8863
|
-
const snapshot = await readIndexMetadataSnapshot(coordinator, indexName);
|
|
8864
|
-
const localUuid = engine.indexUuidOf(indexName);
|
|
8865
|
-
if (snapshot === null || typeof localUuid === "string" && localUuid !== snapshot.record.indexUuid) return;
|
|
8866
|
-
recordGeneration(indexName, snapshot);
|
|
8867
|
-
const { settings } = snapshot;
|
|
8868
|
-
const published = settings?.analysisGeneration ?? 0;
|
|
8869
|
-
const localPending = engine.pendingAnalysisGenerationOf(indexName);
|
|
8870
|
-
if (published > generation) {
|
|
8871
|
-
if (localPending !== published && engine.holdsAnalysisSettings(indexName, namedSettingsOfMetadata(settings))) {
|
|
8872
|
-
if (localPending !== null) await engine.dropHeldAnalysis(indexName);
|
|
8873
|
-
await engine.stampAnalysisGeneration(indexName, published);
|
|
8874
|
-
return;
|
|
8875
|
-
}
|
|
8876
|
-
catchUp(indexName, published, settings);
|
|
8877
|
-
return;
|
|
8878
|
-
}
|
|
8879
|
-
if (localPending !== null && localPending > published && settings?.pendingAnalysis?.generation !== localPending) {
|
|
8880
|
-
await engine.dropHeldAnalysis(indexName);
|
|
8881
|
-
}
|
|
8882
|
-
}
|
|
8883
|
-
function settle(indexName) {
|
|
8884
|
-
const next = (settling.get(indexName) ?? Promise.resolve()).then(() => settleOnce(indexName)).catch(onError);
|
|
8885
|
-
settling.set(indexName, next);
|
|
8886
|
-
return next;
|
|
8887
|
-
}
|
|
8888
|
-
async function reach(indexName, requested) {
|
|
8889
|
-
if (engine.heldAnalysisIsBuilt(indexName, requested)) {
|
|
8890
|
-
await engine.releaseHeldAnalysis(indexName, requested).catch(onError);
|
|
8891
|
-
return;
|
|
8892
|
-
}
|
|
8893
|
-
await settle(indexName);
|
|
8894
|
-
}
|
|
8895
|
-
async function reachThenRead(indexName, requested, generation) {
|
|
8896
|
-
if (generation < requested) await reach(indexName, requested);
|
|
8897
|
-
const reached = engine.analysisGenerationOf(indexName) ?? generation;
|
|
8898
|
-
if (reached !== requested) throw generationMissing(indexName, requested, reached);
|
|
8899
|
-
}
|
|
8900
|
-
return {
|
|
8901
|
-
held(indexName) {
|
|
8902
|
-
return held.get(indexName)?.generation;
|
|
8903
|
-
},
|
|
8904
|
-
async generationFor(indexName) {
|
|
8905
|
-
return held.get(indexName)?.generation ?? refresh(indexName);
|
|
8906
|
-
},
|
|
8907
|
-
refresh,
|
|
8908
|
-
answerAt(indexName, requested, read) {
|
|
8909
|
-
const generation = engine.analysisGenerationOf(indexName);
|
|
8910
|
-
if (generation === void 0) return read();
|
|
8911
|
-
const readAtRequested = (answerOf3) => answerOf3().then((answer2) => {
|
|
8912
|
-
const after = engine.analysisGenerationOf(indexName) ?? requested;
|
|
8913
|
-
if (after !== requested) throw generationMissing(indexName, requested, after);
|
|
8914
|
-
return answer2;
|
|
8915
|
-
});
|
|
8916
|
-
if (generation === requested) return readAtRequested(read);
|
|
8917
|
-
return reachThenRead(indexName, requested, generation).then(() => readAtRequested(read));
|
|
8918
|
-
},
|
|
8919
|
-
settle,
|
|
8920
|
-
async start() {
|
|
8921
|
-
if (unwatch !== null) return;
|
|
8922
|
-
stopped = false;
|
|
8923
|
-
unwatch = await coordinator.watchIndexMetadata((event) => {
|
|
8924
|
-
if (event.metadata === null) {
|
|
8925
|
-
held.delete(event.indexName);
|
|
8926
|
-
return;
|
|
8927
|
-
}
|
|
8928
|
-
try {
|
|
8929
|
-
recordGeneration(event.indexName, indexMetadataSnapshotOf(event.metadata, event.indexName));
|
|
8930
|
-
} catch (error) {
|
|
8931
|
-
onError(error);
|
|
8932
|
-
}
|
|
8933
|
-
void settle(event.indexName);
|
|
8934
|
-
});
|
|
8935
|
-
for (const indexName of await coordinator.listSchemas()) {
|
|
8936
|
-
if (stopped) return;
|
|
8937
|
-
if (!held.has(indexName)) await refresh(indexName).catch(onError);
|
|
8938
|
-
}
|
|
8939
|
-
},
|
|
8940
|
-
stop() {
|
|
8941
|
-
stopped = true;
|
|
8942
|
-
unwatch?.();
|
|
8943
|
-
unwatch = null;
|
|
8944
|
-
held.clear();
|
|
8945
|
-
}
|
|
8946
|
-
};
|
|
8947
|
-
}
|
|
8948
|
-
|
|
8949
|
-
// src/distribution/cluster-node/bm25-view.ts
|
|
8950
|
-
function settingOfMetadata(bytes, indexName) {
|
|
8951
|
-
const decoded = decode(bytes);
|
|
8952
|
-
if (!isRecord2(decoded) || typeof decoded.indexUuid !== "string") return null;
|
|
8953
|
-
const settings = decodeIndexSettings(decoded.settings, indexName);
|
|
8954
|
-
return { indexUuid: decoded.indexUuid, bm25: settings?.bm25, revision: settings?.bm25Revision };
|
|
8955
|
-
}
|
|
8956
|
-
function isLaterThan(next, held) {
|
|
8957
|
-
return (next.revision ?? 0) > (held.revision ?? 0);
|
|
8958
|
-
}
|
|
8959
|
-
function createBm25View(deps) {
|
|
8960
|
-
const { coordinator, engine, onError } = deps;
|
|
8961
|
-
const held = /* @__PURE__ */ new Map();
|
|
8962
|
-
let unwatch = null;
|
|
8963
|
-
function applyToLocalCopy(indexName, indexUuid) {
|
|
8964
|
-
if (engine.indexUuidOf(indexName) !== indexUuid) return Promise.resolve();
|
|
8965
|
-
return engine.applyStoredBm25(indexName, () => {
|
|
8966
|
-
const latest = held.get(indexName);
|
|
8967
|
-
if (latest === void 0 || latest.indexUuid !== indexUuid) return null;
|
|
8968
|
-
return engine.indexUuidOf(indexName) === indexUuid ? { bm25: latest.bm25 } : null;
|
|
8969
|
-
}).then(
|
|
8970
|
-
() => void 0,
|
|
8971
|
-
(error) => {
|
|
8972
|
-
if (error instanceof NarsilError && error.code === ErrorCodes.INDEX_NOT_FOUND) return;
|
|
8973
|
-
onError(error);
|
|
8974
|
-
}
|
|
8975
|
-
);
|
|
8976
|
-
}
|
|
8977
|
-
function hold(indexName, next) {
|
|
8978
|
-
const current2 = held.get(indexName);
|
|
8979
|
-
if (current2 !== void 0 && current2.indexUuid === next.indexUuid && !isLaterThan(next, current2)) return false;
|
|
8980
|
-
held.set(indexName, next);
|
|
8981
|
-
return true;
|
|
8982
|
-
}
|
|
8983
|
-
function receive(indexName, metadata) {
|
|
8984
|
-
if (metadata === null) {
|
|
8985
|
-
held.delete(indexName);
|
|
8986
|
-
return;
|
|
8987
|
-
}
|
|
8988
|
-
let next;
|
|
8989
|
-
try {
|
|
8990
|
-
next = settingOfMetadata(metadata, indexName);
|
|
8991
|
-
} catch (error) {
|
|
8992
|
-
onError(error);
|
|
8993
|
-
return;
|
|
8994
|
-
}
|
|
8995
|
-
if (next !== null && hold(indexName, next)) void applyToLocalCopy(indexName, next.indexUuid);
|
|
8996
|
-
}
|
|
8997
|
-
async function current(indexName) {
|
|
8998
|
-
const known = held.get(indexName);
|
|
8999
|
-
if (known !== void 0) return known;
|
|
9000
|
-
const snapshot = await readIndexMetadataSnapshot(coordinator, indexName);
|
|
9001
|
-
if (snapshot === null) return null;
|
|
9002
|
-
const read = {
|
|
9003
|
-
indexUuid: snapshot.record.indexUuid,
|
|
9004
|
-
bm25: snapshot.settings?.bm25,
|
|
9005
|
-
revision: snapshot.settings?.bm25Revision
|
|
9006
|
-
};
|
|
9007
|
-
const racing = held.get(indexName);
|
|
9008
|
-
if (racing === void 0 || racing.indexUuid === read.indexUuid && isLaterThan(read, racing)) {
|
|
9009
|
-
held.set(indexName, read);
|
|
9010
|
-
return read;
|
|
9011
|
-
}
|
|
9012
|
-
return racing;
|
|
9013
|
-
}
|
|
9014
|
-
return {
|
|
9015
|
-
held: (indexName) => held.get(indexName),
|
|
9016
|
-
current,
|
|
9017
|
-
async start() {
|
|
9018
|
-
if (unwatch !== null) return;
|
|
9019
|
-
unwatch = await coordinator.watchIndexMetadata((event) => receive(event.indexName, event.metadata));
|
|
9020
|
-
for (const indexName of await coordinator.listSchemas()) {
|
|
9021
|
-
if (unwatch === null) return;
|
|
9022
|
-
await current(indexName).catch(onError);
|
|
9023
|
-
}
|
|
9024
|
-
},
|
|
9025
|
-
stop() {
|
|
9026
|
-
unwatch?.();
|
|
9027
|
-
unwatch = null;
|
|
9028
|
-
held.clear();
|
|
9029
|
-
},
|
|
9030
|
-
async applyWritten(indexName, written) {
|
|
9031
|
-
hold(indexName, written);
|
|
9032
|
-
await applyToLocalCopy(indexName, written.indexUuid);
|
|
9033
|
-
},
|
|
9034
|
-
applyToLocalCopy
|
|
9035
|
-
};
|
|
9036
|
-
}
|
|
9037
|
-
|
|
9038
8836
|
// src/distribution/cluster-node/schema-diff.ts
|
|
9039
8837
|
var SCHEMA_DIFF_MAX_DEPTH = MAX_NESTING_DEPTH;
|
|
9040
8838
|
var DEPTH_EXCEEDED_SENTINEL = "depth-exceeded";
|
|
@@ -16159,7 +15957,7 @@ async function openOnce(core, indexName, partitionId) {
|
|
|
16159
15957
|
if (directory === void 0) throw durabilityRequired(indexName, partitionId);
|
|
16160
15958
|
if ((checkpoint.manifest.analysisGeneration ?? 0) !== entry.analysisGeneration) return null;
|
|
16161
15959
|
const config = await encodeIndexConfiguration(entry);
|
|
16162
|
-
const { openPagedPartitionArchive } = await import('./paged-partition-archive-
|
|
15960
|
+
const { openPagedPartitionArchive } = await import('./paged-partition-archive-XCETVLZW.mjs');
|
|
16163
15961
|
const archive = await openPagedPartitionArchive(checkpoint, directory, indexName, partitionId, manager);
|
|
16164
15962
|
handedOver = true;
|
|
16165
15963
|
try {
|
|
@@ -16874,7 +16672,7 @@ async function archiveReplicationPartition(core, indexName, partitionId, lastSeq
|
|
|
16874
16672
|
);
|
|
16875
16673
|
}
|
|
16876
16674
|
try {
|
|
16877
|
-
const { openPagedPartitionArchive } = await import('./paged-partition-archive-
|
|
16675
|
+
const { openPagedPartitionArchive } = await import('./paged-partition-archive-XCETVLZW.mjs');
|
|
16878
16676
|
const archive = await openPagedPartitionArchive(checkpoint2, directory, indexName, partitionId, manager);
|
|
16879
16677
|
return {
|
|
16880
16678
|
stream: archiveStream({ ...archive, config, sourceIndexName: indexName }),
|
|
@@ -17089,7 +16887,7 @@ async function restoreReplicationPartition(core, engine, indexName, partitionId,
|
|
|
17089
16887
|
if (directory !== null) {
|
|
17090
16888
|
const durable = core.durability?.manager;
|
|
17091
16889
|
await (durable?.checkpointFromMemory?.(indexName) ?? durable?.checkpoint(indexName));
|
|
17092
|
-
const { removeReceivedFilesNothingReads } = await import('./paged-partition-archive-
|
|
16890
|
+
const { removeReceivedFilesNothingReads } = await import('./paged-partition-archive-XCETVLZW.mjs');
|
|
17093
16891
|
await removeReceivedFilesNothingReads(
|
|
17094
16892
|
directory,
|
|
17095
16893
|
core.requireManager(indexName),
|
|
@@ -20133,7 +19931,7 @@ async function readPinPresence(deps, indexName, params, allocation) {
|
|
|
20133
19931
|
if (params.pinned === void 0 || params.searchAfter !== void 0) return null;
|
|
20134
19932
|
const pinnedIds = new Set(params.pinned.map((entry) => entry.docId));
|
|
20135
19933
|
const { verifiable, unverifiable } = splitPinnedByReachability(pinnedIds, allocation);
|
|
20136
|
-
const documents = verifiable.length === 0 ? /* @__PURE__ */ new Map() : await readDistributedDocuments(deps.config, deps.nodeId, deps.engine, indexName, verifiable, allocation);
|
|
19934
|
+
const documents = verifiable.length === 0 ? /* @__PURE__ */ new Map() : await readDistributedDocuments(deps.config, deps.nodeId, deps.engine, indexName, verifiable, allocation, false);
|
|
20137
19935
|
return { pinnedIds, stored: new Set(documents.keys()), unverifiable };
|
|
20138
19936
|
}
|
|
20139
19937
|
function dropUnstoredPinnedEntries(distributed, presence) {
|
|
@@ -20163,8 +19961,9 @@ function releaseContextsOfDepartedNodes(coordinator, registry) {
|
|
|
20163
19961
|
}
|
|
20164
19962
|
|
|
20165
19963
|
// src/distribution/cluster-node/search-contexts/documents.ts
|
|
20166
|
-
function documentOfSnapshot(snapshot) {
|
|
19964
|
+
function documentOfSnapshot(snapshot, withVectors = true) {
|
|
20167
19965
|
const document = structuredClone(snapshot.fields);
|
|
19966
|
+
if (!withVectors) return document;
|
|
20168
19967
|
for (const [fieldPath2, vector] of snapshot.vectors) setNestedValue(document, fieldPath2, new Float32Array(vector));
|
|
20169
19968
|
return document;
|
|
20170
19969
|
}
|
|
@@ -20281,7 +20080,7 @@ function createRankedDocumentReader(deps, indexName, allocation) {
|
|
|
20281
20080
|
resolveNodeTargets: deps.resolveNodeTargets
|
|
20282
20081
|
});
|
|
20283
20082
|
}
|
|
20284
|
-
function readOwnContext(context, docIds, last) {
|
|
20083
|
+
function readOwnContext(context, docIds, last, vectors) {
|
|
20285
20084
|
const registry = deps.searchContexts;
|
|
20286
20085
|
const found = registry?.read(context.contextId, docIds) ?? null;
|
|
20287
20086
|
if (last) {
|
|
@@ -20290,10 +20089,10 @@ function createRankedDocumentReader(deps, indexName, allocation) {
|
|
|
20290
20089
|
}
|
|
20291
20090
|
if (found === null) throw contextLost(context, indexName);
|
|
20292
20091
|
const documents = /* @__PURE__ */ new Map();
|
|
20293
|
-
for (const [docId, snapshot] of found) documents.set(docId, documentOfSnapshot(snapshot));
|
|
20092
|
+
for (const [docId, snapshot] of found) documents.set(docId, documentOfSnapshot(snapshot, vectors));
|
|
20294
20093
|
return documents;
|
|
20295
20094
|
}
|
|
20296
|
-
async function readRemoteContext(context, docIds, last, fields) {
|
|
20095
|
+
async function readRemoteContext(context, docIds, last, fields, vectors) {
|
|
20297
20096
|
const documents = /* @__PURE__ */ new Map();
|
|
20298
20097
|
for (let start = 0; start < docIds.length; start += FETCH_GROUP_DOCUMENT_IDS) {
|
|
20299
20098
|
const chunk = docIds.slice(start, start + FETCH_GROUP_DOCUMENT_IDS);
|
|
@@ -20303,6 +20102,7 @@ function createRankedDocumentReader(deps, indexName, allocation) {
|
|
|
20303
20102
|
indexName,
|
|
20304
20103
|
documentIds: chunk.map((docId) => ({ docId, partitionId: resolvePartitionId(docId, partitionCount) })),
|
|
20305
20104
|
fields,
|
|
20105
|
+
...vectors ? {} : { vectors: false },
|
|
20306
20106
|
highlight: null,
|
|
20307
20107
|
contextId: context.contextId,
|
|
20308
20108
|
releaseContext
|
|
@@ -20356,7 +20156,7 @@ function createRankedDocumentReader(deps, indexName, allocation) {
|
|
|
20356
20156
|
const reads = [];
|
|
20357
20157
|
for (const { context, docIds: contextIds } of byContext.values()) {
|
|
20358
20158
|
reads.push(
|
|
20359
|
-
context.nodeId === deps.nodeId && deps.searchContexts !== void 0 ? Promise.resolve().then(() => readOwnContext(context, contextIds, options.last)) : readRemoteContext(context, contextIds, options.last, options.fields)
|
|
20159
|
+
context.nodeId === deps.nodeId && deps.searchContexts !== void 0 ? Promise.resolve().then(() => readOwnContext(context, contextIds, options.last, options.vectors)) : readRemoteContext(context, contextIds, options.last, options.fields, options.vectors)
|
|
20360
20160
|
);
|
|
20361
20161
|
}
|
|
20362
20162
|
if (unsourced.length > 0) {
|
|
@@ -20368,7 +20168,7 @@ function createRankedDocumentReader(deps, indexName, allocation) {
|
|
|
20368
20168
|
indexName,
|
|
20369
20169
|
unsourced,
|
|
20370
20170
|
allocation,
|
|
20371
|
-
options.fields,
|
|
20171
|
+
{ fields: options.fields, vectors: options.vectors },
|
|
20372
20172
|
options.requireEveryPartition
|
|
20373
20173
|
)
|
|
20374
20174
|
);
|
|
@@ -20662,6 +20462,7 @@ function readSortValues(document, fields, modes) {
|
|
|
20662
20462
|
}
|
|
20663
20463
|
|
|
20664
20464
|
// src/distribution/cluster-node/reads/local-list-page.ts
|
|
20465
|
+
var VECTORS_CHOSEN_BY_THE_LIST = { schema: {} };
|
|
20665
20466
|
function toWireSortValue(value) {
|
|
20666
20467
|
if (typeof value === "string") return value;
|
|
20667
20468
|
if (typeof value === "number") return Number.isFinite(value) ? value : null;
|
|
@@ -20679,13 +20480,13 @@ async function listLocalPage(engine, payload) {
|
|
|
20679
20480
|
const sortFieldNames = normalizedSort === null ? null : normalizedSort.map((entry) => entry.field);
|
|
20680
20481
|
const sortModes = normalizedSort === null ? [] : sortModesOf(normalizedSort);
|
|
20681
20482
|
const listedFields = fieldsToList(payload.fields, sortFieldNames);
|
|
20682
|
-
const answerProjection = listedFields === payload.fields || payload.fields === null ? null : resolveProjection({ include: payload.fields });
|
|
20483
|
+
const answerProjection = listedFields === payload.fields || payload.fields === null ? null : resolveProjection({ include: payload.fields }, VECTORS_CHOSEN_BY_THE_LIST);
|
|
20683
20484
|
const listParams = {
|
|
20684
20485
|
cursor: payload.cursor ?? void 0,
|
|
20685
20486
|
limit: payload.limit,
|
|
20686
20487
|
filters: payload.filters ?? void 0,
|
|
20687
20488
|
sort: payload.sort ?? void 0,
|
|
20688
|
-
document: listedFields === null ?
|
|
20489
|
+
document: { ...listedFields === null ? {} : { include: listedFields }, vectors: payload.vectors !== false }
|
|
20689
20490
|
};
|
|
20690
20491
|
const result = await engine.listPartitions(payload.indexName, listParams, payload.partitionIds);
|
|
20691
20492
|
const entries = result.documents.map((listed) => {
|
|
@@ -20733,7 +20534,8 @@ async function listCluster(deps, indexName, params) {
|
|
|
20733
20534
|
const normalizedSort = normalizeSort(params.sort);
|
|
20734
20535
|
const sortFieldNames = normalizedSort.length === 0 ? null : normalizedSort.map((entry) => entry.field);
|
|
20735
20536
|
const directions = normalizedSort.map((entry) => entry.direction);
|
|
20736
|
-
const
|
|
20537
|
+
const settings = await deps.vectorReads.settingsOf(indexName, allocation);
|
|
20538
|
+
const projection = resolveProjection(params.document, settings);
|
|
20737
20539
|
const wireSort = sortFieldNames === null ? null : normalizedSort;
|
|
20738
20540
|
const payload = {
|
|
20739
20541
|
indexName,
|
|
@@ -20742,7 +20544,8 @@ async function listCluster(deps, indexName, params) {
|
|
|
20742
20544
|
limit,
|
|
20743
20545
|
filters: params.filters ?? null,
|
|
20744
20546
|
sort: wireSort,
|
|
20745
|
-
fields: wireFieldsFor(projection, sortFieldNames ?? [])
|
|
20547
|
+
fields: wireFieldsFor(projection, sortFieldNames ?? []),
|
|
20548
|
+
...keepsAnyVectorField(projection, settings) ? {} : { vectors: false }
|
|
20746
20549
|
};
|
|
20747
20550
|
const gathered = await gatherFromEveryPartition(allocation, indexName, (group) => {
|
|
20748
20551
|
if (group.nodeId === deps.nodeId) {
|
|
@@ -20900,12 +20703,27 @@ var clusterRerankHost = {
|
|
|
20900
20703
|
getNestedValue,
|
|
20901
20704
|
applyProjection
|
|
20902
20705
|
};
|
|
20903
|
-
async function readClusterDocuments(deps, indexName, docIds) {
|
|
20706
|
+
async function readClusterDocuments(deps, indexName, docIds, options) {
|
|
20904
20707
|
const allocation = await routableAllocation(deps.config.coordinator, indexName);
|
|
20905
20708
|
if (allocation === null) {
|
|
20906
|
-
return deps.engine.getMultiple(indexName, docIds);
|
|
20907
|
-
}
|
|
20908
|
-
|
|
20709
|
+
return deps.engine.getMultiple(indexName, docIds, options);
|
|
20710
|
+
}
|
|
20711
|
+
const settings = await deps.vectorReads.settingsOf(indexName, allocation);
|
|
20712
|
+
const projection = resolveProjection(options?.document, settings);
|
|
20713
|
+
const found = await readDocumentsFromCopies(
|
|
20714
|
+
deps.config,
|
|
20715
|
+
deps.nodeId,
|
|
20716
|
+
deps.engine,
|
|
20717
|
+
indexName,
|
|
20718
|
+
docIds,
|
|
20719
|
+
allocation,
|
|
20720
|
+
{ fields: fetchFieldsFor(projection), vectors: keepsAnyVectorField(projection, settings) },
|
|
20721
|
+
true
|
|
20722
|
+
);
|
|
20723
|
+
if (projection.kind === "full") return found;
|
|
20724
|
+
const documents = /* @__PURE__ */ new Map();
|
|
20725
|
+
for (const [docId, document] of found) documents.set(docId, applyProjection(document, projection));
|
|
20726
|
+
return documents;
|
|
20909
20727
|
}
|
|
20910
20728
|
function routingConfig(query, searchContexts) {
|
|
20911
20729
|
const config = { searchContexts };
|
|
@@ -20946,33 +20764,37 @@ async function queryCluster(deps, indexName, params, settingName) {
|
|
|
20946
20764
|
deps.analysisView,
|
|
20947
20765
|
indexName,
|
|
20948
20766
|
() => rerunOnLostContext(async () => {
|
|
20767
|
+
const settings = await deps.vectorReads.settingsOf(indexName, allocation);
|
|
20768
|
+
const search = { indexName, settingName, allocation, settings };
|
|
20949
20769
|
if (params.rerank === void 0) {
|
|
20950
20770
|
return withRankedReader(
|
|
20951
20771
|
deps,
|
|
20952
20772
|
indexName,
|
|
20953
20773
|
allocation,
|
|
20954
|
-
(reader) => searchCluster(deps,
|
|
20774
|
+
(reader) => searchCluster(deps, search, params, reader, false)
|
|
20955
20775
|
);
|
|
20956
20776
|
}
|
|
20957
|
-
const { rerankClusterQuery } = await import('./rerank-
|
|
20777
|
+
const { rerankClusterQuery } = await import('./rerank-OJLSVUYV.mjs');
|
|
20958
20778
|
return withRankedReader(
|
|
20959
20779
|
deps,
|
|
20960
20780
|
indexName,
|
|
20961
20781
|
allocation,
|
|
20962
20782
|
(reader) => rerankClusterQuery(
|
|
20963
20783
|
deps,
|
|
20964
|
-
indexName,
|
|
20965
20784
|
params,
|
|
20966
|
-
|
|
20967
|
-
(
|
|
20785
|
+
settings,
|
|
20786
|
+
(firstStage) => searchCluster(deps, search, firstStage, reader, true),
|
|
20787
|
+
(docIds, read) => reader.read(docIds, { ...read, fields: null, requireEveryPartition: true }),
|
|
20968
20788
|
clusterRerankHost
|
|
20969
20789
|
)
|
|
20970
20790
|
);
|
|
20971
20791
|
})
|
|
20972
20792
|
);
|
|
20973
20793
|
}
|
|
20974
|
-
async function searchCluster(deps,
|
|
20975
|
-
const
|
|
20794
|
+
async function searchCluster(deps, search, params, reader, readsLater) {
|
|
20795
|
+
const { indexName, settingName, allocation, settings } = search;
|
|
20796
|
+
const projection = resolveProjection(params.document, settings);
|
|
20797
|
+
const vectors = keepsAnyVectorField(projection, settings);
|
|
20976
20798
|
const groupsRead = params.group !== void 0 && (params.group.reduce !== void 0 || projection.kind !== "none");
|
|
20977
20799
|
const readsDocuments = readsLater || projection.kind !== "none" || groupsRead;
|
|
20978
20800
|
const { wireParams, carriesBm25 } = heldBm25RoutingFor(deps, settingName, params) ?? await bm25RoutingFor(deps, settingName, params);
|
|
@@ -20998,7 +20820,7 @@ async function searchCluster(deps, indexName, settingName, params, allocation, r
|
|
|
20998
20820
|
if (projection.kind !== "none") {
|
|
20999
20821
|
const found = await reader.read(
|
|
21000
20822
|
distributed.scored.map((entry) => entry.docId),
|
|
21001
|
-
{ last: !groupsRead && !readsLater, fields: fetchFieldsFor(projection), requireEveryPartition: false }
|
|
20823
|
+
{ last: !groupsRead && !readsLater, fields: fetchFieldsFor(projection), vectors, requireEveryPartition: false }
|
|
21002
20824
|
);
|
|
21003
20825
|
for (const [docId, document] of found) documents.set(docId, applyProjection(document, projection));
|
|
21004
20826
|
}
|
|
@@ -21021,7 +20843,12 @@ async function searchCluster(deps, indexName, settingName, params, allocation, r
|
|
|
21021
20843
|
params,
|
|
21022
20844
|
distributed,
|
|
21023
20845
|
projection,
|
|
21024
|
-
(docIds, last) => reader.read(docIds, {
|
|
20846
|
+
(docIds, last) => reader.read(docIds, {
|
|
20847
|
+
last: last && !readsLater,
|
|
20848
|
+
fields: null,
|
|
20849
|
+
vectors: vectors || params.group?.reduce !== void 0,
|
|
20850
|
+
requireEveryPartition: true
|
|
20851
|
+
})
|
|
21025
20852
|
);
|
|
21026
20853
|
if (groups !== void 0) {
|
|
21027
20854
|
result.groups = groups;
|
|
@@ -21327,6 +21154,7 @@ async function dropServedIndex(name, coordinator, engine) {
|
|
|
21327
21154
|
}
|
|
21328
21155
|
|
|
21329
21156
|
// src/distribution/cluster-node/node-operations.ts
|
|
21157
|
+
var DOCUMENT_PRESENCE = { document: false };
|
|
21330
21158
|
function createClusterNodeOperations(deps) {
|
|
21331
21159
|
const { trackOp, readDeps, writeDeps, engine, coordinator } = deps;
|
|
21332
21160
|
function served(indexName, fn) {
|
|
@@ -21467,14 +21295,17 @@ function createClusterNodeOperations(deps) {
|
|
|
21467
21295
|
)
|
|
21468
21296
|
);
|
|
21469
21297
|
},
|
|
21470
|
-
async get(indexName, docId) {
|
|
21471
|
-
return read(indexName, async (name) => (await readClusterDocuments(readDeps, name, [docId])).get(docId));
|
|
21298
|
+
async get(indexName, docId, options) {
|
|
21299
|
+
return read(indexName, async (name) => (await readClusterDocuments(readDeps, name, [docId], options)).get(docId));
|
|
21472
21300
|
},
|
|
21473
|
-
async getMultiple(indexName, docIds) {
|
|
21474
|
-
return read(indexName, (name) => readClusterDocuments(readDeps, name, docIds));
|
|
21301
|
+
async getMultiple(indexName, docIds, options) {
|
|
21302
|
+
return read(indexName, (name) => readClusterDocuments(readDeps, name, docIds, options));
|
|
21475
21303
|
},
|
|
21476
21304
|
async has(indexName, docId) {
|
|
21477
|
-
return read(
|
|
21305
|
+
return read(
|
|
21306
|
+
indexName,
|
|
21307
|
+
async (name) => (await readClusterDocuments(readDeps, name, [docId], DOCUMENT_PRESENCE)).has(docId)
|
|
21308
|
+
);
|
|
21478
21309
|
}
|
|
21479
21310
|
};
|
|
21480
21311
|
}
|
|
@@ -21837,7 +21668,7 @@ function createNodeWiring(input) {
|
|
|
21837
21668
|
])
|
|
21838
21669
|
)
|
|
21839
21670
|
};
|
|
21840
|
-
return readDistributedDocuments(routingConfig2, nodeId, engine, indexName, docIds, primariesOnly);
|
|
21671
|
+
return readDistributedDocuments(routingConfig2, nodeId, engine, indexName, docIds, primariesOnly, true);
|
|
21841
21672
|
}
|
|
21842
21673
|
const registration = {
|
|
21843
21674
|
nodeId,
|
|
@@ -22286,7 +22117,7 @@ function readSnapshotPage(deps, job, cursor, onFirstPage) {
|
|
|
22286
22117
|
[job.partitionId]
|
|
22287
22118
|
);
|
|
22288
22119
|
const ids = listed.documents.map((document) => document.id);
|
|
22289
|
-
const stored = await engine.getMultiple(job.sourceGeneration, ids);
|
|
22120
|
+
const stored = await engine.getMultiple(job.sourceGeneration, ids, { document: { vectors: true } });
|
|
22290
22121
|
const operations = [];
|
|
22291
22122
|
for (const id of ids) {
|
|
22292
22123
|
const document = stored.get(id);
|
|
@@ -22782,11 +22613,12 @@ async function convertLocalGroupsToWire(groups, sortFields, sortModes, deps, ind
|
|
|
22782
22613
|
}
|
|
22783
22614
|
return wireGroups;
|
|
22784
22615
|
}
|
|
22616
|
+
var VECTORS_CHOSEN_BY_THE_FETCH = { schema: {} };
|
|
22785
22617
|
async function handleFetch(message, respond, deps) {
|
|
22786
22618
|
const payload = validateFetchPayload(decodeRequestPayload(message.payload, message.type));
|
|
22787
|
-
const
|
|
22619
|
+
const read = { fields: payload.fields, vectors: payload.vectors !== false };
|
|
22788
22620
|
const contextId = payload.contextId ?? null;
|
|
22789
|
-
const documents = contextId === null ? await readCurrentDocuments(deps, payload.indexName, payload.documentIds,
|
|
22621
|
+
const documents = contextId === null ? await readCurrentDocuments(deps, payload.indexName, payload.documentIds, read) : readContextDocuments(deps, contextId, payload.documentIds, payload.releaseContext !== false, read);
|
|
22790
22622
|
refuseEncodeWithoutRoom(documents.map((fetched) => fetched.document));
|
|
22791
22623
|
const resultPayload = { documents };
|
|
22792
22624
|
await respond({
|
|
@@ -22796,21 +22628,24 @@ async function handleFetch(message, respond, deps) {
|
|
|
22796
22628
|
payload: encodeDocumentReply(resultPayload)
|
|
22797
22629
|
});
|
|
22798
22630
|
}
|
|
22799
|
-
async function readCurrentDocuments(deps, indexName, documentIds,
|
|
22631
|
+
async function readCurrentDocuments(deps, indexName, documentIds, read) {
|
|
22800
22632
|
const documents = [];
|
|
22633
|
+
const options = {
|
|
22634
|
+
document: { ...read.fields === null ? {} : { include: read.fields }, vectors: read.vectors }
|
|
22635
|
+
};
|
|
22801
22636
|
for (const docRef of documentIds) {
|
|
22802
|
-
const doc = await deps.engine.get(indexName, docRef.docId);
|
|
22637
|
+
const doc = await deps.engine.get(indexName, docRef.docId, options);
|
|
22803
22638
|
if (doc !== void 0) {
|
|
22804
|
-
documents.push({
|
|
22805
|
-
docId: docRef.docId,
|
|
22806
|
-
document: applyProjection(doc, projection),
|
|
22807
|
-
highlights: null
|
|
22808
|
-
});
|
|
22639
|
+
documents.push({ docId: docRef.docId, document: doc, highlights: null });
|
|
22809
22640
|
}
|
|
22810
22641
|
}
|
|
22811
22642
|
return documents;
|
|
22812
22643
|
}
|
|
22813
|
-
function readContextDocuments(deps, contextId, documentIds, releaseContext,
|
|
22644
|
+
function readContextDocuments(deps, contextId, documentIds, releaseContext, read) {
|
|
22645
|
+
const projection = resolveProjection(
|
|
22646
|
+
read.fields === null ? void 0 : { include: read.fields },
|
|
22647
|
+
VECTORS_CHOSEN_BY_THE_FETCH
|
|
22648
|
+
);
|
|
22814
22649
|
const docIds = documentIds.map((docRef) => docRef.docId);
|
|
22815
22650
|
const found = deps.searchContexts?.read(contextId, docIds) ?? null;
|
|
22816
22651
|
if (releaseContext) deps.searchContexts?.release(contextId);
|
|
@@ -22828,7 +22663,7 @@ function readContextDocuments(deps, contextId, documentIds, releaseContext, proj
|
|
|
22828
22663
|
if (snapshot === void 0) continue;
|
|
22829
22664
|
documents.push({
|
|
22830
22665
|
docId,
|
|
22831
|
-
document: applyProjection(documentOfSnapshot(snapshot), projection),
|
|
22666
|
+
document: applyProjection(documentOfSnapshot(snapshot, read.vectors), projection),
|
|
22832
22667
|
highlights: null
|
|
22833
22668
|
});
|
|
22834
22669
|
}
|
|
@@ -24701,6 +24536,249 @@ function createClusterQuerySuggestions(deps) {
|
|
|
24701
24536
|
};
|
|
24702
24537
|
}
|
|
24703
24538
|
|
|
24539
|
+
// src/distribution/cluster-node/analysis-change/view.ts
|
|
24540
|
+
function namedSettingsOfMetadata(settings) {
|
|
24541
|
+
return {
|
|
24542
|
+
language: settings?.language ?? "english",
|
|
24543
|
+
...settings?.tokenizer !== void 0 ? { tokenizer: settings.tokenizer } : {},
|
|
24544
|
+
...settings?.stopWords !== void 0 ? { stopWords: settings.stopWords } : {},
|
|
24545
|
+
...settings?.stopWordList !== void 0 ? { stopWordList: settings.stopWordList } : {}
|
|
24546
|
+
};
|
|
24547
|
+
}
|
|
24548
|
+
function generationMissing(indexName, requestedGeneration, heldGeneration) {
|
|
24549
|
+
return new NarsilError(
|
|
24550
|
+
ErrorCodes.ANALYSIS_GENERATION_MISSING,
|
|
24551
|
+
`This node holds analysis generation ${heldGeneration} of index '${indexName}', so it cannot answer for generation ${requestedGeneration}`,
|
|
24552
|
+
{ indexName, requestedGeneration, heldGeneration }
|
|
24553
|
+
);
|
|
24554
|
+
}
|
|
24555
|
+
function createAnalysisView(deps) {
|
|
24556
|
+
const { coordinator, engine, onError } = deps;
|
|
24557
|
+
const held = /* @__PURE__ */ new Map();
|
|
24558
|
+
const settling = /* @__PURE__ */ new Map();
|
|
24559
|
+
let unwatch = null;
|
|
24560
|
+
let stopped = false;
|
|
24561
|
+
function recordGeneration(indexName, snapshot) {
|
|
24562
|
+
const generation = snapshot.settings?.analysisGeneration ?? 0;
|
|
24563
|
+
const { indexUuid } = snapshot.record;
|
|
24564
|
+
const known = held.get(indexName);
|
|
24565
|
+
if (known?.indexUuid === indexUuid && known.generation >= generation) return known.generation;
|
|
24566
|
+
held.set(indexName, { indexUuid, generation });
|
|
24567
|
+
return generation;
|
|
24568
|
+
}
|
|
24569
|
+
async function refresh(indexName) {
|
|
24570
|
+
const snapshot = await readIndexMetadataSnapshot(coordinator, indexName);
|
|
24571
|
+
if (snapshot === null) {
|
|
24572
|
+
held.delete(indexName);
|
|
24573
|
+
return 0;
|
|
24574
|
+
}
|
|
24575
|
+
return recordGeneration(indexName, snapshot);
|
|
24576
|
+
}
|
|
24577
|
+
function catchUp(indexName, generation, settings) {
|
|
24578
|
+
engine.buildHeldAnalysis(indexName, { generation, ...namedSettingsOfMetadata(settings) }).then(() => engine.releaseHeldAnalysis(indexName, generation)).catch((error) => {
|
|
24579
|
+
if (!stopped) onError(error);
|
|
24580
|
+
});
|
|
24581
|
+
}
|
|
24582
|
+
async function settleOnce(indexName) {
|
|
24583
|
+
const generation = engine.analysisGenerationOf(indexName);
|
|
24584
|
+
if (stopped || generation === void 0) return;
|
|
24585
|
+
const snapshot = await readIndexMetadataSnapshot(coordinator, indexName);
|
|
24586
|
+
const localUuid = engine.indexUuidOf(indexName);
|
|
24587
|
+
if (snapshot === null || typeof localUuid === "string" && localUuid !== snapshot.record.indexUuid) return;
|
|
24588
|
+
recordGeneration(indexName, snapshot);
|
|
24589
|
+
const { settings } = snapshot;
|
|
24590
|
+
const published = settings?.analysisGeneration ?? 0;
|
|
24591
|
+
const localPending = engine.pendingAnalysisGenerationOf(indexName);
|
|
24592
|
+
if (published > generation) {
|
|
24593
|
+
if (localPending !== published && engine.holdsAnalysisSettings(indexName, namedSettingsOfMetadata(settings))) {
|
|
24594
|
+
if (localPending !== null) await engine.dropHeldAnalysis(indexName);
|
|
24595
|
+
await engine.stampAnalysisGeneration(indexName, published);
|
|
24596
|
+
return;
|
|
24597
|
+
}
|
|
24598
|
+
catchUp(indexName, published, settings);
|
|
24599
|
+
return;
|
|
24600
|
+
}
|
|
24601
|
+
if (localPending !== null && localPending > published && settings?.pendingAnalysis?.generation !== localPending) {
|
|
24602
|
+
await engine.dropHeldAnalysis(indexName);
|
|
24603
|
+
}
|
|
24604
|
+
}
|
|
24605
|
+
function settle(indexName) {
|
|
24606
|
+
const next = (settling.get(indexName) ?? Promise.resolve()).then(() => settleOnce(indexName)).catch(onError);
|
|
24607
|
+
settling.set(indexName, next);
|
|
24608
|
+
return next;
|
|
24609
|
+
}
|
|
24610
|
+
async function reach(indexName, requested) {
|
|
24611
|
+
if (engine.heldAnalysisIsBuilt(indexName, requested)) {
|
|
24612
|
+
await engine.releaseHeldAnalysis(indexName, requested).catch(onError);
|
|
24613
|
+
return;
|
|
24614
|
+
}
|
|
24615
|
+
await settle(indexName);
|
|
24616
|
+
}
|
|
24617
|
+
async function reachThenRead(indexName, requested, generation) {
|
|
24618
|
+
if (generation < requested) await reach(indexName, requested);
|
|
24619
|
+
const reached = engine.analysisGenerationOf(indexName) ?? generation;
|
|
24620
|
+
if (reached !== requested) throw generationMissing(indexName, requested, reached);
|
|
24621
|
+
}
|
|
24622
|
+
return {
|
|
24623
|
+
held(indexName) {
|
|
24624
|
+
return held.get(indexName)?.generation;
|
|
24625
|
+
},
|
|
24626
|
+
async generationFor(indexName) {
|
|
24627
|
+
return held.get(indexName)?.generation ?? refresh(indexName);
|
|
24628
|
+
},
|
|
24629
|
+
refresh,
|
|
24630
|
+
answerAt(indexName, requested, read) {
|
|
24631
|
+
const generation = engine.analysisGenerationOf(indexName);
|
|
24632
|
+
if (generation === void 0) return read();
|
|
24633
|
+
const readAtRequested = (answerOf3) => answerOf3().then((answer2) => {
|
|
24634
|
+
const after = engine.analysisGenerationOf(indexName) ?? requested;
|
|
24635
|
+
if (after !== requested) throw generationMissing(indexName, requested, after);
|
|
24636
|
+
return answer2;
|
|
24637
|
+
});
|
|
24638
|
+
if (generation === requested) return readAtRequested(read);
|
|
24639
|
+
return reachThenRead(indexName, requested, generation).then(() => readAtRequested(read));
|
|
24640
|
+
},
|
|
24641
|
+
settle,
|
|
24642
|
+
async start() {
|
|
24643
|
+
if (unwatch !== null) return;
|
|
24644
|
+
stopped = false;
|
|
24645
|
+
unwatch = await coordinator.watchIndexMetadata((event) => {
|
|
24646
|
+
if (event.metadata === null) {
|
|
24647
|
+
held.delete(event.indexName);
|
|
24648
|
+
return;
|
|
24649
|
+
}
|
|
24650
|
+
try {
|
|
24651
|
+
recordGeneration(event.indexName, indexMetadataSnapshotOf(event.metadata, event.indexName));
|
|
24652
|
+
} catch (error) {
|
|
24653
|
+
onError(error);
|
|
24654
|
+
}
|
|
24655
|
+
void settle(event.indexName);
|
|
24656
|
+
});
|
|
24657
|
+
for (const indexName of await coordinator.listSchemas()) {
|
|
24658
|
+
if (stopped) return;
|
|
24659
|
+
if (!held.has(indexName)) await refresh(indexName).catch(onError);
|
|
24660
|
+
}
|
|
24661
|
+
},
|
|
24662
|
+
stop() {
|
|
24663
|
+
stopped = true;
|
|
24664
|
+
unwatch?.();
|
|
24665
|
+
unwatch = null;
|
|
24666
|
+
held.clear();
|
|
24667
|
+
}
|
|
24668
|
+
};
|
|
24669
|
+
}
|
|
24670
|
+
|
|
24671
|
+
// src/distribution/cluster-node/bm25-view.ts
|
|
24672
|
+
function settingOfMetadata(bytes, indexName) {
|
|
24673
|
+
const decoded = decode(bytes);
|
|
24674
|
+
if (!isRecord2(decoded) || typeof decoded.indexUuid !== "string") return null;
|
|
24675
|
+
const settings = decodeIndexSettings(decoded.settings, indexName);
|
|
24676
|
+
return { indexUuid: decoded.indexUuid, bm25: settings?.bm25, revision: settings?.bm25Revision };
|
|
24677
|
+
}
|
|
24678
|
+
function isLaterThan(next, held) {
|
|
24679
|
+
return (next.revision ?? 0) > (held.revision ?? 0);
|
|
24680
|
+
}
|
|
24681
|
+
function createBm25View(deps) {
|
|
24682
|
+
const { coordinator, engine, onError } = deps;
|
|
24683
|
+
const held = /* @__PURE__ */ new Map();
|
|
24684
|
+
let unwatch = null;
|
|
24685
|
+
function applyToLocalCopy(indexName, indexUuid) {
|
|
24686
|
+
if (engine.indexUuidOf(indexName) !== indexUuid) return Promise.resolve();
|
|
24687
|
+
return engine.applyStoredBm25(indexName, () => {
|
|
24688
|
+
const latest = held.get(indexName);
|
|
24689
|
+
if (latest === void 0 || latest.indexUuid !== indexUuid) return null;
|
|
24690
|
+
return engine.indexUuidOf(indexName) === indexUuid ? { bm25: latest.bm25 } : null;
|
|
24691
|
+
}).then(
|
|
24692
|
+
() => void 0,
|
|
24693
|
+
(error) => {
|
|
24694
|
+
if (error instanceof NarsilError && error.code === ErrorCodes.INDEX_NOT_FOUND) return;
|
|
24695
|
+
onError(error);
|
|
24696
|
+
}
|
|
24697
|
+
);
|
|
24698
|
+
}
|
|
24699
|
+
function hold(indexName, next) {
|
|
24700
|
+
const current2 = held.get(indexName);
|
|
24701
|
+
if (current2 !== void 0 && current2.indexUuid === next.indexUuid && !isLaterThan(next, current2)) return false;
|
|
24702
|
+
held.set(indexName, next);
|
|
24703
|
+
return true;
|
|
24704
|
+
}
|
|
24705
|
+
function receive(indexName, metadata) {
|
|
24706
|
+
if (metadata === null) {
|
|
24707
|
+
held.delete(indexName);
|
|
24708
|
+
return;
|
|
24709
|
+
}
|
|
24710
|
+
let next;
|
|
24711
|
+
try {
|
|
24712
|
+
next = settingOfMetadata(metadata, indexName);
|
|
24713
|
+
} catch (error) {
|
|
24714
|
+
onError(error);
|
|
24715
|
+
return;
|
|
24716
|
+
}
|
|
24717
|
+
if (next !== null && hold(indexName, next)) void applyToLocalCopy(indexName, next.indexUuid);
|
|
24718
|
+
}
|
|
24719
|
+
async function current(indexName) {
|
|
24720
|
+
const known = held.get(indexName);
|
|
24721
|
+
if (known !== void 0) return known;
|
|
24722
|
+
const snapshot = await readIndexMetadataSnapshot(coordinator, indexName);
|
|
24723
|
+
if (snapshot === null) return null;
|
|
24724
|
+
const read = {
|
|
24725
|
+
indexUuid: snapshot.record.indexUuid,
|
|
24726
|
+
bm25: snapshot.settings?.bm25,
|
|
24727
|
+
revision: snapshot.settings?.bm25Revision
|
|
24728
|
+
};
|
|
24729
|
+
const racing = held.get(indexName);
|
|
24730
|
+
if (racing === void 0 || racing.indexUuid === read.indexUuid && isLaterThan(read, racing)) {
|
|
24731
|
+
held.set(indexName, read);
|
|
24732
|
+
return read;
|
|
24733
|
+
}
|
|
24734
|
+
return racing;
|
|
24735
|
+
}
|
|
24736
|
+
return {
|
|
24737
|
+
held: (indexName) => held.get(indexName),
|
|
24738
|
+
current,
|
|
24739
|
+
async start() {
|
|
24740
|
+
if (unwatch !== null) return;
|
|
24741
|
+
unwatch = await coordinator.watchIndexMetadata((event) => receive(event.indexName, event.metadata));
|
|
24742
|
+
for (const indexName of await coordinator.listSchemas()) {
|
|
24743
|
+
if (unwatch === null) return;
|
|
24744
|
+
await current(indexName).catch(onError);
|
|
24745
|
+
}
|
|
24746
|
+
},
|
|
24747
|
+
stop() {
|
|
24748
|
+
unwatch?.();
|
|
24749
|
+
unwatch = null;
|
|
24750
|
+
held.clear();
|
|
24751
|
+
},
|
|
24752
|
+
async applyWritten(indexName, written) {
|
|
24753
|
+
hold(indexName, written);
|
|
24754
|
+
await applyToLocalCopy(indexName, written.indexUuid);
|
|
24755
|
+
},
|
|
24756
|
+
applyToLocalCopy
|
|
24757
|
+
};
|
|
24758
|
+
}
|
|
24759
|
+
|
|
24760
|
+
// src/distribution/cluster-node/read-views.ts
|
|
24761
|
+
function createReadViews(base, coordinator, engine, onError) {
|
|
24762
|
+
const bm25View = createBm25View({ coordinator, engine, onError });
|
|
24763
|
+
const analysisView = createAnalysisView({ coordinator, engine, onError });
|
|
24764
|
+
const vectorReads = createVectorReadView({ coordinator, onError });
|
|
24765
|
+
return {
|
|
24766
|
+
bm25View,
|
|
24767
|
+
analysisView,
|
|
24768
|
+
readDeps: { ...base, bm25View, analysisView, vectorReads },
|
|
24769
|
+
async start() {
|
|
24770
|
+
await bm25View.start();
|
|
24771
|
+
await analysisView.start();
|
|
24772
|
+
await vectorReads.start();
|
|
24773
|
+
},
|
|
24774
|
+
stop() {
|
|
24775
|
+
bm25View.stop();
|
|
24776
|
+
analysisView.stop();
|
|
24777
|
+
vectorReads.stop();
|
|
24778
|
+
}
|
|
24779
|
+
};
|
|
24780
|
+
}
|
|
24781
|
+
|
|
24704
24782
|
// src/distribution/cluster-node/reshard/request.ts
|
|
24705
24783
|
async function waitForSwap(config, indexName, { jobId, targetGeneration }, guard) {
|
|
24706
24784
|
for (; ; ) {
|
|
@@ -24840,11 +24918,11 @@ function clusterNodeEngine(node, options) {
|
|
|
24840
24918
|
async query(indexName, params) {
|
|
24841
24919
|
return node.query(indexName, params);
|
|
24842
24920
|
},
|
|
24843
|
-
async get(indexName, docId) {
|
|
24844
|
-
return node.get(indexName, docId);
|
|
24921
|
+
async get(indexName, docId, options2) {
|
|
24922
|
+
return node.get(indexName, docId, options2);
|
|
24845
24923
|
},
|
|
24846
|
-
async getMultiple(indexName, docIds) {
|
|
24847
|
-
return node.getMultiple(indexName, docIds);
|
|
24924
|
+
async getMultiple(indexName, docIds, options2) {
|
|
24925
|
+
return node.getMultiple(indexName, docIds, options2);
|
|
24848
24926
|
},
|
|
24849
24927
|
async has(indexName, docId) {
|
|
24850
24928
|
return node.has(indexName, docId);
|
|
@@ -25152,9 +25230,9 @@ async function createClusterNode(callerConfig) {
|
|
|
25152
25230
|
extraMessages: (message, respond) => backups.controllerMessages(message, respond) || suggestions.controllerMessages(message, respond)
|
|
25153
25231
|
});
|
|
25154
25232
|
}
|
|
25155
|
-
const
|
|
25156
|
-
const
|
|
25157
|
-
const
|
|
25233
|
+
const readBase = { config: routingConfig2, nodeId, engine, resolveNodeTargets: resolveNodeTargets3, searchContexts };
|
|
25234
|
+
const readViews = createReadViews(readBase, config.coordinator, engine, forwardOnError);
|
|
25235
|
+
const { analysisView, readDeps } = readViews;
|
|
25158
25236
|
const servingName = (indexName) => servingNameOf(allocationView.routing, indexName);
|
|
25159
25237
|
const deviceSync = createClusterDeviceSync({
|
|
25160
25238
|
...cleanupDeps,
|
|
@@ -25218,8 +25296,7 @@ async function createClusterNode(callerConfig) {
|
|
|
25218
25296
|
async start() {
|
|
25219
25297
|
guardShutdown();
|
|
25220
25298
|
await allocationView.start();
|
|
25221
|
-
await
|
|
25222
|
-
await analysisView.start();
|
|
25299
|
+
await readViews.start();
|
|
25223
25300
|
await deviceSync.start();
|
|
25224
25301
|
const reconcileDeps = { engine, coordinator: config.coordinator, nodeId, onError: forwardOnError };
|
|
25225
25302
|
await settleLocalIndexes(reconcileDeps, orphanedIndexes, analysisView.settle);
|
|
@@ -25274,8 +25351,7 @@ async function createClusterNode(callerConfig) {
|
|
|
25274
25351
|
await operationTracker.drain();
|
|
25275
25352
|
await suggestions.stop();
|
|
25276
25353
|
allocationView.stop();
|
|
25277
|
-
|
|
25278
|
-
analysisView.stop();
|
|
25354
|
+
readViews.stop();
|
|
25279
25355
|
await deviceSync.stop();
|
|
25280
25356
|
unregisterHandler?.();
|
|
25281
25357
|
unregisterHandler = null;
|