@delali/narsil 0.3.1-next.104 → 0.3.1-next.109
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/browser/{chunk-CWWD2UHP.mjs → chunk-3RXEGUGO.mjs} +2 -2
- package/dist/browser/{chunk-J3QB4LBD.mjs → chunk-EQ7BQ6VZ.mjs} +1 -0
- package/dist/browser/{chunk-PW7GJFPW.mjs → chunk-RYWURYTW.mjs} +3 -2
- package/dist/browser/{chunk-SFXM7PLV.mjs → chunk-S7KNCVPU.mjs} +2 -2
- package/dist/browser/{chunk-5YMT23EX.mjs → chunk-SCBBEQWD.mjs} +14 -13
- package/dist/browser/{chunk-ECSLQ3S5.mjs → chunk-XHRHOH2P.mjs} +5 -5
- package/dist/browser/{chunk-6E4SWUYZ.mjs → chunk-Z2K5AGZL.mjs} +111 -10
- package/dist/browser/{durability-wiring-HTBWDAOX.mjs → durability-wiring-JJGE7MED.mjs} +6 -6
- package/dist/browser/{module-6FVNK7DO.mjs → module-PCKQADMQ.mjs} +20 -16
- package/dist/browser/{restore-queue-CL43WZTZ.mjs → restore-queue-2STSWU2E.mjs} +1 -1
- package/dist/browser/{runtime-JQYPNOK6.mjs → runtime-ARKW5NZX.mjs} +1 -1
- package/dist/browser/{snapshot-NL4GSDUD.mjs → snapshot-3AZDQKNA.mjs} +7 -7
- package/dist/browser/{write-YFOOPLAN.mjs → write-XXLMPT3Y.mjs} +5 -5
- package/dist/{capabilities-DGaz7cb3.d.ts → capabilities-D4MUmjfe.d.ts} +2 -2
- package/dist/{chunk-35KEGAOS.mjs → chunk-4GD43FSW.mjs} +12 -28
- package/dist/chunk-5DPTHLMX.mjs +350 -0
- package/dist/{chunk-RI5VNFAI.mjs → chunk-5LP7OU6B.mjs} +2 -2
- package/dist/{chunk-KSQS77DK.mjs → chunk-5W7ADHK7.mjs} +1 -1
- package/dist/{chunk-TDUBQ546.mjs → chunk-62GK7UUS.mjs} +3 -3
- package/dist/{chunk-RFVUDM5E.mjs → chunk-6OXNHLNZ.mjs} +3 -15
- package/dist/{chunk-RWXFSGKJ.mjs → chunk-A2HWRBQE.mjs} +2 -2
- package/dist/{chunk-3IQZYNW6.mjs → chunk-AWZVJCCW.mjs} +3 -3
- package/dist/{chunk-2MNJ2HX7.mjs → chunk-C55OFKU6.mjs} +2 -2
- package/dist/{chunk-XWJGJ2SE.mjs → chunk-C57FPCTL.mjs} +6 -6
- package/dist/{chunk-PAWVKEVS.mjs → chunk-CKRP7GCS.mjs} +2 -2
- package/dist/{chunk-IGUTV3MX.mjs → chunk-CYXLPFBG.mjs} +8 -8
- package/dist/{chunk-WCNN2PLX.mjs → chunk-FN75BCXC.mjs} +1 -1
- package/dist/{chunk-77SO7G55.mjs → chunk-FV5CP3E2.mjs} +1 -1
- package/dist/{chunk-WU4P5J4G.mjs → chunk-H5465KUZ.mjs} +3 -3
- package/dist/{chunk-6GFVUXUH.mjs → chunk-HL2GAS5N.mjs} +7 -7
- package/dist/{chunk-MZW4V4DS.mjs → chunk-IBJWO5PF.mjs} +1 -1
- package/dist/{chunk-O372FZM4.mjs → chunk-ICZW4J3I.mjs} +23 -22
- package/dist/{chunk-BLDJ5I6I.mjs → chunk-JKOBHB4A.mjs} +113 -10
- package/dist/{chunk-QALAFGF5.mjs → chunk-LWQTFKRS.mjs} +1 -1
- package/dist/{chunk-PQHR5FHC.mjs → chunk-MVWWAHTR.mjs} +2 -2
- package/dist/{chunk-DQ5NJSBZ.mjs → chunk-N75MAOM7.mjs} +2 -2
- package/dist/{chunk-AEF2O7B4.mjs → chunk-NYWUBTMH.mjs} +1 -1
- package/dist/{chunk-VFEALJEA.mjs → chunk-PA5WZWLC.mjs} +2 -2
- package/dist/{chunk-CYHQNM3T.mjs → chunk-RQ7LOBLD.mjs} +1 -1
- package/dist/{chunk-KNRU5CJ4.mjs → chunk-RSSLOFRP.mjs} +6 -6
- package/dist/{chunk-6JUOMMHQ.mjs → chunk-S2CKO6PP.mjs} +1 -0
- package/dist/{chunk-JSSGO4B5.mjs → chunk-SAZB7ZHT.mjs} +1 -1
- package/dist/{chunk-MJIJG4YT.mjs → chunk-SD2EYVU4.mjs} +2 -2
- package/dist/{chunk-LPQOEU5M.mjs → chunk-SOADQINJ.mjs} +45 -9
- package/dist/{chunk-PFQQ3LCQ.mjs → chunk-SYHS56F7.mjs} +6 -6
- package/dist/{chunk-OIN27OQH.mjs → chunk-U6FT7LKG.mjs} +3 -3
- package/dist/{chunk-J2KRH2RJ.mjs → chunk-VIXXQEX6.mjs} +4 -4
- package/dist/{chunk-HMFNFVP7.mjs → chunk-VJS4JFBQ.mjs} +2 -1
- package/dist/{chunk-MKPE3VZF.mjs → chunk-XY6Z7T5W.mjs} +2 -2
- package/dist/{chunk-5W2ODPPO.mjs → chunk-YP23IAV5.mjs} +177 -8
- package/dist/{chunk-47PZE6CJ.mjs → chunk-ZBZRBM2Z.mjs} +3 -3
- package/dist/chunk-ZXZ26WSO.mjs +14 -0
- package/dist/{client-DI-xIXgR.d.ts → client-Ba9yP-Rr.d.ts} +2 -2
- package/dist/client.d.ts +7 -7
- package/dist/client.mjs +1 -0
- package/dist/{config-DL_bDDD8.d.ts → config-C-xXz8bC.d.ts} +1 -1
- package/dist/{config-DWxtDsJ6.d.ts → config-Dc_8-byz.d.ts} +7 -4
- package/dist/direct-executor-QH2M6AK2.mjs +25 -0
- package/dist/distribution/coordinator/etcd.d.ts +1 -1
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +1 -1
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +7 -7
- package/dist/distribution.mjs +678 -278
- package/dist/{durability-wiring-LD4GWPWD.mjs → durability-wiring-4HWX7OVP.mjs} +26 -25
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-D_SYDsId.d.ts → engine-CMZ5XSLy.d.ts} +1 -1
- package/dist/{events-C7_dboUr.d.ts → events-dnvgprIl.d.ts} +1 -0
- package/dist/{index-Bkdt8nDw.d.ts → index-Cl5qQauj.d.ts} +6 -1
- package/dist/index.browser.mjs +3 -3
- package/dist/index.d.ts +5 -5
- package/dist/index.mjs +19 -18
- package/dist/{module-QFWZ5OBF.mjs → module-IYILOTU3.mjs} +25 -21
- package/dist/partial-words.mjs +6 -6
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.mjs +22 -21
- package/dist/react.d.ts +6 -6
- package/dist/react.mjs +1 -0
- package/dist/{rerank-WVFFZV76.mjs → rerank-EHJVXEDP.mjs} +7 -7
- package/dist/rerankers/http.mjs +2 -2
- package/dist/{restore-queue-7QIKM6PY.mjs → restore-queue-BONSQWCI.mjs} +1 -1
- package/dist/{runtime-AVUUCJWQ.mjs → runtime-LEX2F7OW.mjs} +5 -5
- package/dist/server.d.ts +9 -9
- package/dist/server.mjs +32 -18
- package/dist/{snapshot-JYTT36WI.mjs → snapshot-DWDLN5YW.mjs} +28 -27
- package/dist/synonyms.mjs +3 -3
- package/dist/{tasks-BhjQZ8Qf.d.ts → tasks-B4B2cMK5.d.ts} +1 -1
- package/dist/{thread-AHLISTRZ.mjs → thread-I32PUURR.mjs} +16 -14
- package/dist/vector/search-worker.d.ts +2 -0
- package/dist/vector/search-worker.mjs +5 -5
- package/dist/workers/entry.mjs +7 -6
- package/dist/{write-GPFM2EIX.mjs → write-AOFSHACR.mjs} +21 -20
- package/package.json +7 -7
- package/dist/chunk-B7BSQZ27.mjs +0 -206
- package/dist/direct-executor-5NX2ZIAG.mjs +0 -24
package/dist/distribution.mjs
CHANGED
|
@@ -1,59 +1,60 @@
|
|
|
1
|
-
import { DEFAULT_NODE_LIFECYCLE_CONFIG, bindWriteBlockLookup, bindEngineWriteBlock, bindIndexReads, DEFAULT_CONTROLLER_CONFIG, getIndexMetadata, getClusterIndexConfig, withPartitionCount, validateIndexName, readIndexState, readIndexMetadataSnapshot, storeSynonymSet, deleteSynonymSet, indexSettingsOf, putIndexMetadata, isValidInteger, isRecord, HIDDEN_GENERATION_PREFIX, indexConfigFromSettings, metadataWithSynonymSet, indexConfigKey, RESHARD_RECORD_CAS_ATTEMPTS, indexUuidOfStoredMetadata, RESHARD_FORWARD_BATCH_DOCUMENTS, storedSynonymSetOf, RESHARD_RETRY_DELAY_MS, RESHARD_RECORD_CHECK_INTERVAL_MS, RESHARD_POLL_INTERVAL_MS, ALLOCATION_RETRY_DELAY_MS, TEARDOWN_CAS_ATTEMPTS, TEARDOWN_RETRY_ROUNDS, putIndexReplicationFactor, ALLOCATION_CAS_ATTEMPTS, TEARDOWN_RETRY_DELAY_MS, RESHARD_CONTROLLER_POLL_INTERVAL_MS, MAX_RESHARD_CUTOVER_ATTEMPTS, RECOVERY_CAS_ATTEMPTS, ESTIMATED_PARTITION_BYTES, REBALANCE_THRESHOLD, putGenerationMetadata, RESHARD_CUTOVER_LAG_ENTRIES, RESHARD_CUTOVER_TIMEOUT_MS, PARTITION_STORES_TIMEOUT_MS, LEADERSHIP_IMBALANCE_THRESHOLD, BOOTSTRAP_CAS_ATTEMPTS } from './chunk-
|
|
2
|
-
import { MAX_WAIT_FOR_ACTIVE_REPLICAS, MIN_CONTROLLER_LEASE_TTL_MS, MAX_NODE_LEFT_DELAY_MS, DEFAULT_MAX_PER_SOURCE_SNAPSHOTS, DEFAULT_MAX_STREAMS_PER_INDEX, DEFAULT_CAPACITY, CATCH_UP_TICK_MS, DEFAULT_RESUME_WRITES_BELOW, DEFAULT_BLOCK_WRITES_ABOVE, readMemoryFigures, DEFAULT_IN_FLIGHT_WRITE_FRACTION, onRefusalAfterCollection, MAX_DROP_ACKNOWLEDGE_WAIT_MS, DEFAULT_PARTITION_COUNT, DEFAULT_REPLICATION_FACTOR, DEFAULT_CREATE_INDEX_WAIT_MS, readRecentMemoryFigures, DEFAULT_BOOTSTRAP_SYNC_DEADLINE_MS, MAX_SYNONYM_REPLACEMENT_ATTEMPTS, DROP_ACKNOWLEDGE_POLL_INTERVAL_MS, COORDINATOR_REQUEST_BUCKET_BOUNDS_SECONDS, COPY_FAILURE_ATTEMPTS, DECOMMISSION_RECHECK_INTERVAL_MS, CASE_FOLD_EXPANSION_FACTOR, PATTERN_RUN_HEAP_BYTES, ENUM_VALUE_HEAP_BYTES, MAX_SOURCE_ID_LENGTH, reserveHeapBytes, reservedHeapBytes, DOCUMENT_VALUE_HEAP_BYTES, DOCUMENT_HEAP_BYTES_PER_ESTIMATED_BYTE, DOCUMENT_PROPERTY_HEAP_BYTES, VECTOR_DOCUMENT_HEAP_BYTES, VECTOR_COMPONENT_HEAP_BYTES, NORMALIZATION_EXPANSION_FACTOR, TERM_UNIT_HEAP_BYTES, PARTITION_BYTES_INDEX_OVERHEAD_BYTES, DISTINCT_TERM_HEAP_BYTES, PARTIAL_SPELLING_HEAP_BYTES, PARTITION_BYTES_ENTRY_BYTES, PARTITION_BYTES_REPORT_BUDGET_BYTES,
|
|
3
|
-
export { DEFAULT_CAPACITY, DEFAULT_PARTITION_COUNT, DEFAULT_REPLICATION_FACTOR } from './chunk-
|
|
4
|
-
import { fixedHybridDepthOf, requireWithinHybridDepth, pageByPosition } from './chunk-
|
|
5
|
-
import { closePartitionWriter, replaySyncedWal } from './chunk-
|
|
6
|
-
import { buildArchiveFromMemory, archiveStream, memoryCheckpointStore, readSnapshotArchive } from './chunk-
|
|
7
|
-
import './chunk-
|
|
8
|
-
import { indexMetadataOf, loadPartitionSegments, readSegmentManifest, readVectorFile } from './chunk-
|
|
9
|
-
import { decodeVectorFilePayload, bytesToVectors, isDead } from './chunk-
|
|
1
|
+
import { DEFAULT_NODE_LIFECYCLE_CONFIG, bindWriteBlockLookup, bindEngineWriteBlock, bindIndexReads, DEFAULT_CONTROLLER_CONFIG, getIndexMetadata, getClusterIndexConfig, withPartitionCount, validateIndexName, readIndexState, readIndexMetadataSnapshot, storeSynonymSet, deleteSynonymSet, indexSettingsOf, putIndexMetadata, isValidInteger, isRecord, HIDDEN_GENERATION_PREFIX, indexConfigFromSettings, metadataWithSynonymSet, indexConfigKey, RESHARD_RECORD_CAS_ATTEMPTS, indexUuidOfStoredMetadata, RESHARD_FORWARD_BATCH_DOCUMENTS, storedSynonymSetOf, RESHARD_RETRY_DELAY_MS, RESHARD_RECORD_CHECK_INTERVAL_MS, RESHARD_POLL_INTERVAL_MS, ALLOCATION_RETRY_DELAY_MS, TEARDOWN_CAS_ATTEMPTS, TEARDOWN_RETRY_ROUNDS, putIndexReplicationFactor, ALLOCATION_CAS_ATTEMPTS, TEARDOWN_RETRY_DELAY_MS, RESHARD_CONTROLLER_POLL_INTERVAL_MS, MAX_RESHARD_CUTOVER_ATTEMPTS, RECOVERY_CAS_ATTEMPTS, ESTIMATED_PARTITION_BYTES, REBALANCE_THRESHOLD, putGenerationMetadata, RESHARD_CUTOVER_LAG_ENTRIES, RESHARD_CUTOVER_TIMEOUT_MS, PARTITION_STORES_TIMEOUT_MS, LEADERSHIP_IMBALANCE_THRESHOLD, BOOTSTRAP_CAS_ATTEMPTS } from './chunk-U6FT7LKG.mjs';
|
|
2
|
+
import { MAX_WAIT_FOR_ACTIVE_REPLICAS, MIN_CONTROLLER_LEASE_TTL_MS, MAX_NODE_LEFT_DELAY_MS, DEFAULT_MAX_PER_SOURCE_SNAPSHOTS, DEFAULT_MAX_STREAMS_PER_INDEX, DEFAULT_CAPACITY, CATCH_UP_TICK_MS, DEFAULT_RESUME_WRITES_BELOW, DEFAULT_BLOCK_WRITES_ABOVE, readMemoryFigures, createWritePressure, WRITE_MEMORY_SAMPLE_MAX_AGE_MS, DEFAULT_IN_FLIGHT_WRITE_FRACTION, onRefusalAfterCollection, MAX_DROP_ACKNOWLEDGE_WAIT_MS, DEFAULT_PARTITION_COUNT, DEFAULT_REPLICATION_FACTOR, DEFAULT_CREATE_INDEX_WAIT_MS, refuseEncodeWithoutRoom, readRecentMemoryFigures, DEFAULT_BOOTSTRAP_SYNC_DEADLINE_MS, MAX_SYNONYM_REPLACEMENT_ATTEMPTS, DROP_ACKNOWLEDGE_POLL_INTERVAL_MS, COORDINATOR_REQUEST_BUCKET_BOUNDS_SECONDS, COPY_FAILURE_ATTEMPTS, SEARCH_CONTEXT_REAPER_INTERVAL_MS, DECOMMISSION_RECHECK_INTERVAL_MS, answerRefusalOf, withAdmittedWrite, estimateForwardedBytes, CASE_FOLD_EXPANSION_FACTOR, PATTERN_RUN_HEAP_BYTES, ENUM_VALUE_HEAP_BYTES, MAX_SOURCE_ID_LENGTH, reserveHeapBytes, reservedHeapBytes, DOCUMENT_VALUE_HEAP_BYTES, DOCUMENT_HEAP_BYTES_PER_ESTIMATED_BYTE, DOCUMENT_PROPERTY_HEAP_BYTES, VECTOR_DOCUMENT_HEAP_BYTES, VECTOR_COMPONENT_HEAP_BYTES, NORMALIZATION_EXPANSION_FACTOR, TERM_UNIT_HEAP_BYTES, PARTITION_BYTES_INDEX_OVERHEAD_BYTES, DISTINCT_TERM_HEAP_BYTES, PARTIAL_SPELLING_HEAP_BYTES, PARTITION_BYTES_ENTRY_BYTES, PARTITION_BYTES_REPORT_BUDGET_BYTES, SNAPSHOT_CHUNK_YIELD_INTERVAL_MS, CATCH_UP_FEED_WINDOW_MS, MAX_CATCH_UP_IN_FLIGHT_BYTES, SYNCED_POSITION_REPORT_TIMEOUT_MS, ADMISSION_TIMEOUT_MS, REGISTRATION_LEASE_DRIFT_SHARE, REGISTRATION_LEASE_CONFIRM_TIMEOUT_MS, WRITE_APPLY_YIELD_INTERVAL_MS, ROLLBACK_RETRY_INITIAL_DELAY_MS, ROLLBACK_RETRY_DELAY_CAP_MS, COPY_FAILURE_RETRY_DELAY_MS, FETCH_GROUP_DOCUMENT_IDS, SEARCH_CONTEXT_ENTRY_HEAP_BYTES, collectYoungGenerationBeforeRefusal, collectGarbageBeforeRefusal, FULL_MEMORY_COLLECTION_INTERVAL_MS, MEMORY_SAMPLE_INTERVAL_MS, ANSWER_DECODE_HEAP_BYTES_PER_PAYLOAD_BYTE, CLEAR_PAGE_SIZE, estimateDocumentsBytes, estimateDocumentBytes, PRIMARY_CHANGE_POLL_INTERVAL_MS, PARTITION_WRITE_CONCURRENCY, FORWARD_RETRY_INITIAL_DELAY_MS, PRIMARY_CHANGE_RETRY_WINDOW_MS, FORWARD_RETRY_DELAY_CAP_MS } from './chunk-5DPTHLMX.mjs';
|
|
3
|
+
export { DEFAULT_CAPACITY, DEFAULT_PARTITION_COUNT, DEFAULT_REPLICATION_FACTOR } from './chunk-5DPTHLMX.mjs';
|
|
4
|
+
import { fixedHybridDepthOf, requireWithinHybridDepth, pageByPosition } from './chunk-FN75BCXC.mjs';
|
|
5
|
+
import { closePartitionWriter, replaySyncedWal } from './chunk-H5465KUZ.mjs';
|
|
6
|
+
import { buildArchiveFromMemory, archiveStream, memoryCheckpointStore, readSnapshotArchive } from './chunk-SYHS56F7.mjs';
|
|
7
|
+
import './chunk-ZBZRBM2Z.mjs';
|
|
8
|
+
import { indexMetadataOf, loadPartitionSegments, readSegmentManifest, readVectorFile } from './chunk-RSSLOFRP.mjs';
|
|
9
|
+
import { decodeVectorFilePayload, bytesToVectors, isDead } from './chunk-CKRP7GCS.mjs';
|
|
10
10
|
import { CONTROLLER_LEASE_KEY } from './chunk-44W3CBNG.mjs';
|
|
11
11
|
import { MAX_MESSAGE_SIZE_BYTES } from './chunk-UNEEFTEU.mjs';
|
|
12
|
-
import './chunk-
|
|
12
|
+
import './chunk-JKOBHB4A.mjs';
|
|
13
13
|
import './chunk-DJHHF2L5.mjs';
|
|
14
14
|
import './chunk-37CD3OKS.mjs';
|
|
15
15
|
import { getPath, getFs } from './chunk-HLPRHOSO.mjs';
|
|
16
|
-
import { createForwardBatchResultMessage, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, readListedFile, optionalIndexUuid, computeEntryChecksum, createPrimaryResignMessage, validateInsyncConfirmPayload, buildEntry, createInsyncRemoveMessage, createEntryBatchMessage, createInsyncConfirmMessage, createEntryMessage, validateAckPayload, createInsyncAddMessage, directoryCheckpointStore, checkpointLastSeqNo, checkpointPrimaryTerm,
|
|
17
|
-
import { applyIndexEntry, applyDeleteEntry, validateReplicationEntry } from './chunk-
|
|
16
|
+
import { createForwardBatchResultMessage, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, readListedFile, optionalIndexUuid, computeEntryChecksum, createPrimaryResignMessage, validateInsyncConfirmPayload, buildEntry, createInsyncRemoveMessage, createEntryBatchMessage, createInsyncConfirmMessage, createEntryMessage, validateAckPayload, createInsyncAddMessage, directoryCheckpointStore, checkpointLastSeqNo, checkpointPrimaryTerm, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage } from './chunk-CYXLPFBG.mjs';
|
|
17
|
+
import { applyIndexEntry, applyDeleteEntry, validateReplicationEntry } from './chunk-VIXXQEX6.mjs';
|
|
18
18
|
import { MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR, WIRE_BATCH_BUDGET } from './chunk-P36MUFEJ.mjs';
|
|
19
19
|
import { TransportErrorCodes, ReplicationMessageTypes, ClusterMessageTypes, QueryMessageTypes, TransportError, DEFAULT_TRANSPORT_CONFIG } from './chunk-RPIE4QCL.mjs';
|
|
20
|
-
import { requireKnownConfig, resolveRunnableConfig, createEngineCore, startEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, createEngineIndex, encodeIndexConfiguration, requireConfigurationWithinSizeLimit, prepareConfiguredSynonyms, createByteReader, openChecksummedEnvelope, emptySerializablePartition, crc32Init, crc32Final, crc32Update, yieldToEventLoop } from './chunk-
|
|
21
|
-
import { runtimeHost, requireSurfaceFormsForPartialFields,
|
|
22
|
-
import { bindRerankWindowSource, rerankWindowCapOf,
|
|
23
|
-
import './chunk-
|
|
20
|
+
import { requireKnownConfig, resolveRunnableConfig, createEngineCore, startEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, createEngineIndex, encodeIndexConfiguration, requireConfigurationWithinSizeLimit, prepareConfiguredSynonyms, createByteReader, openChecksummedEnvelope, emptySerializablePartition, crc32Init, crc32Final, crc32Update, yieldToEventLoop } from './chunk-ICZW4J3I.mjs';
|
|
21
|
+
import { runtimeHost, requireSurfaceFormsForPartialFields, now, listBindingOf, clampLimit, hitsKeptPerGroup, queryBindingOf, resolveHybridFusion, dedupePinnedEntries, foldGroupReducer } from './chunk-HL2GAS5N.mjs';
|
|
22
|
+
import { bindRerankWindowSource, rerankWindowCapOf, providedDocId, engineCoreOf } from './chunk-XY6Z7T5W.mjs';
|
|
23
|
+
import './chunk-RQ7LOBLD.mjs';
|
|
24
24
|
import './chunk-AV4IJKD2.mjs';
|
|
25
|
-
import { synonymModule } from './chunk-
|
|
26
|
-
import { oversampledShardSize, normalizeSort, sortModesOf, readSortValues, resolveProjection,
|
|
27
|
-
import { generateId, getLanguage } from './chunk-
|
|
28
|
-
import './chunk-
|
|
25
|
+
import { synonymModule } from './chunk-4GD43FSW.mjs';
|
|
26
|
+
import { oversampledShardSize, normalizeSort, sortModesOf, readSortValues, resolveProjection, fnv1a, applyProjection, FILTER_EXPRESSION_KEYS, filterTestCount, setNestedValue, FIELD_FILTER_OPERATORS, clampRowCount, sortSignatureOf, requireMatchingCursor, decodePageCursor, encodePageCursor, requireWithinResultWindow, sortSignatureEntry, fnv1aUint32s, mergePartitionStats } from './chunk-C57FPCTL.mjs';
|
|
27
|
+
import { generateId, getLanguage } from './chunk-LWQTFKRS.mjs';
|
|
28
|
+
import './chunk-SOADQINJ.mjs';
|
|
29
29
|
import './chunk-HJSN7UYG.mjs';
|
|
30
30
|
import { MAX_DOC_ID_LENGTH, BATCH_CHUNK_SIZE, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING } from './chunk-LXWDQLTW.mjs';
|
|
31
31
|
import './chunk-TGP5M5R7.mjs';
|
|
32
32
|
import './chunk-U2BKYEC7.mjs';
|
|
33
33
|
import './chunk-KOKM4PQN.mjs';
|
|
34
34
|
import { MIN_OVERSAMPLE } from './chunk-QSK35UC2.mjs';
|
|
35
|
-
import './chunk-
|
|
36
|
-
import { isPatternTolerance } from './chunk-
|
|
35
|
+
import './chunk-MVWWAHTR.mjs';
|
|
36
|
+
import { isPatternTolerance } from './chunk-C55OFKU6.mjs';
|
|
37
37
|
import { MAX_FILTER_DEPTH, MAX_FILTER_TESTS, MAX_FILTER_ARRAY_SIZE } from './chunk-WQBCBKY3.mjs';
|
|
38
|
-
import { getNestedValue } from './chunk-
|
|
39
|
-
import { validateSchema, requireCreatableSchema, encodeDocumentReply, encodeDocument, schemaFieldsOf, MIN_POLYGON_POINTS, isLatitude, MAX_LATITUDE_DEGREES, isLongitude, MAX_LONGITUDE_DEGREES } from './chunk-
|
|
38
|
+
import { getNestedValue } from './chunk-62GK7UUS.mjs';
|
|
39
|
+
import { validateSchema, requireCreatableSchema, encodeDocumentReply, encodeDocument, schemaFieldsOf, MIN_POLYGON_POINTS, isLatitude, MAX_LATITUDE_DEGREES, isLongitude, MAX_LONGITUDE_DEGREES } from './chunk-AWZVJCCW.mjs';
|
|
40
40
|
import { MAX_DOCUMENT_ENCODING_DEPTH } from './chunk-U476IXKQ.mjs';
|
|
41
41
|
import { MAX_SORT_FIELDS, MAX_CURSOR_LENGTH, MAX_TOLERANCE, MAX_PREFIX_LENGTH, RESULT_WINDOW, MIN_HYBRID_DEPTH, MAX_HYBRID_DEPTH, DEFAULT_PAGE_SIZE, HYBRID_RESCORE_WINDOW } from './chunk-E74YLAIG.mjs';
|
|
42
|
-
import './chunk-
|
|
42
|
+
import './chunk-VJS4JFBQ.mjs';
|
|
43
43
|
import './chunk-GES7U2PM.mjs';
|
|
44
|
-
import './chunk-
|
|
45
|
-
import { resolveIndexAnalysis } from './chunk-
|
|
44
|
+
import './chunk-FV5CP3E2.mjs';
|
|
45
|
+
import { resolveIndexAnalysis } from './chunk-5LP7OU6B.mjs';
|
|
46
46
|
import './chunk-X5CV2E2B.mjs';
|
|
47
47
|
import './chunk-7X5JST5L.mjs';
|
|
48
|
-
import './chunk-
|
|
48
|
+
import './chunk-NYWUBTMH.mjs';
|
|
49
49
|
import { isSortMode, SORT_MODES, toComparableSortValue, compareComparableKeys, compareSortValues } from './chunk-R6A773WK.mjs';
|
|
50
50
|
import { compareCodePoints, exceedsCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
51
|
-
import {
|
|
51
|
+
import { collectAndReturnPages } from './chunk-6OXNHLNZ.mjs';
|
|
52
52
|
import './chunk-CQUMNIIG.mjs';
|
|
53
|
-
import './chunk-
|
|
53
|
+
import './chunk-5W7ADHK7.mjs';
|
|
54
|
+
import { HEAP_RESUME_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE } from './chunk-ZXZ26WSO.mjs';
|
|
54
55
|
import './chunk-RFBLQLLZ.mjs';
|
|
55
56
|
import { MAX_STORED_VECTOR_DIMENSION, MAX_NESTING_DEPTH } from './chunk-L5ORED7V.mjs';
|
|
56
|
-
import { ErrorCodes, NarsilError, describeError } from './chunk-
|
|
57
|
+
import { ErrorCodes, NarsilError, describeError, ServerErrorCodes } from './chunk-S2CKO6PP.mjs';
|
|
57
58
|
import { encode, decode } from '@msgpack/msgpack';
|
|
58
59
|
import { randomUUID } from 'crypto';
|
|
59
60
|
import { openSync, writeSync, closeSync, rmSync, readSync } from 'fs';
|
|
@@ -7868,6 +7869,29 @@ async function restoreReplicationPartition(core, engine, indexName, partitionId,
|
|
|
7868
7869
|
}
|
|
7869
7870
|
}
|
|
7870
7871
|
|
|
7872
|
+
// src/distribution/cluster-node/search-contexts/snapshots.ts
|
|
7873
|
+
function snapshotOf(manager, docId) {
|
|
7874
|
+
const fields = manager.getRef(docId);
|
|
7875
|
+
if (fields === void 0) return void 0;
|
|
7876
|
+
const vectors = [];
|
|
7877
|
+
for (const [fieldPath2, vectorIndex] of manager.getVectorIndexes()) {
|
|
7878
|
+
const vector = vectorIndex.getVector(docId);
|
|
7879
|
+
if (vector !== null) vectors.push([fieldPath2, vector]);
|
|
7880
|
+
}
|
|
7881
|
+
return { fields, vectors };
|
|
7882
|
+
}
|
|
7883
|
+
function snapshotRankedDocuments(manager, result, into) {
|
|
7884
|
+
const take = (docId) => {
|
|
7885
|
+
if (into.has(docId)) return;
|
|
7886
|
+
const snapshot = snapshotOf(manager, docId);
|
|
7887
|
+
if (snapshot !== void 0) into.set(docId, snapshot);
|
|
7888
|
+
};
|
|
7889
|
+
for (const hit of result.hits) take(hit.id);
|
|
7890
|
+
for (const group of result.groups ?? []) {
|
|
7891
|
+
for (const hit of group.hits) take(hit.id);
|
|
7892
|
+
}
|
|
7893
|
+
}
|
|
7894
|
+
|
|
7871
7895
|
// src/distribution/cluster-node/write-admission.ts
|
|
7872
7896
|
function createAdmittingRecorder(core, admission, durable) {
|
|
7873
7897
|
function admit(indexName, docId, document) {
|
|
@@ -7962,7 +7986,12 @@ async function createClusterLocalEngine(config, hooks) {
|
|
|
7962
7986
|
applyReplicationEntries: (entries) => applyLocalReplicationEntries(core, entries),
|
|
7963
7987
|
archiveReplicationPartition: (indexName, partitionId, lastSeqNo) => archiveReplicationPartition(core, indexName, partitionId, lastSeqNo),
|
|
7964
7988
|
restoreReplicationPartition: (indexName, partitionId, source, indexConfig, partitionCount) => restoreReplicationPartition(core, engine, indexName, partitionId, source, indexConfig, partitionCount),
|
|
7965
|
-
queryPartitions: (indexName, params, partitionIds, globalStats, rescoreWindow) => runEngineQuery(core, indexName, params, {
|
|
7989
|
+
queryPartitions: (indexName, params, partitionIds, globalStats, rescoreWindow, snapshots) => runEngineQuery(core, indexName, params, {
|
|
7990
|
+
partitionIds,
|
|
7991
|
+
globalStats,
|
|
7992
|
+
rescoreWindow,
|
|
7993
|
+
onRanked: snapshots === void 0 ? void 0 : (manager, result) => snapshotRankedDocuments(manager, result, snapshots)
|
|
7994
|
+
}),
|
|
7966
7995
|
preflightPartitions: (indexName, params, partitionIds) => runEnginePreflight(core, indexName, params, { partitionIds }),
|
|
7967
7996
|
suggestPartitions: (indexName, params, partitionIds) => runEngineSuggest(core, indexName, params, partitionIds),
|
|
7968
7997
|
listPartitions: (indexName, params, partitionIds) => runEngineListDocuments(core, indexName, params, partitionIds),
|
|
@@ -8072,6 +8101,9 @@ var MAX_GROUPS_PER_RESPONSE = 65536;
|
|
|
8072
8101
|
var MAX_FACET_SIZE = 1e3;
|
|
8073
8102
|
var DEFAULT_MAX_FACET_BUCKETS = 100;
|
|
8074
8103
|
var MAX_VECTOR_TEXT_LENGTH = 16384;
|
|
8104
|
+
var SEARCH_CONTEXT_KEEP_ALIVE_MS = 6e4;
|
|
8105
|
+
var SEARCH_CONTEXT_KEEP_ALIVE_LIMIT_MS = 864e5;
|
|
8106
|
+
var MAX_SEARCH_CONTEXT_ID_LENGTH = 64;
|
|
8075
8107
|
|
|
8076
8108
|
// src/distribution/query/validators/common.ts
|
|
8077
8109
|
var NULL_BYTE = String.fromCharCode(0);
|
|
@@ -8250,6 +8282,12 @@ function validateFetchPayload(decoded) {
|
|
|
8250
8282
|
if (decoded.highlight !== null) {
|
|
8251
8283
|
validateHighlightConfig(decoded.highlight);
|
|
8252
8284
|
}
|
|
8285
|
+
if (decoded.contextId !== null && decoded.contextId !== void 0) {
|
|
8286
|
+
validateStringField(decoded.contextId, "contextId", MAX_SEARCH_CONTEXT_ID_LENGTH, CONFIG_INVALID);
|
|
8287
|
+
}
|
|
8288
|
+
if (decoded.releaseContext !== null && decoded.releaseContext !== void 0 && typeof decoded.releaseContext !== "boolean") {
|
|
8289
|
+
throwInvalid(CONFIG_INVALID, 'Invalid FetchPayload: "releaseContext" must be a boolean, or null');
|
|
8290
|
+
}
|
|
8253
8291
|
return decoded;
|
|
8254
8292
|
}
|
|
8255
8293
|
function validateFetchResultPayload(decoded) {
|
|
@@ -9081,6 +9119,11 @@ function validateSearchPayload(decoded) {
|
|
|
9081
9119
|
);
|
|
9082
9120
|
}
|
|
9083
9121
|
}
|
|
9122
|
+
if (decoded.keepAliveMs !== null && decoded.keepAliveMs !== void 0) {
|
|
9123
|
+
if (!Number.isInteger(decoded.keepAliveMs) || decoded.keepAliveMs <= 0) {
|
|
9124
|
+
throwInvalid(CONFIG_INVALID, 'Invalid SearchPayload: "keepAliveMs" must be a positive integer, or null');
|
|
9125
|
+
}
|
|
9126
|
+
}
|
|
9084
9127
|
return decoded;
|
|
9085
9128
|
}
|
|
9086
9129
|
|
|
@@ -9483,6 +9526,14 @@ function validateSearchResultPayload(decoded) {
|
|
|
9483
9526
|
if (decoded.groups !== null && decoded.groups !== void 0) {
|
|
9484
9527
|
validateGroupEntries(decoded.groups);
|
|
9485
9528
|
}
|
|
9529
|
+
if (decoded.contextId !== null && decoded.contextId !== void 0) {
|
|
9530
|
+
if (typeof decoded.contextId !== "string" || decoded.contextId.length === 0 || decoded.contextId.length > MAX_SEARCH_CONTEXT_ID_LENGTH) {
|
|
9531
|
+
throwInvalid(
|
|
9532
|
+
CONFIG_INVALID,
|
|
9533
|
+
`Invalid SearchResultPayload: "contextId" must be a string of 1 to ${MAX_SEARCH_CONTEXT_ID_LENGTH} characters, or null`
|
|
9534
|
+
);
|
|
9535
|
+
}
|
|
9536
|
+
}
|
|
9486
9537
|
return decoded;
|
|
9487
9538
|
}
|
|
9488
9539
|
|
|
@@ -9679,6 +9730,10 @@ function nodeRejectionOf(code, message, details) {
|
|
|
9679
9730
|
nodeRejections.add(rejection);
|
|
9680
9731
|
return rejection;
|
|
9681
9732
|
}
|
|
9733
|
+
function markNodeRejection(error) {
|
|
9734
|
+
nodeRejections.add(error);
|
|
9735
|
+
return error;
|
|
9736
|
+
}
|
|
9682
9737
|
function isNodeRejection(error) {
|
|
9683
9738
|
return error instanceof NarsilError && nodeRejections.has(error);
|
|
9684
9739
|
}
|
|
@@ -9827,6 +9882,44 @@ function selectReplicasForQuery(allocationTable, selector = randomSelector) {
|
|
|
9827
9882
|
}
|
|
9828
9883
|
return { nodeToPartitions, copiesByPartition, unavailablePartitions };
|
|
9829
9884
|
}
|
|
9885
|
+
function transportableDetails(error) {
|
|
9886
|
+
if (!(error instanceof NarsilError)) {
|
|
9887
|
+
return void 0;
|
|
9888
|
+
}
|
|
9889
|
+
try {
|
|
9890
|
+
encode(error.details);
|
|
9891
|
+
return error.details;
|
|
9892
|
+
} catch {
|
|
9893
|
+
return void 0;
|
|
9894
|
+
}
|
|
9895
|
+
}
|
|
9896
|
+
function encodeErrorReply(code, message, details) {
|
|
9897
|
+
if (details !== void 0) {
|
|
9898
|
+
try {
|
|
9899
|
+
return encode({ error: true, code, message, details });
|
|
9900
|
+
} catch {
|
|
9901
|
+
return encode({ error: true, code, message });
|
|
9902
|
+
}
|
|
9903
|
+
}
|
|
9904
|
+
return encode({ error: true, code, message });
|
|
9905
|
+
}
|
|
9906
|
+
function encodeErrorReplyFrom(error, fallbackCode) {
|
|
9907
|
+
return encodeErrorReply(
|
|
9908
|
+
error instanceof NarsilError ? error.code : fallbackCode,
|
|
9909
|
+
error instanceof Error ? error.message : String(error),
|
|
9910
|
+
error instanceof NarsilError ? error.details : void 0
|
|
9911
|
+
);
|
|
9912
|
+
}
|
|
9913
|
+
function isPlainRecord(value) {
|
|
9914
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) {
|
|
9915
|
+
return false;
|
|
9916
|
+
}
|
|
9917
|
+
const prototype = Object.getPrototypeOf(value);
|
|
9918
|
+
return prototype === Object.prototype || prototype === null;
|
|
9919
|
+
}
|
|
9920
|
+
function remoteErrorDetails(value) {
|
|
9921
|
+
return isPlainRecord(value) ? value : {};
|
|
9922
|
+
}
|
|
9830
9923
|
|
|
9831
9924
|
// src/distribution/cluster-node/vector-bytes.ts
|
|
9832
9925
|
var FLOAT32_BYTES = 4;
|
|
@@ -9979,141 +10072,6 @@ function requireWritableIndex(table, view) {
|
|
|
9979
10072
|
throw block;
|
|
9980
10073
|
}
|
|
9981
10074
|
}
|
|
9982
|
-
|
|
9983
|
-
// src/distribution/cluster-node/memory/write-pressure.ts
|
|
9984
|
-
var WRITE_BYTES_IN_FLIGHT_REASON = "write_bytes_in_flight";
|
|
9985
|
-
function createWritePressure(options) {
|
|
9986
|
-
let inFlight = 0;
|
|
9987
|
-
return {
|
|
9988
|
-
admit(indexName, bytes) {
|
|
9989
|
-
options.beforeAdmit();
|
|
9990
|
-
const requestBytes = Math.max(0, Math.ceil(bytes));
|
|
9991
|
-
if (inFlight > 0 && inFlight + requestBytes > options.limitBytes) {
|
|
9992
|
-
throw new NarsilError(
|
|
9993
|
-
ErrorCodes.CLUSTER_WRITE_BLOCKED,
|
|
9994
|
-
`Node '${options.nodeId}' refuses this write to index '${indexName}' because its ${requestBytes} bytes would take the writes in flight on the node from ${inFlight} bytes past the limit of ${options.limitBytes} bytes. Retry once the writes in flight finish, or send smaller batches.`,
|
|
9995
|
-
{
|
|
9996
|
-
reason: WRITE_BYTES_IN_FLIGHT_REASON,
|
|
9997
|
-
indexName,
|
|
9998
|
-
nodeId: options.nodeId,
|
|
9999
|
-
requestBytes,
|
|
10000
|
-
inFlightBytes: inFlight,
|
|
10001
|
-
limitBytes: options.limitBytes
|
|
10002
|
-
}
|
|
10003
|
-
);
|
|
10004
|
-
}
|
|
10005
|
-
inFlight += requestBytes;
|
|
10006
|
-
let released = false;
|
|
10007
|
-
return () => {
|
|
10008
|
-
if (released) {
|
|
10009
|
-
return;
|
|
10010
|
-
}
|
|
10011
|
-
released = true;
|
|
10012
|
-
inFlight -= requestBytes;
|
|
10013
|
-
};
|
|
10014
|
-
}
|
|
10015
|
-
};
|
|
10016
|
-
}
|
|
10017
|
-
async function withAdmittedWrite(pressure, indexName, bytes, work) {
|
|
10018
|
-
if (pressure === void 0) {
|
|
10019
|
-
return work();
|
|
10020
|
-
}
|
|
10021
|
-
const release = pressure.admit(indexName, bytes());
|
|
10022
|
-
try {
|
|
10023
|
-
return await work();
|
|
10024
|
-
} finally {
|
|
10025
|
-
release();
|
|
10026
|
-
}
|
|
10027
|
-
}
|
|
10028
|
-
function estimateDocumentBytes(document) {
|
|
10029
|
-
let bytes = 0;
|
|
10030
|
-
const pending = [document];
|
|
10031
|
-
const visited = /* @__PURE__ */ new Set();
|
|
10032
|
-
while (pending.length > 0) {
|
|
10033
|
-
const value = pending.pop();
|
|
10034
|
-
if (typeof value === "string") {
|
|
10035
|
-
bytes += value.length;
|
|
10036
|
-
continue;
|
|
10037
|
-
}
|
|
10038
|
-
if (typeof value === "number" || typeof value === "bigint") {
|
|
10039
|
-
bytes += DOCUMENT_NUMBER_BYTES;
|
|
10040
|
-
continue;
|
|
10041
|
-
}
|
|
10042
|
-
if (value === null || typeof value !== "object") {
|
|
10043
|
-
bytes += DOCUMENT_SCALAR_BYTES;
|
|
10044
|
-
continue;
|
|
10045
|
-
}
|
|
10046
|
-
if (visited.has(value)) {
|
|
10047
|
-
continue;
|
|
10048
|
-
}
|
|
10049
|
-
visited.add(value);
|
|
10050
|
-
if (ArrayBuffer.isView(value)) {
|
|
10051
|
-
bytes += value.byteLength;
|
|
10052
|
-
} else if (Array.isArray(value)) {
|
|
10053
|
-
bytes += DOCUMENT_ENTRY_OVERHEAD_BYTES;
|
|
10054
|
-
for (const item of value) {
|
|
10055
|
-
pending.push(item);
|
|
10056
|
-
}
|
|
10057
|
-
} else {
|
|
10058
|
-
for (const [key, item] of Object.entries(value)) {
|
|
10059
|
-
bytes += key.length + DOCUMENT_ENTRY_OVERHEAD_BYTES;
|
|
10060
|
-
pending.push(item);
|
|
10061
|
-
}
|
|
10062
|
-
}
|
|
10063
|
-
}
|
|
10064
|
-
return bytes;
|
|
10065
|
-
}
|
|
10066
|
-
function estimateDocumentsBytes(documents) {
|
|
10067
|
-
let bytes = 0;
|
|
10068
|
-
for (const document of documents) {
|
|
10069
|
-
bytes += estimateDocumentBytes(document);
|
|
10070
|
-
}
|
|
10071
|
-
return bytes;
|
|
10072
|
-
}
|
|
10073
|
-
function estimateForwardedBytes(operation) {
|
|
10074
|
-
const documentBytes = operation.document?.byteLength ?? 0;
|
|
10075
|
-
return operation.updateFields === null ? documentBytes : documentBytes + estimateDocumentBytes(operation.updateFields);
|
|
10076
|
-
}
|
|
10077
|
-
function transportableDetails(error) {
|
|
10078
|
-
if (!(error instanceof NarsilError)) {
|
|
10079
|
-
return void 0;
|
|
10080
|
-
}
|
|
10081
|
-
try {
|
|
10082
|
-
encode(error.details);
|
|
10083
|
-
return error.details;
|
|
10084
|
-
} catch {
|
|
10085
|
-
return void 0;
|
|
10086
|
-
}
|
|
10087
|
-
}
|
|
10088
|
-
function encodeErrorReply(code, message, details) {
|
|
10089
|
-
if (details !== void 0) {
|
|
10090
|
-
try {
|
|
10091
|
-
return encode({ error: true, code, message, details });
|
|
10092
|
-
} catch {
|
|
10093
|
-
return encode({ error: true, code, message });
|
|
10094
|
-
}
|
|
10095
|
-
}
|
|
10096
|
-
return encode({ error: true, code, message });
|
|
10097
|
-
}
|
|
10098
|
-
function encodeErrorReplyFrom(error, fallbackCode) {
|
|
10099
|
-
return encodeErrorReply(
|
|
10100
|
-
error instanceof NarsilError ? error.code : fallbackCode,
|
|
10101
|
-
error instanceof Error ? error.message : String(error),
|
|
10102
|
-
error instanceof NarsilError ? error.details : void 0
|
|
10103
|
-
);
|
|
10104
|
-
}
|
|
10105
|
-
function isPlainRecord(value) {
|
|
10106
|
-
if (value === null || typeof value !== "object" || Array.isArray(value)) {
|
|
10107
|
-
return false;
|
|
10108
|
-
}
|
|
10109
|
-
const prototype = Object.getPrototypeOf(value);
|
|
10110
|
-
return prototype === Object.prototype || prototype === null;
|
|
10111
|
-
}
|
|
10112
|
-
function remoteErrorDetails(value) {
|
|
10113
|
-
return isPlainRecord(value) ? value : {};
|
|
10114
|
-
}
|
|
10115
|
-
|
|
10116
|
-
// src/distribution/cluster-node/write-routing/forwarding.ts
|
|
10117
10075
|
function assertForwardResponse(response, indexName, primaryNodeId) {
|
|
10118
10076
|
const decoded = decode(response.payload);
|
|
10119
10077
|
if (response.type.endsWith(".error") || response.type === "error" || decoded.error === true) {
|
|
@@ -12270,10 +12228,6 @@ async function sendToNode2(config, targetNodeId, message, failure) {
|
|
|
12270
12228
|
details: { targetNodeId, ...failure?.details }
|
|
12271
12229
|
});
|
|
12272
12230
|
}
|
|
12273
|
-
function fetchFieldsFor(projection) {
|
|
12274
|
-
if (projection.kind !== "fields" || projection.include === null) return null;
|
|
12275
|
-
return projection.include.map((path) => path.join("."));
|
|
12276
|
-
}
|
|
12277
12231
|
function routeDocumentIds(docIds, allocation) {
|
|
12278
12232
|
const partitionCount = allocation.assignments.size;
|
|
12279
12233
|
const routing = {
|
|
@@ -12304,6 +12258,18 @@ function routeDocumentIds(docIds, allocation) {
|
|
|
12304
12258
|
}
|
|
12305
12259
|
return { routing, idsByPartition };
|
|
12306
12260
|
}
|
|
12261
|
+
function readRejectionOf(decoded, targetNodeId, indexName) {
|
|
12262
|
+
const details = { ...remoteErrorDetails(decoded.details), indexName, targetNodeId };
|
|
12263
|
+
return nodeRejectionOf(decoded.code, decoded.message, details) ?? new NarsilError(
|
|
12264
|
+
typeof decoded.code === "string" ? decoded.code : ErrorCodes.QUERY_ROUTING_FAILED,
|
|
12265
|
+
typeof decoded.message === "string" ? decoded.message : `Node '${targetNodeId}' rejected a read request for index '${indexName}'`,
|
|
12266
|
+
details
|
|
12267
|
+
);
|
|
12268
|
+
}
|
|
12269
|
+
function refuseDecodeWithoutRoom(payload) {
|
|
12270
|
+
const refusal = answerRefusalOf(payload.byteLength * ANSWER_DECODE_HEAP_BYTES_PER_PAYLOAD_BYTE);
|
|
12271
|
+
if (refusal !== null) throw markNodeRejection(refusal);
|
|
12272
|
+
}
|
|
12307
12273
|
async function readFromCopies(config, nodeId, engine, indexName, documentRouting, fields) {
|
|
12308
12274
|
const documents = /* @__PURE__ */ new Map();
|
|
12309
12275
|
await sendToEveryPartition(documentRouting.routing, async (targetNodeId, partitionIds) => {
|
|
@@ -12318,14 +12284,19 @@ async function readFromCopies(config, nodeId, engine, indexName, documentRouting
|
|
|
12318
12284
|
);
|
|
12319
12285
|
for (const [docId, document] of local) found.set(docId, document);
|
|
12320
12286
|
} else {
|
|
12321
|
-
for (let start = 0; start < documentIds.length; start +=
|
|
12322
|
-
const chunk = documentIds.slice(start, start +
|
|
12287
|
+
for (let start = 0; start < documentIds.length; start += FETCH_GROUP_DOCUMENT_IDS) {
|
|
12288
|
+
const chunk = documentIds.slice(start, start + FETCH_GROUP_DOCUMENT_IDS);
|
|
12323
12289
|
const fetchMessage = createFetchMessage({ indexName, documentIds: chunk, fields, highlight: null }, nodeId);
|
|
12324
12290
|
const response = await sendToNode2(config, targetNodeId, fetchMessage, {
|
|
12325
12291
|
message: `Node '${targetNodeId}' did not answer a document fetch for index '${indexName}'`,
|
|
12326
12292
|
details: { indexName }
|
|
12327
12293
|
});
|
|
12328
|
-
|
|
12294
|
+
refuseDecodeWithoutRoom(response.payload);
|
|
12295
|
+
const decoded = decode(response.payload);
|
|
12296
|
+
if (response.type.endsWith(".error") || decoded.error === true) {
|
|
12297
|
+
throw readRejectionOf(decoded, targetNodeId, indexName);
|
|
12298
|
+
}
|
|
12299
|
+
const payload = validateFetchResultPayload(decoded);
|
|
12329
12300
|
for (const fetched of payload.documents) {
|
|
12330
12301
|
found.set(fetched.docId, restoreVectorValues(fetched.document));
|
|
12331
12302
|
}
|
|
@@ -12336,30 +12307,20 @@ async function readFromCopies(config, nodeId, engine, indexName, documentRouting
|
|
|
12336
12307
|
});
|
|
12337
12308
|
return documents;
|
|
12338
12309
|
}
|
|
12339
|
-
async function
|
|
12340
|
-
const documents = /* @__PURE__ */ new Map();
|
|
12341
|
-
if (projection.kind === "none") return documents;
|
|
12342
|
-
const documentRouting = routeDocumentIds(
|
|
12343
|
-
result.scored.map((entry) => entry.docId),
|
|
12344
|
-
allocation
|
|
12345
|
-
);
|
|
12346
|
-
const found = await readFromCopies(config, nodeId, engine, indexName, documentRouting, fetchFieldsFor(projection));
|
|
12347
|
-
for (const [docId, document] of found) {
|
|
12348
|
-
documents.set(docId, applyProjection(document, projection));
|
|
12349
|
-
}
|
|
12350
|
-
return documents;
|
|
12351
|
-
}
|
|
12352
|
-
async function readDistributedDocuments(config, nodeId, engine, indexName, docIds, allocation) {
|
|
12310
|
+
async function readDocumentsFromCopies(config, nodeId, engine, indexName, docIds, allocation, fields, requireEveryPartition) {
|
|
12353
12311
|
const documentRouting = routeDocumentIds(docIds, allocation);
|
|
12354
12312
|
const unreachablePartitions = documentRouting.routing.unavailablePartitions;
|
|
12355
|
-
if (unreachablePartitions.length > 0) {
|
|
12313
|
+
if (requireEveryPartition && unreachablePartitions.length > 0) {
|
|
12356
12314
|
throw new NarsilError(
|
|
12357
12315
|
ErrorCodes.QUERY_NO_ACTIVE_REPLICA,
|
|
12358
12316
|
`No active replica serves one or more partitions of index '${indexName}'`,
|
|
12359
12317
|
{ indexName, partitionIds: [...unreachablePartitions].sort((a, b) => a - b) }
|
|
12360
12318
|
);
|
|
12361
12319
|
}
|
|
12362
|
-
return readFromCopies(config, nodeId, engine, indexName, documentRouting,
|
|
12320
|
+
return readFromCopies(config, nodeId, engine, indexName, documentRouting, fields);
|
|
12321
|
+
}
|
|
12322
|
+
async function readDistributedDocuments(config, nodeId, engine, indexName, docIds, allocation) {
|
|
12323
|
+
return readDocumentsFromCopies(config, nodeId, engine, indexName, docIds, allocation, null, true);
|
|
12363
12324
|
}
|
|
12364
12325
|
|
|
12365
12326
|
// src/distribution/cluster-node/synonym-sets.ts
|
|
@@ -13290,7 +13251,8 @@ async function handleIndexSettings(message, respond, deps) {
|
|
|
13290
13251
|
var DEFAULT_QUERY_CONFIG = {
|
|
13291
13252
|
allowPartialResults: true,
|
|
13292
13253
|
partitionTimeout: 5e3,
|
|
13293
|
-
defaultFacetSize: 10
|
|
13254
|
+
defaultFacetSize: 10,
|
|
13255
|
+
searchContexts: false
|
|
13294
13256
|
};
|
|
13295
13257
|
|
|
13296
13258
|
// src/distribution/cluster-node/query-conversion.ts
|
|
@@ -13522,6 +13484,93 @@ function convertWireFacetsToLocal(result, wireFacets, requested) {
|
|
|
13522
13484
|
return converted;
|
|
13523
13485
|
}
|
|
13524
13486
|
|
|
13487
|
+
// src/distribution/cluster-node/search-contexts/departures.ts
|
|
13488
|
+
function releaseContextsOfDepartedNodes(coordinator, registry) {
|
|
13489
|
+
return coordinator.watchNodes((event) => {
|
|
13490
|
+
if (event.type === "node_left") registry.releaseCoordinator(event.nodeId);
|
|
13491
|
+
});
|
|
13492
|
+
}
|
|
13493
|
+
|
|
13494
|
+
// src/distribution/cluster-node/search-contexts/documents.ts
|
|
13495
|
+
function documentOfSnapshot(snapshot) {
|
|
13496
|
+
const document = structuredClone(snapshot.fields);
|
|
13497
|
+
for (const [fieldPath2, vector] of snapshot.vectors) setNestedValue(document, fieldPath2, new Float32Array(vector));
|
|
13498
|
+
return document;
|
|
13499
|
+
}
|
|
13500
|
+
function heldBytesOf(snapshots) {
|
|
13501
|
+
let bytes = 0;
|
|
13502
|
+
for (const snapshot of snapshots.values()) {
|
|
13503
|
+
bytes += SEARCH_CONTEXT_ENTRY_HEAP_BYTES;
|
|
13504
|
+
for (const [, vector] of snapshot.vectors) bytes += vector.byteLength;
|
|
13505
|
+
}
|
|
13506
|
+
return bytes;
|
|
13507
|
+
}
|
|
13508
|
+
|
|
13509
|
+
// src/distribution/cluster-node/search-contexts/registry.ts
|
|
13510
|
+
function keepAliveOf(requested) {
|
|
13511
|
+
return Math.min(requested, SEARCH_CONTEXT_KEEP_ALIVE_LIMIT_MS);
|
|
13512
|
+
}
|
|
13513
|
+
function createSearchContextRegistry(now3 = () => performance.now()) {
|
|
13514
|
+
const contexts = /* @__PURE__ */ new Map();
|
|
13515
|
+
let reaper = null;
|
|
13516
|
+
function reap() {
|
|
13517
|
+
const at = now3();
|
|
13518
|
+
for (const [contextId, context] of contexts) {
|
|
13519
|
+
if (context.expiresAt <= at) contexts.delete(contextId);
|
|
13520
|
+
}
|
|
13521
|
+
if (contexts.size === 0) stopReaper();
|
|
13522
|
+
}
|
|
13523
|
+
function startReaper() {
|
|
13524
|
+
if (reaper !== null) return;
|
|
13525
|
+
reaper = setInterval(reap, SEARCH_CONTEXT_REAPER_INTERVAL_MS);
|
|
13526
|
+
if (typeof reaper === "object" && "unref" in reaper) reaper.unref();
|
|
13527
|
+
}
|
|
13528
|
+
function stopReaper() {
|
|
13529
|
+
if (reaper === null) return;
|
|
13530
|
+
clearInterval(reaper);
|
|
13531
|
+
reaper = null;
|
|
13532
|
+
}
|
|
13533
|
+
return {
|
|
13534
|
+
open(coordinatorNodeId, keepAliveMs, documents) {
|
|
13535
|
+
const contextId = generateId();
|
|
13536
|
+
contexts.set(contextId, { coordinatorNodeId, keepAliveMs, expiresAt: now3() + keepAliveMs, documents });
|
|
13537
|
+
startReaper();
|
|
13538
|
+
return contextId;
|
|
13539
|
+
},
|
|
13540
|
+
read(contextId, docIds) {
|
|
13541
|
+
const context = contexts.get(contextId);
|
|
13542
|
+
if (context === void 0) return null;
|
|
13543
|
+
const at = now3();
|
|
13544
|
+
if (context.expiresAt <= at) {
|
|
13545
|
+
contexts.delete(contextId);
|
|
13546
|
+
return null;
|
|
13547
|
+
}
|
|
13548
|
+
context.expiresAt = at + context.keepAliveMs;
|
|
13549
|
+
const found = /* @__PURE__ */ new Map();
|
|
13550
|
+
for (const docId of docIds) {
|
|
13551
|
+
const snapshot = context.documents.get(docId);
|
|
13552
|
+
if (snapshot !== void 0) found.set(docId, snapshot);
|
|
13553
|
+
}
|
|
13554
|
+
return found;
|
|
13555
|
+
},
|
|
13556
|
+
release(contextId) {
|
|
13557
|
+
contexts.delete(contextId);
|
|
13558
|
+
if (contexts.size === 0) stopReaper();
|
|
13559
|
+
},
|
|
13560
|
+
releaseCoordinator(coordinatorNodeId) {
|
|
13561
|
+
for (const [contextId, context] of contexts) {
|
|
13562
|
+
if (context.coordinatorNodeId === coordinatorNodeId) contexts.delete(contextId);
|
|
13563
|
+
}
|
|
13564
|
+
if (contexts.size === 0) stopReaper();
|
|
13565
|
+
},
|
|
13566
|
+
openCount: () => contexts.size,
|
|
13567
|
+
stop() {
|
|
13568
|
+
contexts.clear();
|
|
13569
|
+
stopReaper();
|
|
13570
|
+
}
|
|
13571
|
+
};
|
|
13572
|
+
}
|
|
13573
|
+
|
|
13525
13574
|
// src/distribution/cluster-node/message-handler/queries.ts
|
|
13526
13575
|
async function handleSearch(message, respond, deps) {
|
|
13527
13576
|
const decoded = decode(message.payload);
|
|
@@ -13531,7 +13580,9 @@ async function handleSearch(message, respond, deps) {
|
|
|
13531
13580
|
queryParams.group = { ...queryParams.group, limit: oversampledShardSize(queryParams.group.limit) };
|
|
13532
13581
|
}
|
|
13533
13582
|
const rescoreWindow = payload.params.vector?.rescoreWindow ?? void 0;
|
|
13534
|
-
|
|
13583
|
+
const keepAliveMs = payload.keepAliveMs ?? null;
|
|
13584
|
+
const snapshots = keepAliveMs === null || deps.searchContexts === void 0 ? void 0 : /* @__PURE__ */ new Map();
|
|
13585
|
+
if (snapshots !== void 0 || rescoreWindow !== void 0 && queryParams.sort === void 0 && queryParams.group === void 0) {
|
|
13535
13586
|
queryParams.document = false;
|
|
13536
13587
|
}
|
|
13537
13588
|
const queryResult = await deps.engine.queryPartitions(
|
|
@@ -13539,15 +13590,17 @@ async function handleSearch(message, respond, deps) {
|
|
|
13539
13590
|
queryParams,
|
|
13540
13591
|
payload.partitionIds,
|
|
13541
13592
|
payload.globalStats ?? void 0,
|
|
13542
|
-
rescoreWindow
|
|
13593
|
+
rescoreWindow,
|
|
13594
|
+
snapshots
|
|
13543
13595
|
);
|
|
13596
|
+
const rankedDocument = (hit) => snapshots?.get(hit.id)?.fields ?? hit.document;
|
|
13544
13597
|
const normalizedSort = queryParams.sort !== void 0 ? normalizeSort(queryParams.sort) : null;
|
|
13545
13598
|
const sortFields = normalizedSort === null ? null : normalizedSort.map((entry) => entry.field);
|
|
13546
13599
|
const sortModes = normalizedSort === null ? [] : sortModesOf(normalizedSort);
|
|
13547
13600
|
const scored = queryResult.hits.map((hit) => ({
|
|
13548
13601
|
docId: hit.id,
|
|
13549
13602
|
score: hit.score ?? null,
|
|
13550
|
-
sortValues: sortFields !== null ? readSortValues(hit
|
|
13603
|
+
sortValues: sortFields !== null ? readSortValues(rankedDocument(hit), sortFields, sortModes).map(toWireSortValue) : null
|
|
13551
13604
|
}));
|
|
13552
13605
|
const results = payload.partitionIds.map((partitionId) => ({
|
|
13553
13606
|
partitionId,
|
|
@@ -13558,8 +13611,18 @@ async function handleSearch(message, respond, deps) {
|
|
|
13558
13611
|
results,
|
|
13559
13612
|
facets: convertLocalFacetsToWire2(queryResult.facets),
|
|
13560
13613
|
facetErrorBounds: convertLocalFacetBoundsToWire(queryResult.facets),
|
|
13561
|
-
groups: await convertLocalGroupsToWire(
|
|
13614
|
+
groups: await convertLocalGroupsToWire(
|
|
13615
|
+
queryResult.groups,
|
|
13616
|
+
sortFields,
|
|
13617
|
+
sortModes,
|
|
13618
|
+
deps,
|
|
13619
|
+
payload.indexName,
|
|
13620
|
+
snapshots
|
|
13621
|
+
)
|
|
13562
13622
|
};
|
|
13623
|
+
if (snapshots !== void 0 && keepAliveMs !== null) {
|
|
13624
|
+
resultPayload.contextId = openSearchContext(deps, message.sourceId, keepAliveMs, snapshots);
|
|
13625
|
+
}
|
|
13563
13626
|
await respond({
|
|
13564
13627
|
type: QueryMessageTypes.SEARCH_RESULT,
|
|
13565
13628
|
sourceId: deps.nodeId,
|
|
@@ -13567,17 +13630,17 @@ async function handleSearch(message, respond, deps) {
|
|
|
13567
13630
|
payload: encodeDocumentReply(resultPayload)
|
|
13568
13631
|
});
|
|
13569
13632
|
}
|
|
13570
|
-
async function convertLocalGroupsToWire(groups, sortFields, sortModes, deps, indexName) {
|
|
13633
|
+
async function convertLocalGroupsToWire(groups, sortFields, sortModes, deps, indexName, snapshots) {
|
|
13571
13634
|
if (groups === void 0) {
|
|
13572
13635
|
return null;
|
|
13573
13636
|
}
|
|
13574
|
-
const unreadIds = sortFields === null ? [] : groups.flatMap((group) => group.hits.filter((hit) => hit.document === void 0).map((hit) => hit.id));
|
|
13637
|
+
const unreadIds = sortFields === null || snapshots !== void 0 ? [] : groups.flatMap((group) => group.hits.filter((hit) => hit.document === void 0).map((hit) => hit.id));
|
|
13575
13638
|
const readDocuments = unreadIds.length > 0 ? await deps.engine.getMultiple(indexName, unreadIds) : /* @__PURE__ */ new Map();
|
|
13576
13639
|
const wireGroups = [];
|
|
13577
13640
|
for (const group of groups) {
|
|
13578
13641
|
const scored = group.hits.map((hit) => {
|
|
13579
13642
|
if (sortFields === null) return { docId: hit.id, score: hit.score ?? null, sortValues: null };
|
|
13580
|
-
const document = hit.document ?? readDocuments.get(hit.id);
|
|
13643
|
+
const document = snapshots?.get(hit.id)?.fields ?? hit.document ?? readDocuments.get(hit.id);
|
|
13581
13644
|
const sortValues = readSortValues(document, sortFields, sortModes).map(toWireSortValue);
|
|
13582
13645
|
return { docId: hit.id, score: hit.score ?? null, sortValues };
|
|
13583
13646
|
});
|
|
@@ -13589,9 +13652,21 @@ async function handleFetch(message, respond, deps) {
|
|
|
13589
13652
|
const decoded = decode(message.payload);
|
|
13590
13653
|
const payload = validateFetchPayload(decoded);
|
|
13591
13654
|
const projection = resolveProjection(payload.fields === null ? void 0 : { include: payload.fields });
|
|
13655
|
+
const contextId = payload.contextId ?? null;
|
|
13656
|
+
const documents = contextId === null ? await readCurrentDocuments(deps, payload.indexName, payload.documentIds, projection) : readContextDocuments(deps, contextId, payload.documentIds, payload.releaseContext !== false, projection);
|
|
13657
|
+
refuseEncodeWithoutRoom(documents.map((fetched) => fetched.document));
|
|
13658
|
+
const resultPayload = { documents };
|
|
13659
|
+
await respond({
|
|
13660
|
+
type: QueryMessageTypes.FETCH_RESULT,
|
|
13661
|
+
sourceId: deps.nodeId,
|
|
13662
|
+
requestId: message.requestId,
|
|
13663
|
+
payload: encodeDocumentReply(resultPayload)
|
|
13664
|
+
});
|
|
13665
|
+
}
|
|
13666
|
+
async function readCurrentDocuments(deps, indexName, documentIds, projection) {
|
|
13592
13667
|
const documents = [];
|
|
13593
|
-
for (const docRef of
|
|
13594
|
-
const doc = await deps.engine.get(
|
|
13668
|
+
for (const docRef of documentIds) {
|
|
13669
|
+
const doc = await deps.engine.get(indexName, docRef.docId);
|
|
13595
13670
|
if (doc !== void 0) {
|
|
13596
13671
|
documents.push({
|
|
13597
13672
|
docId: docRef.docId,
|
|
@@ -13600,13 +13675,37 @@ async function handleFetch(message, respond, deps) {
|
|
|
13600
13675
|
});
|
|
13601
13676
|
}
|
|
13602
13677
|
}
|
|
13603
|
-
|
|
13604
|
-
|
|
13605
|
-
|
|
13606
|
-
|
|
13607
|
-
|
|
13608
|
-
|
|
13609
|
-
|
|
13678
|
+
return documents;
|
|
13679
|
+
}
|
|
13680
|
+
function readContextDocuments(deps, contextId, documentIds, releaseContext, projection) {
|
|
13681
|
+
const docIds = documentIds.map((docRef) => docRef.docId);
|
|
13682
|
+
const found = deps.searchContexts?.read(contextId, docIds) ?? null;
|
|
13683
|
+
if (releaseContext) deps.searchContexts?.release(contextId);
|
|
13684
|
+
if (found === null) {
|
|
13685
|
+
if (docIds.length === 0) return [];
|
|
13686
|
+
throw new NarsilError(
|
|
13687
|
+
ErrorCodes.SEARCH_CONTEXT_MISSING,
|
|
13688
|
+
`Node '${deps.nodeId}' holds no search context '${contextId}', because it expired, the node restarted, or the coordinating node released it`,
|
|
13689
|
+
{ contextId, nodeId: deps.nodeId }
|
|
13690
|
+
);
|
|
13691
|
+
}
|
|
13692
|
+
const documents = [];
|
|
13693
|
+
for (const docId of docIds) {
|
|
13694
|
+
const snapshot = found.get(docId);
|
|
13695
|
+
if (snapshot === void 0) continue;
|
|
13696
|
+
documents.push({
|
|
13697
|
+
docId,
|
|
13698
|
+
document: applyProjection(documentOfSnapshot(snapshot), projection),
|
|
13699
|
+
highlights: null
|
|
13700
|
+
});
|
|
13701
|
+
}
|
|
13702
|
+
return documents;
|
|
13703
|
+
}
|
|
13704
|
+
function openSearchContext(deps, coordinatorNodeId, keepAliveMs, snapshots) {
|
|
13705
|
+
if (deps.searchContexts === void 0 || snapshots.size === 0) return null;
|
|
13706
|
+
const refusal = answerRefusalOf(heldBytesOf(snapshots));
|
|
13707
|
+
if (refusal !== null) throw refusal;
|
|
13708
|
+
return deps.searchContexts.open(coordinatorNodeId, keepAliveOf(keepAliveMs), snapshots);
|
|
13610
13709
|
}
|
|
13611
13710
|
async function handleStats(message, respond, deps) {
|
|
13612
13711
|
const decoded = decode(message.payload);
|
|
@@ -13679,6 +13778,7 @@ async function handleList(message, respond, deps) {
|
|
|
13679
13778
|
document: listed.document,
|
|
13680
13779
|
sortValues: sortFields === null ? null : readSortValues(listed.document, sortFields, sortModes).map(toWireSortValue2)
|
|
13681
13780
|
}));
|
|
13781
|
+
refuseEncodeWithoutRoom(entries.map((entry) => entry.document));
|
|
13682
13782
|
await respond(
|
|
13683
13783
|
createListResultMessage(
|
|
13684
13784
|
{ entries, total: result.total, hasMore: result.cursor !== null },
|
|
@@ -14595,10 +14695,85 @@ async function resolveNodeId(explicitNodeId, directory) {
|
|
|
14595
14695
|
});
|
|
14596
14696
|
}
|
|
14597
14697
|
}
|
|
14698
|
+
|
|
14699
|
+
// src/distribution/query/node-send.ts
|
|
14700
|
+
async function resolveTargets(nodeId, deps) {
|
|
14701
|
+
if (deps.resolveNodeTargets === void 0) {
|
|
14702
|
+
return [nodeId];
|
|
14703
|
+
}
|
|
14704
|
+
const targets = await deps.resolveNodeTargets(nodeId);
|
|
14705
|
+
return targets.length > 0 ? targets : [nodeId];
|
|
14706
|
+
}
|
|
14707
|
+
async function sendToFirstReachableTarget2(transport, targets, message) {
|
|
14708
|
+
let lastError;
|
|
14709
|
+
for (const target of targets) {
|
|
14710
|
+
try {
|
|
14711
|
+
return await transport.send(target, message);
|
|
14712
|
+
} catch (error) {
|
|
14713
|
+
lastError = error;
|
|
14714
|
+
}
|
|
14715
|
+
}
|
|
14716
|
+
throw lastError instanceof Error ? lastError : new Error(String(lastError));
|
|
14717
|
+
}
|
|
14718
|
+
|
|
14719
|
+
// src/distribution/query/search-contexts.ts
|
|
14720
|
+
function searchContextKeepAlive(routing, partitionTimeout, transport) {
|
|
14721
|
+
let copies = 1;
|
|
14722
|
+
for (const partitionCopies of routing.copiesByPartition.values()) copies = Math.max(copies, partitionCopies.length);
|
|
14723
|
+
const requestTimeout = transport.timeouts?.requestTimeout ?? DEFAULT_TRANSPORT_CONFIG.requestTimeout;
|
|
14724
|
+
const longestGap = partitionTimeout * copies + 2 * requestTimeout;
|
|
14725
|
+
return Math.min(SEARCH_CONTEXT_KEEP_ALIVE_LIMIT_MS, Math.max(SEARCH_CONTEXT_KEEP_ALIVE_MS, longestGap));
|
|
14726
|
+
}
|
|
14727
|
+
function collectSearchContexts(...outcomeSets) {
|
|
14728
|
+
const sources = /* @__PURE__ */ new Map();
|
|
14729
|
+
const opened = [];
|
|
14730
|
+
for (const outcomes of outcomeSets) {
|
|
14731
|
+
for (const outcome of outcomes) {
|
|
14732
|
+
const contextId = outcome.status === "success" ? outcome.results?.contextId ?? null : null;
|
|
14733
|
+
if (contextId === null || outcome.results === null) continue;
|
|
14734
|
+
const context = { nodeId: outcome.nodeId, contextId };
|
|
14735
|
+
opened.push(context);
|
|
14736
|
+
for (const partition of outcome.results.results) {
|
|
14737
|
+
for (const entry of partition.scored) if (!sources.has(entry.docId)) sources.set(entry.docId, context);
|
|
14738
|
+
}
|
|
14739
|
+
for (const group of outcome.results.groups ?? []) {
|
|
14740
|
+
for (const entry of group.scored) if (!sources.has(entry.docId)) sources.set(entry.docId, context);
|
|
14741
|
+
}
|
|
14742
|
+
}
|
|
14743
|
+
}
|
|
14744
|
+
return { sources, opened };
|
|
14745
|
+
}
|
|
14746
|
+
function releaseSearchContexts(indexName, contexts, deps) {
|
|
14747
|
+
for (const context of contexts) {
|
|
14748
|
+
const message = createFetchMessage(
|
|
14749
|
+
{
|
|
14750
|
+
indexName,
|
|
14751
|
+
documentIds: [],
|
|
14752
|
+
fields: null,
|
|
14753
|
+
highlight: null,
|
|
14754
|
+
contextId: context.contextId,
|
|
14755
|
+
releaseContext: true
|
|
14756
|
+
},
|
|
14757
|
+
deps.sourceNodeId
|
|
14758
|
+
);
|
|
14759
|
+
resolveTargets(context.nodeId, deps).then((targets) => sendToFirstReachableTarget2(deps.transport, targets, message)).catch(() => void 0);
|
|
14760
|
+
}
|
|
14761
|
+
}
|
|
14762
|
+
|
|
14763
|
+
// src/distribution/query/fan-out.ts
|
|
14598
14764
|
function queryRejectionOf(response, decoded) {
|
|
14599
14765
|
const body = decoded;
|
|
14600
14766
|
if (!response.type.endsWith(".error") && body?.error !== true) return null;
|
|
14601
|
-
return nodeRejectionOf(body?.code, body?.message);
|
|
14767
|
+
if (body?.code !== ServerErrorCodes.TOO_MANY_REQUESTS) return nodeRejectionOf(body?.code, body?.message);
|
|
14768
|
+
const details = typeof body.details === "object" && body.details !== null ? { ...body.details } : {};
|
|
14769
|
+
return new NarsilError(
|
|
14770
|
+
ServerErrorCodes.TOO_MANY_REQUESTS,
|
|
14771
|
+
typeof body.message === "string" ? body.message : "A node had no room in memory to answer its part of the search",
|
|
14772
|
+
details
|
|
14773
|
+
);
|
|
14774
|
+
}
|
|
14775
|
+
function isMemoryRefusal(error) {
|
|
14776
|
+
return error instanceof NarsilError && error.code === ServerErrorCodes.TOO_MANY_REQUESTS;
|
|
14602
14777
|
}
|
|
14603
14778
|
function statsExpansionOf(params) {
|
|
14604
14779
|
return {
|
|
@@ -14667,10 +14842,17 @@ async function collectDistributedStats(indexName, params, routing, deps, config)
|
|
|
14667
14842
|
const leftOut = await countLeftOutTokens(indexName, answered, deps, config);
|
|
14668
14843
|
return mergePartitionStats([...answered.map(([, { result }]) => result), ...leftOut]);
|
|
14669
14844
|
}
|
|
14670
|
-
async function fanOutSearch(indexName, params, globalStats, facetShardSize, routing, deps, config) {
|
|
14845
|
+
async function fanOutSearch(indexName, params, globalStats, facetShardSize, routing, deps, config, keepAliveMs = null) {
|
|
14671
14846
|
const outcomes = await sendWithFailover(routing, async (nodeId, partitionIds) => {
|
|
14672
14847
|
const message = createSearchMessage(
|
|
14673
|
-
{
|
|
14848
|
+
{
|
|
14849
|
+
indexName,
|
|
14850
|
+
partitionIds,
|
|
14851
|
+
params,
|
|
14852
|
+
globalStats,
|
|
14853
|
+
facetShardSize,
|
|
14854
|
+
...keepAliveMs === null ? {} : { keepAliveMs }
|
|
14855
|
+
},
|
|
14674
14856
|
deps.sourceNodeId
|
|
14675
14857
|
);
|
|
14676
14858
|
const targets = await resolveTargets(nodeId, deps);
|
|
@@ -14679,11 +14861,21 @@ async function fanOutSearch(indexName, params, globalStats, facetShardSize, rout
|
|
|
14679
14861
|
targets,
|
|
14680
14862
|
message,
|
|
14681
14863
|
config.partitionTimeout,
|
|
14682
|
-
validateSearchResultPayload
|
|
14864
|
+
validateSearchResultPayload,
|
|
14865
|
+
(late) => {
|
|
14866
|
+
if (typeof late.contextId === "string") {
|
|
14867
|
+
releaseSearchContexts(indexName, [{ nodeId, contextId: late.contextId }], deps);
|
|
14868
|
+
}
|
|
14869
|
+
}
|
|
14683
14870
|
);
|
|
14684
14871
|
});
|
|
14685
|
-
const
|
|
14686
|
-
|
|
14872
|
+
const memoryRefusal = outcomes.find((outcome) => isMemoryRefusal(outcome.error));
|
|
14873
|
+
const everyPartitionFailed = outcomes.every((outcome) => outcome.status !== "success");
|
|
14874
|
+
const rejection = outcomes.find((outcome) => isNodeRejection(outcome.error)) ?? (!config.allowPartialResults || everyPartitionFailed ? memoryRefusal : void 0);
|
|
14875
|
+
if (rejection !== void 0) {
|
|
14876
|
+
releaseSearchContexts(indexName, openedContextsOf(outcomes), deps);
|
|
14877
|
+
throw rejection.error;
|
|
14878
|
+
}
|
|
14687
14879
|
return outcomes.map((outcome) => ({
|
|
14688
14880
|
nodeId: outcome.nodeId,
|
|
14689
14881
|
partitionIds: outcome.partitionIds,
|
|
@@ -14691,6 +14883,14 @@ async function fanOutSearch(indexName, params, globalStats, facetShardSize, rout
|
|
|
14691
14883
|
results: outcome.result
|
|
14692
14884
|
}));
|
|
14693
14885
|
}
|
|
14886
|
+
function openedContextsOf(outcomes) {
|
|
14887
|
+
const opened = [];
|
|
14888
|
+
for (const outcome of outcomes) {
|
|
14889
|
+
const contextId = outcome.result?.contextId;
|
|
14890
|
+
if (typeof contextId === "string") opened.push({ nodeId: outcome.nodeId, contextId });
|
|
14891
|
+
}
|
|
14892
|
+
return opened;
|
|
14893
|
+
}
|
|
14694
14894
|
var OUTCOME_SEVERITY = { success: 0, timeout: 1, failed: 2 };
|
|
14695
14895
|
function buildCoverage(totalPartitions, unavailablePartitions, ...outcomeSets) {
|
|
14696
14896
|
const worstByPartition = /* @__PURE__ */ new Map();
|
|
@@ -14726,7 +14926,7 @@ function buildCoverage(totalPartitions, unavailablePartitions, ...outcomeSets) {
|
|
|
14726
14926
|
failedPartitions
|
|
14727
14927
|
};
|
|
14728
14928
|
}
|
|
14729
|
-
async function sendWithTimeout(transport, targets, message, timeout, validate) {
|
|
14929
|
+
async function sendWithTimeout(transport, targets, message, timeout, validate, onLate) {
|
|
14730
14930
|
let timerId;
|
|
14731
14931
|
const abortPromise = new Promise((resolve) => {
|
|
14732
14932
|
timerId = setTimeout(() => resolve(null), timeout);
|
|
@@ -14741,31 +14941,15 @@ async function sendWithTimeout(transport, targets, message, timeout, validate) {
|
|
|
14741
14941
|
return validate(decoded);
|
|
14742
14942
|
});
|
|
14743
14943
|
try {
|
|
14744
|
-
|
|
14944
|
+
const answered = await Promise.race([sendPromise, abortPromise]);
|
|
14945
|
+
if (answered === null && onLate !== void 0) sendPromise.then(onLate, () => void 0);
|
|
14946
|
+
return answered;
|
|
14745
14947
|
} finally {
|
|
14746
14948
|
if (timerId !== void 0) {
|
|
14747
14949
|
clearTimeout(timerId);
|
|
14748
14950
|
}
|
|
14749
14951
|
}
|
|
14750
14952
|
}
|
|
14751
|
-
async function resolveTargets(nodeId, deps) {
|
|
14752
|
-
if (deps.resolveNodeTargets === void 0) {
|
|
14753
|
-
return [nodeId];
|
|
14754
|
-
}
|
|
14755
|
-
const targets = await deps.resolveNodeTargets(nodeId);
|
|
14756
|
-
return targets.length > 0 ? targets : [nodeId];
|
|
14757
|
-
}
|
|
14758
|
-
async function sendToFirstReachableTarget2(transport, targets, message) {
|
|
14759
|
-
let lastError;
|
|
14760
|
-
for (const target of targets) {
|
|
14761
|
-
try {
|
|
14762
|
-
return await transport.send(target, message);
|
|
14763
|
-
} catch (error) {
|
|
14764
|
-
lastError = error;
|
|
14765
|
-
}
|
|
14766
|
-
}
|
|
14767
|
-
throw lastError instanceof Error ? lastError : new Error(String(lastError));
|
|
14768
|
-
}
|
|
14769
14953
|
|
|
14770
14954
|
// src/distribution/query/merge.ts
|
|
14771
14955
|
function compareByScoreThenId(a, b) {
|
|
@@ -15143,14 +15327,26 @@ async function executeHybridQuery(indexName, params, page, facetShardSize, facet
|
|
|
15143
15327
|
if (params.scoring === "dfs") {
|
|
15144
15328
|
textGlobalStats = await collectDistributedStats(indexName, textParams, routing, deps, config);
|
|
15145
15329
|
}
|
|
15146
|
-
const
|
|
15147
|
-
|
|
15148
|
-
fanOutSearch(indexName,
|
|
15330
|
+
const keepAliveMs = config.searchContexts ? searchContextKeepAlive(routing, config.partitionTimeout, deps.transport) : null;
|
|
15331
|
+
const legs = await Promise.allSettled([
|
|
15332
|
+
fanOutSearch(indexName, textParams, textGlobalStats, facetShardSize, routing, deps, config, keepAliveMs),
|
|
15333
|
+
fanOutSearch(indexName, vectorParams, null, null, routing, deps, config, keepAliveMs)
|
|
15149
15334
|
]);
|
|
15335
|
+
const answered = legs.flatMap((leg) => leg.status === "fulfilled" ? [leg.value] : []);
|
|
15336
|
+
const contexts = collectSearchContexts(...answered);
|
|
15337
|
+
const failedLeg = legs.find((leg) => leg.status === "rejected");
|
|
15338
|
+
if (failedLeg !== void 0) {
|
|
15339
|
+
releaseSearchContexts(indexName, contexts.opened, deps);
|
|
15340
|
+
throw failedLeg.reason;
|
|
15341
|
+
}
|
|
15342
|
+
const [textLeg, vectorLeg] = legs;
|
|
15343
|
+
const textOutcomes = textLeg.status === "fulfilled" ? textLeg.value : [];
|
|
15344
|
+
const vectorOutcomes = vectorLeg.status === "fulfilled" ? vectorLeg.value : [];
|
|
15150
15345
|
const coverage = buildCoverage(totalPartitions, routing.unavailablePartitions, textOutcomes, vectorOutcomes);
|
|
15151
15346
|
if (!config.allowPartialResults) {
|
|
15152
15347
|
const failedCount = coverage.timedOutPartitions + coverage.failedPartitions;
|
|
15153
15348
|
if (failedCount > 0) {
|
|
15349
|
+
releaseSearchContexts(indexName, contexts.opened, deps);
|
|
15154
15350
|
throw new NarsilError(ErrorCodes.QUERY_PARTIAL_FAILURE, "One or more partitions failed during hybrid query", {
|
|
15155
15351
|
coverage
|
|
15156
15352
|
});
|
|
@@ -15212,7 +15408,8 @@ async function executeHybridQuery(indexName, params, page, facetShardSize, facet
|
|
|
15212
15408
|
cursor: paged.cursor,
|
|
15213
15409
|
coverage,
|
|
15214
15410
|
pinnedFromOutside: placement.placedFromOutside,
|
|
15215
|
-
mergeHeldEveryMatch: allMatchesPresent
|
|
15411
|
+
mergeHeldEveryMatch: allMatchesPresent,
|
|
15412
|
+
contexts
|
|
15216
15413
|
};
|
|
15217
15414
|
}
|
|
15218
15415
|
|
|
@@ -15316,7 +15513,18 @@ async function executeSingleFanOut(indexName, params, binding, limit, offset, fa
|
|
|
15316
15513
|
}
|
|
15317
15514
|
const depth = limit + offset;
|
|
15318
15515
|
const searchParams = { ...params, limit: depth, offset: 0 };
|
|
15319
|
-
const
|
|
15516
|
+
const keepAliveMs = config.searchContexts ? searchContextKeepAlive(routing, config.partitionTimeout, deps.transport) : null;
|
|
15517
|
+
const outcomes = await fanOutSearch(
|
|
15518
|
+
indexName,
|
|
15519
|
+
searchParams,
|
|
15520
|
+
globalStats,
|
|
15521
|
+
facetShardSize,
|
|
15522
|
+
routing,
|
|
15523
|
+
deps,
|
|
15524
|
+
config,
|
|
15525
|
+
keepAliveMs
|
|
15526
|
+
);
|
|
15527
|
+
const contexts = collectSearchContexts(outcomes);
|
|
15320
15528
|
const sortFields = params.sort !== null && params.sort.length > 0 ? params.sort : null;
|
|
15321
15529
|
if (sortFields !== null) {
|
|
15322
15530
|
failOutcomesMissingSortValues(outcomes);
|
|
@@ -15325,6 +15533,7 @@ async function executeSingleFanOut(indexName, params, binding, limit, offset, fa
|
|
|
15325
15533
|
if (!config.allowPartialResults) {
|
|
15326
15534
|
const failedCount = coverage.timedOutPartitions + coverage.failedPartitions;
|
|
15327
15535
|
if (failedCount > 0) {
|
|
15536
|
+
releaseSearchContexts(indexName, contexts.opened, deps);
|
|
15328
15537
|
throw new NarsilError(ErrorCodes.QUERY_PARTIAL_FAILURE, "One or more partitions failed during query", {
|
|
15329
15538
|
coverage
|
|
15330
15539
|
});
|
|
@@ -15392,7 +15601,8 @@ async function executeSingleFanOut(indexName, params, binding, limit, offset, fa
|
|
|
15392
15601
|
cursor,
|
|
15393
15602
|
coverage,
|
|
15394
15603
|
pinnedFromOutside: placement.placedFromOutside,
|
|
15395
|
-
mergeHeldEveryMatch: allMatchesPresent
|
|
15604
|
+
mergeHeldEveryMatch: allMatchesPresent,
|
|
15605
|
+
contexts
|
|
15396
15606
|
};
|
|
15397
15607
|
}
|
|
15398
15608
|
function failOutcomesMissingSortValues(outcomes) {
|
|
@@ -15407,7 +15617,7 @@ function failOutcomesMissingSortValues(outcomes) {
|
|
|
15407
15617
|
}
|
|
15408
15618
|
|
|
15409
15619
|
// src/distribution/cluster-node/reads/groups.ts
|
|
15410
|
-
async function assembleDistributedGroups(
|
|
15620
|
+
async function assembleDistributedGroups(params, distributed, projection, readDocuments) {
|
|
15411
15621
|
const wireGroups = distributed.groups;
|
|
15412
15622
|
const group = params.group;
|
|
15413
15623
|
if (wireGroups === null || group === void 0) {
|
|
@@ -15420,19 +15630,19 @@ async function assembleDistributedGroups(deps, indexName, params, distributed, a
|
|
|
15420
15630
|
const results = [];
|
|
15421
15631
|
let batch = [];
|
|
15422
15632
|
let batchDocIds = /* @__PURE__ */ new Set();
|
|
15423
|
-
async function assembleBatch() {
|
|
15424
|
-
const documents = needsDocuments && batchDocIds.size > 0 ? await
|
|
15633
|
+
async function assembleBatch(last) {
|
|
15634
|
+
const documents = needsDocuments && batchDocIds.size > 0 ? await readDocuments([...batchDocIds], last) : /* @__PURE__ */ new Map();
|
|
15425
15635
|
for (const entry of batch) results.push(assembleGroup(entry, documents, keptPerGroup, projection, reduce));
|
|
15426
15636
|
batch = [];
|
|
15427
15637
|
batchDocIds = /* @__PURE__ */ new Set();
|
|
15428
15638
|
}
|
|
15429
15639
|
for (const entry of wireGroups) {
|
|
15430
15640
|
const docIds = docIdsRead(entry);
|
|
15431
|
-
if (batch.length > 0 && batchDocIds.size + docIds.length > MAX_FETCH_DOCUMENT_IDS) await assembleBatch();
|
|
15641
|
+
if (batch.length > 0 && batchDocIds.size + docIds.length > MAX_FETCH_DOCUMENT_IDS) await assembleBatch(false);
|
|
15432
15642
|
batch.push(entry);
|
|
15433
15643
|
for (const docId of docIds) batchDocIds.add(docId);
|
|
15434
15644
|
}
|
|
15435
|
-
if (batch.length > 0) await assembleBatch();
|
|
15645
|
+
if (batch.length > 0) await assembleBatch(true);
|
|
15436
15646
|
return results;
|
|
15437
15647
|
}
|
|
15438
15648
|
function assembleGroup(entry, documents, keptPerGroup, projection, reduce) {
|
|
@@ -15503,6 +15713,147 @@ function countStoredPinsFromOutside(distributed, presence) {
|
|
|
15503
15713
|
}
|
|
15504
15714
|
return { stored: count2, everyPinVerified };
|
|
15505
15715
|
}
|
|
15716
|
+
var NODE_LOSS_CODES = /* @__PURE__ */ new Set([
|
|
15717
|
+
TransportErrorCodes.CONNECT_FAILED,
|
|
15718
|
+
TransportErrorCodes.PEER_UNAVAILABLE,
|
|
15719
|
+
TransportErrorCodes.TIMEOUT
|
|
15720
|
+
]);
|
|
15721
|
+
function contextLost(context, indexName, cause) {
|
|
15722
|
+
return new NarsilError(
|
|
15723
|
+
ErrorCodes.SEARCH_CONTEXT_MISSING,
|
|
15724
|
+
`Node '${context.nodeId}' no longer holds the documents it ranked for a search of index '${indexName}'`,
|
|
15725
|
+
{ indexName, nodeId: context.nodeId, contextId: context.contextId, ...cause ? { cause: cause.code } : {} }
|
|
15726
|
+
);
|
|
15727
|
+
}
|
|
15728
|
+
function lostContextOf(error, context, indexName) {
|
|
15729
|
+
if (!(error instanceof NarsilError)) return null;
|
|
15730
|
+
if (error.code === ErrorCodes.SEARCH_CONTEXT_MISSING) return contextLost(context, indexName, error);
|
|
15731
|
+
if (NODE_LOSS_CODES.has(error.details.transportCode)) return contextLost(context, indexName, error);
|
|
15732
|
+
return null;
|
|
15733
|
+
}
|
|
15734
|
+
function createRankedDocumentReader(deps, indexName, allocation) {
|
|
15735
|
+
const sources = /* @__PURE__ */ new Map();
|
|
15736
|
+
const open = /* @__PURE__ */ new Map();
|
|
15737
|
+
const partitionCount = allocation.assignments.size;
|
|
15738
|
+
function releaseQuietly(contexts) {
|
|
15739
|
+
const remote = [];
|
|
15740
|
+
for (const context of contexts) {
|
|
15741
|
+
open.delete(context.contextId);
|
|
15742
|
+
if (context.nodeId === deps.nodeId && deps.searchContexts !== void 0) {
|
|
15743
|
+
deps.searchContexts.release(context.contextId);
|
|
15744
|
+
} else {
|
|
15745
|
+
remote.push(context);
|
|
15746
|
+
}
|
|
15747
|
+
}
|
|
15748
|
+
releaseSearchContexts(indexName, remote, {
|
|
15749
|
+
transport: deps.config.transport,
|
|
15750
|
+
sourceNodeId: deps.nodeId,
|
|
15751
|
+
resolveNodeTargets: deps.resolveNodeTargets
|
|
15752
|
+
});
|
|
15753
|
+
}
|
|
15754
|
+
function readOwnContext(context, docIds, last) {
|
|
15755
|
+
const registry = deps.searchContexts;
|
|
15756
|
+
const found = registry?.read(context.contextId, docIds) ?? null;
|
|
15757
|
+
if (last) {
|
|
15758
|
+
registry?.release(context.contextId);
|
|
15759
|
+
open.delete(context.contextId);
|
|
15760
|
+
}
|
|
15761
|
+
if (found === null) throw contextLost(context, indexName);
|
|
15762
|
+
const documents = /* @__PURE__ */ new Map();
|
|
15763
|
+
for (const [docId, snapshot] of found) documents.set(docId, documentOfSnapshot(snapshot));
|
|
15764
|
+
return documents;
|
|
15765
|
+
}
|
|
15766
|
+
async function readRemoteContext(context, docIds, last, fields) {
|
|
15767
|
+
const documents = /* @__PURE__ */ new Map();
|
|
15768
|
+
for (let start = 0; start < docIds.length; start += FETCH_GROUP_DOCUMENT_IDS) {
|
|
15769
|
+
const chunk = docIds.slice(start, start + FETCH_GROUP_DOCUMENT_IDS);
|
|
15770
|
+
const releaseContext = last && start + FETCH_GROUP_DOCUMENT_IDS >= docIds.length;
|
|
15771
|
+
const message = createFetchMessage(
|
|
15772
|
+
{
|
|
15773
|
+
indexName,
|
|
15774
|
+
documentIds: chunk.map((docId) => ({ docId, partitionId: resolvePartitionId(docId, partitionCount) })),
|
|
15775
|
+
fields,
|
|
15776
|
+
highlight: null,
|
|
15777
|
+
contextId: context.contextId,
|
|
15778
|
+
releaseContext
|
|
15779
|
+
},
|
|
15780
|
+
deps.nodeId
|
|
15781
|
+
);
|
|
15782
|
+
let response;
|
|
15783
|
+
try {
|
|
15784
|
+
response = await sendToNode2(deps.config, context.nodeId, message, {
|
|
15785
|
+
message: `Node '${context.nodeId}' did not answer a document fetch for index '${indexName}'`,
|
|
15786
|
+
details: { indexName }
|
|
15787
|
+
});
|
|
15788
|
+
} catch (error) {
|
|
15789
|
+
throw lostContextOf(error, context, indexName) ?? error;
|
|
15790
|
+
}
|
|
15791
|
+
if (releaseContext) open.delete(context.contextId);
|
|
15792
|
+
refuseDecodeWithoutRoom(response.payload);
|
|
15793
|
+
const decoded = decode(response.payload);
|
|
15794
|
+
if (response.type.endsWith(".error") || decoded.error === true) {
|
|
15795
|
+
const rejection = readRejectionOf(decoded, context.nodeId, indexName);
|
|
15796
|
+
throw lostContextOf(rejection, context, indexName) ?? rejection;
|
|
15797
|
+
}
|
|
15798
|
+
for (const fetched of validateFetchResultPayload(decoded).documents) {
|
|
15799
|
+
documents.set(fetched.docId, restoreVectorValues(fetched.document));
|
|
15800
|
+
}
|
|
15801
|
+
}
|
|
15802
|
+
return documents;
|
|
15803
|
+
}
|
|
15804
|
+
return {
|
|
15805
|
+
adopt(contexts) {
|
|
15806
|
+
if (contexts === void 0) return;
|
|
15807
|
+
for (const context of contexts.opened) open.set(context.contextId, context);
|
|
15808
|
+
for (const [docId, context] of contexts.sources) if (!sources.has(docId)) sources.set(docId, context);
|
|
15809
|
+
},
|
|
15810
|
+
async read(docIds, options) {
|
|
15811
|
+
const byContext = /* @__PURE__ */ new Map();
|
|
15812
|
+
const unsourced = [];
|
|
15813
|
+
for (const docId of new Set(docIds)) {
|
|
15814
|
+
const context = sources.get(docId);
|
|
15815
|
+
if (context === void 0 || !open.has(context.contextId)) {
|
|
15816
|
+
unsourced.push(docId);
|
|
15817
|
+
continue;
|
|
15818
|
+
}
|
|
15819
|
+
const group = byContext.get(context.contextId);
|
|
15820
|
+
if (group === void 0) byContext.set(context.contextId, { context, docIds: [docId] });
|
|
15821
|
+
else group.docIds.push(docId);
|
|
15822
|
+
}
|
|
15823
|
+
if (options.last) {
|
|
15824
|
+
releaseQuietly([...open.values()].filter((context) => !byContext.has(context.contextId)));
|
|
15825
|
+
}
|
|
15826
|
+
const reads = [];
|
|
15827
|
+
for (const { context, docIds: contextIds } of byContext.values()) {
|
|
15828
|
+
reads.push(
|
|
15829
|
+
context.nodeId === deps.nodeId && deps.searchContexts !== void 0 ? Promise.resolve().then(() => readOwnContext(context, contextIds, options.last)) : readRemoteContext(context, contextIds, options.last, options.fields)
|
|
15830
|
+
);
|
|
15831
|
+
}
|
|
15832
|
+
if (unsourced.length > 0) {
|
|
15833
|
+
reads.push(
|
|
15834
|
+
readDocumentsFromCopies(
|
|
15835
|
+
deps.config,
|
|
15836
|
+
deps.nodeId,
|
|
15837
|
+
deps.engine,
|
|
15838
|
+
indexName,
|
|
15839
|
+
unsourced,
|
|
15840
|
+
allocation,
|
|
15841
|
+
options.fields,
|
|
15842
|
+
options.requireEveryPartition
|
|
15843
|
+
)
|
|
15844
|
+
);
|
|
15845
|
+
}
|
|
15846
|
+
const documents = /* @__PURE__ */ new Map();
|
|
15847
|
+
for (const found of await Promise.all(reads)) {
|
|
15848
|
+
for (const [docId, document] of found) documents.set(docId, document);
|
|
15849
|
+
}
|
|
15850
|
+
return documents;
|
|
15851
|
+
},
|
|
15852
|
+
release() {
|
|
15853
|
+
releaseQuietly([...open.values()]);
|
|
15854
|
+
}
|
|
15855
|
+
};
|
|
15856
|
+
}
|
|
15506
15857
|
async function gatherFromEveryPartition(allocation, indexName, read) {
|
|
15507
15858
|
const routing = selectReplicasForQuery(allocation, randomSelector);
|
|
15508
15859
|
if (routing.unavailablePartitions.length > 0) {
|
|
@@ -15515,19 +15866,15 @@ async function gatherFromEveryPartition(allocation, indexName, read) {
|
|
|
15515
15866
|
const answered = await sendToEveryPartition(routing, (nodeId, partitionIds) => read({ nodeId, partitionIds }));
|
|
15516
15867
|
return answered.map((answer) => answer.result);
|
|
15517
15868
|
}
|
|
15518
|
-
async function sendReadRequest(deps, targetNodeId, message, indexName, validate) {
|
|
15869
|
+
async function sendReadRequest(deps, targetNodeId, message, indexName, validate, boundDecode = false) {
|
|
15519
15870
|
const response = await sendToNode2(deps.config, targetNodeId, message, {
|
|
15520
15871
|
message: `Node '${targetNodeId}' did not answer a read request for index '${indexName}'`,
|
|
15521
15872
|
details: { indexName }
|
|
15522
15873
|
});
|
|
15874
|
+
if (boundDecode) refuseDecodeWithoutRoom(response.payload);
|
|
15523
15875
|
const decoded = decode(response.payload);
|
|
15524
15876
|
if (response.type.endsWith(".error") || response.type === "error" || decoded.error === true) {
|
|
15525
|
-
|
|
15526
|
-
throw nodeRejectionOf(decoded.code, decoded.message, details) ?? new NarsilError(
|
|
15527
|
-
typeof decoded.code === "string" ? decoded.code : ErrorCodes.QUERY_ROUTING_FAILED,
|
|
15528
|
-
typeof decoded.message === "string" ? decoded.message : `Node '${targetNodeId}' rejected a read request for index '${indexName}'`,
|
|
15529
|
-
details
|
|
15530
|
-
);
|
|
15877
|
+
throw readRejectionOf(decoded, targetNodeId, indexName);
|
|
15531
15878
|
}
|
|
15532
15879
|
return validate(decoded);
|
|
15533
15880
|
}
|
|
@@ -15737,7 +16084,7 @@ async function listCluster(deps, indexName, params) {
|
|
|
15737
16084
|
);
|
|
15738
16085
|
}
|
|
15739
16086
|
const message = createListMessage({ ...payload, partitionIds: group.partitionIds }, deps.nodeId);
|
|
15740
|
-
return sendReadRequest(deps, group.nodeId, message, indexName, validateListResultPayload);
|
|
16087
|
+
return sendReadRequest(deps, group.nodeId, message, indexName, validateListResultPayload, true);
|
|
15741
16088
|
});
|
|
15742
16089
|
let total = 0;
|
|
15743
16090
|
let nodeReportsMore = false;
|
|
@@ -15877,35 +16224,70 @@ async function readClusterDocuments(deps, indexName, docIds) {
|
|
|
15877
16224
|
}
|
|
15878
16225
|
return readDistributedDocuments(deps.config, deps.nodeId, deps.engine, indexName, docIds, allocation);
|
|
15879
16226
|
}
|
|
15880
|
-
function routingConfig(query) {
|
|
15881
|
-
|
|
15882
|
-
|
|
15883
|
-
}
|
|
15884
|
-
const config = {};
|
|
15885
|
-
if (query.allowPartialResults !== void 0) {
|
|
16227
|
+
function routingConfig(query, searchContexts) {
|
|
16228
|
+
const config = { searchContexts };
|
|
16229
|
+
if (query?.allowPartialResults !== void 0) {
|
|
15886
16230
|
config.allowPartialResults = query.allowPartialResults;
|
|
15887
16231
|
}
|
|
15888
|
-
if (query
|
|
16232
|
+
if (query?.partitionTimeout !== void 0) {
|
|
15889
16233
|
config.partitionTimeout = query.partitionTimeout;
|
|
15890
16234
|
}
|
|
15891
16235
|
return config;
|
|
15892
16236
|
}
|
|
16237
|
+
function fetchFieldsFor(projection) {
|
|
16238
|
+
if (projection.kind !== "fields" || projection.include === null) return null;
|
|
16239
|
+
return projection.include.map((path) => path.join("."));
|
|
16240
|
+
}
|
|
16241
|
+
async function rerunOnLostContext(run) {
|
|
16242
|
+
try {
|
|
16243
|
+
return await run();
|
|
16244
|
+
} catch (error) {
|
|
16245
|
+
if (!(error instanceof NarsilError) || error.code !== ErrorCodes.SEARCH_CONTEXT_MISSING) throw error;
|
|
16246
|
+
return run();
|
|
16247
|
+
}
|
|
16248
|
+
}
|
|
16249
|
+
async function withRankedReader(deps, indexName, allocation, run) {
|
|
16250
|
+
const reader = createRankedDocumentReader(deps, indexName, allocation);
|
|
16251
|
+
try {
|
|
16252
|
+
return await run(reader);
|
|
16253
|
+
} finally {
|
|
16254
|
+
reader.release();
|
|
16255
|
+
}
|
|
16256
|
+
}
|
|
15893
16257
|
async function queryCluster(deps, indexName, params) {
|
|
15894
16258
|
const allocation = await routableAllocation(deps.config.coordinator, indexName);
|
|
15895
16259
|
if (allocation === null) {
|
|
15896
16260
|
return deps.engine.query(indexName, params);
|
|
15897
16261
|
}
|
|
15898
|
-
|
|
15899
|
-
|
|
15900
|
-
|
|
16262
|
+
return rerunOnLostContext(async () => {
|
|
16263
|
+
if (params.rerank === void 0) {
|
|
16264
|
+
return withRankedReader(
|
|
16265
|
+
deps,
|
|
16266
|
+
indexName,
|
|
16267
|
+
allocation,
|
|
16268
|
+
(reader) => searchCluster(deps, indexName, params, allocation, reader, false)
|
|
16269
|
+
);
|
|
16270
|
+
}
|
|
16271
|
+
const { rerankClusterQuery } = await import('./rerank-EHJVXEDP.mjs');
|
|
16272
|
+
return withRankedReader(
|
|
15901
16273
|
deps,
|
|
15902
16274
|
indexName,
|
|
15903
|
-
|
|
15904
|
-
(
|
|
15905
|
-
|
|
15906
|
-
|
|
16275
|
+
allocation,
|
|
16276
|
+
(reader) => rerankClusterQuery(
|
|
16277
|
+
deps,
|
|
16278
|
+
indexName,
|
|
16279
|
+
params,
|
|
16280
|
+
(firstStage) => searchCluster(deps, indexName, firstStage, allocation, reader, true),
|
|
16281
|
+
(docIds, last) => reader.read(docIds, { last, fields: null, requireEveryPartition: true }),
|
|
16282
|
+
clusterRerankHost
|
|
16283
|
+
)
|
|
15907
16284
|
);
|
|
15908
|
-
}
|
|
16285
|
+
});
|
|
16286
|
+
}
|
|
16287
|
+
async function searchCluster(deps, indexName, params, allocation, reader, readsLater) {
|
|
16288
|
+
const projection = resolveProjection(params.document);
|
|
16289
|
+
const groupsRead = params.group !== void 0 && (params.group.reduce !== void 0 || projection.kind !== "none");
|
|
16290
|
+
const readsDocuments = readsLater || projection.kind !== "none" || groupsRead;
|
|
15909
16291
|
const wireParams = localParamsToWire(params);
|
|
15910
16292
|
const queryDeps = {
|
|
15911
16293
|
transport: deps.config.transport,
|
|
@@ -15913,17 +16295,21 @@ async function queryCluster(deps, indexName, params) {
|
|
|
15913
16295
|
getAllocation: (idx) => deps.config.coordinator.getAllocation(idx),
|
|
15914
16296
|
resolveNodeTargets: deps.resolveNodeTargets
|
|
15915
16297
|
};
|
|
15916
|
-
const distributed = await distributedQuery(
|
|
15917
|
-
const projection = resolveProjection(params.document);
|
|
15918
|
-
const documents = await fetchDistributedDocuments(
|
|
15919
|
-
deps.config,
|
|
15920
|
-
deps.nodeId,
|
|
15921
|
-
deps.engine,
|
|
16298
|
+
const distributed = await distributedQuery(
|
|
15922
16299
|
indexName,
|
|
15923
|
-
|
|
15924
|
-
|
|
15925
|
-
|
|
16300
|
+
wireParams,
|
|
16301
|
+
queryDeps,
|
|
16302
|
+
routingConfig(deps.config.query, readsDocuments)
|
|
15926
16303
|
);
|
|
16304
|
+
reader.adopt(distributed.contexts);
|
|
16305
|
+
const documents = /* @__PURE__ */ new Map();
|
|
16306
|
+
if (projection.kind !== "none") {
|
|
16307
|
+
const found = await reader.read(
|
|
16308
|
+
distributed.scored.map((entry) => entry.docId),
|
|
16309
|
+
{ last: !groupsRead && !readsLater, fields: fetchFieldsFor(projection), requireEveryPartition: false }
|
|
16310
|
+
);
|
|
16311
|
+
for (const [docId, document] of found) documents.set(docId, applyProjection(document, projection));
|
|
16312
|
+
}
|
|
15927
16313
|
const pinPresence = await readPinPresence(deps, indexName, params, allocation);
|
|
15928
16314
|
const scored = dropUnstoredPinnedEntries(distributed, pinPresence);
|
|
15929
16315
|
const pins = countStoredPinsFromOutside(distributed, pinPresence);
|
|
@@ -15940,7 +16326,12 @@ async function queryCluster(deps, indexName, params) {
|
|
|
15940
16326
|
documents,
|
|
15941
16327
|
params.facets
|
|
15942
16328
|
);
|
|
15943
|
-
const groups = await assembleDistributedGroups(
|
|
16329
|
+
const groups = await assembleDistributedGroups(
|
|
16330
|
+
params,
|
|
16331
|
+
distributed,
|
|
16332
|
+
projection,
|
|
16333
|
+
(docIds, last) => reader.read(docIds, { last: last && !readsLater, fields: null, requireEveryPartition: true })
|
|
16334
|
+
);
|
|
15944
16335
|
if (groups !== void 0) {
|
|
15945
16336
|
result.groups = groups;
|
|
15946
16337
|
}
|
|
@@ -17261,6 +17652,8 @@ async function createClusterNode(callerConfig) {
|
|
|
17261
17652
|
let controller = null;
|
|
17262
17653
|
let unregisterHandler = null;
|
|
17263
17654
|
let unregisterNotReadyHandler = null;
|
|
17655
|
+
let unwatchDepartures = null;
|
|
17656
|
+
const searchContexts = hasDataRole ? createSearchContextRegistry() : void 0;
|
|
17264
17657
|
let serving = false;
|
|
17265
17658
|
let isShutdown = false;
|
|
17266
17659
|
const copyFailures = createCopyFailures({
|
|
@@ -17338,7 +17731,7 @@ async function createClusterNode(callerConfig) {
|
|
|
17338
17731
|
});
|
|
17339
17732
|
const trackOp = operationTracker.track;
|
|
17340
17733
|
const transitionIndex = operationTracker.transition;
|
|
17341
|
-
const readDeps = { config: routingConfig2, nodeId, engine, resolveNodeTargets: resolveNodeTargets3 };
|
|
17734
|
+
const readDeps = { config: routingConfig2, nodeId, engine, resolveNodeTargets: resolveNodeTargets3, searchContexts };
|
|
17342
17735
|
const servingName = (indexName) => servingNameOf(allocationView.routing, indexName);
|
|
17343
17736
|
const cluster = createClusterNamespace({
|
|
17344
17737
|
nodeId,
|
|
@@ -17417,12 +17810,16 @@ async function createClusterNode(callerConfig) {
|
|
|
17417
17810
|
allocations: allocationView,
|
|
17418
17811
|
writeDeps,
|
|
17419
17812
|
snapshotSyncState: snapshotSyncHandlerState,
|
|
17813
|
+
searchContexts,
|
|
17420
17814
|
snapshotSpoolDirectory: snapshotSpool.path,
|
|
17421
17815
|
isBootstrapSynced: (indexName, partitionId) => hasCompletedBootstrapSync(bootstrapSyncState, indexName, partitionId),
|
|
17422
17816
|
onError: forwardOnError
|
|
17423
17817
|
});
|
|
17424
17818
|
const listener = controllerTransport !== null ? controllerTransport.createHandler(dataHandler) : dataHandler;
|
|
17425
17819
|
unregisterHandler = await config.transport.listen(listener);
|
|
17820
|
+
if (searchContexts !== void 0) {
|
|
17821
|
+
unwatchDepartures = await releaseContextsOfDepartedNodes(config.coordinator, searchContexts);
|
|
17822
|
+
}
|
|
17426
17823
|
startCatchUpPump(writeDeps.catchUp, writeDeps);
|
|
17427
17824
|
}
|
|
17428
17825
|
serving = true;
|
|
@@ -17452,6 +17849,9 @@ async function createClusterNode(callerConfig) {
|
|
|
17452
17849
|
unregisterNotReadyHandler();
|
|
17453
17850
|
unregisterNotReadyHandler = null;
|
|
17454
17851
|
}
|
|
17852
|
+
unwatchDepartures?.();
|
|
17853
|
+
unwatchDepartures = null;
|
|
17854
|
+
searchContexts?.stop();
|
|
17455
17855
|
await lifecycle.shutdown();
|
|
17456
17856
|
memoryReporting.stop();
|
|
17457
17857
|
if (controller !== null) {
|