@delali/narsil 0.3.1-next.131 → 0.3.1-next.140
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/analysis-change-JN23AZQP.mjs +43 -0
- package/dist/analysis-change-entry-5JSIMEKG.mjs +1 -0
- package/dist/browser/analysis-change-57MIRJWR.mjs +499 -0
- package/dist/browser/analysis-change-entry-ICGRR3IN.mjs +1 -0
- package/dist/browser/{chunk-UDGB2MWT.mjs → chunk-35B42UFM.mjs} +4 -0
- package/dist/browser/{chunk-VBBLTD7O.mjs → chunk-3EYDURFD.mjs} +39 -27
- package/dist/browser/{chunk-AIIJDWMG.mjs → chunk-4MFH226Q.mjs} +1 -1
- package/dist/browser/{chunk-D7WQF6DV.mjs → chunk-5LYWZJE2.mjs} +2 -2
- package/dist/browser/chunk-67KSBUYR.mjs +173 -0
- package/dist/browser/{chunk-M5UAAXGU.mjs → chunk-6BSB4N6B.mjs} +2 -2
- package/dist/browser/{chunk-NSCSR6MA.mjs → chunk-CGVBGHBX.mjs} +81 -6
- package/dist/browser/chunk-E5SFTOQR.mjs +21 -0
- package/dist/browser/{chunk-VEGHFP2P.mjs → chunk-MZI3TEOW.mjs} +4 -2
- package/dist/browser/chunk-OIZS5KIX.mjs +6 -0
- package/dist/browser/chunk-QHNNULPW.mjs +265 -0
- package/dist/browser/chunk-XVMOQQ34.mjs +384 -0
- package/dist/browser/{chunk-2Y3CO4EQ.mjs → chunk-ZAIZQOTM.mjs} +256 -777
- package/dist/browser/{durability-wiring-FNZSRH4N.mjs → durability-wiring-MTIAN6VT.mjs} +145 -50
- package/dist/browser/import-Q7MXD6BS.mjs +8 -0
- package/dist/browser/{module-4HIMOQM5.mjs → module-5J3PEUEB.mjs} +7 -6
- package/dist/browser/{phrase-module-WESYVU7K.mjs → phrase-module-HGVBO24W.mjs} +1 -1
- package/dist/{restore-queue-JAMNRC6T.mjs → browser/restore-queue-DISFECE4.mjs} +8 -2
- package/dist/browser/{runtime-3X7ZJ5T3.mjs → runtime-VEUH674B.mjs} +2 -2
- package/dist/browser/{snapshot-V7VF6BN6.mjs → snapshot-YBJ5MF2M.mjs} +36 -167
- package/dist/browser/{write-YIC2MKRY.mjs → write-RTWI5MXL.mjs} +8 -6
- package/dist/{capabilities-B6WxrWxt.d.ts → capabilities-DXAmysqJ.d.ts} +10 -3
- package/dist/{chunk-NMXRXFGZ.mjs → chunk-4GCN34JV.mjs} +2 -2
- package/dist/{chunk-SOHMMNCH.mjs → chunk-6VHWXAOI.mjs} +4 -4
- package/dist/{chunk-2C3WXBQS.mjs → chunk-74XB2SXI.mjs} +1 -1
- package/dist/{chunk-OGLV4I3L.mjs → chunk-B5I3IL4H.mjs} +12 -7
- package/dist/chunk-BDIDUW2B.mjs +387 -0
- package/dist/{chunk-5LX6MOIB.mjs → chunk-C3YDYX6C.mjs} +1 -1
- package/dist/{chunk-Y4BVTQ53.mjs → chunk-CCCGEPJM.mjs} +2 -2
- package/dist/{chunk-2JWNJSNE.mjs → chunk-CPCGEIOM.mjs} +11 -8
- package/dist/chunk-CZQ33DLW.mjs +78 -0
- package/dist/{chunk-4GDLFDPZ.mjs → chunk-DD2673TX.mjs} +3 -2
- package/dist/{chunk-RNSC4NFI.mjs → chunk-EEBCDP36.mjs} +3 -1
- package/dist/{chunk-HCGJKDZV.mjs → chunk-FFVLPCLH.mjs} +1 -1
- package/dist/{chunk-WVRDRLIC.mjs → chunk-FKARMUWH.mjs} +6 -6
- package/dist/{chunk-PATMDIMN.mjs → chunk-HJJWY6PT.mjs} +1 -1
- package/dist/chunk-HMDT3I63.mjs +21 -0
- package/dist/{chunk-CZ7ISLXA.mjs → chunk-HNHCNTVG.mjs} +4 -0
- package/dist/{chunk-RWESFSJI.mjs → chunk-IBATGS3J.mjs} +87 -61
- package/dist/{chunk-Y2U5VVWX.mjs → chunk-ICW45TTU.mjs} +1 -1
- package/dist/{chunk-T4OFG4IQ.mjs → chunk-IP4HJOGC.mjs} +90 -16
- package/dist/{chunk-EUCX5K74.mjs → chunk-J3EHATEZ.mjs} +3 -3
- package/dist/{chunk-X5CNUKD5.mjs → chunk-K3AQOZVM.mjs} +37 -20
- package/dist/chunk-LK3MOTXF.mjs +174 -0
- package/dist/{chunk-7OIQ2SIO.mjs → chunk-LLDFIKGP.mjs} +4 -4
- package/dist/{chunk-K77FJXKS.mjs → chunk-MTQTX4BP.mjs} +81 -8
- package/dist/{chunk-ZR4VM4A6.mjs → chunk-N37VMC62.mjs} +1 -18
- package/dist/{chunk-2WBY4Y4M.mjs → chunk-NOT5QOHU.mjs} +1 -1
- package/dist/{chunk-D5TFRB5G.mjs → chunk-NPNK57CA.mjs} +1 -1
- package/dist/{chunk-6SSVIW7Y.mjs → chunk-O4LJQI4G.mjs} +31 -5
- package/dist/{chunk-JJ6EUMXB.mjs → chunk-PNNVTKMT.mjs} +2 -2
- package/dist/{chunk-2SEL6SBS.mjs → chunk-RKAVHNRW.mjs} +2 -2
- package/dist/{chunk-RCMNL7RI.mjs → chunk-SBMAWTRG.mjs} +41 -4
- package/dist/chunk-TCWQJL7O.mjs +188 -0
- package/dist/{chunk-2MKX2SCO.mjs → chunk-U4X4AWKL.mjs} +7 -7
- package/dist/{chunk-PU6SKH7O.mjs → chunk-UBXJQJII.mjs} +2 -2
- package/dist/chunk-UEV53BBO.mjs +480 -0
- package/dist/{chunk-NNO7CO3A.mjs → chunk-UIKN2OTQ.mjs} +1 -1
- package/dist/{chunk-F6AAXI7T.mjs → chunk-VSUYTPME.mjs} +3 -3
- package/dist/{chunk-TGHNXN7W.mjs → chunk-W2XDK66Z.mjs} +180 -541
- package/dist/{chunk-7X5JST5L.mjs → chunk-WNBRW4Q2.mjs} +16 -2
- package/dist/{chunk-6M6SDH3S.mjs → chunk-XMQNOD5V.mjs} +8 -8
- package/dist/{chunk-SZGRYEIC.mjs → chunk-XS2VM7UE.mjs} +3 -3
- package/dist/{chunk-JT3YZGCE.mjs → chunk-XXKWEG53.mjs} +3 -188
- package/dist/{chunk-5W7AEVTH.mjs → chunk-ZMBFWQXN.mjs} +2 -2
- package/dist/{chunk-7XY6VD7B.mjs → chunk-ZSWHPKGM.mjs} +1 -1
- package/dist/{client-gNAfaHcE.d.ts → client-DApBeLbk.d.ts} +17 -2
- package/dist/client.d.ts +5 -5
- package/dist/client.mjs +11 -1
- package/dist/{config-D1cDv3sE.d.ts → config-B0H-vp4U.d.ts} +4 -0
- package/dist/{config-CmYJO34X.d.ts → config-Brysl9NO.d.ts} +1 -1
- package/dist/{direct-executor-HQ4QVDX7.mjs → direct-executor-J5UDLPL3.mjs} +14 -14
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +21 -5
- package/dist/distribution.mjs +882 -192
- package/dist/{durability-wiring-S7OX2MAI.mjs → durability-wiring-AYOTAPNQ.mjs} +165 -70
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-m_Ddr1bz.d.ts → engine-CuLMbH1-.d.ts} +63 -6
- package/dist/import-QRZAQ5RH.mjs +43 -0
- package/dist/index.browser.mjs +4 -4
- package/dist/index.d.ts +4 -4
- package/dist/index.mjs +20 -20
- package/dist/{long-query-thread-BSL25PD6.mjs → long-query-thread-U7K5A72G.mjs} +14 -14
- package/dist/{module-2E4XYSGT.mjs → module-L2OZO5PQ.mjs} +11 -11
- package/dist/partial-words.mjs +5 -5
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.d.ts +11 -0
- package/dist/persistence/durability/checkpoint-worker.mjs +23 -23
- package/dist/{phrase-module-JERF2IDG.mjs → phrase-module-5QHBD2VJ.mjs} +1 -1
- package/dist/react.d.ts +7 -7
- package/dist/react.mjs +4 -0
- package/dist/{rerank-DMMT4FZA.mjs → rerank-KTLC3E6N.mjs} +5 -5
- package/dist/rerankers/http.mjs +2 -2
- package/dist/{browser/restore-queue-GEZ54CA5.mjs → restore-queue-RYSRRFS4.mjs} +8 -2
- package/dist/{runtime-7MLQ7J64.mjs → runtime-DYV75DCB.mjs} +6 -6
- package/dist/server.d.ts +10 -8
- package/dist/server.mjs +51 -19
- package/dist/{snapshot-DOQLGGEX.mjs → snapshot-BTRZ3MZK.mjs} +59 -190
- package/dist/synonyms.mjs +6 -4
- package/dist/{tasks-Co8HfRD7.d.ts → tasks-DjGjK70Y.d.ts} +22 -6
- package/dist/{thread-SKO7CN3R.mjs → thread-USB3GOPY.mjs} +13 -13
- package/dist/vector/search-worker.mjs +5 -5
- package/dist/workers/entry.mjs +7 -7
- package/dist/{write-ZKR7SNRT.mjs → write-GHTA56OW.mjs} +25 -22
- package/openapi.json +101 -6
- package/package.json +7 -7
package/dist/distribution.mjs
CHANGED
|
@@ -1,65 +1,69 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
|
|
1
|
+
import { closePartitionWriter, replaySyncedWal } from './chunk-VSUYTPME.mjs';
|
|
2
|
+
import { buildArchiveFromMemory, archiveStream, memoryCheckpointStore, readSnapshotArchive } from './chunk-FKARMUWH.mjs';
|
|
3
|
+
import { indexMetadataOf, loadPartitionSegments, readSegmentManifest, readVectorFile } from './chunk-MTQTX4BP.mjs';
|
|
4
|
+
import { createByteReader } from './chunk-CZQ33DLW.mjs';
|
|
5
|
+
import { DEFAULT_NODE_LIFECYCLE_CONFIG, bindWriteBlockLookup, bindEngineWriteBlock, bindIndexReads, DEFAULT_CONTROLLER_CONFIG, readIndexMetadataSnapshot, indexMetadataSnapshotOf, getIndexMetadata, getClusterIndexConfig, withPartitionCount, validateIndexName, indexConfigFromSettings, metadataWithBm25, indexConfigKey, readIndexState, storeSynonymSet, deleteSynonymSet, indexSettingsOf, putIndexMetadata, isValidInteger, isRecord, isRecord2, decodeIndexSettings, HIDDEN_GENERATION_PREFIX, metadataWithSynonymSet, RESHARD_RECORD_CAS_ATTEMPTS, indexUuidOfStoredMetadata, RESHARD_FORWARD_BATCH_DOCUMENTS, storedSynonymSetOf, RESHARD_RETRY_DELAY_MS, RESHARD_RECORD_CHECK_INTERVAL_MS, RESHARD_POLL_INTERVAL_MS, ALLOCATION_RETRY_DELAY_MS, TEARDOWN_CAS_ATTEMPTS, TEARDOWN_RETRY_ROUNDS, ANALYSIS_CHANGE_RETRY_MS, putIndexReplicationFactor, ALLOCATION_CAS_ATTEMPTS, TEARDOWN_RETRY_DELAY_MS, RESHARD_CONTROLLER_POLL_INTERVAL_MS, MAX_RESHARD_CUTOVER_ATTEMPTS, ANALYSIS_CHANGE_POLL_INTERVAL_MS, RECOVERY_CAS_ATTEMPTS, ESTIMATED_PARTITION_BYTES, REBALANCE_THRESHOLD, putGenerationMetadata, RESHARD_CUTOVER_LAG_ENTRIES, ANALYSIS_REBUILD_RESEND_MS, RESHARD_CUTOVER_TIMEOUT_MS, PARTITION_STORES_TIMEOUT_MS, LEADERSHIP_IMBALANCE_THRESHOLD, BOOTSTRAP_CAS_ATTEMPTS } from './chunk-IP4HJOGC.mjs';
|
|
6
|
+
import './chunk-TCWQJL7O.mjs';
|
|
7
|
+
import { MAX_WAIT_FOR_ACTIVE_REPLICAS, MIN_CONTROLLER_LEASE_TTL_MS, MAX_NODE_LEFT_DELAY_MS, DEFAULT_MAX_PER_SOURCE_SNAPSHOTS, DEFAULT_MAX_STREAMS_PER_INDEX, DEFAULT_CAPACITY, CATCH_UP_TICK_MS, DEFAULT_RESUME_WRITES_BELOW, DEFAULT_BLOCK_WRITES_ABOVE, readMemoryFigures, createWritePressure, WRITE_MEMORY_SAMPLE_MAX_AGE_MS, DEFAULT_IN_FLIGHT_WRITE_FRACTION, MAX_SETTINGS_REPLACEMENT_ATTEMPTS, SETTINGS_CONTENTION_REASON, MAX_DROP_ACKNOWLEDGE_WAIT_MS, DEFAULT_PARTITION_COUNT, DEFAULT_REPLICATION_FACTOR, DEFAULT_CREATE_INDEX_WAIT_MS, refuseEncodeWithoutRoom, readRecentMemoryFigures, DEFAULT_BOOTSTRAP_SYNC_DEADLINE_MS, DROP_ACKNOWLEDGE_POLL_INTERVAL_MS, COORDINATOR_REQUEST_BUCKET_BOUNDS_SECONDS, COPY_FAILURE_ATTEMPTS, SEARCH_CONTEXT_REAPER_INTERVAL_MS, DECOMMISSION_RECHECK_INTERVAL_MS, answerRefusalOf, withAdmittedWrite, estimateForwardedBytes, CASE_FOLD_EXPANSION_FACTOR, PATTERN_RUN_HEAP_BYTES, ENUM_VALUE_HEAP_BYTES, MAX_SOURCE_ID_LENGTH, DOCUMENT_VALUE_HEAP_BYTES, DOCUMENT_HEAP_BYTES_PER_ESTIMATED_BYTE, DOCUMENT_PROPERTY_HEAP_BYTES, VECTOR_DOCUMENT_HEAP_BYTES, VECTOR_COMPONENT_HEAP_BYTES, NORMALIZATION_EXPANSION_FACTOR, TERM_UNIT_HEAP_BYTES, PARTITION_BYTES_INDEX_OVERHEAD_BYTES, DISTINCT_TERM_HEAP_BYTES, PARTIAL_SPELLING_HEAP_BYTES, PARTITION_BYTES_ENTRY_BYTES, PARTITION_BYTES_REPORT_BUDGET_BYTES, SNAPSHOT_CHUNK_YIELD_INTERVAL_MS, CATCH_UP_FEED_WINDOW_MS, MAX_CATCH_UP_IN_FLIGHT_BYTES, SYNCED_POSITION_REPORT_TIMEOUT_MS, ADMISSION_TIMEOUT_MS, REGISTRATION_LEASE_DRIFT_SHARE, REGISTRATION_LEASE_CONFIRM_TIMEOUT_MS, WRITE_APPLY_YIELD_INTERVAL_MS, ROLLBACK_RETRY_INITIAL_DELAY_MS, ROLLBACK_RETRY_DELAY_CAP_MS, COPY_FAILURE_RETRY_DELAY_MS, FETCH_GROUP_DOCUMENT_IDS, SEARCH_CONTEXT_ENTRY_HEAP_BYTES, FULL_MEMORY_COLLECTION_INTERVAL_MS, MEMORY_SAMPLE_INTERVAL_MS, ANSWER_DECODE_HEAP_BYTES_PER_PAYLOAD_BYTE, CLEAR_PAGE_SIZE, estimateDocumentsBytes, estimateDocumentBytes, PRIMARY_CHANGE_RETRY_WINDOW_MS, PRIMARY_CHANGE_POLL_INTERVAL_MS, PARTITION_WRITE_CONCURRENCY, FORWARD_RETRY_INITIAL_DELAY_MS, FORWARD_RETRY_DELAY_CAP_MS } from './chunk-DD2673TX.mjs';
|
|
8
|
+
export { DEFAULT_CAPACITY, DEFAULT_PARTITION_COUNT, DEFAULT_REPLICATION_FACTOR } from './chunk-DD2673TX.mjs';
|
|
5
9
|
import { addBackgroundAdmission, workWaitingForMemory } from './chunk-PELHVTO7.mjs';
|
|
6
|
-
import './chunk-
|
|
10
|
+
import './chunk-6VHWXAOI.mjs';
|
|
11
|
+
import { decodeVectorFilePayload, bytesToVectors, isDead } from './chunk-ICW45TTU.mjs';
|
|
7
12
|
import { onRefusalAfterCollection, reservedHeapBytes, reserveHeapBytes, collectYoungGenerationBeforeRefusal, collectGarbageBeforeRefusal } from './chunk-IWVEG5CX.mjs';
|
|
8
|
-
import {
|
|
9
|
-
import
|
|
10
|
-
import {
|
|
11
|
-
import { decodeVectorFilePayload, bytesToVectors, isDead } from './chunk-Y2U5VVWX.mjs';
|
|
13
|
+
import { stampAnalysisGeneration, holdsNamedSettings, dropHeldAnalysisChange, releaseHeldAnalysisChange, heldAnalysisIsBuilt, buildHeldAnalysisChange, resolveAnalysisTarget, namedSettingsOf, holdsSettings } from './chunk-UEV53BBO.mjs';
|
|
14
|
+
import './chunk-LK3MOTXF.mjs';
|
|
15
|
+
import { fixedHybridDepthOf, requireWithinHybridDepth, pageByPosition } from './chunk-C3YDYX6C.mjs';
|
|
12
16
|
import { CONTROLLER_LEASE_KEY } from './chunk-44W3CBNG.mjs';
|
|
13
17
|
import { MAX_MESSAGE_SIZE_BYTES } from './chunk-UNEEFTEU.mjs';
|
|
14
|
-
import './chunk-
|
|
15
|
-
import './chunk-DJHHF2L5.mjs';
|
|
18
|
+
import './chunk-K3AQOZVM.mjs';
|
|
16
19
|
import './chunk-37CD3OKS.mjs';
|
|
20
|
+
import './chunk-DJHHF2L5.mjs';
|
|
17
21
|
import { getPath, getFs } from './chunk-HLPRHOSO.mjs';
|
|
18
|
-
import { createForwardBatchResultMessage, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, readListedFile, optionalIndexUuid, computeEntryChecksum, createPrimaryResignMessage, validateInsyncConfirmPayload, buildEntry, createInsyncRemoveMessage, createEntryBatchMessage, createInsyncConfirmMessage, createEntryMessage, validateAckPayload, createInsyncAddMessage, directoryCheckpointStore, checkpointLastSeqNo, checkpointPrimaryTerm, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage } from './chunk-
|
|
19
|
-
import { applyIndexEntry, applyDeleteEntry, validateReplicationEntry } from './chunk-
|
|
22
|
+
import { createForwardBatchResultMessage, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, readListedFile, optionalIndexUuid, computeEntryChecksum, createPrimaryResignMessage, validateInsyncConfirmPayload, buildEntry, createInsyncRemoveMessage, createEntryBatchMessage, createInsyncConfirmMessage, createEntryMessage, validateAckPayload, createInsyncAddMessage, directoryCheckpointStore, checkpointLastSeqNo, checkpointPrimaryTerm, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage } from './chunk-CPCGEIOM.mjs';
|
|
23
|
+
import { applyIndexEntry, applyDeleteEntry, validateReplicationEntry } from './chunk-LLDFIKGP.mjs';
|
|
20
24
|
import { MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR, WIRE_BATCH_BUDGET } from './chunk-P36MUFEJ.mjs';
|
|
21
25
|
import { TransportErrorCodes, ReplicationMessageTypes, ClusterMessageTypes, QueryMessageTypes, TransportError, DEFAULT_TRANSPORT_CONFIG } from './chunk-YH5OIL7B.mjs';
|
|
22
|
-
import { requireKnownConfig, resolveRunnableConfig, createEngineCore, startEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, applyStoredIndexBm25, createEngineIndex, encodeIndexConfiguration, requireConfigurationWithinSizeLimit, prepareConfiguredSynonyms,
|
|
23
|
-
import { runtimeHost, validateBM25Params, storedBM25Params, requireSurfaceFormsForPartialFields, validateBM25Shape, now, listBindingOf, clampLimit, hitsKeptPerGroup, queryBindingOf, resolveHybridFusion, dedupePinnedEntries, foldGroupReducer } from './chunk-
|
|
24
|
-
import { bindRerankWindowSource, rerankWindowCapOf, providedDocId, engineCoreOf } from './chunk-
|
|
25
|
-
import './chunk-
|
|
26
|
+
import { requireKnownConfig, resolveRunnableConfig, createEngineCore, startEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, applyStoredIndexBm25, createEngineIndex, encodeIndexConfiguration, requireConfigurationWithinSizeLimit, prepareConfiguredSynonyms, openChecksummedEnvelope, storedArrayPositionGap, emptySerializablePartition, arrayPositionsAreStale, crc32Init, crc32Final, crc32Update, yieldToEventLoop } from './chunk-W2XDK66Z.mjs';
|
|
27
|
+
import { runtimeHost, validateBM25Params, storedBM25Params, requireSurfaceFormsForPartialFields, validateBM25Shape, now, listBindingOf, clampLimit, hitsKeptPerGroup, queryBindingOf, resolveHybridFusion, dedupePinnedEntries, foldGroupReducer } from './chunk-XMQNOD5V.mjs';
|
|
28
|
+
import { bindRerankWindowSource, rerankWindowCapOf, providedDocId, engineCoreOf } from './chunk-XXKWEG53.mjs';
|
|
29
|
+
import './chunk-UIKN2OTQ.mjs';
|
|
26
30
|
import './chunk-AV4IJKD2.mjs';
|
|
27
|
-
import { synonymModule } from './chunk-
|
|
28
|
-
import { oversampledShardSize, bm25SetsValue, normalizeSort, sortModesOf, readSortValues, resolveProjection, fnv1a, applyProjection, FILTER_EXPRESSION_KEYS, filterTestCount, setNestedValue, FIELD_FILTER_OPERATORS, clampRowCount, sortSignatureOf, requireMatchingCursor, decodePageCursor, encodePageCursor,
|
|
29
|
-
import { generateId
|
|
31
|
+
import { synonymModule } from './chunk-IBATGS3J.mjs';
|
|
32
|
+
import { oversampledShardSize, bm25SetsValue, normalizeSort, sortModesOf, readSortValues, resolveProjection, fnv1a, applyProjection, FILTER_EXPRESSION_KEYS, filterTestCount, setNestedValue, FIELD_FILTER_OPERATORS, clampRowCount, sortSignatureOf, requireMatchingCursor, decodePageCursor, encodePageCursor, requireWithinResultWindow, holdsQuotedText, sortSignatureEntry, fnv1aUint32s, mergePartitionStats } from './chunk-U4X4AWKL.mjs';
|
|
33
|
+
import { generateId } from './chunk-N37VMC62.mjs';
|
|
30
34
|
import './chunk-ZATRDAYU.mjs';
|
|
31
35
|
import './chunk-EBIHRL7B.mjs';
|
|
32
36
|
import './chunk-HJSN7UYG.mjs';
|
|
33
|
-
import { MAX_DOC_ID_LENGTH, BATCH_CHUNK_SIZE, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING } from './chunk-
|
|
34
|
-
import './chunk-TGP5M5R7.mjs';
|
|
35
|
-
import './chunk-U2BKYEC7.mjs';
|
|
37
|
+
import { MAX_DOC_ID_LENGTH, BATCH_CHUNK_SIZE, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING } from './chunk-EEBCDP36.mjs';
|
|
36
38
|
import './chunk-KOKM4PQN.mjs';
|
|
37
39
|
import { MIN_OVERSAMPLE } from './chunk-QSK35UC2.mjs';
|
|
38
|
-
import './chunk-
|
|
39
|
-
import { isPatternTolerance } from './chunk-
|
|
40
|
+
import './chunk-74XB2SXI.mjs';
|
|
41
|
+
import { isPatternTolerance } from './chunk-RKAVHNRW.mjs';
|
|
40
42
|
import { MAX_FILTER_DEPTH, MAX_FILTER_TESTS, MAX_FILTER_ARRAY_SIZE } from './chunk-WQBCBKY3.mjs';
|
|
41
43
|
import './chunk-JUKYYXSP.mjs';
|
|
42
|
-
import { getNestedValue } from './chunk-
|
|
43
|
-
import { encodeDocument, validateSchema, requireCreatableSchema, encodeDocumentReply, schemaFieldsOf, MIN_POLYGON_POINTS, isLatitude, MAX_LATITUDE_DEGREES, isLongitude, MAX_LONGITUDE_DEGREES } from './chunk-
|
|
44
|
+
import { getNestedValue } from './chunk-J3EHATEZ.mjs';
|
|
45
|
+
import { encodeDocument, validateSchema, requireCreatableSchema, encodeDocumentReply, schemaFieldsOf, MIN_POLYGON_POINTS, isLatitude, MAX_LATITUDE_DEGREES, isLongitude, MAX_LONGITUDE_DEGREES } from './chunk-UBXJQJII.mjs';
|
|
44
46
|
import { MAX_DOCUMENT_ENCODING_DEPTH } from './chunk-U476IXKQ.mjs';
|
|
45
47
|
import { MAX_SORT_FIELDS, MAX_CURSOR_LENGTH, MAX_TOLERANCE, MAX_PREFIX_LENGTH, RESULT_WINDOW, MIN_HYBRID_DEPTH, MAX_HYBRID_DEPTH, DEFAULT_PAGE_SIZE, HYBRID_RESCORE_WINDOW } from './chunk-FQBFZAWG.mjs';
|
|
46
48
|
import './chunk-VJS4JFBQ.mjs';
|
|
47
49
|
import './chunk-GES7U2PM.mjs';
|
|
48
|
-
import './chunk-
|
|
49
|
-
import { resolveIndexAnalysis } from './chunk-
|
|
50
|
+
import './chunk-ZSWHPKGM.mjs';
|
|
51
|
+
import { resolveIndexAnalysis, configOfNamedSettings } from './chunk-O4LJQI4G.mjs';
|
|
50
52
|
import './chunk-X5CV2E2B.mjs';
|
|
51
|
-
import './chunk-
|
|
52
|
-
import './chunk-
|
|
53
|
+
import { getLanguage } from './chunk-WNBRW4Q2.mjs';
|
|
54
|
+
import './chunk-TGP5M5R7.mjs';
|
|
55
|
+
import './chunk-U2BKYEC7.mjs';
|
|
56
|
+
import './chunk-NPNK57CA.mjs';
|
|
53
57
|
import './chunk-4UMZ7KQL.mjs';
|
|
54
58
|
import { isSortMode, SORT_MODES, toComparableSortValue, compareComparableKeys, compareSortValues } from './chunk-R6A773WK.mjs';
|
|
55
59
|
import { compareCodePoints, exceedsCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
56
60
|
import { collectAndReturnPages } from './chunk-RIIUZJQ3.mjs';
|
|
57
61
|
import './chunk-UJWGQIXU.mjs';
|
|
58
|
-
import './chunk-
|
|
62
|
+
import './chunk-FFVLPCLH.mjs';
|
|
59
63
|
import { HEAP_RESUME_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE } from './chunk-22YPZVMG.mjs';
|
|
60
64
|
import './chunk-RFBLQLLZ.mjs';
|
|
61
65
|
import { MAX_STORED_VECTOR_DIMENSION, MAX_NESTING_DEPTH } from './chunk-L5ORED7V.mjs';
|
|
62
|
-
import { ErrorCodes, NarsilError, describeError, ServerErrorCodes } from './chunk-
|
|
66
|
+
import { ErrorCodes, NarsilError, describeError, ServerErrorCodes } from './chunk-HNHCNTVG.mjs';
|
|
63
67
|
import { encode, decode } from '@msgpack/msgpack';
|
|
64
68
|
import { randomUUID } from 'crypto';
|
|
65
69
|
import { openSync, writeSync, closeSync, rmSync, readSync } from 'fs';
|
|
@@ -1089,6 +1093,134 @@ async function openSnapshotSpoolDirectory(durabilityDirectory) {
|
|
|
1089
1093
|
}
|
|
1090
1094
|
return { path, dispose: () => rm(path, { recursive: true, force: true }) };
|
|
1091
1095
|
}
|
|
1096
|
+
|
|
1097
|
+
// src/distribution/cluster-node/analysis-change/view.ts
|
|
1098
|
+
function namedSettingsOfMetadata(settings) {
|
|
1099
|
+
return {
|
|
1100
|
+
language: settings?.language ?? "english",
|
|
1101
|
+
...settings?.tokenizer !== void 0 ? { tokenizer: settings.tokenizer } : {},
|
|
1102
|
+
...settings?.stopWords !== void 0 ? { stopWords: settings.stopWords } : {},
|
|
1103
|
+
...settings?.stopWordList !== void 0 ? { stopWordList: settings.stopWordList } : {}
|
|
1104
|
+
};
|
|
1105
|
+
}
|
|
1106
|
+
function generationMissing(indexName, requestedGeneration, heldGeneration) {
|
|
1107
|
+
return new NarsilError(
|
|
1108
|
+
ErrorCodes.ANALYSIS_GENERATION_MISSING,
|
|
1109
|
+
`This node holds analysis generation ${heldGeneration} of index '${indexName}', so it cannot answer for generation ${requestedGeneration}`,
|
|
1110
|
+
{ indexName, requestedGeneration, heldGeneration }
|
|
1111
|
+
);
|
|
1112
|
+
}
|
|
1113
|
+
function createAnalysisView(deps) {
|
|
1114
|
+
const { coordinator, engine, onError } = deps;
|
|
1115
|
+
const held = /* @__PURE__ */ new Map();
|
|
1116
|
+
const settling = /* @__PURE__ */ new Map();
|
|
1117
|
+
let unwatch = null;
|
|
1118
|
+
let stopped = false;
|
|
1119
|
+
function recordGeneration(indexName, snapshot) {
|
|
1120
|
+
const generation = snapshot.settings?.analysisGeneration ?? 0;
|
|
1121
|
+
const { indexUuid } = snapshot.record;
|
|
1122
|
+
const known = held.get(indexName);
|
|
1123
|
+
if (known?.indexUuid === indexUuid && known.generation >= generation) return known.generation;
|
|
1124
|
+
held.set(indexName, { indexUuid, generation });
|
|
1125
|
+
return generation;
|
|
1126
|
+
}
|
|
1127
|
+
async function refresh(indexName) {
|
|
1128
|
+
const snapshot = await readIndexMetadataSnapshot(coordinator, indexName);
|
|
1129
|
+
if (snapshot === null) {
|
|
1130
|
+
held.delete(indexName);
|
|
1131
|
+
return 0;
|
|
1132
|
+
}
|
|
1133
|
+
return recordGeneration(indexName, snapshot);
|
|
1134
|
+
}
|
|
1135
|
+
function catchUp(indexName, generation, settings) {
|
|
1136
|
+
engine.buildHeldAnalysis(indexName, { generation, ...namedSettingsOfMetadata(settings) }).then(() => engine.releaseHeldAnalysis(indexName, generation)).catch((error) => {
|
|
1137
|
+
if (!stopped) onError(error);
|
|
1138
|
+
});
|
|
1139
|
+
}
|
|
1140
|
+
async function settleOnce(indexName) {
|
|
1141
|
+
const generation = engine.analysisGenerationOf(indexName);
|
|
1142
|
+
if (stopped || generation === void 0) return;
|
|
1143
|
+
const snapshot = await readIndexMetadataSnapshot(coordinator, indexName);
|
|
1144
|
+
const localUuid = engine.indexUuidOf(indexName);
|
|
1145
|
+
if (snapshot === null || typeof localUuid === "string" && localUuid !== snapshot.record.indexUuid) return;
|
|
1146
|
+
recordGeneration(indexName, snapshot);
|
|
1147
|
+
const { settings } = snapshot;
|
|
1148
|
+
const published = settings?.analysisGeneration ?? 0;
|
|
1149
|
+
const localPending = engine.pendingAnalysisGenerationOf(indexName);
|
|
1150
|
+
if (published > generation) {
|
|
1151
|
+
if (localPending !== published && engine.holdsAnalysisSettings(indexName, namedSettingsOfMetadata(settings))) {
|
|
1152
|
+
if (localPending !== null) await engine.dropHeldAnalysis(indexName);
|
|
1153
|
+
await engine.stampAnalysisGeneration(indexName, published);
|
|
1154
|
+
return;
|
|
1155
|
+
}
|
|
1156
|
+
catchUp(indexName, published, settings);
|
|
1157
|
+
return;
|
|
1158
|
+
}
|
|
1159
|
+
if (localPending !== null && localPending > published && settings?.pendingAnalysis?.generation !== localPending) {
|
|
1160
|
+
await engine.dropHeldAnalysis(indexName);
|
|
1161
|
+
}
|
|
1162
|
+
}
|
|
1163
|
+
function settle(indexName) {
|
|
1164
|
+
const next = (settling.get(indexName) ?? Promise.resolve()).then(() => settleOnce(indexName)).catch(onError);
|
|
1165
|
+
settling.set(indexName, next);
|
|
1166
|
+
return next;
|
|
1167
|
+
}
|
|
1168
|
+
async function reach(indexName, requested) {
|
|
1169
|
+
if (engine.heldAnalysisIsBuilt(indexName, requested)) {
|
|
1170
|
+
await engine.releaseHeldAnalysis(indexName, requested).catch(onError);
|
|
1171
|
+
return;
|
|
1172
|
+
}
|
|
1173
|
+
await settle(indexName);
|
|
1174
|
+
}
|
|
1175
|
+
async function reachThenRead(indexName, requested, generation) {
|
|
1176
|
+
if (generation < requested) await reach(indexName, requested);
|
|
1177
|
+
const reached = engine.analysisGenerationOf(indexName) ?? generation;
|
|
1178
|
+
if (reached !== requested) throw generationMissing(indexName, requested, reached);
|
|
1179
|
+
}
|
|
1180
|
+
return {
|
|
1181
|
+
held(indexName) {
|
|
1182
|
+
return held.get(indexName)?.generation;
|
|
1183
|
+
},
|
|
1184
|
+
async generationFor(indexName) {
|
|
1185
|
+
return held.get(indexName)?.generation ?? refresh(indexName);
|
|
1186
|
+
},
|
|
1187
|
+
refresh,
|
|
1188
|
+
answerAt(indexName, requested, read) {
|
|
1189
|
+
const generation = engine.analysisGenerationOf(indexName);
|
|
1190
|
+
if (generation === void 0) return read();
|
|
1191
|
+
const readAtRequested = (answerOf) => answerOf().then((answer) => {
|
|
1192
|
+
const after = engine.analysisGenerationOf(indexName) ?? requested;
|
|
1193
|
+
if (after !== requested) throw generationMissing(indexName, requested, after);
|
|
1194
|
+
return answer;
|
|
1195
|
+
});
|
|
1196
|
+
if (generation === requested) return readAtRequested(read);
|
|
1197
|
+
return reachThenRead(indexName, requested, generation).then(() => readAtRequested(read));
|
|
1198
|
+
},
|
|
1199
|
+
settle,
|
|
1200
|
+
async start() {
|
|
1201
|
+
if (unwatch !== null) return;
|
|
1202
|
+
stopped = false;
|
|
1203
|
+
unwatch = await coordinator.watchIndexMetadata((event) => {
|
|
1204
|
+
if (event.metadata === null) {
|
|
1205
|
+
held.delete(event.indexName);
|
|
1206
|
+
return;
|
|
1207
|
+
}
|
|
1208
|
+
try {
|
|
1209
|
+
recordGeneration(event.indexName, indexMetadataSnapshotOf(event.metadata, event.indexName));
|
|
1210
|
+
} catch (error) {
|
|
1211
|
+
onError(error);
|
|
1212
|
+
}
|
|
1213
|
+
void settle(event.indexName);
|
|
1214
|
+
});
|
|
1215
|
+
},
|
|
1216
|
+
stop() {
|
|
1217
|
+
stopped = true;
|
|
1218
|
+
unwatch?.();
|
|
1219
|
+
unwatch = null;
|
|
1220
|
+
held.clear();
|
|
1221
|
+
}
|
|
1222
|
+
};
|
|
1223
|
+
}
|
|
1092
1224
|
function settingOfMetadata(bytes, indexName) {
|
|
1093
1225
|
const decoded = decode(bytes);
|
|
1094
1226
|
if (!isRecord2(decoded) || typeof decoded.indexUuid !== "string") return null;
|
|
@@ -2375,14 +2507,16 @@ async function withRoutedIndexUuid(payload, deps) {
|
|
|
2375
2507
|
const indexUuid = (await deps.coordinator.getAllocation(payload.indexName))?.indexUuid;
|
|
2376
2508
|
return indexUuid === void 0 ? payload : { ...payload, indexUuid };
|
|
2377
2509
|
}
|
|
2378
|
-
async function reportSyncedPosition(indexName, partitionId, target,
|
|
2510
|
+
async function reportSyncedPosition(indexName, partitionId, target, deadline, deps) {
|
|
2379
2511
|
const logState = readLocalLogState(indexName, partitionId, deps);
|
|
2380
|
-
|
|
2381
|
-
return;
|
|
2382
|
-
}
|
|
2512
|
+
const details = { indexName, partitionId, target };
|
|
2383
2513
|
const budgetMs = Math.min(SYNCED_POSITION_REPORT_TIMEOUT_MS, deadline - Date.now());
|
|
2384
2514
|
if (budgetMs <= 0) {
|
|
2385
|
-
return
|
|
2515
|
+
return new NarsilError(
|
|
2516
|
+
ErrorCodes.SNAPSHOT_SYNC_TIMEOUT,
|
|
2517
|
+
"bootstrap sync exceeded deadline before reporting the synced position",
|
|
2518
|
+
details
|
|
2519
|
+
);
|
|
2386
2520
|
}
|
|
2387
2521
|
const payload = {
|
|
2388
2522
|
indexName,
|
|
@@ -2396,19 +2530,38 @@ async function reportSyncedPosition(indexName, partitionId, target, knownSeqNo,
|
|
|
2396
2530
|
requestId: generateId(),
|
|
2397
2531
|
payload: encode(await withRoutedIndexUuid(payload, deps))
|
|
2398
2532
|
};
|
|
2533
|
+
const frameState = createLiveSyncFrameState(
|
|
2534
|
+
indexName,
|
|
2535
|
+
partitionId,
|
|
2536
|
+
deps.snapshotSpoolDirectory ?? defaultSpoolDirectory()
|
|
2537
|
+
);
|
|
2399
2538
|
try {
|
|
2400
2539
|
await withDeadline(
|
|
2401
|
-
deps.transport.stream(target, request, () => {
|
|
2540
|
+
deps.transport.stream(target, request, (chunk) => {
|
|
2541
|
+
handleLiveSyncFrame(frameState, chunk);
|
|
2402
2542
|
}),
|
|
2403
2543
|
budgetMs,
|
|
2404
2544
|
indexName,
|
|
2405
2545
|
"synced-position-report"
|
|
2406
2546
|
);
|
|
2407
|
-
} catch (
|
|
2408
|
-
|
|
2409
|
-
|
|
2410
|
-
|
|
2547
|
+
} catch (err) {
|
|
2548
|
+
const cause = err instanceof Error ? err.message : String(err);
|
|
2549
|
+
return new NarsilError(
|
|
2550
|
+
ErrorCodes.SNAPSHOT_SYNC_TRANSPORT_FAILED,
|
|
2551
|
+
`reporting the synced position failed: ${cause}`,
|
|
2552
|
+
{ ...details, cause }
|
|
2553
|
+
);
|
|
2554
|
+
} finally {
|
|
2555
|
+
abandonSnapshotStream(frameState.snapshotState);
|
|
2411
2556
|
}
|
|
2557
|
+
if (frameState.sawSnapshotFrame) {
|
|
2558
|
+
return new NarsilError(
|
|
2559
|
+
ErrorCodes.SNAPSHOT_SYNC_TRANSPORT_FAILED,
|
|
2560
|
+
"the primary no longer retains the synced position, so the copy syncs again",
|
|
2561
|
+
details
|
|
2562
|
+
);
|
|
2563
|
+
}
|
|
2564
|
+
return frameState.error;
|
|
2412
2565
|
}
|
|
2413
2566
|
async function syncFromAnyTarget(state, entry, indexName, partitionId, primaryNodeId, targets, coordinatorConfig, partitionCount, deadline, deps, abortCheck) {
|
|
2414
2567
|
let lastError = new NarsilError(ErrorCodes.SNAPSHOT_SYNC_TRANSPORT_FAILED, "no live sync attempts were made", {
|
|
@@ -2443,7 +2596,6 @@ async function syncFromAnyTarget(state, entry, indexName, partitionId, primaryNo
|
|
|
2443
2596
|
)
|
|
2444
2597
|
};
|
|
2445
2598
|
}
|
|
2446
|
-
const positionBeforeAttempt = readLocalLogState(indexName, partitionId, deps).lastSeqNo;
|
|
2447
2599
|
const attempt = await syncFromTarget(
|
|
2448
2600
|
state,
|
|
2449
2601
|
entry,
|
|
@@ -2456,15 +2608,16 @@ async function syncFromAnyTarget(state, entry, indexName, partitionId, primaryNo
|
|
|
2456
2608
|
deps,
|
|
2457
2609
|
abortCheck
|
|
2458
2610
|
);
|
|
2459
|
-
|
|
2460
|
-
|
|
2611
|
+
const movedThePosition = attempt.ok && (attempt.tier === "snapshot" || attempt.entriesApplied > 0);
|
|
2612
|
+
const failure = !attempt.ok ? attempt.error : movedThePosition ? await reportSyncedPosition(indexName, partitionId, target, deadline, deps) : null;
|
|
2613
|
+
if (failure === null) {
|
|
2461
2614
|
return attempt;
|
|
2462
2615
|
}
|
|
2463
|
-
lastError = new NarsilError(
|
|
2464
|
-
...
|
|
2616
|
+
lastError = new NarsilError(failure.code, failure.message, {
|
|
2617
|
+
...failure.details,
|
|
2465
2618
|
target
|
|
2466
2619
|
});
|
|
2467
|
-
if (!isTransientFailure(
|
|
2620
|
+
if (!isTransientFailure(failure.code, failure.details)) {
|
|
2468
2621
|
return { ok: false, error: lastError };
|
|
2469
2622
|
}
|
|
2470
2623
|
}
|
|
@@ -4861,7 +5014,9 @@ function createEventLoopState(initialIndexNames, nodeLeftDelayMs = DEFAULT_CONTR
|
|
|
4861
5014
|
transport: null,
|
|
4862
5015
|
controllerNodeId: null,
|
|
4863
5016
|
rebalanceDrivers: /* @__PURE__ */ new Map(),
|
|
4864
|
-
rebalanceEpoch: 0
|
|
5017
|
+
rebalanceEpoch: 0,
|
|
5018
|
+
unwatchMetadata: null,
|
|
5019
|
+
analysisDrivers: /* @__PURE__ */ new Map()
|
|
4865
5020
|
};
|
|
4866
5021
|
}
|
|
4867
5022
|
function clearEventLoopWatchers(state) {
|
|
@@ -4893,6 +5048,9 @@ function clearEventLoopWatchers(state) {
|
|
|
4893
5048
|
state.departedAt.clear();
|
|
4894
5049
|
state.rebalanceDrivers.clear();
|
|
4895
5050
|
state.rebalanceEpoch += 1;
|
|
5051
|
+
state.unwatchMetadata?.();
|
|
5052
|
+
state.unwatchMetadata = null;
|
|
5053
|
+
state.analysisDrivers.clear();
|
|
4896
5054
|
}
|
|
4897
5055
|
function inSchemaQueue(state, indexName, task) {
|
|
4898
5056
|
const queued = (state.schemaQueues.get(indexName) ?? Promise.resolve()).then(task);
|
|
@@ -5436,6 +5594,212 @@ async function handleSchemaEvent(event, coordinator, state, isActive, onError) {
|
|
|
5436
5594
|
await tearDownAllocation(event.indexName, coordinator, state, isActive, 0, onError);
|
|
5437
5595
|
}
|
|
5438
5596
|
}
|
|
5597
|
+
|
|
5598
|
+
// src/distribution/query/node-send.ts
|
|
5599
|
+
async function resolveTargets(nodeId, deps) {
|
|
5600
|
+
if (deps.resolveNodeTargets === void 0) {
|
|
5601
|
+
return [nodeId];
|
|
5602
|
+
}
|
|
5603
|
+
const targets = await deps.resolveNodeTargets(nodeId);
|
|
5604
|
+
return targets.length > 0 ? targets : [nodeId];
|
|
5605
|
+
}
|
|
5606
|
+
async function sendToFirstReachableTarget2(transport, targets, message) {
|
|
5607
|
+
let lastError;
|
|
5608
|
+
for (const target of targets) {
|
|
5609
|
+
try {
|
|
5610
|
+
return await transport.send(target, message);
|
|
5611
|
+
} catch (error) {
|
|
5612
|
+
lastError = error;
|
|
5613
|
+
}
|
|
5614
|
+
}
|
|
5615
|
+
throw lastError instanceof Error ? lastError : new Error(String(lastError));
|
|
5616
|
+
}
|
|
5617
|
+
function fieldsOf(snapshot) {
|
|
5618
|
+
const decoded = decode(snapshot.bytes);
|
|
5619
|
+
const fields = isRecord2(decoded) ? decoded : {};
|
|
5620
|
+
const settings = isRecord2(fields.settings) ? fields.settings : { language: snapshot.settings?.language };
|
|
5621
|
+
return { fields, settings };
|
|
5622
|
+
}
|
|
5623
|
+
function encoded(fields, settings) {
|
|
5624
|
+
return new Uint8Array(encode({ ...fields, settings }, { ignoreUndefined: true }));
|
|
5625
|
+
}
|
|
5626
|
+
function metadataWithPendingAnalysis(snapshot, pending) {
|
|
5627
|
+
const { fields, settings } = fieldsOf(snapshot);
|
|
5628
|
+
return encoded(fields, { ...settings, pendingAnalysis: pending });
|
|
5629
|
+
}
|
|
5630
|
+
function metadataWithoutPendingAnalysis(snapshot) {
|
|
5631
|
+
const { fields, settings } = fieldsOf(snapshot);
|
|
5632
|
+
const { pendingAnalysis: _pending, ...rest } = settings;
|
|
5633
|
+
return encoded(fields, rest);
|
|
5634
|
+
}
|
|
5635
|
+
function metadataPublishingAnalysis(snapshot, pending) {
|
|
5636
|
+
const { fields, settings } = fieldsOf(snapshot);
|
|
5637
|
+
const {
|
|
5638
|
+
pendingAnalysis: _pending,
|
|
5639
|
+
analysisFailure: _failure,
|
|
5640
|
+
tokenizer: _tokenizer,
|
|
5641
|
+
stopWords: _stopWords,
|
|
5642
|
+
stopWordList: _stopWordList,
|
|
5643
|
+
...rest
|
|
5644
|
+
} = settings;
|
|
5645
|
+
return encoded(fields, {
|
|
5646
|
+
...rest,
|
|
5647
|
+
language: pending.language,
|
|
5648
|
+
analysisGeneration: pending.generation,
|
|
5649
|
+
...pending.tokenizer !== void 0 ? { tokenizer: pending.tokenizer } : {},
|
|
5650
|
+
...pending.stopWords !== void 0 ? { stopWords: pending.stopWords } : {},
|
|
5651
|
+
...pending.stopWordList !== void 0 ? { stopWordList: pending.stopWordList } : {}
|
|
5652
|
+
});
|
|
5653
|
+
}
|
|
5654
|
+
function metadataWithAnalysisFailure(snapshot, failure) {
|
|
5655
|
+
const { fields, settings } = fieldsOf(snapshot);
|
|
5656
|
+
const { pendingAnalysis: _pending, ...rest } = settings;
|
|
5657
|
+
return encoded(fields, { ...rest, analysisFailure: failure });
|
|
5658
|
+
}
|
|
5659
|
+
|
|
5660
|
+
// src/distribution/cluster/node-targets.ts
|
|
5661
|
+
function targetsOfRegisteredNode(nodeId, address) {
|
|
5662
|
+
return address.length === 0 || address === nodeId ? [nodeId] : [address, nodeId];
|
|
5663
|
+
}
|
|
5664
|
+
|
|
5665
|
+
// src/distribution/cluster/rebalance/pause.ts
|
|
5666
|
+
function pause(milliseconds) {
|
|
5667
|
+
return new Promise((resolve) => {
|
|
5668
|
+
const timer = setTimeout(resolve, milliseconds);
|
|
5669
|
+
timer.unref?.();
|
|
5670
|
+
});
|
|
5671
|
+
}
|
|
5672
|
+
|
|
5673
|
+
// src/distribution/cluster/controller/event-loop/analysis-change.ts
|
|
5674
|
+
var REFUSALS_THAT_FAIL_A_CHANGE = /* @__PURE__ */ new Set([
|
|
5675
|
+
ErrorCodes.ANALYSIS_REBUILD_MEMORY_EXCEEDED,
|
|
5676
|
+
ErrorCodes.LANGUAGE_NOT_SUPPORTED,
|
|
5677
|
+
ErrorCodes.CONFIG_INVALID
|
|
5678
|
+
]);
|
|
5679
|
+
async function registeredHolders(coordinator, indexName) {
|
|
5680
|
+
const table = await coordinator.getAllocation(indexName);
|
|
5681
|
+
if (table === null) return null;
|
|
5682
|
+
const registered = new Map((await coordinator.listNodes()).map((node) => [node.nodeId, node.address]));
|
|
5683
|
+
const holders = /* @__PURE__ */ new Map();
|
|
5684
|
+
const addHolder = (nodeId) => {
|
|
5685
|
+
if (nodeId === null) return;
|
|
5686
|
+
const address = registered.get(nodeId);
|
|
5687
|
+
if (address === void 0) return;
|
|
5688
|
+
holders.set(nodeId, targetsOfRegisteredNode(nodeId, address));
|
|
5689
|
+
};
|
|
5690
|
+
for (const assignment of table.assignments.values()) {
|
|
5691
|
+
addHolder(assignment.primary);
|
|
5692
|
+
for (const replica of assignment.replicas) addHolder(replica);
|
|
5693
|
+
for (const replica of assignment.removingReplicas ?? []) addHolder(replica);
|
|
5694
|
+
}
|
|
5695
|
+
return holders;
|
|
5696
|
+
}
|
|
5697
|
+
async function sendRebuild(ctx, targets, payload) {
|
|
5698
|
+
const transport = ctx.state.transport;
|
|
5699
|
+
if (transport === null) throw new NarsilError(ErrorCodes.NODE_NOT_CONTROLLER, "The controller has no transport");
|
|
5700
|
+
const message = {
|
|
5701
|
+
type: ClusterMessageTypes.REBUILD_ANALYSIS,
|
|
5702
|
+
sourceId: ctx.nodeId,
|
|
5703
|
+
requestId: generateId(),
|
|
5704
|
+
payload: encode(payload)
|
|
5705
|
+
};
|
|
5706
|
+
const response = await sendToFirstReachableTarget2(transport, targets, message);
|
|
5707
|
+
const decoded = decode(response.payload);
|
|
5708
|
+
if (!response.type.endsWith(".error") && !(isRecord(decoded) && decoded.error === true)) return;
|
|
5709
|
+
const reply2 = isRecord(decoded) ? decoded : {};
|
|
5710
|
+
throw new NarsilError(
|
|
5711
|
+
typeof reply2.code === "string" ? reply2.code : ErrorCodes.QUERY_ROUTING_FAILED,
|
|
5712
|
+
typeof reply2.message === "string" ? reply2.message : "The node refused an analysis rebuild"
|
|
5713
|
+
);
|
|
5714
|
+
}
|
|
5715
|
+
function askHolder(ctx, progress, holder, targets, payload) {
|
|
5716
|
+
progress.asking.add(holder);
|
|
5717
|
+
const resend = new Promise((resolve) => {
|
|
5718
|
+
const timer = setTimeout(() => resolve("resend"), ANALYSIS_REBUILD_RESEND_MS);
|
|
5719
|
+
if (typeof timer === "object" && "unref" in timer) timer.unref();
|
|
5720
|
+
});
|
|
5721
|
+
const asked = sendRebuild(ctx, targets, payload).then(
|
|
5722
|
+
() => {
|
|
5723
|
+
progress.answered.add(holder);
|
|
5724
|
+
},
|
|
5725
|
+
(error) => {
|
|
5726
|
+
if (error instanceof NarsilError && REFUSALS_THAT_FAIL_A_CHANGE.has(error.code)) {
|
|
5727
|
+
progress.refusal = {
|
|
5728
|
+
changeId: progress.changeId,
|
|
5729
|
+
code: error.code,
|
|
5730
|
+
message: describeError(error),
|
|
5731
|
+
nodeId: holder
|
|
5732
|
+
};
|
|
5733
|
+
}
|
|
5734
|
+
}
|
|
5735
|
+
);
|
|
5736
|
+
void Promise.race([asked, resend]).finally(() => progress.asking.delete(holder));
|
|
5737
|
+
}
|
|
5738
|
+
async function stepChange(ctx, indexName, snapshot, pending, progress) {
|
|
5739
|
+
const key = indexConfigKey(indexName);
|
|
5740
|
+
if (progress.refusal !== null) {
|
|
5741
|
+
await ctx.coordinator.compareAndSet(key, snapshot.bytes, metadataWithAnalysisFailure(snapshot, progress.refusal));
|
|
5742
|
+
return;
|
|
5743
|
+
}
|
|
5744
|
+
const holders = await registeredHolders(ctx.coordinator, indexName) ?? /* @__PURE__ */ new Map();
|
|
5745
|
+
const payload = {
|
|
5746
|
+
indexName,
|
|
5747
|
+
indexUuid: snapshot.record.indexUuid,
|
|
5748
|
+
generation: pending.generation
|
|
5749
|
+
};
|
|
5750
|
+
let waiting = false;
|
|
5751
|
+
for (const [holder, targets] of holders) {
|
|
5752
|
+
if (progress.answered.has(holder)) continue;
|
|
5753
|
+
waiting = true;
|
|
5754
|
+
if (!progress.asking.has(holder)) askHolder(ctx, progress, holder, targets, payload);
|
|
5755
|
+
}
|
|
5756
|
+
if (!waiting) {
|
|
5757
|
+
await ctx.coordinator.compareAndSet(key, snapshot.bytes, metadataPublishingAnalysis(snapshot, pending));
|
|
5758
|
+
return;
|
|
5759
|
+
}
|
|
5760
|
+
await pause(ANALYSIS_CHANGE_POLL_INTERVAL_MS);
|
|
5761
|
+
}
|
|
5762
|
+
function progressOf(changeId) {
|
|
5763
|
+
return { changeId, answered: /* @__PURE__ */ new Set(), asking: /* @__PURE__ */ new Set(), refusal: null };
|
|
5764
|
+
}
|
|
5765
|
+
async function driveChanges(ctx, indexName) {
|
|
5766
|
+
let progress = progressOf("");
|
|
5767
|
+
while (ctx.isActive()) {
|
|
5768
|
+
const snapshot = await readIndexMetadataSnapshot(ctx.coordinator, indexName);
|
|
5769
|
+
const pending = snapshot?.settings?.pendingAnalysis;
|
|
5770
|
+
if (snapshot === null || pending === void 0) return;
|
|
5771
|
+
if (progress.changeId !== pending.changeId) progress = progressOf(pending.changeId);
|
|
5772
|
+
await stepChange(ctx, indexName, snapshot, pending, progress);
|
|
5773
|
+
}
|
|
5774
|
+
}
|
|
5775
|
+
function startAnalysisDriver(ctx, indexName) {
|
|
5776
|
+
const drivers = ctx.state.analysisDrivers;
|
|
5777
|
+
const running = drivers.get(indexName);
|
|
5778
|
+
if (running !== void 0) {
|
|
5779
|
+
running.again = true;
|
|
5780
|
+
return;
|
|
5781
|
+
}
|
|
5782
|
+
const driver = { again: false };
|
|
5783
|
+
drivers.set(indexName, driver);
|
|
5784
|
+
const drive2 = async () => {
|
|
5785
|
+
do {
|
|
5786
|
+
driver.again = false;
|
|
5787
|
+
await driveChanges(ctx, indexName).catch(async (error) => {
|
|
5788
|
+
ctx.onError?.(indexName, error);
|
|
5789
|
+
driver.again = true;
|
|
5790
|
+
await pause(ANALYSIS_CHANGE_RETRY_MS);
|
|
5791
|
+
});
|
|
5792
|
+
} while (driver.again && ctx.isActive());
|
|
5793
|
+
if (drivers.get(indexName) === driver) drivers.delete(indexName);
|
|
5794
|
+
};
|
|
5795
|
+
void drive2();
|
|
5796
|
+
}
|
|
5797
|
+
async function resumeAnalysisDrivers(ctx) {
|
|
5798
|
+
for (const indexName of await ctx.coordinator.listSchemas()) {
|
|
5799
|
+
const snapshot = await readIndexMetadataSnapshot(ctx.coordinator, indexName);
|
|
5800
|
+
if (snapshot?.settings?.pendingAnalysis !== void 0) startAnalysisDriver(ctx, indexName);
|
|
5801
|
+
}
|
|
5802
|
+
}
|
|
5439
5803
|
function validateBootstrapCompletePayload(decoded) {
|
|
5440
5804
|
if (!isRecord(decoded)) {
|
|
5441
5805
|
return null;
|
|
@@ -5967,7 +6331,10 @@ async function dropGeneration(coordinator, writerNodeId, generation) {
|
|
|
5967
6331
|
const key = indexConfigKey(generation);
|
|
5968
6332
|
const metadata = await coordinator.get(key);
|
|
5969
6333
|
if (metadata !== null && metadata.byteLength > 0) {
|
|
5970
|
-
await coordinator.compareAndSet(key, metadata, new Uint8Array(0))
|
|
6334
|
+
if (await coordinator.compareAndSet(key, metadata, new Uint8Array(0))) {
|
|
6335
|
+
const stored = storedSynonymSetOf(metadata);
|
|
6336
|
+
if (stored !== null) await deleteSynonymSet(coordinator, stored.indexUuid, stored.ref);
|
|
6337
|
+
}
|
|
5971
6338
|
}
|
|
5972
6339
|
if (await coordinator.getSchema(generation) !== null) {
|
|
5973
6340
|
await coordinator.dropSchema(generation);
|
|
@@ -5975,20 +6342,29 @@ async function dropGeneration(coordinator, writerNodeId, generation) {
|
|
|
5975
6342
|
}
|
|
5976
6343
|
await coordinator.deleteAllocation(generation);
|
|
5977
6344
|
}
|
|
6345
|
+
async function dropReplacedNameSynonymSet(coordinator, indexName) {
|
|
6346
|
+
for (let attempt = 0; attempt < TEARDOWN_CAS_ATTEMPTS; attempt++) {
|
|
6347
|
+
const snapshot = await readIndexMetadataSnapshot(coordinator, indexName);
|
|
6348
|
+
const ref = snapshot?.settings?.synonyms;
|
|
6349
|
+
if (snapshot === null || ref === void 0) return;
|
|
6350
|
+
const next = metadataWithSynonymSet(snapshot, null);
|
|
6351
|
+
if (await coordinator.compareAndSet(indexConfigKey(indexName), snapshot.bytes, next)) {
|
|
6352
|
+
await deleteSynonymSet(coordinator, snapshot.record.indexUuid, ref);
|
|
6353
|
+
return;
|
|
6354
|
+
}
|
|
6355
|
+
}
|
|
6356
|
+
throw new NarsilError(
|
|
6357
|
+
ErrorCodes.ALLOCATION_FAILED,
|
|
6358
|
+
`Releasing the replaced synonym set of index '${indexName}' lost every compare-and-set attempt`,
|
|
6359
|
+
{ indexName, attempts: TEARDOWN_CAS_ATTEMPTS }
|
|
6360
|
+
);
|
|
6361
|
+
}
|
|
5978
6362
|
async function clearEveryProgress(coordinator, indexName, partitionCount) {
|
|
5979
6363
|
for (let partitionId = 0; partitionId < partitionCount; partitionId++) {
|
|
5980
6364
|
await clearRebalanceProgress(coordinator, indexName, partitionId);
|
|
5981
6365
|
}
|
|
5982
6366
|
}
|
|
5983
6367
|
|
|
5984
|
-
// src/distribution/cluster/rebalance/pause.ts
|
|
5985
|
-
function pause(milliseconds) {
|
|
5986
|
-
return new Promise((resolve) => {
|
|
5987
|
-
const timer = setTimeout(resolve, milliseconds);
|
|
5988
|
-
timer.unref?.();
|
|
5989
|
-
});
|
|
5990
|
-
}
|
|
5991
|
-
|
|
5992
6368
|
// src/distribution/cluster/controller/event-loop/rebalance/primaries.ts
|
|
5993
6369
|
async function targetsOf(ctx, nodeId) {
|
|
5994
6370
|
const nodes = await ctx.coordinator.listNodes();
|
|
@@ -6204,6 +6580,7 @@ async function finishSwap(ctx, indexName, record) {
|
|
|
6204
6580
|
if (record.sourceGeneration === indexName) {
|
|
6205
6581
|
const metadata = await getIndexMetadata(ctx.coordinator, indexName);
|
|
6206
6582
|
await clearEveryProgress(ctx.coordinator, indexName, metadata?.partitionCount ?? 0);
|
|
6583
|
+
await dropReplacedNameSynonymSet(ctx.coordinator, indexName);
|
|
6207
6584
|
} else {
|
|
6208
6585
|
const source = await ctx.coordinator.getAllocation(record.sourceGeneration);
|
|
6209
6586
|
await clearEveryProgress(ctx.coordinator, indexName, source?.assignments.size ?? 0);
|
|
@@ -6365,6 +6742,15 @@ async function recordRebalance(ctx, message) {
|
|
|
6365
6742
|
{ indexName, jobId: running?.jobId ?? null }
|
|
6366
6743
|
);
|
|
6367
6744
|
}
|
|
6745
|
+
const latest = await readIndexMetadataSnapshot(ctx.coordinator, sourceGeneration);
|
|
6746
|
+
if (latest?.settings?.pendingAnalysis !== void 0) {
|
|
6747
|
+
await updateRebalanceRecord(ctx.coordinator, indexName, record.jobId, () => null);
|
|
6748
|
+
throw new NarsilError(
|
|
6749
|
+
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
6750
|
+
`Index '${indexName}' is changing its analysis, so try its partition count change again once that ends`,
|
|
6751
|
+
{ indexName }
|
|
6752
|
+
);
|
|
6753
|
+
}
|
|
6368
6754
|
startRebalanceDriver(ctx, indexName, record.jobId);
|
|
6369
6755
|
const result = {
|
|
6370
6756
|
indexName,
|
|
@@ -6492,6 +6878,11 @@ async function startEventLoop(state, coordinator, transport, nodeId, isActive, o
|
|
|
6492
6878
|
});
|
|
6493
6879
|
void resumeRebalance({ state, coordinator, nodeId, isActive, onError }, event.indexName);
|
|
6494
6880
|
});
|
|
6881
|
+
state.unwatchMetadata = await coordinator.watchIndexMetadata((event) => {
|
|
6882
|
+
if (event.metadata !== null && isActive()) {
|
|
6883
|
+
startAnalysisDriver({ state, coordinator, nodeId, isActive, onError }, event.indexName);
|
|
6884
|
+
}
|
|
6885
|
+
});
|
|
6495
6886
|
state.unwatchTransport = await transport.listen((message, respond) => {
|
|
6496
6887
|
if (message.type === ClusterMessageTypes.REPLICATION_FACTOR) {
|
|
6497
6888
|
handleReplicationFactorMessage(state, message, respond, coordinator, nodeId, isActive, onError);
|
|
@@ -6520,6 +6911,7 @@ async function startEventLoop(state, coordinator, transport, nodeId, isActive, o
|
|
|
6520
6911
|
}
|
|
6521
6912
|
scheduleDebouncedAllocation(state, coordinator, isActive, onError);
|
|
6522
6913
|
resumeRebalances({ state, coordinator, nodeId, isActive, onError }).catch((error) => onError?.("", error));
|
|
6914
|
+
resumeAnalysisDrivers({ state, coordinator, nodeId, isActive, onError }).catch((error) => onError?.("", error));
|
|
6523
6915
|
}
|
|
6524
6916
|
|
|
6525
6917
|
// src/distribution/cluster/controller/index.ts
|
|
@@ -7248,6 +7640,35 @@ async function servingNameOf(coordinator, indexName) {
|
|
|
7248
7640
|
const table = await coordinator.getAllocation(indexName);
|
|
7249
7641
|
return table?.generation ?? indexName;
|
|
7250
7642
|
}
|
|
7643
|
+
var CODES_OF_A_LOST_LAYOUT = /* @__PURE__ */ new Set([
|
|
7644
|
+
ErrorCodes.INDEX_NOT_FOUND,
|
|
7645
|
+
ErrorCodes.QUERY_NO_ACTIVE_REPLICA,
|
|
7646
|
+
ErrorCodes.QUERY_NODE_TIMEOUT,
|
|
7647
|
+
ErrorCodes.QUERY_PARTIAL_FAILURE,
|
|
7648
|
+
ErrorCodes.QUERY_ROUTING_FAILED
|
|
7649
|
+
]);
|
|
7650
|
+
function leftOutAPartition(coverage) {
|
|
7651
|
+
return coverage.totalPartitions === 0 || coverage.queriedPartitions < coverage.totalPartitions;
|
|
7652
|
+
}
|
|
7653
|
+
async function movedNameOf(names, indexName, servingName) {
|
|
7654
|
+
const live = await names.liveServingName(indexName).catch(() => servingName);
|
|
7655
|
+
return live === servingName ? null : live;
|
|
7656
|
+
}
|
|
7657
|
+
async function readFollowingTheName(names, indexName, read, answerIsPartial) {
|
|
7658
|
+
const servingName = await names.servingName(indexName);
|
|
7659
|
+
let answer;
|
|
7660
|
+
try {
|
|
7661
|
+
answer = await read(servingName);
|
|
7662
|
+
} catch (error) {
|
|
7663
|
+
if (!(error instanceof NarsilError) || !CODES_OF_A_LOST_LAYOUT.has(error.code)) throw error;
|
|
7664
|
+
const moved2 = await movedNameOf(names, indexName, servingName);
|
|
7665
|
+
if (moved2 === null) throw error;
|
|
7666
|
+
return read(moved2);
|
|
7667
|
+
}
|
|
7668
|
+
if (answerIsPartial === void 0 || !answerIsPartial(answer)) return answer;
|
|
7669
|
+
const moved = await movedNameOf(names, indexName, servingName);
|
|
7670
|
+
return moved === null ? answer : read(moved);
|
|
7671
|
+
}
|
|
7251
7672
|
|
|
7252
7673
|
// src/persistence/durability/partition-log.ts
|
|
7253
7674
|
function persistedPosition(manager, indexName, partitionId) {
|
|
@@ -7953,6 +8374,9 @@ function requireRestoreTarget(indexName, partitionId, partitionCount) {
|
|
|
7953
8374
|
async function receiveArchive(indexName, partitionId, source, schema) {
|
|
7954
8375
|
const store = memoryCheckpointStore();
|
|
7955
8376
|
let language = "";
|
|
8377
|
+
let tokenizer;
|
|
8378
|
+
let stopWords;
|
|
8379
|
+
let stopWordList;
|
|
7956
8380
|
let analysisRevision = null;
|
|
7957
8381
|
let arrayPositionGap = null;
|
|
7958
8382
|
let received = null;
|
|
@@ -7966,6 +8390,9 @@ async function receiveArchive(indexName, partitionId, source, schema) {
|
|
|
7966
8390
|
const raw = decode(payload);
|
|
7967
8391
|
if (typeof raw.language !== "string") restoreFailed("partition archive names no language", { indexName });
|
|
7968
8392
|
language = raw.language;
|
|
8393
|
+
tokenizer = raw.tokenizer;
|
|
8394
|
+
stopWords = raw.stopWords;
|
|
8395
|
+
stopWordList = raw.stopWordList;
|
|
7969
8396
|
analysisRevision = typeof raw.analysisRevision === "string" ? raw.analysisRevision : null;
|
|
7970
8397
|
arrayPositionGap = storedArrayPositionGap(raw.arrayPositionGap);
|
|
7971
8398
|
},
|
|
@@ -7994,7 +8421,15 @@ async function receiveArchive(indexName, partitionId, source, schema) {
|
|
|
7994
8421
|
file: async (name, _ref, bytes) => store.write(`${indexName}/${name}`, [bytes])
|
|
7995
8422
|
});
|
|
7996
8423
|
if (received === null) restoreFailed("partition archive holds no manifest", { indexName, partitionId });
|
|
7997
|
-
return { language, analysisRevision, arrayPositionGap, manifest: received, store };
|
|
8424
|
+
return { language, tokenizer, stopWords, stopWordList, analysisRevision, arrayPositionGap, manifest: received, store };
|
|
8425
|
+
}
|
|
8426
|
+
function requireArchiveUnderLocalAnalysis(core, indexName, archive) {
|
|
8427
|
+
const entry = core.requireIndex(indexName);
|
|
8428
|
+
const local = namedSettingsOf(entry.config, entry.language.name);
|
|
8429
|
+
const sameList2 = JSON.stringify(local.stopWordList) === JSON.stringify(archive.stopWordList);
|
|
8430
|
+
if (local.tokenizer !== archive.tokenizer || local.stopWords !== archive.stopWords || !sameList2) {
|
|
8431
|
+
restoreFailed("partition archive holds terms from other analysis settings, so the copy tries again", { indexName });
|
|
8432
|
+
}
|
|
7998
8433
|
}
|
|
7999
8434
|
async function replacePartition(manager, partitionId, archive, writeRecords) {
|
|
8000
8435
|
const vectorIndexes = manager.getVectorIndexes();
|
|
@@ -8070,6 +8505,7 @@ async function restoreReplicationPartition(core, engine, indexName, partitionId,
|
|
|
8070
8505
|
partitionCount
|
|
8071
8506
|
);
|
|
8072
8507
|
if (release === null) release = await core.indexState.acquire(indexName, false);
|
|
8508
|
+
requireArchiveUnderLocalAnalysis(core, indexName, archive);
|
|
8073
8509
|
if (restoreIndex.created) {
|
|
8074
8510
|
const synonyms = core.requireIndex(indexName).synonyms;
|
|
8075
8511
|
await core.orchestrator.replicateToWorkers({
|
|
@@ -8422,6 +8858,14 @@ async function createClusterLocalEngine(config, hooks) {
|
|
|
8422
8858
|
stampIndexUuid: (indexName, indexUuid) => stampIndexUuid(core, indexName, indexUuid),
|
|
8423
8859
|
applyStoredSynonyms: (indexName, load) => synonymModule().applyLoaded(core, indexName, load),
|
|
8424
8860
|
applyStoredBm25: (indexName, load) => applyStoredIndexBm25(core, indexName, load),
|
|
8861
|
+
analysisGenerationOf: (indexName) => core.indexRegistry.get(indexName)?.analysisGeneration,
|
|
8862
|
+
pendingAnalysisGenerationOf: (indexName) => core.indexRegistry.get(indexName)?.pendingAnalysis?.generation ?? null,
|
|
8863
|
+
buildHeldAnalysis: (indexName, pending) => buildHeldAnalysisChange(core, indexName, pending),
|
|
8864
|
+
heldAnalysisIsBuilt: (indexName, generation) => heldAnalysisIsBuilt(core, indexName, generation),
|
|
8865
|
+
releaseHeldAnalysis: (indexName, generation) => releaseHeldAnalysisChange(core, indexName, generation),
|
|
8866
|
+
dropHeldAnalysis: (indexName) => dropHeldAnalysisChange(core, indexName),
|
|
8867
|
+
holdsAnalysisSettings: (indexName, settings) => holdsNamedSettings(core.requireIndex(indexName), settings),
|
|
8868
|
+
stampAnalysisGeneration: (indexName, generation) => stampAnalysisGeneration(core, indexName, generation),
|
|
8425
8869
|
highestPersistedSeqNoOf: (indexName, partitionId) => core.durability?.manager.highestPersistedSeqNo(indexName, partitionId) ?? 0,
|
|
8426
8870
|
persistedPositionOf: (indexName, partitionId) => persistedPositionOf(core, indexName, partitionId),
|
|
8427
8871
|
numberEntriesWith: (numbering, defersCommit) => core.durability?.manager.numberEntriesWith?.(numbering, defersCommit),
|
|
@@ -8610,6 +9054,15 @@ function validateStringField(value, fieldLabel, maxLength, errorCode) {
|
|
|
8610
9054
|
function validateFieldName(value, fieldLabel, errorCode) {
|
|
8611
9055
|
return validateStringField(value, fieldLabel, MAX_FIELD_NAME_LENGTH, errorCode);
|
|
8612
9056
|
}
|
|
9057
|
+
function validateAnalysisGeneration(value, payloadName) {
|
|
9058
|
+
if (value === void 0 || value === null) return;
|
|
9059
|
+
if (!Number.isSafeInteger(value) || value < 0) {
|
|
9060
|
+
throwInvalid(
|
|
9061
|
+
ErrorCodes.CONFIG_INVALID,
|
|
9062
|
+
`Invalid ${payloadName}: "analysisGeneration" must be a non-negative integer`
|
|
9063
|
+
);
|
|
9064
|
+
}
|
|
9065
|
+
}
|
|
8613
9066
|
function validateNonNegativeInteger(value, fieldLabel, maxValue, errorCode) {
|
|
8614
9067
|
if (!isInteger(value)) {
|
|
8615
9068
|
throwInvalid(errorCode, `Invalid payload: "${fieldLabel}" must be a finite non-negative integer`);
|
|
@@ -9506,6 +9959,7 @@ function validateParams(params) {
|
|
|
9506
9959
|
}
|
|
9507
9960
|
if (params.bm25 !== void 0 && params.bm25 !== null) validateBM25Shape(params.bm25, "settable");
|
|
9508
9961
|
if (params.indexBm25 !== void 0 && params.indexBm25 !== null) validateBM25Shape(params.indexBm25, "stored");
|
|
9962
|
+
validateAnalysisGeneration(params.analysisGeneration, "SearchPayload");
|
|
9509
9963
|
if (params.tolerance !== null) {
|
|
9510
9964
|
if (!isInteger(params.tolerance) || params.tolerance < 0 || params.tolerance > MAX_TOLERANCE) {
|
|
9511
9965
|
throwInvalid(
|
|
@@ -9744,6 +10198,7 @@ function validateSuggestPayload(decoded) {
|
|
|
9744
10198
|
validatePartitionIdsArray(decoded.partitionIds, "partitionIds", CONFIG_INVALID);
|
|
9745
10199
|
validateStringField(decoded.prefix, "prefix", MAX_TERM_LENGTH, CONFIG_INVALID);
|
|
9746
10200
|
validatePositiveInteger(decoded.limit, "limit", MAX_SUGGEST_WIRE_LIMIT, CONFIG_INVALID);
|
|
10201
|
+
validateAnalysisGeneration(decoded.analysisGeneration, "SuggestPayload");
|
|
9747
10202
|
return decoded;
|
|
9748
10203
|
}
|
|
9749
10204
|
function validateSuggestResultPayload(decoded) {
|
|
@@ -10079,6 +10534,7 @@ function validateStatsPayload(decoded) {
|
|
|
10079
10534
|
validateStringArray(decoded.terms, "terms", MAX_TERMS_COUNT, MAX_TERM_LENGTH, CONFIG_INVALID);
|
|
10080
10535
|
validateExpansion(decoded.expansion);
|
|
10081
10536
|
if (!isAbsent(decoded.tokens)) validateTokenList(decoded.tokens, "StatsPayload.tokens");
|
|
10537
|
+
validateAnalysisGeneration(decoded.analysisGeneration, "StatsPayload");
|
|
10082
10538
|
return decoded;
|
|
10083
10539
|
}
|
|
10084
10540
|
function validateStatsResultPayload(decoded) {
|
|
@@ -10199,7 +10655,24 @@ var QUERY_REJECTION_CODES = /* @__PURE__ */ new Set([
|
|
|
10199
10655
|
ErrorCodes.CONFIG_INVALID
|
|
10200
10656
|
]);
|
|
10201
10657
|
var nodeRejections = /* @__PURE__ */ new WeakSet();
|
|
10658
|
+
function refusedForNewerGeneration(details) {
|
|
10659
|
+
const held = details?.heldGeneration;
|
|
10660
|
+
const requested = details?.requestedGeneration;
|
|
10661
|
+
return typeof held === "number" && typeof requested === "number" && held > requested;
|
|
10662
|
+
}
|
|
10663
|
+
function generationRefusalOf(code, message, details) {
|
|
10664
|
+
if (code !== ErrorCodes.ANALYSIS_GENERATION_MISSING) return null;
|
|
10665
|
+
const refusal = new NarsilError(
|
|
10666
|
+
ErrorCodes.ANALYSIS_GENERATION_MISSING,
|
|
10667
|
+
typeof message === "string" ? message : code,
|
|
10668
|
+
details
|
|
10669
|
+
);
|
|
10670
|
+
if (refusedForNewerGeneration(details)) nodeRejections.add(refusal);
|
|
10671
|
+
return refusal;
|
|
10672
|
+
}
|
|
10202
10673
|
function nodeRejectionOf(code, message, details) {
|
|
10674
|
+
const generationRefusal = generationRefusalOf(code, message, details);
|
|
10675
|
+
if (generationRefusal !== null) return generationRefusal;
|
|
10203
10676
|
if (typeof code !== "string" || !QUERY_REJECTION_CODES.has(code)) return null;
|
|
10204
10677
|
const rejection = new NarsilError(code, typeof message === "string" ? message : code, details);
|
|
10205
10678
|
nodeRejections.add(rejection);
|
|
@@ -10211,6 +10684,7 @@ function markNodeRejection(error) {
|
|
|
10211
10684
|
}
|
|
10212
10685
|
function rethrowAsRefusal(error) {
|
|
10213
10686
|
if (error instanceof NarsilError && QUERY_REJECTION_CODES.has(error.code)) markNodeRejection(error);
|
|
10687
|
+
if (error instanceof NarsilError && refusedForNewerGeneration(error.details)) markNodeRejection(error);
|
|
10214
10688
|
throw error;
|
|
10215
10689
|
}
|
|
10216
10690
|
function isNodeRejection(error) {
|
|
@@ -13221,9 +13695,7 @@ async function resolveNodeTargets2(config, targetNodeId) {
|
|
|
13221
13695
|
async function resolveRegisteredTargets(config, targetNodeId) {
|
|
13222
13696
|
const nodes = await config.coordinator.listNodes();
|
|
13223
13697
|
const registration = nodes.find((node) => node.nodeId === targetNodeId);
|
|
13224
|
-
|
|
13225
|
-
if (registration.address.length === 0 || registration.address === targetNodeId) return [targetNodeId];
|
|
13226
|
-
return [registration.address, targetNodeId];
|
|
13698
|
+
return registration === void 0 ? [] : targetsOfRegisteredNode(targetNodeId, registration.address);
|
|
13227
13699
|
}
|
|
13228
13700
|
async function sendToNode2(config, targetNodeId, message, failure) {
|
|
13229
13701
|
const targets = await resolveNodeTargets2(config, targetNodeId);
|
|
@@ -13328,7 +13800,6 @@ async function readDistributedDocuments(config, nodeId, engine, indexName, docId
|
|
|
13328
13800
|
}
|
|
13329
13801
|
|
|
13330
13802
|
// src/distribution/cluster-node/synonym-sets.ts
|
|
13331
|
-
var SETTINGS_CONTENTION_REASON = "settings_contention";
|
|
13332
13803
|
function indexNotFound(indexName) {
|
|
13333
13804
|
return new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index '${indexName}' does not exist in the cluster`, {
|
|
13334
13805
|
indexName
|
|
@@ -13336,10 +13807,19 @@ function indexNotFound(indexName) {
|
|
|
13336
13807
|
}
|
|
13337
13808
|
async function readClusterSynonyms(coordinator, indexName) {
|
|
13338
13809
|
validateIndexName(indexName);
|
|
13339
|
-
const state = await readIndexState(coordinator, indexName);
|
|
13810
|
+
const state = await readIndexState(coordinator, await servingNameOf(coordinator, indexName));
|
|
13340
13811
|
if (state === null) throw indexNotFound(indexName);
|
|
13341
13812
|
return state.synonyms;
|
|
13342
13813
|
}
|
|
13814
|
+
async function requireSettledPartitionCount(coordinator, indexName, servingName) {
|
|
13815
|
+
const settled = await readRebalanceRecord(coordinator, indexName) === null && await servingNameOf(coordinator, indexName) === servingName;
|
|
13816
|
+
if (settled) return;
|
|
13817
|
+
throw new NarsilError(
|
|
13818
|
+
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
13819
|
+
`Index '${indexName}' is changing its partition count, so replace its synonym set again once that ends`,
|
|
13820
|
+
{ indexName }
|
|
13821
|
+
);
|
|
13822
|
+
}
|
|
13343
13823
|
function applyStoredSynonyms(engine, coordinator, indexName, indexUuid) {
|
|
13344
13824
|
if (engine.indexUuidOf(indexName) !== indexUuid) return Promise.resolve(false);
|
|
13345
13825
|
return engine.applyStoredSynonyms(indexName, async () => {
|
|
@@ -13352,6 +13832,10 @@ function applyStoredSynonyms(engine, coordinator, indexName, indexUuid) {
|
|
|
13352
13832
|
function preparedSet(snapshot, indexName, synonyms) {
|
|
13353
13833
|
const config = indexConfigFromSettings({}, snapshot.settings, indexName);
|
|
13354
13834
|
const language = getLanguage(config.language ?? "english");
|
|
13835
|
+
const pending = snapshot.settings?.pendingAnalysis;
|
|
13836
|
+
if (pending !== void 0) {
|
|
13837
|
+
prepareConfiguredSynonyms({ ...configOfNamedSettings(config, pending), synonyms }, getLanguage(pending.language));
|
|
13838
|
+
}
|
|
13355
13839
|
return prepareConfiguredSynonyms({ ...config, synonyms }, language)?.encoded ?? null;
|
|
13356
13840
|
}
|
|
13357
13841
|
async function swapStoredSet(coordinator, indexName, first, ref) {
|
|
@@ -13407,14 +13891,16 @@ async function askNodeToApply(deps, nodeId, indexName, indexUuid) {
|
|
|
13407
13891
|
async function replaceClusterSynonyms(deps, indexName, synonyms) {
|
|
13408
13892
|
const { coordinator } = deps;
|
|
13409
13893
|
validateIndexName(indexName);
|
|
13410
|
-
const
|
|
13894
|
+
const servingName = await servingNameOf(coordinator, indexName);
|
|
13895
|
+
await requireSettledPartitionCount(coordinator, indexName, servingName);
|
|
13896
|
+
const first = await readIndexMetadataSnapshot(coordinator, servingName);
|
|
13411
13897
|
if (first === null) throw indexNotFound(indexName);
|
|
13412
|
-
const
|
|
13898
|
+
const encoded2 = preparedSet(first, servingName, synonyms);
|
|
13413
13899
|
const { indexUuid } = first.record;
|
|
13414
|
-
const ref =
|
|
13900
|
+
const ref = encoded2 === null ? null : await storeSynonymSet(coordinator, indexUuid, encoded2);
|
|
13415
13901
|
let replaced;
|
|
13416
13902
|
try {
|
|
13417
|
-
replaced = await swapStoredSet(coordinator,
|
|
13903
|
+
replaced = await swapStoredSet(coordinator, servingName, first, ref);
|
|
13418
13904
|
} catch (err) {
|
|
13419
13905
|
if (ref !== null) await deleteSynonymSet(coordinator, indexUuid, ref).catch(() => void 0);
|
|
13420
13906
|
throw err;
|
|
@@ -13422,13 +13908,14 @@ async function replaceClusterSynonyms(deps, indexName, synonyms) {
|
|
|
13422
13908
|
if (replaced !== null && replaced.generation !== ref?.generation) {
|
|
13423
13909
|
await deleteSynonymSet(coordinator, indexUuid, replaced);
|
|
13424
13910
|
}
|
|
13425
|
-
const allocation = await coordinator.getAllocation(
|
|
13911
|
+
const allocation = await coordinator.getAllocation(servingName);
|
|
13426
13912
|
const holders = /* @__PURE__ */ new Set();
|
|
13427
13913
|
for (const assignment of allocation?.assignments.values() ?? []) {
|
|
13428
13914
|
for (const nodeId of assignment.replicas) holders.add(nodeId);
|
|
13429
13915
|
if (assignment.primary !== null) holders.add(assignment.primary);
|
|
13430
13916
|
}
|
|
13431
|
-
await Promise.all([...holders].map((nodeId) => askNodeToApply(deps, nodeId,
|
|
13917
|
+
await Promise.all([...holders].map((nodeId) => askNodeToApply(deps, nodeId, servingName, indexUuid)));
|
|
13918
|
+
await requireSettledPartitionCount(coordinator, indexName, servingName);
|
|
13432
13919
|
}
|
|
13433
13920
|
|
|
13434
13921
|
// src/distribution/cluster-node/local-index-reconcile.ts
|
|
@@ -13551,13 +14038,40 @@ async function handlePartitionStores(message, respond, deps) {
|
|
|
13551
14038
|
|
|
13552
14039
|
// src/distribution/cluster-node/rebuild-analysis.ts
|
|
13553
14040
|
function validateRebuildPayload(decoded) {
|
|
13554
|
-
if (!isRecord(decoded) || typeof decoded.indexName !== "string" || decoded.indexName.length === 0 || typeof decoded.indexUuid !== "string" || decoded.indexUuid.length === 0) {
|
|
14041
|
+
if (!isRecord(decoded) || typeof decoded.indexName !== "string" || decoded.indexName.length === 0 || typeof decoded.indexUuid !== "string" || decoded.indexUuid.length === 0 || decoded.generation !== void 0 && (!Number.isSafeInteger(decoded.generation) || Number(decoded.generation) < 1)) {
|
|
13555
14042
|
throw new NarsilError(
|
|
13556
14043
|
ErrorCodes.CONFIG_INVALID,
|
|
13557
|
-
"Invalid RebuildAnalysisPayload: it needs an indexName and
|
|
14044
|
+
"Invalid RebuildAnalysisPayload: it needs an indexName, an indexUuid, and a generation of 1 or more where it carries one"
|
|
14045
|
+
);
|
|
14046
|
+
}
|
|
14047
|
+
return {
|
|
14048
|
+
indexName: decoded.indexName,
|
|
14049
|
+
indexUuid: decoded.indexUuid,
|
|
14050
|
+
...typeof decoded.generation === "number" ? { generation: decoded.generation } : {}
|
|
14051
|
+
};
|
|
14052
|
+
}
|
|
14053
|
+
async function buildHeldCopies(engine, coordinator, payload, generation) {
|
|
14054
|
+
const { indexName, indexUuid } = payload;
|
|
14055
|
+
const answer = { indexName, rebuiltPartitions: [], generation };
|
|
14056
|
+
const localGeneration = engine.analysisGenerationOf(indexName);
|
|
14057
|
+
const localUuid = engine.indexUuidOf(indexName);
|
|
14058
|
+
if (localGeneration === void 0 || typeof localUuid === "string" && localUuid !== indexUuid) return answer;
|
|
14059
|
+
if (localGeneration >= generation) return answer;
|
|
14060
|
+
const snapshot = await readIndexMetadataSnapshot(coordinator, indexName);
|
|
14061
|
+
const settings = snapshot?.settings;
|
|
14062
|
+
if (snapshot === null || snapshot.record.indexUuid !== indexUuid) return answer;
|
|
14063
|
+
if ((settings?.analysisGeneration ?? 0) >= generation) return answer;
|
|
14064
|
+
const pending = settings?.pendingAnalysis;
|
|
14065
|
+
if (pending === void 0 || pending.generation !== generation) {
|
|
14066
|
+
throw new NarsilError(
|
|
14067
|
+
ErrorCodes.ANALYSIS_CHANGE_REPLACED,
|
|
14068
|
+
`A later change replaced generation ${generation} of the analysis of index '${indexName}'`,
|
|
14069
|
+
{ indexName }
|
|
13558
14070
|
);
|
|
13559
14071
|
}
|
|
13560
|
-
|
|
14072
|
+
const { changeId: _changeId, ...named } = pending;
|
|
14073
|
+
await engine.buildHeldAnalysis(indexName, named);
|
|
14074
|
+
return { ...answer, rebuiltPartitions: heldPartitionIdsOf(engine, indexName) };
|
|
13561
14075
|
}
|
|
13562
14076
|
function validateRebuildResult(decoded, indexName) {
|
|
13563
14077
|
if (!isRecord(decoded) || decoded.indexName !== indexName || !Array.isArray(decoded.rebuiltPartitions)) {
|
|
@@ -13577,14 +14091,15 @@ function validateRebuildResult(decoded, indexName) {
|
|
|
13577
14091
|
async function rebuildLocalCopies(engine, payload) {
|
|
13578
14092
|
const { indexName, indexUuid } = payload;
|
|
13579
14093
|
const local = engine.listIndexes().find((index) => index.name === indexName);
|
|
13580
|
-
if (local === void 0 || engine.indexUuidOf(indexName) !== indexUuid || local.analysisStale !== true) {
|
|
14094
|
+
if (local === void 0 || engine.indexUuidOf(indexName) !== indexUuid || local.analysisStale !== true || engine.pendingAnalysisGenerationOf(indexName) !== null) {
|
|
13581
14095
|
return { indexName, rebuiltPartitions: [] };
|
|
13582
14096
|
}
|
|
13583
14097
|
await engine.rebuildAnalysis(indexName);
|
|
13584
14098
|
return { indexName, rebuiltPartitions: heldPartitionIdsOf(engine, indexName) };
|
|
13585
14099
|
}
|
|
13586
14100
|
async function handleRebuildAnalysis(message, respond, deps) {
|
|
13587
|
-
const
|
|
14101
|
+
const payload = validateRebuildPayload(decode(message.payload));
|
|
14102
|
+
const result = payload.generation === void 0 ? await rebuildLocalCopies(deps.engine, payload) : await buildHeldCopies(deps.engine, deps.coordinator, payload, payload.generation);
|
|
13588
14103
|
await respond({
|
|
13589
14104
|
type: ClusterMessageTypes.REBUILD_ANALYSIS,
|
|
13590
14105
|
sourceId: deps.nodeId,
|
|
@@ -14595,14 +15110,18 @@ async function handleSearch(message, respond, deps) {
|
|
|
14595
15110
|
if (snapshots !== void 0 || rescoreWindow !== void 0 && queryParams.sort === void 0 && queryParams.group === void 0) {
|
|
14596
15111
|
queryParams.document = false;
|
|
14597
15112
|
}
|
|
14598
|
-
const queryResult = await deps.
|
|
15113
|
+
const queryResult = await deps.analysisView.answerAt(
|
|
14599
15114
|
payload.indexName,
|
|
14600
|
-
|
|
14601
|
-
|
|
14602
|
-
|
|
14603
|
-
|
|
14604
|
-
|
|
14605
|
-
|
|
15115
|
+
payload.params.analysisGeneration ?? 0,
|
|
15116
|
+
() => deps.engine.queryPartitions(
|
|
15117
|
+
payload.indexName,
|
|
15118
|
+
queryParams,
|
|
15119
|
+
payload.partitionIds,
|
|
15120
|
+
payload.globalStats ?? void 0,
|
|
15121
|
+
rescoreWindow,
|
|
15122
|
+
snapshots,
|
|
15123
|
+
payload.params.indexBm25 ?? void 0
|
|
15124
|
+
)
|
|
14606
15125
|
);
|
|
14607
15126
|
const rankedDocument = (hit) => snapshots?.get(hit.id)?.fields ?? hit.document;
|
|
14608
15127
|
const normalizedSort = queryParams.sort !== void 0 ? normalizeSort(queryParams.sort) : null;
|
|
@@ -14721,11 +15240,13 @@ function openSearchContext(deps, coordinatorNodeId, keepAliveMs, snapshots) {
|
|
|
14721
15240
|
async function handleStats(message, respond, deps) {
|
|
14722
15241
|
const decoded = decode(message.payload);
|
|
14723
15242
|
const payload = validateStatsPayload(decoded);
|
|
14724
|
-
const resultPayload = await deps.
|
|
15243
|
+
const resultPayload = await deps.analysisView.answerAt(
|
|
14725
15244
|
payload.indexName,
|
|
14726
|
-
payload.
|
|
14727
|
-
payload.partitionIds,
|
|
14728
|
-
|
|
15245
|
+
payload.analysisGeneration ?? 0,
|
|
15246
|
+
() => deps.engine.collectQueryStats(payload.indexName, payload.terms, payload.partitionIds, {
|
|
15247
|
+
expansion: payload.expansion ?? void 0,
|
|
15248
|
+
tokens: payload.tokens ?? void 0
|
|
15249
|
+
})
|
|
14729
15250
|
);
|
|
14730
15251
|
await respond({
|
|
14731
15252
|
type: QueryMessageTypes.STATS_RESULT,
|
|
@@ -14800,10 +15321,14 @@ async function handleList(message, respond, deps) {
|
|
|
14800
15321
|
}
|
|
14801
15322
|
async function handleSuggest(message, respond, deps) {
|
|
14802
15323
|
const payload = validateSuggestPayload(decode(message.payload));
|
|
14803
|
-
const result = await deps.
|
|
15324
|
+
const result = await deps.analysisView.answerAt(
|
|
14804
15325
|
payload.indexName,
|
|
14805
|
-
|
|
14806
|
-
|
|
15326
|
+
payload.analysisGeneration ?? 0,
|
|
15327
|
+
() => deps.engine.suggestPartitions(
|
|
15328
|
+
payload.indexName,
|
|
15329
|
+
{ prefix: payload.prefix, limit: payload.limit },
|
|
15330
|
+
payload.partitionIds
|
|
15331
|
+
)
|
|
14807
15332
|
);
|
|
14808
15333
|
await respond(
|
|
14809
15334
|
createSuggestResultMessage(
|
|
@@ -14816,11 +15341,15 @@ async function handleSuggest(message, respond, deps) {
|
|
|
14816
15341
|
async function handlePreflight(message, respond, deps) {
|
|
14817
15342
|
const payload = validatePreflightPayload(decode(message.payload));
|
|
14818
15343
|
const params = wireParamsToLocal(payload.params);
|
|
14819
|
-
const result = await deps.
|
|
15344
|
+
const result = await deps.analysisView.answerAt(
|
|
14820
15345
|
payload.indexName,
|
|
14821
|
-
params,
|
|
14822
|
-
|
|
14823
|
-
|
|
15346
|
+
payload.params.analysisGeneration ?? 0,
|
|
15347
|
+
() => deps.engine.preflightPartitions(
|
|
15348
|
+
payload.indexName,
|
|
15349
|
+
params,
|
|
15350
|
+
payload.partitionIds,
|
|
15351
|
+
payload.params.indexBm25 ?? void 0
|
|
15352
|
+
)
|
|
14824
15353
|
);
|
|
14825
15354
|
await respond(
|
|
14826
15355
|
createPreflightResultMessage(
|
|
@@ -15320,9 +15849,7 @@ async function handleSyncRequestMessage(message, respond, deps) {
|
|
|
15320
15849
|
},
|
|
15321
15850
|
afterSnapshot: async (trailingSink, build) => {
|
|
15322
15851
|
const snapshotEnd = build.lastSeqNo ?? snapshotSeqNo;
|
|
15323
|
-
|
|
15324
|
-
await sendSyncEntriesResponse(message, trailingSink, deps, entries);
|
|
15325
|
-
notePosition(entries.at(-1)?.seqNo ?? snapshotEnd);
|
|
15852
|
+
await sendSyncEntriesResponse(message, trailingSink, deps, log.getEntriesFrom(snapshotEnd + 1));
|
|
15326
15853
|
}
|
|
15327
15854
|
}
|
|
15328
15855
|
);
|
|
@@ -15753,24 +16280,130 @@ async function resolveNodeId(explicitNodeId, directory) {
|
|
|
15753
16280
|
}
|
|
15754
16281
|
}
|
|
15755
16282
|
|
|
15756
|
-
// src/distribution/cluster-node/
|
|
15757
|
-
var SETTINGS_CONTENTION_REASON2 = "settings_contention";
|
|
16283
|
+
// src/distribution/cluster-node/analysis-change/request.ts
|
|
15758
16284
|
function indexNotFound2(indexName) {
|
|
15759
16285
|
return new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index '${indexName}' does not exist in the cluster`, {
|
|
15760
16286
|
indexName
|
|
15761
16287
|
});
|
|
15762
16288
|
}
|
|
16289
|
+
function sameList(left, right) {
|
|
16290
|
+
if (left === void 0 || right === void 0) return left === right;
|
|
16291
|
+
return left.length === right.length && left.every((word, position) => word === right[position]);
|
|
16292
|
+
}
|
|
16293
|
+
function asksForPending(pending, target) {
|
|
16294
|
+
if (pending === void 0) return false;
|
|
16295
|
+
const named = namedSettingsOf(target.config, target.language.name);
|
|
16296
|
+
return pending.language === named.language && pending.tokenizer === named.tokenizer && pending.stopWords === named.stopWords && sameList(pending.stopWordList, named.stopWordList);
|
|
16297
|
+
}
|
|
16298
|
+
async function requireTargetAcceptable(coordinator, indexName, target) {
|
|
16299
|
+
const state = await readIndexState(coordinator, indexName);
|
|
16300
|
+
if (state === null || state.synonyms.length === 0) return;
|
|
16301
|
+
prepareConfiguredSynonyms({ ...target.config, synonyms: state.synonyms }, target.language);
|
|
16302
|
+
}
|
|
16303
|
+
async function writePending(coordinator, indexName, target) {
|
|
16304
|
+
const schema = await coordinator.getSchema(indexName);
|
|
16305
|
+
if (schema === null) throw indexNotFound2(indexName);
|
|
16306
|
+
for (let attempt = 1; attempt <= MAX_SETTINGS_REPLACEMENT_ATTEMPTS; attempt++) {
|
|
16307
|
+
const snapshot = await readIndexMetadataSnapshot(coordinator, indexName);
|
|
16308
|
+
if (snapshot === null) throw indexNotFound2(indexName);
|
|
16309
|
+
const running = snapshot.settings?.pendingAnalysis;
|
|
16310
|
+
if (running !== void 0 && asksForPending(running, target)) return running;
|
|
16311
|
+
const current = indexConfigFromSettings(schema, snapshot.settings, indexName);
|
|
16312
|
+
if (running === void 0 && holdsSettings(current, target)) return null;
|
|
16313
|
+
const pending = {
|
|
16314
|
+
changeId: generateId(),
|
|
16315
|
+
generation: Math.max(snapshot.settings?.analysisGeneration ?? 0, running?.generation ?? 0) + 1,
|
|
16316
|
+
...namedSettingsOf(target.config, target.language.name)
|
|
16317
|
+
};
|
|
16318
|
+
const next = metadataWithPendingAnalysis(snapshot, pending);
|
|
16319
|
+
if (await coordinator.compareAndSet(indexConfigKey(indexName), snapshot.bytes, next)) return pending;
|
|
16320
|
+
}
|
|
16321
|
+
throw new NarsilError(
|
|
16322
|
+
ErrorCodes.CLUSTER_WRITE_BLOCKED,
|
|
16323
|
+
`Index '${indexName}' kept changing while its analysis change was recorded ${MAX_SETTINGS_REPLACEMENT_ATTEMPTS} times over, so the change was not stored. Retry the request`,
|
|
16324
|
+
{ reason: SETTINGS_CONTENTION_REASON, indexName, attempts: MAX_SETTINGS_REPLACEMENT_ATTEMPTS }
|
|
16325
|
+
);
|
|
16326
|
+
}
|
|
16327
|
+
async function clearOwnPending(coordinator, indexName, changeId) {
|
|
16328
|
+
for (let attempt = 1; attempt <= MAX_SETTINGS_REPLACEMENT_ATTEMPTS; attempt++) {
|
|
16329
|
+
const snapshot = await readIndexMetadataSnapshot(coordinator, indexName);
|
|
16330
|
+
if (snapshot?.settings?.pendingAnalysis?.changeId !== changeId) return;
|
|
16331
|
+
const next = metadataWithoutPendingAnalysis(snapshot);
|
|
16332
|
+
if (await coordinator.compareAndSet(indexConfigKey(indexName), snapshot.bytes, next)) return;
|
|
16333
|
+
}
|
|
16334
|
+
}
|
|
16335
|
+
function outcomeOf(snapshot, pending, indexName) {
|
|
16336
|
+
const settings = snapshot.settings;
|
|
16337
|
+
if (settings?.analysisGeneration === pending.generation) return "published";
|
|
16338
|
+
const failure = settings?.analysisFailure;
|
|
16339
|
+
if (failure?.changeId === pending.changeId) {
|
|
16340
|
+
throw new NarsilError(failure.code, failure.message, { indexName, nodeId: failure.nodeId });
|
|
16341
|
+
}
|
|
16342
|
+
if (settings?.pendingAnalysis?.changeId === pending.changeId) return "waiting";
|
|
16343
|
+
throw new NarsilError(
|
|
16344
|
+
ErrorCodes.ANALYSIS_CHANGE_REPLACED,
|
|
16345
|
+
`A later change replaced this analysis change of index '${indexName}'`,
|
|
16346
|
+
{ indexName }
|
|
16347
|
+
);
|
|
16348
|
+
}
|
|
16349
|
+
async function changeClusterAnalysis(request) {
|
|
16350
|
+
const { coordinator, view, guard, servingName, publicName, change } = request;
|
|
16351
|
+
validateIndexName(servingName);
|
|
16352
|
+
const schema = await coordinator.getSchema(servingName);
|
|
16353
|
+
const metadata = await readIndexMetadataSnapshot(coordinator, servingName);
|
|
16354
|
+
if (schema === null || metadata === null) throw indexNotFound2(publicName);
|
|
16355
|
+
const target = resolveAnalysisTarget(
|
|
16356
|
+
publicName,
|
|
16357
|
+
indexConfigFromSettings(schema, metadata.settings, servingName),
|
|
16358
|
+
change
|
|
16359
|
+
);
|
|
16360
|
+
await requireTargetAcceptable(coordinator, servingName, target);
|
|
16361
|
+
const olderNodes = registeredNodesBelowBm25Tuning(await coordinator.listNodes());
|
|
16362
|
+
if (olderNodes.length > 0) {
|
|
16363
|
+
throw new NarsilError(
|
|
16364
|
+
ErrorCodes.ANALYSIS_CHANGE_NOT_READY,
|
|
16365
|
+
`The cluster has nodes below the specification version that changes analysis, so index '${publicName}' keeps its settings until they are upgraded`,
|
|
16366
|
+
{ indexName: publicName, nodeIds: olderNodes }
|
|
16367
|
+
);
|
|
16368
|
+
}
|
|
16369
|
+
const pending = await writePending(coordinator, servingName, target);
|
|
16370
|
+
if (pending === null) return;
|
|
16371
|
+
if (await readRebalanceRecord(coordinator, publicName) !== null) {
|
|
16372
|
+
await clearOwnPending(coordinator, servingName, pending.changeId);
|
|
16373
|
+
throw new NarsilError(
|
|
16374
|
+
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
16375
|
+
`Index '${publicName}' is changing its partition count, so try its analysis change again once that ends`,
|
|
16376
|
+
{ indexName: publicName }
|
|
16377
|
+
);
|
|
16378
|
+
}
|
|
16379
|
+
for (; ; ) {
|
|
16380
|
+
guard();
|
|
16381
|
+
const snapshot = await readIndexMetadataSnapshot(coordinator, servingName);
|
|
16382
|
+
if (snapshot === null) throw indexNotFound2(publicName);
|
|
16383
|
+
if (outcomeOf(snapshot, pending, publicName) === "published") break;
|
|
16384
|
+
await pause(ANALYSIS_CHANGE_POLL_INTERVAL_MS);
|
|
16385
|
+
}
|
|
16386
|
+
await view.refresh(servingName);
|
|
16387
|
+
await view.settle(servingName);
|
|
16388
|
+
}
|
|
16389
|
+
|
|
16390
|
+
// src/distribution/cluster-node/bm25-change.ts
|
|
16391
|
+
function indexNotFound3(indexName) {
|
|
16392
|
+
return new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index '${indexName}' does not exist in the cluster`, {
|
|
16393
|
+
indexName
|
|
16394
|
+
});
|
|
16395
|
+
}
|
|
15763
16396
|
async function replaceClusterBm25(coordinator, view, indexName, value) {
|
|
15764
16397
|
validateIndexName(indexName);
|
|
15765
16398
|
const schema = await coordinator.getSchema(indexName);
|
|
15766
|
-
if (schema === null) throw
|
|
16399
|
+
if (schema === null) throw indexNotFound3(indexName);
|
|
15767
16400
|
validateBM25Params(value, schema, ErrorCodes.CONFIG_INVALID);
|
|
15768
16401
|
const bm25 = storedBM25Params(value);
|
|
15769
16402
|
const olderNodes = registeredNodesBelowBm25Tuning(await coordinator.listNodes());
|
|
15770
16403
|
if (olderNodes.length > 0) throw bm25NotReady("The cluster has", olderNodes, indexName);
|
|
15771
16404
|
for (let attempt = 1; attempt <= MAX_SETTINGS_REPLACEMENT_ATTEMPTS; attempt++) {
|
|
15772
16405
|
const current = await readIndexMetadataSnapshot(coordinator, indexName);
|
|
15773
|
-
if (current === null) throw
|
|
16406
|
+
if (current === null) throw indexNotFound3(indexName);
|
|
15774
16407
|
const revision = (current.settings?.bm25Revision ?? 0) + 1;
|
|
15775
16408
|
const next = metadataWithBm25(current, bm25, revision);
|
|
15776
16409
|
if (await coordinator.compareAndSet(indexConfigKey(indexName), current.bytes, next)) {
|
|
@@ -15781,30 +16414,10 @@ async function replaceClusterBm25(coordinator, view, indexName, value) {
|
|
|
15781
16414
|
throw new NarsilError(
|
|
15782
16415
|
ErrorCodes.CLUSTER_WRITE_BLOCKED,
|
|
15783
16416
|
`Index '${indexName}' kept changing while its BM25 setting was replaced ${MAX_SETTINGS_REPLACEMENT_ATTEMPTS} times over, so the new setting was not stored. Retry the request`,
|
|
15784
|
-
{ reason:
|
|
16417
|
+
{ reason: SETTINGS_CONTENTION_REASON, indexName, attempts: MAX_SETTINGS_REPLACEMENT_ATTEMPTS }
|
|
15785
16418
|
);
|
|
15786
16419
|
}
|
|
15787
16420
|
|
|
15788
|
-
// src/distribution/query/node-send.ts
|
|
15789
|
-
async function resolveTargets(nodeId, deps) {
|
|
15790
|
-
if (deps.resolveNodeTargets === void 0) {
|
|
15791
|
-
return [nodeId];
|
|
15792
|
-
}
|
|
15793
|
-
const targets = await deps.resolveNodeTargets(nodeId);
|
|
15794
|
-
return targets.length > 0 ? targets : [nodeId];
|
|
15795
|
-
}
|
|
15796
|
-
async function sendToFirstReachableTarget2(transport, targets, message) {
|
|
15797
|
-
let lastError;
|
|
15798
|
-
for (const target of targets) {
|
|
15799
|
-
try {
|
|
15800
|
-
return await transport.send(target, message);
|
|
15801
|
-
} catch (error) {
|
|
15802
|
-
lastError = error;
|
|
15803
|
-
}
|
|
15804
|
-
}
|
|
15805
|
-
throw lastError instanceof Error ? lastError : new Error(String(lastError));
|
|
15806
|
-
}
|
|
15807
|
-
|
|
15808
16421
|
// src/distribution/query/phrase-routing.ts
|
|
15809
16422
|
function queryReadsPhrases(term) {
|
|
15810
16423
|
return typeof term === "string" && holdsQuotedText(term);
|
|
@@ -15885,11 +16498,13 @@ function releaseSearchContexts(indexName, contexts, deps) {
|
|
|
15885
16498
|
function queryRejectionOf(response, decoded) {
|
|
15886
16499
|
const body = decoded;
|
|
15887
16500
|
if (!response.type.endsWith(".error") && body?.error !== true) return null;
|
|
16501
|
+
const details = typeof body?.details === "object" && body.details !== null ? { ...body.details } : {};
|
|
16502
|
+
const generationRefusal = generationRefusalOf(body?.code, body?.message, details);
|
|
16503
|
+
if (generationRefusal !== null) return generationRefusal;
|
|
15888
16504
|
if (body?.code !== ServerErrorCodes.TOO_MANY_REQUESTS) return nodeRejectionOf(body?.code, body?.message);
|
|
15889
|
-
const details = typeof body.details === "object" && body.details !== null ? { ...body.details } : {};
|
|
15890
16505
|
return new NarsilError(
|
|
15891
16506
|
ServerErrorCodes.TOO_MANY_REQUESTS,
|
|
15892
|
-
typeof body
|
|
16507
|
+
typeof body?.message === "string" ? body.message : "A node had no room in memory to answer its part of the search",
|
|
15893
16508
|
details
|
|
15894
16509
|
);
|
|
15895
16510
|
}
|
|
@@ -15917,7 +16532,7 @@ async function requestStats(nodeId, payload, deps, config) {
|
|
|
15917
16532
|
validateStatsResultPayload
|
|
15918
16533
|
);
|
|
15919
16534
|
}
|
|
15920
|
-
async function countLeftOutTokens(indexName, answered, deps, config) {
|
|
16535
|
+
async function countLeftOutTokens(indexName, answered, analysisGeneration, deps, config) {
|
|
15921
16536
|
const reached = /* @__PURE__ */ new Set();
|
|
15922
16537
|
for (const [, { result }] of answered) {
|
|
15923
16538
|
for (const token of result.expanded ?? []) reached.add(token);
|
|
@@ -15927,7 +16542,12 @@ async function countLeftOutTokens(indexName, answered, deps, config) {
|
|
|
15927
16542
|
answered.map(async ([nodeId, { partitionIds, result }]) => {
|
|
15928
16543
|
const tokens = [...reached].filter((token) => !Object.hasOwn(result.docFrequencies, token));
|
|
15929
16544
|
if (tokens.length === 0) return null;
|
|
15930
|
-
const counted = await requestStats(
|
|
16545
|
+
const counted = await requestStats(
|
|
16546
|
+
nodeId,
|
|
16547
|
+
{ indexName, partitionIds, terms: [], tokens, analysisGeneration },
|
|
16548
|
+
deps,
|
|
16549
|
+
config
|
|
16550
|
+
);
|
|
15931
16551
|
if (counted === null) return null;
|
|
15932
16552
|
const docFrequencies = /* @__PURE__ */ Object.create(null);
|
|
15933
16553
|
for (const token of tokens) {
|
|
@@ -15946,10 +16566,15 @@ async function collectDistributedStats(indexName, params, routing, deps, config)
|
|
|
15946
16566
|
if (routing.nodeToPartitions.size === 0 && queryReadsPhrases(params.term)) return mergePartitionStats([]);
|
|
15947
16567
|
const terms = params.term !== null ? params.term.split(/\s+/).filter((t) => t.length > 0) : [];
|
|
15948
16568
|
const expansion = statsExpansionOf(params);
|
|
16569
|
+
const { analysisGeneration } = params;
|
|
15949
16570
|
const outcomes = await sendWithFailover(
|
|
15950
16571
|
routing,
|
|
15951
|
-
(nodeId, partitionIds) => requestStats(nodeId, { indexName, partitionIds, terms, expansion }, deps, config)
|
|
16572
|
+
(nodeId, partitionIds) => requestStats(nodeId, { indexName, partitionIds, terms, expansion, analysisGeneration }, deps, config)
|
|
15952
16573
|
);
|
|
16574
|
+
const refused = outcomes.find(
|
|
16575
|
+
(outcome) => isNodeRejection(outcome.error) && outcome.error instanceof NarsilError && outcome.error.code === ErrorCodes.ANALYSIS_GENERATION_MISSING
|
|
16576
|
+
);
|
|
16577
|
+
if (refused !== void 0) throw refused.error;
|
|
15953
16578
|
const answered = [];
|
|
15954
16579
|
for (const outcome of outcomes) {
|
|
15955
16580
|
if (outcome.status === "success" && outcome.result !== null) {
|
|
@@ -15961,7 +16586,7 @@ async function collectDistributedStats(indexName, params, routing, deps, config)
|
|
|
15961
16586
|
nodeCount: routing.nodeToPartitions.size
|
|
15962
16587
|
});
|
|
15963
16588
|
}
|
|
15964
|
-
const leftOut = await countLeftOutTokens(indexName, answered, deps, config);
|
|
16589
|
+
const leftOut = await countLeftOutTokens(indexName, answered, analysisGeneration, deps, config);
|
|
15965
16590
|
return mergePartitionStats([...answered.map(([, { result }]) => result), ...leftOut]);
|
|
15966
16591
|
}
|
|
15967
16592
|
async function fanOutSearch(indexName, params, globalStats, facetShardSize, routing, deps, config, keepAliveMs = null) {
|
|
@@ -16740,6 +17365,17 @@ function failOutcomesMissingSortValues(outcomes) {
|
|
|
16740
17365
|
}
|
|
16741
17366
|
}
|
|
16742
17367
|
|
|
17368
|
+
// src/distribution/cluster-node/analysis-change/rerun.ts
|
|
17369
|
+
async function rerunOnNewerGeneration(view, indexName, run) {
|
|
17370
|
+
try {
|
|
17371
|
+
return await run();
|
|
17372
|
+
} catch (error) {
|
|
17373
|
+
if (!(error instanceof NarsilError) || error.code !== ErrorCodes.ANALYSIS_GENERATION_MISSING) throw error;
|
|
17374
|
+
await view.refresh(indexName);
|
|
17375
|
+
return run();
|
|
17376
|
+
}
|
|
17377
|
+
}
|
|
17378
|
+
|
|
16743
17379
|
// src/distribution/cluster-node/reads/bm25.ts
|
|
16744
17380
|
function routingUnder(params, setting) {
|
|
16745
17381
|
const wireParams = localParamsToWire(params);
|
|
@@ -17290,16 +17926,23 @@ async function suggestCluster(deps, indexName, params) {
|
|
|
17290
17926
|
return { terms: [], elapsed: performance.now() - startTime };
|
|
17291
17927
|
}
|
|
17292
17928
|
const nodeLimit = suggestNodeLimit(clientLimit);
|
|
17293
|
-
const gathered = await
|
|
17294
|
-
|
|
17295
|
-
|
|
17296
|
-
|
|
17297
|
-
|
|
17298
|
-
|
|
17299
|
-
|
|
17300
|
-
|
|
17301
|
-
|
|
17302
|
-
|
|
17929
|
+
const gathered = await rerunOnNewerGeneration(deps.analysisView, indexName, async () => {
|
|
17930
|
+
const analysisGeneration = await deps.analysisView.generationFor(indexName);
|
|
17931
|
+
return gatherFromEveryPartition(allocation, indexName, async (group) => {
|
|
17932
|
+
if (group.nodeId === deps.nodeId) {
|
|
17933
|
+
const local = await deps.analysisView.answerAt(
|
|
17934
|
+
indexName,
|
|
17935
|
+
analysisGeneration,
|
|
17936
|
+
() => deps.engine.suggestPartitions(indexName, { prefix, limit: nodeLimit }, group.partitionIds)
|
|
17937
|
+
).catch(rethrowAsRefusal);
|
|
17938
|
+
return { terms: local.terms, analysisStale: local.analysisStale === true };
|
|
17939
|
+
}
|
|
17940
|
+
const message = createSuggestMessage(
|
|
17941
|
+
{ indexName, partitionIds: group.partitionIds, prefix, limit: nodeLimit, analysisGeneration },
|
|
17942
|
+
deps.nodeId
|
|
17943
|
+
);
|
|
17944
|
+
return sendReadRequest(deps, group.nodeId, message, indexName, validateSuggestResultPayload);
|
|
17945
|
+
});
|
|
17303
17946
|
});
|
|
17304
17947
|
const mergedFrequencies = /* @__PURE__ */ new Map();
|
|
17305
17948
|
let analysisStale = false;
|
|
@@ -17325,20 +17968,27 @@ async function preflightCluster(deps, indexName, params, settingName) {
|
|
|
17325
17968
|
return deps.engine.preflight(indexName, params);
|
|
17326
17969
|
}
|
|
17327
17970
|
const startTime = performance.now();
|
|
17328
|
-
const { wireParams, indexBm25, carriesBm25 } = await bm25RoutingFor(deps, settingName, params);
|
|
17329
17971
|
const coordinator = deps.config.coordinator;
|
|
17330
|
-
const
|
|
17331
|
-
|
|
17332
|
-
|
|
17333
|
-
|
|
17334
|
-
|
|
17335
|
-
|
|
17336
|
-
|
|
17337
|
-
|
|
17338
|
-
|
|
17339
|
-
|
|
17340
|
-
|
|
17341
|
-
|
|
17972
|
+
const gathered = await rerunOnNewerGeneration(deps.analysisView, indexName, async () => {
|
|
17973
|
+
const { wireParams, indexBm25, carriesBm25 } = await bm25RoutingFor(deps, settingName, params);
|
|
17974
|
+
wireParams.analysisGeneration = await deps.analysisView.generationFor(indexName);
|
|
17975
|
+
const bm25Allocation = carriesBm25 ? await allocationForBm25Query(indexName, allocation, () => coordinator.listNodes()) : allocation;
|
|
17976
|
+
const routedAllocation = queryReadsPhrases(params.term) ? await allocationForPhraseQuery(indexName, bm25Allocation, () => coordinator.listNodes()) : bm25Allocation;
|
|
17977
|
+
return gatherFromEveryPartition(routedAllocation, indexName, async (group) => {
|
|
17978
|
+
if (group.nodeId === deps.nodeId) {
|
|
17979
|
+
const local = await deps.analysisView.answerAt(
|
|
17980
|
+
indexName,
|
|
17981
|
+
wireParams.analysisGeneration ?? 0,
|
|
17982
|
+
() => deps.engine.preflightPartitions(indexName, params, group.partitionIds, indexBm25)
|
|
17983
|
+
).catch(rethrowAsRefusal);
|
|
17984
|
+
return { count: local.count, analysisStale: local.analysisStale === true };
|
|
17985
|
+
}
|
|
17986
|
+
const message = createPreflightMessage(
|
|
17987
|
+
{ indexName, partitionIds: group.partitionIds, params: wireParams },
|
|
17988
|
+
deps.nodeId
|
|
17989
|
+
);
|
|
17990
|
+
return sendReadRequest(deps, group.nodeId, message, indexName, validatePreflightResultPayload);
|
|
17991
|
+
});
|
|
17342
17992
|
});
|
|
17343
17993
|
let count2 = 0;
|
|
17344
17994
|
let analysisStale = false;
|
|
@@ -17407,36 +18057,41 @@ async function queryCluster(deps, indexName, params, settingName) {
|
|
|
17407
18057
|
if (allocation === null) {
|
|
17408
18058
|
return deps.engine.query(indexName, params);
|
|
17409
18059
|
}
|
|
17410
|
-
return
|
|
17411
|
-
|
|
18060
|
+
return rerunOnNewerGeneration(
|
|
18061
|
+
deps.analysisView,
|
|
18062
|
+
indexName,
|
|
18063
|
+
() => rerunOnLostContext(async () => {
|
|
18064
|
+
if (params.rerank === void 0) {
|
|
18065
|
+
return withRankedReader(
|
|
18066
|
+
deps,
|
|
18067
|
+
indexName,
|
|
18068
|
+
allocation,
|
|
18069
|
+
(reader) => searchCluster(deps, indexName, settingName, params, allocation, reader, false)
|
|
18070
|
+
);
|
|
18071
|
+
}
|
|
18072
|
+
const { rerankClusterQuery } = await import('./rerank-KTLC3E6N.mjs');
|
|
17412
18073
|
return withRankedReader(
|
|
17413
18074
|
deps,
|
|
17414
18075
|
indexName,
|
|
17415
18076
|
allocation,
|
|
17416
|
-
(reader) =>
|
|
18077
|
+
(reader) => rerankClusterQuery(
|
|
18078
|
+
deps,
|
|
18079
|
+
indexName,
|
|
18080
|
+
params,
|
|
18081
|
+
(firstStage) => searchCluster(deps, indexName, settingName, firstStage, allocation, reader, true),
|
|
18082
|
+
(docIds, last) => reader.read(docIds, { last, fields: null, requireEveryPartition: true }),
|
|
18083
|
+
clusterRerankHost
|
|
18084
|
+
)
|
|
17417
18085
|
);
|
|
17418
|
-
}
|
|
17419
|
-
|
|
17420
|
-
return withRankedReader(
|
|
17421
|
-
deps,
|
|
17422
|
-
indexName,
|
|
17423
|
-
allocation,
|
|
17424
|
-
(reader) => rerankClusterQuery(
|
|
17425
|
-
deps,
|
|
17426
|
-
indexName,
|
|
17427
|
-
params,
|
|
17428
|
-
(firstStage) => searchCluster(deps, indexName, settingName, firstStage, allocation, reader, true),
|
|
17429
|
-
(docIds, last) => reader.read(docIds, { last, fields: null, requireEveryPartition: true }),
|
|
17430
|
-
clusterRerankHost
|
|
17431
|
-
)
|
|
17432
|
-
);
|
|
17433
|
-
});
|
|
18086
|
+
})
|
|
18087
|
+
);
|
|
17434
18088
|
}
|
|
17435
18089
|
async function searchCluster(deps, indexName, settingName, params, allocation, reader, readsLater) {
|
|
17436
18090
|
const projection = resolveProjection(params.document);
|
|
17437
18091
|
const groupsRead = params.group !== void 0 && (params.group.reduce !== void 0 || projection.kind !== "none");
|
|
17438
18092
|
const readsDocuments = readsLater || projection.kind !== "none" || groupsRead;
|
|
17439
18093
|
const { wireParams, carriesBm25 } = heldBm25RoutingFor(deps, settingName, params) ?? await bm25RoutingFor(deps, settingName, params);
|
|
18094
|
+
wireParams.analysisGeneration = deps.analysisView.held(indexName) ?? await deps.analysisView.generationFor(indexName);
|
|
17440
18095
|
const queryDeps = {
|
|
17441
18096
|
transport: deps.config.transport,
|
|
17442
18097
|
sourceNodeId: deps.nodeId,
|
|
@@ -17614,6 +18269,13 @@ function createClusterNodeOperations(deps) {
|
|
|
17614
18269
|
function served(indexName, fn) {
|
|
17615
18270
|
return trackOp(indexName, async () => fn(await deps.servingName(indexName)));
|
|
17616
18271
|
}
|
|
18272
|
+
const names = {
|
|
18273
|
+
servingName: deps.servingName,
|
|
18274
|
+
liveServingName: (indexName) => servingNameOf(coordinator, indexName)
|
|
18275
|
+
};
|
|
18276
|
+
function read(indexName, fn) {
|
|
18277
|
+
return trackOp(indexName, () => readFollowingTheName(names, indexName, fn));
|
|
18278
|
+
}
|
|
17617
18279
|
return {
|
|
17618
18280
|
async createIndex(name, indexConfig, options) {
|
|
17619
18281
|
refuseHiddenGeneration(name);
|
|
@@ -17672,16 +18334,16 @@ function createClusterNodeOperations(deps) {
|
|
|
17672
18334
|
return served(indexName, (name) => clearCluster(readDeps, writeDeps, name));
|
|
17673
18335
|
},
|
|
17674
18336
|
async countDocuments(indexName) {
|
|
17675
|
-
return
|
|
18337
|
+
return read(indexName, (name) => countCluster(readDeps, name));
|
|
17676
18338
|
},
|
|
17677
18339
|
async listDocuments(indexName, params) {
|
|
17678
|
-
return
|
|
18340
|
+
return read(indexName, (name) => listCluster(readDeps, name, params ?? {}));
|
|
17679
18341
|
},
|
|
17680
18342
|
async suggest(indexName, params) {
|
|
17681
|
-
return
|
|
18343
|
+
return read(indexName, (name) => suggestCluster(readDeps, name, params));
|
|
17682
18344
|
},
|
|
17683
18345
|
async preflight(indexName, params) {
|
|
17684
|
-
return
|
|
18346
|
+
return read(indexName, (name) => preflightCluster(readDeps, name, params, indexName));
|
|
17685
18347
|
},
|
|
17686
18348
|
async rebuildAnalysis(indexName) {
|
|
17687
18349
|
return served(
|
|
@@ -17689,6 +18351,20 @@ function createClusterNodeOperations(deps) {
|
|
|
17689
18351
|
(name) => rebuildClusterAnalysis({ nodeId: readDeps.nodeId, config: readDeps.config, engine }, name)
|
|
17690
18352
|
);
|
|
17691
18353
|
},
|
|
18354
|
+
async changeAnalysis(indexName, change) {
|
|
18355
|
+
refuseHiddenGeneration(indexName);
|
|
18356
|
+
return trackOp(
|
|
18357
|
+
indexName,
|
|
18358
|
+
async () => changeClusterAnalysis({
|
|
18359
|
+
coordinator,
|
|
18360
|
+
view: readDeps.analysisView,
|
|
18361
|
+
guard: deps.guardShutdown,
|
|
18362
|
+
servingName: await deps.servingName(indexName),
|
|
18363
|
+
publicName: indexName,
|
|
18364
|
+
change
|
|
18365
|
+
})
|
|
18366
|
+
);
|
|
18367
|
+
},
|
|
17692
18368
|
async rebalance(indexName, targetPartitionCount) {
|
|
17693
18369
|
const job = await trackOp(indexName, () => deps.requestRebalance(indexName, targetPartitionCount));
|
|
17694
18370
|
return trackOp(null, () => deps.waitForRebalance(indexName, job));
|
|
@@ -17697,10 +18373,10 @@ function createClusterNodeOperations(deps) {
|
|
|
17697
18373
|
return trackOp(null, () => listIndexesCluster(readDeps));
|
|
17698
18374
|
},
|
|
17699
18375
|
async getStats(indexName) {
|
|
17700
|
-
return
|
|
18376
|
+
return read(indexName, (name) => statsCluster(readDeps, name));
|
|
17701
18377
|
},
|
|
17702
18378
|
async getPartitionStats(indexName) {
|
|
17703
|
-
return
|
|
18379
|
+
return read(indexName, (name) => partitionStatsCluster(readDeps, name));
|
|
17704
18380
|
},
|
|
17705
18381
|
async checkpoint(indexName) {
|
|
17706
18382
|
return served(indexName, (name) => engine.checkpoint(name));
|
|
@@ -17715,16 +18391,24 @@ function createClusterNodeOperations(deps) {
|
|
|
17715
18391
|
engine.off(event, handler);
|
|
17716
18392
|
},
|
|
17717
18393
|
async query(indexName, params) {
|
|
17718
|
-
return
|
|
18394
|
+
return trackOp(
|
|
18395
|
+
indexName,
|
|
18396
|
+
() => readFollowingTheName(
|
|
18397
|
+
names,
|
|
18398
|
+
indexName,
|
|
18399
|
+
(name) => queryCluster(readDeps, name, params, indexName),
|
|
18400
|
+
(answer) => leftOutAPartition(answer.coverage)
|
|
18401
|
+
)
|
|
18402
|
+
);
|
|
17719
18403
|
},
|
|
17720
18404
|
async get(indexName, docId) {
|
|
17721
|
-
return
|
|
18405
|
+
return read(indexName, async (name) => (await readClusterDocuments(readDeps, name, [docId])).get(docId));
|
|
17722
18406
|
},
|
|
17723
18407
|
async getMultiple(indexName, docIds) {
|
|
17724
|
-
return
|
|
18408
|
+
return read(indexName, (name) => readClusterDocuments(readDeps, name, docIds));
|
|
17725
18409
|
},
|
|
17726
18410
|
async has(indexName, docId) {
|
|
17727
|
-
return
|
|
18411
|
+
return read(indexName, async (name) => (await readClusterDocuments(readDeps, name, [docId])).has(docId));
|
|
17728
18412
|
}
|
|
17729
18413
|
};
|
|
17730
18414
|
}
|
|
@@ -18104,9 +18788,9 @@ function createPrimaryNumbering(onNumbered) {
|
|
|
18104
18788
|
admit(indexName, partitionId, documentId, document) {
|
|
18105
18789
|
const scope = scopes.get(replicationLogKey(indexName, partitionId));
|
|
18106
18790
|
if (scope === void 0) return;
|
|
18107
|
-
const
|
|
18108
|
-
scope.log().assertFits({ indexName, documentId, document:
|
|
18109
|
-
scope.admitted.set(documentId,
|
|
18791
|
+
const encoded2 = document === null ? null : encodeDocument(document);
|
|
18792
|
+
scope.log().assertFits({ indexName, documentId, document: encoded2 });
|
|
18793
|
+
scope.admitted.set(documentId, encoded2);
|
|
18110
18794
|
},
|
|
18111
18795
|
numberEntry(record) {
|
|
18112
18796
|
const scope = scopes.get(replicationLogKey(record.indexName, record.partitionId));
|
|
@@ -18902,7 +19586,8 @@ async function createClusterNode(callerConfig) {
|
|
|
18902
19586
|
const trackOp = operationTracker.track;
|
|
18903
19587
|
const transitionIndex = operationTracker.transition;
|
|
18904
19588
|
const bm25View = createBm25View({ coordinator: config.coordinator, engine, onError: forwardOnError });
|
|
18905
|
-
const
|
|
19589
|
+
const analysisView = createAnalysisView({ coordinator: config.coordinator, engine, onError: forwardOnError });
|
|
19590
|
+
const readDeps = { config: routingConfig2, nodeId, engine, resolveNodeTargets: resolveNodeTargets3, searchContexts, bm25View, analysisView };
|
|
18906
19591
|
const servingName = (indexName) => servingNameOf(allocationView.routing, indexName);
|
|
18907
19592
|
const cluster = createClusterNamespace({
|
|
18908
19593
|
nodeId,
|
|
@@ -18938,6 +19623,7 @@ async function createClusterNode(callerConfig) {
|
|
|
18938
19623
|
servingName,
|
|
18939
19624
|
requestRebalance: (indexName, count2) => requestRebalance(config, nodeId, indexName, count2),
|
|
18940
19625
|
waitForRebalance: (indexName, job) => waitForSwap(config, indexName, job, guardShutdown),
|
|
19626
|
+
guardShutdown,
|
|
18941
19627
|
readDeps,
|
|
18942
19628
|
writeDeps,
|
|
18943
19629
|
engine,
|
|
@@ -18955,6 +19641,7 @@ async function createClusterNode(callerConfig) {
|
|
|
18955
19641
|
guardShutdown();
|
|
18956
19642
|
await allocationView.start();
|
|
18957
19643
|
await bm25View.start();
|
|
19644
|
+
await analysisView.start();
|
|
18958
19645
|
const dispositions = await reconcileLocalIndexes({
|
|
18959
19646
|
engine,
|
|
18960
19647
|
coordinator: config.coordinator,
|
|
@@ -18964,6 +19651,8 @@ async function createClusterNode(callerConfig) {
|
|
|
18964
19651
|
for (const [indexName, disposition] of dispositions) {
|
|
18965
19652
|
if (disposition === "orphaned") {
|
|
18966
19653
|
orphanedIndexes.add(indexName);
|
|
19654
|
+
} else if (disposition === "adopted") {
|
|
19655
|
+
await analysisView.settle(indexName);
|
|
18967
19656
|
}
|
|
18968
19657
|
}
|
|
18969
19658
|
if (hasDataRole) {
|
|
@@ -18980,6 +19669,7 @@ async function createClusterNode(callerConfig) {
|
|
|
18980
19669
|
engine,
|
|
18981
19670
|
coordinator: config.coordinator,
|
|
18982
19671
|
allocations: allocationView,
|
|
19672
|
+
analysisView,
|
|
18983
19673
|
writeDeps,
|
|
18984
19674
|
snapshotSyncState: snapshotSyncHandlerState,
|
|
18985
19675
|
searchContexts,
|
|
@@ -19014,14 +19704,11 @@ async function createClusterNode(callerConfig) {
|
|
|
19014
19704
|
await operationTracker.drain();
|
|
19015
19705
|
allocationView.stop();
|
|
19016
19706
|
bm25View.stop();
|
|
19017
|
-
|
|
19018
|
-
|
|
19019
|
-
|
|
19020
|
-
|
|
19021
|
-
|
|
19022
|
-
unregisterNotReadyHandler();
|
|
19023
|
-
unregisterNotReadyHandler = null;
|
|
19024
|
-
}
|
|
19707
|
+
analysisView.stop();
|
|
19708
|
+
unregisterHandler?.();
|
|
19709
|
+
unregisterHandler = null;
|
|
19710
|
+
unregisterNotReadyHandler?.();
|
|
19711
|
+
unregisterNotReadyHandler = null;
|
|
19025
19712
|
unwatchDepartures?.();
|
|
19026
19713
|
unwatchDepartures = null;
|
|
19027
19714
|
searchContexts?.stop();
|
|
@@ -19134,6 +19821,9 @@ function clusterNodeEngine(node, options) {
|
|
|
19134
19821
|
async rebuildAnalysis(indexName) {
|
|
19135
19822
|
return node.rebuildAnalysis(indexName);
|
|
19136
19823
|
},
|
|
19824
|
+
async changeAnalysis(indexName, change) {
|
|
19825
|
+
return node.changeAnalysis(indexName, change);
|
|
19826
|
+
},
|
|
19137
19827
|
async getMemoryStats() {
|
|
19138
19828
|
return node.getMemoryStats();
|
|
19139
19829
|
},
|