@delali/narsil 0.3.1-next.131 → 0.3.1-next.141
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/analysis-change-HQUT2Z3I.mjs +43 -0
- package/dist/analysis-change-entry-TZTG4AZB.mjs +1 -0
- package/dist/browser/analysis-change-JPV775LR.mjs +900 -0
- package/dist/browser/analysis-change-entry-7HR4LWL3.mjs +1 -0
- package/dist/browser/{chunk-UDGB2MWT.mjs → chunk-35B42UFM.mjs} +4 -0
- package/dist/browser/chunk-3ZBJYAS4.mjs +19 -0
- package/dist/browser/{chunk-2Y3CO4EQ.mjs → chunk-5DB2VDGI.mjs} +1149 -1853
- package/dist/browser/chunk-6RHTAVZX.mjs +656 -0
- package/dist/browser/chunk-CCOAOO73.mjs +86 -0
- package/dist/browser/{chunk-AIIJDWMG.mjs → chunk-D33A5VMU.mjs} +16 -16
- package/dist/browser/{chunk-VBBLTD7O.mjs → chunk-HKX2G6WH.mjs} +1520 -2159
- package/dist/browser/{chunk-NSCSR6MA.mjs → chunk-KTOK275N.mjs} +606 -830
- package/dist/browser/{chunk-D7WQF6DV.mjs → chunk-LVY2LI35.mjs} +2 -2
- package/dist/browser/chunk-MVM33ZUL.mjs +993 -0
- package/dist/browser/chunk-MWUK3DA5.mjs +2634 -0
- package/dist/browser/{chunk-VEGHFP2P.mjs → chunk-MZI3TEOW.mjs} +4 -2
- package/dist/browser/chunk-OIZS5KIX.mjs +6 -0
- package/dist/browser/chunk-R4WUDZQV.mjs +176 -0
- package/dist/browser/{chunk-M5UAAXGU.mjs → chunk-TCLEOZ4P.mjs} +3 -3
- package/dist/browser/chunk-VFYMPCOV.mjs +212 -0
- package/dist/browser/chunk-WCFCHIBB.mjs +177 -0
- package/dist/browser/chunk-XNWPP5NS.mjs +21 -0
- package/dist/browser/chunk-ZEV26AQ6.mjs +270 -0
- package/dist/browser/cold-module-MJMKNUYJ.mjs +745 -0
- package/dist/browser/{durability-wiring-FNZSRH4N.mjs → durability-wiring-SNOWIT6Z.mjs} +370 -95
- package/dist/browser/import-TLB2A5LJ.mjs +9 -0
- package/dist/browser/module-BUFNZMEU.mjs +12 -0
- package/dist/browser/{module-4HIMOQM5.mjs → module-BVP4TKZU.mjs} +397 -109
- package/dist/browser/{phrase-module-WESYVU7K.mjs → phrase-module-HGVBO24W.mjs} +1 -1
- package/dist/{restore-queue-JAMNRC6T.mjs → browser/restore-queue-ONYNYUDE.mjs} +8 -2
- package/dist/browser/{runtime-3X7ZJ5T3.mjs → runtime-VEUH674B.mjs} +2 -2
- package/dist/browser/{snapshot-V7VF6BN6.mjs → snapshot-QXBKHJAG.mjs} +135 -183
- package/dist/browser/{write-YIC2MKRY.mjs → write-WHFBDWQS.mjs} +64 -90
- package/dist/{capabilities-B6WxrWxt.d.ts → capabilities-BCdUwK_L.d.ts} +10 -3
- package/dist/chunk-46NZOPHS.mjs +21 -0
- package/dist/{chunk-JT3YZGCE.mjs → chunk-5WYVUPGD.mjs} +3 -188
- package/dist/chunk-6SNBQESJ.mjs +214 -0
- package/dist/{chunk-2C3WXBQS.mjs → chunk-74XB2SXI.mjs} +1 -1
- package/dist/chunk-7AUASRE7.mjs +653 -0
- package/dist/{chunk-NMXRXFGZ.mjs → chunk-AST3SHFA.mjs} +27 -25
- package/dist/chunk-BOJZGUS6.mjs +177 -0
- package/dist/{chunk-T4OFG4IQ.mjs → chunk-BRUUBXCB.mjs} +92 -17
- package/dist/{chunk-5LX6MOIB.mjs → chunk-C3YDYX6C.mjs} +1 -1
- package/dist/chunk-CZQ33DLW.mjs +78 -0
- package/dist/{chunk-4GDLFDPZ.mjs → chunk-DD2673TX.mjs} +3 -2
- package/dist/{chunk-EUCX5K74.mjs → chunk-DQNKVOJ2.mjs} +65 -65
- package/dist/{chunk-RNSC4NFI.mjs → chunk-EEBCDP36.mjs} +3 -1
- package/dist/chunk-FOCV4CBZ.mjs +87 -0
- package/dist/{chunk-HCGJKDZV.mjs → chunk-GLZHNSGB.mjs} +1 -1
- package/dist/chunk-GWOS5DYC.mjs +2637 -0
- package/dist/{chunk-PATMDIMN.mjs → chunk-HJJWY6PT.mjs} +1 -1
- package/dist/{chunk-CZ7ISLXA.mjs → chunk-HNHCNTVG.mjs} +4 -0
- package/dist/{chunk-Y2U5VVWX.mjs → chunk-ICW45TTU.mjs} +1 -1
- package/dist/{chunk-TGHNXN7W.mjs → chunk-IO5F7ZBE.mjs} +261 -948
- package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
- package/dist/{chunk-7OIQ2SIO.mjs → chunk-KEIYVZUX.mjs} +5 -5
- package/dist/{chunk-PU6SKH7O.mjs → chunk-KGOXHLLZ.mjs} +7 -3
- package/dist/{chunk-WVRDRLIC.mjs → chunk-KZL72VTC.mjs} +70 -18
- package/dist/chunk-L76W3IZJ.mjs +193 -0
- package/dist/{chunk-2JWNJSNE.mjs → chunk-LA4WNDHL.mjs} +88 -801
- package/dist/{chunk-ZR4VM4A6.mjs → chunk-N37VMC62.mjs} +1 -18
- package/dist/{chunk-JJ6EUMXB.mjs → chunk-NJJCPFXN.mjs} +2 -2
- package/dist/chunk-NN33CZRE.mjs +86 -0
- package/dist/{chunk-2WBY4Y4M.mjs → chunk-NOT5QOHU.mjs} +1 -1
- package/dist/{chunk-D5TFRB5G.mjs → chunk-NPNK57CA.mjs} +1 -1
- package/dist/chunk-NRUXBYPH.mjs +121 -0
- package/dist/{chunk-6SSVIW7Y.mjs → chunk-O4LJQI4G.mjs} +31 -5
- package/dist/{chunk-UJWGQIXU.mjs → chunk-QHPYS427.mjs} +1 -0
- package/dist/{chunk-F6AAXI7T.mjs → chunk-RHKRTYGO.mjs} +13 -4
- package/dist/{chunk-2SEL6SBS.mjs → chunk-RKAVHNRW.mjs} +2 -2
- package/dist/{chunk-RCMNL7RI.mjs → chunk-SBMAWTRG.mjs} +41 -4
- package/dist/chunk-SRJBFG2T.mjs +881 -0
- package/dist/{chunk-X5CNUKD5.mjs → chunk-TNBIK2LQ.mjs} +1663 -1652
- package/dist/{chunk-NNO7CO3A.mjs → chunk-UIKN2OTQ.mjs} +1 -1
- package/dist/chunk-UUZCQQG2.mjs +209 -0
- package/dist/{chunk-6M6SDH3S.mjs → chunk-VEFN6X6Q.mjs} +8 -8
- package/dist/chunk-VJEGOELZ.mjs +177 -0
- package/dist/chunk-W2COJ5AH.mjs +148 -0
- package/dist/{chunk-RWESFSJI.mjs → chunk-WJGO5PGV.mjs} +317 -174
- package/dist/{chunk-7X5JST5L.mjs → chunk-WNBRW4Q2.mjs} +16 -2
- package/dist/{chunk-SZGRYEIC.mjs → chunk-XS2VM7UE.mjs} +3 -3
- package/dist/{chunk-Y4BVTQ53.mjs → chunk-XZ3SRNDJ.mjs} +25 -64
- package/dist/{chunk-K77FJXKS.mjs → chunk-YHUTF6EA.mjs} +474 -93
- package/dist/{chunk-OGLV4I3L.mjs → chunk-YLNSJQLX.mjs} +12 -7
- package/dist/{chunk-SOHMMNCH.mjs → chunk-Z4HFMYNX.mjs} +32 -27
- package/dist/{chunk-5W7AEVTH.mjs → chunk-ZMBFWQXN.mjs} +2 -2
- package/dist/{chunk-2MKX2SCO.mjs → chunk-ZPWS2SKJ.mjs} +297 -274
- package/dist/{chunk-7XY6VD7B.mjs → chunk-ZSWHPKGM.mjs} +1 -1
- package/dist/{client-gNAfaHcE.d.ts → client-B12YFALP.d.ts} +18 -3
- package/dist/client.d.ts +7 -7
- package/dist/client.mjs +11 -1
- package/dist/cold-module-4X7WJCFM.mjs +783 -0
- package/dist/{config-D1cDv3sE.d.ts → config-BhZxIeb1.d.ts} +10 -1
- package/dist/{config-CmYJO34X.d.ts → config-CIG5yDyg.d.ts} +2 -2
- package/dist/{direct-executor-HQ4QVDX7.mjs → direct-executor-7XOQ7VQP.mjs} +14 -14
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +24 -8
- package/dist/distribution.mjs +982 -211
- package/dist/{durability-wiring-S7OX2MAI.mjs → durability-wiring-324D2DDD.mjs} +362 -94
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-m_Ddr1bz.d.ts → engine-CtzbHOmG.d.ts} +64 -7
- package/dist/import-OOBUW3RR.mjs +44 -0
- package/dist/{index-HYnqS1SE.d.ts → index-DsEurhSl.d.ts} +1 -1
- package/dist/index.browser.mjs +5 -5
- package/dist/index.d.ts +6 -6
- package/dist/index.mjs +22 -22
- package/dist/{long-query-thread-BSL25PD6.mjs → long-query-thread-4SW7RWDE.mjs} +15 -15
- package/dist/{module-2E4XYSGT.mjs → module-DXSJ5LT2.mjs} +378 -87
- package/dist/module-IYPLWHGC.mjs +48 -0
- package/dist/paged-partition-archive-HFP45PLU.mjs +269 -0
- package/dist/partial-words.mjs +5 -5
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.d.ts +16 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +28 -26
- package/dist/{phrase-module-JERF2IDG.mjs → phrase-module-5QHBD2VJ.mjs} +1 -1
- package/dist/react.d.ts +9 -9
- package/dist/react.mjs +4 -0
- package/dist/{rerank-DMMT4FZA.mjs → rerank-KTLC3E6N.mjs} +5 -5
- package/dist/rerankers/http.mjs +2 -2
- package/dist/{browser/restore-queue-GEZ54CA5.mjs → restore-queue-FEYAC4SB.mjs} +8 -2
- package/dist/{runtime-7MLQ7J64.mjs → runtime-DYV75DCB.mjs} +6 -6
- package/dist/{schema-BTXbHCqA.d.ts → schema-B-Runv0n.d.ts} +7 -0
- package/dist/server.d.ts +12 -10
- package/dist/server.mjs +54 -21
- package/dist/{snapshot-DOQLGGEX.mjs → snapshot-CGM65VCD.mjs} +101 -197
- package/dist/synonyms.mjs +6 -4
- package/dist/{tasks-Co8HfRD7.d.ts → tasks-CjRJRGTb.d.ts} +26 -8
- package/dist/{thread-SKO7CN3R.mjs → thread-KGNAIKPD.mjs} +14 -14
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +6 -5
- package/dist/workers/entry.mjs +8 -8
- package/dist/{write-ZKR7SNRT.mjs → write-2DZPEQL2.mjs} +82 -107
- package/openapi.json +114 -6
- package/package.json +12 -8
package/dist/distribution.mjs
CHANGED
|
@@ -1,65 +1,73 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
|
|
1
|
+
import { fixedHybridDepthOf, requireWithinHybridDepth, pageByPosition } from './chunk-C3YDYX6C.mjs';
|
|
2
|
+
import { closePartitionWriter, replaySyncedWal } from './chunk-RHKRTYGO.mjs';
|
|
3
|
+
import { archiveStream, buildArchiveFromMemory, memoryCheckpointStore, readSnapshotArchive, writeArchiveFile } from './chunk-KZL72VTC.mjs';
|
|
4
|
+
import { indexMetadataOf, loadPartitionSegments, readSegmentManifest } from './chunk-YHUTF6EA.mjs';
|
|
5
|
+
import { readVectorFile } from './chunk-NRUXBYPH.mjs';
|
|
6
|
+
import { createByteReader } from './chunk-CZQ33DLW.mjs';
|
|
7
|
+
import { DEFAULT_NODE_LIFECYCLE_CONFIG, bindWriteBlockLookup, bindEngineWriteBlock, bindIndexReads, DEFAULT_CONTROLLER_CONFIG, readIndexMetadataSnapshot, indexMetadataSnapshotOf, getIndexMetadata, getClusterIndexConfig, withPartitionCount, validateIndexName, indexConfigFromSettings, metadataWithBm25, indexConfigKey, readIndexState, storeSynonymSet, deleteSynonymSet, indexSettingsOf, putIndexMetadata, isValidInteger, isRecord, isRecord2, decodeIndexSettings, HIDDEN_GENERATION_PREFIX, metadataWithSynonymSet, RESHARD_RECORD_CAS_ATTEMPTS, indexUuidOfStoredMetadata, RESHARD_FORWARD_BATCH_DOCUMENTS, storedSynonymSetOf, RESHARD_RETRY_DELAY_MS, RESHARD_RECORD_CHECK_INTERVAL_MS, RESHARD_POLL_INTERVAL_MS, ALLOCATION_RETRY_DELAY_MS, TEARDOWN_CAS_ATTEMPTS, TEARDOWN_RETRY_ROUNDS, ANALYSIS_CHANGE_RETRY_MS, putIndexReplicationFactor, ALLOCATION_CAS_ATTEMPTS, TEARDOWN_RETRY_DELAY_MS, RESHARD_CONTROLLER_POLL_INTERVAL_MS, MAX_RESHARD_CUTOVER_ATTEMPTS, ANALYSIS_CHANGE_POLL_INTERVAL_MS, RECOVERY_CAS_ATTEMPTS, ESTIMATED_PARTITION_BYTES, REBALANCE_THRESHOLD, putGenerationMetadata, RESHARD_CUTOVER_LAG_ENTRIES, ANALYSIS_REBUILD_RESEND_MS, RESHARD_CUTOVER_TIMEOUT_MS, PARTITION_STORES_TIMEOUT_MS, LEADERSHIP_IMBALANCE_THRESHOLD, BOOTSTRAP_CAS_ATTEMPTS } from './chunk-BRUUBXCB.mjs';
|
|
8
|
+
import { MAX_WAIT_FOR_ACTIVE_REPLICAS, MIN_CONTROLLER_LEASE_TTL_MS, MAX_NODE_LEFT_DELAY_MS, DEFAULT_MAX_PER_SOURCE_SNAPSHOTS, DEFAULT_MAX_STREAMS_PER_INDEX, DEFAULT_CAPACITY, CATCH_UP_TICK_MS, DEFAULT_RESUME_WRITES_BELOW, DEFAULT_BLOCK_WRITES_ABOVE, readMemoryFigures, createWritePressure, WRITE_MEMORY_SAMPLE_MAX_AGE_MS, DEFAULT_IN_FLIGHT_WRITE_FRACTION, MAX_SETTINGS_REPLACEMENT_ATTEMPTS, SETTINGS_CONTENTION_REASON, MAX_DROP_ACKNOWLEDGE_WAIT_MS, DEFAULT_PARTITION_COUNT, DEFAULT_REPLICATION_FACTOR, DEFAULT_CREATE_INDEX_WAIT_MS, refuseEncodeWithoutRoom, readRecentMemoryFigures, DEFAULT_BOOTSTRAP_SYNC_DEADLINE_MS, DROP_ACKNOWLEDGE_POLL_INTERVAL_MS, COORDINATOR_REQUEST_BUCKET_BOUNDS_SECONDS, COPY_FAILURE_ATTEMPTS, SEARCH_CONTEXT_REAPER_INTERVAL_MS, DECOMMISSION_RECHECK_INTERVAL_MS, answerRefusalOf, withAdmittedWrite, estimateForwardedBytes, CASE_FOLD_EXPANSION_FACTOR, PATTERN_RUN_HEAP_BYTES, ENUM_VALUE_HEAP_BYTES, MAX_SOURCE_ID_LENGTH, DOCUMENT_VALUE_HEAP_BYTES, DOCUMENT_HEAP_BYTES_PER_ESTIMATED_BYTE, DOCUMENT_PROPERTY_HEAP_BYTES, VECTOR_DOCUMENT_HEAP_BYTES, VECTOR_COMPONENT_HEAP_BYTES, NORMALIZATION_EXPANSION_FACTOR, TERM_UNIT_HEAP_BYTES, PARTITION_BYTES_INDEX_OVERHEAD_BYTES, DISTINCT_TERM_HEAP_BYTES, PARTIAL_SPELLING_HEAP_BYTES, PARTITION_BYTES_ENTRY_BYTES, PARTITION_BYTES_REPORT_BUDGET_BYTES, SNAPSHOT_CHUNK_YIELD_INTERVAL_MS, CATCH_UP_FEED_WINDOW_MS, MAX_CATCH_UP_IN_FLIGHT_BYTES, SYNCED_POSITION_REPORT_TIMEOUT_MS, ADMISSION_TIMEOUT_MS, REGISTRATION_LEASE_DRIFT_SHARE, REGISTRATION_LEASE_CONFIRM_TIMEOUT_MS, WRITE_APPLY_YIELD_INTERVAL_MS, ROLLBACK_RETRY_INITIAL_DELAY_MS, ROLLBACK_RETRY_DELAY_CAP_MS, COPY_FAILURE_RETRY_DELAY_MS, FETCH_GROUP_DOCUMENT_IDS, SEARCH_CONTEXT_ENTRY_HEAP_BYTES, FULL_MEMORY_COLLECTION_INTERVAL_MS, MEMORY_SAMPLE_INTERVAL_MS, ANSWER_DECODE_HEAP_BYTES_PER_PAYLOAD_BYTE, CLEAR_PAGE_SIZE, estimateDocumentsBytes, estimateDocumentBytes, PRIMARY_CHANGE_RETRY_WINDOW_MS, PRIMARY_CHANGE_POLL_INTERVAL_MS, PARTITION_WRITE_CONCURRENCY, FORWARD_RETRY_INITIAL_DELAY_MS, FORWARD_RETRY_DELAY_CAP_MS } from './chunk-DD2673TX.mjs';
|
|
9
|
+
export { DEFAULT_CAPACITY, DEFAULT_PARTITION_COUNT, DEFAULT_REPLICATION_FACTOR } from './chunk-DD2673TX.mjs';
|
|
10
|
+
import './chunk-L76W3IZJ.mjs';
|
|
11
|
+
import './chunk-W2COJ5AH.mjs';
|
|
5
12
|
import { addBackgroundAdmission, workWaitingForMemory } from './chunk-PELHVTO7.mjs';
|
|
6
|
-
import './chunk-
|
|
13
|
+
import './chunk-Z4HFMYNX.mjs';
|
|
7
14
|
import { onRefusalAfterCollection, reservedHeapBytes, reserveHeapBytes, collectYoungGenerationBeforeRefusal, collectGarbageBeforeRefusal } from './chunk-IWVEG5CX.mjs';
|
|
8
|
-
import {
|
|
9
|
-
import {
|
|
10
|
-
import
|
|
11
|
-
import { decodeVectorFilePayload, bytesToVectors, isDead } from './chunk-Y2U5VVWX.mjs';
|
|
15
|
+
import { decodeVectorFilePayload, bytesToVectors, isDead } from './chunk-ICW45TTU.mjs';
|
|
16
|
+
import { stampAnalysisGeneration, holdsNamedSettings, dropHeldAnalysisChange, releaseHeldAnalysisChange, heldAnalysisIsBuilt, buildHeldAnalysisChange, resolveAnalysisTarget, namedSettingsOf, holdsSettings } from './chunk-SRJBFG2T.mjs';
|
|
17
|
+
import './chunk-VJEGOELZ.mjs';
|
|
12
18
|
import { CONTROLLER_LEASE_KEY } from './chunk-44W3CBNG.mjs';
|
|
13
19
|
import { MAX_MESSAGE_SIZE_BYTES } from './chunk-UNEEFTEU.mjs';
|
|
14
|
-
import './chunk-
|
|
15
|
-
import './chunk-DJHHF2L5.mjs';
|
|
20
|
+
import './chunk-TNBIK2LQ.mjs';
|
|
16
21
|
import './chunk-37CD3OKS.mjs';
|
|
22
|
+
import './chunk-DJHHF2L5.mjs';
|
|
17
23
|
import { getPath, getFs } from './chunk-HLPRHOSO.mjs';
|
|
18
|
-
import { createForwardBatchResultMessage, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload,
|
|
19
|
-
import {
|
|
20
|
-
import
|
|
24
|
+
import { createForwardBatchResultMessage, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, optionalIndexUuid, computeEntryChecksum, createPrimaryResignMessage, validateInsyncConfirmPayload, buildEntry, createInsyncRemoveMessage, createEntryBatchMessage, createInsyncConfirmMessage, createEntryMessage, validateAckPayload, createInsyncAddMessage, checkpointLastSeqNo, checkpointPrimaryTerm, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage } from './chunk-LA4WNDHL.mjs';
|
|
25
|
+
import { MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR, WIRE_BATCH_BUDGET } from './chunk-JDDXV6UF.mjs';
|
|
26
|
+
import './chunk-7AUASRE7.mjs';
|
|
27
|
+
import { directoryCheckpointStore, readListedFile } from './chunk-UUZCQQG2.mjs';
|
|
28
|
+
import { applyIndexEntry, applyDeleteEntry, validateReplicationEntry } from './chunk-KEIYVZUX.mjs';
|
|
21
29
|
import { TransportErrorCodes, ReplicationMessageTypes, ClusterMessageTypes, QueryMessageTypes, TransportError, DEFAULT_TRANSPORT_CONFIG } from './chunk-YH5OIL7B.mjs';
|
|
22
|
-
import { requireKnownConfig, resolveRunnableConfig, createEngineCore, startEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, applyStoredIndexBm25, createEngineIndex, encodeIndexConfiguration, requireConfigurationWithinSizeLimit, prepareConfiguredSynonyms,
|
|
23
|
-
import { runtimeHost, validateBM25Params, storedBM25Params, requireSurfaceFormsForPartialFields, validateBM25Shape, now, listBindingOf, clampLimit, hitsKeptPerGroup, queryBindingOf, resolveHybridFusion, dedupePinnedEntries, foldGroupReducer } from './chunk-
|
|
24
|
-
import { bindRerankWindowSource, rerankWindowCapOf, providedDocId, engineCoreOf } from './chunk-
|
|
25
|
-
import './chunk-
|
|
30
|
+
import { requireKnownConfig, resolveRunnableConfig, createEngineCore, startEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, applyStoredIndexBm25, createEngineIndex, encodeIndexConfiguration, requireConfigurationWithinSizeLimit, prepareConfiguredSynonyms, openChecksummedEnvelope, storedArrayPositionGap, emptySerializablePartition, arrayPositionsAreStale, crc32Init, crc32Final, crc32Update, yieldToEventLoop } from './chunk-IO5F7ZBE.mjs';
|
|
31
|
+
import { runtimeHost, validateBM25Params, storedBM25Params, requireSurfaceFormsForPartialFields, validateBM25Shape, now, listBindingOf, clampLimit, hitsKeptPerGroup, queryBindingOf, resolveHybridFusion, dedupePinnedEntries, foldGroupReducer } from './chunk-VEFN6X6Q.mjs';
|
|
32
|
+
import { bindRerankWindowSource, rerankWindowCapOf, providedDocId, engineCoreOf } from './chunk-5WYVUPGD.mjs';
|
|
33
|
+
import './chunk-HJSN7UYG.mjs';
|
|
34
|
+
import './chunk-UIKN2OTQ.mjs';
|
|
26
35
|
import './chunk-AV4IJKD2.mjs';
|
|
27
|
-
import {
|
|
28
|
-
import {
|
|
29
|
-
import {
|
|
36
|
+
import { MAX_DOC_ID_LENGTH, BATCH_CHUNK_SIZE, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING } from './chunk-EEBCDP36.mjs';
|
|
37
|
+
import { synonymModule } from './chunk-WJGO5PGV.mjs';
|
|
38
|
+
import { oversampledShardSize, bm25SetsValue, normalizeSort, sortModesOf, readSortValues, resolveProjection, fnv1a, applyProjection, FILTER_EXPRESSION_KEYS, filterTestCount, setNestedValue, FIELD_FILTER_OPERATORS, clampRowCount, sortSignatureOf, requireMatchingCursor, decodePageCursor, encodePageCursor, requireWithinResultWindow, holdsQuotedText, sortSignatureEntry, fnv1aUint32s, mergePartitionStats } from './chunk-ZPWS2SKJ.mjs';
|
|
39
|
+
import { generateId } from './chunk-N37VMC62.mjs';
|
|
30
40
|
import './chunk-ZATRDAYU.mjs';
|
|
31
41
|
import './chunk-EBIHRL7B.mjs';
|
|
32
|
-
import './chunk-HJSN7UYG.mjs';
|
|
33
|
-
import { MAX_DOC_ID_LENGTH, BATCH_CHUNK_SIZE, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING } from './chunk-RNSC4NFI.mjs';
|
|
34
|
-
import './chunk-TGP5M5R7.mjs';
|
|
35
|
-
import './chunk-U2BKYEC7.mjs';
|
|
36
42
|
import './chunk-KOKM4PQN.mjs';
|
|
37
43
|
import { MIN_OVERSAMPLE } from './chunk-QSK35UC2.mjs';
|
|
38
|
-
import './chunk-
|
|
39
|
-
import { isPatternTolerance } from './chunk-
|
|
44
|
+
import './chunk-74XB2SXI.mjs';
|
|
45
|
+
import { isPatternTolerance } from './chunk-RKAVHNRW.mjs';
|
|
40
46
|
import { MAX_FILTER_DEPTH, MAX_FILTER_TESTS, MAX_FILTER_ARRAY_SIZE } from './chunk-WQBCBKY3.mjs';
|
|
41
47
|
import './chunk-JUKYYXSP.mjs';
|
|
42
|
-
import { getNestedValue } from './chunk-
|
|
43
|
-
import { encodeDocument, validateSchema, requireCreatableSchema, encodeDocumentReply, schemaFieldsOf, MIN_POLYGON_POINTS, isLatitude, MAX_LATITUDE_DEGREES, isLongitude, MAX_LONGITUDE_DEGREES } from './chunk-
|
|
48
|
+
import { getNestedValue } from './chunk-DQNKVOJ2.mjs';
|
|
49
|
+
import { encodeDocument, validateSchema, requireCreatableSchema, encodeDocumentReply, schemaFieldsOf, MIN_POLYGON_POINTS, isLatitude, MAX_LATITUDE_DEGREES, isLongitude, MAX_LONGITUDE_DEGREES } from './chunk-KGOXHLLZ.mjs';
|
|
44
50
|
import { MAX_DOCUMENT_ENCODING_DEPTH } from './chunk-U476IXKQ.mjs';
|
|
45
51
|
import { MAX_SORT_FIELDS, MAX_CURSOR_LENGTH, MAX_TOLERANCE, MAX_PREFIX_LENGTH, RESULT_WINDOW, MIN_HYBRID_DEPTH, MAX_HYBRID_DEPTH, DEFAULT_PAGE_SIZE, HYBRID_RESCORE_WINDOW } from './chunk-FQBFZAWG.mjs';
|
|
46
52
|
import './chunk-VJS4JFBQ.mjs';
|
|
47
53
|
import './chunk-GES7U2PM.mjs';
|
|
48
|
-
import './chunk-
|
|
49
|
-
import { resolveIndexAnalysis } from './chunk-
|
|
54
|
+
import './chunk-ZSWHPKGM.mjs';
|
|
55
|
+
import { resolveIndexAnalysis, configOfNamedSettings } from './chunk-O4LJQI4G.mjs';
|
|
50
56
|
import './chunk-X5CV2E2B.mjs';
|
|
51
|
-
import './chunk-
|
|
52
|
-
import './chunk-
|
|
57
|
+
import { getLanguage } from './chunk-WNBRW4Q2.mjs';
|
|
58
|
+
import './chunk-TGP5M5R7.mjs';
|
|
59
|
+
import './chunk-U2BKYEC7.mjs';
|
|
60
|
+
import './chunk-NPNK57CA.mjs';
|
|
53
61
|
import './chunk-4UMZ7KQL.mjs';
|
|
54
62
|
import { isSortMode, SORT_MODES, toComparableSortValue, compareComparableKeys, compareSortValues } from './chunk-R6A773WK.mjs';
|
|
55
63
|
import { compareCodePoints, exceedsCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
56
64
|
import { collectAndReturnPages } from './chunk-RIIUZJQ3.mjs';
|
|
57
|
-
import './chunk-
|
|
58
|
-
import './chunk-
|
|
65
|
+
import './chunk-QHPYS427.mjs';
|
|
66
|
+
import './chunk-GLZHNSGB.mjs';
|
|
59
67
|
import { HEAP_RESUME_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE } from './chunk-22YPZVMG.mjs';
|
|
60
68
|
import './chunk-RFBLQLLZ.mjs';
|
|
61
69
|
import { MAX_STORED_VECTOR_DIMENSION, MAX_NESTING_DEPTH } from './chunk-L5ORED7V.mjs';
|
|
62
|
-
import { ErrorCodes, NarsilError, describeError, ServerErrorCodes } from './chunk-
|
|
70
|
+
import { ErrorCodes, NarsilError, describeError, ServerErrorCodes } from './chunk-HNHCNTVG.mjs';
|
|
63
71
|
import { encode, decode } from '@msgpack/msgpack';
|
|
64
72
|
import { randomUUID } from 'crypto';
|
|
65
73
|
import { openSync, writeSync, closeSync, rmSync, readSync } from 'fs';
|
|
@@ -1089,6 +1097,134 @@ async function openSnapshotSpoolDirectory(durabilityDirectory) {
|
|
|
1089
1097
|
}
|
|
1090
1098
|
return { path, dispose: () => rm(path, { recursive: true, force: true }) };
|
|
1091
1099
|
}
|
|
1100
|
+
|
|
1101
|
+
// src/distribution/cluster-node/analysis-change/view.ts
|
|
1102
|
+
function namedSettingsOfMetadata(settings) {
|
|
1103
|
+
return {
|
|
1104
|
+
language: settings?.language ?? "english",
|
|
1105
|
+
...settings?.tokenizer !== void 0 ? { tokenizer: settings.tokenizer } : {},
|
|
1106
|
+
...settings?.stopWords !== void 0 ? { stopWords: settings.stopWords } : {},
|
|
1107
|
+
...settings?.stopWordList !== void 0 ? { stopWordList: settings.stopWordList } : {}
|
|
1108
|
+
};
|
|
1109
|
+
}
|
|
1110
|
+
function generationMissing(indexName, requestedGeneration, heldGeneration) {
|
|
1111
|
+
return new NarsilError(
|
|
1112
|
+
ErrorCodes.ANALYSIS_GENERATION_MISSING,
|
|
1113
|
+
`This node holds analysis generation ${heldGeneration} of index '${indexName}', so it cannot answer for generation ${requestedGeneration}`,
|
|
1114
|
+
{ indexName, requestedGeneration, heldGeneration }
|
|
1115
|
+
);
|
|
1116
|
+
}
|
|
1117
|
+
function createAnalysisView(deps) {
|
|
1118
|
+
const { coordinator, engine, onError } = deps;
|
|
1119
|
+
const held = /* @__PURE__ */ new Map();
|
|
1120
|
+
const settling = /* @__PURE__ */ new Map();
|
|
1121
|
+
let unwatch = null;
|
|
1122
|
+
let stopped = false;
|
|
1123
|
+
function recordGeneration(indexName, snapshot) {
|
|
1124
|
+
const generation = snapshot.settings?.analysisGeneration ?? 0;
|
|
1125
|
+
const { indexUuid } = snapshot.record;
|
|
1126
|
+
const known = held.get(indexName);
|
|
1127
|
+
if (known?.indexUuid === indexUuid && known.generation >= generation) return known.generation;
|
|
1128
|
+
held.set(indexName, { indexUuid, generation });
|
|
1129
|
+
return generation;
|
|
1130
|
+
}
|
|
1131
|
+
async function refresh(indexName) {
|
|
1132
|
+
const snapshot = await readIndexMetadataSnapshot(coordinator, indexName);
|
|
1133
|
+
if (snapshot === null) {
|
|
1134
|
+
held.delete(indexName);
|
|
1135
|
+
return 0;
|
|
1136
|
+
}
|
|
1137
|
+
return recordGeneration(indexName, snapshot);
|
|
1138
|
+
}
|
|
1139
|
+
function catchUp(indexName, generation, settings) {
|
|
1140
|
+
engine.buildHeldAnalysis(indexName, { generation, ...namedSettingsOfMetadata(settings) }).then(() => engine.releaseHeldAnalysis(indexName, generation)).catch((error) => {
|
|
1141
|
+
if (!stopped) onError(error);
|
|
1142
|
+
});
|
|
1143
|
+
}
|
|
1144
|
+
async function settleOnce(indexName) {
|
|
1145
|
+
const generation = engine.analysisGenerationOf(indexName);
|
|
1146
|
+
if (stopped || generation === void 0) return;
|
|
1147
|
+
const snapshot = await readIndexMetadataSnapshot(coordinator, indexName);
|
|
1148
|
+
const localUuid = engine.indexUuidOf(indexName);
|
|
1149
|
+
if (snapshot === null || typeof localUuid === "string" && localUuid !== snapshot.record.indexUuid) return;
|
|
1150
|
+
recordGeneration(indexName, snapshot);
|
|
1151
|
+
const { settings } = snapshot;
|
|
1152
|
+
const published = settings?.analysisGeneration ?? 0;
|
|
1153
|
+
const localPending = engine.pendingAnalysisGenerationOf(indexName);
|
|
1154
|
+
if (published > generation) {
|
|
1155
|
+
if (localPending !== published && engine.holdsAnalysisSettings(indexName, namedSettingsOfMetadata(settings))) {
|
|
1156
|
+
if (localPending !== null) await engine.dropHeldAnalysis(indexName);
|
|
1157
|
+
await engine.stampAnalysisGeneration(indexName, published);
|
|
1158
|
+
return;
|
|
1159
|
+
}
|
|
1160
|
+
catchUp(indexName, published, settings);
|
|
1161
|
+
return;
|
|
1162
|
+
}
|
|
1163
|
+
if (localPending !== null && localPending > published && settings?.pendingAnalysis?.generation !== localPending) {
|
|
1164
|
+
await engine.dropHeldAnalysis(indexName);
|
|
1165
|
+
}
|
|
1166
|
+
}
|
|
1167
|
+
function settle(indexName) {
|
|
1168
|
+
const next = (settling.get(indexName) ?? Promise.resolve()).then(() => settleOnce(indexName)).catch(onError);
|
|
1169
|
+
settling.set(indexName, next);
|
|
1170
|
+
return next;
|
|
1171
|
+
}
|
|
1172
|
+
async function reach(indexName, requested) {
|
|
1173
|
+
if (engine.heldAnalysisIsBuilt(indexName, requested)) {
|
|
1174
|
+
await engine.releaseHeldAnalysis(indexName, requested).catch(onError);
|
|
1175
|
+
return;
|
|
1176
|
+
}
|
|
1177
|
+
await settle(indexName);
|
|
1178
|
+
}
|
|
1179
|
+
async function reachThenRead(indexName, requested, generation) {
|
|
1180
|
+
if (generation < requested) await reach(indexName, requested);
|
|
1181
|
+
const reached = engine.analysisGenerationOf(indexName) ?? generation;
|
|
1182
|
+
if (reached !== requested) throw generationMissing(indexName, requested, reached);
|
|
1183
|
+
}
|
|
1184
|
+
return {
|
|
1185
|
+
held(indexName) {
|
|
1186
|
+
return held.get(indexName)?.generation;
|
|
1187
|
+
},
|
|
1188
|
+
async generationFor(indexName) {
|
|
1189
|
+
return held.get(indexName)?.generation ?? refresh(indexName);
|
|
1190
|
+
},
|
|
1191
|
+
refresh,
|
|
1192
|
+
answerAt(indexName, requested, read) {
|
|
1193
|
+
const generation = engine.analysisGenerationOf(indexName);
|
|
1194
|
+
if (generation === void 0) return read();
|
|
1195
|
+
const readAtRequested = (answerOf) => answerOf().then((answer) => {
|
|
1196
|
+
const after = engine.analysisGenerationOf(indexName) ?? requested;
|
|
1197
|
+
if (after !== requested) throw generationMissing(indexName, requested, after);
|
|
1198
|
+
return answer;
|
|
1199
|
+
});
|
|
1200
|
+
if (generation === requested) return readAtRequested(read);
|
|
1201
|
+
return reachThenRead(indexName, requested, generation).then(() => readAtRequested(read));
|
|
1202
|
+
},
|
|
1203
|
+
settle,
|
|
1204
|
+
async start() {
|
|
1205
|
+
if (unwatch !== null) return;
|
|
1206
|
+
stopped = false;
|
|
1207
|
+
unwatch = await coordinator.watchIndexMetadata((event) => {
|
|
1208
|
+
if (event.metadata === null) {
|
|
1209
|
+
held.delete(event.indexName);
|
|
1210
|
+
return;
|
|
1211
|
+
}
|
|
1212
|
+
try {
|
|
1213
|
+
recordGeneration(event.indexName, indexMetadataSnapshotOf(event.metadata, event.indexName));
|
|
1214
|
+
} catch (error) {
|
|
1215
|
+
onError(error);
|
|
1216
|
+
}
|
|
1217
|
+
void settle(event.indexName);
|
|
1218
|
+
});
|
|
1219
|
+
},
|
|
1220
|
+
stop() {
|
|
1221
|
+
stopped = true;
|
|
1222
|
+
unwatch?.();
|
|
1223
|
+
unwatch = null;
|
|
1224
|
+
held.clear();
|
|
1225
|
+
}
|
|
1226
|
+
};
|
|
1227
|
+
}
|
|
1092
1228
|
function settingOfMetadata(bytes, indexName) {
|
|
1093
1229
|
const decoded = decode(bytes);
|
|
1094
1230
|
if (!isRecord2(decoded) || typeof decoded.indexUuid !== "string") return null;
|
|
@@ -2375,14 +2511,16 @@ async function withRoutedIndexUuid(payload, deps) {
|
|
|
2375
2511
|
const indexUuid = (await deps.coordinator.getAllocation(payload.indexName))?.indexUuid;
|
|
2376
2512
|
return indexUuid === void 0 ? payload : { ...payload, indexUuid };
|
|
2377
2513
|
}
|
|
2378
|
-
async function reportSyncedPosition(indexName, partitionId, target,
|
|
2514
|
+
async function reportSyncedPosition(indexName, partitionId, target, deadline, deps) {
|
|
2379
2515
|
const logState = readLocalLogState(indexName, partitionId, deps);
|
|
2380
|
-
|
|
2381
|
-
return;
|
|
2382
|
-
}
|
|
2516
|
+
const details = { indexName, partitionId, target };
|
|
2383
2517
|
const budgetMs = Math.min(SYNCED_POSITION_REPORT_TIMEOUT_MS, deadline - Date.now());
|
|
2384
2518
|
if (budgetMs <= 0) {
|
|
2385
|
-
return
|
|
2519
|
+
return new NarsilError(
|
|
2520
|
+
ErrorCodes.SNAPSHOT_SYNC_TIMEOUT,
|
|
2521
|
+
"bootstrap sync exceeded deadline before reporting the synced position",
|
|
2522
|
+
details
|
|
2523
|
+
);
|
|
2386
2524
|
}
|
|
2387
2525
|
const payload = {
|
|
2388
2526
|
indexName,
|
|
@@ -2396,19 +2534,38 @@ async function reportSyncedPosition(indexName, partitionId, target, knownSeqNo,
|
|
|
2396
2534
|
requestId: generateId(),
|
|
2397
2535
|
payload: encode(await withRoutedIndexUuid(payload, deps))
|
|
2398
2536
|
};
|
|
2537
|
+
const frameState = createLiveSyncFrameState(
|
|
2538
|
+
indexName,
|
|
2539
|
+
partitionId,
|
|
2540
|
+
deps.snapshotSpoolDirectory ?? defaultSpoolDirectory()
|
|
2541
|
+
);
|
|
2399
2542
|
try {
|
|
2400
2543
|
await withDeadline(
|
|
2401
|
-
deps.transport.stream(target, request, () => {
|
|
2544
|
+
deps.transport.stream(target, request, (chunk) => {
|
|
2545
|
+
handleLiveSyncFrame(frameState, chunk);
|
|
2402
2546
|
}),
|
|
2403
2547
|
budgetMs,
|
|
2404
2548
|
indexName,
|
|
2405
2549
|
"synced-position-report"
|
|
2406
2550
|
);
|
|
2407
|
-
} catch (
|
|
2408
|
-
|
|
2409
|
-
|
|
2410
|
-
|
|
2551
|
+
} catch (err) {
|
|
2552
|
+
const cause = err instanceof Error ? err.message : String(err);
|
|
2553
|
+
return new NarsilError(
|
|
2554
|
+
ErrorCodes.SNAPSHOT_SYNC_TRANSPORT_FAILED,
|
|
2555
|
+
`reporting the synced position failed: ${cause}`,
|
|
2556
|
+
{ ...details, cause }
|
|
2557
|
+
);
|
|
2558
|
+
} finally {
|
|
2559
|
+
abandonSnapshotStream(frameState.snapshotState);
|
|
2560
|
+
}
|
|
2561
|
+
if (frameState.sawSnapshotFrame) {
|
|
2562
|
+
return new NarsilError(
|
|
2563
|
+
ErrorCodes.SNAPSHOT_SYNC_TRANSPORT_FAILED,
|
|
2564
|
+
"the primary no longer retains the synced position, so the copy syncs again",
|
|
2565
|
+
details
|
|
2566
|
+
);
|
|
2411
2567
|
}
|
|
2568
|
+
return frameState.error;
|
|
2412
2569
|
}
|
|
2413
2570
|
async function syncFromAnyTarget(state, entry, indexName, partitionId, primaryNodeId, targets, coordinatorConfig, partitionCount, deadline, deps, abortCheck) {
|
|
2414
2571
|
let lastError = new NarsilError(ErrorCodes.SNAPSHOT_SYNC_TRANSPORT_FAILED, "no live sync attempts were made", {
|
|
@@ -2443,7 +2600,6 @@ async function syncFromAnyTarget(state, entry, indexName, partitionId, primaryNo
|
|
|
2443
2600
|
)
|
|
2444
2601
|
};
|
|
2445
2602
|
}
|
|
2446
|
-
const positionBeforeAttempt = readLocalLogState(indexName, partitionId, deps).lastSeqNo;
|
|
2447
2603
|
const attempt = await syncFromTarget(
|
|
2448
2604
|
state,
|
|
2449
2605
|
entry,
|
|
@@ -2456,15 +2612,16 @@ async function syncFromAnyTarget(state, entry, indexName, partitionId, primaryNo
|
|
|
2456
2612
|
deps,
|
|
2457
2613
|
abortCheck
|
|
2458
2614
|
);
|
|
2459
|
-
|
|
2460
|
-
|
|
2615
|
+
const movedThePosition = attempt.ok && (attempt.tier === "snapshot" || attempt.entriesApplied > 0);
|
|
2616
|
+
const failure = !attempt.ok ? attempt.error : movedThePosition ? await reportSyncedPosition(indexName, partitionId, target, deadline, deps) : null;
|
|
2617
|
+
if (failure === null) {
|
|
2461
2618
|
return attempt;
|
|
2462
2619
|
}
|
|
2463
|
-
lastError = new NarsilError(
|
|
2464
|
-
...
|
|
2620
|
+
lastError = new NarsilError(failure.code, failure.message, {
|
|
2621
|
+
...failure.details,
|
|
2465
2622
|
target
|
|
2466
2623
|
});
|
|
2467
|
-
if (!isTransientFailure(
|
|
2624
|
+
if (!isTransientFailure(failure.code, failure.details)) {
|
|
2468
2625
|
return { ok: false, error: lastError };
|
|
2469
2626
|
}
|
|
2470
2627
|
}
|
|
@@ -4861,7 +5018,9 @@ function createEventLoopState(initialIndexNames, nodeLeftDelayMs = DEFAULT_CONTR
|
|
|
4861
5018
|
transport: null,
|
|
4862
5019
|
controllerNodeId: null,
|
|
4863
5020
|
rebalanceDrivers: /* @__PURE__ */ new Map(),
|
|
4864
|
-
rebalanceEpoch: 0
|
|
5021
|
+
rebalanceEpoch: 0,
|
|
5022
|
+
unwatchMetadata: null,
|
|
5023
|
+
analysisDrivers: /* @__PURE__ */ new Map()
|
|
4865
5024
|
};
|
|
4866
5025
|
}
|
|
4867
5026
|
function clearEventLoopWatchers(state) {
|
|
@@ -4893,6 +5052,9 @@ function clearEventLoopWatchers(state) {
|
|
|
4893
5052
|
state.departedAt.clear();
|
|
4894
5053
|
state.rebalanceDrivers.clear();
|
|
4895
5054
|
state.rebalanceEpoch += 1;
|
|
5055
|
+
state.unwatchMetadata?.();
|
|
5056
|
+
state.unwatchMetadata = null;
|
|
5057
|
+
state.analysisDrivers.clear();
|
|
4896
5058
|
}
|
|
4897
5059
|
function inSchemaQueue(state, indexName, task) {
|
|
4898
5060
|
const queued = (state.schemaQueues.get(indexName) ?? Promise.resolve()).then(task);
|
|
@@ -5436,6 +5598,212 @@ async function handleSchemaEvent(event, coordinator, state, isActive, onError) {
|
|
|
5436
5598
|
await tearDownAllocation(event.indexName, coordinator, state, isActive, 0, onError);
|
|
5437
5599
|
}
|
|
5438
5600
|
}
|
|
5601
|
+
|
|
5602
|
+
// src/distribution/query/node-send.ts
|
|
5603
|
+
async function resolveTargets(nodeId, deps) {
|
|
5604
|
+
if (deps.resolveNodeTargets === void 0) {
|
|
5605
|
+
return [nodeId];
|
|
5606
|
+
}
|
|
5607
|
+
const targets = await deps.resolveNodeTargets(nodeId);
|
|
5608
|
+
return targets.length > 0 ? targets : [nodeId];
|
|
5609
|
+
}
|
|
5610
|
+
async function sendToFirstReachableTarget2(transport, targets, message) {
|
|
5611
|
+
let lastError;
|
|
5612
|
+
for (const target of targets) {
|
|
5613
|
+
try {
|
|
5614
|
+
return await transport.send(target, message);
|
|
5615
|
+
} catch (error) {
|
|
5616
|
+
lastError = error;
|
|
5617
|
+
}
|
|
5618
|
+
}
|
|
5619
|
+
throw lastError instanceof Error ? lastError : new Error(String(lastError));
|
|
5620
|
+
}
|
|
5621
|
+
function fieldsOf(snapshot) {
|
|
5622
|
+
const decoded = decode(snapshot.bytes);
|
|
5623
|
+
const fields = isRecord2(decoded) ? decoded : {};
|
|
5624
|
+
const settings = isRecord2(fields.settings) ? fields.settings : { language: snapshot.settings?.language };
|
|
5625
|
+
return { fields, settings };
|
|
5626
|
+
}
|
|
5627
|
+
function encoded(fields, settings) {
|
|
5628
|
+
return new Uint8Array(encode({ ...fields, settings }, { ignoreUndefined: true }));
|
|
5629
|
+
}
|
|
5630
|
+
function metadataWithPendingAnalysis(snapshot, pending) {
|
|
5631
|
+
const { fields, settings } = fieldsOf(snapshot);
|
|
5632
|
+
return encoded(fields, { ...settings, pendingAnalysis: pending });
|
|
5633
|
+
}
|
|
5634
|
+
function metadataWithoutPendingAnalysis(snapshot) {
|
|
5635
|
+
const { fields, settings } = fieldsOf(snapshot);
|
|
5636
|
+
const { pendingAnalysis: _pending, ...rest } = settings;
|
|
5637
|
+
return encoded(fields, rest);
|
|
5638
|
+
}
|
|
5639
|
+
function metadataPublishingAnalysis(snapshot, pending) {
|
|
5640
|
+
const { fields, settings } = fieldsOf(snapshot);
|
|
5641
|
+
const {
|
|
5642
|
+
pendingAnalysis: _pending,
|
|
5643
|
+
analysisFailure: _failure,
|
|
5644
|
+
tokenizer: _tokenizer,
|
|
5645
|
+
stopWords: _stopWords,
|
|
5646
|
+
stopWordList: _stopWordList,
|
|
5647
|
+
...rest
|
|
5648
|
+
} = settings;
|
|
5649
|
+
return encoded(fields, {
|
|
5650
|
+
...rest,
|
|
5651
|
+
language: pending.language,
|
|
5652
|
+
analysisGeneration: pending.generation,
|
|
5653
|
+
...pending.tokenizer !== void 0 ? { tokenizer: pending.tokenizer } : {},
|
|
5654
|
+
...pending.stopWords !== void 0 ? { stopWords: pending.stopWords } : {},
|
|
5655
|
+
...pending.stopWordList !== void 0 ? { stopWordList: pending.stopWordList } : {}
|
|
5656
|
+
});
|
|
5657
|
+
}
|
|
5658
|
+
function metadataWithAnalysisFailure(snapshot, failure) {
|
|
5659
|
+
const { fields, settings } = fieldsOf(snapshot);
|
|
5660
|
+
const { pendingAnalysis: _pending, ...rest } = settings;
|
|
5661
|
+
return encoded(fields, { ...rest, analysisFailure: failure });
|
|
5662
|
+
}
|
|
5663
|
+
|
|
5664
|
+
// src/distribution/cluster/node-targets.ts
|
|
5665
|
+
function targetsOfRegisteredNode(nodeId, address) {
|
|
5666
|
+
return address.length === 0 || address === nodeId ? [nodeId] : [address, nodeId];
|
|
5667
|
+
}
|
|
5668
|
+
|
|
5669
|
+
// src/distribution/cluster/rebalance/pause.ts
|
|
5670
|
+
function pause(milliseconds) {
|
|
5671
|
+
return new Promise((resolve) => {
|
|
5672
|
+
const timer = setTimeout(resolve, milliseconds);
|
|
5673
|
+
timer.unref?.();
|
|
5674
|
+
});
|
|
5675
|
+
}
|
|
5676
|
+
|
|
5677
|
+
// src/distribution/cluster/controller/event-loop/analysis-change.ts
|
|
5678
|
+
var REFUSALS_THAT_FAIL_A_CHANGE = /* @__PURE__ */ new Set([
|
|
5679
|
+
ErrorCodes.ANALYSIS_REBUILD_MEMORY_EXCEEDED,
|
|
5680
|
+
ErrorCodes.LANGUAGE_NOT_SUPPORTED,
|
|
5681
|
+
ErrorCodes.CONFIG_INVALID
|
|
5682
|
+
]);
|
|
5683
|
+
async function registeredHolders(coordinator, indexName) {
|
|
5684
|
+
const table = await coordinator.getAllocation(indexName);
|
|
5685
|
+
if (table === null) return null;
|
|
5686
|
+
const registered = new Map((await coordinator.listNodes()).map((node) => [node.nodeId, node.address]));
|
|
5687
|
+
const holders = /* @__PURE__ */ new Map();
|
|
5688
|
+
const addHolder = (nodeId) => {
|
|
5689
|
+
if (nodeId === null) return;
|
|
5690
|
+
const address = registered.get(nodeId);
|
|
5691
|
+
if (address === void 0) return;
|
|
5692
|
+
holders.set(nodeId, targetsOfRegisteredNode(nodeId, address));
|
|
5693
|
+
};
|
|
5694
|
+
for (const assignment of table.assignments.values()) {
|
|
5695
|
+
addHolder(assignment.primary);
|
|
5696
|
+
for (const replica of assignment.replicas) addHolder(replica);
|
|
5697
|
+
for (const replica of assignment.removingReplicas ?? []) addHolder(replica);
|
|
5698
|
+
}
|
|
5699
|
+
return holders;
|
|
5700
|
+
}
|
|
5701
|
+
async function sendRebuild(ctx, targets, payload) {
|
|
5702
|
+
const transport = ctx.state.transport;
|
|
5703
|
+
if (transport === null) throw new NarsilError(ErrorCodes.NODE_NOT_CONTROLLER, "The controller has no transport");
|
|
5704
|
+
const message = {
|
|
5705
|
+
type: ClusterMessageTypes.REBUILD_ANALYSIS,
|
|
5706
|
+
sourceId: ctx.nodeId,
|
|
5707
|
+
requestId: generateId(),
|
|
5708
|
+
payload: encode(payload)
|
|
5709
|
+
};
|
|
5710
|
+
const response = await sendToFirstReachableTarget2(transport, targets, message);
|
|
5711
|
+
const decoded = decode(response.payload);
|
|
5712
|
+
if (!response.type.endsWith(".error") && !(isRecord(decoded) && decoded.error === true)) return;
|
|
5713
|
+
const reply2 = isRecord(decoded) ? decoded : {};
|
|
5714
|
+
throw new NarsilError(
|
|
5715
|
+
typeof reply2.code === "string" ? reply2.code : ErrorCodes.QUERY_ROUTING_FAILED,
|
|
5716
|
+
typeof reply2.message === "string" ? reply2.message : "The node refused an analysis rebuild"
|
|
5717
|
+
);
|
|
5718
|
+
}
|
|
5719
|
+
function askHolder(ctx, progress, holder, targets, payload) {
|
|
5720
|
+
progress.asking.add(holder);
|
|
5721
|
+
const resend = new Promise((resolve) => {
|
|
5722
|
+
const timer = setTimeout(() => resolve("resend"), ANALYSIS_REBUILD_RESEND_MS);
|
|
5723
|
+
if (typeof timer === "object" && "unref" in timer) timer.unref();
|
|
5724
|
+
});
|
|
5725
|
+
const asked = sendRebuild(ctx, targets, payload).then(
|
|
5726
|
+
() => {
|
|
5727
|
+
progress.answered.add(holder);
|
|
5728
|
+
},
|
|
5729
|
+
(error) => {
|
|
5730
|
+
if (error instanceof NarsilError && REFUSALS_THAT_FAIL_A_CHANGE.has(error.code)) {
|
|
5731
|
+
progress.refusal = {
|
|
5732
|
+
changeId: progress.changeId,
|
|
5733
|
+
code: error.code,
|
|
5734
|
+
message: describeError(error),
|
|
5735
|
+
nodeId: holder
|
|
5736
|
+
};
|
|
5737
|
+
}
|
|
5738
|
+
}
|
|
5739
|
+
);
|
|
5740
|
+
void Promise.race([asked, resend]).finally(() => progress.asking.delete(holder));
|
|
5741
|
+
}
|
|
5742
|
+
async function stepChange(ctx, indexName, snapshot, pending, progress) {
|
|
5743
|
+
const key = indexConfigKey(indexName);
|
|
5744
|
+
if (progress.refusal !== null) {
|
|
5745
|
+
await ctx.coordinator.compareAndSet(key, snapshot.bytes, metadataWithAnalysisFailure(snapshot, progress.refusal));
|
|
5746
|
+
return;
|
|
5747
|
+
}
|
|
5748
|
+
const holders = await registeredHolders(ctx.coordinator, indexName) ?? /* @__PURE__ */ new Map();
|
|
5749
|
+
const payload = {
|
|
5750
|
+
indexName,
|
|
5751
|
+
indexUuid: snapshot.record.indexUuid,
|
|
5752
|
+
generation: pending.generation
|
|
5753
|
+
};
|
|
5754
|
+
let waiting = false;
|
|
5755
|
+
for (const [holder, targets] of holders) {
|
|
5756
|
+
if (progress.answered.has(holder)) continue;
|
|
5757
|
+
waiting = true;
|
|
5758
|
+
if (!progress.asking.has(holder)) askHolder(ctx, progress, holder, targets, payload);
|
|
5759
|
+
}
|
|
5760
|
+
if (!waiting) {
|
|
5761
|
+
await ctx.coordinator.compareAndSet(key, snapshot.bytes, metadataPublishingAnalysis(snapshot, pending));
|
|
5762
|
+
return;
|
|
5763
|
+
}
|
|
5764
|
+
await pause(ANALYSIS_CHANGE_POLL_INTERVAL_MS);
|
|
5765
|
+
}
|
|
5766
|
+
function progressOf(changeId) {
|
|
5767
|
+
return { changeId, answered: /* @__PURE__ */ new Set(), asking: /* @__PURE__ */ new Set(), refusal: null };
|
|
5768
|
+
}
|
|
5769
|
+
async function driveChanges(ctx, indexName) {
|
|
5770
|
+
let progress = progressOf("");
|
|
5771
|
+
while (ctx.isActive()) {
|
|
5772
|
+
const snapshot = await readIndexMetadataSnapshot(ctx.coordinator, indexName);
|
|
5773
|
+
const pending = snapshot?.settings?.pendingAnalysis;
|
|
5774
|
+
if (snapshot === null || pending === void 0) return;
|
|
5775
|
+
if (progress.changeId !== pending.changeId) progress = progressOf(pending.changeId);
|
|
5776
|
+
await stepChange(ctx, indexName, snapshot, pending, progress);
|
|
5777
|
+
}
|
|
5778
|
+
}
|
|
5779
|
+
function startAnalysisDriver(ctx, indexName) {
|
|
5780
|
+
const drivers = ctx.state.analysisDrivers;
|
|
5781
|
+
const running = drivers.get(indexName);
|
|
5782
|
+
if (running !== void 0) {
|
|
5783
|
+
running.again = true;
|
|
5784
|
+
return;
|
|
5785
|
+
}
|
|
5786
|
+
const driver = { again: false };
|
|
5787
|
+
drivers.set(indexName, driver);
|
|
5788
|
+
const drive2 = async () => {
|
|
5789
|
+
do {
|
|
5790
|
+
driver.again = false;
|
|
5791
|
+
await driveChanges(ctx, indexName).catch(async (error) => {
|
|
5792
|
+
ctx.onError?.(indexName, error);
|
|
5793
|
+
driver.again = true;
|
|
5794
|
+
await pause(ANALYSIS_CHANGE_RETRY_MS);
|
|
5795
|
+
});
|
|
5796
|
+
} while (driver.again && ctx.isActive());
|
|
5797
|
+
if (drivers.get(indexName) === driver) drivers.delete(indexName);
|
|
5798
|
+
};
|
|
5799
|
+
void drive2();
|
|
5800
|
+
}
|
|
5801
|
+
async function resumeAnalysisDrivers(ctx) {
|
|
5802
|
+
for (const indexName of await ctx.coordinator.listSchemas()) {
|
|
5803
|
+
const snapshot = await readIndexMetadataSnapshot(ctx.coordinator, indexName);
|
|
5804
|
+
if (snapshot?.settings?.pendingAnalysis !== void 0) startAnalysisDriver(ctx, indexName);
|
|
5805
|
+
}
|
|
5806
|
+
}
|
|
5439
5807
|
function validateBootstrapCompletePayload(decoded) {
|
|
5440
5808
|
if (!isRecord(decoded)) {
|
|
5441
5809
|
return null;
|
|
@@ -5967,7 +6335,10 @@ async function dropGeneration(coordinator, writerNodeId, generation) {
|
|
|
5967
6335
|
const key = indexConfigKey(generation);
|
|
5968
6336
|
const metadata = await coordinator.get(key);
|
|
5969
6337
|
if (metadata !== null && metadata.byteLength > 0) {
|
|
5970
|
-
await coordinator.compareAndSet(key, metadata, new Uint8Array(0))
|
|
6338
|
+
if (await coordinator.compareAndSet(key, metadata, new Uint8Array(0))) {
|
|
6339
|
+
const stored = storedSynonymSetOf(metadata);
|
|
6340
|
+
if (stored !== null) await deleteSynonymSet(coordinator, stored.indexUuid, stored.ref);
|
|
6341
|
+
}
|
|
5971
6342
|
}
|
|
5972
6343
|
if (await coordinator.getSchema(generation) !== null) {
|
|
5973
6344
|
await coordinator.dropSchema(generation);
|
|
@@ -5975,20 +6346,29 @@ async function dropGeneration(coordinator, writerNodeId, generation) {
|
|
|
5975
6346
|
}
|
|
5976
6347
|
await coordinator.deleteAllocation(generation);
|
|
5977
6348
|
}
|
|
6349
|
+
async function dropReplacedNameSynonymSet(coordinator, indexName) {
|
|
6350
|
+
for (let attempt = 0; attempt < TEARDOWN_CAS_ATTEMPTS; attempt++) {
|
|
6351
|
+
const snapshot = await readIndexMetadataSnapshot(coordinator, indexName);
|
|
6352
|
+
const ref = snapshot?.settings?.synonyms;
|
|
6353
|
+
if (snapshot === null || ref === void 0) return;
|
|
6354
|
+
const next = metadataWithSynonymSet(snapshot, null);
|
|
6355
|
+
if (await coordinator.compareAndSet(indexConfigKey(indexName), snapshot.bytes, next)) {
|
|
6356
|
+
await deleteSynonymSet(coordinator, snapshot.record.indexUuid, ref);
|
|
6357
|
+
return;
|
|
6358
|
+
}
|
|
6359
|
+
}
|
|
6360
|
+
throw new NarsilError(
|
|
6361
|
+
ErrorCodes.ALLOCATION_FAILED,
|
|
6362
|
+
`Releasing the replaced synonym set of index '${indexName}' lost every compare-and-set attempt`,
|
|
6363
|
+
{ indexName, attempts: TEARDOWN_CAS_ATTEMPTS }
|
|
6364
|
+
);
|
|
6365
|
+
}
|
|
5978
6366
|
async function clearEveryProgress(coordinator, indexName, partitionCount) {
|
|
5979
6367
|
for (let partitionId = 0; partitionId < partitionCount; partitionId++) {
|
|
5980
6368
|
await clearRebalanceProgress(coordinator, indexName, partitionId);
|
|
5981
6369
|
}
|
|
5982
6370
|
}
|
|
5983
6371
|
|
|
5984
|
-
// src/distribution/cluster/rebalance/pause.ts
|
|
5985
|
-
function pause(milliseconds) {
|
|
5986
|
-
return new Promise((resolve) => {
|
|
5987
|
-
const timer = setTimeout(resolve, milliseconds);
|
|
5988
|
-
timer.unref?.();
|
|
5989
|
-
});
|
|
5990
|
-
}
|
|
5991
|
-
|
|
5992
6372
|
// src/distribution/cluster/controller/event-loop/rebalance/primaries.ts
|
|
5993
6373
|
async function targetsOf(ctx, nodeId) {
|
|
5994
6374
|
const nodes = await ctx.coordinator.listNodes();
|
|
@@ -6204,6 +6584,7 @@ async function finishSwap(ctx, indexName, record) {
|
|
|
6204
6584
|
if (record.sourceGeneration === indexName) {
|
|
6205
6585
|
const metadata = await getIndexMetadata(ctx.coordinator, indexName);
|
|
6206
6586
|
await clearEveryProgress(ctx.coordinator, indexName, metadata?.partitionCount ?? 0);
|
|
6587
|
+
await dropReplacedNameSynonymSet(ctx.coordinator, indexName);
|
|
6207
6588
|
} else {
|
|
6208
6589
|
const source = await ctx.coordinator.getAllocation(record.sourceGeneration);
|
|
6209
6590
|
await clearEveryProgress(ctx.coordinator, indexName, source?.assignments.size ?? 0);
|
|
@@ -6365,6 +6746,15 @@ async function recordRebalance(ctx, message) {
|
|
|
6365
6746
|
{ indexName, jobId: running?.jobId ?? null }
|
|
6366
6747
|
);
|
|
6367
6748
|
}
|
|
6749
|
+
const latest = await readIndexMetadataSnapshot(ctx.coordinator, sourceGeneration);
|
|
6750
|
+
if (latest?.settings?.pendingAnalysis !== void 0) {
|
|
6751
|
+
await updateRebalanceRecord(ctx.coordinator, indexName, record.jobId, () => null);
|
|
6752
|
+
throw new NarsilError(
|
|
6753
|
+
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
6754
|
+
`Index '${indexName}' is changing its analysis, so try its partition count change again once that ends`,
|
|
6755
|
+
{ indexName }
|
|
6756
|
+
);
|
|
6757
|
+
}
|
|
6368
6758
|
startRebalanceDriver(ctx, indexName, record.jobId);
|
|
6369
6759
|
const result = {
|
|
6370
6760
|
indexName,
|
|
@@ -6492,6 +6882,11 @@ async function startEventLoop(state, coordinator, transport, nodeId, isActive, o
|
|
|
6492
6882
|
});
|
|
6493
6883
|
void resumeRebalance({ state, coordinator, nodeId, isActive, onError }, event.indexName);
|
|
6494
6884
|
});
|
|
6885
|
+
state.unwatchMetadata = await coordinator.watchIndexMetadata((event) => {
|
|
6886
|
+
if (event.metadata !== null && isActive()) {
|
|
6887
|
+
startAnalysisDriver({ state, coordinator, nodeId, isActive, onError }, event.indexName);
|
|
6888
|
+
}
|
|
6889
|
+
});
|
|
6495
6890
|
state.unwatchTransport = await transport.listen((message, respond) => {
|
|
6496
6891
|
if (message.type === ClusterMessageTypes.REPLICATION_FACTOR) {
|
|
6497
6892
|
handleReplicationFactorMessage(state, message, respond, coordinator, nodeId, isActive, onError);
|
|
@@ -6520,6 +6915,7 @@ async function startEventLoop(state, coordinator, transport, nodeId, isActive, o
|
|
|
6520
6915
|
}
|
|
6521
6916
|
scheduleDebouncedAllocation(state, coordinator, isActive, onError);
|
|
6522
6917
|
resumeRebalances({ state, coordinator, nodeId, isActive, onError }).catch((error) => onError?.("", error));
|
|
6918
|
+
resumeAnalysisDrivers({ state, coordinator, nodeId, isActive, onError }).catch((error) => onError?.("", error));
|
|
6523
6919
|
}
|
|
6524
6920
|
|
|
6525
6921
|
// src/distribution/cluster/controller/index.ts
|
|
@@ -7248,6 +7644,35 @@ async function servingNameOf(coordinator, indexName) {
|
|
|
7248
7644
|
const table = await coordinator.getAllocation(indexName);
|
|
7249
7645
|
return table?.generation ?? indexName;
|
|
7250
7646
|
}
|
|
7647
|
+
var CODES_OF_A_LOST_LAYOUT = /* @__PURE__ */ new Set([
|
|
7648
|
+
ErrorCodes.INDEX_NOT_FOUND,
|
|
7649
|
+
ErrorCodes.QUERY_NO_ACTIVE_REPLICA,
|
|
7650
|
+
ErrorCodes.QUERY_NODE_TIMEOUT,
|
|
7651
|
+
ErrorCodes.QUERY_PARTIAL_FAILURE,
|
|
7652
|
+
ErrorCodes.QUERY_ROUTING_FAILED
|
|
7653
|
+
]);
|
|
7654
|
+
function leftOutAPartition(coverage) {
|
|
7655
|
+
return coverage.totalPartitions === 0 || coverage.queriedPartitions < coverage.totalPartitions;
|
|
7656
|
+
}
|
|
7657
|
+
async function movedNameOf(names, indexName, servingName) {
|
|
7658
|
+
const live = await names.liveServingName(indexName).catch(() => servingName);
|
|
7659
|
+
return live === servingName ? null : live;
|
|
7660
|
+
}
|
|
7661
|
+
async function readFollowingTheName(names, indexName, read, answerIsPartial) {
|
|
7662
|
+
const servingName = await names.servingName(indexName);
|
|
7663
|
+
let answer;
|
|
7664
|
+
try {
|
|
7665
|
+
answer = await read(servingName);
|
|
7666
|
+
} catch (error) {
|
|
7667
|
+
if (!(error instanceof NarsilError) || !CODES_OF_A_LOST_LAYOUT.has(error.code)) throw error;
|
|
7668
|
+
const moved2 = await movedNameOf(names, indexName, servingName);
|
|
7669
|
+
if (moved2 === null) throw error;
|
|
7670
|
+
return read(moved2);
|
|
7671
|
+
}
|
|
7672
|
+
if (answerIsPartial === void 0 || !answerIsPartial(answer)) return answer;
|
|
7673
|
+
const moved = await movedNameOf(names, indexName, servingName);
|
|
7674
|
+
return moved === null ? answer : read(moved);
|
|
7675
|
+
}
|
|
7251
7676
|
|
|
7252
7677
|
// src/persistence/durability/partition-log.ts
|
|
7253
7678
|
function persistedPosition(manager, indexName, partitionId) {
|
|
@@ -7916,6 +8341,30 @@ async function archiveReplicationPartition(core, indexName, partitionId, lastSeq
|
|
|
7916
8341
|
const manager = core.requireManager(indexName);
|
|
7917
8342
|
let captured = 0;
|
|
7918
8343
|
const config = await encodeIndexConfiguration(entry);
|
|
8344
|
+
const durable = core.durability?.manager;
|
|
8345
|
+
if (entry.config.textStorage === "disk" && durable?.openArchiveCheckpoint !== void 0) {
|
|
8346
|
+
const checkpoint2 = await durable.openArchiveCheckpoint(indexName);
|
|
8347
|
+
const { directory } = checkpoint2;
|
|
8348
|
+
if (directory === void 0) {
|
|
8349
|
+
checkpoint2.release();
|
|
8350
|
+
throw new NarsilError(
|
|
8351
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
8352
|
+
`Index "${indexName}" stores its text on disk, which needs a durable directory`,
|
|
8353
|
+
{ indexName }
|
|
8354
|
+
);
|
|
8355
|
+
}
|
|
8356
|
+
try {
|
|
8357
|
+
const { openPagedPartitionArchive } = await import('./paged-partition-archive-HFP45PLU.mjs');
|
|
8358
|
+
const archive = await openPagedPartitionArchive(checkpoint2, directory, indexName, partitionId, manager);
|
|
8359
|
+
return {
|
|
8360
|
+
stream: archiveStream({ ...archive, config, sourceIndexName: indexName }),
|
|
8361
|
+
lastSeqNo: archive.lastSeqNo
|
|
8362
|
+
};
|
|
8363
|
+
} catch (error) {
|
|
8364
|
+
checkpoint2.release();
|
|
8365
|
+
throw error;
|
|
8366
|
+
}
|
|
8367
|
+
}
|
|
7919
8368
|
const checkpoint = await buildArchiveFromMemory({
|
|
7920
8369
|
indexName,
|
|
7921
8370
|
metadata: indexMetadataOf(indexName, entry, manager.partitionCount),
|
|
@@ -7950,9 +8399,13 @@ function requireRestoreTarget(indexName, partitionId, partitionCount) {
|
|
|
7950
8399
|
restoreFailed("partitionId is outside partitionCount", { indexName, partitionId, partitionCount });
|
|
7951
8400
|
}
|
|
7952
8401
|
}
|
|
7953
|
-
async function receiveArchive(indexName, partitionId, source, schema) {
|
|
7954
|
-
const store = memoryCheckpointStore();
|
|
8402
|
+
async function receiveArchive(indexName, partitionId, source, schema, directory) {
|
|
8403
|
+
const store = directory === null ? memoryCheckpointStore() : directoryCheckpointStore(directory);
|
|
8404
|
+
const receivedPrefix = directory === null ? `${indexName}/` : `${indexName}/received/p${partitionId}/${generateId()}/`;
|
|
7955
8405
|
let language = "";
|
|
8406
|
+
let tokenizer;
|
|
8407
|
+
let stopWords;
|
|
8408
|
+
let stopWordList;
|
|
7956
8409
|
let analysisRevision = null;
|
|
7957
8410
|
let arrayPositionGap = null;
|
|
7958
8411
|
let received = null;
|
|
@@ -7966,6 +8419,9 @@ async function receiveArchive(indexName, partitionId, source, schema) {
|
|
|
7966
8419
|
const raw = decode(payload);
|
|
7967
8420
|
if (typeof raw.language !== "string") restoreFailed("partition archive names no language", { indexName });
|
|
7968
8421
|
language = raw.language;
|
|
8422
|
+
tokenizer = raw.tokenizer;
|
|
8423
|
+
stopWords = raw.stopWords;
|
|
8424
|
+
stopWordList = raw.stopWordList;
|
|
7969
8425
|
analysisRevision = typeof raw.analysisRevision === "string" ? raw.analysisRevision : null;
|
|
7970
8426
|
arrayPositionGap = storedArrayPositionGap(raw.arrayPositionGap);
|
|
7971
8427
|
},
|
|
@@ -7978,7 +8434,7 @@ async function receiveArchive(indexName, partitionId, source, schema) {
|
|
|
7978
8434
|
restoreFailed("partition archive holds a graph", { indexName, partitionId });
|
|
7979
8435
|
}
|
|
7980
8436
|
requireSameFlatSchema(indexName, schema, manifest.schema, "partition snapshot");
|
|
7981
|
-
const rename = (key) => `${
|
|
8437
|
+
const rename = (key) => `${receivedPrefix}${key.slice(sourceName.length + 1)}`;
|
|
7982
8438
|
received = {
|
|
7983
8439
|
...manifest,
|
|
7984
8440
|
partitions: manifest.partitions.map((partition) => ({
|
|
@@ -7991,10 +8447,28 @@ async function receiveArchive(indexName, partitionId, source, schema) {
|
|
|
7991
8447
|
}))
|
|
7992
8448
|
};
|
|
7993
8449
|
},
|
|
7994
|
-
file:
|
|
8450
|
+
file: (name, _ref, chunks) => writeArchiveFile(store, `${receivedPrefix}${name}`, chunks)
|
|
7995
8451
|
});
|
|
7996
8452
|
if (received === null) restoreFailed("partition archive holds no manifest", { indexName, partitionId });
|
|
7997
|
-
return {
|
|
8453
|
+
return {
|
|
8454
|
+
language,
|
|
8455
|
+
tokenizer,
|
|
8456
|
+
stopWords,
|
|
8457
|
+
stopWordList,
|
|
8458
|
+
analysisRevision,
|
|
8459
|
+
arrayPositionGap,
|
|
8460
|
+
manifest: received,
|
|
8461
|
+
store,
|
|
8462
|
+
receivedPrefix
|
|
8463
|
+
};
|
|
8464
|
+
}
|
|
8465
|
+
function requireArchiveUnderLocalAnalysis(core, indexName, archive) {
|
|
8466
|
+
const entry = core.requireIndex(indexName);
|
|
8467
|
+
const local = namedSettingsOf(entry.config, entry.language.name);
|
|
8468
|
+
const sameList2 = JSON.stringify(local.stopWordList) === JSON.stringify(archive.stopWordList);
|
|
8469
|
+
if (local.tokenizer !== archive.tokenizer || local.stopWords !== archive.stopWords || !sameList2) {
|
|
8470
|
+
restoreFailed("partition archive holds terms from other analysis settings, so the copy tries again", { indexName });
|
|
8471
|
+
}
|
|
7998
8472
|
}
|
|
7999
8473
|
async function replacePartition(manager, partitionId, archive, writeRecords) {
|
|
8000
8474
|
const vectorIndexes = manager.getVectorIndexes();
|
|
@@ -8058,7 +8532,8 @@ async function restoreReplicationPartition(core, engine, indexName, partitionId,
|
|
|
8058
8532
|
const { config: indexConfig, partitionCount, writeRecords } = target;
|
|
8059
8533
|
core.guardShutdown();
|
|
8060
8534
|
requireRestoreTarget(indexName, partitionId, partitionCount);
|
|
8061
|
-
const
|
|
8535
|
+
const directory = indexConfig.textStorage === "disk" ? core.durability?.manager.partitionRegistry?.directory ?? null : null;
|
|
8536
|
+
const archive = await receiveArchive(indexName, partitionId, source, flattenSchema(indexConfig.schema), directory);
|
|
8062
8537
|
let release = core.indexRegistry.has(indexName) ? await core.indexState.acquire(indexName, false) : null;
|
|
8063
8538
|
try {
|
|
8064
8539
|
const restoreIndex = await ensureReplicationIndex(
|
|
@@ -8070,6 +8545,7 @@ async function restoreReplicationPartition(core, engine, indexName, partitionId,
|
|
|
8070
8545
|
partitionCount
|
|
8071
8546
|
);
|
|
8072
8547
|
if (release === null) release = await core.indexState.acquire(indexName, false);
|
|
8548
|
+
requireArchiveUnderLocalAnalysis(core, indexName, archive);
|
|
8073
8549
|
if (restoreIndex.created) {
|
|
8074
8550
|
const synonyms = core.requireIndex(indexName).synonyms;
|
|
8075
8551
|
await core.orchestrator.replicateToWorkers({
|
|
@@ -8084,6 +8560,17 @@ async function restoreReplicationPartition(core, engine, indexName, partitionId,
|
|
|
8084
8560
|
try {
|
|
8085
8561
|
await replacePartition(core.requireManager(indexName), partitionId, archive, writeRecords);
|
|
8086
8562
|
recordReceivedAnalysis(core, indexName, archive.analysisRevision, archive.arrayPositionGap);
|
|
8563
|
+
if (directory !== null) {
|
|
8564
|
+
const durable = core.durability?.manager;
|
|
8565
|
+
await (durable?.checkpointFromMemory?.(indexName) ?? durable?.checkpoint(indexName));
|
|
8566
|
+
const { removeReceivedFilesNothingReads } = await import('./paged-partition-archive-HFP45PLU.mjs');
|
|
8567
|
+
await removeReceivedFilesNothingReads(
|
|
8568
|
+
directory,
|
|
8569
|
+
core.requireManager(indexName),
|
|
8570
|
+
partitionId,
|
|
8571
|
+
archive.receivedPrefix
|
|
8572
|
+
);
|
|
8573
|
+
}
|
|
8087
8574
|
} finally {
|
|
8088
8575
|
await core.orchestrator.resyncIndex(indexName, wasScaledOut);
|
|
8089
8576
|
}
|
|
@@ -8292,7 +8779,7 @@ function createClusterWriteRecords() {
|
|
|
8292
8779
|
}
|
|
8293
8780
|
let held = partitions.get(partitionId);
|
|
8294
8781
|
if (held === void 0) {
|
|
8295
|
-
held = { table: createWriteRecordTable(), history: createTermHistory() };
|
|
8782
|
+
held = { table: createWriteRecordTable(), history: createTermHistory(), pagedSegments: [] };
|
|
8296
8783
|
partitions.set(partitionId, held);
|
|
8297
8784
|
}
|
|
8298
8785
|
return held;
|
|
@@ -8303,15 +8790,29 @@ function createClusterWriteRecords() {
|
|
|
8303
8790
|
else held.table.remove(entry.documentId);
|
|
8304
8791
|
held.history.noteEntry(entry.seqNo, entry.primaryTerm);
|
|
8305
8792
|
}
|
|
8306
|
-
function
|
|
8793
|
+
function recordOfDocument(indexName, partitionId, documentId) {
|
|
8794
|
+
const held = indexes.get(indexName)?.get(partitionId);
|
|
8795
|
+
if (held === void 0) return null;
|
|
8796
|
+
const record = held.table.recordOf(documentId);
|
|
8797
|
+
if (record !== null) return record;
|
|
8798
|
+
for (let at = held.pagedSegments.length - 1; at >= 0; at--) {
|
|
8799
|
+
const segment = held.pagedSegments[at];
|
|
8800
|
+
if (!segment.hasDocument(documentId) || segment.paged === null) continue;
|
|
8801
|
+
const stored = segment.paged.writeRecordAt(segment.arrays.idTable.ordinalOf(documentId));
|
|
8802
|
+
return stored.seqNo === 0 ? null : stored;
|
|
8803
|
+
}
|
|
8804
|
+
return null;
|
|
8805
|
+
}
|
|
8806
|
+
function captureSegment(indexName, partitionId, segment) {
|
|
8307
8807
|
const count2 = segment.arrays.documentCount;
|
|
8308
8808
|
const seqNos = new Float64Array(count2);
|
|
8309
8809
|
const primaryTerms = new Float64Array(count2);
|
|
8310
8810
|
for (let ordinal = 0; ordinal < count2; ordinal++) {
|
|
8311
8811
|
if (segment.isTombstoned(ordinal)) continue;
|
|
8312
|
-
const
|
|
8313
|
-
|
|
8314
|
-
|
|
8812
|
+
const record = recordOfDocument(indexName, partitionId, segment.arrays.idTable.idAt(ordinal));
|
|
8813
|
+
if (record === null) continue;
|
|
8814
|
+
seqNos[ordinal] = record.seqNo;
|
|
8815
|
+
primaryTerms[ordinal] = record.primaryTerm;
|
|
8315
8816
|
}
|
|
8316
8817
|
return { seqNos, primaryTerms };
|
|
8317
8818
|
}
|
|
@@ -8322,9 +8823,7 @@ function createClusterWriteRecords() {
|
|
|
8322
8823
|
applied,
|
|
8323
8824
|
replayed: applied,
|
|
8324
8825
|
partition,
|
|
8325
|
-
recordOf
|
|
8326
|
-
return indexes.get(indexName)?.get(partitionId)?.table.recordOf(documentId) ?? null;
|
|
8327
|
-
},
|
|
8826
|
+
recordOf: recordOfDocument,
|
|
8328
8827
|
heldEntry(indexName, partitionId, seqNo, primaryTerm) {
|
|
8329
8828
|
const history = indexes.get(indexName)?.get(partitionId)?.history;
|
|
8330
8829
|
if (history === void 0) return "uncovered";
|
|
@@ -8338,6 +8837,23 @@ function createClusterWriteRecords() {
|
|
|
8338
8837
|
if (held === void 0) return;
|
|
8339
8838
|
held.table.clear();
|
|
8340
8839
|
held.history.clear();
|
|
8840
|
+
held.pagedSegments = [];
|
|
8841
|
+
},
|
|
8842
|
+
loadPagedSegment(indexName, partitionId, segment) {
|
|
8843
|
+
const held = partition(indexName, partitionId);
|
|
8844
|
+
for (const documentId of segment.paged?.snapshot.tombstones ?? []) held.table.remove(documentId);
|
|
8845
|
+
held.pagedSegments = [...held.pagedSegments, segment];
|
|
8846
|
+
},
|
|
8847
|
+
adoptPagedSegments(indexName, partitionId, segments, changed) {
|
|
8848
|
+
const held = partition(indexName, partitionId);
|
|
8849
|
+
const kept = [];
|
|
8850
|
+
for (const documentId of changed) {
|
|
8851
|
+
const record = held.table.recordOf(documentId);
|
|
8852
|
+
if (record !== null) kept.push([documentId, record]);
|
|
8853
|
+
}
|
|
8854
|
+
held.table.clear();
|
|
8855
|
+
for (const [documentId, record] of kept) held.table.set(documentId, record.seqNo, record.primaryTerm);
|
|
8856
|
+
held.pagedSegments = segments;
|
|
8341
8857
|
},
|
|
8342
8858
|
loadSegment(indexName, partitionId, segment, contents) {
|
|
8343
8859
|
const { table } = partition(indexName, partitionId);
|
|
@@ -8352,9 +8868,10 @@ function createClusterWriteRecords() {
|
|
|
8352
8868
|
},
|
|
8353
8869
|
captureSegments(indexName, partitionId, segments) {
|
|
8354
8870
|
const captured = /* @__PURE__ */ new Map();
|
|
8355
|
-
|
|
8356
|
-
|
|
8357
|
-
|
|
8871
|
+
if (indexes.get(indexName)?.get(partitionId) === void 0) return captured;
|
|
8872
|
+
for (const segment of segments) {
|
|
8873
|
+
if (segment.paged === null) captured.set(segment, captureSegment(indexName, partitionId, segment));
|
|
8874
|
+
}
|
|
8358
8875
|
return captured;
|
|
8359
8876
|
},
|
|
8360
8877
|
termHistoryOf(indexName, partitionId, lastSeqNo) {
|
|
@@ -8422,6 +8939,14 @@ async function createClusterLocalEngine(config, hooks) {
|
|
|
8422
8939
|
stampIndexUuid: (indexName, indexUuid) => stampIndexUuid(core, indexName, indexUuid),
|
|
8423
8940
|
applyStoredSynonyms: (indexName, load) => synonymModule().applyLoaded(core, indexName, load),
|
|
8424
8941
|
applyStoredBm25: (indexName, load) => applyStoredIndexBm25(core, indexName, load),
|
|
8942
|
+
analysisGenerationOf: (indexName) => core.indexRegistry.get(indexName)?.analysisGeneration,
|
|
8943
|
+
pendingAnalysisGenerationOf: (indexName) => core.indexRegistry.get(indexName)?.pendingAnalysis?.generation ?? null,
|
|
8944
|
+
buildHeldAnalysis: (indexName, pending) => buildHeldAnalysisChange(core, indexName, pending),
|
|
8945
|
+
heldAnalysisIsBuilt: (indexName, generation) => heldAnalysisIsBuilt(core, indexName, generation),
|
|
8946
|
+
releaseHeldAnalysis: (indexName, generation) => releaseHeldAnalysisChange(core, indexName, generation),
|
|
8947
|
+
dropHeldAnalysis: (indexName) => dropHeldAnalysisChange(core, indexName),
|
|
8948
|
+
holdsAnalysisSettings: (indexName, settings) => holdsNamedSettings(core.requireIndex(indexName), settings),
|
|
8949
|
+
stampAnalysisGeneration: (indexName, generation) => stampAnalysisGeneration(core, indexName, generation),
|
|
8425
8950
|
highestPersistedSeqNoOf: (indexName, partitionId) => core.durability?.manager.highestPersistedSeqNo(indexName, partitionId) ?? 0,
|
|
8426
8951
|
persistedPositionOf: (indexName, partitionId) => persistedPositionOf(core, indexName, partitionId),
|
|
8427
8952
|
numberEntriesWith: (numbering, defersCommit) => core.durability?.manager.numberEntriesWith?.(numbering, defersCommit),
|
|
@@ -8610,6 +9135,15 @@ function validateStringField(value, fieldLabel, maxLength, errorCode) {
|
|
|
8610
9135
|
function validateFieldName(value, fieldLabel, errorCode) {
|
|
8611
9136
|
return validateStringField(value, fieldLabel, MAX_FIELD_NAME_LENGTH, errorCode);
|
|
8612
9137
|
}
|
|
9138
|
+
function validateAnalysisGeneration(value, payloadName) {
|
|
9139
|
+
if (value === void 0 || value === null) return;
|
|
9140
|
+
if (!Number.isSafeInteger(value) || value < 0) {
|
|
9141
|
+
throwInvalid(
|
|
9142
|
+
ErrorCodes.CONFIG_INVALID,
|
|
9143
|
+
`Invalid ${payloadName}: "analysisGeneration" must be a non-negative integer`
|
|
9144
|
+
);
|
|
9145
|
+
}
|
|
9146
|
+
}
|
|
8613
9147
|
function validateNonNegativeInteger(value, fieldLabel, maxValue, errorCode) {
|
|
8614
9148
|
if (!isInteger(value)) {
|
|
8615
9149
|
throwInvalid(errorCode, `Invalid payload: "${fieldLabel}" must be a finite non-negative integer`);
|
|
@@ -9506,6 +10040,7 @@ function validateParams(params) {
|
|
|
9506
10040
|
}
|
|
9507
10041
|
if (params.bm25 !== void 0 && params.bm25 !== null) validateBM25Shape(params.bm25, "settable");
|
|
9508
10042
|
if (params.indexBm25 !== void 0 && params.indexBm25 !== null) validateBM25Shape(params.indexBm25, "stored");
|
|
10043
|
+
validateAnalysisGeneration(params.analysisGeneration, "SearchPayload");
|
|
9509
10044
|
if (params.tolerance !== null) {
|
|
9510
10045
|
if (!isInteger(params.tolerance) || params.tolerance < 0 || params.tolerance > MAX_TOLERANCE) {
|
|
9511
10046
|
throwInvalid(
|
|
@@ -9744,6 +10279,7 @@ function validateSuggestPayload(decoded) {
|
|
|
9744
10279
|
validatePartitionIdsArray(decoded.partitionIds, "partitionIds", CONFIG_INVALID);
|
|
9745
10280
|
validateStringField(decoded.prefix, "prefix", MAX_TERM_LENGTH, CONFIG_INVALID);
|
|
9746
10281
|
validatePositiveInteger(decoded.limit, "limit", MAX_SUGGEST_WIRE_LIMIT, CONFIG_INVALID);
|
|
10282
|
+
validateAnalysisGeneration(decoded.analysisGeneration, "SuggestPayload");
|
|
9747
10283
|
return decoded;
|
|
9748
10284
|
}
|
|
9749
10285
|
function validateSuggestResultPayload(decoded) {
|
|
@@ -10079,6 +10615,7 @@ function validateStatsPayload(decoded) {
|
|
|
10079
10615
|
validateStringArray(decoded.terms, "terms", MAX_TERMS_COUNT, MAX_TERM_LENGTH, CONFIG_INVALID);
|
|
10080
10616
|
validateExpansion(decoded.expansion);
|
|
10081
10617
|
if (!isAbsent(decoded.tokens)) validateTokenList(decoded.tokens, "StatsPayload.tokens");
|
|
10618
|
+
validateAnalysisGeneration(decoded.analysisGeneration, "StatsPayload");
|
|
10082
10619
|
return decoded;
|
|
10083
10620
|
}
|
|
10084
10621
|
function validateStatsResultPayload(decoded) {
|
|
@@ -10199,7 +10736,24 @@ var QUERY_REJECTION_CODES = /* @__PURE__ */ new Set([
|
|
|
10199
10736
|
ErrorCodes.CONFIG_INVALID
|
|
10200
10737
|
]);
|
|
10201
10738
|
var nodeRejections = /* @__PURE__ */ new WeakSet();
|
|
10739
|
+
function refusedForNewerGeneration(details) {
|
|
10740
|
+
const held = details?.heldGeneration;
|
|
10741
|
+
const requested = details?.requestedGeneration;
|
|
10742
|
+
return typeof held === "number" && typeof requested === "number" && held > requested;
|
|
10743
|
+
}
|
|
10744
|
+
function generationRefusalOf(code, message, details) {
|
|
10745
|
+
if (code !== ErrorCodes.ANALYSIS_GENERATION_MISSING) return null;
|
|
10746
|
+
const refusal = new NarsilError(
|
|
10747
|
+
ErrorCodes.ANALYSIS_GENERATION_MISSING,
|
|
10748
|
+
typeof message === "string" ? message : code,
|
|
10749
|
+
details
|
|
10750
|
+
);
|
|
10751
|
+
if (refusedForNewerGeneration(details)) nodeRejections.add(refusal);
|
|
10752
|
+
return refusal;
|
|
10753
|
+
}
|
|
10202
10754
|
function nodeRejectionOf(code, message, details) {
|
|
10755
|
+
const generationRefusal = generationRefusalOf(code, message, details);
|
|
10756
|
+
if (generationRefusal !== null) return generationRefusal;
|
|
10203
10757
|
if (typeof code !== "string" || !QUERY_REJECTION_CODES.has(code)) return null;
|
|
10204
10758
|
const rejection = new NarsilError(code, typeof message === "string" ? message : code, details);
|
|
10205
10759
|
nodeRejections.add(rejection);
|
|
@@ -10211,6 +10765,7 @@ function markNodeRejection(error) {
|
|
|
10211
10765
|
}
|
|
10212
10766
|
function rethrowAsRefusal(error) {
|
|
10213
10767
|
if (error instanceof NarsilError && QUERY_REJECTION_CODES.has(error.code)) markNodeRejection(error);
|
|
10768
|
+
if (error instanceof NarsilError && refusedForNewerGeneration(error.details)) markNodeRejection(error);
|
|
10214
10769
|
throw error;
|
|
10215
10770
|
}
|
|
10216
10771
|
function isNodeRejection(error) {
|
|
@@ -13221,9 +13776,7 @@ async function resolveNodeTargets2(config, targetNodeId) {
|
|
|
13221
13776
|
async function resolveRegisteredTargets(config, targetNodeId) {
|
|
13222
13777
|
const nodes = await config.coordinator.listNodes();
|
|
13223
13778
|
const registration = nodes.find((node) => node.nodeId === targetNodeId);
|
|
13224
|
-
|
|
13225
|
-
if (registration.address.length === 0 || registration.address === targetNodeId) return [targetNodeId];
|
|
13226
|
-
return [registration.address, targetNodeId];
|
|
13779
|
+
return registration === void 0 ? [] : targetsOfRegisteredNode(targetNodeId, registration.address);
|
|
13227
13780
|
}
|
|
13228
13781
|
async function sendToNode2(config, targetNodeId, message, failure) {
|
|
13229
13782
|
const targets = await resolveNodeTargets2(config, targetNodeId);
|
|
@@ -13328,7 +13881,6 @@ async function readDistributedDocuments(config, nodeId, engine, indexName, docId
|
|
|
13328
13881
|
}
|
|
13329
13882
|
|
|
13330
13883
|
// src/distribution/cluster-node/synonym-sets.ts
|
|
13331
|
-
var SETTINGS_CONTENTION_REASON = "settings_contention";
|
|
13332
13884
|
function indexNotFound(indexName) {
|
|
13333
13885
|
return new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index '${indexName}' does not exist in the cluster`, {
|
|
13334
13886
|
indexName
|
|
@@ -13336,10 +13888,19 @@ function indexNotFound(indexName) {
|
|
|
13336
13888
|
}
|
|
13337
13889
|
async function readClusterSynonyms(coordinator, indexName) {
|
|
13338
13890
|
validateIndexName(indexName);
|
|
13339
|
-
const state = await readIndexState(coordinator, indexName);
|
|
13891
|
+
const state = await readIndexState(coordinator, await servingNameOf(coordinator, indexName));
|
|
13340
13892
|
if (state === null) throw indexNotFound(indexName);
|
|
13341
13893
|
return state.synonyms;
|
|
13342
13894
|
}
|
|
13895
|
+
async function requireSettledPartitionCount(coordinator, indexName, servingName) {
|
|
13896
|
+
const settled = await readRebalanceRecord(coordinator, indexName) === null && await servingNameOf(coordinator, indexName) === servingName;
|
|
13897
|
+
if (settled) return;
|
|
13898
|
+
throw new NarsilError(
|
|
13899
|
+
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
13900
|
+
`Index '${indexName}' is changing its partition count, so replace its synonym set again once that ends`,
|
|
13901
|
+
{ indexName }
|
|
13902
|
+
);
|
|
13903
|
+
}
|
|
13343
13904
|
function applyStoredSynonyms(engine, coordinator, indexName, indexUuid) {
|
|
13344
13905
|
if (engine.indexUuidOf(indexName) !== indexUuid) return Promise.resolve(false);
|
|
13345
13906
|
return engine.applyStoredSynonyms(indexName, async () => {
|
|
@@ -13352,6 +13913,10 @@ function applyStoredSynonyms(engine, coordinator, indexName, indexUuid) {
|
|
|
13352
13913
|
function preparedSet(snapshot, indexName, synonyms) {
|
|
13353
13914
|
const config = indexConfigFromSettings({}, snapshot.settings, indexName);
|
|
13354
13915
|
const language = getLanguage(config.language ?? "english");
|
|
13916
|
+
const pending = snapshot.settings?.pendingAnalysis;
|
|
13917
|
+
if (pending !== void 0) {
|
|
13918
|
+
prepareConfiguredSynonyms({ ...configOfNamedSettings(config, pending), synonyms }, getLanguage(pending.language));
|
|
13919
|
+
}
|
|
13355
13920
|
return prepareConfiguredSynonyms({ ...config, synonyms }, language)?.encoded ?? null;
|
|
13356
13921
|
}
|
|
13357
13922
|
async function swapStoredSet(coordinator, indexName, first, ref) {
|
|
@@ -13407,14 +13972,16 @@ async function askNodeToApply(deps, nodeId, indexName, indexUuid) {
|
|
|
13407
13972
|
async function replaceClusterSynonyms(deps, indexName, synonyms) {
|
|
13408
13973
|
const { coordinator } = deps;
|
|
13409
13974
|
validateIndexName(indexName);
|
|
13410
|
-
const
|
|
13975
|
+
const servingName = await servingNameOf(coordinator, indexName);
|
|
13976
|
+
await requireSettledPartitionCount(coordinator, indexName, servingName);
|
|
13977
|
+
const first = await readIndexMetadataSnapshot(coordinator, servingName);
|
|
13411
13978
|
if (first === null) throw indexNotFound(indexName);
|
|
13412
|
-
const
|
|
13979
|
+
const encoded2 = preparedSet(first, servingName, synonyms);
|
|
13413
13980
|
const { indexUuid } = first.record;
|
|
13414
|
-
const ref =
|
|
13981
|
+
const ref = encoded2 === null ? null : await storeSynonymSet(coordinator, indexUuid, encoded2);
|
|
13415
13982
|
let replaced;
|
|
13416
13983
|
try {
|
|
13417
|
-
replaced = await swapStoredSet(coordinator,
|
|
13984
|
+
replaced = await swapStoredSet(coordinator, servingName, first, ref);
|
|
13418
13985
|
} catch (err) {
|
|
13419
13986
|
if (ref !== null) await deleteSynonymSet(coordinator, indexUuid, ref).catch(() => void 0);
|
|
13420
13987
|
throw err;
|
|
@@ -13422,13 +13989,14 @@ async function replaceClusterSynonyms(deps, indexName, synonyms) {
|
|
|
13422
13989
|
if (replaced !== null && replaced.generation !== ref?.generation) {
|
|
13423
13990
|
await deleteSynonymSet(coordinator, indexUuid, replaced);
|
|
13424
13991
|
}
|
|
13425
|
-
const allocation = await coordinator.getAllocation(
|
|
13992
|
+
const allocation = await coordinator.getAllocation(servingName);
|
|
13426
13993
|
const holders = /* @__PURE__ */ new Set();
|
|
13427
13994
|
for (const assignment of allocation?.assignments.values() ?? []) {
|
|
13428
13995
|
for (const nodeId of assignment.replicas) holders.add(nodeId);
|
|
13429
13996
|
if (assignment.primary !== null) holders.add(assignment.primary);
|
|
13430
13997
|
}
|
|
13431
|
-
await Promise.all([...holders].map((nodeId) => askNodeToApply(deps, nodeId,
|
|
13998
|
+
await Promise.all([...holders].map((nodeId) => askNodeToApply(deps, nodeId, servingName, indexUuid)));
|
|
13999
|
+
await requireSettledPartitionCount(coordinator, indexName, servingName);
|
|
13432
14000
|
}
|
|
13433
14001
|
|
|
13434
14002
|
// src/distribution/cluster-node/local-index-reconcile.ts
|
|
@@ -13551,13 +14119,40 @@ async function handlePartitionStores(message, respond, deps) {
|
|
|
13551
14119
|
|
|
13552
14120
|
// src/distribution/cluster-node/rebuild-analysis.ts
|
|
13553
14121
|
function validateRebuildPayload(decoded) {
|
|
13554
|
-
if (!isRecord(decoded) || typeof decoded.indexName !== "string" || decoded.indexName.length === 0 || typeof decoded.indexUuid !== "string" || decoded.indexUuid.length === 0) {
|
|
14122
|
+
if (!isRecord(decoded) || typeof decoded.indexName !== "string" || decoded.indexName.length === 0 || typeof decoded.indexUuid !== "string" || decoded.indexUuid.length === 0 || decoded.generation !== void 0 && (!Number.isSafeInteger(decoded.generation) || Number(decoded.generation) < 1)) {
|
|
13555
14123
|
throw new NarsilError(
|
|
13556
14124
|
ErrorCodes.CONFIG_INVALID,
|
|
13557
|
-
"Invalid RebuildAnalysisPayload: it needs an indexName and
|
|
14125
|
+
"Invalid RebuildAnalysisPayload: it needs an indexName, an indexUuid, and a generation of 1 or more where it carries one"
|
|
13558
14126
|
);
|
|
13559
14127
|
}
|
|
13560
|
-
return {
|
|
14128
|
+
return {
|
|
14129
|
+
indexName: decoded.indexName,
|
|
14130
|
+
indexUuid: decoded.indexUuid,
|
|
14131
|
+
...typeof decoded.generation === "number" ? { generation: decoded.generation } : {}
|
|
14132
|
+
};
|
|
14133
|
+
}
|
|
14134
|
+
async function buildHeldCopies(engine, coordinator, payload, generation) {
|
|
14135
|
+
const { indexName, indexUuid } = payload;
|
|
14136
|
+
const answer = { indexName, rebuiltPartitions: [], generation };
|
|
14137
|
+
const localGeneration = engine.analysisGenerationOf(indexName);
|
|
14138
|
+
const localUuid = engine.indexUuidOf(indexName);
|
|
14139
|
+
if (localGeneration === void 0 || typeof localUuid === "string" && localUuid !== indexUuid) return answer;
|
|
14140
|
+
if (localGeneration >= generation) return answer;
|
|
14141
|
+
const snapshot = await readIndexMetadataSnapshot(coordinator, indexName);
|
|
14142
|
+
const settings = snapshot?.settings;
|
|
14143
|
+
if (snapshot === null || snapshot.record.indexUuid !== indexUuid) return answer;
|
|
14144
|
+
if ((settings?.analysisGeneration ?? 0) >= generation) return answer;
|
|
14145
|
+
const pending = settings?.pendingAnalysis;
|
|
14146
|
+
if (pending === void 0 || pending.generation !== generation) {
|
|
14147
|
+
throw new NarsilError(
|
|
14148
|
+
ErrorCodes.ANALYSIS_CHANGE_REPLACED,
|
|
14149
|
+
`A later change replaced generation ${generation} of the analysis of index '${indexName}'`,
|
|
14150
|
+
{ indexName }
|
|
14151
|
+
);
|
|
14152
|
+
}
|
|
14153
|
+
const { changeId: _changeId, ...named } = pending;
|
|
14154
|
+
await engine.buildHeldAnalysis(indexName, named);
|
|
14155
|
+
return { ...answer, rebuiltPartitions: heldPartitionIdsOf(engine, indexName) };
|
|
13561
14156
|
}
|
|
13562
14157
|
function validateRebuildResult(decoded, indexName) {
|
|
13563
14158
|
if (!isRecord(decoded) || decoded.indexName !== indexName || !Array.isArray(decoded.rebuiltPartitions)) {
|
|
@@ -13577,14 +14172,15 @@ function validateRebuildResult(decoded, indexName) {
|
|
|
13577
14172
|
async function rebuildLocalCopies(engine, payload) {
|
|
13578
14173
|
const { indexName, indexUuid } = payload;
|
|
13579
14174
|
const local = engine.listIndexes().find((index) => index.name === indexName);
|
|
13580
|
-
if (local === void 0 || engine.indexUuidOf(indexName) !== indexUuid || local.analysisStale !== true) {
|
|
14175
|
+
if (local === void 0 || engine.indexUuidOf(indexName) !== indexUuid || local.analysisStale !== true || engine.pendingAnalysisGenerationOf(indexName) !== null) {
|
|
13581
14176
|
return { indexName, rebuiltPartitions: [] };
|
|
13582
14177
|
}
|
|
13583
14178
|
await engine.rebuildAnalysis(indexName);
|
|
13584
14179
|
return { indexName, rebuiltPartitions: heldPartitionIdsOf(engine, indexName) };
|
|
13585
14180
|
}
|
|
13586
14181
|
async function handleRebuildAnalysis(message, respond, deps) {
|
|
13587
|
-
const
|
|
14182
|
+
const payload = validateRebuildPayload(decode(message.payload));
|
|
14183
|
+
const result = payload.generation === void 0 ? await rebuildLocalCopies(deps.engine, payload) : await buildHeldCopies(deps.engine, deps.coordinator, payload, payload.generation);
|
|
13588
14184
|
await respond({
|
|
13589
14185
|
type: ClusterMessageTypes.REBUILD_ANALYSIS,
|
|
13590
14186
|
sourceId: deps.nodeId,
|
|
@@ -14595,14 +15191,18 @@ async function handleSearch(message, respond, deps) {
|
|
|
14595
15191
|
if (snapshots !== void 0 || rescoreWindow !== void 0 && queryParams.sort === void 0 && queryParams.group === void 0) {
|
|
14596
15192
|
queryParams.document = false;
|
|
14597
15193
|
}
|
|
14598
|
-
const queryResult = await deps.
|
|
15194
|
+
const queryResult = await deps.analysisView.answerAt(
|
|
14599
15195
|
payload.indexName,
|
|
14600
|
-
|
|
14601
|
-
|
|
14602
|
-
|
|
14603
|
-
|
|
14604
|
-
|
|
14605
|
-
|
|
15196
|
+
payload.params.analysisGeneration ?? 0,
|
|
15197
|
+
() => deps.engine.queryPartitions(
|
|
15198
|
+
payload.indexName,
|
|
15199
|
+
queryParams,
|
|
15200
|
+
payload.partitionIds,
|
|
15201
|
+
payload.globalStats ?? void 0,
|
|
15202
|
+
rescoreWindow,
|
|
15203
|
+
snapshots,
|
|
15204
|
+
payload.params.indexBm25 ?? void 0
|
|
15205
|
+
)
|
|
14606
15206
|
);
|
|
14607
15207
|
const rankedDocument = (hit) => snapshots?.get(hit.id)?.fields ?? hit.document;
|
|
14608
15208
|
const normalizedSort = queryParams.sort !== void 0 ? normalizeSort(queryParams.sort) : null;
|
|
@@ -14721,11 +15321,13 @@ function openSearchContext(deps, coordinatorNodeId, keepAliveMs, snapshots) {
|
|
|
14721
15321
|
async function handleStats(message, respond, deps) {
|
|
14722
15322
|
const decoded = decode(message.payload);
|
|
14723
15323
|
const payload = validateStatsPayload(decoded);
|
|
14724
|
-
const resultPayload = await deps.
|
|
15324
|
+
const resultPayload = await deps.analysisView.answerAt(
|
|
14725
15325
|
payload.indexName,
|
|
14726
|
-
payload.
|
|
14727
|
-
payload.partitionIds,
|
|
14728
|
-
|
|
15326
|
+
payload.analysisGeneration ?? 0,
|
|
15327
|
+
() => deps.engine.collectQueryStats(payload.indexName, payload.terms, payload.partitionIds, {
|
|
15328
|
+
expansion: payload.expansion ?? void 0,
|
|
15329
|
+
tokens: payload.tokens ?? void 0
|
|
15330
|
+
})
|
|
14729
15331
|
);
|
|
14730
15332
|
await respond({
|
|
14731
15333
|
type: QueryMessageTypes.STATS_RESULT,
|
|
@@ -14800,10 +15402,14 @@ async function handleList(message, respond, deps) {
|
|
|
14800
15402
|
}
|
|
14801
15403
|
async function handleSuggest(message, respond, deps) {
|
|
14802
15404
|
const payload = validateSuggestPayload(decode(message.payload));
|
|
14803
|
-
const result = await deps.
|
|
15405
|
+
const result = await deps.analysisView.answerAt(
|
|
14804
15406
|
payload.indexName,
|
|
14805
|
-
|
|
14806
|
-
|
|
15407
|
+
payload.analysisGeneration ?? 0,
|
|
15408
|
+
() => deps.engine.suggestPartitions(
|
|
15409
|
+
payload.indexName,
|
|
15410
|
+
{ prefix: payload.prefix, limit: payload.limit },
|
|
15411
|
+
payload.partitionIds
|
|
15412
|
+
)
|
|
14807
15413
|
);
|
|
14808
15414
|
await respond(
|
|
14809
15415
|
createSuggestResultMessage(
|
|
@@ -14816,11 +15422,15 @@ async function handleSuggest(message, respond, deps) {
|
|
|
14816
15422
|
async function handlePreflight(message, respond, deps) {
|
|
14817
15423
|
const payload = validatePreflightPayload(decode(message.payload));
|
|
14818
15424
|
const params = wireParamsToLocal(payload.params);
|
|
14819
|
-
const result = await deps.
|
|
15425
|
+
const result = await deps.analysisView.answerAt(
|
|
14820
15426
|
payload.indexName,
|
|
14821
|
-
params,
|
|
14822
|
-
|
|
14823
|
-
|
|
15427
|
+
payload.params.analysisGeneration ?? 0,
|
|
15428
|
+
() => deps.engine.preflightPartitions(
|
|
15429
|
+
payload.indexName,
|
|
15430
|
+
params,
|
|
15431
|
+
payload.partitionIds,
|
|
15432
|
+
payload.params.indexBm25 ?? void 0
|
|
15433
|
+
)
|
|
14824
15434
|
);
|
|
14825
15435
|
await respond(
|
|
14826
15436
|
createPreflightResultMessage(
|
|
@@ -15320,9 +15930,7 @@ async function handleSyncRequestMessage(message, respond, deps) {
|
|
|
15320
15930
|
},
|
|
15321
15931
|
afterSnapshot: async (trailingSink, build) => {
|
|
15322
15932
|
const snapshotEnd = build.lastSeqNo ?? snapshotSeqNo;
|
|
15323
|
-
|
|
15324
|
-
await sendSyncEntriesResponse(message, trailingSink, deps, entries);
|
|
15325
|
-
notePosition(entries.at(-1)?.seqNo ?? snapshotEnd);
|
|
15933
|
+
await sendSyncEntriesResponse(message, trailingSink, deps, log.getEntriesFrom(snapshotEnd + 1));
|
|
15326
15934
|
}
|
|
15327
15935
|
}
|
|
15328
15936
|
);
|
|
@@ -15753,24 +16361,130 @@ async function resolveNodeId(explicitNodeId, directory) {
|
|
|
15753
16361
|
}
|
|
15754
16362
|
}
|
|
15755
16363
|
|
|
15756
|
-
// src/distribution/cluster-node/
|
|
15757
|
-
var SETTINGS_CONTENTION_REASON2 = "settings_contention";
|
|
16364
|
+
// src/distribution/cluster-node/analysis-change/request.ts
|
|
15758
16365
|
function indexNotFound2(indexName) {
|
|
15759
16366
|
return new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index '${indexName}' does not exist in the cluster`, {
|
|
15760
16367
|
indexName
|
|
15761
16368
|
});
|
|
15762
16369
|
}
|
|
16370
|
+
function sameList(left, right) {
|
|
16371
|
+
if (left === void 0 || right === void 0) return left === right;
|
|
16372
|
+
return left.length === right.length && left.every((word, position) => word === right[position]);
|
|
16373
|
+
}
|
|
16374
|
+
function asksForPending(pending, target) {
|
|
16375
|
+
if (pending === void 0) return false;
|
|
16376
|
+
const named = namedSettingsOf(target.config, target.language.name);
|
|
16377
|
+
return pending.language === named.language && pending.tokenizer === named.tokenizer && pending.stopWords === named.stopWords && sameList(pending.stopWordList, named.stopWordList);
|
|
16378
|
+
}
|
|
16379
|
+
async function requireTargetAcceptable(coordinator, indexName, target) {
|
|
16380
|
+
const state = await readIndexState(coordinator, indexName);
|
|
16381
|
+
if (state === null || state.synonyms.length === 0) return;
|
|
16382
|
+
prepareConfiguredSynonyms({ ...target.config, synonyms: state.synonyms }, target.language);
|
|
16383
|
+
}
|
|
16384
|
+
async function writePending(coordinator, indexName, target) {
|
|
16385
|
+
const schema = await coordinator.getSchema(indexName);
|
|
16386
|
+
if (schema === null) throw indexNotFound2(indexName);
|
|
16387
|
+
for (let attempt = 1; attempt <= MAX_SETTINGS_REPLACEMENT_ATTEMPTS; attempt++) {
|
|
16388
|
+
const snapshot = await readIndexMetadataSnapshot(coordinator, indexName);
|
|
16389
|
+
if (snapshot === null) throw indexNotFound2(indexName);
|
|
16390
|
+
const running = snapshot.settings?.pendingAnalysis;
|
|
16391
|
+
if (running !== void 0 && asksForPending(running, target)) return running;
|
|
16392
|
+
const current = indexConfigFromSettings(schema, snapshot.settings, indexName);
|
|
16393
|
+
if (running === void 0 && holdsSettings(current, target)) return null;
|
|
16394
|
+
const pending = {
|
|
16395
|
+
changeId: generateId(),
|
|
16396
|
+
generation: Math.max(snapshot.settings?.analysisGeneration ?? 0, running?.generation ?? 0) + 1,
|
|
16397
|
+
...namedSettingsOf(target.config, target.language.name)
|
|
16398
|
+
};
|
|
16399
|
+
const next = metadataWithPendingAnalysis(snapshot, pending);
|
|
16400
|
+
if (await coordinator.compareAndSet(indexConfigKey(indexName), snapshot.bytes, next)) return pending;
|
|
16401
|
+
}
|
|
16402
|
+
throw new NarsilError(
|
|
16403
|
+
ErrorCodes.CLUSTER_WRITE_BLOCKED,
|
|
16404
|
+
`Index '${indexName}' kept changing while its analysis change was recorded ${MAX_SETTINGS_REPLACEMENT_ATTEMPTS} times over, so the change was not stored. Retry the request`,
|
|
16405
|
+
{ reason: SETTINGS_CONTENTION_REASON, indexName, attempts: MAX_SETTINGS_REPLACEMENT_ATTEMPTS }
|
|
16406
|
+
);
|
|
16407
|
+
}
|
|
16408
|
+
async function clearOwnPending(coordinator, indexName, changeId) {
|
|
16409
|
+
for (let attempt = 1; attempt <= MAX_SETTINGS_REPLACEMENT_ATTEMPTS; attempt++) {
|
|
16410
|
+
const snapshot = await readIndexMetadataSnapshot(coordinator, indexName);
|
|
16411
|
+
if (snapshot?.settings?.pendingAnalysis?.changeId !== changeId) return;
|
|
16412
|
+
const next = metadataWithoutPendingAnalysis(snapshot);
|
|
16413
|
+
if (await coordinator.compareAndSet(indexConfigKey(indexName), snapshot.bytes, next)) return;
|
|
16414
|
+
}
|
|
16415
|
+
}
|
|
16416
|
+
function outcomeOf(snapshot, pending, indexName) {
|
|
16417
|
+
const settings = snapshot.settings;
|
|
16418
|
+
if (settings?.analysisGeneration === pending.generation) return "published";
|
|
16419
|
+
const failure = settings?.analysisFailure;
|
|
16420
|
+
if (failure?.changeId === pending.changeId) {
|
|
16421
|
+
throw new NarsilError(failure.code, failure.message, { indexName, nodeId: failure.nodeId });
|
|
16422
|
+
}
|
|
16423
|
+
if (settings?.pendingAnalysis?.changeId === pending.changeId) return "waiting";
|
|
16424
|
+
throw new NarsilError(
|
|
16425
|
+
ErrorCodes.ANALYSIS_CHANGE_REPLACED,
|
|
16426
|
+
`A later change replaced this analysis change of index '${indexName}'`,
|
|
16427
|
+
{ indexName }
|
|
16428
|
+
);
|
|
16429
|
+
}
|
|
16430
|
+
async function changeClusterAnalysis(request) {
|
|
16431
|
+
const { coordinator, view, guard, servingName, publicName, change } = request;
|
|
16432
|
+
validateIndexName(servingName);
|
|
16433
|
+
const schema = await coordinator.getSchema(servingName);
|
|
16434
|
+
const metadata = await readIndexMetadataSnapshot(coordinator, servingName);
|
|
16435
|
+
if (schema === null || metadata === null) throw indexNotFound2(publicName);
|
|
16436
|
+
const target = resolveAnalysisTarget(
|
|
16437
|
+
publicName,
|
|
16438
|
+
indexConfigFromSettings(schema, metadata.settings, servingName),
|
|
16439
|
+
change
|
|
16440
|
+
);
|
|
16441
|
+
await requireTargetAcceptable(coordinator, servingName, target);
|
|
16442
|
+
const olderNodes = registeredNodesBelowBm25Tuning(await coordinator.listNodes());
|
|
16443
|
+
if (olderNodes.length > 0) {
|
|
16444
|
+
throw new NarsilError(
|
|
16445
|
+
ErrorCodes.ANALYSIS_CHANGE_NOT_READY,
|
|
16446
|
+
`The cluster has nodes below the specification version that changes analysis, so index '${publicName}' keeps its settings until they are upgraded`,
|
|
16447
|
+
{ indexName: publicName, nodeIds: olderNodes }
|
|
16448
|
+
);
|
|
16449
|
+
}
|
|
16450
|
+
const pending = await writePending(coordinator, servingName, target);
|
|
16451
|
+
if (pending === null) return;
|
|
16452
|
+
if (await readRebalanceRecord(coordinator, publicName) !== null) {
|
|
16453
|
+
await clearOwnPending(coordinator, servingName, pending.changeId);
|
|
16454
|
+
throw new NarsilError(
|
|
16455
|
+
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
16456
|
+
`Index '${publicName}' is changing its partition count, so try its analysis change again once that ends`,
|
|
16457
|
+
{ indexName: publicName }
|
|
16458
|
+
);
|
|
16459
|
+
}
|
|
16460
|
+
for (; ; ) {
|
|
16461
|
+
guard();
|
|
16462
|
+
const snapshot = await readIndexMetadataSnapshot(coordinator, servingName);
|
|
16463
|
+
if (snapshot === null) throw indexNotFound2(publicName);
|
|
16464
|
+
if (outcomeOf(snapshot, pending, publicName) === "published") break;
|
|
16465
|
+
await pause(ANALYSIS_CHANGE_POLL_INTERVAL_MS);
|
|
16466
|
+
}
|
|
16467
|
+
await view.refresh(servingName);
|
|
16468
|
+
await view.settle(servingName);
|
|
16469
|
+
}
|
|
16470
|
+
|
|
16471
|
+
// src/distribution/cluster-node/bm25-change.ts
|
|
16472
|
+
function indexNotFound3(indexName) {
|
|
16473
|
+
return new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index '${indexName}' does not exist in the cluster`, {
|
|
16474
|
+
indexName
|
|
16475
|
+
});
|
|
16476
|
+
}
|
|
15763
16477
|
async function replaceClusterBm25(coordinator, view, indexName, value) {
|
|
15764
16478
|
validateIndexName(indexName);
|
|
15765
16479
|
const schema = await coordinator.getSchema(indexName);
|
|
15766
|
-
if (schema === null) throw
|
|
16480
|
+
if (schema === null) throw indexNotFound3(indexName);
|
|
15767
16481
|
validateBM25Params(value, schema, ErrorCodes.CONFIG_INVALID);
|
|
15768
16482
|
const bm25 = storedBM25Params(value);
|
|
15769
16483
|
const olderNodes = registeredNodesBelowBm25Tuning(await coordinator.listNodes());
|
|
15770
16484
|
if (olderNodes.length > 0) throw bm25NotReady("The cluster has", olderNodes, indexName);
|
|
15771
16485
|
for (let attempt = 1; attempt <= MAX_SETTINGS_REPLACEMENT_ATTEMPTS; attempt++) {
|
|
15772
16486
|
const current = await readIndexMetadataSnapshot(coordinator, indexName);
|
|
15773
|
-
if (current === null) throw
|
|
16487
|
+
if (current === null) throw indexNotFound3(indexName);
|
|
15774
16488
|
const revision = (current.settings?.bm25Revision ?? 0) + 1;
|
|
15775
16489
|
const next = metadataWithBm25(current, bm25, revision);
|
|
15776
16490
|
if (await coordinator.compareAndSet(indexConfigKey(indexName), current.bytes, next)) {
|
|
@@ -15781,30 +16495,10 @@ async function replaceClusterBm25(coordinator, view, indexName, value) {
|
|
|
15781
16495
|
throw new NarsilError(
|
|
15782
16496
|
ErrorCodes.CLUSTER_WRITE_BLOCKED,
|
|
15783
16497
|
`Index '${indexName}' kept changing while its BM25 setting was replaced ${MAX_SETTINGS_REPLACEMENT_ATTEMPTS} times over, so the new setting was not stored. Retry the request`,
|
|
15784
|
-
{ reason:
|
|
16498
|
+
{ reason: SETTINGS_CONTENTION_REASON, indexName, attempts: MAX_SETTINGS_REPLACEMENT_ATTEMPTS }
|
|
15785
16499
|
);
|
|
15786
16500
|
}
|
|
15787
16501
|
|
|
15788
|
-
// src/distribution/query/node-send.ts
|
|
15789
|
-
async function resolveTargets(nodeId, deps) {
|
|
15790
|
-
if (deps.resolveNodeTargets === void 0) {
|
|
15791
|
-
return [nodeId];
|
|
15792
|
-
}
|
|
15793
|
-
const targets = await deps.resolveNodeTargets(nodeId);
|
|
15794
|
-
return targets.length > 0 ? targets : [nodeId];
|
|
15795
|
-
}
|
|
15796
|
-
async function sendToFirstReachableTarget2(transport, targets, message) {
|
|
15797
|
-
let lastError;
|
|
15798
|
-
for (const target of targets) {
|
|
15799
|
-
try {
|
|
15800
|
-
return await transport.send(target, message);
|
|
15801
|
-
} catch (error) {
|
|
15802
|
-
lastError = error;
|
|
15803
|
-
}
|
|
15804
|
-
}
|
|
15805
|
-
throw lastError instanceof Error ? lastError : new Error(String(lastError));
|
|
15806
|
-
}
|
|
15807
|
-
|
|
15808
16502
|
// src/distribution/query/phrase-routing.ts
|
|
15809
16503
|
function queryReadsPhrases(term) {
|
|
15810
16504
|
return typeof term === "string" && holdsQuotedText(term);
|
|
@@ -15885,11 +16579,13 @@ function releaseSearchContexts(indexName, contexts, deps) {
|
|
|
15885
16579
|
function queryRejectionOf(response, decoded) {
|
|
15886
16580
|
const body = decoded;
|
|
15887
16581
|
if (!response.type.endsWith(".error") && body?.error !== true) return null;
|
|
16582
|
+
const details = typeof body?.details === "object" && body.details !== null ? { ...body.details } : {};
|
|
16583
|
+
const generationRefusal = generationRefusalOf(body?.code, body?.message, details);
|
|
16584
|
+
if (generationRefusal !== null) return generationRefusal;
|
|
15888
16585
|
if (body?.code !== ServerErrorCodes.TOO_MANY_REQUESTS) return nodeRejectionOf(body?.code, body?.message);
|
|
15889
|
-
const details = typeof body.details === "object" && body.details !== null ? { ...body.details } : {};
|
|
15890
16586
|
return new NarsilError(
|
|
15891
16587
|
ServerErrorCodes.TOO_MANY_REQUESTS,
|
|
15892
|
-
typeof body
|
|
16588
|
+
typeof body?.message === "string" ? body.message : "A node had no room in memory to answer its part of the search",
|
|
15893
16589
|
details
|
|
15894
16590
|
);
|
|
15895
16591
|
}
|
|
@@ -15917,7 +16613,7 @@ async function requestStats(nodeId, payload, deps, config) {
|
|
|
15917
16613
|
validateStatsResultPayload
|
|
15918
16614
|
);
|
|
15919
16615
|
}
|
|
15920
|
-
async function countLeftOutTokens(indexName, answered, deps, config) {
|
|
16616
|
+
async function countLeftOutTokens(indexName, answered, analysisGeneration, deps, config) {
|
|
15921
16617
|
const reached = /* @__PURE__ */ new Set();
|
|
15922
16618
|
for (const [, { result }] of answered) {
|
|
15923
16619
|
for (const token of result.expanded ?? []) reached.add(token);
|
|
@@ -15927,7 +16623,12 @@ async function countLeftOutTokens(indexName, answered, deps, config) {
|
|
|
15927
16623
|
answered.map(async ([nodeId, { partitionIds, result }]) => {
|
|
15928
16624
|
const tokens = [...reached].filter((token) => !Object.hasOwn(result.docFrequencies, token));
|
|
15929
16625
|
if (tokens.length === 0) return null;
|
|
15930
|
-
const counted = await requestStats(
|
|
16626
|
+
const counted = await requestStats(
|
|
16627
|
+
nodeId,
|
|
16628
|
+
{ indexName, partitionIds, terms: [], tokens, analysisGeneration },
|
|
16629
|
+
deps,
|
|
16630
|
+
config
|
|
16631
|
+
);
|
|
15931
16632
|
if (counted === null) return null;
|
|
15932
16633
|
const docFrequencies = /* @__PURE__ */ Object.create(null);
|
|
15933
16634
|
for (const token of tokens) {
|
|
@@ -15946,10 +16647,15 @@ async function collectDistributedStats(indexName, params, routing, deps, config)
|
|
|
15946
16647
|
if (routing.nodeToPartitions.size === 0 && queryReadsPhrases(params.term)) return mergePartitionStats([]);
|
|
15947
16648
|
const terms = params.term !== null ? params.term.split(/\s+/).filter((t) => t.length > 0) : [];
|
|
15948
16649
|
const expansion = statsExpansionOf(params);
|
|
16650
|
+
const { analysisGeneration } = params;
|
|
15949
16651
|
const outcomes = await sendWithFailover(
|
|
15950
16652
|
routing,
|
|
15951
|
-
(nodeId, partitionIds) => requestStats(nodeId, { indexName, partitionIds, terms, expansion }, deps, config)
|
|
16653
|
+
(nodeId, partitionIds) => requestStats(nodeId, { indexName, partitionIds, terms, expansion, analysisGeneration }, deps, config)
|
|
16654
|
+
);
|
|
16655
|
+
const refused = outcomes.find(
|
|
16656
|
+
(outcome) => isNodeRejection(outcome.error) && outcome.error instanceof NarsilError && outcome.error.code === ErrorCodes.ANALYSIS_GENERATION_MISSING
|
|
15952
16657
|
);
|
|
16658
|
+
if (refused !== void 0) throw refused.error;
|
|
15953
16659
|
const answered = [];
|
|
15954
16660
|
for (const outcome of outcomes) {
|
|
15955
16661
|
if (outcome.status === "success" && outcome.result !== null) {
|
|
@@ -15961,7 +16667,7 @@ async function collectDistributedStats(indexName, params, routing, deps, config)
|
|
|
15961
16667
|
nodeCount: routing.nodeToPartitions.size
|
|
15962
16668
|
});
|
|
15963
16669
|
}
|
|
15964
|
-
const leftOut = await countLeftOutTokens(indexName, answered, deps, config);
|
|
16670
|
+
const leftOut = await countLeftOutTokens(indexName, answered, analysisGeneration, deps, config);
|
|
15965
16671
|
return mergePartitionStats([...answered.map(([, { result }]) => result), ...leftOut]);
|
|
15966
16672
|
}
|
|
15967
16673
|
async function fanOutSearch(indexName, params, globalStats, facetShardSize, routing, deps, config, keepAliveMs = null) {
|
|
@@ -16740,6 +17446,17 @@ function failOutcomesMissingSortValues(outcomes) {
|
|
|
16740
17446
|
}
|
|
16741
17447
|
}
|
|
16742
17448
|
|
|
17449
|
+
// src/distribution/cluster-node/analysis-change/rerun.ts
|
|
17450
|
+
async function rerunOnNewerGeneration(view, indexName, run) {
|
|
17451
|
+
try {
|
|
17452
|
+
return await run();
|
|
17453
|
+
} catch (error) {
|
|
17454
|
+
if (!(error instanceof NarsilError) || error.code !== ErrorCodes.ANALYSIS_GENERATION_MISSING) throw error;
|
|
17455
|
+
await view.refresh(indexName);
|
|
17456
|
+
return run();
|
|
17457
|
+
}
|
|
17458
|
+
}
|
|
17459
|
+
|
|
16743
17460
|
// src/distribution/cluster-node/reads/bm25.ts
|
|
16744
17461
|
function routingUnder(params, setting) {
|
|
16745
17462
|
const wireParams = localParamsToWire(params);
|
|
@@ -17290,16 +18007,23 @@ async function suggestCluster(deps, indexName, params) {
|
|
|
17290
18007
|
return { terms: [], elapsed: performance.now() - startTime };
|
|
17291
18008
|
}
|
|
17292
18009
|
const nodeLimit = suggestNodeLimit(clientLimit);
|
|
17293
|
-
const gathered = await
|
|
17294
|
-
|
|
17295
|
-
|
|
17296
|
-
|
|
17297
|
-
|
|
17298
|
-
|
|
17299
|
-
|
|
17300
|
-
|
|
17301
|
-
|
|
17302
|
-
|
|
18010
|
+
const gathered = await rerunOnNewerGeneration(deps.analysisView, indexName, async () => {
|
|
18011
|
+
const analysisGeneration = await deps.analysisView.generationFor(indexName);
|
|
18012
|
+
return gatherFromEveryPartition(allocation, indexName, async (group) => {
|
|
18013
|
+
if (group.nodeId === deps.nodeId) {
|
|
18014
|
+
const local = await deps.analysisView.answerAt(
|
|
18015
|
+
indexName,
|
|
18016
|
+
analysisGeneration,
|
|
18017
|
+
() => deps.engine.suggestPartitions(indexName, { prefix, limit: nodeLimit }, group.partitionIds)
|
|
18018
|
+
).catch(rethrowAsRefusal);
|
|
18019
|
+
return { terms: local.terms, analysisStale: local.analysisStale === true };
|
|
18020
|
+
}
|
|
18021
|
+
const message = createSuggestMessage(
|
|
18022
|
+
{ indexName, partitionIds: group.partitionIds, prefix, limit: nodeLimit, analysisGeneration },
|
|
18023
|
+
deps.nodeId
|
|
18024
|
+
);
|
|
18025
|
+
return sendReadRequest(deps, group.nodeId, message, indexName, validateSuggestResultPayload);
|
|
18026
|
+
});
|
|
17303
18027
|
});
|
|
17304
18028
|
const mergedFrequencies = /* @__PURE__ */ new Map();
|
|
17305
18029
|
let analysisStale = false;
|
|
@@ -17325,20 +18049,27 @@ async function preflightCluster(deps, indexName, params, settingName) {
|
|
|
17325
18049
|
return deps.engine.preflight(indexName, params);
|
|
17326
18050
|
}
|
|
17327
18051
|
const startTime = performance.now();
|
|
17328
|
-
const { wireParams, indexBm25, carriesBm25 } = await bm25RoutingFor(deps, settingName, params);
|
|
17329
18052
|
const coordinator = deps.config.coordinator;
|
|
17330
|
-
const
|
|
17331
|
-
|
|
17332
|
-
|
|
17333
|
-
|
|
17334
|
-
|
|
17335
|
-
|
|
17336
|
-
|
|
17337
|
-
|
|
17338
|
-
|
|
17339
|
-
|
|
17340
|
-
|
|
17341
|
-
|
|
18053
|
+
const gathered = await rerunOnNewerGeneration(deps.analysisView, indexName, async () => {
|
|
18054
|
+
const { wireParams, indexBm25, carriesBm25 } = await bm25RoutingFor(deps, settingName, params);
|
|
18055
|
+
wireParams.analysisGeneration = await deps.analysisView.generationFor(indexName);
|
|
18056
|
+
const bm25Allocation = carriesBm25 ? await allocationForBm25Query(indexName, allocation, () => coordinator.listNodes()) : allocation;
|
|
18057
|
+
const routedAllocation = queryReadsPhrases(params.term) ? await allocationForPhraseQuery(indexName, bm25Allocation, () => coordinator.listNodes()) : bm25Allocation;
|
|
18058
|
+
return gatherFromEveryPartition(routedAllocation, indexName, async (group) => {
|
|
18059
|
+
if (group.nodeId === deps.nodeId) {
|
|
18060
|
+
const local = await deps.analysisView.answerAt(
|
|
18061
|
+
indexName,
|
|
18062
|
+
wireParams.analysisGeneration ?? 0,
|
|
18063
|
+
() => deps.engine.preflightPartitions(indexName, params, group.partitionIds, indexBm25)
|
|
18064
|
+
).catch(rethrowAsRefusal);
|
|
18065
|
+
return { count: local.count, analysisStale: local.analysisStale === true };
|
|
18066
|
+
}
|
|
18067
|
+
const message = createPreflightMessage(
|
|
18068
|
+
{ indexName, partitionIds: group.partitionIds, params: wireParams },
|
|
18069
|
+
deps.nodeId
|
|
18070
|
+
);
|
|
18071
|
+
return sendReadRequest(deps, group.nodeId, message, indexName, validatePreflightResultPayload);
|
|
18072
|
+
});
|
|
17342
18073
|
});
|
|
17343
18074
|
let count2 = 0;
|
|
17344
18075
|
let analysisStale = false;
|
|
@@ -17407,36 +18138,41 @@ async function queryCluster(deps, indexName, params, settingName) {
|
|
|
17407
18138
|
if (allocation === null) {
|
|
17408
18139
|
return deps.engine.query(indexName, params);
|
|
17409
18140
|
}
|
|
17410
|
-
return
|
|
17411
|
-
|
|
18141
|
+
return rerunOnNewerGeneration(
|
|
18142
|
+
deps.analysisView,
|
|
18143
|
+
indexName,
|
|
18144
|
+
() => rerunOnLostContext(async () => {
|
|
18145
|
+
if (params.rerank === void 0) {
|
|
18146
|
+
return withRankedReader(
|
|
18147
|
+
deps,
|
|
18148
|
+
indexName,
|
|
18149
|
+
allocation,
|
|
18150
|
+
(reader) => searchCluster(deps, indexName, settingName, params, allocation, reader, false)
|
|
18151
|
+
);
|
|
18152
|
+
}
|
|
18153
|
+
const { rerankClusterQuery } = await import('./rerank-KTLC3E6N.mjs');
|
|
17412
18154
|
return withRankedReader(
|
|
17413
18155
|
deps,
|
|
17414
18156
|
indexName,
|
|
17415
18157
|
allocation,
|
|
17416
|
-
(reader) =>
|
|
18158
|
+
(reader) => rerankClusterQuery(
|
|
18159
|
+
deps,
|
|
18160
|
+
indexName,
|
|
18161
|
+
params,
|
|
18162
|
+
(firstStage) => searchCluster(deps, indexName, settingName, firstStage, allocation, reader, true),
|
|
18163
|
+
(docIds, last) => reader.read(docIds, { last, fields: null, requireEveryPartition: true }),
|
|
18164
|
+
clusterRerankHost
|
|
18165
|
+
)
|
|
17417
18166
|
);
|
|
17418
|
-
}
|
|
17419
|
-
|
|
17420
|
-
return withRankedReader(
|
|
17421
|
-
deps,
|
|
17422
|
-
indexName,
|
|
17423
|
-
allocation,
|
|
17424
|
-
(reader) => rerankClusterQuery(
|
|
17425
|
-
deps,
|
|
17426
|
-
indexName,
|
|
17427
|
-
params,
|
|
17428
|
-
(firstStage) => searchCluster(deps, indexName, settingName, firstStage, allocation, reader, true),
|
|
17429
|
-
(docIds, last) => reader.read(docIds, { last, fields: null, requireEveryPartition: true }),
|
|
17430
|
-
clusterRerankHost
|
|
17431
|
-
)
|
|
17432
|
-
);
|
|
17433
|
-
});
|
|
18167
|
+
})
|
|
18168
|
+
);
|
|
17434
18169
|
}
|
|
17435
18170
|
async function searchCluster(deps, indexName, settingName, params, allocation, reader, readsLater) {
|
|
17436
18171
|
const projection = resolveProjection(params.document);
|
|
17437
18172
|
const groupsRead = params.group !== void 0 && (params.group.reduce !== void 0 || projection.kind !== "none");
|
|
17438
18173
|
const readsDocuments = readsLater || projection.kind !== "none" || groupsRead;
|
|
17439
18174
|
const { wireParams, carriesBm25 } = heldBm25RoutingFor(deps, settingName, params) ?? await bm25RoutingFor(deps, settingName, params);
|
|
18175
|
+
wireParams.analysisGeneration = deps.analysisView.held(indexName) ?? await deps.analysisView.generationFor(indexName);
|
|
17440
18176
|
const queryDeps = {
|
|
17441
18177
|
transport: deps.config.transport,
|
|
17442
18178
|
sourceNodeId: deps.nodeId,
|
|
@@ -17614,6 +18350,13 @@ function createClusterNodeOperations(deps) {
|
|
|
17614
18350
|
function served(indexName, fn) {
|
|
17615
18351
|
return trackOp(indexName, async () => fn(await deps.servingName(indexName)));
|
|
17616
18352
|
}
|
|
18353
|
+
const names = {
|
|
18354
|
+
servingName: deps.servingName,
|
|
18355
|
+
liveServingName: (indexName) => servingNameOf(coordinator, indexName)
|
|
18356
|
+
};
|
|
18357
|
+
function read(indexName, fn) {
|
|
18358
|
+
return trackOp(indexName, () => readFollowingTheName(names, indexName, fn));
|
|
18359
|
+
}
|
|
17617
18360
|
return {
|
|
17618
18361
|
async createIndex(name, indexConfig, options) {
|
|
17619
18362
|
refuseHiddenGeneration(name);
|
|
@@ -17672,16 +18415,16 @@ function createClusterNodeOperations(deps) {
|
|
|
17672
18415
|
return served(indexName, (name) => clearCluster(readDeps, writeDeps, name));
|
|
17673
18416
|
},
|
|
17674
18417
|
async countDocuments(indexName) {
|
|
17675
|
-
return
|
|
18418
|
+
return read(indexName, (name) => countCluster(readDeps, name));
|
|
17676
18419
|
},
|
|
17677
18420
|
async listDocuments(indexName, params) {
|
|
17678
|
-
return
|
|
18421
|
+
return read(indexName, (name) => listCluster(readDeps, name, params ?? {}));
|
|
17679
18422
|
},
|
|
17680
18423
|
async suggest(indexName, params) {
|
|
17681
|
-
return
|
|
18424
|
+
return read(indexName, (name) => suggestCluster(readDeps, name, params));
|
|
17682
18425
|
},
|
|
17683
18426
|
async preflight(indexName, params) {
|
|
17684
|
-
return
|
|
18427
|
+
return read(indexName, (name) => preflightCluster(readDeps, name, params, indexName));
|
|
17685
18428
|
},
|
|
17686
18429
|
async rebuildAnalysis(indexName) {
|
|
17687
18430
|
return served(
|
|
@@ -17689,6 +18432,20 @@ function createClusterNodeOperations(deps) {
|
|
|
17689
18432
|
(name) => rebuildClusterAnalysis({ nodeId: readDeps.nodeId, config: readDeps.config, engine }, name)
|
|
17690
18433
|
);
|
|
17691
18434
|
},
|
|
18435
|
+
async changeAnalysis(indexName, change) {
|
|
18436
|
+
refuseHiddenGeneration(indexName);
|
|
18437
|
+
return trackOp(
|
|
18438
|
+
indexName,
|
|
18439
|
+
async () => changeClusterAnalysis({
|
|
18440
|
+
coordinator,
|
|
18441
|
+
view: readDeps.analysisView,
|
|
18442
|
+
guard: deps.guardShutdown,
|
|
18443
|
+
servingName: await deps.servingName(indexName),
|
|
18444
|
+
publicName: indexName,
|
|
18445
|
+
change
|
|
18446
|
+
})
|
|
18447
|
+
);
|
|
18448
|
+
},
|
|
17692
18449
|
async rebalance(indexName, targetPartitionCount) {
|
|
17693
18450
|
const job = await trackOp(indexName, () => deps.requestRebalance(indexName, targetPartitionCount));
|
|
17694
18451
|
return trackOp(null, () => deps.waitForRebalance(indexName, job));
|
|
@@ -17697,10 +18454,10 @@ function createClusterNodeOperations(deps) {
|
|
|
17697
18454
|
return trackOp(null, () => listIndexesCluster(readDeps));
|
|
17698
18455
|
},
|
|
17699
18456
|
async getStats(indexName) {
|
|
17700
|
-
return
|
|
18457
|
+
return read(indexName, (name) => statsCluster(readDeps, name));
|
|
17701
18458
|
},
|
|
17702
18459
|
async getPartitionStats(indexName) {
|
|
17703
|
-
return
|
|
18460
|
+
return read(indexName, (name) => partitionStatsCluster(readDeps, name));
|
|
17704
18461
|
},
|
|
17705
18462
|
async checkpoint(indexName) {
|
|
17706
18463
|
return served(indexName, (name) => engine.checkpoint(name));
|
|
@@ -17715,16 +18472,24 @@ function createClusterNodeOperations(deps) {
|
|
|
17715
18472
|
engine.off(event, handler);
|
|
17716
18473
|
},
|
|
17717
18474
|
async query(indexName, params) {
|
|
17718
|
-
return
|
|
18475
|
+
return trackOp(
|
|
18476
|
+
indexName,
|
|
18477
|
+
() => readFollowingTheName(
|
|
18478
|
+
names,
|
|
18479
|
+
indexName,
|
|
18480
|
+
(name) => queryCluster(readDeps, name, params, indexName),
|
|
18481
|
+
(answer) => leftOutAPartition(answer.coverage)
|
|
18482
|
+
)
|
|
18483
|
+
);
|
|
17719
18484
|
},
|
|
17720
18485
|
async get(indexName, docId) {
|
|
17721
|
-
return
|
|
18486
|
+
return read(indexName, async (name) => (await readClusterDocuments(readDeps, name, [docId])).get(docId));
|
|
17722
18487
|
},
|
|
17723
18488
|
async getMultiple(indexName, docIds) {
|
|
17724
|
-
return
|
|
18489
|
+
return read(indexName, (name) => readClusterDocuments(readDeps, name, docIds));
|
|
17725
18490
|
},
|
|
17726
18491
|
async has(indexName, docId) {
|
|
17727
|
-
return
|
|
18492
|
+
return read(indexName, async (name) => (await readClusterDocuments(readDeps, name, [docId])).has(docId));
|
|
17728
18493
|
}
|
|
17729
18494
|
};
|
|
17730
18495
|
}
|
|
@@ -18104,9 +18869,9 @@ function createPrimaryNumbering(onNumbered) {
|
|
|
18104
18869
|
admit(indexName, partitionId, documentId, document) {
|
|
18105
18870
|
const scope = scopes.get(replicationLogKey(indexName, partitionId));
|
|
18106
18871
|
if (scope === void 0) return;
|
|
18107
|
-
const
|
|
18108
|
-
scope.log().assertFits({ indexName, documentId, document:
|
|
18109
|
-
scope.admitted.set(documentId,
|
|
18872
|
+
const encoded2 = document === null ? null : encodeDocument(document);
|
|
18873
|
+
scope.log().assertFits({ indexName, documentId, document: encoded2 });
|
|
18874
|
+
scope.admitted.set(documentId, encoded2);
|
|
18110
18875
|
},
|
|
18111
18876
|
numberEntry(record) {
|
|
18112
18877
|
const scope = scopes.get(replicationLogKey(record.indexName, record.partitionId));
|
|
@@ -18902,7 +19667,8 @@ async function createClusterNode(callerConfig) {
|
|
|
18902
19667
|
const trackOp = operationTracker.track;
|
|
18903
19668
|
const transitionIndex = operationTracker.transition;
|
|
18904
19669
|
const bm25View = createBm25View({ coordinator: config.coordinator, engine, onError: forwardOnError });
|
|
18905
|
-
const
|
|
19670
|
+
const analysisView = createAnalysisView({ coordinator: config.coordinator, engine, onError: forwardOnError });
|
|
19671
|
+
const readDeps = { config: routingConfig2, nodeId, engine, resolveNodeTargets: resolveNodeTargets3, searchContexts, bm25View, analysisView };
|
|
18906
19672
|
const servingName = (indexName) => servingNameOf(allocationView.routing, indexName);
|
|
18907
19673
|
const cluster = createClusterNamespace({
|
|
18908
19674
|
nodeId,
|
|
@@ -18938,6 +19704,7 @@ async function createClusterNode(callerConfig) {
|
|
|
18938
19704
|
servingName,
|
|
18939
19705
|
requestRebalance: (indexName, count2) => requestRebalance(config, nodeId, indexName, count2),
|
|
18940
19706
|
waitForRebalance: (indexName, job) => waitForSwap(config, indexName, job, guardShutdown),
|
|
19707
|
+
guardShutdown,
|
|
18941
19708
|
readDeps,
|
|
18942
19709
|
writeDeps,
|
|
18943
19710
|
engine,
|
|
@@ -18955,6 +19722,7 @@ async function createClusterNode(callerConfig) {
|
|
|
18955
19722
|
guardShutdown();
|
|
18956
19723
|
await allocationView.start();
|
|
18957
19724
|
await bm25View.start();
|
|
19725
|
+
await analysisView.start();
|
|
18958
19726
|
const dispositions = await reconcileLocalIndexes({
|
|
18959
19727
|
engine,
|
|
18960
19728
|
coordinator: config.coordinator,
|
|
@@ -18964,6 +19732,8 @@ async function createClusterNode(callerConfig) {
|
|
|
18964
19732
|
for (const [indexName, disposition] of dispositions) {
|
|
18965
19733
|
if (disposition === "orphaned") {
|
|
18966
19734
|
orphanedIndexes.add(indexName);
|
|
19735
|
+
} else if (disposition === "adopted") {
|
|
19736
|
+
await analysisView.settle(indexName);
|
|
18967
19737
|
}
|
|
18968
19738
|
}
|
|
18969
19739
|
if (hasDataRole) {
|
|
@@ -18980,6 +19750,7 @@ async function createClusterNode(callerConfig) {
|
|
|
18980
19750
|
engine,
|
|
18981
19751
|
coordinator: config.coordinator,
|
|
18982
19752
|
allocations: allocationView,
|
|
19753
|
+
analysisView,
|
|
18983
19754
|
writeDeps,
|
|
18984
19755
|
snapshotSyncState: snapshotSyncHandlerState,
|
|
18985
19756
|
searchContexts,
|
|
@@ -19014,14 +19785,11 @@ async function createClusterNode(callerConfig) {
|
|
|
19014
19785
|
await operationTracker.drain();
|
|
19015
19786
|
allocationView.stop();
|
|
19016
19787
|
bm25View.stop();
|
|
19017
|
-
|
|
19018
|
-
|
|
19019
|
-
|
|
19020
|
-
|
|
19021
|
-
|
|
19022
|
-
unregisterNotReadyHandler();
|
|
19023
|
-
unregisterNotReadyHandler = null;
|
|
19024
|
-
}
|
|
19788
|
+
analysisView.stop();
|
|
19789
|
+
unregisterHandler?.();
|
|
19790
|
+
unregisterHandler = null;
|
|
19791
|
+
unregisterNotReadyHandler?.();
|
|
19792
|
+
unregisterNotReadyHandler = null;
|
|
19025
19793
|
unwatchDepartures?.();
|
|
19026
19794
|
unwatchDepartures = null;
|
|
19027
19795
|
searchContexts?.stop();
|
|
@@ -19134,6 +19902,9 @@ function clusterNodeEngine(node, options) {
|
|
|
19134
19902
|
async rebuildAnalysis(indexName) {
|
|
19135
19903
|
return node.rebuildAnalysis(indexName);
|
|
19136
19904
|
},
|
|
19905
|
+
async changeAnalysis(indexName, change) {
|
|
19906
|
+
return node.changeAnalysis(indexName, change);
|
|
19907
|
+
},
|
|
19137
19908
|
async getMemoryStats() {
|
|
19138
19909
|
return node.getMemoryStats();
|
|
19139
19910
|
},
|