@delali/narsil 0.3.1-next.162 → 0.3.1-next.169
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -8
- package/dist/browser/{chunk-MZHQ65VI.mjs → chunk-2Q45HI5H.mjs} +14 -10
- package/dist/browser/{chunk-MMHIIL5B.mjs → chunk-4ZICWS4V.mjs} +1 -1
- package/dist/browser/{chunk-KQXKCMGQ.mjs → chunk-JHFBQKUF.mjs} +119 -90
- package/dist/browser/{core-leaf-SIWXFZLV.mjs → core-leaf-WYFANMKC.mjs} +61 -13
- package/dist/browser/custom-token-results-R5BIX7SL.mjs +59 -0
- package/dist/browser/{embed-RF2MPGAO.mjs → embed-24IPR4LP.mjs} +2 -2
- package/dist/browser/{module-PMQBRESM.mjs → module-XVLHJ7T7.mjs} +358 -62
- package/dist/browser/{optional-writes-WMZX4OEC.mjs → optional-writes-TB34FYUW.mjs} +118 -66
- package/dist/browser/{phrase-module-22TKMXWI.mjs → phrase-module-6UGFVW5J.mjs} +1 -0
- package/dist/browser/{resolve-vector-text-KI65IHMD.mjs → resolve-vector-text-EJ6DHFF5.mjs} +8 -0
- package/dist/browser/{storage-leaf-VGR2BWJJ.mjs → storage-leaf-QFXIWDFK.mjs} +208 -87
- package/dist/browser/{worker-copies-WQCPP3TS.mjs → worker-copies-ICHLYYXM.mjs} +1 -0
- package/dist/{capabilities-DkXAkb-X.d.ts → capabilities-BQIdsaWy.d.ts} +2 -2
- package/dist/{chunk-BKOMO7HT.mjs → chunk-26E5MO3Y.mjs} +5 -5
- package/dist/{chunk-EEYDJB4Z.mjs → chunk-2C3BXZGA.mjs} +213 -149
- package/dist/{chunk-NXMGUDZA.mjs → chunk-2KWTQ2P7.mjs} +2 -2
- package/dist/{chunk-TW543F33.mjs → chunk-34IOOYXA.mjs} +1 -1
- package/dist/{chunk-D5GNFN6X.mjs → chunk-7FC2ZILY.mjs} +98 -40
- package/dist/{chunk-XRBPHCCT.mjs → chunk-A4XYDUVN.mjs} +5 -5
- package/dist/{chunk-R4F7ZTVG.mjs → chunk-BIIROQ5Q.mjs} +2 -2
- package/dist/{chunk-Q2WQSAYH.mjs → chunk-CQX5B6L2.mjs} +54 -11
- package/dist/{chunk-XXWEUXE4.mjs → chunk-D6FYUTFN.mjs} +1 -1
- package/dist/{chunk-OUWZKNO3.mjs → chunk-DG6WIEB3.mjs} +1 -1
- package/dist/{chunk-GJKFXKMI.mjs → chunk-F3FDHXCJ.mjs} +12 -9
- package/dist/{chunk-3UGZBRDN.mjs → chunk-GO6TRSJ6.mjs} +1 -1
- package/dist/{react-native/chunks/chunk-OK3WUA6N.mjs → chunk-I2WJCRIH.mjs} +1 -1
- package/dist/{chunk-Q7CUUSJ2.mjs → chunk-ISJ35523.mjs} +2 -2
- package/dist/{chunk-GQLOEACG.mjs → chunk-JCWC75NH.mjs} +2 -2
- package/dist/{chunk-ZGYF37MG.mjs → chunk-JDIATDPY.mjs} +1 -1
- package/dist/{chunk-3B775F3C.mjs → chunk-JP77OWMI.mjs} +3 -1
- package/dist/{chunk-KSOZJEYU.mjs → chunk-KHJPMZE2.mjs} +1 -1
- package/dist/{chunk-ZQ6QY42L.mjs → chunk-KL6W73BZ.mjs} +8 -0
- package/dist/{chunk-ES5EOXT6.mjs → chunk-LOQHA6QK.mjs} +3 -3
- package/dist/{chunk-AB27FWZM.mjs → chunk-LPRRFCVU.mjs} +2 -2
- package/dist/{chunk-EIR3IQCL.mjs → chunk-QFJMRB7D.mjs} +2 -1
- package/dist/{chunk-S3BSDBF7.mjs → chunk-QJGDB3WR.mjs} +1 -1
- package/dist/{chunk-EI7U43MX.mjs → chunk-R7PLSVK3.mjs} +2 -2
- package/dist/{chunk-5DUTOEK5.mjs → chunk-TBOZHYCF.mjs} +2 -2
- package/dist/{chunk-VJOS6MHF.mjs → chunk-TNEU32EN.mjs} +64 -16
- package/dist/{chunk-RXMR2SIQ.mjs → chunk-UISV46WX.mjs} +4 -4
- package/dist/{chunk-4CAMH7FO.mjs → chunk-X4UMDSB4.mjs} +2 -1
- package/dist/{chunk-R4CE4IVU.mjs → chunk-X5NHWOXT.mjs} +15 -11
- package/dist/{react-native/chunks/chunk-HEELWLKL.mjs → chunk-XFOL77WH.mjs} +1 -1
- package/dist/{chunk-TOZENMM7.mjs → chunk-XITVDRRH.mjs} +5 -5
- package/dist/{chunk-FSLQHOKZ.mjs → chunk-XNFBYAXI.mjs} +1 -1
- package/dist/{chunk-V4YIEMAI.mjs → chunk-XQQKETAF.mjs} +4 -4
- package/dist/{chunk-DSXFN5BF.mjs → chunk-XTSFRWAS.mjs} +8 -8
- package/dist/{chunk-K4BMDPPX.mjs → chunk-Y27E2MGC.mjs} +2 -2
- package/dist/{chunk-JV74RKCY.mjs → chunk-YM2HI5N5.mjs} +1 -1
- package/dist/{chunk-7OE73OO4.mjs → chunk-YPV2L7M5.mjs} +56 -6
- package/dist/{chunk-STLNPJXU.mjs → chunk-YXZDTKQS.mjs} +2 -2
- package/dist/{chunk-5P72V66D.mjs → chunk-ZB5X5WAK.mjs} +24 -20
- package/dist/{client-VJmV_AjZ.d.ts → client-CeUxvIvd.d.ts} +4 -4
- package/dist/client.d.ts +8 -8
- package/dist/{config-ZZg4cwQz.d.ts → config-CIpLXXxm.d.ts} +2 -2
- package/dist/{config-Cb3FOlE_.d.ts → config-DehU-VGL.d.ts} +2 -2
- package/dist/{core-leaf-LOXCD7MY.mjs → core-leaf-VIMO377O.mjs} +3 -3
- package/dist/custom-token-results-NXHBVY7U.mjs +59 -0
- package/dist/{direct-executor-PF4UDSUW.mjs → direct-executor-2WQSGBN3.mjs} +7 -7
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution/transport/tcp.mjs +1 -1
- package/dist/distribution.d.ts +7 -7
- package/dist/distribution.mjs +39 -39
- package/dist/{embed-4H7RTBVL.mjs → embed-UNHDFGDU.mjs} +2 -2
- package/dist/{engine-DdOs682x.d.ts → engine-P-4xhUe4.d.ts} +6 -2
- package/dist/{index-Ck9w5WHz.d.ts → index-CBUl4z3E.d.ts} +1 -1
- package/dist/index.browser.mjs +3 -3
- package/dist/index.d.ts +6 -6
- package/dist/index.mjs +7 -7
- package/dist/index.react-native.mjs +6 -6
- package/dist/{long-query-thread-OTVLP2OI.mjs → long-query-thread-AAYTLYZK.mjs} +19 -19
- package/dist/{module-UIKSHY3G.mjs → module-5SXNUCD5.mjs} +315 -64
- package/dist/{msgpack-leaf-5WFRP6JM.mjs → msgpack-leaf-7KBYOVTF.mjs} +2 -2
- package/dist/{optional-reads-RSLNSGNG.mjs → optional-reads-ENKMVQ4Q.mjs} +1 -1
- package/dist/{optional-writes-4ITCV7YT.mjs → optional-writes-N2DFWWZ3.mjs} +39 -26
- package/dist/{paged-partition-archive-VMJ2AUA6.mjs → paged-partition-archive-X7L27E5Z.mjs} +19 -19
- package/dist/partial-words.browser.mjs +3 -3
- package/dist/partial-words.mjs +7 -7
- package/dist/pattern.browser.mjs +2 -2
- package/dist/pattern.mjs +2 -2
- package/dist/persistence/durability/checkpoint-worker.d.ts +1 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +17 -17
- package/dist/{phrase-module-IWSG2DDZ.mjs → phrase-module-ZY3TLTQA.mjs} +2 -1
- package/dist/{query-actions-EW72OQXY.mjs → query-actions-2JOLPELC.mjs} +7 -7
- package/dist/query-suggestions/merge-worker.mjs +10 -10
- package/dist/query-suggestions.browser.mjs +3 -3
- package/dist/query-suggestions.d.ts +2 -2
- package/dist/query-suggestions.mjs +8 -8
- package/dist/{chunk-ZBVHS5KF.mjs → react-native/chunks/chunk-57R74JWC.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-AUXXPSFB.mjs → chunk-F4MIGBBP.mjs} +214 -150
- package/dist/react-native/chunks/{chunk-3RUXKZJG.mjs → chunk-F6BJYCF2.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-RVHCQJQM.mjs → chunk-GGBATERA.mjs} +3 -3
- package/dist/react-native/chunks/{chunk-I5W7A2JF.mjs → chunk-GVWLACAT.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-XOCM3BFU.mjs → chunk-H344ZXRH.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-VIPNDN4B.mjs → chunk-I7DTUOK2.mjs} +65 -15
- package/dist/react-native/chunks/{chunk-2JMAUU26.mjs → chunk-J2BQTJJE.mjs} +1 -1
- package/dist/{chunk-QBVTYY4J.mjs → react-native/chunks/chunk-OZ3XEM6D.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-4EXKQ2IB.mjs → chunk-SK3DPAI4.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-XBTFK3AZ.mjs → chunk-WFQ6YQZA.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-FTQ7JNMU.mjs → chunk-ZDDJVR7X.mjs} +15 -11
- package/dist/react-native/chunks/{core-leaf-O3236QY3.mjs → core-leaf-UDUMVC3M.mjs} +2 -2
- package/dist/react-native/chunks/custom-token-results-3OZEMRE7.mjs +59 -0
- package/dist/react-native/chunks/{embed-3WQ2CCR5.mjs → embed-U7CIKEY4.mjs} +2 -2
- package/dist/react-native/chunks/{module-LRR7D5ZO.mjs → module-7BJ3VOWQ.mjs} +360 -66
- package/dist/react-native/chunks/{msgpack-leaf-TR66ZVAX.mjs → msgpack-leaf-GTKLF6GJ.mjs} +2 -2
- package/dist/react-native/chunks/{optional-reads-TOWGNEDH.mjs → optional-reads-74HYI4DO.mjs} +1 -1
- package/dist/react-native/chunks/{optional-writes-7AFBIZMN.mjs → optional-writes-TXTJ6HUZ.mjs} +37 -24
- package/dist/react-native/chunks/{phrase-module-3A4AKMKL.mjs → phrase-module-UC4YGEBB.mjs} +2 -1
- package/dist/react-native/chunks/{resolve-vector-text-LXRYXF7M.mjs → resolve-vector-text-FTPD5EY6.mjs} +8 -0
- package/dist/react-native/chunks/{restore-queue-7FSHTGJ5.mjs → restore-queue-SZLWIL3B.mjs} +9 -9
- package/dist/react-native/chunks/{segment-codec-leaf-4QV4HO4R.mjs → segment-codec-leaf-WMUZ5P4F.mjs} +7 -7
- package/dist/react-native/chunks/{segment-merge-leaf-5DUZZSXJ.mjs → segment-merge-leaf-Q2OEJIQ3.mjs} +8 -8
- package/dist/react-native/chunks/{storage-leaf-PNU6HWY7.mjs → storage-leaf-OXYV6O6P.mjs} +213 -100
- package/dist/react-native/chunks/{store-leaf-ESVUGYYO.mjs → store-leaf-ZMDXYK4E.mjs} +8 -8
- package/dist/react-native/chunks/{suggestions-leaf-MOVOPBGD.mjs → suggestions-leaf-FIMGQZ2T.mjs} +8 -8
- package/dist/react-native/chunks/{worker-copies-VQGF3WZX.mjs → worker-copies-72TGFYQB.mjs} +8 -7
- package/dist/react-native/partial-words.mjs +6 -6
- package/dist/react-native/pattern.mjs +3 -3
- package/dist/react-native/query-suggestions.mjs +6 -6
- package/dist/react-native/sync.mjs +6 -6
- package/dist/react-native/synonyms.mjs +6 -6
- package/dist/react.d.ts +9 -9
- package/dist/{rerank-QHJXPSAK.mjs → rerank-E2HFKKGJ.mjs} +7 -7
- package/dist/{resolve-vector-text-4PDRTPNQ.mjs → resolve-vector-text-6WPFFIX5.mjs} +1 -1
- package/dist/{restore-queue-5W5HX44H.mjs → restore-queue-PUVBN6L3.mjs} +10 -10
- package/dist/{schema-C_7yRJLt.d.ts → schema-CzG9OUw5.d.ts} +5 -1
- package/dist/{segment-codec-leaf-7VLVXKWO.mjs → segment-codec-leaf-24Z3MMQI.mjs} +8 -8
- package/dist/{segment-merge-leaf-22XR3IGA.mjs → segment-merge-leaf-FQQDU45E.mjs} +8 -8
- package/dist/server.d.ts +10 -10
- package/dist/server.mjs +26 -26
- package/dist/{storage-leaf-ZZ77C3DX.mjs → storage-leaf-KIA7XO6Y.mjs} +79 -81
- package/dist/{store-leaf-LJ6D2PMF.mjs → store-leaf-ESVLL4JO.mjs} +9 -9
- package/dist/{suggestions-leaf-KTQHO3PY.mjs → suggestions-leaf-FXF4AHAY.mjs} +11 -11
- package/dist/sync/filesystem.d.ts +3 -3
- package/dist/sync/filesystem.mjs +15 -15
- package/dist/sync/indexeddb.browser.mjs +1 -1
- package/dist/sync/indexeddb.d.ts +3 -3
- package/dist/sync/indexeddb.mjs +2 -2
- package/dist/sync.browser.mjs +3 -3
- package/dist/sync.d.ts +4 -4
- package/dist/sync.mjs +7 -7
- package/dist/synonyms.browser.mjs +3 -3
- package/dist/synonyms.mjs +7 -7
- package/dist/{tasks-CxA-J_Ty.d.ts → tasks-CXtQ3Zpp.d.ts} +4 -4
- package/dist/thread-OWXVP2ZA.mjs +30 -0
- package/dist/{types-BboB278g.d.ts → types-CDN70Y_C.d.ts} +2 -2
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +5 -5
- package/dist/{worker-copies-ODAAGQRC.mjs → worker-copies-XXX7PF4J.mjs} +10 -10
- package/dist/workers/entry.mjs +13 -13
- package/package.json +9 -8
- package/dist/thread-XWDK6TAR.mjs +0 -30
- /package/dist/{chunk-77AUVDQ5.mjs → chunk-CV3Q2EKO.mjs} +0 -0
|
@@ -1,20 +1,20 @@
|
|
|
1
1
|
import { sha256 } from './chunk-H3MDVPBO.mjs';
|
|
2
2
|
import { oneSyncChangeAtATime } from './chunk-Q32DDFKX.mjs';
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import { resolveIndexAnalysis, getLanguage, emitEngineEvent, reconstructSchemaFromMetadata, prepareStoredSynonyms, rulesOfStoredSynonyms, resolvePartialWordAnalysis, generateId, wrapFrozenTokenTable, createEncodedDocumentSource, assemblePagedFrozenSegment, isCompositePartition, yieldToEventLoop, decodeStoredDocument, buildPartialWordTableFrom, configurationPayloadOf, withOpenIndex, emitSyncChange, newSyncEpoch, mergePartitionStats, resolvePartitionInsertOptions, createPartitionIndex, createCompositePartition, prepareDocumentVectors, removedSurfaceSpellings, prepareConfiguredSynonyms, decodeSurfaceTable, mergeDocFrequencies, TermLookup, createFrozenRemovals, mergeKeywordSources, createSharedFrozenSegment, ARRAY_POSITION_GAP, createFrozenSegment, createPublicationGate, prepareRemovalText, prepareDocumentText, writePartitionDocument, updateDocumentVectors, oversizedConfigurationPayloadOf, requireEncodedConfigurationWithinSizeLimit, readHeapStatistics, readResidentMemory, holdSyncEpochChange, emptyPendingRegions, mergePostings2, scanPostingBlocks, SegmentRefusal, VERSION, DEFAULT_BM25_B, DEFAULT_BM25_K1, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateRequiredFieldsInSchema, validateEmbeddingConfig, storedArrayPositionGap, endOfValue, compareScores, isSyncEpoch,
|
|
8
|
-
import './chunk-
|
|
9
|
-
import { isDead, VECTOR_FILE_PAYLOAD_VERSION, bytesToVectors, STATISTICS_BROADCAST_INTERVAL_MS, turnsAreBudgeted, turnIsSpent, yieldTurn, EXPORT_CHUNK_BYTES, countWrite, ANALYSIS_CHANGE_MEMBERS, collectAndReturnPages, osqRecordBytes, createFixedBuffer, FOREIGN_STATS_TTL_MS, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, ANALYSIS_HOLD_APPLY_INTERVAL_MS, magnitude, rescoreDepth, defaultEfSearch, computeCodeBlockLayout, toScore, createVectorFileReader, MAX_IMPORT_REFUSALS_REPORTED, decodeVectorIndexParts, signalOfSearch, requireSearchActive, osqQuantize, createOsqScratch, stageQueryLevels, scalarQueryProducts, readTrailer, osqEstimate, osqDistance, toDistance, decodeVectorFilePayload, DEFAULT_VECTOR_METRIC, defaultQuantizationFor, VECTOR_FILE_MAX_VECTORS, deadBytesFor, waitOutsideCountedWrite, MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, vectorsPerFile, vectorsToBytes, decodeVectorGraphPartPayload, assembleGraphParts, nodeThreadId, fnv1a, osqBitsOf, countDead, detectRuntime,
|
|
10
|
-
export { createHNSWIndex, osqBitsOf } from './chunk-
|
|
11
|
-
import './chunk-
|
|
12
|
-
import { nativeSearchCoreIsEnabled, loadNativeCore } from './chunk-
|
|
3
|
+
import { HEADER_SIZE, requireWithinFileLimit, crc32Init, crc32Update, crc32Final, engineVersionParts, writeHeader, crc32, QUERY_HISTORY_ARCHIVE_PREFIX, MAX_HISTORY_FILE_BYTES, openChecksummedEnvelope, packChecksummedEnvelope, hasNrslMagic, readHeader, storedStateOf, attachedRuntimeOf, historyFilesOf, listHistoryFolder, historyFileOf, QUERY_HISTORY_FOLDER, packEnvelopeWithChecksum, checksumOfChunks, storedStateToWire, storedStateFromWire } from './chunk-F6BJYCF2.mjs';
|
|
4
|
+
import { DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE, rewriteSpillOf, REBALANCE_CHUNK_SIZE, rebuildTextIndexes, textRebuildTargetOf } from './chunk-57R74JWC.mjs';
|
|
5
|
+
import { deserializePartition } from './chunk-H344ZXRH.mjs';
|
|
6
|
+
import { mergeFieldIndexes, selectionKeeps, fieldIndexRemapOf, mergeSourceSpellings, mergeSourceSurfaces, mergedPartialSpellings, shareSegmentArrays, freezeSegmentShared, holdsADiscardedField, toError, segmentArraysOf, mergeSegmentContents, buildSegmentPayloads, createRequestId } from './chunk-GGBATERA.mjs';
|
|
7
|
+
import { resolveIndexAnalysis, getLanguage, emitEngineEvent, reconstructSchemaFromMetadata, prepareStoredSynonyms, rulesOfStoredSynonyms, resolvePartialWordAnalysis, loadSchemaCodeFor, generateId, wrapFrozenTokenTable, createEncodedDocumentSource, assemblePagedFrozenSegment, isCompositePartition, yieldToEventLoop, decodeStoredDocument, buildPartialWordTableFrom, configurationPayloadOf, withOpenIndex, emitSyncChange, newSyncEpoch, mergePartitionStats, resolvePartitionInsertOptions, createPartitionIndex, createCompositePartition, prepareDocumentVectors, removedSurfaceSpellings, prepareConfiguredSynonyms, decodeSurfaceTable, mergeDocFrequencies, TermLookup, createFrozenRemovals, mergeKeywordSources, createSharedFrozenSegment, ARRAY_POSITION_GAP, createFrozenSegment, createPublicationGate, prepareRemovalText, prepareDocumentText, writePartitionDocument, updateDocumentVectors, oversizedConfigurationPayloadOf, requireEncodedConfigurationWithinSizeLimit, readHeapStatistics, readResidentMemory, holdSyncEpochChange, emptyPendingRegions, mergePostings2, scanPostingBlocks, SegmentRefusal, VERSION, DEFAULT_BM25_B, DEFAULT_BM25_K1, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateRequiredFieldsInSchema, validateEmbeddingConfig, storedArrayPositionGap, endOfValue, compareScores, isSyncEpoch, decodeEveryStoredDocument, loadGzip, createPartitionManager, createPartitionRouter, arrayPositionsAreStale, insertDocumentVectors, decodeMapWithoutFields, survivorDocumentsOf, partitionStateOf, removeDocumentVectors } from './chunk-F4MIGBBP.mjs';
|
|
8
|
+
import './chunk-J2BQTJJE.mjs';
|
|
9
|
+
import { DEFAULT_MAX_PARTITIONS, isDead, VECTOR_FILE_PAYLOAD_VERSION, bytesToVectors, MAX_PARTITIONS, STATISTICS_BROADCAST_INTERVAL_MS, turnsAreBudgeted, turnIsSpent, yieldTurn, EXPORT_CHUNK_BYTES, countWrite, ANALYSIS_CHANGE_MEMBERS, collectAndReturnPages, requireBoundedUpperLists, osqRecordBytes, createFixedBuffer, FOREIGN_STATS_TTL_MS, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, ANALYSIS_HOLD_APPLY_INTERVAL_MS, magnitude, rescoreDepth, defaultEfSearch, computeCodeBlockLayout, toScore, createVectorFileReader, MAX_IMPORT_REFUSALS_REPORTED, decodeVectorIndexParts, signalOfSearch, requireSearchActive, osqQuantize, createOsqScratch, stageQueryLevels, scalarQueryProducts, readTrailer, osqEstimate, osqDistance, toDistance, decodeVectorFilePayload, DEFAULT_VECTOR_METRIC, defaultQuantizationFor, VECTOR_FILE_MAX_VECTORS, deadBytesFor, waitOutsideCountedWrite, MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, vectorsPerFile, vectorsToBytes, decodeVectorGraphPartPayload, assembleGraphParts, nodeThreadId, fnv1a, osqBitsOf, countDead, detectRuntime, graphPartsOfEarlierPayload, decodeVectorGraphPayload } from './chunk-I7DTUOK2.mjs';
|
|
10
|
+
export { createHNSWIndex, osqBitsOf } from './chunk-I7DTUOK2.mjs';
|
|
11
|
+
import './chunk-SK3DPAI4.mjs';
|
|
12
|
+
import { nativeSearchCoreIsEnabled, loadNativeCore } from './chunk-WFQ6YQZA.mjs';
|
|
13
13
|
import { reactNativeHost } from './chunk-PVMEHBQO.mjs';
|
|
14
|
-
import { setNestedValue, restoreVectorFields } from './chunk-
|
|
14
|
+
import { setNestedValue, restoreVectorFields } from './chunk-OZ3XEM6D.mjs';
|
|
15
15
|
import { decode, decodeMulti, encodeDocument } from './chunk-YOTP2FPE.mjs';
|
|
16
|
-
import { flattenSchema, validateDocument, validateDocumentStrict, schemaFieldsOf, VECTOR_PATTERN, requireValidFieldTypes, parseVectorDimension, extractVectorFieldsFromSchema, raiseEarlierReleaseGraphSetting, validateSchema, validateVectorPromotion, validateVectorStorage, validateTextStorage, messagePackUpperBoundBytes, partialFieldsOf, MAX_DOCUMENT_NESTING_DEPTH, isQuantizationMode, isStorageMode } from './chunk-
|
|
17
|
-
import { compareCodePoints,
|
|
16
|
+
import { flattenSchema, validateDocument, validateDocumentStrict, schemaFieldsOf, VECTOR_PATTERN, requireValidFieldTypes, parseVectorDimension, extractVectorFieldsFromSchema, raiseEarlierReleaseGraphSetting, validateSchema, validateVectorPromotion, validateVectorStorage, validateTextStorage, messagePackUpperBoundBytes, partialFieldsOf, MAX_DOCUMENT_NESTING_DEPTH, isQuantizationMode, isStorageMode } from './chunk-ZDDJVR7X.mjs';
|
|
17
|
+
import { compareCodePoints, MIN_M, MAX_M, MAX_LAYER_CAP } from './chunk-HRDMIQ63.mjs';
|
|
18
18
|
import { MAX_FILE_BYTES, VALID_HNSW_METRICS } from './chunk-VUJCSQG5.mjs';
|
|
19
19
|
import { NarsilError, ErrorCodes } from './chunk-TQ26BKBH.mjs';
|
|
20
20
|
import { encode } from '@msgpack/msgpack';
|
|
@@ -1218,8 +1218,8 @@ async function executeRebalance(manager, indexName, targetPartitionCount, ctx) {
|
|
|
1218
1218
|
);
|
|
1219
1219
|
}
|
|
1220
1220
|
const entry = ctx.requireIndex(indexName);
|
|
1221
|
-
const maxPartitions = entry.config.partitions?.maxPartitions;
|
|
1222
|
-
if (
|
|
1221
|
+
const maxPartitions = entry.config.partitions?.maxPartitions ?? DEFAULT_MAX_PARTITIONS;
|
|
1222
|
+
if (targetPartitionCount > maxPartitions) {
|
|
1223
1223
|
throw new NarsilError(
|
|
1224
1224
|
ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
|
|
1225
1225
|
`Target partition count (${targetPartitionCount}) is above the maximum of ${maxPartitions} partitions`,
|
|
@@ -1524,6 +1524,9 @@ async function driveRun(core, indexName, entry, run, previous, resumedGeneration
|
|
|
1524
1524
|
}
|
|
1525
1525
|
}
|
|
1526
1526
|
async function changeIndexAnalysis(core, indexName, change) {
|
|
1527
|
+
const changedTokenizer = change !== null && typeof change === "object" ? change.tokenizer : void 0;
|
|
1528
|
+
const loadingTokenizerCode = loadSchemaCodeFor(null, changedTokenizer);
|
|
1529
|
+
if (loadingTokenizerCode !== null) await loadingTokenizerCode;
|
|
1527
1530
|
const entry = core.requireIndex(indexName);
|
|
1528
1531
|
requireNotRebalancing(core, indexName);
|
|
1529
1532
|
requireNoRestoreInFlight(core, indexName);
|
|
@@ -1545,6 +1548,8 @@ async function changeIndexAnalysis(core, indexName, change) {
|
|
|
1545
1548
|
}
|
|
1546
1549
|
function resumeAnalysisChange(core, indexName) {
|
|
1547
1550
|
const entry = core.requireIndex(indexName);
|
|
1551
|
+
const loadingTokenizerCode = loadSchemaCodeFor(null, entry.pendingAnalysis?.tokenizer);
|
|
1552
|
+
if (loadingTokenizerCode !== null) return loadingTokenizerCode.then(() => resumeAnalysisChange(core, indexName));
|
|
1548
1553
|
const runs = runsOf(core);
|
|
1549
1554
|
const running = runs.get(indexName);
|
|
1550
1555
|
if (running !== void 0) return running.finished;
|
|
@@ -1594,6 +1599,8 @@ function startHeldRun(core, indexName, pending) {
|
|
|
1594
1599
|
return run;
|
|
1595
1600
|
}
|
|
1596
1601
|
async function buildHeldAnalysisChange(core, indexName, pending) {
|
|
1602
|
+
const loadingTokenizerCode = loadSchemaCodeFor(null, pending.tokenizer);
|
|
1603
|
+
if (loadingTokenizerCode !== null) await loadingTokenizerCode;
|
|
1597
1604
|
const running = runsOf(core).get(indexName);
|
|
1598
1605
|
const entry = core.indexRegistry.get(indexName);
|
|
1599
1606
|
if (running?.hold != null && entry !== void 0 && holdServes(running, entry, pending)) return running.hold.ready;
|
|
@@ -2678,7 +2685,7 @@ function wireToMetadata(raw) {
|
|
|
2678
2685
|
if (raw.partition_limits && typeof raw.partition_limits === "object") {
|
|
2679
2686
|
const limits = {
|
|
2680
2687
|
...isPositiveInteger(raw.partition_limits.max_docs_per_partition) ? { maxDocsPerPartition: raw.partition_limits.max_docs_per_partition } : {},
|
|
2681
|
-
...isPositiveInteger(raw.partition_limits.max_partitions) ? { maxPartitions: raw.partition_limits.max_partitions } : {},
|
|
2688
|
+
...isPositiveInteger(raw.partition_limits.max_partitions) ? { maxPartitions: Math.min(raw.partition_limits.max_partitions, MAX_PARTITIONS) } : {},
|
|
2682
2689
|
...isWatermarkFraction(raw.partition_limits.watermark) ? { watermark: raw.partition_limits.watermark } : {}
|
|
2683
2690
|
};
|
|
2684
2691
|
if (Object.keys(limits).length > 0) {
|
|
@@ -4568,7 +4575,7 @@ function openPagedFile(path, cache) {
|
|
|
4568
4575
|
const native = openNativePagedFile(path);
|
|
4569
4576
|
readersOfPath.set(path, (readersOfPath.get(path) ?? 0) + 1);
|
|
4570
4577
|
try {
|
|
4571
|
-
return { path, fileNumber: cache.fileNumber(), cache, native, closed: false };
|
|
4578
|
+
return { path, fileNumber: cache.fileNumber(), cache, native, closed: false, suspended: false };
|
|
4572
4579
|
} catch (error) {
|
|
4573
4580
|
native?.core.closePagedFile(native.handle);
|
|
4574
4581
|
stopReadingPath(path);
|
|
@@ -4584,27 +4591,31 @@ function stopReadingPath(path) {
|
|
|
4584
4591
|
readersOfPath.delete(path);
|
|
4585
4592
|
fileReaderOfThisThread().retainOnly(new Set(readersOfPath.keys()));
|
|
4586
4593
|
}
|
|
4587
|
-
function
|
|
4588
|
-
if (file.closed) return;
|
|
4589
|
-
file.closed = true;
|
|
4594
|
+
function closeNativeHandle(file) {
|
|
4590
4595
|
if (file.native !== null) file.cache.releasePageFile(file.native.handle.identity);
|
|
4591
4596
|
file.native?.core.closePagedFile(file.native.handle);
|
|
4592
4597
|
file.native = null;
|
|
4598
|
+
}
|
|
4599
|
+
function closePagedFile(file) {
|
|
4600
|
+
if (file.closed) return;
|
|
4601
|
+
file.closed = true;
|
|
4602
|
+
closeNativeHandle(file);
|
|
4593
4603
|
file.cache.releaseFile(file.fileNumber);
|
|
4604
|
+
if (file.suspended) return;
|
|
4594
4605
|
stopReadingPath(file.path);
|
|
4595
4606
|
}
|
|
4596
|
-
function
|
|
4597
|
-
if (
|
|
4598
|
-
|
|
4599
|
-
|
|
4600
|
-
file.native?.core.closePagedFile(file.native.handle);
|
|
4601
|
-
file.native = native;
|
|
4607
|
+
function suspendPagedFile(file) {
|
|
4608
|
+
if (file.closed || file.suspended) return;
|
|
4609
|
+
file.suspended = true;
|
|
4610
|
+
closeNativeHandle(file);
|
|
4602
4611
|
stopReadingPath(file.path);
|
|
4603
|
-
|
|
4604
|
-
|
|
4605
|
-
|
|
4606
|
-
|
|
4612
|
+
}
|
|
4613
|
+
function resumePagedFile(file, path) {
|
|
4614
|
+
if (file.closed || !file.suspended) return;
|
|
4615
|
+
file.suspended = false;
|
|
4607
4616
|
file.path = path;
|
|
4617
|
+
readersOfPath.set(path, (readersOfPath.get(path) ?? 0) + 1);
|
|
4618
|
+
file.native = openNativePagedFile(path);
|
|
4608
4619
|
}
|
|
4609
4620
|
function unreadable(path, offset, target) {
|
|
4610
4621
|
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The file "${path}" cannot be read at byte ${offset}`, {
|
|
@@ -5229,8 +5240,11 @@ function assembleOpenedSegment(file, snapshot, cache) {
|
|
|
5229
5240
|
close() {
|
|
5230
5241
|
closePagedFile(file);
|
|
5231
5242
|
},
|
|
5232
|
-
|
|
5233
|
-
|
|
5243
|
+
suspend() {
|
|
5244
|
+
suspendPagedFile(file);
|
|
5245
|
+
},
|
|
5246
|
+
resume(rewrite) {
|
|
5247
|
+
resumePagedFile(file, rewrite(file.path));
|
|
5234
5248
|
snapshot.path = file.path;
|
|
5235
5249
|
}
|
|
5236
5250
|
};
|
|
@@ -7360,6 +7374,9 @@ function openParts(input) {
|
|
|
7360
7374
|
}
|
|
7361
7375
|
const { entryPoint, maxLayer, m } = layout;
|
|
7362
7376
|
if (typeof maxLayer !== "number" || typeof m !== "number") loadFailed(`The head of "${ref.key}" holds no layers`);
|
|
7377
|
+
if (!Number.isInteger(m) || m < MIN_M || m > MAX_M) {
|
|
7378
|
+
loadFailed(`The head of "${ref.key}" gives m ${m}, while m must be a whole number from ${MIN_M} to ${MAX_M}`);
|
|
7379
|
+
}
|
|
7363
7380
|
if (entryPoint !== null && typeof entryPoint !== "number") loadFailed(`The head of "${ref.key}" holds no entry`);
|
|
7364
7381
|
if (graph === null) graph = { entryPoint, maxLayer, m };
|
|
7365
7382
|
const levels = readWholeSectionChecked(path, layout.levels, false);
|
|
@@ -7371,6 +7388,13 @@ function openParts(input) {
|
|
|
7371
7388
|
neighbours: createSectionReader(file, layout.neighbours)
|
|
7372
7389
|
});
|
|
7373
7390
|
}
|
|
7391
|
+
if (graph !== null) {
|
|
7392
|
+
requireBoundedUpperLists(
|
|
7393
|
+
parts.map((part) => part.levels),
|
|
7394
|
+
graph.m,
|
|
7395
|
+
`The vector graph parts of field "${input.fieldPath}"`
|
|
7396
|
+
);
|
|
7397
|
+
}
|
|
7374
7398
|
} catch (error) {
|
|
7375
7399
|
closeHeldFiles(parts);
|
|
7376
7400
|
throw error;
|
|
@@ -7411,7 +7435,10 @@ function validateColdLayout(input, opened, parts, graph) {
|
|
|
7411
7435
|
// src/persistence/paged/cold-field/index.ts
|
|
7412
7436
|
var FIXSTR2 = 160;
|
|
7413
7437
|
var FIXSTR_MAX_BYTES = 31;
|
|
7438
|
+
var STR8_MAX_BYTES = 255;
|
|
7439
|
+
var STR16_MAX_BYTES = 65535;
|
|
7414
7440
|
var ASCII_MAX2 = 127;
|
|
7441
|
+
var ID_OFFSET_BYTES = 4;
|
|
7415
7442
|
function shortAsciiIdOf(encoded) {
|
|
7416
7443
|
const length = encoded[0] - FIXSTR2;
|
|
7417
7444
|
if (length < 0 || length > FIXSTR_MAX_BYTES || encoded.length !== length + 1) return null;
|
|
@@ -7516,6 +7543,35 @@ function createFieldData(input, opened, graph, parts) {
|
|
|
7516
7543
|
}
|
|
7517
7544
|
};
|
|
7518
7545
|
}
|
|
7546
|
+
function utf8BytesOfEncodedId(encodedBytes) {
|
|
7547
|
+
if (encodedBytes <= FIXSTR_MAX_BYTES + 1) return encodedBytes - 1;
|
|
7548
|
+
if (encodedBytes <= STR8_MAX_BYTES + 2) return encodedBytes - 2;
|
|
7549
|
+
if (encodedBytes <= STR16_MAX_BYTES + 3) return encodedBytes - 3;
|
|
7550
|
+
return encodedBytes - 5;
|
|
7551
|
+
}
|
|
7552
|
+
function claimsOfLoad(data) {
|
|
7553
|
+
const offsets = new Uint8Array(SEGMENT_PAGE_BYTES);
|
|
7554
|
+
const view = new DataView(offsets.buffer);
|
|
7555
|
+
const offsetsPerRead = offsets.byteLength / ID_OFFSET_BYTES - 1;
|
|
7556
|
+
let slots = 0;
|
|
7557
|
+
let docIdBytes = 0;
|
|
7558
|
+
for (const file of data.files) {
|
|
7559
|
+
for (let first = 0; first < file.count; first += offsetsPerRead) {
|
|
7560
|
+
const count = Math.min(offsetsPerRead, file.count - first);
|
|
7561
|
+
file.idOffsets.read(first * ID_OFFSET_BYTES, offsets.subarray(0, (count + 1) * ID_OFFSET_BYTES));
|
|
7562
|
+
for (let at = 0; at < count; at++) {
|
|
7563
|
+
const position = first + at;
|
|
7564
|
+
if (isDead(file.dead, position) && data.topLayerOf(file.firstNumber + position) < 0) continue;
|
|
7565
|
+
const start = view.getUint32(at * ID_OFFSET_BYTES, true);
|
|
7566
|
+
const end = view.getUint32((at + 1) * ID_OFFSET_BYTES, true);
|
|
7567
|
+
if (end <= start) loadFailed(`The ID offsets of "${file.file.path}" hold no ID at position ${position}`);
|
|
7568
|
+
slots += 1;
|
|
7569
|
+
docIdBytes += utf8BytesOfEncodedId(end - start);
|
|
7570
|
+
}
|
|
7571
|
+
}
|
|
7572
|
+
}
|
|
7573
|
+
return { slots, docIdBytes };
|
|
7574
|
+
}
|
|
7519
7575
|
function positionOfId(file, docId, idAt2, slot, view) {
|
|
7520
7576
|
let low = 0;
|
|
7521
7577
|
let high = file.count - 1;
|
|
@@ -7572,6 +7628,9 @@ function openColdVectorField(input) {
|
|
|
7572
7628
|
}
|
|
7573
7629
|
let residentBytes = 0;
|
|
7574
7630
|
for (const part of parts) residentBytes += part.levels.byteLength + part.nodeOffsets.byteLength;
|
|
7631
|
+
let encodedIdBytes = 0;
|
|
7632
|
+
for (const file of opened.files) encodedIdBytes += file.ids.length;
|
|
7633
|
+
let claimsOnceRead = null;
|
|
7575
7634
|
const orderSlot = new Uint8Array(4);
|
|
7576
7635
|
const orderView = new DataView(orderSlot.buffer);
|
|
7577
7636
|
function liveNumberOf(docId) {
|
|
@@ -7590,6 +7649,11 @@ function openColdVectorField(input) {
|
|
|
7590
7649
|
for (const part of parts) bytes += cachedFileBytes(part.file);
|
|
7591
7650
|
return bytes;
|
|
7592
7651
|
},
|
|
7652
|
+
claimsAtMost: { slots: data.total, docIdBytes: encodedIdBytes },
|
|
7653
|
+
claims() {
|
|
7654
|
+
claimsOnceRead ??= claimsOfLoad(data);
|
|
7655
|
+
return claimsOnceRead;
|
|
7656
|
+
},
|
|
7593
7657
|
numberOf: liveNumberOf,
|
|
7594
7658
|
vectorOfNumber: (number) => data.vectorAt(number),
|
|
7595
7659
|
reusedVectorOfNumber: (number) => data.reusedVectorAt(number),
|
|
@@ -10677,6 +10741,21 @@ async function writeIndexCheckpointSegments(input) {
|
|
|
10677
10741
|
});
|
|
10678
10742
|
}
|
|
10679
10743
|
|
|
10744
|
+
// src/partitioning/saved-partitions.ts
|
|
10745
|
+
function requireSavedPartitionCount(partitionCount, source, indexName) {
|
|
10746
|
+
if (partitionCount <= MAX_PARTITIONS) return;
|
|
10747
|
+
throw new NarsilError(
|
|
10748
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
10749
|
+
`The ${source} of index "${indexName}" holds partition ${partitionCount - 1}, and a saved index numbers its partitions from 0 to ${MAX_PARTITIONS - 1}`,
|
|
10750
|
+
{ indexName, partitionId: partitionCount - 1, maxPartitions: MAX_PARTITIONS }
|
|
10751
|
+
);
|
|
10752
|
+
}
|
|
10753
|
+
function highestPartitionIdOf(partitions) {
|
|
10754
|
+
let highest = partitions.length - 1;
|
|
10755
|
+
for (const partition of partitions) if (partition.partitionId > highest) highest = partition.partitionId;
|
|
10756
|
+
return highest;
|
|
10757
|
+
}
|
|
10758
|
+
|
|
10680
10759
|
// src/persistence/paged/read-ahead.ts
|
|
10681
10760
|
var FILE_PAGE_SIZES2 = /* @__PURE__ */ new Set([SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES]);
|
|
10682
10761
|
function rangeOf(path, section) {
|
|
@@ -11115,21 +11194,11 @@ async function readCurrentManifest(store, indexName) {
|
|
|
11115
11194
|
);
|
|
11116
11195
|
}
|
|
11117
11196
|
function growPartitionsTo(manager, highestPartitionId) {
|
|
11118
|
-
|
|
11119
|
-
const before = manager.partitionCount;
|
|
11120
|
-
try {
|
|
11121
|
-
manager.addPartition();
|
|
11122
|
-
} catch {
|
|
11123
|
-
break;
|
|
11124
|
-
}
|
|
11125
|
-
if (manager.partitionCount === before) break;
|
|
11126
|
-
}
|
|
11197
|
+
manager.growToSavedPartitionCount(highestPartitionId + 1);
|
|
11127
11198
|
}
|
|
11128
11199
|
async function loadSegmentedCheckpoint(store, indexName, manifest, deps, decodesEveryDocument = false) {
|
|
11129
|
-
|
|
11130
|
-
|
|
11131
|
-
if (partition.partitionId > highestPartitionId) highestPartitionId = partition.partitionId;
|
|
11132
|
-
}
|
|
11200
|
+
const highestPartitionId = highestPartitionIdOf(manifest.partitions);
|
|
11201
|
+
requireSavedPartitionCount(highestPartitionId + 1, "segment manifest", indexName);
|
|
11133
11202
|
growPartitionsTo(deps.manager, highestPartitionId);
|
|
11134
11203
|
const { directory } = store;
|
|
11135
11204
|
const opensVectorsOnDemand = manifest.vectors.some(
|
|
@@ -11139,13 +11208,6 @@ async function loadSegmentedCheckpoint(store, indexName, manifest, deps, decodes
|
|
|
11139
11208
|
await startReadingOpeningSectionsAhead(directory, manifest);
|
|
11140
11209
|
}
|
|
11141
11210
|
for (const partition of manifest.partitions) {
|
|
11142
|
-
if (partition.partitionId >= deps.manager.partitionCount) {
|
|
11143
|
-
throw new NarsilError(
|
|
11144
|
-
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
11145
|
-
`Segment manifest references partition ${partition.partitionId} beyond the partition count ${deps.manager.partitionCount}`,
|
|
11146
|
-
{ indexName, partitionId: partition.partitionId, partitionCount: deps.manager.partitionCount }
|
|
11147
|
-
);
|
|
11148
|
-
}
|
|
11149
11211
|
await loadPartitionSegments(store, partition, deps.manager, {
|
|
11150
11212
|
schema: manifest.schema,
|
|
11151
11213
|
decodesEveryDocument,
|
|
@@ -13229,7 +13291,8 @@ function countSnapshotBundleDocuments(bundle) {
|
|
|
13229
13291
|
}
|
|
13230
13292
|
async function loadSnapshotBundle(bytes, deps) {
|
|
13231
13293
|
const bundle = await decodeSnapshotBundle(bytes);
|
|
13232
|
-
|
|
13294
|
+
requireSavedPartitionCount(bundle.partitions.length, "snapshot bundle", deps.manager.indexName);
|
|
13295
|
+
deps.manager.growToSavedPartitionCount(bundle.partitions.length);
|
|
13233
13296
|
if (bundle.partitions.length > 0) deps.manager.trimPartitions(bundle.partitions.length);
|
|
13234
13297
|
for (let i = 0; i < bundle.partitions.length; i += 1) {
|
|
13235
13298
|
deps.manager.deserializePartition(i, deserializePayloadV2(bundle.partitions[i]), deserializePartition);
|
|
@@ -13698,9 +13761,7 @@ async function readGraph(store, ref) {
|
|
|
13698
13761
|
what: `graph file "${ref.graphKey}"`,
|
|
13699
13762
|
allowsCompression: true
|
|
13700
13763
|
});
|
|
13701
|
-
|
|
13702
|
-
if (graph.graphs.some((numbered) => numbered.m < MIN_M)) return null;
|
|
13703
|
-
return graphPartsOfEarlierPayload(graph);
|
|
13764
|
+
return graphPartsOfEarlierPayload(decodeVectorGraphPayload(decode(payload)));
|
|
13704
13765
|
}
|
|
13705
13766
|
async function loadVectorField2(store, ref, vectorIndex, fileRefs) {
|
|
13706
13767
|
let liveVectors = 0;
|
|
@@ -13745,21 +13806,10 @@ async function loadVectorField2(store, ref, vectorIndex, fileRefs) {
|
|
|
13745
13806
|
restore.finish();
|
|
13746
13807
|
}
|
|
13747
13808
|
async function loadEarlierLayout(store, indexName, manifest, deps) {
|
|
13748
|
-
|
|
13749
|
-
|
|
13750
|
-
if (partition.partitionId > highestPartitionId) highestPartitionId = partition.partitionId;
|
|
13751
|
-
}
|
|
13809
|
+
const highestPartitionId = highestPartitionIdOf(manifest.partitions);
|
|
13810
|
+
requireSavedPartitionCount(highestPartitionId + 1, "earlier segment manifest", indexName);
|
|
13752
13811
|
growPartitionsTo(deps.manager, highestPartitionId);
|
|
13753
|
-
for (const partition of manifest.partitions)
|
|
13754
|
-
if (partition.partitionId >= deps.manager.partitionCount) {
|
|
13755
|
-
throw new NarsilError(
|
|
13756
|
-
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
13757
|
-
`Segment manifest references partition ${partition.partitionId} beyond the partition count ${deps.manager.partitionCount}`,
|
|
13758
|
-
{ indexName, partitionId: partition.partitionId }
|
|
13759
|
-
);
|
|
13760
|
-
}
|
|
13761
|
-
await loadPartition(store, indexName, partition, deps);
|
|
13762
|
-
}
|
|
13812
|
+
for (const partition of manifest.partitions) await loadPartition(store, indexName, partition, deps);
|
|
13763
13813
|
const vectorFileRefs = /* @__PURE__ */ new Map();
|
|
13764
13814
|
for (const vector of manifest.vectors) {
|
|
13765
13815
|
const vectorIndex = deps.vectorIndexes.get(vector.fieldPath);
|
|
@@ -16124,7 +16174,7 @@ function restoredPartitionConfig(raw, reject = rejectSnapshot) {
|
|
|
16124
16174
|
}
|
|
16125
16175
|
return {
|
|
16126
16176
|
...typeof maxDocsPerPartition === "number" ? { maxDocsPerPartition } : {},
|
|
16127
|
-
...typeof maxPartitions === "number" ? { maxPartitions } : {},
|
|
16177
|
+
...typeof maxPartitions === "number" ? { maxPartitions: Math.min(maxPartitions, MAX_PARTITIONS) } : {},
|
|
16128
16178
|
...typeof watermark === "number" ? { watermark } : {}
|
|
16129
16179
|
};
|
|
16130
16180
|
}
|
|
@@ -16232,6 +16282,59 @@ function restoredConfigFields(envelope, reject = rejectSnapshot) {
|
|
|
16232
16282
|
};
|
|
16233
16283
|
}
|
|
16234
16284
|
|
|
16285
|
+
// src/engine/snapshot/stored-schema.ts
|
|
16286
|
+
function isRecord4(value) {
|
|
16287
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
16288
|
+
}
|
|
16289
|
+
function conflictingStoredPath(path) {
|
|
16290
|
+
return new NarsilError(
|
|
16291
|
+
ErrorCodes.SCHEMA_INVALID_TYPE,
|
|
16292
|
+
`Field path "${path}" is declared both as a field and as an object of fields`,
|
|
16293
|
+
{ path }
|
|
16294
|
+
);
|
|
16295
|
+
}
|
|
16296
|
+
function defineOwn(level, key, value) {
|
|
16297
|
+
Object.defineProperty(level, key, { value, enumerable: true, writable: true, configurable: true });
|
|
16298
|
+
}
|
|
16299
|
+
function placeStoredField(nested, path, type) {
|
|
16300
|
+
const segments = path.split(".");
|
|
16301
|
+
let level = nested;
|
|
16302
|
+
for (let at = 0; at < segments.length; at++) {
|
|
16303
|
+
const segment = segments[at];
|
|
16304
|
+
const held = Object.hasOwn(level, segment) ? level[segment] : void 0;
|
|
16305
|
+
if (at === segments.length - 1) {
|
|
16306
|
+
if (held !== void 0) throw conflictingStoredPath(path);
|
|
16307
|
+
defineOwn(level, segment, type);
|
|
16308
|
+
return;
|
|
16309
|
+
}
|
|
16310
|
+
if (held === void 0) {
|
|
16311
|
+
const child = {};
|
|
16312
|
+
defineOwn(level, segment, child);
|
|
16313
|
+
level = child;
|
|
16314
|
+
} else if (isRecord4(held)) {
|
|
16315
|
+
level = held;
|
|
16316
|
+
} else {
|
|
16317
|
+
throw conflictingStoredPath(segments.slice(0, at + 1).join("."));
|
|
16318
|
+
}
|
|
16319
|
+
}
|
|
16320
|
+
}
|
|
16321
|
+
function placeStoredFields(nested, stored, prefix) {
|
|
16322
|
+
for (const [key, type] of Object.entries(stored)) {
|
|
16323
|
+
const path = prefix ? `${prefix}.${key}` : key;
|
|
16324
|
+
if (isRecord4(type)) {
|
|
16325
|
+
placeStoredFields(nested, type, path);
|
|
16326
|
+
} else {
|
|
16327
|
+
placeStoredField(nested, path, type);
|
|
16328
|
+
}
|
|
16329
|
+
}
|
|
16330
|
+
}
|
|
16331
|
+
function nestedSchemaOf(stored) {
|
|
16332
|
+
if (!isRecord4(stored)) return stored;
|
|
16333
|
+
const nested = {};
|
|
16334
|
+
placeStoredFields(nested, stored, "");
|
|
16335
|
+
return nested;
|
|
16336
|
+
}
|
|
16337
|
+
|
|
16235
16338
|
// src/persistence/archive/byte-reader.ts
|
|
16236
16339
|
async function* chunksOf(source) {
|
|
16237
16340
|
if (source instanceof Uint8Array) {
|
|
@@ -16453,7 +16556,7 @@ function refuse2(path, reason) {
|
|
|
16453
16556
|
field: path
|
|
16454
16557
|
});
|
|
16455
16558
|
}
|
|
16456
|
-
function
|
|
16559
|
+
function isRecord5(value) {
|
|
16457
16560
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
16458
16561
|
}
|
|
16459
16562
|
function startsWithTagPrefix(key) {
|
|
@@ -16520,7 +16623,7 @@ function untagTag(tag, value, path, depth) {
|
|
|
16520
16623
|
case NUMBER_TAG:
|
|
16521
16624
|
return numberOf(value, path);
|
|
16522
16625
|
case OBJECT_TAG:
|
|
16523
|
-
if (!
|
|
16626
|
+
if (!isRecord5(value)) refuse2(path, `holds an "${OBJECT_TAG}" tag whose value is not an object`);
|
|
16524
16627
|
return untagMap(value, path, depth);
|
|
16525
16628
|
default:
|
|
16526
16629
|
return refuse2(path, `holds an object whose only key, "${tag}", names no tag`);
|
|
@@ -16544,9 +16647,6 @@ var DOCUMENT_EXPORT_VERSION = 1;
|
|
|
16544
16647
|
function invalidHeader(message) {
|
|
16545
16648
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, `The document export header ${message}`);
|
|
16546
16649
|
}
|
|
16547
|
-
function isRecord5(value) {
|
|
16548
|
-
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
16549
|
-
}
|
|
16550
16650
|
function configOfHeader(line) {
|
|
16551
16651
|
if (line === void 0) invalidHeader("is missing");
|
|
16552
16652
|
if (!("text" in line)) {
|
|
@@ -16560,14 +16660,14 @@ function configOfHeader(line) {
|
|
|
16560
16660
|
} catch {
|
|
16561
16661
|
invalidHeader("is not JSON");
|
|
16562
16662
|
}
|
|
16563
|
-
if (!
|
|
16663
|
+
if (!isRecord4(raw) || raw.format !== DOCUMENT_EXPORT_FORMAT || raw.version !== DOCUMENT_EXPORT_VERSION) {
|
|
16564
16664
|
invalidHeader(`must name format "${DOCUMENT_EXPORT_FORMAT}" and version ${DOCUMENT_EXPORT_VERSION}`);
|
|
16565
16665
|
}
|
|
16566
16666
|
const configuration = requireConfigurationShape(raw.index, "document export header");
|
|
16567
16667
|
const fields = restoredConfigFields(configuration, (message) => invalidHeader(message));
|
|
16568
16668
|
const embedding = restoredEmbedding(configuration.embedding, (message) => invalidHeader(message));
|
|
16569
16669
|
return {
|
|
16570
|
-
schema: configuration.schema,
|
|
16670
|
+
schema: nestedSchemaOf(configuration.schema),
|
|
16571
16671
|
language: configuration.language,
|
|
16572
16672
|
...fields,
|
|
16573
16673
|
...embedding === void 0 ? {} : {
|
|
@@ -16653,7 +16753,7 @@ async function takeLine(state, entry) {
|
|
|
16653
16753
|
refuse3(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, "The line is not JSON");
|
|
16654
16754
|
return;
|
|
16655
16755
|
}
|
|
16656
|
-
if (!
|
|
16756
|
+
if (!isRecord4(document) || typeof document.id !== "string" || document.id.length === 0) {
|
|
16657
16757
|
refuse3(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, "The line holds no document with a string id");
|
|
16658
16758
|
return;
|
|
16659
16759
|
}
|
|
@@ -17326,9 +17426,9 @@ function attachablePolicy(shared) {
|
|
|
17326
17426
|
}
|
|
17327
17427
|
};
|
|
17328
17428
|
}
|
|
17329
|
-
function createDetachedIndex(indexName, config, language, synonyms, executor) {
|
|
17429
|
+
function createDetachedIndex(indexName, config, language, synonyms, executor, partitionCount) {
|
|
17330
17430
|
const { policy, attach } = attachablePolicy(executor.vectorWorkerCopies);
|
|
17331
|
-
const entry = executor.createDetachedEntry(indexName, config, language, policy);
|
|
17431
|
+
const entry = executor.createDetachedEntry(indexName, config, language, policy, partitionCount);
|
|
17332
17432
|
entry.manager.synonyms = synonyms;
|
|
17333
17433
|
return { indexName, manager: entry.manager, entry, attach };
|
|
17334
17434
|
}
|
|
@@ -17339,7 +17439,7 @@ function prepareRestoredIndex(configuration, deps, reject) {
|
|
|
17339
17439
|
const embedding = restoredEmbedding(configuration.embedding, reject);
|
|
17340
17440
|
const languageName = configuration.language ?? "";
|
|
17341
17441
|
const language = getLanguage(languageName);
|
|
17342
|
-
const schema = validateSchema(configuration.schema);
|
|
17442
|
+
const schema = validateSchema(nestedSchemaOf(configuration.schema));
|
|
17343
17443
|
validatePatternValueLimit(configFields.patternValueLimit);
|
|
17344
17444
|
requireSurfaceFormsForPartialFields(schema, configFields.surfaceForms);
|
|
17345
17445
|
validateVectorPromotion(configFields.vectorPromotion);
|
|
@@ -17389,17 +17489,17 @@ function restoredSynonyms(raw, config, language, reject) {
|
|
|
17389
17489
|
return prepareStoredSynonyms(raw, config, language, (message) => reject(`synonyms: ${message}`));
|
|
17390
17490
|
}
|
|
17391
17491
|
async function prepareDetachedIndex(indexName, prepared, partitionCount, deps) {
|
|
17392
|
-
|
|
17492
|
+
if (partitionCount !== null) requireSavedPartitionCount(partitionCount, "snapshot", indexName);
|
|
17493
|
+
const loadingSchemaCode = loadSchemaCodeFor(prepared.config.schema, prepared.config.tokenizer);
|
|
17393
17494
|
if (loadingSchemaCode !== null) await loadingSchemaCode;
|
|
17394
|
-
|
|
17495
|
+
return createDetachedIndex(
|
|
17395
17496
|
indexName,
|
|
17396
17497
|
prepared.config,
|
|
17397
17498
|
prepared.language,
|
|
17398
17499
|
prepared.synonyms?.lookup ?? null,
|
|
17399
|
-
deps.executor
|
|
17500
|
+
deps.executor,
|
|
17501
|
+
partitionCount ?? void 0
|
|
17400
17502
|
);
|
|
17401
|
-
while (detached.manager.partitionCount < partitionCount) detached.manager.addPartition();
|
|
17402
|
-
return detached;
|
|
17403
17503
|
}
|
|
17404
17504
|
function restoredRegistryEntry(prepared, manager, deps) {
|
|
17405
17505
|
return {
|
|
@@ -17477,30 +17577,42 @@ function persistedCheckpoint(indexName, partitionCount, deps) {
|
|
|
17477
17577
|
primaryTerm: SINGLE_NODE_PRIMARY_TERM
|
|
17478
17578
|
}));
|
|
17479
17579
|
}
|
|
17480
|
-
|
|
17580
|
+
function pagedSegmentsOf(manager) {
|
|
17581
|
+
const paged = [];
|
|
17582
|
+
if (manager === void 0) return paged;
|
|
17583
|
+
for (let partitionId = 0; partitionId < manager.partitionCount; partitionId++) {
|
|
17584
|
+
const partition = manager.getPartition(partitionId);
|
|
17585
|
+
if (!isCompositePartition(partition)) continue;
|
|
17586
|
+
for (const segment of partition.frozenSegments()) if (segment.paged !== null) paged.push(segment.paged);
|
|
17587
|
+
}
|
|
17588
|
+
return paged;
|
|
17589
|
+
}
|
|
17590
|
+
function samePath(path) {
|
|
17591
|
+
return path;
|
|
17592
|
+
}
|
|
17593
|
+
async function commitToStorage(indexName, restored, previous, deps) {
|
|
17481
17594
|
const durable = deps.durability?.manager;
|
|
17482
17595
|
const staging = restored.staging;
|
|
17483
17596
|
if (durable === void 0 || staging === null) return;
|
|
17484
|
-
const
|
|
17597
|
+
const previousPartitions = previous?.partitionCount ?? 0;
|
|
17598
|
+
const previousCheckpoint = persistedCheckpoint(indexName, previousPartitions, deps);
|
|
17485
17599
|
if (previousPartitions > 0) await durable.unloadIndex(indexName);
|
|
17486
17600
|
const vectorIndexes = [...restored.detached.manager.getVectorIndexes().values()];
|
|
17487
17601
|
try {
|
|
17488
17602
|
for (const vectorIndex of vectorIndexes) await vectorIndex.releaseVectorFiles();
|
|
17489
17603
|
await staging.commit();
|
|
17490
17604
|
} catch (error) {
|
|
17491
|
-
if (previousPartitions > 0) durable.adoptRestoredCheckpoint?.(indexName,
|
|
17605
|
+
if (previousPartitions > 0) durable.adoptRestoredCheckpoint?.(indexName, previousCheckpoint);
|
|
17492
17606
|
throw error;
|
|
17493
17607
|
}
|
|
17608
|
+
for (const paged of pagedSegmentsOf(previous)) paged.close();
|
|
17609
|
+
const restoredSegments = pagedSegmentsOf(restored.detached.manager);
|
|
17610
|
+
for (const paged of restoredSegments) paged.suspend();
|
|
17494
17611
|
await staging.finish();
|
|
17495
17612
|
const relocate = await staging.relocation().catch(() => null);
|
|
17613
|
+
for (const paged of restoredSegments) paged.resume(relocate ?? samePath);
|
|
17496
17614
|
if (relocate === null) return;
|
|
17497
17615
|
for (const vectorIndex of vectorIndexes) vectorIndex.relocateVectorFiles(relocate);
|
|
17498
|
-
const { manager } = restored.detached;
|
|
17499
|
-
for (let partitionId = 0; partitionId < manager.partitionCount; partitionId++) {
|
|
17500
|
-
const partition = manager.getPartition(partitionId);
|
|
17501
|
-
if (!isCompositePartition(partition)) continue;
|
|
17502
|
-
for (const segment of partition.frozenSegments()) segment.paged?.relocate(relocate);
|
|
17503
|
-
}
|
|
17504
17616
|
}
|
|
17505
17617
|
function activate(indexName, restored, entry, deps) {
|
|
17506
17618
|
deps.executor.installDetachedIndex(restored.detached);
|
|
@@ -17530,7 +17642,7 @@ async function replaceExisting(indexName, restored, entry, deps) {
|
|
|
17530
17642
|
const vectorIndexes = previous === void 0 ? [] : vectorIndexesOf2(indexName, deps);
|
|
17531
17643
|
await withinLimit(indexName, release, holdVectorCopies(vectorIndexes, release), deps);
|
|
17532
17644
|
for (const vectorIndex of vectorIndexes) await vectorIndex.releaseVectorFiles();
|
|
17533
|
-
await commitToStorage(indexName, restored, previous
|
|
17645
|
+
await commitToStorage(indexName, restored, previous, deps);
|
|
17534
17646
|
if (previous === void 0) activate(indexName, restored, entry, deps);
|
|
17535
17647
|
else await previous.publication.write(() => activate(indexName, restored, entry, deps));
|
|
17536
17648
|
await deps.querySuggestions?.afterActivate(entry);
|
|
@@ -17546,7 +17658,7 @@ async function createFresh(indexName, restored, entry, deps) {
|
|
|
17546
17658
|
deps.indexRegistry.set(indexName, entry);
|
|
17547
17659
|
try {
|
|
17548
17660
|
await deps.indexState.registerReplacing(indexName, async () => {
|
|
17549
|
-
await commitToStorage(indexName, restored, 0, deps);
|
|
17661
|
+
await commitToStorage(indexName, restored, void 0, deps);
|
|
17550
17662
|
activate(indexName, restored, entry, deps);
|
|
17551
17663
|
await deps.querySuggestions?.afterActivate(entry);
|
|
17552
17664
|
});
|
|
@@ -17817,7 +17929,8 @@ async function stageBuiltIndex(indexName, prepared, built, staging, deps) {
|
|
|
17817
17929
|
return archive.manifest;
|
|
17818
17930
|
}
|
|
17819
17931
|
async function buildRestoredIndex(indexName, envelope, prepared, staging, deps) {
|
|
17820
|
-
const
|
|
17932
|
+
const savedPartitions = envelope.partitions?.length ?? 0;
|
|
17933
|
+
const built = await prepareDetachedIndex(indexName, prepared, savedPartitions > 0 ? savedPartitions : null, deps);
|
|
17821
17934
|
let manifest;
|
|
17822
17935
|
try {
|
|
17823
17936
|
insertSnapshotDocuments(envelope, built.manager);
|
|
@@ -18048,4 +18161,4 @@ function createRewriteSpill(storage, indexName, analysis) {
|
|
|
18048
18161
|
};
|
|
18049
18162
|
}
|
|
18050
18163
|
|
|
18051
|
-
export { DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, abandonAnalysisChange, actionPartitionsOf, adoptSavedAnalysis, adoptSegmentsOnWorker, bindStorageHost, buildHeldAnalysisChange, bytesThatSegmentsHoldInMemory, changeIndexAnalysis, checkPagedContents, checkPagedSegmentFully, checkPagedSelections, checkpointForArchive, closeUnusedSegments, countsAsLatestHolder, createByteReader, createDurabilityFromTier, createInvalidationFromConfig, createPagedFrozenSegment, createSnapshot, decodeIndexConfiguration, documentExportStream, dropHeldAnalysisChange, encodeIndexConfiguration, executeRebalance, heldAnalysisIsBuilt, holdRenewedSyncEpoch, holdsNamedSettings, indexConfigurationOf, isPagedSegmentFile, keysThatSegmentsStillRead, loadFilesByParts, mergePagedWindow, openColdVectorField, openFileVectorReads, openPagedSegmentScan, openPagedSnapshot, openRewriteSpill, pagedListingOf, pagedPartitionLines, pagedTombstonesFor, prepareDiskAdoption, readDocumentExport, readSnapshotArchive, releaseHeldAnalysisChange, reloadIndexAndCopies, requireNoQuerySuggestions, requireSyncSetting, restoreFromSnapshot, resumeAnalysisChange, scheduleSyncRetention, stampAnalysisGeneration, startReadingOpeningSectionsAhead, startSyncRetentionSweep, swapPreparedPartitions, syncOf, syncStorageOf, targetOfPending, transitionWalLayout, updateIndexSync, updateSyncOf, waitBeforeColdLoad, writePagedSegment };
|
|
18164
|
+
export { DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, abandonAnalysisChange, actionPartitionsOf, adoptSavedAnalysis, adoptSegmentsOnWorker, bindStorageHost, buildHeldAnalysisChange, bytesThatSegmentsHoldInMemory, changeIndexAnalysis, checkPagedContents, checkPagedSegmentFully, checkPagedSelections, checkpointForArchive, closeUnusedSegments, countsAsLatestHolder, createByteReader, createDurabilityFromTier, createInvalidationFromConfig, createPagedFrozenSegment, createSnapshot, decodeIndexConfiguration, documentExportStream, dropHeldAnalysisChange, encodeIndexConfiguration, executeRebalance, heldAnalysisIsBuilt, holdRenewedSyncEpoch, holdsNamedSettings, indexConfigurationOf, isPagedSegmentFile, keysThatSegmentsStillRead, loadFilesByParts, mergePagedWindow, openColdVectorField, openFileVectorReads, openPagedSegmentScan, openPagedSnapshot, openRewriteSpill, pagedListingOf, pagedPartitionLines, pagedTombstonesFor, prepareDiskAdoption, readDocumentExport, readSnapshotArchive, releaseHeldAnalysisChange, reloadIndexAndCopies, requireNoQuerySuggestions, requireSavedPartitionCount, requireSyncSetting, restoreFromSnapshot, resumeAnalysisChange, scheduleSyncRetention, stampAnalysisGeneration, startReadingOpeningSectionsAhead, startSyncRetentionSweep, swapPreparedPartitions, syncOf, syncStorageOf, targetOfPending, transitionWalLayout, updateIndexSync, updateSyncOf, waitBeforeColdLoad, writePagedSegment };
|
|
@@ -1,14 +1,14 @@
|
|
|
1
|
+
export { DayStore, RestoredHistory, clearLeftoversOfName, dayOf, installSuggestionHooks, openCountsFile, sameBytes, startQuerySuggestions } from './chunk-GVWLACAT.mjs';
|
|
1
2
|
export { oneSyncChangeAtATime } from './chunk-Q32DDFKX.mjs';
|
|
2
|
-
export {
|
|
3
|
-
|
|
4
|
-
import './chunk-
|
|
5
|
-
import './chunk-
|
|
6
|
-
import './chunk-
|
|
7
|
-
import './chunk-
|
|
8
|
-
import './chunk-XBTFK3AZ.mjs';
|
|
3
|
+
export { BUILD_LIST_PAGE, BUILD_WRITE_BATCH, BYTES_PER_MIB, DAY_MS, DEFAULT_PREVIEW_LIMIT, HEADER_SIZE, HISTORY_SWEEP_INTERVAL_MS, KEY_FINGERPRINT_BYTES, KEY_FINGERPRINT_LABEL, MAX_HISTORY_FILE_BYTES, MAX_MANUAL_SUGGESTION_COUNT, MAX_PREVIEW_LIMIT, MAX_SOURCE_INDEX_NAME_LENGTH, MAX_TIMER_DELAY_MS, MIN_KEY_BYTES, MS_PER_SECOND, QUERY_HISTORY_ARCHIVE_PREFIX, QUERY_HISTORY_FOLDER, SCHEDULE_OFF, SCRAMBLED_ID_BYTES, acceptedPhrase, attachRuntime, attachedRuntimeOf, banListFromInput, bindSuggestionStoreHost, buildRecordKey, checksumOfChunks, crc32, crc32Final, crc32Init, crc32Update, engineVersionParts, hasNrslMagic, historyFileOf, historyFilesOf, invalidSetting, isGroup, isPerson, latestMatchAtOrBefore, listHistoryFolder, nextMatchAfter, normaliseBan, openChecksummedEnvelope, packChecksummedEnvelope, packEnvelopeWithChecksum, parseSchedule, readHeader, removeHistoryFolder, removeRestoreLeftovers, requireFieldsInSchema, requireWithinFileLimit, resolveSetting, storedStateFromWire, storedStateOf, storedStateToWire, unionOfBans, validateSetting, withState, writeHeader } from './chunk-F6BJYCF2.mjs';
|
|
4
|
+
import './chunk-F4MIGBBP.mjs';
|
|
5
|
+
import './chunk-J2BQTJJE.mjs';
|
|
6
|
+
import './chunk-I7DTUOK2.mjs';
|
|
7
|
+
import './chunk-SK3DPAI4.mjs';
|
|
8
|
+
import './chunk-WFQ6YQZA.mjs';
|
|
9
9
|
import './chunk-PVMEHBQO.mjs';
|
|
10
10
|
import './chunk-YOTP2FPE.mjs';
|
|
11
|
-
import './chunk-
|
|
11
|
+
import './chunk-ZDDJVR7X.mjs';
|
|
12
12
|
import './chunk-HRDMIQ63.mjs';
|
|
13
13
|
import './chunk-VUJCSQG5.mjs';
|
|
14
14
|
import './chunk-TQ26BKBH.mjs';
|