@delali/narsil 0.3.1-next.185 → 0.3.1-next.193
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/dist/adapters/filesystem.mjs +2 -2
- package/dist/adapters/gcs.mjs +2 -2
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/adapters/s3.mjs +2 -2
- package/dist/browser/chunk-4E5APHTM.mjs +1459 -0
- package/dist/browser/{chunk-4ZICWS4V.mjs → chunk-QAOVAYAG.mjs} +1 -1
- package/dist/browser/{chunk-CPSHS46N.mjs → chunk-UQG4HUGQ.mjs} +82 -50
- package/dist/browser/{chunk-2Q45HI5H.mjs → chunk-YINZFPLT.mjs} +7 -0
- package/dist/browser/{module-XVLHJ7T7.mjs → module-ZZTNG5JZ.mjs} +26 -3
- package/dist/browser/{optional-writes-ZTBHYQMN.mjs → optional-writes-J5BHKEQL.mjs} +21 -8
- package/dist/browser/{storage-leaf-4XMCITJ6.mjs → storage-leaf-N3JMQ6CF.mjs} +14569 -13747
- package/dist/{capabilities-DAHXphRn.d.ts → capabilities-CLnQHIqV.d.ts} +51 -9
- package/dist/{chunk-KLVMK3R5.mjs → chunk-2CKDCC65.mjs} +1 -1
- package/dist/{chunk-CODGHJVW.mjs → chunk-2HVQYOOD.mjs} +40 -12
- package/dist/{chunk-M5I64AQW.mjs → chunk-3N5GUXBI.mjs} +33 -33
- package/dist/{chunk-DU23J5JG.mjs → chunk-5CIHQIA4.mjs} +6 -6
- package/dist/{react-native/chunks/chunk-ZUFOYZ5P.mjs → chunk-62X472AC.mjs} +1 -1
- package/dist/{chunk-QJGDB3WR.mjs → chunk-6GHX35ZM.mjs} +2 -2
- package/dist/{chunk-G3XRHAJA.mjs → chunk-6MC6BCXK.mjs} +3 -3
- package/dist/{chunk-X5NHWOXT.mjs → chunk-7P5YWDRQ.mjs} +2 -2
- package/dist/{chunk-MHBG33R5.mjs → chunk-7X4IIREW.mjs} +76 -46
- package/dist/{chunk-F4OOBPKA.mjs → chunk-AOYQTX6W.mjs} +1 -1
- package/dist/{chunk-D6FYUTFN.mjs → chunk-ASZLNVWW.mjs} +1 -1
- package/dist/{chunk-LR6JJVML.mjs → chunk-D2EJUM4A.mjs} +764 -699
- package/dist/{chunk-FETZYCH7.mjs → chunk-F6F4G322.mjs} +26 -1
- package/dist/{chunk-2YWWHMVC.mjs → chunk-FGOGT273.mjs} +2 -2
- package/dist/{chunk-5BUEBIXR.mjs → chunk-GLXLSPLO.mjs} +18 -5
- package/dist/{chunk-IE3Q62QD.mjs → chunk-GR5GMFJJ.mjs} +1 -1
- package/dist/{chunk-Y7QUFIGV.mjs → chunk-GXED6R3Q.mjs} +2 -2
- package/dist/{chunk-BVZWWMOU.mjs → chunk-HHYQMGPW.mjs} +2 -2
- package/dist/{chunk-SL53DHVE.mjs → chunk-HSA3H3VW.mjs} +121 -68
- package/dist/{chunk-YM2HI5N5.mjs → chunk-IL5DRCNU.mjs} +2 -2
- package/dist/{chunk-JDG6CCXM.mjs → chunk-JENWA4WX.mjs} +2 -2
- package/dist/{chunk-2GONI5HL.mjs → chunk-JXINPXRS.mjs} +5 -4
- package/dist/chunk-K5GNK5GE.mjs +602 -0
- package/dist/{chunk-WLFOLAQI.mjs → chunk-KBU3ZFJN.mjs} +2 -2
- package/dist/{chunk-EDEE7ETQ.mjs → chunk-KOA6T2IC.mjs} +3 -3
- package/dist/{chunk-N3M4NPML.mjs → chunk-M2Z5WVWW.mjs} +1 -1
- package/dist/{chunk-EWGIF6FQ.mjs → chunk-NJJN4FMM.mjs} +1 -1
- package/dist/{chunk-C7YQHCSO.mjs → chunk-P4F4UWLC.mjs} +1 -1
- package/dist/{chunk-VYSWVQCE.mjs → chunk-PAPQGC6I.mjs} +2 -2
- package/dist/{chunk-I2WJCRIH.mjs → chunk-R2PJKSNL.mjs} +1 -1
- package/dist/{chunk-6SZQETSC.mjs → chunk-RORNUNXW.mjs} +5 -2
- package/dist/{chunk-FBQYC5OX.mjs → chunk-SB5HCIIO.mjs} +16 -13
- package/dist/{chunk-CYCVPTTR.mjs → chunk-SCZI7COG.mjs} +2 -2
- package/dist/{chunk-O337MDRJ.mjs → chunk-SHES4VP5.mjs} +38 -13
- package/dist/{chunk-H4QYQ5DE.mjs → chunk-SIIDIGKI.mjs} +2 -2
- package/dist/{chunk-A3OAFUZI.mjs → chunk-SM2SYVIX.mjs} +2 -2
- package/dist/{chunk-II6XF6M4.mjs → chunk-SMLUZGSI.mjs} +9 -8
- package/dist/{chunk-2KCCQK4R.mjs → chunk-TSE2VWPX.mjs} +2 -2
- package/dist/{chunk-WSIU6YXQ.mjs → chunk-YPEJ6ODL.mjs} +1732 -743
- package/dist/{chunk-7WXGUTWK.mjs → chunk-Z6SIJH5O.mjs} +17 -6
- package/dist/{chunk-Q3ZZH5Z7.mjs → chunk-ZKOQR7EA.mjs} +7 -0
- package/dist/{chunk-CKEOTSF5.mjs → chunk-ZS2YUB2L.mjs} +6 -3
- package/dist/{chunk-FCQMZDRI.mjs → chunk-ZVZYJIIP.mjs} +3 -3
- package/dist/{client-DpqtT2XG.d.ts → client-uB0kp2yT.d.ts} +88 -5
- package/dist/client.d.ts +7 -7
- package/dist/client.mjs +3 -1265
- package/dist/{config-D9WR3MhO.d.ts → config-BvHQ-C7U.d.ts} +3 -3
- package/dist/{config-C687-reW.d.ts → config-PqgNY1H7.d.ts} +1 -1
- package/dist/{core-leaf-NEUM7PU3.mjs → core-leaf-TAR3GJVE.mjs} +3 -3
- package/dist/{direct-executor-37IH3ISS.mjs → direct-executor-WSZALYCV.mjs} +8 -8
- package/dist/distribution/coordinator/etcd.d.ts +1 -1
- package/dist/distribution/coordinator/etcd.mjs +35 -15
- package/dist/distribution/coordinator/in-memory.d.ts +1 -1
- package/dist/distribution/coordinator/in-memory.mjs +16 -6
- package/dist/distribution/transport/grpc.mjs +2 -2
- package/dist/distribution/transport/tcp.mjs +2 -2
- package/dist/distribution.d.ts +70 -9
- package/dist/distribution.mjs +1695 -680
- package/dist/{durability-config-NWU2G2SJ.mjs → durability-config-VWS2K24E.mjs} +1 -1
- package/dist/{react-native/chunks/embed-4LBSQD7U.mjs → embed-4PHHKGBK.mjs} +1 -1
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-De6JzTW8.d.ts → engine-qjAxJSKk.d.ts} +206 -7
- package/dist/{index-CBUl4z3E.d.ts → index-BaQIAnk0.d.ts} +137 -111
- package/dist/index.browser.mjs +3 -3
- package/dist/index.d.ts +4 -4
- package/dist/index.mjs +8 -8
- package/dist/index.react-native.mjs +9 -9
- package/dist/{long-query-thread-DIPJBMV7.mjs → long-query-thread-N2DNPEA3.mjs} +21 -21
- package/dist/{module-Z3CL4GJ4.mjs → module-UQFIJQMG.mjs} +33 -13
- package/dist/{msgpack-leaf-7KBYOVTF.mjs → msgpack-leaf-H5ZYW7S7.mjs} +4 -4
- package/dist/{optional-reads-ENKMVQ4Q.mjs → optional-reads-5HFMF2QZ.mjs} +2 -2
- package/dist/{optional-writes-YIVJMQ4D.mjs → optional-writes-CGDBJBHQ.mjs} +31 -18
- package/dist/{paged-partition-archive-IZ6U7DVN.mjs → paged-partition-archive-UK44CVXG.mjs} +20 -20
- package/dist/partial-words.browser.mjs +3 -3
- package/dist/partial-words.mjs +8 -8
- package/dist/pattern.browser.mjs +2 -2
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.d.ts +7 -0
- package/dist/persistence/durability/checkpoint-worker.mjs +18 -18
- package/dist/{phrase-module-ZY3TLTQA.mjs → phrase-module-EFESWOCR.mjs} +2 -2
- package/dist/{query-actions-7EOKT3WD.mjs → query-actions-7BL3YZDW.mjs} +8 -8
- package/dist/query-suggestions/merge-worker.mjs +11 -11
- package/dist/query-suggestions.browser.mjs +3 -3
- package/dist/query-suggestions.d.ts +1 -1
- package/dist/query-suggestions.mjs +9 -9
- package/dist/react-native/chunks/{chunk-TQ26BKBH.mjs → chunk-43R4GCM7.mjs} +7 -0
- package/dist/react-native/chunks/{chunk-CBU4M3EU.mjs → chunk-63DZ26X5.mjs} +277 -75
- package/dist/react-native/chunks/{chunk-BN6FUB7N.mjs → chunk-AN4HKLB4.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-T5HF3FW3.mjs → chunk-BNFGA7TY.mjs} +3 -3
- package/dist/react-native/chunks/{chunk-PVMEHBQO.mjs → chunk-CW2ZFXSE.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-NLEFMPY3.mjs → chunk-G434H5WG.mjs} +15 -4
- package/dist/react-native/chunks/{chunk-SK3DPAI4.mjs → chunk-HTBY4GON.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-ZQPSXN6C.mjs → chunk-IP3QLETQ.mjs} +5 -5
- package/dist/react-native/chunks/{chunk-7TPO4FLE.mjs → chunk-LOKGE4OQ.mjs} +3 -3
- package/dist/react-native/chunks/{chunk-ZDDJVR7X.mjs → chunk-MRH36RN3.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-YOTP2FPE.mjs → chunk-UO44MA5M.mjs} +1 -1
- package/dist/{chunk-4JBTALUC.mjs → react-native/chunks/chunk-UYAZLGEM.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-WFQ6YQZA.mjs → chunk-UYKIV2KT.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-IEY4DARA.mjs → chunk-XHYOQ4K4.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-OZ3XEM6D.mjs → chunk-ZUJCUPVS.mjs} +1 -1
- package/dist/react-native/chunks/{core-leaf-TPBRRWVN.mjs → core-leaf-5PVST5PR.mjs} +4 -4
- package/dist/react-native/chunks/{durability-config-RVTWSXPH.mjs → durability-config-FCSD4MXY.mjs} +1 -1
- package/dist/{embed-2QMYCD2Q.mjs → react-native/chunks/embed-HA2I7RCF.mjs} +1 -1
- package/dist/react-native/chunks/{gzip.react-native-DLO4WW27.mjs → gzip.react-native-CVM4NQG2.mjs} +2 -2
- package/dist/react-native/chunks/{module-COLNJNML.mjs → module-QRCYB7QV.mjs} +32 -9
- package/dist/react-native/chunks/msgpack-leaf-JMNGSTRW.mjs +7 -0
- package/dist/react-native/chunks/{optional-reads-74HYI4DO.mjs → optional-reads-AFPO2FEB.mjs} +3 -3
- package/dist/react-native/chunks/{optional-writes-QETIM6BI.mjs → optional-writes-GC6JWMAF.mjs} +31 -18
- package/dist/react-native/chunks/{phrase-module-UC4YGEBB.mjs → phrase-module-IGYBK7XH.mjs} +3 -3
- package/dist/react-native/chunks/{resolve-vector-text-FU7D4QP5.mjs → resolve-vector-text-GE2CRJKI.mjs} +1 -1
- package/dist/react-native/chunks/{restore-queue-3E2MJIJS.mjs → restore-queue-2ACONS4W.mjs} +12 -12
- package/dist/react-native/chunks/{segment-codec-leaf-IE4DXUZV.mjs → segment-codec-leaf-SSIWXVUG.mjs} +10 -10
- package/dist/react-native/chunks/{segment-merge-leaf-62LQ5WZB.mjs → segment-merge-leaf-I2WDFE6O.mjs} +11 -11
- package/dist/react-native/chunks/{storage-leaf-LNOLKWZB.mjs → storage-leaf-YXXMG4VR.mjs} +14727 -13915
- package/dist/react-native/chunks/{store-leaf-Y7CBOPFZ.mjs → store-leaf-FS7A4PZA.mjs} +11 -11
- package/dist/react-native/chunks/{suggestions-leaf-PSUGL6MI.mjs → suggestions-leaf-PSFCT5EX.mjs} +11 -11
- package/dist/react-native/chunks/{worker-copies-MW5O5U3D.mjs → worker-copies-QRMISMEL.mjs} +10 -10
- package/dist/react-native/partial-words.mjs +9 -9
- package/dist/react-native/pattern.mjs +5 -5
- package/dist/react-native/query-suggestions.mjs +9 -9
- package/dist/react-native/sync.mjs +433 -51
- package/dist/react-native/synonyms.mjs +9 -9
- package/dist/react.d.ts +8 -8
- package/dist/react.mjs +7 -0
- package/dist/{rerank-KTV5L7MZ.mjs → rerank-4GFC26FD.mjs} +8 -8
- package/dist/rerankers/http.mjs +2 -2
- package/dist/{resolve-vector-text-SD6BEMKZ.mjs → resolve-vector-text-OD4J3WHF.mjs} +2 -2
- package/dist/{restore-queue-ES7TDJ5V.mjs → restore-queue-E4EIUYVV.mjs} +11 -11
- package/dist/{segment-codec-leaf-DO75VKEV.mjs → segment-codec-leaf-MP6DILVE.mjs} +9 -9
- package/dist/{segment-merge-leaf-JGND6LG5.mjs → segment-merge-leaf-ED2NMUC5.mjs} +9 -9
- package/dist/server.d.ts +13 -9
- package/dist/server.mjs +875 -348
- package/dist/{storage-leaf-ZI3H2WJY.mjs → storage-leaf-HQFO37W7.mjs} +5234 -5835
- package/dist/{store-leaf-YIH36Y5J.mjs → store-leaf-PL2G7RM5.mjs} +10 -10
- package/dist/{suggestions-leaf-FP5HSTFP.mjs → suggestions-leaf-I3MEHACP.mjs} +12 -12
- package/dist/sync/filesystem.d.ts +2 -2
- package/dist/sync/filesystem.mjs +16 -16
- package/dist/sync/indexeddb.browser.mjs +1 -1
- package/dist/sync/indexeddb.d.ts +2 -2
- package/dist/sync/indexeddb.mjs +3 -3
- package/dist/sync.browser.mjs +35 -1108
- package/dist/sync.d.ts +3 -3
- package/dist/sync.mjs +430 -51
- package/dist/synonyms.browser.mjs +3 -3
- package/dist/synonyms.mjs +8 -8
- package/dist/{tasks-X2IJQc8J.d.ts → tasks-C65V7lIr.d.ts} +3 -3
- package/dist/thread-LTFI3RK5.mjs +31 -0
- package/dist/{types-D0YasjoX.d.ts → types-CpCg-Gq-.d.ts} +1 -1
- package/dist/vector/search-worker.mjs +5 -5
- package/dist/{worker-copies-WYPRCNB7.mjs → worker-copies-P7YWXEP5.mjs} +11 -11
- package/dist/workers/entry.mjs +14 -14
- package/openapi.json +144 -0
- package/package.json +5 -7
- package/dist/chunk-URVH6BLO.mjs +0 -217
- package/dist/react-native/chunks/msgpack-leaf-GTKLF6GJ.mjs +0 -7
- package/dist/thread-NQRNW3IH.mjs +0 -31
- /package/dist/{chunk-CV3Q2EKO.mjs → chunk-S5QMWECL.mjs} +0 -0
|
@@ -1,25 +1,167 @@
|
|
|
1
|
-
import { loadVectorField, VECTOR_FILE_ENVELOPE_VERSION, GRAPH_PART_ENVELOPE_VERSION } from './chunk-
|
|
2
|
-
import { RESTORE_ESTIMATE_MARGIN, RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR_DIMENSION, RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR, RESTORE_WORST_EXTERNAL_BYTES_PER_STORED_BYTE, RESTORE_WORST_HEAP_BYTES_PER_VECTOR_DIMENSION, RESTORE_WORST_HEAP_BYTES_PER_VECTOR, RESTORE_WORST_HEAP_BYTES_PER_STORED_BYTE, RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR_DIMENSION, RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR, RESTORE_FLOOR_EXTERNAL_BYTES_PER_STORED_BYTE, RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR_DIMENSION, RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR, RESTORE_FLOOR_HEAP_BYTES_PER_STORED_BYTE, RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE, RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE, RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE, RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE } from './chunk-
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
1
|
+
import { loadVectorField, openColdVectorField, VECTOR_FILE_ENVELOPE_VERSION, GRAPH_PART_ENVELOPE_VERSION } from './chunk-SMLUZGSI.mjs';
|
|
2
|
+
import { engineChangeSource, ChangeCursor, createSemaphore, changePositionList, changesNotEnabled, previousDocumentRequired, invalidPositions, changePositionsOf, requireConfigurationShape, RESTORE_ESTIMATE_MARGIN, persistEpochStates, RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR_DIMENSION, RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR, RESTORE_WORST_EXTERNAL_BYTES_PER_STORED_BYTE, RESTORE_WORST_HEAP_BYTES_PER_VECTOR_DIMENSION, RESTORE_WORST_HEAP_BYTES_PER_VECTOR, RESTORE_WORST_HEAP_BYTES_PER_STORED_BYTE, RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR_DIMENSION, RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR, RESTORE_FLOOR_EXTERNAL_BYTES_PER_STORED_BYTE, RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR_DIMENSION, RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR, RESTORE_FLOOR_HEAP_BYTES_PER_STORED_BYTE, RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE, RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE, RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE, RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE } from './chunk-K5GNK5GE.mjs';
|
|
3
|
+
import { restoredConfigFields, restoredEmbedding } from './chunk-M2Z5WVWW.mjs';
|
|
4
|
+
import { ENTRIES_MESSAGE_LIMIT_BYTES, FEED_HEAD_RETRY_MS, decodeSegmentManifest, isRecord, isUint32, isSequenceNumber, ARCHIVE_CONFIG_ENTRY, ARCHIVE_MANIFEST_ENTRY, tarEnd, tarEntryHeaders, tarPadding, ARCHIVE_FILE_CHUNK_BYTES } from './chunk-Z6SIJH5O.mjs';
|
|
5
|
+
import { isPagedSegmentFile, openPagedSegmentScan, countsAsLatestHolder, writeCheckpointSegments, commitCheckpointManifest, readSegmentFile, readCommitMarker, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, removeUnusedSegmentFiles, readWalSegments, durableRecordMissing, checkpointLastSeqNo, pagedListingOf, residentSectionsOf, partitionContentsOf, checkPagedSegmentFully, openPagedSnapshot, createPagedFrozenSegment, SEGMENT_HEADER_SIZE, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-D2EJUM4A.mjs';
|
|
6
|
+
import { decodeSections, READS_AHEAD_AT_ONCE, readFileRangeLater, SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES, READ_AHEAD_CHUNK_BYTES, STABLE_HEAD_ATTEMPTS, directoryCheckpointStore } from './chunk-3N5GUXBI.mjs';
|
|
7
|
+
import { manifestKey, segmentsPrefix, vectorFileKey, manifestFileRefs, graphPartKey } from './chunk-TSE2VWPX.mjs';
|
|
7
8
|
import { DEFAULT_COMPACTION_THRESHOLD } from './chunk-MSIN4NM3.mjs';
|
|
8
|
-
import { encode } from './chunk-
|
|
9
|
-
import { restoreVectorFields } from './chunk-
|
|
10
|
-
import { DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE, rewriteSpillOf, REBALANCE_CHUNK_SIZE, rebuildTextIndexes, textRebuildTargetOf } from './chunk-
|
|
11
|
-
import { DEFAULT_VECTOR_METRIC, defaultQuantizationFor, vectorsPerFile, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION } from './chunk-
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
9
|
+
import { encode } from './chunk-S5QMWECL.mjs';
|
|
10
|
+
import { setNestedValue, restoreVectorFields } from './chunk-R2PJKSNL.mjs';
|
|
11
|
+
import { DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE, rewriteSpillOf, REBALANCE_CHUNK_SIZE, rebuildTextIndexes, textRebuildTargetOf } from './chunk-62X472AC.mjs';
|
|
12
|
+
import { DEFAULT_VECTOR_METRIC, defaultQuantizationFor, vectorsPerFile, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-KBU3ZFJN.mjs';
|
|
13
|
+
import { oneSyncChangeAtATime } from './chunk-V6EUBAIT.mjs';
|
|
14
|
+
import { openChecksummedEnvelope, HEADER_SIZE, packChecksummedEnvelope, packEnvelopeWithChecksum, checksumOfChunks, readHeader, crc32, engineVersionParts, writeHeader, storedStateToWire, storedStateFromWire } from './chunk-KOA6T2IC.mjs';
|
|
15
|
+
import { resolveIndexAnalysis, getLanguage, emitEngineEvent, reconstructSchemaFromMetadata, prepareStoredSynonyms, rulesOfStoredSynonyms, resolvePartialWordAnalysis, oversizedConfigurationPayloadOf, requireEncodedConfigurationWithinSizeLimit, loadSchemaCodeFor, VERSION, DEFAULT_BM25_B, DEFAULT_BM25_K1, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors, noteDocumentChurn, createPartitionIndex, writePartitionDocument, yieldToEventLoop, createPublicationGate, removedSurfaceSpellings, readHeapStatistics, readResidentMemory, prepareConfiguredSynonyms, withOpenIndex, loadGzip, prepareRemovalText, prepareDocumentText, updateDocumentVectors, holdSyncEpochChange, decodeStoredDocument, emitSyncChange, ARRAY_POSITION_GAP } from './chunk-HSA3H3VW.mjs';
|
|
14
16
|
import { decode } from './chunk-GXCDWLQI.mjs';
|
|
15
|
-
import { isSyncEpoch } from './chunk-BUMREPLU.mjs';
|
|
16
|
-
import { extractVectorFieldsFromSchema, flattenSchema, VECTOR_PATTERN, validateDocument, validateDocumentStrict, schemaFieldsOf, requireValidFieldTypes, isQuantizationMode, isStorageMode } from './chunk-
|
|
17
|
-
import { collectAndReturnPages, turnIsSpent, yieldTurn, countWrite } from './chunk-UPDNNOTG.mjs';
|
|
18
|
-
import { DEFAULT_MAX_PARTITIONS, ANALYSIS_CHANGE_MEMBERS, MAX_PARTITIONS, ANALYSIS_HOLD_APPLY_INTERVAL_MS } from './chunk-3ECIRFS5.mjs';
|
|
17
|
+
import { isSyncEpoch, newSyncEpoch } from './chunk-BUMREPLU.mjs';
|
|
18
|
+
import { extractVectorFieldsFromSchema, flattenSchema, VECTOR_PATTERN, validateDocument, validateDocumentStrict, schemaFieldsOf, parseVectorDimension, requireValidFieldTypes, isQuantizationMode, isStorageMode, MAX_DOCUMENT_NESTING_DEPTH } from './chunk-7P5YWDRQ.mjs';
|
|
19
|
+
import { collectAndReturnPages, turnsAreBudgeted, turnIsSpent, yieldTurn, countWrite } from './chunk-UPDNNOTG.mjs';
|
|
20
|
+
import { DEFAULT_MAX_PARTITIONS, ANALYSIS_CHANGE_MEMBERS, EXPORT_CHUNK_BYTES, MAX_PARTITIONS, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, ANALYSIS_HOLD_APPLY_INTERVAL_MS, MAX_IMPORT_REFUSALS_REPORTED } from './chunk-3ECIRFS5.mjs';
|
|
19
21
|
import { VALID_HNSW_METRICS } from './chunk-4PY4JYMY.mjs';
|
|
20
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
22
|
+
import { NarsilError, ErrorCodes } from './chunk-ZKOQR7EA.mjs';
|
|
21
23
|
import { compareCodePoints } from './chunk-TGPKS2UM.mjs';
|
|
22
24
|
|
|
25
|
+
// src/changes/constants.ts
|
|
26
|
+
var CHANGES_IDLE_RECHECK_MS = 1e3;
|
|
27
|
+
var MAX_FOLLOWER_CATCH_UPS = 4;
|
|
28
|
+
|
|
29
|
+
// src/changes/follow.ts
|
|
30
|
+
function requestOf(indexName, options) {
|
|
31
|
+
const { epoch, positions, filter } = options;
|
|
32
|
+
if (epoch === void 0 !== (positions === void 0)) {
|
|
33
|
+
throw invalidPositions(indexName, "Pass epoch and positions together, or leave both out");
|
|
34
|
+
}
|
|
35
|
+
const request = filter === void 0 ? {} : { filter };
|
|
36
|
+
if (epoch === void 0) return request;
|
|
37
|
+
if (!isSyncEpoch(epoch)) throw invalidPositions(indexName, "The epoch must be 32 lowercase hexadecimal digits");
|
|
38
|
+
const valid = changePositionsOf(positions);
|
|
39
|
+
if (valid === null) {
|
|
40
|
+
throw invalidPositions(indexName, "Each position needs a whole partitionId and a whole, non-negative seqNo");
|
|
41
|
+
}
|
|
42
|
+
return { ...request, epoch, positions: valid };
|
|
43
|
+
}
|
|
44
|
+
function documentOf(bytes, vectorFieldPaths) {
|
|
45
|
+
const document = decode(bytes);
|
|
46
|
+
restoreVectorFields(document, vectorFieldPaths);
|
|
47
|
+
return document;
|
|
48
|
+
}
|
|
49
|
+
function eventOf(batch, vectorFieldPaths) {
|
|
50
|
+
const changes = batch.changes.map((change) => {
|
|
51
|
+
const shaped = { seqNo: change.seqNo, type: change.type, documentId: change.documentId };
|
|
52
|
+
if (change.document !== null) shaped.document = documentOf(change.document, vectorFieldPaths);
|
|
53
|
+
if (change.previousDocument !== null) {
|
|
54
|
+
shaped.previousDocument = documentOf(change.previousDocument, vectorFieldPaths);
|
|
55
|
+
}
|
|
56
|
+
return shaped;
|
|
57
|
+
});
|
|
58
|
+
return { type: "changes", partitionId: batch.partitionId, changes, scannedSeqNo: batch.scannedSeqNo };
|
|
59
|
+
}
|
|
60
|
+
function wakeupOn(source, indexName, signal) {
|
|
61
|
+
let pending = false;
|
|
62
|
+
let wake = null;
|
|
63
|
+
const fire = () => {
|
|
64
|
+
pending = true;
|
|
65
|
+
wake?.();
|
|
66
|
+
};
|
|
67
|
+
const stopListening = source.listen((change) => {
|
|
68
|
+
if (change.indexName === indexName) fire();
|
|
69
|
+
});
|
|
70
|
+
const stopWatching = source.watchIndex?.(indexName);
|
|
71
|
+
signal?.addEventListener("abort", fire);
|
|
72
|
+
return {
|
|
73
|
+
async wait(ms) {
|
|
74
|
+
if (pending) {
|
|
75
|
+
pending = false;
|
|
76
|
+
return;
|
|
77
|
+
}
|
|
78
|
+
let timer;
|
|
79
|
+
await new Promise((resolve) => {
|
|
80
|
+
wake = resolve;
|
|
81
|
+
timer = setTimeout(resolve, ms);
|
|
82
|
+
});
|
|
83
|
+
clearTimeout(timer);
|
|
84
|
+
wake = null;
|
|
85
|
+
pending = false;
|
|
86
|
+
},
|
|
87
|
+
stop() {
|
|
88
|
+
stopListening();
|
|
89
|
+
stopWatching?.();
|
|
90
|
+
signal?.removeEventListener("abort", fire);
|
|
91
|
+
}
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
function followIndexChanges(core, indexName, options = {}) {
|
|
95
|
+
core.guardShutdown();
|
|
96
|
+
core.requireIndex(indexName);
|
|
97
|
+
return followSourceChanges(
|
|
98
|
+
{ source: engineChangeSource(core), guard: () => core.guardShutdown() },
|
|
99
|
+
indexName,
|
|
100
|
+
options
|
|
101
|
+
);
|
|
102
|
+
}
|
|
103
|
+
async function* followSourceChanges(deps, indexName, options = {}) {
|
|
104
|
+
deps.guard();
|
|
105
|
+
const request = requestOf(indexName, options);
|
|
106
|
+
const { source } = deps;
|
|
107
|
+
const { signal } = options;
|
|
108
|
+
const aborted = () => signal?.aborted === true;
|
|
109
|
+
if (aborted()) return;
|
|
110
|
+
const wakeup = wakeupOn(source, indexName, signal);
|
|
111
|
+
try {
|
|
112
|
+
const opened = await ChangeCursor.open(
|
|
113
|
+
{ source, indexName, reads: createSemaphore(MAX_FOLLOWER_CATCH_UPS) },
|
|
114
|
+
request
|
|
115
|
+
);
|
|
116
|
+
if (typeof opened === "string") {
|
|
117
|
+
yield { type: "resync", reason: opened };
|
|
118
|
+
return;
|
|
119
|
+
}
|
|
120
|
+
const cursor = opened;
|
|
121
|
+
const { state } = cursor;
|
|
122
|
+
const vectorFieldPaths = await source.vectorFieldPaths(indexName);
|
|
123
|
+
yield {
|
|
124
|
+
type: "subscribed",
|
|
125
|
+
epoch: state.epoch,
|
|
126
|
+
partitionCount: state.partitionCount,
|
|
127
|
+
positions: changePositionList(cursor.subscribedSeqNos),
|
|
128
|
+
heads: changePositionList(cursor.heads),
|
|
129
|
+
bytesBehind: cursor.bytesBehind,
|
|
130
|
+
previousDocument: state.previousDocument
|
|
131
|
+
};
|
|
132
|
+
while (!aborted()) {
|
|
133
|
+
deps.guard();
|
|
134
|
+
const current = source.changesState(indexName);
|
|
135
|
+
if (current === null) throw changesNotEnabled(indexName);
|
|
136
|
+
if (current.epoch !== state.epoch) {
|
|
137
|
+
yield { type: "resync", reason: "epoch" };
|
|
138
|
+
return;
|
|
139
|
+
}
|
|
140
|
+
if (cursor.filtered && !current.previousDocument) throw previousDocumentRequired(indexName);
|
|
141
|
+
await cursor.refreshHeads();
|
|
142
|
+
let progressed = false;
|
|
143
|
+
for (let partitionId = 0; partitionId < state.partitionCount; partitionId++) {
|
|
144
|
+
if (aborted() || !cursor.behind(partitionId)) continue;
|
|
145
|
+
const batch = await cursor.readBatch(partitionId, ENTRIES_MESSAGE_LIMIT_BYTES);
|
|
146
|
+
if (batch === "history") {
|
|
147
|
+
yield { type: "resync", reason: "history" };
|
|
148
|
+
return;
|
|
149
|
+
}
|
|
150
|
+
if (batch === null) continue;
|
|
151
|
+
const afterRead = source.changesState(indexName);
|
|
152
|
+
if (afterRead === null || afterRead.epoch !== state.epoch) break;
|
|
153
|
+
if (cursor.filtered && !afterRead.previousDocument) throw previousDocumentRequired(indexName);
|
|
154
|
+
cursor.commit(batch);
|
|
155
|
+
progressed = true;
|
|
156
|
+
yield eventOf(batch, vectorFieldPaths);
|
|
157
|
+
}
|
|
158
|
+
if (!progressed) await wakeup.wait(cursor.awaitsDurability() ? FEED_HEAD_RETRY_MS : CHANGES_IDLE_RECHECK_MS);
|
|
159
|
+
}
|
|
160
|
+
} finally {
|
|
161
|
+
wakeup.stop();
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
|
|
23
165
|
// src/engine/analysis-change/pending-target.ts
|
|
24
166
|
function configUnderAnalysis(base, language, tokenizer, stopWords) {
|
|
25
167
|
const { tokenizer: _tokenizer, stopWords: _stopWords, ...rest } = base;
|
|
@@ -1004,11 +1146,11 @@ function requireRestoreMemory(indexName, footprint, replacesIndex) {
|
|
|
1004
1146
|
// src/engine/analysis-change/run.ts
|
|
1005
1147
|
function createHold(generation) {
|
|
1006
1148
|
let markReady = () => void 0;
|
|
1007
|
-
let
|
|
1149
|
+
let refuse5 = () => void 0;
|
|
1008
1150
|
let wake = () => void 0;
|
|
1009
1151
|
const ready = new Promise((resolve, reject) => {
|
|
1010
1152
|
markReady = resolve;
|
|
1011
|
-
|
|
1153
|
+
refuse5 = reject;
|
|
1012
1154
|
});
|
|
1013
1155
|
ready.catch(() => void 0);
|
|
1014
1156
|
return {
|
|
@@ -1017,7 +1159,7 @@ function createHold(generation) {
|
|
|
1017
1159
|
released: false,
|
|
1018
1160
|
ready,
|
|
1019
1161
|
markReady,
|
|
1020
|
-
refuse:
|
|
1162
|
+
refuse: refuse5,
|
|
1021
1163
|
wake: () => wake(),
|
|
1022
1164
|
pause: () => new Promise((resolve) => {
|
|
1023
1165
|
const timer = setTimeout(resolve, ANALYSIS_HOLD_APPLY_INTERVAL_MS);
|
|
@@ -1483,85 +1625,6 @@ async function stampAnalysisGeneration(core, indexName, generation) {
|
|
|
1483
1625
|
await (durable?.checkpointAfterSwitch?.(indexName, stamp) ?? Promise.resolve(stamp()));
|
|
1484
1626
|
}
|
|
1485
1627
|
|
|
1486
|
-
// src/partitioning/saved-partitions.ts
|
|
1487
|
-
function requireSavedPartitionCount(partitionCount, source, indexName) {
|
|
1488
|
-
if (partitionCount <= MAX_PARTITIONS) return;
|
|
1489
|
-
throw new NarsilError(
|
|
1490
|
-
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
1491
|
-
`The ${source} of index "${indexName}" holds partition ${partitionCount - 1}, and a saved index numbers its partitions from 0 to ${MAX_PARTITIONS - 1}`,
|
|
1492
|
-
{ indexName, partitionId: partitionCount - 1, maxPartitions: MAX_PARTITIONS }
|
|
1493
|
-
);
|
|
1494
|
-
}
|
|
1495
|
-
function highestPartitionIdOf(partitions) {
|
|
1496
|
-
let highest = partitions.length - 1;
|
|
1497
|
-
for (const partition of partitions) if (partition.partitionId > highest) highest = partition.partitionId;
|
|
1498
|
-
return highest;
|
|
1499
|
-
}
|
|
1500
|
-
|
|
1501
|
-
// src/persistence/paged/read-ahead.ts
|
|
1502
|
-
var FILE_PAGE_SIZES = /* @__PURE__ */ new Set([SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES]);
|
|
1503
|
-
function rangeOf(path, section) {
|
|
1504
|
-
return { path, offset: HEADER_SIZE + section.offset, length: section.length };
|
|
1505
|
-
}
|
|
1506
|
-
async function readRangesAhead(ranges) {
|
|
1507
|
-
let rangeAt = 0;
|
|
1508
|
-
let doneInRange = 0;
|
|
1509
|
-
function nextChunk() {
|
|
1510
|
-
while (rangeAt < ranges.length && doneInRange >= ranges[rangeAt].length) {
|
|
1511
|
-
rangeAt += 1;
|
|
1512
|
-
doneInRange = 0;
|
|
1513
|
-
}
|
|
1514
|
-
if (rangeAt >= ranges.length) return null;
|
|
1515
|
-
const { path, offset, length } = ranges[rangeAt];
|
|
1516
|
-
const chunk = { path, offset: offset + doneInRange, length: Math.min(READ_AHEAD_CHUNK_BYTES, length - doneInRange) };
|
|
1517
|
-
doneInRange += chunk.length;
|
|
1518
|
-
return chunk;
|
|
1519
|
-
}
|
|
1520
|
-
const scratch = new Uint8Array(READ_AHEAD_CHUNK_BYTES);
|
|
1521
|
-
async function readChunks() {
|
|
1522
|
-
for (let chunk = nextChunk(); chunk !== null; chunk = nextChunk()) {
|
|
1523
|
-
const target = scratch.subarray(0, chunk.length);
|
|
1524
|
-
if (!await readFileRangeLater(chunk.path, chunk.offset, target)) return;
|
|
1525
|
-
}
|
|
1526
|
-
}
|
|
1527
|
-
await Promise.all(Array.from({ length: READS_AHEAD_AT_ONCE }, readChunks));
|
|
1528
|
-
}
|
|
1529
|
-
async function headOf(path, ref) {
|
|
1530
|
-
const { headOffset, headLength } = ref;
|
|
1531
|
-
if (headOffset === void 0 || headLength === void 0) return null;
|
|
1532
|
-
const head = new Uint8Array(headLength);
|
|
1533
|
-
if (!await readFileRangeLater(path, HEADER_SIZE + headOffset, head)) return null;
|
|
1534
|
-
return decode(head);
|
|
1535
|
-
}
|
|
1536
|
-
async function levelsOfPart(directory, ref) {
|
|
1537
|
-
const path = await directory.pathOf(ref.key);
|
|
1538
|
-
const head = await headOf(path, ref);
|
|
1539
|
-
const graphs = head?.graphs;
|
|
1540
|
-
if (!Array.isArray(graphs) || graphs.length !== 1) return [];
|
|
1541
|
-
const { levels } = graphs[0];
|
|
1542
|
-
return decodeSections([levels], ref.bytes - HEADER_SIZE, `head of "${ref.key}"`, FILE_PAGE_SIZES).map(
|
|
1543
|
-
(section) => rangeOf(path, section)
|
|
1544
|
-
);
|
|
1545
|
-
}
|
|
1546
|
-
async function residentSectionsOfSegment(directory, ref) {
|
|
1547
|
-
if (!await isPagedSegmentFile(directory, ref.key)) return [];
|
|
1548
|
-
const listing = await pagedListingOf(directory, ref);
|
|
1549
|
-
return residentSectionsOf(listing).map((section) => rangeOf(listing.path, section));
|
|
1550
|
-
}
|
|
1551
|
-
async function startReadingOpeningSectionsAhead(directory, manifest) {
|
|
1552
|
-
try {
|
|
1553
|
-
const groups = await Promise.all([
|
|
1554
|
-
...manifest.partitions.flatMap(
|
|
1555
|
-
(partition) => partition.segments.map((ref) => residentSectionsOfSegment(directory, ref))
|
|
1556
|
-
),
|
|
1557
|
-
...manifest.vectors.flatMap((field) => field.graphParts.map((ref) => levelsOfPart(directory, ref)))
|
|
1558
|
-
]);
|
|
1559
|
-
void readRangesAhead(groups.flat());
|
|
1560
|
-
} catch {
|
|
1561
|
-
return;
|
|
1562
|
-
}
|
|
1563
|
-
}
|
|
1564
|
-
|
|
1565
1628
|
// src/persistence/archive/byte-reader.ts
|
|
1566
1629
|
async function* chunksOf(source) {
|
|
1567
1630
|
if (source instanceof Uint8Array) {
|
|
@@ -1639,327 +1702,1017 @@ function createByteReader(source) {
|
|
|
1639
1702
|
};
|
|
1640
1703
|
}
|
|
1641
1704
|
|
|
1642
|
-
// src/engine/
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
1651
|
-
|
|
1652
|
-
|
|
1653
|
-
|
|
1654
|
-
|
|
1705
|
+
// src/engine/snapshot/stored-schema.ts
|
|
1706
|
+
function isRecord2(value) {
|
|
1707
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
1708
|
+
}
|
|
1709
|
+
function conflictingStoredPath(path) {
|
|
1710
|
+
return new NarsilError(
|
|
1711
|
+
ErrorCodes.SCHEMA_INVALID_TYPE,
|
|
1712
|
+
`Field path "${path}" is declared both as a field and as an object of fields`,
|
|
1713
|
+
{ path }
|
|
1714
|
+
);
|
|
1715
|
+
}
|
|
1716
|
+
function defineOwn(level, key, value) {
|
|
1717
|
+
Object.defineProperty(level, key, { value, enumerable: true, writable: true, configurable: true });
|
|
1718
|
+
}
|
|
1719
|
+
function placeStoredField(nested, path, type) {
|
|
1720
|
+
const segments = path.split(".");
|
|
1721
|
+
let level = nested;
|
|
1722
|
+
for (let at = 0; at < segments.length; at++) {
|
|
1723
|
+
const segment = segments[at];
|
|
1724
|
+
const held = Object.hasOwn(level, segment) ? level[segment] : void 0;
|
|
1725
|
+
if (at === segments.length - 1) {
|
|
1726
|
+
if (held !== void 0) throw conflictingStoredPath(path);
|
|
1727
|
+
defineOwn(level, segment, type);
|
|
1728
|
+
return;
|
|
1729
|
+
}
|
|
1730
|
+
if (held === void 0) {
|
|
1731
|
+
const child = {};
|
|
1732
|
+
defineOwn(level, segment, child);
|
|
1733
|
+
level = child;
|
|
1734
|
+
} else if (isRecord2(held)) {
|
|
1735
|
+
level = held;
|
|
1736
|
+
} else {
|
|
1737
|
+
throw conflictingStoredPath(segments.slice(0, at + 1).join("."));
|
|
1738
|
+
}
|
|
1655
1739
|
}
|
|
1656
|
-
return Object.keys(fields).length === 0 ? null : fields;
|
|
1657
1740
|
}
|
|
1658
|
-
function
|
|
1659
|
-
const
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
b: config.b,
|
|
1667
|
-
...config.bm25Fields !== void 0 ? { fields: config.bm25Fields } : {}
|
|
1668
|
-
},
|
|
1669
|
-
createdAt: Date.now(),
|
|
1670
|
-
engineVersion: VERSION
|
|
1671
|
-
};
|
|
1672
|
-
metadata.documentCount = documentCount ?? config.documentCount;
|
|
1673
|
-
const recordedSavedBytes = savedBytes ?? config.savedBytes;
|
|
1674
|
-
if (recordedSavedBytes !== void 0) metadata.savedBytes = recordedSavedBytes;
|
|
1675
|
-
const vectorFields = vectorFieldsOf(config.schema, config.vectorPromotion);
|
|
1676
|
-
if (vectorFields !== null) {
|
|
1677
|
-
metadata.vectorFields = vectorFields;
|
|
1741
|
+
function placeStoredFields(nested, stored, prefix) {
|
|
1742
|
+
for (const [key, type] of Object.entries(stored)) {
|
|
1743
|
+
const path = prefix ? `${prefix}.${key}` : key;
|
|
1744
|
+
if (isRecord2(type)) {
|
|
1745
|
+
placeStoredFields(nested, type, path);
|
|
1746
|
+
} else {
|
|
1747
|
+
placeStoredField(nested, path, type);
|
|
1748
|
+
}
|
|
1678
1749
|
}
|
|
1679
|
-
|
|
1680
|
-
|
|
1750
|
+
}
|
|
1751
|
+
function nestedSchemaOf(stored) {
|
|
1752
|
+
if (!isRecord2(stored)) return stored;
|
|
1753
|
+
const nested = {};
|
|
1754
|
+
placeStoredFields(nested, stored, "");
|
|
1755
|
+
return nested;
|
|
1756
|
+
}
|
|
1757
|
+
|
|
1758
|
+
// src/engine/export/lines.ts
|
|
1759
|
+
var NEWLINE = 10;
|
|
1760
|
+
function joined(parts, total) {
|
|
1761
|
+
if (parts.length === 1) return parts[0];
|
|
1762
|
+
const bytes = new Uint8Array(total);
|
|
1763
|
+
let offset = 0;
|
|
1764
|
+
for (const part of parts) {
|
|
1765
|
+
bytes.set(part, offset);
|
|
1766
|
+
offset += part.byteLength;
|
|
1681
1767
|
}
|
|
1682
|
-
|
|
1683
|
-
|
|
1768
|
+
return bytes;
|
|
1769
|
+
}
|
|
1770
|
+
async function* exportLines(source, maxLineBytes) {
|
|
1771
|
+
const decoder2 = new TextDecoder("utf-8", { fatal: true });
|
|
1772
|
+
let parts = [];
|
|
1773
|
+
let pending = 0;
|
|
1774
|
+
let overflowing = false;
|
|
1775
|
+
let line = 0;
|
|
1776
|
+
const finishLine = () => {
|
|
1777
|
+
line += 1;
|
|
1778
|
+
if (overflowing) {
|
|
1779
|
+
overflowing = false;
|
|
1780
|
+
parts = [];
|
|
1781
|
+
pending = 0;
|
|
1782
|
+
return { line, refused: "too-long" };
|
|
1783
|
+
}
|
|
1784
|
+
const bytes = joined(parts, pending);
|
|
1785
|
+
parts = [];
|
|
1786
|
+
pending = 0;
|
|
1787
|
+
try {
|
|
1788
|
+
return { line, text: decoder2.decode(bytes) };
|
|
1789
|
+
} catch {
|
|
1790
|
+
return { line, refused: "not-utf8" };
|
|
1791
|
+
}
|
|
1792
|
+
};
|
|
1793
|
+
for await (const chunk of chunksOf(source)) {
|
|
1794
|
+
let start = 0;
|
|
1795
|
+
for (; ; ) {
|
|
1796
|
+
const newline = chunk.indexOf(NEWLINE, start);
|
|
1797
|
+
const end = newline < 0 ? chunk.byteLength : newline;
|
|
1798
|
+
const piece = chunk.subarray(start, end);
|
|
1799
|
+
if (!overflowing && pending + piece.byteLength <= maxLineBytes) {
|
|
1800
|
+
if (piece.byteLength > 0) parts.push(piece.slice());
|
|
1801
|
+
pending += piece.byteLength;
|
|
1802
|
+
} else {
|
|
1803
|
+
overflowing = true;
|
|
1804
|
+
parts = [];
|
|
1805
|
+
}
|
|
1806
|
+
if (newline < 0) break;
|
|
1807
|
+
yield finishLine();
|
|
1808
|
+
start = newline + 1;
|
|
1809
|
+
}
|
|
1684
1810
|
}
|
|
1685
|
-
if (
|
|
1686
|
-
|
|
1811
|
+
if (pending > 0 || overflowing) yield finishLine();
|
|
1812
|
+
}
|
|
1813
|
+
|
|
1814
|
+
// src/serialization/base64.ts
|
|
1815
|
+
var ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
|
|
1816
|
+
var PAD = 61;
|
|
1817
|
+
var BYTES_PER_GROUP = 3;
|
|
1818
|
+
var CHARACTERS_PER_GROUP = 4;
|
|
1819
|
+
var ASCII_LIMIT = 128;
|
|
1820
|
+
var ENCODE = new Uint8Array(ALPHABET.length);
|
|
1821
|
+
var DECODE = new Int8Array(ASCII_LIMIT).fill(-1);
|
|
1822
|
+
for (let at = 0; at < ALPHABET.length; at++) {
|
|
1823
|
+
ENCODE[at] = ALPHABET.charCodeAt(at);
|
|
1824
|
+
DECODE[ALPHABET.charCodeAt(at)] = at;
|
|
1825
|
+
}
|
|
1826
|
+
var asciiDecoder = new TextDecoder();
|
|
1827
|
+
function encodeBase64(bytes) {
|
|
1828
|
+
const out = new Uint8Array(Math.ceil(bytes.byteLength / BYTES_PER_GROUP) * CHARACTERS_PER_GROUP);
|
|
1829
|
+
let write = 0;
|
|
1830
|
+
let read = 0;
|
|
1831
|
+
for (; read + BYTES_PER_GROUP <= bytes.byteLength; read += BYTES_PER_GROUP) {
|
|
1832
|
+
const group = bytes[read] << 16 | bytes[read + 1] << 8 | bytes[read + 2];
|
|
1833
|
+
out[write++] = ENCODE[group >>> 18];
|
|
1834
|
+
out[write++] = ENCODE[group >>> 12 & 63];
|
|
1835
|
+
out[write++] = ENCODE[group >>> 6 & 63];
|
|
1836
|
+
out[write++] = ENCODE[group & 63];
|
|
1837
|
+
}
|
|
1838
|
+
const left = bytes.byteLength - read;
|
|
1839
|
+
if (left > 0) {
|
|
1840
|
+
const group = bytes[read] << 16 | (left === 2 ? bytes[read + 1] << 8 : 0);
|
|
1841
|
+
out[write++] = ENCODE[group >>> 18];
|
|
1842
|
+
out[write++] = ENCODE[group >>> 12 & 63];
|
|
1843
|
+
out[write++] = left === 2 ? ENCODE[group >>> 6 & 63] : PAD;
|
|
1844
|
+
out[write++] = PAD;
|
|
1845
|
+
}
|
|
1846
|
+
return asciiDecoder.decode(out);
|
|
1847
|
+
}
|
|
1848
|
+
function sextetAt(text, at) {
|
|
1849
|
+
const code = text.charCodeAt(at);
|
|
1850
|
+
return code < ASCII_LIMIT ? DECODE[code] : -1;
|
|
1851
|
+
}
|
|
1852
|
+
function decodeExactBase64(text) {
|
|
1853
|
+
if (text.length % CHARACTERS_PER_GROUP !== 0) return null;
|
|
1854
|
+
const padding = text.endsWith("==") ? 2 : text.endsWith("=") ? 1 : 0;
|
|
1855
|
+
const out = new Uint8Array(text.length / CHARACTERS_PER_GROUP * BYTES_PER_GROUP - padding);
|
|
1856
|
+
const fullEnd = padding === 0 ? text.length : text.length - CHARACTERS_PER_GROUP;
|
|
1857
|
+
let write = 0;
|
|
1858
|
+
for (let at = 0; at < fullEnd; at += CHARACTERS_PER_GROUP) {
|
|
1859
|
+
const a2 = sextetAt(text, at);
|
|
1860
|
+
const b2 = sextetAt(text, at + 1);
|
|
1861
|
+
const c2 = sextetAt(text, at + 2);
|
|
1862
|
+
const d = sextetAt(text, at + 3);
|
|
1863
|
+
if ((a2 | b2 | c2 | d) < 0) return null;
|
|
1864
|
+
const group = a2 << 18 | b2 << 12 | c2 << 6 | d;
|
|
1865
|
+
out[write++] = group >>> 16;
|
|
1866
|
+
out[write++] = group >>> 8 & 255;
|
|
1867
|
+
out[write++] = group & 255;
|
|
1868
|
+
}
|
|
1869
|
+
if (padding === 0) return out;
|
|
1870
|
+
const a = sextetAt(text, fullEnd);
|
|
1871
|
+
const b = sextetAt(text, fullEnd + 1);
|
|
1872
|
+
const c = padding === 1 ? sextetAt(text, fullEnd + 2) : 0;
|
|
1873
|
+
if ((a | b | c) < 0) return null;
|
|
1874
|
+
if (padding === 2 && (b & 15) !== 0) return null;
|
|
1875
|
+
if (padding === 1 && (c & 3) !== 0) return null;
|
|
1876
|
+
out[write++] = a << 2 | b >>> 4;
|
|
1877
|
+
if (padding === 1) out[write] = (b & 15) << 4 | c >>> 2;
|
|
1878
|
+
return out;
|
|
1879
|
+
}
|
|
1880
|
+
|
|
1881
|
+
// src/engine/export/tags.ts
|
|
1882
|
+
var TAG_PREFIX = "$";
|
|
1883
|
+
var DATE_TAG = "$date";
|
|
1884
|
+
var BYTES_TAG = "$bytes";
|
|
1885
|
+
var NUMBER_TAG = "$number";
|
|
1886
|
+
var OBJECT_TAG = "$object";
|
|
1887
|
+
var MILLISECONDS_PER_SECOND = 1e3;
|
|
1888
|
+
var NANOSECONDS_PER_MILLISECOND = 1e6;
|
|
1889
|
+
var MAX_NANOSECONDS = 999999999;
|
|
1890
|
+
var MAX_DATE_MILLISECONDS = 864e13;
|
|
1891
|
+
var ESCAPED_TAG_PREFIX = "\\u0024";
|
|
1892
|
+
var NON_FINITE_NUMBERS = /* @__PURE__ */ new Map([
|
|
1893
|
+
["NaN", Number.NaN],
|
|
1894
|
+
["Infinity", Number.POSITIVE_INFINITY],
|
|
1895
|
+
["-Infinity", Number.NEGATIVE_INFINITY]
|
|
1896
|
+
]);
|
|
1897
|
+
function refuse2(path, reason) {
|
|
1898
|
+
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path === "" ? "(document)" : path}" ${reason}`, {
|
|
1899
|
+
field: path
|
|
1900
|
+
});
|
|
1901
|
+
}
|
|
1902
|
+
function isRecord3(value) {
|
|
1903
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
1904
|
+
}
|
|
1905
|
+
function startsWithTagPrefix(key) {
|
|
1906
|
+
return key.startsWith(TAG_PREFIX);
|
|
1907
|
+
}
|
|
1908
|
+
function soleTagKey(record) {
|
|
1909
|
+
let sole = null;
|
|
1910
|
+
for (const key of Object.keys(record)) {
|
|
1911
|
+
if (sole !== null) return null;
|
|
1912
|
+
sole = key;
|
|
1913
|
+
}
|
|
1914
|
+
return sole !== null && startsWithTagPrefix(sole) ? sole : null;
|
|
1915
|
+
}
|
|
1916
|
+
function dateOf(value, path) {
|
|
1917
|
+
if (!Array.isArray(value) || value.length !== 2)
|
|
1918
|
+
refuse2(path, `holds a "${DATE_TAG}" tag other than [seconds, nanoseconds]`);
|
|
1919
|
+
const seconds = value[0];
|
|
1920
|
+
const nanoseconds = value[1];
|
|
1921
|
+
if (typeof seconds !== "number" || !Number.isSafeInteger(seconds)) {
|
|
1922
|
+
refuse2(
|
|
1923
|
+
path,
|
|
1924
|
+
`holds a "${DATE_TAG}" tag whose seconds are not an integer from -${Number.MAX_SAFE_INTEGER} to ${Number.MAX_SAFE_INTEGER}`
|
|
1925
|
+
);
|
|
1687
1926
|
}
|
|
1688
|
-
if (
|
|
1689
|
-
|
|
1927
|
+
if (typeof nanoseconds !== "number" || !Number.isInteger(nanoseconds) || nanoseconds < 0 || nanoseconds > MAX_NANOSECONDS) {
|
|
1928
|
+
refuse2(path, `holds a "${DATE_TAG}" tag whose nanoseconds are not an integer from 0 to ${MAX_NANOSECONDS}`);
|
|
1690
1929
|
}
|
|
1691
|
-
|
|
1692
|
-
|
|
1930
|
+
const milliseconds = seconds * MILLISECONDS_PER_SECOND + nanoseconds / NANOSECONDS_PER_MILLISECOND;
|
|
1931
|
+
if (Math.abs(milliseconds) > MAX_DATE_MILLISECONDS)
|
|
1932
|
+
refuse2(path, `holds a "${DATE_TAG}" tag outside the range of a date`);
|
|
1933
|
+
return new Date(milliseconds);
|
|
1934
|
+
}
|
|
1935
|
+
function bytesOf(value, path) {
|
|
1936
|
+
const bytes = typeof value === "string" ? decodeExactBase64(value) : null;
|
|
1937
|
+
if (bytes === null) refuse2(path, `holds a "${BYTES_TAG}" tag whose text is not padded base64`);
|
|
1938
|
+
return bytes;
|
|
1939
|
+
}
|
|
1940
|
+
function numberOf(value, path) {
|
|
1941
|
+
const number = typeof value === "string" ? NON_FINITE_NUMBERS.get(value) : void 0;
|
|
1942
|
+
if (number === void 0) refuse2(path, `holds a "${NUMBER_TAG}" tag other than "NaN", "Infinity", or "-Infinity"`);
|
|
1943
|
+
return number;
|
|
1944
|
+
}
|
|
1945
|
+
function requireDepth(path, depth) {
|
|
1946
|
+
if (depth >= MAX_DOCUMENT_NESTING_DEPTH) refuse2(path, `nests deeper than ${MAX_DOCUMENT_NESTING_DEPTH} levels`);
|
|
1947
|
+
}
|
|
1948
|
+
function untagMap(record, path, depth) {
|
|
1949
|
+
requireDepth(path, depth);
|
|
1950
|
+
for (const key of Object.keys(record)) {
|
|
1951
|
+
record[key] = untag(record[key], path === "" ? key : `${path}.${key}`, depth + 1);
|
|
1952
|
+
}
|
|
1953
|
+
return record;
|
|
1954
|
+
}
|
|
1955
|
+
function untagList(list, path, depth) {
|
|
1956
|
+
requireDepth(path, depth);
|
|
1957
|
+
for (let at = 0; at < list.length; at++) list[at] = untag(list[at], `${path}[${at}]`, depth + 1);
|
|
1958
|
+
return list;
|
|
1959
|
+
}
|
|
1960
|
+
function untagTag(tag, value, path, depth) {
|
|
1961
|
+
switch (tag) {
|
|
1962
|
+
case DATE_TAG:
|
|
1963
|
+
return dateOf(value, path);
|
|
1964
|
+
case BYTES_TAG:
|
|
1965
|
+
return bytesOf(value, path);
|
|
1966
|
+
case NUMBER_TAG:
|
|
1967
|
+
return numberOf(value, path);
|
|
1968
|
+
case OBJECT_TAG:
|
|
1969
|
+
if (!isRecord3(value)) refuse2(path, `holds an "${OBJECT_TAG}" tag whose value is not an object`);
|
|
1970
|
+
return untagMap(value, path, depth);
|
|
1971
|
+
default:
|
|
1972
|
+
return refuse2(path, `holds an object whose only key, "${tag}", names no tag`);
|
|
1973
|
+
}
|
|
1974
|
+
}
|
|
1975
|
+
function untag(value, path, depth) {
|
|
1976
|
+
if (typeof value !== "object" || value === null) return value;
|
|
1977
|
+
if (Array.isArray(value)) return untagList(value, path, depth);
|
|
1978
|
+
const record = value;
|
|
1979
|
+
const tag = soleTagKey(record);
|
|
1980
|
+
return tag === null ? untagMap(record, path, depth) : untagTag(tag, record[tag], path, depth);
|
|
1981
|
+
}
|
|
1982
|
+
function untagDocumentLine(text, parsed) {
|
|
1983
|
+
if (!text.includes(`"${TAG_PREFIX}`) && !text.includes(ESCAPED_TAG_PREFIX)) return parsed;
|
|
1984
|
+
return untagMap(parsed, "", 0);
|
|
1985
|
+
}
|
|
1986
|
+
|
|
1987
|
+
// src/engine/export/import.ts
|
|
1988
|
+
var DOCUMENT_EXPORT_FORMAT = "narsil-export";
|
|
1989
|
+
var DOCUMENT_EXPORT_VERSION = 1;
|
|
1990
|
+
function invalidHeader(message) {
|
|
1991
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, `The document export header ${message}`);
|
|
1992
|
+
}
|
|
1993
|
+
function configOfHeader(line) {
|
|
1994
|
+
if (line === void 0) invalidHeader("is missing");
|
|
1995
|
+
if (!("text" in line)) {
|
|
1996
|
+
invalidHeader(
|
|
1997
|
+
line.refused === "too-long" ? `passes the line limit of ${MAX_EXPORT_LINE_BYTES} bytes` : "is not UTF-8"
|
|
1998
|
+
);
|
|
1693
1999
|
}
|
|
1694
|
-
|
|
1695
|
-
|
|
2000
|
+
let raw;
|
|
2001
|
+
try {
|
|
2002
|
+
raw = JSON.parse(line.text);
|
|
2003
|
+
} catch {
|
|
2004
|
+
invalidHeader("is not JSON");
|
|
1696
2005
|
}
|
|
1697
|
-
if (
|
|
1698
|
-
|
|
2006
|
+
if (!isRecord2(raw) || raw.format !== DOCUMENT_EXPORT_FORMAT || raw.version !== DOCUMENT_EXPORT_VERSION) {
|
|
2007
|
+
invalidHeader(`must name format "${DOCUMENT_EXPORT_FORMAT}" and version ${DOCUMENT_EXPORT_VERSION}`);
|
|
1699
2008
|
}
|
|
1700
|
-
|
|
1701
|
-
|
|
2009
|
+
const configuration = requireConfigurationShape(raw.index, "document export header");
|
|
2010
|
+
const fields = restoredConfigFields(configuration, (message) => invalidHeader(message));
|
|
2011
|
+
const embedding = restoredEmbedding(configuration.embedding, (message) => invalidHeader(message));
|
|
2012
|
+
return {
|
|
2013
|
+
schema: nestedSchemaOf(configuration.schema),
|
|
2014
|
+
language: configuration.language,
|
|
2015
|
+
...fields,
|
|
2016
|
+
...embedding === void 0 ? {} : {
|
|
2017
|
+
embedding: {
|
|
2018
|
+
fields: embedding.fields,
|
|
2019
|
+
...embedding.adapter === void 0 ? {} : { adapter: embedding.adapter }
|
|
2020
|
+
}
|
|
2021
|
+
}
|
|
2022
|
+
};
|
|
2023
|
+
}
|
|
2024
|
+
async function prepareIndex(engine, indexName, config) {
|
|
2025
|
+
if (!engine.listIndexes().some((index) => index.name === indexName)) {
|
|
2026
|
+
await engine.createIndex(indexName, config);
|
|
2027
|
+
return;
|
|
1702
2028
|
}
|
|
1703
|
-
|
|
1704
|
-
|
|
2029
|
+
const existing = flattenSchema(engine.getStats(indexName).schema);
|
|
2030
|
+
const exported = flattenSchema(config.schema);
|
|
2031
|
+
const fields = /* @__PURE__ */ new Set([...Object.keys(existing), ...Object.keys(exported)]);
|
|
2032
|
+
for (const field of fields) {
|
|
2033
|
+
if (existing[field] === exported[field]) continue;
|
|
2034
|
+
throw new NarsilError(
|
|
2035
|
+
ErrorCodes.CONFIG_INVALID,
|
|
2036
|
+
`Index "${indexName}" gives field "${field}" another type than the export does, so the import cannot write into it`,
|
|
2037
|
+
{ indexName, field }
|
|
2038
|
+
);
|
|
1705
2039
|
}
|
|
1706
|
-
|
|
1707
|
-
|
|
2040
|
+
}
|
|
2041
|
+
function highestReportedAt(refused) {
|
|
2042
|
+
let highest = 0;
|
|
2043
|
+
for (let at = 1; at < refused.length; at++) {
|
|
2044
|
+
if (refused[at].line > refused[highest].line) highest = at;
|
|
1708
2045
|
}
|
|
1709
|
-
|
|
1710
|
-
|
|
2046
|
+
return highest;
|
|
2047
|
+
}
|
|
2048
|
+
function refuse3(state, line, code, message) {
|
|
2049
|
+
state.result.refusedCount += 1;
|
|
2050
|
+
const refused = state.result.refused;
|
|
2051
|
+
if (refused.length < MAX_IMPORT_REFUSALS_REPORTED) {
|
|
2052
|
+
refused.push({ line, code, message });
|
|
2053
|
+
if (refused.length === MAX_IMPORT_REFUSALS_REPORTED) state.highestReported = highestReportedAt(refused);
|
|
2054
|
+
return;
|
|
1711
2055
|
}
|
|
1712
|
-
if (
|
|
1713
|
-
|
|
1714
|
-
|
|
1715
|
-
|
|
1716
|
-
|
|
1717
|
-
|
|
1718
|
-
|
|
1719
|
-
|
|
2056
|
+
if (line >= refused[state.highestReported].line) return;
|
|
2057
|
+
refused[state.highestReported] = { line, code, message };
|
|
2058
|
+
state.highestReported = highestReportedAt(refused);
|
|
2059
|
+
}
|
|
2060
|
+
async function flush(state) {
|
|
2061
|
+
if (state.batch.length === 0) return;
|
|
2062
|
+
const batch = state.batch;
|
|
2063
|
+
state.batch = [];
|
|
2064
|
+
state.batchIds = /* @__PURE__ */ new Set();
|
|
2065
|
+
const lineOf = new Map(batch.map((entry) => [entry.docId, entry.line]));
|
|
2066
|
+
const inserts = [];
|
|
2067
|
+
const updates = [];
|
|
2068
|
+
for (const entry of batch) {
|
|
2069
|
+
if (await state.engine.has(state.indexName, entry.docId))
|
|
2070
|
+
updates.push({ docId: entry.docId, document: entry.document });
|
|
2071
|
+
else inserts.push(entry.document);
|
|
2072
|
+
}
|
|
2073
|
+
for (const outcome of [
|
|
2074
|
+
inserts.length === 0 ? null : await state.engine.insertBatch(state.indexName, inserts, { skipClone: true }),
|
|
2075
|
+
updates.length === 0 ? null : await state.engine.updateBatch(state.indexName, updates)
|
|
2076
|
+
]) {
|
|
2077
|
+
if (outcome === null) continue;
|
|
2078
|
+
state.result.imported += outcome.succeeded.length;
|
|
2079
|
+
for (const failure of outcome.failed) {
|
|
2080
|
+
const code = failure.error instanceof NarsilError ? failure.error.code : ErrorCodes.DOC_VALIDATION_FAILED;
|
|
2081
|
+
refuse3(state, lineOf.get(failure.docId) ?? 0, code, failure.error.message);
|
|
2082
|
+
}
|
|
1720
2083
|
}
|
|
1721
|
-
|
|
1722
|
-
|
|
2084
|
+
}
|
|
2085
|
+
async function takeLine(state, entry) {
|
|
2086
|
+
if (!("text" in entry)) {
|
|
2087
|
+
const message = entry.refused === "too-long" ? `The line passes the limit of ${MAX_EXPORT_LINE_BYTES} bytes` : "The line is not UTF-8";
|
|
2088
|
+
refuse3(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, message);
|
|
2089
|
+
return;
|
|
1723
2090
|
}
|
|
1724
|
-
if (
|
|
1725
|
-
|
|
2091
|
+
if (entry.text.length === 0) return;
|
|
2092
|
+
let document;
|
|
2093
|
+
try {
|
|
2094
|
+
document = JSON.parse(entry.text);
|
|
2095
|
+
} catch {
|
|
2096
|
+
refuse3(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, "The line is not JSON");
|
|
2097
|
+
return;
|
|
1726
2098
|
}
|
|
1727
|
-
if (
|
|
1728
|
-
|
|
2099
|
+
if (!isRecord2(document) || typeof document.id !== "string" || document.id.length === 0) {
|
|
2100
|
+
refuse3(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, "The line holds no document with a string id");
|
|
2101
|
+
return;
|
|
1729
2102
|
}
|
|
1730
|
-
|
|
1731
|
-
|
|
2103
|
+
try {
|
|
2104
|
+
untagDocumentLine(entry.text, document);
|
|
2105
|
+
} catch (error) {
|
|
2106
|
+
if (!(error instanceof NarsilError)) throw error;
|
|
2107
|
+
refuse3(state, entry.line, error.code, error.message);
|
|
2108
|
+
return;
|
|
1732
2109
|
}
|
|
1733
|
-
if (
|
|
1734
|
-
|
|
2110
|
+
if (state.batchIds.has(document.id)) await flush(state);
|
|
2111
|
+
state.batch.push({ line: entry.line, docId: document.id, document });
|
|
2112
|
+
state.batchIds.add(document.id);
|
|
2113
|
+
if (state.batch.length >= IMPORT_BATCH_DOCUMENTS) await flush(state);
|
|
2114
|
+
}
|
|
2115
|
+
async function readDocumentExport(engine, indexName, source) {
|
|
2116
|
+
const lines = exportLines(source, MAX_EXPORT_LINE_BYTES);
|
|
2117
|
+
const header = await lines.next();
|
|
2118
|
+
await prepareIndex(engine, indexName, configOfHeader(header.done === true ? void 0 : header.value));
|
|
2119
|
+
const state = {
|
|
2120
|
+
engine,
|
|
2121
|
+
indexName,
|
|
2122
|
+
batch: [],
|
|
2123
|
+
batchIds: /* @__PURE__ */ new Set(),
|
|
2124
|
+
result: { imported: 0, refusedCount: 0, refused: [] },
|
|
2125
|
+
highestReported: 0
|
|
2126
|
+
};
|
|
2127
|
+
for await (const entry of lines) {
|
|
2128
|
+
await takeLine(state, entry);
|
|
2129
|
+
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
1735
2130
|
}
|
|
1736
|
-
|
|
1737
|
-
|
|
2131
|
+
await flush(state);
|
|
2132
|
+
state.result.refused.sort((a, b) => a.line - b.line);
|
|
2133
|
+
return state.result;
|
|
2134
|
+
}
|
|
2135
|
+
|
|
2136
|
+
// src/engine/export/json-line.ts
|
|
2137
|
+
function refuse4(path, kind, docId) {
|
|
2138
|
+
throw new NarsilError(
|
|
2139
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
2140
|
+
`Document "${docId}" holds ${kind} at "${path || "(document)"}", which a document export cannot carry as JSON`,
|
|
2141
|
+
{ docId, field: path }
|
|
2142
|
+
);
|
|
2143
|
+
}
|
|
2144
|
+
function encodeNumber(value, path, target) {
|
|
2145
|
+
if (Number.isFinite(value)) return Object.is(value, -0) ? "-0" : JSON.stringify(value);
|
|
2146
|
+
if (!target.tagged) refuse4(path, String(value), target.docId);
|
|
2147
|
+
return `{"${NUMBER_TAG}":"${String(value)}"}`;
|
|
2148
|
+
}
|
|
2149
|
+
function encodeDate(value, path, target) {
|
|
2150
|
+
const milliseconds = value.getTime();
|
|
2151
|
+
if (!target.tagged || Number.isNaN(milliseconds)) refuse4(path, "a date", target.docId);
|
|
2152
|
+
const seconds = Math.floor(milliseconds / MILLISECONDS_PER_SECOND);
|
|
2153
|
+
const nanoseconds = (milliseconds - seconds * MILLISECONDS_PER_SECOND) * NANOSECONDS_PER_MILLISECOND;
|
|
2154
|
+
return `{"${DATE_TAG}":[${seconds},${nanoseconds}]}`;
|
|
2155
|
+
}
|
|
2156
|
+
function encodeBytes(value, path, target) {
|
|
2157
|
+
if (!target.tagged) refuse4(path, "bytes", target.docId);
|
|
2158
|
+
return `{"${BYTES_TAG}":"${encodeBase64(value)}"}`;
|
|
2159
|
+
}
|
|
2160
|
+
function encodeList(value, path, target) {
|
|
2161
|
+
target.parts.push("[");
|
|
2162
|
+
for (let at = 0; at < value.length; at++) {
|
|
2163
|
+
if (at > 0) target.parts.push(",");
|
|
2164
|
+
encodeValue(value[at], `${path}[${at}]`, target);
|
|
2165
|
+
}
|
|
2166
|
+
target.parts.push("]");
|
|
2167
|
+
}
|
|
2168
|
+
function encodeMap(value, path, target) {
|
|
2169
|
+
const proto = Object.getPrototypeOf(value);
|
|
2170
|
+
if (proto !== Object.prototype && proto !== null) refuse4(path, "a value other than a plain object", target.docId);
|
|
2171
|
+
const entries = Object.entries(value).filter(([, entry]) => entry !== void 0);
|
|
2172
|
+
const wrapped = target.tagged && entries.length === 1 && startsWithTagPrefix(entries[0][0]);
|
|
2173
|
+
target.parts.push(wrapped ? `{"${OBJECT_TAG}":{` : "{");
|
|
2174
|
+
for (let at = 0; at < entries.length; at++) {
|
|
2175
|
+
const [key, entry] = entries[at];
|
|
2176
|
+
if (at > 0) target.parts.push(",");
|
|
2177
|
+
target.parts.push(JSON.stringify(key), ":");
|
|
2178
|
+
encodeValue(entry, path === "" ? key : `${path}.${key}`, target);
|
|
2179
|
+
}
|
|
2180
|
+
target.parts.push(wrapped ? "}}" : "}");
|
|
2181
|
+
}
|
|
2182
|
+
function encodeValue(value, path, target) {
|
|
2183
|
+
if (value === null) {
|
|
2184
|
+
target.parts.push("null");
|
|
2185
|
+
return;
|
|
1738
2186
|
}
|
|
1739
|
-
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
2187
|
+
switch (typeof value) {
|
|
2188
|
+
case "string":
|
|
2189
|
+
target.parts.push(JSON.stringify(value));
|
|
2190
|
+
return;
|
|
2191
|
+
case "boolean":
|
|
2192
|
+
target.parts.push(value ? "true" : "false");
|
|
2193
|
+
return;
|
|
2194
|
+
case "number":
|
|
2195
|
+
target.parts.push(encodeNumber(value, path, target));
|
|
2196
|
+
return;
|
|
2197
|
+
case "object":
|
|
2198
|
+
break;
|
|
2199
|
+
default:
|
|
2200
|
+
refuse4(path, `a ${typeof value}`, target.docId);
|
|
2201
|
+
}
|
|
2202
|
+
if (Array.isArray(value)) encodeList(value, path, target);
|
|
2203
|
+
else if (value instanceof Date) target.parts.push(encodeDate(value, path, target));
|
|
2204
|
+
else if (value instanceof Uint8Array) target.parts.push(encodeBytes(value, path, target));
|
|
2205
|
+
else encodeMap(value, path, target);
|
|
2206
|
+
}
|
|
2207
|
+
function encodeExactJson(value, docId) {
|
|
2208
|
+
const target = { docId, tagged: false, parts: [] };
|
|
2209
|
+
encodeValue(value, "", target);
|
|
2210
|
+
return target.parts.join("");
|
|
2211
|
+
}
|
|
2212
|
+
function encodeDocumentJson(document, docId) {
|
|
2213
|
+
const target = { docId, tagged: true, parts: [] };
|
|
2214
|
+
encodeValue(document, "", target);
|
|
2215
|
+
return target.parts.join("");
|
|
1744
2216
|
}
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
return
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
|
|
1759
|
-
|
|
1760
|
-
|
|
1761
|
-
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
...entry.arrayPositionGap !== null ? { arrayPositionGap: entry.arrayPositionGap } : {},
|
|
1765
|
-
...typeof entry.config.tokenizer === "string" ? { tokenizer: entry.config.tokenizer } : {},
|
|
1766
|
-
...typeof entry.config.stopWords === "string" ? { stopWords: entry.config.stopWords } : {},
|
|
1767
|
-
...entry.config.stopWords instanceof Set ? { stopWordList: [...entry.config.stopWords].sort(compareCodePoints) } : {},
|
|
1768
|
-
...entry.analysisGeneration > 0 ? { analysisGeneration: entry.analysisGeneration } : {},
|
|
1769
|
-
...entry.pendingAnalysis !== null ? { pendingAnalysis: entry.pendingAnalysis } : {},
|
|
1770
|
-
...entry.config.partitions !== void 0 ? { partitionLimits: entry.config.partitions } : {},
|
|
1771
|
-
...entry.config.defaultScoring !== void 0 ? { defaultScoring: entry.config.defaultScoring } : {},
|
|
1772
|
-
...entry.config.trackPositions !== void 0 ? { trackPositions: entry.config.trackPositions } : {},
|
|
1773
|
-
...entry.config.textStorage !== void 0 ? { textStorage: entry.config.textStorage } : {},
|
|
1774
|
-
...entry.config.strict !== void 0 ? { strict: entry.config.strict } : {},
|
|
1775
|
-
...entry.config.required !== void 0 ? { required: entry.config.required } : {},
|
|
1776
|
-
...entry.config.vectorPromotion !== void 0 ? { vectorPromotion: entry.config.vectorPromotion } : {},
|
|
1777
|
-
...entry.config.patternValueLimit !== void 0 ? { patternValueLimit: entry.config.patternValueLimit } : {},
|
|
1778
|
-
...entry.synonyms !== null ? { synonyms: rulesOfStoredSynonyms(entry.synonyms) } : {},
|
|
1779
|
-
...syncMetadataOf(entry),
|
|
1780
|
-
...entry.querySuggestions !== null ? { querySuggestions: entry.querySuggestions } : {}
|
|
1781
|
-
};
|
|
2217
|
+
|
|
2218
|
+
// src/engine/export/paged-lines.ts
|
|
2219
|
+
async function coldFieldOf(checkpoint, field) {
|
|
2220
|
+
const { directory } = checkpoint;
|
|
2221
|
+
if (directory === void 0 || field.files.length === 0 || field.graphParts.length === 0) return null;
|
|
2222
|
+
if (![...field.files, ...field.graphParts].every((file) => file.headOffset !== void 0)) return null;
|
|
2223
|
+
const fieldType = checkpoint.manifest.schema[field.fieldPath];
|
|
2224
|
+
const dimension = fieldType === void 0 ? null : parseVectorDimension(fieldType);
|
|
2225
|
+
if (dimension === null) return null;
|
|
2226
|
+
const withPath = async (ref) => ({ ref, path: await directory.pathOf(ref.key) });
|
|
2227
|
+
return openColdVectorField({
|
|
2228
|
+
fieldPath: field.fieldPath,
|
|
2229
|
+
dimension,
|
|
2230
|
+
metric: "cosine",
|
|
2231
|
+
codeBits: null,
|
|
2232
|
+
files: await Promise.all(field.files.map(withPath)),
|
|
2233
|
+
graphParts: await Promise.all(field.graphParts.map(withPath)),
|
|
2234
|
+
resolvePartition: () => void 0
|
|
2235
|
+
});
|
|
1782
2236
|
}
|
|
1783
|
-
function
|
|
1784
|
-
const
|
|
1785
|
-
|
|
2237
|
+
async function openFileVectorReads(checkpoint, wholeFileReads) {
|
|
2238
|
+
const fields = /* @__PURE__ */ new Map();
|
|
2239
|
+
for (const field of checkpoint.manifest.vectors) {
|
|
2240
|
+
const opened = await coldFieldOf(checkpoint, field);
|
|
2241
|
+
if (opened !== null) fields.set(field.fieldPath, opened);
|
|
2242
|
+
}
|
|
2243
|
+
const fallback = fields.size === checkpoint.manifest.vectors.length ? null : await wholeFileReads();
|
|
1786
2244
|
return {
|
|
1787
|
-
|
|
1788
|
-
|
|
1789
|
-
|
|
1790
|
-
|
|
2245
|
+
async vectorOf(fieldPath, docId) {
|
|
2246
|
+
const field = fields.get(fieldPath);
|
|
2247
|
+
if (field === void 0) return fallback === null ? null : fallback.vectorOf(fieldPath, docId);
|
|
2248
|
+
const number = field.numberOf(docId);
|
|
2249
|
+
return number === void 0 ? null : Array.from(field.vectorOfNumber(number));
|
|
1791
2250
|
},
|
|
1792
|
-
|
|
1793
|
-
|
|
1794
|
-
|
|
1795
|
-
}
|
|
1796
|
-
function durableMetadataConfigOf(entry) {
|
|
1797
|
-
const terms = metadataConfigOf(entry);
|
|
1798
|
-
const durable = entry.durableAnalysis;
|
|
1799
|
-
if (durable === null) return terms;
|
|
1800
|
-
const {
|
|
1801
|
-
language,
|
|
1802
|
-
tokenizer,
|
|
1803
|
-
stopWords,
|
|
1804
|
-
stopWordList,
|
|
1805
|
-
analysisRevision: _analysisRevision,
|
|
1806
|
-
arrayPositionGap: _arrayPositionGap,
|
|
1807
|
-
analysisGeneration: _analysisGeneration,
|
|
1808
|
-
...rest
|
|
1809
|
-
} = terms;
|
|
1810
|
-
return {
|
|
1811
|
-
...rest,
|
|
1812
|
-
language: durable.language,
|
|
1813
|
-
...durable.tokenizer !== void 0 ? { tokenizer: durable.tokenizer } : {},
|
|
1814
|
-
...durable.stopWords !== void 0 ? { stopWords: durable.stopWords } : {},
|
|
1815
|
-
...durable.stopWordList !== void 0 ? { stopWordList: durable.stopWordList } : {},
|
|
1816
|
-
...durable.analysisRevision !== null ? { analysisRevision: durable.analysisRevision } : {},
|
|
1817
|
-
...durable.arrayPositionGap !== null ? { arrayPositionGap: durable.arrayPositionGap } : {},
|
|
1818
|
-
...durable.generation > 0 ? { analysisGeneration: durable.generation } : {},
|
|
1819
|
-
pendingAnalysis: {
|
|
1820
|
-
generation: entry.analysisGeneration,
|
|
1821
|
-
language,
|
|
1822
|
-
...tokenizer !== void 0 ? { tokenizer } : {},
|
|
1823
|
-
...stopWords !== void 0 ? { stopWords } : {},
|
|
1824
|
-
...stopWordList !== void 0 ? { stopWordList } : {}
|
|
2251
|
+
close() {
|
|
2252
|
+
for (const field of fields.values()) field.close();
|
|
2253
|
+
fallback?.close();
|
|
1825
2254
|
}
|
|
1826
2255
|
};
|
|
1827
2256
|
}
|
|
1828
|
-
function
|
|
1829
|
-
|
|
1830
|
-
|
|
1831
|
-
|
|
1832
|
-
|
|
1833
|
-
|
|
1834
|
-
|
|
1835
|
-
|
|
1836
|
-
function conflictingStoredPath(path) {
|
|
1837
|
-
return new NarsilError(
|
|
1838
|
-
ErrorCodes.SCHEMA_INVALID_TYPE,
|
|
1839
|
-
`Field path "${path}" is declared both as a field and as an object of fields`,
|
|
1840
|
-
{ path }
|
|
1841
|
-
);
|
|
1842
|
-
}
|
|
1843
|
-
function defineOwn(level, key, value) {
|
|
1844
|
-
Object.defineProperty(level, key, { value, enumerable: true, writable: true, configurable: true });
|
|
1845
|
-
}
|
|
1846
|
-
function placeStoredField(nested, path, type) {
|
|
1847
|
-
const segments = path.split(".");
|
|
1848
|
-
let level = nested;
|
|
1849
|
-
for (let at = 0; at < segments.length; at++) {
|
|
1850
|
-
const segment = segments[at];
|
|
1851
|
-
const held = Object.hasOwn(level, segment) ? level[segment] : void 0;
|
|
1852
|
-
if (at === segments.length - 1) {
|
|
1853
|
-
if (held !== void 0) throw conflictingStoredPath(path);
|
|
1854
|
-
defineOwn(level, segment, type);
|
|
1855
|
-
return;
|
|
2257
|
+
async function* pagedPartitionLines(checkpoint, partition, vectors, openWholeSegment) {
|
|
2258
|
+
const { directory } = checkpoint;
|
|
2259
|
+
if (directory === void 0) return;
|
|
2260
|
+
const scans = [];
|
|
2261
|
+
try {
|
|
2262
|
+
for (const ref of partition.segments) {
|
|
2263
|
+
const paged = await isPagedSegmentFile(directory, ref.key);
|
|
2264
|
+
scans.push(paged ? await openPagedSegmentScan(directory, ref) : await openWholeSegment(ref));
|
|
1856
2265
|
}
|
|
1857
|
-
|
|
1858
|
-
const
|
|
1859
|
-
|
|
1860
|
-
|
|
1861
|
-
|
|
1862
|
-
|
|
1863
|
-
|
|
1864
|
-
|
|
2266
|
+
for (let at = 0; at < scans.length; at++) {
|
|
2267
|
+
const scan = scans[at];
|
|
2268
|
+
for (let ordinal = 0; ordinal < scan.count; ordinal++) {
|
|
2269
|
+
const docId = scan.idAt(ordinal);
|
|
2270
|
+
if (!countsAsLatestHolder(scans, at, docId)) continue;
|
|
2271
|
+
const document = { id: docId, ...scan.storedDocumentAt(ordinal) };
|
|
2272
|
+
document.id = docId;
|
|
2273
|
+
for (const field of checkpoint.manifest.vectors) {
|
|
2274
|
+
const vector = await vectors.vectorOf(field.fieldPath, docId);
|
|
2275
|
+
if (vector !== null) setNestedValue(document, field.fieldPath, vector);
|
|
2276
|
+
}
|
|
2277
|
+
yield `${encodeDocumentJson(document, docId)}
|
|
2278
|
+
`;
|
|
2279
|
+
}
|
|
1865
2280
|
}
|
|
2281
|
+
} finally {
|
|
2282
|
+
for (const scan of scans) scan.close();
|
|
1866
2283
|
}
|
|
1867
2284
|
}
|
|
1868
|
-
|
|
1869
|
-
|
|
1870
|
-
|
|
1871
|
-
|
|
1872
|
-
|
|
1873
|
-
|
|
1874
|
-
|
|
2285
|
+
|
|
2286
|
+
// src/engine/export/write.ts
|
|
2287
|
+
var VECTOR_FILE_ENVELOPE_VERSION2 = 2;
|
|
2288
|
+
var FLOAT32_BYTES = 4;
|
|
2289
|
+
var ENVELOPE_VERSION_OFFSET = 4;
|
|
2290
|
+
async function locateVectors(checkpoint) {
|
|
2291
|
+
const located = /* @__PURE__ */ new Map();
|
|
2292
|
+
for (const field of checkpoint.manifest.vectors) {
|
|
2293
|
+
const byDocId = /* @__PURE__ */ new Map();
|
|
2294
|
+
for (const ref of field.files) {
|
|
2295
|
+
const { payload } = await openChecksummedEnvelope(await checkpoint.readFile(ref), {
|
|
2296
|
+
envelopeFormatVersion: VECTOR_FILE_ENVELOPE_VERSION2,
|
|
2297
|
+
what: `vector file "${ref.key}"`,
|
|
2298
|
+
allowsCompression: false
|
|
2299
|
+
});
|
|
2300
|
+
const decoded = decodeVectorFilePayload(decode(payload));
|
|
2301
|
+
for (let position = 0; position < decoded.docIds.length; position++) {
|
|
2302
|
+
if (isDead(ref.dead, position)) continue;
|
|
2303
|
+
byDocId.set(decoded.docIds[position], { ref, position, dimension: decoded.dimension });
|
|
2304
|
+
}
|
|
1875
2305
|
}
|
|
2306
|
+
located.set(field.fieldPath, byDocId);
|
|
1876
2307
|
}
|
|
2308
|
+
return located;
|
|
1877
2309
|
}
|
|
1878
|
-
function
|
|
1879
|
-
|
|
1880
|
-
const
|
|
1881
|
-
|
|
1882
|
-
return nested;
|
|
2310
|
+
async function readVector(checkpoint, location) {
|
|
2311
|
+
const width = location.dimension * FLOAT32_BYTES;
|
|
2312
|
+
const offset = location.ref.bytes - (location.ref.count - location.position) * width;
|
|
2313
|
+
return Array.from(bytesToVectors((await checkpoint.readRange(location.ref, offset, width)).slice()));
|
|
1883
2314
|
}
|
|
1884
|
-
|
|
1885
|
-
|
|
1886
|
-
|
|
1887
|
-
|
|
2315
|
+
async function readSegment(checkpoint, ref) {
|
|
2316
|
+
const { payload } = await openChecksummedEnvelope(await checkpoint.readFile(ref), {
|
|
2317
|
+
envelopeFormatVersion: KEYWORD_SEGMENT_ENVELOPE_VERSION,
|
|
2318
|
+
what: `segment "${ref.key}"`,
|
|
2319
|
+
allowsCompression: true
|
|
2320
|
+
});
|
|
2321
|
+
return decodeKeywordSegment(payload, false);
|
|
2322
|
+
}
|
|
2323
|
+
var decoder = new TextDecoder();
|
|
2324
|
+
function idAt(contents, ordinal) {
|
|
2325
|
+
const ids = contents.arrays.idTable;
|
|
2326
|
+
return decoder.decode(ids.blob.subarray(ids.offsets[ordinal], ids.offsets[ordinal + 1]));
|
|
2327
|
+
}
|
|
2328
|
+
async function winnersOf(checkpoint, partition) {
|
|
2329
|
+
const winners = /* @__PURE__ */ new Map();
|
|
2330
|
+
for (let at = 0; at < partition.segments.length; at++) {
|
|
2331
|
+
const contents = await readSegment(checkpoint, partition.segments[at]);
|
|
2332
|
+
for (const docId of contents.tombstones) winners.delete(docId);
|
|
2333
|
+
for (let ordinal = 0; ordinal < contents.arrays.documentCount; ordinal++) winners.set(idAt(contents, ordinal), at);
|
|
2334
|
+
}
|
|
2335
|
+
return winners;
|
|
2336
|
+
}
|
|
2337
|
+
async function wholeSegmentScan(checkpoint, ref) {
|
|
2338
|
+
const contents = await readSegment(checkpoint, ref);
|
|
2339
|
+
const table = contents.arrays.documentTable;
|
|
2340
|
+
const ids = /* @__PURE__ */ new Set();
|
|
2341
|
+
for (let ordinal = 0; ordinal < contents.arrays.documentCount; ordinal++) ids.add(idAt(contents, ordinal));
|
|
2342
|
+
return {
|
|
2343
|
+
count: contents.arrays.documentCount,
|
|
2344
|
+
schema: contents.schema,
|
|
2345
|
+
tombstones: new Set(contents.tombstones),
|
|
2346
|
+
idAt: (ordinal) => idAt(contents, ordinal),
|
|
2347
|
+
holds: (docId) => ids.has(docId),
|
|
2348
|
+
storedDocumentAt: (ordinal) => decodeStoredDocument(
|
|
2349
|
+
table.blob.subarray(table.offsets[ordinal], table.offsets[ordinal + 1]),
|
|
2350
|
+
table.discardedFields
|
|
2351
|
+
),
|
|
2352
|
+
close: () => void 0
|
|
2353
|
+
};
|
|
2354
|
+
}
|
|
2355
|
+
async function wholeFileVectorReads(checkpoint) {
|
|
2356
|
+
const located = await locateVectors(checkpoint);
|
|
2357
|
+
return {
|
|
2358
|
+
async vectorOf(fieldPath, docId) {
|
|
2359
|
+
const location = located.get(fieldPath)?.get(docId);
|
|
2360
|
+
return location === void 0 ? null : readVector(checkpoint, location);
|
|
2361
|
+
},
|
|
2362
|
+
close: () => located.clear()
|
|
2363
|
+
};
|
|
2364
|
+
}
|
|
2365
|
+
async function* pagedDocumentLines(checkpoint) {
|
|
2366
|
+
const vectors = await openFileVectorReads(checkpoint, () => wholeFileVectorReads(checkpoint));
|
|
1888
2367
|
try {
|
|
1889
|
-
for (const
|
|
1890
|
-
|
|
1891
|
-
if (vectorIndex !== void 0) letBuildsIn.push(await vectorIndex.holdGraphComplete());
|
|
2368
|
+
for (const partition of checkpoint.manifest.partitions) {
|
|
2369
|
+
yield* pagedPartitionLines(checkpoint, partition, vectors, (ref) => wholeSegmentScan(checkpoint, ref));
|
|
1892
2370
|
}
|
|
1893
|
-
return await capture2();
|
|
1894
2371
|
} finally {
|
|
1895
|
-
|
|
2372
|
+
vectors.close();
|
|
1896
2373
|
}
|
|
1897
2374
|
}
|
|
1898
|
-
|
|
1899
|
-
|
|
1900
|
-
|
|
1901
|
-
|
|
1902
|
-
|
|
1903
|
-
|
|
1904
|
-
let sum = 0;
|
|
1905
|
-
let rest = vector >>> 0;
|
|
1906
|
-
for (let row = 0; rest !== 0; row++, rest >>>= 1) {
|
|
1907
|
-
if (rest & 1) sum ^= operator[row];
|
|
2375
|
+
async function anyPartitionIsPaged(checkpoint) {
|
|
2376
|
+
for (const partition of checkpoint.manifest.partitions) {
|
|
2377
|
+
for (const ref of partition.segments) {
|
|
2378
|
+
const [version] = await checkpoint.readRange(ref, ENVELOPE_VERSION_OFFSET, 1);
|
|
2379
|
+
if (version !== KEYWORD_SEGMENT_ENVELOPE_VERSION) return true;
|
|
2380
|
+
}
|
|
1908
2381
|
}
|
|
1909
|
-
return
|
|
1910
|
-
}
|
|
1911
|
-
function squared(operator) {
|
|
1912
|
-
const result = new Uint32Array(CRC_BITS);
|
|
1913
|
-
for (let row = 0; row < CRC_BITS; row++) result[row] = timesVector(operator, operator[row]);
|
|
1914
|
-
return result;
|
|
1915
|
-
}
|
|
1916
|
-
function firstOperator() {
|
|
1917
|
-
let oneZeroBit = new Uint32Array(CRC_BITS);
|
|
1918
|
-
oneZeroBit[0] = IEEE_POLYNOMIAL;
|
|
1919
|
-
for (let row = 1; row < CRC_BITS; row++) oneZeroBit[row] = 1 << row - 1;
|
|
1920
|
-
for (let doubling = 0; doubling < 3; doubling++) oneZeroBit = squared(oneZeroBit);
|
|
1921
|
-
return oneZeroBit;
|
|
2382
|
+
return false;
|
|
1922
2383
|
}
|
|
1923
|
-
function
|
|
1924
|
-
if (
|
|
1925
|
-
|
|
1926
|
-
|
|
2384
|
+
async function* documentLines(checkpoint) {
|
|
2385
|
+
if (await anyPartitionIsPaged(checkpoint)) {
|
|
2386
|
+
yield* pagedDocumentLines(checkpoint);
|
|
2387
|
+
return;
|
|
1927
2388
|
}
|
|
1928
|
-
|
|
1929
|
-
|
|
1930
|
-
|
|
1931
|
-
|
|
1932
|
-
|
|
1933
|
-
|
|
1934
|
-
|
|
1935
|
-
|
|
2389
|
+
const vectors = await locateVectors(checkpoint);
|
|
2390
|
+
for (const partition of checkpoint.manifest.partitions) {
|
|
2391
|
+
const winners = await winnersOf(checkpoint, partition);
|
|
2392
|
+
for (let at = 0; at < partition.segments.length; at++) {
|
|
2393
|
+
const contents = await readSegment(checkpoint, partition.segments[at]);
|
|
2394
|
+
const table = contents.arrays.documentTable;
|
|
2395
|
+
for (let ordinal = 0; ordinal < contents.arrays.documentCount; ordinal++) {
|
|
2396
|
+
const docId = idAt(contents, ordinal);
|
|
2397
|
+
if (winners.get(docId) !== at) continue;
|
|
2398
|
+
const stored = decodeStoredDocument(
|
|
2399
|
+
table.blob.subarray(table.offsets[ordinal], table.offsets[ordinal + 1]),
|
|
2400
|
+
table.discardedFields
|
|
2401
|
+
);
|
|
2402
|
+
const document = { id: docId, ...stored };
|
|
2403
|
+
document.id = docId;
|
|
2404
|
+
for (const [fieldPath, located] of vectors) {
|
|
2405
|
+
const location = located.get(docId);
|
|
2406
|
+
if (location !== void 0) setNestedValue(document, fieldPath, await readVector(checkpoint, location));
|
|
2407
|
+
}
|
|
2408
|
+
yield `${encodeDocumentJson(document, docId)}
|
|
2409
|
+
`;
|
|
2410
|
+
}
|
|
2411
|
+
}
|
|
1936
2412
|
}
|
|
1937
|
-
return (crc ^ second) >>> 0;
|
|
1938
2413
|
}
|
|
1939
|
-
|
|
1940
|
-
|
|
1941
|
-
|
|
1942
|
-
|
|
1943
|
-
|
|
1944
|
-
|
|
1945
|
-
|
|
1946
|
-
|
|
1947
|
-
|
|
1948
|
-
|
|
1949
|
-
|
|
1950
|
-
|
|
1951
|
-
|
|
1952
|
-
|
|
1953
|
-
|
|
1954
|
-
|
|
1955
|
-
|
|
1956
|
-
|
|
1957
|
-
|
|
1958
|
-
|
|
1959
|
-
|
|
1960
|
-
|
|
1961
|
-
|
|
1962
|
-
|
|
2414
|
+
async function* exportLines2(header, checkpoint, positions) {
|
|
2415
|
+
const exportHeader = { format: DOCUMENT_EXPORT_FORMAT, version: DOCUMENT_EXPORT_VERSION, index: header };
|
|
2416
|
+
yield `${encodeExactJson(positions === null ? exportHeader : { ...exportHeader, ...positions }, "")}
|
|
2417
|
+
`;
|
|
2418
|
+
yield* documentLines(checkpoint);
|
|
2419
|
+
}
|
|
2420
|
+
function documentExportStream(header, checkpoint, positions = null) {
|
|
2421
|
+
const lines = exportLines2(header, checkpoint, positions);
|
|
2422
|
+
const encoder = new TextEncoder();
|
|
2423
|
+
let released = false;
|
|
2424
|
+
const release = () => {
|
|
2425
|
+
if (released) return;
|
|
2426
|
+
released = true;
|
|
2427
|
+
checkpoint.release();
|
|
2428
|
+
};
|
|
2429
|
+
return new ReadableStream({
|
|
2430
|
+
async pull(controller) {
|
|
2431
|
+
try {
|
|
2432
|
+
let text = "";
|
|
2433
|
+
while (text.length < EXPORT_CHUNK_BYTES) {
|
|
2434
|
+
const step = await lines.next();
|
|
2435
|
+
if (step.done) {
|
|
2436
|
+
if (text.length > 0) controller.enqueue(encoder.encode(text));
|
|
2437
|
+
release();
|
|
2438
|
+
controller.close();
|
|
2439
|
+
return;
|
|
2440
|
+
}
|
|
2441
|
+
text += step.value;
|
|
2442
|
+
}
|
|
2443
|
+
controller.enqueue(encoder.encode(text));
|
|
2444
|
+
} catch (error) {
|
|
2445
|
+
release();
|
|
2446
|
+
controller.error(error);
|
|
2447
|
+
}
|
|
2448
|
+
},
|
|
2449
|
+
async cancel() {
|
|
2450
|
+
release();
|
|
2451
|
+
await lines.return(void 0);
|
|
2452
|
+
}
|
|
2453
|
+
});
|
|
2454
|
+
}
|
|
2455
|
+
|
|
2456
|
+
// src/partitioning/saved-partitions.ts
|
|
2457
|
+
function requireSavedPartitionCount(partitionCount, source, indexName) {
|
|
2458
|
+
if (partitionCount <= MAX_PARTITIONS) return;
|
|
2459
|
+
throw new NarsilError(
|
|
2460
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
2461
|
+
`The ${source} of index "${indexName}" holds partition ${partitionCount - 1}, and a saved index numbers its partitions from 0 to ${MAX_PARTITIONS - 1}`,
|
|
2462
|
+
{ indexName, partitionId: partitionCount - 1, maxPartitions: MAX_PARTITIONS }
|
|
2463
|
+
);
|
|
2464
|
+
}
|
|
2465
|
+
function highestPartitionIdOf(partitions) {
|
|
2466
|
+
let highest = partitions.length - 1;
|
|
2467
|
+
for (const partition of partitions) if (partition.partitionId > highest) highest = partition.partitionId;
|
|
2468
|
+
return highest;
|
|
2469
|
+
}
|
|
2470
|
+
|
|
2471
|
+
// src/persistence/paged/read-ahead.ts
|
|
2472
|
+
var FILE_PAGE_SIZES = /* @__PURE__ */ new Set([SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES]);
|
|
2473
|
+
function rangeOf(path, section) {
|
|
2474
|
+
return { path, offset: HEADER_SIZE + section.offset, length: section.length };
|
|
2475
|
+
}
|
|
2476
|
+
async function readRangesAhead(ranges) {
|
|
2477
|
+
let rangeAt = 0;
|
|
2478
|
+
let doneInRange = 0;
|
|
2479
|
+
function nextChunk() {
|
|
2480
|
+
while (rangeAt < ranges.length && doneInRange >= ranges[rangeAt].length) {
|
|
2481
|
+
rangeAt += 1;
|
|
2482
|
+
doneInRange = 0;
|
|
2483
|
+
}
|
|
2484
|
+
if (rangeAt >= ranges.length) return null;
|
|
2485
|
+
const { path, offset, length } = ranges[rangeAt];
|
|
2486
|
+
const chunk = { path, offset: offset + doneInRange, length: Math.min(READ_AHEAD_CHUNK_BYTES, length - doneInRange) };
|
|
2487
|
+
doneInRange += chunk.length;
|
|
2488
|
+
return chunk;
|
|
2489
|
+
}
|
|
2490
|
+
const scratch = new Uint8Array(READ_AHEAD_CHUNK_BYTES);
|
|
2491
|
+
async function readChunks() {
|
|
2492
|
+
for (let chunk = nextChunk(); chunk !== null; chunk = nextChunk()) {
|
|
2493
|
+
const target = scratch.subarray(0, chunk.length);
|
|
2494
|
+
if (!await readFileRangeLater(chunk.path, chunk.offset, target)) return;
|
|
2495
|
+
}
|
|
2496
|
+
}
|
|
2497
|
+
await Promise.all(Array.from({ length: READS_AHEAD_AT_ONCE }, readChunks));
|
|
2498
|
+
}
|
|
2499
|
+
async function headOf(path, ref) {
|
|
2500
|
+
const { headOffset, headLength } = ref;
|
|
2501
|
+
if (headOffset === void 0 || headLength === void 0) return null;
|
|
2502
|
+
const head = new Uint8Array(headLength);
|
|
2503
|
+
if (!await readFileRangeLater(path, HEADER_SIZE + headOffset, head)) return null;
|
|
2504
|
+
return decode(head);
|
|
2505
|
+
}
|
|
2506
|
+
async function levelsOfPart(directory, ref) {
|
|
2507
|
+
const path = await directory.pathOf(ref.key);
|
|
2508
|
+
const head = await headOf(path, ref);
|
|
2509
|
+
const graphs = head?.graphs;
|
|
2510
|
+
if (!Array.isArray(graphs) || graphs.length !== 1) return [];
|
|
2511
|
+
const { levels } = graphs[0];
|
|
2512
|
+
return decodeSections([levels], ref.bytes - HEADER_SIZE, `head of "${ref.key}"`, FILE_PAGE_SIZES).map(
|
|
2513
|
+
(section) => rangeOf(path, section)
|
|
2514
|
+
);
|
|
2515
|
+
}
|
|
2516
|
+
async function residentSectionsOfSegment(directory, ref) {
|
|
2517
|
+
if (!await isPagedSegmentFile(directory, ref.key)) return [];
|
|
2518
|
+
const listing = await pagedListingOf(directory, ref);
|
|
2519
|
+
return residentSectionsOf(listing).map((section) => rangeOf(listing.path, section));
|
|
2520
|
+
}
|
|
2521
|
+
async function startReadingOpeningSectionsAhead(directory, manifest) {
|
|
2522
|
+
try {
|
|
2523
|
+
const groups = await Promise.all([
|
|
2524
|
+
...manifest.partitions.flatMap(
|
|
2525
|
+
(partition) => partition.segments.map((ref) => residentSectionsOfSegment(directory, ref))
|
|
2526
|
+
),
|
|
2527
|
+
...manifest.vectors.flatMap((field) => field.graphParts.map((ref) => levelsOfPart(directory, ref)))
|
|
2528
|
+
]);
|
|
2529
|
+
void readRangesAhead(groups.flat());
|
|
2530
|
+
} catch {
|
|
2531
|
+
return;
|
|
2532
|
+
}
|
|
2533
|
+
}
|
|
2534
|
+
|
|
2535
|
+
// src/engine/changes/operations.ts
|
|
2536
|
+
function invalidLimit(name, value) {
|
|
2537
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, `The changes setting ${name} must be a positive whole number`, {
|
|
2538
|
+
[name]: String(value)
|
|
2539
|
+
});
|
|
2540
|
+
}
|
|
2541
|
+
function requireChangesSetting(setting) {
|
|
2542
|
+
if (setting === null || typeof setting !== "object" || Array.isArray(setting)) {
|
|
2543
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "The changes setting must be an object");
|
|
2544
|
+
}
|
|
2545
|
+
const { enabled, previousDocument, historyMaxAgeMs, historyMaxBytes } = setting;
|
|
2546
|
+
if (typeof enabled !== "boolean") {
|
|
2547
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "The changes setting needs enabled set to true or false");
|
|
2548
|
+
}
|
|
2549
|
+
if (previousDocument !== void 0 && typeof previousDocument !== "boolean") {
|
|
2550
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "The changes setting previousDocument must be true or false");
|
|
2551
|
+
}
|
|
2552
|
+
const isLimit = (value) => typeof value === "number" && Number.isSafeInteger(value) && value >= 1;
|
|
2553
|
+
if (historyMaxAgeMs !== void 0 && !isLimit(historyMaxAgeMs)) invalidLimit("historyMaxAgeMs", historyMaxAgeMs);
|
|
2554
|
+
if (historyMaxBytes !== void 0 && !isLimit(historyMaxBytes)) invalidLimit("historyMaxBytes", historyMaxBytes);
|
|
2555
|
+
return {
|
|
2556
|
+
enabled,
|
|
2557
|
+
...previousDocument !== void 0 ? { previousDocument } : {},
|
|
2558
|
+
...historyMaxAgeMs !== void 0 ? { historyMaxAgeMs } : {},
|
|
2559
|
+
...historyMaxBytes !== void 0 ? { historyMaxBytes } : {}
|
|
2560
|
+
};
|
|
2561
|
+
}
|
|
2562
|
+
function changesStorageOf(core, indexName) {
|
|
2563
|
+
const storage = core.durability?.manager.syncStorage?.();
|
|
2564
|
+
if (storage === void 0) {
|
|
2565
|
+
throw new NarsilError(
|
|
2566
|
+
ErrorCodes.CHANGES_DURABILITY_REQUIRED,
|
|
2567
|
+
`Index "${indexName}" needs write-ahead log durability for change capture, because the feed reads the log`,
|
|
2568
|
+
{ indexName }
|
|
2569
|
+
);
|
|
2570
|
+
}
|
|
2571
|
+
return storage;
|
|
2572
|
+
}
|
|
2573
|
+
function changesStatusOf(changes) {
|
|
2574
|
+
if (changes === null) return { enabled: false, previousDocument: false, epoch: null };
|
|
2575
|
+
return {
|
|
2576
|
+
enabled: true,
|
|
2577
|
+
previousDocument: changes.previousDocument,
|
|
2578
|
+
...changes.historyMaxAgeMs !== void 0 ? { historyMaxAgeMs: changes.historyMaxAgeMs } : {},
|
|
2579
|
+
...changes.historyMaxBytes !== void 0 ? { historyMaxBytes: changes.historyMaxBytes } : {},
|
|
2580
|
+
epoch: changes.epoch
|
|
2581
|
+
};
|
|
2582
|
+
}
|
|
2583
|
+
function nextState(setting, epoch) {
|
|
2584
|
+
return {
|
|
2585
|
+
previousDocument: setting.previousDocument === true,
|
|
2586
|
+
...setting.historyMaxAgeMs !== void 0 ? { historyMaxAgeMs: setting.historyMaxAgeMs } : {},
|
|
2587
|
+
...setting.historyMaxBytes !== void 0 ? { historyMaxBytes: setting.historyMaxBytes } : {},
|
|
2588
|
+
epoch
|
|
2589
|
+
};
|
|
2590
|
+
}
|
|
2591
|
+
async function turnOff(core, indexName, storage) {
|
|
2592
|
+
const { sync, changes } = core.requireIndex(indexName);
|
|
2593
|
+
if (changes === null) return;
|
|
2594
|
+
await persistEpochStates(core, indexName, { sync, changes: null });
|
|
2595
|
+
if (sync?.active !== true) await storage.release(indexName);
|
|
2596
|
+
emitSyncChange(core.indexRegistry, { kind: "changes-off", indexName });
|
|
2597
|
+
}
|
|
2598
|
+
async function turnOn(core, indexName, storage, setting) {
|
|
2599
|
+
const { sync, changes } = core.requireIndex(indexName);
|
|
2600
|
+
if (changes !== null) {
|
|
2601
|
+
await persistEpochStates(core, indexName, { sync, changes: nextState(setting, changes.epoch) });
|
|
2602
|
+
if (changes.previousDocument && setting.previousDocument !== true) {
|
|
2603
|
+
emitSyncChange(core.indexRegistry, { kind: "previous-document-off", indexName });
|
|
2604
|
+
}
|
|
2605
|
+
return;
|
|
2606
|
+
}
|
|
2607
|
+
if (sync !== null) {
|
|
2608
|
+
await persistEpochStates(core, indexName, { sync, changes: nextState(setting, sync.epoch) });
|
|
2609
|
+
return;
|
|
2610
|
+
}
|
|
2611
|
+
await storage.release(indexName);
|
|
2612
|
+
await persistEpochStates(core, indexName, { sync: null, changes: nextState(setting, newSyncEpoch()) });
|
|
2613
|
+
}
|
|
2614
|
+
function updateIndexChanges(core, indexName, value) {
|
|
2615
|
+
const setting = requireChangesSetting(value);
|
|
2616
|
+
core.requireIndex(indexName);
|
|
2617
|
+
const storage = changesStorageOf(core, indexName);
|
|
2618
|
+
return withOpenIndex(
|
|
2619
|
+
core,
|
|
2620
|
+
indexName,
|
|
2621
|
+
() => oneSyncChangeAtATime(
|
|
2622
|
+
core.indexRegistry,
|
|
2623
|
+
indexName,
|
|
2624
|
+
() => setting.enabled ? turnOn(core, indexName, storage, setting) : turnOff(core, indexName, storage)
|
|
2625
|
+
)
|
|
2626
|
+
);
|
|
2627
|
+
}
|
|
2628
|
+
async function updateChangesOf(core, indexName, setting) {
|
|
2629
|
+
core.guardShutdown();
|
|
2630
|
+
core.requireIndex(indexName);
|
|
2631
|
+
await updateIndexChanges(core, indexName, setting);
|
|
2632
|
+
}
|
|
2633
|
+
function changesOf(core, indexName) {
|
|
2634
|
+
core.guardShutdown();
|
|
2635
|
+
return changesStatusOf(core.requireIndex(indexName).changes);
|
|
2636
|
+
}
|
|
2637
|
+
|
|
2638
|
+
// src/persistence/durability/complete-graphs.ts
|
|
2639
|
+
async function whileGraphsStayComplete(vectorIndexes, capture2) {
|
|
2640
|
+
const letBuildsIn = [];
|
|
2641
|
+
try {
|
|
2642
|
+
for (const fieldPath of [...vectorIndexes.keys()].sort()) {
|
|
2643
|
+
const vectorIndex = vectorIndexes.get(fieldPath);
|
|
2644
|
+
if (vectorIndex !== void 0) letBuildsIn.push(await vectorIndex.holdGraphComplete());
|
|
2645
|
+
}
|
|
2646
|
+
return await capture2();
|
|
2647
|
+
} finally {
|
|
2648
|
+
for (const release of letBuildsIn) release();
|
|
2649
|
+
}
|
|
2650
|
+
}
|
|
2651
|
+
|
|
2652
|
+
// src/serialization/crc32-combine.ts
|
|
2653
|
+
var IEEE_POLYNOMIAL = 3988292384;
|
|
2654
|
+
var CRC_BITS = 32;
|
|
2655
|
+
var zeroByteOperators = [];
|
|
2656
|
+
function timesVector(operator, vector) {
|
|
2657
|
+
let sum = 0;
|
|
2658
|
+
let rest = vector >>> 0;
|
|
2659
|
+
for (let row = 0; rest !== 0; row++, rest >>>= 1) {
|
|
2660
|
+
if (rest & 1) sum ^= operator[row];
|
|
2661
|
+
}
|
|
2662
|
+
return sum >>> 0;
|
|
2663
|
+
}
|
|
2664
|
+
function squared(operator) {
|
|
2665
|
+
const result = new Uint32Array(CRC_BITS);
|
|
2666
|
+
for (let row = 0; row < CRC_BITS; row++) result[row] = timesVector(operator, operator[row]);
|
|
2667
|
+
return result;
|
|
2668
|
+
}
|
|
2669
|
+
function firstOperator() {
|
|
2670
|
+
let oneZeroBit = new Uint32Array(CRC_BITS);
|
|
2671
|
+
oneZeroBit[0] = IEEE_POLYNOMIAL;
|
|
2672
|
+
for (let row = 1; row < CRC_BITS; row++) oneZeroBit[row] = 1 << row - 1;
|
|
2673
|
+
for (let doubling = 0; doubling < 3; doubling++) oneZeroBit = squared(oneZeroBit);
|
|
2674
|
+
return oneZeroBit;
|
|
2675
|
+
}
|
|
2676
|
+
function operatorForZeroBytesPower(power) {
|
|
2677
|
+
if (zeroByteOperators.length === 0) zeroByteOperators.push(firstOperator());
|
|
2678
|
+
while (zeroByteOperators.length <= power) {
|
|
2679
|
+
zeroByteOperators.push(squared(zeroByteOperators[zeroByteOperators.length - 1]));
|
|
2680
|
+
}
|
|
2681
|
+
return zeroByteOperators[power];
|
|
2682
|
+
}
|
|
2683
|
+
function crc32Combine(first, second, secondByteLength) {
|
|
2684
|
+
let crc = first >>> 0;
|
|
2685
|
+
let rest = secondByteLength;
|
|
2686
|
+
for (let power = 0; rest > 0; power++) {
|
|
2687
|
+
if (rest % 2 === 1) crc = timesVector(operatorForZeroBytesPower(power), crc);
|
|
2688
|
+
rest = Math.floor(rest / 2);
|
|
2689
|
+
}
|
|
2690
|
+
return (crc ^ second) >>> 0;
|
|
2691
|
+
}
|
|
2692
|
+
|
|
2693
|
+
// src/persistence/paged/vector-bytes.ts
|
|
2694
|
+
var BIN8 = 196;
|
|
2695
|
+
var BIN16 = 197;
|
|
2696
|
+
var BIN32 = 198;
|
|
2697
|
+
var ARRAY16 = 220;
|
|
2698
|
+
var ARRAY32 = 221;
|
|
2699
|
+
var FIXMAP = 128;
|
|
2700
|
+
var FIXARRAY = 144;
|
|
2701
|
+
var FIXED_LIST_MAX = 15;
|
|
2702
|
+
var FIXSTR = 160;
|
|
2703
|
+
var FIXSTR_MAX_LENGTH = 31;
|
|
2704
|
+
var ASCII_MAX = 127;
|
|
2705
|
+
var UTF8_MAX_BYTES_PER_UNIT = 3;
|
|
2706
|
+
var STRING_HEADER_MAX_BYTES = 5;
|
|
2707
|
+
var BIN8_MAX_BYTES = 255;
|
|
2708
|
+
var UINT16_MAX = 65535;
|
|
2709
|
+
function binHeader(byteLength) {
|
|
2710
|
+
if (byteLength <= BIN8_MAX_BYTES) return Uint8Array.of(BIN8, byteLength);
|
|
2711
|
+
if (byteLength <= UINT16_MAX) return Uint8Array.of(BIN16, byteLength >> 8, byteLength & 255);
|
|
2712
|
+
const header = new Uint8Array(5);
|
|
2713
|
+
header[0] = BIN32;
|
|
2714
|
+
new DataView(header.buffer).setUint32(1, byteLength, false);
|
|
2715
|
+
return header;
|
|
1963
2716
|
}
|
|
1964
2717
|
function arrayHeader(count) {
|
|
1965
2718
|
if (count <= FIXED_LIST_MAX) return Uint8Array.of(FIXARRAY | count);
|
|
@@ -2300,7 +3053,7 @@ async function writeLiveVectors(input) {
|
|
|
2300
3053
|
}
|
|
2301
3054
|
|
|
2302
3055
|
// src/persistence/archive/memory-store.ts
|
|
2303
|
-
function
|
|
3056
|
+
function joined2(chunks) {
|
|
2304
3057
|
let total = 0;
|
|
2305
3058
|
for (const chunk of chunks) total += chunk.byteLength;
|
|
2306
3059
|
const bytes = new Uint8Array(total);
|
|
@@ -2317,7 +3070,7 @@ function memoryCheckpointStore() {
|
|
|
2317
3070
|
files,
|
|
2318
3071
|
read: async (key) => files.get(key) ?? null,
|
|
2319
3072
|
write: async (key, chunks) => {
|
|
2320
|
-
files.set(key,
|
|
3073
|
+
files.set(key, joined2(chunks));
|
|
2321
3074
|
},
|
|
2322
3075
|
remove: async (key) => {
|
|
2323
3076
|
files.delete(key);
|
|
@@ -2358,150 +3111,549 @@ function partitionVectorPlan(vectorIndex, docIds) {
|
|
|
2358
3111
|
};
|
|
2359
3112
|
files.push({ payload, ordinals: new Int32Array(slice.length), dead: null, magnitudes: null });
|
|
2360
3113
|
}
|
|
2361
|
-
return {
|
|
2362
|
-
unchanged: false,
|
|
2363
|
-
storesHeads: false,
|
|
2364
|
-
kept: [],
|
|
2365
|
-
keptFiles: [],
|
|
2366
|
-
newFiles: files.length,
|
|
2367
|
-
graphParts: null,
|
|
2368
|
-
signature: { revision: 0, graphSize: -1, calibration: null },
|
|
2369
|
-
readNewFile: (index) => files[index],
|
|
2370
|
-
release: () => void 0
|
|
2371
|
-
};
|
|
3114
|
+
return {
|
|
3115
|
+
unchanged: false,
|
|
3116
|
+
storesHeads: false,
|
|
3117
|
+
kept: [],
|
|
3118
|
+
keptFiles: [],
|
|
3119
|
+
newFiles: files.length,
|
|
3120
|
+
graphParts: null,
|
|
3121
|
+
signature: { revision: 0, graphSize: -1, calibration: null },
|
|
3122
|
+
readNewFile: (index) => files[index],
|
|
3123
|
+
release: () => void 0
|
|
3124
|
+
};
|
|
3125
|
+
}
|
|
3126
|
+
function capture(input) {
|
|
3127
|
+
const partitionIds = input.partitionId === null ? Array.from({ length: input.manager.partitionCount }, (_, partitionId) => partitionId) : [input.partitionId];
|
|
3128
|
+
const { indexName } = input;
|
|
3129
|
+
const writeRecords = input.writeRecords ?? null;
|
|
3130
|
+
const targets = partitionIds.map((partitionId) => {
|
|
3131
|
+
const target = {
|
|
3132
|
+
partitionId,
|
|
3133
|
+
lastSeqNo: input.appliedSeqNo(partitionId),
|
|
3134
|
+
primaryTerm: SINGLE_NODE_PRIMARY_TERM
|
|
3135
|
+
};
|
|
3136
|
+
if (writeRecords !== null) target.termHistory = writeRecords.termHistoryOf(indexName, partitionId, target.lastSeqNo);
|
|
3137
|
+
return target;
|
|
3138
|
+
});
|
|
3139
|
+
const contents = partitionIds.map(
|
|
3140
|
+
(partitionId) => partitionContentsOf(
|
|
3141
|
+
input.manager,
|
|
3142
|
+
partitionId,
|
|
3143
|
+
writeRecords === null ? void 0 : (segments) => writeRecords.captureSegments(indexName, partitionId, segments)
|
|
3144
|
+
)
|
|
3145
|
+
);
|
|
3146
|
+
const plans = /* @__PURE__ */ new Map();
|
|
3147
|
+
const partitionDocIds = input.partitionId === null ? null : [...input.manager.getPartition(input.partitionId).docIds()];
|
|
3148
|
+
for (const [fieldPath, vectorIndex] of input.vectorIndexes) {
|
|
3149
|
+
plans.set(
|
|
3150
|
+
fieldPath,
|
|
3151
|
+
partitionDocIds === null ? frozenPlan(vectorIndex.planCheckpoint(null)) : partitionVectorPlan(vectorIndex, partitionDocIds)
|
|
3152
|
+
);
|
|
3153
|
+
}
|
|
3154
|
+
return { targets, contents, plans };
|
|
3155
|
+
}
|
|
3156
|
+
async function buildArchiveFromMemory(input) {
|
|
3157
|
+
const captured = input.partitionId === null ? await whileGraphsStayComplete(input.vectorIndexes, () => capture(input)) : capture(input);
|
|
3158
|
+
const store = memoryCheckpointStore();
|
|
3159
|
+
const written = await writeCheckpointSegments({
|
|
3160
|
+
store,
|
|
3161
|
+
metadata: input.metadata,
|
|
3162
|
+
targets: captured.targets,
|
|
3163
|
+
compactionThreshold: DEFAULT_COMPACTION_THRESHOLD,
|
|
3164
|
+
prior: null,
|
|
3165
|
+
wholePartitions: captured.contents,
|
|
3166
|
+
keepsWriteRecords: input.writeRecords != null
|
|
3167
|
+
});
|
|
3168
|
+
const vectors = await writeLiveVectors({ store, indexName: input.indexName, plans: captured.plans, priorVectors: [] });
|
|
3169
|
+
const refs = vectors.refs;
|
|
3170
|
+
await commitCheckpointManifest(store, input.metadata, written, refs);
|
|
3171
|
+
const manifestBytes = store.files.get(manifestKey(input.indexName));
|
|
3172
|
+
const decoded = manifestBytes === void 0 ? null : await decodeSegmentManifest(manifestBytes, input.indexName, input.partitionId);
|
|
3173
|
+
if (manifestBytes === void 0 || decoded === null || decoded.kind !== "current") {
|
|
3174
|
+
throw new NarsilError(
|
|
3175
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
3176
|
+
`The archive of index "${input.indexName}" holds no manifest`
|
|
3177
|
+
);
|
|
3178
|
+
}
|
|
3179
|
+
return {
|
|
3180
|
+
manifestBytes,
|
|
3181
|
+
manifest: decoded.manifest,
|
|
3182
|
+
readFile: async (ref) => {
|
|
3183
|
+
const bytes = store.files.get(ref.key);
|
|
3184
|
+
if (bytes === void 0) {
|
|
3185
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `The archive file "${ref.key}" is missing`, {
|
|
3186
|
+
key: ref.key
|
|
3187
|
+
});
|
|
3188
|
+
}
|
|
3189
|
+
return bytes;
|
|
3190
|
+
},
|
|
3191
|
+
readRange: async (ref, offset, length) => {
|
|
3192
|
+
const bytes = store.files.get(ref.key);
|
|
3193
|
+
if (bytes === void 0 || offset + length > bytes.byteLength) {
|
|
3194
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The archive file "${ref.key}" is missing`, {
|
|
3195
|
+
key: ref.key
|
|
3196
|
+
});
|
|
3197
|
+
}
|
|
3198
|
+
return bytes.subarray(offset, offset + length);
|
|
3199
|
+
},
|
|
3200
|
+
release: () => store.files.clear()
|
|
3201
|
+
};
|
|
3202
|
+
}
|
|
3203
|
+
|
|
3204
|
+
// src/persistence/archive/write.ts
|
|
3205
|
+
function entryChunks(name, bytes) {
|
|
3206
|
+
return [...tarEntryHeaders(name, bytes.byteLength), bytes, tarPadding(bytes.byteLength)].filter(
|
|
3207
|
+
(chunk) => chunk.byteLength > 0
|
|
3208
|
+
);
|
|
3209
|
+
}
|
|
3210
|
+
async function* fileEntryChunks(source, name, ref) {
|
|
3211
|
+
yield* tarEntryHeaders(name, ref.bytes).filter((chunk) => chunk.byteLength > 0);
|
|
3212
|
+
for (let offset = 0; offset < ref.bytes; offset += ARCHIVE_FILE_CHUNK_BYTES) {
|
|
3213
|
+
yield await source.readRange(ref, offset, Math.min(ARCHIVE_FILE_CHUNK_BYTES, ref.bytes - offset));
|
|
3214
|
+
}
|
|
3215
|
+
const padding = tarPadding(ref.bytes);
|
|
3216
|
+
if (padding.byteLength > 0) yield padding;
|
|
3217
|
+
}
|
|
3218
|
+
async function* archiveChunks(source) {
|
|
3219
|
+
yield* entryChunks(ARCHIVE_CONFIG_ENTRY, source.config);
|
|
3220
|
+
yield* entryChunks(ARCHIVE_MANIFEST_ENTRY, source.manifestBytes);
|
|
3221
|
+
const prefix = `${source.sourceIndexName}/`;
|
|
3222
|
+
for (const ref of manifestFileRefs(source.manifest)) {
|
|
3223
|
+
yield* fileEntryChunks(source, ref.key.slice(prefix.length), ref);
|
|
3224
|
+
}
|
|
3225
|
+
if (source.trailingEntries !== void 0) {
|
|
3226
|
+
for await (const entry of source.trailingEntries()) yield* entryChunks(entry.name, entry.bytes);
|
|
3227
|
+
}
|
|
3228
|
+
yield tarEnd();
|
|
3229
|
+
}
|
|
3230
|
+
function archiveStream(source) {
|
|
3231
|
+
const chunks = archiveChunks(source);
|
|
3232
|
+
let released = false;
|
|
3233
|
+
const release = () => {
|
|
3234
|
+
if (released) return;
|
|
3235
|
+
released = true;
|
|
3236
|
+
source.release();
|
|
3237
|
+
};
|
|
3238
|
+
return new ReadableStream({
|
|
3239
|
+
async pull(controller) {
|
|
3240
|
+
try {
|
|
3241
|
+
const step = await chunks.next();
|
|
3242
|
+
if (step.done) {
|
|
3243
|
+
release();
|
|
3244
|
+
controller.close();
|
|
3245
|
+
return;
|
|
3246
|
+
}
|
|
3247
|
+
controller.enqueue(step.value);
|
|
3248
|
+
} catch (error) {
|
|
3249
|
+
release();
|
|
3250
|
+
controller.error(error);
|
|
3251
|
+
}
|
|
3252
|
+
},
|
|
3253
|
+
async cancel() {
|
|
3254
|
+
release();
|
|
3255
|
+
await chunks.return(void 0);
|
|
3256
|
+
}
|
|
3257
|
+
});
|
|
3258
|
+
}
|
|
3259
|
+
|
|
3260
|
+
// src/engine/durability-metadata.ts
|
|
3261
|
+
var VECTOR_TYPE = /^vector\[(\d+)\]$/;
|
|
3262
|
+
function vectorFieldsOf(schema, promotion) {
|
|
3263
|
+
const fields = {};
|
|
3264
|
+
for (const [fieldPath, type] of Object.entries(schema)) {
|
|
3265
|
+
const match = VECTOR_TYPE.exec(type);
|
|
3266
|
+
if (match === null) continue;
|
|
3267
|
+
const dimension = Number(match[1]);
|
|
3268
|
+
fields[fieldPath] = {
|
|
3269
|
+
dimension,
|
|
3270
|
+
metric: promotion?.hnswConfig?.metric ?? DEFAULT_VECTOR_METRIC,
|
|
3271
|
+
quantization: promotion?.quantization ?? defaultQuantizationFor(dimension)
|
|
3272
|
+
};
|
|
3273
|
+
}
|
|
3274
|
+
return Object.keys(fields).length === 0 ? null : fields;
|
|
3275
|
+
}
|
|
3276
|
+
function metadataFromConfig(indexName, config, partitionCount, documentCount, savedBytes) {
|
|
3277
|
+
const metadata = {
|
|
3278
|
+
indexName,
|
|
3279
|
+
schema: config.schema,
|
|
3280
|
+
language: config.language,
|
|
3281
|
+
partitionCount: partitionCount ?? config.partitionCount,
|
|
3282
|
+
bm25Params: {
|
|
3283
|
+
k1: config.k1,
|
|
3284
|
+
b: config.b,
|
|
3285
|
+
...config.bm25Fields !== void 0 ? { fields: config.bm25Fields } : {}
|
|
3286
|
+
},
|
|
3287
|
+
createdAt: Date.now(),
|
|
3288
|
+
engineVersion: VERSION
|
|
3289
|
+
};
|
|
3290
|
+
metadata.documentCount = documentCount ?? config.documentCount;
|
|
3291
|
+
const recordedSavedBytes = savedBytes ?? config.savedBytes;
|
|
3292
|
+
if (recordedSavedBytes !== void 0) metadata.savedBytes = recordedSavedBytes;
|
|
3293
|
+
const vectorFields = vectorFieldsOf(config.schema, config.vectorPromotion);
|
|
3294
|
+
if (vectorFields !== null) {
|
|
3295
|
+
metadata.vectorFields = vectorFields;
|
|
3296
|
+
}
|
|
3297
|
+
if (config.embedding) {
|
|
3298
|
+
metadata.embedding = config.embedding;
|
|
3299
|
+
}
|
|
3300
|
+
if (config.surfaceForms !== void 0) {
|
|
3301
|
+
metadata.surfaceForms = config.surfaceForms;
|
|
3302
|
+
}
|
|
3303
|
+
if (config.analysisRevision !== void 0) {
|
|
3304
|
+
metadata.analysisRevision = config.analysisRevision;
|
|
3305
|
+
}
|
|
3306
|
+
if (config.arrayPositionGap !== void 0) {
|
|
3307
|
+
metadata.arrayPositionGap = config.arrayPositionGap;
|
|
3308
|
+
}
|
|
3309
|
+
if (config.tokenizer !== void 0) {
|
|
3310
|
+
metadata.tokenizer = config.tokenizer;
|
|
3311
|
+
}
|
|
3312
|
+
if (config.stopWords !== void 0) {
|
|
3313
|
+
metadata.stopWords = config.stopWords;
|
|
3314
|
+
}
|
|
3315
|
+
if (config.stopWordList !== void 0) {
|
|
3316
|
+
metadata.stopWordList = config.stopWordList;
|
|
3317
|
+
}
|
|
3318
|
+
if (config.analysisGeneration !== void 0) {
|
|
3319
|
+
metadata.analysisGeneration = config.analysisGeneration;
|
|
3320
|
+
}
|
|
3321
|
+
if (config.pendingAnalysis !== void 0) {
|
|
3322
|
+
metadata.pendingAnalysis = config.pendingAnalysis;
|
|
3323
|
+
}
|
|
3324
|
+
if (config.partitionLimits !== void 0) {
|
|
3325
|
+
metadata.partitionLimits = config.partitionLimits;
|
|
3326
|
+
}
|
|
3327
|
+
if (config.defaultScoring !== void 0) {
|
|
3328
|
+
metadata.defaultScoring = config.defaultScoring;
|
|
3329
|
+
}
|
|
3330
|
+
if (config.trackPositions !== void 0) {
|
|
3331
|
+
metadata.trackPositions = config.trackPositions;
|
|
3332
|
+
}
|
|
3333
|
+
if (config.textStorage !== void 0) {
|
|
3334
|
+
metadata.textStorage = config.textStorage;
|
|
3335
|
+
}
|
|
3336
|
+
if (config.strict !== void 0) {
|
|
3337
|
+
metadata.strict = config.strict;
|
|
3338
|
+
}
|
|
3339
|
+
if (config.required !== void 0) {
|
|
3340
|
+
metadata.required = config.required;
|
|
3341
|
+
}
|
|
3342
|
+
if (config.vectorPromotion !== void 0) {
|
|
3343
|
+
metadata.vectorPromotion = config.vectorPromotion;
|
|
3344
|
+
}
|
|
3345
|
+
if (config.patternValueLimit !== void 0) {
|
|
3346
|
+
metadata.patternValueLimit = config.patternValueLimit;
|
|
3347
|
+
}
|
|
3348
|
+
if (config.indexUuid !== void 0) {
|
|
3349
|
+
metadata.indexUuid = config.indexUuid;
|
|
3350
|
+
}
|
|
3351
|
+
if (config.synonyms !== void 0) {
|
|
3352
|
+
metadata.synonyms = config.synonyms;
|
|
3353
|
+
}
|
|
3354
|
+
if (config.heldPartitions !== void 0) {
|
|
3355
|
+
metadata.heldPartitions = config.heldPartitions;
|
|
3356
|
+
}
|
|
3357
|
+
if (config.sync !== void 0) metadata.sync = config.sync;
|
|
3358
|
+
if (config.syncActive !== void 0) metadata.syncActive = config.syncActive;
|
|
3359
|
+
if (config.syncEpoch !== void 0) metadata.syncEpoch = config.syncEpoch;
|
|
3360
|
+
if (config.changes !== void 0) metadata.changes = config.changes;
|
|
3361
|
+
if (config.querySuggestions !== void 0) metadata.querySuggestions = config.querySuggestions;
|
|
3362
|
+
return metadata;
|
|
3363
|
+
}
|
|
3364
|
+
function metadataConfigOf(entry) {
|
|
3365
|
+
const embedding = entry.config.embedding ? {
|
|
3366
|
+
fields: entry.config.embedding.fields,
|
|
3367
|
+
...entry.embeddingAdapterName !== null ? { adapter: entry.embeddingAdapterName } : {}
|
|
3368
|
+
} : void 0;
|
|
3369
|
+
return {
|
|
3370
|
+
...entry.indexUuid !== null ? { indexUuid: entry.indexUuid } : {},
|
|
3371
|
+
...entry.heldPartitions !== null ? { heldPartitions: entry.heldPartitions } : {},
|
|
3372
|
+
documentCount: entry.documentCount,
|
|
3373
|
+
savedBytes: entry.savedBytes,
|
|
3374
|
+
partitionCount: entry.partitionCount,
|
|
3375
|
+
schema: flattenSchema(entry.config.schema),
|
|
3376
|
+
language: entry.language.name,
|
|
3377
|
+
k1: entry.config.bm25?.k1 ?? DEFAULT_BM25_K1,
|
|
3378
|
+
b: entry.config.bm25?.b ?? DEFAULT_BM25_B,
|
|
3379
|
+
...entry.config.bm25?.fields !== void 0 ? { bm25Fields: entry.config.bm25.fields } : {},
|
|
3380
|
+
...embedding !== void 0 ? { embedding } : {},
|
|
3381
|
+
surfaceForms: entry.config.surfaceForms !== false,
|
|
3382
|
+
...entry.analysisRevision !== null ? { analysisRevision: entry.analysisRevision } : {},
|
|
3383
|
+
...entry.arrayPositionGap !== null ? { arrayPositionGap: entry.arrayPositionGap } : {},
|
|
3384
|
+
...typeof entry.config.tokenizer === "string" ? { tokenizer: entry.config.tokenizer } : {},
|
|
3385
|
+
...typeof entry.config.stopWords === "string" ? { stopWords: entry.config.stopWords } : {},
|
|
3386
|
+
...entry.config.stopWords instanceof Set ? { stopWordList: [...entry.config.stopWords].sort(compareCodePoints) } : {},
|
|
3387
|
+
...entry.analysisGeneration > 0 ? { analysisGeneration: entry.analysisGeneration } : {},
|
|
3388
|
+
...entry.pendingAnalysis !== null ? { pendingAnalysis: entry.pendingAnalysis } : {},
|
|
3389
|
+
...entry.config.partitions !== void 0 ? { partitionLimits: entry.config.partitions } : {},
|
|
3390
|
+
...entry.config.defaultScoring !== void 0 ? { defaultScoring: entry.config.defaultScoring } : {},
|
|
3391
|
+
...entry.config.trackPositions !== void 0 ? { trackPositions: entry.config.trackPositions } : {},
|
|
3392
|
+
...entry.config.textStorage !== void 0 ? { textStorage: entry.config.textStorage } : {},
|
|
3393
|
+
...entry.config.strict !== void 0 ? { strict: entry.config.strict } : {},
|
|
3394
|
+
...entry.config.required !== void 0 ? { required: entry.config.required } : {},
|
|
3395
|
+
...entry.config.vectorPromotion !== void 0 ? { vectorPromotion: entry.config.vectorPromotion } : {},
|
|
3396
|
+
...entry.config.patternValueLimit !== void 0 ? { patternValueLimit: entry.config.patternValueLimit } : {},
|
|
3397
|
+
...entry.synonyms !== null ? { synonyms: rulesOfStoredSynonyms(entry.synonyms) } : {},
|
|
3398
|
+
...syncMetadataOf(entry),
|
|
3399
|
+
...entry.querySuggestions !== null ? { querySuggestions: entry.querySuggestions } : {}
|
|
3400
|
+
};
|
|
3401
|
+
}
|
|
3402
|
+
function syncMetadataOf(entry) {
|
|
3403
|
+
const { sync, changes } = entry;
|
|
3404
|
+
if (sync === null && changes === null) return {};
|
|
3405
|
+
return {
|
|
3406
|
+
...sync === null ? {} : {
|
|
3407
|
+
sync: {
|
|
3408
|
+
enabled: true,
|
|
3409
|
+
...sync.historyMaxAgeMs !== void 0 ? { historyMaxAgeMs: sync.historyMaxAgeMs } : {},
|
|
3410
|
+
...sync.historyMaxBytes !== void 0 ? { historyMaxBytes: sync.historyMaxBytes } : {}
|
|
3411
|
+
},
|
|
3412
|
+
syncActive: sync.active
|
|
3413
|
+
},
|
|
3414
|
+
syncEpoch: sync?.epoch ?? changes?.epoch,
|
|
3415
|
+
...changes === null ? {} : {
|
|
3416
|
+
changes: {
|
|
3417
|
+
enabled: true,
|
|
3418
|
+
...changes.previousDocument ? { previousDocument: true } : {},
|
|
3419
|
+
...changes.historyMaxAgeMs !== void 0 ? { historyMaxAgeMs: changes.historyMaxAgeMs } : {},
|
|
3420
|
+
...changes.historyMaxBytes !== void 0 ? { historyMaxBytes: changes.historyMaxBytes } : {}
|
|
3421
|
+
}
|
|
3422
|
+
}
|
|
3423
|
+
};
|
|
3424
|
+
}
|
|
3425
|
+
function durableMetadataConfigOf(entry) {
|
|
3426
|
+
const terms = metadataConfigOf(entry);
|
|
3427
|
+
const durable = entry.durableAnalysis;
|
|
3428
|
+
if (durable === null) return terms;
|
|
3429
|
+
const {
|
|
3430
|
+
language,
|
|
3431
|
+
tokenizer,
|
|
3432
|
+
stopWords,
|
|
3433
|
+
stopWordList,
|
|
3434
|
+
analysisRevision: _analysisRevision,
|
|
3435
|
+
arrayPositionGap: _arrayPositionGap,
|
|
3436
|
+
analysisGeneration: _analysisGeneration,
|
|
3437
|
+
...rest
|
|
3438
|
+
} = terms;
|
|
3439
|
+
return {
|
|
3440
|
+
...rest,
|
|
3441
|
+
language: durable.language,
|
|
3442
|
+
...durable.tokenizer !== void 0 ? { tokenizer: durable.tokenizer } : {},
|
|
3443
|
+
...durable.stopWords !== void 0 ? { stopWords: durable.stopWords } : {},
|
|
3444
|
+
...durable.stopWordList !== void 0 ? { stopWordList: durable.stopWordList } : {},
|
|
3445
|
+
...durable.analysisRevision !== null ? { analysisRevision: durable.analysisRevision } : {},
|
|
3446
|
+
...durable.arrayPositionGap !== null ? { arrayPositionGap: durable.arrayPositionGap } : {},
|
|
3447
|
+
...durable.generation > 0 ? { analysisGeneration: durable.generation } : {},
|
|
3448
|
+
pendingAnalysis: {
|
|
3449
|
+
generation: entry.analysisGeneration,
|
|
3450
|
+
language,
|
|
3451
|
+
...tokenizer !== void 0 ? { tokenizer } : {},
|
|
3452
|
+
...stopWords !== void 0 ? { stopWords } : {},
|
|
3453
|
+
...stopWordList !== void 0 ? { stopWordList } : {}
|
|
3454
|
+
}
|
|
3455
|
+
};
|
|
3456
|
+
}
|
|
3457
|
+
function indexMetadataOf(indexName, entry, partitionCount) {
|
|
3458
|
+
return metadataFromConfig(indexName, metadataConfigOf(entry), partitionCount);
|
|
3459
|
+
}
|
|
3460
|
+
|
|
3461
|
+
// src/persistence/archive/file-entry.ts
|
|
3462
|
+
async function writeArchiveFile(store, key, chunks) {
|
|
3463
|
+
const { directory } = store;
|
|
3464
|
+
if (directory === void 0) {
|
|
3465
|
+
const held = [];
|
|
3466
|
+
for await (const chunk of chunks) held.push(chunk);
|
|
3467
|
+
await store.write(key, held);
|
|
3468
|
+
return;
|
|
3469
|
+
}
|
|
3470
|
+
const receiving = `${key}.receiving`;
|
|
3471
|
+
await directory.remove(receiving);
|
|
3472
|
+
const handle = await directory.appendHandle(receiving);
|
|
3473
|
+
try {
|
|
3474
|
+
for await (const chunk of chunks) await handle.append(chunk);
|
|
3475
|
+
await handle.sync();
|
|
3476
|
+
} finally {
|
|
3477
|
+
await handle.close();
|
|
3478
|
+
}
|
|
3479
|
+
await directory.move(receiving, key);
|
|
3480
|
+
}
|
|
3481
|
+
|
|
3482
|
+
// src/persistence/durability/segment/load-partition.ts
|
|
3483
|
+
function requireAgreedSchema(agreed, schema, key) {
|
|
3484
|
+
for (const [field, type] of Object.entries(schema)) {
|
|
3485
|
+
const existing = agreed[field];
|
|
3486
|
+
if (existing !== void 0 && existing !== type) {
|
|
3487
|
+
throw new NarsilError(
|
|
3488
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
3489
|
+
`The segment "${key}" gives field "${field}" the type "${type}", while the index gives it "${existing}"`,
|
|
3490
|
+
{ key, field, existing, found: type }
|
|
3491
|
+
);
|
|
3492
|
+
}
|
|
3493
|
+
}
|
|
3494
|
+
}
|
|
3495
|
+
function requireListedCounts(ref, documentCount, tombstoneCount) {
|
|
3496
|
+
if (ref.docCount === documentCount && ref.tombstoneCount === tombstoneCount) return;
|
|
3497
|
+
throw new NarsilError(
|
|
3498
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
3499
|
+
`The segment "${ref.key}" holds ${documentCount} documents and ${tombstoneCount} tombstones where the manifest lists ${ref.docCount} and ${ref.tombstoneCount}`,
|
|
3500
|
+
{
|
|
3501
|
+
key: ref.key,
|
|
3502
|
+
documentCount,
|
|
3503
|
+
tombstoneCount,
|
|
3504
|
+
listedDocCount: ref.docCount,
|
|
3505
|
+
listedTombstoneCount: ref.tombstoneCount
|
|
3506
|
+
}
|
|
3507
|
+
);
|
|
3508
|
+
}
|
|
3509
|
+
function removeWhatTheSegmentReplaces(manager, segment, contents) {
|
|
3510
|
+
manager.beginBatchRemove();
|
|
3511
|
+
try {
|
|
3512
|
+
for (const docId of contents.tombstones) {
|
|
3513
|
+
if (manager.has(docId)) manager.remove(docId);
|
|
3514
|
+
}
|
|
3515
|
+
const ids = segment.arrays.idTable;
|
|
3516
|
+
for (let ordinal = 0; ordinal < ids.count; ordinal++) {
|
|
3517
|
+
const docId = ids.idAt(ordinal);
|
|
3518
|
+
if (manager.has(docId)) manager.remove(docId);
|
|
3519
|
+
}
|
|
3520
|
+
} finally {
|
|
3521
|
+
manager.endBatchRemove();
|
|
3522
|
+
}
|
|
3523
|
+
}
|
|
3524
|
+
function yieldToEventLoop2() {
|
|
3525
|
+
return new Promise((resolve) => setTimeout(resolve, 0));
|
|
3526
|
+
}
|
|
3527
|
+
function removeFromPartition(manager, partitionId, docIds) {
|
|
3528
|
+
const { composite, docPartitionMap, insertOptions } = manager.diskSegmentAccess(partitionId);
|
|
3529
|
+
composite.beginBatch();
|
|
3530
|
+
try {
|
|
3531
|
+
for (const docId of docIds) {
|
|
3532
|
+
if (!composite.has(docId)) continue;
|
|
3533
|
+
composite.remove(docId, manager.config.schema, manager.language, insertOptions);
|
|
3534
|
+
docPartitionMap.delete(docId);
|
|
3535
|
+
}
|
|
3536
|
+
} finally {
|
|
3537
|
+
composite.endBatch();
|
|
3538
|
+
}
|
|
2372
3539
|
}
|
|
2373
|
-
function
|
|
2374
|
-
|
|
2375
|
-
const
|
|
2376
|
-
|
|
2377
|
-
|
|
2378
|
-
|
|
2379
|
-
|
|
2380
|
-
|
|
2381
|
-
|
|
2382
|
-
|
|
2383
|
-
|
|
2384
|
-
return target;
|
|
2385
|
-
});
|
|
2386
|
-
const contents = partitionIds.map(
|
|
2387
|
-
(partitionId) => partitionContentsOf(
|
|
2388
|
-
input.manager,
|
|
2389
|
-
partitionId,
|
|
2390
|
-
writeRecords === null ? void 0 : (segments) => writeRecords.captureSegments(indexName, partitionId, segments)
|
|
2391
|
-
)
|
|
2392
|
-
);
|
|
2393
|
-
const plans = /* @__PURE__ */ new Map();
|
|
2394
|
-
const partitionDocIds = input.partitionId === null ? null : [...input.manager.getPartition(input.partitionId).docIds()];
|
|
2395
|
-
for (const [fieldPath, vectorIndex] of input.vectorIndexes) {
|
|
2396
|
-
plans.set(
|
|
2397
|
-
fieldPath,
|
|
2398
|
-
partitionDocIds === null ? frozenPlan(vectorIndex.planCheckpoint(null)) : partitionVectorPlan(vectorIndex, partitionDocIds)
|
|
2399
|
-
);
|
|
3540
|
+
async function attachPagedSegment(directory, ref, partitionId, manager, options) {
|
|
3541
|
+
if (options.decodesEveryDocument) await checkPagedSegmentFully(directory, ref);
|
|
3542
|
+
const snapshot = openPagedSnapshot(await pagedListingOf(directory, ref));
|
|
3543
|
+
requireListedCounts(ref, snapshot.documentCount, snapshot.tombstones.length);
|
|
3544
|
+
requireAgreedSchema(options.schema, snapshot.schema, ref.key);
|
|
3545
|
+
const segment = createPagedFrozenSegment(snapshot);
|
|
3546
|
+
removeFromPartition(manager, partitionId, snapshot.tombstones);
|
|
3547
|
+
options.writeRecords?.loadPagedSegment(manager.indexName, partitionId, segment);
|
|
3548
|
+
if (snapshot.documentCount > 0) {
|
|
3549
|
+
manager.attachFrozenSegment(partitionId, segment);
|
|
3550
|
+
return segment;
|
|
2400
3551
|
}
|
|
2401
|
-
|
|
3552
|
+
segment.paged?.close();
|
|
3553
|
+
return null;
|
|
2402
3554
|
}
|
|
2403
|
-
|
|
2404
|
-
const
|
|
2405
|
-
|
|
2406
|
-
|
|
2407
|
-
|
|
2408
|
-
|
|
2409
|
-
|
|
2410
|
-
|
|
2411
|
-
|
|
2412
|
-
|
|
2413
|
-
|
|
2414
|
-
|
|
2415
|
-
|
|
2416
|
-
|
|
2417
|
-
await commitCheckpointManifest(store, input.metadata, written, refs);
|
|
2418
|
-
const manifestBytes = store.files.get(manifestKey(input.indexName));
|
|
2419
|
-
const decoded = manifestBytes === void 0 ? null : await decodeSegmentManifest(manifestBytes, input.indexName, input.partitionId);
|
|
2420
|
-
if (manifestBytes === void 0 || decoded === null || decoded.kind !== "current") {
|
|
2421
|
-
throw new NarsilError(
|
|
2422
|
-
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
2423
|
-
`The archive of index "${input.indexName}" holds no manifest`
|
|
2424
|
-
);
|
|
3555
|
+
function documentsHeldBefore(manager, partitionId) {
|
|
3556
|
+
const held = manager.partitionAt(partitionId);
|
|
3557
|
+
return held === void 0 ? [] : [...held.docIds()];
|
|
3558
|
+
}
|
|
3559
|
+
function removeDocumentsMissingFromLoadedSegments(manager, partitionId, heldBefore, loaded) {
|
|
3560
|
+
const absent = heldBefore.filter(
|
|
3561
|
+
(docId) => manager.partitionIdOf(docId) === partitionId && !loaded.some((segment) => segment.hasDocument(docId))
|
|
3562
|
+
);
|
|
3563
|
+
if (absent.length === 0) return;
|
|
3564
|
+
manager.beginBatchRemove();
|
|
3565
|
+
try {
|
|
3566
|
+
for (const docId of absent) manager.remove(docId);
|
|
3567
|
+
} finally {
|
|
3568
|
+
manager.endBatchRemove();
|
|
2425
3569
|
}
|
|
2426
|
-
|
|
2427
|
-
|
|
2428
|
-
|
|
2429
|
-
|
|
2430
|
-
|
|
2431
|
-
|
|
2432
|
-
|
|
2433
|
-
|
|
2434
|
-
|
|
2435
|
-
|
|
2436
|
-
|
|
2437
|
-
|
|
2438
|
-
|
|
2439
|
-
|
|
2440
|
-
|
|
2441
|
-
|
|
2442
|
-
key: ref.key
|
|
2443
|
-
});
|
|
3570
|
+
}
|
|
3571
|
+
async function loadPartitionSegments(store, partition, manager, options) {
|
|
3572
|
+
const heldBefore = documentsHeldBefore(manager, partition.partitionId);
|
|
3573
|
+
const loaded = await attachPartitionSegments(store, partition, manager, options);
|
|
3574
|
+
removeDocumentsMissingFromLoadedSegments(manager, partition.partitionId, heldBefore, loaded);
|
|
3575
|
+
}
|
|
3576
|
+
async function attachPartitionSegments(store, partition, manager, options) {
|
|
3577
|
+
const loaded = [];
|
|
3578
|
+
const { directory } = store;
|
|
3579
|
+
for (const ref of partition.segments) {
|
|
3580
|
+
if (manager.config.textStorage === "disk" && directory !== void 0) {
|
|
3581
|
+
if (await isPagedSegmentFile(directory, ref.key)) {
|
|
3582
|
+
const paged = await attachPagedSegment(directory, ref, partition.partitionId, manager, options);
|
|
3583
|
+
if (paged !== null) loaded.push(paged);
|
|
3584
|
+
await yieldToEventLoop2();
|
|
3585
|
+
continue;
|
|
2444
3586
|
}
|
|
2445
|
-
|
|
2446
|
-
|
|
2447
|
-
|
|
2448
|
-
|
|
3587
|
+
}
|
|
3588
|
+
const contents = await readSegmentFile(store, ref);
|
|
3589
|
+
requireListedCounts(ref, contents.arrays.documentCount, contents.tombstones.length);
|
|
3590
|
+
requireAgreedSchema(options.schema, contents.schema, ref.key);
|
|
3591
|
+
if (options.decodesEveryDocument) decodeEveryStoredDocument(contents.arrays.documentTable);
|
|
3592
|
+
const segment = createSharedFrozenSegment(shareSegmentArrays(contents.arrays, manager.partialWordAnalysis));
|
|
3593
|
+
removeWhatTheSegmentReplaces(manager, segment, contents);
|
|
3594
|
+
options.writeRecords?.loadSegment(manager.indexName, partition.partitionId, segment, contents);
|
|
3595
|
+
if (contents.arrays.documentCount > 0) {
|
|
3596
|
+
manager.attachFrozenSegment(partition.partitionId, segment);
|
|
3597
|
+
loaded.push(segment);
|
|
3598
|
+
}
|
|
3599
|
+
await yieldToEventLoop2();
|
|
3600
|
+
}
|
|
3601
|
+
return loaded;
|
|
2449
3602
|
}
|
|
2450
3603
|
|
|
2451
|
-
// src/persistence/
|
|
2452
|
-
function
|
|
2453
|
-
|
|
2454
|
-
|
|
3604
|
+
// src/persistence/durability/segment/load.ts
|
|
3605
|
+
async function readSegmentManifest(store, indexName) {
|
|
3606
|
+
const bytes = await store.read(manifestKey(indexName));
|
|
3607
|
+
if (bytes === null) return null;
|
|
3608
|
+
return decodeSegmentManifest(bytes, indexName);
|
|
3609
|
+
}
|
|
3610
|
+
async function readCurrentManifest(store, indexName) {
|
|
3611
|
+
const decoded = await readSegmentManifest(store, indexName);
|
|
3612
|
+
if (decoded === null) return null;
|
|
3613
|
+
if (decoded.kind === "current") return decoded.manifest;
|
|
3614
|
+
throw new NarsilError(
|
|
3615
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
3616
|
+
`Index "${indexName}" still holds the layout of an earlier release, which recovery upgrades before any checkpoint`,
|
|
3617
|
+
{ indexName }
|
|
2455
3618
|
);
|
|
2456
3619
|
}
|
|
2457
|
-
|
|
2458
|
-
|
|
2459
|
-
for (let offset = 0; offset < ref.bytes; offset += ARCHIVE_FILE_CHUNK_BYTES) {
|
|
2460
|
-
yield await source.readRange(ref, offset, Math.min(ARCHIVE_FILE_CHUNK_BYTES, ref.bytes - offset));
|
|
2461
|
-
}
|
|
2462
|
-
const padding = tarPadding(ref.bytes);
|
|
2463
|
-
if (padding.byteLength > 0) yield padding;
|
|
3620
|
+
function growPartitionsTo(manager, highestPartitionId) {
|
|
3621
|
+
manager.growToSavedPartitionCount(highestPartitionId + 1);
|
|
2464
3622
|
}
|
|
2465
|
-
async function
|
|
2466
|
-
|
|
2467
|
-
|
|
2468
|
-
|
|
2469
|
-
|
|
2470
|
-
|
|
3623
|
+
async function loadSegmentedCheckpoint(store, indexName, manifest, deps, decodesEveryDocument = false) {
|
|
3624
|
+
const highestPartitionId = highestPartitionIdOf(manifest.partitions);
|
|
3625
|
+
requireSavedPartitionCount(highestPartitionId + 1, "segment manifest", indexName);
|
|
3626
|
+
growPartitionsTo(deps.manager, highestPartitionId);
|
|
3627
|
+
const { directory } = store;
|
|
3628
|
+
const opensVectorsOnDemand = manifest.vectors.some(
|
|
3629
|
+
(field) => deps.vectorIndexes.get(field.fieldPath)?.storage === "disk" && field.files.some((file) => file.headOffset !== void 0)
|
|
3630
|
+
);
|
|
3631
|
+
if (!decodesEveryDocument && directory !== void 0 && opensVectorsOnDemand) {
|
|
3632
|
+
await startReadingOpeningSectionsAhead(directory, manifest);
|
|
2471
3633
|
}
|
|
2472
|
-
|
|
2473
|
-
|
|
3634
|
+
for (const partition of manifest.partitions) {
|
|
3635
|
+
await loadPartitionSegments(store, partition, deps.manager, {
|
|
3636
|
+
schema: manifest.schema,
|
|
3637
|
+
decodesEveryDocument,
|
|
3638
|
+
writeRecords: deps.writeRecords ?? null
|
|
3639
|
+
});
|
|
2474
3640
|
}
|
|
2475
|
-
|
|
3641
|
+
for (const vector of manifest.vectors) {
|
|
3642
|
+
const vectorIndex = deps.vectorIndexes.get(vector.fieldPath);
|
|
3643
|
+
if (vectorIndex === void 0) continue;
|
|
3644
|
+
await loadVectorField(
|
|
3645
|
+
store,
|
|
3646
|
+
vector,
|
|
3647
|
+
vectorIndex,
|
|
3648
|
+
decodesEveryDocument ? null : (docId) => deps.manager.partitionIdOf(docId)
|
|
3649
|
+
);
|
|
3650
|
+
}
|
|
3651
|
+
return manifest.checkpoint;
|
|
2476
3652
|
}
|
|
2477
|
-
function
|
|
2478
|
-
const
|
|
2479
|
-
|
|
2480
|
-
|
|
2481
|
-
if (released) return;
|
|
2482
|
-
released = true;
|
|
2483
|
-
source.release();
|
|
2484
|
-
};
|
|
2485
|
-
return new ReadableStream({
|
|
2486
|
-
async pull(controller) {
|
|
2487
|
-
try {
|
|
2488
|
-
const step = await chunks.next();
|
|
2489
|
-
if (step.done) {
|
|
2490
|
-
release();
|
|
2491
|
-
controller.close();
|
|
2492
|
-
return;
|
|
2493
|
-
}
|
|
2494
|
-
controller.enqueue(step.value);
|
|
2495
|
-
} catch (error) {
|
|
2496
|
-
release();
|
|
2497
|
-
controller.error(error);
|
|
2498
|
-
}
|
|
2499
|
-
},
|
|
2500
|
-
async cancel() {
|
|
2501
|
-
release();
|
|
2502
|
-
await chunks.return(void 0);
|
|
2503
|
-
}
|
|
2504
|
-
});
|
|
3653
|
+
async function reclaimUnlistedKeys(store, indexName, referenced) {
|
|
3654
|
+
for (const key of await store.list(segmentsPrefix(indexName))) {
|
|
3655
|
+
if (!referenced.has(key)) await store.remove(key);
|
|
3656
|
+
}
|
|
2505
3657
|
}
|
|
2506
3658
|
|
|
2507
3659
|
// src/serialization/stored-vector-values.ts
|
|
@@ -2679,38 +3831,71 @@ function deserializePayloadV1(data) {
|
|
|
2679
3831
|
function invalid(message, details) {
|
|
2680
3832
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, `The device sync state in the index metadata ${message}`, details);
|
|
2681
3833
|
}
|
|
3834
|
+
function invalidChanges(message, details) {
|
|
3835
|
+
throw new NarsilError(
|
|
3836
|
+
ErrorCodes.CONFIG_INVALID,
|
|
3837
|
+
`The change capture setting in the index metadata ${message}`,
|
|
3838
|
+
details
|
|
3839
|
+
);
|
|
3840
|
+
}
|
|
2682
3841
|
function isPositiveSafeInteger(value) {
|
|
2683
3842
|
return typeof value === "number" && Number.isSafeInteger(value) && value >= 1;
|
|
2684
3843
|
}
|
|
3844
|
+
function historyLimitsToWire(setting) {
|
|
3845
|
+
return {
|
|
3846
|
+
...setting.historyMaxAgeMs !== void 0 ? { history_max_age_ms: setting.historyMaxAgeMs } : {},
|
|
3847
|
+
...setting.historyMaxBytes !== void 0 ? { history_max_bytes: setting.historyMaxBytes } : {}
|
|
3848
|
+
};
|
|
3849
|
+
}
|
|
2685
3850
|
function syncMetadataToWire(meta) {
|
|
2686
3851
|
const wire = {};
|
|
2687
3852
|
if (meta.sync !== void 0) {
|
|
2688
|
-
wire.sync = {
|
|
2689
|
-
enabled: meta.sync.enabled,
|
|
2690
|
-
...meta.sync.historyMaxAgeMs !== void 0 ? { history_max_age_ms: meta.sync.historyMaxAgeMs } : {},
|
|
2691
|
-
...meta.sync.historyMaxBytes !== void 0 ? { history_max_bytes: meta.sync.historyMaxBytes } : {}
|
|
2692
|
-
};
|
|
3853
|
+
wire.sync = { enabled: meta.sync.enabled, ...historyLimitsToWire(meta.sync) };
|
|
2693
3854
|
}
|
|
2694
3855
|
if (meta.syncActive !== void 0) wire.sync_active = meta.syncActive;
|
|
2695
3856
|
if (meta.syncEpoch !== void 0) wire.sync_epoch = meta.syncEpoch;
|
|
3857
|
+
if (meta.changes !== void 0) {
|
|
3858
|
+
wire.changes = {
|
|
3859
|
+
enabled: meta.changes.enabled,
|
|
3860
|
+
...meta.changes.previousDocument !== void 0 ? { previous_document: meta.changes.previousDocument } : {},
|
|
3861
|
+
...historyLimitsToWire(meta.changes)
|
|
3862
|
+
};
|
|
3863
|
+
}
|
|
2696
3864
|
return wire;
|
|
2697
3865
|
}
|
|
2698
|
-
function
|
|
2699
|
-
|
|
2700
|
-
const { enabled, history_max_age_ms: maxAge, history_max_bytes: maxBytes } = raw;
|
|
2701
|
-
if (typeof enabled !== "boolean") invalid("holds a sync setting without a boolean enabled", {});
|
|
3866
|
+
function historyLimitsFromWire(raw, fail) {
|
|
3867
|
+
const { history_max_age_ms: maxAge, history_max_bytes: maxBytes } = raw;
|
|
2702
3868
|
if (maxAge !== void 0 && !isPositiveSafeInteger(maxAge)) {
|
|
2703
|
-
|
|
3869
|
+
fail("holds a history age limit that is not a positive whole number", { historyMaxAgeMs: String(maxAge) });
|
|
2704
3870
|
}
|
|
2705
3871
|
if (maxBytes !== void 0 && !isPositiveSafeInteger(maxBytes)) {
|
|
2706
|
-
|
|
3872
|
+
fail("holds a history byte limit that is not a positive whole number", { historyMaxBytes: String(maxBytes) });
|
|
2707
3873
|
}
|
|
2708
3874
|
return {
|
|
2709
|
-
enabled,
|
|
2710
3875
|
...maxAge !== void 0 ? { historyMaxAgeMs: maxAge } : {},
|
|
2711
3876
|
...maxBytes !== void 0 ? { historyMaxBytes: maxBytes } : {}
|
|
2712
3877
|
};
|
|
2713
3878
|
}
|
|
3879
|
+
function storedSyncSetting(raw) {
|
|
3880
|
+
if (raw === null || typeof raw !== "object") invalid("holds a sync setting that is not a map", {});
|
|
3881
|
+
const fields = raw;
|
|
3882
|
+
if (typeof fields.enabled !== "boolean") invalid("holds a sync setting without a boolean enabled", {});
|
|
3883
|
+
return { enabled: fields.enabled, ...historyLimitsFromWire(fields, invalid) };
|
|
3884
|
+
}
|
|
3885
|
+
function storedChangesSetting(raw) {
|
|
3886
|
+
if (raw === null || typeof raw !== "object") invalidChanges("is not a map", {});
|
|
3887
|
+
const fields = raw;
|
|
3888
|
+
if (typeof fields.enabled !== "boolean") invalidChanges("lacks a boolean enabled", {});
|
|
3889
|
+
const previousDocument = fields.previous_document;
|
|
3890
|
+
if (previousDocument !== void 0 && typeof previousDocument !== "boolean") {
|
|
3891
|
+
invalidChanges("holds a previous_document that is not a boolean", {});
|
|
3892
|
+
}
|
|
3893
|
+
return {
|
|
3894
|
+
enabled: fields.enabled,
|
|
3895
|
+
...previousDocument !== void 0 ? { previousDocument } : {},
|
|
3896
|
+
...historyLimitsFromWire(fields, invalidChanges)
|
|
3897
|
+
};
|
|
3898
|
+
}
|
|
2714
3899
|
function syncMetadataFromWire(raw, meta) {
|
|
2715
3900
|
if (raw.sync !== void 0 && raw.sync !== null) meta.sync = storedSyncSetting(raw.sync);
|
|
2716
3901
|
if (raw.sync_active !== void 0 && raw.sync_active !== null) {
|
|
@@ -2721,6 +3906,7 @@ function syncMetadataFromWire(raw, meta) {
|
|
|
2721
3906
|
if (!isSyncEpoch(raw.sync_epoch)) invalid("holds an epoch that is not 32 lowercase hexadecimal digits", {});
|
|
2722
3907
|
meta.syncEpoch = raw.sync_epoch;
|
|
2723
3908
|
}
|
|
3909
|
+
if (raw.changes !== void 0 && raw.changes !== null) meta.changes = storedChangesSetting(raw.changes);
|
|
2724
3910
|
}
|
|
2725
3911
|
|
|
2726
3912
|
// src/serialization/payload-v1/metadata.ts
|
|
@@ -3171,202 +4357,32 @@ function applyDeleteEntry(entry, manager, vecIndexes, removedSpellings) {
|
|
|
3171
4357
|
manager.remove(entry.documentId, removedSpellings);
|
|
3172
4358
|
}
|
|
3173
4359
|
|
|
3174
|
-
// src/
|
|
3175
|
-
function
|
|
3176
|
-
|
|
3177
|
-
|
|
3178
|
-
|
|
3179
|
-
|
|
3180
|
-
|
|
3181
|
-
|
|
3182
|
-
|
|
3183
|
-
|
|
3184
|
-
|
|
3185
|
-
|
|
3186
|
-
}
|
|
3187
|
-
function requireListedCounts(ref, documentCount, tombstoneCount) {
|
|
3188
|
-
if (ref.docCount === documentCount && ref.tombstoneCount === tombstoneCount) return;
|
|
3189
|
-
throw new NarsilError(
|
|
3190
|
-
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
3191
|
-
`The segment "${ref.key}" holds ${documentCount} documents and ${tombstoneCount} tombstones where the manifest lists ${ref.docCount} and ${ref.tombstoneCount}`,
|
|
3192
|
-
{
|
|
3193
|
-
key: ref.key,
|
|
3194
|
-
documentCount,
|
|
3195
|
-
tombstoneCount,
|
|
3196
|
-
listedDocCount: ref.docCount,
|
|
3197
|
-
listedTombstoneCount: ref.tombstoneCount
|
|
3198
|
-
}
|
|
3199
|
-
);
|
|
3200
|
-
}
|
|
3201
|
-
function removeWhatTheSegmentReplaces(manager, segment, contents) {
|
|
3202
|
-
manager.beginBatchRemove();
|
|
3203
|
-
try {
|
|
3204
|
-
for (const docId of contents.tombstones) {
|
|
3205
|
-
if (manager.has(docId)) manager.remove(docId);
|
|
3206
|
-
}
|
|
3207
|
-
const ids = segment.arrays.idTable;
|
|
3208
|
-
for (let ordinal = 0; ordinal < ids.count; ordinal++) {
|
|
3209
|
-
const docId = ids.idAt(ordinal);
|
|
3210
|
-
if (manager.has(docId)) manager.remove(docId);
|
|
3211
|
-
}
|
|
3212
|
-
} finally {
|
|
3213
|
-
manager.endBatchRemove();
|
|
3214
|
-
}
|
|
3215
|
-
}
|
|
3216
|
-
function yieldToEventLoop2() {
|
|
3217
|
-
return new Promise((resolve) => setTimeout(resolve, 0));
|
|
3218
|
-
}
|
|
3219
|
-
function removeFromPartition(manager, partitionId, docIds) {
|
|
3220
|
-
const { composite, docPartitionMap, insertOptions } = manager.diskSegmentAccess(partitionId);
|
|
3221
|
-
composite.beginBatch();
|
|
3222
|
-
try {
|
|
3223
|
-
for (const docId of docIds) {
|
|
3224
|
-
if (!composite.has(docId)) continue;
|
|
3225
|
-
composite.remove(docId, manager.config.schema, manager.language, insertOptions);
|
|
3226
|
-
docPartitionMap.delete(docId);
|
|
3227
|
-
}
|
|
3228
|
-
} finally {
|
|
3229
|
-
composite.endBatch();
|
|
3230
|
-
}
|
|
3231
|
-
}
|
|
3232
|
-
async function attachPagedSegment(directory, ref, partitionId, manager, options) {
|
|
3233
|
-
if (options.decodesEveryDocument) await checkPagedSegmentFully(directory, ref);
|
|
3234
|
-
const snapshot = openPagedSnapshot(await pagedListingOf(directory, ref));
|
|
3235
|
-
requireListedCounts(ref, snapshot.documentCount, snapshot.tombstones.length);
|
|
3236
|
-
requireAgreedSchema(options.schema, snapshot.schema, ref.key);
|
|
3237
|
-
const segment = createPagedFrozenSegment(snapshot);
|
|
3238
|
-
removeFromPartition(manager, partitionId, snapshot.tombstones);
|
|
3239
|
-
options.writeRecords?.loadPagedSegment(manager.indexName, partitionId, segment);
|
|
3240
|
-
if (snapshot.documentCount > 0) {
|
|
3241
|
-
manager.attachFrozenSegment(partitionId, segment);
|
|
3242
|
-
return segment;
|
|
3243
|
-
}
|
|
3244
|
-
segment.paged?.close();
|
|
3245
|
-
return null;
|
|
3246
|
-
}
|
|
3247
|
-
function documentsHeldBefore(manager, partitionId) {
|
|
3248
|
-
const held = manager.partitionAt(partitionId);
|
|
3249
|
-
return held === void 0 ? [] : [...held.docIds()];
|
|
3250
|
-
}
|
|
3251
|
-
function removeDocumentsMissingFromLoadedSegments(manager, partitionId, heldBefore, loaded) {
|
|
3252
|
-
const absent = heldBefore.filter(
|
|
3253
|
-
(docId) => manager.partitionIdOf(docId) === partitionId && !loaded.some((segment) => segment.hasDocument(docId))
|
|
3254
|
-
);
|
|
3255
|
-
if (absent.length === 0) return;
|
|
3256
|
-
manager.beginBatchRemove();
|
|
3257
|
-
try {
|
|
3258
|
-
for (const docId of absent) manager.remove(docId);
|
|
3259
|
-
} finally {
|
|
3260
|
-
manager.endBatchRemove();
|
|
3261
|
-
}
|
|
3262
|
-
}
|
|
3263
|
-
async function loadPartitionSegments(store, partition, manager, options) {
|
|
3264
|
-
const heldBefore = documentsHeldBefore(manager, partition.partitionId);
|
|
3265
|
-
const loaded = await attachPartitionSegments(store, partition, manager, options);
|
|
3266
|
-
removeDocumentsMissingFromLoadedSegments(manager, partition.partitionId, heldBefore, loaded);
|
|
3267
|
-
}
|
|
3268
|
-
async function attachPartitionSegments(store, partition, manager, options) {
|
|
3269
|
-
const loaded = [];
|
|
3270
|
-
const { directory } = store;
|
|
3271
|
-
for (const ref of partition.segments) {
|
|
3272
|
-
if (manager.config.textStorage === "disk" && directory !== void 0) {
|
|
3273
|
-
if (await isPagedSegmentFile(directory, ref.key)) {
|
|
3274
|
-
const paged = await attachPagedSegment(directory, ref, partition.partitionId, manager, options);
|
|
3275
|
-
if (paged !== null) loaded.push(paged);
|
|
3276
|
-
await yieldToEventLoop2();
|
|
3277
|
-
continue;
|
|
3278
|
-
}
|
|
4360
|
+
// src/workers/detached-index.ts
|
|
4361
|
+
function attachablePolicy(shared) {
|
|
4362
|
+
let attached = false;
|
|
4363
|
+
const policy = {
|
|
4364
|
+
get enabled() {
|
|
4365
|
+
return attached && shared.enabled;
|
|
4366
|
+
},
|
|
4367
|
+
get count() {
|
|
4368
|
+
return shared.count;
|
|
4369
|
+
},
|
|
4370
|
+
get host() {
|
|
4371
|
+
return shared.host;
|
|
3279
4372
|
}
|
|
3280
|
-
|
|
3281
|
-
|
|
3282
|
-
|
|
3283
|
-
|
|
3284
|
-
|
|
3285
|
-
removeWhatTheSegmentReplaces(manager, segment, contents);
|
|
3286
|
-
options.writeRecords?.loadSegment(manager.indexName, partition.partitionId, segment, contents);
|
|
3287
|
-
if (contents.arrays.documentCount > 0) {
|
|
3288
|
-
manager.attachFrozenSegment(partition.partitionId, segment);
|
|
3289
|
-
loaded.push(segment);
|
|
4373
|
+
};
|
|
4374
|
+
return {
|
|
4375
|
+
policy,
|
|
4376
|
+
attach() {
|
|
4377
|
+
attached = true;
|
|
3290
4378
|
}
|
|
3291
|
-
|
|
3292
|
-
}
|
|
3293
|
-
return loaded;
|
|
3294
|
-
}
|
|
3295
|
-
|
|
3296
|
-
// src/persistence/durability/segment/load.ts
|
|
3297
|
-
async function readSegmentManifest(store, indexName) {
|
|
3298
|
-
const bytes = await store.read(manifestKey(indexName));
|
|
3299
|
-
if (bytes === null) return null;
|
|
3300
|
-
return decodeSegmentManifest(bytes, indexName);
|
|
3301
|
-
}
|
|
3302
|
-
async function readCurrentManifest(store, indexName) {
|
|
3303
|
-
const decoded = await readSegmentManifest(store, indexName);
|
|
3304
|
-
if (decoded === null) return null;
|
|
3305
|
-
if (decoded.kind === "current") return decoded.manifest;
|
|
3306
|
-
throw new NarsilError(
|
|
3307
|
-
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
3308
|
-
`Index "${indexName}" still holds the layout of an earlier release, which recovery upgrades before any checkpoint`,
|
|
3309
|
-
{ indexName }
|
|
3310
|
-
);
|
|
3311
|
-
}
|
|
3312
|
-
function growPartitionsTo(manager, highestPartitionId) {
|
|
3313
|
-
manager.growToSavedPartitionCount(highestPartitionId + 1);
|
|
3314
|
-
}
|
|
3315
|
-
async function loadSegmentedCheckpoint(store, indexName, manifest, deps, decodesEveryDocument = false) {
|
|
3316
|
-
const highestPartitionId = highestPartitionIdOf(manifest.partitions);
|
|
3317
|
-
requireSavedPartitionCount(highestPartitionId + 1, "segment manifest", indexName);
|
|
3318
|
-
growPartitionsTo(deps.manager, highestPartitionId);
|
|
3319
|
-
const { directory } = store;
|
|
3320
|
-
const opensVectorsOnDemand = manifest.vectors.some(
|
|
3321
|
-
(field) => deps.vectorIndexes.get(field.fieldPath)?.storage === "disk" && field.files.some((file) => file.headOffset !== void 0)
|
|
3322
|
-
);
|
|
3323
|
-
if (!decodesEveryDocument && directory !== void 0 && opensVectorsOnDemand) {
|
|
3324
|
-
await startReadingOpeningSectionsAhead(directory, manifest);
|
|
3325
|
-
}
|
|
3326
|
-
for (const partition of manifest.partitions) {
|
|
3327
|
-
await loadPartitionSegments(store, partition, deps.manager, {
|
|
3328
|
-
schema: manifest.schema,
|
|
3329
|
-
decodesEveryDocument,
|
|
3330
|
-
writeRecords: deps.writeRecords ?? null
|
|
3331
|
-
});
|
|
3332
|
-
}
|
|
3333
|
-
for (const vector of manifest.vectors) {
|
|
3334
|
-
const vectorIndex = deps.vectorIndexes.get(vector.fieldPath);
|
|
3335
|
-
if (vectorIndex === void 0) continue;
|
|
3336
|
-
await loadVectorField(
|
|
3337
|
-
store,
|
|
3338
|
-
vector,
|
|
3339
|
-
vectorIndex,
|
|
3340
|
-
decodesEveryDocument ? null : (docId) => deps.manager.partitionIdOf(docId)
|
|
3341
|
-
);
|
|
3342
|
-
}
|
|
3343
|
-
return manifest.checkpoint;
|
|
3344
|
-
}
|
|
3345
|
-
async function reclaimUnlistedKeys(store, indexName, referenced) {
|
|
3346
|
-
for (const key of await store.list(segmentsPrefix(indexName))) {
|
|
3347
|
-
if (!referenced.has(key)) await store.remove(key);
|
|
3348
|
-
}
|
|
4379
|
+
};
|
|
3349
4380
|
}
|
|
3350
|
-
|
|
3351
|
-
|
|
3352
|
-
|
|
3353
|
-
|
|
3354
|
-
|
|
3355
|
-
const held = [];
|
|
3356
|
-
for await (const chunk of chunks) held.push(chunk);
|
|
3357
|
-
await store.write(key, held);
|
|
3358
|
-
return;
|
|
3359
|
-
}
|
|
3360
|
-
const receiving = `${key}.receiving`;
|
|
3361
|
-
await directory.remove(receiving);
|
|
3362
|
-
const handle = await directory.appendHandle(receiving);
|
|
3363
|
-
try {
|
|
3364
|
-
for await (const chunk of chunks) await handle.append(chunk);
|
|
3365
|
-
await handle.sync();
|
|
3366
|
-
} finally {
|
|
3367
|
-
await handle.close();
|
|
3368
|
-
}
|
|
3369
|
-
await directory.move(receiving, key);
|
|
4381
|
+
function createDetachedIndex(indexName, config, language, synonyms, executor, partitionCount) {
|
|
4382
|
+
const { policy, attach } = attachablePolicy(executor.vectorWorkerCopies);
|
|
4383
|
+
const entry = executor.createDetachedEntry(indexName, config, language, policy, partitionCount);
|
|
4384
|
+
entry.manager.synonyms = synonyms;
|
|
4385
|
+
return { indexName, manager: entry.manager, entry, attach };
|
|
3370
4386
|
}
|
|
3371
4387
|
|
|
3372
4388
|
// src/persistence/durability/marker-writer.ts
|
|
@@ -3424,7 +4440,7 @@ async function loadMetadata(directory, indexName) {
|
|
|
3424
4440
|
return metadata;
|
|
3425
4441
|
}
|
|
3426
4442
|
async function replayWal(directory, indexName, partitionId, fromSeqNoExclusive, deps) {
|
|
3427
|
-
await directory
|
|
4443
|
+
await removeUnusedSegmentFiles(directory, indexName, partitionId);
|
|
3428
4444
|
const read = await readWalSegments(directory, indexName, partitionId);
|
|
3429
4445
|
if (read === null) {
|
|
3430
4446
|
return { highestSeqNo: fromSeqNoExclusive, highestPrimaryTerm: 0 };
|
|
@@ -3488,7 +4504,7 @@ async function replayWalUpTo(directory, indexName, partitionId, fromSeqNoExclusi
|
|
|
3488
4504
|
return replayDurableEntries(read, indexName, partitionId, fromSeqNoExclusive, upToSeqNoInclusive, deps);
|
|
3489
4505
|
}
|
|
3490
4506
|
async function replaySyncedWal(directory, indexName, partitionId, fromSeqNoExclusive, deps) {
|
|
3491
|
-
await directory
|
|
4507
|
+
await removeUnusedSegmentFiles(directory, indexName, partitionId);
|
|
3492
4508
|
const read = await readWalSegments(directory, indexName, partitionId);
|
|
3493
4509
|
if (read === null) {
|
|
3494
4510
|
return { highestSeqNo: fromSeqNoExclusive, highestPrimaryTerm: 0 };
|
|
@@ -3548,6 +4564,7 @@ async function truncateSegmentTail(directory, key, durableByteLength) {
|
|
|
3548
4564
|
}
|
|
3549
4565
|
function applyEntry2(entry, deps) {
|
|
3550
4566
|
deps.writeRecords?.replayed(entry);
|
|
4567
|
+
if (entry.operation === "DELETE" || deps.manager.has(entry.documentId)) noteDocumentChurn(entry.indexName, 1);
|
|
3551
4568
|
if (entry.operation === "DELETE") {
|
|
3552
4569
|
applyDeleteEntry(entry, deps.manager, deps.vectorIndexes);
|
|
3553
4570
|
return;
|
|
@@ -3576,32 +4593,4 @@ async function drainIndexStateForUnload(indexState, pendingMetadataWrite, closeW
|
|
|
3576
4593
|
}
|
|
3577
4594
|
}
|
|
3578
4595
|
|
|
3579
|
-
|
|
3580
|
-
function attachablePolicy(shared) {
|
|
3581
|
-
let attached = false;
|
|
3582
|
-
const policy = {
|
|
3583
|
-
get enabled() {
|
|
3584
|
-
return attached && shared.enabled;
|
|
3585
|
-
},
|
|
3586
|
-
get count() {
|
|
3587
|
-
return shared.count;
|
|
3588
|
-
},
|
|
3589
|
-
get host() {
|
|
3590
|
-
return shared.host;
|
|
3591
|
-
}
|
|
3592
|
-
};
|
|
3593
|
-
return {
|
|
3594
|
-
policy,
|
|
3595
|
-
attach() {
|
|
3596
|
-
attached = true;
|
|
3597
|
-
}
|
|
3598
|
-
};
|
|
3599
|
-
}
|
|
3600
|
-
function createDetachedIndex(indexName, config, language, synonyms, executor, partitionCount) {
|
|
3601
|
-
const { policy, attach } = attachablePolicy(executor.vectorWorkerCopies);
|
|
3602
|
-
const entry = executor.createDetachedEntry(indexName, config, language, policy, partitionCount);
|
|
3603
|
-
entry.manager.synonyms = synonyms;
|
|
3604
|
-
return { indexName, manager: entry.manager, entry, attach };
|
|
3605
|
-
}
|
|
3606
|
-
|
|
3607
|
-
export { SINGLE_NODE_PRIMARY_TERM, abandonAnalysisChange, admitDurableWrite, adoptSavedAnalysis, applyDeleteEntry, applyIndexEntry, archiveFootprint, archiveStream, buildArchiveFromMemory, buildHeldAnalysisChange, captureAppliedPrefix, changeIndexAnalysis, checkpointInTurn, chunksOf, closePartitionWriter, configOfNamedSettings, createByteReader, createDetachedIndex, createIndexLoads, createMarkerWriter, createRebalancer, createSeqOwner, deserializePayloadV1, drainIndexStateForUnload, dropHeldAnalysisChange, dropStoredVectorValues, durableMetadataConfigOf, earlierSnapshotFootprint, executeRebalance, growPartitionsTo, heldAnalysisIsBuilt, highestPartitionIdOf, holdScheduledCheckpoint, holdsNamedSettings, holdsSettings, indexMetadataOf, invalidTransition, isRecord2 as isRecord, listPersistedIndexes, loadMetadata, loadPartitionSegments, loadSegmentedCheckpoint, memoryCheckpointStore, metadataConfigOf, metadataFromConfig, metadataPartitionCount, namedSettingsOf, nestedSchemaOf, patternIndexesFromWire, readCurrentManifest, readMetadataEnvelope, readSegmentManifest, readTransition, reclaimUnlistedKeys, releaseHeldAnalysisChange, removeLeftoverRewriteFiles, replaySyncedWal, replayWal, replayWalUpTo, requireCheckpointLayout, requireConfigurationWithinSizeLimit, requireRestoreMemory, requireSavedPartitionCount, requiresLayoutCheckpoint, resolveAnalysisTarget, resumeAnalysisChange, sanitizeSurfaceForms, snapshotCheckpointFor, stampAnalysisGeneration, startReadingOpeningSectionsAhead, storedBytesOfSegments, switchInTurn, targetOfPending, transitionKey, transitionWalLayout, unpackEnvelopeBytes, unpackIndexSnapshotEnvelope, validateReplicationEntry, whileGraphsStayComplete, writeArchiveFile, writeLiveVectors, writeMetadataEnvelope };
|
|
4596
|
+
export { DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, SINGLE_NODE_PRIMARY_TERM, abandonAnalysisChange, admitDurableWrite, adoptSavedAnalysis, applyDeleteEntry, applyIndexEntry, archiveFootprint, archiveStream, buildArchiveFromMemory, buildHeldAnalysisChange, captureAppliedPrefix, changeIndexAnalysis, changesOf, changesStatusOf, checkpointInTurn, closePartitionWriter, configOfNamedSettings, createByteReader, createDetachedIndex, createIndexLoads, createMarkerWriter, createRebalancer, createSeqOwner, deserializePayloadV1, documentExportStream, drainIndexStateForUnload, dropHeldAnalysisChange, dropStoredVectorValues, durableMetadataConfigOf, earlierSnapshotFootprint, executeRebalance, followIndexChanges, followSourceChanges, growPartitionsTo, heldAnalysisIsBuilt, highestPartitionIdOf, holdScheduledCheckpoint, holdsNamedSettings, holdsSettings, indexMetadataOf, invalidTransition, listPersistedIndexes, loadMetadata, loadPartitionSegments, loadSegmentedCheckpoint, memoryCheckpointStore, metadataConfigOf, metadataFromConfig, metadataPartitionCount, namedSettingsOf, nestedSchemaOf, openFileVectorReads, pagedPartitionLines, patternIndexesFromWire, readCurrentManifest, readDocumentExport, readMetadataEnvelope, readSegmentManifest, readTransition, reclaimUnlistedKeys, releaseHeldAnalysisChange, removeLeftoverRewriteFiles, replaySyncedWal, replayWal, replayWalUpTo, requireChangesSetting, requireCheckpointLayout, requireConfigurationWithinSizeLimit, requireRestoreMemory, requireSavedPartitionCount, requiresLayoutCheckpoint, resolveAnalysisTarget, resumeAnalysisChange, sanitizeSurfaceForms, snapshotCheckpointFor, stampAnalysisGeneration, startReadingOpeningSectionsAhead, storedBytesOfSegments, switchInTurn, targetOfPending, transitionKey, transitionWalLayout, unpackEnvelopeBytes, unpackIndexSnapshotEnvelope, updateChangesOf, validateReplicationEntry, whileGraphsStayComplete, writeArchiveFile, writeLiveVectors, writeMetadataEnvelope };
|