@delali/narsil 0.3.1-next.162 → 0.3.1-next.169
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -8
- package/dist/browser/{chunk-MZHQ65VI.mjs → chunk-2Q45HI5H.mjs} +14 -10
- package/dist/browser/{chunk-MMHIIL5B.mjs → chunk-4ZICWS4V.mjs} +1 -1
- package/dist/browser/{chunk-KQXKCMGQ.mjs → chunk-JHFBQKUF.mjs} +119 -90
- package/dist/browser/{core-leaf-SIWXFZLV.mjs → core-leaf-WYFANMKC.mjs} +61 -13
- package/dist/browser/custom-token-results-R5BIX7SL.mjs +59 -0
- package/dist/browser/{embed-RF2MPGAO.mjs → embed-24IPR4LP.mjs} +2 -2
- package/dist/browser/{module-PMQBRESM.mjs → module-XVLHJ7T7.mjs} +358 -62
- package/dist/browser/{optional-writes-WMZX4OEC.mjs → optional-writes-TB34FYUW.mjs} +118 -66
- package/dist/browser/{phrase-module-22TKMXWI.mjs → phrase-module-6UGFVW5J.mjs} +1 -0
- package/dist/browser/{resolve-vector-text-KI65IHMD.mjs → resolve-vector-text-EJ6DHFF5.mjs} +8 -0
- package/dist/browser/{storage-leaf-VGR2BWJJ.mjs → storage-leaf-QFXIWDFK.mjs} +208 -87
- package/dist/browser/{worker-copies-WQCPP3TS.mjs → worker-copies-ICHLYYXM.mjs} +1 -0
- package/dist/{capabilities-DkXAkb-X.d.ts → capabilities-BQIdsaWy.d.ts} +2 -2
- package/dist/{chunk-BKOMO7HT.mjs → chunk-26E5MO3Y.mjs} +5 -5
- package/dist/{chunk-EEYDJB4Z.mjs → chunk-2C3BXZGA.mjs} +213 -149
- package/dist/{chunk-NXMGUDZA.mjs → chunk-2KWTQ2P7.mjs} +2 -2
- package/dist/{chunk-TW543F33.mjs → chunk-34IOOYXA.mjs} +1 -1
- package/dist/{chunk-D5GNFN6X.mjs → chunk-7FC2ZILY.mjs} +98 -40
- package/dist/{chunk-XRBPHCCT.mjs → chunk-A4XYDUVN.mjs} +5 -5
- package/dist/{chunk-R4F7ZTVG.mjs → chunk-BIIROQ5Q.mjs} +2 -2
- package/dist/{chunk-Q2WQSAYH.mjs → chunk-CQX5B6L2.mjs} +54 -11
- package/dist/{chunk-XXWEUXE4.mjs → chunk-D6FYUTFN.mjs} +1 -1
- package/dist/{chunk-OUWZKNO3.mjs → chunk-DG6WIEB3.mjs} +1 -1
- package/dist/{chunk-GJKFXKMI.mjs → chunk-F3FDHXCJ.mjs} +12 -9
- package/dist/{chunk-3UGZBRDN.mjs → chunk-GO6TRSJ6.mjs} +1 -1
- package/dist/{react-native/chunks/chunk-OK3WUA6N.mjs → chunk-I2WJCRIH.mjs} +1 -1
- package/dist/{chunk-Q7CUUSJ2.mjs → chunk-ISJ35523.mjs} +2 -2
- package/dist/{chunk-GQLOEACG.mjs → chunk-JCWC75NH.mjs} +2 -2
- package/dist/{chunk-ZGYF37MG.mjs → chunk-JDIATDPY.mjs} +1 -1
- package/dist/{chunk-3B775F3C.mjs → chunk-JP77OWMI.mjs} +3 -1
- package/dist/{chunk-KSOZJEYU.mjs → chunk-KHJPMZE2.mjs} +1 -1
- package/dist/{chunk-ZQ6QY42L.mjs → chunk-KL6W73BZ.mjs} +8 -0
- package/dist/{chunk-ES5EOXT6.mjs → chunk-LOQHA6QK.mjs} +3 -3
- package/dist/{chunk-AB27FWZM.mjs → chunk-LPRRFCVU.mjs} +2 -2
- package/dist/{chunk-EIR3IQCL.mjs → chunk-QFJMRB7D.mjs} +2 -1
- package/dist/{chunk-S3BSDBF7.mjs → chunk-QJGDB3WR.mjs} +1 -1
- package/dist/{chunk-EI7U43MX.mjs → chunk-R7PLSVK3.mjs} +2 -2
- package/dist/{chunk-5DUTOEK5.mjs → chunk-TBOZHYCF.mjs} +2 -2
- package/dist/{chunk-VJOS6MHF.mjs → chunk-TNEU32EN.mjs} +64 -16
- package/dist/{chunk-RXMR2SIQ.mjs → chunk-UISV46WX.mjs} +4 -4
- package/dist/{chunk-4CAMH7FO.mjs → chunk-X4UMDSB4.mjs} +2 -1
- package/dist/{chunk-R4CE4IVU.mjs → chunk-X5NHWOXT.mjs} +15 -11
- package/dist/{react-native/chunks/chunk-HEELWLKL.mjs → chunk-XFOL77WH.mjs} +1 -1
- package/dist/{chunk-TOZENMM7.mjs → chunk-XITVDRRH.mjs} +5 -5
- package/dist/{chunk-FSLQHOKZ.mjs → chunk-XNFBYAXI.mjs} +1 -1
- package/dist/{chunk-V4YIEMAI.mjs → chunk-XQQKETAF.mjs} +4 -4
- package/dist/{chunk-DSXFN5BF.mjs → chunk-XTSFRWAS.mjs} +8 -8
- package/dist/{chunk-K4BMDPPX.mjs → chunk-Y27E2MGC.mjs} +2 -2
- package/dist/{chunk-JV74RKCY.mjs → chunk-YM2HI5N5.mjs} +1 -1
- package/dist/{chunk-7OE73OO4.mjs → chunk-YPV2L7M5.mjs} +56 -6
- package/dist/{chunk-STLNPJXU.mjs → chunk-YXZDTKQS.mjs} +2 -2
- package/dist/{chunk-5P72V66D.mjs → chunk-ZB5X5WAK.mjs} +24 -20
- package/dist/{client-VJmV_AjZ.d.ts → client-CeUxvIvd.d.ts} +4 -4
- package/dist/client.d.ts +8 -8
- package/dist/{config-ZZg4cwQz.d.ts → config-CIpLXXxm.d.ts} +2 -2
- package/dist/{config-Cb3FOlE_.d.ts → config-DehU-VGL.d.ts} +2 -2
- package/dist/{core-leaf-LOXCD7MY.mjs → core-leaf-VIMO377O.mjs} +3 -3
- package/dist/custom-token-results-NXHBVY7U.mjs +59 -0
- package/dist/{direct-executor-PF4UDSUW.mjs → direct-executor-2WQSGBN3.mjs} +7 -7
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution/transport/tcp.mjs +1 -1
- package/dist/distribution.d.ts +7 -7
- package/dist/distribution.mjs +39 -39
- package/dist/{embed-4H7RTBVL.mjs → embed-UNHDFGDU.mjs} +2 -2
- package/dist/{engine-DdOs682x.d.ts → engine-P-4xhUe4.d.ts} +6 -2
- package/dist/{index-Ck9w5WHz.d.ts → index-CBUl4z3E.d.ts} +1 -1
- package/dist/index.browser.mjs +3 -3
- package/dist/index.d.ts +6 -6
- package/dist/index.mjs +7 -7
- package/dist/index.react-native.mjs +6 -6
- package/dist/{long-query-thread-OTVLP2OI.mjs → long-query-thread-AAYTLYZK.mjs} +19 -19
- package/dist/{module-UIKSHY3G.mjs → module-5SXNUCD5.mjs} +315 -64
- package/dist/{msgpack-leaf-5WFRP6JM.mjs → msgpack-leaf-7KBYOVTF.mjs} +2 -2
- package/dist/{optional-reads-RSLNSGNG.mjs → optional-reads-ENKMVQ4Q.mjs} +1 -1
- package/dist/{optional-writes-4ITCV7YT.mjs → optional-writes-N2DFWWZ3.mjs} +39 -26
- package/dist/{paged-partition-archive-VMJ2AUA6.mjs → paged-partition-archive-X7L27E5Z.mjs} +19 -19
- package/dist/partial-words.browser.mjs +3 -3
- package/dist/partial-words.mjs +7 -7
- package/dist/pattern.browser.mjs +2 -2
- package/dist/pattern.mjs +2 -2
- package/dist/persistence/durability/checkpoint-worker.d.ts +1 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +17 -17
- package/dist/{phrase-module-IWSG2DDZ.mjs → phrase-module-ZY3TLTQA.mjs} +2 -1
- package/dist/{query-actions-EW72OQXY.mjs → query-actions-2JOLPELC.mjs} +7 -7
- package/dist/query-suggestions/merge-worker.mjs +10 -10
- package/dist/query-suggestions.browser.mjs +3 -3
- package/dist/query-suggestions.d.ts +2 -2
- package/dist/query-suggestions.mjs +8 -8
- package/dist/{chunk-ZBVHS5KF.mjs → react-native/chunks/chunk-57R74JWC.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-AUXXPSFB.mjs → chunk-F4MIGBBP.mjs} +214 -150
- package/dist/react-native/chunks/{chunk-3RUXKZJG.mjs → chunk-F6BJYCF2.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-RVHCQJQM.mjs → chunk-GGBATERA.mjs} +3 -3
- package/dist/react-native/chunks/{chunk-I5W7A2JF.mjs → chunk-GVWLACAT.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-XOCM3BFU.mjs → chunk-H344ZXRH.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-VIPNDN4B.mjs → chunk-I7DTUOK2.mjs} +65 -15
- package/dist/react-native/chunks/{chunk-2JMAUU26.mjs → chunk-J2BQTJJE.mjs} +1 -1
- package/dist/{chunk-QBVTYY4J.mjs → react-native/chunks/chunk-OZ3XEM6D.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-4EXKQ2IB.mjs → chunk-SK3DPAI4.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-XBTFK3AZ.mjs → chunk-WFQ6YQZA.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-FTQ7JNMU.mjs → chunk-ZDDJVR7X.mjs} +15 -11
- package/dist/react-native/chunks/{core-leaf-O3236QY3.mjs → core-leaf-UDUMVC3M.mjs} +2 -2
- package/dist/react-native/chunks/custom-token-results-3OZEMRE7.mjs +59 -0
- package/dist/react-native/chunks/{embed-3WQ2CCR5.mjs → embed-U7CIKEY4.mjs} +2 -2
- package/dist/react-native/chunks/{module-LRR7D5ZO.mjs → module-7BJ3VOWQ.mjs} +360 -66
- package/dist/react-native/chunks/{msgpack-leaf-TR66ZVAX.mjs → msgpack-leaf-GTKLF6GJ.mjs} +2 -2
- package/dist/react-native/chunks/{optional-reads-TOWGNEDH.mjs → optional-reads-74HYI4DO.mjs} +1 -1
- package/dist/react-native/chunks/{optional-writes-7AFBIZMN.mjs → optional-writes-TXTJ6HUZ.mjs} +37 -24
- package/dist/react-native/chunks/{phrase-module-3A4AKMKL.mjs → phrase-module-UC4YGEBB.mjs} +2 -1
- package/dist/react-native/chunks/{resolve-vector-text-LXRYXF7M.mjs → resolve-vector-text-FTPD5EY6.mjs} +8 -0
- package/dist/react-native/chunks/{restore-queue-7FSHTGJ5.mjs → restore-queue-SZLWIL3B.mjs} +9 -9
- package/dist/react-native/chunks/{segment-codec-leaf-4QV4HO4R.mjs → segment-codec-leaf-WMUZ5P4F.mjs} +7 -7
- package/dist/react-native/chunks/{segment-merge-leaf-5DUZZSXJ.mjs → segment-merge-leaf-Q2OEJIQ3.mjs} +8 -8
- package/dist/react-native/chunks/{storage-leaf-PNU6HWY7.mjs → storage-leaf-OXYV6O6P.mjs} +213 -100
- package/dist/react-native/chunks/{store-leaf-ESVUGYYO.mjs → store-leaf-ZMDXYK4E.mjs} +8 -8
- package/dist/react-native/chunks/{suggestions-leaf-MOVOPBGD.mjs → suggestions-leaf-FIMGQZ2T.mjs} +8 -8
- package/dist/react-native/chunks/{worker-copies-VQGF3WZX.mjs → worker-copies-72TGFYQB.mjs} +8 -7
- package/dist/react-native/partial-words.mjs +6 -6
- package/dist/react-native/pattern.mjs +3 -3
- package/dist/react-native/query-suggestions.mjs +6 -6
- package/dist/react-native/sync.mjs +6 -6
- package/dist/react-native/synonyms.mjs +6 -6
- package/dist/react.d.ts +9 -9
- package/dist/{rerank-QHJXPSAK.mjs → rerank-E2HFKKGJ.mjs} +7 -7
- package/dist/{resolve-vector-text-4PDRTPNQ.mjs → resolve-vector-text-6WPFFIX5.mjs} +1 -1
- package/dist/{restore-queue-5W5HX44H.mjs → restore-queue-PUVBN6L3.mjs} +10 -10
- package/dist/{schema-C_7yRJLt.d.ts → schema-CzG9OUw5.d.ts} +5 -1
- package/dist/{segment-codec-leaf-7VLVXKWO.mjs → segment-codec-leaf-24Z3MMQI.mjs} +8 -8
- package/dist/{segment-merge-leaf-22XR3IGA.mjs → segment-merge-leaf-FQQDU45E.mjs} +8 -8
- package/dist/server.d.ts +10 -10
- package/dist/server.mjs +26 -26
- package/dist/{storage-leaf-ZZ77C3DX.mjs → storage-leaf-KIA7XO6Y.mjs} +79 -81
- package/dist/{store-leaf-LJ6D2PMF.mjs → store-leaf-ESVLL4JO.mjs} +9 -9
- package/dist/{suggestions-leaf-KTQHO3PY.mjs → suggestions-leaf-FXF4AHAY.mjs} +11 -11
- package/dist/sync/filesystem.d.ts +3 -3
- package/dist/sync/filesystem.mjs +15 -15
- package/dist/sync/indexeddb.browser.mjs +1 -1
- package/dist/sync/indexeddb.d.ts +3 -3
- package/dist/sync/indexeddb.mjs +2 -2
- package/dist/sync.browser.mjs +3 -3
- package/dist/sync.d.ts +4 -4
- package/dist/sync.mjs +7 -7
- package/dist/synonyms.browser.mjs +3 -3
- package/dist/synonyms.mjs +7 -7
- package/dist/{tasks-CxA-J_Ty.d.ts → tasks-CXtQ3Zpp.d.ts} +4 -4
- package/dist/thread-OWXVP2ZA.mjs +30 -0
- package/dist/{types-BboB278g.d.ts → types-CDN70Y_C.d.ts} +2 -2
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +5 -5
- package/dist/{worker-copies-ODAAGQRC.mjs → worker-copies-XXX7PF4J.mjs} +10 -10
- package/dist/workers/entry.mjs +13 -13
- package/package.json +9 -8
- package/dist/thread-XWDK6TAR.mjs +0 -30
- /package/dist/{chunk-77AUVDQ5.mjs → chunk-CV3Q2EKO.mjs} +0 -0
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { openCountsFile } from './chunk-
|
|
2
|
-
import { SCRAMBLED_ID_BYTES } from './chunk-
|
|
1
|
+
import { openCountsFile } from './chunk-YXZDTKQS.mjs';
|
|
2
|
+
import { SCRAMBLED_ID_BYTES } from './chunk-JCWC75NH.mjs';
|
|
3
3
|
export { encode } from '@msgpack/msgpack';
|
|
4
4
|
|
|
5
5
|
function bindSuggestionsHost(_host, _msgpack, _suggestionStore) {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { getFs, getPath } from './chunk-HLPRHOSO.mjs';
|
|
2
2
|
import { DIRECTORY_LOCK_ATTEMPTS, DIRECTORY_LOCK_SETTLE_MS, DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS } from './chunk-QDZJM7NI.mjs';
|
|
3
|
-
import { nodeThreadId } from './chunk-
|
|
3
|
+
import { nodeThreadId } from './chunk-DG6WIEB3.mjs';
|
|
4
4
|
import { NarsilError, ErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
|
|
5
5
|
import { compareCodePoints } from './chunk-TGPKS2UM.mjs';
|
|
6
6
|
import { openSync, close, readFileSync } from 'fs';
|
|
@@ -1,20 +1,20 @@
|
|
|
1
|
-
import { loadVectorField, VECTOR_FILE_ENVELOPE_VERSION, GRAPH_PART_ENVELOPE_VERSION } from './chunk-
|
|
2
|
-
import { RESTORE_ESTIMATE_MARGIN, RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR_DIMENSION, RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR, RESTORE_WORST_EXTERNAL_BYTES_PER_STORED_BYTE, RESTORE_WORST_HEAP_BYTES_PER_VECTOR_DIMENSION, RESTORE_WORST_HEAP_BYTES_PER_VECTOR, RESTORE_WORST_HEAP_BYTES_PER_STORED_BYTE, RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR_DIMENSION, RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR, RESTORE_FLOOR_EXTERNAL_BYTES_PER_STORED_BYTE, RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR_DIMENSION, RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR, RESTORE_FLOOR_HEAP_BYTES_PER_STORED_BYTE, RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE, RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE, RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE, RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE } from './chunk-
|
|
3
|
-
import { decodeSegmentManifest, isRecord, isUint32, isSequenceNumber, ARCHIVE_CONFIG_ENTRY, ARCHIVE_MANIFEST_ENTRY, tarEnd, tarEntryHeaders, tarPadding, ARCHIVE_FILE_CHUNK_BYTES } from './chunk-
|
|
4
|
-
import { openPagedSnapshot, pagedListingOf, createPagedFrozenSegment, requirePagedPayloadChecksum, checkPagedContents, isPagedSegmentFile, bytesOfResidentSections, writeCheckpointSegments, commitCheckpointManifest, readSegmentFile, readCommitMarker, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, nextSegmentKey, readWalSegments, durableRecordMissing, checkpointLastSeqNo, residentSectionsOf, partitionContentsOf, SEGMENT_HEADER_SIZE } from './chunk-
|
|
5
|
-
import { createReadCache, MERGE_READ_CACHE_BYTES, loadFailed, decodeSections, READS_AHEAD_AT_ONCE, readFileRangeLater, SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES, READ_AHEAD_CHUNK_BYTES, STABLE_HEAD_ATTEMPTS, directoryCheckpointStore } from './chunk-
|
|
6
|
-
import { manifestKey, segmentsPrefix, vectorFileKey, manifestFileRefs, graphPartKey } from './chunk-
|
|
1
|
+
import { loadVectorField, VECTOR_FILE_ENVELOPE_VERSION, GRAPH_PART_ENVELOPE_VERSION } from './chunk-YPV2L7M5.mjs';
|
|
2
|
+
import { RESTORE_ESTIMATE_MARGIN, RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR_DIMENSION, RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR, RESTORE_WORST_EXTERNAL_BYTES_PER_STORED_BYTE, RESTORE_WORST_HEAP_BYTES_PER_VECTOR_DIMENSION, RESTORE_WORST_HEAP_BYTES_PER_VECTOR, RESTORE_WORST_HEAP_BYTES_PER_STORED_BYTE, RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR_DIMENSION, RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR, RESTORE_FLOOR_EXTERNAL_BYTES_PER_STORED_BYTE, RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR_DIMENSION, RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR, RESTORE_FLOOR_HEAP_BYTES_PER_STORED_BYTE, RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE, RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE, RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE, RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE } from './chunk-LOQHA6QK.mjs';
|
|
3
|
+
import { decodeSegmentManifest, isRecord, isUint32, isSequenceNumber, ARCHIVE_CONFIG_ENTRY, ARCHIVE_MANIFEST_ENTRY, tarEnd, tarEntryHeaders, tarPadding, ARCHIVE_FILE_CHUNK_BYTES } from './chunk-UISV46WX.mjs';
|
|
4
|
+
import { openPagedSnapshot, pagedListingOf, createPagedFrozenSegment, requirePagedPayloadChecksum, checkPagedContents, isPagedSegmentFile, bytesOfResidentSections, writeCheckpointSegments, commitCheckpointManifest, readSegmentFile, readCommitMarker, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, nextSegmentKey, readWalSegments, durableRecordMissing, checkpointLastSeqNo, residentSectionsOf, partitionContentsOf, SEGMENT_HEADER_SIZE } from './chunk-F3FDHXCJ.mjs';
|
|
5
|
+
import { createReadCache, MERGE_READ_CACHE_BYTES, loadFailed, decodeSections, READS_AHEAD_AT_ONCE, readFileRangeLater, SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES, READ_AHEAD_CHUNK_BYTES, STABLE_HEAD_ATTEMPTS, directoryCheckpointStore } from './chunk-ZB5X5WAK.mjs';
|
|
6
|
+
import { manifestKey, segmentsPrefix, vectorFileKey, manifestFileRefs, graphPartKey } from './chunk-KHJPMZE2.mjs';
|
|
7
7
|
import { DEFAULT_COMPACTION_THRESHOLD } from './chunk-QDZJM7NI.mjs';
|
|
8
|
-
import { encode } from './chunk-
|
|
9
|
-
import { restoreVectorFields } from './chunk-
|
|
10
|
-
import { DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE, rewriteSpillOf, REBALANCE_CHUNK_SIZE, rebuildTextIndexes, textRebuildTargetOf } from './chunk-
|
|
11
|
-
import { DEFAULT_VECTOR_METRIC, defaultQuantizationFor, vectorsPerFile, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION } from './chunk-
|
|
12
|
-
import { openChecksummedEnvelope, HEADER_SIZE, packChecksummedEnvelope, packEnvelopeWithChecksum, checksumOfChunks, readHeader, crc32, engineVersionParts, writeHeader, storedStateToWire, storedStateFromWire } from './chunk-
|
|
13
|
-
import { resolveIndexAnalysis, getLanguage, emitEngineEvent, reconstructSchemaFromMetadata, prepareStoredSynonyms, rulesOfStoredSynonyms, resolvePartialWordAnalysis, oversizedConfigurationPayloadOf, requireEncodedConfigurationWithinSizeLimit, decodeStoredDocument, VERSION, DEFAULT_BM25_B, DEFAULT_BM25_K1, removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, createPartitionIndex, writePartitionDocument, yieldToEventLoop, createPublicationGate, removedSurfaceSpellings, readHeapStatistics, readResidentMemory, prepareConfiguredSynonyms, loadGzip, prepareRemovalText, prepareDocumentText, updateDocumentVectors, holdSyncEpochChange, ARRAY_POSITION_GAP } from './chunk-
|
|
8
|
+
import { encode } from './chunk-CV3Q2EKO.mjs';
|
|
9
|
+
import { restoreVectorFields } from './chunk-I2WJCRIH.mjs';
|
|
10
|
+
import { DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE, rewriteSpillOf, REBALANCE_CHUNK_SIZE, rebuildTextIndexes, textRebuildTargetOf } from './chunk-XFOL77WH.mjs';
|
|
11
|
+
import { DEFAULT_VECTOR_METRIC, defaultQuantizationFor, vectorsPerFile, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION } from './chunk-TNEU32EN.mjs';
|
|
12
|
+
import { openChecksummedEnvelope, HEADER_SIZE, packChecksummedEnvelope, packEnvelopeWithChecksum, checksumOfChunks, readHeader, crc32, engineVersionParts, writeHeader, storedStateToWire, storedStateFromWire } from './chunk-JCWC75NH.mjs';
|
|
13
|
+
import { resolveIndexAnalysis, getLanguage, emitEngineEvent, reconstructSchemaFromMetadata, prepareStoredSynonyms, rulesOfStoredSynonyms, resolvePartialWordAnalysis, oversizedConfigurationPayloadOf, requireEncodedConfigurationWithinSizeLimit, loadSchemaCodeFor, decodeStoredDocument, VERSION, DEFAULT_BM25_B, DEFAULT_BM25_K1, removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, createPartitionIndex, writePartitionDocument, yieldToEventLoop, createPublicationGate, removedSurfaceSpellings, readHeapStatistics, readResidentMemory, prepareConfiguredSynonyms, loadGzip, prepareRemovalText, prepareDocumentText, updateDocumentVectors, holdSyncEpochChange, ARRAY_POSITION_GAP } from './chunk-2C3BXZGA.mjs';
|
|
14
14
|
import { decode } from './chunk-GXCDWLQI.mjs';
|
|
15
15
|
import { isSyncEpoch } from './chunk-BUMREPLU.mjs';
|
|
16
|
-
import { extractVectorFieldsFromSchema, flattenSchema, VECTOR_PATTERN, validateDocument, validateDocumentStrict, schemaFieldsOf, requireValidFieldTypes, isQuantizationMode, isStorageMode } from './chunk-
|
|
17
|
-
import { collectAndReturnPages, ANALYSIS_CHANGE_MEMBERS, turnIsSpent, yieldTurn, countWrite, ANALYSIS_HOLD_APPLY_INTERVAL_MS } from './chunk-
|
|
16
|
+
import { extractVectorFieldsFromSchema, flattenSchema, VECTOR_PATTERN, validateDocument, validateDocumentStrict, schemaFieldsOf, requireValidFieldTypes, isQuantizationMode, isStorageMode } from './chunk-X5NHWOXT.mjs';
|
|
17
|
+
import { DEFAULT_MAX_PARTITIONS, collectAndReturnPages, ANALYSIS_CHANGE_MEMBERS, MAX_PARTITIONS, turnIsSpent, yieldTurn, countWrite, ANALYSIS_HOLD_APPLY_INTERVAL_MS } from './chunk-JP77OWMI.mjs';
|
|
18
18
|
import { VALID_HNSW_METRICS } from './chunk-4PY4JYMY.mjs';
|
|
19
19
|
import { NarsilError, ErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
|
|
20
20
|
import { compareCodePoints } from './chunk-TGPKS2UM.mjs';
|
|
@@ -721,8 +721,8 @@ async function executeRebalance(manager, indexName, targetPartitionCount, ctx) {
|
|
|
721
721
|
);
|
|
722
722
|
}
|
|
723
723
|
const entry = ctx.requireIndex(indexName);
|
|
724
|
-
const maxPartitions = entry.config.partitions?.maxPartitions;
|
|
725
|
-
if (
|
|
724
|
+
const maxPartitions = entry.config.partitions?.maxPartitions ?? DEFAULT_MAX_PARTITIONS;
|
|
725
|
+
if (targetPartitionCount > maxPartitions) {
|
|
726
726
|
throw new NarsilError(
|
|
727
727
|
ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
|
|
728
728
|
`Target partition count (${targetPartitionCount}) is above the maximum of ${maxPartitions} partitions`,
|
|
@@ -1336,6 +1336,9 @@ async function driveRun(core, indexName, entry, run, previous, resumedGeneration
|
|
|
1336
1336
|
}
|
|
1337
1337
|
}
|
|
1338
1338
|
async function changeIndexAnalysis(core, indexName, change) {
|
|
1339
|
+
const changedTokenizer = change !== null && typeof change === "object" ? change.tokenizer : void 0;
|
|
1340
|
+
const loadingTokenizerCode = loadSchemaCodeFor(null, changedTokenizer);
|
|
1341
|
+
if (loadingTokenizerCode !== null) await loadingTokenizerCode;
|
|
1339
1342
|
const entry = core.requireIndex(indexName);
|
|
1340
1343
|
requireNotRebalancing(core, indexName);
|
|
1341
1344
|
requireNoRestoreInFlight(core, indexName);
|
|
@@ -1357,6 +1360,8 @@ async function changeIndexAnalysis(core, indexName, change) {
|
|
|
1357
1360
|
}
|
|
1358
1361
|
function resumeAnalysisChange(core, indexName) {
|
|
1359
1362
|
const entry = core.requireIndex(indexName);
|
|
1363
|
+
const loadingTokenizerCode = loadSchemaCodeFor(null, entry.pendingAnalysis?.tokenizer);
|
|
1364
|
+
if (loadingTokenizerCode !== null) return loadingTokenizerCode.then(() => resumeAnalysisChange(core, indexName));
|
|
1360
1365
|
const runs = runsOf(core);
|
|
1361
1366
|
const running = runs.get(indexName);
|
|
1362
1367
|
if (running !== void 0) return running.finished;
|
|
@@ -1406,6 +1411,8 @@ function startHeldRun(core, indexName, pending) {
|
|
|
1406
1411
|
return run;
|
|
1407
1412
|
}
|
|
1408
1413
|
async function buildHeldAnalysisChange(core, indexName, pending) {
|
|
1414
|
+
const loadingTokenizerCode = loadSchemaCodeFor(null, pending.tokenizer);
|
|
1415
|
+
if (loadingTokenizerCode !== null) await loadingTokenizerCode;
|
|
1409
1416
|
const running = runsOf(core).get(indexName);
|
|
1410
1417
|
const entry = core.indexRegistry.get(indexName);
|
|
1411
1418
|
if (running?.hold != null && entry !== void 0 && holdServes(running, entry, pending)) return running.hold.ready;
|
|
@@ -1457,6 +1464,21 @@ async function stampAnalysisGeneration(core, indexName, generation) {
|
|
|
1457
1464
|
await (durable?.checkpointAfterSwitch?.(indexName, stamp) ?? Promise.resolve(stamp()));
|
|
1458
1465
|
}
|
|
1459
1466
|
|
|
1467
|
+
// src/partitioning/saved-partitions.ts
|
|
1468
|
+
function requireSavedPartitionCount(partitionCount, source, indexName) {
|
|
1469
|
+
if (partitionCount <= MAX_PARTITIONS) return;
|
|
1470
|
+
throw new NarsilError(
|
|
1471
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
1472
|
+
`The ${source} of index "${indexName}" holds partition ${partitionCount - 1}, and a saved index numbers its partitions from 0 to ${MAX_PARTITIONS - 1}`,
|
|
1473
|
+
{ indexName, partitionId: partitionCount - 1, maxPartitions: MAX_PARTITIONS }
|
|
1474
|
+
);
|
|
1475
|
+
}
|
|
1476
|
+
function highestPartitionIdOf(partitions) {
|
|
1477
|
+
let highest = partitions.length - 1;
|
|
1478
|
+
for (const partition of partitions) if (partition.partitionId > highest) highest = partition.partitionId;
|
|
1479
|
+
return highest;
|
|
1480
|
+
}
|
|
1481
|
+
|
|
1460
1482
|
// src/persistence/paged/read-ahead.ts
|
|
1461
1483
|
var FILE_PAGE_SIZES = /* @__PURE__ */ new Set([SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES]);
|
|
1462
1484
|
function rangeOf(path, section) {
|
|
@@ -1840,6 +1862,59 @@ function indexMetadataOf(indexName, entry, partitionCount) {
|
|
|
1840
1862
|
return metadataFromConfig(indexName, metadataConfigOf(entry), partitionCount);
|
|
1841
1863
|
}
|
|
1842
1864
|
|
|
1865
|
+
// src/engine/snapshot/stored-schema.ts
|
|
1866
|
+
function isRecord2(value) {
|
|
1867
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
1868
|
+
}
|
|
1869
|
+
function conflictingStoredPath(path) {
|
|
1870
|
+
return new NarsilError(
|
|
1871
|
+
ErrorCodes.SCHEMA_INVALID_TYPE,
|
|
1872
|
+
`Field path "${path}" is declared both as a field and as an object of fields`,
|
|
1873
|
+
{ path }
|
|
1874
|
+
);
|
|
1875
|
+
}
|
|
1876
|
+
function defineOwn(level, key, value) {
|
|
1877
|
+
Object.defineProperty(level, key, { value, enumerable: true, writable: true, configurable: true });
|
|
1878
|
+
}
|
|
1879
|
+
function placeStoredField(nested, path, type) {
|
|
1880
|
+
const segments = path.split(".");
|
|
1881
|
+
let level = nested;
|
|
1882
|
+
for (let at = 0; at < segments.length; at++) {
|
|
1883
|
+
const segment = segments[at];
|
|
1884
|
+
const held = Object.hasOwn(level, segment) ? level[segment] : void 0;
|
|
1885
|
+
if (at === segments.length - 1) {
|
|
1886
|
+
if (held !== void 0) throw conflictingStoredPath(path);
|
|
1887
|
+
defineOwn(level, segment, type);
|
|
1888
|
+
return;
|
|
1889
|
+
}
|
|
1890
|
+
if (held === void 0) {
|
|
1891
|
+
const child = {};
|
|
1892
|
+
defineOwn(level, segment, child);
|
|
1893
|
+
level = child;
|
|
1894
|
+
} else if (isRecord2(held)) {
|
|
1895
|
+
level = held;
|
|
1896
|
+
} else {
|
|
1897
|
+
throw conflictingStoredPath(segments.slice(0, at + 1).join("."));
|
|
1898
|
+
}
|
|
1899
|
+
}
|
|
1900
|
+
}
|
|
1901
|
+
function placeStoredFields(nested, stored, prefix) {
|
|
1902
|
+
for (const [key, type] of Object.entries(stored)) {
|
|
1903
|
+
const path = prefix ? `${prefix}.${key}` : key;
|
|
1904
|
+
if (isRecord2(type)) {
|
|
1905
|
+
placeStoredFields(nested, type, path);
|
|
1906
|
+
} else {
|
|
1907
|
+
placeStoredField(nested, path, type);
|
|
1908
|
+
}
|
|
1909
|
+
}
|
|
1910
|
+
}
|
|
1911
|
+
function nestedSchemaOf(stored) {
|
|
1912
|
+
if (!isRecord2(stored)) return stored;
|
|
1913
|
+
const nested = {};
|
|
1914
|
+
placeStoredFields(nested, stored, "");
|
|
1915
|
+
return nested;
|
|
1916
|
+
}
|
|
1917
|
+
|
|
1843
1918
|
// src/persistence/durability/complete-graphs.ts
|
|
1844
1919
|
async function whileGraphsStayComplete(vectorIndexes, capture2) {
|
|
1845
1920
|
const letBuildsIn = [];
|
|
@@ -2850,7 +2925,7 @@ function wireToMetadata(raw) {
|
|
|
2850
2925
|
if (raw.partition_limits && typeof raw.partition_limits === "object") {
|
|
2851
2926
|
const limits = {
|
|
2852
2927
|
...isPositiveInteger(raw.partition_limits.max_docs_per_partition) ? { maxDocsPerPartition: raw.partition_limits.max_docs_per_partition } : {},
|
|
2853
|
-
...isPositiveInteger(raw.partition_limits.max_partitions) ? { maxPartitions: raw.partition_limits.max_partitions } : {},
|
|
2928
|
+
...isPositiveInteger(raw.partition_limits.max_partitions) ? { maxPartitions: Math.min(raw.partition_limits.max_partitions, MAX_PARTITIONS) } : {},
|
|
2854
2929
|
...isWatermarkFraction(raw.partition_limits.watermark) ? { watermark: raw.partition_limits.watermark } : {}
|
|
2855
2930
|
};
|
|
2856
2931
|
if (Object.keys(limits).length > 0) {
|
|
@@ -3246,21 +3321,11 @@ async function readCurrentManifest(store, indexName) {
|
|
|
3246
3321
|
);
|
|
3247
3322
|
}
|
|
3248
3323
|
function growPartitionsTo(manager, highestPartitionId) {
|
|
3249
|
-
|
|
3250
|
-
const before = manager.partitionCount;
|
|
3251
|
-
try {
|
|
3252
|
-
manager.addPartition();
|
|
3253
|
-
} catch {
|
|
3254
|
-
break;
|
|
3255
|
-
}
|
|
3256
|
-
if (manager.partitionCount === before) break;
|
|
3257
|
-
}
|
|
3324
|
+
manager.growToSavedPartitionCount(highestPartitionId + 1);
|
|
3258
3325
|
}
|
|
3259
3326
|
async function loadSegmentedCheckpoint(store, indexName, manifest, deps, decodesEveryDocument = false) {
|
|
3260
|
-
|
|
3261
|
-
|
|
3262
|
-
if (partition.partitionId > highestPartitionId) highestPartitionId = partition.partitionId;
|
|
3263
|
-
}
|
|
3327
|
+
const highestPartitionId = highestPartitionIdOf(manifest.partitions);
|
|
3328
|
+
requireSavedPartitionCount(highestPartitionId + 1, "segment manifest", indexName);
|
|
3264
3329
|
growPartitionsTo(deps.manager, highestPartitionId);
|
|
3265
3330
|
const { directory } = store;
|
|
3266
3331
|
const opensVectorsOnDemand = manifest.vectors.some(
|
|
@@ -3270,13 +3335,6 @@ async function loadSegmentedCheckpoint(store, indexName, manifest, deps, decodes
|
|
|
3270
3335
|
await startReadingOpeningSectionsAhead(directory, manifest);
|
|
3271
3336
|
}
|
|
3272
3337
|
for (const partition of manifest.partitions) {
|
|
3273
|
-
if (partition.partitionId >= deps.manager.partitionCount) {
|
|
3274
|
-
throw new NarsilError(
|
|
3275
|
-
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
3276
|
-
`Segment manifest references partition ${partition.partitionId} beyond the partition count ${deps.manager.partitionCount}`,
|
|
3277
|
-
{ indexName, partitionId: partition.partitionId, partitionCount: deps.manager.partitionCount }
|
|
3278
|
-
);
|
|
3279
|
-
}
|
|
3280
3338
|
await loadPartitionSegments(store, partition, deps.manager, {
|
|
3281
3339
|
schema: manifest.schema,
|
|
3282
3340
|
decodesEveryDocument,
|
|
@@ -3550,11 +3608,11 @@ function attachablePolicy(shared) {
|
|
|
3550
3608
|
}
|
|
3551
3609
|
};
|
|
3552
3610
|
}
|
|
3553
|
-
function createDetachedIndex(indexName, config, language, synonyms, executor) {
|
|
3611
|
+
function createDetachedIndex(indexName, config, language, synonyms, executor, partitionCount) {
|
|
3554
3612
|
const { policy, attach } = attachablePolicy(executor.vectorWorkerCopies);
|
|
3555
|
-
const entry = executor.createDetachedEntry(indexName, config, language, policy);
|
|
3613
|
+
const entry = executor.createDetachedEntry(indexName, config, language, policy, partitionCount);
|
|
3556
3614
|
entry.manager.synonyms = synonyms;
|
|
3557
3615
|
return { indexName, manager: entry.manager, entry, attach };
|
|
3558
3616
|
}
|
|
3559
3617
|
|
|
3560
|
-
export { SINGLE_NODE_PRIMARY_TERM, abandonAnalysisChange, admitDurableWrite, adoptSavedAnalysis, applyDeleteEntry, applyIndexEntry, archiveFootprint, archiveStream, buildArchiveFromMemory, buildHeldAnalysisChange, bytesThatSegmentsHoldInMemory, captureAppliedPrefix, changeIndexAnalysis, checkPagedSegmentFully, checkpointInTurn, chunksOf, closePartitionWriter, configOfNamedSettings, countsAsLatestHolder, createByteReader, createDetachedIndex, createIndexLoads, createMarkerWriter, createRebalancer, createSeqOwner, deserializePayloadV1, drainIndexStateForUnload, dropHeldAnalysisChange, dropStoredVectorValues, durableMetadataConfigOf, earlierSnapshotFootprint, executeRebalance, growPartitionsTo, heldAnalysisIsBuilt, holdScheduledCheckpoint, holdsNamedSettings, holdsSettings, indexMetadataOf, invalidTransition, listPersistedIndexes, loadMetadata, loadPartitionSegments, loadSegmentedCheckpoint, memoryCheckpointStore, metadataConfigOf, metadataFromConfig, metadataPartitionCount, namedSettingsOf, openPagedSegmentScan, patternIndexesFromWire, readCurrentManifest, readMetadataEnvelope, readSegmentManifest, readTransition, reclaimUnlistedKeys, releaseHeldAnalysisChange, removeLeftoverRewriteFiles, replaySyncedWal, replayWal, replayWalUpTo, requireCheckpointLayout, requireConfigurationWithinSizeLimit, requireRestoreMemory, requiresLayoutCheckpoint, resolveAnalysisTarget, resumeAnalysisChange, sanitizeSurfaceForms, snapshotCheckpointFor, stampAnalysisGeneration, startReadingOpeningSectionsAhead, storedBytesOfSegments, switchInTurn, targetOfPending, transitionKey, transitionWalLayout, unpackEnvelopeBytes, unpackIndexSnapshotEnvelope, validateReplicationEntry, whileGraphsStayComplete, writeArchiveFile, writeLiveVectors, writeMetadataEnvelope };
|
|
3618
|
+
export { SINGLE_NODE_PRIMARY_TERM, abandonAnalysisChange, admitDurableWrite, adoptSavedAnalysis, applyDeleteEntry, applyIndexEntry, archiveFootprint, archiveStream, buildArchiveFromMemory, buildHeldAnalysisChange, bytesThatSegmentsHoldInMemory, captureAppliedPrefix, changeIndexAnalysis, checkPagedSegmentFully, checkpointInTurn, chunksOf, closePartitionWriter, configOfNamedSettings, countsAsLatestHolder, createByteReader, createDetachedIndex, createIndexLoads, createMarkerWriter, createRebalancer, createSeqOwner, deserializePayloadV1, drainIndexStateForUnload, dropHeldAnalysisChange, dropStoredVectorValues, durableMetadataConfigOf, earlierSnapshotFootprint, executeRebalance, growPartitionsTo, heldAnalysisIsBuilt, highestPartitionIdOf, holdScheduledCheckpoint, holdsNamedSettings, holdsSettings, indexMetadataOf, invalidTransition, isRecord2 as isRecord, listPersistedIndexes, loadMetadata, loadPartitionSegments, loadSegmentedCheckpoint, memoryCheckpointStore, metadataConfigOf, metadataFromConfig, metadataPartitionCount, namedSettingsOf, nestedSchemaOf, openPagedSegmentScan, patternIndexesFromWire, readCurrentManifest, readMetadataEnvelope, readSegmentManifest, readTransition, reclaimUnlistedKeys, releaseHeldAnalysisChange, removeLeftoverRewriteFiles, replaySyncedWal, replayWal, replayWalUpTo, requireCheckpointLayout, requireConfigurationWithinSizeLimit, requireRestoreMemory, requireSavedPartitionCount, requiresLayoutCheckpoint, resolveAnalysisTarget, resumeAnalysisChange, sanitizeSurfaceForms, snapshotCheckpointFor, stampAnalysisGeneration, startReadingOpeningSectionsAhead, storedBytesOfSegments, switchInTurn, targetOfPending, transitionKey, transitionWalLayout, unpackEnvelopeBytes, unpackIndexSnapshotEnvelope, validateReplicationEntry, whileGraphsStayComplete, writeArchiveFile, writeLiveVectors, writeMetadataEnvelope };
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { responseStopOf, sendRelayed, parseJson, validateQuery, rejectInvalid, refuseRegex, withServerRerankWindow, requestSignal, respondJson, encodeQueryResult, respondError, loadUWebSockets, childAppOf, createRouteRunner, corsHeaderReader, createSharedBodyBudget, createSharedRequestGate, registerFeedRoute, registerServerRoutes, closeWhenTheProcessExits, statusLine, sendError, REQUEST_THREAD_MOVE_AFTER_MS, MAX_LONG_QUERIES_IN_FLIGHT, createDocumentReadHandlers, SERVER_CAPABILITIES, versionReport, serverRoutes, validateSuggest, isMainFeedNote, SEARCH_CLOCK_SAMPLE_UNITS, awaitDepartureConfirmation, departureConfirmation, REQUEST_THREAD_DEPARTURE_DRAIN_MS, departureConfirmed, stopResponse } from './chunk-
|
|
1
|
+
import { responseStopOf, sendRelayed, parseJson, validateQuery, rejectInvalid, refuseRegex, withServerRerankWindow, requestSignal, respondJson, encodeQueryResult, respondError, loadUWebSockets, childAppOf, createRouteRunner, corsHeaderReader, createSharedBodyBudget, createSharedRequestGate, registerFeedRoute, registerServerRoutes, closeWhenTheProcessExits, statusLine, sendError, REQUEST_THREAD_MOVE_AFTER_MS, MAX_LONG_QUERIES_IN_FLIGHT, createDocumentReadHandlers, SERVER_CAPABILITIES, versionReport, serverRoutes, validateSuggest, isMainFeedNote, SEARCH_CLOCK_SAMPLE_UNITS, awaitDepartureConfirmation, departureConfirmation, REQUEST_THREAD_DEPARTURE_DRAIN_MS, departureConfirmed, stopResponse } from './chunk-XTSFRWAS.mjs';
|
|
2
2
|
import { SYNC_CLOSE_GOING_AWAY, encodeJson } from './chunk-FETZYCH7.mjs';
|
|
3
|
-
import { importOptionalReads, loadSortingFor, holdsQuotedText, loadQueryCode, executeQuery, lazyQueryBinding, executePreflight, resolveProjection, projectionKeepsField, loadRerankRuntime, setNestedValue } from './chunk-
|
|
4
|
-
import { loadedMessagePack, loadMessagePack, extractVectorFieldsFromSchema } from './chunk-
|
|
5
|
-
import { answerFromOneState } from './chunk-
|
|
3
|
+
import { importOptionalReads, loadSortingFor, holdsQuotedText, loadQueryCode, executeQuery, lazyQueryBinding, executePreflight, resolveProjection, projectionKeepsField, loadRerankRuntime, setNestedValue } from './chunk-2C3BXZGA.mjs';
|
|
4
|
+
import { loadedMessagePack, loadMessagePack, extractVectorFieldsFromSchema } from './chunk-X5NHWOXT.mjs';
|
|
5
|
+
import { answerFromOneState } from './chunk-DG6WIEB3.mjs';
|
|
6
6
|
import { ServerErrorCodes, ErrorCodes, NarsilError, isNarsilError } from './chunk-Q3ZZH5Z7.mjs';
|
|
7
7
|
|
|
8
8
|
// src/server/request-threads/feed-relay.ts
|
|
@@ -895,7 +895,7 @@ function watchForDeparture(options) {
|
|
|
895
895
|
),
|
|
896
896
|
pinVectorField: refusingOnceDeparting((indexName, fieldName) => executor.pinVectorField(indexName, fieldName)),
|
|
897
897
|
createIndex: refusingOnceDeparting(
|
|
898
|
-
(indexName, config, language, analysisStale, synonyms) => executor.createIndex(indexName, config, language, analysisStale, synonyms)
|
|
898
|
+
(indexName, config, language, analysisStale, synonyms, partitionCount) => executor.createIndex(indexName, config, language, analysisStale, synonyms, partitionCount)
|
|
899
899
|
),
|
|
900
900
|
dropIndex: refusingOnceDeparting((indexName) => executor.dropIndex(indexName)),
|
|
901
901
|
listIndexes: refusingOnceDeparting(() => executor.listIndexes())
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { getIndexMetadata } from './chunk-
|
|
2
|
-
import { fnv1a } from './chunk-
|
|
1
|
+
import { getIndexMetadata } from './chunk-XQQKETAF.mjs';
|
|
2
|
+
import { fnv1a } from './chunk-2C3BXZGA.mjs';
|
|
3
3
|
import { NarsilError, ErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
|
|
4
4
|
|
|
5
5
|
// src/distribution/cluster-node/routable-allocation.ts
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, withTransientAdmission, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, VECTOR_MEMORY_LIMITS, sharedVectorStoreBytes, STORE_LIVE_COUNT, IN_MEMORY, STORE_RELEASED_VECTORS_TO_DISK, STORE_SLOTS, writeSequenceOf, openGraphState, fixedView, createSharedVectorStoreHandles, sequenceWritesOf, STORE_DOC_ID_BYTES, unplacedChunkCount, blocksGrowthBytes, oldestHeldPin, coverOrdinal, nextSequence, publishSequence, createOrdinalFilter, addToOrdinalFilter, magnitude, stampBorn, mayFree, retiredAtOf, stampRetired, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, slotByteOffset, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, createVectorFileReader, grownByteLength, growBufferTo, resetStoreHeader, STORE_CODE_COUNT, STORE_CALIBRATED, STORE_CALIBRATION_GENERATION, QUERY_STAGED, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, createVectorBlock, ensureBlockSlots } from './chunk-
|
|
2
|
-
import { createDocIdMap } from './chunk-
|
|
1
|
+
import { openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, withTransientAdmission, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, VECTOR_MEMORY_LIMITS, sharedVectorStoreBytes, STORE_LIVE_COUNT, IN_MEMORY, STORE_RELEASED_VECTORS_TO_DISK, STORE_SLOTS, writeSequenceOf, openGraphState, fixedView, createSharedVectorStoreHandles, sequenceWritesOf, STORE_DOC_ID_BYTES, unplacedChunkCount, blocksGrowthBytes, oldestHeldPin, coverOrdinal, nextSequence, publishSequence, createOrdinalFilter, addToOrdinalFilter, magnitude, stampBorn, mayFree, retiredAtOf, requireBytesWithinLimit, stampRetired, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, slotByteOffset, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, createVectorFileReader, grownByteLength, growBufferTo, resetStoreHeader, STORE_CODE_COUNT, STORE_CALIBRATED, STORE_CALIBRATION_GENERATION, QUERY_STAGED, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, createVectorBlock, ensureBlockSlots } from './chunk-TNEU32EN.mjs';
|
|
2
|
+
import { createDocIdMap } from './chunk-DG6WIEB3.mjs';
|
|
3
3
|
import { NarsilError, ErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
|
|
4
4
|
|
|
5
5
|
// src/vector/vector-store/hot-block-distance.ts
|
|
@@ -615,15 +615,26 @@ function ensureFloatSlot(store, ordinal) {
|
|
|
615
615
|
const block = handles.blocks[blockIndex];
|
|
616
616
|
if (block !== null) ensureBlockSlots(block, ordinal % handles.layout.capacity + 1);
|
|
617
617
|
}
|
|
618
|
-
|
|
618
|
+
var LONGEST_UTF8_BYTES_OF_A_UTF16_UNIT = 3;
|
|
619
|
+
var encoder2 = new TextEncoder();
|
|
620
|
+
function requireOrdinalWithinLimit(ordinal) {
|
|
619
621
|
const maxOrdinals = VECTOR_MEMORY_LIMITS.maxOrdinals;
|
|
620
|
-
if (ordinal
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
622
|
+
if (ordinal < maxOrdinals) return;
|
|
623
|
+
throw new NarsilError(
|
|
624
|
+
ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
|
|
625
|
+
`A vector field holds at most ${maxOrdinals} vectors, deleted ones included, until it is rebuilt`,
|
|
626
|
+
{ ordinal, maxOrdinals }
|
|
627
|
+
);
|
|
628
|
+
}
|
|
629
|
+
function requireRoomForSlot(store, docId) {
|
|
630
|
+
const { header, docIdBytes } = store.handles;
|
|
631
|
+
requireOrdinalWithinLimit(Atomics.load(header, STORE_SLOTS));
|
|
632
|
+
const heldBytes = Atomics.load(header, STORE_DOC_ID_BYTES);
|
|
633
|
+
if (heldBytes + docId.length * LONGEST_UTF8_BYTES_OF_A_UTF16_UNIT <= docIdBytes.maxByteLength) return;
|
|
634
|
+
requireBytesWithinLimit(heldBytes + encoder2.encode(docId).length, docIdBytes.maxByteLength);
|
|
635
|
+
}
|
|
636
|
+
function ensureOrdinal(store, ordinal, cold) {
|
|
637
|
+
requireOrdinalWithinLimit(ordinal);
|
|
627
638
|
const { handles } = store;
|
|
628
639
|
growBufferTo(handles.magnitudes, (ordinal + 1) * 8);
|
|
629
640
|
growBufferTo(handles.present, ordinal + 1);
|
|
@@ -794,6 +805,32 @@ function retireInWrite(store, ordinal, sequence) {
|
|
|
794
805
|
function endWrite(store, sequence) {
|
|
795
806
|
publishSequence(store.sequences, sequence);
|
|
796
807
|
}
|
|
808
|
+
var NOT_RETIRED = 0n;
|
|
809
|
+
function clearRetirement(store, ordinal) {
|
|
810
|
+
const writes = store.sequences;
|
|
811
|
+
coverOrdinal(writes, ordinal);
|
|
812
|
+
Atomics.store(writes.retired, ordinal, NOT_RETIRED);
|
|
813
|
+
}
|
|
814
|
+
function reclaimOrdinal(state, docId, ordinal, partitionId, keepsReplacedVector) {
|
|
815
|
+
const store = state.open;
|
|
816
|
+
if (store === null) return false;
|
|
817
|
+
const current = state.docToOrd.get(docId);
|
|
818
|
+
if (current === ordinal) {
|
|
819
|
+
clearRetirement(store, ordinal);
|
|
820
|
+
return true;
|
|
821
|
+
}
|
|
822
|
+
if (state.ordToDoc[ordinal] !== void 0) return false;
|
|
823
|
+
if (!state.replacedOrdinals.delete(ordinal) && !state.awaitingFree.delete(ordinal)) return false;
|
|
824
|
+
const sequence = beginWrite(state, store);
|
|
825
|
+
clearRetirement(store, ordinal);
|
|
826
|
+
recordPartition(state, store, ordinal, partitionId);
|
|
827
|
+
state.ordToDoc[ordinal] = docId;
|
|
828
|
+
state.docToOrd.set(docId, ordinal);
|
|
829
|
+
if (current !== void 0) retireInWrite(store, current, sequence);
|
|
830
|
+
endWrite(store, sequence);
|
|
831
|
+
settleReplacement(state, store, current, keepsReplacedVector);
|
|
832
|
+
return true;
|
|
833
|
+
}
|
|
797
834
|
function retireStoredOrdinal(state, ordinal) {
|
|
798
835
|
const store = state.open;
|
|
799
836
|
if (store === null) return false;
|
|
@@ -1047,6 +1084,12 @@ function createVectorStore(options) {
|
|
|
1047
1084
|
settleReplacement(state, store, previous, keepsReplacedVector);
|
|
1048
1085
|
return ordinal;
|
|
1049
1086
|
},
|
|
1087
|
+
requireRoomFor(docId) {
|
|
1088
|
+
if (state.open !== null) requireRoomForSlot(state.open, docId);
|
|
1089
|
+
},
|
|
1090
|
+
reclaim(docId, ordinal, partitionId, keepsReplacedVector = false) {
|
|
1091
|
+
return reclaimOrdinal(state, docId, ordinal, partitionId, keepsReplacedVector);
|
|
1092
|
+
},
|
|
1050
1093
|
retire(ordinal) {
|
|
1051
1094
|
retireStoredOrdinal(state, ordinal);
|
|
1052
1095
|
},
|
|
@@ -1211,4 +1254,4 @@ function createVectorStore(options) {
|
|
|
1211
1254
|
};
|
|
1212
1255
|
}
|
|
1213
1256
|
|
|
1214
|
-
export { createVectorStore, openSharedVectorField };
|
|
1257
|
+
export { createVectorStore, openSharedVectorField, requireOrdinalWithinLimit };
|
|
@@ -4,7 +4,7 @@ import { createRequire } from 'module';
|
|
|
4
4
|
import { join, dirname } from 'path';
|
|
5
5
|
|
|
6
6
|
// src/vector/native/types/index.ts
|
|
7
|
-
var NATIVE_CORE_ABI_VERSION =
|
|
7
|
+
var NATIVE_CORE_ABI_VERSION = 51;
|
|
8
8
|
var NATIVE_OK = 0;
|
|
9
9
|
var NATIVE_NEEDS_ROOM = 3;
|
|
10
10
|
var NATIVE_GRAPH_NOT_HELD_ALONE = 4;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { V8_YOUNG_GENERATION_BYTES_BY_MAJOR, SCRATCH_SLOTS_PER_THREAD_POOL } from './chunk-
|
|
1
|
+
import { V8_YOUNG_GENERATION_BYTES_BY_MAJOR, SCRATCH_SLOTS_PER_THREAD_POOL } from './chunk-JP77OWMI.mjs';
|
|
2
2
|
import { NarsilError, ErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
|
|
3
3
|
|
|
4
4
|
// src/core/search-stop.ts
|
|
@@ -1,15 +1,15 @@
|
|
|
1
|
-
import { readWholeSectionChecked, loadFailed, sectionNamed, openPagedFile, closePagedFile, readCacheOfThisThread, PAGED_SEGMENT_ENVELOPE_VERSION, FILE_ASSEMBLY_CHUNK_BYTES, createReadCache, MERGE_READ_CACHE_BYTES, createSectionReader, checkEverySectionPage, encodePagedSegmentHead, SECTION_ALIGNMENT_BYTES, SEGMENT_PAGE_BYTES, readListedFile, requireListedChecksum, encodeSegmentManifest, readFileRange, PAYLOAD_PREFIX_BYTES, decodePagedSegmentHead, BLOCK_SUMMARY_BYTES, SMALLEST_BLOCK_POSTINGS, DOCUMENT_BLOCK_BYTES, MAX_DOCUMENT_BLOCK_RAW_BYTES, ID_SAMPLE_INTERVAL,
|
|
1
|
+
import { readWholeSectionChecked, loadFailed, sectionNamed, openPagedFile, closePagedFile, readCacheOfThisThread, PAGED_SEGMENT_ENVELOPE_VERSION, FILE_ASSEMBLY_CHUNK_BYTES, createReadCache, MERGE_READ_CACHE_BYTES, createSectionReader, checkEverySectionPage, encodePagedSegmentHead, SECTION_ALIGNMENT_BYTES, SEGMENT_PAGE_BYTES, readListedFile, requireListedChecksum, encodeSegmentManifest, readFileRange, PAYLOAD_PREFIX_BYTES, decodePagedSegmentHead, BLOCK_SUMMARY_BYTES, SMALLEST_BLOCK_POSTINGS, DOCUMENT_BLOCK_BYTES, MAX_DOCUMENT_BLOCK_RAW_BYTES, ID_SAMPLE_INTERVAL, resumePagedFile, suspendPagedFile, cachedFileBytes, requireNativePage, LIST_VIEWS_HELD_PER_SEGMENT, WRITER_YIELD_INTERVAL_TOKENS, DOCUMENT_BLOCK_MIN_RAW_BYTES, CACHED_POSITIONS, POSITION_WIDTH_BYTES, CACHED_LIST, POSTING_WIDTH_BYTES, CACHED_DOCUMENT_BLOCK, failNativePage, readFileRangeLater, REMOVAL_BUCKET_WRITE_BUFFER_BYTES, POSTING_WRITE_ROWS, POSTING_WRITE_POSITIONS, SECTION_WRITE_BUFFER_BYTES, LARGEST_REMOVAL_BUCKET_COUNT, REMOVAL_BUCKET_TARGET_BYTES, MERGED_SEGMENT_GROWTH_ALLOWANCE, MERGED_SEGMENT_TARGET_BYTES } from './chunk-ZB5X5WAK.mjs';
|
|
2
2
|
import { ReplicationMessageTypes } from './chunk-5FLRXF73.mjs';
|
|
3
3
|
import { WIRE_BATCH_BUDGET } from './chunk-JDDXV6UF.mjs';
|
|
4
|
-
import { SEGMENT_MANIFEST_VERSION, manifestKey, manifestReferencedKeys, segmentsPrefix, snapshotBundleKey, MAX_SEGMENTS_PER_PARTITION, segmentKey } from './chunk-
|
|
4
|
+
import { SEGMENT_MANIFEST_VERSION, manifestKey, manifestReferencedKeys, segmentsPrefix, snapshotBundleKey, MAX_SEGMENTS_PER_PARTITION, segmentKey } from './chunk-KHJPMZE2.mjs';
|
|
5
5
|
import { DEFAULT_COMPACTION_THRESHOLD, SEGMENT_FILE_FRAMING_SLACK_BYTES, COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT, SEGMENT_TOKEN_TABLE_SHARE_OF_FILE } from './chunk-QDZJM7NI.mjs';
|
|
6
|
-
import { encode } from './chunk-
|
|
7
|
-
import { restoreVectorFields } from './chunk-
|
|
8
|
-
import { readHeader, HEADER_SIZE, crc32Init, crc32Update, crc32Final, requireWithinFileLimit, engineVersionParts, writeHeader, crc32, openChecksummedEnvelope, packChecksummedEnvelope } from './chunk-
|
|
9
|
-
import { generateId, wrapFrozenTokenTable, createEncodedDocumentSource, assemblePagedFrozenSegment, decodeStoredDocument, buildPartialWordTableFrom, mergeFieldIndexes, isCompositePartition, reconstructSchemaFromMetadata, decodeSurfaceTable, mergeDocFrequencies, TermLookup, createFrozenRemovals, selectionKeeps, fieldIndexRemapOf, mergeKeywordSources, mergeSourceSpellings, mergeSourceSurfaces, mergedPartialSpellings, createSharedFrozenSegment, shareSegmentArrays, getLanguage, emptyPendingRegions, mergePostings2, scanPostingBlocks, holdsADiscardedField, SegmentRefusal, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayloads, resolvePartitionInsertOptions, endOfValue, segmentArraysOf, mergeSegmentContents, decodeMapWithoutFields, prepareDocumentVectors, resolveIndexAnalysis } from './chunk-
|
|
6
|
+
import { encode } from './chunk-CV3Q2EKO.mjs';
|
|
7
|
+
import { restoreVectorFields } from './chunk-I2WJCRIH.mjs';
|
|
8
|
+
import { readHeader, HEADER_SIZE, crc32Init, crc32Update, crc32Final, requireWithinFileLimit, engineVersionParts, writeHeader, crc32, openChecksummedEnvelope, packChecksummedEnvelope } from './chunk-JCWC75NH.mjs';
|
|
9
|
+
import { generateId, wrapFrozenTokenTable, createEncodedDocumentSource, assemblePagedFrozenSegment, decodeStoredDocument, buildPartialWordTableFrom, mergeFieldIndexes, isCompositePartition, reconstructSchemaFromMetadata, decodeSurfaceTable, mergeDocFrequencies, TermLookup, createFrozenRemovals, selectionKeeps, fieldIndexRemapOf, mergeKeywordSources, mergeSourceSpellings, mergeSourceSurfaces, mergedPartialSpellings, createSharedFrozenSegment, shareSegmentArrays, getLanguage, emptyPendingRegions, mergePostings2, scanPostingBlocks, holdsADiscardedField, SegmentRefusal, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayloads, resolvePartitionInsertOptions, endOfValue, segmentArraysOf, mergeSegmentContents, decodeMapWithoutFields, prepareDocumentVectors, resolveIndexAnalysis } from './chunk-2C3BXZGA.mjs';
|
|
10
10
|
import { encodeDocument } from './chunk-KLVMK3R5.mjs';
|
|
11
11
|
import { decode } from './chunk-GXCDWLQI.mjs';
|
|
12
|
-
import { extractVectorFieldsFromSchema, VECTOR_PATTERN, requireValidFieldTypes, partialFieldsOf, messagePackUpperBoundBytes } from './chunk-
|
|
12
|
+
import { extractVectorFieldsFromSchema, VECTOR_PATTERN, requireValidFieldTypes, partialFieldsOf, messagePackUpperBoundBytes } from './chunk-X5NHWOXT.mjs';
|
|
13
13
|
import { MAX_FILE_BYTES } from './chunk-4PY4JYMY.mjs';
|
|
14
14
|
import { NarsilError, ErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
|
|
15
15
|
import { compareCodePoints } from './chunk-TGPKS2UM.mjs';
|
|
@@ -1512,8 +1512,11 @@ function assembleOpenedSegment(file, snapshot, cache) {
|
|
|
1512
1512
|
close() {
|
|
1513
1513
|
closePagedFile(file);
|
|
1514
1514
|
},
|
|
1515
|
-
|
|
1516
|
-
|
|
1515
|
+
suspend() {
|
|
1516
|
+
suspendPagedFile(file);
|
|
1517
|
+
},
|
|
1518
|
+
resume(rewrite) {
|
|
1519
|
+
resumePagedFile(file, rewrite(file.path));
|
|
1517
1520
|
snapshot.path = file.path;
|
|
1518
1521
|
}
|
|
1519
1522
|
};
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { withOpenIndex, oneAtATime, validateBM25Params, storedBM25Params, emitSyncChange } from './chunk-
|
|
1
|
+
import { withOpenIndex, oneAtATime, validateBM25Params, storedBM25Params, emitSyncChange } from './chunk-2C3BXZGA.mjs';
|
|
2
2
|
import { ErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
|
|
3
3
|
|
|
4
4
|
// src/engine/bm25-update.ts
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { isValidInteger, isRecord } from './chunk-
|
|
1
|
+
import { isValidInteger, isRecord } from './chunk-XQQKETAF.mjs';
|
|
2
2
|
import { MAX_REPLICATION_FACTOR, MAX_PARTITION_COUNT } from './chunk-JDDXV6UF.mjs';
|
|
3
|
-
import { openChecksummedEnvelope, packChecksummedEnvelope, MAX_HISTORY_FILE_BYTES } from './chunk-
|
|
3
|
+
import { openChecksummedEnvelope, packChecksummedEnvelope, MAX_HISTORY_FILE_BYTES } from './chunk-JCWC75NH.mjs';
|
|
4
4
|
import { decode } from './chunk-GXCDWLQI.mjs';
|
|
5
5
|
import { isSyncEpoch } from './chunk-BUMREPLU.mjs';
|
|
6
6
|
import { joinChunks } from './chunk-IE3Q62QD.mjs';
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { VERSION, GROUP_PATTERN_SOURCE, loadGzip } from './chunk-
|
|
2
|
-
import { flattenSchema } from './chunk-
|
|
1
|
+
import { VERSION, GROUP_PATTERN_SOURCE, loadGzip } from './chunk-2C3BXZGA.mjs';
|
|
2
|
+
import { flattenSchema } from './chunk-X5NHWOXT.mjs';
|
|
3
3
|
import { NATIVE_CRC32_MIN_BYTES, MAX_FILE_BYTES, CHECKSUM_YIELD_CHUNK_BYTES } from './chunk-4PY4JYMY.mjs';
|
|
4
4
|
import { NarsilError, ErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
|
|
5
5
|
import { encode } from '@msgpack/msgpack';
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { boundQuerySuggestionsOf } from './chunk-S2POFTLN.mjs';
|
|
2
|
-
import { engineCoreOf, loadSuggestionsCode, GROUP_PATTERN_SOURCE, GROUP_FORMAT, SUGGESTIONS_INDEX_INFIX } from './chunk-
|
|
2
|
+
import { engineCoreOf, loadSuggestionsCode, GROUP_PATTERN_SOURCE, GROUP_FORMAT, SUGGESTIONS_INDEX_INFIX } from './chunk-2C3BXZGA.mjs';
|
|
3
3
|
import { NarsilError, ErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
|
|
4
4
|
|
|
5
5
|
// src/query-suggestions/index.ts
|
|
@@ -77,6 +77,8 @@ var ABSOLUTE_WINDOWS_PATH_PATTERN = /^[a-zA-Z]:[\\/]/;
|
|
|
77
77
|
var MODULE_URL_SCHEME_PATTERN = /^(file|https?|data|node):/;
|
|
78
78
|
var MAX_INDEX_NAME_LENGTH = 255;
|
|
79
79
|
var MAX_DOC_ID_LENGTH = 512;
|
|
80
|
+
var DEFAULT_MAX_PARTITIONS = 1;
|
|
81
|
+
var MAX_PARTITIONS = 65536;
|
|
80
82
|
var EXPORT_CHUNK_BYTES = 1048576;
|
|
81
83
|
var MAX_EXPORT_LINE_BYTES = 117443592;
|
|
82
84
|
var MAX_INDEX_CONFIGURATION_BYTES = 16777216;
|
|
@@ -249,4 +251,4 @@ var GRAPH_LOCK_WAIT_DEADLINE_MS = 1e4;
|
|
|
249
251
|
var GRAPH_BUILD_DISPATCH_CHUNK = 64;
|
|
250
252
|
var GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER = 2;
|
|
251
253
|
|
|
252
|
-
export { ABSOLUTE_WINDOWS_PATH_PATTERN, ANALYSIS_CHANGE_MEMBERS, ANALYSIS_HOLD_APPLY_INTERVAL_MS, BATCH_CHUNK_SIZE, BM25_DISPLAYED_NAME_LENGTH, CURSOR_BINDING_BUFFER_BYTES, DEFAULT_FUZZY_PREFIX_LENGTH, DEFAULT_LIMIT, DEFAULT_LINEAR_ALPHA, DEFAULT_MAX_PER_GROUP, DEFAULT_OFFSET, DEFAULT_PAGE_SIZE, DEFAULT_RANK_CONSTANT, DEFAULT_SUGGEST_LIMIT, DISK_STORAGE_BLOCK_BYTES, EXPORT_CHUNK_BYTES, FOREIGN_STATS_TTL_MS, FUSION_TIE_INSERTION_LIMIT, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, GRAPH_LOCK_WAIT_DEADLINE_MS, GRAPH_LOCK_WAIT_SLICE_MS, HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, HEAP_PRESSURE_REARM_FRACTION, HEAP_PRESSURE_WARNING_FRACTION, HEAP_RESUME_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, HYBRID_RESCORE_WINDOW, IMPORT_BATCH_DOCUMENTS, INDEX_NAME_PATTERN, LOCK_SPIN_ITERATIONS, LOCK_WAIT_SLICE_MS, MAX_CURSOR_ANCHOR_CODE_POINTS, MAX_CURSOR_LENGTH, MAX_DOC_ID_LENGTH, MAX_DOC_ID_TABLE_BYTES, MAX_EXPORT_LINE_BYTES, MAX_HYBRID_DEPTH, MAX_IMPORT_REFUSALS_REPORTED, MAX_INDEX_CONFIGURATION_BYTES, MAX_INDEX_NAME_LENGTH, MAX_LINEAR_ALPHA, MAX_PHRASE_WORDS, MAX_PREFIX_LENGTH, MAX_QUERY_PHRASES, MAX_RENUMBERED_SEARCH_ATTEMPTS, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, MAX_SWEEP_INTERVAL_MS, MAX_TOLERANCE, MAX_VECTOR_ORDINALS, MAX_WASM_PAGES, MAX_WORKER_COPIES, MIN_DOCUMENTS_FOR_SEGMENTS, MIN_HYBRID_DEPTH, MIN_LINEAR_ALPHA, MIN_RANK_CONSTANT, MIN_SWEEP_INTERVAL_MS, MODULE_URL_SCHEME_PATTERN, NATIVE_ATTACH_RETRY_MS, NATIVE_THREAD_SCAN_CHUNK_ORDINALS, NATIVE_THREAD_SEARCH_SLOTS, PARTIAL_WORDS_ADDED_PER_QUERY, PARTIAL_WORD_MIN_CODE_POINTS, PREFIX_MAX_EXPANSIONS, PROCESS_LIMIT_REFRESH_INTERVAL_MS, REFUSAL_COLLECTION_MIN_INTERVAL_MS, REFUSAL_COLLECTION_TIME_MULTIPLE, REFUSAL_YOUNG_COLLECTION_MIN_INTERVAL_MS, RESULT_WINDOW, SCRATCH_SLOTS_PER_THREAD_POOL, STATISTICS_BROADCAST_INTERVAL_MS, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING, UI_THREAD_BACKGROUND_SLICE_DURING_WRITES_MS, UI_THREAD_HOP_BEFORE_TURN_WITHIN_MS, UI_THREAD_TURN_BUDGET_MS, UNPLACED_FLAG_CHUNK_ORDINALS, V8_YOUNG_GENERATION_BYTES_BY_MAJOR, VECTOR_BLOCK_INITIAL_SLOTS, VECTOR_BLOCK_MAX_BYTES, VECTOR_INSERT_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_STORE_INITIAL_CAPACITY, WASM_PAGE_BYTES, adoptSharedTurns, backgroundTurnIsSpent, changeIndexOutsideExternalWaits, collectAndReturnPages, collectYoungGeneration, countWrite, holdWrites, holdWritesAndBackgroundWork, indexChangeOutsideExternalWaits, noteThreadSlotGone, noteThreadSlotLive, startIdleHeapCleanup, threadSlotGone, threadSlotOfWorker, turnIsSpent, turnsAreBudgeted, waitOutsideCountedWrite, yieldBackgroundTurn, yieldTurn };
|
|
254
|
+
export { ABSOLUTE_WINDOWS_PATH_PATTERN, ANALYSIS_CHANGE_MEMBERS, ANALYSIS_HOLD_APPLY_INTERVAL_MS, BATCH_CHUNK_SIZE, BM25_DISPLAYED_NAME_LENGTH, CURSOR_BINDING_BUFFER_BYTES, DEFAULT_FUZZY_PREFIX_LENGTH, DEFAULT_LIMIT, DEFAULT_LINEAR_ALPHA, DEFAULT_MAX_PARTITIONS, DEFAULT_MAX_PER_GROUP, DEFAULT_OFFSET, DEFAULT_PAGE_SIZE, DEFAULT_RANK_CONSTANT, DEFAULT_SUGGEST_LIMIT, DISK_STORAGE_BLOCK_BYTES, EXPORT_CHUNK_BYTES, FOREIGN_STATS_TTL_MS, FUSION_TIE_INSERTION_LIMIT, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, GRAPH_LOCK_WAIT_DEADLINE_MS, GRAPH_LOCK_WAIT_SLICE_MS, HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, HEAP_PRESSURE_REARM_FRACTION, HEAP_PRESSURE_WARNING_FRACTION, HEAP_RESUME_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, HYBRID_RESCORE_WINDOW, IMPORT_BATCH_DOCUMENTS, INDEX_NAME_PATTERN, LOCK_SPIN_ITERATIONS, LOCK_WAIT_SLICE_MS, MAX_CURSOR_ANCHOR_CODE_POINTS, MAX_CURSOR_LENGTH, MAX_DOC_ID_LENGTH, MAX_DOC_ID_TABLE_BYTES, MAX_EXPORT_LINE_BYTES, MAX_HYBRID_DEPTH, MAX_IMPORT_REFUSALS_REPORTED, MAX_INDEX_CONFIGURATION_BYTES, MAX_INDEX_NAME_LENGTH, MAX_LINEAR_ALPHA, MAX_PARTITIONS, MAX_PHRASE_WORDS, MAX_PREFIX_LENGTH, MAX_QUERY_PHRASES, MAX_RENUMBERED_SEARCH_ATTEMPTS, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, MAX_SWEEP_INTERVAL_MS, MAX_TOLERANCE, MAX_VECTOR_ORDINALS, MAX_WASM_PAGES, MAX_WORKER_COPIES, MIN_DOCUMENTS_FOR_SEGMENTS, MIN_HYBRID_DEPTH, MIN_LINEAR_ALPHA, MIN_RANK_CONSTANT, MIN_SWEEP_INTERVAL_MS, MODULE_URL_SCHEME_PATTERN, NATIVE_ATTACH_RETRY_MS, NATIVE_THREAD_SCAN_CHUNK_ORDINALS, NATIVE_THREAD_SEARCH_SLOTS, PARTIAL_WORDS_ADDED_PER_QUERY, PARTIAL_WORD_MIN_CODE_POINTS, PREFIX_MAX_EXPANSIONS, PROCESS_LIMIT_REFRESH_INTERVAL_MS, REFUSAL_COLLECTION_MIN_INTERVAL_MS, REFUSAL_COLLECTION_TIME_MULTIPLE, REFUSAL_YOUNG_COLLECTION_MIN_INTERVAL_MS, RESULT_WINDOW, SCRATCH_SLOTS_PER_THREAD_POOL, STATISTICS_BROADCAST_INTERVAL_MS, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING, UI_THREAD_BACKGROUND_SLICE_DURING_WRITES_MS, UI_THREAD_HOP_BEFORE_TURN_WITHIN_MS, UI_THREAD_TURN_BUDGET_MS, UNPLACED_FLAG_CHUNK_ORDINALS, V8_YOUNG_GENERATION_BYTES_BY_MAJOR, VECTOR_BLOCK_INITIAL_SLOTS, VECTOR_BLOCK_MAX_BYTES, VECTOR_INSERT_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_STORE_INITIAL_CAPACITY, WASM_PAGE_BYTES, adoptSharedTurns, backgroundTurnIsSpent, changeIndexOutsideExternalWaits, collectAndReturnPages, collectYoungGeneration, countWrite, holdWrites, holdWritesAndBackgroundWork, indexChangeOutsideExternalWaits, noteThreadSlotGone, noteThreadSlotLive, startIdleHeapCleanup, threadSlotGone, threadSlotOfWorker, turnIsSpent, turnsAreBudgeted, waitOutsideCountedWrite, yieldBackgroundTurn, yieldTurn };
|
|
@@ -51,6 +51,14 @@ async function resolveVectorText(params, embeddingAdapter, signal, embeddingAdap
|
|
|
51
51
|
{ expected: embeddingAdapter.dimensions, actual: raw.length }
|
|
52
52
|
);
|
|
53
53
|
}
|
|
54
|
+
for (let index = 0; index < raw.length; index++) {
|
|
55
|
+
if (Number.isFinite(raw[index])) continue;
|
|
56
|
+
throw new NarsilError(
|
|
57
|
+
ErrorCodes.EMBEDDING_FAILED,
|
|
58
|
+
`Adapter returned vector with ${raw[index]} at index ${index} for query embedding`,
|
|
59
|
+
{ index }
|
|
60
|
+
);
|
|
61
|
+
}
|
|
54
62
|
const resolved = { ...params.vector, value: Array.from(raw) };
|
|
55
63
|
delete resolved.text;
|
|
56
64
|
return { ...params, vector: resolved };
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { encode } from './chunk-
|
|
1
|
+
import { encode } from './chunk-CV3Q2EKO.mjs';
|
|
2
2
|
import { oneSyncChangeAtATime } from './chunk-V6EUBAIT.mjs';
|
|
3
|
-
import { openChecksummedEnvelope, packChecksummedEnvelope } from './chunk-
|
|
4
|
-
import { rulesOfStoredSynonyms, configurationPayloadOf, withOpenIndex, emitSyncChange } from './chunk-
|
|
3
|
+
import { openChecksummedEnvelope, packChecksummedEnvelope } from './chunk-JCWC75NH.mjs';
|
|
4
|
+
import { rulesOfStoredSynonyms, configurationPayloadOf, withOpenIndex, emitSyncChange } from './chunk-2C3BXZGA.mjs';
|
|
5
5
|
import { decode } from './chunk-GXCDWLQI.mjs';
|
|
6
6
|
import { newSyncEpoch } from './chunk-BUMREPLU.mjs';
|
|
7
7
|
import { NarsilError, ErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { WORKER_YOUNG_GENERATION_MB } from './chunk-
|
|
2
|
-
import { REFUSAL_YOUNG_COLLECTION_MIN_INTERVAL_MS, collectYoungGeneration, collectAndReturnPages, REFUSAL_COLLECTION_MIN_INTERVAL_MS, REFUSAL_COLLECTION_TIME_MULTIPLE } from './chunk-
|
|
1
|
+
import { WORKER_YOUNG_GENERATION_MB } from './chunk-DG6WIEB3.mjs';
|
|
2
|
+
import { REFUSAL_YOUNG_COLLECTION_MIN_INTERVAL_MS, collectYoungGeneration, collectAndReturnPages, REFUSAL_COLLECTION_MIN_INTERVAL_MS, REFUSAL_COLLECTION_TIME_MULTIPLE } from './chunk-JP77OWMI.mjs';
|
|
3
3
|
|
|
4
4
|
// src/workers/resource-limits.ts
|
|
5
5
|
function workerResourceLimits() {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { isCompositePartition, encodeSegmentState, freezeSegmentShared } from './chunk-
|
|
1
|
+
import { isCompositePartition, encodeSegmentState, freezeSegmentShared } from './chunk-2C3BXZGA.mjs';
|
|
2
2
|
import { NarsilError, ErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
|
|
3
3
|
|
|
4
4
|
// src/core/partition/frozen/share.ts
|
|
@@ -250,6 +250,7 @@ async function loadIndexOntoWorkers(indexName, executors, source, manager, onDro
|
|
|
250
250
|
analysisStale,
|
|
251
251
|
copyLoad,
|
|
252
252
|
publication: held.publication,
|
|
253
|
+
partitionCount: captured.length,
|
|
253
254
|
...synonyms !== null ? { synonyms } : {}
|
|
254
255
|
},
|
|
255
256
|
held.retain()
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { isSyncEpoch } from './chunk-BUMREPLU.mjs';
|
|
2
|
-
import { requireLoadedMessagePack } from './chunk-
|
|
2
|
+
import { requireLoadedMessagePack } from './chunk-X5NHWOXT.mjs';
|
|
3
3
|
import { NarsilError, ErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
|
|
4
4
|
|
|
5
5
|
// src/sync/device/stores/records.ts
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { createNumericIndex, createBooleanIndex, createEnumIndex, createLoadedGeoIndex, rebuildSizeIndexes, requirePatternSearch, patternIndexOf } from './chunk-
|
|
2
|
-
import { getFlatSchema, schemaFieldsOf, getNestedValue } from './chunk-
|
|
1
|
+
import { createNumericIndex, createBooleanIndex, createEnumIndex, createLoadedGeoIndex, rebuildSizeIndexes, requirePatternSearch, patternIndexOf } from './chunk-2C3BXZGA.mjs';
|
|
2
|
+
import { getFlatSchema, schemaFieldsOf, getNestedValue } from './chunk-X5NHWOXT.mjs';
|
|
3
3
|
|
|
4
4
|
// src/core/partition/pattern-snapshot.ts
|
|
5
5
|
function serializePatternIndexes(state, schema) {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { DEFAULT_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, FUSION_TIE_INSERTION_LIMIT, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, MIN_RANK_CONSTANT } from './chunk-
|
|
2
|
-
import { nativeSearchCoreIsEnabled, loadNativeCore, noteFallback } from './chunk-
|
|
1
|
+
import { DEFAULT_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, FUSION_TIE_INSERTION_LIMIT, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, MIN_RANK_CONSTANT } from './chunk-JP77OWMI.mjs';
|
|
2
|
+
import { nativeSearchCoreIsEnabled, loadNativeCore, noteFallback } from './chunk-D6FYUTFN.mjs';
|
|
3
3
|
import { NarsilError, ErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
|
|
4
4
|
import { compareCodePoints } from './chunk-TGPKS2UM.mjs';
|
|
5
5
|
|